Bump to syntect 5.0.0 to e.g. start lazy-loading syntaxes (#2181)

* Bump to syntect 5.0.0 to e.g. start lazy-loading themes

Closes #915
Closes #951
Closes #1846
Closes #1854

* Typo fix formated -> formatted

* Update CHANGELOG.md

CHANGELOG.md

@@ -5,15 +5,21 @@
 - Correctly render tab stops in --show-all, see #2038 (@Synthetica9)
 - Add a `--style=default` option, less verbose than `full`, see #2061 (@IsaacHorvath)
 - Enable BusyBox less as pager, see #2162 (@nfisher1226)
+- File extensions are now matched without taking case into account. See #1854, #2181 (@Enselic)
 
 ## Bugfixes
 
 - Bump `regex` dependency from 1.5.4 to 1.5.5 to fix [CVE-2022-24713](https://blog.rust-lang.org/2022/03/08/cve-2022-24713.html), see #2145, #2139 (@Enselic)
+- `bat` no longer crashes when encountering files that references missing syntaxes. See #915, #2181 (@Enselic)
+
+## Performance
+
+- Skip syntax highlighting on long lines (> 16384 chars) to help improve performance. See #2165 (@keith-hall)
+- Vastly improve startup time by lazy-loading syntaxes via syntect 5.0.0. This makes bat display small files ~75% faster than before. See #951, #2181 (@Enselic)
 
 ## Other
 
 - Include info about custom assets in `--diagnostics` if used. See #2107, #2144 (@Enselic)
-- Skip syntax highlighting on long lines (> 16384 chars) to help improve performance. See #2165 (@keith-hall)
 
 ## Syntaxes
 
@@ -28,6 +34,10 @@
 ## `bat` as a library
 
 - Allow configuration of `show_nonprintable` with `PrettyPrinter`, see #2142
+- The binary format of syntaxes.bin has been changed due to syntaxes now being lazy-loaded via syntect 5.0.0. See #2181 (@Enselic)
+- Mark `bat::error::Error` enum as `#[non_exhaustive]` to allow adding new variants without future semver breakage. See #2181 (@Enselic)
+- Change `Error::SyntectError(syntect::LoadingError)` to `Error::SyntectError(syntect::Error)`. See #2181 (@Enselic)
+- Add `Error::SyntectLoadingError(syntect::LoadingError)` enum variant. See #2181 (@Enselic)
 
 # v0.20.0
 

Cargo.lock

@@ -580,12 +580,6 @@ version = "1.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
 
-[[package]]
-name = "lazycell"
-version = "1.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"
-
 [[package]]
 name = "libc"
 version = "0.2.125"
@@ -1071,9 +1065,9 @@ dependencies = [
 
 [[package]]
 name = "syntect"
-version = "4.6.0"
+version = "5.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8b20815bbe80ee0be06e6957450a841185fcf690fe0178f14d77a05ce2caa031"
+checksum = "c6c454c27d9d7d9a84c7803aaa3c50cd088d2906fe3c6e42da3209aa623576a8"
 dependencies = [
  "bincode",
  "bitflags",
@@ -1081,13 +1075,14 @@ dependencies = [
  "flate2",
  "fnv",
  "lazy_static",
- "lazycell",
+ "once_cell",
  "onig",
  "plist",
  "regex-syntax",
  "serde",
  "serde_derive",
  "serde_json",
+ "thiserror",
  "walkdir",
  "yaml-rust",
 ]

Cargo.toml

@@ -33,8 +33,7 @@ minimal-application = [
 ]
 git = ["git2"] # Support indicating git modifications
 paging = ["shell-words", "grep-cli"] # Support applying a pager on the output
-# Add "syntect/plist-load" when https://github.com/trishume/syntect/pull/345 reaches us
-build-assets = ["syntect/yaml-load", "syntect/dump-create", "regex", "walkdir"]
+build-assets = ["syntect/yaml-load", "syntect/plist-load", "regex", "walkdir"]
 
 # You need to use one of these if you depend on bat as a library:
 regex-onig = ["syntect/regex-onig"] # Use the "oniguruma" regex engine
@@ -73,7 +72,7 @@ optional = true
 default-features = false
 
 [dependencies.syntect]
-version = "4.6.0"
+version = "5.0.0"
 default-features = false
 features = ["parsing"]
 

src/assets.rs

@@ -43,8 +43,9 @@ pub struct SyntaxReferenceInSet<'a> {
     pub syntax_set: &'a SyntaxSet,
 }
 
-/// Compress for size of ~700 kB instead of ~4600 kB at the cost of ~30% longer deserialization time
-pub(crate) const COMPRESS_SYNTAXES: bool = true;
+/// Lazy-loaded syntaxes are already compressed, and we don't want to compress
+/// already compressed data.
+pub(crate) const COMPRESS_SYNTAXES: bool = false;
 
 /// We don't want to compress our [LazyThemeSet] since the lazy-loaded themes
 /// within it are already compressed, and compressing another time just makes
@@ -581,13 +582,22 @@ mod tests {
     }
 
     #[test]
-    fn syntax_detection_is_case_sensitive() {
+    fn syntax_detection_is_case_insensitive() {
         let mut test = SyntaxDetectionTest::new();
 
-        assert_ne!(test.syntax_for_file("README.MD"), "Markdown");
+        assert_eq!(test.syntax_for_file("README.md"), "Markdown");
+        assert_eq!(test.syntax_for_file("README.mD"), "Markdown");
+        assert_eq!(test.syntax_for_file("README.Md"), "Markdown");
+        assert_eq!(test.syntax_for_file("README.MD"), "Markdown");
+
+        // Adding a mapping for "MD" in addition to "md" should not break the mapping
         test.syntax_mapping
             .insert("*.MD", MappingTarget::MapTo("Markdown"))
             .ok();
+
+        assert_eq!(test.syntax_for_file("README.md"), "Markdown");
+        assert_eq!(test.syntax_for_file("README.mD"), "Markdown");
+        assert_eq!(test.syntax_for_file("README.Md"), "Markdown");
         assert_eq!(test.syntax_for_file("README.MD"), "Markdown");
     }
 

src/error.rs

@@ -2,11 +2,14 @@ use std::io::Write;
 use thiserror::Error;
 
 #[derive(Error, Debug)]
+#[non_exhaustive]
 pub enum Error {
     #[error(transparent)]
     Io(#[from] ::std::io::Error),
     #[error(transparent)]
-    SyntectError(#[from] ::syntect::LoadingError),
+    SyntectError(#[from] ::syntect::Error),
+    #[error(transparent)]
+    SyntectLoadingError(#[from] ::syntect::LoadingError),
     #[error(transparent)]
     ParseIntError(#[from] ::std::num::ParseIntError),
     #[error(transparent)]

src/printer.rs

@@ -453,7 +453,7 @@ impl<'a> Printer for InteractivePrinter<'a> {
 
             let mut highlighted_line = highlighter_from_set
                 .highlighter
-                .highlight(for_highlighting, highlighter_from_set.syntax_set);
+                .highlight_line(for_highlighting, highlighter_from_set.syntax_set)?;
 
             if too_long {
                 highlighted_line[0].1 = &line;

tests/syntax-tests/highlighted/Svelte/App.svelte

@@ -27,9 +27,13 @@
 </script>
 
 <script type="text/livescript">
-    // This block is a regression test for a bat panic when a LiveScript syntax definition is missing
+	// This block is a regression test for a bat panic when a LiveScript syntax definition is missing
 </script>
 
+<style lang="text/postcss">
+	/* This block is a regression test for a bat panic when a PostCSS syntax definition is missing */
+</style>
+
 <style>
 	main {
 		position: relative;

tests/syntax-tests/highlighted/Vue/example.vue

@@ -8,6 +8,11 @@
   </div>
 </template>
 
+<template lang='pug'>
+  #container.col
+    p This shall be formatted as Plain Text as long as a Pug syntax definition is missing
+</template>
+
 <script>
 import AppHeader from "@/components/AppHeader";
 import AppLoadingIndicator from "@/components/AppLoadingIndicator";

tests/syntax-tests/source/Svelte/App.svelte

@@ -27,9 +27,13 @@
 </script>
 
 <script type="text/livescript">
-    // This block is a regression test for a bat panic when a LiveScript syntax definition is missing
+	// This block is a regression test for a bat panic when a LiveScript syntax definition is missing
 </script>
 
+<style lang="text/postcss">
+	/* This block is a regression test for a bat panic when a PostCSS syntax definition is missing */
+</style>
+
 <style>
 	main {
 		position: relative;

tests/syntax-tests/source/Vue/example.vue

@@ -8,6 +8,11 @@
   </div>
 </template>
 
+<template lang='pug'>
+  #container.col
+    p This shall be formatted as Plain Text as long as a Pug syntax definition is missing
+</template>
+
 <script>
 import AppHeader from "@/components/AppHeader";
 import AppLoadingIndicator from "@/components/AppLoadingIndicator";