Initial version of syntax regression tests

tests/.gitattributes

@@ -11,3 +11,4 @@ snapshots/* linguist-vendored
 benchmarks/* linguist-vendored
 benchmarks/test-src/* linguist-vendored
 scripts/* linguist-vendored
+syntax-tests/* linguist-vendored

tests/syntax-tests/create_highlighted_versions.py

@@ -0,0 +1,58 @@
+#!/usr/bin/env python
+
+import subprocess
+import glob
+import sys
+import os.path as path
+import os
+
+
+BAT_OPTIONS = [
+    "--no-config",
+    "--style=plain",
+    "--color=always",
+    "--theme='1337'",
+    "--italic-text=always",
+]
+
+
+def create_highlighted_versions():
+    root = os.path.dirname(os.path.abspath(__file__))
+
+    for source in glob.glob(path.join(root, "source", "*", "*")):
+        try:
+            bat_output = subprocess.check_output(
+                ["bat"] + BAT_OPTIONS + [source], stderr=subprocess.PIPE,
+            )
+
+            source_dirname = path.basename(path.dirname(source))
+            source_filename = path.basename(source)
+
+            output_dir = path.join(root, "highlighted", source_dirname)
+            output_path = path.join(output_dir, source_filename)
+
+            os.makedirs(output_dir, exist_ok=True)
+
+            with open(output_path, "wb") as output_file:
+                output_file.write(bat_output)
+
+            relative_path = path.relpath(output_path, root)
+            print("Created '{}'".format(relative_path))
+        except subprocess.CalledProcessError as err:
+            print(
+                "=== Error: Could not highlight source file '{}".format(source),
+                file=sys.stderr,
+            )
+            print(
+                "=== bat stdout:\n{}".format(err.stdout.decode("utf-8")),
+                file=sys.stderr,
+            )
+            print(
+                "=== bat stderr:\n{}".format(err.stderr.decode("utf-8")),
+                file=sys.stderr,
+            )
+            sys.exit(1)
+
+
+if __name__ == "__main__":
+    create_highlighted_versions()

tests/syntax-tests/highlighted/Rust/output.rs

@@ -0,0 +1,161 @@
+use std::io::{self, Write};
+#[cfg(feature = "paging")]
+use std::process::Child;
+
+use crate::error::*;
+#[cfg(feature = "paging")]
+use crate::less::retrieve_less_version;
+#[cfg(feature = "paging")]
+use crate::paging::PagingMode;
+
+#[derive(Debug)]
+pub enum OutputType {
+    #[cfg(feature = "paging")]
+    Pager(Child),
+    Stdout(io::Stdout),
+}
+
+impl OutputType {
+    #[cfg(feature = "paging")]
+    pub fn from_mode(mode: PagingMode, pager: Option<&str>) -> Result<Self> {
+        use self::PagingMode::*;
+        Ok(match mode {
+            Always => OutputType::try_pager(false, pager)?,
+            QuitIfOneScreen => OutputType::try_pager(true, pager)?,
+            _ => OutputType::stdout(),
+        })
+    }
+
+    /// Try to launch the pager. Fall back to stdout in case of errors.
+    #[cfg(feature = "paging")]
+    fn try_pager(quit_if_one_screen: bool, pager_from_config: Option<&str>) -> Result<Self> {
+        use std::env;
+        use std::ffi::OsString;
+        use std::path::PathBuf;
+        use std::process::{Command, Stdio};
+
+        let mut replace_arguments_to_less = false;
+
+        let pager_from_env = match (env::var("BAT_PAGER"), env::var("PAGER")) {
+            (Ok(bat_pager), _) => Some(bat_pager),
+            (_, Ok(pager)) => {
+                // less needs to be called with the '-R' option in order to properly interpret the
+                // ANSI color sequences printed by bat. If someone has set PAGER="less -F", we
+                // therefore need to overwrite the arguments and add '-R'.
+                //
+                // We only do this for PAGER (as it is not specific to 'bat'), not for BAT_PAGER
+                // or bats '--pager' command line option.
+                replace_arguments_to_less = true;
+                Some(pager)
+            }
+            _ => None,
+        };
+
+        let pager_from_config = pager_from_config.map(|p| p.to_string());
+
+        if pager_from_config.is_some() {
+            replace_arguments_to_less = false;
+        }
+
+        let pager = pager_from_config
+            .or(pager_from_env)
+            .unwrap_or_else(|| String::from("less"));
+
+        let pagerflags =
+            shell_words::split(&pager).chain_err(|| "Could not parse pager command.")?;
+
+        match pagerflags.split_first() {
+            Some((pager_name, args)) => {
+                let mut pager_path = PathBuf::from(pager_name);
+
+                if pager_path.file_stem() == Some(&OsString::from("bat")) {
+                    pager_path = PathBuf::from("less");
+                }
+
+                let is_less = pager_path.file_stem() == Some(&OsString::from("less"));
+
+                let mut process = if is_less {
+                    let mut p = Command::new(&pager_path);
+                    if args.is_empty() || replace_arguments_to_less {
+                        p.arg("--RAW-CONTROL-CHARS");
+                        if quit_if_one_screen {
+                            p.arg("--quit-if-one-screen");
+                        }
+
+                        // Passing '--no-init' fixes a bug with '--quit-if-one-screen' in older
+                        // versions of 'less'. Unfortunately, it also breaks mouse-wheel support.
+                        //
+                        // See: http://www.greenwoodsoftware.com/less/news.530.html
+                        //
+                        // For newer versions (530 or 558 on Windows), we omit '--no-init' as it
+                        // is not needed anymore.
+                        match retrieve_less_version() {
+                            None => {
+                                p.arg("--no-init");
+                            }
+                            Some(version)
+                                if (version < 530 || (cfg!(windows) && version < 558)) =>
+                            {
+                                p.arg("--no-init");
+                            }
+                            _ => {}
+                        }
+                    } else {
+                        p.args(args);
+                    }
+                    p.env("LESSCHARSET", "UTF-8");
+                    p
+                } else {
+                    let mut p = Command::new(&pager_path);
+                    p.args(args);
+                    p
+                };
+
+                Ok(process
+                    .stdin(Stdio::piped())
+                    .spawn()
+                    .map(OutputType::Pager)
+                    .unwrap_or_else(|_| OutputType::stdout()))
+            }
+            None => Ok(OutputType::stdout()),
+        }
+    }
+
+    pub(crate) fn stdout() -> Self {
+        OutputType::Stdout(io::stdout())
+    }
+
+    #[cfg(feature = "paging")]
+    pub(crate) fn is_pager(&self) -> bool {
+        if let OutputType::Pager(_) = self {
+            true
+        } else {
+            false
+        }
+    }
+
+    #[cfg(not(feature = "paging"))]
+    pub(crate) fn is_pager(&self) -> bool {
+        false
+    }
+
+    pub fn handle(&mut self) -> Result<&mut dyn Write> {
+        Ok(match *self {
+            #[cfg(feature = "paging")]
+            OutputType::Pager(ref mut command) => command
+                .stdin
+                .as_mut()
+                .chain_err(|| "Could not open stdin for pager")?,
+            OutputType::Stdout(ref mut handle) => handle,
+        })
+    }
+}
+
+#[cfg(feature = "paging")]
+impl Drop for OutputType {
+    fn drop(&mut self) {
+        if let OutputType::Pager(ref mut command) = *self {
+            let _ = command.wait();
+        }
+    }
+}

tests/syntax-tests/highlighted/Rust/test.rs

@@ -0,0 +1,161 @@
+use std::io::{self, Write};
+#[cfg(feature = "paging")]
+use std::process::Child;
+
+use crate::error::*;
+#[cfg(feature = "paging")]
+use crate::less::retrieve_less_version;
+#[cfg(feature = "paging")]
+use crate::paging::PagingMode;
+
+#[derive(Debug)]
+pub enum OutputType {
+    #[cfg(feature = "paging")]
+    Pager(Child),
+    Stdout(io::Stdout),
+}
+
+impl OutputType {
+    #[cfg(feature = "paging")]
+    pub fn from_mode(mode: PagingMode, pager: Option<&str>) -> Result<Self> {
+        use self::PagingMode::*;
+        Ok(match mode {
+            Always => OutputType::try_pager(false, pager)?,
+            QuitIfOneScreen => OutputType::try_pager(true, pager)?,
+            _ => OutputType::stdout(),
+        })
+    }
+
+    /// Try to launch the pager. Fall back to stdout in case of errors.
+    #[cfg(feature = "paging")]
+    fn try_pager(quit_if_one_screen: bool, pager_from_config: Option<&str>) -> Result<Self> {
+        use std::env;
+        use std::ffi::OsString;
+        use std::path::PathBuf;
+        use std::process::{Command, Stdio};
+
+        let mut replace_arguments_to_less = false;
+
+        let pager_from_env = match (env::var("BAT_PAGER"), env::var("PAGER")) {
+            (Ok(bat_pager), _) => Some(bat_pager),
+            (_, Ok(pager)) => {
+                // less needs to be called with the '-R' option in order to properly interpret the
+                // ANSI color sequences printed by bat. If someone has set PAGER="less -F", we
+                // therefore need to overwrite the arguments and add '-R'.
+                //
+                // We only do this for PAGER (as it is not specific to 'bat'), not for BAT_PAGER
+                // or bats '--pager' command line option.
+                replace_arguments_to_less = true;
+                Some(pager)
+            }
+            _ => None,
+        };
+
+        let pager_from_config = pager_from_config.map(|p| p.to_string());
+
+        if pager_from_config.is_some() {
+            replace_arguments_to_less = false;
+        }
+
+        let pager = pager_from_config
+            .or(pager_from_env)
+            .unwrap_or_else(|| String::from("less"));
+
+        let pagerflags =
+            shell_words::split(&pager).chain_err(|| "Could not parse pager command.")?;
+
+        match pagerflags.split_first() {
+            Some((pager_name, args)) => {
+                let mut pager_path = PathBuf::from(pager_name);
+
+                if pager_path.file_stem() == Some(&OsString::from("bat")) {
+                    pager_path = PathBuf::from("less");
+                }
+
+                let is_less = pager_path.file_stem() == Some(&OsString::from("less"));
+
+                let mut process = if is_less {
+                    let mut p = Command::new(&pager_path);
+                    if args.is_empty() || replace_arguments_to_less {
+                        p.arg("--RAW-CONTROL-CHARS");
+                        if quit_if_one_screen {
+                            p.arg("--quit-if-one-screen");
+                        }
+
+                        // Passing '--no-init' fixes a bug with '--quit-if-one-screen' in older
+                        // versions of 'less'. Unfortunately, it also breaks mouse-wheel support.
+                        //
+                        // See: http://www.greenwoodsoftware.com/less/news.530.html
+                        //
+                        // For newer versions (530 or 558 on Windows), we omit '--no-init' as it
+                        // is not needed anymore.
+                        match retrieve_less_version() {
+                            None => {
+                                p.arg("--no-init");
+                            }
+                            Some(version)
+                                if (version < 530 || (cfg!(windows) && version < 558)) =>
+                            {
+                                p.arg("--no-init");
+                            }
+                            _ => {}
+                        }
+                    } else {
+                        p.args(args);
+                    }
+                    p.env("LESSCHARSET", "UTF-8");
+                    p
+                } else {
+                    let mut p = Command::new(&pager_path);
+                    p.args(args);
+                    p
+                };
+
+                Ok(process
+                    .stdin(Stdio::piped())
+                    .spawn()
+                    .map(OutputType::Pager)
+                    .unwrap_or_else(|_| OutputType::stdout()))
+            }
+            None => Ok(OutputType::stdout()),
+        }
+    }
+
+    pub(crate) fn stdout() -> Self {
+        OutputType::Stdout(io::stdout())
+    }
+
+    #[cfg(feature = "paging")]
+    pub(crate) fn is_pager(&self) -> bool {
+        if let OutputType::Pager(_) = self {
+            true
+        } else {
+            false
+        }
+    }
+
+    #[cfg(not(feature = "paging"))]
+    pub(crate) fn is_pager(&self) -> bool {
+        false
+    }
+
+    pub fn handle(&mut self) -> Result<&mut dyn Write> {
+        Ok(match *self {
+            #[cfg(feature = "paging")]
+            OutputType::Pager(ref mut command) => command
+                .stdin
+                .as_mut()
+                .chain_err(|| "Could not open stdin for pager")?,
+            OutputType::Stdout(ref mut handle) => handle,
+        })
+    }
+}
+
+#[cfg(feature = "paging")]
+impl Drop for OutputType {
+    fn drop(&mut self) {
+        if let OutputType::Pager(ref mut command) = *self {
+            let _ = command.wait();
+        }
+    }
+}

tests/syntax-tests/highlighted/SSH Config/ssh_config

@@ -0,0 +1,9 @@
+# A comment
+
+Host example.com
+    User dummy
+    Compression no
+
+Host *.co.uk
+    BatchMode no
+    GlobalKnownHostsFile "/etc/ssh/ssh_known_hosts"

tests/syntax-tests/highlighted/TOML/Cargo.toml

@@ -0,0 +1,87 @@
+[package]
+authors = ["David Peter <mail@david-peter.de>"]
+categories = ["command-line-utilities"]
+description="A cat(1) clone with wings."
+homepage = "https://github.com/sharkdp/bat"
+license = "MIT/Apache-2.0"
+name = "bat"
+readme = "README.md"
+repository = "https://github.com/sharkdp/bat"
+version = "0.15.4"
+exclude = [
+    "assets/syntaxes/*",
+    "assets/themes/*",
+]
+build = "build.rs"
+edition = '2018'
+
+[features]
+default = ["application"]
+# Feature required for bat the application. Should be disabled when depending on
+# bat as a library.
+application = [
+    "atty",
+    "clap",
+    "dirs",
+    "git",
+    "lazy_static",
+    "liquid",
+    "paging",
+    "wild",
+    "regex-onig",
+]
+git = ["git2"] # Support indicating git modifications
+paging = ["shell-words"] # Support applying a pager on the output
+regex-onig = ["syntect/regex-onig"] # Use the "oniguruma" regex engine
+regex-fancy = ["syntect/regex-fancy"] # Use the rust-only "fancy-regex" engine
+
+[dependencies]
+atty = { version = "0.2.14", optional = true }
+ansi_term = "^0.12.1"
+ansi_colours = "^1.0"
+console = "0.11.3"
+dirs = { version = "3.0", optional = true }
+lazy_static = { version = "1.4", optional = true }
+wild = { version = "2.0", optional = true }
+content_inspector = "0.2.4"
+encoding = "0.2"
+shell-words = { version = "1.0.0", optional = true }
+unicode-width = "0.1.8"
+globset = "0.4"
+serde = { version = "1.0", features = ["derive"] }
+serde_yaml = "0.8"
+semver = "0.10"
+path_abs = { version = "0.5", default-features = false }
+
+[dependencies.git2]
+version = "0.13"
+optional = true
+default-features = false
+
+[dependencies.syntect]
+version = "4.2.0"
+default-features = false
+features = ["parsing", "yaml-load", "dump-load", "dump-create"]
+
+[dependencies.clap]
+version = "2.33"
+optional = true
+default-features = false
+features = ["suggestions", "color", "wrap_help"]
+
+[dependencies.error-chain]
+version = "0.12"
+default-features = false
+
+[dev-dependencies]
+tempdir = "0.3"
+assert_cmd = "1.0.1"
+predicates = "1.0.5"
+
+[build-dependencies]
+clap = { version = "2.33", optional = true }
+liquid = { version = "0.21", optional = true }
+
+[profile.release]
+lto = true
+codegen-units = 1

tests/syntax-tests/source/Rust/output.rs

@@ -0,0 +1,161 @@
+use std::io::{self, Write};
+#[cfg(feature = "paging")]
+use std::process::Child;
+
+use crate::error::*;
+#[cfg(feature = "paging")]
+use crate::less::retrieve_less_version;
+#[cfg(feature = "paging")]
+use crate::paging::PagingMode;
+
+#[derive(Debug)]
+pub enum OutputType {
+    #[cfg(feature = "paging")]
+    Pager(Child),
+    Stdout(io::Stdout),
+}
+
+impl OutputType {
+    #[cfg(feature = "paging")]
+    pub fn from_mode(mode: PagingMode, pager: Option<&str>) -> Result<Self> {
+        use self::PagingMode::*;
+        Ok(match mode {
+            Always => OutputType::try_pager(false, pager)?,
+            QuitIfOneScreen => OutputType::try_pager(true, pager)?,
+            _ => OutputType::stdout(),
+        })
+    }
+
+    /// Try to launch the pager. Fall back to stdout in case of errors.
+    #[cfg(feature = "paging")]
+    fn try_pager(quit_if_one_screen: bool, pager_from_config: Option<&str>) -> Result<Self> {
+        use std::env;
+        use std::ffi::OsString;
+        use std::path::PathBuf;
+        use std::process::{Command, Stdio};
+
+        let mut replace_arguments_to_less = false;
+
+        let pager_from_env = match (env::var("BAT_PAGER"), env::var("PAGER")) {
+            (Ok(bat_pager), _) => Some(bat_pager),
+            (_, Ok(pager)) => {
+                // less needs to be called with the '-R' option in order to properly interpret the
+                // ANSI color sequences printed by bat. If someone has set PAGER="less -F", we
+                // therefore need to overwrite the arguments and add '-R'.
+                //
+                // We only do this for PAGER (as it is not specific to 'bat'), not for BAT_PAGER
+                // or bats '--pager' command line option.
+                replace_arguments_to_less = true;
+                Some(pager)
+            }
+            _ => None,
+        };
+
+        let pager_from_config = pager_from_config.map(|p| p.to_string());
+
+        if pager_from_config.is_some() {
+            replace_arguments_to_less = false;
+        }
+
+        let pager = pager_from_config
+            .or(pager_from_env)
+            .unwrap_or_else(|| String::from("less"));
+
+        let pagerflags =
+            shell_words::split(&pager).chain_err(|| "Could not parse pager command.")?;
+
+        match pagerflags.split_first() {
+            Some((pager_name, args)) => {
+                let mut pager_path = PathBuf::from(pager_name);
+
+                if pager_path.file_stem() == Some(&OsString::from("bat")) {
+                    pager_path = PathBuf::from("less");
+                }
+
+                let is_less = pager_path.file_stem() == Some(&OsString::from("less"));
+
+                let mut process = if is_less {
+                    let mut p = Command::new(&pager_path);
+                    if args.is_empty() || replace_arguments_to_less {
+                        p.arg("--RAW-CONTROL-CHARS");
+                        if quit_if_one_screen {
+                            p.arg("--quit-if-one-screen");
+                        }
+
+                        // Passing '--no-init' fixes a bug with '--quit-if-one-screen' in older
+                        // versions of 'less'. Unfortunately, it also breaks mouse-wheel support.
+                        //
+                        // See: http://www.greenwoodsoftware.com/less/news.530.html
+                        //
+                        // For newer versions (530 or 558 on Windows), we omit '--no-init' as it
+                        // is not needed anymore.
+                        match retrieve_less_version() {
+                            None => {
+                                p.arg("--no-init");
+                            }
+                            Some(version)
+                                if (version < 530 || (cfg!(windows) && version < 558)) =>
+                            {
+                                p.arg("--no-init");
+                            }
+                            _ => {}
+                        }
+                    } else {
+                        p.args(args);
+                    }
+                    p.env("LESSCHARSET", "UTF-8");
+                    p
+                } else {
+                    let mut p = Command::new(&pager_path);
+                    p.args(args);
+                    p
+                };
+
+                Ok(process
+                    .stdin(Stdio::piped())
+                    .spawn()
+                    .map(OutputType::Pager)
+                    .unwrap_or_else(|_| OutputType::stdout()))
+            }
+            None => Ok(OutputType::stdout()),
+        }
+    }
+
+    pub(crate) fn stdout() -> Self {
+        OutputType::Stdout(io::stdout())
+    }
+
+    #[cfg(feature = "paging")]
+    pub(crate) fn is_pager(&self) -> bool {
+        if let OutputType::Pager(_) = self {
+            true
+        } else {
+            false
+        }
+    }
+
+    #[cfg(not(feature = "paging"))]
+    pub(crate) fn is_pager(&self) -> bool {
+        false
+    }
+
+    pub fn handle(&mut self) -> Result<&mut dyn Write> {
+        Ok(match *self {
+            #[cfg(feature = "paging")]
+            OutputType::Pager(ref mut command) => command
+                .stdin
+                .as_mut()
+                .chain_err(|| "Could not open stdin for pager")?,
+            OutputType::Stdout(ref mut handle) => handle,
+        })
+    }
+}
+
+#[cfg(feature = "paging")]
+impl Drop for OutputType {
+    fn drop(&mut self) {
+        if let OutputType::Pager(ref mut command) = *self {
+            let _ = command.wait();
+        }
+    }
+}

tests/syntax-tests/source/SSH Config/ssh_config

@@ -0,0 +1,9 @@
+# A comment
+
+Host example.com
+    User dummy
+    Compression no
+
+Host *.co.uk
+    BatchMode no
+    GlobalKnownHostsFile "/etc/ssh/ssh_known_hosts"

tests/syntax-tests/source/TOML/Cargo.toml

@@ -0,0 +1,87 @@
+[package]
+authors = ["David Peter <mail@david-peter.de>"]
+categories = ["command-line-utilities"]
+description="A cat(1) clone with wings."
+homepage = "https://github.com/sharkdp/bat"
+license = "MIT/Apache-2.0"
+name = "bat"
+readme = "README.md"
+repository = "https://github.com/sharkdp/bat"
+version = "0.15.4"
+exclude = [
+    "assets/syntaxes/*",
+    "assets/themes/*",
+]
+build = "build.rs"
+edition = '2018'
+
+[features]
+default = ["application"]
+# Feature required for bat the application. Should be disabled when depending on
+# bat as a library.
+application = [
+    "atty",
+    "clap",
+    "dirs",
+    "git",
+    "lazy_static",
+    "liquid",
+    "paging",
+    "wild",
+    "regex-onig",
+]
+git = ["git2"] # Support indicating git modifications
+paging = ["shell-words"] # Support applying a pager on the output
+regex-onig = ["syntect/regex-onig"] # Use the "oniguruma" regex engine
+regex-fancy = ["syntect/regex-fancy"] # Use the rust-only "fancy-regex" engine
+
+[dependencies]
+atty = { version = "0.2.14", optional = true }
+ansi_term = "^0.12.1"
+ansi_colours = "^1.0"
+console = "0.11.3"
+dirs = { version = "3.0", optional = true }
+lazy_static = { version = "1.4", optional = true }
+wild = { version = "2.0", optional = true }
+content_inspector = "0.2.4"
+encoding = "0.2"
+shell-words = { version = "1.0.0", optional = true }
+unicode-width = "0.1.8"
+globset = "0.4"
+serde = { version = "1.0", features = ["derive"] }
+serde_yaml = "0.8"
+semver = "0.10"
+path_abs = { version = "0.5", default-features = false }
+
+[dependencies.git2]
+version = "0.13"
+optional = true
+default-features = false
+
+[dependencies.syntect]
+version = "4.2.0"
+default-features = false
+features = ["parsing", "yaml-load", "dump-load", "dump-create"]
+
+[dependencies.clap]
+version = "2.33"
+optional = true
+default-features = false
+features = ["suggestions", "color", "wrap_help"]
+
+[dependencies.error-chain]
+version = "0.12"
+default-features = false
+
+[dev-dependencies]
+tempdir = "0.3"
+assert_cmd = "1.0.1"
+predicates = "1.0.5"
+
+[build-dependencies]
+clap = { version = "2.33", optional = true }
+liquid = { version = "0.21", optional = true }
+
+[profile.release]
+lto = true
+codegen-units = 1