diff --git a/CHANGELOG.md b/CHANGELOG.md
index b6e13469..458ad919 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -22,6 +22,7 @@
 - Pull in fix for unsafe-libyaml security advisory, see #2812 (@dtolnay)
 - Update git-version dependency to use Syn v2, see #2816 (@dtolnay)
 - Update git2 dependency to v0.18.2, see #2852 (@eth-p)
+- Improve performance when color output disabled, see #2397 and #2857 (@eth-p)
 
 ## Syntaxes
 
diff --git a/src/printer.rs b/src/printer.rs
index 3ac850fa..8fd6bc8e 100644
--- a/src/printer.rs
+++ b/src/printer.rs
@@ -9,6 +9,7 @@ use bytesize::ByteSize;
 
 use syntect::easy::HighlightLines;
 use syntect::highlighting::Color;
+use syntect::highlighting::FontStyle;
 use syntect::highlighting::Theme;
 use syntect::parsing::SyntaxSet;
 
@@ -48,6 +49,22 @@ const ANSI_UNDERLINE_DISABLE: EscapeSequence = EscapeSequence::CSI {
     final_byte: "m",
 };
 
+const EMPTY_SYNTECT_STYLE: syntect::highlighting::Style = syntect::highlighting::Style {
+    foreground: Color {
+        r: 127,
+        g: 127,
+        b: 127,
+        a: 255,
+    },
+    background: Color {
+        r: 127,
+        g: 127,
+        b: 127,
+        a: 255,
+    },
+    font_style: FontStyle::empty(),
+};
+
 pub enum OutputHandle<'a> {
     IoWrite(&'a mut dyn io::Write),
     FmtWrite(&'a mut dyn fmt::Write),
@@ -222,11 +239,13 @@ impl<'a> InteractivePrinter<'a> {
             panel_width = 0;
         }
 
-        let highlighter_from_set = if input
+        // Get the highlighter for the output.
+        let is_printing_binary = input
             .reader
             .content_type
-            .map_or(false, |c| c.is_binary() && !config.show_nonprintable)
-        {
+            .map_or(false, |c| c.is_binary() && !config.show_nonprintable);
+
+        let highlighter_from_set = if is_printing_binary || config.colored_output == false {
             None
         } else {
             // Determine the type of syntax for highlighting