account for charset when creating data URLs

src/css.rs

@@ -202,10 +202,11 @@ pub fn process_css<'a>(
                             import_contents,
                             import_final_url,
                             import_media_type,
-                            _import_charset,
+                            import_charset,
                         )) => {
                             let mut import_data_url = create_data_url(
                                 &import_media_type,
+                                &import_charset,
                                 embed_css(
                                     cache,
                                     client,
@@ -252,9 +253,9 @@ pub fn process_css<'a>(
                                 options,
                                 depth + 1,
                             ) {
-                                Ok((data, final_url, media_type, _charset)) => {
+                                Ok((data, final_url, media_type, charset)) => {
                                     let mut data_url =
-                                        create_data_url(&media_type, &data, &final_url);
+                                        create_data_url(&media_type, &charset, &data, &final_url);
                                     data_url.set_fragment(resolved_url.fragment());
                                     result.push_str(
                                         format_quoted_string(&data_url.to_string()).as_str(),
@@ -348,9 +349,10 @@ pub fn process_css<'a>(
                         options,
                         depth + 1,
                     ) {
-                        Ok((css, final_url, media_type, _charset)) => {
+                        Ok((css, final_url, media_type, charset)) => {
                             let mut data_url = create_data_url(
                                 &media_type,
+                                &charset,
                                 embed_css(
                                     cache,
                                     client,
@@ -386,8 +388,9 @@ pub fn process_css<'a>(
                             options,
                             depth + 1,
                         ) {
-                            Ok((data, final_url, media_type, _charset)) => {
+                            Ok((data, final_url, media_type, charset)) => {
-                                let mut data_url = create_data_url(&media_type, &data, &final_url);
+                                let mut data_url =
+                                    create_data_url(&media_type, &charset, &data, &final_url);
                                 data_url.set_fragment(full_url.fragment());
                                 result
                                     .push_str(format_quoted_string(&data_url.to_string()).as_str());

src/html.rs

@@ -188,10 +188,13 @@ pub fn embed_srcset(
                 options,
                 depth + 1,
             ) {
-                Ok((image_data, image_final_url, image_media_type, _image_charset)) => {
+                Ok((image_data, image_final_url, image_media_type, image_charset)) => {
-                    // TODO: use image_charset
+                    let mut image_data_url = create_data_url(
-                    let mut image_data_url =
+                        &image_media_type,
-                        create_data_url(&image_media_type, &image_data, &image_final_url);
+                        &image_charset,
+                        &image_data,
+                        &image_final_url,
+                    );
                     // Append retreved asset as a data URL
                     image_data_url.set_fragment(image_full_url.fragment());
                     result.push_str(image_data_url.as_ref());
@@ -637,12 +640,12 @@ pub fn retrieve_and_embed_asset(
                     let css: String = embed_css(cache, client, &final_url, &s, options, depth + 1);
 
                     // Create and embed data URL
-                    // TODO: use charset
+                    let css_data_url =
-                    let css_data_url = create_data_url(&media_type, css.as_bytes(), &final_url);
+                        create_data_url(&media_type, &charset, css.as_bytes(), &final_url);
                     set_node_attr(&node, attr_name, Some(css_data_url.to_string()));
                 } else if node_name == "frame" || node_name == "iframe" {
                     // (I)FRAMEs are also quite different from conventional resources
-                    let frame_dom = html_to_dom(&data, charset);
+                    let frame_dom = html_to_dom(&data, charset.clone());
                     walk_and_embed_assets(
                         cache,
                         client,
@@ -661,8 +664,8 @@ pub fn retrieve_and_embed_asset(
                     .unwrap();
 
                     // Create and embed data URL
-                    // TODO: use charset
+                    let mut frame_data_url =
-                    let mut frame_data_url = create_data_url(&media_type, &frame_data, &final_url);
+                        create_data_url(&media_type, &charset, &frame_data, &final_url);
                     frame_data_url.set_fragment(resolved_url.fragment());
                     set_node_attr(node, attr_name, Some(frame_data_url.to_string()));
                 } else {
@@ -681,8 +684,7 @@ pub fn retrieve_and_embed_asset(
                     }
 
                     // Create and embed data URL
-                    // TODO: use charset
+                    let mut data_url = create_data_url(&media_type, &charset, &data, &final_url);
-                    let mut data_url = create_data_url(&media_type, &data, &final_url);
                     data_url.set_fragment(resolved_url.fragment());
                     set_node_attr(node, attr_name, Some(data_url.to_string()));
                 }

src/main.rs

@@ -299,9 +299,9 @@ fn main() {
             &options,
             0,
         ) {
-            Ok((data, final_url, media_type, _charset)) => {
+            Ok((data, final_url, media_type, charset)) => {
-                // TODO: use charset
+                let favicon_data_url: Url =
-                let favicon_data_url: Url = create_data_url(&media_type, &data, &final_url);
+                    create_data_url(&media_type, &charset, &data, &final_url);
                 dom = add_favicon(&dom.document, favicon_data_url.to_string());
             }
             Err(_) => {

src/tests/url/create_data_url.rs

@@ -13,9 +13,14 @@ mod passing {
 
     #[test]
     fn encode_string_with_specific_media_type() {
-        let mime = "application/javascript";
+        let media_type = "application/javascript";
         let data = "var word = 'hello';\nalert(word);\n";
-        let data_url = url::create_data_url(mime, data.as_bytes(), &Url::parse("data:,").unwrap());
+        let data_url = url::create_data_url(
+            media_type,
+            "",
+            data.as_bytes(),
+            &Url::parse("data:,").unwrap(),
+        );
 
         assert_eq!(
             data_url.as_str(),
@@ -28,6 +33,7 @@ mod passing {
         let data = "<svg></svg>\n";
         let data_url = url::create_data_url(
             "image/svg+xml",
+            "",
             data.as_bytes(),
             &Url::parse("data:,").unwrap(),
         );
@@ -37,4 +43,67 @@ mod passing {
             "data:image/svg+xml;base64,PHN2Zz48L3N2Zz4K"
         );
     }
+
+    #[test]
+    fn encode_string_with_specific_media_type_and_charset() {
+        let media_type = "application/javascript";
+        let charset = "utf8";
+        let data = "var word = 'hello';\nalert(word);\n";
+        let data_url = url::create_data_url(
+            media_type,
+            charset,
+            data.as_bytes(),
+            &Url::parse("data:,").unwrap(),
+        );
+
+        assert_eq!(
+            data_url.as_str(),
+            "data:application/javascript;charset=utf8;base64,dmFyIHdvcmQgPSAnaGVsbG8nOwphbGVydCh3b3JkKTsK"
+        );
+    }
+
+    #[test]
+    fn create_data_url_with_us_ascii_charset() {
+        let media_type = "";
+        let charset = "us-ascii";
+        let data = "";
+        let data_url = url::create_data_url(
+            media_type,
+            charset,
+            data.as_bytes(),
+            &Url::parse("data:,").unwrap(),
+        );
+
+        assert_eq!(data_url.as_str(), "data:;base64,");
+    }
+
+    #[test]
+    fn create_data_url_with_utf8_charset() {
+        let media_type = "";
+        let charset = "utf8";
+        let data = "";
+        let data_url = url::create_data_url(
+            media_type,
+            charset,
+            data.as_bytes(),
+            &Url::parse("data:,").unwrap(),
+        );
+
+        assert_eq!(data_url.as_str(), "data:;charset=utf8;base64,");
+    }
+
+    #[test]
+    fn create_data_url_with_media_type_text_plain_and_utf8_charset() {
+        let media_type = "text/plain";
+        let charset = "utf8";
+        let data = "";
+        let data_url = url::create_data_url(
+            media_type,
+            charset,
+            data.as_bytes(),
+            &Url::parse("data:,").unwrap(),
+        );
+
+        assert_eq!(data_url.as_str(), "data:text/plain;charset=utf8;base64,");
+    }
 }

src/tests/utils/detect_media_type.rs

@@ -195,7 +195,7 @@ mod failing {
         let dummy_url: Url = Url::parse("data:,").unwrap();
         assert_eq!(
             utils::detect_media_type(b"abcdef0123456789", &dummy_url),
-            "application/octet-stream"
+            ""
         );
     }
 }

src/tests/utils/retrieve_asset.rs

@@ -38,7 +38,7 @@ mod passing {
         assert_eq!(&media_type, "text/html");
         assert_eq!(&charset, "US-ASCII");
         assert_eq!(
-            url::create_data_url(&media_type, &data, &final_url),
+            url::create_data_url(&media_type, &charset, &data, &final_url),
             Url::parse("data:text/html;base64,dGFyZ2V0").unwrap(),
         );
         assert_eq!(
@@ -80,7 +80,7 @@ mod passing {
         .unwrap();
         assert_eq!(&media_type, "application/javascript");
         assert_eq!(&charset, "");
-        assert_eq!(url::create_data_url(&media_type, &data, &final_url), Url::parse("data:application/javascript;base64,ZG9jdW1lbnQuYm9keS5zdHlsZS5iYWNrZ3JvdW5kQ29sb3IgPSAiZ3JlZW4iOwpkb2N1bWVudC5ib2R5LnN0eWxlLmNvbG9yID0gInJlZCI7Cg==").unwrap());
+        assert_eq!(url::create_data_url(&media_type, &charset, &data, &final_url), Url::parse("data:application/javascript;base64,ZG9jdW1lbnQuYm9keS5zdHlsZS5iYWNrZ3JvdW5kQ29sb3IgPSAiZ3JlZW4iOwpkb2N1bWVudC5ib2R5LnN0eWxlLmNvbG9yID0gInJlZCI7Cg==").unwrap());
         assert_eq!(
             final_url,
             Url::parse(&format!(

src/url.rs

@@ -12,7 +12,8 @@ pub fn clean_url(url: Url) -> Url {
     url
 }
 
-pub fn create_data_url(media_type: &str, data: &[u8], final_asset_url: &Url) -> Url {
+pub fn create_data_url(media_type: &str, charset: &str, data: &[u8], final_asset_url: &Url) -> Url {
+    // TODO: move this block out of this function
     let media_type: String = if media_type.is_empty() {
         detect_media_type(data, &final_asset_url)
     } else {
@@ -21,7 +22,14 @@ pub fn create_data_url(media_type: &str, data: &[u8], final_asset_url: &Url) ->
 
     let mut data_url: Url = Url::parse("data:,").unwrap();
 
-    data_url.set_path(format!("{};base64,{}", media_type, base64::encode(data)).as_str());
+    let c: String =
+        if !charset.trim().is_empty() && !charset.trim().eq_ignore_ascii_case("US-ASCII") {
+            format!(";charset={}", charset.trim())
+        } else {
+            str!()
+        };
+
+    data_url.set_path(format!("{}{};base64,{}", media_type, c, base64::encode(data)).as_str());
 
     data_url
 }

src/utils.rs

@@ -84,9 +84,9 @@ pub fn detect_media_type_by_file_name(filename: &str) -> String {
             "woff" => "font/woff",
             "woff2" => "font/woff2",
             "xml" => "text/xml",
-            &_ => "application/octet-stream",
+            &_ => "",
         },
-        None => "application/octet-stream",
+        None => "",
     };
 
     mime.to_string()