Merge pull request #97 from rhysd/reqwest-0.10.0

Upgrade reqwest to v0.10.0 for better binary size and build time

Cargo.toml

@@ -17,5 +17,9 @@ clap = "2.33.0"
 html5ever = "0.24.1"
 lazy_static = "1.4.0"
 regex = "1.3.1"
-reqwest = "0.9.20"
 url = "2.1.0"
+
+[dependencies.reqwest]
+version = "0.10.*"
+default-features = false
+features = ["default-tls", "blocking", "gzip"]

src/html.rs

@@ -10,7 +10,7 @@ use html5ever::serialize::{serialize, SerializeOpts};
 use html5ever::tendril::{format_tendril, Tendril, TendrilSink};
 use html5ever::tree_builder::{Attribute, TreeSink};
 use html5ever::{local_name, namespace_url, ns};
-use reqwest::Client;
+use reqwest::blocking::Client;
 use std::collections::HashMap;
 use std::default::Default;
 

src/http.rs

@@ -1,6 +1,6 @@
 use crate::utils::{clean_url, data_to_dataurl, is_data_url};
+use reqwest::blocking::Client;
 use reqwest::header::CONTENT_TYPE;
-use reqwest::Client;
 use std::collections::HashMap;
 
 pub fn retrieve_asset(
@@ -26,17 +26,17 @@ pub fn retrieve_asset(
         } else {
             // url not in cache, we request it
             let mut response = client.get(url).send()?;
+            let res_url = response.url().to_string();
 
             if !opt_silent {
-                let res_url = response.url().as_str();
                 if url == res_url {
                     eprintln!("{}", &url);
                 } else {
-                    eprintln!("{} -> {}", &url, res_url);
+                    eprintln!("{} -> {}", &url, &res_url);
                 }
             }
 
-            let new_cache_key = clean_url(response.url());
+            let new_cache_key = clean_url(&res_url);
 
             if as_dataurl {
                 // Convert response into a byte array
@@ -56,12 +56,12 @@ pub fn retrieve_asset(
                 let dataurl = data_to_dataurl(&mimetype, &data);
                 // insert in cache
                 cache.insert(new_cache_key, dataurl.clone());
-                Ok((dataurl, response.url().to_string()))
+                Ok((dataurl, res_url))
             } else {
                 let content = response.text().unwrap();
                 // insert in cache
                 cache.insert(new_cache_key, content.clone());
-                Ok((content, response.url().to_string()))
+                Ok((content, res_url))
             }
         }
     }

src/main.rs

@@ -8,6 +8,7 @@ use crate::args::AppArgs;
 use monolith::html::{html_to_dom, stringify_document, walk_and_embed_assets};
 use monolith::http::retrieve_asset;
 use monolith::utils::is_valid_url;
+use reqwest::blocking::Client;
 use reqwest::header::{HeaderMap, HeaderValue, USER_AGENT};
 use std::collections::HashMap;
 use std::fs::{remove_file, File};
@@ -53,7 +54,7 @@ fn main() {
                 return;
             }
         };
-        let client = reqwest::Client::builder()
+        let client = Client::builder()
             .timeout(Duration::from_secs(10))
             .danger_accept_invalid_certs(app_args.insecure)
             .default_headers(header_map)

src/tests/html.rs

@@ -3,6 +3,7 @@ use crate::html::{
 };
 use html5ever::rcdom::{Handle, NodeData};
 use html5ever::serialize::{serialize, SerializeOpts};
+use reqwest::blocking::Client;
 use std::collections::HashMap;
 
 #[test]
@@ -72,7 +73,7 @@ fn test_walk_and_embed_assets() {
     let opt_no_images: bool = false;
     let opt_silent = true;
 
-    let client = reqwest::Client::new();
+    let client = Client::new();
 
     walk_and_embed_assets(
         cache,
@@ -108,7 +109,7 @@ fn test_walk_and_embed_assets_ensure_no_recursive_iframe() {
     let opt_no_images: bool = false;
     let opt_silent = true;
 
-    let client = reqwest::Client::new();
+    let client = Client::new();
 
     walk_and_embed_assets(
         cache,
@@ -145,7 +146,7 @@ fn test_walk_and_embed_assets_no_css() {
     let opt_no_js: bool = false;
     let opt_no_images: bool = false;
     let opt_silent = true;
-    let client = reqwest::Client::new();
+    let client = Client::new();
 
     walk_and_embed_assets(
         cache,
@@ -190,7 +191,7 @@ fn test_walk_and_embed_assets_no_images() {
     let opt_no_images: bool = true;
     let opt_silent = true;
 
-    let client = reqwest::Client::new();
+    let client = Client::new();
 
     walk_and_embed_assets(
         cache,
@@ -236,7 +237,7 @@ fn test_walk_and_embed_assets_no_frames() {
     let opt_no_js: bool = false;
     let opt_no_images: bool = false;
     let opt_silent = true;
-    let client = reqwest::Client::new();
+    let client = Client::new();
 
     walk_and_embed_assets(
         cache,
@@ -275,7 +276,7 @@ fn test_walk_and_embed_assets_no_js() {
     let opt_no_images: bool = false;
     let opt_silent = true;
 
-    let client = reqwest::Client::new();
+    let client = Client::new();
 
     walk_and_embed_assets(
         cache,
@@ -307,7 +308,7 @@ fn test_walk_and_embed_with_no_integrity() {
     let dom = html_to_dom(&html);
     let url = "http://localhost";
     let cache = &mut HashMap::new();
-    let client = reqwest::Client::new();
+    let client = Client::new();
     let opt_no_css: bool = true;
     let opt_no_frames: bool = true;
     let opt_no_js: bool = true;

src/tests/http.rs

@@ -1,9 +1,11 @@
 use crate::http::retrieve_asset;
+use reqwest::blocking::Client;
 use std::collections::HashMap;
+
 #[test]
 fn test_retrieve_asset() {
     let cache = &mut HashMap::new();
-    let client = reqwest::Client::new();
+    let client = Client::new();
     let (data, final_url) =
         retrieve_asset(cache, &client, "data:text/html;base64,...", true, "", false).unwrap();
     assert_eq!(&data, "data:text/html;base64,...");

src/utils.rs

@@ -1,7 +1,7 @@
 use crate::http::retrieve_asset;
 use base64::encode;
 use regex::Regex;
-use reqwest::Client;
+use reqwest::blocking::Client;
 use std::collections::HashMap;
 use url::{ParseError, Url};