Rewrite part of function retrieve_asset, include support for brotli and deflate (#312)

do not crash the app if reqwest throws, add support for deflate & brotli

Cargo.lock

@@ -17,6 +17,21 @@ dependencies = [
  "memchr",
 ]
 
+[[package]]
+name = "alloc-no-stdlib"
+version = "2.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "35ef4730490ad1c4eae5c4325b2a95f521d023e5c885853ff7aca0a6a1631db3"
+
+[[package]]
+name = "alloc-stdlib"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "697ed7edc0f1711de49ce108c541623a0af97c6c60b2f6e2b65229847ac843c2"
+dependencies = [
+ "alloc-no-stdlib",
+]
+
 [[package]]
 name = "assert_cmd"
 version = "2.0.4"
@@ -37,6 +52,7 @@ version = "0.3.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "345fd392ab01f746c717b1357165b76f0b67a60192007b234058c9045fdcf695"
 dependencies = [
+ "brotli",
  "flate2",
  "futures-core",
  "memchr",
@@ -88,6 +104,27 @@ dependencies = [
  "generic-array",
 ]
 
+[[package]]
+name = "brotli"
+version = "3.3.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a1a0b1dbcc8ae29329621f8d4f0d835787c1c38bb1401979b49d13b0b305ff68"
+dependencies = [
+ "alloc-no-stdlib",
+ "alloc-stdlib",
+ "brotli-decompressor",
+]
+
+[[package]]
+name = "brotli-decompressor"
+version = "2.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "59ad2d4653bf5ca36ae797b1f4bb4dbddb60ce49ca4aed8a2ce4829f60425b80"
+dependencies = [
+ "alloc-no-stdlib",
+ "alloc-stdlib",
+]
+
 [[package]]
 name = "bstr"
 version = "0.2.17"
@@ -197,9 +234,9 @@ dependencies = [
 
 [[package]]
 name = "crc32fast"
-version = "1.2.1"
+version = "1.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "81156fece84ab6a9f2afdb109ce3ae577e42b1228441eded99bd77f627953b1a"
+checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d"
 dependencies = [
  "cfg-if",
 ]
@@ -295,13 +332,11 @@ dependencies = [
 
 [[package]]
 name = "flate2"
-version = "1.0.22"
+version = "1.0.24"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1e6988e897c1c9c485f43b47a529cef42fde0547f9d8d41a7062518f1d8fc53f"
+checksum = "f82b0f4c27ad9f8bfd1f3208d882da2b09c301bc1c828fd3a00d0216d2fbbff6"
 dependencies = [
- "cfg-if",
  "crc32fast",
- "libc",
  "miniz_oxide",
 ]
 
@@ -369,9 +404,9 @@ checksum = "88d1c26957f23603395cd326b0ffe64124b818f4449552f960d815cfba83a53d"
 
 [[package]]
 name = "futures-io"
-version = "0.3.17"
+version = "0.3.21"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "522de2a0fe3e380f1bc577ba0474108faf3f6b18321dbf60b3b9c39a75073377"
+checksum = "fc4045962a5a5e935ee2fdedaa4e08284547402885ab326734432bed5d12966b"
 
 [[package]]
 name = "futures-sink"
@@ -673,12 +708,11 @@ checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d"
 
 [[package]]
 name = "miniz_oxide"
-version = "0.4.4"
+version = "0.5.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a92518e98c078586bc6c934028adcca4c92a53d6a958196de835170a01d84e4b"
+checksum = "6f5c75688da582b8ffc1f1799e9db273f32133c49e048f614d22ec3256773ccc"
 dependencies = [
  "adler",
- "autocfg 1.1.0",
 ]
 
 [[package]]
@@ -757,9 +791,9 @@ dependencies = [
 
 [[package]]
 name = "num_cpus"
-version = "1.13.0"
+version = "1.13.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3"
+checksum = "19e64526ebdee182341572e50e9ad03965aa510cd94427a4549448f285e957a1"
 dependencies = [
  "hermit-abi",
  "libc",

Cargo.toml

@@ -37,7 +37,7 @@ url = "2.2.2"
 [dependencies.reqwest]
 version = "0.11.11"
 default-features = false
-features = ["default-tls", "blocking", "gzip"]
+features = ["default-tls", "blocking", "gzip", "brotli", "deflate"]
 
 [dev-dependencies]
 assert_cmd = "2.0.4"

src/utils.rs

@@ -238,8 +238,8 @@ pub fn retrieve_asset(
         } else {
             // URL not in cache, we retrieve the file
             match client.get(url.as_str()).send() {
-                Ok(mut response) => {
+                Ok(response) => {
-                    if !options.ignore_errors && response.status() != 200 {
+                    if !options.ignore_errors && response.status() != reqwest::StatusCode::OK {
                         if !options.silent {
                             eprintln!(
                                 "{}{}{} ({}){}",
@@ -258,19 +258,17 @@ pub fn retrieve_asset(
                         return Err(client.get("").send().unwrap_err());
                     }
 
+                    let response_url: Url = response.url().clone();
+
                     if !options.silent {
-                        if url.as_str() == response.url().as_str() {
+                        if url.as_str() == response_url.as_str() {
                             eprintln!("{}{}", indent(depth).as_str(), &url);
                         } else {
-                            eprintln!("{}{} -> {}", indent(depth).as_str(), &url, &response.url());
+                            eprintln!("{}{} -> {}", indent(depth).as_str(), &url, &response_url);
                         }
                     }
 
-                    let new_cache_key: String = clean_url(response.url().clone()).to_string();
+                    let new_cache_key: String = clean_url(response_url.clone()).to_string();
-
-                    // Convert response into a byte array
-                    let mut data: Vec<u8> = vec![];
-                    response.copy_to(&mut data).unwrap();
 
                     // Attempt to obtain media type and charset by reading Content-Type header
                     let content_type: &str = response
@@ -281,11 +279,34 @@ pub fn retrieve_asset(
 
                     let (media_type, charset, _is_base64) = parse_content_type(&content_type);
 
+                    // Convert response into a byte array
+                    let mut data: Vec<u8> = vec![];
+                    match response.bytes() {
+                        Ok(b) => {
+                            data = b.to_vec();
+                        }
+                        Err(error) => {
+                            if !options.silent {
+                                eprintln!(
+                                    "{}{}{}{}",
+                                    indent(depth).as_str(),
+                                    if options.no_color { "" } else { ANSI_COLOR_RED },
+                                    error,
+                                    if options.no_color {
+                                        ""
+                                    } else {
+                                        ANSI_COLOR_RESET
+                                    },
+                                );
+                            }
+                        }
+                    }
+
                     // Add retrieved resource to cache
                     cache.insert(new_cache_key, data.clone());
 
                     // Return
-                    Ok((data, response.url().clone(), media_type, charset))
+                    Ok((data, response_url, media_type, charset))
                 }
                 Err(error) => {
                     if !options.silent {