Rewrite part of function retrieve_asset, include support for brotli and deflate (#312)
do not crash the app if reqwest throws, add support for deflate & brotli
This commit is contained in:
Sunshine
GitHub
parent
68a1531a11
commit
0df8613789
3 changed files with 79 additions and 24 deletions
60
Cargo.lock
generated
60
Cargo.lock
generated
|
|
@ -17,6 +17,21 @@ dependencies = [
|
|||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "alloc-no-stdlib"
|
||||
version = "2.0.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "35ef4730490ad1c4eae5c4325b2a95f521d023e5c885853ff7aca0a6a1631db3"
|
||||
|
||||
[[package]]
|
||||
name = "alloc-stdlib"
|
||||
version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "697ed7edc0f1711de49ce108c541623a0af97c6c60b2f6e2b65229847ac843c2"
|
||||
dependencies = [
|
||||
"alloc-no-stdlib",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "assert_cmd"
|
||||
version = "2.0.4"
|
||||
|
|
@ -37,6 +52,7 @@ version = "0.3.14"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "345fd392ab01f746c717b1357165b76f0b67a60192007b234058c9045fdcf695"
|
||||
dependencies = [
|
||||
"brotli",
|
||||
"flate2",
|
||||
"futures-core",
|
||||
"memchr",
|
||||
|
|
@ -88,6 +104,27 @@ dependencies = [
|
|||
"generic-array",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "brotli"
|
||||
version = "3.3.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a1a0b1dbcc8ae29329621f8d4f0d835787c1c38bb1401979b49d13b0b305ff68"
|
||||
dependencies = [
|
||||
"alloc-no-stdlib",
|
||||
"alloc-stdlib",
|
||||
"brotli-decompressor",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "brotli-decompressor"
|
||||
version = "2.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "59ad2d4653bf5ca36ae797b1f4bb4dbddb60ce49ca4aed8a2ce4829f60425b80"
|
||||
dependencies = [
|
||||
"alloc-no-stdlib",
|
||||
"alloc-stdlib",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bstr"
|
||||
version = "0.2.17"
|
||||
|
|
@ -197,9 +234,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "crc32fast"
|
||||
version = "1.2.1"
|
||||
version = "1.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "81156fece84ab6a9f2afdb109ce3ae577e42b1228441eded99bd77f627953b1a"
|
||||
checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
]
|
||||
|
|
@ -295,13 +332,11 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "flate2"
|
||||
version = "1.0.22"
|
||||
version = "1.0.24"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1e6988e897c1c9c485f43b47a529cef42fde0547f9d8d41a7062518f1d8fc53f"
|
||||
checksum = "f82b0f4c27ad9f8bfd1f3208d882da2b09c301bc1c828fd3a00d0216d2fbbff6"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"crc32fast",
|
||||
"libc",
|
||||
"miniz_oxide",
|
||||
]
|
||||
|
||||
|
|
@ -369,9 +404,9 @@ checksum = "88d1c26957f23603395cd326b0ffe64124b818f4449552f960d815cfba83a53d"
|
|||
|
||||
[[package]]
|
||||
name = "futures-io"
|
||||
version = "0.3.17"
|
||||
version = "0.3.21"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "522de2a0fe3e380f1bc577ba0474108faf3f6b18321dbf60b3b9c39a75073377"
|
||||
checksum = "fc4045962a5a5e935ee2fdedaa4e08284547402885ab326734432bed5d12966b"
|
||||
|
||||
[[package]]
|
||||
name = "futures-sink"
|
||||
|
|
@ -673,12 +708,11 @@ checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d"
|
|||
|
||||
[[package]]
|
||||
name = "miniz_oxide"
|
||||
version = "0.4.4"
|
||||
version = "0.5.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a92518e98c078586bc6c934028adcca4c92a53d6a958196de835170a01d84e4b"
|
||||
checksum = "6f5c75688da582b8ffc1f1799e9db273f32133c49e048f614d22ec3256773ccc"
|
||||
dependencies = [
|
||||
"adler",
|
||||
"autocfg 1.1.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -757,9 +791,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "num_cpus"
|
||||
version = "1.13.0"
|
||||
version = "1.13.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3"
|
||||
checksum = "19e64526ebdee182341572e50e9ad03965aa510cd94427a4549448f285e957a1"
|
||||
dependencies = [
|
||||
"hermit-abi",
|
||||
"libc",
|
||||
|
|
|
|||
|
|
@ -37,7 +37,7 @@ url = "2.2.2"
|
|||
[dependencies.reqwest]
|
||||
version = "0.11.11"
|
||||
default-features = false
|
||||
features = ["default-tls", "blocking", "gzip"]
|
||||
features = ["default-tls", "blocking", "gzip", "brotli", "deflate"]
|
||||
|
||||
[dev-dependencies]
|
||||
assert_cmd = "2.0.4"
|
||||
|
|
|
|||
41
src/utils.rs
41
src/utils.rs
|
|
@ -238,8 +238,8 @@ pub fn retrieve_asset(
|
|||
} else {
|
||||
// URL not in cache, we retrieve the file
|
||||
match client.get(url.as_str()).send() {
|
||||
Ok(mut response) => {
|
||||
if !options.ignore_errors && response.status() != 200 {
|
||||
Ok(response) => {
|
||||
if !options.ignore_errors && response.status() != reqwest::StatusCode::OK {
|
||||
if !options.silent {
|
||||
eprintln!(
|
||||
"{}{}{} ({}){}",
|
||||
|
|
@ -258,19 +258,17 @@ pub fn retrieve_asset(
|
|||
return Err(client.get("").send().unwrap_err());
|
||||
}
|
||||
|
||||
let response_url: Url = response.url().clone();
|
||||
|
||||
if !options.silent {
|
||||
if url.as_str() == response.url().as_str() {
|
||||
if url.as_str() == response_url.as_str() {
|
||||
eprintln!("{}{}", indent(depth).as_str(), &url);
|
||||
} else {
|
||||
eprintln!("{}{} -> {}", indent(depth).as_str(), &url, &response.url());
|
||||
eprintln!("{}{} -> {}", indent(depth).as_str(), &url, &response_url);
|
||||
}
|
||||
}
|
||||
|
||||
let new_cache_key: String = clean_url(response.url().clone()).to_string();
|
||||
|
||||
// Convert response into a byte array
|
||||
let mut data: Vec<u8> = vec![];
|
||||
response.copy_to(&mut data).unwrap();
|
||||
let new_cache_key: String = clean_url(response_url.clone()).to_string();
|
||||
|
||||
// Attempt to obtain media type and charset by reading Content-Type header
|
||||
let content_type: &str = response
|
||||
|
|
@ -281,11 +279,34 @@ pub fn retrieve_asset(
|
|||
|
||||
let (media_type, charset, _is_base64) = parse_content_type(&content_type);
|
||||
|
||||
// Convert response into a byte array
|
||||
let mut data: Vec<u8> = vec![];
|
||||
match response.bytes() {
|
||||
Ok(b) => {
|
||||
data = b.to_vec();
|
||||
}
|
||||
Err(error) => {
|
||||
if !options.silent {
|
||||
eprintln!(
|
||||
"{}{}{}{}",
|
||||
indent(depth).as_str(),
|
||||
if options.no_color { "" } else { ANSI_COLOR_RED },
|
||||
error,
|
||||
if options.no_color {
|
||||
""
|
||||
} else {
|
||||
ANSI_COLOR_RESET
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Add retrieved resource to cache
|
||||
cache.insert(new_cache_key, data.clone());
|
||||
|
||||
// Return
|
||||
Ok((data, response.url().clone(), media_type, charset))
|
||||
Ok((data, response_url, media_type, charset))
|
||||
}
|
||||
Err(error) => {
|
||||
if !options.silent {
|
||||
|
|
|
|||
Loading…
Reference in a new issue