Mirror/monolith
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge pull request #108 from rhysd/revert-manual-proxy-support

Browse source 
Revert #106 since reqwest supports system proxies by default
This commit is contained in:
Sunshine 2020-01-21 02:15:29 -05:00 committed by GitHub
commit d753c83c76
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 15 additions and 35 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
README.md
View file

@ -46,7 +46,7 @@ If compared to saving websites with `wget -mpk`, this tool embeds all assets as
- `-u`: Specify custom User-Agent
## HTTPS and HTTP proxies
Please set `https_proxy` and `http_proxy` environment variables.
Please set `https_proxy`, `http_proxy` and `no_proxy` environment variables.
## Related projects
- `Pagesaver`: https://github.com/distributed-mind/pagesaver

48
src/main.rs
View file

@ -10,9 +10,7 @@ use monolith::http::retrieve_asset;
use monolith::utils::is_valid_url;
use reqwest::blocking::Client;
use reqwest::header::{HeaderMap, HeaderValue, USER_AGENT};
use reqwest::Proxy;
use std::collections::HashMap;
use std::env;
use std::fs::File;
use std::io::{self, Error, Write};
use std::process;
@ -46,37 +44,6 @@ impl Output {
}
}
fn create_http_client(args: &AppArgs) -> Result<Client, reqwest::Error> {
let mut header_map = HeaderMap::new();
header_map.insert(
USER_AGENT,
HeaderValue::from_str(&args.user_agent).expect("Invalid User-Agent header specified"),
);
let mut builder = Client::builder()
.timeout(Duration::from_secs(10))
.danger_accept_invalid_certs(args.insecure)
.default_headers(header_map);
if let Ok(var) = env::var("https_proxy").or_else(|_| env::var("HTTPS_PROXY")) {
if !var.is_empty() {
let proxy = Proxy::https(&var)
.expect("Could not set HTTPS proxy. Please check $https_proxy env var");
builder = builder.proxy(proxy);
}
}
if let Ok(var) = env::var("http_proxy").or_else(|_| env::var("HTTP_PROXY")) {
if !var.is_empty() {
let proxy = Proxy::http(&var)
.expect("Could not set HTTP proxy. Please check $http_proxy env var");
builder = builder.proxy(proxy);
}
}
builder.build()
}
fn main() {
let app_args = AppArgs::get();
@ -89,8 +56,21 @@ fn main() {
}
let mut output = Output::new(&app_args.output).expect("Could not prepare output");
// Initialize client
let mut cache = HashMap::new();
let client = create_http_client(&app_args).expect("Failed to initialize HTTP client");
let mut header_map = HeaderMap::new();
header_map.insert(
USER_AGENT,
HeaderValue::from_str(&app_args.user_agent).expect("Invalid User-Agent header specified"),
);
let client = Client::builder()
.timeout(Duration::from_secs(10))
.danger_accept_invalid_certs(app_args.insecure)
.default_headers(header_map)
.build()
.expect("Failed to initialize HTTP client");
// Retrieve root document
let (data, final_url) = retrieve_asset(