diff --git a/Cargo.lock b/Cargo.lock index a9dff08..78b4e9b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -259,12 +259,14 @@ name = "hardownload" version = "0.1.0" dependencies = [ "clap", + "lazy_static", "log", "pretty_env_logger", "reqwest", "serde", "serde_json", "tokio", + "url", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index a1fde11..322876e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,3 +14,5 @@ serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" log = "0.4.8" pretty_env_logger = "0.4.0" +url = "2.1.1" +lazy_static = "1.4.0" diff --git a/src/cmd_download.rs b/src/cmd_download.rs index 51d5a8c..71b6762 100644 --- a/src/cmd_download.rs +++ b/src/cmd_download.rs @@ -1,9 +1,20 @@ use std::fs; use std::collections::HashSet; use clap::{ Arg, ArgMatches, SubCommand, App, }; +use url::Url; use crate::cmd::{ Command, CommandError, }; use crate::har::*; +lazy_static! { + // ignored header in send request, for 304 -> 200 + static ref IGNORE_HEADER_SET: HashSet<&'static str> = { + let mut s = HashSet::new(); + s.insert("if-none-match"); + s.insert("if-modified-since"); + s + }; +} + pub struct CommandDownload; impl Command for CommandDownload { @@ -29,44 +40,59 @@ impl Command for CommandDownload { return; }, }; - let reqeust = &har_entry.request; - let mut client = match reqeust.method.as_str() { - "GET" => client.get(&reqeust.url), - // "POST" => client.post(&reqeust.url), // TODO ??? + let har_reqeust = &har_entry.request; + let mut request = match har_reqeust.method.as_str() { + "GET" => client.get(&har_reqeust.url), + // "POST" => request.post(&reqeust.url), // TODO ??? _ => { - error!("Method not supported: {}, of url: {}", reqeust.method, reqeust.url); + error!("Method not supported: {}, of url: {}", har_reqeust.method, har_reqeust.url); return; }, }; - let mut ignore_header_names: HashSet<&str> = HashSet::new(); - ignore_header_names.insert("if-none-match"); - ignore_header_names.insert("if-modified-since"); - for header in &reqeust.headers { - if !ignore_header_names.contains(header.name.as_str()) { - client = client.header(&header.name, &header.value); + for header in &har_reqeust.headers { + if !IGNORE_HEADER_SET.contains(header.name.to_lowercase().as_str()) { + request = request.header(&header.name, &header.value); } } - let response = match client.send() { + let response = match request.send() { Ok(r) => r, Err(e) => { - error!("Request url: {}, error: {}", reqeust.url, e); + error!("Request url: {}, error: {}", har_reqeust.url, e); return; }, }; let code = response.status().as_u16(); if code != 200 { - error!("Response url: {}, not success: {}", reqeust.url, code); + error!("Response url: {}, not success: {}", har_reqeust.url, code); return; } - let _bs = match response.bytes() { + let bs = match response.bytes() { Ok(b) => b, Err(e) => { - error!("Response url: {}, error: {}", reqeust.url, e); + error!("Response url: {}, error: {}", har_reqeust.url, e); return; }, }; - todo!(); // write file + println!("GET {}, {} bytes", har_reqeust.url, bs.len()); + // todo!(); // write file })) }); Ok(()) } } + +fn _resolve_name(_base: &str, url: &str) -> String { + let ret = String::with_capacity(url.len()); + let url = match Url::parse(url) { + Ok(u) => u, Err(e) => { + warn!("Url: {}, parse failed: {}", url, e); + return url.chars() + .filter(|c| (*c >= 'a' && *c <= 'z') + || (*c >= 'A' && *c <= 'Z') + || (*c >= '0' && *c <= '9') + || *c == '_' || *c == '-' + ) + .collect::(); + }, + }; + ret +} diff --git a/src/main.rs b/src/main.rs index 253d674..6c91f23 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,4 +1,5 @@ #[macro_use] extern crate log; +#[macro_use] extern crate lazy_static; use clap::App; mod cmd;