add url, lazy_static

Cargo.lock

@@ -259,12 +259,14 @@ name = "hardownload"
 version = "0.1.0"
 dependencies = [
  "clap",
+ "lazy_static",
  "log",
  "pretty_env_logger",
  "reqwest",
  "serde",
  "serde_json",
  "tokio",
+ "url",
 ]
 
 [[package]]

Cargo.toml

@@ -14,3 +14,5 @@ serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 log = "0.4.8"
 pretty_env_logger = "0.4.0"
+url = "2.1.1"
+lazy_static = "1.4.0"

src/cmd_download.rs

@@ -1,9 +1,20 @@
 use std::fs;
 use std::collections::HashSet;
 use clap::{ Arg, ArgMatches, SubCommand, App, };
+use url::Url;
 use crate::cmd::{ Command, CommandError, };
 use crate::har::*;
 
+lazy_static! {
+    // ignored header in send request, for 304 -> 200
+    static ref IGNORE_HEADER_SET: HashSet<&'static str> = {
+        let mut s = HashSet::new();
+        s.insert("if-none-match");
+        s.insert("if-modified-since");
+        s
+    };
+}
+
 pub struct CommandDownload;
 
 impl Command for CommandDownload {
@@ -29,44 +40,59 @@ impl Command for CommandDownload {
                         return;
                     },
                 };
-                let reqeust = &har_entry.request;
+                let har_reqeust = &har_entry.request;
-                let mut client = match reqeust.method.as_str() {
+                let mut request = match har_reqeust.method.as_str() {
-                    "GET" => client.get(&reqeust.url),
+                    "GET" => client.get(&har_reqeust.url),
-                    // "POST" => client.post(&reqeust.url), // TODO ???
+                    // "POST" => request.post(&reqeust.url), // TODO ???
                     _ => {
-                        error!("Method not supported: {}, of url: {}", reqeust.method, reqeust.url);
+                        error!("Method not supported: {}, of url: {}", har_reqeust.method, har_reqeust.url);
                         return;
                     },
                 };
-                let mut ignore_header_names: HashSet<&str> = HashSet::new();
+                for header in &har_reqeust.headers {
-                ignore_header_names.insert("if-none-match");
+                    if !IGNORE_HEADER_SET.contains(header.name.to_lowercase().as_str()) {
-                ignore_header_names.insert("if-modified-since");
+                        request = request.header(&header.name, &header.value);
-                for header in &reqeust.headers {
-                    if !ignore_header_names.contains(header.name.as_str()) {
-                        client = client.header(&header.name, &header.value);
                     }
                 }
-                let response = match client.send() {
+                let response = match request.send() {
                     Ok(r) => r, Err(e) => {
-                        error!("Request url: {}, error: {}", reqeust.url, e);
+                        error!("Request url: {}, error: {}", har_reqeust.url, e);
                         return;
                     },
                 };
                 let code = response.status().as_u16();
                 if code != 200 {
-                    error!("Response  url: {}, not success: {}", reqeust.url, code);
+                    error!("Response  url: {}, not success: {}", har_reqeust.url, code);
                     return;
                 }
-                let _bs = match response.bytes() {
+                let bs = match response.bytes() {
                     Ok(b) => b, Err(e) => {
-                        error!("Response url: {}, error: {}", reqeust.url, e);
+                        error!("Response url: {}, error: {}", har_reqeust.url, e);
                         return;
                     },
                 };
-                todo!(); // write file
+                println!("GET {}, {} bytes", har_reqeust.url, bs.len());
+                // todo!(); // write file
             }))
         });
 
         Ok(())
     }
 }
+
+fn _resolve_name(_base: &str, url: &str) -> String {
+    let ret = String::with_capacity(url.len());
+    let url = match Url::parse(url) {
+        Ok(u) => u, Err(e) => {
+            warn!("Url: {}, parse failed: {}", url, e);
+            return url.chars()
+                      .filter(|c| (*c >= 'a' && *c <= 'z')
+                               || (*c >= 'A' && *c <= 'Z')
+                               || (*c >= '0' && *c <= '9')
+                               ||  *c == '_' || *c == '-'
+                             )
+                      .collect::<String>();
+        },
+    };
+    ret
+}

src/main.rs

@@ -1,4 +1,5 @@
 #[macro_use] extern crate log;
+#[macro_use] extern crate lazy_static;
 use clap::App;
 
 mod cmd;