From ccca8e66bdfe3813d94c5de737b82bfea4f409be Mon Sep 17 00:00:00 2001 From: Hatter Jiang Date: Sat, 31 Aug 2024 20:46:10 +0800 Subject: [PATCH] feat: update osssendfile-rs --- osssendfile-rs/Cargo.lock | 43 ++++++++++++++++++++++-- osssendfile-rs/Cargo.toml | 5 +-- osssendfile-rs/src/main.rs | 67 ++++++++++++++++++++++++++++++++------ script-meta.json | 4 +-- 4 files changed, 103 insertions(+), 16 deletions(-) diff --git a/osssendfile-rs/Cargo.lock b/osssendfile-rs/Cargo.lock index d33f903..a9c64c1 100644 --- a/osssendfile-rs/Cargo.lock +++ b/osssendfile-rs/Cargo.lock @@ -381,6 +381,23 @@ version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" +[[package]] +name = "futures-io" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" + +[[package]] +name = "futures-macro" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "futures-sink" version = "0.3.30" @@ -400,9 +417,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" dependencies = [ "futures-core", + "futures-io", + "futures-macro", + "futures-sink", "futures-task", + "memchr", "pin-project-lite", "pin-utils", + "slab", ] [[package]] @@ -827,15 +849,16 @@ dependencies = [ [[package]] name = "oss" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "095ae6a93b3d51ed05c4c251a6835b7ba61357c8ff6f8e515040abe8a3583996" +checksum = "13022f1f22f5ff353e01e161ecf1d3ae4872c007c8727723975bdc975edc72e4" dependencies = [ "base64 0.11.0", "json", "reqwest", "rust-crypto", "rust_util", + "tokio-util", "urlencoding", ] @@ -855,6 +878,7 @@ dependencies = [ "serde_json", "sha2", "tokio", + "tokio-util", ] [[package]] @@ -1095,10 +1119,12 @@ dependencies = [ "system-configuration", "tokio", "tokio-native-tls", + "tokio-util", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", + "wasm-streams", "web-sys", "windows-registry", ] @@ -1773,6 +1799,19 @@ version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c62a0a307cb4a311d3a07867860911ca130c3494e8c2719593806c08bc5d0484" +[[package]] +name = "wasm-streams" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b65dc4c90b63b118468cf747d8bf3566c1913ef60be765b5730ead9e0a3ba129" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "web-sys" version = "0.3.70" diff --git a/osssendfile-rs/Cargo.toml b/osssendfile-rs/Cargo.toml index e934c0e..81ab60f 100644 --- a/osssendfile-rs/Cargo.toml +++ b/osssendfile-rs/Cargo.toml @@ -8,11 +8,12 @@ aes-gcm-stream = "0.2" base64 = "0.22.1" clap = { version = "4.5", features = ["derive"] } hex = "0.4.3" -oss = "0.3.2" +oss = "0.3.3" rand = "0.8.5" -reqwest = "0.12" +reqwest = { version = "0.12", features = ["stream"] } rust_util = "0.6.47" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" sha2 = "0.10.8" tokio = { version = "1.39", features = ["full"] } +tokio-util = "0.7.11" diff --git a/osssendfile-rs/src/main.rs b/osssendfile-rs/src/main.rs index 6fce897..c69fb25 100755 --- a/osssendfile-rs/src/main.rs +++ b/osssendfile-rs/src/main.rs @@ -8,12 +8,13 @@ //! hex = "0.4.3" //! oss = "0.3.2" //! rand = "0.8.5" -//! reqwest = "0.12" +//! reqwest = { version = "0.12", features = ["stream"] } //! rust_util = "0.6.47" //! serde = { version = "1.0", features = ["derive"] } //! serde_json = "1.0" //! sha2 = "0.10.8" //! tokio = { version = "1.39", features = ["full"] } +//! tokio-util = "0.7.11" //! ``` use aes_gcm_stream::Aes128GcmStreamEncryptor; @@ -21,21 +22,21 @@ use base64::engine::general_purpose::STANDARD; use base64::Engine; use clap::Parser; use oss::OSSClient; -use reqwest::{Client, Response}; +use reqwest::{Body, Client, Response}; use rust_util::util_io::DEFAULT_BUF_SIZE; -use rust_util::{ - failure, information, opt_result, opt_value_result, simple_error, success, - util_file, util_size, util_time, warning, XResult, -}; +use rust_util::{failure, information, opt_result, opt_value_result, simple_error, success, util_file, util_msg, util_size, util_time, warning, XResult}; use serde::{Deserialize, Serialize}; use serde_json::Value; use sha2::Digest; use sha2::Sha256; use std::collections::HashMap; -use std::fs; use std::fs::File; use std::io::{ErrorKind, Read, Write}; use std::path::PathBuf; +use std::{fs, io}; +use tokio::fs::File as TokioFile; +use tokio_util::bytes::BytesMut; +use tokio_util::codec::{Decoder, FramedRead}; const OSS_SEND_FILE_CONFIG_FILE: &str = "~/.jssp/config/osssendfile.json"; const CREATE_STS_URL: &str = "https://hatter.ink/oidc/create_sts.json"; @@ -104,6 +105,45 @@ struct SendFileResponse { sha256: String, } +#[derive(Copy, Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Default)] +pub struct CustomBytesCodec { + read_len: u64, + total_len: u64, +} + +impl CustomBytesCodec { + pub fn new(total_len: u64) -> CustomBytesCodec { + CustomBytesCodec { + read_len: 0, + total_len, + } + } +} + +impl Decoder for CustomBytesCodec { + type Item = BytesMut; + type Error = io::Error; + + fn decode(&mut self, buf: &mut BytesMut) -> Result, io::Error> { + if !buf.is_empty() { + let len = buf.len(); + self.read_len += len as u64; + util_msg::print_lastline(&format!( + "Copied {}, total: {}", + util_size::get_display_size(self.read_len as i64), + if self.total_len == 0 { + "-".to_string() + } else { + util_size::get_display_size(self.total_len as i64) + } + )); + Ok(Some(buf.split_to(len))) + } else { + Ok(None) + } + } +} + #[tokio::main] async fn main() -> XResult<()> { let args = OssSendFileArgs::parse(); @@ -147,10 +187,17 @@ async fn main() -> XResult<()> { let temp_oss_filename = format!("tempfiles/temp_transfer_{}.{}", util_time::get_current_millis(), filename_ext); + let temp_file_len = temp_file.metadata().map(|m| m.len()).unwrap_or(0); information!("Put object {:?} -> {}", &temp_file, &temp_oss_filename); - let temp_file_read = opt_result!(File::open(&temp_file), "Read temp file failed: {}"); - let put_object_response = oss_client.put_file( - &oss_send_file_config.bucket, &temp_oss_filename, 600, temp_file_read).await?; + // thanks: https://stackoverflow.com/questions/65814450/how-to-post-a-file-using-reqwest + let temp_file_read = opt_result!(TokioFile::open(&temp_file).await, "Read temp file failed: {}"); + let temp_file_stream = FramedRead::new(temp_file_read, CustomBytesCodec::new(temp_file_len)); + let temp_file_body = Body::wrap_stream(temp_file_stream); + + let put_object_response = oss_client.put_body( + &oss_send_file_config.bucket, &temp_oss_filename, 600, temp_file_body).await?; + + util_msg::clear_lastline(); if !put_object_response.status().is_success() { return simple_error!("Put object failed, status: {}", put_object_response.status().as_u16()); } diff --git a/script-meta.json b/script-meta.json index b350ee5..1bd1115 100644 --- a/script-meta.json +++ b/script-meta.json @@ -31,8 +31,8 @@ }, "osssendfile-rs": { "script_name": "osssendfile-rs", - "script_length": 12234, - "script_sha256": "ed4c480e97b87b4235b8ca6686f547465c2e8f58df2bfbb6479b88b145bb552e" + "script_length": 13853, + "script_sha256": "218eecef46329fb6d73986d3db1a07dd29fe99269f46290eda165e5f3d663ca9" }, "post-rs": { "script_name": "post-rs",