feat: update osssendfile-rs

2024-08-31 20:46:10 +08:00
parent 2de76ade32
commit ccca8e66bd
4 changed files with 103 additions and 16 deletions
--- a/osssendfile-rs/Cargo.lock
+++ b/osssendfile-rs/Cargo.lock
@@ -381,6 +381,23 @@ version = "0.3.30"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d"

+[[package]]
+name = "futures-io"
+version = "0.3.30"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1"
+
+[[package]]
+name = "futures-macro"
+version = "0.3.30"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
 [[package]]
 name = "futures-sink"
 version = "0.3.30"
@@ -400,9 +417,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48"
 dependencies = [
 "futures-core",
+ "futures-io",
+ "futures-macro",
+ "futures-sink",
 "futures-task",
+ "memchr",
 "pin-project-lite",
 "pin-utils",
+ "slab",
 ]

 [[package]]
@@ -827,15 +849,16 @@ dependencies = [

 [[package]]
 name = "oss"
-version = "0.3.2"
+version = "0.3.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "095ae6a93b3d51ed05c4c251a6835b7ba61357c8ff6f8e515040abe8a3583996"
+checksum = "13022f1f22f5ff353e01e161ecf1d3ae4872c007c8727723975bdc975edc72e4"
 dependencies = [
 "base64 0.11.0",
 "json",
 "reqwest",
 "rust-crypto",
 "rust_util",
+ "tokio-util",
 "urlencoding",
 ]

@@ -855,6 +878,7 @@ dependencies = [
 "serde_json",
 "sha2",
 "tokio",
+ "tokio-util",
 ]

 [[package]]
@@ -1095,10 +1119,12 @@ dependencies = [
 "system-configuration",
 "tokio",
 "tokio-native-tls",
+ "tokio-util",
 "tower-service",
 "url",
 "wasm-bindgen",
 "wasm-bindgen-futures",
+ "wasm-streams",
 "web-sys",
 "windows-registry",
 ]
@@ -1773,6 +1799,19 @@ version = "0.2.93"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c62a0a307cb4a311d3a07867860911ca130c3494e8c2719593806c08bc5d0484"

+[[package]]
+name = "wasm-streams"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b65dc4c90b63b118468cf747d8bf3566c1913ef60be765b5730ead9e0a3ba129"
+dependencies = [
+ "futures-util",
+ "js-sys",
+ "wasm-bindgen",
+ "wasm-bindgen-futures",
+ "web-sys",
+]
+
 [[package]]
 name = "web-sys"
 version = "0.3.70"
--- a/osssendfile-rs/Cargo.toml
+++ b/osssendfile-rs/Cargo.toml
@@ -8,11 +8,12 @@ aes-gcm-stream = "0.2"
 base64 = "0.22.1"
 clap = { version = "4.5", features = ["derive"] }
 hex = "0.4.3"
-oss = "0.3.2"
+oss = "0.3.3"
 rand = "0.8.5"
-reqwest = "0.12"
+reqwest = { version = "0.12", features = ["stream"] }
 rust_util = "0.6.47"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 sha2 = "0.10.8"
 tokio = { version = "1.39", features = ["full"] }
+tokio-util = "0.7.11"
--- a/osssendfile-rs/src/main.rs
+++ b/osssendfile-rs/src/main.rs
@@ -8,12 +8,13 @@
 //! hex = "0.4.3"
 //! oss = "0.3.2"
 //! rand = "0.8.5"
-//! reqwest = "0.12"
+//! reqwest = { version = "0.12", features = ["stream"] }
 //! rust_util = "0.6.47"
 //! serde = { version = "1.0", features = ["derive"] }
 //! serde_json = "1.0"
 //! sha2 = "0.10.8"
 //! tokio = { version = "1.39", features = ["full"] }
+//! tokio-util = "0.7.11"
 //! ```

 use aes_gcm_stream::Aes128GcmStreamEncryptor;
@@ -21,21 +22,21 @@ use base64::engine::general_purpose::STANDARD;
 use base64::Engine;
 use clap::Parser;
 use oss::OSSClient;
-use reqwest::{Client, Response};
+use reqwest::{Body, Client, Response};
 use rust_util::util_io::DEFAULT_BUF_SIZE;
-use rust_util::{
-    failure, information, opt_result, opt_value_result, simple_error, success,
-    util_file, util_size, util_time, warning, XResult,
-};
+use rust_util::{failure, information, opt_result, opt_value_result, simple_error, success, util_file, util_msg, util_size, util_time, warning, XResult};
 use serde::{Deserialize, Serialize};
 use serde_json::Value;
 use sha2::Digest;
 use sha2::Sha256;
 use std::collections::HashMap;
-use std::fs;
 use std::fs::File;
 use std::io::{ErrorKind, Read, Write};
 use std::path::PathBuf;
+use std::{fs, io};
+use tokio::fs::File as TokioFile;
+use tokio_util::bytes::BytesMut;
+use tokio_util::codec::{Decoder, FramedRead};

 const OSS_SEND_FILE_CONFIG_FILE: &str = "~/.jssp/config/osssendfile.json";
 const CREATE_STS_URL: &str = "https://hatter.ink/oidc/create_sts.json";
@@ -104,6 +105,45 @@ struct SendFileResponse {
    sha256: String,
 }

+#[derive(Copy, Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Default)]
+pub struct CustomBytesCodec {
+    read_len: u64,
+    total_len: u64,
+}
+
+impl CustomBytesCodec {
+    pub fn new(total_len: u64) -> CustomBytesCodec {
+        CustomBytesCodec {
+            read_len: 0,
+            total_len,
+        }
+    }
+}
+
+impl Decoder for CustomBytesCodec {
+    type Item = BytesMut;
+    type Error = io::Error;
+
+    fn decode(&mut self, buf: &mut BytesMut) -> Result<Option<BytesMut>, io::Error> {
+        if !buf.is_empty() {
+            let len = buf.len();
+            self.read_len += len as u64;
+            util_msg::print_lastline(&format!(
+                "Copied {}, total: {}",
+                util_size::get_display_size(self.read_len as i64),
+                if self.total_len == 0 {
+                    "-".to_string()
+                } else {
+                    util_size::get_display_size(self.total_len as i64)
+                }
+            ));
+            Ok(Some(buf.split_to(len)))
+        } else {
+            Ok(None)
+        }
+    }
+}
+
 #[tokio::main]
 async fn main() -> XResult<()> {
    let args = OssSendFileArgs::parse();
@@ -147,10 +187,17 @@ async fn main() -> XResult<()> {

    let temp_oss_filename = format!("tempfiles/temp_transfer_{}.{}", util_time::get_current_millis(), filename_ext);

+    let temp_file_len = temp_file.metadata().map(|m| m.len()).unwrap_or(0);
    information!("Put object {:?} -> {}", &temp_file, &temp_oss_filename);
-    let temp_file_read = opt_result!(File::open(&temp_file), "Read temp file failed: {}");
-    let put_object_response = oss_client.put_file(
-        &oss_send_file_config.bucket, &temp_oss_filename, 600, temp_file_read).await?;
+    // thanks: https://stackoverflow.com/questions/65814450/how-to-post-a-file-using-reqwest
+    let temp_file_read = opt_result!(TokioFile::open(&temp_file).await, "Read temp file failed: {}");
+    let temp_file_stream = FramedRead::new(temp_file_read, CustomBytesCodec::new(temp_file_len));
+    let temp_file_body = Body::wrap_stream(temp_file_stream);
+
+    let put_object_response = oss_client.put_body(
+        &oss_send_file_config.bucket, &temp_oss_filename, 600, temp_file_body).await?;
+
+    util_msg::clear_lastline();
    if !put_object_response.status().is_success() {
        return simple_error!("Put object failed, status: {}", put_object_response.status().as_u16());
    }
--- a/script-meta.json
+++ b/script-meta.json
@@ -31,8 +31,8 @@
  },
  "osssendfile-rs": {
    "script_name": "osssendfile-rs",
-    "script_length": 12234,
-    "script_sha256": "ed4c480e97b87b4235b8ca6686f547465c2e8f58df2bfbb6479b88b145bb552e"
+    "script_length": 13853,
+    "script_sha256": "218eecef46329fb6d73986d3db1a07dd29fe99269f46290eda165e5f3d663ca9"
  },
  "post-rs": {
    "script_name": "post-rs",