From 27e107770e543569477500297b84e5ca34a5e429 Mon Sep 17 00:00:00 2001 From: Hatter Jiang Date: Sun, 15 Dec 2024 16:21:22 +0800 Subject: [PATCH] feat: pending add tiny-encrypt encryption method --- .gitignore | 1 + Cargo.lock | 4 +- Cargo.toml | 2 +- src/config_util.rs | 171 ++++++++++++++++++++++++++++++--------------- src/main.rs | 103 ++++++++++++++++----------- src/oss_util.rs | 2 +- src/zip_util.rs | 4 +- 7 files changed, 185 insertions(+), 102 deletions(-) diff --git a/.gitignore b/.gitignore index b7fd2de..a00b4ee 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +.idea/ sample.gpg sample-backupd-config.json # ---> macOS diff --git a/Cargo.lock b/Cargo.lock index 01f3867..19034a1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,5 +1,7 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. +version = 3 + [[package]] name = "addr2line" version = "0.12.1" @@ -1192,7 +1194,7 @@ checksum = "a86ed3f5f244b372d6b1a00b72ef7f8876d0bc6a78a4c9985c53614041512063" [[package]] name = "oss-backupd" -version = "1.1.0" +version = "1.1.1" dependencies = [ "argparse", "base64 0.11.0", diff --git a/Cargo.toml b/Cargo.toml index ea53074..37f21fe 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "oss-backupd" -version = "1.1.0" +version = "1.1.1" authors = ["Hatter Jiang "] edition = "2018" diff --git a/src/config_util.rs b/src/config_util.rs index 05aa59f..20a9a22 100644 --- a/src/config_util.rs +++ b/src/config_util.rs @@ -1,10 +1,10 @@ +use chrono::Utc; +use json::JsonValue; +use rust_util::{new_box_ioerror, XResult}; use std::{ fs, path::Path, - cmp::{ min, max }, }; -use rust_util::{ XResult, new_box_ioerror }; -use chrono::Utc; pub const NAME: &str = env!("CARGO_PKG_NAME"); pub const VERSION: &str = env!("CARGO_PKG_VERSION"); @@ -52,9 +52,16 @@ pub struct OSSBackupdConfigItem { pub file_name: Option, pub oss_config: Option, pub encrypt_pubkey_file: Option, + pub tiny_encrypt: Option, pub backup_count: Option, } +#[derive(Debug, Clone)] +pub struct TinyEncryptConfig { + pub config: String, + pub profile: Option, +} + #[derive(Debug)] pub struct OSSBackupdConfig { pub oss_config: Option, @@ -65,26 +72,24 @@ pub struct OSSBackupdConfig { } impl OSSBackupdConfig { - pub fn get_host(&self) -> String { match &self.host { - Some(h) => remove_start_end_slash(&h), + Some(h) => remove_start_end_slash(h), None => "default_host".to_owned(), } } pub fn get_prefix(&self) -> String { match &self.prefix { - Some(p) => remove_start_end_slash(&p), + Some(p) => remove_start_end_slash(p), None => "default_oss_backupd".to_owned(), } } } impl OSSBackupdConfigItem { - pub fn make_oss_key(&self, oss_backupd_config: &OSSBackupdConfig, suffix: &str) -> String { - real_make_oss_key(oss_backupd_config, &self, suffix) + real_make_oss_key(oss_backupd_config, self, suffix) } pub fn get_file_name(&self) -> String { @@ -95,19 +100,14 @@ impl OSSBackupdConfigItem { } pub fn get_safe_backup_count(&self) -> usize { - min( - max( - self.backup_count.unwrap_or(10u32) as usize, - 1_usize - ), - 1000_usize - ) + (self.backup_count.unwrap_or(10u32) as usize).clamp(1_usize, 1000_usize) } } pub fn parse_config(config_json: &json::JsonValue) -> OSSBackupdConfig { let root_oss_config_object = parse_sub_oss_config(config_json); let encrypt_pubkey_file = get_string_value(config_json, "encrypt_pubkey_file"); + let tiny_encrypt_config = get_tiny_encrypt_value(config_json); let prefix = get_string_value(config_json, "prefix"); let host = get_string_value(config_json, "host"); let backup_count = get_u32_value(config_json, "backup_count"); @@ -115,7 +115,13 @@ pub fn parse_config(config_json: &json::JsonValue) -> OSSBackupdConfig { let mut items_objects: Vec = vec![]; if items.is_array() { for i in 0..items.len() { - items_objects.push(parse_oss_backupd_config_item(&items[i], &root_oss_config_object, &encrypt_pubkey_file, backup_count)); + items_objects.push(parse_oss_backupd_config_item( + &items[i], + &root_oss_config_object, + &encrypt_pubkey_file, + &tiny_encrypt_config, + backup_count, + )); } } @@ -128,13 +134,17 @@ pub fn parse_config(config_json: &json::JsonValue) -> OSSBackupdConfig { } } -pub fn get_config_json(custom_oss_backupd_config: Option<&str>, verbose: bool) -> Option { +pub fn get_config_json( + custom_oss_backupd_config: Option<&str>, + verbose: bool, +) -> Option { let config_content = get_config_content(custom_oss_backupd_config, verbose)?; match json::parse(&config_content) { - Ok(o) => Some(o), Err(e) => { + Ok(o) => Some(o), + Err(e) => { failure!("Parse config json failed: {}", e); None - }, + } } } @@ -149,11 +159,18 @@ fn remove_start_end_slash(s: &str) -> String { ss.to_owned() } -fn parse_oss_backupd_config_item(item: &json::JsonValue, root_oss_config_object: &Option, root_encrypt_pubkey_file: &Option, root_backup_count: Option) -> OSSBackupdConfigItem { +fn parse_oss_backupd_config_item( + item: &json::JsonValue, + root_oss_config_object: &Option, + root_encrypt_pubkey_file: &Option, + root_tiny_encrypt_config: &Option, + root_backup_count: Option, +) -> OSSBackupdConfigItem { let target = get_string_value(item, "target"); let file_name = get_string_value(item, "file_name"); let mut backup_count = get_u32_value(item, "backup_count"); let mut encrypt_pubkey_file = get_string_value(item, "encrypt_pubkey_file"); + let mut tiny_encrypt = get_tiny_encrypt_value(item); let mut oss_config = parse_sub_oss_config(item); if let Some(ref root_oc) = root_oss_config_object { match oss_config { @@ -174,7 +191,7 @@ fn parse_oss_backupd_config_item(item: &json::JsonValue, root_oss_config_object: oc.path = root_oc.path.clone(); } oss_config = Some(oc); - }, + } None => oss_config = root_oss_config_object.clone(), } } @@ -182,6 +199,9 @@ fn parse_oss_backupd_config_item(item: &json::JsonValue, root_oss_config_object: if encrypt_pubkey_file.is_none() && root_encrypt_pubkey_file.is_some() { encrypt_pubkey_file = root_encrypt_pubkey_file.clone(); } + if tiny_encrypt.is_none() && root_tiny_encrypt_config.is_some() { + tiny_encrypt = root_tiny_encrypt_config.clone(); + } if backup_count.is_none() && root_backup_count.is_some() { backup_count = root_backup_count; } @@ -192,16 +212,25 @@ fn parse_oss_backupd_config_item(item: &json::JsonValue, root_oss_config_object: oss_config, backup_count, encrypt_pubkey_file, + tiny_encrypt, } } -fn real_make_oss_key(oss_backupd_config: &OSSBackupdConfig, oss_backupd_config_item: &OSSBackupdConfigItem, suffix: &str) -> String { +fn real_make_oss_key( + oss_backupd_config: &OSSBackupdConfig, + oss_backupd_config_item: &OSSBackupdConfigItem, + suffix: &str, +) -> String { let mut key = String::with_capacity(1024); key.push_str(&oss_backupd_config.get_prefix()); - key.push_str("/"); + key.push('/'); key.push_str(&oss_backupd_config.get_host()); - key.push_str("/"); - key.push_str(&format!("{}_{}", &oss_backupd_config_item.get_file_name(), &get_now_ymdhms())); + key.push('/'); + key.push_str(&format!( + "{}_{}", + &oss_backupd_config_item.get_file_name(), + &get_now_ymdhms() + )); if !suffix.is_empty() { key.push_str(&format!(".{}", suffix)); @@ -212,7 +241,11 @@ fn real_make_oss_key(oss_backupd_config: &OSSBackupdConfig, oss_backupd_config_i fn parse_sub_oss_config(json: &json::JsonValue) -> Option { let root_oss_config = &json["oss_config"]; - iff!(root_oss_config.is_null(), None, Some(parse_oss_config(root_oss_config))) + iff!( + root_oss_config.is_null(), + None, + Some(parse_oss_config(root_oss_config)) + ) } fn parse_oss_config(oss_config: &json::JsonValue) -> OSSConfig { @@ -225,35 +258,55 @@ fn parse_oss_config(oss_config: &json::JsonValue) -> OSSConfig { } } -fn get_string_value(json: &json::JsonValue, key: &str) -> Option { +fn get_string_value(json: &JsonValue, key: &str) -> Option { let value = &json[key]; value.as_str().map(|s| s.to_owned()) } -fn get_u32_value(json: &json::JsonValue, key: &str) -> Option { +fn get_tiny_encrypt_value(json: &JsonValue) -> Option { + let tiny_encrypt = &json["tiny_encrypt"]; + let config = get_string_value(tiny_encrypt, "config"); + let profile = get_string_value(tiny_encrypt, "profile"); + + if let (Some(config), profile) = (config, profile) { + Some(TinyEncryptConfig { config, profile }) + } else { + None + } +} + +fn get_u32_value(json: &JsonValue, key: &str) -> Option { let value = &json[key]; match value { - json::JsonValue::String(v) => v.parse().ok(), - json::JsonValue::Number(v) => Some(f64::from(*v) as u32), + JsonValue::String(v) => v.parse().ok(), + JsonValue::Number(v) => Some(f64::from(*v) as u32), _ => None, } } fn get_config_content(custom_oss_backupd_config: Option<&str>, verbose: bool) -> Option { if let Some(custom_oss_backupd_config_val) = custom_oss_backupd_config { - if verbose { + if verbose { debugging!("Read config from: {}", custom_oss_backupd_config_val); } let custom_oss_backupd_config_path = Path::new(custom_oss_backupd_config_val); if custom_oss_backupd_config_path.exists() { return match fs::read_to_string(custom_oss_backupd_config_path) { - Ok(o) => Some(o), Err(e) => { - failure!("Read config file {} error: {}", custom_oss_backupd_config_val, e); + Ok(o) => Some(o), + Err(e) => { + failure!( + "Read config file {} error: {}", + custom_oss_backupd_config_val, + e + ); None - }, + } }; } else { - failure!("Custom config file not found: {}", custom_oss_backupd_config_val); + failure!( + "Custom config file not found: {}", + custom_oss_backupd_config_val + ); return None; } } @@ -264,18 +317,17 @@ fn get_config_content(custom_oss_backupd_config: Option<&str>, verbose: bool) -> debugging!("Read config from: {}", OSS_BACKUPD_CONFIG); } return match fs::read_to_string(oss_backupd_config_path) { - Ok(o) => Some(o), Err(e) => { + Ok(o) => Some(o), + Err(e) => { failure!("Read config file {} error: {}", OSS_BACKUPD_CONFIG, e); None - }, + } }; } - let home_dot_oss_backupd_config = & match get_user_home_dir(DOT_OSS_BACKUPD_CONFIG) { - Ok(o) => o, Err(e) => { - warning!("Get user home error: {}", e); - String::new() - }, - }; + let home_dot_oss_backupd_config = &get_user_home_dir(DOT_OSS_BACKUPD_CONFIG).unwrap_or_else(|e| { + warning!("Get user home error: {}", e); + String::new() + }); if !home_dot_oss_backupd_config.is_empty() { let home_dot_oss_backupd_config_path = Path::new(home_dot_oss_backupd_config); if home_dot_oss_backupd_config_path.exists() { @@ -283,10 +335,15 @@ fn get_config_content(custom_oss_backupd_config: Option<&str>, verbose: bool) -> debugging!("Read config from: {}", home_dot_oss_backupd_config); } return match fs::read_to_string(home_dot_oss_backupd_config_path) { - Ok(o) => Some(o), Err(e) => { - failure!("Read config file {} error: {}", home_dot_oss_backupd_config, e); + Ok(o) => Some(o), + Err(e) => { + failure!( + "Read config file {} error: {}", + home_dot_oss_backupd_config, + e + ); None - }, + } }; } } @@ -296,10 +353,11 @@ fn get_config_content(custom_oss_backupd_config: Option<&str>, verbose: bool) -> debugging!("Read config from: {}", ETC_OSS_BACKUPD_CONFIG); } return match fs::read_to_string(etc_oss_backupd_config_path) { - Ok(o) => Some(o), Err(e) => { + Ok(o) => Some(o), + Err(e) => { failure!("Read config file {} error: {}", ETC_OSS_BACKUPD_CONFIG, e); None - }, + } }; } @@ -309,14 +367,17 @@ fn get_config_content(custom_oss_backupd_config: Option<&str>, verbose: bool) -> fn get_user_home() -> XResult { match dirs::home_dir() { - None => Err(new_box_ioerror("Home dir not found!")), - Some(home_dir) => home_dir.to_str().map(|h| h.to_owned()).ok_or_else(|| new_box_ioerror("Home dir not found!")), - } - } - + None => Err(new_box_ioerror("Home dir not found!")), + Some(home_dir) => home_dir + .to_str() + .map(|h| h.to_owned()) + .ok_or_else(|| new_box_ioerror("Home dir not found!")), + } +} + fn get_user_home_dir(dir: &str) -> XResult { - Ok(format!("{}/{}", get_user_home()?, dir)) - } + Ok(format!("{}/{}", get_user_home()?, dir)) +} fn get_now_ymdhms() -> String { Utc::now().format("%Y%m%d_%H%M%S").to_string() diff --git a/src/main.rs b/src/main.rs index 13c0b1f..2d220df 100644 --- a/src/main.rs +++ b/src/main.rs @@ -46,7 +46,7 @@ fn main() -> XResult<()> { } for (item_index, config_item) in oss_backupd_config.items.iter().enumerate() { - if let Err(e) = process_config_item(&options, &config_item, &oss_backupd_config, item_index) { + if let Err(e) = process_config_item(&options, config_item, &oss_backupd_config, item_index) { failure!("Config {} not found, at item index: {}", e, item_index); } } @@ -55,15 +55,22 @@ fn main() -> XResult<()> { Ok(()) } -fn process_config_item(options: &Options, config_item: &OSSBackupdConfigItem, +fn process_config_item(options: &Options, config_item: &OSSBackupdConfigItem, oss_backupd_config :&OSSBackupdConfig, item_index: usize) -> Result<(), String> { if options.verbose { debugging!("Process config item index: {}, config: {:?}", item_index, config_item); } - let encrypt_pubkey_file = config_item.encrypt_pubkey_file.as_ref().ok_or_else(|| "encrypt_pubkey_file".to_owned())?; + let encrypt_pubkey_file = config_item.encrypt_pubkey_file.as_ref(); if options.verbose { - debugging!("Encrypt pubkey file: {}", encrypt_pubkey_file); + debugging!("Encrypt pubkey file: {:?}", encrypt_pubkey_file); + } + let tiny_encrypt_config = config_item.tiny_encrypt.as_ref(); + if options.verbose { + debugging!("Encrypt tiny-encrypt config: {:?}", tiny_encrypt_config); + } + if encrypt_pubkey_file.is_none() && tiny_encrypt_config.is_none() { + return Err("Config encrypt_pubkey_file or tiny_encrypt MUST config one".to_string()); } let target = config_item.target.as_ref().ok_or_else(|| "target".to_owned())?; @@ -98,7 +105,7 @@ fn process_config_item(options: &Options, config_item: &OSSBackupdConfigItem, let oss_client = OSSClient::new(endpoint, access_key_id, access_key_secret); let backup_count = config_item.get_safe_backup_count(); let meta_file_name = &format!("{}/ossbackupd_meta_{}_{}.json", &oss_backupd_config.get_prefix(), &oss_backupd_config.get_host(), &config_item.get_file_name()); - let new_file = format!("{}/{}", path, config_item.make_oss_key(&oss_backupd_config, "zip.gpg")); + let new_file = format!("{}/{}", path, config_item.make_oss_key(oss_backupd_config, "zip.gpg")); if options.verbose { debugging!("Backup count: {}", backup_count); @@ -109,9 +116,10 @@ fn process_config_item(options: &Options, config_item: &OSSBackupdConfigItem, let secs = get_current_secs(); let temp_zip_file = &format!("temp_file_{}.zip", secs); let temp_pgp_file = &format!("temp_file_{}.gpg", secs); + let temp_tiny_encrypt_file = &format!("temp_file_{}.tinyenc", secs); let remove_temp_files = || { - for f in &[temp_zip_file, temp_pgp_file] { + for f in &[temp_zip_file, temp_pgp_file, temp_tiny_encrypt_file] { if Path::new(f).is_file() { if options.verbose { debugging!("Remove file: {}", f); @@ -150,42 +158,53 @@ fn process_config_item(options: &Options, config_item: &OSSBackupdConfigItem, return Ok(()); } - #[cfg(feature = "use_sequoia_openpgp")] - let enc_file_by_pgp = || { - let open_pgp_tool = match OpenPGPTool::from_file(encrypt_pubkey_file) { - Ok(t) => t, Err(e) => { - print_message(MessageType::ERROR, &format!("Error in load pgp file: {}, at item index: {}", e, item_index)); - return false; - }, + let temp_encrypted_file; + if let Some(encrypt_pubkey_file) = encrypt_pubkey_file { + #[cfg(feature = "use_sequoia_openpgp")] + let enc_file_by_pgp = || { + let open_pgp_tool = match OpenPGPTool::from_file(encrypt_pubkey_file) { + Ok(t) => t, + Err(e) => { + print_message(MessageType::ERROR, &format!("Error in load pgp file: {}, at item index: {}", e, item_index)); + return false; + }, + }; + if options.verbose { + print_message(MessageType::DEBUG, &format!("Encrypt file: {} -> {}", temp_zip_file, temp_pgp_file)); + } + if let Err(e) = open_pgp_tool.encrypt_file(temp_zip_file, temp_pgp_file, false) { + print_message(MessageType::ERROR, &format!("Error in encrypt file: {}, at item index: {}", e, item_index)); + remove_temp_files(); + false + } else { + true + } + }; + #[cfg(not(feature = "use_sequoia_openpgp"))] + let enc_file_by_pgp = || { + let mut cmd = std::process::Command::new("gpg"); + cmd.args(["-e", "-r", encrypt_pubkey_file, "-o", temp_pgp_file, temp_zip_file]); + if let Err(e) = rust_util::util_cmd::run_command_and_wait(&mut cmd) { + failure!("Error in encrypt file: {}, at item index: {}", e, item_index); + false + } else { + true + } }; - if options.verbose { - print_message(MessageType::DEBUG, &format!("Encrypt file: {} -> {}", temp_zip_file, temp_pgp_file)); - } - if let Err(e) = open_pgp_tool.encrypt_file(temp_zip_file, temp_pgp_file, false) { - print_message(MessageType::ERROR, &format!("Error in encrypt file: {}, at item index: {}", e, item_index)); - remove_temp_files(); - false - } else { - true - } - }; - #[cfg(not(feature = "use_sequoia_openpgp"))] - let enc_file_by_pgp = || { - let mut cmd = std::process::Command::new("gpg"); - cmd.args(&["-e", "-r", encrypt_pubkey_file, "-o", temp_pgp_file, temp_zip_file]); - if let Err(e) = rust_util::util_cmd::run_command_and_wait(&mut cmd) { - failure!("Error in encrypt file: {}, at item index: {}", e, item_index); - false - } else { - true - } - }; - if !enc_file_by_pgp() { - return Ok(()); + if !enc_file_by_pgp() { + return Ok(()); + } + temp_encrypted_file = temp_pgp_file; + } else if let Some(tiny_encrypt_config) = tiny_encrypt_config { + // TODO ... + panic!("TODO to be implemented"); + temp_encrypted_file = temp_tiny_encrypt_file; + } else { + return Err("Config encrypt_pubkey_file or tiny_encrypt MUST config one".to_string()); } - let file_temp_pgp_file = match File::open(temp_pgp_file) { + let file_temp_encrypted_file = match File::open(temp_encrypted_file) { Ok(f) => f, Err(e) => { failure!("Error in open file: {}, at item index: {}", e, item_index); remove_temp_files(); @@ -194,9 +213,9 @@ fn process_config_item(options: &Options, config_item: &OSSBackupdConfigItem, }; if options.verbose { - debugging!("Upload file: {}", temp_pgp_file); + debugging!("Upload file: {}", temp_encrypted_file); } - if let Err(e) = oss_client.put_file(bucket, &new_file, oss_util::DEFAULT_URL_VALID_IN_SECS, file_temp_pgp_file) { + if let Err(e) = oss_client.put_file(bucket, &new_file, oss_util::DEFAULT_URL_VALID_IN_SECS, file_temp_encrypted_file) { failure!("Error in encrypt file: {}, at item index: {}", e, item_index); remove_temp_files(); return Ok(()); @@ -205,7 +224,7 @@ fn process_config_item(options: &Options, config_item: &OSSBackupdConfigItem, if options.verbose { debugging!("Processing meta file: {}", meta_file_name); } - match process_oss_files(&options, &oss_client, bucket, path, meta_file_name, &new_file, backup_count) { + match process_oss_files(options, &oss_client, bucket, path, meta_file_name, &new_file, backup_count) { Err(e) => failure!("Error: {}, at item index: {}", e, item_index), Ok(_) => success!("Success, at item index: {}", item_index), }; @@ -259,7 +278,7 @@ fn stringify_json_array(vec: &[String]) -> XResult { fn parse_json_array(arr: &str) -> XResult> { let mut vec: Vec = vec![]; if !arr.is_empty() { - let json_arr = &json::parse(&arr)?; + let json_arr = &json::parse(arr)?; if json_arr.is_array() { for a in json_arr.members() { if let Some(s) = a.as_str() { diff --git a/src/oss_util.rs b/src/oss_util.rs index 01dac27..fcdc885 100644 --- a/src/oss_util.rs +++ b/src/oss_util.rs @@ -107,7 +107,7 @@ fn get_to_be_signed(verb: &str, expire_secs: u64, bucket_name: &str, key: &str) to_be_signed.push_str(verb); to_be_signed.push_str("\n\n\n"); to_be_signed.push_str(&expire_secs.to_string()); - to_be_signed.push_str("\n"); + to_be_signed.push('\n'); to_be_signed.push_str(&format!("/{}/{}", bucket_name, key)); to_be_signed } diff --git a/src/zip_util.rs b/src/zip_util.rs index bc7e252..ff24057 100644 --- a/src/zip_util.rs +++ b/src/zip_util.rs @@ -57,12 +57,12 @@ pub fn zip_file(target: &str, zip_file: &str) -> XResult<()> { zip.finish()?; } else { let mut_zip = RefCell::new(zip); - walk_dir(&target_path, &|p, e| { + walk_dir(target_path, &|p, e| { warning!("Compress {} failed: {}", &p.display(), &e); }, &|f| { let options = FileOptions::default().compression_method(CompressionMethod::Stored); let mut m_zip = mut_zip.borrow_mut(); - let file_name = get_file_name(&f); // TODO file name! add path + let file_name = get_file_name(f); // TODO file name! add path match m_zip.start_file(&file_name, options) { Ok(_) => match File::open(f) { Err(e) => warning!("Compress {} failed: {}", &f.display(), e),