hatter/oss-backupd
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

feat: pending add tiny-encrypt encryption method

Browse Source
This commit is contained in:
Hatter Jiang
2024-12-15 16:21:22 +08:00
parent 78dc5a228b
commit 27e107770e
7 changed files with 185 additions and 102 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@@ -1,3 +1,4 @@
.idea/
sample.gpg
sample-backupd-config.json
# ---> macOS

4
Cargo.lock generated
View File

@@ -1,5 +1,7 @@
# This file is automatically @generated by Cargo.
# It is not intended for manual editing.
version = 3
[[package]]
name = "addr2line"
version = "0.12.1"
@@ -1192,7 +1194,7 @@ checksum = "a86ed3f5f244b372d6b1a00b72ef7f8876d0bc6a78a4c9985c53614041512063"
[[package]]
name = "oss-backupd"
version = "1.1.0"
version = "1.1.1"
dependencies = [
"argparse",
"base64 0.11.0",

2
Cargo.toml
View File

@@ -1,6 +1,6 @@
[package]
name = "oss-backupd"
version = "1.1.0"
version = "1.1.1"
authors = ["Hatter Jiang <jht5945@gmail.com>"]
edition = "2018"

157
src/config_util.rs
View File

@@ -1,10 +1,10 @@
use chrono::Utc;
use json::JsonValue;
use rust_util::{new_box_ioerror, XResult};
use std::{
fs,
path::Path,
cmp::{ min, max },
};
use rust_util::{ XResult, new_box_ioerror };
use chrono::Utc;
pub const NAME: &str = env!("CARGO_PKG_NAME");
pub const VERSION: &str = env!("CARGO_PKG_VERSION");
@@ -52,9 +52,16 @@ pub struct OSSBackupdConfigItem {
pub file_name: Option<String>,
pub oss_config: Option<OSSConfig>,
pub encrypt_pubkey_file: Option<String>,
pub tiny_encrypt: Option<TinyEncryptConfig>,
pub backup_count: Option<u32>,
}
#[derive(Debug, Clone)]
pub struct TinyEncryptConfig {
pub config: String,
pub profile: Option<String>,
}
#[derive(Debug)]
pub struct OSSBackupdConfig {
pub oss_config: Option<OSSConfig>,
@@ -65,26 +72,24 @@ pub struct OSSBackupdConfig {
}
impl OSSBackupdConfig {
pub fn get_host(&self) -> String {
match &self.host {
Some(h) => remove_start_end_slash(&h),
Some(h) => remove_start_end_slash(h),
None => "default_host".to_owned(),
}
}
pub fn get_prefix(&self) -> String {
match &self.prefix {
Some(p) => remove_start_end_slash(&p),
Some(p) => remove_start_end_slash(p),
None => "default_oss_backupd".to_owned(),
}
}
}
impl OSSBackupdConfigItem {
pub fn make_oss_key(&self, oss_backupd_config: &OSSBackupdConfig, suffix: &str) -> String {
real_make_oss_key(oss_backupd_config, &self, suffix)
real_make_oss_key(oss_backupd_config, self, suffix)
}
pub fn get_file_name(&self) -> String {
@@ -95,19 +100,14 @@ impl OSSBackupdConfigItem {
}
pub fn get_safe_backup_count(&self) -> usize {
min(
max(
self.backup_count.unwrap_or(10u32) as usize,
1_usize
),
1000_usize
)
(self.backup_count.unwrap_or(10u32) as usize).clamp(1_usize, 1000_usize)
}
}
pub fn parse_config(config_json: &json::JsonValue) -> OSSBackupdConfig {
let root_oss_config_object = parse_sub_oss_config(config_json);
let encrypt_pubkey_file = get_string_value(config_json, "encrypt_pubkey_file");
let tiny_encrypt_config = get_tiny_encrypt_value(config_json);
let prefix = get_string_value(config_json, "prefix");
let host = get_string_value(config_json, "host");
let backup_count = get_u32_value(config_json, "backup_count");
@@ -115,7 +115,13 @@ pub fn parse_config(config_json: &json::JsonValue) -> OSSBackupdConfig {
let mut items_objects: Vec<OSSBackupdConfigItem> = vec![];
if items.is_array() {
for i in 0..items.len() {
items_objects.push(parse_oss_backupd_config_item(&items[i], &root_oss_config_object, &encrypt_pubkey_file, backup_count));
items_objects.push(parse_oss_backupd_config_item(
&items[i],
&root_oss_config_object,
&encrypt_pubkey_file,
&tiny_encrypt_config,
backup_count,
));
}
}
@@ -128,13 +134,17 @@ pub fn parse_config(config_json: &json::JsonValue) -> OSSBackupdConfig {
}
}
pub fn get_config_json(custom_oss_backupd_config: Option<&str>, verbose: bool) -> Option<json::JsonValue> {
pub fn get_config_json(
custom_oss_backupd_config: Option<&str>,
verbose: bool,
) -> Option<json::JsonValue> {
let config_content = get_config_content(custom_oss_backupd_config, verbose)?;
match json::parse(&config_content) {
Ok(o) => Some(o), Err(e) => {
Ok(o) => Some(o),
Err(e) => {
failure!("Parse config json failed: {}", e);
None
},
}
}
}
@@ -149,11 +159,18 @@ fn remove_start_end_slash(s: &str) -> String {
ss.to_owned()
}
fn parse_oss_backupd_config_item(item: &json::JsonValue, root_oss_config_object: &Option<OSSConfig>, root_encrypt_pubkey_file: &Option<String>, root_backup_count: Option<u32>) -> OSSBackupdConfigItem {
fn parse_oss_backupd_config_item(
item: &json::JsonValue,
root_oss_config_object: &Option<OSSConfig>,
root_encrypt_pubkey_file: &Option<String>,
root_tiny_encrypt_config: &Option<TinyEncryptConfig>,
root_backup_count: Option<u32>,
) -> OSSBackupdConfigItem {
let target = get_string_value(item, "target");
let file_name = get_string_value(item, "file_name");
let mut backup_count = get_u32_value(item, "backup_count");
let mut encrypt_pubkey_file = get_string_value(item, "encrypt_pubkey_file");
let mut tiny_encrypt = get_tiny_encrypt_value(item);
let mut oss_config = parse_sub_oss_config(item);
if let Some(ref root_oc) = root_oss_config_object {
match oss_config {
@@ -174,7 +191,7 @@ fn parse_oss_backupd_config_item(item: &json::JsonValue, root_oss_config_object:
oc.path = root_oc.path.clone();
}
oss_config = Some(oc);
},
}
None => oss_config = root_oss_config_object.clone(),
}
}
@@ -182,6 +199,9 @@ fn parse_oss_backupd_config_item(item: &json::JsonValue, root_oss_config_object:
if encrypt_pubkey_file.is_none() && root_encrypt_pubkey_file.is_some() {
encrypt_pubkey_file = root_encrypt_pubkey_file.clone();
}
if tiny_encrypt.is_none() && root_tiny_encrypt_config.is_some() {
tiny_encrypt = root_tiny_encrypt_config.clone();
}
if backup_count.is_none() && root_backup_count.is_some() {
backup_count = root_backup_count;
}
@@ -192,16 +212,25 @@ fn parse_oss_backupd_config_item(item: &json::JsonValue, root_oss_config_object:
oss_config,
backup_count,
encrypt_pubkey_file,
tiny_encrypt,
}
}
fn real_make_oss_key(oss_backupd_config: &OSSBackupdConfig, oss_backupd_config_item: &OSSBackupdConfigItem, suffix: &str) -> String {
fn real_make_oss_key(
oss_backupd_config: &OSSBackupdConfig,
oss_backupd_config_item: &OSSBackupdConfigItem,
suffix: &str,
) -> String {
let mut key = String::with_capacity(1024);
key.push_str(&oss_backupd_config.get_prefix());
key.push_str("/");
key.push('/');
key.push_str(&oss_backupd_config.get_host());
key.push_str("/");
key.push_str(&format!("{}_{}", &oss_backupd_config_item.get_file_name(), &get_now_ymdhms()));
key.push('/');
key.push_str(&format!(
"{}_{}",
&oss_backupd_config_item.get_file_name(),
&get_now_ymdhms()
));
if !suffix.is_empty() {
key.push_str(&format!(".{}", suffix));
@@ -212,7 +241,11 @@ fn real_make_oss_key(oss_backupd_config: &OSSBackupdConfig, oss_backupd_config_i
fn parse_sub_oss_config(json: &json::JsonValue) -> Option<OSSConfig> {
let root_oss_config = &json["oss_config"];
iff!(root_oss_config.is_null(), None, Some(parse_oss_config(root_oss_config)))
iff!(
root_oss_config.is_null(),
None,
Some(parse_oss_config(root_oss_config))
)
}
fn parse_oss_config(oss_config: &json::JsonValue) -> OSSConfig {
@@ -225,16 +258,28 @@ fn parse_oss_config(oss_config: &json::JsonValue) -> OSSConfig {
}
}
fn get_string_value(json: &json::JsonValue, key: &str) -> Option<String> {
fn get_string_value(json: &JsonValue, key: &str) -> Option<String> {
let value = &json[key];
value.as_str().map(|s| s.to_owned())
}
fn get_u32_value(json: &json::JsonValue, key: &str) -> Option<u32> {
fn get_tiny_encrypt_value(json: &JsonValue) -> Option<TinyEncryptConfig> {
let tiny_encrypt = &json["tiny_encrypt"];
let config = get_string_value(tiny_encrypt, "config");
let profile = get_string_value(tiny_encrypt, "profile");
if let (Some(config), profile) = (config, profile) {
Some(TinyEncryptConfig { config, profile })
} else {
None
}
}
fn get_u32_value(json: &JsonValue, key: &str) -> Option<u32> {
let value = &json[key];
match value {
json::JsonValue::String(v) => v.parse().ok(),
json::JsonValue::Number(v) => Some(f64::from(*v) as u32),
JsonValue::String(v) => v.parse().ok(),
JsonValue::Number(v) => Some(f64::from(*v) as u32),
_ => None,
}
}
@@ -247,13 +292,21 @@ fn get_config_content(custom_oss_backupd_config: Option<&str>, verbose: bool) ->
let custom_oss_backupd_config_path = Path::new(custom_oss_backupd_config_val);
if custom_oss_backupd_config_path.exists() {
return match fs::read_to_string(custom_oss_backupd_config_path) {
Ok(o) => Some(o), Err(e) => {
failure!("Read config file {} error: {}", custom_oss_backupd_config_val, e);
Ok(o) => Some(o),
Err(e) => {
failure!(
"Read config file {} error: {}",
custom_oss_backupd_config_val,
e
);
None
},
}
};
} else {
failure!("Custom config file not found: {}", custom_oss_backupd_config_val);
failure!(
"Custom config file not found: {}",
custom_oss_backupd_config_val
);
return None;
}
}
@@ -264,18 +317,17 @@ fn get_config_content(custom_oss_backupd_config: Option<&str>, verbose: bool) ->
debugging!("Read config from: {}", OSS_BACKUPD_CONFIG);
}
return match fs::read_to_string(oss_backupd_config_path) {
Ok(o) => Some(o), Err(e) => {
Ok(o) => Some(o),
Err(e) => {
failure!("Read config file {} error: {}", OSS_BACKUPD_CONFIG, e);
None
},
}
};
}
let home_dot_oss_backupd_config = & match get_user_home_dir(DOT_OSS_BACKUPD_CONFIG) {
Ok(o) => o, Err(e) => {
let home_dot_oss_backupd_config = &get_user_home_dir(DOT_OSS_BACKUPD_CONFIG).unwrap_or_else(|e| {
warning!("Get user home error: {}", e);
String::new()
},
};
});
if !home_dot_oss_backupd_config.is_empty() {
let home_dot_oss_backupd_config_path = Path::new(home_dot_oss_backupd_config);
if home_dot_oss_backupd_config_path.exists() {
@@ -283,10 +335,15 @@ fn get_config_content(custom_oss_backupd_config: Option<&str>, verbose: bool) ->
debugging!("Read config from: {}", home_dot_oss_backupd_config);
}
return match fs::read_to_string(home_dot_oss_backupd_config_path) {
Ok(o) => Some(o), Err(e) => {
failure!("Read config file {} error: {}", home_dot_oss_backupd_config, e);
Ok(o) => Some(o),
Err(e) => {
failure!(
"Read config file {} error: {}",
home_dot_oss_backupd_config,
e
);
None
},
}
};
}
}
@@ -296,10 +353,11 @@ fn get_config_content(custom_oss_backupd_config: Option<&str>, verbose: bool) ->
debugging!("Read config from: {}", ETC_OSS_BACKUPD_CONFIG);
}
return match fs::read_to_string(etc_oss_backupd_config_path) {
Ok(o) => Some(o), Err(e) => {
Ok(o) => Some(o),
Err(e) => {
failure!("Read config file {} error: {}", ETC_OSS_BACKUPD_CONFIG, e);
None
},
}
};
}
@@ -310,13 +368,16 @@ fn get_config_content(custom_oss_backupd_config: Option<&str>, verbose: bool) ->
fn get_user_home() -> XResult<String> {
match dirs::home_dir() {
None => Err(new_box_ioerror("Home dir not found!")),
Some(home_dir) => home_dir.to_str().map(|h| h.to_owned()).ok_or_else(|| new_box_ioerror("Home dir not found!")),
}
Some(home_dir) => home_dir
.to_str()
.map(|h| h.to_owned())
.ok_or_else(|| new_box_ioerror("Home dir not found!")),
}
}
fn get_user_home_dir(dir: &str) -> XResult<String> {
Ok(format!("{}/{}", get_user_home()?, dir))
}
}
fn get_now_ymdhms() -> String {
Utc::now().format("%Y%m%d_%H%M%S").to_string()

43
src/main.rs
View File

@@ -46,7 +46,7 @@ fn main() -> XResult<()> {
}
for (item_index, config_item) in oss_backupd_config.items.iter().enumerate() {
if let Err(e) = process_config_item(&options, &config_item, &oss_backupd_config, item_index) {
if let Err(e) = process_config_item(&options, config_item, &oss_backupd_config, item_index) {
failure!("Config {} not found, at item index: {}", e, item_index);
}
}
@@ -61,9 +61,16 @@ fn process_config_item(options: &Options, config_item: &OSSBackupdConfigItem,
debugging!("Process config item index: {}, config: {:?}", item_index, config_item);
}
let encrypt_pubkey_file = config_item.encrypt_pubkey_file.as_ref().ok_or_else(|| "encrypt_pubkey_file".to_owned())?;
let encrypt_pubkey_file = config_item.encrypt_pubkey_file.as_ref();
if options.verbose {
debugging!("Encrypt pubkey file: {}", encrypt_pubkey_file);
debugging!("Encrypt pubkey file: {:?}", encrypt_pubkey_file);
}
let tiny_encrypt_config = config_item.tiny_encrypt.as_ref();
if options.verbose {
debugging!("Encrypt tiny-encrypt config: {:?}", tiny_encrypt_config);
}
if encrypt_pubkey_file.is_none() && tiny_encrypt_config.is_none() {
return Err("Config encrypt_pubkey_file or tiny_encrypt MUST config one".to_string());
}
let target = config_item.target.as_ref().ok_or_else(|| "target".to_owned())?;
@@ -98,7 +105,7 @@ fn process_config_item(options: &Options, config_item: &OSSBackupdConfigItem,
let oss_client = OSSClient::new(endpoint, access_key_id, access_key_secret);
let backup_count = config_item.get_safe_backup_count();
let meta_file_name = &format!("{}/ossbackupd_meta_{}_{}.json", &oss_backupd_config.get_prefix(), &oss_backupd_config.get_host(), &config_item.get_file_name());
let new_file = format!("{}/{}", path, config_item.make_oss_key(&oss_backupd_config, "zip.gpg"));
let new_file = format!("{}/{}", path, config_item.make_oss_key(oss_backupd_config, "zip.gpg"));
if options.verbose {
debugging!("Backup count: {}", backup_count);
@@ -109,9 +116,10 @@ fn process_config_item(options: &Options, config_item: &OSSBackupdConfigItem,
let secs = get_current_secs();
let temp_zip_file = &format!("temp_file_{}.zip", secs);
let temp_pgp_file = &format!("temp_file_{}.gpg", secs);
let temp_tiny_encrypt_file = &format!("temp_file_{}.tinyenc", secs);
let remove_temp_files = || {
for f in &[temp_zip_file, temp_pgp_file] {
for f in &[temp_zip_file, temp_pgp_file, temp_tiny_encrypt_file] {
if Path::new(f).is_file() {
if options.verbose {
debugging!("Remove file: {}", f);
@@ -150,10 +158,13 @@ fn process_config_item(options: &Options, config_item: &OSSBackupdConfigItem,
return Ok(());
}
let temp_encrypted_file;
if let Some(encrypt_pubkey_file) = encrypt_pubkey_file {
#[cfg(feature = "use_sequoia_openpgp")]
let enc_file_by_pgp = || {
let open_pgp_tool = match OpenPGPTool::from_file(encrypt_pubkey_file) {
Ok(t) => t, Err(e) => {
Ok(t) => t,
Err(e) => {
print_message(MessageType::ERROR, &format!("Error in load pgp file: {}, at item index: {}", e, item_index));
return false;
},
@@ -172,7 +183,7 @@ fn process_config_item(options: &Options, config_item: &OSSBackupdConfigItem,
#[cfg(not(feature = "use_sequoia_openpgp"))]
let enc_file_by_pgp = || {
let mut cmd = std::process::Command::new("gpg");
cmd.args(&["-e", "-r", encrypt_pubkey_file, "-o", temp_pgp_file, temp_zip_file]);
cmd.args(["-e", "-r", encrypt_pubkey_file, "-o", temp_pgp_file, temp_zip_file]);
if let Err(e) = rust_util::util_cmd::run_command_and_wait(&mut cmd) {
failure!("Error in encrypt file: {}, at item index: {}", e, item_index);
false
@@ -184,8 +195,16 @@ fn process_config_item(options: &Options, config_item: &OSSBackupdConfigItem,
if !enc_file_by_pgp() {
return Ok(());
}
temp_encrypted_file = temp_pgp_file;
} else if let Some(tiny_encrypt_config) = tiny_encrypt_config {
// TODO ...
panic!("TODO to be implemented");
temp_encrypted_file = temp_tiny_encrypt_file;
} else {
return Err("Config encrypt_pubkey_file or tiny_encrypt MUST config one".to_string());
}
let file_temp_pgp_file = match File::open(temp_pgp_file) {
let file_temp_encrypted_file = match File::open(temp_encrypted_file) {
Ok(f) => f, Err(e) => {
failure!("Error in open file: {}, at item index: {}", e, item_index);
remove_temp_files();
@@ -194,9 +213,9 @@ fn process_config_item(options: &Options, config_item: &OSSBackupdConfigItem,
};
if options.verbose {
debugging!("Upload file: {}", temp_pgp_file);
debugging!("Upload file: {}", temp_encrypted_file);
}
if let Err(e) = oss_client.put_file(bucket, &new_file, oss_util::DEFAULT_URL_VALID_IN_SECS, file_temp_pgp_file) {
if let Err(e) = oss_client.put_file(bucket, &new_file, oss_util::DEFAULT_URL_VALID_IN_SECS, file_temp_encrypted_file) {
failure!("Error in encrypt file: {}, at item index: {}", e, item_index);
remove_temp_files();
return Ok(());
@@ -205,7 +224,7 @@ fn process_config_item(options: &Options, config_item: &OSSBackupdConfigItem,
if options.verbose {
debugging!("Processing meta file: {}", meta_file_name);
}
match process_oss_files(&options, &oss_client, bucket, path, meta_file_name, &new_file, backup_count) {
match process_oss_files(options, &oss_client, bucket, path, meta_file_name, &new_file, backup_count) {
Err(e) => failure!("Error: {}, at item index: {}", e, item_index),
Ok(_) => success!("Success, at item index: {}", item_index),
};
@@ -259,7 +278,7 @@ fn stringify_json_array(vec: &[String]) -> XResult<String> {
fn parse_json_array(arr: &str) -> XResult<Vec<String>> {
let mut vec: Vec<String> = vec![];
if !arr.is_empty() {
let json_arr = &json::parse(&arr)?;
let json_arr = &json::parse(arr)?;
if json_arr.is_array() {
for a in json_arr.members() {
if let Some(s) = a.as_str() {

2
src/oss_util.rs
View File

@@ -107,7 +107,7 @@ fn get_to_be_signed(verb: &str, expire_secs: u64, bucket_name: &str, key: &str)
to_be_signed.push_str(verb);
to_be_signed.push_str("\n\n\n");
to_be_signed.push_str(&expire_secs.to_string());
to_be_signed.push_str("\n");
to_be_signed.push('\n');
to_be_signed.push_str(&format!("/{}/{}", bucket_name, key));
to_be_signed
}

4
src/zip_util.rs
View File

@@ -57,12 +57,12 @@ pub fn zip_file(target: &str, zip_file: &str) -> XResult<()> {
zip.finish()?;
} else {
let mut_zip = RefCell::new(zip);
walk_dir(&target_path, &|p, e| {
walk_dir(target_path, &|p, e| {
warning!("Compress {} failed: {}", &p.display(), &e);
}, &|f| {
let options = FileOptions::default().compression_method(CompressionMethod::Stored);
let mut m_zip = mut_zip.borrow_mut();
let file_name = get_file_name(&f); // TODO file name! add path
let file_name = get_file_name(f); // TODO file name! add path
match m_zip.start_file(&file_name, options) {
Ok(_) => match File::open(f) {
Err(e) => warning!("Compress {} failed: {}", &f.display(), e),