Compare commits

..

16 Commits

Author SHA1 Message Date
d34a15f351 feat: update dependencies 2022-11-27 13:15:20 +08:00
d290249c3c feat: make smaller 2022-11-27 12:17:47 +08:00
abdf6f725c feat: use rust_util 0.6.17 2020-11-22 23:52:16 +08:00
8d6f3c2cb5 style: re-org use 2020-11-22 23:48:03 +08:00
ad6fac1a9f chore: i32->u64 2020-11-08 19:41:45 +08:00
9a4af87669 chore: use less mut variable 2020-11-08 19:38:17 +08:00
6bd539cdbd reflect: add get_check_inverval_secs 2020-10-18 23:16:46 +08:00
f9b1a72018 reflect: use builder 2020-10-18 23:13:09 +08:00
f722c95432 feat: add opt_value 2020-10-18 23:11:25 +08:00
bd82cff4f1 reflect: reflect 2020-10-18 22:57:26 +08:00
fdbb1b9443 reflect: reflect 2020-10-18 22:55:41 +08:00
acd45e75f6 feat: add is_show_debug_output 2020-10-18 22:50:22 +08:00
40748eb3bb style: code style 2020-10-18 22:46:00 +08:00
ad28d12f4c reflect: reflect 2020-10-18 22:42:59 +08:00
25cb508334 fix: fix clippy 2020-10-15 08:42:46 +08:00
e9fdb60a80 feat: do not send notification when notify token is empty 2020-10-15 08:37:30 +08:00
4 changed files with 707 additions and 524 deletions

1097
Cargo.lock generated

File diff suppressed because it is too large Load Diff

View File

@@ -7,7 +7,7 @@ edition = "2018"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies] [dependencies]
tokio = { version = "0.2", features = ["full"] } tokio = { version="1.22", features = ["rt"] }
serde = { version = "1.0", features = ["derive"] } serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0" serde_json = "1.0"
argparse = "0.2" argparse = "0.2"
@@ -23,4 +23,6 @@ file-lock = "1.1"
opt-level = 'z' opt-level = 'z'
lto = true lto = true
codegen-units = 1 codegen-units = 1
# panic = 'abort' panic = 'abort'
strip = true

View File

@@ -1,7 +1,7 @@
{ {
"check_inverval_secs": 1, "check_inverval_secs": 1,
"show_debug_output": true, "show_debug_output": true,
"notify_token": "------------", "notify_token": "",
"lock_file": ".keeprunningd.lock", "lock_file": ".keeprunningd.lock",
"items": [{ "items": [{
"grep_tokens": ["java", "app"], "grep_tokens": ["java", "app"],

View File

@@ -1,12 +1,17 @@
#[macro_use] extern crate lazy_static; #[macro_use] extern crate lazy_static;
#[macro_use] extern crate log; #[macro_use] extern crate log;
#[macro_use] extern crate rust_util;
use std::{ collections::HashMap, fs, panic, thread, time::Duration, process::Command, sync::{ Arc, Mutex } }; use std::collections::HashMap;
use std::{fs, panic, thread};
use std::time::Duration;
use std::process::Command;
use std::sync::{Arc, Mutex};
use log::LevelFilter; use log::LevelFilter;
use chrono::prelude::*; use chrono::prelude::*;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use file_lock::FileLock; use file_lock::FileLock;
use rust_util::{ util_str::read_str_to_lines, util_file::locate_file }; use rust_util::{util_str, util_file};
use dingtalk::DingTalk; use dingtalk::DingTalk;
#[derive(Debug, Clone, Serialize, Deserialize)] #[derive(Debug, Clone, Serialize, Deserialize)]
@@ -18,6 +23,16 @@ struct KeepRunningConfig {
items: Vec<KeepRunningConfigItem>, items: Vec<KeepRunningConfigItem>,
} }
impl KeepRunningConfig {
fn get_check_inverval_secs(&self) -> u64 {
self.check_inverval_secs.unwrap_or(60 * 60)
}
fn is_show_debug_output(&self) -> bool {
self.show_debug_output.unwrap_or(false)
}
}
#[derive(Debug, Clone, Serialize, Deserialize)] #[derive(Debug, Clone, Serialize, Deserialize)]
struct KeepRunningConfigItem { struct KeepRunningConfigItem {
title: String, title: String,
@@ -33,13 +48,10 @@ fn main() {
panic::set_hook(Box::new(|panic_info| { panic::set_hook(Box::new(|panic_info| {
error!("Panic in running keeprunningd: {:?}", panic_info); error!("Panic in running keeprunningd: {:?}", panic_info);
})); }));
let keep_running_config = match parse_keep_running_config() { let keep_running_config = opt_value!(parse_keep_running_config());
Some(c) => c, None => return,
};
let mut the_file_lock = None; let the_file_lock = if let Some(ref lock_file) = keep_running_config.lock_file {
if let Some(ref lock_file) = keep_running_config.lock_file { match FileLock::lock(lock_file, false, true) {
the_file_lock = match FileLock::lock(lock_file, false, true) {
Ok(lock) => { Ok(lock) => {
info!("Lock file success: {}", lock_file); info!("Lock file success: {}", lock_file);
Some(lock) Some(lock)
@@ -48,27 +60,23 @@ fn main() {
warn!("Lock file failed: {}", lock_file); warn!("Lock file failed: {}", lock_file);
return; return;
}, },
};
} }
} else { None };
if let Err(_) = simple_logging::log_to_file("/var/log/keeprunningd.log", LevelFilter::Info) { init_logger();
if let Err(_) = simple_logging::log_to_file("/tmp/keeprunningd.log", LevelFilter::Info) {
simple_logging::log_to_stderr(LevelFilter::Info);
}
}
let keep_running_config = Arc::new(keep_running_config); let keep_running_config = Arc::new(keep_running_config);
for check_cnt in 0.. { for check_cnt in 0_u64.. {
info!("Check index: {} @{:?}", check_cnt, Local::now()); info!("Check index: {} @{:?}", check_cnt, Local::now());
keep_runningd(keep_running_config.clone()); keep_runningd(keep_running_config.clone());
thread::sleep(Duration::from_secs(keep_running_config.check_inverval_secs.unwrap_or(60 * 60))); thread::sleep(Duration::from_secs(keep_running_config.get_check_inverval_secs()));
} }
drop(the_file_lock); drop(the_file_lock);
} }
fn parse_keep_running_config() -> Option<KeepRunningConfig> { fn parse_keep_running_config() -> Option<KeepRunningConfig> {
let config_file_opt = locate_file(&[ let config_file_opt = util_file::locate_file(&[
"keeprunningd.json".into(), "keeprunningd.json".into(),
"~/keeprunningd.json".into(), "~/keeprunningd.json".into(),
"/etc/keeprunningd.json".into() "/etc/keeprunningd.json".into()
@@ -99,7 +107,7 @@ fn parse_keep_running_config() -> Option<KeepRunningConfig> {
}, },
}; };
if keep_running_config.show_debug_output.unwrap_or(false) { if keep_running_config.is_show_debug_output() {
if let Ok(json) = serde_json::to_string_pretty(&keep_running_config) { if let Ok(json) = serde_json::to_string_pretty(&keep_running_config) {
debug!("Config: {}", json); debug!("Config: {}", json);
} }
@@ -109,17 +117,17 @@ fn parse_keep_running_config() -> Option<KeepRunningConfig> {
fn keep_runningd(keep_running_config: Arc<KeepRunningConfig>) { fn keep_runningd(keep_running_config: Arc<KeepRunningConfig>) {
let t = thread::spawn(move || { let t = thread::spawn(move || {
let ps_aux_lines = read_str_to_lines(&match ps_aux() { Some(p) => p, None => return, }); let ps_aux_lines = util_str::read_str_to_lines(&match ps_aux() { Some(p) => p, None => return, });
for keep_running_config_item in &keep_running_config.items { for keep_running_config_item in &keep_running_config.items {
let check_lines = ps_aux_lines.iter().filter(|ln| { let check_lines = ps_aux_lines.iter().filter(|ln| {
keep_running_config_item.grep_tokens.iter().all(|t| ln.contains(t)) keep_running_config_item.grep_tokens.iter().all(|t| ln.contains(t))
}).collect::<Vec<_>>(); }).collect::<Vec<_>>();
let mut fail_count = 0; let mut fail_count = 0;
let check_success = !check_lines.is_empty();
{ // limit `CHECK_FAIL_MAP` lock scope { // limit `CHECK_FAIL_MAP` lock scope
let cloned_title = keep_running_config_item.title.clone(); let cloned_title = keep_running_config_item.title.clone();
let mut check_fail_map = CHECK_FAIL_MAP.lock().unwrap(); let mut check_fail_map = CHECK_FAIL_MAP.lock().unwrap();
let check_success = !check_lines.is_empty();
if check_success { if check_success {
check_fail_map.insert(cloned_title, 0); check_fail_map.insert(cloned_title, 0);
} else { } else {
@@ -130,36 +138,17 @@ fn keep_runningd(keep_running_config: Arc<KeepRunningConfig>) {
} }
} }
if check_lines.is_empty() { // if check fail! if !check_success { // if check fail!
info!("Send DingTalk notification!"); info!("Send DingTalk notification!");
use tokio::runtime; block_send_notify(&keep_running_config.notify_token, &keep_running_config_item.title, fail_count);
match runtime::Builder::new().basic_scheduler().enable_all().build() {
Err(err) => error!("Prepare tokio runtime error: {}", err),
Ok(mut rt) => {
let mut sb = String::with_capacity(1024);
sb.push_str(&format!("Check failed: {}, fail count: {}", &keep_running_config_item.title, fail_count));
sb.push_str(&format!("\ncheck time: {:?}", Local::now()));
rt.block_on(send_notify(&keep_running_config.notify_token, &sb));
},
}
match &keep_running_config_item.restart_command { match &keep_running_config_item.restart_command {
Some(restart_command) if fail_count > 1 => { Some(restart_command) if fail_count > 1 => {
info!("Fail count is {} > 1, try restart command: {:?}", fail_count, restart_command); info!("Fail count is {} > 1, try restart command: {:?}", fail_count, restart_command);
let mut restart_command_iter = restart_command.iter(); restart_cmd(restart_command);
if let Some(program) = restart_command_iter.next() {
let mut cmd = Command::new(program);
while let Some(arg) = restart_command_iter.next() {
cmd.arg(arg);
}
match cmd.spawn() {
Ok(child) => info!("Ran process success, process id: {}", child.id()),
Err(e) => error!("Run restart command failed: {}", e),
}
}
}, },
_ => ( /* IGNORE */ ), _ => ( /* IGNORE */ ),
} }
} else if keep_running_config.show_debug_output.unwrap_or(false) { } else if keep_running_config.is_show_debug_output() {
debug!("Find: {:?}", &check_lines); debug!("Find: {:?}", &check_lines);
} }
} }
@@ -169,7 +158,22 @@ fn keep_runningd(keep_running_config: Arc<KeepRunningConfig>) {
} }
} }
fn block_send_notify(notify_token: &str, title: &str, fail_count: u64) {
use tokio::runtime::Builder;
match Builder::new_current_thread().enable_all().build() {
Err(err) => error!("Prepare tokio runtime error: {}", err),
Ok(rt) => {
let text = format!("Check failed: {}, fail count: {}\ncheck time: {:?}", title, fail_count, Local::now());
rt.block_on(send_notify(notify_token, &text));
},
}
}
async fn send_notify(notify_token: &str, text: &str) { async fn send_notify(notify_token: &str, text: &str) {
if notify_token.is_empty() {
info!("Notify token is empty, do not send DingTalk notification");
return;
}
match DingTalk::from_token(&notify_token) { match DingTalk::from_token(&notify_token) {
Err(err) => error!("Prepare DingTalk error: {}", err), Err(err) => error!("Prepare DingTalk error: {}", err),
Ok(dt) => if let Err(err) = dt.send_text(text).await { Ok(dt) => if let Err(err) = dt.send_text(text).await {
@@ -178,6 +182,27 @@ async fn send_notify(notify_token: &str, text: &str) {
} }
} }
fn restart_cmd(restart_command: &[String]) {
let mut restart_command_iter = restart_command.iter();
if let Some(program) = restart_command_iter.next() {
let mut cmd = Command::new(program);
for arg in restart_command_iter {
cmd.arg(arg);
}
match cmd.spawn() {
Ok(child) => info!("Ran process success, process id: {}", child.id()),
Err(e) => error!("Run restart command failed: {}", e),
}
}
}
fn init_logger() {
if simple_logging::log_to_file("/var/log/keeprunningd.log", LevelFilter::Info).is_err()
&& simple_logging::log_to_file("/tmp/keeprunningd.log", LevelFilter::Info).is_err() {
simple_logging::log_to_stderr(LevelFilter::Info);
}
}
fn ps_aux() -> Option<String> { fn ps_aux() -> Option<String> {
let mut cmd = Command::new("ps"); let mut cmd = Command::new("ps");
cmd.arg("aux"); cmd.arg("aux");
@@ -191,11 +216,8 @@ fn ps_aux() -> Option<String> {
error!("Run 'ps aux' error: {:?}", output.status); error!("Run 'ps aux' error: {:?}", output.status);
return None; return None;
} }
match String::from_utf8(output.stdout) { String::from_utf8(output.stdout).map_or_else(|err| {
Ok(output_str) => Some(output_str),
Err(err) => {
error!("Get ps output as utf8 error: {}", err); error!("Get ps output as utf8 error: {}", err);
None None
}, }, |output_str| Some(output_str))
}
} }