Compare commits

...

18 Commits

Author SHA1 Message Date
d34a15f351 feat: update dependencies 2022-11-27 13:15:20 +08:00
d290249c3c feat: make smaller 2022-11-27 12:17:47 +08:00
abdf6f725c feat: use rust_util 0.6.17 2020-11-22 23:52:16 +08:00
8d6f3c2cb5 style: re-org use 2020-11-22 23:48:03 +08:00
ad6fac1a9f chore: i32->u64 2020-11-08 19:41:45 +08:00
9a4af87669 chore: use less mut variable 2020-11-08 19:38:17 +08:00
6bd539cdbd reflect: add get_check_inverval_secs 2020-10-18 23:16:46 +08:00
f9b1a72018 reflect: use builder 2020-10-18 23:13:09 +08:00
f722c95432 feat: add opt_value 2020-10-18 23:11:25 +08:00
bd82cff4f1 reflect: reflect 2020-10-18 22:57:26 +08:00
fdbb1b9443 reflect: reflect 2020-10-18 22:55:41 +08:00
acd45e75f6 feat: add is_show_debug_output 2020-10-18 22:50:22 +08:00
40748eb3bb style: code style 2020-10-18 22:46:00 +08:00
ad28d12f4c reflect: reflect 2020-10-18 22:42:59 +08:00
25cb508334 fix: fix clippy 2020-10-15 08:42:46 +08:00
e9fdb60a80 feat: do not send notification when notify token is empty 2020-10-15 08:37:30 +08:00
f2ad55691c chore: mv services 2020-10-15 08:30:53 +08:00
01c6041999 chore: mv services 2020-10-15 08:30:46 +08:00
7 changed files with 707 additions and 524 deletions

1097
Cargo.lock generated

File diff suppressed because it is too large Load Diff

View File

@@ -7,7 +7,7 @@ edition = "2018"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
tokio = { version = "0.2", features = ["full"] }
tokio = { version="1.22", features = ["rt"] }
serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0"
argparse = "0.2"
@@ -23,4 +23,6 @@ file-lock = "1.1"
opt-level = 'z'
lto = true
codegen-units = 1
# panic = 'abort'
panic = 'abort'
strip = true

View File

@@ -1,7 +1,7 @@
{
"check_inverval_secs": 1,
"show_debug_output": true,
"notify_token": "------------",
"notify_token": "",
"lock_file": ".keeprunningd.lock",
"items": [{
"grep_tokens": ["java", "app"],

View File

@@ -1,12 +1,17 @@
#[macro_use] extern crate lazy_static;
#[macro_use] extern crate log;
#[macro_use] extern crate rust_util;
use std::{ collections::HashMap, fs, panic, thread, time::Duration, process::Command, sync::{ Arc, Mutex } };
use std::collections::HashMap;
use std::{fs, panic, thread};
use std::time::Duration;
use std::process::Command;
use std::sync::{Arc, Mutex};
use log::LevelFilter;
use chrono::prelude::*;
use serde::{ Deserialize, Serialize };
use serde::{Deserialize, Serialize};
use file_lock::FileLock;
use rust_util::{ util_str::read_str_to_lines, util_file::locate_file };
use rust_util::{util_str, util_file};
use dingtalk::DingTalk;
#[derive(Debug, Clone, Serialize, Deserialize)]
@@ -18,6 +23,16 @@ struct KeepRunningConfig {
items: Vec<KeepRunningConfigItem>,
}
impl KeepRunningConfig {
fn get_check_inverval_secs(&self) -> u64 {
self.check_inverval_secs.unwrap_or(60 * 60)
}
fn is_show_debug_output(&self) -> bool {
self.show_debug_output.unwrap_or(false)
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
struct KeepRunningConfigItem {
title: String,
@@ -33,13 +48,10 @@ fn main() {
panic::set_hook(Box::new(|panic_info| {
error!("Panic in running keeprunningd: {:?}", panic_info);
}));
let keep_running_config = match parse_keep_running_config() {
Some(c) => c, None => return,
};
let keep_running_config = opt_value!(parse_keep_running_config());
let mut the_file_lock = None;
if let Some(ref lock_file) = keep_running_config.lock_file {
the_file_lock = match FileLock::lock(lock_file, false, true) {
let the_file_lock = if let Some(ref lock_file) = keep_running_config.lock_file {
match FileLock::lock(lock_file, false, true) {
Ok(lock) => {
info!("Lock file success: {}", lock_file);
Some(lock)
@@ -48,27 +60,23 @@ fn main() {
warn!("Lock file failed: {}", lock_file);
return;
},
};
}
} else { None };
if let Err(_) = simple_logging::log_to_file("/var/log/keeprunningd.log", LevelFilter::Info) {
if let Err(_) = simple_logging::log_to_file("/tmp/keeprunningd.log", LevelFilter::Info) {
simple_logging::log_to_stderr(LevelFilter::Info);
}
}
init_logger();
let keep_running_config = Arc::new(keep_running_config);
for check_cnt in 0.. {
for check_cnt in 0_u64.. {
info!("Check index: {} @{:?}", check_cnt, Local::now());
keep_runningd(keep_running_config.clone());
thread::sleep(Duration::from_secs(keep_running_config.check_inverval_secs.unwrap_or(60 * 60)));
thread::sleep(Duration::from_secs(keep_running_config.get_check_inverval_secs()));
}
drop(the_file_lock);
}
fn parse_keep_running_config() -> Option<KeepRunningConfig> {
let config_file_opt = locate_file(&[
let config_file_opt = util_file::locate_file(&[
"keeprunningd.json".into(),
"~/keeprunningd.json".into(),
"/etc/keeprunningd.json".into()
@@ -99,7 +107,7 @@ fn parse_keep_running_config() -> Option<KeepRunningConfig> {
},
};
if keep_running_config.show_debug_output.unwrap_or(false) {
if keep_running_config.is_show_debug_output() {
if let Ok(json) = serde_json::to_string_pretty(&keep_running_config) {
debug!("Config: {}", json);
}
@@ -109,17 +117,17 @@ fn parse_keep_running_config() -> Option<KeepRunningConfig> {
fn keep_runningd(keep_running_config: Arc<KeepRunningConfig>) {
let t = thread::spawn(move || {
let ps_aux_lines = read_str_to_lines(&match ps_aux() { Some(p) => p, None => return, });
let ps_aux_lines = util_str::read_str_to_lines(&match ps_aux() { Some(p) => p, None => return, });
for keep_running_config_item in &keep_running_config.items {
let check_lines = ps_aux_lines.iter().filter(|ln| {
keep_running_config_item.grep_tokens.iter().all(|t| ln.contains(t))
}).collect::<Vec<_>>();
let mut fail_count = 0;
let check_success = !check_lines.is_empty();
{ // limit `CHECK_FAIL_MAP` lock scope
let cloned_title = keep_running_config_item.title.clone();
let mut check_fail_map = CHECK_FAIL_MAP.lock().unwrap();
let check_success = !check_lines.is_empty();
if check_success {
check_fail_map.insert(cloned_title, 0);
} else {
@@ -130,36 +138,17 @@ fn keep_runningd(keep_running_config: Arc<KeepRunningConfig>) {
}
}
if check_lines.is_empty() { // if check fail!
if !check_success { // if check fail!
info!("Send DingTalk notification!");
use tokio::runtime;
match runtime::Builder::new().basic_scheduler().enable_all().build() {
Err(err) => error!("Prepare tokio runtime error: {}", err),
Ok(mut rt) => {
let mut sb = String::with_capacity(1024);
sb.push_str(&format!("Check failed: {}, fail count: {}", &keep_running_config_item.title, fail_count));
sb.push_str(&format!("\ncheck time: {:?}", Local::now()));
rt.block_on(send_notify(&keep_running_config.notify_token, &sb));
},
}
block_send_notify(&keep_running_config.notify_token, &keep_running_config_item.title, fail_count);
match &keep_running_config_item.restart_command {
Some(restart_command) if fail_count > 1 => {
info!("Fail count is {} > 1, try restart command: {:?}", fail_count, restart_command);
let mut restart_command_iter = restart_command.iter();
if let Some(program) = restart_command_iter.next() {
let mut cmd = Command::new(program);
while let Some(arg) = restart_command_iter.next() {
cmd.arg(arg);
}
match cmd.spawn() {
Ok(child) => info!("Ran process success, process id: {}", child.id()),
Err(e) => error!("Run restart command failed: {}", e),
}
}
restart_cmd(restart_command);
},
_ => ( /* IGNORE */ ),
}
} else if keep_running_config.show_debug_output.unwrap_or(false) {
} else if keep_running_config.is_show_debug_output() {
debug!("Find: {:?}", &check_lines);
}
}
@@ -169,7 +158,22 @@ fn keep_runningd(keep_running_config: Arc<KeepRunningConfig>) {
}
}
fn block_send_notify(notify_token: &str, title: &str, fail_count: u64) {
use tokio::runtime::Builder;
match Builder::new_current_thread().enable_all().build() {
Err(err) => error!("Prepare tokio runtime error: {}", err),
Ok(rt) => {
let text = format!("Check failed: {}, fail count: {}\ncheck time: {:?}", title, fail_count, Local::now());
rt.block_on(send_notify(notify_token, &text));
},
}
}
async fn send_notify(notify_token: &str, text: &str) {
if notify_token.is_empty() {
info!("Notify token is empty, do not send DingTalk notification");
return;
}
match DingTalk::from_token(&notify_token) {
Err(err) => error!("Prepare DingTalk error: {}", err),
Ok(dt) => if let Err(err) = dt.send_text(text).await {
@@ -178,6 +182,27 @@ async fn send_notify(notify_token: &str, text: &str) {
}
}
fn restart_cmd(restart_command: &[String]) {
let mut restart_command_iter = restart_command.iter();
if let Some(program) = restart_command_iter.next() {
let mut cmd = Command::new(program);
for arg in restart_command_iter {
cmd.arg(arg);
}
match cmd.spawn() {
Ok(child) => info!("Ran process success, process id: {}", child.id()),
Err(e) => error!("Run restart command failed: {}", e),
}
}
}
fn init_logger() {
if simple_logging::log_to_file("/var/log/keeprunningd.log", LevelFilter::Info).is_err()
&& simple_logging::log_to_file("/tmp/keeprunningd.log", LevelFilter::Info).is_err() {
simple_logging::log_to_stderr(LevelFilter::Info);
}
}
fn ps_aux() -> Option<String> {
let mut cmd = Command::new("ps");
cmd.arg("aux");
@@ -191,11 +216,8 @@ fn ps_aux() -> Option<String> {
error!("Run 'ps aux' error: {:?}", output.status);
return None;
}
match String::from_utf8(output.stdout) {
Ok(output_str) => Some(output_str),
Err(err) => {
String::from_utf8(output.stdout).map_or_else(|err| {
error!("Get ps output as utf8 error: {}", err);
None
},
}
}, |output_str| Some(output_str))
}