Compare commits
18 Commits
38ffef632e
...
master
| Author | SHA1 | Date | |
|---|---|---|---|
|
d34a15f351
|
|||
|
d290249c3c
|
|||
| abdf6f725c | |||
| 8d6f3c2cb5 | |||
| ad6fac1a9f | |||
| 9a4af87669 | |||
| 6bd539cdbd | |||
| f9b1a72018 | |||
| f722c95432 | |||
| bd82cff4f1 | |||
| fdbb1b9443 | |||
| acd45e75f6 | |||
| 40748eb3bb | |||
| ad28d12f4c | |||
| 25cb508334 | |||
| e9fdb60a80 | |||
| f2ad55691c | |||
| 01c6041999 |
1097
Cargo.lock
generated
1097
Cargo.lock
generated
File diff suppressed because it is too large
Load Diff
@@ -7,7 +7,7 @@ edition = "2018"
|
||||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||
|
||||
[dependencies]
|
||||
tokio = { version = "0.2", features = ["full"] }
|
||||
tokio = { version="1.22", features = ["rt"] }
|
||||
serde = { version = "1.0", features = ["derive"] }
|
||||
serde_json = "1.0"
|
||||
argparse = "0.2"
|
||||
@@ -23,4 +23,6 @@ file-lock = "1.1"
|
||||
opt-level = 'z'
|
||||
lto = true
|
||||
codegen-units = 1
|
||||
# panic = 'abort'
|
||||
panic = 'abort'
|
||||
strip = true
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"check_inverval_secs": 1,
|
||||
"show_debug_output": true,
|
||||
"notify_token": "------------",
|
||||
"notify_token": "",
|
||||
"lock_file": ".keeprunningd.lock",
|
||||
"items": [{
|
||||
"grep_tokens": ["java", "app"],
|
||||
|
||||
126
src/main.rs
126
src/main.rs
@@ -1,12 +1,17 @@
|
||||
#[macro_use] extern crate lazy_static;
|
||||
#[macro_use] extern crate log;
|
||||
#[macro_use] extern crate rust_util;
|
||||
|
||||
use std::{ collections::HashMap, fs, panic, thread, time::Duration, process::Command, sync::{ Arc, Mutex } };
|
||||
use std::collections::HashMap;
|
||||
use std::{fs, panic, thread};
|
||||
use std::time::Duration;
|
||||
use std::process::Command;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use log::LevelFilter;
|
||||
use chrono::prelude::*;
|
||||
use serde::{ Deserialize, Serialize };
|
||||
use serde::{Deserialize, Serialize};
|
||||
use file_lock::FileLock;
|
||||
use rust_util::{ util_str::read_str_to_lines, util_file::locate_file };
|
||||
use rust_util::{util_str, util_file};
|
||||
use dingtalk::DingTalk;
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
@@ -18,6 +23,16 @@ struct KeepRunningConfig {
|
||||
items: Vec<KeepRunningConfigItem>,
|
||||
}
|
||||
|
||||
impl KeepRunningConfig {
|
||||
fn get_check_inverval_secs(&self) -> u64 {
|
||||
self.check_inverval_secs.unwrap_or(60 * 60)
|
||||
}
|
||||
|
||||
fn is_show_debug_output(&self) -> bool {
|
||||
self.show_debug_output.unwrap_or(false)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
struct KeepRunningConfigItem {
|
||||
title: String,
|
||||
@@ -33,13 +48,10 @@ fn main() {
|
||||
panic::set_hook(Box::new(|panic_info| {
|
||||
error!("Panic in running keeprunningd: {:?}", panic_info);
|
||||
}));
|
||||
let keep_running_config = match parse_keep_running_config() {
|
||||
Some(c) => c, None => return,
|
||||
};
|
||||
let keep_running_config = opt_value!(parse_keep_running_config());
|
||||
|
||||
let mut the_file_lock = None;
|
||||
if let Some(ref lock_file) = keep_running_config.lock_file {
|
||||
the_file_lock = match FileLock::lock(lock_file, false, true) {
|
||||
let the_file_lock = if let Some(ref lock_file) = keep_running_config.lock_file {
|
||||
match FileLock::lock(lock_file, false, true) {
|
||||
Ok(lock) => {
|
||||
info!("Lock file success: {}", lock_file);
|
||||
Some(lock)
|
||||
@@ -48,27 +60,23 @@ fn main() {
|
||||
warn!("Lock file failed: {}", lock_file);
|
||||
return;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
if let Err(_) = simple_logging::log_to_file("/var/log/keeprunningd.log", LevelFilter::Info) {
|
||||
if let Err(_) = simple_logging::log_to_file("/tmp/keeprunningd.log", LevelFilter::Info) {
|
||||
simple_logging::log_to_stderr(LevelFilter::Info);
|
||||
}
|
||||
}
|
||||
} else { None };
|
||||
|
||||
init_logger();
|
||||
|
||||
let keep_running_config = Arc::new(keep_running_config);
|
||||
for check_cnt in 0.. {
|
||||
for check_cnt in 0_u64.. {
|
||||
info!("Check index: {} @{:?}", check_cnt, Local::now());
|
||||
keep_runningd(keep_running_config.clone());
|
||||
thread::sleep(Duration::from_secs(keep_running_config.check_inverval_secs.unwrap_or(60 * 60)));
|
||||
thread::sleep(Duration::from_secs(keep_running_config.get_check_inverval_secs()));
|
||||
}
|
||||
|
||||
drop(the_file_lock);
|
||||
}
|
||||
|
||||
fn parse_keep_running_config() -> Option<KeepRunningConfig> {
|
||||
let config_file_opt = locate_file(&[
|
||||
let config_file_opt = util_file::locate_file(&[
|
||||
"keeprunningd.json".into(),
|
||||
"~/keeprunningd.json".into(),
|
||||
"/etc/keeprunningd.json".into()
|
||||
@@ -99,7 +107,7 @@ fn parse_keep_running_config() -> Option<KeepRunningConfig> {
|
||||
},
|
||||
};
|
||||
|
||||
if keep_running_config.show_debug_output.unwrap_or(false) {
|
||||
if keep_running_config.is_show_debug_output() {
|
||||
if let Ok(json) = serde_json::to_string_pretty(&keep_running_config) {
|
||||
debug!("Config: {}", json);
|
||||
}
|
||||
@@ -109,17 +117,17 @@ fn parse_keep_running_config() -> Option<KeepRunningConfig> {
|
||||
|
||||
fn keep_runningd(keep_running_config: Arc<KeepRunningConfig>) {
|
||||
let t = thread::spawn(move || {
|
||||
let ps_aux_lines = read_str_to_lines(&match ps_aux() { Some(p) => p, None => return, });
|
||||
let ps_aux_lines = util_str::read_str_to_lines(&match ps_aux() { Some(p) => p, None => return, });
|
||||
for keep_running_config_item in &keep_running_config.items {
|
||||
let check_lines = ps_aux_lines.iter().filter(|ln| {
|
||||
keep_running_config_item.grep_tokens.iter().all(|t| ln.contains(t))
|
||||
}).collect::<Vec<_>>();
|
||||
|
||||
let mut fail_count = 0;
|
||||
let check_success = !check_lines.is_empty();
|
||||
{ // limit `CHECK_FAIL_MAP` lock scope
|
||||
let cloned_title = keep_running_config_item.title.clone();
|
||||
let mut check_fail_map = CHECK_FAIL_MAP.lock().unwrap();
|
||||
let check_success = !check_lines.is_empty();
|
||||
if check_success {
|
||||
check_fail_map.insert(cloned_title, 0);
|
||||
} else {
|
||||
@@ -130,36 +138,17 @@ fn keep_runningd(keep_running_config: Arc<KeepRunningConfig>) {
|
||||
}
|
||||
}
|
||||
|
||||
if check_lines.is_empty() { // if check fail!
|
||||
if !check_success { // if check fail!
|
||||
info!("Send DingTalk notification!");
|
||||
use tokio::runtime;
|
||||
match runtime::Builder::new().basic_scheduler().enable_all().build() {
|
||||
Err(err) => error!("Prepare tokio runtime error: {}", err),
|
||||
Ok(mut rt) => {
|
||||
let mut sb = String::with_capacity(1024);
|
||||
sb.push_str(&format!("Check failed: {}, fail count: {}", &keep_running_config_item.title, fail_count));
|
||||
sb.push_str(&format!("\ncheck time: {:?}", Local::now()));
|
||||
rt.block_on(send_notify(&keep_running_config.notify_token, &sb));
|
||||
},
|
||||
}
|
||||
block_send_notify(&keep_running_config.notify_token, &keep_running_config_item.title, fail_count);
|
||||
match &keep_running_config_item.restart_command {
|
||||
Some(restart_command) if fail_count > 1 => {
|
||||
info!("Fail count is {} > 1, try restart command: {:?}", fail_count, restart_command);
|
||||
let mut restart_command_iter = restart_command.iter();
|
||||
if let Some(program) = restart_command_iter.next() {
|
||||
let mut cmd = Command::new(program);
|
||||
while let Some(arg) = restart_command_iter.next() {
|
||||
cmd.arg(arg);
|
||||
}
|
||||
match cmd.spawn() {
|
||||
Ok(child) => info!("Ran process success, process id: {}", child.id()),
|
||||
Err(e) => error!("Run restart command failed: {}", e),
|
||||
}
|
||||
}
|
||||
restart_cmd(restart_command);
|
||||
},
|
||||
_ => ( /* IGNORE */ ),
|
||||
}
|
||||
} else if keep_running_config.show_debug_output.unwrap_or(false) {
|
||||
} else if keep_running_config.is_show_debug_output() {
|
||||
debug!("Find: {:?}", &check_lines);
|
||||
}
|
||||
}
|
||||
@@ -169,7 +158,22 @@ fn keep_runningd(keep_running_config: Arc<KeepRunningConfig>) {
|
||||
}
|
||||
}
|
||||
|
||||
fn block_send_notify(notify_token: &str, title: &str, fail_count: u64) {
|
||||
use tokio::runtime::Builder;
|
||||
match Builder::new_current_thread().enable_all().build() {
|
||||
Err(err) => error!("Prepare tokio runtime error: {}", err),
|
||||
Ok(rt) => {
|
||||
let text = format!("Check failed: {}, fail count: {}\ncheck time: {:?}", title, fail_count, Local::now());
|
||||
rt.block_on(send_notify(notify_token, &text));
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
async fn send_notify(notify_token: &str, text: &str) {
|
||||
if notify_token.is_empty() {
|
||||
info!("Notify token is empty, do not send DingTalk notification");
|
||||
return;
|
||||
}
|
||||
match DingTalk::from_token(¬ify_token) {
|
||||
Err(err) => error!("Prepare DingTalk error: {}", err),
|
||||
Ok(dt) => if let Err(err) = dt.send_text(text).await {
|
||||
@@ -178,6 +182,27 @@ async fn send_notify(notify_token: &str, text: &str) {
|
||||
}
|
||||
}
|
||||
|
||||
fn restart_cmd(restart_command: &[String]) {
|
||||
let mut restart_command_iter = restart_command.iter();
|
||||
if let Some(program) = restart_command_iter.next() {
|
||||
let mut cmd = Command::new(program);
|
||||
for arg in restart_command_iter {
|
||||
cmd.arg(arg);
|
||||
}
|
||||
match cmd.spawn() {
|
||||
Ok(child) => info!("Ran process success, process id: {}", child.id()),
|
||||
Err(e) => error!("Run restart command failed: {}", e),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn init_logger() {
|
||||
if simple_logging::log_to_file("/var/log/keeprunningd.log", LevelFilter::Info).is_err()
|
||||
&& simple_logging::log_to_file("/tmp/keeprunningd.log", LevelFilter::Info).is_err() {
|
||||
simple_logging::log_to_stderr(LevelFilter::Info);
|
||||
}
|
||||
}
|
||||
|
||||
fn ps_aux() -> Option<String> {
|
||||
let mut cmd = Command::new("ps");
|
||||
cmd.arg("aux");
|
||||
@@ -191,11 +216,8 @@ fn ps_aux() -> Option<String> {
|
||||
error!("Run 'ps aux' error: {:?}", output.status);
|
||||
return None;
|
||||
}
|
||||
match String::from_utf8(output.stdout) {
|
||||
Ok(output_str) => Some(output_str),
|
||||
Err(err) => {
|
||||
error!("Get ps output as utf8 error: {}", err);
|
||||
None
|
||||
},
|
||||
}
|
||||
String::from_utf8(output.stdout).map_or_else(|err| {
|
||||
error!("Get ps output as utf8 error: {}", err);
|
||||
None
|
||||
}, |output_str| Some(output_str))
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user