Compare commits
18 Commits
38ffef632e
...
master
| Author | SHA1 | Date | |
|---|---|---|---|
|
d34a15f351
|
|||
|
d290249c3c
|
|||
| abdf6f725c | |||
| 8d6f3c2cb5 | |||
| ad6fac1a9f | |||
| 9a4af87669 | |||
| 6bd539cdbd | |||
| f9b1a72018 | |||
| f722c95432 | |||
| bd82cff4f1 | |||
| fdbb1b9443 | |||
| acd45e75f6 | |||
| 40748eb3bb | |||
| ad28d12f4c | |||
| 25cb508334 | |||
| e9fdb60a80 | |||
| f2ad55691c | |||
| 01c6041999 |
1097
Cargo.lock
generated
1097
Cargo.lock
generated
File diff suppressed because it is too large
Load Diff
@@ -7,7 +7,7 @@ edition = "2018"
|
|||||||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
tokio = { version = "0.2", features = ["full"] }
|
tokio = { version="1.22", features = ["rt"] }
|
||||||
serde = { version = "1.0", features = ["derive"] }
|
serde = { version = "1.0", features = ["derive"] }
|
||||||
serde_json = "1.0"
|
serde_json = "1.0"
|
||||||
argparse = "0.2"
|
argparse = "0.2"
|
||||||
@@ -23,4 +23,6 @@ file-lock = "1.1"
|
|||||||
opt-level = 'z'
|
opt-level = 'z'
|
||||||
lto = true
|
lto = true
|
||||||
codegen-units = 1
|
codegen-units = 1
|
||||||
# panic = 'abort'
|
panic = 'abort'
|
||||||
|
strip = true
|
||||||
|
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
{
|
{
|
||||||
"check_inverval_secs": 1,
|
"check_inverval_secs": 1,
|
||||||
"show_debug_output": true,
|
"show_debug_output": true,
|
||||||
"notify_token": "------------",
|
"notify_token": "",
|
||||||
"lock_file": ".keeprunningd.lock",
|
"lock_file": ".keeprunningd.lock",
|
||||||
"items": [{
|
"items": [{
|
||||||
"grep_tokens": ["java", "app"],
|
"grep_tokens": ["java", "app"],
|
||||||
|
|||||||
120
src/main.rs
120
src/main.rs
@@ -1,12 +1,17 @@
|
|||||||
#[macro_use] extern crate lazy_static;
|
#[macro_use] extern crate lazy_static;
|
||||||
#[macro_use] extern crate log;
|
#[macro_use] extern crate log;
|
||||||
|
#[macro_use] extern crate rust_util;
|
||||||
|
|
||||||
use std::{ collections::HashMap, fs, panic, thread, time::Duration, process::Command, sync::{ Arc, Mutex } };
|
use std::collections::HashMap;
|
||||||
|
use std::{fs, panic, thread};
|
||||||
|
use std::time::Duration;
|
||||||
|
use std::process::Command;
|
||||||
|
use std::sync::{Arc, Mutex};
|
||||||
use log::LevelFilter;
|
use log::LevelFilter;
|
||||||
use chrono::prelude::*;
|
use chrono::prelude::*;
|
||||||
use serde::{ Deserialize, Serialize };
|
use serde::{Deserialize, Serialize};
|
||||||
use file_lock::FileLock;
|
use file_lock::FileLock;
|
||||||
use rust_util::{ util_str::read_str_to_lines, util_file::locate_file };
|
use rust_util::{util_str, util_file};
|
||||||
use dingtalk::DingTalk;
|
use dingtalk::DingTalk;
|
||||||
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
@@ -18,6 +23,16 @@ struct KeepRunningConfig {
|
|||||||
items: Vec<KeepRunningConfigItem>,
|
items: Vec<KeepRunningConfigItem>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
impl KeepRunningConfig {
|
||||||
|
fn get_check_inverval_secs(&self) -> u64 {
|
||||||
|
self.check_inverval_secs.unwrap_or(60 * 60)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn is_show_debug_output(&self) -> bool {
|
||||||
|
self.show_debug_output.unwrap_or(false)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
struct KeepRunningConfigItem {
|
struct KeepRunningConfigItem {
|
||||||
title: String,
|
title: String,
|
||||||
@@ -33,13 +48,10 @@ fn main() {
|
|||||||
panic::set_hook(Box::new(|panic_info| {
|
panic::set_hook(Box::new(|panic_info| {
|
||||||
error!("Panic in running keeprunningd: {:?}", panic_info);
|
error!("Panic in running keeprunningd: {:?}", panic_info);
|
||||||
}));
|
}));
|
||||||
let keep_running_config = match parse_keep_running_config() {
|
let keep_running_config = opt_value!(parse_keep_running_config());
|
||||||
Some(c) => c, None => return,
|
|
||||||
};
|
|
||||||
|
|
||||||
let mut the_file_lock = None;
|
let the_file_lock = if let Some(ref lock_file) = keep_running_config.lock_file {
|
||||||
if let Some(ref lock_file) = keep_running_config.lock_file {
|
match FileLock::lock(lock_file, false, true) {
|
||||||
the_file_lock = match FileLock::lock(lock_file, false, true) {
|
|
||||||
Ok(lock) => {
|
Ok(lock) => {
|
||||||
info!("Lock file success: {}", lock_file);
|
info!("Lock file success: {}", lock_file);
|
||||||
Some(lock)
|
Some(lock)
|
||||||
@@ -48,27 +60,23 @@ fn main() {
|
|||||||
warn!("Lock file failed: {}", lock_file);
|
warn!("Lock file failed: {}", lock_file);
|
||||||
return;
|
return;
|
||||||
},
|
},
|
||||||
};
|
|
||||||
}
|
}
|
||||||
|
} else { None };
|
||||||
|
|
||||||
if let Err(_) = simple_logging::log_to_file("/var/log/keeprunningd.log", LevelFilter::Info) {
|
init_logger();
|
||||||
if let Err(_) = simple_logging::log_to_file("/tmp/keeprunningd.log", LevelFilter::Info) {
|
|
||||||
simple_logging::log_to_stderr(LevelFilter::Info);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
let keep_running_config = Arc::new(keep_running_config);
|
let keep_running_config = Arc::new(keep_running_config);
|
||||||
for check_cnt in 0.. {
|
for check_cnt in 0_u64.. {
|
||||||
info!("Check index: {} @{:?}", check_cnt, Local::now());
|
info!("Check index: {} @{:?}", check_cnt, Local::now());
|
||||||
keep_runningd(keep_running_config.clone());
|
keep_runningd(keep_running_config.clone());
|
||||||
thread::sleep(Duration::from_secs(keep_running_config.check_inverval_secs.unwrap_or(60 * 60)));
|
thread::sleep(Duration::from_secs(keep_running_config.get_check_inverval_secs()));
|
||||||
}
|
}
|
||||||
|
|
||||||
drop(the_file_lock);
|
drop(the_file_lock);
|
||||||
}
|
}
|
||||||
|
|
||||||
fn parse_keep_running_config() -> Option<KeepRunningConfig> {
|
fn parse_keep_running_config() -> Option<KeepRunningConfig> {
|
||||||
let config_file_opt = locate_file(&[
|
let config_file_opt = util_file::locate_file(&[
|
||||||
"keeprunningd.json".into(),
|
"keeprunningd.json".into(),
|
||||||
"~/keeprunningd.json".into(),
|
"~/keeprunningd.json".into(),
|
||||||
"/etc/keeprunningd.json".into()
|
"/etc/keeprunningd.json".into()
|
||||||
@@ -99,7 +107,7 @@ fn parse_keep_running_config() -> Option<KeepRunningConfig> {
|
|||||||
},
|
},
|
||||||
};
|
};
|
||||||
|
|
||||||
if keep_running_config.show_debug_output.unwrap_or(false) {
|
if keep_running_config.is_show_debug_output() {
|
||||||
if let Ok(json) = serde_json::to_string_pretty(&keep_running_config) {
|
if let Ok(json) = serde_json::to_string_pretty(&keep_running_config) {
|
||||||
debug!("Config: {}", json);
|
debug!("Config: {}", json);
|
||||||
}
|
}
|
||||||
@@ -109,17 +117,17 @@ fn parse_keep_running_config() -> Option<KeepRunningConfig> {
|
|||||||
|
|
||||||
fn keep_runningd(keep_running_config: Arc<KeepRunningConfig>) {
|
fn keep_runningd(keep_running_config: Arc<KeepRunningConfig>) {
|
||||||
let t = thread::spawn(move || {
|
let t = thread::spawn(move || {
|
||||||
let ps_aux_lines = read_str_to_lines(&match ps_aux() { Some(p) => p, None => return, });
|
let ps_aux_lines = util_str::read_str_to_lines(&match ps_aux() { Some(p) => p, None => return, });
|
||||||
for keep_running_config_item in &keep_running_config.items {
|
for keep_running_config_item in &keep_running_config.items {
|
||||||
let check_lines = ps_aux_lines.iter().filter(|ln| {
|
let check_lines = ps_aux_lines.iter().filter(|ln| {
|
||||||
keep_running_config_item.grep_tokens.iter().all(|t| ln.contains(t))
|
keep_running_config_item.grep_tokens.iter().all(|t| ln.contains(t))
|
||||||
}).collect::<Vec<_>>();
|
}).collect::<Vec<_>>();
|
||||||
|
|
||||||
let mut fail_count = 0;
|
let mut fail_count = 0;
|
||||||
|
let check_success = !check_lines.is_empty();
|
||||||
{ // limit `CHECK_FAIL_MAP` lock scope
|
{ // limit `CHECK_FAIL_MAP` lock scope
|
||||||
let cloned_title = keep_running_config_item.title.clone();
|
let cloned_title = keep_running_config_item.title.clone();
|
||||||
let mut check_fail_map = CHECK_FAIL_MAP.lock().unwrap();
|
let mut check_fail_map = CHECK_FAIL_MAP.lock().unwrap();
|
||||||
let check_success = !check_lines.is_empty();
|
|
||||||
if check_success {
|
if check_success {
|
||||||
check_fail_map.insert(cloned_title, 0);
|
check_fail_map.insert(cloned_title, 0);
|
||||||
} else {
|
} else {
|
||||||
@@ -130,36 +138,17 @@ fn keep_runningd(keep_running_config: Arc<KeepRunningConfig>) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if check_lines.is_empty() { // if check fail!
|
if !check_success { // if check fail!
|
||||||
info!("Send DingTalk notification!");
|
info!("Send DingTalk notification!");
|
||||||
use tokio::runtime;
|
block_send_notify(&keep_running_config.notify_token, &keep_running_config_item.title, fail_count);
|
||||||
match runtime::Builder::new().basic_scheduler().enable_all().build() {
|
|
||||||
Err(err) => error!("Prepare tokio runtime error: {}", err),
|
|
||||||
Ok(mut rt) => {
|
|
||||||
let mut sb = String::with_capacity(1024);
|
|
||||||
sb.push_str(&format!("Check failed: {}, fail count: {}", &keep_running_config_item.title, fail_count));
|
|
||||||
sb.push_str(&format!("\ncheck time: {:?}", Local::now()));
|
|
||||||
rt.block_on(send_notify(&keep_running_config.notify_token, &sb));
|
|
||||||
},
|
|
||||||
}
|
|
||||||
match &keep_running_config_item.restart_command {
|
match &keep_running_config_item.restart_command {
|
||||||
Some(restart_command) if fail_count > 1 => {
|
Some(restart_command) if fail_count > 1 => {
|
||||||
info!("Fail count is {} > 1, try restart command: {:?}", fail_count, restart_command);
|
info!("Fail count is {} > 1, try restart command: {:?}", fail_count, restart_command);
|
||||||
let mut restart_command_iter = restart_command.iter();
|
restart_cmd(restart_command);
|
||||||
if let Some(program) = restart_command_iter.next() {
|
|
||||||
let mut cmd = Command::new(program);
|
|
||||||
while let Some(arg) = restart_command_iter.next() {
|
|
||||||
cmd.arg(arg);
|
|
||||||
}
|
|
||||||
match cmd.spawn() {
|
|
||||||
Ok(child) => info!("Ran process success, process id: {}", child.id()),
|
|
||||||
Err(e) => error!("Run restart command failed: {}", e),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
},
|
||||||
_ => ( /* IGNORE */ ),
|
_ => ( /* IGNORE */ ),
|
||||||
}
|
}
|
||||||
} else if keep_running_config.show_debug_output.unwrap_or(false) {
|
} else if keep_running_config.is_show_debug_output() {
|
||||||
debug!("Find: {:?}", &check_lines);
|
debug!("Find: {:?}", &check_lines);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -169,7 +158,22 @@ fn keep_runningd(keep_running_config: Arc<KeepRunningConfig>) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn block_send_notify(notify_token: &str, title: &str, fail_count: u64) {
|
||||||
|
use tokio::runtime::Builder;
|
||||||
|
match Builder::new_current_thread().enable_all().build() {
|
||||||
|
Err(err) => error!("Prepare tokio runtime error: {}", err),
|
||||||
|
Ok(rt) => {
|
||||||
|
let text = format!("Check failed: {}, fail count: {}\ncheck time: {:?}", title, fail_count, Local::now());
|
||||||
|
rt.block_on(send_notify(notify_token, &text));
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async fn send_notify(notify_token: &str, text: &str) {
|
async fn send_notify(notify_token: &str, text: &str) {
|
||||||
|
if notify_token.is_empty() {
|
||||||
|
info!("Notify token is empty, do not send DingTalk notification");
|
||||||
|
return;
|
||||||
|
}
|
||||||
match DingTalk::from_token(¬ify_token) {
|
match DingTalk::from_token(¬ify_token) {
|
||||||
Err(err) => error!("Prepare DingTalk error: {}", err),
|
Err(err) => error!("Prepare DingTalk error: {}", err),
|
||||||
Ok(dt) => if let Err(err) = dt.send_text(text).await {
|
Ok(dt) => if let Err(err) = dt.send_text(text).await {
|
||||||
@@ -178,6 +182,27 @@ async fn send_notify(notify_token: &str, text: &str) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn restart_cmd(restart_command: &[String]) {
|
||||||
|
let mut restart_command_iter = restart_command.iter();
|
||||||
|
if let Some(program) = restart_command_iter.next() {
|
||||||
|
let mut cmd = Command::new(program);
|
||||||
|
for arg in restart_command_iter {
|
||||||
|
cmd.arg(arg);
|
||||||
|
}
|
||||||
|
match cmd.spawn() {
|
||||||
|
Ok(child) => info!("Ran process success, process id: {}", child.id()),
|
||||||
|
Err(e) => error!("Run restart command failed: {}", e),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn init_logger() {
|
||||||
|
if simple_logging::log_to_file("/var/log/keeprunningd.log", LevelFilter::Info).is_err()
|
||||||
|
&& simple_logging::log_to_file("/tmp/keeprunningd.log", LevelFilter::Info).is_err() {
|
||||||
|
simple_logging::log_to_stderr(LevelFilter::Info);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn ps_aux() -> Option<String> {
|
fn ps_aux() -> Option<String> {
|
||||||
let mut cmd = Command::new("ps");
|
let mut cmd = Command::new("ps");
|
||||||
cmd.arg("aux");
|
cmd.arg("aux");
|
||||||
@@ -191,11 +216,8 @@ fn ps_aux() -> Option<String> {
|
|||||||
error!("Run 'ps aux' error: {:?}", output.status);
|
error!("Run 'ps aux' error: {:?}", output.status);
|
||||||
return None;
|
return None;
|
||||||
}
|
}
|
||||||
match String::from_utf8(output.stdout) {
|
String::from_utf8(output.stdout).map_or_else(|err| {
|
||||||
Ok(output_str) => Some(output_str),
|
|
||||||
Err(err) => {
|
|
||||||
error!("Get ps output as utf8 error: {}", err);
|
error!("Get ps output as utf8 error: {}", err);
|
||||||
None
|
None
|
||||||
},
|
}, |output_str| Some(output_str))
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user