ComputerSurge/komodo
3
0
Fork 0
forked from github-starred/komodo
Code Pull Requests 1 Actions Packages Releases Activity

prog on monitoring

Browse Source
This commit is contained in:
mbecker20
2023-08-20 04:29:55 -04:00
parent f0deda32dc
commit 080ecf9fd9
6 changed files with 76 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.lock
Generated
+2 -2
View File
@@ -2044,9 +2044,9 @@ dependencies = [
[[package]]
name = "mungos"
version = "0.4.18"
version = "0.4.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e5bab215e70538484f1a21ef673a8c5b49d39f4ba91886852fe60173a5fd2951"
checksum = "ed3fd16ef4c4dc182aa3e058259747a4660736e87dd42d2edbd371a1a8a7b385"
dependencies = [
"anyhow",
"async-trait",
Cargo.toml
+1 -1
View File
@@ -26,7 +26,7 @@ derive_variants = "0.1.1"
make_option = "0.1.7"
resolver_api = "0.1.6"
parse_csl = "0.1.0"
mungos = "0.4.18"
mungos = "0.4.19"
svi = "0.1.4"
# external
tokio = { version = "1.32.0", features = ["full"] }
bin/core/src/helpers/alert.rs
+51 -31
View File
@@ -4,7 +4,7 @@ use monitor_types::entities::{
alert::{Alert, AlertData},
alerter::*,
deployment::DockerContainerState,
server::stats::SystemProcess,
server::stats::{SeverityLevel, SystemProcess},
};
use reqwest::StatusCode;
use slack::types::Block;
@@ -48,25 +48,29 @@ pub async fn send_alert(alerters: &[Alerter], alert: &Alert) {
}
pub async fn send_slack_alert(url: &str, alert: &Alert) -> anyhow::Result<()> {
let level = alert.level;
let level = fmt_level(alert.level);
let (text, blocks): (_, Option<_>) = match &alert.data {
AlertData::ServerUnreachable { name, region, .. } => {
let region = fmt_region(region);
let text = format!("CRITICAL 🚨 | *{name}*{region} is *unreachable* ❌");
let blocks = vec![
Block::header("CRITICAL 🚨"),
Block::section(format!("*{name}*{region} is *unreachable* ❌")),
];
(text, blocks.into())
}
AlertData::ServerReachable { name, region, .. } => {
let region = fmt_region(region);
let text = format!("OK ✅ | *{name}*{region} is now *reachable*");
let blocks = vec![
Block::header("OK ✅"),
Block::section(format!("*{name}*{region} is now *reachable*")),
];
(text, blocks.into())
match alert.level {
SeverityLevel::Ok => {
let text = format!("{level} | *{name}*{region} is now *reachable*");
let blocks = vec![
Block::header(level),
Block::section(format!("*{name}*{region} is now *reachable*")),
];
(text, blocks.into())
}
SeverityLevel::Critical => {
let text = format!("{level} | *{name}*{region} is *unreachable* ❌");
let blocks = vec![
Block::header(level),
Block::section(format!("*{name}*{region} is *unreachable* ❌")),
];
(text, blocks.into())
}
_ => unreachable!(),
}
}
AlertData::ServerCpu {
name,
@@ -76,15 +80,19 @@ pub async fn send_slack_alert(url: &str, alert: &Alert) -> anyhow::Result<()> {
..
} => {
let region = fmt_region(region);
let text =
format!("{level} 🚨 | *{name}*{region} cpu usage at *{percentage:.1}%* 📈 🚨");
let blocks = vec![
let text = format!("{level} | *{name}*{region} cpu usage at *{percentage:.1}%* 📈 🚨");
let mut blocks = vec![
Block::header(format!("{level} 🚨")),
Block::section(format!(
"*{name}*{region} cpu usage at *{percentage:.1}%* 📈 🚨"
)),
Block::section(format!("*top cpu processes*{}", fmt_top_procs(top_procs))),
];
if alert.level != SeverityLevel::Ok {
blocks.push(Block::section(format!(
"*top cpu processes*{}",
fmt_top_procs(top_procs)
)));
}
(text, blocks.into())
}
AlertData::ServerMem {
@@ -98,15 +106,20 @@ pub async fn send_slack_alert(url: &str, alert: &Alert) -> anyhow::Result<()> {
let region = fmt_region(region);
let percentage = 100.0 * used_gb / total_gb;
let text =
format!("{level} 🚨 | *{name}*{region} memory usage at *{percentage:.1}%* 💾 🚨");
let blocks = vec![
Block::header(format!("{level} 🚨")),
format!("{level} | *{name}*{region} memory usage at *{percentage:.1}%* 💾 🚨");
let mut blocks = vec![
Block::header(level),
Block::section(format!(
"*{name}*{region} memory usage at *{percentage:.1}%* 💾 🚨"
)),
Block::section(format!("using *{used_gb:.1} GiB* / *{total_gb:.1} GiB*")),
Block::section(format!("*top cpu processes*{}", fmt_top_procs(top_procs))),
];
if alert.level != SeverityLevel::Ok {
blocks.push(Block::section(format!(
"*top mem processes*{}",
fmt_top_procs(top_procs)
)));
}
(text, blocks.into())
}
AlertData::ServerDisk {
@@ -120,9 +133,9 @@ pub async fn send_slack_alert(url: &str, alert: &Alert) -> anyhow::Result<()> {
let region = fmt_region(region);
let percentage = 100.0 * used_gb / total_gb;
let text =
format!("{level} 🚨 | *{name}*{region} disk usage at *{percentage:.1}%* | mount point: *{path}* 💿 🚨");
format!("{level} | *{name}*{region} disk usage at *{percentage:.1}%* | mount point: *{path}* 💿 🚨");
let blocks = vec![
Block::header(format!("{level} 🚨")),
Block::header(level),
Block::section(format!(
"*{name}*{region} disk usage at *{percentage:.1}%* 💿 🚨"
)),
@@ -140,11 +153,10 @@ pub async fn send_slack_alert(url: &str, alert: &Alert) -> anyhow::Result<()> {
..
} => {
let region = fmt_region(region);
let text = format!(
"{level} 🚨 | *{name}*{region} temp at {temp:.0} °C (max: {max:.0} °C) 🌡️ 🚨"
);
let text =
format!("{level} | *{name}*{region} temp at {temp:.0} °C (max: {max:.0} °C) 🌡️ 🚨");
let blocks = vec![
Block::header(format!("{level} 🚨")),
Block::header(level),
Block::section(format!(
"*{name}*{region} temp at {temp:.0} °C (max: {max:.0} °C) 🌡️ 🚨"
)),
@@ -226,3 +238,11 @@ fn fmt_docker_container_state(state: &DockerContainerState) -> String {
_ => state.to_string(),
}
}
fn fmt_level(level: SeverityLevel) -> &'static str {
match level {
SeverityLevel::Critical => "CRITICAL 🚨",
SeverityLevel::Warning => "WARNING 🚨",
SeverityLevel::Ok => "OK ✅",
}
}
bin/core/src/monitor/alert/server.rs
+19 -15
View File
@@ -77,20 +77,18 @@ impl State {
}
tokio::join!(
self.open_alerts(alerts_to_open),
self.open_alerts(&alerts_to_open),
self.resolve_alerts(&alert_ids_to_close)
);
}
async fn open_alerts(&self, alerts: Vec<Alert>) {
async fn open_alerts(&self, alerts: &[Alert]) {
let open = || async {
// self.db.alerters.create_many(alerts).await?;
self.db.alerts.create_many(alerts).await?;
anyhow::Ok(())
};
let alert = || async { todo!() };
let (res, _) = tokio::join!(open(), alert());
let (res, _) = tokio::join!(open(), self.send_alerts(alerts));
if let Err(e) = res {
error!("failed to create alerts on db | {e:#?}");
@@ -106,7 +104,7 @@ impl State {
self.db
.alerts
.update_many(
doc! { "_id": { "$in": alert_ids } },
doc! { "_id": { "$in": &alert_ids } },
doc! {
"$set": {
"resolved": "true",
@@ -116,17 +114,23 @@ impl State {
)
.await
.context("failed to resolve alerts on db")?;
let mut closed = self
.db
.alerts
.get_some(doc! { "_id": { "$in": &alert_ids } }, None)
.await
.context("failed to get closed alerts from db")?;
for closed in &mut closed {
closed.level = SeverityLevel::Ok;
}
self.send_alerts(&closed).await;
anyhow::Ok(())
};
let alert = || async {
// alert that alert has be resolved
todo!()
};
let (res, _) = tokio::join!(close(), alert());
if let Err(e) = res {
if let Err(e) = close().await {
error!("failed to resolve alerts | {e:#?}");
}
}
lib/types/src/entities/alert.rs
-5
View File
@@ -63,11 +63,6 @@ pub enum AlertData {
name: String,
region: Option<String>,
},
ServerReachable {
id: String,
name: String,
region: Option<String>,
},
ServerCpu {
id: String,
name: String,
lib/types/src/entities/server/stats.rs
+3 -1
View File
@@ -143,7 +143,9 @@ pub struct SystemComponent {
}
#[typeshare]
#[derive(Serialize, Deserialize, Debug, Clone, Copy, Default, Display, EnumString)]
#[derive(
Serialize, Deserialize, Debug, Clone, Copy, Default, Display, EnumString, PartialEq, Eq,
)]
#[serde(rename_all = "UPPERCASE")]
#[strum(serialize_all = "UPPERCASE")]
pub enum SeverityLevel {