mirror of
https://github.com/moghtech/komodo.git
synced 2026-03-11 17:44:19 -05:00
* start on cron schedules * rust 1.86.0 * config periphery directories easier with PERIPHERY_ROOT_DIRECTORY * schedule backend * fix config switch toggling through disabled * procedure schedule working * implement schedules for actions * update schedule immediately after last run * improve config update logs using toml diffs backend * improve the config update logs with TOML diff view * add schedule alerting * version 1.17.2 * Set TZ in core env * dev-1 * better term signal labels * sync configurable pending alert send * fix monaco editor height on larger screen * poll update until complete on client update lib * add logger.pretty option for both core and periphery * fix pretty * configure schedule alert * configure failure alert * dev-3 * 1.17.2 * fmt * added pushover alerter (#421) * fix up pushover * fix some clippy --------- Co-authored-by: Alex Shore <alex@shore.me.uk>
278 lines
7.8 KiB
Rust
278 lines
7.8 KiB
Rust
use std::sync::OnceLock;
|
|
|
|
use serde::Serialize;
|
|
|
|
use super::*;
|
|
|
|
#[instrument(level = "debug")]
|
|
pub async fn send_alert(
|
|
url: &str,
|
|
alert: &Alert,
|
|
) -> anyhow::Result<()> {
|
|
let level = fmt_level(alert.level);
|
|
let content = match &alert.data {
|
|
AlertData::Test { id, name } => {
|
|
let link = resource_link(ResourceTargetVariant::Alerter, id);
|
|
format!(
|
|
"{level} | If you see this message, then Alerter **{name}** is **working**\n{link}"
|
|
)
|
|
}
|
|
AlertData::ServerUnreachable {
|
|
id,
|
|
name,
|
|
region,
|
|
err,
|
|
} => {
|
|
let region = fmt_region(region);
|
|
let link = resource_link(ResourceTargetVariant::Server, id);
|
|
match alert.level {
|
|
SeverityLevel::Ok => {
|
|
format!(
|
|
"{level} | **{name}**{region} is now **reachable**\n{link}"
|
|
)
|
|
}
|
|
SeverityLevel::Critical => {
|
|
let err = err
|
|
.as_ref()
|
|
.map(|e| format!("\n**error**: {e:#?}"))
|
|
.unwrap_or_default();
|
|
format!(
|
|
"{level} | **{name}**{region} is **unreachable** ❌\n{link}{err}"
|
|
)
|
|
}
|
|
_ => unreachable!(),
|
|
}
|
|
}
|
|
AlertData::ServerCpu {
|
|
id,
|
|
name,
|
|
region,
|
|
percentage,
|
|
} => {
|
|
let region = fmt_region(region);
|
|
let link = resource_link(ResourceTargetVariant::Server, id);
|
|
format!(
|
|
"{level} | **{name}**{region} cpu usage at **{percentage:.1}%**\n{link}"
|
|
)
|
|
}
|
|
AlertData::ServerMem {
|
|
id,
|
|
name,
|
|
region,
|
|
used_gb,
|
|
total_gb,
|
|
} => {
|
|
let region = fmt_region(region);
|
|
let link = resource_link(ResourceTargetVariant::Server, id);
|
|
let percentage = 100.0 * used_gb / total_gb;
|
|
format!(
|
|
"{level} | **{name}**{region} memory usage at **{percentage:.1}%** 💾\n\nUsing **{used_gb:.1} GiB** / **{total_gb:.1} GiB**\n{link}"
|
|
)
|
|
}
|
|
AlertData::ServerDisk {
|
|
id,
|
|
name,
|
|
region,
|
|
path,
|
|
used_gb,
|
|
total_gb,
|
|
} => {
|
|
let region = fmt_region(region);
|
|
let link = resource_link(ResourceTargetVariant::Server, id);
|
|
let percentage = 100.0 * used_gb / total_gb;
|
|
format!(
|
|
"{level} | **{name}**{region} disk usage at **{percentage:.1}%** 💿\nmount point: `{path:?}`\nusing **{used_gb:.1} GiB** / **{total_gb:.1} GiB**\n{link}"
|
|
)
|
|
}
|
|
AlertData::ContainerStateChange {
|
|
id,
|
|
name,
|
|
server_id: _server_id,
|
|
server_name,
|
|
from,
|
|
to,
|
|
} => {
|
|
let link = resource_link(ResourceTargetVariant::Deployment, id);
|
|
let to = fmt_docker_container_state(to);
|
|
format!(
|
|
"📦 Deployment **{name}** is now **{to}**\nserver: **{server_name}**\nprevious: **{from}**\n{link}"
|
|
)
|
|
}
|
|
AlertData::DeploymentImageUpdateAvailable {
|
|
id,
|
|
name,
|
|
server_id: _server_id,
|
|
server_name,
|
|
image,
|
|
} => {
|
|
let link = resource_link(ResourceTargetVariant::Deployment, id);
|
|
format!(
|
|
"⬆ Deployment **{name}** has an update available\nserver: **{server_name}**\nimage: **{image}**\n{link}"
|
|
)
|
|
}
|
|
AlertData::DeploymentAutoUpdated {
|
|
id,
|
|
name,
|
|
server_id: _server_id,
|
|
server_name,
|
|
image,
|
|
} => {
|
|
let link = resource_link(ResourceTargetVariant::Deployment, id);
|
|
format!(
|
|
"⬆ Deployment **{name}** was updated automatically ⏫\nserver: **{server_name}**\nimage: **{image}**\n{link}"
|
|
)
|
|
}
|
|
AlertData::StackStateChange {
|
|
id,
|
|
name,
|
|
server_id: _server_id,
|
|
server_name,
|
|
from,
|
|
to,
|
|
} => {
|
|
let link = resource_link(ResourceTargetVariant::Stack, id);
|
|
let to = fmt_stack_state(to);
|
|
format!(
|
|
"🥞 Stack **{name}** is now {to}\nserver: **{server_name}**\nprevious: **{from}**\n{link}"
|
|
)
|
|
}
|
|
AlertData::StackImageUpdateAvailable {
|
|
id,
|
|
name,
|
|
server_id: _server_id,
|
|
server_name,
|
|
service,
|
|
image,
|
|
} => {
|
|
let link = resource_link(ResourceTargetVariant::Stack, id);
|
|
format!(
|
|
"⬆ Stack **{name}** has an update available\nserver: **{server_name}**\nservice: **{service}**\nimage: **{image}**\n{link}"
|
|
)
|
|
}
|
|
AlertData::StackAutoUpdated {
|
|
id,
|
|
name,
|
|
server_id: _server_id,
|
|
server_name,
|
|
images,
|
|
} => {
|
|
let link = resource_link(ResourceTargetVariant::Stack, id);
|
|
let images_label =
|
|
if images.len() > 1 { "images" } else { "image" };
|
|
let images = images.join(", ");
|
|
format!(
|
|
"⬆ Stack **{name}** was updated automatically ⏫\nserver: **{server_name}**\n{images_label}: **{images}**\n{link}"
|
|
)
|
|
}
|
|
AlertData::AwsBuilderTerminationFailed {
|
|
instance_id,
|
|
message,
|
|
} => {
|
|
format!(
|
|
"{level} | Failed to terminated AWS builder instance\ninstance id: **{instance_id}**\n{message}"
|
|
)
|
|
}
|
|
AlertData::ResourceSyncPendingUpdates { id, name } => {
|
|
let link =
|
|
resource_link(ResourceTargetVariant::ResourceSync, id);
|
|
format!(
|
|
"{level} | Pending resource sync updates on **{name}**\n{link}"
|
|
)
|
|
}
|
|
AlertData::BuildFailed { id, name, version } => {
|
|
let link = resource_link(ResourceTargetVariant::Build, id);
|
|
format!(
|
|
"{level} | Build **{name}** failed\nversion: **v{version}**\n{link}"
|
|
)
|
|
}
|
|
AlertData::RepoBuildFailed { id, name } => {
|
|
let link = resource_link(ResourceTargetVariant::Repo, id);
|
|
format!("{level} | Repo build for **{name}** failed\n{link}")
|
|
}
|
|
AlertData::ProcedureFailed { id, name } => {
|
|
let link = resource_link(ResourceTargetVariant::Procedure, id);
|
|
format!("{level} | Procedure **{name}** failed\n{link}")
|
|
}
|
|
AlertData::ActionFailed { id, name } => {
|
|
let link = resource_link(ResourceTargetVariant::Action, id);
|
|
format!("{level} | Action **{name}** failed\n{link}")
|
|
}
|
|
AlertData::ScheduleRun {
|
|
resource_type,
|
|
id,
|
|
name,
|
|
} => {
|
|
let link = resource_link(*resource_type, id);
|
|
format!(
|
|
"{level} | **{name}** ({resource_type}) | Scheduled run started 🕝\n{link}"
|
|
)
|
|
}
|
|
AlertData::None {} => Default::default(),
|
|
};
|
|
if !content.is_empty() {
|
|
let vars_and_secrets = get_variables_and_secrets().await?;
|
|
let mut global_replacers = HashSet::new();
|
|
let mut secret_replacers = HashSet::new();
|
|
let mut url_interpolated = url.to_string();
|
|
|
|
// interpolate variables and secrets into the url
|
|
interpolate_variables_secrets_into_string(
|
|
&vars_and_secrets,
|
|
&mut url_interpolated,
|
|
&mut global_replacers,
|
|
&mut secret_replacers,
|
|
)?;
|
|
|
|
send_message(&url_interpolated, &content)
|
|
.await
|
|
.map_err(|e| {
|
|
let replacers =
|
|
secret_replacers.into_iter().collect::<Vec<_>>();
|
|
let sanitized_error =
|
|
svi::replace_in_string(&format!("{e:?}"), &replacers);
|
|
anyhow::Error::msg(format!(
|
|
"Error with slack request: {}",
|
|
sanitized_error
|
|
))
|
|
})?;
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
async fn send_message(
|
|
url: &str,
|
|
content: &str,
|
|
) -> anyhow::Result<()> {
|
|
let body = DiscordMessageBody { content };
|
|
|
|
let response = http_client()
|
|
.post(url)
|
|
.json(&body)
|
|
.send()
|
|
.await
|
|
.context("Failed to send message")?;
|
|
|
|
let status = response.status();
|
|
|
|
if status.is_success() {
|
|
Ok(())
|
|
} else {
|
|
let text = response.text().await.with_context(|| {
|
|
format!("Failed to send message to Discord | {status} | failed to get response text")
|
|
})?;
|
|
Err(anyhow::anyhow!(
|
|
"Failed to send message to Discord | {status} | {text}"
|
|
))
|
|
}
|
|
}
|
|
|
|
fn http_client() -> &'static reqwest::Client {
|
|
static CLIENT: OnceLock<reqwest::Client> = OnceLock::new();
|
|
CLIENT.get_or_init(reqwest::Client::new)
|
|
}
|
|
|
|
#[derive(Serialize)]
|
|
struct DiscordMessageBody<'a> {
|
|
content: &'a str,
|
|
}
|