test alert implementation

This commit is contained in:
mbecker20
2025-01-14 21:13:24 -08:00
parent bfbcd33c1a
commit 4e4aa8c567
13 changed files with 248 additions and 97 deletions

94
Cargo.lock generated
View File

@@ -228,9 +228,9 @@ dependencies = [
[[package]]
name = "aws-lc-rs"
version = "1.11.1"
version = "1.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f47bb8cc16b669d267eeccf585aea077d0882f4777b1c1f740217885d6e6e5a3"
checksum = "f409eb70b561706bf8abba8ca9c112729c481595893fd06a2dd9af8ed8441148"
dependencies = [
"aws-lc-sys",
"paste",
@@ -239,16 +239,15 @@ dependencies = [
[[package]]
name = "aws-lc-sys"
version = "0.23.1"
version = "0.24.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a2101df3813227bbaaaa0b04cd61c534c7954b22bd68d399b440be937dc63ff7"
checksum = "923ded50f602b3007e5e63e3f094c479d9c8a9b42d7f4034e4afe456aa48bfd2"
dependencies = [
"bindgen",
"cc",
"cmake",
"dunce",
"fs_extra",
"libc",
"paste",
]
@@ -279,9 +278,9 @@ dependencies = [
[[package]]
name = "aws-sdk-ec2"
version = "1.100.0"
version = "1.101.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b326e5f4c3400ae818798134dd3fc0405bb8a5df3d0995f3721980d70ab93f14"
checksum = "de1beb7d38bb03a655b474b604b41152615f761a7e733083ba1f33438e9b3321"
dependencies = [
"aws-credential-types",
"aws-runtime",
@@ -687,7 +686,7 @@ dependencies = [
"hyper 1.5.1",
"hyper-util",
"pin-project-lite",
"rustls 0.23.20",
"rustls 0.23.21",
"rustls-pemfile 2.2.0",
"rustls-pki-types",
"tokio",
@@ -855,7 +854,7 @@ dependencies = [
"serde_json",
"serde_repr",
"serde_urlencoded",
"thiserror 2.0.9",
"thiserror 2.0.11",
"tokio",
"tokio-util",
"tower-service",
@@ -1004,9 +1003,9 @@ dependencies = [
[[package]]
name = "clap"
version = "4.5.23"
version = "4.5.26"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3135e7ec2ef7b10c6ed8950f0f792ed96ee093fa088608f1c76e569722700c84"
checksum = "a8eb5e908ef3a6efbe1ed62520fb7287959888c88485abe072543190ecc66783"
dependencies = [
"clap_builder",
"clap_derive",
@@ -1014,9 +1013,9 @@ dependencies = [
[[package]]
name = "clap_builder"
version = "4.5.23"
version = "4.5.26"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "30582fc632330df2bd26877bde0c1f4470d57c582bbc070376afcd04d8cb4838"
checksum = "96b01801b5fc6a0a232407abc821660c9c6d25a1cafc0d4f85f29fb8d9afc121"
dependencies = [
"anstream",
"anstyle",
@@ -1026,9 +1025,9 @@ dependencies = [
[[package]]
name = "clap_derive"
version = "4.5.18"
version = "4.5.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4ac6a0c7b1a9e9a5186361f67dfa1b88213572f427fb9ab038efb2bd8c582dab"
checksum = "54b755194d6389280185988721fffba69495eed5ee9feeee9a599b53db80318c"
dependencies = [
"heck",
"proc-macro2",
@@ -1059,11 +1058,10 @@ checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990"
[[package]]
name = "colored"
version = "2.2.0"
version = "3.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "117725a109d387c937a1533ce01b450cbde6b88abceea8473c4d7a85853cda3c"
checksum = "fde0e0ec90c9dfb3b4b1a0891a7dcd0e2bffde2f7efed5fe7c9bb00e5bfb915e"
dependencies = [
"lazy_static",
"windows-sys 0.59.0",
]
@@ -1510,7 +1508,7 @@ dependencies = [
name = "environment_file"
version = "1.17.0"
dependencies = [
"thiserror 2.0.9",
"thiserror 2.0.11",
]
[[package]]
@@ -2095,7 +2093,7 @@ dependencies = [
"http 1.1.0",
"hyper 1.5.1",
"hyper-util",
"rustls 0.23.20",
"rustls 0.23.21",
"rustls-native-certs 0.8.1",
"rustls-pki-types",
"tokio",
@@ -2514,7 +2512,7 @@ dependencies = [
"serde_json",
"serror",
"strum",
"thiserror 2.0.9",
"thiserror 2.0.11",
"tokio",
"tokio-tungstenite",
"tokio-util",
@@ -2565,7 +2563,7 @@ dependencies = [
"reqwest 0.12.12",
"resolver_api",
"response",
"rustls 0.23.20",
"rustls 0.23.21",
"serde",
"serde_json",
"serde_yaml",
@@ -2611,7 +2609,7 @@ dependencies = [
"resolver_api",
"response",
"run_command",
"rustls 0.23.20",
"rustls 0.23.21",
"serde",
"serde_json",
"serror",
@@ -2639,9 +2637,9 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"
[[package]]
name = "libc"
version = "0.2.166"
version = "0.2.169"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c2ccc108bbc0b1331bd061864e7cd823c0cab660bbe6970e66e2c0614decde36"
checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a"
[[package]]
name = "libloading"
@@ -3532,9 +3530,9 @@ dependencies = [
"quinn-proto",
"quinn-udp",
"rustc-hash 2.0.0",
"rustls 0.23.20",
"rustls 0.23.21",
"socket2",
"thiserror 2.0.9",
"thiserror 2.0.11",
"tokio",
"tracing",
]
@@ -3550,10 +3548,10 @@ dependencies = [
"rand",
"ring 0.17.8",
"rustc-hash 2.0.0",
"rustls 0.23.20",
"rustls 0.23.21",
"rustls-pki-types",
"slab",
"thiserror 2.0.9",
"thiserror 2.0.11",
"tinyvec",
"tracing",
"web-time",
@@ -3760,7 +3758,7 @@ dependencies = [
"percent-encoding",
"pin-project-lite",
"quinn",
"rustls 0.23.20",
"rustls 0.23.21",
"rustls-native-certs 0.8.1",
"rustls-pemfile 2.2.0",
"rustls-pki-types",
@@ -4029,9 +4027,9 @@ dependencies = [
[[package]]
name = "rustls"
version = "0.23.20"
version = "0.23.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5065c3f250cbd332cd894be57c40fa52387247659b14a2d6041d121547903b1b"
checksum = "8f287924602bf649d949c63dc8ac8b235fa5387d394020705b80c4eb597ce5b8"
dependencies = [
"aws-lc-rs",
"log",
@@ -4289,9 +4287,9 @@ dependencies = [
[[package]]
name = "serde_json"
version = "1.0.134"
version = "1.0.135"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d00f4175c42ee48b15416f6193a959ba3a0d67fc699a0db9ad12df9f83991c7d"
checksum = "2b0d7ba2887406110130a978386c4e1befb98c674b4fba677954e4db976630d9"
dependencies = [
"indexmap 2.6.0",
"itoa",
@@ -4734,11 +4732,11 @@ dependencies = [
[[package]]
name = "thiserror"
version = "2.0.9"
version = "2.0.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f072643fd0190df67a8bab670c20ef5d8737177d6ac6b2e9a236cb096206b2cc"
checksum = "d452f284b73e6d76dd36758a0c8684b1d5be31f92b89d07fd5822175732206fc"
dependencies = [
"thiserror-impl 2.0.9",
"thiserror-impl 2.0.11",
]
[[package]]
@@ -4754,9 +4752,9 @@ dependencies = [
[[package]]
name = "thiserror-impl"
version = "2.0.9"
version = "2.0.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7b50fa271071aae2e6ee85f842e2e28ba8cd2c5fb67f11fcb1fd70b276f9e7d4"
checksum = "26afc1baea8a989337eeb52b6e72a039780ce45c3edfcc9c5b9d112feeb173c2"
dependencies = [
"proc-macro2",
"quote",
@@ -4831,9 +4829,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
[[package]]
name = "tokio"
version = "1.42.0"
version = "1.43.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5cec9b21b0450273377fc97bd4c33a8acffc8c996c987a7c5b319a0083707551"
checksum = "3d61fa4ffa3de412bfea335c6ecff681de2b609ba3c77ef3e00e521813a9ed9e"
dependencies = [
"backtrace",
"bytes",
@@ -4849,9 +4847,9 @@ dependencies = [
[[package]]
name = "tokio-macros"
version = "2.4.0"
version = "2.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752"
checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8"
dependencies = [
"proc-macro2",
"quote",
@@ -4874,7 +4872,7 @@ version = "0.26.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4"
dependencies = [
"rustls 0.23.20",
"rustls 0.23.21",
"rustls-pki-types",
"tokio",
]
@@ -5177,7 +5175,7 @@ dependencies = [
"log",
"rand",
"sha1",
"thiserror 2.0.9",
"thiserror 2.0.11",
"utf-8",
]
@@ -5315,9 +5313,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
[[package]]
name = "uuid"
version = "1.11.0"
version = "1.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f8c5f0a0af699448548ad1a2fbf920fb4bee257eae39953ba95cb84891a0446a"
checksum = "744018581f9a3454a9e15beb8a33b017183f1e7c0cd170232a2d1453b23a51c4"
dependencies = [
"getrandom",
"rand",
@@ -5504,7 +5502,7 @@ version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f9b0540e91e49de3817c314da0dd3bc518093ceacc6ea5327cb0e1eb073e5189"
dependencies = [
"thiserror 2.0.9",
"thiserror 2.0.11",
]
[[package]]

View File

@@ -45,7 +45,7 @@ svi = "1.0.1"
# ASYNC
reqwest = { version = "0.12.12", default-features = false, features = ["json", "rustls-tls"] }
tokio = { version = "1.42.0", features = ["full"] }
tokio = { version = "1.43.0", features = ["full"] }
tokio-util = "0.7.13"
futures = "0.3.31"
futures-util = "0.3.31"
@@ -61,13 +61,13 @@ tokio-tungstenite = "0.26.1"
ordered_hash_map = { version = "0.4.0", features = ["serde"] }
serde = { version = "1.0.217", features = ["derive"] }
strum = { version = "0.26.3", features = ["derive"] }
serde_json = "1.0.134"
serde_json = "1.0.135"
serde_yaml = "0.9.34"
toml = "0.8.19"
# ERROR
anyhow = "1.0.95"
thiserror = "2.0.9"
thiserror = "2.0.11"
# LOGGING
opentelemetry-otlp = { version = "0.27.0", features = ["tls-roots", "reqwest-rustls"] }
@@ -79,18 +79,18 @@ opentelemetry = "0.27.1"
tracing = "0.1.41"
# CONFIG
clap = { version = "4.5.23", features = ["derive"] }
clap = { version = "4.5.26", features = ["derive"] }
dotenvy = "0.15.7"
envy = "0.4.2"
# CRYPTO / AUTH
uuid = { version = "1.11.0", features = ["v4", "fast-rng", "serde"] }
uuid = { version = "1.12.0", features = ["v4", "fast-rng", "serde"] }
openidconnect = "3.5.0"
urlencoding = "2.1.3"
nom_pem = "4.0.0"
bcrypt = "0.16.0"
base64 = "0.22.1"
rustls = "0.23.20"
rustls = "0.23.21"
hmac = "0.12.1"
sha2 = "0.10.8"
rand = "0.8.5"
@@ -103,7 +103,7 @@ sysinfo = "0.33.1"
# CLOUD
aws-config = "1.5.13"
aws-sdk-ec2 = "1.100.0"
aws-sdk-ec2 = "1.101.0"
# MISC
derive_builder = "0.20.2"
@@ -111,6 +111,6 @@ typeshare = "1.0.4"
octorust = "0.9.0"
dashmap = "6.1.0"
wildcard = "0.3.0"
colored = "2.2.0"
colored = "3.0.0"
regex = "1.11.1"
bson = "2.13.0"

View File

@@ -206,6 +206,9 @@ pub async fn run(execution: Execution) -> anyhow::Result<()> {
Execution::BatchDestroyStack(data) => {
println!("{}: {data:?}", "Data".dimmed())
}
Execution::TestAlerter(data) => {
println!("{}: {data:?}", "Data".dimmed())
}
Execution::Sleep(data) => {
println!("{}: {data:?}", "Data".dimmed())
}
@@ -454,6 +457,10 @@ pub async fn run(execution: Execution) -> anyhow::Result<()> {
.execute(request)
.await
.map(ExecutionResult::Batch),
Execution::TestAlerter(request) => komodo_client()
.execute(request)
.await
.map(ExecutionResult::Single),
Execution::Sleep(request) => {
let duration =
Duration::from_millis(request.duration_ms as u64);

View File

@@ -3,7 +3,7 @@ use anyhow::{anyhow, Context};
use derive_variants::ExtractVariant;
use futures::future::join_all;
use komodo_client::entities::{
alert::{Alert, AlertData, SeverityLevel},
alert::{Alert, AlertData, AlertDataVariant, SeverityLevel},
alerter::*,
deployment::DeploymentState,
stack::StackState,
@@ -17,6 +17,7 @@ use crate::{config::core_config, state::db_client};
mod discord;
mod slack;
#[instrument(level = "debug")]
pub async fn send_alerts(alerts: &[Alert]) {
if alerts.is_empty() {
return;
@@ -54,14 +55,31 @@ async fn send_alert(alerters: &[Alerter], alert: &Alert) {
return;
}
let handles = alerters
.iter()
.map(|alerter| send_alert_to_alerter(alerter, alert));
join_all(handles)
.await
.into_iter()
.filter_map(|res| res.err())
.for_each(|e| error!("{e:#}"));
}
pub async fn send_alert_to_alerter(
alerter: &Alerter,
alert: &Alert,
) -> anyhow::Result<()> {
// Don't send if not enabled
if !alerter.config.enabled {
return Ok(());
}
let alert_type = alert.data.extract_variant();
let handles = alerters.iter().map(|alerter| async {
// Don't send if not enabled
if !alerter.config.enabled {
return Ok(());
}
// In the test case, we don't want the filters inside this
// block to stop the test from being sent to the alerting endpoint.
if alert_type != AlertDataVariant::Test {
// Don't send if alert type not configured on the alerter
if !alerter.config.alert_types.is_empty()
&& !alerter.config.alert_types.contains(&alert_type)
@@ -80,40 +98,34 @@ async fn send_alert(alerters: &[Alerter], alert: &Alert) {
{
return Ok(());
}
}
match &alerter.config.endpoint {
AlerterEndpoint::Custom(CustomAlerterEndpoint { url }) => {
send_custom_alert(url, alert).await.with_context(|| {
format!(
"failed to send alert to custom alerter {}",
alerter.name
)
})
}
AlerterEndpoint::Slack(SlackAlerterEndpoint { url }) => {
slack::send_alert(url, alert).await.with_context(|| {
format!(
"failed to send alert to slack alerter {}",
alerter.name
)
})
}
AlerterEndpoint::Discord(DiscordAlerterEndpoint { url }) => {
discord::send_alert(url, alert).await.with_context(|| {
format!(
"failed to send alert to Discord alerter {}",
alerter.name
)
})
}
match &alerter.config.endpoint {
AlerterEndpoint::Custom(CustomAlerterEndpoint { url }) => {
send_custom_alert(url, alert).await.with_context(|| {
format!(
"Failed to send alert to Custom Alerter {}",
alerter.name
)
})
}
});
join_all(handles)
.await
.into_iter()
.filter_map(|res| res.err())
.for_each(|e| error!("{e:#}"));
AlerterEndpoint::Slack(SlackAlerterEndpoint { url }) => {
slack::send_alert(url, alert).await.with_context(|| {
format!(
"Failed to send alert to Slack Alerter {}",
alerter.name
)
})
}
AlerterEndpoint::Discord(DiscordAlerterEndpoint { url }) => {
discord::send_alert(url, alert).await.with_context(|| {
format!(
"Failed to send alert to Discord Alerter {}",
alerter.name
)
})
}
}
}
#[instrument(level = "debug")]

View File

@@ -0,0 +1,73 @@
use formatting::format_serror;
use komodo_client::{
api::execute::TestAlerter,
entities::{
alert::{Alert, AlertData, SeverityLevel},
alerter::Alerter,
komodo_timestamp,
permission::PermissionLevel,
},
};
use resolver_api::Resolve;
use crate::{
alert::send_alert_to_alerter, helpers::update::update_update,
resource::get_check_permissions,
};
use super::ExecuteArgs;
impl Resolve<ExecuteArgs> for TestAlerter {
#[instrument(name = "TestAlerter", skip(user, update), fields(user_id = user.id, update_id = update.id))]
async fn resolve(
self,
ExecuteArgs { user, update }: &ExecuteArgs,
) -> Result<Self::Response, Self::Error> {
let alerter = get_check_permissions::<Alerter>(
&self.alerter,
user,
PermissionLevel::Execute,
)
.await?;
let mut update = update.clone();
if !alerter.config.enabled {
update.push_error_log(
"Test Alerter",
String::from(
"Alerter is disabled. Enable the Alerter to send alerts.",
),
);
update.finalize();
update_update(update.clone()).await?;
return Ok(update);
}
let ts = komodo_timestamp();
let alert = Alert {
id: Default::default(),
ts,
resolved: true,
level: SeverityLevel::Ok,
target: update.target.clone(),
data: AlertData::Test {
id: alerter.id.clone(),
name: alerter.name.clone(),
},
resolved_ts: Some(ts),
};
if let Err(e) = send_alert_to_alerter(&alerter, &alert).await {
update.push_error_log("Test Alerter", format_serror(&e.into()));
} else {
update.push_simple_log("Test Alerter", String::from("Alert sent successfully. It should be visible at your alerting destination."));
};
update.finalize();
update_update(update.clone()).await?;
Ok(update)
}
}

View File

@@ -30,6 +30,7 @@ use crate::{
};
mod action;
mod alerter;
mod build;
mod deployment;
mod procedure;
@@ -132,6 +133,9 @@ pub enum ExecuteRequest {
// ==== SERVER TEMPLATE ====
LaunchServer(LaunchServer),
// ==== ALERTER ====
TestAlerter(TestAlerter),
// ==== SYNC ====
RunSync(RunSync),
}

View File

@@ -1093,6 +1093,23 @@ async fn execute_execution(
"Batch method BatchDestroyStack not implemented correctly"
));
}
Execution::TestAlerter(req) => {
let req = ExecuteRequest::TestAlerter(req);
let update = init_execution_update(&req, &user).await?;
let ExecuteRequest::TestAlerter(req) = req else {
unreachable!()
};
let update_id = update.id.clone();
handle_resolve_result(
req
.resolve(&ExecuteArgs { user, update })
.await
.map_err(|e| e.error)
.context("Failed at TestAlerter"),
&update_id,
)
.await?
}
Execution::Sleep(req) => {
let duration = Duration::from_millis(req.duration_ms as u64);
tokio::time::sleep(duration).await;

View File

@@ -1,6 +1,7 @@
use anyhow::Context;
use komodo_client::entities::{
action::Action,
alerter::Alerter,
build::Build,
deployment::Deployment,
komodo_timestamp,
@@ -498,13 +499,24 @@ pub async fn init_execution_update(
ExecuteRequest::BatchDestroyStack(_data) => {
return Ok(Default::default())
}
// Alerter
ExecuteRequest::TestAlerter(data) => (
Operation::TestAlerter,
ResourceTarget::Alerter(
resource::get::<Alerter>(&data.alerter).await?.id,
),
),
};
let mut update = make_update(target, operation, user);
update.in_progress();
// Hold off on even adding update for DeployStackIfChanged
if !matches!(&request, ExecuteRequest::DeployStackIfChanged(_)) {
// Don't actually send it here, let the handlers send it after they can set action state.
update.id = add_update_without_send(&update).await?;
}
Ok(update)
}

View File

@@ -5,6 +5,7 @@ use komodo_client::{
api::execute::Execution,
entities::{
action::Action,
alerter::Alerter,
build::Build,
deployment::Deployment,
permission::PermissionLevel,
@@ -687,6 +688,15 @@ async fn validate_config(
));
}
}
Execution::TestAlerter(params) => {
let alerter = super::get_check_permissions::<Alerter>(
&params.alerter,
user,
PermissionLevel::Execute,
)
.await?;
params.alerter = alerter.id;
}
Execution::Sleep(_) => {}
}
}

View File

@@ -701,6 +701,13 @@ impl ResourceSyncTrait for Procedure {
.unwrap_or_default();
}
Execution::BatchDestroyStack(_config) => {}
Execution::TestAlerter(config) => {
config.alerter = resources
.alerters
.get(&config.alerter)
.map(|a| a.name.clone())
.unwrap_or_default();
}
Execution::Sleep(_) => {}
}
}

View File

@@ -789,6 +789,13 @@ impl ToToml for Procedure {
.unwrap_or(&String::new()),
),
Execution::BatchDestroyStack(_exec) => {}
Execution::TestAlerter(exec) => exec.alerter.clone_from(
all
.alerters
.get(&exec.alerter)
.map(|a| &a.name)
.unwrap_or(&String::new()),
),
Execution::Sleep(_) | Execution::None(_) => {}
}
}

View File

@@ -17,6 +17,7 @@ mod stack;
mod sync;
pub use action::*;
pub use alerter::*;
pub use build::*;
pub use deployment::*;
pub use procedure::*;
@@ -135,6 +136,9 @@ pub enum Execution {
DestroyStack(DestroyStack),
BatchDestroyStack(BatchDestroyStack),
// ALERTER
TestAlerter(TestAlerter),
// SLEEP
Sleep(Sleep),
}

View File

@@ -1,6 +1,6 @@
//! # Komodo core API
//! # Komodo Core API
//!
//! Komodo core exposes an HTTP api using standard JSON serialization.
//! Komodo Core exposes an HTTP api using standard JSON serialization.
//!
//! All calls share some common HTTP params:
//! - Method: `POST`