From a89bd4a36d97be469a59faf975e0381101bda936 Mon Sep 17 00:00:00 2001 From: mbecker20 Date: Fri, 16 Aug 2024 12:25:35 -0400 Subject: [PATCH] deleting ResourceSync cleans up open alerts use .then in alert cleanup --- bin/core/src/helpers/mod.rs | 75 +++++++++++++++++++++++++++++++-- bin/core/src/main.rs | 6 +-- bin/core/src/resource/server.rs | 2 +- bin/core/src/resource/sync.rs | 14 +++++- lib/logger/src/lib.rs | 2 +- 5 files changed, 88 insertions(+), 11 deletions(-) diff --git a/bin/core/src/helpers/mod.rs b/bin/core/src/helpers/mod.rs index 2159472ff..8af7c7d12 100644 --- a/bin/core/src/helpers/mod.rs +++ b/bin/core/src/helpers/mod.rs @@ -1,20 +1,26 @@ -use std::{collections::HashSet, time::Duration}; +use std::{collections::HashSet, str::FromStr, time::Duration}; use anyhow::{anyhow, Context}; +use futures::future::join_all; use mongo_indexed::Document; use monitor_client::entities::{ + monitor_timestamp, permission::{Permission, PermissionLevel, UserTarget}, server::Server, + sync::ResourceSync, update::{Log, ResourceTarget, Update}, user::User, EnvironmentVar, }; -use mungos::mongodb::bson::{doc, to_document, Bson}; +use mungos::{ + find::find_collect, + mongodb::bson::{doc, oid::ObjectId, to_document, Bson}, +}; use periphery_client::PeripheryClient; use query::get_global_variables; use rand::{distributions::Alphanumeric, thread_rng, Rng}; -use crate::{config::core_config, state::db_client}; +use crate::{config::core_config, resource, state::db_client}; pub mod action_state; pub mod alert; @@ -280,8 +286,15 @@ pub async fn interpolate_variables_secrets_into_environment( Ok(secret_replacers) } +pub async fn startup_cleanup() { + tokio::join!( + startup_in_progress_update_cleanup(), + startup_open_alert_cleanup(), + ); +} + /// Run on startup, as no updates should be in progress on startup -pub async fn startup_in_progress_update_cleanup() { +async fn startup_in_progress_update_cleanup() { let log = Log::error( "monitor shutdown", String::from("Monitor shutdown during execution. If this is a build, the builder may not have been terminated.") @@ -308,3 +321,57 @@ pub async fn startup_in_progress_update_cleanup() { error!("failed to cleanup in progress updates on startup | {e:#}") } } + +/// Run on startup, ensure open alerts pointing to invalid resources are closed. +async fn startup_open_alert_cleanup() { + let db = db_client().await; + let Ok(alerts) = + find_collect(&db.alerts, doc! { "resolved": false }, None) + .await + .inspect_err(|e| { + error!( + "failed to list all alerts for startup open alert cleanup | {e:?}" + ) + }) + else { + return; + }; + let futures = alerts.into_iter().map(|alert| async move { + match alert.target { + ResourceTarget::Server(id) => { + resource::get::(&id) + .await + .is_err() + .then(|| ObjectId::from_str(&alert.id).inspect_err(|e| warn!("failed to clean up alert - id is invalid ObjectId | {e:?}")).ok()).flatten() + } + ResourceTarget::ResourceSync(id) => { + resource::get::(&id) + .await + .is_err() + .then(|| ObjectId::from_str(&alert.id).inspect_err(|e| warn!("failed to clean up alert - id is invalid ObjectId | {e:?}")).ok()).flatten() + } + // No other resources should have open alerts. + _ => ObjectId::from_str(&alert.id).inspect_err(|e| warn!("failed to clean up alert - id is invalid ObjectId | {e:?}")).ok(), + } + }); + let to_update_ids = join_all(futures) + .await + .into_iter() + .flatten() + .collect::>(); + if let Err(e) = db + .alerts + .update_many( + doc! { "_id": { "$in": to_update_ids } }, + doc! { "$set": { + "resolved": true, + "resolved_ts": monitor_timestamp() + } }, + ) + .await + { + error!( + "failed to clean up invalid open alerts on startup | {e:#}" + ) + } +} diff --git a/bin/core/src/main.rs b/bin/core/src/main.rs index a5edf5d6f..e40f6d7c2 100644 --- a/bin/core/src/main.rs +++ b/bin/core/src/main.rs @@ -5,8 +5,6 @@ use std::{net::SocketAddr, str::FromStr}; use anyhow::Context; use axum::Router; -use helpers::startup_in_progress_update_cleanup; -use state::jwt_client; use tower_http::{ cors::{Any, CorsLayer}, services::{ServeDir, ServeFile}, @@ -34,9 +32,9 @@ async fn app() -> anyhow::Result<()> { info!("config: {:?}", config.sanitized()); // includes init db_client check to crash on db init failure - startup_in_progress_update_cleanup().await; + helpers::startup_cleanup().await; // init jwt client to crash on failure - jwt_client(); + state::jwt_client(); // Spawn tasks monitor::spawn_monitor_loop(); diff --git a/bin/core/src/resource/server.rs b/bin/core/src/resource/server.rs index 17ffec2c7..ff8f4beee 100644 --- a/bin/core/src/resource/server.rs +++ b/bin/core/src/resource/server.rs @@ -167,7 +167,7 @@ impl super::MonitorResource for Server { } }, ) .await - .context("failed to detach server from repos")?; + .context("failed to close deleted server alerts")?; Ok(()) } diff --git a/bin/core/src/resource/sync.rs b/bin/core/src/resource/sync.rs index d7c5c1513..6837258d1 100644 --- a/bin/core/src/resource/sync.rs +++ b/bin/core/src/resource/sync.rs @@ -3,6 +3,7 @@ use std::time::Duration; use anyhow::Context; use mongo_indexed::doc; use monitor_client::entities::{ + monitor_timestamp, resource::Resource, sync::{ PartialResourceSyncConfig, PendingSyncUpdatesData, ResourceSync, @@ -126,9 +127,20 @@ impl super::MonitorResource for ResourceSync { } async fn pre_delete( - _resource: &Resource, + resource: &Resource, _update: &mut Update, ) -> anyhow::Result<()> { + db_client().await.alerts + .update_many( + doc! { "target.type": "ResourceSync", "target.id": &resource.id }, + doc! { "$set": { + "resolved": true, + "resolved_ts": monitor_timestamp() + } }, + ) + .await + .context("failed to close deleted sync alerts")?; + Ok(()) } diff --git a/lib/logger/src/lib.rs b/lib/logger/src/lib.rs index 26d06e340..892c0c1b7 100644 --- a/lib/logger/src/lib.rs +++ b/lib/logger/src/lib.rs @@ -52,7 +52,7 @@ pub fn init(config: &LogConfig) -> anyhow::Result<()> { (StdioLogMode::Json, None) => registry .with(tracing_subscriber::fmt::layer().json()) .try_init(), - + (StdioLogMode::None, None) => Ok(()), } .context("failed to init logger")