deleting ResourceSync cleans up open alerts

use .then in alert cleanup

bin/core/src/helpers/mod.rs

@@ -1,20 +1,26 @@
-use std::{collections::HashSet, time::Duration};
+use std::{collections::HashSet, str::FromStr, time::Duration};
 
 use anyhow::{anyhow, Context};
+use futures::future::join_all;
 use mongo_indexed::Document;
 use monitor_client::entities::{
+  monitor_timestamp,
   permission::{Permission, PermissionLevel, UserTarget},
   server::Server,
+  sync::ResourceSync,
   update::{Log, ResourceTarget, Update},
   user::User,
   EnvironmentVar,
 };
-use mungos::mongodb::bson::{doc, to_document, Bson};
+use mungos::{
+  find::find_collect,
+  mongodb::bson::{doc, oid::ObjectId, to_document, Bson},
+};
 use periphery_client::PeripheryClient;
 use query::get_global_variables;
 use rand::{distributions::Alphanumeric, thread_rng, Rng};
 
-use crate::{config::core_config, state::db_client};
+use crate::{config::core_config, resource, state::db_client};
 
 pub mod action_state;
 pub mod alert;
@@ -280,8 +286,15 @@ pub async fn interpolate_variables_secrets_into_environment(
   Ok(secret_replacers)
 }
 
+pub async fn startup_cleanup() {
+  tokio::join!(
+    startup_in_progress_update_cleanup(),
+    startup_open_alert_cleanup(),
+  );
+}
+
 /// Run on startup, as no updates should be in progress on startup
-pub async fn startup_in_progress_update_cleanup() {
+async fn startup_in_progress_update_cleanup() {
   let log = Log::error(
     "monitor shutdown",
     String::from("Monitor shutdown during execution. If this is a build, the builder may not have been terminated.")
@@ -308,3 +321,57 @@ pub async fn startup_in_progress_update_cleanup() {
     error!("failed to cleanup in progress updates on startup | {e:#}")
   }
 }
+
+/// Run on startup, ensure open alerts pointing to invalid resources are closed.
+async fn startup_open_alert_cleanup() {
+  let db = db_client().await;
+  let Ok(alerts) =
+    find_collect(&db.alerts, doc! { "resolved": false }, None)
+      .await
+      .inspect_err(|e| {
+        error!(
+          "failed to list all alerts for startup open alert cleanup | {e:?}"
+        )
+      })
+  else {
+    return;
+  };
+  let futures = alerts.into_iter().map(|alert| async move {
+    match alert.target {
+      ResourceTarget::Server(id) => {
+        resource::get::<Server>(&id)
+          .await
+          .is_err()
+          .then(|| ObjectId::from_str(&alert.id).inspect_err(|e| warn!("failed to clean up alert - id is invalid ObjectId | {e:?}")).ok()).flatten()
+      }
+      ResourceTarget::ResourceSync(id) => {
+        resource::get::<ResourceSync>(&id)
+          .await
+          .is_err()
+          .then(|| ObjectId::from_str(&alert.id).inspect_err(|e| warn!("failed to clean up alert - id is invalid ObjectId | {e:?}")).ok()).flatten()
+      }
+      // No other resources should have open alerts.
+      _ => ObjectId::from_str(&alert.id).inspect_err(|e| warn!("failed to clean up alert - id is invalid ObjectId | {e:?}")).ok(),
+    }
+  });
+  let to_update_ids = join_all(futures)
+    .await
+    .into_iter()
+    .flatten()
+    .collect::<Vec<_>>();
+  if let Err(e) = db
+    .alerts
+    .update_many(
+      doc! { "_id": { "$in": to_update_ids } },
+      doc! { "$set": {
+        "resolved": true,
+        "resolved_ts": monitor_timestamp()
+      } },
+    )
+    .await
+  {
+    error!(
+      "failed to clean up invalid open alerts on startup | {e:#}"
+    )
+  }
+}

bin/core/src/main.rs

@@ -5,8 +5,6 @@ use std::{net::SocketAddr, str::FromStr};
 
 use anyhow::Context;
 use axum::Router;
-use helpers::startup_in_progress_update_cleanup;
-use state::jwt_client;
 use tower_http::{
   cors::{Any, CorsLayer},
   services::{ServeDir, ServeFile},
@@ -34,9 +32,9 @@ async fn app() -> anyhow::Result<()> {
   info!("config: {:?}", config.sanitized());
 
   // includes init db_client check to crash on db init failure
-  startup_in_progress_update_cleanup().await;
+  helpers::startup_cleanup().await;
   // init jwt client to crash on failure
-  jwt_client();
+  state::jwt_client();
 
   // Spawn tasks
   monitor::spawn_monitor_loop();

bin/core/src/resource/server.rs

@@ -167,7 +167,7 @@ impl super::MonitorResource for Server {
         } },
       )
       .await
-      .context("failed to detach server from repos")?;
+      .context("failed to close deleted server alerts")?;
 
     Ok(())
   }

bin/core/src/resource/sync.rs

@@ -3,6 +3,7 @@ use std::time::Duration;
 use anyhow::Context;
 use mongo_indexed::doc;
 use monitor_client::entities::{
+  monitor_timestamp,
   resource::Resource,
   sync::{
     PartialResourceSyncConfig, PendingSyncUpdatesData, ResourceSync,
@@ -126,9 +127,20 @@ impl super::MonitorResource for ResourceSync {
   }
 
   async fn pre_delete(
-    _resource: &Resource<Self::Config, Self::Info>,
+    resource: &Resource<Self::Config, Self::Info>,
     _update: &mut Update,
   ) -> anyhow::Result<()> {
+    db_client().await.alerts
+      .update_many(
+        doc! { "target.type": "ResourceSync", "target.id": &resource.id },
+        doc! { "$set": {
+          "resolved": true,
+          "resolved_ts": monitor_timestamp()
+        } },
+      )
+      .await
+      .context("failed to close deleted sync alerts")?;
+
     Ok(())
   }
 

lib/logger/src/lib.rs

@@ -52,7 +52,7 @@ pub fn init(config: &LogConfig) -> anyhow::Result<()> {
     (StdioLogMode::Json, None) => registry
       .with(tracing_subscriber::fmt::layer().json())
       .try_init(),
-      
+
     (StdioLogMode::None, None) => Ok(()),
   }
   .context("failed to init logger")