add reason (err) server unreachable in alert

2026-03-12 02:04:44 -05:00 · 2024-04-07 04:52:58 -07:00
parent 50866659ea
commit 4c8eb68611
19 changed files with 125 additions and 64 deletions
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -3036,9 +3036,9 @@ dependencies = [

 [[package]]
 name = "serror"
-version = "0.1.8"
+version = "0.1.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9ff25dfba3f34f2b1e53051a9bd99ddaa326cd5ddd88319c4274f7451b99814d"
+checksum = "9d3df9a2b74d806ecbe70d5156f436edca357e458b8d970ce88cab324a599190"
 dependencies = [
 "anyhow",
 "axum",
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -17,7 +17,7 @@ logger = { path = "lib/logger" }

 # MOGH
 run_command = { version = "0.0.6", features = ["async_tokio"] }
-serror = { version = "0.1.8", features = ["axum"] }
+serror = { version = "0.1.9", features = ["axum"] }
 slack = { version = "0.1.0", package = "slack_client_rs" }
 derive_default_builder = "0.1.8"
 derive_empty_traits = "0.1.0"
--- a/bin/core/src/api/execute/build.rs
+++ b/bin/core/src/api/execute/build.rs
@@ -120,7 +120,7 @@ impl Resolve<RunBuild, User> for State {
        update.logs.push(Log::error(
          "get builder",
          // The unwrap is safe, it is inside a block which checks for .is_err()
-          serialize_error_pretty(builder.err().unwrap()),
+          serialize_error_pretty(&builder.err().unwrap()),
        ));
        update.finalize();
        update_update(update.clone()).await?;
@@ -149,7 +149,7 @@ impl Resolve<RunBuild, User> for State {
      match res {
        Ok(clone_logs) => update.logs.extend(clone_logs),
        Err(e) => {
-          update.push_error_log("clone repo", serialize_error(e));
+          update.push_error_log("clone repo", serialize_error(&e));
        }
      }

@@ -174,7 +174,7 @@ impl Resolve<RunBuild, User> for State {
        match res {
          Ok(logs) => update.logs.extend(logs),
          Err(e) => {
-            update.push_error_log("build", serialize_error(e))
+            update.push_error_log("build", serialize_error(&e))
          }
        };
      }
@@ -418,7 +418,7 @@ async fn cleanup_builder_instance(
          format!("terminate instance id {}", instance_id),
        ),
        Err(e) => {
-          Log::error("terminate instance", serialize_error_pretty(e))
+          Log::error("terminate instance", serialize_error_pretty(&e))
        }
      };
      update.logs.push(log);
--- a/bin/core/src/api/execute/deployment.rs
+++ b/bin/core/src/api/execute/deployment.rs
@@ -114,7 +114,7 @@ impl Resolve<Deploy, User> for State {
      {
        Ok(log) => log,
        Err(e) => {
-          Log::error("deploy container", serialize_error_pretty(e))
+          Log::error("deploy container", serialize_error_pretty(&e))
        }
      };

@@ -203,7 +203,7 @@ impl Resolve<StartContainer, User> for State {
      {
        Ok(log) => log,
        Err(e) => {
-          Log::error("start container", serialize_error_pretty(e))
+          Log::error("start container", serialize_error_pretty(&e))
        }
      };

@@ -293,7 +293,7 @@ impl Resolve<StopContainer, User> for State {
      {
        Ok(log) => log,
        Err(e) => {
-          Log::error("stop container", serialize_error_pretty(e))
+          Log::error("stop container", serialize_error_pretty(&e))
        }
      };

@@ -384,7 +384,7 @@ impl Resolve<StopAllContainers, User> for State {
            "stop container failure",
            format!(
              "failed to stop container {name} ({id})\n\n{}",
-              serialize_error_pretty(e)
+              serialize_error_pretty(&e)
            ),
          );
        }
@@ -481,7 +481,7 @@ impl Resolve<RemoveContainer, User> for State {
      {
        Ok(log) => log,
        Err(e) => {
-          Log::error("stop container", serialize_error_pretty(e))
+          Log::error("stop container", serialize_error_pretty(&e))
        }
      };

--- a/bin/core/src/api/execute/procedure.rs
+++ b/bin/core/src/api/execute/procedure.rs
@@ -56,7 +56,7 @@ impl Resolve<RunProcedure, User> for State {
        );
      }
      Err(e) => update
-        .push_error_log("execution error", serialize_error_pretty(e)),
+        .push_error_log("execution error", serialize_error_pretty(&e)),
    }

    update.finalize();
--- a/bin/core/src/api/execute/repo.rs
+++ b/bin/core/src/api/execute/repo.rs
@@ -75,7 +75,7 @@ impl Resolve<CloneRepo, User> for State {
      {
        Ok(logs) => logs,
        Err(e) => {
-          vec![Log::error("clone repo", serialize_error_pretty(e))]
+          vec![Log::error("clone repo", serialize_error_pretty(&e))]
        }
      };

@@ -176,7 +176,7 @@ impl Resolve<PullRepo, User> for State {
      {
        Ok(logs) => logs,
        Err(e) => {
-          vec![Log::error("pull repo", serialize_error_pretty(e))]
+          vec![Log::error("pull repo", serialize_error_pretty(&e))]
        }
      };

--- a/bin/core/src/api/execute/server.rs
+++ b/bin/core/src/api/execute/server.rs
@@ -58,7 +58,7 @@ impl Resolve<PruneDockerContainers, User> for State {
        )) {
        Ok(log) => log,
        Err(e) => {
-          Log::error("prune containers", serialize_error_pretty(e))
+          Log::error("prune containers", serialize_error_pretty(&e))
        }
      };

@@ -127,7 +127,7 @@ impl Resolve<PruneDockerNetworks, User> for State {
        )) {
        Ok(log) => log,
        Err(e) => {
-          Log::error("prune networks", serialize_error_pretty(e))
+          Log::error("prune networks", serialize_error_pretty(&e))
        }
      };

--- a/bin/core/src/api/write/repo.rs
+++ b/bin/core/src/api/write/repo.rs
@@ -252,7 +252,7 @@ impl Resolve<DeleteRepo, User> for State {
          Ok(log) => update.logs.push(log),
          Err(e) => update.logs.push(Log::error(
            "delete repo on periphery",
-            serialize_error_pretty(e),
+            serialize_error_pretty(&e),
          )),
        }
      }
--- a/bin/core/src/api/write/server.rs
+++ b/bin/core/src/api/write/server.rs
@@ -293,7 +293,7 @@ impl Resolve<CreateNetwork, User> for State {
    {
      Ok(log) => update.logs.push(log),
      Err(e) => update
-        .push_error_log("create network", serialize_error_pretty(e)),
+        .push_error_log("create network", serialize_error_pretty(&e)),
    };

    update.finalize();
@@ -330,7 +330,7 @@ impl Resolve<DeleteNetwork, User> for State {
    {
      Ok(log) => update.logs.push(log),
      Err(e) => update
-        .push_error_log("delete network", serialize_error_pretty(e)),
+        .push_error_log("delete network", serialize_error_pretty(&e)),
    };

    update.finalize();
--- a/bin/core/src/monitor/alert/server.rs
+++ b/bin/core/src/monitor/alert/server.rs
@@ -76,11 +76,35 @@ pub async fn alert_servers(
            id: server_status.id.clone(),
            name: server.name.clone(),
            region: optional_string(&server.info.region),
+            err: server_status.err.clone(),
          },
        };
        alerts_to_open
          .push((alert, server.info.send_unreachable_alerts))
      }
+      (ServerStatus::NotOk, Some(alert)) => {
+        // update alert err
+        let mut alert = alert.clone();
+        let (id, name, region) = match alert.data {
+          AlertData::ServerUnreachable {
+            id, name, region, ..
+          } => (id, name, region),
+          data => {
+            error!("got incorrect alert data in ServerStatus handler. got {data:?}");
+            continue;
+          }
+        };
+        alert.data = AlertData::ServerUnreachable {
+          id,
+          name,
+          region,
+          err: server_status.err.clone(),
+        };
+        alerts_to_update
+          .push((alert, server.info.send_unreachable_alerts));
+      }
+
+      // Close an open alert
      (
        ServerStatus::Ok | ServerStatus::Disabled,
        Some(health_alert),
--- a/bin/core/src/monitor/helpers.rs
+++ b/bin/core/src/monitor/helpers.rs
@@ -8,6 +8,7 @@ use monitor_client::entities::{
    Server, ServerConfig, ServerStatus,
  },
 };
+use serror::Serror;

 use crate::helpers::cache::{
  deployment_status_cache, server_status_cache,
@@ -44,6 +45,7 @@ pub async fn insert_server_status(
  status: ServerStatus,
  version: String,
  stats: Option<AllSystemStats>,
+  err: impl Into<Option<Serror>>,
 ) {
  let health = stats.as_ref().map(|s| get_server_health(server, s));
  server_status_cache()
@@ -55,6 +57,7 @@ pub async fn insert_server_status(
        version,
        stats,
        health,
+        err: err.into(),
      }
      .into(),
    )
--- a/bin/core/src/monitor/mod.rs
+++ b/bin/core/src/monitor/mod.rs
@@ -9,6 +9,7 @@ use monitor_client::entities::{
 };
 use mungos::{find::find_collect, mongodb::bson::doc};
 use periphery_client::api;
+use serror::Serror;

 use crate::{
  db::db_client,
@@ -37,6 +38,7 @@ pub struct CachedServerStatus {
  pub version: String,
  pub stats: Option<AllSystemStats>,
  pub health: Option<ServerHealth>,
+  pub err: Option<serror::Serror>,
 }

 #[derive(Default, Clone)]
@@ -74,17 +76,20 @@ pub fn spawn_monitor_loop() {
 }

 pub async fn update_cache_for_server(server: &Server) {
-  let deployments = find_collect(
+  let deployments = match find_collect(
    &db_client().await.deployments,
    doc! { "config.server_id": &server.id },
    None,
  )
-  .await;
-  if let Err(e) = &deployments {
-    error!("failed to get deployments list from mongo (update status cache) | server id: {} | {e:#?}", server.id);
-    return;
-  }
-  let deployments = deployments.unwrap();
+  .await
+  {
+    Ok(deployments) => deployments,
+    Err(e) => {
+      error!("failed to get deployments list from mongo (update status cache) | server id: {} | {e:#?}", server.id);
+      return;
+    }
+  };
+
  if !server.config.enabled {
    insert_deployments_status_unknown(deployments).await;
    insert_server_status(
@@ -92,51 +97,63 @@ pub async fn update_cache_for_server(server: &Server) {
      ServerStatus::Disabled,
      String::from("unknown"),
      None,
+      None,
    )
    .await;
    return;
  }
  // already handle server disabled case above, so using unwrap here
  let periphery = periphery_client(server).unwrap();
-  let version = periphery.request(api::GetVersion {}).await;
-  if version.is_err() {
-    insert_deployments_status_unknown(deployments).await;
-    insert_server_status(
-      server,
-      ServerStatus::NotOk,
-      String::from("unknown"),
-      None,
-    )
-    .await;
-    return;
-  }
+
+  let version = match periphery.request(api::GetVersion {}).await {
+    Ok(version) => version.version,
+    Err(e) => {
+      insert_deployments_status_unknown(deployments).await;
+      insert_server_status(
+        server,
+        ServerStatus::NotOk,
+        String::from("unknown"),
+        None,
+        Serror::from(&e),
+      )
+      .await;
+      return;
+    }
+  };
+
  let stats =
-    periphery.request(api::stats::GetAllSystemStats {}).await;
-  if stats.is_err() {
-    insert_deployments_status_unknown(deployments).await;
-    insert_server_status(
-      server,
-      ServerStatus::NotOk,
-      String::from("unknown"),
-      None,
-    )
-    .await;
-    return;
-  }
-  let stats = stats.unwrap();
+    match periphery.request(api::stats::GetAllSystemStats {}).await {
+      Ok(stats) => stats,
+      Err(e) => {
+        insert_deployments_status_unknown(deployments).await;
+        insert_server_status(
+          server,
+          ServerStatus::NotOk,
+          String::from("unknown"),
+          None,
+          Serror::from(&e),
+        )
+        .await;
+        return;
+      }
+    };
+
  insert_server_status(
    server,
    ServerStatus::Ok,
-    version.unwrap().version,
+    version,
    stats.into(),
+    None,
  )
  .await;
+
  let containers =
    periphery.request(api::container::GetContainerList {}).await;
  if containers.is_err() {
    insert_deployments_status_unknown(deployments).await;
    return;
  }
+  
  let containers = containers.unwrap();
  let status_cache = deployment_status_cache();
  for deployment in deployments {
--- a/bin/core/src/ws.rs
+++ b/bin/core/src/ws.rs
@@ -58,7 +58,7 @@ async fn ws_handler(ws: WebSocketUpgrade) -> impl IntoResponse {
        let user = match user {
          Err(e) => {
            let _ = ws_sender
-              .send(Message::Text(json!({ "type": "INVALID_USER", "msg": serialize_error(e) }).to_string()))
+              .send(Message::Text(json!({ "type": "INVALID_USER", "msg": serialize_error(&e) }).to_string()))
              .await;
            let _ = ws_sender.close().await; 
            return;
--- a/bin/periphery/src/api/build.rs
+++ b/bin/periphery/src/api/build.rs
@@ -30,11 +30,11 @@ impl Resolve<Build> for State {
        match docker::build::build(&build, docker_token).await {
          Ok(logs) => logs,
          Err(e) => {
-            vec![Log::error("build", serialize_error_pretty(e))]
+            vec![Log::error("build", serialize_error_pretty(&e))]
          }
        }
      }
-      Err(e) => vec![Log::error("build", serialize_error_pretty(e))],
+      Err(e) => vec![Log::error("build", serialize_error_pretty(&e))],
    };
    Ok(log)
  }
--- a/bin/periphery/src/helpers/docker/container.rs
+++ b/bin/periphery/src/helpers/docker/container.rs
@@ -169,7 +169,7 @@ pub async fn deploy(
  )) {
    Ok(token) => token,
    Err(e) => {
-      return Log::error("docker login", serialize_error_pretty(e))
+      return Log::error("docker login", serialize_error_pretty(&e))
    }
  };

@@ -179,7 +179,7 @@ pub async fn deploy(
  )
  .await
  {
-    return Log::error("docker login", serialize_error_pretty(e));
+    return Log::error("docker login", serialize_error_pretty(&e));
  }

  let image = if let DeploymentImage::Image { image } =
--- a/client/core/rs/src/entities/alert.rs
+++ b/client/core/rs/src/entities/alert.rs
@@ -11,9 +11,7 @@ use typeshare::typeshare;
 use crate::entities::{MongoId, I64};

 use super::{
-  deployment::DockerContainerState,
-  server::stats::{SeverityLevel, SystemProcess},
-  update::ResourceTarget,
+  _Serror, deployment::DockerContainerState, server::stats::{SeverityLevel, SystemProcess}, update::ResourceTarget
 };

 #[typeshare]
@@ -65,6 +63,7 @@ pub enum AlertData {
    id: String,
    name: String,
    region: Option<String>,
+    err: Option<_Serror>
  },
  ServerCpu {
    id: String,
--- a/client/core/rs/src/entities/mod.rs
+++ b/client/core/rs/src/entities/mod.rs
@@ -1,6 +1,7 @@
 use anyhow::{anyhow, Context};
 use async_timing_util::unix_timestamp_ms;
 use serde::{Deserialize, Serialize};
+use serror::Serror;
 use strum::{Display, EnumString};
 use typeshare::typeshare;

@@ -27,6 +28,8 @@ pub type U64 = u64;
 pub type MongoDocument = mungos::mongodb::bson::Document;
 #[typeshare(serialized_as = "MongoIdObj")]
 pub type MongoId = String;
+#[typeshare(serialized_as = "__Serror")]
+pub type _Serror = Serror;

 pub fn all_logs_success(logs: &[update::Log]) -> bool {
  for log in logs {
@@ -82,6 +85,13 @@ pub struct MongoIdObj {
  pub oid: String,
 }

+#[typeshare]
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct __Serror {
+  pub error: String,
+  pub trace: Vec<String>,
+}
+
 #[typeshare]
 #[derive(
  Serialize, Deserialize, Debug, Clone, Default, PartialEq, Eq,
--- a/client/core/rs/src/ws.rs
+++ b/client/core/rs/src/ws.rs
@@ -90,7 +90,7 @@ impl MonitorClient {

          if let Err(e) = res {
            let _ = tx.send(UpdateWsMessage::Error(
-              UpdateWsError::ConnectionError(serialize_error(e)),
+              UpdateWsError::ConnectionError(serialize_error(&e)),
            ));
            break;
          }
@@ -107,7 +107,7 @@ impl MonitorClient {

          if let Err(e) = login_send_res {
            let _ = tx.send(UpdateWsMessage::Error(
-              UpdateWsError::LoginError(serialize_error(e)),
+              UpdateWsError::LoginError(serialize_error(&e)),
            ));
            break;
          }
@@ -182,7 +182,7 @@ impl MonitorClient {
              }
              Err(e) => {
                let _ = tx.send(UpdateWsMessage::Error(
-                  UpdateWsError::MessageError(serialize_error(e)),
+                  UpdateWsError::MessageError(serialize_error(&e)),
                ));
                let _ = tx.send(UpdateWsMessage::Disconnected);
                let _ = ws.close(None).await;
--- a/client/core/ts/src/types.ts
+++ b/client/core/ts/src/types.ts
@@ -813,6 +813,13 @@ export type U64 = number;

 export type MongoDocument = any;

+export interface __Serror {
+	error: string;
+	trace: string[];
+}
+
+export type _Serror = __Serror;
+
 export interface ProcedureQuerySpecifics {
 	types: ProcedureConfig["type"][];
 }
@@ -967,6 +974,7 @@ export type AlertData =
 	id: string;
 	name: string;
 	region?: string;
+	err?: _Serror;
 }}
 	| { type: "ServerCpu", data: {
 	id: string;