diff --git a/Cargo.lock b/Cargo.lock index e81a050d7..f6658d796 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3036,9 +3036,9 @@ dependencies = [ [[package]] name = "serror" -version = "0.1.8" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ff25dfba3f34f2b1e53051a9bd99ddaa326cd5ddd88319c4274f7451b99814d" +checksum = "9d3df9a2b74d806ecbe70d5156f436edca357e458b8d970ce88cab324a599190" dependencies = [ "anyhow", "axum", diff --git a/Cargo.toml b/Cargo.toml index 9c48f1c19..b03016965 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,7 +17,7 @@ logger = { path = "lib/logger" } # MOGH run_command = { version = "0.0.6", features = ["async_tokio"] } -serror = { version = "0.1.8", features = ["axum"] } +serror = { version = "0.1.9", features = ["axum"] } slack = { version = "0.1.0", package = "slack_client_rs" } derive_default_builder = "0.1.8" derive_empty_traits = "0.1.0" diff --git a/bin/core/src/api/execute/build.rs b/bin/core/src/api/execute/build.rs index ad39fb3e3..cbda3654d 100644 --- a/bin/core/src/api/execute/build.rs +++ b/bin/core/src/api/execute/build.rs @@ -120,7 +120,7 @@ impl Resolve for State { update.logs.push(Log::error( "get builder", // The unwrap is safe, it is inside a block which checks for .is_err() - serialize_error_pretty(builder.err().unwrap()), + serialize_error_pretty(&builder.err().unwrap()), )); update.finalize(); update_update(update.clone()).await?; @@ -149,7 +149,7 @@ impl Resolve for State { match res { Ok(clone_logs) => update.logs.extend(clone_logs), Err(e) => { - update.push_error_log("clone repo", serialize_error(e)); + update.push_error_log("clone repo", serialize_error(&e)); } } @@ -174,7 +174,7 @@ impl Resolve for State { match res { Ok(logs) => update.logs.extend(logs), Err(e) => { - update.push_error_log("build", serialize_error(e)) + update.push_error_log("build", serialize_error(&e)) } }; } @@ -418,7 +418,7 @@ async fn cleanup_builder_instance( format!("terminate instance id {}", instance_id), ), Err(e) => { - Log::error("terminate instance", serialize_error_pretty(e)) + Log::error("terminate instance", serialize_error_pretty(&e)) } }; update.logs.push(log); diff --git a/bin/core/src/api/execute/deployment.rs b/bin/core/src/api/execute/deployment.rs index 48ca605ef..feb3b4607 100644 --- a/bin/core/src/api/execute/deployment.rs +++ b/bin/core/src/api/execute/deployment.rs @@ -114,7 +114,7 @@ impl Resolve for State { { Ok(log) => log, Err(e) => { - Log::error("deploy container", serialize_error_pretty(e)) + Log::error("deploy container", serialize_error_pretty(&e)) } }; @@ -203,7 +203,7 @@ impl Resolve for State { { Ok(log) => log, Err(e) => { - Log::error("start container", serialize_error_pretty(e)) + Log::error("start container", serialize_error_pretty(&e)) } }; @@ -293,7 +293,7 @@ impl Resolve for State { { Ok(log) => log, Err(e) => { - Log::error("stop container", serialize_error_pretty(e)) + Log::error("stop container", serialize_error_pretty(&e)) } }; @@ -384,7 +384,7 @@ impl Resolve for State { "stop container failure", format!( "failed to stop container {name} ({id})\n\n{}", - serialize_error_pretty(e) + serialize_error_pretty(&e) ), ); } @@ -481,7 +481,7 @@ impl Resolve for State { { Ok(log) => log, Err(e) => { - Log::error("stop container", serialize_error_pretty(e)) + Log::error("stop container", serialize_error_pretty(&e)) } }; diff --git a/bin/core/src/api/execute/procedure.rs b/bin/core/src/api/execute/procedure.rs index b5de4f733..033291aec 100644 --- a/bin/core/src/api/execute/procedure.rs +++ b/bin/core/src/api/execute/procedure.rs @@ -56,7 +56,7 @@ impl Resolve for State { ); } Err(e) => update - .push_error_log("execution error", serialize_error_pretty(e)), + .push_error_log("execution error", serialize_error_pretty(&e)), } update.finalize(); diff --git a/bin/core/src/api/execute/repo.rs b/bin/core/src/api/execute/repo.rs index 45df0f6e6..1f0b4efcb 100644 --- a/bin/core/src/api/execute/repo.rs +++ b/bin/core/src/api/execute/repo.rs @@ -75,7 +75,7 @@ impl Resolve for State { { Ok(logs) => logs, Err(e) => { - vec![Log::error("clone repo", serialize_error_pretty(e))] + vec![Log::error("clone repo", serialize_error_pretty(&e))] } }; @@ -176,7 +176,7 @@ impl Resolve for State { { Ok(logs) => logs, Err(e) => { - vec![Log::error("pull repo", serialize_error_pretty(e))] + vec![Log::error("pull repo", serialize_error_pretty(&e))] } }; diff --git a/bin/core/src/api/execute/server.rs b/bin/core/src/api/execute/server.rs index 84c603435..24380a492 100644 --- a/bin/core/src/api/execute/server.rs +++ b/bin/core/src/api/execute/server.rs @@ -58,7 +58,7 @@ impl Resolve for State { )) { Ok(log) => log, Err(e) => { - Log::error("prune containers", serialize_error_pretty(e)) + Log::error("prune containers", serialize_error_pretty(&e)) } }; @@ -127,7 +127,7 @@ impl Resolve for State { )) { Ok(log) => log, Err(e) => { - Log::error("prune networks", serialize_error_pretty(e)) + Log::error("prune networks", serialize_error_pretty(&e)) } }; diff --git a/bin/core/src/api/write/repo.rs b/bin/core/src/api/write/repo.rs index 4c4c53443..b59dd804a 100644 --- a/bin/core/src/api/write/repo.rs +++ b/bin/core/src/api/write/repo.rs @@ -252,7 +252,7 @@ impl Resolve for State { Ok(log) => update.logs.push(log), Err(e) => update.logs.push(Log::error( "delete repo on periphery", - serialize_error_pretty(e), + serialize_error_pretty(&e), )), } } diff --git a/bin/core/src/api/write/server.rs b/bin/core/src/api/write/server.rs index a8a10a7ea..00891090b 100644 --- a/bin/core/src/api/write/server.rs +++ b/bin/core/src/api/write/server.rs @@ -293,7 +293,7 @@ impl Resolve for State { { Ok(log) => update.logs.push(log), Err(e) => update - .push_error_log("create network", serialize_error_pretty(e)), + .push_error_log("create network", serialize_error_pretty(&e)), }; update.finalize(); @@ -330,7 +330,7 @@ impl Resolve for State { { Ok(log) => update.logs.push(log), Err(e) => update - .push_error_log("delete network", serialize_error_pretty(e)), + .push_error_log("delete network", serialize_error_pretty(&e)), }; update.finalize(); diff --git a/bin/core/src/monitor/alert/server.rs b/bin/core/src/monitor/alert/server.rs index b9520b93c..0c9bdc145 100644 --- a/bin/core/src/monitor/alert/server.rs +++ b/bin/core/src/monitor/alert/server.rs @@ -76,11 +76,35 @@ pub async fn alert_servers( id: server_status.id.clone(), name: server.name.clone(), region: optional_string(&server.info.region), + err: server_status.err.clone(), }, }; alerts_to_open .push((alert, server.info.send_unreachable_alerts)) } + (ServerStatus::NotOk, Some(alert)) => { + // update alert err + let mut alert = alert.clone(); + let (id, name, region) = match alert.data { + AlertData::ServerUnreachable { + id, name, region, .. + } => (id, name, region), + data => { + error!("got incorrect alert data in ServerStatus handler. got {data:?}"); + continue; + } + }; + alert.data = AlertData::ServerUnreachable { + id, + name, + region, + err: server_status.err.clone(), + }; + alerts_to_update + .push((alert, server.info.send_unreachable_alerts)); + } + + // Close an open alert ( ServerStatus::Ok | ServerStatus::Disabled, Some(health_alert), diff --git a/bin/core/src/monitor/helpers.rs b/bin/core/src/monitor/helpers.rs index 2918583ab..1418e39b4 100644 --- a/bin/core/src/monitor/helpers.rs +++ b/bin/core/src/monitor/helpers.rs @@ -8,6 +8,7 @@ use monitor_client::entities::{ Server, ServerConfig, ServerStatus, }, }; +use serror::Serror; use crate::helpers::cache::{ deployment_status_cache, server_status_cache, @@ -44,6 +45,7 @@ pub async fn insert_server_status( status: ServerStatus, version: String, stats: Option, + err: impl Into>, ) { let health = stats.as_ref().map(|s| get_server_health(server, s)); server_status_cache() @@ -55,6 +57,7 @@ pub async fn insert_server_status( version, stats, health, + err: err.into(), } .into(), ) diff --git a/bin/core/src/monitor/mod.rs b/bin/core/src/monitor/mod.rs index 9a10237d6..e2c4002b4 100644 --- a/bin/core/src/monitor/mod.rs +++ b/bin/core/src/monitor/mod.rs @@ -9,6 +9,7 @@ use monitor_client::entities::{ }; use mungos::{find::find_collect, mongodb::bson::doc}; use periphery_client::api; +use serror::Serror; use crate::{ db::db_client, @@ -37,6 +38,7 @@ pub struct CachedServerStatus { pub version: String, pub stats: Option, pub health: Option, + pub err: Option, } #[derive(Default, Clone)] @@ -74,17 +76,20 @@ pub fn spawn_monitor_loop() { } pub async fn update_cache_for_server(server: &Server) { - let deployments = find_collect( + let deployments = match find_collect( &db_client().await.deployments, doc! { "config.server_id": &server.id }, None, ) - .await; - if let Err(e) = &deployments { - error!("failed to get deployments list from mongo (update status cache) | server id: {} | {e:#?}", server.id); - return; - } - let deployments = deployments.unwrap(); + .await + { + Ok(deployments) => deployments, + Err(e) => { + error!("failed to get deployments list from mongo (update status cache) | server id: {} | {e:#?}", server.id); + return; + } + }; + if !server.config.enabled { insert_deployments_status_unknown(deployments).await; insert_server_status( @@ -92,51 +97,63 @@ pub async fn update_cache_for_server(server: &Server) { ServerStatus::Disabled, String::from("unknown"), None, + None, ) .await; return; } // already handle server disabled case above, so using unwrap here let periphery = periphery_client(server).unwrap(); - let version = periphery.request(api::GetVersion {}).await; - if version.is_err() { - insert_deployments_status_unknown(deployments).await; - insert_server_status( - server, - ServerStatus::NotOk, - String::from("unknown"), - None, - ) - .await; - return; - } + + let version = match periphery.request(api::GetVersion {}).await { + Ok(version) => version.version, + Err(e) => { + insert_deployments_status_unknown(deployments).await; + insert_server_status( + server, + ServerStatus::NotOk, + String::from("unknown"), + None, + Serror::from(&e), + ) + .await; + return; + } + }; + let stats = - periphery.request(api::stats::GetAllSystemStats {}).await; - if stats.is_err() { - insert_deployments_status_unknown(deployments).await; - insert_server_status( - server, - ServerStatus::NotOk, - String::from("unknown"), - None, - ) - .await; - return; - } - let stats = stats.unwrap(); + match periphery.request(api::stats::GetAllSystemStats {}).await { + Ok(stats) => stats, + Err(e) => { + insert_deployments_status_unknown(deployments).await; + insert_server_status( + server, + ServerStatus::NotOk, + String::from("unknown"), + None, + Serror::from(&e), + ) + .await; + return; + } + }; + insert_server_status( server, ServerStatus::Ok, - version.unwrap().version, + version, stats.into(), + None, ) .await; + let containers = periphery.request(api::container::GetContainerList {}).await; if containers.is_err() { insert_deployments_status_unknown(deployments).await; return; } + let containers = containers.unwrap(); let status_cache = deployment_status_cache(); for deployment in deployments { diff --git a/bin/core/src/ws.rs b/bin/core/src/ws.rs index 73c405984..0b0f6f456 100644 --- a/bin/core/src/ws.rs +++ b/bin/core/src/ws.rs @@ -58,7 +58,7 @@ async fn ws_handler(ws: WebSocketUpgrade) -> impl IntoResponse { let user = match user { Err(e) => { let _ = ws_sender - .send(Message::Text(json!({ "type": "INVALID_USER", "msg": serialize_error(e) }).to_string())) + .send(Message::Text(json!({ "type": "INVALID_USER", "msg": serialize_error(&e) }).to_string())) .await; let _ = ws_sender.close().await; return; diff --git a/bin/periphery/src/api/build.rs b/bin/periphery/src/api/build.rs index c57579e4a..d7604a90c 100644 --- a/bin/periphery/src/api/build.rs +++ b/bin/periphery/src/api/build.rs @@ -30,11 +30,11 @@ impl Resolve for State { match docker::build::build(&build, docker_token).await { Ok(logs) => logs, Err(e) => { - vec![Log::error("build", serialize_error_pretty(e))] + vec![Log::error("build", serialize_error_pretty(&e))] } } } - Err(e) => vec![Log::error("build", serialize_error_pretty(e))], + Err(e) => vec![Log::error("build", serialize_error_pretty(&e))], }; Ok(log) } diff --git a/bin/periphery/src/helpers/docker/container.rs b/bin/periphery/src/helpers/docker/container.rs index fbf411894..b1d0c545b 100644 --- a/bin/periphery/src/helpers/docker/container.rs +++ b/bin/periphery/src/helpers/docker/container.rs @@ -169,7 +169,7 @@ pub async fn deploy( )) { Ok(token) => token, Err(e) => { - return Log::error("docker login", serialize_error_pretty(e)) + return Log::error("docker login", serialize_error_pretty(&e)) } }; @@ -179,7 +179,7 @@ pub async fn deploy( ) .await { - return Log::error("docker login", serialize_error_pretty(e)); + return Log::error("docker login", serialize_error_pretty(&e)); } let image = if let DeploymentImage::Image { image } = diff --git a/client/core/rs/src/entities/alert.rs b/client/core/rs/src/entities/alert.rs index 5b425464d..de224fd72 100644 --- a/client/core/rs/src/entities/alert.rs +++ b/client/core/rs/src/entities/alert.rs @@ -11,9 +11,7 @@ use typeshare::typeshare; use crate::entities::{MongoId, I64}; use super::{ - deployment::DockerContainerState, - server::stats::{SeverityLevel, SystemProcess}, - update::ResourceTarget, + _Serror, deployment::DockerContainerState, server::stats::{SeverityLevel, SystemProcess}, update::ResourceTarget }; #[typeshare] @@ -65,6 +63,7 @@ pub enum AlertData { id: String, name: String, region: Option, + err: Option<_Serror> }, ServerCpu { id: String, diff --git a/client/core/rs/src/entities/mod.rs b/client/core/rs/src/entities/mod.rs index 6b56e2815..1e4b81ac2 100644 --- a/client/core/rs/src/entities/mod.rs +++ b/client/core/rs/src/entities/mod.rs @@ -1,6 +1,7 @@ use anyhow::{anyhow, Context}; use async_timing_util::unix_timestamp_ms; use serde::{Deserialize, Serialize}; +use serror::Serror; use strum::{Display, EnumString}; use typeshare::typeshare; @@ -27,6 +28,8 @@ pub type U64 = u64; pub type MongoDocument = mungos::mongodb::bson::Document; #[typeshare(serialized_as = "MongoIdObj")] pub type MongoId = String; +#[typeshare(serialized_as = "__Serror")] +pub type _Serror = Serror; pub fn all_logs_success(logs: &[update::Log]) -> bool { for log in logs { @@ -82,6 +85,13 @@ pub struct MongoIdObj { pub oid: String, } +#[typeshare] +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct __Serror { + pub error: String, + pub trace: Vec, +} + #[typeshare] #[derive( Serialize, Deserialize, Debug, Clone, Default, PartialEq, Eq, diff --git a/client/core/rs/src/ws.rs b/client/core/rs/src/ws.rs index 29c434642..2cd4b0018 100644 --- a/client/core/rs/src/ws.rs +++ b/client/core/rs/src/ws.rs @@ -90,7 +90,7 @@ impl MonitorClient { if let Err(e) = res { let _ = tx.send(UpdateWsMessage::Error( - UpdateWsError::ConnectionError(serialize_error(e)), + UpdateWsError::ConnectionError(serialize_error(&e)), )); break; } @@ -107,7 +107,7 @@ impl MonitorClient { if let Err(e) = login_send_res { let _ = tx.send(UpdateWsMessage::Error( - UpdateWsError::LoginError(serialize_error(e)), + UpdateWsError::LoginError(serialize_error(&e)), )); break; } @@ -182,7 +182,7 @@ impl MonitorClient { } Err(e) => { let _ = tx.send(UpdateWsMessage::Error( - UpdateWsError::MessageError(serialize_error(e)), + UpdateWsError::MessageError(serialize_error(&e)), )); let _ = tx.send(UpdateWsMessage::Disconnected); let _ = ws.close(None).await; diff --git a/client/core/ts/src/types.ts b/client/core/ts/src/types.ts index 8ec78fe52..596f348ff 100644 --- a/client/core/ts/src/types.ts +++ b/client/core/ts/src/types.ts @@ -813,6 +813,13 @@ export type U64 = number; export type MongoDocument = any; +export interface __Serror { + error: string; + trace: string[]; +} + +export type _Serror = __Serror; + export interface ProcedureQuerySpecifics { types: ProcedureConfig["type"][]; } @@ -967,6 +974,7 @@ export type AlertData = id: string; name: string; region?: string; + err?: _Serror; }} | { type: "ServerCpu", data: { id: string;