ComputerSurge/komodo
3
0
Fork 0
forked from github-starred/komodo
Code Pull Requests Packages Releases Activity

add reason (err) server unreachable in alert

Browse Source
This commit is contained in:
mbecker20
2024-04-07 04:52:58 -07:00
parent 50866659ea
commit 4c8eb68611
19 changed files with 125 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
Cargo.lock generated
View File

@@ -3036,9 +3036,9 @@ dependencies = [
[[package]]
name = "serror"
version = "0.1.8"
version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9ff25dfba3f34f2b1e53051a9bd99ddaa326cd5ddd88319c4274f7451b99814d"
checksum = "9d3df9a2b74d806ecbe70d5156f436edca357e458b8d970ce88cab324a599190"
dependencies = [
"anyhow",
"axum",

2
Cargo.toml
View File

@@ -17,7 +17,7 @@ logger = { path = "lib/logger" }
# MOGH
run_command = { version = "0.0.6", features = ["async_tokio"] }
serror = { version = "0.1.8", features = ["axum"] }
serror = { version = "0.1.9", features = ["axum"] }
slack = { version = "0.1.0", package = "slack_client_rs" }
derive_default_builder = "0.1.8"
derive_empty_traits = "0.1.0"

8
bin/core/src/api/execute/build.rs
View File

@@ -120,7 +120,7 @@ impl Resolve<RunBuild, User> for State {
update.logs.push(Log::error(
"get builder",
// The unwrap is safe, it is inside a block which checks for .is_err()
serialize_error_pretty(builder.err().unwrap()),
serialize_error_pretty(&builder.err().unwrap()),
));
update.finalize();
update_update(update.clone()).await?;
@@ -149,7 +149,7 @@ impl Resolve<RunBuild, User> for State {
match res {
Ok(clone_logs) => update.logs.extend(clone_logs),
Err(e) => {
update.push_error_log("clone repo", serialize_error(e));
update.push_error_log("clone repo", serialize_error(&e));
}
}
@@ -174,7 +174,7 @@ impl Resolve<RunBuild, User> for State {
match res {
Ok(logs) => update.logs.extend(logs),
Err(e) => {
update.push_error_log("build", serialize_error(e))
update.push_error_log("build", serialize_error(&e))
}
};
}
@@ -418,7 +418,7 @@ async fn cleanup_builder_instance(
format!("terminate instance id {}", instance_id),
),
Err(e) => {
Log::error("terminate instance", serialize_error_pretty(e))
Log::error("terminate instance", serialize_error_pretty(&e))
}
};
update.logs.push(log);

10
bin/core/src/api/execute/deployment.rs
View File

@@ -114,7 +114,7 @@ impl Resolve<Deploy, User> for State {
{
Ok(log) => log,
Err(e) => {
Log::error("deploy container", serialize_error_pretty(e))
Log::error("deploy container", serialize_error_pretty(&e))
}
};
@@ -203,7 +203,7 @@ impl Resolve<StartContainer, User> for State {
{
Ok(log) => log,
Err(e) => {
Log::error("start container", serialize_error_pretty(e))
Log::error("start container", serialize_error_pretty(&e))
}
};
@@ -293,7 +293,7 @@ impl Resolve<StopContainer, User> for State {
{
Ok(log) => log,
Err(e) => {
Log::error("stop container", serialize_error_pretty(e))
Log::error("stop container", serialize_error_pretty(&e))
}
};
@@ -384,7 +384,7 @@ impl Resolve<StopAllContainers, User> for State {
"stop container failure",
format!(
"failed to stop container {name} ({id})\n\n{}",
serialize_error_pretty(e)
serialize_error_pretty(&e)
),
);
}
@@ -481,7 +481,7 @@ impl Resolve<RemoveContainer, User> for State {
{
Ok(log) => log,
Err(e) => {
Log::error("stop container", serialize_error_pretty(e))
Log::error("stop container", serialize_error_pretty(&e))
}
};

2
bin/core/src/api/execute/procedure.rs
View File

@@ -56,7 +56,7 @@ impl Resolve<RunProcedure, User> for State {
);
}
Err(e) => update
.push_error_log("execution error", serialize_error_pretty(e)),
.push_error_log("execution error", serialize_error_pretty(&e)),
}
update.finalize();

4
bin/core/src/api/execute/repo.rs
View File

@@ -75,7 +75,7 @@ impl Resolve<CloneRepo, User> for State {
{
Ok(logs) => logs,
Err(e) => {
vec![Log::error("clone repo", serialize_error_pretty(e))]
vec![Log::error("clone repo", serialize_error_pretty(&e))]
}
};
@@ -176,7 +176,7 @@ impl Resolve<PullRepo, User> for State {
{
Ok(logs) => logs,
Err(e) => {
vec![Log::error("pull repo", serialize_error_pretty(e))]
vec![Log::error("pull repo", serialize_error_pretty(&e))]
}
};

4
bin/core/src/api/execute/server.rs
View File

@@ -58,7 +58,7 @@ impl Resolve<PruneDockerContainers, User> for State {
)) {
Ok(log) => log,
Err(e) => {
Log::error("prune containers", serialize_error_pretty(e))
Log::error("prune containers", serialize_error_pretty(&e))
}
};
@@ -127,7 +127,7 @@ impl Resolve<PruneDockerNetworks, User> for State {
)) {
Ok(log) => log,
Err(e) => {
Log::error("prune networks", serialize_error_pretty(e))
Log::error("prune networks", serialize_error_pretty(&e))
}
};

2
bin/core/src/api/write/repo.rs
View File

@@ -252,7 +252,7 @@ impl Resolve<DeleteRepo, User> for State {
Ok(log) => update.logs.push(log),
Err(e) => update.logs.push(Log::error(
"delete repo on periphery",
serialize_error_pretty(e),
serialize_error_pretty(&e),
)),
}
}

4
bin/core/src/api/write/server.rs
View File

@@ -293,7 +293,7 @@ impl Resolve<CreateNetwork, User> for State {
{
Ok(log) => update.logs.push(log),
Err(e) => update
.push_error_log("create network", serialize_error_pretty(e)),
.push_error_log("create network", serialize_error_pretty(&e)),
};
update.finalize();
@@ -330,7 +330,7 @@ impl Resolve<DeleteNetwork, User> for State {
{
Ok(log) => update.logs.push(log),
Err(e) => update
.push_error_log("delete network", serialize_error_pretty(e)),
.push_error_log("delete network", serialize_error_pretty(&e)),
};
update.finalize();

24
bin/core/src/monitor/alert/server.rs
View File

@@ -76,11 +76,35 @@ pub async fn alert_servers(
id: server_status.id.clone(),
name: server.name.clone(),
region: optional_string(&server.info.region),
err: server_status.err.clone(),
},
};
alerts_to_open
.push((alert, server.info.send_unreachable_alerts))
}
(ServerStatus::NotOk, Some(alert)) => {
// update alert err
let mut alert = alert.clone();
let (id, name, region) = match alert.data {
AlertData::ServerUnreachable {
id, name, region, ..
} => (id, name, region),
data => {
error!("got incorrect alert data in ServerStatus handler. got {data:?}");
continue;
}
};
alert.data = AlertData::ServerUnreachable {
id,
name,
region,
err: server_status.err.clone(),
};
alerts_to_update
.push((alert, server.info.send_unreachable_alerts));
}
// Close an open alert
(
ServerStatus::Ok | ServerStatus::Disabled,
Some(health_alert),

3
bin/core/src/monitor/helpers.rs
View File

@@ -8,6 +8,7 @@ use monitor_client::entities::{
Server, ServerConfig, ServerStatus,
},
};
use serror::Serror;
use crate::helpers::cache::{
deployment_status_cache, server_status_cache,
@@ -44,6 +45,7 @@ pub async fn insert_server_status(
status: ServerStatus,
version: String,
stats: Option<AllSystemStats>,
err: impl Into<Option<Serror>>,
) {
let health = stats.as_ref().map(|s| get_server_health(server, s));
server_status_cache()
@@ -55,6 +57,7 @@ pub async fn insert_server_status(
version,
stats,
health,
err: err.into(),
}
.into(),
)

83
bin/core/src/monitor/mod.rs
View File

@@ -9,6 +9,7 @@ use monitor_client::entities::{
};
use mungos::{find::find_collect, mongodb::bson::doc};
use periphery_client::api;
use serror::Serror;
use crate::{
db::db_client,
@@ -37,6 +38,7 @@ pub struct CachedServerStatus {
pub version: String,
pub stats: Option<AllSystemStats>,
pub health: Option<ServerHealth>,
pub err: Option<serror::Serror>,
}
#[derive(Default, Clone)]
@@ -74,17 +76,20 @@ pub fn spawn_monitor_loop() {
}
pub async fn update_cache_for_server(server: &Server) {
let deployments = find_collect(
let deployments = match find_collect(
&db_client().await.deployments,
doc! { "config.server_id": &server.id },
None,
)
.await;
if let Err(e) = &deployments {
error!("failed to get deployments list from mongo (update status cache) | server id: {} | {e:#?}", server.id);
return;
}
let deployments = deployments.unwrap();
.await
{
Ok(deployments) => deployments,
Err(e) => {
error!("failed to get deployments list from mongo (update status cache) | server id: {} | {e:#?}", server.id);
return;
}
};
if !server.config.enabled {
insert_deployments_status_unknown(deployments).await;
insert_server_status(
@@ -92,51 +97,63 @@ pub async fn update_cache_for_server(server: &Server) {
ServerStatus::Disabled,
String::from("unknown"),
None,
None,
)
.await;
return;
}
// already handle server disabled case above, so using unwrap here
let periphery = periphery_client(server).unwrap();
let version = periphery.request(api::GetVersion {}).await;
if version.is_err() {
insert_deployments_status_unknown(deployments).await;
insert_server_status(
server,
ServerStatus::NotOk,
String::from("unknown"),
None,
)
.await;
return;
}
let version = match periphery.request(api::GetVersion {}).await {
Ok(version) => version.version,
Err(e) => {
insert_deployments_status_unknown(deployments).await;
insert_server_status(
server,
ServerStatus::NotOk,
String::from("unknown"),
None,
Serror::from(&e),
)
.await;
return;
}
};
let stats =
periphery.request(api::stats::GetAllSystemStats {}).await;
if stats.is_err() {
insert_deployments_status_unknown(deployments).await;
insert_server_status(
server,
ServerStatus::NotOk,
String::from("unknown"),
None,
)
.await;
return;
}
let stats = stats.unwrap();
match periphery.request(api::stats::GetAllSystemStats {}).await {
Ok(stats) => stats,
Err(e) => {
insert_deployments_status_unknown(deployments).await;
insert_server_status(
server,
ServerStatus::NotOk,
String::from("unknown"),
None,
Serror::from(&e),
)
.await;
return;
}
};
insert_server_status(
server,
ServerStatus::Ok,
version.unwrap().version,
version,
stats.into(),
None,
)
.await;
let containers =
periphery.request(api::container::GetContainerList {}).await;
if containers.is_err() {
insert_deployments_status_unknown(deployments).await;
return;
}
let containers = containers.unwrap();
let status_cache = deployment_status_cache();
for deployment in deployments {

2
bin/core/src/ws.rs
View File

@@ -58,7 +58,7 @@ async fn ws_handler(ws: WebSocketUpgrade) -> impl IntoResponse {
let user = match user {
Err(e) => {
let _ = ws_sender
.send(Message::Text(json!({ "type": "INVALID_USER", "msg": serialize_error(e) }).to_string()))
.send(Message::Text(json!({ "type": "INVALID_USER", "msg": serialize_error(&e) }).to_string()))
.await;
let _ = ws_sender.close().await;
return;

4
bin/periphery/src/api/build.rs
View File

@@ -30,11 +30,11 @@ impl Resolve<Build> for State {
match docker::build::build(&build, docker_token).await {
Ok(logs) => logs,
Err(e) => {
vec![Log::error("build", serialize_error_pretty(e))]
vec![Log::error("build", serialize_error_pretty(&e))]
}
}
}
Err(e) => vec![Log::error("build", serialize_error_pretty(e))],
Err(e) => vec![Log::error("build", serialize_error_pretty(&e))],
};
Ok(log)
}

4
bin/periphery/src/helpers/docker/container.rs
View File

@@ -169,7 +169,7 @@ pub async fn deploy(
)) {
Ok(token) => token,
Err(e) => {
return Log::error("docker login", serialize_error_pretty(e))
return Log::error("docker login", serialize_error_pretty(&e))
}
};
@@ -179,7 +179,7 @@ pub async fn deploy(
)
.await
{
return Log::error("docker login", serialize_error_pretty(e));
return Log::error("docker login", serialize_error_pretty(&e));
}
let image = if let DeploymentImage::Image { image } =

5
client/core/rs/src/entities/alert.rs
View File

@@ -11,9 +11,7 @@ use typeshare::typeshare;
use crate::entities::{MongoId, I64};
use super::{
deployment::DockerContainerState,
server::stats::{SeverityLevel, SystemProcess},
update::ResourceTarget,
_Serror, deployment::DockerContainerState, server::stats::{SeverityLevel, SystemProcess}, update::ResourceTarget
};
#[typeshare]
@@ -65,6 +63,7 @@ pub enum AlertData {
id: String,
name: String,
region: Option<String>,
err: Option<_Serror>
},
ServerCpu {
id: String,

10
client/core/rs/src/entities/mod.rs
View File

@@ -1,6 +1,7 @@
use anyhow::{anyhow, Context};
use async_timing_util::unix_timestamp_ms;
use serde::{Deserialize, Serialize};
use serror::Serror;
use strum::{Display, EnumString};
use typeshare::typeshare;
@@ -27,6 +28,8 @@ pub type U64 = u64;
pub type MongoDocument = mungos::mongodb::bson::Document;
#[typeshare(serialized_as = "MongoIdObj")]
pub type MongoId = String;
#[typeshare(serialized_as = "__Serror")]
pub type _Serror = Serror;
pub fn all_logs_success(logs: &[update::Log]) -> bool {
for log in logs {
@@ -82,6 +85,13 @@ pub struct MongoIdObj {
pub oid: String,
}
#[typeshare]
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct __Serror {
pub error: String,
pub trace: Vec<String>,
}
#[typeshare]
#[derive(
Serialize, Deserialize, Debug, Clone, Default, PartialEq, Eq,

6
client/core/rs/src/ws.rs
View File

@@ -90,7 +90,7 @@ impl MonitorClient {
if let Err(e) = res {
let _ = tx.send(UpdateWsMessage::Error(
UpdateWsError::ConnectionError(serialize_error(e)),
UpdateWsError::ConnectionError(serialize_error(&e)),
));
break;
}
@@ -107,7 +107,7 @@ impl MonitorClient {
if let Err(e) = login_send_res {
let _ = tx.send(UpdateWsMessage::Error(
UpdateWsError::LoginError(serialize_error(e)),
UpdateWsError::LoginError(serialize_error(&e)),
));
break;
}
@@ -182,7 +182,7 @@ impl MonitorClient {
}
Err(e) => {
let _ = tx.send(UpdateWsMessage::Error(
UpdateWsError::MessageError(serialize_error(e)),
UpdateWsError::MessageError(serialize_error(&e)),
));
let _ = tx.send(UpdateWsMessage::Disconnected);
let _ = ws.close(None).await;

8
client/core/ts/src/types.ts
View File

@@ -813,6 +813,13 @@ export type U64 = number;
export type MongoDocument = any;
export interface __Serror {
error: string;
trace: string[];
}
export type _Serror = __Serror;
export interface ProcedureQuerySpecifics {
types: ProcedureConfig["type"][];
}
@@ -967,6 +974,7 @@ export type AlertData =
id: string;
name: string;
region?: string;
err?: _Serror;
}}
| { type: "ServerCpu", data: {
id: string;