From 0ceecee60459290d7573096b687a95af15b619e9 Mon Sep 17 00:00:00 2001 From: mbecker20 Date: Sat, 31 Dec 2022 05:42:44 +0000 Subject: [PATCH] a bunch of stuff --- Cargo.lock | 4 +- core/src/actions/build.rs | 4 +- core/src/actions/deployment.rs | 17 ++++-- core/src/api/server.rs | 10 ++-- core/src/config.rs | 2 +- core/src/helpers.rs | 10 ---- frontend/index.html | 10 ++-- frontend/package.json | 2 +- frontend/post-build.mjs | 11 ++++ frontend/src/components/shared/Icon.tsx | 2 +- .../src/components/topbar/Search/Search.tsx | 27 +++++---- frontend/src/components/topbar/Topbar.tsx | 6 +- frontend/src/style/index.scss | 24 ++++++++ lib/helpers/src/docker/build.rs | 11 ++-- lib/helpers/src/git.rs | 37 ++++++++++++- lib/helpers/src/lib.rs | 9 +++ lib/monitor_client/src/server.rs | 17 +++--- lib/periphery_client/src/git.rs | 7 ++- lib/periphery_client/src/lib.rs | 20 +++++-- lib/types/src/deployment.rs | 8 +++ lib/types/src/server.rs | 28 ++++++++++ periphery/Cargo.toml | 2 +- periphery/src/api/git.rs | 15 +++-- periphery/src/api/stats.rs | 55 +++++++++++++------ tests/src/tests.rs | 2 +- 25 files changed, 247 insertions(+), 93 deletions(-) create mode 100644 frontend/post-build.mjs diff --git a/Cargo.lock b/Cargo.lock index 4b9aa7c51..cf4057053 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2249,9 +2249,9 @@ checksum = "20518fe4a4c9acf048008599e464deb21beeae3d3578418951a189c235a7a9a8" [[package]] name = "sysinfo" -version = "0.27.0" +version = "0.27.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d08ba83d6dde63d053e42d7230f0dc7f8d8efeb8d30d3681580d158156461ba" +checksum = "17351d0e9eb8841897b14e9669378f3c69fb57779cc04f8ca9a9d512edfb2563" dependencies = [ "cfg-if", "core-foundation-sys", diff --git a/core/src/actions/build.rs b/core/src/actions/build.rs index 0a354b470..01b60c279 100644 --- a/core/src/actions/build.rs +++ b/core/src/actions/build.rs @@ -1,6 +1,6 @@ use anyhow::{anyhow, Context}; use diff::Diff; -use helpers::to_monitor_name; +use helpers::{all_logs_success, to_monitor_name}; use mungos::{doc, to_bson}; use types::{ monitor_timestamp, @@ -10,7 +10,7 @@ use types::{ use crate::{ auth::RequestUser, - helpers::{all_logs_success, any_option_diff_is_some, option_diff_is_some}, + helpers::{any_option_diff_is_some, option_diff_is_some}, state::State, }; diff --git a/core/src/actions/deployment.rs b/core/src/actions/deployment.rs index fc6b704ba..e7bb3c19d 100644 --- a/core/src/actions/deployment.rs +++ b/core/src/actions/deployment.rs @@ -1,6 +1,6 @@ use anyhow::{anyhow, Context}; use diff::Diff; -use helpers::to_monitor_name; +use helpers::{all_logs_success, to_monitor_name}; use types::{ monitor_timestamp, traits::{Busy, Permissioned}, @@ -9,7 +9,7 @@ use types::{ use crate::{ auth::RequestUser, - helpers::{all_logs_success, any_option_diff_is_some, option_diff_is_some}, + helpers::{any_option_diff_is_some, option_diff_is_some}, state::State, }; @@ -633,13 +633,18 @@ impl State { update.id = self.add_update(update.clone()).await?; - let log = self + let logs = self .periphery - .pull_repo(&server, &deployment.name, &deployment.branch) + .pull_repo( + &server, + &deployment.name, + &deployment.branch, + &deployment.on_pull, + ) .await?; - update.success = log.success; - update.logs.push(log); + update.success = all_logs_success(&logs); + update.logs.extend(logs); update.end_ts = Some(monitor_timestamp()); update.status = UpdateStatus::Complete; diff --git a/core/src/api/server.rs b/core/src/api/server.rs index f6df07917..b46b22706 100644 --- a/core/src/api/server.rs +++ b/core/src/api/server.rs @@ -9,7 +9,7 @@ use helpers::handle_anyhow_error; use mungos::{Deserialize, Document, Serialize}; use types::{ traits::Permissioned, BasicContainerInfo, ImageSummary, Network, PermissionLevel, Server, - ServerActionState, ServerStatus, ServerWithStatus, SystemStats, + ServerActionState, ServerStatus, ServerWithStatus, SystemStats, SystemStatsQuery, }; use typeshare::typeshare; @@ -122,9 +122,10 @@ pub fn router() -> Router { get( |Extension(state): StateExtension, Extension(user): RequestUserExtension, - Path(ServerId { id }): Path| async move { + Path(ServerId { id }): Path, + Query(query): Query| async move { let stats = state - .get_server_stats(&id, &user) + .get_server_stats(&id, &user, &query) .await .map_err(handle_anyhow_error)?; response!(Json(stats)) @@ -323,13 +324,14 @@ impl State { &self, server_id: &str, user: &RequestUser, + query: &SystemStatsQuery, ) -> anyhow::Result { let server = self .get_server_check_permissions(server_id, user, PermissionLevel::Read) .await?; let stats = self .periphery - .get_system_stats(&server) + .get_system_stats(&server, query) .await .context(format!("failed to get stats from server {}", server.name))?; Ok(stats) diff --git a/core/src/config.rs b/core/src/config.rs index 23469dd67..4fc694c3a 100644 --- a/core/src/config.rs +++ b/core/src/config.rs @@ -16,7 +16,7 @@ pub fn load() -> (CoreConfig, SpaRouter) { dotenv().ok(); let env: Env = envy::from_env().expect("failed to parse environment variables"); let config = parse_config_file(&env.config_path).expect("failed to parse config"); - let spa_router = SpaRouter::new("/", env.frontend_path); + let spa_router = SpaRouter::new("/assets", env.frontend_path); (config, spa_router) } diff --git a/core/src/helpers.rs b/core/src/helpers.rs index 272eaa898..da57fafd1 100644 --- a/core/src/helpers.rs +++ b/core/src/helpers.rs @@ -1,5 +1,4 @@ use diff::{Diff, OptionDiff}; -use types::Log; #[macro_export] macro_rules! response { @@ -26,12 +25,3 @@ where } return false; } - -pub fn all_logs_success(logs: &Vec) -> bool { - for log in logs { - if !log.success { - return false; - } - } - true -} diff --git a/frontend/index.html b/frontend/index.html index 1a41de7c9..ecfded206 100644 --- a/frontend/index.html +++ b/frontend/index.html @@ -6,11 +6,11 @@ - - - - - + + + + + monitor diff --git a/frontend/package.json b/frontend/package.json index 383249894..760835c7f 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -5,7 +5,7 @@ "scripts": { "start": "vite", "dev": "vite", - "build": "vite build", + "build": "vite build && node post-build.mjs", "serve": "vite preview" }, "license": "GPL v3.0", diff --git a/frontend/post-build.mjs b/frontend/post-build.mjs new file mode 100644 index 000000000..17eb73415 --- /dev/null +++ b/frontend/post-build.mjs @@ -0,0 +1,11 @@ +import { readdirSync, renameSync, rmdirSync } from "fs"; + +const files = readdirSync("./build/assets"); + +for (const file of files) { + renameSync("./build/assets/" + file, "./build/" + file); +} + +rmdirSync("./build/assets"); + +console.log("\npost build complete\n") \ No newline at end of file diff --git a/frontend/src/components/shared/Icon.tsx b/frontend/src/components/shared/Icon.tsx index 748a5ade1..957f31cc5 100644 --- a/frontend/src/components/shared/Icon.tsx +++ b/frontend/src/components/shared/Icon.tsx @@ -59,7 +59,7 @@ const Icon: Component<{ return ( {p.alt void }> = (p) => { style={{ "max-height": "70vh", "padding-right": "0.5rem" }} > - no results + + no results + {(deployment, index) => ( - + )} @@ -183,14 +186,15 @@ const Builds: Component<{ close: () => void }> = (p) => { {(build, index) => ( - + )} @@ -206,7 +210,7 @@ const Builds: Component<{ close: () => void }> = (p) => { }; const Servers: Component<{ close: () => void }> = (p) => { - const navigate = useNavigate(); + // const navigate = useNavigate(); const { highlighted, filteredServers } = useSearchState(); return ( void }> = (p) => { {(server, index) => ( - + )} diff --git a/frontend/src/components/topbar/Topbar.tsx b/frontend/src/components/topbar/Topbar.tsx index e7d808519..939add539 100644 --- a/frontend/src/components/topbar/Topbar.tsx +++ b/frontend/src/components/topbar/Topbar.tsx @@ -1,4 +1,4 @@ -import { useNavigate } from "@solidjs/router"; +import { A, useNavigate } from "@solidjs/router"; import { Component, createSignal, JSX, Show } from "solid-js"; import { TOPBAR_HEIGHT } from "../.."; import { useAppDimensions } from "../../state/DimensionProvider"; @@ -47,9 +47,9 @@ const LeftSide: Component = () => { alignItems="center" style={{ padding: "0rem 0.5rem", "place-self": "center start" }} > - + , branch: Option, on_clone: Option, + on_pull: Option, pub github_account: Option, } @@ -23,6 +24,7 @@ impl From<&Deployment> for CloneArgs { repo: d.repo.clone(), branch: d.branch.clone(), on_clone: d.on_clone.clone(), + on_pull: d.on_pull.clone(), github_account: d.github_account.clone(), } } @@ -35,18 +37,37 @@ impl From<&Build> for CloneArgs { repo: b.repo.clone(), branch: b.branch.clone(), on_clone: b.on_clone.clone(), + on_pull: None, github_account: b.github_account.clone(), } } } -pub async fn pull(path: &str, branch: &Option) -> Log { +pub async fn pull(path: &str, branch: &Option, on_pull: &Option) -> Vec { let branch = match branch { Some(branch) => branch.to_owned(), None => "main".to_string(), }; let command = format!("cd {path} && git pull origin {branch}"); - run_monitor_command("git pull", command).await + let mut logs = Vec::new(); + let pull_log = run_monitor_command("git pull", command).await; + if !pull_log.success { + logs.push(pull_log); + return logs; + } + logs.push(pull_log); + if let Some(on_pull) = on_pull { + let mut path = PathBuf::from_str(path).unwrap(); + path.push(&on_pull.path); + let path = path.display().to_string(); + let on_pull_log = run_monitor_command( + "on pull command", + format!("cd {path} && {}", on_pull.command), + ) + .await; + logs.push(on_pull_log); + } + logs } pub async fn clone_repo( @@ -59,6 +80,7 @@ pub async fn clone_repo( repo, branch, on_clone, + on_pull, .. } = clone_args.into(); let repo = repo.as_ref().ok_or(anyhow!("build has no repo attached"))?; @@ -76,6 +98,17 @@ pub async fn clone_repo( ) .await; logs.push(on_clone_log); + repo_dir.pop(); + } + if let Some(command) = on_pull { + repo_dir.push(&command.path); + let on_clone_log = run_monitor_command( + "on clone", + format!("cd {} && {}", repo_dir.display(), command.command), + ) + .await; + logs.push(on_clone_log); + repo_dir.pop(); } Ok(logs) } diff --git a/lib/helpers/src/lib.rs b/lib/helpers/src/lib.rs index fc0bbdc98..37a10bf22 100644 --- a/lib/helpers/src/lib.rs +++ b/lib/helpers/src/lib.rs @@ -71,3 +71,12 @@ pub fn generate_secret(length: usize) -> String { .map(char::from) .collect() } + +pub fn all_logs_success(logs: &Vec) -> bool { + for log in logs { + if !log.success { + return false; + } + } + true +} diff --git a/lib/monitor_client/src/server.rs b/lib/monitor_client/src/server.rs index 89e62f68c..212096007 100644 --- a/lib/monitor_client/src/server.rs +++ b/lib/monitor_client/src/server.rs @@ -1,7 +1,7 @@ use anyhow::Context; use monitor_types::{ BasicContainerInfo, ImageSummary, Log, Network, Server, ServerActionState, ServerWithStatus, - SystemStats, + SystemStats, SystemStatsQuery, }; use serde_json::{json, Value}; @@ -78,13 +78,14 @@ impl MonitorClient { .context("failed at update server") } - pub async fn get_server_stats(&self, server_id: &str) -> anyhow::Result { - self.get( - &format!("/api/server/{server_id}/stats"), - Option::<()>::None, - ) - .await - .context(format!("failed to get server stats at id {server_id}")) + pub async fn get_server_stats( + &self, + server_id: &str, + query: impl Into>, + ) -> anyhow::Result { + self.get(&format!("/api/server/{server_id}/stats"), query.into()) + .await + .context(format!("failed to get server stats at id {server_id}")) } pub async fn get_docker_networks(&self, server_id: &str) -> anyhow::Result> { diff --git a/lib/periphery_client/src/git.rs b/lib/periphery_client/src/git.rs index 2103f0af6..9fac1432e 100644 --- a/lib/periphery_client/src/git.rs +++ b/lib/periphery_client/src/git.rs @@ -1,7 +1,7 @@ use anyhow::Context; use helpers::git::CloneArgs; use serde_json::json; -use types::{Log, Server}; +use types::{Command, Log, Server}; use crate::PeripheryClient; @@ -22,11 +22,12 @@ impl PeripheryClient { server: &Server, name: &str, branch: &Option, - ) -> anyhow::Result { + on_pull: &Option, + ) -> anyhow::Result> { self.post_json( server, "/git/pull", - &json!({ "name": name, "branch": branch }), + &json!({ "name": name, "branch": branch, "on_pull": on_pull }), ) .await .context("failed to pull repo on periphery") diff --git a/lib/periphery_client/src/lib.rs b/lib/periphery_client/src/lib.rs index ac98ba2e9..833b91a6b 100644 --- a/lib/periphery_client/src/lib.rs +++ b/lib/periphery_client/src/lib.rs @@ -1,7 +1,7 @@ use anyhow::{anyhow, Context}; use reqwest::StatusCode; use serde::{de::DeserializeOwned, Serialize}; -use types::{Server, SystemStats}; +use types::{Server, SystemStats, SystemStatsQuery}; mod build; mod container; @@ -33,10 +33,20 @@ impl PeripheryClient { .context("failed to get docker accounts from periphery") } - pub async fn get_system_stats(&self, server: &Server) -> anyhow::Result { - self.get_json(server, "/stats/system") - .await - .context("failed to get system stats from periphery") + pub async fn get_system_stats( + &self, + server: &Server, + query: &SystemStatsQuery, + ) -> anyhow::Result { + self.get_json( + server, + &format!( + "/stats/system?networks={}&components={}&processes={}", + query.networks, query.components, query.processes + ), + ) + .await + .context("failed to get system stats from periphery") } async fn get_text(&self, server: &Server, endpoint: &str) -> anyhow::Result { diff --git a/lib/types/src/deployment.rs b/lib/types/src/deployment.rs index 877723d11..84796b1b2 100644 --- a/lib/types/src/deployment.rs +++ b/lib/types/src/deployment.rs @@ -60,6 +60,14 @@ pub struct Deployment { #[diff(attr(#[serde(skip_serializing_if = "option_diff_no_change")]))] pub on_clone: Option, + #[serde(skip_serializing_if = "Option::is_none")] + #[diff(attr(#[serde(skip_serializing_if = "option_diff_no_change")]))] + pub on_pull: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + #[diff(attr(#[serde(skip_serializing_if = "option_diff_no_change")]))] + pub repo_mount: Option, + #[serde(default)] #[diff(attr(#[serde(skip)]))] #[builder(setter(skip))] diff --git a/lib/types/src/server.rs b/lib/types/src/server.rs index 6d1c96075..0e20d5b02 100644 --- a/lib/types/src/server.rs +++ b/lib/types/src/server.rs @@ -128,6 +128,27 @@ pub enum ServerStatus { Disabled, } +#[typeshare] +#[derive(Serialize, Deserialize, Debug, Clone, Default)] +pub struct SystemStatsQuery { + #[serde(default)] + pub networks: bool, + #[serde(default)] + pub components: bool, + #[serde(default)] + pub processes: bool, +} + +impl SystemStatsQuery { + pub fn all() -> SystemStatsQuery { + SystemStatsQuery { + networks: true, + components: true, + processes: true, + } + } +} + #[typeshare] #[derive(Serialize, Deserialize, Debug)] pub struct SystemStats { @@ -184,4 +205,11 @@ pub struct SystemComponent { pub struct SystemProcess { pub pid: u32, pub name: String, + #[serde(default, skip_serializing_if = "String::is_empty")] + pub exe: String, + pub cmd: Vec, + pub cpu_perc: f32, + pub mem_mb: f64, + pub disk_read_kb: f64, + pub disk_write_kb: f64, } diff --git a/periphery/Cargo.toml b/periphery/Cargo.toml index 3cfdd62fc..f370244c1 100644 --- a/periphery/Cargo.toml +++ b/periphery/Cargo.toml @@ -24,7 +24,7 @@ serde_json = "1.0" bollard = "0.13" anyhow = "1.0" envy = "0.4" -sysinfo = "0.27.0" +sysinfo = "0.27.2" toml = "0.5" daemonize = "0.4" clap = { version = "4.0", features = ["derive"] } diff --git a/periphery/src/api/git.rs b/periphery/src/api/git.rs index 055717608..8f1125d20 100644 --- a/periphery/src/api/git.rs +++ b/periphery/src/api/git.rs @@ -6,7 +6,7 @@ use helpers::{ handle_anyhow_error, to_monitor_name, }; use serde::Deserialize; -use types::Log; +use types::{Command, Log}; use crate::{helpers::get_github_token, PeripheryConfigExtension}; @@ -19,6 +19,7 @@ pub struct DeleteRepoBody { pub struct PullBody { name: String, branch: Option, + on_pull: Option, } pub fn router() -> Router { @@ -73,12 +74,16 @@ async fn delete_repo( async fn pull_repo( Extension(config): PeripheryConfigExtension, - Json(PullBody { name, branch }): Json, -) -> anyhow::Result> { + Json(PullBody { + name, + branch, + on_pull, + }): Json, +) -> anyhow::Result>> { let mut repo_dir = PathBuf::from_str(&config.repo_dir)?; let name = to_monitor_name(&name); repo_dir.push(&name); let path = repo_dir.display().to_string(); - let log = git::pull(&path, &branch).await; - Ok(Json(log)) + let logs = git::pull(&path, &branch, &on_pull).await; + Ok(Json(logs)) } diff --git a/periphery/src/api/stats.rs b/periphery/src/api/stats.rs index 123012f88..f3a3aa46e 100644 --- a/periphery/src/api/stats.rs +++ b/periphery/src/api/stats.rs @@ -1,23 +1,23 @@ use std::sync::{Arc, RwLock}; use async_timing_util::wait_until_timelength; -use axum::{routing::get, Extension, Json, Router}; -use sysinfo::{ - ComponentExt, CpuExt, DiskExt, NetworkExt, ProcessExt, ProcessRefreshKind, SystemExt, PidExt, -}; +use axum::{extract::Query, routing::get, Extension, Json, Router}; +use sysinfo::{ComponentExt, CpuExt, DiskExt, NetworkExt, PidExt, ProcessExt, SystemExt}; use types::{ DiskUsage, SingleDiskUsage, SystemComponent, SystemNetwork, SystemProcess, SystemStats, - Timelength, + SystemStatsQuery, Timelength, }; pub fn router(stats_polling_rate: Timelength) -> Router { Router::new() .route( "/system", - get(|Extension(sys): StatsExtension| async move { - let stats = sys.read().unwrap().get_stats(); - Json(stats) - }), + get( + |Extension(sys): StatsExtension, Query(query): Query| async move { + let stats = sys.read().unwrap().get_stats(query); + Json(stats) + }, + ), ) .layer(StatsClient::extension(stats_polling_rate)) } @@ -32,6 +32,7 @@ struct StatsClient { } const BYTES_PER_GB: f64 = 1073741824.0; +const BYTES_PER_MB: f64 = 1048576.0; const BYTES_PER_KB: f64 = 1024.0; impl StatsClient { @@ -80,15 +81,27 @@ impl StatsClient { self.sys.refresh_components_list(); } - pub fn get_stats(&self) -> SystemStats { + pub fn get_stats(&self, query: SystemStatsQuery) -> SystemStats { SystemStats { cpu_perc: self.sys.global_cpu_info().cpu_usage(), mem_used_gb: self.sys.used_memory() as f64 / BYTES_PER_GB, mem_total_gb: self.sys.total_memory() as f64 / BYTES_PER_GB, disk: self.get_disk_usage(), - networks: self.get_networks(), - components: self.get_components(), - processes: self.get_processes(), + networks: if query.networks { + self.get_networks() + } else { + vec![] + }, + components: if query.components { + self.get_components() + } else { + vec![] + }, + processes: if query.processes { + self.get_processes() + } else { + vec![] + }, polling_rate: self.polling_rate, refresh_ts: self.refresh_ts, refresh_list_ts: self.refresh_list_ts, @@ -161,10 +174,18 @@ impl StatsClient { self.sys .processes() .into_iter() - .map(|(pid, p)| SystemProcess { - pid: pid.as_u32(), - name: p.name().to_string(), - + .map(|(pid, p)| { + let disk_usage = p.disk_usage(); + SystemProcess { + pid: pid.as_u32(), + name: p.name().to_string(), + exe: p.exe().to_str().unwrap_or("").to_string(), + cmd: p.cmd().to_vec(), + cpu_perc: p.cpu_usage(), + mem_mb: p.memory() as f64 / BYTES_PER_MB, + disk_read_kb: disk_usage.read_bytes as f64 / BYTES_PER_KB, + disk_write_kb: disk_usage.written_bytes as f64 / BYTES_PER_KB, + } }) .collect() } diff --git a/tests/src/tests.rs b/tests/src/tests.rs index c882dbcb0..e21c678a4 100644 --- a/tests/src/tests.rs +++ b/tests/src/tests.rs @@ -48,7 +48,7 @@ pub async fn get_server_stats(monitor: &MonitorClient) -> anyhow::Result