ComputerSurge/komodo
3
0
Fork 0
forked from github-starred/komodo
Code Pull Requests Packages Releases Activity

a bunch of stuff

Browse Source
This commit is contained in:
mbecker20
2022-12-31 05:42:44 +00:00
parent 5d0c14201d
commit 0ceecee604
25 changed files with 247 additions and 93 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
Cargo.lock generated
View File

@@ -2249,9 +2249,9 @@ checksum = "20518fe4a4c9acf048008599e464deb21beeae3d3578418951a189c235a7a9a8"
[[package]]
name = "sysinfo"
version = "0.27.0"
version = "0.27.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0d08ba83d6dde63d053e42d7230f0dc7f8d8efeb8d30d3681580d158156461ba"
checksum = "17351d0e9eb8841897b14e9669378f3c69fb57779cc04f8ca9a9d512edfb2563"
dependencies = [
"cfg-if",
"core-foundation-sys",

4
core/src/actions/build.rs
View File

@@ -1,6 +1,6 @@
use anyhow::{anyhow, Context};
use diff::Diff;
use helpers::to_monitor_name;
use helpers::{all_logs_success, to_monitor_name};
use mungos::{doc, to_bson};
use types::{
monitor_timestamp,
@@ -10,7 +10,7 @@ use types::{
use crate::{
auth::RequestUser,
helpers::{all_logs_success, any_option_diff_is_some, option_diff_is_some},
helpers::{any_option_diff_is_some, option_diff_is_some},
state::State,
};

17
core/src/actions/deployment.rs
View File

@@ -1,6 +1,6 @@
use anyhow::{anyhow, Context};
use diff::Diff;
use helpers::to_monitor_name;
use helpers::{all_logs_success, to_monitor_name};
use types::{
monitor_timestamp,
traits::{Busy, Permissioned},
@@ -9,7 +9,7 @@ use types::{
use crate::{
auth::RequestUser,
helpers::{all_logs_success, any_option_diff_is_some, option_diff_is_some},
helpers::{any_option_diff_is_some, option_diff_is_some},
state::State,
};
@@ -633,13 +633,18 @@ impl State {
update.id = self.add_update(update.clone()).await?;
let log = self
let logs = self
.periphery
.pull_repo(&server, &deployment.name, &deployment.branch)
.pull_repo(
&server,
&deployment.name,
&deployment.branch,
&deployment.on_pull,
)
.await?;
update.success = log.success;
update.logs.push(log);
update.success = all_logs_success(&logs);
update.logs.extend(logs);
update.end_ts = Some(monitor_timestamp());
update.status = UpdateStatus::Complete;

10
core/src/api/server.rs
View File

@@ -9,7 +9,7 @@ use helpers::handle_anyhow_error;
use mungos::{Deserialize, Document, Serialize};
use types::{
traits::Permissioned, BasicContainerInfo, ImageSummary, Network, PermissionLevel, Server,
ServerActionState, ServerStatus, ServerWithStatus, SystemStats,
ServerActionState, ServerStatus, ServerWithStatus, SystemStats, SystemStatsQuery,
};
use typeshare::typeshare;
@@ -122,9 +122,10 @@ pub fn router() -> Router {
get(
|Extension(state): StateExtension,
Extension(user): RequestUserExtension,
Path(ServerId { id }): Path<ServerId>| async move {
Path(ServerId { id }): Path<ServerId>,
Query(query): Query<SystemStatsQuery>| async move {
let stats = state
.get_server_stats(&id, &user)
.get_server_stats(&id, &user, &query)
.await
.map_err(handle_anyhow_error)?;
response!(Json(stats))
@@ -323,13 +324,14 @@ impl State {
&self,
server_id: &str,
user: &RequestUser,
query: &SystemStatsQuery,
) -> anyhow::Result<SystemStats> {
let server = self
.get_server_check_permissions(server_id, user, PermissionLevel::Read)
.await?;
let stats = self
.periphery
.get_system_stats(&server)
.get_system_stats(&server, query)
.await
.context(format!("failed to get stats from server {}", server.name))?;
Ok(stats)

2
core/src/config.rs
View File

@@ -16,7 +16,7 @@ pub fn load() -> (CoreConfig, SpaRouter) {
dotenv().ok();
let env: Env = envy::from_env().expect("failed to parse environment variables");
let config = parse_config_file(&env.config_path).expect("failed to parse config");
let spa_router = SpaRouter::new("/", env.frontend_path);
let spa_router = SpaRouter::new("/assets", env.frontend_path);
(config, spa_router)
}

10
core/src/helpers.rs
View File

@@ -1,5 +1,4 @@
use diff::{Diff, OptionDiff};
use types::Log;
#[macro_export]
macro_rules! response {
@@ -26,12 +25,3 @@ where
}
return false;
}
pub fn all_logs_success(logs: &Vec<Log>) -> bool {
for log in logs {
if !log.success {
return false;
}
}
true
}

10
frontend/index.html
View File

@@ -6,11 +6,11 @@
<meta name="viewport" content="width=device-width, initial-scale=1" />
<meta name="theme-color" content="#000000" />
<link rel="shortcut icon" type="image/ico" href="/favicon.ico" />
<link rel="apple-touch-icon" sizes="180x180" href="/apple-touch-icon.png" />
<link rel="icon" type="image/png" sizes="32x32" href="/favicon-32x32.png">
<link rel="icon" type="image/png" sizes="16x16" href="/favicon-16x16.png">
<link rel="manifest" href="/manifest.json" />
<link rel="shortcut icon" type="image/ico" href="/assets/favicon.ico" />
<link rel="apple-touch-icon" sizes="180x180" href="/assets/apple-touch-icon.png" />
<link rel="icon" type="image/png" sizes="32x32" href="/assets/favicon-32x32.png">
<link rel="icon" type="image/png" sizes="16x16" href="/assets/favicon-16x16.png">
<link rel="manifest" href="/assets/manifest.json" />
<title>monitor</title>
</head>

2
frontend/package.json
View File

@@ -5,7 +5,7 @@
"scripts": {
"start": "vite",
"dev": "vite",
"build": "vite build",
"build": "vite build && node post-build.mjs",
"serve": "vite preview"
},
"license": "GPL v3.0",

11
frontend/post-build.mjs Normal file
View File

@@ -0,0 +1,11 @@
import { readdirSync, renameSync, rmdirSync } from "fs";
const files = readdirSync("./build/assets");
for (const file of files) {
renameSync("./build/assets/" + file, "./build/" + file);
}
rmdirSync("./build/assets");
console.log("\npost build complete\n")

2
frontend/src/components/shared/Icon.tsx
View File

@@ -59,7 +59,7 @@ const Icon: Component<{
return (
<img
class={p.class}
src={`/icons/${p.type}.svg`}
src={`/assets/icons/${p.type}.svg`}
alt={p.alt || p.type}
title={p.title}
style={{

27
frontend/src/components/topbar/Search/Search.tsx
View File

@@ -16,7 +16,7 @@ import Circle from "../../shared/Circle";
import { ControlledTabs } from "../../shared/tabs/Tabs";
import { useAppDimensions } from "../../../state/DimensionProvider";
import Grid from "../../shared/layout/Grid";
import { useNavigate } from "@solidjs/router";
import { A, useNavigate } from "@solidjs/router";
import { ServerStatus } from "../../../types";
const mobileStyle: JSX.CSSProperties = {
@@ -127,11 +127,14 @@ const Deployments: Component<{ close: () => void }> = (p) => {
style={{ "max-height": "70vh", "padding-right": "0.5rem" }}
>
<Show when={filteredDeployments()?.length === 0}>
<Flex alignItems="center" justifyContent="center">no results</Flex>
<Flex alignItems="center" justifyContent="center">
no results
</Flex>
</Show>
<For each={filteredDeployments()}>
{(deployment, index) => (
<button
<A
href={`/deployment/${getId(deployment.deployment)}`}
class={combineClasses(
s.SearchItem,
index() === highlighted.value() && "selected",
@@ -160,7 +163,7 @@ const Deployments: Component<{ close: () => void }> = (p) => {
)}
size={1.25}
/>
</button>
</A>
)}
</For>
</Grid>
@@ -183,14 +186,15 @@ const Builds: Component<{ close: () => void }> = (p) => {
</Show>
<For each={filteredBuilds()}>
{(build, index) => (
<button
<A
href={`/build/${getId(build)}`}
class={combineClasses(
s.SearchItem,
index() === highlighted.value() && "selected",
"grey"
)}
onClick={() => {
navigate(`/build/${getId(build)}`);
// navigate(`/build/${getId(build)}`);
p.close();
}}
>
@@ -198,7 +202,7 @@ const Builds: Component<{ close: () => void }> = (p) => {
{build.name}
<Flex style={{ opacity: 0.6, "font-size": "0.9rem" }}>build</Flex>
</Grid>
</button>
</A>
)}
</For>
</Grid>
@@ -206,7 +210,7 @@ const Builds: Component<{ close: () => void }> = (p) => {
};
const Servers: Component<{ close: () => void }> = (p) => {
const navigate = useNavigate();
// const navigate = useNavigate();
const { highlighted, filteredServers } = useSearchState();
return (
<Grid
@@ -221,14 +225,15 @@ const Servers: Component<{ close: () => void }> = (p) => {
</Show>
<For each={filteredServers()}>
{(server, index) => (
<button
<A
href={`/server/${getId(server.server)}`}
class={combineClasses(
s.SearchItem,
index() === highlighted.value() && "selected",
"grey"
)}
onClick={() => {
navigate(`/server/${getId(server.server)}`);
// navigate(`/server/${getId(server.server)}`);
p.close();
}}
>
@@ -263,7 +268,7 @@ const Servers: Component<{ close: () => void }> = (p) => {
>
{server.status.replaceAll("_", " ").toUpperCase()}
</div>
</button>
</A>
)}
</For>
</Grid>

6
frontend/src/components/topbar/Topbar.tsx
View File

@@ -1,4 +1,4 @@
import { useNavigate } from "@solidjs/router";
import { A, useNavigate } from "@solidjs/router";
import { Component, createSignal, JSX, Show } from "solid-js";
import { TOPBAR_HEIGHT } from "../..";
import { useAppDimensions } from "../../state/DimensionProvider";
@@ -47,9 +47,9 @@ const LeftSide: Component = () => {
alignItems="center"
style={{ padding: "0rem 0.5rem", "place-self": "center start" }}
>
<button class="grey" onClick={() => navigate("/")}>
<A href="/" class="grey">
<Icon type="home" width="1.15rem" />
</button>
</A>
<HoverMenu
target={
<Circle

24
frontend/src/style/index.scss
View File

@@ -40,6 +40,30 @@ button:focus {
outline: none;
}
a {
color: inherit;
padding: 0.5rem;
margin: 0;
cursor: pointer;
border: none;
width: fit-content;
border-radius: 0.25rem;
font-family: inherit;
font-size: 0.9rem;
font-weight: 500;
display: flex;
gap: 0.5rem;
justify-content: center;
align-items: center;
background-color: transparent;
transition: background-color 500ms ease-in-out;
text-decoration: none;
}
a:focus {
outline: none;
}
input {
font-family: inherit;
font-size: 1.05rem;

11
lib/helpers/src/docker/build.rs
View File

@@ -3,7 +3,7 @@ use std::{path::PathBuf, str::FromStr};
use anyhow::{anyhow, Context};
use types::{Build, DockerBuildArgs, EnvironmentVar, Log, Version};
use crate::{git, run_monitor_command, to_monitor_name};
use crate::{all_logs_success, git, run_monitor_command, to_monitor_name};
use super::docker_login;
@@ -40,18 +40,19 @@ pub async fn build(
let repo_dir = PathBuf::from_str(repo_dir)
.context(format!("invalid repo dir: {repo_dir}"))?
.join(&name);
let pull_log = git::pull(
let pull_logs = git::pull(
&repo_dir
.to_str()
.context(format!("invalid repo dir: {}", repo_dir.display()))?,
branch,
&None,
)
.await;
if !pull_log.success {
logs.push(pull_log);
if !all_logs_success(&pull_logs) {
logs.extend(pull_logs);
return Ok(logs);
}
logs.push(pull_log);
logs.extend(pull_logs);
if let Some(command) = pre_build {
let mut repo_dir = repo_dir.clone();
repo_dir.push(&command.path);

37
lib/helpers/src/git.rs
View File

@@ -13,6 +13,7 @@ pub struct CloneArgs {
repo: Option<String>,
branch: Option<String>,
on_clone: Option<Command>,
on_pull: Option<Command>,
pub github_account: Option<GithubUsername>,
}
@@ -23,6 +24,7 @@ impl From<&Deployment> for CloneArgs {
repo: d.repo.clone(),
branch: d.branch.clone(),
on_clone: d.on_clone.clone(),
on_pull: d.on_pull.clone(),
github_account: d.github_account.clone(),
}
}
@@ -35,18 +37,37 @@ impl From<&Build> for CloneArgs {
repo: b.repo.clone(),
branch: b.branch.clone(),
on_clone: b.on_clone.clone(),
on_pull: None,
github_account: b.github_account.clone(),
}
}
}
pub async fn pull(path: &str, branch: &Option<String>) -> Log {
pub async fn pull(path: &str, branch: &Option<String>, on_pull: &Option<Command>) -> Vec<Log> {
let branch = match branch {
Some(branch) => branch.to_owned(),
None => "main".to_string(),
};
let command = format!("cd {path} && git pull origin {branch}");
run_monitor_command("git pull", command).await
let mut logs = Vec::new();
let pull_log = run_monitor_command("git pull", command).await;
if !pull_log.success {
logs.push(pull_log);
return logs;
}
logs.push(pull_log);
if let Some(on_pull) = on_pull {
let mut path = PathBuf::from_str(path).unwrap();
path.push(&on_pull.path);
let path = path.display().to_string();
let on_pull_log = run_monitor_command(
"on pull command",
format!("cd {path} && {}", on_pull.command),
)
.await;
logs.push(on_pull_log);
}
logs
}
pub async fn clone_repo(
@@ -59,6 +80,7 @@ pub async fn clone_repo(
repo,
branch,
on_clone,
on_pull,
..
} = clone_args.into();
let repo = repo.as_ref().ok_or(anyhow!("build has no repo attached"))?;
@@ -76,6 +98,17 @@ pub async fn clone_repo(
)
.await;
logs.push(on_clone_log);
repo_dir.pop();
}
if let Some(command) = on_pull {
repo_dir.push(&command.path);
let on_clone_log = run_monitor_command(
"on clone",
format!("cd {} && {}", repo_dir.display(), command.command),
)
.await;
logs.push(on_clone_log);
repo_dir.pop();
}
Ok(logs)
}

9
lib/helpers/src/lib.rs
View File

@@ -71,3 +71,12 @@ pub fn generate_secret(length: usize) -> String {
.map(char::from)
.collect()
}
pub fn all_logs_success(logs: &Vec<Log>) -> bool {
for log in logs {
if !log.success {
return false;
}
}
true
}

17
lib/monitor_client/src/server.rs
View File

@@ -1,7 +1,7 @@
use anyhow::Context;
use monitor_types::{
BasicContainerInfo, ImageSummary, Log, Network, Server, ServerActionState, ServerWithStatus,
SystemStats,
SystemStats, SystemStatsQuery,
};
use serde_json::{json, Value};
@@ -78,13 +78,14 @@ impl MonitorClient {
.context("failed at update server")
}
pub async fn get_server_stats(&self, server_id: &str) -> anyhow::Result<SystemStats> {
self.get(
&format!("/api/server/{server_id}/stats"),
Option::<()>::None,
)
.await
.context(format!("failed to get server stats at id {server_id}"))
pub async fn get_server_stats(
&self,
server_id: &str,
query: impl Into<Option<&SystemStatsQuery>>,
) -> anyhow::Result<SystemStats> {
self.get(&format!("/api/server/{server_id}/stats"), query.into())
.await
.context(format!("failed to get server stats at id {server_id}"))
}
pub async fn get_docker_networks(&self, server_id: &str) -> anyhow::Result<Vec<Network>> {

7
lib/periphery_client/src/git.rs
View File

@@ -1,7 +1,7 @@
use anyhow::Context;
use helpers::git::CloneArgs;
use serde_json::json;
use types::{Log, Server};
use types::{Command, Log, Server};
use crate::PeripheryClient;
@@ -22,11 +22,12 @@ impl PeripheryClient {
server: &Server,
name: &str,
branch: &Option<String>,
) -> anyhow::Result<Log> {
on_pull: &Option<Command>,
) -> anyhow::Result<Vec<Log>> {
self.post_json(
server,
"/git/pull",
&json!({ "name": name, "branch": branch }),
&json!({ "name": name, "branch": branch, "on_pull": on_pull }),
)
.await
.context("failed to pull repo on periphery")

20
lib/periphery_client/src/lib.rs
View File

@@ -1,7 +1,7 @@
use anyhow::{anyhow, Context};
use reqwest::StatusCode;
use serde::{de::DeserializeOwned, Serialize};
use types::{Server, SystemStats};
use types::{Server, SystemStats, SystemStatsQuery};
mod build;
mod container;
@@ -33,10 +33,20 @@ impl PeripheryClient {
.context("failed to get docker accounts from periphery")
}
pub async fn get_system_stats(&self, server: &Server) -> anyhow::Result<SystemStats> {
self.get_json(server, "/stats/system")
.await
.context("failed to get system stats from periphery")
pub async fn get_system_stats(
&self,
server: &Server,
query: &SystemStatsQuery,
) -> anyhow::Result<SystemStats> {
self.get_json(
server,
&format!(
"/stats/system?networks={}&components={}&processes={}",
query.networks, query.components, query.processes
),
)
.await
.context("failed to get system stats from periphery")
}
async fn get_text(&self, server: &Server, endpoint: &str) -> anyhow::Result<String> {

8
lib/types/src/deployment.rs
View File

@@ -60,6 +60,14 @@ pub struct Deployment {
#[diff(attr(#[serde(skip_serializing_if = "option_diff_no_change")]))]
pub on_clone: Option<Command>,
#[serde(skip_serializing_if = "Option::is_none")]
#[diff(attr(#[serde(skip_serializing_if = "option_diff_no_change")]))]
pub on_pull: Option<Command>,
#[serde(skip_serializing_if = "Option::is_none")]
#[diff(attr(#[serde(skip_serializing_if = "option_diff_no_change")]))]
pub repo_mount: Option<Conversion>,
#[serde(default)]
#[diff(attr(#[serde(skip)]))]
#[builder(setter(skip))]

28
lib/types/src/server.rs
View File

@@ -128,6 +128,27 @@ pub enum ServerStatus {
Disabled,
}
#[typeshare]
#[derive(Serialize, Deserialize, Debug, Clone, Default)]
pub struct SystemStatsQuery {
#[serde(default)]
pub networks: bool,
#[serde(default)]
pub components: bool,
#[serde(default)]
pub processes: bool,
}
impl SystemStatsQuery {
pub fn all() -> SystemStatsQuery {
SystemStatsQuery {
networks: true,
components: true,
processes: true,
}
}
}
#[typeshare]
#[derive(Serialize, Deserialize, Debug)]
pub struct SystemStats {
@@ -184,4 +205,11 @@ pub struct SystemComponent {
pub struct SystemProcess {
pub pid: u32,
pub name: String,
#[serde(default, skip_serializing_if = "String::is_empty")]
pub exe: String,
pub cmd: Vec<String>,
pub cpu_perc: f32,
pub mem_mb: f64,
pub disk_read_kb: f64,
pub disk_write_kb: f64,
}

2
periphery/Cargo.toml
View File

@@ -24,7 +24,7 @@ serde_json = "1.0"
bollard = "0.13"
anyhow = "1.0"
envy = "0.4"
sysinfo = "0.27.0"
sysinfo = "0.27.2"
toml = "0.5"
daemonize = "0.4"
clap = { version = "4.0", features = ["derive"] }

15
periphery/src/api/git.rs
View File

@@ -6,7 +6,7 @@ use helpers::{
handle_anyhow_error, to_monitor_name,
};
use serde::Deserialize;
use types::Log;
use types::{Command, Log};
use crate::{helpers::get_github_token, PeripheryConfigExtension};
@@ -19,6 +19,7 @@ pub struct DeleteRepoBody {
pub struct PullBody {
name: String,
branch: Option<String>,
on_pull: Option<Command>,
}
pub fn router() -> Router {
@@ -73,12 +74,16 @@ async fn delete_repo(
async fn pull_repo(
Extension(config): PeripheryConfigExtension,
Json(PullBody { name, branch }): Json<PullBody>,
) -> anyhow::Result<Json<Log>> {
Json(PullBody {
name,
branch,
on_pull,
}): Json<PullBody>,
) -> anyhow::Result<Json<Vec<Log>>> {
let mut repo_dir = PathBuf::from_str(&config.repo_dir)?;
let name = to_monitor_name(&name);
repo_dir.push(&name);
let path = repo_dir.display().to_string();
let log = git::pull(&path, &branch).await;
Ok(Json(log))
let logs = git::pull(&path, &branch, &on_pull).await;
Ok(Json(logs))
}

55
periphery/src/api/stats.rs
View File

@@ -1,23 +1,23 @@
use std::sync::{Arc, RwLock};
use async_timing_util::wait_until_timelength;
use axum::{routing::get, Extension, Json, Router};
use sysinfo::{
ComponentExt, CpuExt, DiskExt, NetworkExt, ProcessExt, ProcessRefreshKind, SystemExt, PidExt,
};
use axum::{extract::Query, routing::get, Extension, Json, Router};
use sysinfo::{ComponentExt, CpuExt, DiskExt, NetworkExt, PidExt, ProcessExt, SystemExt};
use types::{
DiskUsage, SingleDiskUsage, SystemComponent, SystemNetwork, SystemProcess, SystemStats,
Timelength,
SystemStatsQuery, Timelength,
};
pub fn router(stats_polling_rate: Timelength) -> Router {
Router::new()
.route(
"/system",
get(|Extension(sys): StatsExtension| async move {
let stats = sys.read().unwrap().get_stats();
Json(stats)
}),
get(
|Extension(sys): StatsExtension, Query(query): Query<SystemStatsQuery>| async move {
let stats = sys.read().unwrap().get_stats(query);
Json(stats)
},
),
)
.layer(StatsClient::extension(stats_polling_rate))
}
@@ -32,6 +32,7 @@ struct StatsClient {
}
const BYTES_PER_GB: f64 = 1073741824.0;
const BYTES_PER_MB: f64 = 1048576.0;
const BYTES_PER_KB: f64 = 1024.0;
impl StatsClient {
@@ -80,15 +81,27 @@ impl StatsClient {
self.sys.refresh_components_list();
}
pub fn get_stats(&self) -> SystemStats {
pub fn get_stats(&self, query: SystemStatsQuery) -> SystemStats {
SystemStats {
cpu_perc: self.sys.global_cpu_info().cpu_usage(),
mem_used_gb: self.sys.used_memory() as f64 / BYTES_PER_GB,
mem_total_gb: self.sys.total_memory() as f64 / BYTES_PER_GB,
disk: self.get_disk_usage(),
networks: self.get_networks(),
components: self.get_components(),
processes: self.get_processes(),
networks: if query.networks {
self.get_networks()
} else {
vec![]
},
components: if query.components {
self.get_components()
} else {
vec![]
},
processes: if query.processes {
self.get_processes()
} else {
vec![]
},
polling_rate: self.polling_rate,
refresh_ts: self.refresh_ts,
refresh_list_ts: self.refresh_list_ts,
@@ -161,10 +174,18 @@ impl StatsClient {
self.sys
.processes()
.into_iter()
.map(|(pid, p)| SystemProcess {
pid: pid.as_u32(),
name: p.name().to_string(),
.map(|(pid, p)| {
let disk_usage = p.disk_usage();
SystemProcess {
pid: pid.as_u32(),
name: p.name().to_string(),
exe: p.exe().to_str().unwrap_or("").to_string(),
cmd: p.cmd().to_vec(),
cpu_perc: p.cpu_usage(),
mem_mb: p.memory() as f64 / BYTES_PER_MB,
disk_read_kb: disk_usage.read_bytes as f64 / BYTES_PER_KB,
disk_write_kb: disk_usage.written_bytes as f64 / BYTES_PER_KB,
}
})
.collect()
}

2
tests/src/tests.rs
View File

@@ -48,7 +48,7 @@ pub async fn get_server_stats(monitor: &MonitorClient) -> anyhow::Result<SystemS
.context("failed at list servers")?;
let server = &servers.get(0).ok_or(anyhow!("no servers"))?.server;
let stats = monitor
.get_server_stats(&server.id)
.get_server_stats(&server.id, None)
.await
.context("failed at get server stats")?;
Ok(stats)