Compare commits

..

10 Commits

Author SHA1 Message Date
mbecker20
e385c6e722 use ferretdb:1 2025-02-26 14:55:34 -08:00
Maxwell Becker
9ef25e7575 Create FUNDING.yml 2025-02-13 12:07:48 -08:00
boomam
f945a3014a Update index.mdx (#306)
Added small note on initial login steps.
2025-02-11 00:03:39 -08:00
mbecker20
fdad04d6cb fix KOMODO_DB_USERNAME compose files 2025-02-08 18:45:02 -08:00
mbecker20
c914f23aa8 update compose files re #180 2025-02-08 12:29:26 -08:00
Maarten Kossen
82b2e68cd3 Adding Resource Sync documentation. (#259) 2025-01-11 21:32:02 -08:00
rita7lopes
e274d6f7c8 Network Usage - Ingress Egress per interface and global usage (#229)
* Add network io stats

Add network usage graph and current status

Change network graphs to use network interface from drop down menu

Adjust the type to be able to get general network stats for the general
UI view

Working setup with a working builder

remove changes to these dockerfile

remove lock changes

* change network hashmap to Vector

* adjust all the network_usage_interface to be a vector rather than a hash map

* PR requested changes applied

* Change net_ingress_bytes and egress to network_ingress_bytes egress respectively

* final gen-client types

---------

Co-authored-by: mbecker20 <becker.maxh@gmail.com>
2024-12-21 08:15:21 -08:00
Maxwell Becker
ab8777460d Simplify periphery aio.Dockerfile 2024-12-14 09:13:01 -08:00
Maxwell Becker
7e030e702f Simplify aio.Dockerfile 2024-12-14 09:12:32 -08:00
Maxwell Becker
a869a74002 Fix test.compose.yaml (use aio.Dockerfiles) 2024-12-14 09:11:24 -08:00
20 changed files with 347 additions and 158 deletions

2
.github/FUNDING.yml vendored Normal file
View File

@@ -0,0 +1,2 @@
# https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/displaying-a-sponsor-button-in-your-repository
open_collective: komodo

View File

@@ -8,13 +8,6 @@ COPY Cargo.toml Cargo.lock ./
COPY ./lib ./lib COPY ./lib ./lib
COPY ./client/core/rs ./client/core/rs COPY ./client/core/rs ./client/core/rs
COPY ./client/periphery ./client/periphery COPY ./client/periphery ./client/periphery
# Pre compile dependencies
COPY ./bin/core/Cargo.toml ./bin/core/Cargo.toml
RUN mkdir ./bin/core/src && \
echo "fn main() {}" >> ./bin/core/src/main.rs && \
cargo build -p komodo_core --release && \
rm -r ./bin/core
COPY ./bin/core ./bin/core COPY ./bin/core ./bin/core
# Compile app # Compile app
@@ -59,4 +52,4 @@ LABEL org.opencontainers.image.source=https://github.com/mbecker20/komodo
LABEL org.opencontainers.image.description="Komodo Core" LABEL org.opencontainers.image.description="Komodo Core"
LABEL org.opencontainers.image.licenses=GPL-3.0 LABEL org.opencontainers.image.licenses=GPL-3.0
ENTRYPOINT [ "core" ] ENTRYPOINT [ "core" ]

View File

@@ -26,6 +26,9 @@ pub async fn record_server_stats(ts: i64) {
disk_total_gb, disk_total_gb,
disk_used_gb, disk_used_gb,
disks: stats.disks.clone(), disks: stats.disks.clone(),
network_ingress_bytes: stats.network_ingress_bytes,
network_egress_bytes: stats.network_egress_bytes,
network_usage_interface: stats.network_usage_interface.clone(),
}) })
}) })
.collect::<Vec<_>>(); .collect::<Vec<_>>();

View File

@@ -7,13 +7,6 @@ COPY Cargo.toml Cargo.lock ./
COPY ./lib ./lib COPY ./lib ./lib
COPY ./client/core/rs ./client/core/rs COPY ./client/core/rs ./client/core/rs
COPY ./client/periphery ./client/periphery COPY ./client/periphery ./client/periphery
# Pre compile dependencies
COPY ./bin/periphery/Cargo.toml ./bin/periphery/Cargo.toml
RUN mkdir ./bin/periphery/src && \
echo "fn main() {}" >> ./bin/periphery/src/main.rs && \
cargo build -p komodo_periphery --release && \
rm -r ./bin/periphery
COPY ./bin/periphery ./bin/periphery COPY ./bin/periphery ./bin/periphery
# Compile app # Compile app
@@ -33,4 +26,4 @@ LABEL org.opencontainers.image.source=https://github.com/mbecker20/komodo
LABEL org.opencontainers.image.description="Komodo Periphery" LABEL org.opencontainers.image.description="Komodo Periphery"
LABEL org.opencontainers.image.licenses=GPL-3.0 LABEL org.opencontainers.image.licenses=GPL-3.0
CMD [ "periphery" ] CMD [ "periphery" ]

View File

@@ -56,3 +56,4 @@ impl ResolveToString<GetSystemProcesses> for State {
.context("failed to serialize response to string") .context("failed to serialize response to string")
} }
} }

View File

@@ -2,7 +2,7 @@ use std::{cmp::Ordering, sync::OnceLock};
use async_timing_util::wait_until_timelength; use async_timing_util::wait_until_timelength;
use komodo_client::entities::stats::{ use komodo_client::entities::stats::{
SingleDiskUsage, SystemInformation, SystemProcess, SystemStats, SingleDiskUsage, SystemInformation, SystemProcess, SystemStats, SingleNetworkInterfaceUsage,
}; };
use sysinfo::{ProcessesToUpdate, System}; use sysinfo::{ProcessesToUpdate, System};
use tokio::sync::RwLock; use tokio::sync::RwLock;
@@ -48,6 +48,7 @@ pub fn spawn_system_stats_polling_threads() {
}); });
} }
pub struct StatsClient { pub struct StatsClient {
/// Cached system stats /// Cached system stats
pub stats: SystemStats, pub stats: SystemStats,
@@ -57,6 +58,7 @@ pub struct StatsClient {
// the handles used to get the stats // the handles used to get the stats
system: sysinfo::System, system: sysinfo::System,
disks: sysinfo::Disks, disks: sysinfo::Disks,
networks: sysinfo::Networks,
} }
const BYTES_PER_GB: f64 = 1073741824.0; const BYTES_PER_GB: f64 = 1073741824.0;
@@ -67,6 +69,7 @@ impl Default for StatsClient {
fn default() -> Self { fn default() -> Self {
let system = sysinfo::System::new_all(); let system = sysinfo::System::new_all();
let disks = sysinfo::Disks::new_with_refreshed_list(); let disks = sysinfo::Disks::new_with_refreshed_list();
let networks = sysinfo::Networks::new_with_refreshed_list();
let stats = SystemStats { let stats = SystemStats {
polling_rate: periphery_config().stats_polling_rate, polling_rate: periphery_config().stats_polling_rate,
..Default::default() ..Default::default()
@@ -75,6 +78,7 @@ impl Default for StatsClient {
info: get_system_information(&system), info: get_system_information(&system),
system, system,
disks, disks,
networks,
stats, stats,
} }
} }
@@ -86,20 +90,51 @@ impl StatsClient {
self.system.refresh_memory(); self.system.refresh_memory();
self.system.refresh_processes(ProcessesToUpdate::All, true); self.system.refresh_processes(ProcessesToUpdate::All, true);
self.disks.refresh(); self.disks.refresh();
self.networks.refresh();
} }
fn refresh_lists(&mut self) { fn refresh_lists(&mut self) {
self.disks.refresh_list(); self.disks.refresh_list();
self.networks.refresh_list();
} }
pub fn get_system_stats(&self) -> SystemStats { pub fn get_system_stats(&self) -> SystemStats {
let total_mem = self.system.total_memory(); let total_mem = self.system.total_memory();
let available_mem = self.system.available_memory(); let available_mem = self.system.available_memory();
let mut total_ingress: u64 = 0;
let mut total_egress: u64 = 0;
// Fetch network data (Ingress and Egress)
let network_usage: Vec<SingleNetworkInterfaceUsage> = self.networks
.iter()
.map(|(interface_name, network)| {
let ingress = network.received();
let egress = network.transmitted();
// Update total ingress and egress
total_ingress += ingress;
total_egress += egress;
// Return per-interface network stats
SingleNetworkInterfaceUsage {
name: interface_name.clone(),
ingress_bytes: ingress as f64,
egress_bytes: egress as f64,
}
})
.collect();
SystemStats { SystemStats {
cpu_perc: self.system.global_cpu_usage(), cpu_perc: self.system.global_cpu_usage(),
mem_free_gb: self.system.free_memory() as f64 / BYTES_PER_GB, mem_free_gb: self.system.free_memory() as f64 / BYTES_PER_GB,
mem_used_gb: (total_mem - available_mem) as f64 / BYTES_PER_GB, mem_used_gb: (total_mem - available_mem) as f64 / BYTES_PER_GB,
mem_total_gb: total_mem as f64 / BYTES_PER_GB, mem_total_gb: total_mem as f64 / BYTES_PER_GB,
// Added total ingress and egress
network_ingress_bytes: total_ingress as f64,
network_egress_bytes: total_egress as f64,
network_usage_interface: network_usage,
disks: self.get_disks(), disks: self.get_disks(),
polling_rate: self.stats.polling_rate, polling_rate: self.stats.polling_rate,
refresh_ts: self.stats.refresh_ts, refresh_ts: self.stats.refresh_ts,

View File

@@ -51,6 +51,15 @@ pub struct SystemStatsRecord {
pub disk_total_gb: f64, pub disk_total_gb: f64,
/// Breakdown of individual disks, ie their usages, sizes, and mount points /// Breakdown of individual disks, ie their usages, sizes, and mount points
pub disks: Vec<SingleDiskUsage>, pub disks: Vec<SingleDiskUsage>,
/// Network ingress usage in bytes
#[serde(default)]
pub network_ingress_bytes: f64,
/// Network egress usage in bytes
#[serde(default)]
pub network_egress_bytes: f64,
/// Network usage by interface name (ingress, egress in bytes)
#[serde(default)]
pub network_usage_interface: Vec<SingleNetworkInterfaceUsage>, // interface -> (ingress, egress)
} }
/// Realtime system stats data. /// Realtime system stats data.
@@ -71,7 +80,15 @@ pub struct SystemStats {
pub mem_total_gb: f64, pub mem_total_gb: f64,
/// Breakdown of individual disks, ie their usages, sizes, and mount points /// Breakdown of individual disks, ie their usages, sizes, and mount points
pub disks: Vec<SingleDiskUsage>, pub disks: Vec<SingleDiskUsage>,
/// Network ingress usage in MB
#[serde(default)]
pub network_ingress_bytes: f64,
/// Network egress usage in MB
#[serde(default)]
pub network_egress_bytes: f64,
/// Network usage by interface name (ingress, egress in bytes)
#[serde(default)]
pub network_usage_interface: Vec<SingleNetworkInterfaceUsage>, // interface -> (ingress, egress)
// metadata // metadata
/// The rate the system stats are being polled from the system /// The rate the system stats are being polled from the system
pub polling_rate: Timelength, pub polling_rate: Timelength,
@@ -95,6 +112,18 @@ pub struct SingleDiskUsage {
pub total_gb: f64, pub total_gb: f64,
} }
/// Info for network interface usage.
#[typeshare]
#[derive(Serialize, Deserialize, Debug, Clone)]
pub struct SingleNetworkInterfaceUsage {
/// The network interface name
pub name: String,
/// The ingress in bytes
pub ingress_bytes: f64,
/// The egress in bytes
pub egress_bytes: f64,
}
pub fn sum_disk_usage(disks: &[SingleDiskUsage]) -> TotalDiskUsage { pub fn sum_disk_usage(disks: &[SingleDiskUsage]) -> TotalDiskUsage {
disks disks
.iter() .iter()

View File

@@ -1809,6 +1809,16 @@ export interface SingleDiskUsage {
total_gb: number; total_gb: number;
} }
/** Info for network interface usage. */
export interface SingleNetworkInterfaceUsage {
/** The network interface name */
name: string;
/** The ingress in bytes */
ingress_bytes: number;
/** The egress in bytes */
egress_bytes: number;
}
export enum Timelength { export enum Timelength {
OneSecond = "1-sec", OneSecond = "1-sec",
FiveSeconds = "5-sec", FiveSeconds = "5-sec",
@@ -1850,6 +1860,12 @@ export interface SystemStats {
mem_total_gb: number; mem_total_gb: number;
/** Breakdown of individual disks, ie their usages, sizes, and mount points */ /** Breakdown of individual disks, ie their usages, sizes, and mount points */
disks: SingleDiskUsage[]; disks: SingleDiskUsage[];
/** Network ingress usage in MB */
network_ingress_bytes?: number;
/** Network egress usage in MB */
network_egress_bytes?: number;
/** Network usage by interface name (ingress, egress in bytes) */
network_usage_interface?: SingleNetworkInterfaceUsage[];
/** The rate the system stats are being polled from the system */ /** The rate the system stats are being polled from the system */
polling_rate: Timelength; polling_rate: Timelength;
/** Unix timestamp in milliseconds when stats were last polled */ /** Unix timestamp in milliseconds when stats were last polled */
@@ -5095,6 +5111,12 @@ export interface SystemStatsRecord {
disk_total_gb: number; disk_total_gb: number;
/** Breakdown of individual disks, ie their usages, sizes, and mount points */ /** Breakdown of individual disks, ie their usages, sizes, and mount points */
disks: SingleDiskUsage[]; disks: SingleDiskUsage[];
/** Network ingress usage in bytes */
network_ingress_bytes?: number;
/** Network egress usage in bytes */
network_egress_bytes?: number;
/** Network usage by interface name (ingress, egress in bytes) */
network_usage_interface?: SingleNetworkInterfaceUsage[];
} }
/** Response to [GetHistoricalServerStats]. */ /** Response to [GetHistoricalServerStats]. */

View File

@@ -20,6 +20,4 @@ pub struct GetSystemStats {}
#[derive(Serialize, Deserialize, Debug, Clone, Request)] #[derive(Serialize, Deserialize, Debug, Clone, Request)]
#[response(Vec<SystemProcess>)] #[response(Vec<SystemProcess>)]
pub struct GetSystemProcesses {} pub struct GetSystemProcesses {}
//

View File

@@ -15,11 +15,11 @@ COMPOSE_KOMODO_IMAGE_TAG=latest
COMPOSE_LOGGING_DRIVER=local # Enable log rotation with the local driver. COMPOSE_LOGGING_DRIVER=local # Enable log rotation with the local driver.
## DB credentials - Ignored for Sqlite ## DB credentials - Ignored for Sqlite
DB_USERNAME=admin KOMODO_DB_USERNAME=admin
DB_PASSWORD=admin KOMODO_DB_PASSWORD=admin
## Configure a secure passkey to authenticate between Core / Periphery. ## Configure a secure passkey to authenticate between Core / Periphery.
PASSKEY=a_random_passkey KOMODO_PASSKEY=a_random_passkey
#=-------------------------=# #=-------------------------=#
#= Komodo Core Environment =# #= Komodo Core Environment =#
@@ -52,8 +52,6 @@ KOMODO_MONITORING_INTERVAL="15-sec"
## Default: 5-min ## Default: 5-min
KOMODO_RESOURCE_POLL_INTERVAL="5-min" KOMODO_RESOURCE_POLL_INTERVAL="5-min"
## Used to auth against periphery. Alt: KOMODO_PASSKEY_FILE
KOMODO_PASSKEY=${PASSKEY}
## Used to auth incoming webhooks. Alt: KOMODO_WEBHOOK_SECRET_FILE ## Used to auth incoming webhooks. Alt: KOMODO_WEBHOOK_SECRET_FILE
KOMODO_WEBHOOK_SECRET=a_random_secret KOMODO_WEBHOOK_SECRET=a_random_secret
## Used to generate jwt. Alt: KOMODO_JWT_SECRET_FILE ## Used to generate jwt. Alt: KOMODO_JWT_SECRET_FILE
@@ -115,8 +113,11 @@ KOMODO_HETZNER_TOKEN= # Alt: KOMODO_HETZNER_TOKEN_FILE
## Full variable list + descriptions are available here: ## Full variable list + descriptions are available here:
## 🦎 https://github.com/mbecker20/komodo/blob/main/config/periphery.config.toml 🦎 ## 🦎 https://github.com/mbecker20/komodo/blob/main/config/periphery.config.toml 🦎
## Periphery passkeys must include KOMODO_PASSKEY to authenticate ## Periphery passkeys must include KOMODO_PASSKEY to authenticate.
PERIPHERY_PASSKEYS=${PASSKEY} PERIPHERY_PASSKEYS=${KOMODO_PASSKEY}
## Specify the root directory used by Periphery agent.
PERIPHERY_ROOT_DIRECTORY=/etc/komodo
## Enable SSL using self signed certificates. ## Enable SSL using self signed certificates.
## Connect to Periphery at https://address:8120. ## Connect to Periphery at https://address:8120.

View File

@@ -24,8 +24,8 @@ services:
- mongo-data:/data/db - mongo-data:/data/db
- mongo-config:/data/configdb - mongo-config:/data/configdb
environment: environment:
MONGO_INITDB_ROOT_USERNAME: ${DB_USERNAME} MONGO_INITDB_ROOT_USERNAME: ${KOMODO_DB_USERNAME}
MONGO_INITDB_ROOT_PASSWORD: ${DB_PASSWORD} MONGO_INITDB_ROOT_PASSWORD: ${KOMODO_DB_PASSWORD}
core: core:
image: ghcr.io/mbecker20/komodo:${COMPOSE_KOMODO_IMAGE_TAG:-latest} image: ghcr.io/mbecker20/komodo:${COMPOSE_KOMODO_IMAGE_TAG:-latest}
@@ -43,8 +43,8 @@ services:
env_file: ./compose.env env_file: ./compose.env
environment: environment:
KOMODO_DATABASE_ADDRESS: mongo:27017 KOMODO_DATABASE_ADDRESS: mongo:27017
KOMODO_DATABASE_USERNAME: ${DB_USERNAME} KOMODO_DATABASE_USERNAME: ${KOMODO_DB_USERNAME}
KOMODO_DATABASE_PASSWORD: ${DB_PASSWORD} KOMODO_DATABASE_PASSWORD: ${KOMODO_DB_PASSWORD}
volumes: volumes:
## Core cache for repos for latest commit hash / contents ## Core cache for repos for latest commit hash / contents
- repo-cache:/repo-cache - repo-cache:/repo-cache
@@ -70,22 +70,21 @@ services:
networks: networks:
- default - default
env_file: ./compose.env env_file: ./compose.env
environment:
PERIPHERY_REPO_DIR: ${PERIPHERY_ROOT_DIRECTORY:-/etc/komodo}/repos
PERIPHERY_STACK_DIR: ${PERIPHERY_ROOT_DIRECTORY:-/etc/komodo}/stacks
PERIPHERY_SSL_KEY_FILE: ${PERIPHERY_ROOT_DIRECTORY:-/etc/komodo}/ssl/key.pem
PERIPHERY_SSL_CERT_FILE: ${PERIPHERY_ROOT_DIRECTORY:-/etc/komodo}/ssl/cert.pem
volumes: volumes:
## Mount external docker socket ## Mount external docker socket
- /var/run/docker.sock:/var/run/docker.sock - /var/run/docker.sock:/var/run/docker.sock
## Allow Periphery to see processes outside of container ## Allow Periphery to see processes outside of container
- /proc:/proc - /proc:/proc
## use self signed certs in docker volume, ## Specify the Periphery agent root directory.
## or mount your own signed certs. ## Must be the same inside and outside the container,
- ssl-certs:/etc/komodo/ssl ## or docker will get confused. See https://github.com/mbecker20/komodo/discussions/180.
## manage repos in a docker volume, ## Default: /etc/komodo.
## or change it to an accessible host directory. - ${PERIPHERY_ROOT_DIRECTORY:-/etc/komodo}:${PERIPHERY_ROOT_DIRECTORY:-/etc/komodo}
- repos:/etc/komodo/repos
## manage stack files in a docker volume,
## or change it to an accessible host directory.
- stacks:/etc/komodo/stacks
## Optionally mount a path to store compose files
# - /path/to/compose:/host/compose
volumes: volumes:
# Mongo # Mongo
@@ -93,10 +92,6 @@ volumes:
mongo-config: mongo-config:
# Core # Core
repo-cache: repo-cache:
# Periphery
ssl-certs:
repos:
stacks:
networks: networks:
default: {} default: {}

View File

@@ -3,13 +3,13 @@
################################### ###################################
## This compose file will deploy: ## This compose file will deploy:
## 1. Postgres + FerretDB Mongo adapter ## 1. Postgres + FerretDB Mongo adapter (https://www.ferretdb.com)
## 2. Komodo Core ## 2. Komodo Core
## 3. Komodo Periphery ## 3. Komodo Periphery
services: services:
postgres: postgres:
image: postgres image: postgres:17
labels: labels:
komodo.skip: # Prevent Komodo from stopping with StopAllContainers komodo.skip: # Prevent Komodo from stopping with StopAllContainers
restart: unless-stopped restart: unless-stopped
@@ -22,12 +22,12 @@ services:
volumes: volumes:
- pg-data:/var/lib/postgresql/data - pg-data:/var/lib/postgresql/data
environment: environment:
- POSTGRES_USER=${DB_USERNAME} - POSTGRES_USER=${KOMODO_DB_USERNAME}
- POSTGRES_PASSWORD=${DB_PASSWORD} - POSTGRES_PASSWORD=${KOMODO_DB_PASSWORD}
- POSTGRES_DB=${KOMODO_DATABASE_DB_NAME:-komodo} - POSTGRES_DB=${KOMODO_DATABASE_DB_NAME:-komodo}
ferretdb: ferretdb:
image: ghcr.io/ferretdb/ferretdb image: ghcr.io/ferretdb/ferretdb:1
labels: labels:
komodo.skip: # Prevent Komodo from stopping with StopAllContainers komodo.skip: # Prevent Komodo from stopping with StopAllContainers
restart: unless-stopped restart: unless-stopped
@@ -57,7 +57,7 @@ services:
- 9120:9120 - 9120:9120
env_file: ./compose.env env_file: ./compose.env
environment: environment:
KOMODO_DATABASE_URI: mongodb://${DB_USERNAME}:${DB_PASSWORD}@ferretdb:27017/${KOMODO_DATABASE_DB_NAME:-komodo}?authMechanism=PLAIN KOMODO_DATABASE_URI: mongodb://${KOMODO_DB_USERNAME}:${KOMODO_DB_PASSWORD}@ferretdb:27017/${KOMODO_DATABASE_DB_NAME:-komodo}?authMechanism=PLAIN
volumes: volumes:
## Core cache for repos for latest commit hash / contents ## Core cache for repos for latest commit hash / contents
- repo-cache:/repo-cache - repo-cache:/repo-cache
@@ -83,32 +83,27 @@ services:
networks: networks:
- default - default
env_file: ./compose.env env_file: ./compose.env
environment:
PERIPHERY_REPO_DIR: ${PERIPHERY_ROOT_DIRECTORY:-/etc/komodo}/repos
PERIPHERY_STACK_DIR: ${PERIPHERY_ROOT_DIRECTORY:-/etc/komodo}/stacks
PERIPHERY_SSL_KEY_FILE: ${PERIPHERY_ROOT_DIRECTORY:-/etc/komodo}/ssl/key.pem
PERIPHERY_SSL_CERT_FILE: ${PERIPHERY_ROOT_DIRECTORY:-/etc/komodo}/ssl/cert.pem
volumes: volumes:
## Mount external docker socket ## Mount external docker socket
- /var/run/docker.sock:/var/run/docker.sock - /var/run/docker.sock:/var/run/docker.sock
## Allow Periphery to see processes outside of container ## Allow Periphery to see processes outside of container
- /proc:/proc - /proc:/proc
## use self signed certs in docker volume, ## Specify the Periphery agent root directory.
## or mount your own signed certs. ## Must be the same inside and outside the container,
- ssl-certs:/etc/komodo/ssl ## or docker will get confused. See https://github.com/mbecker20/komodo/discussions/180.
## manage repos in a docker volume, ## Default: /etc/komodo.
## or change it to an accessible host directory. - ${PERIPHERY_ROOT_DIRECTORY:-/etc/komodo}:${PERIPHERY_ROOT_DIRECTORY:-/etc/komodo}
- repos:/etc/komodo/repos
## manage stack files in a docker volume,
## or change it to an accessible host directory.
- stacks:/etc/komodo/stacks
## Optionally mount a path to store compose files
# - /path/to/compose:/host/compose
volumes: volumes:
# Postgres # Postgres
pg-data: pg-data:
# Core # Core
repo-cache: repo-cache:
# Periphery
ssl-certs:
repos:
stacks:
networks: networks:
default: {} default: {}

View File

@@ -3,13 +3,13 @@
################################# #################################
## This compose file will deploy: ## This compose file will deploy:
## 1. Sqlite + FerretDB Mongo adapter ## 1. Sqlite + FerretDB Mongo adapter (https://www.ferretdb.com)
## 2. Komodo Core ## 2. Komodo Core
## 3. Komodo Periphery ## 3. Komodo Periphery
services: services:
ferretdb: ferretdb:
image: ghcr.io/ferretdb/ferretdb image: ghcr.io/ferretdb/ferretdb:1
labels: labels:
komodo.skip: # Prevent Komodo from stopping with StopAllContainers komodo.skip: # Prevent Komodo from stopping with StopAllContainers
restart: unless-stopped restart: unless-stopped
@@ -65,32 +65,27 @@ services:
networks: networks:
- default - default
env_file: ./compose.env env_file: ./compose.env
environment:
PERIPHERY_REPO_DIR: ${PERIPHERY_ROOT_DIRECTORY:-/etc/komodo}/repos
PERIPHERY_STACK_DIR: ${PERIPHERY_ROOT_DIRECTORY:-/etc/komodo}/stacks
PERIPHERY_SSL_KEY_FILE: ${PERIPHERY_ROOT_DIRECTORY:-/etc/komodo}/ssl/key.pem
PERIPHERY_SSL_CERT_FILE: ${PERIPHERY_ROOT_DIRECTORY:-/etc/komodo}/ssl/cert.pem
volumes: volumes:
## Mount external docker socket ## Mount external docker socket
- /var/run/docker.sock:/var/run/docker.sock - /var/run/docker.sock:/var/run/docker.sock
## Allow Periphery to see processes outside of container ## Allow Periphery to see processes outside of container
- /proc:/proc - /proc:/proc
## use self signed certs in docker volume, ## Specify the Periphery agent root directory.
## or mount your own signed certs. ## Must be the same inside and outside the container,
- ssl-certs:/etc/komodo/ssl ## or docker will get confused. See https://github.com/mbecker20/komodo/discussions/180.
## manage repos in a docker volume, ## Default: /etc/komodo.
## or change it to an accessible host directory. - ${PERIPHERY_ROOT_DIRECTORY:-/etc/komodo}:${PERIPHERY_ROOT_DIRECTORY:-/etc/komodo}
- repos:/etc/komodo/repos
## manage stack files in a docker volume,
## or change it to an accessible host directory.
- stacks:/etc/komodo/stacks
## Optionally mount a path to store compose files
# - /path/to/compose:/host/compose
volumes: volumes:
# Sqlite # Sqlite
sqlite-data: sqlite-data:
# Core # Core
repo-cache: repo-cache:
# Periphery
ssl-certs:
repos:
stacks:
networks: networks:
default: {} default: {}

View File

@@ -14,9 +14,9 @@ Komodo is able to support Postgres and Sqlite by utilizing the [FerretDB Mongo A
### First login ### First login
Core should now be accessible on the specified port, so navigating to `http://<address>:<port>` will display the login page. Core should now be accessible on the specified port, so navigating to `http://<address>:<port>` will display the login page.
On first login, you need to click `sign-up`, _not_ login, to create an initial admin user for Komodo.
The first user to log in will be auto enabled and made an admin. Any additional users to create accounts will be disabled by default, and must be enabled by an admin. Any additional users to create accounts will be disabled by default, and must be enabled by an admin.
### Https ### Https

View File

@@ -220,6 +220,20 @@ cp ./target/release/periphery /root/periphery
""" """
``` ```
### Resource sync
- [Resource sync config schema](https://docs.rs/komodo_client/latest/komodo_client/entities/sync/type.ResourceSync.html)
```toml
[[resource_sync]]
name = "resource-sync"
[resource_sync.config]
git_provider = "git.mogh.tech" # use an alternate git provider (default is github.com)
git_account = "mbecker20"
repo = "mbecker20/komodo"
resource_path = ["stacks.toml", "repos.toml"]
```
### User Group: ### User Group:
- [UserGroup schema](https://docs.rs/komodo_client/latest/komodo_client/entities/toml/struct.UserGroupToml.html) - [UserGroup schema](https://docs.rs/komodo_client/latest/komodo_client/entities/toml/struct.UserGroupToml.html)

View File

@@ -1894,6 +1894,15 @@ export interface SingleDiskUsage {
/** Total size of the disk in GB */ /** Total size of the disk in GB */
total_gb: number; total_gb: number;
} }
/** Info for network interface usage. */
export interface SingleNetworkInterfaceUsage {
/** The network interface name */
name: string;
/** The ingress in bytes */
ingress_bytes: number;
/** The egress in bytes */
egress_bytes: number;
}
export declare enum Timelength { export declare enum Timelength {
OneSecond = "1-sec", OneSecond = "1-sec",
FiveSeconds = "5-sec", FiveSeconds = "5-sec",
@@ -1934,6 +1943,12 @@ export interface SystemStats {
mem_total_gb: number; mem_total_gb: number;
/** Breakdown of individual disks, ie their usages, sizes, and mount points */ /** Breakdown of individual disks, ie their usages, sizes, and mount points */
disks: SingleDiskUsage[]; disks: SingleDiskUsage[];
/** Network ingress usage in MB */
network_ingress_bytes?: number;
/** Network egress usage in MB */
network_egress_bytes?: number;
/** Network usage by interface name (ingress, egress in bytes) */
network_usage_interface?: SingleNetworkInterfaceUsage[];
/** The rate the system stats are being polled from the system */ /** The rate the system stats are being polled from the system */
polling_rate: Timelength; polling_rate: Timelength;
/** Unix timestamp in milliseconds when stats were last polled */ /** Unix timestamp in milliseconds when stats were last polled */
@@ -4842,6 +4857,12 @@ export interface SystemStatsRecord {
disk_total_gb: number; disk_total_gb: number;
/** Breakdown of individual disks, ie their usages, sizes, and mount points */ /** Breakdown of individual disks, ie their usages, sizes, and mount points */
disks: SingleDiskUsage[]; disks: SingleDiskUsage[];
/** Network ingress usage in bytes */
network_ingress_bytes?: number;
/** Network egress usage in bytes */
network_egress_bytes?: number;
/** Network usage by interface name (ingress, egress in bytes) */
network_usage_interface?: SingleNetworkInterfaceUsage[];
} }
/** Response to [GetHistoricalServerStats]. */ /** Response to [GetHistoricalServerStats]. */
export interface GetHistoricalServerStatsResponse { export interface GetHistoricalServerStatsResponse {

View File

@@ -8,3 +8,10 @@ const statsGranularityAtom = atomWithStorage<Types.Timelength>(
); );
export const useStatsGranularity = () => useAtom(statsGranularityAtom); export const useStatsGranularity = () => useAtom(statsGranularityAtom);
const selectedNetworkInterfaceAtom = atomWithStorage<string | undefined>(
"selected-network-interface-v0",
undefined // Default value is `undefined` (Global view)
);
export const useSelectedNetworkInterface = () => useAtom(selectedNetworkInterfaceAtom);

View File

@@ -2,14 +2,14 @@ import { hex_color_by_intention } from "@lib/color";
import { useRead } from "@lib/hooks"; import { useRead } from "@lib/hooks";
import { Types } from "komodo_client"; import { Types } from "komodo_client";
import { useMemo } from "react"; import { useMemo } from "react";
import { useStatsGranularity } from "./hooks"; import { useStatsGranularity, useSelectedNetworkInterface } from "./hooks";
import { Loader2 } from "lucide-react"; import { Loader2 } from "lucide-react";
import { AxisOptions, Chart } from "react-charts"; import { AxisOptions, Chart } from "react-charts";
import { convertTsMsToLocalUnixTsInMs } from "@lib/utils"; import { convertTsMsToLocalUnixTsInMs } from "@lib/utils";
import { useTheme } from "@ui/theme"; import { useTheme } from "@ui/theme";
import { fmt_utc_date } from "@lib/formatting"; import { fmt_utc_date } from "@lib/formatting";
type StatType = "cpu" | "mem" | "disk"; type StatType = "cpu" | "mem" | "disk" | "network_ingress" | "network_egress" | "network_interface_ingress" | "network_interface_egress";
type StatDatapoint = { date: number; value: number }; type StatDatapoint = { date: number; value: number };
@@ -22,11 +22,13 @@ export const StatChart = ({
type: StatType; type: StatType;
className?: string; className?: string;
}) => { }) => {
const [selectedInterface] = useSelectedNetworkInterface();
const [granularity] = useStatsGranularity(); const [granularity] = useStatsGranularity();
const { data, isPending } = useRead("GetHistoricalServerStats", { const { data, isPending } = useRead("GetHistoricalServerStats", {
server: server_id, server: server_id,
granularity, granularity,
selectedInterface,
}); });
const stats = useMemo( const stats = useMemo(
@@ -35,7 +37,7 @@ export const StatChart = ({
.map((stat) => { .map((stat) => {
return { return {
date: convertTsMsToLocalUnixTsInMs(stat.ts), date: convertTsMsToLocalUnixTsInMs(stat.ts),
value: getStat(stat, type), value: getStat(stat, type, selectedInterface),
}; };
}) })
.reverse(), .reverse(),
@@ -70,6 +72,9 @@ export const InnerStatChart = ({
? "dark" ? "dark"
: "light" : "light"
: _theme; : _theme;
const BYTES_PER_GB = 1073741824.0;
const BYTES_PER_MB = 1048576.0;
const BYTES_PER_KB = 1024.0;
const min = stats?.[0]?.date ?? 0; const min = stats?.[0]?.date ?? 0;
const max = stats?.[stats.length - 1]?.date ?? 0; const max = stats?.[stats.length - 1]?.date ?? 0;
const diff = max - min; const diff = max - min;
@@ -90,25 +95,65 @@ export const InnerStatChart = ({
}, },
}; };
}, []); }, []);
// Determine the dynamic scaling for network-related types
const maxStatValue = Math.max(...(stats?.map((d) => d.value) ?? [0]));
const { unit, maxUnitValue } = useMemo(() => {
if (type === "network_ingress" || type === "network_egress") {
if (maxStatValue <= BYTES_PER_KB) {
return { unit: "KB", maxUnitValue: BYTES_PER_KB };
} else if (maxStatValue <= BYTES_PER_MB) {
return { unit: "MB", maxUnitValue: BYTES_PER_MB };
} else if (maxStatValue <= BYTES_PER_GB) {
return { unit: "GB", maxUnitValue: BYTES_PER_GB };
} else {
return { unit: "TB", maxUnitValue: BYTES_PER_GB * 1024 }; // Larger scale for high values
}
}
return { unit: "", maxUnitValue: 100 }; // Default for CPU, memory, disk
}, [type, maxStatValue]);
const valueAxis = useMemo( const valueAxis = useMemo(
(): AxisOptions<StatDatapoint>[] => [ (): AxisOptions<StatDatapoint>[] => [
{ {
getValue: (datum) => datum.value, getValue: (datum) => datum.value,
elementType: "area", elementType: "area",
min: 0, min: 0,
max: 100, max: maxUnitValue,
formatters: { formatters: {
tooltip: (value?: number) => ( tooltip: (value?: number) => (
<div className="text-lg font-mono"> <div className="text-lg font-mono">
{(value ?? 0) >= 10 ? value?.toFixed(2) : "0" + value?.toFixed(2)} {(type === "network_ingress" || type === "network_egress") && unit
% ? `${(value ?? 0) / (maxUnitValue / 1024)} ${unit}`
: `${value?.toFixed(2)}%`}
</div> </div>
), ),
}, },
}, },
], ],
[] [type, maxUnitValue, unit]
); );
// const valueAxis = useMemo(
// (): AxisOptions<StatDatapoint>[] => [
// {
// getValue: (datum) => datum.value,
// elementType: "area",
// min: 0,
// max: 100,
// formatters: {
// tooltip: (value?: number) => (
// <div className="text-lg font-mono">
// {(value ?? 0) >= 10 ? value?.toFixed(2) : "0" + value?.toFixed(2)}
// %
// </div>
// ),
// },
// },
// ],
// []
// );
return ( return (
<Chart <Chart
options={{ options={{
@@ -133,68 +178,26 @@ export const InnerStatChart = ({
/> />
); );
// const container_ref = useRef<HTMLDivElement>(null);
// const line_ref = useRef<IChartApi>();
// const series_ref = useRef<ISeriesApi<"Area">>();
// const lineColor = getColor(type);
// const handleResize = () =>
// line_ref.current?.applyOptions({
// width: container_ref.current?.clientWidth,
// });
// useEffect(() => {
// if (!stats) return;
// if (line_ref.current) line_ref.current.remove();
// const init = () => {
// if (!container_ref.current) return;
// // INIT LINE
// line_ref.current = createChart(container_ref.current, {
// width: container_ref.current.clientWidth,
// height: container_ref.current.clientHeight,
// layout: {
// background: { type: ColorType.Solid, color: "transparent" },
// // textColor: "grey",
// fontSize: 12,
// },
// grid: {
// horzLines: { color: "#3f454d25" },
// vertLines: { color: "#3f454d25" },
// },
// timeScale: { timeVisible: true },
// handleScale: false,
// handleScroll: false,
// });
// line_ref.current.timeScale().fitContent();
// // INIT SERIES
// series_ref.current = line_ref.current.addAreaSeries({
// priceLineVisible: false,
// title: `${type} %`,
// lineColor,
// topColor: `${lineColor}B3`,
// bottomColor: `${lineColor}0D`,
// });
// series_ref.current.setData(stats);
// };
// // Run the effect
// init();
// window.addEventListener("resize", handleResize);
// return () => {
// window.removeEventListener("resize", handleResize);
// };
// }, [stats]);
// return <div className="w-full max-w-full h-full" ref={container_ref} />;
}; };
const getStat = (stat: Types.SystemStatsRecord, type: StatType) => { const getStat = (stat: Types.SystemStatsRecord, type: StatType, selectedInterface?: string) => {
if (type === "cpu") return stat.cpu_perc || 0; if (type === "cpu") return stat.cpu_perc || 0;
if (type === "mem") return (100 * stat.mem_used_gb) / stat.mem_total_gb; if (type === "mem") return (100 * stat.mem_used_gb) / stat.mem_total_gb;
if (type === "disk") return (100 * stat.disk_used_gb) / stat.disk_total_gb; if (type === "disk") return (100 * stat.disk_used_gb) / stat.disk_total_gb;
if (type === "network_ingress") return stat.network_ingress_bytes || 0;
if (type === "network_egress") return stat.network_egress_bytes || 0;
if (type === "network_interface_ingress")
return selectedInterface
? stat.network_usage_interface?.find(
(networkInterface) => networkInterface.name === selectedInterface
)?.ingress_bytes || 0
: stat.network_ingress_bytes || 0;
if (type === "network_interface_egress")
return selectedInterface
? stat.network_usage_interface?.find(
(networkInterface) => networkInterface.name === selectedInterface
)?.egress_bytes || 0
: stat.network_egress_bytes || 0;
return 0; return 0;
}; };
@@ -202,5 +205,7 @@ const getColor = (type: StatType) => {
if (type === "cpu") return hex_color_by_intention("Good"); if (type === "cpu") return hex_color_by_intention("Good");
if (type === "mem") return hex_color_by_intention("Warning"); if (type === "mem") return hex_color_by_intention("Warning");
if (type === "disk") return hex_color_by_intention("Neutral"); if (type === "disk") return hex_color_by_intention("Neutral");
if (type === "network_interface_ingress") return hex_color_by_intention("Critical");
if (type === "network_interface_egress") return hex_color_by_intention("Unknown");
return hex_color_by_intention("Unknown"); return hex_color_by_intention("Unknown");
}; };

View File

@@ -14,7 +14,7 @@ import { DataTable, SortableHeader } from "@ui/data-table";
import { ReactNode, useMemo, useState } from "react"; import { ReactNode, useMemo, useState } from "react";
import { Input } from "@ui/input"; import { Input } from "@ui/input";
import { StatChart } from "./stat-chart"; import { StatChart } from "./stat-chart";
import { useStatsGranularity } from "./hooks"; import { useStatsGranularity, useSelectedNetworkInterface } from "./hooks";
import { import {
Select, Select,
SelectContent, SelectContent,
@@ -32,6 +32,8 @@ export const ServerStats = ({
titleOther?: ReactNode; titleOther?: ReactNode;
}) => { }) => {
const [interval, setInterval] = useStatsGranularity(); const [interval, setInterval] = useStatsGranularity();
const [networkInterface, setNetworkInterface] = useSelectedNetworkInterface();
const stats = useRead( const stats = useRead(
"GetSystemStats", "GetSystemStats",
{ server: id }, { server: id },
@@ -94,6 +96,7 @@ export const ServerStats = ({
<div className="flex flex-col lg:flex-row gap-4"> <div className="flex flex-col lg:flex-row gap-4">
<CPU stats={stats} /> <CPU stats={stats} />
<RAM stats={stats} /> <RAM stats={stats} />
<NETWORK stats={stats} />
<DISK stats={stats} /> <DISK stats={stats} />
</div> </div>
</Section> </Section>
@@ -101,7 +104,9 @@ export const ServerStats = ({
<Section <Section
title="Historical" title="Historical"
actions={ actions={
<div className="flex gap-2 items-center"> <div className="flex gap-4 items-center">
{/* Granularity Dropdown */}
<div className="flex items-center gap-2">
<div className="text-muted-foreground">Interval:</div> <div className="text-muted-foreground">Interval:</div>
<Select <Select
value={interval} value={interval}
@@ -131,6 +136,36 @@ export const ServerStats = ({
</SelectContent> </SelectContent>
</Select> </Select>
</div> </div>
{/* Network Interface Dropdown */}
<div className="flex items-center gap-2">
<div className="text-muted-foreground">Interface:</div>
<Select
value={networkInterface ?? "all"} // Show "all" if networkInterface is undefined
onValueChange={(interfaceName) => {
if (interfaceName === "all") {
setNetworkInterface(undefined); // Set undefined for "All" option
} else {
setNetworkInterface(interfaceName);
}
}}
>
<SelectTrigger className="w-[150px]">
<SelectValue />
</SelectTrigger>
<SelectContent>
<SelectItem value="all">All</SelectItem>
{/* Iterate over the vector and access the `name` property */}
{(stats?.network_usage_interface ?? []).map((networkInterface) => (
<SelectItem key={networkInterface.name} value={networkInterface.name}>
{networkInterface.name}
</SelectItem>
))}
</SelectContent>
</Select>
</div>
</div>
} }
> >
<div className="flex flex-col gap-8"> <div className="flex flex-col gap-8">
@@ -141,6 +176,8 @@ export const ServerStats = ({
type="disk" type="disk"
className="w-full h-[250px]" className="w-full h-[250px]"
/> />
<StatChart server_id={id} type="network_ingress" className="w-full h-[250px]" />
<StatChart server_id={id} type="network_egress" className="w-full h-[250px]" />
</div> </div>
</Section> </Section>
@@ -347,6 +384,49 @@ const RAM = ({ stats }: { stats: Types.SystemStats | undefined }) => {
); );
}; };
const formatBytes = (bytes: number) => {
const BYTES_PER_KB = 1024;
const BYTES_PER_MB = 1024 * BYTES_PER_KB;
const BYTES_PER_GB = 1024 * BYTES_PER_MB;
if (bytes >= BYTES_PER_GB) {
return { value: bytes / BYTES_PER_GB, unit: "GB" };
} else if (bytes >= BYTES_PER_MB) {
return { value: bytes / BYTES_PER_MB, unit: "MB" };
} else if (bytes >= BYTES_PER_KB) {
return { value: bytes / BYTES_PER_KB, unit: "KB" };
} else {
return { value: bytes, unit: "bytes" };
}
};
const NETWORK = ({ stats }: { stats: Types.SystemStats | undefined }) => {
const ingress = stats?.network_ingress_bytes ?? 0;
const egress = stats?.network_egress_bytes ?? 0;
const formattedIngress = formatBytes(ingress);
const formattedEgress = formatBytes(egress);
return (
<Card className="w-full">
<CardHeader className="flex-row justify-between">
<CardTitle>Network Usage</CardTitle>
</CardHeader>
<CardContent>
<div className="flex justify-between items-center mb-4">
<p className="font-medium">Ingress</p>
<span className="text-sm text-gray-600">{formattedIngress.value.toFixed(2)} {formattedIngress.unit}</span>
</div>
<div className="flex justify-between items-center">
<p className="font-medium">Egress</p>
<span className="text-sm text-gray-600">{formattedEgress.value.toFixed(2)} {formattedEgress.unit}</span>
</div>
</CardContent>
</Card>
);
};
const DISK = ({ stats }: { stats: Types.SystemStats | undefined }) => { const DISK = ({ stats }: { stats: Types.SystemStats | undefined }) => {
const used = stats?.disks.reduce((acc, curr) => (acc += curr.used_gb), 0); const used = stats?.disks.reduce((acc, curr) => (acc += curr.used_gb), 0);
const total = stats?.disks.reduce((acc, curr) => (acc += curr.total_gb), 0); const total = stats?.disks.reduce((acc, curr) => (acc += curr.total_gb), 0);

View File

@@ -2,7 +2,7 @@ services:
core: core:
build: build:
context: . context: .
dockerfile: bin/core/debian.Dockerfile dockerfile: bin/core/aio.Dockerfile
restart: unless-stopped restart: unless-stopped
logging: logging:
driver: local driver: local
@@ -20,7 +20,7 @@ services:
periphery: periphery:
build: build:
context: . context: .
dockerfile: bin/periphery/debian.Dockerfile dockerfile: bin/periphery/aio.Dockerfile
restart: unless-stopped restart: unless-stopped
logging: logging:
driver: local driver: local
@@ -53,4 +53,4 @@ volumes:
data: data:
repo-cache: repo-cache:
repos: repos:
stacks: stacks: