1.17.4 (#446)

* add terminal (ssh) apis

* add core terminal exec method

* terminal typescript client method

* terminals WIP

* backend for pty

* add ts responses

* about wire everything

* add new blog

* credit Skyfay

* working

* regen lock

* 1.17.4-dev-1

* pty history

* replace the test terminal impl with websocket (pty)

* create api and improve frontend

* fix fe

* terminals

* disable terminal api on periphery

* implement write level terminal perms

* remove unneeded

* fix clippy

* delete unneeded

* fix waste cpu cycles

* set TERM and COLORTERM for shell environment

* fix xterm scrolling behavior

* starship promp in periphery container terminal

* kill all terminals on periphery shutdown signal

* improve starship config and enable ssl in compose

* use same scrollTop setter

* fix periphery container distribution link

* support custom command / args to init terminal

* allow fully configurable init command

* docker exec into container

* add permissioning for container exec

* add starship to core container

* add delete all terminals

* dev-2

* finished gen client

* core need curl

* hide Terminal trigger if disabled

* 1.17.4

Cargo.toml

@@ -8,7 +8,7 @@ members = [
 ]
 
 [workspace.package]
-version = "1.17.1"
+version = "1.17.4"
 edition = "2024"
 authors = ["mbecker20 <becker.maxh@gmail.com>"]
 license = "GPL-3.0-or-later"
@@ -44,19 +44,19 @@ mungos = "3.2.0"
 svi = "1.0.1"
 
 # ASYNC
-reqwest = { version = "0.12.15", default-features = false, features = ["json", "rustls-tls-native-roots"] }
+reqwest = { version = "0.12.15", default-features = false, features = ["json", "stream", "rustls-tls-native-roots"] }
 tokio = { version = "1.44.1", features = ["full"] }
-tokio-util = "0.7.14"
+tokio-util = { version = "0.7.14", features = ["io", "codec"] }
 futures = "0.3.31"
 futures-util = "0.3.31"
 arc-swap = "1.7.1"
 
 # SERVER
+tokio-tungstenite = { version = "0.26.2", features = ["rustls-tls-native-roots"] }
 axum-extra = { version = "0.10.0", features = ["typed-header"] }
 tower-http = { version = "0.6.2", features = ["fs", "cors"] }
 axum-server = { version = "0.7.2", features = ["tls-rustls"] }
 axum = { version = "0.8.1", features = ["ws", "json", "macros"] }
-tokio-tungstenite = "0.26.2"
 
 # SER/DE
 ordered_hash_map = { version = "0.4.0", features = ["serde"] }
@@ -64,10 +64,11 @@ serde = { version = "1.0.219", features = ["derive"] }
 strum = { version = "0.27.1", features = ["derive"] }
 serde_json = "1.0.140"
 serde_yaml = "0.9.34"
+serde_qs = "0.14.0"
 toml = "0.8.20"
 
 # ERROR
-anyhow = "1.0.97"
+anyhow = "1.0.98"
 thiserror = "2.0.12"
 
 # LOGGING
@@ -80,7 +81,7 @@ opentelemetry = "0.29.0"
 tracing = "0.1.41"
 
 # CONFIG
-clap = { version = "4.5.36", features = ["derive"] }
+clap = { version = "4.5.37", features = ["derive"] }
 dotenvy = "0.15.7"
 envy = "0.4.2"
 
@@ -95,10 +96,11 @@ base64 = "0.22.1"
 rustls = "0.23.26"
 hmac = "0.12.1"
 sha2 = "0.10.8"
-rand = "0.9.0"
+rand = "0.9.1"
 hex = "0.4.3"
 
 # SYSTEM
+portable-pty = "0.9.0"
 bollard = "0.18.1"
 sysinfo = "0.34.2"
 
@@ -107,6 +109,12 @@ aws-config = "1.6.1"
 aws-sdk-ec2 = "1.121.1"
 aws-credential-types = "1.2.2"
 
+## CRON
+english-to-cron = "0.1.4"
+chrono-tz = "0.10.3"
+chrono = "0.4.40"
+croner = "2.1.0"
+
 # MISC
 derive_builder = "0.20.2"
 typeshare = "1.0.4"
@@ -115,4 +123,5 @@ dashmap = "6.1.0"
 wildcard = "0.3.0"
 colored = "3.0.0"
 regex = "1.11.1"
+bytes = "1.10.1"
 bson = "2.14.0"

bin/binaries.Dockerfile

@@ -1,7 +1,7 @@
 ## Builds the Komodo Core and Periphery binaries
 ## for a specific architecture.
 
-FROM rust:1.85.1-bullseye AS builder
+FROM rust:1.86.0-bullseye AS builder
 
 WORKDIR /builder
 COPY Cargo.toml Cargo.lock ./
@@ -23,5 +23,5 @@ COPY --from=builder /builder/target/release/core /core
 COPY --from=builder /builder/target/release/periphery /periphery
 
 LABEL org.opencontainers.image.source=https://github.com/moghtech/komodo
-LABEL org.opencontainers.image.description="Komodo Periphery"
+LABEL org.opencontainers.image.description="Komodo Binaries"
 LABEL org.opencontainers.image.licenses=GPL-3.0

bin/core/Cargo.toml

@@ -38,8 +38,11 @@ slack.workspace = true
 svi.workspace = true
 # external
 aws-credential-types.workspace = true
+tokio-tungstenite.workspace = true
 ordered_hash_map.workspace = true
+english-to-cron.workspace = true
 openidconnect.workspace = true
+jsonwebtoken.workspace = true
 axum-server.workspace = true
 urlencoding.workspace = true
 aws-sdk-ec2.workspace = true
@@ -50,6 +53,7 @@ tower-http.workspace = true
 serde_json.workspace = true
 serde_yaml.workspace = true
 typeshare.workspace = true
+chrono-tz.workspace = true
 octorust.workspace = true
 wildcard.workspace = true
 arc-swap.workspace = true
@@ -60,6 +64,8 @@ futures.workspace = true
 nom_pem.workspace = true
 dotenvy.workspace = true
 anyhow.workspace = true
+croner.workspace = true
+chrono.workspace = true
 bcrypt.workspace = true
 base64.workspace = true
 rustls.workspace = true
@@ -73,5 +79,4 @@ envy.workspace = true
 rand.workspace = true
 hmac.workspace = true
 sha2.workspace = true
-jsonwebtoken.workspace = true
 hex.workspace = true

bin/core/aio.Dockerfile

@@ -1,7 +1,7 @@
 ## All in one, multi stage compile + runtime Docker build for your architecture.
 
 # Build Core
-FROM rust:1.85.1-bullseye AS core-builder
+FROM rust:1.86.0-bullseye AS core-builder
 
 WORKDIR /builder
 COPY Cargo.toml Cargo.lock ./
@@ -24,10 +24,9 @@ RUN cd frontend && yarn link komodo_client && yarn && yarn build
 # Final Image
 FROM debian:bullseye-slim
 
-# Install Deps
-RUN apt update && \
-  apt install -y git ca-certificates && \
-  rm -rf /var/lib/apt/lists/*
+COPY ./bin/core/starship.toml /config/starship.toml
+COPY ./bin/core/debian-deps.sh .
+RUN sh ./debian-deps.sh && rm ./debian-deps.sh
 
 # Setup an application directory
 WORKDIR /app

bin/core/debian-deps.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+## Core deps installer
+
+apt-get update
+apt-get install -y git curl ca-certificates
+
+rm -rf /var/lib/apt/lists/*
+
+# Starship prompt
+curl -sS https://starship.rs/install.sh | sh -s -- --yes --bin-dir /usr/local/bin
+echo 'export STARSHIP_CONFIG=/config/starship.toml' >> /root/.bashrc
+echo 'eval "$(starship init bash)"' >> /root/.bashrc
+

bin/core/multi-arch.Dockerfile

@@ -15,10 +15,9 @@ FROM ${FRONTEND_IMAGE} AS frontend
 # Final Image
 FROM debian:bullseye-slim
 
-# Install Deps
-RUN apt update && \
-  apt install -y git ca-certificates && \
-  rm -rf /var/lib/apt/lists/*
+COPY ./bin/core/starship.toml /config/starship.toml
+COPY ./bin/core/debian-deps.sh .
+RUN sh ./debian-deps.sh && rm ./debian-deps.sh
 
 WORKDIR /app
 

bin/core/single-arch.Dockerfile

@@ -16,10 +16,9 @@ RUN cd frontend && yarn link komodo_client && yarn && yarn build
 
 FROM debian:bullseye-slim
 
-# Install Deps
-RUN apt update && \
-	apt install -y git ca-certificates && \
-	rm -rf /var/lib/apt/lists/*
+COPY ./bin/core/starship.toml /config/starship.toml
+COPY ./bin/core/debian-deps.sh .
+RUN sh ./debian-deps.sh && rm ./debian-deps.sh
 	
 # Copy
 COPY ./config/core.config.toml /config/config.toml

bin/core/src/alert/discord.rs

@@ -189,6 +189,24 @@ pub async fn send_alert(
       let link = resource_link(ResourceTargetVariant::Repo, id);
       format!("{level} | Repo build for **{name}** failed\n{link}")
     }
+    AlertData::ProcedureFailed { id, name } => {
+      let link = resource_link(ResourceTargetVariant::Procedure, id);
+      format!("{level} | Procedure **{name}** failed\n{link}")
+    }
+    AlertData::ActionFailed { id, name } => {
+      let link = resource_link(ResourceTargetVariant::Action, id);
+      format!("{level} | Action **{name}** failed\n{link}")
+    }
+    AlertData::ScheduleRun {
+      resource_type,
+      id,
+      name,
+    } => {
+      let link = resource_link(*resource_type, id);
+      format!(
+        "{level} | **{name}** ({resource_type}) | Scheduled run started 🕝\n{link}"
+      )
+    }
     AlertData::None {} => Default::default(),
   };
   if !content.is_empty() {

bin/core/src/alert/mod.rs

@@ -18,8 +18,9 @@ use crate::helpers::query::get_variables_and_secrets;
 use crate::{config::core_config, state::db_client};
 
 mod discord;
-mod slack;
 mod ntfy;
+mod pushover;
+mod slack;
 
 #[instrument(level = "debug")]
 pub async fn send_alerts(alerts: &[Alert]) {
@@ -131,7 +132,18 @@ pub async fn send_alert_to_alerter(
     }
     AlerterEndpoint::Ntfy(NtfyAlerterEndpoint { url }) => {
       ntfy::send_alert(url, alert).await.with_context(|| {
-        format!("Failed to send alert to ntfy Alerter {}", alerter.name)
+        format!(
+          "Failed to send alert to ntfy Alerter {}",
+          alerter.name
+        )
+      })
+    }
+    AlerterEndpoint::Pushover(PushoverAlerterEndpoint { url }) => {
+      pushover::send_alert(url, alert).await.with_context(|| {
+        format!(
+          "Failed to send alert to Pushover Alerter {}",
+          alerter.name
+        )
       })
     }
   }

bin/core/src/alert/ntfy.rs

@@ -202,6 +202,24 @@ pub async fn send_alert(
       let link = resource_link(ResourceTargetVariant::Repo, id);
       format!("{level} | Repo build for {} failed\n{link}", name,)
     }
+    AlertData::ProcedureFailed { id, name } => {
+      let link = resource_link(ResourceTargetVariant::Procedure, id);
+      format!("{level} | Procedure {name} failed\n{link}")
+    }
+    AlertData::ActionFailed { id, name } => {
+      let link = resource_link(ResourceTargetVariant::Action, id);
+      format!("{level} | Action {name} failed\n{link}")
+    }
+    AlertData::ScheduleRun {
+      resource_type,
+      id,
+      name,
+    } => {
+      let link = resource_link(*resource_type, id);
+      format!(
+        "{level} | {name} ({resource_type}) | Scheduled run started 🕝\n{link}"
+      )
+    }
     AlertData::None {} => Default::default(),
   };
 

bin/core/src/alert/pushover.rs

@@ -0,0 +1,270 @@
+use std::sync::OnceLock;
+
+use super::*;
+
+#[instrument(level = "debug")]
+pub async fn send_alert(
+  url: &str,
+  alert: &Alert,
+) -> anyhow::Result<()> {
+  let level = fmt_level(alert.level);
+  let content = match &alert.data {
+    AlertData::Test { id, name } => {
+      let link = resource_link(ResourceTargetVariant::Alerter, id);
+      format!(
+        "{level} | If you see this message, then Alerter {} is working\n{link}",
+        name,
+      )
+    }
+    AlertData::ServerUnreachable {
+      id,
+      name,
+      region,
+      err,
+    } => {
+      let region = fmt_region(region);
+      let link = resource_link(ResourceTargetVariant::Server, id);
+      match alert.level {
+        SeverityLevel::Ok => {
+          format!(
+            "{level} | {}{} is now reachable\n{link}",
+            name, region
+          )
+        }
+        SeverityLevel::Critical => {
+          let err = err
+            .as_ref()
+            .map(|e| format!("\nerror: {:#?}", e))
+            .unwrap_or_default();
+          format!(
+            "{level} | {}{} is unreachable ❌\n{link}{err}",
+            name, region
+          )
+        }
+        _ => unreachable!(),
+      }
+    }
+    AlertData::ServerCpu {
+      id,
+      name,
+      region,
+      percentage,
+    } => {
+      let region = fmt_region(region);
+      let link = resource_link(ResourceTargetVariant::Server, id);
+      format!(
+        "{level} | {}{} cpu usage at {percentage:.1}%\n{link}",
+        name, region,
+      )
+    }
+    AlertData::ServerMem {
+      id,
+      name,
+      region,
+      used_gb,
+      total_gb,
+    } => {
+      let region = fmt_region(region);
+      let link = resource_link(ResourceTargetVariant::Server, id);
+      let percentage = 100.0 * used_gb / total_gb;
+      format!(
+        "{level} | {}{} memory usage at {percentage:.1}%💾\n\nUsing {used_gb:.1} GiB / {total_gb:.1} GiB\n{link}",
+        name, region,
+      )
+    }
+    AlertData::ServerDisk {
+      id,
+      name,
+      region,
+      path,
+      used_gb,
+      total_gb,
+    } => {
+      let region = fmt_region(region);
+      let link = resource_link(ResourceTargetVariant::Server, id);
+      let percentage = 100.0 * used_gb / total_gb;
+      format!(
+        "{level} | {}{} disk usage at {percentage:.1}%💿\nmount point: {:?}\nusing {used_gb:.1} GiB / {total_gb:.1} GiB\n{link}",
+        name, region, path,
+      )
+    }
+    AlertData::ContainerStateChange {
+      id,
+      name,
+      server_id: _server_id,
+      server_name,
+      from,
+      to,
+    } => {
+      let link = resource_link(ResourceTargetVariant::Deployment, id);
+      let to_state = fmt_docker_container_state(to);
+      format!(
+        "📦Deployment {} is now {}\nserver: {}\nprevious: {}\n{link}",
+        name, to_state, server_name, from,
+      )
+    }
+    AlertData::DeploymentImageUpdateAvailable {
+      id,
+      name,
+      server_id: _server_id,
+      server_name,
+      image,
+    } => {
+      let link = resource_link(ResourceTargetVariant::Deployment, id);
+      format!(
+        "⬆ Deployment {} has an update available\nserver: {}\nimage: {}\n{link}",
+        name, server_name, image,
+      )
+    }
+    AlertData::DeploymentAutoUpdated {
+      id,
+      name,
+      server_id: _server_id,
+      server_name,
+      image,
+    } => {
+      let link = resource_link(ResourceTargetVariant::Deployment, id);
+      format!(
+        "⬆ Deployment {} was updated automatically\nserver: {}\nimage: {}\n{link}",
+        name, server_name, image,
+      )
+    }
+    AlertData::StackStateChange {
+      id,
+      name,
+      server_id: _server_id,
+      server_name,
+      from,
+      to,
+    } => {
+      let link = resource_link(ResourceTargetVariant::Stack, id);
+      let to_state = fmt_stack_state(to);
+      format!(
+        "🥞 Stack {} is now {}\nserver: {}\nprevious: {}\n{link}",
+        name, to_state, server_name, from,
+      )
+    }
+    AlertData::StackImageUpdateAvailable {
+      id,
+      name,
+      server_id: _server_id,
+      server_name,
+      service,
+      image,
+    } => {
+      let link = resource_link(ResourceTargetVariant::Stack, id);
+      format!(
+        "⬆ Stack {} has an update available\nserver: {}\nservice: {}\nimage: {}\n{link}",
+        name, server_name, service, image,
+      )
+    }
+    AlertData::StackAutoUpdated {
+      id,
+      name,
+      server_id: _server_id,
+      server_name,
+      images,
+    } => {
+      let link = resource_link(ResourceTargetVariant::Stack, id);
+      let images_label =
+        if images.len() > 1 { "images" } else { "image" };
+      let images_str = images.join(", ");
+      format!(
+        "⬆ Stack {} was updated automatically ⏫\nserver: {}\n{}: {}\n{link}",
+        name, server_name, images_label, images_str,
+      )
+    }
+    AlertData::AwsBuilderTerminationFailed {
+      instance_id,
+      message,
+    } => {
+      format!(
+        "{level} | Failed to terminate AWS builder instance\ninstance id: {}\n{}",
+        instance_id, message,
+      )
+    }
+    AlertData::ResourceSyncPendingUpdates { id, name } => {
+      let link =
+        resource_link(ResourceTargetVariant::ResourceSync, id);
+      format!(
+        "{level} | Pending resource sync updates on {}\n{link}",
+        name,
+      )
+    }
+    AlertData::BuildFailed { id, name, version } => {
+      let link = resource_link(ResourceTargetVariant::Build, id);
+      format!(
+        "{level} | Build {name} failed\nversion: v{version}\n{link}",
+      )
+    }
+    AlertData::RepoBuildFailed { id, name } => {
+      let link = resource_link(ResourceTargetVariant::Repo, id);
+      format!("{level} | Repo build for {} failed\n{link}", name,)
+    }
+    AlertData::ProcedureFailed { id, name } => {
+      let link = resource_link(ResourceTargetVariant::Procedure, id);
+      format!("{level} | Procedure {name} failed\n{link}")
+    }
+    AlertData::ActionFailed { id, name } => {
+      let link = resource_link(ResourceTargetVariant::Action, id);
+      format!("{level} | Action {name} failed\n{link}")
+    }
+    AlertData::ScheduleRun {
+      resource_type,
+      id,
+      name,
+    } => {
+      let link = resource_link(*resource_type, id);
+      format!(
+        "{level} | {name} ({resource_type}) | Scheduled run started 🕝\n{link}"
+      )
+    }
+    AlertData::None {} => Default::default(),
+  };
+
+  if !content.is_empty() {
+    send_message(url, content).await?;
+  }
+  Ok(())
+}
+
+async fn send_message(
+  url: &str,
+  content: String,
+) -> anyhow::Result<()> {
+  // pushover needs all information to be encoded in the URL. At minimum they need
+  // the user key, the application token, and the message (url encoded).
+  // other optional params here: https://pushover.net/api (just add them to the
+  // webhook url along with the application token and the user key).
+  let content = [("message", content)];
+
+  let response = http_client()
+    .post(url)
+    .form(&content)
+    .send()
+    .await
+    .context("Failed to send message")?;
+
+  let status = response.status();
+  if status.is_success() {
+    debug!("pushover alert sent successfully: {}", status);
+    Ok(())
+  } else {
+    let text = response.text().await.with_context(|| {
+      format!(
+        "Failed to send message to pushover | {} | failed to get response text",
+        status
+      )
+    })?;
+    Err(anyhow!(
+      "Failed to send message to pushover | {} | {}",
+      status,
+      text
+    ))
+  }
+}
+
+fn http_client() -> &'static reqwest::Client {
+  static CLIENT: OnceLock<reqwest::Client> = OnceLock::new();
+  CLIENT.get_or_init(reqwest::Client::new)
+}

bin/core/src/alert/slack.rs

@@ -373,9 +373,7 @@ pub async fn send_alert(
       let text = format!("{level} | Build {name} has failed");
       let blocks = vec![
         Block::header(text.clone()),
-        Block::section(format!(
-          "build name: *{name}*\nversion: *v{version}*",
-        )),
+        Block::section(format!("version: *v{version}*",)),
         Block::section(resource_link(
           ResourceTargetVariant::Build,
           id,
@@ -388,7 +386,6 @@ pub async fn send_alert(
         format!("{level} | Repo build for *{name}* has *failed*");
       let blocks = vec![
         Block::header(text.clone()),
-        Block::section(format!("repo name: *{name}*",)),
         Block::section(resource_link(
           ResourceTargetVariant::Repo,
           id,
@@ -396,6 +393,42 @@ pub async fn send_alert(
       ];
       (text, blocks.into())
     }
+    AlertData::ProcedureFailed { id, name } => {
+      let text = format!("{level} | Procedure *{name}* has *failed*");
+      let blocks = vec![
+        Block::header(text.clone()),
+        Block::section(resource_link(
+          ResourceTargetVariant::Procedure,
+          id,
+        )),
+      ];
+      (text, blocks.into())
+    }
+    AlertData::ActionFailed { id, name } => {
+      let text = format!("{level} | Action *{name}* has *failed*");
+      let blocks = vec![
+        Block::header(text.clone()),
+        Block::section(resource_link(
+          ResourceTargetVariant::Action,
+          id,
+        )),
+      ];
+      (text, blocks.into())
+    }
+    AlertData::ScheduleRun {
+      resource_type,
+      id,
+      name,
+    } => {
+      let text = format!(
+        "{level} | *{name}* ({resource_type}) | Scheduled run started 🕝"
+      );
+      let blocks = vec![
+        Block::header(text.clone()),
+        Block::section(resource_link(*resource_type, id)),
+      ];
+      (text, blocks.into())
+    }
     AlertData::None {} => Default::default(),
   };
   if !text.is_empty() {
@@ -411,7 +444,7 @@ pub async fn send_alert(
       &mut global_replacers,
       &mut secret_replacers,
     )?;
-    
+
     let slack = ::slack::Client::new(url_interpolated);
     slack.send_message(text, blocks).await.map_err(|e| {
       let replacers =

bin/core/src/api/execute/action.rs

@@ -13,8 +13,13 @@ use komodo_client::{
     user::{CreateApiKey, CreateApiKeyResponse, DeleteApiKey},
   },
   entities::{
-    action::Action, config::core::CoreConfig,
-    permission::PermissionLevel, update::Update, user::action_user,
+    action::Action,
+    alert::{Alert, AlertData, SeverityLevel},
+    config::core::CoreConfig,
+    komodo_timestamp,
+    permission::PermissionLevel,
+    update::Update,
+    user::action_user,
   },
 };
 use mungos::{by_id::update_one_by_id, mongodb::bson::to_document};
@@ -22,6 +27,7 @@ use resolver_api::Resolve;
 use tokio::fs;
 
 use crate::{
+  alert::send_alerts,
   api::{execute::ExecuteRequest, user::UserArgs},
   config::core_config,
   helpers::{
@@ -178,6 +184,26 @@ impl Resolve<ExecuteArgs> for RunAction {
 
     update_update(update.clone()).await?;
 
+    if !update.success && action.config.failure_alert {
+      warn!("action unsuccessful, alerting...");
+      let target = update.target.clone();
+      tokio::spawn(async move {
+        let alert = Alert {
+          id: Default::default(),
+          target,
+          ts: komodo_timestamp(),
+          resolved_ts: Some(komodo_timestamp()),
+          resolved: true,
+          level: SeverityLevel::Warning,
+          data: AlertData::ActionFailed {
+            id: action.id,
+            name: action.name,
+          },
+        };
+        send_alerts(&[alert]).await
+      });
+    }
+
     Ok(update)
   }
 }

bin/core/src/api/execute/procedure.rs

@@ -6,8 +6,12 @@ use komodo_client::{
     BatchExecutionResponse, BatchRunProcedure, RunProcedure,
   },
   entities::{
-    permission::PermissionLevel, procedure::Procedure,
-    update::Update, user::User,
+    alert::{Alert, AlertData, SeverityLevel},
+    komodo_timestamp,
+    permission::PermissionLevel,
+    procedure::Procedure,
+    update::Update,
+    user::User,
   },
 };
 use mungos::{by_id::update_one_by_id, mongodb::bson::to_document};
@@ -15,6 +19,7 @@ use resolver_api::Resolve;
 use tokio::sync::Mutex;
 
 use crate::{
+  alert::send_alerts,
   helpers::{procedure::execute_procedure, update::update_update},
   resource::{self, refresh_procedure_state_cache},
   state::{action_states, db_client},
@@ -137,6 +142,26 @@ fn resolve_inner(
 
     update_update(update.clone()).await?;
 
+    if !update.success && procedure.config.failure_alert {
+      warn!("procedure unsuccessful, alerting...");
+      let target = update.target.clone();
+      tokio::spawn(async move {
+        let alert = Alert {
+          id: Default::default(),
+          target,
+          ts: komodo_timestamp(),
+          resolved_ts: Some(komodo_timestamp()),
+          resolved: true,
+          level: SeverityLevel::Warning,
+          data: AlertData::ProcedureFailed {
+            id: procedure.id,
+            name: procedure.name,
+          },
+        };
+        send_alerts(&[alert]).await
+      });
+    }
+
     Ok(update)
   })
 }

bin/core/src/api/read/mod.rs

@@ -123,6 +123,7 @@ enum ReadRequest {
   ListDockerImages(ListDockerImages),
   ListDockerVolumes(ListDockerVolumes),
   ListComposeProjects(ListComposeProjects),
+  ListTerminals(ListTerminals),
 
   // ==== DEPLOYMENT ====
   GetDeploymentsSummary(GetDeploymentsSummary),

bin/core/src/api/read/server.rs

@@ -21,9 +21,11 @@ use komodo_client::{
       network::Network,
       volume::Volume,
     },
+    komodo_timestamp,
     permission::PermissionLevel,
     server::{
       Server, ServerActionState, ServerListItem, ServerState,
+      TerminalInfo,
     },
     stack::{Stack, StackServiceNames},
     stats::{SystemInformation, SystemProcess},
@@ -45,7 +47,10 @@ use resolver_api::Resolve;
 use tokio::sync::Mutex;
 
 use crate::{
-  helpers::{periphery_client, query::get_all_tags},
+  helpers::{
+    periphery_client,
+    query::{get_all_tags, get_system_info},
+  },
   resource,
   stack::compose_container_match_regex,
   state::{action_states, db_client, server_status_cache},
@@ -198,16 +203,6 @@ impl Resolve<ReadArgs> for GetServerActionState {
   }
 }
 
-// This protects the peripheries from spam requests
-const SYSTEM_INFO_EXPIRY: u128 = FIFTEEN_SECONDS_MS;
-type SystemInfoCache =
-  Mutex<HashMap<String, Arc<(SystemInformation, u128)>>>;
-fn system_info_cache() -> &'static SystemInfoCache {
-  static SYSTEM_INFO_CACHE: OnceLock<SystemInfoCache> =
-    OnceLock::new();
-  SYSTEM_INFO_CACHE.get_or_init(Default::default)
-}
-
 impl Resolve<ReadArgs> for GetSystemInformation {
   async fn resolve(
     self,
@@ -219,25 +214,7 @@ impl Resolve<ReadArgs> for GetSystemInformation {
       PermissionLevel::Read,
     )
     .await?;
-
-    let mut lock = system_info_cache().lock().await;
-    let res = match lock.get(&server.id) {
-      Some(cached) if cached.1 > unix_timestamp_ms() => {
-        cached.0.clone()
-      }
-      _ => {
-        let stats = periphery_client(&server)?
-          .request(periphery::stats::GetSystemInformation {})
-          .await?;
-        lock.insert(
-          server.id,
-          (stats.clone(), unix_timestamp_ms() + SYSTEM_INFO_EXPIRY)
-            .into(),
-        );
-        stats
-      }
-    };
-    Ok(res)
+    get_system_info(&server).await.map_err(Into::into)
   }
 }
 
@@ -812,3 +789,66 @@ impl Resolve<ReadArgs> for ListComposeProjects {
     }
   }
 }
+
+#[derive(Default)]
+struct TerminalCacheItem {
+  list: Vec<TerminalInfo>,
+  ttl: i64,
+}
+
+const TERMINAL_CACHE_TIMEOUT: i64 = 30_000;
+
+#[derive(Default)]
+struct TerminalCache(
+  std::sync::Mutex<
+    HashMap<String, Arc<tokio::sync::Mutex<TerminalCacheItem>>>,
+  >,
+);
+
+impl TerminalCache {
+  fn get_or_insert(
+    &self,
+    server_id: String,
+  ) -> Arc<tokio::sync::Mutex<TerminalCacheItem>> {
+    if let Some(cached) =
+      self.0.lock().unwrap().get(&server_id).cloned()
+    {
+      return cached;
+    }
+    let to_cache =
+      Arc::new(tokio::sync::Mutex::new(TerminalCacheItem::default()));
+    self.0.lock().unwrap().insert(server_id, to_cache.clone());
+    to_cache
+  }
+}
+
+fn terminals_cache() -> &'static TerminalCache {
+  static TERMINALS: OnceLock<TerminalCache> = OnceLock::new();
+  TERMINALS.get_or_init(Default::default)
+}
+
+impl Resolve<ReadArgs> for ListTerminals {
+  async fn resolve(
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<ListTerminalsResponse> {
+    let server = resource::get_check_permissions::<Server>(
+      &self.server,
+      user,
+      PermissionLevel::Read,
+    )
+    .await?;
+    let cache = terminals_cache().get_or_insert(server.id.clone());
+    let mut cache = cache.lock().await;
+    if self.fresh || komodo_timestamp() > cache.ttl {
+      cache.list = periphery_client(&server)?
+        .request(periphery_client::api::terminal::ListTerminals {})
+        .await
+        .context("Failed to get fresh terminal list")?;
+      cache.ttl = komodo_timestamp() + TERMINAL_CACHE_TIMEOUT;
+      Ok(cache.list.clone())
+    } else {
+      Ok(cache.list.clone())
+    }
+  }
+}

bin/core/src/api/write/build.rs

@@ -432,9 +432,7 @@ async fn get_on_host_periphery(
 
   match builder.config {
     BuilderConfig::Aws(_) => {
-      return Err(anyhow!(
-        "Files on host doesn't work with AWS builder"
-      ));
+      Err(anyhow!("Files on host doesn't work with AWS builder"))
     }
     BuilderConfig::Url(config) => {
       let periphery = PeripheryClient::new(

bin/core/src/api/write/mod.rs

@@ -81,6 +81,9 @@ pub enum WriteRequest {
   UpdateServer(UpdateServer),
   RenameServer(RenameServer),
   CreateNetwork(CreateNetwork),
+  CreateTerminal(CreateTerminal),
+  DeleteTerminal(DeleteTerminal),
+  DeleteAllTerminals(DeleteAllTerminals),
 
   // ==== DEPLOYMENT ====
   CreateDeployment(CreateDeployment),
@@ -208,10 +211,6 @@ async fn handler(
     .await
     .context("failure in spawned task");
 
-  if let Err(e) = &res {
-    warn!("/write request {req_id} spawn error: {e:#}");
-  }
-
   res?
 }
 

bin/core/src/api/write/server.rs

@@ -1,8 +1,9 @@
+use anyhow::Context;
 use formatting::format_serror;
 use komodo_client::{
   api::write::*,
   entities::{
-    Operation,
+    NoData, Operation,
     permission::PermissionLevel,
     server::Server,
     update::{Update, UpdateStatus},
@@ -101,3 +102,81 @@ impl Resolve<WriteArgs> for CreateNetwork {
     Ok(update)
   }
 }
+
+impl Resolve<WriteArgs> for CreateTerminal {
+  #[instrument(name = "CreateTerminal", skip(user))]
+  async fn resolve(
+    self,
+    WriteArgs { user }: &WriteArgs,
+  ) -> serror::Result<NoData> {
+    let server = resource::get_check_permissions::<Server>(
+      &self.server,
+      user,
+      PermissionLevel::Write,
+    )
+    .await?;
+
+    let periphery = periphery_client(&server)?;
+
+    periphery
+      .request(api::terminal::CreateTerminal {
+        name: self.name,
+        command: self.command,
+        recreate: self.recreate,
+      })
+      .await
+      .context("Failed to create terminal on periphery")?;
+
+    Ok(NoData {})
+  }
+}
+
+impl Resolve<WriteArgs> for DeleteTerminal {
+  #[instrument(name = "DeleteTerminal", skip(user))]
+  async fn resolve(
+    self,
+    WriteArgs { user }: &WriteArgs,
+  ) -> serror::Result<NoData> {
+    let server = resource::get_check_permissions::<Server>(
+      &self.server,
+      user,
+      PermissionLevel::Write,
+    )
+    .await?;
+
+    let periphery = periphery_client(&server)?;
+
+    periphery
+      .request(api::terminal::DeleteTerminal {
+        terminal: self.terminal,
+      })
+      .await
+      .context("Failed to delete terminal on periphery")?;
+
+    Ok(NoData {})
+  }
+}
+
+impl Resolve<WriteArgs> for DeleteAllTerminals {
+  #[instrument(name = "DeleteAllTerminals", skip(user))]
+  async fn resolve(
+    self,
+    WriteArgs { user }: &WriteArgs,
+  ) -> serror::Result<NoData> {
+    let server = resource::get_check_permissions::<Server>(
+      &self.server,
+      user,
+      PermissionLevel::Write,
+    )
+    .await?;
+
+    let periphery = periphery_client(&server)?;
+
+    periphery
+      .request(api::terminal::DeleteAllTerminals {})
+      .await
+      .context("Failed to delete all terminals on periphery")?;
+
+    Ok(NoData {})
+  }
+}

bin/core/src/api/write/sync.rs

@@ -829,7 +829,9 @@ impl Resolve<WriteArgs> for RefreshResourceSyncPending {
             .context("failed to open existing pending resource sync updates alert")
             .inspect_err(|e| warn!("{e:#}"))
             .ok();
-          send_alerts(&[alert]).await;
+          if sync.config.pending_alert {
+            send_alerts(&[alert]).await;
+          }
         }
         // CLOSE ALERT
         (Some(existing), false) => {

bin/core/src/config.rs

@@ -194,6 +194,7 @@ pub fn core_config() -> &'static CoreConfig {
         stdio: env
           .komodo_logging_stdio
           .unwrap_or(config.logging.stdio),
+        pretty: env.komodo_logging_pretty.unwrap_or(config.logging.pretty),
         otlp_endpoint: env
           .komodo_logging_otlp_endpoint
           .unwrap_or(config.logging.otlp_endpoint),

bin/core/src/helpers/query.rs

@@ -1,6 +1,11 @@
-use std::{collections::HashMap, str::FromStr};
+use std::{
+  collections::HashMap,
+  str::FromStr,
+  sync::{Arc, OnceLock},
+};
 
 use anyhow::{Context, anyhow};
+use async_timing_util::{ONE_MIN_MS, unix_timestamp_ms};
 use komodo_client::entities::{
   Operation, ResourceTarget, ResourceTargetVariant,
   action::Action,
@@ -15,6 +20,7 @@ use komodo_client::entities::{
   server::{Server, ServerState},
   server_template::ServerTemplate,
   stack::{Stack, StackServiceNames, StackState},
+  stats::SystemInformation,
   sync::ResourceSync,
   tag::Tag,
   update::Update,
@@ -29,6 +35,8 @@ use mungos::{
     options::FindOneOptions,
   },
 };
+use periphery_client::api::stats;
+use tokio::sync::Mutex;
 
 use crate::{
   config::core_config,
@@ -37,6 +45,8 @@ use crate::{
   state::{db_client, deployment_status_cache, stack_status_cache},
 };
 
+use super::periphery_client;
+
 // user: Id or username
 #[instrument(level = "debug")]
 pub async fn get_user(user: &str) -> anyhow::Result<User> {
@@ -382,3 +392,36 @@ pub async fn get_variables_and_secrets()
 
   Ok(VariablesAndSecrets { variables, secrets })
 }
+
+// This protects the peripheries from spam requests
+const SYSTEM_INFO_EXPIRY: u128 = ONE_MIN_MS;
+type SystemInfoCache =
+  Mutex<HashMap<String, Arc<(SystemInformation, u128)>>>;
+fn system_info_cache() -> &'static SystemInfoCache {
+  static SYSTEM_INFO_CACHE: OnceLock<SystemInfoCache> =
+    OnceLock::new();
+  SYSTEM_INFO_CACHE.get_or_init(Default::default)
+}
+
+pub async fn get_system_info(
+  server: &Server,
+) -> anyhow::Result<SystemInformation> {
+  let mut lock = system_info_cache().lock().await;
+  let res = match lock.get(&server.id) {
+    Some(cached) if cached.1 > unix_timestamp_ms() => {
+      cached.0.clone()
+    }
+    _ => {
+      let stats = periphery_client(server)?
+        .request(stats::GetSystemInformation {})
+        .await?;
+      lock.insert(
+        server.id.clone(),
+        (stats.clone(), unix_timestamp_ms() + SYSTEM_INFO_EXPIRY)
+          .into(),
+      );
+      stats
+    }
+  };
+  Ok(res)
+}

bin/core/src/main.rs

@@ -23,6 +23,7 @@ mod helpers;
 mod listener;
 mod monitor;
 mod resource;
+mod schedule;
 mod stack;
 mod state;
 mod sync;
@@ -33,6 +34,12 @@ async fn app() -> anyhow::Result<()> {
   dotenvy::dotenv().ok();
   let config = core_config();
   logger::init(&config.logging)?;
+  if let Err(e) =
+    rustls::crypto::aws_lc_rs::default_provider().install_default()
+  {
+    error!("Failed to install default crypto provider | {e:?}");
+    std::process::exit(1);
+  };
 
   info!("Komodo Core version: v{}", env!("CARGO_PKG_VERSION"));
   info!("{:?}", config.sanitized());
@@ -59,6 +66,7 @@ async fn app() -> anyhow::Result<()> {
   resource::spawn_repo_state_refresh_loop();
   resource::spawn_procedure_state_refresh_loop();
   resource::spawn_action_state_refresh_loop();
+  schedule::spawn_schedule_executor();
   helpers::prune::spawn_prune_loop();
 
   // Setup static frontend services
@@ -86,9 +94,9 @@ async fn app() -> anyhow::Result<()> {
     )
     .into_make_service();
 
-  let addr = format!("{}:{}", core_config().bind_ip, core_config().port);
-  let socket_addr =
-    SocketAddr::from_str(&addr)
+  let addr =
+    format!("{}:{}", core_config().bind_ip, core_config().port);
+  let socket_addr = SocketAddr::from_str(&addr)
     .context("failed to parse listen address")?;
 
   if config.ssl_enabled {

bin/core/src/resource/action.rs

@@ -2,7 +2,7 @@ use std::time::Duration;
 
 use anyhow::Context;
 use komodo_client::entities::{
-  Operation, ResourceTargetVariant,
+  Operation, ResourceTarget, ResourceTargetVariant,
   action::{
     Action, ActionConfig, ActionConfigDiff, ActionInfo,
     ActionListItem, ActionListItemInfo, ActionQuerySpecifics,
@@ -17,7 +17,12 @@ use mungos::{
   mongodb::{Collection, bson::doc, options::FindOneOptions},
 };
 
-use crate::state::{action_state_cache, action_states, db_client};
+use crate::{
+  schedule::{
+    cancel_schedule, get_schedule_item_info, update_schedule,
+  },
+  state::{action_state_cache, action_states, db_client},
+};
 
 impl super::KomodoResource for Action {
   type Config = ActionConfig;
@@ -31,6 +36,10 @@ impl super::KomodoResource for Action {
     ResourceTargetVariant::Action
   }
 
+  fn resource_target(id: impl Into<String>) -> ResourceTarget {
+    ResourceTarget::Action(id.into())
+  }
+
   fn coll() -> &'static Collection<Resource<Self::Config, Self::Info>>
   {
     &db_client().actions
@@ -40,6 +49,9 @@ impl super::KomodoResource for Action {
     action: Resource<Self::Config, Self::Info>,
   ) -> Self::ListItem {
     let state = get_action_state(&action.id).await;
+    let (next_scheduled_run, schedule_error) = get_schedule_item_info(
+      &ResourceTarget::Action(action.id.clone()),
+    );
     ActionListItem {
       name: action.name,
       id: action.id,
@@ -48,6 +60,8 @@ impl super::KomodoResource for Action {
       info: ActionListItemInfo {
         state,
         last_run_at: action.info.last_run_at,
+        next_scheduled_run,
+        schedule_error,
       },
     }
   }
@@ -83,9 +97,10 @@ impl super::KomodoResource for Action {
   }
 
   async fn post_create(
-    _created: &Resource<Self::Config, Self::Info>,
+    created: &Resource<Self::Config, Self::Info>,
     _update: &mut Update,
   ) -> anyhow::Result<()> {
+    update_schedule(created);
     refresh_action_state_cache().await;
     Ok(())
   }
@@ -131,9 +146,10 @@ impl super::KomodoResource for Action {
   }
 
   async fn post_delete(
-    _resource: &Resource<Self::Config, Self::Info>,
+    resource: &Resource<Self::Config, Self::Info>,
     _update: &mut Update,
   ) -> anyhow::Result<()> {
+    cancel_schedule(&ResourceTarget::Action(resource.id.clone()));
     Ok(())
   }
 }

bin/core/src/resource/alerter.rs

@@ -1,6 +1,6 @@
 use derive_variants::ExtractVariant;
 use komodo_client::entities::{
-  Operation, ResourceTargetVariant,
+  Operation, ResourceTarget, ResourceTargetVariant,
   alerter::{
     Alerter, AlerterConfig, AlerterConfigDiff, AlerterListItem,
     AlerterListItemInfo, AlerterQuerySpecifics, PartialAlerterConfig,
@@ -25,6 +25,10 @@ impl super::KomodoResource for Alerter {
     ResourceTargetVariant::Alerter
   }
 
+  fn resource_target(id: impl Into<String>) -> ResourceTarget {
+    ResourceTarget::Alerter(id.into())
+  }
+
   fn coll() -> &'static Collection<Resource<Self::Config, Self::Info>>
   {
     &db_client().alerters

bin/core/src/resource/build.rs

@@ -5,7 +5,7 @@ use formatting::format_serror;
 use komodo_client::{
   api::write::RefreshBuildCache,
   entities::{
-    Operation, ResourceTargetVariant,
+    Operation, ResourceTarget, ResourceTargetVariant,
     build::{
       Build, BuildConfig, BuildConfigDiff, BuildInfo, BuildListItem,
       BuildListItemInfo, BuildQuerySpecifics, BuildState,
@@ -44,6 +44,10 @@ impl super::KomodoResource for Build {
     ResourceTargetVariant::Build
   }
 
+  fn resource_target(id: impl Into<String>) -> ResourceTarget {
+    ResourceTarget::Build(id.into())
+  }
+
   fn coll() -> &'static Collection<Resource<Self::Config, Self::Info>>
   {
     &db_client().builds

bin/core/src/resource/builder.rs

@@ -1,6 +1,6 @@
 use anyhow::Context;
 use komodo_client::entities::{
-  MergePartial, Operation, ResourceTargetVariant,
+  MergePartial, Operation, ResourceTarget, ResourceTargetVariant,
   builder::{
     Builder, BuilderConfig, BuilderConfigDiff, BuilderConfigVariant,
     BuilderListItem, BuilderListItemInfo, BuilderQuerySpecifics,
@@ -31,6 +31,10 @@ impl super::KomodoResource for Builder {
     ResourceTargetVariant::Builder
   }
 
+  fn resource_target(id: impl Into<String>) -> ResourceTarget {
+    ResourceTarget::Builder(id.into())
+  }
+
   fn coll() -> &'static Collection<Resource<Self::Config, Self::Info>>
   {
     &db_client().builders

bin/core/src/resource/deployment.rs

@@ -1,7 +1,7 @@
 use anyhow::Context;
 use formatting::format_serror;
 use komodo_client::entities::{
-  Operation, ResourceTargetVariant,
+  Operation, ResourceTarget, ResourceTargetVariant,
   build::Build,
   deployment::{
     Deployment, DeploymentConfig, DeploymentConfigDiff,
@@ -43,6 +43,10 @@ impl super::KomodoResource for Deployment {
     ResourceTargetVariant::Deployment
   }
 
+  fn resource_target(id: impl Into<String>) -> ResourceTarget {
+    ResourceTarget::Deployment(id.into())
+  }
+
   fn coll() -> &'static Collection<Resource<Self::Config, Self::Info>>
   {
     &db_client().deployments

bin/core/src/resource/mod.rs

@@ -29,7 +29,7 @@ use mungos::{
     options::FindOptions,
   },
 };
-use partial_derive2::{Diff, FieldDiff, MaybeNone, PartialDiff};
+use partial_derive2::{Diff, MaybeNone, PartialDiff};
 use resolver_api::Resolve;
 use serde::{Serialize, de::DeserializeOwned};
 
@@ -107,6 +107,7 @@ pub trait KomodoResource {
   type QuerySpecifics: AddFilters + Default + std::fmt::Debug;
 
   fn resource_type() -> ResourceTargetVariant;
+  fn resource_target(id: impl Into<String>) -> ResourceTarget;
 
   fn coll() -> &'static Collection<Resource<Self::Config, Self::Info>>;
 
@@ -693,13 +694,17 @@ pub async fn update<T: KomodoResource>(
     return Ok(resource);
   }
 
-  let mut diff_log = String::from("diff");
-
-  for FieldDiff { field, from, to } in diff.iter_field_diffs() {
-    diff_log.push_str(&format!(
-      "\n\n<span class=\"text-muted-foreground\">field</span>: '{field}'\n<span class=\"text-muted-foreground\">from</span>:  <span class=\"text-red-700 dark:text-red-400\">{from}</span>\n<span class=\"text-muted-foreground\">to</span>:    <span class=\"text-green-700 dark:text-green-400\">{to}</span>",
-    ));
+  // Leave this Result unhandled for now
+  let prev_toml = ExportResourcesToToml {
+    targets: vec![T::resource_target(&resource.id)],
+    ..Default::default()
   }
+  .resolve(&ReadArgs {
+    user: system_user().to_owned(),
+  })
+  .await
+  .map_err(|e| e.error)
+  .context("Failed to export resource toml before update");
 
   // This minimizes the update against the existing config
   let config: T::PartialConfig = diff.into();
@@ -715,13 +720,35 @@ pub async fn update<T: KomodoResource>(
     .await
     .context("failed to update resource on database")?;
 
+  let curr_toml = ExportResourcesToToml {
+    targets: vec![T::resource_target(&id)],
+    ..Default::default()
+  }
+  .resolve(&ReadArgs {
+    user: system_user().to_owned(),
+  })
+  .await
+  .map_err(|e| e.error)
+  .context("Failed to export resource toml after update");
+
   let mut update = make_update(
     resource_target::<T>(id),
     T::update_operation(),
     user,
   );
 
-  update.push_simple_log("update config", diff_log);
+  match prev_toml {
+    Ok(res) => update.prev_toml = res.toml,
+    Err(e) => update
+      // These logs are pushed with success == true, so user still knows the update was succesful.
+      .push_simple_log("Failed export", format_serror(&e.into())),
+  }
+  match curr_toml {
+    Ok(res) => update.current_toml = res.toml,
+    Err(e) => update
+      // These logs are pushed with success == true, so user still knows the update was succesful.
+      .push_simple_log("Failed export", format_serror(&e.into())),
+  }
 
   let updated = get::<T>(id_or_name).await?;
 

bin/core/src/resource/procedure.rs

@@ -4,7 +4,7 @@ use anyhow::{Context, anyhow};
 use komodo_client::{
   api::execute::Execution,
   entities::{
-    Operation, ResourceTargetVariant,
+    Operation, ResourceTarget, ResourceTargetVariant,
     action::Action,
     alerter::Alerter,
     build::Build,
@@ -31,6 +31,9 @@ use mungos::{
 
 use crate::{
   config::core_config,
+  schedule::{
+    cancel_schedule, get_schedule_item_info, update_schedule,
+  },
   state::{action_states, db_client, procedure_state_cache},
 };
 
@@ -46,6 +49,10 @@ impl super::KomodoResource for Procedure {
     ResourceTargetVariant::Procedure
   }
 
+  fn resource_target(id: impl Into<String>) -> ResourceTarget {
+    ResourceTarget::Procedure(id.into())
+  }
+
   fn coll() -> &'static Collection<Resource<Self::Config, Self::Info>>
   {
     &db_client().procedures
@@ -55,6 +62,9 @@ impl super::KomodoResource for Procedure {
     procedure: Resource<Self::Config, Self::Info>,
   ) -> Self::ListItem {
     let state = get_procedure_state(&procedure.id).await;
+    let (next_scheduled_run, schedule_error) = get_schedule_item_info(
+      &ResourceTarget::Procedure(procedure.id.clone()),
+    );
     ProcedureListItem {
       name: procedure.name,
       id: procedure.id,
@@ -63,6 +73,8 @@ impl super::KomodoResource for Procedure {
       info: ProcedureListItemInfo {
         stages: procedure.config.stages.len() as i64,
         state,
+        next_scheduled_run,
+        schedule_error,
       },
     }
   }
@@ -94,9 +106,10 @@ impl super::KomodoResource for Procedure {
   }
 
   async fn post_create(
-    _created: &Resource<Self::Config, Self::Info>,
+    created: &Resource<Self::Config, Self::Info>,
     _update: &mut Update,
   ) -> anyhow::Result<()> {
+    update_schedule(created);
     refresh_procedure_state_cache().await;
     Ok(())
   }
@@ -142,9 +155,10 @@ impl super::KomodoResource for Procedure {
   }
 
   async fn post_delete(
-    _resource: &Resource<Self::Config, Self::Info>,
+    resource: &Resource<Self::Config, Self::Info>,
     _update: &mut Update,
   ) -> anyhow::Result<()> {
+    cancel_schedule(&ResourceTarget::Procedure(resource.id.clone()));
     Ok(())
   }
 }

bin/core/src/resource/repo.rs

@@ -3,7 +3,7 @@ use std::time::Duration;
 use anyhow::Context;
 use formatting::format_serror;
 use komodo_client::entities::{
-  Operation, ResourceTargetVariant,
+  Operation, ResourceTarget, ResourceTargetVariant,
   builder::Builder,
   permission::PermissionLevel,
   repo::{
@@ -44,6 +44,10 @@ impl super::KomodoResource for Repo {
     ResourceTargetVariant::Repo
   }
 
+  fn resource_target(id: impl Into<String>) -> ResourceTarget {
+    ResourceTarget::Repo(id.into())
+  }
+
   fn coll() -> &'static Collection<Resource<Self::Config, Self::Info>>
   {
     &db_client().repos

bin/core/src/resource/server.rs

@@ -1,6 +1,6 @@
 use anyhow::Context;
 use komodo_client::entities::{
-  Operation, ResourceTargetVariant, komodo_timestamp,
+  Operation, ResourceTarget, ResourceTargetVariant, komodo_timestamp,
   resource::Resource,
   server::{
     PartialServerConfig, Server, ServerConfig, ServerConfigDiff,
@@ -13,6 +13,7 @@ use mungos::mongodb::{Collection, bson::doc};
 
 use crate::{
   config::core_config,
+  helpers::query::get_system_info,
   monitor::update_cache_for_server,
   state::{action_states, db_client, server_status_cache},
 };
@@ -29,6 +30,10 @@ impl super::KomodoResource for Server {
     ResourceTargetVariant::Server
   }
 
+  fn resource_target(id: impl Into<String>) -> ResourceTarget {
+    ResourceTarget::Server(id.into())
+  }
+
   fn coll() -> &'static Collection<Resource<Self::Config, Self::Info>>
   {
     &db_client().servers
@@ -38,6 +43,10 @@ impl super::KomodoResource for Server {
     server: Resource<Self::Config, Self::Info>,
   ) -> Self::ListItem {
     let status = server_status_cache().get(&server.id).await;
+    let terminals_disabled = get_system_info(&server)
+      .await
+      .map(|i| i.terminals_disabled)
+      .unwrap_or(true);
     ServerListItem {
       name: server.name,
       id: server.id,
@@ -53,6 +62,7 @@ impl super::KomodoResource for Server {
         send_cpu_alerts: server.config.send_cpu_alerts,
         send_mem_alerts: server.config.send_mem_alerts,
         send_disk_alerts: server.config.send_disk_alerts,
+        terminals_disabled,
       },
     }
   }

bin/core/src/resource/server_template.rs

@@ -1,5 +1,5 @@
 use komodo_client::entities::{
-  MergePartial, Operation, ResourceTargetVariant,
+  MergePartial, Operation, ResourceTarget, ResourceTargetVariant,
   resource::Resource,
   server_template::{
     PartialServerTemplateConfig, ServerTemplate,
@@ -29,6 +29,10 @@ impl super::KomodoResource for ServerTemplate {
     ResourceTargetVariant::ServerTemplate
   }
 
+  fn resource_target(id: impl Into<String>) -> ResourceTarget {
+    ResourceTarget::ServerTemplate(id.into())
+  }
+
   fn coll() -> &'static Collection<Resource<Self::Config, Self::Info>>
   {
     &db_client().server_templates

bin/core/src/resource/stack.rs

@@ -3,7 +3,7 @@ use formatting::format_serror;
 use komodo_client::{
   api::write::RefreshStackCache,
   entities::{
-    Operation, ResourceTargetVariant,
+    Operation, ResourceTarget, ResourceTargetVariant,
     permission::PermissionLevel,
     resource::Resource,
     server::Server,
@@ -44,6 +44,10 @@ impl super::KomodoResource for Stack {
     ResourceTargetVariant::Stack
   }
 
+  fn resource_target(id: impl Into<String>) -> ResourceTarget {
+    ResourceTarget::Stack(id.into())
+  }
+
   fn coll() -> &'static Collection<Resource<Self::Config, Self::Info>>
   {
     &db_client().stacks

bin/core/src/resource/sync.rs

@@ -3,7 +3,8 @@ use formatting::format_serror;
 use komodo_client::{
   api::write::RefreshResourceSyncPending,
   entities::{
-    Operation, ResourceTargetVariant, komodo_timestamp,
+    Operation, ResourceTarget, ResourceTargetVariant,
+    komodo_timestamp,
     resource::Resource,
     sync::{
       PartialResourceSyncConfig, ResourceSync, ResourceSyncConfig,
@@ -36,6 +37,10 @@ impl super::KomodoResource for ResourceSync {
     ResourceTargetVariant::ResourceSync
   }
 
+  fn resource_target(id: impl Into<String>) -> ResourceTarget {
+    ResourceTarget::ResourceSync(id.into())
+  }
+
   fn coll() -> &'static Collection<Resource<Self::Config, Self::Info>>
   {
     &db_client().resource_syncs

bin/core/src/schedule.rs

@@ -0,0 +1,378 @@
+use std::{
+  collections::HashMap,
+  sync::{OnceLock, RwLock},
+};
+
+use anyhow::{Context, anyhow};
+use async_timing_util::Timelength;
+use chrono::Local;
+use formatting::format_serror;
+use komodo_client::{
+  api::execute::{RunAction, RunProcedure},
+  entities::{
+    ResourceTarget, ResourceTargetVariant, ScheduleFormat,
+    action::Action,
+    alert::{Alert, AlertData, SeverityLevel},
+    komodo_timestamp,
+    procedure::Procedure,
+    user::{action_user, procedure_user},
+  },
+};
+use mungos::find::find_collect;
+use resolver_api::Resolve;
+
+use crate::{
+  alert::send_alerts,
+  api::execute::{ExecuteArgs, ExecuteRequest},
+  helpers::update::init_execution_update,
+  state::db_client,
+};
+
+pub fn spawn_schedule_executor() {
+  // Executor thread
+  tokio::spawn(async move {
+    loop {
+      let current_time = async_timing_util::wait_until_timelength(
+        Timelength::OneSecond,
+        0,
+      )
+      .await as i64;
+      let mut lock = schedules().write().unwrap();
+      let drained = lock.drain().collect::<Vec<_>>();
+      for (target, next_run) in drained {
+        match next_run {
+          Ok(next_run_time) if current_time >= next_run_time => {
+            tokio::spawn(async move {
+              match &target {
+                ResourceTarget::Action(id) => {
+                  let action = match crate::resource::get::<Action>(
+                    id,
+                  )
+                  .await
+                  {
+                    Ok(action) => action,
+                    Err(e) => {
+                      warn!(
+                        "Scheduled action run on {id} failed | failed to get procedure | {e:?}"
+                      );
+                      return;
+                    }
+                  };
+                  let request =
+                    ExecuteRequest::RunAction(RunAction {
+                      action: id.clone(),
+                    });
+                  let update = match init_execution_update(
+                    &request,
+                    action_user(),
+                  )
+                  .await
+                  {
+                    Ok(update) => update,
+                    Err(e) => {
+                      error!(
+                        "Failed to make update for scheduled action run, action {id} is not being run | {e:#}"
+                      );
+                      return;
+                    }
+                  };
+                  let ExecuteRequest::RunAction(request) = request
+                  else {
+                    unreachable!()
+                  };
+                  if let Err(e) = request
+                    .resolve(&ExecuteArgs {
+                      user: action_user().to_owned(),
+                      update,
+                    })
+                    .await
+                  {
+                    warn!(
+                      "Scheduled action run on {id} failed | {e:?}"
+                    );
+                  }
+                  update_schedule(&action);
+                  if action.config.schedule_alert {
+                    let alert = Alert {
+                      id: Default::default(),
+                      target,
+                      ts: komodo_timestamp(),
+                      resolved_ts: Some(komodo_timestamp()),
+                      resolved: true,
+                      level: SeverityLevel::Ok,
+                      data: AlertData::ScheduleRun {
+                        resource_type: ResourceTargetVariant::Action,
+                        id: action.id,
+                        name: action.name,
+                      },
+                    };
+                    send_alerts(&[alert]).await
+                  }
+                }
+                ResourceTarget::Procedure(id) => {
+                  let procedure = match crate::resource::get::<
+                    Procedure,
+                  >(id)
+                  .await
+                  {
+                    Ok(procedure) => procedure,
+                    Err(e) => {
+                      warn!(
+                        "Scheduled procedure run on {id} failed | failed to get procedure | {e:?}"
+                      );
+                      return;
+                    }
+                  };
+                  let request =
+                    ExecuteRequest::RunProcedure(RunProcedure {
+                      procedure: id.clone(),
+                    });
+                  let update = match init_execution_update(
+                    &request,
+                    procedure_user(),
+                  )
+                  .await
+                  {
+                    Ok(update) => update,
+                    Err(e) => {
+                      error!(
+                        "Failed to make update for scheduled procedure run, procedure {id} is not being run | {e:#}"
+                      );
+                      return;
+                    }
+                  };
+                  let ExecuteRequest::RunProcedure(request) = request
+                  else {
+                    unreachable!()
+                  };
+                  if let Err(e) = request
+                    .resolve(&ExecuteArgs {
+                      user: procedure_user().to_owned(),
+                      update,
+                    })
+                    .await
+                  {
+                    warn!(
+                      "Scheduled procedure run on {id} failed | {e:?}"
+                    );
+                  }
+                  update_schedule(&procedure);
+                  if procedure.config.schedule_alert {
+                    let alert = Alert {
+                      id: Default::default(),
+                      target,
+                      ts: komodo_timestamp(),
+                      resolved_ts: Some(komodo_timestamp()),
+                      resolved: true,
+                      level: SeverityLevel::Ok,
+                      data: AlertData::ScheduleRun {
+                        resource_type:
+                          ResourceTargetVariant::Procedure,
+                        id: procedure.id,
+                        name: procedure.name,
+                      },
+                    };
+                    send_alerts(&[alert]).await
+                  }
+                }
+                _ => unreachable!(),
+              }
+            });
+          }
+          other => {
+            lock.insert(target, other);
+            continue;
+          }
+        };
+      }
+    }
+  });
+  // Updater thread
+  tokio::spawn(async move {
+    update_schedules().await;
+    loop {
+      async_timing_util::wait_until_timelength(
+        Timelength::FiveMinutes,
+        500,
+      )
+      .await;
+      update_schedules().await
+    }
+  });
+}
+
+type UnixTimestampMs = i64;
+type Schedules =
+  HashMap<ResourceTarget, Result<UnixTimestampMs, String>>;
+
+fn schedules() -> &'static RwLock<Schedules> {
+  static SCHEDULES: OnceLock<RwLock<Schedules>> = OnceLock::new();
+  SCHEDULES.get_or_init(Default::default)
+}
+
+pub fn get_schedule_item_info(
+  target: &ResourceTarget,
+) -> (Option<i64>, Option<String>) {
+  match schedules().read().unwrap().get(target) {
+    Some(Ok(time)) => (Some(*time), None),
+    Some(Err(e)) => (None, Some(e.clone())),
+    None => (None, None),
+  }
+}
+
+pub fn cancel_schedule(target: &ResourceTarget) {
+  schedules().write().unwrap().remove(target);
+}
+
+pub async fn update_schedules() {
+  let (procedures, actions) = tokio::join!(
+    find_collect(&db_client().procedures, None, None),
+    find_collect(&db_client().actions, None, None),
+  );
+  let procedures = match procedures
+    .context("failed to get all procedures from db")
+  {
+    Ok(procedures) => procedures,
+    Err(e) => {
+      error!("failed to get procedures for schedule update | {e:#}");
+      Vec::new()
+    }
+  };
+  let actions =
+    match actions.context("failed to get all actions from db") {
+      Ok(actions) => actions,
+      Err(e) => {
+        error!("failed to get actions for schedule update | {e:#}");
+        Vec::new()
+      }
+    };
+  // clear out any schedules which don't match to existing resources
+  {
+    let mut lock = schedules().write().unwrap();
+    lock.retain(|target, _| match target {
+      ResourceTarget::Action(id) => {
+        actions.iter().any(|action| &action.id == id)
+      }
+      ResourceTarget::Procedure(id) => {
+        procedures.iter().any(|procedure| &procedure.id == id)
+      }
+      _ => unreachable!(),
+    });
+  }
+  for procedure in procedures {
+    update_schedule(&procedure);
+  }
+  for action in actions {
+    update_schedule(&action);
+  }
+}
+
+/// Re/spawns the schedule for the given procedure
+pub fn update_schedule(schedule: impl HasSchedule) {
+  // Cancel any existing schedule for the procedure
+  cancel_schedule(&schedule.target());
+
+  if !schedule.enabled() || schedule.schedule().is_empty() {
+    return;
+  }
+
+  schedules().write().unwrap().insert(
+    schedule.target(),
+    find_next_occurrence(schedule)
+      .map_err(|e| format_serror(&e.into())),
+  );
+}
+
+/// Finds the next run occurence in UTC ms.
+fn find_next_occurrence(
+  schedule: impl HasSchedule,
+) -> anyhow::Result<i64> {
+  let cron = match schedule.format() {
+    ScheduleFormat::Cron => croner::Cron::new(schedule.schedule())
+      .with_seconds_required()
+      .with_dom_and_dow()
+      .parse()
+      .context("Failed to parse schedule CRON")?,
+    ScheduleFormat::English => {
+      let cron =
+        english_to_cron::str_cron_syntax(schedule.schedule())
+          .map_err(|e| {
+            anyhow!("Failed to parse english to cron | {e:?}")
+          })?
+          .split(' ')
+          // croner does not accept year
+          .take(6)
+          .collect::<Vec<_>>()
+          .join(" ");
+      croner::Cron::new(&cron)
+        .with_seconds_required()
+        .with_dom_and_dow()
+        .parse()
+        .with_context(|| {
+          format!("Failed to parse schedule CRON: {cron}")
+        })?
+    }
+  };
+  let next = if schedule.timezone().is_empty() {
+    let tz_time = chrono::Local::now().with_timezone(&Local);
+    cron
+      .find_next_occurrence(&tz_time, false)
+      .context("Failed to find next run time")?
+      .timestamp_millis()
+  } else {
+    let tz: chrono_tz::Tz = schedule
+      .timezone()
+      .parse()
+      .context("Failed to parse schedule timezone")?;
+    let tz_time = chrono::Local::now().with_timezone(&tz);
+    cron
+      .find_next_occurrence(&tz_time, false)
+      .context("Failed to find next run time")?
+      .timestamp_millis()
+  };
+  Ok(next)
+}
+
+pub trait HasSchedule {
+  fn target(&self) -> ResourceTarget;
+  fn enabled(&self) -> bool;
+  fn format(&self) -> ScheduleFormat;
+  fn schedule(&self) -> &str;
+  fn timezone(&self) -> &str;
+}
+
+impl HasSchedule for &Procedure {
+  fn target(&self) -> ResourceTarget {
+    ResourceTarget::Procedure(self.id.clone())
+  }
+  fn enabled(&self) -> bool {
+    self.config.schedule_enabled
+  }
+  fn format(&self) -> ScheduleFormat {
+    self.config.schedule_format
+  }
+  fn schedule(&self) -> &str {
+    &self.config.schedule
+  }
+  fn timezone(&self) -> &str {
+    &self.config.schedule_timezone
+  }
+}
+
+impl HasSchedule for &Action {
+  fn target(&self) -> ResourceTarget {
+    ResourceTarget::Action(self.id.clone())
+  }
+  fn enabled(&self) -> bool {
+    self.config.schedule_enabled
+  }
+  fn format(&self) -> ScheduleFormat {
+    self.config.schedule_format
+  }
+  fn schedule(&self) -> &str {
+    &self.config.schedule
+  }
+  fn timezone(&self) -> &str {
+    &self.config.schedule_timezone
+  }
+}

bin/core/src/sync/deploy.rs

@@ -424,7 +424,7 @@ fn build_cache_for_deployment<'a>(
         let deployed_version = status
           .container
           .as_ref()
-          .and_then(|c| c.image.as_ref()?.split(':').last())
+          .and_then(|c| c.image.as_ref()?.split(':').next_back())
           .unwrap_or("0.0.0");
         match build_version_cache.get(build_id) {
           Some(version) if deployed_version != version => {

bin/core/src/sync/mod.rs

@@ -2,7 +2,7 @@ use std::{collections::HashMap, str::FromStr};
 
 use anyhow::anyhow;
 use komodo_client::entities::{
-  ResourceTarget, ResourceTargetVariant,
+  ResourceTargetVariant,
   action::Action,
   alerter::Alerter,
   build::Build,
@@ -55,8 +55,6 @@ pub struct ToUpdateItem<T: Default> {
 }
 
 pub trait ResourceSyncTrait: ToToml + Sized {
-  fn resource_target(id: String) -> ResourceTarget;
-
   /// To exclude resource syncs with "file_contents" (they aren't compatible)
   fn include_resource(
     name: &String,

bin/core/src/sync/resources.rs

@@ -4,7 +4,7 @@ use formatting::{Color, bold, colored, muted};
 use komodo_client::{
   api::execute::Execution,
   entities::{
-    ResourceTarget, ResourceTargetVariant,
+    ResourceTargetVariant,
     action::Action,
     alerter::Alerter,
     build::Build,
@@ -40,10 +40,6 @@ use super::{
 };
 
 impl ResourceSyncTrait for Server {
-  fn resource_target(id: String) -> ResourceTarget {
-    ResourceTarget::Server(id)
-  }
-
   fn get_diff(
     original: Self::Config,
     update: Self::PartialConfig,
@@ -56,10 +52,6 @@ impl ResourceSyncTrait for Server {
 impl ExecuteResourceSync for Server {}
 
 impl ResourceSyncTrait for Deployment {
-  fn resource_target(id: String) -> ResourceTarget {
-    ResourceTarget::Deployment(id)
-  }
-
   fn get_diff(
     mut original: Self::Config,
     update: Self::PartialConfig,
@@ -93,10 +85,6 @@ impl ResourceSyncTrait for Deployment {
 impl ExecuteResourceSync for Deployment {}
 
 impl ResourceSyncTrait for Stack {
-  fn resource_target(id: String) -> ResourceTarget {
-    ResourceTarget::Stack(id)
-  }
-
   fn get_diff(
     mut original: Self::Config,
     update: Self::PartialConfig,
@@ -116,10 +104,6 @@ impl ResourceSyncTrait for Stack {
 impl ExecuteResourceSync for Stack {}
 
 impl ResourceSyncTrait for Build {
-  fn resource_target(id: String) -> ResourceTarget {
-    ResourceTarget::Build(id)
-  }
-
   fn get_diff(
     mut original: Self::Config,
     update: Self::PartialConfig,
@@ -149,10 +133,6 @@ impl ResourceSyncTrait for Build {
 impl ExecuteResourceSync for Build {}
 
 impl ResourceSyncTrait for Repo {
-  fn resource_target(id: String) -> ResourceTarget {
-    ResourceTarget::Repo(id)
-  }
-
   fn get_diff(
     mut original: Self::Config,
     update: Self::PartialConfig,
@@ -179,10 +159,6 @@ impl ResourceSyncTrait for Repo {
 impl ExecuteResourceSync for Repo {}
 
 impl ResourceSyncTrait for Alerter {
-  fn resource_target(id: String) -> ResourceTarget {
-    ResourceTarget::Alerter(id)
-  }
-
   fn get_diff(
     original: Self::Config,
     update: Self::PartialConfig,
@@ -195,10 +171,6 @@ impl ResourceSyncTrait for Alerter {
 impl ExecuteResourceSync for Alerter {}
 
 impl ResourceSyncTrait for Builder {
-  fn resource_target(id: String) -> ResourceTarget {
-    ResourceTarget::Builder(id)
-  }
-
   fn get_diff(
     mut original: Self::Config,
     update: Self::PartialConfig,
@@ -220,10 +192,6 @@ impl ResourceSyncTrait for Builder {
 impl ExecuteResourceSync for Builder {}
 
 impl ResourceSyncTrait for ServerTemplate {
-  fn resource_target(id: String) -> ResourceTarget {
-    ResourceTarget::ServerTemplate(id)
-  }
-
   fn get_diff(
     original: Self::Config,
     update: Self::PartialConfig,
@@ -236,10 +204,6 @@ impl ResourceSyncTrait for ServerTemplate {
 impl ExecuteResourceSync for ServerTemplate {}
 
 impl ResourceSyncTrait for Action {
-  fn resource_target(id: String) -> ResourceTarget {
-    ResourceTarget::Action(id)
-  }
-
   fn get_diff(
     original: Self::Config,
     update: Self::PartialConfig,
@@ -252,10 +216,6 @@ impl ResourceSyncTrait for Action {
 impl ExecuteResourceSync for Action {}
 
 impl ResourceSyncTrait for ResourceSync {
-  fn resource_target(id: String) -> ResourceTarget {
-    ResourceTarget::ResourceSync(id)
-  }
-
   fn include_resource(
     name: &String,
     config: &Self::Config,
@@ -341,10 +301,6 @@ impl ResourceSyncTrait for ResourceSync {
 impl ExecuteResourceSync for ResourceSync {}
 
 impl ResourceSyncTrait for Procedure {
-  fn resource_target(id: String) -> ResourceTarget {
-    ResourceTarget::Procedure(id)
-  }
-
   fn get_diff(
     mut original: Self::Config,
     update: Self::PartialConfig,

bin/core/src/ws.rs

@@ -1,209 +0,0 @@
-use anyhow::anyhow;
-use axum::{
-  Router,
-  extract::{
-    WebSocketUpgrade,
-    ws::{Message, WebSocket},
-  },
-  response::IntoResponse,
-  routing::get,
-};
-use futures::{SinkExt, StreamExt};
-use komodo_client::{
-  entities::{
-    ResourceTarget, permission::PermissionLevel, user::User,
-  },
-  ws::WsLoginMessage,
-};
-use serde_json::json;
-use serror::serialize_error;
-use tokio::select;
-use tokio_util::sync::CancellationToken;
-
-use crate::{
-  auth::{auth_api_key_check_enabled, auth_jwt_check_enabled},
-  helpers::{
-    channel::update_channel,
-    query::{get_user, get_user_permission_on_target},
-  },
-};
-
-pub fn router() -> Router {
-  Router::new().route("/update", get(ws_handler))
-}
-
-#[instrument(level = "debug")]
-async fn ws_handler(ws: WebSocketUpgrade) -> impl IntoResponse {
-  // get a reveiver for internal update messages.
-  let mut receiver = update_channel().receiver.resubscribe();
-
-  // handle http -> ws updgrade
-  ws.on_upgrade(|socket| async move {
-    let Some((socket, user)) = ws_login(socket).await else {
-      return
-    };
-
-    let (mut ws_sender, mut ws_reciever) = socket.split();
-
-    let cancel = CancellationToken::new();
-    let cancel_clone = cancel.clone();
-
-    tokio::spawn(async move {
-      loop {
-        // poll for updates off the receiver / await cancel.
-        let update = select! {
-          _ = cancel_clone.cancelled() => break,
-          update = receiver.recv() => {update.expect("failed to recv update msg")}
-        };
-
-        // before sending every update, verify user is still valid.
-        // kill the connection is user if found to be invalid.
-        let user = check_user_valid(&user.id).await;
-        let user = match user {
-          Err(e) => {
-            let _ = ws_sender
-              .send(Message::text(json!({ "type": "INVALID_USER", "msg": serialize_error(&e) }).to_string()))
-              .await;
-            let _ = ws_sender.close().await;
-            return;
-          },
-          Ok(user) => user,
-        };
-
-        // Only send if user has permission on the target resource.
-        if user_can_see_update(&user, &update.target).await.is_ok() {
-          let _ = ws_sender
-            .send(Message::text(serde_json::to_string(&update).unwrap()))
-            .await;
-        }
-      }
-    });
-
-    // Handle messages from the client.
-    // After login, only handles close message.
-    while let Some(msg) = ws_reciever.next().await {
-      match msg {
-        Ok(msg) => {
-          if let Message::Close(_) = msg {
-            cancel.cancel();
-            return;
-          }
-        }
-        Err(_) => {
-          cancel.cancel();
-          return;
-        }
-      }
-    }
-    })
-}
-
-#[instrument(level = "debug")]
-async fn ws_login(
-  mut socket: WebSocket,
-) -> Option<(WebSocket, User)> {
-  let login_msg = match socket.recv().await {
-    Some(Ok(Message::Text(login_msg))) => {
-      LoginMessage::Ok(login_msg.to_string())
-    }
-    Some(Ok(msg)) => {
-      LoginMessage::Err(format!("invalid login message: {msg:?}"))
-    }
-    Some(Err(e)) => {
-      LoginMessage::Err(format!("failed to get login message: {e:?}"))
-    }
-    None => {
-      LoginMessage::Err("failed to get login message".to_string())
-    }
-  };
-  let login_msg = match login_msg {
-    LoginMessage::Ok(login_msg) => login_msg,
-    LoginMessage::Err(msg) => {
-      let _ = socket.send(Message::text(msg)).await;
-      let _ = socket.close().await;
-      return None;
-    }
-  };
-  match WsLoginMessage::from_json_str(&login_msg) {
-    // Login using a jwt
-    Ok(WsLoginMessage::Jwt { jwt }) => {
-      match auth_jwt_check_enabled(&jwt).await {
-        Ok(user) => {
-          let _ = socket.send(Message::text("LOGGED_IN")).await;
-          Some((socket, user))
-        }
-        Err(e) => {
-          let _ = socket
-            .send(Message::text(format!(
-              "failed to authenticate user using jwt | {e:#}"
-            )))
-            .await;
-          let _ = socket.close().await;
-          None
-        }
-      }
-    }
-    // login using api keys
-    Ok(WsLoginMessage::ApiKeys { key, secret }) => {
-      match auth_api_key_check_enabled(&key, &secret).await {
-        Ok(user) => {
-          let _ = socket.send(Message::text("LOGGED_IN")).await;
-          Some((socket, user))
-        }
-        Err(e) => {
-          let _ = socket
-            .send(Message::text(format!(
-              "failed to authenticate user using api keys | {e:#}"
-            )))
-            .await;
-          let _ = socket.close().await;
-          None
-        }
-      }
-    }
-    Err(e) => {
-      let _ = socket
-        .send(Message::text(format!(
-          "failed to parse login message: {e:#}"
-        )))
-        .await;
-      let _ = socket.close().await;
-      None
-    }
-  }
-}
-
-enum LoginMessage {
-  /// The text message
-  Ok(String),
-  /// The err message
-  Err(String),
-}
-
-#[instrument(level = "debug")]
-async fn check_user_valid(user_id: &str) -> anyhow::Result<User> {
-  let user = get_user(user_id).await?;
-  if !user.enabled {
-    return Err(anyhow!("user not enabled"));
-  }
-  Ok(user)
-}
-
-#[instrument(level = "debug")]
-async fn user_can_see_update(
-  user: &User,
-  update_target: &ResourceTarget,
-) -> anyhow::Result<()> {
-  if user.admin {
-    return Ok(());
-  }
-  let permissions =
-    get_user_permission_on_target(user, update_target).await?;
-  if permissions > PermissionLevel::None {
-    Ok(())
-  } else {
-    Err(anyhow!(
-      "user does not have permissions on {update_target:?}"
-    ))
-  }
-}

bin/core/src/ws/mod.rs

@@ -0,0 +1,112 @@
+use crate::{
+  auth::{auth_api_key_check_enabled, auth_jwt_check_enabled},
+  helpers::query::get_user,
+};
+use anyhow::anyhow;
+use axum::{
+  Router,
+  extract::ws::{Message, WebSocket},
+  routing::get,
+};
+use futures::SinkExt;
+use komodo_client::{entities::user::User, ws::WsLoginMessage};
+
+mod terminal;
+mod update;
+
+pub fn router() -> Router {
+  Router::new()
+    .route("/update", get(update::handler))
+    .route("/terminal", get(terminal::handler))
+}
+
+#[instrument(level = "debug")]
+async fn ws_login(
+  mut socket: WebSocket,
+) -> Option<(WebSocket, User)> {
+  let login_msg = match socket.recv().await {
+    Some(Ok(Message::Text(login_msg))) => {
+      LoginMessage::Ok(login_msg.to_string())
+    }
+    Some(Ok(msg)) => {
+      LoginMessage::Err(format!("invalid login message: {msg:?}"))
+    }
+    Some(Err(e)) => {
+      LoginMessage::Err(format!("failed to get login message: {e:?}"))
+    }
+    None => {
+      LoginMessage::Err("failed to get login message".to_string())
+    }
+  };
+  let login_msg = match login_msg {
+    LoginMessage::Ok(login_msg) => login_msg,
+    LoginMessage::Err(msg) => {
+      let _ = socket.send(Message::text(msg)).await;
+      let _ = socket.close().await;
+      return None;
+    }
+  };
+  match WsLoginMessage::from_json_str(&login_msg) {
+    // Login using a jwt
+    Ok(WsLoginMessage::Jwt { jwt }) => {
+      match auth_jwt_check_enabled(&jwt).await {
+        Ok(user) => {
+          let _ = socket.send(Message::text("LOGGED_IN")).await;
+          Some((socket, user))
+        }
+        Err(e) => {
+          let _ = socket
+            .send(Message::text(format!(
+              "failed to authenticate user using jwt | {e:#}"
+            )))
+            .await;
+          let _ = socket.close().await;
+          None
+        }
+      }
+    }
+    // login using api keys
+    Ok(WsLoginMessage::ApiKeys { key, secret }) => {
+      match auth_api_key_check_enabled(&key, &secret).await {
+        Ok(user) => {
+          let _ = socket.send(Message::text("LOGGED_IN")).await;
+          Some((socket, user))
+        }
+        Err(e) => {
+          let _ = socket
+            .send(Message::text(format!(
+              "failed to authenticate user using api keys | {e:#}"
+            )))
+            .await;
+          let _ = socket.close().await;
+          None
+        }
+      }
+    }
+    Err(e) => {
+      let _ = socket
+        .send(Message::text(format!(
+          "failed to parse login message: {e:#}"
+        )))
+        .await;
+      let _ = socket.close().await;
+      None
+    }
+  }
+}
+
+enum LoginMessage {
+  /// The text message
+  Ok(String),
+  /// The err message
+  Err(String),
+}
+
+#[instrument(level = "debug")]
+async fn check_user_valid(user_id: &str) -> anyhow::Result<User> {
+  let user = get_user(user_id).await?;
+  if !user.enabled {
+    return Err(anyhow!("user not enabled"));
+  }
+  Ok(user)
+}

bin/core/src/ws/terminal.rs

@@ -0,0 +1,200 @@
+use axum::{
+  extract::{
+    Query, WebSocketUpgrade,
+    ws::{CloseFrame, Message, Utf8Bytes},
+  },
+  response::IntoResponse,
+};
+use futures::{SinkExt, StreamExt};
+use komodo_client::{
+  api::terminal::ConnectTerminalQuery,
+  entities::{permission::PermissionLevel, server::Server},
+};
+use tokio_tungstenite::tungstenite;
+use tokio_util::sync::CancellationToken;
+
+use crate::{helpers::periphery_client, resource};
+
+#[instrument(name = "ConnectTerminal", skip(ws))]
+pub async fn handler(
+  Query(ConnectTerminalQuery {
+    server,
+    terminal,
+    init,
+  }): Query<ConnectTerminalQuery>,
+  ws: WebSocketUpgrade,
+) -> impl IntoResponse {
+  ws.on_upgrade(|socket| async move {
+    let Some((mut socket, user)) = super::ws_login(socket).await
+    else {
+      return;
+    };
+
+    let server = match resource::get_check_permissions::<Server>(
+      &server,
+      &user,
+      PermissionLevel::Write,
+    )
+    .await
+    {
+      Ok(server) => server,
+      Err(e) => {
+        debug!("could not get server | {e:#}");
+        let _ =
+          socket.send(Message::text(format!("ERROR: {e:#}"))).await;
+        let _ = socket.close().await;
+        return;
+      }
+    };
+
+    let periphery = match periphery_client(&server) {
+      Ok(periphery) => periphery,
+      Err(e) => {
+        debug!("couldn't get periphery | {e:#}");
+        let _ =
+          socket.send(Message::text(format!("ERROR: {e:#}"))).await;
+        let _ = socket.close().await;
+        return;
+      }
+    };
+
+    trace!("connecting to periphery terminal");
+
+    let periphery_socket = match periphery
+      .connect_terminal(
+        terminal,
+        init,
+      )
+      .await
+    {
+      Ok(ws) => ws,
+      Err(e) => {
+        debug!("Failed connect to periphery terminal | {e:#}");
+        let _ =
+          socket.send(Message::text(format!("ERROR: {e:#}"))).await;
+        let _ = socket.close().await;
+        return;
+      }
+    };
+
+    trace!("connected to periphery terminal socket");
+
+    let (mut periphery_send, mut periphery_receive) =
+      periphery_socket.split();
+    let (mut core_send, mut core_receive) = socket.split();
+    let cancel = CancellationToken::new();
+
+    trace!("starting ws exchange");
+
+    let core_to_periphery = async {
+      loop {
+        let res = tokio::select! {
+          res = core_receive.next() => res,
+          _ = cancel.cancelled() => {
+            trace!("core to periphery read: cancelled from inside");
+            break;
+          }
+        };
+        match res {
+          Some(Ok(msg)) => {
+            if let Err(e) =
+              periphery_send.send(axum_to_tungstenite(msg)).await
+            {
+              debug!("Failed to send terminal message to {} | {e:?}", server.name);
+              cancel.cancel();
+              break;
+            };
+          }
+          Some(Err(_e)) => {
+            cancel.cancel();
+            break;
+          }
+          None => {
+            cancel.cancel();
+            break;
+          }
+        }
+      }
+    };
+
+    let periphery_to_core = async {
+      loop {
+        let res = tokio::select! {
+          res = periphery_receive.next() => res,
+          _ = cancel.cancelled() => {
+            trace!("periphery to core read: cancelled from inside");
+            break;
+          }
+        };
+        match res {
+          Some(Ok(msg)) => {
+            if let Err(e) =
+              core_send.send(tungstenite_to_axum(msg)).await
+            {
+              debug!("{e:?}");
+              cancel.cancel();
+              break;
+            };
+          }
+          Some(Err(e)) => {
+            let _ = core_send
+              .send(Message::text(format!(
+                "ERROR: Failed to receive message from periphery | {e:?}"
+              )))
+              .await;
+            cancel.cancel();
+            break;
+          }
+          None => {
+            let _ = core_send
+              .send(Message::text("STREAM EOF"))
+              .await;
+            cancel.cancel();
+            break;
+          }
+        }
+      }
+    };
+
+    tokio::join!(core_to_periphery, periphery_to_core);
+  })
+}
+
+fn axum_to_tungstenite(msg: Message) -> tungstenite::Message {
+  match msg {
+    Message::Text(text) => tungstenite::Message::Text(
+      tungstenite::Utf8Bytes::from(text.to_string()),
+    ),
+    Message::Binary(bytes) => tungstenite::Message::Binary(bytes),
+    Message::Ping(bytes) => tungstenite::Message::Ping(bytes),
+    Message::Pong(bytes) => tungstenite::Message::Pong(bytes),
+    Message::Close(close_frame) => {
+      tungstenite::Message::Close(close_frame.map(|cf| {
+        tungstenite::protocol::CloseFrame {
+          code: cf.code.into(),
+          reason: tungstenite::Utf8Bytes::from(cf.reason.to_string()),
+        }
+      }))
+    }
+  }
+}
+
+fn tungstenite_to_axum(msg: tungstenite::Message) -> Message {
+  match msg {
+    tungstenite::Message::Text(text) => {
+      Message::Text(Utf8Bytes::from(text.to_string()))
+    }
+    tungstenite::Message::Binary(bytes) => Message::Binary(bytes),
+    tungstenite::Message::Ping(bytes) => Message::Ping(bytes),
+    tungstenite::Message::Pong(bytes) => Message::Pong(bytes),
+    tungstenite::Message::Close(close_frame) => {
+      Message::Close(close_frame.map(|cf| CloseFrame {
+        code: cf.code.into(),
+        reason: Utf8Bytes::from(cf.reason.to_string()),
+      }))
+    }
+    tungstenite::Message::Frame(_) => {
+      unreachable!()
+    }
+  }
+}

bin/core/src/ws/update.rs

@@ -0,0 +1,102 @@
+use anyhow::anyhow;
+use axum::{
+  extract::{WebSocketUpgrade, ws::Message},
+  response::IntoResponse,
+};
+use futures::{SinkExt, StreamExt};
+use komodo_client::entities::{
+  ResourceTarget, permission::PermissionLevel, user::User,
+};
+use serde_json::json;
+use serror::serialize_error;
+use tokio::select;
+use tokio_util::sync::CancellationToken;
+
+use crate::helpers::{
+  channel::update_channel, query::get_user_permission_on_target,
+};
+
+#[instrument(level = "debug")]
+pub async fn handler(ws: WebSocketUpgrade) -> impl IntoResponse {
+  // get a reveiver for internal update messages.
+  let mut receiver = update_channel().receiver.resubscribe();
+
+  // handle http -> ws updgrade
+  ws.on_upgrade(|socket| async move {
+    let Some((socket, user)) = super::ws_login(socket).await else {
+      return
+    };
+
+    let (mut ws_sender, mut ws_reciever) = socket.split();
+
+    let cancel = CancellationToken::new();
+    let cancel_clone = cancel.clone();
+
+    tokio::spawn(async move {
+      loop {
+        // poll for updates off the receiver / await cancel.
+        let update = select! {
+          _ = cancel_clone.cancelled() => break,
+          update = receiver.recv() => {update.expect("failed to recv update msg")}
+        };
+
+        // before sending every update, verify user is still valid.
+        // kill the connection is user if found to be invalid.
+        let user = super::check_user_valid(&user.id).await;
+        let user = match user {
+          Err(e) => {
+            let _ = ws_sender
+              .send(Message::text(json!({ "type": "INVALID_USER", "msg": serialize_error(&e) }).to_string()))
+              .await;
+            let _ = ws_sender.close().await;
+            return;
+          },
+          Ok(user) => user,
+        };
+
+        // Only send if user has permission on the target resource.
+        if user_can_see_update(&user, &update.target).await.is_ok() {
+          let _ = ws_sender
+            .send(Message::text(serde_json::to_string(&update).unwrap()))
+            .await;
+        }
+      }
+    });
+
+    // Handle messages from the client.
+    // After login, only handles close message.
+    while let Some(msg) = ws_reciever.next().await {
+      match msg {
+        Ok(msg) => {
+          if let Message::Close(_) = msg {
+            cancel.cancel();
+            return;
+          }
+        }
+        Err(_) => {
+          cancel.cancel();
+          return;
+        }
+      }
+    }
+    })
+}
+
+#[instrument(level = "debug")]
+async fn user_can_see_update(
+  user: &User,
+  update_target: &ResourceTarget,
+) -> anyhow::Result<()> {
+  if user.admin {
+    return Ok(());
+  }
+  let permissions =
+    get_user_permission_on_target(user, update_target).await?;
+  if permissions > PermissionLevel::None {
+    Ok(())
+  } else {
+    Err(anyhow!(
+      "user does not have permissions on {update_target:?}"
+    ))
+  }
+}

bin/core/starship.toml

@@ -0,0 +1,67 @@
+## This is used to customize the shell prompt in Periphery container for Terminals
+
+"$schema" = 'https://starship.rs/config-schema.json'
+
+add_newline = true
+
+format = "$time$hostname$container$memory_usage$all"
+
+[character]
+success_symbol = "[❯](bright-blue bold)"
+error_symbol = "[❯](bright-red bold)"
+
+[package]
+disabled = true
+
+[time]
+format = "[❯$time](white dimmed) "
+time_format = "%l:%M %p"
+utc_time_offset = '-5'
+disabled = true
+
+[username]
+format = "[❯ $user]($style) "
+style_user = "bright-green"
+show_always = true
+
+[hostname]
+format = "[❯ $hostname]($style) "
+style = "bright-blue"
+ssh_only = false
+
+[directory]
+format = "[❯ $path]($style)[$read_only]($read_only_style) "
+style = "bright-cyan"
+
+[git_branch]
+format = "[❯ $symbol$branch(:$remote_branch)]($style) "
+style = "bright-purple"
+
+[git_status]
+style = "bright-purple"
+
+[rust]
+format = "[❯ $symbol($version )]($style)"
+symbol = "rustc "
+style = "bright-red"
+
+[nodejs]
+format = "[❯ $symbol($version )]($style)"
+symbol = "nodejs "
+style = "bright-red"
+
+[memory_usage]
+format = "[❯ mem ${ram} ${ram_pct}]($style) "
+threshold = -1
+style = "white"
+
+[cmd_duration]
+format = "[❯ $duration]($style)"
+style = "bright-yellow"
+
+[container]
+format = "[❯ 🦎 core container ]($style)"
+style = "bright-green"
+
+[aws]
+disabled = true

bin/periphery/Cargo.toml

@@ -33,9 +33,11 @@ resolver_api.workspace = true
 run_command.workspace = true
 svi.workspace = true
 # external
+portable-pty.workspace = true
 axum-server.workspace = true
 serde_json.workspace = true
 serde_yaml.workspace = true
+tokio-util.workspace = true
 futures.workspace = true
 tracing.workspace = true
 bollard.workspace = true
@@ -45,7 +47,9 @@ anyhow.workspace = true
 rustls.workspace = true
 tokio.workspace = true
 serde.workspace = true
+bytes.workspace = true
 axum.workspace = true
 clap.workspace = true
 envy.workspace = true
 uuid.workspace = true
+rand.workspace = true

bin/periphery/aio.Dockerfile

@@ -1,6 +1,6 @@
 ## All in one, multi stage compile + runtime Docker build for your architecture.
 
-FROM rust:1.85.1-bullseye AS builder
+FROM rust:1.86.0-bullseye AS builder
 
 WORKDIR /builder
 COPY Cargo.toml Cargo.lock ./
@@ -15,6 +15,7 @@ RUN cargo build -p komodo_periphery --release
 # Final Image
 FROM debian:bullseye-slim
 
+COPY ./bin/periphery/starship.toml /config/starship.toml
 COPY ./bin/periphery/debian-deps.sh .
 RUN sh ./debian-deps.sh && rm ./debian-deps.sh
 

bin/periphery/debian-deps.sh

@@ -1,6 +1,10 @@
 #!/bin/bash
+
+## Periphery deps installer
+
 apt-get update
 apt-get install -y git curl wget ca-certificates
+
 install -m 0755 -d /etc/apt/keyrings
 curl -fsSL https://download.docker.com/linux/debian/gpg -o /etc/apt/keyrings/docker.asc
 chmod a+r /etc/apt/keyrings/docker.asc
@@ -15,4 +19,10 @@ apt-get update
 # apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
 apt-get install -y docker-ce-cli docker-buildx-plugin docker-compose-plugin
 
-rm -rf /var/lib/apt/lists/*
+rm -rf /var/lib/apt/lists/*
+
+# Starship prompt
+curl -sS https://starship.rs/install.sh | sh -s -- --yes --bin-dir /usr/local/bin
+echo 'export STARSHIP_CONFIG=/config/starship.toml' >> /root/.bashrc
+echo 'eval "$(starship init bash)"' >> /root/.bashrc
+

bin/periphery/multi-arch.Dockerfile

@@ -12,6 +12,7 @@ FROM ${AARCH64_BINARIES} AS aarch64
 
 FROM debian:bullseye-slim
 
+COPY ./bin/periphery/starship.toml /config/starship.toml
 COPY ./bin/periphery/debian-deps.sh .
 RUN sh ./debian-deps.sh && rm ./debian-deps.sh
 

bin/periphery/single-arch.Dockerfile

@@ -8,10 +8,10 @@ FROM ${BINARIES_IMAGE} AS binaries
 
 FROM debian:bullseye-slim
 
+COPY ./bin/periphery/starship.toml /config/starship.toml
 COPY ./bin/periphery/debian-deps.sh .
 RUN sh ./debian-deps.sh && rm ./debian-deps.sh
 
-WORKDIR /app
 COPY --from=binaries /periphery /usr/local/bin/periphery
 
 EXPOSE 8120

bin/periphery/src/api/build.rs

@@ -46,7 +46,7 @@ impl Resolve<super::Args> for GetDockerfileContentsOnHost {
     } = self;
 
     let root =
-      periphery_config().build_dir.join(to_komodo_name(&name));
+      periphery_config().build_dir().join(to_komodo_name(&name));
     let build_dir =
       root.join(&build_path).components().collect::<PathBuf>();
 
@@ -91,7 +91,7 @@ impl Resolve<super::Args> for WriteDockerfileContentsToHost {
       contents,
     } = self;
     let full_path = periphery_config()
-      .build_dir
+      .build_dir()
       .join(to_komodo_name(&name))
       .join(&build_path)
       .join(dockerfile_path)
@@ -180,7 +180,7 @@ impl Resolve<super::Args> for build::Build {
     let name = to_komodo_name(name);
 
     let build_path =
-      periphery_config().build_dir.join(&name).join(build_path);
+      periphery_config().build_dir().join(&name).join(build_path);
     let dockerfile_path = optional_string(dockerfile_path)
       .unwrap_or("Dockerfile".to_owned());
 
@@ -248,7 +248,7 @@ impl Resolve<super::Args> for build::Build {
       } {
         let success = log.success;
         logs.push(log);
-        if success {
+        if !success {
           return Ok(logs);
         }
       };

bin/periphery/src/api/compose.rs

@@ -138,7 +138,7 @@ impl Resolve<super::Args> for GetComposeContentsOnHost {
       file_paths,
     } = self;
     let root =
-      periphery_config().stack_dir.join(to_komodo_name(&name));
+      periphery_config().stack_dir().join(to_komodo_name(&name));
     let run_directory =
       root.join(&run_directory).components().collect::<PathBuf>();
 
@@ -196,7 +196,7 @@ impl Resolve<super::Args> for WriteComposeContentsToHost {
       contents,
     } = self;
     let file_path = periphery_config()
-      .stack_dir
+      .stack_dir()
       .join(to_komodo_name(&name))
       .join(&run_directory)
       .join(file_path)

bin/periphery/src/api/git.rs

@@ -20,7 +20,7 @@ impl Resolve<super::Args> for GetLatestCommit {
   ) -> serror::Result<LatestCommit> {
     let repo_path = match self.path {
       Some(p) => PathBuf::from(p),
-      None => periphery_config().repo_dir.join(self.name),
+      None => periphery_config().repo_dir().join(self.name),
     };
     if !repo_path.is_dir() {
       return Err(
@@ -70,13 +70,13 @@ impl Resolve<super::Args> for CloneRepo {
       ),
     };
     let parent_dir = if args.is_build {
-      &periphery_config().build_dir
+      periphery_config().build_dir()
     } else {
-      &periphery_config().repo_dir
+      periphery_config().repo_dir()
     };
     git::clone(
       args,
-      parent_dir,
+      &parent_dir,
       token,
       &environment,
       &env_file_path,
@@ -140,13 +140,13 @@ impl Resolve<super::Args> for PullRepo {
       ),
     };
     let parent_dir = if args.is_build {
-      &periphery_config().build_dir
+      periphery_config().build_dir()
     } else {
-      &periphery_config().repo_dir
+      periphery_config().repo_dir()
     };
     git::pull(
       args,
-      parent_dir,
+      &parent_dir,
       token,
       &environment,
       &env_file_path,
@@ -210,13 +210,13 @@ impl Resolve<super::Args> for PullOrCloneRepo {
       ),
     };
     let parent_dir = if args.is_build {
-      &periphery_config().build_dir
+      periphery_config().build_dir()
     } else {
-      &periphery_config().repo_dir
+      periphery_config().repo_dir()
     };
     git::pull_or_clone(
       args,
-      parent_dir,
+      &parent_dir,
       token,
       &environment,
       &env_file_path,
@@ -252,11 +252,10 @@ impl Resolve<super::Args> for RenameRepo {
       curr_name,
       new_name,
     } = self;
-    let renamed = fs::rename(
-      periphery_config().repo_dir.join(&curr_name),
-      periphery_config().repo_dir.join(&new_name),
-    )
-    .await;
+    let repo_dir = periphery_config().repo_dir();
+    let renamed =
+      fs::rename(repo_dir.join(&curr_name), repo_dir.join(&new_name))
+        .await;
     let msg = match renamed {
       Ok(_) => String::from("Renamed Repo directory on Server"),
       Err(_) => format!("No Repo cloned at {curr_name} to rename"),
@@ -274,9 +273,9 @@ impl Resolve<super::Args> for DeleteRepo {
     // If using custom clone path, it will be passed by core instead of name.
     // So the join will resolve to just the absolute path.
     let root = if is_build {
-      &periphery_config().build_dir
+      periphery_config().build_dir()
     } else {
-      &periphery_config().repo_dir
+      periphery_config().repo_dir()
     };
     let full_path = root.join(&name);
     let deleted =

bin/periphery/src/api/mod.rs

@@ -8,11 +8,8 @@ use komodo_client::entities::{
   update::Log,
 };
 use periphery_client::api::{
-  GetDockerLists, GetDockerListsResponse, GetHealth,
-  GetHealthResponse, GetVersion, GetVersionResponse,
-  ListDockerRegistries, ListGitProviders, ListSecrets, PruneSystem,
-  RunCommand, build::*, compose::*, container::*, git::*, image::*,
-  network::*, stats::*, volume::*,
+  build::*, compose::*, container::*, git::*, image::*, network::*,
+  stats::*, terminal::*, volume::*, *,
 };
 use resolver_api::Resolve;
 use response::Response;
@@ -27,9 +24,13 @@ mod deploy;
 mod git;
 mod image;
 mod network;
+mod router;
 mod stats;
+mod terminal;
 mod volume;
 
+pub use router::router;
+
 pub struct Args;
 
 #[derive(
@@ -136,6 +137,13 @@ pub enum PeripheryRequest {
 
   // All in one (Write)
   PruneSystem(PruneSystem),
+
+  // Terminal
+  ListTerminals(ListTerminals),
+  CreateTerminal(CreateTerminal),
+  DeleteTerminal(DeleteTerminal),
+  DeleteAllTerminals(DeleteAllTerminals),
+  CreateTerminalAuthToken(CreateTerminalAuthToken),
 }
 
 //

bin/periphery/src/api/router.rs

@@ -8,7 +8,7 @@ use axum::{
   http::{Request, StatusCode},
   middleware::{self, Next},
   response::Response,
-  routing::post,
+  routing::{get, post},
 };
 use derive_variants::ExtractVariant;
 use resolver_api::Resolve;
@@ -19,9 +19,13 @@ use crate::config::periphery_config;
 
 pub fn router() -> Router {
   Router::new()
-    .route("/", post(handler))
+    .merge(
+      Router::new()
+        .route("/", post(handler))
+        .layer(middleware::from_fn(guard_request_by_passkey)),
+    )
+    .route("/terminal", get(super::terminal::connect_terminal))
     .layer(middleware::from_fn(guard_request_by_ip))
-    .layer(middleware::from_fn(guard_request_by_passkey))
 }
 
 async fn handler(

bin/periphery/src/api/terminal.rs

@@ -0,0 +1,333 @@
+use std::{collections::HashMap, sync::OnceLock};
+
+use anyhow::{Context, anyhow};
+use axum::{
+  extract::{
+    Query, WebSocketUpgrade,
+    ws::{Message, Utf8Bytes},
+  },
+  http::StatusCode,
+  response::Response,
+};
+use bytes::Bytes;
+use futures::{SinkExt, StreamExt};
+use komodo_client::entities::{
+  NoData, komodo_timestamp, server::TerminalInfo,
+};
+use periphery_client::api::terminal::{
+  ConnectTerminalQuery, CreateTerminal, CreateTerminalAuthToken,
+  CreateTerminalAuthTokenResponse, DeleteAllTerminals,
+  DeleteTerminal, ListTerminals,
+};
+use rand::Rng;
+use resolver_api::Resolve;
+use serror::AddStatusCodeError;
+use tokio_util::sync::CancellationToken;
+
+use crate::{
+  config::periphery_config,
+  terminal::{
+    ResizeDimensions, StdinMsg, clean_up_terminals, create_terminal,
+    delete_all_terminals, delete_terminal, get_terminal,
+    list_terminals,
+  },
+};
+
+impl Resolve<super::Args> for ListTerminals {
+  #[instrument(name = "ListTerminals", level = "debug")]
+  async fn resolve(
+    self,
+    _: &super::Args,
+  ) -> serror::Result<Vec<TerminalInfo>> {
+    if periphery_config().disable_terminals {
+      return Err(
+        anyhow!("Terminals are disabled in the periphery config")
+          .status_code(StatusCode::FORBIDDEN),
+      );
+    }
+    clean_up_terminals().await;
+    Ok(list_terminals().await)
+  }
+}
+
+impl Resolve<super::Args> for CreateTerminal {
+  #[instrument(name = "CreateTerminal", level = "debug")]
+  async fn resolve(self, _: &super::Args) -> serror::Result<NoData> {
+    if periphery_config().disable_terminals {
+      return Err(
+        anyhow!("Terminals are disabled in the periphery config")
+          .status_code(StatusCode::FORBIDDEN),
+      );
+    }
+    create_terminal(self.name, self.command, self.recreate)
+      .await
+      .map(|_| NoData {})
+      .map_err(Into::into)
+  }
+}
+
+impl Resolve<super::Args> for DeleteTerminal {
+  #[instrument(name = "DeleteTerminal", level = "debug")]
+  async fn resolve(self, _: &super::Args) -> serror::Result<NoData> {
+    if periphery_config().disable_terminals {
+      return Err(
+        anyhow!("Terminals are disabled in the periphery config")
+          .status_code(StatusCode::FORBIDDEN),
+      );
+    }
+    delete_terminal(&self.terminal).await;
+    Ok(NoData {})
+  }
+}
+
+impl Resolve<super::Args> for DeleteAllTerminals {
+  #[instrument(name = "DeleteAllTerminals", level = "debug")]
+  async fn resolve(self, _: &super::Args) -> serror::Result<NoData> {
+    if periphery_config().disable_terminals {
+      return Err(
+        anyhow!("Terminals are disabled in the periphery config")
+          .status_code(StatusCode::FORBIDDEN),
+      );
+    }
+    delete_all_terminals().await;
+    Ok(NoData {})
+  }
+}
+
+impl Resolve<super::Args> for CreateTerminalAuthToken {
+  #[instrument(name = "CreateTerminalAuthToken", level = "debug")]
+  async fn resolve(
+    self,
+    _: &super::Args,
+  ) -> serror::Result<CreateTerminalAuthTokenResponse> {
+    if periphery_config().disable_terminals {
+      return Err(
+        anyhow!("Terminals are disabled in the periphery config")
+          .status_code(StatusCode::FORBIDDEN),
+      );
+    }
+    Ok(CreateTerminalAuthTokenResponse {
+      token: auth_tokens().create_auth_token(),
+    })
+  }
+}
+
+/// Tokens valid for 3 seconds
+const TOKEN_VALID_FOR_MS: i64 = 3_000;
+
+fn auth_tokens() -> &'static AuthTokens {
+  static AUTH_TOKENS: OnceLock<AuthTokens> = OnceLock::new();
+  AUTH_TOKENS.get_or_init(Default::default)
+}
+
+#[derive(Default)]
+struct AuthTokens {
+  map: std::sync::Mutex<HashMap<String, i64>>,
+}
+
+impl AuthTokens {
+  pub fn create_auth_token(&self) -> String {
+    let token: String = rand::rng()
+      .sample_iter(&rand::distr::Alphanumeric)
+      .take(30)
+      .map(char::from)
+      .collect();
+    self
+      .map
+      .lock()
+      .unwrap()
+      .insert(token.clone(), komodo_timestamp() + TOKEN_VALID_FOR_MS);
+    token
+  }
+
+  pub fn check_token(&self, token: String) -> serror::Result<()> {
+    let Some(valid_until) = self.map.lock().unwrap().remove(&token)
+    else {
+      return Err(
+        anyhow!("Terminal auth token not found")
+          .status_code(StatusCode::UNAUTHORIZED),
+      );
+    };
+    if komodo_timestamp() <= valid_until {
+      Ok(())
+    } else {
+      Err(
+        anyhow!("Terminal token is expired")
+          .status_code(StatusCode::UNAUTHORIZED),
+      )
+    }
+  }
+}
+
+pub async fn connect_terminal(
+  Query(ConnectTerminalQuery {
+    token,
+    terminal,
+    init,
+  }): Query<ConnectTerminalQuery>,
+  ws: WebSocketUpgrade,
+) -> serror::Result<Response> {
+  if periphery_config().disable_terminals {
+    return Err(
+      anyhow!("Terminals are disabled in the periphery config")
+        .status_code(StatusCode::FORBIDDEN),
+    );
+  }
+
+  // Auth the connection with single use token
+  auth_tokens().check_token(token)?;
+
+  clean_up_terminals().await;
+  let terminal = get_terminal(&terminal).await?;
+
+  Ok(ws.on_upgrade(|mut socket| async move {
+    let init_res = async {
+      let (a, b) = terminal.history.bytes_parts();
+      if !a.is_empty() {
+        socket.send(Message::Binary(a)).await.context("Failed to send history part a")?;
+      }
+      if !b.is_empty() {
+        socket.send(Message::Binary(b)).await.context("Failed to send history part b")?;
+      }
+
+      if let Some(init) = init {
+        terminal
+          .stdin
+          .send(StdinMsg::Bytes(Bytes::from(init + "\n")))
+          .await
+          .context("Failed to run init command")?
+      }
+      anyhow::Ok(())
+    }.await;
+
+    if let Err(e) = init_res {
+      let _ = socket.send(Message::Text(format!("ERROR: {e:#}").into())).await;
+      let _ = socket.close().await;
+      return;
+    }
+
+    let (mut ws_write, mut ws_read) = socket.split();
+
+    let cancel = CancellationToken::new();
+
+    let ws_read = async {
+      loop {
+        let res = tokio::select! {
+          res = ws_read.next() => res,
+          _ = terminal.cancel.cancelled() => {
+            trace!("ws read: cancelled from outside");
+            break
+          },
+          _ = cancel.cancelled() => {
+            trace!("ws read: cancelled from inside");
+            break;
+          }
+        };
+        match res {
+          Some(Ok(Message::Binary(bytes)))
+            if bytes.first() == Some(&0x00) =>
+          {
+            // println!("Got ws read bytes - for stdin");
+            if let Err(e) = terminal.stdin.send(StdinMsg::Bytes(
+              Bytes::copy_from_slice(&bytes[1..]),
+            )).await {
+              debug!("WS -> PTY channel send error: {e:}");
+              terminal.cancel();
+              break;
+            };
+          }
+          Some(Ok(Message::Binary(bytes)))
+            if bytes.first() == Some(&0xFF) =>
+          {
+            // println!("Got ws read bytes - for resize");
+            if let Ok(dimensions) =
+              serde_json::from_slice::<ResizeDimensions>(&bytes[1..])
+            {
+              if let Err(e) =
+                terminal.stdin.send(StdinMsg::Resize(dimensions)).await
+              {
+                debug!("WS -> PTY channel send error: {e:}");
+                terminal.cancel();
+                break;
+              };
+            }
+          }
+          Some(Ok(Message::Text(text))) => {
+            trace!("Got ws read text");
+            if let Err(e) =
+              terminal.stdin.send(StdinMsg::Bytes(Bytes::from(text))).await
+            {
+              debug!("WS -> PTY channel send error: {e:?}");
+              terminal.cancel();
+              break;
+            };
+          }
+          Some(Ok(Message::Close(_))) => {
+            debug!("got ws read close");
+            cancel.cancel();
+            break;
+          }
+          Some(Ok(_)) => {
+            // Do nothing (ping, non-prefixed bytes, etc.)
+          }
+          Some(Err(e)) => {
+            debug!("Got ws read error: {e:?}");
+            cancel.cancel();
+            break;
+          }
+          None => {
+            debug!("Got ws read none");
+            cancel.cancel();
+            break;
+          }
+        }
+      }
+    };
+
+    let ws_write = async {
+      let mut stdout = terminal.stdout.resubscribe();
+      loop {
+        let res = tokio::select! {
+          res = stdout.recv() => res.context("Failed to get message over stdout receiver"),
+          _ = terminal.cancel.cancelled() => {
+            trace!("ws write: cancelled from outside");
+            let _ = ws_write.send(Message::Text(Utf8Bytes::from_static("PTY KILLED"))).await;
+            if let Err(e) = ws_write.close().await {
+              debug!("Failed to close ws: {e:?}");
+            };
+            break
+          },
+          _ = cancel.cancelled() => {
+            let _ = ws_write.send(Message::Text(Utf8Bytes::from_static("WS KILLED"))).await;
+            if let Err(e) = ws_write.close().await {
+              debug!("Failed to close ws: {e:?}");
+            };
+            break
+          }
+        };
+        match res {
+          Ok(bytes) => {
+            if let Err(e) =
+              ws_write.send(Message::Binary(bytes)).await
+            {
+              debug!("Failed to send to WS: {e:?}");
+              cancel.cancel();
+              break;
+            }
+          }
+          Err(e) => {
+            debug!("PTY -> WS channel read error: {e:?}");
+            let _ = ws_write.send(Message::Text(Utf8Bytes::from(format!("ERROR: {e:#}")))).await;
+            let _ = ws_write.close().await;
+            terminal.cancel();
+            break;
+          }
+        }
+      }
+    };
+
+    tokio::join!(ws_read, ws_write);
+
+    clean_up_terminals().await;
+  }))
+}

bin/periphery/src/compose.rs

@@ -430,7 +430,7 @@ pub async fn write_stack(
   Option<Vec<(String, String)>>,
 )> {
   let root = periphery_config()
-    .stack_dir
+    .stack_dir()
     .join(to_komodo_name(&stack.name));
   let run_directory = root.join(&stack.config.run_directory);
   // This will remove any intermediate '/./' in the path, which is a problem for some OS.

bin/periphery/src/config.rs

@@ -36,9 +36,15 @@ pub fn periphery_config() -> &'static PeripheryConfig {
     PeripheryConfig {
       port: env.periphery_port.unwrap_or(config.port),
       bind_ip: env.periphery_bind_ip.unwrap_or(config.bind_ip),
-      repo_dir: env.periphery_repo_dir.unwrap_or(config.repo_dir),
-      stack_dir: env.periphery_stack_dir.unwrap_or(config.stack_dir),
-      build_dir: env.periphery_build_dir.unwrap_or(config.build_dir),
+      root_directory: env
+        .periphery_root_directory
+        .unwrap_or(config.root_directory),
+      repo_dir: env.periphery_repo_dir.or(config.repo_dir),
+      stack_dir: env.periphery_stack_dir.or(config.stack_dir),
+      build_dir: env.periphery_build_dir.or(config.build_dir),
+      disable_terminals: env
+        .periphery_disable_terminals
+        .unwrap_or(config.disable_terminals),
       stats_polling_rate: env
         .periphery_stats_polling_rate
         .unwrap_or(config.stats_polling_rate),
@@ -54,6 +60,9 @@ pub fn periphery_config() -> &'static PeripheryConfig {
         stdio: env
           .periphery_logging_stdio
           .unwrap_or(config.logging.stdio),
+        pretty: env
+          .periphery_logging_pretty
+          .unwrap_or(config.logging.pretty),
         otlp_endpoint: env
           .periphery_logging_otlp_endpoint
           .unwrap_or(config.logging.otlp_endpoint),
@@ -80,10 +89,10 @@ pub fn periphery_config() -> &'static PeripheryConfig {
         .unwrap_or(config.ssl_enabled),
       ssl_key_file: env
         .periphery_ssl_key_file
-        .unwrap_or(config.ssl_key_file),
+        .or(config.ssl_key_file),
       ssl_cert_file: env
         .periphery_ssl_cert_file
-        .unwrap_or(config.ssl_cert_file),
+        .or(config.ssl_cert_file),
       secrets: config.secrets,
       git_providers: config.git_providers,
       docker_registries: config.docker_registries,

bin/periphery/src/helpers.rs

@@ -101,7 +101,7 @@ pub async fn pull_or_clone_stack(
   }
 
   let root = periphery_config()
-    .stack_dir
+    .stack_dir()
     .join(to_komodo_name(&stack.name));
 
   let mut args: CloneArgs = stack.into();

bin/periphery/src/main.rs

@@ -12,9 +12,9 @@ mod compose;
 mod config;
 mod docker;
 mod helpers;
-mod router;
 mod ssl;
 mod stats;
+mod terminal;
 
 async fn app() -> anyhow::Result<()> {
   dotenvy::dotenv().ok();
@@ -26,14 +26,17 @@ async fn app() -> anyhow::Result<()> {
 
   stats::spawn_system_stats_polling_thread();
 
-  let addr = format!("{}:{}", config::periphery_config().bind_ip, config::periphery_config().port);
+  let addr = format!(
+    "{}:{}",
+    config::periphery_config().bind_ip,
+    config::periphery_config().port
+  );
 
-  let socket_addr = 
-    SocketAddr::from_str(&addr)
+  let socket_addr = SocketAddr::from_str(&addr)
     .context("failed to parse listen address")?;
 
-  let app = router::router()
-    .into_make_service_with_connect_info::<SocketAddr>();
+  let app =
+    api::router().into_make_service_with_connect_info::<SocketAddr>();
 
   if config.ssl_enabled {
     info!("🔒 Periphery SSL Enabled");
@@ -43,8 +46,8 @@ async fn app() -> anyhow::Result<()> {
     ssl::ensure_certs().await;
     info!("Komodo Periphery starting on https://{}", socket_addr);
     let ssl_config = RustlsConfig::from_pem_file(
-      &config.ssl_cert_file,
-      &config.ssl_key_file,
+      config.ssl_cert_file(),
+      config.ssl_key_file(),
     )
     .await
     .context("Invalid ssl cert / key")?;
@@ -70,7 +73,10 @@ async fn main() -> anyhow::Result<()> {
 
   tokio::select! {
     res = app => return res?,
-    _ = term_signal.recv() => {},
+    _ = term_signal.recv() => {
+      info!("Exiting all active Terminals for shutdown");
+      terminal::delete_all_terminals().await;
+    },
   }
 
   Ok(())

bin/periphery/src/ssl.rs

@@ -2,7 +2,8 @@ use crate::config::periphery_config;
 
 pub async fn ensure_certs() {
   let config = periphery_config();
-  if !config.ssl_cert_file.is_file() || !config.ssl_key_file.is_file()
+  if !config.ssl_cert_file().is_file()
+    || !config.ssl_key_file().is_file()
   {
     generate_self_signed_ssl_certs().await
   }
@@ -14,16 +15,19 @@ async fn generate_self_signed_ssl_certs() {
 
   let config = periphery_config();
 
+  let ssl_key_file = config.ssl_key_file();
+  let ssl_cert_file = config.ssl_cert_file();
+
   // ensure cert folders exist
-  if let Some(parent) = config.ssl_key_file.parent() {
+  if let Some(parent) = ssl_key_file.parent() {
     let _ = std::fs::create_dir_all(parent);
   }
-  if let Some(parent) = config.ssl_cert_file.parent() {
+  if let Some(parent) = ssl_cert_file.parent() {
     let _ = std::fs::create_dir_all(parent);
   }
 
-  let key_path = &config.ssl_key_file.display();
-  let cert_path = &config.ssl_cert_file.display();
+  let key_path = ssl_key_file.display();
+  let cert_path = ssl_cert_file.display();
 
   let command = format!(
     "openssl req -x509 -newkey rsa:4096 -keyout {key_path} -out {cert_path} -sha256 -days 3650 -nodes -subj \"/C=XX/CN=periphery\""

bin/periphery/src/stats.rs

@@ -201,5 +201,6 @@ fn get_system_information(
       .next()
       .map(|cpu| cpu.brand().to_string())
       .unwrap_or_default(),
+    terminals_disabled: periphery_config().disable_terminals,
   }
 }

bin/periphery/src/terminal.rs

@@ -0,0 +1,346 @@
+use std::{
+  collections::{HashMap, VecDeque},
+  sync::{Arc, OnceLock},
+  time::Duration,
+};
+
+use anyhow::{Context, anyhow};
+use bytes::Bytes;
+use komodo_client::{
+  api::write::TerminalRecreateMode, entities::server::TerminalInfo,
+};
+use portable_pty::{CommandBuilder, PtySize, native_pty_system};
+use tokio::sync::{broadcast, mpsc};
+use tokio_util::sync::CancellationToken;
+
+type PtyName = String;
+type PtyMap = tokio::sync::RwLock<HashMap<PtyName, Arc<Terminal>>>;
+type StdinSender = mpsc::Sender<StdinMsg>;
+type StdoutReceiver = broadcast::Receiver<Bytes>;
+
+pub async fn create_terminal(
+  name: String,
+  command: String,
+  recreate: TerminalRecreateMode,
+) -> anyhow::Result<()> {
+  trace!(
+    "CreateTerminal: {name} | command: {command} | recreate: {recreate:?}"
+  );
+  let mut terminals = terminals().write().await;
+  use TerminalRecreateMode::*;
+  if matches!(recreate, Never | DifferentCommand) {
+    if let Some(terminal) = terminals.get(&name) {
+      if terminal.command == command {
+        return Ok(());
+      } else if matches!(recreate, Never) {
+        return Err(anyhow!(
+          "Terminal {name} already exists, but has command {} instead of {command}",
+          terminal.command
+        ));
+      }
+    }
+  }
+  if let Some(prev) = terminals.insert(
+    name,
+    Terminal::new(command)
+      .await
+      .context("Failed to init terminal")?
+      .into(),
+  ) {
+    prev.cancel();
+  }
+  Ok(())
+}
+
+pub async fn delete_terminal(name: &str) {
+  if let Some(terminal) = terminals().write().await.remove(name) {
+    terminal.cancel.cancel();
+  }
+}
+
+pub async fn list_terminals() -> Vec<TerminalInfo> {
+  let mut terminals = terminals()
+    .read()
+    .await
+    .iter()
+    .map(|(name, terminal)| TerminalInfo {
+      name: name.to_string(),
+      command: terminal.command.clone(),
+      stored_size_kb: terminal.history.size_kb(),
+    })
+    .collect::<Vec<_>>();
+  terminals.sort_by(|a, b| a.name.cmp(&b.name));
+  terminals
+}
+
+pub async fn get_terminal(
+  name: &str,
+) -> anyhow::Result<Arc<Terminal>> {
+  terminals()
+    .read()
+    .await
+    .get(name)
+    .cloned()
+    .with_context(|| format!("No terminal at {name}"))
+}
+
+pub async fn clean_up_terminals() {
+  terminals()
+    .write()
+    .await
+    .retain(|_, terminal| !terminal.cancel.is_cancelled());
+}
+
+pub async fn delete_all_terminals() {
+  terminals()
+    .write()
+    .await
+    .drain()
+    .for_each(|(_, terminal)| terminal.cancel());
+  // The terminals poll cancel every 500 millis, need to wait for them
+  // to finish cancelling.
+  tokio::time::sleep(Duration::from_millis(100)).await;
+}
+
+fn terminals() -> &'static PtyMap {
+  static TERMINALS: OnceLock<PtyMap> = OnceLock::new();
+  TERMINALS.get_or_init(Default::default)
+}
+
+#[derive(Clone, serde::Deserialize)]
+pub struct ResizeDimensions {
+  rows: u16,
+  cols: u16,
+}
+
+#[derive(Clone)]
+pub enum StdinMsg {
+  Bytes(Bytes),
+  Resize(ResizeDimensions),
+}
+
+pub struct Terminal {
+  /// The command that was used as the root command, eg `shell`
+  command: String,
+
+  pub cancel: CancellationToken,
+
+  pub stdin: StdinSender,
+  pub stdout: StdoutReceiver,
+
+  pub history: Arc<History>,
+}
+
+impl Terminal {
+  async fn new(command: String) -> anyhow::Result<Terminal> {
+    trace!("Creating terminal with command: {command}");
+
+    let terminal = native_pty_system()
+      .openpty(PtySize::default())
+      .context("Failed to open terminal")?;
+
+    let mut command_split = command.split(' ').map(|arg| arg.trim());
+    let cmd =
+      command_split.next().context("Command cannot be empty")?;
+
+    let mut cmd = CommandBuilder::new(cmd);
+
+    for arg in command_split {
+      cmd.arg(arg);
+    }
+
+    cmd.env("TERM", "xterm-256color");
+    cmd.env("COLORTERM", "truecolor");
+
+    let mut child = terminal
+      .slave
+      .spawn_command(cmd)
+      .context("Failed to spawn child command")?;
+
+    // Check the child didn't stop immediately (after a little wait) with error
+    tokio::time::sleep(Duration::from_millis(100)).await;
+    if let Some(status) = child
+      .try_wait()
+      .context("Failed to check child process exit status")?
+    {
+      return Err(anyhow!(
+        "Child process exited immediately with code {}",
+        status.exit_code()
+      ));
+    }
+
+    let mut terminal_write = terminal
+      .master
+      .take_writer()
+      .context("Failed to take terminal writer")?;
+    let mut terminal_read = terminal
+      .master
+      .try_clone_reader()
+      .context("Failed to clone terminal reader")?;
+
+    let cancel = CancellationToken::new();
+
+    // CHILD WAIT TASK
+    let _cancel = cancel.clone();
+    tokio::task::spawn_blocking(move || {
+      loop {
+        if _cancel.is_cancelled() {
+          trace!("child wait handle cancelled from outside");
+          if let Err(e) = child.kill() {
+            debug!("Failed to kill child | {e:?}");
+          }
+          break;
+        }
+        match child.try_wait() {
+          Ok(Some(code)) => {
+            debug!("child exited with code {code}");
+            _cancel.cancel();
+            break;
+          }
+          Ok(None) => {
+            std::thread::sleep(Duration::from_millis(500));
+          }
+          Err(e) => {
+            debug!("failed to wait for child | {e:?}");
+            _cancel.cancel();
+            break;
+          }
+        }
+      }
+    });
+
+    // WS (channel) -> STDIN TASK
+    // Theres only one consumer here, so use mpsc
+    let (stdin, mut channel_read) =
+      tokio::sync::mpsc::channel::<StdinMsg>(8192);
+    let _cancel = cancel.clone();
+    tokio::task::spawn_blocking(move || {
+      loop {
+        if _cancel.is_cancelled() {
+          trace!("terminal write: cancelled from outside");
+          break;
+        }
+        match channel_read.blocking_recv() {
+          Some(StdinMsg::Bytes(bytes)) => {
+            if let Err(e) = terminal_write.write_all(&bytes) {
+              debug!("Failed to write to PTY: {e:?}");
+              _cancel.cancel();
+              break;
+            }
+          }
+          Some(StdinMsg::Resize(dimensions)) => {
+            if let Err(e) = terminal.master.resize(PtySize {
+              cols: dimensions.cols,
+              rows: dimensions.rows,
+              pixel_width: 0,
+              pixel_height: 0,
+            }) {
+              debug!("Failed to resize | {e:?}");
+              _cancel.cancel();
+              break;
+            };
+          }
+          None => {
+            debug!("WS -> PTY channel read error: Disconnected");
+            _cancel.cancel();
+            break;
+          }
+        }
+      }
+    });
+
+    let history = Arc::new(History::default());
+
+    // PTY -> WS (channel) TASK
+    // Uses broadcast to output to multiple client simultaneously
+    let (write, stdout) =
+      tokio::sync::broadcast::channel::<Bytes>(8192);
+    let _cancel = cancel.clone();
+    let _history = history.clone();
+    tokio::task::spawn_blocking(move || {
+      let mut buf = [0u8; 8192];
+      loop {
+        if _cancel.is_cancelled() {
+          trace!("terminal read: cancelled from outside");
+          break;
+        }
+        match terminal_read.read(&mut buf) {
+          Ok(0) => {
+            // EOF
+            trace!("Got PTY read EOF");
+            _cancel.cancel();
+            break;
+          }
+          Ok(n) => {
+            _history.push(&buf[..n]);
+            if let Err(e) =
+              write.send(Bytes::copy_from_slice(&buf[..n]))
+            {
+              debug!("PTY -> WS channel send error: {e:?}");
+              _cancel.cancel();
+              break;
+            }
+          }
+          Err(e) => {
+            debug!("Failed to read for PTY: {e:?}");
+            _cancel.cancel();
+            break;
+          }
+        }
+      }
+    });
+
+    trace!("terminal tasks spawned");
+
+    Ok(Terminal {
+      command,
+      cancel,
+      stdin,
+      stdout,
+      history,
+    })
+  }
+
+  pub fn cancel(&self) {
+    trace!("Cancel called");
+    self.cancel.cancel();
+  }
+}
+
+/// 1 MiB max history size per terminal
+const MAX_BYTES: usize = 1024 * 1024;
+
+pub struct History {
+  buf: std::sync::RwLock<VecDeque<u8>>,
+}
+
+impl Default for History {
+  fn default() -> Self {
+    History {
+      buf: VecDeque::with_capacity(MAX_BYTES).into(),
+    }
+  }
+}
+
+impl History {
+  /// Push some bytes, evicting the oldest when full.
+  fn push(&self, bytes: &[u8]) {
+    let mut buf = self.buf.write().unwrap();
+    for byte in bytes {
+      if buf.len() == MAX_BYTES {
+        buf.pop_front();
+      }
+      buf.push_back(*byte);
+    }
+  }
+
+  pub fn bytes_parts(&self) -> (Bytes, Bytes) {
+    let buf = self.buf.read().unwrap();
+    let (a, b) = buf.as_slices();
+    (Bytes::copy_from_slice(a), Bytes::copy_from_slice(b))
+  }
+
+  pub fn size_kb(&self) -> f64 {
+    self.buf.read().unwrap().len() as f64 / 1024.0
+  }
+}

bin/periphery/starship.toml

@@ -0,0 +1,67 @@
+## This is used to customize the shell prompt in Periphery container for Terminals
+
+"$schema" = 'https://starship.rs/config-schema.json'
+
+add_newline = true
+
+format = "$time$hostname$container$memory_usage$all"
+
+[character]
+success_symbol = "[❯](bright-blue bold)"
+error_symbol = "[❯](bright-red bold)"
+
+[package]
+disabled = true
+
+[time]
+format = "[❯$time](white dimmed) "
+time_format = "%l:%M %p"
+utc_time_offset = '-5'
+disabled = true
+
+[username]
+format = "[❯ $user]($style) "
+style_user = "bright-green"
+show_always = true
+
+[hostname]
+format = "[❯ $hostname]($style) "
+style = "bright-blue"
+ssh_only = false
+
+[directory]
+format = "[❯ $path]($style)[$read_only]($read_only_style) "
+style = "bright-cyan"
+
+[git_branch]
+format = "[❯ $symbol$branch(:$remote_branch)]($style) "
+style = "bright-purple"
+
+[git_status]
+style = "bright-purple"
+
+[rust]
+format = "[❯ $symbol($version )]($style)"
+symbol = "rustc "
+style = "bright-red"
+
+[nodejs]
+format = "[❯ $symbol($version )]($style)"
+symbol = "nodejs "
+style = "bright-red"
+
+[memory_usage]
+format = "[❯ mem ${ram} ${ram_pct}]($style) "
+threshold = -1
+style = "white"
+
+[cmd_duration]
+format = "[❯ $duration]($style)"
+style = "bright-yellow"
+
+[container]
+format = "[❯ 🦎 periphery container ]($style)"
+style = "bright-green"
+
+[aws]
+disabled = true

client/core/rs/src/api/mod.rs

@@ -68,6 +68,7 @@
 
 pub mod auth;
 pub mod execute;
+pub mod terminal;
 pub mod read;
 pub mod user;
 pub mod write;

client/core/rs/src/api/read/server.rs

@@ -13,7 +13,7 @@ use crate::entities::{
   },
   server::{
     Server, ServerActionState, ServerListItem, ServerQuery,
-    ServerState,
+    ServerState, TerminalInfo,
   },
   stack::ComposeProject,
   stats::{
@@ -628,3 +628,27 @@ pub struct GetServersSummaryResponse {
   /// The number of disabled servers.
   pub disabled: I64,
 }
+
+//
+
+/// List the current terminals on specified server.
+/// Response: [ListTerminalsResponse].
+#[typeshare]
+#[derive(
+  Serialize, Deserialize, Debug, Clone, Default, Resolve, EmptyTraits,
+)]
+#[empty_traits(KomodoReadRequest)]
+#[response(ListTerminalsResponse)]
+#[error(serror::Error)]
+pub struct ListTerminals {
+  /// Id or name
+  #[serde(alias = "id", alias = "name")]
+  pub server: String,
+  /// Force a fresh call to Periphery for the list.
+  /// Otherwise the response will be cached for 30s
+  #[serde(default)]
+  pub fresh: bool,
+}
+
+#[typeshare]
+pub type ListTerminalsResponse = Vec<TerminalInfo>;

client/core/rs/src/api/terminal.rs

@@ -0,0 +1,20 @@
+use serde::{Deserialize, Serialize};
+use typeshare::typeshare;
+
+/// Query to connect to a terminal (interactive shell over websocket) on the given server.
+/// TODO: Document calling.
+#[typeshare]
+#[derive(Serialize, Deserialize, Debug, Clone)]
+pub struct ConnectTerminalQuery {
+  /// Server Id or name
+  pub server: String,
+  /// Each periphery can keep multiple terminals open.
+  /// If a terminals with the specified name already exists,
+  /// it will be attached to.
+  /// Otherwise a new terminal will be created for the command,
+  /// which will persist until it is deleted using
+  /// [DeleteTerminal][crate::api::write::server::DeleteTerminal]
+  pub terminal: String,
+  /// Optional. The initial command to execute on connection to the shell.
+  pub init: Option<String>,
+}

client/core/rs/src/api/write/server.rs

@@ -4,6 +4,7 @@ use serde::{Deserialize, Serialize};
 use typeshare::typeshare;
 
 use crate::entities::{
+  NoData,
   server::{_PartialServerConfig, Server},
   update::Update,
 };
@@ -105,3 +106,83 @@ pub struct CreateNetwork {
   /// The name of the network to create.
   pub name: String,
 }
+
+//
+
+/// Configures the behavior of [CreateTerminal] if the
+/// specified terminal name already exists.
+#[typeshare]
+#[derive(Serialize, Deserialize, Debug, Clone, Copy, Default)]
+pub enum TerminalRecreateMode {
+  /// Never kill the old terminal if it already exists.
+  /// If the command is different, returns error.
+  #[default]
+  Never,
+  /// Always kill the old terminal and create new one
+  Always,
+  /// Only kill and recreate if the command is different.
+  DifferentCommand,
+}
+
+/// Create a terminal on the server.
+/// Response: [NoData]
+#[typeshare]
+#[derive(
+  Serialize, Deserialize, Debug, Clone, Resolve, EmptyTraits,
+)]
+#[empty_traits(KomodoWriteRequest)]
+#[response(NoData)]
+#[error(serror::Error)]
+pub struct CreateTerminal {
+  /// Server Id or name
+  pub server: String,
+  /// The name of the terminal on the server to create.
+  pub name: String,
+  /// The shell command (eg `bash`) to init the shell.
+  ///
+  /// This can also include args:
+  /// `docker exec -it container sh`
+  ///
+  /// Default: `bash`
+  #[serde(default = "default_command")]
+  pub command: String,
+  /// Default: `Never`
+  #[serde(default)]
+  pub recreate: TerminalRecreateMode,
+}
+
+fn default_command() -> String {
+  String::from("bash")
+}
+
+//
+
+/// Delete a terminal on the server.
+/// Response: [NoData]
+#[typeshare]
+#[derive(
+  Serialize, Deserialize, Debug, Clone, Resolve, EmptyTraits,
+)]
+#[empty_traits(KomodoWriteRequest)]
+#[response(NoData)]
+#[error(serror::Error)]
+pub struct DeleteTerminal {
+  /// Server Id or name
+  pub server: String,
+  /// The name of the terminal on the server to delete.
+  pub terminal: String,
+}
+
+/// Delete all terminals on the server.
+/// Response: [NoData]
+#[typeshare]
+#[derive(
+  Serialize, Deserialize, Debug, Clone, Resolve, EmptyTraits,
+)]
+#[empty_traits(KomodoWriteRequest)]
+#[response(NoData)]
+#[error(serror::Error)]
+pub struct DeleteAllTerminals {
+  /// Server Id or name
+  pub server: String,
+}

client/core/rs/src/entities/action.rs

@@ -13,7 +13,10 @@ use crate::{
   entities::I64,
 };
 
-use super::resource::{Resource, ResourceListItem, ResourceQuery};
+use super::{
+  ScheduleFormat,
+  resource::{Resource, ResourceListItem, ResourceQuery},
+};
 
 #[typeshare]
 pub type ActionListItem = ResourceListItem<ActionListItemInfo>;
@@ -25,6 +28,12 @@ pub struct ActionListItemInfo {
   pub last_run_at: I64,
   /// Whether last action run successful
   pub state: ActionState,
+  /// If the procedure has schedule enabled, this is the
+  /// next scheduled run time in unix ms.
+  pub next_scheduled_run: Option<I64>,
+  /// If there is an error parsing schedule expression,
+  /// it will be given here.
+  pub schedule_error: Option<String>,
 }
 
 #[typeshare]
@@ -62,15 +71,55 @@ pub type _PartialActionConfig = PartialActionConfig;
 #[partial_derive(Serialize, Deserialize, Debug, Clone, Default)]
 #[partial(skip_serializing_none, from, diff)]
 pub struct ActionConfig {
-  /// Typescript file contents using pre-initialized `komodo` client.
-  /// Supports variable / secret interpolation.
-  #[serde(default, deserialize_with = "file_contents_deserializer")]
-  #[partial_attr(serde(
-    default,
-    deserialize_with = "option_file_contents_deserializer"
-  ))]
+  /// Choose whether to specify schedule as regular CRON, or using the english to CRON parser.
+  #[serde(default)]
   #[builder(default)]
-  pub file_contents: String,
+  pub schedule_format: ScheduleFormat,
+
+  /// Optionally provide a schedule for the procedure to run on.
+  ///
+  /// There are 2 ways to specify a schedule:
+  ///
+  /// 1. Regular CRON expression:
+  ///
+  /// (second, minute, hour, day, month, day-of-week)
+  /// ```
+  /// 0 0 0 1,15 * ?
+  /// ```
+  ///
+  /// 2. "English" expression via [english-to-cron](https://crates.io/crates/english-to-cron):
+  ///
+  /// ```
+  /// at midnight on the 1st and 15th of the month
+  /// ```
+  #[serde(default)]
+  #[builder(default)]
+  pub schedule: String,
+
+  /// Whether schedule is enabled if one is provided.
+  /// Can be used to temporarily disable the schedule.
+  #[serde(default = "default_schedule_enabled")]
+  #[builder(default = "default_schedule_enabled()")]
+  #[partial_default(default_schedule_enabled())]
+  pub schedule_enabled: bool,
+
+  /// Optional. A TZ Identifier. If not provided, will use Core local timezone.
+  /// https://en.wikipedia.org/wiki/List_of_tz_database_time_zones.
+  #[serde(default)]
+  #[builder(default)]
+  pub schedule_timezone: String,
+
+  /// Whether to send alerts when the schedule was run.
+  #[serde(default = "default_schedule_alert")]
+  #[builder(default = "default_schedule_alert()")]
+  #[partial_default(default_schedule_alert())]
+  pub schedule_alert: bool,
+
+  /// Whether to send alerts when this action fails.
+  #[serde(default = "default_failure_alert")]
+  #[builder(default = "default_failure_alert()")]
+  #[partial_default(default_failure_alert())]
+  pub failure_alert: bool,
 
   /// Whether incoming webhooks actually trigger action.
   #[serde(default = "default_webhook_enabled")]
@@ -83,6 +132,28 @@ pub struct ActionConfig {
   #[serde(default)]
   #[builder(default)]
   pub webhook_secret: String,
+
+  /// Typescript file contents using pre-initialized `komodo` client.
+  /// Supports variable / secret interpolation.
+  #[serde(default, deserialize_with = "file_contents_deserializer")]
+  #[partial_attr(serde(
+    default,
+    deserialize_with = "option_file_contents_deserializer"
+  ))]
+  #[builder(default)]
+  pub file_contents: String,
+}
+
+fn default_schedule_enabled() -> bool {
+  true
+}
+
+fn default_schedule_alert() -> bool {
+  true
+}
+
+fn default_failure_alert() -> bool {
+  true
 }
 
 fn default_webhook_enabled() -> bool {
@@ -98,9 +169,15 @@ impl ActionConfig {
 impl Default for ActionConfig {
   fn default() -> Self {
     Self {
-      file_contents: Default::default(),
+      schedule_format: Default::default(),
+      schedule: Default::default(),
+      schedule_enabled: default_schedule_enabled(),
+      schedule_timezone: Default::default(),
+      schedule_alert: default_schedule_alert(),
+      failure_alert: default_failure_alert(),
       webhook_enabled: default_webhook_enabled(),
       webhook_secret: Default::default(),
+      file_contents: Default::default(),
     }
   }
 }

client/core/rs/src/entities/alert.rs

@@ -8,8 +8,8 @@ use typeshare::typeshare;
 use crate::entities::{I64, MongoId};
 
 use super::{
-  _Serror, ResourceTarget, Version, deployment::DeploymentState,
-  stack::StackState,
+  _Serror, ResourceTarget, ResourceTargetVariant, Version,
+  deployment::DeploymentState, stack::StackState,
 };
 
 /// Representation of an alert in the system.
@@ -260,6 +260,32 @@ pub enum AlertData {
     /// The name of the repo
     name: String,
   },
+
+  /// A procedure has failed
+  ProcedureFailed {
+    /// The id of the procedure
+    id: String,
+    /// The name of the procedure
+    name: String,
+  },
+
+  /// An action has failed
+  ActionFailed {
+    /// The id of the action
+    id: String,
+    /// The name of the action
+    name: String,
+  },
+
+  /// A schedule was run
+  ScheduleRun {
+    /// Procedure or Action
+    resource_type: ResourceTargetVariant,
+    /// The resource id
+    id: String,
+    /// The resource name
+    name: String,
+  },
 }
 
 impl Default for AlertData {

client/core/rs/src/entities/alerter.rs

@@ -114,6 +114,9 @@ pub enum AlerterEndpoint {
 
   /// Send alert to Ntfy
   Ntfy(NtfyAlerterEndpoint),
+
+  /// Send alert to Pushover
+  Pushover(PushoverAlerterEndpoint),
 }
 
 impl Default for AlerterEndpoint {
@@ -222,9 +225,33 @@ fn default_ntfy_url() -> String {
   String::from("http://localhost:8080/komodo")
 }
 
+/// Configuration for a Pushover alerter.
+#[typeshare]
+#[derive(
+  Debug, Clone, PartialEq, Serialize, Deserialize, Builder,
+)]
+pub struct PushoverAlerterEndpoint {
+  /// The pushover URL including application and user tokens in parameters.
+  #[serde(default = "default_pushover_url")]
+  #[builder(default = "default_pushover_url()")]
+  pub url: String,
+}
+
+impl Default for PushoverAlerterEndpoint {
+  fn default() -> Self {
+    Self {
+      url: default_pushover_url(),
+    }
+  }
+}
+
+fn default_pushover_url() -> String {
+  String::from(
+    "https://api.pushover.net/1/messages.json?token=XXXXXXXXXXXXX&user=XXXXXXXXXXXXX",
+  )
+}
 
 // QUERY
-
 #[typeshare]
 pub type AlerterQuery = ResourceQuery<AlerterQuerySpecifics>;
 

client/core/rs/src/entities/build.rs

@@ -43,7 +43,7 @@ pub struct BuildListItemInfo {
 
   /// Whether build is in files on host mode.
   pub files_on_host: bool,
-  
+
   /// The git provider domain
   pub git_provider: Option<String>,
   /// The repo used as the source of the build

client/core/rs/src/entities/config/core.rs

@@ -85,6 +85,8 @@ pub struct Env {
   pub komodo_logging_level: Option<LogLevel>,
   /// Override `logging.stdio`
   pub komodo_logging_stdio: Option<StdioLogMode>,
+  /// Override `logging.pretty`
+  pub komodo_logging_pretty: Option<bool>,
   /// Override `logging.otlp_endpoint`
   pub komodo_logging_otlp_endpoint: Option<String>,
   /// Override `logging.opentelemetry_service_name`
@@ -260,7 +262,7 @@ pub struct CoreConfig {
 
   /// IP address the core server binds to.
   /// Default: [::].
-   #[serde(default = "default_core_bind_ip")]
+  #[serde(default = "default_core_bind_ip")]
   pub bind_ip: String,
 
   /// Sent in auth header with req to periphery.

client/core/rs/src/entities/config/periphery.rs

@@ -116,12 +116,16 @@ pub struct Env {
   pub periphery_port: Option<u16>,
   /// Override `bind_ip`
   pub periphery_bind_ip: Option<String>,
+  /// Override `root_directory`
+  pub periphery_root_directory: Option<PathBuf>,
   /// Override `repo_dir`
   pub periphery_repo_dir: Option<PathBuf>,
   /// Override `stack_dir`
   pub periphery_stack_dir: Option<PathBuf>,
   /// Override `build_dir`
   pub periphery_build_dir: Option<PathBuf>,
+  /// Override `disable_terminals`
+  pub periphery_disable_terminals: Option<bool>,
   /// Override `stats_polling_rate`
   pub periphery_stats_polling_rate: Option<Timelength>,
   /// Override `legacy_compose_cli`
@@ -132,6 +136,8 @@ pub struct Env {
   pub periphery_logging_level: Option<LogLevel>,
   /// Override `logging.stdio`
   pub periphery_logging_stdio: Option<StdioLogMode>,
+  /// Override `logging.pretty`
+  pub periphery_logging_pretty: Option<bool>,
   /// Override `logging.otlp_endpoint`
   pub periphery_logging_otlp_endpoint: Option<String>,
   /// Override `logging.opentelemetry_service_name`
@@ -171,20 +177,39 @@ pub struct PeripheryConfig {
   #[serde(default = "default_periphery_bind_ip")]
   pub bind_ip: String,
 
+  /// The directory Komodo will use as the default root for the specific (repo, stack, build) directories.
+  ///
+  /// repo: ${root_directory}/repos
+  /// stack: ${root_directory}/stacks
+  /// build: ${root_directory}/builds
+  ///
+  /// Note. These can each be overridden with a specific directory
+  /// by specifying `repo_dir`, `stack_dir`, or `build_dir` explicitly
+  ///
+  /// Default: `/etc/komodo`
+  #[serde(default = "default_root_directory")]
+  pub root_directory: PathBuf,
+
   /// The system directory where Komodo managed repos will be cloned.
-  /// Default: `/etc/komodo/repos`
-  #[serde(default = "default_repo_dir")]
-  pub repo_dir: PathBuf,
+  /// If not provided, will default to `${root_directory}/repos`.
+  /// Default: empty
+  pub repo_dir: Option<PathBuf>,
 
   /// The system directory where stacks will managed.
-  /// Default: `/etc/komodo/stacks`
-  #[serde(default = "default_stack_dir")]
-  pub stack_dir: PathBuf,
+  /// If not provided, will default to `${root_directory}/stacks`.
+  /// Default: empty
+  pub stack_dir: Option<PathBuf>,
 
   /// The system directory where builds will managed.
-  /// Default: `/etc/komodo/builds`
-  #[serde(default = "default_build_dir")]
-  pub build_dir: PathBuf,
+  /// If not provided, will default to `${root_directory}/builds`.
+  /// Default: empty
+  pub build_dir: Option<PathBuf>,
+
+  /// Whether to disable the terminal APIs
+  /// and disallow remote shell access.
+  /// Default: false
+  #[serde(default)]
+  pub disable_terminals: bool,
 
   /// The rate at which the system stats will be polled to update the cache.
   /// Default: `5-sec`
@@ -244,14 +269,12 @@ pub struct PeripheryConfig {
   pub ssl_enabled: bool,
 
   /// Path to the ssl key.
-  /// Default: `/etc/komodo/ssl/periphery/key.pem`.
-  #[serde(default = "default_ssl_key_file")]
-  pub ssl_key_file: PathBuf,
+  /// Default: `${root_directory}/ssl/key.pem`.
+  pub ssl_key_file: Option<PathBuf>,
 
   /// Path to the ssl cert.
-  /// Default: `/etc/komodo/ssl/periphery/cert.pem`.
-  #[serde(default = "default_ssl_cert_file")]
-  pub ssl_cert_file: PathBuf,
+  /// Default: `${root_directory}/ssl/cert.pem`.
+  pub ssl_cert_file: Option<PathBuf>,
 }
 
 fn default_periphery_port() -> u16 {
@@ -262,16 +285,8 @@ fn default_periphery_bind_ip() -> String {
   "[::]".to_string()
 }
 
-fn default_repo_dir() -> PathBuf {
-  "/etc/komodo/repos".parse().unwrap()
-}
-
-fn default_stack_dir() -> PathBuf {
-  "/etc/komodo/stacks".parse().unwrap()
-}
-
-fn default_build_dir() -> PathBuf {
-  "/etc/komodo/builds".parse().unwrap()
+fn default_root_directory() -> PathBuf {
+  "/etc/komodo".parse().unwrap()
 }
 
 fn default_stats_polling_rate() -> Timelength {
@@ -282,22 +297,16 @@ fn default_ssl_enabled() -> bool {
   false
 }
 
-fn default_ssl_key_file() -> PathBuf {
-  "/etc/komodo/ssl/key.pem".parse().unwrap()
-}
-
-fn default_ssl_cert_file() -> PathBuf {
-  "/etc/komodo/ssl/cert.pem".parse().unwrap()
-}
-
 impl Default for PeripheryConfig {
   fn default() -> Self {
     Self {
       port: default_periphery_port(),
       bind_ip: default_periphery_bind_ip(),
-      repo_dir: default_repo_dir(),
-      stack_dir: default_stack_dir(),
-      build_dir: default_build_dir(),
+      root_directory: default_root_directory(),
+      repo_dir: None,
+      stack_dir: None,
+      build_dir: None,
+      disable_terminals: Default::default(),
       stats_polling_rate: default_stats_polling_rate(),
       legacy_compose_cli: Default::default(),
       logging: Default::default(),
@@ -309,8 +318,8 @@ impl Default for PeripheryConfig {
       git_providers: Default::default(),
       docker_registries: Default::default(),
       ssl_enabled: default_ssl_enabled(),
-      ssl_key_file: default_ssl_key_file(),
-      ssl_cert_file: default_ssl_cert_file(),
+      ssl_key_file: None,
+      ssl_cert_file: None,
     }
   }
 }
@@ -320,9 +329,11 @@ impl PeripheryConfig {
     PeripheryConfig {
       port: self.port,
       bind_ip: self.bind_ip.clone(),
+      root_directory: self.root_directory.clone(),
       repo_dir: self.repo_dir.clone(),
       stack_dir: self.stack_dir.clone(),
       build_dir: self.build_dir.clone(),
+      disable_terminals: self.disable_terminals,
       stats_polling_rate: self.stats_polling_rate,
       legacy_compose_cli: self.legacy_compose_cli,
       logging: self.logging.clone(),
@@ -378,4 +389,44 @@ impl PeripheryConfig {
       ssl_cert_file: self.ssl_cert_file.clone(),
     }
   }
+
+  pub fn repo_dir(&self) -> PathBuf {
+    if let Some(dir) = &self.repo_dir {
+      dir.to_owned()
+    } else {
+      self.root_directory.join("repos")
+    }
+  }
+
+  pub fn stack_dir(&self) -> PathBuf {
+    if let Some(dir) = &self.stack_dir {
+      dir.to_owned()
+    } else {
+      self.root_directory.join("stacks")
+    }
+  }
+
+  pub fn build_dir(&self) -> PathBuf {
+    if let Some(dir) = &self.build_dir {
+      dir.to_owned()
+    } else {
+      self.root_directory.join("builds")
+    }
+  }
+
+  pub fn ssl_key_file(&self) -> PathBuf {
+    if let Some(dir) = &self.ssl_key_file {
+      dir.to_owned()
+    } else {
+      self.root_directory.join("ssl/key.pem")
+    }
+  }
+
+  pub fn ssl_cert_file(&self) -> PathBuf {
+    if let Some(dir) = &self.ssl_cert_file {
+      dir.to_owned()
+    } else {
+      self.root_directory.join("ssl/cert.pem")
+    }
+  }
 }

client/core/rs/src/entities/logger.rs

@@ -10,6 +10,9 @@ pub struct LogConfig {
   #[serde(default)]
   pub stdio: StdioLogMode,
 
+  #[serde(default)]
+  pub pretty: bool,
+
   /// Enable opentelemetry exporting
   #[serde(default)]
   pub otlp_endpoint: String,
@@ -27,6 +30,7 @@ impl Default for LogConfig {
     Self {
       level: Default::default(),
       stdio: Default::default(),
+      pretty: Default::default(),
       otlp_endpoint: Default::default(),
       opentelemetry_service_name: default_opentelemetry_service_name(
       ),

client/core/rs/src/entities/mod.rs

@@ -999,3 +999,15 @@ impl ResourceTargetVariant {
     }
   }
 }
+
+#[typeshare]
+#[derive(
+  Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize,
+)]
+pub enum ScheduleFormat {
+  #[default]
+  English,
+  Cron,
+}
+
+pub const KOMODO_EXIT_DATA: &str = "__KOMODO_EXIT_DATA:";

client/core/rs/src/entities/procedure.rs

@@ -9,7 +9,7 @@ use typeshare::typeshare;
 use crate::api::execute::Execution;
 
 use super::{
-  I64,
+  I64, ScheduleFormat,
   resource::{Resource, ResourceListItem, ResourceQuery},
 };
 
@@ -23,6 +23,12 @@ pub struct ProcedureListItemInfo {
   pub stages: I64,
   /// Reflect whether last run successful / currently running.
   pub state: ProcedureState,
+  /// If the procedure has schedule enabled, this is the
+  /// next scheduled run time in unix ms.
+  pub next_scheduled_run: Option<I64>,
+  /// If there is an error parsing schedule expression,
+  /// it will be given here.
+  pub schedule_error: Option<String>,
 }
 
 #[typeshare]
@@ -61,6 +67,56 @@ pub struct ProcedureConfig {
   #[builder(default)]
   pub stages: Vec<ProcedureStage>,
 
+  /// Choose whether to specify schedule as regular CRON, or using the english to CRON parser.
+  #[serde(default)]
+  #[builder(default)]
+  pub schedule_format: ScheduleFormat,
+
+  /// Optionally provide a schedule for the procedure to run on.
+  ///
+  /// There are 2 ways to specify a schedule:
+  ///
+  /// 1. Regular CRON expression:
+  ///
+  /// (second, minute, hour, day, month, day-of-week)
+  /// ```
+  /// 0 0 0 1,15 * ?
+  /// ```
+  ///
+  /// 2. "English" expression via [english-to-cron](https://crates.io/crates/english-to-cron):
+  ///
+  /// ```
+  /// at midnight on the 1st and 15th of the month
+  /// ```
+  #[serde(default)]
+  #[builder(default)]
+  pub schedule: String,
+
+  /// Whether schedule is enabled if one is provided.
+  /// Can be used to temporarily disable the schedule.
+  #[serde(default = "default_schedule_enabled")]
+  #[builder(default = "default_schedule_enabled()")]
+  #[partial_default(default_schedule_enabled())]
+  pub schedule_enabled: bool,
+
+  /// Optional. A TZ Identifier. If not provided, will use Core local timezone.
+  /// https://en.wikipedia.org/wiki/List_of_tz_database_time_zones.
+  #[serde(default)]
+  #[builder(default)]
+  pub schedule_timezone: String,
+
+  /// Whether to send alerts when the schedule was run.
+  #[serde(default = "default_schedule_alert")]
+  #[builder(default = "default_schedule_alert()")]
+  #[partial_default(default_schedule_alert())]
+  pub schedule_alert: bool,
+
+  /// Whether to send alerts when this procedure fails.
+  #[serde(default = "default_failure_alert")]
+  #[builder(default = "default_failure_alert()")]
+  #[partial_default(default_failure_alert())]
+  pub failure_alert: bool,
+
   /// Whether incoming webhooks actually trigger action.
   #[serde(default = "default_webhook_enabled")]
   #[builder(default = "default_webhook_enabled()")]
@@ -80,6 +136,18 @@ impl ProcedureConfig {
   }
 }
 
+fn default_schedule_enabled() -> bool {
+  true
+}
+
+fn default_schedule_alert() -> bool {
+  true
+}
+
+fn default_failure_alert() -> bool {
+  true
+}
+
 fn default_webhook_enabled() -> bool {
   true
 }
@@ -88,6 +156,12 @@ impl Default for ProcedureConfig {
   fn default() -> Self {
     Self {
       stages: Default::default(),
+      schedule_format: Default::default(),
+      schedule: Default::default(),
+      schedule_enabled: default_schedule_enabled(),
+      schedule_timezone: Default::default(),
+      schedule_alert: default_schedule_alert(),
+      failure_alert: default_failure_alert(),
       webhook_enabled: default_webhook_enabled(),
       webhook_secret: Default::default(),
     }

client/core/rs/src/entities/server.rs

@@ -38,6 +38,8 @@ pub struct ServerListItemInfo {
   pub send_mem_alerts: bool,
   /// Whether server is configured to send disk alerts.
   pub send_disk_alerts: bool,
+  /// Whether terminals are disabled for this Server.
+  pub terminals_disabled: bool,
 }
 
 #[typeshare(serialized_as = "Partial<ServerConfig>")]
@@ -276,6 +278,19 @@ pub struct ServerHealth {
   pub disks: HashMap<PathBuf, ServerHealthState>,
 }
 
+/// Info about an active terminal on a server.
+/// Retrieve with [ListTerminals][crate::api::read::server::ListTerminals].
+#[typeshare]
+#[derive(Serialize, Deserialize, Default, Debug, Clone)]
+pub struct TerminalInfo {
+  /// The name of the terminal.
+  pub name: String,
+  /// The root program / args of the pty
+  pub command: String,
+  /// The size of the terminal history in memory.
+  pub stored_size_kb: f64,
+}
+
 /// Current pending actions on the server.
 #[typeshare]
 #[derive(Serialize, Deserialize, Debug, Clone, Copy, Default)]

client/core/rs/src/entities/stats.rs

@@ -21,6 +21,8 @@ pub struct SystemInformation {
   pub host_name: Option<String>,
   /// The CPU's brand
   pub cpu_brand: String,
+  /// Whether terminals are disabled on this Periphery
+  pub terminals_disabled: bool,
 }
 
 /// System stats stored on the database.

client/core/rs/src/entities/sync.rs

@@ -271,6 +271,13 @@ pub struct ResourceSyncConfig {
   #[builder(default)]
   pub include_user_groups: bool,
 
+  /// Whether sync should send alert when it enters Pending state.
+  /// Default: true
+  #[serde(default = "default_pending_alert")]
+  #[builder(default = "default_pending_alert()")]
+  #[partial_default(default_pending_alert())]
+  pub pending_alert: bool,
+
   /// Manage the file contents in the UI.
   #[serde(default, deserialize_with = "file_contents_deserializer")]
   #[partial_attr(serde(
@@ -318,6 +325,10 @@ fn default_include_resources() -> bool {
   true
 }
 
+fn default_pending_alert() -> bool {
+  true
+}
+
 impl Default for ResourceSyncConfig {
   fn default() -> Self {
     Self {
@@ -338,6 +349,7 @@ impl Default for ResourceSyncConfig {
       delete: Default::default(),
       webhook_enabled: default_webhook_enabled(),
       webhook_secret: Default::default(),
+      pending_alert: default_pending_alert(),
     }
   }
 }

client/core/rs/src/entities/update.rs

@@ -72,6 +72,12 @@ pub struct Update {
   /// Some unstructured, operation specific data. Not for general usage.
   #[serde(default, skip_serializing_if = "String::is_empty")]
   pub other_data: String,
+  /// If the update is for resource config update, give the previous toml contents
+  #[serde(default, skip_serializing_if = "String::is_empty")]
+  pub prev_toml: String,
+  /// If the update is for resource config update, give the current (at time of Update) toml contents
+  #[serde(default, skip_serializing_if = "String::is_empty")]
+  pub current_toml: String,
 }
 
 impl Update {

client/core/rs/src/lib.rs

@@ -170,4 +170,42 @@ impl KomodoClient {
     self.reqwest = reqwest;
     self
   }
+
+  /// Poll an [Update][entities::update::Update] (returned by the `execute` calls) until the
+  /// [UpdateStatus][entities::update::UpdateStatus] is `Complete`, and then return it.
+  #[cfg(not(feature = "blocking"))]
+  pub async fn poll_update_until_complete(
+    &self,
+    update_id: impl Into<String>,
+  ) -> anyhow::Result<entities::update::Update> {
+    let update_id = update_id.into();
+    loop {
+      let update = self
+        .read(api::read::GetUpdate {
+          id: update_id.clone(),
+        })
+        .await?;
+      if update.status == entities::update::UpdateStatus::Complete {
+        return Ok(update);
+      }
+    }
+  }
+
+  /// Poll an [Update][entities::update::Update] (returned by the `execute` calls) until the
+  /// [UpdateStatus][entities::update::UpdateStatus] is `Complete`, and then return it.
+  #[cfg(feature = "blocking")]
+  pub fn poll_update_until_complete(
+    &self,
+    update_id: impl Into<String>,
+  ) -> anyhow::Result<entities::update::Update> {
+    let update_id = update_id.into();
+    loop {
+      let update = self.read(api::read::GetUpdate {
+        id: update_id.clone(),
+      })?;
+      if update.status == entities::update::UpdateStatus::Complete {
+        return Ok(update);
+      }
+    }
+  }
 }

client/core/rs/src/request.rs

@@ -1,5 +1,4 @@
 use anyhow::{Context, anyhow};
-use reqwest::StatusCode;
 use serde::{Serialize, de::DeserializeOwned};
 use serde_json::json;
 use serror::deserialize_error;
@@ -208,7 +207,7 @@ impl KomodoClient {
     let res =
       req.send().await.context("failed to reach Komodo API")?;
     let status = res.status();
-    if status == StatusCode::OK {
+    if status.is_success() {
       match res.json().await {
         Ok(res) => Ok(res),
         Err(e) => Err(anyhow!("{e:#?}").context(status)),
@@ -236,7 +235,7 @@ impl KomodoClient {
       .json(&body);
     let res = req.send().context("failed to reach Komodo API")?;
     let status = res.status();
-    if status == StatusCode::OK {
+    if status.is_success() {
       match res.json() {
         Ok(res) => Ok(res),
         Err(e) => Err(anyhow!("{e:#?}").context(status)),

client/core/ts/package.json

@@ -1,6 +1,6 @@
 {
   "name": "komodo_client",
-  "version": "1.17.1",
+  "version": "1.17.4",
   "description": "Komodo client package",
   "homepage": "https://komo.do",
   "main": "dist/lib.js",

client/core/ts/src/lib.ts

@@ -7,9 +7,13 @@ import {
 } from "./responses.js";
 import {
   AuthRequest,
+  BatchExecutionResponse,
+  ConnectTerminalQuery,
   ExecuteRequest,
   ReadRequest,
+  Update,
   UpdateListItem,
+  UpdateStatus,
   UserRequest,
   WriteRequest,
   WsLoginMessage,
@@ -39,7 +43,7 @@ export function KomodoClient(url: string, options: InitOptions) {
     secret: options.type === "api-key" ? options.params.secret : undefined,
   };
 
-  const request = async <Req, Res>(
+  const request = <Req, Res>(
     path: "/auth" | "/user" | "/read" | "/execute" | "/write",
     request: Req
   ): Promise<Res> =>
@@ -155,6 +159,42 @@ export function KomodoClient(url: string, options: InitOptions) {
       ExecuteResponses[Req["type"]]
     >("/execute", { type, params });
 
+  const execute_and_poll = async <
+    T extends ExecuteRequest["type"],
+    Req extends Extract<ExecuteRequest, { type: T }>
+  >(
+    type: T,
+    params: Req["params"]
+  ) => {
+    const res = await execute(type, params);
+    // Check if its a batch of updates or a single update;
+    if (Array.isArray(res)) {
+      const batch = res as any as BatchExecutionResponse;
+      return await Promise.all(
+        batch.map(async (item) => {
+          if (item.status === "Err") {
+            return item;
+          }
+          return await poll_update_until_complete(item.data._id?.$oid!);
+        })
+      );
+    } else {
+      // it is a single update
+      const update = res as any as Update;
+      return await poll_update_until_complete(update._id?.$oid!);
+    }
+  };
+
+  const poll_update_until_complete = async (update_id: string) => {
+    while (true) {
+      await new Promise((resolve) => setTimeout(resolve, 1000));
+      const update = await read("GetUpdate", { id: update_id });
+      if (update.status === UpdateStatus.Complete) {
+        return update;
+      }
+    }
+  };
+
   const core_version = () => read("GetVersion", {}).then((res) => res.version);
 
   const subscribe_to_update_websocket = async ({
@@ -231,6 +271,62 @@ export function KomodoClient(url: string, options: InitOptions) {
     }
   };
 
+  const connect_terminal = ({
+    query,
+    on_message,
+    on_login,
+    on_open,
+    on_close,
+  }: {
+    query: ConnectTerminalQuery;
+    on_message?: (e: MessageEvent<any>) => void;
+    on_login?: () => void;
+    on_open?: () => void;
+    on_close?: () => void;
+  }) => {
+    const url_query = new URLSearchParams(
+      query as any as Record<string, string>
+    ).toString();
+    const ws = new WebSocket(
+      url.replace("http", "ws") + "/ws/terminal?" + url_query
+    );
+    // Handle login on websocket open
+    ws.onopen = () => {
+      const login_msg: WsLoginMessage =
+        options.type === "jwt"
+          ? {
+              type: "Jwt",
+              params: {
+                jwt: options.params.jwt,
+              },
+            }
+          : {
+              type: "ApiKeys",
+              params: {
+                key: options.params.key,
+                secret: options.params.secret,
+              },
+            };
+      ws.send(JSON.stringify(login_msg));
+      on_open?.();
+    };
+
+    ws.onmessage = (e) => {
+      if (e.data == "LOGGED_IN") {
+        ws.binaryType = "arraybuffer";
+        ws.onmessage = (e) => on_message?.(e);
+        on_login?.();
+        return;
+      } else {
+        on_message?.(e);
+      }
+    };
+
+    ws.onclose = () => on_close?.();
+
+    return ws;
+  };
+
   return {
     /**
      * Call the `/auth` api.
@@ -290,9 +386,29 @@ export function KomodoClient(url: string, options: InitOptions) {
      * });
      * ```
      *
+     * NOTE. These calls return immediately when the update is created, NOT when the execution task finishes.
+     * To have the call only return when the task finishes, use [execute_and_poll_until_complete].
+     *
      * https://docs.rs/komodo_client/latest/komodo_client/api/execute/index.html
      */
     execute,
+    /**
+     * Call the `/execute` api, and poll the update until the task has completed.
+     *
+     * ```
+     * const update = await komodo.execute_and_poll("DeployStack", {
+     *   stack: "my-stack"
+     * });
+     * ```
+     *
+     * https://docs.rs/komodo_client/latest/komodo_client/api/execute/index.html
+     */
+    execute_and_poll,
+    /**
+     * Poll an Update (returned by the `execute` calls) until the `status` is `Complete`.
+     * https://docs.rs/komodo_client/latest/komodo_client/entities/update/struct.Update.html#structfield.status.
+     */
+    poll_update_until_complete,
     /** Returns the version of Komodo Core the client is calling to. */
     core_version,
     /**
@@ -301,5 +417,10 @@ export function KomodoClient(url: string, options: InitOptions) {
      * Note. Awaiting this method will never finish.
      */
     subscribe_to_update_websocket,
+    /**
+     * Subscribes to terminal io over websocket message,
+     * for use with xtermjs.
+     */
+    connect_terminal,
   };
 }

client/core/ts/src/responses.ts

@@ -80,6 +80,7 @@ export type ReadResponses = {
   GetHistoricalServerStats: Types.GetHistoricalServerStatsResponse;
   ListServers: Types.ListServersResponse;
   ListFullServers: Types.ListFullServersResponse;
+  ListTerminals: Types.ListTerminalsResponse;
 
   // ==== DEPLOYMENT ====
   GetDeploymentsSummary: Types.GetDeploymentsSummaryResponse;
@@ -213,6 +214,9 @@ export type WriteResponses = {
   UpdateServer: Types.Server;
   RenameServer: Types.Update;
   CreateNetwork: Types.Update;
+  CreateTerminal: Types.NoData;
+  DeleteTerminal: Types.NoData;
+  DeleteAllTerminals: Types.NoData;
 
   // ==== DEPLOYMENT ====
   CreateDeployment: Types.Deployment;

client/core/ts/src/types.ts

@@ -51,12 +51,47 @@ export interface Resource<Config, Info> {
 	base_permission?: PermissionLevel;
 }
 
+export enum ScheduleFormat {
+	English = "English",
+	Cron = "Cron",
+}
+
 export interface ActionConfig {
+	/** Choose whether to specify schedule as regular CRON, or using the english to CRON parser. */
+	schedule_format?: ScheduleFormat;
 	/**
-	 * Typescript file contents using pre-initialized `komodo` client.
-	 * Supports variable / secret interpolation.
+	 * Optionally provide a schedule for the procedure to run on.
+	 * 
+	 * There are 2 ways to specify a schedule:
+	 * 
+	 * 1. Regular CRON expression:
+	 * 
+	 * (second, minute, hour, day, month, day-of-week)
+	 * ```
+	 * 0 0 0 1,15 * ?
+	 * ```
+	 * 
+	 * 2. "English" expression via [english-to-cron](https://crates.io/crates/english-to-cron):
+	 * 
+	 * ```
+	 * at midnight on the 1st and 15th of the month
+	 * ```
 	 */
-	file_contents?: string;
+	schedule?: string;
+	/**
+	 * Whether schedule is enabled if one is provided.
+	 * Can be used to temporarily disable the schedule.
+	 */
+	schedule_enabled: boolean;
+	/**
+	 * Optional. A TZ Identifier. If not provided, will use Core local timezone.
+	 * https://en.wikipedia.org/wiki/List_of_tz_database_time_zones.
+	 */
+	schedule_timezone?: string;
+	/** Whether to send alerts when the schedule was run. */
+	schedule_alert: boolean;
+	/** Whether to send alerts when this action fails. */
+	failure_alert: boolean;
 	/** Whether incoming webhooks actually trigger action. */
 	webhook_enabled: boolean;
 	/**
@@ -64,6 +99,11 @@ export interface ActionConfig {
 	 * If its an empty string, use the default secret from the config.
 	 */
 	webhook_secret?: string;
+	/**
+	 * Typescript file contents using pre-initialized `komodo` client.
+	 * Supports variable / secret interpolation.
+	 */
+	file_contents?: string;
 }
 
 export interface ActionInfo {
@@ -102,6 +142,16 @@ export interface ActionListItemInfo {
 	last_run_at: I64;
 	/** Whether last action run successful */
 	state: ActionState;
+	/**
+	 * If the procedure has schedule enabled, this is the
+	 * next scheduled run time in unix ms.
+	 */
+	next_scheduled_run?: I64;
+	/**
+	 * If there is an error parsing schedule expression,
+	 * it will be given here.
+	 */
+	schedule_error?: string;
 }
 
 export type ActionListItem = ResourceListItem<ActionListItemInfo>;
@@ -135,7 +185,9 @@ export type AlerterEndpoint =
 	/** Send alert to a Discord app */
 	| { type: "Discord", params: DiscordAlerterEndpoint }
 	/** Send alert to Ntfy */
-	| { type: "Ntfy", params: NtfyAlerterEndpoint };
+	| { type: "Ntfy", params: NtfyAlerterEndpoint }
+	/** Send alert to Pushover */
+	| { type: "Pushover", params: PushoverAlerterEndpoint };
 
 /** Used to reference a specific resource across all resource types */
 export type ResourceTarget = 
@@ -544,6 +596,41 @@ export interface ProcedureStage {
 export interface ProcedureConfig {
 	/** The stages to be run by the procedure. */
 	stages?: ProcedureStage[];
+	/** Choose whether to specify schedule as regular CRON, or using the english to CRON parser. */
+	schedule_format?: ScheduleFormat;
+	/**
+	 * Optionally provide a schedule for the procedure to run on.
+	 * 
+	 * There are 2 ways to specify a schedule:
+	 * 
+	 * 1. Regular CRON expression:
+	 * 
+	 * (second, minute, hour, day, month, day-of-week)
+	 * ```
+	 * 0 0 0 1,15 * ?
+	 * ```
+	 * 
+	 * 2. "English" expression via [english-to-cron](https://crates.io/crates/english-to-cron):
+	 * 
+	 * ```
+	 * at midnight on the 1st and 15th of the month
+	 * ```
+	 */
+	schedule?: string;
+	/**
+	 * Whether schedule is enabled if one is provided.
+	 * Can be used to temporarily disable the schedule.
+	 */
+	schedule_enabled: boolean;
+	/**
+	 * Optional. A TZ Identifier. If not provided, will use Core local timezone.
+	 * https://en.wikipedia.org/wiki/List_of_tz_database_time_zones.
+	 */
+	schedule_timezone?: string;
+	/** Whether to send alerts when the schedule was run. */
+	schedule_alert: boolean;
+	/** Whether to send alerts when this procedure fails. */
+	failure_alert: boolean;
 	/** Whether incoming webhooks actually trigger action. */
 	webhook_enabled: boolean;
 	/**
@@ -1146,6 +1233,29 @@ export type AlertData =
 	id: string;
 	/** The name of the repo */
 	name: string;
+}}
+	/** A procedure has failed */
+	| { type: "ProcedureFailed", data: {
+	/** The id of the procedure */
+	id: string;
+	/** The name of the procedure */
+	name: string;
+}}
+	/** An action has failed */
+	| { type: "ActionFailed", data: {
+	/** The id of the action */
+	id: string;
+	/** The name of the action */
+	name: string;
+}}
+	/** A schedule was run */
+	| { type: "ScheduleRun", data: {
+	/** Procedure or Action */
+	resource_type: ResourceTarget["type"];
+	/** The resource id */
+	id: string;
+	/** The resource name */
+	name: string;
 }};
 
 /** Representation of an alert in the system. */
@@ -1424,6 +1534,11 @@ export interface ResourceSyncConfig {
 	include_variables?: boolean;
 	/** Whether sync should include user groups. */
 	include_user_groups?: boolean;
+	/**
+	 * Whether sync should send alert when it enters Pending state.
+	 * Default: true
+	 */
+	pending_alert: boolean;
 	/** Manage the file contents in the UI. */
 	file_contents?: string;
 }
@@ -1884,6 +1999,8 @@ export interface SystemInformation {
 	host_name?: string;
 	/** The CPU's brand */
 	cpu_brand: string;
+	/** Whether terminals are disabled on this Periphery */
+	terminals_disabled: boolean;
 }
 
 export type GetSystemInformationResponse = SystemInformation;
@@ -2198,6 +2315,10 @@ export interface Update {
 	commit_hash?: string;
 	/** Some unstructured, operation specific data. Not for general usage. */
 	other_data?: string;
+	/** If the update is for resource config update, give the previous toml contents */
+	prev_toml?: string;
+	/** If the update is for resource config update, give the current (at time of Update) toml contents */
+	current_toml?: string;
 }
 
 export type GetUpdateResponse = Update;
@@ -3286,6 +3407,16 @@ export interface ProcedureListItemInfo {
 	stages: I64;
 	/** Reflect whether last run successful / currently running. */
 	state: ProcedureState;
+	/**
+	 * If the procedure has schedule enabled, this is the
+	 * next scheduled run time in unix ms.
+	 */
+	next_scheduled_run?: I64;
+	/**
+	 * If there is an error parsing schedule expression,
+	 * it will be given here.
+	 */
+	schedule_error?: string;
 }
 
 export type ProcedureListItem = ResourceListItem<ProcedureListItemInfo>;
@@ -3417,6 +3548,8 @@ export interface ServerListItemInfo {
 	send_mem_alerts: boolean;
 	/** Whether server is configured to send disk alerts. */
 	send_disk_alerts: boolean;
+	/** Whether terminals are disabled for this Server. */
+	terminals_disabled: boolean;
 }
 
 export type ServerListItem = ResourceListItem<ServerListItemInfo>;
@@ -3541,6 +3674,21 @@ export type ListSystemProcessesResponse = SystemProcess[];
 
 export type ListTagsResponse = Tag[];
 
+/**
+ * Info about an active terminal on a server.
+ * Retrieve with [ListTerminals][crate::api::read::server::ListTerminals].
+ */
+export interface TerminalInfo {
+	/** The name of the terminal. */
+	name: string;
+	/** The root program / args of the pty */
+	command: string;
+	/** The size of the terminal history in memory. */
+	stored_size_kb: number;
+}
+
+export type ListTerminalsResponse = TerminalInfo[];
+
 export type ListUserGroupsResponse = UserGroup[];
 
 export type ListUserTargetPermissionsResponse = Permission[];
@@ -4088,6 +4236,26 @@ export interface CommitSync {
 	sync: string;
 }
 
+/**
+ * Query to connect to a terminal (interactive shell over websocket) on the given server.
+ * TODO: Document calling.
+ */
+export interface ConnectTerminalQuery {
+	/** Server Id or name */
+	server: string;
+	/**
+	 * Each periphery can keep multiple terminals open.
+	 * If a terminals with the specified name already exists,
+	 * it will be attached to.
+	 * Otherwise a new terminal will be created for the command,
+	 * which will persist until it is deleted using
+	 * [DeleteTerminal][crate::api::write::server::DeleteTerminal]
+	 */
+	terminal: string;
+	/** Optional. The initial command to execute on connection to the shell. */
+	init?: string;
+}
+
 export interface Conversion {
 	/** reference on the server. */
 	local: string;
@@ -4478,6 +4646,44 @@ export interface CreateTag {
 	name: string;
 }
 
+/**
+ * Configures the behavior of [CreateTerminal] if the
+ * specified terminal name already exists.
+ */
+export enum TerminalRecreateMode {
+	/**
+	 * Never kill the old terminal if it already exists.
+	 * If the command is different, returns error.
+	 */
+	Never = "Never",
+	/** Always kill the old terminal and create new one */
+	Always = "Always",
+	/** Only kill and recreate if the command is different. */
+	DifferentCommand = "DifferentCommand",
+}
+
+/**
+ * Create a terminal on the server.
+ * Response: [NoData]
+ */
+export interface CreateTerminal {
+	/** Server Id or name */
+	server: string;
+	/** The name of the terminal on the server to create. */
+	name: string;
+	/**
+	 * The shell command (eg `bash`) to init the shell.
+	 * 
+	 * This can also include args:
+	 * `docker exec -it container sh`
+	 * 
+	 * Default: `bash`
+	 */
+	command: string;
+	/** Default: `Never` */
+	recreate?: TerminalRecreateMode;
+}
+
 /** **Admin only.** Create a user group. Response: [UserGroup] */
 export interface CreateUserGroup {
 	/** The name to assign to the new UserGroup */
@@ -4529,6 +4735,15 @@ export interface DeleteAlerter {
 	id: string;
 }
 
+/**
+ * Delete all terminals on the server.
+ * Response: [NoData]
+ */
+export interface DeleteAllTerminals {
+	/** Server Id or name */
+	server: string;
+}
+
 /**
  * Delete an api key for the calling user.
  * Response: [NoData]
@@ -4722,6 +4937,17 @@ export interface DeleteTag {
 	id: string;
 }
 
+/**
+ * Delete a terminal on the server.
+ * Response: [NoData]
+ */
+export interface DeleteTerminal {
+	/** Server Id or name */
+	server: string;
+	/** The name of the terminal on the server to delete. */
+	terminal: string;
+}
+
 /**
  * **Admin only**. Delete a user.
  * Admins can delete any non-admin user.
@@ -6308,6 +6534,20 @@ export interface ListTags {
 	query?: MongoDocument;
 }
 
+/**
+ * List the current terminals on specified server.
+ * Response: [ListTerminalsResponse].
+ */
+export interface ListTerminals {
+	/** Id or name */
+	server: string;
+	/**
+	 * Force a fresh call to Periphery for the list.
+	 * Otherwise the response will be cached for 30s
+	 */
+	fresh?: boolean;
+}
+
 /**
  * Paginated endpoint for updates matching optional query.
  * More recent updates will be returned first.
@@ -6609,6 +6849,12 @@ export interface PushRecentlyViewed {
 	resource: ResourceTarget;
 }
 
+/** Configuration for a Pushover alerter. */
+export interface PushoverAlerterEndpoint {
+	/** The pushover URL including application and user tokens in parameters. */
+	url: string;
+}
+
 /** Trigger a refresh of the cached latest hash and message. */
 export interface RefreshBuildCache {
 	/** Id or name */
@@ -7735,6 +7981,7 @@ export type ReadRequest =
 	| { type: "ListDockerImages", params: ListDockerImages }
 	| { type: "ListDockerVolumes", params: ListDockerVolumes }
 	| { type: "ListComposeProjects", params: ListComposeProjects }
+	| { type: "ListTerminals", params: ListTerminals }
 	| { type: "GetDeploymentsSummary", params: GetDeploymentsSummary }
 	| { type: "GetDeployment", params: GetDeployment }
 	| { type: "GetDeploymentContainer", params: GetDeploymentContainer }
@@ -7833,6 +8080,9 @@ export type WriteRequest =
 	| { type: "UpdateServer", params: UpdateServer }
 	| { type: "RenameServer", params: RenameServer }
 	| { type: "CreateNetwork", params: CreateNetwork }
+	| { type: "CreateTerminal", params: CreateTerminal }
+	| { type: "DeleteTerminal", params: DeleteTerminal }
+	| { type: "DeleteAllTerminals", params: DeleteAllTerminals }
 	| { type: "CreateDeployment", params: CreateDeployment }
 	| { type: "CopyDeployment", params: CopyDeployment }
 	| { type: "CreateDeploymentFromContainer", params: CreateDeploymentFromContainer }

client/periphery/rs/Cargo.toml

@@ -13,11 +13,15 @@ repository.workspace = true
 # local
 komodo_client.workspace = true
 # mogh
-serror.workspace = true
 resolver_api.workspace = true
+serror.workspace = true
 # external
-reqwest.workspace = true
-anyhow.workspace = true
-serde.workspace = true
+tokio-tungstenite.workspace = true
 serde_json.workspace = true
-tracing.workspace = true
+serde_qs.workspace = true
+reqwest.workspace = true
+tracing.workspace = true
+anyhow.workspace = true
+rustls.workspace = true
+tokio.workspace = true
+serde.workspace = true

client/periphery/rs/src/api/git.rs

@@ -106,6 +106,6 @@ pub struct RenameRepo {
 #[error(serror::Error)]
 pub struct DeleteRepo {
   pub name: String,
-  /// Clears 
+  /// Clears
   pub is_build: bool,
 }

client/periphery/rs/src/api/mod.rs

@@ -19,6 +19,7 @@ pub mod git;
 pub mod image;
 pub mod network;
 pub mod stats;
+pub mod terminal;
 pub mod volume;
 
 //

client/periphery/rs/src/api/terminal.rs

@@ -0,0 +1,80 @@
+use komodo_client::{
+  api::write::TerminalRecreateMode,
+  entities::{NoData, server::TerminalInfo},
+};
+use resolver_api::Resolve;
+use serde::{Deserialize, Serialize};
+
+#[derive(Serialize, Deserialize, Debug, Clone, Resolve)]
+#[response(Vec<TerminalInfo>)]
+#[error(serror::Error)]
+pub struct ListTerminals {}
+
+#[derive(Serialize, Deserialize, Debug, Clone, Resolve)]
+#[response(NoData)]
+#[error(serror::Error)]
+pub struct CreateTerminal {
+  /// The name of the terminal to create
+  pub name: String,
+  /// The shell command (eg `bash`) to init the shell.
+  ///
+  /// This can also include args:
+  /// `docker exec -it container sh`
+  #[serde(default = "default_command")]
+  pub command: String,
+  /// Default: `Never`
+  #[serde(default)]
+  pub recreate: TerminalRecreateMode,
+}
+
+fn default_command() -> String {
+  String::from("bash")
+}
+
+//
+
+#[derive(Serialize, Deserialize, Debug, Clone, Resolve)]
+#[response(NoData)]
+#[error(serror::Error)]
+pub struct DeleteTerminal {
+  /// The name of the terminal to delete
+  pub terminal: String,
+}
+
+//
+
+#[derive(Serialize, Deserialize, Debug, Clone, Resolve)]
+#[response(NoData)]
+#[error(serror::Error)]
+pub struct DeleteAllTerminals {}
+
+//
+
+/// Create a single use auth token to connect to periphery terminal websocket.
+#[derive(Serialize, Deserialize, Debug, Clone, Resolve)]
+#[response(CreateTerminalAuthTokenResponse)]
+#[error(serror::Error)]
+pub struct CreateTerminalAuthToken {}
+
+#[derive(Debug, Clone, Default, Serialize, Deserialize)]
+pub struct CreateTerminalAuthTokenResponse {
+  pub token: String,
+}
+
+//
+
+#[derive(Serialize, Deserialize, Debug, Clone)]
+pub struct ConnectTerminalQuery {
+  /// Use [CreateTerminalAuthToken] to create a single-use
+  /// token to send in the query.
+  pub token: String,
+  /// Each periphery can keep multiple terminals open.
+  /// If a terminal with the specified name already exists,
+  /// it will be attached to.
+  /// Otherwise a new terminal will be created,
+  /// which will persist until it is either exited via command (ie `exit`),
+  /// or deleted using [DeleteTerminal]
+  pub terminal: String,
+  /// Optional. The initial command to execute on connection to the shell.
+  pub init: Option<String>,
+}

client/periphery/rs/src/lib.rs

@@ -8,6 +8,8 @@ use serde_json::json;
 
 pub mod api;
 
+mod terminal;
+
 fn periphery_http_client() -> &'static reqwest::Client {
   static PERIPHERY_HTTP_CLIENT: OnceLock<reqwest::Client> =
     OnceLock::new();
@@ -95,12 +97,12 @@ impl PeripheryClient {
       req.send().await.context("failed at request to periphery")?;
     let status = res.status();
     tracing::debug!(
-      "got response | type: {req_type} | {status} | body: {res:?}",
+      "got response | type: {req_type} | {status} | response: {res:?}",
     );
     if status == StatusCode::OK {
       tracing::debug!("response ok, deserializing");
       res.json().await.with_context(|| format!(
-        "failed to parse response to json | type: {req_type} | body: {request:?}"
+        "failed to parse response to json | type: {req_type} | request: {request:?}"
       ))
     } else {
       tracing::debug!("response is non-200");

client/periphery/rs/src/terminal.rs

@@ -0,0 +1,125 @@
+use std::sync::Arc;
+
+use anyhow::Context;
+use rustls::{ClientConfig, client::danger::ServerCertVerifier};
+use tokio::net::TcpStream;
+use tokio_tungstenite::{Connector, MaybeTlsStream, WebSocketStream};
+
+use crate::{
+  PeripheryClient,
+  api::terminal::{ConnectTerminalQuery, CreateTerminalAuthToken},
+};
+
+impl PeripheryClient {
+  /// Handles ws connect and login.
+  /// Does not handle reconnect.
+  pub async fn connect_terminal(
+    &self,
+    terminal: String,
+    init: Option<String>,
+  ) -> anyhow::Result<WebSocketStream<MaybeTlsStream<TcpStream>>> {
+    tracing::trace!(
+      "request | type: ConnectTerminal | terminal name: {terminal} | init command: {init:?}",
+    );
+
+    let token = self
+      .request(CreateTerminalAuthToken {})
+      .await
+      .context("Failed to create terminal auth token")?;
+
+    let query_str = serde_qs::to_string(&ConnectTerminalQuery {
+      token: token.token,
+      terminal,
+      init,
+    })
+    .context("Failed to serialize query string")?;
+
+    let url = format!(
+      "{}/terminal?{query_str}",
+      self.address.replacen("http", "ws", 1)
+    );
+
+    let (stream, _) = if url.starts_with("wss") {
+      tokio_tungstenite::connect_async_tls_with_config(
+        url,
+        None,
+        false,
+        Some(Connector::Rustls(Arc::new(
+          ClientConfig::builder()
+            .dangerous()
+            .with_custom_certificate_verifier(Arc::new(
+              InsecureVerifier,
+            ))
+            .with_no_client_auth(),
+        ))),
+      )
+      .await
+      .context("failed to connect to websocket")?
+    } else {
+      tokio_tungstenite::connect_async(url)
+        .await
+        .context("failed to connect to websocket")?
+    };
+
+    Ok(stream)
+  }
+}
+
+#[derive(Debug)]
+struct InsecureVerifier;
+
+impl ServerCertVerifier for InsecureVerifier {
+  fn verify_server_cert(
+    &self,
+    _end_entity: &rustls::pki_types::CertificateDer<'_>,
+    _intermediates: &[rustls::pki_types::CertificateDer<'_>],
+    _server_name: &rustls::pki_types::ServerName<'_>,
+    _ocsp_response: &[u8],
+    _now: rustls::pki_types::UnixTime,
+  ) -> Result<rustls::client::danger::ServerCertVerified, rustls::Error>
+  {
+    Ok(rustls::client::danger::ServerCertVerified::assertion())
+  }
+
+  fn verify_tls12_signature(
+    &self,
+    _message: &[u8],
+    _cert: &rustls::pki_types::CertificateDer<'_>,
+    _dss: &rustls::DigitallySignedStruct,
+  ) -> Result<
+    rustls::client::danger::HandshakeSignatureValid,
+    rustls::Error,
+  > {
+    Ok(rustls::client::danger::HandshakeSignatureValid::assertion())
+  }
+
+  fn verify_tls13_signature(
+    &self,
+    _message: &[u8],
+    _cert: &rustls::pki_types::CertificateDer<'_>,
+    _dss: &rustls::DigitallySignedStruct,
+  ) -> Result<
+    rustls::client::danger::HandshakeSignatureValid,
+    rustls::Error,
+  > {
+    Ok(rustls::client::danger::HandshakeSignatureValid::assertion())
+  }
+
+  fn supported_verify_schemes(&self) -> Vec<rustls::SignatureScheme> {
+    vec![
+      rustls::SignatureScheme::RSA_PKCS1_SHA1,
+      rustls::SignatureScheme::ECDSA_SHA1_Legacy,
+      rustls::SignatureScheme::RSA_PKCS1_SHA256,
+      rustls::SignatureScheme::ECDSA_NISTP256_SHA256,
+      rustls::SignatureScheme::RSA_PKCS1_SHA384,
+      rustls::SignatureScheme::ECDSA_NISTP384_SHA384,
+      rustls::SignatureScheme::RSA_PKCS1_SHA512,
+      rustls::SignatureScheme::ECDSA_NISTP521_SHA512,
+      rustls::SignatureScheme::RSA_PSS_SHA256,
+      rustls::SignatureScheme::RSA_PSS_SHA384,
+      rustls::SignatureScheme::RSA_PSS_SHA512,
+      rustls::SignatureScheme::ED25519,
+      rustls::SignatureScheme::ED448,
+    ]
+  }
+}

compose/compose.env

@@ -21,6 +21,10 @@ KOMODO_DB_PASSWORD=admin
 ## Configure a secure passkey to authenticate between Core / Periphery.
 KOMODO_PASSKEY=a_random_passkey
 
+## Set your time zone for schedules
+## https://en.wikipedia.org/wiki/List_of_tz_database_time_zones
+TZ=Etc/UTC
+
 #=-------------------------=#
 #= Komodo Core Environment =#
 #=-------------------------=#
@@ -116,21 +120,19 @@ KOMODO_HETZNER_TOKEN= # Alt: KOMODO_HETZNER_TOKEN_FILE
 ## Full variable list + descriptions are available here:
 ## 🦎 https://github.com/moghtech/komodo/blob/main/config/periphery.config.toml 🦎
 
-## Periphery passkeys must include KOMODO_PASSKEY to authenticate.
-PERIPHERY_PASSKEYS=${KOMODO_PASSKEY}
-
 ## Specify the root directory used by Periphery agent.
 PERIPHERY_ROOT_DIRECTORY=/etc/komodo
 
-PERIPHERY_REPO_DIR=$PERIPHERY_ROOT_DIRECTORY/repos
-PERIPHERY_STACK_DIR=$PERIPHERY_ROOT_DIRECTORY/stacks
-PERIPHERY_BUILD_DIR=$PERIPHERY_ROOT_DIRECTORY/builds
+## Periphery passkeys must include KOMODO_PASSKEY to authenticate.
+PERIPHERY_PASSKEYS=${KOMODO_PASSKEY}
+
+## Specify whether to disable the terminals feature
+## and disallow remote shell access (inside the Periphery container).
+PERIPHERY_DISABLE_TERMINALS=false
 
 ## Enable SSL using self signed certificates.
 ## Connect to Periphery at https://address:8120.
 PERIPHERY_SSL_ENABLED=true
-PERIPHERY_SSL_KEY_FILE=$PERIPHERY_ROOT_DIRECTORY/ssl/key.pem
-PERIPHERY_SSL_CERT_FILE=$PERIPHERY_ROOT_DIRECTORY/ssl/cert.pem
 
 ## If the disk size is overreporting, can use one of these to 
 ## whitelist / blacklist the disks to filter them, whichever is easier.

compose/periphery.compose.yaml

@@ -15,13 +15,14 @@ services:
       driver: ${COMPOSE_LOGGING_DRIVER:-local}
     ## https://komo.do/docs/connect-servers#configuration
     environment:
-      PERIPHERY_REPO_DIR: ${PERIPHERY_ROOT_DIRECTORY:-/etc/komodo}/repos
-      PERIPHERY_STACK_DIR: ${PERIPHERY_ROOT_DIRECTORY:-/etc/komodo}/stacks
-      PERIPHERY_BUILD_DIR: ${PERIPHERY_ROOT_DIRECTORY:-/etc/komodo}/builds
-      PERIPHERY_SSL_KEY_FILE: ${PERIPHERY_ROOT_DIRECTORY:-/etc/komodo}/ssl/key.pem
-      PERIPHERY_SSL_CERT_FILE: ${PERIPHERY_ROOT_DIRECTORY:-/etc/komodo}/ssl/cert.pem
+      PERIPHERY_ROOT_DIRECTORY: ${PERIPHERY_ROOT_DIRECTORY:-/etc/komodo}
       ## Pass the same passkey as used by the Komodo Core connecting to this Periphery agent.
       PERIPHERY_PASSKEYS: abc123
+      ## Make server run over https
+      PERIPHERY_SSL_ENABLED: true
+      ## Specify whether to disable the terminals feature
+      ## and disallow remote shell access (inside the Periphery container).
+      PERIPHERY_DISABLE_TERMINALS: false
       ## If the disk size is overreporting, can use one of these to 
       ## whitelist / blacklist the disks to filter them, whichever is easier.
       ## Accepts comma separated list of paths.

config/core.config.toml

@@ -341,12 +341,18 @@ webhook_base_url = ""
 ## Default: info
 logging.level = "info"
 
-## Specify the logging format for stdout / stderr.
+## Specify the logging format.
 ## Env: KOMODO_LOGGING_STDIO
 ## Options: standard, json, none
 ## Default: standard
 logging.stdio = "standard"
 
+## Specify whether logging is more human readable.
+## Note. Single logs will span multiple lines.
+## Env: KOMODO_LOGGING_PRETTY
+## Default: false
+logging.pretty = false
+
 ## Optionally specify a opentelemetry otlp endpoint to send traces to.
 ## Example: http://localhost:4317
 ## Env: KOMODO_LOGGING_OTLP_ENDPOINT

config/periphery.config.toml

@@ -24,28 +24,40 @@ port = 8120
 ## Default: [::]
 bind_ip = "[::]"
 
-## The directory periphery will use to manage repos.
+## The directory periphery will use as the default base for the directories it uses.
+## The periphery user must have write access to this directory.
+## Env: PERIPHERY_ROOT_DIRECTORY
+## Default: /etc/komodo
+root_directory = "/etc/komodo"
+
+## Optional. Override the directory periphery will use to manage repos.
 ## The periphery user must have write access to this directory.
 ## Env: PERIPHERY_REPO_DIR
-## Default: /etc/komodo/repos
-repo_dir = "/etc/komodo/repos"
+## Default: ${root_directory}/repos
+# repo_dir = "/etc/komodo/repos"
 
-## The directory periphery will use to manage stacks.
+## Optional. Override the directory periphery will use to manage stacks.
 ## The periphery user must have write access to this directory.
 ## Env: PERIPHERY_STACK_DIR
-## Default: /etc/komodo/stacks
-stack_dir = "/etc/komodo/stacks"
+## Default: ${root_directory}/stacks
+# stack_dir = "/etc/komodo/stacks"
 
-## The directory periphery will use to manage builds.
+## Optional. Override the directory periphery will use to manage builds.
 ## The periphery user must have write access to this directory.
 ## Env: PERIPHERY_BUILD_DIR
-## Default: /etc/komodo/builds
-build_dir = "/etc/komodo/builds"
+## Default: ${root_directory}/builds
+# build_dir = "/etc/komodo/builds"
+
+## Disable the terminal APIs and disallow remote shell access through Periphery.
+## Env: PERIPHERY_DISABLE_TERMINALS
+## Default: false
+disable_terminals = false
 
 ## How often Periphery polls the host for system stats,
-## like CPU / memory usage.
+## like CPU / memory usage. To effectively disable polling,
+## set this to something like 1-hr.
 ## Env: PERIPHERY_STATS_POLLING_RATE
-## Options: 1-sec, 5-sec, 10-sec, 30-sec, 1-min
+## Options: 1-sec, 5-sec, 10-sec, 30-sec, 1-min, 5-min, 30-min, 1-hr
 ## Default: 5-sec
 stats_polling_rate = "5-sec"
 
@@ -116,6 +128,12 @@ logging.level = "info"
 ## Default: standard
 logging.stdio = "standard"
 
+## Specify whether logging is more human readable.
+## Note. Single logs will span multiple lines.
+## Env: PERIPHERY_LOGGING_PRETTY
+## Default: false
+logging.pretty = false
+
 ## Specify a opentelemetry otlp endpoint to send traces to.
 ## Example: http://localhost:4317.
 ## Env: PERIPHERY_LOGGING_OTLP_ENDPOINT

docsite/docs/connect-servers.mdx

@@ -11,7 +11,7 @@ Connecting a server to Komodo has 2 steps:
 
 ## Install Periphery
 
-You can install Periphery as a systemd managed process, run it as a [docker container](https://github.com/moghtech/komodo/pkgs/container/periphery), or do whatever you want with the binary.
+You can install Periphery as a systemd managed process, run it as a [docker container](https://github.com/moghtech/komodo/pkgs/container/komodo-periphery), or do whatever you want with the binary.
 
 :::warning
 Allowing unintended access to the Periphery agent API is a security risk.

docsite/docs/other-resources.md

@@ -5,7 +5,7 @@
 - [FAQ, Tips, and Tricks](https://blog.foxxmd.dev/posts/komodo-tips-tricks) by [FoxxMD](https://github.com/FoxxMD)
 - [Compose Environments Explained](https://blog.foxxmd.dev/posts/compose-envs-explained) by [FoxxMD](https://github.com/FoxxMD)
 - [How To: Automate version updates for your self-hosted Docker containers with Gitea, Renovate, and Komodo](https://nickcunningh.am/blog/how-to-automate-version-updates-for-your-self-hosted-docker-containers-with-gitea-renovate-and-komodo) by [TheNickOfTime](https://github.com/TheNickOfTime)
-
+- [Setting up Komodo, comparison to Portainer, and FAQ](https://skyblog.one/komodo-the-better-alternative-to-portainer-for-container-management) by [Skyfay](https://skyblog.one/authors/)
 ### Community Alerters
 These provide alerting implementations which can be used with the `Custom` Alerter type.
 - [Discord](https://github.com/FoxxMD/deploy-discord-alerter) by [FoxxMD](https://github.com/FoxxMD)

frontend/Dockerfile

@@ -19,5 +19,5 @@ FROM scratch
 COPY --from=builder /builder/frontend/dist /frontend
 
 LABEL org.opencontainers.image.source=https://github.com/moghtech/komodo
-LABEL org.opencontainers.image.description="Komodo Periphery"
+LABEL org.opencontainers.image.description="Komodo Frontend"
 LABEL org.opencontainers.image.licenses=GPL-3.0