improve 2fa ux

Cargo.toml

@@ -8,7 +8,7 @@ members = [
 ]
 
 [workspace.package]
-version = "2.0.0-dev-91"
+version = "2.0.0-dev-98"
 edition = "2024"
 authors = ["mbecker20 <becker.maxh@gmail.com>"]
 license = "GPL-3.0-or-later"
@@ -26,7 +26,9 @@ environment_file = { path = "lib/environment_file" }
 environment = { path = "lib/environment" }
 interpolate = { path = "lib/interpolate" }
 secret_file = { path = "lib/secret_file" }
+validations = { path = "lib/validations" }
 formatting = { path = "lib/formatting" }
+rate_limit = { path = "lib/rate_limit" }
 transport = { path = "lib/transport" }
 database = { path = "lib/database" }
 encoding = { path = "lib/encoding" }
@@ -63,14 +65,15 @@ arc-swap = "1.7.1"
 
 # SERVER
 tokio-tungstenite = { version = "0.28.0", features = ["rustls-tls-native-roots"] }
-axum-extra = { version = "0.12.2", features = ["typed-header"] }
-tower-http = { version = "0.6.6", features = ["fs", "cors"] }
-axum-server = { version = "0.7.3", features = ["tls-rustls"] }
+tower-http = { version = "0.6.8", features = ["fs", "cors", "set-header"] }
 axum = { version = "0.8.7", features = ["ws", "json", "macros"] }
+axum-extra = { version = "0.12.2", features = ["typed-header"] }
+axum-server = { version = "0.8.0", features = ["tls-rustls"] }
+tower-sessions = "0.14.0"
 
 # SER/DE
 ipnetwork = { version = "0.21.1", features = ["serde"] }
-indexmap = { version = "2.12.0", features = ["serde"] }
+indexmap = { version = "2.12.1", features = ["serde"] }
 serde = { version = "1.0.227", features = ["derive"] }
 strum = { version = "0.27.2", features = ["derive"] }
 bson = { version = "2.15.0" } # must keep in sync with mongodb version
@@ -87,35 +90,39 @@ thiserror = "2.0.17"
 # LOGGING
 opentelemetry-otlp = { version = "0.31.0", features = ["tls-roots", "reqwest-rustls"] }
 opentelemetry_sdk = { version = "0.31.0", features = ["rt-tokio"] }
-tracing-subscriber = { version = "0.3.20", features = ["json"] }
+tracing-subscriber = { version = "0.3.22", features = ["json"] }
 opentelemetry-semantic-conventions = "0.31.0"
 tracing-opentelemetry = "0.32.0"
 opentelemetry = "0.31.0"
-tracing = "0.1.41"
+tracing = "0.1.43"
 
 # CONFIG
-clap = { version = "4.5.52", features = ["derive"] }
+clap = { version = "4.5.53", features = ["derive"] }
 dotenvy = "0.15.7"
 envy = "0.4.2"
 
 # CRYPTO / AUTH
+webauthn-rs = { version = "0.5.4", features = ["danger-allow-state-serialisation"] }
 openidconnect = { version = "4.0.1", features = ["accept-rfc3339-timestamps"] }
-uuid = { version = "1.18.1", features = ["v4", "fast-rng", "serde"] }
+uuid = { version = "1.19.0", features = ["v4", "fast-rng", "serde"] }
 jsonwebtoken = { version = "10.2.0", features = ["aws_lc_rs"] } # locked back with octorust
 rustls = { version = "0.23.35", features = ["aws-lc-rs"] }
 pem-rfc7468 = { version = "1.0.0", features = ["alloc"] }
+totp-rs = { version = "5.7.0", features = ["qr"] }
+webauthn-rs-proto = "0.5.4"
+base64urlsafedata = "0.5.4"
+data-encoding = "2.9.0"
 urlencoding = "2.1.3"
 bcrypt = "0.17.1"
-base64 = "0.22.1"
 pkcs8 = "0.10.2"
 snow = "0.10.0"
 hmac = "0.12.1"
 sha1 = "0.10.6"
 sha2 = "0.10.9"
 rand = "0.9.2"
-hex = "0.4.3"
 spki = "0.7.3"
 der = "0.7.10"
+hex = "0.4.3"
 
 # SYSTEM
 hickory-resolver = "0.25.2"
@@ -127,18 +134,18 @@ sysinfo = "0.37.1"
 shlex = "1.3.0"
 
 # CLOUD
-aws-config = "1.8.10"
-aws-sdk-ec2 = "1.188.0"
-aws-credential-types = "1.2.9"
+aws-config = "1.8.12"
+aws-sdk-ec2 = "1.196.0"
+aws-credential-types = "1.2.11"
 
 ## CRON
-english-to-cron = "0.1.6"
+english-to-cron = "0.1.7"
 chrono-tz = "0.10.4"
 chrono = "0.4.42"
 croner = "3.0.1"
 
 # MISC
-async-compression = { version = "0.4.33", features = ["tokio", "gzip"] }
+async-compression = { version = "0.4.35", features = ["tokio", "gzip"] }
 derive_builder = "0.20.2"
 comfy-table = "7.2.1"
 typeshare = "1.0.4"

bin/binaries.Dockerfile

@@ -1,7 +1,7 @@
 ## Builds the Komodo Core, Periphery, and Util binaries
-## for a specific architecture.
+## for a specific architecture. Requires OpenSSL 3 or later.
 
-FROM rust:1.90.0-bullseye AS builder
+FROM rust:1.91.1-bookworm AS builder
 RUN cargo install cargo-strip
 
 WORKDIR /builder

bin/chef.binaries.Dockerfile

@@ -1,9 +1,9 @@
 ## Builds the Komodo Core, Periphery, and Util binaries
-## for a specific architecture.
+## for a specific architecture. Requires OpenSSL 3 or later.
 
 ## Uses chef for dependency caching to help speed up back-to-back builds.
 
-FROM lukemathwalker/cargo-chef:latest-rust-1.90.0-bullseye AS chef
+FROM lukemathwalker/cargo-chef:latest-rust-1.90.0-bookworm AS chef
 WORKDIR /builder
 
 # Plan just the RECIPE to see if things have changed

bin/cli/aio.Dockerfile

@@ -1,4 +1,4 @@
-FROM rust:1.90.0-bullseye AS builder
+FROM rust:1.91.1-bullseye AS builder
 RUN cargo install cargo-strip
 
 WORKDIR /builder

bin/cli/src/command/execute.rs

@@ -221,6 +221,21 @@ pub async fn handle(
     Execution::SendAlert(data) => {
       println!("{}: {data:?}", "Data".dimmed())
     }
+    Execution::RemoveSwarmNodes(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
+    Execution::RemoveSwarmStacks(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
+    Execution::RemoveSwarmServices(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
+    Execution::RemoveSwarmConfigs(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
+    Execution::RemoveSwarmSecrets(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
     Execution::ClearRepoCache(data) => {
       println!("{}: {data:?}", "Data".dimmed())
     }
@@ -488,6 +503,26 @@ pub async fn handle(
       .execute(request)
       .await
       .map(|u| ExecutionResult::Single(u.into())),
+    Execution::RemoveSwarmNodes(request) => client
+      .execute(request)
+      .await
+      .map(|u| ExecutionResult::Single(u.into())),
+    Execution::RemoveSwarmStacks(request) => client
+      .execute(request)
+      .await
+      .map(|u| ExecutionResult::Single(u.into())),
+    Execution::RemoveSwarmServices(request) => client
+      .execute(request)
+      .await
+      .map(|u| ExecutionResult::Single(u.into())),
+    Execution::RemoveSwarmConfigs(request) => client
+      .execute(request)
+      .await
+      .map(|u| ExecutionResult::Single(u.into())),
+    Execution::RemoveSwarmSecrets(request) => client
+      .execute(request)
+      .await
+      .map(|u| ExecutionResult::Single(u.into())),
     Execution::ClearRepoCache(request) => client
       .execute(request)
       .await

bin/cli/src/command/update/user.rs

@@ -26,6 +26,9 @@ pub async fn update(
     UpdateUserCommand::SuperAdmin { enabled, yes } => {
       update_super_admin(username, *enabled, *yes).await
     }
+    UpdateUserCommand::Clear2fa { yes } => {
+      clear_2fa(username, *yes).await
+    }
   }
 }
 
@@ -120,3 +123,20 @@ async fn update_super_admin(
 
   Ok(())
 }
+
+async fn clear_2fa(username: &str, yes: bool) -> anyhow::Result<()> {
+  println!("\n{}: Clear 2FA Methods\n", "Mode".dimmed());
+  println!(" - {}: {username}", "Username".dimmed());
+
+  crate::command::wait_for_enter("clear user 2FA methods", yes)?;
+
+  info!("Clearing 2FA methods...");
+
+  let db = database::Client::new(&cli_config().database).await?;
+
+  db.clear_user_2fa_methods(username).await?;
+
+  info!("2FA methods cleared ✅");
+
+  Ok(())
+}

bin/core/Cargo.toml

@@ -20,7 +20,9 @@ periphery_client.workspace = true
 environment_file.workspace = true
 interpolate.workspace = true
 secret_file.workspace = true
+validations.workspace = true
 formatting.workspace = true
+rate_limit.workspace = true
 transport.workspace = true
 database.workspace = true
 encoding.workspace = true
@@ -43,34 +45,37 @@ svi.workspace = true
 # external
 aws-credential-types.workspace = true
 english-to-cron.workspace = true
+tower-sessions.workspace = true
 openidconnect.workspace = true
+data-encoding.workspace = true
+serde_yaml_ng.workspace = true
 jsonwebtoken.workspace = true
 futures-util.workspace = true
 axum-server.workspace = true
-urlencoding.workspace = true
 aws-sdk-ec2.workspace = true
+urlencoding.workspace = true
+webauthn-rs.workspace = true
 aws-config.workspace = true
 tokio-util.workspace = true
 axum-extra.workspace = true
 tower-http.workspace = true
 serde_json.workspace = true
-serde_yaml_ng.workspace = true
-serde_qs.workspace = true
 typeshare.workspace = true
 chrono-tz.workspace = true
 indexmap.workspace = true
 wildcard.workspace = true
 arc-swap.workspace = true
+serde_qs.workspace = true
 colored.workspace = true
 dashmap.workspace = true
 tracing.workspace = true
 reqwest.workspace = true
 dotenvy.workspace = true
+totp-rs.workspace = true
 anyhow.workspace = true
 croner.workspace = true
 chrono.workspace = true
 bcrypt.workspace = true
-base64.workspace = true
 rustls.workspace = true
 bytes.workspace = true
 tokio.workspace = true

bin/core/aio.Dockerfile

@@ -1,7 +1,7 @@
 ## All in one, multi stage compile + runtime Docker build for your architecture.
 
 # Build Core
-FROM rust:1.90.0-trixie AS core-builder
+FROM rust:1.91.1-trixie AS core-builder
 RUN cargo install cargo-strip
 
 WORKDIR /builder

bin/core/src/alert/discord.rs

@@ -232,9 +232,9 @@ pub async fn send_alert(
       format!(
         "{level} | {message}{}",
         if details.is_empty() {
-          format_args!("")
+          String::new()
         } else {
-          format_args!("\n{details}")
+          format!("\n{details}")
         }
       )
     }

bin/core/src/alert/mod.rs

@@ -466,9 +466,9 @@ fn standard_alert_content(alert: &Alert) -> String {
       format!(
         "{level} | {message}{}",
         if details.is_empty() {
-          format_args!("")
+          String::new()
         } else {
-          format_args!("\n{details}")
+          format!("\n{details}")
         }
       )
     }

bin/core/src/api/auth.rs

@@ -1,34 +1,61 @@
-use std::{sync::OnceLock, time::Instant};
+use std::{
+  net::{IpAddr, SocketAddr},
+  sync::OnceLock,
+  time::Instant,
+};
 
-use axum::{Router, extract::Path, http::HeaderMap, routing::post};
+use anyhow::{Context, anyhow};
+use axum::{
+  Router,
+  extract::{ConnectInfo, Path},
+  http::HeaderMap,
+  routing::post,
+};
+use data_encoding::BASE32_NOPAD;
+use database::{
+  bson::{doc, to_bson},
+  mungos::by_id::update_one_by_id,
+};
 use derive_variants::{EnumVariants, ExtractVariant};
 use komodo_client::{api::auth::*, entities::user::User};
+use rate_limit::WithFailureRateLimit;
 use reqwest::StatusCode;
 use resolver_api::Resolve;
 use response::Response;
 use serde::{Deserialize, Serialize};
 use serde_json::json;
-use serror::{AddStatusCode, Json};
+use serror::{AddStatusCode, AddStatusCodeError, Json};
+use tower_sessions::Session;
 use typeshare::typeshare;
 use uuid::Uuid;
+use webauthn_rs::prelude::PasskeyAuthentication;
 
 use crate::{
+  api::{
+    SESSION_KEY_PASSKEY_LOGIN, SESSION_KEY_TOTP_LOGIN,
+    SESSION_KEY_USER_ID, memory_session_layer,
+  },
   auth::{
     get_user_id_from_headers,
     github::{self, client::github_oauth_client},
     google::{self, client::google_oauth_client},
     oidc::{self, client::oidc_client},
+    totp::make_totp,
   },
   config::core_config,
   helpers::query::get_user,
-  state::jwt_client,
+  state::{auth_rate_limiter, db_client, jwt_client, webauthn},
 };
 
 use super::Variant;
 
-#[derive(Default)]
 pub struct AuthArgs {
   pub headers: HeaderMap,
+  /// Prefer extracting IP from headers.
+  /// This IP will be the IP of reverse proxy itself.
+  pub ip: IpAddr,
+  /// Per-client session state
+  pub session: Option<Session>,
 }
 
 #[typeshare]
@@ -46,6 +73,8 @@ pub enum AuthRequest {
   SignUpLocalUser(SignUpLocalUser),
   LoginLocalUser(LoginLocalUser),
   ExchangeForJwt(ExchangeForJwt),
+  CompleteTotpLogin(CompleteTotpLogin),
+  CompletePasskeyLogin(CompletePasskeyLogin),
   GetUser(GetUser),
 }
 
@@ -73,11 +102,13 @@ pub fn router() -> Router {
     router = router.nest("/oidc", oidc::router())
   }
 
-  router
+  router.layer(memory_session_layer(60))
 }
 
 async fn variant_handler(
   headers: HeaderMap,
+  session: Session,
+  info: ConnectInfo<SocketAddr>,
   Path(Variant { variant }): Path<Variant>,
   Json(params): Json<serde_json::Value>,
 ) -> serror::Result<axum::response::Response> {
@@ -85,11 +116,13 @@ async fn variant_handler(
     "type": variant,
     "params": params,
   }))?;
-  handler(headers, Json(req)).await
+  handler(headers, session, info, Json(req)).await
 }
 
 async fn handler(
   headers: HeaderMap,
+  session: Session,
+  ConnectInfo(info): ConnectInfo<SocketAddr>,
   Json(request): Json<AuthRequest>,
 ) -> serror::Result<axum::response::Response> {
   let timer = Instant::now();
@@ -98,7 +131,13 @@ async fn handler(
     "/auth request {req_id} | METHOD: {:?}",
     request.extract_variant()
   );
-  let res = request.resolve(&AuthArgs { headers }).await;
+  let res = request
+    .resolve(&AuthArgs {
+      headers,
+      ip: info.ip(),
+      session: Some(session),
+    })
+    .await;
   if let Err(e) = &res {
     debug!("/auth request {req_id} | error: {:#}", e.error);
   }
@@ -135,25 +174,189 @@ impl Resolve<AuthArgs> for GetLoginOptions {
 impl Resolve<AuthArgs> for ExchangeForJwt {
   async fn resolve(
     self,
-    _: &AuthArgs,
+    AuthArgs {
+      headers,
+      ip,
+      session,
+    }: &AuthArgs,
   ) -> serror::Result<ExchangeForJwtResponse> {
-    jwt_client()
-      .redeem_exchange_token(&self.token)
+    async {
+      let session = session.as_ref().context(
+        "Method called in invalid context. This should not happen",
+      )?;
+
+      let user_id = session
+      .remove::<String>(SESSION_KEY_USER_ID)
       .await
-      .map_err(Into::into)
+      .context("Internal session type error")?
+      .context("Authentication steps must be completed before JWT can be retrieved")?;
+
+      jwt_client().encode(user_id).map_err(Into::into)
+    }
+    .with_failure_rate_limit_using_headers(
+      auth_rate_limiter(),
+      headers,
+      Some(*ip),
+    )
+    .await
+  }
+}
+
+impl Resolve<AuthArgs> for CompleteTotpLogin {
+  async fn resolve(
+    self,
+    AuthArgs {
+      headers,
+      ip,
+      session,
+    }: &AuthArgs,
+  ) -> serror::Result<CompleteTotpLoginResponse> {
+    async {
+      let session = session.as_ref().context(
+        "Method called in invalid context. This should not happen",
+      )?;
+
+      let user_id = session
+        .get::<String>(SESSION_KEY_TOTP_LOGIN)
+        .await
+        .context("Internal session type error")?
+        .context(
+          "Totp login has not been initiated for this session",
+        )?;
+
+      let user = get_user(&user_id)
+        .await
+        .status_code(StatusCode::UNAUTHORIZED)?;
+
+      if user.totp.secret.is_empty() {
+        return Err(
+          anyhow!("User is not enrolled in totp")
+            .status_code(StatusCode::BAD_REQUEST),
+        );
+      }
+
+      let secret_bytes = BASE32_NOPAD
+        .decode(user.totp.secret.as_bytes())
+        .context("Failed to decode totp secret to bytes")?;
+
+      let totp = make_totp(secret_bytes, None)?;
+
+      let valid = totp
+        .check_current(&self.code)
+        .context("Failed to check TOTP code validity")?;
+
+      if !valid {
+        return Err(
+          anyhow!("Invalid totp code")
+            .status_code(StatusCode::UNAUTHORIZED),
+        );
+      }
+
+      jwt_client().encode(user_id).map_err(Into::into)
+    }
+    .with_failure_rate_limit_using_headers(
+      auth_rate_limiter(),
+      headers,
+      Some(*ip),
+    )
+    .await
+  }
+}
+
+impl Resolve<AuthArgs> for CompletePasskeyLogin {
+  async fn resolve(
+    self,
+    AuthArgs {
+      headers,
+      ip,
+      session,
+    }: &AuthArgs,
+  ) -> serror::Result<CompletePasskeyLoginResponse> {
+    async {
+      let session = session.as_ref().context(
+        "Method called in invalid context. This should not happen",
+      )?;
+
+      let webauthn = webauthn().context(
+        "No webauthn provider available, invalid KOMODO_HOST config",
+      )?;
+
+      let (user_id, server_state) = session
+        .get::<(String, PasskeyAuthentication)>(
+          SESSION_KEY_PASSKEY_LOGIN,
+        )
+        .await
+        .context("Internal session type error")?
+        .context(
+          "Passkey login has not been initiated for this session",
+        )?;
+
+      // The result of this call must be used to
+      // update the stored passkey info on database.
+      let update = webauthn
+        .finish_passkey_authentication(
+          &self.credential,
+          &server_state,
+        )
+        .context("Failed to validate passkey")?;
+
+      let mut passkey = get_user(&user_id)
+        .await?
+        .passkey
+        .passkey
+        .context("Could not find passkey on database.")?;
+
+      passkey.update_credential(&update);
+
+      let passkey = to_bson(&passkey)
+        .context("Failed to serialize passkey to BSON")?;
+
+      let update = doc! { "$set": { "passkey.passkey": passkey } };
+
+      let _ =
+        update_one_by_id(&db_client().users, &user_id, update, None)
+          .await
+          .context(
+            "Failed to update user passkey on database after login",
+          )
+          .inspect_err(|e| warn!("{e:#}"));
+
+      jwt_client().encode(user_id).map_err(Into::into)
+    }
+    .with_failure_rate_limit_using_headers(
+      auth_rate_limiter(),
+      headers,
+      Some(*ip),
+    )
+    .await
   }
 }
 
 impl Resolve<AuthArgs> for GetUser {
   async fn resolve(
     self,
-    AuthArgs { headers }: &AuthArgs,
+    AuthArgs {
+      headers,
+      ip,
+      session: _,
+    }: &AuthArgs,
   ) -> serror::Result<User> {
-    let user_id = get_user_id_from_headers(headers)
-      .await
-      .status_code(StatusCode::UNAUTHORIZED)?;
-    get_user(&user_id)
-      .await
-      .status_code(StatusCode::UNAUTHORIZED)
+    async {
+      let user_id = get_user_id_from_headers(headers)
+        .await
+        .status_code(StatusCode::UNAUTHORIZED)?;
+      let mut user = get_user(&user_id)
+        .await
+        .status_code(StatusCode::UNAUTHORIZED)?;
+      // Sanitize before sending to client.
+      user.sanitize();
+      Ok(user)
+    }
+    .with_failure_rate_limit_using_headers(
+      auth_rate_limiter(),
+      headers,
+      Some(*ip),
+    )
+    .await
   }
 }

bin/core/src/api/execute/action.rs

@@ -141,6 +141,7 @@ impl Resolve<ExecuteArgs> for RunAction {
     }
     .resolve(&UserArgs {
       user: action_user().to_owned(),
+      session: None,
     })
     .await?;
 
@@ -199,6 +200,7 @@ impl Resolve<ExecuteArgs> for RunAction {
     if let Err(e) = (DeleteApiKey { key })
       .resolve(&UserArgs {
         user: action_user().to_owned(),
+        session: None,
       })
       .await
     {

bin/core/src/api/execute/deployment.rs

@@ -7,7 +7,7 @@ use interpolate::Interpolator;
 use komodo_client::{
   api::execute::*,
   entities::{
-    Version,
+    SwarmOrServer, Version,
     build::{Build, ImageRegistryConfig},
     deployment::{
       Deployment, DeploymentImage, extract_registry_domain,
@@ -16,22 +16,23 @@ use komodo_client::{
     permission::PermissionLevel,
     server::Server,
     update::{Log, Update},
-    user::User,
   },
 };
 use periphery_client::api;
+use reqwest::StatusCode;
 use resolver_api::Resolve;
+use serror::AddStatusCodeError;
 
 use crate::{
   helpers::{
     periphery_client,
     query::{VariablesAndSecrets, get_variables_and_secrets},
     registry_token,
+    swarm::swarm_request,
     update::update_update,
   },
-  monitor::update_cache_for_server,
-  permission::get_check_permissions,
-  resource,
+  monitor::{update_cache_for_server, update_cache_for_swarm},
+  resource::{self, setup_deployment_execution},
   state::action_states,
 };
 
@@ -69,32 +70,6 @@ impl Resolve<ExecuteArgs> for BatchDeploy {
   }
 }
 
-#[instrument("SetupDeploy", skip_all)]
-async fn setup_deployment_execution(
-  deployment: &str,
-  user: &User,
-) -> anyhow::Result<(Deployment, Server)> {
-  let deployment = get_check_permissions::<Deployment>(
-    deployment,
-    user,
-    PermissionLevel::Execute.into(),
-  )
-  .await?;
-
-  if deployment.config.server_id.is_empty() {
-    return Err(anyhow!("Deployment has no Server configured"));
-  }
-
-  let server =
-    resource::get::<Server>(&deployment.config.server_id).await?;
-
-  if !server.config.enabled {
-    return Err(anyhow!("Attached Server is not enabled"));
-  }
-
-  Ok((deployment, server))
-}
-
 impl Resolve<ExecuteArgs> for Deploy {
   #[instrument(
     "Deploy",
@@ -112,8 +87,13 @@ impl Resolve<ExecuteArgs> for Deploy {
     self,
     ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
-    let (mut deployment, server) =
-      setup_deployment_execution(&self.deployment, user).await?;
+    let (mut deployment, swarm_or_server) =
+      setup_deployment_execution(
+        &self.deployment,
+        user,
+        PermissionLevel::Execute.into(),
+      )
+      .await?;
 
     // get the action state for the deployment (or insert default).
     let action_state = action_states()
@@ -223,27 +203,55 @@ impl Resolve<ExecuteArgs> for Deploy {
     update.version = version;
     update_update(update.clone()).await?;
 
-    match periphery_client(&server)
-      .await?
-      .request(api::container::Deploy {
-        deployment,
-        stop_signal: self.stop_signal,
-        stop_time: self.stop_time,
-        registry_token,
-        replacers: secret_replacers.into_iter().collect(),
-      })
-      .await
-    {
-      Ok(log) => update.logs.push(log),
-      Err(e) => {
-        update.push_error_log(
-          "Deploy Container",
-          format_serror(&e.into()),
-        );
+    match swarm_or_server {
+      SwarmOrServer::Swarm(swarm) => {
+        match swarm_request(
+          &swarm.config.server_ids,
+          api::swarm::CreateSwarmService {
+            deployment,
+            registry_token,
+            replacers: secret_replacers.into_iter().collect(),
+          },
+        )
+        .await
+        {
+          Ok(logs) => {
+            update_cache_for_swarm(&swarm, true).await;
+            update.logs.extend(logs)
+          }
+          Err(e) => {
+            update.push_error_log(
+              "Create Swarm Service",
+              format_serror(&e.into()),
+            );
+          }
+        };
       }
-    };
-
-    update_cache_for_server(&server, true).await;
+      SwarmOrServer::Server(server) => {
+        match periphery_client(&server)
+          .await?
+          .request(api::container::RunContainer {
+            deployment,
+            stop_signal: self.stop_signal,
+            stop_time: self.stop_time,
+            registry_token,
+            replacers: secret_replacers.into_iter().collect(),
+          })
+          .await
+        {
+          Ok(log) => {
+            update_cache_for_server(&server, true).await;
+            update.logs.push(log)
+          }
+          Err(e) => {
+            update.push_error_log(
+              "Deploy Container",
+              format_serror(&e.into()),
+            );
+          }
+        };
+      }
+    }
 
     update.finalize();
     update_update(update.clone()).await?;
@@ -400,8 +408,19 @@ impl Resolve<ExecuteArgs> for PullDeployment {
     self,
     ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
-    let (deployment, server) =
-      setup_deployment_execution(&self.deployment, user).await?;
+    let (deployment, swarm_or_server) = setup_deployment_execution(
+      &self.deployment,
+      user,
+      PermissionLevel::Execute.into(),
+    )
+    .await?;
+
+    let SwarmOrServer::Server(server) = swarm_or_server else {
+      return Err(
+        anyhow!("PullDeployment should not be called for Deployment in Swarm Mode")
+          .status_code(StatusCode::BAD_REQUEST),
+      );
+    };
 
     // get the action state for the deployment (or insert default).
     let action_state = action_states()
@@ -443,8 +462,19 @@ impl Resolve<ExecuteArgs> for StartDeployment {
     self,
     ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
-    let (deployment, server) =
-      setup_deployment_execution(&self.deployment, user).await?;
+    let (deployment, swarm_or_server) = setup_deployment_execution(
+      &self.deployment,
+      user,
+      PermissionLevel::Execute.into(),
+    )
+    .await?;
+
+    let SwarmOrServer::Server(server) = swarm_or_server else {
+      return Err(
+        anyhow!("StartDeployment should not be called for Deployment in Swarm Mode")
+          .status_code(StatusCode::BAD_REQUEST),
+      );
+    };
 
     // get the action state for the deployment (or insert default).
     let action_state = action_states()
@@ -500,8 +530,19 @@ impl Resolve<ExecuteArgs> for RestartDeployment {
     self,
     ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
-    let (deployment, server) =
-      setup_deployment_execution(&self.deployment, user).await?;
+    let (deployment, swarm_or_server) = setup_deployment_execution(
+      &self.deployment,
+      user,
+      PermissionLevel::Execute.into(),
+    )
+    .await?;
+
+    let SwarmOrServer::Server(server) = swarm_or_server else {
+      return Err(
+        anyhow!("RestartDeployment should not be called for Deployment in Swarm Mode")
+          .status_code(StatusCode::BAD_REQUEST),
+      );
+    };
 
     // get the action state for the deployment (or insert default).
     let action_state = action_states()
@@ -559,8 +600,19 @@ impl Resolve<ExecuteArgs> for PauseDeployment {
     self,
     ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
-    let (deployment, server) =
-      setup_deployment_execution(&self.deployment, user).await?;
+    let (deployment, swarm_or_server) = setup_deployment_execution(
+      &self.deployment,
+      user,
+      PermissionLevel::Execute.into(),
+    )
+    .await?;
+
+    let SwarmOrServer::Server(server) = swarm_or_server else {
+      return Err(
+        anyhow!("PauseDeployment should not be called for Deployment in Swarm Mode")
+          .status_code(StatusCode::BAD_REQUEST),
+      );
+    };
 
     // get the action state for the deployment (or insert default).
     let action_state = action_states()
@@ -616,8 +668,19 @@ impl Resolve<ExecuteArgs> for UnpauseDeployment {
     self,
     ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
-    let (deployment, server) =
-      setup_deployment_execution(&self.deployment, user).await?;
+    let (deployment, swarm_or_server) = setup_deployment_execution(
+      &self.deployment,
+      user,
+      PermissionLevel::Execute.into(),
+    )
+    .await?;
+
+    let SwarmOrServer::Server(server) = swarm_or_server else {
+      return Err(
+        anyhow!("UnpauseDeployment should not be called for Deployment in Swarm Mode")
+          .status_code(StatusCode::BAD_REQUEST),
+      );
+    };
 
     // get the action state for the deployment (or insert default).
     let action_state = action_states()
@@ -677,8 +740,19 @@ impl Resolve<ExecuteArgs> for StopDeployment {
     self,
     ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
-    let (deployment, server) =
-      setup_deployment_execution(&self.deployment, user).await?;
+    let (deployment, swarm_or_server) = setup_deployment_execution(
+      &self.deployment,
+      user,
+      PermissionLevel::Execute.into(),
+    )
+    .await?;
+
+    let SwarmOrServer::Server(server) = swarm_or_server else {
+      return Err(
+        anyhow!("StopDeployment should not be called for Deployment in Swarm Mode")
+          .status_code(StatusCode::BAD_REQUEST),
+      );
+    };
 
     // get the action state for the deployment (or insert default).
     let action_state = action_states()
@@ -779,8 +853,12 @@ impl Resolve<ExecuteArgs> for DestroyDeployment {
     self,
     ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
-    let (deployment, server) =
-      setup_deployment_execution(&self.deployment, user).await?;
+    let (deployment, swarm_or_server) = setup_deployment_execution(
+      &self.deployment,
+      user,
+      PermissionLevel::Execute.into(),
+    )
+    .await?;
 
     // get the action state for the deployment (or insert default).
     let action_state = action_states()
@@ -798,31 +876,61 @@ impl Resolve<ExecuteArgs> for DestroyDeployment {
     // Send update after setting action state, this way frontend gets correct state.
     update_update(update.clone()).await?;
 
-    let log = match periphery_client(&server)
-      .await?
-      .request(api::container::RemoveContainer {
-        name: deployment.name,
-        signal: self
-          .signal
-          .unwrap_or(deployment.config.termination_signal)
-          .into(),
-        time: self
-          .time
-          .unwrap_or(deployment.config.termination_timeout)
-          .into(),
-      })
-      .await
-    {
-      Ok(log) => log,
-      Err(e) => Log::error(
-        "stop container",
-        format_serror(&e.context("failed to stop container").into()),
-      ),
+    let log = match swarm_or_server {
+      SwarmOrServer::Swarm(swarm) => {
+        match swarm_request(
+          &swarm.config.server_ids,
+          api::swarm::RemoveSwarmServices {
+            services: vec![deployment.name],
+          },
+        )
+        .await
+        {
+          Ok(log) => {
+            update_cache_for_swarm(&swarm, true).await;
+            log
+          }
+          Err(e) => Log::error(
+            "Remove Swarm Service",
+            format_serror(
+              &e.context("Failed to remove swarm service").into(),
+            ),
+          ),
+        }
+      }
+      SwarmOrServer::Server(server) => {
+        match periphery_client(&server)
+          .await?
+          .request(api::container::RemoveContainer {
+            name: deployment.name,
+            signal: self
+              .signal
+              .unwrap_or(deployment.config.termination_signal)
+              .into(),
+            time: self
+              .time
+              .unwrap_or(deployment.config.termination_timeout)
+              .into(),
+          })
+          .await
+        {
+          Ok(log) => {
+            update_cache_for_server(&server, true).await;
+            log
+          }
+          Err(e) => Log::error(
+            "Destroy Container",
+            format_serror(
+              &e.context("Failed to destroy container").into(),
+            ),
+          ),
+        }
+      }
     };
 
     update.logs.push(log);
     update.finalize();
-    update_cache_for_server(&server, true).await;
+
     update_update(update.clone()).await?;
 
     Ok(update)

bin/core/src/api/execute/mod.rs

@@ -43,6 +43,7 @@ mod procedure;
 mod repo;
 mod server;
 mod stack;
+mod swarm;
 mod sync;
 
 use super::Variant;
@@ -69,29 +70,6 @@ pub struct ExecuteArgs {
 #[error(serror::Error)]
 #[serde(tag = "type", content = "params")]
 pub enum ExecuteRequest {
-  // ==== SERVER ====
-  StartContainer(StartContainer),
-  RestartContainer(RestartContainer),
-  PauseContainer(PauseContainer),
-  UnpauseContainer(UnpauseContainer),
-  StopContainer(StopContainer),
-  DestroyContainer(DestroyContainer),
-  StartAllContainers(StartAllContainers),
-  RestartAllContainers(RestartAllContainers),
-  PauseAllContainers(PauseAllContainers),
-  UnpauseAllContainers(UnpauseAllContainers),
-  StopAllContainers(StopAllContainers),
-  PruneContainers(PruneContainers),
-  DeleteNetwork(DeleteNetwork),
-  PruneNetworks(PruneNetworks),
-  DeleteImage(DeleteImage),
-  PruneImages(PruneImages),
-  DeleteVolume(DeleteVolume),
-  PruneVolumes(PruneVolumes),
-  PruneDockerBuilders(PruneDockerBuilders),
-  PruneBuildx(PruneBuildx),
-  PruneSystem(PruneSystem),
-
   // ==== STACK ====
   DeployStack(DeployStack),
   BatchDeployStack(BatchDeployStack),
@@ -149,6 +127,36 @@ pub enum ExecuteRequest {
   TestAlerter(TestAlerter),
   SendAlert(SendAlert),
 
+  // ==== SERVER ====
+  StartContainer(StartContainer),
+  RestartContainer(RestartContainer),
+  PauseContainer(PauseContainer),
+  UnpauseContainer(UnpauseContainer),
+  StopContainer(StopContainer),
+  DestroyContainer(DestroyContainer),
+  StartAllContainers(StartAllContainers),
+  RestartAllContainers(RestartAllContainers),
+  PauseAllContainers(PauseAllContainers),
+  UnpauseAllContainers(UnpauseAllContainers),
+  StopAllContainers(StopAllContainers),
+  PruneContainers(PruneContainers),
+  DeleteNetwork(DeleteNetwork),
+  PruneNetworks(PruneNetworks),
+  DeleteImage(DeleteImage),
+  PruneImages(PruneImages),
+  DeleteVolume(DeleteVolume),
+  PruneVolumes(PruneVolumes),
+  PruneDockerBuilders(PruneDockerBuilders),
+  PruneBuildx(PruneBuildx),
+  PruneSystem(PruneSystem),
+
+  // ==== SWARM ====
+  RemoveSwarmNodes(RemoveSwarmNodes),
+  RemoveSwarmStacks(RemoveSwarmStacks),
+  RemoveSwarmServices(RemoveSwarmServices),
+  RemoveSwarmConfigs(RemoveSwarmConfigs),
+  RemoveSwarmSecrets(RemoveSwarmSecrets),
+
   // ==== MAINTENANCE ====
   ClearRepoCache(ClearRepoCache),
   BackupCoreDatabase(BackupCoreDatabase),

bin/core/src/api/execute/server.rs

@@ -70,8 +70,8 @@ impl Resolve<ExecuteArgs> for StartContainer {
     {
       Ok(log) => log,
       Err(e) => Log::error(
-        "start container",
-        format_serror(&e.context("failed to start container").into()),
+        "Start Container",
+        format_serror(&e.context("Failed to start container").into()),
       ),
     };
 
@@ -134,9 +134,9 @@ impl Resolve<ExecuteArgs> for RestartContainer {
     {
       Ok(log) => log,
       Err(e) => Log::error(
-        "restart container",
+        "Restart Container",
         format_serror(
-          &e.context("failed to restart container").into(),
+          &e.context("Failed to restart container").into(),
         ),
       ),
     };
@@ -200,8 +200,8 @@ impl Resolve<ExecuteArgs> for PauseContainer {
     {
       Ok(log) => log,
       Err(e) => Log::error(
-        "pause container",
-        format_serror(&e.context("failed to pause container").into()),
+        "Pause Container",
+        format_serror(&e.context("Failed to pause container").into()),
       ),
     };
 
@@ -264,9 +264,9 @@ impl Resolve<ExecuteArgs> for UnpauseContainer {
     {
       Ok(log) => log,
       Err(e) => Log::error(
-        "unpause container",
+        "Unpause Container",
         format_serror(
-          &e.context("failed to unpause container").into(),
+          &e.context("Failed to unpause container").into(),
         ),
       ),
     };
@@ -334,8 +334,8 @@ impl Resolve<ExecuteArgs> for StopContainer {
     {
       Ok(log) => log,
       Err(e) => Log::error(
-        "stop container",
-        format_serror(&e.context("failed to stop container").into()),
+        "Stop Container",
+        format_serror(&e.context("Failed to stop container").into()),
       ),
     };
 
@@ -408,8 +408,10 @@ impl Resolve<ExecuteArgs> for DestroyContainer {
     {
       Ok(log) => log,
       Err(e) => Log::error(
-        "stop container",
-        format_serror(&e.context("failed to stop container").into()),
+        "Remove Container",
+        format_serror(
+          &e.context("Failed to remove container").into(),
+        ),
       ),
     };
 
@@ -464,13 +466,13 @@ impl Resolve<ExecuteArgs> for StartAllContainers {
       .await?
       .request(api::container::StartAllContainers {})
       .await
-      .context("failed to start all containers on host")?;
+      .context("Failed to start all containers on host")?;
 
     update.logs.extend(logs);
 
     if all_logs_success(&update.logs) {
       update.push_simple_log(
-        "start all containers",
+        "Start All Containers",
         String::from("All containers have been started on the host."),
       );
     }
@@ -524,13 +526,13 @@ impl Resolve<ExecuteArgs> for RestartAllContainers {
       .await?
       .request(api::container::RestartAllContainers {})
       .await
-      .context("failed to restart all containers on host")?;
+      .context("Failed to restart all containers on host")?;
 
     update.logs.extend(logs);
 
     if all_logs_success(&update.logs) {
       update.push_simple_log(
-        "restart all containers",
+        "Restart All Containers",
         String::from(
           "All containers have been restarted on the host.",
         ),
@@ -586,13 +588,13 @@ impl Resolve<ExecuteArgs> for PauseAllContainers {
       .await?
       .request(api::container::PauseAllContainers {})
       .await
-      .context("failed to pause all containers on host")?;
+      .context("Failed to pause all containers on host")?;
 
     update.logs.extend(logs);
 
     if all_logs_success(&update.logs) {
       update.push_simple_log(
-        "pause all containers",
+        "Pause All Containers",
         String::from("All containers have been paused on the host."),
       );
     }
@@ -646,13 +648,13 @@ impl Resolve<ExecuteArgs> for UnpauseAllContainers {
       .await?
       .request(api::container::UnpauseAllContainers {})
       .await
-      .context("failed to unpause all containers on host")?;
+      .context("Failed to unpause all containers on host")?;
 
     update.logs.extend(logs);
 
     if all_logs_success(&update.logs) {
       update.push_simple_log(
-        "unpause all containers",
+        "Unpause All Containers",
         String::from(
           "All containers have been unpaused on the host.",
         ),
@@ -708,13 +710,13 @@ impl Resolve<ExecuteArgs> for StopAllContainers {
       .await?
       .request(api::container::StopAllContainers {})
       .await
-      .context("failed to stop all containers on host")?;
+      .context("Failed to stop all containers on host")?;
 
     update.logs.extend(logs);
 
     if all_logs_success(&update.logs) {
       update.push_simple_log(
-        "stop all containers",
+        "Stop All Containers",
         String::from("All containers have been stopped on the host."),
       );
     }
@@ -770,14 +772,14 @@ impl Resolve<ExecuteArgs> for PruneContainers {
       .request(api::container::PruneContainers {})
       .await
       .context(format!(
-        "failed to prune containers on server {}",
+        "Failed to prune containers on server {}",
         server.name
       )) {
       Ok(log) => log,
       Err(e) => Log::error(
-        "prune containers",
+        "Prune Containers",
         format_serror(
-          &e.context("failed to prune containers").into(),
+          &e.context("Failed to prune containers").into(),
         ),
       ),
     };
@@ -827,15 +829,15 @@ impl Resolve<ExecuteArgs> for DeleteNetwork {
       })
       .await
       .context(format!(
-        "failed to delete network {} on server {}",
+        "Failed to delete network {} on server {}",
         self.name, server.name
       )) {
       Ok(log) => log,
       Err(e) => Log::error(
-        "delete network",
+        "Delete Network",
         format_serror(
           &e.context(format!(
-            "failed to delete network {}",
+            "Failed to delete network {}",
             self.name
           ))
           .into(),
@@ -896,13 +898,13 @@ impl Resolve<ExecuteArgs> for PruneNetworks {
       .request(api::docker::PruneNetworks {})
       .await
       .context(format!(
-        "failed to prune networks on server {}",
+        "Failed to prune networks on server {}",
         server.name
       )) {
       Ok(log) => log,
       Err(e) => Log::error(
-        "prune networks",
-        format_serror(&e.context("failed to prune networks").into()),
+        "Prune Networks",
+        format_serror(&e.context("Failed to prune networks").into()),
       ),
     };
 
@@ -951,14 +953,14 @@ impl Resolve<ExecuteArgs> for DeleteImage {
       })
       .await
       .context(format!(
-        "failed to delete image {} on server {}",
+        "Failed to delete image {} on server {}",
         self.name, server.name
       )) {
       Ok(log) => log,
       Err(e) => Log::error(
         "delete image",
         format_serror(
-          &e.context(format!("failed to delete image {}", self.name))
+          &e.context(format!("Failed to delete image {}", self.name))
             .into(),
         ),
       ),
@@ -1017,9 +1019,9 @@ impl Resolve<ExecuteArgs> for PruneImages {
       match periphery.request(api::docker::PruneImages {}).await {
         Ok(log) => log,
         Err(e) => Log::error(
-          "prune images",
+          "Prune Images",
           format!(
-            "failed to prune images on server {} | {e:#?}",
+            "Failed to prune images on server {} | {e:#?}",
             server.name
           ),
         ),
@@ -1070,7 +1072,7 @@ impl Resolve<ExecuteArgs> for DeleteVolume {
       })
       .await
       .context(format!(
-        "failed to delete volume {} on server {}",
+        "Failed to delete volume {} on server {}",
         self.name, server.name
       )) {
       Ok(log) => log,
@@ -1078,7 +1080,7 @@ impl Resolve<ExecuteArgs> for DeleteVolume {
         "delete volume",
         format_serror(
           &e.context(format!(
-            "failed to delete volume {}",
+            "Failed to delete volume {}",
             self.name
           ))
           .into(),
@@ -1139,9 +1141,9 @@ impl Resolve<ExecuteArgs> for PruneVolumes {
       match periphery.request(api::docker::PruneVolumes {}).await {
         Ok(log) => log,
         Err(e) => Log::error(
-          "prune volumes",
+          "Prune Volumes",
           format!(
-            "failed to prune volumes on server {} | {e:#?}",
+            "Failed to prune volumes on server {} | {e:#?}",
             server.name
           ),
         ),
@@ -1200,9 +1202,9 @@ impl Resolve<ExecuteArgs> for PruneDockerBuilders {
       match periphery.request(api::build::PruneBuilders {}).await {
         Ok(log) => log,
         Err(e) => Log::error(
-          "prune builders",
+          "Prune Builders",
           format!(
-            "failed to docker builder prune on server {} | {e:#?}",
+            "Failed to docker builder prune on server {} | {e:#?}",
             server.name
           ),
         ),
@@ -1261,9 +1263,9 @@ impl Resolve<ExecuteArgs> for PruneBuildx {
       match periphery.request(api::build::PruneBuildx {}).await {
         Ok(log) => log,
         Err(e) => Log::error(
-          "prune buildx",
+          "Prune Buildx",
           format!(
-            "failed to docker buildx prune on server {} | {e:#?}",
+            "Failed to docker buildx prune on server {} | {e:#?}",
             server.name
           ),
         ),
@@ -1321,9 +1323,9 @@ impl Resolve<ExecuteArgs> for PruneSystem {
     let log = match periphery.request(api::PruneSystem {}).await {
       Ok(log) => log,
       Err(e) => Log::error(
-        "prune system",
+        "Prune System",
         format!(
-          "failed to docker system prune on server {} | {e:#?}",
+          "Failed to docker system prune on server {} | {e:#?}",
           server.name
         ),
       ),

bin/core/src/api/execute/stack.rs

@@ -1,6 +1,6 @@
 use std::{collections::HashSet, str::FromStr};
 
-use anyhow::Context;
+use anyhow::{Context, anyhow};
 use database::mungos::mongodb::bson::{
   doc, oid::ObjectId, to_bson, to_document,
 };
@@ -9,7 +9,7 @@ use interpolate::Interpolator;
 use komodo_client::{
   api::{execute::*, write::RefreshStackCache},
   entities::{
-    FileContents,
+    FileContents, SwarmOrServer,
     permission::PermissionLevel,
     repo::Repo,
     server::Server,
@@ -20,8 +20,12 @@ use komodo_client::{
     user::User,
   },
 };
-use periphery_client::api::compose::*;
+use periphery_client::api::{
+  DeployStackResponse, compose::*, swarm::DeploySwarmStack,
+};
+use reqwest::StatusCode;
 use resolver_api::Resolve;
+use serror::AddStatusCodeError as _;
 use uuid::Uuid;
 
 use crate::{
@@ -30,14 +34,20 @@ use crate::{
     periphery_client,
     query::{VariablesAndSecrets, get_variables_and_secrets},
     stack_git_token,
+    swarm::swarm_request,
     update::{
       add_update_without_send, init_execution_update, update_update,
     },
   },
-  monitor::update_cache_for_server,
+  monitor::{update_cache_for_server, update_cache_for_swarm},
   permission::get_check_permissions,
   resource,
-  stack::{execute::execute_compose, get_stack_and_server},
+  stack::{
+    execute::{
+      execute_compose, execute_compose_with_stack_and_server,
+    },
+    setup_stack_execution,
+  },
   state::{action_states, db_client},
 };
 
@@ -92,11 +102,10 @@ impl Resolve<ExecuteArgs> for DeployStack {
     self,
     ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
-    let (mut stack, server) = get_stack_and_server(
+    let (mut stack, swarm_or_server) = setup_stack_execution(
       &self.stack,
       user,
       PermissionLevel::Execute.into(),
-      true,
     )
     .await?;
 
@@ -165,27 +174,44 @@ impl Resolve<ExecuteArgs> for DeployStack {
       Default::default()
     };
 
-    let ComposeUpResponse {
+    let DeployStackResponse {
       logs,
       deployed,
       services,
       file_contents,
       missing_files,
       remote_errors,
-      compose_config,
+      merged_config,
       commit_hash,
       commit_message,
-    } = periphery_client(&server)
-      .await?
-      .request(ComposeUp {
-        stack: stack.clone(),
-        services: self.services,
-        repo,
-        git_token,
-        registry_token,
-        replacers: secret_replacers.into_iter().collect(),
-      })
-      .await?;
+    } = match &swarm_or_server {
+      SwarmOrServer::Swarm(swarm) => {
+        swarm_request(
+          &swarm.config.server_ids,
+          DeploySwarmStack {
+            stack: stack.clone(),
+            repo,
+            git_token,
+            registry_token,
+            replacers: secret_replacers.into_iter().collect(),
+          },
+        )
+        .await?
+      }
+      SwarmOrServer::Server(server) => {
+        periphery_client(server)
+          .await?
+          .request(ComposeUp {
+            stack: stack.clone(),
+            services: self.services,
+            repo,
+            git_token,
+            registry_token,
+            replacers: secret_replacers.into_iter().collect(),
+          })
+          .await?
+      }
+    };
 
     update.logs.extend(logs);
 
@@ -219,7 +245,7 @@ impl Resolve<ExecuteArgs> for DeployStack {
               })
               .collect(),
           ),
-          compose_config,
+          merged_config,
           commit_hash.clone(),
           commit_message.clone(),
         )
@@ -257,7 +283,7 @@ impl Resolve<ExecuteArgs> for DeployStack {
       };
 
       let info = to_document(&info)
-        .context("failed to serialize stack info to bson")?;
+        .context("Failed to serialize stack info to bson")?;
 
       db_client()
         .stacks
@@ -266,22 +292,29 @@ impl Resolve<ExecuteArgs> for DeployStack {
           doc! { "$set": { "info": info } },
         )
         .await
-        .context("failed to update stack info on db")?;
+        .context("Failed to update stack info on db")?;
       anyhow::Ok(())
     };
 
     // This will be weird with single service deploys. Come back to it.
     if let Err(e) = update_info.await {
       update.push_error_log(
-        "refresh stack info",
+        "Refresh Stack Info",
         format_serror(
-          &e.context("failed to refresh stack info on db").into(),
+          &e.context("Failed to refresh stack info on db").into(),
         ),
       )
     }
 
     // Ensure cached stack state up to date by updating server cache
-    update_cache_for_server(&server, true).await;
+    match swarm_or_server {
+      SwarmOrServer::Swarm(swarm) => {
+        update_cache_for_swarm(&swarm, true).await;
+      }
+      SwarmOrServer::Server(server) => {
+        update_cache_for_server(&server, true).await;
+      }
+    }
 
     update.finalize();
     update_update(update.clone()).await?;
@@ -862,14 +895,22 @@ impl Resolve<ExecuteArgs> for PullStack {
     self,
     ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
-    let (stack, server) = get_stack_and_server(
+    let (stack, swarm_or_server) = setup_stack_execution(
       &self.stack,
       user,
       PermissionLevel::Execute.into(),
-      true,
     )
     .await?;
 
+    let SwarmOrServer::Server(server) = swarm_or_server else {
+      return Err(
+        anyhow!(
+          "PullStack should not be called for Stack in Swarm Mode"
+        )
+        .status_code(StatusCode::BAD_REQUEST),
+      );
+    };
+
     let repo = if !stack.config.files_on_host
       && !stack.config.linked_repo.is_empty()
     {
@@ -1106,16 +1147,77 @@ impl Resolve<ExecuteArgs> for DestroyStack {
     self,
     ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
-    execute_compose::<DestroyStack>(
+    let (stack, swarm_or_server) = setup_stack_execution(
       &self.stack,
-      self.services,
       user,
-      |state| state.destroying = true,
-      update.clone(),
-      (self.stop_time, self.remove_orphans),
+      PermissionLevel::Execute.into(),
     )
-    .await
-    .map_err(Into::into)
+    .await?;
+
+    match swarm_or_server {
+      SwarmOrServer::Swarm(swarm) => {
+        if !self.services.is_empty() {
+          return Err(
+            anyhow!("Cannot destroy specific Stack services when in Swarm mode.")
+              .status_code(StatusCode::BAD_REQUEST)
+          );
+        }
+
+        // get the action state for the stack (or insert default).
+        let action_state = action_states()
+          .stack
+          .get_or_insert_default(&stack.id)
+          .await;
+
+        // Will check to ensure stack not already busy before updating, and return Err if so.
+        // The returned guard will set the action state back to default when dropped.
+        let _action_guard =
+          action_state.update(|state| state.destroying = true)?;
+
+        let mut update = update.clone();
+        // Send update here for frontend to recheck action state
+        update_update(update.clone()).await?;
+
+        match swarm_request(
+          &swarm.config.server_ids,
+          periphery_client::api::swarm::RemoveSwarmStacks {
+            stacks: vec![stack.project_name(false)],
+            detach: false,
+          },
+        )
+        .await
+        {
+          Ok(log) => update.logs.push(log),
+          Err(e) => update.push_error_log(
+            "Destroy Stack",
+            format_serror(
+              &e.context("Failed to 'docker stack rm' on swarm")
+                .into(),
+            ),
+          ),
+        }
+
+        // Ensure cached stack state up to date by updating swarm cache
+        update_cache_for_swarm(&swarm, true).await;
+
+        update.finalize();
+        update_update(update.clone()).await?;
+
+        Ok(update)
+      }
+      SwarmOrServer::Server(server) => {
+        execute_compose_with_stack_and_server::<DestroyStack>(
+          stack,
+          server,
+          self.services,
+          |state| state.destroying = true,
+          update.clone(),
+          (self.stop_time, self.remove_orphans),
+        )
+        .await
+        .map_err(Into::into)
+      }
+    }
   }
 }
 
@@ -1136,14 +1238,22 @@ impl Resolve<ExecuteArgs> for RunStackService {
     self,
     ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
-    let (mut stack, server) = get_stack_and_server(
+    let (mut stack, swarm_or_server) = setup_stack_execution(
       &self.stack,
       user,
       PermissionLevel::Execute.into(),
-      true,
     )
     .await?;
 
+    let SwarmOrServer::Server(server) = swarm_or_server else {
+      return Err(
+        anyhow!(
+          "RunStackService should not be called for Stack in Swarm Mode"
+        )
+        .status_code(StatusCode::BAD_REQUEST),
+      );
+    };
+
     let mut repo = if !stack.config.files_on_host
       && !stack.config.linked_repo.is_empty()
     {

bin/core/src/api/execute/swarm.rs

@@ -0,0 +1,274 @@
+use formatting::format_serror;
+use komodo_client::{
+  api::execute::{
+    RemoveSwarmConfigs, RemoveSwarmNodes, RemoveSwarmSecrets,
+    RemoveSwarmServices, RemoveSwarmStacks,
+  },
+  entities::{permission::PermissionLevel, swarm::Swarm},
+};
+use resolver_api::Resolve;
+
+use crate::{
+  api::execute::ExecuteArgs,
+  helpers::{swarm::swarm_request, update::update_update},
+  permission::get_check_permissions,
+};
+
+impl Resolve<ExecuteArgs> for RemoveSwarmNodes {
+  #[instrument(
+    "RemoveSwarmNodes",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      swarm = self.swarm,
+      nodes = serde_json::to_string(&self.nodes).unwrap_or_else(|e| e.to_string()),
+      force = self.force,
+    )
+  )]
+  async fn resolve(
+    self,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
+  ) -> Result<Self::Response, Self::Error> {
+    let swarm = get_check_permissions::<Swarm>(
+      &self.swarm,
+      user,
+      PermissionLevel::Execute.into(),
+    )
+    .await?;
+
+    update_update(update.clone()).await?;
+
+    let mut update = update.clone();
+
+    match swarm_request(
+      &swarm.config.server_ids,
+      periphery_client::api::swarm::RemoveSwarmNodes {
+        nodes: self.nodes,
+        force: self.force,
+      },
+    )
+    .await
+    {
+      Ok(log) => update.logs.push(log),
+      Err(e) => update.push_error_log(
+        "Remove Swarm Nodes",
+        format_serror(
+          &e.context("Failed to remove swarm nodes").into(),
+        ),
+      ),
+    };
+
+    update.finalize();
+    update_update(update.clone()).await?;
+
+    Ok(update)
+  }
+}
+
+impl Resolve<ExecuteArgs> for RemoveSwarmStacks {
+  #[instrument(
+    "RemoveSwarmStacks",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      swarm = self.swarm,
+      stacks = serde_json::to_string(&self.stacks).unwrap_or_else(|e| e.to_string()),
+      detach = self.detach,
+    )
+  )]
+  async fn resolve(
+    self,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
+  ) -> Result<Self::Response, Self::Error> {
+    let swarm = get_check_permissions::<Swarm>(
+      &self.swarm,
+      user,
+      PermissionLevel::Execute.into(),
+    )
+    .await?;
+
+    update_update(update.clone()).await?;
+
+    let mut update = update.clone();
+
+    match swarm_request(
+      &swarm.config.server_ids,
+      periphery_client::api::swarm::RemoveSwarmStacks {
+        stacks: self.stacks,
+        detach: self.detach,
+      },
+    )
+    .await
+    {
+      Ok(log) => update.logs.push(log),
+      Err(e) => update.push_error_log(
+        "Remove Swarm Stacks",
+        format_serror(
+          &e.context("Failed to remove swarm stacks").into(),
+        ),
+      ),
+    };
+
+    update.finalize();
+    update_update(update.clone()).await?;
+
+    Ok(update)
+  }
+}
+
+impl Resolve<ExecuteArgs> for RemoveSwarmServices {
+  #[instrument(
+    "RemoveSwarmServices",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      swarm = self.swarm,
+      services = serde_json::to_string(&self.services).unwrap_or_else(|e| e.to_string()),
+    )
+  )]
+  async fn resolve(
+    self,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
+  ) -> Result<Self::Response, Self::Error> {
+    let swarm = get_check_permissions::<Swarm>(
+      &self.swarm,
+      user,
+      PermissionLevel::Execute.into(),
+    )
+    .await?;
+
+    update_update(update.clone()).await?;
+
+    let mut update = update.clone();
+
+    match swarm_request(
+      &swarm.config.server_ids,
+      periphery_client::api::swarm::RemoveSwarmServices {
+        services: self.services,
+      },
+    )
+    .await
+    {
+      Ok(log) => update.logs.push(log),
+      Err(e) => update.push_error_log(
+        "Remove Swarm Services",
+        format_serror(
+          &e.context("Failed to remove swarm services").into(),
+        ),
+      ),
+    };
+
+    update.finalize();
+    update_update(update.clone()).await?;
+
+    Ok(update)
+  }
+}
+
+impl Resolve<ExecuteArgs> for RemoveSwarmConfigs {
+  #[instrument(
+    "RemoveSwarmConfigs",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      swarm = self.swarm,
+      configs = serde_json::to_string(&self.configs).unwrap_or_else(|e| e.to_string()),
+    )
+  )]
+  async fn resolve(
+    self,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
+  ) -> Result<Self::Response, Self::Error> {
+    let swarm = get_check_permissions::<Swarm>(
+      &self.swarm,
+      user,
+      PermissionLevel::Execute.into(),
+    )
+    .await?;
+
+    update_update(update.clone()).await?;
+
+    let mut update = update.clone();
+
+    match swarm_request(
+      &swarm.config.server_ids,
+      periphery_client::api::swarm::RemoveSwarmConfigs {
+        configs: self.configs,
+      },
+    )
+    .await
+    {
+      Ok(log) => update.logs.push(log),
+      Err(e) => update.push_error_log(
+        "Remove Swarm Configs",
+        format_serror(
+          &e.context("Failed to remove swarm configs").into(),
+        ),
+      ),
+    };
+
+    update.finalize();
+    update_update(update.clone()).await?;
+
+    Ok(update)
+  }
+}
+
+impl Resolve<ExecuteArgs> for RemoveSwarmSecrets {
+  #[instrument(
+    "RemoveSwarmSecrets",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      swarm = self.swarm,
+      secrets = serde_json::to_string(&self.secrets).unwrap_or_else(|e| e.to_string()),
+    )
+  )]
+  async fn resolve(
+    self,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
+  ) -> Result<Self::Response, Self::Error> {
+    let swarm = get_check_permissions::<Swarm>(
+      &self.swarm,
+      user,
+      PermissionLevel::Execute.into(),
+    )
+    .await?;
+
+    update_update(update.clone()).await?;
+
+    let mut update = update.clone();
+
+    match swarm_request(
+      &swarm.config.server_ids,
+      periphery_client::api::swarm::RemoveSwarmSecrets {
+        secrets: self.secrets,
+      },
+    )
+    .await
+    {
+      Ok(log) => update.logs.push(log),
+      Err(e) => update.push_error_log(
+        "Remove Swarm Secrets",
+        format_serror(
+          &e.context("Failed to remove swarm secrets").into(),
+        ),
+      ),
+    };
+
+    update.finalize();
+    update_update(update.clone()).await?;
+
+    Ok(update)
+  }
+}

bin/core/src/api/listener/integrations/github.rs

@@ -5,10 +5,9 @@ use hmac::{Hmac, Mac};
 use serde::Deserialize;
 use sha2::Sha256;
 
-use crate::{
-  config::core_config,
-  listener::{ExtractBranch, VerifySecret},
-};
+use crate::config::core_config;
+
+use super::{ExtractBranch, VerifySecret};
 
 type HmacSha256 = Hmac<Sha256>;
 
@@ -18,7 +17,7 @@ pub struct Github;
 impl VerifySecret for Github {
   #[instrument("VerifyGithubSecret", skip_all)]
   fn verify_secret(
-    headers: HeaderMap,
+    headers: &HeaderMap,
     body: &str,
     custom_secret: &str,
   ) -> anyhow::Result<()> {

bin/core/src/api/listener/integrations/gitlab.rs

@@ -1,10 +1,10 @@
 use anyhow::{Context, anyhow};
+use axum::http::HeaderMap;
 use serde::Deserialize;
 
-use crate::{
-  config::core_config,
-  listener::{ExtractBranch, VerifySecret},
-};
+use crate::config::core_config;
+
+use super::{ExtractBranch, VerifySecret};
 
 /// Listener implementation for Gitlab type API
 pub struct Gitlab;
@@ -12,7 +12,7 @@ pub struct Gitlab;
 impl VerifySecret for Gitlab {
   #[instrument("VerifyGitlabSecret", skip_all)]
   fn verify_secret(
-    headers: axum::http::HeaderMap,
+    headers: &HeaderMap,
     _body: &str,
     custom_secret: &str,
   ) -> anyhow::Result<()> {

bin/core/src/api/listener/integrations/mod.rs

@@ -0,0 +1,4 @@
+pub mod github;
+pub mod gitlab;
+
+use super::{ExtractBranch, VerifySecret};

bin/core/src/api/listener/mod.rs

@@ -32,7 +32,7 @@ trait CustomSecret: KomodoResource {
 /// Implemented on the integration struct, eg [integrations::github::Github]
 trait VerifySecret {
   fn verify_secret(
-    headers: HeaderMap,
+    headers: &HeaderMap,
     body: &str,
     custom_secret: &str,
   ) -> anyhow::Result<()>;

bin/core/src/api/listener/router.rs

@@ -1,14 +1,22 @@
-use axum::{Router, extract::Path, http::HeaderMap, routing::post};
+use std::net::{IpAddr, SocketAddr};
+
+use axum::{
+  Router,
+  extract::{ConnectInfo, Path},
+  http::HeaderMap,
+  routing::post,
+};
 use komodo_client::entities::{
   action::Action, build::Build, procedure::Procedure, repo::Repo,
   resource::Resource, stack::Stack, sync::ResourceSync,
 };
+use rate_limit::WithFailureRateLimit;
 use reqwest::StatusCode;
 use serde::Deserialize;
 use serror::AddStatusCode;
 use tracing::Instrument;
 
-use crate::resource::KomodoResource;
+use crate::{resource::KomodoResource, state::auth_rate_limiter};
 
 use super::{
   CustomSecret, ExtractBranch, VerifySecret,
@@ -47,9 +55,9 @@ pub fn router<P: VerifySecret + ExtractBranch>() -> Router {
   .route(
     "/build/{id}",
     post(
-      |Path(Id { id }), headers: HeaderMap, body: String| async move {
+      |Path(Id { id }), headers: HeaderMap, ConnectInfo(info): ConnectInfo<SocketAddr>, body: String| async move {
         let build =
-          auth_webhook::<P, Build>(&id, headers, &body).await?;
+          auth_webhook::<P, Build>(&id, &headers, info.ip(), &body).await?;
         tokio::spawn(async move {
           let span = info_span!("BuildWebhook", id);
           async {
@@ -73,9 +81,9 @@ pub fn router<P: VerifySecret + ExtractBranch>() -> Router {
   .route(
     "/repo/{id}/{option}",
     post(
-      |Path(IdAndOption::<RepoWebhookOption> { id, option }), headers: HeaderMap, body: String| async move {
+      |Path(IdAndOption::<RepoWebhookOption> { id, option }), headers: HeaderMap, ConnectInfo(info): ConnectInfo<SocketAddr>, body: String| async move {
         let repo =
-          auth_webhook::<P, Repo>(&id, headers, &body).await?;
+          auth_webhook::<P, Repo>(&id, &headers, info.ip(), &body).await?;
         tokio::spawn(async move {
           let span = info_span!("RepoWebhook", id);
           async {
@@ -99,9 +107,9 @@ pub fn router<P: VerifySecret + ExtractBranch>() -> Router {
   .route(
     "/stack/{id}/{option}",
     post(
-      |Path(IdAndOption::<StackWebhookOption> { id, option }), headers: HeaderMap, body: String| async move {
+      |Path(IdAndOption::<StackWebhookOption> { id, option }), headers: HeaderMap, ConnectInfo(info): ConnectInfo<SocketAddr>, body: String| async move {
         let stack =
-          auth_webhook::<P, Stack>(&id, headers, &body).await?;
+          auth_webhook::<P, Stack>(&id, &headers, info.ip(), &body).await?;
         tokio::spawn(async move {
           let span = info_span!("StackWebhook", id);
           async {
@@ -125,9 +133,9 @@ pub fn router<P: VerifySecret + ExtractBranch>() -> Router {
   .route(
     "/sync/{id}/{option}",
     post(
-      |Path(IdAndOption::<SyncWebhookOption> { id, option }), headers: HeaderMap, body: String| async move {
+      |Path(IdAndOption::<SyncWebhookOption> { id, option }), headers: HeaderMap, ConnectInfo(info): ConnectInfo<SocketAddr>, body: String| async move {
         let sync =
-          auth_webhook::<P, ResourceSync>(&id, headers, &body).await?;
+          auth_webhook::<P, ResourceSync>(&id, &headers, info.ip(), &body).await?;
         tokio::spawn(async move {
           let span = info_span!("ResourceSyncWebhook", id);
           async {
@@ -151,9 +159,9 @@ pub fn router<P: VerifySecret + ExtractBranch>() -> Router {
   .route(
     "/procedure/{id}/{branch}",
     post(
-      |Path(IdAndBranch { id, branch }), headers: HeaderMap, body: String| async move {
+      |Path(IdAndBranch { id, branch }), headers: HeaderMap, ConnectInfo(info): ConnectInfo<SocketAddr>, body: String| async move {
         let procedure =
-          auth_webhook::<P, Procedure>(&id, headers, &body).await?;
+          auth_webhook::<P, Procedure>(&id, &headers, info.ip(), &body).await?;
         tokio::spawn(async move {
           let span = info_span!("ProcedureWebhook", id);
           async {
@@ -177,9 +185,9 @@ pub fn router<P: VerifySecret + ExtractBranch>() -> Router {
   .route(
     "/action/{id}/{branch}",
     post(
-      |Path(IdAndBranch { id, branch }), headers: HeaderMap, body: String| async move {
+      |Path(IdAndBranch { id, branch }), headers: HeaderMap, ConnectInfo(info): ConnectInfo<SocketAddr>, body: String| async move {
         let action =
-          auth_webhook::<P, Action>(&id, headers, &body).await?;
+          auth_webhook::<P, Action>(&id, &headers, info.ip(), &body).await?;
         tokio::spawn(async move {
           let span = info_span!("ActionWebhook", id);
           async {
@@ -204,17 +212,26 @@ pub fn router<P: VerifySecret + ExtractBranch>() -> Router {
 
 async fn auth_webhook<P, R>(
   id: &str,
-  headers: HeaderMap,
+  headers: &HeaderMap,
+  ip: IpAddr,
   body: &str,
 ) -> serror::Result<Resource<R::Config, R::Info>>
 where
   P: VerifySecret,
   R: KomodoResource + CustomSecret,
 {
-  let resource = crate::resource::get::<R>(id)
-    .await
-    .status_code(StatusCode::BAD_REQUEST)?;
-  P::verify_secret(headers, body, R::custom_secret(&resource))
-    .status_code(StatusCode::UNAUTHORIZED)?;
-  Ok(resource)
+  async {
+    let resource = crate::resource::get::<R>(id)
+      .await
+      .status_code(StatusCode::BAD_REQUEST)?;
+    P::verify_secret(headers, body, R::custom_secret(&resource))
+      .status_code(StatusCode::UNAUTHORIZED)?;
+    serror::Result::Ok(resource)
+  }
+  .with_failure_rate_limit_using_headers(
+    auth_rate_limiter(),
+    headers,
+    Some(ip),
+  )
+  .await
 }

bin/core/src/api/mod.rs

@@ -1,11 +1,92 @@
+use axum::{
+  Router,
+  http::{HeaderName, HeaderValue},
+  routing::get,
+};
+use tower_http::{
+  services::{ServeDir, ServeFile},
+  set_header::SetResponseHeaderLayer,
+};
+use tower_sessions::{
+  Expiry, MemoryStore, SessionManagerLayer, cookie::time::Duration,
+};
+
+use crate::{
+  config::{core_config, core_host, cors_layer},
+  ts_client,
+};
+
 pub mod auth;
 pub mod execute;
 pub mod read;
-pub mod terminal;
 pub mod user;
 pub mod write;
 
+mod listener;
+mod terminal;
+mod ws;
+
 #[derive(serde::Deserialize)]
 struct Variant {
   variant: String,
 }
+
+pub fn app() -> Router {
+  let config = core_config();
+
+  // Setup static frontend services
+  let frontend_path = &config.frontend_path;
+  let frontend_index =
+    ServeFile::new(format!("{frontend_path}/index.html"));
+  let serve_frontend = ServeDir::new(frontend_path)
+    .not_found_service(frontend_index.clone());
+
+  Router::new()
+    .route("/version", get(|| async { env!("CARGO_PKG_VERSION") }))
+    .nest("/auth", auth::router())
+    .nest("/user", user::router())
+    .nest("/read", read::router())
+    .nest("/write", write::router())
+    .nest("/execute", execute::router())
+    .nest("/terminal", terminal::router())
+    .nest("/listener", listener::router())
+    .nest("/ws", ws::router())
+    .nest("/client", ts_client::router())
+    .fallback_service(serve_frontend)
+    .layer(cors_layer())
+    .layer(SetResponseHeaderLayer::overriding(
+      HeaderName::from_static("x-content-type-options"),
+      HeaderValue::from_static("nosniff"),
+    ))
+    .layer(SetResponseHeaderLayer::overriding(
+      HeaderName::from_static("x-frame-options"),
+      HeaderValue::from_static("DENY"),
+    ))
+    .layer(SetResponseHeaderLayer::overriding(
+      HeaderName::from_static("x-xss-protection"),
+      HeaderValue::from_static("1; mode=block"),
+    ))
+    .layer(SetResponseHeaderLayer::overriding(
+      HeaderName::from_static("referrer-policy"),
+      HeaderValue::from_static("strict-origin-when-cross-origin"),
+    ))
+}
+
+fn memory_session_layer(
+  expiry: i64,
+) -> SessionManagerLayer<MemoryStore> {
+  let config = core_config();
+  let mut layer = SessionManagerLayer::new(MemoryStore::default())
+    .with_expiry(Expiry::OnInactivity(Duration::seconds(expiry)))
+    .with_secure(config.host.starts_with("https://"));
+  if let Some(domain) = core_host().and_then(|url| url.domain()) {
+    layer = layer.with_domain(domain);
+  }
+  layer
+}
+
+pub const SESSION_KEY_USER_ID: &str = "user-id";
+pub const SESSION_KEY_TOTP_LOGIN: &str = "totp-user-id";
+pub const SESSION_KEY_TOTP_ENROLLMENT: &str = "totp-enrollment";
+pub const SESSION_KEY_PASSKEY_LOGIN: &str = "passkey-user-id";
+pub const SESSION_KEY_PASSKEY_ENROLLMENT: &str = "passkey-enrollment";

bin/core/src/api/read/alert.rs

@@ -8,15 +8,15 @@ use komodo_client::{
   api::read::{
     GetAlert, GetAlertResponse, ListAlerts, ListAlertsResponse,
   },
-  entities::{
-    deployment::Deployment, permission::PermissionLevel,
-    server::Server, stack::Stack, sync::ResourceSync,
-  },
+  entities::permission::PermissionLevel,
 };
 use resolver_api::Resolve;
 
 use crate::{
-  config::core_config, permission::list_resource_ids_for_user,
+  config::core_config,
+  permission::{
+    check_user_target_access, user_resource_target_query,
+  },
   state::db_client,
 };
 
@@ -29,40 +29,10 @@ impl Resolve<ReadArgs> for ListAlerts {
     self,
     ReadArgs { user }: &ReadArgs,
   ) -> serror::Result<ListAlertsResponse> {
-    let mut query = self.query.unwrap_or_default();
-    if !user.admin && !core_config().transparent_mode {
-      let (server_ids, stack_ids, deployment_ids, sync_ids) = tokio::try_join!(
-        list_resource_ids_for_user::<Server>(
-          None,
-          user,
-          PermissionLevel::Read.into(),
-        ),
-        list_resource_ids_for_user::<Stack>(
-          None,
-          user,
-          PermissionLevel::Read.into(),
-        ),
-        list_resource_ids_for_user::<Deployment>(
-          None,
-          user,
-          PermissionLevel::Read.into(),
-        ),
-        list_resource_ids_for_user::<ResourceSync>(
-          None,
-          user,
-          PermissionLevel::Read.into(),
-        )
-      )?;
-      // All of the vecs will be non-none if !admin and !transparent mode.
-      query.extend(doc! {
-        "$or": [
-          { "target.type": "Server", "target.id": { "$in": &server_ids } },
-          { "target.type": "Stack", "target.id": { "$in": &stack_ids } },
-          { "target.type": "Deployment", "target.id": { "$in": &deployment_ids } },
-          { "target.type": "ResourceSync", "target.id": { "$in": &sync_ids } },
-        ]
-      });
-    }
+    // Alerts
+    let query = user_resource_target_query(user, self.query)
+      .await?
+      .unwrap_or_default();
 
     let alerts = find_collect(
       &db_client().alerts,
@@ -91,13 +61,21 @@ impl Resolve<ReadArgs> for ListAlerts {
 impl Resolve<ReadArgs> for GetAlert {
   async fn resolve(
     self,
-    _: &ReadArgs,
+    ReadArgs { user }: &ReadArgs,
   ) -> serror::Result<GetAlertResponse> {
-    Ok(
-      find_one_by_id(&db_client().alerts, &self.id)
-        .await
-        .context("failed to query db for alert")?
-        .context("no alert found with given id")?,
+    let alert = find_one_by_id(&db_client().alerts, &self.id)
+      .await
+      .context("failed to query db for alert")?
+      .context("no alert found with given id")?;
+    if user.admin || core_config().transparent_mode {
+      return Ok(alert);
+    }
+    check_user_target_access(
+      &alert.target,
+      user,
+      PermissionLevel::Read.into(),
     )
+    .await?;
+    Ok(alert)
   }
 }

bin/core/src/api/read/deployment.rs

@@ -4,23 +4,31 @@ use anyhow::{Context, anyhow};
 use komodo_client::{
   api::read::*,
   entities::{
+    SwarmOrServer,
     deployment::{
       Deployment, DeploymentActionState, DeploymentConfig,
       DeploymentListItem, DeploymentState,
     },
-    docker::container::{Container, ContainerStats},
+    docker::{
+      container::{Container, ContainerStats},
+      service::SwarmService,
+    },
     permission::PermissionLevel,
     server::{Server, ServerState},
     update::Log,
   },
 };
 use periphery_client::api::{self, container::InspectContainer};
+use reqwest::StatusCode;
 use resolver_api::Resolve;
+use serror::AddStatusCodeError as _;
 
 use crate::{
-  helpers::{periphery_client, query::get_all_tags},
+  helpers::{
+    periphery_client, query::get_all_tags, swarm::swarm_request,
+  },
   permission::get_check_permissions,
-  resource,
+  resource::{self, setup_deployment_execution},
   state::{
     action_states, deployment_status_cache, server_status_cache,
   },
@@ -131,30 +139,40 @@ impl Resolve<ReadArgs> for GetDeploymentLog {
       tail,
       timestamps,
     } = self;
-    let Deployment {
-      name,
-      config: DeploymentConfig { server_id, .. },
-      ..
-    } = get_check_permissions::<Deployment>(
+
+    let (deployment, swarm_or_server) = setup_deployment_execution(
       &deployment,
       user,
       PermissionLevel::Read.logs(),
     )
     .await?;
-    if server_id.is_empty() {
-      return Ok(Log::default());
-    }
-    let server = resource::get::<Server>(&server_id).await?;
-    let res = periphery_client(&server)
-      .await?
-      .request(api::container::GetContainerLog {
-        name,
-        tail: cmp::min(tail, MAX_LOG_LENGTH),
-        timestamps,
-      })
+
+    let log = match swarm_or_server {
+      SwarmOrServer::Swarm(swarm) => swarm_request(
+        &swarm.config.server_ids,
+        periphery_client::api::swarm::GetSwarmServiceLog {
+          service: deployment.name,
+          tail,
+          timestamps,
+          no_task_ids: false,
+          no_resolve: false,
+          details: false,
+        },
+      )
       .await
-      .context("failed at call to periphery")?;
-    Ok(res)
+      .context("Failed to get service log from swarm")?,
+      SwarmOrServer::Server(server) => periphery_client(&server)
+        .await?
+        .request(api::container::GetContainerLog {
+          name: deployment.name,
+          tail: cmp::min(tail, MAX_LOG_LENGTH),
+          timestamps,
+        })
+        .await
+        .context("failed at call to periphery")?,
+    };
+
+    Ok(log)
   }
 }
 
@@ -170,32 +188,44 @@ impl Resolve<ReadArgs> for SearchDeploymentLog {
       invert,
       timestamps,
     } = self;
-    let Deployment {
-      name,
-      config: DeploymentConfig { server_id, .. },
-      ..
-    } = get_check_permissions::<Deployment>(
+
+    let (deployment, swarm_or_server) = setup_deployment_execution(
       &deployment,
       user,
       PermissionLevel::Read.logs(),
     )
     .await?;
-    if server_id.is_empty() {
-      return Ok(Log::default());
-    }
-    let server = resource::get::<Server>(&server_id).await?;
-    let res = periphery_client(&server)
-      .await?
-      .request(api::container::GetContainerLogSearch {
-        name,
-        terms,
-        combinator,
-        invert,
-        timestamps,
-      })
+
+    let log = match swarm_or_server {
+      SwarmOrServer::Swarm(swarm) => swarm_request(
+        &swarm.config.server_ids,
+        periphery_client::api::swarm::GetSwarmServiceLogSearch {
+          service: deployment.name,
+          terms,
+          combinator,
+          invert,
+          timestamps,
+          no_task_ids: false,
+          no_resolve: false,
+          details: false,
+        },
+      )
       .await
-      .context("failed at call to periphery")?;
-    Ok(res)
+      .context("Failed to search service log from swarm")?,
+      SwarmOrServer::Server(server) => periphery_client(&server)
+        .await?
+        .request(api::container::GetContainerLogSearch {
+          name: deployment.name,
+          terms,
+          combinator,
+          invert,
+          timestamps,
+        })
+        .await
+        .context("Failed to search container log from server")?,
+    };
+
+    Ok(log)
   }
 }
 
@@ -205,42 +235,78 @@ impl Resolve<ReadArgs> for InspectDeploymentContainer {
     ReadArgs { user }: &ReadArgs,
   ) -> serror::Result<Container> {
     let InspectDeploymentContainer { deployment } = self;
-    let Deployment {
-      name,
-      config: DeploymentConfig { server_id, .. },
-      ..
-    } = get_check_permissions::<Deployment>(
+    let (deployment, swarm_or_server) = setup_deployment_execution(
       &deployment,
       user,
       PermissionLevel::Read.inspect(),
     )
     .await?;
-    if server_id.is_empty() {
+
+    let SwarmOrServer::Server(server) = swarm_or_server else {
       return Err(
         anyhow!(
-          "Cannot inspect deployment, not attached to any server"
+          "InspectDeploymentContainer should not be called for Deployment in Swarm Mode"
         )
-        .into(),
+        .status_code(StatusCode::BAD_REQUEST),
       );
-    }
-    let server = resource::get::<Server>(&server_id).await?;
+    };
+
     let cache = server_status_cache()
       .get_or_insert_default(&server.id)
       .await;
+
     if cache.state != ServerState::Ok {
       return Err(
         anyhow!(
-          "Cannot inspect container: server is {:?}",
+          "Cannot inspect container: Server is {:?}",
           cache.state
         )
         .into(),
       );
     }
-    let res = periphery_client(&server)
+
+    periphery_client(&server)
       .await?
-      .request(InspectContainer { name })
-      .await?;
-    Ok(res)
+      .request(InspectContainer {
+        name: deployment.name,
+      })
+      .await
+      .context("Failed to inspect container on server")
+      .map_err(Into::into)
+  }
+}
+
+impl Resolve<ReadArgs> for InspectDeploymentSwarmService {
+  async fn resolve(
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<SwarmService> {
+    let InspectDeploymentSwarmService { deployment } = self;
+    let (deployment, swarm_or_server) = setup_deployment_execution(
+      &deployment,
+      user,
+      PermissionLevel::Read.logs(),
+    )
+    .await?;
+
+    let SwarmOrServer::Swarm(swarm) = swarm_or_server else {
+      return Err(
+        anyhow!(
+          "InspectDeploymentSwarmService should only be called for Deployment in Swarm Mode"
+        )
+        .status_code(StatusCode::BAD_REQUEST),
+      );
+    };
+
+    swarm_request(
+      &swarm.config.server_ids,
+      periphery_client::api::swarm::InspectSwarmService {
+        service: deployment.name,
+      },
+    )
+    .await
+    .context("Failed to inspect service on swarm")
+    .map_err(Into::into)
   }
 }
 

bin/core/src/api/read/mod.rs

@@ -97,6 +97,7 @@ enum ReadRequest {
   InspectSwarmService(InspectSwarmService),
   GetSwarmServiceLog(GetSwarmServiceLog),
   SearchSwarmServiceLog(SearchSwarmServiceLog),
+  ListSwarmNetworks(ListSwarmNetworks),
 
   // ==== SERVER ====
   GetServersSummary(GetServersSummary),
@@ -140,6 +141,7 @@ enum ReadRequest {
   GetStackLog(GetStackLog),
   SearchStackLog(SearchStackLog),
   InspectStackContainer(InspectStackContainer),
+  InspectStackSwarmService(InspectStackSwarmService),
   ListStacks(ListStacks),
   ListFullStacks(ListFullStacks),
   ListStackServices(ListStackServices),
@@ -155,6 +157,7 @@ enum ReadRequest {
   GetDeploymentLog(GetDeploymentLog),
   SearchDeploymentLog(SearchDeploymentLog),
   InspectDeploymentContainer(InspectDeploymentContainer),
+  InspectDeploymentSwarmService(InspectDeploymentSwarmService),
   ListDeployments(ListDeployments),
   ListFullDeployments(ListFullDeployments),
   ListCommonDeploymentExtraArgs(ListCommonDeploymentExtraArgs),

bin/core/src/api/read/stack.rs

@@ -4,9 +4,11 @@ use anyhow::{Context, anyhow};
 use komodo_client::{
   api::read::*,
   entities::{
-    docker::container::Container,
+    SwarmOrServer,
+    docker::{
+      container::Container, service::SwarmService, stack::SwarmStack,
+    },
     permission::PermissionLevel,
-    server::{Server, ServerState},
     stack::{Stack, StackActionState, StackListItem, StackState},
   },
 };
@@ -14,14 +16,18 @@ use periphery_client::api::{
   compose::{GetComposeLog, GetComposeLogSearch},
   container::InspectContainer,
 };
+use reqwest::StatusCode;
 use resolver_api::Resolve;
+use serror::AddStatusCodeError as _;
 
 use crate::{
-  helpers::{periphery_client, query::get_all_tags},
+  helpers::{
+    periphery_client, query::get_all_tags, swarm::swarm_request,
+  },
   permission::get_check_permissions,
   resource,
-  stack::get_stack_and_server,
-  state::{action_states, server_status_cache, stack_status_cache},
+  stack::setup_stack_execution,
+  state::{action_states, stack_status_cache},
 };
 
 use super::ReadArgs;
@@ -73,28 +79,53 @@ impl Resolve<ReadArgs> for GetStackLog {
   ) -> serror::Result<GetStackLogResponse> {
     let GetStackLog {
       stack,
-      services,
+      mut services,
       tail,
       timestamps,
     } = self;
-    let (stack, server) = get_stack_and_server(
+    let (stack, swarm_or_server) = setup_stack_execution(
       &stack,
       user,
       PermissionLevel::Read.logs(),
-      true,
     )
     .await?;
-    let res = periphery_client(&server)
-      .await?
-      .request(GetComposeLog {
-        project: stack.project_name(false),
-        services,
-        tail,
-        timestamps,
-      })
-      .await
-      .context("Failed to get stack log from periphery")?;
-    Ok(res)
+
+    let log = match swarm_or_server {
+      SwarmOrServer::Swarm(swarm) => {
+        let service = services.pop().context(
+          "Must pass single service for Swarm mode Stack logs",
+        )?;
+        swarm_request(
+          &swarm.config.server_ids,
+          periphery_client::api::swarm::GetSwarmServiceLog {
+            // The actual service name on swarm will be stackname_servicename
+            service: format!(
+              "{}_{service}",
+              stack.project_name(false)
+            ),
+            tail,
+            timestamps,
+            no_task_ids: false,
+            no_resolve: false,
+            details: false,
+          },
+        )
+        .await
+        .context("Failed to get stack service log from swarm")?
+      }
+      SwarmOrServer::Server(server) => periphery_client(&server)
+        .await?
+        .request(GetComposeLog {
+          project: stack.project_name(false),
+          services,
+          tail,
+          timestamps,
+        })
+        .await
+        .context("Failed to get stack log from periphery")?,
+    };
+
+    Ok(log)
   }
 }
 
@@ -105,32 +136,55 @@ impl Resolve<ReadArgs> for SearchStackLog {
   ) -> serror::Result<SearchStackLogResponse> {
     let SearchStackLog {
       stack,
-      services,
+      mut services,
       terms,
       combinator,
       invert,
       timestamps,
     } = self;
-    let (stack, server) = get_stack_and_server(
+    let (stack, swarm_or_server) = setup_stack_execution(
       &stack,
       user,
       PermissionLevel::Read.logs(),
-      true,
     )
     .await?;
-    let res = periphery_client(&server)
-      .await?
-      .request(GetComposeLogSearch {
-        project: stack.project_name(false),
-        services,
-        terms,
-        combinator,
-        invert,
-        timestamps,
-      })
-      .await
-      .context("Failed to search stack log from periphery")?;
-    Ok(res)
+
+    let log = match swarm_or_server {
+      SwarmOrServer::Swarm(swarm) => {
+        let service = services.pop().context(
+          "Must pass single service for Swarm mode Stack logs",
+        )?;
+        swarm_request(
+          &swarm.config.server_ids,
+          periphery_client::api::swarm::GetSwarmServiceLogSearch {
+            service,
+            terms,
+            combinator,
+            invert,
+            timestamps,
+            no_task_ids: false,
+            no_resolve: false,
+            details: false,
+          },
+        )
+        .await
+        .context("Failed to get stack service log from swarm")?
+      }
+      SwarmOrServer::Server(server) => periphery_client(&server)
+        .await?
+        .request(GetComposeLogSearch {
+          project: stack.project_name(false),
+          services,
+          terms,
+          combinator,
+          invert,
+          timestamps,
+        })
+        .await
+        .context("Failed to search stack log from periphery")?,
+    };
+
+    Ok(log)
   }
 }
 
@@ -140,38 +194,29 @@ impl Resolve<ReadArgs> for InspectStackContainer {
     ReadArgs { user }: &ReadArgs,
   ) -> serror::Result<Container> {
     let InspectStackContainer { stack, service } = self;
-    let stack = get_check_permissions::<Stack>(
+    let (stack, swarm_or_server) = setup_stack_execution(
       &stack,
       user,
       PermissionLevel::Read.inspect(),
     )
     .await?;
-    if stack.config.server_id.is_empty() {
-      return Err(
-        anyhow!("Cannot inspect stack, not attached to any server")
-          .into(),
-      );
-    }
-    let server =
-      resource::get::<Server>(&stack.config.server_id).await?;
-    let cache = server_status_cache()
-      .get_or_insert_default(&server.id)
-      .await;
-    if cache.state != ServerState::Ok {
+
+    let SwarmOrServer::Server(server) = swarm_or_server else {
       return Err(
         anyhow!(
-          "Cannot inspect container: server is {:?}",
-          cache.state
+          "InspectStackContainer should not be called for Stack in Swarm Mode"
         )
-        .into(),
+        .status_code(StatusCode::BAD_REQUEST),
       );
-    }
+    };
+
     let services = &stack_status_cache()
       .get(&stack.id)
       .await
       .unwrap_or_default()
       .curr
       .services;
+
     let Some(name) = services
       .iter()
       .find(|s| s.service == service)
@@ -181,14 +226,101 @@ impl Resolve<ReadArgs> for InspectStackContainer {
         "No service found matching '{service}'. Was the stack last deployed manually?"
       ).into());
     };
+
     let res = periphery_client(&server)
       .await?
       .request(InspectContainer { name })
-      .await?;
+      .await
+      .context("Failed to inspect container on server")?;
+
     Ok(res)
   }
 }
 
+impl Resolve<ReadArgs> for InspectStackSwarmService {
+  async fn resolve(
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<SwarmService> {
+    let InspectStackSwarmService { stack, service } = self;
+    let (stack, swarm_or_server) = setup_stack_execution(
+      &stack,
+      user,
+      PermissionLevel::Read.inspect(),
+    )
+    .await?;
+
+    let SwarmOrServer::Swarm(swarm) = swarm_or_server else {
+      return Err(
+        anyhow!(
+          "InspectStackSwarmService should only be called for Stack in Swarm Mode"
+        )
+        .status_code(StatusCode::BAD_REQUEST),
+      );
+    };
+
+    let services = &stack_status_cache()
+      .get(&stack.id)
+      .await
+      .unwrap_or_default()
+      .curr
+      .services;
+
+    let Some(service) = services
+      .iter()
+      .find(|s| s.service == service)
+      .and_then(|s| {
+        s.swarm_service.as_ref().and_then(|c| c.name.clone())
+      })
+    else {
+      return Err(anyhow!(
+        "No service found matching '{service}'. Was the stack last deployed manually?"
+      ).into());
+    };
+
+    swarm_request(
+      &swarm.config.server_ids,
+      periphery_client::api::swarm::InspectSwarmService { service },
+    )
+    .await
+    .context("Failed to inspect service on swarm")
+    .map_err(Into::into)
+  }
+}
+
+impl Resolve<ReadArgs> for InspectStackSwarmInfo {
+  async fn resolve(
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<SwarmStack> {
+    let (stack, swarm_or_server) = setup_stack_execution(
+      &self.stack,
+      user,
+      PermissionLevel::Read.inspect(),
+    )
+    .await?;
+
+    let SwarmOrServer::Swarm(swarm) = swarm_or_server else {
+      return Err(
+        anyhow!(
+          "InspectStackSwarmInfo should only be called for Stack in Swarm Mode"
+        )
+        .status_code(StatusCode::BAD_REQUEST),
+      );
+    };
+
+    swarm_request(
+      &swarm.config.server_ids,
+      periphery_client::api::swarm::InspectSwarmStack {
+        stack: stack.project_name(false),
+      },
+    )
+    .await
+    .context("Failed to inspect stack info on swarm")
+    .map_err(Into::into)
+  }
+}
+
 impl Resolve<ReadArgs> for ListCommonStackExtraArgs {
   async fn resolve(
     self,
@@ -206,7 +338,7 @@ impl Resolve<ReadArgs> for ListCommonStackExtraArgs {
       &all_tags,
     )
     .await
-    .context("failed to get resources matching query")?;
+    .context("Failed to get resources matching query")?;
 
     // first collect with guaranteed uniqueness
     let mut res = HashSet::<String>::new();
@@ -240,7 +372,7 @@ impl Resolve<ReadArgs> for ListCommonStackBuildExtraArgs {
       &all_tags,
     )
     .await
-    .context("failed to get resources matching query")?;
+    .context("Failed to get resources matching query")?;
 
     // first collect with guaranteed uniqueness
     let mut res = HashSet::<String>::new();
@@ -348,7 +480,7 @@ impl Resolve<ReadArgs> for GetStacksSummary {
       &[],
     )
     .await
-    .context("failed to get stacks from db")?;
+    .context("Failed to get stacks from database")?;
 
     let mut res = GetStacksSummaryResponse::default();
 

bin/core/src/api/read/swarm.rs

@@ -1,4 +1,4 @@
-use anyhow::Context;
+use anyhow::{Context, anyhow};
 use komodo_client::{
   api::read::*,
   entities::{
@@ -12,7 +12,7 @@ use crate::{
   helpers::{query::get_all_tags, swarm::swarm_request},
   permission::get_check_permissions,
   resource,
-  state::{action_states, swarm_status_cache},
+  state::{action_states, server_status_cache, swarm_status_cache},
 };
 
 use super::ReadArgs;
@@ -149,7 +149,7 @@ impl Resolve<ReadArgs> for InspectSwarm {
     let swarm = get_check_permissions::<Swarm>(
       &self.swarm,
       user,
-      PermissionLevel::Read.into(),
+      PermissionLevel::Read.inspect(),
     )
     .await?;
     let cache =
@@ -192,7 +192,7 @@ impl Resolve<ReadArgs> for InspectSwarmNode {
     let swarm = get_check_permissions::<Swarm>(
       &self.swarm,
       user,
-      PermissionLevel::Read.into(),
+      PermissionLevel::Read.inspect(),
     )
     .await?;
     swarm_request(
@@ -235,7 +235,7 @@ impl Resolve<ReadArgs> for InspectSwarmService {
     let swarm = get_check_permissions::<Swarm>(
       &self.swarm,
       user,
-      PermissionLevel::Read.into(),
+      PermissionLevel::Read.inspect(),
     )
     .await?;
     swarm_request(
@@ -334,7 +334,7 @@ impl Resolve<ReadArgs> for InspectSwarmTask {
     let swarm = get_check_permissions::<Swarm>(
       &self.swarm,
       user,
-      PermissionLevel::Read.into(),
+      PermissionLevel::Read.inspect(),
     )
     .await?;
     swarm_request(
@@ -377,7 +377,7 @@ impl Resolve<ReadArgs> for InspectSwarmSecret {
     let swarm = get_check_permissions::<Swarm>(
       &self.swarm,
       user,
-      PermissionLevel::Read.into(),
+      PermissionLevel::Read.inspect(),
     )
     .await?;
     swarm_request(
@@ -420,7 +420,7 @@ impl Resolve<ReadArgs> for InspectSwarmConfig {
     let swarm = get_check_permissions::<Swarm>(
       &self.swarm,
       user,
-      PermissionLevel::Read.into(),
+      PermissionLevel::Read.inspect(),
     )
     .await?;
     swarm_request(
@@ -463,7 +463,7 @@ impl Resolve<ReadArgs> for InspectSwarmStack {
     let swarm = get_check_permissions::<Swarm>(
       &self.swarm,
       user,
-      PermissionLevel::Read.into(),
+      PermissionLevel::Read.inspect(),
     )
     .await?;
     swarm_request(
@@ -476,3 +476,44 @@ impl Resolve<ReadArgs> for InspectSwarmStack {
     .map_err(Into::into)
   }
 }
+
+impl Resolve<ReadArgs> for ListSwarmNetworks {
+  async fn resolve(
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> Result<Self::Response, Self::Error> {
+    let swarm = get_check_permissions::<Swarm>(
+      &self.swarm,
+      user,
+      PermissionLevel::Read.into(),
+    )
+    .await?;
+
+    let cache = server_status_cache();
+
+    for server_id in swarm.config.server_ids {
+      let Some(status) = cache.get(&server_id).await else {
+        continue;
+      };
+      let Some(docker) = &status.docker else {
+        continue;
+      };
+      let networks = docker
+        .networks
+        .iter()
+        .filter(|network| {
+          network.driver.as_deref() == Some("overlay")
+        })
+        .cloned()
+        .collect::<Vec<_>>();
+      return Ok(networks);
+    }
+
+    Err(
+      anyhow!(
+        "Failed to retrieve swarm networks from any manager node."
+      )
+      .into(),
+    )
+  }
+}

bin/core/src/api/read/update.rs

@@ -1,6 +1,6 @@
 use std::collections::HashMap;
 
-use anyhow::{Context, anyhow};
+use anyhow::Context;
 use database::mungos::{
   by_id::find_one_by_id,
   find::find_collect,
@@ -9,19 +9,7 @@ use database::mungos::{
 use komodo_client::{
   api::read::{GetUpdate, ListUpdates, ListUpdatesResponse},
   entities::{
-    ResourceTarget,
-    action::Action,
-    alerter::Alerter,
-    build::Build,
-    builder::Builder,
-    deployment::Deployment,
     permission::PermissionLevel,
-    procedure::Procedure,
-    repo::Repo,
-    server::Server,
-    stack::Stack,
-    swarm::Swarm,
-    sync::ResourceSync,
     update::{Update, UpdateListItem},
     user::User,
   },
@@ -30,7 +18,9 @@ use resolver_api::Resolve;
 
 use crate::{
   config::core_config,
-  permission::{get_check_permissions, list_resource_ids_for_user},
+  permission::{
+    check_user_target_access, user_resource_target_query,
+  },
   state::db_client,
 };
 
@@ -43,158 +33,7 @@ impl Resolve<ReadArgs> for ListUpdates {
     self,
     ReadArgs { user }: &ReadArgs,
   ) -> serror::Result<ListUpdatesResponse> {
-    let query = if user.admin || core_config().transparent_mode {
-      self.query
-    } else {
-      let server_query = list_resource_ids_for_user::<Server>(
-        None,
-        user,
-        PermissionLevel::Read.into(),
-      )
-      .await?
-      .map(|ids| {
-        doc! {
-          "target.type": "Server", "target.id": { "$in": ids }
-        }
-      })
-      .unwrap_or_else(|| doc! { "target.type": "Server" });
-
-      let deployment_query =
-        list_resource_ids_for_user::<Deployment>(
-          None,
-          user,
-          PermissionLevel::Read.into(),
-        )
-        .await?
-        .map(|ids| {
-          doc! {
-            "target.type": "Deployment", "target.id": { "$in": ids }
-          }
-        })
-        .unwrap_or_else(|| doc! { "target.type": "Deployment" });
-
-      let stack_query = list_resource_ids_for_user::<Stack>(
-        None,
-        user,
-        PermissionLevel::Read.into(),
-      )
-      .await?
-      .map(|ids| {
-        doc! {
-          "target.type": "Stack", "target.id": { "$in": ids }
-        }
-      })
-      .unwrap_or_else(|| doc! { "target.type": "Stack" });
-
-      let build_query = list_resource_ids_for_user::<Build>(
-        None,
-        user,
-        PermissionLevel::Read.into(),
-      )
-      .await?
-      .map(|ids| {
-        doc! {
-          "target.type": "Build", "target.id": { "$in": ids }
-        }
-      })
-      .unwrap_or_else(|| doc! { "target.type": "Build" });
-
-      let repo_query = list_resource_ids_for_user::<Repo>(
-        None,
-        user,
-        PermissionLevel::Read.into(),
-      )
-      .await?
-      .map(|ids| {
-        doc! {
-          "target.type": "Repo", "target.id": { "$in": ids }
-        }
-      })
-      .unwrap_or_else(|| doc! { "target.type": "Repo" });
-
-      let procedure_query = list_resource_ids_for_user::<Procedure>(
-        None,
-        user,
-        PermissionLevel::Read.into(),
-      )
-      .await?
-      .map(|ids| {
-        doc! {
-          "target.type": "Procedure", "target.id": { "$in": ids }
-        }
-      })
-      .unwrap_or_else(|| doc! { "target.type": "Procedure" });
-
-      let action_query = list_resource_ids_for_user::<Action>(
-        None,
-        user,
-        PermissionLevel::Read.into(),
-      )
-      .await?
-      .map(|ids| {
-        doc! {
-          "target.type": "Action", "target.id": { "$in": ids }
-        }
-      })
-      .unwrap_or_else(|| doc! { "target.type": "Action" });
-
-      let builder_query = list_resource_ids_for_user::<Builder>(
-        None,
-        user,
-        PermissionLevel::Read.into(),
-      )
-      .await?
-      .map(|ids| {
-        doc! {
-          "target.type": "Builder", "target.id": { "$in": ids }
-        }
-      })
-      .unwrap_or_else(|| doc! { "target.type": "Builder" });
-
-      let alerter_query = list_resource_ids_for_user::<Alerter>(
-        None,
-        user,
-        PermissionLevel::Read.into(),
-      )
-      .await?
-      .map(|ids| {
-        doc! {
-          "target.type": "Alerter", "target.id": { "$in": ids }
-        }
-      })
-      .unwrap_or_else(|| doc! { "target.type": "Alerter" });
-
-      let resource_sync_query =
-        list_resource_ids_for_user::<ResourceSync>(
-          None,
-          user,
-          PermissionLevel::Read.into(),
-        )
-        .await?
-        .map(|ids| {
-          doc! {
-            "target.type": "ResourceSync", "target.id": { "$in": ids }
-          }
-        })
-        .unwrap_or_else(|| doc! { "target.type": "ResourceSync" });
-
-      let mut query = self.query.unwrap_or_default();
-      query.extend(doc! {
-        "$or": [
-          server_query,
-          deployment_query,
-          stack_query,
-          build_query,
-          repo_query,
-          procedure_query,
-          action_query,
-          alerter_query,
-          builder_query,
-          resource_sync_query,
-        ]
-      });
-      query.into()
-    };
+    let query = user_resource_target_query(user, self.query).await?;
 
     let usernames = find_collect(&db_client().users, None, None)
       .await
@@ -261,101 +100,12 @@ impl Resolve<ReadArgs> for GetUpdate {
     if user.admin || core_config().transparent_mode {
       return Ok(update);
     }
-    match &update.target {
-      ResourceTarget::System(_) => {
-        return Err(
-          anyhow!("user must be admin to view system updates").into(),
-        );
-      }
-      ResourceTarget::Swarm(id) => {
-        get_check_permissions::<Swarm>(
-          id,
-          user,
-          PermissionLevel::Read.into(),
-        )
-        .await?;
-      }
-      ResourceTarget::Server(id) => {
-        get_check_permissions::<Server>(
-          id,
-          user,
-          PermissionLevel::Read.into(),
-        )
-        .await?;
-      }
-      ResourceTarget::Stack(id) => {
-        get_check_permissions::<Stack>(
-          id,
-          user,
-          PermissionLevel::Read.into(),
-        )
-        .await?;
-      }
-      ResourceTarget::Deployment(id) => {
-        get_check_permissions::<Deployment>(
-          id,
-          user,
-          PermissionLevel::Read.into(),
-        )
-        .await?;
-      }
-      ResourceTarget::Build(id) => {
-        get_check_permissions::<Build>(
-          id,
-          user,
-          PermissionLevel::Read.into(),
-        )
-        .await?;
-      }
-      ResourceTarget::Repo(id) => {
-        get_check_permissions::<Repo>(
-          id,
-          user,
-          PermissionLevel::Read.into(),
-        )
-        .await?;
-      }
-      ResourceTarget::Procedure(id) => {
-        get_check_permissions::<Procedure>(
-          id,
-          user,
-          PermissionLevel::Read.into(),
-        )
-        .await?;
-      }
-      ResourceTarget::Action(id) => {
-        get_check_permissions::<Action>(
-          id,
-          user,
-          PermissionLevel::Read.into(),
-        )
-        .await?;
-      }
-      ResourceTarget::ResourceSync(id) => {
-        get_check_permissions::<ResourceSync>(
-          id,
-          user,
-          PermissionLevel::Read.into(),
-        )
-        .await?;
-      }
-      ResourceTarget::Builder(id) => {
-        get_check_permissions::<Builder>(
-          id,
-          user,
-          PermissionLevel::Read.into(),
-        )
-        .await?;
-      }
-      ResourceTarget::Alerter(id) => {
-        get_check_permissions::<Alerter>(
-          id,
-          user,
-          PermissionLevel::Read.into(),
-        )
-        .await?;
-      }
-    }
+    check_user_target_access(
+      &update.target,
+      user,
+      PermissionLevel::Read.into(),
+    )
+    .await?;
     Ok(update)
   }
 }

bin/core/src/api/user.rs

@@ -4,23 +4,37 @@ use anyhow::{Context, anyhow};
 use axum::{
   Extension, Json, Router, extract::Path, middleware, routing::post,
 };
+use data_encoding::BASE32_NOPAD;
+use database::hash_password;
 use database::mongo_indexed::doc;
 use database::mungos::{
   by_id::update_one_by_id, mongodb::bson::to_bson,
 };
 use derive_variants::EnumVariants;
-use komodo_client::entities::random_string;
+use komodo_client::entities::{random_bytes, random_string};
 use komodo_client::{
   api::user::*,
   entities::{api_key::ApiKey, komodo_timestamp, user::User},
 };
+use reqwest::StatusCode;
 use resolver_api::Resolve;
 use response::Response;
 use serde::{Deserialize, Serialize};
 use serde_json::json;
+use serror::{AddStatusCode, AddStatusCodeError};
+use tower_sessions::Session;
 use typeshare::typeshare;
 use uuid::Uuid;
+use webauthn_rs::prelude::PasskeyRegistration;
 
+use crate::api::{
+  SESSION_KEY_PASSKEY_ENROLLMENT, SESSION_KEY_TOTP_ENROLLMENT,
+  memory_session_layer,
+};
+use crate::auth::totp::make_totp;
+use crate::config::core_config;
+use crate::helpers::validations::validate_api_key_name;
+use crate::state::webauthn;
 use crate::{
   auth::auth_request, helpers::query::get_user, state::db_client,
 };
@@ -29,6 +43,8 @@ use super::Variant;
 
 pub struct UserArgs {
   pub user: User,
+  /// Per-client session state
+  pub session: Option<Session>,
 }
 
 #[typeshare]
@@ -44,16 +60,24 @@ enum UserRequest {
   SetLastSeenUpdate(SetLastSeenUpdate),
   CreateApiKey(CreateApiKey),
   DeleteApiKey(DeleteApiKey),
+  BeginTotpEnrollment(BeginTotpEnrollment),
+  ConfirmTotpEnrollment(ConfirmTotpEnrollment),
+  UnenrollTotp(UnenrollTotp),
+  BeginPasskeyEnrollment(BeginPasskeyEnrollment),
+  ConfirmPasskeyEnrollment(ConfirmPasskeyEnrollment),
+  UnenrollPasskey(UnenrollPasskey),
 }
 
 pub fn router() -> Router {
   Router::new()
     .route("/", post(handler))
     .route("/{variant}", post(variant_handler))
+    .layer(memory_session_layer(60))
     .layer(middleware::from_fn(auth_request))
 }
 
 async fn variant_handler(
+  session: Session,
   user: Extension<User>,
   Path(Variant { variant }): Path<Variant>,
   Json(params): Json<serde_json::Value>,
@@ -62,10 +86,11 @@ async fn variant_handler(
     "type": variant,
     "params": params,
   }))?;
-  handler(user, Json(req)).await
+  handler(session, user, Json(req)).await
 }
 
 async fn handler(
+  session: Session,
   Extension(user): Extension<User>,
   Json(request): Json<UserRequest>,
 ) -> serror::Result<axum::response::Response> {
@@ -75,7 +100,12 @@ async fn handler(
     "/user request {req_id} | user: {} ({})",
     user.username, user.id
   );
-  let res = request.resolve(&UserArgs { user }).await;
+  let res = request
+    .resolve(&UserArgs {
+      user,
+      session: Some(session),
+    })
+    .await;
   if let Err(e) = &res {
     warn!("/user request {req_id} error: {:#}", e.error);
   }
@@ -89,11 +119,14 @@ const RECENTLY_VIEWED_MAX: usize = 10;
 impl Resolve<UserArgs> for PushRecentlyViewed {
   async fn resolve(
     self,
-    UserArgs { user }: &UserArgs,
+    UserArgs { user, .. }: &UserArgs,
   ) -> serror::Result<PushRecentlyViewedResponse> {
     let user = get_user(&user.id).await?;
 
     let (resource_type, id) = self.resource.extract_variant_id();
+
+    let field = format!("recents.{resource_type}");
+
     let update = match user.recents.get(&resource_type) {
       Some(recents) => {
         let mut recents = recents
@@ -101,13 +134,16 @@ impl Resolve<UserArgs> for PushRecentlyViewed {
           .filter(|_id| !id.eq(*_id))
           .take(RECENTLY_VIEWED_MAX - 1)
           .collect::<VecDeque<_>>();
+
         recents.push_front(id);
-        doc! { format!("recents.{resource_type}"): to_bson(&recents)? }
+
+        doc! { &field: to_bson(&recents)? }
       }
       None => {
-        doc! { format!("recents.{resource_type}"): [id] }
+        doc! { &field: [id] }
       }
     };
+
     update_one_by_id(
       &db_client().users,
       &user.id,
@@ -115,9 +151,7 @@ impl Resolve<UserArgs> for PushRecentlyViewed {
       None,
     )
     .await
-    .with_context(|| {
-      format!("failed to update recents.{resource_type}")
-    })?;
+    .with_context(|| format!("Failed to update user '{field}'"))?;
 
     Ok(PushRecentlyViewedResponse {})
   }
@@ -126,7 +160,7 @@ impl Resolve<UserArgs> for PushRecentlyViewed {
 impl Resolve<UserArgs> for SetLastSeenUpdate {
   async fn resolve(
     self,
-    UserArgs { user }: &UserArgs,
+    UserArgs { user, .. }: &UserArgs,
   ) -> serror::Result<SetLastSeenUpdateResponse> {
     update_one_by_id(
       &db_client().users,
@@ -137,7 +171,8 @@ impl Resolve<UserArgs> for SetLastSeenUpdate {
       None,
     )
     .await
-    .context("failed to update user last_update_view")?;
+    .context("Failed to update user 'last_update_view'")?;
+
     Ok(SetLastSeenUpdateResponse {})
   }
 }
@@ -153,14 +188,17 @@ impl Resolve<UserArgs> for CreateApiKey {
   )]
   async fn resolve(
     self,
-    UserArgs { user }: &UserArgs,
+    UserArgs { user, .. }: &UserArgs,
   ) -> serror::Result<CreateApiKeyResponse> {
     let user = get_user(&user.id).await?;
 
+    validate_api_key_name(&self.name)
+      .status_code(StatusCode::BAD_REQUEST)?;
+
     let key = format!("K-{}", random_string(SECRET_LENGTH));
     let secret = format!("S-{}", random_string(SECRET_LENGTH));
     let secret_hash = bcrypt::hash(&secret, BCRYPT_COST)
-      .context("failed at hashing secret string")?;
+      .context("Failed at hashing secret string")?;
 
     let api_key = ApiKey {
       name: self.name,
@@ -170,11 +208,13 @@ impl Resolve<UserArgs> for CreateApiKey {
       created_at: komodo_timestamp(),
       expires: self.expires,
     };
+
     db_client()
       .api_keys
       .insert_one(api_key)
       .await
-      .context("failed to create api key on db")?;
+      .context("Failed to create api key on database")?;
+
     Ok(CreateApiKeyResponse { key, secret })
   }
 }
@@ -187,23 +227,297 @@ impl Resolve<UserArgs> for DeleteApiKey {
   )]
   async fn resolve(
     self,
-    UserArgs { user }: &UserArgs,
+    UserArgs { user, .. }: &UserArgs,
   ) -> serror::Result<DeleteApiKeyResponse> {
     let client = db_client();
+
     let key = client
       .api_keys
       .find_one(doc! { "key": &self.key })
       .await
-      .context("failed at db query")?
-      .context("no api key with key found")?;
+      .context("Failed at database query")?
+      .context("No api key with key found")?;
+
     if user.id != key.user_id {
-      return Err(anyhow!("api key does not belong to user").into());
+      return Err(
+        anyhow!("Api key does not belong to user")
+          .status_code(StatusCode::FORBIDDEN),
+      );
     }
+
     client
       .api_keys
       .delete_one(doc! { "key": key.key })
       .await
-      .context("failed to delete api key from db")?;
+      .context("Failed to delete api key from database")?;
+
     Ok(DeleteApiKeyResponse {})
   }
 }
+
+const TOTP_ENROLLMENT_SECRET_LENGTH: usize = 20;
+
+impl Resolve<UserArgs> for BeginTotpEnrollment {
+  #[instrument(
+    "BeginTotpEnrollment",
+    skip_all,
+    fields(operator = user.id)
+  )]
+  async fn resolve(
+    self,
+    UserArgs { user, session }: &UserArgs,
+  ) -> serror::Result<BeginTotpEnrollmentResponse> {
+    for locked_username in &core_config().lock_login_credentials_for {
+      if *locked_username == user.username {
+        return Err(
+          anyhow!("User not allowed to enroll in TOTP 2FA.").into(),
+        );
+      }
+    }
+
+    let session = session.as_ref().context(
+      "Method called in invalid context. This should not happen",
+    )?;
+
+    let secret_bytes = random_bytes(TOTP_ENROLLMENT_SECRET_LENGTH);
+    let totp = make_totp(secret_bytes.clone(), user.id.clone())?;
+    let png = totp
+      .get_qr_base64()
+      .map_err(|e| anyhow::Error::msg(e))
+      .context("Failed to generate QR code png")?;
+    session
+      .insert(SESSION_KEY_TOTP_ENROLLMENT, secret_bytes)
+      .await?;
+
+    Ok(BeginTotpEnrollmentResponse {
+      uri: totp.get_url(),
+      png,
+    })
+  }
+}
+
+impl Resolve<UserArgs> for ConfirmTotpEnrollment {
+  #[instrument(
+    "ConfirmTotpEnrollment",
+    skip_all,
+    fields(operator = user.id)
+  )]
+  async fn resolve(
+    self,
+    UserArgs { user, session }: &UserArgs,
+  ) -> serror::Result<ConfirmTotpEnrollmentResponse> {
+    let session = session.as_ref().context(
+      "Method called in invalid context. This should not happen",
+    )?;
+
+    let secret_bytes = session
+      .remove::<Vec<u8>>(SESSION_KEY_TOTP_ENROLLMENT)
+      .await
+      .context("Totp enrollment was not initiated correctly")?
+      .context(
+        "Totp enrollment was not initiated correctly or timed out",
+      )?;
+
+    let encoded_secret = BASE32_NOPAD.encode(&secret_bytes);
+
+    let totp = make_totp(secret_bytes, None)?;
+
+    let valid = totp
+      .check_current(&self.code)
+      .context("Failed to check code validity")?;
+
+    if !valid {
+      return Err(anyhow!(
+        "The provided code was not valid. Please try BeginTotpEnrollment flow again."
+      ).status_code(StatusCode::BAD_REQUEST));
+    }
+
+    let recovery_codes =
+      (0..10).map(|_| random_string(20)).collect::<Vec<_>>();
+    let hashed_recovery_codes = recovery_codes
+      .iter()
+      .map(|code| hash_password(code))
+      .collect::<anyhow::Result<Vec<_>>>()
+      .context("Failed to generate valid recovery codes")?;
+
+    update_one_by_id(
+      &db_client().users,
+      &user.id,
+      doc! {
+        "$set": {
+          "totp.secret": encoded_secret,
+          "totp.confirmed_at": komodo_timestamp(),
+          "totp.recovery_codes": hashed_recovery_codes,
+        }
+      },
+      None,
+    )
+    .await
+    .context("Failed to update user totp fields on database")?;
+
+    Ok(ConfirmTotpEnrollmentResponse { recovery_codes })
+  }
+}
+
+impl Resolve<UserArgs> for UnenrollTotp {
+  #[instrument(
+    "UnenrollTotp",
+    skip_all,
+    fields(operator = user.id)
+  )]
+  async fn resolve(
+    self,
+    UserArgs { user, .. }: &UserArgs,
+  ) -> serror::Result<UnenrollTotpResponse> {
+    update_one_by_id(
+      &db_client().users,
+      &user.id,
+      doc! {
+        "$set": {
+          "totp.secret": "",
+          "totp.confirmed_at": 0,
+          "totp.recovery_codes": [],
+        }
+      },
+      None,
+    )
+    .await
+    .context("Failed to clear user totp fields on database")?;
+    Ok(UnenrollTotpResponse {})
+  }
+}
+
+//
+
+impl Resolve<UserArgs> for BeginPasskeyEnrollment {
+  #[instrument(
+    "BeginPasskeyEnrollment",
+    skip_all,
+    fields(operator = user.id)
+  )]
+  async fn resolve(
+    self,
+    UserArgs { user, session }: &UserArgs,
+  ) -> serror::Result<BeginPasskeyEnrollmentResponse> {
+    for locked_username in &core_config().lock_login_credentials_for {
+      if *locked_username == user.username {
+        return Err(
+          anyhow!(
+            "User not allowed to enroll in Passkey authentication."
+          )
+          .into(),
+        );
+      }
+    }
+
+    let session = session.as_ref().context(
+      "Method called in invalid context. This should not happen",
+    )?;
+
+    let webauthn = webauthn().context(
+      "No webauthn provider available, invalid KOMODO_HOST config",
+    )?;
+
+    // Get two parts from this, the first is returned to the client.
+    // The second must stay server side and is used in confirmation flow.
+    let (challenge, server_state) = webauthn
+      .start_passkey_registration(
+        Uuid::new_v4(),
+        &user.username,
+        &user.username,
+        None,
+      )?;
+
+    session
+      .insert(
+        SESSION_KEY_PASSKEY_ENROLLMENT,
+        (&user.id, server_state),
+      )
+      .await
+      .context(
+        "Failed to store passkey enrollment state in server side client session",
+      )?;
+
+    Ok(challenge.into())
+  }
+}
+
+//
+
+impl Resolve<UserArgs> for ConfirmPasskeyEnrollment {
+  #[instrument(
+    "ConfirmPasskeyEnrollment",
+    skip_all,
+    fields(operator = user.id)
+  )]
+  async fn resolve(
+    self,
+    UserArgs { user, session }: &UserArgs,
+  ) -> serror::Result<ConfirmPasskeyEnrollmentResponse> {
+    let session = session.as_ref().context(
+      "Method called in invalid context. This should not happen",
+    )?;
+
+    let webauthn = webauthn().context(
+      "No webauthn provider available, invalid KOMODO_HOST config",
+    )?;
+
+    let (user_id, server_state) = session
+      .remove::<(String, PasskeyRegistration)>(
+        SESSION_KEY_PASSKEY_ENROLLMENT,
+      )
+      .await
+      .context("Passkey enrollment was not initiated correctly")?
+      .context(
+        "Passkey enrollment was not initiated correctly or timed out",
+      )?;
+
+    let passkey = webauthn
+      .finish_passkey_registration(
+        &self.credential.into(),
+        &server_state,
+      )
+      .context("Failed to finish passkey registration")?;
+
+    let passkey = to_bson(&passkey)
+      .context("Failed to serialize passkey to BSON")?;
+
+    let update = doc! {
+      "$set": {
+        "passkey.passkey": passkey,
+        "passkey.created_at": komodo_timestamp()
+      }
+    };
+
+    update_one_by_id(&db_client().users, &user_id, update, None)
+      .await
+      .context("Failed to update user passkey options on database")?;
+
+    Ok(ConfirmPasskeyEnrollmentResponse {})
+  }
+}
+
+//
+
+impl Resolve<UserArgs> for UnenrollPasskey {
+  #[instrument(
+    "UnenrollPasskey",
+    skip_all,
+    fields(operator = user.id)
+  )]
+  async fn resolve(
+    self,
+    UserArgs { user, .. }: &UserArgs,
+  ) -> serror::Result<UnenrollPasskeyResponse> {
+    let update = doc! {
+      "$set": {
+        "passkey.passkey": null,
+        "passkey.created_at": 0
+      }
+    };
+    update_one_by_id(&db_client().users, &user.id, update, None)
+      .await
+      .context("Failed to update user passkey options on database")?;
+    Ok(UnenrollPasskeyResponse {})
+  }
+}

bin/core/src/api/write/service_user.rs

@@ -1,10 +1,5 @@
-use std::str::FromStr;
-
 use anyhow::{Context, anyhow};
-use database::mungos::{
-  by_id::find_one_by_id,
-  mongodb::bson::{doc, oid::ObjectId},
-};
+use database::mungos::{by_id::find_one_by_id, mongodb::bson::doc};
 use komodo_client::{
   api::{user::CreateApiKey, write::*},
   entities::{
@@ -12,9 +7,15 @@ use komodo_client::{
     user::{User, UserConfig},
   },
 };
+use reqwest::StatusCode;
 use resolver_api::Resolve;
+use serror::{AddStatusCode as _, AddStatusCodeError as _};
 
-use crate::{api::user::UserArgs, state::db_client};
+use crate::{
+  api::user::UserArgs,
+  helpers::validations::{validate_api_key_name, validate_username},
+  state::db_client,
+};
 
 use super::WriteArgs;
 
@@ -33,20 +34,25 @@ impl Resolve<WriteArgs> for CreateServiceUser {
     WriteArgs { user }: &WriteArgs,
   ) -> serror::Result<CreateServiceUserResponse> {
     if !user.admin {
-      return Err(anyhow!("user not admin").into());
-    }
-    if ObjectId::from_str(&self.username).is_ok() {
       return Err(
-        anyhow!("username cannot be valid ObjectId").into(),
+        anyhow!("Only Admins can manage Service Users")
+          .status_code(StatusCode::FORBIDDEN),
       );
     }
+
+    validate_username(&self.username)
+      .status_code(StatusCode::BAD_REQUEST)?;
+
     let config = UserConfig::Service {
       description: self.description,
     };
+
     let mut user = User {
       id: Default::default(),
       username: self.username,
       config,
+      totp: Default::default(),
+      passkey: Default::default(),
       enabled: true,
       admin: false,
       super_admin: false,
@@ -57,6 +63,7 @@ impl Resolve<WriteArgs> for CreateServiceUser {
       all: Default::default(),
       updated_at: komodo_timestamp(),
     };
+
     user.id = db_client()
       .users
       .insert_one(&user)
@@ -66,6 +73,7 @@ impl Resolve<WriteArgs> for CreateServiceUser {
       .as_object_id()
       .context("inserted id is not object id")?
       .to_string();
+
     Ok(user)
   }
 }
@@ -85,18 +93,28 @@ impl Resolve<WriteArgs> for UpdateServiceUserDescription {
     WriteArgs { user }: &WriteArgs,
   ) -> serror::Result<UpdateServiceUserDescriptionResponse> {
     if !user.admin {
-      return Err(anyhow!("user not admin").into());
+      return Err(
+        anyhow!("Only Admins can manage Service Users")
+          .status_code(StatusCode::FORBIDDEN),
+      );
     }
+
     let db = db_client();
+
     let service_user = db
       .users
       .find_one(doc! { "username": &self.username })
       .await
-      .context("failed to query db for user")?
-      .context("no user with given username")?;
+      .context("Failed to query db for user")?
+      .context("No user with given username")?;
+
     let UserConfig::Service { .. } = &service_user.config else {
-      return Err(anyhow!("user is not service user").into());
+      return Err(
+        anyhow!("Target user is not Service User")
+          .status_code(StatusCode::FORBIDDEN),
+      );
     };
+
     db.users
       .update_one(
         doc! { "username": &self.username },
@@ -104,13 +122,15 @@ impl Resolve<WriteArgs> for UpdateServiceUserDescription {
       )
       .await
       .context("failed to update user on db")?;
-    let res = db
+
+    let service_user = db
       .users
       .find_one(doc! { "username": &self.username })
       .await
       .context("failed to query db for user")?
       .context("user with username not found")?;
-    Ok(res)
+
+    Ok(service_user)
   }
 }
 
@@ -130,21 +150,36 @@ impl Resolve<WriteArgs> for CreateApiKeyForServiceUser {
     WriteArgs { user }: &WriteArgs,
   ) -> serror::Result<CreateApiKeyForServiceUserResponse> {
     if !user.admin {
-      return Err(anyhow!("user not admin").into());
+      return Err(
+        anyhow!("Only Admins can manage Service Users")
+          .status_code(StatusCode::FORBIDDEN),
+      );
     }
+
+    validate_api_key_name(&self.name)
+      .status_code(StatusCode::BAD_REQUEST)?;
+
     let service_user =
       find_one_by_id(&db_client().users, &self.user_id)
         .await
-        .context("failed to query db for user")?
-        .context("no user found with id")?;
+        .context("Failed to query db for user")?
+        .context("No user found with id")?;
+
     let UserConfig::Service { .. } = &service_user.config else {
-      return Err(anyhow!("user is not service user").into());
+      return Err(
+        anyhow!("Target user is not Service User")
+          .status_code(StatusCode::FORBIDDEN),
+      );
     };
+
     CreateApiKey {
       name: self.name,
       expires: self.expires,
     }
-    .resolve(&UserArgs { user: service_user })
+    .resolve(&UserArgs {
+      user: service_user,
+      session: None,
+    })
     .await
   }
 }
@@ -163,23 +198,34 @@ impl Resolve<WriteArgs> for DeleteApiKeyForServiceUser {
     WriteArgs { user }: &WriteArgs,
   ) -> serror::Result<DeleteApiKeyForServiceUserResponse> {
     if !user.admin {
-      return Err(anyhow!("user not admin").into());
+      return Err(
+        anyhow!("Only Admins can manage Service Users")
+          .status_code(StatusCode::FORBIDDEN),
+      );
     }
+
     let db = db_client();
+
     let api_key = db
       .api_keys
       .find_one(doc! { "key": &self.key })
       .await
       .context("failed to query db for api key")?
       .context("did not find matching api key")?;
+
     let service_user =
       find_one_by_id(&db_client().users, &api_key.user_id)
         .await
         .context("failed to query db for user")?
         .context("no user found with id")?;
+
     let UserConfig::Service { .. } = &service_user.config else {
-      return Err(anyhow!("user is not service user").into());
+      return Err(
+        anyhow!("Target user is not Service User")
+          .status_code(StatusCode::FORBIDDEN),
+      );
     };
+
     db.api_keys
       .delete_one(doc! { "key": self.key })
       .await

bin/core/src/api/write/stack.rs

@@ -10,7 +10,6 @@ use komodo_client::{
     all_logs_success,
     permission::PermissionLevel,
     repo::Repo,
-    server::ServerState,
     stack::{Stack, StackInfo},
     update::Update,
     user::stack_user,
@@ -25,9 +24,8 @@ use resolver_api::Resolve;
 use crate::{
   config::core_config,
   helpers::{
-    periphery_client,
-    query::get_server_with_state,
-    stack_git_token,
+    query::get_swarm_or_server,
+    stack_git_token, swarm_or_server_request,
     update::{add_update, make_update},
   },
   permission::get_check_permissions,
@@ -204,32 +202,24 @@ async fn write_stack_file_contents_on_host(
   contents: String,
   mut update: Update,
 ) -> serror::Result<Update> {
-  if stack.config.server_id.is_empty() {
-    return Err(anyhow!(
-      "Cannot write file, Files on host Stack has not configured a Server"
-    ).into());
-  }
-  let (server, state) =
-    get_server_with_state(&stack.config.server_id).await?;
-  if state != ServerState::Ok {
-    return Err(
-      anyhow!(
-        "Cannot write file when server is unreachable or disabled"
-      )
-      .into(),
-    );
-  }
-  match periphery_client(&server)
-    .await?
-    .request(WriteComposeContentsToHost {
+  let swarm_or_server = get_swarm_or_server(
+    &stack.config.swarm_id,
+    &stack.config.server_id,
+  )
+  .await?;
+
+  let res = swarm_or_server_request(
+    &swarm_or_server,
+    WriteComposeContentsToHost {
       name: stack.name,
       run_directory: stack.config.run_directory,
       file_path,
       contents,
-    })
-    .await
-    .context("Failed to write contents to host")
-  {
+    },
+  )
+  .await;
+
+  match res {
     Ok(log) => {
       update.logs.push(log);
     }
@@ -239,7 +229,7 @@ async fn write_stack_file_contents_on_host(
         format_serror(&e.into()),
       );
     }
-  };
+  }
 
   if !all_logs_success(&update.logs) {
     update.finalize();
@@ -459,26 +449,22 @@ impl Resolve<WriteArgs> for RefreshStackCache {
       // =============
       // FILES ON HOST
       // =============
-      let (server, state) = if stack.config.server_id.is_empty() {
-        (None, ServerState::Disabled)
-      } else {
-        let (server, state) =
-          get_server_with_state(&stack.config.server_id).await?;
-        (Some(server), state)
-      };
-      if state != ServerState::Ok {
-        (vec![], None, None, None, None)
-      } else if let Some(server) = server {
+      if let Ok(swarm_or_server) = get_swarm_or_server(
+        &stack.config.swarm_id,
+        &stack.config.server_id,
+      )
+      .await
+      {
         let GetComposeContentsOnHostResponse { contents, errors } =
-          match periphery_client(&server)
-            .await?
-            .request(GetComposeContentsOnHost {
+          match swarm_or_server_request(
+            &swarm_or_server,
+            GetComposeContentsOnHost {
               file_paths: stack.all_file_dependencies(),
               name: stack.name.clone(),
               run_directory: stack.config.run_directory.clone(),
-            })
-            .await
-            .context("failed to get compose file contents from host")
+            },
+          )
+          .await
           {
             Ok(res) => res,
             Err(e) => GetComposeContentsOnHostResponse {
@@ -489,7 +475,6 @@ impl Resolve<WriteArgs> for RefreshStackCache {
               }],
             },
           };
-
         let project_name = stack.project_name(true);
 
         let mut services = Vec::new();

bin/core/src/api/write/user.rs

@@ -15,9 +15,13 @@ use komodo_client::{
 };
 use reqwest::StatusCode;
 use resolver_api::Resolve;
-use serror::AddStatusCodeError;
+use serror::{AddStatusCode as _, AddStatusCodeError};
 
-use crate::{config::core_config, state::db_client};
+use crate::{
+  config::core_config,
+  helpers::validations::{validate_password, validate_username},
+  state::db_client,
+};
 
 use super::WriteArgs;
 
@@ -38,24 +42,15 @@ impl Resolve<WriteArgs> for CreateLocalUser {
   ) -> serror::Result<CreateLocalUserResponse> {
     if !admin.admin {
       return Err(
-        anyhow!("This method is admin-only.")
+        anyhow!("This method is Admin Only.")
           .status_code(StatusCode::FORBIDDEN),
       );
     }
 
-    if self.username.is_empty() {
-      return Err(anyhow!("Username cannot be empty.").into());
-    }
-
-    if ObjectId::from_str(&self.username).is_ok() {
-      return Err(
-        anyhow!("Username cannot be valid ObjectId").into(),
-      );
-    }
-
-    if self.password.is_empty() {
-      return Err(anyhow!("Password cannot be empty.").into());
-    }
+    validate_username(&self.username)
+      .status_code(StatusCode::BAD_REQUEST)?;
+    validate_password(&self.password)
+      .status_code(StatusCode::BAD_REQUEST)?;
 
     let db = db_client();
 
@@ -87,6 +82,8 @@ impl Resolve<WriteArgs> for CreateLocalUser {
       config: UserConfig::Local {
         password: hashed_password,
       },
+      totp: Default::default(),
+      passkey: Default::default(),
     };
 
     user.id = db_client()
@@ -130,17 +127,11 @@ impl Resolve<WriteArgs> for UpdateUserUsername {
         );
       }
     }
-    if self.username.is_empty() {
-      return Err(anyhow!("Username cannot be empty.").into());
-    }
 
-    if ObjectId::from_str(&self.username).is_ok() {
-      return Err(
-        anyhow!("Username cannot be valid ObjectId").into(),
-      );
-    }
+    validate_username(&self.username)?;
 
     let db = db_client();
+
     if db
       .users
       .find_one(doc! { "username": &self.username })
@@ -150,8 +141,10 @@ impl Resolve<WriteArgs> for UpdateUserUsername {
     {
       return Err(anyhow!("Username already taken.").into());
     }
+
     let id = ObjectId::from_str(&user.id)
       .context("User id not valid ObjectId.")?;
+
     db.users
       .update_one(
         doc! { "_id": id },
@@ -159,6 +152,7 @@ impl Resolve<WriteArgs> for UpdateUserUsername {
       )
       .await
       .context("Failed to update user username on database.")?;
+
     Ok(NoData {})
   }
 }
@@ -185,7 +179,12 @@ impl Resolve<WriteArgs> for UpdateUserPassword {
         );
       }
     }
+
+    validate_password(&self.password)
+      .status_code(StatusCode::BAD_REQUEST)?;
+
     db_client().set_user_password(user, &self.password).await?;
+
     Ok(NoData {})
   }
 }
@@ -211,15 +210,19 @@ impl Resolve<WriteArgs> for DeleteUser {
           .status_code(StatusCode::FORBIDDEN),
       );
     }
+
     if admin.username == self.user || admin.id == self.user {
       return Err(anyhow!("User cannot delete themselves.").into());
     }
+
     let query = if let Ok(id) = ObjectId::from_str(&self.user) {
       doc! { "_id": id }
     } else {
       doc! { "username": self.user }
     };
+
     let db = db_client();
+
     let Some(user) = db
       .users
       .find_one(query.clone())
@@ -230,21 +233,25 @@ impl Resolve<WriteArgs> for DeleteUser {
         anyhow!("No user found with given id / username").into(),
       );
     };
+
     if user.super_admin {
       return Err(
         anyhow!("Cannot delete a super admin user.").into(),
       );
     }
+
     if user.admin && !admin.super_admin {
       return Err(
         anyhow!("Only a Super Admin can delete an admin user.")
           .into(),
       );
     }
+
     db.users
       .delete_one(query)
       .await
       .context("Failed to delete user from database")?;
+
     // Also remove user id from all user groups
     if let Err(e) = db
       .user_groups
@@ -253,6 +260,7 @@ impl Resolve<WriteArgs> for DeleteUser {
     {
       warn!("Failed to remove deleted user from user groups | {e:?}");
     };
+
     Ok(user)
   }
 }

bin/core/src/api/write/variable.rs

@@ -6,12 +6,13 @@ use komodo_client::{
 };
 use reqwest::StatusCode;
 use resolver_api::Resolve;
-use serror::AddStatusCodeError;
+use serror::{AddStatusCode as _, AddStatusCodeError};
 
 use crate::{
   helpers::{
     query::get_variable,
     update::{add_update, make_update},
+    validations::{validate_variable_name, validate_variable_value},
   },
   state::db_client,
 };
@@ -35,7 +36,7 @@ impl Resolve<WriteArgs> for CreateVariable {
   ) -> serror::Result<CreateVariableResponse> {
     if !user.admin {
       return Err(
-        anyhow!("Only admins can create variables")
+        anyhow!("Only Admins can create Variables")
           .status_code(StatusCode::FORBIDDEN),
       );
     }
@@ -47,6 +48,11 @@ impl Resolve<WriteArgs> for CreateVariable {
       is_secret,
     } = self;
 
+    validate_variable_name(&name)
+      .status_code(StatusCode::BAD_REQUEST)?;
+    validate_variable_value(&value)
+      .status_code(StatusCode::BAD_REQUEST)?;
+
     let variable = Variable {
       name,
       value,
@@ -58,7 +64,7 @@ impl Resolve<WriteArgs> for CreateVariable {
       .variables
       .insert_one(&variable)
       .await
-      .context("Failed to create variable on db")?;
+      .context("Failed to create Variable on db")?;
 
     let mut update = make_update(
       ResourceTarget::system(),
@@ -67,7 +73,8 @@ impl Resolve<WriteArgs> for CreateVariable {
     );
 
     update
-      .push_simple_log("create variable", format!("{variable:#?}"));
+      .push_simple_log("Create Variable", format!("{variable:#?}"));
+
     update.finalize();
 
     add_update(update).await?;
@@ -91,13 +98,18 @@ impl Resolve<WriteArgs> for UpdateVariableValue {
   ) -> serror::Result<UpdateVariableValueResponse> {
     if !user.admin {
       return Err(
-        anyhow!("Only admins can update variables")
+        anyhow!("Only Admins can update Variables")
           .status_code(StatusCode::FORBIDDEN),
       );
     }
 
     let UpdateVariableValue { name, value } = self;
 
+    validate_variable_name(&name)
+      .status_code(StatusCode::BAD_REQUEST)?;
+    validate_variable_value(&value)
+      .status_code(StatusCode::BAD_REQUEST)?;
+
     let variable = get_variable(&name).await?;
 
     if value == variable.value {
@@ -156,10 +168,11 @@ impl Resolve<WriteArgs> for UpdateVariableDescription {
   ) -> serror::Result<UpdateVariableDescriptionResponse> {
     if !user.admin {
       return Err(
-        anyhow!("Only admins can update variables")
+        anyhow!("Only Admins can update Variables")
           .status_code(StatusCode::FORBIDDEN),
       );
     }
+
     db_client()
       .variables
       .update_one(
@@ -168,6 +181,7 @@ impl Resolve<WriteArgs> for UpdateVariableDescription {
       )
       .await
       .context("Failed to update variable description on db")?;
+
     Ok(get_variable(&self.name).await?)
   }
 }
@@ -188,10 +202,11 @@ impl Resolve<WriteArgs> for UpdateVariableIsSecret {
   ) -> serror::Result<UpdateVariableIsSecretResponse> {
     if !user.admin {
       return Err(
-        anyhow!("Only admins can update variables")
+        anyhow!("Only Admins can update Variables")
           .status_code(StatusCode::FORBIDDEN),
       );
     }
+
     db_client()
       .variables
       .update_one(
@@ -199,7 +214,8 @@ impl Resolve<WriteArgs> for UpdateVariableIsSecret {
         doc! { "$set": { "is_secret": self.is_secret } },
       )
       .await
-      .context("Failed to update variable is secret on db")?;
+      .context("Failed to update Variable 'is_secret' on db")?;
+
     Ok(get_variable(&self.name).await?)
   }
 }
@@ -219,16 +235,18 @@ impl Resolve<WriteArgs> for DeleteVariable {
   ) -> serror::Result<DeleteVariableResponse> {
     if !user.admin {
       return Err(
-        anyhow!("Only admins can delete variables")
+        anyhow!("Only Admins can delete Variables")
           .status_code(StatusCode::FORBIDDEN),
       );
     }
+
     let variable = get_variable(&self.name).await?;
+
     db_client()
       .variables
       .delete_one(doc! { "name": &self.name })
       .await
-      .context("Failed to delete variable on db")?;
+      .context("Failed to delete Variable on db")?;
 
     let mut update = make_update(
       ResourceTarget::system(),

bin/core/src/api/ws/mod.rs

@@ -0,0 +1,101 @@
+use std::net::IpAddr;
+
+use crate::{
+  auth::{auth_api_key_check_enabled, auth_jwt_check_enabled},
+  helpers::query::get_user,
+  state::auth_rate_limiter,
+};
+use anyhow::{Context, anyhow};
+use axum::{
+  Router,
+  extract::ws::{self, WebSocket},
+  http::HeaderMap,
+  routing::get,
+};
+use komodo_client::{entities::user::User, ws::WsLoginMessage};
+use rate_limit::WithFailureRateLimit;
+use reqwest::StatusCode;
+use serror::{AddStatusCode, AddStatusCodeError};
+
+mod terminal;
+mod update;
+
+pub fn router() -> Router {
+  Router::new()
+    // Periphery facing
+    .route("/periphery", get(crate::connection::server::handler))
+    // User facing
+    .route("/update", get(update::handler))
+    .route("/terminal", get(terminal::handler))
+}
+
+async fn user_ws_login(
+  mut socket: WebSocket,
+  headers: &HeaderMap,
+  fallback_ip: IpAddr,
+) -> Option<(WebSocket, User)> {
+  let res = async {
+    let message = match socket
+      .recv()
+      .await
+      .context("Failed to receive message over socket: Closed")
+      .status_code(StatusCode::BAD_REQUEST)?
+      .context("Failed to recieve message over socket: Error")
+      .status_code(StatusCode::BAD_REQUEST)?
+    {
+      ws::Message::Text(utf8_bytes) => utf8_bytes.to_string(),
+      ws::Message::Binary(bytes) => String::from_utf8(bytes.into())
+        .context("Received invalid message bytes: Not UTF-8")
+        .status_code(StatusCode::BAD_REQUEST)?,
+      message => {
+        return Err(
+          anyhow!("Received invalid message: {message:?}")
+            .status_code(StatusCode::BAD_REQUEST),
+        );
+      }
+    };
+
+    match WsLoginMessage::from_json_str(&message)
+      .context("Invalid login message")
+      .status_code(StatusCode::BAD_REQUEST)?
+    {
+      WsLoginMessage::Jwt { jwt } => auth_jwt_check_enabled(&jwt)
+        .await
+        .status_code(StatusCode::UNAUTHORIZED),
+      WsLoginMessage::ApiKeys { key, secret } => {
+        auth_api_key_check_enabled(&key, &secret)
+          .await
+          .status_code(StatusCode::UNAUTHORIZED)
+      }
+    }
+  }
+  .with_failure_rate_limit_using_headers(
+    auth_rate_limiter(),
+    headers,
+    Some(fallback_ip),
+  )
+  .await;
+  match res {
+    Ok(user) => {
+      let _ = socket.send(ws::Message::text("LOGGED_IN")).await;
+      Some((socket, user))
+    }
+    Err(e) => {
+      let _ = socket
+        .send(ws::Message::text(format!(
+          "[{}]: {:#}",
+          e.status, e.error
+        )))
+        .await;
+      None
+    }
+  }
+}
+
+async fn check_user_valid(user_id: &str) -> anyhow::Result<User> {
+  let user = get_user(user_id).await?;
+  if !user.enabled {
+    return Err(anyhow!("User not enabled"));
+  }
+  Ok(user)
+}

bin/core/src/api/ws/terminal.rs

@@ -1,7 +1,9 @@
+use std::net::SocketAddr;
+
 use anyhow::anyhow;
 use axum::{
-  extract::{FromRequestParts, WebSocketUpgrade, ws},
-  http::request,
+  extract::{ConnectInfo, FromRequestParts, WebSocketUpgrade, ws},
+  http::{HeaderMap, request},
   response::IntoResponse,
 };
 use bytes::Bytes;
@@ -22,11 +24,14 @@ use crate::{
 #[instrument("ConnectTerminal", skip(ws))]
 pub async fn handler(
   Qs(query): Qs<ConnectTerminalQuery>,
+  ConnectInfo(info): ConnectInfo<SocketAddr>,
+  headers: HeaderMap,
   ws: WebSocketUpgrade,
 ) -> impl IntoResponse {
-  ws.on_upgrade(|socket| async move {
+  let ip = info.ip();
+  ws.on_upgrade(move |socket| async move {
     let Some((mut client_socket, user)) =
-      super::user_ws_login(socket).await
+      super::user_ws_login(socket, &headers, ip).await
     else {
       return;
     };

bin/core/src/api/ws/update.rs

@@ -1,6 +1,9 @@
+use std::net::SocketAddr;
+
 use anyhow::anyhow;
 use axum::{
-  extract::{WebSocketUpgrade, ws::Message},
+  extract::{ConnectInfo, WebSocketUpgrade, ws::Message},
+  http::HeaderMap,
   response::IntoResponse,
 };
 use futures_util::{SinkExt, StreamExt};
@@ -16,17 +19,24 @@ use crate::helpers::{
   channel::update_channel, query::get_user_permission_on_target,
 };
 
-pub async fn handler(ws: WebSocketUpgrade) -> impl IntoResponse {
+pub async fn handler(
+  headers: HeaderMap,
+  ConnectInfo(info): ConnectInfo<SocketAddr>,
+  ws: WebSocketUpgrade,
+) -> impl IntoResponse {
   // get a reveiver for internal update messages.
   let mut receiver = update_channel().receiver.resubscribe();
+  let ip = info.ip();
 
   // handle http -> ws updgrade
-  ws.on_upgrade(|socket| async move {
-    let Some((socket, user)) = super::user_ws_login(socket).await else {
-      return
+  ws.on_upgrade(move |socket| async move {
+    let Some((client_socket, user)) =
+      super::user_ws_login(socket, &headers, ip).await
+    else {
+      return;
     };
 
-    let (mut ws_sender, mut ws_reciever) = socket.split();
+    let (mut ws_sender, mut ws_reciever) = client_socket.split();
 
     let cancel = CancellationToken::new();
     let cancel_clone = cancel.clone();

bin/core/src/auth/github/mod.rs

@@ -1,20 +1,37 @@
+use std::net::SocketAddr;
+
 use anyhow::{Context, anyhow};
 use axum::{
-  Router, extract::Query, response::Redirect, routing::get,
+  Router,
+  extract::{ConnectInfo, Query},
+  http::HeaderMap,
+  response::Redirect,
+  routing::get,
 };
 use database::mongo_indexed::Document;
 use database::mungos::mongodb::bson::doc;
-use komodo_client::entities::{
-  komodo_timestamp, random_string,
-  user::{User, UserConfig},
+use futures_util::TryFutureExt;
+use komodo_client::{
+  api::auth::UserIdOrTwoFactor,
+  entities::{
+    komodo_timestamp, random_string,
+    user::{User, UserConfig},
+  },
 };
+use rate_limit::WithFailureRateLimit;
 use reqwest::StatusCode;
 use serde::Deserialize;
-use serror::AddStatusCode;
+use serror::{AddStatusCode, AddStatusCodeError as _};
+use tower_sessions::Session;
 
 use crate::{
+  api::{
+    SESSION_KEY_PASSKEY_LOGIN, SESSION_KEY_TOTP_LOGIN,
+    SESSION_KEY_USER_ID,
+  },
+  auth::format_redirect,
   config::core_config,
-  state::{db_client, jwt_client},
+  state::{auth_rate_limiter, db_client, webauthn},
 };
 
 use self::client::github_oauth_client;
@@ -28,21 +45,32 @@ pub fn router() -> Router {
     .route(
       "/login",
       get(|Query(query): Query<RedirectQuery>| async {
-        Redirect::to(
-          &github_oauth_client()
-            .as_ref()
-            // OK: the router is only mounted in case that the client is populated
-            .unwrap()
-            .get_login_redirect_url(query.redirect)
-            .await,
-        )
+        let uri = github_oauth_client()
+          .as_ref()
+          .context("Github Oauth not configured")
+          .status_code(StatusCode::UNAUTHORIZED)?
+          .get_login_redirect_url(query.redirect)
+          .await;
+        serror::Result::Ok(Redirect::to(&uri))
       }),
     )
     .route(
       "/callback",
-      get(|query| async {
-        callback(query).await.status_code(StatusCode::UNAUTHORIZED)
-      }),
+      get(
+        |query,
+         session: Session,
+         headers: HeaderMap,
+         ConnectInfo(info): ConnectInfo<SocketAddr>| async move {
+          callback(query, session)
+            .map_err(|e| e.status_code(StatusCode::UNAUTHORIZED))
+            .with_failure_rate_limit_using_headers(
+              auth_rate_limiter(),
+              &headers,
+              Some(info.ip()),
+            )
+            .await
+        },
+      ),
     )
 }
 
@@ -54,6 +82,7 @@ struct CallbackQuery {
 
 async fn callback(
   Query(query): Query<CallbackQuery>,
+  session: Session,
 ) -> anyhow::Result<Redirect> {
   let client = github_oauth_client().as_ref().unwrap();
   if !client.check_state(&query.state).await {
@@ -69,10 +98,39 @@ async fn callback(
     .find_one(doc! { "config.data.github_id": &github_id })
     .await
     .context("failed at find user query from database")?;
-  let jwt = match user {
-    Some(user) => jwt_client()
-      .encode(user.id)
-      .context("failed to generate jwt")?,
+  let user_id_or_two_factor = match user {
+    Some(user) => {
+      match (user.passkey.passkey, user.totp.enrolled()) {
+        // WebAuthn Passkey 2FA
+        (Some(passkey), _) => {
+          let webauthn = webauthn().context(
+            "No webauthn provider available, invalid KOMODO_HOST config",
+          )?;
+          let (response, server_state) = webauthn
+            .start_passkey_authentication(&[passkey])
+            .context("Failed to start passkey authentication flow")?;
+          session
+            .insert(
+              SESSION_KEY_PASSKEY_LOGIN,
+              (user.id, server_state),
+            )
+            .await?;
+          UserIdOrTwoFactor::Passkey(response)
+        }
+        // TOTP 2FA
+        (None, true) => {
+          session
+            .insert(SESSION_KEY_TOTP_LOGIN, user.id)
+            .await
+            .context(
+              "Failed to store totp login state in for user session",
+            )?;
+          UserIdOrTwoFactor::Totp {}
+        }
+        // No 2FA
+        (None, false) => UserIdOrTwoFactor::UserId(user.id),
+      }
+    }
     None => {
       let ts = komodo_timestamp();
       let no_users_exist =
@@ -111,28 +169,38 @@ async fn callback(
           github_id,
           avatar: github_user.avatar_url,
         },
+        totp: Default::default(),
+        passkey: Default::default(),
       };
       let user_id = db_client
         .users
         .insert_one(user)
         .await
-        .context("failed to create user on mongo")?
+        .context("Failed to create user on mongo")?
         .inserted_id
         .as_object_id()
         .context("inserted_id is not ObjectId")?
         .to_string();
-      jwt_client()
-        .encode(user_id)
-        .context("failed to generate jwt")?
+      UserIdOrTwoFactor::UserId(user_id)
     }
   };
-  let exchange_token = jwt_client().create_exchange_token(jwt).await;
-  let redirect = &query.state[STATE_PREFIX_LENGTH..];
-  let redirect_url = if redirect.is_empty() {
-    format!("{}?token={exchange_token}", core_config().host)
-  } else {
-    let splitter = if redirect.contains('?') { '&' } else { '?' };
-    format!("{redirect}{splitter}token={exchange_token}")
-  };
-  Ok(Redirect::to(&redirect_url))
+  let redirect = Some(&query.state[STATE_PREFIX_LENGTH..]);
+  match user_id_or_two_factor {
+    UserIdOrTwoFactor::UserId(user_id) => {
+      session
+        .insert(SESSION_KEY_USER_ID, user_id)
+        .await
+        .context("Failed to store user id for client session")?;
+      Ok(format_redirect(redirect, "redeem_ready=true"))
+    }
+    UserIdOrTwoFactor::Totp {} => {
+      Ok(format_redirect(redirect, "totp=true"))
+    }
+    UserIdOrTwoFactor::Passkey(passkey) => {
+      let passkey = serde_json::to_string(&passkey)
+        .context("Failed to serialize passkey response")?;
+      let passkey = urlencoding::encode(&passkey);
+      Ok(format_redirect(redirect, &format!("passkey={passkey}")))
+    }
+  }
 }

bin/core/src/auth/google/mod.rs

@@ -1,21 +1,35 @@
+use std::net::SocketAddr;
+
 use anyhow::{Context, anyhow};
 use async_timing_util::unix_timestamp_ms;
 use axum::{
-  Router, extract::Query, response::Redirect, routing::get,
+  Router,
+  extract::{ConnectInfo, Query},
+  http::HeaderMap,
+  response::Redirect,
+  routing::get,
 };
 use database::mongo_indexed::Document;
 use database::mungos::mongodb::bson::doc;
-use komodo_client::entities::{
-  random_string,
-  user::{User, UserConfig},
+use futures_util::TryFutureExt;
+use komodo_client::{
+  api::auth::UserIdOrTwoFactor,
+  entities::{
+    random_string,
+    user::{User, UserConfig},
+  },
 };
+use rate_limit::WithFailureRateLimit;
 use reqwest::StatusCode;
 use serde::Deserialize;
-use serror::AddStatusCode;
+use serror::{AddStatusCode, AddStatusCodeError as _};
+use tower_sessions::Session;
 
 use crate::{
+  api::{SESSION_KEY_PASSKEY_LOGIN, SESSION_KEY_TOTP_LOGIN, SESSION_KEY_USER_ID},
+  auth::format_redirect,
   config::core_config,
-  state::{db_client, jwt_client},
+  state::{auth_rate_limiter, db_client, webauthn},
 };
 
 use self::client::google_oauth_client;
@@ -29,21 +43,32 @@ pub fn router() -> Router {
     .route(
       "/login",
       get(|Query(query): Query<RedirectQuery>| async move {
-        Redirect::to(
-          &google_oauth_client()
-            .as_ref()
-            // OK: its not mounted unless the client is populated
-            .unwrap()
-            .get_login_redirect_url(query.redirect)
-            .await,
-        )
+        let uri = google_oauth_client()
+          .as_ref()
+          .context("Google Oauth not configured")
+          .status_code(StatusCode::UNAUTHORIZED)?
+          .get_login_redirect_url(query.redirect)
+          .await;
+        serror::Result::Ok(Redirect::to(&uri))
       }),
     )
     .route(
       "/callback",
-      get(|query| async {
-        callback(query).await.status_code(StatusCode::UNAUTHORIZED)
-      }),
+      get(
+        |query,
+         session: Session,
+         headers: HeaderMap,
+         ConnectInfo(info): ConnectInfo<SocketAddr>| async move {
+          callback(query, session)
+            .map_err(|e| e.status_code(StatusCode::UNAUTHORIZED))
+            .with_failure_rate_limit_using_headers(
+              auth_rate_limiter(),
+              &headers,
+              Some(info.ip()),
+            )
+            .await
+        },
+      ),
     )
 }
 
@@ -56,6 +81,7 @@ struct CallbackQuery {
 
 async fn callback(
   Query(query): Query<CallbackQuery>,
+  session: Session,
 ) -> anyhow::Result<Redirect> {
   // Safe: the method is only called after the client is_some
   let client = google_oauth_client().as_ref().unwrap();
@@ -81,10 +107,39 @@ async fn callback(
     .find_one(doc! { "config.data.google_id": &google_id })
     .await
     .context("failed at find user query from mongo")?;
-  let jwt = match user {
-    Some(user) => jwt_client()
-      .encode(user.id)
-      .context("failed to generate jwt")?,
+  let user_id_or_two_factor = match user {
+    Some(user) => {
+      match (user.passkey.passkey, user.totp.enrolled()) {
+        // WebAuthn Passkey 2FA
+        (Some(passkey), _) => {
+          let webauthn = webauthn().context(
+            "No webauthn provider available, invalid KOMODO_HOST config",
+          )?;
+          let (response, server_state) = webauthn
+            .start_passkey_authentication(&[passkey])
+            .context("Failed to start passkey authentication flow")?;
+          session
+            .insert(
+              SESSION_KEY_PASSKEY_LOGIN,
+              (user.id, server_state),
+            )
+            .await?;
+          UserIdOrTwoFactor::Passkey(response)
+        }
+        // TOTP 2FA
+        (None, true) => {
+          session
+            .insert(SESSION_KEY_TOTP_LOGIN, user.id)
+            .await
+            .context(
+              "Failed to store totp login state in for user session",
+            )?;
+          UserIdOrTwoFactor::Totp {}
+        }
+        // No 2FA
+        (None, false) => UserIdOrTwoFactor::UserId(user.id),
+      }
+    }
     None => {
       let ts = unix_timestamp_ms() as i64;
       let no_users_exist =
@@ -128,28 +183,38 @@ async fn callback(
           google_id,
           avatar: google_user.picture,
         },
+        totp: Default::default(),
+        passkey: Default::default(),
       };
       let user_id = db_client
         .users
         .insert_one(user)
         .await
-        .context("failed to create user on mongo")?
+        .context("Failed to create user on mongo")?
         .inserted_id
         .as_object_id()
         .context("inserted_id is not ObjectId")?
         .to_string();
-      jwt_client()
-        .encode(user_id)
-        .context("failed to generate jwt")?
+      UserIdOrTwoFactor::UserId(user_id)
     }
   };
-  let exchange_token = jwt_client().create_exchange_token(jwt).await;
-  let redirect = &state[STATE_PREFIX_LENGTH..];
-  let redirect_url = if redirect.is_empty() {
-    format!("{}?token={exchange_token}", core_config().host)
-  } else {
-    let splitter = if redirect.contains('?') { '&' } else { '?' };
-    format!("{redirect}{splitter}token={exchange_token}")
-  };
-  Ok(Redirect::to(&redirect_url))
+  let redirect = Some(&state[STATE_PREFIX_LENGTH..]);
+  match user_id_or_two_factor {
+    UserIdOrTwoFactor::UserId(user_id) => {
+      session
+        .insert(SESSION_KEY_USER_ID, user_id)
+        .await
+        .context("Failed to store user id for client session")?;
+      Ok(format_redirect(redirect, "redeem_ready=true"))
+    }
+    UserIdOrTwoFactor::Totp {} => {
+      Ok(format_redirect(redirect, "totp=true"))
+    }
+    UserIdOrTwoFactor::Passkey(passkey) => {
+      let passkey = serde_json::to_string(&passkey)
+        .context("Failed to serialize passkey response")?;
+      let passkey = urlencoding::encode(&passkey);
+      Ok(format_redirect(redirect, &format!("passkey={passkey}")))
+    }
+  }
 }

bin/core/src/auth/jwt.rs

@@ -1,9 +1,5 @@
-use std::collections::HashMap;
-
-use anyhow::{Context, anyhow};
-use async_timing_util::{
-  Timelength, get_timelength_in_ms, unix_timestamp_ms,
-};
+use anyhow::Context;
+use async_timing_util::{get_timelength_in_ms, unix_timestamp_ms};
 use database::mungos::mongodb::bson::doc;
 use jsonwebtoken::{
   DecodingKey, EncodingKey, Header, Validation, decode, encode,
@@ -13,9 +9,6 @@ use komodo_client::{
   entities::{config::core::CoreConfig, random_string},
 };
 use serde::{Deserialize, Serialize};
-use tokio::sync::Mutex;
-
-type ExchangeTokenMap = Mutex<HashMap<String, (JwtResponse, u128)>>;
 
 #[derive(Serialize, Deserialize, Clone)]
 pub struct JwtClaims {
@@ -30,7 +23,6 @@ pub struct JwtClient {
   encoding_key: EncodingKey,
   decoding_key: DecodingKey,
   ttl_ms: u128,
-  exchange_tokens: ExchangeTokenMap,
 }
 
 impl JwtClient {
@@ -48,7 +40,6 @@ impl JwtClient {
       ttl_ms: get_timelength_in_ms(
         config.jwt_ttl.to_string().parse()?,
       ),
-      exchange_tokens: Default::default(),
     })
   }
 
@@ -64,46 +55,13 @@ impl JwtClient {
       exp,
     };
     let jwt = encode(&self.header, &claims, &self.encoding_key)
-      .context("failed at signing claim")?;
+      .context("Failed at signing claim")?;
     Ok(JwtResponse { user_id, jwt })
   }
 
   pub fn decode(&self, jwt: &str) -> anyhow::Result<JwtClaims> {
     decode::<JwtClaims>(jwt, &self.decoding_key, &self.validation)
       .map(|res| res.claims)
-      .context("failed to decode token claims")
-  }
-
-  pub async fn create_exchange_token(
-    &self,
-    jwt: JwtResponse,
-  ) -> String {
-    let exchange_token = random_string(40);
-    self.exchange_tokens.lock().await.insert(
-      exchange_token.clone(),
-      (
-        jwt,
-        unix_timestamp_ms()
-          + get_timelength_in_ms(Timelength::OneMinute),
-      ),
-    );
-    exchange_token
-  }
-
-  pub async fn redeem_exchange_token(
-    &self,
-    exchange_token: &str,
-  ) -> anyhow::Result<JwtResponse> {
-    let (jwt, valid_until) = self
-      .exchange_tokens
-      .lock()
-      .await
-      .remove(exchange_token)
-      .context("invalid exchange token: unrecognized")?;
-    if unix_timestamp_ms() < valid_until {
-      Ok(jwt)
-    } else {
-      Err(anyhow!("invalid exchange token: expired"))
-    }
+      .context("Failed to decode token claims")
   }
 }

bin/core/src/auth/local.rs

@@ -1,10 +1,10 @@
-use std::str::FromStr;
+use std::sync::{Arc, OnceLock};
 
 use anyhow::{Context, anyhow};
 use async_timing_util::unix_timestamp_ms;
 use database::{
   hash_password,
-  mungos::mongodb::bson::{Document, doc, oid::ObjectId},
+  mungos::mongodb::bson::{Document, doc},
 };
 use komodo_client::{
   api::auth::{
@@ -13,136 +13,233 @@ use komodo_client::{
   },
   entities::user::{User, UserConfig},
 };
+use rate_limit::{RateLimiter, WithFailureRateLimit};
+use reqwest::StatusCode;
 use resolver_api::Resolve;
+use serror::{AddStatusCode as _, AddStatusCodeError};
+use tower_sessions::Session;
 
 use crate::{
-  api::auth::AuthArgs,
+  api::{
+    SESSION_KEY_PASSKEY_LOGIN, SESSION_KEY_TOTP_LOGIN, auth::AuthArgs,
+  },
   config::core_config,
-  state::{db_client, jwt_client},
+  helpers::validations::{validate_password, validate_username},
+  state::{auth_rate_limiter, db_client, jwt_client, webauthn},
 };
 
 impl Resolve<AuthArgs> for SignUpLocalUser {
   #[instrument("SignUpLocalUser", skip(self))]
   async fn resolve(
     self,
-    _: &AuthArgs,
+    AuthArgs {
+      headers,
+      ip,
+      session,
+    }: &AuthArgs,
   ) -> serror::Result<SignUpLocalUserResponse> {
-    let core_config = core_config();
-
-    if !core_config.local_auth {
-      return Err(anyhow!("Local auth is not enabled").into());
-    }
-
-    if self.username.is_empty() {
-      return Err(anyhow!("Username cannot be empty string").into());
-    }
-
-    if ObjectId::from_str(&self.username).is_ok() {
-      return Err(
-        anyhow!("Username cannot be valid ObjectId").into(),
-      );
-    }
-
-    if self.password.is_empty() {
-      return Err(anyhow!("Password cannot be empty string").into());
-    }
-
-    let db = db_client();
-
-    let no_users_exist =
-      db.users.find_one(Document::new()).await?.is_none();
-
-    if !no_users_exist && core_config.disable_user_registration {
-      return Err(anyhow!("User registration is disabled").into());
-    }
-
-    if db
-      .users
-      .find_one(doc! { "username": &self.username })
+    sign_up_local_user(self)
+      .with_failure_rate_limit_using_headers(
+        auth_rate_limiter(),
+        headers,
+        Some(*ip),
+      )
       .await
-      .context("Failed to query for existing users")?
-      .is_some()
-    {
-      return Err(anyhow!("Username already taken.").into());
-    }
-
-    let ts = unix_timestamp_ms() as i64;
-    let hashed_password = hash_password(self.password)?;
-
-    let user = User {
-      id: Default::default(),
-      username: self.username,
-      enabled: no_users_exist || core_config.enable_new_users,
-      admin: no_users_exist,
-      super_admin: no_users_exist,
-      create_server_permissions: no_users_exist,
-      create_build_permissions: no_users_exist,
-      updated_at: ts,
-      last_update_view: 0,
-      recents: Default::default(),
-      all: Default::default(),
-      config: UserConfig::Local {
-        password: hashed_password,
-      },
-    };
-
-    let user_id = db_client()
-      .users
-      .insert_one(user)
-      .await
-      .context("failed to create user")?
-      .inserted_id
-      .as_object_id()
-      .context("inserted_id is not ObjectId")?
-      .to_string();
-
-    jwt_client()
-      .encode(user_id.clone())
-      .context("failed to generate jwt for user")
-      .map_err(Into::into)
   }
 }
 
+async fn sign_up_local_user(
+  req: SignUpLocalUser,
+) -> serror::Result<SignUpLocalUserResponse> {
+  let config = core_config();
+
+  if !config.local_auth {
+    return Err(anyhow!("Local auth is not enabled").into());
+  }
+
+  validate_username(&req.username)
+    .status_code(StatusCode::BAD_REQUEST)?;
+  validate_password(&req.password)
+    .status_code(StatusCode::BAD_REQUEST)?;
+
+  let db = db_client();
+
+  let no_users_exist =
+    db.users.find_one(Document::new()).await?.is_none();
+
+  if !no_users_exist && config.disable_user_registration {
+    return Err(
+      anyhow!("User registration is disabled")
+        .status_code(StatusCode::UNAUTHORIZED),
+    );
+  }
+
+  if db
+    .users
+    .find_one(doc! { "username": &req.username })
+    .await
+    .context("Failed to query for existing users")?
+    .is_some()
+  {
+    // When user registration is enabled, there is no way around allowing
+    // potential attackers to gain some insight about which usernames exist
+    // if they are allowed to register accounts. Since this can be easily inferred,
+    // might as well be clear. The auth rate limiter is critical here.
+    return Err(
+      anyhow!("Username already taken.")
+        .status_code(StatusCode::BAD_REQUEST),
+    );
+  }
+
+  let ts = unix_timestamp_ms() as i64;
+  let hashed_password = hash_password(req.password)?;
+
+  let user = User {
+    id: Default::default(),
+    username: req.username,
+    enabled: no_users_exist || config.enable_new_users,
+    admin: no_users_exist,
+    super_admin: no_users_exist,
+    create_server_permissions: no_users_exist,
+    create_build_permissions: no_users_exist,
+    updated_at: ts,
+    last_update_view: 0,
+    recents: Default::default(),
+    all: Default::default(),
+    config: UserConfig::Local {
+      password: hashed_password,
+    },
+    totp: Default::default(),
+    passkey: Default::default(),
+  };
+
+  let user_id = db_client()
+    .users
+    .insert_one(user)
+    .await
+    .context("Failed to create user on database")?
+    .inserted_id
+    .as_object_id()
+    .context("The 'inserted_id' is not ObjectId")?
+    .to_string();
+
+  jwt_client()
+    .encode(user_id)
+    .context("Failed to generate JWT for user")
+    .map_err(Into::into)
+}
+
+/// Local login method has a dedicated rate limiter
+/// so the UI background calls using existing JWT do
+/// not influence the number of attempts user has
+/// to log in.
+fn login_local_user_rate_limiter() -> &'static RateLimiter {
+  static LOGIN_LOCAL_USER_RATE_LIMITER: OnceLock<Arc<RateLimiter>> =
+    OnceLock::new();
+  LOGIN_LOCAL_USER_RATE_LIMITER.get_or_init(|| {
+    let config = core_config();
+    RateLimiter::new(
+      config.auth_rate_limit_disabled,
+      config.auth_rate_limit_max_attempts as usize,
+      config.auth_rate_limit_window_seconds,
+    )
+  })
+}
+
 impl Resolve<AuthArgs> for LoginLocalUser {
   async fn resolve(
     self,
-    _: &AuthArgs,
+    AuthArgs {
+      headers,
+      ip,
+      session,
+    }: &AuthArgs,
   ) -> serror::Result<LoginLocalUserResponse> {
-    if !core_config().local_auth {
-      return Err(anyhow!("local auth is not enabled").into());
-    }
+    login_local_user(
+      self,
+      session
+        .as_ref()
+        .context("Method called in context without session")?,
+    )
+    .with_failure_rate_limit_using_headers(
+      login_local_user_rate_limiter(),
+      headers,
+      Some(*ip),
+    )
+    .await
+  }
+}
+
+async fn login_local_user(
+  req: LoginLocalUser,
+  session: &Session,
+) -> serror::Result<LoginLocalUserResponse> {
+  if !core_config().local_auth {
+    return Err(
+      anyhow!("Local auth is not enabled")
+        .status_code(StatusCode::UNAUTHORIZED),
+    );
+  }
+
+  validate_username(&req.username)
+    .status_code(StatusCode::BAD_REQUEST)?;
+
+  let user = db_client()
+    .users
+    .find_one(doc! { "username": &req.username })
+    .await
+    .context("Failed at db query for users")?
+    .context("Invalid login credentials")
+    .status_code(StatusCode::UNAUTHORIZED)?;
 
-    let user = db_client()
-      .users
-      .find_one(doc! { "username": &self.username })
-      .await
-      .context("failed at db query for users")?
-      .with_context(|| {
-        format!("did not find user with username {}", self.username)
-      })?;
+  let UserConfig::Local {
+    password: user_pw_hash,
+  } = user.config
+  else {
+    return Err(
+      anyhow!("Invalid login credentials")
+        .status_code(StatusCode::UNAUTHORIZED),
+    );
+  };
 
-    let UserConfig::Local {
-      password: user_pw_hash,
-    } = user.config
-    else {
-      return Err(
-        anyhow!(
-          "non-local auth users can not log in with a password"
-        )
-        .into(),
-      );
-    };
+  let verified = bcrypt::verify(req.password, &user_pw_hash)
+    .context("Invalid login credentials")
+    .status_code(StatusCode::UNAUTHORIZED)?;
 
-    let verified = bcrypt::verify(self.password, &user_pw_hash)
-      .context("failed at verify password")?;
+  if !verified {
+    return Err(
+      anyhow!("Invalid login credentials")
+        .status_code(StatusCode::UNAUTHORIZED),
+    );
+  }
 
-    if !verified {
-      return Err(anyhow!("invalid credentials").into());
+  match (user.passkey.passkey, user.totp.enrolled()) {
+    // WebAuthn 2FA
+    (Some(passkey), _) => {
+      let webauthn = webauthn().context(
+        "No webauthn provider available, invalid KOMODO_HOST config",
+      )?;
+      let (response, server_state) = webauthn
+        .start_passkey_authentication(&[passkey])
+        .context("Failed to start passkey authentication flow")?;
+      session
+        .insert(SESSION_KEY_PASSKEY_LOGIN, (user.id, server_state))
+        .await?;
+      Ok(LoginLocalUserResponse::Passkey(response))
     }
-
-    jwt_client()
-      .encode(user.id.clone())
-      .context("failed at generating jwt for user")
-      .map_err(Into::into)
+    // TOTP 2FA
+    (None, true) => {
+      session.insert(SESSION_KEY_TOTP_LOGIN, user.id).await?;
+      Ok(LoginLocalUserResponse::Totp {})
+    }
+    // No 2FA, can return JWT immediately
+    (None, false) => {
+      jwt_client()
+        .encode(user.id)
+        // This is in internal error (500), not auth error
+        .context("Failed to generate JWT for user")
+        .map(|jwt| LoginLocalUserResponse::Jwt(jwt))
+        .map_err(Into::into)
+    }
   }
 }

bin/core/src/auth/mod.rs

@@ -1,18 +1,25 @@
+use std::net::SocketAddr;
+
 use anyhow::{Context, anyhow};
 use async_timing_util::unix_timestamp_ms;
 use axum::{
-  extract::Request, http::HeaderMap, middleware::Next,
-  response::Response,
+  extract::{ConnectInfo, Request},
+  http::HeaderMap,
+  middleware::Next,
+  response::{Redirect, Response},
 };
 use database::mungos::mongodb::bson::doc;
+use futures_util::TryFutureExt;
 use komodo_client::entities::{komodo_timestamp, user::User};
+use rate_limit::WithFailureRateLimit;
 use reqwest::StatusCode;
 use serde::Deserialize;
-use serror::AddStatusCode;
+use serror::AddStatusCodeError as _;
 
 use crate::{
+  config::core_config,
   helpers::query::get_user,
-  state::{db_client, jwt_client},
+  state::{auth_rate_limiter, db_client, jwt_client},
 };
 
 use self::jwt::JwtClaims;
@@ -21,10 +28,16 @@ pub mod github;
 pub mod google;
 pub mod jwt;
 pub mod oidc;
+pub mod totp;
 
 mod local;
 
+/// Length of random token in Oauth / OIDC 'state'
 const STATE_PREFIX_LENGTH: usize = 20;
+/// JWT Clock skew tolerance in milliseconds (10 seconds for JWTs)
+const JWT_CLOCK_SKEW_TOLERANCE_MS: u128 = 10 * 1000;
+/// Api Key Clock skew tolerance in milliseconds (5 minutes for Api Keys)
+const API_KEY_CLOCK_SKEW_TOLERANCE_MS: i64 = 5 * 60 * 1000;
 
 #[derive(Debug, Deserialize)]
 struct RedirectQuery {
@@ -36,9 +49,21 @@ pub async fn auth_request(
   mut req: Request,
   next: Next,
 ) -> serror::Result<Response> {
-  let user = authenticate_check_enabled(&headers)
-    .await
-    .status_code(StatusCode::UNAUTHORIZED)?;
+  let fallback = req
+    .extensions()
+    .get::<ConnectInfo<SocketAddr>>()
+    .map(|addr| addr.ip());
+  let mut user = authenticate_check_enabled(&headers)
+    .map_err(|e| e.status_code(StatusCode::UNAUTHORIZED))
+    .with_failure_rate_limit_using_headers(
+      auth_rate_limiter(),
+      &headers,
+      fallback,
+    )
+    .await?;
+  // Sanitize the user for safety before
+  // attaching to the request handlers. 
+  user.sanitize();
   req.extensions_mut().insert(user);
   Ok(next.run(req).await)
 }
@@ -53,23 +78,21 @@ pub async fn get_user_id_from_headers(
   ) {
     (Some(jwt), _, _) => {
       // USE JWT
-      let jwt = jwt.to_str().context("jwt is not str")?;
-      auth_jwt_get_user_id(jwt)
-        .await
-        .context("failed to authenticate jwt")
+      let jwt = jwt.to_str().context("JWT is not valid UTF-8")?;
+      auth_jwt_get_user_id(jwt).await
     }
     (None, Some(key), Some(secret)) => {
       // USE API KEY / SECRET
-      let key = key.to_str().context("key is not str")?;
-      let secret = secret.to_str().context("secret is not str")?;
-      auth_api_key_get_user_id(key, secret)
-        .await
-        .context("failed to authenticate api key")
+      let key =
+        key.to_str().context("X-API-KEY is not valid UTF-8")?;
+      let secret =
+        secret.to_str().context("X-API-SECRET is not valid UTF-8")?;
+      auth_api_key_get_user_id(key, secret).await
     }
     _ => {
       // AUTH FAIL
       Err(anyhow!(
-        "must attach either AUTHORIZATION header with jwt OR pass X-API-KEY and X-API-SECRET"
+        "Must attach either AUTHORIZATION header with jwt OR pass X-API-KEY and X-API-SECRET"
       ))
     }
   }
@@ -79,22 +102,30 @@ pub async fn authenticate_check_enabled(
   headers: &HeaderMap,
 ) -> anyhow::Result<User> {
   let user_id = get_user_id_from_headers(headers).await?;
-  let user = get_user(&user_id).await?;
+  let user = get_user(&user_id)
+    .await
+    .map_err(|_| anyhow!("Invalid user credentials"))?;
   if user.enabled {
     Ok(user)
   } else {
-    Err(anyhow!("user not enabled"))
+    Err(anyhow!("Invalid user credentials"))
   }
 }
 
 pub async fn auth_jwt_get_user_id(
   jwt: &str,
 ) -> anyhow::Result<String> {
-  let claims: JwtClaims = jwt_client().decode(jwt)?;
-  if claims.exp > unix_timestamp_ms() {
+  let claims: JwtClaims = jwt_client()
+    .decode(jwt)
+    .map_err(|_| anyhow!("Invalid user credentials"))?;
+  // Apply clock skew tolerance.
+  // Token is valid if expiration is greater than (now - tolerance)
+  if claims.exp
+    > unix_timestamp_ms().saturating_sub(JWT_CLOCK_SKEW_TOLERANCE_MS)
+  {
     Ok(claims.id)
   } else {
-    Err(anyhow!("token has expired"))
+    Err(anyhow!("Invalid user credentials"))
   }
 }
 
@@ -113,19 +144,25 @@ pub async fn auth_api_key_get_user_id(
     .api_keys
     .find_one(doc! { "key": key })
     .await
-    .context("failed to query db")?
-    .context("no api key matching key")?;
-  if key.expires != 0 && key.expires < komodo_timestamp() {
-    return Err(anyhow!("api key expired"));
+    .context("Failed to query db")?
+    .context("Invalid user credentials")?;
+  // Apply clock skew tolerance.
+  // Token is invalid if expiration is less than (now - tolerance)
+  if key.expires != 0
+    && key.expires
+      < komodo_timestamp()
+        .saturating_sub(API_KEY_CLOCK_SKEW_TOLERANCE_MS)
+  {
+    return Err(anyhow!("Invalid user credentials"));
   }
   if bcrypt::verify(secret, &key.secret)
-    .context("failed to verify secret hash")?
+    .map_err(|_| anyhow!("Invalid user credentials"))?
   {
     // secret matches
     Ok(key.user_id)
   } else {
     // secret mismatch
-    Err(anyhow!("invalid api secret"))
+    Err(anyhow!("Invalid user credentials"))
   }
 }
 
@@ -142,6 +179,18 @@ async fn check_enabled(user_id: String) -> anyhow::Result<User> {
   if user.enabled {
     Ok(user)
   } else {
-    Err(anyhow!("user not enabled"))
+    Err(anyhow!("Invalid user credentials"))
   }
 }
+
+fn format_redirect(redirect: Option<&str>, extra: &str) -> Redirect {
+  let redirect_url = if let Some(redirect) = redirect
+    && !redirect.is_empty()
+  {
+    let splitter = if redirect.contains('?') { '&' } else { '?' };
+    format!("{redirect}{splitter}{extra}")
+  } else {
+    format!("{}?{extra}", core_config().host)
+  };
+  Redirect::to(&redirect_url)
+}

bin/core/src/auth/oidc/mod.rs

@@ -1,15 +1,23 @@
-use std::sync::OnceLock;
+use std::{net::SocketAddr, sync::OnceLock};
 
 use anyhow::{Context, anyhow};
 use axum::{
-  Router, extract::Query, response::Redirect, routing::get,
+  Router,
+  extract::{ConnectInfo, Query},
+  http::HeaderMap,
+  response::Redirect,
+  routing::get,
 };
 use client::oidc_client;
 use dashmap::DashMap;
 use database::mungos::mongodb::bson::{Document, doc};
-use komodo_client::entities::{
-  komodo_timestamp, random_string,
-  user::{User, UserConfig},
+use futures_util::TryFutureExt;
+use komodo_client::{
+  api::auth::UserIdOrTwoFactor,
+  entities::{
+    komodo_timestamp, random_string,
+    user::{User, UserConfig},
+  },
 };
 use openidconnect::{
   AccessTokenHash, AuthorizationCode, CsrfToken,
@@ -17,13 +25,20 @@ use openidconnect::{
   PkceCodeChallenge, PkceCodeVerifier, Scope, TokenResponse,
   core::{CoreAuthenticationFlow, CoreGenderClaim},
 };
+use rate_limit::WithFailureRateLimit;
 use reqwest::StatusCode;
 use serde::Deserialize;
-use serror::AddStatusCode;
+use serror::{AddStatusCode as _, AddStatusCodeError};
+use tower_sessions::Session;
 
 use crate::{
+  api::{
+    SESSION_KEY_PASSKEY_LOGIN, SESSION_KEY_TOTP_LOGIN,
+    SESSION_KEY_USER_ID,
+  },
+  auth::format_redirect,
   config::core_config,
-  state::{db_client, jwt_client},
+  state::{auth_rate_limiter, db_client, webauthn},
 };
 
 use super::RedirectQuery;
@@ -68,9 +83,21 @@ pub fn router() -> Router {
     )
     .route(
       "/callback",
-      get(|query| async {
-        callback(query).await.status_code(StatusCode::UNAUTHORIZED)
-      }),
+      get(
+        |query,
+         session: Session,
+         headers: HeaderMap,
+         ConnectInfo(info): ConnectInfo<SocketAddr>| async move {
+          callback(query, session)
+            .map_err(|e| e.status_code(StatusCode::UNAUTHORIZED))
+            .with_failure_rate_limit_using_headers(
+              auth_rate_limiter(),
+              &headers,
+              Some(info.ip()),
+            )
+            .await
+        },
+      ),
     )
 }
 
@@ -138,6 +165,7 @@ struct CallbackQuery {
 
 async fn callback(
   Query(query): Query<CallbackQuery>,
+  session: Session,
 ) -> anyhow::Result<Redirect> {
   let client = oidc_client().load();
   let client =
@@ -217,12 +245,41 @@ async fn callback(
       "config.data.user_id": user_id
     })
     .await
-    .context("failed at find user query from database")?;
+    .context("Failed at find user query from database")?;
 
-  let jwt = match user {
-    Some(user) => jwt_client()
-      .encode(user.id)
-      .context("failed to generate jwt")?,
+  let user_id_or_two_factor = match user {
+    Some(user) => {
+      match (user.passkey.passkey, user.totp.enrolled()) {
+        // WebAuthn Passkey 2FA
+        (Some(passkey), _) => {
+          let webauthn = webauthn().context(
+            "No webauthn provider available, invalid KOMODO_HOST config",
+          )?;
+          let (response, server_state) = webauthn
+            .start_passkey_authentication(&[passkey])
+            .context("Failed to start passkey authentication flow")?;
+          session
+            .insert(
+              SESSION_KEY_PASSKEY_LOGIN,
+              (user.id, server_state),
+            )
+            .await?;
+          UserIdOrTwoFactor::Passkey(response)
+        }
+        // TOTP 2FA
+        (None, true) => {
+          session
+            .insert(SESSION_KEY_TOTP_LOGIN, user.id)
+            .await
+            .context(
+              "Failed to store totp login state in for user session",
+            )?;
+          UserIdOrTwoFactor::Totp {}
+        }
+        // No 2FA
+        (None, false) => UserIdOrTwoFactor::UserId(user.id),
+      }
+    }
     None => {
       let ts = komodo_timestamp();
       let no_users_exist =
@@ -293,6 +350,8 @@ async fn callback(
           provider: core_config.oidc_provider.clone(),
           user_id: user_id.to_string(),
         },
+        totp: Default::default(),
+        passkey: Default::default(),
       };
 
       let user_id = db_client
@@ -305,17 +364,29 @@ async fn callback(
         .context("inserted_id is not ObjectId")?
         .to_string();
 
-      jwt_client()
-        .encode(user_id)
-        .context("failed to generate jwt")?
+      UserIdOrTwoFactor::UserId(user_id)
     }
   };
-  let exchange_token = jwt_client().create_exchange_token(jwt).await;
-  let redirect_url = if let Some(redirect) = redirect {
-    let splitter = if redirect.contains('?') { '&' } else { '?' };
-    format!("{redirect}{splitter}token={exchange_token}")
-  } else {
-    format!("{}?token={exchange_token}", core_config().host)
-  };
-  Ok(Redirect::to(&redirect_url))
+
+  match user_id_or_two_factor {
+    UserIdOrTwoFactor::UserId(user_id) => {
+      session
+        .insert(SESSION_KEY_USER_ID, user_id)
+        .await
+        .context("Failed to store user id for client session")?;
+      Ok(format_redirect(redirect.as_deref(), "redeem_ready=true"))
+    }
+    UserIdOrTwoFactor::Totp {} => {
+      Ok(format_redirect(redirect.as_deref(), "totp=true"))
+    }
+    UserIdOrTwoFactor::Passkey(passkey) => {
+      let passkey = serde_json::to_string(&passkey)
+        .context("Failed to serialize passkey response")?;
+      let passkey = urlencoding::encode(&passkey);
+      Ok(format_redirect(
+        redirect.as_deref(),
+        &format!("passkey={passkey}"),
+      ))
+    }
+  }
 }

bin/core/src/auth/totp.rs

@@ -0,0 +1,17 @@
+use anyhow::Context as _;
+
+pub fn make_totp(
+  secret_bytes: Vec<u8>,
+  account_name: impl Into<Option<String>>,
+) -> anyhow::Result<totp_rs::TOTP> {
+  totp_rs::TOTP::new(
+    totp_rs::Algorithm::SHA1,
+    6,
+    1,
+    30,
+    secret_bytes,
+    Some(String::from("Komodo")),
+    account_name.into().unwrap_or_default(),
+  )
+  .context("Failed to construct TOTP")
+}

bin/core/src/cloud/aws/ec2.rs

@@ -11,7 +11,7 @@ use aws_sdk_ec2::{
     Tag, TagSpecification,
   },
 };
-use base64::Engine;
+use data_encoding::BASE64_NOPAD;
 use komodo_client::entities::{
   ResourceTarget,
   alert::{Alert, AlertData, SeverityLevel},
@@ -127,10 +127,7 @@ pub async fn launch_ec2_instance(
     )
     .min_count(1)
     .max_count(1)
-    .user_data(
-      base64::engine::general_purpose::STANDARD_NO_PAD
-        .encode(user_data),
-    );
+    .user_data(BASE64_NOPAD.encode(user_data.as_bytes()));
 
   let res = req
     .send()

bin/core/src/config.rs

@@ -1,6 +1,7 @@
 use std::{path::PathBuf, sync::OnceLock};
 
 use anyhow::Context;
+use axum::http::HeaderValue;
 use colored::Colorize;
 use config::ConfigLoader;
 use environment_file::{
@@ -14,6 +15,7 @@ use komodo_client::entities::{
   logger::LogConfig,
 };
 use noise::key::{RotatableKeyPair, SpkiPublicKey};
+use tower_http::cors::CorsLayer;
 
 /// Should call in startup to ensure Core errors without valid private key.
 pub fn core_keys() -> &'static RotatableKeyPair {
@@ -89,6 +91,70 @@ pub fn periphery_public_keys() -> Option<&'static [SpkiPublicKey]> {
     .as_deref()
 }
 
+/// Creates a CORS layer based on the Core configuration.
+///
+/// - If `cors_allowed_origins` is empty: Allows all origins (backward compatibility)
+/// - If `cors_allowed_origins` is set: Only allows the specified origins
+/// - Methods and headers are always allowed (Any)
+/// - Credentials are only allowed if `cors_allow_credentials` is true
+pub fn cors_layer() -> CorsLayer {
+  let config = core_config();
+  let mut cors = CorsLayer::new()
+    .allow_methods(tower_http::cors::AllowMethods::mirror_request())
+    .allow_headers(tower_http::cors::AllowHeaders::mirror_request())
+    .allow_credentials(config.cors_allow_credentials);
+  if config.cors_allowed_origins.is_empty() {
+    warn!(
+      "CORS using allowed origin 'Any' (*). Use KOMODO_CORS_ALLOWED_ORIGINS to configure specific origins."
+    );
+    cors = cors.allow_origin(tower_http::cors::Any)
+  } else {
+    let allowed_origins = config
+      .cors_allowed_origins
+      .iter()
+      .filter_map(|origin| {
+        HeaderValue::from_str(origin)
+          .inspect_err(|e| {
+            warn!("Invalid CORS allowed origin: {origin} | {e:?}")
+          })
+          .ok()
+      })
+      .collect::<Vec<_>>();
+    info!("CORS using allowed origin/s: {allowed_origins:?}");
+    cors = cors.allow_origin(allowed_origins);
+  };
+  cors
+}
+
+pub fn monitoring_interval() -> async_timing_util::Timelength {
+  static MONITORING_INTERVAL: OnceLock<
+    async_timing_util::Timelength,
+  > = OnceLock::new();
+  *MONITORING_INTERVAL.get_or_init(|| {
+    core_config().monitoring_interval.try_into().unwrap_or_else(
+      |_| {
+        error!("Invalid 'monitoring_interval', using default 15-sec");
+        async_timing_util::Timelength::FifteenSeconds
+      },
+    )
+  })
+}
+
+pub fn core_host() -> Option<&'static url::Url> {
+  static CORE_URL: OnceLock<Option<url::Url>> = OnceLock::new();
+  CORE_URL
+    .get_or_init(|| {
+      url::Url::parse(&core_config().host)
+      .inspect_err(|e| {
+        warn!(
+          "Invalid KOMODO_HOST: not URL. Passkeys won't work. | {e:?}"
+        )
+      })
+      .ok()
+    })
+    .as_ref()
+}
+
 pub fn core_config() -> &'static CoreConfig {
   static CORE_CONFIG: OnceLock<CoreConfig> = OnceLock::new();
   CORE_CONFIG.get_or_init(|| {
@@ -281,6 +347,21 @@ pub fn core_config() -> &'static CoreConfig {
         .komodo_frontend_path
         .unwrap_or(config.frontend_path),
       jwt_ttl: env.komodo_jwt_ttl.unwrap_or(config.jwt_ttl),
+      auth_rate_limit_disabled: env
+        .komodo_auth_rate_limit_disabled
+        .unwrap_or(config.auth_rate_limit_disabled),
+      auth_rate_limit_max_attempts: env
+        .komodo_auth_rate_limit_max_attempts
+        .unwrap_or(config.auth_rate_limit_max_attempts),
+      auth_rate_limit_window_seconds: env
+        .komodo_auth_rate_limit_window_seconds
+        .unwrap_or(config.auth_rate_limit_window_seconds),
+      cors_allowed_origins: env
+        .komodo_cors_allowed_origins
+        .unwrap_or(config.cors_allowed_origins),
+      cors_allow_credentials: env
+        .komodo_cors_allow_credentials
+        .unwrap_or(config.cors_allow_credentials),
       sync_directory: env
         .komodo_sync_directory
         .unwrap_or(config.sync_directory),
@@ -336,6 +417,9 @@ pub fn core_config() -> &'static CoreConfig {
         .komodo_lock_login_credentials_for
         .unwrap_or(config.lock_login_credentials_for),
       local_auth: env.komodo_local_auth.unwrap_or(config.local_auth),
+      min_password_length: env
+        .komodo_min_password_length
+        .unwrap_or(config.min_password_length),
       logging: LogConfig {
         level: env
           .komodo_logging_level

bin/core/src/helpers/action_state.rs

@@ -1,3 +1,16 @@
+//! # Action State Management
+//!
+//! This module provides thread-safe state management for resource exections.
+//! It prevents concurrent execution of exections on the same resource using
+//! a Mutex-based locking mechanism with RAII guards.
+//!
+//! ## Safety
+//!
+//! - Uses RAII pattern to ensure locks are always released
+//! - Handles lock poisoning gracefully
+//! - Prevents race conditions through per-resource locks
+//! - No deadlock risk: each resource has independent locks
+
 use std::sync::{Arc, Mutex};
 
 use anyhow::anyhow;
@@ -29,7 +42,16 @@ pub struct ActionStates {
     CloneCache<String, Arc<ActionState<ResourceSyncActionState>>>,
 }
 
-/// Need to be able to check "busy" with write lock acquired.
+/// Thread-safe state container for resource executions.
+///
+/// Uses a Mutex to prevent concurrent executions and provides
+/// RAII-based locking through [UpdateGuard].
+///
+/// # Safety
+///
+/// - Each resource has its own ActionState instance
+/// - State is reset to default when [UpdateGuard] is dropped
+/// - Lock poisoning error handling is handled gracefully with anyhow::Error
 #[derive(Default)]
 pub struct ActionState<States: Default + Send + 'static>(
   Mutex<States>,
@@ -43,7 +65,7 @@ impl<States: Default + Busy + Copy + Send + 'static>
       *self
         .0
         .lock()
-        .map_err(|e| anyhow!("action state lock poisoned | {e:?}"))?,
+        .map_err(|e| anyhow!("Action state lock poisoned | {e:?}"))?,
     )
   }
 
@@ -52,14 +74,33 @@ impl<States: Default + Busy + Copy + Send + 'static>
       self
         .0
         .lock()
-        .map_err(|e| anyhow!("action state lock poisoned | {e:?}"))?
+        .map_err(|e| anyhow!("Action state lock poisoned | {e:?}"))?
         .busy(),
     )
   }
 
-  /// Will acquire lock, check busy, and if not will
-  /// run the provided update function on the states.
-  /// Returns a guard that returns the states to default (not busy) when dropped.
+  /// Acquires lock, checks if resource is busy, and if not,
+  /// runs the provided update function on the states.
+  ///
+  /// Returns an `UpdateGuard` that automatically resets the state
+  /// to default (not busy) when dropped.
+  ///
+  /// # Errors
+  ///
+  /// Returns an error if:
+  /// - The lock is poisoned
+  /// - The resource is currently busy
+  ///
+  /// # Example
+  ///
+  /// ```rust
+  /// let guard = action_state.update(|state| {
+  ///   *state = SomeNewState;
+  /// })?;
+  /// // State is locked and marked as busy
+  /// // ... perform work ...
+  /// drop(guard) // Guard is dropped, state returns to default
+  /// ```
   pub fn update(
     &self,
     update_fn: impl Fn(&mut States),
@@ -92,10 +133,22 @@ impl<States: Default + Busy + Copy + Send + 'static>
   }
 }
 
-/// When dropped will return the inner state to default.
-/// The inner mutex guard must already be dropped BEFORE this is dropped,
-/// which is guaranteed as the inner guard is dropped by all public methods before
-/// user could drop UpdateGuard.
+/// RAII guard that automatically resets the action state when dropped.
+///
+/// # Safety
+///
+/// The inner mutex guard is guaranteed to be dropped before this guard
+/// is dropped, preventing deadlocks. This is ensured by all public methods
+/// that create UpdateGuard instances.
+///
+/// # Behavior
+///
+/// When dropped, this guard will:
+/// 1. Re-acquire the lock
+/// 2. Call the provided return function (typically resetting to default)
+/// 3. Release the lock
+///
+/// If the lock is poisoned, an error is logged but the drop continues.
 pub struct UpdateGuard<'a, States: Default + Send + 'static>(
   &'a Mutex<States>,
   Box<dyn Fn(&mut States) + Send>,

bin/core/src/helpers/mod.rs

@@ -4,6 +4,7 @@ use anyhow::{Context, anyhow};
 use database::mongo_indexed::Document;
 use database::mungos::mongodb::bson::{Bson, doc};
 use indexmap::IndexSet;
+use komodo_client::entities::SwarmOrServer;
 use komodo_client::entities::{
   ResourceTarget,
   build::Build,
@@ -15,7 +16,11 @@ use komodo_client::entities::{
   stack::Stack,
   user::User,
 };
+use resolver_api::HasResponse;
+use serde::Serialize;
+use serde::de::DeserializeOwned;
 
+use crate::helpers::swarm::swarm_request;
 use crate::{
   config::core_config, connection::PeripheryConnectionArgs,
   periphery::PeripheryClient, state::db_client,
@@ -33,8 +38,7 @@ pub mod query;
 pub mod swarm;
 pub mod terminal;
 pub mod update;
-
-// pub mod resource;
+pub mod validations;
 
 pub fn empty_or_only_spaces(word: &str) -> bool {
   if word.is_empty() {
@@ -265,3 +269,21 @@ pub fn repo_link(
   }
   res
 }
+
+pub async fn swarm_or_server_request<T>(
+  swarm_or_server: &SwarmOrServer,
+  request: T,
+) -> anyhow::Result<T::Response>
+where
+  T: std::fmt::Debug + Clone + Serialize + HasResponse,
+  T::Response: DeserializeOwned,
+{
+  match swarm_or_server {
+    SwarmOrServer::Swarm(swarm) => {
+      swarm_request(&swarm.config.server_ids, request).await
+    }
+    SwarmOrServer::Server(server) => {
+      periphery_client(server).await?.request(request).await
+    }
+  }
+}

bin/core/src/helpers/procedure.rs

@@ -457,7 +457,7 @@ async fn execute_execution(
           .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
-          .context("Failed at RemoveDeployment"),
+          .context("Failed at DestroyDeployment"),
         &update_id,
       )
       .await?
@@ -1162,6 +1162,91 @@ async fn execute_execution(
       )
       .await?
     }
+    Execution::RemoveSwarmNodes(req) => {
+      let req = ExecuteRequest::RemoveSwarmNodes(req);
+      let update = init_execution_update(&req, &user).await?;
+      let ExecuteRequest::RemoveSwarmNodes(req) = req else {
+        unreachable!()
+      };
+      let update_id = update.id.clone();
+      handle_resolve_result(
+        req
+          .resolve(&ExecuteArgs { user, update, id })
+          .await
+          .map_err(|e| e.error)
+          .context("Failed at RemoveSwarmNodes"),
+        &update_id,
+      )
+      .await?
+    }
+    Execution::RemoveSwarmStacks(req) => {
+      let req = ExecuteRequest::RemoveSwarmStacks(req);
+      let update = init_execution_update(&req, &user).await?;
+      let ExecuteRequest::RemoveSwarmStacks(req) = req else {
+        unreachable!()
+      };
+      let update_id = update.id.clone();
+      handle_resolve_result(
+        req
+          .resolve(&ExecuteArgs { user, update, id })
+          .await
+          .map_err(|e| e.error)
+          .context("Failed at RemoveSwarmStacks"),
+        &update_id,
+      )
+      .await?
+    }
+    Execution::RemoveSwarmServices(req) => {
+      let req = ExecuteRequest::RemoveSwarmServices(req);
+      let update = init_execution_update(&req, &user).await?;
+      let ExecuteRequest::RemoveSwarmServices(req) = req else {
+        unreachable!()
+      };
+      let update_id = update.id.clone();
+      handle_resolve_result(
+        req
+          .resolve(&ExecuteArgs { user, update, id })
+          .await
+          .map_err(|e| e.error)
+          .context("Failed at RemoveSwarmServices"),
+        &update_id,
+      )
+      .await?
+    }
+    Execution::RemoveSwarmConfigs(req) => {
+      let req = ExecuteRequest::RemoveSwarmConfigs(req);
+      let update = init_execution_update(&req, &user).await?;
+      let ExecuteRequest::RemoveSwarmConfigs(req) = req else {
+        unreachable!()
+      };
+      let update_id = update.id.clone();
+      handle_resolve_result(
+        req
+          .resolve(&ExecuteArgs { user, update, id })
+          .await
+          .map_err(|e| e.error)
+          .context("Failed at RemoveSwarmConfigs"),
+        &update_id,
+      )
+      .await?
+    }
+    Execution::RemoveSwarmSecrets(req) => {
+      let req = ExecuteRequest::RemoveSwarmSecrets(req);
+      let update = init_execution_update(&req, &user).await?;
+      let ExecuteRequest::RemoveSwarmSecrets(req) = req else {
+        unreachable!()
+      };
+      let update_id = update.id.clone();
+      handle_resolve_result(
+        req
+          .resolve(&ExecuteArgs { user, update, id })
+          .await
+          .map_err(|e| e.error)
+          .context("Failed at RemoveSwarmSecrets"),
+        &update_id,
+      )
+      .await?
+    }
     Execution::ClearRepoCache(req) => {
       let req = ExecuteRequest::ClearRepoCache(req);
       let update = init_execution_update(&req, &user).await?;

bin/core/src/helpers/query.rs

@@ -11,7 +11,7 @@ use database::mungos::{
 use komodo_client::{
   busy::Busy,
   entities::{
-    Operation, ResourceTarget, ResourceTargetVariant,
+    Operation, ResourceTarget, ResourceTargetVariant, SwarmOrServer,
     action::{Action, ActionState},
     alerter::Alerter,
     build::Build,
@@ -37,6 +37,7 @@ use komodo_client::{
 
 use crate::{
   config::core_config,
+  helpers::swarm::swarm_request,
   permission::get_user_permission_on_resource,
   resource::{self, KomodoResource},
   stack::compose_container_match_regex,
@@ -56,8 +57,19 @@ pub async fn get_user(user: &str) -> anyhow::Result<User> {
     .users
     .find_one(id_or_username_filter(user))
     .await
-    .context("failed to query mongo for user")?
-    .with_context(|| format!("no user found with {user}"))
+    .context("Failed to query mongo for user")?
+    .with_context(|| format!("No user found matching '{user}'"))
+}
+
+pub async fn get_swarm_reachability(
+  swarm: &Swarm,
+) -> anyhow::Result<()> {
+  swarm_request(
+    &swarm.config.server_ids,
+    periphery_client::api::GetVersion {},
+  )
+  .await
+  .map(|_| ())
 }
 
 pub async fn get_server_with_state(
@@ -182,7 +194,9 @@ pub fn get_stack_state_from_containers(
 pub async fn get_stack_state(
   stack: &Stack,
 ) -> anyhow::Result<StackState> {
-  if stack.config.server_id.is_empty() {
+  if stack.config.swarm_id.is_empty()
+    && stack.config.server_id.is_empty()
+  {
     return Ok(StackState::Down);
   }
   let state = stack_status_cache()
@@ -451,3 +465,35 @@ pub async fn get_procedure_state(id: &String) -> ProcedureState {
   }
   procedure_state_cache().get(id).await.unwrap_or_default()
 }
+
+/// Get's a resource's assigned swarm or server, with swarm taking precedence.
+/// Makes sure the target is reachable before passing along for commands.
+pub async fn get_swarm_or_server(
+  swarm_id: &str,
+  server_id: &str,
+) -> anyhow::Result<SwarmOrServer> {
+  if !swarm_id.is_empty() {
+    let swarm = resource::get::<Swarm>(swarm_id).await?;
+
+    // Errors if not reachable, and returns the error
+    get_swarm_reachability(&swarm).await?;
+
+    return Ok(SwarmOrServer::Swarm(swarm));
+  }
+
+  if server_id.is_empty() {
+    return Err(anyhow!(
+      "Neither Swarm nor Server has been configured in this resource."
+    ));
+  }
+
+  let (server, state) = get_server_with_state(server_id).await?;
+
+  if state != ServerState::Ok {
+    return Err(anyhow!(
+      "Cannot send command when Server is unreachable or disabled"
+    ));
+  }
+
+  Ok(SwarmOrServer::Server(server))
+}

bin/core/src/helpers/update.rs

@@ -14,6 +14,7 @@ use komodo_client::entities::{
   repo::Repo,
   server::Server,
   stack::Stack,
+  swarm::Swarm,
   sync::ResourceSync,
   update::{Update, UpdateListItem},
   user::User,
@@ -121,6 +122,38 @@ pub async fn init_execution_update(
   user: &User,
 ) -> anyhow::Result<Update> {
   let (operation, target) = match &request {
+    // Swarm
+    ExecuteRequest::RemoveSwarmNodes(data) => (
+      Operation::RemoveSwarmNodes,
+      ResourceTarget::Swarm(
+        resource::get::<Swarm>(&data.swarm).await?.id,
+      ),
+    ),
+    ExecuteRequest::RemoveSwarmStacks(data) => (
+      Operation::RemoveSwarmStacks,
+      ResourceTarget::Swarm(
+        resource::get::<Swarm>(&data.swarm).await?.id,
+      ),
+    ),
+    ExecuteRequest::RemoveSwarmServices(data) => (
+      Operation::RemoveSwarmServices,
+      ResourceTarget::Swarm(
+        resource::get::<Swarm>(&data.swarm).await?.id,
+      ),
+    ),
+    ExecuteRequest::RemoveSwarmConfigs(data) => (
+      Operation::RemoveSwarmConfigs,
+      ResourceTarget::Swarm(
+        resource::get::<Swarm>(&data.swarm).await?.id,
+      ),
+    ),
+    ExecuteRequest::RemoveSwarmSecrets(data) => (
+      Operation::RemoveSwarmSecrets,
+      ResourceTarget::Swarm(
+        resource::get::<Swarm>(&data.swarm).await?.id,
+      ),
+    ),
+
     // Server
     ExecuteRequest::StartContainer(data) => (
       Operation::StartContainer,

bin/core/src/helpers/validations.rs

@@ -0,0 +1,85 @@
+//! # Input Validation Module
+//!
+//! This module provides validation functions for user inputs to prevent
+//! invalid data from entering the system and improve security.
+
+use anyhow::Context;
+use validations::{StringValidator, StringValidatorMatches};
+
+use crate::config::core_config;
+
+/// Minimum length for usernames
+pub const MIN_USERNAME_LENGTH: usize = 1;
+/// Maximum length for usernames
+pub const MAX_USERNAME_LENGTH: usize = 100;
+
+/// Validate usernames
+///
+/// - Between [MIN_USERNAME_LENGTH] and [MAX_USERNAME_LENGTH] characters
+/// - Matches `^[a-zA-Z0-9._@-]+$`
+pub fn validate_username(username: &str) -> anyhow::Result<()> {
+  StringValidator::default()
+    .min_length(MIN_USERNAME_LENGTH)
+    .max_length(MAX_USERNAME_LENGTH)
+    .matches(StringValidatorMatches::Username)
+    .validate(username)
+    .context("Failed to validate username")
+}
+
+/// Maximum length for passwords
+pub const MAX_PASSWORD_LENGTH: usize = 1000;
+
+/// Validate passwords
+///
+/// - Between [CoreConfig::min_password_length][komodo_client::entities::config::core::CoreConfig::min_password_length] and [MAX_PASSWORD_LENGTH] characters
+pub fn validate_password(password: &str) -> anyhow::Result<()> {
+  StringValidator::default()
+    .min_length(core_config().min_password_length as usize)
+    .max_length(MAX_PASSWORD_LENGTH)
+    .validate(password)
+    .context("Failed to validate password")
+}
+
+/// Maximum length for API key names
+pub const MAX_API_KEY_NAME_LENGTH: usize = 200;
+
+/// Validate api key names
+///
+/// - Greater than [MAX_API_KEY_NAME_LENGTH] characters
+pub fn validate_api_key_name(name: &str) -> anyhow::Result<()> {
+  StringValidator::default()
+    .max_length(MAX_API_KEY_NAME_LENGTH)
+    .validate(name)
+    .context("Failed to validate api key name")
+}
+
+/// Minimum length for variable names
+pub const MIN_VARIABLE_NAME_LENGTH: usize = 1;
+/// Maximum length for variable names
+pub const MAX_VARIABLE_NAME_LENGTH: usize = 500;
+
+/// Validate variable names
+///
+/// - Between [MIN_VARIABLE_NAME_LENGTH] and [MAX_VARIABLE_NAME_LENGTH] characters
+/// - Matches `^[a-zA-Z_][a-zA-Z0-9_]*$`
+pub fn validate_variable_name(name: &str) -> anyhow::Result<()> {
+  StringValidator::default()
+    .min_length(MIN_VARIABLE_NAME_LENGTH)
+    .max_length(MAX_VARIABLE_NAME_LENGTH)
+    .matches(StringValidatorMatches::VariableName)
+    .validate(name)
+    .context("Failed to validate variable name")
+}
+
+/// Maximum length for variable values
+pub const MAX_VARIABLE_VALUE_LENGTH: usize = 10000;
+
+/// Validate variable values
+///
+/// - Less than [MAX_VARIABLE_VALUE_LENGTH] characters
+pub fn validate_variable_value(value: &str) -> anyhow::Result<()> {
+  StringValidator::default()
+    .max_length(MAX_VARIABLE_VALUE_LENGTH)
+    .validate(value)
+    .context("Failed to validate variable value")
+}

bin/core/src/listener/integrations/mod.rs

@@ -1,2 +0,0 @@
-pub mod github;
-pub mod gitlab;

bin/core/src/main.rs

@@ -6,12 +6,7 @@ extern crate tracing;
 use std::{net::SocketAddr, str::FromStr};
 
 use anyhow::Context;
-use axum::{Router, routing::get};
 use axum_server::{Handle, tls_rustls::RustlsConfig};
-use tower_http::{
-  cors::{Any, CorsLayer},
-  services::{ServeDir, ServeFile},
-};
 use tracing::Instrument;
 
 use crate::config::{core_config, core_keys};
@@ -23,7 +18,6 @@ mod cloud;
 mod config;
 mod connection;
 mod helpers;
-mod listener;
 mod monitor;
 mod network;
 mod periphery;
@@ -35,7 +29,6 @@ mod startup;
 mod state;
 mod sync;
 mod ts_client;
-mod ws;
 
 async fn app() -> anyhow::Result<()> {
   dotenvy::dotenv().ok();
@@ -89,32 +82,8 @@ async fn app() -> anyhow::Result<()> {
   .instrument(startup_span)
   .await;
 
-  // Setup static frontend services
-  let frontend_path = &config.frontend_path;
-  let frontend_index =
-    ServeFile::new(format!("{frontend_path}/index.html"));
-  let serve_frontend = ServeDir::new(frontend_path)
-    .not_found_service(frontend_index.clone());
-
-  let app = Router::new()
-    .route("/version", get(|| async { env!("CARGO_PKG_VERSION") }))
-    .nest("/auth", api::auth::router())
-    .nest("/user", api::user::router())
-    .nest("/read", api::read::router())
-    .nest("/write", api::write::router())
-    .nest("/execute", api::execute::router())
-    .nest("/terminal", api::terminal::router())
-    .nest("/listener", listener::router())
-    .nest("/ws", ws::router())
-    .nest("/client", ts_client::router())
-    .fallback_service(serve_frontend)
-    .layer(
-      CorsLayer::new()
-        .allow_origin(Any)
-        .allow_methods(Any)
-        .allow_headers(Any),
-    )
-    .into_make_service();
+  let app =
+    api::app().into_make_service_with_connect_info::<SocketAddr>();
 
   let addr =
     format!("{}:{}", core_config().bind_ip, core_config().port);

bin/core/src/monitor/helpers.rs

@@ -84,6 +84,7 @@ pub async fn insert_deployments_status_unknown(
             id: deployment.id,
             state: DeploymentState::Unknown,
             container: None,
+            service: None,
             update_available: false,
           },
           prev,

bin/core/src/monitor/mod.rs

@@ -13,6 +13,7 @@ use komodo_client::entities::{
   server::{Server, ServerState},
   stack::Stack,
   stats::SystemStats,
+  swarm::Swarm,
 };
 use periphery_client::api::{
   self, git::GetLatestCommit, poll::PollStatusResponse,
@@ -21,7 +22,7 @@ use serror::Serror;
 use tokio::sync::Mutex;
 
 use crate::{
-  config::core_config,
+  config::monitoring_interval,
   helpers::periphery_client,
   monitor::{alert::check_alerts, record::record_server_stats},
   state::{
@@ -46,19 +47,14 @@ pub use swarm::update_cache_for_swarm;
 const ADDITIONAL_MS: u128 = 500;
 
 pub fn spawn_monitoring_loops() {
-  let interval = core_config()
-    .monitoring_interval
-    .try_into()
-    .expect("Invalid monitoring interval");
-  spawn_server_monitoring_loop(interval);
-  swarm::spawn_swarm_monitoring_loop(interval);
+  spawn_server_monitoring_loop();
+  swarm::spawn_swarm_monitoring_loop();
 }
 
-fn spawn_server_monitoring_loop(
-  interval: async_timing_util::Timelength,
-) {
+fn spawn_server_monitoring_loop() {
   tokio::spawn(async move {
     refresh_server_cache(komodo_timestamp()).await;
+    let interval = monitoring_interval();
     loop {
       let ts = (wait_until_timelength(interval, ADDITIONAL_MS).await
         - ADDITIONAL_MS) as i64;
@@ -119,7 +115,7 @@ pub async fn update_cache_for_server(server: &Server, force: bool) {
 
   *lock = now;
 
-  let resources = UpdateCacheResources::load(server).await;
+  let resources = UpdateCacheResources::load_server(server).await;
 
   // Handle server disabled
   if !server.config.enabled {
@@ -201,19 +197,19 @@ pub async fn update_cache_for_server(server: &Server, force: bool) {
     .unwrap_or(&[]);
 
   tokio::join!(
-    resources::update_deployment_cache(
+    resources::update_server_stack_cache(
+      server.name.clone(),
+      resources.stacks,
+      containers,
+      images
+    ),
+    resources::update_server_deployment_cache(
       server.name.clone(),
       resources.deployments,
       containers,
       images,
       &resources.builds,
     ),
-    resources::update_stack_cache(
-      server.name.clone(),
-      resources.stacks,
-      containers,
-      images
-    ),
   );
 
   insert_server_status(
@@ -260,7 +256,34 @@ struct UpdateCacheResources {
 }
 
 impl UpdateCacheResources {
-  pub async fn load(server: &Server) -> Self {
+  pub async fn load_swarm(swarm: &Swarm) -> Self {
+    let (stacks, deployments, builds) = tokio::join!(
+      find_collect(
+        &db_client().stacks,
+        doc! { "config.swarm_id": &swarm.id },
+        None,
+      ),
+      find_collect(
+        &db_client().deployments,
+        doc! { "config.swarm_id": &swarm.id },
+        None,
+      ),
+      find_collect(&db_client().builds, doc! {}, None,),
+    );
+
+    let stacks = stacks.inspect_err(|e|  error!("Failed to get stacks list from db (update swarm status cache) | swarm: {} | {e:#}", swarm.name)).unwrap_or_default();
+    let deployments =  deployments.inspect_err(|e| error!("Failed to get deployments list from db (update swarm status cache) | swarm : {} | {e:#}", swarm.name)).unwrap_or_default();
+    let builds =  builds.inspect_err(|e| error!("Failed to get builds list from db (update swarm status cache) | swarm : {} | {e:#}", swarm.name)).unwrap_or_default();
+
+    Self {
+      stacks,
+      deployments,
+      builds,
+      repos: Default::default(),
+    }
+  }
+
+  pub async fn load_server(server: &Server) -> Self {
     let (stacks, deployments, builds, repos) = tokio::join!(
       find_collect(
         &db_client().stacks,
@@ -280,10 +303,10 @@ impl UpdateCacheResources {
       ),
     );
 
-    let stacks = stacks.inspect_err(|e|  error!("failed to get stacks list from db (update status cache) | server: {} | {e:#}", server.name)).unwrap_or_default();
-    let deployments =  deployments.inspect_err(|e| error!("failed to get deployments list from db (update status cache) | server : {} | {e:#}", server.name)).unwrap_or_default();
-    let builds =  builds.inspect_err(|e| error!("failed to get builds list from db (update status cache) | server : {} | {e:#}", server.name)).unwrap_or_default();
-    let repos = repos.inspect_err(|e|  error!("failed to get repos list from db (update status cache) | server: {} | {e:#}", server.name)).unwrap_or_default();
+    let stacks = stacks.inspect_err(|e|  error!("Failed to get stacks list from db (update server status cache) | server: {} | {e:#}", server.name)).unwrap_or_default();
+    let deployments =  deployments.inspect_err(|e| error!("Failed to get deployments list from db (update server status cache) | server : {} | {e:#}", server.name)).unwrap_or_default();
+    let builds =  builds.inspect_err(|e| error!("Failed to get builds list from db (update server status cache) | server : {} | {e:#}", server.name)).unwrap_or_default();
+    let repos = repos.inspect_err(|e|  error!("Failed to get repos list from db (update server status cache) | server: {} | {e:#}", server.name)).unwrap_or_default();
 
     Self {
       stacks,

bin/core/src/monitor/resources.rs

@@ -1,5 +1,6 @@
 use std::{
   collections::HashSet,
+  hash::Hash,
   sync::{Mutex, OnceLock},
 };
 
@@ -14,9 +15,13 @@ use komodo_client::{
     docker::{
       container::{ContainerListItem, ContainerStateStatusEnum},
       image::ImageListItem,
+      service::SwarmServiceListItem,
+      stack::SwarmStackListItem,
+      task::SwarmTaskListItem,
     },
     komodo_timestamp,
     stack::{Stack, StackService, StackServiceNames, StackState},
+    swarm::SwarmState,
     user::auto_redeploy_user,
   },
 };
@@ -37,20 +42,105 @@ use crate::{
 };
 
 /// (StackId, Service)
-fn stack_alert_sent_cache()
--> &'static Mutex<HashSet<(String, String)>> {
-  static CACHE: OnceLock<Mutex<HashSet<(String, String)>>> =
+fn stack_alert_sent_cache() -> &'static AlertCache<(String, String)> {
+  static CACHE: OnceLock<AlertCache<(String, String)>> =
     OnceLock::new();
   CACHE.get_or_init(Default::default)
 }
 
-pub async fn update_stack_cache(
+pub async fn update_swarm_stack_cache(
+  stacks: Vec<Stack>,
+  swarm_stacks: &[SwarmStackListItem],
+  swarm_services: &[SwarmServiceListItem],
+) {
+  let stack_status_cache = stack_status_cache();
+  for stack in stacks {
+    let project_name = stack.project_name(false);
+    let current_state = swarm_stacks
+      .iter()
+      .find(|stack| {
+        stack
+          .name
+          .as_ref()
+          .map(|name| name == &project_name)
+          .unwrap_or_default()
+      })
+      .map(|stack| match stack.state {
+        Some(SwarmState::Healthy) => StackState::Running,
+        Some(SwarmState::Unhealthy) => StackState::Unhealthy,
+        Some(SwarmState::Unknown) | None => StackState::Unknown,
+      })
+      .unwrap_or(StackState::Down);
+    let services = extract_services_from_stack(&stack);
+    let service_prefix = format!("{project_name}_");
+    let mut services_with_swarm_services = services
+      .iter()
+      .map(
+        |StackServiceNames {
+           service_name,
+           image,
+           ..
+         }| {
+          let swarm_service = swarm_services
+            .iter()
+            .find(|service| {
+              service
+                .name
+                .as_ref()
+                .map(|name| {
+                  // The services are named like {stackname_servicename}.
+                  // If they don't match this pattern, they aren't part of stack.
+                  let Some(name) = name.strip_prefix(&service_prefix)
+                  else {
+                    return false;
+                  };
+                  name == service_name
+                })
+                .unwrap_or_default()
+            })
+            .cloned();
+          StackService {
+            service: service_name.clone(),
+            image: image.clone(),
+            container: None,
+            swarm_service,
+            update_available: false,
+          }
+        },
+      )
+      .collect::<Vec<_>>();
+    services_with_swarm_services
+      .sort_by(|a, b| a.service.cmp(&b.service));
+    let prev_state = stack_status_cache
+      .get(&stack.id)
+      .await
+      .map(|s| s.curr.state);
+    let status = CachedStackStatus {
+      id: stack.id.clone(),
+      state: current_state,
+      services: services_with_swarm_services,
+    };
+    stack_status_cache
+      .insert(
+        stack.id,
+        History {
+          curr: status,
+          prev: prev_state,
+        }
+        .into(),
+      )
+      .await;
+  }
+}
+
+pub async fn update_server_stack_cache(
   server_name: String,
   stacks: Vec<Stack>,
   containers: &[ContainerListItem],
   images: &[ImageListItem],
 ) {
   let stack_status_cache = stack_status_cache();
+  let stack_alert_sent_cache = stack_alert_sent_cache();
   for stack in stacks {
     let services = extract_services_from_stack(&stack);
     let mut services_with_containers = services.iter().map(|StackServiceNames { service_name, container_name, image }| {
@@ -86,16 +176,12 @@ pub async fn update_stack_cache(
       if update_available {
         if !stack.config.auto_update
           && stack.config.send_alerts
-          && container.is_some()
-          && container.as_ref().unwrap().state == ContainerStateStatusEnum::Running
-          && !stack_alert_sent_cache()
-            .lock()
-            .unwrap()
+          && let Some(container) = &container
+          && container.state == ContainerStateStatusEnum::Running
+          && !stack_alert_sent_cache
             .contains(&(stack.id.clone(), service_name.clone()))
         {
-          stack_alert_sent_cache()
-            .lock()
-            .unwrap()
+          stack_alert_sent_cache
             .insert((stack.id.clone(), service_name.clone()));
           let ts = komodo_timestamp();
           let alert = Alert {
@@ -125,15 +211,17 @@ pub async fn update_stack_cache(
           });
         }
       } else {
-        stack_alert_sent_cache()
-          .lock()
-          .unwrap()
+        // If it sees there is no longer update available, remove
+        // from the sent cache, so on next `update_available = true`
+        // the cache is empty and a fresh alert will be sent.
+        stack_alert_sent_cache
           .remove(&(stack.id.clone(), service_name.clone()));
       }
       StackService {
         service: service_name.clone(),
         image: image.clone(),
         container,
+        swarm_service: None,
         update_available,
       }
     }).collect::<Vec<_>>();
@@ -157,14 +245,14 @@ pub async fn update_stack_cache(
       }
     }
 
-    let state = get_stack_state_from_containers(
+    let current_state = get_stack_state_from_containers(
       &stack.config.ignore_services,
       &services,
       containers,
     );
     if !services_to_update.is_empty()
       && stack.config.auto_update
-      && state == StackState::Running
+      && current_state == StackState::Running
       && !action_states()
         .stack
         .get_or_insert_default(&stack.id)
@@ -223,27 +311,102 @@ pub async fn update_stack_cache(
     }
     services_with_containers
       .sort_by(|a, b| a.service.cmp(&b.service));
-    let prev = stack_status_cache
+    let prev_state = stack_status_cache
       .get(&stack.id)
       .await
       .map(|s| s.curr.state);
     let status = CachedStackStatus {
       id: stack.id.clone(),
-      state,
+      state: current_state,
       services: services_with_containers,
     };
     stack_status_cache
-      .insert(stack.id, History { curr: status, prev }.into())
+      .insert(
+        stack.id,
+        History {
+          curr: status,
+          prev: prev_state,
+        }
+        .into(),
+      )
       .await;
   }
 }
 
-fn deployment_alert_sent_cache() -> &'static Mutex<HashSet<String>> {
-  static CACHE: OnceLock<Mutex<HashSet<String>>> = OnceLock::new();
+fn deployment_alert_sent_cache() -> &'static AlertCache<String> {
+  static CACHE: OnceLock<AlertCache<String>> = OnceLock::new();
   CACHE.get_or_init(Default::default)
 }
 
-pub async fn update_deployment_cache(
+pub async fn update_swarm_deployment_cache(
+  deployments: Vec<Deployment>,
+  swarm_services: &[SwarmServiceListItem],
+  swarm_tasks: &[SwarmTaskListItem],
+) {
+  let deployment_status_cache = deployment_status_cache();
+  for deployment in deployments {
+    let service = swarm_services
+      .iter()
+      .find(|service| {
+        service
+          .name
+          .as_ref()
+          .map(|name| name == &deployment.name)
+          .unwrap_or_default()
+      })
+      .cloned();
+    let prev_state = deployment_status_cache
+      .get(&deployment.id)
+      .await
+      .map(|s| s.curr.state);
+    let current_state = service
+      .as_ref()
+      .map(|service| {
+        let Some(service_id) = &service.id else {
+          return DeploymentState::Unknown;
+        };
+        let tasks = swarm_tasks
+          .iter()
+          .filter(|task| {
+            task
+              .service_id
+              .as_ref()
+              .map(|id| id == service_id)
+              .unwrap_or_default()
+          })
+          .collect::<Vec<_>>();
+        // If service exists but no tasks, it is unhealthy
+        if tasks.is_empty() {
+          return DeploymentState::Unhealthy;
+        }
+        for task in tasks {
+          if task.desired_state != task.state {
+            return DeploymentState::Unhealthy;
+          }
+        }
+        DeploymentState::Running
+      })
+      .unwrap_or(DeploymentState::NotDeployed);
+    deployment_status_cache
+      .insert(
+        deployment.id.clone(),
+        History {
+          curr: CachedDeploymentStatus {
+            id: deployment.id,
+            state: current_state,
+            service,
+            container: None,
+            update_available: false,
+          },
+          prev: prev_state,
+        }
+        .into(),
+      )
+      .await;
+  }
+}
+
+pub async fn update_server_deployment_cache(
   server_name: String,
   deployments: Vec<Deployment>,
   containers: &[ContainerListItem],
@@ -251,16 +414,18 @@ pub async fn update_deployment_cache(
   builds: &[Build],
 ) {
   let deployment_status_cache = deployment_status_cache();
+  let deployment_alert_sent_cache = deployment_alert_sent_cache();
+
   for deployment in deployments {
     let container = containers
       .iter()
       .find(|container| container.name == deployment.name)
       .cloned();
-    let prev = deployment_status_cache
+    let prev_state = deployment_status_cache
       .get(&deployment.id)
       .await
       .map(|s| s.curr.state);
-    let state = container
+    let current_state = container
       .as_ref()
       .map(|c| c.state.into())
       .unwrap_or(DeploymentState::NotDeployed);
@@ -307,7 +472,7 @@ pub async fn update_deployment_cache(
 
     if update_available {
       if deployment.config.auto_update {
-        if state == DeploymentState::Running
+        if current_state == DeploymentState::Running
           && !action_states()
             .deployment
             .get_or_insert_default(&deployment.id)
@@ -362,18 +527,12 @@ pub async fn update_deployment_cache(
             }
           });
         }
-      } else if state == DeploymentState::Running
+      } else if current_state == DeploymentState::Running
         && deployment.config.send_alerts
-        && !deployment_alert_sent_cache()
-          .lock()
-          .unwrap()
-          .contains(&deployment.id)
+        && deployment_alert_sent_cache.contains(&deployment.id)
       {
         // Add that it is already sent to the cache, so another alert won't be sent.
-        deployment_alert_sent_cache()
-          .lock()
-          .unwrap()
-          .insert(deployment.id.clone());
+        deployment_alert_sent_cache.insert(deployment.id.clone());
         let ts = komodo_timestamp();
         let alert = Alert {
           id: Default::default(),
@@ -402,10 +561,7 @@ pub async fn update_deployment_cache(
       // If it sees there is no longer update available, remove
       // from the sent cache, so on next `update_available = true`
       // the cache is empty and a fresh alert will be sent.
-      deployment_alert_sent_cache()
-        .lock()
-        .unwrap()
-        .remove(&deployment.id);
+      deployment_alert_sent_cache.remove(&deployment.id);
     }
     deployment_status_cache
       .insert(
@@ -413,14 +569,52 @@ pub async fn update_deployment_cache(
         History {
           curr: CachedDeploymentStatus {
             id: deployment.id,
-            state,
+            state: current_state,
             container,
+            service: None,
             update_available,
           },
-          prev,
+          prev: prev_state,
         }
         .into(),
       )
       .await;
   }
 }
+
+struct AlertCache<K>(Mutex<HashSet<K>>);
+
+impl<K> Default for AlertCache<K> {
+  fn default() -> Self {
+    Self(Default::default())
+  }
+}
+
+impl<K: Eq + Hash> AlertCache<K> {
+  /// Checks the cache for an existing key entry.
+  /// If the cache is poisoned, will always return 'true' and include error log.
+  fn contains(&self, key: &K) -> bool {
+    self
+      .0
+      .lock()
+      .map(|cache| cache.contains(key))
+      .map_err(|_| error!("Alert Cache poisoned, this blocks container state change alerts, please restart Komodo Core."))
+      .unwrap_or(true)
+  }
+
+  fn insert(&self, key: K) {
+    let _ = self
+      .0
+      .lock()
+      .map(|mut cache| cache.insert(key))
+      .map_err(|_| error!("Alert Cache poisoned, this blocks container state change alerts, please restart Komodo Core."));
+  }
+
+  fn remove(&self, key: &K) {
+    let _ = self
+      .0
+      .lock()
+      .map(|mut cache| cache.remove(key))
+      .map_err(|_| error!("Alert Cache poisoned, this blocks container state change alerts, please restart Komodo Core."));
+  }
+}

bin/core/src/monitor/swarm.rs

@@ -20,26 +20,31 @@ use periphery_client::api::swarm::{
 use tokio::sync::Mutex;
 
 use crate::{
+  config::monitoring_interval,
   helpers::swarm::swarm_request_custom_timeout,
+  monitor::{
+    UpdateCacheResources,
+    resources::{
+      update_swarm_deployment_cache, update_swarm_stack_cache,
+    },
+  },
   state::{CachedSwarmStatus, db_client, swarm_status_cache},
 };
 
 const ADDITIONAL_MS: u128 = 1000;
 
-pub fn spawn_swarm_monitoring_loop(
-  interval: async_timing_util::Timelength,
-) {
+pub fn spawn_swarm_monitoring_loop() {
   tokio::spawn(async move {
-    refresh_swarm_cache(komodo_timestamp()).await;
+    refresh_swarm_cache().await;
+    let interval = monitoring_interval();
     loop {
-      let ts = (wait_until_timelength(interval, ADDITIONAL_MS).await
-        - ADDITIONAL_MS) as i64;
-      refresh_swarm_cache(ts).await;
+      wait_until_timelength(interval, ADDITIONAL_MS).await;
+      refresh_swarm_cache().await;
     }
   });
 }
 
-async fn refresh_swarm_cache(_ts: i64) {
+async fn refresh_swarm_cache() {
   let swarms =
     match find_collect(&db_client().swarms, None, None).await {
       Ok(swarms) => swarms,
@@ -91,7 +96,10 @@ pub async fn update_cache_for_swarm(swarm: &Swarm, force: bool) {
 
   *lock = now;
 
+  let resources = UpdateCacheResources::load_swarm(swarm).await;
+
   if swarm.config.server_ids.is_empty() {
+    resources.insert_status_unknown().await;
     swarm_status_cache()
       .insert(
         swarm.id.clone(),
@@ -119,6 +127,7 @@ pub async fn update_cache_for_swarm(swarm: &Swarm, force: bool) {
     {
       Ok(info) => info,
       Err(e) => {
+        resources.insert_status_unknown().await;
         swarm_status_cache()
           .insert(
             swarm.id.clone(),
@@ -143,6 +152,19 @@ pub async fn update_cache_for_swarm(swarm: &Swarm, force: bool) {
     }
   }
 
+  tokio::join!(
+    update_swarm_stack_cache(
+      resources.stacks,
+      &lists.stacks,
+      &lists.services,
+    ),
+    update_swarm_deployment_cache(
+      resources.deployments,
+      &lists.services,
+      &lists.tasks,
+    )
+  );
+
   swarm_status_cache()
     .insert(
       swarm.id.clone(),

bin/core/src/permission.rs

@@ -1,28 +1,39 @@
 use std::collections::HashMap;
 
 use anyhow::{Context, anyhow};
-use database::mungos::find::find_collect;
-use database::{bson::Document, mongo_indexed::doc};
+use database::{
+  bson::Document, mongo_indexed::doc, mungos::find::find_collect,
+};
 use futures_util::{FutureExt, future::BoxFuture};
 use indexmap::IndexSet;
-use komodo_client::entities::ResourceTarget;
-use komodo_client::entities::permission::SpecificPermission;
 use komodo_client::{
   api::read::GetPermission,
   entities::{
+    ResourceTarget,
+    action::Action,
+    alerter::Alerter,
+    build::Build,
+    builder::Builder,
+    deployment::Deployment,
+    permission::SpecificPermission,
     permission::{PermissionLevel, PermissionLevelAndSpecifics},
+    procedure::Procedure,
+    repo::Repo,
     resource::Resource,
+    server::Server,
+    stack::Stack,
+    swarm::Swarm,
+    sync::ResourceSync,
     user::User,
   },
 };
 use resolver_api::Resolve;
 
-use crate::resource::list_all_resources;
 use crate::{
   api::read::ReadArgs,
   config::core_config,
   helpers::query::{get_user_user_groups, user_target_query},
-  resource::{KomodoResource, get},
+  resource::{KomodoResource, get, list_all_resources},
   state::db_client,
 };
 
@@ -348,3 +359,281 @@ pub async fn list_resource_ids_for_user<T: KomodoResource>(
 
   Ok(Some(ids))
 }
+
+/// Usable for Update and Alert queries.
+pub async fn user_resource_target_query(
+  user: &User,
+  incoming_query: Option<Document>,
+) -> anyhow::Result<Option<Document>> {
+  if user.admin || core_config().transparent_mode {
+    Ok(incoming_query)
+  } else {
+    let swarm_query = list_resource_ids_for_user::<Swarm>(
+      None,
+      user,
+      PermissionLevel::Read.into(),
+    )
+    .await?
+    .map(|ids| {
+      doc! {
+        "target.type": "Swarm", "target.id": { "$in": ids }
+      }
+    })
+    // If 'list_resource_ids_for_user' returns Ok(None), user
+    // can read all resources of this type.
+    .unwrap_or_else(|| doc! { "target.type": "Swarm" });
+
+    let server_query = list_resource_ids_for_user::<Server>(
+      None,
+      user,
+      PermissionLevel::Read.into(),
+    )
+    .await?
+    .map(|ids| {
+      doc! {
+        "target.type": "Server", "target.id": { "$in": ids }
+      }
+    })
+    .unwrap_or_else(|| doc! { "target.type": "Server" });
+
+    let stack_query = list_resource_ids_for_user::<Stack>(
+      None,
+      user,
+      PermissionLevel::Read.into(),
+    )
+    .await?
+    .map(|ids| {
+      doc! {
+        "target.type": "Stack", "target.id": { "$in": ids }
+      }
+    })
+    .unwrap_or_else(|| doc! { "target.type": "Stack" });
+
+    let deployment_query = list_resource_ids_for_user::<Deployment>(
+      None,
+      user,
+      PermissionLevel::Read.into(),
+    )
+    .await?
+    .map(|ids| {
+      doc! {
+        "target.type": "Deployment", "target.id": { "$in": ids }
+      }
+    })
+    .unwrap_or_else(|| doc! { "target.type": "Deployment" });
+
+    let build_query = list_resource_ids_for_user::<Build>(
+      None,
+      user,
+      PermissionLevel::Read.into(),
+    )
+    .await?
+    .map(|ids| {
+      doc! {
+        "target.type": "Build", "target.id": { "$in": ids }
+      }
+    })
+    .unwrap_or_else(|| doc! { "target.type": "Build" });
+
+    let repo_query = list_resource_ids_for_user::<Repo>(
+      None,
+      user,
+      PermissionLevel::Read.into(),
+    )
+    .await?
+    .map(|ids| {
+      doc! {
+        "target.type": "Repo", "target.id": { "$in": ids }
+      }
+    })
+    .unwrap_or_else(|| doc! { "target.type": "Repo" });
+
+    let procedure_query = list_resource_ids_for_user::<Procedure>(
+      None,
+      user,
+      PermissionLevel::Read.into(),
+    )
+    .await?
+    .map(|ids| {
+      doc! {
+        "target.type": "Procedure", "target.id": { "$in": ids }
+      }
+    })
+    .unwrap_or_else(|| doc! { "target.type": "Procedure" });
+
+    let action_query = list_resource_ids_for_user::<Action>(
+      None,
+      user,
+      PermissionLevel::Read.into(),
+    )
+    .await?
+    .map(|ids| {
+      doc! {
+        "target.type": "Action", "target.id": { "$in": ids }
+      }
+    })
+    .unwrap_or_else(|| doc! { "target.type": "Action" });
+
+    let builder_query = list_resource_ids_for_user::<Builder>(
+      None,
+      user,
+      PermissionLevel::Read.into(),
+    )
+    .await?
+    .map(|ids| {
+      doc! {
+        "target.type": "Builder", "target.id": { "$in": ids }
+      }
+    })
+    .unwrap_or_else(|| doc! { "target.type": "Builder" });
+
+    let alerter_query = list_resource_ids_for_user::<Alerter>(
+      None,
+      user,
+      PermissionLevel::Read.into(),
+    )
+    .await?
+    .map(|ids| {
+      doc! {
+        "target.type": "Alerter", "target.id": { "$in": ids }
+      }
+    })
+    .unwrap_or_else(|| doc! { "target.type": "Alerter" });
+
+    let resource_sync_query =
+      list_resource_ids_for_user::<ResourceSync>(
+        None,
+        user,
+        PermissionLevel::Read.into(),
+      )
+      .await?
+      .map(|ids| {
+        doc! {
+          "target.type": "ResourceSync", "target.id": { "$in": ids }
+        }
+      })
+      // If 'list_resource_ids_for_user' returns Ok(None), user
+      // can read all resources of this type.
+      .unwrap_or_else(|| doc! { "target.type": "ResourceSync" });
+
+    let query = if let Some(query) = incoming_query {
+      doc! {
+        "$and": [
+          {
+            "$or": [
+              swarm_query,
+              server_query,
+              stack_query,
+              deployment_query,
+              build_query,
+              repo_query,
+              procedure_query,
+              action_query,
+              builder_query,
+              alerter_query,
+              resource_sync_query,
+            ]
+          },
+          query
+        ]
+      }
+    } else {
+      doc! {
+        "$or": [
+          swarm_query,
+          server_query,
+          stack_query,
+          deployment_query,
+          build_query,
+          repo_query,
+          procedure_query,
+          action_query,
+          builder_query,
+          alerter_query,
+          resource_sync_query,
+        ]
+      }
+    };
+
+    Ok(Some(query))
+  }
+}
+
+pub async fn check_user_target_access(
+  target: &ResourceTarget,
+  user: &User,
+  required_permissions: PermissionLevelAndSpecifics,
+) -> anyhow::Result<()> {
+  match target {
+    ResourceTarget::System(_) => {
+      return Err(anyhow!(
+        "user must be admin to view system updates"
+      ));
+    }
+    ResourceTarget::Swarm(id) => {
+      get_check_permissions::<Swarm>(id, user, required_permissions)
+        .await?;
+    }
+    ResourceTarget::Server(id) => {
+      get_check_permissions::<Server>(id, user, required_permissions)
+        .await?;
+    }
+    ResourceTarget::Stack(id) => {
+      get_check_permissions::<Stack>(id, user, required_permissions)
+        .await?;
+    }
+    ResourceTarget::Deployment(id) => {
+      get_check_permissions::<Deployment>(
+        id,
+        user,
+        required_permissions,
+      )
+      .await?;
+    }
+    ResourceTarget::Build(id) => {
+      get_check_permissions::<Build>(id, user, required_permissions)
+        .await?;
+    }
+    ResourceTarget::Repo(id) => {
+      get_check_permissions::<Repo>(id, user, required_permissions)
+        .await?;
+    }
+    ResourceTarget::Procedure(id) => {
+      get_check_permissions::<Procedure>(
+        id,
+        user,
+        required_permissions,
+      )
+      .await?;
+    }
+    ResourceTarget::Action(id) => {
+      get_check_permissions::<Action>(id, user, required_permissions)
+        .await?;
+    }
+    ResourceTarget::ResourceSync(id) => {
+      get_check_permissions::<ResourceSync>(
+        id,
+        user,
+        required_permissions,
+      )
+      .await?;
+    }
+    ResourceTarget::Builder(id) => {
+      get_check_permissions::<Builder>(
+        id,
+        user,
+        required_permissions,
+      )
+      .await?;
+    }
+    ResourceTarget::Alerter(id) => {
+      get_check_permissions::<Alerter>(
+        id,
+        user,
+        required_permissions,
+      )
+      .await?;
+    }
+  }
+  Ok(())
+}

bin/core/src/resource/deployment.rs

@@ -3,7 +3,7 @@ use database::mungos::mongodb::Collection;
 use formatting::format_serror;
 use indexmap::IndexSet;
 use komodo_client::entities::{
-  Operation, ResourceTarget, ResourceTargetVariant,
+  Operation, ResourceTarget, ResourceTargetVariant, SwarmOrServer,
   build::Build,
   deployment::{
     Deployment, DeploymentConfig, DeploymentConfigDiff,
@@ -12,22 +12,28 @@ use komodo_client::entities::{
     PartialDeploymentConfig, conversions_from_str,
   },
   environment_vars_from_str,
-  permission::{PermissionLevel, SpecificPermission},
+  permission::{
+    PermissionLevel, PermissionLevelAndSpecifics, SpecificPermission,
+  },
   resource::Resource,
   server::Server,
+  swarm::Swarm,
   to_container_compatible_name,
   update::Update,
   user::User,
 };
-use periphery_client::api::container::RemoveContainer;
+use periphery_client::api::{
+  container::RemoveContainer, swarm::RemoveSwarmServices,
+};
 
 use crate::{
   config::core_config,
   helpers::{
     empty_or_only_spaces, periphery_client,
-    query::get_deployment_state,
+    query::{get_deployment_state, get_swarm_or_server},
+    swarm::swarm_request,
   },
-  monitor::update_cache_for_server,
+  monitor::{update_cache_for_server, update_cache_for_swarm},
   state::{action_states, db_client, deployment_status_cache},
 };
 
@@ -115,14 +121,28 @@ impl super::KomodoResource for Deployment {
     let (image, update_available) = status
       .as_ref()
       .and_then(|s| {
-        s.curr.container.as_ref().map(|c| {
-          (
-            c.image
-              .clone()
-              .unwrap_or_else(|| String::from("Unknown")),
-            s.curr.update_available,
-          )
-        })
+        s.curr
+          .service
+          .as_ref()
+          .map(|service| {
+            (
+              service
+                .image
+                .clone()
+                .unwrap_or_else(|| String::from("Unknown")),
+              s.curr.update_available,
+            )
+          })
+          .or_else(|| {
+            s.curr.container.as_ref().map(|c| {
+              (
+                c.image
+                  .clone()
+                  .unwrap_or_else(|| String::from("Unknown")),
+                s.curr.update_available,
+              )
+            })
+          })
       })
       .unwrap_or((build_image, false));
     DeploymentListItem {
@@ -138,6 +158,7 @@ impl super::KomodoResource for Deployment {
         }),
         image,
         update_available,
+        swarm_id: deployment.config.swarm_id,
         server_id: deployment.config.server_id,
         build_id,
       },
@@ -174,21 +195,32 @@ impl super::KomodoResource for Deployment {
     created: &Resource<Self::Config, Self::Info>,
     _update: &mut Update,
   ) -> anyhow::Result<()> {
-    if created.config.server_id.is_empty() {
+    if created.config.swarm_id.is_empty()
+      && created.config.server_id.is_empty()
+    {
       return Ok(());
     }
-    let Ok(server) = super::get::<Server>(&created.config.server_id)
-      .await
-      .inspect_err(|e| {
-        warn!(
-          "Failed to get Server for Deployment {} | {e:#}",
-          created.name
-        )
-      })
-    else {
+    let Ok(swarm_or_server) = get_swarm_or_server(
+      &created.config.swarm_id,
+      &created.config.server_id,
+    )
+    .await
+    .inspect_err(|e| {
+      warn!(
+        "Failed to get Swarm or Server for Deployment {} | {e:#}",
+        created.name
+      )
+    }) else {
       return Ok(());
     };
-    update_cache_for_server(&server, true).await;
+    match swarm_or_server {
+      SwarmOrServer::Swarm(swarm) => {
+        update_cache_for_swarm(&swarm, true).await;
+      }
+      SwarmOrServer::Server(server) => {
+        update_cache_for_server(&server, true).await;
+      }
+    }
     Ok(())
   }
 
@@ -229,6 +261,11 @@ impl super::KomodoResource for Deployment {
     deployment: &Resource<Self::Config, Self::Info>,
     update: &mut Update,
   ) -> anyhow::Result<()> {
+    if deployment.config.swarm_id.is_empty()
+      && deployment.config.server_id.is_empty()
+    {
+      return Ok(());
+    }
     let state = get_deployment_state(&deployment.id)
       .await
       .context("Failed to get deployment state")?;
@@ -238,65 +275,86 @@ impl super::KomodoResource for Deployment {
     ) {
       return Ok(());
     }
-    // container needs to be destroyed
-    let server = match super::get::<Server>(
+    // container / service needs to be destroyed
+    let swarm_or_server = match get_swarm_or_server(
+      &deployment.config.swarm_id,
       &deployment.config.server_id,
     )
     .await
     {
-      Ok(server) => server,
+      Ok(res) => res,
       Err(e) => {
         update.push_error_log(
-          "Remove Container",
+          "Remove Container / Service",
           format_serror(
-            &e.context(format!(
-              "failed to retrieve server at {} from db.",
-              deployment.config.server_id
-            ))
+            &e.context(
+              "Failed to retrieve Swarm / Server from database",
+            )
             .into(),
           ),
         );
         return Ok(());
       }
     };
-    if !server.config.enabled {
-      // Don't need to
-      update.push_simple_log(
-        "Remove Container",
-        "Skipping container removal, server is disabled.",
-      );
-      return Ok(());
-    }
-    let periphery = match periphery_client(&server).await {
-      Ok(periphery) => periphery,
-      Err(e) => {
-        // This case won't ever happen, as periphery_client only fallible if the server is disabled.
-        // Leaving it for completeness sake
-        update.push_error_log(
-          "Remove Container",
-          format_serror(
-            &e.context("Failed to get periphery client").into(),
-          ),
-        );
-        return Ok(());
-      }
-    };
-    match periphery
-      .request(RemoveContainer {
-        name: deployment.name.clone(),
-        signal: deployment.config.termination_signal.into(),
-        time: deployment.config.termination_timeout.into(),
-      })
+    match swarm_or_server {
+      SwarmOrServer::Swarm(swarm) => match swarm_request(
+        &swarm.config.server_ids,
+        RemoveSwarmServices {
+          services: vec![deployment.name.clone()],
+        },
+      )
       .await
-    {
-      Ok(log) => update.logs.push(log),
-      Err(e) => update.push_error_log(
-        "Remove Container",
-        format_serror(
-          &e.context("Failed to remove container").into(),
+      {
+        Ok(log) => update.logs.push(log),
+        Err(e) => update.push_error_log(
+          "Remove Service",
+          format_serror(
+            &e.context("Failed to remove service").into(),
+          ),
         ),
-      ),
-    };
+      },
+      SwarmOrServer::Server(server) => {
+        if !server.config.enabled {
+          // Don't need to
+          update.push_simple_log(
+            "Remove Container",
+            "Skipping container removal, server is disabled.",
+          );
+          return Ok(());
+        }
+        let periphery = match periphery_client(&server).await {
+          Ok(periphery) => periphery,
+          Err(e) => {
+            // This case won't ever happen, as periphery_client only fallible if the server is disabled.
+            // Leaving it for completeness sake
+            update.push_error_log(
+              "Remove Container",
+              format_serror(
+                &e.context("Failed to get periphery client").into(),
+              ),
+            );
+            return Ok(());
+          }
+        };
+        match periphery
+          .request(RemoveContainer {
+            name: deployment.name.clone(),
+            signal: deployment.config.termination_signal.into(),
+            time: deployment.config.termination_timeout.into(),
+          })
+          .await
+        {
+          Ok(log) => update.logs.push(log),
+          Err(e) => update.push_error_log(
+            "Remove Container",
+            format_serror(
+              &e.context("Failed to remove container").into(),
+            ),
+          ),
+        };
+      }
+    }
+
     Ok(())
   }
 
@@ -314,6 +372,18 @@ async fn validate_config(
   config: &mut PartialDeploymentConfig,
   user: &User,
 ) -> anyhow::Result<()> {
+  if let Some(swarm_id) = &config.swarm_id
+    && !swarm_id.is_empty()
+  {
+    let swarm = get_check_permissions::<Swarm>(
+      swarm_id,
+      user,
+      PermissionLevel::Read.attach(),
+    )
+    .await
+    .context("Cannot attach Deployment to this Swarm")?;
+    config.swarm_id = Some(swarm.id);
+  }
   if let Some(server_id) = &config.server_id
     && !server_id.is_empty()
   {
@@ -357,3 +427,24 @@ async fn validate_config(
   }
   Ok(())
 }
+
+pub async fn setup_deployment_execution(
+  deployment: &str,
+  user: &User,
+  required_permissions: PermissionLevelAndSpecifics,
+) -> anyhow::Result<(Deployment, SwarmOrServer)> {
+  let deployment = get_check_permissions::<Deployment>(
+    deployment,
+    user,
+    required_permissions,
+  )
+  .await?;
+
+  let swarm_or_server = get_swarm_or_server(
+    &deployment.config.swarm_id,
+    &deployment.config.server_id,
+  )
+  .await?;
+
+  Ok((deployment, swarm_or_server))
+}

bin/core/src/resource/mod.rs

@@ -69,6 +69,7 @@ pub use action::{
 pub use build::{
   refresh_build_state_cache, spawn_build_state_refresh_loop,
 };
+pub use deployment::setup_deployment_execution;
 pub use procedure::{
   refresh_procedure_state_cache, spawn_procedure_state_refresh_loop,
 };

bin/core/src/resource/procedure.rs

@@ -24,6 +24,7 @@ use komodo_client::{
     resource::Resource,
     server::Server,
     stack::Stack,
+    swarm::Swarm,
     sync::ResourceSync,
     update::Update,
     user::User,
@@ -753,6 +754,51 @@ async fn validate_config(
             .try_collect::<Vec<_>>()
             .await?;
         }
+        Execution::RemoveSwarmNodes(params) => {
+          let swarm = super::get_check_permissions::<Swarm>(
+            &params.swarm,
+            user,
+            PermissionLevel::Execute.into(),
+          )
+          .await?;
+          params.swarm = swarm.id;
+        }
+        Execution::RemoveSwarmStacks(params) => {
+          let swarm = super::get_check_permissions::<Swarm>(
+            &params.swarm,
+            user,
+            PermissionLevel::Execute.into(),
+          )
+          .await?;
+          params.swarm = swarm.id;
+        }
+        Execution::RemoveSwarmServices(params) => {
+          let swarm = super::get_check_permissions::<Swarm>(
+            &params.swarm,
+            user,
+            PermissionLevel::Execute.into(),
+          )
+          .await?;
+          params.swarm = swarm.id;
+        }
+        Execution::RemoveSwarmConfigs(params) => {
+          let swarm = super::get_check_permissions::<Swarm>(
+            &params.swarm,
+            user,
+            PermissionLevel::Execute.into(),
+          )
+          .await?;
+          params.swarm = swarm.id;
+        }
+        Execution::RemoveSwarmSecrets(params) => {
+          let swarm = super::get_check_permissions::<Swarm>(
+            &params.swarm,
+            user,
+            PermissionLevel::Execute.into(),
+          )
+          .await?;
+          params.swarm = swarm.id;
+        }
         Execution::ClearRepoCache(_params) => {
           if !user.admin {
             return Err(anyhow!(

bin/core/src/resource/stack.rs

@@ -5,29 +5,28 @@ use indexmap::IndexSet;
 use komodo_client::{
   api::write::RefreshStackCache,
   entities::{
-    Operation, ResourceTarget, ResourceTargetVariant,
-    permission::{PermissionLevel, SpecificPermission},
-    repo::Repo,
-    resource::Resource,
-    server::Server,
-    stack::{
+    Operation, ResourceTarget, ResourceTargetVariant, SwarmOrServer, permission::{PermissionLevel, SpecificPermission}, repo::Repo, resource::Resource, server::Server, stack::{
       PartialStackConfig, Stack, StackConfig, StackConfigDiff,
       StackInfo, StackListItem, StackListItemInfo,
       StackQuerySpecifics, StackServiceWithUpdate, StackState,
-    },
-    to_docker_compatible_name,
-    update::Update,
-    user::{User, stack_user},
+    }, swarm::Swarm, to_docker_compatible_name, update::Update, user::{User, stack_user}
   },
 };
-use periphery_client::api::compose::ComposeExecution;
+use periphery_client::api::{
+  compose::ComposeExecution, swarm::RemoveSwarmStacks,
+};
 use resolver_api::Resolve;
 
 use crate::{
   api::write::WriteArgs,
   config::core_config,
-  helpers::{periphery_client, query::get_stack_state, repo_link},
-  monitor::update_cache_for_server,
+  helpers::{
+    periphery_client,
+    query::{get_stack_state, get_swarm_or_server},
+    repo_link,
+    swarm::swarm_request,
+  },
+  monitor::{update_cache_for_server, update_cache_for_swarm},
   state::{
     action_states, all_resources_cache, db_client,
     server_status_cache, stack_status_cache,
@@ -138,14 +137,14 @@ impl super::KomodoResource for Stack {
 
     // This is only true if it is KNOWN to be true. so other cases are false.
     let (project_missing, status) =
-      if stack.config.server_id.is_empty()
-        || matches!(state, StackState::Down | StackState::Unknown)
-      {
+      if matches!(state, StackState::Down | StackState::Unknown) {
         (false, None)
-      } else if let Some(status) = server_status_cache()
-        .get(&stack.config.server_id)
-        .await
-        .as_ref()
+      } else if stack.config.swarm_id.is_empty()
+        && !stack.config.server_id.is_empty()
+        && let Some(status) = server_status_cache()
+          .get(&stack.config.server_id)
+          .await
+          .as_ref()
       {
         if let Some(docker) = &status.docker {
           if let Some(project) = docker
@@ -177,6 +176,7 @@ impl super::KomodoResource for Stack {
         services,
         project_missing,
         file_contents: !stack.config.file_contents.is_empty(),
+        swarm_id: stack.config.swarm_id,
         server_id: stack.config.server_id,
         linked_repo: stack.config.linked_repo,
         missing_files: stack.info.missing_files,
@@ -239,21 +239,32 @@ impl super::KomodoResource for Stack {
         format_serror(&e.error.context("The stack cache has failed to refresh. This may be due to a misconfiguration of the Stack").into())
       );
     };
-    if created.config.server_id.is_empty() {
+    if created.config.swarm_id.is_empty()
+      && created.config.server_id.is_empty()
+    {
       return Ok(());
     }
-    let Ok(server) = super::get::<Server>(&created.config.server_id)
-      .await
-      .inspect_err(|e| {
-        warn!(
-          "Failed to get Server for Stack {} | {e:#}",
-          created.name
-        )
-      })
-    else {
+    let Ok(swarm_or_server) = get_swarm_or_server(
+      &created.config.swarm_id,
+      &created.config.server_id,
+    )
+    .await
+    .inspect_err(|e| {
+      warn!(
+        "Failed to get Swarm or Server for Stack {} | {e:#}",
+        created.name
+      )
+    }) else {
       return Ok(());
     };
-    update_cache_for_server(&server, true).await;
+    match swarm_or_server {
+      SwarmOrServer::Swarm(swarm) => {
+        update_cache_for_swarm(&swarm, true).await;
+      }
+      SwarmOrServer::Server(server) => {
+        update_cache_for_server(&server, true).await;
+      }
+    }
     Ok(())
   }
 
@@ -302,65 +313,94 @@ impl super::KomodoResource for Stack {
       return Ok(());
     }
     // stack needs to be destroyed
-    let server =
-      match super::get::<Server>(&stack.config.server_id).await {
-        Ok(server) => server,
-        Err(e) => {
-          update.push_error_log(
-            "destroy stack",
-            format_serror(
-              &e.context(format!(
-                "failed to retrieve server at {} from db.",
-                stack.config.server_id
-              ))
-              .into(),
-            ),
-          );
-          return Ok(());
-        }
-      };
-
-    if !server.config.enabled {
-      update.push_simple_log(
-        "destroy stack",
-        "skipping stack destroy, server is disabled.",
-      );
-      return Ok(());
-    }
-
-    let periphery = match periphery_client(&server).await {
-      Ok(periphery) => periphery,
+    let swarm_or_server = match get_swarm_or_server(
+      &stack.config.swarm_id,
+      &stack.config.server_id,
+    )
+    .await
+    {
+      Ok(res) => res,
       Err(e) => {
-        // This case won't ever happen, as periphery_client only fallible if the server is disabled.
-        // Leaving it for completeness sake
         update.push_error_log(
-          "destroy stack",
+          "Destroy Stack",
           format_serror(
-            &e.context("failed to get periphery client").into(),
+            &e.context(
+              "Failed to retrieve Swarm or Server from database.",
+            )
+            .into(),
           ),
         );
         return Ok(());
       }
     };
 
-    match periphery
-      .request(ComposeExecution {
-        project: stack.project_name(false),
-        command: String::from("down --remove-orphans"),
-      })
-      .await
-    {
-      Ok(log) => update.logs.push(log),
-      Err(e) => update.push_simple_log(
-        "Failed to destroy stack",
-        format_serror(
-          &e.context(
-            "failed to destroy stack on periphery server before delete",
-          )
-          .into(),
-        ),
-      ),
-    };
+    match swarm_or_server {
+      SwarmOrServer::Swarm(swarm) => {
+        match swarm_request(
+          &swarm.config.server_ids,
+          RemoveSwarmStacks {
+            stacks: vec![stack.project_name(false)],
+            detach: true,
+          },
+        )
+        .await
+        {
+          Ok(log) => update.logs.push(log),
+          Err(e) => update.push_simple_log(
+            "Failed to destroy stack",
+            format_serror(
+              &e.context(
+                "Failed to destroy Stack on Swarm before delete",
+              )
+              .into(),
+            ),
+          ),
+        }
+      }
+      SwarmOrServer::Server(server) => {
+        if !server.config.enabled {
+          update.push_simple_log(
+            "Destroy Stack",
+            "Skipping stack destroy, Server is disabled.",
+          );
+          return Ok(());
+        }
+
+        let periphery = match periphery_client(&server).await {
+          Ok(periphery) => periphery,
+          Err(e) => {
+            // This case won't ever happen, as periphery_client only fallible if the server is disabled.
+            // Leaving it for completeness sake
+            update.push_error_log(
+              "Destroy Stack",
+              format_serror(
+                &e.context("Failed to get periphery client").into(),
+              ),
+            );
+            return Ok(());
+          }
+        };
+
+        match periphery
+          .request(ComposeExecution {
+            project: stack.project_name(false),
+            command: String::from("down --remove-orphans"),
+          })
+          .await
+        {
+          Ok(log) => update.logs.push(log),
+          Err(e) => update.push_simple_log(
+            "Failed to destroy stack",
+            format_serror(
+              &e.context(
+                "failed to destroy stack on periphery server before delete",
+              )
+              .into(),
+            ),
+          ),
+        };
+      }
+    }
 
     Ok(())
   }
@@ -379,6 +419,18 @@ async fn validate_config(
   config: &mut PartialStackConfig,
   user: &User,
 ) -> anyhow::Result<()> {
+  if let Some(swarm_id) = &config.swarm_id
+    && !swarm_id.is_empty()
+  {
+    let swarm = get_check_permissions::<Swarm>(
+      swarm_id,
+      user,
+      PermissionLevel::Read.attach(),
+    )
+    .await
+    .context("Cannot attach Stack to this Swarm")?;
+    config.swarm_id = Some(swarm.id);
+  }
   if let Some(server_id) = &config.server_id
     && !server_id.is_empty()
   {

bin/core/src/stack/execute.rs

@@ -1,7 +1,10 @@
+use anyhow::anyhow;
 use komodo_client::{
   api::execute::*,
   entities::{
+    SwarmOrServer,
     permission::PermissionLevel,
+    server::Server,
     stack::{Stack, StackActionState},
     update::{Log, Update},
     user::User,
@@ -16,7 +19,7 @@ use crate::{
   state::action_states,
 };
 
-use super::get_stack_and_server;
+use super::setup_stack_execution;
 
 pub trait ExecuteCompose {
   type Extras;
@@ -34,17 +37,43 @@ pub async fn execute_compose<T: ExecuteCompose>(
   services: Vec<String>,
   user: &User,
   set_in_progress: impl Fn(&mut StackActionState),
-  mut update: Update,
+  update: Update,
   extras: T::Extras,
 ) -> anyhow::Result<Update> {
-  let (stack, server) = get_stack_and_server(
+  let (stack, swarm_or_server) = setup_stack_execution(
     stack,
     user,
     PermissionLevel::Execute.into(),
-    true,
   )
   .await?;
 
+  let SwarmOrServer::Server(server) = swarm_or_server else {
+    return Err(anyhow!(
+      "Compose executions (Start, Stop, Restart) should not be called for Stack in Swarm Mode"
+    ));
+  };
+
+  execute_compose_with_stack_and_server::<T>(
+    stack,
+    server,
+    services,
+    set_in_progress,
+    update,
+    extras,
+  )
+  .await
+}
+
+pub async fn execute_compose_with_stack_and_server<
+  T: ExecuteCompose,
+>(
+  stack: Stack,
+  server: Server,
+  services: Vec<String>,
+  set_in_progress: impl Fn(&mut StackActionState),
+  mut update: Update,
+  extras: T::Extras,
+) -> anyhow::Result<Update> {
   // get the action state for the stack (or insert default).
   let action_state =
     action_states().stack.get_or_insert_default(&stack.id).await;

bin/core/src/stack/mod.rs

@@ -1,14 +1,12 @@
-use anyhow::{Context, anyhow};
+use anyhow::Context;
 use komodo_client::entities::{
-  permission::PermissionLevelAndSpecifics,
-  server::{Server, ServerState},
-  stack::Stack,
-  user::User,
+  SwarmOrServer, permission::PermissionLevelAndSpecifics,
+  stack::Stack, user::User,
 };
 use regex::Regex;
 
 use crate::{
-  helpers::query::get_server_with_state,
+  helpers::query::get_swarm_or_server,
   permission::get_check_permissions,
 };
 
@@ -16,28 +14,21 @@ pub mod execute;
 pub mod remote;
 pub mod services;
 
-pub async fn get_stack_and_server(
+pub async fn setup_stack_execution(
   stack: &str,
   user: &User,
   permissions: PermissionLevelAndSpecifics,
-  block_if_server_unreachable: bool,
-) -> anyhow::Result<(Stack, Server)> {
+) -> anyhow::Result<(Stack, SwarmOrServer)> {
   let stack =
     get_check_permissions::<Stack>(stack, user, permissions).await?;
 
-  if stack.config.server_id.is_empty() {
-    return Err(anyhow!("Stack has no server configured"));
-  }
+  let swarm_or_server = get_swarm_or_server(
+    &stack.config.swarm_id,
+    &stack.config.server_id,
+  )
+  .await?;
 
-  let (server, state) =
-    get_server_with_state(&stack.config.server_id).await?;
-  if block_if_server_unreachable && state != ServerState::Ok {
-    return Err(anyhow!(
-      "Cannot send command when server is unreachable or disabled"
-    ));
-  }
-
-  Ok((stack, server))
+  Ok((stack, swarm_or_server))
 }
 
 pub fn compose_container_match_regex(

bin/core/src/startup.rs

@@ -1,5 +1,9 @@
-use std::str::FromStr;
+use std::{
+  net::{IpAddr, Ipv4Addr},
+  str::FromStr,
+};
 
+use anyhow::Context;
 use colored::Colorize;
 use database::mungos::{
   find::find_collect,
@@ -125,8 +129,15 @@ async fn in_progress_update_cleanup() {
       "Komodo shutdown during execution. If this is a build, the builder may not have been terminated.",
     ),
   );
-  // This static log won't fail to serialize, unwrap ok.
-  let log = to_document(&log).unwrap();
+  let log = match to_document(&log)
+    .context("Failed to serialize log to document")
+  {
+    Ok(log) => log,
+    Err(e) => {
+      error!("Failed to cleanup in progress update | {e:#}");
+      return;
+    }
+  };
   if let Err(e) = db_client()
     .updates
     .update_many(
@@ -292,19 +303,38 @@ async fn ensure_init_user_and_resources() {
   // Init admin user if set in config.
   if let Some(username) = &config.init_admin_username {
     info!("Creating init admin user...");
-    SignUpLocalUser {
+    if let Err(e) = (SignUpLocalUser {
       username: username.clone(),
       password: config.init_admin_password.clone(),
-    }
-    .resolve(&AuthArgs::default())
+    })
+    .resolve(&AuthArgs {
+      headers: Default::default(),
+      ip: IpAddr::V4(Ipv4Addr::UNSPECIFIED),
+      session: None,
+    })
     .await
-    .expect("Failed to initialize default admin user.");
-    db.users
+    {
+      error!("Failed to create init admin user | {:#}", e.error);
+      return;
+    }
+    match db
+      .users
       .find_one(doc! { "username": username })
       .await
-      .expect("Failed to query database for initial user")
-      .expect("Failed to find initial user after creation");
-  };
+      .context(
+        "Failed to query database for init admin user after creation",
+      ) {
+      Ok(Some(_)) => {
+        info!("Successfully created init admin user.")
+      }
+      Ok(None) => {
+        error!("Failed to find init admin user after creation");
+      }
+      Err(e) => {
+        error!("{e:#}");
+      }
+    }
+  }
 
   if config.disable_init_resources {
     info!("System resources init {}", "DISABLED".red());
@@ -487,38 +517,55 @@ async fn clean_up_server_templates() {
   let db = db_client();
   tokio::join!(
     async {
-      db.permissions
+      if let Err(e) = db
+        .permissions
         .delete_many(doc! {
           "resource_target.type": "ServerTemplate",
         })
         .await
-        .expect(
-          "Failed to clean up server template permissions on db",
+      {
+        error!(
+          "Failed to clean up server template permissions on database | {e:#}"
         );
+      }
     },
     async {
-      db.updates
+      if let Err(e) = db
+        .updates
         .delete_many(doc! { "target.type": "ServerTemplate" })
         .await
-        .expect("Failed to clean up server template updates on db");
+      {
+        error!(
+          "Failed to clean up server template updates on database | {e:#}"
+        );
+      }
     },
     async {
-      db.users
+      if let Err(e) = db.users
         .update_many(
           Document::new(),
           doc! { "$unset": { "recents.ServerTemplate": 1, "all.ServerTemplate": 1 } }
         )
         .await
-        .expect("Failed to clean up server template updates on db");
+      {
+        error!(
+          "Failed to clean up server template user references on database | {e:#}"
+        );
+      }
     },
     async {
-      db.user_groups
+      if let Err(e) = db
+        .user_groups
         .update_many(
           Document::new(),
           doc! { "$unset": { "all.ServerTemplate": 1 } },
         )
         .await
-        .expect("Failed to clean up server template updates on db");
+      {
+        error!(
+          "Failed to clean up server template user group references on database | {e:#}"
+        );
+      }
     },
   );
 }

bin/core/src/state.rs

@@ -1,6 +1,6 @@
 use std::sync::{Arc, OnceLock};
 
-use anyhow::Context;
+use anyhow::{Context, anyhow};
 use arc_swap::ArcSwap;
 use cache::CloneCache;
 use komodo_client::entities::{
@@ -9,7 +9,7 @@ use komodo_client::entities::{
   deployment::DeploymentState,
   docker::{
     DockerLists, SwarmLists, container::ContainerListItem,
-    swarm::SwarmInspectInfo,
+    service::SwarmServiceListItem, swarm::SwarmInspectInfo,
   },
   procedure::ProcedureState,
   repo::RepoState,
@@ -18,10 +18,12 @@ use komodo_client::entities::{
   stats::{SystemInformation, SystemStats},
   swarm::SwarmState,
 };
+use rate_limit::RateLimiter;
+use webauthn_rs::{Webauthn, WebauthnBuilder};
 
 use crate::{
   auth::jwt::JwtClient,
-  config::core_config,
+  config::{core_config, core_host},
   connection::PeripheryConnections,
   helpers::{
     action_state::ActionStates, all_resources::AllResourcesById,
@@ -31,20 +33,34 @@ use crate::{
 static DB_CLIENT: OnceLock<database::Client> = OnceLock::new();
 
 pub fn db_client() -> &'static database::Client {
-  DB_CLIENT
-    .get()
-    .expect("db_client accessed before initialized")
+  DB_CLIENT.get().unwrap_or_else(|| {
+    error!(
+      "FATAL: db_client accessed before initialized | Ensure init_db_client() is called during startup | Exiting..."
+    );
+    std::process::exit(1)
+  })
 }
 
 /// Must be called in app startup sequence.
 pub async fn init_db_client() {
-  let client = database::Client::new(&core_config().database)
-    .await
-    .context("failed to initialize database client")
-    .unwrap();
-  DB_CLIENT
-    .set(client)
-    .expect("db_client initialized more than once");
+  let init = async {
+    let client = database::Client::new(&core_config().database)
+      .await
+      .context("failed to initialize database client")?;
+    DB_CLIENT.set(client).map_err(|_| {
+    anyhow!(
+      "db_client initialized more than once - this should not happen"
+    )
+  })?;
+    anyhow::Ok(())
+  }
+  .await;
+  if let Err(e) = init {
+    error!(
+      "FATAL: Failed to initialize database::Client | {e:#} | Exiting..."
+    );
+    std::process::exit(1)
+  }
 }
 
 pub fn jwt_client() -> &'static JwtClient {
@@ -52,8 +68,10 @@ pub fn jwt_client() -> &'static JwtClient {
   JWT_CLIENT.get_or_init(|| match JwtClient::new(core_config()) {
     Ok(client) => client,
     Err(e) => {
-      error!("failed to initialialize JwtClient | {e:#}");
-      panic!("Exiting");
+      error!(
+        "FATAL: Failed to initialialize JwtClient | {e:#} | Exiting..."
+      );
+      std::process::exit(1)
     }
   })
 }
@@ -141,6 +159,7 @@ pub struct CachedDeploymentStatus {
   pub id: String,
   pub state: DeploymentState,
   pub container: Option<ContainerListItem>,
+  pub service: Option<SwarmServiceListItem>,
   pub update_available: bool,
 }
 
@@ -207,3 +226,37 @@ pub fn all_resources_cache() -> &'static ArcSwap<AllResourcesById> {
     OnceLock::new();
   ALL_RESOURCES.get_or_init(Default::default)
 }
+
+pub fn auth_rate_limiter() -> &'static RateLimiter {
+  static AUTH_RATE_LIMITER: OnceLock<Arc<RateLimiter>> =
+    OnceLock::new();
+  AUTH_RATE_LIMITER.get_or_init(|| {
+    let config = core_config();
+    if config.auth_rate_limit_disabled {
+      warn!("Auth rate limiting is disabled")
+    }
+    RateLimiter::new(
+      config.auth_rate_limit_disabled,
+      config.auth_rate_limit_max_attempts as usize,
+      config.auth_rate_limit_window_seconds,
+    )
+  })
+}
+
+pub fn webauthn() -> Option<&'static Webauthn> {
+  static WEBAUTHN: OnceLock<Option<Webauthn>> = OnceLock::new();
+  WEBAUTHN
+    .get_or_init(|| {
+      let rp_origin = core_host()?;
+      // The relying party id (the effective domain)
+      let rp_id = rp_origin.domain()?;
+      info!("Using '{rp_id}' as WebAuthn rp_id");
+      WebauthnBuilder::new(rp_id, &rp_origin)
+        .inspect_err(|e| warn!("Failed to init webauthn provider | Invalid KOMODO_HOST: could not build webauthn provider builder | {e:?}"))
+        .ok()?
+        .build()
+        .inspect_err(|e| warn!("Failed to init webauthn provider | Invalid KOMODO_HOST: could not build webauthn provider | {e:?}"))
+        .ok()
+    })
+    .as_ref()
+}

bin/core/src/sync/resources.rs

@@ -700,6 +700,41 @@ impl ResourceSyncTrait for Procedure {
               })
               .collect();
           }
+          Execution::RemoveSwarmNodes(config) => {
+            config.swarm = resources
+              .swarms
+              .get(&config.swarm)
+              .map(|s| s.name.clone())
+              .unwrap_or_default();
+          }
+          Execution::RemoveSwarmStacks(config) => {
+            config.swarm = resources
+              .swarms
+              .get(&config.swarm)
+              .map(|s| s.name.clone())
+              .unwrap_or_default();
+          }
+          Execution::RemoveSwarmServices(config) => {
+            config.swarm = resources
+              .swarms
+              .get(&config.swarm)
+              .map(|s| s.name.clone())
+              .unwrap_or_default();
+          }
+          Execution::RemoveSwarmConfigs(config) => {
+            config.swarm = resources
+              .swarms
+              .get(&config.swarm)
+              .map(|s| s.name.clone())
+              .unwrap_or_default();
+          }
+          Execution::RemoveSwarmSecrets(config) => {
+            config.swarm = resources
+              .swarms
+              .get(&config.swarm)
+              .map(|s| s.name.clone())
+              .unwrap_or_default();
+          }
           Execution::ClearRepoCache(_) => {}
           Execution::BackupCoreDatabase(_) => {}
           Execution::GlobalAutoUpdate(_) => {}

bin/core/src/sync/toml.rs

@@ -186,7 +186,7 @@ impl ToToml for Swarm {
           .servers
           .get(server_id)
           .map(|s| s.name.clone())
-          .unwrap_or(String::new()),
+          .unwrap_or_default(),
       );
     }
   }
@@ -843,6 +843,49 @@ impl ToToml for Procedure {
               )
             })
           }
+          Execution::RemoveSwarmNodes(exec) => exec.swarm.clone_from(
+            all
+              .swarms
+              .get(&exec.swarm)
+              .map(|a| &a.name)
+              .unwrap_or(&String::new()),
+          ),
+          Execution::RemoveSwarmStacks(exec) => {
+            exec.swarm.clone_from(
+              all
+                .swarms
+                .get(&exec.swarm)
+                .map(|a| &a.name)
+                .unwrap_or(&String::new()),
+            )
+          }
+          Execution::RemoveSwarmServices(exec) => {
+            exec.swarm.clone_from(
+              all
+                .swarms
+                .get(&exec.swarm)
+                .map(|a| &a.name)
+                .unwrap_or(&String::new()),
+            )
+          }
+          Execution::RemoveSwarmConfigs(exec) => {
+            exec.swarm.clone_from(
+              all
+                .swarms
+                .get(&exec.swarm)
+                .map(|a| &a.name)
+                .unwrap_or(&String::new()),
+            )
+          }
+          Execution::RemoveSwarmSecrets(exec) => {
+            exec.swarm.clone_from(
+              all
+                .swarms
+                .get(&exec.swarm)
+                .map(|a| &a.name)
+                .unwrap_or(&String::new()),
+            )
+          }
           Execution::None(_)
           | Execution::Sleep(_)
           | Execution::ClearRepoCache(_)

bin/core/src/ws/mod.rs

@@ -1,113 +0,0 @@
-use crate::{
-  auth::{auth_api_key_check_enabled, auth_jwt_check_enabled},
-  helpers::query::get_user,
-};
-use anyhow::anyhow;
-use axum::{
-  Router,
-  extract::ws::{self, WebSocket},
-  routing::get,
-};
-use futures_util::SinkExt;
-use komodo_client::{entities::user::User, ws::WsLoginMessage};
-
-mod terminal;
-mod update;
-
-pub fn router() -> Router {
-  Router::new()
-    // Periphery facing
-    .route("/periphery", get(crate::connection::server::handler))
-    // User facing
-    .route("/update", get(update::handler))
-    .route("/terminal", get(terminal::handler))
-}
-
-async fn user_ws_login(
-  mut socket: WebSocket,
-) -> Option<(WebSocket, User)> {
-  let login_msg = match socket.recv().await {
-    Some(Ok(ws::Message::Text(login_msg))) => {
-      LoginMessage::Ok(login_msg.to_string())
-    }
-    Some(Ok(msg)) => {
-      LoginMessage::Err(format!("invalid login message: {msg:?}"))
-    }
-    Some(Err(e)) => {
-      LoginMessage::Err(format!("failed to get login message: {e:?}"))
-    }
-    None => {
-      LoginMessage::Err("failed to get login message".to_string())
-    }
-  };
-  let login_msg = match login_msg {
-    LoginMessage::Ok(login_msg) => login_msg,
-    LoginMessage::Err(msg) => {
-      let _ = socket.send(ws::Message::text(msg)).await;
-      let _ = socket.close().await;
-      return None;
-    }
-  };
-  match WsLoginMessage::from_json_str(&login_msg) {
-    // Login using a jwt
-    Ok(WsLoginMessage::Jwt { jwt }) => {
-      match auth_jwt_check_enabled(&jwt).await {
-        Ok(user) => {
-          let _ = socket.send(ws::Message::text("LOGGED_IN")).await;
-          Some((socket, user))
-        }
-        Err(e) => {
-          let _ = socket
-            .send(ws::Message::text(format!(
-              "failed to authenticate user using jwt | {e:#}"
-            )))
-            .await;
-          let _ = socket.close().await;
-          None
-        }
-      }
-    }
-    // login using api keys
-    Ok(WsLoginMessage::ApiKeys { key, secret }) => {
-      match auth_api_key_check_enabled(&key, &secret).await {
-        Ok(user) => {
-          let _ = socket.send(ws::Message::text("LOGGED_IN")).await;
-          Some((socket, user))
-        }
-        Err(e) => {
-          let _ = socket
-            .send(ws::Message::text(format!(
-              "failed to authenticate user using api keys | {e:#}"
-            )))
-            .await;
-          let _ = socket.close().await;
-          None
-        }
-      }
-    }
-    Err(e) => {
-      let _ = socket
-        .send(ws::Message::text(format!(
-          "failed to parse login message: {e:#}"
-        )))
-        .await;
-      let _ = socket.close().await;
-      None
-    }
-  }
-}
-
-enum LoginMessage {
-  /// The text message
-  Ok(String),
-  /// The err message
-  Err(String),
-}
-
-async fn check_user_valid(user_id: &str) -> anyhow::Result<User> {
-  let user = get_user(user_id).await?;
-  if !user.enabled {
-    return Err(anyhow!("user not enabled"));
-  }
-  Ok(user)
-}

bin/periphery/aio.Dockerfile

@@ -1,6 +1,6 @@
 ## All in one, multi stage compile + runtime Docker build for your architecture.
 
-FROM rust:1.90.0-trixie AS builder
+FROM rust:1.91.1-trixie AS builder
 RUN cargo install cargo-strip
 
 WORKDIR /builder

bin/periphery/src/api/build/mod.rs

@@ -35,10 +35,10 @@ use crate::{
 mod helpers;
 use helpers::*;
 
-impl Resolve<super::Args> for GetDockerfileContentsOnHost {
+impl Resolve<crate::api::Args> for GetDockerfileContentsOnHost {
   async fn resolve(
     self,
-    _: &super::Args,
+    _: &crate::api::Args,
   ) -> anyhow::Result<GetDockerfileContentsOnHostResponse> {
     let GetDockerfileContentsOnHost {
       name,
@@ -75,7 +75,7 @@ impl Resolve<super::Args> for GetDockerfileContentsOnHost {
   }
 }
 
-impl Resolve<super::Args> for WriteDockerfileContentsToHost {
+impl Resolve<crate::api::Args> for WriteDockerfileContentsToHost {
   #[instrument(
     "WriteDockerfileContentsToHost",
     skip_all,
@@ -87,7 +87,10 @@ impl Resolve<super::Args> for WriteDockerfileContentsToHost {
       dockerfile_path = &self.dockerfile_path,
     )
   )]
-  async fn resolve(self, args: &super::Args) -> anyhow::Result<Log> {
+  async fn resolve(
+    self,
+    args: &crate::api::Args,
+  ) -> anyhow::Result<Log> {
     let WriteDockerfileContentsToHost {
       name,
       build_path,
@@ -123,7 +126,7 @@ impl Resolve<super::Args> for WriteDockerfileContentsToHost {
   }
 }
 
-impl Resolve<super::Args> for build::Build {
+impl Resolve<crate::api::Args> for build::Build {
   #[instrument(
     "Build",
     skip_all,
@@ -136,7 +139,7 @@ impl Resolve<super::Args> for build::Build {
   )]
   async fn resolve(
     self,
-    args: &super::Args,
+    args: &crate::api::Args,
   ) -> anyhow::Result<Vec<Log>> {
     let build::Build {
       mut build,
@@ -339,7 +342,7 @@ impl Resolve<super::Args> for build::Build {
 
 //
 
-impl Resolve<super::Args> for PruneBuilders {
+impl Resolve<crate::api::Args> for PruneBuilders {
   #[instrument(
     "PruneBuilders",
     skip_all,
@@ -348,7 +351,10 @@ impl Resolve<super::Args> for PruneBuilders {
       core = args.core,
     )
   )]
-  async fn resolve(self, args: &super::Args) -> anyhow::Result<Log> {
+  async fn resolve(
+    self,
+    args: &crate::api::Args,
+  ) -> anyhow::Result<Log> {
     let command = String::from("docker builder prune -a -f");
     Ok(
       run_komodo_standard_command("Prune Builders", None, command)
@@ -359,7 +365,7 @@ impl Resolve<super::Args> for PruneBuilders {
 
 //
 
-impl Resolve<super::Args> for PruneBuildx {
+impl Resolve<crate::api::Args> for PruneBuildx {
   #[instrument(
     "PruneBuildx",
     skip_all,
@@ -368,7 +374,10 @@ impl Resolve<super::Args> for PruneBuildx {
       core = args.core,
     )
   )]
-  async fn resolve(self, args: &super::Args) -> anyhow::Result<Log> {
+  async fn resolve(
+    self,
+    args: &crate::api::Args,
+  ) -> anyhow::Result<Log> {
     let command = String::from("docker buildx prune -a -f");
     Ok(
       run_komodo_standard_command("Prune Buildx", None, command)

bin/periphery/src/api/compose.rs

@@ -1,4 +1,4 @@
-use std::{borrow::Cow, path::PathBuf};
+use std::{borrow::Cow, fmt::Write, path::PathBuf};
 
 use anyhow::{Context, anyhow};
 use command::{
@@ -12,15 +12,16 @@ use komodo_client::{
   entities::{
     FileContents, RepoExecutionResponse, all_logs_success,
     stack::{
-      ComposeFile, ComposeService, ComposeServiceDeploy,
-      StackRemoteFileContents, StackServiceNames,
+      AdditionalEnvFile, ComposeFile, ComposeService,
+      ComposeServiceDeploy, StackRemoteFileContents,
+      StackServiceNames,
     },
     to_path_compatible_name,
     update::Log,
   },
   parsers::parse_multiline_command,
 };
-use periphery_client::api::compose::*;
+use periphery_client::api::{DeployStackResponse, compose::*};
 use resolver_api::Resolve;
 use shell_escape::unix::escape;
 use tracing::Instrument;
@@ -29,15 +30,17 @@ use crate::{
   config::periphery_config,
   docker::compose::docker_compose,
   helpers::{format_extra_args, format_log_grep},
+  stack::{
+    maybe_login_registry, pull_or_clone_stack, validate_files,
+    write::write_stack,
+  },
 };
 
-mod helpers;
-mod write;
-
-use helpers::*;
-
-impl Resolve<super::Args> for GetComposeLog {
-  async fn resolve(self, _: &super::Args) -> anyhow::Result<Log> {
+impl Resolve<crate::api::Args> for GetComposeLog {
+  async fn resolve(
+    self,
+    _: &crate::api::Args,
+  ) -> anyhow::Result<Log> {
     let GetComposeLog {
       project,
       services,
@@ -61,8 +64,11 @@ impl Resolve<super::Args> for GetComposeLog {
   }
 }
 
-impl Resolve<super::Args> for GetComposeLogSearch {
-  async fn resolve(self, _: &super::Args) -> anyhow::Result<Log> {
+impl Resolve<crate::api::Args> for GetComposeLogSearch {
+  async fn resolve(
+    self,
+    _: &crate::api::Args,
+  ) -> anyhow::Result<Log> {
     let GetComposeLogSearch {
       project,
       services,
@@ -91,10 +97,10 @@ impl Resolve<super::Args> for GetComposeLogSearch {
 
 //
 
-impl Resolve<super::Args> for GetComposeContentsOnHost {
+impl Resolve<crate::api::Args> for GetComposeContentsOnHost {
   async fn resolve(
     self,
-    _: &super::Args,
+    _: &crate::api::Args,
   ) -> anyhow::Result<GetComposeContentsOnHostResponse> {
     let GetComposeContentsOnHost {
       name,
@@ -146,7 +152,7 @@ impl Resolve<super::Args> for GetComposeContentsOnHost {
 
 //
 
-impl Resolve<super::Args> for WriteComposeContentsToHost {
+impl Resolve<crate::api::Args> for WriteComposeContentsToHost {
   #[instrument(
     "WriteComposeContentsToHost",
     skip_all,
@@ -158,7 +164,10 @@ impl Resolve<super::Args> for WriteComposeContentsToHost {
       file_path = self.file_path,
     )
   )]
-  async fn resolve(self, args: &super::Args) -> anyhow::Result<Log> {
+  async fn resolve(
+    self,
+    args: &crate::api::Args,
+  ) -> anyhow::Result<Log> {
     let WriteComposeContentsToHost {
       name,
       run_directory,
@@ -188,7 +197,7 @@ impl Resolve<super::Args> for WriteComposeContentsToHost {
 
 //
 
-impl Resolve<super::Args> for WriteCommitComposeContents {
+impl Resolve<crate::api::Args> for WriteCommitComposeContents {
   #[instrument(
     "WriteCommitComposeContents",
     skip_all,
@@ -202,7 +211,7 @@ impl Resolve<super::Args> for WriteCommitComposeContents {
   )]
   async fn resolve(
     self,
-    args: &super::Args,
+    args: &crate::api::Args,
   ) -> anyhow::Result<RepoExecutionResponse> {
     let WriteCommitComposeContents {
       stack,
@@ -243,7 +252,7 @@ impl Resolve<super::Args> for WriteCommitComposeContents {
 
 //
 
-impl Resolve<super::Args> for ComposePull {
+impl Resolve<crate::api::Args> for ComposePull {
   #[instrument(
     "ComposePull",
     skip_all,
@@ -256,7 +265,7 @@ impl Resolve<super::Args> for ComposePull {
   )]
   async fn resolve(
     self,
-    args: &super::Args,
+    args: &crate::api::Args,
   ) -> anyhow::Result<ComposePullResponse> {
     let ComposePull {
       mut stack,
@@ -278,7 +287,7 @@ impl Resolve<super::Args> for ComposePull {
       .push_logs(&mut res.logs);
     replacers.extend(interpolator.secret_replacers);
 
-    let (run_directory, env_file_path) = match write::stack(
+    let (run_directory, env_file_path) = match write_stack(
       &stack,
       repo.as_ref(),
       git_token,
@@ -362,7 +371,7 @@ impl Resolve<super::Args> for ComposePull {
 
 //
 
-impl Resolve<super::Args> for ComposeUp {
+impl Resolve<crate::api::Args> for ComposeUp {
   #[instrument(
     "ComposeUp",
     skip_all,
@@ -376,8 +385,8 @@ impl Resolve<super::Args> for ComposeUp {
   )]
   async fn resolve(
     self,
-    args: &super::Args,
-  ) -> anyhow::Result<ComposeUpResponse> {
+    args: &crate::api::Args,
+  ) -> anyhow::Result<DeployStackResponse> {
     let ComposeUp {
       mut stack,
       repo,
@@ -387,7 +396,7 @@ impl Resolve<super::Args> for ComposeUp {
       mut replacers,
     } = self;
 
-    let mut res = ComposeUpResponse::default();
+    let mut res = DeployStackResponse::default();
 
     let mut interpolator =
       Interpolator::new(None, &periphery_config().secrets);
@@ -398,7 +407,7 @@ impl Resolve<super::Args> for ComposeUp {
       .push_logs(&mut res.logs);
     replacers.extend(interpolator.secret_replacers);
 
-    let (run_directory, env_file_path) = match write::stack(
+    let (run_directory, env_file_path) = match write_stack(
       &stack,
       repo.as_ref(),
       git_token,
@@ -436,7 +445,7 @@ impl Resolve<super::Args> for ComposeUp {
     if !stack.config.pre_deploy.is_none() {
       let pre_deploy_path =
         run_directory.join(&stack.config.pre_deploy.path);
-      let span = info_span!("RunPreDeploy");
+      let span = info_span!("ExecutePreDeploy");
       if let Some(log) = run_komodo_command_with_sanitization(
         "Pre Deploy",
         pre_deploy_path.as_path(),
@@ -502,8 +511,8 @@ impl Resolve<super::Args> for ComposeUp {
       let compose =
         serde_yaml_ng::from_str::<ComposeFile>(&config_log.stdout)
           .context("Failed to parse compose contents")?;
-      // Record sanitized compose config output
-      res.compose_config = Some(config_log.stdout);
+      // Store sanitized compose config output
+      res.merged_config = Some(config_log.stdout);
       for (
         service_name,
         ComposeService {
@@ -547,7 +556,7 @@ impl Resolve<super::Args> for ComposeUp {
       let command = format!(
         "{docker_compose} -p {project_name} -f {file_args}{env_file_args} build{build_extra_args}{service_args}",
       );
-      let span = info_span!("RunComposeBuild");
+      let span = info_span!("ExecuteComposeBuild");
       let Some(log) = run_komodo_command_with_sanitization(
         "Compose Build",
         run_directory.as_path(),
@@ -591,11 +600,11 @@ impl Resolve<super::Args> for ComposeUp {
       // Also check if project name changed, which also requires taking down.
       || last_project_name != project_name
     {
-      // Take down the existing containers.
+      // Take down the existing compose stack.
       // This one tries to use the previously deployed service name, to ensure the right stack is taken down.
-      helpers::compose_down(&last_project_name, &services, &mut res)
+      compose_down(&last_project_name, &services, &mut res)
         .await
-        .context("failed to destroy existing containers")?;
+        .context("Failed to take down existing compose stack")?;
     }
 
     // Run compose up
@@ -619,7 +628,7 @@ impl Resolve<super::Args> for ComposeUp {
         compose_cmd_wrapper.replace("[[COMPOSE_COMMAND]]", &command);
     }
 
-    let span = info_span!("RunComposeUp");
+    let span = info_span!("ExecuteComposeUp");
     let Some(log) = run_komodo_command_with_sanitization(
       "Compose Up",
       run_directory.as_path(),
@@ -639,7 +648,7 @@ impl Resolve<super::Args> for ComposeUp {
     if res.deployed && !stack.config.post_deploy.is_none() {
       let post_deploy_path =
         run_directory.join(&stack.config.post_deploy.path);
-      let span = info_span!("RunPostDeploy");
+      let span = info_span!("ExecutePostDeploy");
       if let Some(log) = run_komodo_command_with_sanitization(
         "Post Deploy",
         post_deploy_path.as_path(),
@@ -660,7 +669,7 @@ impl Resolve<super::Args> for ComposeUp {
 
 //
 
-impl Resolve<super::Args> for ComposeExecution {
+impl Resolve<crate::api::Args> for ComposeExecution {
   #[instrument(
     "ComposeExecution",
     skip_all,
@@ -671,7 +680,10 @@ impl Resolve<super::Args> for ComposeExecution {
       command = self.command,
     )
   )]
-  async fn resolve(self, args: &super::Args) -> anyhow::Result<Log> {
+  async fn resolve(
+    self,
+    args: &crate::api::Args,
+  ) -> anyhow::Result<Log> {
     let ComposeExecution { project, command } = self;
     let docker_compose = docker_compose();
     let log = run_komodo_standard_command(
@@ -686,7 +698,7 @@ impl Resolve<super::Args> for ComposeExecution {
 
 //
 
-impl Resolve<super::Args> for ComposeRun {
+impl Resolve<crate::api::Args> for ComposeRun {
   #[instrument(
     "ComposeRun",
     skip_all,
@@ -698,7 +710,10 @@ impl Resolve<super::Args> for ComposeRun {
       service = &self.service
     )
   )]
-  async fn resolve(self, args: &super::Args) -> anyhow::Result<Log> {
+  async fn resolve(
+    self,
+    args: &crate::api::Args,
+  ) -> anyhow::Result<Log> {
     let ComposeRun {
       mut stack,
       repo,
@@ -726,7 +741,7 @@ impl Resolve<super::Args> for ComposeRun {
     replacers.extend(interpolator.secret_replacers);
 
     let mut res = ComposeRunResponse::default();
-    let (run_directory, env_file_path) = match write::stack(
+    let (run_directory, env_file_path) = match write_stack(
       &stack,
       repo.as_ref(),
       git_token,
@@ -843,3 +858,59 @@ impl Resolve<super::Args> for ComposeRun {
     Ok(log)
   }
 }
+
+fn env_file_args(
+  env_file_path: Option<&str>,
+  additional_env_files: &[AdditionalEnvFile],
+) -> anyhow::Result<String> {
+  let mut res = String::new();
+
+  // Add additional env files (except komodo's own, which comes last)
+  for file in additional_env_files
+    .iter()
+    .filter(|f| env_file_path != Some(f.path.as_str()))
+  {
+    let path = &file.path;
+    write!(res, " --env-file {path}").with_context(|| {
+      format!("Failed to write --env-file arg for {path}")
+    })?;
+  }
+
+  // Add komodo's env file last for highest priority
+  if let Some(file) = env_file_path {
+    write!(res, " --env-file {file}").with_context(|| {
+      format!("Failed to write --env-file arg for {file}")
+    })?;
+  }
+
+  Ok(res)
+}
+
+#[instrument("ComposeDown", skip(res))]
+async fn compose_down(
+  project: &str,
+  services: &[String],
+  res: &mut DeployStackResponse,
+) -> anyhow::Result<()> {
+  let docker_compose = docker_compose();
+  let service_args = if services.is_empty() {
+    String::new()
+  } else {
+    format!(" {}", services.join(" "))
+  };
+  let log = run_komodo_standard_command(
+    "Compose Down",
+    None,
+    format!("{docker_compose} -p {project} down{service_args}"),
+  )
+  .await;
+  let success = log.success;
+  res.logs.push(log);
+  if !success {
+    return Err(anyhow!(
+      "Failed to bring down existing container(s) with docker compose down. Stopping run."
+    ));
+  }
+
+  Ok(())
+}

bin/periphery/src/api/container/mod.rs

@@ -19,16 +19,18 @@ use crate::{
   state::docker_client,
 };
 
+mod run;
+
 // ======
 //  READ
 // ======
 
 //
 
-impl Resolve<super::Args> for InspectContainer {
+impl Resolve<crate::api::Args> for InspectContainer {
   async fn resolve(
     self,
-    _: &super::Args,
+    _: &crate::api::Args,
   ) -> anyhow::Result<Container> {
     let client = docker_client().load();
     let client = client
@@ -41,8 +43,11 @@ impl Resolve<super::Args> for InspectContainer {
 
 //
 
-impl Resolve<super::Args> for GetContainerLog {
-  async fn resolve(self, _: &super::Args) -> anyhow::Result<Log> {
+impl Resolve<crate::api::Args> for GetContainerLog {
+  async fn resolve(
+    self,
+    _: &crate::api::Args,
+  ) -> anyhow::Result<Log> {
     let GetContainerLog {
       name,
       tail,
@@ -64,8 +69,11 @@ impl Resolve<super::Args> for GetContainerLog {
 
 //
 
-impl Resolve<super::Args> for GetContainerLogSearch {
-  async fn resolve(self, _: &super::Args) -> anyhow::Result<Log> {
+impl Resolve<crate::api::Args> for GetContainerLogSearch {
+  async fn resolve(
+    self,
+    _: &crate::api::Args,
+  ) -> anyhow::Result<Log> {
     let GetContainerLogSearch {
       name,
       terms,
@@ -95,10 +103,10 @@ impl Resolve<super::Args> for GetContainerLogSearch {
 
 //
 
-impl Resolve<super::Args> for GetContainerStats {
+impl Resolve<crate::api::Args> for GetContainerStats {
   async fn resolve(
     self,
-    _: &super::Args,
+    _: &crate::api::Args,
   ) -> anyhow::Result<ContainerStats> {
     let mut stats = get_container_stats(Some(self.name)).await?;
     let stats =
@@ -109,10 +117,10 @@ impl Resolve<super::Args> for GetContainerStats {
 
 //
 
-impl Resolve<super::Args> for GetFullContainerStats {
+impl Resolve<crate::api::Args> for GetFullContainerStats {
   async fn resolve(
     self,
-    _: &super::Args,
+    _: &crate::api::Args,
   ) -> anyhow::Result<FullContainerStats> {
     let client = docker_client().load();
     let client = client
@@ -125,10 +133,10 @@ impl Resolve<super::Args> for GetFullContainerStats {
 
 //
 
-impl Resolve<super::Args> for GetContainerStatsList {
+impl Resolve<crate::api::Args> for GetContainerStatsList {
   async fn resolve(
     self,
-    _: &super::Args,
+    _: &crate::api::Args,
   ) -> anyhow::Result<Vec<ContainerStats>> {
     get_container_stats(None).await
   }
@@ -138,7 +146,7 @@ impl Resolve<super::Args> for GetContainerStatsList {
 //  ACTIONS
 // =========
 
-impl Resolve<super::Args> for StartContainer {
+impl Resolve<crate::api::Args> for StartContainer {
   #[instrument(
     "StartContainer",
     skip_all,
@@ -148,7 +156,10 @@ impl Resolve<super::Args> for StartContainer {
       container = self.name,
     )
   )]
-  async fn resolve(self, args: &super::Args) -> anyhow::Result<Log> {
+  async fn resolve(
+    self,
+    args: &crate::api::Args,
+  ) -> anyhow::Result<Log> {
     Ok(
       run_komodo_standard_command(
         "Docker Start",
@@ -162,7 +173,7 @@ impl Resolve<super::Args> for StartContainer {
 
 //
 
-impl Resolve<super::Args> for RestartContainer {
+impl Resolve<crate::api::Args> for RestartContainer {
   #[instrument(
     "RestartContainer",
     skip_all,
@@ -172,7 +183,10 @@ impl Resolve<super::Args> for RestartContainer {
       container = self.name,
     )
   )]
-  async fn resolve(self, args: &super::Args) -> anyhow::Result<Log> {
+  async fn resolve(
+    self,
+    args: &crate::api::Args,
+  ) -> anyhow::Result<Log> {
     Ok(
       run_komodo_standard_command(
         "Docker Restart",
@@ -186,7 +200,7 @@ impl Resolve<super::Args> for RestartContainer {
 
 //
 
-impl Resolve<super::Args> for PauseContainer {
+impl Resolve<crate::api::Args> for PauseContainer {
   #[instrument(
     "PauseContainer",
     skip_all,
@@ -196,7 +210,10 @@ impl Resolve<super::Args> for PauseContainer {
       container = self.name,
     )
   )]
-  async fn resolve(self, args: &super::Args) -> anyhow::Result<Log> {
+  async fn resolve(
+    self,
+    args: &crate::api::Args,
+  ) -> anyhow::Result<Log> {
     Ok(
       run_komodo_standard_command(
         "Docker Pause",
@@ -208,7 +225,7 @@ impl Resolve<super::Args> for PauseContainer {
   }
 }
 
-impl Resolve<super::Args> for UnpauseContainer {
+impl Resolve<crate::api::Args> for UnpauseContainer {
   #[instrument(
     "UnpauseContainer",
     skip_all,
@@ -218,7 +235,10 @@ impl Resolve<super::Args> for UnpauseContainer {
       container = self.name,
     )
   )]
-  async fn resolve(self, args: &super::Args) -> anyhow::Result<Log> {
+  async fn resolve(
+    self,
+    args: &crate::api::Args,
+  ) -> anyhow::Result<Log> {
     Ok(
       run_komodo_standard_command(
         "Docker Unpause",
@@ -232,7 +252,7 @@ impl Resolve<super::Args> for UnpauseContainer {
 
 //
 
-impl Resolve<super::Args> for StopContainer {
+impl Resolve<crate::api::Args> for StopContainer {
   #[instrument(
     "StopContainer",
     skip_all,
@@ -242,7 +262,10 @@ impl Resolve<super::Args> for StopContainer {
       container = self.name,
     )
   )]
-  async fn resolve(self, args: &super::Args) -> anyhow::Result<Log> {
+  async fn resolve(
+    self,
+    args: &crate::api::Args,
+  ) -> anyhow::Result<Log> {
     let StopContainer { name, signal, time } = self;
     let command = stop_container_command(&name, signal, time);
     let log =
@@ -269,7 +292,7 @@ impl Resolve<super::Args> for StopContainer {
 
 //
 
-impl Resolve<super::Args> for RemoveContainer {
+impl Resolve<crate::api::Args> for RemoveContainer {
   #[instrument(
     "RemoveContainer",
     skip_all,
@@ -279,7 +302,10 @@ impl Resolve<super::Args> for RemoveContainer {
       container = self.name,
     )
   )]
-  async fn resolve(self, args: &super::Args) -> anyhow::Result<Log> {
+  async fn resolve(
+    self,
+    args: &crate::api::Args,
+  ) -> anyhow::Result<Log> {
     let RemoveContainer { name, signal, time } = self;
     let stop_command = stop_container_command(&name, signal, time);
     let command =
@@ -317,7 +343,7 @@ impl Resolve<super::Args> for RemoveContainer {
 
 //
 
-impl Resolve<super::Args> for RenameContainer {
+impl Resolve<crate::api::Args> for RenameContainer {
   #[instrument(
     "RenameContainer",
     skip_all,
@@ -328,7 +354,10 @@ impl Resolve<super::Args> for RenameContainer {
       new = self.new_name,
     )
   )]
-  async fn resolve(self, args: &super::Args) -> anyhow::Result<Log> {
+  async fn resolve(
+    self,
+    args: &crate::api::Args,
+  ) -> anyhow::Result<Log> {
     let RenameContainer {
       curr_name,
       new_name,
@@ -343,7 +372,7 @@ impl Resolve<super::Args> for RenameContainer {
 
 //
 
-impl Resolve<super::Args> for PruneContainers {
+impl Resolve<crate::api::Args> for PruneContainers {
   #[instrument(
     "PruneContainers",
     skip_all,
@@ -352,7 +381,10 @@ impl Resolve<super::Args> for PruneContainers {
       core = args.core
     )
   )]
-  async fn resolve(self, args: &super::Args) -> anyhow::Result<Log> {
+  async fn resolve(
+    self,
+    args: &crate::api::Args,
+  ) -> anyhow::Result<Log> {
     let command = String::from("docker container prune -f");
     Ok(
       run_komodo_standard_command("Prune Containers", None, command)
@@ -363,7 +395,7 @@ impl Resolve<super::Args> for PruneContainers {
 
 //
 
-impl Resolve<super::Args> for StartAllContainers {
+impl Resolve<crate::api::Args> for StartAllContainers {
   #[instrument(
     "StartAllContainers",
     skip_all,
@@ -374,7 +406,7 @@ impl Resolve<super::Args> for StartAllContainers {
   )]
   async fn resolve(
     self,
-    args: &super::Args,
+    args: &crate::api::Args,
   ) -> anyhow::Result<Vec<Log>> {
     let client = docker_client().load();
     let client = client
@@ -405,7 +437,7 @@ impl Resolve<super::Args> for StartAllContainers {
 
 //
 
-impl Resolve<super::Args> for RestartAllContainers {
+impl Resolve<crate::api::Args> for RestartAllContainers {
   #[instrument(
     "RestartAllContainers",
     skip_all,
@@ -416,7 +448,7 @@ impl Resolve<super::Args> for RestartAllContainers {
   )]
   async fn resolve(
     self,
-    args: &super::Args,
+    args: &crate::api::Args,
   ) -> anyhow::Result<Vec<Log>> {
     let client = docker_client().load();
     let client = client
@@ -447,7 +479,7 @@ impl Resolve<super::Args> for RestartAllContainers {
 
 //
 
-impl Resolve<super::Args> for PauseAllContainers {
+impl Resolve<crate::api::Args> for PauseAllContainers {
   #[instrument(
     "PauseAllContainers",
     skip_all,
@@ -458,7 +490,7 @@ impl Resolve<super::Args> for PauseAllContainers {
   )]
   async fn resolve(
     self,
-    args: &super::Args,
+    args: &crate::api::Args,
   ) -> anyhow::Result<Vec<Log>> {
     let client = docker_client().load();
     let client = client
@@ -489,7 +521,7 @@ impl Resolve<super::Args> for PauseAllContainers {
 
 //
 
-impl Resolve<super::Args> for UnpauseAllContainers {
+impl Resolve<crate::api::Args> for UnpauseAllContainers {
   #[instrument(
     "UnpauseAllContainers",
     skip_all,
@@ -500,7 +532,7 @@ impl Resolve<super::Args> for UnpauseAllContainers {
   )]
   async fn resolve(
     self,
-    args: &super::Args,
+    args: &crate::api::Args,
   ) -> anyhow::Result<Vec<Log>> {
     let client = docker_client().load();
     let client = client
@@ -531,7 +563,7 @@ impl Resolve<super::Args> for UnpauseAllContainers {
 
 //
 
-impl Resolve<super::Args> for StopAllContainers {
+impl Resolve<crate::api::Args> for StopAllContainers {
   #[instrument(
     "StopAllContainers",
     skip_all,
@@ -542,7 +574,7 @@ impl Resolve<super::Args> for StopAllContainers {
   )]
   async fn resolve(
     self,
-    args: &super::Args,
+    args: &crate::api::Args,
   ) -> anyhow::Result<Vec<Log>> {
     let client = docker_client().load();
     let client = client

bin/periphery/src/api/container/run.rs

@@ -1,34 +1,36 @@
+use std::fmt::Write;
+
 use anyhow::Context;
 use command::{
   KomodoCommandMode, run_komodo_command_with_sanitization,
 };
 use formatting::format_serror;
 use interpolate::Interpolator;
-use komodo_client::{
-  entities::{
-    EnvironmentVar,
-    deployment::{
-      Conversion, Deployment, DeploymentConfig, DeploymentImage,
-      RestartMode, conversions_from_str, extract_registry_domain,
-    },
-    environment_vars_from_str,
-    update::Log,
+use komodo_client::entities::{
+  deployment::{
+    Deployment, DeploymentConfig, DeploymentImage, RestartMode,
+    conversions_from_str, extract_registry_domain,
   },
-  parsers::QUOTE_PATTERN,
+  environment_vars_from_str,
+  update::Log,
+};
+use periphery_client::api::container::{
+  RemoveContainer, RunContainer,
 };
-use periphery_client::api::container::{Deploy, RemoveContainer};
 use resolver_api::Resolve;
 use tracing::Instrument;
 
 use crate::{
   config::periphery_config,
   docker::{docker_login, pull_image},
-  helpers::{format_extra_args, format_labels},
+  helpers::{
+    push_conversions, push_environment, push_extra_args, push_labels,
+  },
 };
 
-impl Resolve<super::Args> for Deploy {
+impl Resolve<crate::api::Args> for RunContainer {
   #[instrument(
-    "Deploy",
+    "DeployContainer",
     skip_all,
     fields(
       id = args.id.to_string(),
@@ -38,8 +40,11 @@ impl Resolve<super::Args> for Deploy {
       stop_time = self.stop_time,
     )
   )]
-  async fn resolve(self, args: &super::Args) -> anyhow::Result<Log> {
-    let Deploy {
+  async fn resolve(
+    self,
+    args: &crate::api::Args,
+  ) -> anyhow::Result<Log> {
+    let RunContainer {
       mut deployment,
       stop_signal,
       stop_time,
@@ -57,15 +62,15 @@ impl Resolve<super::Args> for Deploy {
     {
       if image.is_empty() {
         return Ok(Log::error(
-          "get image",
-          String::from("deployment does not have image attached"),
+          "Get Image",
+          String::from("Deployment does not have image attached"),
         ));
       }
       image
     } else {
       return Ok(Log::error(
-        "get image",
-        String::from("deployment does not have image attached"),
+        "Get Image",
+        String::from("Deployment does not have image attached"),
       ));
     };
 
@@ -77,9 +82,9 @@ impl Resolve<super::Args> for Deploy {
     .await
     {
       return Ok(Log::error(
-        "docker login",
+        "Docker Login",
         format_serror(
-          &e.context("failed to login to docker registry").into(),
+          &e.context("Failed to login to docker registry").into(),
         ),
       ));
     }
@@ -99,7 +104,7 @@ impl Resolve<super::Args> for Deploy {
     let command = docker_run_command(&deployment, image)
       .context("Unable to generate valid docker run command")?;
 
-    let span = info_span!("RunDockerRun");
+    let span = info_span!("ExecuteDockerRun");
     let Some(log) = run_komodo_command_with_sanitization(
       "Docker Run",
       None,
@@ -138,75 +143,53 @@ fn docker_run_command(
   }: &Deployment,
   image: &str,
 ) -> anyhow::Result<String> {
-  let ports = parse_conversions(
+  let mut res =
+    format!("docker run -d --name {name} --network {network}");
+
+  push_conversions(
+    &mut res,
     &conversions_from_str(ports).context("Invalid ports")?,
     "-p",
-  );
-  let volumes = parse_conversions(
+  )?;
+
+  push_conversions(
+    &mut res,
     &conversions_from_str(volumes).context("Invalid volumes")?,
     "-v",
-  );
-  let network = parse_network(network);
-  let restart = parse_restart(restart);
-  let environment = parse_environment(
+  )?;
+
+  push_environment(
+    &mut res,
     &environment_vars_from_str(environment)
       .context("Invalid environment")?,
-  );
-  let labels = format_labels(
+  )?;
+
+  push_restart(&mut res, restart)?;
+
+  push_labels(
+    &mut res,
     &environment_vars_from_str(labels).context("Invalid labels")?,
-  );
-  let command = parse_command(command);
-  let extra_args = format_extra_args(extra_args);
-  let command = format!(
-    "docker run -d --name {name}{ports}{volumes}{network}{restart}{environment}{labels}{extra_args} {image}{command}"
-  );
-  Ok(command)
-}
+  )?;
 
-fn parse_conversions(
-  conversions: &[Conversion],
-  flag: &str,
-) -> String {
-  conversions
-    .iter()
-    .map(|p| format!(" {flag} {}:{}", p.local, p.container))
-    .collect::<Vec<_>>()
-    .join("")
-}
+  push_extra_args(&mut res, extra_args)?;
 
-fn parse_environment(environment: &[EnvironmentVar]) -> String {
-  environment
-    .iter()
-    .map(|p| {
-      if p.value.starts_with(QUOTE_PATTERN)
-        && p.value.ends_with(QUOTE_PATTERN)
-      {
-        // If the value already wrapped in quotes, don't wrap it again
-        format!(" --env {}={}", p.variable, p.value)
-      } else {
-        format!(" --env {}=\"{}\"", p.variable, p.value)
-      }
-    })
-    .collect::<Vec<_>>()
-    .join("")
-}
+  write!(&mut res, " {image}")?;
 
-fn parse_network(network: &str) -> String {
-  format!(" --network {network}")
-}
-
-fn parse_restart(restart: &RestartMode) -> String {
-  let restart = match restart {
-    RestartMode::OnFailure => "on-failure:10".to_string(),
-    _ => restart.to_string(),
-  };
-  format!(" --restart {restart}")
-}
-
-fn parse_command(command: &str) -> String {
-  if command.is_empty() {
-    String::new()
-  } else {
-    format!(" {command}")
+  if !command.is_empty() {
+    write!(&mut res, " {command}")?;
   }
+
+  Ok(res)
+}
+
+fn push_restart(
+  command: &mut String,
+  restart: &RestartMode,
+) -> anyhow::Result<()> {
+  let restart = match restart {
+    RestartMode::OnFailure => "on-failure:10",
+    _ => restart.as_ref(),
+  };
+  write!(command, " --restart {restart}")
+    .context("Failed to write restart mode")
 }

bin/periphery/src/api/docker.rs

@@ -22,8 +22,11 @@ use crate::{docker::docker_login, state::docker_client};
 // IMAGE
 // =====
 
-impl Resolve<super::Args> for InspectImage {
-  async fn resolve(self, _: &super::Args) -> anyhow::Result<Image> {
+impl Resolve<crate::api::Args> for InspectImage {
+  async fn resolve(
+    self,
+    _: &crate::api::Args,
+  ) -> anyhow::Result<Image> {
     let client = docker_client().load();
     let client = client
       .iter()
@@ -35,10 +38,10 @@ impl Resolve<super::Args> for InspectImage {
 
 //
 
-impl Resolve<super::Args> for ImageHistory {
+impl Resolve<crate::api::Args> for ImageHistory {
   async fn resolve(
     self,
-    _: &super::Args,
+    _: &crate::api::Args,
   ) -> anyhow::Result<Vec<ImageHistoryResponseItem>> {
     let client = docker_client().load();
     let client = client
@@ -60,7 +63,7 @@ fn pull_cache() -> &'static TimeoutCache<String, Log> {
   PULL_CACHE.get_or_init(Default::default)
 }
 
-impl Resolve<super::Args> for PullImage {
+impl Resolve<crate::api::Args> for PullImage {
   #[instrument(
     "PullImage",
     skip_all,
@@ -71,7 +74,10 @@ impl Resolve<super::Args> for PullImage {
       account = self.account,
     )
   )]
-  async fn resolve(self, args: &super::Args) -> anyhow::Result<Log> {
+  async fn resolve(
+    self,
+    args: &crate::api::Args,
+  ) -> anyhow::Result<Log> {
     let PullImage {
       name,
       account,
@@ -118,7 +124,7 @@ impl Resolve<super::Args> for PullImage {
 
 //
 
-impl Resolve<super::Args> for DeleteImage {
+impl Resolve<crate::api::Args> for DeleteImage {
   #[instrument(
     "DeleteImage",
     skip_all,
@@ -128,7 +134,10 @@ impl Resolve<super::Args> for DeleteImage {
       image = self.name,
     )
   )]
-  async fn resolve(self, args: &super::Args) -> anyhow::Result<Log> {
+  async fn resolve(
+    self,
+    args: &crate::api::Args,
+  ) -> anyhow::Result<Log> {
     let command = format!("docker image rm {}", self.name);
     Ok(
       run_komodo_standard_command("Delete Image", None, command)
@@ -139,7 +148,7 @@ impl Resolve<super::Args> for DeleteImage {
 
 //
 
-impl Resolve<super::Args> for PruneImages {
+impl Resolve<crate::api::Args> for PruneImages {
   #[instrument(
     "PruneImages",
     skip_all,
@@ -148,7 +157,10 @@ impl Resolve<super::Args> for PruneImages {
       core = args.core,
     )
   )]
-  async fn resolve(self, args: &super::Args) -> anyhow::Result<Log> {
+  async fn resolve(
+    self,
+    args: &crate::api::Args,
+  ) -> anyhow::Result<Log> {
     let command = String::from("docker image prune -a -f");
     Ok(
       run_komodo_standard_command("Prune Images", None, command)
@@ -161,8 +173,11 @@ impl Resolve<super::Args> for PruneImages {
 // NETWORK
 // =======
 
-impl Resolve<super::Args> for InspectNetwork {
-  async fn resolve(self, _: &super::Args) -> anyhow::Result<Network> {
+impl Resolve<crate::api::Args> for InspectNetwork {
+  async fn resolve(
+    self,
+    _: &crate::api::Args,
+  ) -> anyhow::Result<Network> {
     let client = docker_client().load();
     let client = client
       .iter()
@@ -174,7 +189,7 @@ impl Resolve<super::Args> for InspectNetwork {
 
 //
 
-impl Resolve<super::Args> for CreateNetwork {
+impl Resolve<crate::api::Args> for CreateNetwork {
   #[instrument(
     "CreateNetwork",
     skip_all,
@@ -185,7 +200,10 @@ impl Resolve<super::Args> for CreateNetwork {
       driver = self.driver,
     )
   )]
-  async fn resolve(self, args: &super::Args) -> anyhow::Result<Log> {
+  async fn resolve(
+    self,
+    args: &crate::api::Args,
+  ) -> anyhow::Result<Log> {
     let CreateNetwork { name, driver } = self;
     let driver = match driver {
       Some(driver) => format!(" -d {driver}"),
@@ -201,7 +219,7 @@ impl Resolve<super::Args> for CreateNetwork {
 
 //
 
-impl Resolve<super::Args> for DeleteNetwork {
+impl Resolve<crate::api::Args> for DeleteNetwork {
   #[instrument(
     "DeleteNetwork",
     skip_all,
@@ -211,7 +229,10 @@ impl Resolve<super::Args> for DeleteNetwork {
       network = self.name,
     )
   )]
-  async fn resolve(self, args: &super::Args) -> anyhow::Result<Log> {
+  async fn resolve(
+    self,
+    args: &crate::api::Args,
+  ) -> anyhow::Result<Log> {
     let command = format!("docker network rm {}", self.name);
     Ok(
       run_komodo_standard_command("Delete Network", None, command)
@@ -222,7 +243,7 @@ impl Resolve<super::Args> for DeleteNetwork {
 
 //
 
-impl Resolve<super::Args> for PruneNetworks {
+impl Resolve<crate::api::Args> for PruneNetworks {
   #[instrument(
     "PruneNetworks",
     skip_all,
@@ -231,7 +252,10 @@ impl Resolve<super::Args> for PruneNetworks {
       core = args.core,
     )
   )]
-  async fn resolve(self, args: &super::Args) -> anyhow::Result<Log> {
+  async fn resolve(
+    self,
+    args: &crate::api::Args,
+  ) -> anyhow::Result<Log> {
     let command = String::from("docker network prune -f");
     Ok(
       run_komodo_standard_command("Prune Networks", None, command)
@@ -244,8 +268,11 @@ impl Resolve<super::Args> for PruneNetworks {
 // VOLUME
 // ======
 
-impl Resolve<super::Args> for InspectVolume {
-  async fn resolve(self, _: &super::Args) -> anyhow::Result<Volume> {
+impl Resolve<crate::api::Args> for InspectVolume {
+  async fn resolve(
+    self,
+    _: &crate::api::Args,
+  ) -> anyhow::Result<Volume> {
     let client = docker_client().load();
     let client = client
       .iter()
@@ -257,7 +284,7 @@ impl Resolve<super::Args> for InspectVolume {
 
 //
 
-impl Resolve<super::Args> for DeleteVolume {
+impl Resolve<crate::api::Args> for DeleteVolume {
   #[instrument(
     "DeleteVolume",
     skip_all,
@@ -267,7 +294,10 @@ impl Resolve<super::Args> for DeleteVolume {
       volume = self.name,
     )
   )]
-  async fn resolve(self, args: &super::Args) -> anyhow::Result<Log> {
+  async fn resolve(
+    self,
+    args: &crate::api::Args,
+  ) -> anyhow::Result<Log> {
     let command = format!("docker volume rm {}", self.name);
     Ok(
       run_komodo_standard_command("Delete Volume", None, command)
@@ -278,7 +308,7 @@ impl Resolve<super::Args> for DeleteVolume {
 
 //
 
-impl Resolve<super::Args> for PruneVolumes {
+impl Resolve<crate::api::Args> for PruneVolumes {
   #[instrument(
     "PruneVolumes",
     skip_all,
@@ -287,7 +317,10 @@ impl Resolve<super::Args> for PruneVolumes {
       core = args.core,
     )
   )]
-  async fn resolve(self, args: &super::Args) -> anyhow::Result<Log> {
+  async fn resolve(
+    self,
+    args: &crate::api::Args,
+  ) -> anyhow::Result<Log> {
     let command = String::from("docker volume prune -a -f");
     Ok(
       run_komodo_standard_command("Prune Volumes", None, command)

bin/periphery/src/api/git.rs

@@ -16,10 +16,10 @@ use crate::{
   config::periphery_config, helpers::handle_post_repo_execution,
 };
 
-impl Resolve<super::Args> for GetLatestCommit {
+impl Resolve<crate::api::Args> for GetLatestCommit {
   async fn resolve(
     self,
-    _: &super::Args,
+    _: &crate::api::Args,
   ) -> anyhow::Result<Option<LatestCommit>> {
     let repo_path = match self.path {
       Some(p) => PathBuf::from(p),
@@ -33,7 +33,7 @@ impl Resolve<super::Args> for GetLatestCommit {
   }
 }
 
-impl Resolve<super::Args> for CloneRepo {
+impl Resolve<crate::api::Args> for CloneRepo {
   #[instrument(
     "CloneRepo",
     skip_all,
@@ -46,7 +46,7 @@ impl Resolve<super::Args> for CloneRepo {
   )]
   async fn resolve(
     self,
-    args: &super::Args,
+    args: &crate::api::Args,
   ) -> anyhow::Result<PeripheryRepoExecutionResponse> {
     let CloneRepo {
       args,
@@ -79,7 +79,7 @@ impl Resolve<super::Args> for CloneRepo {
 
 //
 
-impl Resolve<super::Args> for PullRepo {
+impl Resolve<crate::api::Args> for PullRepo {
   #[instrument(
     "PullRepo",
     skip_all,
@@ -92,7 +92,7 @@ impl Resolve<super::Args> for PullRepo {
   )]
   async fn resolve(
     self,
-    args: &super::Args,
+    args: &crate::api::Args,
   ) -> anyhow::Result<PeripheryRepoExecutionResponse> {
     let PullRepo {
       args,
@@ -124,7 +124,7 @@ impl Resolve<super::Args> for PullRepo {
 
 //
 
-impl Resolve<super::Args> for PullOrCloneRepo {
+impl Resolve<crate::api::Args> for PullOrCloneRepo {
   #[instrument(
     "PullOrCloneRepo",
     skip_all,
@@ -137,7 +137,7 @@ impl Resolve<super::Args> for PullOrCloneRepo {
   )]
   async fn resolve(
     self,
-    args: &super::Args,
+    args: &crate::api::Args,
   ) -> anyhow::Result<PeripheryRepoExecutionResponse> {
     let PullOrCloneRepo {
       args,
@@ -171,7 +171,7 @@ impl Resolve<super::Args> for PullOrCloneRepo {
 
 //
 
-impl Resolve<super::Args> for RenameRepo {
+impl Resolve<crate::api::Args> for RenameRepo {
   #[instrument(
     "RenameRepo",
     skip_all,
@@ -182,7 +182,10 @@ impl Resolve<super::Args> for RenameRepo {
       new_name = self.new_name,
     )
   )]
-  async fn resolve(self, args: &super::Args) -> anyhow::Result<Log> {
+  async fn resolve(
+    self,
+    args: &crate::api::Args,
+  ) -> anyhow::Result<Log> {
     let RenameRepo {
       curr_name,
       new_name,
@@ -201,7 +204,7 @@ impl Resolve<super::Args> for RenameRepo {
 
 //
 
-impl Resolve<super::Args> for DeleteRepo {
+impl Resolve<crate::api::Args> for DeleteRepo {
   #[instrument(
     "DeleteRepo",
     skip_all,
@@ -212,7 +215,10 @@ impl Resolve<super::Args> for DeleteRepo {
       is_build = self.is_build,
     )
   )]
-  async fn resolve(self, args: &super::Args) -> anyhow::Result<Log> {
+  async fn resolve(
+    self,
+    args: &crate::api::Args,
+  ) -> anyhow::Result<Log> {
     let DeleteRepo { name, is_build } = self;
     // If using custom clone path, it will be passed by core instead of name.
     // So the join will resolve to just the absolute path.

bin/periphery/src/api/keys.rs

@@ -12,7 +12,7 @@ use crate::{
 
 //
 
-impl Resolve<super::Args> for RotatePrivateKey {
+impl Resolve<crate::api::Args> for RotatePrivateKey {
   #[instrument(
     "RotatePrivateKey",
     skip_all,
@@ -23,7 +23,7 @@ impl Resolve<super::Args> for RotatePrivateKey {
   )]
   async fn resolve(
     self,
-    args: &super::Args,
+    args: &crate::api::Args,
   ) -> anyhow::Result<RotatePrivateKeyResponse> {
     let public_key = periphery_keys().rotate().await?.into_inner();
     info!("New Public Key: {public_key}");
@@ -33,7 +33,7 @@ impl Resolve<super::Args> for RotatePrivateKey {
 
 //
 
-impl Resolve<super::Args> for RotateCorePublicKey {
+impl Resolve<crate::api::Args> for RotateCorePublicKey {
   #[instrument(
     "RotateCorePublicKey",
     skip_all,
@@ -45,7 +45,7 @@ impl Resolve<super::Args> for RotateCorePublicKey {
   )]
   async fn resolve(
     self,
-    args: &super::Args,
+    args: &crate::api::Args,
   ) -> anyhow::Result<NoData> {
     let config = periphery_config();
 

bin/periphery/src/api/mod.rs

@@ -21,7 +21,6 @@ pub mod terminal;
 mod build;
 mod compose;
 mod container;
-mod deploy;
 mod docker;
 mod git;
 mod keys;
@@ -94,7 +93,7 @@ pub enum PeripheryRequest {
   GetFullContainerStats(GetFullContainerStats),
 
   // Container (Write)
-  Deploy(Deploy),
+  RunContainer(RunContainer),
   StartContainer(StartContainer),
   RestartContainer(RestartContainer),
   PauseContainer(PauseContainer),
@@ -136,16 +135,27 @@ pub enum PeripheryRequest {
   // All in one (Write)
   PruneSystem(PruneSystem),
 
-  // Swarm
+  // Swarm (Read)
   PollSwarmStatus(PollSwarmStatus),
   InspectSwarmNode(InspectSwarmNode),
-  InspectSwarmConfig(InspectSwarmConfig),
-  InspectSwarmSecret(InspectSwarmSecret),
   InspectSwarmStack(InspectSwarmStack),
-  InspectSwarmTask(InspectSwarmTask),
   InspectSwarmService(InspectSwarmService),
   GetSwarmServiceLog(GetSwarmServiceLog),
   GetSwarmServiceLogSearch(GetSwarmServiceLogSearch),
+  InspectSwarmTask(InspectSwarmTask),
+  InspectSwarmConfig(InspectSwarmConfig),
+  InspectSwarmSecret(InspectSwarmSecret),
+
+  // Swarm (Write)
+  UpdateSwarmNode(UpdateSwarmNode),
+  RemoveSwarmNodes(RemoveSwarmNodes),
+  DeploySwarmStack(DeploySwarmStack),
+  RemoveSwarmStacks(RemoveSwarmStacks),
+  CreateSwarmService(CreateSwarmService),
+  UpdateSwarmService(UpdateSwarmService),
+  RemoveSwarmServices(RemoveSwarmServices),
+  RemoveSwarmConfigs(RemoveSwarmConfigs),
+  RemoveSwarmSecrets(RemoveSwarmSecrets),
 
   // Terminal
   ListTerminals(ListTerminals),

bin/periphery/src/api/poll.rs

@@ -13,10 +13,10 @@ use crate::{
   },
 };
 
-impl Resolve<super::Args> for PollStatus {
+impl Resolve<crate::api::Args> for PollStatus {
   async fn resolve(
     self,
-    _: &super::Args,
+    _: &crate::api::Args,
   ) -> anyhow::Result<PollStatusResponse> {
     let stats_client = stats_client().read().await;
 

bin/periphery/src/api/swarm.rs

@@ -1,219 +0,0 @@
-use anyhow::Context as _;
-use command::{
-  run_komodo_shell_command, run_komodo_standard_command,
-};
-use komodo_client::entities::{
-  docker::{
-    SwarmLists, config::SwarmConfig, node::SwarmNode,
-    secret::SwarmSecret, service::SwarmService,
-    stack::SwarmStackLists, task::SwarmTask,
-  },
-  update::Log,
-};
-use periphery_client::api::swarm::*;
-use resolver_api::Resolve;
-
-use crate::{
-  docker::{
-    config::{inspect_swarm_config, list_swarm_configs},
-    stack::{inspect_swarm_stack, list_swarm_stacks},
-  },
-  helpers::format_log_grep,
-  state::docker_client,
-};
-
-impl Resolve<super::Args> for PollSwarmStatus {
-  async fn resolve(
-    self,
-    _: &super::Args,
-  ) -> anyhow::Result<PollSwarmStatusResponse> {
-    let client = docker_client().load();
-    let client = client
-      .iter()
-      .next()
-      .context("Could not connect to docker client")?;
-    let (inspect, nodes, services, tasks, secrets, configs, stacks) = tokio::join!(
-      client.inspect_swarm(),
-      client.list_swarm_nodes(),
-      client.list_swarm_services(),
-      client.list_swarm_tasks(),
-      client.list_swarm_secrets(),
-      list_swarm_configs(),
-      list_swarm_stacks(),
-    );
-    Ok(PollSwarmStatusResponse {
-      inspect: inspect.ok(),
-      lists: SwarmLists {
-        nodes: nodes.unwrap_or_default(),
-        services: services.unwrap_or_default(),
-        tasks: tasks.unwrap_or_default(),
-        secrets: secrets.unwrap_or_default(),
-        configs: configs.unwrap_or_default(),
-        stacks: stacks.unwrap_or_default(),
-      },
-    })
-  }
-}
-
-// ======
-//  Node
-// ======
-
-impl Resolve<super::Args> for InspectSwarmNode {
-  async fn resolve(
-    self,
-    _: &super::Args,
-  ) -> anyhow::Result<SwarmNode> {
-    let client = docker_client().load();
-    let client = client
-      .iter()
-      .next()
-      .context("Could not connect to docker client")?;
-    client.inspect_swarm_node(&self.node).await
-  }
-}
-
-// =========
-//  Service
-// =========
-
-impl Resolve<super::Args> for InspectSwarmService {
-  async fn resolve(
-    self,
-    _: &super::Args,
-  ) -> anyhow::Result<SwarmService> {
-    let client = docker_client().load();
-    let client = client
-      .iter()
-      .next()
-      .context("Could not connect to docker client")?;
-    client.inspect_swarm_service(&self.service).await
-  }
-}
-
-impl Resolve<super::Args> for GetSwarmServiceLog {
-  async fn resolve(self, _: &super::Args) -> anyhow::Result<Log> {
-    let GetSwarmServiceLog {
-      service,
-      tail,
-      timestamps,
-      no_task_ids,
-      no_resolve,
-      details,
-    } = self;
-    let timestamps =
-      timestamps.then_some(" --timestamps").unwrap_or_default();
-    let no_task_ids =
-      no_task_ids.then_some(" --no-task-ids").unwrap_or_default();
-    let no_resolve =
-      no_resolve.then_some(" --no-resolve").unwrap_or_default();
-    let details = details.then_some(" --details").unwrap_or_default();
-    let command = format!(
-      "docker service logs --tail {tail}{timestamps}{no_task_ids}{no_resolve}{details} {service}",
-    );
-    Ok(
-      run_komodo_standard_command(
-        "Get Swarm Service Log",
-        None,
-        command,
-      )
-      .await,
-    )
-  }
-}
-
-impl Resolve<super::Args> for GetSwarmServiceLogSearch {
-  async fn resolve(self, _: &super::Args) -> anyhow::Result<Log> {
-    let GetSwarmServiceLogSearch {
-      service,
-      terms,
-      combinator,
-      invert,
-      timestamps,
-      no_task_ids,
-      no_resolve,
-      details,
-    } = self;
-    let timestamps =
-      timestamps.then_some(" --timestamps").unwrap_or_default();
-    let no_task_ids =
-      no_task_ids.then_some(" --no-task-ids").unwrap_or_default();
-    let no_resolve =
-      no_resolve.then_some(" --no-resolve").unwrap_or_default();
-    let details = details.then_some(" --details").unwrap_or_default();
-    let grep = format_log_grep(&terms, combinator, invert);
-    let command = format!(
-      "docker service logs --tail 5000{timestamps}{no_task_ids}{no_resolve}{details} {service} 2>&1 | {grep}",
-    );
-    Ok(
-      run_komodo_shell_command(
-        "Search Swarm Service Log",
-        None,
-        command,
-      )
-      .await,
-    )
-  }
-}
-
-// ======
-//  Task
-// ======
-
-impl Resolve<super::Args> for InspectSwarmTask {
-  async fn resolve(
-    self,
-    _: &super::Args,
-  ) -> anyhow::Result<SwarmTask> {
-    let client = docker_client().load();
-    let client = client
-      .iter()
-      .next()
-      .context("Could not connect to docker client")?;
-    client.inspect_swarm_task(&self.task).await
-  }
-}
-
-// ========
-//  Secret
-// ========
-
-impl Resolve<super::Args> for InspectSwarmSecret {
-  async fn resolve(
-    self,
-    _: &super::Args,
-  ) -> anyhow::Result<SwarmSecret> {
-    let client = docker_client().load();
-    let client = client
-      .iter()
-      .next()
-      .context("Could not connect to docker client")?;
-    client.inspect_swarm_secret(&self.secret).await
-  }
-}
-
-// ========
-//  Config
-// ========
-
-impl Resolve<super::Args> for InspectSwarmConfig {
-  async fn resolve(
-    self,
-    _: &super::Args,
-  ) -> anyhow::Result<Vec<SwarmConfig>> {
-    inspect_swarm_config(&self.config).await
-  }
-}
-
-// =======
-//  Stack
-// =======
-
-impl Resolve<super::Args> for InspectSwarmStack {
-  async fn resolve(
-    self,
-    _: &super::Args,
-  ) -> anyhow::Result<SwarmStackLists> {
-    inspect_swarm_stack(self.stack).await
-  }
-}

bin/periphery/src/api/swarm/mod.rs

@@ -0,0 +1,272 @@
+use anyhow::Context as _;
+use command::run_komodo_standard_command;
+use komodo_client::entities::{
+  docker::{
+    SwarmLists, config::SwarmConfig, node::SwarmNode,
+    secret::SwarmSecret, task::SwarmTask,
+  },
+  update::Log,
+};
+use periphery_client::api::swarm::*;
+use resolver_api::Resolve;
+
+use crate::{
+  docker::{
+    config::{inspect_swarm_config, list_swarm_configs},
+    stack::list_swarm_stacks,
+  },
+  state::docker_client,
+};
+
+mod service;
+mod stack;
+
+impl Resolve<crate::api::Args> for PollSwarmStatus {
+  async fn resolve(
+    self,
+    _: &crate::api::Args,
+  ) -> anyhow::Result<PollSwarmStatusResponse> {
+    let client = docker_client().load();
+    let client = client
+      .iter()
+      .next()
+      .context("Could not connect to docker client")?;
+    let (inspect, nodes, tasks, secrets, configs, stacks) = tokio::join!(
+      client.inspect_swarm(),
+      client.list_swarm_nodes(),
+      client.list_swarm_tasks(),
+      client.list_swarm_secrets(),
+      list_swarm_configs(),
+      list_swarm_stacks(),
+    );
+    let tasks = tasks.unwrap_or_default();
+    let services = client.list_swarm_services(&tasks).await;
+    Ok(PollSwarmStatusResponse {
+      inspect: inspect.ok(),
+      lists: SwarmLists {
+        nodes: nodes.unwrap_or_default(),
+        services: services.unwrap_or_default(),
+        secrets: secrets.unwrap_or_default(),
+        configs: configs.unwrap_or_default(),
+        stacks: stacks.unwrap_or_default(),
+        tasks,
+      },
+    })
+  }
+}
+
+// ======
+//  Node
+// ======
+
+impl Resolve<crate::api::Args> for InspectSwarmNode {
+  async fn resolve(
+    self,
+    _: &crate::api::Args,
+  ) -> anyhow::Result<SwarmNode> {
+    let client = docker_client().load();
+    let client = client
+      .iter()
+      .next()
+      .context("Could not connect to docker client")?;
+    client.inspect_swarm_node(&self.node).await
+  }
+}
+
+impl Resolve<crate::api::Args> for UpdateSwarmNode {
+  #[instrument(
+    "UpdateSwarmNode",
+    skip_all,
+    fields(
+      id = args.id.to_string(),
+      core = args.core,
+      node = self.node,
+      update = serde_json::to_string(&self).unwrap_or_else(|e| e.to_string())
+    )
+  )]
+  async fn resolve(
+    self,
+    args: &crate::api::Args,
+  ) -> anyhow::Result<Log> {
+    let mut command = String::from("docker node update");
+
+    if let Some(role) = self.role {
+      command += " --role=";
+      command += role.as_ref();
+    }
+
+    if let Some(availability) = self.availability {
+      command += " --availability=";
+      command += availability.as_ref();
+    }
+
+    if let Some(label_add) = self.label_add {
+      for (key, value) in label_add {
+        command += " --label-add ";
+        command += &key;
+        if let Some(value) = value {
+          command += "=";
+          command += &value;
+        }
+      }
+    }
+
+    if let Some(label_rm) = self.label_rm {
+      for key in label_rm {
+        command += " --label-rm ";
+        command += &key;
+      }
+    }
+
+    command += " ";
+    command += &self.node;
+
+    Ok(
+      run_komodo_standard_command("Update Swarm Node", None, command)
+        .await,
+    )
+  }
+}
+
+impl Resolve<crate::api::Args> for RemoveSwarmNodes {
+  #[instrument(
+    "RemoveSwarmNodes",
+    skip_all,
+    fields(
+      id = args.id.to_string(),
+      core = args.core,
+      nodes = serde_json::to_string(&self.nodes).unwrap_or_else(|e| e.to_string()),
+      force = self.force,
+    )
+  )]
+  async fn resolve(
+    self,
+    args: &crate::api::Args,
+  ) -> anyhow::Result<Log> {
+    let mut command = String::from("docker node rm");
+    if self.force {
+      command += " --force"
+    }
+    for node in self.nodes {
+      command += " ";
+      command += &node;
+    }
+    Ok(
+      run_komodo_standard_command(
+        "Remove Swarm Nodes",
+        None,
+        command,
+      )
+      .await,
+    )
+  }
+}
+
+// ======
+//  Task
+// ======
+
+impl Resolve<crate::api::Args> for InspectSwarmTask {
+  async fn resolve(
+    self,
+    _: &crate::api::Args,
+  ) -> anyhow::Result<SwarmTask> {
+    let client = docker_client().load();
+    let client = client
+      .iter()
+      .next()
+      .context("Could not connect to docker client")?;
+    client.inspect_swarm_task(&self.task).await
+  }
+}
+
+// ========
+//  Config
+// ========
+
+impl Resolve<crate::api::Args> for InspectSwarmConfig {
+  async fn resolve(
+    self,
+    _: &crate::api::Args,
+  ) -> anyhow::Result<Vec<SwarmConfig>> {
+    inspect_swarm_config(&self.config).await
+  }
+}
+
+impl Resolve<crate::api::Args> for RemoveSwarmConfigs {
+  #[instrument(
+    "RemoveSwarmConfigs",
+    skip_all,
+    fields(
+      id = args.id.to_string(),
+      core = args.core,
+      configs = serde_json::to_string(&self.configs).unwrap_or_else(|e| e.to_string()),
+    )
+  )]
+  async fn resolve(
+    self,
+    args: &crate::api::Args,
+  ) -> anyhow::Result<Log> {
+    let mut command = String::from("docker config rm");
+    for config in self.configs {
+      command += " ";
+      command += &config;
+    }
+    Ok(
+      run_komodo_standard_command(
+        "Remove Swarm Configs",
+        None,
+        command,
+      )
+      .await,
+    )
+  }
+}
+
+// ========
+//  Secret
+// ========
+
+impl Resolve<crate::api::Args> for InspectSwarmSecret {
+  async fn resolve(
+    self,
+    _: &crate::api::Args,
+  ) -> anyhow::Result<SwarmSecret> {
+    let client = docker_client().load();
+    let client = client
+      .iter()
+      .next()
+      .context("Could not connect to docker client")?;
+    client.inspect_swarm_secret(&self.secret).await
+  }
+}
+
+impl Resolve<crate::api::Args> for RemoveSwarmSecrets {
+  #[instrument(
+    "RemoveSwarmSecrets",
+    skip_all,
+    fields(
+      id = args.id.to_string(),
+      core = args.core,
+      secrets = serde_json::to_string(&self.secrets).unwrap_or_else(|e| e.to_string()),
+    )
+  )]
+  async fn resolve(
+    self,
+    args: &crate::api::Args,
+  ) -> anyhow::Result<Log> {
+    let mut command = String::from("docker secret rm");
+    for secret in self.secrets {
+      command += " ";
+      command += &secret;
+    }
+    Ok(
+      run_komodo_standard_command(
+        "Remove Swarm Secrets",
+        None,
+        command,
+      )
+      .await,
+    )
+  }
+}

bin/periphery/src/api/swarm/service.rs

@@ -0,0 +1,421 @@
+use std::fmt::Write;
+
+use anyhow::Context as _;
+use command::{
+  KomodoCommandMode, run_komodo_command_with_sanitization,
+  run_komodo_shell_command, run_komodo_standard_command,
+};
+use formatting::format_serror;
+use interpolate::Interpolator;
+use komodo_client::entities::{
+  deployment::{
+    Deployment, DeploymentConfig, DeploymentImage,
+    conversions_from_str, extract_registry_domain,
+  },
+  docker::service::SwarmService,
+  environment_vars_from_str,
+  update::Log,
+};
+use periphery_client::api::swarm::{
+  CreateSwarmService, GetSwarmServiceLog, GetSwarmServiceLogSearch,
+  InspectSwarmService, RemoveSwarmServices, UpdateSwarmService,
+};
+use resolver_api::Resolve;
+use tracing::Instrument;
+
+use crate::{
+  config::periphery_config,
+  docker::docker_login,
+  helpers::{
+    format_log_grep, push_conversions, push_environment,
+    push_extra_args, push_labels,
+  },
+  state::docker_client,
+};
+
+impl Resolve<crate::api::Args> for InspectSwarmService {
+  async fn resolve(
+    self,
+    _: &crate::api::Args,
+  ) -> anyhow::Result<SwarmService> {
+    let client = docker_client().load();
+    let client = client
+      .iter()
+      .next()
+      .context("Could not connect to docker client")?;
+    client.inspect_swarm_service(&self.service).await
+  }
+}
+
+impl Resolve<crate::api::Args> for GetSwarmServiceLog {
+  async fn resolve(
+    self,
+    _: &crate::api::Args,
+  ) -> anyhow::Result<Log> {
+    let GetSwarmServiceLog {
+      service,
+      tail,
+      timestamps,
+      no_task_ids,
+      no_resolve,
+      details,
+    } = self;
+    let timestamps = if timestamps {
+      " --timestamps"
+    } else {
+      Default::default()
+    };
+    let no_task_ids = if no_task_ids {
+      " --no-task-ids"
+    } else {
+      Default::default()
+    };
+    let no_resolve = if no_resolve {
+      " --no-resolve"
+    } else {
+      Default::default()
+    };
+    let details = if details {
+      " --details"
+    } else {
+      Default::default()
+    };
+    let command = format!(
+      "docker service logs --tail {tail}{timestamps}{no_task_ids}{no_resolve}{details} {service}",
+    );
+    Ok(
+      run_komodo_standard_command(
+        "Get Swarm Service Log",
+        None,
+        command,
+      )
+      .await,
+    )
+  }
+}
+
+impl Resolve<crate::api::Args> for GetSwarmServiceLogSearch {
+  async fn resolve(
+    self,
+    _: &crate::api::Args,
+  ) -> anyhow::Result<Log> {
+    let GetSwarmServiceLogSearch {
+      service,
+      terms,
+      combinator,
+      invert,
+      timestamps,
+      no_task_ids,
+      no_resolve,
+      details,
+    } = self;
+    let timestamps = if timestamps {
+      " --timestamps"
+    } else {
+      Default::default()
+    };
+    let no_task_ids = if no_task_ids {
+      " --no-task-ids"
+    } else {
+      Default::default()
+    };
+    let no_resolve = if no_resolve {
+      " --no-resolve"
+    } else {
+      Default::default()
+    };
+    let details = if details {
+      " --details"
+    } else {
+      Default::default()
+    };
+    let grep = format_log_grep(&terms, combinator, invert);
+    let command = format!(
+      "docker service logs --tail 5000{timestamps}{no_task_ids}{no_resolve}{details} {service} 2>&1 | {grep}",
+    );
+    Ok(
+      run_komodo_shell_command(
+        "Search Swarm Service Log",
+        None,
+        command,
+      )
+      .await,
+    )
+  }
+}
+
+impl Resolve<crate::api::Args> for RemoveSwarmServices {
+  #[instrument(
+    "RemoveSwarmServices",
+    skip_all,
+    fields(
+      id = args.id.to_string(),
+      core = args.core,
+      services = serde_json::to_string(&self.services).unwrap_or_else(|e| e.to_string()),
+    )
+  )]
+  async fn resolve(
+    self,
+    args: &crate::api::Args,
+  ) -> anyhow::Result<Log> {
+    let mut command = String::from("docker service rm");
+    for service in self.services {
+      command += " ";
+      command += &service;
+    }
+    Ok(
+      run_komodo_standard_command(
+        "Remove Swarm Services",
+        None,
+        command,
+      )
+      .await,
+    )
+  }
+}
+
+impl Resolve<crate::api::Args> for CreateSwarmService {
+  #[instrument(
+    "CreateSwarmService",
+    skip_all,
+    fields(
+      id = args.id.to_string(),
+      core = args.core,
+      deployment = &self.deployment.name,
+    )
+  )]
+  async fn resolve(
+    self,
+    args: &crate::api::Args,
+  ) -> Result<Self::Response, Self::Error> {
+    let CreateSwarmService {
+      mut deployment,
+      registry_token,
+      mut replacers,
+    } = self;
+
+    let mut logs = Vec::new();
+
+    let mut interpolator =
+      Interpolator::new(None, &periphery_config().secrets);
+    interpolator.interpolate_deployment(&mut deployment)?;
+    replacers.extend(interpolator.secret_replacers);
+
+    let image = if let DeploymentImage::Image { image } =
+      &deployment.config.image
+    {
+      if image.is_empty() {
+        logs.push(Log::error(
+          "Get Image",
+          String::from("Deployment does not have image attached"),
+        ));
+        return Ok(logs);
+      }
+      image
+    } else {
+      logs.push(Log::error(
+        "Get Image",
+        String::from(
+          "Deployment does not have build replaced by Core",
+        ),
+      ));
+      return Ok(logs);
+    };
+
+    let use_with_registry_auth = match docker_login(
+      &extract_registry_domain(image)?,
+      &deployment.config.image_registry_account,
+      registry_token.as_deref(),
+    )
+    .await
+    {
+      Ok(res) => res,
+      Err(e) => {
+        logs.push(Log::error(
+          "Docker Login",
+          format_serror(
+            &e.context("Failed to login to docker registry").into(),
+          ),
+        ));
+        return Ok(logs);
+      }
+    };
+
+    let log = (RemoveSwarmServices {
+      services: vec![deployment.name.clone()],
+    })
+    .resolve(args)
+    .await;
+
+    // This is only necessary when it successfully
+    // takes down existing service before redeploy.
+    if let Ok(log) = log
+      && log.success
+    {
+      logs.push(log)
+    }
+
+    let command = docker_service_create_command(
+      &deployment,
+      image,
+      use_with_registry_auth,
+    )
+    .context(
+      "Unable to generate valid docker service create command",
+    )?;
+
+    let span = info_span!("ExecuteDockerServiceCreate");
+    if let Some(log) = run_komodo_command_with_sanitization(
+      "Docker Service Create",
+      None,
+      command,
+      KomodoCommandMode::Shell,
+      &replacers,
+    )
+    .instrument(span)
+    .await
+    {
+      logs.push(log)
+    };
+
+    Ok(logs)
+  }
+}
+
+fn docker_service_create_command(
+  Deployment {
+    name,
+    config:
+      DeploymentConfig {
+        volumes,
+        ports,
+        network,
+        command,
+        environment,
+        labels,
+        extra_args,
+        ..
+      },
+    ..
+  }: &Deployment,
+  image: &str,
+  use_with_registry_auth: bool,
+) -> anyhow::Result<String> {
+  let mut res = format!(
+    "docker service create --name {name} --network {network}"
+  );
+
+  push_conversions(
+    &mut res,
+    &conversions_from_str(ports).context("Invalid ports")?,
+    "-p",
+  )?;
+
+  push_conversions(
+    &mut res,
+    &conversions_from_str(volumes).context("Invalid volumes")?,
+    "--mount",
+  )?;
+
+  push_environment(
+    &mut res,
+    &environment_vars_from_str(environment)
+      .context("Invalid environment")?,
+  )?;
+
+  push_labels(
+    &mut res,
+    &environment_vars_from_str(labels).context("Invalid labels")?,
+  )?;
+
+  if use_with_registry_auth {
+    res += " --with-registry-auth";
+  }
+
+  push_extra_args(&mut res, extra_args)?;
+
+  write!(&mut res, " {image}")?;
+
+  if !command.is_empty() {
+    write!(&mut res, " {command}")?;
+  }
+
+  Ok(res)
+}
+
+impl Resolve<crate::api::Args> for UpdateSwarmService {
+  #[instrument(
+    "UpdateSwarmService",
+    skip_all,
+    fields(
+      id = args.id.to_string(),
+      core = args.core,
+      service = self.service,
+      update = serde_json::to_string(&self).unwrap_or_else(|e| e.to_string()),
+    )
+  )]
+  async fn resolve(
+    self,
+    args: &crate::api::Args,
+  ) -> Result<Self::Response, Self::Error> {
+    let UpdateSwarmService {
+      service,
+      registry_account,
+      registry_token,
+      image,
+      replicas,
+      rollback,
+      extra_args,
+    } = self;
+
+    let mut command = format!("docker service update");
+
+    if let Some(image) = image {
+      write!(&mut command, " --image {image}")?;
+
+      match docker_login(
+        &extract_registry_domain(&image)?,
+        &registry_account.unwrap_or_default(),
+        registry_token.as_deref(),
+      )
+      .await
+      {
+        Ok(res) if res => {
+          write!(&mut command, " --with-registry-auth")?;
+        }
+        Ok(_) => {}
+        Err(e) => {
+          return Ok(Log::error(
+            "Docker Login",
+            format_serror(
+              &e.context("Failed to login to docker registry").into(),
+            ),
+          ));
+        }
+      };
+    }
+
+    if let Some(replicas) = replicas {
+      write!(&mut command, " --replicas={replicas}")?;
+    }
+
+    if rollback {
+      write!(&mut command, " --rollback")?;
+    }
+
+    push_extra_args(&mut command, &extra_args)?;
+
+    write!(&mut command, " {service}")?;
+
+    let span = info_span!("ExecuteDockerServiceCreate");
+    let log = run_komodo_standard_command(
+      "Docker Service Create",
+      None,
+      command,
+    )
+    .instrument(span)
+    .await;
+
+    Ok(log)
+  }
+}

bin/periphery/src/api/swarm/stack.rs

@@ -0,0 +1,309 @@
+use std::fmt::Write;
+
+use anyhow::{Context as _, anyhow};
+use command::{
+  KomodoCommandMode, run_komodo_command_with_sanitization,
+  run_komodo_standard_command,
+};
+use formatting::format_serror;
+use interpolate::Interpolator;
+use komodo_client::{
+  entities::{
+    all_logs_success,
+    docker::stack::SwarmStack,
+    stack::{ComposeFile, ComposeService, StackServiceNames},
+    update::Log,
+  },
+  parsers::parse_multiline_command,
+};
+use periphery_client::api::{
+  DeployStackResponse,
+  swarm::{DeploySwarmStack, InspectSwarmStack, RemoveSwarmStacks},
+};
+use resolver_api::Resolve;
+use tracing::Instrument as _;
+
+use crate::{
+  config::periphery_config,
+  docker::stack::inspect_swarm_stack,
+  helpers::push_extra_args,
+  stack::{maybe_login_registry, validate_files, write::write_stack},
+};
+
+impl Resolve<crate::api::Args> for InspectSwarmStack {
+  async fn resolve(
+    self,
+    _: &crate::api::Args,
+  ) -> anyhow::Result<SwarmStack> {
+    inspect_swarm_stack(self.stack).await
+  }
+}
+
+impl Resolve<crate::api::Args> for RemoveSwarmStacks {
+  #[instrument(
+    "RemoveSwarmStacks",
+    skip_all,
+    fields(
+      id = args.id.to_string(),
+      core = args.core,
+      stacks = serde_json::to_string(&self.stacks).unwrap_or_else(|e| e.to_string()),
+    )
+  )]
+  async fn resolve(
+    self,
+    args: &crate::api::Args,
+  ) -> anyhow::Result<Log> {
+    let mut command = String::from("docker stack rm");
+    // This defaults to true, only need when false
+    if !self.detach {
+      command += " --detach=false"
+    }
+    for stack in self.stacks {
+      command += " ";
+      command += &stack;
+    }
+    Ok(
+      run_komodo_standard_command(
+        "Remove Swarm Stacks",
+        None,
+        command,
+      )
+      .await,
+    )
+  }
+}
+
+impl Resolve<crate::api::Args> for DeploySwarmStack {
+  #[instrument(
+    "DeploySwarmStack",
+    skip_all,
+    fields(
+      id = args.id.to_string(),
+      core = args.core,
+      stack = self.stack.name,
+      repo = self.repo.as_ref().map(|repo| &repo.name),
+    )
+  )]
+  async fn resolve(
+    self,
+    args: &crate::api::Args,
+  ) -> Result<Self::Response, Self::Error> {
+    let DeploySwarmStack {
+      mut stack,
+      repo,
+      git_token,
+      registry_token,
+      mut replacers,
+    } = self;
+
+    let mut res = DeployStackResponse::default();
+
+    let mut interpolator =
+      Interpolator::new(None, &periphery_config().secrets);
+    // Only interpolate Stack. Repo interpolation will be handled
+    // by the CloneRepo / PullOrCloneRepo call.
+    interpolator
+      .interpolate_stack(&mut stack)?
+      .push_logs(&mut res.logs);
+    replacers.extend(interpolator.secret_replacers);
+
+    // Env files are not supported by docker stack deploy so are ignored.
+    let (run_directory, _) = match write_stack(
+      &stack,
+      repo.as_ref(),
+      git_token,
+      replacers.clone(),
+      &mut res,
+      args,
+    )
+    .await
+    {
+      Ok(res) => res,
+      Err(e) => {
+        res
+          .logs
+          .push(Log::error("Write Stack", format_serror(&e.into())));
+        return Ok(res);
+      }
+    };
+
+    // Canonicalize the path to ensure it exists, and is the cleanest path to the run directory.
+    let run_directory = run_directory.canonicalize().context(
+      "Failed to validate run directory on host after stack write (canonicalize error)",
+    )?;
+
+    validate_files(&stack, &run_directory, &mut res).await;
+    if !all_logs_success(&res.logs) {
+      return Ok(res);
+    }
+
+    let use_with_registry_auth =
+      maybe_login_registry(&stack, registry_token, &mut res.logs)
+        .await;
+    if !all_logs_success(&res.logs) {
+      return Ok(res);
+    }
+
+    // Pre deploy
+    if !stack.config.pre_deploy.is_none() {
+      let pre_deploy_path =
+        run_directory.join(&stack.config.pre_deploy.path);
+      let span = info_span!("ExecutePreDeploy");
+      if let Some(log) = run_komodo_command_with_sanitization(
+        "Pre Deploy",
+        pre_deploy_path.as_path(),
+        &stack.config.pre_deploy.command,
+        KomodoCommandMode::Multiline,
+        &replacers,
+      )
+      .instrument(span)
+      .await
+      {
+        res.logs.push(log);
+        if !all_logs_success(&res.logs) {
+          return Ok(res);
+        }
+      };
+    }
+
+    let file_args = stack.compose_file_paths().join(" -c ");
+
+    // This will be the last project name, which is the one that needs to be destroyed.
+    // Might be different from the current project name, if user renames stack / changes to custom project name.
+    let last_project_name = stack.project_name(false);
+    let project_name = stack.project_name(true);
+
+    // Uses 'docker stack config' command to extract services (including image)
+    // after performing interpolation
+    {
+      let command = format!("docker stack config -c {file_args}",);
+      let span = info_span!("GetStackConfig", command);
+      let Some(config_log) = run_komodo_command_with_sanitization(
+        "Stack Config",
+        run_directory.as_path(),
+        command,
+        KomodoCommandMode::Standard,
+        &replacers,
+      )
+      .instrument(span)
+      .await
+      else {
+        // Only reachable if command is empty,
+        // not the case since it is provided above.
+        unreachable!()
+      };
+      if !config_log.success {
+        res.logs.push(config_log);
+        return Ok(res);
+      }
+      let compose =
+        serde_yaml_ng::from_str::<ComposeFile>(&config_log.stdout)
+          .context("Failed to parse compose contents")?;
+      // Store sanitized stack config output
+      res.merged_config = Some(config_log.stdout);
+      for (service_name, ComposeService { image, .. }) in
+        compose.services
+      {
+        let image = image.unwrap_or_default();
+        res.services.push(StackServiceNames {
+          container_name: format!("{project_name}-{service_name}"),
+          service_name,
+          image,
+        });
+      }
+    }
+
+    if stack.config.destroy_before_deploy
+      // Also check if project name changed, which also requires taking down.
+      || last_project_name != project_name
+    {
+      // Take down the existing stack.
+      // This one tries to use the previously deployed project name, to ensure the right stack is taken down.
+      remove_stack(&last_project_name, &mut res)
+        .await
+        .context("Failed to destroy existing stack")?;
+    }
+
+    // Run stack deploy
+    let mut command =
+      format!("docker stack deploy --detach=false -c {file_args}");
+    if use_with_registry_auth {
+      command += " --with-registry-auth";
+    }
+    push_extra_args(&mut command, &stack.config.extra_args)?;
+    write!(&mut command, " {project_name}")?;
+
+    // Apply compose cmd wrapper if configured
+    let compose_cmd_wrapper =
+      parse_multiline_command(&stack.config.compose_cmd_wrapper);
+    if !compose_cmd_wrapper.is_empty() {
+      if !compose_cmd_wrapper.contains("[[COMPOSE_COMMAND]]") {
+        res.logs.push(Log::error(
+          "Compose Command Wrapper",
+          "compose_cmd_wrapper is configured but does not contain [[COMPOSE_COMMAND]] placeholder. The placeholder is required to inject the compose command.".to_string(),
+        ));
+        return Ok(res);
+      }
+      command =
+        compose_cmd_wrapper.replace("[[COMPOSE_COMMAND]]", &command);
+    }
+
+    let span = info_span!("ExecuteStackDeploy");
+    let Some(log) = run_komodo_command_with_sanitization(
+      "Deploy Swarm Stack",
+      run_directory.as_path(),
+      command,
+      KomodoCommandMode::Shell,
+      &replacers,
+    )
+    .instrument(span)
+    .await
+    else {
+      unreachable!()
+    };
+
+    res.deployed = log.success;
+    res.logs.push(log);
+
+    if res.deployed && !stack.config.post_deploy.is_none() {
+      let post_deploy_path =
+        run_directory.join(&stack.config.post_deploy.path);
+      let span = info_span!("ExecutePostDeploy");
+      if let Some(log) = run_komodo_command_with_sanitization(
+        "Post Deploy",
+        post_deploy_path.as_path(),
+        &stack.config.post_deploy.command,
+        KomodoCommandMode::Multiline,
+        &replacers,
+      )
+      .instrument(span)
+      .await
+      {
+        res.logs.push(log);
+      };
+    }
+
+    Ok(res)
+  }
+}
+
+#[instrument("RemoveStack", skip(res))]
+async fn remove_stack(
+  stack: &str,
+  res: &mut DeployStackResponse,
+) -> anyhow::Result<()> {
+  let log = run_komodo_standard_command(
+    "Remove Stack",
+    None,
+    format!("docker stack rm --detach=false {stack}"),
+  )
+  .await;
+  let success = log.success;
+  res.logs.push(log);
+  if !success {
+    return Err(anyhow!(
+      "Failed to remove existing stack with docker stack rm. Stopping run."
+    ));
+  }
+  Ok(())
+}

bin/periphery/src/api/terminal.rs

@@ -25,10 +25,10 @@ use crate::{
 
 //
 
-impl Resolve<super::Args> for ListTerminals {
+impl Resolve<crate::api::Args> for ListTerminals {
   async fn resolve(
     self,
-    _: &super::Args,
+    _: &crate::api::Args,
   ) -> anyhow::Result<Vec<Terminal>> {
     clean_up_terminals().await;
     Ok(list_terminals(self.target.as_ref()).await)
@@ -37,7 +37,7 @@ impl Resolve<super::Args> for ListTerminals {
 
 //
 
-impl Resolve<super::Args> for CreateServerTerminal {
+impl Resolve<crate::api::Args> for CreateServerTerminal {
   #[instrument(
     "CreateServerTerminal",
     skip_all,
@@ -51,7 +51,7 @@ impl Resolve<super::Args> for CreateServerTerminal {
   )]
   async fn resolve(
     self,
-    args: &super::Args,
+    args: &crate::api::Args,
   ) -> anyhow::Result<NoData> {
     if periphery_config().disable_terminals {
       return Err(anyhow!(
@@ -71,7 +71,7 @@ impl Resolve<super::Args> for CreateServerTerminal {
 
 //
 
-impl Resolve<super::Args> for CreateContainerExecTerminal {
+impl Resolve<crate::api::Args> for CreateContainerExecTerminal {
   #[instrument(
     "CreateContainerExecTerminal",
     skip_all,
@@ -86,7 +86,7 @@ impl Resolve<super::Args> for CreateContainerExecTerminal {
   )]
   async fn resolve(
     self,
-    args: &super::Args,
+    args: &crate::api::Args,
   ) -> anyhow::Result<NoData> {
     if periphery_config().disable_container_terminals {
       return Err(anyhow!(
@@ -119,7 +119,7 @@ impl Resolve<super::Args> for CreateContainerExecTerminal {
 
 //
 
-impl Resolve<super::Args> for CreateContainerAttachTerminal {
+impl Resolve<crate::api::Args> for CreateContainerAttachTerminal {
   #[instrument(
     "CreateContainerAttachTerminal",
     skip_all,
@@ -133,7 +133,7 @@ impl Resolve<super::Args> for CreateContainerAttachTerminal {
   )]
   async fn resolve(
     self,
-    args: &super::Args,
+    args: &crate::api::Args,
   ) -> anyhow::Result<NoData> {
     if periphery_config().disable_container_terminals {
       return Err(anyhow!(
@@ -164,7 +164,7 @@ impl Resolve<super::Args> for CreateContainerAttachTerminal {
 
 //
 
-impl Resolve<super::Args> for DeleteTerminal {
+impl Resolve<crate::api::Args> for DeleteTerminal {
   #[instrument(
     "DeleteTerminal",
     skip_all,
@@ -176,7 +176,7 @@ impl Resolve<super::Args> for DeleteTerminal {
   )]
   async fn resolve(
     self,
-    args: &super::Args,
+    args: &crate::api::Args,
   ) -> anyhow::Result<NoData> {
     delete_terminal(&self.target, &self.terminal).await;
     Ok(NoData {})
@@ -185,7 +185,7 @@ impl Resolve<super::Args> for DeleteTerminal {
 
 //
 
-impl Resolve<super::Args> for DeleteAllTerminals {
+impl Resolve<crate::api::Args> for DeleteAllTerminals {
   #[instrument(
     "DeleteAllTerminals",
     skip_all,
@@ -196,7 +196,7 @@ impl Resolve<super::Args> for DeleteAllTerminals {
   )]
   async fn resolve(
     self,
-    args: &super::Args,
+    args: &crate::api::Args,
   ) -> anyhow::Result<NoData> {
     delete_all_terminals().await;
     Ok(NoData {})
@@ -205,7 +205,7 @@ impl Resolve<super::Args> for DeleteAllTerminals {
 
 //
 
-impl Resolve<super::Args> for ConnectTerminal {
+impl Resolve<crate::api::Args> for ConnectTerminal {
   #[instrument(
     "ConnectTerminal",
     skip_all,
@@ -215,7 +215,10 @@ impl Resolve<super::Args> for ConnectTerminal {
       terminal = self.terminal,
     )
   )]
-  async fn resolve(self, args: &super::Args) -> anyhow::Result<Uuid> {
+  async fn resolve(
+    self,
+    args: &crate::api::Args,
+  ) -> anyhow::Result<Uuid> {
     let connection =
       core_connections().get(&args.core).await.with_context(
         || format!("Failed to find channel for {}", args.core),
@@ -234,7 +237,7 @@ impl Resolve<super::Args> for ConnectTerminal {
 
 //
 
-impl Resolve<super::Args> for DisconnectTerminal {
+impl Resolve<crate::api::Args> for DisconnectTerminal {
   #[instrument(
     "DisconnectTerminal",
     skip_all,
@@ -246,7 +249,7 @@ impl Resolve<super::Args> for DisconnectTerminal {
   )]
   async fn resolve(
     self,
-    args: &super::Args,
+    args: &crate::api::Args,
   ) -> anyhow::Result<NoData> {
     terminal_channels().remove(&self.channel).await;
     Ok(NoData {})
@@ -255,7 +258,7 @@ impl Resolve<super::Args> for DisconnectTerminal {
 
 //
 
-impl Resolve<super::Args> for ExecuteTerminal {
+impl Resolve<crate::api::Args> for ExecuteTerminal {
   #[instrument(
     "ExecuteTerminal",
     skip_all,
@@ -266,7 +269,10 @@ impl Resolve<super::Args> for ExecuteTerminal {
       command = self.command,
     )
   )]
-  async fn resolve(self, args: &super::Args) -> anyhow::Result<Uuid> {
+  async fn resolve(
+    self,
+    args: &crate::api::Args,
+  ) -> anyhow::Result<Uuid> {
     let channel =
       core_connections().get(&args.core).await.with_context(
         || format!("Failed to find channel for {}", args.core),

bin/periphery/src/docker/compose.rs

@@ -29,7 +29,7 @@ pub async fn list_compose_projects()
     )));
   }
 
-  let res =
+  let mut res =
     serde_json::from_str::<Vec<DockerComposeLsItem>>(&res.stdout)
       .with_context(|| res.stdout.clone())
       .with_context(|| {
@@ -48,7 +48,11 @@ pub async fn list_compose_projects()
           .map(str::to_string)
           .collect(),
       })
-      .collect();
+      .collect::<Vec<_>>();
+
+  res.sort_by(|a, b| {
+    a.status.cmp(&b.status).then_with(|| a.name.cmp(&b.name))
+  });
 
   Ok(res)
 }

bin/periphery/src/docker/config.rs

@@ -14,17 +14,24 @@ pub async fn list_swarm_configs()
   .await;
 
   if !res.success {
-    return Err(anyhow!("{}", res.combined()).context(format!(
-      "Failed to list swarm configs using 'docker config ls'"
-    )));
+    return Err(anyhow!("{}", res.combined()).context(
+      "Failed to list swarm configs using 'docker config ls'",
+    ));
   }
 
   // The output is in JSONL, need to convert to standard JSON vec.
-  serde_json::from_str(&format!(
-    "[{}]",
-    res.stdout.trim().replace('\n', ",")
-  ))
-  .context("Failed to parse 'docker config ls' response from json")
+  let mut res = serde_json::from_str::<Vec<SwarmConfigListItem>>(
+    &format!("[{}]", res.stdout.trim().replace('\n', ",")),
+  )
+  .context("Failed to parse 'docker config ls' response from json")?;
+
+  res.sort_by(|a, b| {
+    a.name
+      .cmp(&b.name)
+      .then_with(|| b.updated_at.cmp(&a.updated_at))
+  });
+
+  Ok(res)
 }
 
 pub async fn inspect_swarm_config(

bin/periphery/src/docker/container.rs

@@ -100,6 +100,9 @@ impl DockerClient {
       container.network_mode =
         container_id_to_network.get(container_id).cloned();
     });
+    containers.sort_by(|a, b| {
+      a.state.cmp(&b.state).then_with(|| a.name.cmp(&b.name))
+    });
     Ok(containers)
   }
 

bin/periphery/src/docker/image.rs

@@ -11,7 +11,7 @@ impl DockerClient {
     &self,
     containers: &[ContainerListItem],
   ) -> anyhow::Result<Vec<ImageListItem>> {
-    let images = self
+    let mut images = self
       .docker
       .list_images(Option::<ListImagesOptions>::None)
       .await?
@@ -37,7 +37,12 @@ impl DockerClient {
           in_use,
         }
       })
-      .collect();
+      .collect::<Vec<_>>();
+
+    images.sort_by(|a, b| {
+      a.in_use.cmp(&b.in_use).then_with(|| a.name.cmp(&b.name))
+    });
+
     Ok(images)
   }
 

bin/periphery/src/docker/mod.rs

@@ -34,7 +34,7 @@ impl DockerClient {
   }
 }
 
-/// Returns whether build result should be pushed after build
+/// Returns whether login was actually performed.
 #[instrument("DockerLogin", skip(registry_token))]
 pub async fn docker_login(
   domain: &str,
@@ -45,30 +45,33 @@ pub async fn docker_login(
   if domain.is_empty() || account.is_empty() {
     return Ok(false);
   }
+
   let registry_token = match registry_token {
     Some(token) => token,
     None => crate::helpers::registry_token(domain, account)?,
   };
+
   let log = run_shell_command(&format!(
     "echo {registry_token} | docker login {domain} --username '{account}' --password-stdin",
   ), None)
   .await;
+
   if log.success() {
-    Ok(true)
-  } else {
-    let mut e = anyhow!("End of trace");
-    for line in
-      log.stderr.split('\n').filter(|line| !line.is_empty()).rev()
-    {
-      e = e.context(line.to_string());
-    }
-    for line in
-      log.stdout.split('\n').filter(|line| !line.is_empty()).rev()
-    {
-      e = e.context(line.to_string());
-    }
-    Err(e.context(format!("Registry {domain} login error")))
+    return Ok(true);
   }
+
+  let mut e = anyhow!("End of trace");
+  for line in
+    log.stderr.split('\n').filter(|line| !line.is_empty()).rev()
+  {
+    e = e.context(line.to_string());
+  }
+  for line in
+    log.stdout.split('\n').filter(|line| !line.is_empty()).rev()
+  {
+    e = e.context(line.to_string());
+  }
+  Err(e.context(format!("Registry {domain} login error")))
 }
 
 #[instrument("PullImage")]

bin/periphery/src/docker/network.rs

@@ -12,7 +12,7 @@ impl DockerClient {
     &self,
     containers: &[ContainerListItem],
   ) -> anyhow::Result<Vec<NetworkListItem>> {
-    let networks = self
+    let mut networks = self
       .docker
       .list_networks(Option::<ListNetworksOptions>::None)
       .await?
@@ -54,7 +54,12 @@ impl DockerClient {
           in_use,
         }
       })
-      .collect();
+      .collect::<Vec<_>>();
+
+    networks.sort_by(|a, b| {
+      a.in_use.cmp(&b.in_use).then_with(|| a.name.cmp(&b.name))
+    });
+
     Ok(networks)
   }
 

bin/periphery/src/docker/node.rs

@@ -11,14 +11,22 @@ impl DockerClient {
   pub async fn list_swarm_nodes(
     &self,
   ) -> anyhow::Result<Vec<SwarmNodeListItem>> {
-    let nodes = self
+    let mut nodes = self
       .docker
       .list_nodes(Option::<ListNodesOptions>::None)
       .await
       .context("Failed to query for swarm node list")?
       .into_iter()
       .map(convert_node_list_item)
-      .collect();
+      .collect::<Vec<_>>();
+
+    nodes.sort_by(|a, b| {
+      a.state
+        .cmp(&b.state)
+        .then_with(|| a.name.cmp(&b.name))
+        .then_with(|| a.hostname.cmp(&b.hostname))
+    });
+
     Ok(nodes)
   }
 

bin/periphery/src/docker/secret.rs

@@ -10,14 +10,21 @@ impl DockerClient {
   pub async fn list_swarm_secrets(
     &self,
   ) -> anyhow::Result<Vec<SwarmSecretListItem>> {
-    let secrets = self
+    let mut secrets = self
       .docker
       .list_secrets(Option::<ListSecretsOptions>::None)
       .await
       .context("Failed to query for swarm secret list")?
       .into_iter()
       .map(convert_secret_list_item)
-      .collect();
+      .collect::<Vec<_>>();
+
+    secrets.sort_by(|a, b| {
+      a.name
+        .cmp(&b.name)
+        .then_with(|| b.updated_at.cmp(&a.updated_at))
+    });
+
     Ok(secrets)
   }
 

bin/periphery/src/docker/service.rs

@@ -2,8 +2,9 @@ use anyhow::Context;
 use bollard::query_parameters::{
   InspectServiceOptions, ListServicesOptions,
 };
-use komodo_client::entities::docker::{
-  NetworkAttachmentConfig, service::*,
+use komodo_client::entities::{
+  docker::{NetworkAttachmentConfig, service::*},
+  swarm::SwarmState,
 };
 
 use super::*;
@@ -12,15 +13,23 @@ impl DockerClient {
   /// List swarm services
   pub async fn list_swarm_services(
     &self,
+    tasks: &[SwarmTaskListItem],
   ) -> anyhow::Result<Vec<SwarmServiceListItem>> {
-    let services = self
+    let mut services = self
       .docker
       .list_services(Option::<ListServicesOptions>::None)
       .await
       .context("Failed to query for swarm service list")?
       .into_iter()
-      .map(convert_service_list_item)
-      .collect();
+      .map(|service| convert_service_list_item(service, tasks))
+      .collect::<Vec<_>>();
+
+    services.sort_by(|a, b| {
+      a.name
+        .cmp(&b.name)
+        .then_with(|| b.updated_at.cmp(&a.updated_at))
+    });
+
     Ok(services)
   }
 
@@ -48,6 +57,7 @@ impl DockerClient {
 
 fn convert_service_list_item(
   service: bollard::models::Service,
+  tasks: &[SwarmTaskListItem],
 ) -> SwarmServiceListItem {
   let (name, (image, restart, runtime), replicas) = service
     .spec
@@ -74,9 +84,15 @@ fn convert_service_list_item(
       )
     })
     .unwrap_or_default();
+  let state = service
+    .id
+    .as_ref()
+    .map(|service_id| service_state_from_tasks(service_id, tasks))
+    .unwrap_or(SwarmState::Unknown);
   SwarmServiceListItem {
     id: service.id,
     name,
+    state,
     replicas,
     image,
     restart,
@@ -86,6 +102,25 @@ fn convert_service_list_item(
   }
 }
 
+fn service_state_from_tasks(
+  service_id: &str,
+  tasks: &[SwarmTaskListItem],
+) -> SwarmState {
+  let tasks = tasks.iter().filter(|task| {
+    task
+      .service_id
+      .as_ref()
+      .map(|sid| sid == service_id)
+      .unwrap_or_default()
+  });
+  for task in tasks {
+    if task.state != task.desired_state {
+      return SwarmState::Unhealthy;
+    }
+  }
+  SwarmState::Healthy
+}
+
 fn convert_service(
   service: bollard::models::Service,
 ) -> SwarmService {

bin/periphery/src/docker/stack.rs

@@ -1,19 +1,25 @@
 use anyhow::{Context, anyhow};
 use command::run_komodo_standard_command;
-use komodo_client::entities::docker::stack::{
-  SwarmStackListItem, SwarmStackLists, SwarmStackServiceListItem,
-  SwarmStackTaskListItem,
+use futures_util::{StreamExt, stream::FuturesOrdered};
+use komodo_client::entities::{
+  docker::stack::{
+    SwarmStack, SwarmStackListItem, SwarmStackServiceListItem,
+    SwarmStackTaskListItem,
+  },
+  swarm::SwarmState,
 };
 
 pub async fn inspect_swarm_stack(
   name: String,
-) -> anyhow::Result<SwarmStackLists> {
-  let (services, tasks) = tokio::try_join!(
-    list_swarm_stack_services(&name),
+) -> anyhow::Result<SwarmStack> {
+  let (tasks, services) = tokio::try_join!(
     list_swarm_stack_tasks(&name),
+    list_swarm_stack_services(&name)
   )?;
-  Ok(SwarmStackLists {
+  let state = state_from_tasks(&tasks);
+  Ok(SwarmStack {
     name,
+    state,
     services,
     tasks,
   })
@@ -35,11 +41,35 @@ pub async fn list_swarm_stacks()
   }
 
   // The output is in JSONL, need to convert to standard JSON vec.
-  serde_json::from_str(&format!(
-    "[{}]",
-    res.stdout.trim().replace('\n', ",")
-  ))
-  .context("Failed to parse 'docker stack ls' response from json")
+  let mut stacks = serde_json::from_str::<Vec<SwarmStackListItem>>(
+    &format!("[{}]", res.stdout.trim().replace('\n', ",")),
+  )
+  .context("Failed to parse 'docker stack ls' response from json")?
+  // Attach state concurrently from tasks. Still include stack
+  // if it fails, just with None state.
+  .into_iter()
+  .map(|mut stack| async move {
+    let res = async {
+      let tasks =
+        list_swarm_stack_tasks(stack.name.as_ref()?).await.ok()?;
+      Some(state_from_tasks(&tasks))
+    }
+    .await;
+    if let Some(state) = res {
+      stack.state = Some(state);
+    }
+    stack
+  })
+  .collect::<FuturesOrdered<_>>()
+  .collect::<Vec<_>>()
+  .await;
+
+  stacks.sort_by(|a, b| {
+    cmp_option(a.state, b.state)
+      .then_with(|| cmp_option(a.name.as_ref(), b.name.as_ref()))
+  });
+
+  Ok(stacks)
 }
 
 pub async fn list_swarm_stack_services(
@@ -59,13 +89,18 @@ pub async fn list_swarm_stack_services(
   }
 
   // The output is in JSONL, need to convert to standard JSON vec.
-  serde_json::from_str(&format!(
-    "[{}]",
-    res.stdout.trim().replace('\n', ",")
-  ))
-  .context(
-    "Failed to parse 'docker stack services' response from json",
-  )
+  let mut services =
+    serde_json::from_str::<Vec<SwarmStackServiceListItem>>(&format!(
+      "[{}]",
+      res.stdout.trim().replace('\n', ",")
+    ))
+    .context(
+      "Failed to parse 'docker stack services' response from json",
+    )?;
+
+  services.sort_by(|a, b| a.name.cmp(&b.name));
+
+  Ok(services)
 }
 
 pub async fn list_swarm_stack_tasks(
@@ -74,7 +109,7 @@ pub async fn list_swarm_stack_tasks(
   let res = run_komodo_standard_command(
     "List Swarm Stack Tasks",
     None,
-    format!("docker stack ps --format json {stack}"),
+    format!("docker stack ps --format json --no-trunc {stack}"),
   )
   .await;
 
@@ -85,9 +120,53 @@ pub async fn list_swarm_stack_tasks(
   }
 
   // The output is in JSONL, need to convert to standard JSON vec.
-  serde_json::from_str(&format!(
-    "[{}]",
-    res.stdout.trim().replace('\n', ",")
-  ))
-  .context("Failed to parse 'docker stack ps' response from json")
+  let mut tasks =
+    serde_json::from_str::<Vec<SwarmStackTaskListItem>>(&format!(
+      "[{}]",
+      res.stdout.trim().replace('\n', ",")
+    ))
+    .context(
+      "Failed to parse 'docker stack ps' response from json",
+    )?;
+
+  tasks.sort_by(|a, b| {
+    a.desired_state
+      .cmp(&b.desired_state)
+      .then_with(|| a.name.cmp(&b.name))
+  });
+
+  Ok(tasks)
+}
+
+pub fn state_from_tasks<'a>(
+  tasks: impl IntoIterator<Item = &'a SwarmStackTaskListItem>,
+) -> SwarmState {
+  for task in tasks {
+    let (Some(current), Some(desired)) =
+      (&task.current_state, &task.desired_state)
+    else {
+      continue;
+    };
+    // CurrentState example: 'Running 44 minutes ago'.
+    // Only want first "word"
+    let Some(current) = current.split(" ").next() else {
+      continue;
+    };
+    if current != desired {
+      return SwarmState::Unhealthy;
+    }
+  }
+  SwarmState::Healthy
+}
+
+fn cmp_option<T: Ord>(
+  a: Option<T>,
+  b: Option<T>,
+) -> std::cmp::Ordering {
+  match (a, b) {
+    (Some(a), Some(b)) => a.cmp(&b),
+    (Some(_), None) => std::cmp::Ordering::Less,
+    (None, Some(_)) => std::cmp::Ordering::Greater,
+    (None, None) => std::cmp::Ordering::Equal,
+  }
 }

bin/periphery/src/docker/task.rs

@@ -8,14 +8,19 @@ impl DockerClient {
   pub async fn list_swarm_tasks(
     &self,
   ) -> anyhow::Result<Vec<SwarmTaskListItem>> {
-    let tasks = self
+    let mut tasks = self
       .docker
       .list_tasks(Option::<ListTasksOptions>::None)
       .await
       .context("Failed to query for swarm tasks list")?
       .into_iter()
       .map(convert_task_list_item)
-      .collect();
+      .collect::<Vec<_>>();
+
+    tasks.sort_by(|a, b| {
+      a.state.cmp(&b.state).then_with(|| a.name.cmp(&b.name))
+    });
+
     Ok(tasks)
   }
 

bin/periphery/src/docker/volume.rs

@@ -10,7 +10,7 @@ impl DockerClient {
     &self,
     containers: &[ContainerListItem],
   ) -> anyhow::Result<Vec<VolumeListItem>> {
-    let volumes = self
+    let mut volumes = self
       .docker
       .list_volumes(Option::<ListVolumesOptions>::None)
       .await?
@@ -45,7 +45,12 @@ impl DockerClient {
           in_use,
         }
       })
-      .collect();
+      .collect::<Vec<_>>();
+
+    volumes.sort_by(|a, b| {
+      a.in_use.cmp(&b.in_use).then_with(|| a.name.cmp(&b.name))
+    });
+
     Ok(volumes)
   }
 

bin/periphery/src/helpers.rs

@@ -1,6 +1,6 @@
 use std::{
-  net::IpAddr, path::PathBuf, str::FromStr as _, sync::OnceLock,
-  time::Duration,
+  fmt::Write, net::IpAddr, path::PathBuf, str::FromStr as _,
+  sync::OnceLock, time::Duration,
 };
 
 use anyhow::Context;
@@ -14,6 +14,7 @@ use komodo_client::{
   entities::{
     EnvironmentVar, RepoExecutionArgs, RepoExecutionResponse,
     SearchCombinator, SystemCommand, all_logs_success,
+    deployment::Conversion,
   },
   parsers::QUOTE_PATTERN,
 };
@@ -34,6 +35,17 @@ pub fn format_extra_args(extra_args: &[String]) -> String {
   }
 }
 
+pub fn push_extra_args(
+  command: &mut String,
+  extra_args: &[String],
+) -> anyhow::Result<()> {
+  for arg in extra_args {
+    write!(command, " {arg}")
+      .context("Failed to write extra args to command")?
+  }
+  Ok(())
+}
+
 pub fn format_labels(labels: &[EnvironmentVar]) -> String {
   labels
     .iter()
@@ -51,6 +63,56 @@ pub fn format_labels(labels: &[EnvironmentVar]) -> String {
     .join("")
 }
 
+pub fn push_labels(
+  command: &mut String,
+  labels: &[EnvironmentVar],
+) -> anyhow::Result<()> {
+  for label in labels {
+    if label.value.starts_with(QUOTE_PATTERN)
+      && label.value.ends_with(QUOTE_PATTERN)
+    {
+      write!(command, " --label {}={}", label.variable, label.value)
+    } else {
+      write!(
+        command,
+        " --label {}=\"{}\"",
+        label.variable, label.value
+      )
+    }
+    .context("Failed to write labels to command")?;
+  }
+  Ok(())
+}
+
+pub fn push_conversions(
+  command: &mut String,
+  conversions: &[Conversion],
+  flag: &str,
+) -> anyhow::Result<()> {
+  for Conversion { local, container } in conversions {
+    write!(command, " {flag} {local}:{container}")
+      .context("Failed to format conversions")?;
+  }
+  Ok(())
+}
+
+pub fn push_environment(
+  command: &mut String,
+  environment: &[EnvironmentVar],
+) -> anyhow::Result<()> {
+  for EnvironmentVar { variable, value } in environment {
+    if value.starts_with(QUOTE_PATTERN)
+      && value.ends_with(QUOTE_PATTERN)
+    {
+      write!(command, " --env {variable}={value}")
+    } else {
+      write!(command, " --env {variable}=\"{value}\"")
+    }
+    .context("Failed to format environment")?;
+  }
+  Ok(())
+}
+
 pub fn format_log_grep(
   terms: &[String],
   combinator: SearchCombinator,