2.1.2 (#1347)

* 2.1.2 fix multi file service override UI crash

* deploy 2.1.2-dev-1

* 2.1.2

.github/workflows/ci.yml

@@ -22,16 +22,16 @@ jobs:
         with:
           toolchain: stable
 
-      - name: Cache cargo registry
-        uses: actions/cache@v4
-        with:
-          path: |
-            ~/.cargo/registry
-            ~/.cargo/git
-            target
-          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
-          restore-keys: |
-            ${{ runner.os }}-cargo-
+      # - name: Cache cargo registry
+      #   uses: actions/cache@v4
+      #   with:
+      #     path: |
+      #       ~/.cargo/registry
+      #       ~/.cargo/git
+      #       target
+      #     key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
+      #     restore-keys: |
+      #       ${{ runner.os }}-cargo-
 
       - name: Build
         run: cargo build --verbose

Cargo.lock

@@ -324,9 +324,9 @@ dependencies = [
 
 [[package]]
 name = "aws-sdk-ec2"
-version = "1.220.0"
+version = "1.220.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a4d073e665c1303edc348511ec2509b58a2cee148196f72db33aef5cd47c3573"
+checksum = "1065222c6fe7bed0ef49acf2bfdba8ab9b59cc14bc534772d575b365601cd557"
 dependencies = [
  "aws-credential-types",
  "aws-runtime",
@@ -1149,7 +1149,7 @@ dependencies = [
 
 [[package]]
 name = "command"
-version = "2.0.0"
+version = "2.1.2"
 dependencies = [
  "komodo_client",
  "shlex",
@@ -1489,7 +1489,7 @@ checksum = "d7a1e2f27636f116493b8b860f5546edb47c8d8f8ea73e1d2a20be88e28d1fea"
 
 [[package]]
 name = "database"
-version = "2.0.0"
+version = "2.1.2"
 dependencies = [
  "anyhow",
  "async-compression",
@@ -1759,7 +1759,7 @@ dependencies = [
 
 [[package]]
 name = "encoding"
-version = "2.0.0"
+version = "2.1.2"
 dependencies = [
  "anyhow",
  "bytes",
@@ -1801,7 +1801,7 @@ dependencies = [
 
 [[package]]
 name = "environment"
-version = "2.0.0"
+version = "2.1.2"
 dependencies = [
  "anyhow",
  "formatting",
@@ -1930,7 +1930,7 @@ dependencies = [
 
 [[package]]
 name = "formatting"
-version = "2.0.0"
+version = "2.1.2"
 dependencies = [
  "mogh_error",
 ]
@@ -2109,7 +2109,7 @@ dependencies = [
 
 [[package]]
 name = "git"
-version = "2.0.0"
+version = "2.1.2"
 dependencies = [
  "anyhow",
  "command",
@@ -2709,7 +2709,7 @@ dependencies = [
 
 [[package]]
 name = "interpolate"
-version = "2.0.0"
+version = "2.1.2"
 dependencies = [
  "anyhow",
  "komodo_client",
@@ -2835,7 +2835,7 @@ dependencies = [
 
 [[package]]
 name = "komodo_cli"
-version = "2.0.0"
+version = "2.1.2"
 dependencies = [
  "anyhow",
  "bcrypt",
@@ -2865,7 +2865,7 @@ dependencies = [
 
 [[package]]
 name = "komodo_client"
-version = "2.0.0"
+version = "2.1.2"
 dependencies = [
  "anyhow",
  "async_timing_util",
@@ -2904,7 +2904,7 @@ dependencies = [
 
 [[package]]
 name = "komodo_core"
-version = "2.0.0"
+version = "2.1.2"
 dependencies = [
  "anyhow",
  "arc-swap",
@@ -2977,7 +2977,7 @@ dependencies = [
 
 [[package]]
 name = "komodo_periphery"
-version = "2.0.0"
+version = "2.1.2"
 dependencies = [
  "anyhow",
  "arc-swap",
@@ -3271,9 +3271,9 @@ dependencies = [
 
 [[package]]
 name = "mogh_config"
-version = "1.0.4"
+version = "1.0.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7a75ba6d01ac3bb726f95ee40c9d6b37251a1d538cbaf9ea0945744cfcec4842"
+checksum = "b2f42709806fc4822dd9f116f68a43433ca89663e8e1842b6859faced1fc64c1"
 dependencies = [
  "colored",
  "indexmap 2.13.0",
@@ -3301,9 +3301,9 @@ dependencies = [
 
 [[package]]
 name = "mogh_logger"
-version = "1.3.2"
+version = "1.3.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cc71b30d39325fd768c98f0dc60487365f40eb30f3565f0d986fc2d555f95148"
+checksum = "b0aac766e2f34066511e67bf330a88874a9ac9d58732c9c4eb0687546c7a206e"
 dependencies = [
  "anyhow",
  "opentelemetry",
@@ -4032,7 +4032,7 @@ checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220"
 
 [[package]]
 name = "periphery_client"
-version = "2.0.0"
+version = "2.1.2"
 dependencies = [
  "anyhow",
  "encoding",
@@ -5016,9 +5016,9 @@ dependencies = [
 
 [[package]]
 name = "serde_spanned"
-version = "1.1.0"
+version = "1.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "876ac351060d4f882bb1032b6369eb0aef79ad9df1ea8bc404874d8cc3d0cd98"
+checksum = "6662b5879511e06e8999a8a235d848113e942c9124f211511b16466ee2995f26"
 dependencies = [
  "serde_core",
 ]
@@ -5677,9 +5677,9 @@ dependencies = [
 
 [[package]]
 name = "toml"
-version = "1.1.0+spec-1.1.0"
+version = "1.1.2+spec-1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f8195ca05e4eb728f4ba94f3e3291661320af739c4e43779cbdfae82ab239fcc"
+checksum = "81f3d15e84cbcd896376e6730314d59fb5a87f31e4b038454184435cd57defee"
 dependencies = [
  "indexmap 2.13.0",
  "serde_core",
@@ -5692,18 +5692,18 @@ dependencies = [
 
 [[package]]
 name = "toml_datetime"
-version = "1.1.0+spec-1.1.0"
+version = "1.1.1+spec-1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "97251a7c317e03ad83774a8752a7e81fb6067740609f75ea2b585b569a59198f"
+checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7"
 dependencies = [
  "serde_core",
 ]
 
 [[package]]
 name = "toml_parser"
-version = "1.1.0+spec-1.1.0"
+version = "1.1.2+spec-1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2334f11ee363607eb04df9b8fc8a13ca1715a72ba8662a26ac285c98aabb4011"
+checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526"
 dependencies = [
  "winnow",
 ]
@@ -5722,9 +5722,9 @@ dependencies = [
 
 [[package]]
 name = "toml_writer"
-version = "1.1.0+spec-1.1.0"
+version = "1.1.1+spec-1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d282ade6016312faf3e41e57ebbba0c073e4056dab1232ab1cb624199648f8ed"
+checksum = "756daf9b1013ebe47a8776667b466417e2d4c5679d441c26230efd9ef78692db"
 
 [[package]]
 name = "tonic"
@@ -5997,7 +5997,7 @@ dependencies = [
 
 [[package]]
 name = "transport"
-version = "2.0.0"
+version = "2.1.2"
 dependencies = [
  "anyhow",
  "axum",
@@ -6264,9 +6264,9 @@ dependencies = [
 
 [[package]]
 name = "uuid"
-version = "1.22.0"
+version = "1.23.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a68d3c8f01c0cfa54a75291d83601161799e4a89a39e0929f4b0354d88757a37"
+checksum = "5ac8b6f42ead25368cf5b098aeb3dc8a1a2c05a3eee8a9a1a68c640edbfc79d9"
 dependencies = [
  "getrandom 0.4.1",
  "js-sys",

Cargo.toml

@@ -8,7 +8,7 @@ members = [
 ]
 
 [workspace.package]
-version = "2.0.0"
+version = "2.1.2"
 edition = "2024"
 authors = ["mbecker20 <becker.maxh@gmail.com>"]
 license = "GPL-3.0-or-later"
@@ -44,8 +44,8 @@ mogh_rate_limit = "1.0.1"
 partial_derive2 = "0.4.5"
 mongo_indexed = "2.0.2"
 mogh_resolver = "1.0.0"
-mogh_config = "1.0.4"
-mogh_logger = "1.3.2"
+mogh_config = "1.0.5"
+mogh_logger = "1.3.3"
 mogh_server = "1.4.5"
 toml_pretty = "2.0.0"
 mogh_cache = "1.1.1"
@@ -77,7 +77,7 @@ indexmap = { version = "2.13.0", features = ["serde"] }
 serde = { version = "1.0.227", features = ["derive"] }
 strum = { version = "0.28.0", features = ["derive"] }
 bson = { version = "2.15.0" } # must keep in sync with mongodb version
-toml = "1.1.0"
+toml = "1.1.2"
 serde_yaml_ng = "0.10.0"
 serde_json = "1.0.149"
 serde_qs = "1.1.0"
@@ -96,7 +96,7 @@ dotenvy = "0.15.7"
 envy = "0.4.2"
 
 # CRYPTO / AUTH
-uuid = { version = "1.21.0", features = ["v4", "fast-rng", "serde"] }
+uuid = { version = "1.23.0", features = ["v4", "fast-rng", "serde"] }
 rustls = { version = "0.23.37", features = ["aws-lc-rs"] }
 data-encoding = "2.10.0"
 urlencoding = "2.1.3"
@@ -118,7 +118,7 @@ shlex = "1.3.0"
 
 # CLOUD
 aws-config = "1.8.15"
-aws-sdk-ec2 = "1.220.0"
+aws-sdk-ec2 = "1.220.1"
 aws-credential-types = "1.2.14"
 
 ## CRON

bin/binaries.Dockerfile

@@ -1,7 +1,7 @@
 ## Builds the Komodo Core, Periphery, and Util binaries
 ## for a specific architecture. Requires OpenSSL 3 or later.
 
-FROM rust:1.94.0-bookworm AS builder
+FROM rust:1.94.1-bookworm AS builder
 RUN cargo install cargo-strip
 
 WORKDIR /builder

bin/chef.binaries.Dockerfile

@@ -3,7 +3,7 @@
 
 ## Uses chef for dependency caching to help speed up back-to-back builds.
 
-FROM lukemathwalker/cargo-chef:latest-rust-1.94.0-bookworm AS chef
+FROM lukemathwalker/cargo-chef:latest-rust-1.94.1-bookworm AS chef
 WORKDIR /builder
 
 # Plan just the RECIPE to see if things have changed

bin/cli/aio.Dockerfile

@@ -1,4 +1,4 @@
-FROM rust:1.94.0-bullseye AS builder
+FROM rust:1.94.1-bullseye AS builder
 RUN cargo install cargo-strip
 
 WORKDIR /builder

bin/cli/src/command/execute.rs

@@ -28,558 +28,139 @@ pub async fn handle(
   }
 
   println!("\n{}: Execution", "Mode".dimmed());
-  match execution {
-    Execution::None(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::RunAction(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::BatchRunAction(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::RunProcedure(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::BatchRunProcedure(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::RunBuild(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::BatchRunBuild(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::CancelBuild(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::Deploy(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::BatchDeploy(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::PullDeployment(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::StartDeployment(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::RestartDeployment(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::PauseDeployment(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::UnpauseDeployment(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::StopDeployment(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::DestroyDeployment(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::BatchDestroyDeployment(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::CloneRepo(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::BatchCloneRepo(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::PullRepo(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::BatchPullRepo(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::BuildRepo(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::BatchBuildRepo(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::CancelRepoBuild(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::StartContainer(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::RestartContainer(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::PauseContainer(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::UnpauseContainer(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::StopContainer(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::DestroyContainer(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::StartAllContainers(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::RestartAllContainers(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::PauseAllContainers(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::UnpauseAllContainers(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::StopAllContainers(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::PruneContainers(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::DeleteNetwork(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::PruneNetworks(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::DeleteImage(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::PruneImages(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::DeleteVolume(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::PruneVolumes(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::PruneDockerBuilders(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::PruneBuildx(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::PruneSystem(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::RunSync(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::CommitSync(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::DeployStack(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::BatchDeployStack(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::DeployStackIfChanged(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::BatchDeployStackIfChanged(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::PullStack(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::BatchPullStack(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::StartStack(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::RestartStack(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::PauseStack(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::UnpauseStack(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::StopStack(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::DestroyStack(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::BatchDestroyStack(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::RunStackService(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::TestAlerter(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::SendAlert(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::RemoveSwarmNodes(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::RemoveSwarmStacks(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::RemoveSwarmServices(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::CreateSwarmConfig(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::RotateSwarmConfig(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::RemoveSwarmConfigs(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::CreateSwarmSecret(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::RotateSwarmSecret(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::RemoveSwarmSecrets(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::ClearRepoCache(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::BackupCoreDatabase(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::GlobalAutoUpdate(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::RotateAllServerKeys(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::RotateCoreKeys(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::Sleep(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
+
+  macro_rules! handle_execution {
+    (
+      execute: [$($ExecVariant:ident),* $(,)?],
+      batch: [$($BatchVariant:ident),* $(,)?],
+    ) => {{
+      // Print data
+      match execution {
+        $(Execution::$ExecVariant(data) => println!("{}: {data:?}", "Data".dimmed()),)*
+        $(Execution::$BatchVariant(data) => println!("{}: {data:?}", "Data".dimmed()),)*
+        Execution::CommitSync(data) => println!("{}: {data:?}", "Data".dimmed()),
+        Execution::Sleep(data) => println!("{}: {data:?}", "Data".dimmed()),
+        Execution::None(data) => println!("{}: {data:?}", "Data".dimmed()),
+      }
+
+      $crate::command::wait_for_enter("run execution", yes)?;
+
+      info!("Running Execution...");
+
+      let client = $crate::command::komodo_client().await?;
+
+      // Execute and get result
+      match execution.clone() {
+        $(
+          Execution::$ExecVariant(request) => client
+            .execute(request)
+            .await
+            .map(|u| ExecutionResult::Single(u.into())),
+        )*
+        $(
+          Execution::$BatchVariant(request) => {
+            client.execute(request).await.map(ExecutionResult::Batch)
+          }
+        )*
+        Execution::CommitSync(request) => client
+          .write(request)
+          .await
+          .map(|u| ExecutionResult::Single(u.into())),
+        Execution::Sleep(request) => {
+          let duration =
+            Duration::from_millis(request.duration_ms as u64);
+          tokio::time::sleep(duration).await;
+          println!("Finished sleeping!");
+          std::process::exit(0)
+        }
+        Execution::None(_) => unreachable!(),
+      }
+    }};
   }
 
-  super::wait_for_enter("run execution", yes)?;
-
-  info!("Running Execution...");
-
-  let client = super::komodo_client().await?;
-
-  let res = match execution.clone() {
-    Execution::RunAction(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::BatchRunAction(request) => {
-      client.execute(request).await.map(ExecutionResult::Batch)
-    }
-    Execution::RunProcedure(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::BatchRunProcedure(request) => {
-      client.execute(request).await.map(ExecutionResult::Batch)
-    }
-    Execution::RunBuild(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::BatchRunBuild(request) => {
-      client.execute(request).await.map(ExecutionResult::Batch)
-    }
-    Execution::CancelBuild(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::Deploy(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::BatchDeploy(request) => {
-      client.execute(request).await.map(ExecutionResult::Batch)
-    }
-    Execution::PullDeployment(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::StartDeployment(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::RestartDeployment(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::PauseDeployment(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::UnpauseDeployment(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::StopDeployment(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::DestroyDeployment(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::BatchDestroyDeployment(request) => {
-      client.execute(request).await.map(ExecutionResult::Batch)
-    }
-    Execution::CloneRepo(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::BatchCloneRepo(request) => {
-      client.execute(request).await.map(ExecutionResult::Batch)
-    }
-    Execution::PullRepo(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::BatchPullRepo(request) => {
-      client.execute(request).await.map(ExecutionResult::Batch)
-    }
-    Execution::BuildRepo(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::BatchBuildRepo(request) => {
-      client.execute(request).await.map(ExecutionResult::Batch)
-    }
-    Execution::CancelRepoBuild(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::StartContainer(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::RestartContainer(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::PauseContainer(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::UnpauseContainer(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::StopContainer(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::DestroyContainer(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::StartAllContainers(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::RestartAllContainers(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::PauseAllContainers(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::UnpauseAllContainers(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::StopAllContainers(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::PruneContainers(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::DeleteNetwork(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::PruneNetworks(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::DeleteImage(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::PruneImages(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::DeleteVolume(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::PruneVolumes(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::PruneDockerBuilders(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::PruneBuildx(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::PruneSystem(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::RunSync(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::CommitSync(request) => client
-      .write(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::DeployStack(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::BatchDeployStack(request) => {
-      client.execute(request).await.map(ExecutionResult::Batch)
-    }
-    Execution::DeployStackIfChanged(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::BatchDeployStackIfChanged(request) => {
-      client.execute(request).await.map(ExecutionResult::Batch)
-    }
-    Execution::PullStack(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::BatchPullStack(request) => {
-      client.execute(request).await.map(ExecutionResult::Batch)
-    }
-    Execution::StartStack(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::RestartStack(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::PauseStack(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::UnpauseStack(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::StopStack(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::DestroyStack(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::BatchDestroyStack(request) => {
-      client.execute(request).await.map(ExecutionResult::Batch)
-    }
-    Execution::RunStackService(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::TestAlerter(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::SendAlert(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::RemoveSwarmNodes(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::RemoveSwarmStacks(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::RemoveSwarmServices(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::CreateSwarmConfig(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::RotateSwarmConfig(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::RemoveSwarmConfigs(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::CreateSwarmSecret(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::RotateSwarmSecret(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::RemoveSwarmSecrets(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::ClearRepoCache(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::BackupCoreDatabase(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::GlobalAutoUpdate(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::RotateAllServerKeys(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::RotateCoreKeys(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::Sleep(request) => {
-      let duration =
-        Duration::from_millis(request.duration_ms as u64);
-      tokio::time::sleep(duration).await;
-      println!("Finished sleeping!");
-      std::process::exit(0)
-    }
-    Execution::None(_) => unreachable!(),
-  };
+  let res = handle_execution!(
+    execute: [
+      RunAction,
+      RunProcedure,
+      RunBuild,
+      CancelBuild,
+      Deploy,
+      PullDeployment,
+      StartDeployment,
+      RestartDeployment,
+      PauseDeployment,
+      UnpauseDeployment,
+      StopDeployment,
+      DestroyDeployment,
+      CloneRepo,
+      PullRepo,
+      BuildRepo,
+      CancelRepoBuild,
+      StartContainer,
+      RestartContainer,
+      PauseContainer,
+      UnpauseContainer,
+      StopContainer,
+      DestroyContainer,
+      StartAllContainers,
+      RestartAllContainers,
+      PauseAllContainers,
+      UnpauseAllContainers,
+      StopAllContainers,
+      PruneContainers,
+      DeleteNetwork,
+      PruneNetworks,
+      DeleteImage,
+      PruneImages,
+      DeleteVolume,
+      PruneVolumes,
+      PruneDockerBuilders,
+      PruneBuildx,
+      PruneSystem,
+      RunSync,
+      DeployStack,
+      DeployStackIfChanged,
+      PullStack,
+      StartStack,
+      RestartStack,
+      PauseStack,
+      UnpauseStack,
+      StopStack,
+      DestroyStack,
+      RunStackService,
+      TestAlerter,
+      SendAlert,
+      RemoveSwarmNodes,
+      UpdateSwarmNode,
+      RemoveSwarmStacks,
+      RemoveSwarmServices,
+      CreateSwarmConfig,
+      RotateSwarmConfig,
+      RemoveSwarmConfigs,
+      CreateSwarmSecret,
+      RotateSwarmSecret,
+      RemoveSwarmSecrets,
+      ClearRepoCache,
+      BackupCoreDatabase,
+      GlobalAutoUpdate,
+      RotateAllServerKeys,
+      RotateCoreKeys,
+    ],
+    batch: [
+      BatchRunAction,
+      BatchRunProcedure,
+      BatchRunBuild,
+      BatchDeploy,
+      BatchDestroyDeployment,
+      BatchCloneRepo,
+      BatchPullRepo,
+      BatchBuildRepo,
+      BatchDeployStack,
+      BatchDeployStackIfChanged,
+      BatchPullStack,
+      BatchDestroyStack,
+    ],
+  );
 
   match res {
     Ok(ExecutionResult::Single(update)) => {

bin/cli/src/command/terminal.rs

@@ -2,7 +2,7 @@ use anyhow::{Context, anyhow};
 use colored::Colorize;
 use komodo_client::{
   api::{
-    read::{ListAllDockerContainers, ListServers},
+    read::{GetServer, ListAllDockerContainers, ListServers},
     terminal::InitTerminal,
   },
   entities::{
@@ -138,11 +138,14 @@ async fn get_server(
   }
 
   if containers.len() == 1 {
-    return containers
+    let server_id = containers
       .pop()
       .context("Shouldn't happen")?
       .server_id
-      .context("Container doesn't have server_id");
+      .context("Container doesn't have server_id")?;
+    let server_name =
+      client.read(GetServer { server: server_id }).await?.name;
+    return Ok(server_name);
   }
 
   let servers = containers

bin/cli/src/config.rs

@@ -264,6 +264,9 @@ pub fn cli_config() -> &'static CliConfig {
         ansi: env
           .komodo_cli_logging_ansi
           .unwrap_or(config.cli_logging.ansi),
+        timestamps: env
+          .komodo_cli_logging_timestamps
+          .unwrap_or(config.cli_logging.timestamps),
         otlp_endpoint: env
           .komodo_cli_logging_otlp_endpoint
           .unwrap_or(config.cli_logging.otlp_endpoint),

bin/core/aio.Dockerfile

@@ -1,7 +1,7 @@
 ## All in one, multi stage compile + runtime Docker build for your architecture.
 
 # Build Core
-FROM rust:1.94.0-trixie AS core-builder
+FROM rust:1.94.1-trixie AS core-builder
 RUN cargo install cargo-strip
 
 WORKDIR /builder

bin/core/src/alert/discord.rs

@@ -27,7 +27,7 @@ pub async fn send_alert(
         SeverityLevel::Critical => {
           let err = err
             .as_ref()
-            .map(|e| format!("\n**error**: {e}"))
+            .map(|e| format!("\n**error**: {e:#?}"))
             .unwrap_or_default();
           format!(
             "{level} | Swarm **{name}** is **unhealthy** ❌\n{link}{err}"
@@ -36,28 +36,6 @@ pub async fn send_alert(
         _ => unreachable!(),
       }
     }
-    AlertData::ServerVersionMismatch {
-      id,
-      name,
-      region,
-      server_version,
-      core_version,
-    } => {
-      let region = fmt_region(region);
-      let link = resource_link(ResourceTargetVariant::Server, id);
-      match alert.level {
-        SeverityLevel::Ok => {
-          format!(
-            "{level} | **{name}**{region} | Periphery version now matches Core version ✅\n{link}"
-          )
-        }
-        _ => {
-          format!(
-            "{level} | **{name}**{region} | Version mismatch detected ⚠️\nPeriphery: **{server_version}** | Core: **{core_version}**\n{link}"
-          )
-        }
-      }
-    }
     AlertData::ServerUnreachable {
       id,
       name,
@@ -84,6 +62,28 @@ pub async fn send_alert(
         _ => unreachable!(),
       }
     }
+    AlertData::ServerVersionMismatch {
+      id,
+      name,
+      region,
+      server_version,
+      core_version,
+    } => {
+      let region = fmt_region(region);
+      let link = resource_link(ResourceTargetVariant::Server, id);
+      match alert.level {
+        SeverityLevel::Ok => {
+          format!(
+            "{level} | **{name}**{region} | Periphery version now matches Core version ✅\n{link}"
+          )
+        }
+        _ => {
+          format!(
+            "{level} | **{name}**{region} | Version mismatch detected ⚠️\nPeriphery: **{server_version}** | Core: **{core_version}**\n{link}"
+          )
+        }
+      }
+    }
     AlertData::ServerCpu {
       id,
       name,

bin/core/src/alert/mod.rs

@@ -260,7 +260,7 @@ fn standard_alert_content(alert: &Alert) -> String {
         SeverityLevel::Critical => {
           let err = err
             .as_ref()
-            .map(|e| format!("\nerror: {e}"))
+            .map(|e| format!("\nerror: {e:#?}"))
             .unwrap_or_default();
           format!(
             "{level} | Swarm {name} is unhealthy ❌\n{link}{err}"
@@ -269,6 +269,30 @@ fn standard_alert_content(alert: &Alert) -> String {
         _ => unreachable!(),
       }
     }
+    AlertData::ServerUnreachable {
+      id,
+      name,
+      region,
+      err,
+    } => {
+      let region = fmt_region(region);
+      let link = resource_link(ResourceTargetVariant::Server, id);
+      match alert.level {
+        SeverityLevel::Ok => {
+          format!("{level} | {name}{region} is now connected\n{link}")
+        }
+        SeverityLevel::Critical => {
+          let err = err
+            .as_ref()
+            .map(|e| format!("\nerror: {e:#?}"))
+            .unwrap_or_default();
+          format!(
+            "{level} | {name}{region} is unreachable ❌\n{link}{err}"
+          )
+        }
+        _ => unreachable!(),
+      }
+    }
     AlertData::ServerVersionMismatch {
       id,
       name,
@@ -291,30 +315,6 @@ fn standard_alert_content(alert: &Alert) -> String {
         }
       }
     }
-    AlertData::ServerUnreachable {
-      id,
-      name,
-      region,
-      err,
-    } => {
-      let region = fmt_region(region);
-      let link = resource_link(ResourceTargetVariant::Server, id);
-      match alert.level {
-        SeverityLevel::Ok => {
-          format!("{level} | {name}{region} is now connected\n{link}")
-        }
-        SeverityLevel::Critical => {
-          let err = err
-            .as_ref()
-            .map(|e| format!("\nerror: {e:#?}"))
-            .unwrap_or_default();
-          format!(
-            "{level} | {name}{region} is unreachable ❌\n{link}{err}"
-          )
-        }
-        _ => unreachable!(),
-      }
-    }
     AlertData::ServerCpu {
       id,
       name,

bin/core/src/alert/slack.rs

@@ -42,7 +42,7 @@ pub async fn send_alert(
             format!("{level} | Swarm *{name}* is *unhealthy* ❌");
           let err = err
             .as_ref()
-            .map(|e| format!("\nerror: {e}"))
+            .map(|e| format!("\nerror: {e:#?}"))
             .unwrap_or_default();
           let blocks = vec![
             Block::header(level),
@@ -59,35 +59,6 @@ pub async fn send_alert(
         _ => unreachable!(),
       }
     }
-    AlertData::ServerVersionMismatch {
-      id,
-      name,
-      region,
-      server_version,
-      core_version,
-    } => {
-      let region = fmt_region(region);
-      let text = match alert.level {
-        SeverityLevel::Ok => {
-          format!(
-            "{level} | *{name}*{region} | Periphery version now matches Core version ✅"
-          )
-        }
-        _ => {
-          format!(
-            "{level} | *{name}*{region} | Version mismatch detected ⚠️\nPeriphery: {server_version} | Core: {core_version}"
-          )
-        }
-      };
-      let blocks = vec![
-        Block::header(text.clone()),
-        Block::section(resource_link(
-          ResourceTargetVariant::Server,
-          id,
-        )),
-      ];
-      (text, blocks.into())
-    }
     AlertData::ServerUnreachable {
       id,
       name,
@@ -129,6 +100,35 @@ pub async fn send_alert(
         _ => unreachable!(),
       }
     }
+    AlertData::ServerVersionMismatch {
+      id,
+      name,
+      region,
+      server_version,
+      core_version,
+    } => {
+      let region = fmt_region(region);
+      let text = match alert.level {
+        SeverityLevel::Ok => {
+          format!(
+            "{level} | *{name}*{region} | Periphery version now matches Core version ✅"
+          )
+        }
+        _ => {
+          format!(
+            "{level} | *{name}*{region} | Version mismatch detected ⚠️\nPeriphery: {server_version} | Core: {core_version}"
+          )
+        }
+      };
+      let blocks = vec![
+        Block::header(text.clone()),
+        Block::section(resource_link(
+          ResourceTargetVariant::Server,
+          id,
+        )),
+      ];
+      (text, blocks.into())
+    }
     AlertData::ServerCpu {
       id,
       name,

bin/core/src/api/execute/maintenance.rs

@@ -15,7 +15,7 @@ use komodo_client::{
   },
   entities::{
     SwarmOrServer, deployment::DeploymentState, server::ServerState,
-    stack::StackState,
+    stack::StackState, swarm::SwarmState,
   },
 };
 use mogh_error::AddStatusCodeError;
@@ -39,7 +39,7 @@ use crate::{
   resource::rotate_server_keys,
   state::{
     db_client, deployment_status_cache, server_status_cache,
-    stack_status_cache,
+    stack_status_cache, swarm_status_cache,
   },
 };
 
@@ -244,6 +244,7 @@ impl Resolve<ExecuteArgs> for GlobalAutoUpdate {
         .context("Failed to query for stacks from database")?;
 
     let server_status_cache = server_status_cache();
+    let swarm_status_cache = swarm_status_cache();
     let stack_status_cache = stack_status_cache();
 
     // Will be edited later at update.logs[0]
@@ -255,7 +256,7 @@ impl Resolve<ExecuteArgs> for GlobalAutoUpdate {
         continue;
       };
 
-      // Only pull running stacks.
+      // Only check running stacks.
       if !matches!(status.curr.state, StackState::Running) {
         continue;
       }
@@ -267,43 +268,58 @@ impl Resolve<ExecuteArgs> for GlobalAutoUpdate {
         &servers,
       )?;
 
-      if let SwarmOrServer::None = &swarm_or_server {
-        continue;
+      // Ensure server / swarm is reachable
+      match &swarm_or_server {
+        SwarmOrServer::None => continue,
+        SwarmOrServer::Server(server) => {
+          if !server_status_cache
+            .get(&server.id)
+            .await
+            .map(|s| matches!(s.state, ServerState::Ok))
+            .unwrap_or_default()
+          {
+            continue;
+          }
+        }
+        SwarmOrServer::Swarm(swarm) => {
+          if !swarm_status_cache
+            .get(&swarm.id)
+            .await
+            .map(|s| {
+              matches!(
+                s.state,
+                SwarmState::Healthy | SwarmState::Unhealthy
+              )
+            })
+            .unwrap_or_default()
+          {
+            continue;
+          }
+        }
       }
 
-      if let Some(server) =
-        servers.iter().find(|s| s.id == stack.config.server_id)
-        // This check is probably redundant along with running check
-        // but shouldn't hurt
-        && server_status_cache
-          .get(&server.id)
-          .await
-          .map(|s| matches!(s.state, ServerState::Ok))
-          .unwrap_or_default()
+      if let Err(e) = check_stack_for_update_inner(
+        stack.id,
+        &swarm_or_server,
+        self.skip_auto_update,
+        true,
+        false,
+      )
+      .await
       {
-        if let Err(e) = check_stack_for_update_inner(
-          stack.id,
-          &swarm_or_server,
-          self.skip_auto_update,
-          true,
-          false,
-        )
-        .await
-        {
-          update.push_error_log(
-            &format!("Check Stack {}", stack.name),
-            format_serror(&e.into()),
-          );
-        } else {
-          if !update.logs[0].stdout.is_empty() {
-            update.logs[0].stdout.push('\n');
-          }
-
-          update.logs[0].stdout.push_str(&format!(
-            "Checked Stack {} ✅",
-            bold(&stack.name)
-          ));
+        update.push_error_log(
+          &format!("Check Stack {}", stack.name),
+          format_serror(&e.into()),
+        );
+      } else {
+        if !update.logs[0].stdout.is_empty() {
+          update.logs[0].stdout.push('\n');
         }
+
+        update.logs[0].stdout.push_str(&format!(
+          "Checked Stack {} ✅",
+          bold(&stack.name)
+        ));
       }
     }
 
@@ -320,7 +336,7 @@ impl Resolve<ExecuteArgs> for GlobalAutoUpdate {
         continue;
       };
 
-      // Only pull running deployments.
+      // Only check running deployments.
       if !matches!(status.curr.state, DeploymentState::Running) {
         continue;
       }
@@ -332,8 +348,34 @@ impl Resolve<ExecuteArgs> for GlobalAutoUpdate {
         &servers,
       )?;
 
-      if let SwarmOrServer::None = &swarm_or_server {
-        continue;
+      // Ensure server / swarm is reachable
+      match &swarm_or_server {
+        SwarmOrServer::None => continue,
+        SwarmOrServer::Server(server) => {
+          if !server_status_cache
+            .get(&server.id)
+            .await
+            .map(|s| matches!(s.state, ServerState::Ok))
+            .unwrap_or_default()
+          {
+            continue;
+          }
+        }
+        SwarmOrServer::Swarm(swarm) => {
+          if !swarm_status_cache
+            .get(&swarm.id)
+            .await
+            .map(|s| {
+              matches!(
+                s.state,
+                SwarmState::Healthy | SwarmState::Unhealthy
+              )
+            })
+            .unwrap_or_default()
+          {
+            continue;
+          }
+        }
       }
 
       let name = deployment.name.clone();

bin/core/src/api/execute/mod.rs

@@ -148,6 +148,7 @@ pub enum ExecuteRequest {
 
   // ==== SWARM ====
   RemoveSwarmNodes(RemoveSwarmNodes),
+  UpdateSwarmNode(UpdateSwarmNode),
   RemoveSwarmStacks(RemoveSwarmStacks),
   RemoveSwarmServices(RemoveSwarmServices),
   CreateSwarmConfig(CreateSwarmConfig),

bin/core/src/api/execute/swarm.rs

@@ -1,10 +1,6 @@
 use formatting::format_serror;
 use komodo_client::{
-  api::execute::{
-    CreateSwarmConfig, CreateSwarmSecret, RemoveSwarmConfigs,
-    RemoveSwarmNodes, RemoveSwarmSecrets, RemoveSwarmServices,
-    RemoveSwarmStacks, RotateSwarmConfig, RotateSwarmSecret,
-  },
+  api::execute::*,
   entities::{permission::PermissionLevel, swarm::Swarm},
 };
 use mogh_resolver::Resolve;
@@ -76,6 +72,72 @@ impl Resolve<ExecuteArgs> for RemoveSwarmNodes {
   }
 }
 
+impl Resolve<ExecuteArgs> for UpdateSwarmNode {
+  #[instrument(
+    "UpdateSwarmNode",
+    skip_all,
+    fields(
+      task_id = task_id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      swarm = self.swarm,
+      node = self.node,
+      availability = format!("{:?}", self.availability),
+      label_add = format!("{:?}", self.label_add),
+      label_rm = format!("{:?}", self.label_rm),
+      role = format!("{:?}", self.role),
+    )
+  )]
+  async fn resolve(
+    self,
+    ExecuteArgs {
+      user,
+      update,
+      task_id,
+    }: &ExecuteArgs,
+  ) -> Result<Self::Response, Self::Error> {
+    let swarm = get_check_permissions::<Swarm>(
+      &self.swarm,
+      user,
+      PermissionLevel::Execute.into(),
+    )
+    .await?;
+
+    update_update(update.clone()).await?;
+
+    let mut update = update.clone();
+
+    match swarm_request(
+      &swarm.config.server_ids,
+      periphery_client::api::swarm::UpdateSwarmNode {
+        node: self.node,
+        availability: self.availability,
+        label_add: self.label_add,
+        label_rm: self.label_rm,
+        role: self.role,
+      },
+    )
+    .await
+    {
+      Ok(log) => {
+        update.logs.push(log);
+        refresh_swarm_cache(&swarm, true).await;
+      }
+      Err(e) => update.push_error_log(
+        "Update Swarm Node",
+        format_serror(
+          &e.context("Failed to update swarm node").into(),
+        ),
+      ),
+    };
+
+    update.finalize();
+    update_update(update.clone()).await?;
+
+    Ok(update)
+  }
+}
+
 impl Resolve<ExecuteArgs> for RemoveSwarmStacks {
   #[instrument(
     "RemoveSwarmStacks",

bin/core/src/api/read/toml.rs

@@ -1,5 +1,8 @@
 use anyhow::Context;
-use database::mungos::find::find_collect;
+use database::{
+  bson::doc,
+  mungos::{find::find_collect, mongodb::options::FindOptions},
+};
 use komodo_client::{
   api::read::{
     ExportAllResourcesToToml, ExportAllResourcesToTomlResponse,
@@ -215,18 +218,21 @@ impl Resolve<ReadArgs> for ExportResourcesToToml {
       .context("failed to add user groups")?;
 
     if include_variables {
-      res.variables =
-        find_collect(&db_client().variables, None, None)
-          .await
-          .context("failed to get variables from db")?
-          .into_iter()
-          .map(|mut variable| {
-            if !user.admin && variable.is_secret {
-              variable.value = "#".repeat(variable.value.len())
-            }
-            variable
-          })
-          .collect();
+      res.variables = find_collect(
+        &db_client().variables,
+        None,
+        FindOptions::builder().sort(doc! { "name": 1 }).build(),
+      )
+      .await
+      .context("failed to get variables from db")?
+      .into_iter()
+      .map(|mut variable| {
+        if !user.admin && variable.is_secret {
+          variable.value = "#".repeat(variable.value.len())
+        }
+        variable
+      })
+      .collect();
     }
 
     let toml = serialize_resources_toml(res)

bin/core/src/api/write/stack.rs

@@ -555,8 +555,9 @@ impl Resolve<WriteArgs> for RefreshStackCache {
             &mut services,
           ) {
             warn!(
-              "failed to extract stack services, things won't works correctly. stack: {} | {e:#}",
-              stack.name
+              stack = stack.id,
+              stack_name = stack.name,
+              "Failed to extract stack services | {e:#}",
             );
           }
         }
@@ -939,7 +940,10 @@ pub async fn check_stack_for_update_inner(
     .retain(|(stack_id, _)| stack_id != &stack.id)
     .await;
 
-  let deploy_services = if stack.config.auto_update_all_services {
+  let deploy_services = if stack.config.auto_update_all_services
+    // Swarm stacks don't support individual service deploy
+    || !stack.config.swarm_id.is_empty()
+  {
     Vec::new()
   } else {
     services_with_update

bin/core/src/config.rs

@@ -375,6 +375,9 @@ pub fn core_config() -> &'static CoreConfig {
           .komodo_logging_location
           .unwrap_or(config.logging.location),
         ansi: env.komodo_logging_ansi.unwrap_or(config.logging.ansi),
+        timestamps: env
+          .komodo_logging_timestamps
+          .unwrap_or(config.logging.timestamps),
         otlp_endpoint: env
           .komodo_logging_otlp_endpoint
           .unwrap_or(config.logging.otlp_endpoint),

bin/core/src/connection/server.rs

@@ -307,6 +307,13 @@ async fn fix_server(
     } else {
       None
     },
+    // Move address to external_address if not set.
+    // This helps preserve container port link behavior.
+    external_address: if !server.config.external_address.is_empty() {
+      Some(server.config.address)
+    } else {
+      None
+    },
     ..Default::default()
   };
   if !config.is_none() {

bin/core/src/helpers/update.rs

@@ -123,464 +123,151 @@ pub async fn init_execution_update(
   request: &ExecuteRequest,
   user: &User,
 ) -> anyhow::Result<Update> {
-  let (operation, target) = match &request {
-    // Swarm
-    ExecuteRequest::RemoveSwarmNodes(data) => (
-      Operation::RemoveSwarmNodes,
-      ResourceTarget::Swarm(
-        resource::get::<Swarm>(&data.swarm).await?.id,
-      ),
-    ),
-    ExecuteRequest::RemoveSwarmStacks(data) => (
-      Operation::RemoveSwarmStacks,
-      ResourceTarget::Swarm(
-        resource::get::<Swarm>(&data.swarm).await?.id,
-      ),
-    ),
-    ExecuteRequest::RemoveSwarmServices(data) => (
-      Operation::RemoveSwarmServices,
-      ResourceTarget::Swarm(
-        resource::get::<Swarm>(&data.swarm).await?.id,
-      ),
-    ),
-    ExecuteRequest::CreateSwarmConfig(data) => (
-      Operation::CreateSwarmConfig,
-      ResourceTarget::Swarm(
-        resource::get::<Swarm>(&data.swarm).await?.id,
-      ),
-    ),
-    ExecuteRequest::RotateSwarmConfig(data) => (
-      Operation::RotateSwarmConfig,
-      ResourceTarget::Swarm(
-        resource::get::<Swarm>(&data.swarm).await?.id,
-      ),
-    ),
-    ExecuteRequest::RemoveSwarmConfigs(data) => (
-      Operation::RemoveSwarmConfigs,
-      ResourceTarget::Swarm(
-        resource::get::<Swarm>(&data.swarm).await?.id,
-      ),
-    ),
-    ExecuteRequest::CreateSwarmSecret(data) => (
-      Operation::CreateSwarmSecret,
-      ResourceTarget::Swarm(
-        resource::get::<Swarm>(&data.swarm).await?.id,
-      ),
-    ),
-    ExecuteRequest::RotateSwarmSecret(data) => (
-      Operation::RotateSwarmSecret,
-      ResourceTarget::Swarm(
-        resource::get::<Swarm>(&data.swarm).await?.id,
-      ),
-    ),
-    ExecuteRequest::RemoveSwarmSecrets(data) => (
-      Operation::RemoveSwarmSecrets,
-      ResourceTarget::Swarm(
-        resource::get::<Swarm>(&data.swarm).await?.id,
-      ),
-    ),
+  macro_rules! init_execution_match {
+    (
+      resource: [$(($Variant:ident, $ResType:ident, $field:ident)),* $(,)?],
+      batch: [$($BatchVariant:ident),* $(,)?],
+      stack_service: [$(($StackVariant:ident, $ServiceOp:ident)),* $(,)?],
+      system: [$($SysVariant:ident),* $(,)?],
+    ) => {
+      match &request {
+        $(
+          ExecuteRequest::$Variant(data) => (
+            Operation::$Variant,
+            ResourceTarget::$ResType(
+              resource::get::<$ResType>(&data.$field).await?.id,
+            ),
+          ),
+        )*
+        $(
+          ExecuteRequest::$BatchVariant(_data) => {
+            return Ok(Default::default());
+          }
+        )*
+        $(
+          ExecuteRequest::$StackVariant(data) => (
+            if !data.services.is_empty() {
+              Operation::$ServiceOp
+            } else {
+              Operation::$StackVariant
+            },
+            ResourceTarget::Stack(
+              resource::get::<Stack>(&data.stack).await?.id,
+            ),
+          ),
+        )*
+        // DeployStackIfChanged doesn't have a service variant
+        ExecuteRequest::DeployStackIfChanged(data) => (
+          Operation::DeployStack,
+          ResourceTarget::Stack(
+            resource::get::<Stack>(&data.stack).await?.id,
+          ),
+        ),
+        $(
+          ExecuteRequest::$SysVariant(_data) => {
+            (Operation::$SysVariant, ResourceTarget::system())
+          }
+        )*
+      }
+    };
+  }
 
-    // Server
-    ExecuteRequest::StartContainer(data) => (
-      Operation::StartContainer,
-      ResourceTarget::Server(
-        resource::get::<Server>(&data.server).await?.id,
-      ),
-    ),
-    ExecuteRequest::RestartContainer(data) => (
-      Operation::RestartContainer,
-      ResourceTarget::Server(
-        resource::get::<Server>(&data.server).await?.id,
-      ),
-    ),
-    ExecuteRequest::PauseContainer(data) => (
-      Operation::PauseContainer,
-      ResourceTarget::Server(
-        resource::get::<Server>(&data.server).await?.id,
-      ),
-    ),
-    ExecuteRequest::UnpauseContainer(data) => (
-      Operation::UnpauseContainer,
-      ResourceTarget::Server(
-        resource::get::<Server>(&data.server).await?.id,
-      ),
-    ),
-    ExecuteRequest::StopContainer(data) => (
-      Operation::StopContainer,
-      ResourceTarget::Server(
-        resource::get::<Server>(&data.server).await?.id,
-      ),
-    ),
-    ExecuteRequest::DestroyContainer(data) => (
-      Operation::DestroyContainer,
-      ResourceTarget::Server(
-        resource::get::<Server>(&data.server).await?.id,
-      ),
-    ),
-    ExecuteRequest::StartAllContainers(data) => (
-      Operation::StartAllContainers,
-      ResourceTarget::Server(
-        resource::get::<Server>(&data.server).await?.id,
-      ),
-    ),
-    ExecuteRequest::RestartAllContainers(data) => (
-      Operation::RestartAllContainers,
-      ResourceTarget::Server(
-        resource::get::<Server>(&data.server).await?.id,
-      ),
-    ),
-    ExecuteRequest::PauseAllContainers(data) => (
-      Operation::PauseAllContainers,
-      ResourceTarget::Server(
-        resource::get::<Server>(&data.server).await?.id,
-      ),
-    ),
-    ExecuteRequest::UnpauseAllContainers(data) => (
-      Operation::UnpauseAllContainers,
-      ResourceTarget::Server(
-        resource::get::<Server>(&data.server).await?.id,
-      ),
-    ),
-    ExecuteRequest::StopAllContainers(data) => (
-      Operation::StopAllContainers,
-      ResourceTarget::Server(
-        resource::get::<Server>(&data.server).await?.id,
-      ),
-    ),
-    ExecuteRequest::PruneContainers(data) => (
-      Operation::PruneContainers,
-      ResourceTarget::Server(
-        resource::get::<Server>(&data.server).await?.id,
-      ),
-    ),
-    ExecuteRequest::DeleteNetwork(data) => (
-      Operation::DeleteNetwork,
-      ResourceTarget::Server(
-        resource::get::<Server>(&data.server).await?.id,
-      ),
-    ),
-    ExecuteRequest::PruneNetworks(data) => (
-      Operation::PruneNetworks,
-      ResourceTarget::Server(
-        resource::get::<Server>(&data.server).await?.id,
-      ),
-    ),
-    ExecuteRequest::DeleteImage(data) => (
-      Operation::DeleteImage,
-      ResourceTarget::Server(
-        resource::get::<Server>(&data.server).await?.id,
-      ),
-    ),
-    ExecuteRequest::PruneImages(data) => (
-      Operation::PruneImages,
-      ResourceTarget::Server(
-        resource::get::<Server>(&data.server).await?.id,
-      ),
-    ),
-    ExecuteRequest::DeleteVolume(data) => (
-      Operation::DeleteVolume,
-      ResourceTarget::Server(
-        resource::get::<Server>(&data.server).await?.id,
-      ),
-    ),
-    ExecuteRequest::PruneVolumes(data) => (
-      Operation::PruneVolumes,
-      ResourceTarget::Server(
-        resource::get::<Server>(&data.server).await?.id,
-      ),
-    ),
-    ExecuteRequest::PruneDockerBuilders(data) => (
-      Operation::PruneDockerBuilders,
-      ResourceTarget::Server(
-        resource::get::<Server>(&data.server).await?.id,
-      ),
-    ),
-    ExecuteRequest::PruneBuildx(data) => (
-      Operation::PruneBuildx,
-      ResourceTarget::Server(
-        resource::get::<Server>(&data.server).await?.id,
-      ),
-    ),
-    ExecuteRequest::PruneSystem(data) => (
-      Operation::PruneSystem,
-      ResourceTarget::Server(
-        resource::get::<Server>(&data.server).await?.id,
-      ),
-    ),
-
-    // Deployment
-    ExecuteRequest::Deploy(data) => (
-      Operation::Deploy,
-      ResourceTarget::Deployment(
-        resource::get::<Deployment>(&data.deployment).await?.id,
-      ),
-    ),
-    ExecuteRequest::BatchDeploy(_data) => {
-      return Ok(Default::default());
-    }
-    ExecuteRequest::PullDeployment(data) => (
-      Operation::PullDeployment,
-      ResourceTarget::Deployment(
-        resource::get::<Deployment>(&data.deployment).await?.id,
-      ),
-    ),
-    ExecuteRequest::StartDeployment(data) => (
-      Operation::StartDeployment,
-      ResourceTarget::Deployment(
-        resource::get::<Deployment>(&data.deployment).await?.id,
-      ),
-    ),
-    ExecuteRequest::RestartDeployment(data) => (
-      Operation::RestartDeployment,
-      ResourceTarget::Deployment(
-        resource::get::<Deployment>(&data.deployment).await?.id,
-      ),
-    ),
-    ExecuteRequest::PauseDeployment(data) => (
-      Operation::PauseDeployment,
-      ResourceTarget::Deployment(
-        resource::get::<Deployment>(&data.deployment).await?.id,
-      ),
-    ),
-    ExecuteRequest::UnpauseDeployment(data) => (
-      Operation::UnpauseDeployment,
-      ResourceTarget::Deployment(
-        resource::get::<Deployment>(&data.deployment).await?.id,
-      ),
-    ),
-    ExecuteRequest::StopDeployment(data) => (
-      Operation::StopDeployment,
-      ResourceTarget::Deployment(
-        resource::get::<Deployment>(&data.deployment).await?.id,
-      ),
-    ),
-    ExecuteRequest::DestroyDeployment(data) => (
-      Operation::DestroyDeployment,
-      ResourceTarget::Deployment(
-        resource::get::<Deployment>(&data.deployment).await?.id,
-      ),
-    ),
-    ExecuteRequest::BatchDestroyDeployment(_data) => {
-      return Ok(Default::default());
-    }
-
-    // Build
-    ExecuteRequest::RunBuild(data) => (
-      Operation::RunBuild,
-      ResourceTarget::Build(
-        resource::get::<Build>(&data.build).await?.id,
-      ),
-    ),
-    ExecuteRequest::BatchRunBuild(_data) => {
-      return Ok(Default::default());
-    }
-    ExecuteRequest::CancelBuild(data) => (
-      Operation::CancelBuild,
-      ResourceTarget::Build(
-        resource::get::<Build>(&data.build).await?.id,
-      ),
-    ),
-
-    // Repo
-    ExecuteRequest::CloneRepo(data) => (
-      Operation::CloneRepo,
-      ResourceTarget::Repo(
-        resource::get::<Repo>(&data.repo).await?.id,
-      ),
-    ),
-    ExecuteRequest::BatchCloneRepo(_data) => {
-      return Ok(Default::default());
-    }
-    ExecuteRequest::PullRepo(data) => (
-      Operation::PullRepo,
-      ResourceTarget::Repo(
-        resource::get::<Repo>(&data.repo).await?.id,
-      ),
-    ),
-    ExecuteRequest::BatchPullRepo(_data) => {
-      return Ok(Default::default());
-    }
-    ExecuteRequest::BuildRepo(data) => (
-      Operation::BuildRepo,
-      ResourceTarget::Repo(
-        resource::get::<Repo>(&data.repo).await?.id,
-      ),
-    ),
-    ExecuteRequest::BatchBuildRepo(_data) => {
-      return Ok(Default::default());
-    }
-    ExecuteRequest::CancelRepoBuild(data) => (
-      Operation::CancelRepoBuild,
-      ResourceTarget::Repo(
-        resource::get::<Repo>(&data.repo).await?.id,
-      ),
-    ),
-
-    // Procedure
-    ExecuteRequest::RunProcedure(data) => (
-      Operation::RunProcedure,
-      ResourceTarget::Procedure(
-        resource::get::<Procedure>(&data.procedure).await?.id,
-      ),
-    ),
-    ExecuteRequest::BatchRunProcedure(_) => {
-      return Ok(Default::default());
-    }
-
-    // Action
-    ExecuteRequest::RunAction(data) => (
-      Operation::RunAction,
-      ResourceTarget::Action(
-        resource::get::<Action>(&data.action).await?.id,
-      ),
-    ),
-    ExecuteRequest::BatchRunAction(_) => {
-      return Ok(Default::default());
-    }
-
-    // Resource Sync
-    ExecuteRequest::RunSync(data) => (
-      Operation::RunSync,
-      ResourceTarget::ResourceSync(
-        resource::get::<ResourceSync>(&data.sync).await?.id,
-      ),
-    ),
-
-    // Stack
-    ExecuteRequest::DeployStack(data) => (
-      if !data.services.is_empty() {
-        Operation::DeployStackService
-      } else {
-        Operation::DeployStack
-      },
-      ResourceTarget::Stack(
-        resource::get::<Stack>(&data.stack).await?.id,
-      ),
-    ),
-    ExecuteRequest::BatchDeployStack(_data) => {
-      return Ok(Default::default());
-    }
-    ExecuteRequest::DeployStackIfChanged(data) => (
-      Operation::DeployStack,
-      ResourceTarget::Stack(
-        resource::get::<Stack>(&data.stack).await?.id,
-      ),
-    ),
-    ExecuteRequest::BatchDeployStackIfChanged(_data) => {
-      return Ok(Default::default());
-    }
-    ExecuteRequest::StartStack(data) => (
-      if !data.services.is_empty() {
-        Operation::StartStackService
-      } else {
-        Operation::StartStack
-      },
-      ResourceTarget::Stack(
-        resource::get::<Stack>(&data.stack).await?.id,
-      ),
-    ),
-    ExecuteRequest::PullStack(data) => (
-      if !data.services.is_empty() {
-        Operation::PullStackService
-      } else {
-        Operation::PullStack
-      },
-      ResourceTarget::Stack(
-        resource::get::<Stack>(&data.stack).await?.id,
-      ),
-    ),
-    ExecuteRequest::BatchPullStack(_data) => {
-      return Ok(Default::default());
-    }
-    ExecuteRequest::RestartStack(data) => (
-      if !data.services.is_empty() {
-        Operation::RestartStackService
-      } else {
-        Operation::RestartStack
-      },
-      ResourceTarget::Stack(
-        resource::get::<Stack>(&data.stack).await?.id,
-      ),
-    ),
-    ExecuteRequest::PauseStack(data) => (
-      if !data.services.is_empty() {
-        Operation::PauseStackService
-      } else {
-        Operation::PauseStack
-      },
-      ResourceTarget::Stack(
-        resource::get::<Stack>(&data.stack).await?.id,
-      ),
-    ),
-    ExecuteRequest::UnpauseStack(data) => (
-      if !data.services.is_empty() {
-        Operation::UnpauseStackService
-      } else {
-        Operation::UnpauseStack
-      },
-      ResourceTarget::Stack(
-        resource::get::<Stack>(&data.stack).await?.id,
-      ),
-    ),
-    ExecuteRequest::StopStack(data) => (
-      if !data.services.is_empty() {
-        Operation::StopStackService
-      } else {
-        Operation::StopStack
-      },
-      ResourceTarget::Stack(
-        resource::get::<Stack>(&data.stack).await?.id,
-      ),
-    ),
-    ExecuteRequest::DestroyStack(data) => (
-      if !data.services.is_empty() {
-        Operation::DestroyStackService
-      } else {
-        Operation::DestroyStack
-      },
-      ResourceTarget::Stack(
-        resource::get::<Stack>(&data.stack).await?.id,
-      ),
-    ),
-    ExecuteRequest::BatchDestroyStack(_data) => {
-      return Ok(Default::default());
-    }
-
-    ExecuteRequest::RunStackService(data) => (
-      Operation::RunStackService,
-      ResourceTarget::Stack(
-        resource::get::<Stack>(&data.stack).await?.id,
-      ),
-    ),
-
-    // Alerter
-    ExecuteRequest::TestAlerter(data) => (
-      Operation::TestAlerter,
-      ResourceTarget::Alerter(
-        resource::get::<Alerter>(&data.alerter).await?.id,
-      ),
-    ),
-    ExecuteRequest::SendAlert(_) => {
-      (Operation::SendAlert, ResourceTarget::system())
-    }
-
-    // Maintenance
-    ExecuteRequest::ClearRepoCache(_data) => {
-      (Operation::ClearRepoCache, ResourceTarget::system())
-    }
-    ExecuteRequest::BackupCoreDatabase(_data) => {
-      (Operation::BackupCoreDatabase, ResourceTarget::system())
-    }
-    ExecuteRequest::GlobalAutoUpdate(_data) => {
-      (Operation::GlobalAutoUpdate, ResourceTarget::system())
-    }
-    ExecuteRequest::RotateAllServerKeys(_data) => {
-      (Operation::RotateAllServerKeys, ResourceTarget::system())
-    }
-    ExecuteRequest::RotateCoreKeys(_data) => {
-      (Operation::RotateCoreKeys, ResourceTarget::system())
-    }
-  };
+  let (operation, target) = init_execution_match!(
+    resource: [
+      // Swarm
+      (RemoveSwarmNodes, Swarm, swarm),
+      (UpdateSwarmNode, Swarm, swarm),
+      (RemoveSwarmStacks, Swarm, swarm),
+      (RemoveSwarmServices, Swarm, swarm),
+      (CreateSwarmConfig, Swarm, swarm),
+      (RotateSwarmConfig, Swarm, swarm),
+      (RemoveSwarmConfigs, Swarm, swarm),
+      (CreateSwarmSecret, Swarm, swarm),
+      (RotateSwarmSecret, Swarm, swarm),
+      (RemoveSwarmSecrets, Swarm, swarm),
+      // Server
+      (StartContainer, Server, server),
+      (RestartContainer, Server, server),
+      (PauseContainer, Server, server),
+      (UnpauseContainer, Server, server),
+      (StopContainer, Server, server),
+      (DestroyContainer, Server, server),
+      (StartAllContainers, Server, server),
+      (RestartAllContainers, Server, server),
+      (PauseAllContainers, Server, server),
+      (UnpauseAllContainers, Server, server),
+      (StopAllContainers, Server, server),
+      (PruneContainers, Server, server),
+      (DeleteNetwork, Server, server),
+      (PruneNetworks, Server, server),
+      (DeleteImage, Server, server),
+      (PruneImages, Server, server),
+      (DeleteVolume, Server, server),
+      (PruneVolumes, Server, server),
+      (PruneDockerBuilders, Server, server),
+      (PruneBuildx, Server, server),
+      (PruneSystem, Server, server),
+      // Deployment
+      (Deploy, Deployment, deployment),
+      (PullDeployment, Deployment, deployment),
+      (StartDeployment, Deployment, deployment),
+      (RestartDeployment, Deployment, deployment),
+      (PauseDeployment, Deployment, deployment),
+      (UnpauseDeployment, Deployment, deployment),
+      (StopDeployment, Deployment, deployment),
+      (DestroyDeployment, Deployment, deployment),
+      // Build
+      (RunBuild, Build, build),
+      (CancelBuild, Build, build),
+      // Repo
+      (CloneRepo, Repo, repo),
+      (PullRepo, Repo, repo),
+      (BuildRepo, Repo, repo),
+      (CancelRepoBuild, Repo, repo),
+      // Procedure
+      (RunProcedure, Procedure, procedure),
+      // Action
+      (RunAction, Action, action),
+      // Resource Sync
+      (RunSync, ResourceSync, sync),
+      // Stack (simple)
+      (RunStackService, Stack, stack),
+      // Alerter
+      (TestAlerter, Alerter, alerter),
+    ],
+    batch: [
+      BatchDeploy,
+      BatchDestroyDeployment,
+      BatchRunBuild,
+      BatchCloneRepo,
+      BatchPullRepo,
+      BatchBuildRepo,
+      BatchRunProcedure,
+      BatchRunAction,
+      BatchDeployStack,
+      BatchDeployStackIfChanged,
+      BatchPullStack,
+      BatchDestroyStack,
+    ],
+    stack_service: [
+      (DeployStack, DeployStackService),
+      (PullStack, PullStackService),
+      (StartStack, StartStackService),
+      (RestartStack, RestartStackService),
+      (PauseStack, PauseStackService),
+      (UnpauseStack, UnpauseStackService),
+      (StopStack, StopStackService),
+      (DestroyStack, DestroyStackService),
+    ],
+    system: [
+      SendAlert,
+      ClearRepoCache,
+      BackupCoreDatabase,
+      GlobalAutoUpdate,
+      RotateAllServerKeys,
+      RotateCoreKeys,
+    ],
+  );
 
   let mut update = make_update(target, operation, user);
   update.in_progress();

bin/core/src/monitor/resources.rs

@@ -46,8 +46,10 @@ pub async fn update_swarm_stack_cache(
         Some(SwarmState::Unknown) | None => StackState::Unknown,
       })
       .unwrap_or(StackState::Down);
+
     let services = extract_services_from_stack(&stack);
     let service_prefix = format!("{project_name}_");
+
     let mut services_with_swarm_services = services
       .iter()
       .map(
@@ -74,27 +76,50 @@ pub async fn update_swarm_stack_cache(
                 .unwrap_or_default()
             })
             .cloned();
+
+          let (image, image_digests) = swarm_service
+            .as_ref()
+            .and_then(|swarm_service| swarm_service.image.as_ref())
+            .map(|image| {
+              (
+                image.clone(),
+                ImageDigest::parse(image).map(|d| vec![d]),
+              )
+            })
+            .unwrap_or((
+              if image.contains(':') {
+                image.to_string()
+              } else {
+                format!("{image}:latest")
+              },
+              None,
+            ));
+
           StackService {
             service: service_name.clone(),
-            image: image.clone(),
             container: None,
             swarm_service,
-            image_digests: Default::default(),
+            image,
+            image_digests,
           }
         },
       )
       .collect::<Vec<_>>();
+
     services_with_swarm_services
       .sort_by(|a, b| a.service.cmp(&b.service));
+
     let prev_state = stack_status_cache
       .get(&stack.id)
       .await
       .map(|s| s.curr.state);
+
     let status = CachedStackStatus {
       id: stack.id.clone(),
       state: current_state,
       services: services_with_swarm_services,
     };
+
     stack_status_cache
       .insert(
         stack.id,
@@ -207,10 +232,17 @@ pub async fn update_swarm_deployment_cache(
           .unwrap_or_default()
       })
       .cloned();
+
+    let image_digests = service
+      .as_ref()
+      .and_then(|service| service.image.as_ref())
+      .and_then(|image| ImageDigest::parse(image).map(|d| vec![d]));
+
     let prev_state = deployment_status_cache
       .get(&deployment.id)
       .await
       .map(|s| s.curr.state);
+
     let current_state = service
       .as_ref()
       .map(|service| match service.state {
@@ -220,6 +252,7 @@ pub async fn update_swarm_deployment_cache(
         SwarmState::Unknown => DeploymentState::Unknown,
       })
       .unwrap_or(DeploymentState::NotDeployed);
+
     deployment_status_cache
       .insert(
         deployment.id.clone(),
@@ -227,9 +260,9 @@ pub async fn update_swarm_deployment_cache(
           curr: CachedDeploymentStatus {
             id: deployment.id,
             state: current_state,
-            service,
             container: None,
-            image_digests: None,
+            service,
+            image_digests,
           },
           prev: prev_state,
         }
@@ -251,6 +284,7 @@ pub async fn update_server_deployment_cache(
       .iter()
       .find(|container| container.name == deployment.name)
       .cloned();
+
     let image_digests = container
       .as_ref()
       .and_then(|container| container.image_id.as_ref())
@@ -263,10 +297,12 @@ pub async fn update_server_deployment_cache(
           }
         })
       });
+
     let prev_state = deployment_status_cache
       .get(&deployment.id)
       .await
       .map(|s| s.curr.state);
+
     let current_state = container
       .as_ref()
       .map(|c| c.state.into())

bin/core/src/monitor/swarm.rs

@@ -6,7 +6,6 @@ use std::{
 use anyhow::anyhow;
 use async_timing_util::wait_until_timelength;
 use database::mungos::find::find_collect;
-use formatting::format_serror;
 use futures_util::future::join_all;
 use komodo_client::entities::{
   docker::node::NodeState,
@@ -108,9 +107,9 @@ pub async fn refresh_swarm_cache(swarm: &Swarm, force: bool) {
           state: SwarmState::Unknown,
           inspect: None,
           lists: None,
-          err: Some(format_serror(
-            &anyhow!("No Servers configured as manager nodes").into(),
-          )),
+          err: Some(
+            anyhow!("No servers configured as manager nodes").into(),
+          ),
         }
         .into(),
       )
@@ -137,7 +136,7 @@ pub async fn refresh_swarm_cache(swarm: &Swarm, force: bool) {
               state: SwarmState::Unknown,
               inspect: None,
               lists: None,
-              err: Some(format_serror(&e.into())),
+              err: Some(e.into()),
             }
             .into(),
           )

bin/core/src/resource/procedure.rs

@@ -183,695 +183,183 @@ async fn validate_config(
   };
   for stage in stages {
     for exec in &mut stage.executions {
-      match &mut exec.execution {
-        Execution::None(_) => {}
-        Execution::RunProcedure(params) => {
-          let procedure = super::get_check_permissions::<Procedure>(
-            &params.procedure,
-            user,
-            PermissionLevel::Execute.into(),
-          )
-          .await?;
-          match id {
-            Some(id) if procedure.id == id => {
-              return Err(anyhow!(
-                "Cannot have self-referential procedure"
-              ));
-            }
-            _ => {}
-          }
-          params.procedure = procedure.id;
-        }
-        Execution::BatchRunProcedure(_params) => {
-          if !user.admin {
-            return Err(anyhow!(
-              "Non admin user cannot configure Batch executions"
-            ));
-          }
-        }
-        Execution::RunAction(params) => {
-          let action = super::get_check_permissions::<Action>(
-            &params.action,
-            user,
-            PermissionLevel::Execute.into(),
-          )
-          .await?;
-          params.action = action.id;
-        }
-        Execution::BatchRunAction(_params) => {
-          if !user.admin {
-            return Err(anyhow!(
-              "Non admin user cannot configure Batch executions"
-            ));
-          }
-        }
-        Execution::RunBuild(params) => {
-          let build = super::get_check_permissions::<Build>(
-            &params.build,
-            user,
-            PermissionLevel::Execute.into(),
-          )
-          .await?;
-          params.build = build.id;
-        }
-        Execution::BatchRunBuild(_params) => {
-          if !user.admin {
-            return Err(anyhow!(
-              "Non admin user cannot configure Batch executions"
-            ));
-          }
-        }
-        Execution::CancelBuild(params) => {
-          let build = super::get_check_permissions::<Build>(
-            &params.build,
-            user,
-            PermissionLevel::Execute.into(),
-          )
-          .await?;
-          params.build = build.id;
-        }
-        Execution::Deploy(params) => {
-          let deployment =
-            super::get_check_permissions::<Deployment>(
-              &params.deployment,
-              user,
-              PermissionLevel::Execute.into(),
-            )
-            .await?;
-          params.deployment = deployment.id;
-        }
-        Execution::BatchDeploy(_params) => {
-          if !user.admin {
-            return Err(anyhow!(
-              "Non admin user cannot configure Batch executions"
-            ));
-          }
-        }
-        Execution::PullDeployment(params) => {
-          let deployment =
-            super::get_check_permissions::<Deployment>(
-              &params.deployment,
-              user,
-              PermissionLevel::Execute.into(),
-            )
-            .await?;
-          params.deployment = deployment.id;
-        }
-        Execution::StartDeployment(params) => {
-          let deployment =
-            super::get_check_permissions::<Deployment>(
-              &params.deployment,
-              user,
-              PermissionLevel::Execute.into(),
-            )
-            .await?;
-          params.deployment = deployment.id;
-        }
-        Execution::RestartDeployment(params) => {
-          let deployment =
-            super::get_check_permissions::<Deployment>(
-              &params.deployment,
-              user,
-              PermissionLevel::Execute.into(),
-            )
-            .await?;
-          params.deployment = deployment.id;
-        }
-        Execution::PauseDeployment(params) => {
-          let deployment =
-            super::get_check_permissions::<Deployment>(
-              &params.deployment,
-              user,
-              PermissionLevel::Execute.into(),
-            )
-            .await?;
-          params.deployment = deployment.id;
-        }
-        Execution::UnpauseDeployment(params) => {
-          let deployment =
-            super::get_check_permissions::<Deployment>(
-              &params.deployment,
-              user,
-              PermissionLevel::Execute.into(),
-            )
-            .await?;
-          params.deployment = deployment.id;
-        }
-        Execution::StopDeployment(params) => {
-          let deployment =
-            super::get_check_permissions::<Deployment>(
-              &params.deployment,
-              user,
-              PermissionLevel::Execute.into(),
-            )
-            .await?;
-          params.deployment = deployment.id;
-        }
-        Execution::DestroyDeployment(params) => {
-          let deployment =
-            super::get_check_permissions::<Deployment>(
-              &params.deployment,
-              user,
-              PermissionLevel::Execute.into(),
-            )
-            .await?;
-          params.deployment = deployment.id;
-        }
-        Execution::BatchDestroyDeployment(_params) => {
-          if !user.admin {
-            return Err(anyhow!(
-              "Non admin user cannot configure Batch executions"
-            ));
-          }
-        }
-        Execution::CloneRepo(params) => {
-          let repo = super::get_check_permissions::<Repo>(
-            &params.repo,
-            user,
-            PermissionLevel::Execute.into(),
-          )
-          .await?;
-          params.repo = repo.id;
-        }
-        Execution::BatchCloneRepo(_params) => {
-          if !user.admin {
-            return Err(anyhow!(
-              "Non admin user cannot configure Batch executions"
-            ));
-          }
-        }
-        Execution::PullRepo(params) => {
-          let repo = super::get_check_permissions::<Repo>(
-            &params.repo,
-            user,
-            PermissionLevel::Execute.into(),
-          )
-          .await?;
-          params.repo = repo.id;
-        }
-        Execution::BatchPullRepo(_params) => {
-          if !user.admin {
-            return Err(anyhow!(
-              "Non admin user cannot configure Batch executions"
-            ));
-          }
-        }
-        Execution::BuildRepo(params) => {
-          let repo = super::get_check_permissions::<Repo>(
-            &params.repo,
-            user,
-            PermissionLevel::Execute.into(),
-          )
-          .await?;
-          params.repo = repo.id;
-        }
-        Execution::BatchBuildRepo(_params) => {
-          if !user.admin {
-            return Err(anyhow!(
-              "Non admin user cannot configure Batch executions"
-            ));
-          }
-        }
-        Execution::CancelRepoBuild(params) => {
-          let repo = super::get_check_permissions::<Repo>(
-            &params.repo,
-            user,
-            PermissionLevel::Execute.into(),
-          )
-          .await?;
-          params.repo = repo.id;
-        }
-        Execution::StartContainer(params) => {
-          let server = super::get_check_permissions::<Server>(
-            &params.server,
-            user,
-            PermissionLevel::Execute.into(),
-          )
-          .await?;
-          params.server = server.id;
-        }
-        Execution::RestartContainer(params) => {
-          let server = super::get_check_permissions::<Server>(
-            &params.server,
-            user,
-            PermissionLevel::Execute.into(),
-          )
-          .await?;
-          params.server = server.id;
-        }
-        Execution::PauseContainer(params) => {
-          let server = super::get_check_permissions::<Server>(
-            &params.server,
-            user,
-            PermissionLevel::Execute.into(),
-          )
-          .await?;
-          params.server = server.id;
-        }
-        Execution::UnpauseContainer(params) => {
-          let server = super::get_check_permissions::<Server>(
-            &params.server,
-            user,
-            PermissionLevel::Execute.into(),
-          )
-          .await?;
-          params.server = server.id;
-        }
-        Execution::StopContainer(params) => {
-          let server = super::get_check_permissions::<Server>(
-            &params.server,
-            user,
-            PermissionLevel::Execute.into(),
-          )
-          .await?;
-          params.server = server.id;
-        }
-        Execution::DestroyContainer(params) => {
-          let server = super::get_check_permissions::<Server>(
-            &params.server,
-            user,
-            PermissionLevel::Execute.into(),
-          )
-          .await?;
-          params.server = server.id;
-        }
-        Execution::StartAllContainers(params) => {
-          let server = super::get_check_permissions::<Server>(
-            &params.server,
-            user,
-            PermissionLevel::Execute.into(),
-          )
-          .await?;
-          params.server = server.id;
-        }
-        Execution::RestartAllContainers(params) => {
-          let server = super::get_check_permissions::<Server>(
-            &params.server,
-            user,
-            PermissionLevel::Execute.into(),
-          )
-          .await?;
-          params.server = server.id;
-        }
-        Execution::PauseAllContainers(params) => {
-          let server = super::get_check_permissions::<Server>(
-            &params.server,
-            user,
-            PermissionLevel::Execute.into(),
-          )
-          .await?;
-          params.server = server.id;
-        }
-        Execution::UnpauseAllContainers(params) => {
-          let server = super::get_check_permissions::<Server>(
-            &params.server,
-            user,
-            PermissionLevel::Execute.into(),
-          )
-          .await?;
-          params.server = server.id;
-        }
-        Execution::StopAllContainers(params) => {
-          let server = super::get_check_permissions::<Server>(
-            &params.server,
-            user,
-            PermissionLevel::Execute.into(),
-          )
-          .await?;
-          params.server = server.id;
-        }
-        Execution::PruneContainers(params) => {
-          let server = super::get_check_permissions::<Server>(
-            &params.server,
-            user,
-            PermissionLevel::Execute.into(),
-          )
-          .await?;
-          params.server = server.id;
-        }
-        Execution::DeleteNetwork(params) => {
-          let server = super::get_check_permissions::<Server>(
-            &params.server,
-            user,
-            PermissionLevel::Execute.into(),
-          )
-          .await?;
-          params.server = server.id;
-        }
-        Execution::PruneNetworks(params) => {
-          let server = super::get_check_permissions::<Server>(
-            &params.server,
-            user,
-            PermissionLevel::Execute.into(),
-          )
-          .await?;
-          params.server = server.id;
-        }
-        Execution::DeleteImage(params) => {
-          let server = super::get_check_permissions::<Server>(
-            &params.server,
-            user,
-            PermissionLevel::Execute.into(),
-          )
-          .await?;
-          params.server = server.id;
-        }
-        Execution::PruneImages(params) => {
-          let server = super::get_check_permissions::<Server>(
-            &params.server,
-            user,
-            PermissionLevel::Execute.into(),
-          )
-          .await?;
-          params.server = server.id;
-        }
-        Execution::DeleteVolume(params) => {
-          let server = super::get_check_permissions::<Server>(
-            &params.server,
-            user,
-            PermissionLevel::Execute.into(),
-          )
-          .await?;
-          params.server = server.id;
-        }
-        Execution::PruneVolumes(params) => {
-          let server = super::get_check_permissions::<Server>(
-            &params.server,
-            user,
-            PermissionLevel::Execute.into(),
-          )
-          .await?;
-          params.server = server.id;
-        }
-        Execution::PruneDockerBuilders(params) => {
-          let server = super::get_check_permissions::<Server>(
-            &params.server,
-            user,
-            PermissionLevel::Execute.into(),
-          )
-          .await?;
-          params.server = server.id;
-        }
-        Execution::PruneBuildx(params) => {
-          let server = super::get_check_permissions::<Server>(
-            &params.server,
-            user,
-            PermissionLevel::Execute.into(),
-          )
-          .await?;
-          params.server = server.id;
-        }
-        Execution::PruneSystem(params) => {
-          let server = super::get_check_permissions::<Server>(
-            &params.server,
-            user,
-            PermissionLevel::Execute.into(),
-          )
-          .await?;
-          params.server = server.id;
-        }
-        Execution::RunSync(params) => {
-          let sync = super::get_check_permissions::<ResourceSync>(
-            &params.sync,
-            user,
-            PermissionLevel::Execute.into(),
-          )
-          .await?;
-          params.sync = sync.id;
-        }
-        Execution::CommitSync(params) => {
-          // This one is actually a write operation.
-          let sync = super::get_check_permissions::<ResourceSync>(
-            &params.sync,
-            user,
-            PermissionLevel::Write.into(),
-          )
-          .await?;
-          params.sync = sync.id;
-        }
-        Execution::DeployStack(params) => {
-          let stack = super::get_check_permissions::<Stack>(
-            &params.stack,
-            user,
-            PermissionLevel::Execute.into(),
-          )
-          .await?;
-          params.stack = stack.id;
-        }
-        Execution::BatchDeployStack(_params) => {
-          if !user.admin {
-            return Err(anyhow!(
-              "Non admin user cannot configure Batch executions"
-            ));
-          }
-        }
-        Execution::DeployStackIfChanged(params) => {
-          let stack = super::get_check_permissions::<Stack>(
-            &params.stack,
-            user,
-            PermissionLevel::Execute.into(),
-          )
-          .await?;
-          params.stack = stack.id;
-        }
-        Execution::BatchDeployStackIfChanged(_params) => {
-          if !user.admin {
-            return Err(anyhow!(
-              "Non admin user cannot configure Batch executions"
-            ));
-          }
-        }
-        Execution::PullStack(params) => {
-          let stack = super::get_check_permissions::<Stack>(
-            &params.stack,
-            user,
-            PermissionLevel::Execute.into(),
-          )
-          .await?;
-          params.stack = stack.id;
-        }
-        Execution::BatchPullStack(_params) => {
-          if !user.admin {
-            return Err(anyhow!(
-              "Non admin user cannot configure Batch executions"
-            ));
-          }
-        }
-        Execution::StartStack(params) => {
-          let stack = super::get_check_permissions::<Stack>(
-            &params.stack,
-            user,
-            PermissionLevel::Execute.into(),
-          )
-          .await?;
-          params.stack = stack.id;
-        }
-        Execution::RestartStack(params) => {
-          let stack = super::get_check_permissions::<Stack>(
-            &params.stack,
-            user,
-            PermissionLevel::Execute.into(),
-          )
-          .await?;
-          params.stack = stack.id;
-        }
-        Execution::PauseStack(params) => {
-          let stack = super::get_check_permissions::<Stack>(
-            &params.stack,
-            user,
-            PermissionLevel::Execute.into(),
-          )
-          .await?;
-          params.stack = stack.id;
-        }
-        Execution::UnpauseStack(params) => {
-          let stack = super::get_check_permissions::<Stack>(
-            &params.stack,
-            user,
-            PermissionLevel::Execute.into(),
-          )
-          .await?;
-          params.stack = stack.id;
-        }
-        Execution::StopStack(params) => {
-          let stack = super::get_check_permissions::<Stack>(
-            &params.stack,
-            user,
-            PermissionLevel::Execute.into(),
-          )
-          .await?;
-          params.stack = stack.id;
-        }
-        Execution::DestroyStack(params) => {
-          let stack = super::get_check_permissions::<Stack>(
-            &params.stack,
-            user,
-            PermissionLevel::Execute.into(),
-          )
-          .await?;
-          params.stack = stack.id;
-        }
-        Execution::RunStackService(params) => {
-          let stack = super::get_check_permissions::<Stack>(
-            &params.stack,
-            user,
-            PermissionLevel::Execute.into(),
-          )
-          .await?;
-          params.stack = stack.id;
-        }
-        Execution::BatchDestroyStack(_params) => {
-          if !user.admin {
-            return Err(anyhow!(
-              "Non admin user cannot configure Batch executions"
-            ));
-          }
-        }
-        Execution::TestAlerter(params) => {
-          let alerter = super::get_check_permissions::<Alerter>(
-            &params.alerter,
-            user,
-            PermissionLevel::Execute.into(),
-          )
-          .await?;
-          params.alerter = alerter.id;
-        }
-        Execution::SendAlert(params) => {
-          params.alerters = params
-            .alerters
-            .iter()
-            .map(async |alerter| {
-              let id = super::get_check_permissions::<Alerter>(
-                alerter,
+      macro_rules! check_execution_perms {
+        (
+          execute: [$(($Variant:ident, $Type:ident, $field:ident)),* $(,)?],
+          batch_admin: [$($BatchVariant:ident),* $(,)?],
+          admin_only: [$(($AdminVariant:ident, $msg:literal)),* $(,)?],
+        ) => {
+          match &mut exec.execution {
+            $(
+              Execution::$Variant(params) => {
+                let resource = super::get_check_permissions::<$Type>(
+                  &params.$field,
+                  user,
+                  PermissionLevel::Execute.into(),
+                )
+                .await?;
+                params.$field = resource.id;
+              }
+            )*
+            $(
+              Execution::$BatchVariant(_params) => {
+                if !user.admin {
+                  return Err(anyhow!(
+                    "Non admin user cannot configure Batch executions"
+                  ));
+                }
+              }
+            )*
+            $(
+              Execution::$AdminVariant(_params) => {
+                if !user.admin {
+                  return Err(anyhow!($msg));
+                }
+              }
+            )*
+            // Special: self-referential procedure check
+            Execution::RunProcedure(params) => {
+              let procedure = super::get_check_permissions::<Procedure>(
+                &params.procedure,
                 user,
                 PermissionLevel::Execute.into(),
               )
-              .await?
-              .id;
-              anyhow::Ok(id)
-            })
-            .collect::<FuturesUnordered<_>>()
-            .try_collect::<Vec<_>>()
-            .await?;
-        }
-        Execution::RemoveSwarmNodes(params) => {
-          let swarm = super::get_check_permissions::<Swarm>(
-            &params.swarm,
-            user,
-            PermissionLevel::Execute.into(),
-          )
-          .await?;
-          params.swarm = swarm.id;
-        }
-        Execution::RemoveSwarmStacks(params) => {
-          let swarm = super::get_check_permissions::<Swarm>(
-            &params.swarm,
-            user,
-            PermissionLevel::Execute.into(),
-          )
-          .await?;
-          params.swarm = swarm.id;
-        }
-        Execution::RemoveSwarmServices(params) => {
-          let swarm = super::get_check_permissions::<Swarm>(
-            &params.swarm,
-            user,
-            PermissionLevel::Execute.into(),
-          )
-          .await?;
-          params.swarm = swarm.id;
-        }
-        Execution::CreateSwarmConfig(params) => {
-          let swarm = super::get_check_permissions::<Swarm>(
-            &params.swarm,
-            user,
-            PermissionLevel::Execute.into(),
-          )
-          .await?;
-          params.swarm = swarm.id;
-        }
-        Execution::RotateSwarmConfig(params) => {
-          let swarm = super::get_check_permissions::<Swarm>(
-            &params.swarm,
-            user,
-            PermissionLevel::Execute.into(),
-          )
-          .await?;
-          params.swarm = swarm.id;
-        }
-        Execution::RemoveSwarmConfigs(params) => {
-          let swarm = super::get_check_permissions::<Swarm>(
-            &params.swarm,
-            user,
-            PermissionLevel::Execute.into(),
-          )
-          .await?;
-          params.swarm = swarm.id;
-        }
-        Execution::CreateSwarmSecret(params) => {
-          let swarm = super::get_check_permissions::<Swarm>(
-            &params.swarm,
-            user,
-            PermissionLevel::Execute.into(),
-          )
-          .await?;
-          params.swarm = swarm.id;
-        }
-        Execution::RotateSwarmSecret(params) => {
-          let swarm = super::get_check_permissions::<Swarm>(
-            &params.swarm,
-            user,
-            PermissionLevel::Execute.into(),
-          )
-          .await?;
-          params.swarm = swarm.id;
-        }
-        Execution::RemoveSwarmSecrets(params) => {
-          let swarm = super::get_check_permissions::<Swarm>(
-            &params.swarm,
-            user,
-            PermissionLevel::Execute.into(),
-          )
-          .await?;
-          params.swarm = swarm.id;
-        }
-        Execution::ClearRepoCache(_params) => {
-          if !user.admin {
-            return Err(anyhow!(
-              "Non admin user cannot clear repo cache"
-            ));
+              .await?;
+              match id {
+                Some(id) if procedure.id == id => {
+                  return Err(anyhow!(
+                    "Cannot have self-referential procedure"
+                  ));
+                }
+                _ => {}
+              }
+              params.procedure = procedure.id;
+            }
+            // Special: CommitSync uses Write permission
+            Execution::CommitSync(params) => {
+              let sync = super::get_check_permissions::<ResourceSync>(
+                &params.sync,
+                user,
+                PermissionLevel::Write.into(),
+              )
+              .await?;
+              params.sync = sync.id;
+            }
+            // Special: SendAlert checks a Vec of alerters
+            Execution::SendAlert(params) => {
+              params.alerters = params
+                .alerters
+                .iter()
+                .map(async |alerter| {
+                  let id = super::get_check_permissions::<Alerter>(
+                    alerter,
+                    user,
+                    PermissionLevel::Execute.into(),
+                  )
+                  .await?
+                  .id;
+                  anyhow::Ok(id)
+                })
+                .collect::<FuturesUnordered<_>>()
+                .try_collect::<Vec<_>>()
+                .await?;
+            }
+            Execution::None(_) | Execution::Sleep(_) => {}
           }
-        }
-        Execution::BackupCoreDatabase(_params) => {
-          if !user.admin {
-            return Err(anyhow!(
-              "Non admin user cannot trigger core database backup"
-            ));
-          }
-        }
-        Execution::GlobalAutoUpdate(_params) => {
-          if !user.admin {
-            return Err(anyhow!(
-              "Non admin user cannot trigger global auto update"
-            ));
-          }
-        }
-        Execution::RotateAllServerKeys(_params) => {
-          if !user.admin {
-            return Err(anyhow!(
-              "Non admin user cannot trigger rotate all server keys"
-            ));
-          }
-        }
-        Execution::RotateCoreKeys(_params) => {
-          if !user.admin {
-            return Err(anyhow!(
-              "Non admin user cannot trigger rotate core keys"
-            ));
-          }
-        }
-        Execution::Sleep(_) => {}
+        };
       }
+      check_execution_perms!(
+        execute: [
+          // Action
+          (RunAction, Action, action),
+          // Build
+          (RunBuild, Build, build),
+          (CancelBuild, Build, build),
+          // Deployment
+          (Deploy, Deployment, deployment),
+          (PullDeployment, Deployment, deployment),
+          (StartDeployment, Deployment, deployment),
+          (RestartDeployment, Deployment, deployment),
+          (PauseDeployment, Deployment, deployment),
+          (UnpauseDeployment, Deployment, deployment),
+          (StopDeployment, Deployment, deployment),
+          (DestroyDeployment, Deployment, deployment),
+          // Repo
+          (CloneRepo, Repo, repo),
+          (PullRepo, Repo, repo),
+          (BuildRepo, Repo, repo),
+          (CancelRepoBuild, Repo, repo),
+          // Server
+          (StartContainer, Server, server),
+          (RestartContainer, Server, server),
+          (PauseContainer, Server, server),
+          (UnpauseContainer, Server, server),
+          (StopContainer, Server, server),
+          (DestroyContainer, Server, server),
+          (StartAllContainers, Server, server),
+          (RestartAllContainers, Server, server),
+          (PauseAllContainers, Server, server),
+          (UnpauseAllContainers, Server, server),
+          (StopAllContainers, Server, server),
+          (PruneContainers, Server, server),
+          (DeleteNetwork, Server, server),
+          (PruneNetworks, Server, server),
+          (DeleteImage, Server, server),
+          (PruneImages, Server, server),
+          (DeleteVolume, Server, server),
+          (PruneVolumes, Server, server),
+          (PruneDockerBuilders, Server, server),
+          (PruneBuildx, Server, server),
+          (PruneSystem, Server, server),
+          // Resource Sync
+          (RunSync, ResourceSync, sync),
+          // Stack
+          (DeployStack, Stack, stack),
+          (DeployStackIfChanged, Stack, stack),
+          (PullStack, Stack, stack),
+          (StartStack, Stack, stack),
+          (RestartStack, Stack, stack),
+          (PauseStack, Stack, stack),
+          (UnpauseStack, Stack, stack),
+          (StopStack, Stack, stack),
+          (DestroyStack, Stack, stack),
+          (RunStackService, Stack, stack),
+          // Alerter
+          (TestAlerter, Alerter, alerter),
+          // Swarm
+          (RemoveSwarmNodes, Swarm, swarm),
+          (UpdateSwarmNode, Swarm, swarm),
+          (RemoveSwarmStacks, Swarm, swarm),
+          (RemoveSwarmServices, Swarm, swarm),
+          (CreateSwarmConfig, Swarm, swarm),
+          (RotateSwarmConfig, Swarm, swarm),
+          (RemoveSwarmConfigs, Swarm, swarm),
+          (CreateSwarmSecret, Swarm, swarm),
+          (RotateSwarmSecret, Swarm, swarm),
+          (RemoveSwarmSecrets, Swarm, swarm),
+        ],
+        batch_admin: [
+          BatchRunProcedure,
+          BatchRunAction,
+          BatchRunBuild,
+          BatchDeploy,
+          BatchDestroyDeployment,
+          BatchCloneRepo,
+          BatchPullRepo,
+          BatchBuildRepo,
+          BatchDeployStack,
+          BatchDeployStackIfChanged,
+          BatchPullStack,
+          BatchDestroyStack,
+        ],
+        admin_only: [
+          (ClearRepoCache, "Non admin user cannot clear repo cache"),
+          (BackupCoreDatabase, "Non admin user cannot trigger core database backup"),
+          (GlobalAutoUpdate, "Non admin user cannot trigger global auto update"),
+          (RotateAllServerKeys, "Non admin user cannot trigger rotate all server keys"),
+          (RotateCoreKeys, "Non admin user cannot trigger rotate core keys"),
+        ],
+      );
     }
   }
 

bin/core/src/resource/server.rs

@@ -93,6 +93,7 @@ impl super::KomodoResource for Server {
       resource_type: ResourceTargetVariant::Server,
       info: ServerListItemInfo {
         state: status.as_ref().map(|s| s.state).unwrap_or_default(),
+        err: status.as_ref().and_then(|s| s.err.clone()),
         region: server.config.region,
         address: optional_string(server.config.address),
         external_address: optional_string(

bin/core/src/stack/services.rs

@@ -38,8 +38,6 @@ pub fn extract_services_into_res(
         "failed to parse service names from compose contents",
       )?;
 
-  let mut services = Vec::with_capacity(compose.services.capacity());
-
   for (
     service_name,
     ComposeService {
@@ -49,17 +47,29 @@ pub fn extract_services_into_res(
     },
   ) in compose.services
   {
-    let image = image.unwrap_or_default();
-    services.push(StackServiceNames {
-      container_name: container_name
-        .unwrap_or_else(|| format!("{project_name}-{service_name}")),
-      image_digest: service_image_digests.get(&service_name).cloned(),
-      service_name,
-      image,
-    });
+    if let Some(existing) =
+      res.iter_mut().find(|s| s.service_name == service_name)
+    {
+      // Override any defined fields
+      if let Some(container_name) = container_name {
+        existing.container_name = container_name;
+      }
+      if let Some(image) = image {
+        existing.image = image;
+      }
+    } else {
+      res.push(StackServiceNames {
+        container_name: container_name.unwrap_or_else(|| {
+          format!("{project_name}-{service_name}")
+        }),
+        image_digest: service_image_digests
+          .get(&service_name)
+          .cloned(),
+        image: image.unwrap_or_default(),
+        service_name,
+      });
+    }
   }
 
-  res.extend(services);
-
   Ok(())
 }

bin/core/src/state.rs

@@ -87,7 +87,7 @@ pub struct CachedSwarmStatus {
   pub inspect: Option<SwarmInspectInfo>,
   pub lists: Option<SwarmLists>,
   /// Store the error in communicating with Swarm
-  pub err: Option<String>,
+  pub err: Option<mogh_error::Serror>,
 }
 
 pub type SwarmStatusCache =

bin/core/src/sync/file.rs

@@ -243,36 +243,27 @@ pub fn extend_resources(
   more: ResourcesToml,
   match_tags: &[String],
 ) {
-  resources
-    .servers
-    .extend(filter_by_tag(more.servers, match_tags));
-  resources
-    .stacks
-    .extend(filter_by_tag(more.stacks, match_tags));
-  resources
-    .deployments
-    .extend(filter_by_tag(more.deployments, match_tags));
-  resources
-    .builds
-    .extend(filter_by_tag(more.builds, match_tags));
-  resources
-    .repos
-    .extend(filter_by_tag(more.repos, match_tags));
-  resources
-    .procedures
-    .extend(filter_by_tag(more.procedures, match_tags));
-  resources
-    .actions
-    .extend(filter_by_tag(more.actions, match_tags));
-  resources
-    .alerters
-    .extend(filter_by_tag(more.alerters, match_tags));
-  resources
-    .builders
-    .extend(filter_by_tag(more.builders, match_tags));
-  resources
-    .resource_syncs
-    .extend(filter_by_tag(more.resource_syncs, match_tags));
+  macro_rules! extend_filtered {
+    ($($field:ident),* $(,)?) => {
+      $(
+        resources.$field.extend(filter_by_tag(more.$field, match_tags));
+      )*
+    };
+  }
+  // New resource types need to be added here manually.
+  extend_filtered!(
+    servers,
+    swarms,
+    stacks,
+    deployments,
+    builds,
+    repos,
+    procedures,
+    actions,
+    alerters,
+    builders,
+    resource_syncs,
+  );
   resources.user_groups.extend(more.user_groups);
   resources.variables.extend(more.variables);
 }

bin/core/src/sync/resources.rs

@@ -1,29 +1,27 @@
 use std::collections::HashMap;
 
 use formatting::{Color, bold, colored, muted};
-use komodo_client::{
-  api::execute::Execution,
-  entities::{
-    ResourceTargetVariant,
-    action::Action,
-    alerter::Alerter,
-    build::Build,
-    builder::{Builder, BuilderConfig},
-    deployment::{Deployment, DeploymentImage},
-    procedure::Procedure,
-    repo::Repo,
-    server::Server,
-    stack::Stack,
-    swarm::Swarm,
-    sync::ResourceSync,
-    tag::Tag,
-    update::Log,
-    user::sync_user,
-  },
+use komodo_client::entities::{
+  ResourceTargetVariant,
+  action::Action,
+  alerter::Alerter,
+  build::Build,
+  builder::{Builder, BuilderConfig},
+  deployment::{Deployment, DeploymentImage},
+  procedure::Procedure,
+  repo::Repo,
+  server::Server,
+  stack::Stack,
+  swarm::Swarm,
+  sync::ResourceSync,
+  tag::Tag,
+  update::Log,
+  user::sync_user,
 };
 use partial_derive2::{MaybeNone, PartialDiff};
 
 use crate::{
+  helpers::procedure::replace_procedure_stage_ids_with_names,
   resource::{KomodoResource, ResourceMetaUpdate},
   state::all_resources_cache,
   sync::{ToUpdateItem, execute::run_update_meta},
@@ -341,123 +339,7 @@ impl ResourceSyncTrait for Procedure {
     mut original: Self::Config,
     update: Self::PartialConfig,
   ) -> anyhow::Result<Self::ConfigDiff> {
-    let all = all_resources_cache().load();
-    for stage in &mut original.stages {
-      for execution in &mut stage.executions {
-        // Replaces an id field on config with the resource name from `all`.
-        macro_rules! replace_id_with_name {
-          ($($Variant:ident => $field:ident, $collection:ident);* $(;)?) => {
-            match &mut execution.execution {
-              $(
-                Execution::$Variant(config) => {
-                  config.$field = all
-                    .$collection
-                    .get(&config.$field)
-                    .map(|r| r.name.clone())
-                    .unwrap_or_default();
-                }
-              )*
-              // SendAlert maps a Vec of alerter ids
-              Execution::SendAlert(config) => {
-                config.alerters = config
-                  .alerters
-                  .iter()
-                  .map(|alerter| {
-                    all
-                      .alerters
-                      .get(alerter)
-                      .map(|a| a.name.clone())
-                      .unwrap_or_default()
-                  })
-                  .collect();
-              }
-              // No-op variants
-              Execution::None(_)
-              | Execution::BatchRunProcedure(_)
-              | Execution::BatchRunAction(_)
-              | Execution::BatchRunBuild(_)
-              | Execution::BatchDeploy(_)
-              | Execution::BatchDestroyDeployment(_)
-              | Execution::BatchCloneRepo(_)
-              | Execution::BatchPullRepo(_)
-              | Execution::BatchBuildRepo(_)
-              | Execution::BatchDeployStack(_)
-              | Execution::BatchDeployStackIfChanged(_)
-              | Execution::BatchPullStack(_)
-              | Execution::BatchDestroyStack(_)
-              | Execution::ClearRepoCache(_)
-              | Execution::BackupCoreDatabase(_)
-              | Execution::GlobalAutoUpdate(_)
-              | Execution::RotateAllServerKeys(_)
-              | Execution::RotateCoreKeys(_)
-              | Execution::Sleep(_) => {}
-            }
-          };
-        }
-
-        replace_id_with_name!(
-          RunProcedure => procedure, procedures;
-          RunAction => action, actions;
-          RunBuild => build, builds;
-          CancelBuild => build, builds;
-          Deploy => deployment, deployments;
-          PullDeployment => deployment, deployments;
-          StartDeployment => deployment, deployments;
-          RestartDeployment => deployment, deployments;
-          PauseDeployment => deployment, deployments;
-          UnpauseDeployment => deployment, deployments;
-          StopDeployment => deployment, deployments;
-          DestroyDeployment => deployment, deployments;
-          CloneRepo => repo, repos;
-          PullRepo => repo, repos;
-          BuildRepo => repo, repos;
-          CancelRepoBuild => repo, repos;
-          StartContainer => server, servers;
-          RestartContainer => server, servers;
-          PauseContainer => server, servers;
-          UnpauseContainer => server, servers;
-          StopContainer => server, servers;
-          DestroyContainer => server, servers;
-          StartAllContainers => server, servers;
-          RestartAllContainers => server, servers;
-          PauseAllContainers => server, servers;
-          UnpauseAllContainers => server, servers;
-          StopAllContainers => server, servers;
-          PruneContainers => server, servers;
-          DeleteNetwork => server, servers;
-          PruneNetworks => server, servers;
-          DeleteImage => server, servers;
-          PruneImages => server, servers;
-          DeleteVolume => server, servers;
-          PruneVolumes => server, servers;
-          PruneDockerBuilders => server, servers;
-          PruneBuildx => server, servers;
-          PruneSystem => server, servers;
-          RunSync => sync, syncs;
-          CommitSync => sync, syncs;
-          DeployStack => stack, stacks;
-          DeployStackIfChanged => stack, stacks;
-          PullStack => stack, stacks;
-          StartStack => stack, stacks;
-          RestartStack => stack, stacks;
-          PauseStack => stack, stacks;
-          UnpauseStack => stack, stacks;
-          StopStack => stack, stacks;
-          DestroyStack => stack, stacks;
-          RunStackService => stack, stacks;
-          TestAlerter => alerter, alerters;
-          RemoveSwarmNodes => swarm, swarms;
-          RemoveSwarmStacks => swarm, swarms;
-          RemoveSwarmServices => swarm, swarms;
-          CreateSwarmConfig => swarm, swarms;
-          RotateSwarmConfig => swarm, swarms;
-          RemoveSwarmConfigs => swarm, swarms;
-          CreateSwarmSecret => swarm, swarms;
-          RotateSwarmSecret => swarm, swarms;
-          RemoveSwarmSecrets => swarm, swarms;
-        );
-      }
-    }
+    replace_procedure_stage_ids_with_names(&mut original.stages);
     Ok(original.partial_diff(update))
   }
 }

bin/core/src/sync/toml.rs

@@ -2,28 +2,28 @@ use std::collections::HashMap;
 
 use anyhow::Context;
 use indexmap::IndexMap;
-use komodo_client::{
-  api::execute::Execution,
-  entities::{
-    action::Action,
-    alerter::Alerter,
-    build::Build,
-    builder::{Builder, BuilderConfig, PartialBuilderConfig},
-    deployment::{Deployment, DeploymentImage},
-    procedure::Procedure,
-    repo::Repo,
-    resource::Resource,
-    server::Server,
-    stack::Stack,
-    swarm::Swarm,
-    sync::ResourceSync,
-    tag::Tag,
-    toml::ResourceToml,
-  },
+use komodo_client::entities::{
+  action::Action,
+  alerter::Alerter,
+  build::Build,
+  builder::{Builder, BuilderConfig, PartialBuilderConfig},
+  deployment::{Deployment, DeploymentImage},
+  procedure::Procedure,
+  repo::Repo,
+  resource::Resource,
+  server::Server,
+  stack::Stack,
+  swarm::Swarm,
+  sync::ResourceSync,
+  tag::Tag,
+  toml::ResourceToml,
 };
 use partial_derive2::{MaybeNone, PartialDiff};
 
-use crate::{resource::KomodoResource, state::all_resources_cache};
+use crate::{
+  helpers::procedure::replace_procedure_stage_ids_with_names,
+  resource::KomodoResource, state::all_resources_cache,
+};
 
 pub const TOML_PRETTY_OPTIONS: toml_pretty::Options =
   toml_pretty::Options {
@@ -462,506 +462,9 @@ impl ToToml for Builder {
 
 impl ToToml for Procedure {
   fn replace_ids(resource: &mut Resource<Self::Config, Self::Info>) {
-    let all = all_resources_cache().load();
-    for stage in &mut resource.config.stages {
-      for execution in &mut stage.executions {
-        match &mut execution.execution {
-          Execution::RunProcedure(exec) => exec.procedure.clone_from(
-            all
-              .procedures
-              .get(&exec.procedure)
-              .map(|r| &r.name)
-              .unwrap_or(&String::new()),
-          ),
-          Execution::BatchRunProcedure(_exec) => {}
-          Execution::RunAction(exec) => exec.action.clone_from(
-            all
-              .actions
-              .get(&exec.action)
-              .map(|r| &r.name)
-              .unwrap_or(&String::new()),
-          ),
-          Execution::BatchRunAction(_exec) => {}
-          Execution::RunBuild(exec) => exec.build.clone_from(
-            all
-              .builds
-              .get(&exec.build)
-              .map(|r| &r.name)
-              .unwrap_or(&String::new()),
-          ),
-          Execution::BatchRunBuild(_exec) => {}
-          Execution::CancelBuild(exec) => exec.build.clone_from(
-            all
-              .builds
-              .get(&exec.build)
-              .map(|r| &r.name)
-              .unwrap_or(&String::new()),
-          ),
-          Execution::Deploy(exec) => exec.deployment.clone_from(
-            all
-              .deployments
-              .get(&exec.deployment)
-              .map(|r| &r.name)
-              .unwrap_or(&String::new()),
-          ),
-          Execution::BatchDeploy(_exec) => {}
-          Execution::PullDeployment(exec) => {
-            exec.deployment.clone_from(
-              all
-                .deployments
-                .get(&exec.deployment)
-                .map(|r| &r.name)
-                .unwrap_or(&String::new()),
-            )
-          }
-          Execution::StartDeployment(exec) => {
-            exec.deployment.clone_from(
-              all
-                .deployments
-                .get(&exec.deployment)
-                .map(|r| &r.name)
-                .unwrap_or(&String::new()),
-            )
-          }
-          Execution::RestartDeployment(exec) => {
-            exec.deployment.clone_from(
-              all
-                .deployments
-                .get(&exec.deployment)
-                .map(|r| &r.name)
-                .unwrap_or(&String::new()),
-            )
-          }
-          Execution::PauseDeployment(exec) => {
-            exec.deployment.clone_from(
-              all
-                .deployments
-                .get(&exec.deployment)
-                .map(|r| &r.name)
-                .unwrap_or(&String::new()),
-            )
-          }
-          Execution::UnpauseDeployment(exec) => {
-            exec.deployment.clone_from(
-              all
-                .deployments
-                .get(&exec.deployment)
-                .map(|r| &r.name)
-                .unwrap_or(&String::new()),
-            )
-          }
-          Execution::StopDeployment(exec) => {
-            exec.deployment.clone_from(
-              all
-                .deployments
-                .get(&exec.deployment)
-                .map(|r| &r.name)
-                .unwrap_or(&String::new()),
-            )
-          }
-          Execution::DestroyDeployment(exec) => {
-            exec.deployment.clone_from(
-              all
-                .deployments
-                .get(&exec.deployment)
-                .map(|r| &r.name)
-                .unwrap_or(&String::new()),
-            )
-          }
-          Execution::BatchDestroyDeployment(_exec) => {}
-          Execution::CloneRepo(exec) => exec.repo.clone_from(
-            all
-              .repos
-              .get(&exec.repo)
-              .map(|r| &r.name)
-              .unwrap_or(&String::new()),
-          ),
-          Execution::BatchCloneRepo(_exec) => {}
-          Execution::PullRepo(exec) => exec.repo.clone_from(
-            all
-              .repos
-              .get(&exec.repo)
-              .map(|r| &r.name)
-              .unwrap_or(&String::new()),
-          ),
-          Execution::BatchPullRepo(_exec) => {}
-          Execution::BuildRepo(exec) => exec.repo.clone_from(
-            all
-              .repos
-              .get(&exec.repo)
-              .map(|r| &r.name)
-              .unwrap_or(&String::new()),
-          ),
-          Execution::BatchBuildRepo(_exec) => {}
-          Execution::CancelRepoBuild(exec) => exec.repo.clone_from(
-            all
-              .repos
-              .get(&exec.repo)
-              .map(|r| &r.name)
-              .unwrap_or(&String::new()),
-          ),
-          Execution::StartContainer(exec) => exec.server.clone_from(
-            all
-              .servers
-              .get(&exec.server)
-              .map(|r| &r.name)
-              .unwrap_or(&String::new()),
-          ),
-          Execution::RestartContainer(exec) => {
-            exec.server.clone_from(
-              all
-                .servers
-                .get(&exec.server)
-                .map(|r| &r.name)
-                .unwrap_or(&String::new()),
-            )
-          }
-          Execution::PauseContainer(exec) => exec.server.clone_from(
-            all
-              .servers
-              .get(&exec.server)
-              .map(|r| &r.name)
-              .unwrap_or(&String::new()),
-          ),
-          Execution::UnpauseContainer(exec) => {
-            exec.server.clone_from(
-              all
-                .servers
-                .get(&exec.server)
-                .map(|r| &r.name)
-                .unwrap_or(&String::new()),
-            )
-          }
-          Execution::StopContainer(exec) => exec.server.clone_from(
-            all
-              .servers
-              .get(&exec.server)
-              .map(|r| &r.name)
-              .unwrap_or(&String::new()),
-          ),
-          Execution::DestroyContainer(exec) => {
-            exec.server.clone_from(
-              all
-                .servers
-                .get(&exec.server)
-                .map(|r| &r.name)
-                .unwrap_or(&String::new()),
-            )
-          }
-          Execution::StartAllContainers(exec) => {
-            exec.server.clone_from(
-              all
-                .servers
-                .get(&exec.server)
-                .map(|r| &r.name)
-                .unwrap_or(&String::new()),
-            )
-          }
-          Execution::RestartAllContainers(exec) => {
-            exec.server.clone_from(
-              all
-                .servers
-                .get(&exec.server)
-                .map(|r| &r.name)
-                .unwrap_or(&String::new()),
-            )
-          }
-          Execution::PauseAllContainers(exec) => {
-            exec.server.clone_from(
-              all
-                .servers
-                .get(&exec.server)
-                .map(|r| &r.name)
-                .unwrap_or(&String::new()),
-            )
-          }
-          Execution::UnpauseAllContainers(exec) => {
-            exec.server.clone_from(
-              all
-                .servers
-                .get(&exec.server)
-                .map(|r| &r.name)
-                .unwrap_or(&String::new()),
-            )
-          }
-          Execution::StopAllContainers(exec) => {
-            exec.server.clone_from(
-              all
-                .servers
-                .get(&exec.server)
-                .map(|r| &r.name)
-                .unwrap_or(&String::new()),
-            )
-          }
-          Execution::PruneContainers(exec) => exec.server.clone_from(
-            all
-              .servers
-              .get(&exec.server)
-              .map(|r| &r.name)
-              .unwrap_or(&String::new()),
-          ),
-          Execution::DeleteNetwork(exec) => exec.server.clone_from(
-            all
-              .servers
-              .get(&exec.server)
-              .map(|r| &r.name)
-              .unwrap_or(&String::new()),
-          ),
-          Execution::PruneNetworks(exec) => exec.server.clone_from(
-            all
-              .servers
-              .get(&exec.server)
-              .map(|r| &r.name)
-              .unwrap_or(&String::new()),
-          ),
-          Execution::DeleteImage(exec) => exec.server.clone_from(
-            all
-              .servers
-              .get(&exec.server)
-              .map(|r| &r.name)
-              .unwrap_or(&String::new()),
-          ),
-          Execution::PruneImages(exec) => exec.server.clone_from(
-            all
-              .servers
-              .get(&exec.server)
-              .map(|r| &r.name)
-              .unwrap_or(&String::new()),
-          ),
-          Execution::DeleteVolume(exec) => exec.server.clone_from(
-            all
-              .servers
-              .get(&exec.server)
-              .map(|r| &r.name)
-              .unwrap_or(&String::new()),
-          ),
-          Execution::PruneVolumes(exec) => exec.server.clone_from(
-            all
-              .servers
-              .get(&exec.server)
-              .map(|r| &r.name)
-              .unwrap_or(&String::new()),
-          ),
-          Execution::PruneDockerBuilders(exec) => {
-            exec.server.clone_from(
-              all
-                .servers
-                .get(&exec.server)
-                .map(|r| &r.name)
-                .unwrap_or(&String::new()),
-            )
-          }
-          Execution::PruneBuildx(exec) => exec.server.clone_from(
-            all
-              .servers
-              .get(&exec.server)
-              .map(|r| &r.name)
-              .unwrap_or(&String::new()),
-          ),
-          Execution::PruneSystem(exec) => exec.server.clone_from(
-            all
-              .servers
-              .get(&exec.server)
-              .map(|r| &r.name)
-              .unwrap_or(&String::new()),
-          ),
-          Execution::RunSync(exec) => exec.sync.clone_from(
-            all
-              .syncs
-              .get(&exec.sync)
-              .map(|r| &r.name)
-              .unwrap_or(&String::new()),
-          ),
-          Execution::CommitSync(exec) => exec.sync.clone_from(
-            all
-              .syncs
-              .get(&exec.sync)
-              .map(|r| &r.name)
-              .unwrap_or(&String::new()),
-          ),
-          Execution::DeployStack(exec) => exec.stack.clone_from(
-            all
-              .stacks
-              .get(&exec.stack)
-              .map(|r| &r.name)
-              .unwrap_or(&String::new()),
-          ),
-          Execution::BatchDeployStack(_exec) => {}
-          Execution::DeployStackIfChanged(exec) => {
-            exec.stack.clone_from(
-              all
-                .stacks
-                .get(&exec.stack)
-                .map(|r| &r.name)
-                .unwrap_or(&String::new()),
-            )
-          }
-          Execution::BatchDeployStackIfChanged(_exec) => {}
-          Execution::PullStack(exec) => exec.stack.clone_from(
-            all
-              .stacks
-              .get(&exec.stack)
-              .map(|r| &r.name)
-              .unwrap_or(&String::new()),
-          ),
-          Execution::BatchPullStack(_exec) => {}
-          Execution::StartStack(exec) => exec.stack.clone_from(
-            all
-              .stacks
-              .get(&exec.stack)
-              .map(|r| &r.name)
-              .unwrap_or(&String::new()),
-          ),
-          Execution::RestartStack(exec) => exec.stack.clone_from(
-            all
-              .stacks
-              .get(&exec.stack)
-              .map(|r| &r.name)
-              .unwrap_or(&String::new()),
-          ),
-          Execution::RunStackService(exec) => exec.stack.clone_from(
-            all
-              .stacks
-              .get(&exec.stack)
-              .map(|r| &r.name)
-              .unwrap_or(&String::new()),
-          ),
-          Execution::PauseStack(exec) => exec.stack.clone_from(
-            all
-              .stacks
-              .get(&exec.stack)
-              .map(|r| &r.name)
-              .unwrap_or(&String::new()),
-          ),
-          Execution::UnpauseStack(exec) => exec.stack.clone_from(
-            all
-              .stacks
-              .get(&exec.stack)
-              .map(|r| &r.name)
-              .unwrap_or(&String::new()),
-          ),
-          Execution::StopStack(exec) => exec.stack.clone_from(
-            all
-              .stacks
-              .get(&exec.stack)
-              .map(|r| &r.name)
-              .unwrap_or(&String::new()),
-          ),
-          Execution::DestroyStack(exec) => exec.stack.clone_from(
-            all
-              .stacks
-              .get(&exec.stack)
-              .map(|r| &r.name)
-              .unwrap_or(&String::new()),
-          ),
-          Execution::BatchDestroyStack(_exec) => {}
-          Execution::TestAlerter(exec) => exec.alerter.clone_from(
-            all
-              .alerters
-              .get(&exec.alerter)
-              .map(|a| &a.name)
-              .unwrap_or(&String::new()),
-          ),
-          Execution::SendAlert(exec) => {
-            exec.alerters.iter_mut().for_each(|a| {
-              a.clone_from(
-                all
-                  .alerters
-                  .get(a)
-                  .map(|a| &a.name)
-                  .unwrap_or(&String::new()),
-              )
-            })
-          }
-          Execution::RemoveSwarmNodes(exec) => exec.swarm.clone_from(
-            all
-              .swarms
-              .get(&exec.swarm)
-              .map(|a| &a.name)
-              .unwrap_or(&String::new()),
-          ),
-          Execution::RemoveSwarmStacks(exec) => {
-            exec.swarm.clone_from(
-              all
-                .swarms
-                .get(&exec.swarm)
-                .map(|a| &a.name)
-                .unwrap_or(&String::new()),
-            )
-          }
-          Execution::RemoveSwarmServices(exec) => {
-            exec.swarm.clone_from(
-              all
-                .swarms
-                .get(&exec.swarm)
-                .map(|a| &a.name)
-                .unwrap_or(&String::new()),
-            )
-          }
-          Execution::CreateSwarmConfig(exec) => {
-            exec.swarm.clone_from(
-              all
-                .swarms
-                .get(&exec.swarm)
-                .map(|a| &a.name)
-                .unwrap_or(&String::new()),
-            )
-          }
-          Execution::RotateSwarmConfig(exec) => {
-            exec.swarm.clone_from(
-              all
-                .swarms
-                .get(&exec.swarm)
-                .map(|a| &a.name)
-                .unwrap_or(&String::new()),
-            )
-          }
-          Execution::RemoveSwarmConfigs(exec) => {
-            exec.swarm.clone_from(
-              all
-                .swarms
-                .get(&exec.swarm)
-                .map(|a| &a.name)
-                .unwrap_or(&String::new()),
-            )
-          }
-          Execution::CreateSwarmSecret(exec) => {
-            exec.swarm.clone_from(
-              all
-                .swarms
-                .get(&exec.swarm)
-                .map(|a| &a.name)
-                .unwrap_or(&String::new()),
-            )
-          }
-          Execution::RotateSwarmSecret(exec) => {
-            exec.swarm.clone_from(
-              all
-                .swarms
-                .get(&exec.swarm)
-                .map(|a| &a.name)
-                .unwrap_or(&String::new()),
-            )
-          }
-          Execution::RemoveSwarmSecrets(exec) => {
-            exec.swarm.clone_from(
-              all
-                .swarms
-                .get(&exec.swarm)
-                .map(|a| &a.name)
-                .unwrap_or(&String::new()),
-            )
-          }
-          Execution::None(_)
-          | Execution::Sleep(_)
-          | Execution::ClearRepoCache(_)
-          | Execution::BackupCoreDatabase(_)
-          | Execution::GlobalAutoUpdate(_)
-          | Execution::RotateAllServerKeys(_)
-          | Execution::RotateCoreKeys(_) => {}
-        }
-      }
-    }
+    replace_procedure_stage_ids_with_names(
+      &mut resource.config.stages,
+    );
   }
 
   fn push_to_toml_string(

bin/periphery/aio.Dockerfile

@@ -1,6 +1,6 @@
 ## All in one, multi stage compile + runtime Docker build for your architecture.
 
-FROM rust:1.94.0-trixie AS builder
+FROM rust:1.94.1-trixie AS builder
 RUN cargo install cargo-strip
 
 WORKDIR /builder

bin/periphery/src/api/swarm/mod.rs

@@ -99,13 +99,9 @@ impl Resolve<crate::api::Args> for UpdateSwarmNode {
     }
 
     if let Some(label_add) = self.label_add {
-      for (key, value) in label_add {
+      for key_value in label_add {
         command += " --label-add ";
-        command += &key;
-        if let Some(value) = value {
-          command += "=";
-          command += &value;
-        }
+        command += &key_value;
       }
     }
 

bin/periphery/src/api/swarm/stack.rs

@@ -11,7 +11,10 @@ use komodo_client::{
   entities::{
     all_logs_success,
     docker::stack::SwarmStack,
-    stack::{ComposeFile, ComposeService, StackServiceNames},
+    stack::{
+      AdditionalEnvFile, ComposeFile, ComposeService,
+      StackServiceNames,
+    },
     update::Log,
   },
   parsers::parse_multiline_command,
@@ -139,7 +142,7 @@ impl Resolve<crate::api::Args> for DeploySwarmStack {
     replacers.extend(interpolator.secret_replacers);
 
     // Env files are not supported by docker stack deploy so are ignored.
-    let (run_directory, _) = match write_stack(
+    let (run_directory, env_file_path) = match write_stack(
       &stack,
       repo.as_ref(),
       git_token,
@@ -218,10 +221,16 @@ impl Resolve<crate::api::Args> for DeploySwarmStack {
         &stack.config.compose_cmd_wrapper_include
       };
 
+    let env_file_args = env_file_args(
+      env_file_path,
+      &stack.config.additional_env_files,
+    )?;
+
     // Uses 'docker stack config' command to extract services (including image)
     // after performing interpolation
     {
-      let command = format!("docker stack config -c {file_args}",);
+      let command =
+        format!("{env_file_args}docker stack config -c {file_args}",);
       let (command, wrapped) = match maybe_wrap_command(
         command,
         &compose_cmd_wrapper,
@@ -234,7 +243,7 @@ impl Resolve<crate::api::Args> for DeploySwarmStack {
           return Ok(res);
         }
       };
-      let mode = if wrapped {
+      let mode = if wrapped || !env_file_args.is_empty() {
         KomodoCommandMode::Shell
       } else {
         KomodoCommandMode::Standard
@@ -288,8 +297,9 @@ impl Resolve<crate::api::Args> for DeploySwarmStack {
     }
 
     // Run stack deploy
-    let mut command =
-      format!("docker stack deploy --detach=false -c {file_args}");
+    let mut command = format!(
+      "{env_file_args}docker stack deploy --detach=true -c {file_args}"
+    );
     if use_with_registry_auth {
       command += " --with-registry-auth";
     }
@@ -349,6 +359,43 @@ impl Resolve<crate::api::Args> for DeploySwarmStack {
   }
 }
 
+/// Prefix docker stack deploy command.
+///
+/// Produces either
+///     - Empty string (no env files defined)
+///     - 'set -a && . .env1 && . .env2 && '
+fn env_file_args(
+  env_file_path: Option<&str>,
+  additional_env_files: &[AdditionalEnvFile],
+) -> anyhow::Result<String> {
+  let mut res = String::new();
+
+  // Add additional env files (except komodo's own, which comes last)
+  for file in additional_env_files
+    .iter()
+    .filter(|f| env_file_path != Some(f.path.as_str()))
+  {
+    let path = &file.path;
+    write!(res, ". {path} && ").with_context(|| {
+      format!("Failed to write env source arg for {path}")
+    })?;
+  }
+
+  // Add komodo's env file last for highest priority
+  if let Some(file) = env_file_path {
+    write!(res, ". {file} && ").with_context(|| {
+      format!("Failed to write env source arg for {file}")
+    })?;
+  }
+
+  if !res.is_empty() {
+    // Add 'set -a' so vars are auto exported to child 'docker stack' process.
+    Ok(format!("set -a && {res}"))
+  } else {
+    Ok(res)
+  }
+}
+
 #[instrument("RemoveStack", skip(res))]
 async fn remove_stack(
   stack: &str,

bin/periphery/src/config.rs

@@ -144,6 +144,9 @@ pub fn periphery_config() -> &'static PeripheryConfig {
         ansi: env
           .periphery_logging_ansi
           .unwrap_or(config.logging.ansi),
+        timestamps: env
+          .periphery_logging_timestamps
+          .unwrap_or(config.logging.timestamps),
         otlp_endpoint: env
           .periphery_logging_otlp_endpoint
           .unwrap_or(config.logging.otlp_endpoint),

bin/periphery/src/docker/node.rs

@@ -51,24 +51,24 @@ impl DockerClient {
 fn convert_node_list_item(
   node: bollard::models::Node,
 ) -> SwarmNodeListItem {
-  let (name, role, availability) = node
+  let (name, role, availability, labels) = node
     .spec
     .map(|spec| {
       (
         spec.name,
         spec.role.map(convert_role),
         spec.availability.map(convert_availability),
+        spec.labels,
       )
     })
     .unwrap_or_default();
   SwarmNodeListItem {
     id: node.id,
     name,
+    hostname: node.description.and_then(|d| d.hostname),
     role,
     availability,
-    hostname: node
-      .description
-      .and_then(|description| description.hostname),
+    labels,
     state: node
       .status
       .and_then(|status| status.state.map(convert_state)),

client/core/rs/src/api/execute/mod.rs

@@ -183,6 +183,7 @@ pub enum Execution {
 
   // SWARM
   RemoveSwarmNodes(RemoveSwarmNodes),
+  UpdateSwarmNode(UpdateSwarmNode),
   RemoveSwarmStacks(RemoveSwarmStacks),
   RemoveSwarmServices(RemoveSwarmServices),
   CreateSwarmConfig(CreateSwarmConfig),

client/core/rs/src/api/execute/openapi.rs

@@ -7,6 +7,7 @@ use crate::api::execute;
   paths(
     // swarm
     execute::remove_swarm_nodes,
+    execute::update_swarm_node,
     execute::remove_swarm_stacks,
     execute::remove_swarm_services,
     execute::create_swarm_config,

client/core/rs/src/api/execute/swarm.rs

@@ -4,7 +4,11 @@ use serde::{Deserialize, Serialize};
 use typeshare::typeshare;
 
 use crate::{
-  api::execute::KomodoExecuteRequest, entities::update::Update,
+  api::execute::KomodoExecuteRequest,
+  entities::{
+    docker::node::{NodeSpecAvailabilityEnum, NodeSpecRoleEnum},
+    update::Update,
+  },
 };
 
 // ========
@@ -45,6 +49,48 @@ pub struct RemoveSwarmNodes {
   pub force: bool,
 }
 
+#[cfg(feature = "utoipa")]
+#[utoipa::path(
+  post,
+  path = "/UpdateSwarmNode",
+  description = "Update a swarm node's availability, labels, and role.",
+  request_body(content = UpdateSwarmNode),
+  responses(
+    (status = 200, description = "The update", body = Update),
+  ),
+)]
+pub fn update_swarm_node() {}
+
+/// `docker node update [OPTIONS] NODE`
+///
+/// https://docs.docker.com/reference/cli/docker/node/update/
+#[typeshare]
+#[derive(
+  Serialize, Deserialize, Debug, Clone, PartialEq, Resolve, Parser,
+)]
+#[cfg_attr(feature = "utoipa", derive(utoipa::ToSchema))]
+#[empty_traits(KomodoExecuteRequest)]
+#[response(Update)]
+#[error(mogh_error::Error)]
+pub struct UpdateSwarmNode {
+  /// Name or id
+  pub swarm: String,
+  /// Node hostname or id
+  pub node: String,
+  /// Update the node's availability: 'active', 'pause', or 'drain'
+  #[arg(long, short = 'a')]
+  pub availability: Option<NodeSpecAvailabilityEnum>,
+  /// Add labels to node (`key=value`).
+  #[arg(long, short = 'l')]
+  pub label_add: Option<Vec<String>>,
+  /// Add labels to node (`key=value`). (alias: `lr`)
+  #[arg(long, alias = "lr")]
+  pub label_rm: Option<Vec<String>>,
+  /// Update the node's role: 'worker' or 'manager'
+  #[arg(long, short = 'r')]
+  pub role: Option<NodeSpecRoleEnum>,
+}
+
 // =========
 // = Stack =
 // =========

client/core/rs/src/entities/alert.rs

@@ -95,7 +95,7 @@ pub enum AlertData {
     /// The name of the swarm
     name: String,
     /// The error data
-    err: Option<String>,
+    err: Option<_Serror>,
   },
 
   /// A server could not be reached.

client/core/rs/src/entities/config/cli/mod.rs

@@ -96,6 +96,8 @@ pub struct Env {
   pub komodo_cli_logging_pretty: Option<bool>,
   /// Override `logging.ansi`
   pub komodo_cli_logging_ansi: Option<bool>,
+  /// Override `logging.timestamps`
+  pub komodo_cli_logging_timestamps: Option<bool>,
   /// Override `logging.otlp_endpoint`
   pub komodo_cli_logging_otlp_endpoint: Option<String>,
   /// Override `logging.opentelemetry_service_name`

client/core/rs/src/entities/config/core.rs

@@ -116,27 +116,6 @@ pub struct Env {
   /// Override `webhook_base_url`
   pub komodo_webhook_base_url: Option<String>,
 
-  /// Override `logging.level`
-  pub komodo_logging_level: Option<LogLevel>,
-  /// Override `logging.stdio`
-  pub komodo_logging_stdio: Option<StdioLogMode>,
-  /// Override `logging.pretty`
-  pub komodo_logging_pretty: Option<bool>,
-  /// Override `logging.location`
-  pub komodo_logging_location: Option<bool>,
-  /// Override `logging.ansi`
-  pub komodo_logging_ansi: Option<bool>,
-  /// Override `logging.otlp_endpoint`
-  pub komodo_logging_otlp_endpoint: Option<String>,
-  /// Override `logging.opentelemetry_service_name`
-  pub komodo_logging_opentelemetry_service_name: Option<String>,
-  /// Override `logging.opentelemetry_scope_name`
-  pub komodo_logging_opentelemetry_scope_name: Option<String>,
-  /// Override `pretty_startup_config`
-  pub komodo_pretty_startup_config: Option<bool>,
-  /// Override `unsafe_unsanitized_startup_config`
-  pub komodo_unsafe_unsanitized_startup_config: Option<bool>,
-
   /// Override `transparent_mode`
   pub komodo_transparent_mode: Option<bool>,
   /// Override `ui_write_disabled`
@@ -265,6 +244,29 @@ pub struct Env {
   /// Override `aws.secret_access_key` with file
   pub komodo_aws_secret_access_key_file: Option<PathBuf>,
 
+  /// Override `logging.level`
+  pub komodo_logging_level: Option<LogLevel>,
+  /// Override `logging.stdio`
+  pub komodo_logging_stdio: Option<StdioLogMode>,
+  /// Override `logging.pretty`
+  pub komodo_logging_pretty: Option<bool>,
+  /// Override `logging.location`
+  pub komodo_logging_location: Option<bool>,
+  /// Override `logging.ansi`
+  pub komodo_logging_ansi: Option<bool>,
+  /// Override `logging.timestamps`
+  pub komodo_logging_timestamps: Option<bool>,
+  /// Override `logging.otlp_endpoint`
+  pub komodo_logging_otlp_endpoint: Option<String>,
+  /// Override `logging.opentelemetry_service_name`
+  pub komodo_logging_opentelemetry_service_name: Option<String>,
+  /// Override `logging.opentelemetry_scope_name`
+  pub komodo_logging_opentelemetry_scope_name: Option<String>,
+  /// Override `pretty_startup_config`
+  pub komodo_pretty_startup_config: Option<bool>,
+  /// Override `unsafe_unsanitized_startup_config`
+  pub komodo_unsafe_unsanitized_startup_config: Option<bool>,
+
   /// Override `internet_interface`
   pub komodo_internet_interface: Option<String>,
 

client/core/rs/src/entities/config/periphery.rs

@@ -199,6 +199,8 @@ pub struct Env {
   pub periphery_logging_location: Option<bool>,
   /// Override `logging.ansi`
   pub periphery_logging_ansi: Option<bool>,
+  /// Override `logging.timestamps`
+  pub periphery_logging_timestamps: Option<bool>,
   /// Override `logging.otlp_endpoint`
   pub periphery_logging_otlp_endpoint: Option<String>,
   /// Override `logging.opentelemetry_service_name`

client/core/rs/src/entities/docker/node.rs

@@ -2,6 +2,7 @@
 
 use std::collections::HashMap;
 
+use clap::ValueEnum;
 use serde::{Deserialize, Serialize};
 use strum::AsRefStr;
 use typeshare::typeshare;
@@ -22,6 +23,7 @@ pub struct SwarmNodeListItem {
   #[serde(rename = "Name")]
   pub name: Option<String>,
 
+  /// Node hostname, more commonly used than Name
   #[serde(rename = "Hostname")]
   pub hostname: Option<String>,
 
@@ -33,6 +35,10 @@ pub struct SwarmNodeListItem {
   #[serde(rename = "Availability")]
   pub availability: Option<NodeSpecAvailabilityEnum>,
 
+  /// Labels of the node
+  #[serde(rename = "Labels")]
+  pub labels: Option<HashMap<String, String>>,
+
   /// State of the node
   #[serde(rename = "State")]
   pub state: Option<NodeState>,
@@ -120,6 +126,7 @@ pub struct NodeSpec {
   Serialize,
   Deserialize,
   AsRefStr,
+  ValueEnum,
 )]
 #[cfg_attr(feature = "utoipa", derive(utoipa::ToSchema))]
 pub enum NodeSpecRoleEnum {
@@ -145,6 +152,7 @@ pub enum NodeSpecRoleEnum {
   Serialize,
   Deserialize,
   AsRefStr,
+  ValueEnum,
 )]
 #[cfg_attr(feature = "utoipa", derive(utoipa::ToSchema))]
 pub enum NodeSpecAvailabilityEnum {

client/core/rs/src/entities/logger.rs

@@ -17,14 +17,18 @@ pub struct LogConfig {
   pub pretty: bool,
 
   /// Including information about the log location (ie the function which produced the log).
-  /// Tracing refers to this as the 'target'.
+  /// Tracing refers to this as the 'target'. (default: false)
   #[serde(default = "default_location")]
   pub location: bool,
 
-  /// Logs use ansi colors for readability.
+  /// Logs use ansi colors for readability. (default: true)
   #[serde(default = "default_ansi")]
   pub ansi: bool,
 
+  /// Logs include timestamps. (default: true)
+  #[serde(default = "default_timestamps")]
+  pub timestamps: bool,
+
   /// Enable opentelemetry exporting
   #[serde(default)]
   pub otlp_endpoint: String,
@@ -52,6 +56,10 @@ fn default_ansi() -> bool {
   true
 }
 
+fn default_timestamps() -> bool {
+  true
+}
+
 impl Default for LogConfig {
   fn default() -> Self {
     Self {
@@ -60,6 +68,7 @@ impl Default for LogConfig {
       pretty: Default::default(),
       location: default_location(),
       ansi: default_ansi(),
+      timestamps: default_timestamps(),
       otlp_endpoint: Default::default(),
       opentelemetry_service_name: default_opentelemetry_service_name(
       ),
@@ -166,15 +175,18 @@ impl mogh_logger::LogConfig for &LogConfig {
     &TARGETS
   }
 
-  fn ansi(&self) -> bool {
-    self.ansi
-  }
   fn level(&self) -> tracing::Level {
     self.level.into()
   }
   fn location(&self) -> bool {
     self.location
   }
+  fn ansi(&self) -> bool {
+    self.ansi
+  }
+  fn timestamps(&self) -> bool {
+    self.timestamps
+  }
   fn opentelemetry_scope_name(&self) -> String {
     self.opentelemetry_scope_name.clone()
   }

client/core/rs/src/entities/mod.rs

@@ -1167,6 +1167,7 @@ pub enum Operation {
   RenameSwarm,
   DeleteSwarm,
   RemoveSwarmNodes,
+  UpdateSwarmNode,
   RemoveSwarmStacks,
   RemoveSwarmServices,
   CreateSwarmConfig,

client/core/rs/src/entities/server.rs

@@ -10,7 +10,7 @@ use crate::{
   deserializers::{
     option_string_list_deserializer, string_list_deserializer,
   },
-  entities::{MaintenanceWindow, Timelength},
+  entities::{_Serror, MaintenanceWindow, Timelength},
 };
 
 use super::{
@@ -37,6 +37,9 @@ pub type ServerListItem = ResourceListItem<ServerListItemInfo>;
 pub struct ServerListItemInfo {
   /// The server's state.
   pub state: ServerState,
+  /// If there is an error reaching
+  /// the server, message will be given here.
+  pub err: Option<_Serror>,
   /// Region of the server.
   pub region: String,
   /// Address of the server, or null if empty.

client/core/rs/src/entities/swarm.rs

@@ -10,7 +10,7 @@ use crate::{
   deserializers::{
     option_string_list_deserializer, string_list_deserializer,
   },
-  entities::MaintenanceWindow,
+  entities::{_Serror, MaintenanceWindow},
 };
 
 use super::resource::{Resource, ResourceListItem, ResourceQuery};
@@ -28,7 +28,7 @@ pub struct SwarmListItemInfo {
   pub state: SwarmState,
   /// If there is an error reaching
   /// Swarm, message will be given here.
-  pub err: Option<String>,
+  pub err: Option<_Serror>,
 }
 
 #[typeshare]

client/core/ts/package.json

@@ -1,6 +1,6 @@
 {
   "name": "komodo_client",
-  "version": "2.0.0",
+  "version": "2.1.2",
   "description": "Komodo client package",
   "homepage": "https://komo.do",
   "main": "dist/lib.js",
@@ -16,7 +16,7 @@
     "mogh_auth_client": "^1.2.1"
   },
   "devDependencies": {
-    "typescript": "^5.6.3"
+    "typescript": "^6.0.2"
   },
   "packageManager": "yarn@1.22.22+sha512.a6b2f7906b721bba3d67d4aff083df04dad64c399707841b7acf00f6b133b7ac24255f2652fa22ae3534329dc6180534e98d17432037ff6fd140556e2bb3137e"
 }

client/core/ts/src/responses.ts

@@ -433,6 +433,7 @@ export type ExecuteResponses = {
 
   // ==== SWARM ====
   RemoveSwarmNodes: Types.Update;
+  UpdateSwarmNode: Types.Update;
   RemoveSwarmStacks: Types.Update;
   RemoveSwarmServices: Types.Update;
   CreateSwarmConfig: Types.Update;

client/core/ts/src/types.ts

@@ -378,6 +378,7 @@ export enum Operation {
 	RenameSwarm = "RenameSwarm",
 	DeleteSwarm = "DeleteSwarm",
 	RemoveSwarmNodes = "RemoveSwarmNodes",
+	UpdateSwarmNode = "UpdateSwarmNode",
 	RemoveSwarmStacks = "RemoveSwarmStacks",
 	RemoveSwarmServices = "RemoveSwarmServices",
 	CreateSwarmConfig = "CreateSwarmConfig",
@@ -927,6 +928,7 @@ export type Execution =
 	| { type: "PruneBuildx", params: PruneBuildx }
 	| { type: "PruneSystem", params: PruneSystem }
 	| { type: "RemoveSwarmNodes", params: RemoveSwarmNodes }
+	| { type: "UpdateSwarmNode", params: UpdateSwarmNode }
 	| { type: "RemoveSwarmStacks", params: RemoveSwarmStacks }
 	| { type: "RemoveSwarmServices", params: RemoveSwarmServices }
 	| { type: "CreateSwarmConfig", params: CreateSwarmConfig }
@@ -1561,7 +1563,7 @@ export type AlertData =
 	/** The name of the swarm */
 	name: string;
 	/** The error data */
-	err?: string;
+	err?: _Serror;
 }}
 	/** A server could not be reached. */
 	| { type: "ServerUnreachable", data: {
@@ -5328,9 +5330,21 @@ export enum ServerState {
 	Disabled = "Disabled",
 }
 
+export interface __Serror {
+	error: string;
+	trace: string[];
+}
+
+export type _Serror = __Serror;
+
 export interface ServerListItemInfo {
 	/** The server's state. */
 	state: ServerState;
+	/**
+	 * If there is an error reaching
+	 * the server, message will be given here.
+	 */
+	err?: _Serror;
 	/** Region of the server. */
 	region: string;
 	/** Address of the server, or null if empty. */
@@ -5492,11 +5506,14 @@ export interface SwarmNodeListItem {
 	ID?: string;
 	/** Name for the node. */
 	Name?: string;
+	/** Node hostname, more commonly used than Name */
 	Hostname?: string;
 	/** Role of the node. */
 	Role?: NodeSpecRoleEnum;
 	/** Availability of the node. */
 	Availability?: NodeSpecAvailabilityEnum;
+	/** Labels of the node */
+	Labels?: Record<string, string>;
 	/** State of the node */
 	State?: NodeState;
 	/** For manager nodes, include the manager addr. */
@@ -5569,7 +5586,7 @@ export interface SwarmListItemInfo {
 	 * If there is an error reaching
 	 * Swarm, message will be given here.
 	 */
-	err?: string;
+	err?: _Serror;
 }
 
 export type SwarmListItem = ResourceListItem<SwarmListItemInfo>;
@@ -5778,13 +5795,6 @@ export type _PartialTag = Partial<Tag>;
 
 export type _PartialUrlBuilderConfig = Partial<UrlBuilderConfig>;
 
-export interface __Serror {
-	error: string;
-	trace: string[];
-}
-
-export type _Serror = __Serror;
-
 /** **Admin only.** Add a user to a user group. Response: [UserGroup] */
 export interface AddUserToUserGroup {
 	/** The name or id of UserGroup that user should be added to. */
@@ -10477,6 +10487,26 @@ export interface UpdateSwarm {
 	config: _PartialSwarmConfig;
 }
 
+/**
+ * `docker node update [OPTIONS] NODE`
+ * 
+ * https://docs.docker.com/reference/cli/docker/node/update/
+ */
+export interface UpdateSwarmNode {
+	/** Name or id */
+	swarm: string;
+	/** Node hostname or id */
+	node: string;
+	/** Update the node's availability: 'active', 'pause', or 'drain' */
+	availability?: NodeSpecAvailabilityEnum;
+	/** Add labels to node (`key=value`). */
+	label_add?: string[];
+	/** Add labels to node (`key=value`). (alias: `lr`) */
+	label_rm?: string[];
+	/** Update the node's role: 'worker' or 'manager' */
+	role?: NodeSpecRoleEnum;
+}
+
 /** Update color for tag. Response: [Tag]. */
 export interface UpdateTagColor {
 	/** The name or id of the tag to update. */
@@ -10666,6 +10696,7 @@ export type ExecuteRequest =
 	| { type: "PruneBuildx", params: PruneBuildx }
 	| { type: "PruneSystem", params: PruneSystem }
 	| { type: "RemoveSwarmNodes", params: RemoveSwarmNodes }
+	| { type: "UpdateSwarmNode", params: UpdateSwarmNode }
 	| { type: "RemoveSwarmStacks", params: RemoveSwarmStacks }
 	| { type: "RemoveSwarmServices", params: RemoveSwarmServices }
 	| { type: "CreateSwarmConfig", params: CreateSwarmConfig }

client/core/ts/tsconfig.json

@@ -3,10 +3,11 @@
 		"strict": true,
 		"target": "ESNext",
 		"module": "ESNext",
-		"moduleResolution": "node",
+		"moduleResolution": "bundler",
 		"allowSyntheticDefaultImports": true,
 		"esModuleInterop": true,
 		"isolatedModules": true,
+		"rootDir": "src/",
 		"outDir": "dist",
 		"declaration": true
 	}

client/core/ts/yarn.lock

@@ -14,7 +14,7 @@ mogh_auth_client@^1.2.1:
   dependencies:
     jwt-decode "^4.0.0"
 
-typescript@^5.6.3:
-  version "5.6.3"
-  resolved "https://registry.yarnpkg.com/typescript/-/typescript-5.6.3.tgz#5f3449e31c9d94febb17de03cc081dd56d81db5b"
-  integrity sha512-hjcS1mhfuyi4WW8IWtjP7brDrG2cuDZukyrYrSauoXGNgx0S7zceP07adYkJycEr56BOUTNPzbInooiN3fn1qw==
+typescript@^6.0.2:
+  version "6.0.2"
+  resolved "https://registry.yarnpkg.com/typescript/-/typescript-6.0.2.tgz#0b1bfb15f68c64b97032f3d78abbf98bdbba501f"
+  integrity sha512-bGdAIrZ0wiGDo5l8c++HWtbaNCWTS4UTv7RaTH/ThVIgjkveJt83m74bBHMJkuCbslY8ixgLBVZJIOiQlQTjfQ==

client/periphery/rs/src/api/swarm.rs

@@ -1,5 +1,3 @@
-use std::collections::HashMap;
-
 use komodo_client::entities::{
   SearchCombinator,
   deployment::Deployment,
@@ -63,11 +61,12 @@ pub struct RemoveSwarmNodes {
 #[response(Log)]
 #[error(anyhow::Error)]
 pub struct UpdateSwarmNode {
+  /// Node hostname or id
   pub node: String,
+  /// Update the node's availability: 'active', 'pause', or 'drain'
   pub availability: Option<NodeSpecAvailabilityEnum>,
   /// Add metadata to a swarm node using node labels (`key=value`).
-  /// You can specify a node label as a key with an empty value.
-  pub label_add: Option<HashMap<String, Option<String>>>,
+  pub label_add: Option<Vec<String>>,
   /// Remove labels by the label key.
   pub label_rm: Option<Vec<String>>,
   /// Update the node role (`worker`, `manager`)

docsite/docs/ecosystem/cli.mdx

@@ -2,8 +2,9 @@
 
 The Komodo CLI, `km`, can be used to:
   - Quickly **run executions** and update **resources** and **variables**.
-  - **Reset user passwords** and elevate users to **Super Admin**. 
+  - **Reset user passwords** and elevate users to **Super Admin**.
   - Perform Database **backup**, **restore**, and **copy**.
+  - **Connect to server / container shell** via Komodo Terminals.
 
 The Komodo Core image comes packaged with the Komodo CLI,
 and is available for usage inside running container with `docker exec -it komodo-core km ...`.
@@ -21,6 +22,61 @@ This way, it inherits the Core database config in order to easily perform backup
   - `km x commit my-sync`
   - `km set user mbecks super-admin true`
   - `km set user mbecks password "temp-password"`
+  - `km ssh my-server`
+  - `km connect my-server bash -n my-session`
+  - `km exec my-container bash`
+  - `km attach my-container -s my-server`
+
+### Terminals
+
+All terminal commands share the same disconnect shortcut: press **Alt+Q** to end the session.
+The session itself stays running and will keep the history if you connect again, or connect from a
+different device.
+
+#### SSH / Connect
+
+The `km ssh` command (alias for `km connect`) opens an interactive shell on a server managed by Komodo Periphery. Analogous to `ssh`.
+
+```
+km ssh <server> [command] [options]
+km connect <server> [command] [options]
+```
+
+| Argument | Description |
+|---|---|
+| `server` | Server name (required) |
+| `command` | Shell command to start, e.g. `bash` (optional, defaults to Periphery default) |
+| `-n`, `--name` | Terminal session name (default: `ssh`) |
+| `-r`, `--recreate` | Force a fresh terminal, replacing any existing session with the same name |
+
+#### Exec
+
+The `km exec` command opens an interactive shell inside a running container. Analogous to `docker exec`.
+
+```
+km exec <container> <shell> [options]
+```
+
+| Argument | Description |
+|---|---|
+| `container` | Container name (required) |
+| `shell` | Shell to use, e.g. `bash` or `sh` (required) |
+| `-s`, `--server` | Server name — required if multiple servers have a container with the same name |
+| `-r`, `--recreate` | Force a fresh terminal, replacing any existing session |
+
+#### Attach
+
+The `km attach` command attaches to a running container's main process stdio. Analogous to `docker attach`.
+
+```
+km attach <container> [options]
+```
+
+| Argument | Description |
+|---|---|
+| `container` | Container name (required) |
+| `-s`, `--server` | Server name — required if multiple servers have a container with the same name |
+| `-r`, `--recreate` | Force a fresh terminal, replacing any existing session |
 
 ### Install
 
@@ -53,7 +109,7 @@ brew tap moghtech/komodo && \
 
 You can alias a docker run command:
 ```bash
-alias km='docker run --rm -v $HOME/.config/komodo:/config ghcr.io/moghtech/komodo-cli km'
+alias km='docker run --rm -v $HOME/.config/komodo:/config ghcr.io/moghtech/komodo-cli:2 km'
 km config
 ```
 

docsite/docs/releases/v2.0.0.md

@@ -9,11 +9,18 @@ Komodo v2 is a major release with significant architectural changes and new feat
 - **PKI authentication**: Core and Periphery now authenticate with auto-generated key pairs and automatic rotation. Passkeys are deprecated.
 - **Onboarding keys**: streamlined server onboarding with reusable keys.
 - **Improved terminals**: Terminals dashboard, `km ssh`, and improved Action scripting.
+- **Improved auto updates**: Uses a better system to handle auto updates based on image digest.
 - **New UI**: Improved look with higher contrast and better UI primitives.
 - **Passkey / TOTP 2FA**: Built in two factor authentication for username / password login.
 - **Multi-login Linking**: Users can now link multiple login providers (Local, OIDC, Github, etc) to their account.
 - **Full OpenAPI documentation**: Interactive API docs [now available](https://demo.komo.do/docs).
 
+### 🚨 Breaking
+
+- **Debian Bullseye EOL**: Bullseye stopped recieving security updates in 2024 and is slated to be EOL. The binaries are now built with Debian Bookworm which uses openssl v3,
+and Periphery may have compatibility issues running on systems running **openssl v1** including **Debian Bullseye / Ubuntu 20.04 and before**. You can still compile Periphery
+yourself for these systems if needed (or use containerized periphery), but the official distribution will use the more secure and up-to-date option.
+
 ## Upgrading to Komodo v2
 
 Komodo v2 introduces a new connection and authentication method between Komodo Core and the Periphery agents running on your Servers. Find more information on [Komodo v2 here](../releases/v2.0.0).
@@ -22,7 +29,8 @@ It is largely backward compatible with Komodo v1 configuration, and users can up
 
 :::note
 Starting with v2, Komodo will not publish images with the `latest` tag in favor of Semver (`2`, `2.0`, `2.0.0`).
-This prevents unintented major version upgrades when using auto updaters.
+This prevents unintented major version upgrades when using auto updaters. Use `:2` to stay up to date
+with all future Komodo v2 releases.
 :::
 
 ### 1. Upgrade Core and Periphery to v2
@@ -49,22 +57,37 @@ volumes:
 ```
 
 :::warning
-Ensure the Komodo Core service includes `init: true`, as shown above.
-Failing to do so may cause a build up of zombie processes, and it wasn't
-included in earlier releases example compose file.
+Ensure the Komodo Core service **and** all Periphery services / containers include `init: true`,
+as shown above. Failing to do so may cause a build up of zombie processes,
+and it wasn't included in earlier releases example compose file.
 :::
 
-#### Upgrade Periphery
+#### Upgrade Periphery: Container
 
-If you are running Komodo Periphery in a container, you also need the `:2` tag for the image, and the **keys will default to being stored in `$PERIPHERY_ROOT_DIRECTORY/keys` which should already be mounted**.
-Still, if running Periphery in container, **ensure you see the private / public keys in the key directory**, you may need to add a mount if you customized mount directories.
+If you are running Komodo Periphery in a container, you also need the `:2` tag for the image,
+and the keys will default to being stored in `/config/keys`.
+
+```yaml
+services:
+  periphery:
+    image: ghcr.io/moghtech/komodo-periphery:2
+    init: true # This should be added regardless of version
+    volumes:
+      - keys:/config/keys
+      - (...unchanged)
+    (...unchanged)
+
+volumes:
+  keys:
+  (...unchanged)
+```
+
+#### Upgrade Periphery: Systemd
 
 Systemd Periphery users just need to update their Periphery binary version. The keys will be stored in your `root_directory` (default: `/etc/komodo/keys`).
 
 After getting both Core and Periphery running, everything should already work correctly at this point.
 
-
-
 ### 2a. Move to public key authentication
 
 If you want to **reverse the agent connection, [skip this step and go to 2b](#2b-reversing-the-agent-connection)**.
@@ -119,7 +142,7 @@ onboarding_key = "<YOUR_ONBOARDING_KEY>"
 
 Upon connecting, the privileged onboarding key will allow the existing server's expected public key
 to be updated, allowing the Periphery agent to connect.
-**In general when onboarding _new_ servers, privilidged mode is not needed.**
+**In general when onboarding _new_ servers, privileged mode is not needed.**
 
 ### 3. Fix any `komodo.execute_terminal` in Actions
 

docsite/docs/terminals.md

@@ -28,6 +28,16 @@ You can create multiple named terminal sessions on the same resource. Each sessi
 
 Each terminal maintains a rolling 1 MiB output buffer. When you reconnect to an existing session, the history is replayed so you can see previous output.
 
+## CLI
+
+Terminal sessions can also be accessed from the command line using the [Komodo CLI](./ecosystem/cli.mdx#terminals).
+
+- `km ssh <server>` — open a shell on a server
+- `km exec <container> <shell>` — exec into a container
+- `km attach <container>` — attach to a container's main process
+
+Press **Alt+Q** to disconnect from any CLI terminal session while the session itself stays running.
+
 ## Execute Terminal
 
 The `execute_terminal` API method allows you to run a command on a terminal and stream the output back over HTTP. This is useful for:

docsite/static/CNAME

@@ -0,0 +1 @@
+komo.do

example/alerter/Dockerfile

@@ -1,4 +1,4 @@
-FROM rust:1.94.0-bullseye as builder
+FROM rust:1.94.1-bullseye as builder
 WORKDIR /builder
 
 COPY . .

example/update_logger/Dockerfile

@@ -1,4 +1,4 @@
-FROM rust:1.94.0-bullseye as builder
+FROM rust:1.94.1-bullseye as builder
 WORKDIR /builder
 
 COPY . .

lib/command/src/lib.rs

@@ -163,12 +163,12 @@ pub async fn run_standard_command(
 fn shell() -> &'static str {
   static DEFAULT_SHELL: OnceLock<String> = OnceLock::new();
   DEFAULT_SHELL.get_or_init(|| {
-    if PathBuf::from("/bin/bash").exists()
-      || PathBuf::from("/usr/bin/bash").exists()
-    {
-      String::from("bash")
+    if PathBuf::from("/bin/bash").exists() {
+      String::from("/bin/bash")
+    } else if PathBuf::from("/usr/bin/bash").exists() {
+      String::from("/usr/bin/bash")
     } else {
-      String::from("sh")
+      String::from("/bin/sh")
     }
   })
 }

lib/environment/src/lib.rs

@@ -32,6 +32,12 @@ pub async fn write_env_file(
     .collect::<Vec<_>>()
     .join("\n");
 
+  let contents = if contents.is_empty() || contents.ends_with('\n') {
+    contents
+  } else {
+    contents + "\n"
+  };
+
   if let Err(e) =
     mogh_secret_file::write_async(&env_file_path, contents)
       .await

roadmap.md

@@ -13,7 +13,7 @@ If you have an idea for Komodo, feel free to open an issue beginning with the `[
 - **v1.16**: "Action" resource: Run requests on the Komodo API using snippets of typescript. ✅
 - **v1.17**: Procedure Schedules: Run procedures / Actions at scheduled times, like CRON job. Connect to host terminals and exec into containers ✅
 - **v1.18**: Upgrade granular role based access control system ✅
-- **Undecided**: Support "Swarm" resource - Manage docker swarms, attach Deployments / Stacks to "Swarm". 
+- **v2.0**: Support "Swarm" resource - Manage docker swarms, attach Deployments / Stacks to "Swarm". 
 - **Undecided**: Support "Cluster" resource - Manage Kubernetes cluster, can attach deployments to "Cluster"
 
 **Note. The specific versions associated with these features are not final.**

ui/package.json

@@ -24,7 +24,7 @@
     "jotai": "^2.18.0",
     "jotai-family": "^1.0.1",
     "jotai-location": "^0.6.2",
-    "lucide-react": "^0.575.0",
+    "lucide-react": "^1.6.0",
     "monaco-editor": "^0.55.1",
     "monaco-yaml": "^5.4.1",
     "react": "^19.2.4",
@@ -42,13 +42,14 @@
     "@types/react-dom": "^19.2.3",
     "@types/sanitize-html": "^2.16.0",
     "@types/shell-quote": "^1.7.5",
-    "@vitejs/plugin-react": "^5.1.4",
+    "@vitejs/plugin-react": "^6.0.1",
     "dotenv": "^17.3.1",
     "postcss": "^8.5.6",
     "postcss-preset-mantine": "^1.18.0",
     "postcss-simple-vars": "^7.0.1",
     "sass-embedded": "^1.97.3",
-    "typescript": "^5.9.3",
-    "vite": "^7.3.1"
-  }
+    "typescript": "^6.0.2",
+    "vite": "^8.0.2"
+  },
+  "packageManager": "yarn@1.22.22+sha512.a6b2f7906b721bba3d67d4aff083df04dad64c399707841b7acf00f6b133b7ac24255f2652fa22ae3534329dc6180534e98d17432037ff6fd140556e2bb3137e"
 }

ui/public/client/responses.d.ts

@@ -320,6 +320,7 @@ export type ExecuteResponses = {
     PruneBuildx: Types.Update;
     PruneSystem: Types.Update;
     RemoveSwarmNodes: Types.Update;
+    UpdateSwarmNode: Types.Update;
     RemoveSwarmStacks: Types.Update;
     RemoveSwarmServices: Types.Update;
     CreateSwarmConfig: Types.Update;

ui/public/client/types.d.ts

@@ -380,6 +380,7 @@ export declare enum Operation {
     RenameSwarm = "RenameSwarm",
     DeleteSwarm = "DeleteSwarm",
     RemoveSwarmNodes = "RemoveSwarmNodes",
+    UpdateSwarmNode = "UpdateSwarmNode",
     RemoveSwarmStacks = "RemoveSwarmStacks",
     RemoveSwarmServices = "RemoveSwarmServices",
     CreateSwarmConfig = "CreateSwarmConfig",
@@ -1054,6 +1055,9 @@ export type Execution =
 } | {
     type: "RemoveSwarmNodes";
     params: RemoveSwarmNodes;
+} | {
+    type: "UpdateSwarmNode";
+    params: UpdateSwarmNode;
 } | {
     type: "RemoveSwarmStacks";
     params: RemoveSwarmStacks;
@@ -1690,7 +1694,7 @@ export type AlertData =
         /** The name of the swarm */
         name: string;
         /** The error data */
-        err?: string;
+        err?: _Serror;
     };
 }
 /** A server could not be reached. */
@@ -5209,9 +5213,19 @@ export declare enum ServerState {
     /** Server is disabled. */
     Disabled = "Disabled"
 }
+export interface __Serror {
+    error: string;
+    trace: string[];
+}
+export type _Serror = __Serror;
 export interface ServerListItemInfo {
     /** The server's state. */
     state: ServerState;
+    /**
+     * If there is an error reaching
+     * the server, message will be given here.
+     */
+    err?: _Serror;
     /** Region of the server. */
     region: string;
     /** Address of the server, or null if empty. */
@@ -5361,11 +5375,14 @@ export interface SwarmNodeListItem {
     ID?: string;
     /** Name for the node. */
     Name?: string;
+    /** Node hostname, more commonly used than Name */
     Hostname?: string;
     /** Role of the node. */
     Role?: NodeSpecRoleEnum;
     /** Availability of the node. */
     Availability?: NodeSpecAvailabilityEnum;
+    /** Labels of the node */
+    Labels?: Record<string, string>;
     /** State of the node */
     State?: NodeState;
     /** For manager nodes, include the manager addr. */
@@ -5430,7 +5447,7 @@ export interface SwarmListItemInfo {
      * If there is an error reaching
      * Swarm, message will be given here.
      */
-    err?: string;
+    err?: _Serror;
 }
 export type SwarmListItem = ResourceListItem<SwarmListItemInfo>;
 export type ListSwarmsResponse = SwarmListItem[];
@@ -5585,11 +5602,6 @@ export type _PartialStackConfig = Partial<StackConfig>;
 export type _PartialSwarmConfig = Partial<SwarmConfig>;
 export type _PartialTag = Partial<Tag>;
 export type _PartialUrlBuilderConfig = Partial<UrlBuilderConfig>;
-export interface __Serror {
-    error: string;
-    trace: string[];
-}
-export type _Serror = __Serror;
 /** **Admin only.** Add a user to a user group. Response: [UserGroup] */
 export interface AddUserToUserGroup {
     /** The name or id of UserGroup that user should be added to. */
@@ -9894,6 +9906,25 @@ export interface UpdateSwarm {
     /** The partial config update to apply. */
     config: _PartialSwarmConfig;
 }
+/**
+ * `docker node update [OPTIONS] NODE`
+ *
+ * https://docs.docker.com/reference/cli/docker/node/update/
+ */
+export interface UpdateSwarmNode {
+    /** Name or id */
+    swarm: string;
+    /** Node hostname or id */
+    node: string;
+    /** Update the node's availability: 'active', 'pause', or 'drain' */
+    availability?: NodeSpecAvailabilityEnum;
+    /** Add labels to node (`key=value`). */
+    label_add?: string[];
+    /** Add labels to node (`key=value`). (alias: `lr`) */
+    label_rm?: string[];
+    /** Update the node's role: 'worker' or 'manager' */
+    role?: NodeSpecRoleEnum;
+}
 /** Update color for tag. Response: [Tag]. */
 export interface UpdateTagColor {
     /** The name or id of the tag to update. */
@@ -10197,6 +10228,9 @@ export type ExecuteRequest = {
 } | {
     type: "RemoveSwarmNodes";
     params: RemoveSwarmNodes;
+} | {
+    type: "UpdateSwarmNode";
+    params: UpdateSwarmNode;
 } | {
     type: "RemoveSwarmStacks";
     params: RemoveSwarmStacks;

ui/public/client/types.js

@@ -70,6 +70,7 @@ export var Operation;
     Operation["RenameSwarm"] = "RenameSwarm";
     Operation["DeleteSwarm"] = "DeleteSwarm";
     Operation["RemoveSwarmNodes"] = "RemoveSwarmNodes";
+    Operation["UpdateSwarmNode"] = "UpdateSwarmNode";
     Operation["RemoveSwarmStacks"] = "RemoveSwarmStacks";
     Operation["RemoveSwarmServices"] = "RemoveSwarmServices";
     Operation["CreateSwarmConfig"] = "CreateSwarmConfig";

ui/src/components/config/organization-selector.tsx

@@ -25,6 +25,7 @@ export default function OrganizationSelector({
   if (customMode) {
     return (
       <TextInput
+        label="Organization"
         placeholder="Input custom organization name"
         w={{ base: "85%", lg: 400 }}
         value={selected}
@@ -65,6 +66,7 @@ export default function OrganizationSelector({
           onSelect(value);
         }
       }}
+      searchable
       {...selectorProps}
     />
   );

ui/src/components/config/provider-selector.tsx

@@ -36,7 +36,7 @@ export default function ProviderSelector({
   if (customMode) {
     return (
       <TextInput
-        label="Domain"
+        label={showLabel && "Domain"}
         placeholder="Input custom provider domain"
         w={{ base: "85%", lg: 400 }}
         value={selected}

ui/src/components/docker/container-ports.tsx

@@ -1,5 +1,13 @@
 import { useContainerPortsMap } from "@/lib/hooks";
-import { Group, GroupProps, HoverCard, Stack, Text } from "@mantine/core";
+import {
+  Box,
+  Group,
+  GroupProps,
+  HoverCard,
+  HoverCardProps,
+  Stack,
+  Text,
+} from "@mantine/core";
 import { Types } from "komodo_client";
 import { EthernetPort } from "lucide-react";
 import { colorByIntention } from "@/lib/color";
@@ -11,19 +19,32 @@ import DividedChildren from "@/ui/divided-children";
 export interface ContainerPortsProps extends GroupProps {
   ports: Types.Port[];
   serverId: string | undefined;
+  hoverCardProps?: HoverCardProps;
 }
 
 export default function ContainerPorts({
   ports,
   serverId,
+  hoverCardProps,
+  gap = "xs",
+  wrap = "nowrap",
   ...groupProps
 }: ContainerPortsProps) {
   const portsMap = useContainerPortsMap(ports);
+
+  if (Object.keys(portsMap).length > 0) {
+    console.log("mapped", portsMap);
+  }
+
   const sortedNumericPorts = Object.keys(portsMap)
     .map(Number)
     .filter((port) => !Number.isNaN(port))
     .sort((a, b) => a - b);
 
+  if (Object.keys(portsMap).length > 0) {
+    console.log("sorted", sortedNumericPorts);
+  }
+
   type Group = { start: number; end: number; ports: Types.Port[] };
 
   const groupedPorts = sortedNumericPorts.reduce<Group[]>((acc, port) => {
@@ -42,21 +63,25 @@ export default function ContainerPorts({
     return null;
   }
 
+  console.log("grouped", groupedPorts);
+
   return (
-    <DividedChildren {...groupProps}>
+    <DividedChildren gap={gap} wrap="nowrap" {...groupProps}>
       {groupedPorts.map((group) => (
-        <ContainerPort
-          key={group.start}
-          ports={group.ports}
-          hostPort={String(group.start)}
-          serverId={serverId}
-        />
+        <Box key={group.start}>
+          <ContainerPort
+            ports={group.ports}
+            hostPort={String(group.start)}
+            serverId={serverId}
+            {...hoverCardProps}
+          />
+        </Box>
       ))}
     </DividedChildren>
   );
 }
 
-export interface ContainerPortProps {
+export interface ContainerPortProps extends HoverCardProps {
   ports: Types.Port[];
   hostPort: string;
   serverId: string | undefined;
@@ -66,16 +91,17 @@ export function ContainerPort({
   ports,
   hostPort,
   serverId,
+  position = "bottom",
+  ...hoverCardProps
 }: ContainerPortProps) {
   const serverAddress = useServerAddress(serverId);
 
-  if (!serverAddress) return null;
-
-  const isHttps = serverAddress.protocol === "https:";
+  const isHttps = serverAddress?.protocol === "https:";
   const link =
-    hostPort === "443" && isHttps
+    serverAddress &&
+    (hostPort === "443" && isHttps
       ? `https://${serverAddress.hostname}`
-      : `http://${serverAddress.hostname}:${hostPort}`;
+      : `http://${serverAddress.hostname}:${hostPort}`);
 
   const uniqueHostPorts = Array.from(
     new Set(
@@ -92,11 +118,18 @@ export function ContainerPort({
       : `${uniqueHostPorts[0]}-${uniqueHostPorts[uniqueHostPorts.length - 1]}`;
 
   return (
-    <HoverCard>
+    <HoverCard position={position} {...hoverCardProps}>
       <HoverCard.Target>
         <Group
-          renderRoot={(props) => <a target="_blank" href={link} {...props} />}
+          renderRoot={(props) =>
+            link ? (
+              <a target="_blank" href={link} {...props} />
+            ) : (
+              <div {...props} />
+            )
+          }
           gap="sm"
+          wrap="nowrap"
         >
           <EthernetPort size="1rem" color={colorByIntention("Good")} />
           {displayText}
@@ -105,10 +138,23 @@ export function ContainerPort({
       <HoverCard.Dropdown>
         <Stack gap="xs">
           <Group
-            renderRoot={(props) => <a target="_blank" href={link} {...props} />}
+            renderRoot={(props) =>
+              link ? (
+                <a target="_blank" href={link} {...props} />
+              ) : (
+                <div {...props} />
+              )
+            }
+            wrap="nowrap"
           >
-            <ICONS.Link size="1rem" color={colorByIntention("Good")} />
-            {link}
+            {link ? (
+              <>
+                <ICONS.Link size="1rem" color={colorByIntention("Good")} />
+                {link}
+              </>
+            ) : (
+              "Missing external address"
+            )}
           </Group>
 
           {ports.slice(0, 10).map((port, i) => (

ui/src/components/docker/container-selector.tsx

@@ -43,7 +43,7 @@ export default function ContainerSelector({
     server: serverId,
   }).data?.filter((container) => !state || container.state === state);
 
-  const firstContainer = containers?.[0].name;
+  const firstContainer = containers?.[0]?.name;
   useEffect(() => {
     !clearable && firstContainer && !selected && onSelect?.(firstContainer);
   }, [firstContainer]);

ui/src/components/docker/containers-section.tsx

@@ -170,7 +170,6 @@ export default function ContainersSection({
                 <ContainerPorts
                   ports={row.original.ports ?? []}
                   serverId={row.original.server_id}
-                  wrap="nowrap"
                 />
               ),
             },

ui/src/components/hash-compare.tsx

@@ -55,7 +55,9 @@ export default function HashCompare({
               <Badge color="accent.9">message</Badge>
               <Text c="dimmed">{lastHash}</Text>
             </Group>
-            <Code>{lastMessage || latestMessage}</Code>
+            <Code maw="calc(100vh - 150px)" style={{ overflow: "auto" }}>
+              {lastMessage || latestMessage}
+            </Code>
           </Stack>
           {outOfDate && (
             <Stack gap="xs">
@@ -65,7 +67,9 @@ export default function HashCompare({
                 </Badge>
                 <Text c="dimmed">{latestHash}</Text>
               </Group>
-              <Code>{latestMessage}</Code>
+              <Code maw="calc(100vh - 150px)" style={{ overflow: "auto" }}>
+                {latestMessage}
+              </Code>
             </Stack>
           )}
         </Stack>

ui/src/components/stack-service-selector.tsx

@@ -51,7 +51,7 @@ export default function StackServiceSelector({
     stack: stackId,
   }).data?.filter((service) => !state || service?.container?.state === state);
 
-  const firstService = services?.[0].service;
+  const firstService = services?.[0]?.service;
   useEffect(() => {
     !clearable && firstService && !selected && onSelect?.(firstService);
   }, [firstService]);

ui/src/lib/utils.ts

@@ -357,6 +357,11 @@ export function listsEqual(a: string[], b: string[]) {
  * @returns a === b
  */
 export function deepCompare(a: any, b: any) {
+  // Short path for falsy. Important to catch typeof null === "object" edge case.
+  if (!a || !b) {
+    return a === b;
+  }
+
   const ta = typeof a;
   const tb = typeof b;
 

ui/src/pages/settings/tags/color-selector.tsx

@@ -77,7 +77,7 @@ export default function TagColorSelector({
             <Combobox.Option key={color} value={color}>
               <Group justify="space-between">
                 <Text>{fmtUpperCamelcase(color)}</Text>
-                <Box w={25} h={25} bg={"Tag" + color} bdrs="md" />
+                <Box w={25} h={25} bg={`Tag${color}.9`} bdrs="md" />
               </Group>
             </Combobox.Option>
           ))}

ui/src/pages/swarm/node/index.tsx

@@ -7,6 +7,7 @@ import RemoveSwarmResource from "@/components/swarm/remove";
 import ResourceSubPage from "@/resources/sub-page";
 import SwarmNodeTabs from "./tabs";
 import PageGuard from "@/ui/page-guard";
+import UpdateSwarmNodes from "@/resources/swarm/docker/nodes/update";
 
 export default function SwarmNode() {
   const { id: swarmId, node: __node } = useParams() as {
@@ -52,12 +53,20 @@ export default function SwarmNode() {
         status={node?.Spec?.Role}
         executions={
           node?.ID && (
-            <RemoveSwarmResource
-              swarmId={swarmId}
-              type="Node"
-              resourceId={node.ID}
-              resourceName={node.Description?.Hostname}
-            />
+            <>
+              <UpdateSwarmNodes
+                swarm={swarmId}
+                nodes={
+                  node.Description?.Hostname ? [node.Description?.Hostname] : []
+                }
+              />
+              <RemoveSwarmResource
+                swarmId={swarmId}
+                type="Node"
+                resourceId={node.ID}
+                resourceName={node.Description?.Hostname}
+              />
+            </>
           )
         }
       >

ui/src/pages/terminals/new.tsx

@@ -253,6 +253,7 @@ function CreateServerTerminal({
       finalize={(baseRequest) => ({
         name: baseRequest.name,
         target: { type: "Server", params: { server: server } },
+        command: baseRequest.command,
       })}
       onSuccess={(terminal) => {
         nav(`/servers/${server}/terminal/${terminal.name}`);
@@ -307,11 +308,12 @@ function CreateContainerTerminal({
       setType={setType}
       finalize={(baseRequest) => ({
         name: baseRequest.name,
-        mode: baseRequest.mode,
         target: {
           type: "Container",
           params: { server: params.server, container: params.container },
         },
+        command: baseRequest.command,
+        mode: baseRequest.mode,
       })}
       onSuccess={(terminal) => {
         nav(
@@ -382,11 +384,12 @@ function CreateStackServiceTerminal({
       setType={setType}
       finalize={(baseRequest) => ({
         name: baseRequest.name,
-        mode: baseRequest.mode,
         target: {
           type: "Stack",
           params: { stack: params.stack, service: params.service },
         },
+        command: baseRequest.command,
+        mode: baseRequest.mode,
       })}
       onSuccess={(terminal) => {
         nav(
@@ -459,6 +462,8 @@ function CreateDeploymentTerminal({
       finalize={(baseRequest) => ({
         name: baseRequest.name,
         target: { type: "Deployment", params: { deployment: deployment } },
+        command: baseRequest.command,
+        mode: baseRequest.mode,
       })}
       onSuccess={(terminal) => {
         nav(`/deployments/${deployment}/terminal/${terminal.name}`);

ui/src/resources/build/config.tsx

@@ -242,12 +242,7 @@ export default function BuildConfig({
 
             {image_registries?.map((registry, index) => (
               <ImageRegistryConfig
-                key={
-                  (registry.domain ?? "") +
-                  (registry.organization ?? "") +
-                  (registry.account ?? "") +
-                  index
-                }
+                key={index}
                 registry={registry}
                 imageName={imageName}
                 setRegistry={(registry) =>

ui/src/resources/build/info.tsx

@@ -168,8 +168,7 @@ export default function BuildInfo({
                   />
                 </>
               )}
-              {/* The toggle onClick is given to entire header */}
-              <ShowHideButton show={show} setShow={() => {}} />
+              <ShowHideButton show={show} setShow={() => setShow((show) => !show)} />
             </Group>
           </Group>
 

ui/src/resources/deployment/update-available.tsx

@@ -45,7 +45,6 @@ export default function DeploymentUpdateAvailable({
   const state = info?.state ?? Types.DeploymentState.Unknown;
   if (
     !info ||
-    info.swarm_id ||
     info.build_id ||
     [Types.DeploymentState.NotDeployed, Types.DeploymentState.Unknown].includes(
       state,

ui/src/resources/procedure/config/executions.tsx

@@ -43,6 +43,7 @@ export type ProcedureMinExecutionType = Exclude<
   | "DeleteVolume"
   | "TestAlerter"
   | "RemoveSwarmNodes"
+  | "UpdateSwarmNode"
   | "RemoveSwarmStacks"
   | "RemoveSwarmServices"
   | "CreateSwarmConfig"
@@ -772,7 +773,7 @@ export const PROCEDURE_EXECUTIONS: ProcedureExecutions = {
               <Stack gap="0">
                 <Text>Command</Text>
                 <TextInput
-                  placeholder="Enter command (Required)"
+                  placeholder="Enter command (Optional)"
                   value={commandText}
                   onChange={(e) => setCommand(e.target.value)}
                   disabled={disabled}

ui/src/resources/server/index.tsx

@@ -9,7 +9,7 @@ import NewResource from "@/resources/new";
 import ConfirmButton from "@/ui/confirm-button";
 import { Prune } from "./executions";
 import ServerVersion from "./version";
-import { Group, HoverCard } from "@mantine/core";
+import { Box, Group, HoverCard } from "@mantine/core";
 import { notifications } from "@mantine/notifications";
 import ConfirmServerPubkey from "./confirm-pubkey";
 import ServerTabs from "./tabs";
@@ -22,6 +22,7 @@ import { ServerLoadAverage } from "./stats/current/load-average";
 import { ServerRamUsage } from "./stats/current/ram";
 import ServerDiskUsage from "./diskUsage";
 import ServerCpuUsage from "./stats/current/cpu";
+import HoverError from "@/ui/hover-error";
 
 export function useServer(id: string | undefined, useName?: boolean) {
   return useRead("ListServers", {}).data?.find((r) =>
@@ -288,6 +289,15 @@ export const ServerComponents: RequiredResourceComponents<
         </HoverCard>
       );
     },
+    Err: ({ id }) => {
+      const err = useServer(id)?.info.err;
+      if (!err) return null;
+      return (
+        <Box>
+          <HoverError {...err} />
+        </Box>
+      );
+    },
     ConfirmServerPubkey,
   },
 

ui/src/resources/stack/config/index.tsx

@@ -237,8 +237,7 @@ export default function StackConfig({
 
   const environment: ConfigGroupArgs<Types.StackConfig> = {
     label: "Environment",
-    hidden: !!currSwarmId,
-    description: "Pass these variables to the compose command",
+    description: `Pass these variables to the docker ${currSwarmId ? "stack" : "compose"} command`,
     actions: (
       <ShowHideButton
         show={show.env}
@@ -334,9 +333,9 @@ export default function StackConfig({
                         ],
                       });
                     }}
-                    w="fit-content"
+                    leftSection={<ICONS.Add size="1rem" />}
+                    w={{ base: "85%", lg: 400 }}
                   >
-                    <ICONS.Add size="1rem" />
                     Add Env File
                   </Button>
                 )}
@@ -532,8 +531,8 @@ export default function StackConfig({
             description={
               <Group gap="xs">
                 <Text>
-                  Pass extra arguments to '
-                  {currSwarmId ? "docker stack deploy" : "docker compose up"}
+                  Pass extra arguments to docker '
+                  {currSwarmId ? "stack deploy" : "compose up"}
                   '.
                 </Text>
                 <Text
@@ -693,7 +692,7 @@ export default function StackConfig({
         destroy_before_deploy: {
           label: "Destroy Before Deploy",
           description: `Ensure '${
-            currSwarmId ? "docker service rm" : "docker compose down"
+            currSwarmId ? "docker stack rm" : "docker compose down"
           }' is run before redeploying the Stack.`,
         },
       },

ui/src/resources/stack/info.tsx

@@ -180,8 +180,7 @@ export default function StackInfo({
                       />
                     </>
                   )}
-                  {/* The toggle onClick is given to entire header */}
-                  <ShowHideButton show={showContents} setShow={() => {}} />
+                  <ShowHideButton show={showContents} setShow={handleToggleShow} />
                 </Group>
               </Group>
 

ui/src/resources/stack/update-available.tsx

@@ -52,7 +52,6 @@ export default function StackUpdateAvailable({
 
   if (
     !info ||
-    info.swarm_id ||
     [Types.StackState.Down, Types.StackState.Unknown].includes(state)
   ) {
     return null;

ui/src/resources/swarm/config.tsx

@@ -58,8 +58,9 @@ export default function SwarmConfig({
                         <Group
                           key={index}
                           gap="xs"
-                          w={{ base: "85%", lg: 400 }}
+                          w={{ base: "100%", lg: 400 }}
                           justify="space-between"
+                          wrap="nowrap"
                         >
                           <ResourceSelector
                             type="Server"
@@ -105,7 +106,7 @@ export default function SwarmConfig({
                           })
                         }
                         leftSection={<ICONS.Add size="1rem" />}
-                        w={{ base: "85%", lg: 400 }}
+                        w={{ base: "100%", lg: 400 }}
                       >
                         Add Server
                       </Button>
@@ -118,7 +119,6 @@ export default function SwarmConfig({
           {
             label: "Alerts",
             labelHidden: true,
-            contentHidden: ((update.links ?? config.links)?.length ?? 0) === 0,
             fields: {
               send_unhealthy_alerts: {
                 description: "Send alerts when the Swarm is unhealthy",
@@ -150,11 +150,13 @@ export default function SwarmConfig({
           },
           {
             label: "Links",
-            description: "Add quick links in the resource header",
-            contentHidden: ((update.links ?? config.links)?.length ?? 0) === 0,
+            labelHidden: true,
             fields: {
               links: (values, set) => (
                 <ConfigList
+                  label="Links"
+                  addLabel="Add Link"
+                  description="Add quick links in the resource header"
                   field="links"
                   values={values ?? []}
                   set={set}

ui/src/resources/swarm/docker/nodes/index.tsx

@@ -1,7 +1,7 @@
 import { useRead } from "@/lib/hooks";
 import { filterBySplit } from "@/lib/utils";
-import { ReactNode } from "react";
-import { useSwarmDockerSearch } from ".";
+import { ReactNode, useState } from "react";
+import { useSwarmDockerSearch } from "..";
 import Section from "@/ui/section";
 import { DataTable, SortableHeader } from "@/ui/data-table";
 import SwarmResourceLink from "@/components/swarm/link";
@@ -12,6 +12,10 @@ import {
   swarmNodeStateIntention,
 } from "@/lib/color";
 import SearchInput from "@/ui/search-input";
+import { RowSelectionState } from "@tanstack/react-table";
+import UpdateSwarmNode from "./update";
+import { HoverCard } from "@mantine/core";
+import LabelsGroup from "@/ui/labels-group";
 
 export default function SwarmNodes({
   id,
@@ -25,6 +29,8 @@ export default function SwarmNodes({
     useRead("ListSwarmNodes", { swarm: id }, { refetchInterval: 10_000 })
       .data ?? [];
 
+  const selectState = useState<RowSelectionState>({});
+
   const filtered = filterBySplit(
     nodes,
     search,
@@ -34,11 +40,19 @@ export default function SwarmNodes({
   return (
     <Section
       titleOther={titleOther}
+      titleRight={
+        <UpdateSwarmNode swarm={id} nodes={Object.keys(selectState[0])} />
+      }
       actions={<SearchInput value={search} onSearch={setSearch} />}
     >
       <DataTable
         tableKey="swarm-nodes"
         data={filtered}
+        selectOptions={{
+          selectKey: (node) =>
+            node.Name ?? node.Hostname ?? node.ID ?? "Unknown",
+          state: selectState,
+        }}
         columns={[
           {
             accessorKey: "Hostname",
@@ -101,6 +115,31 @@ export default function SwarmNodes({
               />
             ),
           },
+          {
+            accessorKey: "Labels",
+            header: ({ column }) => (
+              <SortableHeader column={column} title="Labels" />
+            ),
+            cell: ({ row }) => {
+              const labels = Object.entries(row.original.Labels ?? {}).sort();
+              return (
+                <HoverCard position="bottom-start" disabled={labels.length < 3}>
+                  <HoverCard.Target>
+                    <LabelsGroup
+                      labels={labels.slice(0, 2)}
+                      showEllipsis={labels.length > 2}
+                      wrap="nowrap"
+                    />
+                  </HoverCard.Target>
+                  <HoverCard.Dropdown
+                    maw={{ base: "calc(100vw - 100px)", xs: 400 }}
+                  >
+                    <LabelsGroup labels={labels.slice(2)} />
+                  </HoverCard.Dropdown>
+                </HoverCard>
+              );
+            },
+          },
           {
             accessorKey: "UpdatedAt",
             header: ({ column }) => (

ui/src/resources/swarm/docker/nodes/update.tsx

@@ -0,0 +1,189 @@
+import { useExecute, useRead } from "@/lib/hooks";
+import {
+  Button,
+  Group,
+  Loader,
+  Modal,
+  Select,
+  SimpleGrid,
+  Stack,
+  TagsInput,
+  Text,
+  useMatches,
+} from "@mantine/core";
+import { useDisclosure } from "@mantine/hooks";
+import { useMemo, useState } from "react";
+import { Types } from "komodo_client";
+import { ICONS } from "@/theme/icons";
+import LabelsGroup from "@/ui/labels-group";
+
+export interface UpdateSwarmNodeProps {
+  swarm: string;
+  nodes: string[];
+}
+
+export default function UpdateSwarmNodes({
+  swarm,
+  nodes,
+}: UpdateSwarmNodeProps) {
+  const [opened, { open, close }] = useDisclosure();
+  const { data: inspect, isPending } = useRead(
+    "InspectSwarmNode",
+    { swarm, node: nodes[0] },
+    { enabled: opened && nodes.length === 1 },
+  );
+
+  const currentLabels = useMemo(() => {
+    const labels = inspect?.Spec?.Labels ?? {};
+    return Object.entries(labels).sort();
+  }, [inspect]);
+
+  const currentRole = inspect?.Spec?.Role ?? "";
+  const currentAvailability = inspect?.Spec?.Availability ?? "";
+
+  const [role, setRole] = useState<string | null>(null);
+  const [availability, setAvailability] = useState<string | null>(null);
+  const [labelsToAdd, setLabelsToAdd] = useState<string[]>([]);
+  const [labelsToRemove, setLabelsToRemove] = useState<string[]>([]);
+
+  const { mutate: update, isPending: updatePending } = useExecute(
+    "UpdateSwarmNode",
+    {
+      onSuccess: () => {
+        close();
+        resetForm();
+      },
+    },
+  );
+
+  const resetForm = () => {
+    setRole(null);
+    setAvailability(null);
+    setLabelsToAdd([]);
+    setLabelsToRemove([]);
+  };
+
+  const onSubmit = () => {
+    for (const node of nodes) {
+      update({
+        swarm,
+        node,
+        role: role ? (role as Types.NodeSpecRoleEnum) : undefined,
+        availability: availability
+          ? (availability as Types.NodeSpecAvailabilityEnum)
+          : undefined,
+        label_add: labelsToAdd.length > 0 ? labelsToAdd : undefined,
+        label_rm: labelsToRemove.length > 0 ? labelsToRemove : undefined,
+      });
+    }
+  };
+
+  const tagSpan = useMatches({ base: 1, sm: 2 });
+
+  return (
+    <>
+      <Modal
+        opened={opened}
+        onClose={() => {
+          close();
+          resetForm();
+        }}
+        title={`Update Node${nodes.length === 1 ? "" : "s"}: ${nodes.join(", ")}`}
+        size="xl"
+      >
+        {nodes.length === 1 && isPending ? (
+          <Group justify="center" p="xl">
+            <Loader />
+          </Group>
+        ) : (
+          <Stack>
+            <SimpleGrid cols={{ base: 1, sm: 2 }}>
+              <Select
+                label="Role"
+                placeholder={
+                  (nodes.length === 1 &&
+                    currentRole &&
+                    `Current: ${currentRole}`) ||
+                  "Select role"
+                }
+                data={[
+                  { value: "worker", label: "Worker" },
+                  { value: "manager", label: "Manager" },
+                ]}
+                value={role}
+                onChange={setRole}
+                clearable
+              />
+
+              <Select
+                label="Availability"
+                placeholder={
+                  (nodes.length === 1 &&
+                    currentAvailability &&
+                    `Current: ${currentAvailability}`) ||
+                  "Select availability"
+                }
+                data={[
+                  { value: "active", label: "Active" },
+                  { value: "pause", label: "Pause" },
+                  { value: "drain", label: "Drain" },
+                ]}
+                value={availability}
+                onChange={setAvailability}
+                clearable
+              />
+
+              <TagsInput
+                label="Labels to Add. Press enter to add multiple."
+                placeholder="key=value"
+                value={labelsToAdd}
+                onChange={setLabelsToAdd}
+              />
+
+              <TagsInput
+                label="Labels to Remove"
+                placeholder="key"
+                data={currentLabels.map(([key, _]) => key)}
+                value={labelsToRemove}
+                onChange={setLabelsToRemove}
+              />
+
+              {currentLabels.length > 0 && (
+                <Stack gap="0.25rem" style={{ gridColumn: `span ${tagSpan}` }}>
+                  <Text size="sm" fw={500}>
+                    Current Labels
+                  </Text>
+                  <LabelsGroup labels={currentLabels} />
+                </Stack>
+              )}
+            </SimpleGrid>
+            <Group justify="flex-end" mt="md">
+              <Button
+                variant="default"
+                onClick={() => {
+                  close();
+                  resetForm();
+                }}
+              >
+                Cancel
+              </Button>
+              <Button onClick={onSubmit} loading={updatePending}>
+                Update
+              </Button>
+            </Group>
+          </Stack>
+        )}
+      </Modal>
+
+      <Button
+        onClick={open}
+        disabled={!nodes.length}
+        rightSection={<ICONS.Edit size="1rem" />}
+        justify="space-between"
+        w={{ base: "100%", xs: 190 }}
+      >
+        Update Node{nodes.length === 1 ? "" : "s"}
+      </Button>
+    </>
+  );
+}

ui/src/resources/swarm/index.tsx

@@ -13,12 +13,12 @@ import SwarmTable from "./table";
 import NewResource from "@/resources/new";
 import SwarmTabs from "./tabs";
 import { useDisclosure } from "@mantine/hooks";
-import { Box, Button, HoverCard, Modal, Text } from "@mantine/core";
+import { Box, Button, Modal, Text } from "@mantine/core";
 import JoinSwarmCommands from "./join-commands";
-import { updateLogToHtml } from "@/lib/utils";
 import ResourceHeader from "../header";
 import BatchExecutions from "@/components/batch-executions";
 import SwarmHeaderInfo from "./header-info";
+import HoverError from "@/ui/hover-error";
 
 export function useSwarm(id: string | undefined, useName?: boolean) {
   return useRead("ListSwarms", {}).data?.find((r) =>
@@ -202,28 +202,7 @@ export const SwarmComponents: RequiredResourceComponents<
       if (!err) return null;
       return (
         <Box>
-          <HoverCard position="bottom-start">
-            <HoverCard.Target>
-              <Button
-                variant="filled"
-                color="red"
-                leftSection={<ICONS.Alert size="1rem" />}
-              >
-                Error
-              </Button>
-            </HoverCard.Target>
-            <HoverCard.Dropdown>
-              <Text
-                component="pre"
-                fz="sm"
-                mah={500}
-                dangerouslySetInnerHTML={{
-                  __html: updateLogToHtml(err),
-                }}
-                style={{ overflowY: "auto" }}
-              />
-            </HoverCard.Dropdown>
-          </HoverCard>
+          <HoverError {...err} />
         </Box>
       );
     },

ui/src/resources/sync/info.tsx

@@ -190,8 +190,7 @@ export default function ResourceSyncInfo({
                       />
                     </>
                   )}
-                  {/* The toggle onClick is given to entire header */}
-                  <ShowHideButton show={showContents} setShow={() => {}} />
+                  <ShowHideButton show={showContents} setShow={handleToggleShow} />
                 </Group>
               </Group>
 

ui/src/ui/hover-error.tsx

@@ -0,0 +1,48 @@
+import { ICONS } from "@/theme/icons";
+import { Button, Code, Group, HoverCard, Stack, Text } from "@mantine/core";
+
+export interface HoverErrorProps {
+  error: string;
+  trace: string[];
+}
+
+export default function HoverError({ error, trace }: HoverErrorProps) {
+  return (
+    <HoverCard position="bottom-start">
+      <HoverCard.Target>
+        <Button
+          variant="filled"
+          color="red"
+          leftSection={<ICONS.Alert size="1rem" />}
+        >
+          Error
+        </Button>
+      </HoverCard.Target>
+      <HoverCard.Dropdown>
+        <Stack gap="sm" maw={{ base: 400, xs: 500, sm: 600, md: 800 }}>
+          <Stack component={Code} gap="0">
+            <Text fw="bold" c="red" fz="sm">
+              ERROR:
+            </Text>
+            <Text fz="sm">{error}</Text>
+          </Stack>
+          {trace.length > 0 && (
+            <Stack component={Code} gap="0">
+              <Text c="dimmed" fz="sm">
+                TRACE:
+              </Text>
+              {trace.map((error, i) => (
+                <Group key={i} wrap="nowrap" gap="xs" align="start" pl="1rem">
+                  <Text c="dimmed" fz="sm">
+                    {i + 1}:
+                  </Text>
+                  <Text fz="sm">{error}</Text>
+                </Group>
+              ))}
+            </Stack>
+          )}
+        </Stack>
+      </HoverCard.Dropdown>
+    </HoverCard>
+  );
+}

ui/src/ui/labels-group.tsx

@@ -0,0 +1,27 @@
+import { Badge, Group, GroupProps } from "@mantine/core";
+
+export interface LabelsGroupProps extends GroupProps {
+  labels: [string, string][];
+  showEllipsis?: boolean;
+}
+
+export default function LabelsGroup({
+  labels,
+  showEllipsis,
+  ...groupProps
+}: LabelsGroupProps) {
+  return (
+    <Group gap="xs" {...groupProps}>
+      {labels.map(([key, val]) => (
+        <Badge key={key} tt="none" fw="normal" fz={{ base: "sm", sm: "md" }}>
+          {key}=<b>{val}</b>
+        </Badge>
+      ))}
+      {showEllipsis && (
+        <Badge tt="none" fw="normal" fz={{ base: "sm", sm: "md" }}>
+          ...
+        </Badge>
+      )}
+    </Group>
+  );
+}

ui/src/ui/section.tsx

@@ -70,6 +70,11 @@ const Section = createPolymorphicComponent<"div", SectionProps>(
               {titleNode}
               {titleRight}
             </Group>
+          ) : titleRight ? (
+            <Group c={titleDimmed ? "dimmed" : undefined} gap="xs">
+              {titleOther}
+              {titleRight}
+            </Group>
           ) : (
             titleOther
           )}

ui/tsconfig.json

@@ -21,7 +21,6 @@
     "noUnusedParameters": true,
     "noFallthroughCasesInSwitch": true,
     /* Module resolution */
-    "baseUrl": ".",
     "paths": {
       "@/*": [
         "./src/*"