2.3.1 (#1553)

* add static connection user agent

* include server alerting thresholds in list info to reduce query

* fix database registry accounts lost

* dev-1

* deploy 2.3.1-dev-2

* CoreReport -> KomodoReport

* stack update available avoid many GetStack call on tables

* deploy 2.3.1-dev-3

* dashboard tables toml respects tag filter

* add post execution invalidate for better feedback when ws update is missed

* resource selector select first result on settle

* skip GetPermission call for admin users

* fmt

* 2.3.1

* fix clippy lint

* ts 2.3.1

.cargo/config.toml

@@ -1,2 +1,5 @@
+[alias]
+xtask = "run --package xtask --"
+
 [build]
-rustflags = ["-Wunused-crate-dependencies"]
+rustflags = ["-Wunused-crate-dependencies"]

.github/workflows/ci.yml

@@ -22,16 +22,16 @@ jobs:
         with:
           toolchain: stable
 
-      - name: Cache cargo registry
-        uses: actions/cache@v4
-        with:
-          path: |
-            ~/.cargo/registry
-            ~/.cargo/git
-            target
-          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
-          restore-keys: |
-            ${{ runner.os }}-cargo-
+      # - name: Cache cargo registry
+      #   uses: actions/cache@v4
+      #   with:
+      #     path: |
+      #       ~/.cargo/registry
+      #       ~/.cargo/git
+      #       target
+      #     key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
+      #     restore-keys: |
+      #       ${{ runner.os }}-cargo-
 
       - name: Build
         run: cargo build --verbose

Cargo.toml

@@ -3,21 +3,18 @@ resolver = "2"
 members = [
 	"bin/*",
 	"lib/*",
+	"xtask",
 	"client/core/rs",
 	"client/periphery/rs",
 ]
 
 [workspace.package]
-version = "2.0.0"
+version = "2.3.1"
 edition = "2024"
-authors = ["mbecker20 <becker.maxh@gmail.com>"]
 license = "GPL-3.0-or-later"
 repository = "https://github.com/moghtech/komodo"
 homepage = "https://komo.do"
 
-[profile.release]
-strip = "debuginfo"
-
 [workspace.dependencies]
 # LOCAL
 komodo_client = { path = "client/core/rs" }
@@ -33,106 +30,113 @@ git = { path = "lib/git" }
 
 # MOGH
 slack = { version = "2.0.0", package = "slack_client_rs", default-features = false, features = ["rustls"] }
-mogh_error = { version = "1.0.3", default-features = false }
+mogh_error = { version = "1.0.4", default-features = false }
 derive_default_builder = "0.1.8"
 async_timing_util = "1.1.0"
-mogh_auth_client = "1.2.2"
-mogh_auth_server = "1.2.13"
+mogh_auth_client = "1.7.0"
+mogh_auth_server = "1.7.0"
 mogh_secret_file = "1.0.1"
 mogh_validations = "1.0.1"
 mogh_rate_limit = "1.0.1"
-partial_derive2 = "0.4.5"
-mongo_indexed = "2.0.2"
+partial_derive2 = "0.5.0"
+mongo_indexed = "2.1.0"
 mogh_resolver = "1.0.0"
-mogh_config = "1.0.4"
-mogh_logger = "1.3.2"
-mogh_server = "1.4.5"
+mogh_config = "1.1.0"
+mogh_logger = "1.3.4"
+mogh_server = "1.5.0"
 toml_pretty = "2.0.0"
-mogh_cache = "1.1.1"
+mogh_cache = "1.1.2"
 mogh_pki = "1.1.3"
-mungos = "3.2.2"
+mungos = "3.7.0"
 svi = "1.2.0"
 
 # ASYNC
-reqwest = { version = "0.13.2", default-features = false, features = ["json", "stream", "form", "query", "rustls"] }
-tokio = { version = "1.50.0", features = ["full"] }
-tokio-util = { version = "0.7.18", features = ["io", "codec"] }
-tokio-stream = { version = "0.1.18", features = ["sync"] }
+reqwest = { version = "0.13.4", default-features = false, features = ["json", "stream", "form", "query", "rustls"] }
+tokio = { version = "1.53.1", features = ["full"] }
+tokio-util = { version = "0.7.19", features = ["io", "codec"] }
+tokio-stream = { version = "0.1.19", features = ["sync"] }
 pin-project-lite = "0.2.17"
-futures-util = "0.3.32"
-arc-swap = "1.9.0"
+futures-util = "0.3.33"
+arc-swap = "1.9.2"
 
 # SERVER
-tokio-tungstenite = { version = "0.29.0", features = ["rustls-tls-native-roots"] }
-axum = { version = "0.8.8", features = ["ws", "json", "macros"] }
-axum-extra = { version = "0.12.5", features = ["typed-header"] }
+tokio-tungstenite = { version = "0.30.0", features = ["rustls-tls-native-roots"] }
+axum = { version = "0.8.9", features = ["ws", "json", "macros"] }
+axum-extra = { version = "0.12.6", features = ["typed-header"] }
 
 # OPENAPI
 utoipa-scalar = { version = "0.3.0", features = ["axum"] }
-utoipa = "5.4.0"
+utoipa = "5.5.0"
 
 # SER/DE
 ipnetwork = { version = "0.21.1", features = ["serde"] }
-indexmap = { version = "2.13.0", features = ["serde"] }
-serde = { version = "1.0.227", features = ["derive"] }
+indexmap = { version = "2.14.0", features = ["serde"] }
+serde = { version = "1.0.229", features = ["derive"] }
 strum = { version = "0.28.0", features = ["derive"] }
 bson = { version = "2.15.0" } # must keep in sync with mongodb version
-toml = "1.1.0"
+toml = "1.1.3"
 serde_yaml_ng = "0.10.0"
-serde_json = "1.0.149"
-serde_qs = "1.1.0"
+serde_json = "1.0.151"
+serde_qs = "1.1.2"
 url = "2.5.8"
 
 # ERROR
-anyhow = "1.0.102"
-thiserror = "2.0.18"
+anyhow = "1.0.104"
+thiserror = "2.0.19"
+
+# SYSTEM
+libc = "0.2.189"
 
 # LOGGING
 tracing = "0.1.44"
 
 # CONFIG
-clap = { version = "4.5.60", features = ["derive"] }
+clap = { version = "4.6.4", features = ["derive"] }
 dotenvy = "0.15.7"
 envy = "0.4.2"
 
 # CRYPTO / AUTH
-uuid = { version = "1.21.0", features = ["v4", "fast-rng", "serde"] }
-rustls = { version = "0.23.37", features = ["aws-lc-rs"] }
-data-encoding = "2.10.0"
+uuid = { version = "1.24.0", features = ["v4", "fast-rng", "serde"] }
+rustls = { version = "0.23.42", features = ["aws-lc-rs"] }
+data-encoding = "2.11.0"
 urlencoding = "2.1.3"
-bcrypt = "0.19.0"
-hmac = "0.12.1"
-sha1 = "0.10.6"
-sha2 = "0.10.9"
-rand = "0.10.0"
+bcrypt = "0.19.2"
+hmac = "0.13.0"
+sha1 = "0.11.0"
+sha2 = "0.11.0"
+rand = "0.10.2"
 hex = "0.4.3"
 
 # SYSTEM
-hickory-resolver = "0.25.2"
+hickory-resolver = "0.26.1"
 portable-pty = "0.9.0"
 shell-escape = "0.1.5"
 crossterm = "0.29.0"
-bollard = "0.20.2"
-sysinfo = "0.38.4"
-shlex = "1.3.0"
+bollard = "0.21.0"
+sysinfo = "0.39.6"
+shlex = "2.0.1"
 
 # CLOUD
-aws-config = "1.8.15"
-aws-sdk-ec2 = "1.220.0"
-aws-credential-types = "1.2.14"
+aws-config = "1.10.0"
+aws-sdk-ec2 = "1.241.0"
+aws-credential-types = "1.3.0"
 
 ## CRON
 english-to-cron = "0.1.7"
 chrono-tz = "0.10.4"
-chrono = "0.4.44"
+chrono = "0.4.45"
 croner = "3.0.1"
 
 # MISC
-async-compression = { version = "0.4.41", features = ["tokio", "gzip"] }
+async-compression = { version = "0.4.42", features = ["tokio", "gzip"] }
+schemars = { version = "1.2.1", features = ["indexmap2"] }
 derive_builder = "0.20.2"
 comfy-table = "7.2.2"
 typeshare = "1.0.5"
 wildcard = "0.3.0"
 colored = "3.1.1"
-bytes = "1.11.1"
-regex = "1.12.3"
+bytes = "1.12.1"
+regex = "1.13.1"
+
+[profile.release]
+strip = "debuginfo"

action/deploy-ui.ts

@@ -1,4 +1,4 @@
-const cmd = "km run -y action deploy-komodo-fe-change";
+const cmd = "km run -y action deploy-komodo-ui-change";
 new Deno.Command("bash", {
   args: ["-c", cmd],
 }).spawn();

bin/binaries.Dockerfile

@@ -1,7 +1,7 @@
 ## Builds the Komodo Core, Periphery, and Util binaries
 ## for a specific architecture. Requires OpenSSL 3 or later.
 
-FROM rust:1.94.0-bookworm AS builder
+FROM rust:1.97.1-bookworm AS builder
 RUN cargo install cargo-strip
 
 WORKDIR /builder
@@ -12,6 +12,7 @@ COPY ./client/periphery ./client/periphery
 COPY ./bin/core ./bin/core
 COPY ./bin/periphery ./bin/periphery
 COPY ./bin/cli ./bin/cli
+COPY ./xtask ./xtask
 
 # Compile bin
 RUN \

bin/chef.binaries.Dockerfile

@@ -3,7 +3,7 @@
 
 ## Uses chef for dependency caching to help speed up back-to-back builds.
 
-FROM lukemathwalker/cargo-chef:latest-rust-1.94.0-bookworm AS chef
+FROM lukemathwalker/cargo-chef:latest-rust-1.97.1-bookworm AS chef
 WORKDIR /builder
 
 # Plan just the RECIPE to see if things have changed

bin/cli/Cargo.toml

@@ -3,7 +3,6 @@ name = "komodo_cli"
 description = "Command line tool for Komodo"
 version.workspace = true
 edition.workspace = true
-authors.workspace = true
 license.workspace = true
 repository.workspace = true
 homepage.workspace = true

bin/cli/aio.Dockerfile

@@ -1,4 +1,4 @@
-FROM rust:1.94.0-bullseye AS builder
+FROM rust:1.97.1-trixie AS builder
 RUN cargo install cargo-strip
 
 WORKDIR /builder

bin/cli/multi-arch.Dockerfile

@@ -9,7 +9,7 @@ ARG AARCH64_BINARIES=${BINARIES_IMAGE}-aarch64
 FROM ${X86_64_BINARIES} AS x86_64
 FROM ${AARCH64_BINARIES} AS aarch64
 
-FROM debian:bullseye-slim
+FROM debian:trixie-slim
 
 WORKDIR /app
 

bin/cli/src/command/container.rs

@@ -7,16 +7,16 @@ use futures_util::{
   FutureExt, TryStreamExt, stream::FuturesUnordered,
 };
 use komodo_client::{
-  api::read::{
-    InspectDockerContainer, ListAllDockerContainers, ListServers,
-  },
+  api::read::{self, ListAllContainers, ListServers},
   entities::{
     config::cli::args::container::{
       Container, ContainerCommand, InspectContainer,
     },
     docker::{
       self,
-      container::{ContainerListItem, ContainerStateStatusEnum},
+      container::{
+        ContainerListItem, ContainerSortBy, ContainerStateStatusEnum,
+      },
     },
   },
 };
@@ -44,11 +44,12 @@ async fn list_containers(
     down,
     links,
     reverse,
-    containers: names,
+    containers: terms,
     images,
     networks,
     servers,
     format,
+    page,
     command: _,
   }: &Container,
 ) -> anyhow::Result<()> {
@@ -60,9 +61,16 @@ async fn list_containers(
         .into_iter()
         .map(|s| (s.id.clone(), s))
         .collect::<HashMap<_, _>>())),
-    client.read(ListAllDockerContainers {
+    client.read(ListAllContainers {
       servers: Default::default(),
-      containers: Default::default(),
+      tags: Default::default(),
+      terms: terms.clone(),
+      state: Default::default(),
+      limit: Some(100),
+      // Page is more naturally given starting as 1, 2, 3.
+      page: if *page == 0 { 0 } else { *page - 1 },
+      sort_by: ContainerSortBy::Server,
+      sort_desc: false,
     }),
   )?;
 
@@ -78,7 +86,6 @@ async fn list_containers(
     (Some(server.name.as_str()), c)
   });
 
-  let names = parse_wildcards(names);
   let servers = parse_wildcards(servers);
   let images = parse_wildcards(images);
   let networks = parse_wildcards(networks);
@@ -105,7 +112,6 @@ async fn list_containers(
       );
       state_check
         && network_check
-        && matches_wildcards(&names, &[c.name.as_str()])
         && matches_wildcards(
           &servers,
           &server_name
@@ -145,9 +151,15 @@ pub async fn inspect_container(
         .into_iter()
         .map(|s| (s.id.clone(), s))
         .collect::<HashMap<_, _>>())),
-    client.read(ListAllDockerContainers {
-      servers: Default::default(),
-      containers: Default::default()
+    client.read(ListAllContainers {
+      servers: inspect.servers.clone(),
+      tags: Default::default(),
+      terms: vec![inspect.container.clone()],
+      state: Default::default(),
+      limit: Some(0),
+      page: 0,
+      sort_by: ContainerSortBy::Server,
+      sort_desc: false,
     }),
   )?;
 
@@ -162,25 +174,11 @@ pub async fn inspect_container(
     c.server_id = Some(server.name.clone());
   });
 
-  let names = [inspect.container.to_string()];
-  let names = parse_wildcards(&names);
-  let servers = parse_wildcards(&inspect.servers);
-
   let mut containers = containers
     .into_iter()
-    .filter(|c| {
-      matches_wildcards(&names, &[c.name.as_str()])
-        && matches_wildcards(
-          &servers,
-          &c.server_id
-            .as_deref()
-            .map(|i| vec![i])
-            .unwrap_or_default(),
-        )
-    })
     .map(|c| async move {
       client
-        .read(InspectDockerContainer {
+        .read(read::InspectContainer {
           container: c.name,
           server: c.server_id.context("No server...")?,
         })

bin/cli/src/command/execute.rs

@@ -28,558 +28,141 @@ pub async fn handle(
   }
 
   println!("\n{}: Execution", "Mode".dimmed());
-  match execution {
-    Execution::None(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::RunAction(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::BatchRunAction(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::RunProcedure(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::BatchRunProcedure(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::RunBuild(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::BatchRunBuild(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::CancelBuild(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::Deploy(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::BatchDeploy(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::PullDeployment(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::StartDeployment(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::RestartDeployment(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::PauseDeployment(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::UnpauseDeployment(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::StopDeployment(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::DestroyDeployment(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::BatchDestroyDeployment(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::CloneRepo(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::BatchCloneRepo(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::PullRepo(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::BatchPullRepo(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::BuildRepo(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::BatchBuildRepo(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::CancelRepoBuild(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::StartContainer(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::RestartContainer(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::PauseContainer(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::UnpauseContainer(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::StopContainer(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::DestroyContainer(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::StartAllContainers(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::RestartAllContainers(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::PauseAllContainers(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::UnpauseAllContainers(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::StopAllContainers(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::PruneContainers(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::DeleteNetwork(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::PruneNetworks(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::DeleteImage(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::PruneImages(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::DeleteVolume(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::PruneVolumes(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::PruneDockerBuilders(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::PruneBuildx(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::PruneSystem(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::RunSync(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::CommitSync(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::DeployStack(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::BatchDeployStack(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::DeployStackIfChanged(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::BatchDeployStackIfChanged(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::PullStack(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::BatchPullStack(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::StartStack(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::RestartStack(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::PauseStack(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::UnpauseStack(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::StopStack(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::DestroyStack(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::BatchDestroyStack(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::RunStackService(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::TestAlerter(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::SendAlert(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::RemoveSwarmNodes(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::RemoveSwarmStacks(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::RemoveSwarmServices(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::CreateSwarmConfig(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::RotateSwarmConfig(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::RemoveSwarmConfigs(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::CreateSwarmSecret(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::RotateSwarmSecret(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::RemoveSwarmSecrets(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::ClearRepoCache(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::BackupCoreDatabase(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::GlobalAutoUpdate(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::RotateAllServerKeys(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::RotateCoreKeys(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::Sleep(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
+
+  macro_rules! handle_execution {
+    (
+      execute: [$($ExecVariant:ident),* $(,)?],
+      batch: [$($BatchVariant:ident),* $(,)?],
+    ) => {{
+      // Print data
+      match execution {
+        $(Execution::$ExecVariant(data) => println!("{}: {data:?}", "Data".dimmed()),)*
+        $(Execution::$BatchVariant(data) => println!("{}: {data:?}", "Data".dimmed()),)*
+        Execution::CommitSync(data) => println!("{}: {data:?}", "Data".dimmed()),
+        Execution::Sleep(data) => println!("{}: {data:?}", "Data".dimmed()),
+        Execution::None(data) => println!("{}: {data:?}", "Data".dimmed()),
+      }
+
+      $crate::command::wait_for_enter("run execution", yes)?;
+
+      info!("Running Execution...");
+
+      let client = $crate::command::komodo_client().await?;
+
+      // Execute and get result
+      match execution.clone() {
+        $(
+          Execution::$ExecVariant(request) => client
+            .execute(request)
+            .await
+            .map(|u| ExecutionResult::Single(u.into())),
+        )*
+        $(
+          Execution::$BatchVariant(request) => {
+            client.execute(request).await.map(ExecutionResult::Batch)
+          }
+        )*
+        Execution::CommitSync(request) => client
+          .write(request)
+          .await
+          .map(|u| ExecutionResult::Single(u.into())),
+        Execution::Sleep(request) => {
+          let duration =
+            Duration::from_millis(request.duration_ms as u64);
+          tokio::time::sleep(duration).await;
+          println!("Finished sleeping!");
+          std::process::exit(0)
+        }
+        Execution::None(_) => unreachable!(),
+      }
+    }};
   }
 
-  super::wait_for_enter("run execution", yes)?;
-
-  info!("Running Execution...");
-
-  let client = super::komodo_client().await?;
-
-  let res = match execution.clone() {
-    Execution::RunAction(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::BatchRunAction(request) => {
-      client.execute(request).await.map(ExecutionResult::Batch)
-    }
-    Execution::RunProcedure(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::BatchRunProcedure(request) => {
-      client.execute(request).await.map(ExecutionResult::Batch)
-    }
-    Execution::RunBuild(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::BatchRunBuild(request) => {
-      client.execute(request).await.map(ExecutionResult::Batch)
-    }
-    Execution::CancelBuild(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::Deploy(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::BatchDeploy(request) => {
-      client.execute(request).await.map(ExecutionResult::Batch)
-    }
-    Execution::PullDeployment(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::StartDeployment(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::RestartDeployment(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::PauseDeployment(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::UnpauseDeployment(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::StopDeployment(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::DestroyDeployment(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::BatchDestroyDeployment(request) => {
-      client.execute(request).await.map(ExecutionResult::Batch)
-    }
-    Execution::CloneRepo(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::BatchCloneRepo(request) => {
-      client.execute(request).await.map(ExecutionResult::Batch)
-    }
-    Execution::PullRepo(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::BatchPullRepo(request) => {
-      client.execute(request).await.map(ExecutionResult::Batch)
-    }
-    Execution::BuildRepo(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::BatchBuildRepo(request) => {
-      client.execute(request).await.map(ExecutionResult::Batch)
-    }
-    Execution::CancelRepoBuild(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::StartContainer(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::RestartContainer(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::PauseContainer(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::UnpauseContainer(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::StopContainer(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::DestroyContainer(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::StartAllContainers(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::RestartAllContainers(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::PauseAllContainers(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::UnpauseAllContainers(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::StopAllContainers(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::PruneContainers(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::DeleteNetwork(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::PruneNetworks(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::DeleteImage(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::PruneImages(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::DeleteVolume(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::PruneVolumes(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::PruneDockerBuilders(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::PruneBuildx(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::PruneSystem(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::RunSync(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::CommitSync(request) => client
-      .write(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::DeployStack(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::BatchDeployStack(request) => {
-      client.execute(request).await.map(ExecutionResult::Batch)
-    }
-    Execution::DeployStackIfChanged(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::BatchDeployStackIfChanged(request) => {
-      client.execute(request).await.map(ExecutionResult::Batch)
-    }
-    Execution::PullStack(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::BatchPullStack(request) => {
-      client.execute(request).await.map(ExecutionResult::Batch)
-    }
-    Execution::StartStack(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::RestartStack(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::PauseStack(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::UnpauseStack(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::StopStack(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::DestroyStack(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::BatchDestroyStack(request) => {
-      client.execute(request).await.map(ExecutionResult::Batch)
-    }
-    Execution::RunStackService(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::TestAlerter(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::SendAlert(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::RemoveSwarmNodes(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::RemoveSwarmStacks(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::RemoveSwarmServices(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::CreateSwarmConfig(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::RotateSwarmConfig(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::RemoveSwarmConfigs(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::CreateSwarmSecret(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::RotateSwarmSecret(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::RemoveSwarmSecrets(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::ClearRepoCache(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::BackupCoreDatabase(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::GlobalAutoUpdate(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::RotateAllServerKeys(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::RotateCoreKeys(request) => client
-      .execute(request)
-      .await
-      .map(|u| ExecutionResult::Single(u.into())),
-    Execution::Sleep(request) => {
-      let duration =
-        Duration::from_millis(request.duration_ms as u64);
-      tokio::time::sleep(duration).await;
-      println!("Finished sleeping!");
-      std::process::exit(0)
-    }
-    Execution::None(_) => unreachable!(),
-  };
+  let res = handle_execution!(
+    execute: [
+      RunAction,
+      CancelAction,
+      RunProcedure,
+      CancelProcedure,
+      RunBuild,
+      CancelBuild,
+      Deploy,
+      PullDeployment,
+      StartDeployment,
+      RestartDeployment,
+      PauseDeployment,
+      UnpauseDeployment,
+      StopDeployment,
+      DestroyDeployment,
+      CloneRepo,
+      PullRepo,
+      BuildRepo,
+      CancelRepoBuild,
+      StartContainer,
+      RestartContainer,
+      PauseContainer,
+      UnpauseContainer,
+      StopContainer,
+      DestroyContainer,
+      StartAllContainers,
+      RestartAllContainers,
+      PauseAllContainers,
+      UnpauseAllContainers,
+      StopAllContainers,
+      PruneContainers,
+      DeleteNetwork,
+      PruneNetworks,
+      DeleteImage,
+      PruneImages,
+      DeleteVolume,
+      PruneVolumes,
+      PruneDockerBuilders,
+      PruneBuildx,
+      PruneSystem,
+      RunSync,
+      DeployStack,
+      DeployStackIfChanged,
+      PullStack,
+      StartStack,
+      RestartStack,
+      PauseStack,
+      UnpauseStack,
+      StopStack,
+      DestroyStack,
+      RunStackService,
+      TestAlerter,
+      SendAlert,
+      RemoveSwarmNodes,
+      UpdateSwarmNode,
+      RemoveSwarmStacks,
+      RemoveSwarmServices,
+      CreateSwarmConfig,
+      RotateSwarmConfig,
+      RemoveSwarmConfigs,
+      CreateSwarmSecret,
+      RotateSwarmSecret,
+      RemoveSwarmSecrets,
+      ClearRepoCache,
+      BackupCoreDatabase,
+      GlobalAutoUpdate,
+      RotateAllServerKeys,
+      RotateCoreKeys,
+    ],
+    batch: [
+      BatchRunAction,
+      BatchRunProcedure,
+      BatchRunBuild,
+      BatchDeploy,
+      BatchDestroyDeployment,
+      BatchCloneRepo,
+      BatchPullRepo,
+      BatchBuildRepo,
+      BatchDeployStack,
+      BatchDeployStackIfChanged,
+      BatchPullStack,
+      BatchDestroyStack,
+    ],
+  );
 
   match res {
     Ok(ExecutionResult::Single(update)) => {

bin/cli/src/command/list.rs

@@ -7,7 +7,8 @@ use komodo_client::{
   api::read::{
     ListActions, ListAlerters, ListBuilders, ListBuilds,
     ListDeployments, ListProcedures, ListRepos, ListResourceSyncs,
-    ListSchedules, ListServers, ListStacks, ListTags, ListTerminals,
+    ListSchedules, ListServers, ListStacks, ListSwarms, ListTags,
+    ListTerminals,
   },
   entities::{
     ResourceTargetVariant,
@@ -31,6 +32,7 @@ use komodo_client::{
     schedule::Schedule,
     server::{ServerListItem, ServerListItemInfo, ServerState},
     stack::{StackListItem, StackListItemInfo, StackState},
+    swarm::{SwarmListItem, SwarmListItemInfo, SwarmState},
     sync::{
       ResourceSyncListItem, ResourceSyncListItemInfo,
       ResourceSyncState,
@@ -52,28 +54,46 @@ pub async fn handle(list: &args::list::List) -> anyhow::Result<()> {
   match &list.command {
     None => list_all(list).await,
     Some(ListCommand::Servers(filters)) => {
-      list_resources::<ServerListItem>(filters, false).await
+      list_resources::<ServerListItem>(filters, list.page, false)
+        .await
+    }
+    Some(ListCommand::Swarms(filters)) => {
+      list_resources::<SwarmListItem>(filters, list.page, false).await
     }
     Some(ListCommand::Stacks(filters)) => {
-      list_resources::<StackListItem>(filters, false).await
+      list_resources::<StackListItem>(filters, list.page, false).await
     }
     Some(ListCommand::Deployments(filters)) => {
-      list_resources::<DeploymentListItem>(filters, false).await
+      list_resources::<DeploymentListItem>(filters, list.page, false)
+        .await
     }
     Some(ListCommand::Builds(filters)) => {
-      list_resources::<BuildListItem>(filters, false).await
+      list_resources::<BuildListItem>(filters, list.page, false).await
     }
     Some(ListCommand::Repos(filters)) => {
-      list_resources::<RepoListItem>(filters, false).await
+      list_resources::<RepoListItem>(filters, list.page, false).await
     }
     Some(ListCommand::Procedures(filters)) => {
-      list_resources::<ProcedureListItem>(filters, false).await
+      list_resources::<ProcedureListItem>(filters, list.page, false)
+        .await
     }
     Some(ListCommand::Actions(filters)) => {
-      list_resources::<ActionListItem>(filters, false).await
+      list_resources::<ActionListItem>(filters, list.page, false)
+        .await
     }
     Some(ListCommand::Syncs(filters)) => {
-      list_resources::<ResourceSyncListItem>(filters, false).await
+      list_resources::<ResourceSyncListItem>(
+        filters, list.page, false,
+      )
+      .await
+    }
+    Some(ListCommand::Builders(filters)) => {
+      list_resources::<BuilderListItem>(filters, list.page, false)
+        .await
+    }
+    Some(ListCommand::Alerters(filters)) => {
+      list_resources::<AlerterListItem>(filters, list.page, false)
+        .await
     }
     Some(ListCommand::Terminals(filters)) => {
       list_terminals(filters).await
@@ -81,12 +101,6 @@ pub async fn handle(list: &args::list::List) -> anyhow::Result<()> {
     Some(ListCommand::Schedules(filters)) => {
       list_schedules(filters).await
     }
-    Some(ListCommand::Builders(filters)) => {
-      list_resources::<BuilderListItem>(filters, false).await
-    }
-    Some(ListCommand::Alerters(filters)) => {
-      list_resources::<AlerterListItem>(filters, false).await
-    }
   }
 }
 
@@ -97,6 +111,7 @@ async fn list_all(list: &args::list::List) -> anyhow::Result<()> {
   let (
     tags,
     mut servers,
+    mut swarms,
     mut stacks,
     mut deployments,
     mut builds,
@@ -109,14 +124,15 @@ async fn list_all(list: &args::list::List) -> anyhow::Result<()> {
       .into_iter()
       .map(|t| (t.id, t.name))
       .collect::<HashMap<_, _>>())),
-    ServerListItem::list(client, &filters, true),
-    StackListItem::list(client, &filters, true),
-    DeploymentListItem::list(client, &filters, true),
-    BuildListItem::list(client, &filters, true),
-    RepoListItem::list(client, &filters, true),
-    ProcedureListItem::list(client, &filters, true),
-    ActionListItem::list(client, &filters, true),
-    ResourceSyncListItem::list(client, &filters, true),
+    ServerListItem::list(client, &filters, list.page, true),
+    SwarmListItem::list(client, &filters, list.page, true),
+    StackListItem::list(client, &filters, list.page, true),
+    DeploymentListItem::list(client, &filters, list.page, true),
+    BuildListItem::list(client, &filters, list.page, true),
+    RepoListItem::list(client, &filters, list.page, true),
+    ProcedureListItem::list(client, &filters, list.page, true),
+    ActionListItem::list(client, &filters, list.page, true),
+    ResourceSyncListItem::list(client, &filters, list.page, true),
   )?;
 
   if !servers.is_empty() {
@@ -125,6 +141,12 @@ async fn list_all(list: &args::list::List) -> anyhow::Result<()> {
     println!();
   }
 
+  if !swarms.is_empty() {
+    fix_tags(&mut swarms, &tags);
+    print_items(swarms, filters.format, list.links)?;
+    println!();
+  }
+
   if !stacks.is_empty() {
     fix_tags(&mut stacks, &tags);
     print_items(stacks, filters.format, list.links)?;
@@ -172,6 +194,7 @@ async fn list_all(list: &args::list::List) -> anyhow::Result<()> {
 
 async fn list_resources<T>(
   filters: &ResourceFilters,
+  page: u64,
   minimal: bool,
 ) -> anyhow::Result<()>
 where
@@ -180,7 +203,7 @@ where
 {
   let client = crate::command::komodo_client().await?;
   let (mut resources, tags) = tokio::try_join!(
-    T::list(client, filters, minimal),
+    T::list(client, filters, page, minimal),
     client.read(ListTags::default()).map(|res| res.map(|res| res
       .into_iter()
       .map(|t| (t.id, t.name))
@@ -205,6 +228,8 @@ async fn list_terminals(
     .read(ListTerminals {
       target: None,
       use_names: true,
+      limit: Some(0),
+      ..Default::default()
     })
     .await?;
   if !terminals.is_empty() {
@@ -222,6 +247,8 @@ async fn list_schedules(
       .read(ListSchedules {
         tags: filters.tags.clone(),
         tag_behavior: Default::default(),
+        limit: Some(0),
+        ..Default::default()
       })
       .map(|res| res.map(|res| res
         .into_iter()
@@ -279,6 +306,7 @@ where
   async fn list(
     client: &KomodoClient,
     filters: &ResourceFilters,
+    page: u64,
     // For use with root `km ls`
     minimal: bool,
   ) -> anyhow::Result<Vec<ResourceListItem<Self::Info>>>;
@@ -291,6 +319,7 @@ impl ListResources for ServerListItem {
   async fn list(
     client: &KomodoClient,
     filters: &ResourceFilters,
+    page: u64,
     _minimal: bool,
   ) -> anyhow::Result<Vec<Self>> {
     let servers = client
@@ -300,6 +329,10 @@ impl ListResources for ServerListItem {
           // .tag_behavior(TagQueryBehavior::Any)
           .templates(filters.templates)
           .build(),
+        limit: None,
+        page: page.saturating_sub(1),
+        sort_by: Default::default(),
+        sort_desc: false,
       })
       .await?;
     let names = parse_wildcards(&filters.names);
@@ -329,17 +362,68 @@ impl ListResources for ServerListItem {
   }
 }
 
+impl ListResources for SwarmListItem {
+  type Info = SwarmListItemInfo;
+  async fn list(
+    client: &KomodoClient,
+    filters: &ResourceFilters,
+    page: u64,
+    _minimal: bool,
+  ) -> anyhow::Result<Vec<Self>> {
+    let servers = client
+      .read(ListSwarms {
+        query: ResourceQuery::builder()
+          .tags(filters.tags.clone())
+          // .tag_behavior(TagQueryBehavior::Any)
+          .templates(filters.templates)
+          .build(),
+        limit: None,
+        page: page.saturating_sub(1),
+        sort_by: Default::default(),
+        sort_desc: false,
+      })
+      .await?;
+    let names = parse_wildcards(&filters.names);
+    let server_wildcards = parse_wildcards(&filters.servers);
+    let mut servers = servers
+      .into_iter()
+      .filter(|server| {
+        let state_check = if filters.all {
+          true
+        } else if filters.down {
+          !matches!(server.info.state, SwarmState::Healthy)
+        } else if filters.in_progress {
+          false
+        } else {
+          matches!(server.info.state, SwarmState::Healthy)
+        };
+        let name_items = &[server.name.as_str()];
+        state_check
+          && matches_wildcards(&names, name_items)
+          && matches_wildcards(&server_wildcards, name_items)
+      })
+      .collect::<Vec<_>>();
+    servers.sort_by(|a, b| {
+      a.info.state.cmp(&b.info.state).then(a.name.cmp(&b.name))
+    });
+    Ok(servers)
+  }
+}
+
 impl ListResources for StackListItem {
   type Info = StackListItemInfo;
   async fn list(
     client: &KomodoClient,
     filters: &ResourceFilters,
+    page: u64,
     _minimal: bool,
   ) -> anyhow::Result<Vec<Self>> {
     let (servers, mut stacks) = tokio::try_join!(
       client
         .read(ListServers {
           query: ResourceQuery::builder().build(),
+          limit: Some(0),
+          ..Default::default()
         })
         .map(|res| res.map(|res| res
           .into_iter()
@@ -351,6 +435,10 @@ impl ListResources for StackListItem {
           // .tag_behavior(TagQueryBehavior::Any)
           .templates(filters.templates)
           .build(),
+        limit: None,
+        page: page.saturating_sub(1),
+        sort_by: Default::default(),
+        sort_desc: false
       })
     )?;
     stacks.iter_mut().for_each(|stack| {
@@ -406,12 +494,15 @@ impl ListResources for DeploymentListItem {
   async fn list(
     client: &KomodoClient,
     filters: &ResourceFilters,
+    page: u64,
     _minimal: bool,
   ) -> anyhow::Result<Vec<Self>> {
     let (servers, mut deployments) = tokio::try_join!(
       client
         .read(ListServers {
           query: ResourceQuery::builder().build(),
+          limit: Some(0),
+          ..Default::default()
         })
         .map(|res| res.map(|res| res
           .into_iter()
@@ -423,6 +514,10 @@ impl ListResources for DeploymentListItem {
           // .tag_behavior(TagQueryBehavior::Any)
           .templates(filters.templates)
           .build(),
+        limit: None,
+        page: page.saturating_sub(1),
+        sort_by: Default::default(),
+        sort_desc: false
       })
     )?;
     deployments.iter_mut().for_each(|deployment| {
@@ -479,12 +574,15 @@ impl ListResources for BuildListItem {
   async fn list(
     client: &KomodoClient,
     filters: &ResourceFilters,
+    page: u64,
     minimal: bool,
   ) -> anyhow::Result<Vec<Self>> {
     let (builders, mut builds) = tokio::try_join!(
       client
         .read(ListBuilders {
           query: ResourceQuery::builder().build(),
+          limit: Some(0),
+          ..Default::default()
         })
         .map(|res| res.map(|res| res
           .into_iter()
@@ -496,6 +594,10 @@ impl ListResources for BuildListItem {
           // .tag_behavior(TagQueryBehavior::Any)
           .templates(filters.templates)
           .build(),
+        limit: None,
+        page: page.saturating_sub(1),
+        sort_by: Default::default(),
+        sort_desc: false
       })
     )?;
     builds.iter_mut().for_each(|build| {
@@ -533,10 +635,7 @@ impl ListResources for BuildListItem {
       })
       .collect::<Vec<_>>();
     builds.sort_by(|a, b| {
-      a.name
-        .cmp(&b.name)
-        .then(a.info.builder_id.cmp(&b.info.builder_id))
-        .then(a.info.state.cmp(&b.info.state))
+      a.info.state.cmp(&b.info.state).then(a.name.cmp(&b.name))
     });
     Ok(builds)
   }
@@ -547,6 +646,7 @@ impl ListResources for RepoListItem {
   async fn list(
     client: &KomodoClient,
     filters: &ResourceFilters,
+    page: u64,
     minimal: bool,
   ) -> anyhow::Result<Vec<Self>> {
     let names = parse_wildcards(&filters.names);
@@ -557,6 +657,10 @@ impl ListResources for RepoListItem {
           // .tag_behavior(TagQueryBehavior::Any)
           .templates(filters.templates)
           .build(),
+        limit: None,
+        page: page.saturating_sub(1),
+        sort_by: Default::default(),
+        sort_desc: false,
       })
       .await?
       .into_iter()
@@ -595,6 +699,7 @@ impl ListResources for ProcedureListItem {
   async fn list(
     client: &KomodoClient,
     filters: &ResourceFilters,
+    page: u64,
     minimal: bool,
   ) -> anyhow::Result<Vec<Self>> {
     let names = parse_wildcards(&filters.names);
@@ -605,6 +710,10 @@ impl ListResources for ProcedureListItem {
           // .tag_behavior(TagQueryBehavior::Any)
           .templates(filters.templates)
           .build(),
+        limit: None,
+        page: page.saturating_sub(1),
+        sort_by: Default::default(),
+        sort_desc: false,
       })
       .await?
       .into_iter()
@@ -632,7 +741,7 @@ impl ListResources for ProcedureListItem {
         (Some(a), Some(b)) => return a.cmp(&b),
         (None, None) => {}
       }
-      a.name.cmp(&b.name).then(a.info.state.cmp(&b.info.state))
+      a.info.state.cmp(&b.info.state).then(a.name.cmp(&b.name))
     });
     Ok(procedures)
   }
@@ -643,6 +752,7 @@ impl ListResources for ActionListItem {
   async fn list(
     client: &KomodoClient,
     filters: &ResourceFilters,
+    page: u64,
     minimal: bool,
   ) -> anyhow::Result<Vec<Self>> {
     let names = parse_wildcards(&filters.names);
@@ -653,6 +763,10 @@ impl ListResources for ActionListItem {
           // .tag_behavior(TagQueryBehavior::Any)
           .templates(filters.templates)
           .build(),
+        limit: None,
+        page: page.saturating_sub(1),
+        sort_by: Default::default(),
+        sort_desc: false,
       })
       .await?
       .into_iter()
@@ -680,7 +794,7 @@ impl ListResources for ActionListItem {
         (Some(a), Some(b)) => return a.cmp(&b),
         (None, None) => {}
       }
-      a.name.cmp(&b.name).then(a.info.state.cmp(&b.info.state))
+      a.info.state.cmp(&b.info.state).then(a.name.cmp(&b.name))
     });
     Ok(actions)
   }
@@ -691,6 +805,7 @@ impl ListResources for ResourceSyncListItem {
   async fn list(
     client: &KomodoClient,
     filters: &ResourceFilters,
+    page: u64,
     minimal: bool,
   ) -> anyhow::Result<Vec<Self>> {
     let names = parse_wildcards(&filters.names);
@@ -701,6 +816,10 @@ impl ListResources for ResourceSyncListItem {
           // .tag_behavior(TagQueryBehavior::Any)
           .templates(filters.templates)
           .build(),
+        limit: None,
+        page: page.saturating_sub(1),
+        sort_by: Default::default(),
+        sort_desc: false,
       })
       .await?
       .into_iter()
@@ -725,7 +844,7 @@ impl ListResources for ResourceSyncListItem {
       })
       .collect::<Vec<_>>();
     syncs.sort_by(|a, b| {
-      a.name.cmp(&b.name).then(a.info.state.cmp(&b.info.state))
+      a.info.state.cmp(&b.info.state).then(a.name.cmp(&b.name))
     });
     Ok(syncs)
   }
@@ -736,6 +855,7 @@ impl ListResources for BuilderListItem {
   async fn list(
     client: &KomodoClient,
     filters: &ResourceFilters,
+    page: u64,
     minimal: bool,
   ) -> anyhow::Result<Vec<Self>> {
     let names = parse_wildcards(&filters.names);
@@ -746,6 +866,10 @@ impl ListResources for BuilderListItem {
           // .tag_behavior(TagQueryBehavior::Any)
           .templates(filters.templates)
           .build(),
+        limit: None,
+        page: page.saturating_sub(1),
+        sort_by: Default::default(),
+        sort_desc: false,
       })
       .await?
       .into_iter()
@@ -768,6 +892,7 @@ impl ListResources for AlerterListItem {
   async fn list(
     client: &KomodoClient,
     filters: &ResourceFilters,
+    page: u64,
     minimal: bool,
   ) -> anyhow::Result<Vec<Self>> {
     let names = parse_wildcards(&filters.names);
@@ -778,6 +903,10 @@ impl ListResources for AlerterListItem {
           // .tag_behavior(TagQueryBehavior::Any)
           .templates(filters.templates)
           .build(),
+        limit: None,
+        page: page.saturating_sub(1),
+        sort_by: Default::default(),
+        sort_desc: false,
       })
       .await?
       .into_iter()
@@ -832,6 +961,39 @@ impl PrintTable for ResourceListItem<ServerListItemInfo> {
   }
 }
 
+impl PrintTable for ResourceListItem<SwarmListItemInfo> {
+  fn header(links: bool) -> &'static [&'static str] {
+    if links {
+      &["Swarm", "State", "Tags", "Link"]
+    } else {
+      &["Swarm", "State", "Tags"]
+    }
+  }
+  fn row(self, links: bool) -> Vec<Cell> {
+    let color = match self.info.state {
+      SwarmState::Healthy => Color::Green,
+      SwarmState::Unhealthy => Color::Red,
+      SwarmState::Down => Color::Blue,
+      SwarmState::Unknown => Color::Magenta,
+    };
+    let mut res = vec![
+      Cell::new(self.name).add_attribute(Attribute::Bold),
+      Cell::new(self.info.state.to_string())
+        .fg(color)
+        .add_attribute(Attribute::Bold),
+      Cell::new(self.tags.join(", ")),
+    ];
+    if links {
+      res.push(Cell::new(resource_link(
+        &cli_config().host,
+        ResourceTargetVariant::Server,
+        &self.id,
+      )))
+    }
+    res
+  }
+}
+
 impl PrintTable for ResourceListItem<StackListItemInfo> {
   fn header(links: bool) -> &'static [&'static str] {
     if links {

bin/cli/src/command/terminal.rs

@@ -2,7 +2,7 @@ use anyhow::{Context, anyhow};
 use colored::Colorize;
 use komodo_client::{
   api::{
-    read::{ListAllDockerContainers, ListServers},
+    read::{GetServer, ListAllContainers, ListServers},
     terminal::InitTerminal,
   },
   entities::{
@@ -118,6 +118,8 @@ async fn get_server(
   server: Option<String>,
   container: &str,
 ) -> anyhow::Result<String> {
+  // Don't need to find server for terminal,
+  // user provided it, early return.
   if let Some(server) = server {
     return Ok(server);
   }
@@ -125,9 +127,15 @@ async fn get_server(
   let client = super::komodo_client().await?;
 
   let mut containers = client
-    .read(ListAllDockerContainers {
+    .read(ListAllContainers {
       servers: Default::default(),
-      containers: vec![container.to_string()],
+      tags: Default::default(),
+      terms: vec![container.to_string()],
+      state: Default::default(),
+      limit: Some(0),
+      page: 0,
+      sort_by: Default::default(),
+      sort_desc: false,
     })
     .await?;
 
@@ -138,11 +146,14 @@ async fn get_server(
   }
 
   if containers.len() == 1 {
-    return containers
+    let server_id = containers
       .pop()
       .context("Shouldn't happen")?
       .server_id
-      .context("Container doesn't have server_id");
+      .context("Container doesn't have server_id")?;
+    let server_name =
+      client.read(GetServer { server: server_id }).await?.name;
+    return Ok(server_name);
   }
 
   let servers = containers
@@ -153,6 +164,8 @@ async fn get_server(
   let servers = client
     .read(ListServers {
       query: ServerQuery::builder().names(servers).build(),
+      limit: Some(0),
+      ..Default::default()
     })
     .await?
     .into_iter()

bin/cli/src/config.rs

@@ -264,6 +264,9 @@ pub fn cli_config() -> &'static CliConfig {
         ansi: env
           .komodo_cli_logging_ansi
           .unwrap_or(config.cli_logging.ansi),
+        timestamps: env
+          .komodo_cli_logging_timestamps
+          .unwrap_or(config.cli_logging.timestamps),
         otlp_endpoint: env
           .komodo_cli_logging_otlp_endpoint
           .unwrap_or(config.cli_logging.otlp_endpoint),

bin/core/Cargo.toml

@@ -2,7 +2,6 @@
 name = "komodo_core"
 version.workspace = true
 edition.workspace = true
-authors.workspace = true
 license.workspace = true
 homepage.workspace = true
 repository.workspace = true

bin/core/aio.Dockerfile

@@ -1,7 +1,7 @@
 ## All in one, multi stage compile + runtime Docker build for your architecture.
 
 # Build Core
-FROM rust:1.94.0-trixie AS core-builder
+FROM rust:1.97.1-trixie AS core-builder
 RUN cargo install cargo-strip
 
 WORKDIR /builder
@@ -11,6 +11,7 @@ COPY ./client/core/rs ./client/core/rs
 COPY ./client/periphery ./client/periphery
 COPY ./bin/core ./bin/core
 COPY ./bin/cli ./bin/cli
+COPY ./xtask ./xtask
 
 # Compile app
 RUN cargo build -p komodo_core --release && \
@@ -58,7 +59,8 @@ ENV KOMODO_CLI_CONFIG_PATHS="/config"
 # This ensures any `komodo.cli.*` takes precedence over the Core `/config/*config.*`
 ENV KOMODO_CLI_CONFIG_KEYWORDS="*config.*,*komodo.cli*.*"
 
-CMD [ "/bin/bash", "-c", "update-ca-certificates && core" ]
+ENTRYPOINT [ "entrypoint.sh" ]
+CMD [ "core" ]
 
 # Label to prevent Komodo from stopping with StopAllContainers
 LABEL komodo.skip="true"

bin/core/src/alert/discord.rs

@@ -27,7 +27,7 @@ pub async fn send_alert(
         SeverityLevel::Critical => {
           let err = err
             .as_ref()
-            .map(|e| format!("\n**error**: {e}"))
+            .map(|e| format!("\n**error**: {e:#?}"))
             .unwrap_or_default();
           format!(
             "{level} | Swarm **{name}** is **unhealthy** ❌\n{link}{err}"
@@ -36,28 +36,6 @@ pub async fn send_alert(
         _ => unreachable!(),
       }
     }
-    AlertData::ServerVersionMismatch {
-      id,
-      name,
-      region,
-      server_version,
-      core_version,
-    } => {
-      let region = fmt_region(region);
-      let link = resource_link(ResourceTargetVariant::Server, id);
-      match alert.level {
-        SeverityLevel::Ok => {
-          format!(
-            "{level} | **{name}**{region} | Periphery version now matches Core version ✅\n{link}"
-          )
-        }
-        _ => {
-          format!(
-            "{level} | **{name}**{region} | Version mismatch detected ⚠️\nPeriphery: **{server_version}** | Core: **{core_version}**\n{link}"
-          )
-        }
-      }
-    }
     AlertData::ServerUnreachable {
       id,
       name,
@@ -84,6 +62,28 @@ pub async fn send_alert(
         _ => unreachable!(),
       }
     }
+    AlertData::ServerVersionMismatch {
+      id,
+      name,
+      region,
+      server_version,
+      core_version,
+    } => {
+      let region = fmt_region(region);
+      let link = resource_link(ResourceTargetVariant::Server, id);
+      match alert.level {
+        SeverityLevel::Ok => {
+          format!(
+            "{level} | **{name}**{region} | Periphery version now matches Core version ✅\n{link}"
+          )
+        }
+        _ => {
+          format!(
+            "{level} | **{name}**{region} | Version mismatch detected ⚠️\nPeriphery: **{server_version}** | Core: **{core_version}**\n{link}"
+          )
+        }
+      }
+    }
     AlertData::ServerCpu {
       id,
       name,

bin/core/src/alert/mod.rs

@@ -260,7 +260,7 @@ fn standard_alert_content(alert: &Alert) -> String {
         SeverityLevel::Critical => {
           let err = err
             .as_ref()
-            .map(|e| format!("\nerror: {e}"))
+            .map(|e| format!("\nerror: {e:#?}"))
             .unwrap_or_default();
           format!(
             "{level} | Swarm {name} is unhealthy ❌\n{link}{err}"
@@ -269,6 +269,30 @@ fn standard_alert_content(alert: &Alert) -> String {
         _ => unreachable!(),
       }
     }
+    AlertData::ServerUnreachable {
+      id,
+      name,
+      region,
+      err,
+    } => {
+      let region = fmt_region(region);
+      let link = resource_link(ResourceTargetVariant::Server, id);
+      match alert.level {
+        SeverityLevel::Ok => {
+          format!("{level} | {name}{region} is now connected\n{link}")
+        }
+        SeverityLevel::Critical => {
+          let err = err
+            .as_ref()
+            .map(|e| format!("\nerror: {e:#?}"))
+            .unwrap_or_default();
+          format!(
+            "{level} | {name}{region} is unreachable ❌\n{link}{err}"
+          )
+        }
+        _ => unreachable!(),
+      }
+    }
     AlertData::ServerVersionMismatch {
       id,
       name,
@@ -291,30 +315,6 @@ fn standard_alert_content(alert: &Alert) -> String {
         }
       }
     }
-    AlertData::ServerUnreachable {
-      id,
-      name,
-      region,
-      err,
-    } => {
-      let region = fmt_region(region);
-      let link = resource_link(ResourceTargetVariant::Server, id);
-      match alert.level {
-        SeverityLevel::Ok => {
-          format!("{level} | {name}{region} is now connected\n{link}")
-        }
-        SeverityLevel::Critical => {
-          let err = err
-            .as_ref()
-            .map(|e| format!("\nerror: {e:#?}"))
-            .unwrap_or_default();
-          format!(
-            "{level} | {name}{region} is unreachable ❌\n{link}{err}"
-          )
-        }
-        _ => unreachable!(),
-      }
-    }
     AlertData::ServerCpu {
       id,
       name,

bin/core/src/alert/slack.rs

@@ -42,7 +42,7 @@ pub async fn send_alert(
             format!("{level} | Swarm *{name}* is *unhealthy* ❌");
           let err = err
             .as_ref()
-            .map(|e| format!("\nerror: {e}"))
+            .map(|e| format!("\nerror: {e:#?}"))
             .unwrap_or_default();
           let blocks = vec![
             Block::header(level),
@@ -59,35 +59,6 @@ pub async fn send_alert(
         _ => unreachable!(),
       }
     }
-    AlertData::ServerVersionMismatch {
-      id,
-      name,
-      region,
-      server_version,
-      core_version,
-    } => {
-      let region = fmt_region(region);
-      let text = match alert.level {
-        SeverityLevel::Ok => {
-          format!(
-            "{level} | *{name}*{region} | Periphery version now matches Core version ✅"
-          )
-        }
-        _ => {
-          format!(
-            "{level} | *{name}*{region} | Version mismatch detected ⚠️\nPeriphery: {server_version} | Core: {core_version}"
-          )
-        }
-      };
-      let blocks = vec![
-        Block::header(text.clone()),
-        Block::section(resource_link(
-          ResourceTargetVariant::Server,
-          id,
-        )),
-      ];
-      (text, blocks.into())
-    }
     AlertData::ServerUnreachable {
       id,
       name,
@@ -129,6 +100,35 @@ pub async fn send_alert(
         _ => unreachable!(),
       }
     }
+    AlertData::ServerVersionMismatch {
+      id,
+      name,
+      region,
+      server_version,
+      core_version,
+    } => {
+      let region = fmt_region(region);
+      let text = match alert.level {
+        SeverityLevel::Ok => {
+          format!(
+            "{level} | *{name}*{region} | Periphery version now matches Core version ✅"
+          )
+        }
+        _ => {
+          format!(
+            "{level} | *{name}*{region} | Version mismatch detected ⚠️\nPeriphery: {server_version} | Core: {core_version}"
+          )
+        }
+      };
+      let blocks = vec![
+        Block::header(text.clone()),
+        Block::section(resource_link(
+          ResourceTargetVariant::Server,
+          id,
+        )),
+      ];
+      (text, blocks.into())
+    }
     AlertData::ServerCpu {
       id,
       name,

bin/core/src/api/execute/action.rs

@@ -4,23 +4,26 @@ use std::{
   sync::OnceLock,
 };
 
-use anyhow::Context;
-use command::run_komodo_standard_command;
-use database::mungos::{
-  by_id::update_one_by_id, mongodb::bson::to_document,
+use anyhow::Context as _;
+use command::{CommandOptions, run_komodo_standard_command};
+use database::{
+  bson::doc,
+  mungos::{by_id::update_one_by_id, mongodb::bson::to_document},
 };
 use interpolate::Interpolator;
 use komodo_client::{
-  api::execute::{BatchExecutionResponse, BatchRunAction, RunAction},
+  api::execute::{
+    BatchExecutionResponse, BatchRunAction, CancelAction, RunAction,
+  },
   entities::{
-    FileFormat, JsonObject,
+    FileFormat, JsonObject, Operation,
     action::Action,
     alert::{Alert, AlertData, SeverityLevel},
     config::core::CoreConfig,
     komodo_timestamp,
     permission::PermissionLevel,
     random_string,
-    update::Update,
+    update::{Update, UpdateStatus},
     user::action_user,
   },
   parsers::parse_key_value_list,
@@ -34,6 +37,7 @@ use mogh_auth_server::api::manage::api_key::{
 use mogh_config::merge_objects;
 use mogh_resolver::Resolve;
 use tokio::fs;
+use tokio_util::sync::CancellationToken;
 
 use crate::{
   alert::send_alerts,
@@ -46,7 +50,7 @@ use crate::{
   },
   permission::get_check_permissions,
   resource::refresh_action_state_cache,
-  state::{action_states, db_client},
+  state::{action_cancel_cache, action_states, db_client},
 };
 
 use super::ExecuteArgs;
@@ -69,6 +73,7 @@ impl Resolve<ExecuteArgs> for BatchRunAction {
       task_id = task_id.to_string(),
       operator = user.id,
       pattern = self.pattern,
+      tags = self.tags.join(","),
     )
   )]
   async fn resolve(
@@ -76,8 +81,12 @@ impl Resolve<ExecuteArgs> for BatchRunAction {
     ExecuteArgs { user, task_id, .. }: &ExecuteArgs,
   ) -> mogh_error::Result<BatchExecutionResponse> {
     Ok(
-      super::batch_execute::<BatchRunAction>(&self.pattern, user)
-        .await?,
+      super::batch_execute::<BatchRunAction>(
+        &self.pattern,
+        self.tags,
+        user,
+      )
+      .await?,
     )
   }
 }
@@ -116,7 +125,7 @@ impl Resolve<ExecuteArgs> for RunAction {
 
     // This will set action state back to default when dropped.
     // Will also check to ensure action not already busy before updating.
-    let _action_guard = action_state.update_custom(
+    let action_guard = action_state.update_custom(
       |state| state.running += 1,
       |state| state.running -= 1,
       false,
@@ -197,14 +206,20 @@ impl Resolve<ExecuteArgs> for RunAction {
         ""
       };
 
+      let cancel = CancellationToken::new();
+
+      action_cancel_cache()
+        .insert(update.id.clone(), cancel.clone())
+        .await;
+
       let mut res = run_komodo_standard_command(
         // Keep this stage name as is, the UI will find the latest update log by matching the stage name
         "Execute Action",
-        None,
         format!(
           "deno run --allow-all{https_cert_flag}{reload} {}",
           path.display()
         ),
+        CommandOptions::default().cancel(cancel),
       )
       .await;
 
@@ -218,7 +233,7 @@ impl Resolve<ExecuteArgs> for RunAction {
       update.logs.push(res);
       update.finalize();
 
-      mogh_error::Ok(update)
+      mogh_error::Ok(())
     }
     .await;
 
@@ -231,7 +246,13 @@ impl Resolve<ExecuteArgs> for RunAction {
       );
     };
 
-    let update = res?;
+    action_cancel_cache().remove(&update.id).await;
+
+    // Drop action guard before updating
+    // clients to requery action state
+    drop(action_guard);
+
+    res?;
 
     // Need to manually update the update before cache refresh,
     // and before broadcast with update_update.
@@ -459,3 +480,65 @@ fn parse_action_arguments(
       .context("Failed to parse Json to action args"),
   }
 }
+
+impl Resolve<ExecuteArgs> for CancelAction {
+  #[instrument(
+    "CancelAction",
+    skip_all,
+    fields(
+      task_id = task_id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      action = self.action,
+    )
+  )]
+  async fn resolve(
+    self,
+    ExecuteArgs {
+      user,
+      update,
+      task_id,
+    }: &ExecuteArgs,
+  ) -> Result<Self::Response, Self::Error> {
+    let action = get_check_permissions::<Action>(
+      &self.action,
+      user,
+      PermissionLevel::Execute.into(),
+    )
+    .await?;
+
+    let update_id = if let Some(update_id) = self.update_id
+      && !update_id.is_empty()
+    {
+      update_id
+    } else {
+      db_client()
+        .updates
+        .find_one(doc! {
+          "target.type": "Action",
+          "target.id": &action.id,
+          "operation": Operation::RunAction.as_ref(),
+          "status": UpdateStatus::InProgress.as_ref(),
+        })
+        .await?
+        .context("No active run to cancel found")?
+        .id
+    };
+
+    action_cancel_cache()
+      .get(&update_id)
+      .await
+      .context("Action run cancel token not found")?
+      .cancel();
+
+    let mut update = update.clone();
+    update.push_simple_log(
+      "Cancel Triggered",
+      "The action cancel has been triggered.",
+    );
+    update.finalize();
+    update_update(update.clone()).await?;
+
+    Ok(update)
+  }
+}

bin/core/src/api/execute/alerter.rs

@@ -114,6 +114,8 @@ impl Resolve<ExecuteArgs> for SendAlert {
   ) -> Result<Self::Response, Self::Error> {
     let alerters = list_full_for_user::<Alerter>(
       Default::default(),
+      None,
+      None,
       user,
       PermissionLevel::Read.into(),
       &[],

bin/core/src/api/execute/build.rs

@@ -28,7 +28,7 @@ use komodo_client::{
     alert::{Alert, AlertData, SeverityLevel},
     all_logs_success,
     build::{Build, BuildConfig},
-    builder::{Builder, BuilderConfig},
+    builder::Builder,
     deployment::DeploymentState,
     komodo_timestamp, optional_string,
     permission::PermissionLevel,
@@ -78,6 +78,7 @@ impl Resolve<ExecuteArgs> for BatchRunBuild {
       task_id = task_id.to_string(),
       operator = user.id,
       pattern = self.pattern,
+      tags = self.tags.join(","),
     )
   )]
   async fn resolve(
@@ -85,8 +86,12 @@ impl Resolve<ExecuteArgs> for BatchRunBuild {
     ExecuteArgs { user, task_id, .. }: &ExecuteArgs,
   ) -> mogh_error::Result<BatchExecutionResponse> {
     Ok(
-      super::batch_execute::<BatchRunBuild>(&self.pattern, user)
-        .await?,
+      super::batch_execute::<BatchRunBuild>(
+        &self.pattern,
+        self.tags,
+        user,
+      )
+      .await?,
     )
   }
 }
@@ -148,7 +153,7 @@ impl Resolve<ExecuteArgs> for RunBuild {
 
     // This will set action state back to default when dropped.
     // Will also check to ensure build not already busy before updating.
-    let _action_guard =
+    let action_guard =
       action_state.update(|state| state.building = true)?;
 
     if build.config.auto_increment_version {
@@ -175,9 +180,6 @@ impl Resolve<ExecuteArgs> for RunBuild {
     let builder =
       resource::get::<Builder>(&build.config.builder_id).await?;
 
-    let is_server_builder =
-      matches!(&builder.config, BuilderConfig::Server(_));
-
     tokio::spawn(async move {
       let poll = async {
         loop {
@@ -186,19 +188,13 @@ impl Resolve<ExecuteArgs> for RunBuild {
             id = cancel_recv.recv() => id?
           };
           if incoming_build_id == build_id {
-            if is_server_builder {
-              update.push_error_log("Cancel acknowledged", "Build cancellation is not possible on server builders at this time. Use an AWS builder to enable this feature.");
-            } else {
-              update.push_simple_log("Cancel acknowledged", "The build cancellation has been queued, it may still take some time.");
-            }
+            update.push_simple_log("Cancel acknowledged", "The build cancellation has been queued, it may still take some time.");
             update.finalize();
             let id = update.id.clone();
             if let Err(e) = update_update(update).await {
               warn!("Failed to modify Update {id} on db | {e:#}");
             }
-            if !is_server_builder {
-              cancel_clone.cancel();
-            }
+            cancel_clone.cancel();
             return Ok(());
           }
         }
@@ -216,7 +212,7 @@ impl Resolve<ExecuteArgs> for RunBuild {
       build.name.clone(),
       Some(build.config.version),
       builder,
-      &mut update,
+      Some(&mut update),
     )
     .await
     {
@@ -273,11 +269,11 @@ impl Resolve<ExecuteArgs> for RunBuild {
             replacers: Default::default(),
           }) => res,
         _ = cancel.cancelled() => {
-          debug!("Build cancelled during clone, cleaning up builder");
-          update.push_error_log("Build cancelled", String::from("user cancelled build during repo clone"));
+          debug!("Build cancelled during repo clone, cleaning up builder");
+          update.push_error_log("Build cancelled", String::from("Build cancelled during repo clone"));
           cleanup_builder_instance(periphery, cleanup_data, &mut update)
             .await;
-          info!("Builder cleaned up");
+          debug!("Builder cleaned up");
           return handle_early_return(update, build.id, build.name, true).await
         },
       };
@@ -323,7 +319,14 @@ impl Resolve<ExecuteArgs> for RunBuild {
           }) => res.context("Failed at call to Periphery to build"),
         _ = cancel.cancelled() => {
           info!("Build cancelled during build, cleaning up builder");
-          update.push_error_log("Build cancelled", String::from("User cancelled build during docker build"));
+          if let Err(e) = periphery.request(api::build::CancelBuild {
+            id: build.id.clone()
+          })
+          .await
+          .context("Failed to cancel build execution on Server") {
+            update.push_error_log("Cancel Build", format_serror(&e.into()));
+          }
+          update.push_error_log("Build Cancelled", String::from("User cancelled build during image build step"));
           cleanup_builder_instance(periphery, cleanup_data, &mut update)
             .await;
           return handle_early_return(update, build.id, build.name, true).await
@@ -373,6 +376,10 @@ impl Resolve<ExecuteArgs> for RunBuild {
     cleanup_builder_instance(periphery, cleanup_data, &mut update)
       .await;
 
+    // Drop action guard before updating
+    // clients to requery action state
+    drop(action_guard);
+
     // Need to manually update the update before cache refresh,
     // and before broadcast with add_update.
     // The Err case of to_document should be unreachable,
@@ -509,10 +516,10 @@ pub async fn validate_cancel_build(
     )?;
 
     match (latest_build, latest_cancel) {
-      (Some(build), Some(cancel)) => {
-        if cancel.start_ts > build.start_ts {
-          return Err(anyhow!("Build has already been cancelled"));
-        }
+      (Some(build), Some(cancel))
+        if cancel.start_ts > build.start_ts =>
+      {
+        return Err(anyhow!("Build has already been cancelled"));
       }
       (None, _) => return Err(anyhow!("No build in progress")),
       _ => {}
@@ -561,8 +568,8 @@ impl Resolve<ExecuteArgs> for CancelBuild {
     let mut update = update.clone();
 
     update.push_simple_log(
-      "cancel triggered",
-      "the build cancel has been triggered",
+      "Cancel Triggered",
+      "The build cancel has been triggered",
     );
     update_update(update.clone()).await?;
 
@@ -617,7 +624,13 @@ async fn handle_post_build_redeploy(build_id: &str) {
         let state = get_deployment_state(&deployment.id)
           .await
           .unwrap_or_default();
-        if state == DeploymentState::Running {
+        if ![
+          DeploymentState::NotDeployed,
+          DeploymentState::Exited,
+          DeploymentState::Unknown,
+        ]
+        .contains(&state)
+        {
           let req = super::ExecuteRequest::Deploy(Deploy {
             deployment: deployment.id.clone(),
             stop_signal: None,

bin/core/src/api/execute/deployment.rs

@@ -58,6 +58,7 @@ impl Resolve<ExecuteArgs> for BatchDeploy {
       task_id = task_id.to_string(),
       operator = user.id,
       pattern = self.pattern,
+      tags = self.tags.join(","),
     )
   )]
   async fn resolve(
@@ -65,8 +66,12 @@ impl Resolve<ExecuteArgs> for BatchDeploy {
     ExecuteArgs { user, task_id, .. }: &ExecuteArgs,
   ) -> mogh_error::Result<BatchExecutionResponse> {
     Ok(
-      super::batch_execute::<BatchDeploy>(&self.pattern, user)
-        .await?,
+      super::batch_execute::<BatchDeploy>(
+        &self.pattern,
+        self.tags,
+        user,
+      )
+      .await?,
     )
   }
 }
@@ -110,7 +115,7 @@ impl Resolve<ExecuteArgs> for Deploy {
 
     // Will check to ensure deployment not already busy before updating, and return Err if so.
     // The returned guard will set the action state back to default when dropped.
-    let _action_guard =
+    let action_guard =
       action_state.update(|state| state.deploying = true)?;
 
     let mut update = update.clone();
@@ -122,26 +127,16 @@ impl Resolve<ExecuteArgs> for Deploy {
     let (version, registry_token) = match &deployment.config.image {
       DeploymentImage::Build { build_id, version } => {
         let build = resource::get::<Build>(build_id).await?;
-        let image_names = build.get_image_names();
-        let image_name = image_names
-          .first()
-          .context("No image name could be created")
-          .context("Failed to create image name")?;
         let version = if version.is_none() {
           build.config.version
         } else {
           *version
         };
-        let version_str = version.to_string();
-        // Potentially add the build image_tag postfix
-        let version_str = if build.config.image_tag.is_empty() {
-          version_str
-        } else {
-          format!("{version_str}-{}", build.config.image_tag)
-        };
+        let image_name = build.get_deployment_image_name();
+        let image_tag = build.get_deployment_image_tag(version);
         // replace image with corresponding build image.
         deployment.config.image = DeploymentImage::Image {
-          image: format!("{image_name}:{version_str}"),
+          image: format!("{image_name}:{image_tag}"),
         };
         let first_registry = build
           .config
@@ -211,6 +206,12 @@ impl Resolve<ExecuteArgs> for Deploy {
     update_update(update.clone()).await?;
 
     let deployment_id = deployment.id.clone();
+    // Track the name the container / service is deployed under,
+    // so the Deployment stays matched to it even if the
+    // name configuration changes before the next deploy.
+    let fresh_name = deployment.custom_name().to_string();
+    let prev_deployed_name = deployment.info.deployed_name.clone();
+    let mut deployed = false;
 
     match swarm_or_server {
       SwarmOrServer::None => unreachable!(),
@@ -227,6 +228,7 @@ impl Resolve<ExecuteArgs> for Deploy {
         {
           Ok(logs) => {
             refresh_swarm_cache(&swarm, true).await;
+            deployed = logs.iter().all(|log| log.success);
             update.logs.extend(logs)
           }
           Err(e) => {
@@ -251,6 +253,7 @@ impl Resolve<ExecuteArgs> for Deploy {
         {
           Ok(log) => {
             refresh_server_cache(&server, true).await;
+            deployed = log.success;
             update.logs.push(log)
           }
           Err(e) => {
@@ -267,6 +270,11 @@ impl Resolve<ExecuteArgs> for Deploy {
       &deployment_id,
       &DeploymentInfo {
         latest_image_digest: Default::default(),
+        deployed_name: if deployed {
+          fresh_name
+        } else {
+          prev_deployed_name
+        },
       },
     )
     .await
@@ -278,6 +286,11 @@ impl Resolve<ExecuteArgs> for Deploy {
     }
 
     update.finalize();
+
+    // Drop action guard before updating
+    // clients to requery action state
+    drop(action_guard);
+
     update_update(update.clone()).await?;
 
     Ok(update)
@@ -458,7 +471,7 @@ impl Resolve<ExecuteArgs> for PullDeployment {
 
     // Will check to ensure deployment not already busy before updating, and return Err if so.
     // The returned guard will set the action state back to default when dropped.
-    let _action_guard =
+    let action_guard =
       action_state.update(|state| state.pulling = true)?;
 
     let mut update = update.clone();
@@ -469,6 +482,10 @@ impl Resolve<ExecuteArgs> for PullDeployment {
 
     update.logs.push(log);
     update.finalize();
+
+    // Drop action guard before updating
+    // clients to requery action state
+    drop(action_guard);
     update_update(update.clone()).await?;
 
     Ok(update)
@@ -516,7 +533,7 @@ impl Resolve<ExecuteArgs> for StartDeployment {
 
     // Will check to ensure deployment not already busy before updating, and return Err if so.
     // The returned guard will set the action state back to default when dropped.
-    let _action_guard =
+    let action_guard =
       action_state.update(|state| state.starting = true)?;
 
     let mut update = update.clone();
@@ -527,7 +544,7 @@ impl Resolve<ExecuteArgs> for StartDeployment {
     let log = match periphery_client(&server)
       .await?
       .request(api::container::StartContainer {
-        name: deployment.name,
+        name: deployment.deployed_name().to_string(),
       })
       .await
     {
@@ -541,6 +558,10 @@ impl Resolve<ExecuteArgs> for StartDeployment {
     update.logs.push(log);
     refresh_server_cache(&server, true).await;
     update.finalize();
+
+    // Drop action guard before updating
+    // clients to requery action state
+    drop(action_guard);
     update_update(update.clone()).await?;
 
     Ok(update)
@@ -588,7 +609,7 @@ impl Resolve<ExecuteArgs> for RestartDeployment {
 
     // Will check to ensure deployment not already busy before updating, and return Err if so.
     // The returned guard will set the action state back to default when dropped.
-    let _action_guard =
+    let action_guard =
       action_state.update(|state| state.restarting = true)?;
 
     let mut update = update.clone();
@@ -599,7 +620,7 @@ impl Resolve<ExecuteArgs> for RestartDeployment {
     let log = match periphery_client(&server)
       .await?
       .request(api::container::RestartContainer {
-        name: deployment.name,
+        name: deployment.deployed_name().to_string(),
       })
       .await
     {
@@ -615,6 +636,10 @@ impl Resolve<ExecuteArgs> for RestartDeployment {
     update.logs.push(log);
     refresh_server_cache(&server, true).await;
     update.finalize();
+
+    // Drop action guard before updating
+    // clients to requery action state
+    drop(action_guard);
     update_update(update.clone()).await?;
 
     Ok(update)
@@ -662,7 +687,7 @@ impl Resolve<ExecuteArgs> for PauseDeployment {
 
     // Will check to ensure deployment not already busy before updating, and return Err if so.
     // The returned guard will set the action state back to default when dropped.
-    let _action_guard =
+    let action_guard =
       action_state.update(|state| state.pausing = true)?;
 
     let mut update = update.clone();
@@ -673,7 +698,7 @@ impl Resolve<ExecuteArgs> for PauseDeployment {
     let log = match periphery_client(&server)
       .await?
       .request(api::container::PauseContainer {
-        name: deployment.name,
+        name: deployment.deployed_name().to_string(),
       })
       .await
     {
@@ -687,6 +712,10 @@ impl Resolve<ExecuteArgs> for PauseDeployment {
     update.logs.push(log);
     refresh_server_cache(&server, true).await;
     update.finalize();
+
+    // Drop action guard before updating
+    // clients to requery action state
+    drop(action_guard);
     update_update(update.clone()).await?;
 
     Ok(update)
@@ -734,7 +763,7 @@ impl Resolve<ExecuteArgs> for UnpauseDeployment {
 
     // Will check to ensure deployment not already busy before updating, and return Err if so.
     // The returned guard will set the action state back to default when dropped.
-    let _action_guard =
+    let action_guard =
       action_state.update(|state| state.unpausing = true)?;
 
     let mut update = update.clone();
@@ -745,7 +774,7 @@ impl Resolve<ExecuteArgs> for UnpauseDeployment {
     let log = match periphery_client(&server)
       .await?
       .request(api::container::UnpauseContainer {
-        name: deployment.name,
+        name: deployment.deployed_name().to_string(),
       })
       .await
     {
@@ -761,6 +790,10 @@ impl Resolve<ExecuteArgs> for UnpauseDeployment {
     update.logs.push(log);
     refresh_server_cache(&server, true).await;
     update.finalize();
+
+    // Drop action guard before updating
+    // clients to requery action state
+    drop(action_guard);
     update_update(update.clone()).await?;
 
     Ok(update)
@@ -810,7 +843,7 @@ impl Resolve<ExecuteArgs> for StopDeployment {
 
     // Will check to ensure deployment not already busy before updating, and return Err if so.
     // The returned guard will set the action state back to default when dropped.
-    let _action_guard =
+    let action_guard =
       action_state.update(|state| state.stopping = true)?;
 
     let mut update = update.clone();
@@ -821,7 +854,7 @@ impl Resolve<ExecuteArgs> for StopDeployment {
     let log = match periphery_client(&server)
       .await?
       .request(api::container::StopContainer {
-        name: deployment.name,
+        name: deployment.deployed_name().to_string(),
         signal: self
           .signal
           .unwrap_or(deployment.config.termination_signal)
@@ -843,6 +876,10 @@ impl Resolve<ExecuteArgs> for StopDeployment {
     update.logs.push(log);
     refresh_server_cache(&server, true).await;
     update.finalize();
+
+    // Drop action guard before updating
+    // clients to requery action state
+    drop(action_guard);
     update_update(update.clone()).await?;
 
     Ok(update)
@@ -868,6 +905,7 @@ impl Resolve<ExecuteArgs> for BatchDestroyDeployment {
       task_id = task_id.to_string(),
       operator = user.id,
       pattern = self.pattern,
+      tags = self.tags.join(","),
     )
   )]
   async fn resolve(
@@ -877,6 +915,7 @@ impl Resolve<ExecuteArgs> for BatchDestroyDeployment {
     Ok(
       super::batch_execute::<BatchDestroyDeployment>(
         &self.pattern,
+        self.tags,
         user,
       )
       .await?,
@@ -922,7 +961,7 @@ impl Resolve<ExecuteArgs> for DestroyDeployment {
 
     // Will check to ensure deployment not already busy before updating, and return Err if so.
     // The returned guard will set the action state back to default when dropped.
-    let _action_guard =
+    let action_guard =
       action_state.update(|state| state.destroying = true)?;
 
     let mut update = update.clone();
@@ -936,7 +975,7 @@ impl Resolve<ExecuteArgs> for DestroyDeployment {
         match swarm_request(
           &swarm.config.server_ids,
           api::swarm::RemoveSwarmServices {
-            services: vec![deployment.name],
+            services: vec![deployment.deployed_name().to_string()],
           },
         )
         .await
@@ -957,7 +996,7 @@ impl Resolve<ExecuteArgs> for DestroyDeployment {
         match periphery_client(&server)
           .await?
           .request(api::container::RemoveContainer {
-            name: deployment.name,
+            name: deployment.deployed_name().to_string(),
             signal: self
               .signal
               .unwrap_or(deployment.config.termination_signal)
@@ -983,9 +1022,33 @@ impl Resolve<ExecuteArgs> for DestroyDeployment {
       }
     };
 
+    // Clear the tracked deployed name once the container / service
+    // is confirmed removed, so the next deploy uses the fresh name.
+    if log.success
+      && let Err(e) = resource::update_info::<Deployment>(
+        &deployment.id,
+        &DeploymentInfo {
+          latest_image_digest: deployment
+            .info
+            .latest_image_digest
+            .clone(),
+          deployed_name: Default::default(),
+        },
+      )
+      .await
+    {
+      warn!(
+        "Failed to clear deployment {} ({}) deployed name after destroy | {e:#}",
+        deployment.name, deployment.id,
+      );
+    }
+
     update.logs.push(log);
     update.finalize();
 
+    // Drop action guard before updating
+    // clients to requery action state
+    drop(action_guard);
     update_update(update.clone()).await?;
 
     Ok(update)

bin/core/src/api/execute/maintenance.rs

@@ -1,7 +1,7 @@
 use std::{fmt::Write as _, sync::OnceLock};
 
 use anyhow::{Context, anyhow};
-use command::run_komodo_standard_command;
+use command::{CommandOptions, run_komodo_standard_command};
 use database::{
   bson::{Document, doc},
   mungos::find::find_collect,
@@ -15,7 +15,7 @@ use komodo_client::{
   },
   entities::{
     SwarmOrServer, deployment::DeploymentState, server::ServerState,
-    stack::StackState,
+    stack::StackState, swarm::SwarmState,
   },
 };
 use mogh_error::AddStatusCodeError;
@@ -39,7 +39,7 @@ use crate::{
   resource::rotate_server_keys,
   state::{
     db_client, deployment_status_cache, server_status_cache,
-    stack_status_cache,
+    stack_status_cache, swarm_status_cache,
   },
 };
 
@@ -167,8 +167,8 @@ impl Resolve<ExecuteArgs> for BackupCoreDatabase {
 
     let res = run_komodo_standard_command(
       "Backup Core Database",
-      None,
       "km database backup --yes",
+      CommandOptions::default(),
     )
     .await;
 
@@ -244,6 +244,7 @@ impl Resolve<ExecuteArgs> for GlobalAutoUpdate {
         .context("Failed to query for stacks from database")?;
 
     let server_status_cache = server_status_cache();
+    let swarm_status_cache = swarm_status_cache();
     let stack_status_cache = stack_status_cache();
 
     // Will be edited later at update.logs[0]
@@ -255,7 +256,7 @@ impl Resolve<ExecuteArgs> for GlobalAutoUpdate {
         continue;
       };
 
-      // Only pull running stacks.
+      // Only check running stacks.
       if !matches!(status.curr.state, StackState::Running) {
         continue;
       }
@@ -267,43 +268,58 @@ impl Resolve<ExecuteArgs> for GlobalAutoUpdate {
         &servers,
       )?;
 
-      if let SwarmOrServer::None = &swarm_or_server {
-        continue;
+      // Ensure server / swarm is reachable
+      match &swarm_or_server {
+        SwarmOrServer::None => continue,
+        SwarmOrServer::Server(server) => {
+          if !server_status_cache
+            .get(&server.id)
+            .await
+            .map(|s| matches!(s.state, ServerState::Ok))
+            .unwrap_or_default()
+          {
+            continue;
+          }
+        }
+        SwarmOrServer::Swarm(swarm) => {
+          if !swarm_status_cache
+            .get(&swarm.id)
+            .await
+            .map(|s| {
+              matches!(
+                s.state,
+                SwarmState::Healthy | SwarmState::Unhealthy
+              )
+            })
+            .unwrap_or_default()
+          {
+            continue;
+          }
+        }
       }
 
-      if let Some(server) =
-        servers.iter().find(|s| s.id == stack.config.server_id)
-        // This check is probably redundant along with running check
-        // but shouldn't hurt
-        && server_status_cache
-          .get(&server.id)
-          .await
-          .map(|s| matches!(s.state, ServerState::Ok))
-          .unwrap_or_default()
+      if let Err(e) = check_stack_for_update_inner(
+        stack.id,
+        &swarm_or_server,
+        self.skip_auto_update,
+        true,
+        false,
+      )
+      .await
       {
-        if let Err(e) = check_stack_for_update_inner(
-          stack.id,
-          &swarm_or_server,
-          self.skip_auto_update,
-          true,
-          false,
-        )
-        .await
-        {
-          update.push_error_log(
-            &format!("Check Stack {}", stack.name),
-            format_serror(&e.into()),
-          );
-        } else {
-          if !update.logs[0].stdout.is_empty() {
-            update.logs[0].stdout.push('\n');
-          }
-
-          update.logs[0].stdout.push_str(&format!(
-            "Checked Stack {} ✅",
-            bold(&stack.name)
-          ));
+        update.push_error_log(
+          &format!("Check Stack {}", stack.name),
+          format_serror(&e.into()),
+        );
+      } else {
+        if !update.logs[0].stdout.is_empty() {
+          update.logs[0].stdout.push('\n');
         }
+
+        update.logs[0].stdout.push_str(&format!(
+          "Checked Stack {} ✅",
+          bold(&stack.name)
+        ));
       }
     }
 
@@ -320,7 +336,7 @@ impl Resolve<ExecuteArgs> for GlobalAutoUpdate {
         continue;
       };
 
-      // Only pull running deployments.
+      // Only check running deployments.
       if !matches!(status.curr.state, DeploymentState::Running) {
         continue;
       }
@@ -332,8 +348,34 @@ impl Resolve<ExecuteArgs> for GlobalAutoUpdate {
         &servers,
       )?;
 
-      if let SwarmOrServer::None = &swarm_or_server {
-        continue;
+      // Ensure server / swarm is reachable
+      match &swarm_or_server {
+        SwarmOrServer::None => continue,
+        SwarmOrServer::Server(server) => {
+          if !server_status_cache
+            .get(&server.id)
+            .await
+            .map(|s| matches!(s.state, ServerState::Ok))
+            .unwrap_or_default()
+          {
+            continue;
+          }
+        }
+        SwarmOrServer::Swarm(swarm) => {
+          if !swarm_status_cache
+            .get(&swarm.id)
+            .await
+            .map(|s| {
+              matches!(
+                s.state,
+                SwarmState::Healthy | SwarmState::Unhealthy
+              )
+            })
+            .unwrap_or_default()
+          {
+            continue;
+          }
+        }
       }
 
       let name = deployment.name.clone();

bin/core/src/api/execute/mod.rs

@@ -13,6 +13,7 @@ use komodo_client::{
   entities::{
     Operation,
     permission::PermissionLevel,
+    resource::ResourceQuery,
     update::{Log, Update},
     user::User,
   },
@@ -29,7 +30,10 @@ use uuid::Uuid;
 
 use crate::{
   auth::KomodoAuthImpl,
-  helpers::update::{init_execution_update, update_update},
+  helpers::{
+    query::get_all_tags,
+    update::{init_execution_update, update_update},
+  },
   resource::{KomodoResource, list_full_for_user_using_pattern},
   state::db_client,
 };
@@ -111,10 +115,12 @@ pub enum ExecuteRequest {
   // ==== PROCEDURE ====
   RunProcedure(RunProcedure),
   BatchRunProcedure(BatchRunProcedure),
+  CancelProcedure(CancelProcedure),
 
   // ==== ACTION ====
   RunAction(RunAction),
   BatchRunAction(BatchRunAction),
+  CancelAction(CancelAction),
 
   // ==== SYNC ====
   RunSync(RunSync),
@@ -148,6 +154,7 @@ pub enum ExecuteRequest {
 
   // ==== SWARM ====
   RemoveSwarmNodes(RemoveSwarmNodes),
+  UpdateSwarmNode(UpdateSwarmNode),
   RemoveSwarmStacks(RemoveSwarmStacks),
   RemoveSwarmServices(RemoveSwarmServices),
   CreateSwarmConfig(CreateSwarmConfig),
@@ -355,14 +362,26 @@ trait BatchExecute {
 #[instrument("BatchExecute", skip(user))]
 async fn batch_execute<E: BatchExecute>(
   pattern: &str,
+  tags: Vec<String>,
   user: &User,
 ) -> anyhow::Result<BatchExecutionResponse> {
+  let all_tags = if tags.is_empty() {
+    vec![]
+  } else {
+    get_all_tags(None).await?
+  };
+
   let resources = list_full_for_user_using_pattern::<E::Resource>(
     pattern,
-    Default::default(),
+    ResourceQuery {
+      tags,
+      ..Default::default()
+    },
+    None,
+    None,
     user,
     PermissionLevel::Execute.into(),
-    &[],
+    &all_tags,
   )
   .await?;
 

bin/core/src/api/execute/procedure.rs

@@ -1,12 +1,14 @@
 use std::pin::Pin;
 
+use anyhow::Context as _;
 use database::mungos::{
   by_id::update_one_by_id, mongodb::bson::to_document,
 };
 use formatting::{Color, bold, colored, format_serror, muted};
 use komodo_client::{
   api::execute::{
-    BatchExecutionResponse, BatchRunProcedure, RunProcedure,
+    BatchExecutionResponse, BatchRunProcedure, CancelProcedure,
+    RunProcedure,
   },
   entities::{
     alert::{Alert, AlertData, SeverityLevel},
@@ -19,13 +21,14 @@ use komodo_client::{
 };
 use mogh_resolver::Resolve;
 use tokio::sync::Mutex;
+use tokio_util::sync::CancellationToken;
 
 use crate::{
   alert::send_alerts,
   helpers::{procedure::execute_procedure, update::update_update},
   permission::get_check_permissions,
   resource::refresh_procedure_state_cache,
-  state::{action_states, db_client},
+  state::{action_states, db_client, procedure_cancel_cache},
 };
 
 use super::{ExecuteArgs, ExecuteRequest};
@@ -48,8 +51,12 @@ impl Resolve<ExecuteArgs> for BatchRunProcedure {
     ExecuteArgs { user, .. }: &ExecuteArgs,
   ) -> mogh_error::Result<BatchExecutionResponse> {
     Ok(
-      super::batch_execute::<BatchRunProcedure>(&self.pattern, user)
-        .await?,
+      super::batch_execute::<BatchRunProcedure>(
+        &self.pattern,
+        self.tags,
+        user,
+      )
+      .await?,
     )
   }
 }
@@ -117,14 +124,22 @@ fn resolve_inner(
 
     // This will set action state back to default when dropped.
     // Will also check to ensure procedure not already busy before updating.
-    let _action_guard =
+    let action_guard =
       action_state.update(|state| state.running = true)?;
 
     update_update(update.clone()).await?;
 
+    let cancel = CancellationToken::new();
+
+    procedure_cancel_cache()
+      .insert(procedure.id.clone(), cancel.clone())
+      .await;
+
     let update = Mutex::new(update);
 
-    let res = execute_procedure(&procedure, &update).await;
+    let res = execute_procedure(&procedure, &update, cancel).await;
+
+    procedure_cancel_cache().remove(&procedure.id).await;
 
     let mut update = update.into_inner();
 
@@ -140,11 +155,15 @@ fn resolve_inner(
         );
       }
       Err(e) => update
-        .push_error_log("execution error", format_serror(&e.into())),
+        .push_error_log("Execution error", format_serror(&e.into())),
     }
 
     update.finalize();
 
+    // Drop action guard before updating
+    // clients to requery action state
+    drop(action_guard);
+
     // Need to manually update the update before cache refresh,
     // and before broadcast with add_update.
     // The Err case of to_document should be unreachable,
@@ -184,3 +203,47 @@ fn resolve_inner(
     Ok(update)
   })
 }
+
+impl Resolve<ExecuteArgs> for CancelProcedure {
+  #[instrument(
+    "CancelProcedure",
+    skip_all,
+    fields(
+      task_id = task_id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      procedure = self.procedure,
+    )
+  )]
+  async fn resolve(
+    self,
+    ExecuteArgs {
+      user,
+      update,
+      task_id,
+    }: &ExecuteArgs,
+  ) -> Result<Self::Response, Self::Error> {
+    let procedure = get_check_permissions::<Procedure>(
+      &self.procedure,
+      user,
+      PermissionLevel::Execute.into(),
+    )
+    .await?;
+
+    procedure_cancel_cache()
+      .get(&procedure.id)
+      .await
+      .context("Procedure run cancel token not found")?
+      .cancel();
+
+    let mut update = update.clone();
+    update.push_simple_log(
+      "Cancel Triggered",
+      "Procedure cancel has been triggered. The procedure will exit after the currently running stage is complete.",
+    );
+    update.finalize();
+    update_update(update.clone()).await?;
+
+    Ok(update)
+  }
+}

bin/core/src/api/execute/repo.rs

@@ -58,6 +58,7 @@ impl Resolve<ExecuteArgs> for BatchCloneRepo {
       task_id = task_id.to_string(),
       operator = user.id,
       pattern = self.pattern,
+      tags = self.tags.join(","),
     )
   )]
   async fn resolve(
@@ -65,8 +66,12 @@ impl Resolve<ExecuteArgs> for BatchCloneRepo {
     ExecuteArgs { user, task_id, .. }: &ExecuteArgs,
   ) -> mogh_error::Result<BatchExecutionResponse> {
     Ok(
-      super::batch_execute::<BatchCloneRepo>(&self.pattern, user)
-        .await?,
+      super::batch_execute::<BatchCloneRepo>(
+        &self.pattern,
+        self.tags,
+        user,
+      )
+      .await?,
     )
   }
 }
@@ -103,7 +108,7 @@ impl Resolve<ExecuteArgs> for CloneRepo {
 
     // This will set action state back to default when dropped.
     // Will also check to ensure repo not already busy before updating.
-    let _action_guard =
+    let action_guard =
       action_state.update(|state| state.cloning = true)?;
 
     let mut update = update.clone();
@@ -174,6 +179,10 @@ impl Resolve<ExecuteArgs> for CloneRepo {
       );
     };
 
+    // Drop action guard before updating
+    // clients to requery action state
+    drop(action_guard);
+
     handle_repo_update_return(update).await
   }
 }
@@ -192,7 +201,8 @@ impl Resolve<ExecuteArgs> for BatchPullRepo {
     fields(
       task_id = task_id.to_string(),
       operator = user.id,
-      pattern = self.pattern
+      pattern = self.pattern,
+      tags = self.tags.join(","),
     )
   )]
   async fn resolve(
@@ -200,8 +210,12 @@ impl Resolve<ExecuteArgs> for BatchPullRepo {
     ExecuteArgs { user, task_id, .. }: &ExecuteArgs,
   ) -> mogh_error::Result<BatchExecutionResponse> {
     Ok(
-      super::batch_execute::<BatchPullRepo>(&self.pattern, user)
-        .await?,
+      super::batch_execute::<BatchPullRepo>(
+        &self.pattern,
+        self.tags,
+        user,
+      )
+      .await?,
     )
   }
 }
@@ -238,7 +252,7 @@ impl Resolve<ExecuteArgs> for PullRepo {
 
     // This will set action state back to default when dropped.
     // Will also check to ensure repo not already busy before updating.
-    let _action_guard =
+    let action_guard =
       action_state.update(|state| state.pulling = true)?;
 
     let mut update = update.clone();
@@ -313,6 +327,10 @@ impl Resolve<ExecuteArgs> for PullRepo {
       );
     };
 
+    // Drop action guard before updating
+    // clients to requery action state
+    drop(action_guard);
+
     handle_repo_update_return(update).await
   }
 }
@@ -374,6 +392,7 @@ impl Resolve<ExecuteArgs> for BatchBuildRepo {
       task_id = task_id.to_string(),
       operator = user.id,
       pattern = self.pattern,
+      tags = self.tags.join(","),
     )
   )]
   async fn resolve(
@@ -381,8 +400,12 @@ impl Resolve<ExecuteArgs> for BatchBuildRepo {
     ExecuteArgs { user, task_id, .. }: &ExecuteArgs,
   ) -> mogh_error::Result<BatchExecutionResponse> {
     Ok(
-      super::batch_execute::<BatchBuildRepo>(&self.pattern, user)
-        .await?,
+      super::batch_execute::<BatchBuildRepo>(
+        &self.pattern,
+        self.tags,
+        user,
+      )
+      .await?,
     )
   }
 }
@@ -423,7 +446,7 @@ impl Resolve<ExecuteArgs> for BuildRepo {
 
     // This will set action state back to default when dropped.
     // Will also check to ensure repo not already busy before updating.
-    let _action_guard =
+    let action_guard =
       action_state.update(|state| state.building = true)?;
 
     let mut update = update.clone();
@@ -490,7 +513,7 @@ impl Resolve<ExecuteArgs> for BuildRepo {
       repo.name.clone(),
       None,
       builder,
-      &mut update,
+      Some(&mut update),
     )
     .await
     {
@@ -501,6 +524,9 @@ impl Resolve<ExecuteArgs> for BuildRepo {
           "get builder",
           format_serror(&e.context("failed to get builder").into()),
         ));
+        // Drop action guard before updating
+        // clients to requery action state
+        drop(action_guard);
         return handle_builder_early_return(
           update, repo.id, repo.name, false,
         )
@@ -533,6 +559,9 @@ impl Resolve<ExecuteArgs> for BuildRepo {
         cleanup_builder_instance(periphery, cleanup_data, &mut update)
           .await;
         info!("builder cleaned up");
+        // Drop action guard before updating
+        // clients to requery action state
+        drop(action_guard);
         return handle_builder_early_return(update, repo.id, repo.name, true).await
       },
     };
@@ -580,6 +609,10 @@ impl Resolve<ExecuteArgs> for BuildRepo {
     cleanup_builder_instance(periphery, cleanup_data, &mut update)
       .await;
 
+    // Drop action guard before updating
+    // clients to requery action state
+    drop(action_guard);
+
     // Need to manually update the update before cache refresh,
     // and before broadcast with add_update.
     // The Err case of to_document should be unreachable,
@@ -698,13 +731,12 @@ pub async fn validate_cancel_repo_build(
     )?;
 
     match (latest_build, latest_cancel) {
-      (Some(build), Some(cancel)) => {
-        if cancel.start_ts > build.start_ts {
-          return Err(anyhow!(
-            "Repo build has already been cancelled"
-          ));
-        }
+      (Some(build), Some(cancel))
+        if cancel.start_ts > build.start_ts =>
+      {
+        return Err(anyhow!("Repo build has already been cancelled"));
       }
+
       (None, _) => return Err(anyhow!("No repo build in progress")),
       _ => {}
     };

bin/core/src/api/execute/server.rs

@@ -56,8 +56,11 @@ impl Resolve<ExecuteArgs> for StartContainer {
 
     // Will check to ensure deployment not already busy before updating, and return Err if so.
     // The returned guard will set the action state back to default when dropped.
-    let _action_guard = action_state
-      .update(|state| state.starting_containers = true)?;
+    let action_guard = action_state.update_custom(
+      |state| state.starting_containers += 1,
+      |state| state.starting_containers -= 1,
+      false,
+    )?;
 
     let mut update = update.clone();
 
@@ -83,6 +86,10 @@ impl Resolve<ExecuteArgs> for StartContainer {
     refresh_server_cache(&server, true).await;
 
     update.finalize();
+
+    // Drop action guard before updating
+    // clients to requery action state
+    drop(action_guard);
     update_update(update.clone()).await?;
 
     Ok(update)
@@ -124,8 +131,11 @@ impl Resolve<ExecuteArgs> for RestartContainer {
 
     // Will check to ensure server not already busy before updating, and return Err if so.
     // The returned guard will set the action state back to default when dropped.
-    let _action_guard = action_state
-      .update(|state| state.restarting_containers = true)?;
+    let action_guard = action_state.update_custom(
+      |state| state.restarting_containers += 1,
+      |state| state.restarting_containers -= 1,
+      false,
+    )?;
 
     let mut update = update.clone();
 
@@ -153,6 +163,10 @@ impl Resolve<ExecuteArgs> for RestartContainer {
     refresh_server_cache(&server, true).await;
 
     update.finalize();
+
+    // Drop action guard before updating
+    // clients to requery action state
+    drop(action_guard);
     update_update(update.clone()).await?;
 
     Ok(update)
@@ -194,8 +208,11 @@ impl Resolve<ExecuteArgs> for PauseContainer {
 
     // Will check to ensure server not already busy before updating, and return Err if so.
     // The returned guard will set the action state back to default when dropped.
-    let _action_guard =
-      action_state.update(|state| state.pausing_containers = true)?;
+    let action_guard = action_state.update_custom(
+      |state| state.pausing_containers += 1,
+      |state| state.pausing_containers -= 1,
+      false,
+    )?;
 
     let mut update = update.clone();
 
@@ -221,6 +238,10 @@ impl Resolve<ExecuteArgs> for PauseContainer {
     refresh_server_cache(&server, true).await;
 
     update.finalize();
+
+    // Drop action guard before updating
+    // clients to requery action state
+    drop(action_guard);
     update_update(update.clone()).await?;
 
     Ok(update)
@@ -262,8 +283,11 @@ impl Resolve<ExecuteArgs> for UnpauseContainer {
 
     // Will check to ensure server not already busy before updating, and return Err if so.
     // The returned guard will set the action state back to default when dropped.
-    let _action_guard = action_state
-      .update(|state| state.unpausing_containers = true)?;
+    let action_guard = action_state.update_custom(
+      |state| state.unpausing_containers += 1,
+      |state| state.unpausing_containers -= 1,
+      false,
+    )?;
 
     let mut update = update.clone();
 
@@ -291,6 +315,10 @@ impl Resolve<ExecuteArgs> for UnpauseContainer {
     refresh_server_cache(&server, true).await;
 
     update.finalize();
+
+    // Drop action guard before updating
+    // clients to requery action state
+    drop(action_guard);
     update_update(update.clone()).await?;
 
     Ok(update)
@@ -334,8 +362,11 @@ impl Resolve<ExecuteArgs> for StopContainer {
 
     // Will check to ensure server not already busy before updating, and return Err if so.
     // The returned guard will set the action state back to default when dropped.
-    let _action_guard = action_state
-      .update(|state| state.stopping_containers = true)?;
+    let action_guard = action_state.update_custom(
+      |state| state.stopping_containers += 1,
+      |state| state.stopping_containers -= 1,
+      false,
+    )?;
 
     let mut update = update.clone();
 
@@ -363,6 +394,10 @@ impl Resolve<ExecuteArgs> for StopContainer {
     refresh_server_cache(&server, true).await;
 
     update.finalize();
+
+    // Drop action guard before updating
+    // clients to requery action state
+    drop(action_guard);
     update_update(update.clone()).await?;
 
     Ok(update)
@@ -412,8 +447,11 @@ impl Resolve<ExecuteArgs> for DestroyContainer {
 
     // Will check to ensure server not already busy before updating, and return Err if so.
     // The returned guard will set the action state back to default when dropped.
-    let _action_guard =
-      action_state.update(|state| state.pruning_containers = true)?;
+    let action_guard = action_state.update_custom(
+      |state| state.destroying_containers += 1,
+      |state| state.destroying_containers -= 1,
+      false,
+    )?;
 
     let mut update = update.clone();
 
@@ -443,6 +481,10 @@ impl Resolve<ExecuteArgs> for DestroyContainer {
     refresh_server_cache(&server, true).await;
 
     update.finalize();
+
+    // Drop action guard before updating
+    // clients to requery action state
+    drop(action_guard);
     update_update(update.clone()).await?;
 
     Ok(update)
@@ -483,8 +525,11 @@ impl Resolve<ExecuteArgs> for StartAllContainers {
 
     // Will check to ensure server not already busy before updating, and return Err if so.
     // The returned guard will set the action state back to default when dropped.
-    let _action_guard = action_state
-      .update(|state| state.starting_containers = true)?;
+    let action_guard = action_state.update_custom(
+      |state| state.starting_containers += 1,
+      |state| state.starting_containers -= 1,
+      false,
+    )?;
 
     let mut update = update.clone();
 
@@ -507,6 +552,10 @@ impl Resolve<ExecuteArgs> for StartAllContainers {
 
     refresh_server_cache(&server, true).await;
     update.finalize();
+
+    // Drop action guard before updating
+    // clients to requery action state
+    drop(action_guard);
     update_update(update.clone()).await?;
 
     Ok(update)
@@ -547,8 +596,11 @@ impl Resolve<ExecuteArgs> for RestartAllContainers {
 
     // Will check to ensure server not already busy before updating, and return Err if so.
     // The returned guard will set the action state back to default when dropped.
-    let _action_guard = action_state
-      .update(|state| state.restarting_containers = true)?;
+    let action_guard = action_state.update_custom(
+      |state| state.restarting_containers += 1,
+      |state| state.restarting_containers -= 1,
+      false,
+    )?;
 
     let mut update = update.clone();
 
@@ -573,6 +625,10 @@ impl Resolve<ExecuteArgs> for RestartAllContainers {
 
     refresh_server_cache(&server, true).await;
     update.finalize();
+
+    // Drop action guard before updating
+    // clients to requery action state
+    drop(action_guard);
     update_update(update.clone()).await?;
 
     Ok(update)
@@ -613,8 +669,11 @@ impl Resolve<ExecuteArgs> for PauseAllContainers {
 
     // Will check to ensure server not already busy before updating, and return Err if so.
     // The returned guard will set the action state back to default when dropped.
-    let _action_guard =
-      action_state.update(|state| state.pausing_containers = true)?;
+    let action_guard = action_state.update_custom(
+      |state| state.pausing_containers += 1,
+      |state| state.pausing_containers -= 1,
+      false,
+    )?;
 
     let mut update = update.clone();
 
@@ -637,6 +696,10 @@ impl Resolve<ExecuteArgs> for PauseAllContainers {
 
     refresh_server_cache(&server, true).await;
     update.finalize();
+
+    // Drop action guard before updating
+    // clients to requery action state
+    drop(action_guard);
     update_update(update.clone()).await?;
 
     Ok(update)
@@ -677,8 +740,11 @@ impl Resolve<ExecuteArgs> for UnpauseAllContainers {
 
     // Will check to ensure server not already busy before updating, and return Err if so.
     // The returned guard will set the action state back to default when dropped.
-    let _action_guard = action_state
-      .update(|state| state.unpausing_containers = true)?;
+    let action_guard = action_state.update_custom(
+      |state| state.unpausing_containers += 1,
+      |state| state.unpausing_containers -= 1,
+      false,
+    )?;
 
     let mut update = update.clone();
 
@@ -703,6 +769,10 @@ impl Resolve<ExecuteArgs> for UnpauseAllContainers {
 
     refresh_server_cache(&server, true).await;
     update.finalize();
+
+    // Drop action guard before updating
+    // clients to requery action state
+    drop(action_guard);
     update_update(update.clone()).await?;
 
     Ok(update)
@@ -743,8 +813,11 @@ impl Resolve<ExecuteArgs> for StopAllContainers {
 
     // Will check to ensure server not already busy before updating, and return Err if so.
     // The returned guard will set the action state back to default when dropped.
-    let _action_guard = action_state
-      .update(|state| state.stopping_containers = true)?;
+    let action_guard = action_state.update_custom(
+      |state| state.stopping_containers += 1,
+      |state| state.stopping_containers -= 1,
+      false,
+    )?;
 
     let mut update = update.clone();
 
@@ -767,6 +840,10 @@ impl Resolve<ExecuteArgs> for StopAllContainers {
 
     refresh_server_cache(&server, true).await;
     update.finalize();
+
+    // Drop action guard before updating
+    // clients to requery action state
+    drop(action_guard);
     update_update(update.clone()).await?;
 
     Ok(update)
@@ -807,7 +884,7 @@ impl Resolve<ExecuteArgs> for PruneContainers {
 
     // Will check to ensure server not already busy before updating, and return Err if so.
     // The returned guard will set the action state back to default when dropped.
-    let _action_guard =
+    let action_guard =
       action_state.update(|state| state.pruning_containers = true)?;
 
     let mut update = update.clone();
@@ -836,6 +913,10 @@ impl Resolve<ExecuteArgs> for PruneContainers {
     refresh_server_cache(&server, true).await;
 
     update.finalize();
+
+    // Drop action guard before updating
+    // clients to requery action state
+    drop(action_guard);
     update_update(update.clone()).await?;
 
     Ok(update)
@@ -941,7 +1022,7 @@ impl Resolve<ExecuteArgs> for PruneNetworks {
 
     // Will check to ensure server not already busy before updating, and return Err if so.
     // The returned guard will set the action state back to default when dropped.
-    let _action_guard =
+    let action_guard =
       action_state.update(|state| state.pruning_networks = true)?;
 
     let mut update = update.clone();
@@ -968,6 +1049,10 @@ impl Resolve<ExecuteArgs> for PruneNetworks {
     refresh_server_cache(&server, true).await;
 
     update.finalize();
+
+    // Drop action guard before updating
+    // clients to requery action state
+    drop(action_guard);
     update_update(update.clone()).await?;
 
     Ok(update)
@@ -1070,7 +1155,7 @@ impl Resolve<ExecuteArgs> for PruneImages {
 
     // Will check to ensure server not already busy before updating, and return Err if so.
     // The returned guard will set the action state back to default when dropped.
-    let _action_guard =
+    let action_guard =
       action_state.update(|state| state.pruning_images = true)?;
 
     let mut update = update.clone();
@@ -1095,6 +1180,10 @@ impl Resolve<ExecuteArgs> for PruneImages {
     refresh_server_cache(&server, true).await;
 
     update.finalize();
+
+    // Drop action guard before updating
+    // clients to requery action state
+    drop(action_guard);
     update_update(update.clone()).await?;
 
     Ok(update)
@@ -1200,7 +1289,7 @@ impl Resolve<ExecuteArgs> for PruneVolumes {
 
     // Will check to ensure server not already busy before updating, and return Err if so.
     // The returned guard will set the action state back to default when dropped.
-    let _action_guard =
+    let action_guard =
       action_state.update(|state| state.pruning_volumes = true)?;
 
     let mut update = update.clone();
@@ -1225,6 +1314,10 @@ impl Resolve<ExecuteArgs> for PruneVolumes {
     refresh_server_cache(&server, true).await;
 
     update.finalize();
+
+    // Drop action guard before updating
+    // clients to requery action state
+    drop(action_guard);
     update_update(update.clone()).await?;
 
     Ok(update)
@@ -1265,7 +1358,7 @@ impl Resolve<ExecuteArgs> for PruneDockerBuilders {
 
     // Will check to ensure server not already busy before updating, and return Err if so.
     // The returned guard will set the action state back to default when dropped.
-    let _action_guard =
+    let action_guard =
       action_state.update(|state| state.pruning_builders = true)?;
 
     let mut update = update.clone();
@@ -1290,6 +1383,10 @@ impl Resolve<ExecuteArgs> for PruneDockerBuilders {
     refresh_server_cache(&server, true).await;
 
     update.finalize();
+
+    // Drop action guard before updating
+    // clients to requery action state
+    drop(action_guard);
     update_update(update.clone()).await?;
 
     Ok(update)
@@ -1330,7 +1427,7 @@ impl Resolve<ExecuteArgs> for PruneBuildx {
 
     // Will check to ensure server not already busy before updating, and return Err if so.
     // The returned guard will set the action state back to default when dropped.
-    let _action_guard =
+    let action_guard =
       action_state.update(|state| state.pruning_buildx = true)?;
 
     let mut update = update.clone();
@@ -1355,6 +1452,10 @@ impl Resolve<ExecuteArgs> for PruneBuildx {
     refresh_server_cache(&server, true).await;
 
     update.finalize();
+
+    // Drop action guard before updating
+    // clients to requery action state
+    drop(action_guard);
     update_update(update.clone()).await?;
 
     Ok(update)
@@ -1395,7 +1496,7 @@ impl Resolve<ExecuteArgs> for PruneSystem {
 
     // Will check to ensure server not already busy before updating, and return Err if so.
     // The returned guard will set the action state back to default when dropped.
-    let _action_guard =
+    let action_guard =
       action_state.update(|state| state.pruning_system = true)?;
 
     let mut update = update.clone();
@@ -1419,6 +1520,10 @@ impl Resolve<ExecuteArgs> for PruneSystem {
     refresh_server_cache(&server, true).await;
 
     update.finalize();
+
+    // Drop action guard before updating
+    // clients to requery action state
+    drop(action_guard);
     update_update(update.clone()).await?;
 
     Ok(update)

bin/core/src/api/execute/stack.rs

@@ -72,6 +72,7 @@ impl Resolve<ExecuteArgs> for BatchDeployStack {
       task_id = task_id.to_string(),
       operator = user.id,
       pattern = self.pattern,
+      tags = self.tags.join(","),
     )
   )]
   async fn resolve(
@@ -79,8 +80,12 @@ impl Resolve<ExecuteArgs> for BatchDeployStack {
     ExecuteArgs { user, task_id, .. }: &ExecuteArgs,
   ) -> mogh_error::Result<BatchExecutionResponse> {
     Ok(
-      super::batch_execute::<BatchDeployStack>(&self.pattern, user)
-        .await?,
+      super::batch_execute::<BatchDeployStack>(
+        &self.pattern,
+        self.tags,
+        user,
+      )
+      .await?,
     )
   }
 }
@@ -131,7 +136,7 @@ impl Resolve<ExecuteArgs> for DeployStack {
 
     // Will check to ensure stack not already busy before updating, and return Err if so.
     // The returned guard will set the action state back to default when dropped.
-    let _action_guard =
+    let action_guard =
       action_state.update(|state| state.deploying = true)?;
 
     let mut update = update.clone();
@@ -325,6 +330,10 @@ impl Resolve<ExecuteArgs> for DeployStack {
     }
 
     update.finalize();
+
+    // Drop action guard before updating
+    // clients to requery action state
+    drop(action_guard);
     update_update(update.clone()).await?;
 
     Ok(update)
@@ -349,6 +358,7 @@ impl Resolve<ExecuteArgs> for BatchDeployStackIfChanged {
       task_id = task_id.to_string(),
       operator = user.id,
       pattern = self.pattern,
+      tags = self.tags.join(","),
     )
   )]
   async fn resolve(
@@ -358,6 +368,7 @@ impl Resolve<ExecuteArgs> for BatchDeployStackIfChanged {
     Ok(
       super::batch_execute::<BatchDeployStackIfChanged>(
         &self.pattern,
+        self.tags,
         user,
       )
       .await?,
@@ -777,6 +788,7 @@ impl Resolve<ExecuteArgs> for BatchPullStack {
       task_id = task_id.to_string(),
       operator = user.id,
       pattern = self.pattern,
+      tags = self.tags.join(","),
     )
   )]
   async fn resolve(
@@ -784,8 +796,12 @@ impl Resolve<ExecuteArgs> for BatchPullStack {
     ExecuteArgs { user, task_id, .. }: &ExecuteArgs,
   ) -> mogh_error::Result<BatchExecutionResponse> {
     Ok(
-      super::batch_execute::<BatchPullStack>(&self.pattern, user)
-        .await?,
+      super::batch_execute::<BatchPullStack>(
+        &self.pattern,
+        self.tags,
+        user,
+      )
+      .await?,
     )
   }
 }
@@ -943,7 +959,7 @@ impl Resolve<ExecuteArgs> for PullStack {
 
     // Will check to ensure stack not already busy before updating, and return Err if so.
     // The returned guard will set the action state back to default when dropped.
-    let _action_guard =
+    let action_guard =
       action_state.update(|state| state.pulling = true)?;
 
     let mut update = update.clone();
@@ -960,6 +976,10 @@ impl Resolve<ExecuteArgs> for PullStack {
 
     update.logs.extend(res.logs);
     update.finalize();
+
+    // Drop action guard before updating
+    // clients to requery action state
+    drop(action_guard);
     update_update(update.clone()).await?;
 
     Ok(update)
@@ -1153,15 +1173,20 @@ impl Resolve<ExecuteArgs> for BatchDestroyStack {
       task_id = task_id.to_string(),
       operator = user.id,
       pattern = self.pattern,
+      tags = self.tags.join(","),
     )
   )]
   async fn resolve(
     self,
     ExecuteArgs { user, task_id, .. }: &ExecuteArgs,
   ) -> mogh_error::Result<BatchExecutionResponse> {
-    super::batch_execute::<BatchDestroyStack>(&self.pattern, user)
-      .await
-      .map_err(Into::into)
+    super::batch_execute::<BatchDestroyStack>(
+      &self.pattern,
+      self.tags,
+      user,
+    )
+    .await
+    .map_err(Into::into)
   }
 }
 
@@ -1214,7 +1239,7 @@ impl Resolve<ExecuteArgs> for DestroyStack {
 
         // Will check to ensure stack not already busy before updating, and return Err if so.
         // The returned guard will set the action state back to default when dropped.
-        let _action_guard =
+        let action_guard =
           action_state.update(|state| state.destroying = true)?;
 
         let mut update = update.clone();
@@ -1244,6 +1269,10 @@ impl Resolve<ExecuteArgs> for DestroyStack {
         refresh_swarm_cache(&swarm, true).await;
 
         update.finalize();
+
+        // Drop action guard before updating
+        // clients to requery action state
+        drop(action_guard);
         update_update(update.clone()).await?;
 
         Ok(update)
@@ -1314,7 +1343,7 @@ impl Resolve<ExecuteArgs> for RunStackService {
     let action_state =
       action_states().stack.get_or_insert_default(&stack.id).await;
 
-    let _action_guard =
+    let action_guard =
       action_state.update(|state| state.deploying = true)?;
 
     let mut update = update.clone();
@@ -1374,6 +1403,10 @@ impl Resolve<ExecuteArgs> for RunStackService {
 
     update.logs.push(log);
     update.finalize();
+
+    // Drop action guard before updating
+    // clients to requery action state
+    drop(action_guard);
     update_update(update.clone()).await?;
 
     Ok(update)

bin/core/src/api/execute/swarm.rs

@@ -1,10 +1,6 @@
 use formatting::format_serror;
 use komodo_client::{
-  api::execute::{
-    CreateSwarmConfig, CreateSwarmSecret, RemoveSwarmConfigs,
-    RemoveSwarmNodes, RemoveSwarmSecrets, RemoveSwarmServices,
-    RemoveSwarmStacks, RotateSwarmConfig, RotateSwarmSecret,
-  },
+  api::execute::*,
   entities::{permission::PermissionLevel, swarm::Swarm},
 };
 use mogh_resolver::Resolve;
@@ -76,6 +72,72 @@ impl Resolve<ExecuteArgs> for RemoveSwarmNodes {
   }
 }
 
+impl Resolve<ExecuteArgs> for UpdateSwarmNode {
+  #[instrument(
+    "UpdateSwarmNode",
+    skip_all,
+    fields(
+      task_id = task_id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      swarm = self.swarm,
+      node = self.node,
+      availability = format!("{:?}", self.availability),
+      label_add = format!("{:?}", self.label_add),
+      label_rm = format!("{:?}", self.label_rm),
+      role = format!("{:?}", self.role),
+    )
+  )]
+  async fn resolve(
+    self,
+    ExecuteArgs {
+      user,
+      update,
+      task_id,
+    }: &ExecuteArgs,
+  ) -> Result<Self::Response, Self::Error> {
+    let swarm = get_check_permissions::<Swarm>(
+      &self.swarm,
+      user,
+      PermissionLevel::Execute.into(),
+    )
+    .await?;
+
+    update_update(update.clone()).await?;
+
+    let mut update = update.clone();
+
+    match swarm_request(
+      &swarm.config.server_ids,
+      periphery_client::api::swarm::UpdateSwarmNode {
+        node: self.node,
+        availability: self.availability,
+        label_add: self.label_add,
+        label_rm: self.label_rm,
+        role: self.role,
+      },
+    )
+    .await
+    {
+      Ok(log) => {
+        update.logs.push(log);
+        refresh_swarm_cache(&swarm, true).await;
+      }
+      Err(e) => update.push_error_log(
+        "Update Swarm Node",
+        format_serror(
+          &e.context("Failed to update swarm node").into(),
+        ),
+      ),
+    };
+
+    update.finalize();
+    update_update(update.clone()).await?;
+
+    Ok(update)
+  }
+}
+
 impl Resolve<ExecuteArgs> for RemoveSwarmStacks {
   #[instrument(
     "RemoveSwarmStacks",

bin/core/src/api/execute/sync.rs

@@ -98,7 +98,7 @@ impl Resolve<ExecuteArgs> for RunSync {
 
     // This will set action state back to default when dropped.
     // Will also check to ensure sync not already busy before updating.
-    let _action_guard =
+    let action_guard =
       action_state.update(|state| state.syncing = true)?;
 
     let mut update = update.clone();
@@ -283,6 +283,7 @@ impl Resolve<ExecuteArgs> for RunSync {
       Default::default()
     };
 
+    // New resource types need to be added here manually.
     if deploy_cache.is_empty()
       && resource_sync_deltas.no_changes()
       && server_deltas.no_changes()
@@ -310,6 +311,10 @@ impl Resolve<ExecuteArgs> for RunSync {
         ),
       );
       update.finalize();
+
+      // Drop action guard before updating
+      // clients to requery action state
+      drop(action_guard);
       update_update(update.clone()).await?;
       return Ok(update);
     }
@@ -339,10 +344,7 @@ impl Resolve<ExecuteArgs> for RunSync {
       )
       .await,
     );
-    maybe_extend(
-      &mut update.logs,
-      ResourceSync::execute_sync_updates(resource_sync_deltas).await,
-    );
+
     maybe_extend(
       &mut update.logs,
       Server::execute_sync_updates(server_deltas).await,
@@ -356,38 +358,43 @@ impl Resolve<ExecuteArgs> for RunSync {
       Action::execute_sync_updates(action_deltas).await,
     );
 
-    // Dependent on server
+    // Depends on server
     maybe_extend(
       &mut update.logs,
       Swarm::execute_sync_updates(swarm_deltas).await,
     );
+    // Depends on server
     maybe_extend(
       &mut update.logs,
       Builder::execute_sync_updates(builder_deltas).await,
     );
+    // Depends on server / builder
     maybe_extend(
       &mut update.logs,
       Repo::execute_sync_updates(repo_deltas).await,
     );
 
-    // Dependant on builder
+    // Depends on builder / repo
     maybe_extend(
       &mut update.logs,
       Build::execute_sync_updates(build_deltas).await,
     );
-
-    // Dependant on server / build
-    maybe_extend(
-      &mut update.logs,
-      Deployment::execute_sync_updates(deployment_deltas).await,
-    );
-    // stack only depends on server, but maybe will depend on build later.
+    // Depends on server / repo
     maybe_extend(
       &mut update.logs,
       Stack::execute_sync_updates(stack_deltas).await,
     );
-
-    // Dependant on everything
+    // Depends on repo
+    maybe_extend(
+      &mut update.logs,
+      ResourceSync::execute_sync_updates(resource_sync_deltas).await,
+    );
+    // Depends on server / build
+    maybe_extend(
+      &mut update.logs,
+      Deployment::execute_sync_updates(deployment_deltas).await,
+    );
+    // Depends on everything
     maybe_extend(
       &mut update.logs,
       Procedure::execute_sync_updates(procedure_deltas).await,
@@ -439,6 +446,10 @@ impl Resolve<ExecuteArgs> for RunSync {
     }
 
     update.finalize();
+
+    // Drop action guard before updating
+    // clients to requery action state
+    drop(action_guard);
     update_update(update.clone()).await?;
 
     Ok(update)

bin/core/src/api/listener/integrations/github.rs

@@ -1,7 +1,7 @@
 use anyhow::{Context, anyhow};
 use axum::http::HeaderMap;
 use hex::ToHex;
-use hmac::{Hmac, Mac};
+use hmac::{Hmac, KeyInit as _, Mac};
 use serde::Deserialize;
 use sha2::Sha256;
 

bin/core/src/api/listener/mod.rs

@@ -1,6 +1,5 @@
 use std::sync::Arc;
 
-use anyhow::anyhow;
 use axum::{Router, http::HeaderMap};
 use komodo_client::entities::resource::Resource;
 use mogh_cache::CloneCache;
@@ -41,12 +40,21 @@ trait VerifySecret {
 /// Implemented on the integration struct, eg [integrations::github::Github]
 trait ExtractBranch {
   fn extract_branch(body: &str) -> anyhow::Result<String>;
-  fn verify_branch(body: &str, expected: &str) -> anyhow::Result<()> {
+  /// Whether the webhook body's branch matches `expected`.
+  /// A mismatch is routine and only logged at debug; errors
+  /// only when the branch cannot be extracted from the body.
+  fn branch_matches(
+    body: &str,
+    expected: &str,
+  ) -> anyhow::Result<bool> {
     let branch = Self::extract_branch(body)?;
     if branch == expected {
-      Ok(())
+      Ok(true)
     } else {
-      Err(anyhow!("request branch does not match expected"))
+      debug!(
+        "Ignoring webhook | push to branch '{branch}' does not match expected branch '{expected}'"
+      );
+      Ok(false)
     }
   }
 }

bin/core/src/api/listener/resources.rs

@@ -1,4 +1,4 @@
-use std::{str::FromStr, sync::OnceLock};
+use std::{str::FromStr, sync::OnceLock, time::Duration};
 
 use anyhow::{Context, anyhow};
 use komodo_client::{
@@ -23,6 +23,7 @@ use crate::{
   },
   helpers::update::init_execution_update,
   resource,
+  state::action_states,
 };
 
 use super::{ANY_BRANCH, ListenerLockCache};
@@ -37,11 +38,6 @@ impl super::CustomSecret for Build {
   }
 }
 
-fn build_locks() -> &'static ListenerLockCache {
-  static BUILD_LOCKS: OnceLock<ListenerLockCache> = OnceLock::new();
-  BUILD_LOCKS.get_or_init(Default::default)
-}
-
 pub async fn handle_build_webhook<B: super::ExtractBranch>(
   build: Build,
   body: String,
@@ -50,12 +46,6 @@ pub async fn handle_build_webhook<B: super::ExtractBranch>(
     return Ok(());
   }
 
-  // Acquire and hold lock to make a task queue for
-  // subsequent listener calls on same resource.
-  // It would fail if we let it go through from action state busy.
-  let lock = build_locks().get_or_insert_default(&build.id).await;
-  let _lock = lock.lock().await;
-
   // Use the correct target branch when using linked repo.
   let branch = if build.config.linked_repo.is_empty() {
     build.config.branch
@@ -67,15 +57,49 @@ pub async fn handle_build_webhook<B: super::ExtractBranch>(
       .branch
   };
 
-  B::verify_branch(&body, &branch)?;
+  if !B::branch_matches(&body, &branch)? {
+    return Ok(());
+  }
+
+  // Cancel if currently building
+  if action_states()
+    .build
+    .get(&build.id)
+    .await
+    .and_then(|states| {
+      states.get().ok().map(|states| states.building)
+    })
+    .unwrap_or_default()
+  {
+    let user = git_webhook_user().to_owned();
+    let cancel = ExecuteRequest::CancelBuild(CancelBuild {
+      build: build.id.clone(),
+    });
+    let update = init_execution_update(&cancel, &user).await?;
+    let ExecuteRequest::CancelBuild(cancel) = cancel else {
+      unreachable!()
+    };
+
+    cancel
+      .resolve(&ExecuteArgs {
+        user,
+        update,
+        task_id: Uuid::new_v4(),
+      })
+      .await
+      .ok();
+
+    poll_build_until_cancelled(&build.id).await?;
+  }
 
   let user = git_webhook_user().to_owned();
-  let req = ExecuteRequest::RunBuild(RunBuild { build: build.id });
-  let update = init_execution_update(&req, &user).await?;
-  let ExecuteRequest::RunBuild(req) = req else {
+  let run = ExecuteRequest::RunBuild(RunBuild { build: build.id });
+  let update = init_execution_update(&run, &user).await?;
+  let ExecuteRequest::RunBuild(run) = run else {
     unreachable!()
   };
-  req
+
+  run
     .resolve(&ExecuteArgs {
       user,
       update,
@@ -83,9 +107,32 @@ pub async fn handle_build_webhook<B: super::ExtractBranch>(
     })
     .await
     .map_err(|e| e.error)?;
+
   Ok(())
 }
 
+async fn poll_build_until_cancelled(
+  build_id: &String,
+) -> anyhow::Result<()> {
+  let action_states = action_states();
+  // Poll to ensure cancelled
+  for _ in 0..10 {
+    if !action_states
+      .build
+      .get(build_id)
+      .await
+      .and_then(|states| {
+        states.get().ok().map(|states| states.building)
+      })
+      .unwrap_or_default()
+    {
+      return Ok(());
+    }
+    tokio::time::sleep(Duration::from_secs(1)).await;
+  }
+  Err(anyhow!("Build still running after cancel"))
+}
+
 // ======
 //  REPO
 // ======
@@ -220,7 +267,9 @@ async fn handle_repo_webhook_inner<
   let lock = repo_locks().get_or_insert_default(&repo.id).await;
   let _lock = lock.lock().await;
 
-  B::verify_branch(&body, &repo.config.branch)?;
+  if !B::branch_matches(&body, &repo.config.branch)? {
+    return Ok(());
+  }
 
   E::resolve(repo).await
 }
@@ -351,7 +400,9 @@ pub async fn handle_stack_webhook_inner<
       .branch
   };
 
-  B::verify_branch(&body, &branch)?;
+  if !B::branch_matches(&body, &branch)? {
+    return Ok(());
+  }
 
   E::resolve(stack).await.map_err(|e| e.error)
 }
@@ -464,7 +515,9 @@ async fn handle_sync_webhook_inner<
       .branch
   };
 
-  B::verify_branch(&body, &branch)?;
+  if !B::branch_matches(&body, &branch)? {
+    return Ok(());
+  }
 
   E::resolve(sync).await
 }
@@ -501,8 +554,10 @@ pub async fn handle_procedure_webhook<B: super::ExtractBranch>(
     procedure_locks().get_or_insert_default(&procedure.id).await;
   let _lock = lock.lock().await;
 
-  if target_branch != ANY_BRANCH {
-    B::verify_branch(&body, target_branch)?;
+  if target_branch != ANY_BRANCH
+    && !B::branch_matches(&body, target_branch)?
+  {
+    return Ok(());
   }
 
   let user = git_webhook_user().to_owned();
@@ -557,7 +612,10 @@ pub async fn handle_action_webhook<B: super::ExtractBranch>(
   let branch = B::extract_branch(&body)?;
 
   if target_branch != ANY_BRANCH && branch != target_branch {
-    return Err(anyhow!("request branch does not match expected"));
+    debug!(
+      "Ignoring webhook | push to branch '{branch}' does not match expected branch '{target_branch}'"
+    );
+    return Ok(());
   }
 
   let user = git_webhook_user().to_owned();

bin/core/src/api/read/action.rs

@@ -3,7 +3,8 @@ use komodo_client::{
   api::read::*,
   entities::{
     action::{
-      Action, ActionActionState, ActionListItem, ActionState,
+      Action, ActionActionState, ActionListItem, ActionSortBy,
+      ActionState,
     },
     permission::PermissionLevel,
   },
@@ -11,13 +12,13 @@ use komodo_client::{
 use mogh_resolver::Resolve;
 
 use crate::{
-  helpers::query::get_all_tags,
+  helpers::query::{get_action_state, get_all_tags},
   permission::get_check_permissions,
   resource,
   state::{action_state_cache, action_states},
 };
 
-use super::ReadArgs;
+use super::{ReadArgs, list_limit};
 
 impl Resolve<ReadArgs> for GetAction {
   async fn resolve(
@@ -45,15 +46,45 @@ impl Resolve<ReadArgs> for ListActions {
     } else {
       get_all_tags(None).await?
     };
-    Ok(
-      resource::list_for_user::<Action>(
-        self.query,
-        user,
-        PermissionLevel::Read.into(),
-        &all_tags,
-      )
-      .await?,
+    let states = self.query.specific.states.clone();
+    let limit = list_limit(self.limit);
+    let sort_by: resource::ListItemSort<ActionListItem> =
+      match self.sort_by {
+        ActionSortBy::Name => resource::ListItemSort::Name,
+        ActionSortBy::State => {
+          resource::ListItemSort::InMemory(Box::new(|a, b| {
+            a.info
+              .state
+              .cmp(&b.info.state)
+              .then_with(|| a.name.cmp(&b.name))
+          }))
+        }
+        ActionSortBy::NextRun => {
+          resource::ListItemSort::InMemory(Box::new(|a, b| {
+            a.info
+              .next_scheduled_run
+              .cmp(&b.info.next_scheduled_run)
+              .then_with(|| a.name.cmp(&b.name))
+          }))
+        }
+      };
+    let actions = resource::list_items_for_user::<Action>(
+      self.query,
+      resource::ListItemsQueryOptions {
+        limit,
+        page: self.page,
+        sort_desc: self.sort_desc,
+        sort_by,
+      },
+      user,
+      PermissionLevel::Read.into(),
+      &all_tags,
+      |action| {
+        states.is_empty() || states.contains(&action.info.state)
+      },
     )
+    .await?;
+    Ok(actions)
   }
 }
 
@@ -67,12 +98,28 @@ impl Resolve<ReadArgs> for ListFullActions {
     } else {
       get_all_tags(None).await?
     };
+    let states = self.query.specific.states.clone();
+    let limit = list_limit(self.limit);
     Ok(
-      resource::list_full_for_user::<Action>(
+      resource::list_full_for_user_filtered::<Action, _>(
         self.query,
+        limit,
+        self.page,
         user,
         PermissionLevel::Read.into(),
         &all_tags,
+        |action| {
+          let states = states.clone();
+          async move {
+            if states.is_empty()
+              || states.contains(&get_action_state(&action.id).await)
+            {
+              Some(action)
+            } else {
+              None
+            }
+          }
+        },
       )
       .await?,
     )
@@ -107,6 +154,8 @@ impl Resolve<ReadArgs> for GetActionsSummary {
   ) -> mogh_error::Result<GetActionsSummaryResponse> {
     let actions = resource::list_full_for_user::<Action>(
       Default::default(),
+      None,
+      None,
       user,
       PermissionLevel::Read.into(),
       &[],

bin/core/src/api/read/alert.rs

@@ -40,7 +40,7 @@ impl Resolve<ReadArgs> for ListAlerts {
       FindOptions::builder()
         .sort(doc! { "ts": -1 })
         .limit(NUM_ALERTS_PER_PAGE as i64)
-        .skip(self.page * NUM_ALERTS_PER_PAGE)
+        .skip(self.page.saturating_mul(NUM_ALERTS_PER_PAGE))
         .build(),
     )
     .await

bin/core/src/api/read/alerter.rs

@@ -4,7 +4,7 @@ use database::mungos::mongodb::bson::doc;
 use komodo_client::{
   api::read::*,
   entities::{
-    alerter::{Alerter, AlerterListItem},
+    alerter::{Alerter, AlerterListItem, AlerterSortBy},
     permission::PermissionLevel,
   },
 };
@@ -17,7 +17,7 @@ use crate::{
   state::db_client,
 };
 
-use super::ReadArgs;
+use super::{ReadArgs, list_limit};
 
 impl Resolve<ReadArgs> for GetAlerter {
   async fn resolve(
@@ -45,12 +45,30 @@ impl Resolve<ReadArgs> for ListAlerters {
     } else {
       get_all_tags(None).await?
     };
+    let limit = list_limit(self.limit);
+    let sort_by: resource::ListItemSort<AlerterListItem> =
+      match self.sort_by {
+        AlerterSortBy::Name => resource::ListItemSort::Name,
+        AlerterSortBy::Type => {
+          resource::ListItemSort::DbField("config.endpoint.type")
+        }
+        AlerterSortBy::Enabled => {
+          resource::ListItemSort::DbField("config.enabled")
+        }
+      };
     Ok(
-      resource::list_for_user::<Alerter>(
+      resource::list_items_for_user::<Alerter>(
         self.query,
+        resource::ListItemsQueryOptions {
+          limit,
+          page: self.page,
+          sort_desc: self.sort_desc,
+          sort_by,
+        },
         user,
         PermissionLevel::Read.into(),
         &all_tags,
+        |_| true,
       )
       .await?,
     )
@@ -67,9 +85,12 @@ impl Resolve<ReadArgs> for ListFullAlerters {
     } else {
       get_all_tags(None).await?
     };
+    let limit = list_limit(self.limit);
     Ok(
       resource::list_full_for_user::<Alerter>(
         self.query,
+        limit as i64,
+        self.page.saturating_mul(limit),
         user,
         PermissionLevel::Read.into(),
         &all_tags,
@@ -85,6 +106,8 @@ impl Resolve<ReadArgs> for GetAlertersSummary {
     ReadArgs { user }: &ReadArgs,
   ) -> mogh_error::Result<GetAlertersSummaryResponse> {
     let query = match list_resource_ids_for_user::<Alerter>(
+      Default::default(),
+      None,
       None,
       user,
       PermissionLevel::Read.into(),

bin/core/src/api/read/build.rs

@@ -11,7 +11,9 @@ use komodo_client::{
   api::read::*,
   entities::{
     Operation,
-    build::{Build, BuildActionState, BuildListItem, BuildState},
+    build::{
+      Build, BuildActionState, BuildListItem, BuildSortBy, BuildState,
+    },
     permission::PermissionLevel,
     update::UpdateStatus,
   },
@@ -25,7 +27,7 @@ use crate::{
   state::{action_states, build_state_cache, db_client},
 };
 
-use super::ReadArgs;
+use super::{ReadArgs, list_limit};
 
 impl Resolve<ReadArgs> for GetBuild {
   async fn resolve(
@@ -53,15 +55,47 @@ impl Resolve<ReadArgs> for ListBuilds {
     } else {
       get_all_tags(None).await?
     };
-    Ok(
-      resource::list_for_user::<Build>(
-        self.query,
-        user,
-        PermissionLevel::Read.into(),
-        &all_tags,
-      )
-      .await?,
+    let states = self.query.specific.states.clone();
+    let limit = list_limit(self.limit);
+    let sort_by: resource::ListItemSort<BuildListItem> =
+      match self.sort_by {
+        BuildSortBy::Name => resource::ListItemSort::Name,
+        BuildSortBy::Source => {
+          resource::ListItemSort::InMemory(Box::new(|a, b| {
+            a.info
+              .files_on_host
+              .cmp(&b.info.files_on_host)
+              .then_with(|| {
+                a.info.linked_repo_name.cmp(&b.info.linked_repo_name)
+              })
+              .then_with(|| a.info.repo.cmp(&b.info.repo))
+              .then_with(|| a.name.cmp(&b.name))
+          }))
+        }
+        BuildSortBy::State => {
+          resource::ListItemSort::InMemory(Box::new(|a, b| {
+            a.info
+              .state
+              .cmp(&b.info.state)
+              .then_with(|| a.name.cmp(&b.name))
+          }))
+        }
+      };
+    let builds = resource::list_items_for_user::<Build>(
+      self.query,
+      resource::ListItemsQueryOptions {
+        limit,
+        page: self.page,
+        sort_desc: self.sort_desc,
+        sort_by,
+      },
+      user,
+      PermissionLevel::Read.into(),
+      &all_tags,
+      |build| states.is_empty() || states.contains(&build.info.state),
     )
+    .await?;
+    Ok(builds)
   }
 }
 
@@ -75,12 +109,29 @@ impl Resolve<ReadArgs> for ListFullBuilds {
     } else {
       get_all_tags(None).await?
     };
+    let states = self.query.specific.states.clone();
+    let limit = list_limit(self.limit);
     Ok(
-      resource::list_full_for_user::<Build>(
+      resource::list_full_for_user_filtered::<Build, _>(
         self.query,
+        limit,
+        self.page,
         user,
         PermissionLevel::Read.into(),
         &all_tags,
+        |build| {
+          let states = states.clone();
+          async move {
+            if states.is_empty()
+              || states
+                .contains(&resource::get_build_state(&build.id).await)
+            {
+              Some(build)
+            } else {
+              None
+            }
+          }
+        },
       )
       .await?,
     )
@@ -115,6 +166,8 @@ impl Resolve<ReadArgs> for GetBuildsSummary {
   ) -> mogh_error::Result<GetBuildsSummaryResponse> {
     let builds = resource::list_full_for_user::<Build>(
       Default::default(),
+      None,
+      None,
       user,
       PermissionLevel::Read.into(),
       &[],
@@ -164,7 +217,8 @@ impl Resolve<ReadArgs> for GetBuildMonthlyStats {
     let curr_ts = unix_timestamp_ms() as i64;
     let next_day = curr_ts - curr_ts % ONE_DAY_MS + ONE_DAY_MS;
 
-    let close_ts = next_day - self.page as i64 * 30 * ONE_DAY_MS;
+    let close_ts =
+      next_day - (self.page as i64).saturating_mul(30 * ONE_DAY_MS);
     let open_ts = close_ts - 30 * ONE_DAY_MS;
 
     let mut build_updates = db_client()
@@ -281,6 +335,8 @@ impl Resolve<ReadArgs> for ListCommonBuildExtraArgs {
     };
     let builds = resource::list_full_for_user::<Build>(
       self.query,
+      None,
+      None,
       user,
       PermissionLevel::Read.into(),
       &all_tags,

bin/core/src/api/read/builder.rs

@@ -4,7 +4,7 @@ use database::mungos::mongodb::bson::doc;
 use komodo_client::{
   api::read::*,
   entities::{
-    builder::{Builder, BuilderListItem},
+    builder::{Builder, BuilderListItem, BuilderSortBy},
     permission::PermissionLevel,
   },
 };
@@ -17,7 +17,7 @@ use crate::{
   state::db_client,
 };
 
-use super::ReadArgs;
+use super::{ReadArgs, list_limit};
 
 impl Resolve<ReadArgs> for GetBuilder {
   async fn resolve(
@@ -45,12 +45,35 @@ impl Resolve<ReadArgs> for ListBuilders {
     } else {
       get_all_tags(None).await?
     };
+    let limit = list_limit(self.limit);
+    let sort_by: resource::ListItemSort<BuilderListItem> =
+      match self.sort_by {
+        BuilderSortBy::Name => resource::ListItemSort::Name,
+        BuilderSortBy::Provider => {
+          resource::ListItemSort::DbField("config.type")
+        }
+        BuilderSortBy::InstanceType => {
+          resource::ListItemSort::InMemory(Box::new(|a, b| {
+            a.info
+              .instance_type
+              .cmp(&b.info.instance_type)
+              .then_with(|| a.name.cmp(&b.name))
+          }))
+        }
+      };
     Ok(
-      resource::list_for_user::<Builder>(
+      resource::list_items_for_user::<Builder>(
         self.query,
+        resource::ListItemsQueryOptions {
+          limit,
+          page: self.page,
+          sort_desc: self.sort_desc,
+          sort_by,
+        },
         user,
         PermissionLevel::Read.into(),
         &all_tags,
+        |_| true,
       )
       .await?,
     )
@@ -67,9 +90,12 @@ impl Resolve<ReadArgs> for ListFullBuilders {
     } else {
       get_all_tags(None).await?
     };
+    let limit = list_limit(self.limit);
     Ok(
       resource::list_full_for_user::<Builder>(
         self.query,
+        limit as i64,
+        self.page.saturating_mul(limit),
         user,
         PermissionLevel::Read.into(),
         &all_tags,
@@ -85,6 +111,8 @@ impl Resolve<ReadArgs> for GetBuildersSummary {
     ReadArgs { user }: &ReadArgs,
   ) -> mogh_error::Result<GetBuildersSummaryResponse> {
     let query = match list_resource_ids_for_user::<Builder>(
+      None,
+      None,
       None,
       user,
       PermissionLevel::Read.into(),

bin/core/src/api/read/deployment.rs

@@ -6,8 +6,8 @@ use komodo_client::{
   entities::{
     SwarmOrServer,
     deployment::{
-      Deployment, DeploymentActionState, DeploymentConfig,
-      DeploymentListItem, DeploymentState,
+      Deployment, DeploymentActionState, DeploymentListItem,
+      DeploymentSortBy, DeploymentState,
     },
     docker::{
       container::{Container, ContainerStats},
@@ -25,7 +25,9 @@ use reqwest::StatusCode;
 
 use crate::{
   helpers::{
-    periphery_client, query::get_all_tags, swarm::swarm_request,
+    periphery_client,
+    query::{get_all_tags, get_deployment_state},
+    swarm::swarm_request,
   },
   permission::get_check_permissions,
   resource::{self, setup_deployment_execution},
@@ -34,7 +36,7 @@ use crate::{
   },
 };
 
-use super::ReadArgs;
+use super::{ReadArgs, list_limit};
 
 impl Resolve<ReadArgs> for GetDeployment {
   async fn resolve(
@@ -63,21 +65,66 @@ impl Resolve<ReadArgs> for ListDeployments {
       get_all_tags(None).await?
     };
     let only_update_available = self.query.specific.update_available;
-    let deployments = resource::list_for_user::<Deployment>(
+    let states = self.query.specific.states.clone();
+    let limit = list_limit(self.limit);
+    let sort_by: resource::ListItemSort<DeploymentListItem> =
+      match self.sort_by {
+        DeploymentSortBy::Name => resource::ListItemSort::Name,
+        DeploymentSortBy::Image => {
+          resource::ListItemSort::InMemory(Box::new(|a, b| {
+            a.info
+              .image
+              .cmp(&b.info.image)
+              .then_with(|| a.name.cmp(&b.name))
+          }))
+        }
+        DeploymentSortBy::Host => {
+          resource::ListItemSort::InMemory(Box::new(|a, b| {
+            let host_a = if a.info.swarm_id.is_empty() {
+              &a.info.server_name
+            } else {
+              &a.info.swarm_name
+            };
+            let host_b = if b.info.swarm_id.is_empty() {
+              &b.info.server_name
+            } else {
+              &b.info.swarm_name
+            };
+            host_a.cmp(host_b).then_with(|| a.name.cmp(&b.name))
+          }))
+        }
+        DeploymentSortBy::State => {
+          resource::ListItemSort::InMemory(Box::new(|a, b| {
+            a.info
+              .state
+              .cmp(&b.info.state)
+              .then_with(|| {
+                // Use ! with update available to order 'true' first
+                (!a.info.update_available)
+                  .cmp(&!b.info.update_available)
+              })
+              .then_with(|| a.name.cmp(&b.name))
+          }))
+        }
+      };
+    let deployments = resource::list_items_for_user::<Deployment>(
       self.query,
+      resource::ListItemsQueryOptions {
+        limit,
+        page: self.page,
+        sort_desc: self.sort_desc,
+        sort_by,
+      },
       user,
       PermissionLevel::Read.into(),
       &all_tags,
+      |deployment| {
+        (!only_update_available || deployment.info.update_available)
+          && (states.is_empty()
+            || states.contains(&deployment.info.state))
+      },
     )
     .await?;
-    let deployments = if only_update_available {
-      deployments
-        .into_iter()
-        .filter(|deployment| deployment.info.update_available)
-        .collect()
-    } else {
-      deployments
-    };
     Ok(deployments)
   }
 }
@@ -92,12 +139,32 @@ impl Resolve<ReadArgs> for ListFullDeployments {
     } else {
       get_all_tags(None).await?
     };
+    let states = self.query.specific.states.clone();
+    let limit = list_limit(self.limit);
     Ok(
-      resource::list_full_for_user::<Deployment>(
+      resource::list_full_for_user_filtered::<Deployment, _>(
         self.query,
+        limit,
+        self.page,
         user,
         PermissionLevel::Read.into(),
         &all_tags,
+        |deployment| {
+          let states = states.clone();
+          async move {
+            if states.is_empty()
+              || states.contains(
+                &get_deployment_state(&deployment.id)
+                  .await
+                  .unwrap_or_default(),
+              )
+            {
+              Some(deployment)
+            } else {
+              None
+            }
+          }
+        },
       )
       .await?,
     )
@@ -154,7 +221,7 @@ impl Resolve<ReadArgs> for GetDeploymentLog {
       SwarmOrServer::Swarm(swarm) => swarm_request(
         &swarm.config.server_ids,
         periphery_client::api::swarm::GetSwarmServiceLog {
-          service: deployment.name,
+          service: deployment.deployed_name().to_string(),
           tail,
           timestamps,
           no_task_ids: false,
@@ -167,7 +234,7 @@ impl Resolve<ReadArgs> for GetDeploymentLog {
       SwarmOrServer::Server(server) => periphery_client(&server)
         .await?
         .request(api::container::GetContainerLog {
-          name: deployment.name,
+          name: deployment.deployed_name().to_string(),
           tail: cmp::min(tail, MAX_LOG_LENGTH),
           timestamps,
         })
@@ -206,7 +273,7 @@ impl Resolve<ReadArgs> for SearchDeploymentLog {
       SwarmOrServer::Swarm(swarm) => swarm_request(
         &swarm.config.server_ids,
         periphery_client::api::swarm::GetSwarmServiceLogSearch {
-          service: deployment.name,
+          service: deployment.deployed_name().to_string(),
           terms,
           combinator,
           invert,
@@ -221,7 +288,7 @@ impl Resolve<ReadArgs> for SearchDeploymentLog {
       SwarmOrServer::Server(server) => periphery_client(&server)
         .await?
         .request(api::container::GetContainerLogSearch {
-          name: deployment.name,
+          name: deployment.deployed_name().to_string(),
           terms,
           combinator,
           invert,
@@ -274,7 +341,7 @@ impl Resolve<ReadArgs> for InspectDeploymentContainer {
     periphery_client(&server)
       .await?
       .request(InspectContainer {
-        name: deployment.name,
+        name: deployment.deployed_name().to_string(),
       })
       .await
       .context("Failed to inspect container on server")
@@ -307,7 +374,7 @@ impl Resolve<ReadArgs> for InspectDeploymentSwarmService {
     swarm_request(
       &swarm.config.server_ids,
       periphery_client::api::swarm::InspectSwarmService {
-        service: deployment.name,
+        service: deployment.deployed_name().to_string(),
       },
     )
     .await
@@ -321,25 +388,24 @@ impl Resolve<ReadArgs> for GetDeploymentStats {
     self,
     ReadArgs { user }: &ReadArgs,
   ) -> mogh_error::Result<ContainerStats> {
-    let Deployment {
-      name,
-      config: DeploymentConfig { server_id, .. },
-      ..
-    } = get_check_permissions::<Deployment>(
+    let deployment = get_check_permissions::<Deployment>(
       &self.deployment,
       user,
       PermissionLevel::Read.into(),
     )
     .await?;
-    if server_id.is_empty() {
+    if deployment.config.server_id.is_empty() {
       return Err(
-        anyhow!("deployment has no server attached").into(),
+        anyhow!("Deployment has no Server attached").into(),
       );
     }
-    let server = resource::get::<Server>(&server_id).await?;
+    let server =
+      resource::get::<Server>(&deployment.config.server_id).await?;
     let res = periphery_client(&server)
       .await?
-      .request(api::container::GetContainerStats { name })
+      .request(api::container::GetContainerStats {
+        name: deployment.deployed_name().to_string(),
+      })
       .await
       .context("failed to get stats from periphery")?;
     Ok(res)
@@ -374,6 +440,8 @@ impl Resolve<ReadArgs> for GetDeploymentsSummary {
   ) -> mogh_error::Result<GetDeploymentsSummaryResponse> {
     let deployments = resource::list_full_for_user::<Deployment>(
       Default::default(),
+      None,
+      None,
       user,
       PermissionLevel::Read.into(),
       &[],
@@ -422,6 +490,8 @@ impl Resolve<ReadArgs> for ListCommonDeploymentExtraArgs {
     };
     let deployments = resource::list_full_for_user::<Deployment>(
       self.query,
+      None,
+      None,
       user,
       PermissionLevel::Read.into(),
       &all_tags,

bin/core/src/api/read/docker.rs

@@ -0,0 +1,590 @@
+use std::cmp;
+
+use anyhow::{Context as _, anyhow};
+use database::bson::doc;
+use komodo_client::{
+  api::read::*,
+  entities::{
+    ResourceTarget,
+    deployment::Deployment,
+    docker::{
+      container::{
+        Container, ContainerListItem, ContainerSortBy,
+        ContainerStateStatusEnum,
+      },
+      image::{Image, ImageHistoryResponseItem},
+      network::Network,
+      volume::Volume,
+    },
+    permission::PermissionLevel,
+    server::{Server, ServerQuery, ServerState},
+    stack::{Stack, StackServiceNames},
+    update::Log,
+  },
+};
+use mogh_resolver::Resolve;
+use periphery_client::api as periphery;
+
+use crate::{
+  api::read::{ReadArgs, list_limit},
+  helpers::{periphery_client, query::get_all_tags},
+  permission::{get_check_permissions, list_resources_for_user},
+  resource,
+  stack::compose_container_match_regex,
+  state::{db_client, server_status_cache},
+};
+
+impl Resolve<ReadArgs> for GetContainersSummary {
+  async fn resolve(
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> mogh_error::Result<GetContainersSummaryResponse> {
+    let servers = resource::list_full_for_user::<Server>(
+      Default::default(),
+      None,
+      None,
+      user,
+      PermissionLevel::Read.into(),
+      &[],
+    )
+    .await
+    .context("failed to get servers from db")?;
+
+    let mut res = GetContainersSummaryResponse::default();
+
+    for server in servers {
+      let cache = server_status_cache()
+        .get_or_insert_default(&server.id)
+        .await;
+
+      if let Some(docker) = &cache.docker {
+        for container in &docker.containers {
+          res.total += 1;
+          match container.state {
+            ContainerStateStatusEnum::Created
+            | ContainerStateStatusEnum::Paused
+            | ContainerStateStatusEnum::Exited => res.stopped += 1,
+            ContainerStateStatusEnum::Running => res.running += 1,
+            ContainerStateStatusEnum::Empty => res.unknown += 1,
+            _ => res.unhealthy += 1,
+          }
+        }
+      }
+    }
+
+    Ok(res)
+  }
+}
+
+impl Resolve<ReadArgs> for ListAllContainers {
+  async fn resolve(
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> mogh_error::Result<ListAllContainersResponse> {
+    let all_tags = if self.tags.is_empty() {
+      vec![]
+    } else {
+      get_all_tags(None).await?
+    };
+    let servers = resource::list_for_user::<Server>(
+      ServerQuery::builder()
+        .names(self.servers.clone())
+        .tags(self.tags)
+        .build(),
+      None,
+      None,
+      user,
+      PermissionLevel::Read.into(),
+      &all_tags,
+    )
+    .await?;
+
+    let mut containers = Vec::<ContainerListItem>::new();
+    let limit = list_limit(self.limit);
+    // Match terms case insensitively.
+    let terms = self
+      .terms
+      .iter()
+      .map(|term| term.to_lowercase())
+      .collect::<Vec<_>>();
+
+    for server in servers {
+      let cache = server_status_cache()
+        .get_or_insert_default(&server.id)
+        .await;
+      let Some(docker) = &cache.docker else {
+        continue;
+      };
+      containers.extend(
+        docker
+          .containers
+          .iter()
+          .filter(|container| {
+            // Apply state filter if defined.
+            (self.state.is_empty() || self.state.contains(&container.state)) &&
+            // Apply terms filter if defined
+            (terms.is_empty()
+              // Match when all terms contained within a name.
+              || {
+                let name = container.name.to_lowercase();
+                terms.iter().all(|term| name.contains(term))
+              })
+          })
+          .cloned(),
+      );
+    }
+
+    // The containers all come from the in memory status cache,
+    // so all matching containers are collected and sorted
+    // before applying pagination.
+    let compare = |a: &ContainerListItem, b: &ContainerListItem| {
+      match self.sort_by {
+        ContainerSortBy::Name => a.name.cmp(&b.name),
+        ContainerSortBy::Server => a.server_name.cmp(&b.server_name),
+        ContainerSortBy::State => a.state.cmp(&b.state),
+        ContainerSortBy::Image => a.image.cmp(&b.image),
+        ContainerSortBy::Networks => {
+          a.networks.first().cmp(&b.networks.first())
+        }
+        ContainerSortBy::Ports => a
+          .ports
+          .first()
+          .map(|port| port.private_port)
+          .cmp(&b.ports.first().map(|port| port.private_port)),
+        ContainerSortBy::Volumes => {
+          a.volumes.first().cmp(&b.volumes.first())
+        }
+      }
+      // Fall back to name based sorting for equal sort keys.
+      // Inside `compare`, so descending sorts are fully descending,
+      // matching the List<Resource> apis.
+      .then_with(|| a.name.cmp(&b.name))
+    };
+    if self.sort_desc {
+      containers.sort_by(|a, b| compare(b, a));
+    } else {
+      containers.sort_by(|a, b| compare(a, b));
+    }
+
+    let skip = limit.saturating_mul(self.page) as usize;
+    let take = if limit == 0 {
+      usize::MAX
+    } else {
+      limit as usize
+    };
+    Ok(containers.into_iter().skip(skip).take(take).collect())
+  }
+}
+
+impl Resolve<ReadArgs> for ListContainers {
+  async fn resolve(
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> mogh_error::Result<ListContainersResponse> {
+    let server = get_check_permissions::<Server>(
+      &self.server,
+      user,
+      PermissionLevel::Read.into(),
+    )
+    .await?;
+    let cache = server_status_cache()
+      .get_or_insert_default(&server.id)
+      .await;
+    if let Some(docker) = &cache.docker {
+      Ok(docker.containers.clone())
+    } else {
+      Ok(Vec::new())
+    }
+  }
+}
+
+impl Resolve<ReadArgs> for InspectContainer {
+  async fn resolve(
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> mogh_error::Result<Container> {
+    let server = get_check_permissions::<Server>(
+      &self.server,
+      user,
+      PermissionLevel::Read.inspect(),
+    )
+    .await?;
+    let cache = server_status_cache()
+      .get_or_insert_default(&server.id)
+      .await;
+    if cache.state != ServerState::Ok {
+      return Err(
+        anyhow!(
+          "Cannot inspect container: server is {:?}",
+          cache.state
+        )
+        .into(),
+      );
+    }
+    let res = periphery_client(&server)
+      .await?
+      .request(periphery::container::InspectContainer {
+        name: self.container,
+      })
+      .await?;
+    Ok(res)
+  }
+}
+
+impl Resolve<ReadArgs> for GetResourceMatchingContainer {
+  async fn resolve(
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> mogh_error::Result<GetResourceMatchingContainerResponse> {
+    let server = get_check_permissions::<Server>(
+      &self.server,
+      user,
+      PermissionLevel::Read.into(),
+    )
+    .await?;
+
+    // First check deployments with a matching deployed / custom container name.
+    // The empty check is required to avoid matching
+    // deployments with no custom name configured.
+    if !self.container.is_empty()
+      && let Ok(Some(deployment)) = db_client()
+        .deployments
+        .find_one(doc! {
+          "$or": [
+            { "info.deployed_name": &self.container },
+            { "config.custom_name": &self.container },
+          ]
+        })
+        .await
+    {
+      return Ok(GetResourceMatchingContainerResponse {
+        resource: ResourceTarget::Deployment(deployment.id).into(),
+      });
+    }
+
+    // Then check deployments matching by name
+    if let Ok(deployment) =
+      resource::get::<Deployment>(&self.container).await
+      && deployment.custom_name() == self.container
+    {
+      return Ok(GetResourceMatchingContainerResponse {
+        resource: ResourceTarget::Deployment(deployment.id).into(),
+      });
+    }
+
+    // then check stacks
+    let stacks = list_resources_for_user::<Stack>(
+      doc! { "config.server_id": &server.id },
+      None,
+      None,
+      user,
+      PermissionLevel::Read.into(),
+    )
+    .await?;
+
+    // check matching stack
+    for stack in stacks {
+      for StackServiceNames {
+        service_name,
+        container_name,
+        ..
+      } in stack
+        .info
+        .deployed_services
+        .unwrap_or(stack.info.latest_services)
+      {
+        let is_match = match compose_container_match_regex(&container_name)
+          .with_context(|| format!("failed to construct container name matching regex for service {service_name}")) 
+        {
+          Ok(regex) => regex,
+          Err(e) => {
+            warn!("{e:#}");
+            continue;
+          }
+        }.is_match(&self.container);
+
+        if is_match {
+          return Ok(GetResourceMatchingContainerResponse {
+            resource: ResourceTarget::Stack(stack.id).into(),
+          });
+        }
+      }
+    }
+
+    Ok(GetResourceMatchingContainerResponse { resource: None })
+  }
+}
+
+const MAX_LOG_LENGTH: u64 = 5000;
+
+impl Resolve<ReadArgs> for GetContainerLog {
+  async fn resolve(
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> mogh_error::Result<Log> {
+    let GetContainerLog {
+      server,
+      container,
+      tail,
+      timestamps,
+    } = self;
+    let server = get_check_permissions::<Server>(
+      &server,
+      user,
+      PermissionLevel::Read.logs(),
+    )
+    .await?;
+    let res = periphery_client(&server)
+      .await?
+      .request(periphery::container::GetContainerLog {
+        name: container,
+        tail: cmp::min(tail, MAX_LOG_LENGTH),
+        timestamps,
+      })
+      .await
+      .context("failed at call to periphery")?;
+    Ok(res)
+  }
+}
+
+impl Resolve<ReadArgs> for SearchContainerLog {
+  async fn resolve(
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> mogh_error::Result<Log> {
+    let SearchContainerLog {
+      server,
+      container,
+      terms,
+      combinator,
+      invert,
+      timestamps,
+    } = self;
+    let server = get_check_permissions::<Server>(
+      &server,
+      user,
+      PermissionLevel::Read.logs(),
+    )
+    .await?;
+    let res = periphery_client(&server)
+      .await?
+      .request(periphery::container::GetContainerLogSearch {
+        name: container,
+        terms,
+        combinator,
+        invert,
+        timestamps,
+      })
+      .await
+      .context("failed at call to periphery")?;
+    Ok(res)
+  }
+}
+
+impl Resolve<ReadArgs> for ListComposeProjects {
+  async fn resolve(
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> mogh_error::Result<ListComposeProjectsResponse> {
+    let server = get_check_permissions::<Server>(
+      &self.server,
+      user,
+      PermissionLevel::Read.into(),
+    )
+    .await?;
+    let cache = server_status_cache()
+      .get_or_insert_default(&server.id)
+      .await;
+    if let Some(docker) = &cache.docker {
+      Ok(docker.projects.clone())
+    } else {
+      Ok(Vec::new())
+    }
+  }
+}
+
+impl Resolve<ReadArgs> for ListNetworks {
+  async fn resolve(
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> mogh_error::Result<ListNetworksResponse> {
+    let server = get_check_permissions::<Server>(
+      &self.server,
+      user,
+      PermissionLevel::Read.into(),
+    )
+    .await?;
+    let cache = server_status_cache()
+      .get_or_insert_default(&server.id)
+      .await;
+    if let Some(docker) = &cache.docker {
+      Ok(docker.networks.clone())
+    } else {
+      Ok(Vec::new())
+    }
+  }
+}
+
+impl Resolve<ReadArgs> for InspectNetwork {
+  async fn resolve(
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> mogh_error::Result<Network> {
+    let server = get_check_permissions::<Server>(
+      &self.server,
+      user,
+      PermissionLevel::Read.into(),
+    )
+    .await?;
+    let cache = server_status_cache()
+      .get_or_insert_default(&server.id)
+      .await;
+    if cache.state != ServerState::Ok {
+      return Err(
+        anyhow!(
+          "Cannot inspect network: server is {:?}",
+          cache.state
+        )
+        .into(),
+      );
+    }
+    let res = periphery_client(&server)
+      .await?
+      .request(periphery::docker::InspectNetwork {
+        name: self.network,
+      })
+      .await?;
+    Ok(res)
+  }
+}
+
+impl Resolve<ReadArgs> for ListImages {
+  async fn resolve(
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> mogh_error::Result<ListImagesResponse> {
+    let server = get_check_permissions::<Server>(
+      &self.server,
+      user,
+      PermissionLevel::Read.into(),
+    )
+    .await?;
+    let cache = server_status_cache()
+      .get_or_insert_default(&server.id)
+      .await;
+    if let Some(docker) = &cache.docker {
+      Ok(docker.images.clone())
+    } else {
+      Ok(Vec::new())
+    }
+  }
+}
+
+impl Resolve<ReadArgs> for InspectImage {
+  async fn resolve(
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> mogh_error::Result<Image> {
+    let server = get_check_permissions::<Server>(
+      &self.server,
+      user,
+      PermissionLevel::Read.into(),
+    )
+    .await?;
+    let cache = server_status_cache()
+      .get_or_insert_default(&server.id)
+      .await;
+    if cache.state != ServerState::Ok {
+      return Err(
+        anyhow!("Cannot inspect image: server is {:?}", cache.state)
+          .into(),
+      );
+    }
+    let res = periphery_client(&server)
+      .await?
+      .request(periphery::docker::InspectImage { name: self.image })
+      .await?;
+    Ok(res)
+  }
+}
+
+impl Resolve<ReadArgs> for ListImageHistory {
+  async fn resolve(
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> mogh_error::Result<Vec<ImageHistoryResponseItem>> {
+    let server = get_check_permissions::<Server>(
+      &self.server,
+      user,
+      PermissionLevel::Read.into(),
+    )
+    .await?;
+    let cache = server_status_cache()
+      .get_or_insert_default(&server.id)
+      .await;
+    if cache.state != ServerState::Ok {
+      return Err(
+        anyhow!(
+          "Cannot get image history: server is {:?}",
+          cache.state
+        )
+        .into(),
+      );
+    }
+    let res = periphery_client(&server)
+      .await?
+      .request(periphery::docker::ImageHistory { name: self.image })
+      .await?;
+    Ok(res)
+  }
+}
+
+impl Resolve<ReadArgs> for ListVolumes {
+  async fn resolve(
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> mogh_error::Result<ListVolumesResponse> {
+    let server = get_check_permissions::<Server>(
+      &self.server,
+      user,
+      PermissionLevel::Read.into(),
+    )
+    .await?;
+    let cache = server_status_cache()
+      .get_or_insert_default(&server.id)
+      .await;
+    if let Some(docker) = &cache.docker {
+      Ok(docker.volumes.clone())
+    } else {
+      Ok(Vec::new())
+    }
+  }
+}
+
+impl Resolve<ReadArgs> for InspectVolume {
+  async fn resolve(
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> mogh_error::Result<Volume> {
+    let server = get_check_permissions::<Server>(
+      &self.server,
+      user,
+      PermissionLevel::Read.into(),
+    )
+    .await?;
+    let cache = server_status_cache()
+      .get_or_insert_default(&server.id)
+      .await;
+    if cache.state != ServerState::Ok {
+      return Err(
+        anyhow!("Cannot inspect volume: server is {:?}", cache.state)
+          .into(),
+      );
+    }
+    let res = periphery_client(&server)
+      .await?
+      .request(periphery::docker::InspectVolume { name: self.volume })
+      .await?;
+    Ok(res)
+  }
+}

bin/core/src/api/read/mod.rs

@@ -10,7 +10,7 @@ use komodo_client::{
     ResourceTarget,
     build::Build,
     builder::{Builder, BuilderConfig},
-    config::{DockerRegistry, GitProvider},
+    config::{GitProvider, ImageRegistry},
     permission::PermissionLevel,
     repo::Repo,
     server::Server,
@@ -44,6 +44,7 @@ mod alerter;
 mod build;
 mod builder;
 mod deployment;
+mod docker;
 mod onboarding_key;
 mod permission;
 mod procedure;
@@ -66,6 +67,12 @@ pub struct ReadArgs {
   pub user: User,
 }
 
+/// Resolve the page limit for List apis, falling back to the
+/// configured `default_pagination_limit` when not provided.
+fn list_limit(limit: Option<u64>) -> u64 {
+  limit.unwrap_or(core_config().default_pagination_limit)
+}
+
 #[typeshare]
 #[derive(
   Serialize, Deserialize, Debug, Clone, Resolve, EnumDiscriminants,
@@ -80,7 +87,8 @@ enum ReadRequest {
   GetCoreInfo(GetCoreInfo),
   ListSecrets(ListSecrets),
   ListGitProvidersFromConfig(ListGitProvidersFromConfig),
-  ListDockerRegistriesFromConfig(ListDockerRegistriesFromConfig),
+  #[serde(alias = "ListDockerRegistriesFromConfig")]
+  ListImageRegistriesFromConfig(ListImageRegistriesFromConfig),
 
   // ==== SWARM ====
   GetSwarmsSummary(GetSwarmsSummary),
@@ -117,22 +125,33 @@ enum ReadRequest {
   // ==== TERMINAL ====
   ListTerminals(ListTerminals),
 
-  // ==== DOCKER ====
-  GetDockerContainersSummary(GetDockerContainersSummary),
-  ListAllDockerContainers(ListAllDockerContainers),
-  ListDockerContainers(ListDockerContainers),
-  InspectDockerContainer(InspectDockerContainer),
+  // ==== CONTAINER ====
+  #[serde(alias = "GetDockerContainersSummary")]
+  GetContainersSummary(GetContainersSummary),
+  #[serde(alias = "ListAllDockerContainers")]
+  ListAllContainers(ListAllContainers),
+  #[serde(alias = "ListDockerContainers")]
+  ListContainers(ListContainers),
+  #[serde(alias = "InspectDockerContainer")]
+  InspectContainer(InspectContainer),
   GetResourceMatchingContainer(GetResourceMatchingContainer),
   GetContainerLog(GetContainerLog),
   SearchContainerLog(SearchContainerLog),
   ListComposeProjects(ListComposeProjects),
-  ListDockerNetworks(ListDockerNetworks),
-  InspectDockerNetwork(InspectDockerNetwork),
-  ListDockerImages(ListDockerImages),
-  InspectDockerImage(InspectDockerImage),
-  ListDockerImageHistory(ListDockerImageHistory),
-  ListDockerVolumes(ListDockerVolumes),
-  InspectDockerVolume(InspectDockerVolume),
+  #[serde(alias = "ListDockerNetworks")]
+  ListNetworks(ListNetworks),
+  #[serde(alias = "InspectDockerNetwork")]
+  InspectNetwork(InspectNetwork),
+  #[serde(alias = "ListDockerImages")]
+  ListImages(ListImages),
+  #[serde(alias = "InspectDockerImage")]
+  InspectImage(InspectImage),
+  #[serde(alias = "ListDockerImageHistory")]
+  ListImageHistory(ListImageHistory),
+  #[serde(alias = "ListDockerVolumes")]
+  ListVolumes(ListVolumes),
+  #[serde(alias = "InspectDockerVolume")]
+  InspectVolume(InspectVolume),
 
   // ==== SERVER STATS ====
   GetSystemInformation(GetSystemInformation),
@@ -151,6 +170,7 @@ enum ReadRequest {
   ListStacks(ListStacks),
   ListFullStacks(ListFullStacks),
   ListStackServices(ListStackServices),
+  ListAllStackServices(ListAllStackServices),
   ListCommonStackExtraArgs(ListCommonStackExtraArgs),
   ListCommonStackBuildExtraArgs(ListCommonStackBuildExtraArgs),
 
@@ -258,8 +278,10 @@ enum ReadRequest {
   // ==== PROVIDER ====
   GetGitProviderAccount(GetGitProviderAccount),
   ListGitProviderAccounts(ListGitProviderAccounts),
-  GetDockerRegistryAccount(GetDockerRegistryAccount),
-  ListDockerRegistryAccounts(ListDockerRegistryAccounts),
+  #[serde(alias = "GetDockerRegistryAccount")]
+  GetImageRegistryAccount(GetImageRegistryAccount),
+  #[serde(alias = "ListDockerRegistryAccounts")]
+  ListImageRegistryAccounts(ListImageRegistryAccounts),
 
   // ==== ONBOARDING KEY ====
   ListOnboardingKeys(ListOnboardingKeys),
@@ -354,6 +376,7 @@ impl Resolve<ReadArgs> for GetCoreInfo {
       disable_websocket_reconnect: config.disable_websocket_reconnect,
       enable_fancy_toml: config.enable_fancy_toml,
       timezone: config.timezone.clone(),
+      default_pagination_limit: config.default_pagination_limit,
       public_key: core_keys().load().public.to_string(),
     };
     Ok(info)
@@ -379,7 +402,9 @@ impl Resolve<ReadArgs> for ListSecrets {
         ResourceTarget::Builder(id) => {
           match resource::get::<Builder>(&id).await?.config {
             BuilderConfig::Url(_) => None,
-            BuilderConfig::Server(config) => Some(config.server_id),
+            BuilderConfig::Server(config) => {
+              config.server_ids.first().cloned()
+            }
             BuilderConfig::Aws(config) => {
               secrets.extend(config.secrets);
               None
@@ -434,11 +459,13 @@ impl Resolve<ReadArgs> for ListGitProvidersFromConfig {
           match resource::get::<Builder>(&id).await?.config {
             BuilderConfig::Url(_) => {}
             BuilderConfig::Server(config) => {
-              merge_git_providers_for_server(
-                &mut providers,
-                &config.server_id,
-              )
-              .await?;
+              if let Some(server_id) = config.server_ids.first() {
+                merge_git_providers_for_server(
+                  &mut providers,
+                  server_id,
+                )
+                .await?;
+              }
             }
             BuilderConfig::Aws(config) => {
               merge_git_providers(
@@ -460,18 +487,24 @@ impl Resolve<ReadArgs> for ListGitProvidersFromConfig {
     let (builds, repos, syncs) = tokio::try_join!(
       resource::list_full_for_user::<Build>(
         Default::default(),
+        None,
+        None,
         user,
         PermissionLevel::Read.into(),
         &[]
       ),
       resource::list_full_for_user::<Repo>(
         Default::default(),
+        None,
+        None,
         user,
         PermissionLevel::Read.into(),
         &[]
       ),
       resource::list_full_for_user::<ResourceSync>(
         Default::default(),
+        None,
+        None,
         user,
         PermissionLevel::Read.into(),
         &[]
@@ -523,33 +556,35 @@ impl Resolve<ReadArgs> for ListGitProvidersFromConfig {
 
 //
 
-impl Resolve<ReadArgs> for ListDockerRegistriesFromConfig {
+impl Resolve<ReadArgs> for ListImageRegistriesFromConfig {
   async fn resolve(
     self,
     _: &ReadArgs,
-  ) -> mogh_error::Result<ListDockerRegistriesFromConfigResponse> {
-    let mut registries = core_config().docker_registries.clone();
+  ) -> mogh_error::Result<ListImageRegistriesFromConfigResponse> {
+    let mut registries = core_config().image_registries.clone();
 
     if let Some(target) = self.target {
       match target {
         ResourceTarget::Server(id) => {
-          merge_docker_registries_for_server(&mut registries, &id)
+          merge_image_registries_for_server(&mut registries, &id)
             .await?;
         }
         ResourceTarget::Builder(id) => {
           match resource::get::<Builder>(&id).await?.config {
             BuilderConfig::Url(_) => {}
             BuilderConfig::Server(config) => {
-              merge_docker_registries_for_server(
-                &mut registries,
-                &config.server_id,
-              )
-              .await?;
+              if let Some(server_id) = config.server_ids.first() {
+                merge_image_registries_for_server(
+                  &mut registries,
+                  server_id,
+                )
+                .await?;
+              }
             }
             BuilderConfig::Aws(config) => {
-              merge_docker_registries(
+              merge_image_registries(
                 &mut registries,
-                config.docker_registries,
+                config.image_registries,
               );
             }
           }
@@ -607,14 +642,14 @@ fn merge_git_providers(
   }
 }
 
-async fn merge_docker_registries_for_server(
-  registries: &mut Vec<DockerRegistry>,
+async fn merge_image_registries_for_server(
+  registries: &mut Vec<ImageRegistry>,
   server_id: &str,
 ) -> mogh_error::Result<()> {
   let server = resource::get::<Server>(server_id).await?;
   let more = periphery_client(&server)
     .await?
-    .request(periphery_client::api::ListDockerRegistries {})
+    .request(periphery_client::api::ListImageRegistries {})
     .await
     .with_context(|| {
       format!(
@@ -622,13 +657,13 @@ async fn merge_docker_registries_for_server(
         server.name
       )
     })?;
-  merge_docker_registries(registries, more);
+  merge_image_registries(registries, more);
   Ok(())
 }
 
-fn merge_docker_registries(
-  registries: &mut Vec<DockerRegistry>,
-  more: Vec<DockerRegistry>,
+fn merge_image_registries(
+  registries: &mut Vec<ImageRegistry>,
+  more: Vec<ImageRegistry>,
 ) {
   for incoming_registry in more {
     if let Some(registry) = registries

bin/core/src/api/read/procedure.rs

@@ -3,19 +3,21 @@ use komodo_client::{
   api::read::*,
   entities::{
     permission::PermissionLevel,
-    procedure::{Procedure, ProcedureState},
+    procedure::{
+      Procedure, ProcedureListItem, ProcedureSortBy, ProcedureState,
+    },
   },
 };
 use mogh_resolver::Resolve;
 
 use crate::{
-  helpers::query::get_all_tags,
+  helpers::query::{get_all_tags, get_procedure_state},
   permission::get_check_permissions,
   resource,
   state::{action_states, procedure_state_cache},
 };
 
-use super::ReadArgs;
+use super::{ReadArgs, list_limit};
 
 impl Resolve<ReadArgs> for GetProcedure {
   async fn resolve(
@@ -43,15 +45,45 @@ impl Resolve<ReadArgs> for ListProcedures {
     } else {
       get_all_tags(None).await?
     };
-    Ok(
-      resource::list_for_user::<Procedure>(
-        self.query,
-        user,
-        PermissionLevel::Read.into(),
-        &all_tags,
-      )
-      .await?,
+    let states = self.query.specific.states.clone();
+    let limit = list_limit(self.limit);
+    let sort_by: resource::ListItemSort<ProcedureListItem> =
+      match self.sort_by {
+        ProcedureSortBy::Name => resource::ListItemSort::Name,
+        ProcedureSortBy::State => {
+          resource::ListItemSort::InMemory(Box::new(|a, b| {
+            a.info
+              .state
+              .cmp(&b.info.state)
+              .then_with(|| a.name.cmp(&b.name))
+          }))
+        }
+        ProcedureSortBy::NextRun => {
+          resource::ListItemSort::InMemory(Box::new(|a, b| {
+            a.info
+              .next_scheduled_run
+              .cmp(&b.info.next_scheduled_run)
+              .then_with(|| a.name.cmp(&b.name))
+          }))
+        }
+      };
+    let procedures = resource::list_items_for_user::<Procedure>(
+      self.query,
+      resource::ListItemsQueryOptions {
+        limit,
+        page: self.page,
+        sort_desc: self.sort_desc,
+        sort_by,
+      },
+      user,
+      PermissionLevel::Read.into(),
+      &all_tags,
+      |procedure| {
+        states.is_empty() || states.contains(&procedure.info.state)
+      },
     )
+    .await?;
+    Ok(procedures)
   }
 }
 
@@ -65,12 +97,29 @@ impl Resolve<ReadArgs> for ListFullProcedures {
     } else {
       get_all_tags(None).await?
     };
+    let states = self.query.specific.states.clone();
+    let limit = list_limit(self.limit);
     Ok(
-      resource::list_full_for_user::<Procedure>(
+      resource::list_full_for_user_filtered::<Procedure, _>(
         self.query,
+        limit,
+        self.page,
         user,
         PermissionLevel::Read.into(),
         &all_tags,
+        |procedure| {
+          let states = states.clone();
+          async move {
+            if states.is_empty()
+              || states
+                .contains(&get_procedure_state(&procedure.id).await)
+            {
+              Some(procedure)
+            } else {
+              None
+            }
+          }
+        },
       )
       .await?,
     )
@@ -84,6 +133,8 @@ impl Resolve<ReadArgs> for GetProceduresSummary {
   ) -> mogh_error::Result<GetProceduresSummaryResponse> {
     let procedures = resource::list_full_for_user::<Procedure>(
       Default::default(),
+      None,
+      None,
       user,
       PermissionLevel::Read.into(),
       &[],

bin/core/src/api/read/provider.rs

@@ -61,11 +61,11 @@ impl Resolve<ReadArgs> for ListGitProviderAccounts {
   }
 }
 
-impl Resolve<ReadArgs> for GetDockerRegistryAccount {
+impl Resolve<ReadArgs> for GetImageRegistryAccount {
   async fn resolve(
     self,
     ReadArgs { user }: &ReadArgs,
-  ) -> mogh_error::Result<GetDockerRegistryAccountResponse> {
+  ) -> mogh_error::Result<GetImageRegistryAccountResponse> {
     if !user.admin {
       return Err(
         anyhow!("Only admins can read docker registry accounts")
@@ -83,11 +83,11 @@ impl Resolve<ReadArgs> for GetDockerRegistryAccount {
   }
 }
 
-impl Resolve<ReadArgs> for ListDockerRegistryAccounts {
+impl Resolve<ReadArgs> for ListImageRegistryAccounts {
   async fn resolve(
     self,
     ReadArgs { user }: &ReadArgs,
-  ) -> mogh_error::Result<ListDockerRegistryAccountsResponse> {
+  ) -> mogh_error::Result<ListImageRegistryAccountsResponse> {
     if !user.admin {
       return Err(
         anyhow!("Only admins can read docker registry accounts")

bin/core/src/api/read/repo.rs

@@ -3,7 +3,9 @@ use komodo_client::{
   api::read::*,
   entities::{
     permission::PermissionLevel,
-    repo::{Repo, RepoActionState, RepoListItem, RepoState},
+    repo::{
+      Repo, RepoActionState, RepoListItem, RepoSortBy, RepoState,
+    },
   },
 };
 use mogh_resolver::Resolve;
@@ -15,7 +17,7 @@ use crate::{
   state::{action_states, repo_state_cache},
 };
 
-use super::ReadArgs;
+use super::{ReadArgs, list_limit};
 
 impl Resolve<ReadArgs> for GetRepo {
   async fn resolve(
@@ -43,15 +45,41 @@ impl Resolve<ReadArgs> for ListRepos {
     } else {
       get_all_tags(None).await?
     };
-    Ok(
-      resource::list_for_user::<Repo>(
-        self.query,
-        user,
-        PermissionLevel::Read.into(),
-        &all_tags,
-      )
-      .await?,
+    let states = self.query.specific.states.clone();
+    let limit = list_limit(self.limit);
+    let sort_by: resource::ListItemSort<RepoListItem> =
+      match self.sort_by {
+        RepoSortBy::Name => resource::ListItemSort::Name,
+        RepoSortBy::Repo => {
+          resource::ListItemSort::DbField("config.repo")
+        }
+        RepoSortBy::Branch => {
+          resource::ListItemSort::DbField("config.branch")
+        }
+        RepoSortBy::State => {
+          resource::ListItemSort::InMemory(Box::new(|a, b| {
+            a.info
+              .state
+              .cmp(&b.info.state)
+              .then_with(|| a.name.cmp(&b.name))
+          }))
+        }
+      };
+    let repos = resource::list_items_for_user::<Repo>(
+      self.query,
+      resource::ListItemsQueryOptions {
+        limit,
+        page: self.page,
+        sort_desc: self.sort_desc,
+        sort_by,
+      },
+      user,
+      PermissionLevel::Read.into(),
+      &all_tags,
+      |repo| states.is_empty() || states.contains(&repo.info.state),
     )
+    .await?;
+    Ok(repos)
   }
 }
 
@@ -65,12 +93,29 @@ impl Resolve<ReadArgs> for ListFullRepos {
     } else {
       get_all_tags(None).await?
     };
+    let states = self.query.specific.states.clone();
+    let limit = list_limit(self.limit);
     Ok(
-      resource::list_full_for_user::<Repo>(
+      resource::list_full_for_user_filtered::<Repo, _>(
         self.query,
+        limit,
+        self.page,
         user,
         PermissionLevel::Read.into(),
         &all_tags,
+        |repo| {
+          let states = states.clone();
+          async move {
+            if states.is_empty()
+              || states
+                .contains(&resource::get_repo_state(&repo.id).await)
+            {
+              Some(repo)
+            } else {
+              None
+            }
+          }
+        },
       )
       .await?,
     )
@@ -105,6 +150,8 @@ impl Resolve<ReadArgs> for GetReposSummary {
   ) -> mogh_error::Result<GetReposSummaryResponse> {
     let repos = resource::list_full_for_user::<Repo>(
       Default::default(),
+      None,
+      None,
       user,
       PermissionLevel::Read.into(),
       &[],

bin/core/src/api/read/schedule.rs

@@ -1,13 +1,15 @@
+use std::cmp::Ordering;
+
 use futures_util::future::join_all;
 use komodo_client::{
   api::read::*,
   entities::{
     ResourceTarget,
-    action::Action,
+    action::{Action, ActionQuerySpecifics},
     permission::PermissionLevel,
-    procedure::Procedure,
+    procedure::{Procedure, ProcedureQuerySpecifics},
     resource::{ResourceQuery, TemplatesQueryBehavior},
-    schedule::Schedule,
+    schedule::{Schedule, ScheduleSortBy},
   },
 };
 use mogh_resolver::Resolve;
@@ -18,7 +20,7 @@ use crate::{
   schedule::get_schedule_item_info,
 };
 
-use super::ReadArgs;
+use super::{ReadArgs, list_limit};
 
 impl Resolve<ReadArgs> for ListSchedules {
   async fn resolve(
@@ -29,24 +31,36 @@ impl Resolve<ReadArgs> for ListSchedules {
     let (actions, procedures) = tokio::try_join!(
       list_full_for_user::<Action>(
         ResourceQuery {
-          names: Default::default(),
           templates: TemplatesQueryBehavior::Include,
           tag_behavior: self.tag_behavior,
           tags: self.tags.clone(),
-          specific: Default::default(),
+          terms: self.terms.clone(),
+          specific: ActionQuerySpecifics {
+            scheduled: Some(true),
+            ..Default::default()
+          },
+          ..Default::default()
         },
+        None,
+        None,
         &args.user,
         PermissionLevel::Read.into(),
         &all_tags,
       ),
       list_full_for_user::<Procedure>(
         ResourceQuery {
-          names: Default::default(),
           templates: TemplatesQueryBehavior::Include,
           tag_behavior: self.tag_behavior,
           tags: self.tags.clone(),
-          specific: Default::default(),
+          terms: self.terms,
+          specific: ProcedureQuerySpecifics {
+            scheduled: Some(true),
+            ..Default::default()
+          },
+          ..Default::default()
         },
+        None,
+        None,
         &args.user,
         PermissionLevel::Read.into(),
         &all_tags,
@@ -96,12 +110,51 @@ impl Resolve<ReadArgs> for ListSchedules {
     let (actions, procedures) =
       tokio::join!(join_all(actions), join_all(procedures));
 
-    Ok(
-      actions
-        .into_iter()
-        .chain(procedures)
-        .filter(|s| !s.schedule.is_empty())
-        .collect(),
-    )
+    // The terms / scheduled filters are already applied
+    // at the db level by the queries above.
+    let mut schedules =
+      actions.into_iter().chain(procedures).collect::<Vec<_>>();
+
+    // The schedules are composed in memory across resource types,
+    // so all matching schedules are collected and sorted
+    // before applying pagination.
+    // All comparators fall back to name based sorting for equal
+    // sort keys, inside `compare`, so descending sorts are fully
+    // descending, matching the List<Resource> apis.
+    let compare: fn(&Schedule, &Schedule) -> Ordering =
+      match self.sort_by {
+        ScheduleSortBy::Name => |a, b| a.name.cmp(&b.name),
+        ScheduleSortBy::Schedule => |a, b| {
+          a.schedule
+            .cmp(&b.schedule)
+            .then_with(|| a.name.cmp(&b.name))
+        },
+        // Order unscheduled (None) last, matching the UI.
+        ScheduleSortBy::NextRun => |a, b| {
+          (a.next_scheduled_run.is_none(), a.next_scheduled_run)
+            .cmp(&(
+              b.next_scheduled_run.is_none(),
+              b.next_scheduled_run,
+            ))
+            .then_with(|| a.name.cmp(&b.name))
+        },
+        ScheduleSortBy::Enabled => |a, b| {
+          a.enabled.cmp(&b.enabled).then_with(|| a.name.cmp(&b.name))
+        },
+      };
+    if self.sort_desc {
+      schedules.sort_by(|a, b| compare(b, a));
+    } else {
+      schedules.sort_by(compare);
+    }
+
+    let limit = list_limit(self.limit);
+    let skip = limit.saturating_mul(self.page) as usize;
+    let take = if limit == 0 {
+      usize::MAX
+    } else {
+      limit as usize
+    };
+    Ok(schedules.into_iter().skip(skip).take(take).collect())
   }
 }

bin/core/src/api/read/server.rs

@@ -1,5 +1,5 @@
 use std::{
-  cmp,
+  cmp::Ordering,
   collections::HashMap,
   sync::{Arc, OnceLock},
 };
@@ -15,47 +15,31 @@ use database::mungos::{
 use komodo_client::{
   api::read::*,
   entities::{
-    ResourceTarget,
-    deployment::Deployment,
-    docker::{
-      container::{
-        Container, ContainerListItem, ContainerStateStatusEnum,
-      },
-      image::{Image, ImageHistoryResponseItem},
-      network::Network,
-      volume::Volume,
-    },
     permission::PermissionLevel,
     server::{
-      Server, ServerActionState, ServerListItem, ServerQuery,
+      Server, ServerActionState, ServerListItem, ServerSortBy,
       ServerState,
     },
-    stack::{Stack, StackServiceNames},
-    stats::{SystemInformation, SystemProcess},
-    update::Log,
+    stats::{MinimalSystemStats, SystemInformation, SystemProcess},
   },
 };
 use mogh_error::AddStatusCode;
 use mogh_resolver::Resolve;
-use periphery_client::api::{
-  self as periphery,
-  container::InspectContainer,
-  docker::{
-    ImageHistory, InspectImage, InspectNetwork, InspectVolume,
-  },
-};
+use periphery_client::api as periphery;
 use reqwest::StatusCode;
 use tokio::sync::Mutex;
 
 use crate::{
-  helpers::{periphery_client, query::get_all_tags},
-  permission::{get_check_permissions, list_resources_for_user},
+  helpers::{
+    periphery_client,
+    query::{get_all_tags, get_cached_server_state},
+  },
+  permission::get_check_permissions,
   resource,
-  stack::compose_container_match_regex,
   state::{action_states, db_client, server_status_cache},
 };
 
-use super::ReadArgs;
+use super::{ReadArgs, list_limit};
 
 impl Resolve<ReadArgs> for GetServersSummary {
   async fn resolve(
@@ -64,6 +48,8 @@ impl Resolve<ReadArgs> for GetServersSummary {
   ) -> mogh_error::Result<GetServersSummaryResponse> {
     let servers = resource::list_for_user::<Server>(
       Default::default(),
+      None,
+      None,
       user,
       PermissionLevel::Read.into(),
       &[],
@@ -125,15 +111,89 @@ impl Resolve<ReadArgs> for ListServers {
     } else {
       get_all_tags(None).await?
     };
-    Ok(
-      resource::list_for_user::<Server>(
-        self.query,
-        user,
-        PermissionLevel::Read.into(),
-        &all_tags,
-      )
-      .await?,
+    let states = self.query.specific.states.clone();
+    let limit = list_limit(self.limit);
+    let sort_by: resource::ListItemSort<ServerListItem> = match self
+      .sort_by
+    {
+      ServerSortBy::Name => resource::ListItemSort::Name,
+      ServerSortBy::Region => {
+        resource::ListItemSort::DbField("config.region")
+      }
+      ServerSortBy::Version => {
+        resource::ListItemSort::InMemory(Box::new(|a, b| {
+          a.info
+            .version
+            .cmp(&b.info.version)
+            .then_with(|| a.name.cmp(&b.name))
+        }))
+      }
+      ServerSortBy::State => {
+        resource::ListItemSort::InMemory(Box::new(|a, b| {
+          a.info
+            .state
+            .cmp(&b.info.state)
+            .then_with(|| a.name.cmp(&b.name))
+        }))
+      }
+      ServerSortBy::Cpu => stats_sort(|stats| stats.cpu_perc as f64),
+      ServerSortBy::Memory => stats_sort(|stats| {
+        usage_percent(stats.mem_used_gb, stats.mem_total_gb)
+      }),
+      ServerSortBy::Disk => stats_sort(|stats| {
+        usage_percent(stats.disk_used_gb, stats.disk_total_gb)
+      }),
+      ServerSortBy::LoadAverage => {
+        stats_sort(|stats| stats.load_average.one)
+      }
+      ServerSortBy::Network => stats_sort(|stats| {
+        stats.network_ingress_bytes + stats.network_egress_bytes
+      }),
+    };
+    let servers = resource::list_items_for_user::<Server>(
+      self.query,
+      resource::ListItemsQueryOptions {
+        limit,
+        page: self.page,
+        sort_desc: self.sort_desc,
+        sort_by,
+      },
+      user,
+      PermissionLevel::Read.into(),
+      &all_tags,
+      |server| {
+        states.is_empty() || states.contains(&server.info.state)
+      },
     )
+    .await?;
+    Ok(servers)
+  }
+}
+
+/// Build an in memory sort on the list item stats,
+/// matching the stats displayed on the server stats table.
+/// Servers without stats (unreachable / disabled) order last.
+fn stats_sort(
+  metric: fn(&MinimalSystemStats) -> f64,
+) -> resource::ListItemSort<ServerListItem> {
+  resource::ListItemSort::InMemory(Box::new(move |a, b| {
+    match (a.info.stats.as_ref(), b.info.stats.as_ref()) {
+      (Some(a_stats), Some(b_stats)) => {
+        metric(a_stats).total_cmp(&metric(b_stats))
+      }
+      (Some(_), None) => Ordering::Greater,
+      (None, Some(_)) => Ordering::Less,
+      (None, None) => Ordering::Equal,
+    }
+    .then_with(|| a.name.cmp(&b.name))
+  }))
+}
+
+fn usage_percent(used: f64, total: f64) -> f64 {
+  if total > 0.0 {
+    100.0 * used / total
+  } else {
+    0.0
   }
 }
 
@@ -147,12 +207,29 @@ impl Resolve<ReadArgs> for ListFullServers {
     } else {
       get_all_tags(None).await?
     };
+    let states = self.query.specific.states.clone();
+    let limit = list_limit(self.limit);
     Ok(
-      resource::list_full_for_user::<Server>(
+      resource::list_full_for_user_filtered::<Server, _>(
         self.query,
+        limit,
+        self.page,
         user,
         PermissionLevel::Read.into(),
         &all_tags,
+        |server| {
+          let states = states.clone();
+          async move {
+            if states.is_empty()
+              || states
+                .contains(&get_cached_server_state(&server.id).await)
+            {
+              Some(server)
+            } else {
+              None
+            }
+          }
+        },
       )
       .await?,
     )
@@ -339,7 +416,7 @@ impl Resolve<ReadArgs> for GetHistoricalServerStats {
     let curr_ts = unix_timestamp_ms() as i64;
     let mut curr_ts = curr_ts
       - curr_ts % granularity
-      - granularity * STATS_PER_PAGE * page as i64;
+      - granularity * (page as i64).saturating_mul(STATS_PER_PAGE);
     for _ in 0..STATS_PER_PAGE {
       ts_vec.push(curr_ts);
       curr_ts -= granularity;
@@ -353,7 +430,6 @@ impl Resolve<ReadArgs> for GetHistoricalServerStats {
       },
       FindOptions::builder()
         .sort(doc! { "ts": -1 })
-        .skip(page as u64 * STATS_PER_PAGE as u64)
         .limit(STATS_PER_PAGE)
         .build(),
     )
@@ -368,512 +444,3 @@ impl Resolve<ReadArgs> for GetHistoricalServerStats {
     Ok(res)
   }
 }
-
-impl Resolve<ReadArgs> for ListDockerContainers {
-  async fn resolve(
-    self,
-    ReadArgs { user }: &ReadArgs,
-  ) -> mogh_error::Result<ListDockerContainersResponse> {
-    let server = get_check_permissions::<Server>(
-      &self.server,
-      user,
-      PermissionLevel::Read.into(),
-    )
-    .await?;
-    let cache = server_status_cache()
-      .get_or_insert_default(&server.id)
-      .await;
-    if let Some(docker) = &cache.docker {
-      Ok(docker.containers.clone())
-    } else {
-      Ok(Vec::new())
-    }
-  }
-}
-
-impl Resolve<ReadArgs> for ListAllDockerContainers {
-  async fn resolve(
-    self,
-    ReadArgs { user }: &ReadArgs,
-  ) -> mogh_error::Result<ListAllDockerContainersResponse> {
-    let servers = resource::list_for_user::<Server>(
-      ServerQuery::builder().names(self.servers.clone()).build(),
-      user,
-      PermissionLevel::Read.into(),
-      &[],
-    )
-    .await?;
-
-    let mut containers = Vec::<ContainerListItem>::new();
-
-    for server in servers {
-      let cache = server_status_cache()
-        .get_or_insert_default(&server.id)
-        .await;
-      let Some(docker) = &cache.docker else {
-        continue;
-      };
-      let more = docker
-        .containers
-        .iter()
-        .filter(|container| {
-          self.containers.is_empty()
-            || self.containers.contains(&container.name)
-        })
-        .cloned();
-      containers.extend(more);
-    }
-
-    Ok(containers)
-  }
-}
-
-impl Resolve<ReadArgs> for GetDockerContainersSummary {
-  async fn resolve(
-    self,
-    ReadArgs { user }: &ReadArgs,
-  ) -> mogh_error::Result<GetDockerContainersSummaryResponse> {
-    let servers = resource::list_full_for_user::<Server>(
-      Default::default(),
-      user,
-      PermissionLevel::Read.into(),
-      &[],
-    )
-    .await
-    .context("failed to get servers from db")?;
-
-    let mut res = GetDockerContainersSummaryResponse::default();
-
-    for server in servers {
-      let cache = server_status_cache()
-        .get_or_insert_default(&server.id)
-        .await;
-
-      if let Some(docker) = &cache.docker {
-        for container in &docker.containers {
-          res.total += 1;
-          match container.state {
-            ContainerStateStatusEnum::Created
-            | ContainerStateStatusEnum::Paused
-            | ContainerStateStatusEnum::Exited => res.stopped += 1,
-            ContainerStateStatusEnum::Running => res.running += 1,
-            ContainerStateStatusEnum::Empty => res.unknown += 1,
-            _ => res.unhealthy += 1,
-          }
-        }
-      }
-    }
-
-    Ok(res)
-  }
-}
-
-impl Resolve<ReadArgs> for InspectDockerContainer {
-  async fn resolve(
-    self,
-    ReadArgs { user }: &ReadArgs,
-  ) -> mogh_error::Result<Container> {
-    let server = get_check_permissions::<Server>(
-      &self.server,
-      user,
-      PermissionLevel::Read.inspect(),
-    )
-    .await?;
-    let cache = server_status_cache()
-      .get_or_insert_default(&server.id)
-      .await;
-    if cache.state != ServerState::Ok {
-      return Err(
-        anyhow!(
-          "Cannot inspect container: server is {:?}",
-          cache.state
-        )
-        .into(),
-      );
-    }
-    let res = periphery_client(&server)
-      .await?
-      .request(InspectContainer {
-        name: self.container,
-      })
-      .await?;
-    Ok(res)
-  }
-}
-
-const MAX_LOG_LENGTH: u64 = 5000;
-
-impl Resolve<ReadArgs> for GetContainerLog {
-  async fn resolve(
-    self,
-    ReadArgs { user }: &ReadArgs,
-  ) -> mogh_error::Result<Log> {
-    let GetContainerLog {
-      server,
-      container,
-      tail,
-      timestamps,
-    } = self;
-    let server = get_check_permissions::<Server>(
-      &server,
-      user,
-      PermissionLevel::Read.logs(),
-    )
-    .await?;
-    let res = periphery_client(&server)
-      .await?
-      .request(periphery::container::GetContainerLog {
-        name: container,
-        tail: cmp::min(tail, MAX_LOG_LENGTH),
-        timestamps,
-      })
-      .await
-      .context("failed at call to periphery")?;
-    Ok(res)
-  }
-}
-
-impl Resolve<ReadArgs> for SearchContainerLog {
-  async fn resolve(
-    self,
-    ReadArgs { user }: &ReadArgs,
-  ) -> mogh_error::Result<Log> {
-    let SearchContainerLog {
-      server,
-      container,
-      terms,
-      combinator,
-      invert,
-      timestamps,
-    } = self;
-    let server = get_check_permissions::<Server>(
-      &server,
-      user,
-      PermissionLevel::Read.logs(),
-    )
-    .await?;
-    let res = periphery_client(&server)
-      .await?
-      .request(periphery::container::GetContainerLogSearch {
-        name: container,
-        terms,
-        combinator,
-        invert,
-        timestamps,
-      })
-      .await
-      .context("failed at call to periphery")?;
-    Ok(res)
-  }
-}
-
-impl Resolve<ReadArgs> for GetResourceMatchingContainer {
-  async fn resolve(
-    self,
-    ReadArgs { user }: &ReadArgs,
-  ) -> mogh_error::Result<GetResourceMatchingContainerResponse> {
-    let server = get_check_permissions::<Server>(
-      &self.server,
-      user,
-      PermissionLevel::Read.into(),
-    )
-    .await?;
-    // first check deployments
-    if let Ok(deployment) =
-      resource::get::<Deployment>(&self.container).await
-    {
-      return Ok(GetResourceMatchingContainerResponse {
-        resource: ResourceTarget::Deployment(deployment.id).into(),
-      });
-    }
-
-    // then check stacks
-    let stacks = list_resources_for_user::<Stack>(
-      doc! { "config.server_id": &server.id },
-      user,
-      PermissionLevel::Read.into(),
-    )
-    .await?;
-
-    // check matching stack
-    for stack in stacks {
-      for StackServiceNames {
-        service_name,
-        container_name,
-        ..
-      } in stack
-        .info
-        .deployed_services
-        .unwrap_or(stack.info.latest_services)
-      {
-        let is_match = match compose_container_match_regex(&container_name)
-          .with_context(|| format!("failed to construct container name matching regex for service {service_name}")) 
-        {
-          Ok(regex) => regex,
-          Err(e) => {
-            warn!("{e:#}");
-            continue;
-          }
-        }.is_match(&self.container);
-
-        if is_match {
-          return Ok(GetResourceMatchingContainerResponse {
-            resource: ResourceTarget::Stack(stack.id).into(),
-          });
-        }
-      }
-    }
-
-    Ok(GetResourceMatchingContainerResponse { resource: None })
-  }
-}
-
-impl Resolve<ReadArgs> for ListDockerNetworks {
-  async fn resolve(
-    self,
-    ReadArgs { user }: &ReadArgs,
-  ) -> mogh_error::Result<ListDockerNetworksResponse> {
-    let server = get_check_permissions::<Server>(
-      &self.server,
-      user,
-      PermissionLevel::Read.into(),
-    )
-    .await?;
-    let cache = server_status_cache()
-      .get_or_insert_default(&server.id)
-      .await;
-    if let Some(docker) = &cache.docker {
-      Ok(docker.networks.clone())
-    } else {
-      Ok(Vec::new())
-    }
-  }
-}
-
-impl Resolve<ReadArgs> for InspectDockerNetwork {
-  async fn resolve(
-    self,
-    ReadArgs { user }: &ReadArgs,
-  ) -> mogh_error::Result<Network> {
-    let server = get_check_permissions::<Server>(
-      &self.server,
-      user,
-      PermissionLevel::Read.into(),
-    )
-    .await?;
-    let cache = server_status_cache()
-      .get_or_insert_default(&server.id)
-      .await;
-    if cache.state != ServerState::Ok {
-      return Err(
-        anyhow!(
-          "Cannot inspect network: server is {:?}",
-          cache.state
-        )
-        .into(),
-      );
-    }
-    let res = periphery_client(&server)
-      .await?
-      .request(InspectNetwork { name: self.network })
-      .await?;
-    Ok(res)
-  }
-}
-
-impl Resolve<ReadArgs> for ListDockerImages {
-  async fn resolve(
-    self,
-    ReadArgs { user }: &ReadArgs,
-  ) -> mogh_error::Result<ListDockerImagesResponse> {
-    let server = get_check_permissions::<Server>(
-      &self.server,
-      user,
-      PermissionLevel::Read.into(),
-    )
-    .await?;
-    let cache = server_status_cache()
-      .get_or_insert_default(&server.id)
-      .await;
-    if let Some(docker) = &cache.docker {
-      Ok(docker.images.clone())
-    } else {
-      Ok(Vec::new())
-    }
-  }
-}
-
-impl Resolve<ReadArgs> for InspectDockerImage {
-  async fn resolve(
-    self,
-    ReadArgs { user }: &ReadArgs,
-  ) -> mogh_error::Result<Image> {
-    let server = get_check_permissions::<Server>(
-      &self.server,
-      user,
-      PermissionLevel::Read.into(),
-    )
-    .await?;
-    let cache = server_status_cache()
-      .get_or_insert_default(&server.id)
-      .await;
-    if cache.state != ServerState::Ok {
-      return Err(
-        anyhow!("Cannot inspect image: server is {:?}", cache.state)
-          .into(),
-      );
-    }
-    let res = periphery_client(&server)
-      .await?
-      .request(InspectImage { name: self.image })
-      .await?;
-    Ok(res)
-  }
-}
-
-impl Resolve<ReadArgs> for ListDockerImageHistory {
-  async fn resolve(
-    self,
-    ReadArgs { user }: &ReadArgs,
-  ) -> mogh_error::Result<Vec<ImageHistoryResponseItem>> {
-    let server = get_check_permissions::<Server>(
-      &self.server,
-      user,
-      PermissionLevel::Read.into(),
-    )
-    .await?;
-    let cache = server_status_cache()
-      .get_or_insert_default(&server.id)
-      .await;
-    if cache.state != ServerState::Ok {
-      return Err(
-        anyhow!(
-          "Cannot get image history: server is {:?}",
-          cache.state
-        )
-        .into(),
-      );
-    }
-    let res = periphery_client(&server)
-      .await?
-      .request(ImageHistory { name: self.image })
-      .await?;
-    Ok(res)
-  }
-}
-
-impl Resolve<ReadArgs> for ListDockerVolumes {
-  async fn resolve(
-    self,
-    ReadArgs { user }: &ReadArgs,
-  ) -> mogh_error::Result<ListDockerVolumesResponse> {
-    let server = get_check_permissions::<Server>(
-      &self.server,
-      user,
-      PermissionLevel::Read.into(),
-    )
-    .await?;
-    let cache = server_status_cache()
-      .get_or_insert_default(&server.id)
-      .await;
-    if let Some(docker) = &cache.docker {
-      Ok(docker.volumes.clone())
-    } else {
-      Ok(Vec::new())
-    }
-  }
-}
-
-impl Resolve<ReadArgs> for InspectDockerVolume {
-  async fn resolve(
-    self,
-    ReadArgs { user }: &ReadArgs,
-  ) -> mogh_error::Result<Volume> {
-    let server = get_check_permissions::<Server>(
-      &self.server,
-      user,
-      PermissionLevel::Read.into(),
-    )
-    .await?;
-    let cache = server_status_cache()
-      .get_or_insert_default(&server.id)
-      .await;
-    if cache.state != ServerState::Ok {
-      return Err(
-        anyhow!("Cannot inspect volume: server is {:?}", cache.state)
-          .into(),
-      );
-    }
-    let res = periphery_client(&server)
-      .await?
-      .request(InspectVolume { name: self.volume })
-      .await?;
-    Ok(res)
-  }
-}
-
-impl Resolve<ReadArgs> for ListComposeProjects {
-  async fn resolve(
-    self,
-    ReadArgs { user }: &ReadArgs,
-  ) -> mogh_error::Result<ListComposeProjectsResponse> {
-    let server = get_check_permissions::<Server>(
-      &self.server,
-      user,
-      PermissionLevel::Read.into(),
-    )
-    .await?;
-    let cache = server_status_cache()
-      .get_or_insert_default(&server.id)
-      .await;
-    if let Some(docker) = &cache.docker {
-      Ok(docker.projects.clone())
-    } else {
-      Ok(Vec::new())
-    }
-  }
-}
-
-// impl Resolve<ReadArgs> for ListAllTerminals {
-//   async fn resolve(
-//     self,
-//     args: &ReadArgs,
-//   ) -> Result<Self::Response, Self::Error> {
-//     // match self.tar
-//     let mut terminals = resource::list_full_for_user::<Server>(
-//       self.query, &args.user, &all_tags,
-//     )
-//     .await?
-//     .into_iter()
-//     .map(|server| async move {
-//       (
-//         list_terminals_inner(&server, self.fresh).await,
-//         (server.id, server.name),
-//       )
-//     })
-//     .collect::<FuturesUnordered<_>>()
-//     .collect::<Vec<_>>()
-//     .await
-//     .into_iter()
-//     .flat_map(|(terminals, server)| {
-//       let terminals = terminals.ok()?;
-//       Some((terminals, server))
-//     })
-//     .flat_map(|(terminals, (server_id, server_name))| {
-//       terminals.into_iter().map(move |info| {
-//         TerminalInfoWithServer::from_terminal_info(
-//           &server_id,
-//           &server_name,
-//           info,
-//         )
-//       })
-//     })
-//     .collect::<Vec<_>>();
-
-//     terminals.sort_by(|a, b| {
-//       a.server_name.cmp(&b.server_name).then(a.name.cmp(&b.name))
-//     });
-
-//     Ok(terminals)
-//   }
-// }

bin/core/src/api/read/stack.rs

@@ -9,7 +9,10 @@ use komodo_client::{
       container::Container, service::SwarmService, stack::SwarmStack,
     },
     permission::PermissionLevel,
-    stack::{Stack, StackActionState, StackListItem, StackState},
+    stack::{
+      Stack, StackActionState, StackListItem, StackQuery,
+      StackService, StackSortBy, StackState,
+    },
   },
 };
 use mogh_error::AddStatusCodeError as _;
@@ -22,7 +25,9 @@ use reqwest::StatusCode;
 
 use crate::{
   helpers::{
-    periphery_client, query::get_all_tags, swarm::swarm_request,
+    periphery_client,
+    query::{get_all_tags, get_cached_stack_state},
+    swarm::swarm_request,
   },
   permission::get_check_permissions,
   resource,
@@ -30,7 +35,7 @@ use crate::{
   state::{action_states, stack_status_cache},
 };
 
-use super::ReadArgs;
+use super::{ReadArgs, list_limit};
 
 impl Resolve<ReadArgs> for GetStack {
   async fn resolve(
@@ -48,6 +53,128 @@ impl Resolve<ReadArgs> for GetStack {
   }
 }
 
+impl Resolve<ReadArgs> for ListStacks {
+  async fn resolve(
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> mogh_error::Result<Vec<StackListItem>> {
+    let all_tags = if self.query.tags.is_empty() {
+      vec![]
+    } else {
+      get_all_tags(None).await?
+    };
+    let only_update_available = self.query.specific.update_available;
+    let states = self.query.specific.states.clone();
+    let limit = list_limit(self.limit);
+    let sort_by: resource::ListItemSort<StackListItem> =
+      match self.sort_by {
+        StackSortBy::Name => resource::ListItemSort::Name,
+        StackSortBy::Source => {
+          resource::ListItemSort::InMemory(Box::new(|a, b| {
+            a.info
+              .files_on_host
+              .cmp(&b.info.files_on_host)
+              .then_with(|| {
+                a.info.linked_repo_name.cmp(&b.info.linked_repo_name)
+              })
+              .then_with(|| a.info.repo.cmp(&b.info.repo))
+              .then_with(|| a.name.cmp(&b.name))
+          }))
+        }
+        StackSortBy::Host => {
+          resource::ListItemSort::InMemory(Box::new(|a, b| {
+            let host_a = if a.info.swarm_id.is_empty() {
+              &a.info.server_name
+            } else {
+              &a.info.swarm_name
+            };
+            let host_b = if b.info.swarm_id.is_empty() {
+              &b.info.server_name
+            } else {
+              &b.info.swarm_name
+            };
+            host_a.cmp(host_b).then_with(|| a.name.cmp(&b.name))
+          }))
+        }
+        StackSortBy::State => {
+          resource::ListItemSort::InMemory(Box::new(|a, b| {
+            a.info
+              .state
+              .cmp(&b.info.state)
+              .then_with(|| {
+                // Use ! with update available to order 'true' first
+                (!a.info.update_available())
+                  .cmp(&!b.info.update_available())
+              })
+              .then_with(|| a.name.cmp(&b.name))
+          }))
+        }
+      };
+    let stacks = resource::list_items_for_user::<Stack>(
+      self.query,
+      resource::ListItemsQueryOptions {
+        limit,
+        page: self.page,
+        sort_desc: self.sort_desc,
+        sort_by,
+      },
+      user,
+      PermissionLevel::Read.into(),
+      &all_tags,
+      |stack| {
+        (!only_update_available
+          || stack
+            .info
+            .services
+            .iter()
+            .any(|service| service.update_available))
+          && (states.is_empty() || states.contains(&stack.info.state))
+      },
+    )
+    .await?;
+    Ok(stacks)
+  }
+}
+
+impl Resolve<ReadArgs> for ListFullStacks {
+  async fn resolve(
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> mogh_error::Result<ListFullStacksResponse> {
+    let all_tags = if self.query.tags.is_empty() {
+      vec![]
+    } else {
+      get_all_tags(None).await?
+    };
+    let states = self.query.specific.states.clone();
+    let limit = list_limit(self.limit);
+    Ok(
+      resource::list_full_for_user_filtered::<Stack, _>(
+        self.query,
+        limit,
+        self.page,
+        user,
+        PermissionLevel::Read.into(),
+        &all_tags,
+        |stack| {
+          let states = states.clone();
+          async move {
+            if states.is_empty()
+              || states
+                .contains(&get_cached_stack_state(&stack.id).await)
+            {
+              Some(stack)
+            } else {
+              None
+            }
+          }
+        },
+      )
+      .await?,
+    )
+  }
+}
+
 impl Resolve<ReadArgs> for ListStackServices {
   async fn resolve(
     self,
@@ -72,6 +199,75 @@ impl Resolve<ReadArgs> for ListStackServices {
   }
 }
 
+impl Resolve<ReadArgs> for ListAllStackServices {
+  async fn resolve(
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> mogh_error::Result<ListStackServicesResponse> {
+    let all_tags = if self.tags.is_empty() {
+      vec![]
+    } else {
+      get_all_tags(None).await?
+    };
+    let stacks = resource::list_for_user::<Stack>(
+      StackQuery::builder()
+        .names(self.stacks.clone())
+        .tags(self.tags)
+        .build(),
+      None,
+      None,
+      user,
+      PermissionLevel::Read.into(),
+      &all_tags,
+    )
+    .await?;
+
+    let mut services = Vec::<StackService>::new();
+    let mut skipped = 0;
+    let limit = list_limit(self.limit);
+    let limit_usize = limit as usize;
+    // Eg. page 1 skips until after 100 services, page 2 after 200.
+    let skip = limit.saturating_mul(self.page);
+    // Match terms case insensitively.
+    let terms = self
+      .terms
+      .iter()
+      .map(|term| term.to_lowercase())
+      .collect::<Vec<_>>();
+
+    for stack in stacks {
+      let cache =
+        stack_status_cache().get_or_insert_default(&stack.id).await;
+      let more = cache.curr.services
+        .iter()
+        .filter(|service| {
+          // Apply state filter if defined.
+          (self.state.is_empty() || self.state.contains(&service.state)) &&
+          // Apply terms filter if defined
+          (terms.is_empty()
+            // Match when all terms contained within a name.
+            || {
+              let name = service.service.to_lowercase();
+              terms.iter().all(|term| name.contains(term))
+            })
+        });
+      for service in more {
+        if skipped < skip {
+          skipped += 1;
+        } else {
+          // push and maybe early return
+          services.push(service.clone());
+          if limit > 0 && services.len() >= limit_usize {
+            return Ok(services);
+          }
+        }
+      }
+    }
+
+    Ok(services)
+  }
+}
+
 impl Resolve<ReadArgs> for GetStackLog {
   async fn resolve(
     self,
@@ -339,6 +535,8 @@ impl Resolve<ReadArgs> for ListCommonStackExtraArgs {
     };
     let stacks = resource::list_full_for_user::<Stack>(
       self.query,
+      None,
+      None,
       user,
       PermissionLevel::Read.into(),
       &all_tags,
@@ -373,6 +571,8 @@ impl Resolve<ReadArgs> for ListCommonStackBuildExtraArgs {
     };
     let stacks = resource::list_full_for_user::<Stack>(
       self.query,
+      None,
+      None,
       user,
       PermissionLevel::Read.into(),
       &all_tags,
@@ -395,64 +595,6 @@ impl Resolve<ReadArgs> for ListCommonStackBuildExtraArgs {
   }
 }
 
-impl Resolve<ReadArgs> for ListStacks {
-  async fn resolve(
-    self,
-    ReadArgs { user }: &ReadArgs,
-  ) -> mogh_error::Result<Vec<StackListItem>> {
-    let all_tags = if self.query.tags.is_empty() {
-      vec![]
-    } else {
-      get_all_tags(None).await?
-    };
-    let only_update_available = self.query.specific.update_available;
-    let stacks = resource::list_for_user::<Stack>(
-      self.query,
-      user,
-      PermissionLevel::Read.into(),
-      &all_tags,
-    )
-    .await?;
-    let stacks = if only_update_available {
-      stacks
-        .into_iter()
-        .filter(|stack| {
-          stack
-            .info
-            .services
-            .iter()
-            .any(|service| service.update_available)
-        })
-        .collect()
-    } else {
-      stacks
-    };
-    Ok(stacks)
-  }
-}
-
-impl Resolve<ReadArgs> for ListFullStacks {
-  async fn resolve(
-    self,
-    ReadArgs { user }: &ReadArgs,
-  ) -> mogh_error::Result<ListFullStacksResponse> {
-    let all_tags = if self.query.tags.is_empty() {
-      vec![]
-    } else {
-      get_all_tags(None).await?
-    };
-    Ok(
-      resource::list_full_for_user::<Stack>(
-        self.query,
-        user,
-        PermissionLevel::Read.into(),
-        &all_tags,
-      )
-      .await?,
-    )
-  }
-}
-
 impl Resolve<ReadArgs> for GetStackActionState {
   async fn resolve(
     self,
@@ -481,6 +623,8 @@ impl Resolve<ReadArgs> for GetStacksSummary {
   ) -> mogh_error::Result<GetStacksSummaryResponse> {
     let stacks = resource::list_full_for_user::<Stack>(
       Default::default(),
+      None,
+      None,
       user,
       PermissionLevel::Read.into(),
       &[],

bin/core/src/api/read/swarm.rs

@@ -3,7 +3,9 @@ use komodo_client::{
   api::read::*,
   entities::{
     permission::PermissionLevel,
-    swarm::{Swarm, SwarmActionState, SwarmListItem, SwarmState},
+    swarm::{
+      Swarm, SwarmActionState, SwarmListItem, SwarmSortBy, SwarmState,
+    },
   },
 };
 use mogh_resolver::Resolve;
@@ -15,7 +17,7 @@ use crate::{
   state::{action_states, server_status_cache, swarm_status_cache},
 };
 
-use super::ReadArgs;
+use super::{ReadArgs, list_limit};
 
 impl Resolve<ReadArgs> for GetSwarm {
   async fn resolve(
@@ -43,12 +45,32 @@ impl Resolve<ReadArgs> for ListSwarms {
     } else {
       get_all_tags(None).await?
     };
+    let limit = list_limit(self.limit);
+    let sort_by: resource::ListItemSort<SwarmListItem> =
+      match self.sort_by {
+        SwarmSortBy::Name => resource::ListItemSort::Name,
+        SwarmSortBy::State => {
+          resource::ListItemSort::InMemory(Box::new(|a, b| {
+            a.info
+              .state
+              .cmp(&b.info.state)
+              .then_with(|| a.name.cmp(&b.name))
+          }))
+        }
+      };
     Ok(
-      resource::list_for_user::<Swarm>(
+      resource::list_items_for_user::<Swarm>(
         self.query,
+        resource::ListItemsQueryOptions {
+          limit,
+          page: self.page,
+          sort_desc: self.sort_desc,
+          sort_by,
+        },
         user,
         PermissionLevel::Read.into(),
         &all_tags,
+        |_| true,
       )
       .await?,
     )
@@ -65,9 +87,12 @@ impl Resolve<ReadArgs> for ListFullSwarms {
     } else {
       get_all_tags(None).await?
     };
+    let limit = list_limit(self.limit);
     Ok(
       resource::list_full_for_user::<Swarm>(
         self.query,
+        limit as i64,
+        self.page.saturating_mul(limit),
         user,
         PermissionLevel::Read.into(),
         &all_tags,
@@ -105,6 +130,8 @@ impl Resolve<ReadArgs> for GetSwarmsSummary {
   ) -> mogh_error::Result<GetSwarmsSummaryResponse> {
     let swarms = resource::list_full_for_user::<Swarm>(
       Default::default(),
+      None,
+      None,
       user,
       PermissionLevel::Read.into(),
       &[],

bin/core/src/api/read/sync.rs

@@ -5,6 +5,7 @@ use komodo_client::{
     permission::PermissionLevel,
     sync::{
       ResourceSync, ResourceSyncActionState, ResourceSyncListItem,
+      ResourceSyncSortBy,
     },
   },
 };
@@ -15,7 +16,7 @@ use crate::{
   resource, state::action_states,
 };
 
-use super::ReadArgs;
+use super::{ReadArgs, list_limit};
 
 impl Resolve<ReadArgs> for GetResourceSync {
   async fn resolve(
@@ -43,12 +44,52 @@ impl Resolve<ReadArgs> for ListResourceSyncs {
     } else {
       get_all_tags(None).await?
     };
+    let limit = list_limit(self.limit);
+    let sort_by: resource::ListItemSort<ResourceSyncListItem> =
+      match self.sort_by {
+        ResourceSyncSortBy::Name => resource::ListItemSort::Name,
+        ResourceSyncSortBy::Source => {
+          resource::ListItemSort::InMemory(Box::new(|a, b| {
+            a.info
+              .files_on_host
+              .cmp(&b.info.files_on_host)
+              .then_with(|| {
+                a.info.linked_repo_name.cmp(&b.info.linked_repo_name)
+              })
+              .then_with(|| a.info.repo.cmp(&b.info.repo))
+              .then_with(|| a.name.cmp(&b.name))
+          }))
+        }
+        ResourceSyncSortBy::Branch => {
+          resource::ListItemSort::InMemory(Box::new(|a, b| {
+            a.info
+              .branch
+              .cmp(&b.info.branch)
+              .then_with(|| a.name.cmp(&b.name))
+          }))
+        }
+        ResourceSyncSortBy::State => {
+          resource::ListItemSort::InMemory(Box::new(|a, b| {
+            a.info
+              .state
+              .cmp(&b.info.state)
+              .then_with(|| a.name.cmp(&b.name))
+          }))
+        }
+      };
     Ok(
-      resource::list_for_user::<ResourceSync>(
+      resource::list_items_for_user::<ResourceSync>(
         self.query,
+        resource::ListItemsQueryOptions {
+          limit,
+          page: self.page,
+          sort_desc: self.sort_desc,
+          sort_by,
+        },
         user,
         PermissionLevel::Read.into(),
         &all_tags,
+        |_| true,
       )
       .await?,
     )
@@ -65,9 +106,12 @@ impl Resolve<ReadArgs> for ListFullResourceSyncs {
     } else {
       get_all_tags(None).await?
     };
+    let limit = list_limit(self.limit);
     Ok(
       resource::list_full_for_user::<ResourceSync>(
         self.query,
+        limit as i64,
+        self.page.saturating_mul(limit),
         user,
         PermissionLevel::Read.into(),
         &all_tags,
@@ -106,6 +150,8 @@ impl Resolve<ReadArgs> for GetResourceSyncsSummary {
     let resource_syncs =
       resource::list_full_for_user::<ResourceSync>(
         Default::default(),
+        None,
+        None,
         user,
         PermissionLevel::Read.into(),
         &[],

bin/core/src/api/read/terminal.rs

@@ -1,3 +1,5 @@
+use std::cmp::Ordering;
+
 use anyhow::Context as _;
 use futures_util::{
   FutureExt, StreamExt as _, stream::FuturesUnordered,
@@ -9,7 +11,7 @@ use komodo_client::{
     permission::PermissionLevel,
     server::Server,
     stack::Stack,
-    terminal::{Terminal, TerminalTarget},
+    terminal::{Terminal, TerminalSortBy, TerminalTarget},
     user::User,
   },
 };
@@ -22,7 +24,7 @@ use crate::{
   resource,
 };
 
-use super::ReadArgs;
+use super::{ReadArgs, list_limit};
 
 //
 
@@ -31,57 +33,111 @@ impl Resolve<ReadArgs> for ListTerminals {
     self,
     ReadArgs { user }: &ReadArgs,
   ) -> mogh_error::Result<ListTerminalsResponse> {
-    let Some(target) = self.target else {
-      return list_all_terminals_for_user(user, self.use_names).await;
+    let mut terminals = match self.target {
+      None => {
+        list_all_terminals_for_user(user, self.use_names).await?
+      }
+      Some(target) => match &target {
+        TerminalTarget::Server { server } => {
+          let server = server
+            .as_ref()
+            .context("Must provide 'target.params.server'")
+            .status_code(StatusCode::BAD_REQUEST)?;
+          let server = get_check_permissions::<Server>(
+            server,
+            user,
+            PermissionLevel::Read.terminal(),
+          )
+          .await?;
+          list_terminals_on_server(&server, Some(target)).await?
+        }
+        TerminalTarget::Container { server, .. } => {
+          let server = get_check_permissions::<Server>(
+            server,
+            user,
+            PermissionLevel::Read.terminal(),
+          )
+          .await?;
+          list_terminals_on_server(&server, Some(target)).await?
+        }
+        TerminalTarget::Stack { stack, .. } => {
+          let server = get_check_permissions::<Stack>(
+            stack,
+            user,
+            PermissionLevel::Read.terminal(),
+          )
+          .await?
+          .config
+          .server_id;
+          let server = resource::get::<Server>(&server).await?;
+          list_terminals_on_server(&server, Some(target)).await?
+        }
+        TerminalTarget::Deployment { deployment } => {
+          let server = get_check_permissions::<Deployment>(
+            deployment,
+            user,
+            PermissionLevel::Read.terminal(),
+          )
+          .await?
+          .config
+          .server_id;
+          let server = resource::get::<Server>(&server).await?;
+          list_terminals_on_server(&server, Some(target)).await?
+        }
+      },
     };
-    match &target {
-      TerminalTarget::Server { server } => {
-        let server = server
-          .as_ref()
-          .context("Must provide 'target.params.server'")
-          .status_code(StatusCode::BAD_REQUEST)?;
-        let server = get_check_permissions::<Server>(
-          server,
-          user,
-          PermissionLevel::Read.terminal(),
-        )
-        .await?;
-        list_terminals_on_server(&server, Some(target)).await
-      }
-      TerminalTarget::Container { server, .. } => {
-        let server = get_check_permissions::<Server>(
-          server,
-          user,
-          PermissionLevel::Read.terminal(),
-        )
-        .await?;
-        list_terminals_on_server(&server, Some(target)).await
-      }
-      TerminalTarget::Stack { stack, .. } => {
-        let server = get_check_permissions::<Stack>(
-          stack,
-          user,
-          PermissionLevel::Read.terminal(),
-        )
-        .await?
-        .config
-        .server_id;
-        let server = resource::get::<Server>(&server).await?;
-        list_terminals_on_server(&server, Some(target)).await
-      }
-      TerminalTarget::Deployment { deployment } => {
-        let server = get_check_permissions::<Deployment>(
-          deployment,
-          user,
-          PermissionLevel::Read.terminal(),
-        )
-        .await?
-        .config
-        .server_id;
-        let server = resource::get::<Server>(&server).await?;
-        list_terminals_on_server(&server, Some(target)).await
-      }
+
+    // The terminals come from Periphery agents rather than the db,
+    // so the terms filter / sort / pagination are applied in memory.
+    if !self.terms.is_empty() {
+      let terms = self
+        .terms
+        .iter()
+        .map(|term| term.to_lowercase())
+        .collect::<Vec<_>>();
+      terminals.retain(|terminal| {
+        let name = terminal.name.to_lowercase();
+        terms.iter().all(|term| name.contains(term))
+      });
     }
+
+    // All comparators fall back to name based sorting for equal
+    // sort keys, inside `compare`, so descending sorts are fully
+    // descending, matching the List<Resource> apis.
+    let compare: fn(&Terminal, &Terminal) -> Ordering =
+      match self.sort_by {
+        TerminalSortBy::Name => |a, b| a.name.cmp(&b.name),
+        TerminalSortBy::Target => |a, b| {
+          a.target.cmp(&b.target).then_with(|| a.name.cmp(&b.name))
+        },
+        TerminalSortBy::Command => |a, b| {
+          a.command.cmp(&b.command).then_with(|| a.name.cmp(&b.name))
+        },
+        TerminalSortBy::Size => |a, b| {
+          a.stored_size_kb
+            .total_cmp(&b.stored_size_kb)
+            .then_with(|| a.name.cmp(&b.name))
+        },
+        TerminalSortBy::Created => |a, b| {
+          a.created_at
+            .cmp(&b.created_at)
+            .then_with(|| a.name.cmp(&b.name))
+        },
+      };
+    if self.sort_desc {
+      terminals.sort_by(|a, b| compare(b, a));
+    } else {
+      terminals.sort_by(compare);
+    }
+
+    let limit = list_limit(self.limit);
+    let skip = limit.saturating_mul(self.page) as usize;
+    let take = if limit == 0 {
+      usize::MAX
+    } else {
+      limit as usize
+    };
+    Ok(terminals.into_iter().skip(skip).take(take).collect())
   }
 }
 
@@ -92,6 +148,8 @@ async fn list_all_terminals_for_user(
   let (mut servers, stacks, deployments) = tokio::try_join!(
     resource::list_full_for_user::<Server>(
       Default::default(),
+      None,
+      None,
       user,
       PermissionLevel::Read.terminal(),
       &[]
@@ -103,12 +161,16 @@ async fn list_all_terminals_for_user(
       .collect::<Vec<_>>())),
     resource::list_full_for_user::<Stack>(
       Default::default(),
+      None,
+      None,
       user,
       PermissionLevel::Read.terminal(),
       &[]
     ),
     resource::list_full_for_user::<Deployment>(
       Default::default(),
+      None,
+      None,
       user,
       PermissionLevel::Read.terminal(),
       &[]
@@ -139,7 +201,7 @@ async fn list_all_terminals_for_user(
     }
   }
 
-  let mut terminals = servers
+  let terminals = servers
     .into_iter()
     .map(|(server, server_permission)| async move {
       (
@@ -168,6 +230,7 @@ async fn list_all_terminals_for_user(
                       server_id.clone()
                     }),
                   };
+                  terminal.target_name = Some(server_name.clone());
                   terminal
                 }),
               TerminalTarget::Container { container, .. } => {
@@ -180,6 +243,7 @@ async fn list_all_terminals_for_user(
                     },
                     container,
                   };
+                  terminal.target_name = Some(server_name.clone());
                   terminal
                 })
               }
@@ -193,6 +257,7 @@ async fn list_all_terminals_for_user(
                     },
                     service,
                   };
+                  terminal.target_name = Some(s.name.clone());
                   terminal
                 })
               }
@@ -206,6 +271,7 @@ async fn list_all_terminals_for_user(
                         d.id.clone()
                       },
                     };
+                    terminal.target_name = Some(d.name.clone());
                     terminal
                   },
                 )
@@ -220,10 +286,6 @@ async fn list_all_terminals_for_user(
     .flatten()
     .collect::<Vec<_>>();
 
-  terminals.sort_by(|a, b| {
-    a.target.cmp(&b.target).then(a.name.cmp(&b.name))
-  });
-
   Ok(terminals)
 }
 

bin/core/src/api/read/toml.rs

@@ -1,5 +1,8 @@
 use anyhow::Context;
-use database::mungos::find::find_collect;
+use database::{
+  bson::doc,
+  mungos::{find::find_collect, mongodb::options::FindOptions},
+};
 use komodo_client::{
   api::read::{
     ExportAllResourcesToToml, ExportAllResourcesToTomlResponse,
@@ -25,6 +28,7 @@ use crate::{
   resource,
   state::db_client,
   sync::{
+    replace_ids::ReplaceIds,
     toml::{ToToml, convert_resource},
     user_groups::{convert_user_groups, user_group_to_toml},
     variables::variable_to_toml,
@@ -51,6 +55,8 @@ async fn get_all_targets(
         targets.extend(
           resource::list_full_for_user::<$Type>(
             ResourceQuery::builder().tags(tags).build(),
+            None,
+            None,
             user,
             PermissionLevel::Read.into(),
             &all_tags,
@@ -139,7 +145,7 @@ impl Resolve<ReadArgs> for ExportResourcesToToml {
           PermissionLevel::Read.into(),
         )
         .await?;
-        $Type::replace_ids(&mut resource);
+        $Type::replace_ids(&mut resource.config);
         let (deploy, after) = existing
           .as_ref()
           .and_then(|e| {
@@ -197,7 +203,7 @@ impl Resolve<ReadArgs> for ExportResourcesToToml {
               || !sync.config.repo.is_empty()
               || !sync.config.linked_repo.is_empty())
           {
-            ResourceSync::replace_ids(&mut sync);
+            ResourceSync::replace_ids(&mut sync.config);
             res.resource_syncs.push(convert_resource::<ResourceSync>(
               sync,
               false,
@@ -215,18 +221,21 @@ impl Resolve<ReadArgs> for ExportResourcesToToml {
       .context("failed to add user groups")?;
 
     if include_variables {
-      res.variables =
-        find_collect(&db_client().variables, None, None)
-          .await
-          .context("failed to get variables from db")?
-          .into_iter()
-          .map(|mut variable| {
-            if !user.admin && variable.is_secret {
-              variable.value = "#".repeat(variable.value.len())
-            }
-            variable
-          })
-          .collect();
+      res.variables = find_collect(
+        &db_client().variables,
+        None,
+        FindOptions::builder().sort(doc! { "name": 1 }).build(),
+      )
+      .await
+      .context("failed to get variables from db")?
+      .into_iter()
+      .map(|mut variable| {
+        if !user.admin && variable.is_secret {
+          variable.value = "#".repeat(variable.value.len())
+        }
+        variable
+      })
+      .collect();
     }
 
     let toml = serialize_resources_toml(res)

bin/core/src/api/read/update.rs

@@ -47,7 +47,9 @@ impl Resolve<ReadArgs> for ListUpdates {
       query,
       FindOptions::builder()
         .sort(doc! { "start_ts": -1 })
-        .skip(self.page as u64 * UPDATES_PER_PAGE as u64)
+        .skip(
+          (self.page as u64).saturating_mul(UPDATES_PER_PAGE as u64),
+        )
         .limit(UPDATES_PER_PAGE)
         .build(),
     )

bin/core/src/api/write/build.rs

@@ -1,10 +1,8 @@
-use std::{path::PathBuf, time::Duration};
+use std::path::PathBuf;
 
 use anyhow::{Context, anyhow};
+use database::mongo_indexed::doc;
 use database::mungos::mongodb::bson::to_document;
-use database::{
-  mongo_indexed::doc, mungos::mongodb::bson::oid::ObjectId,
-};
 use formatting::format_serror;
 use komodo_client::{
   api::write::*,
@@ -15,7 +13,6 @@ use komodo_client::{
     builder::{Builder, BuilderConfig},
     permission::PermissionLevel,
     repo::Repo,
-    server::ServerState,
     update::Update,
   },
 };
@@ -25,12 +22,11 @@ use periphery_client::api::build::{
 };
 use tokio::fs;
 
+use crate::helpers::builder::connect_builder_periphery;
 use crate::{
   config::core_config,
-  connection::PeripheryConnectionArgs,
   helpers::{
-    git_token, periphery_client,
-    query::get_server_with_state,
+    git_token,
     update::{add_update, make_update},
   },
   periphery::PeripheryClient,
@@ -476,48 +472,15 @@ async fn get_on_host_periphery(
     .await
     .context("Failed to get builder")?;
 
-  match builder.config {
-    BuilderConfig::Aws(_) => {
-      Err(anyhow!("Files on host doesn't work with AWS builder"))
-    }
-    BuilderConfig::Url(config) => {
-      // TODO: Ensure connection is actually established.
-      // Builder id no good because it may be active for multiple connections.
-      let periphery = PeripheryClient::new(
-        PeripheryConnectionArgs::from_url_builder(
-          &ObjectId::new().to_hex(),
-          &config,
-        ),
-        config.insecure_tls,
-      )
-      .await?;
-      // Poll for connection to be estalished
-      let mut err = None;
-      for _ in 0..10 {
-        tokio::time::sleep(Duration::from_secs(1)).await;
-        match periphery.health_check().await {
-          Ok(_) => return Ok(periphery),
-          Err(e) => err = Some(e),
-        };
-      }
-      Err(err.context("Missing error")?)
-    }
-    BuilderConfig::Server(config) => {
-      if config.server_id.is_empty() {
-        return Err(anyhow!(
-          "Builder is type server, but has no server attached"
-        ));
-      }
-      let (server, state) =
-        get_server_with_state(&config.server_id).await?;
-      if state != ServerState::Ok {
-        return Err(anyhow!(
-          "Builder server is disabled or not reachable"
-        ));
-      };
-      periphery_client(&server).await
-    }
+  if let BuilderConfig::Aws(_) = builder.config {
+    return Err(anyhow!(
+      "Files on host doesn't work with AWS builder"
+    ));
   }
+
+  connect_builder_periphery(build.name.clone(), None, builder, None)
+    .await
+    .map(|(periphery, _)| periphery)
 }
 
 /// The successful case will be included as Some(remote_contents).

bin/core/src/api/write/deployment.rs

@@ -15,6 +15,7 @@ use komodo_client::{
     docker::container::RestartPolicyNameEnum,
     komodo_timestamp, optional_string,
     permission::PermissionLevel,
+    resource::ResourceQuery,
     server::{Server, ServerState},
     to_container_compatible_name,
     update::Update,
@@ -30,7 +31,9 @@ use crate::{
   api::execute::{self, ExecuteRequest, ExecutionResult},
   helpers::{
     periphery_client,
-    query::{get_deployment_state, get_swarm_or_server},
+    query::{
+      get_all_tags, get_deployment_state, get_swarm_or_server,
+    },
     registry_token,
     update::{add_update, make_update, poll_update_until_complete},
   },
@@ -300,10 +303,24 @@ impl Resolve<WriteArgs> for RenameDeployment {
 
     let name = to_container_compatible_name(&self.name);
 
-    let container_state =
-      get_deployment_state(&deployment.id).await?;
+    let state = get_deployment_state(&deployment.id).await?;
 
-    if container_state == DeploymentState::Unknown {
+    // When no custom name is configured, the container / service
+    // follows the Deployment name, and must be kept matched:
+    //   - Server mode: rename the container to the new name.
+    //   - Swarm mode: services cannot be renamed, so the current
+    //     service name is pinned in info.deployed_name, and stays
+    //     in use until the next deploy recreates the service.
+    // A configured custom name is unaffected by Deployment rename.
+    let follows_name =
+      deployment.config.custom_name.trim().is_empty();
+    let deployed = state != DeploymentState::NotDeployed;
+
+    let rename_container = follows_name
+      && deployed
+      && !deployment.config.server_id.is_empty();
+
+    if rename_container && state == DeploymentState::Unknown {
       return Err(
         anyhow!(
           "Cannot rename Deployment when container status is unknown"
@@ -312,27 +329,51 @@ impl Resolve<WriteArgs> for RenameDeployment {
       );
     }
 
+    // Includes the Unknown state, in case a service
+    // is still running under the current name.
+    let pin_deployed_name = follows_name
+      && deployed
+      && !deployment.config.swarm_id.is_empty()
+      && deployment.info.deployed_name.is_empty();
+
     let mut update =
       make_update(&deployment, Operation::RenameDeployment, user);
 
+    let mut set =
+      doc! { "name": &name, "updated_at": komodo_timestamp() };
+    if rename_container {
+      // Container is renamed to the new name below.
+      set.insert("info.deployed_name", name.as_str());
+    } else if pin_deployed_name {
+      set.insert("info.deployed_name", deployment.deployed_name());
+    }
+
     update_one_by_id(
       &db_client().deployments,
       &deployment.id,
-      database::mungos::update::Update::Set(
-        doc! { "name": &name, "updated_at": komodo_timestamp() },
-      ),
+      database::mungos::update::Update::Set(set),
       None,
     )
     .await
     .context("Failed to update Deployment name on db")?;
 
-    if container_state != DeploymentState::NotDeployed {
+    if pin_deployed_name {
+      update.push_simple_log(
+        "Pin Service Name",
+        format!(
+          "Swarm services cannot be renamed, so the current service name '{}' stays in use until the next deploy recreates the service under the new name.",
+          deployment.deployed_name()
+        ),
+      );
+    }
+
+    if rename_container {
       let server =
         resource::get::<Server>(&deployment.config.server_id).await?;
       let log = periphery_client(&server)
         .await?
         .request(api::container::RenameContainer {
-          curr_name: deployment.name.clone(),
+          curr_name: deployment.deployed_name().to_string(),
           new_name: name.clone(),
         })
         .await
@@ -465,6 +506,7 @@ pub async fn check_deployment_for_update_inner(
     &deployment.id,
     &DeploymentInfo {
       latest_image_digest: latest_digest.clone(),
+      deployed_name: deployment.info.deployed_name.clone(),
     },
   )
   .await?;
@@ -621,6 +663,7 @@ impl Resolve<WriteArgs> for BatchCheckDeploymentForUpdate {
     fields(
       operator = user.id,
       pattern = self.pattern,
+      tags = self.tags.join(","),
       skip_auto_update = self.skip_auto_update,
       wait_for_auto_update = self.wait_for_auto_update,
     )
@@ -629,12 +672,23 @@ impl Resolve<WriteArgs> for BatchCheckDeploymentForUpdate {
     self,
     WriteArgs { user }: &WriteArgs,
   ) -> Result<Self::Response, Self::Error> {
+    let all_tags = if self.tags.is_empty() {
+      vec![]
+    } else {
+      get_all_tags(None).await?
+    };
+
     let deployments = list_full_for_user_using_pattern::<Deployment>(
       &self.pattern,
-      Default::default(),
+      ResourceQuery {
+        tags: self.tags,
+        ..Default::default()
+      },
+      None,
+      None,
       user,
       PermissionLevel::Execute.into(),
-      &[],
+      &all_tags,
     )
     .await?;
 

bin/core/src/api/write/mod.rs

@@ -211,9 +211,12 @@ pub enum WriteRequest {
   CreateGitProviderAccount(CreateGitProviderAccount),
   UpdateGitProviderAccount(UpdateGitProviderAccount),
   DeleteGitProviderAccount(DeleteGitProviderAccount),
-  CreateDockerRegistryAccount(CreateDockerRegistryAccount),
-  UpdateDockerRegistryAccount(UpdateDockerRegistryAccount),
-  DeleteDockerRegistryAccount(DeleteDockerRegistryAccount),
+  #[serde(alias = "CreateDockerRegistryAccount")]
+  CreateImageRegistryAccount(CreateImageRegistryAccount),
+  #[serde(alias = "UpdateDockerRegistryAccount")]
+  UpdateImageRegistryAccount(UpdateImageRegistryAccount),
+  #[serde(alias = "DeleteDockerRegistryAccount")]
+  DeleteImageRegistryAccount(DeleteImageRegistryAccount),
 
   // ==== ALERT ====
   CloseAlert(CloseAlert),

bin/core/src/api/write/permissions.rs

@@ -347,9 +347,9 @@ async fn extract_resource_target_with_validation(
         .find_one(filter)
         .await
         .context("Failed to query db for swarms")?
-        .context("No matching server found")?
+        .context("No matching swarm found")?
         .id;
-      Ok((ResourceTargetVariant::Server, id))
+      Ok((ResourceTargetVariant::Swarm, id))
     }
     ResourceTarget::Server(ident) => {
       let filter = match ObjectId::from_str(ident) {

bin/core/src/api/write/provider.rs

@@ -7,7 +7,7 @@ use komodo_client::{
   api::write::*,
   entities::{
     Operation, ResourceTarget,
-    provider::{DockerRegistryAccount, GitProviderAccount},
+    provider::{GitProviderAccount, ImageRegistryAccount},
   },
 };
 use mogh_error::AddStatusCodeError;
@@ -248,7 +248,7 @@ impl Resolve<WriteArgs> for DeleteGitProviderAccount {
   }
 }
 
-impl Resolve<WriteArgs> for CreateDockerRegistryAccount {
+impl Resolve<WriteArgs> for CreateImageRegistryAccount {
   #[instrument(
     "CreateDockerRegistryAccount",
     skip_all,
@@ -261,7 +261,7 @@ impl Resolve<WriteArgs> for CreateDockerRegistryAccount {
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
-  ) -> mogh_error::Result<CreateDockerRegistryAccountResponse> {
+  ) -> mogh_error::Result<CreateImageRegistryAccountResponse> {
     if !user.admin {
       return Err(
         anyhow!(
@@ -271,7 +271,7 @@ impl Resolve<WriteArgs> for CreateDockerRegistryAccount {
       );
     }
 
-    let mut account: DockerRegistryAccount = self.account.into();
+    let mut account: ImageRegistryAccount = self.account.into();
 
     if account.domain.is_empty() {
       return Err(
@@ -326,7 +326,7 @@ impl Resolve<WriteArgs> for CreateDockerRegistryAccount {
   }
 }
 
-impl Resolve<WriteArgs> for UpdateDockerRegistryAccount {
+impl Resolve<WriteArgs> for UpdateImageRegistryAccount {
   #[instrument(
     "UpdateDockerRegistryAccount",
     skip_all,
@@ -340,7 +340,7 @@ impl Resolve<WriteArgs> for UpdateDockerRegistryAccount {
   async fn resolve(
     mut self,
     WriteArgs { user }: &WriteArgs,
-  ) -> mogh_error::Result<UpdateDockerRegistryAccountResponse> {
+  ) -> mogh_error::Result<UpdateImageRegistryAccountResponse> {
     if !user.admin {
       return Err(
         anyhow!("Only admins can update docker registry accounts")
@@ -423,7 +423,7 @@ impl Resolve<WriteArgs> for UpdateDockerRegistryAccount {
   }
 }
 
-impl Resolve<WriteArgs> for DeleteDockerRegistryAccount {
+impl Resolve<WriteArgs> for DeleteImageRegistryAccount {
   #[instrument(
     "DeleteDockerRegistryAccount",
     skip_all,
@@ -435,7 +435,7 @@ impl Resolve<WriteArgs> for DeleteDockerRegistryAccount {
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
-  ) -> mogh_error::Result<DeleteDockerRegistryAccountResponse> {
+  ) -> mogh_error::Result<DeleteImageRegistryAccountResponse> {
     if !user.admin {
       return Err(
         anyhow!("Only admins can delete docker registry accounts")

bin/core/src/api/write/stack.rs

@@ -19,7 +19,11 @@ use komodo_client::{
     all_logs_success, komodo_timestamp,
     permission::PermissionLevel,
     repo::Repo,
-    stack::{Stack, StackInfo, StackServiceWithUpdate, StackState},
+    resource::ResourceQuery,
+    stack::{
+      Stack, StackInfo, StackServiceNames, StackServiceWithUpdate,
+      StackState,
+    },
     update::Update,
     user::{auto_redeploy_user, stack_user, system_user},
   },
@@ -36,7 +40,7 @@ use crate::{
   api::execute::{self, ExecuteRequest, ExecutionResult},
   config::core_config,
   helpers::{
-    query::get_swarm_or_server,
+    query::{get_all_tags, get_swarm_or_server},
     stack_git_token, swarm_or_server_request,
     update::{add_update, make_update, poll_update_until_complete},
   },
@@ -555,8 +559,9 @@ impl Resolve<WriteArgs> for RefreshStackCache {
             &mut services,
           ) {
             warn!(
-              "failed to extract stack services, things won't works correctly. stack: {} | {e:#}",
-              stack.name
+              stack = stack.id,
+              stack_name = stack.name,
+              "Failed to extract stack services | {e:#}",
             );
           }
         }
@@ -775,7 +780,13 @@ pub async fn check_stack_for_update_inner(
 
     if image.is_empty() ||
       // Images with a hardcoded digest can't have update.
-      image.contains('@')
+      image.contains('@') ||
+      // Services explicitly excluded from global auto-update checks.
+      // Manual checks should still evaluate all services.
+      (wait_for_auto_update &&
+        stack.config.auto_update_skip_services.contains(
+          &service.service_name,
+        ))
     {
       service.image_digest = None;
       continue;
@@ -814,6 +825,11 @@ pub async fn check_stack_for_update_inner(
       services: extract_services_from_stack(&stack)
         .into_iter()
         .map(|service| StackServiceWithUpdate {
+          latest_image: find_latest_image(
+            &service.service_name,
+            &service.image,
+            &stack.info.latest_services,
+          ),
           service: service.service_name,
           image: service.image,
           update_available: false,
@@ -834,6 +850,11 @@ pub async fn check_stack_for_update_inner(
         .services
         .iter()
         .map(|service| StackServiceWithUpdate {
+          latest_image: find_latest_image(
+            &service.service,
+            &service.image,
+            &stack.info.latest_services,
+          ),
           service: service.service.clone(),
           image: service.image.clone(),
           update_available: false,
@@ -849,6 +870,11 @@ pub async fn check_stack_for_update_inner(
       service: service.service.clone(),
       image: service.image.clone(),
       update_available: false,
+      latest_image: find_latest_image(
+        &service.service,
+        &service.image,
+        &stack.info.latest_services,
+      ),
     };
 
     let Some(current_digests) = &service.image_digests else {
@@ -939,7 +965,10 @@ pub async fn check_stack_for_update_inner(
     .retain(|(stack_id, _)| stack_id != &stack.id)
     .await;
 
-  let deploy_services = if stack.config.auto_update_all_services {
+  let deploy_services = if stack.config.auto_update_all_services
+    // Swarm stacks don't support individual service deploy
+    || !stack.config.swarm_id.is_empty()
+  {
     Vec::new()
   } else {
     services_with_update
@@ -1021,6 +1050,22 @@ pub async fn check_stack_for_update_inner(
   })
 }
 
+fn find_latest_image(
+  service_name: &str,
+  current_image: &str,
+  latest_services: &[StackServiceNames],
+) -> Option<String> {
+  latest_services.iter().find_map(|latest| {
+    if latest.service_name == service_name
+      && latest.image != current_image
+    {
+      Some(latest.image.clone())
+    } else {
+      None
+    }
+  })
+}
+
 //
 
 impl Resolve<WriteArgs> for BatchCheckStackForUpdate {
@@ -1030,6 +1075,7 @@ impl Resolve<WriteArgs> for BatchCheckStackForUpdate {
     fields(
       operator = user.id,
       pattern = self.pattern,
+      tags = self.tags.join(","),
       skip_auto_update = self.skip_auto_update,
       wait_for_auto_update = self.wait_for_auto_update,
     )
@@ -1038,12 +1084,23 @@ impl Resolve<WriteArgs> for BatchCheckStackForUpdate {
     self,
     WriteArgs { user }: &WriteArgs,
   ) -> Result<Self::Response, Self::Error> {
+    let all_tags = if self.tags.is_empty() {
+      vec![]
+    } else {
+      get_all_tags(None).await?
+    };
+
     let stacks = list_full_for_user_using_pattern::<Stack>(
       &self.pattern,
-      Default::default(),
+      ResourceQuery {
+        tags: self.tags,
+        ..Default::default()
+      },
+      None,
+      None,
       user,
       PermissionLevel::Execute.into(),
-      &[],
+      &all_tags,
     )
     .await?;
 

bin/core/src/api/write/terminal.rs

@@ -171,10 +171,10 @@ impl Resolve<WriteArgs> for DeleteTerminal {
     )
   )]
   async fn resolve(
-    self,
+    mut self,
     WriteArgs { user }: &WriteArgs,
   ) -> mogh_error::Result<NoData> {
-    let server = match &self.target {
+    let server = match &mut self.target {
       TerminalTarget::Server { server } => {
         let server = server
           .as_ref()
@@ -195,27 +195,31 @@ impl Resolve<WriteArgs> for DeleteTerminal {
         )
         .await?
       }
-      TerminalTarget::Stack { stack, .. } => {
-        let server = get_check_permissions::<Stack>(
-          stack,
+      TerminalTarget::Stack {
+        stack: stack_id, ..
+      } => {
+        let stack = get_check_permissions::<Stack>(
+          stack_id,
           user,
           PermissionLevel::Read.terminal(),
         )
-        .await?
-        .config
-        .server_id;
-        resource::get::<Server>(&server).await?
+        .await?;
+        // Must fix any incoming stack name to id
+        *stack_id = stack.id;
+        resource::get::<Server>(&stack.config.server_id).await?
       }
-      TerminalTarget::Deployment { deployment } => {
-        let server = get_check_permissions::<Deployment>(
-          deployment,
+      TerminalTarget::Deployment {
+        deployment: deployment_id,
+      } => {
+        let deployment = get_check_permissions::<Deployment>(
+          deployment_id,
           user,
           PermissionLevel::Read.terminal(),
         )
-        .await?
-        .config
-        .server_id;
-        resource::get::<Server>(&server).await?
+        .await?;
+        // Must fix any incoming deployment name to id
+        *deployment_id = deployment.id;
+        resource::get::<Server>(&deployment.config.server_id).await?
       }
     };
 
@@ -289,6 +293,8 @@ impl Resolve<WriteArgs> for BatchDeleteAllTerminals {
 
     resource::list_full_for_user::<Server>(
       self.query,
+      None,
+      None,
       user,
       PermissionLevel::Read.terminal(),
       &all_tags,

bin/core/src/api/ws/mod.rs

@@ -24,6 +24,9 @@ use crate::{
 mod terminal;
 mod update;
 
+pub(super) const WS_KEEP_ALIVE_INTERVAL: std::time::Duration =
+  std::time::Duration::from_secs(30);
+
 pub fn router() -> Router {
   Router::new()
     // Periphery facing

bin/core/src/api/ws/terminal.rs

@@ -117,9 +117,25 @@ async fn forward_ws_channel(
   };
 
   let periphery_to_core = async {
+    let mut keep_alive =
+      tokio::time::interval(super::WS_KEEP_ALIVE_INTERVAL);
     loop {
       // Already adheres to cancellation token
-      match periphery_receiver.recv().await {
+      let recv = tokio::select! {
+        // Websocket keep-alive ping
+        _ = keep_alive.tick() => {
+          if client_send
+            .send(ws::Message::Ping(Bytes::new()))
+            .await
+            .is_err()
+          {
+            break;
+          }
+          continue;
+        }
+        recv = periphery_receiver.recv() => recv,
+      };
+      match recv {
         Ok(Ok(bytes)) => {
           if let Err(e) =
             client_send.send(ws::Message::Binary(bytes.into())).await

bin/core/src/api/ws/update.rs

@@ -3,6 +3,7 @@ use axum::{
   extract::{WebSocketUpgrade, ws::Message},
   response::IntoResponse,
 };
+use bytes::Bytes;
 use futures_util::{SinkExt, StreamExt};
 use komodo_client::entities::{
   ResourceTarget, permission::PermissionLevel, user::User,
@@ -38,10 +39,19 @@ pub async fn handler(
     let cancel_clone = cancel.clone();
 
     tokio::spawn(async move {
+      let mut keep_alive =
+        tokio::time::interval(super::WS_KEEP_ALIVE_INTERVAL);
       loop {
-        // poll for updates off the receiver / await cancel.
+        // poll for updates off the receiver / await cancel / send keep-alive ping.
         let update = select! {
           _ = cancel_clone.cancelled() => break,
+          _ = keep_alive.tick() => {
+            if ws_sender.send(Message::Ping(Bytes::new())).await.is_err() {
+              cancel_clone.cancel();
+              break;
+            }
+            continue;
+          }
           update = receiver.recv() => {update.expect("failed to recv update msg")}
         };
 

bin/core/src/auth/mod.rs

@@ -205,6 +205,20 @@ impl AuthImpl for KomodoAuthImpl {
     core_config().disable_user_registration
   }
 
+  fn local_registration_disabled(&self) -> bool {
+    let config = core_config();
+    config
+      .disable_local_user_registration
+      .unwrap_or(config.disable_user_registration)
+  }
+
+  fn oidc_registration_disabled(&self) -> bool {
+    let config = core_config();
+    config
+      .disable_oidc_user_registration
+      .unwrap_or(config.disable_user_registration)
+  }
+
   fn validate_username(
     &self,
     username: &str,
@@ -348,6 +362,7 @@ impl AuthImpl for KomodoAuthImpl {
         additional_audiences: config
           .oidc_additional_audiences
           .clone(),
+        auto_redirect: config.oidc_auto_redirect,
       }
     });
     Some(&OIDC_CONFIG)

bin/core/src/cloud/aws/ec2.rs

@@ -88,7 +88,7 @@ pub async fn launch_ec2_instance(
     port: _,
     use_https: _,
     git_providers: _,
-    docker_registries: _,
+    image_registries: _,
     secrets: _,
   } = config;
   let instance_type = handle_unknown_instance_type(

bin/core/src/cloud/mod.rs

@@ -2,8 +2,9 @@ pub mod aws;
 
 #[derive(Debug)]
 pub enum BuildCleanupData {
-  /// Nothing to clean up
-  Server,
+  /// Store the builder id if it needs
+  /// to be removed from builder_usage_cache.
+  Server(Option<String>),
   /// Cleanup Periphery connection
   Url,
   /// Clean up AWS instance

bin/core/src/config.rs

@@ -235,6 +235,9 @@ pub fn core_config() -> &'static CoreConfig {
         env.komodo_oidc_additional_audiences,
       )
       .unwrap_or(config.oidc_additional_audiences),
+      oidc_auto_redirect: env
+        .komodo_oidc_auto_redirect
+        .unwrap_or(config.oidc_auto_redirect),
       google_oauth: NamedOauthConfig {
         enabled: env
           .komodo_google_oauth_enabled
@@ -312,6 +315,21 @@ pub fn core_config() -> &'static CoreConfig {
       session_allow_cross_site: env
         .komodo_session_allow_cross_site
         .unwrap_or(config.session_allow_cross_site),
+      x_content_type_options: env
+        .komodo_x_content_type_options
+        .unwrap_or(config.x_content_type_options),
+      x_frame_options: env
+        .komodo_x_frame_options
+        .unwrap_or(config.x_frame_options),
+      x_xss_protection: env
+        .komodo_x_xss_protection
+        .unwrap_or(config.x_xss_protection),
+      referrer_policy: env
+        .komodo_referrer_policy
+        .unwrap_or(config.referrer_policy),
+      content_security_policy: env
+        .komodo_content_security_policy
+        .unwrap_or(config.content_security_policy),
       resource_poll_interval: env
         .komodo_resource_poll_interval
         .unwrap_or(config.resource_poll_interval),
@@ -330,6 +348,9 @@ pub fn core_config() -> &'static CoreConfig {
       transparent_mode: env
         .komodo_transparent_mode
         .unwrap_or(config.transparent_mode),
+      default_pagination_limit: env
+        .komodo_default_pagination_limit
+        .unwrap_or(config.default_pagination_limit),
       ui_write_disabled: env
         .komodo_ui_write_disabled
         .unwrap_or(config.ui_write_disabled),
@@ -345,6 +366,12 @@ pub fn core_config() -> &'static CoreConfig {
       disable_user_registration: env
         .komodo_disable_user_registration
         .unwrap_or(config.disable_user_registration),
+      disable_local_user_registration: env
+        .komodo_disable_local_user_registration
+        .or(config.disable_local_user_registration),
+      disable_oidc_user_registration: env
+        .komodo_disable_oidc_user_registration
+        .or(config.disable_oidc_user_registration),
       disable_non_admin_create: env
         .komodo_disable_non_admin_create
         .unwrap_or(config.disable_non_admin_create),
@@ -375,6 +402,9 @@ pub fn core_config() -> &'static CoreConfig {
           .komodo_logging_location
           .unwrap_or(config.logging.location),
         ansi: env.komodo_logging_ansi.unwrap_or(config.logging.ansi),
+        timestamps: env
+          .komodo_logging_timestamps
+          .unwrap_or(config.logging.timestamps),
         otlp_endpoint: env
           .komodo_logging_otlp_endpoint
           .unwrap_or(config.logging.otlp_endpoint),
@@ -403,6 +433,12 @@ pub fn core_config() -> &'static CoreConfig {
       ssl_cert_file: env
         .komodo_ssl_cert_file
         .unwrap_or(config.ssl_cert_file),
+      reporting_enabled: env
+        .komodo_reporting_enabled
+        .unwrap_or(config.reporting_enabled),
+      reporting_private_key: env
+        .komodo_reporting_private_key
+        .unwrap_or(config.reporting_private_key),
       ui_path: env.komodo_ui_path.unwrap_or(config.ui_path),
       ui_index_force_no_cache: env
         .komodo_ui_index_force_no_cache
@@ -420,7 +456,7 @@ pub fn core_config() -> &'static CoreConfig {
       // These can't be overridden on env
       secrets: config.secrets,
       git_providers: config.git_providers,
-      docker_registries: config.docker_registries,
+      image_registries: config.image_registries,
     }
   })
 }

bin/core/src/connection/server.rs

@@ -307,6 +307,13 @@ async fn fix_server(
     } else {
       None
     },
+    // Move address to external_address if not set.
+    // This helps preserve container port link behavior.
+    external_address: if !server.config.external_address.is_empty() {
+      Some(server.config.address)
+    } else {
+      None
+    },
     ..Default::default()
   };
   if !config.is_none() {
@@ -496,7 +503,7 @@ async fn create_server_maybe_builder(
       name: server_query,
       config: PartialBuilderConfig::Server(
         PartialServerBuilderConfig {
-          server_id: Some(server.id.clone()),
+          server_ids: Some(vec![server.id.clone()]),
         },
       ),
     })

bin/core/src/helpers/builder.rs

@@ -1,16 +1,18 @@
-use std::time::Duration;
+use std::{collections::HashMap, time::Duration};
 
 use anyhow::{Context, anyhow};
 use database::mungos::mongodb::bson::oid::ObjectId;
 use formatting::muted;
+use futures_util::{StreamExt, stream::FuturesOrdered};
 use komodo_client::entities::{
   Version,
   builder::{AwsBuilderConfig, Builder, BuilderConfig},
   komodo_timestamp,
-  server::Server,
+  server::{Server, ServerState},
   update::{Log, Update},
 };
 use periphery_client::api::{self, GetVersionResponse};
+use tokio::sync::Mutex;
 
 use crate::{
   cloud::{
@@ -24,6 +26,7 @@ use crate::{
   helpers::update::update_update,
   periphery::PeripheryClient,
   resource,
+  state::{builder_usage_cache, server_status_cache},
 };
 
 use super::periphery_client;
@@ -37,25 +40,26 @@ const BUILDER_POLL_MAX_TRIES: usize = 60;
   fields(
     resource_name,
     builder_id = builder.id,
-    update_id = update.id
+    update_id = update.as_ref().map(|u| u.id.as_str())
   )
 )]
 pub async fn connect_builder_periphery(
-  // build: &Build,
   resource_name: String,
   version: Option<Version>,
   builder: Builder,
-  update: &mut Update,
+  update: Option<&mut Update>,
 ) -> anyhow::Result<(PeripheryClient, BuildCleanupData)> {
   match builder.config {
+    BuilderConfig::Aws(config) => {
+      get_aws_builder(&resource_name, version, config, update).await
+    }
     BuilderConfig::Url(config) => {
       if config.address.is_empty() {
         return Err(anyhow!(
           "Builder has not yet configured an address"
         ));
       }
-      // TODO: Dont use builder id, or will be problems
-      // with simultaneous spawned builders.
+      // Builder id no good because it may be active for multiple connections.
       let periphery = PeripheryClient::new(
         PeripheryConnectionArgs::from_url_builder(
           &ObjectId::new().to_hex(),
@@ -64,22 +68,66 @@ pub async fn connect_builder_periphery(
         config.insecure_tls,
       )
       .await?;
-      periphery
-        .health_check()
-        .await
-        .context("Url Builder failed health check")?;
-      Ok((periphery, BuildCleanupData::Url))
+      // Poll for connection to be estalished
+      let mut err = None;
+      for _ in 0..10 {
+        tokio::time::sleep(Duration::from_secs(1)).await;
+        match periphery
+          .health_check()
+          .await
+          .context("Url Builder failed health check")
+        {
+          Ok(_) => return Ok((periphery, BuildCleanupData::Url)),
+          Err(e) => err = Some(e),
+        };
+      }
+      Err(err.context("Missing error")?)
     }
     BuilderConfig::Server(config) => {
-      if config.server_id.is_empty() {
-        return Err(anyhow!("Builder has not configured a server"));
+      if config.server_ids.is_empty() {
+        return Err(anyhow!(
+          "Server Builder has no configured Servers"
+        ));
       }
-      let server = resource::get::<Server>(&config.server_id).await?;
+
+      // Short path for single configured builder
+      if config.server_ids.len() == 1 {
+        let server =
+          resource::get::<Server>(&config.server_ids[0]).await?;
+        let periphery = periphery_client(&server).await?;
+        return Ok((periphery, BuildCleanupData::Server(None)));
+      }
+
+      // Get filtered list of available Servers
+      let server_status_cache = server_status_cache();
+      let available_server_ids = config
+        .server_ids
+        .iter()
+        .map(|server_id| async move {
+          server_status_cache
+            .get(server_id)
+            .await
+            .map(|s| matches!(s.state, ServerState::Ok))
+            .unwrap_or_default()
+            .then_some(server_id)
+        })
+        .collect::<FuturesOrdered<_>>()
+        .collect::<Vec<_>>()
+        .await
+        .into_iter()
+        .flatten()
+        .collect::<Vec<_>>();
+
+      let selected = builder_usage_cache()
+        .get_or_insert_default(&builder.id)
+        .await
+        .select(&available_server_ids)
+        .await
+        .context("Server builder has no available servers")?;
+
+      let server = resource::get::<Server>(selected).await?;
       let periphery = periphery_client(&server).await?;
-      Ok((periphery, BuildCleanupData::Server))
-    }
-    BuilderConfig::Aws(config) => {
-      get_aws_builder(&resource_name, version, config, update).await
+      Ok((periphery, BuildCleanupData::Server(Some(server.id))))
     }
   }
 }
@@ -89,14 +137,14 @@ pub async fn connect_builder_periphery(
   skip_all,
   fields(
     resource_name,
-    update_id = update.id,
+    update_id = update.as_ref().map(|u| u.id.as_str()),
   )
 )]
 async fn get_aws_builder(
   resource_name: &str,
   version: Option<Version>,
   config: AwsBuilderConfig,
-  update: &mut Update,
+  mut update: Option<&mut Update>,
 ) -> anyhow::Result<(PeripheryClient, BuildCleanupData)> {
   let start_create_ts = komodo_timestamp();
 
@@ -114,9 +162,10 @@ async fn get_aws_builder(
     ..Default::default()
   };
 
-  update.logs.push(log);
-
-  update_update(update.clone()).await?;
+  if let Some(update) = &mut update {
+    update.logs.push(log);
+    update_update((*update).clone()).await?;
+  }
 
   let protocol = if config.use_https { "wss" } else { "ws" };
 
@@ -153,8 +202,10 @@ async fn get_aws_builder(
         end_ts: komodo_timestamp(),
         ..Default::default()
       };
-      update.logs.push(connect_log);
-      update_update(update.clone()).await?;
+      if let Some(update) = update {
+        update.logs.push(connect_log);
+        update_update(update.clone()).await?;
+      }
       return Ok((
         periphery,
         BuildCleanupData::Aws {
@@ -194,9 +245,17 @@ pub async fn cleanup_builder_instance(
   update: &mut Update,
 ) {
   match cleanup_data {
-    BuildCleanupData::Server => {
+    BuildCleanupData::Server(None) => {
       // Nothing to clean up
     }
+    BuildCleanupData::Server(Some(builder_id)) => {
+      // Release periphery (server) id from builder
+      builder_usage_cache()
+        .get_or_insert_default(&builder_id)
+        .await
+        .release(&periphery.id)
+        .await
+    }
     BuildCleanupData::Url => {
       periphery.cleanup().await;
     }
@@ -252,3 +311,29 @@ pub fn start_aws_builder_log(
   ]
   .join("\n")
 }
+
+#[derive(Default)]
+pub struct BuilderUsage(Mutex<HashMap<String, usize>>);
+
+impl BuilderUsage {
+  pub async fn select<'a>(
+    &self,
+    available: &'a [&String],
+  ) -> Option<&'a str> {
+    let mut lock = self.0.lock().await;
+    let selected = *available.iter().min_by_key(|key| {
+      lock.get(key.as_str()).copied().unwrap_or(0)
+    })?;
+    *lock.entry(selected.clone()).or_insert(0) += 1;
+    Some(selected.as_str())
+  }
+
+  pub async fn release(&self, key: &str) {
+    let mut lock = self.0.lock().await;
+    if let Some(count) = lock.get_mut(key)
+      && *count > 0
+    {
+      *count -= 1;
+    }
+  }
+}

bin/core/src/helpers/mod.rs

@@ -169,7 +169,7 @@ pub async fn registry_token(
   }
   Ok(
     core_config()
-      .docker_registries
+      .image_registries
       .iter()
       .find(|provider| provider.domain == provider_domain)
       .and_then(|provider| {

bin/core/src/helpers/prune.rs

@@ -55,7 +55,7 @@ async fn prune_images() -> anyhow::Result<()> {
   while let Some((res, server)) = futures.next().await {
     if let Err(e) = res {
       warn!(
-        "failed to prune images on Server {} ({}) | {e:#}",
+        "Failed to prune images on Server {} ({}) | {e:#}",
         server.name, server.id
       )
     }

bin/core/src/helpers/query.rs

@@ -45,7 +45,7 @@ use crate::{
   state::{
     action_state_cache, action_states, db_client,
     deployment_status_cache, procedure_state_cache,
-    stack_status_cache,
+    server_status_cache, stack_status_cache,
   },
 };
 
@@ -97,6 +97,38 @@ pub async fn get_server_state(server: &Server) -> ServerState {
   }
 }
 
+/// The Server state as computed for the server list items,
+/// from the in memory status cache.
+pub async fn get_cached_server_state(id: &String) -> ServerState {
+  server_status_cache()
+    .get(id)
+    .await
+    .map(|status| status.state)
+    .unwrap_or_default()
+}
+
+/// The Stack state as computed for the stack list items,
+/// from the in memory action states / status cache.
+pub async fn get_cached_stack_state(id: &String) -> StackState {
+  if action_states()
+    .stack
+    .get(id)
+    .await
+    .map(|s| s.get().map(|s| s.deploying))
+    .transpose()
+    .ok()
+    .flatten()
+    .unwrap_or_default()
+  {
+    return StackState::Deploying;
+  }
+  stack_status_cache()
+    .get(id)
+    .await
+    .map(|status| status.curr.state)
+    .unwrap_or_default()
+}
+
 pub async fn get_deployment_state(
   id: &String,
 ) -> anyhow::Result<DeploymentState> {

bin/core/src/helpers/swarm.rs

@@ -1,5 +1,3 @@
-use std::time::Duration;
-
 use anyhow::anyhow;
 use komodo_client::entities::server::{Server, ServerState};
 use mogh_resolver::HasResponse;
@@ -13,23 +11,6 @@ pub async fn swarm_request<T>(
   server_ids: &[String],
   request: T,
 ) -> anyhow::Result<T::Response>
-where
-  T: std::fmt::Debug + Clone + Serialize + HasResponse,
-  T::Response: DeserializeOwned,
-{
-  swarm_request_custom_timeout(
-    server_ids,
-    request,
-    Duration::from_secs(10),
-  )
-  .await
-}
-
-pub async fn swarm_request_custom_timeout<T>(
-  server_ids: &[String],
-  request: T,
-  timeout: Duration,
-) -> anyhow::Result<T::Response>
 where
   T: std::fmt::Debug + Clone + Serialize + HasResponse,
   T::Response: DeserializeOwned,
@@ -54,10 +35,7 @@ where
     else {
       continue;
     };
-    match periphery
-      .request_custom_timeout(request.clone(), timeout)
-      .await
-    {
+    match periphery.request(request.clone()).await {
       Ok(res) => return Ok(res),
       Err(e) => err = Some(e),
     }

bin/core/src/helpers/terminal.rs

@@ -332,7 +332,7 @@ pub async fn get_deployment_periphery_container(
 
   let periphery = periphery_client(&server).await?;
 
-  let container = deployment.name.clone();
+  let container = deployment.deployed_name().to_string();
 
   Ok((
     TerminalTarget::Deployment {

bin/core/src/helpers/update.rs

@@ -123,464 +123,153 @@ pub async fn init_execution_update(
   request: &ExecuteRequest,
   user: &User,
 ) -> anyhow::Result<Update> {
-  let (operation, target) = match &request {
-    // Swarm
-    ExecuteRequest::RemoveSwarmNodes(data) => (
-      Operation::RemoveSwarmNodes,
-      ResourceTarget::Swarm(
-        resource::get::<Swarm>(&data.swarm).await?.id,
-      ),
-    ),
-    ExecuteRequest::RemoveSwarmStacks(data) => (
-      Operation::RemoveSwarmStacks,
-      ResourceTarget::Swarm(
-        resource::get::<Swarm>(&data.swarm).await?.id,
-      ),
-    ),
-    ExecuteRequest::RemoveSwarmServices(data) => (
-      Operation::RemoveSwarmServices,
-      ResourceTarget::Swarm(
-        resource::get::<Swarm>(&data.swarm).await?.id,
-      ),
-    ),
-    ExecuteRequest::CreateSwarmConfig(data) => (
-      Operation::CreateSwarmConfig,
-      ResourceTarget::Swarm(
-        resource::get::<Swarm>(&data.swarm).await?.id,
-      ),
-    ),
-    ExecuteRequest::RotateSwarmConfig(data) => (
-      Operation::RotateSwarmConfig,
-      ResourceTarget::Swarm(
-        resource::get::<Swarm>(&data.swarm).await?.id,
-      ),
-    ),
-    ExecuteRequest::RemoveSwarmConfigs(data) => (
-      Operation::RemoveSwarmConfigs,
-      ResourceTarget::Swarm(
-        resource::get::<Swarm>(&data.swarm).await?.id,
-      ),
-    ),
-    ExecuteRequest::CreateSwarmSecret(data) => (
-      Operation::CreateSwarmSecret,
-      ResourceTarget::Swarm(
-        resource::get::<Swarm>(&data.swarm).await?.id,
-      ),
-    ),
-    ExecuteRequest::RotateSwarmSecret(data) => (
-      Operation::RotateSwarmSecret,
-      ResourceTarget::Swarm(
-        resource::get::<Swarm>(&data.swarm).await?.id,
-      ),
-    ),
-    ExecuteRequest::RemoveSwarmSecrets(data) => (
-      Operation::RemoveSwarmSecrets,
-      ResourceTarget::Swarm(
-        resource::get::<Swarm>(&data.swarm).await?.id,
-      ),
-    ),
+  macro_rules! init_execution_match {
+    (
+      resource: [$(($Variant:ident, $ResType:ident, $field:ident)),* $(,)?],
+      batch: [$($BatchVariant:ident),* $(,)?],
+      stack_service: [$(($StackVariant:ident, $ServiceOp:ident)),* $(,)?],
+      system: [$($SysVariant:ident),* $(,)?],
+    ) => {
+      match &request {
+        $(
+          ExecuteRequest::$Variant(data) => (
+            Operation::$Variant,
+            ResourceTarget::$ResType(
+              resource::get::<$ResType>(&data.$field).await?.id,
+            ),
+          ),
+        )*
+        $(
+          ExecuteRequest::$BatchVariant(_data) => {
+            return Ok(Default::default());
+          }
+        )*
+        $(
+          ExecuteRequest::$StackVariant(data) => (
+            if !data.services.is_empty() {
+              Operation::$ServiceOp
+            } else {
+              Operation::$StackVariant
+            },
+            ResourceTarget::Stack(
+              resource::get::<Stack>(&data.stack).await?.id,
+            ),
+          ),
+        )*
+        // DeployStackIfChanged doesn't have a service variant
+        ExecuteRequest::DeployStackIfChanged(data) => (
+          Operation::DeployStack,
+          ResourceTarget::Stack(
+            resource::get::<Stack>(&data.stack).await?.id,
+          ),
+        ),
+        $(
+          ExecuteRequest::$SysVariant(_data) => {
+            (Operation::$SysVariant, ResourceTarget::system())
+          }
+        )*
+      }
+    };
+  }
 
-    // Server
-    ExecuteRequest::StartContainer(data) => (
-      Operation::StartContainer,
-      ResourceTarget::Server(
-        resource::get::<Server>(&data.server).await?.id,
-      ),
-    ),
-    ExecuteRequest::RestartContainer(data) => (
-      Operation::RestartContainer,
-      ResourceTarget::Server(
-        resource::get::<Server>(&data.server).await?.id,
-      ),
-    ),
-    ExecuteRequest::PauseContainer(data) => (
-      Operation::PauseContainer,
-      ResourceTarget::Server(
-        resource::get::<Server>(&data.server).await?.id,
-      ),
-    ),
-    ExecuteRequest::UnpauseContainer(data) => (
-      Operation::UnpauseContainer,
-      ResourceTarget::Server(
-        resource::get::<Server>(&data.server).await?.id,
-      ),
-    ),
-    ExecuteRequest::StopContainer(data) => (
-      Operation::StopContainer,
-      ResourceTarget::Server(
-        resource::get::<Server>(&data.server).await?.id,
-      ),
-    ),
-    ExecuteRequest::DestroyContainer(data) => (
-      Operation::DestroyContainer,
-      ResourceTarget::Server(
-        resource::get::<Server>(&data.server).await?.id,
-      ),
-    ),
-    ExecuteRequest::StartAllContainers(data) => (
-      Operation::StartAllContainers,
-      ResourceTarget::Server(
-        resource::get::<Server>(&data.server).await?.id,
-      ),
-    ),
-    ExecuteRequest::RestartAllContainers(data) => (
-      Operation::RestartAllContainers,
-      ResourceTarget::Server(
-        resource::get::<Server>(&data.server).await?.id,
-      ),
-    ),
-    ExecuteRequest::PauseAllContainers(data) => (
-      Operation::PauseAllContainers,
-      ResourceTarget::Server(
-        resource::get::<Server>(&data.server).await?.id,
-      ),
-    ),
-    ExecuteRequest::UnpauseAllContainers(data) => (
-      Operation::UnpauseAllContainers,
-      ResourceTarget::Server(
-        resource::get::<Server>(&data.server).await?.id,
-      ),
-    ),
-    ExecuteRequest::StopAllContainers(data) => (
-      Operation::StopAllContainers,
-      ResourceTarget::Server(
-        resource::get::<Server>(&data.server).await?.id,
-      ),
-    ),
-    ExecuteRequest::PruneContainers(data) => (
-      Operation::PruneContainers,
-      ResourceTarget::Server(
-        resource::get::<Server>(&data.server).await?.id,
-      ),
-    ),
-    ExecuteRequest::DeleteNetwork(data) => (
-      Operation::DeleteNetwork,
-      ResourceTarget::Server(
-        resource::get::<Server>(&data.server).await?.id,
-      ),
-    ),
-    ExecuteRequest::PruneNetworks(data) => (
-      Operation::PruneNetworks,
-      ResourceTarget::Server(
-        resource::get::<Server>(&data.server).await?.id,
-      ),
-    ),
-    ExecuteRequest::DeleteImage(data) => (
-      Operation::DeleteImage,
-      ResourceTarget::Server(
-        resource::get::<Server>(&data.server).await?.id,
-      ),
-    ),
-    ExecuteRequest::PruneImages(data) => (
-      Operation::PruneImages,
-      ResourceTarget::Server(
-        resource::get::<Server>(&data.server).await?.id,
-      ),
-    ),
-    ExecuteRequest::DeleteVolume(data) => (
-      Operation::DeleteVolume,
-      ResourceTarget::Server(
-        resource::get::<Server>(&data.server).await?.id,
-      ),
-    ),
-    ExecuteRequest::PruneVolumes(data) => (
-      Operation::PruneVolumes,
-      ResourceTarget::Server(
-        resource::get::<Server>(&data.server).await?.id,
-      ),
-    ),
-    ExecuteRequest::PruneDockerBuilders(data) => (
-      Operation::PruneDockerBuilders,
-      ResourceTarget::Server(
-        resource::get::<Server>(&data.server).await?.id,
-      ),
-    ),
-    ExecuteRequest::PruneBuildx(data) => (
-      Operation::PruneBuildx,
-      ResourceTarget::Server(
-        resource::get::<Server>(&data.server).await?.id,
-      ),
-    ),
-    ExecuteRequest::PruneSystem(data) => (
-      Operation::PruneSystem,
-      ResourceTarget::Server(
-        resource::get::<Server>(&data.server).await?.id,
-      ),
-    ),
-
-    // Deployment
-    ExecuteRequest::Deploy(data) => (
-      Operation::Deploy,
-      ResourceTarget::Deployment(
-        resource::get::<Deployment>(&data.deployment).await?.id,
-      ),
-    ),
-    ExecuteRequest::BatchDeploy(_data) => {
-      return Ok(Default::default());
-    }
-    ExecuteRequest::PullDeployment(data) => (
-      Operation::PullDeployment,
-      ResourceTarget::Deployment(
-        resource::get::<Deployment>(&data.deployment).await?.id,
-      ),
-    ),
-    ExecuteRequest::StartDeployment(data) => (
-      Operation::StartDeployment,
-      ResourceTarget::Deployment(
-        resource::get::<Deployment>(&data.deployment).await?.id,
-      ),
-    ),
-    ExecuteRequest::RestartDeployment(data) => (
-      Operation::RestartDeployment,
-      ResourceTarget::Deployment(
-        resource::get::<Deployment>(&data.deployment).await?.id,
-      ),
-    ),
-    ExecuteRequest::PauseDeployment(data) => (
-      Operation::PauseDeployment,
-      ResourceTarget::Deployment(
-        resource::get::<Deployment>(&data.deployment).await?.id,
-      ),
-    ),
-    ExecuteRequest::UnpauseDeployment(data) => (
-      Operation::UnpauseDeployment,
-      ResourceTarget::Deployment(
-        resource::get::<Deployment>(&data.deployment).await?.id,
-      ),
-    ),
-    ExecuteRequest::StopDeployment(data) => (
-      Operation::StopDeployment,
-      ResourceTarget::Deployment(
-        resource::get::<Deployment>(&data.deployment).await?.id,
-      ),
-    ),
-    ExecuteRequest::DestroyDeployment(data) => (
-      Operation::DestroyDeployment,
-      ResourceTarget::Deployment(
-        resource::get::<Deployment>(&data.deployment).await?.id,
-      ),
-    ),
-    ExecuteRequest::BatchDestroyDeployment(_data) => {
-      return Ok(Default::default());
-    }
-
-    // Build
-    ExecuteRequest::RunBuild(data) => (
-      Operation::RunBuild,
-      ResourceTarget::Build(
-        resource::get::<Build>(&data.build).await?.id,
-      ),
-    ),
-    ExecuteRequest::BatchRunBuild(_data) => {
-      return Ok(Default::default());
-    }
-    ExecuteRequest::CancelBuild(data) => (
-      Operation::CancelBuild,
-      ResourceTarget::Build(
-        resource::get::<Build>(&data.build).await?.id,
-      ),
-    ),
-
-    // Repo
-    ExecuteRequest::CloneRepo(data) => (
-      Operation::CloneRepo,
-      ResourceTarget::Repo(
-        resource::get::<Repo>(&data.repo).await?.id,
-      ),
-    ),
-    ExecuteRequest::BatchCloneRepo(_data) => {
-      return Ok(Default::default());
-    }
-    ExecuteRequest::PullRepo(data) => (
-      Operation::PullRepo,
-      ResourceTarget::Repo(
-        resource::get::<Repo>(&data.repo).await?.id,
-      ),
-    ),
-    ExecuteRequest::BatchPullRepo(_data) => {
-      return Ok(Default::default());
-    }
-    ExecuteRequest::BuildRepo(data) => (
-      Operation::BuildRepo,
-      ResourceTarget::Repo(
-        resource::get::<Repo>(&data.repo).await?.id,
-      ),
-    ),
-    ExecuteRequest::BatchBuildRepo(_data) => {
-      return Ok(Default::default());
-    }
-    ExecuteRequest::CancelRepoBuild(data) => (
-      Operation::CancelRepoBuild,
-      ResourceTarget::Repo(
-        resource::get::<Repo>(&data.repo).await?.id,
-      ),
-    ),
-
-    // Procedure
-    ExecuteRequest::RunProcedure(data) => (
-      Operation::RunProcedure,
-      ResourceTarget::Procedure(
-        resource::get::<Procedure>(&data.procedure).await?.id,
-      ),
-    ),
-    ExecuteRequest::BatchRunProcedure(_) => {
-      return Ok(Default::default());
-    }
-
-    // Action
-    ExecuteRequest::RunAction(data) => (
-      Operation::RunAction,
-      ResourceTarget::Action(
-        resource::get::<Action>(&data.action).await?.id,
-      ),
-    ),
-    ExecuteRequest::BatchRunAction(_) => {
-      return Ok(Default::default());
-    }
-
-    // Resource Sync
-    ExecuteRequest::RunSync(data) => (
-      Operation::RunSync,
-      ResourceTarget::ResourceSync(
-        resource::get::<ResourceSync>(&data.sync).await?.id,
-      ),
-    ),
-
-    // Stack
-    ExecuteRequest::DeployStack(data) => (
-      if !data.services.is_empty() {
-        Operation::DeployStackService
-      } else {
-        Operation::DeployStack
-      },
-      ResourceTarget::Stack(
-        resource::get::<Stack>(&data.stack).await?.id,
-      ),
-    ),
-    ExecuteRequest::BatchDeployStack(_data) => {
-      return Ok(Default::default());
-    }
-    ExecuteRequest::DeployStackIfChanged(data) => (
-      Operation::DeployStack,
-      ResourceTarget::Stack(
-        resource::get::<Stack>(&data.stack).await?.id,
-      ),
-    ),
-    ExecuteRequest::BatchDeployStackIfChanged(_data) => {
-      return Ok(Default::default());
-    }
-    ExecuteRequest::StartStack(data) => (
-      if !data.services.is_empty() {
-        Operation::StartStackService
-      } else {
-        Operation::StartStack
-      },
-      ResourceTarget::Stack(
-        resource::get::<Stack>(&data.stack).await?.id,
-      ),
-    ),
-    ExecuteRequest::PullStack(data) => (
-      if !data.services.is_empty() {
-        Operation::PullStackService
-      } else {
-        Operation::PullStack
-      },
-      ResourceTarget::Stack(
-        resource::get::<Stack>(&data.stack).await?.id,
-      ),
-    ),
-    ExecuteRequest::BatchPullStack(_data) => {
-      return Ok(Default::default());
-    }
-    ExecuteRequest::RestartStack(data) => (
-      if !data.services.is_empty() {
-        Operation::RestartStackService
-      } else {
-        Operation::RestartStack
-      },
-      ResourceTarget::Stack(
-        resource::get::<Stack>(&data.stack).await?.id,
-      ),
-    ),
-    ExecuteRequest::PauseStack(data) => (
-      if !data.services.is_empty() {
-        Operation::PauseStackService
-      } else {
-        Operation::PauseStack
-      },
-      ResourceTarget::Stack(
-        resource::get::<Stack>(&data.stack).await?.id,
-      ),
-    ),
-    ExecuteRequest::UnpauseStack(data) => (
-      if !data.services.is_empty() {
-        Operation::UnpauseStackService
-      } else {
-        Operation::UnpauseStack
-      },
-      ResourceTarget::Stack(
-        resource::get::<Stack>(&data.stack).await?.id,
-      ),
-    ),
-    ExecuteRequest::StopStack(data) => (
-      if !data.services.is_empty() {
-        Operation::StopStackService
-      } else {
-        Operation::StopStack
-      },
-      ResourceTarget::Stack(
-        resource::get::<Stack>(&data.stack).await?.id,
-      ),
-    ),
-    ExecuteRequest::DestroyStack(data) => (
-      if !data.services.is_empty() {
-        Operation::DestroyStackService
-      } else {
-        Operation::DestroyStack
-      },
-      ResourceTarget::Stack(
-        resource::get::<Stack>(&data.stack).await?.id,
-      ),
-    ),
-    ExecuteRequest::BatchDestroyStack(_data) => {
-      return Ok(Default::default());
-    }
-
-    ExecuteRequest::RunStackService(data) => (
-      Operation::RunStackService,
-      ResourceTarget::Stack(
-        resource::get::<Stack>(&data.stack).await?.id,
-      ),
-    ),
-
-    // Alerter
-    ExecuteRequest::TestAlerter(data) => (
-      Operation::TestAlerter,
-      ResourceTarget::Alerter(
-        resource::get::<Alerter>(&data.alerter).await?.id,
-      ),
-    ),
-    ExecuteRequest::SendAlert(_) => {
-      (Operation::SendAlert, ResourceTarget::system())
-    }
-
-    // Maintenance
-    ExecuteRequest::ClearRepoCache(_data) => {
-      (Operation::ClearRepoCache, ResourceTarget::system())
-    }
-    ExecuteRequest::BackupCoreDatabase(_data) => {
-      (Operation::BackupCoreDatabase, ResourceTarget::system())
-    }
-    ExecuteRequest::GlobalAutoUpdate(_data) => {
-      (Operation::GlobalAutoUpdate, ResourceTarget::system())
-    }
-    ExecuteRequest::RotateAllServerKeys(_data) => {
-      (Operation::RotateAllServerKeys, ResourceTarget::system())
-    }
-    ExecuteRequest::RotateCoreKeys(_data) => {
-      (Operation::RotateCoreKeys, ResourceTarget::system())
-    }
-  };
+  let (operation, target) = init_execution_match!(
+    resource: [
+      // Swarm
+      (RemoveSwarmNodes, Swarm, swarm),
+      (UpdateSwarmNode, Swarm, swarm),
+      (RemoveSwarmStacks, Swarm, swarm),
+      (RemoveSwarmServices, Swarm, swarm),
+      (CreateSwarmConfig, Swarm, swarm),
+      (RotateSwarmConfig, Swarm, swarm),
+      (RemoveSwarmConfigs, Swarm, swarm),
+      (CreateSwarmSecret, Swarm, swarm),
+      (RotateSwarmSecret, Swarm, swarm),
+      (RemoveSwarmSecrets, Swarm, swarm),
+      // Server
+      (StartContainer, Server, server),
+      (RestartContainer, Server, server),
+      (PauseContainer, Server, server),
+      (UnpauseContainer, Server, server),
+      (StopContainer, Server, server),
+      (DestroyContainer, Server, server),
+      (StartAllContainers, Server, server),
+      (RestartAllContainers, Server, server),
+      (PauseAllContainers, Server, server),
+      (UnpauseAllContainers, Server, server),
+      (StopAllContainers, Server, server),
+      (PruneContainers, Server, server),
+      (DeleteNetwork, Server, server),
+      (PruneNetworks, Server, server),
+      (DeleteImage, Server, server),
+      (PruneImages, Server, server),
+      (DeleteVolume, Server, server),
+      (PruneVolumes, Server, server),
+      (PruneDockerBuilders, Server, server),
+      (PruneBuildx, Server, server),
+      (PruneSystem, Server, server),
+      // Deployment
+      (Deploy, Deployment, deployment),
+      (PullDeployment, Deployment, deployment),
+      (StartDeployment, Deployment, deployment),
+      (RestartDeployment, Deployment, deployment),
+      (PauseDeployment, Deployment, deployment),
+      (UnpauseDeployment, Deployment, deployment),
+      (StopDeployment, Deployment, deployment),
+      (DestroyDeployment, Deployment, deployment),
+      // Build
+      (RunBuild, Build, build),
+      (CancelBuild, Build, build),
+      // Repo
+      (CloneRepo, Repo, repo),
+      (PullRepo, Repo, repo),
+      (BuildRepo, Repo, repo),
+      (CancelRepoBuild, Repo, repo),
+      // Procedure
+      (RunProcedure, Procedure, procedure),
+      (CancelProcedure, Procedure, procedure),
+      // Action
+      (RunAction, Action, action),
+      (CancelAction, Action, action),
+      // Resource Sync
+      (RunSync, ResourceSync, sync),
+      // Stack (simple)
+      (RunStackService, Stack, stack),
+      // Alerter
+      (TestAlerter, Alerter, alerter),
+    ],
+    batch: [
+      BatchDeploy,
+      BatchDestroyDeployment,
+      BatchRunBuild,
+      BatchCloneRepo,
+      BatchPullRepo,
+      BatchBuildRepo,
+      BatchRunProcedure,
+      BatchRunAction,
+      BatchDeployStack,
+      BatchDeployStackIfChanged,
+      BatchPullStack,
+      BatchDestroyStack,
+    ],
+    stack_service: [
+      (DeployStack, DeployStackService),
+      (PullStack, PullStackService),
+      (StartStack, StartStackService),
+      (RestartStack, RestartStackService),
+      (PauseStack, PauseStackService),
+      (UnpauseStack, UnpauseStackService),
+      (StopStack, StopStackService),
+      (DestroyStack, DestroyStackService),
+    ],
+    system: [
+      SendAlert,
+      ClearRepoCache,
+      BackupCoreDatabase,
+      GlobalAutoUpdate,
+      RotateAllServerKeys,
+      RotateCoreKeys,
+    ],
+  );
 
   let mut update = make_update(target, operation, user);
   update.in_progress();

bin/core/src/main.rs

@@ -19,6 +19,7 @@ mod monitor;
 mod network;
 mod periphery;
 mod permission;
+mod report;
 mod resource;
 mod schedule;
 mod stack;
@@ -71,6 +72,7 @@ async fn app() -> anyhow::Result<()> {
     resource::spawn_action_state_refresh_loop();
     schedule::spawn_schedule_executor();
     helpers::prune::spawn_prune_loop();
+    report::spawn_reporting_loop();
   }
   .instrument(startup_span)
   .await;

bin/core/src/monitor/alert/mod.rs

@@ -36,6 +36,8 @@ async fn get_all_swarms_map()
 -> anyhow::Result<(HashMap<String, Swarm>, HashMap<String, String>)> {
   let swarms = resource::list_full_for_user::<Swarm>(
     ResourceQuery::default(),
+    None,
+    None,
     system_user(),
     PermissionLevel::Read.into(),
     &[],
@@ -61,6 +63,8 @@ async fn get_all_servers_map()
 {
   let servers = resource::list_full_for_user::<Server>(
     ResourceQuery::default(),
+    None,
+    None,
     system_user(),
     PermissionLevel::Read.into(),
     &[],

bin/core/src/monitor/mod.rs

@@ -52,17 +52,18 @@ pub fn spawn_monitoring_loops() {
 
 fn spawn_server_monitoring_loop() {
   tokio::spawn(async move {
-    refresh_all_server_cache(komodo_timestamp()).await;
+    refresh_all_server_cache().await;
     let interval = monitoring_interval();
     loop {
       let ts = (wait_until_timelength(interval, ADDITIONAL_MS).await
         - ADDITIONAL_MS) as i64;
-      refresh_all_server_cache(ts).await;
+      refresh_all_server_cache().await;
+      tokio::join!(check_alerts(ts), record_server_stats(ts));
     }
   });
 }
 
-async fn refresh_all_server_cache(ts: i64) {
+async fn refresh_all_server_cache() {
   let servers =
     match find_collect(&db_client().servers, None, None).await {
       Ok(servers) => servers,
@@ -77,7 +78,6 @@ async fn refresh_all_server_cache(ts: i64) {
     refresh_server_cache(&server, false).await;
   });
   join_all(futures).await;
-  tokio::join!(check_alerts(ts), record_server_stats(ts));
 }
 
 /// Makes sure cache for server doesn't update too frequently / simultaneously.
@@ -193,7 +193,8 @@ pub fn refresh_server_cache(
 
     if let Some(docker) = &mut docker {
       docker.containers.iter_mut().for_each(|container| {
-        container.server_id = Some(server.id.clone())
+        container.server_id = Some(server.id.clone());
+        container.server_name = Some(server.name.clone());
       });
     }
 

bin/core/src/monitor/record.rs

@@ -20,9 +20,13 @@ pub async fn record_server_stats(ts: i64) {
         ts,
         sid: status.id.clone(),
         cpu_perc: stats.cpu_perc,
-        load_average: stats.load_average.clone(),
+        load_average: stats.load_average,
         mem_total_gb: stats.mem_total_gb,
         mem_used_gb: stats.mem_used_gb,
+        mem_buff_cache_gb: stats.mem_buff_cache_gb,
+        mem_zfs_arc_gb: stats.mem_zfs_arc_gb,
+        swap_total_gb: stats.swap_total_gb,
+        swap_used_gb: stats.swap_used_gb,
         disk_total_gb,
         disk_used_gb,
         disks: stats.disks.clone(),

bin/core/src/monitor/resources.rs

@@ -6,7 +6,10 @@ use komodo_client::entities::{
     container::ContainerListItem, image::ImageListItem,
     service::SwarmServiceListItem, stack::SwarmStackListItem,
   },
-  stack::{Stack, StackService, StackServiceNames, StackState},
+  stack::{
+    Stack, StackService, StackServiceNames, StackServiceState,
+    StackState,
+  },
   swarm::SwarmState,
 };
 
@@ -46,8 +49,10 @@ pub async fn update_swarm_stack_cache(
         Some(SwarmState::Unknown) | None => StackState::Unknown,
       })
       .unwrap_or(StackState::Down);
+
     let services = extract_services_from_stack(&stack);
     let service_prefix = format!("{project_name}_");
+
     let mut services_with_swarm_services = services
       .iter()
       .map(
@@ -74,27 +79,58 @@ pub async fn update_swarm_stack_cache(
                 .unwrap_or_default()
             })
             .cloned();
+
+          let (image, image_digests) = swarm_service
+            .as_ref()
+            .and_then(|swarm_service| swarm_service.image.as_ref())
+            .map(|image| {
+              (
+                image.clone(),
+                ImageDigest::parse(image).map(|d| vec![d]),
+              )
+            })
+            .unwrap_or((
+              if image.contains(':') {
+                image.to_string()
+              } else {
+                format!("{image}:latest")
+              },
+              None,
+            ));
+
+          let state = swarm_service
+            .as_ref()
+            .map(|c| c.state.into())
+            .unwrap_or(StackServiceState::Unknown);
+
           StackService {
+            stack_id: stack.id.clone(),
+            stack_name: stack.name.clone(),
             service: service_name.clone(),
-            image: image.clone(),
             container: None,
             swarm_service,
-            image_digests: Default::default(),
+            state,
+            image,
+            image_digests,
           }
         },
       )
       .collect::<Vec<_>>();
+
     services_with_swarm_services
       .sort_by(|a, b| a.service.cmp(&b.service));
+
     let prev_state = stack_status_cache
       .get(&stack.id)
       .await
       .map(|s| s.curr.state);
+
     let status = CachedStackStatus {
       id: stack.id.clone(),
       state: current_state,
       services: services_with_swarm_services,
     };
+
     stack_status_cache
       .insert(
         stack.id,
@@ -151,11 +187,19 @@ pub async fn update_server_stack_cache(
           None
         ));
 
+      let state = container
+        .as_ref()
+        .map(|c| c.state.into())
+        .unwrap_or(StackServiceState::Unknown);
+
       StackService {
+        stack_id: stack.id.clone(),
+        stack_name: stack.name.clone(),
         service: service_name.clone(),
         image: image.clone(),
         container,
         swarm_service: None,
+        state,
         image_digests,
       }
     }).collect::<Vec<_>>();
@@ -203,14 +247,21 @@ pub async fn update_swarm_deployment_cache(
         service
           .name
           .as_ref()
-          .map(|name| name == &deployment.name)
+          .map(|name| name == deployment.deployed_name())
           .unwrap_or_default()
       })
       .cloned();
+
+    let image_digests = service
+      .as_ref()
+      .and_then(|service| service.image.as_ref())
+      .and_then(|image| ImageDigest::parse(image).map(|d| vec![d]));
+
     let prev_state = deployment_status_cache
       .get(&deployment.id)
       .await
       .map(|s| s.curr.state);
+
     let current_state = service
       .as_ref()
       .map(|service| match service.state {
@@ -220,6 +271,7 @@ pub async fn update_swarm_deployment_cache(
         SwarmState::Unknown => DeploymentState::Unknown,
       })
       .unwrap_or(DeploymentState::NotDeployed);
+
     deployment_status_cache
       .insert(
         deployment.id.clone(),
@@ -227,9 +279,9 @@ pub async fn update_swarm_deployment_cache(
           curr: CachedDeploymentStatus {
             id: deployment.id,
             state: current_state,
-            service,
             container: None,
-            image_digests: None,
+            service,
+            image_digests,
           },
           prev: prev_state,
         }
@@ -249,8 +301,9 @@ pub async fn update_server_deployment_cache(
   for deployment in deployments {
     let container = containers
       .iter()
-      .find(|container| container.name == deployment.name)
+      .find(|container| container.name == deployment.deployed_name())
       .cloned();
+
     let image_digests = container
       .as_ref()
       .and_then(|container| container.image_id.as_ref())
@@ -263,10 +316,12 @@ pub async fn update_server_deployment_cache(
           }
         })
       });
+
     let prev_state = deployment_status_cache
       .get(&deployment.id)
       .await
       .map(|s| s.curr.state);
+
     let current_state = container
       .as_ref()
       .map(|c| c.state.into())

bin/core/src/monitor/swarm.rs

@@ -1,12 +1,8 @@
-use std::{
-  sync::{Arc, OnceLock},
-  time::Duration,
-};
+use std::sync::{Arc, OnceLock};
 
 use anyhow::anyhow;
 use async_timing_util::wait_until_timelength;
 use database::mungos::find::find_collect;
-use formatting::format_serror;
 use futures_util::future::join_all;
 use komodo_client::entities::{
   docker::node::NodeState,
@@ -21,7 +17,7 @@ use tokio::sync::Mutex;
 
 use crate::{
   config::monitoring_interval,
-  helpers::swarm::swarm_request_custom_timeout,
+  helpers::swarm::swarm_request,
   monitor::{
     RefreshCacheResources,
     resources::{
@@ -108,9 +104,9 @@ pub async fn refresh_swarm_cache(swarm: &Swarm, force: bool) {
           state: SwarmState::Unknown,
           inspect: None,
           lists: None,
-          err: Some(format_serror(
-            &anyhow!("No Servers configured as manager nodes").into(),
-          )),
+          err: Some(
+            anyhow!("No servers configured as manager nodes").into(),
+          ),
         }
         .into(),
       )
@@ -119,12 +115,8 @@ pub async fn refresh_swarm_cache(swarm: &Swarm, force: bool) {
   }
 
   let PollSwarmStatusResponse { inspect, lists } =
-    match swarm_request_custom_timeout(
-      &swarm.config.server_ids,
-      PollSwarmStatus {},
-      Duration::from_secs(1),
-    )
-    .await
+    match swarm_request(&swarm.config.server_ids, PollSwarmStatus {})
+      .await
     {
       Ok(info) => info,
       Err(e) => {
@@ -137,7 +129,7 @@ pub async fn refresh_swarm_cache(swarm: &Swarm, force: bool) {
               state: SwarmState::Unknown,
               inspect: None,
               lists: None,
-              err: Some(format_serror(&e.into())),
+              err: Some(e.into()),
             }
             .into(),
           )

bin/core/src/periphery/mod.rs

@@ -85,20 +85,6 @@ impl PeripheryClient {
     &self,
     request: T,
   ) -> anyhow::Result<T::Response>
-  where
-    T: std::fmt::Debug + Serialize + HasResponse,
-    T::Response: DeserializeOwned,
-  {
-    self
-      .request_custom_timeout(request, Duration::from_secs(10))
-      .await
-  }
-
-  pub async fn request_custom_timeout<T>(
-    &self,
-    request: T,
-    timeout: Duration,
-  ) -> anyhow::Result<T::Response>
   where
     T: std::fmt::Debug + Serialize + HasResponse,
     T::Response: DeserializeOwned,
@@ -134,11 +120,17 @@ impl PeripheryClient {
     let res = async {
       // Poll for the associated response
       loop {
-        let message =
-          response_receiever.recv().with_timeout(timeout).await?;
+        let message = response_receiever
+          .recv()
+          // Periphery request handler sends pings every 4s
+          // *on this channel specifically* so Core knows
+          // request is being processed. Hardcoded 11s
+          // allows for missed 5s ping due to network reconnect.
+          .with_timeout(Duration::from_secs(10))
+          .await?;
 
-        // Still in progress, sent to avoid timeout.
         let Some(message) = message.decode()? else {
+          // Just a ping from periphery request handler
           continue;
         };
 

bin/core/src/permission.rs

@@ -1,10 +1,10 @@
-use std::collections::HashMap;
+use std::{collections::HashMap, future::Future};
 
 use anyhow::{Context, anyhow};
 use database::{
   bson::Document, mongo_indexed::doc, mungos::find::find_collect,
 };
-use futures_util::{FutureExt, future::BoxFuture};
+use futures_util::{TryStreamExt, future::BoxFuture};
 use indexmap::IndexSet;
 use komodo_client::{
   api::read::GetPermission,
@@ -16,7 +16,9 @@ use komodo_client::{
     builder::Builder,
     deployment::Deployment,
     permission::SpecificPermission,
-    permission::{PermissionLevel, PermissionLevelAndSpecifics},
+    permission::{
+      Permission, PermissionLevel, PermissionLevelAndSpecifics,
+    },
     procedure::Procedure,
     repo::Repo,
     resource::Resource,
@@ -176,14 +178,34 @@ pub fn get_user_permission_on_resource<'a, T: KomodoResource>(
   })
 }
 
-pub async fn list_resources_for_user<T: KomodoResource>(
-  filters: impl Into<Option<Document>>,
+/// Precomputed user permissions for listing resources of a type.
+/// Load with [load_list_permits], then check visibility of
+/// each resource with [ListPermits::permitted].
+pub enum ListPermits {
+  /// The user can see all resources of the type,
+  /// eg. admin, transparent mode, or 'all' access on the variant.
+  Unrestricted,
+  /// Visibility must be checked per-resource
+  /// against the permissions table.
+  Fine(Box<FineListPermits>),
+}
+
+pub struct FineListPermits {
+  required: PermissionLevelAndSpecifics,
+  base: PermissionLevelAndSpecifics,
+  permission_by_resource_id: HashMap<String, Permission>,
+  additional_specific_cache:
+    HashMap<ResourceTarget, IndexSet<SpecificPermission>>,
+  user: User,
+}
+
+pub async fn load_list_permits<T: KomodoResource>(
   user: &User,
-  permission: PermissionLevelAndSpecifics,
-) -> anyhow::Result<Vec<Resource<T::Config, T::Info>>> {
+  required: PermissionLevelAndSpecifics,
+) -> anyhow::Result<ListPermits> {
   // Check admin
   if user.admin {
-    return list_all_resources::<T>(filters).await;
+    return Ok(ListPermits::Unrestricted);
   }
 
   let mut base = PermissionLevelAndSpecifics {
@@ -196,8 +218,8 @@ pub async fn list_resources_for_user<T: KomodoResource>(
   };
 
   // 'transparent_mode' early return.
-  if base.fulfills(&permission) {
-    return list_all_resources::<T>(filters).await;
+  if base.fulfills(&required) {
+    return Ok(ListPermits::Unrestricted);
   }
 
   let resource_type = T::resource_type();
@@ -206,8 +228,8 @@ pub async fn list_resources_for_user<T: KomodoResource>(
   if let Some(all_permission) = user.all.get(&resource_type) {
     base.elevate(all_permission);
     // 'user.all' early return.
-    if base.fulfills(&permission) {
-      return list_all_resources::<T>(filters).await;
+    if base.fulfills(&required) {
+      return Ok(ListPermits::Unrestricted);
     }
   }
 
@@ -217,25 +239,23 @@ pub async fn list_resources_for_user<T: KomodoResource>(
     if let Some(all_permission) = group.all.get(&resource_type) {
       base.elevate(all_permission);
       // 'group.all' early return.
-      if base.fulfills(&permission) {
-        return list_all_resources::<T>(filters).await;
+      if base.fulfills(&required) {
+        return Ok(ListPermits::Unrestricted);
       }
     }
   }
 
-  let (all, permissions) = tokio::try_join!(
-    list_all_resources::<T>(filters),
-    // And any ids using the permissions table
-    find_collect(
-      &db_client().permissions,
-      doc! {
-        "$or": user_target_query(&user.id, &groups)?,
-        "resource_target.type": resource_type.as_ref(),
-      },
-      None,
-    )
-    .map(|res| res.context("failed to query permissions on db"))
-  )?;
+  // Pull any permissions on the variant using the permissions table
+  let permissions = find_collect(
+    &db_client().permissions,
+    doc! {
+      "$or": user_target_query(&user.id, &groups)?,
+      "resource_target.type": resource_type.as_ref(),
+    },
+    None,
+  )
+  .await
+  .context("failed to query permissions on db")?;
 
   let permission_by_resource_id = permissions
     .into_iter()
@@ -247,34 +267,54 @@ pub async fn list_resources_for_user<T: KomodoResource>(
     })
     .collect::<HashMap<_, _>>();
 
-  let mut resources = Vec::new();
-  let mut additional_specific_cache =
-    HashMap::<ResourceTarget, IndexSet<SpecificPermission>>::new();
+  Ok(ListPermits::Fine(
+    FineListPermits {
+      required,
+      base,
+      permission_by_resource_id,
+      additional_specific_cache: Default::default(),
+      user: user.clone(),
+    }
+    .into(),
+  ))
+}
 
-  for resource in all {
-    let mut perm = if let Some(perm) =
-      permission_by_resource_id.get(&resource.id)
-    {
-      base.join(perm)
-    } else {
-      base.clone()
+impl ListPermits {
+  /// Check whether the user can see the given resource.
+  pub async fn permitted<T: KomodoResource>(
+    &mut self,
+    resource: &Resource<T::Config, T::Info>,
+  ) -> anyhow::Result<bool> {
+    let ListPermits::Fine(fine) = self else {
+      return Ok(true);
     };
+
+    let mut perm = if let Some(perm) =
+      fine.permission_by_resource_id.get(&resource.id)
+    {
+      fine.base.join(perm)
+    } else {
+      fine.base.clone()
+    };
+    // Add in the resource level base permissions,
+    // matching [get_user_permission_on_resource].
+    perm.elevate(&resource.base_permission);
     // Check if already fulfils
-    if perm.fulfills(&permission) {
-      resources.push(resource);
-      continue;
+    if perm.fulfills(&fine.required) {
+      return Ok(true);
     }
 
     // Also check if fulfills with inherited specific
     let additional_target = if let Some(additional_target) =
-      T::inherit_specific_permissions_from(&resource)
+      T::inherit_specific_permissions_from(resource)
       && !additional_target.is_empty()
     {
       additional_target
     } else {
-      continue;
+      return Ok(false);
     };
-    let additional_specific = match additional_specific_cache
+    let additional_specific = match fine
+      .additional_specific_cache
       .get(&additional_target)
       .cloned()
     {
@@ -283,21 +323,99 @@ pub async fn list_resources_for_user<T: KomodoResource>(
         let specific = GetPermission {
           target: additional_target.clone(),
         }
-        .resolve(&ReadArgs { user: user.clone() })
+        .resolve(&ReadArgs {
+          user: fine.user.clone(),
+        })
         .await
         .map_err(|e| e.error)
         .context(
           "failed to get user permission on additional target",
         )?
         .specific;
-        additional_specific_cache
+        fine
+          .additional_specific_cache
           .insert(additional_target, specific.clone());
         specific
       }
     };
     perm.specific.extend(additional_specific);
-    if perm.fulfills(&permission) {
-      resources.push(resource);
+    Ok(perm.fulfills(&fine.required))
+  }
+}
+
+pub async fn list_resources_for_user<T: KomodoResource>(
+  filters: impl Into<Option<Document>>,
+  limit: impl Into<Option<i64>>,
+  skip: impl Into<Option<u64>>,
+  user: &User,
+  permission: PermissionLevelAndSpecifics,
+) -> anyhow::Result<Vec<Resource<T::Config, T::Info>>> {
+  let mut permits = load_list_permits::<T>(user, permission).await?;
+
+  if let ListPermits::Unrestricted = permits {
+    return list_all_resources::<T>(filters, limit, skip).await;
+  }
+
+  list_resources_with_permits::<T, _>(
+    &mut permits,
+    filters.into(),
+    limit.into(),
+    skip.into(),
+    |resource| async move { Some(resource) },
+  )
+  .await
+}
+
+/// Drive the cursor directly, checking each resource against the
+/// user permissions and the additional `filter` (eg. by state
+/// computed from the in memory caches), with limit / skip applied
+/// in memory after the filters. Stops pulling from the cursor as
+/// soon as the limit is reached, and avoids collecting resources
+/// the user cannot see.
+///
+/// `filter` receives each resource by value, and returns
+/// `Some(resource)` to keep it in the results, or `None` to drop it.
+pub async fn list_resources_with_permits<T: KomodoResource, F>(
+  permits: &mut ListPermits,
+  filters: Option<Document>,
+  limit: Option<i64>,
+  skip: Option<u64>,
+  filter: impl Fn(Resource<T::Config, T::Info>) -> F,
+) -> anyhow::Result<Vec<Resource<T::Config, T::Info>>>
+where
+  F: Future<Output = Option<Resource<T::Config, T::Info>>> + Send,
+{
+  let mut cursor = T::coll()
+    .find(filters.unwrap_or_default())
+    .sort(doc! { "name": 1 })
+    .await
+    .with_context(|| {
+      format!("Failed to query db for {}s", T::resource_type())
+    })?;
+
+  let limit = limit.unwrap_or_default().max(0) as usize;
+  let skip = skip.unwrap_or_default();
+  let mut skipped = 0;
+  let mut resources = Vec::new();
+
+  while let Some(resource) = cursor
+    .try_next()
+    .await
+    .context("Failed to pull next resource from db cursor")?
+  {
+    if !permits.permitted::<T>(&resource).await? {
+      continue;
+    }
+    let Some(resource) = filter(resource).await else {
+      continue;
+    };
+    if skipped < skip {
+      skipped += 1;
+      continue;
+    }
+    resources.push(resource);
+    if limit != 0 && resources.len() >= limit {
+      break;
     }
   }
 
@@ -307,55 +425,28 @@ pub async fn list_resources_for_user<T: KomodoResource>(
 /// Returns None if still no need to filter by resource id (eg transparent mode, group membership with all access).
 pub async fn list_resource_ids_for_user<T: KomodoResource>(
   filters: Option<Document>,
+  limit: impl Into<Option<i64>>,
+  skip: impl Into<Option<u64>>,
   user: &User,
   permission: PermissionLevelAndSpecifics,
 ) -> anyhow::Result<Option<Vec<String>>> {
-  // Check admin
-  if user.admin {
+  let mut permits = load_list_permits::<T>(user, permission).await?;
+
+  if let ListPermits::Unrestricted = permits {
     return Ok(None);
   }
 
-  let mut base = PermissionLevelAndSpecifics {
-    level: if core_config().transparent_mode {
-      PermissionLevel::Read
-    } else {
-      PermissionLevel::None
-    },
-    specific: Default::default(),
-  };
-
-  // 'transparent_mode' early return.
-  if base.fulfills(&permission) {
-    return Ok(None);
-  }
-
-  let resource_type = T::resource_type();
-
-  if let Some(all) = user.all.get(&resource_type) {
-    base.elevate(all);
-    // 'user.all' early return.
-    if base.fulfills(&permission) {
-      return Ok(None);
-    }
-  }
-
-  // Check user groups 'all' on variant
-  let groups = get_user_user_groups(&user.id).await?;
-  for group in &groups {
-    if let Some(all) = group.all.get(&resource_type) {
-      base.elevate(all);
-      // 'group.all' early return.
-      if base.fulfills(&permission) {
-        return Ok(None);
-      }
-    }
-  }
-
-  let ids = list_resources_for_user::<T>(filters, user, permission)
-    .await?
-    .into_iter()
-    .map(|resource| resource.id)
-    .collect();
+  let ids = list_resources_with_permits::<T, _>(
+    &mut permits,
+    filters,
+    limit.into(),
+    skip.into(),
+    |resource| async move { Some(resource) },
+  )
+  .await?
+  .into_iter()
+  .map(|resource| resource.id)
+  .collect();
 
   Ok(Some(ids))
 }
@@ -369,6 +460,8 @@ pub async fn user_resource_target_query(
     Ok(incoming_query)
   } else {
     let swarm_query = list_resource_ids_for_user::<Swarm>(
+      None,
+      None,
       None,
       user,
       PermissionLevel::Read.into(),
@@ -384,6 +477,8 @@ pub async fn user_resource_target_query(
     .unwrap_or_else(|| doc! { "target.type": "Swarm" });
 
     let server_query = list_resource_ids_for_user::<Server>(
+      None,
+      None,
       None,
       user,
       PermissionLevel::Read.into(),
@@ -397,6 +492,8 @@ pub async fn user_resource_target_query(
     .unwrap_or_else(|| doc! { "target.type": "Server" });
 
     let stack_query = list_resource_ids_for_user::<Stack>(
+      None,
+      None,
       None,
       user,
       PermissionLevel::Read.into(),
@@ -410,6 +507,8 @@ pub async fn user_resource_target_query(
     .unwrap_or_else(|| doc! { "target.type": "Stack" });
 
     let deployment_query = list_resource_ids_for_user::<Deployment>(
+      None,
+      None,
       None,
       user,
       PermissionLevel::Read.into(),
@@ -423,6 +522,8 @@ pub async fn user_resource_target_query(
     .unwrap_or_else(|| doc! { "target.type": "Deployment" });
 
     let build_query = list_resource_ids_for_user::<Build>(
+      None,
+      None,
       None,
       user,
       PermissionLevel::Read.into(),
@@ -436,6 +537,8 @@ pub async fn user_resource_target_query(
     .unwrap_or_else(|| doc! { "target.type": "Build" });
 
     let repo_query = list_resource_ids_for_user::<Repo>(
+      None,
+      None,
       None,
       user,
       PermissionLevel::Read.into(),
@@ -449,6 +552,8 @@ pub async fn user_resource_target_query(
     .unwrap_or_else(|| doc! { "target.type": "Repo" });
 
     let procedure_query = list_resource_ids_for_user::<Procedure>(
+      None,
+      None,
       None,
       user,
       PermissionLevel::Read.into(),
@@ -462,6 +567,8 @@ pub async fn user_resource_target_query(
     .unwrap_or_else(|| doc! { "target.type": "Procedure" });
 
     let action_query = list_resource_ids_for_user::<Action>(
+      None,
+      None,
       None,
       user,
       PermissionLevel::Read.into(),
@@ -475,6 +582,8 @@ pub async fn user_resource_target_query(
     .unwrap_or_else(|| doc! { "target.type": "Action" });
 
     let builder_query = list_resource_ids_for_user::<Builder>(
+      None,
+      None,
       None,
       user,
       PermissionLevel::Read.into(),
@@ -488,6 +597,8 @@ pub async fn user_resource_target_query(
     .unwrap_or_else(|| doc! { "target.type": "Builder" });
 
     let alerter_query = list_resource_ids_for_user::<Alerter>(
+      None,
+      None,
       None,
       user,
       PermissionLevel::Read.into(),
@@ -502,6 +613,8 @@ pub async fn user_resource_target_query(
 
     let resource_sync_query =
       list_resource_ids_for_user::<ResourceSync>(
+        None,
+        None,
         None,
         user,
         PermissionLevel::Read.into(),

bin/core/src/report.rs

@@ -0,0 +1,154 @@
+use std::time::{SystemTime, UNIX_EPOCH};
+
+use anyhow::Context;
+use async_timing_util::{Timelength, wait_until_timelength};
+use komodo_client::entities::{
+  ResourceTargetVariant, report::KomodoReport,
+};
+use mogh_pki::{
+  PkiKind, RotatableKeyPair, SpkiPublicKey,
+  one_way::OneWayNoiseHandshake,
+};
+
+use crate::{config::core_config, state::db_client};
+
+pub fn spawn_reporting_loop() {
+  let config = core_config();
+
+  if !config.reporting_enabled {
+    return;
+  }
+
+  let keys = match RotatableKeyPair::from_private_key_spec(
+    PkiKind::OneWay,
+    &config.reporting_private_key,
+  ) {
+    Ok(keys) => keys,
+    Err(e) => {
+      warn!(
+        "Failed to initialize reporting key pair. Reporting is disabled. {e:#}"
+      );
+      return;
+    }
+  };
+
+  let client = reqwest::Client::default();
+
+  tokio::spawn(async move {
+    loop {
+      wait_until_timelength(Timelength::OneDay, 1100).await;
+      if let Err(e) = report(&client, &keys).await {
+        warn!("Reporting failed. {e:#}");
+      }
+    }
+  });
+}
+
+async fn report(
+  client: &reqwest::Client,
+  keys: &RotatableKeyPair,
+) -> anyhow::Result<()> {
+  let db = db_client();
+
+  let keys = keys.load();
+  let private_key_bytes = keys
+    .private
+    .as_raw_bytes()
+    .context("Invalid reporting private key.")?;
+
+  let endpoint_public_key = client
+    .get("https://mogh.tech/report/public_key")
+    .send()
+    .await
+    .context("Failed to query for reporting endpoint public key for signature.")?
+    .error_for_status()
+    .context("Failed response for reporting endpoint public key for signature.")?
+    .text()
+    .await
+    .context("Failed to get reporting endpoint public key for signature.")?;
+
+  let endpoint_public_key =
+    SpkiPublicKey::maybe_pem_to_raw_bytes(&endpoint_public_key)
+      .context("Invalid reporting endpoint public key.")?;
+
+  let users = db
+    .users
+    .count_documents(Default::default())
+    .await
+    .context("Failed to query database for users")?;
+  let servers = db
+    .servers
+    .count_documents(Default::default())
+    .await
+    .context("Failed to query database for servers")?;
+  let swarms = db
+    .swarms
+    .count_documents(Default::default())
+    .await
+    .context("Failed to query database for swarms")?;
+  let stacks = db
+    .stacks
+    .count_documents(Default::default())
+    .await
+    .context("Failed to query database for stacks")?;
+  let deployments = db
+    .deployments
+    .count_documents(Default::default())
+    .await
+    .context("Failed to query database for deployments")?;
+  let builds = db
+    .builds
+    .count_documents(Default::default())
+    .await
+    .context("Failed to query database for builds")?;
+
+  let count = [
+    (ResourceTargetVariant::Server, servers),
+    (ResourceTargetVariant::Swarm, swarms),
+    (ResourceTargetVariant::Stack, stacks),
+    (ResourceTargetVariant::Deployment, deployments),
+    (ResourceTargetVariant::Build, builds),
+  ]
+  .into_iter()
+  .collect();
+
+  let report = KomodoReport {
+    public_key: keys.public().to_string(),
+    version: String::from(env!("CARGO_PKG_VERSION")),
+    users,
+    count,
+  };
+
+  let serialized = serde_json::to_string(&report)
+    .context("Failed to serialize report JSON")?;
+
+  let timestamp =
+    SystemTime::now().duration_since(UNIX_EPOCH)?.as_millis() as i64;
+
+  let prologue =
+    format!("POST|/report/komodo|{serialized}|{timestamp}");
+
+  let mut handshake = OneWayNoiseHandshake::new_initiator(
+    &private_key_bytes,
+    &endpoint_public_key,
+    prologue.as_bytes(),
+  )?;
+
+  let signature = handshake
+    .generate_signature()
+    .context("Failed to generate report signature")?;
+
+  client
+    .post("https://mogh.tech/report/komodo")
+    .header("x-api-signature", signature)
+    .header("x-api-timestamp", timestamp)
+    .header("content-type", "application/json")
+    .body(serialized)
+    .send()
+    .await
+    .context("Failed to post report.")?
+    .error_for_status()
+    .context("Failed response for report post.")?;
+
+  Ok(())
+}

bin/core/src/resource/build.rs

@@ -74,8 +74,9 @@ impl super::KomodoResource for Build {
       build.config.repo,
       build.config.branch,
       build.config.git_https,
+      String::new(),
     );
-    let (git_provider, repo, branch, git_https) =
+    let (git_provider, repo, branch, git_https, linked_repo_name) =
       if build.config.linked_repo.is_empty() {
         default_git
       } else {
@@ -89,6 +90,7 @@ impl super::KomodoResource for Build {
               r.config.repo.clone(),
               r.config.branch.clone(),
               r.config.git_https,
+              r.name.clone(),
             )
           })
           .unwrap_or(default_git)
@@ -107,6 +109,7 @@ impl super::KomodoResource for Build {
         files_on_host: build.config.files_on_host,
         dockerfile_contents: !build.config.dockerfile.is_empty(),
         linked_repo: build.config.linked_repo,
+        linked_repo_name,
         repo_link: repo_link(
           &git_provider,
           &repo,
@@ -297,7 +300,9 @@ async fn validate_config(
   Ok(())
 }
 
-async fn get_build_state(id: &String) -> BuildState {
+/// The Build state as computed for the build list items,
+/// from the in memory action states / state cache.
+pub async fn get_build_state(id: &String) -> BuildState {
   if action_states()
     .build
     .get(id)

bin/core/src/resource/builder.rs

@@ -54,7 +54,7 @@ impl super::KomodoResource for Builder {
       }
       BuilderConfig::Server(config) => (
         BuilderConfigVariant::Server.to_string(),
-        Some(config.server_id),
+        Some(config.server_ids.join(", ")),
       ),
       BuilderConfig::Aws(config) => (
         BuilderConfigVariant::Aws.to_string(),
@@ -184,10 +184,11 @@ async fn validate_config(
   config: &mut PartialBuilderConfig,
   user: &User,
 ) -> anyhow::Result<()> {
-  match config {
-    PartialBuilderConfig::Server(PartialServerBuilderConfig {
-      server_id: Some(server_id),
-    }) if !server_id.is_empty() => {
+  if let PartialBuilderConfig::Server(PartialServerBuilderConfig {
+    server_ids: Some(server_ids),
+  }) = config
+  {
+    for server_id in server_ids {
       let server = super::get_check_permissions::<Server>(
         server_id,
         user,
@@ -196,7 +197,6 @@ async fn validate_config(
       .await?;
       *server_id = server.id;
     }
-    _ => {}
   }
   Ok(())
 }

bin/core/src/resource/deployment.rs

@@ -1,5 +1,7 @@
-use anyhow::Context;
-use database::mungos::mongodb::Collection;
+use anyhow::{Context, anyhow};
+use database::mungos::{
+  by_id::update_one_by_id, mongodb::Collection, mongodb::bson::doc,
+};
 use formatting::format_serror;
 use indexmap::IndexSet;
 use komodo_client::entities::{
@@ -11,7 +13,7 @@ use komodo_client::entities::{
     DeploymentListItemInfo, DeploymentQuerySpecifics,
     DeploymentState, PartialDeploymentConfig, conversions_from_str,
   },
-  environment_vars_from_str,
+  environment_vars_from_str, optional_string,
   permission::{
     PermissionLevel, PermissionLevelAndSpecifics, SpecificPermission,
   },
@@ -23,7 +25,8 @@ use komodo_client::entities::{
   user::User,
 };
 use periphery_client::api::{
-  container::RemoveContainer, swarm::RemoveSwarmServices,
+  container::{RemoveContainer, RenameContainer},
+  swarm::RemoveSwarmServices,
 };
 
 use crate::{
@@ -34,7 +37,10 @@ use crate::{
     swarm::swarm_request,
   },
   monitor::{refresh_server_cache, refresh_swarm_cache},
-  state::{action_states, db_client, deployment_status_cache},
+  state::{
+    action_states, all_resources_cache, db_client,
+    deployment_status_cache,
+  },
 };
 
 use super::get_check_permissions;
@@ -90,31 +96,31 @@ impl super::KomodoResource for Deployment {
     deployment: Resource<Self::Config, Self::Info>,
   ) -> Self::ListItem {
     let status = deployment_status_cache().get(&deployment.id).await;
-    let state = if action_states()
-      .deployment
-      .get(&deployment.id)
+    let state = get_deployment_state(&deployment.id)
       .await
-      .map(|s| s.get().map(|s| s.deploying))
-      .transpose()
-      .ok()
-      .flatten()
-      .unwrap_or_default()
-    {
-      DeploymentState::Deploying
-    } else {
-      status.as_ref().map(|s| s.curr.state).unwrap_or_default()
-    };
+      .unwrap_or_default();
+    let all = all_resources_cache().load();
+    let server_name = all
+      .servers
+      .get(&deployment.config.server_id)
+      .map(|server| server.name.clone())
+      .unwrap_or_default();
+    let swarm_name = all
+      .swarms
+      .get(&deployment.config.swarm_id)
+      .map(|swarm| swarm.name.clone())
+      .unwrap_or_default();
     let (build_image, build_id) = match deployment.config.image {
       DeploymentImage::Build { build_id, version } => {
-        let (build_name, build_id, build_version) =
-          super::get::<Build>(&build_id)
-            .await
-            .map(|b| (b.name, b.id, b.config.version))
-            .unwrap_or((
-              String::from("unknown"),
-              String::new(),
-              Default::default(),
-            ));
+        let (build_name, build_id, build_version) = all
+          .builds
+          .get(&build_id)
+          .map(|b| (b.name.clone(), b.id.clone(), b.config.version))
+          .unwrap_or((
+            String::from("unknown"),
+            String::new(),
+            Default::default(),
+          ));
         let version = if version.is_none() {
           build_version.to_string()
         } else {
@@ -168,10 +174,14 @@ impl super::KomodoResource for Deployment {
         status: status.as_ref().and_then(|s| {
           s.curr.container.as_ref().and_then(|c| c.status.to_owned())
         }),
+        custom_name: optional_string(deployment.info.deployed_name)
+          .unwrap_or(deployment.config.custom_name),
         image,
         update_available,
         swarm_id: deployment.config.swarm_id,
+        swarm_name,
         server_id: deployment.config.server_id,
+        server_name,
         build_id,
       },
     }
@@ -244,11 +254,12 @@ impl super::KomodoResource for Deployment {
   }
 
   async fn validate_update_config(
-    _id: &str,
+    id: &str,
     config: &mut Self::PartialConfig,
     user: &User,
   ) -> anyhow::Result<()> {
-    validate_config(config, user).await
+    validate_config(config, user).await?;
+    handle_custom_name_update(id, config).await
   }
 
   async fn post_update(
@@ -314,7 +325,7 @@ impl super::KomodoResource for Deployment {
       SwarmOrServer::Swarm(swarm) => match swarm_request(
         &swarm.config.server_ids,
         RemoveSwarmServices {
-          services: vec![deployment.name.clone()],
+          services: vec![deployment.deployed_name().to_string()],
         },
       )
       .await
@@ -352,7 +363,7 @@ impl super::KomodoResource for Deployment {
         };
         match periphery
           .request(RemoveContainer {
-            name: deployment.name.clone(),
+            name: deployment.deployed_name().to_string(),
             signal: deployment.config.termination_signal.into(),
             time: deployment.config.termination_timeout.into(),
           })
@@ -436,12 +447,100 @@ async fn validate_config(
     environment_vars_from_str(environment)
       .context("Invalid environment")?;
   }
+  if let Some(custom_name) = &config.custom_name
+    && !custom_name.trim().is_empty()
+  {
+    config.custom_name =
+      Some(to_container_compatible_name(custom_name));
+  }
   if let Some(extra_args) = &mut config.extra_args {
     extra_args.retain(|v| !empty_or_only_spaces(v))
   }
   Ok(())
 }
 
+/// Handles a config update which changes the custom container name:
+///   - Server mode: the container is renamed to keep the Deployment
+///     matched to it. This runs before the update is written to the
+///     database, so a failed rename rejects the update and nothing
+///     is left mismatched.
+///   - Swarm mode: services cannot be renamed, so the current
+///     service name is pinned in info.deployed_name, and stays
+///     in use until the next deploy recreates the service.
+async fn handle_custom_name_update(
+  id: &str,
+  config: &PartialDeploymentConfig,
+) -> anyhow::Result<()> {
+  let Some(custom_name) = &config.custom_name else {
+    return Ok(());
+  };
+  let deployment = super::get::<Deployment>(id).await?;
+  // custom_name is already made container compatible by validate_config.
+  let new_container_name = if custom_name.is_empty() {
+    deployment.name.as_str()
+  } else {
+    custom_name.as_str()
+  };
+  if deployment.custom_name() == new_container_name {
+    return Ok(());
+  }
+  if deployment.config.swarm_id.is_empty()
+    && !deployment.config.server_id.is_empty()
+  {
+    let container_state =
+      get_deployment_state(&deployment.id).await?;
+    if container_state == DeploymentState::Unknown {
+      return Err(anyhow!(
+        "Cannot change custom container name when container status is unknown"
+      ));
+    }
+    if container_state == DeploymentState::NotDeployed {
+      return Ok(());
+    }
+    let server =
+      super::get::<Server>(&deployment.config.server_id).await?;
+    periphery_client(&server)
+      .await?
+      .request(RenameContainer {
+        curr_name: deployment.deployed_name().to_string(),
+        new_name: new_container_name.to_string(),
+      })
+      .await
+      .context("Failed to rename container to the new custom name")?;
+    // Keep the tracked deployed name matched to the renamed container.
+    update_one_by_id(
+      &db_client().deployments,
+      &deployment.id,
+      database::mungos::update::Update::Set(
+        doc! { "info.deployed_name": new_container_name },
+      ),
+      None,
+    )
+    .await
+    .context(
+      "Failed to update deployed name on db after container rename",
+    )?;
+  } else if !deployment.config.swarm_id.is_empty()
+    && deployment.info.deployed_name.is_empty()
+    // Includes the Unknown state, in case a service
+    // is still running under the current name.
+    && get_deployment_state(&deployment.id).await?
+      != DeploymentState::NotDeployed
+  {
+    update_one_by_id(
+      &db_client().deployments,
+      &deployment.id,
+      database::mungos::update::Update::Set(
+        doc! { "info.deployed_name": deployment.deployed_name() },
+      ),
+      None,
+    )
+    .await
+    .context("Failed to pin deployed service name on db")?;
+  }
+  Ok(())
+}
+
 pub async fn setup_deployment_execution(
   deployment: &str,
   user: &User,

bin/core/src/resource/mod.rs

@@ -1,5 +1,6 @@
 use std::{
   collections::{HashMap, HashSet},
+  future::Future,
   str::FromStr,
 };
 
@@ -17,7 +18,7 @@ use database::{
   },
 };
 use formatting::format_serror;
-use futures_util::future::join_all;
+use futures_util::{TryStreamExt, future::join_all};
 use indexmap::IndexSet;
 use komodo_client::{
   api::{read::ExportResourcesToToml, write::CreateTag},
@@ -70,7 +71,8 @@ pub use action::{
   refresh_action_state_cache, spawn_action_state_refresh_loop,
 };
 pub use build::{
-  refresh_build_state_cache, spawn_build_state_refresh_loop,
+  get_build_state, refresh_build_state_cache,
+  spawn_build_state_refresh_loop,
 };
 pub use deployment::setup_deployment_execution;
 pub use procedure::{
@@ -82,7 +84,8 @@ pub use refresh::{
   spawn_resource_refresh_loop,
 };
 pub use repo::{
-  refresh_repo_state_cache, spawn_repo_state_refresh_loop,
+  get_repo_state, refresh_repo_state_cache,
+  spawn_repo_state_refresh_loop,
 };
 pub use server::{rotate_server_keys, update_server_public_key};
 
@@ -262,20 +265,156 @@ pub async fn get<T: KomodoResource>(
 /// Get full resource list with no permissions check.
 pub async fn list_all_resources<T: KomodoResource>(
   filters: impl Into<Option<Document>>,
+  limit: impl Into<Option<i64>>,
+  skip: impl Into<Option<u64>>,
 ) -> anyhow::Result<Vec<Resource<T::Config, T::Info>>> {
-  find_collect(
-    T::coll(),
-    filters,
-    FindOptions::builder().sort(doc! { "name": 1 }).build(),
-  )
-  .await
-  .with_context(|| {
-    format!("Failed to pull {}s from mongo", T::resource_type())
-  })
+  let options = FindOptions::builder()
+    .sort(doc! { "name": 1 })
+    .limit(limit)
+    .skip(skip);
+  find_collect(T::coll(), filters, options.build())
+    .await
+    .with_context(|| {
+      format!("Failed to pull {}s from mongo", T::resource_type())
+    })
 }
 
+pub struct ListItemsQueryOptions<T: KomodoResource> {
+  pub limit: u64,
+  pub page: u64,
+  pub sort_desc: bool,
+  pub sort_by: ListItemSort<T::ListItem>,
+}
+
+/// List item pagination for the `List<Resource>` apis, driving the
+/// mongo cursor directly instead of collecting the full resource
+/// list in memory. Each resource pulled from the cursor is checked
+/// against the user permissions and converted to its list item, and
+/// `filter` is applied before the item counts toward `limit` / `page`.
+/// This is required because some list item fields (eg. state,
+/// update available) are computed from in-memory caches rather than
+/// stored on the database, so they cannot be part of the db query.
+/// Stops pulling from the cursor as soon as the page is full.
+pub async fn list_items_for_user<T: KomodoResource>(
+  mut query: ResourceQuery<T::QuerySpecifics>,
+  ListItemsQueryOptions {
+    limit,
+    page,
+    sort_desc,
+    sort_by,
+  }: ListItemsQueryOptions<T>,
+  user: &User,
+  permission: PermissionLevelAndSpecifics,
+  all_tags: &[Tag],
+  filter: impl Fn(&T::ListItem) -> bool,
+) -> anyhow::Result<Vec<T::ListItem>> {
+  validate_resource_query_tags(&mut query, all_tags)?;
+  let mut filters = Document::new();
+  query.add_filters(&mut filters);
+
+  let mut permits =
+    crate::permission::load_list_permits::<T>(user, permission)
+      .await?;
+
+  let direction = if sort_desc { -1 } else { 1 };
+  let sort = match &sort_by {
+    ListItemSort::Name => doc! { "name": direction },
+    // Db field sorts use ascending name as the secondary sort.
+    ListItemSort::DbField(field) => {
+      doc! { *field: direction, "name": 1 }
+    }
+    // In-memory sorts pull items in ascending name db order,
+    // keeping name as the stable tiebreak for equal sort keys.
+    ListItemSort::InMemory(_) => doc! { "name": 1 },
+  };
+
+  let mut cursor =
+    T::coll().find(filters).sort(sort).await.with_context(|| {
+      format!("Failed to query db for {}s", T::resource_type())
+    })?;
+
+  let skip = page.saturating_mul(limit) as usize;
+  let take = if limit == 0 {
+    usize::MAX
+  } else {
+    limit as usize
+  };
+  let mut items = Vec::new();
+
+  if let ListItemSort::InMemory(compare) = sort_by {
+    // The sort can only be applied after all matching items
+    // are collected, so pagination also happens after the sort.
+    while let Some(resource) = cursor
+      .try_next()
+      .await
+      .context("Failed to pull next resource from db cursor")?
+    {
+      if !permits.permitted::<T>(&resource).await? {
+        continue;
+      }
+      let item = T::to_list_item(resource).await;
+      if !filter(&item) {
+        continue;
+      }
+      items.push(item);
+    }
+    // Stable sort keeps equal sort keys in ascending
+    // name order for both directions.
+    if sort_desc {
+      items.sort_by(|a, b| compare(b, a));
+    } else {
+      items.sort_by(|a, b| compare(a, b));
+    }
+    Ok(items.into_iter().skip(skip).take(take).collect())
+  } else {
+    let mut skipped = 0;
+    while let Some(resource) = cursor
+      .try_next()
+      .await
+      .context("Failed to pull next resource from db cursor")?
+    {
+      if !permits.permitted::<T>(&resource).await? {
+        continue;
+      }
+      let item = T::to_list_item(resource).await;
+      if !filter(&item) {
+        continue;
+      }
+      if skipped < skip {
+        skipped += 1;
+        continue;
+      }
+      items.push(item);
+      if take != usize::MAX && items.len() >= take {
+        break;
+      }
+    }
+    Ok(items)
+  }
+}
+
+/// How the `List<Resource>` apis sort the list items.
+pub enum ListItemSort<I> {
+  /// Sort by name at the db level. Default.
+  Name,
+  /// Sort on a db stored field, keeping db level sort
+  /// and streaming pagination. Only usable when the db field
+  /// exactly matches the displayed list item field.
+  DbField(&'static str),
+  /// Compare list items in memory, required for fields computed
+  /// from the in memory caches (eg. state), or which diverge
+  /// from the db field (eg. linked repo sources).
+  /// Collects all matching items before applying pagination.
+  InMemory(InMemoryListItemSortFn<I>),
+}
+
+pub type InMemoryListItemSortFn<I> =
+  Box<dyn Fn(&I, &I) -> std::cmp::Ordering + Send>;
+
 pub async fn list_for_user<T: KomodoResource>(
   mut query: ResourceQuery<T::QuerySpecifics>,
+  limit: impl Into<Option<i64>>,
+  skip: impl Into<Option<u64>>,
   user: &User,
   permission: PermissionLevelAndSpecifics,
   all_tags: &[Tag],
@@ -283,38 +422,25 @@ pub async fn list_for_user<T: KomodoResource>(
   validate_resource_query_tags(&mut query, all_tags)?;
   let mut filters = Document::new();
   query.add_filters(&mut filters);
-  list_for_user_using_document::<T>(filters, user, permission).await
+  list_for_user_using_document::<T>(
+    filters, limit, skip, user, permission,
+  )
+  .await
 }
 
-// // pub async fn list_for_user_using_pattern<T: KomodoResource>(
-//   pattern: &str,
-//   query: ResourceQuery<T::QuerySpecifics>,
-//   user: &User,
-//   permissions: PermissionLevelAndSpecifics,
-//   all_tags: &[Tag],
-// ) -> anyhow::Result<Vec<T::ListItem>> {
-//   let list = list_full_for_user_using_pattern::<T>(
-//     pattern,
-//     query,
-//     user,
-//     permissions,
-//     all_tags,
-//   )
-//   .await?
-//   .into_iter()
-//   .map(|resource| T::to_list_item(resource));
-//   Ok(join_all(list).await)
-// }
-
 pub async fn list_for_user_using_document<T: KomodoResource>(
   filters: Document,
+  limit: impl Into<Option<i64>>,
+  skip: impl Into<Option<u64>>,
   user: &User,
   permission: PermissionLevelAndSpecifics,
 ) -> anyhow::Result<Vec<T::ListItem>> {
-  let list = list_resources_for_user::<T>(filters, user, permission)
-    .await?
-    .into_iter()
-    .map(|resource| T::to_list_item(resource));
+  let list = list_resources_for_user::<T>(
+    filters, limit, skip, user, permission,
+  )
+  .await?
+  .into_iter()
+  .map(|resource| T::to_list_item(resource));
   Ok(join_all(list).await)
 }
 
@@ -329,13 +455,16 @@ pub async fn list_for_user_using_document<T: KomodoResource>(
 pub async fn list_full_for_user_using_pattern<T: KomodoResource>(
   pattern: &str,
   query: ResourceQuery<T::QuerySpecifics>,
+  limit: impl Into<Option<i64>>,
+  skip: impl Into<Option<u64>>,
   user: &User,
   permission: PermissionLevelAndSpecifics,
   all_tags: &[Tag],
 ) -> anyhow::Result<Vec<Resource<T::Config, T::Info>>> {
-  let resources =
-    list_full_for_user::<T>(query, user, permission, all_tags)
-      .await?;
+  let resources = list_full_for_user::<T>(
+    query, limit, skip, user, permission, all_tags,
+  )
+  .await?;
 
   let patterns = parse_string_list(pattern);
   let mut names = HashSet::<String>::new();
@@ -368,8 +497,45 @@ pub async fn list_full_for_user_using_pattern<T: KomodoResource>(
   )
 }
 
+/// Same as [list_full_for_user], but applies an additional in-memory
+/// `filter` (eg. by state computed from the in memory caches), with
+/// limit / page applied after the filter. Required because such
+/// filters cannot be expressed in the db level query.
+///
+/// `filter` receives each resource by value, and returns
+/// `Some(resource)` to keep it in the results, or `None` to drop it.
+pub async fn list_full_for_user_filtered<T: KomodoResource, F>(
+  mut query: ResourceQuery<T::QuerySpecifics>,
+  limit: u64,
+  page: u64,
+  user: &User,
+  permission: PermissionLevelAndSpecifics,
+  all_tags: &[Tag],
+  filter: impl Fn(Resource<T::Config, T::Info>) -> F,
+) -> anyhow::Result<Vec<Resource<T::Config, T::Info>>>
+where
+  F: Future<Output = Option<Resource<T::Config, T::Info>>> + Send,
+{
+  validate_resource_query_tags(&mut query, all_tags)?;
+  let mut filters = Document::new();
+  query.add_filters(&mut filters);
+  let mut permits =
+    crate::permission::load_list_permits::<T>(user, permission)
+      .await?;
+  crate::permission::list_resources_with_permits::<T, _>(
+    &mut permits,
+    Some(filters),
+    Some(limit as i64),
+    Some(page.saturating_mul(limit)),
+    filter,
+  )
+  .await
+}
+
 pub async fn list_full_for_user<T: KomodoResource>(
   mut query: ResourceQuery<T::QuerySpecifics>,
+  limit: impl Into<Option<i64>>,
+  skip: impl Into<Option<u64>>,
   user: &User,
   permissions: PermissionLevelAndSpecifics,
   all_tags: &[Tag],
@@ -377,7 +543,14 @@ pub async fn list_full_for_user<T: KomodoResource>(
   validate_resource_query_tags(&mut query, all_tags)?;
   let mut filters = Document::new();
   query.add_filters(&mut filters);
-  list_resources_for_user::<T>(filters, user, permissions).await
+  list_resources_for_user::<T>(
+    filters,
+    limit,
+    skip,
+    user,
+    permissions,
+  )
+  .await
 }
 
 pub type IdResourceMap<T> = HashMap<

bin/core/src/resource/procedure.rs

@@ -183,695 +183,186 @@ async fn validate_config(
   };
   for stage in stages {
     for exec in &mut stage.executions {
-      match &mut exec.execution {
-        Execution::None(_) => {}
-        Execution::RunProcedure(params) => {
-          let procedure = super::get_check_permissions::<Procedure>(
-            &params.procedure,
-            user,
-            PermissionLevel::Execute.into(),
-          )
-          .await?;
-          match id {
-            Some(id) if procedure.id == id => {
-              return Err(anyhow!(
-                "Cannot have self-referential procedure"
-              ));
-            }
-            _ => {}
-          }
-          params.procedure = procedure.id;
-        }
-        Execution::BatchRunProcedure(_params) => {
-          if !user.admin {
-            return Err(anyhow!(
-              "Non admin user cannot configure Batch executions"
-            ));
-          }
-        }
-        Execution::RunAction(params) => {
-          let action = super::get_check_permissions::<Action>(
-            &params.action,
-            user,
-            PermissionLevel::Execute.into(),
-          )
-          .await?;
-          params.action = action.id;
-        }
-        Execution::BatchRunAction(_params) => {
-          if !user.admin {
-            return Err(anyhow!(
-              "Non admin user cannot configure Batch executions"
-            ));
-          }
-        }
-        Execution::RunBuild(params) => {
-          let build = super::get_check_permissions::<Build>(
-            &params.build,
-            user,
-            PermissionLevel::Execute.into(),
-          )
-          .await?;
-          params.build = build.id;
-        }
-        Execution::BatchRunBuild(_params) => {
-          if !user.admin {
-            return Err(anyhow!(
-              "Non admin user cannot configure Batch executions"
-            ));
-          }
-        }
-        Execution::CancelBuild(params) => {
-          let build = super::get_check_permissions::<Build>(
-            &params.build,
-            user,
-            PermissionLevel::Execute.into(),
-          )
-          .await?;
-          params.build = build.id;
-        }
-        Execution::Deploy(params) => {
-          let deployment =
-            super::get_check_permissions::<Deployment>(
-              &params.deployment,
-              user,
-              PermissionLevel::Execute.into(),
-            )
-            .await?;
-          params.deployment = deployment.id;
-        }
-        Execution::BatchDeploy(_params) => {
-          if !user.admin {
-            return Err(anyhow!(
-              "Non admin user cannot configure Batch executions"
-            ));
-          }
-        }
-        Execution::PullDeployment(params) => {
-          let deployment =
-            super::get_check_permissions::<Deployment>(
-              &params.deployment,
-              user,
-              PermissionLevel::Execute.into(),
-            )
-            .await?;
-          params.deployment = deployment.id;
-        }
-        Execution::StartDeployment(params) => {
-          let deployment =
-            super::get_check_permissions::<Deployment>(
-              &params.deployment,
-              user,
-              PermissionLevel::Execute.into(),
-            )
-            .await?;
-          params.deployment = deployment.id;
-        }
-        Execution::RestartDeployment(params) => {
-          let deployment =
-            super::get_check_permissions::<Deployment>(
-              &params.deployment,
-              user,
-              PermissionLevel::Execute.into(),
-            )
-            .await?;
-          params.deployment = deployment.id;
-        }
-        Execution::PauseDeployment(params) => {
-          let deployment =
-            super::get_check_permissions::<Deployment>(
-              &params.deployment,
-              user,
-              PermissionLevel::Execute.into(),
-            )
-            .await?;
-          params.deployment = deployment.id;
-        }
-        Execution::UnpauseDeployment(params) => {
-          let deployment =
-            super::get_check_permissions::<Deployment>(
-              &params.deployment,
-              user,
-              PermissionLevel::Execute.into(),
-            )
-            .await?;
-          params.deployment = deployment.id;
-        }
-        Execution::StopDeployment(params) => {
-          let deployment =
-            super::get_check_permissions::<Deployment>(
-              &params.deployment,
-              user,
-              PermissionLevel::Execute.into(),
-            )
-            .await?;
-          params.deployment = deployment.id;
-        }
-        Execution::DestroyDeployment(params) => {
-          let deployment =
-            super::get_check_permissions::<Deployment>(
-              &params.deployment,
-              user,
-              PermissionLevel::Execute.into(),
-            )
-            .await?;
-          params.deployment = deployment.id;
-        }
-        Execution::BatchDestroyDeployment(_params) => {
-          if !user.admin {
-            return Err(anyhow!(
-              "Non admin user cannot configure Batch executions"
-            ));
-          }
-        }
-        Execution::CloneRepo(params) => {
-          let repo = super::get_check_permissions::<Repo>(
-            &params.repo,
-            user,
-            PermissionLevel::Execute.into(),
-          )
-          .await?;
-          params.repo = repo.id;
-        }
-        Execution::BatchCloneRepo(_params) => {
-          if !user.admin {
-            return Err(anyhow!(
-              "Non admin user cannot configure Batch executions"
-            ));
-          }
-        }
-        Execution::PullRepo(params) => {
-          let repo = super::get_check_permissions::<Repo>(
-            &params.repo,
-            user,
-            PermissionLevel::Execute.into(),
-          )
-          .await?;
-          params.repo = repo.id;
-        }
-        Execution::BatchPullRepo(_params) => {
-          if !user.admin {
-            return Err(anyhow!(
-              "Non admin user cannot configure Batch executions"
-            ));
-          }
-        }
-        Execution::BuildRepo(params) => {
-          let repo = super::get_check_permissions::<Repo>(
-            &params.repo,
-            user,
-            PermissionLevel::Execute.into(),
-          )
-          .await?;
-          params.repo = repo.id;
-        }
-        Execution::BatchBuildRepo(_params) => {
-          if !user.admin {
-            return Err(anyhow!(
-              "Non admin user cannot configure Batch executions"
-            ));
-          }
-        }
-        Execution::CancelRepoBuild(params) => {
-          let repo = super::get_check_permissions::<Repo>(
-            &params.repo,
-            user,
-            PermissionLevel::Execute.into(),
-          )
-          .await?;
-          params.repo = repo.id;
-        }
-        Execution::StartContainer(params) => {
-          let server = super::get_check_permissions::<Server>(
-            &params.server,
-            user,
-            PermissionLevel::Execute.into(),
-          )
-          .await?;
-          params.server = server.id;
-        }
-        Execution::RestartContainer(params) => {
-          let server = super::get_check_permissions::<Server>(
-            &params.server,
-            user,
-            PermissionLevel::Execute.into(),
-          )
-          .await?;
-          params.server = server.id;
-        }
-        Execution::PauseContainer(params) => {
-          let server = super::get_check_permissions::<Server>(
-            &params.server,
-            user,
-            PermissionLevel::Execute.into(),
-          )
-          .await?;
-          params.server = server.id;
-        }
-        Execution::UnpauseContainer(params) => {
-          let server = super::get_check_permissions::<Server>(
-            &params.server,
-            user,
-            PermissionLevel::Execute.into(),
-          )
-          .await?;
-          params.server = server.id;
-        }
-        Execution::StopContainer(params) => {
-          let server = super::get_check_permissions::<Server>(
-            &params.server,
-            user,
-            PermissionLevel::Execute.into(),
-          )
-          .await?;
-          params.server = server.id;
-        }
-        Execution::DestroyContainer(params) => {
-          let server = super::get_check_permissions::<Server>(
-            &params.server,
-            user,
-            PermissionLevel::Execute.into(),
-          )
-          .await?;
-          params.server = server.id;
-        }
-        Execution::StartAllContainers(params) => {
-          let server = super::get_check_permissions::<Server>(
-            &params.server,
-            user,
-            PermissionLevel::Execute.into(),
-          )
-          .await?;
-          params.server = server.id;
-        }
-        Execution::RestartAllContainers(params) => {
-          let server = super::get_check_permissions::<Server>(
-            &params.server,
-            user,
-            PermissionLevel::Execute.into(),
-          )
-          .await?;
-          params.server = server.id;
-        }
-        Execution::PauseAllContainers(params) => {
-          let server = super::get_check_permissions::<Server>(
-            &params.server,
-            user,
-            PermissionLevel::Execute.into(),
-          )
-          .await?;
-          params.server = server.id;
-        }
-        Execution::UnpauseAllContainers(params) => {
-          let server = super::get_check_permissions::<Server>(
-            &params.server,
-            user,
-            PermissionLevel::Execute.into(),
-          )
-          .await?;
-          params.server = server.id;
-        }
-        Execution::StopAllContainers(params) => {
-          let server = super::get_check_permissions::<Server>(
-            &params.server,
-            user,
-            PermissionLevel::Execute.into(),
-          )
-          .await?;
-          params.server = server.id;
-        }
-        Execution::PruneContainers(params) => {
-          let server = super::get_check_permissions::<Server>(
-            &params.server,
-            user,
-            PermissionLevel::Execute.into(),
-          )
-          .await?;
-          params.server = server.id;
-        }
-        Execution::DeleteNetwork(params) => {
-          let server = super::get_check_permissions::<Server>(
-            &params.server,
-            user,
-            PermissionLevel::Execute.into(),
-          )
-          .await?;
-          params.server = server.id;
-        }
-        Execution::PruneNetworks(params) => {
-          let server = super::get_check_permissions::<Server>(
-            &params.server,
-            user,
-            PermissionLevel::Execute.into(),
-          )
-          .await?;
-          params.server = server.id;
-        }
-        Execution::DeleteImage(params) => {
-          let server = super::get_check_permissions::<Server>(
-            &params.server,
-            user,
-            PermissionLevel::Execute.into(),
-          )
-          .await?;
-          params.server = server.id;
-        }
-        Execution::PruneImages(params) => {
-          let server = super::get_check_permissions::<Server>(
-            &params.server,
-            user,
-            PermissionLevel::Execute.into(),
-          )
-          .await?;
-          params.server = server.id;
-        }
-        Execution::DeleteVolume(params) => {
-          let server = super::get_check_permissions::<Server>(
-            &params.server,
-            user,
-            PermissionLevel::Execute.into(),
-          )
-          .await?;
-          params.server = server.id;
-        }
-        Execution::PruneVolumes(params) => {
-          let server = super::get_check_permissions::<Server>(
-            &params.server,
-            user,
-            PermissionLevel::Execute.into(),
-          )
-          .await?;
-          params.server = server.id;
-        }
-        Execution::PruneDockerBuilders(params) => {
-          let server = super::get_check_permissions::<Server>(
-            &params.server,
-            user,
-            PermissionLevel::Execute.into(),
-          )
-          .await?;
-          params.server = server.id;
-        }
-        Execution::PruneBuildx(params) => {
-          let server = super::get_check_permissions::<Server>(
-            &params.server,
-            user,
-            PermissionLevel::Execute.into(),
-          )
-          .await?;
-          params.server = server.id;
-        }
-        Execution::PruneSystem(params) => {
-          let server = super::get_check_permissions::<Server>(
-            &params.server,
-            user,
-            PermissionLevel::Execute.into(),
-          )
-          .await?;
-          params.server = server.id;
-        }
-        Execution::RunSync(params) => {
-          let sync = super::get_check_permissions::<ResourceSync>(
-            &params.sync,
-            user,
-            PermissionLevel::Execute.into(),
-          )
-          .await?;
-          params.sync = sync.id;
-        }
-        Execution::CommitSync(params) => {
-          // This one is actually a write operation.
-          let sync = super::get_check_permissions::<ResourceSync>(
-            &params.sync,
-            user,
-            PermissionLevel::Write.into(),
-          )
-          .await?;
-          params.sync = sync.id;
-        }
-        Execution::DeployStack(params) => {
-          let stack = super::get_check_permissions::<Stack>(
-            &params.stack,
-            user,
-            PermissionLevel::Execute.into(),
-          )
-          .await?;
-          params.stack = stack.id;
-        }
-        Execution::BatchDeployStack(_params) => {
-          if !user.admin {
-            return Err(anyhow!(
-              "Non admin user cannot configure Batch executions"
-            ));
-          }
-        }
-        Execution::DeployStackIfChanged(params) => {
-          let stack = super::get_check_permissions::<Stack>(
-            &params.stack,
-            user,
-            PermissionLevel::Execute.into(),
-          )
-          .await?;
-          params.stack = stack.id;
-        }
-        Execution::BatchDeployStackIfChanged(_params) => {
-          if !user.admin {
-            return Err(anyhow!(
-              "Non admin user cannot configure Batch executions"
-            ));
-          }
-        }
-        Execution::PullStack(params) => {
-          let stack = super::get_check_permissions::<Stack>(
-            &params.stack,
-            user,
-            PermissionLevel::Execute.into(),
-          )
-          .await?;
-          params.stack = stack.id;
-        }
-        Execution::BatchPullStack(_params) => {
-          if !user.admin {
-            return Err(anyhow!(
-              "Non admin user cannot configure Batch executions"
-            ));
-          }
-        }
-        Execution::StartStack(params) => {
-          let stack = super::get_check_permissions::<Stack>(
-            &params.stack,
-            user,
-            PermissionLevel::Execute.into(),
-          )
-          .await?;
-          params.stack = stack.id;
-        }
-        Execution::RestartStack(params) => {
-          let stack = super::get_check_permissions::<Stack>(
-            &params.stack,
-            user,
-            PermissionLevel::Execute.into(),
-          )
-          .await?;
-          params.stack = stack.id;
-        }
-        Execution::PauseStack(params) => {
-          let stack = super::get_check_permissions::<Stack>(
-            &params.stack,
-            user,
-            PermissionLevel::Execute.into(),
-          )
-          .await?;
-          params.stack = stack.id;
-        }
-        Execution::UnpauseStack(params) => {
-          let stack = super::get_check_permissions::<Stack>(
-            &params.stack,
-            user,
-            PermissionLevel::Execute.into(),
-          )
-          .await?;
-          params.stack = stack.id;
-        }
-        Execution::StopStack(params) => {
-          let stack = super::get_check_permissions::<Stack>(
-            &params.stack,
-            user,
-            PermissionLevel::Execute.into(),
-          )
-          .await?;
-          params.stack = stack.id;
-        }
-        Execution::DestroyStack(params) => {
-          let stack = super::get_check_permissions::<Stack>(
-            &params.stack,
-            user,
-            PermissionLevel::Execute.into(),
-          )
-          .await?;
-          params.stack = stack.id;
-        }
-        Execution::RunStackService(params) => {
-          let stack = super::get_check_permissions::<Stack>(
-            &params.stack,
-            user,
-            PermissionLevel::Execute.into(),
-          )
-          .await?;
-          params.stack = stack.id;
-        }
-        Execution::BatchDestroyStack(_params) => {
-          if !user.admin {
-            return Err(anyhow!(
-              "Non admin user cannot configure Batch executions"
-            ));
-          }
-        }
-        Execution::TestAlerter(params) => {
-          let alerter = super::get_check_permissions::<Alerter>(
-            &params.alerter,
-            user,
-            PermissionLevel::Execute.into(),
-          )
-          .await?;
-          params.alerter = alerter.id;
-        }
-        Execution::SendAlert(params) => {
-          params.alerters = params
-            .alerters
-            .iter()
-            .map(async |alerter| {
-              let id = super::get_check_permissions::<Alerter>(
-                alerter,
+      macro_rules! check_execution_perms {
+        (
+          execute: [$(($Variant:ident, $Type:ident, $field:ident)),* $(,)?],
+          batch_admin: [$($BatchVariant:ident),* $(,)?],
+          admin_only: [$(($AdminVariant:ident, $msg:literal)),* $(,)?],
+        ) => {
+          match &mut exec.execution {
+            $(
+              Execution::$Variant(params) => {
+                let resource = super::get_check_permissions::<$Type>(
+                  &params.$field,
+                  user,
+                  PermissionLevel::Execute.into(),
+                )
+                .await?;
+                params.$field = resource.id;
+              }
+            )*
+            $(
+              Execution::$BatchVariant(_params) => {
+                if !user.admin {
+                  return Err(anyhow!(
+                    "Non admin user cannot configure Batch executions"
+                  ));
+                }
+              }
+            )*
+            $(
+              Execution::$AdminVariant(_params) => {
+                if !user.admin {
+                  return Err(anyhow!($msg));
+                }
+              }
+            )*
+            // Special: self-referential procedure check
+            Execution::RunProcedure(params) => {
+              let procedure = super::get_check_permissions::<Procedure>(
+                &params.procedure,
                 user,
                 PermissionLevel::Execute.into(),
               )
-              .await?
-              .id;
-              anyhow::Ok(id)
-            })
-            .collect::<FuturesUnordered<_>>()
-            .try_collect::<Vec<_>>()
-            .await?;
-        }
-        Execution::RemoveSwarmNodes(params) => {
-          let swarm = super::get_check_permissions::<Swarm>(
-            &params.swarm,
-            user,
-            PermissionLevel::Execute.into(),
-          )
-          .await?;
-          params.swarm = swarm.id;
-        }
-        Execution::RemoveSwarmStacks(params) => {
-          let swarm = super::get_check_permissions::<Swarm>(
-            &params.swarm,
-            user,
-            PermissionLevel::Execute.into(),
-          )
-          .await?;
-          params.swarm = swarm.id;
-        }
-        Execution::RemoveSwarmServices(params) => {
-          let swarm = super::get_check_permissions::<Swarm>(
-            &params.swarm,
-            user,
-            PermissionLevel::Execute.into(),
-          )
-          .await?;
-          params.swarm = swarm.id;
-        }
-        Execution::CreateSwarmConfig(params) => {
-          let swarm = super::get_check_permissions::<Swarm>(
-            &params.swarm,
-            user,
-            PermissionLevel::Execute.into(),
-          )
-          .await?;
-          params.swarm = swarm.id;
-        }
-        Execution::RotateSwarmConfig(params) => {
-          let swarm = super::get_check_permissions::<Swarm>(
-            &params.swarm,
-            user,
-            PermissionLevel::Execute.into(),
-          )
-          .await?;
-          params.swarm = swarm.id;
-        }
-        Execution::RemoveSwarmConfigs(params) => {
-          let swarm = super::get_check_permissions::<Swarm>(
-            &params.swarm,
-            user,
-            PermissionLevel::Execute.into(),
-          )
-          .await?;
-          params.swarm = swarm.id;
-        }
-        Execution::CreateSwarmSecret(params) => {
-          let swarm = super::get_check_permissions::<Swarm>(
-            &params.swarm,
-            user,
-            PermissionLevel::Execute.into(),
-          )
-          .await?;
-          params.swarm = swarm.id;
-        }
-        Execution::RotateSwarmSecret(params) => {
-          let swarm = super::get_check_permissions::<Swarm>(
-            &params.swarm,
-            user,
-            PermissionLevel::Execute.into(),
-          )
-          .await?;
-          params.swarm = swarm.id;
-        }
-        Execution::RemoveSwarmSecrets(params) => {
-          let swarm = super::get_check_permissions::<Swarm>(
-            &params.swarm,
-            user,
-            PermissionLevel::Execute.into(),
-          )
-          .await?;
-          params.swarm = swarm.id;
-        }
-        Execution::ClearRepoCache(_params) => {
-          if !user.admin {
-            return Err(anyhow!(
-              "Non admin user cannot clear repo cache"
-            ));
+              .await?;
+              match id {
+                Some(id) if procedure.id == id => {
+                  return Err(anyhow!(
+                    "Cannot have self-referential procedure"
+                  ));
+                }
+                _ => {}
+              }
+              params.procedure = procedure.id;
+            }
+            // Special: CommitSync uses Write permission
+            Execution::CommitSync(params) => {
+              let sync = super::get_check_permissions::<ResourceSync>(
+                &params.sync,
+                user,
+                PermissionLevel::Write.into(),
+              )
+              .await?;
+              params.sync = sync.id;
+            }
+            // Special: SendAlert checks a Vec of alerters
+            Execution::SendAlert(params) => {
+              params.alerters = params
+                .alerters
+                .iter()
+                .map(async |alerter| {
+                  let id = super::get_check_permissions::<Alerter>(
+                    alerter,
+                    user,
+                    PermissionLevel::Execute.into(),
+                  )
+                  .await?
+                  .id;
+                  anyhow::Ok(id)
+                })
+                .collect::<FuturesUnordered<_>>()
+                .try_collect::<Vec<_>>()
+                .await?;
+            }
+            Execution::None(_) | Execution::Sleep(_) => {}
           }
-        }
-        Execution::BackupCoreDatabase(_params) => {
-          if !user.admin {
-            return Err(anyhow!(
-              "Non admin user cannot trigger core database backup"
-            ));
-          }
-        }
-        Execution::GlobalAutoUpdate(_params) => {
-          if !user.admin {
-            return Err(anyhow!(
-              "Non admin user cannot trigger global auto update"
-            ));
-          }
-        }
-        Execution::RotateAllServerKeys(_params) => {
-          if !user.admin {
-            return Err(anyhow!(
-              "Non admin user cannot trigger rotate all server keys"
-            ));
-          }
-        }
-        Execution::RotateCoreKeys(_params) => {
-          if !user.admin {
-            return Err(anyhow!(
-              "Non admin user cannot trigger rotate core keys"
-            ));
-          }
-        }
-        Execution::Sleep(_) => {}
+        };
       }
+      check_execution_perms!(
+        execute: [
+          // Action
+          (RunAction, Action, action),
+          (CancelAction, Action, action),
+          // Procedure
+          (CancelProcedure, Procedure, procedure),
+          // Build
+          (RunBuild, Build, build),
+          (CancelBuild, Build, build),
+          // Deployment
+          (Deploy, Deployment, deployment),
+          (PullDeployment, Deployment, deployment),
+          (StartDeployment, Deployment, deployment),
+          (RestartDeployment, Deployment, deployment),
+          (PauseDeployment, Deployment, deployment),
+          (UnpauseDeployment, Deployment, deployment),
+          (StopDeployment, Deployment, deployment),
+          (DestroyDeployment, Deployment, deployment),
+          // Repo
+          (CloneRepo, Repo, repo),
+          (PullRepo, Repo, repo),
+          (BuildRepo, Repo, repo),
+          (CancelRepoBuild, Repo, repo),
+          // Server
+          (StartContainer, Server, server),
+          (RestartContainer, Server, server),
+          (PauseContainer, Server, server),
+          (UnpauseContainer, Server, server),
+          (StopContainer, Server, server),
+          (DestroyContainer, Server, server),
+          (StartAllContainers, Server, server),
+          (RestartAllContainers, Server, server),
+          (PauseAllContainers, Server, server),
+          (UnpauseAllContainers, Server, server),
+          (StopAllContainers, Server, server),
+          (PruneContainers, Server, server),
+          (DeleteNetwork, Server, server),
+          (PruneNetworks, Server, server),
+          (DeleteImage, Server, server),
+          (PruneImages, Server, server),
+          (DeleteVolume, Server, server),
+          (PruneVolumes, Server, server),
+          (PruneDockerBuilders, Server, server),
+          (PruneBuildx, Server, server),
+          (PruneSystem, Server, server),
+          // Resource Sync
+          (RunSync, ResourceSync, sync),
+          // Stack
+          (DeployStack, Stack, stack),
+          (DeployStackIfChanged, Stack, stack),
+          (PullStack, Stack, stack),
+          (StartStack, Stack, stack),
+          (RestartStack, Stack, stack),
+          (PauseStack, Stack, stack),
+          (UnpauseStack, Stack, stack),
+          (StopStack, Stack, stack),
+          (DestroyStack, Stack, stack),
+          (RunStackService, Stack, stack),
+          // Alerter
+          (TestAlerter, Alerter, alerter),
+          // Swarm
+          (RemoveSwarmNodes, Swarm, swarm),
+          (UpdateSwarmNode, Swarm, swarm),
+          (RemoveSwarmStacks, Swarm, swarm),
+          (RemoveSwarmServices, Swarm, swarm),
+          (CreateSwarmConfig, Swarm, swarm),
+          (RotateSwarmConfig, Swarm, swarm),
+          (RemoveSwarmConfigs, Swarm, swarm),
+          (CreateSwarmSecret, Swarm, swarm),
+          (RotateSwarmSecret, Swarm, swarm),
+          (RemoveSwarmSecrets, Swarm, swarm),
+        ],
+        batch_admin: [
+          BatchRunProcedure,
+          BatchRunAction,
+          BatchRunBuild,
+          BatchDeploy,
+          BatchDestroyDeployment,
+          BatchCloneRepo,
+          BatchPullRepo,
+          BatchBuildRepo,
+          BatchDeployStack,
+          BatchDeployStackIfChanged,
+          BatchPullStack,
+          BatchDestroyStack,
+        ],
+        admin_only: [
+          (ClearRepoCache, "Non admin user cannot clear repo cache"),
+          (BackupCoreDatabase, "Non admin user cannot trigger core database backup"),
+          (GlobalAutoUpdate, "Non admin user cannot trigger global auto update"),
+          (RotateAllServerKeys, "Non admin user cannot trigger rotate all server keys"),
+          (RotateCoreKeys, "Non admin user cannot trigger rotate core keys"),
+        ],
+      );
     }
   }
 

bin/core/src/resource/repo.rs

@@ -26,7 +26,8 @@ use crate::{
   config::core_config,
   helpers::{periphery_client, repo_link},
   state::{
-    action_states, db_client, repo_state_cache, repo_status_cache,
+    action_states, all_resources_cache, db_client, repo_state_cache,
+    repo_status_cache,
   },
 };
 
@@ -63,6 +64,12 @@ impl super::KomodoResource for Repo {
     let state = get_repo_state(&repo.id).await;
     let status =
       repo_status_cache().get(&repo.id).await.unwrap_or_default();
+    let server_name = all_resources_cache()
+      .load()
+      .servers
+      .get(&repo.config.server_id)
+      .map(|server| server.name.clone())
+      .unwrap_or_default();
     RepoListItem {
       name: repo.name,
       id: repo.id,
@@ -71,6 +78,7 @@ impl super::KomodoResource for Repo {
       resource_type: ResourceTargetVariant::Repo,
       info: RepoListItemInfo {
         server_id: repo.config.server_id,
+        server_name,
         builder_id: repo.config.builder_id,
         last_pulled_at: repo.info.last_pulled_at,
         last_built_at: repo.info.last_built_at,
@@ -263,7 +271,9 @@ async fn validate_config(
   Ok(())
 }
 
-async fn get_repo_state(id: &String) -> RepoState {
+/// The Repo state as computed for the repo list items,
+/// from the in memory action states / state cache.
+pub async fn get_repo_state(id: &String) -> RepoState {
   if let Some(state) = action_states()
     .repo
     .get(id)

bin/core/src/resource/server.rs

@@ -68,14 +68,23 @@ impl super::KomodoResource for Server {
     server: Resource<Self::Config, Self::Info>,
   ) -> Self::ListItem {
     let status = server_status_cache().get(&server.id).await;
+    let (periphery_info, system_info, system_stats) = status
+      .as_ref()
+      .map(|s| {
+        (
+          s.periphery_info.as_ref(),
+          s.system_info.as_ref(),
+          s.system_stats.as_ref(),
+        )
+      })
+      .unwrap_or_default();
     let (
       version,
       public_key,
       public_ip,
       terminals_disabled,
       container_terminals_disabled,
-    ) = match status.as_ref().and_then(|s| s.periphery_info.as_ref())
-    {
+    ) = match periphery_info {
       Some(info) => (
         Some(info.version.clone()),
         Some(info.public_key.clone()),
@@ -93,6 +102,12 @@ impl super::KomodoResource for Server {
       resource_type: ResourceTargetVariant::Server,
       info: ServerListItemInfo {
         state: status.as_ref().map(|s| s.state).unwrap_or_default(),
+        err: status.as_ref().and_then(|s| s.err.clone()),
+        stats: system_stats.map(Into::into),
+        alerting_thresholds: (&server.config).into(),
+        core_count: system_info.and_then(|i| i.core_count),
+        logical_core_count: system_info
+          .and_then(|i| i.logical_core_count),
         region: server.config.region,
         address: optional_string(server.config.address),
         external_address: optional_string(

bin/core/src/resource/stack.rs

@@ -31,7 +31,9 @@ use crate::{
   config::core_config,
   helpers::{
     periphery_client,
-    query::{get_stack_state, get_swarm_or_server},
+    query::{
+      get_cached_stack_state, get_stack_state, get_swarm_or_server,
+    },
     repo_link,
     swarm::swarm_request,
   },
@@ -95,20 +97,7 @@ impl super::KomodoResource for Stack {
     stack: Resource<Self::Config, Self::Info>,
   ) -> Self::ListItem {
     let status = stack_status_cache().get(&stack.id).await;
-    let state = if action_states()
-      .stack
-      .get(&stack.id)
-      .await
-      .map(|s| s.get().map(|s| s.deploying))
-      .transpose()
-      .ok()
-      .flatten()
-      .unwrap_or_default()
-    {
-      StackState::Deploying
-    } else {
-      status.as_ref().map(|s| s.curr.state).unwrap_or_default()
-    };
+    let state = get_cached_stack_state(&stack.id).await;
     let project_name = stack.project_name(false);
     let services = status
       .as_ref()
@@ -117,33 +106,38 @@ impl super::KomodoResource for Stack {
           .services
           .iter()
           .map(|current_service| {
+            let latest_service = stack
+              .info
+              .latest_services
+              .iter()
+              .find(|latest_service| {
+                current_service.service == latest_service.service_name
+              });
+            let latest_image = if let Some(latest_image) =
+              latest_service.as_ref().map(|s| &s.image)
+              && latest_image != &current_service.image
+            {
+              Some(latest_image.to_string())
+            } else {
+              None
+            };
             let update_available = current_service
               .image_digests
               .as_ref()
-              .map(|current_digests| {
-                stack
-                  .info
-                  .latest_services
-                  .iter()
-                  .find_map(|latest_service| {
-                    if current_service.service
-                      == latest_service.service_name
-                    {
-                      latest_service
-                        .image_digest
-                        .as_ref()?
-                        .update_available(current_digests)
-                        .into()
-                    } else {
-                      None
-                    }
-                  })
-                  .unwrap_or_default()
+              .and_then(|current_digests| {
+                latest_service.as_ref().and_then(|latest_service| {
+                  latest_service
+                    .image_digest
+                    .as_ref()?
+                    .update_available(current_digests)
+                    .into()
+                })
               })
               .unwrap_or_default();
             StackServiceWithUpdate {
               service: current_service.service.clone(),
               image: current_service.image.clone(),
+              latest_image,
               update_available,
             }
           })
@@ -156,8 +150,9 @@ impl super::KomodoResource for Stack {
       stack.config.repo,
       stack.config.branch,
       stack.config.git_https,
+      String::new(),
     );
-    let (git_provider, repo, branch, git_https) =
+    let (git_provider, repo, branch, git_https, linked_repo_name) =
       if stack.config.linked_repo.is_empty() {
         default_git
       } else {
@@ -171,6 +166,7 @@ impl super::KomodoResource for Stack {
               r.config.repo.clone(),
               r.config.branch.clone(),
               r.config.git_https,
+              r.name.clone(),
             )
           })
           .unwrap_or(default_git)
@@ -205,6 +201,18 @@ impl super::KomodoResource for Stack {
         (false, None)
       };
 
+    let all = all_resources_cache().load();
+    let server_name = all
+      .servers
+      .get(&stack.config.server_id)
+      .map(|server| server.name.clone())
+      .unwrap_or_default();
+    let swarm_name = all
+      .swarms
+      .get(&stack.config.swarm_id)
+      .map(|swarm| swarm.name.clone())
+      .unwrap_or_default();
+
     StackListItem {
       name: stack.name,
       id: stack.id,
@@ -218,8 +226,11 @@ impl super::KomodoResource for Stack {
         project_missing,
         file_contents: !stack.config.file_contents.is_empty(),
         swarm_id: stack.config.swarm_id,
+        swarm_name,
         server_id: stack.config.server_id,
+        server_name,
         linked_repo: stack.config.linked_repo,
+        linked_repo_name,
         missing_files: stack.info.missing_files,
         files_on_host: stack.config.files_on_host,
         repo_link: repo_link(
@@ -233,6 +244,9 @@ impl super::KomodoResource for Stack {
         branch,
         latest_hash: stack.info.latest_hash,
         deployed_hash: stack.info.deployed_hash,
+        auto_update_all_services: stack
+          .config
+          .auto_update_all_services,
       },
     }
   }

bin/core/src/resource/sync.rs

@@ -57,8 +57,9 @@ impl super::KomodoResource for ResourceSync {
       resource_sync.config.repo,
       resource_sync.config.branch,
       resource_sync.config.git_https,
+      String::new(),
     );
-    let (git_provider, repo, branch, git_https) =
+    let (git_provider, repo, branch, git_https, linked_repo_name) =
       if resource_sync.config.linked_repo.is_empty() {
         default_git
       } else {
@@ -72,6 +73,7 @@ impl super::KomodoResource for ResourceSync {
               r.config.repo.clone(),
               r.config.branch.clone(),
               r.config.git_https,
+              r.name.clone(),
             )
           })
           .unwrap_or(default_git)
@@ -88,6 +90,7 @@ impl super::KomodoResource for ResourceSync {
         files_on_host: resource_sync.config.files_on_host,
         managed: resource_sync.config.managed,
         linked_repo: resource_sync.config.linked_repo,
+        linked_repo_name,
         repo_link: repo_link(
           &git_provider,
           &repo,

bin/core/src/stack/execute.rs

@@ -80,7 +80,7 @@ pub async fn execute_compose_with_stack_and_server<
 
   // Will check to ensure stack not already busy before updating, and return Err if so.
   // The returned guard will set the action state back to default when dropped.
-  let _action_guard = action_state.update(set_in_progress)?;
+  let action_guard = action_state.update(set_in_progress)?;
 
   // Send update here for UI to recheck action state
   update_update(update.clone()).await?;
@@ -105,6 +105,10 @@ pub async fn execute_compose_with_stack_and_server<
   refresh_server_cache(&server, true).await;
 
   update.finalize();
+
+  // Drop action guard before updating
+  // clients to requery action state
+  drop(action_guard);
   update_update(update.clone()).await?;
 
   Ok(update)

bin/core/src/stack/services.rs

@@ -38,8 +38,6 @@ pub fn extract_services_into_res(
         "failed to parse service names from compose contents",
       )?;
 
-  let mut services = Vec::with_capacity(compose.services.capacity());
-
   for (
     service_name,
     ComposeService {
@@ -49,17 +47,29 @@ pub fn extract_services_into_res(
     },
   ) in compose.services
   {
-    let image = image.unwrap_or_default();
-    services.push(StackServiceNames {
-      container_name: container_name
-        .unwrap_or_else(|| format!("{project_name}-{service_name}")),
-      image_digest: service_image_digests.get(&service_name).cloned(),
-      service_name,
-      image,
-    });
+    if let Some(existing) =
+      res.iter_mut().find(|s| s.service_name == service_name)
+    {
+      // Override any defined fields
+      if let Some(container_name) = container_name {
+        existing.container_name = container_name;
+      }
+      if let Some(image) = image {
+        existing.image = image;
+      }
+    } else {
+      res.push(StackServiceNames {
+        container_name: container_name.unwrap_or_else(|| {
+          format!("{project_name}-{service_name}")
+        }),
+        image_digest: service_image_digests
+          .get(&service_name)
+          .cloned(),
+        image: image.unwrap_or_default(),
+        service_name,
+      });
+    }
   }
 
-  res.extend(services);
-
   Ok(())
 }

bin/core/src/startup.rs

@@ -281,7 +281,7 @@ async fn ensure_first_server_and_builder() {
     name: name.to_string(),
     config: PartialBuilderConfig::Server(
       PartialServerBuilderConfig {
-        server_id: Some(server.id),
+        server_ids: Some(vec![server.id]),
       },
     ),
   })

bin/core/src/state.rs

@@ -19,13 +19,14 @@ use komodo_client::entities::{
   swarm::SwarmState,
 };
 use mogh_cache::CloneCache;
+use tokio_util::sync::CancellationToken;
 
 use crate::{
   config::core_config,
   connection::PeripheryConnections,
   helpers::{
     action_state::ActionStates, all_resources::AllResourcesById,
-    image_digest::ImageDigestCache,
+    builder::BuilderUsage, image_digest::ImageDigestCache,
   },
 };
 
@@ -87,7 +88,7 @@ pub struct CachedSwarmStatus {
   pub inspect: Option<SwarmInspectInfo>,
   pub lists: Option<SwarmLists>,
   /// Store the error in communicating with Swarm
-  pub err: Option<String>,
+  pub err: Option<mogh_error::Serror>,
 }
 
 pub type SwarmStatusCache =
@@ -208,6 +209,7 @@ pub fn action_state_cache() -> &'static ActionStateCache {
   ACTION_STATE_CACHE.get_or_init(Default::default)
 }
 
+/// Store all resources in local cache for fast lookup
 pub fn all_resources_cache() -> &'static ArcSwap<AllResourcesById> {
   static ALL_RESOURCES: OnceLock<ArcSwap<AllResourcesById>> =
     OnceLock::new();
@@ -222,3 +224,30 @@ pub fn image_digest_cache() -> &'static ImageDigestCache {
     OnceLock::new();
   IMAGE_DIGEST_CACHE.get_or_init(ImageDigestCache::new)
 }
+
+/// Maps Builder id => downstream count map (eg server id => active count)
+type BuilderUsageCache = CloneCache<String, Arc<BuilderUsage>>;
+
+/// For builders with multiple downstream machines to choose.
+/// Stores active build count for each downstream.
+/// Maps Builder id => downstream count map
+pub fn builder_usage_cache() -> &'static BuilderUsageCache {
+  static BUILDER_USAGE_CACHE: OnceLock<BuilderUsageCache> =
+    OnceLock::new();
+  BUILDER_USAGE_CACHE.get_or_init(Default::default)
+}
+
+type CancelCache = CloneCache<String, CancellationToken>;
+
+/// Maps procedure id => CancellationToken
+pub fn procedure_cancel_cache() -> &'static CancelCache {
+  static PROCEDURE_CANCEL_CACHE: OnceLock<CancelCache> =
+    OnceLock::new();
+  PROCEDURE_CANCEL_CACHE.get_or_init(Default::default)
+}
+
+/// Maps update id => CancellationToken
+pub fn action_cancel_cache() -> &'static CancelCache {
+  static ACTION_CANCEL_CACHE: OnceLock<CancelCache> = OnceLock::new();
+  ACTION_CANCEL_CACHE.get_or_init(Default::default)
+}

bin/core/src/sync/deploy.rs

@@ -338,9 +338,7 @@ fn build_cache_for_deployment<'a>(
         // Makes sure things that aren't defined in toml (come through as None) actually get removed.
         let config: DeploymentConfig =
           deployment.config.clone().into();
-        let mut config: PartialDeploymentConfig = config.into();
-
-        Deployment::validate_partial_config(&mut config);
+        let config: PartialDeploymentConfig = config.into();
 
         let mut diff =
           Deployment::get_diff(original.config.clone(), config)?;
@@ -572,9 +570,7 @@ fn build_cache_for_stack<'a>(
         // Merge toml resource config (partial) onto default resource config.
         // Makes sure things that aren't defined in toml (come through as None) actually get removed.
         let config: StackConfig = stack.config.clone().into();
-        let mut config: PartialStackConfig = config.into();
-
-        Stack::validate_partial_config(&mut config);
+        let config: PartialStackConfig = config.into();
 
         let mut diff =
           Stack::get_diff(original.config.clone(), config)?;