Initial plan

* start 1.19.5

* deploy 1.19.5-dev-1

* avoid execute_and_poll error when update is already complete or has no id

* improve image tagging customization

* 1.19.5 release

Cargo.toml

@@ -8,13 +8,16 @@ members = [
 ]
 
 [workspace.package]
-version = "1.19.0"
+version = "1.19.5"
 edition = "2024"
 authors = ["mbecker20 <becker.maxh@gmail.com>"]
 license = "GPL-3.0-or-later"
 repository = "https://github.com/moghtech/komodo"
 homepage = "https://komo.do"
 
+[profile.release]
+strip = "debuginfo"
+
 [workspace.dependencies]
 # LOCAL
 komodo_client = { path = "client/core/rs" }
@@ -33,7 +36,7 @@ git = { path = "lib/git" }
 
 # MOGH
 run_command = { version = "0.0.6", features = ["async_tokio"] }
-serror = { version = "0.5.0", default-features = false }
+serror = { version = "0.5.1", default-features = false }
 slack = { version = "0.4.0", package = "slack_client_rs", default-features = false, features = ["rustls"] }
 derive_default_builder = "0.1.8"
 derive_empty_traits = "0.1.0"
@@ -43,11 +46,11 @@ derive_variants = "1.0.0"
 mongo_indexed = "2.0.2"
 resolver_api = "3.0.0"
 toml_pretty = "1.2.0"
-mungos = "3.2.1"
+mungos = "3.2.2"
 svi = "1.2.0"
 
 # ASYNC
-reqwest = { version = "0.12.22", default-features = false, features = ["json", "stream", "rustls-tls-native-roots"] }
+reqwest = { version = "0.12.23", default-features = false, features = ["json", "stream", "rustls-tls-native-roots"] }
 tokio = { version = "1.47.1", features = ["full"] }
 tokio-util = { version = "0.7.16", features = ["io", "codec"] }
 tokio-stream = { version = "0.1.17", features = ["sync"] }
@@ -59,45 +62,46 @@ arc-swap = "1.7.1"
 # SERVER
 tokio-tungstenite = { version = "0.27.0", features = ["rustls-tls-native-roots"] }
 axum-extra = { version = "0.10.1", features = ["typed-header"] }
-tower-http = { version = "0.6.4", features = ["fs", "cors"] }
+tower-http = { version = "0.6.6", features = ["fs", "cors"] }
 axum-server = { version = "0.7.2", features = ["tls-rustls"] }
 axum = { version = "0.8.4", features = ["ws", "json", "macros"] }
 
 # SER/DE
 ipnetwork = { version = "0.21.1", features = ["serde"] }
-indexmap = { version = "2.10.0", features = ["serde"] }
+indexmap = { version = "2.11.1", features = ["serde"] }
 serde = { version = "1.0.219", features = ["derive"] }
 strum = { version = "0.27.2", features = ["derive"] }
+bson = { version = "2.15.0" } # must keep in sync with mongodb version
 serde_yaml_ng = "0.10.0"
-serde_json = "1.0.142"
+serde_json = "1.0.145"
 serde_qs = "0.15.0"
 toml = "0.9.5"
 
 # ERROR
 anyhow = "1.0.99"
-thiserror = "2.0.14"
+thiserror = "2.0.16"
 
 # LOGGING
 opentelemetry-otlp = { version = "0.30.0", features = ["tls-roots", "reqwest-rustls"] }
 opentelemetry_sdk = { version = "0.30.0", features = ["rt-tokio"] }
-tracing-subscriber = { version = "0.3.19", features = ["json"] }
+tracing-subscriber = { version = "0.3.20", features = ["json"] }
 opentelemetry-semantic-conventions = "0.30.0"
 tracing-opentelemetry = "0.31.0"
 opentelemetry = "0.30.0"
 tracing = "0.1.41"
 
 # CONFIG
-clap = { version = "4.5.43", features = ["derive"] }
+clap = { version = "4.5.47", features = ["derive"] }
 dotenvy = "0.15.7"
 envy = "0.4.2"
 
 # CRYPTO / AUTH
-uuid = { version = "1.17.0", features = ["v4", "fast-rng", "serde"] }
+uuid = { version = "1.18.1", features = ["v4", "fast-rng", "serde"] }
 jsonwebtoken = { version = "9.3.1", default-features = false }
 openidconnect = "4.0.1"
 urlencoding = "2.1.3"
 nom_pem = "4.0.0"
-bcrypt = "0.17.0"
+bcrypt = "0.17.1"
 base64 = "0.22.1"
 rustls = "0.23.31"
 hmac = "0.12.1"
@@ -111,25 +115,25 @@ bollard = "0.19.2"
 sysinfo = "0.37.0"
 
 # CLOUD
-aws-config = "1.8.5"
-aws-sdk-ec2 = "1.159.0"
-aws-credential-types = "1.2.5"
+aws-config = "1.8.6"
+aws-sdk-ec2 = "1.167.0"
+aws-credential-types = "1.2.6"
 
 ## CRON
 english-to-cron = "0.1.6"
 chrono-tz = "0.10.4"
-chrono = "0.4.41"
+chrono = "0.4.42"
 croner = "3.0.0"
 
 # MISC
-async-compression = { version = "0.4.27", features = ["tokio", "gzip"] }
+async-compression = { version = "0.4.30", features = ["tokio", "gzip"] }
 derive_builder = "0.20.2"
-comfy-table = "7.1.4"
+comfy-table = "7.2.1"
 typeshare = "1.0.4"
 octorust = "0.10.0"
 dashmap = "6.1.0"
 wildcard = "0.3.0"
 colored = "3.0.0"
-regex = "1.11.1"
+regex = "1.11.2"
 bytes = "1.10.1"
-bson = "2.15.0"
+shell-escape = "0.1.5"

bin/binaries.Dockerfile

@@ -2,6 +2,7 @@
 ## for a specific architecture.
 
 FROM rust:1.89.0-bullseye AS builder
+RUN cargo install cargo-strip
 
 WORKDIR /builder
 COPY Cargo.toml Cargo.lock ./
@@ -16,7 +17,8 @@ COPY ./bin/cli ./bin/cli
 RUN \
   cargo build -p komodo_core --release && \
   cargo build -p komodo_periphery --release && \
-  cargo build -p komodo_cli --release
+  cargo build -p komodo_cli --release && \
+  cargo strip
 
 # Copy just the binaries to scratch image
 FROM scratch

bin/chef.binaries.Dockerfile

@@ -12,6 +12,7 @@ COPY . .
 RUN cargo chef prepare --recipe-path recipe.json
 
 FROM chef AS builder
+RUN cargo install cargo-strip
 COPY --from=planner /builder/recipe.json recipe.json
 # Build JUST dependencies - cached layer
 RUN cargo chef cook --release --recipe-path recipe.json
@@ -20,7 +21,8 @@ COPY . .
 RUN \
   cargo build --release --bin core && \
   cargo build --release --bin periphery && \
-  cargo build --release --bin km
+  cargo build --release --bin km && \
+  cargo strip
 
 # Copy just the binaries to scratch image
 FROM scratch

bin/cli/aio.Dockerfile

@@ -1,4 +1,5 @@
 FROM rust:1.89.0-bullseye AS builder
+RUN cargo install cargo-strip
 
 WORKDIR /builder
 COPY Cargo.toml Cargo.lock ./
@@ -8,7 +9,7 @@ COPY ./client/periphery ./client/periphery
 COPY ./bin/cli ./bin/cli
 
 # Compile bin
-RUN cargo build -p komodo_cli --release
+RUN cargo build -p komodo_cli --release && cargo strip
 
 # Copy binaries to distroless base
 FROM gcr.io/distroless/cc

bin/cli/src/command/container.rs

@@ -292,7 +292,7 @@ impl PrintTable for (Option<&'_ str>, ContainerListItem) {
       Cell::new(self.0.unwrap_or("Unknown")),
       ports,
       Cell::new(networks.join(", ")),
-      Cell::new(clamp_sha(&image)),
+      Cell::new(clamp_sha(image)),
     ];
     if !links {
       return res;

bin/cli/src/command/execute.rs

@@ -212,9 +212,15 @@ pub async fn handle(
     Execution::BatchDestroyStack(data) => {
       println!("{}: {data:?}", "Data".dimmed())
     }
+    Execution::RunStackService(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
     Execution::TestAlerter(data) => {
       println!("{}: {data:?}", "Data".dimmed())
     }
+    Execution::SendAlert(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
     Execution::ClearRepoCache(data) => {
       println!("{}: {data:?}", "Data".dimmed())
     }
@@ -464,10 +470,18 @@ pub async fn handle(
     Execution::BatchDestroyStack(request) => {
       client.execute(request).await.map(ExecutionResult::Batch)
     }
+    Execution::RunStackService(request) => client
+      .execute(request)
+      .await
+      .map(|u| ExecutionResult::Single(u.into())),
     Execution::TestAlerter(request) => client
       .execute(request)
       .await
       .map(|u| ExecutionResult::Single(u.into())),
+    Execution::SendAlert(request) => client
+      .execute(request)
+      .await
+      .map(|u| ExecutionResult::Single(u.into())),
     Execution::ClearRepoCache(request) => client
       .execute(request)
       .await
@@ -535,20 +549,20 @@ async fn poll_update_until_complete(
   } else {
     format!("{}/updates/{}", cli_config().host, update.id)
   };
-  info!("Link: '{}'", link.bold());
+  println!("Link: '{}'", link.bold());
 
   let client = super::komodo_client().await?;
 
   let timer = tokio::time::Instant::now();
   let update = client.poll_update_until_complete(&update.id).await?;
   if update.success {
-    info!(
+    println!(
       "FINISHED in {}: {}",
       format!("{:.1?}", timer.elapsed()).bold(),
       "EXECUTION SUCCESSFUL".green(),
     );
   } else {
-    warn!(
+    eprintln!(
       "FINISHED in {}: {}",
       format!("{:.1?}", timer.elapsed()).bold(),
       "EXECUTION FAILED".red(),

bin/cli/src/command/list.rs

@@ -233,7 +233,7 @@ async fn list_schedules(
 }
 
 fn fix_tags<T>(
-  resources: &mut Vec<ResourceListItem<T>>,
+  resources: &mut [ResourceListItem<T>],
   tags: &HashMap<String, String>,
 ) {
   resources.iter_mut().for_each(|resource| {

bin/cli/src/command/mod.rs

@@ -111,7 +111,7 @@ fn print_items<T: PrintTable + Serialize>(
       };
       table.load_preset(preset).set_header(
         T::header(links)
-          .into_iter()
+          .iter()
           .map(|h| Cell::new(h).add_attribute(Attribute::Bold)),
       );
       for item in items {

bin/core/aio.Dockerfile

@@ -2,6 +2,7 @@
 
 # Build Core
 FROM rust:1.89.0-bullseye AS core-builder
+RUN cargo install cargo-strip
 
 WORKDIR /builder
 COPY Cargo.toml Cargo.lock ./
@@ -13,7 +14,8 @@ COPY ./bin/cli ./bin/cli
 
 # Compile app
 RUN cargo build -p komodo_core --release && \
-  cargo build -p komodo_cli --release
+  cargo build -p komodo_cli --release && \
+  cargo strip
 
 # Build Frontend
 FROM node:20.12-alpine AS frontend-builder

bin/core/src/alert/discord.rs

@@ -17,6 +17,28 @@ pub async fn send_alert(
         "{level} | If you see this message, then Alerter **{name}** is **working**\n{link}"
       )
     }
+    AlertData::ServerVersionMismatch {
+      id,
+      name,
+      region,
+      server_version,
+      core_version,
+    } => {
+      let region = fmt_region(region);
+      let link = resource_link(ResourceTargetVariant::Server, id);
+      match alert.level {
+        SeverityLevel::Ok => {
+          format!(
+            "{level} | **{name}**{region} | Periphery version now matches Core version ✅\n{link}"
+          )
+        }
+        _ => {
+          format!(
+            "{level} | **{name}**{region} | Version mismatch detected ⚠️\nPeriphery: **{server_version}** | Core: **{core_version}**\n{link}"
+          )
+        }
+      }
+    }
     AlertData::ServerUnreachable {
       id,
       name,
@@ -207,6 +229,16 @@ pub async fn send_alert(
         "{level} | **{name}** ({resource_type}) | Scheduled run started 🕝\n{link}"
       )
     }
+    AlertData::Custom { message, details } => {
+      format!(
+        "{level} | {message}{}",
+        if details.is_empty() {
+          format_args!("")
+        } else {
+          format_args!("\n{details}")
+        }
+      )
+    }
     AlertData::None {} => Default::default(),
   };
   if !content.is_empty() {

bin/core/src/alert/mod.rs

@@ -48,8 +48,9 @@ pub async fn send_alerts(alerts: &[Alert]) {
       return;
     };
 
-    let handles =
-      alerts.iter().map(|alert| send_alert(&alerters, alert));
+    let handles = alerts
+      .iter()
+      .map(|alert| send_alert_to_alerters(&alerters, alert));
 
     join_all(handles).await;
   }
@@ -58,7 +59,7 @@ pub async fn send_alerts(alerts: &[Alert]) {
 }
 
 #[instrument(level = "debug")]
-async fn send_alert(alerters: &[Alerter], alert: &Alert) {
+async fn send_alert_to_alerters(alerters: &[Alerter], alert: &Alert) {
   if alerters.is_empty() {
     return;
   }
@@ -250,3 +251,238 @@ fn resource_link(
     id,
   )
 }
+
+/// Standard message content format
+/// used by Ntfy, Pushover.
+fn standard_alert_content(alert: &Alert) -> String {
+  let level = fmt_level(alert.level);
+  match &alert.data {
+    AlertData::Test { id, name } => {
+      let link = resource_link(ResourceTargetVariant::Alerter, id);
+      format!(
+        "{level} | If you see this message, then Alerter {name} is working\n{link}",
+      )
+    }
+    AlertData::ServerVersionMismatch {
+      id,
+      name,
+      region,
+      server_version,
+      core_version,
+    } => {
+      let region = fmt_region(region);
+      let link = resource_link(ResourceTargetVariant::Server, id);
+      match alert.level {
+        SeverityLevel::Ok => {
+          format!(
+            "{level} | {name}{region} | Periphery version now matches Core version ✅\n{link}"
+          )
+        }
+        _ => {
+          format!(
+            "{level} | {name}{region} | Version mismatch detected ⚠️\nPeriphery: {server_version} | Core: {core_version}\n{link}"
+          )
+        }
+      }
+    }
+    AlertData::ServerUnreachable {
+      id,
+      name,
+      region,
+      err,
+    } => {
+      let region = fmt_region(region);
+      let link = resource_link(ResourceTargetVariant::Server, id);
+      match alert.level {
+        SeverityLevel::Ok => {
+          format!("{level} | {name}{region} is now reachable\n{link}")
+        }
+        SeverityLevel::Critical => {
+          let err = err
+            .as_ref()
+            .map(|e| format!("\nerror: {e:#?}"))
+            .unwrap_or_default();
+          format!(
+            "{level} | {name}{region} is unreachable ❌\n{link}{err}"
+          )
+        }
+        _ => unreachable!(),
+      }
+    }
+    AlertData::ServerCpu {
+      id,
+      name,
+      region,
+      percentage,
+    } => {
+      let region = fmt_region(region);
+      let link = resource_link(ResourceTargetVariant::Server, id);
+      format!(
+        "{level} | {name}{region} cpu usage at {percentage:.1}%\n{link}",
+      )
+    }
+    AlertData::ServerMem {
+      id,
+      name,
+      region,
+      used_gb,
+      total_gb,
+    } => {
+      let region = fmt_region(region);
+      let link = resource_link(ResourceTargetVariant::Server, id);
+      let percentage = 100.0 * used_gb / total_gb;
+      format!(
+        "{level} | {name}{region} memory usage at {percentage:.1}%💾\n\nUsing {used_gb:.1} GiB / {total_gb:.1} GiB\n{link}",
+      )
+    }
+    AlertData::ServerDisk {
+      id,
+      name,
+      region,
+      path,
+      used_gb,
+      total_gb,
+    } => {
+      let region = fmt_region(region);
+      let link = resource_link(ResourceTargetVariant::Server, id);
+      let percentage = 100.0 * used_gb / total_gb;
+      format!(
+        "{level} | {name}{region} disk usage at {percentage:.1}%💿\nmount point: {path:?}\nusing {used_gb:.1} GiB / {total_gb:.1} GiB\n{link}",
+      )
+    }
+    AlertData::ContainerStateChange {
+      id,
+      name,
+      server_id: _server_id,
+      server_name,
+      from,
+      to,
+    } => {
+      let link = resource_link(ResourceTargetVariant::Deployment, id);
+      let to_state = fmt_docker_container_state(to);
+      format!(
+        "📦Deployment {name} is now {to_state}\nserver: {server_name}\nprevious: {from}\n{link}",
+      )
+    }
+    AlertData::DeploymentImageUpdateAvailable {
+      id,
+      name,
+      server_id: _server_id,
+      server_name,
+      image,
+    } => {
+      let link = resource_link(ResourceTargetVariant::Deployment, id);
+      format!(
+        "⬆ Deployment {name} has an update available\nserver: {server_name}\nimage: {image}\n{link}",
+      )
+    }
+    AlertData::DeploymentAutoUpdated {
+      id,
+      name,
+      server_id: _server_id,
+      server_name,
+      image,
+    } => {
+      let link = resource_link(ResourceTargetVariant::Deployment, id);
+      format!(
+        "⬆ Deployment {name} was updated automatically\nserver: {server_name}\nimage: {image}\n{link}",
+      )
+    }
+    AlertData::StackStateChange {
+      id,
+      name,
+      server_id: _server_id,
+      server_name,
+      from,
+      to,
+    } => {
+      let link = resource_link(ResourceTargetVariant::Stack, id);
+      let to_state = fmt_stack_state(to);
+      format!(
+        "🥞 Stack {name} is now {to_state}\nserver: {server_name}\nprevious: {from}\n{link}",
+      )
+    }
+    AlertData::StackImageUpdateAvailable {
+      id,
+      name,
+      server_id: _server_id,
+      server_name,
+      service,
+      image,
+    } => {
+      let link = resource_link(ResourceTargetVariant::Stack, id);
+      format!(
+        "⬆ Stack {name} has an update available\nserver: {server_name}\nservice: {service}\nimage: {image}\n{link}",
+      )
+    }
+    AlertData::StackAutoUpdated {
+      id,
+      name,
+      server_id: _server_id,
+      server_name,
+      images,
+    } => {
+      let link = resource_link(ResourceTargetVariant::Stack, id);
+      let images_label =
+        if images.len() > 1 { "images" } else { "image" };
+      let images_str = images.join(", ");
+      format!(
+        "⬆ Stack {name} was updated automatically ⏫\nserver: {server_name}\n{images_label}: {images_str}\n{link}",
+      )
+    }
+    AlertData::AwsBuilderTerminationFailed {
+      instance_id,
+      message,
+    } => {
+      format!(
+        "{level} | Failed to terminate AWS builder instance\ninstance id: {instance_id}\n{message}",
+      )
+    }
+    AlertData::ResourceSyncPendingUpdates { id, name } => {
+      let link =
+        resource_link(ResourceTargetVariant::ResourceSync, id);
+      format!(
+        "{level} | Pending resource sync updates on {name}\n{link}",
+      )
+    }
+    AlertData::BuildFailed { id, name, version } => {
+      let link = resource_link(ResourceTargetVariant::Build, id);
+      format!(
+        "{level} | Build {name} failed\nversion: v{version}\n{link}",
+      )
+    }
+    AlertData::RepoBuildFailed { id, name } => {
+      let link = resource_link(ResourceTargetVariant::Repo, id);
+      format!("{level} | Repo build for {name} failed\n{link}",)
+    }
+    AlertData::ProcedureFailed { id, name } => {
+      let link = resource_link(ResourceTargetVariant::Procedure, id);
+      format!("{level} | Procedure {name} failed\n{link}")
+    }
+    AlertData::ActionFailed { id, name } => {
+      let link = resource_link(ResourceTargetVariant::Action, id);
+      format!("{level} | Action {name} failed\n{link}")
+    }
+    AlertData::ScheduleRun {
+      resource_type,
+      id,
+      name,
+    } => {
+      let link = resource_link(*resource_type, id);
+      format!(
+        "{level} | {name} ({resource_type}) | Scheduled run started 🕝\n{link}"
+      )
+    }
+    AlertData::Custom { message, details } => {
+      format!(
+        "{level} | {message}{}",
+        if details.is_empty() {
+          format_args!("")
+        } else {
+          format_args!("\n{details}")
+        }
+      )
+    }
+    AlertData::None {} => Default::default(),
+  }
+}

bin/core/src/alert/ntfy.rs

@@ -8,205 +8,7 @@ pub async fn send_alert(
   email: Option<&str>,
   alert: &Alert,
 ) -> anyhow::Result<()> {
-  let level = fmt_level(alert.level);
-  let content = match &alert.data {
-    AlertData::Test { id, name } => {
-      let link = resource_link(ResourceTargetVariant::Alerter, id);
-      format!(
-        "{level} | If you see this message, then Alerter {name} is working\n{link}",
-      )
-    }
-    AlertData::ServerUnreachable {
-      id,
-      name,
-      region,
-      err,
-    } => {
-      let region = fmt_region(region);
-      let link = resource_link(ResourceTargetVariant::Server, id);
-      match alert.level {
-        SeverityLevel::Ok => {
-          format!("{level} | {name}{region} is now reachable\n{link}")
-        }
-        SeverityLevel::Critical => {
-          let err = err
-            .as_ref()
-            .map(|e| format!("\nerror: {e:#?}"))
-            .unwrap_or_default();
-          format!(
-            "{level} | {name}{region} is unreachable ❌\n{link}{err}"
-          )
-        }
-        _ => unreachable!(),
-      }
-    }
-    AlertData::ServerCpu {
-      id,
-      name,
-      region,
-      percentage,
-    } => {
-      let region = fmt_region(region);
-      let link = resource_link(ResourceTargetVariant::Server, id);
-      format!(
-        "{level} | {name}{region} cpu usage at {percentage:.1}%\n{link}",
-      )
-    }
-    AlertData::ServerMem {
-      id,
-      name,
-      region,
-      used_gb,
-      total_gb,
-    } => {
-      let region = fmt_region(region);
-      let link = resource_link(ResourceTargetVariant::Server, id);
-      let percentage = 100.0 * used_gb / total_gb;
-      format!(
-        "{level} | {name}{region} memory usage at {percentage:.1}%💾\n\nUsing {used_gb:.1} GiB / {total_gb:.1} GiB\n{link}",
-      )
-    }
-    AlertData::ServerDisk {
-      id,
-      name,
-      region,
-      path,
-      used_gb,
-      total_gb,
-    } => {
-      let region = fmt_region(region);
-      let link = resource_link(ResourceTargetVariant::Server, id);
-      let percentage = 100.0 * used_gb / total_gb;
-      format!(
-        "{level} | {name}{region} disk usage at {percentage:.1}%💿\nmount point: {path:?}\nusing {used_gb:.1} GiB / {total_gb:.1} GiB\n{link}",
-      )
-    }
-    AlertData::ContainerStateChange {
-      id,
-      name,
-      server_id: _server_id,
-      server_name,
-      from,
-      to,
-    } => {
-      let link = resource_link(ResourceTargetVariant::Deployment, id);
-      let to_state = fmt_docker_container_state(to);
-      format!(
-        "📦Deployment {name} is now {to_state}\nserver: {server_name}\nprevious: {from}\n{link}",
-      )
-    }
-    AlertData::DeploymentImageUpdateAvailable {
-      id,
-      name,
-      server_id: _server_id,
-      server_name,
-      image,
-    } => {
-      let link = resource_link(ResourceTargetVariant::Deployment, id);
-      format!(
-        "⬆ Deployment {name} has an update available\nserver: {server_name}\nimage: {image}\n{link}",
-      )
-    }
-    AlertData::DeploymentAutoUpdated {
-      id,
-      name,
-      server_id: _server_id,
-      server_name,
-      image,
-    } => {
-      let link = resource_link(ResourceTargetVariant::Deployment, id);
-      format!(
-        "⬆ Deployment {name} was updated automatically\nserver: {server_name}\nimage: {image}\n{link}",
-      )
-    }
-    AlertData::StackStateChange {
-      id,
-      name,
-      server_id: _server_id,
-      server_name,
-      from,
-      to,
-    } => {
-      let link = resource_link(ResourceTargetVariant::Stack, id);
-      let to_state = fmt_stack_state(to);
-      format!(
-        "🥞 Stack {name} is now {to_state}\nserver: {server_name}\nprevious: {from}\n{link}",
-      )
-    }
-    AlertData::StackImageUpdateAvailable {
-      id,
-      name,
-      server_id: _server_id,
-      server_name,
-      service,
-      image,
-    } => {
-      let link = resource_link(ResourceTargetVariant::Stack, id);
-      format!(
-        "⬆ Stack {name} has an update available\nserver: {server_name}\nservice: {service}\nimage: {image}\n{link}",
-      )
-    }
-    AlertData::StackAutoUpdated {
-      id,
-      name,
-      server_id: _server_id,
-      server_name,
-      images,
-    } => {
-      let link = resource_link(ResourceTargetVariant::Stack, id);
-      let images_label =
-        if images.len() > 1 { "images" } else { "image" };
-      let images_str = images.join(", ");
-      format!(
-        "⬆ Stack {name} was updated automatically ⏫\nserver: {server_name}\n{images_label}: {images_str}\n{link}",
-      )
-    }
-    AlertData::AwsBuilderTerminationFailed {
-      instance_id,
-      message,
-    } => {
-      format!(
-        "{level} | Failed to terminate AWS builder instance\ninstance id: {instance_id}\n{message}",
-      )
-    }
-    AlertData::ResourceSyncPendingUpdates { id, name } => {
-      let link =
-        resource_link(ResourceTargetVariant::ResourceSync, id);
-      format!(
-        "{level} | Pending resource sync updates on {name}\n{link}",
-      )
-    }
-    AlertData::BuildFailed { id, name, version } => {
-      let link = resource_link(ResourceTargetVariant::Build, id);
-      format!(
-        "{level} | Build {name} failed\nversion: v{version}\n{link}",
-      )
-    }
-    AlertData::RepoBuildFailed { id, name } => {
-      let link = resource_link(ResourceTargetVariant::Repo, id);
-      format!("{level} | Repo build for {name} failed\n{link}",)
-    }
-    AlertData::ProcedureFailed { id, name } => {
-      let link = resource_link(ResourceTargetVariant::Procedure, id);
-      format!("{level} | Procedure {name} failed\n{link}")
-    }
-    AlertData::ActionFailed { id, name } => {
-      let link = resource_link(ResourceTargetVariant::Action, id);
-      format!("{level} | Action {name} failed\n{link}")
-    }
-    AlertData::ScheduleRun {
-      resource_type,
-      id,
-      name,
-    } => {
-      let link = resource_link(*resource_type, id);
-      format!(
-        "{level} | {name} ({resource_type}) | Scheduled run started 🕝\n{link}"
-      )
-    }
-    AlertData::None {} => Default::default(),
-  };
-
+  let content = standard_alert_content(alert);
   if !content.is_empty() {
     send_message(url, email, content).await?;
   }

bin/core/src/alert/pushover.rs

@@ -7,205 +7,7 @@ pub async fn send_alert(
   url: &str,
   alert: &Alert,
 ) -> anyhow::Result<()> {
-  let level = fmt_level(alert.level);
-  let content = match &alert.data {
-    AlertData::Test { id, name } => {
-      let link = resource_link(ResourceTargetVariant::Alerter, id);
-      format!(
-        "{level} | If you see this message, then Alerter {name} is working\n{link}",
-      )
-    }
-    AlertData::ServerUnreachable {
-      id,
-      name,
-      region,
-      err,
-    } => {
-      let region = fmt_region(region);
-      let link = resource_link(ResourceTargetVariant::Server, id);
-      match alert.level {
-        SeverityLevel::Ok => {
-          format!("{level} | {name}{region} is now reachable\n{link}")
-        }
-        SeverityLevel::Critical => {
-          let err = err
-            .as_ref()
-            .map(|e| format!("\nerror: {e:#?}"))
-            .unwrap_or_default();
-          format!(
-            "{level} | {name}{region} is unreachable ❌\n{link}{err}"
-          )
-        }
-        _ => unreachable!(),
-      }
-    }
-    AlertData::ServerCpu {
-      id,
-      name,
-      region,
-      percentage,
-    } => {
-      let region = fmt_region(region);
-      let link = resource_link(ResourceTargetVariant::Server, id);
-      format!(
-        "{level} | {name}{region} cpu usage at {percentage:.1}%\n{link}",
-      )
-    }
-    AlertData::ServerMem {
-      id,
-      name,
-      region,
-      used_gb,
-      total_gb,
-    } => {
-      let region = fmt_region(region);
-      let link = resource_link(ResourceTargetVariant::Server, id);
-      let percentage = 100.0 * used_gb / total_gb;
-      format!(
-        "{level} | {name}{region} memory usage at {percentage:.1}%💾\n\nUsing {used_gb:.1} GiB / {total_gb:.1} GiB\n{link}",
-      )
-    }
-    AlertData::ServerDisk {
-      id,
-      name,
-      region,
-      path,
-      used_gb,
-      total_gb,
-    } => {
-      let region = fmt_region(region);
-      let link = resource_link(ResourceTargetVariant::Server, id);
-      let percentage = 100.0 * used_gb / total_gb;
-      format!(
-        "{level} | {name}{region} disk usage at {percentage:.1}%💿\nmount point: {path:?}\nusing {used_gb:.1} GiB / {total_gb:.1} GiB\n{link}",
-      )
-    }
-    AlertData::ContainerStateChange {
-      id,
-      name,
-      server_id: _server_id,
-      server_name,
-      from,
-      to,
-    } => {
-      let link = resource_link(ResourceTargetVariant::Deployment, id);
-      let to_state = fmt_docker_container_state(to);
-      format!(
-        "📦Deployment {name} is now {to_state}\nserver: {server_name}\nprevious: {from}\n{link}",
-      )
-    }
-    AlertData::DeploymentImageUpdateAvailable {
-      id,
-      name,
-      server_id: _server_id,
-      server_name,
-      image,
-    } => {
-      let link = resource_link(ResourceTargetVariant::Deployment, id);
-      format!(
-        "⬆ Deployment {name} has an update available\nserver: {server_name}\nimage: {image}\n{link}",
-      )
-    }
-    AlertData::DeploymentAutoUpdated {
-      id,
-      name,
-      server_id: _server_id,
-      server_name,
-      image,
-    } => {
-      let link = resource_link(ResourceTargetVariant::Deployment, id);
-      format!(
-        "⬆ Deployment {name} was updated automatically\nserver: {server_name}\nimage: {image}\n{link}",
-      )
-    }
-    AlertData::StackStateChange {
-      id,
-      name,
-      server_id: _server_id,
-      server_name,
-      from,
-      to,
-    } => {
-      let link = resource_link(ResourceTargetVariant::Stack, id);
-      let to_state = fmt_stack_state(to);
-      format!(
-        "🥞 Stack {name} is now {to_state}\nserver: {server_name}\nprevious: {from}\n{link}",
-      )
-    }
-    AlertData::StackImageUpdateAvailable {
-      id,
-      name,
-      server_id: _server_id,
-      server_name,
-      service,
-      image,
-    } => {
-      let link = resource_link(ResourceTargetVariant::Stack, id);
-      format!(
-        "⬆ Stack {name} has an update available\nserver: {server_name}\nservice: {service}\nimage: {image}\n{link}",
-      )
-    }
-    AlertData::StackAutoUpdated {
-      id,
-      name,
-      server_id: _server_id,
-      server_name,
-      images,
-    } => {
-      let link = resource_link(ResourceTargetVariant::Stack, id);
-      let images_label =
-        if images.len() > 1 { "images" } else { "image" };
-      let images_str = images.join(", ");
-      format!(
-        "⬆ Stack {name} was updated automatically ⏫\nserver: {server_name}\n{images_label}: {images_str}\n{link}",
-      )
-    }
-    AlertData::AwsBuilderTerminationFailed {
-      instance_id,
-      message,
-    } => {
-      format!(
-        "{level} | Failed to terminate AWS builder instance\ninstance id: {instance_id}\n{message}",
-      )
-    }
-    AlertData::ResourceSyncPendingUpdates { id, name } => {
-      let link =
-        resource_link(ResourceTargetVariant::ResourceSync, id);
-      format!(
-        "{level} | Pending resource sync updates on {name}\n{link}",
-      )
-    }
-    AlertData::BuildFailed { id, name, version } => {
-      let link = resource_link(ResourceTargetVariant::Build, id);
-      format!(
-        "{level} | Build {name} failed\nversion: v{version}\n{link}",
-      )
-    }
-    AlertData::RepoBuildFailed { id, name } => {
-      let link = resource_link(ResourceTargetVariant::Repo, id);
-      format!("{level} | Repo build for {name} failed\n{link}",)
-    }
-    AlertData::ProcedureFailed { id, name } => {
-      let link = resource_link(ResourceTargetVariant::Procedure, id);
-      format!("{level} | Procedure {name} failed\n{link}")
-    }
-    AlertData::ActionFailed { id, name } => {
-      let link = resource_link(ResourceTargetVariant::Action, id);
-      format!("{level} | Action {name} failed\n{link}")
-    }
-    AlertData::ScheduleRun {
-      resource_type,
-      id,
-      name,
-    } => {
-      let link = resource_link(*resource_type, id);
-      format!(
-        "{level} | {name} ({resource_type}) | Scheduled run started 🕝\n{link}"
-      )
-    }
-    AlertData::None {} => Default::default(),
-  };
-
+  let content = standard_alert_content(alert);
   if !content.is_empty() {
     send_message(url, content).await?;
   }

bin/core/src/alert/slack.rs

@@ -23,6 +23,35 @@ pub async fn send_alert(
       ];
       (text, blocks.into())
     }
+    AlertData::ServerVersionMismatch {
+      id,
+      name,
+      region,
+      server_version,
+      core_version,
+    } => {
+      let region = fmt_region(region);
+      let text = match alert.level {
+        SeverityLevel::Ok => {
+          format!(
+            "{level} | *{name}*{region} | Periphery version now matches Core version ✅"
+          )
+        }
+        _ => {
+          format!(
+            "{level} | *{name}*{region} | Version mismatch detected ⚠️\nPeriphery: {server_version} | Core: {core_version}"
+          )
+        }
+      };
+      let blocks = vec![
+        Block::header(text.clone()),
+        Block::section(resource_link(
+          ResourceTargetVariant::Server,
+          id,
+        )),
+      ];
+      (text, blocks.into())
+    }
     AlertData::ServerUnreachable {
       id,
       name,
@@ -429,6 +458,12 @@ pub async fn send_alert(
       ];
       (text, blocks.into())
     }
+    AlertData::Custom { message, details } => {
+      let text = format!("{level} | {message}");
+      let blocks =
+        vec![Block::header(text.clone()), Block::section(details)];
+      (text, blocks.into())
+    }
     AlertData::None {} => Default::default(),
   };
   if !text.is_empty() {

bin/core/src/api/auth.rs

@@ -3,11 +3,12 @@ use std::{sync::OnceLock, time::Instant};
 use axum::{Router, extract::Path, http::HeaderMap, routing::post};
 use derive_variants::{EnumVariants, ExtractVariant};
 use komodo_client::{api::auth::*, entities::user::User};
+use reqwest::StatusCode;
 use resolver_api::Resolve;
 use response::Response;
 use serde::{Deserialize, Serialize};
 use serde_json::json;
-use serror::Json;
+use serror::{AddStatusCode, Json};
 use typeshare::typeshare;
 use uuid::Uuid;
 
@@ -152,7 +153,11 @@ impl Resolve<AuthArgs> for GetUser {
     self,
     AuthArgs { headers }: &AuthArgs,
   ) -> serror::Result<User> {
-    let user_id = get_user_id_from_headers(headers).await?;
-    Ok(get_user(&user_id).await?)
+    let user_id = get_user_id_from_headers(headers)
+      .await
+      .status_code(StatusCode::UNAUTHORIZED)?;
+    get_user(&user_id)
+      .await
+      .status_code(StatusCode::UNAUTHORIZED)
   }
 }

bin/core/src/api/execute/action.rs

@@ -92,8 +92,11 @@ impl Resolve<ExecuteArgs> for RunAction {
 
     // This will set action state back to default when dropped.
     // Will also check to ensure action not already busy before updating.
-    let _action_guard =
-      action_state.update(|state| state.running = true)?;
+    let _action_guard = action_state.update_custom(
+      |state| state.running += 1,
+      |state| state.running -= 1,
+      false,
+    )?;
 
     let mut update = update.clone();
 

bin/core/src/api/execute/alerter.rs

@@ -1,18 +1,22 @@
+use anyhow::{Context, anyhow};
 use formatting::format_serror;
+use futures::{TryStreamExt, stream::FuturesUnordered};
 use komodo_client::{
-  api::execute::TestAlerter,
+  api::execute::{SendAlert, TestAlerter},
   entities::{
-    alert::{Alert, AlertData, SeverityLevel},
+    alert::{Alert, AlertData, AlertDataVariant, SeverityLevel},
     alerter::Alerter,
     komodo_timestamp,
     permission::PermissionLevel,
   },
 };
+use reqwest::StatusCode;
 use resolver_api::Resolve;
+use serror::AddStatusCodeError;
 
 use crate::{
   alert::send_alert_to_alerter, helpers::update::update_update,
-  permission::get_check_permissions,
+  permission::get_check_permissions, resource::list_full_for_user,
 };
 
 use super::ExecuteArgs;
@@ -71,3 +75,75 @@ impl Resolve<ExecuteArgs> for TestAlerter {
     Ok(update)
   }
 }
+
+//
+
+impl Resolve<ExecuteArgs> for SendAlert {
+  #[instrument(name = "SendAlert", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+  async fn resolve(
+    self,
+    ExecuteArgs { user, update }: &ExecuteArgs,
+  ) -> Result<Self::Response, Self::Error> {
+    let alerters = list_full_for_user::<Alerter>(
+      Default::default(),
+      user,
+      PermissionLevel::Execute.into(),
+      &[],
+    )
+    .await?
+    .into_iter()
+    .filter(|a| {
+      a.config.enabled
+        && (self.alerters.is_empty()
+          || self.alerters.contains(&a.name)
+          || self.alerters.contains(&a.id))
+        && (a.config.alert_types.is_empty()
+          || a.config.alert_types.contains(&AlertDataVariant::Custom))
+    })
+    .collect::<Vec<_>>();
+
+    if alerters.is_empty() {
+      return Err(anyhow!(
+        "Could not find any valid alerters to send to, this required Execute permissions on the Alerter"
+      ).status_code(StatusCode::BAD_REQUEST));
+    }
+
+    let mut update = update.clone();
+
+    let ts = komodo_timestamp();
+
+    let alert = Alert {
+      id: Default::default(),
+      ts,
+      resolved: true,
+      level: self.level,
+      target: update.target.clone(),
+      data: AlertData::Custom {
+        message: self.message,
+        details: self.details,
+      },
+      resolved_ts: Some(ts),
+    };
+
+    update.push_simple_log(
+      "Send alert",
+      serde_json::to_string_pretty(&alert)
+        .context("Failed to serialize alert to JSON")?,
+    );
+
+    if let Err(e) = alerters
+      .iter()
+      .map(|alerter| send_alert_to_alerter(alerter, &alert))
+      .collect::<FuturesUnordered<_>>()
+      .try_collect::<Vec<_>>()
+      .await
+    {
+      update.push_error_log("Send Error", format_serror(&e.into()));
+    };
+
+    update.finalize();
+    update_update(update.clone()).await?;
+
+    Ok(update)
+  }
+}

bin/core/src/api/execute/build.rs

@@ -1,4 +1,8 @@
-use std::{future::IntoFuture, time::Duration};
+use std::{
+  collections::{HashMap, HashSet},
+  future::IntoFuture,
+  time::Duration,
+};
 
 use anyhow::{Context, anyhow};
 use database::mungos::{
@@ -20,10 +24,10 @@ use komodo_client::{
   entities::{
     alert::{Alert, AlertData, SeverityLevel},
     all_logs_success,
-    build::{Build, BuildConfig, ImageRegistryConfig},
+    build::{Build, BuildConfig},
     builder::{Builder, BuilderConfig},
     deployment::DeploymentState,
-    komodo_timestamp,
+    komodo_timestamp, optional_string,
     permission::PermissionLevel,
     repo::Repo,
     update::{Log, Update},
@@ -133,8 +137,8 @@ impl Resolve<ExecuteArgs> for RunBuild {
     let git_token =
       build_git_token(&mut build, repo.as_mut()).await?;
 
-    let registry_token =
-      validate_account_extract_registry_token(&build).await?;
+    let registry_tokens =
+      validate_account_extract_registry_tokens(&build).await?;
 
     let cancel = CancellationToken::new();
     let cancel_clone = cancel.clone();
@@ -284,14 +288,12 @@ impl Resolve<ExecuteArgs> for RunBuild {
           .request(api::build::Build {
             build: build.clone(),
             repo,
-            registry_token,
+            registry_tokens,
             replacers: secret_replacers.into_iter().collect(),
-            // Push a commit hash tagged image
-            additional_tags: if update.commit_hash.is_empty() {
-              Default::default()
-            } else {
-              vec![update.commit_hash.clone()]
-            },
+            // To push a commit hash tagged image
+            commit_hash: optional_string(&update.commit_hash),
+            // Unused for now
+            additional_tags: Default::default(),
           }) => res.context("failed at call to periphery to build"),
         _ = cancel.cancelled() => {
           info!("build cancelled during build, cleaning up builder");
@@ -608,34 +610,48 @@ async fn handle_post_build_redeploy(build_id: &str) {
 /// This will make sure that a build with non-none image registry has an account attached,
 /// and will check the core config for a token matching requirements.
 /// Otherwise it is left to periphery.
-async fn validate_account_extract_registry_token(
+async fn validate_account_extract_registry_tokens(
   Build {
-    config:
-      BuildConfig {
-        image_registry:
-          ImageRegistryConfig {
-            domain, account, ..
-          },
-        ..
-      },
+    config: BuildConfig { image_registry, .. },
     ..
   }: &Build,
-) -> serror::Result<Option<String>> {
-  if domain.is_empty() {
-    return Ok(None);
-  }
-  if account.is_empty() {
-    return Err(
-      anyhow!(
-        "Must attach account to use registry provider {domain}"
-      )
-      .into(),
+  // Maps (domain, account) -> token
+) -> serror::Result<Vec<(String, String, String)>> {
+  let mut res = HashMap::with_capacity(image_registry.capacity());
+
+  for (domain, account) in image_registry
+    .iter()
+    .map(|r| (r.domain.as_str(), r.account.as_str()))
+    // This ensures uniqueness / prevents redundant logins
+    .collect::<HashSet<_>>()
+  {
+    if domain.is_empty() {
+      continue;
+    }
+    if account.is_empty() {
+      return Err(
+        anyhow!(
+          "Must attach account to use registry provider {domain}"
+        )
+        .into(),
+      );
+    }
+    let Some(registry_token) = registry_token(domain, account).await.with_context(
+      || format!("Failed to get registry token in call to db. Stopping run. | {domain} | {account}"),
+    )? else {
+      continue;
+    };
+
+    res.insert(
+      (domain.to_string(), account.to_string()),
+      registry_token,
     );
   }
 
-  let registry_token = registry_token(domain, account).await.with_context(
-    || format!("Failed to get registry token in call to db. Stopping run. | {domain} | {account}"),
-  )?;
-
-  Ok(registry_token)
+  Ok(
+    res
+      .into_iter()
+      .map(|((domain, account), token)| (domain, account, token))
+      .collect(),
+  )
 }

bin/core/src/api/execute/deployment.rs

@@ -12,7 +12,7 @@ use komodo_client::{
     deployment::{
       Deployment, DeploymentImage, extract_registry_domain,
     },
-    get_image_name, komodo_timestamp, optional_string,
+    komodo_timestamp, optional_string,
     permission::PermissionLevel,
     server::Server,
     update::{Log, Update},
@@ -115,8 +115,11 @@ impl Resolve<ExecuteArgs> for Deploy {
     let (version, registry_token) = match &deployment.config.image {
       DeploymentImage::Build { build_id, version } => {
         let build = resource::get::<Build>(build_id).await?;
-        let image_name = get_image_name(&build)
-          .context("failed to create image name")?;
+        let image_names = build.get_image_names();
+        let image_name = image_names
+          .first()
+          .context("No image name could be created")
+          .context("Failed to create image name")?;
         let version = if version.is_none() {
           build.config.version
         } else {
@@ -133,21 +136,27 @@ impl Resolve<ExecuteArgs> for Deploy {
         deployment.config.image = DeploymentImage::Image {
           image: format!("{image_name}:{version_str}"),
         };
-        if build.config.image_registry.domain.is_empty() {
+        let first_registry = build
+          .config
+          .image_registry
+          .first()
+          .unwrap_or(ImageRegistryConfig::static_default());
+        if first_registry.domain.is_empty() {
           (version, None)
         } else {
           let ImageRegistryConfig {
             domain, account, ..
-          } = build.config.image_registry;
+          } = first_registry;
           if deployment.config.image_registry_account.is_empty() {
-            deployment.config.image_registry_account = account
+            deployment.config.image_registry_account =
+              account.to_string();
           }
           let token = if !deployment
             .config
             .image_registry_account
             .is_empty()
           {
-            registry_token(&domain, &deployment.config.image_registry_account).await.with_context(
+            registry_token(domain, &deployment.config.image_registry_account).await.with_context(
               || format!("Failed to get git token in call to db. Stopping run. | {domain} | {}", deployment.config.image_registry_account),
             )?
           } else {
@@ -213,7 +222,7 @@ impl Resolve<ExecuteArgs> for Deploy {
       }
     };
 
-    update_cache_for_server(&server).await;
+    update_cache_for_server(&server, true).await;
 
     update.finalize();
     update_update(update.clone()).await?;
@@ -240,8 +249,11 @@ pub async fn pull_deployment_inner(
   let (image, account, token) = match deployment.config.image {
     DeploymentImage::Build { build_id, version } => {
       let build = resource::get::<Build>(&build_id).await?;
-      let image_name = get_image_name(&build)
-        .context("failed to create image name")?;
+      let image_names = build.get_image_names();
+      let image_name = image_names
+        .first()
+        .context("No image name could be created")
+        .context("Failed to create image name")?;
       let version = if version.is_none() {
         build.config.version.to_string()
       } else {
@@ -255,26 +267,31 @@ pub async fn pull_deployment_inner(
       };
       // replace image with corresponding build image.
       let image = format!("{image_name}:{version}");
-      if build.config.image_registry.domain.is_empty() {
+      let first_registry = build
+        .config
+        .image_registry
+        .first()
+        .unwrap_or(ImageRegistryConfig::static_default());
+      if first_registry.domain.is_empty() {
         (image, None, None)
       } else {
         let ImageRegistryConfig {
           domain, account, ..
-        } = build.config.image_registry;
+        } = first_registry;
         let account =
           if deployment.config.image_registry_account.is_empty() {
             account
           } else {
-            deployment.config.image_registry_account
+            &deployment.config.image_registry_account
           };
         let token = if !account.is_empty() {
-          registry_token(&domain, &account).await.with_context(
+          registry_token(domain, account).await.with_context(
               || format!("Failed to get git token in call to db. Stopping run. | {domain} | {account}"),
             )?
         } else {
           None
         };
-        (image, optional_string(&account), token)
+        (image, optional_string(account), token)
       }
     }
     DeploymentImage::Image { image } => {
@@ -326,7 +343,7 @@ pub async fn pull_deployment_inner(
       Err(e) => Log::error("Pull image", format_serror(&e.into())),
     };
 
-    update_cache_for_server(server).await;
+    update_cache_for_server(server, true).await;
     anyhow::Ok(log)
   }
   .await;
@@ -411,7 +428,7 @@ impl Resolve<ExecuteArgs> for StartDeployment {
     };
 
     update.logs.push(log);
-    update_cache_for_server(&server).await;
+    update_cache_for_server(&server, true).await;
     update.finalize();
     update_update(update.clone()).await?;
 
@@ -460,7 +477,7 @@ impl Resolve<ExecuteArgs> for RestartDeployment {
     };
 
     update.logs.push(log);
-    update_cache_for_server(&server).await;
+    update_cache_for_server(&server, true).await;
     update.finalize();
     update_update(update.clone()).await?;
 
@@ -507,7 +524,7 @@ impl Resolve<ExecuteArgs> for PauseDeployment {
     };
 
     update.logs.push(log);
-    update_cache_for_server(&server).await;
+    update_cache_for_server(&server, true).await;
     update.finalize();
     update_update(update.clone()).await?;
 
@@ -556,7 +573,7 @@ impl Resolve<ExecuteArgs> for UnpauseDeployment {
     };
 
     update.logs.push(log);
-    update_cache_for_server(&server).await;
+    update_cache_for_server(&server, true).await;
     update.finalize();
     update_update(update.clone()).await?;
 
@@ -611,7 +628,7 @@ impl Resolve<ExecuteArgs> for StopDeployment {
     };
 
     update.logs.push(log);
-    update_cache_for_server(&server).await;
+    update_cache_for_server(&server, true).await;
     update.finalize();
     update_update(update.clone()).await?;
 
@@ -694,7 +711,7 @@ impl Resolve<ExecuteArgs> for DestroyDeployment {
 
     update.logs.push(log);
     update.finalize();
-    update_cache_for_server(&server).await;
+    update_cache_for_server(&server, true).await;
     update_update(update.clone()).await?;
 
     Ok(update)

bin/core/src/api/execute/maintenance.rs

@@ -49,7 +49,7 @@ impl Resolve<ExecuteArgs> for ClearRepoCache {
     if !user.admin {
       return Err(
         anyhow!("This method is admin only.")
-          .status_code(StatusCode::UNAUTHORIZED),
+          .status_code(StatusCode::FORBIDDEN),
       );
     }
 
@@ -124,7 +124,7 @@ impl Resolve<ExecuteArgs> for BackupCoreDatabase {
     if !user.admin {
       return Err(
         anyhow!("This method is admin only.")
-          .status_code(StatusCode::UNAUTHORIZED),
+          .status_code(StatusCode::FORBIDDEN),
       );
     }
 
@@ -173,7 +173,7 @@ impl Resolve<ExecuteArgs> for GlobalAutoUpdate {
     if !user.admin {
       return Err(
         anyhow!("This method is admin only.")
-          .status_code(StatusCode::UNAUTHORIZED),
+          .status_code(StatusCode::FORBIDDEN),
       );
     }
 

bin/core/src/api/execute/mod.rs

@@ -102,6 +102,7 @@ pub enum ExecuteRequest {
   UnpauseStack(UnpauseStack),
   DestroyStack(DestroyStack),
   BatchDestroyStack(BatchDestroyStack),
+  RunStackService(RunStackService),
 
   // ==== DEPLOYMENT ====
   Deploy(Deploy),
@@ -139,6 +140,7 @@ pub enum ExecuteRequest {
 
   // ==== ALERTER ====
   TestAlerter(TestAlerter),
+  SendAlert(SendAlert),
 
   // ==== SYNC ====
   RunSync(RunSync),
@@ -219,24 +221,33 @@ pub fn inner_handler(
       ));
     }
 
+    // Spawn a task for the execution which continues
+    // running after this method returns.
     let handle =
       tokio::spawn(task(req_id, request, user, update.clone()));
 
+    // Spawns another task to monitor the first for failures,
+    // and add the log to Update about it (which primary task can't do because it errored out)
     tokio::spawn({
       let update_id = update.id.clone();
       async move {
         let log = match handle.await {
           Ok(Err(e)) => {
             warn!("/execute request {req_id} task error: {e:#}",);
-            Log::error("task error", format_serror(&e.into()))
+            Log::error("Task Error", format_serror(&e.into()))
           }
           Err(e) => {
             warn!("/execute request {req_id} spawn error: {e:?}",);
-            Log::error("spawn error", format!("{e:#?}"))
+            Log::error("Spawn Error", format!("{e:#?}"))
           }
           _ => return,
         };
         let res = async {
+          // Nothing to do if update was never actually created,
+          // which is the case when the id is empty.
+          if update_id.is_empty() {
+            return Ok(());
+          }
           let mut update =
             find_one_by_id(&db_client().updates, &update_id)
               .await

bin/core/src/api/execute/server.rs

@@ -66,7 +66,7 @@ impl Resolve<ExecuteArgs> for StartContainer {
     };
 
     update.logs.push(log);
-    update_cache_for_server(&server).await;
+    update_cache_for_server(&server, true).await;
 
     update.finalize();
     update_update(update.clone()).await?;
@@ -122,7 +122,7 @@ impl Resolve<ExecuteArgs> for RestartContainer {
     };
 
     update.logs.push(log);
-    update_cache_for_server(&server).await;
+    update_cache_for_server(&server, true).await;
 
     update.finalize();
     update_update(update.clone()).await?;
@@ -176,7 +176,7 @@ impl Resolve<ExecuteArgs> for PauseContainer {
     };
 
     update.logs.push(log);
-    update_cache_for_server(&server).await;
+    update_cache_for_server(&server, true).await;
 
     update.finalize();
     update_update(update.clone()).await?;
@@ -232,7 +232,7 @@ impl Resolve<ExecuteArgs> for UnpauseContainer {
     };
 
     update.logs.push(log);
-    update_cache_for_server(&server).await;
+    update_cache_for_server(&server, true).await;
 
     update.finalize();
     update_update(update.clone()).await?;
@@ -288,7 +288,7 @@ impl Resolve<ExecuteArgs> for StopContainer {
     };
 
     update.logs.push(log);
-    update_cache_for_server(&server).await;
+    update_cache_for_server(&server, true).await;
 
     update.finalize();
     update_update(update.clone()).await?;
@@ -350,7 +350,7 @@ impl Resolve<ExecuteArgs> for DestroyContainer {
     };
 
     update.logs.push(log);
-    update_cache_for_server(&server).await;
+    update_cache_for_server(&server, true).await;
 
     update.finalize();
     update_update(update.clone()).await?;
@@ -401,7 +401,7 @@ impl Resolve<ExecuteArgs> for StartAllContainers {
       );
     }
 
-    update_cache_for_server(&server).await;
+    update_cache_for_server(&server, true).await;
     update.finalize();
     update_update(update.clone()).await?;
 
@@ -453,7 +453,7 @@ impl Resolve<ExecuteArgs> for RestartAllContainers {
       );
     }
 
-    update_cache_for_server(&server).await;
+    update_cache_for_server(&server, true).await;
     update.finalize();
     update_update(update.clone()).await?;
 
@@ -503,7 +503,7 @@ impl Resolve<ExecuteArgs> for PauseAllContainers {
       );
     }
 
-    update_cache_for_server(&server).await;
+    update_cache_for_server(&server, true).await;
     update.finalize();
     update_update(update.clone()).await?;
 
@@ -555,7 +555,7 @@ impl Resolve<ExecuteArgs> for UnpauseAllContainers {
       );
     }
 
-    update_cache_for_server(&server).await;
+    update_cache_for_server(&server, true).await;
     update.finalize();
     update_update(update.clone()).await?;
 
@@ -605,7 +605,7 @@ impl Resolve<ExecuteArgs> for StopAllContainers {
       );
     }
 
-    update_cache_for_server(&server).await;
+    update_cache_for_server(&server, true).await;
     update.finalize();
     update_update(update.clone()).await?;
 
@@ -660,7 +660,7 @@ impl Resolve<ExecuteArgs> for PruneContainers {
     };
 
     update.logs.push(log);
-    update_cache_for_server(&server).await;
+    update_cache_for_server(&server, true).await;
 
     update.finalize();
     update_update(update.clone()).await?;
@@ -711,7 +711,7 @@ impl Resolve<ExecuteArgs> for DeleteNetwork {
     };
 
     update.logs.push(log);
-    update_cache_for_server(&server).await;
+    update_cache_for_server(&server, true).await;
 
     update.finalize();
     update_update(update.clone()).await?;
@@ -765,7 +765,7 @@ impl Resolve<ExecuteArgs> for PruneNetworks {
     };
 
     update.logs.push(log);
-    update_cache_for_server(&server).await;
+    update_cache_for_server(&server, true).await;
 
     update.finalize();
     update_update(update.clone()).await?;
@@ -813,7 +813,7 @@ impl Resolve<ExecuteArgs> for DeleteImage {
     };
 
     update.logs.push(log);
-    update_cache_for_server(&server).await;
+    update_cache_for_server(&server, true).await;
 
     update.finalize();
     update_update(update.clone()).await?;
@@ -865,7 +865,7 @@ impl Resolve<ExecuteArgs> for PruneImages {
       };
 
     update.logs.push(log);
-    update_cache_for_server(&server).await;
+    update_cache_for_server(&server, true).await;
 
     update.finalize();
     update_update(update.clone()).await?;
@@ -916,7 +916,7 @@ impl Resolve<ExecuteArgs> for DeleteVolume {
     };
 
     update.logs.push(log);
-    update_cache_for_server(&server).await;
+    update_cache_for_server(&server, true).await;
 
     update.finalize();
     update_update(update.clone()).await?;
@@ -968,7 +968,7 @@ impl Resolve<ExecuteArgs> for PruneVolumes {
       };
 
     update.logs.push(log);
-    update_cache_for_server(&server).await;
+    update_cache_for_server(&server, true).await;
 
     update.finalize();
     update_update(update.clone()).await?;
@@ -1020,7 +1020,7 @@ impl Resolve<ExecuteArgs> for PruneDockerBuilders {
       };
 
     update.logs.push(log);
-    update_cache_for_server(&server).await;
+    update_cache_for_server(&server, true).await;
 
     update.finalize();
     update_update(update.clone()).await?;
@@ -1072,7 +1072,7 @@ impl Resolve<ExecuteArgs> for PruneBuildx {
       };
 
     update.logs.push(log);
-    update_cache_for_server(&server).await;
+    update_cache_for_server(&server, true).await;
 
     update.finalize();
     update_update(update.clone()).await?;
@@ -1123,7 +1123,7 @@ impl Resolve<ExecuteArgs> for PruneSystem {
     };
 
     update.logs.push(log);
-    update_cache_for_server(&server).await;
+    update_cache_for_server(&server, true).await;
 
     update.finalize();
     update_update(update.clone()).await?;

bin/core/src/api/execute/stack.rs

@@ -1,15 +1,23 @@
+use std::{collections::HashSet, str::FromStr};
+
 use anyhow::Context;
-use database::mungos::mongodb::bson::{doc, to_document};
+use database::mungos::mongodb::bson::{
+  doc, oid::ObjectId, to_bson, to_document,
+};
 use formatting::format_serror;
 use interpolate::Interpolator;
 use komodo_client::{
   api::{execute::*, write::RefreshStackCache},
   entities::{
+    FileContents,
     permission::PermissionLevel,
     repo::Repo,
     server::Server,
-    stack::{Stack, StackInfo},
+    stack::{
+      Stack, StackFileRequires, StackInfo, StackRemoteFileContents,
+    },
     update::{Log, Update},
+    user::User,
   },
 };
 use periphery_client::api::compose::*;
@@ -21,7 +29,9 @@ use crate::{
     periphery_client,
     query::{VariablesAndSecrets, get_variables_and_secrets},
     stack_git_token,
-    update::{add_update_without_send, update_update},
+    update::{
+      add_update_without_send, init_execution_update, update_update,
+    },
   },
   monitor::update_cache_for_server,
   permission::get_check_permissions,
@@ -179,7 +189,15 @@ impl Resolve<ExecuteArgs> for DeployStack {
       ) = if deployed {
         (
           Some(latest_services.clone()),
-          Some(file_contents.clone()),
+          Some(
+            file_contents
+              .iter()
+              .map(|f| FileContents {
+                path: f.path.clone(),
+                contents: f.contents.clone(),
+              })
+              .collect(),
+          ),
           compose_config,
           commit_hash.clone(),
           commit_message.clone(),
@@ -242,7 +260,7 @@ impl Resolve<ExecuteArgs> for DeployStack {
     }
 
     // Ensure cached stack state up to date by updating server cache
-    update_cache_for_server(&server).await;
+    update_cache_for_server(&server, true).await;
 
     update.finalize();
     update_update(update.clone()).await?;
@@ -289,62 +307,347 @@ impl Resolve<ExecuteArgs> for DeployStackIfChanged {
       PermissionLevel::Execute.into(),
     )
     .await?;
+
     RefreshStackCache {
       stack: stack.id.clone(),
     }
     .resolve(&WriteArgs { user: user.clone() })
     .await?;
+
     let stack = resource::get::<Stack>(&stack.id).await?;
-    let changed = match (
+
+    let action = match (
       &stack.info.deployed_contents,
       &stack.info.remote_contents,
     ) {
       (Some(deployed_contents), Some(latest_contents)) => {
-        let changed = || {
-          for latest in latest_contents {
-            let Some(deployed) = deployed_contents
-              .iter()
-              .find(|c| c.path == latest.path)
-            else {
-              return true;
-            };
-            if latest.contents != deployed.contents {
-              return true;
-            }
-          }
-          false
-        };
-        changed()
+        let services = stack
+          .info
+          .latest_services
+          .iter()
+          .map(|s| s.service_name.clone())
+          .collect::<Vec<_>>();
+        resolve_deploy_if_changed_action(
+          deployed_contents,
+          latest_contents,
+          &services,
+        )
       }
-      (None, _) => true,
-      _ => false,
+      (None, _) => DeployIfChangedAction::FullDeploy,
+      _ => DeployIfChangedAction::Services {
+        deploy: Vec::new(),
+        restart: Vec::new(),
+      },
     };
 
     let mut update = update.clone();
 
-    if !changed {
-      update.push_simple_log(
-        "Diff compose files",
-        String::from("Deploy cancelled after no changes detected."),
-      );
-      update.finalize();
-      return Ok(update);
-    }
+    match action {
+      // Existing path pre 1.19.1
+      DeployIfChangedAction::FullDeploy => {
+        // Don't actually send it here, let the handler send it after it can set action state.
+        // This is usually done in crate::helpers::update::init_execution_update.
+        update.id = add_update_without_send(&update).await?;
 
-    // Don't actually send it here, let the handler send it after it can set action state.
-    // This is usually done in crate::helpers::update::init_execution_update.
-    update.id = add_update_without_send(&update).await?;
+        DeployStack {
+          stack: stack.name,
+          services: Vec::new(),
+          stop_time: self.stop_time,
+        }
+        .resolve(&ExecuteArgs {
+          user: user.clone(),
+          update,
+        })
+        .await
+      }
+      DeployIfChangedAction::FullRestart => {
+        // For git repo based stacks, need to do a
+        // PullStack in order to ensure latest repo contents on the
+        // host before restart.
+        maybe_pull_stack(&stack, Some(&mut update)).await?;
 
-    DeployStack {
-      stack: stack.name,
-      services: Vec::new(),
-      stop_time: self.stop_time,
+        let mut update =
+          restart_services(stack.name, Vec::new(), user).await?;
+
+        if update.success {
+          // Need to update 'info.deployed_contents' with the
+          // latest contents so next check doesn't read the same diff.
+          update_deployed_contents_with_latest(
+            &stack.id,
+            stack.info.remote_contents,
+            &mut update,
+          )
+          .await;
+        }
+
+        Ok(update)
+      }
+      DeployIfChangedAction::Services { deploy, restart } => {
+        match (deploy.is_empty(), restart.is_empty()) {
+          // Both empty, nothing to do
+          (true, true) => {
+            update.push_simple_log(
+              "Diff compose files",
+              String::from(
+                "Deploy cancelled after no changes detected.",
+              ),
+            );
+            update.finalize();
+            Ok(update)
+          }
+          // Only restart
+          (true, false) => {
+            // For git repo based stacks, need to do a
+            // PullStack in order to ensure latest repo contents on the
+            // host before restart. Only necessary if no "deploys" (deploy already pulls stack).
+            maybe_pull_stack(&stack, Some(&mut update)).await?;
+
+            let mut update =
+              restart_services(stack.name, restart, user).await?;
+
+            if update.success {
+              // Need to update 'info.deployed_contents' with the
+              // latest contents so next check doesn't read the same diff.
+              update_deployed_contents_with_latest(
+                &stack.id,
+                stack.info.remote_contents,
+                &mut update,
+              )
+              .await;
+            }
+
+            Ok(update)
+          }
+          // Only deploy
+          (false, true) => {
+            deploy_services(stack.name, deploy, user).await
+          }
+          // Deploy then restart, returning non-db update with executed services.
+          (false, false) => {
+            update.push_simple_log(
+              "Execute Deploys",
+              format!("Deploying: {}", deploy.join(", "),),
+            );
+            // This already updates 'stack.info.deployed_services',
+            // restart doesn't require this again.
+            let deploy_update =
+              deploy_services(stack.name.clone(), deploy, user)
+                .await?;
+            if !deploy_update.success {
+              update.push_error_log(
+                "Execute Deploys",
+                String::from("There was a failure in service deploy"),
+              );
+              update.finalize();
+              return Ok(update);
+            }
+
+            update.push_simple_log(
+              "Execute Restarts",
+              format!("Restarting: {}", restart.join(", "),),
+            );
+            let restart_update =
+              restart_services(stack.name, restart, user).await?;
+            if !restart_update.success {
+              update.push_error_log(
+                "Execute Restarts",
+                String::from(
+                  "There was a failure in a service restart",
+                ),
+              );
+            }
+
+            update.finalize();
+            Ok(update)
+          }
+        }
+      }
     }
+  }
+}
+
+async fn deploy_services(
+  stack: String,
+  services: Vec<String>,
+  user: &User,
+) -> serror::Result<Update> {
+  // The existing update is initialized to DeployStack,
+  // but also has not been created on database.
+  // Setup a new update here.
+  let req = ExecuteRequest::DeployStack(DeployStack {
+    stack,
+    services,
+    stop_time: None,
+  });
+  let update = init_execution_update(&req, user).await?;
+  let ExecuteRequest::DeployStack(req) = req else {
+    unreachable!()
+  };
+  req
     .resolve(&ExecuteArgs {
       user: user.clone(),
       update,
     })
     .await
+}
+
+async fn restart_services(
+  stack: String,
+  services: Vec<String>,
+  user: &User,
+) -> serror::Result<Update> {
+  // The existing update is initialized to DeployStack,
+  // but also has not been created on database.
+  // Setup a new update here.
+  let req =
+    ExecuteRequest::RestartStack(RestartStack { stack, services });
+  let update = init_execution_update(&req, user).await?;
+  let ExecuteRequest::RestartStack(req) = req else {
+    unreachable!()
+  };
+  req
+    .resolve(&ExecuteArgs {
+      user: user.clone(),
+      update,
+    })
+    .await
+}
+
+/// This can safely be called in [DeployStackIfChanged]
+/// when there are ONLY changes to config files requiring restart,
+/// AFTER the restart has been successfully completed.
+///
+/// In the case the if changed action is not FullDeploy,
+/// the only file diff possible is to config files.
+/// Also note either full or service deploy will already update 'deployed_contents'
+/// making this method unnecessary in those cases.
+///
+/// Changes to config files after restart is applied should
+/// be taken as the deployed contents, otherwise next changed check
+/// will restart service again for no reason.
+async fn update_deployed_contents_with_latest(
+  id: &str,
+  contents: Option<Vec<StackRemoteFileContents>>,
+  update: &mut Update,
+) {
+  let Some(contents) = contents else {
+    return;
+  };
+  let contents = contents
+    .into_iter()
+    .map(|f| FileContents {
+      path: f.path,
+      contents: f.contents,
+    })
+    .collect::<Vec<_>>();
+  if let Err(e) = (async {
+    let contents = to_bson(&contents)
+      .context("Failed to serialize contents to bson")?;
+    let id =
+      ObjectId::from_str(id).context("Id is not valid ObjectId")?;
+    db_client()
+      .stacks
+      .update_one(
+        doc! { "_id": id },
+        doc! { "$set": { "info.deployed_contents": contents } },
+      )
+      .await
+      .context("Failed to update stack 'deployed_contents'")?;
+    anyhow::Ok(())
+  })
+  .await
+  {
+    update.push_error_log(
+      "Update content cache",
+      format_serror(&e.into()),
+    );
+    update.finalize();
+    let _ = update_update(update.clone()).await;
+  }
+}
+
+enum DeployIfChangedAction {
+  /// Changes to any compose or env files
+  /// always lead to this.
+  FullDeploy,
+  /// If the above is not met, then changes to
+  /// any changed additional file with `requires = "Restart"`
+  /// and empty services array will lead to this.
+  FullRestart,
+  /// If all changed additional files have specific services
+  /// they depend on, collect the final necessary
+  /// services to deploy / restart.
+  /// If eg `deploy` is empty, no services will be redeployed, same for `restart`.
+  /// If both are empty, nothing is to be done.
+  Services {
+    deploy: Vec<String>,
+    restart: Vec<String>,
+  },
+}
+
+fn resolve_deploy_if_changed_action(
+  deployed_contents: &[FileContents],
+  latest_contents: &[StackRemoteFileContents],
+  all_services: &[String],
+) -> DeployIfChangedAction {
+  let mut full_restart = false;
+  let mut deploy = HashSet::<String>::new();
+  let mut restart = HashSet::<String>::new();
+
+  for latest in latest_contents {
+    let Some(deployed) =
+      deployed_contents.iter().find(|c| c.path == latest.path)
+    else {
+      // If file doesn't exist in deployed contents, do full
+      // deploy to align this.
+      return DeployIfChangedAction::FullDeploy;
+    };
+    // Ignore unchanged files
+    if latest.contents == deployed.contents {
+      continue;
+    }
+    match (latest.requires, latest.services.is_empty()) {
+      (StackFileRequires::Redeploy, true) => {
+        // File has requires = "Redeploy" at global level.
+        // Can do early return here.
+        return DeployIfChangedAction::FullDeploy;
+      }
+      (StackFileRequires::Redeploy, false) => {
+        // Requires redeploy on specific services
+        deploy.extend(latest.services.clone());
+      }
+      (StackFileRequires::Restart, true) => {
+        // Services empty -> Full restart
+        full_restart = true;
+      }
+      (StackFileRequires::Restart, false) => {
+        restart.extend(latest.services.clone());
+      }
+      (StackFileRequires::None, _) => {
+        // File can be ignored even with changes.
+        continue;
+      }
+    }
+  }
+
+  match (full_restart, deploy.is_empty()) {
+    // Full restart required with NO deploys needed -> Full Restart
+    (true, true) => DeployIfChangedAction::FullRestart,
+    // Full restart required WITH deploys needed -> Deploy those, restart all others
+    (true, false) => DeployIfChangedAction::Services {
+      restart: all_services
+        .iter()
+        // Only keep ones that don't need deploy
+        .filter(|&s| !deploy.contains(s))
+        .cloned()
+        .collect(),
+      deploy: deploy.into_iter().collect(),
+    },
+    // No full restart needed -> Deploy / restart as. pickedup.
+    (false, _) => DeployIfChangedAction::Services {
+      deploy: deploy.into_iter().collect(),
+      restart: restart.into_iter().collect(),
+    },
   }
 }
 
@@ -371,6 +674,31 @@ impl Resolve<ExecuteArgs> for BatchPullStack {
   }
 }
 
+async fn maybe_pull_stack(
+  stack: &Stack,
+  update: Option<&mut Update>,
+) -> anyhow::Result<()> {
+  if stack.config.files_on_host
+    || (stack.config.repo.is_empty()
+      && stack.config.linked_repo.is_empty())
+  {
+    // Not repo based, no pull necessary
+    return Ok(());
+  }
+  let server =
+    resource::get::<Server>(&stack.config.server_id).await?;
+  let repo = if stack.config.repo.is_empty()
+    && !stack.config.linked_repo.is_empty()
+  {
+    Some(resource::get::<Repo>(&stack.config.linked_repo).await?)
+  } else {
+    None
+  };
+  pull_stack_inner(stack.clone(), Vec::new(), &server, repo, update)
+    .await?;
+  Ok(())
+}
+
 pub async fn pull_stack_inner(
   mut stack: Stack,
   services: Vec<String>,
@@ -433,7 +761,7 @@ pub async fn pull_stack_inner(
     .await?;
 
   // Ensure cached stack state up to date by updating server cache
-  update_cache_for_server(server).await;
+  update_cache_for_server(server, true).await;
 
   Ok(res)
 }
@@ -630,3 +958,95 @@ impl Resolve<ExecuteArgs> for DestroyStack {
     .map_err(Into::into)
   }
 }
+
+impl Resolve<ExecuteArgs> for RunStackService {
+  #[instrument(name = "RunStackService", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+  async fn resolve(
+    self,
+    ExecuteArgs { user, update }: &ExecuteArgs,
+  ) -> serror::Result<Update> {
+    let (mut stack, server) = get_stack_and_server(
+      &self.stack,
+      user,
+      PermissionLevel::Execute.into(),
+      true,
+    )
+    .await?;
+
+    let mut repo = if !stack.config.files_on_host
+      && !stack.config.linked_repo.is_empty()
+    {
+      crate::resource::get::<Repo>(&stack.config.linked_repo)
+        .await?
+        .into()
+    } else {
+      None
+    };
+
+    let action_state =
+      action_states().stack.get_or_insert_default(&stack.id).await;
+
+    let _action_guard =
+      action_state.update(|state| state.deploying = true)?;
+
+    let mut update = update.clone();
+    update_update(update.clone()).await?;
+
+    let git_token =
+      stack_git_token(&mut stack, repo.as_mut()).await?;
+
+    let registry_token = crate::helpers::registry_token(
+      &stack.config.registry_provider,
+      &stack.config.registry_account,
+    ).await.with_context(
+      || format!("Failed to get registry token in call to db. Stopping run. | {} | {}", stack.config.registry_provider, stack.config.registry_account),
+    )?;
+
+    let secret_replacers = if !stack.config.skip_secret_interp {
+      let VariablesAndSecrets { variables, secrets } =
+        get_variables_and_secrets().await?;
+
+      let mut interpolator =
+        Interpolator::new(Some(&variables), &secrets);
+
+      interpolator.interpolate_stack(&mut stack)?;
+      if let Some(repo) = repo.as_mut()
+        && !repo.config.skip_secret_interp
+      {
+        interpolator.interpolate_repo(repo)?;
+      }
+      interpolator.push_logs(&mut update.logs);
+
+      interpolator.secret_replacers
+    } else {
+      Default::default()
+    };
+
+    let log = periphery_client(&server)?
+      .request(ComposeRun {
+        stack,
+        repo,
+        git_token,
+        registry_token,
+        replacers: secret_replacers.into_iter().collect(),
+        service: self.service,
+        command: self.command,
+        no_tty: self.no_tty,
+        no_deps: self.no_deps,
+        detach: self.detach,
+        service_ports: self.service_ports,
+        env: self.env,
+        workdir: self.workdir,
+        user: self.user,
+        entrypoint: self.entrypoint,
+        pull: self.pull,
+      })
+      .await?;
+
+    update.logs.push(log);
+    update.finalize();
+    update_update(update.clone()).await?;
+
+    Ok(update)
+  }
+}

bin/core/src/api/execute/sync.rs

@@ -77,10 +77,8 @@ impl Resolve<ExecuteArgs> for RunSync {
     };
 
     // get the action state for the sync (or insert default).
-    let action_state = action_states()
-      .resource_sync
-      .get_or_insert_default(&sync.id)
-      .await;
+    let action_state =
+      action_states().sync.get_or_insert_default(&sync.id).await;
 
     // This will set action state back to default when dropped.
     // Will also check to ensure sync not already busy before updating.

bin/core/src/api/read/action.rs

@@ -131,8 +131,8 @@ impl Resolve<ReadArgs> for GetActionsSummary {
           .unwrap_or_default()
           .get()?,
       ) {
-        (_, action_states) if action_states.running => {
-          res.running += 1;
+        (_, action_states) if action_states.running > 0 => {
+          res.running += action_states.running;
         }
         (ActionState::Ok, _) => res.ok += 1,
         (ActionState::Failed, _) => res.failed += 1,

bin/core/src/api/read/server.rs

@@ -71,12 +71,24 @@ impl Resolve<ReadArgs> for GetServersSummary {
       &[],
     )
     .await?;
+
+    let core_version = env!("CARGO_PKG_VERSION");
     let mut res = GetServersSummaryResponse::default();
+
     for server in servers {
       res.total += 1;
       match server.info.state {
         ServerState::Ok => {
-          res.healthy += 1;
+          // Check for version mismatch
+          let has_version_mismatch = !server.info.version.is_empty()
+            && server.info.version != "Unknown"
+            && server.info.version != core_version;
+
+          if has_version_mismatch {
+            res.warning += 1;
+          } else {
+            res.healthy += 1;
+          }
         }
         ServerState::NotOk => {
           res.unhealthy += 1;

bin/core/src/api/read/sync.rs

@@ -93,7 +93,7 @@ impl Resolve<ReadArgs> for GetResourceSyncActionState {
     )
     .await?;
     let action_state = action_states()
-      .resource_sync
+      .sync
       .get(&sync.id)
       .await
       .unwrap_or_default()
@@ -138,7 +138,7 @@ impl Resolve<ReadArgs> for GetResourceSyncsSummary {
         continue;
       }
       if action_states
-        .resource_sync
+        .sync
         .get(&resource_sync.id)
         .await
         .unwrap_or_default()

bin/core/src/api/write/action.rs

@@ -16,10 +16,7 @@ impl Resolve<WriteArgs> for CreateAction {
     self,
     WriteArgs { user }: &WriteArgs,
   ) -> serror::Result<Action> {
-    Ok(
-      resource::create::<Action>(&self.name, self.config, user)
-        .await?,
-    )
+    resource::create::<Action>(&self.name, self.config, user).await
   }
 }
 
@@ -35,10 +32,7 @@ impl Resolve<WriteArgs> for CopyAction {
       PermissionLevel::Write.into(),
     )
     .await?;
-    Ok(
-      resource::create::<Action>(&self.name, config.into(), user)
-        .await?,
-    )
+    resource::create::<Action>(&self.name, config.into(), user).await
   }
 }
 

bin/core/src/api/write/alerter.rs

@@ -16,10 +16,7 @@ impl Resolve<WriteArgs> for CreateAlerter {
     self,
     WriteArgs { user }: &WriteArgs,
   ) -> serror::Result<Alerter> {
-    Ok(
-      resource::create::<Alerter>(&self.name, self.config, user)
-        .await?,
-    )
+    resource::create::<Alerter>(&self.name, self.config, user).await
   }
 }
 
@@ -35,10 +32,7 @@ impl Resolve<WriteArgs> for CopyAlerter {
       PermissionLevel::Write.into(),
     )
     .await?;
-    Ok(
-      resource::create::<Alerter>(&self.name, config.into(), user)
-        .await?,
-    )
+    resource::create::<Alerter>(&self.name, config.into(), user).await
   }
 }
 

bin/core/src/api/write/build.rs

@@ -50,10 +50,7 @@ impl Resolve<WriteArgs> for CreateBuild {
     self,
     WriteArgs { user }: &WriteArgs,
   ) -> serror::Result<Build> {
-    Ok(
-      resource::create::<Build>(&self.name, self.config, user)
-        .await?,
-    )
+    resource::create::<Build>(&self.name, self.config, user).await
   }
 }
 
@@ -71,10 +68,7 @@ impl Resolve<WriteArgs> for CopyBuild {
     .await?;
     // reset version to 0.0.0
     config.version = Default::default();
-    Ok(
-      resource::create::<Build>(&self.name, config.into(), user)
-        .await?,
-    )
+    resource::create::<Build>(&self.name, config.into(), user).await
   }
 }
 
@@ -186,7 +180,7 @@ async fn write_dockerfile_contents_git(
 ) -> serror::Result<Update> {
   let WriteBuildFileContents { build: _, contents } = req;
 
-  let mut clone_args: RepoExecutionArgs = if !build
+  let mut repo_args: RepoExecutionArgs = if !build
     .config
     .files_on_host
     && !build.config.linked_repo.is_empty()
@@ -196,8 +190,8 @@ async fn write_dockerfile_contents_git(
   } else {
     (&build).into()
   };
-  let root = clone_args.unique_path(&core_config().repo_directory)?;
-  clone_args.destination = Some(root.display().to_string());
+  let root = repo_args.unique_path(&core_config().repo_directory)?;
+  repo_args.destination = Some(root.display().to_string());
 
   let build_path = build
     .config
@@ -220,11 +214,11 @@ async fn write_dockerfile_contents_git(
     })?;
   }
 
-  let access_token = if let Some(account) = &clone_args.account {
-    git_token(&clone_args.provider, account, |https| clone_args.https = https)
+  let access_token = if let Some(account) = &repo_args.account {
+    git_token(&repo_args.provider, account, |https| repo_args.https = https)
     .await
     .with_context(
-      || format!("Failed to get git token in call to db. Stopping run. | {} | {account}", clone_args.provider),
+      || format!("Failed to get git token in call to db. Stopping run. | {} | {account}", repo_args.provider),
     )?
   } else {
     None
@@ -235,7 +229,7 @@ async fn write_dockerfile_contents_git(
   if !root.join(".git").exists() {
     git::init_folder_as_repo(
       &root,
-      &clone_args,
+      &repo_args,
       access_token.as_deref(),
       &mut update.logs,
     )
@@ -249,9 +243,11 @@ async fn write_dockerfile_contents_git(
     }
   }
 
+  // Save this for later -- repo_args moved next.
+  let branch = repo_args.branch.clone();
   // Pull latest changes to repo to ensure linear commit history
   match git::pull_or_clone(
-    clone_args,
+    repo_args,
     &core_config().repo_directory,
     access_token,
   )
@@ -298,7 +294,7 @@ async fn write_dockerfile_contents_git(
     &format!("{}: Commit Dockerfile", args.user.username),
     &root,
     &build_path.join(&dockerfile_path),
-    &build.config.branch,
+    &branch,
   )
   .await;
 

bin/core/src/api/write/builder.rs

@@ -16,10 +16,7 @@ impl Resolve<WriteArgs> for CreateBuilder {
     self,
     WriteArgs { user }: &WriteArgs,
   ) -> serror::Result<Builder> {
-    Ok(
-      resource::create::<Builder>(&self.name, self.config, user)
-        .await?,
-    )
+    resource::create::<Builder>(&self.name, self.config, user).await
   }
 }
 
@@ -35,10 +32,7 @@ impl Resolve<WriteArgs> for CopyBuilder {
       PermissionLevel::Write.into(),
     )
     .await?;
-    Ok(
-      resource::create::<Builder>(&self.name, config.into(), user)
-        .await?,
-    )
+    resource::create::<Builder>(&self.name, config.into(), user).await
   }
 }
 

bin/core/src/api/write/deployment.rs

@@ -38,10 +38,8 @@ impl Resolve<WriteArgs> for CreateDeployment {
     self,
     WriteArgs { user }: &WriteArgs,
   ) -> serror::Result<Deployment> {
-    Ok(
-      resource::create::<Deployment>(&self.name, self.config, user)
-        .await?,
-    )
+    resource::create::<Deployment>(&self.name, self.config, user)
+      .await
   }
 }
 
@@ -58,10 +56,8 @@ impl Resolve<WriteArgs> for CopyDeployment {
         PermissionLevel::Read.into(),
       )
       .await?;
-    Ok(
-      resource::create::<Deployment>(&self.name, config.into(), user)
-        .await?,
-    )
+    resource::create::<Deployment>(&self.name, config.into(), user)
+      .await
   }
 }
 
@@ -153,10 +149,7 @@ impl Resolve<WriteArgs> for CreateDeploymentFromContainer {
       });
     }
 
-    Ok(
-      resource::create::<Deployment>(&self.name, config, user)
-        .await?,
-    )
+    resource::create::<Deployment>(&self.name, config, user).await
   }
 }
 

bin/core/src/api/write/procedure.rs

@@ -16,10 +16,7 @@ impl Resolve<WriteArgs> for CreateProcedure {
     self,
     WriteArgs { user }: &WriteArgs,
   ) -> serror::Result<CreateProcedureResponse> {
-    Ok(
-      resource::create::<Procedure>(&self.name, self.config, user)
-        .await?,
-    )
+    resource::create::<Procedure>(&self.name, self.config, user).await
   }
 }
 
@@ -36,10 +33,8 @@ impl Resolve<WriteArgs> for CopyProcedure {
         PermissionLevel::Write.into(),
       )
       .await?;
-    Ok(
-      resource::create::<Procedure>(&self.name, config.into(), user)
-        .await?,
-    )
+    resource::create::<Procedure>(&self.name, config.into(), user)
+      .await
   }
 }
 

bin/core/src/api/write/repo.rs

@@ -42,7 +42,7 @@ impl Resolve<WriteArgs> for CreateRepo {
     self,
     WriteArgs { user }: &WriteArgs,
   ) -> serror::Result<Repo> {
-    Ok(resource::create::<Repo>(&self.name, self.config, user).await?)
+    resource::create::<Repo>(&self.name, self.config, user).await
   }
 }
 
@@ -58,10 +58,7 @@ impl Resolve<WriteArgs> for CopyRepo {
       PermissionLevel::Read.into(),
     )
     .await?;
-    Ok(
-      resource::create::<Repo>(&self.name, config.into(), user)
-        .await?,
-    )
+    resource::create::<Repo>(&self.name, config.into(), user).await
   }
 }
 

bin/core/src/api/write/server.rs

@@ -30,10 +30,7 @@ impl Resolve<WriteArgs> for CreateServer {
     self,
     WriteArgs { user }: &WriteArgs,
   ) -> serror::Result<Server> {
-    Ok(
-      resource::create::<Server>(&self.name, self.config, user)
-        .await?,
-    )
+    resource::create::<Server>(&self.name, self.config, user).await
   }
 }
 
@@ -49,10 +46,8 @@ impl Resolve<WriteArgs> for CopyServer {
       PermissionLevel::Read.into(),
     )
     .await?;
-    Ok(
-      resource::create::<Server>(&self.name, config.into(), user)
-        .await?,
-    )
+
+    resource::create::<Server>(&self.name, config.into(), user).await
   }
 }
 

bin/core/src/api/write/stack.rs

@@ -51,10 +51,7 @@ impl Resolve<WriteArgs> for CreateStack {
     self,
     WriteArgs { user }: &WriteArgs,
   ) -> serror::Result<Stack> {
-    Ok(
-      resource::create::<Stack>(&self.name, self.config, user)
-        .await?,
-    )
+    resource::create::<Stack>(&self.name, self.config, user).await
   }
 }
 
@@ -70,10 +67,8 @@ impl Resolve<WriteArgs> for CopyStack {
       PermissionLevel::Read.into(),
     )
     .await?;
-    Ok(
-      resource::create::<Stack>(&self.name, config.into(), user)
-        .await?,
-    )
+
+    resource::create::<Stack>(&self.name, config.into(), user).await
   }
 }
 
@@ -284,6 +279,8 @@ async fn write_stack_file_contents_git(
     }
   }
 
+  // Save this for later -- repo_args moved next.
+  let branch = repo_args.branch.clone();
   // Pull latest changes to repo to ensure linear commit history
   match git::pull_or_clone(
     repo_args,
@@ -334,7 +331,7 @@ async fn write_stack_file_contents_git(
     &format!("{username}: Write Stack File"),
     &root,
     &file_path,
-    &stack.config.branch,
+    &branch,
   )
   .await;
 
@@ -429,7 +426,7 @@ impl Resolve<WriteArgs> for RefreshStackCache {
         let GetComposeContentsOnHostResponse { contents, errors } =
           match periphery_client(&server)?
             .request(GetComposeContentsOnHost {
-              file_paths: stack.file_paths().to_vec(),
+              file_paths: stack.all_file_dependencies(),
               name: stack.name.clone(),
               run_directory: stack.config.run_directory.clone(),
             })
@@ -451,6 +448,10 @@ impl Resolve<WriteArgs> for RefreshStackCache {
         let mut services = Vec::new();
 
         for contents in &contents {
+          // Don't include additional files in service parsing
+          if !stack.is_compose_file(&contents.path) {
+            continue;
+          }
           if let Err(e) = extract_services_into_res(
             &project_name,
             &contents.contents,
@@ -489,6 +490,10 @@ impl Resolve<WriteArgs> for RefreshStackCache {
       let mut services = Vec::new();
 
       for contents in &remote_contents {
+        // Don't include additional files in service parsing
+        if !stack.is_compose_file(&contents.path) {
+          continue;
+        }
         if let Err(e) = extract_services_into_res(
           &project_name,
           &contents.contents,

bin/core/src/api/write/sync.rs

@@ -68,10 +68,8 @@ impl Resolve<WriteArgs> for CreateResourceSync {
     self,
     WriteArgs { user }: &WriteArgs,
   ) -> serror::Result<ResourceSync> {
-    Ok(
-      resource::create::<ResourceSync>(&self.name, self.config, user)
-        .await?,
-    )
+    resource::create::<ResourceSync>(&self.name, self.config, user)
+      .await
   }
 }
 
@@ -88,14 +86,8 @@ impl Resolve<WriteArgs> for CopyResourceSync {
         PermissionLevel::Write.into(),
       )
       .await?;
-    Ok(
-      resource::create::<ResourceSync>(
-        &self.name,
-        config.into(),
-        user,
-      )
-      .await?,
-    )
+    resource::create::<ResourceSync>(&self.name, config.into(), user)
+      .await
   }
 }
 
@@ -323,6 +315,8 @@ async fn write_sync_file_contents_git(
     }
   }
 
+  // Save this for later -- repo_args moved next.
+  let branch = repo_args.branch.clone();
   // Pull latest changes to repo to ensure linear commit history
   match git::pull_or_clone(
     repo_args,
@@ -373,7 +367,7 @@ async fn write_sync_file_contents_git(
     &format!("{}: Commit Resource File", args.user.username),
     &root,
     &resource_path.join(&file_path),
-    &sync.config.branch,
+    &branch,
   )
   .await;
 

bin/core/src/api/write/tag.rs

@@ -13,9 +13,12 @@ use komodo_client::{
     server::Server, stack::Stack, sync::ResourceSync, tag::Tag,
   },
 };
+use reqwest::StatusCode;
 use resolver_api::Resolve;
+use serror::AddStatusCodeError;
 
 use crate::{
+  config::core_config,
   helpers::query::{get_tag, get_tag_check_owner},
   resource,
   state::db_client,
@@ -29,8 +32,18 @@ impl Resolve<WriteArgs> for CreateTag {
     self,
     WriteArgs { user }: &WriteArgs,
   ) -> serror::Result<Tag> {
+    if core_config().disable_non_admin_create && !user.admin {
+      return Err(
+        anyhow!("Non admins cannot create tags")
+          .status_code(StatusCode::FORBIDDEN),
+      );
+    }
+
     if ObjectId::from_str(&self.name).is_ok() {
-      return Err(anyhow!("tag name cannot be ObjectId").into());
+      return Err(
+        anyhow!("Tag name cannot be ObjectId")
+          .status_code(StatusCode::BAD_REQUEST),
+      );
     }
 
     let mut tag = Tag {

bin/core/src/api/write/user.rs

@@ -32,7 +32,7 @@ impl Resolve<WriteArgs> for CreateLocalUser {
     if !admin.admin {
       return Err(
         anyhow!("This method is admin-only.")
-          .status_code(StatusCode::UNAUTHORIZED),
+          .status_code(StatusCode::FORBIDDEN),
       );
     }
 
@@ -183,7 +183,7 @@ impl Resolve<WriteArgs> for DeleteUser {
     if !admin.admin {
       return Err(
         anyhow!("This method is admin-only.")
-          .status_code(StatusCode::UNAUTHORIZED),
+          .status_code(StatusCode::FORBIDDEN),
       );
     }
     if admin.username == self.user || admin.id == self.user {
@@ -220,6 +220,14 @@ impl Resolve<WriteArgs> for DeleteUser {
       .delete_one(query)
       .await
       .context("Failed to delete user from database")?;
+    // Also remove user id from all user groups
+    if let Err(e) = db
+      .user_groups
+      .update_many(doc! {}, doc! { "$pull": { "users": &user.id } })
+      .await
+    {
+      warn!("Failed to remove deleted user from user groups | {e:?}");
+    };
     Ok(user)
   }
 }

bin/core/src/api/write/user_group.rs

@@ -10,7 +10,9 @@ use komodo_client::{
   api::write::*,
   entities::{komodo_timestamp, user_group::UserGroup},
 };
+use reqwest::StatusCode;
 use resolver_api::Resolve;
+use serror::AddStatusCodeError;
 
 use crate::state::db_client;
 
@@ -23,7 +25,10 @@ impl Resolve<WriteArgs> for CreateUserGroup {
     WriteArgs { user: admin }: &WriteArgs,
   ) -> serror::Result<UserGroup> {
     if !admin.admin {
-      return Err(anyhow!("This call is admin-only").into());
+      return Err(
+        anyhow!("This call is admin-only")
+          .status_code(StatusCode::FORBIDDEN),
+      );
     }
     let user_group = UserGroup {
       name: self.name,
@@ -58,7 +63,10 @@ impl Resolve<WriteArgs> for RenameUserGroup {
     WriteArgs { user: admin }: &WriteArgs,
   ) -> serror::Result<UserGroup> {
     if !admin.admin {
-      return Err(anyhow!("This call is admin-only").into());
+      return Err(
+        anyhow!("This call is admin-only")
+          .status_code(StatusCode::FORBIDDEN),
+      );
     }
     let db = db_client();
     update_one_by_id(
@@ -84,7 +92,10 @@ impl Resolve<WriteArgs> for DeleteUserGroup {
     WriteArgs { user: admin }: &WriteArgs,
   ) -> serror::Result<UserGroup> {
     if !admin.admin {
-      return Err(anyhow!("This call is admin-only").into());
+      return Err(
+        anyhow!("This call is admin-only")
+          .status_code(StatusCode::FORBIDDEN),
+      );
     }
 
     let db = db_client();
@@ -117,7 +128,10 @@ impl Resolve<WriteArgs> for AddUserToUserGroup {
     WriteArgs { user: admin }: &WriteArgs,
   ) -> serror::Result<UserGroup> {
     if !admin.admin {
-      return Err(anyhow!("This call is admin-only").into());
+      return Err(
+        anyhow!("This call is admin-only")
+          .status_code(StatusCode::FORBIDDEN),
+      );
     }
 
     let db = db_client();
@@ -161,7 +175,10 @@ impl Resolve<WriteArgs> for RemoveUserFromUserGroup {
     WriteArgs { user: admin }: &WriteArgs,
   ) -> serror::Result<UserGroup> {
     if !admin.admin {
-      return Err(anyhow!("This call is admin-only").into());
+      return Err(
+        anyhow!("This call is admin-only")
+          .status_code(StatusCode::FORBIDDEN),
+      );
     }
 
     let db = db_client();
@@ -205,7 +222,10 @@ impl Resolve<WriteArgs> for SetUsersInUserGroup {
     WriteArgs { user: admin }: &WriteArgs,
   ) -> serror::Result<UserGroup> {
     if !admin.admin {
-      return Err(anyhow!("This call is admin-only").into());
+      return Err(
+        anyhow!("This call is admin-only")
+          .status_code(StatusCode::FORBIDDEN),
+      );
     }
 
     let db = db_client();
@@ -252,7 +272,10 @@ impl Resolve<WriteArgs> for SetEveryoneUserGroup {
     WriteArgs { user: admin }: &WriteArgs,
   ) -> serror::Result<UserGroup> {
     if !admin.admin {
-      return Err(anyhow!("This call is admin-only").into());
+      return Err(
+        anyhow!("This call is admin-only")
+          .status_code(StatusCode::FORBIDDEN),
+      );
     }
 
     let db = db_client();

bin/core/src/api/write/variable.rs

@@ -4,7 +4,9 @@ use komodo_client::{
   api::write::*,
   entities::{Operation, ResourceTarget, variable::Variable},
 };
+use reqwest::StatusCode;
 use resolver_api::Resolve;
+use serror::AddStatusCodeError;
 
 use crate::{
   helpers::{
@@ -22,6 +24,13 @@ impl Resolve<WriteArgs> for CreateVariable {
     self,
     WriteArgs { user }: &WriteArgs,
   ) -> serror::Result<CreateVariableResponse> {
+    if !user.admin {
+      return Err(
+        anyhow!("Only admins can create variables")
+          .status_code(StatusCode::FORBIDDEN),
+      );
+    }
+
     let CreateVariable {
       name,
       value,
@@ -29,10 +38,6 @@ impl Resolve<WriteArgs> for CreateVariable {
       is_secret,
     } = self;
 
-    if !user.admin {
-      return Err(anyhow!("only admins can create variables").into());
-    }
-
     let variable = Variable {
       name,
       value,
@@ -44,7 +49,7 @@ impl Resolve<WriteArgs> for CreateVariable {
       .variables
       .insert_one(&variable)
       .await
-      .context("failed to create variable on db")?;
+      .context("Failed to create variable on db")?;
 
     let mut update = make_update(
       ResourceTarget::system(),
@@ -69,7 +74,10 @@ impl Resolve<WriteArgs> for UpdateVariableValue {
     WriteArgs { user }: &WriteArgs,
   ) -> serror::Result<UpdateVariableValueResponse> {
     if !user.admin {
-      return Err(anyhow!("only admins can update variables").into());
+      return Err(
+        anyhow!("Only admins can update variables")
+          .status_code(StatusCode::FORBIDDEN),
+      );
     }
 
     let UpdateVariableValue { name, value } = self;
@@ -87,7 +95,7 @@ impl Resolve<WriteArgs> for UpdateVariableValue {
         doc! { "$set": { "value": &value } },
       )
       .await
-      .context("failed to update variable value on db")?;
+      .context("Failed to update variable value on db")?;
 
     let mut update = make_update(
       ResourceTarget::system(),
@@ -107,7 +115,7 @@ impl Resolve<WriteArgs> for UpdateVariableValue {
       )
     };
 
-    update.push_simple_log("update variable value", log);
+    update.push_simple_log("Update Variable Value", log);
     update.finalize();
 
     add_update(update).await?;
@@ -123,7 +131,10 @@ impl Resolve<WriteArgs> for UpdateVariableDescription {
     WriteArgs { user }: &WriteArgs,
   ) -> serror::Result<UpdateVariableDescriptionResponse> {
     if !user.admin {
-      return Err(anyhow!("only admins can update variables").into());
+      return Err(
+        anyhow!("Only admins can update variables")
+          .status_code(StatusCode::FORBIDDEN),
+      );
     }
     db_client()
       .variables
@@ -132,7 +143,7 @@ impl Resolve<WriteArgs> for UpdateVariableDescription {
         doc! { "$set": { "description": &self.description } },
       )
       .await
-      .context("failed to update variable description on db")?;
+      .context("Failed to update variable description on db")?;
     Ok(get_variable(&self.name).await?)
   }
 }
@@ -144,7 +155,10 @@ impl Resolve<WriteArgs> for UpdateVariableIsSecret {
     WriteArgs { user }: &WriteArgs,
   ) -> serror::Result<UpdateVariableIsSecretResponse> {
     if !user.admin {
-      return Err(anyhow!("only admins can update variables").into());
+      return Err(
+        anyhow!("Only admins can update variables")
+          .status_code(StatusCode::FORBIDDEN),
+      );
     }
     db_client()
       .variables
@@ -153,7 +167,7 @@ impl Resolve<WriteArgs> for UpdateVariableIsSecret {
         doc! { "$set": { "is_secret": self.is_secret } },
       )
       .await
-      .context("failed to update variable is secret on db")?;
+      .context("Failed to update variable is secret on db")?;
     Ok(get_variable(&self.name).await?)
   }
 }
@@ -164,14 +178,17 @@ impl Resolve<WriteArgs> for DeleteVariable {
     WriteArgs { user }: &WriteArgs,
   ) -> serror::Result<DeleteVariableResponse> {
     if !user.admin {
-      return Err(anyhow!("only admins can delete variables").into());
+      return Err(
+        anyhow!("Only admins can delete variables")
+          .status_code(StatusCode::FORBIDDEN),
+      );
     }
     let variable = get_variable(&self.name).await?;
     db_client()
       .variables
       .delete_one(doc! { "name": &self.name })
       .await
-      .context("failed to delete variable on db")?;
+      .context("Failed to delete variable on db")?;
 
     let mut update = make_update(
       ResourceTarget::system(),
@@ -180,7 +197,7 @@ impl Resolve<WriteArgs> for DeleteVariable {
     );
 
     update
-      .push_simple_log("delete variable", format!("{variable:#?}"));
+      .push_simple_log("Delete Variable", format!("{variable:#?}"));
     update.finalize();
 
     add_update(update).await?;

bin/core/src/auth/google/client.rs

@@ -187,8 +187,8 @@ impl GoogleOauthClient {
       Ok(body)
     } else {
       let text = res.text().await.context(format!(
-                "method: POST | status: {status} | failed to get response text"
-            ))?;
+        "method: POST | status: {status} | failed to get response text"
+      ))?;
       Err(anyhow!("method: POST | status: {status} | text: {text}"))
     }
   }
@@ -207,5 +207,6 @@ pub struct GoogleUser {
   #[serde(rename = "sub")]
   pub id: String,
   pub email: String,
+  #[serde(default)]
   pub picture: String,
 }

bin/core/src/config.rs

@@ -250,6 +250,7 @@ pub fn core_config() -> &'static CoreConfig {
           .unwrap_or(config.logging.opentelemetry_service_name),
       },
       pretty_startup_config: env.komodo_pretty_startup_config.unwrap_or(config.pretty_startup_config),
+      unsafe_unsanitized_startup_config: env.komodo_unsafe_unsanitized_startup_config.unwrap_or(config.unsafe_unsanitized_startup_config),
       internet_interface: env.komodo_internet_interface.unwrap_or(config.internet_interface),
       ssl_enabled: env.komodo_ssl_enabled.unwrap_or(config.ssl_enabled),
       ssl_key_file: env.komodo_ssl_key_file.unwrap_or(config.ssl_key_file),

bin/core/src/helpers/action_state.rs

@@ -16,17 +16,16 @@ use super::cache::Cache;
 
 #[derive(Default)]
 pub struct ActionStates {
-  pub build: Cache<String, Arc<ActionState<BuildActionState>>>,
+  pub server: Cache<String, Arc<ActionState<ServerActionState>>>,
+  pub stack: Cache<String, Arc<ActionState<StackActionState>>>,
   pub deployment:
     Cache<String, Arc<ActionState<DeploymentActionState>>>,
-  pub server: Cache<String, Arc<ActionState<ServerActionState>>>,
+  pub build: Cache<String, Arc<ActionState<BuildActionState>>>,
   pub repo: Cache<String, Arc<ActionState<RepoActionState>>>,
   pub procedure:
     Cache<String, Arc<ActionState<ProcedureActionState>>>,
   pub action: Cache<String, Arc<ActionState<ActionActionState>>>,
-  pub resource_sync:
-    Cache<String, Arc<ActionState<ResourceSyncActionState>>>,
-  pub stack: Cache<String, Arc<ActionState<StackActionState>>>,
+  pub sync: Cache<String, Arc<ActionState<ResourceSyncActionState>>>,
 }
 
 /// Need to be able to check "busy" with write lock acquired.
@@ -62,17 +61,33 @@ impl<States: Default + Busy + Copy + Send + 'static>
   /// Returns a guard that returns the states to default (not busy) when dropped.
   pub fn update(
     &self,
-    handler: impl Fn(&mut States),
+    update_fn: impl Fn(&mut States),
+  ) -> anyhow::Result<UpdateGuard<'_, States>> {
+    self.update_custom(
+      update_fn,
+      |states| *states = Default::default(),
+      true,
+    )
+  }
+
+  /// Will acquire lock, optionally check busy, and if not will
+  /// run the provided update function on the states.
+  /// Returns a guard that calls the provided return_fn when dropped.
+  pub fn update_custom(
+    &self,
+    update_fn: impl Fn(&mut States),
+    return_fn: impl Fn(&mut States) + Send + 'static,
+    busy_check: bool,
   ) -> anyhow::Result<UpdateGuard<'_, States>> {
     let mut lock = self
       .0
       .lock()
-      .map_err(|e| anyhow!("action state lock poisoned | {e:?}"))?;
-    if lock.busy() {
-      return Err(anyhow!("resource is busy"));
+      .map_err(|e| anyhow!("Action state lock poisoned | {e:?}"))?;
+    if busy_check && lock.busy() {
+      return Err(anyhow!("Resource is busy"));
     }
-    handler(&mut *lock);
-    Ok(UpdateGuard(&self.0))
+    update_fn(&mut *lock);
+    Ok(UpdateGuard(&self.0, Box::new(return_fn)))
   }
 }
 
@@ -82,6 +97,7 @@ impl<States: Default + Busy + Copy + Send + 'static>
 /// user could drop UpdateGuard.
 pub struct UpdateGuard<'a, States: Default + Send + 'static>(
   &'a Mutex<States>,
+  Box<dyn Fn(&mut States) + Send>,
 );
 
 impl<States: Default + Send + 'static> Drop
@@ -95,6 +111,6 @@ impl<States: Default + Send + 'static> Drop
         return;
       }
     };
-    *lock = States::default();
+    self.1(&mut *lock);
   }
 }

bin/core/src/helpers/procedure.rs

@@ -1101,6 +1101,23 @@ async fn execute_execution(
       )
       .await?
     }
+    Execution::RunStackService(req) => {
+      let req = ExecuteRequest::RunStackService(req);
+      let update = init_execution_update(&req, &user).await?;
+      let ExecuteRequest::RunStackService(req) = req else {
+        unreachable!()
+      };
+      let update_id = update.id.clone();
+      handle_resolve_result(
+        req
+          .resolve(&ExecuteArgs { user, update })
+          .await
+          .map_err(|e| e.error)
+          .context("Failed at RunStackService"),
+        &update_id,
+      )
+      .await?
+    }
     Execution::BatchDestroyStack(_) => {
       // All batch executions must be expanded in `execute_stage`
       return Err(anyhow!(
@@ -1124,6 +1141,23 @@ async fn execute_execution(
       )
       .await?
     }
+    Execution::SendAlert(req) => {
+      let req = ExecuteRequest::SendAlert(req);
+      let update = init_execution_update(&req, &user).await?;
+      let ExecuteRequest::SendAlert(req) = req else {
+        unreachable!()
+      };
+      let update_id = update.id.clone();
+      handle_resolve_result(
+        req
+          .resolve(&ExecuteArgs { user, update })
+          .await
+          .map_err(|e| e.error)
+          .context("Failed at SendAlert"),
+        &update_id,
+      )
+      .await?
+    }
     Execution::ClearRepoCache(req) => {
       let req = ExecuteRequest::ClearRepoCache(req);
       let update = init_execution_update(&req, &user).await?;

bin/core/src/helpers/query.rs

@@ -13,26 +13,31 @@ use database::mungos::{
     options::FindOneOptions,
   },
 };
-use komodo_client::entities::{
-  Operation, ResourceTarget, ResourceTargetVariant,
-  action::{Action, ActionState},
-  alerter::Alerter,
-  build::Build,
-  builder::Builder,
-  deployment::{Deployment, DeploymentState},
-  docker::container::{ContainerListItem, ContainerStateStatusEnum},
-  permission::{PermissionLevel, PermissionLevelAndSpecifics},
-  procedure::{Procedure, ProcedureState},
-  repo::Repo,
-  server::{Server, ServerState},
-  stack::{Stack, StackServiceNames, StackState},
-  stats::SystemInformation,
-  sync::ResourceSync,
-  tag::Tag,
-  update::Update,
-  user::{User, admin_service_user},
-  user_group::UserGroup,
-  variable::Variable,
+use komodo_client::{
+  busy::Busy,
+  entities::{
+    Operation, ResourceTarget, ResourceTargetVariant,
+    action::{Action, ActionState},
+    alerter::Alerter,
+    build::Build,
+    builder::Builder,
+    deployment::{Deployment, DeploymentState},
+    docker::container::{
+      ContainerListItem, ContainerStateStatusEnum,
+    },
+    permission::{PermissionLevel, PermissionLevelAndSpecifics},
+    procedure::{Procedure, ProcedureState},
+    repo::Repo,
+    server::{Server, ServerState},
+    stack::{Stack, StackServiceNames, StackState},
+    stats::SystemInformation,
+    sync::ResourceSync,
+    tag::Tag,
+    update::Update,
+    user::{User, admin_service_user},
+    user_group::UserGroup,
+    variable::Variable,
+  },
 };
 use periphery_client::api::stats;
 use tokio::sync::Mutex;
@@ -467,7 +472,7 @@ pub async fn get_action_state(id: &String) -> ActionState {
     .action
     .get(id)
     .await
-    .map(|s| s.get().map(|s| s.running))
+    .map(|s| s.get().map(|s| s.busy()))
     .transpose()
     .ok()
     .flatten()
@@ -483,7 +488,7 @@ pub async fn get_procedure_state(id: &String) -> ProcedureState {
     .procedure
     .get(id)
     .await
-    .map(|s| s.get().map(|s| s.running))
+    .map(|s| s.get().map(|s| s.busy()))
     .transpose()
     .ok()
     .flatten()

bin/core/src/helpers/update.rs

@@ -492,6 +492,13 @@ pub async fn init_execution_update(
       return Ok(Default::default());
     }
 
+    ExecuteRequest::RunStackService(data) => (
+      Operation::RunStackService,
+      ResourceTarget::Stack(
+        resource::get::<Stack>(&data.stack).await?.id,
+      ),
+    ),
+
     // Alerter
     ExecuteRequest::TestAlerter(data) => (
       Operation::TestAlerter,
@@ -499,6 +506,9 @@ pub async fn init_execution_update(
         resource::get::<Alerter>(&data.alerter).await?.id,
       ),
     ),
+    ExecuteRequest::SendAlert(_) => {
+      (Operation::SendAlert, ResourceTarget::system())
+    }
 
     // Maintenance
     ExecuteRequest::ClearRepoCache(_data) => {

bin/core/src/main.rs

@@ -45,10 +45,14 @@ async fn app() -> anyhow::Result<()> {
 
   info!("Komodo Core version: v{}", env!("CARGO_PKG_VERSION"));
 
-  if core_config().pretty_startup_config {
-    info!("{:#?}", config.sanitized());
-  } else {
-    info!("{:?}", config.sanitized());
+  match (
+    config.pretty_startup_config,
+    config.unsafe_unsanitized_startup_config,
+  ) {
+    (true, true) => info!("{:#?}", config),
+    (true, false) => info!("{:#?}", config.sanitized()),
+    (false, true) => info!("{:?}", config),
+    (false, false) => info!("{:?}", config.sanitized()),
   }
 
   // Init jwt client to crash on failure

bin/core/src/monitor/alert/server.rs

@@ -178,6 +178,77 @@ pub async fn alert_servers(
         ),
     }
 
+    // ===================
+    // SERVER VERSION MISMATCH
+    // ===================
+    let core_version = env!("CARGO_PKG_VERSION");
+    let has_version_mismatch = server_status.state == ServerState::Ok
+      && !server_status.version.is_empty()
+      && server_status.version != "Unknown"
+      && server_status.version != core_version;
+
+    let version_alert = server_alerts.as_ref().and_then(|alerts| {
+      alerts.get(&AlertDataVariant::ServerVersionMismatch)
+    });
+
+    match (has_version_mismatch, version_alert) {
+      (true, None) => {
+        // Only open version mismatch alert if not in maintenance and buffer is ready
+        if !in_maintenance
+          && buffer.ready_to_open(
+            server_status.id.clone(),
+            AlertDataVariant::ServerVersionMismatch,
+          )
+        {
+          let alert = Alert {
+            id: Default::default(),
+            ts,
+            resolved: false,
+            resolved_ts: None,
+            level: SeverityLevel::Warning,
+            target: ResourceTarget::Server(server_status.id.clone()),
+            data: AlertData::ServerVersionMismatch {
+              id: server_status.id.clone(),
+              name: server.name.clone(),
+              region: optional_string(&server.config.region),
+              server_version: server_status.version.clone(),
+              core_version: core_version.to_string(),
+            },
+          };
+          // Use send_unreachable_alerts as a proxy for general server alerts
+          alerts_to_open
+            .push((alert, server.config.send_version_mismatch_alerts))
+        }
+      }
+      (true, Some(alert)) => {
+        // Update existing alert with current version info
+        let mut alert = alert.clone();
+        alert.data = AlertData::ServerVersionMismatch {
+          id: server_status.id.clone(),
+          name: server.name.clone(),
+          region: optional_string(&server.config.region),
+          server_version: server_status.version.clone(),
+          core_version: core_version.to_string(),
+        };
+        // Don't send notification for updates
+        alerts_to_update.push((alert, false));
+      }
+      (false, Some(alert)) => {
+        // Version is now correct, close the alert
+        alert_ids_to_close.push((
+          alert.clone(),
+          server.config.send_version_mismatch_alerts,
+        ));
+      }
+      (false, None) => {
+        // Reset buffer state when no mismatch and no alert
+        buffer.reset(
+          server_status.id.clone(),
+          AlertDataVariant::ServerVersionMismatch,
+        )
+      }
+    }
+
     let Some(health) = &server_status.health else {
       continue;
     };

bin/core/src/monitor/mod.rs

@@ -1,3 +1,5 @@
+use std::sync::{Arc, OnceLock};
+
 use async_timing_util::wait_until_timelength;
 use database::mungos::{find::find_collect, mongodb::bson::doc};
 use futures::future::join_all;
@@ -15,10 +17,11 @@ use komodo_client::entities::{
 };
 use periphery_client::api::{self, git::GetLatestCommit};
 use serror::Serror;
+use tokio::sync::Mutex;
 
 use crate::{
   config::core_config,
-  helpers::periphery_client,
+  helpers::{cache::Cache, periphery_client},
   monitor::{alert::check_alerts, record::record_server_stats},
   state::{db_client, deployment_status_cache, repo_status_cache},
 };
@@ -110,14 +113,47 @@ async fn refresh_server_cache(ts: i64) {
       }
     };
   let futures = servers.into_iter().map(|server| async move {
-    update_cache_for_server(&server).await;
+    update_cache_for_server(&server, false).await;
   });
   join_all(futures).await;
   tokio::join!(check_alerts(ts), record_server_stats(ts));
 }
 
+/// Makes sure cache for server doesn't update too frequently / simultaneously.
+/// If forced, will still block against simultaneous update.
+fn update_cache_for_server_controller()
+-> &'static Cache<String, Arc<Mutex<i64>>> {
+  static CACHE: OnceLock<Cache<String, Arc<Mutex<i64>>>> =
+    OnceLock::new();
+  CACHE.get_or_init(Default::default)
+}
+
+/// The background loop will call this with force: false,
+/// which exits early if the lock is busy or it was completed too recently.
+/// If force is true, it will wait on simultaneous calls, and will
+/// ignore the restriction on being completed too recently.
 #[instrument(level = "debug")]
-pub async fn update_cache_for_server(server: &Server) {
+pub async fn update_cache_for_server(server: &Server, force: bool) {
+  // Concurrency controller to ensure it isn't done too often
+  // when it happens in other contexts.
+  let controller = update_cache_for_server_controller()
+    .get_or_insert_default(&server.id)
+    .await;
+  let mut lock = match controller.try_lock() {
+    Ok(lock) => lock,
+    Err(_) if force => controller.lock().await,
+    Err(_) => return,
+  };
+
+  let now = komodo_timestamp();
+
+  // early return if called again sooner than 1s.
+  if !force && *lock > now - 1_000 {
+    return;
+  }
+
+  *lock = now;
+
   let (deployments, builds, repos, stacks) = tokio::join!(
     find_collect(
       &db_client().deployments,

bin/core/src/monitor/record.rs

@@ -21,6 +21,7 @@ pub async fn record_server_stats(ts: i64) {
         ts,
         sid: status.id.clone(),
         cpu_perc: stats.cpu_perc,
+        load_average: stats.load_average.clone(),
         mem_total_gb: stats.mem_total_gb,
         mem_used_gb: stats.mem_used_gb,
         disk_total_gb,

bin/core/src/monitor/resources.rs

@@ -81,7 +81,7 @@ pub async fn update_deployment_cache(
         // If image already has tag, leave it,
         // otherwise default the tag to latest
         if image.contains(':') {
-          image
+          image.to_string()
         } else {
           format!("{image}:latest")
         }
@@ -92,6 +92,9 @@ pub async fn update_deployment_cache(
       ..
     }) = &container
     {
+      // Docker will automatically strip `docker.io` from incoming image names re #468.
+      // Need to strip it in order to match by image name and find available updates.
+      let image = image.strip_prefix("docker.io/").unwrap_or(&image);
       images
         .iter()
         .find(|i| i.name == image)
@@ -250,20 +253,21 @@ pub async fn update_stack_cache(
           }
         }.is_match(&container.name)
       }).cloned();
-      // If image already has tag, leave it,
-      // otherwise default the tag to latest
-      let image = image.clone();
       let image = if image.contains(':') {
-        image
+        image.to_string()
       } else {
-        image + ":latest"
+        format!("{image}:latest")
       };
       let update_available = if let Some(ContainerListItem { image_id: Some(curr_image_id), .. }) = &container {
+        // Docker will automatically strip `docker.io` from incoming image names re #468.
+        // Need to strip it in order to match by image tag and find available update.
+        let image =
+          image.strip_prefix("docker.io/").unwrap_or(&image);
         images
-        .iter()
-        .find(|i| i.name == image)
-        .map(|i| &i.id != curr_image_id)
-        .unwrap_or_default()
+          .iter()
+          .find(|i| i.name == image)
+          .map(|i| &i.id != curr_image_id)
+          .unwrap_or_default()
       } else {
         false
       };

bin/core/src/network.rs

@@ -36,10 +36,10 @@ fn is_container_environment() -> bool {
   }
 
   // Check cgroup for container runtime indicators
-  if let Ok(content) = std::fs::read_to_string(CGROUP_FILE) {
-    if content.contains("docker") || content.contains("containerd") {
-      return true;
-    }
+  if let Ok(content) = std::fs::read_to_string(CGROUP_FILE)
+    && (content.contains("docker") || content.contains("containerd"))
+  {
+    return true;
   }
 
   false
@@ -142,7 +142,7 @@ async fn find_gateway(
   }
 
   let ip_cidr = ip_cidr.ok_or_else(|| anyhow!(
-        "Could not find IP address for interface '{}'. Ensure interface has a valid IPv4 address", 
+        "Could not find IP address for interface '{}'. Ensure interface has a valid IPv4 address",
         interface_name
     ))?;
 
@@ -167,14 +167,13 @@ async fn find_gateway(
       if line.contains("via") {
         let parts: Vec<&str> = line.split_whitespace().collect();
         if let Some(via_idx) = parts.iter().position(|&x| x == "via")
+          && let Some(&gateway) = parts.get(via_idx + 1)
         {
-          if let Some(&gateway) = parts.get(via_idx + 1) {
-            trace!(
-              "Found gateway {} for {} from routing table",
-              gateway, interface_name
-            );
-            return Ok(gateway.to_string());
-          }
+          trace!(
+            "Found gateway {} for {} from routing table",
+            gateway, interface_name
+          );
+          return Ok(gateway.to_string());
         }
       }
     }
@@ -206,14 +205,14 @@ async fn find_gateway(
           .output()
           .await;
 
-        if let Ok(output) = route_test {
-          if output.status.success() {
-            trace!(
-              "Gateway {} is reachable via {}",
-              gateway, interface_name
-            );
-            return Ok(gateway.to_string());
-          }
+        if let Ok(output) = route_test
+          && output.status.success()
+        {
+          trace!(
+            "Gateway {} is reachable via {}",
+            gateway, interface_name
+          );
+          return Ok(gateway.to_string());
         }
 
         // Fallback: assume .1 is gateway (Docker standard)
@@ -266,10 +265,10 @@ async fn set_default_gateway(
     .output()
     .await;
 
-  if let Ok(output) = remove_default {
-    if output.status.success() {
-      trace!("Removed existing default routes");
-    }
+  if let Ok(output) = remove_default
+    && output.status.success()
+  {
+    trace!("Removed existing default routes");
   }
 
   // Add new default route

bin/core/src/resource/build.rs

@@ -116,9 +116,11 @@ impl super::KomodoResource for Build {
         git_provider,
         repo,
         branch,
-        image_registry_domain: optional_string(
-          build.config.image_registry.domain,
-        ),
+        image_registry_domain: build
+          .config
+          .image_registry
+          .first()
+          .and_then(|r| optional_string(&r.domain)),
         built_hash: build.info.built_hash,
         latest_hash: build.info.latest_hash,
         state,

bin/core/src/resource/deployment.rs

@@ -188,7 +188,7 @@ impl super::KomodoResource for Deployment {
     else {
       return Ok(());
     };
-    update_cache_for_server(&server).await;
+    update_cache_for_server(&server, true).await;
     Ok(())
   }
 

bin/core/src/resource/mod.rs

@@ -34,8 +34,10 @@ use komodo_client::{
   parsers::parse_string_list,
 };
 use partial_derive2::{Diff, MaybeNone, PartialDiff};
+use reqwest::StatusCode;
 use resolver_api::Resolve;
 use serde::{Serialize, de::DeserializeOwned};
+use serror::AddStatusCodeError;
 
 use crate::{
   api::{read::ReadArgs, write::WriteArgs},
@@ -229,6 +231,12 @@ pub trait KomodoResource {
 pub async fn get<T: KomodoResource>(
   id_or_name: &str,
 ) -> anyhow::Result<Resource<T::Config, T::Info>> {
+  if id_or_name.is_empty() {
+    return Err(anyhow!(
+      "Cannot find {} with empty name / id",
+      T::resource_type()
+    ));
+  }
   T::coll()
     .find_one(id_or_name_filter(id_or_name))
     .await
@@ -452,22 +460,31 @@ pub async fn create<T: KomodoResource>(
   name: &str,
   mut config: T::PartialConfig,
   user: &User,
-) -> anyhow::Result<Resource<T::Config, T::Info>> {
+) -> serror::Result<Resource<T::Config, T::Info>> {
   if !T::user_can_create(user) {
-    return Err(anyhow!(
-      "User does not have permissions to create {}.",
-      T::resource_type()
-    ));
+    return Err(
+      anyhow!(
+        "User does not have permissions to create {}.",
+        T::resource_type()
+      )
+      .status_code(StatusCode::FORBIDDEN),
+    );
   }
 
   if name.is_empty() {
-    return Err(anyhow!("Must provide non-empty name for resource."));
+    return Err(
+      anyhow!("Must provide non-empty name for resource")
+        .status_code(StatusCode::BAD_REQUEST),
+    );
   }
 
   let name = T::validated_name(name);
 
   if ObjectId::from_str(&name).is_ok() {
-    return Err(anyhow!("valid ObjectIds cannot be used as names."));
+    return Err(
+      anyhow!("Valid ObjectIds cannot be used as names")
+        .status_code(StatusCode::BAD_REQUEST),
+    );
   }
 
   // Ensure an existing resource with same name doesn't already exist
@@ -483,7 +500,10 @@ pub async fn create<T: KomodoResource>(
   .into_iter()
   .any(|r| r.name == name)
   {
-    return Err(anyhow!("Must provide unique name for resource."));
+    return Err(
+      anyhow!("Resource with name '{}' already exists", name)
+        .status_code(StatusCode::CONFLICT),
+    );
   }
 
   let start_ts = komodo_timestamp();
@@ -847,9 +867,15 @@ pub async fn delete<T: KomodoResource>(
   );
   update.push_simple_log("Deleted Toml", toml);
 
-  if let Err(e) = T::post_delete(&resource, &mut update).await {
-    update.push_error_log("post delete", format_serror(&e.into()));
-  }
+  tokio::join!(
+    async {
+      if let Err(e) = T::post_delete(&resource, &mut update).await {
+        update
+          .push_error_log("post delete", format_serror(&e.into()));
+      }
+    },
+    delete_from_alerters::<T>(&resource.id)
+  );
 
   refresh_all_resources_cache().await;
 
@@ -859,6 +885,26 @@ pub async fn delete<T: KomodoResource>(
   Ok(resource)
 }
 
+async fn delete_from_alerters<T: KomodoResource>(id: &str) {
+  let target_bson = doc! {
+    "type": T::resource_type().as_ref(),
+    "id": id,
+  };
+  if let Err(e) = db_client()
+    .alerters
+    .update_many(Document::new(), doc! {
+      "$pull": {
+        "config.resources": &target_bson,
+        "config.except_resources": target_bson,
+      }
+    })
+    .await
+    .context("Failed to clear deleted resource from alerter whitelist / blacklist")
+  {
+    warn!("{e:#}");
+  }
+}
+
 // =======
 
 #[instrument(level = "debug")]

bin/core/src/resource/procedure.rs

@@ -5,6 +5,7 @@ use database::mungos::{
   find::find_collect,
   mongodb::{Collection, bson::doc, options::FindOneOptions},
 };
+use futures::{TryStreamExt, stream::FuturesUnordered};
 use komodo_client::{
   api::execute::Execution,
   entities::{
@@ -709,6 +710,15 @@ async fn validate_config(
           .await?;
           params.stack = stack.id;
         }
+        Execution::RunStackService(params) => {
+          let stack = super::get_check_permissions::<Stack>(
+            &params.stack,
+            user,
+            PermissionLevel::Execute.into(),
+          )
+          .await?;
+          params.stack = stack.id;
+        }
         Execution::BatchDestroyStack(_params) => {
           if !user.admin {
             return Err(anyhow!(
@@ -725,6 +735,24 @@ async fn validate_config(
           .await?;
           params.alerter = alerter.id;
         }
+        Execution::SendAlert(params) => {
+          params.alerters = params
+            .alerters
+            .iter()
+            .map(async |alerter| {
+              let id = super::get_check_permissions::<Alerter>(
+                alerter,
+                user,
+                PermissionLevel::Execute.into(),
+              )
+              .await?
+              .id;
+              anyhow::Ok(id)
+            })
+            .collect::<FuturesUnordered<_>>()
+            .try_collect::<Vec<_>>()
+            .await?;
+        }
         Execution::ClearRepoCache(_params) => {
           if !user.admin {
             return Err(anyhow!(

bin/core/src/resource/repo.rs

@@ -300,6 +300,7 @@ async fn get_repo_state_from_db(id: &str) -> RepoState {
         "$or": [
           { "operation": "CloneRepo" },
           { "operation": "PullRepo" },
+          { "operation": "BuildRepo" },
         ],
       })
       .with_options(

bin/core/src/resource/server.rs

@@ -82,6 +82,9 @@ impl super::KomodoResource for Server {
         send_cpu_alerts: server.config.send_cpu_alerts,
         send_mem_alerts: server.config.send_mem_alerts,
         send_disk_alerts: server.config.send_disk_alerts,
+        send_version_mismatch_alerts: server
+          .config
+          .send_version_mismatch_alerts,
         terminals_disabled,
         container_exec_disabled,
       },
@@ -120,7 +123,7 @@ impl super::KomodoResource for Server {
     created: &Resource<Self::Config, Self::Info>,
     _update: &mut Update,
   ) -> anyhow::Result<()> {
-    update_cache_for_server(created).await;
+    update_cache_for_server(created, true).await;
     Ok(())
   }
 
@@ -142,7 +145,7 @@ impl super::KomodoResource for Server {
     updated: &Self,
     _update: &mut Update,
   ) -> anyhow::Result<()> {
-    update_cache_for_server(updated).await;
+    update_cache_for_server(updated, true).await;
     Ok(())
   }
 

bin/core/src/resource/stack.rs

@@ -252,7 +252,7 @@ impl super::KomodoResource for Stack {
     else {
       return Ok(());
     };
-    update_cache_for_server(&server).await;
+    update_cache_for_server(&server, true).await;
     Ok(())
   }
 

bin/core/src/resource/sync.rs

@@ -113,7 +113,7 @@ impl super::KomodoResource for ResourceSync {
 
   async fn busy(id: &String) -> anyhow::Result<bool> {
     action_states()
-      .resource_sync
+      .sync
       .get(id)
       .await
       .unwrap_or_default()
@@ -242,7 +242,7 @@ async fn get_resource_sync_state(
   data: &ResourceSyncInfo,
 ) -> ResourceSyncState {
   if let Some(state) = action_states()
-    .resource_sync
+    .sync
     .get(id)
     .await
     .and_then(|s| {

bin/core/src/stack/execute.rs

@@ -72,7 +72,7 @@ pub async fn execute_compose<T: ExecuteCompose>(
     .push(T::execute(periphery, stack, services, extras).await?);
 
   // Ensure cached stack state up to date by updating server cache
-  update_cache_for_server(&server).await;
+  update_cache_for_server(&server, true).await;
 
   update.finalize();
   update_update(update.clone()).await?;

bin/core/src/stack/remote.rs

@@ -3,14 +3,16 @@ use std::{fs, path::PathBuf};
 use anyhow::Context;
 use formatting::format_serror;
 use komodo_client::entities::{
-  FileContents, RepoExecutionArgs, repo::Repo, stack::Stack,
+  FileContents, RepoExecutionArgs,
+  repo::Repo,
+  stack::{Stack, StackRemoteFileContents},
   update::Log,
 };
 
 use crate::{config::core_config, helpers::git_token};
 
 pub struct RemoteComposeContents {
-  pub successful: Vec<FileContents>,
+  pub successful: Vec<StackRemoteFileContents>,
   pub errored: Vec<FileContents>,
   pub hash: Option<String>,
   pub message: Option<String>,
@@ -38,23 +40,25 @@ pub async fn get_repo_compose_contents(
   let mut successful = Vec::new();
   let mut errored = Vec::new();
 
-  for path in stack.file_paths() {
-    let file_path = run_directory.join(path);
+  for file in stack.all_file_dependencies() {
+    let file_path = run_directory.join(&file.path);
     if !file_path.exists()
       && let Some(missing_files) = &mut missing_files
     {
-      missing_files.push(path.to_string());
+      missing_files.push(file.path.clone());
     }
     // If file does not exist, will show up in err case so the log is handled
     match fs::read_to_string(&file_path).with_context(|| {
       format!("Failed to read file contents from {file_path:?}")
     }) {
-      Ok(contents) => successful.push(FileContents {
-        path: path.to_string(),
+      Ok(contents) => successful.push(StackRemoteFileContents {
+        path: file.path,
         contents,
+        services: file.services,
+        requires: file.requires,
       }),
       Err(e) => errored.push(FileContents {
-        path: path.to_string(),
+        path: file.path,
         contents: format_serror(&e.into()),
       }),
     }

bin/core/src/startup.rs

@@ -242,7 +242,7 @@ async fn ensure_first_server_and_builder() {
       return;
     };
   if let Err(e) = (CreateBuilder {
-    name: String::from("Local"),
+    name: config.first_server_name.clone(),
     config: PartialBuilderConfig::Server(
       PartialServerBuilderConfig {
         server_id: Some(server.id),
@@ -285,8 +285,7 @@ async fn ensure_init_user_and_resources() {
     }
     .resolve(&AuthArgs::default())
     .await
-    .expect("Failed to initialize default admin user.")
-    .jwt;
+    .expect("Failed to initialize default admin user.");
     db.users
       .find_one(doc! { "username": username })
       .await

bin/core/src/state.rs

@@ -40,6 +40,7 @@ pub fn db_client() -> &'static Client {
     .expect("db_client accessed before initialized")
 }
 
+/// Must be called in app startup sequence.
 pub async fn init_db_client() {
   let client = Client::new(&core_config().database)
     .await

bin/core/src/sync/deploy.rs

@@ -9,12 +9,15 @@ use komodo_client::{
     read::ListBuildVersions,
   },
   entities::{
-    FileContents, ResourceTarget,
+    ResourceTarget,
     deployment::{
       Deployment, DeploymentConfig, DeploymentImage, DeploymentState,
       PartialDeploymentConfig,
     },
-    stack::{PartialStackConfig, Stack, StackConfig, StackState},
+    stack::{
+      PartialStackConfig, Stack, StackConfig,
+      StackRemoteFileContents, StackState,
+    },
     sync::SyncDeployUpdate,
     toml::ResourceToml,
     update::Log,
@@ -554,7 +557,13 @@ fn build_cache_for_stack<'a>(
           &original.info.remote_contents,
         ) {
           (Some(deployed_contents), Some(remote_contents)) => {
-            for FileContents { path, contents } in remote_contents {
+            for StackRemoteFileContents {
+              path,
+              contents,
+              services: _services,
+              requires: _requires,
+            } in remote_contents
+            {
               if let Some(deployed) =
                 deployed_contents.iter().find(|c| &c.path == path)
               {

bin/core/src/sync/execute.rs

@@ -147,6 +147,7 @@ pub trait ExecuteResourceSync: ResourceSyncTrait {
         sync_user(),
       )
       .await
+      .map_err(|e| e.error)
       {
         Ok(resource) => resource.id,
         Err(e) => {

bin/core/src/sync/resources.rs

@@ -661,6 +661,13 @@ impl ResourceSyncTrait for Procedure {
               .map(|s| s.name.clone())
               .unwrap_or_default();
           }
+          Execution::RunStackService(config) => {
+            config.stack = resources
+              .stacks
+              .get(&config.stack)
+              .map(|s| s.name.clone())
+              .unwrap_or_default();
+          }
           Execution::BatchDestroyStack(_config) => {}
           Execution::TestAlerter(config) => {
             config.alerter = resources
@@ -669,6 +676,19 @@ impl ResourceSyncTrait for Procedure {
               .map(|a| a.name.clone())
               .unwrap_or_default();
           }
+          Execution::SendAlert(config) => {
+            config.alerters = config
+              .alerters
+              .iter()
+              .map(|alerter| {
+                resources
+                  .alerters
+                  .get(alerter)
+                  .map(|a| a.name.clone())
+                  .unwrap_or_default()
+              })
+              .collect();
+          }
           Execution::ClearRepoCache(_) => {}
           Execution::BackupCoreDatabase(_) => {}
           Execution::GlobalAutoUpdate(_) => {}
@@ -805,6 +825,7 @@ impl ExecuteResourceSync for Procedure {
           sync_user(),
         )
         .await
+        .map_err(|e| e.error)
         {
           Ok(resource) => resource.id,
           Err(e) => {

bin/core/src/sync/toml.rs

@@ -752,6 +752,13 @@ impl ToToml for Procedure {
               .map(|r| &r.name)
               .unwrap_or(&String::new()),
           ),
+          Execution::RunStackService(exec) => exec.stack.clone_from(
+            all
+              .stacks
+              .get(&exec.stack)
+              .map(|r| &r.name)
+              .unwrap_or(&String::new()),
+          ),
           Execution::PauseStack(exec) => exec.stack.clone_from(
             all
               .stacks
@@ -788,6 +795,17 @@ impl ToToml for Procedure {
               .map(|a| &a.name)
               .unwrap_or(&String::new()),
           ),
+          Execution::SendAlert(exec) => {
+            exec.alerters.iter_mut().for_each(|a| {
+              a.clone_from(
+                all
+                  .alerters
+                  .get(a)
+                  .map(|a| &a.name)
+                  .unwrap_or(&String::new()),
+              )
+            })
+          }
           Execution::None(_)
           | Execution::Sleep(_)
           | Execution::ClearRepoCache(_)

bin/periphery/Cargo.toml

@@ -57,4 +57,5 @@ axum.workspace = true
 clap.workspace = true
 envy.workspace = true
 uuid.workspace = true
-rand.workspace = true
+rand.workspace = true
+shell-escape.workspace = true

bin/periphery/aio.Dockerfile

@@ -1,6 +1,7 @@
 ## All in one, multi stage compile + runtime Docker build for your architecture.
 
 FROM rust:1.89.0-bullseye AS builder
+RUN cargo install cargo-strip
 
 WORKDIR /builder
 COPY Cargo.toml Cargo.lock ./
@@ -10,7 +11,7 @@ COPY ./client/periphery ./client/periphery
 COPY ./bin/periphery ./bin/periphery
 
 # Compile app
-RUN cargo build -p komodo_periphery --release
+RUN cargo build -p komodo_periphery --release && cargo strip
 
 # Final Image
 FROM debian:bullseye-slim

bin/periphery/src/api/build.rs

@@ -1,4 +1,7 @@
-use std::path::PathBuf;
+use std::{
+  collections::{HashMap, HashSet},
+  path::PathBuf,
+};
 
 use anyhow::{Context, anyhow};
 use command::{
@@ -9,7 +12,7 @@ use interpolate::Interpolator;
 use komodo_client::entities::{
   EnvironmentVar, all_logs_success,
   build::{Build, BuildConfig},
-  environment_vars_from_str, get_image_name, optional_string,
+  environment_vars_from_str, optional_string,
   to_path_compatible_name,
   update::Log,
 };
@@ -22,9 +25,7 @@ use resolver_api::Resolve;
 use tokio::fs;
 
 use crate::{
-  build::{
-    image_tags, parse_build_args, parse_secret_args, write_dockerfile,
-  },
+  build::{parse_build_args, parse_secret_args, write_dockerfile},
   config::periphery_config,
   docker::docker_login,
   helpers::{parse_extra_args, parse_labels},
@@ -122,9 +123,10 @@ impl Resolve<super::Args> for build::Build {
     let build::Build {
       mut build,
       repo: linked_repo,
-      registry_token,
-      additional_tags,
+      registry_tokens,
       mut replacers,
+      commit_hash,
+      additional_tags,
     } = self;
 
     let mut logs = Vec::new();
@@ -142,8 +144,6 @@ impl Resolve<super::Args> for build::Build {
       name,
       config:
         BuildConfig {
-          version,
-          image_tag,
           build_path,
           dockerfile_path,
           build_args,
@@ -169,25 +169,41 @@ impl Resolve<super::Args> for build::Build {
       return Err(anyhow!("Build must be files on host mode, have a repo attached, or have dockerfile contents set to build").into());
     }
 
+    let registry_tokens = registry_tokens
+      .iter()
+      .map(|(domain, account, token)| {
+        ((domain.as_str(), account.as_str()), token.as_str())
+      })
+      .collect::<HashMap<_, _>>();
+
     // Maybe docker login
-    let should_push = match docker_login(
-      &image_registry.domain,
-      &image_registry.account,
-      registry_token.as_deref(),
-    )
-    .await
+    let mut should_push = false;
+    for (domain, account) in image_registry
+      .iter()
+      .map(|r| (r.domain.as_str(), r.account.as_str()))
+      // This ensures uniqueness / prevents redundant logins
+      .collect::<HashSet<_>>()
     {
-      Ok(should_push) => should_push,
-      Err(e) => {
-        logs.push(Log::error(
-          "Docker Login",
-          format_serror(
-            &e.context("failed to login to docker registry").into(),
-          ),
-        ));
-        return Ok(logs);
-      }
-    };
+      match docker_login(
+        domain,
+        account,
+        registry_tokens.get(&(domain, account)).copied(),
+      )
+      .await
+      {
+        Ok(logged_in) if logged_in => should_push = true,
+        Ok(_) => {}
+        Err(e) => {
+          logs.push(Log::error(
+            "Docker Login",
+            format_serror(
+              &e.context("failed to login to docker registry").into(),
+            ),
+          ));
+          return Ok(logs);
+        }
+      };
+    }
 
     let build_path = if let Some(repo) = &linked_repo {
       periphery_config()
@@ -245,8 +261,6 @@ impl Resolve<super::Args> for build::Build {
     }
 
     // Get command parts
-    let image_name =
-      get_image_name(&build).context("failed to make image name")?;
 
     // Add VERSION to build args (if not already there)
     let mut build_args = environment_vars_from_str(build_args)
@@ -267,10 +281,15 @@ impl Resolve<super::Args> for build::Build {
     let labels = parse_labels(
       &environment_vars_from_str(labels).context("Invalid labels")?,
     );
+
     let extra_args = parse_extra_args(extra_args);
+
     let buildx = if *use_buildx { " buildx" } else { "" };
-    let image_tags =
-      image_tags(&image_name, image_tag, version, &additional_tags);
+
+    let image_tags = build
+      .get_image_tags_as_arg(commit_hash.as_deref(), &additional_tags)
+      .context("Failed to parse image tags into command")?;
+
     let maybe_push = if should_push { " --push" } else { "" };
 
     // Construct command

bin/periphery/src/api/compose.rs

@@ -1,5 +1,3 @@
-use std::{fmt::Write, path::PathBuf};
-
 use anyhow::{Context, anyhow};
 use command::{
   run_komodo_command, run_komodo_command_with_sanitization,
@@ -11,7 +9,7 @@ use komodo_client::entities::{
   FileContents, RepoExecutionResponse, all_logs_success,
   stack::{
     ComposeFile, ComposeProject, ComposeService,
-    ComposeServiceDeploy, StackServiceNames,
+    ComposeServiceDeploy, StackRemoteFileContents, StackServiceNames,
   },
   to_path_compatible_name,
   update::Log,
@@ -19,11 +17,13 @@ use komodo_client::entities::{
 use periphery_client::api::compose::*;
 use resolver_api::Resolve;
 use serde::{Deserialize, Serialize};
+use shell_escape::unix::escape;
+use std::{borrow::Cow, path::PathBuf};
 use tokio::fs;
 
 use crate::{
   compose::{
-    docker_compose, pull_or_clone_stack,
+    docker_compose, env_file_args, pull_or_clone_stack,
     up::{maybe_login_registry, validate_files},
     write::write_stack,
   },
@@ -169,9 +169,11 @@ impl Resolve<super::Args> for GetComposeContentsOnHost {
 
     let mut res = GetComposeContentsOnHostResponse::default();
 
-    for path in file_paths {
-      let full_path =
-        run_directory.join(&path).components().collect::<PathBuf>();
+    for file in file_paths {
+      let full_path = run_directory
+        .join(&file.path)
+        .components()
+        .collect::<PathBuf>();
       match fs::read_to_string(&full_path).await.with_context(|| {
         format!(
           "Failed to read compose file contents at {full_path:?}"
@@ -180,11 +182,16 @@ impl Resolve<super::Args> for GetComposeContentsOnHost {
         Ok(contents) => {
           // The path we store here has to be the same as incoming file path in the array,
           // in order for WriteComposeContentsToHost to write to the correct path.
-          res.contents.push(FileContents { path, contents });
+          res.contents.push(StackRemoteFileContents {
+            path: file.path,
+            contents,
+            services: file.services,
+            requires: file.requires,
+          });
         }
         Err(e) => {
           res.errors.push(FileContents {
-            path,
+            path: file.path,
             contents: format_serror(&e.into()),
           });
         }
@@ -351,19 +358,19 @@ impl Resolve<super::Args> for ComposePull {
     )?;
 
     let file_paths = stack
-      .file_paths()
-      .iter()
+      .all_file_paths()
+      .into_iter()
       .map(|path| {
         (
-          path,
           // This will remove any intermediate uneeded '/./' in the path
-          run_directory.join(path).components().collect::<PathBuf>(),
+          run_directory.join(&path).components().collect::<PathBuf>(),
+          path,
         )
       })
       .collect::<Vec<_>>();
 
     // Validate files
-    for (path, full_path) in &file_paths {
+    for (full_path, path) in &file_paths {
       if !full_path.exists() {
         return Err(anyhow!("Missing compose file at {path}").into());
       }
@@ -382,24 +389,12 @@ impl Resolve<super::Args> for ComposePull {
       format!(" {}", services.join(" "))
     };
 
-    let file_args = if stack.config.file_paths.is_empty() {
-      String::from("compose.yaml")
-    } else {
-      stack.config.file_paths.join(" -f ")
-    };
+    let file_args = stack.compose_file_paths().join(" -f ");
 
-    let env_file = env_file_path
-      .map(|path| format!(" --env-file {path}"))
-      .unwrap_or_default();
-
-    let additional_env_files = stack
-      .config
-      .additional_env_files
-      .iter()
-      .fold(String::new(), |mut output, file| {
-        let _ = write!(output, " --env-file {file}");
-        output
-      });
+    let env_file_args = env_file_args(
+      env_file_path,
+      &stack.config.additional_env_files,
+    )?;
 
     let project_name = stack.project_name(false);
 
@@ -407,7 +402,7 @@ impl Resolve<super::Args> for ComposePull {
       "Compose Pull",
       run_directory.as_ref(),
       format!(
-        "{docker_compose} -p {project_name} -f {file_args}{additional_env_files}{env_file} pull{service_args}",
+        "{docker_compose} -p {project_name} -f {file_args}{env_file_args} pull{service_args}",
       ),
     )
     .await;
@@ -514,35 +509,23 @@ impl Resolve<super::Args> for ComposeUp {
       format!(" {}", services.join(" "))
     };
 
-    let file_args = if stack.config.file_paths.is_empty() {
-      String::from("compose.yaml")
-    } else {
-      stack.config.file_paths.join(" -f ")
-    };
+    let file_args = stack.compose_file_paths().join(" -f ");
 
     // This will be the last project name, which is the one that needs to be destroyed.
     // Might be different from the current project name, if user renames stack / changes to custom project name.
     let last_project_name = stack.project_name(false);
     let project_name = stack.project_name(true);
 
-    let env_file = env_file_path
-      .map(|path| format!(" --env-file {path}"))
-      .unwrap_or_default();
-
-    let additional_env_files = stack
-      .config
-      .additional_env_files
-      .iter()
-      .fold(String::new(), |mut output, file| {
-        let _ = write!(output, " --env-file {file}");
-        output
-      });
+    let env_file_args = env_file_args(
+      env_file_path,
+      &stack.config.additional_env_files,
+    )?;
 
     // Uses 'docker compose config' command to extract services (including image)
     // after performing interpolation
     {
       let command = format!(
-        "{docker_compose} -p {project_name} -f {file_args}{additional_env_files}{env_file} config",
+        "{docker_compose} -p {project_name} -f {file_args}{env_file_args} config",
       );
       let Some(config_log) = run_komodo_command_with_sanitization(
         "Compose Config",
@@ -607,7 +590,7 @@ impl Resolve<super::Args> for ComposeUp {
       let build_extra_args =
         parse_extra_args(&stack.config.build_extra_args);
       let command = format!(
-        "{docker_compose} -p {project_name} -f {file_args}{additional_env_files}{env_file} build{build_extra_args}{service_args}",
+        "{docker_compose} -p {project_name} -f {file_args}{env_file_args} build{build_extra_args}{service_args}",
       );
       let Some(log) = run_komodo_command_with_sanitization(
         "Compose Build",
@@ -631,7 +614,7 @@ impl Resolve<super::Args> for ComposeUp {
       // Pull images before destroying to minimize downtime.
       // If this fails, do not continue.
       let command = format!(
-        "{docker_compose} -p {project_name} -f {file_args}{additional_env_files}{env_file} pull{service_args}",
+        "{docker_compose} -p {project_name} -f {file_args}{env_file_args} pull{service_args}",
       );
       let log = run_komodo_command(
         "Compose Pull",
@@ -659,7 +642,7 @@ impl Resolve<super::Args> for ComposeUp {
     // Run compose up
     let extra_args = parse_extra_args(&stack.config.extra_args);
     let command = format!(
-      "{docker_compose} -p {project_name} -f {file_args}{additional_env_files}{env_file} up -d{extra_args}{service_args}",
+      "{docker_compose} -p {project_name} -f {file_args}{env_file_args} up -d{extra_args}{service_args}",
     );
 
     let Some(log) = run_komodo_command_with_sanitization(
@@ -713,3 +696,150 @@ impl Resolve<super::Args> for ComposeExecution {
     Ok(log)
   }
 }
+
+//
+
+impl Resolve<super::Args> for ComposeRun {
+  #[instrument(name = "ComposeRun", level = "debug", skip_all, fields(stack = &self.stack.name, service = &self.service))]
+  async fn resolve(self, _: &super::Args) -> serror::Result<Log> {
+    let ComposeRun {
+      mut stack,
+      repo,
+      git_token,
+      registry_token,
+      mut replacers,
+      service,
+      command,
+      no_tty,
+      no_deps,
+      detach,
+      service_ports,
+      env,
+      workdir,
+      user,
+      entrypoint,
+      pull,
+    } = self;
+
+    let mut interpolator =
+      Interpolator::new(None, &periphery_config().secrets);
+    interpolator
+      .interpolate_stack(&mut stack)?
+      .push_logs(&mut Vec::new());
+    replacers.extend(interpolator.secret_replacers);
+
+    let mut res = ComposeRunResponse::default();
+    let (run_directory, env_file_path) = match write_stack(
+      &stack,
+      repo.as_ref(),
+      git_token,
+      replacers.clone(),
+      &mut res,
+    )
+    .await
+    {
+      Ok(res) => res,
+      Err(e) => {
+        return Ok(Log::error(
+          "Write Stack",
+          format_serror(&e.into()),
+        ));
+      }
+    };
+
+    let run_directory = run_directory.canonicalize().context(
+      "Failed to validate run directory on host after stack write (canonicalize error)",
+    )?;
+
+    maybe_login_registry(&stack, registry_token, &mut Vec::new())
+      .await;
+
+    let docker_compose = docker_compose();
+
+    let file_args = if stack.config.file_paths.is_empty() {
+      String::from("compose.yaml")
+    } else {
+      stack.config.file_paths.join(" -f ")
+    };
+
+    let env_file_args = env_file_args(
+      env_file_path,
+      &stack.config.additional_env_files,
+    )?;
+
+    let project_name = stack.project_name(true);
+
+    if pull.unwrap_or_default() {
+      let pull_log = run_komodo_command(
+        "Compose Pull",
+        run_directory.as_ref(),
+        format!(
+          "{docker_compose} -p {project_name} -f {file_args}{env_file_args} pull {service}",
+        ),
+      )
+      .await;
+      if !pull_log.success {
+        return Ok(pull_log);
+      }
+    }
+
+    let mut run_flags = String::from(" --rm");
+    if detach.unwrap_or_default() {
+      run_flags.push_str(" -d");
+    }
+    if no_tty.unwrap_or_default() {
+      run_flags.push_str(" --no-tty");
+    }
+    if no_deps.unwrap_or_default() {
+      run_flags.push_str(" --no-deps");
+    }
+    if service_ports.unwrap_or_default() {
+      run_flags.push_str(" --service-ports");
+    }
+    if let Some(dir) = workdir.as_ref() {
+      run_flags.push_str(&format!(" --workdir {dir}"));
+    }
+    if let Some(user) = user.as_ref() {
+      run_flags.push_str(&format!(" --user {user}"));
+    }
+    if let Some(entrypoint) = entrypoint.as_ref() {
+      run_flags.push_str(&format!(" --entrypoint {entrypoint}"));
+    }
+    if let Some(env) = env {
+      for (k, v) in env {
+        run_flags.push_str(&format!(" -e {}={} ", k, v));
+      }
+    }
+
+    let command_args = command
+      .as_ref()
+      .filter(|v| !v.is_empty())
+      .map(|argv| {
+        let joined = argv
+          .iter()
+          .map(|s| escape(Cow::Borrowed(s)).into_owned())
+          .collect::<Vec<_>>()
+          .join(" ");
+        format!(" {joined}")
+      })
+      .unwrap_or_default();
+
+    let command = format!(
+      "{docker_compose} -p {project_name} -f {file_args}{env_file_args} run{run_flags} {service}{command_args}",
+    );
+
+    let Some(log) = run_komodo_command_with_sanitization(
+      "Compose Run",
+      run_directory.as_path(),
+      command,
+      false,
+      &replacers,
+    )
+    .await
+    else {
+      unreachable!()
+    };
+
+    Ok(log)
+  }
+}

bin/periphery/src/api/mod.rs

@@ -85,6 +85,7 @@ pub enum PeripheryRequest {
   ComposePull(ComposePull),
   ComposeUp(ComposeUp),
   ComposeExecution(ComposeExecution),
+  ComposeRun(ComposeRun),
 
   // Container (Read)
   InspectContainer(InspectContainer),

bin/periphery/src/api/terminal.rs

@@ -347,7 +347,7 @@ async fn execute_command_on_terminal(
   );
 
   let full_command = format!(
-    "printf '\n{START_OF_OUTPUT}\n\n'; {command}; rc=$? printf '\n{KOMODO_EXIT_CODE}%d\n{END_OF_OUTPUT}\n' \"$rc\"\n"
+    "printf '\n{START_OF_OUTPUT}\n\n'; {command}; rc=$?; printf '\n{KOMODO_EXIT_CODE}%d\n{END_OF_OUTPUT}\n' \"$rc\"\n"
   );
 
   terminal

bin/periphery/src/build.rs

@@ -6,7 +6,7 @@ use std::{
 use anyhow::{Context, anyhow};
 use formatting::format_serror;
 use komodo_client::{
-  entities::{EnvironmentVar, Version, update::Log},
+  entities::{EnvironmentVar, update::Log},
   parsers::QUOTE_PATTERN,
 };
 
@@ -52,28 +52,6 @@ pub async fn write_dockerfile(
   }
 }
 
-pub fn image_tags(
-  image_name: &str,
-  custom_tag: &str,
-  version: &Version,
-  additional: &[String],
-) -> String {
-  let Version { major, minor, .. } = version;
-  let custom_tag = if custom_tag.is_empty() {
-    String::new()
-  } else {
-    format!("-{custom_tag}")
-  };
-  let additional = additional
-    .iter()
-    .map(|tag| format!(" -t {image_name}:{tag}{custom_tag}"))
-    .collect::<Vec<_>>()
-    .join("");
-  format!(
-    " -t {image_name}:latest{custom_tag} -t {image_name}:{version}{custom_tag} -t {image_name}:{major}.{minor}{custom_tag} -t {image_name}:{major}{custom_tag}{additional}",
-  )
-}
-
 pub fn parse_build_args(build_args: &[EnvironmentVar]) -> String {
   build_args
     .iter()

bin/periphery/src/compose/mod.rs

@@ -1,6 +1,6 @@
-use std::path::PathBuf;
+use std::{fmt::Write, path::PathBuf};
 
-use anyhow::anyhow;
+use anyhow::{Context, anyhow};
 use command::run_komodo_command;
 use komodo_client::entities::{
   RepoExecutionArgs, repo::Repo, stack::Stack,
@@ -24,6 +24,35 @@ pub fn docker_compose() -> &'static str {
   }
 }
 
+pub fn env_file_args(
+  env_file_path: Option<&str>,
+  additional_env_files: &[String],
+) -> anyhow::Result<String> {
+  let mut res = String::new();
+
+  for file in additional_env_files.iter().filter(|&path| {
+    let Some(komodo_path) = env_file_path else {
+      return true;
+    };
+    // Filter komodo env out of additional env file if its also in there.
+    // It will be always be added last / have highest priority.
+    path != komodo_path
+  }) {
+    write!(res, " --env-file {file}").with_context(|| {
+      format!("Failed to write --env-file arg for {file}")
+    })?;
+  }
+
+  // Add this last, so it is applied on top
+  if let Some(file) = env_file_path {
+    write!(res, " --env-file {file}").with_context(|| {
+      format!("Failed to write --env-file arg for {file}")
+    })?;
+  }
+
+  Ok(res)
+}
+
 pub async fn down(
   project: &str,
   services: &[String],

bin/periphery/src/compose/up.rs

@@ -3,7 +3,9 @@ use std::path::{Path, PathBuf};
 use anyhow::{Context, anyhow};
 use formatting::format_serror;
 use komodo_client::entities::{
-  FileContents, stack::Stack, update::Log,
+  FileContents,
+  stack::{Stack, StackRemoteFileContents},
+  update::Log,
 };
 use periphery_client::api::compose::ComposeUpResponse;
 use tokio::fs;
@@ -16,20 +18,24 @@ pub async fn validate_files(
   res: &mut ComposeUpResponse,
 ) {
   let file_paths = stack
-    .file_paths()
-    .iter()
-    .map(|path| {
+    .all_file_dependencies()
+    .into_iter()
+    .map(|file| {
       (
-        path,
         // This will remove any intermediate uneeded '/./' in the path
-        run_directory.join(path).components().collect::<PathBuf>(),
+        run_directory
+          .join(&file.path)
+          .components()
+          .collect::<PathBuf>(),
+        file,
       )
     })
     .collect::<Vec<_>>();
 
-  for (path, full_path) in &file_paths {
+  // First validate no missing files
+  for (full_path, file) in &file_paths {
     if !full_path.exists() {
-      res.missing_files.push(path.to_string());
+      res.missing_files.push(file.path.clone());
     }
   }
   if !res.missing_files.is_empty() {
@@ -37,21 +43,20 @@ pub async fn validate_files(
       "Validate Files",
       format_serror(
         &anyhow!(
-          "Ensure the run_directory and file_paths are correct."
+          "Missing files: {}", res.missing_files.join(", ")
         )
-        .context("A compose file doesn't exist after writing stack.")
+        .context("Ensure the run_directory and all file paths are correct.")
+        .context("A file doesn't exist after writing stack.")
         .into(),
       ),
     ));
     return;
   }
 
-  for (path, full_path) in &file_paths {
+  for (full_path, file) in file_paths {
     let file_contents =
       match fs::read_to_string(&full_path).await.with_context(|| {
-        format!(
-          "Failed to read compose file contents at {full_path:?}"
-        )
+        format!("Failed to read file contents at {full_path:?}")
       }) {
         Ok(res) => res,
         Err(e) => {
@@ -61,15 +66,17 @@ pub async fn validate_files(
             .push(Log::error("Read Compose File", error.clone()));
           // This should only happen for repo stacks, ie remote error
           res.remote_errors.push(FileContents {
-            path: path.to_string(),
+            path: file.path,
             contents: error,
           });
           return;
         }
       };
-    res.file_contents.push(FileContents {
-      path: path.to_string(),
+    res.file_contents.push(StackRemoteFileContents {
+      path: file.path,
       contents: file_contents,
+      services: file.services,
+      requires: file.requires,
     });
   }
 }

bin/periphery/src/compose/write.rs

@@ -7,7 +7,9 @@ use komodo_client::entities::{
   stack::Stack, to_path_compatible_name, update::Log,
 };
 use periphery_client::api::{
-  compose::{ComposePullResponse, ComposeUpResponse},
+  compose::{
+    ComposePullResponse, ComposeRunResponse, ComposeUpResponse,
+  },
   git::{CloneRepo, PullOrCloneRepo},
 };
 use resolver_api::Resolve;
@@ -43,6 +45,12 @@ impl WriteStackRes for &mut ComposePullResponse {
   }
 }
 
+impl WriteStackRes for &mut ComposeRunResponse {
+  fn logs(&mut self) -> &mut Vec<Log> {
+    &mut self.logs
+  }
+}
+
 /// Either writes the stack file_contents to a file, or clones the repo.
 /// Asssumes all interpolation is already complete.
 /// Returns (run_directory, env_file_path, periphery_replacers)

bin/periphery/src/stats.rs

@@ -2,7 +2,8 @@ use std::{cmp::Ordering, sync::OnceLock};
 
 use async_timing_util::wait_until_timelength;
 use komodo_client::entities::stats::{
-  SingleDiskUsage, SystemInformation, SystemProcess, SystemStats,
+  SingleDiskUsage, SystemInformation, SystemLoadAverage,
+  SystemProcess, SystemStats,
 };
 use sysinfo::{ProcessRefreshKind, ProcessesToUpdate, System};
 use tokio::sync::RwLock;
@@ -95,8 +96,15 @@ impl StatsClient {
       network_egress_bytes += network.transmitted();
     }
 
+    let load_avg = System::load_average();
+
     SystemStats {
       cpu_perc: self.system.global_cpu_usage(),
+      load_average: SystemLoadAverage {
+        one: load_avg.one,
+        five: load_avg.five,
+        fifteen: load_avg.fifteen,
+      },
       mem_free_gb: self.system.free_memory() as f64 / BYTES_PER_GB,
       mem_used_gb: (total_mem - available_mem) as f64 / BYTES_PER_GB,
       mem_total_gb: total_mem as f64 / BYTES_PER_GB,

client/core/rs/Cargo.toml

@@ -28,6 +28,7 @@ resolver_api.workspace = true
 # external
 tokio-tungstenite.workspace = true
 derive_builder.workspace = true
+urlencoding.workspace = true
 serde_json.workspace = true
 tokio-util.workspace = true
 thiserror.workspace = true

client/core/rs/src/api/execute/action.rs

@@ -60,7 +60,7 @@ pub struct BatchRunAction {
   /// Supports multiline and comma delineated combinations of the above.
   ///
   /// Example:
-  /// ```
+  /// ```text
   /// # match all foo-* actions
   /// foo-*
   /// # add some more

client/core/rs/src/api/execute/alerter.rs

@@ -4,7 +4,7 @@ use resolver_api::Resolve;
 use serde::{Deserialize, Serialize};
 use typeshare::typeshare;
 
-use crate::entities::update::Update;
+use crate::entities::{alert::SeverityLevel, update::Update};
 
 use super::KomodoExecuteRequest;
 
@@ -27,3 +27,39 @@ pub struct TestAlerter {
   /// Name or id
   pub alerter: String,
 }
+
+//
+
+/// Send a custom alert message to configured Alerters. Response: [Update]
+#[typeshare]
+#[derive(
+  Serialize,
+  Deserialize,
+  Debug,
+  Clone,
+  PartialEq,
+  Resolve,
+  EmptyTraits,
+  Parser,
+)]
+#[empty_traits(KomodoExecuteRequest)]
+#[response(Update)]
+#[error(serror::Error)]
+pub struct SendAlert {
+  /// The alert level.
+  #[serde(default)]
+  #[clap(long, short = 'l', default_value_t = SeverityLevel::Ok)]
+  pub level: SeverityLevel,
+  /// The alert message. Required.
+  pub message: String,
+  /// The alert details. Optional.
+  #[serde(default)]
+  #[arg(long, short = 'd', default_value_t = String::new())]
+  pub details: String,
+  /// Specific alerter names or ids.
+  /// If empty / not passed, sends to all configured alerters
+  /// with the `Custom` alert type whitelisted / not blacklisted.
+  #[serde(default)]
+  #[arg(long, short = 'a')]
+  pub alerters: Vec<String>,
+}

client/core/rs/src/api/execute/build.rs

@@ -64,7 +64,7 @@ pub struct BatchRunBuild {
   /// Supports multiline and comma delineated combinations of the above.
   ///
   /// Example:
-  /// ```
+  /// ```text
   /// # match all foo-* builds
   /// foo-*
   /// # add some more

client/core/rs/src/api/execute/deployment.rs

@@ -62,7 +62,7 @@ pub struct BatchDeploy {
   /// Supports multiline and comma delineated combinations of the above.
   ///
   /// Example:
-  /// ```
+  /// ```text
   /// # match all foo-* deployments
   /// foo-*
   /// # add some more
@@ -270,7 +270,7 @@ pub struct BatchDestroyDeployment {
   /// Supports multiline and comma delineated combinations of the above.
   ///
   /// Example:
-  /// ```
+  /// ```text
   /// # match all foo-* deployments
   /// foo-*
   /// # add some more

client/core/rs/src/api/execute/maintenance.rs

@@ -50,12 +50,12 @@ pub struct ClearRepoCache {}
 #[error(serror::Error)]
 pub struct BackupCoreDatabase {}
 
-/// Trigger a global poll for image updateson Stacks and Deployments
+/// Trigger a global poll for image updates on Stacks and Deployments
 /// with `poll_for_updates` or `auto_update` enabled.
 /// Admin only. Response: [Update]
 ///
 /// 1. `docker compose pull` any Stacks / Deployments with `poll_for_updates` or `auto_update` enabled. This will pick up any available updates.
-/// 2. Redeploy Stacks / Deployments that have updates found.
+/// 2. Redeploy Stacks / Deployments that have updates found and 'auto_update' enabled.
 #[typeshare]
 #[derive(
   Debug,

client/core/rs/src/api/execute/mod.rs

@@ -152,9 +152,12 @@ pub enum Execution {
   StopStack(StopStack),
   DestroyStack(DestroyStack),
   BatchDestroyStack(BatchDestroyStack),
+  RunStackService(RunStackService),
 
   // ALERTER
   TestAlerter(TestAlerter),
+  #[clap(alias = "alert")]
+  SendAlert(SendAlert),
 
   // MAINTENANCE
   ClearRepoCache(ClearRepoCache),

client/core/rs/src/api/execute/procedure.rs

@@ -48,7 +48,7 @@ pub struct BatchRunProcedure {
   /// Supports multiline and comma delineated combinations of the above.
   ///
   /// Example:
-  /// ```
+  /// ```text
   /// # match all foo-* procedures
   /// foo-*
   /// # add some more

client/core/rs/src/api/execute/repo.rs

@@ -60,7 +60,7 @@ pub struct BatchCloneRepo {
   /// Supports multiline and comma delineated combinations of the above.
   ///
   /// Example:
-  /// ```
+  /// ```text
   /// # match all foo-* repos
   /// foo-*
   /// # add some more
@@ -118,7 +118,7 @@ pub struct BatchPullRepo {
   /// Supports multiline and comma delineated combinations of the above.
   ///
   /// Example:
-  /// ```
+  /// ```text
   /// # match all foo-* repos
   /// foo-*
   /// # add some more
@@ -180,7 +180,7 @@ pub struct BatchBuildRepo {
   /// Supports multiline and comma delineated combinations of the above.
   ///
   /// Example:
-  /// ```
+  /// ```text
   /// # match all foo-* repos
   /// foo-*
   /// # add some more

client/core/rs/src/api/execute/stack.rs

@@ -1,11 +1,13 @@
+use crate::entities::update::Update;
+use anyhow::Context;
+use clap::ArgAction::SetTrue;
 use clap::Parser;
 use derive_empty_traits::EmptyTraits;
 use resolver_api::Resolve;
 use serde::{Deserialize, Serialize};
+use std::collections::HashMap;
 use typeshare::typeshare;
 
-use crate::entities::update::Update;
-
 use super::{BatchExecutionResponse, KomodoExecuteRequest};
 
 /// Deploys the target stack. `docker compose up`. Response: [Update]
@@ -57,7 +59,7 @@ pub struct BatchDeployStack {
   /// Supports multiline and comma delineated combinations of the above.
   ///
   /// Example:
-  /// ```
+  /// ```text
   /// # match all foo-* stacks
   /// foo-*
   /// # add some more
@@ -115,7 +117,7 @@ pub struct BatchDeployStackIfChanged {
   /// Supports multiline and comma delineated combinations of the above.
   ///
   /// Example:
-  /// ```
+  /// ```text
   /// # match all foo-* stacks
   /// foo-*
   /// # add some more
@@ -172,7 +174,7 @@ pub struct BatchPullStack {
   /// Supports multiline and comma delineated combinations of the above.
   ///
   /// Example:
-  /// ```
+  /// ```text
   /// # match all foo-* stacks
   /// foo-*
   /// # add some more
@@ -348,6 +350,64 @@ pub struct DestroyStack {
 
 //
 
+/// Runs a one-time command against a service using `docker compose run`. Response: [Update]
+#[typeshare]
+#[derive(
+  Debug,
+  Clone,
+  PartialEq,
+  Serialize,
+  Deserialize,
+  Resolve,
+  EmptyTraits,
+  Parser,
+)]
+#[empty_traits(KomodoExecuteRequest)]
+#[response(Update)]
+#[error(serror::Error)]
+pub struct RunStackService {
+  /// Id or name
+  pub stack: String,
+  /// Service to run
+  pub service: String,
+  /// Command and args to pass to the service container
+  #[arg(trailing_var_arg = true, num_args = 1.., allow_hyphen_values = true)]
+  pub command: Option<Vec<String>>,
+  /// Do not allocate TTY
+  #[arg(long = "no-tty", action = SetTrue)]
+  pub no_tty: Option<bool>,
+  /// Do not start linked services
+  #[arg(long = "no-deps", action = SetTrue)]
+  pub no_deps: Option<bool>,
+  /// Detach container on run
+  #[arg(long = "detach", action = SetTrue)]
+  pub detach: Option<bool>,
+  /// Map service ports to the host
+  #[arg(long = "service-ports", action = SetTrue)]
+  pub service_ports: Option<bool>,
+  /// Extra environment variables for the run
+  #[arg(long = "env", short = 'e', value_parser = env_parser)]
+  pub env: Option<HashMap<String, String>>,
+  /// Working directory inside the container
+  #[arg(long = "workdir")]
+  pub workdir: Option<String>,
+  /// User to run as inside the container
+  #[arg(long = "user")]
+  pub user: Option<String>,
+  /// Override the default entrypoint
+  #[arg(long = "entrypoint")]
+  pub entrypoint: Option<String>,
+  /// Pull the image before running
+  #[arg(long = "pull", action = SetTrue)]
+  pub pull: Option<bool>,
+}
+
+fn env_parser(args: &str) -> anyhow::Result<HashMap<String, String>> {
+  serde_qs::from_str(args).context("Failed to parse env")
+}
+
+//
+
 /// Destroys multiple Stacks in parallel that match pattern. Response: [BatchExecutionResponse].
 #[typeshare]
 #[derive(
@@ -368,7 +428,7 @@ pub struct BatchDestroyStack {
   /// Supports multiline and comma delineated combinations of the above.
   ///d
   /// Example:
-  /// ```
+  /// ```text
   /// # match all foo-* stacks
   /// foo-*
   /// # add some more

client/core/rs/src/api/mod.rs

@@ -20,7 +20,7 @@
 //! the schemas given in [read], [mod@write], [execute], and so on.
 //!
 //! For example, this is an example body for [read::GetDeployment]:
-//! ```
+//! ```json
 //! {
 //!   "type": "GetDeployment",
 //!   "params": {
@@ -56,7 +56,7 @@
 //!
 //! Request errors will be returned with a JSON body containing information about the error.
 //! They will have the following common format:
-//! ```
+//! ```json
 //! {
 //!   "error": "top level error message",
 //!   "trace": [

client/core/rs/src/api/read/alert.rs

@@ -20,7 +20,7 @@ pub struct ListAlerts {
   /// Pass a custom mongo query to filter the alerts.
   ///
   /// ## Example JSON
-  /// ```
+  /// ```json
   /// {
   ///   "resolved": "false",
   ///   "level": "CRITICAL",

client/core/rs/src/api/read/server.rs

@@ -623,6 +623,8 @@ pub struct GetServersSummaryResponse {
   pub total: I64,
   /// The number of healthy (`status: OK`) servers.
   pub healthy: I64,
+  /// The number of servers with warnings (e.g., version mismatch).
+  pub warning: I64,
   /// The number of unhealthy servers.
   pub unhealthy: I64,
   /// The number of disabled servers.

client/core/rs/src/busy.rs

@@ -68,7 +68,7 @@ impl Busy for ProcedureActionState {
 
 impl Busy for ActionActionState {
   fn busy(&self) -> bool {
-    self.running
+    self.running > 0
   }
 }
 

client/core/rs/src/deserializers/conversion.rs

@@ -35,11 +35,10 @@ impl<'de> Visitor<'de> for ConversionVisitor {
     write!(formatter, "string or Vec<Conversion>")
   }
 
-  fn visit_str<E>(self, v: &str) -> Result<Self::Value, E>
+  fn visit_string<E>(self, out: String) -> Result<Self::Value, E>
   where
     E: serde::de::Error,
   {
-    let out = v.to_string();
     if out.is_empty() || out.ends_with('\n') {
       Ok(out)
     } else {
@@ -47,6 +46,13 @@ impl<'de> Visitor<'de> for ConversionVisitor {
     }
   }
 
+  fn visit_str<E>(self, v: &str) -> Result<Self::Value, E>
+  where
+    E: serde::de::Error,
+  {
+    Self::visit_string(self, v.to_string())
+  }
+
   fn visit_seq<A>(self, seq: A) -> Result<Self::Value, A::Error>
   where
     A: serde::de::SeqAccess<'de>,
@@ -78,6 +84,13 @@ impl<'de> Visitor<'de> for OptionConversionVisitor {
     write!(formatter, "null or string or Vec<Conversion>")
   }
 
+  fn visit_string<E>(self, v: String) -> Result<Self::Value, E>
+  where
+    E: serde::de::Error,
+  {
+    ConversionVisitor.visit_string(v).map(Some)
+  }
+
   fn visit_str<E>(self, v: &str) -> Result<Self::Value, E>
   where
     E: serde::de::Error,

client/core/rs/src/deserializers/forgiving_vec.rs

@@ -1,5 +1,4 @@
 use serde::{
-  __private::de::{Content, ContentDeserializer},
   Deserialize, Deserializer,
   de::{IntoDeserializer, Visitor},
 };
@@ -69,17 +68,15 @@ impl<'de, T: Deserialize<'de>> Visitor<'de>
     let mut res =
       Vec::with_capacity(seq.size_hint().unwrap_or_default());
     loop {
-      match seq.next_element::<Content>() {
-        Ok(Some(content)) => {
-          match T::deserialize::<ContentDeserializer<'_, S::Error>>(
-            content.clone().into_deserializer(),
-          ) {
+      match seq.next_element::<serde_json::Value>() {
+        Ok(Some(value)) => {
+          match T::deserialize(value.clone().into_deserializer()) {
             Ok(item) => res.push(item),
             Err(e) => {
               // Since this is used to parse startup config (including logging config),
               // the tracing logging is not initialized. Need to use eprintln.
               eprintln!(
-                "WARN: failed to parse item in list | {content:?} | {e:?}",
+                "WARN: failed to parse item in list | {value:?} | {e:?}",
               )
             }
           }

client/core/rs/src/deserializers/item_or_vec.rs

@@ -0,0 +1,104 @@
+//! # Item or Vec<Item> deserializer.
+//!
+//! Used to convert `item: T` (struct / map) -> `item: Vec<T>` (seq) in schemas with backward compatibility.
+//! Supports deserializing either a T as Vec<T> with length 1, or a seq as Vec<T> directly.
+
+use serde::{
+  Deserialize, Deserializer,
+  de::{
+    DeserializeOwned, IntoDeserializer, Visitor,
+    value::{MapAccessDeserializer, SeqAccessDeserializer},
+  },
+};
+
+pub fn item_or_vec_deserializer<'de, D, T>(
+  deserializer: D,
+) -> Result<Vec<T>, D::Error>
+where
+  D: Deserializer<'de>,
+  T: DeserializeOwned,
+{
+  deserializer
+    .deserialize_any(ItemOrVecVisitor::<T>(std::marker::PhantomData))
+}
+
+pub fn option_item_or_vec_deserializer<'de, D, T>(
+  deserializer: D,
+) -> Result<Option<Vec<T>>, D::Error>
+where
+  D: Deserializer<'de>,
+  T: DeserializeOwned,
+{
+  deserializer.deserialize_any(OptionItemOrVecVisitor::<T>(
+    std::marker::PhantomData,
+  ))
+}
+
+struct ItemOrVecVisitor<T>(std::marker::PhantomData<T>);
+
+impl<'de, T> Visitor<'de> for ItemOrVecVisitor<T>
+where
+  T: Deserialize<'de>,
+{
+  type Value = Vec<T>;
+
+  fn expecting(
+    &self,
+    formatter: &mut std::fmt::Formatter,
+  ) -> std::fmt::Result {
+    write!(formatter, "Item or Vec<Item>")
+  }
+
+  fn visit_map<A>(self, map: A) -> Result<Self::Value, A::Error>
+  where
+    A: serde::de::MapAccess<'de>,
+  {
+    T::deserialize(
+      MapAccessDeserializer::new(map).into_deserializer(),
+    )
+    .map(|r| vec![r])
+  }
+
+  fn visit_seq<A>(self, seq: A) -> Result<Self::Value, A::Error>
+  where
+    A: serde::de::SeqAccess<'de>,
+  {
+    Vec::<T>::deserialize(
+      SeqAccessDeserializer::new(seq).into_deserializer(),
+    )
+  }
+}
+
+struct OptionItemOrVecVisitor<T>(std::marker::PhantomData<T>);
+
+impl<'de, T> Visitor<'de> for OptionItemOrVecVisitor<T>
+where
+  T: Deserialize<'de>,
+{
+  type Value = Option<Vec<T>>;
+
+  fn expecting(
+    &self,
+    formatter: &mut std::fmt::Formatter,
+  ) -> std::fmt::Result {
+    write!(formatter, "null or Item or Vec<Item>")
+  }
+
+  fn visit_map<A>(self, map: A) -> Result<Self::Value, A::Error>
+  where
+    A: serde::de::MapAccess<'de>,
+  {
+    ItemOrVecVisitor::<T>(std::marker::PhantomData)
+      .visit_map(map)
+      .map(Some)
+  }
+
+  fn visit_seq<A>(self, seq: A) -> Result<Self::Value, A::Error>
+  where
+    A: serde::de::SeqAccess<'de>,
+  {
+    ItemOrVecVisitor::<T>(std::marker::PhantomData)
+      .visit_seq(seq)
+      .map(Some)
+  }
+}

client/core/rs/src/deserializers/mod.rs

@@ -4,6 +4,7 @@ mod conversion;
 mod environment;
 mod file_contents;
 mod forgiving_vec;
+mod item_or_vec;
 mod labels;
 mod maybe_string_i64;
 mod permission;
@@ -14,6 +15,7 @@ pub use conversion::*;
 pub use environment::*;
 pub use file_contents::*;
 pub use forgiving_vec::*;
+pub use item_or_vec::*;
 pub use labels::*;
 pub use maybe_string_i64::*;
 pub use string_list::*;

client/core/rs/src/entities/action.rs

@@ -91,13 +91,13 @@ pub struct ActionConfig {
   /// 1. Regular CRON expression:
   ///
   /// (second, minute, hour, day, month, day-of-week)
-  /// ```
+  /// ```text
   /// 0 0 0 1,15 * ?
   /// ```
   ///
   /// 2. "English" expression via [english-to-cron](https://crates.io/crates/english-to-cron):
   ///
-  /// ```
+  /// ```text
   /// at midnight on the 1st and 15th of the month
   /// ```
   #[serde(default)]
@@ -222,8 +222,8 @@ impl Default for ActionConfig {
 #[typeshare]
 #[derive(Serialize, Deserialize, Debug, Clone, Copy, Default)]
 pub struct ActionActionState {
-  /// Whether the action is currently running.
-  pub running: bool,
+  /// Number of instances of the Action currently running
+  pub running: u32,
 }
 
 #[typeshare]