fmt

* 1.19.2-dev-0

* deploy 1.19.2-dev-1

* Add option to make run command detachable (#766)

* improve missing files log to include the missing paths

* bump mungos for urlencoding mongo creds

* Update permissioning.md - typo: "priviledges" -> "privileges" (#770)

* Add support for monaco-yaml and docker compose spec validatiaon (#772)

* deploy 1.19.2-dev-2

* on delete user, remove from all user groups

* fix Google login issues around `picture`

* unsafe_unsanitized_startup_config

* improve git provider support re #355

* should fix #468

* should fix exit code re #597

* deploy 1.19.2-dev-3

* fix container ports sorting (#776)

* missing serde default

* deploy 1.19.2-dev-4

* ensure git tokens trimmed in remote url

* Add link to Authentik support docs

* Fix incorrect commit branch when using linked repo re #634

* Better display container port ranges (#786)

* ensure build and sync also commit to correct branch. re #634

* deploy 1.19.2-dev-5

* Improve login form (#788)

* Use proper form for login, add autocomplete and names to input fields

* Do not return null if loading

* Remove unused function

* Cleanup and streamline

* improve login screen flash on reload

* first builder given same name as first server

* 1.19.2

---------

Co-authored-by: mbecker20 <max@mogh.tech>
Co-authored-by: Brian Bradley <brian.bradley.p@gmail.com>
Co-authored-by: Ravi Wolter-Krishan <rkn@gedikas.net>
Co-authored-by: Christopher Hoage <iam@chrishoage.com>
Co-authored-by: jack <45038833+jackra1n@users.noreply.github.com>

Cargo.toml

@@ -8,7 +8,7 @@ members = [
 ]
 
 [workspace.package]
-version = "1.19.0"
+version = "1.19.2"
 edition = "2024"
 authors = ["mbecker20 <becker.maxh@gmail.com>"]
 license = "GPL-3.0-or-later"
@@ -43,11 +43,11 @@ derive_variants = "1.0.0"
 mongo_indexed = "2.0.2"
 resolver_api = "3.0.0"
 toml_pretty = "1.2.0"
-mungos = "3.2.1"
+mungos = "3.2.2"
 svi = "1.2.0"
 
 # ASYNC
-reqwest = { version = "0.12.22", default-features = false, features = ["json", "stream", "rustls-tls-native-roots"] }
+reqwest = { version = "0.12.23", default-features = false, features = ["json", "stream", "rustls-tls-native-roots"] }
 tokio = { version = "1.47.1", features = ["full"] }
 tokio-util = { version = "0.7.16", features = ["io", "codec"] }
 tokio-stream = { version = "0.1.17", features = ["sync"] }
@@ -59,23 +59,24 @@ arc-swap = "1.7.1"
 # SERVER
 tokio-tungstenite = { version = "0.27.0", features = ["rustls-tls-native-roots"] }
 axum-extra = { version = "0.10.1", features = ["typed-header"] }
-tower-http = { version = "0.6.4", features = ["fs", "cors"] }
+tower-http = { version = "0.6.6", features = ["fs", "cors"] }
 axum-server = { version = "0.7.2", features = ["tls-rustls"] }
 axum = { version = "0.8.4", features = ["ws", "json", "macros"] }
 
 # SER/DE
 ipnetwork = { version = "0.21.1", features = ["serde"] }
-indexmap = { version = "2.10.0", features = ["serde"] }
+indexmap = { version = "2.11.0", features = ["serde"] }
 serde = { version = "1.0.219", features = ["derive"] }
 strum = { version = "0.27.2", features = ["derive"] }
+bson = { version = "2.15.0" } # must keep in sync with mongodb version
 serde_yaml_ng = "0.10.0"
-serde_json = "1.0.142"
+serde_json = "1.0.143"
 serde_qs = "0.15.0"
 toml = "0.9.5"
 
 # ERROR
 anyhow = "1.0.99"
-thiserror = "2.0.14"
+thiserror = "2.0.16"
 
 # LOGGING
 opentelemetry-otlp = { version = "0.30.0", features = ["tls-roots", "reqwest-rustls"] }
@@ -87,17 +88,17 @@ opentelemetry = "0.30.0"
 tracing = "0.1.41"
 
 # CONFIG
-clap = { version = "4.5.43", features = ["derive"] }
+clap = { version = "4.5.45", features = ["derive"] }
 dotenvy = "0.15.7"
 envy = "0.4.2"
 
 # CRYPTO / AUTH
-uuid = { version = "1.17.0", features = ["v4", "fast-rng", "serde"] }
+uuid = { version = "1.18.0", features = ["v4", "fast-rng", "serde"] }
 jsonwebtoken = { version = "9.3.1", default-features = false }
 openidconnect = "4.0.1"
 urlencoding = "2.1.3"
 nom_pem = "4.0.0"
-bcrypt = "0.17.0"
+bcrypt = "0.17.1"
 base64 = "0.22.1"
 rustls = "0.23.31"
 hmac = "0.12.1"
@@ -112,7 +113,7 @@ sysinfo = "0.37.0"
 
 # CLOUD
 aws-config = "1.8.5"
-aws-sdk-ec2 = "1.159.0"
+aws-sdk-ec2 = "1.161.0"
 aws-credential-types = "1.2.5"
 
 ## CRON
@@ -122,7 +123,7 @@ chrono = "0.4.41"
 croner = "3.0.0"
 
 # MISC
-async-compression = { version = "0.4.27", features = ["tokio", "gzip"] }
+async-compression = { version = "0.4.28", features = ["tokio", "gzip"] }
 derive_builder = "0.20.2"
 comfy-table = "7.1.4"
 typeshare = "1.0.4"
@@ -130,6 +131,6 @@ octorust = "0.10.0"
 dashmap = "6.1.0"
 wildcard = "0.3.0"
 colored = "3.0.0"
-regex = "1.11.1"
+regex = "1.11.2"
 bytes = "1.10.1"
-bson = "2.15.0"
+shell-escape = "0.1.5"

bin/cli/src/command/container.rs

@@ -292,7 +292,7 @@ impl PrintTable for (Option<&'_ str>, ContainerListItem) {
       Cell::new(self.0.unwrap_or("Unknown")),
       ports,
       Cell::new(networks.join(", ")),
-      Cell::new(clamp_sha(&image)),
+      Cell::new(clamp_sha(image)),
     ];
     if !links {
       return res;

bin/cli/src/command/execute.rs

@@ -212,9 +212,15 @@ pub async fn handle(
     Execution::BatchDestroyStack(data) => {
       println!("{}: {data:?}", "Data".dimmed())
     }
+    Execution::RunStackService(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
     Execution::TestAlerter(data) => {
       println!("{}: {data:?}", "Data".dimmed())
     }
+    Execution::SendAlert(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
     Execution::ClearRepoCache(data) => {
       println!("{}: {data:?}", "Data".dimmed())
     }
@@ -464,10 +470,18 @@ pub async fn handle(
     Execution::BatchDestroyStack(request) => {
       client.execute(request).await.map(ExecutionResult::Batch)
     }
+    Execution::RunStackService(request) => client
+      .execute(request)
+      .await
+      .map(|u| ExecutionResult::Single(u.into())),
     Execution::TestAlerter(request) => client
       .execute(request)
       .await
       .map(|u| ExecutionResult::Single(u.into())),
+    Execution::SendAlert(request) => client
+      .execute(request)
+      .await
+      .map(|u| ExecutionResult::Single(u.into())),
     Execution::ClearRepoCache(request) => client
       .execute(request)
       .await

bin/cli/src/command/list.rs

@@ -233,7 +233,7 @@ async fn list_schedules(
 }
 
 fn fix_tags<T>(
-  resources: &mut Vec<ResourceListItem<T>>,
+  resources: &mut [ResourceListItem<T>],
   tags: &HashMap<String, String>,
 ) {
   resources.iter_mut().for_each(|resource| {

bin/cli/src/command/mod.rs

@@ -111,7 +111,7 @@ fn print_items<T: PrintTable + Serialize>(
       };
       table.load_preset(preset).set_header(
         T::header(links)
-          .into_iter()
+          .iter()
           .map(|h| Cell::new(h).add_attribute(Attribute::Bold)),
       );
       for item in items {

bin/core/src/alert/discord.rs

@@ -17,6 +17,28 @@ pub async fn send_alert(
         "{level} | If you see this message, then Alerter **{name}** is **working**\n{link}"
       )
     }
+    AlertData::ServerVersionMismatch {
+      id,
+      name,
+      region,
+      server_version,
+      core_version,
+    } => {
+      let region = fmt_region(region);
+      let link = resource_link(ResourceTargetVariant::Server, id);
+      match alert.level {
+        SeverityLevel::Ok => {
+          format!(
+            "{level} | **{name}** ({region}) | Server version now matches core version ✅\n{link}"
+          )
+        }
+        _ => {
+          format!(
+            "{level} | **{name}** ({region}) | Version mismatch detected ⚠️\nServer: **{server_version}** | Core: **{core_version}**\n{link}"
+          )
+        }
+      }
+    }
     AlertData::ServerUnreachable {
       id,
       name,
@@ -207,6 +229,16 @@ pub async fn send_alert(
         "{level} | **{name}** ({resource_type}) | Scheduled run started 🕝\n{link}"
       )
     }
+    AlertData::Custom { message, details } => {
+      format!(
+        "{level} | {message}{}",
+        if details.is_empty() {
+          format_args!("")
+        } else {
+          format_args!("\n{details}")
+        }
+      )
+    }
     AlertData::None {} => Default::default(),
   };
   if !content.is_empty() {

bin/core/src/alert/mod.rs

@@ -48,8 +48,9 @@ pub async fn send_alerts(alerts: &[Alert]) {
       return;
     };
 
-    let handles =
-      alerts.iter().map(|alert| send_alert(&alerters, alert));
+    let handles = alerts
+      .iter()
+      .map(|alert| send_alert_to_alerters(&alerters, alert));
 
     join_all(handles).await;
   }
@@ -58,7 +59,7 @@ pub async fn send_alerts(alerts: &[Alert]) {
 }
 
 #[instrument(level = "debug")]
-async fn send_alert(alerters: &[Alerter], alert: &Alert) {
+async fn send_alert_to_alerters(alerters: &[Alerter], alert: &Alert) {
   if alerters.is_empty() {
     return;
   }
@@ -250,3 +251,238 @@ fn resource_link(
     id,
   )
 }
+
+/// Standard message content format
+/// used by Ntfy, Pushover.
+fn standard_alert_content(alert: &Alert) -> String {
+  let level = fmt_level(alert.level);
+  match &alert.data {
+    AlertData::Test { id, name } => {
+      let link = resource_link(ResourceTargetVariant::Alerter, id);
+      format!(
+        "{level} | If you see this message, then Alerter {name} is working\n{link}",
+      )
+    }
+    AlertData::ServerVersionMismatch {
+      id,
+      name,
+      region,
+      server_version,
+      core_version,
+    } => {
+      let region = fmt_region(region);
+      let link = resource_link(ResourceTargetVariant::Server, id);
+      match alert.level {
+        SeverityLevel::Ok => {
+          format!(
+            "{level} | {name} ({region}) | Server version now matches core version ✅\n{link}"
+          )
+        }
+        _ => {
+          format!(
+            "{level} | {name} ({region}) | Version mismatch detected ⚠️\nServer: {server_version} | Core: {core_version}\n{link}"
+          )
+        }
+      }
+    }
+    AlertData::ServerUnreachable {
+      id,
+      name,
+      region,
+      err,
+    } => {
+      let region = fmt_region(region);
+      let link = resource_link(ResourceTargetVariant::Server, id);
+      match alert.level {
+        SeverityLevel::Ok => {
+          format!("{level} | {name}{region} is now reachable\n{link}")
+        }
+        SeverityLevel::Critical => {
+          let err = err
+            .as_ref()
+            .map(|e| format!("\nerror: {e:#?}"))
+            .unwrap_or_default();
+          format!(
+            "{level} | {name}{region} is unreachable ❌\n{link}{err}"
+          )
+        }
+        _ => unreachable!(),
+      }
+    }
+    AlertData::ServerCpu {
+      id,
+      name,
+      region,
+      percentage,
+    } => {
+      let region = fmt_region(region);
+      let link = resource_link(ResourceTargetVariant::Server, id);
+      format!(
+        "{level} | {name}{region} cpu usage at {percentage:.1}%\n{link}",
+      )
+    }
+    AlertData::ServerMem {
+      id,
+      name,
+      region,
+      used_gb,
+      total_gb,
+    } => {
+      let region = fmt_region(region);
+      let link = resource_link(ResourceTargetVariant::Server, id);
+      let percentage = 100.0 * used_gb / total_gb;
+      format!(
+        "{level} | {name}{region} memory usage at {percentage:.1}%💾\n\nUsing {used_gb:.1} GiB / {total_gb:.1} GiB\n{link}",
+      )
+    }
+    AlertData::ServerDisk {
+      id,
+      name,
+      region,
+      path,
+      used_gb,
+      total_gb,
+    } => {
+      let region = fmt_region(region);
+      let link = resource_link(ResourceTargetVariant::Server, id);
+      let percentage = 100.0 * used_gb / total_gb;
+      format!(
+        "{level} | {name}{region} disk usage at {percentage:.1}%💿\nmount point: {path:?}\nusing {used_gb:.1} GiB / {total_gb:.1} GiB\n{link}",
+      )
+    }
+    AlertData::ContainerStateChange {
+      id,
+      name,
+      server_id: _server_id,
+      server_name,
+      from,
+      to,
+    } => {
+      let link = resource_link(ResourceTargetVariant::Deployment, id);
+      let to_state = fmt_docker_container_state(to);
+      format!(
+        "📦Deployment {name} is now {to_state}\nserver: {server_name}\nprevious: {from}\n{link}",
+      )
+    }
+    AlertData::DeploymentImageUpdateAvailable {
+      id,
+      name,
+      server_id: _server_id,
+      server_name,
+      image,
+    } => {
+      let link = resource_link(ResourceTargetVariant::Deployment, id);
+      format!(
+        "⬆ Deployment {name} has an update available\nserver: {server_name}\nimage: {image}\n{link}",
+      )
+    }
+    AlertData::DeploymentAutoUpdated {
+      id,
+      name,
+      server_id: _server_id,
+      server_name,
+      image,
+    } => {
+      let link = resource_link(ResourceTargetVariant::Deployment, id);
+      format!(
+        "⬆ Deployment {name} was updated automatically\nserver: {server_name}\nimage: {image}\n{link}",
+      )
+    }
+    AlertData::StackStateChange {
+      id,
+      name,
+      server_id: _server_id,
+      server_name,
+      from,
+      to,
+    } => {
+      let link = resource_link(ResourceTargetVariant::Stack, id);
+      let to_state = fmt_stack_state(to);
+      format!(
+        "🥞 Stack {name} is now {to_state}\nserver: {server_name}\nprevious: {from}\n{link}",
+      )
+    }
+    AlertData::StackImageUpdateAvailable {
+      id,
+      name,
+      server_id: _server_id,
+      server_name,
+      service,
+      image,
+    } => {
+      let link = resource_link(ResourceTargetVariant::Stack, id);
+      format!(
+        "⬆ Stack {name} has an update available\nserver: {server_name}\nservice: {service}\nimage: {image}\n{link}",
+      )
+    }
+    AlertData::StackAutoUpdated {
+      id,
+      name,
+      server_id: _server_id,
+      server_name,
+      images,
+    } => {
+      let link = resource_link(ResourceTargetVariant::Stack, id);
+      let images_label =
+        if images.len() > 1 { "images" } else { "image" };
+      let images_str = images.join(", ");
+      format!(
+        "⬆ Stack {name} was updated automatically ⏫\nserver: {server_name}\n{images_label}: {images_str}\n{link}",
+      )
+    }
+    AlertData::AwsBuilderTerminationFailed {
+      instance_id,
+      message,
+    } => {
+      format!(
+        "{level} | Failed to terminate AWS builder instance\ninstance id: {instance_id}\n{message}",
+      )
+    }
+    AlertData::ResourceSyncPendingUpdates { id, name } => {
+      let link =
+        resource_link(ResourceTargetVariant::ResourceSync, id);
+      format!(
+        "{level} | Pending resource sync updates on {name}\n{link}",
+      )
+    }
+    AlertData::BuildFailed { id, name, version } => {
+      let link = resource_link(ResourceTargetVariant::Build, id);
+      format!(
+        "{level} | Build {name} failed\nversion: v{version}\n{link}",
+      )
+    }
+    AlertData::RepoBuildFailed { id, name } => {
+      let link = resource_link(ResourceTargetVariant::Repo, id);
+      format!("{level} | Repo build for {name} failed\n{link}",)
+    }
+    AlertData::ProcedureFailed { id, name } => {
+      let link = resource_link(ResourceTargetVariant::Procedure, id);
+      format!("{level} | Procedure {name} failed\n{link}")
+    }
+    AlertData::ActionFailed { id, name } => {
+      let link = resource_link(ResourceTargetVariant::Action, id);
+      format!("{level} | Action {name} failed\n{link}")
+    }
+    AlertData::ScheduleRun {
+      resource_type,
+      id,
+      name,
+    } => {
+      let link = resource_link(*resource_type, id);
+      format!(
+        "{level} | {name} ({resource_type}) | Scheduled run started 🕝\n{link}"
+      )
+    }
+    AlertData::Custom { message, details } => {
+      format!(
+        "{level} | {message}{}",
+        if details.is_empty() {
+          format_args!("")
+        } else {
+          format_args!("\n{details}")
+        }
+      )
+    }
+    AlertData::None {} => Default::default(),
+  }
+}

bin/core/src/alert/ntfy.rs

@@ -8,205 +8,7 @@ pub async fn send_alert(
   email: Option<&str>,
   alert: &Alert,
 ) -> anyhow::Result<()> {
-  let level = fmt_level(alert.level);
-  let content = match &alert.data {
-    AlertData::Test { id, name } => {
-      let link = resource_link(ResourceTargetVariant::Alerter, id);
-      format!(
-        "{level} | If you see this message, then Alerter {name} is working\n{link}",
-      )
-    }
-    AlertData::ServerUnreachable {
-      id,
-      name,
-      region,
-      err,
-    } => {
-      let region = fmt_region(region);
-      let link = resource_link(ResourceTargetVariant::Server, id);
-      match alert.level {
-        SeverityLevel::Ok => {
-          format!("{level} | {name}{region} is now reachable\n{link}")
-        }
-        SeverityLevel::Critical => {
-          let err = err
-            .as_ref()
-            .map(|e| format!("\nerror: {e:#?}"))
-            .unwrap_or_default();
-          format!(
-            "{level} | {name}{region} is unreachable ❌\n{link}{err}"
-          )
-        }
-        _ => unreachable!(),
-      }
-    }
-    AlertData::ServerCpu {
-      id,
-      name,
-      region,
-      percentage,
-    } => {
-      let region = fmt_region(region);
-      let link = resource_link(ResourceTargetVariant::Server, id);
-      format!(
-        "{level} | {name}{region} cpu usage at {percentage:.1}%\n{link}",
-      )
-    }
-    AlertData::ServerMem {
-      id,
-      name,
-      region,
-      used_gb,
-      total_gb,
-    } => {
-      let region = fmt_region(region);
-      let link = resource_link(ResourceTargetVariant::Server, id);
-      let percentage = 100.0 * used_gb / total_gb;
-      format!(
-        "{level} | {name}{region} memory usage at {percentage:.1}%💾\n\nUsing {used_gb:.1} GiB / {total_gb:.1} GiB\n{link}",
-      )
-    }
-    AlertData::ServerDisk {
-      id,
-      name,
-      region,
-      path,
-      used_gb,
-      total_gb,
-    } => {
-      let region = fmt_region(region);
-      let link = resource_link(ResourceTargetVariant::Server, id);
-      let percentage = 100.0 * used_gb / total_gb;
-      format!(
-        "{level} | {name}{region} disk usage at {percentage:.1}%💿\nmount point: {path:?}\nusing {used_gb:.1} GiB / {total_gb:.1} GiB\n{link}",
-      )
-    }
-    AlertData::ContainerStateChange {
-      id,
-      name,
-      server_id: _server_id,
-      server_name,
-      from,
-      to,
-    } => {
-      let link = resource_link(ResourceTargetVariant::Deployment, id);
-      let to_state = fmt_docker_container_state(to);
-      format!(
-        "📦Deployment {name} is now {to_state}\nserver: {server_name}\nprevious: {from}\n{link}",
-      )
-    }
-    AlertData::DeploymentImageUpdateAvailable {
-      id,
-      name,
-      server_id: _server_id,
-      server_name,
-      image,
-    } => {
-      let link = resource_link(ResourceTargetVariant::Deployment, id);
-      format!(
-        "⬆ Deployment {name} has an update available\nserver: {server_name}\nimage: {image}\n{link}",
-      )
-    }
-    AlertData::DeploymentAutoUpdated {
-      id,
-      name,
-      server_id: _server_id,
-      server_name,
-      image,
-    } => {
-      let link = resource_link(ResourceTargetVariant::Deployment, id);
-      format!(
-        "⬆ Deployment {name} was updated automatically\nserver: {server_name}\nimage: {image}\n{link}",
-      )
-    }
-    AlertData::StackStateChange {
-      id,
-      name,
-      server_id: _server_id,
-      server_name,
-      from,
-      to,
-    } => {
-      let link = resource_link(ResourceTargetVariant::Stack, id);
-      let to_state = fmt_stack_state(to);
-      format!(
-        "🥞 Stack {name} is now {to_state}\nserver: {server_name}\nprevious: {from}\n{link}",
-      )
-    }
-    AlertData::StackImageUpdateAvailable {
-      id,
-      name,
-      server_id: _server_id,
-      server_name,
-      service,
-      image,
-    } => {
-      let link = resource_link(ResourceTargetVariant::Stack, id);
-      format!(
-        "⬆ Stack {name} has an update available\nserver: {server_name}\nservice: {service}\nimage: {image}\n{link}",
-      )
-    }
-    AlertData::StackAutoUpdated {
-      id,
-      name,
-      server_id: _server_id,
-      server_name,
-      images,
-    } => {
-      let link = resource_link(ResourceTargetVariant::Stack, id);
-      let images_label =
-        if images.len() > 1 { "images" } else { "image" };
-      let images_str = images.join(", ");
-      format!(
-        "⬆ Stack {name} was updated automatically ⏫\nserver: {server_name}\n{images_label}: {images_str}\n{link}",
-      )
-    }
-    AlertData::AwsBuilderTerminationFailed {
-      instance_id,
-      message,
-    } => {
-      format!(
-        "{level} | Failed to terminate AWS builder instance\ninstance id: {instance_id}\n{message}",
-      )
-    }
-    AlertData::ResourceSyncPendingUpdates { id, name } => {
-      let link =
-        resource_link(ResourceTargetVariant::ResourceSync, id);
-      format!(
-        "{level} | Pending resource sync updates on {name}\n{link}",
-      )
-    }
-    AlertData::BuildFailed { id, name, version } => {
-      let link = resource_link(ResourceTargetVariant::Build, id);
-      format!(
-        "{level} | Build {name} failed\nversion: v{version}\n{link}",
-      )
-    }
-    AlertData::RepoBuildFailed { id, name } => {
-      let link = resource_link(ResourceTargetVariant::Repo, id);
-      format!("{level} | Repo build for {name} failed\n{link}",)
-    }
-    AlertData::ProcedureFailed { id, name } => {
-      let link = resource_link(ResourceTargetVariant::Procedure, id);
-      format!("{level} | Procedure {name} failed\n{link}")
-    }
-    AlertData::ActionFailed { id, name } => {
-      let link = resource_link(ResourceTargetVariant::Action, id);
-      format!("{level} | Action {name} failed\n{link}")
-    }
-    AlertData::ScheduleRun {
-      resource_type,
-      id,
-      name,
-    } => {
-      let link = resource_link(*resource_type, id);
-      format!(
-        "{level} | {name} ({resource_type}) | Scheduled run started 🕝\n{link}"
-      )
-    }
-    AlertData::None {} => Default::default(),
-  };
-
+  let content = standard_alert_content(alert);
   if !content.is_empty() {
     send_message(url, email, content).await?;
   }

bin/core/src/alert/pushover.rs

@@ -7,205 +7,7 @@ pub async fn send_alert(
   url: &str,
   alert: &Alert,
 ) -> anyhow::Result<()> {
-  let level = fmt_level(alert.level);
-  let content = match &alert.data {
-    AlertData::Test { id, name } => {
-      let link = resource_link(ResourceTargetVariant::Alerter, id);
-      format!(
-        "{level} | If you see this message, then Alerter {name} is working\n{link}",
-      )
-    }
-    AlertData::ServerUnreachable {
-      id,
-      name,
-      region,
-      err,
-    } => {
-      let region = fmt_region(region);
-      let link = resource_link(ResourceTargetVariant::Server, id);
-      match alert.level {
-        SeverityLevel::Ok => {
-          format!("{level} | {name}{region} is now reachable\n{link}")
-        }
-        SeverityLevel::Critical => {
-          let err = err
-            .as_ref()
-            .map(|e| format!("\nerror: {e:#?}"))
-            .unwrap_or_default();
-          format!(
-            "{level} | {name}{region} is unreachable ❌\n{link}{err}"
-          )
-        }
-        _ => unreachable!(),
-      }
-    }
-    AlertData::ServerCpu {
-      id,
-      name,
-      region,
-      percentage,
-    } => {
-      let region = fmt_region(region);
-      let link = resource_link(ResourceTargetVariant::Server, id);
-      format!(
-        "{level} | {name}{region} cpu usage at {percentage:.1}%\n{link}",
-      )
-    }
-    AlertData::ServerMem {
-      id,
-      name,
-      region,
-      used_gb,
-      total_gb,
-    } => {
-      let region = fmt_region(region);
-      let link = resource_link(ResourceTargetVariant::Server, id);
-      let percentage = 100.0 * used_gb / total_gb;
-      format!(
-        "{level} | {name}{region} memory usage at {percentage:.1}%💾\n\nUsing {used_gb:.1} GiB / {total_gb:.1} GiB\n{link}",
-      )
-    }
-    AlertData::ServerDisk {
-      id,
-      name,
-      region,
-      path,
-      used_gb,
-      total_gb,
-    } => {
-      let region = fmt_region(region);
-      let link = resource_link(ResourceTargetVariant::Server, id);
-      let percentage = 100.0 * used_gb / total_gb;
-      format!(
-        "{level} | {name}{region} disk usage at {percentage:.1}%💿\nmount point: {path:?}\nusing {used_gb:.1} GiB / {total_gb:.1} GiB\n{link}",
-      )
-    }
-    AlertData::ContainerStateChange {
-      id,
-      name,
-      server_id: _server_id,
-      server_name,
-      from,
-      to,
-    } => {
-      let link = resource_link(ResourceTargetVariant::Deployment, id);
-      let to_state = fmt_docker_container_state(to);
-      format!(
-        "📦Deployment {name} is now {to_state}\nserver: {server_name}\nprevious: {from}\n{link}",
-      )
-    }
-    AlertData::DeploymentImageUpdateAvailable {
-      id,
-      name,
-      server_id: _server_id,
-      server_name,
-      image,
-    } => {
-      let link = resource_link(ResourceTargetVariant::Deployment, id);
-      format!(
-        "⬆ Deployment {name} has an update available\nserver: {server_name}\nimage: {image}\n{link}",
-      )
-    }
-    AlertData::DeploymentAutoUpdated {
-      id,
-      name,
-      server_id: _server_id,
-      server_name,
-      image,
-    } => {
-      let link = resource_link(ResourceTargetVariant::Deployment, id);
-      format!(
-        "⬆ Deployment {name} was updated automatically\nserver: {server_name}\nimage: {image}\n{link}",
-      )
-    }
-    AlertData::StackStateChange {
-      id,
-      name,
-      server_id: _server_id,
-      server_name,
-      from,
-      to,
-    } => {
-      let link = resource_link(ResourceTargetVariant::Stack, id);
-      let to_state = fmt_stack_state(to);
-      format!(
-        "🥞 Stack {name} is now {to_state}\nserver: {server_name}\nprevious: {from}\n{link}",
-      )
-    }
-    AlertData::StackImageUpdateAvailable {
-      id,
-      name,
-      server_id: _server_id,
-      server_name,
-      service,
-      image,
-    } => {
-      let link = resource_link(ResourceTargetVariant::Stack, id);
-      format!(
-        "⬆ Stack {name} has an update available\nserver: {server_name}\nservice: {service}\nimage: {image}\n{link}",
-      )
-    }
-    AlertData::StackAutoUpdated {
-      id,
-      name,
-      server_id: _server_id,
-      server_name,
-      images,
-    } => {
-      let link = resource_link(ResourceTargetVariant::Stack, id);
-      let images_label =
-        if images.len() > 1 { "images" } else { "image" };
-      let images_str = images.join(", ");
-      format!(
-        "⬆ Stack {name} was updated automatically ⏫\nserver: {server_name}\n{images_label}: {images_str}\n{link}",
-      )
-    }
-    AlertData::AwsBuilderTerminationFailed {
-      instance_id,
-      message,
-    } => {
-      format!(
-        "{level} | Failed to terminate AWS builder instance\ninstance id: {instance_id}\n{message}",
-      )
-    }
-    AlertData::ResourceSyncPendingUpdates { id, name } => {
-      let link =
-        resource_link(ResourceTargetVariant::ResourceSync, id);
-      format!(
-        "{level} | Pending resource sync updates on {name}\n{link}",
-      )
-    }
-    AlertData::BuildFailed { id, name, version } => {
-      let link = resource_link(ResourceTargetVariant::Build, id);
-      format!(
-        "{level} | Build {name} failed\nversion: v{version}\n{link}",
-      )
-    }
-    AlertData::RepoBuildFailed { id, name } => {
-      let link = resource_link(ResourceTargetVariant::Repo, id);
-      format!("{level} | Repo build for {name} failed\n{link}",)
-    }
-    AlertData::ProcedureFailed { id, name } => {
-      let link = resource_link(ResourceTargetVariant::Procedure, id);
-      format!("{level} | Procedure {name} failed\n{link}")
-    }
-    AlertData::ActionFailed { id, name } => {
-      let link = resource_link(ResourceTargetVariant::Action, id);
-      format!("{level} | Action {name} failed\n{link}")
-    }
-    AlertData::ScheduleRun {
-      resource_type,
-      id,
-      name,
-    } => {
-      let link = resource_link(*resource_type, id);
-      format!(
-        "{level} | {name} ({resource_type}) | Scheduled run started 🕝\n{link}"
-      )
-    }
-    AlertData::None {} => Default::default(),
-  };
-
+  let content = standard_alert_content(alert);
   if !content.is_empty() {
     send_message(url, content).await?;
   }

bin/core/src/alert/slack.rs

@@ -23,6 +23,35 @@ pub async fn send_alert(
       ];
       (text, blocks.into())
     }
+    AlertData::ServerVersionMismatch {
+      id,
+      name,
+      region,
+      server_version,
+      core_version,
+    } => {
+      let region = fmt_region(region);
+      let text = match alert.level {
+        SeverityLevel::Ok => {
+          format!(
+            "{level} | {name} ({region}) | Server version now matches core version ✅"
+          )
+        }
+        _ => {
+          format!(
+            "{level} | {name} ({region}) | Version mismatch detected ⚠️\nServer: {server_version} | Core: {core_version}"
+          )
+        }
+      };
+      let blocks = vec![
+        Block::header(text.clone()),
+        Block::section(resource_link(
+          ResourceTargetVariant::Server,
+          id,
+        )),
+      ];
+      (text, blocks.into())
+    }
     AlertData::ServerUnreachable {
       id,
       name,
@@ -429,6 +458,12 @@ pub async fn send_alert(
       ];
       (text, blocks.into())
     }
+    AlertData::Custom { message, details } => {
+      let text = format!("{level} | {message}");
+      let blocks =
+        vec![Block::header(text.clone()), Block::section(details)];
+      (text, blocks.into())
+    }
     AlertData::None {} => Default::default(),
   };
   if !text.is_empty() {

bin/core/src/api/execute/alerter.rs

@@ -1,18 +1,22 @@
+use anyhow::{Context, anyhow};
 use formatting::format_serror;
+use futures::{TryStreamExt, stream::FuturesUnordered};
 use komodo_client::{
-  api::execute::TestAlerter,
+  api::execute::{SendAlert, TestAlerter},
   entities::{
-    alert::{Alert, AlertData, SeverityLevel},
+    alert::{Alert, AlertData, AlertDataVariant, SeverityLevel},
     alerter::Alerter,
     komodo_timestamp,
     permission::PermissionLevel,
   },
 };
+use reqwest::StatusCode;
 use resolver_api::Resolve;
+use serror::AddStatusCodeError;
 
 use crate::{
   alert::send_alert_to_alerter, helpers::update::update_update,
-  permission::get_check_permissions,
+  permission::get_check_permissions, resource::list_full_for_user,
 };
 
 use super::ExecuteArgs;
@@ -71,3 +75,75 @@ impl Resolve<ExecuteArgs> for TestAlerter {
     Ok(update)
   }
 }
+
+//
+
+impl Resolve<ExecuteArgs> for SendAlert {
+  #[instrument(name = "SendAlert", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+  async fn resolve(
+    self,
+    ExecuteArgs { user, update }: &ExecuteArgs,
+  ) -> Result<Self::Response, Self::Error> {
+    let alerters = list_full_for_user::<Alerter>(
+      Default::default(),
+      user,
+      PermissionLevel::Execute.into(),
+      &[],
+    )
+    .await?
+    .into_iter()
+    .filter(|a| {
+      a.config.enabled
+        && (self.alerters.is_empty()
+          || self.alerters.contains(&a.name)
+          || self.alerters.contains(&a.id))
+        && (a.config.alert_types.is_empty()
+          || a.config.alert_types.contains(&AlertDataVariant::Custom))
+    })
+    .collect::<Vec<_>>();
+
+    if alerters.is_empty() {
+      return Err(anyhow!(
+        "Could not find any valid alerters to send to, this required Execute permissions on the Alerter"
+      ).status_code(StatusCode::BAD_REQUEST));
+    }
+
+    let mut update = update.clone();
+
+    let ts = komodo_timestamp();
+
+    let alert = Alert {
+      id: Default::default(),
+      ts,
+      resolved: true,
+      level: self.level,
+      target: update.target.clone(),
+      data: AlertData::Custom {
+        message: self.message,
+        details: self.details,
+      },
+      resolved_ts: Some(ts),
+    };
+
+    update.push_simple_log(
+      "Send alert",
+      serde_json::to_string_pretty(&alert)
+        .context("Failed to serialize alert to JSON")?,
+    );
+
+    if let Err(e) = alerters
+      .iter()
+      .map(|alerter| send_alert_to_alerter(alerter, &alert))
+      .collect::<FuturesUnordered<_>>()
+      .try_collect::<Vec<_>>()
+      .await
+    {
+      update.push_error_log("Send Error", format_serror(&e.into()));
+    };
+
+    update.finalize();
+    update_update(update.clone()).await?;
+
+    Ok(update)
+  }
+}

bin/core/src/api/execute/build.rs

@@ -1,4 +1,8 @@
-use std::{future::IntoFuture, time::Duration};
+use std::{
+  collections::{HashMap, HashSet},
+  future::IntoFuture,
+  time::Duration,
+};
 
 use anyhow::{Context, anyhow};
 use database::mungos::{
@@ -20,7 +24,7 @@ use komodo_client::{
   entities::{
     alert::{Alert, AlertData, SeverityLevel},
     all_logs_success,
-    build::{Build, BuildConfig, ImageRegistryConfig},
+    build::{Build, BuildConfig},
     builder::{Builder, BuilderConfig},
     deployment::DeploymentState,
     komodo_timestamp,
@@ -133,8 +137,8 @@ impl Resolve<ExecuteArgs> for RunBuild {
     let git_token =
       build_git_token(&mut build, repo.as_mut()).await?;
 
-    let registry_token =
-      validate_account_extract_registry_token(&build).await?;
+    let registry_tokens =
+      validate_account_extract_registry_tokens(&build).await?;
 
     let cancel = CancellationToken::new();
     let cancel_clone = cancel.clone();
@@ -284,7 +288,7 @@ impl Resolve<ExecuteArgs> for RunBuild {
           .request(api::build::Build {
             build: build.clone(),
             repo,
-            registry_token,
+            registry_tokens,
             replacers: secret_replacers.into_iter().collect(),
             // Push a commit hash tagged image
             additional_tags: if update.commit_hash.is_empty() {
@@ -608,34 +612,48 @@ async fn handle_post_build_redeploy(build_id: &str) {
 /// This will make sure that a build with non-none image registry has an account attached,
 /// and will check the core config for a token matching requirements.
 /// Otherwise it is left to periphery.
-async fn validate_account_extract_registry_token(
+async fn validate_account_extract_registry_tokens(
   Build {
-    config:
-      BuildConfig {
-        image_registry:
-          ImageRegistryConfig {
-            domain, account, ..
-          },
-        ..
-      },
+    config: BuildConfig { image_registry, .. },
     ..
   }: &Build,
-) -> serror::Result<Option<String>> {
-  if domain.is_empty() {
-    return Ok(None);
-  }
-  if account.is_empty() {
-    return Err(
-      anyhow!(
-        "Must attach account to use registry provider {domain}"
-      )
-      .into(),
+  // Maps (domain, account) -> token
+) -> serror::Result<Vec<(String, String, String)>> {
+  let mut res = HashMap::with_capacity(image_registry.capacity());
+
+  for (domain, account) in image_registry
+    .iter()
+    .map(|r| (r.domain.as_str(), r.account.as_str()))
+    // This ensures uniqueness / prevents redundant logins
+    .collect::<HashSet<_>>()
+  {
+    if domain.is_empty() {
+      continue;
+    }
+    if account.is_empty() {
+      return Err(
+        anyhow!(
+          "Must attach account to use registry provider {domain}"
+        )
+        .into(),
+      );
+    }
+    let Some(registry_token) = registry_token(domain, account).await.with_context(
+      || format!("Failed to get registry token in call to db. Stopping run. | {domain} | {account}"),
+    )? else {
+      continue;
+    };
+
+    res.insert(
+      (domain.to_string(), account.to_string()),
+      registry_token,
     );
   }
 
-  let registry_token = registry_token(domain, account).await.with_context(
-    || format!("Failed to get registry token in call to db. Stopping run. | {domain} | {account}"),
-  )?;
-
-  Ok(registry_token)
+  Ok(
+    res
+      .into_iter()
+      .map(|((domain, account), token)| (domain, account, token))
+      .collect(),
+  )
 }

bin/core/src/api/execute/deployment.rs

@@ -12,7 +12,7 @@ use komodo_client::{
     deployment::{
       Deployment, DeploymentImage, extract_registry_domain,
     },
-    get_image_name, komodo_timestamp, optional_string,
+    get_image_names, komodo_timestamp, optional_string,
     permission::PermissionLevel,
     server::Server,
     update::{Log, Update},
@@ -115,8 +115,11 @@ impl Resolve<ExecuteArgs> for Deploy {
     let (version, registry_token) = match &deployment.config.image {
       DeploymentImage::Build { build_id, version } => {
         let build = resource::get::<Build>(build_id).await?;
-        let image_name = get_image_name(&build)
-          .context("failed to create image name")?;
+        let image_names = get_image_names(&build);
+        let image_name = image_names
+          .first()
+          .context("No image name could be created")
+          .context("Failed to create image name")?;
         let version = if version.is_none() {
           build.config.version
         } else {
@@ -133,21 +136,27 @@ impl Resolve<ExecuteArgs> for Deploy {
         deployment.config.image = DeploymentImage::Image {
           image: format!("{image_name}:{version_str}"),
         };
-        if build.config.image_registry.domain.is_empty() {
+        let first_registry = build
+          .config
+          .image_registry
+          .first()
+          .unwrap_or(ImageRegistryConfig::static_default());
+        if first_registry.domain.is_empty() {
           (version, None)
         } else {
           let ImageRegistryConfig {
             domain, account, ..
-          } = build.config.image_registry;
+          } = first_registry;
           if deployment.config.image_registry_account.is_empty() {
-            deployment.config.image_registry_account = account
+            deployment.config.image_registry_account =
+              account.to_string();
           }
           let token = if !deployment
             .config
             .image_registry_account
             .is_empty()
           {
-            registry_token(&domain, &deployment.config.image_registry_account).await.with_context(
+            registry_token(domain, &deployment.config.image_registry_account).await.with_context(
               || format!("Failed to get git token in call to db. Stopping run. | {domain} | {}", deployment.config.image_registry_account),
             )?
           } else {
@@ -240,8 +249,11 @@ pub async fn pull_deployment_inner(
   let (image, account, token) = match deployment.config.image {
     DeploymentImage::Build { build_id, version } => {
       let build = resource::get::<Build>(&build_id).await?;
-      let image_name = get_image_name(&build)
-        .context("failed to create image name")?;
+      let image_names = get_image_names(&build);
+      let image_name = image_names
+        .first()
+        .context("No image name could be created")
+        .context("Failed to create image name")?;
       let version = if version.is_none() {
         build.config.version.to_string()
       } else {
@@ -255,26 +267,31 @@ pub async fn pull_deployment_inner(
       };
       // replace image with corresponding build image.
       let image = format!("{image_name}:{version}");
-      if build.config.image_registry.domain.is_empty() {
+      let first_registry = build
+        .config
+        .image_registry
+        .first()
+        .unwrap_or(ImageRegistryConfig::static_default());
+      if first_registry.domain.is_empty() {
         (image, None, None)
       } else {
         let ImageRegistryConfig {
           domain, account, ..
-        } = build.config.image_registry;
+        } = first_registry;
         let account =
           if deployment.config.image_registry_account.is_empty() {
             account
           } else {
-            deployment.config.image_registry_account
+            &deployment.config.image_registry_account
           };
         let token = if !account.is_empty() {
-          registry_token(&domain, &account).await.with_context(
+          registry_token(domain, account).await.with_context(
               || format!("Failed to get git token in call to db. Stopping run. | {domain} | {account}"),
             )?
         } else {
           None
         };
-        (image, optional_string(&account), token)
+        (image, optional_string(account), token)
       }
     }
     DeploymentImage::Image { image } => {

bin/core/src/api/execute/mod.rs

@@ -102,6 +102,7 @@ pub enum ExecuteRequest {
   UnpauseStack(UnpauseStack),
   DestroyStack(DestroyStack),
   BatchDestroyStack(BatchDestroyStack),
+  RunStackService(RunStackService),
 
   // ==== DEPLOYMENT ====
   Deploy(Deploy),
@@ -139,6 +140,7 @@ pub enum ExecuteRequest {
 
   // ==== ALERTER ====
   TestAlerter(TestAlerter),
+  SendAlert(SendAlert),
 
   // ==== SYNC ====
   RunSync(RunSync),
@@ -219,24 +221,33 @@ pub fn inner_handler(
       ));
     }
 
+    // Spawn a task for the execution which continues
+    // running after this method returns.
     let handle =
       tokio::spawn(task(req_id, request, user, update.clone()));
 
+    // Spawns another task to monitor the first for failures,
+    // and add the log to Update about it (which primary task can't do because it errored out)
     tokio::spawn({
       let update_id = update.id.clone();
       async move {
         let log = match handle.await {
           Ok(Err(e)) => {
             warn!("/execute request {req_id} task error: {e:#}",);
-            Log::error("task error", format_serror(&e.into()))
+            Log::error("Task Error", format_serror(&e.into()))
           }
           Err(e) => {
             warn!("/execute request {req_id} spawn error: {e:?}",);
-            Log::error("spawn error", format!("{e:#?}"))
+            Log::error("Spawn Error", format!("{e:#?}"))
           }
           _ => return,
         };
         let res = async {
+          // Nothing to do if update was never actually created,
+          // which is the case when the id is empty.
+          if update_id.is_empty() {
+            return Ok(());
+          }
           let mut update =
             find_one_by_id(&db_client().updates, &update_id)
               .await

bin/core/src/api/execute/stack.rs

@@ -1,15 +1,23 @@
+use std::{collections::HashSet, str::FromStr};
+
 use anyhow::Context;
-use database::mungos::mongodb::bson::{doc, to_document};
+use database::mungos::mongodb::bson::{
+  doc, oid::ObjectId, to_bson, to_document,
+};
 use formatting::format_serror;
 use interpolate::Interpolator;
 use komodo_client::{
   api::{execute::*, write::RefreshStackCache},
   entities::{
+    FileContents,
     permission::PermissionLevel,
     repo::Repo,
     server::Server,
-    stack::{Stack, StackInfo},
+    stack::{
+      Stack, StackFileRequires, StackInfo, StackRemoteFileContents,
+    },
     update::{Log, Update},
+    user::User,
   },
 };
 use periphery_client::api::compose::*;
@@ -21,7 +29,9 @@ use crate::{
     periphery_client,
     query::{VariablesAndSecrets, get_variables_and_secrets},
     stack_git_token,
-    update::{add_update_without_send, update_update},
+    update::{
+      add_update_without_send, init_execution_update, update_update,
+    },
   },
   monitor::update_cache_for_server,
   permission::get_check_permissions,
@@ -179,7 +189,15 @@ impl Resolve<ExecuteArgs> for DeployStack {
       ) = if deployed {
         (
           Some(latest_services.clone()),
-          Some(file_contents.clone()),
+          Some(
+            file_contents
+              .iter()
+              .map(|f| FileContents {
+                path: f.path.clone(),
+                contents: f.contents.clone(),
+              })
+              .collect(),
+          ),
           compose_config,
           commit_hash.clone(),
           commit_message.clone(),
@@ -289,62 +307,347 @@ impl Resolve<ExecuteArgs> for DeployStackIfChanged {
       PermissionLevel::Execute.into(),
     )
     .await?;
+
     RefreshStackCache {
       stack: stack.id.clone(),
     }
     .resolve(&WriteArgs { user: user.clone() })
     .await?;
+
     let stack = resource::get::<Stack>(&stack.id).await?;
-    let changed = match (
+
+    let action = match (
       &stack.info.deployed_contents,
       &stack.info.remote_contents,
     ) {
       (Some(deployed_contents), Some(latest_contents)) => {
-        let changed = || {
-          for latest in latest_contents {
-            let Some(deployed) = deployed_contents
-              .iter()
-              .find(|c| c.path == latest.path)
-            else {
-              return true;
-            };
-            if latest.contents != deployed.contents {
-              return true;
-            }
-          }
-          false
-        };
-        changed()
+        let services = stack
+          .info
+          .latest_services
+          .iter()
+          .map(|s| s.service_name.clone())
+          .collect::<Vec<_>>();
+        resolve_deploy_if_changed_action(
+          deployed_contents,
+          latest_contents,
+          &services,
+        )
       }
-      (None, _) => true,
-      _ => false,
+      (None, _) => DeployIfChangedAction::FullDeploy,
+      _ => DeployIfChangedAction::Services {
+        deploy: Vec::new(),
+        restart: Vec::new(),
+      },
     };
 
     let mut update = update.clone();
 
-    if !changed {
-      update.push_simple_log(
-        "Diff compose files",
-        String::from("Deploy cancelled after no changes detected."),
-      );
-      update.finalize();
-      return Ok(update);
-    }
+    match action {
+      // Existing path pre 1.19.1
+      DeployIfChangedAction::FullDeploy => {
+        // Don't actually send it here, let the handler send it after it can set action state.
+        // This is usually done in crate::helpers::update::init_execution_update.
+        update.id = add_update_without_send(&update).await?;
 
-    // Don't actually send it here, let the handler send it after it can set action state.
-    // This is usually done in crate::helpers::update::init_execution_update.
-    update.id = add_update_without_send(&update).await?;
+        DeployStack {
+          stack: stack.name,
+          services: Vec::new(),
+          stop_time: self.stop_time,
+        }
+        .resolve(&ExecuteArgs {
+          user: user.clone(),
+          update,
+        })
+        .await
+      }
+      DeployIfChangedAction::FullRestart => {
+        // For git repo based stacks, need to do a
+        // PullStack in order to ensure latest repo contents on the
+        // host before restart.
+        maybe_pull_stack(&stack, Some(&mut update)).await?;
 
-    DeployStack {
-      stack: stack.name,
-      services: Vec::new(),
-      stop_time: self.stop_time,
+        let mut update =
+          restart_services(stack.name, Vec::new(), user).await?;
+
+        if update.success {
+          // Need to update 'info.deployed_contents' with the
+          // latest contents so next check doesn't read the same diff.
+          update_deployed_contents_with_latest(
+            &stack.id,
+            stack.info.remote_contents,
+            &mut update,
+          )
+          .await;
+        }
+
+        Ok(update)
+      }
+      DeployIfChangedAction::Services { deploy, restart } => {
+        match (deploy.is_empty(), restart.is_empty()) {
+          // Both empty, nothing to do
+          (true, true) => {
+            update.push_simple_log(
+              "Diff compose files",
+              String::from(
+                "Deploy cancelled after no changes detected.",
+              ),
+            );
+            update.finalize();
+            Ok(update)
+          }
+          // Only restart
+          (true, false) => {
+            // For git repo based stacks, need to do a
+            // PullStack in order to ensure latest repo contents on the
+            // host before restart. Only necessary if no "deploys" (deploy already pulls stack).
+            maybe_pull_stack(&stack, Some(&mut update)).await?;
+
+            let mut update =
+              restart_services(stack.name, restart, user).await?;
+
+            if update.success {
+              // Need to update 'info.deployed_contents' with the
+              // latest contents so next check doesn't read the same diff.
+              update_deployed_contents_with_latest(
+                &stack.id,
+                stack.info.remote_contents,
+                &mut update,
+              )
+              .await;
+            }
+
+            Ok(update)
+          }
+          // Only deploy
+          (false, true) => {
+            deploy_services(stack.name, deploy, user).await
+          }
+          // Deploy then restart, returning non-db update with executed services.
+          (false, false) => {
+            update.push_simple_log(
+              "Execute Deploys",
+              format!("Deploying: {}", deploy.join(", "),),
+            );
+            // This already updates 'stack.info.deployed_services',
+            // restart doesn't require this again.
+            let deploy_update =
+              deploy_services(stack.name.clone(), deploy, user)
+                .await?;
+            if !deploy_update.success {
+              update.push_error_log(
+                "Execute Deploys",
+                String::from("There was a failure in service deploy"),
+              );
+              update.finalize();
+              return Ok(update);
+            }
+
+            update.push_simple_log(
+              "Execute Restarts",
+              format!("Restarting: {}", restart.join(", "),),
+            );
+            let restart_update =
+              restart_services(stack.name, restart, user).await?;
+            if !restart_update.success {
+              update.push_error_log(
+                "Execute Restarts",
+                String::from(
+                  "There was a failure in a service restart",
+                ),
+              );
+            }
+
+            update.finalize();
+            Ok(update)
+          }
+        }
+      }
     }
+  }
+}
+
+async fn deploy_services(
+  stack: String,
+  services: Vec<String>,
+  user: &User,
+) -> serror::Result<Update> {
+  // The existing update is initialized to DeployStack,
+  // but also has not been created on database.
+  // Setup a new update here.
+  let req = ExecuteRequest::DeployStack(DeployStack {
+    stack,
+    services,
+    stop_time: None,
+  });
+  let update = init_execution_update(&req, user).await?;
+  let ExecuteRequest::DeployStack(req) = req else {
+    unreachable!()
+  };
+  req
     .resolve(&ExecuteArgs {
       user: user.clone(),
       update,
     })
     .await
+}
+
+async fn restart_services(
+  stack: String,
+  services: Vec<String>,
+  user: &User,
+) -> serror::Result<Update> {
+  // The existing update is initialized to DeployStack,
+  // but also has not been created on database.
+  // Setup a new update here.
+  let req =
+    ExecuteRequest::RestartStack(RestartStack { stack, services });
+  let update = init_execution_update(&req, user).await?;
+  let ExecuteRequest::RestartStack(req) = req else {
+    unreachable!()
+  };
+  req
+    .resolve(&ExecuteArgs {
+      user: user.clone(),
+      update,
+    })
+    .await
+}
+
+/// This can safely be called in [DeployStackIfChanged]
+/// when there are ONLY changes to config files requiring restart,
+/// AFTER the restart has been successfully completed.
+///
+/// In the case the if changed action is not FullDeploy,
+/// the only file diff possible is to config files.
+/// Also note either full or service deploy will already update 'deployed_contents'
+/// making this method unnecessary in those cases.
+///
+/// Changes to config files after restart is applied should
+/// be taken as the deployed contents, otherwise next changed check
+/// will restart service again for no reason.
+async fn update_deployed_contents_with_latest(
+  id: &str,
+  contents: Option<Vec<StackRemoteFileContents>>,
+  update: &mut Update,
+) {
+  let Some(contents) = contents else {
+    return;
+  };
+  let contents = contents
+    .into_iter()
+    .map(|f| FileContents {
+      path: f.path,
+      contents: f.contents,
+    })
+    .collect::<Vec<_>>();
+  if let Err(e) = (async {
+    let contents = to_bson(&contents)
+      .context("Failed to serialize contents to bson")?;
+    let id =
+      ObjectId::from_str(id).context("Id is not valid ObjectId")?;
+    db_client()
+      .stacks
+      .update_one(
+        doc! { "_id": id },
+        doc! { "$set": { "info.deployed_contents": contents } },
+      )
+      .await
+      .context("Failed to update stack 'deployed_contents'")?;
+    anyhow::Ok(())
+  })
+  .await
+  {
+    update.push_error_log(
+      "Update content cache",
+      format_serror(&e.into()),
+    );
+    update.finalize();
+    let _ = update_update(update.clone()).await;
+  }
+}
+
+enum DeployIfChangedAction {
+  /// Changes to any compose or env files
+  /// always lead to this.
+  FullDeploy,
+  /// If the above is not met, then changes to
+  /// any changed additional file with `requires = "Restart"`
+  /// and empty services array will lead to this.
+  FullRestart,
+  /// If all changed additional files have specific services
+  /// they depend on, collect the final necessary
+  /// services to deploy / restart.
+  /// If eg `deploy` is empty, no services will be redeployed, same for `restart`.
+  /// If both are empty, nothing is to be done.
+  Services {
+    deploy: Vec<String>,
+    restart: Vec<String>,
+  },
+}
+
+fn resolve_deploy_if_changed_action(
+  deployed_contents: &[FileContents],
+  latest_contents: &[StackRemoteFileContents],
+  all_services: &[String],
+) -> DeployIfChangedAction {
+  let mut full_restart = false;
+  let mut deploy = HashSet::<String>::new();
+  let mut restart = HashSet::<String>::new();
+
+  for latest in latest_contents {
+    let Some(deployed) =
+      deployed_contents.iter().find(|c| c.path == latest.path)
+    else {
+      // If file doesn't exist in deployed contents, do full
+      // deploy to align this.
+      return DeployIfChangedAction::FullDeploy;
+    };
+    // Ignore unchanged files
+    if latest.contents == deployed.contents {
+      continue;
+    }
+    match (latest.requires, latest.services.is_empty()) {
+      (StackFileRequires::Redeploy, true) => {
+        // File has requires = "Redeploy" at global level.
+        // Can do early return here.
+        return DeployIfChangedAction::FullDeploy;
+      }
+      (StackFileRequires::Redeploy, false) => {
+        // Requires redeploy on specific services
+        deploy.extend(latest.services.clone());
+      }
+      (StackFileRequires::Restart, true) => {
+        // Services empty -> Full restart
+        full_restart = true;
+      }
+      (StackFileRequires::Restart, false) => {
+        restart.extend(latest.services.clone());
+      }
+      (StackFileRequires::None, _) => {
+        // File can be ignored even with changes.
+        continue;
+      }
+    }
+  }
+
+  match (full_restart, deploy.is_empty()) {
+    // Full restart required with NO deploys needed -> Full Restart
+    (true, true) => DeployIfChangedAction::FullRestart,
+    // Full restart required WITH deploys needed -> Deploy those, restart all others
+    (true, false) => DeployIfChangedAction::Services {
+      restart: all_services
+        .iter()
+        // Only keep ones that don't need deploy
+        .filter(|&s| !deploy.contains(s))
+        .cloned()
+        .collect(),
+      deploy: deploy.into_iter().collect(),
+    },
+    // No full restart needed -> Deploy / restart as. pickedup.
+    (false, _) => DeployIfChangedAction::Services {
+      deploy: deploy.into_iter().collect(),
+      restart: restart.into_iter().collect(),
+    },
   }
 }
 
@@ -371,6 +674,31 @@ impl Resolve<ExecuteArgs> for BatchPullStack {
   }
 }
 
+async fn maybe_pull_stack(
+  stack: &Stack,
+  update: Option<&mut Update>,
+) -> anyhow::Result<()> {
+  if stack.config.files_on_host
+    || (stack.config.repo.is_empty()
+      && stack.config.linked_repo.is_empty())
+  {
+    // Not repo based, no pull necessary
+    return Ok(());
+  }
+  let server =
+    resource::get::<Server>(&stack.config.server_id).await?;
+  let repo = if stack.config.repo.is_empty()
+    && !stack.config.linked_repo.is_empty()
+  {
+    Some(resource::get::<Repo>(&stack.config.linked_repo).await?)
+  } else {
+    None
+  };
+  pull_stack_inner(stack.clone(), Vec::new(), &server, repo, update)
+    .await?;
+  Ok(())
+}
+
 pub async fn pull_stack_inner(
   mut stack: Stack,
   services: Vec<String>,
@@ -630,3 +958,95 @@ impl Resolve<ExecuteArgs> for DestroyStack {
     .map_err(Into::into)
   }
 }
+
+impl Resolve<ExecuteArgs> for RunStackService {
+  #[instrument(name = "RunStackService", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+  async fn resolve(
+    self,
+    ExecuteArgs { user, update }: &ExecuteArgs,
+  ) -> serror::Result<Update> {
+    let (mut stack, server) = get_stack_and_server(
+      &self.stack,
+      user,
+      PermissionLevel::Execute.into(),
+      true,
+    )
+    .await?;
+
+    let mut repo = if !stack.config.files_on_host
+      && !stack.config.linked_repo.is_empty()
+    {
+      crate::resource::get::<Repo>(&stack.config.linked_repo)
+        .await?
+        .into()
+    } else {
+      None
+    };
+
+    let action_state =
+      action_states().stack.get_or_insert_default(&stack.id).await;
+
+    let _action_guard =
+      action_state.update(|state| state.deploying = true)?;
+
+    let mut update = update.clone();
+    update_update(update.clone()).await?;
+
+    let git_token =
+      stack_git_token(&mut stack, repo.as_mut()).await?;
+
+    let registry_token = crate::helpers::registry_token(
+      &stack.config.registry_provider,
+      &stack.config.registry_account,
+    ).await.with_context(
+      || format!("Failed to get registry token in call to db. Stopping run. | {} | {}", stack.config.registry_provider, stack.config.registry_account),
+    )?;
+
+    let secret_replacers = if !stack.config.skip_secret_interp {
+      let VariablesAndSecrets { variables, secrets } =
+        get_variables_and_secrets().await?;
+
+      let mut interpolator =
+        Interpolator::new(Some(&variables), &secrets);
+
+      interpolator.interpolate_stack(&mut stack)?;
+      if let Some(repo) = repo.as_mut()
+        && !repo.config.skip_secret_interp
+      {
+        interpolator.interpolate_repo(repo)?;
+      }
+      interpolator.push_logs(&mut update.logs);
+
+      interpolator.secret_replacers
+    } else {
+      Default::default()
+    };
+
+    let log = periphery_client(&server)?
+      .request(ComposeRun {
+        stack,
+        repo,
+        git_token,
+        registry_token,
+        replacers: secret_replacers.into_iter().collect(),
+        service: self.service,
+        command: self.command,
+        no_tty: self.no_tty,
+        no_deps: self.no_deps,
+        detach: self.detach,
+        service_ports: self.service_ports,
+        env: self.env,
+        workdir: self.workdir,
+        user: self.user,
+        entrypoint: self.entrypoint,
+        pull: self.pull,
+      })
+      .await?;
+
+    update.logs.push(log);
+    update.finalize();
+    update_update(update.clone()).await?;
+
+    Ok(update)
+  }
+}

bin/core/src/api/read/server.rs

@@ -71,12 +71,24 @@ impl Resolve<ReadArgs> for GetServersSummary {
       &[],
     )
     .await?;
+
+    let core_version = env!("CARGO_PKG_VERSION");
     let mut res = GetServersSummaryResponse::default();
+
     for server in servers {
       res.total += 1;
       match server.info.state {
         ServerState::Ok => {
-          res.healthy += 1;
+          // Check for version mismatch
+          let has_version_mismatch = !server.info.version.is_empty()
+            && server.info.version != "Unknown"
+            && server.info.version != core_version;
+
+          if has_version_mismatch {
+            res.warning += 1;
+          } else {
+            res.healthy += 1;
+          }
         }
         ServerState::NotOk => {
           res.unhealthy += 1;

bin/core/src/api/write/build.rs

@@ -186,7 +186,7 @@ async fn write_dockerfile_contents_git(
 ) -> serror::Result<Update> {
   let WriteBuildFileContents { build: _, contents } = req;
 
-  let mut clone_args: RepoExecutionArgs = if !build
+  let mut repo_args: RepoExecutionArgs = if !build
     .config
     .files_on_host
     && !build.config.linked_repo.is_empty()
@@ -196,8 +196,8 @@ async fn write_dockerfile_contents_git(
   } else {
     (&build).into()
   };
-  let root = clone_args.unique_path(&core_config().repo_directory)?;
-  clone_args.destination = Some(root.display().to_string());
+  let root = repo_args.unique_path(&core_config().repo_directory)?;
+  repo_args.destination = Some(root.display().to_string());
 
   let build_path = build
     .config
@@ -220,11 +220,11 @@ async fn write_dockerfile_contents_git(
     })?;
   }
 
-  let access_token = if let Some(account) = &clone_args.account {
-    git_token(&clone_args.provider, account, |https| clone_args.https = https)
+  let access_token = if let Some(account) = &repo_args.account {
+    git_token(&repo_args.provider, account, |https| repo_args.https = https)
     .await
     .with_context(
-      || format!("Failed to get git token in call to db. Stopping run. | {} | {account}", clone_args.provider),
+      || format!("Failed to get git token in call to db. Stopping run. | {} | {account}", repo_args.provider),
     )?
   } else {
     None
@@ -235,7 +235,7 @@ async fn write_dockerfile_contents_git(
   if !root.join(".git").exists() {
     git::init_folder_as_repo(
       &root,
-      &clone_args,
+      &repo_args,
       access_token.as_deref(),
       &mut update.logs,
     )
@@ -249,9 +249,11 @@ async fn write_dockerfile_contents_git(
     }
   }
 
+  // Save this for later -- repo_args moved next.
+  let branch = repo_args.branch.clone();
   // Pull latest changes to repo to ensure linear commit history
   match git::pull_or_clone(
-    clone_args,
+    repo_args,
     &core_config().repo_directory,
     access_token,
   )
@@ -298,7 +300,7 @@ async fn write_dockerfile_contents_git(
     &format!("{}: Commit Dockerfile", args.user.username),
     &root,
     &build_path.join(&dockerfile_path),
-    &build.config.branch,
+    &branch,
   )
   .await;
 

bin/core/src/api/write/stack.rs

@@ -284,6 +284,8 @@ async fn write_stack_file_contents_git(
     }
   }
 
+  // Save this for later -- repo_args moved next.
+  let branch = repo_args.branch.clone();
   // Pull latest changes to repo to ensure linear commit history
   match git::pull_or_clone(
     repo_args,
@@ -334,7 +336,7 @@ async fn write_stack_file_contents_git(
     &format!("{username}: Write Stack File"),
     &root,
     &file_path,
-    &stack.config.branch,
+    &branch,
   )
   .await;
 
@@ -429,7 +431,7 @@ impl Resolve<WriteArgs> for RefreshStackCache {
         let GetComposeContentsOnHostResponse { contents, errors } =
           match periphery_client(&server)?
             .request(GetComposeContentsOnHost {
-              file_paths: stack.file_paths().to_vec(),
+              file_paths: stack.all_file_dependencies(),
               name: stack.name.clone(),
               run_directory: stack.config.run_directory.clone(),
             })
@@ -451,6 +453,10 @@ impl Resolve<WriteArgs> for RefreshStackCache {
         let mut services = Vec::new();
 
         for contents in &contents {
+          // Don't include additional files in service parsing
+          if !stack.is_compose_file(&contents.path) {
+            continue;
+          }
           if let Err(e) = extract_services_into_res(
             &project_name,
             &contents.contents,
@@ -489,6 +495,10 @@ impl Resolve<WriteArgs> for RefreshStackCache {
       let mut services = Vec::new();
 
       for contents in &remote_contents {
+        // Don't include additional files in service parsing
+        if !stack.is_compose_file(&contents.path) {
+          continue;
+        }
         if let Err(e) = extract_services_into_res(
           &project_name,
           &contents.contents,

bin/core/src/api/write/sync.rs

@@ -323,6 +323,8 @@ async fn write_sync_file_contents_git(
     }
   }
 
+  // Save this for later -- repo_args moved next.
+  let branch = repo_args.branch.clone();
   // Pull latest changes to repo to ensure linear commit history
   match git::pull_or_clone(
     repo_args,
@@ -373,7 +375,7 @@ async fn write_sync_file_contents_git(
     &format!("{}: Commit Resource File", args.user.username),
     &root,
     &resource_path.join(&file_path),
-    &sync.config.branch,
+    &branch,
   )
   .await;
 

bin/core/src/api/write/user.rs

@@ -220,6 +220,14 @@ impl Resolve<WriteArgs> for DeleteUser {
       .delete_one(query)
       .await
       .context("Failed to delete user from database")?;
+    // Also remove user id from all user groups
+    if let Err(e) = db
+      .user_groups
+      .update_many(doc! {}, doc! { "$pull": { "users": &user.id } })
+      .await
+    {
+      warn!("Failed to remove deleted user from user groups | {e:?}");
+    };
     Ok(user)
   }
 }

bin/core/src/auth/google/client.rs

@@ -187,8 +187,8 @@ impl GoogleOauthClient {
       Ok(body)
     } else {
       let text = res.text().await.context(format!(
-                "method: POST | status: {status} | failed to get response text"
-            ))?;
+        "method: POST | status: {status} | failed to get response text"
+      ))?;
       Err(anyhow!("method: POST | status: {status} | text: {text}"))
     }
   }
@@ -207,5 +207,6 @@ pub struct GoogleUser {
   #[serde(rename = "sub")]
   pub id: String,
   pub email: String,
+  #[serde(default)]
   pub picture: String,
 }

bin/core/src/config.rs

@@ -250,6 +250,7 @@ pub fn core_config() -> &'static CoreConfig {
           .unwrap_or(config.logging.opentelemetry_service_name),
       },
       pretty_startup_config: env.komodo_pretty_startup_config.unwrap_or(config.pretty_startup_config),
+      unsafe_unsanitized_startup_config: env.komodo_unsafe_unsanitized_startup_config.unwrap_or(config.unsafe_unsanitized_startup_config),
       internet_interface: env.komodo_internet_interface.unwrap_or(config.internet_interface),
       ssl_enabled: env.komodo_ssl_enabled.unwrap_or(config.ssl_enabled),
       ssl_key_file: env.komodo_ssl_key_file.unwrap_or(config.ssl_key_file),

bin/core/src/helpers/procedure.rs

@@ -1101,6 +1101,23 @@ async fn execute_execution(
       )
       .await?
     }
+    Execution::RunStackService(req) => {
+      let req = ExecuteRequest::RunStackService(req);
+      let update = init_execution_update(&req, &user).await?;
+      let ExecuteRequest::RunStackService(req) = req else {
+        unreachable!()
+      };
+      let update_id = update.id.clone();
+      handle_resolve_result(
+        req
+          .resolve(&ExecuteArgs { user, update })
+          .await
+          .map_err(|e| e.error)
+          .context("Failed at RunStackService"),
+        &update_id,
+      )
+      .await?
+    }
     Execution::BatchDestroyStack(_) => {
       // All batch executions must be expanded in `execute_stage`
       return Err(anyhow!(
@@ -1124,6 +1141,23 @@ async fn execute_execution(
       )
       .await?
     }
+    Execution::SendAlert(req) => {
+      let req = ExecuteRequest::SendAlert(req);
+      let update = init_execution_update(&req, &user).await?;
+      let ExecuteRequest::SendAlert(req) = req else {
+        unreachable!()
+      };
+      let update_id = update.id.clone();
+      handle_resolve_result(
+        req
+          .resolve(&ExecuteArgs { user, update })
+          .await
+          .map_err(|e| e.error)
+          .context("Failed at SendAlert"),
+        &update_id,
+      )
+      .await?
+    }
     Execution::ClearRepoCache(req) => {
       let req = ExecuteRequest::ClearRepoCache(req);
       let update = init_execution_update(&req, &user).await?;

bin/core/src/helpers/update.rs

@@ -492,6 +492,13 @@ pub async fn init_execution_update(
       return Ok(Default::default());
     }
 
+    ExecuteRequest::RunStackService(data) => (
+      Operation::RunStackService,
+      ResourceTarget::Stack(
+        resource::get::<Stack>(&data.stack).await?.id,
+      ),
+    ),
+
     // Alerter
     ExecuteRequest::TestAlerter(data) => (
       Operation::TestAlerter,
@@ -499,6 +506,9 @@ pub async fn init_execution_update(
         resource::get::<Alerter>(&data.alerter).await?.id,
       ),
     ),
+    ExecuteRequest::SendAlert(_) => {
+      (Operation::SendAlert, ResourceTarget::system())
+    }
 
     // Maintenance
     ExecuteRequest::ClearRepoCache(_data) => {

bin/core/src/main.rs

@@ -45,10 +45,14 @@ async fn app() -> anyhow::Result<()> {
 
   info!("Komodo Core version: v{}", env!("CARGO_PKG_VERSION"));
 
-  if core_config().pretty_startup_config {
-    info!("{:#?}", config.sanitized());
-  } else {
-    info!("{:?}", config.sanitized());
+  match (
+    config.pretty_startup_config,
+    config.unsafe_unsanitized_startup_config,
+  ) {
+    (true, true) => info!("{:#?}", config),
+    (true, false) => info!("{:#?}", config.sanitized()),
+    (false, true) => info!("{:?}", config),
+    (false, false) => info!("{:?}", config.sanitized()),
   }
 
   // Init jwt client to crash on failure

bin/core/src/monitor/alert/server.rs

@@ -178,6 +178,77 @@ pub async fn alert_servers(
         ),
     }
 
+    // ===================
+    // SERVER VERSION MISMATCH
+    // ===================
+    let core_version = env!("CARGO_PKG_VERSION");
+    let has_version_mismatch = server_status.state == ServerState::Ok
+      && !server_status.version.is_empty()
+      && server_status.version != "Unknown"
+      && server_status.version != core_version;
+
+    let version_alert = server_alerts.as_ref().and_then(|alerts| {
+      alerts.get(&AlertDataVariant::ServerVersionMismatch)
+    });
+
+    match (has_version_mismatch, version_alert) {
+      (true, None) => {
+        // Only open version mismatch alert if not in maintenance and buffer is ready
+        if !in_maintenance
+          && buffer.ready_to_open(
+            server_status.id.clone(),
+            AlertDataVariant::ServerVersionMismatch,
+          )
+        {
+          let alert = Alert {
+            id: Default::default(),
+            ts,
+            resolved: false,
+            resolved_ts: None,
+            level: SeverityLevel::Warning,
+            target: ResourceTarget::Server(server_status.id.clone()),
+            data: AlertData::ServerVersionMismatch {
+              id: server_status.id.clone(),
+              name: server.name.clone(),
+              region: optional_string(&server.config.region),
+              server_version: server_status.version.clone(),
+              core_version: core_version.to_string(),
+            },
+          };
+          // Use send_unreachable_alerts as a proxy for general server alerts
+          alerts_to_open
+            .push((alert, server.config.send_version_mismatch_alerts))
+        }
+      }
+      (true, Some(alert)) => {
+        // Update existing alert with current version info
+        let mut alert = alert.clone();
+        alert.data = AlertData::ServerVersionMismatch {
+          id: server_status.id.clone(),
+          name: server.name.clone(),
+          region: optional_string(&server.config.region),
+          server_version: server_status.version.clone(),
+          core_version: core_version.to_string(),
+        };
+        // Don't send notification for updates
+        alerts_to_update.push((alert, false));
+      }
+      (false, Some(alert)) => {
+        // Version is now correct, close the alert
+        alert_ids_to_close.push((
+          alert.clone(),
+          server.config.send_version_mismatch_alerts,
+        ));
+      }
+      (false, None) => {
+        // Reset buffer state when no mismatch and no alert
+        buffer.reset(
+          server_status.id.clone(),
+          AlertDataVariant::ServerVersionMismatch,
+        )
+      }
+    }
+
     let Some(health) = &server_status.health else {
       continue;
     };

bin/core/src/monitor/record.rs

@@ -21,6 +21,7 @@ pub async fn record_server_stats(ts: i64) {
         ts,
         sid: status.id.clone(),
         cpu_perc: stats.cpu_perc,
+        load_average: stats.load_average.clone(),
         mem_total_gb: stats.mem_total_gb,
         mem_used_gb: stats.mem_used_gb,
         disk_total_gb,

bin/core/src/monitor/resources.rs

@@ -81,7 +81,7 @@ pub async fn update_deployment_cache(
         // If image already has tag, leave it,
         // otherwise default the tag to latest
         if image.contains(':') {
-          image
+          image.to_string()
         } else {
           format!("{image}:latest")
         }
@@ -92,6 +92,9 @@ pub async fn update_deployment_cache(
       ..
     }) = &container
     {
+      // Docker will automatically strip `docker.io` from incoming image names re #468.
+      // Need to strip it in order to match by image name and find available updates.
+      let image = image.strip_prefix("docker.io/").unwrap_or(&image);
       images
         .iter()
         .find(|i| i.name == image)
@@ -250,20 +253,21 @@ pub async fn update_stack_cache(
           }
         }.is_match(&container.name)
       }).cloned();
-      // If image already has tag, leave it,
-      // otherwise default the tag to latest
-      let image = image.clone();
       let image = if image.contains(':') {
-        image
+        image.to_string()
       } else {
-        image + ":latest"
+        format!("{image}:latest")
       };
       let update_available = if let Some(ContainerListItem { image_id: Some(curr_image_id), .. }) = &container {
+        // Docker will automatically strip `docker.io` from incoming image names re #468.
+        // Need to strip it in order to match by image tag and find available update.
+        let image =
+          image.strip_prefix("docker.io/").unwrap_or(&image);
         images
-        .iter()
-        .find(|i| i.name == image)
-        .map(|i| &i.id != curr_image_id)
-        .unwrap_or_default()
+          .iter()
+          .find(|i| i.name == image)
+          .map(|i| &i.id != curr_image_id)
+          .unwrap_or_default()
       } else {
         false
       };

bin/core/src/network.rs

@@ -36,10 +36,10 @@ fn is_container_environment() -> bool {
   }
 
   // Check cgroup for container runtime indicators
-  if let Ok(content) = std::fs::read_to_string(CGROUP_FILE) {
-    if content.contains("docker") || content.contains("containerd") {
-      return true;
-    }
+  if let Ok(content) = std::fs::read_to_string(CGROUP_FILE)
+    && (content.contains("docker") || content.contains("containerd"))
+  {
+    return true;
   }
 
   false
@@ -142,7 +142,7 @@ async fn find_gateway(
   }
 
   let ip_cidr = ip_cidr.ok_or_else(|| anyhow!(
-        "Could not find IP address for interface '{}'. Ensure interface has a valid IPv4 address", 
+        "Could not find IP address for interface '{}'. Ensure interface has a valid IPv4 address",
         interface_name
     ))?;
 
@@ -167,14 +167,13 @@ async fn find_gateway(
       if line.contains("via") {
         let parts: Vec<&str> = line.split_whitespace().collect();
         if let Some(via_idx) = parts.iter().position(|&x| x == "via")
+          && let Some(&gateway) = parts.get(via_idx + 1)
         {
-          if let Some(&gateway) = parts.get(via_idx + 1) {
-            trace!(
-              "Found gateway {} for {} from routing table",
-              gateway, interface_name
-            );
-            return Ok(gateway.to_string());
-          }
+          trace!(
+            "Found gateway {} for {} from routing table",
+            gateway, interface_name
+          );
+          return Ok(gateway.to_string());
         }
       }
     }
@@ -206,14 +205,14 @@ async fn find_gateway(
           .output()
           .await;
 
-        if let Ok(output) = route_test {
-          if output.status.success() {
-            trace!(
-              "Gateway {} is reachable via {}",
-              gateway, interface_name
-            );
-            return Ok(gateway.to_string());
-          }
+        if let Ok(output) = route_test
+          && output.status.success()
+        {
+          trace!(
+            "Gateway {} is reachable via {}",
+            gateway, interface_name
+          );
+          return Ok(gateway.to_string());
         }
 
         // Fallback: assume .1 is gateway (Docker standard)
@@ -266,10 +265,10 @@ async fn set_default_gateway(
     .output()
     .await;
 
-  if let Ok(output) = remove_default {
-    if output.status.success() {
-      trace!("Removed existing default routes");
-    }
+  if let Ok(output) = remove_default
+    && output.status.success()
+  {
+    trace!("Removed existing default routes");
   }
 
   // Add new default route

bin/core/src/resource/build.rs

@@ -116,9 +116,11 @@ impl super::KomodoResource for Build {
         git_provider,
         repo,
         branch,
-        image_registry_domain: optional_string(
-          build.config.image_registry.domain,
-        ),
+        image_registry_domain: build
+          .config
+          .image_registry
+          .first()
+          .and_then(|r| optional_string(&r.domain)),
         built_hash: build.info.built_hash,
         latest_hash: build.info.latest_hash,
         state,

bin/core/src/resource/mod.rs

@@ -229,6 +229,12 @@ pub trait KomodoResource {
 pub async fn get<T: KomodoResource>(
   id_or_name: &str,
 ) -> anyhow::Result<Resource<T::Config, T::Info>> {
+  if id_or_name.is_empty() {
+    return Err(anyhow!(
+      "Cannot find {} with empty name / id",
+      T::resource_type()
+    ));
+  }
   T::coll()
     .find_one(id_or_name_filter(id_or_name))
     .await

bin/core/src/resource/procedure.rs

@@ -5,6 +5,7 @@ use database::mungos::{
   find::find_collect,
   mongodb::{Collection, bson::doc, options::FindOneOptions},
 };
+use futures::{TryStreamExt, stream::FuturesUnordered};
 use komodo_client::{
   api::execute::Execution,
   entities::{
@@ -709,6 +710,15 @@ async fn validate_config(
           .await?;
           params.stack = stack.id;
         }
+        Execution::RunStackService(params) => {
+          let stack = super::get_check_permissions::<Stack>(
+            &params.stack,
+            user,
+            PermissionLevel::Execute.into(),
+          )
+          .await?;
+          params.stack = stack.id;
+        }
         Execution::BatchDestroyStack(_params) => {
           if !user.admin {
             return Err(anyhow!(
@@ -725,6 +735,24 @@ async fn validate_config(
           .await?;
           params.alerter = alerter.id;
         }
+        Execution::SendAlert(params) => {
+          params.alerters = params
+            .alerters
+            .iter()
+            .map(async |alerter| {
+              let id = super::get_check_permissions::<Alerter>(
+                alerter,
+                user,
+                PermissionLevel::Execute.into(),
+              )
+              .await?
+              .id;
+              anyhow::Ok(id)
+            })
+            .collect::<FuturesUnordered<_>>()
+            .try_collect::<Vec<_>>()
+            .await?;
+        }
         Execution::ClearRepoCache(_params) => {
           if !user.admin {
             return Err(anyhow!(

bin/core/src/resource/server.rs

@@ -82,6 +82,9 @@ impl super::KomodoResource for Server {
         send_cpu_alerts: server.config.send_cpu_alerts,
         send_mem_alerts: server.config.send_mem_alerts,
         send_disk_alerts: server.config.send_disk_alerts,
+        send_version_mismatch_alerts: server
+          .config
+          .send_version_mismatch_alerts,
         terminals_disabled,
         container_exec_disabled,
       },

bin/core/src/stack/remote.rs

@@ -3,14 +3,16 @@ use std::{fs, path::PathBuf};
 use anyhow::Context;
 use formatting::format_serror;
 use komodo_client::entities::{
-  FileContents, RepoExecutionArgs, repo::Repo, stack::Stack,
+  FileContents, RepoExecutionArgs,
+  repo::Repo,
+  stack::{Stack, StackRemoteFileContents},
   update::Log,
 };
 
 use crate::{config::core_config, helpers::git_token};
 
 pub struct RemoteComposeContents {
-  pub successful: Vec<FileContents>,
+  pub successful: Vec<StackRemoteFileContents>,
   pub errored: Vec<FileContents>,
   pub hash: Option<String>,
   pub message: Option<String>,
@@ -38,23 +40,25 @@ pub async fn get_repo_compose_contents(
   let mut successful = Vec::new();
   let mut errored = Vec::new();
 
-  for path in stack.file_paths() {
-    let file_path = run_directory.join(path);
+  for file in stack.all_file_dependencies() {
+    let file_path = run_directory.join(&file.path);
     if !file_path.exists()
       && let Some(missing_files) = &mut missing_files
     {
-      missing_files.push(path.to_string());
+      missing_files.push(file.path.clone());
     }
     // If file does not exist, will show up in err case so the log is handled
     match fs::read_to_string(&file_path).with_context(|| {
       format!("Failed to read file contents from {file_path:?}")
     }) {
-      Ok(contents) => successful.push(FileContents {
-        path: path.to_string(),
+      Ok(contents) => successful.push(StackRemoteFileContents {
+        path: file.path,
         contents,
+        services: file.services,
+        requires: file.requires,
       }),
       Err(e) => errored.push(FileContents {
-        path: path.to_string(),
+        path: file.path,
         contents: format_serror(&e.into()),
       }),
     }

bin/core/src/startup.rs

@@ -242,7 +242,7 @@ async fn ensure_first_server_and_builder() {
       return;
     };
   if let Err(e) = (CreateBuilder {
-    name: String::from("Local"),
+    name: config.first_server_name.clone(),
     config: PartialBuilderConfig::Server(
       PartialServerBuilderConfig {
         server_id: Some(server.id),
@@ -285,8 +285,7 @@ async fn ensure_init_user_and_resources() {
     }
     .resolve(&AuthArgs::default())
     .await
-    .expect("Failed to initialize default admin user.")
-    .jwt;
+    .expect("Failed to initialize default admin user.");
     db.users
       .find_one(doc! { "username": username })
       .await

bin/core/src/sync/deploy.rs

@@ -9,12 +9,15 @@ use komodo_client::{
     read::ListBuildVersions,
   },
   entities::{
-    FileContents, ResourceTarget,
+    ResourceTarget,
     deployment::{
       Deployment, DeploymentConfig, DeploymentImage, DeploymentState,
       PartialDeploymentConfig,
     },
-    stack::{PartialStackConfig, Stack, StackConfig, StackState},
+    stack::{
+      PartialStackConfig, Stack, StackConfig,
+      StackRemoteFileContents, StackState,
+    },
     sync::SyncDeployUpdate,
     toml::ResourceToml,
     update::Log,
@@ -554,7 +557,13 @@ fn build_cache_for_stack<'a>(
           &original.info.remote_contents,
         ) {
           (Some(deployed_contents), Some(remote_contents)) => {
-            for FileContents { path, contents } in remote_contents {
+            for StackRemoteFileContents {
+              path,
+              contents,
+              services: _services,
+              requires: _requires,
+            } in remote_contents
+            {
               if let Some(deployed) =
                 deployed_contents.iter().find(|c| &c.path == path)
               {

bin/core/src/sync/resources.rs

@@ -661,6 +661,13 @@ impl ResourceSyncTrait for Procedure {
               .map(|s| s.name.clone())
               .unwrap_or_default();
           }
+          Execution::RunStackService(config) => {
+            config.stack = resources
+              .stacks
+              .get(&config.stack)
+              .map(|s| s.name.clone())
+              .unwrap_or_default();
+          }
           Execution::BatchDestroyStack(_config) => {}
           Execution::TestAlerter(config) => {
             config.alerter = resources
@@ -669,6 +676,19 @@ impl ResourceSyncTrait for Procedure {
               .map(|a| a.name.clone())
               .unwrap_or_default();
           }
+          Execution::SendAlert(config) => {
+            config.alerters = config
+              .alerters
+              .iter()
+              .map(|alerter| {
+                resources
+                  .alerters
+                  .get(alerter)
+                  .map(|a| a.name.clone())
+                  .unwrap_or_default()
+              })
+              .collect();
+          }
           Execution::ClearRepoCache(_) => {}
           Execution::BackupCoreDatabase(_) => {}
           Execution::GlobalAutoUpdate(_) => {}

bin/core/src/sync/toml.rs

@@ -752,6 +752,13 @@ impl ToToml for Procedure {
               .map(|r| &r.name)
               .unwrap_or(&String::new()),
           ),
+          Execution::RunStackService(exec) => exec.stack.clone_from(
+            all
+              .stacks
+              .get(&exec.stack)
+              .map(|r| &r.name)
+              .unwrap_or(&String::new()),
+          ),
           Execution::PauseStack(exec) => exec.stack.clone_from(
             all
               .stacks
@@ -788,6 +795,17 @@ impl ToToml for Procedure {
               .map(|a| &a.name)
               .unwrap_or(&String::new()),
           ),
+          Execution::SendAlert(exec) => {
+            exec.alerters.iter_mut().for_each(|a| {
+              a.clone_from(
+                all
+                  .alerters
+                  .get(a)
+                  .map(|a| &a.name)
+                  .unwrap_or(&String::new()),
+              )
+            })
+          }
           Execution::None(_)
           | Execution::Sleep(_)
           | Execution::ClearRepoCache(_)

bin/periphery/Cargo.toml

@@ -57,4 +57,5 @@ axum.workspace = true
 clap.workspace = true
 envy.workspace = true
 uuid.workspace = true
-rand.workspace = true
+rand.workspace = true
+shell-escape.workspace = true

bin/periphery/src/api/build.rs

@@ -1,4 +1,7 @@
-use std::path::PathBuf;
+use std::{
+  collections::{HashMap, HashSet},
+  path::PathBuf,
+};
 
 use anyhow::{Context, anyhow};
 use command::{
@@ -9,7 +12,7 @@ use interpolate::Interpolator;
 use komodo_client::entities::{
   EnvironmentVar, all_logs_success,
   build::{Build, BuildConfig},
-  environment_vars_from_str, get_image_name, optional_string,
+  environment_vars_from_str, get_image_names, optional_string,
   to_path_compatible_name,
   update::Log,
 };
@@ -122,7 +125,7 @@ impl Resolve<super::Args> for build::Build {
     let build::Build {
       mut build,
       repo: linked_repo,
-      registry_token,
+      registry_tokens,
       additional_tags,
       mut replacers,
     } = self;
@@ -169,25 +172,41 @@ impl Resolve<super::Args> for build::Build {
       return Err(anyhow!("Build must be files on host mode, have a repo attached, or have dockerfile contents set to build").into());
     }
 
+    let registry_tokens = registry_tokens
+      .iter()
+      .map(|(domain, account, token)| {
+        ((domain.as_str(), account.as_str()), token.as_str())
+      })
+      .collect::<HashMap<_, _>>();
+
     // Maybe docker login
-    let should_push = match docker_login(
-      &image_registry.domain,
-      &image_registry.account,
-      registry_token.as_deref(),
-    )
-    .await
+    let mut should_push = false;
+    for (domain, account) in image_registry
+      .iter()
+      .map(|r| (r.domain.as_str(), r.account.as_str()))
+      // This ensures uniqueness / prevents redundant logins
+      .collect::<HashSet<_>>()
     {
-      Ok(should_push) => should_push,
-      Err(e) => {
-        logs.push(Log::error(
-          "Docker Login",
-          format_serror(
-            &e.context("failed to login to docker registry").into(),
-          ),
-        ));
-        return Ok(logs);
-      }
-    };
+      match docker_login(
+        domain,
+        account,
+        registry_tokens.get(&(domain, account)).copied(),
+      )
+      .await
+      {
+        Ok(logged_in) if logged_in => should_push = true,
+        Ok(_) => {}
+        Err(e) => {
+          logs.push(Log::error(
+            "Docker Login",
+            format_serror(
+              &e.context("failed to login to docker registry").into(),
+            ),
+          ));
+          return Ok(logs);
+        }
+      };
+    }
 
     let build_path = if let Some(repo) = &linked_repo {
       periphery_config()
@@ -245,8 +264,8 @@ impl Resolve<super::Args> for build::Build {
     }
 
     // Get command parts
-    let image_name =
-      get_image_name(&build).context("failed to make image name")?;
+
+    let image_names = get_image_names(&build);
 
     // Add VERSION to build args (if not already there)
     let mut build_args = environment_vars_from_str(build_args)
@@ -267,10 +286,15 @@ impl Resolve<super::Args> for build::Build {
     let labels = parse_labels(
       &environment_vars_from_str(labels).context("Invalid labels")?,
     );
+
     let extra_args = parse_extra_args(extra_args);
+
     let buildx = if *use_buildx { " buildx" } else { "" };
+
     let image_tags =
-      image_tags(&image_name, image_tag, version, &additional_tags);
+      image_tags(&image_names, image_tag, version, &additional_tags)
+        .context("Failed to parse image tags into command")?;
+
     let maybe_push = if should_push { " --push" } else { "" };
 
     // Construct command

bin/periphery/src/api/compose.rs

@@ -1,5 +1,3 @@
-use std::{fmt::Write, path::PathBuf};
-
 use anyhow::{Context, anyhow};
 use command::{
   run_komodo_command, run_komodo_command_with_sanitization,
@@ -11,7 +9,7 @@ use komodo_client::entities::{
   FileContents, RepoExecutionResponse, all_logs_success,
   stack::{
     ComposeFile, ComposeProject, ComposeService,
-    ComposeServiceDeploy, StackServiceNames,
+    ComposeServiceDeploy, StackRemoteFileContents, StackServiceNames,
   },
   to_path_compatible_name,
   update::Log,
@@ -19,11 +17,13 @@ use komodo_client::entities::{
 use periphery_client::api::compose::*;
 use resolver_api::Resolve;
 use serde::{Deserialize, Serialize};
+use shell_escape::unix::escape;
+use std::{borrow::Cow, path::PathBuf};
 use tokio::fs;
 
 use crate::{
   compose::{
-    docker_compose, pull_or_clone_stack,
+    docker_compose, env_file_args, pull_or_clone_stack,
     up::{maybe_login_registry, validate_files},
     write::write_stack,
   },
@@ -169,9 +169,11 @@ impl Resolve<super::Args> for GetComposeContentsOnHost {
 
     let mut res = GetComposeContentsOnHostResponse::default();
 
-    for path in file_paths {
-      let full_path =
-        run_directory.join(&path).components().collect::<PathBuf>();
+    for file in file_paths {
+      let full_path = run_directory
+        .join(&file.path)
+        .components()
+        .collect::<PathBuf>();
       match fs::read_to_string(&full_path).await.with_context(|| {
         format!(
           "Failed to read compose file contents at {full_path:?}"
@@ -180,11 +182,16 @@ impl Resolve<super::Args> for GetComposeContentsOnHost {
         Ok(contents) => {
           // The path we store here has to be the same as incoming file path in the array,
           // in order for WriteComposeContentsToHost to write to the correct path.
-          res.contents.push(FileContents { path, contents });
+          res.contents.push(StackRemoteFileContents {
+            path: file.path,
+            contents,
+            services: file.services,
+            requires: file.requires,
+          });
         }
         Err(e) => {
           res.errors.push(FileContents {
-            path,
+            path: file.path,
             contents: format_serror(&e.into()),
           });
         }
@@ -351,19 +358,19 @@ impl Resolve<super::Args> for ComposePull {
     )?;
 
     let file_paths = stack
-      .file_paths()
-      .iter()
+      .all_file_paths()
+      .into_iter()
       .map(|path| {
         (
-          path,
           // This will remove any intermediate uneeded '/./' in the path
-          run_directory.join(path).components().collect::<PathBuf>(),
+          run_directory.join(&path).components().collect::<PathBuf>(),
+          path,
         )
       })
       .collect::<Vec<_>>();
 
     // Validate files
-    for (path, full_path) in &file_paths {
+    for (full_path, path) in &file_paths {
       if !full_path.exists() {
         return Err(anyhow!("Missing compose file at {path}").into());
       }
@@ -382,24 +389,12 @@ impl Resolve<super::Args> for ComposePull {
       format!(" {}", services.join(" "))
     };
 
-    let file_args = if stack.config.file_paths.is_empty() {
-      String::from("compose.yaml")
-    } else {
-      stack.config.file_paths.join(" -f ")
-    };
+    let file_args = stack.compose_file_paths().join(" -f ");
 
-    let env_file = env_file_path
-      .map(|path| format!(" --env-file {path}"))
-      .unwrap_or_default();
-
-    let additional_env_files = stack
-      .config
-      .additional_env_files
-      .iter()
-      .fold(String::new(), |mut output, file| {
-        let _ = write!(output, " --env-file {file}");
-        output
-      });
+    let env_file_args = env_file_args(
+      env_file_path,
+      &stack.config.additional_env_files,
+    )?;
 
     let project_name = stack.project_name(false);
 
@@ -407,7 +402,7 @@ impl Resolve<super::Args> for ComposePull {
       "Compose Pull",
       run_directory.as_ref(),
       format!(
-        "{docker_compose} -p {project_name} -f {file_args}{additional_env_files}{env_file} pull{service_args}",
+        "{docker_compose} -p {project_name} -f {file_args}{env_file_args} pull{service_args}",
       ),
     )
     .await;
@@ -514,35 +509,23 @@ impl Resolve<super::Args> for ComposeUp {
       format!(" {}", services.join(" "))
     };
 
-    let file_args = if stack.config.file_paths.is_empty() {
-      String::from("compose.yaml")
-    } else {
-      stack.config.file_paths.join(" -f ")
-    };
+    let file_args = stack.compose_file_paths().join(" -f ");
 
     // This will be the last project name, which is the one that needs to be destroyed.
     // Might be different from the current project name, if user renames stack / changes to custom project name.
     let last_project_name = stack.project_name(false);
     let project_name = stack.project_name(true);
 
-    let env_file = env_file_path
-      .map(|path| format!(" --env-file {path}"))
-      .unwrap_or_default();
-
-    let additional_env_files = stack
-      .config
-      .additional_env_files
-      .iter()
-      .fold(String::new(), |mut output, file| {
-        let _ = write!(output, " --env-file {file}");
-        output
-      });
+    let env_file_args = env_file_args(
+      env_file_path,
+      &stack.config.additional_env_files,
+    )?;
 
     // Uses 'docker compose config' command to extract services (including image)
     // after performing interpolation
     {
       let command = format!(
-        "{docker_compose} -p {project_name} -f {file_args}{additional_env_files}{env_file} config",
+        "{docker_compose} -p {project_name} -f {file_args}{env_file_args} config",
       );
       let Some(config_log) = run_komodo_command_with_sanitization(
         "Compose Config",
@@ -607,7 +590,7 @@ impl Resolve<super::Args> for ComposeUp {
       let build_extra_args =
         parse_extra_args(&stack.config.build_extra_args);
       let command = format!(
-        "{docker_compose} -p {project_name} -f {file_args}{additional_env_files}{env_file} build{build_extra_args}{service_args}",
+        "{docker_compose} -p {project_name} -f {file_args}{env_file_args} build{build_extra_args}{service_args}",
       );
       let Some(log) = run_komodo_command_with_sanitization(
         "Compose Build",
@@ -631,7 +614,7 @@ impl Resolve<super::Args> for ComposeUp {
       // Pull images before destroying to minimize downtime.
       // If this fails, do not continue.
       let command = format!(
-        "{docker_compose} -p {project_name} -f {file_args}{additional_env_files}{env_file} pull{service_args}",
+        "{docker_compose} -p {project_name} -f {file_args}{env_file_args} pull{service_args}",
       );
       let log = run_komodo_command(
         "Compose Pull",
@@ -659,7 +642,7 @@ impl Resolve<super::Args> for ComposeUp {
     // Run compose up
     let extra_args = parse_extra_args(&stack.config.extra_args);
     let command = format!(
-      "{docker_compose} -p {project_name} -f {file_args}{additional_env_files}{env_file} up -d{extra_args}{service_args}",
+      "{docker_compose} -p {project_name} -f {file_args}{env_file_args} up -d{extra_args}{service_args}",
     );
 
     let Some(log) = run_komodo_command_with_sanitization(
@@ -713,3 +696,150 @@ impl Resolve<super::Args> for ComposeExecution {
     Ok(log)
   }
 }
+
+//
+
+impl Resolve<super::Args> for ComposeRun {
+  #[instrument(name = "ComposeRun", level = "debug", skip_all, fields(stack = &self.stack.name, service = &self.service))]
+  async fn resolve(self, _: &super::Args) -> serror::Result<Log> {
+    let ComposeRun {
+      mut stack,
+      repo,
+      git_token,
+      registry_token,
+      mut replacers,
+      service,
+      command,
+      no_tty,
+      no_deps,
+      detach,
+      service_ports,
+      env,
+      workdir,
+      user,
+      entrypoint,
+      pull,
+    } = self;
+
+    let mut interpolator =
+      Interpolator::new(None, &periphery_config().secrets);
+    interpolator
+      .interpolate_stack(&mut stack)?
+      .push_logs(&mut Vec::new());
+    replacers.extend(interpolator.secret_replacers);
+
+    let mut res = ComposeRunResponse::default();
+    let (run_directory, env_file_path) = match write_stack(
+      &stack,
+      repo.as_ref(),
+      git_token,
+      replacers.clone(),
+      &mut res,
+    )
+    .await
+    {
+      Ok(res) => res,
+      Err(e) => {
+        return Ok(Log::error(
+          "Write Stack",
+          format_serror(&e.into()),
+        ));
+      }
+    };
+
+    let run_directory = run_directory.canonicalize().context(
+      "Failed to validate run directory on host after stack write (canonicalize error)",
+    )?;
+
+    maybe_login_registry(&stack, registry_token, &mut Vec::new())
+      .await;
+
+    let docker_compose = docker_compose();
+
+    let file_args = if stack.config.file_paths.is_empty() {
+      String::from("compose.yaml")
+    } else {
+      stack.config.file_paths.join(" -f ")
+    };
+
+    let env_file_args = env_file_args(
+      env_file_path,
+      &stack.config.additional_env_files,
+    )?;
+
+    let project_name = stack.project_name(true);
+
+    if pull.unwrap_or_default() {
+      let pull_log = run_komodo_command(
+        "Compose Pull",
+        run_directory.as_ref(),
+        format!(
+          "{docker_compose} -p {project_name} -f {file_args}{env_file_args} pull {service}",
+        ),
+      )
+      .await;
+      if !pull_log.success {
+        return Ok(pull_log);
+      }
+    }
+
+    let mut run_flags = String::from(" --rm");
+    if detach.unwrap_or_default() {
+      run_flags.push_str(" -d");
+    }
+    if no_tty.unwrap_or_default() {
+      run_flags.push_str(" --no-tty");
+    }
+    if no_deps.unwrap_or_default() {
+      run_flags.push_str(" --no-deps");
+    }
+    if service_ports.unwrap_or_default() {
+      run_flags.push_str(" --service-ports");
+    }
+    if let Some(dir) = workdir.as_ref() {
+      run_flags.push_str(&format!(" --workdir {dir}"));
+    }
+    if let Some(user) = user.as_ref() {
+      run_flags.push_str(&format!(" --user {user}"));
+    }
+    if let Some(entrypoint) = entrypoint.as_ref() {
+      run_flags.push_str(&format!(" --entrypoint {entrypoint}"));
+    }
+    if let Some(env) = env {
+      for (k, v) in env {
+        run_flags.push_str(&format!(" -e {}={} ", k, v));
+      }
+    }
+
+    let command_args = command
+      .as_ref()
+      .filter(|v| !v.is_empty())
+      .map(|argv| {
+        let joined = argv
+          .iter()
+          .map(|s| escape(Cow::Borrowed(s)).into_owned())
+          .collect::<Vec<_>>()
+          .join(" ");
+        format!(" {joined}")
+      })
+      .unwrap_or_default();
+
+    let command = format!(
+      "{docker_compose} -p {project_name} -f {file_args}{env_file_args} run{run_flags} {service}{command_args}",
+    );
+
+    let Some(log) = run_komodo_command_with_sanitization(
+      "Compose Run",
+      run_directory.as_path(),
+      command,
+      false,
+      &replacers,
+    )
+    .await
+    else {
+      unreachable!()
+    };
+
+    Ok(log)
+  }
+}

bin/periphery/src/api/mod.rs

@@ -85,6 +85,7 @@ pub enum PeripheryRequest {
   ComposePull(ComposePull),
   ComposeUp(ComposeUp),
   ComposeExecution(ComposeExecution),
+  ComposeRun(ComposeRun),
 
   // Container (Read)
   InspectContainer(InspectContainer),

bin/periphery/src/api/terminal.rs

@@ -347,7 +347,7 @@ async fn execute_command_on_terminal(
   );
 
   let full_command = format!(
-    "printf '\n{START_OF_OUTPUT}\n\n'; {command}; rc=$? printf '\n{KOMODO_EXIT_CODE}%d\n{END_OF_OUTPUT}\n' \"$rc\"\n"
+    "printf '\n{START_OF_OUTPUT}\n\n'; {command}; rc=$?; printf '\n{KOMODO_EXIT_CODE}%d\n{END_OF_OUTPUT}\n' \"$rc\"\n"
   );
 
   terminal

bin/periphery/src/build.rs

@@ -53,25 +53,31 @@ pub async fn write_dockerfile(
 }
 
 pub fn image_tags(
-  image_name: &str,
+  image_names: &[String],
   custom_tag: &str,
   version: &Version,
   additional: &[String],
-) -> String {
+) -> anyhow::Result<String> {
   let Version { major, minor, .. } = version;
   let custom_tag = if custom_tag.is_empty() {
     String::new()
   } else {
     format!("-{custom_tag}")
   };
-  let additional = additional
-    .iter()
-    .map(|tag| format!(" -t {image_name}:{tag}{custom_tag}"))
-    .collect::<Vec<_>>()
-    .join("");
-  format!(
-    " -t {image_name}:latest{custom_tag} -t {image_name}:{version}{custom_tag} -t {image_name}:{major}.{minor}{custom_tag} -t {image_name}:{major}{custom_tag}{additional}",
-  )
+
+  let mut res = String::new();
+
+  for image_name in image_names {
+    write!(
+      &mut res,
+      " -t {image_name}:latest{custom_tag} -t {image_name}:{version}{custom_tag} -t {image_name}:{major}.{minor}{custom_tag} -t {image_name}:{major}{custom_tag}"
+    )?;
+    for tag in additional {
+      write!(&mut res, " -t {image_name}:{tag}{custom_tag}")?;
+    }
+  }
+
+  Ok(res)
 }
 
 pub fn parse_build_args(build_args: &[EnvironmentVar]) -> String {

bin/periphery/src/compose/mod.rs

@@ -1,6 +1,6 @@
-use std::path::PathBuf;
+use std::{fmt::Write, path::PathBuf};
 
-use anyhow::anyhow;
+use anyhow::{Context, anyhow};
 use command::run_komodo_command;
 use komodo_client::entities::{
   RepoExecutionArgs, repo::Repo, stack::Stack,
@@ -24,6 +24,35 @@ pub fn docker_compose() -> &'static str {
   }
 }
 
+pub fn env_file_args(
+  env_file_path: Option<&str>,
+  additional_env_files: &[String],
+) -> anyhow::Result<String> {
+  let mut res = String::new();
+
+  for file in additional_env_files.iter().filter(|&path| {
+    let Some(komodo_path) = env_file_path else {
+      return true;
+    };
+    // Filter komodo env out of additional env file if its also in there.
+    // It will be always be added last / have highest priority.
+    path != komodo_path
+  }) {
+    write!(res, " --env-file {file}").with_context(|| {
+      format!("Failed to write --env-file arg for {file}")
+    })?;
+  }
+
+  // Add this last, so it is applied on top
+  if let Some(file) = env_file_path {
+    write!(res, " --env-file {file}").with_context(|| {
+      format!("Failed to write --env-file arg for {file}")
+    })?;
+  }
+
+  Ok(res)
+}
+
 pub async fn down(
   project: &str,
   services: &[String],

bin/periphery/src/compose/up.rs

@@ -3,7 +3,9 @@ use std::path::{Path, PathBuf};
 use anyhow::{Context, anyhow};
 use formatting::format_serror;
 use komodo_client::entities::{
-  FileContents, stack::Stack, update::Log,
+  FileContents,
+  stack::{Stack, StackRemoteFileContents},
+  update::Log,
 };
 use periphery_client::api::compose::ComposeUpResponse;
 use tokio::fs;
@@ -16,20 +18,24 @@ pub async fn validate_files(
   res: &mut ComposeUpResponse,
 ) {
   let file_paths = stack
-    .file_paths()
-    .iter()
-    .map(|path| {
+    .all_file_dependencies()
+    .into_iter()
+    .map(|file| {
       (
-        path,
         // This will remove any intermediate uneeded '/./' in the path
-        run_directory.join(path).components().collect::<PathBuf>(),
+        run_directory
+          .join(&file.path)
+          .components()
+          .collect::<PathBuf>(),
+        file,
       )
     })
     .collect::<Vec<_>>();
 
-  for (path, full_path) in &file_paths {
+  // First validate no missing files
+  for (full_path, file) in &file_paths {
     if !full_path.exists() {
-      res.missing_files.push(path.to_string());
+      res.missing_files.push(file.path.clone());
     }
   }
   if !res.missing_files.is_empty() {
@@ -37,21 +43,20 @@ pub async fn validate_files(
       "Validate Files",
       format_serror(
         &anyhow!(
-          "Ensure the run_directory and file_paths are correct."
+          "Missing files: {}", res.missing_files.join(", ")
         )
-        .context("A compose file doesn't exist after writing stack.")
+        .context("Ensure the run_directory and all file paths are correct.")
+        .context("A file doesn't exist after writing stack.")
         .into(),
       ),
     ));
     return;
   }
 
-  for (path, full_path) in &file_paths {
+  for (full_path, file) in file_paths {
     let file_contents =
       match fs::read_to_string(&full_path).await.with_context(|| {
-        format!(
-          "Failed to read compose file contents at {full_path:?}"
-        )
+        format!("Failed to read file contents at {full_path:?}")
       }) {
         Ok(res) => res,
         Err(e) => {
@@ -61,15 +66,17 @@ pub async fn validate_files(
             .push(Log::error("Read Compose File", error.clone()));
           // This should only happen for repo stacks, ie remote error
           res.remote_errors.push(FileContents {
-            path: path.to_string(),
+            path: file.path,
             contents: error,
           });
           return;
         }
       };
-    res.file_contents.push(FileContents {
-      path: path.to_string(),
+    res.file_contents.push(StackRemoteFileContents {
+      path: file.path,
       contents: file_contents,
+      services: file.services,
+      requires: file.requires,
     });
   }
 }

bin/periphery/src/compose/write.rs

@@ -7,7 +7,9 @@ use komodo_client::entities::{
   stack::Stack, to_path_compatible_name, update::Log,
 };
 use periphery_client::api::{
-  compose::{ComposePullResponse, ComposeUpResponse},
+  compose::{
+    ComposePullResponse, ComposeRunResponse, ComposeUpResponse,
+  },
   git::{CloneRepo, PullOrCloneRepo},
 };
 use resolver_api::Resolve;
@@ -43,6 +45,12 @@ impl WriteStackRes for &mut ComposePullResponse {
   }
 }
 
+impl WriteStackRes for &mut ComposeRunResponse {
+  fn logs(&mut self) -> &mut Vec<Log> {
+    &mut self.logs
+  }
+}
+
 /// Either writes the stack file_contents to a file, or clones the repo.
 /// Asssumes all interpolation is already complete.
 /// Returns (run_directory, env_file_path, periphery_replacers)

bin/periphery/src/stats.rs

@@ -2,7 +2,8 @@ use std::{cmp::Ordering, sync::OnceLock};
 
 use async_timing_util::wait_until_timelength;
 use komodo_client::entities::stats::{
-  SingleDiskUsage, SystemInformation, SystemProcess, SystemStats,
+  SingleDiskUsage, SystemInformation, SystemLoadAverage,
+  SystemProcess, SystemStats,
 };
 use sysinfo::{ProcessRefreshKind, ProcessesToUpdate, System};
 use tokio::sync::RwLock;
@@ -95,8 +96,15 @@ impl StatsClient {
       network_egress_bytes += network.transmitted();
     }
 
+    let load_avg = System::load_average();
+
     SystemStats {
       cpu_perc: self.system.global_cpu_usage(),
+      load_average: SystemLoadAverage {
+        one: load_avg.one,
+        five: load_avg.five,
+        fifteen: load_avg.fifteen,
+      },
       mem_free_gb: self.system.free_memory() as f64 / BYTES_PER_GB,
       mem_used_gb: (total_mem - available_mem) as f64 / BYTES_PER_GB,
       mem_total_gb: total_mem as f64 / BYTES_PER_GB,

client/core/rs/Cargo.toml

@@ -28,6 +28,7 @@ resolver_api.workspace = true
 # external
 tokio-tungstenite.workspace = true
 derive_builder.workspace = true
+urlencoding.workspace = true
 serde_json.workspace = true
 tokio-util.workspace = true
 thiserror.workspace = true

client/core/rs/src/api/execute/action.rs

@@ -60,7 +60,7 @@ pub struct BatchRunAction {
   /// Supports multiline and comma delineated combinations of the above.
   ///
   /// Example:
-  /// ```
+  /// ```text
   /// # match all foo-* actions
   /// foo-*
   /// # add some more

client/core/rs/src/api/execute/alerter.rs

@@ -4,7 +4,7 @@ use resolver_api::Resolve;
 use serde::{Deserialize, Serialize};
 use typeshare::typeshare;
 
-use crate::entities::update::Update;
+use crate::entities::{alert::SeverityLevel, update::Update};
 
 use super::KomodoExecuteRequest;
 
@@ -27,3 +27,39 @@ pub struct TestAlerter {
   /// Name or id
   pub alerter: String,
 }
+
+//
+
+/// Send a custom alert message to configured Alerters. Response: [Update]
+#[typeshare]
+#[derive(
+  Serialize,
+  Deserialize,
+  Debug,
+  Clone,
+  PartialEq,
+  Resolve,
+  EmptyTraits,
+  Parser,
+)]
+#[empty_traits(KomodoExecuteRequest)]
+#[response(Update)]
+#[error(serror::Error)]
+pub struct SendAlert {
+  /// The alert level.
+  #[serde(default)]
+  #[clap(long, short = 'l', default_value_t = SeverityLevel::Ok)]
+  pub level: SeverityLevel,
+  /// The alert message. Required.
+  pub message: String,
+  /// The alert details. Optional.
+  #[serde(default)]
+  #[arg(long, short = 'd', default_value_t = String::new())]
+  pub details: String,
+  /// Specific alerter names or ids.
+  /// If empty / not passed, sends to all configured alerters
+  /// with the `Custom` alert type whitelisted / not blacklisted.
+  #[serde(default)]
+  #[arg(long, short = 'a')]
+  pub alerters: Vec<String>,
+}

client/core/rs/src/api/execute/build.rs

@@ -64,7 +64,7 @@ pub struct BatchRunBuild {
   /// Supports multiline and comma delineated combinations of the above.
   ///
   /// Example:
-  /// ```
+  /// ```text
   /// # match all foo-* builds
   /// foo-*
   /// # add some more

client/core/rs/src/api/execute/deployment.rs

@@ -62,7 +62,7 @@ pub struct BatchDeploy {
   /// Supports multiline and comma delineated combinations of the above.
   ///
   /// Example:
-  /// ```
+  /// ```text
   /// # match all foo-* deployments
   /// foo-*
   /// # add some more
@@ -270,7 +270,7 @@ pub struct BatchDestroyDeployment {
   /// Supports multiline and comma delineated combinations of the above.
   ///
   /// Example:
-  /// ```
+  /// ```text
   /// # match all foo-* deployments
   /// foo-*
   /// # add some more

client/core/rs/src/api/execute/maintenance.rs

@@ -50,12 +50,12 @@ pub struct ClearRepoCache {}
 #[error(serror::Error)]
 pub struct BackupCoreDatabase {}
 
-/// Trigger a global poll for image updateson Stacks and Deployments
+/// Trigger a global poll for image updates on Stacks and Deployments
 /// with `poll_for_updates` or `auto_update` enabled.
 /// Admin only. Response: [Update]
 ///
 /// 1. `docker compose pull` any Stacks / Deployments with `poll_for_updates` or `auto_update` enabled. This will pick up any available updates.
-/// 2. Redeploy Stacks / Deployments that have updates found.
+/// 2. Redeploy Stacks / Deployments that have updates found and 'auto_update' enabled.
 #[typeshare]
 #[derive(
   Debug,

client/core/rs/src/api/execute/mod.rs

@@ -152,9 +152,12 @@ pub enum Execution {
   StopStack(StopStack),
   DestroyStack(DestroyStack),
   BatchDestroyStack(BatchDestroyStack),
+  RunStackService(RunStackService),
 
   // ALERTER
   TestAlerter(TestAlerter),
+  #[clap(alias = "alert")]
+  SendAlert(SendAlert),
 
   // MAINTENANCE
   ClearRepoCache(ClearRepoCache),

client/core/rs/src/api/execute/procedure.rs

@@ -48,7 +48,7 @@ pub struct BatchRunProcedure {
   /// Supports multiline and comma delineated combinations of the above.
   ///
   /// Example:
-  /// ```
+  /// ```text
   /// # match all foo-* procedures
   /// foo-*
   /// # add some more

client/core/rs/src/api/execute/repo.rs

@@ -60,7 +60,7 @@ pub struct BatchCloneRepo {
   /// Supports multiline and comma delineated combinations of the above.
   ///
   /// Example:
-  /// ```
+  /// ```text
   /// # match all foo-* repos
   /// foo-*
   /// # add some more
@@ -118,7 +118,7 @@ pub struct BatchPullRepo {
   /// Supports multiline and comma delineated combinations of the above.
   ///
   /// Example:
-  /// ```
+  /// ```text
   /// # match all foo-* repos
   /// foo-*
   /// # add some more
@@ -180,7 +180,7 @@ pub struct BatchBuildRepo {
   /// Supports multiline and comma delineated combinations of the above.
   ///
   /// Example:
-  /// ```
+  /// ```text
   /// # match all foo-* repos
   /// foo-*
   /// # add some more

client/core/rs/src/api/execute/stack.rs

@@ -1,11 +1,13 @@
+use crate::entities::update::Update;
+use anyhow::Context;
+use clap::ArgAction::SetTrue;
 use clap::Parser;
 use derive_empty_traits::EmptyTraits;
 use resolver_api::Resolve;
 use serde::{Deserialize, Serialize};
+use std::collections::HashMap;
 use typeshare::typeshare;
 
-use crate::entities::update::Update;
-
 use super::{BatchExecutionResponse, KomodoExecuteRequest};
 
 /// Deploys the target stack. `docker compose up`. Response: [Update]
@@ -57,7 +59,7 @@ pub struct BatchDeployStack {
   /// Supports multiline and comma delineated combinations of the above.
   ///
   /// Example:
-  /// ```
+  /// ```text
   /// # match all foo-* stacks
   /// foo-*
   /// # add some more
@@ -115,7 +117,7 @@ pub struct BatchDeployStackIfChanged {
   /// Supports multiline and comma delineated combinations of the above.
   ///
   /// Example:
-  /// ```
+  /// ```text
   /// # match all foo-* stacks
   /// foo-*
   /// # add some more
@@ -172,7 +174,7 @@ pub struct BatchPullStack {
   /// Supports multiline and comma delineated combinations of the above.
   ///
   /// Example:
-  /// ```
+  /// ```text
   /// # match all foo-* stacks
   /// foo-*
   /// # add some more
@@ -348,6 +350,64 @@ pub struct DestroyStack {
 
 //
 
+/// Runs a one-time command against a service using `docker compose run`. Response: [Update]
+#[typeshare]
+#[derive(
+  Debug,
+  Clone,
+  PartialEq,
+  Serialize,
+  Deserialize,
+  Resolve,
+  EmptyTraits,
+  Parser,
+)]
+#[empty_traits(KomodoExecuteRequest)]
+#[response(Update)]
+#[error(serror::Error)]
+pub struct RunStackService {
+  /// Id or name
+  pub stack: String,
+  /// Service to run
+  pub service: String,
+  /// Command and args to pass to the service container
+  #[arg(trailing_var_arg = true, num_args = 1.., allow_hyphen_values = true)]
+  pub command: Option<Vec<String>>,
+  /// Do not allocate TTY
+  #[arg(long = "no-tty", action = SetTrue)]
+  pub no_tty: Option<bool>,
+  /// Do not start linked services
+  #[arg(long = "no-deps", action = SetTrue)]
+  pub no_deps: Option<bool>,
+  /// Detach container on run
+  #[arg(long = "detach", action = SetTrue)]
+  pub detach: Option<bool>,
+  /// Map service ports to the host
+  #[arg(long = "service-ports", action = SetTrue)]
+  pub service_ports: Option<bool>,
+  /// Extra environment variables for the run
+  #[arg(long = "env", short = 'e', value_parser = env_parser)]
+  pub env: Option<HashMap<String, String>>,
+  /// Working directory inside the container
+  #[arg(long = "workdir")]
+  pub workdir: Option<String>,
+  /// User to run as inside the container
+  #[arg(long = "user")]
+  pub user: Option<String>,
+  /// Override the default entrypoint
+  #[arg(long = "entrypoint")]
+  pub entrypoint: Option<String>,
+  /// Pull the image before running
+  #[arg(long = "pull", action = SetTrue)]
+  pub pull: Option<bool>,
+}
+
+fn env_parser(args: &str) -> anyhow::Result<HashMap<String, String>> {
+  serde_qs::from_str(args).context("Failed to parse env")
+}
+
+//
+
 /// Destroys multiple Stacks in parallel that match pattern. Response: [BatchExecutionResponse].
 #[typeshare]
 #[derive(
@@ -368,7 +428,7 @@ pub struct BatchDestroyStack {
   /// Supports multiline and comma delineated combinations of the above.
   ///d
   /// Example:
-  /// ```
+  /// ```text
   /// # match all foo-* stacks
   /// foo-*
   /// # add some more

client/core/rs/src/api/mod.rs

@@ -20,7 +20,7 @@
 //! the schemas given in [read], [mod@write], [execute], and so on.
 //!
 //! For example, this is an example body for [read::GetDeployment]:
-//! ```
+//! ```json
 //! {
 //!   "type": "GetDeployment",
 //!   "params": {
@@ -56,7 +56,7 @@
 //!
 //! Request errors will be returned with a JSON body containing information about the error.
 //! They will have the following common format:
-//! ```
+//! ```json
 //! {
 //!   "error": "top level error message",
 //!   "trace": [

client/core/rs/src/api/read/alert.rs

@@ -20,7 +20,7 @@ pub struct ListAlerts {
   /// Pass a custom mongo query to filter the alerts.
   ///
   /// ## Example JSON
-  /// ```
+  /// ```json
   /// {
   ///   "resolved": "false",
   ///   "level": "CRITICAL",

client/core/rs/src/api/read/server.rs

@@ -623,6 +623,8 @@ pub struct GetServersSummaryResponse {
   pub total: I64,
   /// The number of healthy (`status: OK`) servers.
   pub healthy: I64,
+  /// The number of servers with warnings (e.g., version mismatch).
+  pub warning: I64,
   /// The number of unhealthy servers.
   pub unhealthy: I64,
   /// The number of disabled servers.

client/core/rs/src/deserializers/conversion.rs

@@ -35,11 +35,10 @@ impl<'de> Visitor<'de> for ConversionVisitor {
     write!(formatter, "string or Vec<Conversion>")
   }
 
-  fn visit_str<E>(self, v: &str) -> Result<Self::Value, E>
+  fn visit_string<E>(self, out: String) -> Result<Self::Value, E>
   where
     E: serde::de::Error,
   {
-    let out = v.to_string();
     if out.is_empty() || out.ends_with('\n') {
       Ok(out)
     } else {
@@ -47,6 +46,13 @@ impl<'de> Visitor<'de> for ConversionVisitor {
     }
   }
 
+  fn visit_str<E>(self, v: &str) -> Result<Self::Value, E>
+  where
+    E: serde::de::Error,
+  {
+    Self::visit_string(self, v.to_string())
+  }
+
   fn visit_seq<A>(self, seq: A) -> Result<Self::Value, A::Error>
   where
     A: serde::de::SeqAccess<'de>,
@@ -78,6 +84,13 @@ impl<'de> Visitor<'de> for OptionConversionVisitor {
     write!(formatter, "null or string or Vec<Conversion>")
   }
 
+  fn visit_string<E>(self, v: String) -> Result<Self::Value, E>
+  where
+    E: serde::de::Error,
+  {
+    ConversionVisitor.visit_string(v).map(Some)
+  }
+
   fn visit_str<E>(self, v: &str) -> Result<Self::Value, E>
   where
     E: serde::de::Error,

client/core/rs/src/deserializers/item_or_vec.rs

@@ -0,0 +1,104 @@
+//! # Item or Vec<Item> deserializer.
+//!
+//! Used to convert `item: T` (struct / map) -> `item: Vec<T>` (seq) in schemas with backward compatibility.
+//! Supports deserializing either a T as Vec<T> with length 1, or a seq as Vec<T> directly.
+
+use serde::{
+  Deserialize, Deserializer,
+  de::{
+    DeserializeOwned, IntoDeserializer, Visitor,
+    value::{MapAccessDeserializer, SeqAccessDeserializer},
+  },
+};
+
+pub fn item_or_vec_deserializer<'de, D, T>(
+  deserializer: D,
+) -> Result<Vec<T>, D::Error>
+where
+  D: Deserializer<'de>,
+  T: DeserializeOwned,
+{
+  deserializer
+    .deserialize_any(ItemOrVecVisitor::<T>(std::marker::PhantomData))
+}
+
+pub fn option_item_or_vec_deserializer<'de, D, T>(
+  deserializer: D,
+) -> Result<Option<Vec<T>>, D::Error>
+where
+  D: Deserializer<'de>,
+  T: DeserializeOwned,
+{
+  deserializer.deserialize_any(OptionItemOrVecVisitor::<T>(
+    std::marker::PhantomData,
+  ))
+}
+
+struct ItemOrVecVisitor<T>(std::marker::PhantomData<T>);
+
+impl<'de, T> Visitor<'de> for ItemOrVecVisitor<T>
+where
+  T: Deserialize<'de>,
+{
+  type Value = Vec<T>;
+
+  fn expecting(
+    &self,
+    formatter: &mut std::fmt::Formatter,
+  ) -> std::fmt::Result {
+    write!(formatter, "Item or Vec<Item>")
+  }
+
+  fn visit_map<A>(self, map: A) -> Result<Self::Value, A::Error>
+  where
+    A: serde::de::MapAccess<'de>,
+  {
+    T::deserialize(
+      MapAccessDeserializer::new(map).into_deserializer(),
+    )
+    .map(|r| vec![r])
+  }
+
+  fn visit_seq<A>(self, seq: A) -> Result<Self::Value, A::Error>
+  where
+    A: serde::de::SeqAccess<'de>,
+  {
+    Vec::<T>::deserialize(
+      SeqAccessDeserializer::new(seq).into_deserializer(),
+    )
+  }
+}
+
+struct OptionItemOrVecVisitor<T>(std::marker::PhantomData<T>);
+
+impl<'de, T> Visitor<'de> for OptionItemOrVecVisitor<T>
+where
+  T: Deserialize<'de>,
+{
+  type Value = Option<Vec<T>>;
+
+  fn expecting(
+    &self,
+    formatter: &mut std::fmt::Formatter,
+  ) -> std::fmt::Result {
+    write!(formatter, "null or Item or Vec<Item>")
+  }
+
+  fn visit_map<A>(self, map: A) -> Result<Self::Value, A::Error>
+  where
+    A: serde::de::MapAccess<'de>,
+  {
+    ItemOrVecVisitor::<T>(std::marker::PhantomData)
+      .visit_map(map)
+      .map(Some)
+  }
+
+  fn visit_seq<A>(self, seq: A) -> Result<Self::Value, A::Error>
+  where
+    A: serde::de::SeqAccess<'de>,
+  {
+    ItemOrVecVisitor::<T>(std::marker::PhantomData)
+      .visit_seq(seq)
+      .map(Some)
+  }
+}

client/core/rs/src/deserializers/mod.rs

@@ -4,6 +4,7 @@ mod conversion;
 mod environment;
 mod file_contents;
 mod forgiving_vec;
+mod item_or_vec;
 mod labels;
 mod maybe_string_i64;
 mod permission;
@@ -14,6 +15,7 @@ pub use conversion::*;
 pub use environment::*;
 pub use file_contents::*;
 pub use forgiving_vec::*;
+pub use item_or_vec::*;
 pub use labels::*;
 pub use maybe_string_i64::*;
 pub use string_list::*;

client/core/rs/src/entities/action.rs

@@ -91,13 +91,13 @@ pub struct ActionConfig {
   /// 1. Regular CRON expression:
   ///
   /// (second, minute, hour, day, month, day-of-week)
-  /// ```
+  /// ```text
   /// 0 0 0 1,15 * ?
   /// ```
   ///
   /// 2. "English" expression via [english-to-cron](https://crates.io/crates/english-to-cron):
   ///
-  /// ```
+  /// ```text
   /// at midnight on the 1st and 15th of the month
   /// ```
   #[serde(default)]

client/core/rs/src/entities/alert.rs

@@ -137,6 +137,20 @@ pub enum AlertData {
     total_gb: f64,
   },
 
+  /// A server has a version mismatch with the core.
+  ServerVersionMismatch {
+    /// The id of the server
+    id: String,
+    /// The name of the server
+    name: String,
+    /// The region of the server
+    region: Option<String>,
+    /// The actual server version
+    server_version: String,
+    /// The core version
+    core_version: String,
+  },
+
   /// A container's state has changed unexpectedly.
   ContainerStateChange {
     /// The id of the deployment
@@ -286,6 +300,16 @@ pub enum AlertData {
     /// The resource name
     name: String,
   },
+
+  /// Custom header / body.
+  /// Produced using `/execute/SendAlert`
+  Custom {
+    /// The alert message.
+    message: String,
+    /// Message details. May be empty string.
+    #[serde(default)]
+    details: String,
+  },
 }
 
 impl Default for AlertData {
@@ -320,10 +344,29 @@ impl Default for AlertDataVariant {
 #[strum(serialize_all = "UPPERCASE")]
 pub enum SeverityLevel {
   /// No problem.
+  ///
+  /// Aliases: ok, low, l
   #[default]
+  #[strum(serialize = "ok", serialize = "low", serialize = "l")]
   Ok,
   /// Problem is imminent.
+  ///
+  /// Aliases: warning, w, medium, m
+  #[strum(
+    serialize = "warning",
+    serialize = "w",
+    serialize = "medium",
+    serialize = "m"
+  )]
   Warning,
   /// Problem fully realized.
+  ///
+  /// Aliases: critical, c, high, h
+  #[strum(
+    serialize = "critical",
+    serialize = "c",
+    serialize = "high",
+    serialize = "h"
+  )]
   Critical,
 }

client/core/rs/src/entities/build.rs

@@ -1,25 +1,25 @@
+use std::sync::OnceLock;
+
 use bson::{Document, doc};
 use derive_builder::Builder;
 use derive_default_builder::DefaultBuilder;
 use partial_derive2::Partial;
-use serde::{
-  Deserialize, Deserializer, Serialize,
-  de::{Visitor, value::MapAccessDeserializer},
-};
+use serde::{Deserialize, Serialize};
 use strum::Display;
 use typeshare::typeshare;
 
 use crate::{
   deserializers::{
-    env_vars_deserializer, labels_deserializer,
-    option_env_vars_deserializer, option_labels_deserializer,
+    env_vars_deserializer, item_or_vec_deserializer,
+    labels_deserializer, option_env_vars_deserializer,
+    option_item_or_vec_deserializer, option_labels_deserializer,
     option_string_list_deserializer, string_list_deserializer,
   },
   entities::I64,
 };
 
 use super::{
-  NoData, SystemCommand, Version,
+  SystemCommand, Version,
   resource::{Resource, ResourceListItem, ResourceQuery},
 };
 
@@ -62,7 +62,7 @@ pub struct BuildListItemInfo {
   /// Latest short commit hash, or null. Only for repo based stacks
   pub latest_hash: Option<String>,
 
-  /// The image registry domain
+  /// The first listed image registry domain
   pub image_registry_domain: Option<String>,
 }
 
@@ -255,14 +255,15 @@ pub struct BuildConfig {
   #[partial_default(default_dockerfile_path())]
   pub dockerfile_path: String,
 
-  /// Configuration for the registry to push the built image to.
-  #[serde(default, deserialize_with = "image_registry_deserializer")]
+  /// Configuration for the registry/s to push the built image to.
+  /// The first registry in this list will be used with attached Deployments.
+  #[serde(default, deserialize_with = "item_or_vec_deserializer")]
   #[partial_attr(serde(
     default,
-    deserialize_with = "option_image_registry_deserializer"
+    deserialize_with = "option_item_or_vec_deserializer"
   ))]
   #[builder(default)]
-  pub image_registry: ImageRegistryConfig,
+  pub image_registry: Vec<ImageRegistryConfig>,
 
   /// Whether to skip secret interpolation in the build_args.
   #[serde(default)]
@@ -422,125 +423,10 @@ pub struct ImageRegistryConfig {
   pub organization: String,
 }
 
-pub fn image_registry_deserializer<'de, D>(
-  deserializer: D,
-) -> Result<ImageRegistryConfig, D::Error>
-where
-  D: Deserializer<'de>,
-{
-  deserializer.deserialize_any(ImageRegistryVisitor)
-}
-
-pub fn option_image_registry_deserializer<'de, D>(
-  deserializer: D,
-) -> Result<Option<ImageRegistryConfig>, D::Error>
-where
-  D: Deserializer<'de>,
-{
-  deserializer.deserialize_any(OptionImageRegistryVisitor)
-}
-
-struct ImageRegistryVisitor;
-
-impl<'de> Visitor<'de> for ImageRegistryVisitor {
-  type Value = ImageRegistryConfig;
-
-  fn expecting(
-    &self,
-    formatter: &mut std::fmt::Formatter,
-  ) -> std::fmt::Result {
-    write!(
-      formatter,
-      "{{ \"domain\": string, \"account\": string, \"organization\": string }}"
-    )
-  }
-
-  fn visit_map<A>(self, map: A) -> Result<Self::Value, A::Error>
-  where
-    A: serde::de::MapAccess<'de>,
-  {
-    // Need to use Value intermediate to get something cloneable.
-    let value = serde_json::Value::deserialize(
-      MapAccessDeserializer::new(map),
-    )?;
-    // 1.14 and before: try to use ImageRegistry1_14 syntax
-    if let Ok(config) =
-      serde_json::from_value::<ImageRegistryLegacy1_14>(value.clone())
-    {
-      return Ok(config.into());
-    }
-    serde_json::from_value(value).map_err(serde::de::Error::custom)
-  }
-}
-
-struct OptionImageRegistryVisitor;
-
-impl<'de> Visitor<'de> for OptionImageRegistryVisitor {
-  type Value = Option<ImageRegistryConfig>;
-
-  fn expecting(
-    &self,
-    formatter: &mut std::fmt::Formatter,
-  ) -> std::fmt::Result {
-    write!(
-      formatter,
-      "null or {{ \"domain\": string, \"account\": string, \"organization\": string }}"
-    )
-  }
-
-  fn visit_map<A>(self, map: A) -> Result<Self::Value, A::Error>
-  where
-    A: serde::de::MapAccess<'de>,
-  {
-    ImageRegistryVisitor.visit_map(map).map(Some)
-  }
-
-  fn visit_none<E>(self) -> Result<Self::Value, E>
-  where
-    E: serde::de::Error,
-  {
-    Ok(None)
-  }
-
-  fn visit_unit<E>(self) -> Result<Self::Value, E>
-  where
-    E: serde::de::Error,
-  {
-    Ok(None)
-  }
-}
-
-/// Configuration for the registry to push the built image to.
-#[typeshare]
-#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
-#[serde(tag = "type", content = "params")]
-pub enum ImageRegistryLegacy1_14 {
-  /// Don't push the image to any registry
-  None(NoData),
-  /// Push the image to a standard image registry (any domain)
-  Standard(ImageRegistryConfig),
-}
-
-impl Default for ImageRegistryLegacy1_14 {
-  fn default() -> Self {
-    Self::None(NoData {})
-  }
-}
-
-impl From<ImageRegistryLegacy1_14> for ImageRegistryConfig {
-  fn from(value: ImageRegistryLegacy1_14) -> Self {
-    match value {
-      ImageRegistryLegacy1_14::None(_) => {
-        ImageRegistryConfig::default()
-      }
-      ImageRegistryLegacy1_14::Standard(mut config) => {
-        // This version had domain defaulted to docker.io
-        if config.domain.is_empty() {
-          config.domain = String::from("docker.io");
-        }
-        config
-      }
-    }
+impl ImageRegistryConfig {
+  pub fn static_default() -> &'static ImageRegistryConfig {
+    static DEFAULT: OnceLock<ImageRegistryConfig> = OnceLock::new();
+    DEFAULT.get_or_init(Default::default)
   }
 }
 

client/core/rs/src/entities/config/cli/args/mod.rs

@@ -63,8 +63,14 @@ pub enum Command {
   #[clap(alias = "ls", alias = "resources")]
   List(list::List),
 
-  /// Run Komodo executions. (aliases: `x`, `run`, `deploy`, `dep`)
-  #[clap(alias = "x", alias = "run", alias = "deploy", alias = "dep")]
+  /// Run Komodo executions. (aliases: `x`, `run`, `deploy`, `dep`, `send`)
+  #[clap(
+    alias = "x",
+    alias = "run",
+    alias = "deploy",
+    alias = "dep",
+    alias = "send"
+  )]
   Execute(Execute),
 
   /// Update resource configuration. (alias: `set`)

client/core/rs/src/entities/config/cli/mod.rs

@@ -70,7 +70,7 @@ pub struct Env {
   pub komodo_cli_backups_folder: Option<PathBuf>,
   /// Override `max_backups`
   pub komodo_cli_max_backups: Option<u16>,
-  
+
   /// Override `database_target_uri`
   #[serde(alias = "komodo_cli_database_copy_uri")]
   pub komodo_cli_database_target_uri: Option<String>,

client/core/rs/src/entities/config/core.rs

@@ -125,6 +125,8 @@ pub struct Env {
   pub komodo_logging_opentelemetry_service_name: Option<String>,
   /// Override `pretty_startup_config`
   pub komodo_pretty_startup_config: Option<bool>,
+  /// Override `unsafe_unsanitized_startup_config`
+  pub komodo_unsafe_unsanitized_startup_config: Option<bool>,
 
   /// Override `transparent_mode`
   pub komodo_transparent_mode: Option<bool>,
@@ -509,6 +511,12 @@ pub struct CoreConfig {
   #[serde(default)]
   pub pretty_startup_config: bool,
 
+  /// Unsafe: logs unsanitized config on startup,
+  /// in order to verify everything is being
+  /// passed correctly.
+  #[serde(default)]
+  pub unsafe_unsanitized_startup_config: bool,
+
   // ===========
   // = Pruning =
   // ===========
@@ -729,6 +737,7 @@ impl Default for CoreConfig {
       github_webhook_app: Default::default(),
       logging: Default::default(),
       pretty_startup_config: Default::default(),
+      unsafe_unsanitized_startup_config: Default::default(),
       keep_stats_for_days: default_prune_days(),
       keep_alerts_for_days: default_prune_days(),
       resource_poll_interval: default_poll_interval(),
@@ -772,6 +781,8 @@ impl CoreConfig {
       keep_alerts_for_days: config.keep_alerts_for_days,
       logging: config.logging,
       pretty_startup_config: config.pretty_startup_config,
+      unsafe_unsanitized_startup_config: config
+        .unsafe_unsanitized_startup_config,
       transparent_mode: config.transparent_mode,
       ui_write_disabled: config.ui_write_disabled,
       disable_confirm_dialog: config.disable_confirm_dialog,

client/core/rs/src/entities/mod.rs

@@ -129,43 +129,52 @@ pub fn optional_string(string: impl Into<String>) -> Option<String> {
   }
 }
 
-pub fn get_image_name(
+pub fn get_image_names(
   build::Build {
     name,
     config:
       build::BuildConfig {
         image_name,
-        image_registry:
-          ImageRegistryConfig {
-            domain,
-            account,
-            organization,
-          },
+        image_registry,
         ..
       },
     ..
   }: &build::Build,
-) -> anyhow::Result<String> {
+) -> Vec<String> {
   let name = if image_name.is_empty() {
     name
   } else {
     image_name
   };
-  let name = match (
-    !domain.is_empty(),
-    !organization.is_empty(),
-    !account.is_empty(),
-  ) {
-    // If organization and account provided, name under organization.
-    (true, true, true) => {
-      format!("{domain}/{organization}/{name}")
-    }
-    // Just domain / account provided
-    (true, false, true) => format!("{domain}/{account}/{name}"),
-    // Otherwise, just use name
-    _ => name.to_string(),
-  };
-  Ok(name)
+  // Local only
+  if image_registry.is_empty() {
+    return vec![name.to_string()];
+  }
+  image_registry
+    .iter()
+    .map(
+      |ImageRegistryConfig {
+         domain,
+         account,
+         organization,
+       }| {
+        match (
+          !domain.is_empty(),
+          !organization.is_empty(),
+          !account.is_empty(),
+        ) {
+          // If organization and account provided, name under organization.
+          (true, true, true) => {
+            format!("{domain}/{organization}/{name}")
+          }
+          // Just domain / account provided
+          (true, false, true) => format!("{domain}/{account}/{name}"),
+          // Otherwise, just use name (local only)
+          _ => name.to_string(),
+        }
+      },
+    )
+    .collect()
 }
 
 pub fn to_general_name(name: &str) -> String {
@@ -396,7 +405,7 @@ pub struct LatestCommit {
 #[typeshare]
 #[derive(Debug, Clone, Default, Serialize, Deserialize)]
 pub struct FileContents {
-  /// The path of the file on the host
+  /// The path to the file
   pub path: String,
   /// The contents of the file
   pub contents: String,
@@ -502,8 +511,17 @@ impl RepoExecutionArgs {
     &self,
     access_token: Option<&str>,
   ) -> anyhow::Result<String> {
-    let access_token_at = match &access_token {
-      Some(token) => format!("token:{token}@"),
+    let access_token_at = match access_token {
+      Some(token) => match token.split_once(':') {
+        Some((username, token)) => format!(
+          "{}:{}@",
+          urlencoding::encode(username.trim()),
+          urlencoding::encode(token.trim())
+        ),
+        None => {
+          format!("token:{}@", urlencoding::encode(token.trim()))
+        }
+      },
       None => String::new(),
     };
     let protocol = if self.https { "https" } else { "http" };
@@ -1083,6 +1101,7 @@ pub enum Operation {
   UnpauseStack,
   StopStack,
   DestroyStack,
+  RunStackService,
 
   // stack (service)
   DeployStackService,
@@ -1153,6 +1172,7 @@ pub enum Operation {
   RenameAlerter,
   DeleteAlerter,
   TestAlerter,
+  SendAlert,
 
   // sync
   CreateResourceSync,

client/core/rs/src/entities/procedure.rs

@@ -91,13 +91,13 @@ pub struct ProcedureConfig {
   /// 1. Regular CRON expression:
   ///
   /// (second, minute, hour, day, month, day-of-week)
-  /// ```
+  /// ```text
   /// 0 0 0 1,15 * ?
   /// ```
   ///
   /// 2. "English" expression via [english-to-cron](https://crates.io/crates/english-to-cron):
   ///
-  /// ```
+  /// ```text
   /// at midnight on the 1st and 15th of the month
   /// ```
   #[serde(default)]

client/core/rs/src/entities/server.rs

@@ -48,6 +48,8 @@ pub struct ServerListItemInfo {
   pub send_mem_alerts: bool,
   /// Whether server is configured to send disk alerts.
   pub send_disk_alerts: bool,
+  /// Whether server is configured to send version mismatch alerts.
+  pub send_version_mismatch_alerts: bool,
   /// Whether terminals are disabled for this Server.
   pub terminals_disabled: bool,
   /// Whether container exec is disabled for this Server.
@@ -84,7 +86,7 @@ pub struct ServerConfig {
   /// Whether a server is enabled.
   /// If a server is disabled,
   /// you won't be able to perform any actions on it or see deployment's status.
-  /// default: true
+  /// Default: false
   #[serde(default = "default_enabled")]
   #[builder(default = "default_enabled()")]
   #[partial_default(default_enabled())]
@@ -161,6 +163,12 @@ pub struct ServerConfig {
   #[partial_default(default_send_alerts())]
   pub send_disk_alerts: bool,
 
+  /// Whether to send alerts about the servers version mismatch with core
+  #[serde(default = "default_send_alerts")]
+  #[builder(default = "default_send_alerts()")]
+  #[partial_default(default_send_alerts())]
+  pub send_version_mismatch_alerts: bool,
+
   /// The percentage threshhold which triggers WARNING state for CPU.
   #[serde(default = "default_cpu_warning")]
   #[builder(default = "default_cpu_warning()")]
@@ -272,6 +280,7 @@ impl Default for ServerConfig {
       send_cpu_alerts: default_send_alerts(),
       send_mem_alerts: default_send_alerts(),
       send_disk_alerts: default_send_alerts(),
+      send_version_mismatch_alerts: default_send_alerts(),
       region: Default::default(),
       passkey: Default::default(),
       cpu_warning: default_cpu_warning(),

client/core/rs/src/entities/stack.rs

@@ -4,8 +4,12 @@ use anyhow::Context;
 use bson::{Document, doc};
 use derive_builder::Builder;
 use derive_default_builder::DefaultBuilder;
+use indexmap::IndexSet;
 use partial_derive2::Partial;
-use serde::{Deserialize, Serialize};
+use serde::{
+  Deserialize, Serialize,
+  de::{IntoDeserializer, Visitor, value::MapAccessDeserializer},
+};
 use strum::Display;
 use typeshare::typeshare;
 
@@ -44,13 +48,56 @@ impl Stack {
     }
   }
 
-  pub fn file_paths(&self) -> &[String] {
+  pub fn compose_file_paths(&self) -> &[String] {
     if self.config.file_paths.is_empty() {
       default_stack_file_paths()
     } else {
       &self.config.file_paths
     }
   }
+
+  pub fn is_compose_file(&self, path: &str) -> bool {
+    for compose_path in self.compose_file_paths() {
+      if path.ends_with(compose_path) {
+        return true;
+      }
+    }
+    false
+  }
+
+  pub fn all_file_paths(&self) -> Vec<String> {
+    let mut res = self
+      .compose_file_paths()
+      .iter()
+      .cloned()
+      // Makes sure to dedup them, while maintaining ordering
+      .collect::<IndexSet<_>>();
+    res.extend(self.config.additional_env_files.clone());
+    res.extend(
+      self.config.config_files.iter().map(|f| f.path.clone()),
+    );
+    res.into_iter().collect()
+  }
+
+  pub fn all_file_dependencies(&self) -> Vec<StackFileDependency> {
+    let mut res = self
+      .compose_file_paths()
+      .iter()
+      .cloned()
+      .map(StackFileDependency::full_redeploy)
+      // Makes sure to dedup them, while maintaining ordering
+      .collect::<IndexSet<_>>();
+    res.extend(
+      self
+        .config
+        .additional_env_files
+        .iter()
+        .cloned()
+        .map(StackFileDependency::full_redeploy),
+    );
+    res.extend(self.config.config_files.clone());
+    res.into_iter().collect()
+  }
 }
 
 fn default_stack_file_paths() -> &'static [String] {
@@ -159,7 +206,7 @@ pub enum StackState {
 #[typeshare]
 #[derive(Debug, Clone, Default, Serialize, Deserialize)]
 pub struct StackInfo {
-  /// If any of the expected files are missing in the repo,
+  /// If any of the expected compose / additional files are missing in the repo,
   /// they will be stored here.
   #[serde(default)]
   pub missing_files: Vec<String>,
@@ -174,7 +221,7 @@ pub struct StackInfo {
   pub deployed_hash: Option<String>,
   /// Deployed commit message, or null. Only for repo based stacks
   pub deployed_message: Option<String>,
-  /// The deployed compose file contents.
+  /// The deployed compose / additional file contents.
   /// This is updated whenever Komodo successfully deploys the stack.
   pub deployed_contents: Option<Vec<FileContents>>,
   /// The deployed service names.
@@ -188,10 +235,10 @@ pub struct StackInfo {
   #[serde(default)]
   pub latest_services: Vec<StackServiceNames>,
 
-  /// The remote compose file contents, whether on host or in repo.
+  /// The remote compose / additional file contents, whether on host or in repo.
   /// This is updated whenever Komodo refreshes the stack cache.
   /// It will be empty if the file is defined directly in the stack config.
-  pub remote_contents: Option<Vec<FileContents>>,
+  pub remote_contents: Option<Vec<StackRemoteFileContents>>,
   /// If there was an error in getting the remote contents, it will be here.
   pub remote_errors: Option<Vec<FileContents>>,
 
@@ -386,6 +433,9 @@ pub struct StackConfig {
 
   /// Add additional env files to attach with `--env-file`.
   /// Relative to the run directory root.
+  ///
+  /// Note. It is already included as an `additional_file`.
+  /// Don't add it again there.
   #[serde(default, deserialize_with = "string_list_deserializer")]
   #[partial_attr(serde(
     default,
@@ -394,6 +444,18 @@ pub struct StackConfig {
   #[builder(default)]
   pub additional_env_files: Vec<String>,
 
+  /// Add additional config files either in repo or on host to track.
+  /// Can add any files associated with the stack to enable editing them in the UI.
+  /// Doing so will also include diffing these when deciding to deploy in `DeployStackIfChanged`.
+  /// Relative to the run directory.
+  ///
+  /// Note. If the config file is .env and should be included in compose command
+  /// using `--env-file`, add it to `additional_env_files` instead.
+  #[serde(default)]
+  #[partial_attr(serde(default))]
+  #[builder(default)]
+  pub config_files: Vec<StackFileDependency>,
+
   /// Whether to send StackStateChange alerts for this stack.
   #[serde(default = "default_send_alerts")]
   #[builder(default = "default_send_alerts()")]
@@ -539,6 +601,7 @@ impl Default for StackConfig {
       environment: Default::default(),
       env_file_path: default_env_file_path(),
       additional_env_files: Default::default(),
+      config_files: Default::default(),
       run_build: Default::default(),
       destroy_before_deploy: Default::default(),
       build_extra_args: Default::default(),
@@ -638,6 +701,10 @@ pub struct StackQuerySpecifics {
   /// Only accepts Server id (not name).
   #[serde(default)]
   pub server_ids: Vec<String>,
+  /// Query only for Stacks with these linked repos.
+  /// Only accepts Repo id (not name).
+  #[serde(default)]
+  pub linked_repos: Vec<String>,
   /// Filter syncs by their repo.
   #[serde(default)]
   pub repos: Vec<String>,
@@ -652,6 +719,12 @@ impl super::resource::AddFilters for StackQuerySpecifics {
       filters
         .insert("config.server_id", doc! { "$in": &self.server_ids });
     }
+    if !self.linked_repos.is_empty() {
+      filters.insert(
+        "config.linked_repo",
+        doc! { "$in": &self.linked_repos },
+      );
+    }
     if !self.repos.is_empty() {
       filters.insert("config.repo", doc! { "$in": &self.repos });
     }
@@ -683,3 +756,165 @@ pub struct ComposeServiceDeploy {
   )]
   pub replicas: Option<i64>,
 }
+
+// PRE-1.19.1 BACKWARD COMPAT NOTE
+// This was split from general FileContents in 1.19.1,
+// and must maintain 2 way de/ser backward compatibility
+// with the mentioned struct.
+/// Same as [FileContents] with some extra
+/// info specific to Stacks.
+#[typeshare]
+#[derive(Debug, Clone, Default, Serialize, Deserialize)]
+pub struct StackRemoteFileContents {
+  /// The path to the file
+  pub path: String,
+  /// The contents of the file
+  pub contents: String,
+  /// The services depending on this file,
+  /// or empty for global requirement (eg all compose files and env files).
+  #[serde(default)]
+  pub services: Vec<String>,
+  /// Whether diff requires Redeploy / Restart / None
+  #[serde(default)]
+  pub requires: StackFileRequires,
+}
+
+#[typeshare]
+#[derive(
+  Debug,
+  Clone,
+  Copy,
+  PartialEq,
+  Eq,
+  Hash,
+  Default,
+  Serialize,
+  Deserialize,
+)]
+pub enum StackFileRequires {
+  /// Diff requires service redeploy.
+  #[serde(alias = "redeploy")]
+  Redeploy,
+  /// Diff requires service restart
+  #[serde(alias = "restart")]
+  Restart,
+  /// Diff requires no action. Default.
+  #[default]
+  #[serde(alias = "none")]
+  None,
+}
+
+/// Configure additional file dependencies of the Stack.
+#[typeshare]
+#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize)]
+pub struct StackFileDependency {
+  /// Specify the file
+  pub path: String,
+  /// Specify specific service/s
+  #[serde(default, skip_serializing_if = "Vec::is_empty")]
+  pub services: Vec<String>,
+  /// Specify
+  #[serde(default, skip_serializing_if = "is_none")]
+  pub requires: StackFileRequires,
+}
+
+impl StackFileDependency {
+  pub fn full_redeploy(path: String) -> StackFileDependency {
+    StackFileDependency {
+      path,
+      services: Vec::new(),
+      requires: StackFileRequires::Redeploy,
+    }
+  }
+}
+
+fn is_none(requires: &StackFileRequires) -> bool {
+  matches!(requires, StackFileRequires::None)
+}
+
+/// Used with custom de/serializer for [StackFileDependency]
+#[derive(Deserialize)]
+struct __StackFileDependency {
+  path: String,
+  #[serde(
+    default,
+    alias = "service",
+    deserialize_with = "string_list_deserializer"
+  )]
+  services: Vec<String>,
+  #[serde(default, alias = "req")]
+  requires: StackFileRequires,
+}
+
+impl<'de> Deserialize<'de> for StackFileDependency {
+  fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+  where
+    D: serde::Deserializer<'de>,
+  {
+    struct StackFileDependencyVisitor;
+
+    impl<'de> Visitor<'de> for StackFileDependencyVisitor {
+      type Value = StackFileDependency;
+
+      fn expecting(
+        &self,
+        formatter: &mut std::fmt::Formatter,
+      ) -> std::fmt::Result {
+        write!(formatter, "string or StackFileDependency (object)")
+      }
+
+      fn visit_string<E>(self, path: String) -> Result<Self::Value, E>
+      where
+        E: serde::de::Error,
+      {
+        Ok(StackFileDependency {
+          path,
+          services: Vec::new(),
+          requires: StackFileRequires::None,
+        })
+      }
+
+      fn visit_str<E>(self, v: &str) -> Result<Self::Value, E>
+      where
+        E: serde::de::Error,
+      {
+        Self::visit_string(self, v.to_string())
+      }
+
+      fn visit_map<A>(self, map: A) -> Result<Self::Value, A::Error>
+      where
+        A: serde::de::MapAccess<'de>,
+      {
+        __StackFileDependency::deserialize(
+          MapAccessDeserializer::new(map).into_deserializer(),
+        )
+        .map(|v| StackFileDependency {
+          path: v.path,
+          services: v.services,
+          requires: v.requires,
+        })
+      }
+    }
+
+    deserializer.deserialize_any(StackFileDependencyVisitor)
+  }
+}
+
+// // This one is nice for TOML, but annoying to use on frontend
+// impl Serialize for StackFileDependency {
+//   fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+//   where
+//     S: serde::Serializer,
+//   {
+//     // Serialize to string in default case
+//     if is_redeploy(&self.requires) && self.services.is_empty() {
+//       return serializer.serialize_str(&self.path);
+//     }
+//     __StackFileDependency {
+//       path: self.path.clone(),
+//       services: self.services.clone(),
+//       requires: self.requires,
+//     }
+//     .serialize(serializer)
+//   }
+// }

client/core/rs/src/entities/stats.rs

@@ -45,6 +45,9 @@ pub struct SystemStatsRecord {
   // basic stats
   /// Cpu usage percentage
   pub cpu_perc: f32,
+  /// Load average (1m, 5m, 15m)
+  #[serde(default)]
+  pub load_average: SystemLoadAverage,
   /// Memory used in GB
   pub mem_used_gb: f64,
   /// Total memory in GB
@@ -72,6 +75,9 @@ pub struct SystemStatsRecord {
 pub struct SystemStats {
   /// Cpu usage percentage
   pub cpu_perc: f32,
+  ///  Load average (1m, 5m, 15m)
+  #[serde(default)]
+  pub load_average: SystemLoadAverage,
   /// [1.15.9+]
   /// Free memory in GB.
   /// This is really the 'Free' memory, not the 'Available' memory.
@@ -175,3 +181,14 @@ pub struct SystemProcess {
   /// Process disk write in KB/s
   pub disk_write_kb: f64,
 }
+
+#[typeshare]
+#[derive(Serialize, Deserialize, Debug, Default, Clone)]
+pub struct SystemLoadAverage {
+  /// 1m load average
+  pub one: f64,
+  /// 5m load average
+  pub five: f64,
+  /// 15m load average
+  pub fifteen: f64,
+}

client/core/rs/src/lib.rs

@@ -15,7 +15,7 @@
 //! - `KOMODO_API_SECRET`
 //!
 //! ## Client Example
-//! ```
+//! ```text
 //! dotenvy::dotenv().ok();
 //!
 //! let client = KomodoClient::new_from_env()?;
@@ -109,7 +109,7 @@ impl KomodoClient {
 
   /// Add a healthcheck in the initialization pipeline:
   ///
-  /// ```rust
+  /// ```text
   /// let komodo = KomodoClient::new_from_env()?
   ///   .with_healthcheck().await?;
   /// ```
@@ -121,7 +121,7 @@ impl KomodoClient {
 
   /// Add a healthcheck in the initialization pipeline:
   ///
-  /// ```rust
+  /// ```text
   /// let komodo = KomodoClient::new_from_env()?
   ///   .with_healthcheck().await?;
   /// ```

client/core/rs/src/ws.rs

@@ -61,7 +61,7 @@ impl KomodoClient {
   /// and forwards the updates over a channel.
   /// Handles reconnection internally.
   ///
-  /// ```
+  /// ```text
   /// let (mut rx, _) = komodo.subscribe_to_updates()?;
   /// loop {
   ///   let update = match rx.recv().await {

client/core/ts/package.json

@@ -1,6 +1,6 @@
 {
   "name": "komodo_client",
-  "version": "1.19.0",
+  "version": "1.19.2",
   "description": "Komodo client package",
   "homepage": "https://komo.do",
   "main": "dist/lib.js",

client/core/ts/src/responses.ts

@@ -405,9 +405,11 @@ export type ExecuteResponses = {
   PauseStackService: Types.Update;
   UnpauseStackService: Types.Update;
   DestroyStackService: Types.Update;
-
+  RunStackService: Types.Update;
+  
   // ==== ALERTER ====
   TestAlerter: Types.Update;
+  SendAlert: Types.Update;
 
   // ==== MAINTENANCE ====
   ClearRepoCache: Types.Update;

client/core/ts/src/types.ts

@@ -83,13 +83,13 @@ export interface ActionConfig {
 	 * 1. Regular CRON expression:
 	 * 
 	 * (second, minute, hour, day, month, day-of-week)
-	 * ```
+	 * ```text
 	 * 0 0 0 1,15 * ?
 	 * ```
 	 * 
 	 * 2. "English" expression via [english-to-cron](https://crates.io/crates/english-to-cron):
 	 * 
-	 * ```
+	 * ```text
 	 * at midnight on the 1st and 15th of the month
 	 * ```
 	 */
@@ -386,6 +386,7 @@ export enum Operation {
 	UnpauseStack = "UnpauseStack",
 	StopStack = "StopStack",
 	DestroyStack = "DestroyStack",
+	RunStackService = "RunStackService",
 	DeployStackService = "DeployStackService",
 	PullStackService = "PullStackService",
 	StartStackService = "StartStackService",
@@ -440,6 +441,7 @@ export enum Operation {
 	RenameAlerter = "RenameAlerter",
 	DeleteAlerter = "DeleteAlerter",
 	TestAlerter = "TestAlerter",
+	SendAlert = "SendAlert",
 	CreateResourceSync = "CreateResourceSync",
 	UpdateResourceSync = "UpdateResourceSync",
 	RenameResourceSync = "RenameResourceSync",
@@ -643,8 +645,11 @@ export interface BuildConfig {
 	build_path: string;
 	/** The path of the dockerfile relative to the build path. */
 	dockerfile_path: string;
-	/** Configuration for the registry to push the built image to. */
-	image_registry?: ImageRegistryConfig;
+	/**
+	 * Configuration for the registry/s to push the built image to.
+	 * The first registry in this list will be used with attached Deployments.
+	 */
+	image_registry?: ImageRegistryConfig[];
 	/** Whether to skip secret interpolation in the build_args. */
 	skip_secret_interp?: boolean;
 	/** Whether to use buildx to build (eg `docker buildx build ...`) */
@@ -749,7 +754,7 @@ export interface BuildListItemInfo {
 	built_hash?: string;
 	/** Latest short commit hash, or null. Only for repo based stacks */
 	latest_hash?: string;
-	/** The image registry domain */
+	/** The first listed image registry domain */
 	image_registry_domain?: string;
 }
 
@@ -867,7 +872,9 @@ export type Execution =
 	| { type: "StopStack", params: StopStack }
 	| { type: "DestroyStack", params: DestroyStack }
 	| { type: "BatchDestroyStack", params: BatchDestroyStack }
+	| { type: "RunStackService", params: RunStackService }
 	| { type: "TestAlerter", params: TestAlerter }
+	| { type: "SendAlert", params: SendAlert }
 	| { type: "ClearRepoCache", params: ClearRepoCache }
 	| { type: "BackupCoreDatabase", params: BackupCoreDatabase }
 	| { type: "GlobalAutoUpdate", params: GlobalAutoUpdate }
@@ -905,13 +912,13 @@ export interface ProcedureConfig {
 	 * 1. Regular CRON expression:
 	 * 
 	 * (second, minute, hour, day, month, day-of-week)
-	 * ```
+	 * ```text
 	 * 0 0 0 1,15 * ?
 	 * ```
 	 * 
 	 * 2. "English" expression via [english-to-cron](https://crates.io/crates/english-to-cron):
 	 * 
-	 * ```
+	 * ```text
 	 * at midnight on the 1st and 15th of the month
 	 * ```
 	 */
@@ -1354,11 +1361,23 @@ export type GetActionResponse = Action;
 
 /** Severity level of problem. */
 export enum SeverityLevel {
-	/** No problem. */
+	/**
+	 * No problem.
+	 * 
+	 * Aliases: ok, low, l
+	 */
 	Ok = "OK",
-	/** Problem is imminent. */
+	/**
+	 * Problem is imminent.
+	 * 
+	 * Aliases: warning, w, medium, m
+	 */
 	Warning = "WARNING",
-	/** Problem fully realized. */
+	/**
+	 * Problem fully realized.
+	 * 
+	 * Aliases: critical, c, high, h
+	 */
 	Critical = "CRITICAL",
 }
 
@@ -1426,6 +1445,19 @@ export type AlertData =
 	used_gb: number;
 	/** The total size of the disk in GB */
 	total_gb: number;
+}}
+	/** A server has a version mismatch with the core. */
+	| { type: "ServerVersionMismatch", data: {
+	/** The id of the server */
+	id: string;
+	/** The name of the server */
+	name: string;
+	/** The region of the server */
+	region?: string;
+	/** The actual server version */
+	server_version: string;
+	/** The core version */
+	core_version: string;
 }}
 	/** A container's state has changed unexpectedly. */
 	| { type: "ContainerStateChange", data: {
@@ -1563,6 +1595,16 @@ export type AlertData =
 	id: string;
 	/** The resource name */
 	name: string;
+}}
+	/**
+	 * Custom header / body.
+	 * Produced using `/execute/SendAlert`
+	 */
+	| { type: "Custom", data: {
+	/** The alert message. */
+	message: string;
+	/** Message details. May be empty string. */
+	details?: string;
 }};
 
 /** Representation of an alert in the system. */
@@ -1964,7 +2006,7 @@ export interface ServerConfig {
 	 * Whether a server is enabled.
 	 * If a server is disabled,
 	 * you won't be able to perform any actions on it or see deployment's status.
-	 * default: true
+	 * Default: false
 	 */
 	enabled: boolean;
 	/**
@@ -2003,6 +2045,8 @@ export interface ServerConfig {
 	send_mem_alerts: boolean;
 	/** Whether to send alerts about the servers DISK status */
 	send_disk_alerts: boolean;
+	/** Whether to send alerts about the servers version mismatch with core */
+	send_version_mismatch_alerts: boolean;
 	/** The percentage threshhold which triggers WARNING state for CPU. */
 	cpu_warning: number;
 	/** The percentage threshhold which triggers CRITICAL state for CPU. */
@@ -2038,6 +2082,25 @@ export type GetStackActionStateResponse = StackActionState;
 
 export type GetStackLogResponse = Log;
 
+export enum StackFileRequires {
+	/** Diff requires service redeploy. */
+	Redeploy = "Redeploy",
+	/** Diff requires service restart */
+	Restart = "Restart",
+	/** Diff requires no action. Default. */
+	None = "None",
+}
+
+/** Configure additional file dependencies of the Stack. */
+export interface StackFileDependency {
+	/** Specify the file */
+	path: string;
+	/** Specify specific service/s */
+	services?: string[];
+	/** Specify */
+	requires?: StackFileRequires;
+}
+
 /** The compose file configuration. */
 export interface StackConfig {
 	/** The server to deploy the stack on. */
@@ -2152,8 +2215,21 @@ export interface StackConfig {
 	/**
 	 * Add additional env files to attach with `--env-file`.
 	 * Relative to the run directory root.
+	 * 
+	 * Note. It is already included as an `additional_file`.
+	 * Don't add it again there.
 	 */
 	additional_env_files?: string[];
+	/**
+	 * Add additional config files either in repo or on host to track.
+	 * Can add any files associated with the stack to enable editing them in the UI.
+	 * Doing so will also include diffing these when deciding to deploy in `DeployStackIfChanged`.
+	 * Relative to the run directory.
+	 * 
+	 * Note. If the config file is .env and should be included in compose command
+	 * using `--env-file`, add it to `additional_env_files` instead.
+	 */
+	config_files?: StackFileDependency[];
 	/** Whether to send StackStateChange alerts for this stack. */
 	send_alerts: boolean;
 	/** Used with `registry_account` to login to a registry before docker compose up. */
@@ -2199,7 +2275,7 @@ export interface StackConfig {
 }
 
 export interface FileContents {
-	/** The path of the file on the host */
+	/** The path to the file */
 	path: string;
 	/** The contents of the file */
 	contents: string;
@@ -2231,9 +2307,27 @@ export interface StackServiceNames {
 	image?: string;
 }
 
+/**
+ * Same as [FileContents] with some extra
+ * info specific to Stacks.
+ */
+export interface StackRemoteFileContents {
+	/** The path to the file */
+	path: string;
+	/** The contents of the file */
+	contents: string;
+	/**
+	 * The services depending on this file,
+	 * or empty for global requirement (eg all compose files and env files).
+	 */
+	services?: string[];
+	/** Whether diff requires Redeploy / Restart / None */
+	requires?: StackFileRequires;
+}
+
 export interface StackInfo {
 	/**
-	 * If any of the expected files are missing in the repo,
+	 * If any of the expected compose / additional files are missing in the repo,
 	 * they will be stored here.
 	 */
 	missing_files?: string[];
@@ -2249,7 +2343,7 @@ export interface StackInfo {
 	/** Deployed commit message, or null. Only for repo based stacks */
 	deployed_message?: string;
 	/**
-	 * The deployed compose file contents.
+	 * The deployed compose / additional file contents.
 	 * This is updated whenever Komodo successfully deploys the stack.
 	 */
 	deployed_contents?: FileContents[];
@@ -2269,11 +2363,11 @@ export interface StackInfo {
 	 */
 	latest_services?: StackServiceNames[];
 	/**
-	 * The remote compose file contents, whether on host or in repo.
+	 * The remote compose / additional file contents, whether on host or in repo.
 	 * This is updated whenever Komodo refreshes the stack cache.
 	 * It will be empty if the file is defined directly in the stack config.
 	 */
-	remote_contents?: FileContents[];
+	remote_contents?: StackRemoteFileContents[];
 	/** If there was an error in getting the remote contents, it will be here. */
 	remote_errors?: FileContents[];
 	/** Latest commit hash, or null */
@@ -2308,6 +2402,15 @@ export interface SystemInformation {
 
 export type GetSystemInformationResponse = SystemInformation;
 
+export interface SystemLoadAverage {
+	/** 1m load average */
+	one: number;
+	/** 5m load average */
+	five: number;
+	/** 15m load average */
+	fifteen: number;
+}
+
 /** Info for a single disk mounted on the system. */
 export interface SingleDiskUsage {
 	/** The mount point of the disk */
@@ -2369,6 +2472,8 @@ export enum Timelength {
 export interface SystemStats {
 	/** Cpu usage percentage */
 	cpu_perc: number;
+	/** Load average (1m, 5m, 15m) */
+	load_average?: SystemLoadAverage;
 	/**
 	 * [1.15.9+]
 	 * Free memory in GB.
@@ -3760,6 +3865,8 @@ export interface ServerListItemInfo {
 	send_mem_alerts: boolean;
 	/** Whether server is configured to send disk alerts. */
 	send_disk_alerts: boolean;
+	/** Whether server is configured to send version mismatch alerts. */
+	send_version_mismatch_alerts: boolean;
 	/** Whether terminals are disabled for this Server. */
 	terminals_disabled: boolean;
 	/** Whether container exec is disabled for this Server. */
@@ -3967,6 +4074,11 @@ export interface StackQuerySpecifics {
 	 * Only accepts Server id (not name).
 	 */
 	server_ids?: string[];
+	/**
+	 * Query only for Stacks with these linked repos.
+	 * Only accepts Repo id (not name).
+	 */
+	linked_repos?: string[];
 	/** Filter syncs by their repo. */
 	repos?: string[];
 	/** Query only for Stack with available image updates. */
@@ -4107,7 +4219,7 @@ export interface AwsBuilderConfig {
  * timestamped database dumps, which can be restored using
  * the Komodo CLI.
  * 
- * TODO: Link to docs
+ * https://komo.do/docs/setup/backup
  */
 export interface BackupCoreDatabase {
 }
@@ -4119,7 +4231,7 @@ export interface BatchBuildRepo {
 	 * Supports multiline and comma delineated combinations of the above.
 	 * 
 	 * Example:
-	 * ```
+	 * ```text
 	 * # match all foo-* repos
 	 * foo-*
 	 * # add some more
@@ -4136,7 +4248,7 @@ export interface BatchCloneRepo {
 	 * Supports multiline and comma delineated combinations of the above.
 	 * 
 	 * Example:
-	 * ```
+	 * ```text
 	 * # match all foo-* repos
 	 * foo-*
 	 * # add some more
@@ -4153,7 +4265,7 @@ export interface BatchDeploy {
 	 * Supports multiline and comma delineated combinations of the above.
 	 * 
 	 * Example:
-	 * ```
+	 * ```text
 	 * # match all foo-* deployments
 	 * foo-*
 	 * # add some more
@@ -4170,7 +4282,7 @@ export interface BatchDeployStack {
 	 * Supports multiline and comma delineated combinations of the above.
 	 * 
 	 * Example:
-	 * ```
+	 * ```text
 	 * # match all foo-* stacks
 	 * foo-*
 	 * # add some more
@@ -4187,7 +4299,7 @@ export interface BatchDeployStackIfChanged {
 	 * Supports multiline and comma delineated combinations of the above.
 	 * 
 	 * Example:
-	 * ```
+	 * ```text
 	 * # match all foo-* stacks
 	 * foo-*
 	 * # add some more
@@ -4204,7 +4316,7 @@ export interface BatchDestroyDeployment {
 	 * Supports multiline and comma delineated combinations of the above.
 	 * 
 	 * Example:
-	 * ```
+	 * ```text
 	 * # match all foo-* deployments
 	 * foo-*
 	 * # add some more
@@ -4221,7 +4333,7 @@ export interface BatchDestroyStack {
 	 * Supports multiline and comma delineated combinations of the above.
 	 * d
 	 * Example:
-	 * ```
+	 * ```text
 	 * # match all foo-* stacks
 	 * foo-*
 	 * # add some more
@@ -4243,7 +4355,7 @@ export interface BatchPullRepo {
 	 * Supports multiline and comma delineated combinations of the above.
 	 * 
 	 * Example:
-	 * ```
+	 * ```text
 	 * # match all foo-* repos
 	 * foo-*
 	 * # add some more
@@ -4260,7 +4372,7 @@ export interface BatchPullStack {
 	 * Supports multiline and comma delineated combinations of the above.
 	 * 
 	 * Example:
-	 * ```
+	 * ```text
 	 * # match all foo-* stacks
 	 * foo-*
 	 * # add some more
@@ -4277,7 +4389,7 @@ export interface BatchRunAction {
 	 * Supports multiline and comma delineated combinations of the above.
 	 * 
 	 * Example:
-	 * ```
+	 * ```text
 	 * # match all foo-* actions
 	 * foo-*
 	 * # add some more
@@ -4294,7 +4406,7 @@ export interface BatchRunBuild {
 	 * Supports multiline and comma delineated combinations of the above.
 	 * 
 	 * Example:
-	 * ```
+	 * ```text
 	 * # match all foo-* builds
 	 * foo-*
 	 * # add some more
@@ -4311,7 +4423,7 @@ export interface BatchRunProcedure {
 	 * Supports multiline and comma delineated combinations of the above.
 	 * 
 	 * Example:
-	 * ```
+	 * ```text
 	 * # match all foo-* procedures
 	 * foo-*
 	 * # add some more
@@ -5996,6 +6108,8 @@ export interface SystemStatsRecord {
 	sid: string;
 	/** Cpu usage percentage */
 	cpu_perc: number;
+	/** Load average (1m, 5m, 15m) */
+	load_average?: SystemLoadAverage;
 	/** Memory used in GB */
 	mem_used_gb: number;
 	/** Total memory in GB */
@@ -6243,6 +6357,8 @@ export interface GetServersSummaryResponse {
 	total: I64;
 	/** The number of healthy (`status: OK`) servers. */
 	healthy: I64;
+	/** The number of servers with warnings (e.g., version mismatch). */
+	warning: I64;
 	/** The number of unhealthy servers. */
 	unhealthy: I64;
 	/** The number of disabled servers. */
@@ -6440,12 +6556,12 @@ export interface GetVersionResponse {
 }
 
 /**
- * Trigger a global poll for image updateson Stacks and Deployments
+ * Trigger a global poll for image updates on Stacks and Deployments
  * with `poll_for_updates` or `auto_update` enabled.
  * Admin only. Response: [Update]
  * 
  * 1. `docker compose pull` any Stacks / Deployments with `poll_for_updates` or `auto_update` enabled. This will pick up any available updates.
- * 2. Redeploy Stacks / Deployments that have updates found.
+ * 2. Redeploy Stacks / Deployments that have updates found and 'auto_update' enabled.
  */
 export interface GlobalAutoUpdate {
 }
@@ -6528,7 +6644,7 @@ export interface ListAlerts {
 	 * Pass a custom mongo query to filter the alerts.
 	 * 
 	 * ## Example JSON
-	 * ```
+	 * ```json
 	 * {
 	 * "resolved": "false",
 	 * "level": "CRITICAL",
@@ -7585,6 +7701,34 @@ export interface RunProcedure {
 	procedure: string;
 }
 
+/** Runs a one-time command against a service using `docker compose run`. Response: [Update] */
+export interface RunStackService {
+	/** Id or name */
+	stack: string;
+	/** Service to run */
+	service: string;
+	/** Command and args to pass to the service container */
+	command?: string[];
+	/** Do not allocate TTY */
+	no_tty?: boolean;
+	/** Do not start linked services */
+	no_deps?: boolean;
+	/** Detach container on run */
+	detach?: boolean;
+	/** Map service ports to the host */
+	service_ports?: boolean;
+	/** Extra environment variables for the run */
+	env?: Record<string, string>;
+	/** Working directory inside the container */
+	workdir?: string;
+	/** User to run as inside the container */
+	user?: string;
+	/** Override the default entrypoint */
+	entrypoint?: string;
+	/** Pull the image before running */
+	pull?: boolean;
+}
+
 /** Runs the target resource sync. Response: [Update] */
 export interface RunSync {
 	/** Id or name */
@@ -7686,6 +7830,22 @@ export interface SearchStackLog {
 	timestamps?: boolean;
 }
 
+/** Send a custom alert message to configured Alerters. Response: [Update] */
+export interface SendAlert {
+	/** The alert level. */
+	level?: SeverityLevel;
+	/** The alert message. Required. */
+	message: string;
+	/** The alert details. Optional. */
+	details?: string;
+	/**
+	 * Specific alerter names or ids.
+	 * If empty / not passed, sends to all configured alerters
+	 * with the `Custom` alert type whitelisted / not blacklisted.
+	 */
+	alerters?: string[];
+}
+
 /** Configuration for a Komodo Server Builder. */
 export interface ServerBuilderConfig {
 	/** The server id of the builder */
@@ -8363,6 +8523,7 @@ export type ExecuteRequest =
 	| { type: "UnpauseStack", params: UnpauseStack }
 	| { type: "DestroyStack", params: DestroyStack }
 	| { type: "BatchDestroyStack", params: BatchDestroyStack }
+	| { type: "RunStackService", params: RunStackService }
 	| { type: "Deploy", params: Deploy }
 	| { type: "BatchDeploy", params: BatchDeploy }
 	| { type: "PullDeployment", params: PullDeployment }
@@ -8388,6 +8549,7 @@ export type ExecuteRequest =
 	| { type: "RunAction", params: RunAction }
 	| { type: "BatchRunAction", params: BatchRunAction }
 	| { type: "TestAlerter", params: TestAlerter }
+	| { type: "SendAlert", params: SendAlert }
 	| { type: "RunSync", params: RunSync }
 	| { type: "ClearRepoCache", params: ClearRepoCache }
 	| { type: "BackupCoreDatabase", params: BackupCoreDatabase }
@@ -8479,13 +8641,6 @@ export enum IanaTimezone {
 	PacificKiritimati = "Pacific/Kiritimati",
 }
 
-/** Configuration for the registry to push the built image to. */
-export type ImageRegistryLegacy1_14 = 
-	/** Don't push the image to any registry */
-	| { type: "None", params: NoData }
-	/** Push the image to a standard image registry (any domain) */
-	| { type: "Standard", params: ImageRegistryConfig };
-
 export type ReadRequest = 
 	| { type: "GetVersion", params: GetVersion }
 	| { type: "GetCoreInfo", params: GetCoreInfo }

client/periphery/rs/src/api/build.rs

@@ -11,8 +11,10 @@ pub struct Build {
   pub build: komodo_client::entities::build::Build,
   /// Send the linked repo if it exists.
   pub repo: Option<Repo>,
-  /// Override registry token with one sent from core.
-  pub registry_token: Option<String>,
+  /// Override registry tokens with ones sent from core.
+  /// maps (domain, account) -> token.
+  #[serde(default)]
+  pub registry_tokens: Vec<(String, String, String)>,
   /// Propogate any secret replacers from core interpolation.
   #[serde(default)]
   pub replacers: Vec<(String, String)>,

client/periphery/rs/src/api/compose.rs

@@ -1,11 +1,15 @@
 use komodo_client::entities::{
   FileContents, RepoExecutionResponse, SearchCombinator,
   repo::Repo,
-  stack::{ComposeProject, Stack, StackServiceNames},
+  stack::{
+    ComposeProject, Stack, StackFileDependency,
+    StackRemoteFileContents, StackServiceNames,
+  },
   update::Log,
 };
 use resolver_api::Resolve;
 use serde::{Deserialize, Serialize};
+use std::collections::HashMap;
 
 /// List the compose project names that are on the host.
 /// List running `docker compose ls`
@@ -28,12 +32,13 @@ pub struct GetComposeContentsOnHost {
   /// The name of the stack
   pub name: String,
   pub run_directory: String,
-  pub file_paths: Vec<String>,
+  /// Both compose files and env / additional files, all relative to run directory.
+  pub file_paths: Vec<StackFileDependency>,
 }
 
 #[derive(Debug, Clone, Default, Serialize, Deserialize)]
 pub struct GetComposeContentsOnHostResponse {
-  pub contents: Vec<FileContents>,
+  pub contents: Vec<StackRemoteFileContents>,
   pub errors: Vec<FileContents>,
 }
 
@@ -91,7 +96,7 @@ pub struct GetComposeLogSearch {
 
 //
 
-/// Write the compose contents to the file on the host, for stacks using
+/// Write the compose / additional file contents to the file on the host, for stacks using
 /// `files_on_host`.
 #[derive(Debug, Clone, Serialize, Deserialize, Resolve)]
 #[response(Log)]
@@ -204,7 +209,7 @@ pub struct ComposeUpResponse {
   #[serde(default)]
   pub services: Vec<StackServiceNames>,
   /// The deploy compose file contents if they could be acquired, or empty vec.
-  pub file_contents: Vec<FileContents>,
+  pub file_contents: Vec<StackRemoteFileContents>,
   /// The error in getting remote file contents at the path, or null
   pub remote_errors: Vec<FileContents>,
   /// The output of `docker compose config` at deploy time
@@ -217,6 +222,14 @@ pub struct ComposeUpResponse {
 
 //
 
+#[derive(Debug, Clone, Default, Serialize, Deserialize)]
+pub struct ComposeRunResponse {
+  /// Logs produced during stack write/prepare for the run
+  pub logs: Vec<Log>,
+}
+
+//
+
 /// General compose command runner
 #[derive(Debug, Clone, Serialize, Deserialize, Resolve)]
 #[response(Log)]
@@ -228,3 +241,56 @@ pub struct ComposeExecution {
   /// The command in `docker compose -p {project} {command}`
   pub command: String,
 }
+
+//
+
+/// docker compose run one-time service execution.
+#[derive(Debug, Clone, Serialize, Deserialize, Resolve)]
+#[response(Log)]
+#[error(serror::Error)]
+pub struct ComposeRun {
+  /// The stack to run a service for
+  pub stack: Stack,
+  /// The linked repo, if it exists.
+  pub repo: Option<Repo>,
+  /// If provided, use it to login in. Otherwise check periphery local registries.
+  pub git_token: Option<String>,
+  /// If provided, use it to login in. Otherwise check periphery local git providers.
+  pub registry_token: Option<String>,
+  /// Propogate any secret replacers from core interpolation.
+  #[serde(default)]
+  pub replacers: Vec<(String, String)>,
+
+  /// Service to run
+  pub service: String,
+  /// Command
+  #[serde(default)]
+  pub command: Option<Vec<String>>,
+  /// Do not allocate TTY
+  #[serde(default)]
+  pub no_tty: Option<bool>,
+  /// Do not start linked services
+  #[serde(default)]
+  pub no_deps: Option<bool>,
+  /// Detach container on run
+  #[serde(default)]
+  pub detach: Option<bool>,
+  /// Map service ports to the host
+  #[serde(default)]
+  pub service_ports: Option<bool>,
+  /// Extra environment variables for the run
+  #[serde(default)]
+  pub env: Option<HashMap<String, String>>,
+  /// Working directory inside the container
+  #[serde(default)]
+  pub workdir: Option<String>,
+  /// User to run as inside the container
+  #[serde(default)]
+  pub user: Option<String>,
+  /// Override the default entrypoint
+  #[serde(default)]
+  pub entrypoint: Option<String>,
+  /// Pull the image before running
+  #[serde(default)]
+  pub pull: Option<bool>,
+}

client/periphery/rs/src/terminal.rs

@@ -44,7 +44,7 @@ impl PeripheryClient {
   /// sentinal value as the expected final line of the stream.
   ///
   /// Example final line:
-  /// ```
+  /// ```text
   /// __KOMODO_EXIT_CODE:0
   /// ```
   ///
@@ -104,7 +104,7 @@ impl PeripheryClient {
   /// sentinal value as the expected final line of the stream.
   ///
   /// Example final line:
-  /// ```
+  /// ```text
   /// __KOMODO_EXIT_CODE:0
   /// ```
   ///

compose/compose.env

@@ -9,6 +9,8 @@
 
 ## Stick to a specific version, or use `latest`
 COMPOSE_KOMODO_IMAGE_TAG=latest
+## Store dated database backups on the host - https://komo.do/docs/setup/backup
+COMPOSE_KOMODO_BACKUPS_PATH=/etc/komodo/backups
 
 ## DB credentials
 KOMODO_DB_USERNAME=admin

compose/ferretdb.compose.yaml

@@ -54,11 +54,13 @@ services:
       KOMODO_DATABASE_ADDRESS: ferretdb:27017
       KOMODO_DATABASE_USERNAME: ${KOMODO_DB_USERNAME}
       KOMODO_DATABASE_PASSWORD: ${KOMODO_DB_PASSWORD}
-    # volumes:
-    #   ## Store sync files on server
-    #   - /path/to/syncs:/syncs
-    #   ## Optionally mount a custom core.config.toml
-    #   - /path/to/core.config.toml:/config/config.toml
+    volumes:
+      ## Store dated backups of the database - https://komo.do/docs/setup/backup
+      - ${COMPOSE_KOMODO_BACKUPS_PATH}:/backups
+      ## Store sync files on server
+      # - /path/to/syncs:/syncs
+      ## Optionally mount a custom core.config.toml
+      # - /path/to/core.config.toml:/config/config.toml
     ## Allows for systemd Periphery connection at 
     ## "https://host.docker.internal:8120"
     # extra_hosts:
@@ -88,6 +90,4 @@ volumes:
   # Postgres
   postgres-data:
   # FerretDB
-  ferretdb-state:
-  # Core
-  repo-cache:
+  ferretdb-state:

compose/mongo.compose.yaml

@@ -37,11 +37,13 @@ services:
       KOMODO_DATABASE_ADDRESS: mongo:27017
       KOMODO_DATABASE_USERNAME: ${KOMODO_DB_USERNAME}
       KOMODO_DATABASE_PASSWORD: ${KOMODO_DB_PASSWORD}
-    # volumes:
-    #   ## Store sync files on server
-    #   - /path/to/syncs:/syncs
-    #   ## Optionally mount a custom core.config.toml
-    #   - /path/to/core.config.toml:/config/config.toml
+    volumes:
+      ## Store dated backups of the database - https://komo.do/docs/setup/backup
+      - ${COMPOSE_KOMODO_BACKUPS_PATH}:/backups
+      ## Store sync files on server
+      # - /path/to/syncs:/syncs
+      ## Optionally mount a custom core.config.toml
+      # - /path/to/core.config.toml:/config/config.toml
     ## Allows for systemd Periphery connection at 
     ## "https://host.docker.internal:8120"
     # extra_hosts:
@@ -70,6 +72,4 @@ services:
 volumes:
   # Mongo
   mongo-data:
-  mongo-config:
-  # Core
-  repo-cache:
+  mongo-config:

config/core.config.toml

@@ -53,7 +53,7 @@ passkey = "default-passkey-changeme"
 ## upon first startup. Example: `https://periphery:8120`
 ## Env: KOMODO_FIRST_SERVER
 ## Optional, no default.
-first_server = ""
+# first_server = ""
 
 ## Give the first server a custom name.
 ## Env: KOMODO_FIRST_SERVER_NAME

config/periphery.config.toml

@@ -75,10 +75,10 @@ disable_container_exec = false
 stats_polling_rate = "5-sec"
 
 ## How often Periphery polls the host for container stats,
-## Env: PERIPHERY_STATS_POLLING_RATE
+## Env: PERIPHERY_CONTAINER_STATS_POLLING_RATE
 ## Options: https://docs.rs/komodo_client/latest/komodo_client/entities/enum.Timelength.html
-## Default: 5-sec
-container_stats_polling_rate = "1-min"
+## Default: 30-sec
+container_stats_polling_rate = "30-sec"
 
 ## Whether stack actions should use `docker-compose ...`
 ## instead of `docker compose ...`.

docsite/docs/ecosystem/cli.mdx

@@ -75,14 +75,14 @@ in the merge ordering, meaning they are **lower priority** than those detected i
 You can also override these default paths by passing `km -c /path/to/1/base.config.yaml -c ./overrides ...`.
 
 If you want `km` to find configuration files in another directory,
-you can use make `.kminclude` file inside one of the configured directories.
+you can make a `.kminclude` file inside one of the configured directories.
 
 ```
 # Supports comments
 
-./.komodo    # relative to directory containing `kmignore`
+./.komodo    # relative to directory containing `.kminclude`
 
-/etc/komodo/komodo.cli.toml
+/etc/komodo/komodo.cli.toml  # also supports absolute path
 ```
 
 Note that wildcards in these paths are **not supported**.

docsite/docs/resources/deploy-containers/configuration.md

@@ -30,7 +30,7 @@ If you do select select a network other than host, you can specify port bindings
 
 In this case, you would access mongo from outside of the container on port `27018`.
 
-Note that this is not the only affect of using a network other than `host`. For example, containers running on different networks can not communicate, and ones on the same network can not reach other containers on `localhost` even when they are running on the same system. This behavior can be a bit confusing if you are not familiar with it, and it can be bypassed entirely by just using `host` network.
+Note that this is not the only effect of using a network other than `host`. For example, containers running on different networks can not communicate, and ones on the same network can not reach other containers on `localhost` even when they are running on the same system. This behavior can be a bit confusing if you are not familiar with it, and it can be bypassed entirely by just using `host` network.
 
 ## Configuring restart behavior
 
@@ -81,4 +81,4 @@ Sometimes you need to override the default command in the image, or specify some
 docker run -d --name mongo-db mongo:6.0.3 --quiet
 ```
 
-In order to achieve this with Komodo, just pass `--quiet` to 'command'.
+In order to achieve this with Komodo, just pass `--quiet` to 'command'.

docsite/docs/resources/permissioning.md

@@ -71,7 +71,7 @@ permissions = [
 
 ## Administration
 
-Users can be given Admin priviledges by a `Super Admin` (only the first user is given this status, set with `super_admin: true` on a User document in database). Super admins will see the "Make Admin" button when on a User page `/users/${user_id}`.
+Users can be given Admin privileges by a `Super Admin` (only the first user is given this status, set with `super_admin: true` on a User document in database). Super admins will see the "Make Admin" button when on a User page `/users/${user_id}`.
 
 These users have unrestricted access to all Komodo Resources. Additionally, these users can update other (non-admin) user's permissions on resources.
 
@@ -82,4 +82,4 @@ Users also have some configurable global permissions, these are:
  - create server permission
  - create build permission
 
-Only users with these permissions (as well as admins) can add additional servers to Komodo, and can create additional builds, respectively.
+Only users with these permissions (as well as admins) can add additional servers to Komodo, and can create additional builds, respectively.

docsite/docs/setup/advanced.mdx

@@ -14,6 +14,10 @@ Komodo also supports self hosted Oauth2 providers like [Authentik](https://docs.
 	- `<KOMODO_HOST>/auth/google/callback` for Google.
 	- `<KOMODO_HOST>/auth/oidc/callback` for OIDC.
 
+### Authentik
+
+Check out the [Authentik official support documentation](https://integrations.goauthentik.io/infrastructure/komodo/).
+
 ### Keycloak
 - Create an [OIDC client](https://www.keycloak.org/docs/latest/server_admin/index.html#proc-creating-oidc-client_server_administration_guide) in Keycloak.
   - Note down the `Client ID` that you enter (e.g.: "komodo"), you will need it for Komodo configuration

docsite/docs/setup/ferretdb.mdx

@@ -2,7 +2,7 @@
 
 :::info
 - If you setup Komodo using **Postgres** or **Sqlite** options prior to [**Komodo v1.18.0**](https://github.com/moghtech/komodo/releases/tag/v1.18.0), you are using **FerretDB v1**.
-- Komodo now uses **FerretDB v2**. For existing users, [**upgrading requires a migration**](https://github.com/moghtech/komodo/blob/main/bin/util/docs/copy-database.md#ferretdb-v2-update-guide).
+- Komodo now uses **FerretDB v2**. For existing users, [**upgrading requires a migration**](https://github.com/moghtech/komodo/blob/main/bin/cli/docs/copy-database.md#ferretdb-v2-update-guide).
 :::
 
 [**FerretDB**](https://www.ferretdb.com) is a MongoDB-compatible database backed by [Postgres + DocumentDB extension](https://github.com/microsoft/documentdb).

docsite/docs/setup/index.mdx

@@ -12,7 +12,7 @@ To run Komodo, you will need Docker. See [the docker install docs](https://docs.
 
 :::info
 **FerretDB v1** users:
-There is an [**upgrade guide for FerretDB v2** available here](https://github.com/moghtech/komodo/blob/main/bin/util/docs/copy-database.md#ferretdb-v2-update-guide).
+There is an [**upgrade guide for FerretDB v2** available here](https://github.com/moghtech/komodo/blob/main/bin/cli/docs/copy-database.md#ferretdb-v2-update-guide).
 :::
 
 ### First login

frontend/package.json

@@ -40,6 +40,7 @@
     "jotai": "2.12.5",
     "lucide-react": "0.511.0",
     "monaco-editor": "0.52.2",
+    "monaco-yaml": "5.4.0",
     "prettier": "3.5.3",
     "react": "19.1.0",
     "react-charts": "3.0.0-beta.57",
@@ -49,7 +50,8 @@
     "react-xtermjs": "1.0.10",
     "sanitize-html": "2.17.0",
     "tailwind-merge": "2.6.0",
-    "tailwindcss-animate": "1.0.7"
+    "tailwindcss-animate": "1.0.7",
+    "shell-quote": "1.8.1"
   },
   "devDependencies": {
     "@types/react": "19.1.6",

frontend/public/client/responses.d.ts

@@ -220,7 +220,7 @@ export type WriteResponses = {
     DeleteResourceSync: Types.ResourceSync;
     UpdateResourceSync: Types.ResourceSync;
     RenameResourceSync: Types.Update;
-    CommitSync: Types.ResourceSync;
+    CommitSync: Types.Update;
     WriteSyncFileContents: Types.Update;
     RefreshResourceSyncPending: Types.ResourceSync;
     CreateSyncWebhook: Types.CreateSyncWebhookResponse;
@@ -308,7 +308,9 @@ export type ExecuteResponses = {
     PauseStackService: Types.Update;
     UnpauseStackService: Types.Update;
     DestroyStackService: Types.Update;
+    RunStackService: Types.Update;
     TestAlerter: Types.Update;
+    SendAlert: Types.Update;
     ClearRepoCache: Types.Update;
     BackupCoreDatabase: Types.Update;
     GlobalAutoUpdate: Types.Update;

frontend/public/client/types.d.ts

@@ -71,13 +71,13 @@ export interface ActionConfig {
      * 1. Regular CRON expression:
      *
      * (second, minute, hour, day, month, day-of-week)
-     * ```
+     * ```text
      * 0 0 0 1,15 * ?
      * ```
      *
      * 2. "English" expression via [english-to-cron](https://crates.io/crates/english-to-cron):
      *
-     * ```
+     * ```text
      * at midnight on the 1st and 15th of the month
      * ```
      */
@@ -391,6 +391,7 @@ export declare enum Operation {
     UnpauseStack = "UnpauseStack",
     StopStack = "StopStack",
     DestroyStack = "DestroyStack",
+    RunStackService = "RunStackService",
     DeployStackService = "DeployStackService",
     PullStackService = "PullStackService",
     StartStackService = "StartStackService",
@@ -445,6 +446,7 @@ export declare enum Operation {
     RenameAlerter = "RenameAlerter",
     DeleteAlerter = "DeleteAlerter",
     TestAlerter = "TestAlerter",
+    SendAlert = "SendAlert",
     CreateResourceSync = "CreateResourceSync",
     UpdateResourceSync = "UpdateResourceSync",
     RenameResourceSync = "RenameResourceSync",
@@ -640,8 +642,11 @@ export interface BuildConfig {
     build_path: string;
     /** The path of the dockerfile relative to the build path. */
     dockerfile_path: string;
-    /** Configuration for the registry to push the built image to. */
-    image_registry?: ImageRegistryConfig;
+    /**
+     * Configuration for the registry/s to push the built image to.
+     * The first registry in this list will be used with attached Deployments.
+     */
+    image_registry?: ImageRegistryConfig[];
     /** Whether to skip secret interpolation in the build_args. */
     skip_secret_interp?: boolean;
     /** Whether to use buildx to build (eg `docker buildx build ...`) */
@@ -742,7 +747,7 @@ export interface BuildListItemInfo {
     built_hash?: string;
     /** Latest short commit hash, or null. Only for repo based stacks */
     latest_hash?: string;
-    /** The image registry domain */
+    /** The first listed image registry domain */
     image_registry_domain?: string;
 }
 export type BuildListItem = ResourceListItem<BuildListItemInfo>;
@@ -989,9 +994,15 @@ export type Execution =
 } | {
     type: "BatchDestroyStack";
     params: BatchDestroyStack;
+} | {
+    type: "RunStackService";
+    params: RunStackService;
 } | {
     type: "TestAlerter";
     params: TestAlerter;
+} | {
+    type: "SendAlert";
+    params: SendAlert;
 } | {
     type: "ClearRepoCache";
     params: ClearRepoCache;
@@ -1035,13 +1046,13 @@ export interface ProcedureConfig {
      * 1. Regular CRON expression:
      *
      * (second, minute, hour, day, month, day-of-week)
-     * ```
+     * ```text
      * 0 0 0 1,15 * ?
      * ```
      *
      * 2. "English" expression via [english-to-cron](https://crates.io/crates/english-to-cron):
      *
-     * ```
+     * ```text
      * at midnight on the 1st and 15th of the month
      * ```
      */
@@ -1453,11 +1464,23 @@ export type GetActionActionStateResponse = ActionActionState;
 export type GetActionResponse = Action;
 /** Severity level of problem. */
 export declare enum SeverityLevel {
-    /** No problem. */
+    /**
+     * No problem.
+     *
+     * Aliases: ok, low, l
+     */
     Ok = "OK",
-    /** Problem is imminent. */
+    /**
+     * Problem is imminent.
+     *
+     * Aliases: warning, w, medium, m
+     */
     Warning = "WARNING",
-    /** Problem fully realized. */
+    /**
+     * Problem fully realized.
+     *
+     * Aliases: critical, c, high, h
+     */
     Critical = "CRITICAL"
 }
 /** The variants of data related to the alert. */
@@ -1542,6 +1565,22 @@ export type AlertData =
         total_gb: number;
     };
 }
+/** A server has a version mismatch with the core. */
+ | {
+    type: "ServerVersionMismatch";
+    data: {
+        /** The id of the server */
+        id: string;
+        /** The name of the server */
+        name: string;
+        /** The region of the server */
+        region?: string;
+        /** The actual server version */
+        server_version: string;
+        /** The core version */
+        core_version: string;
+    };
+}
 /** A container's state has changed unexpectedly. */
  | {
     type: "ContainerStateChange";
@@ -1717,6 +1756,19 @@ export type AlertData =
         /** The resource name */
         name: string;
     };
+}
+/**
+ * Custom header / body.
+ * Produced using `/execute/SendAlert`
+ */
+ | {
+    type: "Custom";
+    data: {
+        /** The alert message. */
+        message: string;
+        /** Message details. May be empty string. */
+        details?: string;
+    };
 };
 /** Representation of an alert in the system. */
 export interface Alert {
@@ -2086,7 +2138,7 @@ export interface ServerConfig {
      * Whether a server is enabled.
      * If a server is disabled,
      * you won't be able to perform any actions on it or see deployment's status.
-     * default: true
+     * Default: false
      */
     enabled: boolean;
     /**
@@ -2125,6 +2177,8 @@ export interface ServerConfig {
     send_mem_alerts: boolean;
     /** Whether to send alerts about the servers DISK status */
     send_disk_alerts: boolean;
+    /** Whether to send alerts about the servers version mismatch with core */
+    send_version_mismatch_alerts: boolean;
     /** The percentage threshhold which triggers WARNING state for CPU. */
     cpu_warning: number;
     /** The percentage threshhold which triggers CRITICAL state for CPU. */
@@ -2154,6 +2208,23 @@ export interface StackActionState {
 }
 export type GetStackActionStateResponse = StackActionState;
 export type GetStackLogResponse = Log;
+export declare enum StackFileRequires {
+    /** Diff requires service redeploy. */
+    Redeploy = "Redeploy",
+    /** Diff requires service restart */
+    Restart = "Restart",
+    /** Diff requires no action. Default. */
+    None = "None"
+}
+/** Configure additional file dependencies of the Stack. */
+export interface StackFileDependency {
+    /** Specify the file */
+    path: string;
+    /** Specify specific service/s */
+    services?: string[];
+    /** Specify */
+    requires?: StackFileRequires;
+}
 /** The compose file configuration. */
 export interface StackConfig {
     /** The server to deploy the stack on. */
@@ -2268,8 +2339,21 @@ export interface StackConfig {
     /**
      * Add additional env files to attach with `--env-file`.
      * Relative to the run directory root.
+     *
+     * Note. It is already included as an `additional_file`.
+     * Don't add it again there.
      */
     additional_env_files?: string[];
+    /**
+     * Add additional config files either in repo or on host to track.
+     * Can add any files associated with the stack to enable editing them in the UI.
+     * Doing so will also include diffing these when deciding to deploy in `DeployStackIfChanged`.
+     * Relative to the run directory.
+     *
+     * Note. If the config file is .env and should be included in compose command
+     * using `--env-file`, add it to `additional_env_files` instead.
+     */
+    config_files?: StackFileDependency[];
     /** Whether to send StackStateChange alerts for this stack. */
     send_alerts: boolean;
     /** Used with `registry_account` to login to a registry before docker compose up. */
@@ -2314,7 +2398,7 @@ export interface StackConfig {
     environment?: string;
 }
 export interface FileContents {
-    /** The path of the file on the host */
+    /** The path to the file */
     path: string;
     /** The contents of the file */
     contents: string;
@@ -2344,9 +2428,26 @@ export interface StackServiceNames {
     /** The services image. */
     image?: string;
 }
+/**
+ * Same as [FileContents] with some extra
+ * info specific to Stacks.
+ */
+export interface StackRemoteFileContents {
+    /** The path to the file */
+    path: string;
+    /** The contents of the file */
+    contents: string;
+    /**
+     * The services depending on this file,
+     * or empty for global requirement (eg all compose files and env files).
+     */
+    services?: string[];
+    /** Whether diff requires Redeploy / Restart / None */
+    requires?: StackFileRequires;
+}
 export interface StackInfo {
     /**
-     * If any of the expected files are missing in the repo,
+     * If any of the expected compose / additional files are missing in the repo,
      * they will be stored here.
      */
     missing_files?: string[];
@@ -2362,7 +2463,7 @@ export interface StackInfo {
     /** Deployed commit message, or null. Only for repo based stacks */
     deployed_message?: string;
     /**
-     * The deployed compose file contents.
+     * The deployed compose / additional file contents.
      * This is updated whenever Komodo successfully deploys the stack.
      */
     deployed_contents?: FileContents[];
@@ -2382,11 +2483,11 @@ export interface StackInfo {
      */
     latest_services?: StackServiceNames[];
     /**
-     * The remote compose file contents, whether on host or in repo.
+     * The remote compose / additional file contents, whether on host or in repo.
      * This is updated whenever Komodo refreshes the stack cache.
      * It will be empty if the file is defined directly in the stack config.
      */
-    remote_contents?: FileContents[];
+    remote_contents?: StackRemoteFileContents[];
     /** If there was an error in getting the remote contents, it will be here. */
     remote_errors?: FileContents[];
     /** Latest commit hash, or null */
@@ -2416,6 +2517,14 @@ export interface SystemInformation {
     container_exec_disabled: boolean;
 }
 export type GetSystemInformationResponse = SystemInformation;
+export interface SystemLoadAverage {
+    /** 1m load average */
+    one: number;
+    /** 5m load average */
+    five: number;
+    /** 15m load average */
+    fifteen: number;
+}
 /** Info for a single disk mounted on the system. */
 export interface SingleDiskUsage {
     /** The mount point of the disk */
@@ -2475,6 +2584,8 @@ export declare enum Timelength {
 export interface SystemStats {
     /** Cpu usage percentage */
     cpu_perc: number;
+    /** Load average (1m, 5m, 15m) */
+    load_average?: SystemLoadAverage;
     /**
      * [1.15.9+]
      * Free memory in GB.
@@ -3731,6 +3842,8 @@ export interface ServerListItemInfo {
     send_mem_alerts: boolean;
     /** Whether server is configured to send disk alerts. */
     send_disk_alerts: boolean;
+    /** Whether server is configured to send version mismatch alerts. */
+    send_version_mismatch_alerts: boolean;
     /** Whether terminals are disabled for this Server. */
     terminals_disabled: boolean;
     /** Whether container exec is disabled for this Server. */
@@ -3903,6 +4016,11 @@ export interface StackQuerySpecifics {
      * Only accepts Server id (not name).
      */
     server_ids?: string[];
+    /**
+     * Query only for Stacks with these linked repos.
+     * Only accepts Repo id (not name).
+     */
+    linked_repos?: string[];
     /** Filter syncs by their repo. */
     repos?: string[];
     /** Query only for Stack with available image updates. */
@@ -4007,7 +4125,7 @@ export interface AwsBuilderConfig {
  * timestamped database dumps, which can be restored using
  * the Komodo CLI.
  *
- * TODO: Link to docs
+ * https://komo.do/docs/setup/backup
  */
 export interface BackupCoreDatabase {
 }
@@ -4018,7 +4136,7 @@ export interface BatchBuildRepo {
      * Supports multiline and comma delineated combinations of the above.
      *
      * Example:
-     * ```
+     * ```text
      * # match all foo-* repos
      * foo-*
      * # add some more
@@ -4034,7 +4152,7 @@ export interface BatchCloneRepo {
      * Supports multiline and comma delineated combinations of the above.
      *
      * Example:
-     * ```
+     * ```text
      * # match all foo-* repos
      * foo-*
      * # add some more
@@ -4050,7 +4168,7 @@ export interface BatchDeploy {
      * Supports multiline and comma delineated combinations of the above.
      *
      * Example:
-     * ```
+     * ```text
      * # match all foo-* deployments
      * foo-*
      * # add some more
@@ -4066,7 +4184,7 @@ export interface BatchDeployStack {
      * Supports multiline and comma delineated combinations of the above.
      *
      * Example:
-     * ```
+     * ```text
      * # match all foo-* stacks
      * foo-*
      * # add some more
@@ -4082,7 +4200,7 @@ export interface BatchDeployStackIfChanged {
      * Supports multiline and comma delineated combinations of the above.
      *
      * Example:
-     * ```
+     * ```text
      * # match all foo-* stacks
      * foo-*
      * # add some more
@@ -4098,7 +4216,7 @@ export interface BatchDestroyDeployment {
      * Supports multiline and comma delineated combinations of the above.
      *
      * Example:
-     * ```
+     * ```text
      * # match all foo-* deployments
      * foo-*
      * # add some more
@@ -4114,7 +4232,7 @@ export interface BatchDestroyStack {
      * Supports multiline and comma delineated combinations of the above.
      * d
      * Example:
-     * ```
+     * ```text
      * # match all foo-* stacks
      * foo-*
      * # add some more
@@ -4134,7 +4252,7 @@ export interface BatchPullRepo {
      * Supports multiline and comma delineated combinations of the above.
      *
      * Example:
-     * ```
+     * ```text
      * # match all foo-* repos
      * foo-*
      * # add some more
@@ -4150,7 +4268,7 @@ export interface BatchPullStack {
      * Supports multiline and comma delineated combinations of the above.
      *
      * Example:
-     * ```
+     * ```text
      * # match all foo-* stacks
      * foo-*
      * # add some more
@@ -4166,7 +4284,7 @@ export interface BatchRunAction {
      * Supports multiline and comma delineated combinations of the above.
      *
      * Example:
-     * ```
+     * ```text
      * # match all foo-* actions
      * foo-*
      * # add some more
@@ -4182,7 +4300,7 @@ export interface BatchRunBuild {
      * Supports multiline and comma delineated combinations of the above.
      *
      * Example:
-     * ```
+     * ```text
      * # match all foo-* builds
      * foo-*
      * # add some more
@@ -4198,7 +4316,7 @@ export interface BatchRunProcedure {
      * Supports multiline and comma delineated combinations of the above.
      *
      * Example:
-     * ```
+     * ```text
      * # match all foo-* procedures
      * foo-*
      * # add some more
@@ -5744,6 +5862,8 @@ export interface SystemStatsRecord {
     sid: string;
     /** Cpu usage percentage */
     cpu_perc: number;
+    /** Load average (1m, 5m, 15m) */
+    load_average?: SystemLoadAverage;
     /** Memory used in GB */
     mem_used_gb: number;
     /** Total memory in GB */
@@ -5963,6 +6083,8 @@ export interface GetServersSummaryResponse {
     total: I64;
     /** The number of healthy (`status: OK`) servers. */
     healthy: I64;
+    /** The number of servers with warnings (e.g., version mismatch). */
+    warning: I64;
     /** The number of unhealthy servers. */
     unhealthy: I64;
     /** The number of disabled servers. */
@@ -6139,12 +6261,12 @@ export interface GetVersionResponse {
     version: string;
 }
 /**
- * Trigger a global poll for image updateson Stacks and Deployments
+ * Trigger a global poll for image updates on Stacks and Deployments
  * with `poll_for_updates` or `auto_update` enabled.
  * Admin only. Response: [Update]
  *
  * 1. `docker compose pull` any Stacks / Deployments with `poll_for_updates` or `auto_update` enabled. This will pick up any available updates.
- * 2. Redeploy Stacks / Deployments that have updates found.
+ * 2. Redeploy Stacks / Deployments that have updates found and 'auto_update' enabled.
  */
 export interface GlobalAutoUpdate {
 }
@@ -6217,7 +6339,7 @@ export interface ListAlerts {
      * Pass a custom mongo query to filter the alerts.
      *
      * ## Example JSON
-     * ```
+     * ```json
      * {
      * "resolved": "false",
      * "level": "CRITICAL",
@@ -7172,6 +7294,33 @@ export interface RunProcedure {
     /** Id or name */
     procedure: string;
 }
+/** Runs a one-time command against a service using `docker compose run`. Response: [Update] */
+export interface RunStackService {
+    /** Id or name */
+    stack: string;
+    /** Service to run */
+    service: string;
+    /** Command and args to pass to the service container */
+    command?: string[];
+    /** Do not allocate TTY */
+    no_tty?: boolean;
+    /** Do not start linked services */
+    no_deps?: boolean;
+    /** Detach container on run */
+    detach?: boolean;
+    /** Map service ports to the host */
+    service_ports?: boolean;
+    /** Extra environment variables for the run */
+    env?: Record<string, string>;
+    /** Working directory inside the container */
+    workdir?: string;
+    /** User to run as inside the container */
+    user?: string;
+    /** Override the default entrypoint */
+    entrypoint?: string;
+    /** Pull the image before running */
+    pull?: boolean;
+}
 /** Runs the target resource sync. Response: [Update] */
 export interface RunSync {
     /** Id or name */
@@ -7268,6 +7417,21 @@ export interface SearchStackLog {
     /** Enable `--timestamps` */
     timestamps?: boolean;
 }
+/** Send a custom alert message to configured Alerters. Response: [Update] */
+export interface SendAlert {
+    /** The alert level. */
+    level?: SeverityLevel;
+    /** The alert message. Required. */
+    message: string;
+    /** The alert details. Optional. */
+    details?: string;
+    /**
+     * Specific alerter names or ids.
+     * If empty / not passed, sends to all configured alerters
+     * with the `Custom` alert type whitelisted / not blacklisted.
+     */
+    alerters?: string[];
+}
 /** Configuration for a Komodo Server Builder. */
 export interface ServerBuilderConfig {
     /** The server id of the builder */
@@ -7967,6 +8131,9 @@ export type ExecuteRequest = {
 } | {
     type: "BatchDestroyStack";
     params: BatchDestroyStack;
+} | {
+    type: "RunStackService";
+    params: RunStackService;
 } | {
     type: "Deploy";
     params: Deploy;
@@ -8042,6 +8209,9 @@ export type ExecuteRequest = {
 } | {
     type: "TestAlerter";
     params: TestAlerter;
+} | {
+    type: "SendAlert";
+    params: SendAlert;
 } | {
     type: "RunSync";
     params: RunSync;
@@ -8140,18 +8310,6 @@ export declare enum IanaTimezone {
     /** UTC+14:00 */
     PacificKiritimati = "Pacific/Kiritimati"
 }
-/** Configuration for the registry to push the built image to. */
-export type ImageRegistryLegacy1_14 = 
-/** Don't push the image to any registry */
-{
-    type: "None";
-    params: NoData;
-}
-/** Push the image to a standard image registry (any domain) */
- | {
-    type: "Standard";
-    params: ImageRegistryConfig;
-};
 export type ReadRequest = {
     type: "GetVersion";
     params: GetVersion;

frontend/public/client/types.js

@@ -105,6 +105,7 @@ export var Operation;
     Operation["UnpauseStack"] = "UnpauseStack";
     Operation["StopStack"] = "StopStack";
     Operation["DestroyStack"] = "DestroyStack";
+    Operation["RunStackService"] = "RunStackService";
     Operation["DeployStackService"] = "DeployStackService";
     Operation["PullStackService"] = "PullStackService";
     Operation["StartStackService"] = "StartStackService";
@@ -159,6 +160,7 @@ export var Operation;
     Operation["RenameAlerter"] = "RenameAlerter";
     Operation["DeleteAlerter"] = "DeleteAlerter";
     Operation["TestAlerter"] = "TestAlerter";
+    Operation["SendAlert"] = "SendAlert";
     Operation["CreateResourceSync"] = "CreateResourceSync";
     Operation["UpdateResourceSync"] = "UpdateResourceSync";
     Operation["RenameResourceSync"] = "RenameResourceSync";
@@ -248,13 +250,34 @@ export var DeploymentState;
 /** Severity level of problem. */
 export var SeverityLevel;
 (function (SeverityLevel) {
-    /** No problem. */
+    /**
+     * No problem.
+     *
+     * Aliases: ok, low, l
+     */
     SeverityLevel["Ok"] = "OK";
-    /** Problem is imminent. */
+    /**
+     * Problem is imminent.
+     *
+     * Aliases: warning, w, medium, m
+     */
     SeverityLevel["Warning"] = "WARNING";
-    /** Problem fully realized. */
+    /**
+     * Problem fully realized.
+     *
+     * Aliases: critical, c, high, h
+     */
     SeverityLevel["Critical"] = "CRITICAL";
 })(SeverityLevel || (SeverityLevel = {}));
+export var StackFileRequires;
+(function (StackFileRequires) {
+    /** Diff requires service redeploy. */
+    StackFileRequires["Redeploy"] = "Redeploy";
+    /** Diff requires service restart */
+    StackFileRequires["Restart"] = "Restart";
+    /** Diff requires no action. Default. */
+    StackFileRequires["None"] = "None";
+})(StackFileRequires || (StackFileRequires = {}));
 export var Timelength;
 (function (Timelength) {
     /** `1-sec` */