fix multi arch built var reference

* inc version

* Komodo interp in ui compose file

* fix auto update when image doesn't specify tag by defaulting to latest

* Pull image buttons don't need safety dialog

* WIP crosscompile

* rename

* entrypoint

* fix copy

* remove example/* from workspace

* add targets

* multiarch pkg config

* use specific COPY

* update deps

* multiarch build command

* pre compile deps

* cross compile

* enable-linger

* remove spammed log when server doesn't have docker

* add multiarch.Dockerfile

* fix casing

* fix tag

* try not let COPY fail

* try

* ARG TARGETPLATFORM

* use /app for consistency

* try

* delete cross-compile approach

* add multiarch core build

* multiarch Deno

* single arch multi arch

* typeshare cli note

* new typeshare

* remove note about aarch64 image

* test configs

* fix config file headers

* binaries dockerfile

* update cargo build

* docs

* simple

* just simple

* use -p

* add configurable binaries tag

* add multi-arch

* allow copy to fail

* fix binary paths

* frontend Dockerfiel

* use dedicated static frontend build

* auto retry getting instance state from aws

* retry 5 times

* cleanup

* simplify binary build

* try alpine and musl

* install alpine deps

* back to debian, try rustls

* move fully to rustls

* single arch builds using single binary image

* default IMAGE_TAG

* cleanup

* try caching deps

* single arch add frontend build

* rustls::crypto::ring::default_provider()

* back to simple

* comment dockerfile

* add select options prop, render checkboxes if present

* add allowSelectedIf to enable / disable rows where necessary

* rename allowSelectIf to isSelectable, allow false as global disable, disable checkboxes when not allowed

* rename isSelectable to disableRow (it works the oppsite way lol)

* selected resources hook, start deployment batch execute component

* add deployment group actions

* add deployment group actions

* add default (empty) group actions for other resources

* fix checkbox header styles

* explicitly check if disableRow is passed (this prop is cursed)

* don't disable row selection for deployments table

* don't need id for groupactions

* add group actions to resources page

* fix row checkbox (prop not cursed, i dumb)

* re-implement group action list using dropdown menu

* only make group actions clickable when at least one row selected

* add loading indicator

* gap betwen new resource and group actions

* refactor group actions

* remove "Batch" from action labels

* add group actions for relevant resources

* fix hardcode

* add selectOptions to relevant tables

* select by name not id

* expect selected to be names

* add note re selection state init for future reference

* multi select working nicely for all resources

* configure server health check timeout

* config message

* refresh processes remove dead processes

* simplify the build args

* default timeout seconds 3

---------

Co-authored-by: kv <karamvir.singh98@gmail.com>

.devcontainer/dev.compose.yaml

@@ -0,0 +1,33 @@
+services:
+  dev:
+    image: mcr.microsoft.com/devcontainers/rust:1-1-bullseye
+    volumes:
+      # Mount the root folder that contains .git
+      - ../:/workspace:cached
+      - /var/run/docker.sock:/var/run/docker.sock
+      - /proc:/proc
+      - repos:/etc/komodo/repos
+      - stacks:/etc/komodo/stacks
+    command: sleep infinity
+    ports:
+      - "9121:9121"
+    environment:
+      KOMODO_FIRST_SERVER: http://localhost:8120
+      KOMODO_DATABASE_ADDRESS: db
+      KOMODO_ENABLE_NEW_USERS: true
+      KOMODO_LOCAL_AUTH: true
+      KOMODO_JWT_SECRET: a_random_secret
+    links:
+      - db
+    # ...
+
+  db:
+    extends:
+      file: ../test.compose.yaml
+      service: ferretdb
+
+volumes:
+  data:
+  repo-cache:
+  repos:
+  stacks:

.devcontainer/devcontainer.json

@@ -0,0 +1,46 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the
+// README at: https://github.com/devcontainers/templates/tree/main/src/rust
+{
+	"name": "Komodo",
+	// Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile
+	//"image": "mcr.microsoft.com/devcontainers/rust:1-1-bullseye",
+	"dockerComposeFile": ["dev.compose.yaml"],
+	"workspaceFolder": "/workspace",
+  	"service": "dev",
+	// Features to add to the dev container. More info: https://containers.dev/features.
+	"features": {
+		"ghcr.io/devcontainers/features/node:1": {
+			"version": "18.18.0"
+		},
+		"ghcr.io/devcontainers-community/features/deno:1": {
+
+		}
+	},
+
+	// Use 'mounts' to make the cargo cache persistent in a Docker Volume.
+	"mounts": [
+		{
+			"source": "devcontainer-cargo-cache-${devcontainerId}",
+			"target": "/usr/local/cargo",
+			"type": "volume"
+		}
+	],
+
+	// Use 'forwardPorts' to make a list of ports inside the container available locally.
+	"forwardPorts": [
+		9121
+	],
+
+	// Use 'postCreateCommand' to run commands after the container is created.
+	"postCreateCommand": "./.devcontainer/postCreate.sh",
+
+	"runServices": [
+		"db"
+	]
+
+	// Configure tool-specific properties.
+	// "customizations": {},
+
+	// Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root.
+	// "remoteUser": "root"
+}

.devcontainer/postCreate.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+cargo install typeshare-cli

.dockerignore

@@ -5,4 +5,10 @@ LICENSE
 *.code-workspace
 
 */node_modules
-*/dist
+*/dist
+
+creds.toml
+.core-repos
+.repos
+.stacks
+.ssl

.gitignore

@@ -1,11 +1,11 @@
 target
 /frontend/build
-node_modules
 /lib/ts_client/build
+node_modules
 dist
 .env
 .env.development
+.DS_Store
+
 creds.toml
-.syncs
-.stacks
-.DS_Store
+.komodo

.vscode/extensions.json

@@ -0,0 +1,8 @@
+{
+    "recommendations": [
+        "rust-lang.rust-analyzer",
+        "tamasfe.even-better-toml",
+        "vadimcn.vscode-lldb",
+        "denoland.vscode-deno"
+    ]
+}

.vscode/tasks.json

@@ -0,0 +1,179 @@
+{
+    "version": "2.0.0",
+    "tasks": [
+        {
+            "label": "Run Core",
+            "command": "cargo",
+            "args": [
+                "run",
+                "-p",
+                "komodo_core",
+                "--release"
+            ],
+            "options": {
+                "cwd": "${workspaceFolder}",
+                "env": {
+                    "KOMODO_CONFIG_PATH": "test.core.config.toml"
+                }
+            },
+            "problemMatcher": [
+                "$rustc"
+            ]
+        },
+        {
+            "label": "Build Core",
+            "command": "cargo",
+            "args": [
+                "build",
+                "-p",
+                "komodo_core",
+                "--release"
+            ],
+            "options": {
+                "cwd": "${workspaceFolder}",
+                "env": {
+                    "KOMODO_CONFIG_PATH": "test.core.config.toml"
+                }
+            },
+            "problemMatcher": [
+                "$rustc"
+            ]
+        },
+        {
+            "label": "Run Periphery",
+            "command": "cargo",
+            "args": [
+                "run",
+                "-p",
+                "komodo_periphery",
+                "--release"
+            ],
+            "options": {
+                "cwd": "${workspaceFolder}",
+                "env": {
+                    "KOMODO_CONFIG_PATH": "test.periphery.config.toml"
+                }
+            },
+            "problemMatcher": [
+                "$rustc"
+            ]
+        },
+        {
+            "label": "Build Periphery",
+            "command": "cargo",
+            "args": [
+                "build",
+                "-p",
+                "komodo_periphery",
+                "--release"
+            ],
+            "options": {
+                "cwd": "${workspaceFolder}",
+                "env": {
+                    "KOMODO_CONFIG_PATH": "test.periphery.config.toml"
+                }
+            },
+            "problemMatcher": [
+                "$rustc"
+            ]
+        },
+        {
+            "label": "Run Backend",
+            "dependsOn": [
+                "Run Core",
+                "Run Periphery"
+            ],
+            "problemMatcher": [
+                "$rustc"
+            ]
+        },
+        {
+            "label": "Build TS Client Types",
+            "type": "process",
+            "command": "node",
+            "args": [
+                "./client/core/ts/generate_types.mjs"
+            ],
+            "problemMatcher": []
+        },
+        {
+            "label": "Init TS Client",
+            "type": "shell",
+            "command": "yarn && yarn build && yarn link",
+            "options": {
+                "cwd": "${workspaceFolder}/client/core/ts",
+            },
+            "problemMatcher": []
+        },
+        {
+            "label": "Init Frontend Client",
+            "type": "shell",
+            "command": "yarn link komodo_client && yarn install",
+            "options": {
+                "cwd": "${workspaceFolder}/frontend",
+            },
+            "problemMatcher": []
+        },
+        {
+            "label": "Init Frontend",
+            "dependsOn": [
+                "Build TS Client Types",
+                "Init TS Client",
+                "Init Frontend Client"
+            ],
+            "dependsOrder": "sequence",
+            "problemMatcher": []
+        },
+        {
+            "label": "Build Frontend",
+            "type": "shell",
+            "command": "yarn build",
+            "options": {
+                "cwd": "${workspaceFolder}/frontend",
+            },
+            "problemMatcher": []
+        },
+        {
+            "label": "Prepare Frontend For Run",
+            "type": "shell",
+            "command": "cp -r ./client/core/ts/dist/. frontend/public/client/.",
+            "options": {
+                "cwd": "${workspaceFolder}",
+            },
+            "dependsOn": [
+                "Build TS Client Types",
+                "Build Frontend"
+            ],
+            "dependsOrder": "sequence",
+            "problemMatcher": []
+        },
+        {
+            "label": "Run Frontend",
+            "type": "shell",
+            "command": "yarn dev",
+            "options": {
+                "cwd": "${workspaceFolder}/frontend",
+            },
+            "dependsOn": ["Prepare Frontend For Run"],
+            "problemMatcher": []
+        },
+        {
+            "label": "Init",
+            "dependsOn": [
+                "Build Backend",
+                "Init Frontend"
+            ],
+            "dependsOrder": "sequence",
+            "problemMatcher": []
+        },
+        {
+            "label": "Run Komodo",
+            "dependsOn": [
+                "Run Core",
+                "Run Periphery",
+                "Run Frontend"
+            ],
+            "problemMatcher": []
+        },
+    ]
+  }

Cargo.toml

@@ -1,9 +1,14 @@
 [workspace]
 resolver = "2"
-members = ["bin/*", "lib/*", "client/core/rs", "client/periphery/rs"]
+members = [
+	"bin/*",
+	"lib/*",
+	"client/core/rs",
+	"client/periphery/rs",
+]
 
 [workspace.package]
-version = "1.15.2"
+version = "1.16.12"
 edition = "2021"
 authors = ["mbecker20 <becker.maxh@gmail.com>"]
 license = "GPL-3.0-or-later"
@@ -15,19 +20,20 @@ homepage = "https://komo.do"
 
 [workspace.dependencies]
 # LOCAL
-# komodo_client = "1.14.3"
+# komodo_client = "1.15.6"
 komodo_client = { path = "client/core/rs" }
 periphery_client = { path = "client/periphery/rs" }
 environment_file = { path = "lib/environment_file" }
 formatting = { path = "lib/formatting" }
 command = { path = "lib/command" }
 logger = { path = "lib/logger" }
+cache = { path = "lib/cache" }
 git = { path = "lib/git" }
 
 # MOGH
 run_command = { version = "0.0.6", features = ["async_tokio"] }
-serror = { version = "0.4.6", default-features = false }
-slack = { version = "0.2.0", package = "slack_client_rs" }
+serror = { version = "0.4.7", default-features = false }
+slack = { version = "0.3.0", package = "slack_client_rs", default-features = false, features = ["rustls"] }
 derive_default_builder = "0.1.8"
 derive_empty_traits = "0.1.0"
 merge_config_files = "0.1.5"
@@ -41,52 +47,53 @@ mungos = "1.1.0"
 svi = "1.0.1"
 
 # ASYNC
-reqwest = { version = "0.12.8", features = ["json"] }
-tokio = { version = "1.38.1", features = ["full"] }
+reqwest = { version = "0.12.9", default-features = false, features = ["json", "rustls-tls"] }
+tokio = { version = "1.41.1", features = ["full"] }
 tokio-util = "0.7.12"
-futures = "0.3.30"
-futures-util = "0.3.30"
+futures = "0.3.31"
+futures-util = "0.3.31"
 
 # SERVER
-axum-extra = { version = "0.9.4", features = ["typed-header"] }
-tower-http = { version = "0.6.1", features = ["fs", "cors"] }
-axum-server = { version = "0.7.1", features = ["tls-openssl"] }
-axum = { version = "0.7.7", features = ["ws", "json"] }
+axum-extra = { version = "0.9.6", features = ["typed-header"] }
+tower-http = { version = "0.6.2", features = ["fs", "cors"] }
+axum-server = { version = "0.7.1", features = ["tls-rustls"] }
+axum = { version = "0.7.9", features = ["ws", "json"] }
 tokio-tungstenite = "0.24.0"
 
 # SER/DE
 ordered_hash_map = { version = "0.4.0", features = ["serde"] }
-serde = { version = "1.0.210", features = ["derive"] }
+serde = { version = "1.0.215", features = ["derive"] }
 strum = { version = "0.26.3", features = ["derive"] }
-serde_json = "1.0.128"
+serde_json = "1.0.133"
 serde_yaml = "0.9.34"
 toml = "0.8.19"
 
 # ERROR
-anyhow = "1.0.89"
-thiserror = "1.0.64"
+anyhow = "1.0.93"
+thiserror = "2.0.3"
 
 # LOGGING
-opentelemetry_sdk = { version = "0.25.0", features = ["rt-tokio"] }
+opentelemetry-otlp = { version = "0.27.0", features = ["tls-roots", "reqwest-rustls"] }
+opentelemetry_sdk = { version = "0.27.0", features = ["rt-tokio"] }
 tracing-subscriber = { version = "0.3.18", features = ["json"] }
-opentelemetry-semantic-conventions = "0.25.0"
-tracing-opentelemetry = "0.26.0"
-opentelemetry-otlp = "0.25.0"
-opentelemetry = "0.25.0"
+opentelemetry-semantic-conventions = "0.27.0"
+tracing-opentelemetry = "0.28.0"
+opentelemetry = "0.27.0"
 tracing = "0.1.40"
 
 # CONFIG
-clap = { version = "4.5.19", features = ["derive"] }
+clap = { version = "4.5.21", features = ["derive"] }
 dotenvy = "0.15.7"
 envy = "0.4.2"
 
 # CRYPTO / AUTH
-uuid = { version = "1.10.0", features = ["v4", "fast-rng", "serde"] }
+uuid = { version = "1.11.0", features = ["v4", "fast-rng", "serde"] }
 openidconnect = "3.5.0"
 urlencoding = "2.1.3"
 nom_pem = "4.0.0"
-bcrypt = "0.15.1"
+bcrypt = "0.16.0"
 base64 = "0.22.1"
+rustls = "0.23.18"
 hmac = "0.12.1"
 sha2 = "0.10.8"
 rand = "0.8.5"
@@ -94,18 +101,19 @@ jwt = "0.16.0"
 hex = "0.4.3"
 
 # SYSTEM
-bollard = "0.17.1"
-sysinfo = "0.31.4"
+bollard = "0.18.1"
+sysinfo = "0.32.0"
 
 # CLOUD
-aws-config = "1.5.7"
-aws-sdk-ec2 = "1.75.0"
+aws-config = "1.5.10"
+aws-sdk-ec2 = "1.91.0"
 
 # MISC
-derive_builder = "0.20.1"
-typeshare = "1.0.3"
+derive_builder = "0.20.2"
+typeshare = "1.0.4"
 octorust = "0.7.0"
 dashmap = "6.1.0"
+wildcard = "0.3.0"
 colored = "2.1.0"
-regex = "1.11.0"
-bson = "2.13.0"
+regex = "1.11.1"
+bson = "2.13.0"

bin/binaries.Dockerfile

@@ -0,0 +1,27 @@
+## Builds the Komodo Core and Periphery binaries
+## for a specific architecture.
+
+FROM rust:1.82.0-bullseye AS builder
+
+WORKDIR /builder
+COPY Cargo.toml Cargo.lock ./
+COPY ./lib ./lib
+COPY ./client/core/rs ./client/core/rs
+COPY ./client/periphery ./client/periphery
+COPY ./bin/core ./bin/core
+COPY ./bin/periphery ./bin/periphery
+
+# Compile bin
+RUN \
+  cargo build -p komodo_core --release && \
+  cargo build -p komodo_periphery --release
+
+# Copy just the binaries to scratch image
+FROM scratch
+
+COPY --from=builder /builder/target/release/core /core
+COPY --from=builder /builder/target/release/periphery /periphery
+
+LABEL org.opencontainers.image.source=https://github.com/mbecker20/komodo
+LABEL org.opencontainers.image.description="Komodo Periphery"
+LABEL org.opencontainers.image.licenses=GPL-3.0

bin/cli/src/exec.rs

@@ -1,13 +1,21 @@
 use std::time::Duration;
 
 use colored::Colorize;
-use komodo_client::api::execute::Execution;
+use komodo_client::{
+  api::execute::{BatchExecutionResponse, Execution},
+  entities::update::Update,
+};
 
 use crate::{
   helpers::wait_for_enter,
   state::{cli_args, komodo_client},
 };
 
+pub enum ExecutionResult {
+  Single(Update),
+  Batch(BatchExecutionResponse),
+}
+
 pub async fn run(execution: Execution) -> anyhow::Result<()> {
   if matches!(execution, Execution::None(_)) {
     println!("Got 'none' execution. Doing nothing...");
@@ -21,18 +29,36 @@ pub async fn run(execution: Execution) -> anyhow::Result<()> {
     Execution::None(data) => {
       println!("{}: {data:?}", "Data".dimmed())
     }
+    Execution::RunAction(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
+    Execution::BatchRunAction(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
     Execution::RunProcedure(data) => {
       println!("{}: {data:?}", "Data".dimmed())
     }
+    Execution::BatchRunProcedure(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
     Execution::RunBuild(data) => {
       println!("{}: {data:?}", "Data".dimmed())
     }
+    Execution::BatchRunBuild(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
     Execution::CancelBuild(data) => {
       println!("{}: {data:?}", "Data".dimmed())
     }
     Execution::Deploy(data) => {
       println!("{}: {data:?}", "Data".dimmed())
     }
+    Execution::BatchDeploy(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
+    Execution::PullDeployment(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
     Execution::StartDeployment(data) => {
       println!("{}: {data:?}", "Data".dimmed())
     }
@@ -51,15 +77,27 @@ pub async fn run(execution: Execution) -> anyhow::Result<()> {
     Execution::DestroyDeployment(data) => {
       println!("{}: {data:?}", "Data".dimmed())
     }
+    Execution::BatchDestroyDeployment(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
     Execution::CloneRepo(data) => {
       println!("{}: {data:?}", "Data".dimmed())
     }
+    Execution::BatchCloneRepo(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
     Execution::PullRepo(data) => {
       println!("{}: {data:?}", "Data".dimmed())
     }
+    Execution::BatchPullRepo(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
     Execution::BuildRepo(data) => {
       println!("{}: {data:?}", "Data".dimmed())
     }
+    Execution::BatchBuildRepo(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
     Execution::CancelRepoBuild(data) => {
       println!("{}: {data:?}", "Data".dimmed())
     }
@@ -129,9 +167,24 @@ pub async fn run(execution: Execution) -> anyhow::Result<()> {
     Execution::RunSync(data) => {
       println!("{}: {data:?}", "Data".dimmed())
     }
+    Execution::CommitSync(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
     Execution::DeployStack(data) => {
       println!("{}: {data:?}", "Data".dimmed())
     }
+    Execution::BatchDeployStack(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
+    Execution::DeployStackIfChanged(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
+    Execution::BatchDeployStackIfChanged(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
+    Execution::PullStack(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
     Execution::StartStack(data) => {
       println!("{}: {data:?}", "Data".dimmed())
     }
@@ -150,6 +203,9 @@ pub async fn run(execution: Execution) -> anyhow::Result<()> {
     Execution::DestroyStack(data) => {
       println!("{}: {data:?}", "Data".dimmed())
     }
+    Execution::BatchDestroyStack(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
     Execution::Sleep(data) => {
       println!("{}: {data:?}", "Data".dimmed())
     }
@@ -162,135 +218,242 @@ pub async fn run(execution: Execution) -> anyhow::Result<()> {
   info!("Running Execution...");
 
   let res = match execution {
-    Execution::RunProcedure(request) => {
-      komodo_client().execute(request).await
-    }
-    Execution::RunBuild(request) => {
-      komodo_client().execute(request).await
-    }
-    Execution::CancelBuild(request) => {
-      komodo_client().execute(request).await
-    }
-    Execution::Deploy(request) => {
-      komodo_client().execute(request).await
-    }
-    Execution::StartDeployment(request) => {
-      komodo_client().execute(request).await
-    }
-    Execution::RestartDeployment(request) => {
-      komodo_client().execute(request).await
-    }
-    Execution::PauseDeployment(request) => {
-      komodo_client().execute(request).await
-    }
-    Execution::UnpauseDeployment(request) => {
-      komodo_client().execute(request).await
-    }
-    Execution::StopDeployment(request) => {
-      komodo_client().execute(request).await
-    }
-    Execution::DestroyDeployment(request) => {
-      komodo_client().execute(request).await
-    }
-    Execution::CloneRepo(request) => {
-      komodo_client().execute(request).await
-    }
-    Execution::PullRepo(request) => {
-      komodo_client().execute(request).await
-    }
-    Execution::BuildRepo(request) => {
-      komodo_client().execute(request).await
-    }
-    Execution::CancelRepoBuild(request) => {
-      komodo_client().execute(request).await
-    }
-    Execution::StartContainer(request) => {
-      komodo_client().execute(request).await
-    }
-    Execution::RestartContainer(request) => {
-      komodo_client().execute(request).await
-    }
-    Execution::PauseContainer(request) => {
-      komodo_client().execute(request).await
-    }
-    Execution::UnpauseContainer(request) => {
-      komodo_client().execute(request).await
-    }
-    Execution::StopContainer(request) => {
-      komodo_client().execute(request).await
-    }
-    Execution::DestroyContainer(request) => {
-      komodo_client().execute(request).await
-    }
-    Execution::StartAllContainers(request) => {
-      komodo_client().execute(request).await
-    }
-    Execution::RestartAllContainers(request) => {
-      komodo_client().execute(request).await
-    }
-    Execution::PauseAllContainers(request) => {
-      komodo_client().execute(request).await
-    }
-    Execution::UnpauseAllContainers(request) => {
-      komodo_client().execute(request).await
-    }
-    Execution::StopAllContainers(request) => {
-      komodo_client().execute(request).await
-    }
-    Execution::PruneContainers(request) => {
-      komodo_client().execute(request).await
-    }
-    Execution::DeleteNetwork(request) => {
-      komodo_client().execute(request).await
-    }
-    Execution::PruneNetworks(request) => {
-      komodo_client().execute(request).await
-    }
-    Execution::DeleteImage(request) => {
-      komodo_client().execute(request).await
-    }
-    Execution::PruneImages(request) => {
-      komodo_client().execute(request).await
-    }
-    Execution::DeleteVolume(request) => {
-      komodo_client().execute(request).await
-    }
-    Execution::PruneVolumes(request) => {
-      komodo_client().execute(request).await
-    }
-    Execution::PruneDockerBuilders(request) => {
-      komodo_client().execute(request).await
-    }
-    Execution::PruneBuildx(request) => {
-      komodo_client().execute(request).await
-    }
-    Execution::PruneSystem(request) => {
-      komodo_client().execute(request).await
-    }
-    Execution::RunSync(request) => {
-      komodo_client().execute(request).await
-    }
-    Execution::DeployStack(request) => {
-      komodo_client().execute(request).await
-    }
-    Execution::StartStack(request) => {
-      komodo_client().execute(request).await
-    }
-    Execution::RestartStack(request) => {
-      komodo_client().execute(request).await
-    }
-    Execution::PauseStack(request) => {
-      komodo_client().execute(request).await
-    }
-    Execution::UnpauseStack(request) => {
-      komodo_client().execute(request).await
-    }
-    Execution::StopStack(request) => {
-      komodo_client().execute(request).await
-    }
-    Execution::DestroyStack(request) => {
-      komodo_client().execute(request).await
-    }
+    Execution::RunAction(request) => komodo_client()
+      .execute(request)
+      .await
+      .map(ExecutionResult::Single),
+    Execution::BatchRunAction(request) => komodo_client()
+      .execute(request)
+      .await
+      .map(ExecutionResult::Batch),
+    Execution::RunProcedure(request) => komodo_client()
+      .execute(request)
+      .await
+      .map(ExecutionResult::Single),
+    Execution::BatchRunProcedure(request) => komodo_client()
+      .execute(request)
+      .await
+      .map(ExecutionResult::Batch),
+    Execution::RunBuild(request) => komodo_client()
+      .execute(request)
+      .await
+      .map(ExecutionResult::Single),
+    Execution::BatchRunBuild(request) => komodo_client()
+      .execute(request)
+      .await
+      .map(ExecutionResult::Batch),
+    Execution::CancelBuild(request) => komodo_client()
+      .execute(request)
+      .await
+      .map(ExecutionResult::Single),
+    Execution::Deploy(request) => komodo_client()
+      .execute(request)
+      .await
+      .map(ExecutionResult::Single),
+    Execution::BatchDeploy(request) => komodo_client()
+      .execute(request)
+      .await
+      .map(ExecutionResult::Batch),
+    Execution::PullDeployment(request) => komodo_client()
+      .execute(request)
+      .await
+      .map(ExecutionResult::Single),
+    Execution::StartDeployment(request) => komodo_client()
+      .execute(request)
+      .await
+      .map(ExecutionResult::Single),
+    Execution::RestartDeployment(request) => komodo_client()
+      .execute(request)
+      .await
+      .map(ExecutionResult::Single),
+    Execution::PauseDeployment(request) => komodo_client()
+      .execute(request)
+      .await
+      .map(ExecutionResult::Single),
+    Execution::UnpauseDeployment(request) => komodo_client()
+      .execute(request)
+      .await
+      .map(ExecutionResult::Single),
+    Execution::StopDeployment(request) => komodo_client()
+      .execute(request)
+      .await
+      .map(ExecutionResult::Single),
+    Execution::DestroyDeployment(request) => komodo_client()
+      .execute(request)
+      .await
+      .map(ExecutionResult::Single),
+    Execution::BatchDestroyDeployment(request) => komodo_client()
+      .execute(request)
+      .await
+      .map(ExecutionResult::Batch),
+    Execution::CloneRepo(request) => komodo_client()
+      .execute(request)
+      .await
+      .map(ExecutionResult::Single),
+    Execution::BatchCloneRepo(request) => komodo_client()
+      .execute(request)
+      .await
+      .map(ExecutionResult::Batch),
+    Execution::PullRepo(request) => komodo_client()
+      .execute(request)
+      .await
+      .map(ExecutionResult::Single),
+    Execution::BatchPullRepo(request) => komodo_client()
+      .execute(request)
+      .await
+      .map(ExecutionResult::Batch),
+    Execution::BuildRepo(request) => komodo_client()
+      .execute(request)
+      .await
+      .map(ExecutionResult::Single),
+    Execution::BatchBuildRepo(request) => komodo_client()
+      .execute(request)
+      .await
+      .map(ExecutionResult::Batch),
+    Execution::CancelRepoBuild(request) => komodo_client()
+      .execute(request)
+      .await
+      .map(ExecutionResult::Single),
+    Execution::StartContainer(request) => komodo_client()
+      .execute(request)
+      .await
+      .map(ExecutionResult::Single),
+    Execution::RestartContainer(request) => komodo_client()
+      .execute(request)
+      .await
+      .map(ExecutionResult::Single),
+    Execution::PauseContainer(request) => komodo_client()
+      .execute(request)
+      .await
+      .map(ExecutionResult::Single),
+    Execution::UnpauseContainer(request) => komodo_client()
+      .execute(request)
+      .await
+      .map(ExecutionResult::Single),
+    Execution::StopContainer(request) => komodo_client()
+      .execute(request)
+      .await
+      .map(ExecutionResult::Single),
+    Execution::DestroyContainer(request) => komodo_client()
+      .execute(request)
+      .await
+      .map(ExecutionResult::Single),
+    Execution::StartAllContainers(request) => komodo_client()
+      .execute(request)
+      .await
+      .map(ExecutionResult::Single),
+    Execution::RestartAllContainers(request) => komodo_client()
+      .execute(request)
+      .await
+      .map(ExecutionResult::Single),
+    Execution::PauseAllContainers(request) => komodo_client()
+      .execute(request)
+      .await
+      .map(ExecutionResult::Single),
+    Execution::UnpauseAllContainers(request) => komodo_client()
+      .execute(request)
+      .await
+      .map(ExecutionResult::Single),
+    Execution::StopAllContainers(request) => komodo_client()
+      .execute(request)
+      .await
+      .map(ExecutionResult::Single),
+    Execution::PruneContainers(request) => komodo_client()
+      .execute(request)
+      .await
+      .map(ExecutionResult::Single),
+    Execution::DeleteNetwork(request) => komodo_client()
+      .execute(request)
+      .await
+      .map(ExecutionResult::Single),
+    Execution::PruneNetworks(request) => komodo_client()
+      .execute(request)
+      .await
+      .map(ExecutionResult::Single),
+    Execution::DeleteImage(request) => komodo_client()
+      .execute(request)
+      .await
+      .map(ExecutionResult::Single),
+    Execution::PruneImages(request) => komodo_client()
+      .execute(request)
+      .await
+      .map(ExecutionResult::Single),
+    Execution::DeleteVolume(request) => komodo_client()
+      .execute(request)
+      .await
+      .map(ExecutionResult::Single),
+    Execution::PruneVolumes(request) => komodo_client()
+      .execute(request)
+      .await
+      .map(ExecutionResult::Single),
+    Execution::PruneDockerBuilders(request) => komodo_client()
+      .execute(request)
+      .await
+      .map(ExecutionResult::Single),
+    Execution::PruneBuildx(request) => komodo_client()
+      .execute(request)
+      .await
+      .map(ExecutionResult::Single),
+    Execution::PruneSystem(request) => komodo_client()
+      .execute(request)
+      .await
+      .map(ExecutionResult::Single),
+    Execution::RunSync(request) => komodo_client()
+      .execute(request)
+      .await
+      .map(ExecutionResult::Single),
+    Execution::CommitSync(request) => komodo_client()
+      .write(request)
+      .await
+      .map(ExecutionResult::Single),
+    Execution::DeployStack(request) => komodo_client()
+      .execute(request)
+      .await
+      .map(ExecutionResult::Single),
+    Execution::BatchDeployStack(request) => komodo_client()
+      .execute(request)
+      .await
+      .map(ExecutionResult::Batch),
+    Execution::DeployStackIfChanged(request) => komodo_client()
+      .execute(request)
+      .await
+      .map(ExecutionResult::Single),
+    Execution::BatchDeployStackIfChanged(request) => komodo_client()
+      .execute(request)
+      .await
+      .map(ExecutionResult::Batch),
+    Execution::PullStack(request) => komodo_client()
+      .execute(request)
+      .await
+      .map(ExecutionResult::Single),
+    Execution::StartStack(request) => komodo_client()
+      .execute(request)
+      .await
+      .map(ExecutionResult::Single),
+    Execution::RestartStack(request) => komodo_client()
+      .execute(request)
+      .await
+      .map(ExecutionResult::Single),
+    Execution::PauseStack(request) => komodo_client()
+      .execute(request)
+      .await
+      .map(ExecutionResult::Single),
+    Execution::UnpauseStack(request) => komodo_client()
+      .execute(request)
+      .await
+      .map(ExecutionResult::Single),
+    Execution::StopStack(request) => komodo_client()
+      .execute(request)
+      .await
+      .map(ExecutionResult::Single),
+    Execution::DestroyStack(request) => komodo_client()
+      .execute(request)
+      .await
+      .map(ExecutionResult::Single),
+    Execution::BatchDestroyStack(request) => komodo_client()
+      .execute(request)
+      .await
+      .map(ExecutionResult::Batch),
     Execution::Sleep(request) => {
       let duration =
         Duration::from_millis(request.duration_ms as u64);
@@ -302,7 +465,12 @@ pub async fn run(execution: Execution) -> anyhow::Result<()> {
   };
 
   match res {
-    Ok(update) => println!("\n{}: {update:#?}", "SUCCESS".green()),
+    Ok(ExecutionResult::Single(update)) => {
+      println!("\n{}: {update:#?}", "SUCCESS".green())
+    }
+    Ok(ExecutionResult::Batch(update)) => {
+      println!("\n{}: {update:#?}", "SUCCESS".green())
+    }
     Err(e) => println!("{}\n\n{e:#?}", "ERROR".red()),
   }
 

bin/cli/src/state.rs

@@ -40,7 +40,9 @@ pub fn komodo_client() -> &'static KomodoClient {
           creds
         }
       };
-    futures::executor::block_on(KomodoClient::new(url, key, secret))
-      .expect("failed to initialize Komodo client")
+    futures::executor::block_on(
+      KomodoClient::new(url, key, secret).with_healthcheck(),
+    )
+    .expect("failed to initialize Komodo client")
   })
 }

bin/core/Cargo.toml

@@ -19,7 +19,9 @@ komodo_client = { workspace = true, features = ["mongo"] }
 periphery_client.workspace = true
 environment_file.workspace = true
 formatting.workspace = true
+command.workspace = true
 logger.workspace = true
+cache.workspace = true
 git.workspace = true
 # mogh
 serror = { workspace = true, features = ["axum"] }
@@ -47,15 +49,17 @@ serde_json.workspace = true
 serde_yaml.workspace = true
 typeshare.workspace = true
 octorust.workspace = true
+wildcard.workspace = true
 dashmap.workspace = true
 tracing.workspace = true
 reqwest.workspace = true
 futures.workspace = true
 nom_pem.workspace = true
-anyhow.workspace = true
 dotenvy.workspace = true
+anyhow.workspace = true
 bcrypt.workspace = true
 base64.workspace = true
+rustls.workspace = true
 tokio.workspace = true
 serde.workspace = true
 regex.workspace = true

bin/core/aio.Dockerfile

@@ -0,0 +1,62 @@
+## All in one, multi stage compile + runtime Docker build for your architecture.
+
+# Build Core
+FROM rust:1.82.0-bullseye AS core-builder
+
+WORKDIR /builder
+COPY Cargo.toml Cargo.lock ./
+COPY ./lib ./lib
+COPY ./client/core/rs ./client/core/rs
+COPY ./client/periphery ./client/periphery
+
+# Pre compile dependencies
+COPY ./bin/core/Cargo.toml ./bin/core/Cargo.toml
+RUN mkdir ./bin/core/src && \
+  echo "fn main() {}" >> ./bin/core/src/main.rs && \
+  cargo build -p komodo_core --release && \
+  rm -r ./bin/core
+COPY ./bin/core ./bin/core
+
+# Compile app
+RUN cargo build -p komodo_core --release
+
+# Build Frontend
+FROM node:20.12-alpine AS frontend-builder
+WORKDIR /builder
+COPY ./frontend ./frontend
+COPY ./client/core/ts ./client
+RUN cd client && yarn && yarn build && yarn link
+RUN cd frontend && yarn link komodo_client && yarn && yarn build
+
+# Final Image
+FROM debian:bullseye-slim
+
+# Install Deps
+RUN apt update && \
+  apt install -y git ca-certificates && \
+  rm -rf /var/lib/apt/lists/*
+
+# Setup an application directory
+WORKDIR /app
+
+# Copy
+COPY ./config/core.config.toml /config/config.toml
+COPY --from=frontend-builder /builder/frontend/dist /app/frontend
+COPY --from=core-builder /builder/target/release/core /usr/local/bin/core
+COPY --from=denoland/deno:bin /deno /usr/local/bin/deno
+
+# Set $DENO_DIR and preload external Deno deps
+ENV DENO_DIR=/action-cache/deno
+RUN mkdir /action-cache && \
+  cd /action-cache && \
+  deno install jsr:@std/yaml jsr:@std/toml
+
+# Hint at the port
+EXPOSE 9120
+
+# Label for Ghcr
+LABEL org.opencontainers.image.source=https://github.com/mbecker20/komodo
+LABEL org.opencontainers.image.description="Komodo Core"
+LABEL org.opencontainers.image.licenses=GPL-3.0
+
+ENTRYPOINT [ "core" ]

bin/core/alpine.Dockerfile

@@ -1,45 +0,0 @@
-## This one produces smaller images,
-## but alpine uses `musl` instead of `glibc`.
-## This makes it take longer / more resources to build,
-## and may negatively affect runtime performance.
-
-# Build Core
-FROM rust:1.81.0-alpine AS core-builder
-WORKDIR /builder
-RUN apk update && apk --no-cache add musl-dev openssl-dev openssl-libs-static
-COPY . .
-RUN cargo build -p komodo_core --release
-
-# Build Frontend
-FROM node:20.12-alpine AS frontend-builder
-WORKDIR /builder
-COPY ./frontend ./frontend
-COPY ./client/core/ts ./client
-RUN cd client && yarn && yarn build && yarn link
-RUN cd frontend && yarn link @komodo/client && yarn && yarn build
-
-# Final Image
-FROM alpine:3.20
-
-# Install Deps
-RUN apk update && apk add --no-cache --virtual .build-deps \
-	openssl ca-certificates git git-lfs
-
-# Setup an application directory
-WORKDIR /app
-
-# Copy
-COPY ./config/core.config.toml /config/config.toml
-COPY --from=core-builder /builder/target/release/core /app
-COPY --from=frontend-builder /builder/frontend/dist /app/frontend
-
-# Hint at the port
-EXPOSE 9120 
-
-# Label for Ghcr
-LABEL org.opencontainers.image.source=https://github.com/mbecker20/komodo
-LABEL org.opencontainers.image.description="Komodo Core"
-LABEL org.opencontainers.image.licenses=GPL-3.0
-
-# Using ENTRYPOINT allows cli args to be passed, eg using "command" in docker compose.
-ENTRYPOINT [ "/app/core" ]

bin/core/multi-arch.Dockerfile

@@ -0,0 +1,50 @@
+## Assumes the latest binaries for x86_64 and aarch64 are already built (by binaries.Dockerfile).
+## Sets up the necessary runtime container dependencies for Komodo Core.
+## Since theres no heavy build here, QEMU multi-arch builds are fine for this image.
+
+ARG BINARIES_IMAGE=ghcr.io/mbecker20/komodo-binaries:latest
+ARG FRONTEND_IMAGE=ghcr.io/mbecker20/komodo-frontend:latest
+ARG X86_64_BINARIES=${BINARIES_IMAGE}-x86_64
+ARG AARCH64_BINARIES=${BINARIES_IMAGE}-aarch64
+
+# This is required to work with COPY --from
+FROM ${X86_64_BINARIES} AS x86_64
+FROM ${AARCH64_BINARIES} AS aarch64
+FROM ${FRONTEND_IMAGE} AS frontend
+
+# Final Image
+FROM debian:bullseye-slim
+
+# Install Deps
+RUN apt update && \
+  apt install -y git ca-certificates && \
+  rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+
+# Copy both binaries initially, but only keep appropriate one for the TARGETPLATFORM.
+COPY --from=x86_64 /core /app/arch/linux/amd64
+COPY --from=aarch64 /core /app/arch/linux/arm64
+ARG TARGETPLATFORM
+RUN mv /app/arch/${TARGETPLATFORM} /usr/local/bin/core && rm -r /app/arch
+
+# Copy default config / static frontend / deno binary
+COPY ./config/core.config.toml /config/config.toml
+COPY --from=frontend /frontend /app/frontend
+COPY --from=denoland/deno:bin /deno /usr/local/bin/deno
+
+# Set $DENO_DIR and preload external Deno deps
+ENV DENO_DIR=/action-cache/deno
+RUN mkdir /action-cache && \
+  cd /action-cache && \
+  deno install jsr:@std/yaml jsr:@std/toml
+
+# Hint at the port
+EXPOSE 9120
+
+# Label for Ghcr
+LABEL org.opencontainers.image.source=https://github.com/mbecker20/komodo
+LABEL org.opencontainers.image.description="Komodo Core"
+LABEL org.opencontainers.image.licenses=GPL-3.0
+
+ENTRYPOINT [ "core" ]

bin/core/single-arch.Dockerfile

@@ -1,8 +1,10 @@
-# Build Core
-FROM rust:1.81.0-bullseye AS core-builder
-WORKDIR /builder
-COPY . .
-RUN cargo build -p komodo_core --release
+## Assumes the latest binaries for the required arch are already built (by binaries.Dockerfile).
+## Sets up the necessary runtime container dependencies for Komodo Core.
+
+ARG BINARIES_IMAGE=ghcr.io/mbecker20/komodo-binaries:latest
+
+# This is required to work with COPY --from
+FROM ${BINARIES_IMAGE} AS binaries
 
 # Build Frontend
 FROM node:20.12-alpine AS frontend-builder
@@ -10,23 +12,26 @@ WORKDIR /builder
 COPY ./frontend ./frontend
 COPY ./client/core/ts ./client
 RUN cd client && yarn && yarn build && yarn link
-RUN cd frontend && yarn link @komodo/client && yarn && yarn build
+RUN cd frontend && yarn link komodo_client && yarn && yarn build
 
-# Final Image
 FROM debian:bullseye-slim
 
 # Install Deps
 RUN apt update && \
 	apt install -y git ca-certificates && \
 	rm -rf /var/lib/apt/lists/*
-
-# Setup an application directory
-WORKDIR /app
 	
 # Copy
 COPY ./config/core.config.toml /config/config.toml
-COPY --from=core-builder /builder/target/release/core /app
 COPY --from=frontend-builder /builder/frontend/dist /app/frontend
+COPY --from=binaries /core /usr/local/bin/core
+COPY --from=denoland/deno:bin /deno /usr/local/bin/deno
+
+# Set $DENO_DIR and preload external Deno deps
+ENV DENO_DIR=/action-cache/deno
+RUN mkdir /action-cache && \
+	cd /action-cache && \
+	deno install jsr:@std/yaml jsr:@std/toml
 
 # Hint at the port
 EXPOSE 9120
@@ -36,4 +41,4 @@ LABEL org.opencontainers.image.source=https://github.com/mbecker20/komodo
 LABEL org.opencontainers.image.description="Komodo Core"
 LABEL org.opencontainers.image.licenses=GPL-3.0
 
-ENTRYPOINT [ "/app/core" ]
+ENTRYPOINT [ "core" ]

bin/core/src/alert/discord.rs

@@ -22,7 +22,7 @@ pub async fn send_alert(
       match alert.level {
         SeverityLevel::Ok => {
           format!(
-            "{level} | *{name}*{region} is now *reachable*\n{link}"
+            "{level} | **{name}**{region} is now **reachable**\n{link}"
           )
         }
         SeverityLevel::Critical => {
@@ -31,7 +31,7 @@ pub async fn send_alert(
             .map(|e| format!("\n**error**: {e:#?}"))
             .unwrap_or_default();
           format!(
-            "{level} | *{name}*{region} is *unreachable* ❌\n{link}{err}"
+            "{level} | **{name}**{region} is **unreachable** ❌\n{link}{err}"
           )
         }
         _ => unreachable!(),
@@ -46,7 +46,7 @@ pub async fn send_alert(
       let region = fmt_region(region);
       let link = resource_link(ResourceTargetVariant::Server, id);
       format!(
-        "{level} | *{name}*{region} cpu usage at *{percentage:.1}%*\n{link}"
+        "{level} | **{name}**{region} cpu usage at **{percentage:.1}%**\n{link}"
       )
     }
     AlertData::ServerMem {
@@ -60,7 +60,7 @@ pub async fn send_alert(
       let link = resource_link(ResourceTargetVariant::Server, id);
       let percentage = 100.0 * used_gb / total_gb;
       format!(
-        "{level} | *{name}*{region} memory usage at *{percentage:.1}%* 💾\n\nUsing *{used_gb:.1} GiB* / *{total_gb:.1} GiB*\n{link}"
+        "{level} | **{name}**{region} memory usage at **{percentage:.1}%** 💾\n\nUsing **{used_gb:.1} GiB** / **{total_gb:.1} GiB**\n{link}"
       )
     }
     AlertData::ServerDisk {
@@ -75,7 +75,7 @@ pub async fn send_alert(
       let link = resource_link(ResourceTargetVariant::Server, id);
       let percentage = 100.0 * used_gb / total_gb;
       format!(
-        "{level} | *{name}*{region} disk usage at *{percentage:.1}%* 💿\nmount point: `{path:?}`\nusing *{used_gb:.1} GiB* / *{total_gb:.1} GiB*\n{link}"
+        "{level} | **{name}**{region} disk usage at **{percentage:.1}%** 💿\nmount point: `{path:?}`\nusing **{used_gb:.1} GiB** / **{total_gb:.1} GiB**\n{link}"
       )
     }
     AlertData::ContainerStateChange {
@@ -88,7 +88,27 @@ pub async fn send_alert(
     } => {
       let link = resource_link(ResourceTargetVariant::Deployment, id);
       let to = fmt_docker_container_state(to);
-      format!("📦 Deployment *{name}* is now {to}\nserver: {server_name}\nprevious: {from}\n{link}")
+      format!("📦 Deployment **{name}** is now **{to}**\nserver: **{server_name}**\nprevious: **{from}**\n{link}")
+    }
+    AlertData::DeploymentImageUpdateAvailable {
+      id,
+      name,
+      server_id: _server_id,
+      server_name,
+      image,
+    } => {
+      let link = resource_link(ResourceTargetVariant::Deployment, id);
+      format!("⬆ Deployment **{name}** has an update available\nserver: **{server_name}**\nimage: **{image}**\n{link}")
+    }
+    AlertData::DeploymentAutoUpdated {
+      id,
+      name,
+      server_id: _server_id,
+      server_name,
+      image,
+    } => {
+      let link = resource_link(ResourceTargetVariant::Deployment, id);
+      format!("⬆ Deployment **{name}** was updated automatically ⏫\nserver: **{server_name}**\nimage: **{image}**\n{link}")
     }
     AlertData::StackStateChange {
       id,
@@ -100,28 +120,52 @@ pub async fn send_alert(
     } => {
       let link = resource_link(ResourceTargetVariant::Stack, id);
       let to = fmt_stack_state(to);
-      format!("🥞 Stack *{name}* is now {to}\nserver: {server_name}\nprevious: {from}\n{link}")
+      format!("🥞 Stack **{name}** is now {to}\nserver: **{server_name}**\nprevious: **{from}**\n{link}")
+    }
+    AlertData::StackImageUpdateAvailable {
+      id,
+      name,
+      server_id: _server_id,
+      server_name,
+      service,
+      image,
+    } => {
+      let link = resource_link(ResourceTargetVariant::Stack, id);
+      format!("⬆ Stack **{name}** has an update available\nserver: **{server_name}**\nservice: **{service}**\nimage: **{image}**\n{link}")
+    }
+    AlertData::StackAutoUpdated {
+      id,
+      name,
+      server_id: _server_id,
+      server_name,
+      images,
+    } => {
+      let link = resource_link(ResourceTargetVariant::Stack, id);
+      let images_label =
+        if images.len() > 1 { "images" } else { "image" };
+      let images = images.join(", ");
+      format!("⬆ Stack **{name}** was updated automatically ⏫\nserver: **{server_name}**\n{images_label}: **{images}**\n{link}")
     }
     AlertData::AwsBuilderTerminationFailed {
       instance_id,
       message,
     } => {
-      format!("{level} | Failed to terminated AWS builder instance\ninstance id: *{instance_id}*\n{message}")
+      format!("{level} | Failed to terminated AWS builder instance\ninstance id: **{instance_id}**\n{message}")
     }
     AlertData::ResourceSyncPendingUpdates { id, name } => {
       let link =
         resource_link(ResourceTargetVariant::ResourceSync, id);
       format!(
-        "{level} | Pending resource sync updates on *{name}*\n{link}"
+        "{level} | Pending resource sync updates on **{name}**\n{link}"
       )
     }
     AlertData::BuildFailed { id, name, version } => {
       let link = resource_link(ResourceTargetVariant::Build, id);
-      format!("{level} | Build *{name}* failed\nversion: v{version}\n{link}")
+      format!("{level} | Build **{name}** failed\nversion: **v{version}**\n{link}")
     }
     AlertData::RepoBuildFailed { id, name } => {
       let link = resource_link(ResourceTargetVariant::Repo, id);
-      format!("{level} | Repo build for *{name}* failed\n{link}")
+      format!("{level} | Repo build for **{name}** failed\n{link}")
     }
     AlertData::None {} => Default::default(),
   };

bin/core/src/alert/mod.rs

@@ -10,36 +10,42 @@ use komodo_client::entities::{
   ResourceTargetVariant,
 };
 use mungos::{find::find_collect, mongodb::bson::doc};
+use tracing::Instrument;
 
 use crate::{config::core_config, state::db_client};
 
 mod discord;
 mod slack;
 
-#[instrument]
 pub async fn send_alerts(alerts: &[Alert]) {
   if alerts.is_empty() {
     return;
   }
 
-  let Ok(alerters) = find_collect(
-    &db_client().alerters,
-    doc! { "config.enabled": true },
-    None,
-  )
-  .await
-  .inspect_err(|e| {
-    error!(
+  let span =
+    info_span!("send_alerts", alerts = format!("{alerts:?}"));
+  async {
+    let Ok(alerters) = find_collect(
+      &db_client().alerters,
+      doc! { "config.enabled": true },
+      None,
+    )
+    .await
+    .inspect_err(|e| {
+      error!(
       "ERROR sending alerts | failed to get alerters from db | {e:#}"
     )
-  }) else {
-    return;
-  };
+    }) else {
+      return;
+    };
 
-  let handles =
-    alerts.iter().map(|alert| send_alert(&alerters, alert));
+    let handles =
+      alerts.iter().map(|alert| send_alert(&alerters, alert));
 
-  join_all(handles).await;
+    join_all(handles).await;
+  }
+  .instrument(span)
+  .await
 }
 
 #[instrument(level = "debug")]
@@ -195,6 +201,9 @@ fn resource_link(
     ResourceTargetVariant::Procedure => {
       format!("/procedures/{id}")
     }
+    ResourceTargetVariant::Action => {
+      format!("/actions/{id}")
+    }
     ResourceTargetVariant::ServerTemplate => {
       format!("/server-templates/{id}")
     }

bin/core/src/alert/slack.rs

@@ -182,7 +182,7 @@ pub async fn send_alert(
       ..
     } => {
       let to = fmt_docker_container_state(to);
-      let text = format!("📦 Container *{name}* is now {to}");
+      let text = format!("📦 Container *{name}* is now *{to}*");
       let blocks = vec![
         Block::header(text.clone()),
         Block::section(format!(
@@ -195,6 +195,48 @@ pub async fn send_alert(
       ];
       (text, blocks.into())
     }
+    AlertData::DeploymentImageUpdateAvailable {
+      id,
+      name,
+      server_name,
+      server_id: _server_id,
+      image,
+    } => {
+      let text =
+        format!("⬆ Deployment *{name}* has an update available");
+      let blocks = vec![
+        Block::header(text.clone()),
+        Block::section(format!(
+          "server: *{server_name}*\nimage: *{image}*",
+        )),
+        Block::section(resource_link(
+          ResourceTargetVariant::Deployment,
+          id,
+        )),
+      ];
+      (text, blocks.into())
+    }
+    AlertData::DeploymentAutoUpdated {
+      id,
+      name,
+      server_name,
+      server_id: _server_id,
+      image,
+    } => {
+      let text =
+        format!("⬆ Deployment *{name}* was updated automatically ⏫");
+      let blocks = vec![
+        Block::header(text.clone()),
+        Block::section(format!(
+          "server: *{server_name}*\nimage: *{image}*",
+        )),
+        Block::section(resource_link(
+          ResourceTargetVariant::Deployment,
+          id,
+        )),
+      ];
+      (text, blocks.into())
+    }
     AlertData::StackStateChange {
       name,
       server_name,
@@ -204,11 +246,56 @@ pub async fn send_alert(
       ..
     } => {
       let to = fmt_stack_state(to);
-      let text = format!("🥞 Stack *{name}* is now {to}");
+      let text = format!("🥞 Stack *{name}* is now *{to}*");
       let blocks = vec![
         Block::header(text.clone()),
         Block::section(format!(
-          "server: {server_name}\nprevious: {from}",
+          "server: *{server_name}*\nprevious: *{from}*",
+        )),
+        Block::section(resource_link(
+          ResourceTargetVariant::Stack,
+          id,
+        )),
+      ];
+      (text, blocks.into())
+    }
+    AlertData::StackImageUpdateAvailable {
+      id,
+      name,
+      server_name,
+      server_id: _server_id,
+      service,
+      image,
+    } => {
+      let text = format!("⬆ Stack *{name}* has an update available");
+      let blocks = vec![
+        Block::header(text.clone()),
+        Block::section(format!(
+          "server: *{server_name}*\nservice: *{service}*\nimage: *{image}*",
+        )),
+        Block::section(resource_link(
+          ResourceTargetVariant::Stack,
+          id,
+        )),
+      ];
+      (text, blocks.into())
+    }
+    AlertData::StackAutoUpdated {
+      id,
+      name,
+      server_name,
+      server_id: _server_id,
+      images,
+    } => {
+      let text =
+        format!("⬆ Stack *{name}* was updated automatically ⏫");
+      let images_label =
+        if images.len() > 1 { "images" } else { "image" };
+      let images = images.join(", ");
+      let blocks = vec![
+        Block::header(text.clone()),
+        Block::section(format!(
+          "server: *{server_name}*\n{images_label}: *{images}*",
         )),
         Block::section(resource_link(
           ResourceTargetVariant::Stack,
@@ -233,8 +320,9 @@ pub async fn send_alert(
       (text, blocks.into())
     }
     AlertData::ResourceSyncPendingUpdates { id, name } => {
-      let text =
-        format!("{level} | Pending resource sync updates on {name}");
+      let text = format!(
+        "{level} | Pending resource sync updates on *{name}*"
+      );
       let blocks = vec![
         Block::header(text.clone()),
         Block::section(format!(
@@ -252,20 +340,21 @@ pub async fn send_alert(
       let blocks = vec![
         Block::header(text.clone()),
         Block::section(format!(
-          "build id: *{id}*\nbuild name: *{name}*\nversion: v{version}",
+          "build name: *{name}*\nversion: *v{version}*",
+        )),
+        Block::section(resource_link(
+          ResourceTargetVariant::Build,
+          id,
         )),
-        Block::section(resource_link(ResourceTargetVariant::Build, id))
       ];
       (text, blocks.into())
     }
     AlertData::RepoBuildFailed { id, name } => {
       let text =
-        format!("{level} | Repo build for {name} has failed");
+        format!("{level} | Repo build for *{name}* has *failed*");
       let blocks = vec![
         Block::header(text.clone()),
-        Block::section(format!(
-          "repo id: *{id}*\nrepo name: *{name}*",
-        )),
+        Block::section(format!("repo name: *{name}*",)),
         Block::section(resource_link(
           ResourceTargetVariant::Repo,
           id,

bin/core/src/api/execute/action.rs

@@ -0,0 +1,328 @@
+use std::{
+  collections::HashSet,
+  path::{Path, PathBuf},
+  str::FromStr,
+  sync::OnceLock,
+};
+
+use anyhow::Context;
+use command::run_komodo_command;
+use komodo_client::{
+  api::{
+    execute::{BatchExecutionResponse, BatchRunAction, RunAction},
+    user::{CreateApiKey, CreateApiKeyResponse, DeleteApiKey},
+  },
+  entities::{
+    action::Action,
+    config::core::CoreConfig,
+    permission::PermissionLevel,
+    update::Update,
+    user::{action_user, User},
+  },
+};
+use mungos::{by_id::update_one_by_id, mongodb::bson::to_document};
+use resolver_api::Resolve;
+use tokio::fs;
+
+use crate::{
+  api::execute::ExecuteRequest,
+  config::core_config,
+  helpers::{
+    interpolate::{
+      add_interp_update_log,
+      interpolate_variables_secrets_into_string,
+    },
+    query::get_variables_and_secrets,
+    random_string,
+    update::update_update,
+  },
+  resource::{self, refresh_action_state_cache},
+  state::{action_states, db_client, State},
+};
+
+impl super::BatchExecute for BatchRunAction {
+  type Resource = Action;
+  fn single_request(action: String) -> ExecuteRequest {
+    ExecuteRequest::RunAction(RunAction { action })
+  }
+}
+
+impl Resolve<BatchRunAction, (User, Update)> for State {
+  #[instrument(name = "BatchRunAction", skip(self, user), fields(user_id = user.id))]
+  async fn resolve(
+    &self,
+    BatchRunAction { pattern }: BatchRunAction,
+    (user, _): (User, Update),
+  ) -> anyhow::Result<BatchExecutionResponse> {
+    super::batch_execute::<BatchRunAction>(&pattern, &user).await
+  }
+}
+
+impl Resolve<RunAction, (User, Update)> for State {
+  #[instrument(name = "RunAction", skip(self, user, update), fields(user_id = user.id, update_id = update.id))]
+  async fn resolve(
+    &self,
+    RunAction { action }: RunAction,
+    (user, mut update): (User, Update),
+  ) -> anyhow::Result<Update> {
+    let mut action = resource::get_check_permissions::<Action>(
+      &action,
+      &user,
+      PermissionLevel::Execute,
+    )
+    .await?;
+
+    // get the action state for the action (or insert default).
+    let action_state = action_states()
+      .action
+      .get_or_insert_default(&action.id)
+      .await;
+
+    // This will set action state back to default when dropped.
+    // Will also check to ensure action not already busy before updating.
+    let _action_guard =
+      action_state.update(|state| state.running = true)?;
+
+    update_update(update.clone()).await?;
+
+    let CreateApiKeyResponse { key, secret } = State
+      .resolve(
+        CreateApiKey {
+          name: update.id.clone(),
+          expires: 0,
+        },
+        action_user().to_owned(),
+      )
+      .await?;
+
+    let contents = &mut action.config.file_contents;
+
+    // Wrap the file contents in the execution context.
+    *contents = full_contents(contents, &key, &secret);
+
+    let replacers =
+      interpolate(contents, &mut update, key.clone(), secret.clone())
+        .await?
+        .into_iter()
+        .collect::<Vec<_>>();
+
+    let file = format!("{}.ts", random_string(10));
+    let path = core_config().action_directory.join(&file);
+
+    if let Some(parent) = path.parent() {
+      let _ = fs::create_dir_all(parent).await;
+    }
+
+    fs::write(&path, contents).await.with_context(|| {
+      format!("Failed to write action file to {path:?}")
+    })?;
+
+    let mut res = run_komodo_command(
+      // Keep this stage name as is, the UI will find the latest update log by matching the stage name
+      "Execute Action",
+      None,
+      format!("deno run --allow-all {}", path.display()),
+      false,
+    )
+    .await;
+
+    res.stdout = svi::replace_in_string(&res.stdout, &replacers)
+      .replace(&key, "<ACTION_API_KEY>");
+    res.stderr = svi::replace_in_string(&res.stderr, &replacers)
+      .replace(&secret, "<ACTION_API_SECRET>");
+
+    cleanup_run(file + ".js", &path).await;
+
+    if let Err(e) = State
+      .resolve(DeleteApiKey { key }, action_user().to_owned())
+      .await
+    {
+      warn!(
+        "Failed to delete API key after action execution | {e:#}"
+      );
+    };
+
+    update.logs.push(res);
+    update.finalize();
+
+    // Need to manually update the update before cache refresh,
+    // and before broadcast with update_update.
+    // The Err case of to_document should be unreachable,
+    // but will fail to update cache in that case.
+    if let Ok(update_doc) = to_document(&update) {
+      let _ = update_one_by_id(
+        &db_client().updates,
+        &update.id,
+        mungos::update::Update::Set(update_doc),
+        None,
+      )
+      .await;
+      refresh_action_state_cache().await;
+    }
+
+    update_update(update.clone()).await?;
+
+    Ok(update)
+  }
+}
+
+async fn interpolate(
+  contents: &mut String,
+  update: &mut Update,
+  key: String,
+  secret: String,
+) -> anyhow::Result<HashSet<(String, String)>> {
+  let mut vars_and_secrets = get_variables_and_secrets().await?;
+
+  vars_and_secrets
+    .secrets
+    .insert(String::from("ACTION_API_KEY"), key);
+  vars_and_secrets
+    .secrets
+    .insert(String::from("ACTION_API_SECRET"), secret);
+
+  let mut global_replacers = HashSet::new();
+  let mut secret_replacers = HashSet::new();
+
+  interpolate_variables_secrets_into_string(
+    &vars_and_secrets,
+    contents,
+    &mut global_replacers,
+    &mut secret_replacers,
+  )?;
+
+  add_interp_update_log(update, &global_replacers, &secret_replacers);
+
+  Ok(secret_replacers)
+}
+
+fn full_contents(contents: &str, key: &str, secret: &str) -> String {
+  let CoreConfig {
+    port, ssl_enabled, ..
+  } = core_config();
+  let protocol = if *ssl_enabled { "https" } else { "http" };
+  let base_url = format!("{protocol}://localhost:{port}");
+  format!(
+    "import {{ KomodoClient }} from '{base_url}/client/lib.js';
+import * as __YAML__ from 'jsr:@std/yaml';
+import * as __TOML__ from 'jsr:@std/toml';
+
+const YAML = {{
+  stringify: __YAML__.stringify,
+  parse: __YAML__.parse,
+  parseAll: __YAML__.parseAll,
+  parseDockerCompose: __YAML__.parse,
+}}
+
+const TOML = {{
+  stringify: __TOML__.stringify,
+  parse: __TOML__.parse,
+  parseResourceToml: __TOML__.parse,
+  parseCargoToml: __TOML__.parse,
+}}
+
+const komodo = KomodoClient('{base_url}', {{
+  type: 'api-key',
+  params: {{ key: '{key}', secret: '{secret}' }}
+}});
+
+async function main() {{
+{contents}
+
+console.log('🦎 Action completed successfully 🦎');
+}}
+
+main()
+.catch(error => {{
+  console.error('🚨 Action exited early with errors 🚨')
+  if (error.status !== undefined && error.result !== undefined) {{
+    console.error('Status:', error.status);
+    console.error(JSON.stringify(error.result, null, 2));
+  }} else {{
+    console.error(JSON.stringify(error, null, 2));
+  }}
+  Deno.exit(1)
+}});"
+  )
+}
+
+/// Cleans up file at given path.
+/// ALSO if $DENO_DIR is set,
+/// will clean up the generated file matching "file"
+async fn cleanup_run(file: String, path: &Path) {
+  if let Err(e) = fs::remove_file(path).await {
+    warn!(
+      "Failed to delete action file after action execution | {e:#}"
+    );
+  }
+  // If $DENO_DIR is set (will be in container),
+  // will clean up the generated file matching "file" (NOT under path)
+  let Some(deno_dir) = deno_dir() else {
+    return;
+  };
+  delete_file(deno_dir.join("gen/file"), file).await;
+}
+
+fn deno_dir() -> Option<&'static Path> {
+  static DENO_DIR: OnceLock<Option<PathBuf>> = OnceLock::new();
+  DENO_DIR
+    .get_or_init(|| {
+      let deno_dir = std::env::var("DENO_DIR").ok()?;
+      PathBuf::from_str(&deno_dir).ok()
+    })
+    .as_deref()
+}
+
+/// file is just the terminating file path,
+/// it may be nested multiple folder under path,
+/// this will find the nested file and delete it.
+/// Assumes the file is only there once.
+fn delete_file(
+  dir: PathBuf,
+  file: String,
+) -> std::pin::Pin<Box<dyn std::future::Future<Output = bool> + Send>>
+{
+  Box::pin(async move {
+    let Ok(mut dir) = fs::read_dir(dir).await else {
+      return false;
+    };
+    // Collect the nested folders for recursing
+    // only after checking all the files in directory.
+    let mut folders = Vec::<PathBuf>::new();
+
+    while let Ok(Some(entry)) = dir.next_entry().await {
+      let Ok(meta) = entry.metadata().await else {
+        continue;
+      };
+      if meta.is_file() {
+        let Ok(name) = entry.file_name().into_string() else {
+          continue;
+        };
+        if name == file {
+          if let Err(e) = fs::remove_file(entry.path()).await {
+            warn!(
+            "Failed to clean up generated file after action execution | {e:#}"
+          );
+          };
+          return true;
+        }
+      } else {
+        folders.push(entry.path());
+      }
+    }
+
+    if folders.len() == 1 {
+      // unwrap ok, folders definitely is not empty
+      let folder = folders.pop().unwrap();
+      delete_file(folder, file).await
+    } else {
+      // Check folders with file.clone
+      for folder in folders {
+        if delete_file(folder, file.clone()).await {
+          return true;
+        }
+      }
+      false
+    }
+  })
+}

bin/core/src/api/execute/build.rs

@@ -4,7 +4,10 @@ use anyhow::{anyhow, Context};
 use formatting::format_serror;
 use futures::future::join_all;
 use komodo_client::{
-  api::execute::{CancelBuild, Deploy, RunBuild},
+  api::execute::{
+    BatchExecutionResponse, BatchRunBuild, CancelBuild, Deploy,
+    RunBuild,
+  },
   entities::{
     alert::{Alert, AlertData, SeverityLevel},
     all_logs_success,
@@ -51,6 +54,24 @@ use crate::{
 
 use super::ExecuteRequest;
 
+impl super::BatchExecute for BatchRunBuild {
+  type Resource = Build;
+  fn single_request(build: String) -> ExecuteRequest {
+    ExecuteRequest::RunBuild(RunBuild { build })
+  }
+}
+
+impl Resolve<BatchRunBuild, (User, Update)> for State {
+  #[instrument(name = "BatchRunBuild", skip(self, user), fields(user_id = user.id))]
+  async fn resolve(
+    &self,
+    BatchRunBuild { pattern }: BatchRunBuild,
+    (user, _): (User, Update),
+  ) -> anyhow::Result<BatchExecutionResponse> {
+    super::batch_execute::<BatchRunBuild>(&pattern, &user).await
+  }
+}
+
 impl Resolve<RunBuild, (User, Update)> for State {
   #[instrument(name = "RunBuild", skip(self, user, update), fields(user_id = user.id, update_id = update.id))]
   async fn resolve(
@@ -64,7 +85,7 @@ impl Resolve<RunBuild, (User, Update)> for State {
       PermissionLevel::Execute,
     )
     .await?;
-    let vars_and_secrets = get_variables_and_secrets().await?;
+    let mut vars_and_secrets = get_variables_and_secrets().await?;
 
     if build.config.builder_id.is_empty() {
       return Err(anyhow!("Must attach builder to RunBuild"));
@@ -85,6 +106,14 @@ impl Resolve<RunBuild, (User, Update)> for State {
     update.version = build.config.version;
     update_update(update.clone()).await?;
 
+    // Add the $VERSION to variables. Use with [[$VERSION]]
+    if !vars_and_secrets.variables.contains_key("$VERSION") {
+      vars_and_secrets.variables.insert(
+        String::from("$VERSION"),
+        build.config.version.to_string(),
+      );
+    }
+
     let git_token = git_token(
       &build.config.git_provider,
       &build.config.git_account,
@@ -171,7 +200,6 @@ impl Resolve<RunBuild, (User, Update)> for State {
     };
 
     // CLONE REPO
-
     let secret_replacers = if !build.config.skip_secret_interp {
       // Interpolate variables / secrets into pre build command
       let mut global_replacers = HashSet::new();
@@ -431,7 +459,6 @@ async fn handle_early_return(
   Ok(update)
 }
 
-#[instrument(skip_all)]
 pub async fn validate_cancel_build(
   request: &ExecuteRequest,
 ) -> anyhow::Result<()> {

bin/core/src/api/execute/deployment.rs

@@ -1,6 +1,7 @@
-use std::collections::HashSet;
+use std::{collections::HashSet, sync::OnceLock};
 
 use anyhow::{anyhow, Context};
+use cache::TimeoutCache;
 use formatting::format_serror;
 use komodo_client::{
   api::execute::*,
@@ -9,7 +10,7 @@ use komodo_client::{
     deployment::{
       extract_registry_domain, Deployment, DeploymentImage,
     },
-    get_image_name,
+    get_image_name, komodo_timestamp, optional_string,
     permission::PermissionLevel,
     server::Server,
     update::{Log, Update},
@@ -37,6 +38,30 @@ use crate::{
   state::{action_states, State},
 };
 
+use super::ExecuteRequest;
+
+impl super::BatchExecute for BatchDeploy {
+  type Resource = Deployment;
+  fn single_request(deployment: String) -> ExecuteRequest {
+    ExecuteRequest::Deploy(Deploy {
+      deployment,
+      stop_signal: None,
+      stop_time: None,
+    })
+  }
+}
+
+impl Resolve<BatchDeploy, (User, Update)> for State {
+  #[instrument(name = "BatchDeploy", skip(self, user), fields(user_id = user.id))]
+  async fn resolve(
+    &self,
+    BatchDeploy { pattern }: BatchDeploy,
+    (user, _): (User, Update),
+  ) -> anyhow::Result<BatchExecutionResponse> {
+    super::batch_execute::<BatchDeploy>(&pattern, &user).await
+  }
+}
+
 async fn setup_deployment_execution(
   deployment: &str,
   user: &User,
@@ -49,12 +74,16 @@ async fn setup_deployment_execution(
   .await?;
 
   if deployment.config.server_id.is_empty() {
-    return Err(anyhow!("deployment has no server configured"));
+    return Err(anyhow!("Deployment has no Server configured"));
   }
 
   let server =
     resource::get::<Server>(&deployment.config.server_id).await?;
 
+  if !server.config.enabled {
+    return Err(anyhow!("Attached Server is not enabled"));
+  }
+
   Ok((deployment, server))
 }
 
@@ -86,13 +115,6 @@ impl Resolve<Deploy, (User, Update)> for State {
     // Send update after setting action state, this way frontend gets correct state.
     update_update(update.clone()).await?;
 
-    let periphery = periphery_client(&server)?;
-
-    periphery
-      .health_check()
-      .await
-      .context("Failed server health check, stopping run.")?;
-
     // This block resolves the attached Build to an actual versioned image
     let (version, registry_token) = match &deployment.config.image {
       DeploymentImage::Build { build_id, version } => {
@@ -104,12 +126,7 @@ impl Resolve<Deploy, (User, Update)> for State {
         } else {
           *version
         };
-        // Remove ending patch if it is 0, this means use latest patch.
-        let version_str = if version.patch == 0 {
-          format!("{}.{}", version.major, version.minor)
-        } else {
-          version.to_string()
-        };
+        let version_str = version.to_string();
         // Potentially add the build image_tag postfix
         let version_str = if build.config.image_tag.is_empty() {
           version_str
@@ -217,7 +234,7 @@ impl Resolve<Deploy, (User, Update)> for State {
     update.version = version;
     update_update(update.clone()).await?;
 
-    match periphery
+    match periphery_client(&server)?
       .request(api::container::Deploy {
         deployment,
         stop_signal,
@@ -230,10 +247,8 @@ impl Resolve<Deploy, (User, Update)> for State {
       Ok(log) => update.logs.push(log),
       Err(e) => {
         update.push_error_log(
-          "deploy container",
-          format_serror(
-            &e.context("failed to deploy container").into(),
-          ),
+          "Deploy Container",
+          format_serror(&e.into()),
         );
       }
     };
@@ -247,6 +262,155 @@ impl Resolve<Deploy, (User, Update)> for State {
   }
 }
 
+/// Wait this long after a pull to allow another pull through
+const PULL_TIMEOUT: i64 = 5_000;
+type ServerId = String;
+type Image = String;
+type PullCache = TimeoutCache<(ServerId, Image), Log>;
+
+fn pull_cache() -> &'static PullCache {
+  static PULL_CACHE: OnceLock<PullCache> = OnceLock::new();
+  PULL_CACHE.get_or_init(Default::default)
+}
+
+pub async fn pull_deployment_inner(
+  deployment: Deployment,
+  server: &Server,
+) -> anyhow::Result<Log> {
+  let (image, account, token) = match deployment.config.image {
+    DeploymentImage::Build { build_id, version } => {
+      let build = resource::get::<Build>(&build_id).await?;
+      let image_name = get_image_name(&build)
+        .context("failed to create image name")?;
+      let version = if version.is_none() {
+        build.config.version.to_string()
+      } else {
+        version.to_string()
+      };
+      // Potentially add the build image_tag postfix
+      let version = if build.config.image_tag.is_empty() {
+        version
+      } else {
+        format!("{version}-{}", build.config.image_tag)
+      };
+      // replace image with corresponding build image.
+      let image = format!("{image_name}:{version}");
+      if build.config.image_registry.domain.is_empty() {
+        (image, None, None)
+      } else {
+        let ImageRegistryConfig {
+          domain, account, ..
+        } = build.config.image_registry;
+        let account =
+          if deployment.config.image_registry_account.is_empty() {
+            account
+          } else {
+            deployment.config.image_registry_account
+          };
+        let token = if !account.is_empty() {
+          registry_token(&domain, &account).await.with_context(
+              || format!("Failed to get git token in call to db. Stopping run. | {domain} | {account}"),
+            )?
+        } else {
+          None
+        };
+        (image, optional_string(&account), token)
+      }
+    }
+    DeploymentImage::Image { image } => {
+      let domain = extract_registry_domain(&image)?;
+      let token = if !deployment
+        .config
+        .image_registry_account
+        .is_empty()
+      {
+        registry_token(&domain, &deployment.config.image_registry_account).await.with_context(
+            || format!("Failed to get git token in call to db. Stopping run. | {domain} | {}", deployment.config.image_registry_account),
+          )?
+      } else {
+        None
+      };
+      (
+        image,
+        optional_string(&deployment.config.image_registry_account),
+        token,
+      )
+    }
+  };
+
+  // Acquire the pull lock for this image on the server
+  let lock = pull_cache()
+    .get_lock((server.id.clone(), image.clone()))
+    .await;
+
+  // Lock the path lock, prevents simultaneous pulls by
+  // ensuring simultaneous pulls will wait for first to finish
+  // and checking cached results.
+  let mut locked = lock.lock().await;
+
+  // Early return from cache if lasted pulled with PULL_TIMEOUT
+  if locked.last_ts + PULL_TIMEOUT > komodo_timestamp() {
+    return locked.clone_res();
+  }
+
+  let res = async {
+    let log = match periphery_client(server)?
+      .request(api::image::PullImage {
+        name: image,
+        account,
+        token,
+      })
+      .await
+    {
+      Ok(log) => log,
+      Err(e) => Log::error("Pull image", format_serror(&e.into())),
+    };
+
+    update_cache_for_server(server).await;
+    anyhow::Ok(log)
+  }
+  .await;
+
+  // Set the cache with results. Any other calls waiting on the lock will
+  // then immediately also use this same result.
+  locked.set(&res, komodo_timestamp());
+
+  res
+}
+
+impl Resolve<PullDeployment, (User, Update)> for State {
+  async fn resolve(
+    &self,
+    PullDeployment { deployment }: PullDeployment,
+    (user, mut update): (User, Update),
+  ) -> anyhow::Result<Update> {
+    let (deployment, server) =
+      setup_deployment_execution(&deployment, &user).await?;
+
+    // get the action state for the deployment (or insert default).
+    let action_state = action_states()
+      .deployment
+      .get_or_insert_default(&deployment.id)
+      .await;
+
+    // Will check to ensure deployment not already busy before updating, and return Err if so.
+    // The returned guard will set the action state back to default when dropped.
+    let _action_guard =
+      action_state.update(|state| state.pulling = true)?;
+
+    // Send update after setting action state, this way frontend gets correct state.
+    update_update(update.clone()).await?;
+
+    let log = pull_deployment_inner(deployment, &server).await?;
+
+    update.logs.push(log);
+    update.finalize();
+    update_update(update.clone()).await?;
+
+    Ok(update)
+  }
+}
+
 impl Resolve<StartDeployment, (User, Update)> for State {
   #[instrument(name = "StartDeployment", skip(self, user, update), fields(user_id = user.id, update_id = update.id))]
   async fn resolve(
@@ -271,9 +435,7 @@ impl Resolve<StartDeployment, (User, Update)> for State {
     // Send update after setting action state, this way frontend gets correct state.
     update_update(update.clone()).await?;
 
-    let periphery = periphery_client(&server)?;
-
-    let log = match periphery
+    let log = match periphery_client(&server)?
       .request(api::container::StartContainer {
         name: deployment.name,
       })
@@ -319,9 +481,7 @@ impl Resolve<RestartDeployment, (User, Update)> for State {
     // Send update after setting action state, this way frontend gets correct state.
     update_update(update.clone()).await?;
 
-    let periphery = periphery_client(&server)?;
-
-    let log = match periphery
+    let log = match periphery_client(&server)?
       .request(api::container::RestartContainer {
         name: deployment.name,
       })
@@ -369,9 +529,7 @@ impl Resolve<PauseDeployment, (User, Update)> for State {
     // Send update after setting action state, this way frontend gets correct state.
     update_update(update.clone()).await?;
 
-    let periphery = periphery_client(&server)?;
-
-    let log = match periphery
+    let log = match periphery_client(&server)?
       .request(api::container::PauseContainer {
         name: deployment.name,
       })
@@ -417,9 +575,7 @@ impl Resolve<UnpauseDeployment, (User, Update)> for State {
     // Send update after setting action state, this way frontend gets correct state.
     update_update(update.clone()).await?;
 
-    let periphery = periphery_client(&server)?;
-
-    let log = match periphery
+    let log = match periphery_client(&server)?
       .request(api::container::UnpauseContainer {
         name: deployment.name,
       })
@@ -471,9 +627,7 @@ impl Resolve<StopDeployment, (User, Update)> for State {
     // Send update after setting action state, this way frontend gets correct state.
     update_update(update.clone()).await?;
 
-    let periphery = periphery_client(&server)?;
-
-    let log = match periphery
+    let log = match periphery_client(&server)?
       .request(api::container::StopContainer {
         name: deployment.name,
         signal: signal
@@ -501,6 +655,29 @@ impl Resolve<StopDeployment, (User, Update)> for State {
   }
 }
 
+impl super::BatchExecute for BatchDestroyDeployment {
+  type Resource = Deployment;
+  fn single_request(deployment: String) -> ExecuteRequest {
+    ExecuteRequest::DestroyDeployment(DestroyDeployment {
+      deployment,
+      signal: None,
+      time: None,
+    })
+  }
+}
+
+impl Resolve<BatchDestroyDeployment, (User, Update)> for State {
+  #[instrument(name = "BatchDestroyDeployment", skip(self, user), fields(user_id = user.id))]
+  async fn resolve(
+    &self,
+    BatchDestroyDeployment { pattern }: BatchDestroyDeployment,
+    (user, _): (User, Update),
+  ) -> anyhow::Result<BatchExecutionResponse> {
+    super::batch_execute::<BatchDestroyDeployment>(&pattern, &user)
+      .await
+  }
+}
+
 impl Resolve<DestroyDeployment, (User, Update)> for State {
   #[instrument(name = "DestroyDeployment", skip(self, user, update), fields(user_id = user.id, update_id = update.id))]
   async fn resolve(
@@ -529,9 +706,7 @@ impl Resolve<DestroyDeployment, (User, Update)> for State {
     // Send update after setting action state, this way frontend gets correct state.
     update_update(update.clone()).await?;
 
-    let periphery = periphery_client(&server)?;
-
-    let log = match periphery
+    let log = match periphery_client(&server)?
       .request(api::container::RemoveContainer {
         name: deployment.name,
         signal: signal

bin/core/src/api/execute/mod.rs

@@ -2,12 +2,16 @@ use std::time::Instant;
 
 use anyhow::{anyhow, Context};
 use axum::{middleware, routing::post, Extension, Router};
+use axum_extra::{headers::ContentType, TypedHeader};
+use derive_variants::{EnumVariants, ExtractVariant};
 use formatting::format_serror;
+use futures::future::join_all;
 use komodo_client::{
   api::execute::*,
   entities::{
     update::{Log, Update},
     user::User,
+    Operation,
   },
 };
 use mungos::by_id::find_one_by_id;
@@ -20,9 +24,11 @@ use uuid::Uuid;
 use crate::{
   auth::auth_request,
   helpers::update::{init_execution_update, update_update},
+  resource::{list_full_for_user_using_pattern, KomodoResource},
   state::{db_client, State},
 };
 
+mod action;
 mod build;
 mod deployment;
 mod procedure;
@@ -32,8 +38,15 @@ mod server_template;
 mod stack;
 mod sync;
 
+pub use {
+  deployment::pull_deployment_inner, stack::pull_stack_inner,
+};
+
 #[typeshare]
-#[derive(Serialize, Deserialize, Debug, Clone, Resolver)]
+#[derive(
+  Serialize, Deserialize, Debug, Clone, Resolver, EnumVariants,
+)]
+#[variant_derive(Debug)]
 #[resolver_target(State)]
 #[resolver_args((User, Update))]
 #[serde(tag = "type", content = "params")]
@@ -63,34 +76,51 @@ pub enum ExecuteRequest {
 
   // ==== DEPLOYMENT ====
   Deploy(Deploy),
+  BatchDeploy(BatchDeploy),
+  PullDeployment(PullDeployment),
   StartDeployment(StartDeployment),
   RestartDeployment(RestartDeployment),
   PauseDeployment(PauseDeployment),
   UnpauseDeployment(UnpauseDeployment),
   StopDeployment(StopDeployment),
   DestroyDeployment(DestroyDeployment),
+  BatchDestroyDeployment(BatchDestroyDeployment),
 
   // ==== STACK ====
   DeployStack(DeployStack),
+  BatchDeployStack(BatchDeployStack),
+  DeployStackIfChanged(DeployStackIfChanged),
+  BatchDeployStackIfChanged(BatchDeployStackIfChanged),
+  PullStack(PullStack),
   StartStack(StartStack),
   RestartStack(RestartStack),
   StopStack(StopStack),
   PauseStack(PauseStack),
   UnpauseStack(UnpauseStack),
   DestroyStack(DestroyStack),
+  BatchDestroyStack(BatchDestroyStack),
 
   // ==== BUILD ====
   RunBuild(RunBuild),
+  BatchRunBuild(BatchRunBuild),
   CancelBuild(CancelBuild),
 
   // ==== REPO ====
   CloneRepo(CloneRepo),
+  BatchCloneRepo(BatchCloneRepo),
   PullRepo(PullRepo),
+  BatchPullRepo(BatchPullRepo),
   BuildRepo(BuildRepo),
+  BatchBuildRepo(BatchBuildRepo),
   CancelRepoBuild(CancelRepoBuild),
 
   // ==== PROCEDURE ====
   RunProcedure(RunProcedure),
+  BatchRunProcedure(BatchRunProcedure),
+
+  // ==== ACTION ====
+  RunAction(RunAction),
+  BatchRunAction(BatchRunAction),
 
   // ==== SERVER TEMPLATE ====
   LaunchServer(LaunchServer),
@@ -108,7 +138,25 @@ pub fn router() -> Router {
 async fn handler(
   Extension(user): Extension<User>,
   Json(request): Json<ExecuteRequest>,
-) -> serror::Result<Json<Update>> {
+) -> serror::Result<(TypedHeader<ContentType>, String)> {
+  let res = match inner_handler(request, user).await? {
+    ExecutionResult::Single(update) => serde_json::to_string(&update)
+      .context("Failed to serialize Update")?,
+    ExecutionResult::Batch(res) => res,
+  };
+  Ok((TypedHeader(ContentType::json()), res))
+}
+
+pub enum ExecutionResult {
+  Single(Update),
+  /// The batch contents will be pre serialized here
+  Batch(String),
+}
+
+pub async fn inner_handler(
+  request: ExecuteRequest,
+  user: User,
+) -> anyhow::Result<ExecutionResult> {
   let req_id = Uuid::new_v4();
 
   // need to validate no cancel is active before any update is created.
@@ -116,6 +164,17 @@ async fn handler(
 
   let update = init_execution_update(&request, &user).await?;
 
+  // This will be the case for the Batch exections,
+  // they don't have their own updates.
+  // The batch calls also call "inner_handler" themselves,
+  // and in their case will spawn tasks, so that isn't necessary
+  // here either.
+  if update.operation == Operation::None {
+    return Ok(ExecutionResult::Batch(
+      task(req_id, request, user, update).await?,
+    ));
+  }
+
   let handle =
     tokio::spawn(task(req_id, request, user, update.clone()));
 
@@ -151,10 +210,18 @@ async fn handler(
     }
   });
 
-  Ok(Json(update))
+  Ok(ExecutionResult::Single(update))
 }
 
-#[instrument(name = "ExecuteRequest", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+#[instrument(
+  name = "ExecuteRequest",
+  skip(user, update),
+  fields(
+    user_id = user.id,
+    update_id = update.id,
+    request = format!("{:?}", request.extract_variant()))
+  )
+]
 async fn task(
   req_id: Uuid,
   request: ExecuteRequest,
@@ -183,3 +250,40 @@ async fn task(
 
   res
 }
+
+trait BatchExecute {
+  type Resource: KomodoResource;
+  fn single_request(name: String) -> ExecuteRequest;
+}
+
+async fn batch_execute<E: BatchExecute>(
+  pattern: &str,
+  user: &User,
+) -> anyhow::Result<BatchExecutionResponse> {
+  let resources = list_full_for_user_using_pattern::<E::Resource>(
+    pattern,
+    Default::default(),
+    user,
+    &[],
+  )
+  .await?;
+  let futures = resources.into_iter().map(|resource| {
+    let user = user.clone();
+    async move {
+      inner_handler(E::single_request(resource.name.clone()), user)
+        .await
+        .map(|r| {
+          let ExecutionResult::Single(update) = r else {
+            unreachable!()
+          };
+          update
+        })
+        .map_err(|e| BatchExecutionResponseItemErr {
+          name: resource.name,
+          error: e.into(),
+        })
+        .into()
+    }
+  });
+  Ok(join_all(futures).await)
+}

bin/core/src/api/execute/procedure.rs

@@ -2,7 +2,9 @@ use std::pin::Pin;
 
 use formatting::{bold, colored, format_serror, muted, Color};
 use komodo_client::{
-  api::execute::RunProcedure,
+  api::execute::{
+    BatchExecutionResponse, BatchRunProcedure, RunProcedure,
+  },
   entities::{
     permission::PermissionLevel, procedure::Procedure,
     update::Update, user::User,
@@ -18,6 +20,26 @@ use crate::{
   state::{action_states, db_client, State},
 };
 
+use super::ExecuteRequest;
+
+impl super::BatchExecute for BatchRunProcedure {
+  type Resource = Procedure;
+  fn single_request(procedure: String) -> ExecuteRequest {
+    ExecuteRequest::RunProcedure(RunProcedure { procedure })
+  }
+}
+
+impl Resolve<BatchRunProcedure, (User, Update)> for State {
+  #[instrument(name = "BatchRunProcedure", skip(self, user), fields(user_id = user.id))]
+  async fn resolve(
+    &self,
+    BatchRunProcedure { pattern }: BatchRunProcedure,
+    (user, _): (User, Update),
+  ) -> anyhow::Result<BatchExecutionResponse> {
+    super::batch_execute::<BatchRunProcedure>(&pattern, &user).await
+  }
+}
+
 impl Resolve<RunProcedure, (User, Update)> for State {
   #[instrument(name = "RunProcedure", skip(self, user, update), fields(user_id = user.id, update_id = update.id))]
   async fn resolve(

bin/core/src/api/execute/repo.rs

@@ -3,7 +3,7 @@ use std::{collections::HashSet, future::IntoFuture, time::Duration};
 use anyhow::{anyhow, Context};
 use formatting::format_serror;
 use komodo_client::{
-  api::execute::*,
+  api::{execute::*, write::RefreshRepoCache},
   entities::{
     alert::{Alert, AlertData, SeverityLevel},
     builder::{Builder, BuilderConfig},
@@ -47,6 +47,24 @@ use crate::{
 
 use super::ExecuteRequest;
 
+impl super::BatchExecute for BatchCloneRepo {
+  type Resource = Repo;
+  fn single_request(repo: String) -> ExecuteRequest {
+    ExecuteRequest::CloneRepo(CloneRepo { repo })
+  }
+}
+
+impl Resolve<BatchCloneRepo, (User, Update)> for State {
+  #[instrument(name = "BatchCloneRepo", skip(self, user), fields(user_id = user.id))]
+  async fn resolve(
+    &self,
+    BatchCloneRepo { pattern }: BatchCloneRepo,
+    (user, _): (User, Update),
+  ) -> anyhow::Result<BatchExecutionResponse> {
+    super::batch_execute::<BatchCloneRepo>(&pattern, &user).await
+  }
+}
+
 impl Resolve<CloneRepo, (User, Update)> for State {
   #[instrument(name = "CloneRepo", skip(self, user, update), fields(user_id = user.id, update_id = update.id))]
   async fn resolve(
@@ -123,10 +141,39 @@ impl Resolve<CloneRepo, (User, Update)> for State {
       update_last_pulled_time(&repo.name).await;
     }
 
+    if let Err(e) = State
+      .resolve(RefreshRepoCache { repo: repo.id }, user)
+      .await
+      .context("Failed to refresh repo cache")
+    {
+      update.push_error_log(
+        "Refresh Repo cache",
+        format_serror(&e.into()),
+      );
+    };
+
     handle_server_update_return(update).await
   }
 }
 
+impl super::BatchExecute for BatchPullRepo {
+  type Resource = Repo;
+  fn single_request(repo: String) -> ExecuteRequest {
+    ExecuteRequest::CloneRepo(CloneRepo { repo })
+  }
+}
+
+impl Resolve<BatchPullRepo, (User, Update)> for State {
+  #[instrument(name = "BatchPullRepo", skip(self, user), fields(user_id = user.id))]
+  async fn resolve(
+    &self,
+    BatchPullRepo { pattern }: BatchPullRepo,
+    (user, _): (User, Update),
+  ) -> anyhow::Result<BatchExecutionResponse> {
+    super::batch_execute::<BatchPullRepo>(&pattern, &user).await
+  }
+}
+
 impl Resolve<PullRepo, (User, Update)> for State {
   #[instrument(name = "PullRepo", skip(self, user, update), fields(user_id = user.id, update_id = update.id))]
   async fn resolve(
@@ -207,6 +254,17 @@ impl Resolve<PullRepo, (User, Update)> for State {
       update_last_pulled_time(&repo.name).await;
     }
 
+    if let Err(e) = State
+      .resolve(RefreshRepoCache { repo: repo.id }, user)
+      .await
+      .context("Failed to refresh repo cache")
+    {
+      update.push_error_log(
+        "Refresh Repo cache",
+        format_serror(&e.into()),
+      );
+    };
+
     handle_server_update_return(update).await
   }
 }
@@ -249,6 +307,24 @@ async fn update_last_pulled_time(repo_name: &str) {
   }
 }
 
+impl super::BatchExecute for BatchBuildRepo {
+  type Resource = Repo;
+  fn single_request(repo: String) -> ExecuteRequest {
+    ExecuteRequest::CloneRepo(CloneRepo { repo })
+  }
+}
+
+impl Resolve<BatchBuildRepo, (User, Update)> for State {
+  #[instrument(name = "BatchBuildRepo", skip(self, user), fields(user_id = user.id))]
+  async fn resolve(
+    &self,
+    BatchBuildRepo { pattern }: BatchBuildRepo,
+    (user, _): (User, Update),
+  ) -> anyhow::Result<BatchExecutionResponse> {
+    super::batch_execute::<BatchBuildRepo>(&pattern, &user).await
+  }
+}
+
 impl Resolve<BuildRepo, (User, Update)> for State {
   #[instrument(name = "BuildRepo", skip(self, user, update), fields(user_id = user.id, update_id = update.id))]
   async fn resolve(

bin/core/src/api/execute/server.rs

@@ -425,7 +425,7 @@ impl Resolve<RestartAllContainers, (User, Update)> for State {
     update_update(update.clone()).await?;
 
     let logs = periphery_client(&server)?
-      .request(api::container::StartAllContainers {})
+      .request(api::container::RestartAllContainers {})
       .await
       .context("failed to restart all containers on host")?;
 
@@ -520,12 +520,12 @@ impl Resolve<UnpauseAllContainers, (User, Update)> for State {
     // Will check to ensure server not already busy before updating, and return Err if so.
     // The returned guard will set the action state back to default when dropped.
     let _action_guard = action_state
-      .update(|state| state.starting_containers = true)?;
+      .update(|state| state.unpausing_containers = true)?;
 
     update_update(update.clone()).await?;
 
     let logs = periphery_client(&server)?
-      .request(api::container::StartAllContainers {})
+      .request(api::container::UnpauseAllContainers {})
       .await
       .context("failed to unpause all containers on host")?;
 

bin/core/src/api/execute/stack.rs

@@ -3,9 +3,12 @@ use std::collections::HashSet;
 use anyhow::Context;
 use formatting::format_serror;
 use komodo_client::{
-  api::execute::*,
+  api::{execute::*, write::RefreshStackCache},
   entities::{
-    permission::PermissionLevel, stack::StackInfo, update::Update,
+    permission::PermissionLevel,
+    server::Server,
+    stack::{Stack, StackInfo},
+    update::{Log, Update},
     user::User,
   },
 };
@@ -19,24 +22,51 @@ use crate::{
       add_interp_update_log,
       interpolate_variables_secrets_into_extra_args,
       interpolate_variables_secrets_into_string,
+      interpolate_variables_secrets_into_system_command,
     },
     periphery_client,
     query::get_variables_and_secrets,
-    update::update_update,
+    update::{add_update_without_send, update_update},
   },
   monitor::update_cache_for_server,
-  stack::{
-    execute::execute_compose, get_stack_and_server,
-    services::extract_services_into_res,
-  },
+  resource,
+  stack::{execute::execute_compose, get_stack_and_server},
   state::{action_states, db_client, State},
 };
 
+use super::ExecuteRequest;
+
+impl super::BatchExecute for BatchDeployStack {
+  type Resource = Stack;
+  fn single_request(stack: String) -> ExecuteRequest {
+    ExecuteRequest::DeployStack(DeployStack {
+      stack,
+      service: None,
+      stop_time: None,
+    })
+  }
+}
+
+impl Resolve<BatchDeployStack, (User, Update)> for State {
+  #[instrument(name = "BatchDeployStack", skip(self, user), fields(user_id = user.id))]
+  async fn resolve(
+    &self,
+    BatchDeployStack { pattern }: BatchDeployStack,
+    (user, _): (User, Update),
+  ) -> anyhow::Result<BatchExecutionResponse> {
+    super::batch_execute::<BatchDeployStack>(&pattern, &user).await
+  }
+}
+
 impl Resolve<DeployStack, (User, Update)> for State {
   #[instrument(name = "DeployStack", skip(self, user, update), fields(user_id = user.id, update_id = update.id))]
   async fn resolve(
     &self,
-    DeployStack { stack, stop_time }: DeployStack,
+    DeployStack {
+      stack,
+      service,
+      stop_time,
+    }: DeployStack,
     (user, mut update): (User, Update),
   ) -> anyhow::Result<Update> {
     let (mut stack, server) = get_stack_and_server(
@@ -58,6 +88,13 @@ impl Resolve<DeployStack, (User, Update)> for State {
 
     update_update(update.clone()).await?;
 
+    if let Some(service) = &service {
+      update.logs.push(Log::simple(
+        &format!("Service: {service}"),
+        format!("Execution requested for Stack service {service}"),
+      ))
+    }
+
     let git_token = crate::helpers::git_token(
       &stack.config.git_provider,
       &stack.config.git_account,
@@ -81,6 +118,13 @@ impl Resolve<DeployStack, (User, Update)> for State {
       let mut global_replacers = HashSet::new();
       let mut secret_replacers = HashSet::new();
 
+      interpolate_variables_secrets_into_string(
+        &vars_and_secrets,
+        &mut stack.config.file_contents,
+        &mut global_replacers,
+        &mut secret_replacers,
+      )?;
+
       interpolate_variables_secrets_into_string(
         &vars_and_secrets,
         &mut stack.config.environment,
@@ -102,6 +146,13 @@ impl Resolve<DeployStack, (User, Update)> for State {
         &mut secret_replacers,
       )?;
 
+      interpolate_variables_secrets_into_system_command(
+        &vars_and_secrets,
+        &mut stack.config.pre_deploy,
+        &mut global_replacers,
+        &mut secret_replacers,
+      )?;
+
       add_interp_update_log(
         &mut update,
         &global_replacers,
@@ -116,6 +167,7 @@ impl Resolve<DeployStack, (User, Update)> for State {
     let ComposeUpResponse {
       logs,
       deployed,
+      services,
       file_contents,
       missing_files,
       remote_errors,
@@ -124,7 +176,7 @@ impl Resolve<DeployStack, (User, Update)> for State {
     } = periphery_client(&server)?
       .request(ComposeUp {
         stack: stack.clone(),
-        service: None,
+        service,
         git_token,
         registry_token,
         replacers: secret_replacers.into_iter().collect(),
@@ -134,24 +186,11 @@ impl Resolve<DeployStack, (User, Update)> for State {
     update.logs.extend(logs);
 
     let update_info = async {
-      let latest_services = if !file_contents.is_empty() {
-        let mut services = Vec::new();
-        for contents in &file_contents {
-          if let Err(e) = extract_services_into_res(
-            &stack.project_name(true),
-            &contents.contents,
-            &mut services,
-          ) {
-            update.push_error_log(
-              "extract services",
-              format_serror(&e.context(format!("Failed to extract stack services for compose file path {}. Things probably won't work correctly", contents.path)).into())
-            );
-          }
-        }
-        services
-      } else {
+      let latest_services = if services.is_empty() {
         // maybe better to do something else here for services.
         stack.info.latest_services.clone()
+      } else {
+        services
       };
 
       // This ensures to get the latest project name,
@@ -235,6 +274,182 @@ impl Resolve<DeployStack, (User, Update)> for State {
   }
 }
 
+impl super::BatchExecute for BatchDeployStackIfChanged {
+  type Resource = Stack;
+  fn single_request(stack: String) -> ExecuteRequest {
+    ExecuteRequest::DeployStackIfChanged(DeployStackIfChanged {
+      stack,
+      stop_time: None,
+    })
+  }
+}
+
+impl Resolve<BatchDeployStackIfChanged, (User, Update)> for State {
+  #[instrument(name = "BatchDeployStackIfChanged", skip(self, user), fields(user_id = user.id))]
+  async fn resolve(
+    &self,
+    BatchDeployStackIfChanged { pattern }: BatchDeployStackIfChanged,
+    (user, _): (User, Update),
+  ) -> anyhow::Result<BatchExecutionResponse> {
+    super::batch_execute::<BatchDeployStackIfChanged>(&pattern, &user)
+      .await
+  }
+}
+
+impl Resolve<DeployStackIfChanged, (User, Update)> for State {
+  #[instrument(name = "DeployStackIfChanged", skip(self, user, update), fields(user_id = user.id, update_id = update.id))]
+  async fn resolve(
+    &self,
+    DeployStackIfChanged { stack, stop_time }: DeployStackIfChanged,
+    (user, mut update): (User, Update),
+  ) -> anyhow::Result<Update> {
+    let stack = resource::get_check_permissions::<Stack>(
+      &stack,
+      &user,
+      PermissionLevel::Execute,
+    )
+    .await?;
+    State
+      .resolve(
+        RefreshStackCache {
+          stack: stack.id.clone(),
+        },
+        user.clone(),
+      )
+      .await?;
+    let stack = resource::get::<Stack>(&stack.id).await?;
+    let changed = match (
+      &stack.info.deployed_contents,
+      &stack.info.remote_contents,
+    ) {
+      (Some(deployed_contents), Some(latest_contents)) => {
+        let changed = || {
+          for latest in latest_contents {
+            let Some(deployed) = deployed_contents
+              .iter()
+              .find(|c| c.path == latest.path)
+            else {
+              return true;
+            };
+            if latest.contents != deployed.contents {
+              return true;
+            }
+          }
+          false
+        };
+        changed()
+      }
+      (None, _) => true,
+      _ => false,
+    };
+
+    if !changed {
+      update.push_simple_log(
+        "Diff compose files",
+        String::from("Deploy cancelled after no changes detected."),
+      );
+      update.finalize();
+      return Ok(update);
+    }
+
+    // Don't actually send it here, let the handler send it after it can set action state.
+    // This is usually done in crate::helpers::update::init_execution_update.
+    update.id = add_update_without_send(&update).await?;
+
+    State
+      .resolve(
+        DeployStack {
+          stack: stack.name,
+          service: None,
+          stop_time,
+        },
+        (user, update),
+      )
+      .await
+  }
+}
+
+pub async fn pull_stack_inner(
+  mut stack: Stack,
+  service: Option<String>,
+  server: &Server,
+  update: Option<&mut Update>,
+) -> anyhow::Result<ComposePullResponse> {
+  if let (Some(service), Some(update)) = (&service, update) {
+    update.logs.push(Log::simple(
+      &format!("Service: {service}"),
+      format!("Execution requested for Stack service {service}"),
+    ))
+  }
+
+  let git_token = crate::helpers::git_token(
+      &stack.config.git_provider,
+      &stack.config.git_account,
+      |https| stack.config.git_https = https,
+    ).await.with_context(
+      || format!("Failed to get git token in call to db. Stopping run. | {} | {}", stack.config.git_provider, stack.config.git_account),
+    )?;
+
+  let registry_token = crate::helpers::registry_token(
+      &stack.config.registry_provider,
+      &stack.config.registry_account,
+    ).await.with_context(
+      || format!("Failed to get registry token in call to db. Stopping run. | {} | {}", stack.config.registry_provider, stack.config.registry_account),
+    )?;
+
+  let res = periphery_client(server)?
+    .request(ComposePull {
+      stack,
+      service,
+      git_token,
+      registry_token,
+    })
+    .await?;
+
+  // Ensure cached stack state up to date by updating server cache
+  update_cache_for_server(server).await;
+
+  Ok(res)
+}
+
+impl Resolve<PullStack, (User, Update)> for State {
+  #[instrument(name = "PullStack", skip(self, user, update), fields(user_id = user.id, update_id = update.id))]
+  async fn resolve(
+    &self,
+    PullStack { stack, service }: PullStack,
+    (user, mut update): (User, Update),
+  ) -> anyhow::Result<Update> {
+    let (stack, server) = get_stack_and_server(
+      &stack,
+      &user,
+      PermissionLevel::Execute,
+      true,
+    )
+    .await?;
+
+    // get the action state for the stack (or insert default).
+    let action_state =
+      action_states().stack.get_or_insert_default(&stack.id).await;
+
+    // Will check to ensure stack not already busy before updating, and return Err if so.
+    // The returned guard will set the action state back to default when dropped.
+    let _action_guard =
+      action_state.update(|state| state.pulling = true)?;
+
+    update_update(update.clone()).await?;
+
+    let res =
+      pull_stack_inner(stack, service, &server, Some(&mut update))
+        .await?;
+
+    update.logs.extend(res.logs);
+    update.finalize();
+    update_update(update.clone()).await?;
+
+    Ok(update)
+  }
+}
+
 impl Resolve<StartStack, (User, Update)> for State {
   #[instrument(name = "StartStack", skip(self, user, update), fields(user_id = user.id, update_id = update.id))]
   async fn resolve(
@@ -336,12 +551,36 @@ impl Resolve<StopStack, (User, Update)> for State {
   }
 }
 
+impl super::BatchExecute for BatchDestroyStack {
+  type Resource = Stack;
+  fn single_request(stack: String) -> ExecuteRequest {
+    ExecuteRequest::DestroyStack(DestroyStack {
+      stack,
+      service: None,
+      remove_orphans: false,
+      stop_time: None,
+    })
+  }
+}
+
+impl Resolve<BatchDestroyStack, (User, Update)> for State {
+  #[instrument(name = "BatchDestroyStack", skip(self, user), fields(user_id = user.id))]
+  async fn resolve(
+    &self,
+    BatchDestroyStack { pattern }: BatchDestroyStack,
+    (user, _): (User, Update),
+  ) -> anyhow::Result<BatchExecutionResponse> {
+    super::batch_execute::<BatchDestroyStack>(&pattern, &user).await
+  }
+}
+
 impl Resolve<DestroyStack, (User, Update)> for State {
   #[instrument(name = "DestroyStack", skip(self, user, update), fields(user_id = user.id, update_id = update.id))]
   async fn resolve(
     &self,
     DestroyStack {
       stack,
+      service,
       remove_orphans,
       stop_time,
     }: DestroyStack,
@@ -349,7 +588,7 @@ impl Resolve<DestroyStack, (User, Update)> for State {
   ) -> anyhow::Result<Update> {
     execute_compose::<DestroyStack>(
       &stack,
-      None,
+      service,
       &user,
       |state| state.destroying = true,
       update,

bin/core/src/api/execute/sync.rs

@@ -1,4 +1,4 @@
-use std::collections::HashMap;
+use std::{collections::HashMap, str::FromStr};
 
 use anyhow::{anyhow, Context};
 use formatting::{colored, format_serror, Color};
@@ -6,6 +6,7 @@ use komodo_client::{
   api::{execute::RunSync, write::RefreshResourceSyncPending},
   entities::{
     self,
+    action::Action,
     alerter::Alerter,
     build::Build,
     builder::Builder,
@@ -20,23 +21,27 @@ use komodo_client::{
     sync::ResourceSync,
     update::{Log, Update},
     user::{sync_user, User},
+    ResourceTargetVariant,
   },
 };
 use mongo_indexed::doc;
-use mungos::{by_id::update_one_by_id, mongodb::bson::to_document};
+use mungos::{
+  by_id::update_one_by_id,
+  mongodb::bson::{oid::ObjectId, to_document},
+};
 use resolver_api::Resolve;
 
 use crate::{
   helpers::{query::get_id_to_tags, update::update_update},
   resource::{self, refresh_resource_sync_state_cache},
-  state::{db_client, State},
+  state::{action_states, db_client, State},
   sync::{
     deploy::{
       build_deploy_cache, deploy_from_cache, SyncDeployParams,
     },
     execute::{get_updates_for_execution, ExecuteResourceSync},
     remote::RemoteResources,
-    AllResourcesById,
+    AllResourcesById, ResourceSyncTrait,
   },
 };
 
@@ -44,7 +49,11 @@ impl Resolve<RunSync, (User, Update)> for State {
   #[instrument(name = "RunSync", skip(self, user, update), fields(user_id = user.id, update_id = update.id))]
   async fn resolve(
     &self,
-    RunSync { sync }: RunSync,
+    RunSync {
+      sync,
+      resource_type: match_resource_type,
+      resources: match_resources,
+    }: RunSync,
     (user, mut update): (User, Update),
   ) -> anyhow::Result<Update> {
     let sync = resource::get_check_permissions::<
@@ -52,6 +61,17 @@ impl Resolve<RunSync, (User, Update)> for State {
     >(&sync, &user, PermissionLevel::Execute)
     .await?;
 
+    // get the action state for the sync (or insert default).
+    let action_state = action_states()
+      .resource_sync
+      .get_or_insert_default(&sync.id)
+      .await;
+
+    // This will set action state back to default when dropped.
+    // Will also check to ensure sync not already busy before updating.
+    let _action_guard =
+      action_state.update(|state| state.syncing = true)?;
+
     // Send update here for FE to recheck action state
     update_update(update.clone()).await?;
 
@@ -70,22 +90,105 @@ impl Resolve<RunSync, (User, Update)> for State {
     update_update(update.clone()).await?;
 
     if !file_errors.is_empty() {
-      return Err(anyhow!("Found file errors. Cannot execute sync."))
+      return Err(anyhow!("Found file errors. Cannot execute sync."));
     }
 
     let resources = resources?;
 
     let id_to_tags = get_id_to_tags(None).await?;
     let all_resources = AllResourcesById::load().await?;
+    // Convert all match_resources to names
+    let match_resources = match_resources.map(|resources| {
+      resources
+        .into_iter()
+        .filter_map(|name_or_id| {
+          let Some(resource_type) = match_resource_type else {
+            return Some(name_or_id);
+          };
+          match ObjectId::from_str(&name_or_id) {
+            Ok(_) => match resource_type {
+              ResourceTargetVariant::Alerter => all_resources
+                .alerters
+                .get(&name_or_id)
+                .map(|a| a.name.clone()),
+              ResourceTargetVariant::Build => all_resources
+                .builds
+                .get(&name_or_id)
+                .map(|b| b.name.clone()),
+              ResourceTargetVariant::Builder => all_resources
+                .builders
+                .get(&name_or_id)
+                .map(|b| b.name.clone()),
+              ResourceTargetVariant::Deployment => all_resources
+                .deployments
+                .get(&name_or_id)
+                .map(|d| d.name.clone()),
+              ResourceTargetVariant::Procedure => all_resources
+                .procedures
+                .get(&name_or_id)
+                .map(|p| p.name.clone()),
+              ResourceTargetVariant::Action => all_resources
+                .actions
+                .get(&name_or_id)
+                .map(|p| p.name.clone()),
+              ResourceTargetVariant::Repo => all_resources
+                .repos
+                .get(&name_or_id)
+                .map(|r| r.name.clone()),
+              ResourceTargetVariant::Server => all_resources
+                .servers
+                .get(&name_or_id)
+                .map(|s| s.name.clone()),
+              ResourceTargetVariant::ServerTemplate => all_resources
+                .templates
+                .get(&name_or_id)
+                .map(|t| t.name.clone()),
+              ResourceTargetVariant::Stack => all_resources
+                .stacks
+                .get(&name_or_id)
+                .map(|s| s.name.clone()),
+              ResourceTargetVariant::ResourceSync => all_resources
+                .syncs
+                .get(&name_or_id)
+                .map(|s| s.name.clone()),
+              ResourceTargetVariant::System => None,
+            },
+            Err(_) => Some(name_or_id),
+          }
+        })
+        .collect::<Vec<_>>()
+    });
 
     let deployments_by_name = all_resources
       .deployments
       .values()
+      .filter(|deployment| {
+        Deployment::include_resource(
+          &deployment.name,
+          &deployment.config,
+          match_resource_type,
+          match_resources.as_deref(),
+          &deployment.tags,
+          &id_to_tags,
+          &sync.config.match_tags,
+        )
+      })
       .map(|deployment| (deployment.name.clone(), deployment.clone()))
       .collect::<HashMap<_, _>>();
     let stacks_by_name = all_resources
       .stacks
       .values()
+      .filter(|stack| {
+        Stack::include_resource(
+          &stack.name,
+          &stack.config,
+          match_resource_type,
+          match_resources.as_deref(),
+          &stack.tags,
+          &id_to_tags,
+          &sync.config.match_tags,
+        )
+      })
       .map(|stack| (stack.name.clone(), stack.clone()))
       .collect::<HashMap<_, _>>();
 
@@ -105,6 +208,8 @@ impl Resolve<RunSync, (User, Update)> for State {
         resources.servers,
         delete,
         &all_resources,
+        match_resource_type,
+        match_resources.as_deref(),
         &id_to_tags,
         &sync.config.match_tags,
       )
@@ -117,6 +222,8 @@ impl Resolve<RunSync, (User, Update)> for State {
       resources.deployments,
       delete,
       &all_resources,
+      match_resource_type,
+      match_resources.as_deref(),
       &id_to_tags,
       &sync.config.match_tags,
     )
@@ -126,6 +233,8 @@ impl Resolve<RunSync, (User, Update)> for State {
         resources.stacks,
         delete,
         &all_resources,
+        match_resource_type,
+        match_resources.as_deref(),
         &id_to_tags,
         &sync.config.match_tags,
       )
@@ -135,6 +244,8 @@ impl Resolve<RunSync, (User, Update)> for State {
         resources.builds,
         delete,
         &all_resources,
+        match_resource_type,
+        match_resources.as_deref(),
         &id_to_tags,
         &sync.config.match_tags,
       )
@@ -144,6 +255,8 @@ impl Resolve<RunSync, (User, Update)> for State {
         resources.repos,
         delete,
         &all_resources,
+        match_resource_type,
+        match_resources.as_deref(),
         &id_to_tags,
         &sync.config.match_tags,
       )
@@ -156,15 +269,30 @@ impl Resolve<RunSync, (User, Update)> for State {
       resources.procedures,
       delete,
       &all_resources,
+      match_resource_type,
+      match_resources.as_deref(),
       &id_to_tags,
       &sync.config.match_tags,
     )
     .await?;
+    let (actions_to_create, actions_to_update, actions_to_delete) =
+      get_updates_for_execution::<Action>(
+        resources.actions,
+        delete,
+        &all_resources,
+        match_resource_type,
+        match_resources.as_deref(),
+        &id_to_tags,
+        &sync.config.match_tags,
+      )
+      .await?;
     let (builders_to_create, builders_to_update, builders_to_delete) =
       get_updates_for_execution::<Builder>(
         resources.builders,
         delete,
         &all_resources,
+        match_resource_type,
+        match_resources.as_deref(),
         &id_to_tags,
         &sync.config.match_tags,
       )
@@ -174,6 +302,8 @@ impl Resolve<RunSync, (User, Update)> for State {
         resources.alerters,
         delete,
         &all_resources,
+        match_resource_type,
+        match_resources.as_deref(),
         &id_to_tags,
         &sync.config.match_tags,
       )
@@ -186,6 +316,8 @@ impl Resolve<RunSync, (User, Update)> for State {
       resources.server_templates,
       delete,
       &all_resources,
+      match_resource_type,
+      match_resources.as_deref(),
       &id_to_tags,
       &sync.config.match_tags,
     )
@@ -198,31 +330,48 @@ impl Resolve<RunSync, (User, Update)> for State {
       resources.resource_syncs,
       delete,
       &all_resources,
+      match_resource_type,
+      match_resources.as_deref(),
       &id_to_tags,
       &sync.config.match_tags,
     )
     .await?;
+
     let (
       variables_to_create,
       variables_to_update,
       variables_to_delete,
-    ) = crate::sync::variables::get_updates_for_execution(
-      resources.variables,
-      // Delete doesn't work with variables when match tags are set
-      sync.config.match_tags.is_empty() && delete,
-    )
-    .await?;
+    ) = if match_resource_type.is_none()
+      && match_resources.is_none()
+      && sync.config.match_tags.is_empty()
+    {
+      crate::sync::variables::get_updates_for_execution(
+        resources.variables,
+        // Delete doesn't work with variables when match tags are set
+        sync.config.match_tags.is_empty() && delete,
+      )
+      .await?
+    } else {
+      Default::default()
+    };
     let (
       user_groups_to_create,
       user_groups_to_update,
       user_groups_to_delete,
-    ) = crate::sync::user_groups::get_updates_for_execution(
-      resources.user_groups,
-      // Delete doesn't work with user groups when match tags are set
-      sync.config.match_tags.is_empty() && delete,
-      &all_resources,
-    )
-    .await?;
+    ) = if match_resource_type.is_none()
+      && match_resources.is_none()
+      && sync.config.match_tags.is_empty()
+    {
+      crate::sync::user_groups::get_updates_for_execution(
+        resources.user_groups,
+        // Delete doesn't work with user groups when match tags are set
+        sync.config.match_tags.is_empty() && delete,
+        &all_resources,
+      )
+      .await?
+    } else {
+      Default::default()
+    };
 
     if deploy_cache.is_empty()
       && resource_syncs_to_create.is_empty()
@@ -255,6 +404,9 @@ impl Resolve<RunSync, (User, Update)> for State {
       && procedures_to_create.is_empty()
       && procedures_to_update.is_empty()
       && procedures_to_delete.is_empty()
+      && actions_to_create.is_empty()
+      && actions_to_update.is_empty()
+      && actions_to_delete.is_empty()
       && user_groups_to_create.is_empty()
       && user_groups_to_update.is_empty()
       && user_groups_to_delete.is_empty()
@@ -331,6 +483,15 @@ impl Resolve<RunSync, (User, Update)> for State {
       )
       .await,
     );
+    maybe_extend(
+      &mut update.logs,
+      Action::execute_sync_updates(
+        actions_to_create,
+        actions_to_update,
+        actions_to_delete,
+      )
+      .await,
+    );
 
     // Dependent on server
     maybe_extend(

bin/core/src/api/read/action.rs

@@ -0,0 +1,132 @@
+use anyhow::Context;
+use komodo_client::{
+  api::read::*,
+  entities::{
+    action::{
+      Action, ActionActionState, ActionListItem, ActionState,
+    },
+    permission::PermissionLevel,
+    user::User,
+  },
+};
+use resolver_api::Resolve;
+
+use crate::{
+  helpers::query::get_all_tags,
+  resource,
+  state::{action_state_cache, action_states, State},
+};
+
+impl Resolve<GetAction, User> for State {
+  async fn resolve(
+    &self,
+    GetAction { action }: GetAction,
+    user: User,
+  ) -> anyhow::Result<Action> {
+    resource::get_check_permissions::<Action>(
+      &action,
+      &user,
+      PermissionLevel::Read,
+    )
+    .await
+  }
+}
+
+impl Resolve<ListActions, User> for State {
+  async fn resolve(
+    &self,
+    ListActions { query }: ListActions,
+    user: User,
+  ) -> anyhow::Result<Vec<ActionListItem>> {
+    let all_tags = if query.tags.is_empty() {
+      vec![]
+    } else {
+      get_all_tags(None).await?
+    };
+    resource::list_for_user::<Action>(query, &user, &all_tags).await
+  }
+}
+
+impl Resolve<ListFullActions, User> for State {
+  async fn resolve(
+    &self,
+    ListFullActions { query }: ListFullActions,
+    user: User,
+  ) -> anyhow::Result<ListFullActionsResponse> {
+    let all_tags = if query.tags.is_empty() {
+      vec![]
+    } else {
+      get_all_tags(None).await?
+    };
+    resource::list_full_for_user::<Action>(query, &user, &all_tags)
+      .await
+  }
+}
+
+impl Resolve<GetActionActionState, User> for State {
+  async fn resolve(
+    &self,
+    GetActionActionState { action }: GetActionActionState,
+    user: User,
+  ) -> anyhow::Result<ActionActionState> {
+    let action = resource::get_check_permissions::<Action>(
+      &action,
+      &user,
+      PermissionLevel::Read,
+    )
+    .await?;
+    let action_state = action_states()
+      .action
+      .get(&action.id)
+      .await
+      .unwrap_or_default()
+      .get()?;
+    Ok(action_state)
+  }
+}
+
+impl Resolve<GetActionsSummary, User> for State {
+  async fn resolve(
+    &self,
+    GetActionsSummary {}: GetActionsSummary,
+    user: User,
+  ) -> anyhow::Result<GetActionsSummaryResponse> {
+    let actions = resource::list_full_for_user::<Action>(
+      Default::default(),
+      &user,
+      &[],
+    )
+    .await
+    .context("failed to get actions from db")?;
+
+    let mut res = GetActionsSummaryResponse::default();
+
+    let cache = action_state_cache();
+    let action_states = action_states();
+
+    for action in actions {
+      res.total += 1;
+
+      match (
+        cache.get(&action.id).await.unwrap_or_default(),
+        action_states
+          .action
+          .get(&action.id)
+          .await
+          .unwrap_or_default()
+          .get()?,
+      ) {
+        (_, action_states) if action_states.running => {
+          res.running += 1;
+        }
+        (ActionState::Ok, _) => res.ok += 1,
+        (ActionState::Failed, _) => res.failed += 1,
+        (ActionState::Unknown, _) => res.unknown += 1,
+        // will never come off the cache in the running state, since that comes from action states
+        (ActionState::Running, _) => unreachable!(),
+      }
+    }
+
+    Ok(res)
+  }
+}

bin/core/src/api/read/alert.rs

@@ -3,7 +3,10 @@ use komodo_client::{
   api::read::{
     GetAlert, GetAlertResponse, ListAlerts, ListAlertsResponse,
   },
-  entities::{deployment::Deployment, server::Server, user::User},
+  entities::{
+    deployment::Deployment, server::Server, stack::Stack,
+    sync::ResourceSync, user::User,
+  },
 };
 use mungos::{
   by_id::find_one_by_id,
@@ -30,12 +33,18 @@ impl Resolve<ListAlerts, User> for State {
     if !user.admin && !core_config().transparent_mode {
       let server_ids =
         get_resource_ids_for_user::<Server>(&user).await?;
+      let stack_ids =
+        get_resource_ids_for_user::<Stack>(&user).await?;
       let deployment_ids =
         get_resource_ids_for_user::<Deployment>(&user).await?;
+      let sync_ids =
+        get_resource_ids_for_user::<ResourceSync>(&user).await?;
       query.extend(doc! {
         "$or": [
           { "target.type": "Server", "target.id": { "$in": &server_ids } },
+          { "target.type": "Stack", "target.id": { "$in": &stack_ids } },
           { "target.type": "Deployment", "target.id": { "$in": &deployment_ids } },
+          { "target.type": "ResourceSync", "target.id": { "$in": &sync_ids } },
         ]
       });
     }

bin/core/src/api/read/alerter.rs

@@ -12,6 +12,7 @@ use mungos::mongodb::bson::doc;
 use resolver_api::Resolve;
 
 use crate::{
+  helpers::query::get_all_tags,
   resource,
   state::{db_client, State},
 };
@@ -37,7 +38,12 @@ impl Resolve<ListAlerters, User> for State {
     ListAlerters { query }: ListAlerters,
     user: User,
   ) -> anyhow::Result<Vec<AlerterListItem>> {
-    resource::list_for_user::<Alerter>(query, &user).await
+    let all_tags = if query.tags.is_empty() {
+      vec![]
+    } else {
+      get_all_tags(None).await?
+    };
+    resource::list_for_user::<Alerter>(query, &user, &all_tags).await
   }
 }
 
@@ -47,7 +53,13 @@ impl Resolve<ListFullAlerters, User> for State {
     ListFullAlerters { query }: ListFullAlerters,
     user: User,
   ) -> anyhow::Result<ListFullAlertersResponse> {
-    resource::list_full_for_user::<Alerter>(query, &user).await
+    let all_tags = if query.tags.is_empty() {
+      vec![]
+    } else {
+      get_all_tags(None).await?
+    };
+    resource::list_full_for_user::<Alerter>(query, &user, &all_tags)
+      .await
   }
 }
 
@@ -57,15 +69,16 @@ impl Resolve<GetAlertersSummary, User> for State {
     GetAlertersSummary {}: GetAlertersSummary,
     user: User,
   ) -> anyhow::Result<GetAlertersSummaryResponse> {
-    let query =
-      match resource::get_resource_ids_for_user::<Alerter>(&user)
-        .await?
-      {
-        Some(ids) => doc! {
-          "_id": { "$in": ids }
-        },
-        None => Document::new(),
-      };
+    let query = match resource::get_resource_object_ids_for_user::<
+      Alerter,
+    >(&user)
+    .await?
+    {
+      Some(ids) => doc! {
+        "_id": { "$in": ids }
+      },
+      None => Document::new(),
+    };
     let total = db_client()
       .alerters
       .count_documents(query)

bin/core/src/api/read/build.rs

@@ -22,6 +22,7 @@ use resolver_api::Resolve;
 
 use crate::{
   config::core_config,
+  helpers::query::get_all_tags,
   resource,
   state::{
     action_states, build_state_cache, db_client, github_client, State,
@@ -49,7 +50,12 @@ impl Resolve<ListBuilds, User> for State {
     ListBuilds { query }: ListBuilds,
     user: User,
   ) -> anyhow::Result<Vec<BuildListItem>> {
-    resource::list_for_user::<Build>(query, &user).await
+    let all_tags = if query.tags.is_empty() {
+      vec![]
+    } else {
+      get_all_tags(None).await?
+    };
+    resource::list_for_user::<Build>(query, &user, &all_tags).await
   }
 }
 
@@ -59,7 +65,13 @@ impl Resolve<ListFullBuilds, User> for State {
     ListFullBuilds { query }: ListFullBuilds,
     user: User,
   ) -> anyhow::Result<ListFullBuildsResponse> {
-    resource::list_full_for_user::<Build>(query, &user).await
+    let all_tags = if query.tags.is_empty() {
+      vec![]
+    } else {
+      get_all_tags(None).await?
+    };
+    resource::list_full_for_user::<Build>(query, &user, &all_tags)
+      .await
   }
 }
 
@@ -94,6 +106,7 @@ impl Resolve<GetBuildsSummary, User> for State {
     let builds = resource::list_full_for_user::<Build>(
       Default::default(),
       &user,
+      &[],
     )
     .await
     .context("failed to get all builds")?;
@@ -252,9 +265,15 @@ impl Resolve<ListCommonBuildExtraArgs, User> for State {
     ListCommonBuildExtraArgs { query }: ListCommonBuildExtraArgs,
     user: User,
   ) -> anyhow::Result<ListCommonBuildExtraArgsResponse> {
-    let builds = resource::list_full_for_user::<Build>(query, &user)
-      .await
-      .context("failed to get resources matching query")?;
+    let all_tags = if query.tags.is_empty() {
+      vec![]
+    } else {
+      get_all_tags(None).await?
+    };
+    let builds =
+      resource::list_full_for_user::<Build>(query, &user, &all_tags)
+        .await
+        .context("failed to get resources matching query")?;
 
     // first collect with guaranteed uniqueness
     let mut res = HashSet::<String>::new();

bin/core/src/api/read/builder.rs

@@ -12,6 +12,7 @@ use mungos::mongodb::bson::doc;
 use resolver_api::Resolve;
 
 use crate::{
+  helpers::query::get_all_tags,
   resource,
   state::{db_client, State},
 };
@@ -37,7 +38,12 @@ impl Resolve<ListBuilders, User> for State {
     ListBuilders { query }: ListBuilders,
     user: User,
   ) -> anyhow::Result<Vec<BuilderListItem>> {
-    resource::list_for_user::<Builder>(query, &user).await
+    let all_tags = if query.tags.is_empty() {
+      vec![]
+    } else {
+      get_all_tags(None).await?
+    };
+    resource::list_for_user::<Builder>(query, &user, &all_tags).await
   }
 }
 
@@ -47,7 +53,13 @@ impl Resolve<ListFullBuilders, User> for State {
     ListFullBuilders { query }: ListFullBuilders,
     user: User,
   ) -> anyhow::Result<ListFullBuildersResponse> {
-    resource::list_full_for_user::<Builder>(query, &user).await
+    let all_tags = if query.tags.is_empty() {
+      vec![]
+    } else {
+      get_all_tags(None).await?
+    };
+    resource::list_full_for_user::<Builder>(query, &user, &all_tags)
+      .await
   }
 }
 
@@ -57,15 +69,16 @@ impl Resolve<GetBuildersSummary, User> for State {
     GetBuildersSummary {}: GetBuildersSummary,
     user: User,
   ) -> anyhow::Result<GetBuildersSummaryResponse> {
-    let query =
-      match resource::get_resource_ids_for_user::<Builder>(&user)
-        .await?
-      {
-        Some(ids) => doc! {
-          "_id": { "$in": ids }
-        },
-        None => Document::new(),
-      };
+    let query = match resource::get_resource_object_ids_for_user::<
+      Builder,
+    >(&user)
+    .await?
+    {
+      Some(ids) => doc! {
+        "_id": { "$in": ids }
+      },
+      None => Document::new(),
+    };
     let total = db_client()
       .builders
       .count_documents(query)

bin/core/src/api/read/deployment.rs

@@ -19,7 +19,7 @@ use periphery_client::api;
 use resolver_api::Resolve;
 
 use crate::{
-  helpers::periphery_client,
+  helpers::{periphery_client, query::get_all_tags},
   resource,
   state::{action_states, deployment_status_cache, State},
 };
@@ -45,7 +45,13 @@ impl Resolve<ListDeployments, User> for State {
     ListDeployments { query }: ListDeployments,
     user: User,
   ) -> anyhow::Result<Vec<DeploymentListItem>> {
-    resource::list_for_user::<Deployment>(query, &user).await
+    let all_tags = if query.tags.is_empty() {
+      vec![]
+    } else {
+      get_all_tags(None).await?
+    };
+    resource::list_for_user::<Deployment>(query, &user, &all_tags)
+      .await
   }
 }
 
@@ -55,7 +61,15 @@ impl Resolve<ListFullDeployments, User> for State {
     ListFullDeployments { query }: ListFullDeployments,
     user: User,
   ) -> anyhow::Result<ListFullDeploymentsResponse> {
-    resource::list_full_for_user::<Deployment>(query, &user).await
+    let all_tags = if query.tags.is_empty() {
+      vec![]
+    } else {
+      get_all_tags(None).await?
+    };
+    resource::list_full_for_user::<Deployment>(
+      query, &user, &all_tags,
+    )
+    .await
   }
 }
 
@@ -88,7 +102,11 @@ const MAX_LOG_LENGTH: u64 = 5000;
 impl Resolve<GetDeploymentLog, User> for State {
   async fn resolve(
     &self,
-    GetDeploymentLog { deployment, tail }: GetDeploymentLog,
+    GetDeploymentLog {
+      deployment,
+      tail,
+      timestamps,
+    }: GetDeploymentLog,
     user: User,
   ) -> anyhow::Result<Log> {
     let Deployment {
@@ -109,6 +127,7 @@ impl Resolve<GetDeploymentLog, User> for State {
       .request(api::container::GetContainerLog {
         name,
         tail: cmp::min(tail, MAX_LOG_LENGTH),
+        timestamps,
       })
       .await
       .context("failed at call to periphery")
@@ -123,6 +142,7 @@ impl Resolve<SearchDeploymentLog, User> for State {
       terms,
       combinator,
       invert,
+      timestamps,
     }: SearchDeploymentLog,
     user: User,
   ) -> anyhow::Result<Log> {
@@ -146,6 +166,7 @@ impl Resolve<SearchDeploymentLog, User> for State {
         terms,
         combinator,
         invert,
+        timestamps,
       })
       .await
       .context("failed at call to periphery")
@@ -210,6 +231,7 @@ impl Resolve<GetDeploymentsSummary, User> for State {
     let deployments = resource::list_full_for_user::<Deployment>(
       Default::default(),
       &user,
+      &[],
     )
     .await
     .context("failed to get deployments from db")?;
@@ -247,10 +269,16 @@ impl Resolve<ListCommonDeploymentExtraArgs, User> for State {
     ListCommonDeploymentExtraArgs { query }: ListCommonDeploymentExtraArgs,
     user: User,
   ) -> anyhow::Result<ListCommonDeploymentExtraArgsResponse> {
-    let deployments =
-      resource::list_full_for_user::<Deployment>(query, &user)
-        .await
-        .context("failed to get resources matching query")?;
+    let all_tags = if query.tags.is_empty() {
+      vec![]
+    } else {
+      get_all_tags(None).await?
+    };
+    let deployments = resource::list_full_for_user::<Deployment>(
+      query, &user, &all_tags,
+    )
+    .await
+    .context("failed to get resources matching query")?;
 
     // first collect with guaranteed uniqueness
     let mut res = HashSet::<String>::new();

bin/core/src/api/read/mod.rs

@@ -29,6 +29,7 @@ use crate::{
   resource, state::State,
 };
 
+mod action;
 mod alert;
 mod alerter;
 mod build;
@@ -88,6 +89,13 @@ enum ReadRequest {
   ListProcedures(ListProcedures),
   ListFullProcedures(ListFullProcedures),
 
+  // ==== ACTION ====
+  GetActionsSummary(GetActionsSummary),
+  GetAction(GetAction),
+  GetActionActionState(GetActionActionState),
+  ListActions(ListActions),
+  ListFullActions(ListFullActions),
+
   // ==== SERVER TEMPLATE ====
   GetServerTemplate(GetServerTemplate),
   GetServerTemplatesSummary(GetServerTemplatesSummary),
@@ -111,6 +119,7 @@ enum ReadRequest {
   InspectDockerImage(InspectDockerImage),
   ListDockerImageHistory(ListDockerImageHistory),
   InspectDockerVolume(InspectDockerVolume),
+  ListAllDockerContainers(ListAllDockerContainers),
   #[to_string_resolver]
   ListDockerContainers(ListDockerContainers),
   #[to_string_resolver]
@@ -330,6 +339,7 @@ impl Resolve<ListSecrets, User> for State {
         ResourceTarget::Server(id) => Some(id),
         ResourceTarget::Builder(id) => {
           match resource::get::<Builder>(&id).await?.config {
+            BuilderConfig::Url(_) => None,
             BuilderConfig::Server(config) => Some(config.server_id),
             BuilderConfig::Aws(config) => {
               secrets.extend(config.secrets);
@@ -378,6 +388,7 @@ impl Resolve<ListGitProvidersFromConfig, User> for State {
         }
         ResourceTarget::Builder(id) => {
           match resource::get::<Builder>(&id).await?.config {
+            BuilderConfig::Url(_) => {}
             BuilderConfig::Server(config) => {
               merge_git_providers_for_server(
                 &mut providers,
@@ -402,12 +413,18 @@ impl Resolve<ListGitProvidersFromConfig, User> for State {
     let (builds, repos, syncs) = tokio::try_join!(
       resource::list_full_for_user::<Build>(
         Default::default(),
-        &user
+        &user,
+        &[]
+      ),
+      resource::list_full_for_user::<Repo>(
+        Default::default(),
+        &user,
+        &[]
       ),
-      resource::list_full_for_user::<Repo>(Default::default(), &user),
       resource::list_full_for_user::<ResourceSync>(
         Default::default(),
-        &user
+        &user,
+        &[]
       ),
     )?;
 
@@ -470,6 +487,7 @@ impl Resolve<ListDockerRegistriesFromConfig, User> for State {
         }
         ResourceTarget::Builder(id) => {
           match resource::get::<Builder>(&id).await?.config {
+            BuilderConfig::Url(_) => {}
             BuilderConfig::Server(config) => {
               merge_docker_registries_for_server(
                 &mut registries,

bin/core/src/api/read/procedure.rs

@@ -10,6 +10,7 @@ use komodo_client::{
 use resolver_api::Resolve;
 
 use crate::{
+  helpers::query::get_all_tags,
   resource,
   state::{action_states, procedure_state_cache, State},
 };
@@ -35,7 +36,13 @@ impl Resolve<ListProcedures, User> for State {
     ListProcedures { query }: ListProcedures,
     user: User,
   ) -> anyhow::Result<ListProceduresResponse> {
-    resource::list_for_user::<Procedure>(query, &user).await
+    let all_tags = if query.tags.is_empty() {
+      vec![]
+    } else {
+      get_all_tags(None).await?
+    };
+    resource::list_for_user::<Procedure>(query, &user, &all_tags)
+      .await
   }
 }
 
@@ -45,7 +52,13 @@ impl Resolve<ListFullProcedures, User> for State {
     ListFullProcedures { query }: ListFullProcedures,
     user: User,
   ) -> anyhow::Result<ListFullProceduresResponse> {
-    resource::list_full_for_user::<Procedure>(query, &user).await
+    let all_tags = if query.tags.is_empty() {
+      vec![]
+    } else {
+      get_all_tags(None).await?
+    };
+    resource::list_full_for_user::<Procedure>(query, &user, &all_tags)
+      .await
   }
 }
 
@@ -58,6 +71,7 @@ impl Resolve<GetProceduresSummary, User> for State {
     let procedures = resource::list_full_for_user::<Procedure>(
       Default::default(),
       &user,
+      &[],
     )
     .await
     .context("failed to get procedures from db")?;

bin/core/src/api/read/repo.rs

@@ -12,6 +12,7 @@ use resolver_api::Resolve;
 
 use crate::{
   config::core_config,
+  helpers::query::get_all_tags,
   resource,
   state::{action_states, github_client, repo_state_cache, State},
 };
@@ -37,7 +38,12 @@ impl Resolve<ListRepos, User> for State {
     ListRepos { query }: ListRepos,
     user: User,
   ) -> anyhow::Result<Vec<RepoListItem>> {
-    resource::list_for_user::<Repo>(query, &user).await
+    let all_tags = if query.tags.is_empty() {
+      vec![]
+    } else {
+      get_all_tags(None).await?
+    };
+    resource::list_for_user::<Repo>(query, &user, &all_tags).await
   }
 }
 
@@ -47,7 +53,13 @@ impl Resolve<ListFullRepos, User> for State {
     ListFullRepos { query }: ListFullRepos,
     user: User,
   ) -> anyhow::Result<ListFullReposResponse> {
-    resource::list_full_for_user::<Repo>(query, &user).await
+    let all_tags = if query.tags.is_empty() {
+      vec![]
+    } else {
+      get_all_tags(None).await?
+    };
+    resource::list_full_for_user::<Repo>(query, &user, &all_tags)
+      .await
   }
 }
 
@@ -79,10 +91,13 @@ impl Resolve<GetReposSummary, User> for State {
     GetReposSummary {}: GetReposSummary,
     user: User,
   ) -> anyhow::Result<GetReposSummaryResponse> {
-    let repos =
-      resource::list_full_for_user::<Repo>(Default::default(), &user)
-        .await
-        .context("failed to get repos from db")?;
+    let repos = resource::list_full_for_user::<Repo>(
+      Default::default(),
+      &user,
+      &[],
+    )
+    .await
+    .context("failed to get repos from db")?;
 
     let mut res = GetReposSummaryResponse::default();
 

bin/core/src/api/read/server.rs

@@ -13,7 +13,7 @@ use komodo_client::{
   entities::{
     deployment::Deployment,
     docker::{
-      container::Container,
+      container::{Container, ContainerListItem},
       image::{Image, ImageHistoryResponseItem},
       network::Network,
       volume::Volume,
@@ -43,7 +43,7 @@ use resolver_api::{Resolve, ResolveToString};
 use tokio::sync::Mutex;
 
 use crate::{
-  helpers::periphery_client,
+  helpers::{periphery_client, query::get_all_tags},
   resource,
   stack::compose_container_match_regex,
   state::{action_states, db_client, server_status_cache, State},
@@ -55,9 +55,12 @@ impl Resolve<GetServersSummary, User> for State {
     GetServersSummary {}: GetServersSummary,
     user: User,
   ) -> anyhow::Result<GetServersSummaryResponse> {
-    let servers =
-      resource::list_for_user::<Server>(Default::default(), &user)
-        .await?;
+    let servers = resource::list_for_user::<Server>(
+      Default::default(),
+      &user,
+      &[],
+    )
+    .await?;
     let mut res = GetServersSummaryResponse::default();
     for server in servers {
       res.total += 1;
@@ -119,7 +122,12 @@ impl Resolve<ListServers, User> for State {
     ListServers { query }: ListServers,
     user: User,
   ) -> anyhow::Result<Vec<ServerListItem>> {
-    resource::list_for_user::<Server>(query, &user).await
+    let all_tags = if query.tags.is_empty() {
+      vec![]
+    } else {
+      get_all_tags(None).await?
+    };
+    resource::list_for_user::<Server>(query, &user, &all_tags).await
   }
 }
 
@@ -129,7 +137,13 @@ impl Resolve<ListFullServers, User> for State {
     ListFullServers { query }: ListFullServers,
     user: User,
   ) -> anyhow::Result<ListFullServersResponse> {
-    resource::list_full_for_user::<Server>(query, &user).await
+    let all_tags = if query.tags.is_empty() {
+      vec![]
+    } else {
+      get_all_tags(None).await?
+    };
+    resource::list_full_for_user::<Server>(query, &user, &all_tags)
+      .await
   }
 }
 
@@ -289,7 +303,7 @@ impl ResolveToString<ListSystemProcesses, User> for State {
   }
 }
 
-const STATS_PER_PAGE: i64 = 500;
+const STATS_PER_PAGE: i64 = 200;
 
 impl Resolve<GetHistoricalServerStats, User> for State {
   async fn resolve(
@@ -368,6 +382,40 @@ impl ResolveToString<ListDockerContainers, User> for State {
   }
 }
 
+impl Resolve<ListAllDockerContainers, User> for State {
+  async fn resolve(
+    &self,
+    ListAllDockerContainers { servers }: ListAllDockerContainers,
+    user: User,
+  ) -> anyhow::Result<Vec<ContainerListItem>> {
+    let servers = resource::list_for_user::<Server>(
+      Default::default(),
+      &user,
+      &[],
+    )
+    .await?
+    .into_iter()
+    .filter(|server| {
+      servers.is_empty()
+        || servers.contains(&server.id)
+        || servers.contains(&server.name)
+    });
+
+    let mut containers = Vec::<ContainerListItem>::new();
+
+    for server in servers {
+      let cache = server_status_cache()
+        .get_or_insert_default(&server.id)
+        .await;
+      if let Some(more_containers) = &cache.containers {
+        containers.extend(more_containers.clone());
+      }
+    }
+
+    Ok(containers)
+  }
+}
+
 impl Resolve<InspectDockerContainer, User> for State {
   async fn resolve(
     &self,
@@ -404,6 +452,7 @@ impl Resolve<GetContainerLog, User> for State {
       server,
       container,
       tail,
+      timestamps,
     }: GetContainerLog,
     user: User,
   ) -> anyhow::Result<Log> {
@@ -417,6 +466,7 @@ impl Resolve<GetContainerLog, User> for State {
       .request(periphery::container::GetContainerLog {
         name: container,
         tail: cmp::min(tail, MAX_LOG_LENGTH),
+        timestamps,
       })
       .await
       .context("failed at call to periphery")
@@ -432,6 +482,7 @@ impl Resolve<SearchContainerLog, User> for State {
       terms,
       combinator,
       invert,
+      timestamps,
     }: SearchContainerLog,
     user: User,
   ) -> anyhow::Result<Log> {
@@ -447,6 +498,7 @@ impl Resolve<SearchContainerLog, User> for State {
         terms,
         combinator,
         invert,
+        timestamps,
       })
       .await
       .context("failed at call to periphery")
@@ -487,20 +539,21 @@ impl Resolve<GetResourceMatchingContainer, User> for State {
       for StackServiceNames {
         service_name,
         container_name,
+        ..
       } in stack
         .info
         .deployed_services
         .unwrap_or(stack.info.latest_services)
       {
         let is_match = match compose_container_match_regex(&container_name)
-					.with_context(|| format!("failed to construct container name matching regex for service {service_name}")) 
-				{
-					Ok(regex) => regex,
-					Err(e) => {
-						warn!("{e:#}");
-						continue;
-					}
-				}.is_match(&container);
+          .with_context(|| format!("failed to construct container name matching regex for service {service_name}")) 
+        {
+          Ok(regex) => regex,
+          Err(e) => {
+            warn!("{e:#}");
+            continue;
+          }
+        }.is_match(&container);
 
         if is_match {
           return Ok(GetResourceMatchingContainerResponse {

bin/core/src/api/read/server_template.rs

@@ -11,6 +11,7 @@ use mungos::mongodb::bson::doc;
 use resolver_api::Resolve;
 
 use crate::{
+  helpers::query::get_all_tags,
   resource,
   state::{db_client, State},
 };
@@ -36,7 +37,13 @@ impl Resolve<ListServerTemplates, User> for State {
     ListServerTemplates { query }: ListServerTemplates,
     user: User,
   ) -> anyhow::Result<ListServerTemplatesResponse> {
-    resource::list_for_user::<ServerTemplate>(query, &user).await
+    let all_tags = if query.tags.is_empty() {
+      vec![]
+    } else {
+      get_all_tags(None).await?
+    };
+    resource::list_for_user::<ServerTemplate>(query, &user, &all_tags)
+      .await
   }
 }
 
@@ -46,7 +53,15 @@ impl Resolve<ListFullServerTemplates, User> for State {
     ListFullServerTemplates { query }: ListFullServerTemplates,
     user: User,
   ) -> anyhow::Result<ListFullServerTemplatesResponse> {
-    resource::list_full_for_user::<ServerTemplate>(query, &user).await
+    let all_tags = if query.tags.is_empty() {
+      vec![]
+    } else {
+      get_all_tags(None).await?
+    };
+    resource::list_full_for_user::<ServerTemplate>(
+      query, &user, &all_tags,
+    )
+    .await
   }
 }
 
@@ -56,7 +71,7 @@ impl Resolve<GetServerTemplatesSummary, User> for State {
     GetServerTemplatesSummary {}: GetServerTemplatesSummary,
     user: User,
   ) -> anyhow::Result<GetServerTemplatesSummaryResponse> {
-    let query = match resource::get_resource_ids_for_user::<
+    let query = match resource::get_resource_object_ids_for_user::<
       ServerTemplate,
     >(&user)
     .await?

bin/core/src/api/read/stack.rs

@@ -17,7 +17,7 @@ use resolver_api::Resolve;
 
 use crate::{
   config::core_config,
-  helpers::periphery_client,
+  helpers::{periphery_client, query::get_all_tags},
   resource,
   stack::get_stack_and_server,
   state::{action_states, github_client, stack_status_cache, State},
@@ -70,6 +70,7 @@ impl Resolve<GetStackServiceLog, User> for State {
       stack,
       service,
       tail,
+      timestamps,
     }: GetStackServiceLog,
     user: User,
   ) -> anyhow::Result<GetStackServiceLogResponse> {
@@ -85,6 +86,7 @@ impl Resolve<GetStackServiceLog, User> for State {
         project: stack.project_name(false),
         service,
         tail,
+        timestamps,
       })
       .await
       .context("failed to get stack service log from periphery")
@@ -100,6 +102,7 @@ impl Resolve<SearchStackServiceLog, User> for State {
       terms,
       combinator,
       invert,
+      timestamps,
     }: SearchStackServiceLog,
     user: User,
   ) -> anyhow::Result<SearchStackServiceLogResponse> {
@@ -117,6 +120,7 @@ impl Resolve<SearchStackServiceLog, User> for State {
         terms,
         combinator,
         invert,
+        timestamps,
       })
       .await
       .context("failed to get stack service log from periphery")
@@ -129,9 +133,15 @@ impl Resolve<ListCommonStackExtraArgs, User> for State {
     ListCommonStackExtraArgs { query }: ListCommonStackExtraArgs,
     user: User,
   ) -> anyhow::Result<ListCommonStackExtraArgsResponse> {
-    let stacks = resource::list_full_for_user::<Stack>(query, &user)
-      .await
-      .context("failed to get resources matching query")?;
+    let all_tags = if query.tags.is_empty() {
+      vec![]
+    } else {
+      get_all_tags(None).await?
+    };
+    let stacks =
+      resource::list_full_for_user::<Stack>(query, &user, &all_tags)
+        .await
+        .context("failed to get resources matching query")?;
 
     // first collect with guaranteed uniqueness
     let mut res = HashSet::<String>::new();
@@ -154,9 +164,15 @@ impl Resolve<ListCommonStackBuildExtraArgs, User> for State {
     ListCommonStackBuildExtraArgs { query }: ListCommonStackBuildExtraArgs,
     user: User,
   ) -> anyhow::Result<ListCommonStackBuildExtraArgsResponse> {
-    let stacks = resource::list_full_for_user::<Stack>(query, &user)
-      .await
-      .context("failed to get resources matching query")?;
+    let all_tags = if query.tags.is_empty() {
+      vec![]
+    } else {
+      get_all_tags(None).await?
+    };
+    let stacks =
+      resource::list_full_for_user::<Stack>(query, &user, &all_tags)
+        .await
+        .context("failed to get resources matching query")?;
 
     // first collect with guaranteed uniqueness
     let mut res = HashSet::<String>::new();
@@ -179,7 +195,12 @@ impl Resolve<ListStacks, User> for State {
     ListStacks { query }: ListStacks,
     user: User,
   ) -> anyhow::Result<Vec<StackListItem>> {
-    resource::list_for_user::<Stack>(query, &user).await
+    let all_tags = if query.tags.is_empty() {
+      vec![]
+    } else {
+      get_all_tags(None).await?
+    };
+    resource::list_for_user::<Stack>(query, &user, &all_tags).await
   }
 }
 
@@ -189,7 +210,13 @@ impl Resolve<ListFullStacks, User> for State {
     ListFullStacks { query }: ListFullStacks,
     user: User,
   ) -> anyhow::Result<ListFullStacksResponse> {
-    resource::list_full_for_user::<Stack>(query, &user).await
+    let all_tags = if query.tags.is_empty() {
+      vec![]
+    } else {
+      get_all_tags(None).await?
+    };
+    resource::list_full_for_user::<Stack>(query, &user, &all_tags)
+      .await
   }
 }
 
@@ -224,6 +251,7 @@ impl Resolve<GetStacksSummary, User> for State {
     let stacks = resource::list_full_for_user::<Stack>(
       Default::default(),
       &user,
+      &[],
     )
     .await
     .context("failed to get stacks from db")?;

bin/core/src/api/read/sync.rs

@@ -15,6 +15,7 @@ use resolver_api::Resolve;
 
 use crate::{
   config::core_config,
+  helpers::query::get_all_tags,
   resource,
   state::{
     action_states, github_client, resource_sync_state_cache, State,
@@ -42,7 +43,13 @@ impl Resolve<ListResourceSyncs, User> for State {
     ListResourceSyncs { query }: ListResourceSyncs,
     user: User,
   ) -> anyhow::Result<Vec<ResourceSyncListItem>> {
-    resource::list_for_user::<ResourceSync>(query, &user).await
+    let all_tags = if query.tags.is_empty() {
+      vec![]
+    } else {
+      get_all_tags(None).await?
+    };
+    resource::list_for_user::<ResourceSync>(query, &user, &all_tags)
+      .await
   }
 }
 
@@ -52,7 +59,15 @@ impl Resolve<ListFullResourceSyncs, User> for State {
     ListFullResourceSyncs { query }: ListFullResourceSyncs,
     user: User,
   ) -> anyhow::Result<ListFullResourceSyncsResponse> {
-    resource::list_full_for_user::<ResourceSync>(query, &user).await
+    let all_tags = if query.tags.is_empty() {
+      vec![]
+    } else {
+      get_all_tags(None).await?
+    };
+    resource::list_full_for_user::<ResourceSync>(
+      query, &user, &all_tags,
+    )
+    .await
   }
 }
 
@@ -88,6 +103,7 @@ impl Resolve<GetResourceSyncsSummary, User> for State {
       resource::list_full_for_user::<ResourceSync>(
         Default::default(),
         &user,
+        &[],
       )
       .await
       .context("failed to get resource_syncs from db")?;

bin/core/src/api/read/toml.rs

@@ -1,27 +1,16 @@
-use std::collections::HashMap;
-
 use anyhow::Context;
 use komodo_client::{
   api::read::{
     ExportAllResourcesToToml, ExportAllResourcesToTomlResponse,
     ExportResourcesToToml, ExportResourcesToTomlResponse,
-    GetUserGroup, ListUserTargetPermissions,
+    ListUserGroups,
   },
   entities::{
-    alerter::Alerter,
-    build::Build,
-    builder::Builder,
-    deployment::Deployment,
-    permission::{PermissionLevel, UserTarget},
-    procedure::Procedure,
-    repo::Repo,
-    resource::ResourceQuery,
-    server::Server,
-    server_template::ServerTemplate,
-    stack::Stack,
-    sync::ResourceSync,
-    toml::{PermissionToml, ResourcesToml, UserGroupToml},
-    user::User,
+    action::Action, alerter::Alerter, build::Build, builder::Builder,
+    deployment::Deployment, permission::PermissionLevel,
+    procedure::Procedure, repo::Repo, resource::ResourceQuery,
+    server::Server, server_template::ServerTemplate, stack::Stack,
+    sync::ResourceSync, toml::ResourcesToml, user::User,
     ResourceTarget,
   },
 };
@@ -29,11 +18,14 @@ use mungos::find::find_collect;
 use resolver_api::Resolve;
 
 use crate::{
-  helpers::query::{get_id_to_tags, get_user_user_group_ids},
+  helpers::query::{
+    get_all_tags, get_id_to_tags, get_user_user_group_ids,
+  },
   resource,
   state::{db_client, State},
   sync::{
     toml::{convert_resource, ToToml, TOML_PRETTY_OPTIONS},
+    user_groups::convert_user_groups,
     AllResourcesById,
   },
 };
@@ -46,10 +38,17 @@ impl Resolve<ExportAllResourcesToToml, User> for State {
   ) -> anyhow::Result<ExportAllResourcesToTomlResponse> {
     let mut targets = Vec::<ResourceTarget>::new();
 
+    let all_tags = if tags.is_empty() {
+      vec![]
+    } else {
+      get_all_tags(None).await?
+    };
+
     targets.extend(
       resource::list_for_user::<Alerter>(
         ResourceQuery::builder().tags(tags.clone()).build(),
         &user,
+        &all_tags,
       )
       .await?
       .into_iter()
@@ -59,6 +58,7 @@ impl Resolve<ExportAllResourcesToToml, User> for State {
       resource::list_for_user::<Builder>(
         ResourceQuery::builder().tags(tags.clone()).build(),
         &user,
+        &all_tags,
       )
       .await?
       .into_iter()
@@ -68,6 +68,7 @@ impl Resolve<ExportAllResourcesToToml, User> for State {
       resource::list_for_user::<Server>(
         ResourceQuery::builder().tags(tags.clone()).build(),
         &user,
+        &all_tags,
       )
       .await?
       .into_iter()
@@ -77,6 +78,7 @@ impl Resolve<ExportAllResourcesToToml, User> for State {
       resource::list_for_user::<Deployment>(
         ResourceQuery::builder().tags(tags.clone()).build(),
         &user,
+        &all_tags,
       )
       .await?
       .into_iter()
@@ -86,6 +88,7 @@ impl Resolve<ExportAllResourcesToToml, User> for State {
       resource::list_for_user::<Stack>(
         ResourceQuery::builder().tags(tags.clone()).build(),
         &user,
+        &all_tags,
       )
       .await?
       .into_iter()
@@ -95,6 +98,7 @@ impl Resolve<ExportAllResourcesToToml, User> for State {
       resource::list_for_user::<Build>(
         ResourceQuery::builder().tags(tags.clone()).build(),
         &user,
+        &all_tags,
       )
       .await?
       .into_iter()
@@ -104,6 +108,7 @@ impl Resolve<ExportAllResourcesToToml, User> for State {
       resource::list_for_user::<Repo>(
         ResourceQuery::builder().tags(tags.clone()).build(),
         &user,
+        &all_tags,
       )
       .await?
       .into_iter()
@@ -113,15 +118,27 @@ impl Resolve<ExportAllResourcesToToml, User> for State {
       resource::list_for_user::<Procedure>(
         ResourceQuery::builder().tags(tags.clone()).build(),
         &user,
+        &all_tags,
       )
       .await?
       .into_iter()
       .map(|resource| ResourceTarget::Procedure(resource.id)),
     );
+    targets.extend(
+      resource::list_for_user::<Action>(
+        ResourceQuery::builder().tags(tags.clone()).build(),
+        &user,
+        &all_tags,
+      )
+      .await?
+      .into_iter()
+      .map(|resource| ResourceTarget::Action(resource.id)),
+    );
     targets.extend(
       resource::list_for_user::<ServerTemplate>(
         ResourceQuery::builder().tags(tags.clone()).build(),
         &user,
+        &all_tags,
       )
       .await?
       .into_iter()
@@ -131,6 +148,7 @@ impl Resolve<ExportAllResourcesToToml, User> for State {
       resource::list_full_for_user::<ResourceSync>(
         ResourceQuery::builder().tags(tags.clone()).build(),
         &user,
+        &all_tags,
       )
       .await?
       .into_iter()
@@ -184,9 +202,12 @@ impl Resolve<ExportResourcesToToml, User> for State {
             PermissionLevel::Read,
           )
           .await?;
-          res
-            .alerters
-            .push(convert_resource::<Alerter>(alerter, &id_to_tags))
+          res.alerters.push(convert_resource::<Alerter>(
+            alerter,
+            false,
+            vec![],
+            &id_to_tags,
+          ))
         }
         ResourceTarget::ResourceSync(id) => {
           let sync = resource::get_check_permissions::<ResourceSync>(
@@ -201,6 +222,8 @@ impl Resolve<ExportResourcesToToml, User> for State {
           {
             res.resource_syncs.push(convert_resource::<ResourceSync>(
               sync,
+              false,
+              vec![],
               &id_to_tags,
             ))
           }
@@ -213,7 +236,12 @@ impl Resolve<ExportResourcesToToml, User> for State {
           )
           .await?;
           res.server_templates.push(
-            convert_resource::<ServerTemplate>(template, &id_to_tags),
+            convert_resource::<ServerTemplate>(
+              template,
+              false,
+              vec![],
+              &id_to_tags,
+            ),
           )
         }
         ResourceTarget::Server(id) => {
@@ -223,9 +251,12 @@ impl Resolve<ExportResourcesToToml, User> for State {
             PermissionLevel::Read,
           )
           .await?;
-          res
-            .servers
-            .push(convert_resource::<Server>(server, &id_to_tags))
+          res.servers.push(convert_resource::<Server>(
+            server,
+            false,
+            vec![],
+            &id_to_tags,
+          ))
         }
         ResourceTarget::Builder(id) => {
           let mut builder =
@@ -236,9 +267,12 @@ impl Resolve<ExportResourcesToToml, User> for State {
             )
             .await?;
           Builder::replace_ids(&mut builder, &all);
-          res
-            .builders
-            .push(convert_resource::<Builder>(builder, &id_to_tags))
+          res.builders.push(convert_resource::<Builder>(
+            builder,
+            false,
+            vec![],
+            &id_to_tags,
+          ))
         }
         ResourceTarget::Build(id) => {
           let mut build = resource::get_check_permissions::<Build>(
@@ -248,9 +282,12 @@ impl Resolve<ExportResourcesToToml, User> for State {
           )
           .await?;
           Build::replace_ids(&mut build, &all);
-          res
-            .builds
-            .push(convert_resource::<Build>(build, &id_to_tags))
+          res.builds.push(convert_resource::<Build>(
+            build,
+            false,
+            vec![],
+            &id_to_tags,
+          ))
         }
         ResourceTarget::Deployment(id) => {
           let mut deployment = resource::get_check_permissions::<
@@ -262,6 +299,8 @@ impl Resolve<ExportResourcesToToml, User> for State {
           Deployment::replace_ids(&mut deployment, &all);
           res.deployments.push(convert_resource::<Deployment>(
             deployment,
+            false,
+            vec![],
             &id_to_tags,
           ))
         }
@@ -273,7 +312,12 @@ impl Resolve<ExportResourcesToToml, User> for State {
           )
           .await?;
           Repo::replace_ids(&mut repo, &all);
-          res.repos.push(convert_resource::<Repo>(repo, &id_to_tags))
+          res.repos.push(convert_resource::<Repo>(
+            repo,
+            false,
+            vec![],
+            &id_to_tags,
+          ))
         }
         ResourceTarget::Stack(id) => {
           let mut stack = resource::get_check_permissions::<Stack>(
@@ -283,9 +327,12 @@ impl Resolve<ExportResourcesToToml, User> for State {
           )
           .await?;
           Stack::replace_ids(&mut stack, &all);
-          res
-            .stacks
-            .push(convert_resource::<Stack>(stack, &id_to_tags))
+          res.stacks.push(convert_resource::<Stack>(
+            stack,
+            false,
+            vec![],
+            &id_to_tags,
+          ))
         }
         ResourceTarget::Procedure(id) => {
           let mut procedure = resource::get_check_permissions::<
@@ -297,6 +344,23 @@ impl Resolve<ExportResourcesToToml, User> for State {
           Procedure::replace_ids(&mut procedure, &all);
           res.procedures.push(convert_resource::<Procedure>(
             procedure,
+            false,
+            vec![],
+            &id_to_tags,
+          ));
+        }
+        ResourceTarget::Action(id) => {
+          let mut action = resource::get_check_permissions::<Action>(
+            &id,
+            &user,
+            PermissionLevel::Read,
+          )
+          .await?;
+          Action::replace_ids(&mut action, &all);
+          res.actions.push(convert_resource::<Action>(
+            action,
+            false,
+            vec![],
             &id_to_tags,
           ));
         }
@@ -336,122 +400,17 @@ async fn add_user_groups(
   all: &AllResourcesById,
   user: &User,
 ) -> anyhow::Result<()> {
-  let db = db_client();
-
-  let usernames = find_collect(&db.users, None, None)
+  let user_groups = State
+    .resolve(ListUserGroups {}, user.clone())
     .await?
     .into_iter()
-    .map(|user| (user.id, user.username))
-    .collect::<HashMap<_, _>>();
-
-  for user_group in user_groups {
-    let ug = State
-      .resolve(GetUserGroup { user_group }, user.clone())
-      .await?;
-    // this method is admin only, but we already know user can see user group if above does not return Err
-    let permissions = State
-      .resolve(
-        ListUserTargetPermissions {
-          user_target: UserTarget::UserGroup(ug.id),
-        },
-        User {
-          admin: true,
-          ..Default::default()
-        },
-      )
-      .await?
-      .into_iter()
-      .map(|mut permission| {
-        match &mut permission.resource_target {
-          ResourceTarget::Build(id) => {
-            *id = all
-              .builds
-              .get(id)
-              .map(|r| r.name.clone())
-              .unwrap_or_default()
-          }
-          ResourceTarget::Builder(id) => {
-            *id = all
-              .builders
-              .get(id)
-              .map(|r| r.name.clone())
-              .unwrap_or_default()
-          }
-          ResourceTarget::Deployment(id) => {
-            *id = all
-              .deployments
-              .get(id)
-              .map(|r| r.name.clone())
-              .unwrap_or_default()
-          }
-          ResourceTarget::Server(id) => {
-            *id = all
-              .servers
-              .get(id)
-              .map(|r| r.name.clone())
-              .unwrap_or_default()
-          }
-          ResourceTarget::Repo(id) => {
-            *id = all
-              .repos
-              .get(id)
-              .map(|r| r.name.clone())
-              .unwrap_or_default()
-          }
-          ResourceTarget::Alerter(id) => {
-            *id = all
-              .alerters
-              .get(id)
-              .map(|r| r.name.clone())
-              .unwrap_or_default()
-          }
-          ResourceTarget::Procedure(id) => {
-            *id = all
-              .procedures
-              .get(id)
-              .map(|r| r.name.clone())
-              .unwrap_or_default()
-          }
-          ResourceTarget::ServerTemplate(id) => {
-            *id = all
-              .templates
-              .get(id)
-              .map(|r| r.name.clone())
-              .unwrap_or_default()
-          }
-          ResourceTarget::ResourceSync(id) => {
-            *id = all
-              .syncs
-              .get(id)
-              .map(|r| r.name.clone())
-              .unwrap_or_default()
-          }
-          ResourceTarget::Stack(id) => {
-            *id = all
-              .stacks
-              .get(id)
-              .map(|r| r.name.clone())
-              .unwrap_or_default()
-          }
-          ResourceTarget::System(_) => {}
-        }
-        PermissionToml {
-          target: permission.resource_target,
-          level: permission.level,
-        }
-      })
-      .collect();
-    res.user_groups.push(UserGroupToml {
-      name: ug.name,
-      users: ug
-        .users
-        .into_iter()
-        .filter_map(|user_id| usernames.get(&user_id).cloned())
-        .collect(),
-      all: ug.all,
-      permissions,
+    .filter(|ug| {
+      user_groups.contains(&ug.name) || user_groups.contains(&ug.id)
     });
-  }
+  let mut ug = Vec::with_capacity(user_groups.size_hint().0);
+  convert_user_groups(user_groups, all, &mut ug).await?;
+  res.user_groups = ug.into_iter().map(|ug| ug.1).collect();
+
   Ok(())
 }
 
@@ -508,6 +467,14 @@ fn serialize_resources_toml(
     Procedure::push_to_toml_string(procedure, &mut toml)?;
   }
 
+  for action in resources.actions {
+    if !toml.is_empty() {
+      toml.push_str("\n\n##\n\n");
+    }
+    toml.push_str("[[action]]\n");
+    Action::push_to_toml_string(action, &mut toml)?;
+  }
+
   for alerter in resources.alerters {
     if !toml.is_empty() {
       toml.push_str("\n\n##\n\n");

bin/core/src/api/read/update.rs

@@ -4,6 +4,7 @@ use anyhow::{anyhow, Context};
 use komodo_client::{
   api::read::{GetUpdate, ListUpdates, ListUpdatesResponse},
   entities::{
+    action::Action,
     alerter::Alerter,
     build::Build,
     builder::Builder,
@@ -104,6 +105,16 @@ impl Resolve<ListUpdates, User> for State {
           })
           .unwrap_or_else(|| doc! { "target.type": "Procedure" });
 
+      let action_query =
+        resource::get_resource_ids_for_user::<Action>(&user)
+          .await?
+          .map(|ids| {
+            doc! {
+              "target.type": "Action", "target.id": { "$in": ids }
+            }
+          })
+          .unwrap_or_else(|| doc! { "target.type": "Action" });
+
       let builder_query =
         resource::get_resource_ids_for_user::<Builder>(&user)
           .await?
@@ -124,27 +135,27 @@ impl Resolve<ListUpdates, User> for State {
           })
           .unwrap_or_else(|| doc! { "target.type": "Alerter" });
 
-      let server_template_query = resource::get_resource_ids_for_user::<ServerTemplate>(
-        &user,
-      )
-      .await?
-      .map(|ids| {
-        doc! {
-          "target.type": "ServerTemplate", "target.id": { "$in": ids }
-        }
-      })
-      .unwrap_or_else(|| doc! { "target.type": "ServerTemplate" });
+      let server_template_query =
+        resource::get_resource_ids_for_user::<ServerTemplate>(&user)
+          .await?
+          .map(|ids| {
+            doc! {
+              "target.type": "ServerTemplate", "target.id": { "$in": ids }
+            }
+          })
+          .unwrap_or_else(|| doc! { "target.type": "ServerTemplate" });
 
-      let resource_sync_query = resource::get_resource_ids_for_user::<ResourceSync>(
-        &user,
-      )
-      .await?
-      .map(|ids| {
-        doc! {
-          "target.type": "ResourceSync", "target.id": { "$in": ids }
-        }
-      })
-      .unwrap_or_else(|| doc! { "target.type": "ResourceSync" });
+      let resource_sync_query =
+        resource::get_resource_ids_for_user::<ResourceSync>(
+          &user,
+        )
+        .await?
+        .map(|ids| {
+          doc! {
+            "target.type": "ResourceSync", "target.id": { "$in": ids }
+          }
+        })
+        .unwrap_or_else(|| doc! { "target.type": "ResourceSync" });
 
       let mut query = query.unwrap_or_default();
       query.extend(doc! {
@@ -155,6 +166,7 @@ impl Resolve<ListUpdates, User> for State {
           build_query,
           repo_query,
           procedure_query,
+          action_query,
           alerter_query,
           builder_query,
           server_template_query,
@@ -292,6 +304,14 @@ impl Resolve<GetUpdate, User> for State {
         )
         .await?;
       }
+      ResourceTarget::Action(id) => {
+        resource::get_check_permissions::<Action>(
+          id,
+          &user,
+          PermissionLevel::Read,
+        )
+        .await?;
+      }
       ResourceTarget::ServerTemplate(id) => {
         resource::get_check_permissions::<ServerTemplate>(
           id,

bin/core/src/api/read/user.rs

@@ -6,7 +6,7 @@ use komodo_client::{
     ListApiKeysForServiceUserResponse, ListApiKeysResponse,
     ListUsers, ListUsersResponse,
   },
-  entities::user::{User, UserConfig},
+  entities::user::{admin_service_user, User, UserConfig},
 };
 use mungos::{
   by_id::find_one_by_id,
@@ -26,6 +26,13 @@ impl Resolve<GetUsername, User> for State {
     GetUsername { user_id }: GetUsername,
     _: User,
   ) -> anyhow::Result<GetUsernameResponse> {
+    if let Some(user) = admin_service_user(&user_id) {
+      return Ok(GetUsernameResponse {
+        username: user.username,
+        avatar: None,
+      });
+    }
+
     let user = find_one_by_id(&db_client().users, &user_id)
       .await
       .context("failed at mongo query for user")?

bin/core/src/api/write/action.rs

@@ -0,0 +1,71 @@
+use komodo_client::{
+  api::write::*,
+  entities::{
+    action::Action, permission::PermissionLevel, update::Update,
+    user::User,
+  },
+};
+use resolver_api::Resolve;
+
+use crate::{resource, state::State};
+
+impl Resolve<CreateAction, User> for State {
+  #[instrument(name = "CreateAction", skip(self, user))]
+  async fn resolve(
+    &self,
+    CreateAction { name, config }: CreateAction,
+    user: User,
+  ) -> anyhow::Result<Action> {
+    resource::create::<Action>(&name, config, &user).await
+  }
+}
+
+impl Resolve<CopyAction, User> for State {
+  #[instrument(name = "CopyAction", skip(self, user))]
+  async fn resolve(
+    &self,
+    CopyAction { name, id }: CopyAction,
+    user: User,
+  ) -> anyhow::Result<Action> {
+    let Action { config, .. } = resource::get_check_permissions::<
+      Action,
+    >(
+      &id, &user, PermissionLevel::Write
+    )
+    .await?;
+    resource::create::<Action>(&name, config.into(), &user).await
+  }
+}
+
+impl Resolve<UpdateAction, User> for State {
+  #[instrument(name = "UpdateAction", skip(self, user))]
+  async fn resolve(
+    &self,
+    UpdateAction { id, config }: UpdateAction,
+    user: User,
+  ) -> anyhow::Result<Action> {
+    resource::update::<Action>(&id, config, &user).await
+  }
+}
+
+impl Resolve<RenameAction, User> for State {
+  #[instrument(name = "RenameAction", skip(self, user))]
+  async fn resolve(
+    &self,
+    RenameAction { id, name }: RenameAction,
+    user: User,
+  ) -> anyhow::Result<Update> {
+    resource::rename::<Action>(&id, &name, &user).await
+  }
+}
+
+impl Resolve<DeleteAction, User> for State {
+  #[instrument(name = "DeleteAction", skip(self, user))]
+  async fn resolve(
+    &self,
+    DeleteAction { id }: DeleteAction,
+    user: User,
+  ) -> anyhow::Result<Action> {
+    resource::delete::<Action>(&id, &user).await
+  }
+}

bin/core/src/api/write/alerter.rs

@@ -1,9 +1,8 @@
 use komodo_client::{
-  api::write::{
-    CopyAlerter, CreateAlerter, DeleteAlerter, UpdateAlerter,
-  },
+  api::write::*,
   entities::{
-    alerter::Alerter, permission::PermissionLevel, user::User,
+    alerter::Alerter, permission::PermissionLevel, update::Update,
+    user::User,
   },
 };
 use resolver_api::Resolve;
@@ -59,3 +58,14 @@ impl Resolve<UpdateAlerter, User> for State {
     resource::update::<Alerter>(&id, config, &user).await
   }
 }
+
+impl Resolve<RenameAlerter, User> for State {
+  #[instrument(name = "RenameAlerter", skip(self, user))]
+  async fn resolve(
+    &self,
+    RenameAlerter { id, name }: RenameAlerter,
+    user: User,
+  ) -> anyhow::Result<Update> {
+    resource::rename::<Alerter>(&id, &name, &user).await
+  }
+}

bin/core/src/api/write/build.rs

@@ -6,6 +6,7 @@ use komodo_client::{
     build::{Build, BuildInfo, PartialBuildConfig},
     config::core::CoreConfig,
     permission::PermissionLevel,
+    update::Update,
     user::User,
     CloneArgs, NoData,
   },
@@ -77,6 +78,17 @@ impl Resolve<UpdateBuild, User> for State {
   }
 }
 
+impl Resolve<RenameBuild, User> for State {
+  #[instrument(name = "RenameBuild", skip(self, user))]
+  async fn resolve(
+    &self,
+    RenameBuild { id, name }: RenameBuild,
+    user: User,
+  ) -> anyhow::Result<Update> {
+    resource::rename::<Build>(&id, &name, &user).await
+  }
+}
+
 impl Resolve<RefreshBuildCache, User> for State {
   #[instrument(
     name = "RefreshBuildCache",

bin/core/src/api/write/builder.rs

@@ -1,7 +1,8 @@
 use komodo_client::{
   api::write::*,
   entities::{
-    builder::Builder, permission::PermissionLevel, user::User,
+    builder::Builder, permission::PermissionLevel, update::Update,
+    user::User,
   },
 };
 use resolver_api::Resolve;
@@ -57,3 +58,14 @@ impl Resolve<UpdateBuilder, User> for State {
     resource::update::<Builder>(&id, config, &user).await
   }
 }
+
+impl Resolve<RenameBuilder, User> for State {
+  #[instrument(name = "RenameBuilder", skip(self, user))]
+  async fn resolve(
+    &self,
+    RenameBuilder { id, name }: RenameBuilder,
+    user: User,
+  ) -> anyhow::Result<Update> {
+    resource::rename::<Builder>(&id, &name, &user).await
+  }
+}

bin/core/src/api/write/deployment.rs

@@ -2,10 +2,14 @@ use anyhow::{anyhow, Context};
 use komodo_client::{
   api::write::*,
   entities::{
-    deployment::{Deployment, DeploymentState},
+    deployment::{
+      Deployment, DeploymentImage, DeploymentState,
+      PartialDeploymentConfig, RestartMode,
+    },
+    docker::container::RestartPolicyNameEnum,
     komodo_timestamp,
     permission::PermissionLevel,
-    server::Server,
+    server::{Server, ServerState},
     to_komodo_name,
     update::Update,
     user::User,
@@ -13,7 +17,7 @@ use komodo_client::{
   },
 };
 use mungos::{by_id::update_one_by_id, mongodb::bson::doc};
-use periphery_client::api;
+use periphery_client::api::{self, container::InspectContainer};
 use resolver_api::Resolve;
 
 use crate::{
@@ -23,7 +27,7 @@ use crate::{
     update::{add_update, make_update},
   },
   resource,
-  state::{action_states, db_client, State},
+  state::{action_states, db_client, server_status_cache, State},
 };
 
 impl Resolve<CreateDeployment, User> for State {
@@ -55,6 +59,97 @@ impl Resolve<CopyDeployment, User> for State {
   }
 }
 
+impl Resolve<CreateDeploymentFromContainer, User> for State {
+  #[instrument(
+    name = "CreateDeploymentFromContainer",
+    skip(self, user)
+  )]
+  async fn resolve(
+    &self,
+    CreateDeploymentFromContainer { name, server }: CreateDeploymentFromContainer,
+    user: User,
+  ) -> anyhow::Result<Deployment> {
+    let server = resource::get_check_permissions::<Server>(
+      &server,
+      &user,
+      PermissionLevel::Write,
+    )
+    .await?;
+    let cache = server_status_cache()
+      .get_or_insert_default(&server.id)
+      .await;
+    if cache.state != ServerState::Ok {
+      return Err(anyhow!(
+        "Cannot inspect container: server is {:?}",
+        cache.state
+      ));
+    }
+    let container = periphery_client(&server)?
+      .request(InspectContainer { name: name.clone() })
+      .await
+      .context("Failed to inspect container")?;
+
+    let mut config = PartialDeploymentConfig {
+      server_id: server.id.into(),
+      ..Default::default()
+    };
+
+    if let Some(container_config) = container.config {
+      config.image = container_config
+        .image
+        .map(|image| DeploymentImage::Image { image });
+      config.command = container_config.cmd.join(" ").into();
+      config.environment = container_config
+        .env
+        .into_iter()
+        .map(|env| format!("  {env}"))
+        .collect::<Vec<_>>()
+        .join("\n")
+        .into();
+      config.labels = container_config
+        .labels
+        .into_iter()
+        .map(|(key, val)| format!("  {key}: {val}"))
+        .collect::<Vec<_>>()
+        .join("\n")
+        .into();
+    }
+    if let Some(host_config) = container.host_config {
+      config.volumes = host_config
+        .binds
+        .into_iter()
+        .map(|bind| format!("  {bind}"))
+        .collect::<Vec<_>>()
+        .join("\n")
+        .into();
+      config.network = host_config.network_mode;
+      config.ports = host_config
+        .port_bindings
+        .into_iter()
+        .filter_map(|(container, mut host)| {
+          let host = host.pop()?.host_port?;
+          Some(format!("  {host}:{}", container.replace("/tcp", "")))
+        })
+        .collect::<Vec<_>>()
+        .join("\n")
+        .into();
+      config.restart = host_config.restart_policy.map(|restart| {
+        match restart.name {
+          RestartPolicyNameEnum::Always => RestartMode::Always,
+          RestartPolicyNameEnum::No
+          | RestartPolicyNameEnum::Empty => RestartMode::NoRestart,
+          RestartPolicyNameEnum::UnlessStopped => {
+            RestartMode::UnlessStopped
+          }
+          RestartPolicyNameEnum::OnFailure => RestartMode::OnFailure,
+        }
+      });
+    }
+
+    resource::create::<Deployment>(&name, config, &user).await
+  }
+}
+
 impl Resolve<DeleteDeployment, User> for State {
   #[instrument(name = "DeleteDeployment", skip(self, user))]
   async fn resolve(
@@ -108,7 +203,7 @@ impl Resolve<RenameDeployment, User> for State {
 
     if container_state == DeploymentState::Unknown {
       return Err(anyhow!(
-        "cannot rename deployment when container status is unknown"
+        "Cannot rename Deployment when container status is unknown"
       ));
     }
 
@@ -124,7 +219,7 @@ impl Resolve<RenameDeployment, User> for State {
       None,
     )
     .await
-    .context("failed to update deployment name on db")?;
+    .context("Failed to update Deployment name on db")?;
 
     if container_state != DeploymentState::NotDeployed {
       let server =
@@ -135,20 +230,19 @@ impl Resolve<RenameDeployment, User> for State {
           new_name: name.clone(),
         })
         .await
-        .context("failed to rename container on server")?;
+        .context("Failed to rename container on server")?;
       update.logs.push(log);
     }
 
     update.push_simple_log(
-      "rename deployment",
+      "Rename Deployment",
       format!(
-        "renamed deployment from {} to {}",
+        "Renamed Deployment from {} to {}",
         deployment.name, name
       ),
     );
     update.finalize();
-
-    add_update(update.clone()).await?;
+    update.id = add_update(update.clone()).await?;
 
     Ok(update)
   }

bin/core/src/api/write/description.rs

@@ -2,7 +2,7 @@ use anyhow::anyhow;
 use komodo_client::{
   api::write::{UpdateDescription, UpdateDescriptionResponse},
   entities::{
-    alerter::Alerter, build::Build, builder::Builder,
+    action::Action, alerter::Alerter, build::Build, builder::Builder,
     deployment::Deployment, procedure::Procedure, repo::Repo,
     server::Server, server_template::ServerTemplate, stack::Stack,
     sync::ResourceSync, user::User, ResourceTarget,
@@ -84,6 +84,14 @@ impl Resolve<UpdateDescription, User> for State {
         )
         .await?;
       }
+      ResourceTarget::Action(id) => {
+        resource::update_description::<Action>(
+          &id,
+          &description,
+          &user,
+        )
+        .await?;
+      }
       ResourceTarget::ServerTemplate(id) => {
         resource::update_description::<ServerTemplate>(
           &id,

bin/core/src/api/write/mod.rs

@@ -13,6 +13,7 @@ use uuid::Uuid;
 
 use crate::{auth::auth_request, state::State};
 
+mod action;
 mod alerter;
 mod build;
 mod builder;
@@ -28,6 +29,7 @@ mod service_user;
 mod stack;
 mod sync;
 mod tag;
+mod user;
 mod user_group;
 mod variable;
 
@@ -40,6 +42,11 @@ mod variable;
 #[resolver_args(User)]
 #[serde(tag = "type", content = "params")]
 pub enum WriteRequest {
+  // ==== USER ====
+  UpdateUserUsername(UpdateUserUsername),
+  UpdateUserPassword(UpdateUserPassword),
+  DeleteUser(DeleteUser),
+
   // ==== SERVICE USER ====
   CreateServiceUser(CreateServiceUser),
   UpdateServiceUserDescription(UpdateServiceUserDescription),
@@ -73,6 +80,7 @@ pub enum WriteRequest {
   // ==== DEPLOYMENT ====
   CreateDeployment(CreateDeployment),
   CopyDeployment(CopyDeployment),
+  CreateDeploymentFromContainer(CreateDeploymentFromContainer),
   DeleteDeployment(DeleteDeployment),
   UpdateDeployment(UpdateDeployment),
   RenameDeployment(RenameDeployment),
@@ -82,6 +90,7 @@ pub enum WriteRequest {
   CopyBuild(CopyBuild),
   DeleteBuild(DeleteBuild),
   UpdateBuild(UpdateBuild),
+  RenameBuild(RenameBuild),
   RefreshBuildCache(RefreshBuildCache),
   CreateBuildWebhook(CreateBuildWebhook),
   DeleteBuildWebhook(DeleteBuildWebhook),
@@ -91,18 +100,21 @@ pub enum WriteRequest {
   CopyBuilder(CopyBuilder),
   DeleteBuilder(DeleteBuilder),
   UpdateBuilder(UpdateBuilder),
+  RenameBuilder(RenameBuilder),
 
   // ==== SERVER TEMPLATE ====
   CreateServerTemplate(CreateServerTemplate),
   CopyServerTemplate(CopyServerTemplate),
   DeleteServerTemplate(DeleteServerTemplate),
   UpdateServerTemplate(UpdateServerTemplate),
+  RenameServerTemplate(RenameServerTemplate),
 
   // ==== REPO ====
   CreateRepo(CreateRepo),
   CopyRepo(CopyRepo),
   DeleteRepo(DeleteRepo),
   UpdateRepo(UpdateRepo),
+  RenameRepo(RenameRepo),
   RefreshRepoCache(RefreshRepoCache),
   CreateRepoWebhook(CreateRepoWebhook),
   DeleteRepoWebhook(DeleteRepoWebhook),
@@ -112,20 +124,31 @@ pub enum WriteRequest {
   CopyAlerter(CopyAlerter),
   DeleteAlerter(DeleteAlerter),
   UpdateAlerter(UpdateAlerter),
+  RenameAlerter(RenameAlerter),
 
   // ==== PROCEDURE ====
   CreateProcedure(CreateProcedure),
   CopyProcedure(CopyProcedure),
   DeleteProcedure(DeleteProcedure),
   UpdateProcedure(UpdateProcedure),
+  RenameProcedure(RenameProcedure),
+
+  // ==== ACTION ====
+  CreateAction(CreateAction),
+  CopyAction(CopyAction),
+  DeleteAction(DeleteAction),
+  UpdateAction(UpdateAction),
+  RenameAction(RenameAction),
 
   // ==== SYNC ====
   CreateResourceSync(CreateResourceSync),
   CopyResourceSync(CopyResourceSync),
   DeleteResourceSync(DeleteResourceSync),
   UpdateResourceSync(UpdateResourceSync),
-  RefreshResourceSyncPending(RefreshResourceSyncPending),
+  RenameResourceSync(RenameResourceSync),
+  WriteSyncFileContents(WriteSyncFileContents),
   CommitSync(CommitSync),
+  RefreshResourceSyncPending(RefreshResourceSyncPending),
   CreateSyncWebhook(CreateSyncWebhook),
   DeleteSyncWebhook(DeleteSyncWebhook),
 
@@ -188,7 +211,10 @@ async fn handler(
 #[instrument(
   name = "WriteRequest",
   skip(user, request),
-  fields(user_id = user.id, request = format!("{:?}", request.extract_variant()))
+  fields(
+    user_id = user.id,
+    request = format!("{:?}", request.extract_variant())
+  )
 )]
 async fn task(
   req_id: Uuid,

bin/core/src/api/write/permissions.rs

@@ -387,6 +387,20 @@ async fn extract_resource_target_with_validation(
         .id;
       Ok((ResourceTargetVariant::Procedure, id))
     }
+    ResourceTarget::Action(ident) => {
+      let filter = match ObjectId::from_str(ident) {
+        Ok(id) => doc! { "_id": id },
+        Err(_) => doc! { "name": ident },
+      };
+      let id = db_client()
+        .actions
+        .find_one(filter)
+        .await
+        .context("failed to query db for actions")?
+        .context("no matching action found")?
+        .id;
+      Ok((ResourceTargetVariant::Action, id))
+    }
     ResourceTarget::ServerTemplate(ident) => {
       let filter = match ObjectId::from_str(ident) {
         Ok(id) => doc! { "_id": id },

bin/core/src/api/write/procedure.rs

@@ -1,7 +1,8 @@
 use komodo_client::{
   api::write::*,
   entities::{
-    permission::PermissionLevel, procedure::Procedure, user::User,
+    permission::PermissionLevel, procedure::Procedure,
+    update::Update, user::User,
   },
 };
 use resolver_api::Resolve;
@@ -48,6 +49,17 @@ impl Resolve<UpdateProcedure, User> for State {
   }
 }
 
+impl Resolve<RenameProcedure, User> for State {
+  #[instrument(name = "RenameProcedure", skip(self, user))]
+  async fn resolve(
+    &self,
+    RenameProcedure { id, name }: RenameProcedure,
+    user: User,
+  ) -> anyhow::Result<Update> {
+    resource::rename::<Procedure>(&id, &name, &user).await
+  }
+}
+
 impl Resolve<DeleteProcedure, User> for State {
   #[instrument(name = "DeleteProcedure", skip(self, user))]
   async fn resolve(

bin/core/src/api/write/repo.rs

@@ -1,27 +1,36 @@
 use anyhow::{anyhow, Context};
+use formatting::format_serror;
 use git::GitRes;
 use komodo_client::{
   api::write::*,
   entities::{
     config::core::CoreConfig,
+    komodo_timestamp,
     permission::PermissionLevel,
     repo::{PartialRepoConfig, Repo, RepoInfo},
+    server::Server,
+    to_komodo_name,
+    update::{Log, Update},
     user::User,
-    CloneArgs, NoData,
+    CloneArgs, NoData, Operation,
   },
 };
 use mongo_indexed::doc;
-use mungos::mongodb::bson::to_document;
+use mungos::{by_id::update_one_by_id, mongodb::bson::to_document};
 use octorust::types::{
   ReposCreateWebhookRequest, ReposCreateWebhookRequestConfig,
 };
+use periphery_client::api;
 use resolver_api::Resolve;
 
 use crate::{
   config::core_config,
-  helpers::git_token,
+  helpers::{
+    git_token, periphery_client,
+    update::{add_update, make_update},
+  },
   resource,
-  state::{db_client, github_client, State},
+  state::{action_states, db_client, github_client, State},
 };
 
 impl Resolve<CreateRepo, User> for State {
@@ -75,6 +84,81 @@ impl Resolve<UpdateRepo, User> for State {
   }
 }
 
+impl Resolve<RenameRepo, User> for State {
+  #[instrument(name = "RenameRepo", skip(self, user))]
+  async fn resolve(
+    &self,
+    RenameRepo { id, name }: RenameRepo,
+    user: User,
+  ) -> anyhow::Result<Update> {
+    let repo = resource::get_check_permissions::<Repo>(
+      &id,
+      &user,
+      PermissionLevel::Write,
+    )
+    .await?;
+
+    if repo.config.server_id.is_empty()
+      || !repo.config.path.is_empty()
+    {
+      return resource::rename::<Repo>(&repo.id, &name, &user).await;
+    }
+
+    // get the action state for the repo (or insert default).
+    let action_state =
+      action_states().repo.get_or_insert_default(&repo.id).await;
+
+    // Will check to ensure repo not already busy before updating, and return Err if so.
+    // The returned guard will set the action state back to default when dropped.
+    let _action_guard =
+      action_state.update(|state| state.renaming = true)?;
+
+    let name = to_komodo_name(&name);
+
+    let mut update = make_update(&repo, Operation::RenameRepo, &user);
+
+    update_one_by_id(
+      &db_client().repos,
+      &repo.id,
+      mungos::update::Update::Set(
+        doc! { "name": &name, "updated_at": komodo_timestamp() },
+      ),
+      None,
+    )
+    .await
+    .context("Failed to update Repo name on db")?;
+
+    let server =
+      resource::get::<Server>(&repo.config.server_id).await?;
+
+    let log = match periphery_client(&server)?
+      .request(api::git::RenameRepo {
+        curr_name: to_komodo_name(&repo.name),
+        new_name: name.clone(),
+      })
+      .await
+      .context("Failed to rename Repo directory on Server")
+    {
+      Ok(log) => log,
+      Err(e) => Log::error(
+        "Rename Repo directory failure",
+        format_serror(&e.into()),
+      ),
+    };
+
+    update.logs.push(log);
+
+    update.push_simple_log(
+      "Rename Repo",
+      format!("Renamed Repo from {} to {}", repo.name, name),
+    );
+    update.finalize();
+    update.id = add_update(update.clone()).await?;
+
+    Ok(update)
+  }
+}
+
 impl Resolve<RefreshRepoCache, User> for State {
   #[instrument(
     name = "RefreshRepoCache",

bin/core/src/api/write/server.rs

@@ -1,9 +1,7 @@
-use anyhow::Context;
 use formatting::format_serror;
 use komodo_client::{
   api::write::*,
   entities::{
-    komodo_timestamp,
     permission::PermissionLevel,
     server::Server,
     update::{Update, UpdateStatus},
@@ -11,7 +9,6 @@ use komodo_client::{
     Operation,
   },
 };
-use mungos::{by_id::update_one_by_id, mongodb::bson::doc};
 use periphery_client::api;
 use resolver_api::Resolve;
 
@@ -21,7 +18,7 @@ use crate::{
     update::{add_update, make_update, update_update},
   },
   resource,
-  state::{db_client, State},
+  state::State,
 };
 
 impl Resolve<CreateServer, User> for State {
@@ -64,25 +61,7 @@ impl Resolve<RenameServer, User> for State {
     RenameServer { id, name }: RenameServer,
     user: User,
   ) -> anyhow::Result<Update> {
-    let server = resource::get_check_permissions::<Server>(
-      &id,
-      &user,
-      PermissionLevel::Write,
-    )
-    .await?;
-    let mut update =
-      make_update(&server, Operation::RenameServer, &user);
-
-    update_one_by_id(&db_client().servers, &id, mungos::update::Update::Set(doc! { "name": &name, "updated_at": komodo_timestamp() }), None)
-      .await
-      .context("failed to update server on db. this name may already be taken.")?;
-    update.push_simple_log(
-      "rename server",
-      format!("renamed server {id} from {} to {name}", server.name),
-    );
-    update.finalize();
-    update.id = add_update(update.clone()).await?;
-    Ok(update)
+    resource::rename::<Server>(&id, &name, &user).await
   }
 }
 

bin/core/src/api/write/server_template.rs

@@ -1,11 +1,11 @@
 use komodo_client::{
   api::write::{
     CopyServerTemplate, CreateServerTemplate, DeleteServerTemplate,
-    UpdateServerTemplate,
+    RenameServerTemplate, UpdateServerTemplate,
   },
   entities::{
     permission::PermissionLevel, server_template::ServerTemplate,
-    user::User,
+    update::Update, user::User,
   },
 };
 use resolver_api::Resolve;
@@ -63,3 +63,14 @@ impl Resolve<UpdateServerTemplate, User> for State {
     resource::update::<ServerTemplate>(&id, config, &user).await
   }
 }
+
+impl Resolve<RenameServerTemplate, User> for State {
+  #[instrument(name = "RenameServerTemplate", skip(self, user))]
+  async fn resolve(
+    &self,
+    RenameServerTemplate { id, name }: RenameServerTemplate,
+    user: User,
+  ) -> anyhow::Result<Update> {
+    resource::rename::<ServerTemplate>(&id, &name, &user).await
+  }
+}

bin/core/src/api/write/stack.rs

@@ -4,7 +4,6 @@ use komodo_client::{
   api::write::*,
   entities::{
     config::core::CoreConfig,
-    komodo_timestamp,
     permission::PermissionLevel,
     server::ServerState,
     stack::{PartialStackConfig, Stack, StackInfo},
@@ -13,30 +12,28 @@ use komodo_client::{
     FileContents, NoData, Operation,
   },
 };
-use mungos::{
-  by_id::update_one_by_id,
-  mongodb::bson::{doc, to_document},
-};
+use mungos::mongodb::bson::{doc, to_document};
 use octorust::types::{
   ReposCreateWebhookRequest, ReposCreateWebhookRequestConfig,
 };
 use periphery_client::api::compose::{
   GetComposeContentsOnHost, GetComposeContentsOnHostResponse,
-  WriteComposeContentsToHost,
+  WriteCommitComposeContents, WriteComposeContentsToHost,
 };
 use resolver_api::Resolve;
 
 use crate::{
+  api::execute::pull_stack_inner,
   config::core_config,
   helpers::{
-    periphery_client,
+    git_token, periphery_client,
     query::get_server_with_state,
     update::{add_update, make_update},
   },
   resource,
   stack::{
     get_stack_and_server,
-    remote::{get_remote_compose_contents, RemoteComposeContents},
+    remote::{get_repo_compose_contents, RemoteComposeContents},
     services::extract_services_into_res,
   },
   state::{db_client, github_client, State},
@@ -100,36 +97,7 @@ impl Resolve<RenameStack, User> for State {
     RenameStack { id, name }: RenameStack,
     user: User,
   ) -> anyhow::Result<Update> {
-    let stack = resource::get_check_permissions::<Stack>(
-      &id,
-      &user,
-      PermissionLevel::Write,
-    )
-    .await?;
-
-    let mut update =
-      make_update(&stack, Operation::RenameStack, &user);
-
-    update_one_by_id(
-      &db_client().stacks,
-      &stack.id,
-      mungos::update::Update::Set(
-        doc! { "name": &name, "updated_at": komodo_timestamp() },
-      ),
-      None,
-    )
-    .await
-    .context("failed to update stack name on db")?;
-
-    update.push_simple_log(
-      "rename stack",
-      format!("renamed stack from {} to {}", stack.name, name),
-    );
-    update.finalize();
-
-    add_update(update.clone()).await?;
-
-    Ok(update)
+    resource::rename::<Stack>(&id, &name, &user).await
   }
 }
 
@@ -143,7 +111,7 @@ impl Resolve<WriteStackFileContents, User> for State {
     }: WriteStackFileContents,
     user: User,
   ) -> anyhow::Result<Update> {
-    let (stack, server) = get_stack_and_server(
+    let (mut stack, server) = get_stack_and_server(
       &stack,
       &user,
       PermissionLevel::Write,
@@ -151,9 +119,9 @@ impl Resolve<WriteStackFileContents, User> for State {
     )
     .await?;
 
-    if !stack.config.files_on_host {
+    if !stack.config.files_on_host && stack.config.repo.is_empty() {
       return Err(anyhow!(
-        "Stack is not configured to use files on host, can't write file contents"
+        "Stack is not configured to use Files on Host or Git Repo, can't write file contents"
       ));
     }
 
@@ -162,30 +130,72 @@ impl Resolve<WriteStackFileContents, User> for State {
 
     update.push_simple_log("File contents to write", &contents);
 
-    match periphery_client(&server)?
-      .request(WriteComposeContentsToHost {
-        name: stack.name,
-        run_directory: stack.config.run_directory,
-        file_path,
-        contents,
-      })
-      .await
-      .context("Failed to write contents to host")
-    {
-      Ok(log) => {
-        update.logs.push(log);
-      }
-      Err(e) => {
-        update.push_error_log(
-          "Write file contents",
-          format_serror(&e.into()),
-        );
-      }
-    };
+    let stack_id = stack.id.clone();
+
+    if stack.config.files_on_host {
+      match periphery_client(&server)?
+        .request(WriteComposeContentsToHost {
+          name: stack.name,
+          run_directory: stack.config.run_directory,
+          file_path,
+          contents,
+        })
+        .await
+        .context("Failed to write contents to host")
+      {
+        Ok(log) => {
+          update.logs.push(log);
+        }
+        Err(e) => {
+          update.push_error_log(
+            "Write file contents",
+            format_serror(&e.into()),
+          );
+        }
+      };
+    } else {
+      let git_token = if !stack.config.git_account.is_empty() {
+        git_token(
+          &stack.config.git_provider,
+          &stack.config.git_account,
+          |https| stack.config.git_https = https,
+        )
+        .await
+        .with_context(|| {
+          format!(
+            "Failed to get git token. | {} | {}",
+            stack.config.git_account, stack.config.git_provider
+          )
+        })?
+      } else {
+        None
+      };
+      match periphery_client(&server)?
+        .request(WriteCommitComposeContents {
+          stack,
+          username: Some(user.username),
+          file_path,
+          contents,
+          git_token,
+        })
+        .await
+        .context("Failed to write contents to host")
+      {
+        Ok(res) => {
+          update.logs.extend(res.logs);
+        }
+        Err(e) => {
+          update.push_error_log(
+            "Write file contents",
+            format_serror(&e.into()),
+          );
+        }
+      };
+    }
 
     if let Err(e) = State
       .resolve(
-        RefreshStackCache { stack: stack.id },
+        RefreshStackCache { stack: stack_id },
         stack_user().to_owned(),
       )
       .await
@@ -227,10 +237,11 @@ impl Resolve<RefreshStackCache, User> for State {
     .await?;
 
     let file_contents_empty = stack.config.file_contents.is_empty();
+    let repo_empty = stack.config.repo.is_empty();
 
     if !stack.config.files_on_host
       && file_contents_empty
-      && stack.config.repo.is_empty()
+      && repo_empty
     {
       // Nothing to do without one of these
       return Ok(NoData {});
@@ -248,56 +259,58 @@ impl Resolve<RefreshStackCache, User> for State {
       // =============
       // FILES ON HOST
       // =============
-      if stack.config.server_id.is_empty() {
-        (vec![], None, None, None, None)
+      let (server, state) = if stack.config.server_id.is_empty() {
+        (None, ServerState::Disabled)
       } else {
-        let (server, status) =
+        let (server, state) =
           get_server_with_state(&stack.config.server_id).await?;
-        if status != ServerState::Ok {
-          (vec![], None, None, None, None)
-        } else {
-          let GetComposeContentsOnHostResponse { contents, errors } =
-            match periphery_client(&server)?
-              .request(GetComposeContentsOnHost {
-                file_paths: stack.file_paths().to_vec(),
-                name: stack.name.clone(),
-                run_directory: stack.config.run_directory.clone(),
-              })
-              .await
-              .context(
-                "failed to get compose file contents from host",
-              ) {
-              Ok(res) => res,
-              Err(e) => GetComposeContentsOnHostResponse {
-                contents: Default::default(),
-                errors: vec![FileContents {
-                  path: stack.config.run_directory.clone(),
-                  contents: format_serror(&e.into()),
-                }],
-              },
-            };
+        (Some(server), state)
+      };
+      if state != ServerState::Ok {
+        (vec![], None, None, None, None)
+      } else if let Some(server) = server {
+        let GetComposeContentsOnHostResponse { contents, errors } =
+          match periphery_client(&server)?
+            .request(GetComposeContentsOnHost {
+              file_paths: stack.file_paths().to_vec(),
+              name: stack.name.clone(),
+              run_directory: stack.config.run_directory.clone(),
+            })
+            .await
+            .context("failed to get compose file contents from host")
+          {
+            Ok(res) => res,
+            Err(e) => GetComposeContentsOnHostResponse {
+              contents: Default::default(),
+              errors: vec![FileContents {
+                path: stack.config.run_directory.clone(),
+                contents: format_serror(&e.into()),
+              }],
+            },
+          };
 
-          let project_name = stack.project_name(true);
+        let project_name = stack.project_name(true);
 
-          let mut services = Vec::new();
+        let mut services = Vec::new();
 
-          for contents in &contents {
-            if let Err(e) = extract_services_into_res(
-              &project_name,
-              &contents.contents,
-              &mut services,
-            ) {
-              warn!(
+        for contents in &contents {
+          if let Err(e) = extract_services_into_res(
+            &project_name,
+            &contents.contents,
+            &mut services,
+          ) {
+            warn!(
                 "failed to extract stack services, things won't works correctly. stack: {} | {e:#}",
                 stack.name
               );
-            }
           }
-
-          (services, Some(contents), Some(errors), None, None)
         }
+
+        (services, Some(contents), Some(errors), None, None)
+      } else {
+        (vec![], None, None, None, None)
       }
-    } else if file_contents_empty {
+    } else if !repo_empty {
       // ================
       // REPO BASED STACK
       // ================
@@ -307,9 +320,8 @@ impl Resolve<RefreshStackCache, User> for State {
         hash: latest_hash,
         message: latest_message,
         ..
-      } =
-        get_remote_compose_contents(&stack, Some(&mut missing_files))
-          .await?;
+      } = get_repo_compose_contents(&stack, Some(&mut missing_files))
+        .await?;
 
       let project_name = stack.project_name(true);
 
@@ -347,21 +359,21 @@ impl Resolve<RefreshStackCache, User> for State {
         &mut services,
       ) {
         warn!(
-          "failed to extract stack services, things won't works correctly. stack: {} | {e:#}",
+          "Failed to extract Stack services for {}, things may not work correctly. | {e:#}",
           stack.name
         );
-        services.extend(stack.info.latest_services);
+        services.extend(stack.info.latest_services.clone());
       };
       (services, None, None, None, None)
     };
 
     let info = StackInfo {
       missing_files,
-      deployed_services: stack.info.deployed_services,
-      deployed_project_name: stack.info.deployed_project_name,
-      deployed_contents: stack.info.deployed_contents,
-      deployed_hash: stack.info.deployed_hash,
-      deployed_message: stack.info.deployed_message,
+      deployed_services: stack.info.deployed_services.clone(),
+      deployed_project_name: stack.info.deployed_project_name.clone(),
+      deployed_contents: stack.info.deployed_contents.clone(),
+      deployed_hash: stack.info.deployed_hash.clone(),
+      deployed_message: stack.info.deployed_message.clone(),
       latest_services,
       remote_contents,
       remote_errors,
@@ -381,6 +393,23 @@ impl Resolve<RefreshStackCache, User> for State {
       .await
       .context("failed to update stack info on db")?;
 
+    if (stack.config.poll_for_updates || stack.config.auto_update)
+      && !stack.config.server_id.is_empty()
+    {
+      let (server, state) =
+        get_server_with_state(&stack.config.server_id).await?;
+      if state == ServerState::Ok {
+        let name = stack.name.clone();
+        if let Err(e) =
+          pull_stack_inner(stack, None, &server, None).await
+        {
+          warn!(
+            "Failed to pull latest images for Stack {name} | {e:#}",
+          );
+        }
+      }
+    }
+
     Ok(NoData {})
   }
 }

bin/core/src/api/write/sync.rs

@@ -6,8 +6,10 @@ use komodo_client::{
   api::{read::ExportAllResourcesToToml, write::*},
   entities::{
     self,
+    action::Action,
     alert::{Alert, AlertData, SeverityLevel},
     alerter::Alerter,
+    all_logs_success,
     build::Build,
     builder::Builder,
     config::core::CoreConfig,
@@ -22,9 +24,10 @@ use komodo_client::{
     sync::{
       PartialResourceSyncConfig, ResourceSync, ResourceSyncInfo,
     },
-    update::Log,
+    to_komodo_name,
+    update::{Log, Update},
     user::{sync_user, User},
-    NoData, Operation, ResourceTarget,
+    CloneArgs, NoData, Operation, ResourceTarget,
   },
 };
 use mungos::{
@@ -35,6 +38,7 @@ use octorust::types::{
   ReposCreateWebhookRequest, ReposCreateWebhookRequestConfig,
 };
 use resolver_api::Resolve;
+use tokio::fs;
 
 use crate::{
   alert::send_alerts,
@@ -103,6 +107,297 @@ impl Resolve<UpdateResourceSync, User> for State {
   }
 }
 
+impl Resolve<RenameResourceSync, User> for State {
+  #[instrument(name = "RenameResourceSync", skip(self, user))]
+  async fn resolve(
+    &self,
+    RenameResourceSync { id, name }: RenameResourceSync,
+    user: User,
+  ) -> anyhow::Result<Update> {
+    resource::rename::<ResourceSync>(&id, &name, &user).await
+  }
+}
+
+impl Resolve<WriteSyncFileContents, User> for State {
+  async fn resolve(
+    &self,
+    WriteSyncFileContents {
+      sync,
+      resource_path,
+      file_path,
+      contents,
+    }: WriteSyncFileContents,
+    user: User,
+  ) -> anyhow::Result<Update> {
+    let sync = resource::get_check_permissions::<ResourceSync>(
+      &sync,
+      &user,
+      PermissionLevel::Write,
+    )
+    .await?;
+
+    if !sync.config.files_on_host && sync.config.repo.is_empty() {
+      return Err(anyhow!(
+        "This method is only for files on host, or repo based syncs."
+      ));
+    }
+
+    let mut update =
+      make_update(&sync, Operation::WriteSyncContents, &user);
+
+    update.push_simple_log("File contents", &contents);
+
+    let root = if sync.config.files_on_host {
+      core_config()
+        .sync_directory
+        .join(to_komodo_name(&sync.name))
+    } else {
+      let clone_args: CloneArgs = (&sync).into();
+      clone_args.unique_path(&core_config().repo_directory)?
+    };
+    let file_path =
+      file_path.parse::<PathBuf>().context("Invalid file path")?;
+    let resource_path = resource_path
+      .parse::<PathBuf>()
+      .context("Invalid resource path")?;
+    let full_path = root.join(&resource_path).join(&file_path);
+
+    if let Some(parent) = full_path.parent() {
+      let _ = fs::create_dir_all(parent).await;
+    }
+
+    if let Err(e) =
+      fs::write(&full_path, &contents).await.with_context(|| {
+        format!("Failed to write file contents to {full_path:?}")
+      })
+    {
+      update.push_error_log("Write file", format_serror(&e.into()));
+    } else {
+      update.push_simple_log(
+        "Write file",
+        format!("File written to {full_path:?}"),
+      );
+    };
+
+    if !all_logs_success(&update.logs) {
+      update.finalize();
+      update.id = add_update(update.clone()).await?;
+
+      return Ok(update);
+    }
+
+    if sync.config.files_on_host {
+      if let Err(e) = State
+        .resolve(RefreshResourceSyncPending { sync: sync.name }, user)
+        .await
+      {
+        update
+          .push_error_log("Refresh failed", format_serror(&e.into()));
+      }
+
+      update.finalize();
+      update.id = add_update(update.clone()).await?;
+
+      return Ok(update);
+    }
+
+    let commit_res = git::commit_file(
+      &format!("{}: Commit Resource File", user.username),
+      &root,
+      &resource_path.join(&file_path),
+    )
+    .await;
+
+    update.logs.extend(commit_res.logs);
+
+    if let Err(e) = State
+      .resolve(RefreshResourceSyncPending { sync: sync.name }, user)
+      .await
+    {
+      update
+        .push_error_log("Refresh failed", format_serror(&e.into()));
+    }
+
+    update.finalize();
+    update.id = add_update(update.clone()).await?;
+
+    Ok(update)
+  }
+}
+
+impl Resolve<CommitSync, User> for State {
+  #[instrument(name = "CommitSync", skip(self, user))]
+  async fn resolve(
+    &self,
+    CommitSync { sync }: CommitSync,
+    user: User,
+  ) -> anyhow::Result<Update> {
+    let sync = resource::get_check_permissions::<
+      entities::sync::ResourceSync,
+    >(&sync, &user, PermissionLevel::Write)
+    .await?;
+
+    let file_contents_empty = sync.config.file_contents_empty();
+
+    let fresh_sync = !sync.config.files_on_host
+      && sync.config.repo.is_empty()
+      && file_contents_empty;
+
+    if !sync.config.managed && !fresh_sync {
+      return Err(anyhow!(
+        "Cannot commit to sync. Enabled 'managed' mode."
+      ));
+    }
+
+    // Get this here so it can fail before update created.
+    let resource_path =
+      if sync.config.files_on_host || !sync.config.repo.is_empty() {
+        let resource_path = sync
+          .config
+          .resource_path
+          .first()
+          .context("Sync does not have resource path configured.")?
+          .parse::<PathBuf>()
+          .context("Invalid resource path")?;
+
+        if resource_path
+          .extension()
+          .context("Resource path missing '.toml' extension")?
+          != "toml"
+        {
+          return Err(anyhow!(
+            "Resource path missing '.toml' extension"
+          ));
+        }
+        Some(resource_path)
+      } else {
+        None
+      };
+
+    let res = State
+      .resolve(
+        ExportAllResourcesToToml {
+          tags: sync.config.match_tags.clone(),
+        },
+        sync_user().to_owned(),
+      )
+      .await?;
+
+    let mut update = make_update(&sync, Operation::CommitSync, &user);
+    update.id = add_update(update.clone()).await?;
+
+    update.logs.push(Log::simple("Resources", res.toml.clone()));
+
+    if sync.config.files_on_host {
+      let Some(resource_path) = resource_path else {
+        // Resource path checked above for files_on_host mode.
+        unreachable!()
+      };
+      let file_path = core_config()
+        .sync_directory
+        .join(to_komodo_name(&sync.name))
+        .join(&resource_path);
+      if let Some(parent) = file_path.parent() {
+        let _ = tokio::fs::create_dir_all(&parent).await;
+      };
+      if let Err(e) = tokio::fs::write(&file_path, &res.toml)
+        .await
+        .with_context(|| {
+          format!("Failed to write resource file to {file_path:?}",)
+        })
+      {
+        update.push_error_log(
+          "Write resource file",
+          format_serror(&e.into()),
+        );
+        update.finalize();
+        add_update(update.clone()).await?;
+        return Ok(update);
+      } else {
+        update.push_simple_log(
+          "Write contents",
+          format!("File contents written to {file_path:?}"),
+        );
+      }
+    } else if !sync.config.repo.is_empty() {
+      let Some(resource_path) = resource_path else {
+        // Resource path checked above for repo mode.
+        unreachable!()
+      };
+      // GIT REPO
+      let args: CloneArgs = (&sync).into();
+      let root = args.unique_path(&core_config().repo_directory)?;
+      match git::write_commit_file(
+        "Commit Sync",
+        &root,
+        &resource_path,
+        &res.toml,
+      )
+      .await
+      {
+        Ok(res) => update.logs.extend(res.logs),
+        Err(e) => {
+          update.push_error_log(
+            "Write resource file",
+            format_serror(&e.into()),
+          );
+          update.finalize();
+          add_update(update.clone()).await?;
+          return Ok(update);
+        }
+      }
+      // ===========
+      // UI DEFINED
+    } else if let Err(e) = db_client()
+      .resource_syncs
+      .update_one(
+        doc! { "name": &sync.name },
+        doc! { "$set": { "config.file_contents": res.toml } },
+      )
+      .await
+      .context("failed to update file_contents on db")
+    {
+      update.push_error_log(
+        "Write resource to database",
+        format_serror(&e.into()),
+      );
+      update.finalize();
+      add_update(update.clone()).await?;
+      return Ok(update);
+    }
+
+    if let Err(e) = State
+      .resolve(RefreshResourceSyncPending { sync: sync.name }, user)
+      .await
+    {
+      update.push_error_log(
+        "Refresh sync pending",
+        format_serror(&(&e).into()),
+      );
+    };
+
+    update.finalize();
+
+    // Need to manually update the update before cache refresh,
+    // and before broadcast with add_update.
+    // The Err case of to_document should be unreachable,
+    // but will fail to update cache in that case.
+    if let Ok(update_doc) = to_document(&update) {
+      let _ = update_one_by_id(
+        &db_client().updates,
+        &update.id,
+        mungos::update::Update::Set(update_doc),
+        None,
+      )
+      .await;
+      refresh_resource_sync_state_cache().await;
+    }
+    update_update(update.clone()).await?;
+
+    Ok(update)
+  }
+}
+
 impl Resolve<RefreshResourceSyncPending, User> for State {
   #[instrument(
     name = "RefreshResourceSyncPending",
@@ -190,6 +485,8 @@ impl Resolve<RefreshResourceSyncPending, User> for State {
           resources.servers,
           delete,
           &all_resources,
+          None,
+          None,
           &id_to_tags,
           &sync.config.match_tags,
           &mut diffs,
@@ -199,6 +496,8 @@ impl Resolve<RefreshResourceSyncPending, User> for State {
           resources.stacks,
           delete,
           &all_resources,
+          None,
+          None,
           &id_to_tags,
           &sync.config.match_tags,
           &mut diffs,
@@ -208,6 +507,8 @@ impl Resolve<RefreshResourceSyncPending, User> for State {
           resources.deployments,
           delete,
           &all_resources,
+          None,
+          None,
           &id_to_tags,
           &sync.config.match_tags,
           &mut diffs,
@@ -217,6 +518,8 @@ impl Resolve<RefreshResourceSyncPending, User> for State {
           resources.builds,
           delete,
           &all_resources,
+          None,
+          None,
           &id_to_tags,
           &sync.config.match_tags,
           &mut diffs,
@@ -226,6 +529,8 @@ impl Resolve<RefreshResourceSyncPending, User> for State {
           resources.repos,
           delete,
           &all_resources,
+          None,
+          None,
           &id_to_tags,
           &sync.config.match_tags,
           &mut diffs,
@@ -235,6 +540,19 @@ impl Resolve<RefreshResourceSyncPending, User> for State {
           resources.procedures,
           delete,
           &all_resources,
+          None,
+          None,
+          &id_to_tags,
+          &sync.config.match_tags,
+          &mut diffs,
+        )
+        .await?;
+        push_updates_for_view::<Action>(
+          resources.actions,
+          delete,
+          &all_resources,
+          None,
+          None,
           &id_to_tags,
           &sync.config.match_tags,
           &mut diffs,
@@ -244,6 +562,8 @@ impl Resolve<RefreshResourceSyncPending, User> for State {
           resources.builders,
           delete,
           &all_resources,
+          None,
+          None,
           &id_to_tags,
           &sync.config.match_tags,
           &mut diffs,
@@ -253,6 +573,8 @@ impl Resolve<RefreshResourceSyncPending, User> for State {
           resources.alerters,
           delete,
           &all_resources,
+          None,
+          None,
           &id_to_tags,
           &sync.config.match_tags,
           &mut diffs,
@@ -262,6 +584,8 @@ impl Resolve<RefreshResourceSyncPending, User> for State {
           resources.server_templates,
           delete,
           &all_resources,
+          None,
+          None,
           &id_to_tags,
           &sync.config.match_tags,
           &mut diffs,
@@ -271,6 +595,8 @@ impl Resolve<RefreshResourceSyncPending, User> for State {
           resources.resource_syncs,
           delete,
           &all_resources,
+          None,
+          None,
           &id_to_tags,
           &sync.config.match_tags,
           &mut diffs,
@@ -278,22 +604,26 @@ impl Resolve<RefreshResourceSyncPending, User> for State {
         .await?;
       }
 
-      let variable_updates =
+      let variable_updates = if sync.config.match_tags.is_empty() {
         crate::sync::variables::get_updates_for_view(
           &resources.variables,
-          // Delete doesn't work with variables when match tags are set
-          sync.config.match_tags.is_empty() && delete,
+          delete,
         )
-        .await?;
+        .await?
+      } else {
+        Default::default()
+      };
 
-      let user_group_updates =
+      let user_group_updates = if sync.config.match_tags.is_empty() {
         crate::sync::user_groups::get_updates_for_view(
           resources.user_groups,
-          // Delete doesn't work with user groups when match tags are set
-          sync.config.match_tags.is_empty() && delete,
+          delete,
           &all_resources,
         )
-        .await?;
+        .await?
+      } else {
+        Default::default()
+      };
 
       anyhow::Ok((
         diffs,
@@ -418,135 +748,6 @@ impl Resolve<RefreshResourceSyncPending, User> for State {
   }
 }
 
-impl Resolve<CommitSync, User> for State {
-  #[instrument(name = "CommitSync", skip(self, user))]
-  async fn resolve(
-    &self,
-    CommitSync { sync }: CommitSync,
-    user: User,
-  ) -> anyhow::Result<ResourceSync> {
-    let sync = resource::get_check_permissions::<
-      entities::sync::ResourceSync,
-    >(&sync, &user, PermissionLevel::Write)
-    .await?;
-
-    let fresh_sync = !sync.config.files_on_host
-      && sync.config.file_contents.is_empty()
-      && sync.config.repo.is_empty();
-
-    if !sync.config.managed && !fresh_sync {
-      return Err(anyhow!(
-        "Cannot commit to sync. Enabled 'managed' mode."
-      ));
-    }
-
-    let res = State
-      .resolve(
-        ExportAllResourcesToToml {
-          tags: sync.config.match_tags,
-        },
-        sync_user().to_owned(),
-      )
-      .await?;
-
-    let mut update = make_update(
-      ResourceTarget::ResourceSync(sync.id),
-      Operation::CommitSync,
-      &user,
-    );
-    update.id = add_update(update.clone()).await?;
-
-    if sync.config.files_on_host {
-      let path = sync
-        .config
-        .resource_path
-        .parse::<PathBuf>()
-        .context("Resource path is not valid file path")?;
-      let extension = path
-        .extension()
-        .context("Resource path missing '.toml' extension")?;
-      if extension != "toml" {
-        return Err(anyhow!("Wrong file extension. Expected '.toml', got '.{extension:?}'"));
-      }
-      if let Some(parent) = path.parent() {
-        let _ = tokio::fs::create_dir_all(&parent).await;
-      };
-      if let Err(e) =
-        tokio::fs::write(&sync.config.resource_path, &res.toml)
-          .await
-          .with_context(|| {
-            format!(
-              "Failed to write resource file to {}",
-              sync.config.resource_path
-            )
-          })
-      {
-        update.push_error_log(
-          "Write resource file",
-          format_serror(&e.into()),
-        );
-        update.finalize();
-        add_update(update).await?;
-        return resource::get::<ResourceSync>(&sync.name).await;
-      }
-    } else if let Err(e) = db_client()
-      .resource_syncs
-      .update_one(
-        doc! { "name": &sync.name },
-        doc! { "$set": { "config.file_contents": &res.toml } },
-      )
-      .await
-      .context("failed to update file_contents on db")
-    {
-      update.push_error_log(
-        "Write resource to database",
-        format_serror(&e.into()),
-      );
-      update.finalize();
-      add_update(update).await?;
-      return resource::get::<ResourceSync>(&sync.name).await;
-    }
-
-    update
-      .logs
-      .push(Log::simple("Committed resources", res.toml));
-
-    let res = match State
-      .resolve(RefreshResourceSyncPending { sync: sync.name }, user)
-      .await
-    {
-      Ok(sync) => Ok(sync),
-      Err(e) => {
-        update.push_error_log(
-          "Refresh sync pending",
-          format_serror(&(&e).into()),
-        );
-        Err(e)
-      }
-    };
-
-    update.finalize();
-
-    // Need to manually update the update before cache refresh,
-    // and before broadcast with add_update.
-    // The Err case of to_document should be unreachable,
-    // but will fail to update cache in that case.
-    if let Ok(update_doc) = to_document(&update) {
-      let _ = update_one_by_id(
-        &db_client().updates,
-        &update.id,
-        mungos::update::Update::Set(update_doc),
-        None,
-      )
-      .await;
-      refresh_resource_sync_state_cache().await;
-    }
-    update_update(update).await?;
-
-    res
-  }
-}
-
 impl Resolve<CreateSyncWebhook, User> for State {
   #[instrument(name = "CreateSyncWebhook", skip(self, user))]
   async fn resolve(

bin/core/src/api/write/tag.rs

@@ -7,7 +7,7 @@ use komodo_client::{
     UpdateTagsOnResourceResponse,
   },
   entities::{
-    alerter::Alerter, build::Build, builder::Builder,
+    action::Action, alerter::Alerter, build::Build, builder::Builder,
     deployment::Deployment, permission::PermissionLevel,
     procedure::Procedure, repo::Repo, server::Server,
     server_template::ServerTemplate, stack::Stack,
@@ -182,6 +182,15 @@ impl Resolve<UpdateTagsOnResource, User> for State {
         .await?;
         resource::update_tags::<Procedure>(&id, tags, user).await?
       }
+      ResourceTarget::Action(id) => {
+        resource::get_check_permissions::<Action>(
+          &id,
+          &user,
+          PermissionLevel::Write,
+        )
+        .await?;
+        resource::update_tags::<Action>(&id, tags, user).await?
+      }
       ResourceTarget::ServerTemplate(id) => {
         resource::get_check_permissions::<ServerTemplate>(
           &id,

bin/core/src/api/write/user.rs

@@ -0,0 +1,130 @@
+use std::str::FromStr;
+
+use anyhow::{anyhow, Context};
+use komodo_client::{
+  api::write::{
+    DeleteUser, DeleteUserResponse, UpdateUserPassword,
+    UpdateUserPasswordResponse, UpdateUserUsername,
+    UpdateUserUsernameResponse,
+  },
+  entities::{
+    user::{User, UserConfig},
+    NoData,
+  },
+};
+use mungos::mongodb::bson::{doc, oid::ObjectId};
+use resolver_api::Resolve;
+
+use crate::{
+  helpers::hash_password,
+  state::{db_client, State},
+};
+
+//
+
+impl Resolve<UpdateUserUsername, User> for State {
+  async fn resolve(
+    &self,
+    UpdateUserUsername { username }: UpdateUserUsername,
+    user: User,
+  ) -> anyhow::Result<UpdateUserUsernameResponse> {
+    if username.is_empty() {
+      return Err(anyhow!("Username cannot be empty."));
+    }
+    let db = db_client();
+    if db
+      .users
+      .find_one(doc! { "username": &username })
+      .await
+      .context("Failed to query for existing users")?
+      .is_some()
+    {
+      return Err(anyhow!("Username already taken."));
+    }
+    let id = ObjectId::from_str(&user.id)
+      .context("User id not valid ObjectId.")?;
+    db.users
+      .update_one(
+        doc! { "_id": id },
+        doc! { "$set": { "username": username } },
+      )
+      .await
+      .context("Failed to update user username on database.")?;
+    Ok(NoData {})
+  }
+}
+
+//
+
+impl Resolve<UpdateUserPassword, User> for State {
+  async fn resolve(
+    &self,
+    UpdateUserPassword { password }: UpdateUserPassword,
+    user: User,
+  ) -> anyhow::Result<UpdateUserPasswordResponse> {
+    let UserConfig::Local { .. } = user.config else {
+      return Err(anyhow!("User is not local user"));
+    };
+    if password.is_empty() {
+      return Err(anyhow!("Password cannot be empty."));
+    }
+    let id = ObjectId::from_str(&user.id)
+      .context("User id not valid ObjectId.")?;
+    let hashed_password = hash_password(password)?;
+    db_client()
+      .users
+      .update_one(
+        doc! { "_id": id },
+        doc! { "$set": {
+          "config.data.password": hashed_password
+        } },
+      )
+      .await
+      .context("Failed to update user password on database.")?;
+    Ok(NoData {})
+  }
+}
+
+//
+
+impl Resolve<DeleteUser, User> for State {
+  async fn resolve(
+    &self,
+    DeleteUser { user }: DeleteUser,
+    admin: User,
+  ) -> anyhow::Result<DeleteUserResponse> {
+    if !admin.admin {
+      return Err(anyhow!("Calling user is not admin."));
+    }
+    if admin.username == user || admin.id == user {
+      return Err(anyhow!("User cannot delete themselves."));
+    }
+    let query = if let Ok(id) = ObjectId::from_str(&user) {
+      doc! { "_id": id }
+    } else {
+      doc! { "username": user }
+    };
+    let db = db_client();
+    let Some(user) = db
+      .users
+      .find_one(query.clone())
+      .await
+      .context("Failed to query database for users.")?
+    else {
+      return Err(anyhow!("No user found with given id / username"));
+    };
+    if user.super_admin {
+      return Err(anyhow!("Cannot delete a super admin user."));
+    }
+    if user.admin && !admin.super_admin {
+      return Err(anyhow!(
+        "Only a Super Admin can delete an admin user."
+      ));
+    }
+    db.users
+      .delete_one(query)
+      .await
+      .context("Failed to delete user from database")?;
+    Ok(user)
+  }
+}

bin/core/src/api/write/variable.rs

@@ -81,7 +81,7 @@ impl Resolve<UpdateVariableValue, User> for State {
     let variable = get_variable(&name).await?;
 
     if value == variable.value {
-      return Err(anyhow!("no change"));
+      return Ok(variable);
     }
 
     db_client()

bin/core/src/auth/local.rs

@@ -16,12 +16,10 @@ use resolver_api::Resolve;
 
 use crate::{
   config::core_config,
-  state::State,
-  state::{db_client, jwt_client},
+  helpers::hash_password,
+  state::{db_client, jwt_client, State},
 };
 
-const BCRYPT_COST: u32 = 10;
-
 impl Resolve<CreateLocalUser, HeaderMap> for State {
   #[instrument(name = "CreateLocalUser", skip(self))]
   async fn resolve(
@@ -47,8 +45,7 @@ impl Resolve<CreateLocalUser, HeaderMap> for State {
       return Err(anyhow!("Password cannot be empty string"));
     }
 
-    let password = bcrypt::hash(password, BCRYPT_COST)
-      .context("failed to hash password")?;
+    let hashed_password = hash_password(password)?;
 
     let no_users_exist =
       db_client().users.find_one(Document::new()).await?.is_none();
@@ -71,7 +68,9 @@ impl Resolve<CreateLocalUser, HeaderMap> for State {
       last_update_view: 0,
       recents: Default::default(),
       all: Default::default(),
-      config: UserConfig::Local { password },
+      config: UserConfig::Local {
+        password: hashed_password,
+      },
     };
 
     let user_id = db_client()

bin/core/src/auth/oidc/mod.rs

@@ -92,13 +92,19 @@ async fn login(
   );
 
   let config = core_config();
-  let redirect = if !config.oidc_redirect.is_empty() {
-    Redirect::to(
-      auth_url
-        .as_str()
-        .replace(&config.oidc_provider, &config.oidc_redirect)
-        .as_str(),
-    )
+  let redirect = if !config.oidc_redirect_host.is_empty() {
+    let auth_url = auth_url.as_str();
+    let (protocol, rest) = auth_url
+      .split_once("://")
+      .context("Invalid URL: Missing protocol (eg 'https://')")?;
+    let host = rest
+      .split_once(['/', '?'])
+      .map(|(host, _)| host)
+      .unwrap_or(rest);
+    Redirect::to(&auth_url.replace(
+      &format!("{protocol}://{host}"),
+      &config.oidc_redirect_host,
+    ))
   } else {
     Redirect::to(auth_url.as_str())
   };

bin/core/src/cloud/aws/ec2.rs

@@ -212,21 +212,37 @@ async fn terminate_ec2_instance_inner(
   Ok(res)
 }
 
+/// Automatically retries 5 times, waiting 2 sec in between
 #[instrument(level = "debug")]
 async fn get_ec2_instance_status(
   client: &Client,
   instance_id: &str,
 ) -> anyhow::Result<Option<InstanceStatus>> {
-  let status = client
-    .describe_instance_status()
-    .instance_ids(instance_id)
-    .send()
+  let mut try_count = 1;
+  loop {
+    match async {
+      anyhow::Ok(
+        client
+          .describe_instance_status()
+          .instance_ids(instance_id)
+          .send()
+          .await
+          .context("failed to describe instance status from aws")?
+          .instance_statuses()
+          .first()
+          .cloned(),
+      )
+    }
     .await
-    .context("failed to get instance status from aws")?
-    .instance_statuses()
-    .first()
-    .cloned();
-  Ok(status)
+    {
+      Ok(res) => return Ok(res),
+      Err(e) if try_count > 4 => return Err(e),
+      Err(_) => {
+        tokio::time::sleep(Duration::from_secs(2)).await;
+        try_count += 1;
+      }
+    }
+  }
 }
 
 #[instrument(level = "debug")]
@@ -248,28 +264,43 @@ async fn get_ec2_instance_state_name(
   Ok(Some(state))
 }
 
+/// Automatically retries 5 times, waiting 2 sec in between
 #[instrument(level = "debug")]
 async fn get_ec2_instance_public_ip(
   client: &Client,
   instance_id: &str,
 ) -> anyhow::Result<String> {
-  let ip = client
-    .describe_instances()
-    .instance_ids(instance_id)
-    .send()
+  let mut try_count = 1;
+  loop {
+    match async {
+      anyhow::Ok(
+        client
+          .describe_instances()
+          .instance_ids(instance_id)
+          .send()
+          .await
+          .context("failed to describe instances from aws")?
+          .reservations()
+          .first()
+          .context("instance reservations is empty")?
+          .instances()
+          .first()
+          .context("instances is empty")?
+          .public_ip_address()
+          .context("instance has no public ip")?
+          .to_string(),
+      )
+    }
     .await
-    .context("failed to get instance status from aws")?
-    .reservations()
-    .first()
-    .context("instance reservations is empty")?
-    .instances()
-    .first()
-    .context("instances is empty")?
-    .public_ip_address()
-    .context("instance has no public ip")?
-    .to_string();
-
-  Ok(ip)
+    {
+      Ok(res) => return Ok(res),
+      Err(e) if try_count > 4 => return Err(e),
+      Err(_) => {
+        tokio::time::sleep(Duration::from_secs(2)).await;
+        try_count += 1;
+      }
+    }
+  }
 }
 
 fn handle_unknown_instance_type(

bin/core/src/config.rs

@@ -78,7 +78,7 @@ pub fn core_config() -> &'static CoreConfig {
       },
       oidc_enabled: env.komodo_oidc_enabled.unwrap_or(config.oidc_enabled),
       oidc_provider: env.komodo_oidc_provider.unwrap_or(config.oidc_provider),
-      oidc_redirect: env.komodo_oidc_redirect.unwrap_or(config.oidc_redirect),
+      oidc_redirect_host: env.komodo_oidc_redirect_host.unwrap_or(config.oidc_redirect_host),
       oidc_client_id: maybe_read_item_from_file(env.komodo_oidc_client_id_file,env
         .komodo_oidc_client_id)
         .unwrap_or(config.oidc_client_id),
@@ -144,9 +144,15 @@ pub fn core_config() -> &'static CoreConfig {
       jwt_ttl: env
         .komodo_jwt_ttl
         .unwrap_or(config.jwt_ttl),
+      sync_directory: env
+        .komodo_sync_directory
+        .unwrap_or(config.sync_directory),
       repo_directory: env
         .komodo_repo_directory
         .unwrap_or(config.repo_directory),
+      action_directory: env
+        .komodo_action_directory
+        .unwrap_or(config.action_directory),
       resource_poll_interval: env
         .komodo_resource_poll_interval
         .unwrap_or(config.resource_poll_interval),

bin/core/src/db.rs

@@ -1,4 +1,5 @@
 use komodo_client::entities::{
+  action::Action,
   alert::Alert,
   alerter::Alerter,
   api_key::ApiKey,
@@ -47,6 +48,7 @@ pub struct DbClient {
   pub builders: Collection<Builder>,
   pub repos: Collection<Repo>,
   pub procedures: Collection<Procedure>,
+  pub actions: Collection<Action>,
   pub alerters: Collection<Alerter>,
   pub server_templates: Collection<ServerTemplate>,
   pub resource_syncs: Collection<ResourceSync>,
@@ -115,6 +117,7 @@ impl DbClient {
       repos: resource_collection(&db, "Repo").await?,
       alerters: resource_collection(&db, "Alerter").await?,
       procedures: resource_collection(&db, "Procedure").await?,
+      actions: resource_collection(&db, "Action").await?,
       server_templates: resource_collection(&db, "ServerTemplate")
         .await?,
       resource_syncs: resource_collection(&db, "ResourceSync")

bin/core/src/helpers/action_state.rs

@@ -4,7 +4,8 @@ use anyhow::anyhow;
 use komodo_client::{
   busy::Busy,
   entities::{
-    build::BuildActionState, deployment::DeploymentActionState,
+    action::ActionActionState, build::BuildActionState,
+    deployment::DeploymentActionState,
     procedure::ProcedureActionState, repo::RepoActionState,
     server::ServerActionState, stack::StackActionState,
     sync::ResourceSyncActionState,
@@ -22,6 +23,7 @@ pub struct ActionStates {
   pub repo: Cache<String, Arc<ActionState<RepoActionState>>>,
   pub procedure:
     Cache<String, Arc<ActionState<ProcedureActionState>>>,
+  pub action: Cache<String, Arc<ActionState<ActionActionState>>>,
   pub resource_sync:
     Cache<String, Arc<ActionState<ResourceSyncActionState>>>,
   pub stack: Cache<String, Arc<ActionState<StackActionState>>>,

bin/core/src/helpers/builder.rs

@@ -31,7 +31,7 @@ use crate::{
 use super::periphery_client;
 
 const BUILDER_POLL_RATE_SECS: u64 = 2;
-const BUILDER_POLL_MAX_TRIES: usize = 30;
+const BUILDER_POLL_MAX_TRIES: usize = 60;
 
 #[instrument(skip_all, fields(builder_id = builder.id, update_id = update.id))]
 pub async fn get_builder_periphery(
@@ -42,9 +42,35 @@ pub async fn get_builder_periphery(
   update: &mut Update,
 ) -> anyhow::Result<(PeripheryClient, BuildCleanupData)> {
   match builder.config {
+    BuilderConfig::Url(config) => {
+      if config.address.is_empty() {
+        return Err(anyhow!(
+          "Builder has not yet configured an address"
+        ));
+      }
+      let periphery = PeripheryClient::new(
+        config.address,
+        if config.passkey.is_empty() {
+          core_config().passkey.clone()
+        } else {
+          config.passkey
+        },
+        Duration::from_secs(3),
+      );
+      periphery
+        .health_check()
+        .await
+        .context("Url Builder failed health check")?;
+      Ok((
+        periphery,
+        BuildCleanupData::Server {
+          repo_name: resource_name,
+        },
+      ))
+    }
     BuilderConfig::Server(config) => {
       if config.server_id.is_empty() {
-        return Err(anyhow!("builder has not configured a server"));
+        return Err(anyhow!("Builder has not configured a server"));
       }
       let server = resource::get::<Server>(&config.server_id).await?;
       let periphery = periphery_client(&server)?;
@@ -97,7 +123,7 @@ async fn get_aws_builder(
   let periphery_address =
     format!("{protocol}://{ip}:{}", config.port);
   let periphery =
-    PeripheryClient::new(&periphery_address, &core_config().passkey);
+    PeripheryClient::new(&periphery_address, &core_config().passkey, Duration::from_secs(3));
 
   let start_connect_ts = komodo_timestamp();
   let mut res = Ok(GetVersionResponse {

bin/core/src/helpers/mod.rs

@@ -3,8 +3,9 @@ use std::{str::FromStr, time::Duration};
 use anyhow::{anyhow, Context};
 use futures::future::join_all;
 use komodo_client::{
-  api::write::CreateServer,
+  api::write::{CreateBuilder, CreateServer},
   entities::{
+    builder::{PartialBuilderConfig, PartialServerBuilderConfig},
     komodo_timestamp,
     permission::{Permission, PermissionLevel, UserTarget},
     server::{PartialServerConfig, Server},
@@ -53,10 +54,6 @@ pub fn empty_or_only_spaces(word: &str) -> bool {
   true
 }
 
-pub fn random_duration(min_ms: u64, max_ms: u64) -> Duration {
-  Duration::from_millis(thread_rng().gen_range(min_ms..max_ms))
-}
-
 pub fn random_string(length: usize) -> String {
   thread_rng()
     .sample_iter(&Alphanumeric)
@@ -65,6 +62,15 @@ pub fn random_string(length: usize) -> String {
     .collect()
 }
 
+const BCRYPT_COST: u32 = 10;
+pub fn hash_password<P>(password: P) -> anyhow::Result<String>
+where
+  P: AsRef<[u8]>,
+{
+  bcrypt::hash(password, BCRYPT_COST)
+    .context("failed to hash password")
+}
+
 /// First checks db for token, then checks core config.
 /// Only errors if db call errors.
 /// Returns (token, use_https)
@@ -139,6 +145,7 @@ pub fn periphery_client(
   let client = PeripheryClient::new(
     &server.config.address,
     &core_config().passkey,
+    Duration::from_secs(server.config.timeout_seconds as u64),
   );
 
   Ok(client)
@@ -280,8 +287,8 @@ async fn startup_open_alert_cleanup() {
   }
 }
 
-/// Ensures a default server exists with the defined address
-pub async fn ensure_first_server() {
+/// Ensures a default server / builder exists with the defined address
+pub async fn ensure_first_server_and_builder() {
   let first_server = &core_config().first_server;
   if first_server.is_empty() {
     return;
@@ -295,23 +302,49 @@ pub async fn ensure_first_server() {
   else {
     return;
   };
-  if server.is_some() {
-    return;
-  }
+  let server = if let Some(server) = server {
+    server
+  } else {
+    match State
+      .resolve(
+        CreateServer {
+          name: format!("server-{}", random_string(5)),
+          config: PartialServerConfig {
+            address: Some(first_server.to_string()),
+            enabled: Some(true),
+            ..Default::default()
+          },
+        },
+        system_user().to_owned(),
+      )
+      .await
+    {
+      Ok(server) => server,
+      Err(e) => {
+        error!("Failed to initialize 'first_server'. Failed to CreateServer. {e:?}");
+        return;
+      }
+    }
+  };
+  let Ok(None) = db.builders
+    .find_one(Document::new()).await
+    .inspect_err(|e| error!("Failed to initialize 'first_builder'. Failed to query db. {e:?}")) else {
+      return;
+    };
   if let Err(e) = State
     .resolve(
-      CreateServer {
-        name: format!("server-{}", random_string(5)),
-        config: PartialServerConfig {
-          address: Some(first_server.to_string()),
-          enabled: Some(true),
-          ..Default::default()
-        },
+      CreateBuilder {
+        name: String::from("local"),
+        config: PartialBuilderConfig::Server(
+          PartialServerBuilderConfig {
+            server_id: Some(server.id),
+          },
+        ),
       },
       system_user().to_owned(),
     )
     .await
   {
-    error!("Failed to initialize 'first_server'. Failed to CreateServer. {e:?}");
+    error!("Failed to initialize 'first_builder'. Failed to CreateBuilder. {e:?}");
   }
 }

bin/core/src/helpers/procedure.rs

@@ -4,9 +4,14 @@ use anyhow::{anyhow, Context};
 use formatting::{bold, colored, format_serror, muted, Color};
 use futures::future::join_all;
 use komodo_client::{
-  api::execute::Execution,
+  api::execute::*,
   entities::{
+    action::Action,
+    build::Build,
+    deployment::Deployment,
     procedure::Procedure,
+    repo::Repo,
+    stack::Stack,
     update::{Log, Update},
     user::procedure_user,
   },
@@ -17,6 +22,7 @@ use tokio::sync::Mutex;
 
 use crate::{
   api::execute::ExecuteRequest,
+  resource::{list_full_for_user_using_pattern, KomodoResource},
   state::{db_client, State},
 };
 
@@ -79,11 +85,94 @@ pub async fn execute_procedure(
 #[allow(dependency_on_unit_never_type_fallback)]
 #[instrument(skip(update))]
 async fn execute_stage(
-  executions: Vec<Execution>,
+  _executions: Vec<Execution>,
   parent_id: &str,
   parent_name: &str,
   update: &Mutex<Update>,
 ) -> anyhow::Result<()> {
+  let mut executions = Vec::with_capacity(_executions.capacity());
+  for execution in _executions {
+    match execution {
+      Execution::BatchRunAction(exec) => {
+        extend_batch_exection::<BatchRunAction>(
+          &exec.pattern,
+          &mut executions,
+        )
+        .await?;
+      }
+      Execution::BatchRunProcedure(exec) => {
+        extend_batch_exection::<BatchRunProcedure>(
+          &exec.pattern,
+          &mut executions,
+        )
+        .await?;
+      }
+      Execution::BatchRunBuild(exec) => {
+        extend_batch_exection::<BatchRunBuild>(
+          &exec.pattern,
+          &mut executions,
+        )
+        .await?;
+      }
+      Execution::BatchCloneRepo(exec) => {
+        extend_batch_exection::<BatchCloneRepo>(
+          &exec.pattern,
+          &mut executions,
+        )
+        .await?;
+      }
+      Execution::BatchPullRepo(exec) => {
+        extend_batch_exection::<BatchPullRepo>(
+          &exec.pattern,
+          &mut executions,
+        )
+        .await?;
+      }
+      Execution::BatchBuildRepo(exec) => {
+        extend_batch_exection::<BatchBuildRepo>(
+          &exec.pattern,
+          &mut executions,
+        )
+        .await?;
+      }
+      Execution::BatchDeploy(exec) => {
+        extend_batch_exection::<BatchDeploy>(
+          &exec.pattern,
+          &mut executions,
+        )
+        .await?;
+      }
+      Execution::BatchDestroyDeployment(exec) => {
+        extend_batch_exection::<BatchDestroyDeployment>(
+          &exec.pattern,
+          &mut executions,
+        )
+        .await?;
+      }
+      Execution::BatchDeployStack(exec) => {
+        extend_batch_exection::<BatchDeployStack>(
+          &exec.pattern,
+          &mut executions,
+        )
+        .await?;
+      }
+      Execution::BatchDeployStackIfChanged(exec) => {
+        extend_batch_exection::<BatchDeployStackIfChanged>(
+          &exec.pattern,
+          &mut executions,
+        )
+        .await?;
+      }
+      Execution::BatchDestroyStack(exec) => {
+        extend_batch_exection::<BatchDestroyStack>(
+          &exec.pattern,
+          &mut executions,
+        )
+        .await?;
+      }
+      execution => executions.push(execution),
+    }
+  }
   let futures = executions.into_iter().map(|execution| async move {
     let now = Instant::now();
     add_line_to_update(
@@ -146,6 +235,34 @@ async fn execute_execution(
       )
       .await?
     }
+    Execution::BatchRunProcedure(_) => {
+      // All batch executions must be expanded in `execute_stage`
+      return Err(anyhow!(
+        "Batch method BatchRunProcedure not implemented correctly"
+      ));
+    }
+    Execution::RunAction(req) => {
+      let req = ExecuteRequest::RunAction(req);
+      let update = init_execution_update(&req, &user).await?;
+      let ExecuteRequest::RunAction(req) = req else {
+        unreachable!()
+      };
+      let update_id = update.id.clone();
+      handle_resolve_result(
+        State
+          .resolve(req, (user, update))
+          .await
+          .context("Failed at RunAction"),
+        &update_id,
+      )
+      .await?
+    }
+    Execution::BatchRunAction(_) => {
+      // All batch executions must be expanded in `execute_stage`
+      return Err(anyhow!(
+        "Batch method BatchRunAction not implemented correctly"
+      ));
+    }
     Execution::RunBuild(req) => {
       let req = ExecuteRequest::RunBuild(req);
       let update = init_execution_update(&req, &user).await?;
@@ -162,6 +279,12 @@ async fn execute_execution(
       )
       .await?
     }
+    Execution::BatchRunBuild(_) => {
+      // All batch executions must be expanded in `execute_stage`
+      return Err(anyhow!(
+        "Batch method BatchRunBuild not implemented correctly"
+      ));
+    }
     Execution::CancelBuild(req) => {
       let req = ExecuteRequest::CancelBuild(req);
       let update = init_execution_update(&req, &user).await?;
@@ -194,6 +317,28 @@ async fn execute_execution(
       )
       .await?
     }
+    Execution::BatchDeploy(_) => {
+      // All batch executions must be expanded in `execute_stage`
+      return Err(anyhow!(
+        "Batch method BatchDeploy not implemented correctly"
+      ));
+    }
+    Execution::PullDeployment(req) => {
+      let req = ExecuteRequest::PullDeployment(req);
+      let update = init_execution_update(&req, &user).await?;
+      let ExecuteRequest::PullDeployment(req) = req else {
+        unreachable!()
+      };
+      let update_id = update.id.clone();
+      handle_resolve_result(
+        State
+          .resolve(req, (user, update))
+          .await
+          .context("Failed at PullDeployment"),
+        &update_id,
+      )
+      .await?
+    }
     Execution::StartDeployment(req) => {
       let req = ExecuteRequest::StartDeployment(req);
       let update = init_execution_update(&req, &user).await?;
@@ -290,6 +435,12 @@ async fn execute_execution(
       )
       .await?
     }
+    Execution::BatchDestroyDeployment(_) => {
+      // All batch executions must be expanded in `execute_stage`
+      return Err(anyhow!(
+        "Batch method BatchDestroyDeployment not implemented correctly"
+      ));
+    }
     Execution::CloneRepo(req) => {
       let req = ExecuteRequest::CloneRepo(req);
       let update = init_execution_update(&req, &user).await?;
@@ -306,6 +457,12 @@ async fn execute_execution(
       )
       .await?
     }
+    Execution::BatchCloneRepo(_) => {
+      // All batch executions must be expanded in `execute_stage`
+      return Err(anyhow!(
+        "Batch method BatchCloneRepo not implemented correctly"
+      ));
+    }
     Execution::PullRepo(req) => {
       let req = ExecuteRequest::PullRepo(req);
       let update = init_execution_update(&req, &user).await?;
@@ -322,6 +479,12 @@ async fn execute_execution(
       )
       .await?
     }
+    Execution::BatchPullRepo(_) => {
+      // All batch executions must be expanded in `execute_stage`
+      return Err(anyhow!(
+        "Batch method BatchPullRepo not implemented correctly"
+      ));
+    }
     Execution::BuildRepo(req) => {
       let req = ExecuteRequest::BuildRepo(req);
       let update = init_execution_update(&req, &user).await?;
@@ -338,6 +501,12 @@ async fn execute_execution(
       )
       .await?
     }
+    Execution::BatchBuildRepo(_) => {
+      // All batch executions must be expanded in `execute_stage`
+      return Err(anyhow!(
+        "Batch method BatchBuildRepo not implemented correctly"
+      ));
+    }
     Execution::CancelRepoBuild(req) => {
       let req = ExecuteRequest::CancelRepoBuild(req);
       let update = init_execution_update(&req, &user).await?;
@@ -706,6 +875,11 @@ async fn execute_execution(
       )
       .await?
     }
+    // Exception: This is a write operation.
+    Execution::CommitSync(req) => State
+      .resolve(req, user)
+      .await
+      .context("Failed at CommitSync")?,
     Execution::DeployStack(req) => {
       let req = ExecuteRequest::DeployStack(req);
       let update = init_execution_update(&req, &user).await?;
@@ -722,6 +896,50 @@ async fn execute_execution(
       )
       .await?
     }
+    Execution::BatchDeployStack(_) => {
+      // All batch executions must be expanded in `execute_stage`
+      return Err(anyhow!(
+        "Batch method BatchDeployStack not implemented correctly"
+      ));
+    }
+    Execution::DeployStackIfChanged(req) => {
+      let req = ExecuteRequest::DeployStackIfChanged(req);
+      let update = init_execution_update(&req, &user).await?;
+      let ExecuteRequest::DeployStackIfChanged(req) = req else {
+        unreachable!()
+      };
+      let update_id = update.id.clone();
+      handle_resolve_result(
+        State
+          .resolve(req, (user, update))
+          .await
+          .context("Failed at DeployStackIfChanged"),
+        &update_id,
+      )
+      .await?
+    }
+    Execution::BatchDeployStackIfChanged(_) => {
+      // All batch executions must be expanded in `execute_stage`
+      return Err(anyhow!(
+        "Batch method BatchDeployStackIfChanged not implemented correctly"
+      ));
+    }
+    Execution::PullStack(req) => {
+      let req = ExecuteRequest::PullStack(req);
+      let update = init_execution_update(&req, &user).await?;
+      let ExecuteRequest::PullStack(req) = req else {
+        unreachable!()
+      };
+      let update_id = update.id.clone();
+      handle_resolve_result(
+        State
+          .resolve(req, (user, update))
+          .await
+          .context("Failed at PullStack"),
+        &update_id,
+      )
+      .await?
+    }
     Execution::StartStack(req) => {
       let req = ExecuteRequest::StartStack(req);
       let update = init_execution_update(&req, &user).await?;
@@ -818,6 +1036,12 @@ async fn execute_execution(
       )
       .await?
     }
+    Execution::BatchDestroyStack(_) => {
+      // All batch executions must be expanded in `execute_stage`
+      return Err(anyhow!(
+        "Batch method BatchDestroyStack not implemented correctly"
+      ));
+    }
     Execution::Sleep(req) => {
       let duration = Duration::from_millis(req.duration_ms as u64);
       tokio::time::sleep(duration).await;
@@ -875,3 +1099,122 @@ async fn add_line_to_update(update: &Mutex<Update>, line: &str) {
     error!("Failed to update an update during procedure | {e:#}");
   };
 }
+
+async fn extend_batch_exection<E: ExtendBatch>(
+  pattern: &str,
+  executions: &mut Vec<Execution>,
+) -> anyhow::Result<()> {
+  let more = list_full_for_user_using_pattern::<E::Resource>(
+    pattern,
+    Default::default(),
+    procedure_user(),
+    &[],
+  )
+  .await?
+  .into_iter()
+  .map(|resource| E::single_execution(resource.name));
+  executions.extend(more);
+  Ok(())
+}
+
+trait ExtendBatch {
+  type Resource: KomodoResource;
+  fn single_execution(name: String) -> Execution;
+}
+
+impl ExtendBatch for BatchRunProcedure {
+  type Resource = Procedure;
+  fn single_execution(procedure: String) -> Execution {
+    Execution::RunProcedure(RunProcedure { procedure })
+  }
+}
+
+impl ExtendBatch for BatchRunAction {
+  type Resource = Action;
+  fn single_execution(action: String) -> Execution {
+    Execution::RunAction(RunAction { action })
+  }
+}
+
+impl ExtendBatch for BatchRunBuild {
+  type Resource = Build;
+  fn single_execution(build: String) -> Execution {
+    Execution::RunBuild(RunBuild { build })
+  }
+}
+
+impl ExtendBatch for BatchCloneRepo {
+  type Resource = Repo;
+  fn single_execution(repo: String) -> Execution {
+    Execution::CloneRepo(CloneRepo { repo })
+  }
+}
+
+impl ExtendBatch for BatchPullRepo {
+  type Resource = Repo;
+  fn single_execution(repo: String) -> Execution {
+    Execution::PullRepo(PullRepo { repo })
+  }
+}
+
+impl ExtendBatch for BatchBuildRepo {
+  type Resource = Repo;
+  fn single_execution(repo: String) -> Execution {
+    Execution::BuildRepo(BuildRepo { repo })
+  }
+}
+
+impl ExtendBatch for BatchDeploy {
+  type Resource = Deployment;
+  fn single_execution(deployment: String) -> Execution {
+    Execution::Deploy(Deploy {
+      deployment,
+      stop_signal: None,
+      stop_time: None,
+    })
+  }
+}
+
+impl ExtendBatch for BatchDestroyDeployment {
+  type Resource = Deployment;
+  fn single_execution(deployment: String) -> Execution {
+    Execution::DestroyDeployment(DestroyDeployment {
+      deployment,
+      signal: None,
+      time: None,
+    })
+  }
+}
+
+impl ExtendBatch for BatchDeployStack {
+  type Resource = Stack;
+  fn single_execution(stack: String) -> Execution {
+    Execution::DeployStack(DeployStack {
+      stack,
+      service: None,
+      stop_time: None,
+    })
+  }
+}
+
+impl ExtendBatch for BatchDeployStackIfChanged {
+  type Resource = Stack;
+  fn single_execution(stack: String) -> Execution {
+    Execution::DeployStackIfChanged(DeployStackIfChanged {
+      stack,
+      stop_time: None,
+    })
+  }
+}
+
+impl ExtendBatch for BatchDestroyStack {
+  type Resource = Stack;
+  fn single_execution(stack: String) -> Execution {
+    Execution::DestroyStack(DestroyStack {
+      stack,
+      service: None,
+      remove_orphans: false,
+      stop_time: None,
+    })
+  }
+}

bin/core/src/helpers/query.rs

@@ -2,6 +2,7 @@ use std::{collections::HashMap, str::FromStr};
 
 use anyhow::{anyhow, Context};
 use komodo_client::entities::{
+  action::Action,
   alerter::Alerter,
   build::Build,
   builder::Builder,
@@ -102,7 +103,7 @@ pub fn get_stack_state_from_containers(
     })
     .collect::<Vec<_>>();
   let containers = containers.iter().filter(|container| {
-    services.iter().any(|StackServiceNames { service_name, container_name }| {
+    services.iter().any(|StackServiceNames { service_name, container_name, .. }| {
       match compose_container_match_regex(container_name)
         .with_context(|| format!("failed to construct container name matching regex for service {service_name}")) 
       {
@@ -117,7 +118,7 @@ pub fn get_stack_state_from_containers(
   if containers.is_empty() {
     return StackState::Down;
   }
-  if services.len() != containers.len() {
+  if services.len() > containers.len() {
     return StackState::Unhealthy;
   }
   let running = containers.iter().all(|container| {
@@ -201,6 +202,14 @@ pub async fn get_tag_check_owner(
   Err(anyhow!("user must be tag owner or admin"))
 }
 
+pub async fn get_all_tags(
+  filter: impl Into<Option<Document>>,
+) -> anyhow::Result<Vec<Tag>> {
+  find_collect(&db_client().tags, filter, None)
+    .await
+    .context("failed to query db for tags")
+}
+
 pub async fn get_id_to_tags(
   filter: impl Into<Option<Document>>,
 ) -> anyhow::Result<HashMap<String, Tag>> {
@@ -283,6 +292,9 @@ pub async fn get_user_permission_on_target(
     ResourceTarget::Procedure(id) => {
       get_user_permission_on_resource::<Procedure>(user, id).await
     }
+    ResourceTarget::Action(id) => {
+      get_user_permission_on_resource::<Action>(user, id).await
+    }
     ResourceTarget::ServerTemplate(id) => {
       get_user_permission_on_resource::<ServerTemplate>(user, id)
         .await

bin/core/src/helpers/update.rs

@@ -1,5 +1,6 @@
 use anyhow::Context;
 use komodo_client::entities::{
+  action::Action,
   build::Build,
   deployment::Deployment,
   komodo_timestamp,
@@ -260,6 +261,15 @@ pub async fn init_execution_update(
         resource::get::<Deployment>(&data.deployment).await?.id,
       ),
     ),
+    ExecuteRequest::BatchDeploy(_data) => {
+      return Ok(Default::default())
+    }
+    ExecuteRequest::PullDeployment(data) => (
+      Operation::PullDeployment,
+      ResourceTarget::Deployment(
+        resource::get::<Deployment>(&data.deployment).await?.id,
+      ),
+    ),
     ExecuteRequest::StartDeployment(data) => (
       Operation::StartDeployment,
       ResourceTarget::Deployment(
@@ -296,6 +306,9 @@ pub async fn init_execution_update(
         resource::get::<Deployment>(&data.deployment).await?.id,
       ),
     ),
+    ExecuteRequest::BatchDestroyDeployment(_data) => {
+      return Ok(Default::default())
+    }
 
     // Build
     ExecuteRequest::RunBuild(data) => (
@@ -304,6 +317,9 @@ pub async fn init_execution_update(
         resource::get::<Build>(&data.build).await?.id,
       ),
     ),
+    ExecuteRequest::BatchRunBuild(_data) => {
+      return Ok(Default::default())
+    }
     ExecuteRequest::CancelBuild(data) => (
       Operation::CancelBuild,
       ResourceTarget::Build(
@@ -318,18 +334,27 @@ pub async fn init_execution_update(
         resource::get::<Repo>(&data.repo).await?.id,
       ),
     ),
+    ExecuteRequest::BatchCloneRepo(_data) => {
+      return Ok(Default::default())
+    }
     ExecuteRequest::PullRepo(data) => (
       Operation::PullRepo,
       ResourceTarget::Repo(
         resource::get::<Repo>(&data.repo).await?.id,
       ),
     ),
+    ExecuteRequest::BatchPullRepo(_data) => {
+      return Ok(Default::default())
+    }
     ExecuteRequest::BuildRepo(data) => (
       Operation::BuildRepo,
       ResourceTarget::Repo(
         resource::get::<Repo>(&data.repo).await?.id,
       ),
     ),
+    ExecuteRequest::BatchBuildRepo(_data) => {
+      return Ok(Default::default())
+    }
     ExecuteRequest::CancelRepoBuild(data) => (
       Operation::CancelRepoBuild,
       ResourceTarget::Repo(
@@ -344,6 +369,20 @@ pub async fn init_execution_update(
         resource::get::<Procedure>(&data.procedure).await?.id,
       ),
     ),
+    ExecuteRequest::BatchRunProcedure(_) => {
+      return Ok(Default::default())
+    }
+
+    // Action
+    ExecuteRequest::RunAction(data) => (
+      Operation::RunAction,
+      ResourceTarget::Action(
+        resource::get::<Action>(&data.action).await?.id,
+      ),
+    ),
+    ExecuteRequest::BatchRunAction(_) => {
+      return Ok(Default::default())
+    }
 
     // Server template
     ExecuteRequest::LaunchServer(data) => (
@@ -365,11 +404,27 @@ pub async fn init_execution_update(
 
     // Stack
     ExecuteRequest::DeployStack(data) => (
+      if data.service.is_some() {
+        Operation::DeployStackService
+      } else {
+        Operation::DeployStack
+      },
+      ResourceTarget::Stack(
+        resource::get::<Stack>(&data.stack).await?.id,
+      ),
+    ),
+    ExecuteRequest::BatchDeployStack(_data) => {
+      return Ok(Default::default())
+    }
+    ExecuteRequest::DeployStackIfChanged(data) => (
       Operation::DeployStack,
       ResourceTarget::Stack(
         resource::get::<Stack>(&data.stack).await?.id,
       ),
     ),
+    ExecuteRequest::BatchDeployStackIfChanged(_data) => {
+      return Ok(Default::default())
+    }
     ExecuteRequest::StartStack(data) => (
       if data.service.is_some() {
         Operation::StartStackService
@@ -380,6 +435,16 @@ pub async fn init_execution_update(
         resource::get::<Stack>(&data.stack).await?.id,
       ),
     ),
+    ExecuteRequest::PullStack(data) => (
+      if data.service.is_some() {
+        Operation::PullStackService
+      } else {
+        Operation::PullStack
+      },
+      ResourceTarget::Stack(
+        resource::get::<Stack>(&data.stack).await?.id,
+      ),
+    ),
     ExecuteRequest::RestartStack(data) => (
       if data.service.is_some() {
         Operation::RestartStackService
@@ -421,15 +486,25 @@ pub async fn init_execution_update(
       ),
     ),
     ExecuteRequest::DestroyStack(data) => (
-      Operation::DestroyStack,
+      if data.service.is_some() {
+        Operation::DestroyStackService
+      } else {
+        Operation::DestroyStack
+      },
       ResourceTarget::Stack(
         resource::get::<Stack>(&data.stack).await?.id,
       ),
     ),
+    ExecuteRequest::BatchDestroyStack(_data) => {
+      return Ok(Default::default())
+    }
   };
   let mut update = make_update(target, operation, user);
   update.in_progress();
-  // Don't actually send it here, let the handlers send it after they can set action state.
-  update.id = add_update_without_send(&update).await?;
+  // Hold off on even adding update for DeployStackIfChanged
+  if !matches!(&request, ExecuteRequest::DeployStackIfChanged(_)) {
+    // Don't actually send it here, let the handlers send it after they can set action state.
+    update.id = add_update_without_send(&update).await?;
+  }
   Ok(update)
 }

bin/core/src/listener/github/build.rs

@@ -1,56 +0,0 @@
-use std::sync::OnceLock;
-
-use anyhow::anyhow;
-use axum::http::HeaderMap;
-use komodo_client::{
-  api::execute::RunBuild,
-  entities::{build::Build, user::git_webhook_user},
-};
-use resolver_api::Resolve;
-
-use crate::{
-  api::execute::ExecuteRequest,
-  helpers::update::init_execution_update, resource, state::State,
-};
-
-use super::{extract_branch, verify_gh_signature, ListenerLockCache};
-
-fn build_locks() -> &'static ListenerLockCache {
-  static BUILD_LOCKS: OnceLock<ListenerLockCache> = OnceLock::new();
-  BUILD_LOCKS.get_or_init(Default::default)
-}
-
-pub async fn handle_build_webhook(
-  build_id: String,
-  headers: HeaderMap,
-  body: String,
-) -> anyhow::Result<()> {
-  // Acquire and hold lock to make a task queue for
-  // subsequent listener calls on same resource.
-  // It would fail if we let it go through from action state busy.
-  let lock = build_locks().get_or_insert_default(&build_id).await;
-  let _lock = lock.lock().await;
-
-  let build = resource::get::<Build>(&build_id).await?;
-
-  verify_gh_signature(headers, &body, &build.config.webhook_secret)
-    .await?;
-
-  if !build.config.webhook_enabled {
-    return Err(anyhow!("build does not have webhook enabled"));
-  }
-
-  let request_branch = extract_branch(&body)?;
-  if request_branch != build.config.branch {
-    return Err(anyhow!("request branch does not match expected"));
-  }
-
-  let user = git_webhook_user().to_owned();
-  let req = ExecuteRequest::RunBuild(RunBuild { build: build_id });
-  let update = init_execution_update(&req, &user).await?;
-  let ExecuteRequest::RunBuild(req) = req else {
-    unreachable!()
-  };
-  State.resolve(req, (user, update)).await?;
-  Ok(())
-}

bin/core/src/listener/github/mod.rs

@@ -1,263 +0,0 @@
-use std::sync::Arc;
-
-use anyhow::{anyhow, Context};
-use axum::{extract::Path, http::HeaderMap, routing::post, Router};
-use hex::ToHex;
-use hmac::{Hmac, Mac};
-use serde::Deserialize;
-use sha2::Sha256;
-use tokio::sync::Mutex;
-use tracing::Instrument;
-
-use crate::{
-  config::core_config,
-  helpers::{cache::Cache, random_duration},
-};
-
-mod build;
-mod procedure;
-mod repo;
-mod stack;
-mod sync;
-
-type HmacSha256 = Hmac<Sha256>;
-
-#[derive(Deserialize)]
-struct Id {
-  id: String,
-}
-
-#[derive(Deserialize)]
-struct IdBranch {
-  id: String,
-  branch: Option<String>,
-}
-
-pub fn router() -> Router {
-  Router::new()
-		.route(
-			"/build/:id",
-			post(
-				|Path(Id { id }), headers: HeaderMap, body: String| async move {
-					tokio::spawn(async move {
-            let span = info_span!("build_webhook", id);
-            async {
-              let res = build::handle_build_webhook(id.clone(), headers, body).await;
-              if let Err(e) = res {
-                warn!("failed to run build webook for build {id} | {e:#}");
-              }
-            }
-              .instrument(span)
-              .await
-					});
-				},
-			),
-		)
-		.route(
-			"/repo/:id/clone", 
-			post(
-				|Path(Id { id }), headers: HeaderMap, body: String| async move {
-					tokio::spawn(async move {
-						let span = info_span!("repo_clone_webhook", id);
-            async {
-              let res = repo::handle_repo_clone_webhook(id.clone(), headers, body).await;
-              if let Err(e) = res {
-                warn!("failed to run repo clone webook for repo {id} | {e:#}");
-              }
-            }
-              .instrument(span)
-              .await
-					});
-				},
-			)
-		)
-		.route(
-			"/repo/:id/pull", 
-			post(
-				|Path(Id { id }), headers: HeaderMap, body: String| async move {
-					tokio::spawn(async move {
-            let span = info_span!("repo_pull_webhook", id);
-            async {
-              let res = repo::handle_repo_pull_webhook(id.clone(), headers, body).await;
-              if let Err(e) = res {
-                warn!("failed to run repo pull webook for repo {id} | {e:#}");
-              }
-            }
-              .instrument(span)
-              .await
-					});
-				},
-			)
-		)
-		.route(
-			"/repo/:id/build", 
-			post(
-				|Path(Id { id }), headers: HeaderMap, body: String| async move {
-					tokio::spawn(async move {
-            let span = info_span!("repo_build_webhook", id);
-            async {
-              let res = repo::handle_repo_build_webhook(id.clone(), headers, body).await;
-              if let Err(e) = res {
-                warn!("failed to run repo build webook for repo {id} | {e:#}");
-              }
-            }
-              .instrument(span)
-              .await
-					});
-				},
-			)
-		)
-    .route(
-			"/stack/:id/refresh", 
-			post(
-				|Path(Id { id }), headers: HeaderMap, body: String| async move {
-					tokio::spawn(async move {
-						let span = info_span!("stack_clone_webhook", id);
-            async {
-              let res = stack::handle_stack_refresh_webhook(id.clone(), headers, body).await;
-              if let Err(e) = res {
-                warn!("failed to run stack clone webook for stack {id} | {e:#}");
-              }
-            }
-              .instrument(span)
-              .await
-					});
-				},
-			)
-		)
-		.route(
-			"/stack/:id/deploy", 
-			post(
-				|Path(Id { id }), headers: HeaderMap, body: String| async move {
-					tokio::spawn(async move {
-            let span = info_span!("stack_pull_webhook", id);
-            async {
-              let res = stack::handle_stack_deploy_webhook(id.clone(), headers, body).await;
-              if let Err(e) = res {
-                warn!("failed to run stack pull webook for stack {id} | {e:#}");
-              }
-            }
-              .instrument(span)
-              .await
-					});
-				},
-			)
-		)
-    .route(
-			"/procedure/:id/:branch", 
-			post(
-				|Path(IdBranch { id, branch }), headers: HeaderMap, body: String| async move {
-					tokio::spawn(async move {
-            let span = info_span!("procedure_webhook", id, branch);
-            async {
-              let res = procedure::handle_procedure_webhook(
-                id.clone(),
-                branch.unwrap_or_else(|| String::from("main")),
-                headers,
-                body
-              ).await;
-              if let Err(e) = res {
-                warn!("failed to run procedure webook for procedure {id} | {e:#}");
-              }
-            }
-              .instrument(span)
-              .await
-					});
-				},
-			)
-		)
-    .route(
-			"/sync/:id/refresh", 
-			post(
-				|Path(Id { id }), headers: HeaderMap, body: String| async move {
-					tokio::spawn(async move {
-            let span = info_span!("sync_refresh_webhook", id);
-            async {
-              let res = sync::handle_sync_refresh_webhook(
-                id.clone(),
-                headers,
-                body
-              ).await;
-              if let Err(e) = res {
-                warn!("failed to run sync webook for sync {id} | {e:#}");
-              }
-            }
-              .instrument(span)
-              .await
-					});
-				},
-			)
-		)
-    .route(
-			"/sync/:id/sync", 
-			post(
-				|Path(Id { id }), headers: HeaderMap, body: String| async move {
-					tokio::spawn(async move {
-            let span = info_span!("sync_execute_webhook", id);
-            async {
-              let res = sync::handle_sync_execute_webhook(
-                id.clone(),
-                headers,
-                body
-              ).await;
-              if let Err(e) = res {
-                warn!("failed to run sync webook for sync {id} | {e:#}");
-              }
-            }
-              .instrument(span)
-              .await
-					});
-				},
-			)
-		)
-}
-
-#[instrument(skip_all)]
-async fn verify_gh_signature(
-  headers: HeaderMap,
-  body: &str,
-  custom_secret: &str,
-) -> anyhow::Result<()> {
-  // wait random amount of time
-  tokio::time::sleep(random_duration(0, 500)).await;
-
-  let signature = headers.get("x-hub-signature-256");
-  if signature.is_none() {
-    return Err(anyhow!("no signature in headers"));
-  }
-  let signature = signature.unwrap().to_str();
-  if signature.is_err() {
-    return Err(anyhow!("failed to unwrap signature"));
-  }
-  let signature = signature.unwrap().replace("sha256=", "");
-  let secret_bytes = if custom_secret.is_empty() {
-    core_config().webhook_secret.as_bytes()
-  } else {
-    custom_secret.as_bytes()
-  };
-  let mut mac = HmacSha256::new_from_slice(secret_bytes)
-    .expect("github webhook | failed to create hmac sha256");
-  mac.update(body.as_bytes());
-  let expected = mac.finalize().into_bytes().encode_hex::<String>();
-  if signature == expected {
-    Ok(())
-  } else {
-    Err(anyhow!("signature does not equal expected"))
-  }
-}
-
-#[derive(Deserialize)]
-struct GithubWebhookBody {
-  #[serde(rename = "ref")]
-  branch: String,
-}
-
-fn extract_branch(body: &str) -> anyhow::Result<String> {
-  let branch = serde_json::from_str::<GithubWebhookBody>(body)
-    .context("failed to parse github request body")?
-    .branch
-    .replace("refs/heads/", "");
-  Ok(branch)
-}
-
-type ListenerLockCache = Cache<String, Arc<Mutex<()>>>;

bin/core/src/listener/github/procedure.rs

@@ -1,64 +0,0 @@
-use std::sync::OnceLock;
-
-use anyhow::anyhow;
-use axum::http::HeaderMap;
-use komodo_client::{
-  api::execute::RunProcedure,
-  entities::{procedure::Procedure, user::git_webhook_user},
-};
-use resolver_api::Resolve;
-
-use crate::{
-  api::execute::ExecuteRequest,
-  helpers::update::init_execution_update, resource, state::State,
-};
-
-use super::{extract_branch, verify_gh_signature, ListenerLockCache};
-
-fn procedure_locks() -> &'static ListenerLockCache {
-  static BUILD_LOCKS: OnceLock<ListenerLockCache> = OnceLock::new();
-  BUILD_LOCKS.get_or_init(Default::default)
-}
-
-pub async fn handle_procedure_webhook(
-  procedure_id: String,
-  target_branch: String,
-  headers: HeaderMap,
-  body: String,
-) -> anyhow::Result<()> {
-  // Acquire and hold lock to make a task queue for
-  // subsequent listener calls on same resource.
-  // It would fail if we let it go through from action state busy.
-  let lock =
-    procedure_locks().get_or_insert_default(&procedure_id).await;
-  let _lock = lock.lock().await;
-
-  let procedure = resource::get::<Procedure>(&procedure_id).await?;
-
-  verify_gh_signature(
-    headers,
-    &body,
-    &procedure.config.webhook_secret,
-  )
-  .await?;
-
-  if !procedure.config.webhook_enabled {
-    return Err(anyhow!("procedure does not have webhook enabled"));
-  }
-
-  let request_branch = extract_branch(&body)?;
-  if request_branch != target_branch {
-    return Err(anyhow!("request branch does not match expected"));
-  }
-
-  let user = git_webhook_user().to_owned();
-  let req = ExecuteRequest::RunProcedure(RunProcedure {
-    procedure: procedure_id,
-  });
-  let update = init_execution_update(&req, &user).await?;
-  let ExecuteRequest::RunProcedure(req) = req else {
-    unreachable!()
-  };
-  State.resolve(req, (user, update)).await?;
-  Ok(())
-}

bin/core/src/listener/github/repo.rs

@@ -1,135 +0,0 @@
-use std::sync::OnceLock;
-
-use anyhow::anyhow;
-use axum::http::HeaderMap;
-use komodo_client::{
-  api::execute::{BuildRepo, CloneRepo, PullRepo},
-  entities::{repo::Repo, user::git_webhook_user},
-};
-use resolver_api::Resolve;
-
-use crate::{
-  helpers::update::init_execution_update, resource, state::State,
-};
-
-use super::{extract_branch, verify_gh_signature, ListenerLockCache};
-
-fn repo_locks() -> &'static ListenerLockCache {
-  static REPO_LOCKS: OnceLock<ListenerLockCache> = OnceLock::new();
-  REPO_LOCKS.get_or_init(Default::default)
-}
-
-pub async fn handle_repo_clone_webhook(
-  repo_id: String,
-  headers: HeaderMap,
-  body: String,
-) -> anyhow::Result<()> {
-  // Acquire and hold lock to make a task queue for
-  // subsequent listener calls on same resource.
-  // It would fail if we let it go through from action state busy.
-  let lock = repo_locks().get_or_insert_default(&repo_id).await;
-  let _lock = lock.lock().await;
-
-  let repo = resource::get::<Repo>(&repo_id).await?;
-
-  verify_gh_signature(headers, &body, &repo.config.webhook_secret)
-    .await?;
-
-  if !repo.config.webhook_enabled {
-    return Err(anyhow!("repo does not have webhook enabled"));
-  }
-
-  let request_branch = extract_branch(&body)?;
-  if request_branch != repo.config.branch {
-    return Err(anyhow!("request branch does not match expected"));
-  }
-
-  let user = git_webhook_user().to_owned();
-  let req =
-    crate::api::execute::ExecuteRequest::CloneRepo(CloneRepo {
-      repo: repo_id,
-    });
-  let update = init_execution_update(&req, &user).await?;
-  let crate::api::execute::ExecuteRequest::CloneRepo(req) = req
-  else {
-    unreachable!()
-  };
-  State.resolve(req, (user, update)).await?;
-  Ok(())
-}
-
-pub async fn handle_repo_pull_webhook(
-  repo_id: String,
-  headers: HeaderMap,
-  body: String,
-) -> anyhow::Result<()> {
-  // Acquire and hold lock to make a task queue for
-  // subsequent listener calls on same resource.
-  // It would fail if we let it go through from action state busy.
-  let lock = repo_locks().get_or_insert_default(&repo_id).await;
-  let _lock = lock.lock().await;
-
-  let repo = resource::get::<Repo>(&repo_id).await?;
-
-  verify_gh_signature(headers, &body, &repo.config.webhook_secret)
-    .await?;
-
-  if !repo.config.webhook_enabled {
-    return Err(anyhow!("repo does not have webhook enabled"));
-  }
-
-  let request_branch = extract_branch(&body)?;
-  if request_branch != repo.config.branch {
-    return Err(anyhow!("request branch does not match expected"));
-  }
-
-  let user = git_webhook_user().to_owned();
-  let req = crate::api::execute::ExecuteRequest::PullRepo(PullRepo {
-    repo: repo_id,
-  });
-  let update = init_execution_update(&req, &user).await?;
-  let crate::api::execute::ExecuteRequest::PullRepo(req) = req else {
-    unreachable!()
-  };
-  State.resolve(req, (user, update)).await?;
-  Ok(())
-}
-
-pub async fn handle_repo_build_webhook(
-  repo_id: String,
-  headers: HeaderMap,
-  body: String,
-) -> anyhow::Result<()> {
-  // Acquire and hold lock to make a task queue for
-  // subsequent listener calls on same resource.
-  // It would fail if we let it go through from action state busy.
-  let lock = repo_locks().get_or_insert_default(&repo_id).await;
-  let _lock = lock.lock().await;
-
-  let repo = resource::get::<Repo>(&repo_id).await?;
-
-  verify_gh_signature(headers, &body, &repo.config.webhook_secret)
-    .await?;
-
-  if !repo.config.webhook_enabled {
-    return Err(anyhow!("repo does not have webhook enabled"));
-  }
-
-  let request_branch = extract_branch(&body)?;
-  if request_branch != repo.config.branch {
-    return Err(anyhow!("request branch does not match expected"));
-  }
-
-  let user = git_webhook_user().to_owned();
-  let req =
-    crate::api::execute::ExecuteRequest::BuildRepo(BuildRepo {
-      repo: repo_id,
-    });
-  let update = init_execution_update(&req, &user).await?;
-  let crate::api::execute::ExecuteRequest::BuildRepo(req) = req
-  else {
-    unreachable!()
-  };
-  State.resolve(req, (user, update)).await?;
-  Ok(())
-}

bin/core/src/listener/github/stack.rs

@@ -1,91 +0,0 @@
-use std::sync::OnceLock;
-
-use anyhow::anyhow;
-use axum::http::HeaderMap;
-use komodo_client::{
-  api::{execute::DeployStack, write::RefreshStackCache},
-  entities::{stack::Stack, user::git_webhook_user},
-};
-use resolver_api::Resolve;
-
-use crate::{
-  api::execute::ExecuteRequest,
-  helpers::update::init_execution_update, resource, state::State,
-};
-
-use super::{extract_branch, verify_gh_signature, ListenerLockCache};
-
-fn stack_locks() -> &'static ListenerLockCache {
-  static STACK_LOCKS: OnceLock<ListenerLockCache> = OnceLock::new();
-  STACK_LOCKS.get_or_init(Default::default)
-}
-
-pub async fn handle_stack_refresh_webhook(
-  stack_id: String,
-  headers: HeaderMap,
-  body: String,
-) -> anyhow::Result<()> {
-  // Acquire and hold lock to make a task queue for
-  // subsequent listener calls on same resource.
-  // It would fail if we let it go through, from "action state busy".
-  let lock = stack_locks().get_or_insert_default(&stack_id).await;
-  let _lock = lock.lock().await;
-
-  let stack = resource::get::<Stack>(&stack_id).await?;
-
-  verify_gh_signature(headers, &body, &stack.config.webhook_secret)
-    .await?;
-
-  if !stack.config.webhook_enabled {
-    return Err(anyhow!("stack does not have webhook enabled"));
-  }
-
-  let request_branch = extract_branch(&body)?;
-  if request_branch != stack.config.branch {
-    return Err(anyhow!("request branch does not match expected"));
-  }
-
-  let user = git_webhook_user().to_owned();
-  State
-    .resolve(RefreshStackCache { stack: stack.id }, user)
-    .await?;
-  Ok(())
-}
-
-pub async fn handle_stack_deploy_webhook(
-  stack_id: String,
-  headers: HeaderMap,
-  body: String,
-) -> anyhow::Result<()> {
-  // Acquire and hold lock to make a task queue for
-  // subsequent listener calls on same resource.
-  // It would fail if we let it go through from action state busy.
-  let lock = stack_locks().get_or_insert_default(&stack_id).await;
-  let _lock = lock.lock().await;
-
-  let stack = resource::get::<Stack>(&stack_id).await?;
-
-  verify_gh_signature(headers, &body, &stack.config.webhook_secret)
-    .await?;
-
-  if !stack.config.webhook_enabled {
-    return Err(anyhow!("stack does not have webhook enabled"));
-  }
-
-  let request_branch = extract_branch(&body)?;
-  if request_branch != stack.config.branch {
-    return Err(anyhow!("request branch does not match expected"));
-  }
-
-  let user = git_webhook_user().to_owned();
-  let req = ExecuteRequest::DeployStack(DeployStack {
-    stack: stack_id,
-    stop_time: None,
-  });
-  let update = init_execution_update(&req, &user).await?;
-  let ExecuteRequest::DeployStack(req) = req else {
-    unreachable!()
-  };
-  State.resolve(req, (user, update)).await?;
-  Ok(())
-}

bin/core/src/listener/github/sync.rs

@@ -1,88 +0,0 @@
-use std::sync::OnceLock;
-
-use anyhow::anyhow;
-use axum::http::HeaderMap;
-use komodo_client::{
-  api::{execute::RunSync, write::RefreshResourceSyncPending},
-  entities::{sync::ResourceSync, user::git_webhook_user},
-};
-use resolver_api::Resolve;
-
-use crate::{
-  api::execute::ExecuteRequest,
-  helpers::update::init_execution_update, resource, state::State,
-};
-
-use super::{extract_branch, verify_gh_signature, ListenerLockCache};
-
-fn sync_locks() -> &'static ListenerLockCache {
-  static SYNC_LOCKS: OnceLock<ListenerLockCache> = OnceLock::new();
-  SYNC_LOCKS.get_or_init(Default::default)
-}
-
-pub async fn handle_sync_refresh_webhook(
-  sync_id: String,
-  headers: HeaderMap,
-  body: String,
-) -> anyhow::Result<()> {
-  // Acquire and hold lock to make a task queue for
-  // subsequent listener calls on same resource.
-  // It would fail if we let it go through from action state busy.
-  let lock = sync_locks().get_or_insert_default(&sync_id).await;
-  let _lock = lock.lock().await;
-
-  let sync = resource::get::<ResourceSync>(&sync_id).await?;
-
-  verify_gh_signature(headers, &body, &sync.config.webhook_secret)
-    .await?;
-
-  if !sync.config.webhook_enabled {
-    return Err(anyhow!("sync does not have webhook enabled"));
-  }
-
-  let request_branch = extract_branch(&body)?;
-  if request_branch != sync.config.branch {
-    return Err(anyhow!("request branch does not match expected"));
-  }
-
-  let user = git_webhook_user().to_owned();
-  State
-    .resolve(RefreshResourceSyncPending { sync: sync_id }, user)
-    .await?;
-  Ok(())
-}
-
-pub async fn handle_sync_execute_webhook(
-  sync_id: String,
-  headers: HeaderMap,
-  body: String,
-) -> anyhow::Result<()> {
-  // Acquire and hold lock to make a task queue for
-  // subsequent listener calls on same resource.
-  // It would fail if we let it go through from action state busy.
-  let lock = sync_locks().get_or_insert_default(&sync_id).await;
-  let _lock = lock.lock().await;
-
-  let sync = resource::get::<ResourceSync>(&sync_id).await?;
-
-  verify_gh_signature(headers, &body, &sync.config.webhook_secret)
-    .await?;
-
-  if !sync.config.webhook_enabled {
-    return Err(anyhow!("sync does not have webhook enabled"));
-  }
-
-  let request_branch = extract_branch(&body)?;
-  if request_branch != sync.config.branch {
-    return Err(anyhow!("request branch does not match expected"));
-  }
-
-  let user = git_webhook_user().to_owned();
-  let req = ExecuteRequest::RunSync(RunSync { sync: sync_id });
-  let update = init_execution_update(&req, &user).await?;
-  let ExecuteRequest::RunSync(req) = req else {
-    unreachable!()
-  };
-  State.resolve(req, (user, update)).await?;
-  Ok(())
-}

bin/core/src/listener/integrations/github.rs

@@ -0,0 +1,71 @@
+use anyhow::{anyhow, Context};
+use axum::http::HeaderMap;
+use hex::ToHex;
+use hmac::{Hmac, Mac};
+use serde::Deserialize;
+use sha2::Sha256;
+
+use crate::{
+  config::core_config,
+  listener::{VerifyBranch, VerifySecret},
+};
+
+type HmacSha256 = Hmac<Sha256>;
+
+/// Listener implementation for Github type API, including Gitea
+pub struct Github;
+
+impl VerifySecret for Github {
+  #[instrument("VerifyGithubSecret", skip_all)]
+  fn verify_secret(
+    headers: HeaderMap,
+    body: &str,
+    custom_secret: &str,
+  ) -> anyhow::Result<()> {
+    let signature = headers
+      .get("x-hub-signature-256")
+      .context("No github signature in headers")?;
+    let signature = signature
+      .to_str()
+      .context("Failed to get signature as string")?;
+    let signature =
+      signature.strip_prefix("sha256=").unwrap_or(signature);
+    let secret_bytes = if custom_secret.is_empty() {
+      core_config().webhook_secret.as_bytes()
+    } else {
+      custom_secret.as_bytes()
+    };
+    let mut mac = HmacSha256::new_from_slice(secret_bytes)
+      .context("Failed to create hmac sha256 from secret")?;
+    mac.update(body.as_bytes());
+    let expected = mac.finalize().into_bytes().encode_hex::<String>();
+    if signature == expected {
+      Ok(())
+    } else {
+      Err(anyhow!("Signature does not equal expected"))
+    }
+  }
+}
+
+#[derive(Deserialize)]
+struct GithubWebhookBody {
+  #[serde(rename = "ref")]
+  branch: String,
+}
+
+impl VerifyBranch for Github {
+  fn verify_branch(
+    body: &str,
+    expected_branch: &str,
+  ) -> anyhow::Result<()> {
+    let branch = serde_json::from_str::<GithubWebhookBody>(body)
+      .context("Failed to parse github request body")?
+      .branch
+      .replace("refs/heads/", "");
+    if branch == expected_branch {
+      Ok(())
+    } else {
+      Err(anyhow!("request branch does not match expected"))
+    }
+  }
+}

bin/core/src/listener/integrations/gitlab.rs

@@ -0,0 +1,58 @@
+use anyhow::{anyhow, Context};
+use serde::Deserialize;
+
+use crate::{
+  config::core_config,
+  listener::{VerifyBranch, VerifySecret},
+};
+
+/// Listener implementation for Gitlab type API
+pub struct Gitlab;
+
+impl VerifySecret for Gitlab {
+  #[instrument("VerifyGitlabSecret", skip_all)]
+  fn verify_secret(
+    headers: axum::http::HeaderMap,
+    _body: &str,
+    custom_secret: &str,
+  ) -> anyhow::Result<()> {
+    let token = headers
+      .get("x-gitlab-token")
+      .context("No gitlab token in headers")?;
+    let token =
+      token.to_str().context("Failed to get token as string")?;
+    let secret = if custom_secret.is_empty() {
+      core_config().webhook_secret.as_str()
+    } else {
+      custom_secret
+    };
+    if token == secret {
+      Ok(())
+    } else {
+      Err(anyhow!("Webhook secret does not match expected."))
+    }
+  }
+}
+
+#[derive(Deserialize)]
+struct GitlabWebhookBody {
+  #[serde(rename = "ref")]
+  branch: String,
+}
+
+impl VerifyBranch for Gitlab {
+  fn verify_branch(
+    body: &str,
+    expected_branch: &str,
+  ) -> anyhow::Result<()> {
+    let branch = serde_json::from_str::<GitlabWebhookBody>(body)
+      .context("Failed to parse gitlab request body")?
+      .branch
+      .replace("refs/heads/", "");
+    if branch == expected_branch {
+      Ok(())
+    } else {
+      Err(anyhow!("request branch does not match expected"))
+    }
+  }
+}

bin/core/src/listener/integrations/mod.rs

@@ -0,0 +1,2 @@
+pub mod github;
+pub mod gitlab;

bin/core/src/listener/mod.rs

@@ -1,7 +1,52 @@
-use axum::Router;
+use std::sync::Arc;
 
-mod github;
+use axum::{http::HeaderMap, Router};
+use komodo_client::entities::resource::Resource;
+use tokio::sync::Mutex;
+
+use crate::{helpers::cache::Cache, resource::KomodoResource};
+
+mod integrations;
+mod resources;
+mod router;
+
+use integrations::*;
 
 pub fn router() -> Router {
-  Router::new().nest("/github", github::router())
+  Router::new()
+    .nest("/github", router::router::<github::Github>())
+    .nest("/gitlab", router::router::<gitlab::Gitlab>())
 }
+
+type ListenerLockCache = Cache<String, Arc<Mutex<()>>>;
+
+/// Implemented for all resources which can recieve webhook.
+trait CustomSecret: KomodoResource {
+  fn custom_secret(
+    resource: &Resource<Self::Config, Self::Info>,
+  ) -> &str;
+}
+
+/// Implemented on the integration struct, eg [integrations::github::Github]
+trait VerifySecret {
+  fn verify_secret(
+    headers: HeaderMap,
+    body: &str,
+    custom_secret: &str,
+  ) -> anyhow::Result<()>;
+}
+
+/// Implemented on the integration struct, eg [integrations::github::Github]
+trait VerifyBranch {
+  /// Returns Err if the branch extracted from request
+  /// body does not match the expected branch.
+  fn verify_branch(
+    body: &str,
+    expected_branch: &str,
+  ) -> anyhow::Result<()>;
+}
+
+/// For Procedures and Actions, incoming webhook
+/// can be triggered by any branch by using `__ANY__`
+/// as the branch in the webhook URL.
+const ANY_BRANCH: &str = "__ANY__";

bin/core/src/listener/resources.rs

@@ -0,0 +1,472 @@
+use std::sync::OnceLock;
+
+use anyhow::anyhow;
+use komodo_client::{
+  api::{
+    execute::*,
+    write::{RefreshResourceSyncPending, RefreshStackCache},
+  },
+  entities::{
+    action::Action, build::Build, procedure::Procedure, repo::Repo,
+    stack::Stack, sync::ResourceSync, user::git_webhook_user,
+  },
+};
+use resolver_api::Resolve;
+use serde::Deserialize;
+
+use crate::{
+  api::execute::ExecuteRequest,
+  helpers::update::init_execution_update, state::State,
+};
+
+use super::{ListenerLockCache, ANY_BRANCH};
+
+// =======
+//  BUILD
+// =======
+
+impl super::CustomSecret for Build {
+  fn custom_secret(resource: &Self) -> &str {
+    &resource.config.webhook_secret
+  }
+}
+
+fn build_locks() -> &'static ListenerLockCache {
+  static BUILD_LOCKS: OnceLock<ListenerLockCache> = OnceLock::new();
+  BUILD_LOCKS.get_or_init(Default::default)
+}
+
+pub async fn handle_build_webhook<B: super::VerifyBranch>(
+  build: Build,
+  body: String,
+) -> anyhow::Result<()> {
+  // Acquire and hold lock to make a task queue for
+  // subsequent listener calls on same resource.
+  // It would fail if we let it go through from action state busy.
+  let lock = build_locks().get_or_insert_default(&build.id).await;
+  let _lock = lock.lock().await;
+
+  if !build.config.webhook_enabled {
+    return Err(anyhow!("build does not have webhook enabled"));
+  }
+
+  B::verify_branch(&body, &build.config.branch)?;
+
+  let user = git_webhook_user().to_owned();
+  let req = ExecuteRequest::RunBuild(RunBuild { build: build.id });
+  let update = init_execution_update(&req, &user).await?;
+  let ExecuteRequest::RunBuild(req) = req else {
+    unreachable!()
+  };
+  State.resolve(req, (user, update)).await?;
+  Ok(())
+}
+
+// ======
+//  REPO
+// ======
+
+impl super::CustomSecret for Repo {
+  fn custom_secret(resource: &Self) -> &str {
+    &resource.config.webhook_secret
+  }
+}
+
+fn repo_locks() -> &'static ListenerLockCache {
+  static REPO_LOCKS: OnceLock<ListenerLockCache> = OnceLock::new();
+  REPO_LOCKS.get_or_init(Default::default)
+}
+
+pub trait RepoExecution {
+  async fn resolve(repo: Repo) -> anyhow::Result<()>;
+}
+
+impl RepoExecution for CloneRepo {
+  async fn resolve(repo: Repo) -> anyhow::Result<()> {
+    let user = git_webhook_user().to_owned();
+    let req =
+      crate::api::execute::ExecuteRequest::CloneRepo(CloneRepo {
+        repo: repo.id,
+      });
+    let update = init_execution_update(&req, &user).await?;
+    let crate::api::execute::ExecuteRequest::CloneRepo(req) = req
+    else {
+      unreachable!()
+    };
+    State.resolve(req, (user, update)).await?;
+    Ok(())
+  }
+}
+
+impl RepoExecution for PullRepo {
+  async fn resolve(repo: Repo) -> anyhow::Result<()> {
+    let user = git_webhook_user().to_owned();
+    let req =
+      crate::api::execute::ExecuteRequest::PullRepo(PullRepo {
+        repo: repo.id,
+      });
+    let update = init_execution_update(&req, &user).await?;
+    let crate::api::execute::ExecuteRequest::PullRepo(req) = req
+    else {
+      unreachable!()
+    };
+    State.resolve(req, (user, update)).await?;
+    Ok(())
+  }
+}
+
+impl RepoExecution for BuildRepo {
+  async fn resolve(repo: Repo) -> anyhow::Result<()> {
+    let user = git_webhook_user().to_owned();
+    let req =
+      crate::api::execute::ExecuteRequest::BuildRepo(BuildRepo {
+        repo: repo.id,
+      });
+    let update = init_execution_update(&req, &user).await?;
+    let crate::api::execute::ExecuteRequest::BuildRepo(req) = req
+    else {
+      unreachable!()
+    };
+    State.resolve(req, (user, update)).await?;
+    Ok(())
+  }
+}
+
+#[derive(Deserialize)]
+#[serde(rename_all = "lowercase")]
+pub enum RepoWebhookOption {
+  Clone,
+  Pull,
+  Build,
+}
+
+pub async fn handle_repo_webhook<B: super::VerifyBranch>(
+  option: RepoWebhookOption,
+  repo: Repo,
+  body: String,
+) -> anyhow::Result<()> {
+  match option {
+    RepoWebhookOption::Clone => {
+      handle_repo_webhook_inner::<B, CloneRepo>(repo, body).await
+    }
+    RepoWebhookOption::Pull => {
+      handle_repo_webhook_inner::<B, PullRepo>(repo, body).await
+    }
+    RepoWebhookOption::Build => {
+      handle_repo_webhook_inner::<B, BuildRepo>(repo, body).await
+    }
+  }
+}
+
+async fn handle_repo_webhook_inner<
+  B: super::VerifyBranch,
+  E: RepoExecution,
+>(
+  repo: Repo,
+  body: String,
+) -> anyhow::Result<()> {
+  // Acquire and hold lock to make a task queue for
+  // subsequent listener calls on same resource.
+  // It would fail if we let it go through from action state busy.
+  let lock = repo_locks().get_or_insert_default(&repo.id).await;
+  let _lock = lock.lock().await;
+
+  if !repo.config.webhook_enabled {
+    return Err(anyhow!("repo does not have webhook enabled"));
+  }
+
+  B::verify_branch(&body, &repo.config.branch)?;
+
+  E::resolve(repo).await
+}
+
+// =======
+//  STACK
+// =======
+
+impl super::CustomSecret for Stack {
+  fn custom_secret(resource: &Self) -> &str {
+    &resource.config.webhook_secret
+  }
+}
+
+fn stack_locks() -> &'static ListenerLockCache {
+  static STACK_LOCKS: OnceLock<ListenerLockCache> = OnceLock::new();
+  STACK_LOCKS.get_or_init(Default::default)
+}
+
+pub trait StackExecution {
+  async fn resolve(stack: Stack) -> anyhow::Result<()>;
+}
+
+impl StackExecution for RefreshStackCache {
+  async fn resolve(stack: Stack) -> anyhow::Result<()> {
+    let user = git_webhook_user().to_owned();
+    State
+      .resolve(RefreshStackCache { stack: stack.id }, user)
+      .await?;
+    Ok(())
+  }
+}
+
+impl StackExecution for DeployStack {
+  async fn resolve(stack: Stack) -> anyhow::Result<()> {
+    let user = git_webhook_user().to_owned();
+    if stack.config.webhook_force_deploy {
+      let req = ExecuteRequest::DeployStack(DeployStack {
+        stack: stack.id,
+        service: None,
+        stop_time: None,
+      });
+      let update = init_execution_update(&req, &user).await?;
+      let ExecuteRequest::DeployStack(req) = req else {
+        unreachable!()
+      };
+      State.resolve(req, (user, update)).await?;
+    } else {
+      let req =
+        ExecuteRequest::DeployStackIfChanged(DeployStackIfChanged {
+          stack: stack.id,
+          stop_time: None,
+        });
+      let update = init_execution_update(&req, &user).await?;
+      let ExecuteRequest::DeployStackIfChanged(req) = req else {
+        unreachable!()
+      };
+      State.resolve(req, (user, update)).await?;
+    }
+
+    Ok(())
+  }
+}
+
+#[derive(Deserialize)]
+#[serde(rename_all = "lowercase")]
+pub enum StackWebhookOption {
+  Refresh,
+  Deploy,
+}
+
+pub async fn handle_stack_webhook<B: super::VerifyBranch>(
+  option: StackWebhookOption,
+  stack: Stack,
+  body: String,
+) -> anyhow::Result<()> {
+  match option {
+    StackWebhookOption::Refresh => {
+      handle_stack_webhook_inner::<B, RefreshStackCache>(stack, body)
+        .await
+    }
+    StackWebhookOption::Deploy => {
+      handle_stack_webhook_inner::<B, DeployStack>(stack, body).await
+    }
+  }
+}
+
+pub async fn handle_stack_webhook_inner<
+  B: super::VerifyBranch,
+  E: StackExecution,
+>(
+  stack: Stack,
+  body: String,
+) -> anyhow::Result<()> {
+  // Acquire and hold lock to make a task queue for
+  // subsequent listener calls on same resource.
+  // It would fail if we let it go through, from "action state busy".
+  let lock = stack_locks().get_or_insert_default(&stack.id).await;
+  let _lock = lock.lock().await;
+
+  if !stack.config.webhook_enabled {
+    return Err(anyhow!("stack does not have webhook enabled"));
+  }
+
+  B::verify_branch(&body, &stack.config.branch)?;
+
+  E::resolve(stack).await
+}
+
+// ======
+//  SYNC
+// ======
+
+impl super::CustomSecret for ResourceSync {
+  fn custom_secret(resource: &Self) -> &str {
+    &resource.config.webhook_secret
+  }
+}
+
+fn sync_locks() -> &'static ListenerLockCache {
+  static SYNC_LOCKS: OnceLock<ListenerLockCache> = OnceLock::new();
+  SYNC_LOCKS.get_or_init(Default::default)
+}
+
+pub trait SyncExecution {
+  async fn resolve(sync: ResourceSync) -> anyhow::Result<()>;
+}
+
+impl SyncExecution for RefreshResourceSyncPending {
+  async fn resolve(sync: ResourceSync) -> anyhow::Result<()> {
+    let user = git_webhook_user().to_owned();
+    State
+      .resolve(RefreshResourceSyncPending { sync: sync.id }, user)
+      .await?;
+    Ok(())
+  }
+}
+
+impl SyncExecution for RunSync {
+  async fn resolve(sync: ResourceSync) -> anyhow::Result<()> {
+    let user = git_webhook_user().to_owned();
+    let req = ExecuteRequest::RunSync(RunSync {
+      sync: sync.id,
+      resource_type: None,
+      resources: None,
+    });
+    let update = init_execution_update(&req, &user).await?;
+    let ExecuteRequest::RunSync(req) = req else {
+      unreachable!()
+    };
+    State.resolve(req, (user, update)).await?;
+    Ok(())
+  }
+}
+
+#[derive(Deserialize)]
+#[serde(rename_all = "lowercase")]
+pub enum SyncWebhookOption {
+  Refresh,
+  Sync,
+}
+
+pub async fn handle_sync_webhook<B: super::VerifyBranch>(
+  option: SyncWebhookOption,
+  sync: ResourceSync,
+  body: String,
+) -> anyhow::Result<()> {
+  match option {
+    SyncWebhookOption::Refresh => {
+      handle_sync_webhook_inner::<B, RefreshResourceSyncPending>(
+        sync, body,
+      )
+      .await
+    }
+    SyncWebhookOption::Sync => {
+      handle_sync_webhook_inner::<B, RunSync>(sync, body).await
+    }
+  }
+}
+
+async fn handle_sync_webhook_inner<
+  B: super::VerifyBranch,
+  E: SyncExecution,
+>(
+  sync: ResourceSync,
+  body: String,
+) -> anyhow::Result<()> {
+  // Acquire and hold lock to make a task queue for
+  // subsequent listener calls on same resource.
+  // It would fail if we let it go through from action state busy.
+  let lock = sync_locks().get_or_insert_default(&sync.id).await;
+  let _lock = lock.lock().await;
+
+  if !sync.config.webhook_enabled {
+    return Err(anyhow!("sync does not have webhook enabled"));
+  }
+
+  B::verify_branch(&body, &sync.config.branch)?;
+
+  E::resolve(sync).await
+}
+
+// ===========
+//  PROCEDURE
+// ===========
+
+impl super::CustomSecret for Procedure {
+  fn custom_secret(resource: &Self) -> &str {
+    &resource.config.webhook_secret
+  }
+}
+
+fn procedure_locks() -> &'static ListenerLockCache {
+  static PROCEDURE_LOCKS: OnceLock<ListenerLockCache> =
+    OnceLock::new();
+  PROCEDURE_LOCKS.get_or_init(Default::default)
+}
+
+pub async fn handle_procedure_webhook<B: super::VerifyBranch>(
+  procedure: Procedure,
+  target_branch: &str,
+  body: String,
+) -> anyhow::Result<()> {
+  // Acquire and hold lock to make a task queue for
+  // subsequent listener calls on same resource.
+  // It would fail if we let it go through from action state busy.
+  let lock =
+    procedure_locks().get_or_insert_default(&procedure.id).await;
+  let _lock = lock.lock().await;
+
+  if !procedure.config.webhook_enabled {
+    return Err(anyhow!("procedure does not have webhook enabled"));
+  }
+
+  if target_branch != ANY_BRANCH {
+    B::verify_branch(&body, target_branch)?;
+  }
+
+  let user = git_webhook_user().to_owned();
+  let req = ExecuteRequest::RunProcedure(RunProcedure {
+    procedure: procedure.id,
+  });
+  let update = init_execution_update(&req, &user).await?;
+  let ExecuteRequest::RunProcedure(req) = req else {
+    unreachable!()
+  };
+  State.resolve(req, (user, update)).await?;
+  Ok(())
+}
+
+// ========
+//  ACTION
+// ========
+
+impl super::CustomSecret for Action {
+  fn custom_secret(resource: &Self) -> &str {
+    &resource.config.webhook_secret
+  }
+}
+
+fn action_locks() -> &'static ListenerLockCache {
+  static ACTION_LOCKS: OnceLock<ListenerLockCache> = OnceLock::new();
+  ACTION_LOCKS.get_or_init(Default::default)
+}
+
+pub async fn handle_action_webhook<B: super::VerifyBranch>(
+  action: Action,
+  target_branch: &str,
+  body: String,
+) -> anyhow::Result<()> {
+  // Acquire and hold lock to make a task queue for
+  // subsequent listener calls on same resource.
+  // It would fail if we let it go through from action state busy.
+  let lock = action_locks().get_or_insert_default(&action.id).await;
+  let _lock = lock.lock().await;
+
+  if !action.config.webhook_enabled {
+    return Err(anyhow!("action does not have webhook enabled"));
+  }
+
+  if target_branch != ANY_BRANCH {
+    B::verify_branch(&body, target_branch)?;
+  }
+
+  let user = git_webhook_user().to_owned();
+  let req =
+    ExecuteRequest::RunAction(RunAction { action: action.id });
+  let update = init_execution_update(&req, &user).await?;
+  let ExecuteRequest::RunAction(req) = req else {
+    unreachable!()
+  };
+  State.resolve(req, (user, update)).await?;
+  Ok(())
+}

bin/core/src/listener/router.rs

@@ -0,0 +1,220 @@
+use axum::{extract::Path, http::HeaderMap, routing::post, Router};
+use komodo_client::entities::{
+  action::Action, build::Build, procedure::Procedure, repo::Repo,
+  resource::Resource, stack::Stack, sync::ResourceSync,
+};
+use reqwest::StatusCode;
+use serde::Deserialize;
+use serror::AddStatusCode;
+use tracing::Instrument;
+
+use crate::resource::KomodoResource;
+
+use super::{
+  resources::{
+    handle_action_webhook, handle_build_webhook,
+    handle_procedure_webhook, handle_repo_webhook,
+    handle_stack_webhook, handle_sync_webhook, RepoWebhookOption,
+    StackWebhookOption, SyncWebhookOption,
+  },
+  CustomSecret, VerifyBranch, VerifySecret,
+};
+
+#[derive(Deserialize)]
+struct Id {
+  id: String,
+}
+
+#[derive(Deserialize)]
+struct IdAndOption<T> {
+  id: String,
+  option: T,
+}
+
+#[derive(Deserialize)]
+struct IdAndBranch {
+  id: String,
+  #[serde(default = "default_branch")]
+  branch: String,
+}
+
+fn default_branch() -> String {
+  String::from("main")
+}
+
+pub fn router<P: VerifySecret + VerifyBranch>() -> Router {
+  Router::new()
+  .route(
+    "/build/:id",
+    post(
+      |Path(Id { id }), headers: HeaderMap, body: String| async move {
+        let build =
+          auth_webhook::<P, Build>(&id, headers, &body).await?;
+        tokio::spawn(async move {
+          let span = info_span!("BuildWebhook", id);
+          async {
+            let res = handle_build_webhook::<P>(
+              build, body,
+            )
+            .await;
+            if let Err(e) = res {
+              warn!(
+                "Failed at running webhook for build {id} | {e:#}"
+              );
+            }
+          }
+          .instrument(span)
+          .await
+        });
+        serror::Result::Ok(())
+      },
+    ),
+  )
+  .route(
+    "/repo/:id/:option",
+    post(
+      |Path(IdAndOption::<RepoWebhookOption> { id, option }), headers: HeaderMap, body: String| async move {
+        let repo =
+          auth_webhook::<P, Repo>(&id, headers, &body).await?;
+        tokio::spawn(async move {
+          let span = info_span!("RepoWebhook", id);
+          async {
+            let res = handle_repo_webhook::<P>(
+              option, repo, body,
+            )
+            .await;
+            if let Err(e) = res {
+              warn!(
+                "Failed at running webhook for repo {id} | {e:#}"
+              );
+            }
+          }
+          .instrument(span)
+          .await
+        });
+        serror::Result::Ok(())
+      },
+    ),
+  )
+  .route(
+    "/stack/:id/:option",
+    post(
+      |Path(IdAndOption::<StackWebhookOption> { id, option }), headers: HeaderMap, body: String| async move {
+        let stack =
+          auth_webhook::<P, Stack>(&id, headers, &body).await?;
+        tokio::spawn(async move {
+          let span = info_span!("StackWebhook", id);
+          async {
+            let res = handle_stack_webhook::<P>(
+              option, stack, body,
+            )
+            .await;
+            if let Err(e) = res {
+              warn!(
+                "Failed at running webhook for stack {id} | {e:#}"
+              );
+            }
+          }
+          .instrument(span)
+          .await
+        });
+        serror::Result::Ok(())
+      },
+    ),
+  )
+  .route(
+    "/sync/:id/:option",
+    post(
+      |Path(IdAndOption::<SyncWebhookOption> { id, option }), headers: HeaderMap, body: String| async move {
+        let sync =
+          auth_webhook::<P, ResourceSync>(&id, headers, &body).await?;
+        tokio::spawn(async move {
+          let span = info_span!("ResourceSyncWebhook", id);
+          async {
+            let res = handle_sync_webhook::<P>(
+              option, sync, body,
+            )
+            .await;
+            if let Err(e) = res {
+              warn!(
+                "Failed at running webhook for resource sync {id} | {e:#}"
+              );
+            }
+          }
+          .instrument(span)
+          .await
+        });
+        serror::Result::Ok(())
+      },
+    ),
+  )
+  .route(
+    "/procedure/:id/:branch",
+    post(
+      |Path(IdAndBranch { id, branch }), headers: HeaderMap, body: String| async move {
+        let procedure =
+          auth_webhook::<P, Procedure>(&id, headers, &body).await?;
+        tokio::spawn(async move {
+          let span = info_span!("ProcedureWebhook", id);
+          async {
+            let res = handle_procedure_webhook::<P>(
+              procedure, &branch, body,
+            )
+            .await;
+            if let Err(e) = res {
+              warn!(
+                "Failed at running webhook for procedure {id} | target branch: {branch} | {e:#}"
+              );
+            }
+          }
+          .instrument(span)
+          .await
+        });
+        serror::Result::Ok(())
+      },
+    ),
+  )
+  .route(
+    "/action/:id/:branch",
+    post(
+      |Path(IdAndBranch { id, branch }), headers: HeaderMap, body: String| async move {
+        let action =
+          auth_webhook::<P, Action>(&id, headers, &body).await?;
+        tokio::spawn(async move {
+          let span = info_span!("ActionWebhook", id);
+          async {
+            let res = handle_action_webhook::<P>(
+              action, &branch, body,
+            )
+            .await;
+            if let Err(e) = res {
+              warn!(
+                "Failed at running webhook for action {id} | target branch: {branch} | {e:#}"
+              );
+            }
+          }
+          .instrument(span)
+          .await
+        });
+        serror::Result::Ok(())
+      },
+    ),
+  )
+}
+
+async fn auth_webhook<P, R>(
+  id: &str,
+  headers: HeaderMap,
+  body: &str,
+) -> serror::Result<Resource<R::Config, R::Info>>
+where
+  P: VerifySecret,
+  R: KomodoResource + CustomSecret,
+{
+  let resource = crate::resource::get::<R>(id)
+    .await
+    .status_code(StatusCode::BAD_REQUEST)?;
+  P::verify_secret(headers, body, R::custom_secret(&resource))
+    .status_code(StatusCode::UNAUTHORIZED)?;
+  Ok(resource)
+}

bin/core/src/main.rs

@@ -5,7 +5,7 @@ use std::{net::SocketAddr, str::FromStr};
 
 use anyhow::Context;
 use axum::Router;
-use axum_server::tls_openssl::OpenSSLConfig;
+use axum_server::tls_rustls::RustlsConfig;
 use tower_http::{
   cors::{Any, CorsLayer},
   services::{ServeDir, ServeFile},
@@ -26,6 +26,7 @@ mod resource;
 mod stack;
 mod state;
 mod sync;
+mod ts_client;
 mod ws;
 
 async fn app() -> anyhow::Result<()> {
@@ -44,7 +45,7 @@ async fn app() -> anyhow::Result<()> {
   );
   tokio::join!(
     // Maybe initialize first server
-    helpers::ensure_first_server(),
+    helpers::ensure_first_server_and_builder(),
     // Cleanup open updates / invalid alerts
     helpers::startup_cleanup(),
   );
@@ -57,6 +58,7 @@ async fn app() -> anyhow::Result<()> {
   resource::spawn_build_state_refresh_loop();
   resource::spawn_repo_state_refresh_loop();
   resource::spawn_procedure_state_refresh_loop();
+  resource::spawn_action_state_refresh_loop();
   resource::spawn_resource_sync_state_refresh_loop();
   helpers::prune::spawn_prune_loop();
 
@@ -75,6 +77,7 @@ async fn app() -> anyhow::Result<()> {
     .nest("/execute", api::execute::router())
     .nest("/listener", listener::router())
     .nest("/ws", ws::router())
+    .nest("/client", ts_client::router())
     .nest_service("/", serve_dir)
     .fallback_service(frontend_index)
     .layer(cors()?)
@@ -86,13 +89,17 @@ async fn app() -> anyhow::Result<()> {
 
   if config.ssl_enabled {
     info!("🔒 Core SSL Enabled");
+    rustls::crypto::ring::default_provider()
+      .install_default()
+      .expect("failed to install default rustls CryptoProvider");
     info!("Komodo Core starting on https://{socket_addr}");
-    let ssl_config = OpenSSLConfig::from_pem_file(
+    let ssl_config = RustlsConfig::from_pem_file(
       &config.ssl_cert_file,
       &config.ssl_key_file,
     )
-    .context("Failed to parse ssl ")?;
-    axum_server::bind_openssl(socket_addr, ssl_config)
+    .await
+    .context("Invalid ssl cert / key")?;
+    axum_server::bind_rustls(socket_addr, ssl_config)
       .serve(app)
       .await?
   } else {

bin/core/src/monitor/alert/mod.rs

@@ -2,9 +2,7 @@ use std::collections::HashMap;
 
 use anyhow::Context;
 use komodo_client::entities::{
-  resource::ResourceQuery,
-  server::{Server, ServerListItem},
-  user::User,
+  resource::ResourceQuery, server::Server, user::User,
 };
 
 use crate::resource;
@@ -32,16 +30,16 @@ pub async fn check_alerts(ts: i64) {
 }
 
 #[instrument(level = "debug")]
-async fn get_all_servers_map() -> anyhow::Result<(
-  HashMap<String, ServerListItem>,
-  HashMap<String, String>,
-)> {
-  let servers = resource::list_for_user::<Server>(
+async fn get_all_servers_map(
+) -> anyhow::Result<(HashMap<String, Server>, HashMap<String, String>)>
+{
+  let servers = resource::list_full_for_user::<Server>(
     ResourceQuery::default(),
     &User {
       admin: true,
       ..Default::default()
     },
+    &[],
   )
   .await
   .context("failed to get servers from db (in alert_servers)")?;

bin/core/src/monitor/alert/server.rs

@@ -5,7 +5,7 @@ use derive_variants::ExtractVariant;
 use komodo_client::entities::{
   alert::{Alert, AlertData, AlertDataVariant, SeverityLevel},
   komodo_timestamp, optional_string,
-  server::{ServerListItem, ServerState},
+  server::{Server, ServerState},
   ResourceTarget,
 };
 use mongo_indexed::Indexed;
@@ -28,7 +28,7 @@ type OpenDiskAlertMap = OpenAlertMap<PathBuf>;
 #[instrument(level = "debug")]
 pub async fn alert_servers(
   ts: i64,
-  mut servers: HashMap<String, ServerListItem>,
+  mut servers: HashMap<String, Server>,
 ) {
   let server_statuses = server_status_cache().get_list().await;
 
@@ -70,12 +70,12 @@ pub async fn alert_servers(
           data: AlertData::ServerUnreachable {
             id: server_status.id.clone(),
             name: server.name.clone(),
-            region: optional_string(&server.info.region),
+            region: optional_string(&server.config.region),
             err: server_status.err.clone(),
           },
         };
         alerts_to_open
-          .push((alert, server.info.send_unreachable_alerts))
+          .push((alert, server.config.send_unreachable_alerts))
       }
       (ServerState::NotOk, Some(alert)) => {
         // update alert err
@@ -102,8 +102,10 @@ pub async fn alert_servers(
 
       // Close an open alert
       (ServerState::Ok | ServerState::Disabled, Some(alert)) => {
-        alert_ids_to_close
-          .push((alert.clone(), server.info.send_unreachable_alerts));
+        alert_ids_to_close.push((
+          alert.clone(),
+          server.config.send_unreachable_alerts,
+        ));
       }
       _ => {}
     }
@@ -119,20 +121,21 @@ pub async fn alert_servers(
       .as_ref()
       .and_then(|alerts| alerts.get(&AlertDataVariant::ServerCpu))
       .cloned();
-    match (health.cpu, cpu_alert) {
-      (SeverityLevel::Warning | SeverityLevel::Critical, None) => {
+    match (health.cpu.level, cpu_alert, health.cpu.should_close_alert)
+    {
+      (SeverityLevel::Warning | SeverityLevel::Critical, None, _) => {
         // open alert
         let alert = Alert {
           id: Default::default(),
           ts,
           resolved: false,
           resolved_ts: None,
-          level: health.cpu,
+          level: health.cpu.level,
           target: ResourceTarget::Server(server_status.id.clone()),
           data: AlertData::ServerCpu {
             id: server_status.id.clone(),
             name: server.name.clone(),
-            region: optional_string(&server.info.region),
+            region: optional_string(&server.config.region),
             percentage: server_status
               .stats
               .as_ref()
@@ -140,41 +143,44 @@ pub async fn alert_servers(
               .unwrap_or(0.0),
           },
         };
-        alerts_to_open.push((alert, server.info.send_cpu_alerts));
+        alerts_to_open.push((alert, server.config.send_cpu_alerts));
       }
       (
         SeverityLevel::Warning | SeverityLevel::Critical,
         Some(mut alert),
+        _,
       ) => {
         // modify alert level only if it has increased
-        if alert.level < health.cpu {
-          alert.level = health.cpu;
+        if alert.level < health.cpu.level {
+          alert.level = health.cpu.level;
           alert.data = AlertData::ServerCpu {
             id: server_status.id.clone(),
             name: server.name.clone(),
-            region: optional_string(&server.info.region),
+            region: optional_string(&server.config.region),
             percentage: server_status
               .stats
               .as_ref()
               .map(|s| s.cpu_perc as f64)
               .unwrap_or(0.0),
           };
-          alerts_to_update.push((alert, server.info.send_cpu_alerts));
+          alerts_to_update
+            .push((alert, server.config.send_cpu_alerts));
         }
       }
-      (SeverityLevel::Ok, Some(alert)) => {
+      (SeverityLevel::Ok, Some(alert), true) => {
         let mut alert = alert.clone();
         alert.data = AlertData::ServerCpu {
           id: server_status.id.clone(),
           name: server.name.clone(),
-          region: optional_string(&server.info.region),
+          region: optional_string(&server.config.region),
           percentage: server_status
             .stats
             .as_ref()
             .map(|s| s.cpu_perc as f64)
             .unwrap_or(0.0),
         };
-        alert_ids_to_close.push((alert, server.info.send_cpu_alerts))
+        alert_ids_to_close
+          .push((alert, server.config.send_cpu_alerts))
       }
       _ => {}
     }
@@ -186,20 +192,21 @@ pub async fn alert_servers(
       .as_ref()
       .and_then(|alerts| alerts.get(&AlertDataVariant::ServerMem))
       .cloned();
-    match (health.mem, mem_alert) {
-      (SeverityLevel::Warning | SeverityLevel::Critical, None) => {
+    match (health.mem.level, mem_alert, health.mem.should_close_alert)
+    {
+      (SeverityLevel::Warning | SeverityLevel::Critical, None, _) => {
         // open alert
         let alert = Alert {
           id: Default::default(),
           ts,
           resolved: false,
           resolved_ts: None,
-          level: health.mem,
+          level: health.mem.level,
           target: ResourceTarget::Server(server_status.id.clone()),
           data: AlertData::ServerMem {
             id: server_status.id.clone(),
             name: server.name.clone(),
-            region: optional_string(&server.info.region),
+            region: optional_string(&server.config.region),
             total_gb: server_status
               .stats
               .as_ref()
@@ -212,19 +219,20 @@ pub async fn alert_servers(
               .unwrap_or(0.0),
           },
         };
-        alerts_to_open.push((alert, server.info.send_mem_alerts));
+        alerts_to_open.push((alert, server.config.send_mem_alerts));
       }
       (
         SeverityLevel::Warning | SeverityLevel::Critical,
         Some(mut alert),
+        _,
       ) => {
         // modify alert level only if it has increased
-        if alert.level < health.mem {
-          alert.level = health.mem;
+        if alert.level < health.mem.level {
+          alert.level = health.mem.level;
           alert.data = AlertData::ServerMem {
             id: server_status.id.clone(),
             name: server.name.clone(),
-            region: optional_string(&server.info.region),
+            region: optional_string(&server.config.region),
             total_gb: server_status
               .stats
               .as_ref()
@@ -236,15 +244,16 @@ pub async fn alert_servers(
               .map(|s| s.mem_used_gb)
               .unwrap_or(0.0),
           };
-          alerts_to_update.push((alert, server.info.send_mem_alerts));
+          alerts_to_update
+            .push((alert, server.config.send_mem_alerts));
         }
       }
-      (SeverityLevel::Ok, Some(alert)) => {
+      (SeverityLevel::Ok, Some(alert), true) => {
         let mut alert = alert.clone();
         alert.data = AlertData::ServerMem {
           id: server_status.id.clone(),
           name: server.name.clone(),
-          region: optional_string(&server.info.region),
+          region: optional_string(&server.config.region),
           total_gb: server_status
             .stats
             .as_ref()
@@ -256,7 +265,8 @@ pub async fn alert_servers(
             .map(|s| s.mem_used_gb)
             .unwrap_or(0.0),
         };
-        alert_ids_to_close.push((alert, server.info.send_mem_alerts))
+        alert_ids_to_close
+          .push((alert, server.config.send_mem_alerts))
       }
       _ => {}
     }
@@ -273,8 +283,12 @@ pub async fn alert_servers(
         .as_ref()
         .and_then(|alerts| alerts.get(path))
         .cloned();
-      match (*health, disk_alert) {
-        (SeverityLevel::Warning | SeverityLevel::Critical, None) => {
+      match (health.level, disk_alert, health.should_close_alert) {
+        (
+          SeverityLevel::Warning | SeverityLevel::Critical,
+          None,
+          _,
+        ) => {
           let disk = server_status.stats.as_ref().and_then(|stats| {
             stats.disks.iter().find(|disk| disk.mount == *path)
           });
@@ -283,58 +297,60 @@ pub async fn alert_servers(
             ts,
             resolved: false,
             resolved_ts: None,
-            level: *health,
+            level: health.level,
             target: ResourceTarget::Server(server_status.id.clone()),
             data: AlertData::ServerDisk {
               id: server_status.id.clone(),
               name: server.name.clone(),
-              region: optional_string(&server.info.region),
+              region: optional_string(&server.config.region),
               path: path.to_owned(),
               total_gb: disk.map(|d| d.total_gb).unwrap_or_default(),
               used_gb: disk.map(|d| d.used_gb).unwrap_or_default(),
             },
           };
-          alerts_to_open.push((alert, server.info.send_disk_alerts));
+          alerts_to_open
+            .push((alert, server.config.send_disk_alerts));
         }
         (
           SeverityLevel::Warning | SeverityLevel::Critical,
           Some(mut alert),
+          _,
         ) => {
-          // Disk is persistent, update alert if health changes regardless of direction
-          if *health != alert.level {
+          // modify alert level only if it has increased
+          if health.level < alert.level {
             let disk =
               server_status.stats.as_ref().and_then(|stats| {
                 stats.disks.iter().find(|disk| disk.mount == *path)
               });
-            alert.level = *health;
+            alert.level = health.level;
             alert.data = AlertData::ServerDisk {
               id: server_status.id.clone(),
               name: server.name.clone(),
-              region: optional_string(&server.info.region),
+              region: optional_string(&server.config.region),
               path: path.to_owned(),
               total_gb: disk.map(|d| d.total_gb).unwrap_or_default(),
               used_gb: disk.map(|d| d.used_gb).unwrap_or_default(),
             };
             alerts_to_update
-              .push((alert, server.info.send_disk_alerts));
+              .push((alert, server.config.send_disk_alerts));
           }
         }
-        (SeverityLevel::Ok, Some(alert)) => {
+        (SeverityLevel::Ok, Some(alert), true) => {
           let mut alert = alert.clone();
           let disk = server_status.stats.as_ref().and_then(|stats| {
             stats.disks.iter().find(|disk| disk.mount == *path)
           });
-          alert.level = *health;
+          alert.level = health.level;
           alert.data = AlertData::ServerDisk {
             id: server_status.id.clone(),
             name: server.name.clone(),
-            region: optional_string(&server.info.region),
+            region: optional_string(&server.config.region),
             path: path.to_owned(),
             total_gb: disk.map(|d| d.total_gb).unwrap_or_default(),
             used_gb: disk.map(|d| d.used_gb).unwrap_or_default(),
           };
           alert_ids_to_close
-            .push((alert, server.info.send_disk_alerts))
+            .push((alert, server.config.send_disk_alerts))
         }
         _ => {}
       }
@@ -347,7 +363,7 @@ pub async fn alert_servers(
           let mut alert = alert.clone();
           alert.level = SeverityLevel::Ok;
           alert_ids_to_close
-            .push((alert, server.info.send_disk_alerts));
+            .push((alert, server.config.send_disk_alerts));
         }
       }
     }

bin/core/src/monitor/helpers.rs

@@ -6,7 +6,10 @@ use komodo_client::entities::{
     network::NetworkListItem, volume::VolumeListItem,
   },
   repo::Repo,
-  server::{Server, ServerConfig, ServerHealth, ServerState},
+  server::{
+    Server, ServerConfig, ServerHealth, ServerHealthState,
+    ServerState,
+  },
   stack::{ComposeProject, Stack, StackState},
   stats::{SingleDiskUsage, SystemStats},
 };
@@ -38,6 +41,7 @@ pub async fn insert_deployments_status_unknown(
             id: deployment.id,
             state: DeploymentState::Unknown,
             container: None,
+            update_available: false,
           },
           prev,
         }
@@ -126,6 +130,8 @@ pub async fn insert_server_status(
     .await;
 }
 
+const ALERT_PERCENTAGE_THRESHOLD: f32 = 5.0;
+
 fn get_server_health(
   server: &Server,
   SystemStats {
@@ -148,16 +154,22 @@ fn get_server_health(
   let mut health = ServerHealth::default();
 
   if cpu_perc >= cpu_critical {
-    health.cpu = SeverityLevel::Critical
+    health.cpu.level = SeverityLevel::Critical;
   } else if cpu_perc >= cpu_warning {
-    health.cpu = SeverityLevel::Warning
+    health.cpu.level = SeverityLevel::Warning
+  } else if *cpu_perc < cpu_warning - ALERT_PERCENTAGE_THRESHOLD {
+    health.cpu.should_close_alert = true
   }
 
   let mem_perc = 100.0 * mem_used_gb / mem_total_gb;
   if mem_perc >= *mem_critical {
-    health.mem = SeverityLevel::Critical
+    health.mem.level = SeverityLevel::Critical
   } else if mem_perc >= *mem_warning {
-    health.mem = SeverityLevel::Warning
+    health.mem.level = SeverityLevel::Warning
+  } else if mem_perc
+    < mem_warning - (ALERT_PERCENTAGE_THRESHOLD as f64)
+  {
+    health.mem.should_close_alert = true
   }
 
   for SingleDiskUsage {
@@ -168,14 +180,17 @@ fn get_server_health(
   } in disks
   {
     let perc = 100.0 * used_gb / total_gb;
-    let stats_state = if perc >= *disk_critical {
-      SeverityLevel::Critical
+    let mut state = ServerHealthState::default();
+    if perc >= *disk_critical {
+      state.level = SeverityLevel::Critical;
     } else if perc >= *disk_warning {
-      SeverityLevel::Warning
-    } else {
-      SeverityLevel::Ok
+      state.level = SeverityLevel::Warning;
+    } else if perc
+      < disk_warning - (ALERT_PERCENTAGE_THRESHOLD as f64)
+    {
+      state.should_close_alert = true;
     };
-    health.disks.insert(mount.clone(), stats_state);
+    health.disks.insert(mount.clone(), state);
   }
 
   health

bin/core/src/monitor/mod.rs

@@ -62,6 +62,7 @@ pub struct CachedDeploymentStatus {
   pub id: String,
   pub state: DeploymentState,
   pub container: Option<ContainerListItem>,
+  pub update_available: bool,
 }
 
 #[derive(Default, Clone, Debug)]
@@ -117,12 +118,13 @@ async fn refresh_server_cache(ts: i64) {
 
 #[instrument(level = "debug")]
 pub async fn update_cache_for_server(server: &Server) {
-  let (deployments, repos, stacks) = tokio::join!(
+  let (deployments, builds, repos, stacks) = tokio::join!(
     find_collect(
       &db_client().deployments,
       doc! { "config.server_id": &server.id },
       None,
     ),
+    find_collect(&db_client().builds, doc! {}, None,),
     find_collect(
       &db_client().repos,
       doc! { "config.server_id": &server.id },
@@ -136,6 +138,7 @@ pub async fn update_cache_for_server(server: &Server) {
   );
 
   let deployments =  deployments.inspect_err(|e| error!("failed to get deployments list from db (update status cache) | server : {} | {e:#}", server.name)).unwrap_or_default();
+  let builds =  builds.inspect_err(|e| error!("failed to get builds list from db (update status cache) | server : {} | {e:#}", server.name)).unwrap_or_default();
   let repos = repos.inspect_err(|e|  error!("failed to get repos list from db (update status cache) | server: {} | {e:#}", server.name)).unwrap_or_default();
   let stacks = stacks.inspect_err(|e|  error!("failed to get stacks list from db (update status cache) | server: {} | {e:#}", server.name)).unwrap_or_default();
 
@@ -206,10 +209,24 @@ pub async fn update_cache_for_server(server: &Server) {
   };
 
   match lists::get_docker_lists(&periphery).await {
-    Ok((containers, networks, images, volumes, projects)) => {
+    Ok((mut containers, networks, images, volumes, projects)) => {
+      containers.iter_mut().for_each(|container| {
+        container.server_id = Some(server.id.clone())
+      });
       tokio::join!(
-        resources::update_deployment_cache(deployments, &containers),
-        resources::update_stack_cache(stacks, &containers),
+        resources::update_deployment_cache(
+          server.name.clone(),
+          deployments,
+          &containers,
+          &images,
+          &builds,
+        ),
+        resources::update_stack_cache(
+          server.name.clone(),
+          stacks,
+          &containers,
+          &images
+        ),
       );
       insert_server_status(
         server,
@@ -228,9 +245,6 @@ pub async fn update_cache_for_server(server: &Server) {
       .await;
     }
     Err(e) => {
-      warn!(
-        "could not get docker lists | (update status cache) | {e:#}"
-      );
       insert_deployments_status_unknown(deployments).await;
       insert_stacks_status_unknown(stacks).await;
       insert_server_status(

bin/core/src/monitor/resources.rs

@@ -1,24 +1,53 @@
+use std::{
+  collections::HashSet,
+  sync::{Mutex, OnceLock},
+};
+
 use anyhow::Context;
-use komodo_client::entities::{
-  deployment::{Deployment, DeploymentState},
-  docker::container::ContainerListItem,
-  stack::{Stack, StackService, StackServiceNames},
+use komodo_client::{
+  api::execute::{Deploy, DeployStack},
+  entities::{
+    alert::{Alert, AlertData, SeverityLevel},
+    build::Build,
+    deployment::{Deployment, DeploymentImage, DeploymentState},
+    docker::{
+      container::{ContainerListItem, ContainerStateStatusEnum},
+      image::ImageListItem,
+    },
+    komodo_timestamp,
+    stack::{Stack, StackService, StackServiceNames, StackState},
+    user::auto_redeploy_user,
+    ResourceTarget,
+  },
 };
 
 use crate::{
+  alert::send_alerts,
+  api::execute::{self, ExecuteRequest},
   helpers::query::get_stack_state_from_containers,
   stack::{
     compose_container_match_regex,
     services::extract_services_from_stack,
   },
-  state::{deployment_status_cache, stack_status_cache},
+  state::{
+    action_states, db_client, deployment_status_cache,
+    stack_status_cache,
+  },
 };
 
 use super::{CachedDeploymentStatus, CachedStackStatus, History};
 
+fn deployment_alert_sent_cache() -> &'static Mutex<HashSet<String>> {
+  static CACHE: OnceLock<Mutex<HashSet<String>>> = OnceLock::new();
+  CACHE.get_or_init(Default::default)
+}
+
 pub async fn update_deployment_cache(
+  server_name: String,
   deployments: Vec<Deployment>,
   containers: &[ContainerListItem],
+  images: &[ImageListItem],
+  builds: &[Build],
 ) {
   let deployment_status_cache = deployment_status_cache();
   for deployment in deployments {
@@ -34,6 +63,146 @@ pub async fn update_deployment_cache(
       .as_ref()
       .map(|c| c.state.into())
       .unwrap_or(DeploymentState::NotDeployed);
+    let image = match deployment.config.image {
+      DeploymentImage::Build { build_id, version } => {
+        let (build_name, build_version) = builds
+          .iter()
+          .find(|build| build.id == build_id)
+          .map(|b| (b.name.as_ref(), b.config.version))
+          .unwrap_or(("Unknown", Default::default()));
+        let version = if version.is_none() {
+          build_version.to_string()
+        } else {
+          version.to_string()
+        };
+        format!("{build_name}:{version}")
+      }
+      DeploymentImage::Image { image } => {
+        // If image already has tag, leave it,
+        // otherwise default the tag to latest
+        if image.contains(':') {
+          image
+        } else {
+          format!("{image}:latest")
+        }
+      }
+    };
+    let update_available = if let Some(ContainerListItem {
+      image_id: Some(curr_image_id),
+      ..
+    }) = &container
+    {
+      images
+        .iter()
+        .find(|i| i.name == image)
+        .map(|i| &i.id != curr_image_id)
+        .unwrap_or_default()
+    } else {
+      false
+    };
+
+    if update_available {
+      if deployment.config.auto_update {
+        if state == DeploymentState::Running
+          && !action_states()
+            .deployment
+            .get_or_insert_default(&deployment.id)
+            .await
+            .busy()
+            .unwrap_or(true)
+        {
+          let id = deployment.id.clone();
+          let server_name = server_name.clone();
+          tokio::spawn(async move {
+            match execute::inner_handler(
+              ExecuteRequest::Deploy(Deploy {
+                deployment: deployment.name.clone(),
+                stop_time: None,
+                stop_signal: None,
+              }),
+              auto_redeploy_user().to_owned(),
+            )
+            .await
+            {
+              Ok(_) => {
+                let ts = komodo_timestamp();
+                let alert = Alert {
+                  id: Default::default(),
+                  ts,
+                  resolved: true,
+                  resolved_ts: ts.into(),
+                  level: SeverityLevel::Ok,
+                  target: ResourceTarget::Deployment(id.clone()),
+                  data: AlertData::DeploymentAutoUpdated {
+                    id,
+                    name: deployment.name,
+                    server_name,
+                    server_id: deployment.config.server_id,
+                    image,
+                  },
+                };
+                let res = db_client().alerts.insert_one(&alert).await;
+                if let Err(e) = res {
+                  error!(
+                    "Failed to record DeploymentAutoUpdated to db | {e:#}"
+                  );
+                }
+                send_alerts(&[alert]).await;
+              }
+              Err(e) => {
+                warn!(
+                  "Failed to auto update Deployment {} | {e:#}",
+                  deployment.name
+                )
+              }
+            }
+          });
+        }
+      } else if state == DeploymentState::Running
+        && deployment.config.send_alerts
+        && !deployment_alert_sent_cache()
+          .lock()
+          .unwrap()
+          .contains(&deployment.id)
+      {
+        // Add that it is already sent to the cache, so another alert won't be sent.
+        deployment_alert_sent_cache()
+          .lock()
+          .unwrap()
+          .insert(deployment.id.clone());
+        let ts = komodo_timestamp();
+        let alert = Alert {
+          id: Default::default(),
+          ts,
+          resolved: true,
+          resolved_ts: ts.into(),
+          level: SeverityLevel::Ok,
+          target: ResourceTarget::Deployment(deployment.id.clone()),
+          data: AlertData::DeploymentImageUpdateAvailable {
+            id: deployment.id.clone(),
+            name: deployment.name,
+            server_name: server_name.clone(),
+            server_id: deployment.config.server_id,
+            image,
+          },
+        };
+        let res = db_client().alerts.insert_one(&alert).await;
+        if let Err(e) = res {
+          error!(
+            "Failed to record DeploymentImageUpdateAvailable to db | {e:#}"
+          );
+        }
+        send_alerts(&[alert]).await;
+      }
+    } else {
+      // If it sees there is no longer update available, remove
+      // from the sent cache, so on next `update_available = true`
+      // the cache is empty and a fresh alert will be sent.
+      deployment_alert_sent_cache()
+        .lock()
+        .unwrap()
+        .remove(&deployment.id);
+    }
     deployment_status_cache
       .insert(
         deployment.id.clone(),
@@ -42,6 +211,7 @@ pub async fn update_deployment_cache(
             id: deployment.id,
             state,
             container,
+            update_available,
           },
           prev,
         }
@@ -51,38 +221,185 @@ pub async fn update_deployment_cache(
   }
 }
 
+/// (StackId, Service)
+fn stack_alert_sent_cache(
+) -> &'static Mutex<HashSet<(String, String)>> {
+  static CACHE: OnceLock<Mutex<HashSet<(String, String)>>> =
+    OnceLock::new();
+  CACHE.get_or_init(Default::default)
+}
+
 pub async fn update_stack_cache(
+  server_name: String,
   stacks: Vec<Stack>,
   containers: &[ContainerListItem],
+  images: &[ImageListItem],
 ) {
   let stack_status_cache = stack_status_cache();
   for stack in stacks {
-    let services = match extract_services_from_stack(&stack, false)
-      .await
-    {
-      Ok(services) => services,
-      Err(e) => {
-        warn!("failed to extract services for stack {}. cannot match services to containers. (update status cache) | {e:?}", stack.name);
-        continue;
+    let services = extract_services_from_stack(&stack);
+    let mut services_with_containers = services.iter().map(|StackServiceNames { service_name, container_name, image }| {
+      let container = containers.iter().find(|container| {
+        match compose_container_match_regex(container_name)
+          .with_context(|| format!("failed to construct container name matching regex for service {service_name}")) 
+        {
+          Ok(regex) => regex,
+          Err(e) => {
+            warn!("{e:#}");
+            return false
+          }
+        }.is_match(&container.name)
+      }).cloned();
+      // If image already has tag, leave it,
+      // otherwise default the tag to latest
+      let image = image.clone();
+      let image = if image.contains(':') {
+        image
+      } else {
+        image + ":latest"
+      };
+      let update_available = if let Some(ContainerListItem { image_id: Some(curr_image_id), .. }) = &container {
+        images
+        .iter()
+        .find(|i| i.name == image)
+        .map(|i| &i.id != curr_image_id)
+        .unwrap_or_default()
+      } else {
+        false
+      };
+      if update_available {
+        if !stack.config.auto_update
+          && stack.config.send_alerts
+          && container.is_some()
+          && container.as_ref().unwrap().state == ContainerStateStatusEnum::Running
+          && !stack_alert_sent_cache()
+            .lock()
+            .unwrap()
+            .contains(&(stack.id.clone(), service_name.clone()))
+        {
+          stack_alert_sent_cache()
+            .lock()
+            .unwrap()
+            .insert((stack.id.clone(), service_name.clone()));
+          let ts = komodo_timestamp();
+          let alert = Alert {
+            id: Default::default(),
+            ts,
+            resolved: true,
+            resolved_ts: ts.into(),
+            level: SeverityLevel::Ok,
+            target: ResourceTarget::Stack(stack.id.clone()),
+            data: AlertData::StackImageUpdateAvailable {
+              id: stack.id.clone(),
+              name: stack.name.clone(),
+              server_name: server_name.clone(),
+              server_id: stack.config.server_id.clone(),
+              service: service_name.clone(),
+              image: image.clone(),
+            },
+          };
+          tokio::spawn(async move {
+            let res = db_client().alerts.insert_one(&alert).await;
+            if let Err(e) = res {
+              error!(
+                "Failed to record StackImageUpdateAvailable to db | {e:#}"
+              );
+            }
+            send_alerts(&[alert]).await;
+          });
+        }
+      } else {
+        stack_alert_sent_cache()
+          .lock()
+          .unwrap()
+          .remove(&(stack.id.clone(), service_name.clone()));
       }
-    };
-    let mut services_with_containers = services.iter().map(|StackServiceNames { service_name, container_name }| {
-			let container = containers.iter().find(|container| {
-				match compose_container_match_regex(container_name)
-					.with_context(|| format!("failed to construct container name matching regex for service {service_name}")) 
-				{
-					Ok(regex) => regex,
-					Err(e) => {
-						warn!("{e:#}");
-						return false
-					}
-				}.is_match(&container.name)
-			}).cloned();
-			StackService {
-				service: service_name.clone(),
-				container,
-			}
-		}).collect::<Vec<_>>();
+      StackService {
+        service: service_name.clone(),
+        image: image.clone(),
+        container,
+        update_available,
+      }
+    }).collect::<Vec<_>>();
+
+    let mut update_available = false;
+    let mut images_with_update = Vec::new();
+
+    for service in services_with_containers.iter() {
+      if service.update_available {
+        images_with_update.push(service.image.clone());
+        // Only allow it to actually trigger an auto update deploy
+        // if the service is running.
+        if service
+          .container
+          .as_ref()
+          .map(|c| c.state == ContainerStateStatusEnum::Running)
+          .unwrap_or_default()
+        {
+          update_available = true
+        }
+      }
+    }
+
+    let state = get_stack_state_from_containers(
+      &stack.config.ignore_services,
+      &services,
+      containers,
+    );
+    if update_available
+      && stack.config.auto_update
+      && state == StackState::Running
+      && !action_states()
+        .stack
+        .get_or_insert_default(&stack.id)
+        .await
+        .busy()
+        .unwrap_or(true)
+    {
+      let id = stack.id.clone();
+      let server_name = server_name.clone();
+      tokio::spawn(async move {
+        match execute::inner_handler(
+          ExecuteRequest::DeployStack(DeployStack {
+            stack: stack.name.clone(),
+            service: None,
+            stop_time: None,
+          }),
+          auto_redeploy_user().to_owned(),
+        )
+        .await
+        {
+          Ok(_) => {
+            let ts = komodo_timestamp();
+            let alert = Alert {
+              id: Default::default(),
+              ts,
+              resolved: true,
+              resolved_ts: ts.into(),
+              level: SeverityLevel::Ok,
+              target: ResourceTarget::Stack(id.clone()),
+              data: AlertData::StackAutoUpdated {
+                id,
+                name: stack.name.clone(),
+                server_name,
+                server_id: stack.config.server_id,
+                images: images_with_update,
+              },
+            };
+            let res = db_client().alerts.insert_one(&alert).await;
+            if let Err(e) = res {
+              error!(
+                "Failed to record StackAutoUpdated to db | {e:#}"
+              );
+            }
+            send_alerts(&[alert]).await;
+          }
+          Err(e) => {
+            warn!("Failed auto update Stack {} | {e:#}", stack.name)
+          }
+        }
+      });
+    }
     services_with_containers
       .sort_by(|a, b| a.service.cmp(&b.service));
     let prev = stack_status_cache
@@ -91,11 +408,7 @@ pub async fn update_stack_cache(
       .map(|s| s.curr.state);
     let status = CachedStackStatus {
       id: stack.id.clone(),
-      state: get_stack_state_from_containers(
-        &stack.config.ignore_services,
-        &services,
-        containers,
-      ),
+      state,
       services: services_with_containers,
     };
     stack_status_cache

bin/core/src/resource/action.rs

@@ -0,0 +1,219 @@
+use std::time::Duration;
+
+use anyhow::Context;
+use komodo_client::entities::{
+  action::{
+    Action, ActionConfig, ActionConfigDiff, ActionInfo,
+    ActionListItem, ActionListItemInfo, ActionQuerySpecifics,
+    ActionState, PartialActionConfig,
+  },
+  resource::Resource,
+  update::Update,
+  user::User,
+  Operation, ResourceTargetVariant,
+};
+use mungos::{
+  find::find_collect,
+  mongodb::{bson::doc, options::FindOneOptions, Collection},
+};
+
+use crate::state::{action_state_cache, action_states, db_client};
+
+impl super::KomodoResource for Action {
+  type Config = ActionConfig;
+  type PartialConfig = PartialActionConfig;
+  type ConfigDiff = ActionConfigDiff;
+  type Info = ActionInfo;
+  type ListItem = ActionListItem;
+  type QuerySpecifics = ActionQuerySpecifics;
+
+  fn resource_type() -> ResourceTargetVariant {
+    ResourceTargetVariant::Action
+  }
+
+  fn coll() -> &'static Collection<Resource<Self::Config, Self::Info>>
+  {
+    &db_client().actions
+  }
+
+  async fn to_list_item(
+    action: Resource<Self::Config, Self::Info>,
+  ) -> Self::ListItem {
+    let state = get_action_state(&action.id).await;
+    ActionListItem {
+      name: action.name,
+      id: action.id,
+      tags: action.tags,
+      resource_type: ResourceTargetVariant::Action,
+      info: ActionListItemInfo {
+        state,
+        last_run_at: action.info.last_run_at,
+      },
+    }
+  }
+
+  async fn busy(id: &String) -> anyhow::Result<bool> {
+    action_states()
+      .action
+      .get(id)
+      .await
+      .unwrap_or_default()
+      .busy()
+  }
+
+  // CREATE
+
+  fn create_operation() -> Operation {
+    Operation::CreateAction
+  }
+
+  fn user_can_create(user: &User) -> bool {
+    user.admin
+  }
+
+  async fn validate_create_config(
+    config: &mut Self::PartialConfig,
+    _user: &User,
+  ) -> anyhow::Result<()> {
+    if config.file_contents.is_none() {
+      config.file_contents =
+        Some(DEFAULT_ACTION_FILE_CONTENTS.to_string());
+    }
+    Ok(())
+  }
+
+  async fn post_create(
+    _created: &Resource<Self::Config, Self::Info>,
+    _update: &mut Update,
+  ) -> anyhow::Result<()> {
+    refresh_action_state_cache().await;
+    Ok(())
+  }
+
+  // UPDATE
+
+  fn update_operation() -> Operation {
+    Operation::UpdateAction
+  }
+
+  async fn validate_update_config(
+    _id: &str,
+    _config: &mut Self::PartialConfig,
+    _user: &User,
+  ) -> anyhow::Result<()> {
+    Ok(())
+  }
+
+  async fn post_update(
+    updated: &Self,
+    update: &mut Update,
+  ) -> anyhow::Result<()> {
+    Self::post_create(updated, update).await
+  }
+
+  // RENAME
+
+  fn rename_operation() -> Operation {
+    Operation::RenameAction
+  }
+
+  // DELETE
+
+  fn delete_operation() -> Operation {
+    Operation::DeleteAction
+  }
+
+  async fn pre_delete(
+    _resource: &Resource<Self::Config, Self::Info>,
+    _update: &mut Update,
+  ) -> anyhow::Result<()> {
+    Ok(())
+  }
+
+  async fn post_delete(
+    _resource: &Resource<Self::Config, Self::Info>,
+    _update: &mut Update,
+  ) -> anyhow::Result<()> {
+    Ok(())
+  }
+}
+
+pub fn spawn_action_state_refresh_loop() {
+  tokio::spawn(async move {
+    loop {
+      refresh_action_state_cache().await;
+      tokio::time::sleep(Duration::from_secs(60)).await;
+    }
+  });
+}
+
+pub async fn refresh_action_state_cache() {
+  let _ = async {
+    let actions = find_collect(&db_client().actions, None, None)
+      .await
+      .context("Failed to get Actions from db")?;
+    let cache = action_state_cache();
+    for action in actions {
+      let state = get_action_state_from_db(&action.id).await;
+      cache.insert(action.id, state).await;
+    }
+    anyhow::Ok(())
+  }
+  .await
+  .inspect_err(|e| {
+    error!("Failed to refresh Action state cache | {e:#}")
+  });
+}
+
+async fn get_action_state(id: &String) -> ActionState {
+  if action_states()
+    .action
+    .get(id)
+    .await
+    .map(|s| s.get().map(|s| s.running))
+    .transpose()
+    .ok()
+    .flatten()
+    .unwrap_or_default()
+  {
+    return ActionState::Running;
+  }
+  action_state_cache().get(id).await.unwrap_or_default()
+}
+
+async fn get_action_state_from_db(id: &str) -> ActionState {
+  async {
+    let state = db_client()
+      .updates
+      .find_one(doc! {
+        "target.type": "Action",
+        "target.id": id,
+        "operation": "RunAction"
+      })
+      .with_options(
+        FindOneOptions::builder()
+          .sort(doc! { "start_ts": -1 })
+          .build(),
+      )
+      .await?
+      .map(|u| {
+        if u.success {
+          ActionState::Ok
+        } else {
+          ActionState::Failed
+        }
+      })
+      .unwrap_or(ActionState::Ok);
+    anyhow::Ok(state)
+  }
+  .await
+  .inspect_err(|e| {
+    warn!("Failed to get Action state for {id} | {e:#}")
+  })
+  .unwrap_or(ActionState::Unknown)
+}
+
+const DEFAULT_ACTION_FILE_CONTENTS: &str =
+  "// Run actions using the pre initialized 'komodo' client.
+const version: Types.GetVersionResponse = await komodo.read('GetVersion', {});
+console.log('🦎 Komodo version:', version.version, '🦎\\n');";

bin/core/src/resource/alerter.rs

@@ -25,8 +25,8 @@ impl super::KomodoResource for Alerter {
     ResourceTargetVariant::Alerter
   }
 
-  async fn coll(
-  ) -> &'static Collection<Resource<Self::Config, Self::Info>> {
+  fn coll() -> &'static Collection<Resource<Self::Config, Self::Info>>
+  {
     &db_client().alerters
   }
 
@@ -94,6 +94,12 @@ impl super::KomodoResource for Alerter {
     Ok(())
   }
 
+  // RENAME
+
+  fn rename_operation() -> Operation {
+    Operation::RenameAlerter
+  }
+
   // DELETE
 
   fn delete_operation() -> Operation {

bin/core/src/resource/build.rs

@@ -38,8 +38,8 @@ impl super::KomodoResource for Build {
     ResourceTargetVariant::Build
   }
 
-  async fn coll(
-  ) -> &'static Collection<Resource<Self::Config, Self::Info>> {
+  fn coll() -> &'static Collection<Resource<Self::Config, Self::Info>>
+  {
     &db_client().builds
   }
 
@@ -57,6 +57,7 @@ impl super::KomodoResource for Build {
         version: build.config.version,
         builder_id: build.config.builder_id,
         git_provider: build.config.git_provider,
+        image_registry_domain: build.config.image_registry.domain,
         repo: build.config.repo,
         branch: build.config.branch,
         built_hash: build.info.built_hash,
@@ -117,11 +118,16 @@ impl super::KomodoResource for Build {
   }
 
   async fn post_update(
-    _updated: &Self,
-    _update: &mut Update,
+    updated: &Self,
+    update: &mut Update,
   ) -> anyhow::Result<()> {
-    refresh_build_state_cache().await;
-    Ok(())
+    Self::post_create(updated, update).await
+  }
+
+  // RENAME
+
+  fn rename_operation() -> Operation {
+    Operation::RenameBuild
   }
 
   // DELETE
@@ -179,9 +185,13 @@ async fn validate_config(
 ) -> anyhow::Result<()> {
   if let Some(builder_id) = &config.builder_id {
     if !builder_id.is_empty() {
-      let builder = super::get_check_permissions::<Builder>(builder_id, user, PermissionLevel::Read)
-        .await
-        .context("cannot create build using this builder. user must have at least read permissions on the builder.")?;
+      let builder = super::get_check_permissions::<Builder>(
+        builder_id,
+        user,
+        PermissionLevel::Read,
+      )
+      .await
+      .context("Cannot attach Build to this Builder")?;
       config.builder_id = Some(builder.id)
     }
   }

bin/core/src/resource/builder.rs

@@ -31,8 +31,8 @@ impl super::KomodoResource for Builder {
     ResourceTargetVariant::Builder
   }
 
-  async fn coll(
-  ) -> &'static Collection<Resource<Self::Config, Self::Info>> {
+  fn coll() -> &'static Collection<Resource<Self::Config, Self::Info>>
+  {
     &db_client().builders
   }
 
@@ -40,6 +40,9 @@ impl super::KomodoResource for Builder {
     builder: Resource<Self::Config, Self::Info>,
   ) -> Self::ListItem {
     let (builder_type, instance_type) = match builder.config {
+      BuilderConfig::Url(_) => {
+        (BuilderConfigVariant::Url.to_string(), None)
+      }
       BuilderConfig::Server(config) => (
         BuilderConfigVariant::Server.to_string(),
         Some(config.server_id),
@@ -118,6 +121,12 @@ impl super::KomodoResource for Builder {
     Ok(())
   }
 
+  // RENAME
+
+  fn rename_operation() -> Operation {
+    Operation::RenameBuilder
+  }
+
   // DELETE
 
   fn delete_operation() -> Operation {
@@ -128,17 +137,22 @@ impl super::KomodoResource for Builder {
     resource: &Resource<Self::Config, Self::Info>,
     _update: &mut Update,
   ) -> anyhow::Result<()> {
-    // remove the builder from any attached builds
     db_client()
       .builds
       .update_many(
-        doc! { "config.builder.params.builder_id": &resource.id },
-        mungos::update::Update::Set(
-          doc! { "config.builder.params.builder_id": "" },
-        ),
+        doc! { "config.builder_id": &resource.id },
+        mungos::update::Update::Set(doc! { "config.builder_id": "" }),
       )
       .await
       .context("failed to update_many builds on database")?;
+    db_client()
+      .repos
+      .update_many(
+        doc! { "config.builder_id": &resource.id },
+        mungos::update::Update::Set(doc! { "config.builder_id": "" }),
+      )
+      .await
+      .context("failed to update_many repos on database")?;
     Ok(())
   }
 

bin/core/src/resource/deployment.rs

@@ -26,7 +26,6 @@ use crate::{
     query::get_deployment_state,
   },
   monitor::update_cache_for_server,
-  resource,
   state::{action_states, db_client, deployment_status_cache},
 };
 
@@ -44,8 +43,8 @@ impl super::KomodoResource for Deployment {
     ResourceTargetVariant::Deployment
   }
 
-  async fn coll(
-  ) -> &'static Collection<Resource<Self::Config, Self::Info>> {
+  fn coll() -> &'static Collection<Resource<Self::Config, Self::Info>>
+  {
     &db_client().deployments
   }
 
@@ -73,6 +72,19 @@ impl super::KomodoResource for Deployment {
       }
       DeploymentImage::Image { image } => (image, None),
     };
+    let (image, update_available) = status
+      .as_ref()
+      .and_then(|s| {
+        s.curr.container.as_ref().map(|c| {
+          (
+            c.image
+              .clone()
+              .unwrap_or_else(|| String::from("Unknown")),
+            s.curr.update_available,
+          )
+        })
+      })
+      .unwrap_or((build_image, false));
     DeploymentListItem {
       name: deployment.name,
       id: deployment.id,
@@ -86,16 +98,8 @@ impl super::KomodoResource for Deployment {
         status: status.as_ref().and_then(|s| {
           s.curr.container.as_ref().and_then(|c| c.status.to_owned())
         }),
-        image: status
-          .as_ref()
-          .and_then(|s| {
-            s.curr.container.as_ref().map(|c| {
-              c.image
-                .clone()
-                .unwrap_or_else(|| String::from("Unknown"))
-            })
-          })
-          .unwrap_or(build_image),
+        image,
+        update_available,
         server_id: deployment.config.server_id,
         build_id,
       },
@@ -132,11 +136,21 @@ impl super::KomodoResource for Deployment {
     created: &Resource<Self::Config, Self::Info>,
     _update: &mut Update,
   ) -> anyhow::Result<()> {
-    if !created.config.server_id.is_empty() {
-      let server =
-        resource::get::<Server>(&created.config.server_id).await?;
-      update_cache_for_server(&server).await;
+    if created.config.server_id.is_empty() {
+      return Ok(());
     }
+    let Ok(server) = super::get::<Server>(&created.config.server_id)
+      .await
+      .inspect_err(|e| {
+        warn!(
+          "Failed to get Server for Deployment {} | {e:#}",
+          created.name
+        )
+      })
+    else {
+      return Ok(());
+    };
+    update_cache_for_server(&server).await;
     Ok(())
   }
 
@@ -156,14 +170,15 @@ impl super::KomodoResource for Deployment {
 
   async fn post_update(
     updated: &Self,
-    _update: &mut Update,
+    update: &mut Update,
   ) -> anyhow::Result<()> {
-    if !updated.config.server_id.is_empty() {
-      let server =
-        resource::get::<Server>(&updated.config.server_id).await?;
-      update_cache_for_server(&server).await;
-    }
-    Ok(())
+    Self::post_create(updated, update).await
+  }
+
+  // RENAME
+
+  fn rename_operation() -> Operation {
+    Operation::RenameDeployment
   }
 
   // DELETE
@@ -262,9 +277,13 @@ async fn validate_config(
 ) -> anyhow::Result<()> {
   if let Some(server_id) = &config.server_id {
     if !server_id.is_empty() {
-      let server = get_check_permissions::<Server>(server_id, user, PermissionLevel::Write)
-          .await
-          .context("cannot create deployment on this server. user must have update permissions on the server to perform this action.")?;
+      let server = get_check_permissions::<Server>(
+        server_id,
+        user,
+        PermissionLevel::Write,
+      )
+      .await
+      .context("Cannot attach Deployment to this Server")?;
       config.server_id = Some(server.id);
     }
   }
@@ -272,9 +291,15 @@ async fn validate_config(
     &config.image
   {
     if !build_id.is_empty() {
-      let build = get_check_permissions::<Build>(build_id, user, PermissionLevel::Read)
-          .await
-          .context("cannot create deployment with this build attached. user must have at least read permissions on the build to perform this action.")?;
+      let build = get_check_permissions::<Build>(
+        build_id,
+        user,
+        PermissionLevel::Read,
+      )
+      .await
+      .context(
+        "Cannot update deployment with this build attached.",
+      )?;
       config.image = Some(DeploymentImage::Build {
         build_id: build.id,
         version: *version,

bin/core/src/resource/mod.rs

@@ -7,7 +7,7 @@ use anyhow::{anyhow, Context};
 use formatting::format_serror;
 use futures::{future::join_all, FutureExt};
 use komodo_client::{
-  api::write::CreateTag,
+  api::{read::ExportResourcesToToml, write::CreateTag},
   entities::{
     komodo_timestamp,
     permission::PermissionLevel,
@@ -15,9 +15,10 @@ use komodo_client::{
     tag::Tag,
     to_komodo_name,
     update::Update,
-    user::User,
+    user::{system_user, User},
     Operation, ResourceTarget, ResourceTargetVariant,
   },
+  parsers::parse_string_list,
 };
 use mungos::{
   by_id::{delete_one_by_id, update_one_by_id},
@@ -45,6 +46,7 @@ use crate::{
   state::{db_client, State},
 };
 
+mod action;
 mod alerter;
 mod build;
 mod builder;
@@ -57,6 +59,9 @@ mod server_template;
 mod stack;
 mod sync;
 
+pub use action::{
+  refresh_action_state_cache, spawn_action_state_refresh_loop,
+};
 pub use build::{
   refresh_build_state_cache, spawn_build_state_refresh_loop,
 };
@@ -106,8 +111,7 @@ pub trait KomodoResource {
 
   fn resource_type() -> ResourceTargetVariant;
 
-  async fn coll(
-  ) -> &'static Collection<Resource<Self::Config, Self::Info>>;
+  fn coll() -> &'static Collection<Resource<Self::Config, Self::Info>>;
 
   async fn to_list_item(
     resource: Resource<Self::Config, Self::Info>,
@@ -165,6 +169,12 @@ pub trait KomodoResource {
     update: &mut Update,
   ) -> anyhow::Result<()>;
 
+  // =======
+  // RENAME
+  // =======
+
+  fn rename_operation() -> Operation;
+
   // =======
   // DELETE
   // =======
@@ -195,7 +205,6 @@ pub async fn get<T: KomodoResource>(
   id_or_name: &str,
 ) -> anyhow::Result<Resource<T::Config, T::Info>> {
   T::coll()
-    .await
     .find_one(id_or_name_filter(id_or_name))
     .await
     .context("failed to query db for resource")?
@@ -228,83 +237,12 @@ pub async fn get_check_permissions<T: KomodoResource>(
     Ok(resource)
   } else {
     Err(anyhow!(
-      "user does not have required permissions on this {}",
+      "User does not have required permissions on this {}. Must have at least {permission_level} permissions",
       T::resource_type()
     ))
   }
 }
 
-// ======
-// LIST
-// ======
-
-/// Returns None if still no need to filter by resource id (eg transparent mode, group membership with all access).
-#[instrument(level = "debug")]
-pub async fn get_resource_ids_for_user<T: KomodoResource>(
-  user: &User,
-) -> anyhow::Result<Option<Vec<ObjectId>>> {
-  // Check admin or transparent mode
-  if user.admin || core_config().transparent_mode {
-    return Ok(None);
-  }
-
-  let resource_type = T::resource_type();
-
-  // Check user 'all' on variant
-  if let Some(level) = user.all.get(&resource_type).cloned() {
-    if level > PermissionLevel::None {
-      return Ok(None);
-    }
-  }
-
-  // Check user groups 'all' on variant
-  let groups = get_user_user_groups(&user.id).await?;
-  for group in &groups {
-    if let Some(level) = group.all.get(&resource_type).cloned() {
-      if level > PermissionLevel::None {
-        return Ok(None);
-      }
-    }
-  }
-
-  let (base, perms) = tokio::try_join!(
-    // Get any resources with non-none base permission,
-    find_collect(
-      T::coll().await,
-      doc! { "base_permission": { "$ne": "None" } },
-      None,
-    )
-    .map(|res| res.with_context(|| format!(
-      "failed to query {resource_type} on db"
-    ))),
-    // And any ids using the permissions table
-    find_collect(
-      &db_client().permissions,
-      doc! {
-        "$or": user_target_query(&user.id, &groups)?,
-        "resource_target.type": resource_type.as_ref(),
-        "level": { "$in": ["Read", "Execute", "Write"] }
-      },
-      None,
-    )
-    .map(|res| res.context("failed to query permissions on db"))
-  )?;
-
-  // Add specific ids
-  let ids = perms
-    .into_iter()
-    .map(|p| p.resource_target.extract_variant_id().1.to_string())
-    // Chain in the ones with non-None base permissions
-    .chain(base.into_iter().map(|res| res.id))
-    // collect into hashset first to remove any duplicates
-    .collect::<HashSet<_>>()
-    .into_iter()
-    .flat_map(|id| ObjectId::from_str(&id))
-    .collect::<HashSet<_>>();
-
-  Ok(Some(ids.into_iter().collect()))
-}
-
 #[instrument(level = "debug")]
 pub async fn get_user_permission_on_resource<T: KomodoResource>(
   user: &User,
@@ -378,17 +316,118 @@ pub async fn get_user_permission_on_resource<T: KomodoResource>(
   Ok(permission)
 }
 
+// ======
+// LIST
+// ======
+
+/// Returns None if still no need to filter by resource id (eg transparent mode, group membership with all access).
+#[instrument(level = "debug")]
+pub async fn get_resource_object_ids_for_user<T: KomodoResource>(
+  user: &User,
+) -> anyhow::Result<Option<Vec<ObjectId>>> {
+  get_resource_ids_for_user::<T>(user).await.map(|ids| {
+    ids.map(|ids| {
+      ids
+        .into_iter()
+        .flat_map(|id| ObjectId::from_str(&id))
+        .collect()
+    })
+  })
+}
+
+/// Returns None if still no need to filter by resource id (eg transparent mode, group membership with all access).
+#[instrument(level = "debug")]
+pub async fn get_resource_ids_for_user<T: KomodoResource>(
+  user: &User,
+) -> anyhow::Result<Option<Vec<String>>> {
+  // Check admin or transparent mode
+  if user.admin || core_config().transparent_mode {
+    return Ok(None);
+  }
+
+  let resource_type = T::resource_type();
+
+  // Check user 'all' on variant
+  if let Some(level) = user.all.get(&resource_type).cloned() {
+    if level > PermissionLevel::None {
+      return Ok(None);
+    }
+  }
+
+  // Check user groups 'all' on variant
+  let groups = get_user_user_groups(&user.id).await?;
+  for group in &groups {
+    if let Some(level) = group.all.get(&resource_type).cloned() {
+      if level > PermissionLevel::None {
+        return Ok(None);
+      }
+    }
+  }
+
+  let (base, perms) = tokio::try_join!(
+    // Get any resources with non-none base permission,
+    find_collect(
+      T::coll(),
+      doc! { "base_permission": { "$exists": true, "$ne": "None" } },
+      None,
+    )
+    .map(|res| res.with_context(|| format!(
+      "failed to query {resource_type} on db"
+    ))),
+    // And any ids using the permissions table
+    find_collect(
+      &db_client().permissions,
+      doc! {
+        "$or": user_target_query(&user.id, &groups)?,
+        "resource_target.type": resource_type.as_ref(),
+        "level": { "$exists": true, "$ne": "None" }
+      },
+      None,
+    )
+    .map(|res| res.context("failed to query permissions on db"))
+  )?;
+
+  // Add specific ids
+  let ids = perms
+    .into_iter()
+    .map(|p| p.resource_target.extract_variant_id().1.to_string())
+    // Chain in the ones with non-None base permissions
+    .chain(base.into_iter().map(|res| res.id))
+    // collect into hashset first to remove any duplicates
+    .collect::<HashSet<_>>();
+
+  Ok(Some(ids.into_iter().collect()))
+}
+
 #[instrument(level = "debug")]
 pub async fn list_for_user<T: KomodoResource>(
   mut query: ResourceQuery<T::QuerySpecifics>,
   user: &User,
+  all_tags: &[Tag],
 ) -> anyhow::Result<Vec<T::ListItem>> {
-  validate_resource_query_tags(&mut query).await;
+  validate_resource_query_tags(&mut query, all_tags)?;
   let mut filters = Document::new();
   query.add_filters(&mut filters);
   list_for_user_using_document::<T>(filters, user).await
 }
 
+#[instrument(level = "debug")]
+pub async fn list_for_user_using_pattern<T: KomodoResource>(
+  pattern: &str,
+  query: ResourceQuery<T::QuerySpecifics>,
+  user: &User,
+  all_tags: &[Tag],
+) -> anyhow::Result<Vec<T::ListItem>> {
+  let list = list_full_for_user_using_pattern::<T>(
+    pattern, query, user, all_tags,
+  )
+  .await?
+  .into_iter()
+  .map(|resource| T::to_list_item(resource));
+  Ok(join_all(list).await)
+}
+
+#[instrument(level = "debug")]
 pub async fn list_for_user_using_document<T: KomodoResource>(
   filters: Document,
   user: &User,
@@ -400,12 +439,62 @@ pub async fn list_for_user_using_document<T: KomodoResource>(
   Ok(join_all(list).await)
 }
 
+/// Lists full resource matching wildcard syntax,
+/// or regex if wrapped with "\\"
+///
+/// ## Example
+/// ```
+/// let items = list_full_for_user_using_match_string::<Build>("foo-*", Default::default(), user, all_tags).await?;
+/// let items = list_full_for_user_using_match_string::<Build>("\\^foo-.*$\\", Default::default(), user, all_tags).await?;
+/// ```
+#[instrument(level = "debug")]
+pub async fn list_full_for_user_using_pattern<T: KomodoResource>(
+  pattern: &str,
+  query: ResourceQuery<T::QuerySpecifics>,
+  user: &User,
+  all_tags: &[Tag],
+) -> anyhow::Result<Vec<Resource<T::Config, T::Info>>> {
+  let resources =
+    list_full_for_user::<T>(query, user, all_tags).await?;
+
+  let patterns = parse_string_list(pattern);
+  let mut names = HashSet::<String>::new();
+
+  for pattern in patterns {
+    if pattern.starts_with('\\') && pattern.ends_with('\\') {
+      let regex = regex::Regex::new(&pattern[1..(pattern.len() - 1)])
+        .context("Regex matching string invalid")?;
+      for resource in &resources {
+        if regex.is_match(&resource.name) {
+          names.insert(resource.name.clone());
+        }
+      }
+    } else {
+      let wildcard = wildcard::Wildcard::new(pattern.as_bytes())
+        .context("Wildcard matching string invalid")?;
+      for resource in &resources {
+        if wildcard.is_match(resource.name.as_bytes()) {
+          names.insert(resource.name.clone());
+        }
+      }
+    };
+  }
+
+  Ok(
+    resources
+      .into_iter()
+      .filter(|resource| names.contains(resource.name.as_str()))
+      .collect(),
+  )
+}
+
 #[instrument(level = "debug")]
 pub async fn list_full_for_user<T: KomodoResource>(
   mut query: ResourceQuery<T::QuerySpecifics>,
   user: &User,
+  all_tags: &[Tag],
 ) -> anyhow::Result<Vec<Resource<T::Config, T::Info>>> {
-  validate_resource_query_tags(&mut query).await;
+  validate_resource_query_tags(&mut query, all_tags)?;
   let mut filters = Document::new();
   query.add_filters(&mut filters);
   list_full_for_user_using_document::<T>(filters, user).await
@@ -416,11 +505,13 @@ pub async fn list_full_for_user_using_document<T: KomodoResource>(
   mut filters: Document,
   user: &User,
 ) -> anyhow::Result<Vec<Resource<T::Config, T::Info>>> {
-  if let Some(ids) = get_resource_ids_for_user::<T>(user).await? {
+  if let Some(ids) =
+    get_resource_object_ids_for_user::<T>(user).await?
+  {
     filters.insert("_id", doc! { "$in": ids });
   }
   find_collect(
-    T::coll().await,
+    T::coll(),
     filters,
     FindOptions::builder().sort(doc! { "name": 1 }).build(),
   )
@@ -443,7 +534,7 @@ pub async fn get_id_to_resource_map<T: KomodoResource>(
   id_to_tags: &HashMap<String, Tag>,
   match_tags: &[String],
 ) -> anyhow::Result<IdResourceMap<T>> {
-  let res = find_collect(T::coll().await, None, None)
+  let res = find_collect(T::coll(), None, None)
     .await
     .with_context(|| {
       format!("failed to pull {}s from mongo", T::resource_type())
@@ -508,6 +599,17 @@ pub async fn create<T: KomodoResource>(
     return Err(anyhow!("valid ObjectIds cannot be used as names."));
   }
 
+  // Ensure an existing resource with same name doesn't already exist
+  // The database indexing also ensures this but doesn't give a good error message.
+  if list_full_for_user::<T>(Default::default(), system_user(), &[])
+    .await
+    .context("Failed to list all resources for duplicate name check")?
+    .into_iter()
+    .any(|r| r.name == name)
+  {
+    return Err(anyhow!("Must provide unique name for resource."));
+  }
+
   let start_ts = komodo_timestamp();
 
   T::validate_create_config(&mut config, user).await?;
@@ -524,7 +626,6 @@ pub async fn create<T: KomodoResource>(
   };
 
   let resource_id = T::coll()
-    .await
     .insert_one(&resource)
     .await
     .with_context(|| {
@@ -592,7 +693,7 @@ pub async fn update<T: KomodoResource>(
   let diff = resource.config.partial_diff(config);
 
   if diff.is_none() {
-    return Err(anyhow!("update has no changes"));
+    return Ok(resource);
   }
 
   let mut diff_log = String::from("diff");
@@ -613,14 +714,9 @@ pub async fn update<T: KomodoResource>(
 
   let update_doc = flatten_document(doc! { "config": config_doc });
 
-  update_one_by_id(
-    T::coll().await,
-    &id,
-    doc! { "$set": update_doc },
-    None,
-  )
-  .await
-  .context("failed to update resource on database")?;
+  update_one_by_id(T::coll(), &id, doc! { "$set": update_doc }, None)
+    .await
+    .context("failed to update resource on database")?;
 
   let mut update = make_update(
     resource_target::<T>(id),
@@ -660,6 +756,7 @@ fn resource_target<T: KomodoResource>(id: String) -> ResourceTarget {
       ResourceTarget::ResourceSync(id)
     }
     ResourceTargetVariant::Stack => ResourceTarget::Stack(id),
+    ResourceTargetVariant::Action => ResourceTarget::Action(id),
   }
 }
 
@@ -675,7 +772,6 @@ pub async fn update_description<T: KomodoResource>(
   )
   .await?;
   T::coll()
-    .await
     .update_one(
       id_or_name_filter(id_or_name),
       doc! { "$set": { "description": description } },
@@ -709,7 +805,6 @@ pub async fn update_tags<T: KomodoResource>(
     .flatten()
     .collect::<Vec<_>>();
   T::coll()
-    .await
     .update_one(
       id_or_name_filter(id_or_name),
       doc! { "$set": { "tags": tags } },
@@ -722,13 +817,67 @@ pub async fn remove_tag_from_all<T: KomodoResource>(
   tag_id: &str,
 ) -> anyhow::Result<()> {
   T::coll()
-    .await
     .update_many(doc! {}, doc! { "$pull": { "tags": tag_id } })
     .await
     .context("failed to remove tag from resources")?;
   Ok(())
 }
 
+// =======
+// RENAME
+// =======
+
+pub async fn rename<T: KomodoResource>(
+  id_or_name: &str,
+  name: &str,
+  user: &User,
+) -> anyhow::Result<Update> {
+  let resource = get_check_permissions::<T>(
+    id_or_name,
+    user,
+    PermissionLevel::Write,
+  )
+  .await?;
+
+  let mut update = make_update(
+    resource_target::<T>(resource.id.clone()),
+    T::rename_operation(),
+    user,
+  );
+
+  let name = to_komodo_name(name);
+
+  update_one_by_id(
+    T::coll(),
+    &resource.id,
+    mungos::update::Update::Set(
+      doc! { "name": &name, "updated_at": komodo_timestamp() },
+    ),
+    None,
+  )
+  .await
+  .with_context(|| {
+    format!(
+      "Failed to update {ty} on db. This name may already be taken.",
+      ty = T::resource_type()
+    )
+  })?;
+
+  update.push_simple_log(
+    &format!("Rename {}", T::resource_type()),
+    format!(
+      "Renamed {ty} {id} from {prev_name} to {name}",
+      ty = T::resource_type(),
+      id = resource.id,
+      prev_name = resource.name
+    ),
+  );
+
+  update.finalize();
+  update.id = add_update(update.clone()).await?;
+  Ok(update)
+}
+
 // =======
 // DELETE
 // =======
@@ -749,6 +898,16 @@ pub async fn delete<T: KomodoResource>(
   }
 
   let target = resource_target::<T>(resource.id.clone());
+  let toml = State
+    .resolve(
+      ExportResourcesToToml {
+        targets: vec![target.clone()],
+        ..Default::default()
+      },
+      user.clone(),
+    )
+    .await?
+    .toml;
 
   let mut update =
     make_update(target.clone(), T::delete_operation(), user);
@@ -758,16 +917,17 @@ pub async fn delete<T: KomodoResource>(
   delete_all_permissions_on_resource(target.clone()).await;
   remove_from_recently_viewed(target.clone()).await;
 
-  delete_one_by_id(T::coll().await, &resource.id, None)
+  delete_one_by_id(T::coll(), &resource.id, None)
     .await
     .with_context(|| {
-      format!("failed to delete {} from database", T::resource_type())
+      format!("Failed to delete {} from database", T::resource_type())
     })?;
 
   update.push_simple_log(
-    &format!("delete {}", T::resource_type()),
-    format!("deleted {} {}", T::resource_type(), resource.name),
+    &format!("Delete {}", T::resource_type()),
+    format!("Deleted {} {}", T::resource_type(), resource.name),
   );
+  update.push_simple_log("Deleted Toml", toml);
 
   if let Err(e) = T::post_delete(&resource, &mut update).await {
     update.push_error_log("post delete", format_serror(&e.into()));
@@ -782,14 +942,24 @@ pub async fn delete<T: KomodoResource>(
 // =======
 
 #[instrument(level = "debug")]
-pub async fn validate_resource_query_tags<
-  T: Default + std::fmt::Debug,
->(
+pub fn validate_resource_query_tags<T: Default + std::fmt::Debug>(
   query: &mut ResourceQuery<T>,
-) {
-  let futures = query.tags.iter().map(|tag| get_tag(tag));
-  let res = join_all(futures).await;
-  query.tags = res.into_iter().flatten().map(|tag| tag.id).collect();
+  all_tags: &[Tag],
+) -> anyhow::Result<()> {
+  query.tags = query
+    .tags
+    .iter()
+    .map(|tag| {
+      all_tags
+        .iter()
+        .find(|t| t.name == *tag || t.id == *tag)
+        .map(|tag| tag.id.clone())
+        .with_context(|| {
+          format!("No tag found matching name or id: {}", tag)
+        })
+    })
+    .collect::<anyhow::Result<Vec<_>>>()?;
+  Ok(())
 }
 
 #[instrument]
@@ -823,6 +993,7 @@ where
     ResourceTarget::Build(id) => ("recents.Build", id),
     ResourceTarget::Repo(id) => ("recents.Repo", id),
     ResourceTarget::Procedure(id) => ("recents.Procedure", id),
+    ResourceTarget::Action(id) => ("recents.Action", id),
     ResourceTarget::Stack(id) => ("recents.Stack", id),
     ResourceTarget::Builder(id) => ("recents.Builder", id),
     ResourceTarget::Alerter(id) => ("recents.Alerter", id),

bin/core/src/resource/procedure.rs

@@ -4,6 +4,7 @@ use anyhow::{anyhow, Context};
 use komodo_client::{
   api::execute::Execution,
   entities::{
+    action::Action,
     build::Build,
     deployment::Deployment,
     permission::PermissionLevel,
@@ -44,8 +45,8 @@ impl super::KomodoResource for Procedure {
     ResourceTargetVariant::Procedure
   }
 
-  async fn coll(
-  ) -> &'static Collection<Resource<Self::Config, Self::Info>> {
+  fn coll() -> &'static Collection<Resource<Self::Config, Self::Info>>
+  {
     &db_client().procedures
   }
 
@@ -114,11 +115,16 @@ impl super::KomodoResource for Procedure {
   }
 
   async fn post_update(
-    _updated: &Self,
-    _update: &mut Update,
+    updated: &Self,
+    update: &mut Update,
   ) -> anyhow::Result<()> {
-    refresh_procedure_state_cache().await;
-    Ok(())
+    Self::post_create(updated, update).await
+  }
+
+  // RENAME
+
+  fn rename_operation() -> Operation {
+    Operation::RenameProcedure
   }
 
   // DELETE
@@ -172,6 +178,29 @@ async fn validate_config(
           }
           params.procedure = procedure.id;
         }
+        Execution::BatchRunProcedure(_params) => {
+          if !user.admin {
+            return Err(anyhow!(
+              "Non admin user cannot configure Batch executions"
+            ));
+          }
+        }
+        Execution::RunAction(params) => {
+          let action = super::get_check_permissions::<Action>(
+            &params.action,
+            user,
+            PermissionLevel::Execute,
+          )
+          .await?;
+          params.action = action.id;
+        }
+        Execution::BatchRunAction(_params) => {
+          if !user.admin {
+            return Err(anyhow!(
+              "Non admin user cannot configure Batch executions"
+            ));
+          }
+        }
         Execution::RunBuild(params) => {
           let build = super::get_check_permissions::<Build>(
             &params.build,
@@ -181,6 +210,13 @@ async fn validate_config(
           .await?;
           params.build = build.id;
         }
+        Execution::BatchRunBuild(_params) => {
+          if !user.admin {
+            return Err(anyhow!(
+              "Non admin user cannot configure Batch executions"
+            ));
+          }
+        }
         Execution::CancelBuild(params) => {
           let build = super::get_check_permissions::<Build>(
             &params.build,
@@ -200,6 +236,23 @@ async fn validate_config(
             .await?;
           params.deployment = deployment.id;
         }
+        Execution::BatchDeploy(_params) => {
+          if !user.admin {
+            return Err(anyhow!(
+              "Non admin user cannot configure Batch executions"
+            ));
+          }
+        }
+        Execution::PullDeployment(params) => {
+          let deployment =
+            super::get_check_permissions::<Deployment>(
+              &params.deployment,
+              user,
+              PermissionLevel::Execute,
+            )
+            .await?;
+          params.deployment = deployment.id;
+        }
         Execution::StartDeployment(params) => {
           let deployment =
             super::get_check_permissions::<Deployment>(
@@ -260,6 +313,13 @@ async fn validate_config(
             .await?;
           params.deployment = deployment.id;
         }
+        Execution::BatchDestroyDeployment(_params) => {
+          if !user.admin {
+            return Err(anyhow!(
+              "Non admin user cannot configure Batch executions"
+            ));
+          }
+        }
         Execution::CloneRepo(params) => {
           let repo = super::get_check_permissions::<Repo>(
             &params.repo,
@@ -269,6 +329,13 @@ async fn validate_config(
           .await?;
           params.repo = repo.id;
         }
+        Execution::BatchCloneRepo(_params) => {
+          if !user.admin {
+            return Err(anyhow!(
+              "Non admin user cannot configure Batch executions"
+            ));
+          }
+        }
         Execution::PullRepo(params) => {
           let repo = super::get_check_permissions::<Repo>(
             &params.repo,
@@ -278,6 +345,13 @@ async fn validate_config(
           .await?;
           params.repo = repo.id;
         }
+        Execution::BatchPullRepo(_params) => {
+          if !user.admin {
+            return Err(anyhow!(
+              "Non admin user cannot configure Batch executions"
+            ));
+          }
+        }
         Execution::BuildRepo(params) => {
           let repo = super::get_check_permissions::<Repo>(
             &params.repo,
@@ -287,6 +361,13 @@ async fn validate_config(
           .await?;
           params.repo = repo.id;
         }
+        Execution::BatchBuildRepo(_params) => {
+          if !user.admin {
+            return Err(anyhow!(
+              "Non admin user cannot configure Batch executions"
+            ));
+          }
+        }
         Execution::CancelRepoBuild(params) => {
           let repo = super::get_check_permissions::<Repo>(
             &params.repo,
@@ -494,6 +575,16 @@ async fn validate_config(
           .await?;
           params.sync = sync.id;
         }
+        Execution::CommitSync(params) => {
+          // This one is actually a write operation.
+          let sync = super::get_check_permissions::<ResourceSync>(
+            &params.sync,
+            user,
+            PermissionLevel::Write,
+          )
+          .await?;
+          params.sync = sync.id;
+        }
         Execution::DeployStack(params) => {
           let stack = super::get_check_permissions::<Stack>(
             &params.stack,
@@ -503,6 +594,38 @@ async fn validate_config(
           .await?;
           params.stack = stack.id;
         }
+        Execution::BatchDeployStack(_params) => {
+          if !user.admin {
+            return Err(anyhow!(
+              "Non admin user cannot configure Batch executions"
+            ));
+          }
+        }
+        Execution::DeployStackIfChanged(params) => {
+          let stack = super::get_check_permissions::<Stack>(
+            &params.stack,
+            user,
+            PermissionLevel::Execute,
+          )
+          .await?;
+          params.stack = stack.id;
+        }
+        Execution::BatchDeployStackIfChanged(_params) => {
+          if !user.admin {
+            return Err(anyhow!(
+              "Non admin user cannot configure Batch executions"
+            ));
+          }
+        }
+        Execution::PullStack(params) => {
+          let stack = super::get_check_permissions::<Stack>(
+            &params.stack,
+            user,
+            PermissionLevel::Execute,
+          )
+          .await?;
+          params.stack = stack.id;
+        }
         Execution::StartStack(params) => {
           let stack = super::get_check_permissions::<Stack>(
             &params.stack,
@@ -557,6 +680,13 @@ async fn validate_config(
           .await?;
           params.stack = stack.id;
         }
+        Execution::BatchDestroyStack(_params) => {
+          if !user.admin {
+            return Err(anyhow!(
+              "Non admin user cannot configure Batch executions"
+            ));
+          }
+        }
         Execution::Sleep(_) => {}
       }
     }
@@ -579,7 +709,7 @@ pub async fn refresh_procedure_state_cache() {
     let procedures =
       find_collect(&db_client().procedures, None, None)
         .await
-        .context("failed to get procedures from db")?;
+        .context("Failed to get Procedures from db")?;
     let cache = procedure_state_cache();
     for procedure in procedures {
       let state = get_procedure_state_from_db(&procedure.id).await;
@@ -589,7 +719,7 @@ pub async fn refresh_procedure_state_cache() {
   }
   .await
   .inspect_err(|e| {
-    error!("failed to refresh build state cache | {e:#}")
+    error!("Failed to refresh Procedure state cache | {e:#}")
   });
 }
 
@@ -636,7 +766,7 @@ async fn get_procedure_state_from_db(id: &str) -> ProcedureState {
   }
   .await
   .inspect_err(|e| {
-    warn!("failed to get procedure state for {id} | {e:#}")
+    warn!("Failed to get Procedure state for {id} | {e:#}")
   })
   .unwrap_or(ProcedureState::Unknown)
 }

bin/core/src/resource/refresh.rs

@@ -1,4 +1,6 @@
-use async_timing_util::{wait_until_timelength, Timelength};
+use std::time::Duration;
+
+use async_timing_util::{get_timelength_in_ms, Timelength};
 use komodo_client::{
   api::write::{
     RefreshBuildCache, RefreshRepoCache, RefreshResourceSyncPending,
@@ -10,6 +12,7 @@ use mungos::find::find_collect;
 use resolver_api::Resolve;
 
 use crate::{
+  api::execute::pull_deployment_inner,
   config::core_config,
   state::{db_client, State},
 };
@@ -20,9 +23,11 @@ pub fn spawn_resource_refresh_loop() {
     .try_into()
     .expect("Invalid resource poll interval");
   tokio::spawn(async move {
-    refresh_all().await;
+    let mut interval = tokio::time::interval(Duration::from_millis(
+      get_timelength_in_ms(interval) as u64,
+    ));
     loop {
-      wait_until_timelength(interval, 3000).await;
+      interval.tick().await;
       refresh_all().await;
     }
   });
@@ -30,6 +35,7 @@ pub fn spawn_resource_refresh_loop() {
 
 async fn refresh_all() {
   refresh_stacks().await;
+  refresh_deployments().await;
   refresh_builds().await;
   refresh_repos().await;
   refresh_syncs().await;
@@ -60,6 +66,43 @@ async fn refresh_stacks() {
   }
 }
 
+async fn refresh_deployments() {
+  let servers = find_collect(&db_client().servers, None, None)
+    .await
+    .inspect_err(|e| {
+      warn!(
+        "Failed to get Servers from database in refresh task | {e:#}"
+      )
+    })
+    .unwrap_or_default();
+  let Ok(deployments) = find_collect(&db_client().deployments, None, None)
+    .await
+    .inspect_err(|e| {
+      warn!(
+        "Failed to get Deployments from database in refresh task | {e:#}"
+      )
+    })
+  else {
+    return;
+  };
+  for deployment in deployments {
+    if deployment.config.poll_for_updates
+      || deployment.config.auto_update
+    {
+      if let Some(server) =
+        servers.iter().find(|s| s.id == deployment.config.server_id)
+      {
+        let name = deployment.name.clone();
+        if let Err(e) =
+          pull_deployment_inner(deployment, server).await
+        {
+          warn!("Failed to pull latest image for Deployment {name} | {e:#}");
+        }
+      }
+    }
+  }
+}
+
 async fn refresh_builds() {
   let Ok(builds) = find_collect(&db_client().builds, None, None)
     .await

bin/core/src/resource/repo.rs

@@ -11,6 +11,7 @@ use komodo_client::entities::{
   },
   resource::Resource,
   server::Server,
+  to_komodo_name,
   update::Update,
   user::User,
   Operation, ResourceTargetVariant,
@@ -43,8 +44,8 @@ impl super::KomodoResource for Repo {
     ResourceTargetVariant::Repo
   }
 
-  async fn coll(
-  ) -> &'static Collection<Resource<Self::Config, Self::Info>> {
+  fn coll() -> &'static Collection<Resource<Self::Config, Self::Info>>
+  {
     &db_client().repos
   }
 
@@ -132,6 +133,12 @@ impl super::KomodoResource for Repo {
     Ok(())
   }
 
+  // RENAME
+
+  fn rename_operation() -> Operation {
+    Operation::RenameRepo
+  }
+
   // DELETE
 
   fn delete_operation() -> Operation {
@@ -158,7 +165,11 @@ impl super::KomodoResource for Repo {
 
     match periphery
       .request(DeleteRepo {
-        name: repo.name.clone(),
+        name: if repo.config.path.is_empty() {
+          to_komodo_name(&repo.name)
+        } else {
+          repo.config.path.clone()
+        },
       })
       .await
     {
@@ -213,15 +224,19 @@ async fn validate_config(
         PermissionLevel::Write,
       )
       .await
-      .context("Cannot attach repo to this server. User must have write permissions on the server.")?;
+      .context("Cannot attach Repo to this Server")?;
       config.server_id = Some(server.id);
     }
   }
   if let Some(builder_id) = &config.builder_id {
     if !builder_id.is_empty() {
-      let builder = super::get_check_permissions::<Builder>(builder_id, user, PermissionLevel::Read)
-        .await
-        .context("Cannot attach repo to this builder. User must have at least read permissions on the builder.")?;
+      let builder = super::get_check_permissions::<Builder>(
+        builder_id,
+        user,
+        PermissionLevel::Read,
+      )
+      .await
+      .context("Cannot attach Repo to this Builder")?;
       config.builder_id = Some(builder.id);
     }
   }