Merge branch 'fix-issue-1022'

The GetLoginOptions response was cached in a OnceLock, meaning it was
computed once at startup and never updated. When the OIDC client was
re-initialized (e.g. after config changes), the login options endpoint
would still return the stale cached value, causing the UI to show
outdated OIDC button state.

Replace the static OnceLock cache with a function that reads the
current OIDC client state on each request, so the response always
reflects whether OIDC is currently available.

Also add a 30-second polling interval to the frontend's login options
query so the login page picks up server-side changes without requiring
a hard refresh.

Fixes #1022

.devcontainer/dev.compose.yaml

@@ -0,0 +1,33 @@
+services:
+  dev:
+    image: mcr.microsoft.com/devcontainers/rust:1-1-bullseye
+    volumes:
+      # Mount the root folder that contains .git
+      - ../:/workspace:cached
+      - /var/run/docker.sock:/var/run/docker.sock
+      - /proc:/proc
+      - repos:/etc/komodo/repos
+      - stacks:/etc/komodo/stacks
+    command: sleep infinity
+    ports:
+      - "9121:9121"
+    environment:
+      KOMODO_FIRST_SERVER: http://localhost:8120
+      KOMODO_DATABASE_ADDRESS: db
+      KOMODO_ENABLE_NEW_USERS: true
+      KOMODO_LOCAL_AUTH: true
+      KOMODO_JWT_SECRET: a_random_secret
+    links:
+      - db
+    # ...
+
+  db:
+    extends:
+      file: ../dev.compose.yaml
+      service: ferretdb
+
+volumes:
+  data:
+  repo-cache:
+  repos:
+  stacks:

.devcontainer/devcontainer.json

@@ -0,0 +1,46 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the
+// README at: https://github.com/devcontainers/templates/tree/main/src/rust
+{
+	"name": "Komodo",
+	// Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile
+	//"image": "mcr.microsoft.com/devcontainers/rust:1-1-bullseye",
+	"dockerComposeFile": ["dev.compose.yaml"],
+	"workspaceFolder": "/workspace",
+  	"service": "dev",
+	// Features to add to the dev container. More info: https://containers.dev/features.
+	"features": {
+		"ghcr.io/devcontainers/features/node:1": {
+			"version": "20.12.2"
+		},
+		"ghcr.io/devcontainers-community/features/deno:1": {
+
+		}
+	},
+
+	// Use 'mounts' to make the cargo cache persistent in a Docker Volume.
+	"mounts": [
+		{
+			"source": "devcontainer-cargo-cache-${devcontainerId}",
+			"target": "/usr/local/cargo",
+			"type": "volume"
+		}
+	],
+
+	// Use 'forwardPorts' to make a list of ports inside the container available locally.
+	"forwardPorts": [
+		9121
+	],
+
+	// Use 'postCreateCommand' to run commands after the container is created.
+	"postCreateCommand": "./.devcontainer/postCreate.sh",
+
+	"runServices": [
+		"db"
+	]
+
+	// Configure tool-specific properties.
+	// "customizations": {},
+
+	// Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root.
+	// "remoteUser": "root"
+}

.devcontainer/postCreate.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+cargo install typeshare-cli

.dockerignore

@@ -1,8 +0,0 @@
-/target
-readme.md
-typeshare.toml
-LICENSE
-*.code-workspace
-
-*/node_modules
-*/dist

.github/FUNDING.yml

@@ -0,0 +1,2 @@
+# https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/displaying-a-sponsor-button-in-your-repository
+open_collective: komodo

.gitignore

@@ -1,11 +1,13 @@
 target
-/frontend/build
 node_modules
-/lib/ts_client/build
 dist
+deno.lock
 .env
 .env.development
-creds.toml
-core.config.toml
-.syncs
-.stacks
+.DS_Store
+.idea
+
+/frontend/build
+/lib/ts_client/build
+
+.dev

.kminclude

@@ -0,0 +1 @@
+.dev

.vscode/extensions.json

@@ -0,0 +1,8 @@
+{
+    "recommendations": [
+        "rust-lang.rust-analyzer",
+        "tamasfe.even-better-toml",
+        "vadimcn.vscode-lldb",
+        "denoland.vscode-deno"
+    ]
+}

.vscode/tasks.json

@@ -1,93 +1,179 @@
-{
-	"version": "2.0.0",
-	"tasks": [
-		{
-			"type": "cargo",
-			"command": "build",
-			"group": {
-				"kind": "build",
-				"isDefault": true
-			},
-			"label": "rust: cargo build"
-		},
-		{
-			"type": "cargo",
-			"command": "fmt",
-			"label": "rust: cargo fmt"
-		},
-		{
-			"type": "cargo",
-			"command": "check",
-			"label": "rust: cargo check"
-		},
-		{
-			"label": "start dev",
-			"dependsOn": [
-				"run core",
-				"start frontend"
-			],
-			"problemMatcher": []
-		},
-		{
-			"type": "shell",
-			"command": "yarn start",
-			"label": "start frontend",
-			"options": {
-				"cwd": "${workspaceFolder}/frontend"
-			},
-			"presentation": {
-				"group": "start"
-			}
-		},
-		{
-			"type": "cargo",
-			"command": "run",
-			"label": "run core",
-			"options": {
-				"cwd": "${workspaceFolder}/bin/core"
-			},
-			"presentation": {
-				"group": "start"
-			}
-		},
-		{
-			"type": "cargo",
-			"command": "run",
-			"label": "run periphery",
-			"options": {
-				"cwd": "${workspaceFolder}/bin/periphery"
-			}
-		},
-		{
-			"type": "cargo",
-			"command": "run",
-			"label": "run tests",
-			"options": {
-				"cwd": "${workspaceFolder}/bin/tests"
-			}
-		},
-		{
-			"type": "cargo",
-			"command": "publish",
-			"args": ["--allow-dirty"],
-			"label": "publish types",
-			"options": {
-				"cwd": "${workspaceFolder}/lib/types"
-			}
-		},
-		{
-			"type": "cargo",
-			"command": "publish",
-			"label": "publish rs client",
-			"options": {
-				"cwd": "${workspaceFolder}/lib/rs_client"
-			}
-		},
-		{
-			"type": "shell",
-			"command": "node ./client/ts/generate_types.mjs",
-			"label": "generate typescript types",
-			"problemMatcher": []
-		}
-	]
-}
+{
+    "version": "2.0.0",
+    "tasks": [
+        {
+            "label": "Run Core",
+            "command": "cargo",
+            "args": [
+                "run",
+                "-p",
+                "komodo_core",
+                "--release"
+            ],
+            "options": {
+                "cwd": "${workspaceFolder}",
+                "env": {
+                    "KOMODO_CONFIG_PATH": "test.core.config.toml"
+                }
+            },
+            "problemMatcher": [
+                "$rustc"
+            ]
+        },
+        {
+            "label": "Build Core",
+            "command": "cargo",
+            "args": [
+                "build",
+                "-p",
+                "komodo_core",
+                "--release"
+            ],
+            "options": {
+                "cwd": "${workspaceFolder}",
+                "env": {
+                    "KOMODO_CONFIG_PATH": "test.core.config.toml"
+                }
+            },
+            "problemMatcher": [
+                "$rustc"
+            ]
+        },
+        {
+            "label": "Run Periphery",
+            "command": "cargo",
+            "args": [
+                "run",
+                "-p",
+                "komodo_periphery",
+                "--release"
+            ],
+            "options": {
+                "cwd": "${workspaceFolder}",
+                "env": {
+                    "KOMODO_CONFIG_PATH": "test.periphery.config.toml"
+                }
+            },
+            "problemMatcher": [
+                "$rustc"
+            ]
+        },
+        {
+            "label": "Build Periphery",
+            "command": "cargo",
+            "args": [
+                "build",
+                "-p",
+                "komodo_periphery",
+                "--release"
+            ],
+            "options": {
+                "cwd": "${workspaceFolder}",
+                "env": {
+                    "KOMODO_CONFIG_PATH": "test.periphery.config.toml"
+                }
+            },
+            "problemMatcher": [
+                "$rustc"
+            ]
+        },
+        {
+            "label": "Run Backend",
+            "dependsOn": [
+                "Run Core",
+                "Run Periphery"
+            ],
+            "problemMatcher": [
+                "$rustc"
+            ]
+        },
+        {
+            "label": "Build TS Client Types",
+            "type": "process",
+            "command": "node",
+            "args": [
+                "./client/core/ts/generate_types.mjs"
+            ],
+            "problemMatcher": []
+        },
+        {
+            "label": "Init TS Client",
+            "type": "shell",
+            "command": "yarn && yarn build && yarn link",
+            "options": {
+                "cwd": "${workspaceFolder}/client/core/ts",
+            },
+            "problemMatcher": []
+        },
+        {
+            "label": "Init Frontend Client",
+            "type": "shell",
+            "command": "yarn link komodo_client && yarn install",
+            "options": {
+                "cwd": "${workspaceFolder}/frontend",
+            },
+            "problemMatcher": []
+        },
+        {
+            "label": "Init Frontend",
+            "dependsOn": [
+                "Build TS Client Types",
+                "Init TS Client",
+                "Init Frontend Client"
+            ],
+            "dependsOrder": "sequence",
+            "problemMatcher": []
+        },
+        {
+            "label": "Build Frontend",
+            "type": "shell",
+            "command": "yarn build",
+            "options": {
+                "cwd": "${workspaceFolder}/frontend",
+            },
+            "problemMatcher": []
+        },
+        {
+            "label": "Prepare Frontend For Run",
+            "type": "shell",
+            "command": "cp -r ./client/core/ts/dist/. frontend/public/client/.",
+            "options": {
+                "cwd": "${workspaceFolder}",
+            },
+            "dependsOn": [
+                "Build TS Client Types",
+                "Build Frontend"
+            ],
+            "dependsOrder": "sequence",
+            "problemMatcher": []
+        },
+        {
+            "label": "Run Frontend",
+            "type": "shell",
+            "command": "yarn dev",
+            "options": {
+                "cwd": "${workspaceFolder}/frontend",
+            },
+            "dependsOn": ["Prepare Frontend For Run"],
+            "problemMatcher": []
+        },
+        {
+            "label": "Init",
+            "dependsOn": [
+                "Build Backend",
+                "Init Frontend"
+            ],
+            "dependsOrder": "sequence",
+            "problemMatcher": []
+        },
+        {
+            "label": "Run Komodo",
+            "dependsOn": [
+                "Run Core",
+                "Run Periphery",
+                "Run Frontend"
+            ],
+            "problemMatcher": []
+        },
+    ]
+  }

Cargo.toml

@@ -1,108 +1,139 @@
 [workspace]
 resolver = "2"
-members = ["bin/*", "lib/*", "client/core/rs", "client/periphery/rs"]
+members = [
+	"bin/*",
+	"lib/*",
+	"client/core/rs",
+	"client/periphery/rs",
+]
 
 [workspace.package]
-version = "1.13.4"
-edition = "2021"
+version = "1.19.5"
+edition = "2024"
 authors = ["mbecker20 <becker.maxh@gmail.com>"]
 license = "GPL-3.0-or-later"
-repository = "https://github.com/mbecker20/monitor"
-homepage = "https://docs.monitor.dev"
+repository = "https://github.com/moghtech/komodo"
+homepage = "https://komo.do"
 
-[patch.crates-io]
-monitor_client = { path = "client/core/rs" }
+[profile.release]
+strip = "debuginfo"
 
 [workspace.dependencies]
 # LOCAL
-monitor_client = "1.13.3"
+komodo_client = { path = "client/core/rs" }
 periphery_client = { path = "client/periphery/rs" }
+environment_file = { path = "lib/environment_file" }
+environment = { path = "lib/environment" }
+interpolate = { path = "lib/interpolate" }
 formatting = { path = "lib/formatting" }
+database = { path = "lib/database" }
+response = { path = "lib/response" }
 command = { path = "lib/command" }
+config = { path = "lib/config" }
 logger = { path = "lib/logger" }
+cache = { path = "lib/cache" }
 git = { path = "lib/git" }
 
 # MOGH
 run_command = { version = "0.0.6", features = ["async_tokio"] }
-serror = { version = "0.4.6", default-features = false }
-slack = { version = "0.1.0", package = "slack_client_rs" }
+serror = { version = "0.5.1", default-features = false }
+slack = { version = "0.4.0", package = "slack_client_rs", default-features = false, features = ["rustls"] }
 derive_default_builder = "0.1.8"
 derive_empty_traits = "0.1.0"
-merge_config_files = "0.1.5"
 async_timing_util = "1.0.0"
 partial_derive2 = "0.4.3"
 derive_variants = "1.0.0"
-mongo_indexed = "2.0.1"
-resolver_api = "1.1.1"
-toml_pretty = "1.1.2"
-mungos = "1.0.1"
-svi = "1.0.1"
+mongo_indexed = "2.0.2"
+resolver_api = "3.0.0"
+toml_pretty = "1.2.0"
+mungos = "3.2.2"
+svi = "1.2.0"
 
 # ASYNC
-tokio = { version = "1.39.2", features = ["full"] }
-reqwest = { version = "0.12.5", features = ["json"] }
-tokio-util = "0.7.11"
-futures = "0.3.30"
-futures-util = "0.3.30"
+reqwest = { version = "0.12.23", default-features = false, features = ["json", "stream", "rustls-tls-native-roots"] }
+tokio = { version = "1.47.1", features = ["full"] }
+tokio-util = { version = "0.7.16", features = ["io", "codec"] }
+tokio-stream = { version = "0.1.17", features = ["sync"] }
+pin-project-lite = "0.2.16"
+futures = "0.3.31"
+futures-util = "0.3.31"
+arc-swap = "1.7.1"
 
 # SERVER
-axum = { version = "0.7.5", features = ["ws", "json"] }
-axum-extra = { version = "0.9.3", features = ["typed-header"] }
-tower-http = { version = "0.5.2", features = ["fs", "cors"] }
-tokio-tungstenite = "0.23.1"
+tokio-tungstenite = { version = "0.27.0", features = ["rustls-tls-native-roots"] }
+axum-extra = { version = "0.10.1", features = ["typed-header"] }
+tower-http = { version = "0.6.6", features = ["fs", "cors"] }
+axum-server = { version = "0.7.2", features = ["tls-rustls"] }
+axum = { version = "0.8.4", features = ["ws", "json", "macros"] }
 
 # SER/DE
-ordered_hash_map = { version = "0.4.0", features = ["serde"] }
-serde = { version = "1.0.208", features = ["derive"] }
-strum = { version = "0.26.3", features = ["derive"] }
-serde_json = "1.0.125"
-serde_yaml = "0.9.34"
-toml = "0.8.19"
+ipnetwork = { version = "0.21.1", features = ["serde"] }
+indexmap = { version = "2.11.1", features = ["serde"] }
+serde = { version = "1.0.219", features = ["derive"] }
+strum = { version = "0.27.2", features = ["derive"] }
+bson = { version = "2.15.0" } # must keep in sync with mongodb version
+serde_yaml_ng = "0.10.0"
+serde_json = "1.0.145"
+serde_qs = "0.15.0"
+toml = "0.9.5"
 
 # ERROR
-anyhow = "1.0.86"
-thiserror = "1.0.63"
+anyhow = "1.0.99"
+thiserror = "2.0.16"
 
 # LOGGING
-opentelemetry_sdk = { version = "0.24.1", features = ["rt-tokio"] }
-tracing-subscriber = { version = "0.3.18", features = ["json"] }
-opentelemetry-semantic-conventions = "0.16.0"
-tracing-opentelemetry = "0.25.0"
-opentelemetry-otlp = "0.17.0"
-opentelemetry = "0.24.0"
-tracing = "0.1.40"
+opentelemetry-otlp = { version = "0.30.0", features = ["tls-roots", "reqwest-rustls"] }
+opentelemetry_sdk = { version = "0.30.0", features = ["rt-tokio"] }
+tracing-subscriber = { version = "0.3.20", features = ["json"] }
+opentelemetry-semantic-conventions = "0.30.0"
+tracing-opentelemetry = "0.31.0"
+opentelemetry = "0.30.0"
+tracing = "0.1.41"
 
 # CONFIG
-clap = { version = "4.5.16", features = ["derive"] }
+clap = { version = "4.5.47", features = ["derive"] }
 dotenvy = "0.15.7"
 envy = "0.4.2"
 
-# CRYPTO
-uuid = { version = "1.10.0", features = ["v4", "fast-rng", "serde"] }
+# CRYPTO / AUTH
+uuid = { version = "1.18.1", features = ["v4", "fast-rng", "serde"] }
+jsonwebtoken = { version = "9.3.1", default-features = false }
+openidconnect = "4.0.1"
 urlencoding = "2.1.3"
 nom_pem = "4.0.0"
-bcrypt = "0.15.1"
+bcrypt = "0.17.1"
 base64 = "0.22.1"
+rustls = "0.23.31"
 hmac = "0.12.1"
-sha2 = "0.10.8"
-rand = "0.8.5"
-jwt = "0.16.0"
+sha2 = "0.10.9"
+rand = "0.9.2"
 hex = "0.4.3"
 
 # SYSTEM
-bollard = "0.17.0"
-sysinfo = "0.31.2"
+portable-pty = "0.9.0"
+bollard = "0.19.2"
+sysinfo = "0.37.0"
 
 # CLOUD
-aws-config = "1.5.5"
-aws-sdk-ec2 = "1.66.0"
-aws-sdk-ecr = "1.40.0"
+aws-config = "1.8.6"
+aws-sdk-ec2 = "1.167.0"
+aws-credential-types = "1.2.6"
+
+## CRON
+english-to-cron = "0.1.6"
+chrono-tz = "0.10.4"
+chrono = "0.4.42"
+croner = "3.0.0"
 
 # MISC
-derive_builder = "0.20.0"
-typeshare = "1.0.3"
-octorust = "0.7.0"
-colored = "2.1.0"
-regex = "1.10.6"
-bson = "2.11.0"
-
+async-compression = { version = "0.4.30", features = ["tokio", "gzip"] }
+derive_builder = "0.20.2"
+comfy-table = "7.2.1"
+typeshare = "1.0.4"
+octorust = "0.10.0"
+dashmap = "6.1.0"
+wildcard = "0.3.0"
+colored = "3.0.0"
+regex = "1.11.2"
+bytes = "1.10.1"
+shell-escape = "0.1.5"

bin/alerter/README.md

@@ -1,4 +0,0 @@
-# Alerter
-
-This crate sets up a basic axum server that listens for incoming alert POSTs.
-It can be used as a monitor alerting endpoint, and serves as a template for other custom alerter implementations.

bin/binaries.Dockerfile

@@ -0,0 +1,32 @@
+## Builds the Komodo Core, Periphery, and Util binaries
+## for a specific architecture.
+
+FROM rust:1.89.0-bullseye AS builder
+RUN cargo install cargo-strip
+
+WORKDIR /builder
+COPY Cargo.toml Cargo.lock ./
+COPY ./lib ./lib
+COPY ./client/core/rs ./client/core/rs
+COPY ./client/periphery ./client/periphery
+COPY ./bin/core ./bin/core
+COPY ./bin/periphery ./bin/periphery
+COPY ./bin/cli ./bin/cli
+
+# Compile bin
+RUN \
+  cargo build -p komodo_core --release && \
+  cargo build -p komodo_periphery --release && \
+  cargo build -p komodo_cli --release && \
+  cargo strip
+
+# Copy just the binaries to scratch image
+FROM scratch
+
+COPY --from=builder /builder/target/release/core /core
+COPY --from=builder /builder/target/release/periphery /periphery
+COPY --from=builder /builder/target/release/km /km
+
+LABEL org.opencontainers.image.source=https://github.com/moghtech/komodo
+LABEL org.opencontainers.image.description="Komodo Binaries"
+LABEL org.opencontainers.image.licenses=GPL-3.0

bin/chef.binaries.Dockerfile

@@ -0,0 +1,36 @@
+## Builds the Komodo Core, Periphery, and Util binaries
+## for a specific architecture.
+
+## Uses chef for dependency caching to help speed up back-to-back builds.
+
+FROM lukemathwalker/cargo-chef:latest-rust-1.89.0-bullseye AS chef
+WORKDIR /builder
+
+# Plan just the RECIPE to see if things have changed
+FROM chef AS planner
+COPY . .
+RUN cargo chef prepare --recipe-path recipe.json
+
+FROM chef AS builder
+RUN cargo install cargo-strip
+COPY --from=planner /builder/recipe.json recipe.json
+# Build JUST dependencies - cached layer
+RUN cargo chef cook --release --recipe-path recipe.json
+# NOW copy again (this time into builder) and build app
+COPY . .
+RUN \
+  cargo build --release --bin core && \
+  cargo build --release --bin periphery && \
+  cargo build --release --bin km && \
+  cargo strip
+
+# Copy just the binaries to scratch image
+FROM scratch
+
+COPY --from=builder /builder/target/release/core /core
+COPY --from=builder /builder/target/release/periphery /periphery
+COPY --from=builder /builder/target/release/km /km
+
+LABEL org.opencontainers.image.source=https://github.com/moghtech/komodo
+LABEL org.opencontainers.image.description="Komodo Binaries"
+LABEL org.opencontainers.image.licenses=GPL-3.0

bin/cli/Cargo.toml

@@ -1,29 +1,36 @@
 [package]
-name = "monitor_cli"
-description = "Command line tool to sync monitor resources and execute file defined procedures"
+name = "komodo_cli"
+description = "Command line tool for Komodo"
 version.workspace = true
 edition.workspace = true
 authors.workspace = true
 license.workspace = true
-homepage.workspace = true
 repository.workspace = true
+homepage.workspace = true
 
 [[bin]]
-name = "monitor"
+name = "km"
 path = "src/main.rs"
 
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
-
 [dependencies]
 # local
-monitor_client.workspace = true
+environment_file.workspace = true
+komodo_client.workspace = true
+database.workspace = true
+config.workspace = true
+logger.workspace = true
 # external
-tracing-subscriber.workspace = true
-merge_config_files.workspace = true
-futures.workspace = true
+futures-util.workspace = true
+comfy-table.workspace = true
+serde_json.workspace = true
+serde_qs.workspace = true
+wildcard.workspace = true
 tracing.workspace = true
 colored.workspace = true
+dotenvy.workspace = true
 anyhow.workspace = true
+chrono.workspace = true
 tokio.workspace = true
 serde.workspace = true
 clap.workspace = true
+envy.workspace = true

bin/cli/README.md

@@ -1,11 +1,11 @@
-# Monitor CLI
+# Komodo CLI
 
-Monitor CLI is a tool to sync monitor resources and execute operations.
+Komodo CLI is a tool to execute actions on your Komodo instance from shell scripts.
 
 ## Install
 
 ```sh
-cargo install monitor_cli
+cargo install komodo_cli
 ```
 
 Note: On Ubuntu, also requires `apt install build-essential pkg-config libssl-dev`.
@@ -14,9 +14,9 @@ Note: On Ubuntu, also requires `apt install build-essential pkg-config libssl-de
 
 ### Credentials
 
-Configure a file `~/.config/monitor/creds.toml` file with contents:
+Configure a file `~/.config/komodo/creds.toml` file with contents:
 ```toml
-url = "https://your.monitor.address"
+url = "https://your.komodo.address"
 key = "YOUR-API-KEY"
 secret = "YOUR-API-SECRET"
 ```
@@ -25,52 +25,88 @@ Note. You can specify a different creds file by using `--creds ./other/path.toml
 You can also bypass using any file and pass the information using `--url`, `--key`, `--secret`:
 
 ```sh
-monitor --url "https://your.monitor.address" --key "YOUR-API-KEY" --secret "YOUR-API-SECRET" ...
+komodo --url "https://your.komodo.address" --key "YOUR-API-KEY" --secret "YOUR-API-SECRET" ...
 ```
 
 ### Run Executions
 
 ```sh
 # Triggers an example build
-monitor execute run-build test_build
+komodo execute run-build test_build
 ```
 
 #### Manual
+`komodo --help`
+```md
+Command line tool to execute Komodo actions
+
+Usage: komodo [OPTIONS] <COMMAND>
+
+Commands:
+  execute  Runs an execution
+  help     Print this message or the help of the given subcommand(s)
+
+Options:
+      --creds <CREDS>    The path to a creds file [default: /Users/max/.config/komodo/creds.toml]
+      --url <URL>        Pass url in args instead of creds file
+      --key <KEY>        Pass api key in args instead of creds file
+      --secret <SECRET>  Pass api secret in args instead of creds file
+  -y, --yes              Always continue on user confirmation prompts
+  -h, --help             Print help (see more with '--help')
+  -V, --version          Print version
+```
+
+`komodo execute --help`
 ```md
 Runs an execution
 
-Usage: monitor execute <COMMAND>
+Usage: komodo execute <COMMAND>
 
 Commands:
-  none                 The "null" execution. Does nothing
-  run-procedure        Runs the target procedure. Response: [Update]
-  run-build            Runs the target build. Response: [Update]
-  cancel-build         Cancels the target build. Only does anything if the build is `building` when called. Response: [Update]
-  deploy               Deploys the container for the target deployment. Response: [Update]
-  start-container      Starts the container for the target deployment. Response: [Update]
-  restart-container    Restarts the container for the target deployment. Response: [Update]
-  pause-container      Pauses the container for the target deployment. Response: [Update]
-  unpause-container    Unpauses the container for the target deployment. Response: [Update]
-  stop-container       Stops the container for the target deployment. Response: [Update]
-  remove-container     Stops and removes the container for the target deployment. Reponse: [Update]
-  clone-repo           Clones the target repo. Response: [Update]
-  pull-repo            Pulls the target repo. Response: [Update]
-  build-repo           Builds the target repo, using the attached builder. Response: [Update]
-  cancel-repo-build    Cancels the target repo build. Only does anything if the repo build is `building` when called. Response: [Update]
-  stop-all-containers  Stops all containers on the target server. Response: [Update]
-  prune-networks       Prunes the docker networks on the target server. Response: [Update]
-  prune-images         Prunes the docker images on the target server. Response: [Update]
-  prune-containers     Prunes the docker containers on the target server. Response: [Update]
-  run-sync             Runs the target resource sync. Response: [Update]
-  deploy-stack         Deploys the target stack. `docker compose up`. Response: [Update]
-  start-stack          Starts the target stack. `docker compose start`. Response: [Update]
-  restart-stack        Restarts the target stack. `docker compose restart`. Response: [Update]
-  pause-stack          Pauses the target stack. `docker compose pause`. Response: [Update]
-  unpause-stack        Unpauses the target stack. `docker compose unpause`. Response: [Update]
-  stop-stack           Starts the target stack. `docker compose stop`. Response: [Update]
-  destroy-stack        Destoys the target stack. `docker compose down`. Response: [Update]
-  sleep                
-  help                 Print this message or the help of the given subcommand(s)
+  none                    The "null" execution. Does nothing
+  run-procedure           Runs the target procedure. Response: [Update]
+  run-build               Runs the target build. Response: [Update]
+  cancel-build            Cancels the target build. Only does anything if the build is `building` when called. Response: [Update]
+  deploy                  Deploys the container for the target deployment. Response: [Update]
+  start-deployment        Starts the container for the target deployment. Response: [Update]
+  restart-deployment      Restarts the container for the target deployment. Response: [Update]
+  pause-deployment        Pauses the container for the target deployment. Response: [Update]
+  unpause-deployment      Unpauses the container for the target deployment. Response: [Update]
+  stop-deployment         Stops the container for the target deployment. Response: [Update]
+  destroy-deployment      Stops and destroys the container for the target deployment. Reponse: [Update]
+  clone-repo              Clones the target repo. Response: [Update]
+  pull-repo               Pulls the target repo. Response: [Update]
+  build-repo              Builds the target repo, using the attached builder. Response: [Update]
+  cancel-repo-build       Cancels the target repo build. Only does anything if the repo build is `building` when called. Response: [Update]
+  start-container         Starts the container on the target server. Response: [Update]
+  restart-container       Restarts the container on the target server. Response: [Update]
+  pause-container         Pauses the container on the target server. Response: [Update]
+  unpause-container       Unpauses the container on the target server. Response: [Update]
+  stop-container          Stops the container on the target server. Response: [Update]
+  destroy-container       Stops and destroys the container on the target server. Reponse: [Update]
+  start-all-containers    Starts all containers on the target server. Response: [Update]
+  restart-all-containers  Restarts all containers on the target server. Response: [Update]
+  pause-all-containers    Pauses all containers on the target server. Response: [Update]
+  unpause-all-containers  Unpauses all containers on the target server. Response: [Update]
+  stop-all-containers     Stops all containers on the target server. Response: [Update]
+  prune-containers        Prunes the docker containers on the target server. Response: [Update]
+  delete-network          Delete a docker network. Response: [Update]
+  prune-networks          Prunes the docker networks on the target server. Response: [Update]
+  delete-image            Delete a docker image. Response: [Update]
+  prune-images            Prunes the docker images on the target server. Response: [Update]
+  delete-volume           Delete a docker volume. Response: [Update]
+  prune-volumes           Prunes the docker volumes on the target server. Response: [Update]
+  prune-system            Prunes the docker system on the target server, including volumes. Response: [Update]
+  run-sync                Runs the target resource sync. Response: [Update]
+  deploy-stack            Deploys the target stack. `docker compose up`. Response: [Update]
+  start-stack             Starts the target stack. `docker compose start`. Response: [Update]
+  restart-stack           Restarts the target stack. `docker compose restart`. Response: [Update]
+  pause-stack             Pauses the target stack. `docker compose pause`. Response: [Update]
+  unpause-stack           Unpauses the target stack. `docker compose unpause`. Response: [Update]
+  stop-stack              Starts the target stack. `docker compose stop`. Response: [Update]
+  destroy-stack           Destoys the target stack. `docker compose down`. Response: [Update]
+  sleep                   
+  help                    Print this message or the help of the given subcommand(s)
 
 Options:
   -h, --help  Print help

bin/cli/aio.Dockerfile

@@ -0,0 +1,25 @@
+FROM rust:1.89.0-bullseye AS builder
+RUN cargo install cargo-strip
+
+WORKDIR /builder
+COPY Cargo.toml Cargo.lock ./
+COPY ./lib ./lib
+COPY ./client/core/rs ./client/core/rs
+COPY ./client/periphery ./client/periphery
+COPY ./bin/cli ./bin/cli
+
+# Compile bin
+RUN cargo build -p komodo_cli --release && cargo strip
+
+# Copy binaries to distroless base
+FROM gcr.io/distroless/cc
+
+COPY --from=builder /builder/target/release/km /usr/local/bin/km
+
+ENV KOMODO_CLI_CONFIG_PATHS="/config"
+
+CMD [ "km" ]
+
+LABEL org.opencontainers.image.source=https://github.com/moghtech/komodo
+LABEL org.opencontainers.image.description="Komodo CLI"
+LABEL org.opencontainers.image.licenses=GPL-3.0

bin/cli/docs/copy-database.md

@@ -0,0 +1,135 @@
+# Copy Database Utility
+
+Copy the Komodo database contents between running, mongo-compatible databases.
+Can be used to move between MongoDB / FerretDB, or upgrade from FerretDB v1 to v2.
+
+```yaml
+services:
+
+  copy_database:
+    image: ghcr.io/moghtech/komodo-cli
+    command: km database copy -y
+    environment:
+      KOMODO_DATABASE_URI: mongodb://${KOMODO_DB_USERNAME}:${KOMODO_DB_PASSWORD}@source:27017
+      KOMODO_DATABASE_DB_NAME: ${KOMODO_DATABASE_DB_NAME:-komodo}
+      KOMODO_CLI_DATABASE_TARGET_URI: mongodb://${KOMODO_DB_USERNAME}:${KOMODO_DB_PASSWORD}@target:27017
+      KOMODO_CLI_DATABASE_TARGET_DB_NAME: ${KOMODO_DATABASE_DB_NAME:-komodo}
+
+```
+
+## FerretDB v2 Update Guide
+
+Up to Komodo 1.17.5, users who wanted to use Postgres / Sqlite were instructed to deploy FerretDB v1.
+Now that v2 is out however, v1 will go largely unsupported. Users are recommended to migrate to v2 for
+the best performance and ongoing support / updates, however the internal data structures
+have changed and this cannot be done in-place. 
+
+Also note that FerretDB v2 no longer supports Sqlite, and only supports 
+a [customized Postgres distribution](https://docs.ferretdb.io/installation/documentdb/docker/).
+Nonetheless, it remains a solid option for hosts which [do not support mongo](https://github.com/moghtech/komodo/issues/59).
+
+Also note, the same basic process outlined below can also be used to move between MongoDB and FerretDB, just replace FerretDB v2
+with the database you wish to move to.
+
+### **Step 1**: *Add* the new database to the top of your existing Komodo compose file.
+
+**Don't forget to also add the new volumes.**
+
+```yaml
+## In Komodo compose.yaml
+services:
+  postgres2:
+    # Recommended: Pin to a specific version
+    # https://github.com/FerretDB/documentdb/pkgs/container/postgres-documentdb
+    image: ghcr.io/ferretdb/postgres-documentdb
+    labels:
+      komodo.skip: # Prevent Komodo from stopping with StopAllContainers
+    restart: unless-stopped
+    # ports:
+    #   - 5432:5432
+    volumes:
+      - postgres-data:/var/lib/postgresql/data
+    environment:
+      POSTGRES_USER: ${KOMODO_DB_USERNAME}
+      POSTGRES_PASSWORD: ${KOMODO_DB_PASSWORD}
+      POSTGRES_DB: postgres # Do not change
+
+  ferretdb2:
+    # Recommended: Pin to a specific version
+    # https://github.com/FerretDB/FerretDB/pkgs/container/ferretdb
+    image: ghcr.io/ferretdb/ferretdb
+    labels:
+      komodo.skip: # Prevent Komodo from stopping with StopAllContainers
+    restart: unless-stopped
+    depends_on:
+      - postgres2
+    # ports:
+    #   - 27017:27017
+    volumes:
+      - ferretdb-state:/state
+    environment:
+      FERRETDB_POSTGRESQL_URL: postgres://${KOMODO_DB_USERNAME}:${KOMODO_DB_PASSWORD}@postgres2:5432/postgres
+
+  ...(unchanged)
+
+volumes:
+  ...(unchanged)
+  postgres-data:
+  ferretdb-state:
+```
+
+### **Step 2**: *Add* the database copy utility to Komodo compose file.
+
+The SOURCE_URI points to the existing database, ie the old FerretDB v1, and it depends
+on whether it was deployed using Postgres or Sqlite. The example below uses the Postgres one,
+but if you use Sqlite it should just be something like `mongodb://ferretdb:27017`.
+
+```yaml
+## In Komodo compose.yaml
+services:
+  ...(new database)
+
+  copy_database:
+    image: ghcr.io/moghtech/komodo-cli
+    command: km database copy -y
+    environment:
+      KOMODO_DATABASE_URI: mongodb://${KOMODO_DB_USERNAME}:${KOMODO_DB_PASSWORD}@ferretdb:27017/${KOMODO_DATABASE_DB_NAME:-komodo}?authMechanism=PLAIN
+      KOMODO_DATABASE_DB_NAME: ${KOMODO_DATABASE_DB_NAME:-komodo}
+      KOMODO_CLI_DATABASE_TARGET_URI: mongodb://${KOMODO_DB_USERNAME}:${KOMODO_DB_PASSWORD}@ferretdb2:27017
+      KOMODO_CLI_DATABASE_TARGET_DB_NAME: ${KOMODO_DATABASE_DB_NAME:-komodo}
+
+  ...(unchanged)
+```
+
+### **Step 3**: *Compose Up* the new additions
+
+Run `docker compose -p komodo --env-file compose.env -f xxxxx.compose.yaml up -d`, filling in the name of your compose.yaml.
+This will start up both the old and new database, and copy the data to the new one.
+
+Wait a few moments for the `copy_database` service to finish. When it exits,
+confirm the logs show the data was moved successfully, and move on to the next step.
+
+### **Step 4**: Point Komodo Core to the new database
+
+In your Komodo compose.yaml, first *comment out* the `copy_database` service and old ferretdb v1 service/s.
+Then update the `core` service environment to point to `ferretdb2`.
+
+```yaml
+services:
+  ...
+
+  core:
+    ...(unchanged)
+    environment:
+      KOMODO_DATABASE_ADDRESS: ferretdb2:27017
+      KOMODO_DATABASE_USERNAME: ${KOMODO_DB_USERNAME}
+      KOMODO_DATABASE_PASSWORD: ${KOMODO_DB_PASSWORD}
+```
+
+### **Step 5**: Final *Compose Up*
+
+Repeat the same `docker compose` command as before to apply the changes, and then try navigating to your Komodo web page.
+If it works, congrats, **you are done**. You can clean up the compose file if you would like, removing the old volumes etc.
+
+If it does not work, check the logs for any obvious issues, and if necessary you can undo the previous steps
+to go back to using the previous database.

bin/cli/multi-arch.Dockerfile

@@ -0,0 +1,29 @@
+## Assumes the latest binaries for x86_64 and aarch64 are already built (by binaries.Dockerfile).
+## Since theres no heavy build here, QEMU multi-arch builds are fine for this image.
+
+ARG BINARIES_IMAGE=ghcr.io/moghtech/komodo-binaries:latest
+ARG X86_64_BINARIES=${BINARIES_IMAGE}-x86_64
+ARG AARCH64_BINARIES=${BINARIES_IMAGE}-aarch64
+
+# This is required to work with COPY --from
+FROM ${X86_64_BINARIES} AS x86_64
+FROM ${AARCH64_BINARIES} AS aarch64
+
+FROM debian:bullseye-slim
+
+WORKDIR /app
+
+## Copy both binaries initially, but only keep appropriate one for the TARGETPLATFORM.
+COPY --from=x86_64 /km /app/arch/linux/amd64
+COPY --from=aarch64 /km /app/arch/linux/arm64
+
+ARG TARGETPLATFORM
+RUN mv /app/arch/${TARGETPLATFORM} /usr/local/bin/km && rm -r /app/arch
+
+ENV KOMODO_CLI_CONFIG_PATHS="/config"
+
+CMD [ "km" ]
+
+LABEL org.opencontainers.image.source=https://github.com/moghtech/komodo
+LABEL org.opencontainers.image.description="Komodo CLI"
+LABEL org.opencontainers.image.licenses=GPL-3.0

bin/cli/runfile.toml

@@ -0,0 +1,4 @@
+[install-cli]
+alias = "ic"
+description = "installs the komodo-cli, available on the command line as 'km'"
+cmd = "cargo install --path ."

bin/cli/single-arch.Dockerfile

@@ -0,0 +1,18 @@
+## Assumes the latest binaries for the required arch are already built (by binaries.Dockerfile).
+
+ARG BINARIES_IMAGE=ghcr.io/moghtech/komodo-binaries:latest
+
+# This is required to work with COPY --from
+FROM ${BINARIES_IMAGE} AS binaries
+
+FROM gcr.io/distroless/cc
+
+COPY --from=binaries /km /usr/local/bin/km
+
+ENV KOMODO_CLI_CONFIG_PATHS="/config"
+
+CMD [ "km" ]
+
+LABEL org.opencontainers.image.source=https://github.com/moghtech/komodo
+LABEL org.opencontainers.image.description="Komodo CLI"
+LABEL org.opencontainers.image.licenses=GPL-3.0

bin/cli/src/args.rs

@@ -1,55 +0,0 @@
-use clap::{Parser, Subcommand};
-use monitor_client::api::execute::Execution;
-use serde::Deserialize;
-
-#[derive(Parser, Debug)]
-#[command(version, about, long_about = None)]
-pub struct CliArgs {
-  /// Sync or Exec
-  #[command(subcommand)]
-  pub command: Command,
-
-  /// The path to a creds file.
-  ///
-  /// Note: If each of `url`, `key` and `secret` are passed,
-  /// no file is required at this path.
-  #[arg(long, default_value_t = default_creds())]
-  pub creds: String,
-
-  /// Pass url in args instead of creds file
-  #[arg(long)]
-  pub url: Option<String>,
-  /// Pass api key in args instead of creds file
-  #[arg(long)]
-  pub key: Option<String>,
-  /// Pass api secret in args instead of creds file
-  #[arg(long)]
-  pub secret: Option<String>,
-
-  /// Always continue on user confirmation prompts.
-  #[arg(long, short, default_value_t = false)]
-  pub yes: bool,
-}
-
-fn default_creds() -> String {
-  let home =
-    std::env::var("HOME").unwrap_or_else(|_| String::from("/root"));
-  format!("{home}/.config/monitor/creds.toml")
-}
-
-#[derive(Debug, Clone, Subcommand)]
-pub enum Command {
-  /// Runs an execution
-  Execute {
-    #[command(subcommand)]
-    execution: Execution,
-  },
-  // Room for more
-}
-
-#[derive(Debug, Deserialize)]
-pub struct CredsFile {
-  pub url: String,
-  pub key: String,
-  pub secret: String,
-}

bin/cli/src/command/container.rs

@@ -0,0 +1,312 @@
+use std::collections::{HashMap, HashSet};
+
+use anyhow::Context;
+use colored::Colorize;
+use comfy_table::{Attribute, Cell, Color};
+use futures_util::{
+  FutureExt, TryStreamExt, stream::FuturesUnordered,
+};
+use komodo_client::{
+  api::read::{
+    InspectDockerContainer, ListAllDockerContainers, ListServers,
+  },
+  entities::{
+    config::cli::args::container::{
+      Container, ContainerCommand, InspectContainer,
+    },
+    docker::{
+      self,
+      container::{ContainerListItem, ContainerStateStatusEnum},
+    },
+  },
+};
+
+use crate::{
+  command::{
+    PrintTable, clamp_sha, matches_wildcards, parse_wildcards,
+    print_items,
+  },
+  config::cli_config,
+};
+
+pub async fn handle(container: &Container) -> anyhow::Result<()> {
+  match &container.command {
+    None => list_containers(container).await,
+    Some(ContainerCommand::Inspect(inspect)) => {
+      inspect_container(inspect).await
+    }
+  }
+}
+
+async fn list_containers(
+  Container {
+    all,
+    down,
+    links,
+    reverse,
+    containers: names,
+    images,
+    networks,
+    servers,
+    format,
+    command: _,
+  }: &Container,
+) -> anyhow::Result<()> {
+  let client = super::komodo_client().await?;
+  let (server_map, containers) = tokio::try_join!(
+    client
+      .read(ListServers::default())
+      .map(|res| res.map(|res| res
+        .into_iter()
+        .map(|s| (s.id.clone(), s))
+        .collect::<HashMap<_, _>>())),
+    client.read(ListAllDockerContainers {
+      servers: Default::default()
+    }),
+  )?;
+
+  // (Option<Server Name>, Container)
+  let containers = containers.into_iter().map(|c| {
+    let server = if let Some(server_id) = c.server_id.as_ref()
+      && let Some(server) = server_map.get(server_id)
+    {
+      server
+    } else {
+      return (None, c);
+    };
+    (Some(server.name.as_str()), c)
+  });
+
+  let names = parse_wildcards(names);
+  let servers = parse_wildcards(servers);
+  let images = parse_wildcards(images);
+  let networks = parse_wildcards(networks);
+
+  let mut containers = containers
+    .into_iter()
+    .filter(|(server_name, c)| {
+      let state_check = if *all {
+        true
+      } else if *down {
+        !matches!(c.state, ContainerStateStatusEnum::Running)
+      } else {
+        matches!(c.state, ContainerStateStatusEnum::Running)
+      };
+      let network_check = matches_wildcards(
+        &networks,
+        &c.network_mode
+          .as_deref()
+          .map(|n| vec![n])
+          .unwrap_or_default(),
+      ) || matches_wildcards(
+        &networks,
+        &c.networks.iter().map(String::as_str).collect::<Vec<_>>(),
+      );
+      state_check
+        && network_check
+        && matches_wildcards(&names, &[c.name.as_str()])
+        && matches_wildcards(
+          &servers,
+          &server_name
+            .as_deref()
+            .map(|i| vec![i])
+            .unwrap_or_default(),
+        )
+        && matches_wildcards(
+          &images,
+          &c.image.as_deref().map(|i| vec![i]).unwrap_or_default(),
+        )
+    })
+    .collect::<Vec<_>>();
+  containers.sort_by(|(a_s, a), (b_s, b)| {
+    a.state
+      .cmp(&b.state)
+      .then(a.name.cmp(&b.name))
+      .then(a_s.cmp(b_s))
+      .then(a.network_mode.cmp(&b.network_mode))
+      .then(a.image.cmp(&b.image))
+  });
+  if *reverse {
+    containers.reverse();
+  }
+  print_items(containers, *format, *links)?;
+  Ok(())
+}
+
+pub async fn inspect_container(
+  inspect: &InspectContainer,
+) -> anyhow::Result<()> {
+  let client = super::komodo_client().await?;
+  let (server_map, mut containers) = tokio::try_join!(
+    client
+      .read(ListServers::default())
+      .map(|res| res.map(|res| res
+        .into_iter()
+        .map(|s| (s.id.clone(), s))
+        .collect::<HashMap<_, _>>())),
+    client.read(ListAllDockerContainers {
+      servers: Default::default()
+    }),
+  )?;
+
+  containers.iter_mut().for_each(|c| {
+    let Some(server_id) = c.server_id.as_ref() else {
+      return;
+    };
+    let Some(server) = server_map.get(server_id) else {
+      c.server_id = Some(String::from("Unknown"));
+      return;
+    };
+    c.server_id = Some(server.name.clone());
+  });
+
+  let names = [inspect.container.to_string()];
+  let names = parse_wildcards(&names);
+  let servers = parse_wildcards(&inspect.servers);
+
+  let mut containers = containers
+    .into_iter()
+    .filter(|c| {
+      matches_wildcards(&names, &[c.name.as_str()])
+        && matches_wildcards(
+          &servers,
+          &c.server_id
+            .as_deref()
+            .map(|i| vec![i])
+            .unwrap_or_default(),
+        )
+    })
+    .map(|c| async move {
+      client
+        .read(InspectDockerContainer {
+          container: c.name,
+          server: c.server_id.context("No server...")?,
+        })
+        .await
+    })
+    .collect::<FuturesUnordered<_>>()
+    .try_collect::<Vec<_>>()
+    .await?;
+
+  containers.sort_by(|a, b| a.name.cmp(&b.name));
+
+  match containers.len() {
+    0 => {
+      println!(
+        "{}: Did not find any containers matching '{}'",
+        "INFO".green(),
+        inspect.container.bold()
+      );
+    }
+    1 => {
+      println!("{}", serialize_container(inspect, &containers[0])?);
+    }
+    _ => {
+      let containers = containers
+        .iter()
+        .map(|c| serialize_container(inspect, c))
+        .collect::<anyhow::Result<Vec<_>>>()?
+        .join("\n");
+      println!("{containers}");
+    }
+  }
+
+  Ok(())
+}
+
+fn serialize_container(
+  inspect: &InspectContainer,
+  container: &docker::container::Container,
+) -> anyhow::Result<String> {
+  let res = if inspect.state {
+    serde_json::to_string_pretty(&container.state)
+  } else if inspect.mounts {
+    serde_json::to_string_pretty(&container.mounts)
+  } else if inspect.host_config {
+    serde_json::to_string_pretty(&container.host_config)
+  } else if inspect.config {
+    serde_json::to_string_pretty(&container.config)
+  } else if inspect.network_settings {
+    serde_json::to_string_pretty(&container.network_settings)
+  } else {
+    serde_json::to_string_pretty(container)
+  }
+  .context("Failed to serialize items to JSON")?;
+  Ok(res)
+}
+
+// (Option<Server Name>, Container)
+impl PrintTable for (Option<&'_ str>, ContainerListItem) {
+  fn header(links: bool) -> &'static [&'static str] {
+    if links {
+      &[
+        "Container",
+        "State",
+        "Server",
+        "Ports",
+        "Networks",
+        "Image",
+        "Link",
+      ]
+    } else {
+      &["Container", "State", "Server", "Ports", "Networks", "Image"]
+    }
+  }
+  fn row(self, links: bool) -> Vec<Cell> {
+    let color = match self.1.state {
+      ContainerStateStatusEnum::Running => Color::Green,
+      ContainerStateStatusEnum::Paused => Color::DarkYellow,
+      ContainerStateStatusEnum::Empty => Color::Grey,
+      _ => Color::Red,
+    };
+    let mut networks = HashSet::new();
+    if let Some(network) = self.1.network_mode {
+      networks.insert(network);
+    }
+    for network in self.1.networks {
+      networks.insert(network);
+    }
+    let mut networks = networks.into_iter().collect::<Vec<_>>();
+    networks.sort();
+    let mut ports = self
+      .1
+      .ports
+      .into_iter()
+      .flat_map(|p| p.public_port.map(|p| p.to_string()))
+      .collect::<HashSet<_>>()
+      .into_iter()
+      .collect::<Vec<_>>();
+    ports.sort();
+    let ports = if ports.is_empty() {
+      Cell::new("")
+    } else {
+      Cell::new(format!(":{}", ports.join(", :")))
+    };
+
+    let image = self.1.image.as_deref().unwrap_or("Unknown");
+    let mut res = vec![
+      Cell::new(self.1.name.clone()).add_attribute(Attribute::Bold),
+      Cell::new(self.1.state.to_string())
+        .fg(color)
+        .add_attribute(Attribute::Bold),
+      Cell::new(self.0.unwrap_or("Unknown")),
+      ports,
+      Cell::new(networks.join(", ")),
+      Cell::new(clamp_sha(image)),
+    ];
+    if !links {
+      return res;
+    }
+    let link = if let Some(server_id) = self.1.server_id {
+      format!(
+        "{}/servers/{server_id}/container/{}",
+        cli_config().host,
+        self.1.name
+      )
+    } else {
+      String::new()
+    };
+    res.push(Cell::new(link));
+    res
+  }
+}

bin/cli/src/command/database.rs

@@ -0,0 +1,320 @@
+use std::path::Path;
+
+use anyhow::Context;
+use colored::Colorize;
+use komodo_client::entities::{
+  config::cli::args::database::DatabaseCommand, optional_string,
+};
+
+use crate::{command::sanitize_uri, config::cli_config};
+
+pub async fn handle(command: &DatabaseCommand) -> anyhow::Result<()> {
+  match command {
+    DatabaseCommand::Backup { yes, .. } => backup(*yes).await,
+    DatabaseCommand::Restore {
+      restore_folder,
+      index,
+      yes,
+      ..
+    } => restore(restore_folder.as_deref(), *index, *yes).await,
+    DatabaseCommand::Prune { yes, .. } => prune(*yes).await,
+    DatabaseCommand::Copy { yes, index, .. } => {
+      copy(*index, *yes).await
+    }
+  }
+}
+
+async fn backup(yes: bool) -> anyhow::Result<()> {
+  let config = cli_config();
+
+  println!(
+    "\n🦎  {} Database {} Utility  🦎",
+    "Komodo".bold(),
+    "Backup".green().bold()
+  );
+  println!(
+    "\n{}\n",
+    " - Backup all database contents to gzip compressed files."
+      .dimmed()
+  );
+  if let Some(uri) = optional_string(&config.database.uri) {
+    println!("{}: {}", " - Source URI".dimmed(), sanitize_uri(&uri));
+  }
+  if let Some(address) = optional_string(&config.database.address) {
+    println!("{}: {address}", " - Source Address".dimmed());
+  }
+  if let Some(username) = optional_string(&config.database.username) {
+    println!("{}: {username}", " - Source Username".dimmed());
+  }
+  println!(
+    "{}: {}\n",
+    " - Source Db Name".dimmed(),
+    config.database.db_name,
+  );
+  println!(
+    "{}: {:?}",
+    " - Backups Folder".dimmed(),
+    config.backups_folder
+  );
+  if config.max_backups == 0 {
+    println!(
+      "{}{}",
+      " - Backup pruning".dimmed(),
+      "disabled".red().dimmed()
+    );
+  } else {
+    println!("{}: {}", " - Max Backups".dimmed(), config.max_backups);
+  }
+
+  crate::command::wait_for_enter("start backup", yes)?;
+
+  let db = database::init(&config.database).await?;
+
+  database::utils::backup(&db, &config.backups_folder).await?;
+
+  // Early return if backup pruning disabled
+  if config.max_backups == 0 {
+    return Ok(());
+  }
+
+  // Know that new backup was taken successfully at this point,
+  // safe to prune old backup folders
+
+  prune_inner().await
+}
+
+async fn restore(
+  restore_folder: Option<&Path>,
+  index: bool,
+  yes: bool,
+) -> anyhow::Result<()> {
+  let config = cli_config();
+
+  println!(
+    "\n🦎  {} Database {} Utility  🦎",
+    "Komodo".bold(),
+    "Restore".purple().bold()
+  );
+  println!(
+    "\n{}\n",
+    " - Restores database contents from gzip compressed files."
+      .dimmed()
+  );
+  if let Some(uri) = optional_string(&config.database_target.uri) {
+    println!("{}: {}", " - Target URI".dimmed(), sanitize_uri(&uri));
+  }
+  if let Some(address) =
+    optional_string(&config.database_target.address)
+  {
+    println!("{}: {address}", " - Target Address".dimmed());
+  }
+  if let Some(username) =
+    optional_string(&config.database_target.username)
+  {
+    println!("{}: {username}", " - Target Username".dimmed());
+  }
+  println!(
+    "{}: {}",
+    " - Target Db Name".dimmed(),
+    config.database_target.db_name,
+  );
+  if !index {
+    println!(
+      "{}: {}",
+      " - Target Db Indexing".dimmed(),
+      "DISABLED".red(),
+    );
+  }
+  println!(
+    "\n{}: {:?}",
+    " - Backups Folder".dimmed(),
+    config.backups_folder
+  );
+  if let Some(restore_folder) = restore_folder {
+    println!("{}: {restore_folder:?}", " - Restore Folder".dimmed());
+  }
+
+  crate::command::wait_for_enter("start restore", yes)?;
+
+  let db = if index {
+    database::Client::new(&config.database_target).await?.db
+  } else {
+    database::init(&config.database_target).await?
+  };
+
+  database::utils::restore(
+    &db,
+    &config.backups_folder,
+    restore_folder,
+  )
+  .await
+}
+
+async fn prune(yes: bool) -> anyhow::Result<()> {
+  let config = cli_config();
+
+  println!(
+    "\n🦎  {} Database {} Utility  🦎",
+    "Komodo".bold(),
+    "Backup Prune".cyan().bold()
+  );
+  println!(
+    "\n{}\n",
+    " - Prunes database backup folders when greater than the configured amount."
+      .dimmed()
+  );
+  println!(
+    "{}: {:?}",
+    " - Backups Folder".dimmed(),
+    config.backups_folder
+  );
+  if config.max_backups == 0 {
+    println!(
+      "{}{}",
+      " - Backup pruning".dimmed(),
+      "disabled".red().dimmed()
+    );
+  } else {
+    println!("{}: {}", " - Max Backups".dimmed(), config.max_backups);
+  }
+
+  // Early return if backup pruning disabled
+  if config.max_backups == 0 {
+    info!(
+      "Backup pruning is disabled, enabled using 'max_backups' (KOMODO_CLI_MAX_BACKUPS)"
+    );
+    return Ok(());
+  }
+
+  crate::command::wait_for_enter("start backup prune", yes)?;
+
+  prune_inner().await
+}
+
+async fn prune_inner() -> anyhow::Result<()> {
+  let config = cli_config();
+
+  let mut backups_dir =
+    match tokio::fs::read_dir(&config.backups_folder)
+      .await
+      .context("Failed to read backups folder for prune")
+    {
+      Ok(backups_dir) => backups_dir,
+      Err(e) => {
+        warn!("{e:#}");
+        return Ok(());
+      }
+    };
+
+  let mut backup_folders = Vec::new();
+  loop {
+    match backups_dir.next_entry().await {
+      Ok(Some(entry)) => {
+        let Ok(metadata) = entry.metadata().await else {
+          continue;
+        };
+        if metadata.is_dir() {
+          backup_folders.push(entry.path());
+        }
+      }
+      Ok(None) => break,
+      Err(_) => {
+        continue;
+      }
+    }
+  }
+  // Ordered from oldest -> newest
+  backup_folders.sort();
+
+  let max_backups = config.max_backups as usize;
+  let backup_folders_len = backup_folders.len();
+
+  // Early return if under the backup count threshold
+  if backup_folders_len <= max_backups {
+    info!("No backups to prune");
+    return Ok(());
+  }
+
+  let to_delete =
+    &backup_folders[..(backup_folders_len - max_backups)];
+
+  info!("Pruning old backups: {to_delete:?}");
+
+  for path in to_delete {
+    if let Err(e) =
+      tokio::fs::remove_dir_all(path).await.with_context(|| {
+        format!("Failed to delete backup folder at {path:?}")
+      })
+    {
+      warn!("{e:#}");
+    }
+  }
+
+  Ok(())
+}
+
+async fn copy(index: bool, yes: bool) -> anyhow::Result<()> {
+  let config = cli_config();
+
+  println!(
+    "\n🦎  {} Database {} Utility  🦎",
+    "Komodo".bold(),
+    "Copy".blue().bold()
+  );
+  println!(
+    "\n{}\n",
+    " - Copies database contents to another database.".dimmed()
+  );
+
+  if let Some(uri) = optional_string(&config.database.uri) {
+    println!("{}: {}", " - Source URI".dimmed(), sanitize_uri(&uri));
+  }
+  if let Some(address) = optional_string(&config.database.address) {
+    println!("{}: {address}", " - Source Address".dimmed());
+  }
+  if let Some(username) = optional_string(&config.database.username) {
+    println!("{}: {username}", " - Source Username".dimmed());
+  }
+  println!(
+    "{}: {}\n",
+    " - Source Db Name".dimmed(),
+    config.database.db_name,
+  );
+
+  if let Some(uri) = optional_string(&config.database_target.uri) {
+    println!("{}: {}", " - Target URI".dimmed(), sanitize_uri(&uri));
+  }
+  if let Some(address) =
+    optional_string(&config.database_target.address)
+  {
+    println!("{}: {address}", " - Target Address".dimmed());
+  }
+  if let Some(username) =
+    optional_string(&config.database_target.username)
+  {
+    println!("{}: {username}", " - Target Username".dimmed());
+  }
+  println!(
+    "{}: {}",
+    " - Target Db Name".dimmed(),
+    config.database_target.db_name,
+  );
+  if !index {
+    println!(
+      "{}: {}",
+      " - Target Db Indexing".dimmed(),
+      "DISABLED".red(),
+    );
+  }
+
+  crate::command::wait_for_enter("start copy", yes)?;
+
+  let source_db = database::init(&config.database).await?;
+  let target_db = if index {
+    database::Client::new(&config.database_target).await?.db
+  } else {
+    database::init(&config.database_target).await?
+  };
+
+  database::utils::copy(&source_db, &target_db).await
+}

bin/cli/src/command/execute.rs

@@ -0,0 +1,572 @@
+use std::time::Duration;
+
+use colored::Colorize;
+use futures_util::{StreamExt, stream::FuturesUnordered};
+use komodo_client::{
+  api::execute::{
+    BatchExecutionResponse, BatchExecutionResponseItem, Execution,
+  },
+  entities::{resource_link, update::Update},
+};
+
+use crate::config::cli_config;
+
+enum ExecutionResult {
+  Single(Box<Update>),
+  Batch(BatchExecutionResponse),
+}
+
+pub async fn handle(
+  execution: &Execution,
+  yes: bool,
+) -> anyhow::Result<()> {
+  if matches!(execution, Execution::None(_)) {
+    println!("Got 'none' execution. Doing nothing...");
+    tokio::time::sleep(Duration::from_secs(3)).await;
+    println!("Finished doing nothing. Exiting...");
+    std::process::exit(0);
+  }
+
+  println!("\n{}: Execution", "Mode".dimmed());
+  match execution {
+    Execution::None(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
+    Execution::RunAction(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
+    Execution::BatchRunAction(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
+    Execution::RunProcedure(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
+    Execution::BatchRunProcedure(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
+    Execution::RunBuild(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
+    Execution::BatchRunBuild(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
+    Execution::CancelBuild(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
+    Execution::Deploy(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
+    Execution::BatchDeploy(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
+    Execution::PullDeployment(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
+    Execution::StartDeployment(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
+    Execution::RestartDeployment(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
+    Execution::PauseDeployment(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
+    Execution::UnpauseDeployment(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
+    Execution::StopDeployment(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
+    Execution::DestroyDeployment(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
+    Execution::BatchDestroyDeployment(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
+    Execution::CloneRepo(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
+    Execution::BatchCloneRepo(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
+    Execution::PullRepo(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
+    Execution::BatchPullRepo(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
+    Execution::BuildRepo(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
+    Execution::BatchBuildRepo(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
+    Execution::CancelRepoBuild(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
+    Execution::StartContainer(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
+    Execution::RestartContainer(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
+    Execution::PauseContainer(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
+    Execution::UnpauseContainer(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
+    Execution::StopContainer(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
+    Execution::DestroyContainer(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
+    Execution::StartAllContainers(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
+    Execution::RestartAllContainers(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
+    Execution::PauseAllContainers(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
+    Execution::UnpauseAllContainers(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
+    Execution::StopAllContainers(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
+    Execution::PruneContainers(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
+    Execution::DeleteNetwork(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
+    Execution::PruneNetworks(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
+    Execution::DeleteImage(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
+    Execution::PruneImages(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
+    Execution::DeleteVolume(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
+    Execution::PruneVolumes(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
+    Execution::PruneDockerBuilders(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
+    Execution::PruneBuildx(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
+    Execution::PruneSystem(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
+    Execution::RunSync(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
+    Execution::CommitSync(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
+    Execution::DeployStack(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
+    Execution::BatchDeployStack(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
+    Execution::DeployStackIfChanged(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
+    Execution::BatchDeployStackIfChanged(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
+    Execution::PullStack(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
+    Execution::BatchPullStack(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
+    Execution::StartStack(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
+    Execution::RestartStack(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
+    Execution::PauseStack(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
+    Execution::UnpauseStack(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
+    Execution::StopStack(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
+    Execution::DestroyStack(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
+    Execution::BatchDestroyStack(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
+    Execution::RunStackService(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
+    Execution::TestAlerter(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
+    Execution::SendAlert(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
+    Execution::ClearRepoCache(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
+    Execution::BackupCoreDatabase(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
+    Execution::GlobalAutoUpdate(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
+    Execution::Sleep(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
+  }
+
+  super::wait_for_enter("run execution", yes)?;
+
+  info!("Running Execution...");
+
+  let client = super::komodo_client().await?;
+
+  let res = match execution.clone() {
+    Execution::RunAction(request) => client
+      .execute(request)
+      .await
+      .map(|u| ExecutionResult::Single(u.into())),
+    Execution::BatchRunAction(request) => {
+      client.execute(request).await.map(ExecutionResult::Batch)
+    }
+    Execution::RunProcedure(request) => client
+      .execute(request)
+      .await
+      .map(|u| ExecutionResult::Single(u.into())),
+    Execution::BatchRunProcedure(request) => {
+      client.execute(request).await.map(ExecutionResult::Batch)
+    }
+    Execution::RunBuild(request) => client
+      .execute(request)
+      .await
+      .map(|u| ExecutionResult::Single(u.into())),
+    Execution::BatchRunBuild(request) => {
+      client.execute(request).await.map(ExecutionResult::Batch)
+    }
+    Execution::CancelBuild(request) => client
+      .execute(request)
+      .await
+      .map(|u| ExecutionResult::Single(u.into())),
+    Execution::Deploy(request) => client
+      .execute(request)
+      .await
+      .map(|u| ExecutionResult::Single(u.into())),
+    Execution::BatchDeploy(request) => {
+      client.execute(request).await.map(ExecutionResult::Batch)
+    }
+    Execution::PullDeployment(request) => client
+      .execute(request)
+      .await
+      .map(|u| ExecutionResult::Single(u.into())),
+    Execution::StartDeployment(request) => client
+      .execute(request)
+      .await
+      .map(|u| ExecutionResult::Single(u.into())),
+    Execution::RestartDeployment(request) => client
+      .execute(request)
+      .await
+      .map(|u| ExecutionResult::Single(u.into())),
+    Execution::PauseDeployment(request) => client
+      .execute(request)
+      .await
+      .map(|u| ExecutionResult::Single(u.into())),
+    Execution::UnpauseDeployment(request) => client
+      .execute(request)
+      .await
+      .map(|u| ExecutionResult::Single(u.into())),
+    Execution::StopDeployment(request) => client
+      .execute(request)
+      .await
+      .map(|u| ExecutionResult::Single(u.into())),
+    Execution::DestroyDeployment(request) => client
+      .execute(request)
+      .await
+      .map(|u| ExecutionResult::Single(u.into())),
+    Execution::BatchDestroyDeployment(request) => {
+      client.execute(request).await.map(ExecutionResult::Batch)
+    }
+    Execution::CloneRepo(request) => client
+      .execute(request)
+      .await
+      .map(|u| ExecutionResult::Single(u.into())),
+    Execution::BatchCloneRepo(request) => {
+      client.execute(request).await.map(ExecutionResult::Batch)
+    }
+    Execution::PullRepo(request) => client
+      .execute(request)
+      .await
+      .map(|u| ExecutionResult::Single(u.into())),
+    Execution::BatchPullRepo(request) => {
+      client.execute(request).await.map(ExecutionResult::Batch)
+    }
+    Execution::BuildRepo(request) => client
+      .execute(request)
+      .await
+      .map(|u| ExecutionResult::Single(u.into())),
+    Execution::BatchBuildRepo(request) => {
+      client.execute(request).await.map(ExecutionResult::Batch)
+    }
+    Execution::CancelRepoBuild(request) => client
+      .execute(request)
+      .await
+      .map(|u| ExecutionResult::Single(u.into())),
+    Execution::StartContainer(request) => client
+      .execute(request)
+      .await
+      .map(|u| ExecutionResult::Single(u.into())),
+    Execution::RestartContainer(request) => client
+      .execute(request)
+      .await
+      .map(|u| ExecutionResult::Single(u.into())),
+    Execution::PauseContainer(request) => client
+      .execute(request)
+      .await
+      .map(|u| ExecutionResult::Single(u.into())),
+    Execution::UnpauseContainer(request) => client
+      .execute(request)
+      .await
+      .map(|u| ExecutionResult::Single(u.into())),
+    Execution::StopContainer(request) => client
+      .execute(request)
+      .await
+      .map(|u| ExecutionResult::Single(u.into())),
+    Execution::DestroyContainer(request) => client
+      .execute(request)
+      .await
+      .map(|u| ExecutionResult::Single(u.into())),
+    Execution::StartAllContainers(request) => client
+      .execute(request)
+      .await
+      .map(|u| ExecutionResult::Single(u.into())),
+    Execution::RestartAllContainers(request) => client
+      .execute(request)
+      .await
+      .map(|u| ExecutionResult::Single(u.into())),
+    Execution::PauseAllContainers(request) => client
+      .execute(request)
+      .await
+      .map(|u| ExecutionResult::Single(u.into())),
+    Execution::UnpauseAllContainers(request) => client
+      .execute(request)
+      .await
+      .map(|u| ExecutionResult::Single(u.into())),
+    Execution::StopAllContainers(request) => client
+      .execute(request)
+      .await
+      .map(|u| ExecutionResult::Single(u.into())),
+    Execution::PruneContainers(request) => client
+      .execute(request)
+      .await
+      .map(|u| ExecutionResult::Single(u.into())),
+    Execution::DeleteNetwork(request) => client
+      .execute(request)
+      .await
+      .map(|u| ExecutionResult::Single(u.into())),
+    Execution::PruneNetworks(request) => client
+      .execute(request)
+      .await
+      .map(|u| ExecutionResult::Single(u.into())),
+    Execution::DeleteImage(request) => client
+      .execute(request)
+      .await
+      .map(|u| ExecutionResult::Single(u.into())),
+    Execution::PruneImages(request) => client
+      .execute(request)
+      .await
+      .map(|u| ExecutionResult::Single(u.into())),
+    Execution::DeleteVolume(request) => client
+      .execute(request)
+      .await
+      .map(|u| ExecutionResult::Single(u.into())),
+    Execution::PruneVolumes(request) => client
+      .execute(request)
+      .await
+      .map(|u| ExecutionResult::Single(u.into())),
+    Execution::PruneDockerBuilders(request) => client
+      .execute(request)
+      .await
+      .map(|u| ExecutionResult::Single(u.into())),
+    Execution::PruneBuildx(request) => client
+      .execute(request)
+      .await
+      .map(|u| ExecutionResult::Single(u.into())),
+    Execution::PruneSystem(request) => client
+      .execute(request)
+      .await
+      .map(|u| ExecutionResult::Single(u.into())),
+    Execution::RunSync(request) => client
+      .execute(request)
+      .await
+      .map(|u| ExecutionResult::Single(u.into())),
+    Execution::CommitSync(request) => client
+      .write(request)
+      .await
+      .map(|u| ExecutionResult::Single(u.into())),
+    Execution::DeployStack(request) => client
+      .execute(request)
+      .await
+      .map(|u| ExecutionResult::Single(u.into())),
+    Execution::BatchDeployStack(request) => {
+      client.execute(request).await.map(ExecutionResult::Batch)
+    }
+    Execution::DeployStackIfChanged(request) => client
+      .execute(request)
+      .await
+      .map(|u| ExecutionResult::Single(u.into())),
+    Execution::BatchDeployStackIfChanged(request) => {
+      client.execute(request).await.map(ExecutionResult::Batch)
+    }
+    Execution::PullStack(request) => client
+      .execute(request)
+      .await
+      .map(|u| ExecutionResult::Single(u.into())),
+    Execution::BatchPullStack(request) => {
+      client.execute(request).await.map(ExecutionResult::Batch)
+    }
+    Execution::StartStack(request) => client
+      .execute(request)
+      .await
+      .map(|u| ExecutionResult::Single(u.into())),
+    Execution::RestartStack(request) => client
+      .execute(request)
+      .await
+      .map(|u| ExecutionResult::Single(u.into())),
+    Execution::PauseStack(request) => client
+      .execute(request)
+      .await
+      .map(|u| ExecutionResult::Single(u.into())),
+    Execution::UnpauseStack(request) => client
+      .execute(request)
+      .await
+      .map(|u| ExecutionResult::Single(u.into())),
+    Execution::StopStack(request) => client
+      .execute(request)
+      .await
+      .map(|u| ExecutionResult::Single(u.into())),
+    Execution::DestroyStack(request) => client
+      .execute(request)
+      .await
+      .map(|u| ExecutionResult::Single(u.into())),
+    Execution::BatchDestroyStack(request) => {
+      client.execute(request).await.map(ExecutionResult::Batch)
+    }
+    Execution::RunStackService(request) => client
+      .execute(request)
+      .await
+      .map(|u| ExecutionResult::Single(u.into())),
+    Execution::TestAlerter(request) => client
+      .execute(request)
+      .await
+      .map(|u| ExecutionResult::Single(u.into())),
+    Execution::SendAlert(request) => client
+      .execute(request)
+      .await
+      .map(|u| ExecutionResult::Single(u.into())),
+    Execution::ClearRepoCache(request) => client
+      .execute(request)
+      .await
+      .map(|u| ExecutionResult::Single(u.into())),
+    Execution::BackupCoreDatabase(request) => client
+      .execute(request)
+      .await
+      .map(|u| ExecutionResult::Single(u.into())),
+    Execution::GlobalAutoUpdate(request) => client
+      .execute(request)
+      .await
+      .map(|u| ExecutionResult::Single(u.into())),
+    Execution::Sleep(request) => {
+      let duration =
+        Duration::from_millis(request.duration_ms as u64);
+      tokio::time::sleep(duration).await;
+      println!("Finished sleeping!");
+      std::process::exit(0)
+    }
+    Execution::None(_) => unreachable!(),
+  };
+
+  match res {
+    Ok(ExecutionResult::Single(update)) => {
+      poll_update_until_complete(&update).await
+    }
+    Ok(ExecutionResult::Batch(updates)) => {
+      let mut handles = updates
+        .iter()
+        .map(|update| async move {
+          match update {
+            BatchExecutionResponseItem::Ok(update) => {
+              poll_update_until_complete(update).await
+            }
+            BatchExecutionResponseItem::Err(e) => {
+              error!("{e:#?}");
+              Ok(())
+            }
+          }
+        })
+        .collect::<FuturesUnordered<_>>();
+      while let Some(res) = handles.next().await {
+        match res {
+          Ok(()) => {}
+          Err(e) => {
+            error!("{e:#?}");
+          }
+        }
+      }
+      Ok(())
+    }
+    Err(e) => {
+      error!("{e:#?}");
+      Ok(())
+    }
+  }
+}
+
+async fn poll_update_until_complete(
+  update: &Update,
+) -> anyhow::Result<()> {
+  let link = if update.id.is_empty() {
+    let (resource_type, id) = update.target.extract_variant_id();
+    resource_link(&cli_config().host, resource_type, id)
+  } else {
+    format!("{}/updates/{}", cli_config().host, update.id)
+  };
+  println!("Link: '{}'", link.bold());
+
+  let client = super::komodo_client().await?;
+
+  let timer = tokio::time::Instant::now();
+  let update = client.poll_update_until_complete(&update.id).await?;
+  if update.success {
+    println!(
+      "FINISHED in {}: {}",
+      format!("{:.1?}", timer.elapsed()).bold(),
+      "EXECUTION SUCCESSFUL".green(),
+    );
+  } else {
+    eprintln!(
+      "FINISHED in {}: {}",
+      format!("{:.1?}", timer.elapsed()).bold(),
+      "EXECUTION FAILED".red(),
+    );
+  }
+  Ok(())
+}

bin/cli/src/command/mod.rs

@@ -0,0 +1,181 @@
+use std::io::Read;
+
+use anyhow::{Context, anyhow};
+use chrono::TimeZone;
+use colored::Colorize;
+use comfy_table::{Attribute, Cell, Table};
+use komodo_client::{
+  KomodoClient,
+  entities::config::cli::{CliTableBorders, args::CliFormat},
+};
+use serde::Serialize;
+use tokio::sync::OnceCell;
+use wildcard::Wildcard;
+
+use crate::config::cli_config;
+
+pub mod container;
+pub mod database;
+pub mod execute;
+pub mod list;
+pub mod update;
+
+async fn komodo_client() -> anyhow::Result<&'static KomodoClient> {
+  static KOMODO_CLIENT: OnceCell<KomodoClient> =
+    OnceCell::const_new();
+  KOMODO_CLIENT
+    .get_or_try_init(|| async {
+      let config = cli_config();
+      let (Some(key), Some(secret)) =
+        (&config.cli_key, &config.cli_secret)
+      else {
+        return Err(anyhow!(
+          "Must provide both cli_key and cli_secret"
+        ));
+      };
+      KomodoClient::new(&config.host, key, secret)
+        .with_healthcheck()
+        .await
+    })
+    .await
+}
+
+fn wait_for_enter(
+  press_enter_to: &str,
+  skip: bool,
+) -> anyhow::Result<()> {
+  if skip {
+    println!();
+    return Ok(());
+  }
+  println!(
+    "\nPress {} to {}\n",
+    "ENTER".green(),
+    press_enter_to.bold()
+  );
+  let buffer = &mut [0u8];
+  std::io::stdin()
+    .read_exact(buffer)
+    .context("failed to read ENTER")?;
+  Ok(())
+}
+
+/// Sanitizes uris of the form:
+/// `protocol://username:password@address`
+fn sanitize_uri(uri: &str) -> String {
+  // protocol: `mongodb`
+  // credentials_address: `username:password@address`
+  let Some((protocol, credentials_address)) = uri.split_once("://")
+  else {
+    // If no protocol, return as-is
+    return uri.to_string();
+  };
+
+  // credentials: `username:password`
+  let Some((credentials, address)) =
+    credentials_address.split_once('@')
+  else {
+    // If no credentials, return as-is
+    return uri.to_string();
+  };
+
+  match credentials.split_once(':') {
+    Some((username, _)) => {
+      format!("{protocol}://{username}:*****@{address}")
+    }
+    None => {
+      format!("{protocol}://*****@{address}")
+    }
+  }
+}
+
+fn print_items<T: PrintTable + Serialize>(
+  items: Vec<T>,
+  format: CliFormat,
+  links: bool,
+) -> anyhow::Result<()> {
+  match format {
+    CliFormat::Table => {
+      let mut table = Table::new();
+      let preset = {
+        use comfy_table::presets::*;
+        match cli_config().table_borders {
+          None | Some(CliTableBorders::Horizontal) => {
+            UTF8_HORIZONTAL_ONLY
+          }
+          Some(CliTableBorders::Vertical) => UTF8_FULL_CONDENSED,
+          Some(CliTableBorders::Inside) => UTF8_NO_BORDERS,
+          Some(CliTableBorders::Outside) => UTF8_BORDERS_ONLY,
+          Some(CliTableBorders::All) => UTF8_FULL,
+        }
+      };
+      table.load_preset(preset).set_header(
+        T::header(links)
+          .iter()
+          .map(|h| Cell::new(h).add_attribute(Attribute::Bold)),
+      );
+      for item in items {
+        table.add_row(item.row(links));
+      }
+      println!("{table}");
+    }
+    CliFormat::Json => {
+      println!(
+        "{}",
+        serde_json::to_string_pretty(&items)
+          .context("Failed to serialize items to JSON")?
+      );
+    }
+  }
+  Ok(())
+}
+
+trait PrintTable {
+  fn header(links: bool) -> &'static [&'static str];
+  fn row(self, links: bool) -> Vec<Cell>;
+}
+
+fn parse_wildcards(items: &[String]) -> Vec<Wildcard<'_>> {
+  items
+    .iter()
+    .flat_map(|i| {
+      Wildcard::new(i.as_bytes()).inspect_err(|e| {
+        warn!("Failed to parse wildcard: {i} | {e:?}")
+      })
+    })
+    .collect::<Vec<_>>()
+}
+
+fn matches_wildcards(
+  wildcards: &[Wildcard<'_>],
+  items: &[&str],
+) -> bool {
+  if wildcards.is_empty() {
+    return true;
+  }
+  items.iter().any(|item| {
+    wildcards.iter().any(|wc| wc.is_match(item.as_bytes()))
+  })
+}
+
+fn format_timetamp(ts: i64) -> anyhow::Result<String> {
+  let ts = chrono::Local
+    .timestamp_millis_opt(ts)
+    .single()
+    .context("Invalid ts")?
+    .format("%m/%d %H:%M:%S")
+    .to_string();
+  Ok(ts)
+}
+
+fn clamp_sha(maybe_sha: &str) -> String {
+  if maybe_sha.starts_with("sha256:") {
+    maybe_sha[0..20].to_string() + "..."
+  } else {
+    maybe_sha.to_string()
+  }
+}
+
+// fn text_link(link: &str, text: &str) -> String {
+//   format!("\x1b]8;;{link}\x07{text}\x1b]8;;\x07")
+// }

bin/cli/src/command/update/mod.rs

@@ -0,0 +1,43 @@
+use komodo_client::entities::{
+  build::PartialBuildConfig,
+  config::cli::args::update::UpdateCommand,
+  deployment::PartialDeploymentConfig, repo::PartialRepoConfig,
+  server::PartialServerConfig, stack::PartialStackConfig,
+  sync::PartialResourceSyncConfig,
+};
+
+mod resource;
+mod user;
+mod variable;
+
+pub async fn handle(command: &UpdateCommand) -> anyhow::Result<()> {
+  match command {
+    UpdateCommand::Build(update) => {
+      resource::update::<PartialBuildConfig>(update).await
+    }
+    UpdateCommand::Deployment(update) => {
+      resource::update::<PartialDeploymentConfig>(update).await
+    }
+    UpdateCommand::Repo(update) => {
+      resource::update::<PartialRepoConfig>(update).await
+    }
+    UpdateCommand::Server(update) => {
+      resource::update::<PartialServerConfig>(update).await
+    }
+    UpdateCommand::Stack(update) => {
+      resource::update::<PartialStackConfig>(update).await
+    }
+    UpdateCommand::Sync(update) => {
+      resource::update::<PartialResourceSyncConfig>(update).await
+    }
+    UpdateCommand::Variable {
+      name,
+      value,
+      secret,
+      yes,
+    } => variable::update(name, value, *secret, *yes).await,
+    UpdateCommand::User { username, command } => {
+      user::update(username, command).await
+    }
+  }
+}

bin/cli/src/command/update/resource.rs

@@ -0,0 +1,152 @@
+use anyhow::Context;
+use colored::Colorize;
+use komodo_client::{
+  api::write::{
+    UpdateBuild, UpdateDeployment, UpdateRepo, UpdateResourceSync,
+    UpdateServer, UpdateStack,
+  },
+  entities::{
+    build::PartialBuildConfig,
+    config::cli::args::update::UpdateResource,
+    deployment::PartialDeploymentConfig, repo::PartialRepoConfig,
+    server::PartialServerConfig, stack::PartialStackConfig,
+    sync::PartialResourceSyncConfig,
+  },
+};
+use serde::{Serialize, de::DeserializeOwned};
+
+pub async fn update<
+  T: std::fmt::Debug + Serialize + DeserializeOwned + ResourceUpdate,
+>(
+  UpdateResource {
+    resource,
+    update,
+    yes,
+  }: &UpdateResource,
+) -> anyhow::Result<()> {
+  println!("\n{}: Update {}\n", "Mode".dimmed(), T::resource_type());
+  println!(" - {}: {resource}", "Name".dimmed());
+
+  let config = serde_qs::from_str::<T>(update)
+    .context("Failed to deserialize config")?;
+
+  match serde_json::to_string_pretty(&config) {
+    Ok(config) => {
+      println!(" - {}: {config}", "Update".dimmed());
+    }
+    Err(_) => {
+      println!(" - {}: {config:#?}", "Update".dimmed());
+    }
+  }
+
+  crate::command::wait_for_enter("update resource", *yes)?;
+
+  config.apply(resource).await
+}
+
+pub trait ResourceUpdate {
+  fn resource_type() -> &'static str;
+  async fn apply(self, resource: &str) -> anyhow::Result<()>;
+}
+
+impl ResourceUpdate for PartialBuildConfig {
+  fn resource_type() -> &'static str {
+    "Build"
+  }
+  async fn apply(self, resource: &str) -> anyhow::Result<()> {
+    let client = crate::command::komodo_client().await?;
+    client
+      .write(UpdateBuild {
+        id: resource.to_string(),
+        config: self,
+      })
+      .await
+      .context("Failed to update build config")?;
+    Ok(())
+  }
+}
+
+impl ResourceUpdate for PartialDeploymentConfig {
+  fn resource_type() -> &'static str {
+    "Deployment"
+  }
+  async fn apply(self, resource: &str) -> anyhow::Result<()> {
+    let client = crate::command::komodo_client().await?;
+    client
+      .write(UpdateDeployment {
+        id: resource.to_string(),
+        config: self,
+      })
+      .await
+      .context("Failed to update deployment config")?;
+    Ok(())
+  }
+}
+
+impl ResourceUpdate for PartialRepoConfig {
+  fn resource_type() -> &'static str {
+    "Repo"
+  }
+  async fn apply(self, resource: &str) -> anyhow::Result<()> {
+    let client = crate::command::komodo_client().await?;
+    client
+      .write(UpdateRepo {
+        id: resource.to_string(),
+        config: self,
+      })
+      .await
+      .context("Failed to update repo config")?;
+    Ok(())
+  }
+}
+
+impl ResourceUpdate for PartialServerConfig {
+  fn resource_type() -> &'static str {
+    "Server"
+  }
+  async fn apply(self, resource: &str) -> anyhow::Result<()> {
+    let client = crate::command::komodo_client().await?;
+    client
+      .write(UpdateServer {
+        id: resource.to_string(),
+        config: self,
+      })
+      .await
+      .context("Failed to update server config")?;
+    Ok(())
+  }
+}
+
+impl ResourceUpdate for PartialStackConfig {
+  fn resource_type() -> &'static str {
+    "Stack"
+  }
+  async fn apply(self, resource: &str) -> anyhow::Result<()> {
+    let client = crate::command::komodo_client().await?;
+    client
+      .write(UpdateStack {
+        id: resource.to_string(),
+        config: self,
+      })
+      .await
+      .context("Failed to update stack config")?;
+    Ok(())
+  }
+}
+
+impl ResourceUpdate for PartialResourceSyncConfig {
+  fn resource_type() -> &'static str {
+    "Sync"
+  }
+  async fn apply(self, resource: &str) -> anyhow::Result<()> {
+    let client = crate::command::komodo_client().await?;
+    client
+      .write(UpdateResourceSync {
+        id: resource.to_string(),
+        config: self,
+      })
+      .await
+      .context("Failed to update sync config")?;
+    Ok(())
+  }
+}

bin/cli/src/command/update/user.rs

@@ -0,0 +1,122 @@
+use anyhow::Context;
+use colored::Colorize;
+use database::mungos::mongodb::bson::doc;
+use komodo_client::entities::{
+  config::{
+    cli::args::{CliEnabled, update::UpdateUserCommand},
+    empty_or_redacted,
+  },
+  optional_string,
+};
+
+use crate::{command::sanitize_uri, config::cli_config};
+
+pub async fn update(
+  username: &str,
+  command: &UpdateUserCommand,
+) -> anyhow::Result<()> {
+  match command {
+    UpdateUserCommand::Password {
+      password,
+      unsanitized,
+      yes,
+    } => {
+      update_password(username, password, *unsanitized, *yes).await
+    }
+    UpdateUserCommand::SuperAdmin { enabled, yes } => {
+      update_super_admin(username, *enabled, *yes).await
+    }
+  }
+}
+
+async fn update_password(
+  username: &str,
+  password: &str,
+  unsanitized: bool,
+  yes: bool,
+) -> anyhow::Result<()> {
+  println!("\n{}: Update Password\n", "Mode".dimmed());
+  println!(" - {}: {username}", "Username".dimmed());
+  if unsanitized {
+    println!(" - {}: {password}", "Password".dimmed());
+  } else {
+    println!(
+      " - {}: {}",
+      "Password".dimmed(),
+      empty_or_redacted(password)
+    );
+  }
+
+  crate::command::wait_for_enter("update password", yes)?;
+
+  info!("Updating password...");
+
+  let db = database::Client::new(&cli_config().database).await?;
+
+  let user = db
+    .users
+    .find_one(doc! { "username": username })
+    .await
+    .context("Failed to query database for user")?
+    .context("No user found with given username")?;
+
+  db.set_user_password(&user, password).await?;
+
+  info!("Password updated ✅");
+
+  Ok(())
+}
+
+async fn update_super_admin(
+  username: &str,
+  super_admin: CliEnabled,
+  yes: bool,
+) -> anyhow::Result<()> {
+  let config = cli_config();
+
+  println!("\n{}: Update Super Admin\n", "Mode".dimmed());
+  println!(" - {}: {username}", "Username".dimmed());
+  println!(" - {}: {super_admin}\n", "Super Admin".dimmed());
+
+  if let Some(uri) = optional_string(&config.database.uri) {
+    println!("{}: {}", " - Source URI".dimmed(), sanitize_uri(&uri));
+  }
+  if let Some(address) = optional_string(&config.database.address) {
+    println!("{}: {address}", " - Source Address".dimmed());
+  }
+  if let Some(username) = optional_string(&config.database.username) {
+    println!("{}: {username}", " - Source Username".dimmed());
+  }
+  println!(
+    "{}: {}",
+    " - Source Db Name".dimmed(),
+    config.database.db_name,
+  );
+
+  crate::command::wait_for_enter("update super admin", yes)?;
+
+  info!("Updating super admin...");
+
+  let db = database::Client::new(&config.database).await?;
+
+  // Make sure the user exists first before saying it is successful.
+  let user = db
+    .users
+    .find_one(doc! { "username": username })
+    .await
+    .context("Failed to query database for user")?
+    .context("No user found with given username")?;
+
+  let super_admin: bool = super_admin.into();
+  db.users
+    .update_one(
+      doc! { "username": user.username },
+      doc! { "$set": { "super_admin": super_admin } },
+    )
+    .await
+    .context("Failed to update user super admin on db")?;
+
+  info!("Super admin updated ✅");
+
+  Ok(())
+}

bin/cli/src/command/update/variable.rs

@@ -0,0 +1,70 @@
+use anyhow::Context;
+use colored::Colorize;
+use komodo_client::api::{
+  read::GetVariable,
+  write::{
+    CreateVariable, UpdateVariableIsSecret, UpdateVariableValue,
+  },
+};
+
+pub async fn update(
+  name: &str,
+  value: &str,
+  secret: Option<bool>,
+  yes: bool,
+) -> anyhow::Result<()> {
+  println!("\n{}: Update Variable\n", "Mode".dimmed());
+  println!(" - {}:  {name}", "Name".dimmed());
+  println!(" - {}: {value}", "Value".dimmed());
+  if let Some(secret) = secret {
+    println!(" - {}: {secret}", "Is Secret".dimmed());
+  }
+
+  crate::command::wait_for_enter("update variable", yes)?;
+
+  let client = crate::command::komodo_client().await?;
+
+  let Ok(existing) = client
+    .read(GetVariable {
+      name: name.to_string(),
+    })
+    .await
+  else {
+    // Create the variable
+    client
+      .write(CreateVariable {
+        name: name.to_string(),
+        value: value.to_string(),
+        is_secret: secret.unwrap_or_default(),
+        description: Default::default(),
+      })
+      .await
+      .context("Failed to create variable")?;
+    info!("Variable created ✅");
+    return Ok(());
+  };
+
+  client
+    .write(UpdateVariableValue {
+      name: name.to_string(),
+      value: value.to_string(),
+    })
+    .await
+    .context("Failed to update variable 'value'")?;
+  info!("Variable 'value' updated ✅");
+
+  let Some(secret) = secret else { return Ok(()) };
+
+  if secret != existing.is_secret {
+    client
+      .write(UpdateVariableIsSecret {
+        name: name.to_string(),
+        is_secret: secret,
+      })
+      .await
+      .context("Failed to update variable 'is_secret'")?;
+    info!("Variable 'is_secret' updated to {secret} ✅");
+  }
+
+  Ok(())
+}

bin/cli/src/config.rs

@@ -0,0 +1,274 @@
+use std::{path::PathBuf, sync::OnceLock};
+
+use anyhow::Context;
+use clap::Parser;
+use colored::Colorize;
+use environment_file::maybe_read_item_from_file;
+use komodo_client::entities::{
+  config::{
+    DatabaseConfig,
+    cli::{
+      CliConfig, Env,
+      args::{CliArgs, Command, Execute, database::DatabaseCommand},
+    },
+  },
+  logger::LogConfig,
+};
+
+pub fn cli_args() -> &'static CliArgs {
+  static CLI_ARGS: OnceLock<CliArgs> = OnceLock::new();
+  CLI_ARGS.get_or_init(CliArgs::parse)
+}
+
+pub fn cli_env() -> &'static Env {
+  static CLI_ARGS: OnceLock<Env> = OnceLock::new();
+  CLI_ARGS.get_or_init(|| {
+    match envy::from_env()
+      .context("Failed to parse Komodo CLI environment")
+    {
+      Ok(env) => env,
+      Err(e) => {
+        panic!("{e:?}");
+      }
+    }
+  })
+}
+
+pub fn cli_config() -> &'static CliConfig {
+  static CLI_CONFIG: OnceLock<CliConfig> = OnceLock::new();
+  CLI_CONFIG.get_or_init(|| {
+    let args = cli_args();
+    let env = cli_env().clone();
+    let config_paths = args
+      .config_path
+      .clone()
+      .unwrap_or(env.komodo_cli_config_paths);
+    let debug_startup =
+      args.debug_startup.unwrap_or(env.komodo_cli_debug_startup);
+
+    if debug_startup {
+      println!(
+        "{}: Komodo CLI version: {}",
+        "DEBUG".cyan(),
+        env!("CARGO_PKG_VERSION").blue().bold()
+      );
+      println!(
+        "{}: {}: {config_paths:?}",
+        "DEBUG".cyan(),
+        "Config Paths".dimmed(),
+      );
+    }
+
+    let config_keywords = args
+      .config_keyword
+      .clone()
+      .unwrap_or(env.komodo_cli_config_keywords);
+    let config_keywords = config_keywords
+      .iter()
+      .map(String::as_str)
+      .collect::<Vec<_>>();
+    if debug_startup {
+      println!(
+        "{}: {}: {config_keywords:?}",
+        "DEBUG".cyan(),
+        "Config File Keywords".dimmed(),
+      );
+    }
+    let mut unparsed_config = (config::ConfigLoader {
+      paths: &config_paths
+        .iter()
+        .map(PathBuf::as_path)
+        .collect::<Vec<_>>(),
+      match_wildcards: &config_keywords,
+      include_file_name: ".kminclude",
+      merge_nested: env.komodo_cli_merge_nested_config,
+      extend_array: env.komodo_cli_extend_config_arrays,
+      debug_print: debug_startup,
+    })
+    .load::<serde_json::Map<String, serde_json::Value>>()
+    .expect("failed at parsing config from paths");
+    let init_parsed_config = serde_json::from_value::<CliConfig>(
+      serde_json::Value::Object(unparsed_config.clone()),
+    )
+    .context("Failed to parse config")
+    .unwrap();
+
+    let (host, key, secret) = match &args.command {
+      Command::Execute(Execute {
+        host, key, secret, ..
+      }) => (host.clone(), key.clone(), secret.clone()),
+      _ => (None, None, None),
+    };
+
+    let backups_folder = match &args.command {
+      Command::Database {
+        command: DatabaseCommand::Backup { backups_folder, .. },
+      } => backups_folder.clone(),
+      Command::Database {
+        command: DatabaseCommand::Restore { backups_folder, .. },
+      } => backups_folder.clone(),
+      _ => None,
+    };
+    let (uri, address, username, password, db_name) =
+      match &args.command {
+        Command::Database {
+          command:
+            DatabaseCommand::Copy {
+              uri,
+              address,
+              username,
+              password,
+              db_name,
+              ..
+            },
+        } => (
+          uri.clone(),
+          address.clone(),
+          username.clone(),
+          password.clone(),
+          db_name.clone(),
+        ),
+        _ => (None, None, None, None, None),
+      };
+
+    let profile = args
+      .profile
+      .as_ref()
+      .or(init_parsed_config.default_profile.as_ref());
+
+    let unparsed_config = if let Some(profile) = profile
+      && !profile.is_empty()
+    {
+      // Find the profile config,
+      // then merge it with the Default config.
+      let serde_json::Value::Array(profiles) = unparsed_config
+        .remove("profile")
+        .context("Config has no profiles, but a profile is required")
+        .unwrap()
+      else {
+        panic!("`config.profile` is not array");
+      };
+      let Some(profile_config) = profiles.into_iter().find(|p| {
+        let Ok(parsed) =
+          serde_json::from_value::<CliConfig>(p.clone())
+        else {
+          return false;
+        };
+        &parsed.config_profile == profile
+          || parsed
+            .config_aliases
+            .iter()
+            .any(|alias| alias == profile)
+      }) else {
+        panic!("No profile matching '{profile}' was found.");
+      };
+      let serde_json::Value::Object(profile_config) = profile_config
+      else {
+        panic!("Profile config is not Object type.");
+      };
+      config::merge_config(
+        unparsed_config,
+        profile_config.clone(),
+        env.komodo_cli_merge_nested_config,
+        env.komodo_cli_extend_config_arrays,
+      )
+      .unwrap_or(profile_config)
+    } else {
+      unparsed_config
+    };
+    let config = serde_json::from_value::<CliConfig>(
+      serde_json::Value::Object(unparsed_config),
+    )
+    .context("Failed to parse final config")
+    .unwrap();
+    let config_profile = if config.config_profile.is_empty() {
+      String::from("None")
+    } else {
+      config.config_profile
+    };
+
+    CliConfig {
+      config_profile,
+      config_aliases: config.config_aliases,
+      default_profile: config.default_profile,
+      table_borders: env
+        .komodo_cli_table_borders
+        .or(config.table_borders),
+      host: host
+        .or(env.komodo_cli_host)
+        .or(env.komodo_host)
+        .unwrap_or(config.host),
+      cli_key: key.or(env.komodo_cli_key).or(config.cli_key),
+      cli_secret: secret
+        .or(env.komodo_cli_secret)
+        .or(config.cli_secret),
+      backups_folder: backups_folder
+        .or(env.komodo_cli_backups_folder)
+        .unwrap_or(config.backups_folder),
+      max_backups: env
+        .komodo_cli_max_backups
+        .unwrap_or(config.max_backups),
+      database_target: DatabaseConfig {
+        uri: uri
+          .or(env.komodo_cli_database_target_uri)
+          .unwrap_or(config.database_target.uri),
+        address: address
+          .or(env.komodo_cli_database_target_address)
+          .unwrap_or(config.database_target.address),
+        username: username
+          .or(env.komodo_cli_database_target_username)
+          .unwrap_or(config.database_target.username),
+        password: password
+          .or(env.komodo_cli_database_target_password)
+          .unwrap_or(config.database_target.password),
+        db_name: db_name
+          .or(env.komodo_cli_database_target_db_name)
+          .unwrap_or(config.database_target.db_name),
+        app_name: config.database_target.app_name,
+      },
+      database: DatabaseConfig {
+        uri: maybe_read_item_from_file(
+          env.komodo_database_uri_file,
+          env.komodo_database_uri,
+        )
+        .unwrap_or(config.database.uri),
+        address: env
+          .komodo_database_address
+          .unwrap_or(config.database.address),
+        username: maybe_read_item_from_file(
+          env.komodo_database_username_file,
+          env.komodo_database_username,
+        )
+        .unwrap_or(config.database.username),
+        password: maybe_read_item_from_file(
+          env.komodo_database_password_file,
+          env.komodo_database_password,
+        )
+        .unwrap_or(config.database.password),
+        db_name: env
+          .komodo_database_db_name
+          .unwrap_or(config.database.db_name),
+        app_name: config.database.app_name,
+      },
+      cli_logging: LogConfig {
+        level: env
+          .komodo_cli_logging_level
+          .unwrap_or(config.cli_logging.level),
+        stdio: env
+          .komodo_cli_logging_stdio
+          .unwrap_or(config.cli_logging.stdio),
+        pretty: env
+          .komodo_cli_logging_pretty
+          .unwrap_or(config.cli_logging.pretty),
+        location: false,
+        otlp_endpoint: env
+          .komodo_cli_logging_otlp_endpoint
+          .unwrap_or(config.cli_logging.otlp_endpoint),
+        opentelemetry_service_name: env
+          .komodo_cli_logging_opentelemetry_service_name
+          .unwrap_or(config.cli_logging.opentelemetry_service_name),
+      },
+      profile: config.profile,
+    }
+  })
+}

bin/cli/src/exec.rs

@@ -1,208 +0,0 @@
-use std::time::Duration;
-
-use colored::Colorize;
-use monitor_client::api::execute::Execution;
-
-use crate::{
-  helpers::wait_for_enter,
-  state::{cli_args, monitor_client},
-};
-
-pub async fn run(execution: Execution) -> anyhow::Result<()> {
-  if matches!(execution, Execution::None(_)) {
-    println!("Got 'none' execution. Doing nothing...");
-    tokio::time::sleep(Duration::from_secs(3)).await;
-    println!("Finished doing nothing. Exiting...");
-    std::process::exit(0);
-  }
-
-  println!("\n{}: Execution", "Mode".dimmed());
-  match &execution {
-    Execution::None(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::RunProcedure(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::RunBuild(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::CancelBuild(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::Deploy(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::StartContainer(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::RestartContainer(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::PauseContainer(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::UnpauseContainer(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::StopContainer(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::StopAllContainers(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::RemoveContainer(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::CloneRepo(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::PullRepo(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::BuildRepo(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::CancelRepoBuild(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::PruneNetworks(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::PruneImages(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::PruneContainers(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::RunSync(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::DeployStack(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::StartStack(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::RestartStack(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::PauseStack(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::UnpauseStack(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::StopStack(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::DestroyStack(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-    Execution::Sleep(data) => {
-      println!("{}: {data:?}", "Data".dimmed())
-    }
-  }
-
-  if !cli_args().yes {
-    wait_for_enter("run execution")?;
-  }
-
-  info!("Running Execution...");
-
-  let res = match execution {
-    Execution::RunProcedure(request) => {
-      monitor_client().execute(request).await
-    }
-    Execution::RunBuild(request) => {
-      monitor_client().execute(request).await
-    }
-    Execution::CancelBuild(request) => {
-      monitor_client().execute(request).await
-    }
-    Execution::Deploy(request) => {
-      monitor_client().execute(request).await
-    }
-    Execution::StartContainer(request) => {
-      monitor_client().execute(request).await
-    }
-    Execution::RestartContainer(request) => {
-      monitor_client().execute(request).await
-    }
-    Execution::PauseContainer(request) => {
-      monitor_client().execute(request).await
-    }
-    Execution::UnpauseContainer(request) => {
-      monitor_client().execute(request).await
-    }
-    Execution::StopContainer(request) => {
-      monitor_client().execute(request).await
-    }
-    Execution::StopAllContainers(request) => {
-      monitor_client().execute(request).await
-    }
-    Execution::RemoveContainer(request) => {
-      monitor_client().execute(request).await
-    }
-    Execution::CloneRepo(request) => {
-      monitor_client().execute(request).await
-    }
-    Execution::PullRepo(request) => {
-      monitor_client().execute(request).await
-    }
-    Execution::BuildRepo(request) => {
-      monitor_client().execute(request).await
-    }
-    Execution::CancelRepoBuild(request) => {
-      monitor_client().execute(request).await
-    }
-    Execution::PruneNetworks(request) => {
-      monitor_client().execute(request).await
-    }
-    Execution::PruneImages(request) => {
-      monitor_client().execute(request).await
-    }
-    Execution::PruneContainers(request) => {
-      monitor_client().execute(request).await
-    }
-    Execution::RunSync(request) => {
-      monitor_client().execute(request).await
-    }
-    Execution::DeployStack(request) => {
-      monitor_client().execute(request).await
-    }
-    Execution::StartStack(request) => {
-      monitor_client().execute(request).await
-    }
-    Execution::RestartStack(request) => {
-      monitor_client().execute(request).await
-    }
-    Execution::PauseStack(request) => {
-      monitor_client().execute(request).await
-    }
-    Execution::UnpauseStack(request) => {
-      monitor_client().execute(request).await
-    }
-    Execution::StopStack(request) => {
-      monitor_client().execute(request).await
-    }
-    Execution::DestroyStack(request) => {
-      monitor_client().execute(request).await
-    }
-    Execution::Sleep(request) => {
-      let duration =
-        Duration::from_millis(request.duration_ms as u64);
-      tokio::time::sleep(duration).await;
-      println!("Finished sleeping!");
-      std::process::exit(0)
-    }
-    Execution::None(_) => unreachable!(),
-  };
-
-  match res {
-    Ok(update) => println!("\n{}: {update:#?}", "SUCCESS".green()),
-    Err(e) => println!("{}\n\n{e:#?}", "ERROR".red()),
-  }
-
-  Ok(())
-}

bin/cli/src/helpers.rs

@@ -1,17 +0,0 @@
-use std::io::Read;
-
-use anyhow::Context;
-use colored::Colorize;
-
-pub fn wait_for_enter(press_enter_to: &str) -> anyhow::Result<()> {
-  println!(
-    "\nPress {} to {}\n",
-    "ENTER".green(),
-    press_enter_to.bold()
-  );
-  let buffer = &mut [0u8];
-  std::io::stdin()
-    .read_exact(buffer)
-    .context("failed to read ENTER")?;
-  Ok(())
-}

bin/cli/src/main.rs

@@ -1,27 +1,72 @@
 #[macro_use]
 extern crate tracing;
 
-use colored::Colorize;
-use monitor_client::api::read::GetVersion;
+use anyhow::Context;
+use komodo_client::entities::config::cli::args;
 
-mod args;
-mod exec;
-mod helpers;
-mod state;
+use crate::config::cli_config;
+
+mod command;
+mod config;
+
+async fn app() -> anyhow::Result<()> {
+  dotenvy::dotenv().ok();
+  logger::init(&config::cli_config().cli_logging)?;
+  let args = config::cli_args();
+  let env = config::cli_env();
+  let debug_load =
+    args.debug_startup.unwrap_or(env.komodo_cli_debug_startup);
+
+  match &args.command {
+    args::Command::Config {
+      all_profiles,
+      unsanitized,
+    } => {
+      let mut config = if *unsanitized {
+        cli_config().clone()
+      } else {
+        cli_config().sanitized()
+      };
+      if !*all_profiles {
+        config.profile = Default::default();
+      }
+      if debug_load {
+        println!("\n{config:#?}");
+      } else {
+        println!(
+          "\nCLI Config {}",
+          serde_json::to_string_pretty(&config)
+            .context("Failed to serialize config for pretty print")?
+        );
+      }
+      Ok(())
+    }
+    args::Command::Container(container) => {
+      command::container::handle(container).await
+    }
+    args::Command::Inspect(inspect) => {
+      command::container::inspect_container(inspect).await
+    }
+    args::Command::List(list) => command::list::handle(list).await,
+    args::Command::Execute(args) => {
+      command::execute::handle(&args.execution, args.yes).await
+    }
+    args::Command::Update { command } => {
+      command::update::handle(command).await
+    }
+    args::Command::Database { command } => {
+      command::database::handle(command).await
+    }
+  }
+}
 
 #[tokio::main]
 async fn main() -> anyhow::Result<()> {
-  tracing_subscriber::fmt().with_target(false).init();
-
-  let version =
-    state::monitor_client().read(GetVersion {}).await?.version;
-  info!("monitor version: {}", version.to_string().blue().bold());
-
-  match &state::cli_args().command {
-    args::Command::Execute { execution } => {
-      exec::run(execution.to_owned()).await?
-    }
+  let mut term_signal = tokio::signal::unix::signal(
+    tokio::signal::unix::SignalKind::terminate(),
+  )?;
+  tokio::select! {
+    res = tokio::spawn(app()) => res?,
+    _ = term_signal.recv() => Ok(()),
   }
-
-  Ok(())
 }

bin/cli/src/state.rs

@@ -1,46 +0,0 @@
-use std::sync::OnceLock;
-
-use clap::Parser;
-use merge_config_files::parse_config_file;
-use monitor_client::MonitorClient;
-
-pub fn cli_args() -> &'static crate::args::CliArgs {
-  static CLI_ARGS: OnceLock<crate::args::CliArgs> = OnceLock::new();
-  CLI_ARGS.get_or_init(crate::args::CliArgs::parse)
-}
-
-pub fn monitor_client() -> &'static MonitorClient {
-  static MONITOR_CLIENT: OnceLock<MonitorClient> = OnceLock::new();
-  MONITOR_CLIENT.get_or_init(|| {
-    let args = cli_args();
-    let crate::args::CredsFile { url, key, secret } =
-      match (&args.url, &args.key, &args.secret) {
-        (Some(url), Some(key), Some(secret)) => {
-          crate::args::CredsFile {
-            url: url.clone(),
-            key: key.clone(),
-            secret: secret.clone(),
-          }
-        }
-        (url, key, secret) => {
-          let mut creds: crate::args::CredsFile =
-            parse_config_file(cli_args().creds.as_str())
-              .expect("failed to parse monitor credentials");
-
-          if let Some(url) = url {
-            creds.url.clone_from(url);
-          }
-          if let Some(key) = key {
-            creds.key.clone_from(key);
-          }
-          if let Some(secret) = secret {
-            creds.secret.clone_from(secret);
-          }
-
-          creds
-        }
-      };
-    futures::executor::block_on(MonitorClient::new(url, key, secret))
-      .expect("failed to initialize monitor client")
-  })
-}

bin/core/Cargo.toml

@@ -1,5 +1,5 @@
 [package]
-name = "monitor_core"
+name = "komodo_core"
 version.workspace = true
 edition.workspace = true
 authors.workspace = true
@@ -15,45 +15,61 @@ path = "src/main.rs"
 
 [dependencies]
 # local
-monitor_client = { workspace = true, features = ["mongo"] }
+komodo_client = { workspace = true, features = ["mongo"] }
 periphery_client.workspace = true
+environment_file.workspace = true
+interpolate.workspace = true
 formatting.workspace = true
+database.workspace = true
+response.workspace = true
+command.workspace = true
+config.workspace = true
 logger.workspace = true
+cache.workspace = true
 git.workspace = true
 # mogh
 serror = { workspace = true, features = ["axum"] }
-merge_config_files.workspace = true
 async_timing_util.workspace = true
 partial_derive2.workspace = true
 derive_variants.workspace = true
-mongo_indexed.workspace = true
 resolver_api.workspace = true
 toml_pretty.workspace = true
-run_command.workspace = true
-mungos.workspace = true
 slack.workspace = true
 svi.workspace = true
 # external
-ordered_hash_map.workspace = true
+aws-credential-types.workspace = true
+tokio-tungstenite.workspace = true
+english-to-cron.workspace = true
+openidconnect.workspace = true
+jsonwebtoken.workspace = true
+axum-server.workspace = true
 urlencoding.workspace = true
 aws-sdk-ec2.workspace = true
-aws-sdk-ecr.workspace = true
 aws-config.workspace = true
 tokio-util.workspace = true
 axum-extra.workspace = true
 tower-http.workspace = true
 serde_json.workspace = true
-serde_yaml.workspace = true
+serde_yaml_ng.workspace = true
 typeshare.workspace = true
+chrono-tz.workspace = true
+indexmap.workspace = true
 octorust.workspace = true
+wildcard.workspace = true
+arc-swap.workspace = true
+colored.workspace = true
+dashmap.workspace = true
 tracing.workspace = true
 reqwest.workspace = true
 futures.workspace = true
 nom_pem.workspace = true
-anyhow.workspace = true
 dotenvy.workspace = true
+anyhow.workspace = true
+croner.workspace = true
+chrono.workspace = true
 bcrypt.workspace = true
 base64.workspace = true
+rustls.workspace = true
 tokio.workspace = true
 serde.workspace = true
 regex.workspace = true
@@ -64,5 +80,4 @@ envy.workspace = true
 rand.workspace = true
 hmac.workspace = true
 sha2.workspace = true
-jwt.workspace = true
 hex.workspace = true

bin/core/Dockerfile

@@ -1,39 +0,0 @@
-# Build Core
-FROM rust:1.80.1-bookworm AS core-builder
-WORKDIR /builder
-COPY . .
-RUN cargo build -p monitor_core --release
-
-# Build Frontend
-FROM node:20.12-alpine AS frontend-builder
-WORKDIR /builder
-COPY ./frontend ./frontend
-COPY ./client/core/ts ./client
-RUN cd client && yarn && yarn build && yarn link
-RUN cd frontend && yarn link @monitor/client && yarn && yarn build
-
-# Final Image
-FROM debian:bookworm-slim
-
-# Install Deps
-RUN apt update && apt install -y git curl unzip ca-certificates && \
-	curl -SL https://github.com/docker/compose/releases/download/v2.29.1/docker-compose-linux-x86_64 -o /usr/local/bin/docker-compose && \
-	chmod +x /usr/local/bin/docker-compose && \
-	curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" && \
-	unzip awscliv2.zip && \
-	./aws/install
-	
-# Copy
-COPY ./config_example/core.config.example.toml /config/config.toml
-COPY --from=core-builder /builder/target/release/core /
-COPY --from=frontend-builder /builder/frontend/dist /frontend
-
-# Hint at the port
-EXPOSE 9000
-
-# Label for Ghcr
-LABEL org.opencontainers.image.source=https://github.com/mbecker20/monitor
-LABEL org.opencontainers.image.description="Monitor Core"
-LABEL org.opencontainers.image.licenses=GPL-3.0
-
-CMD ["./core"]

bin/core/aio.Dockerfile

@@ -0,0 +1,63 @@
+## All in one, multi stage compile + runtime Docker build for your architecture.
+
+# Build Core
+FROM rust:1.89.0-bullseye AS core-builder
+RUN cargo install cargo-strip
+
+WORKDIR /builder
+COPY Cargo.toml Cargo.lock ./
+COPY ./lib ./lib
+COPY ./client/core/rs ./client/core/rs
+COPY ./client/periphery ./client/periphery
+COPY ./bin/core ./bin/core
+COPY ./bin/cli ./bin/cli
+
+# Compile app
+RUN cargo build -p komodo_core --release && \
+  cargo build -p komodo_cli --release && \
+  cargo strip
+
+# Build Frontend
+FROM node:20.12-alpine AS frontend-builder
+WORKDIR /builder
+COPY ./frontend ./frontend
+COPY ./client/core/ts ./client
+RUN cd client && yarn && yarn build && yarn link
+RUN cd frontend && yarn link komodo_client && yarn && yarn build
+
+# Final Image
+FROM debian:bullseye-slim
+
+COPY ./bin/core/starship.toml /starship.toml
+COPY ./bin/core/debian-deps.sh .
+RUN sh ./debian-deps.sh && rm ./debian-deps.sh
+
+# Setup an application directory
+WORKDIR /app
+
+# Copy
+COPY ./config/core.config.toml /config/.default.config.toml
+COPY --from=frontend-builder /builder/frontend/dist /app/frontend
+COPY --from=core-builder /builder/target/release/core /usr/local/bin/core
+COPY --from=core-builder /builder/target/release/km /usr/local/bin/km
+COPY --from=denoland/deno:bin /deno /usr/local/bin/deno
+
+# Set $DENO_DIR and preload external Deno deps
+ENV DENO_DIR=/action-cache/deno
+RUN mkdir /action-cache && \
+  cd /action-cache && \
+  deno install jsr:@std/yaml jsr:@std/toml
+
+# Hint at the port
+EXPOSE 9120
+
+ENV KOMODO_CLI_CONFIG_PATHS="/config"
+# This ensures any `komodo.cli.*` takes precedence over the Core `/config/*config.*`
+ENV KOMODO_CLI_CONFIG_KEYWORDS="*config.*,*komodo.cli*.*"
+
+CMD [ "core" ]
+
+# Label for Ghcr
+LABEL org.opencontainers.image.source=https://github.com/moghtech/komodo
+LABEL org.opencontainers.image.description="Komodo Core"
+LABEL org.opencontainers.image.licenses=GPL-3.0

bin/core/debian-deps.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+## Core deps installer
+
+apt-get update
+apt-get install -y git curl ca-certificates iproute2
+
+rm -rf /var/lib/apt/lists/*
+
+# Starship prompt
+curl -sS https://starship.rs/install.sh | sh -s -- --yes --bin-dir /usr/local/bin
+echo 'export STARSHIP_CONFIG=/starship.toml' >> /root/.bashrc
+echo 'eval "$(starship init bash)"' >> /root/.bashrc
+

bin/core/multi-arch.Dockerfile

@@ -0,0 +1,59 @@
+## Assumes the latest binaries for x86_64 and aarch64 are already built (by binaries.Dockerfile).
+## Sets up the necessary runtime container dependencies for Komodo Core.
+## Since theres no heavy build here, QEMU multi-arch builds are fine for this image.
+
+ARG BINARIES_IMAGE=ghcr.io/moghtech/komodo-binaries:latest
+ARG FRONTEND_IMAGE=ghcr.io/moghtech/komodo-frontend:latest
+ARG X86_64_BINARIES=${BINARIES_IMAGE}-x86_64
+ARG AARCH64_BINARIES=${BINARIES_IMAGE}-aarch64
+
+# This is required to work with COPY --from
+FROM ${X86_64_BINARIES} AS x86_64
+FROM ${AARCH64_BINARIES} AS aarch64
+FROM ${FRONTEND_IMAGE} AS frontend
+
+# Final Image
+FROM debian:bullseye-slim
+
+COPY ./bin/core/starship.toml /starship.toml
+COPY ./bin/core/debian-deps.sh .
+RUN sh ./debian-deps.sh && rm ./debian-deps.sh
+
+WORKDIR /app
+
+ARG TARGETPLATFORM
+
+# Copy both binaries initially, but only keep appropriate one for the TARGETPLATFORM.
+COPY --from=x86_64 /core /app/core/linux/amd64
+COPY --from=aarch64 /core /app/core/linux/arm64
+RUN mv /app/core/${TARGETPLATFORM} /usr/local/bin/core && rm -r /app/core
+
+# Same for util
+COPY --from=x86_64 /km /app/km/linux/amd64
+COPY --from=aarch64 /km /app/km/linux/arm64
+RUN mv /app/km/${TARGETPLATFORM} /usr/local/bin/km && rm -r /app/km
+
+# Copy default config / static frontend / deno binary
+COPY ./config/core.config.toml /config/.default.config.toml
+COPY --from=frontend /frontend /app/frontend
+COPY --from=denoland/deno:bin /deno /usr/local/bin/deno
+
+# Set $DENO_DIR and preload external Deno deps
+ENV DENO_DIR=/action-cache/deno
+RUN mkdir /action-cache && \
+  cd /action-cache && \
+  deno install jsr:@std/yaml jsr:@std/toml
+
+# Hint at the port
+EXPOSE 9120
+
+ENV KOMODO_CLI_CONFIG_PATHS="/config"
+# This ensures any `komodo.cli.*` takes precedence over the Core `/config/*config.*`
+ENV KOMODO_CLI_CONFIG_KEYWORDS="*config.*,*komodo.cli*.*"
+
+CMD [ "core" ]
+
+# Label for Ghcr
+LABEL org.opencontainers.image.source=https://github.com/moghtech/komodo
+LABEL org.opencontainers.image.description="Komodo Core"
+LABEL org.opencontainers.image.licenses=GPL-3.0

bin/core/single-arch.Dockerfile

@@ -0,0 +1,48 @@
+## Assumes the latest binaries for the required arch are already built (by binaries.Dockerfile).
+## Sets up the necessary runtime container dependencies for Komodo Core.
+
+ARG BINARIES_IMAGE=ghcr.io/moghtech/komodo-binaries:latest
+
+# This is required to work with COPY --from
+FROM ${BINARIES_IMAGE} AS binaries
+
+# Build Frontend
+FROM node:20.12-alpine AS frontend-builder
+WORKDIR /builder
+COPY ./frontend ./frontend
+COPY ./client/core/ts ./client
+RUN cd client && yarn && yarn build && yarn link
+RUN cd frontend && yarn link komodo_client && yarn && yarn build
+
+FROM debian:bullseye-slim
+
+COPY ./bin/core/starship.toml /starship.toml
+COPY ./bin/core/debian-deps.sh .
+RUN sh ./debian-deps.sh && rm ./debian-deps.sh
+	
+# Copy
+COPY ./config/core.config.toml /config/.default.config.toml
+COPY --from=frontend-builder /builder/frontend/dist /app/frontend
+COPY --from=binaries /core /usr/local/bin/core
+COPY --from=binaries /km /usr/local/bin/km
+COPY --from=denoland/deno:bin /deno /usr/local/bin/deno
+
+# Set $DENO_DIR and preload external Deno deps
+ENV DENO_DIR=/action-cache/deno
+RUN mkdir /action-cache && \
+	cd /action-cache && \
+	deno install jsr:@std/yaml jsr:@std/toml
+
+# Hint at the port
+EXPOSE 9120
+
+ENV KOMODO_CLI_CONFIG_PATHS="/config"
+# This ensures any `komodo.cli.*` takes precedence over the Core `/config/*config.*`
+ENV KOMODO_CLI_CONFIG_KEYWORDS="*config.*,*komodo.cli*.*"
+
+CMD [ "core" ]
+
+# Label for Ghcr
+LABEL org.opencontainers.image.source=https://github.com/moghtech/komodo
+LABEL org.opencontainers.image.description="Komodo Core"
+LABEL org.opencontainers.image.licenses=GPL-3.0

bin/core/src/alert/discord.rs

@@ -0,0 +1,306 @@
+use std::sync::OnceLock;
+
+use serde::Serialize;
+
+use super::*;
+
+#[instrument(level = "debug")]
+pub async fn send_alert(
+  url: &str,
+  alert: &Alert,
+) -> anyhow::Result<()> {
+  let level = fmt_level(alert.level);
+  let content = match &alert.data {
+    AlertData::Test { id, name } => {
+      let link = resource_link(ResourceTargetVariant::Alerter, id);
+      format!(
+        "{level} | If you see this message, then Alerter **{name}** is **working**\n{link}"
+      )
+    }
+    AlertData::ServerVersionMismatch {
+      id,
+      name,
+      region,
+      server_version,
+      core_version,
+    } => {
+      let region = fmt_region(region);
+      let link = resource_link(ResourceTargetVariant::Server, id);
+      match alert.level {
+        SeverityLevel::Ok => {
+          format!(
+            "{level} | **{name}**{region} | Periphery version now matches Core version ✅\n{link}"
+          )
+        }
+        _ => {
+          format!(
+            "{level} | **{name}**{region} | Version mismatch detected ⚠️\nPeriphery: **{server_version}** | Core: **{core_version}**\n{link}"
+          )
+        }
+      }
+    }
+    AlertData::ServerUnreachable {
+      id,
+      name,
+      region,
+      err,
+    } => {
+      let region = fmt_region(region);
+      let link = resource_link(ResourceTargetVariant::Server, id);
+      match alert.level {
+        SeverityLevel::Ok => {
+          format!(
+            "{level} | **{name}**{region} is now **reachable**\n{link}"
+          )
+        }
+        SeverityLevel::Critical => {
+          let err = err
+            .as_ref()
+            .map(|e| format!("\n**error**: {e:#?}"))
+            .unwrap_or_default();
+          format!(
+            "{level} | **{name}**{region} is **unreachable** ❌\n{link}{err}"
+          )
+        }
+        _ => unreachable!(),
+      }
+    }
+    AlertData::ServerCpu {
+      id,
+      name,
+      region,
+      percentage,
+    } => {
+      let region = fmt_region(region);
+      let link = resource_link(ResourceTargetVariant::Server, id);
+      format!(
+        "{level} | **{name}**{region} cpu usage at **{percentage:.1}%**\n{link}"
+      )
+    }
+    AlertData::ServerMem {
+      id,
+      name,
+      region,
+      used_gb,
+      total_gb,
+    } => {
+      let region = fmt_region(region);
+      let link = resource_link(ResourceTargetVariant::Server, id);
+      let percentage = 100.0 * used_gb / total_gb;
+      format!(
+        "{level} | **{name}**{region} memory usage at **{percentage:.1}%** 💾\n\nUsing **{used_gb:.1} GiB** / **{total_gb:.1} GiB**\n{link}"
+      )
+    }
+    AlertData::ServerDisk {
+      id,
+      name,
+      region,
+      path,
+      used_gb,
+      total_gb,
+    } => {
+      let region = fmt_region(region);
+      let link = resource_link(ResourceTargetVariant::Server, id);
+      let percentage = 100.0 * used_gb / total_gb;
+      format!(
+        "{level} | **{name}**{region} disk usage at **{percentage:.1}%** 💿\nmount point: `{path:?}`\nusing **{used_gb:.1} GiB** / **{total_gb:.1} GiB**\n{link}"
+      )
+    }
+    AlertData::ContainerStateChange {
+      id,
+      name,
+      server_id: _server_id,
+      server_name,
+      from,
+      to,
+    } => {
+      let link = resource_link(ResourceTargetVariant::Deployment, id);
+      let to = fmt_docker_container_state(to);
+      format!(
+        "📦 Deployment **{name}** is now **{to}**\nserver: **{server_name}**\nprevious: **{from}**\n{link}"
+      )
+    }
+    AlertData::DeploymentImageUpdateAvailable {
+      id,
+      name,
+      server_id: _server_id,
+      server_name,
+      image,
+    } => {
+      let link = resource_link(ResourceTargetVariant::Deployment, id);
+      format!(
+        "⬆ Deployment **{name}** has an update available\nserver: **{server_name}**\nimage: **{image}**\n{link}"
+      )
+    }
+    AlertData::DeploymentAutoUpdated {
+      id,
+      name,
+      server_id: _server_id,
+      server_name,
+      image,
+    } => {
+      let link = resource_link(ResourceTargetVariant::Deployment, id);
+      format!(
+        "⬆ Deployment **{name}** was updated automatically ⏫\nserver: **{server_name}**\nimage: **{image}**\n{link}"
+      )
+    }
+    AlertData::StackStateChange {
+      id,
+      name,
+      server_id: _server_id,
+      server_name,
+      from,
+      to,
+    } => {
+      let link = resource_link(ResourceTargetVariant::Stack, id);
+      let to = fmt_stack_state(to);
+      format!(
+        "🥞 Stack **{name}** is now {to}\nserver: **{server_name}**\nprevious: **{from}**\n{link}"
+      )
+    }
+    AlertData::StackImageUpdateAvailable {
+      id,
+      name,
+      server_id: _server_id,
+      server_name,
+      service,
+      image,
+    } => {
+      let link = resource_link(ResourceTargetVariant::Stack, id);
+      format!(
+        "⬆ Stack **{name}** has an update available\nserver: **{server_name}**\nservice: **{service}**\nimage: **{image}**\n{link}"
+      )
+    }
+    AlertData::StackAutoUpdated {
+      id,
+      name,
+      server_id: _server_id,
+      server_name,
+      images,
+    } => {
+      let link = resource_link(ResourceTargetVariant::Stack, id);
+      let images_label =
+        if images.len() > 1 { "images" } else { "image" };
+      let images = images.join(", ");
+      format!(
+        "⬆ Stack **{name}** was updated automatically ⏫\nserver: **{server_name}**\n{images_label}: **{images}**\n{link}"
+      )
+    }
+    AlertData::AwsBuilderTerminationFailed {
+      instance_id,
+      message,
+    } => {
+      format!(
+        "{level} | Failed to terminated AWS builder instance\ninstance id: **{instance_id}**\n{message}"
+      )
+    }
+    AlertData::ResourceSyncPendingUpdates { id, name } => {
+      let link =
+        resource_link(ResourceTargetVariant::ResourceSync, id);
+      format!(
+        "{level} | Pending resource sync updates on **{name}**\n{link}"
+      )
+    }
+    AlertData::BuildFailed { id, name, version } => {
+      let link = resource_link(ResourceTargetVariant::Build, id);
+      format!(
+        "{level} | Build **{name}** failed\nversion: **v{version}**\n{link}"
+      )
+    }
+    AlertData::RepoBuildFailed { id, name } => {
+      let link = resource_link(ResourceTargetVariant::Repo, id);
+      format!("{level} | Repo build for **{name}** failed\n{link}")
+    }
+    AlertData::ProcedureFailed { id, name } => {
+      let link = resource_link(ResourceTargetVariant::Procedure, id);
+      format!("{level} | Procedure **{name}** failed\n{link}")
+    }
+    AlertData::ActionFailed { id, name } => {
+      let link = resource_link(ResourceTargetVariant::Action, id);
+      format!("{level} | Action **{name}** failed\n{link}")
+    }
+    AlertData::ScheduleRun {
+      resource_type,
+      id,
+      name,
+    } => {
+      let link = resource_link(*resource_type, id);
+      format!(
+        "{level} | **{name}** ({resource_type}) | Scheduled run started 🕝\n{link}"
+      )
+    }
+    AlertData::Custom { message, details } => {
+      format!(
+        "{level} | {message}{}",
+        if details.is_empty() {
+          format_args!("")
+        } else {
+          format_args!("\n{details}")
+        }
+      )
+    }
+    AlertData::None {} => Default::default(),
+  };
+  if !content.is_empty() {
+    let VariablesAndSecrets { variables, secrets } =
+      get_variables_and_secrets().await?;
+    let mut url_interpolated = url.to_string();
+
+    let mut interpolator =
+      Interpolator::new(Some(&variables), &secrets);
+
+    interpolator.interpolate_string(&mut url_interpolated)?;
+
+    send_message(&url_interpolated, &content)
+      .await
+      .map_err(|e| {
+        let replacers = interpolator
+          .secret_replacers
+          .into_iter()
+          .collect::<Vec<_>>();
+        let sanitized_error =
+          svi::replace_in_string(&format!("{e:?}"), &replacers);
+        anyhow::Error::msg(format!(
+          "Error with slack request: {sanitized_error}"
+        ))
+      })?;
+  }
+  Ok(())
+}
+
+async fn send_message(
+  url: &str,
+  content: &str,
+) -> anyhow::Result<()> {
+  let body = DiscordMessageBody { content };
+
+  let response = http_client()
+    .post(url)
+    .json(&body)
+    .send()
+    .await
+    .context("Failed to send message")?;
+
+  let status = response.status();
+
+  if status.is_success() {
+    Ok(())
+  } else {
+    let text = response.text().await.with_context(|| {
+      format!("Failed to send message to Discord | {status} | failed to get response text")
+    })?;
+    Err(anyhow::anyhow!(
+      "Failed to send message to Discord | {status} | {text}"
+    ))
+  }
+}
+
+fn http_client() -> &'static reqwest::Client {
+  static CLIENT: OnceLock<reqwest::Client> = OnceLock::new();
+  CLIENT.get_or_init(reqwest::Client::new)
+}
+
+#[derive(Serialize)]
+struct DiscordMessageBody<'a> {
+  content: &'a str,
+}

bin/core/src/alert/mod.rs

@@ -0,0 +1,488 @@
+use ::slack::types::Block;
+use anyhow::{Context, anyhow};
+use database::mungos::{find::find_collect, mongodb::bson::doc};
+use derive_variants::ExtractVariant;
+use futures::future::join_all;
+use interpolate::Interpolator;
+use komodo_client::entities::{
+  ResourceTargetVariant,
+  alert::{Alert, AlertData, AlertDataVariant, SeverityLevel},
+  alerter::*,
+  deployment::DeploymentState,
+  komodo_timestamp,
+  stack::StackState,
+};
+use tracing::Instrument;
+
+use crate::helpers::query::get_variables_and_secrets;
+use crate::helpers::{
+  maintenance::is_in_maintenance, query::VariablesAndSecrets,
+};
+use crate::{config::core_config, state::db_client};
+
+mod discord;
+mod ntfy;
+mod pushover;
+mod slack;
+
+#[instrument(level = "debug")]
+pub async fn send_alerts(alerts: &[Alert]) {
+  if alerts.is_empty() {
+    return;
+  }
+
+  let span =
+    info_span!("send_alerts", alerts = format!("{alerts:?}"));
+  async {
+    let Ok(alerters) = find_collect(
+      &db_client().alerters,
+      doc! { "config.enabled": true },
+      None,
+    )
+    .await
+    .inspect_err(|e| {
+      error!(
+      "ERROR sending alerts | failed to get alerters from db | {e:#}"
+    )
+    }) else {
+      return;
+    };
+
+    let handles = alerts
+      .iter()
+      .map(|alert| send_alert_to_alerters(&alerters, alert));
+
+    join_all(handles).await;
+  }
+  .instrument(span)
+  .await
+}
+
+#[instrument(level = "debug")]
+async fn send_alert_to_alerters(alerters: &[Alerter], alert: &Alert) {
+  if alerters.is_empty() {
+    return;
+  }
+
+  let handles = alerters
+    .iter()
+    .map(|alerter| send_alert_to_alerter(alerter, alert));
+
+  join_all(handles)
+    .await
+    .into_iter()
+    .filter_map(|res| res.err())
+    .for_each(|e| error!("{e:#}"));
+}
+
+pub async fn send_alert_to_alerter(
+  alerter: &Alerter,
+  alert: &Alert,
+) -> anyhow::Result<()> {
+  // Don't send if not enabled
+  if !alerter.config.enabled {
+    return Ok(());
+  }
+
+  if is_in_maintenance(
+    &alerter.config.maintenance_windows,
+    komodo_timestamp(),
+  ) {
+    return Ok(());
+  }
+
+  let alert_type = alert.data.extract_variant();
+
+  // In the test case, we don't want the filters inside this
+  // block to stop the test from being sent to the alerting endpoint.
+  if alert_type != AlertDataVariant::Test {
+    // Don't send if alert type not configured on the alerter
+    if !alerter.config.alert_types.is_empty()
+      && !alerter.config.alert_types.contains(&alert_type)
+    {
+      return Ok(());
+    }
+
+    // Don't send if resource is in the blacklist
+    if alerter.config.except_resources.contains(&alert.target) {
+      return Ok(());
+    }
+
+    // Don't send if whitelist configured and target is not included
+    if !alerter.config.resources.is_empty()
+      && !alerter.config.resources.contains(&alert.target)
+    {
+      return Ok(());
+    }
+  }
+
+  match &alerter.config.endpoint {
+    AlerterEndpoint::Custom(CustomAlerterEndpoint { url }) => {
+      send_custom_alert(url, alert).await.with_context(|| {
+        format!(
+          "Failed to send alert to Custom Alerter {}",
+          alerter.name
+        )
+      })
+    }
+    AlerterEndpoint::Slack(SlackAlerterEndpoint { url }) => {
+      slack::send_alert(url, alert).await.with_context(|| {
+        format!(
+          "Failed to send alert to Slack Alerter {}",
+          alerter.name
+        )
+      })
+    }
+    AlerterEndpoint::Discord(DiscordAlerterEndpoint { url }) => {
+      discord::send_alert(url, alert).await.with_context(|| {
+        format!(
+          "Failed to send alert to Discord Alerter {}",
+          alerter.name
+        )
+      })
+    }
+    AlerterEndpoint::Ntfy(NtfyAlerterEndpoint { url, email }) => {
+      ntfy::send_alert(url, email.as_deref(), alert)
+        .await
+        .with_context(|| {
+          format!(
+            "Failed to send alert to ntfy Alerter {}",
+            alerter.name
+          )
+        })
+    }
+    AlerterEndpoint::Pushover(PushoverAlerterEndpoint { url }) => {
+      pushover::send_alert(url, alert).await.with_context(|| {
+        format!(
+          "Failed to send alert to Pushover Alerter {}",
+          alerter.name
+        )
+      })
+    }
+  }
+}
+
+#[instrument(level = "debug")]
+async fn send_custom_alert(
+  url: &str,
+  alert: &Alert,
+) -> anyhow::Result<()> {
+  let VariablesAndSecrets { variables, secrets } =
+    get_variables_and_secrets().await?;
+  let mut url_interpolated = url.to_string();
+
+  let mut interpolator =
+    Interpolator::new(Some(&variables), &secrets);
+
+  interpolator.interpolate_string(&mut url_interpolated)?;
+
+  let res = reqwest::Client::new()
+    .post(url_interpolated)
+    .json(alert)
+    .send()
+    .await
+    .map_err(|e| {
+      let replacers = interpolator
+        .secret_replacers
+        .into_iter()
+        .collect::<Vec<_>>();
+      let sanitized_error =
+        svi::replace_in_string(&format!("{e:?}"), &replacers);
+      anyhow::Error::msg(format!(
+        "Error with request: {sanitized_error}"
+      ))
+    })
+    .context("failed at post request to alerter")?;
+  let status = res.status();
+  if !status.is_success() {
+    let text = res
+      .text()
+      .await
+      .context("failed to get response text on alerter response")?;
+    return Err(anyhow!(
+      "post to alerter failed | {status} | {text}"
+    ));
+  }
+  Ok(())
+}
+
+fn fmt_region(region: &Option<String>) -> String {
+  match region {
+    Some(region) => format!(" ({region})"),
+    None => String::new(),
+  }
+}
+
+fn fmt_docker_container_state(state: &DeploymentState) -> String {
+  match state {
+    DeploymentState::Running => String::from("Running ▶️"),
+    DeploymentState::Exited => String::from("Exited 🛑"),
+    DeploymentState::Restarting => String::from("Restarting 🔄"),
+    DeploymentState::NotDeployed => String::from("Not Deployed"),
+    _ => state.to_string(),
+  }
+}
+
+fn fmt_stack_state(state: &StackState) -> String {
+  match state {
+    StackState::Running => String::from("Running ▶️"),
+    StackState::Stopped => String::from("Stopped 🛑"),
+    StackState::Restarting => String::from("Restarting 🔄"),
+    StackState::Down => String::from("Down ⬇️"),
+    _ => state.to_string(),
+  }
+}
+
+fn fmt_level(level: SeverityLevel) -> &'static str {
+  match level {
+    SeverityLevel::Critical => "CRITICAL 🚨",
+    SeverityLevel::Warning => "WARNING ‼️",
+    SeverityLevel::Ok => "OK ✅",
+  }
+}
+
+fn resource_link(
+  resource_type: ResourceTargetVariant,
+  id: &str,
+) -> String {
+  komodo_client::entities::resource_link(
+    &core_config().host,
+    resource_type,
+    id,
+  )
+}
+
+/// Standard message content format
+/// used by Ntfy, Pushover.
+fn standard_alert_content(alert: &Alert) -> String {
+  let level = fmt_level(alert.level);
+  match &alert.data {
+    AlertData::Test { id, name } => {
+      let link = resource_link(ResourceTargetVariant::Alerter, id);
+      format!(
+        "{level} | If you see this message, then Alerter {name} is working\n{link}",
+      )
+    }
+    AlertData::ServerVersionMismatch {
+      id,
+      name,
+      region,
+      server_version,
+      core_version,
+    } => {
+      let region = fmt_region(region);
+      let link = resource_link(ResourceTargetVariant::Server, id);
+      match alert.level {
+        SeverityLevel::Ok => {
+          format!(
+            "{level} | {name}{region} | Periphery version now matches Core version ✅\n{link}"
+          )
+        }
+        _ => {
+          format!(
+            "{level} | {name}{region} | Version mismatch detected ⚠️\nPeriphery: {server_version} | Core: {core_version}\n{link}"
+          )
+        }
+      }
+    }
+    AlertData::ServerUnreachable {
+      id,
+      name,
+      region,
+      err,
+    } => {
+      let region = fmt_region(region);
+      let link = resource_link(ResourceTargetVariant::Server, id);
+      match alert.level {
+        SeverityLevel::Ok => {
+          format!("{level} | {name}{region} is now reachable\n{link}")
+        }
+        SeverityLevel::Critical => {
+          let err = err
+            .as_ref()
+            .map(|e| format!("\nerror: {e:#?}"))
+            .unwrap_or_default();
+          format!(
+            "{level} | {name}{region} is unreachable ❌\n{link}{err}"
+          )
+        }
+        _ => unreachable!(),
+      }
+    }
+    AlertData::ServerCpu {
+      id,
+      name,
+      region,
+      percentage,
+    } => {
+      let region = fmt_region(region);
+      let link = resource_link(ResourceTargetVariant::Server, id);
+      format!(
+        "{level} | {name}{region} cpu usage at {percentage:.1}%\n{link}",
+      )
+    }
+    AlertData::ServerMem {
+      id,
+      name,
+      region,
+      used_gb,
+      total_gb,
+    } => {
+      let region = fmt_region(region);
+      let link = resource_link(ResourceTargetVariant::Server, id);
+      let percentage = 100.0 * used_gb / total_gb;
+      format!(
+        "{level} | {name}{region} memory usage at {percentage:.1}%💾\n\nUsing {used_gb:.1} GiB / {total_gb:.1} GiB\n{link}",
+      )
+    }
+    AlertData::ServerDisk {
+      id,
+      name,
+      region,
+      path,
+      used_gb,
+      total_gb,
+    } => {
+      let region = fmt_region(region);
+      let link = resource_link(ResourceTargetVariant::Server, id);
+      let percentage = 100.0 * used_gb / total_gb;
+      format!(
+        "{level} | {name}{region} disk usage at {percentage:.1}%💿\nmount point: {path:?}\nusing {used_gb:.1} GiB / {total_gb:.1} GiB\n{link}",
+      )
+    }
+    AlertData::ContainerStateChange {
+      id,
+      name,
+      server_id: _server_id,
+      server_name,
+      from,
+      to,
+    } => {
+      let link = resource_link(ResourceTargetVariant::Deployment, id);
+      let to_state = fmt_docker_container_state(to);
+      format!(
+        "📦Deployment {name} is now {to_state}\nserver: {server_name}\nprevious: {from}\n{link}",
+      )
+    }
+    AlertData::DeploymentImageUpdateAvailable {
+      id,
+      name,
+      server_id: _server_id,
+      server_name,
+      image,
+    } => {
+      let link = resource_link(ResourceTargetVariant::Deployment, id);
+      format!(
+        "⬆ Deployment {name} has an update available\nserver: {server_name}\nimage: {image}\n{link}",
+      )
+    }
+    AlertData::DeploymentAutoUpdated {
+      id,
+      name,
+      server_id: _server_id,
+      server_name,
+      image,
+    } => {
+      let link = resource_link(ResourceTargetVariant::Deployment, id);
+      format!(
+        "⬆ Deployment {name} was updated automatically\nserver: {server_name}\nimage: {image}\n{link}",
+      )
+    }
+    AlertData::StackStateChange {
+      id,
+      name,
+      server_id: _server_id,
+      server_name,
+      from,
+      to,
+    } => {
+      let link = resource_link(ResourceTargetVariant::Stack, id);
+      let to_state = fmt_stack_state(to);
+      format!(
+        "🥞 Stack {name} is now {to_state}\nserver: {server_name}\nprevious: {from}\n{link}",
+      )
+    }
+    AlertData::StackImageUpdateAvailable {
+      id,
+      name,
+      server_id: _server_id,
+      server_name,
+      service,
+      image,
+    } => {
+      let link = resource_link(ResourceTargetVariant::Stack, id);
+      format!(
+        "⬆ Stack {name} has an update available\nserver: {server_name}\nservice: {service}\nimage: {image}\n{link}",
+      )
+    }
+    AlertData::StackAutoUpdated {
+      id,
+      name,
+      server_id: _server_id,
+      server_name,
+      images,
+    } => {
+      let link = resource_link(ResourceTargetVariant::Stack, id);
+      let images_label =
+        if images.len() > 1 { "images" } else { "image" };
+      let images_str = images.join(", ");
+      format!(
+        "⬆ Stack {name} was updated automatically ⏫\nserver: {server_name}\n{images_label}: {images_str}\n{link}",
+      )
+    }
+    AlertData::AwsBuilderTerminationFailed {
+      instance_id,
+      message,
+    } => {
+      format!(
+        "{level} | Failed to terminate AWS builder instance\ninstance id: {instance_id}\n{message}",
+      )
+    }
+    AlertData::ResourceSyncPendingUpdates { id, name } => {
+      let link =
+        resource_link(ResourceTargetVariant::ResourceSync, id);
+      format!(
+        "{level} | Pending resource sync updates on {name}\n{link}",
+      )
+    }
+    AlertData::BuildFailed { id, name, version } => {
+      let link = resource_link(ResourceTargetVariant::Build, id);
+      format!(
+        "{level} | Build {name} failed\nversion: v{version}\n{link}",
+      )
+    }
+    AlertData::RepoBuildFailed { id, name } => {
+      let link = resource_link(ResourceTargetVariant::Repo, id);
+      format!("{level} | Repo build for {name} failed\n{link}",)
+    }
+    AlertData::ProcedureFailed { id, name } => {
+      let link = resource_link(ResourceTargetVariant::Procedure, id);
+      format!("{level} | Procedure {name} failed\n{link}")
+    }
+    AlertData::ActionFailed { id, name } => {
+      let link = resource_link(ResourceTargetVariant::Action, id);
+      format!("{level} | Action {name} failed\n{link}")
+    }
+    AlertData::ScheduleRun {
+      resource_type,
+      id,
+      name,
+    } => {
+      let link = resource_link(*resource_type, id);
+      format!(
+        "{level} | {name} ({resource_type}) | Scheduled run started 🕝\n{link}"
+      )
+    }
+    AlertData::Custom { message, details } => {
+      format!(
+        "{level} | {message}{}",
+        if details.is_empty() {
+          format_args!("")
+        } else {
+          format_args!("\n{details}")
+        }
+      )
+    }
+    AlertData::None {} => Default::default(),
+  }
+}

bin/core/src/alert/ntfy.rs

@@ -0,0 +1,56 @@
+use std::sync::OnceLock;
+
+use super::*;
+
+#[instrument(level = "debug")]
+pub async fn send_alert(
+  url: &str,
+  email: Option<&str>,
+  alert: &Alert,
+) -> anyhow::Result<()> {
+  let content = standard_alert_content(alert);
+  if !content.is_empty() {
+    send_message(url, email, content).await?;
+  }
+  Ok(())
+}
+
+async fn send_message(
+  url: &str,
+  email: Option<&str>,
+  content: String,
+) -> anyhow::Result<()> {
+  let mut request = http_client()
+    .post(url)
+    .header("Title", "ntfy Alert")
+    .body(content);
+
+  if let Some(email) = email {
+    request = request.header("X-Email", email);
+  }
+
+  let response =
+    request.send().await.context("Failed to send message")?;
+
+  let status = response.status();
+  if status.is_success() {
+    debug!("ntfy alert sent successfully: {}", status);
+    Ok(())
+  } else {
+    let text = response.text().await.with_context(|| {
+      format!(
+        "Failed to send message to ntfy | {status} | failed to get response text"
+      )
+    })?;
+    Err(anyhow!(
+      "Failed to send message to ntfy | {} | {}",
+      status,
+      text
+    ))
+  }
+}
+
+fn http_client() -> &'static reqwest::Client {
+  static CLIENT: OnceLock<reqwest::Client> = OnceLock::new();
+  CLIENT.get_or_init(reqwest::Client::new)
+}

bin/core/src/alert/pushover.rs

@@ -0,0 +1,55 @@
+use std::sync::OnceLock;
+
+use super::*;
+
+#[instrument(level = "debug")]
+pub async fn send_alert(
+  url: &str,
+  alert: &Alert,
+) -> anyhow::Result<()> {
+  let content = standard_alert_content(alert);
+  if !content.is_empty() {
+    send_message(url, content).await?;
+  }
+  Ok(())
+}
+
+async fn send_message(
+  url: &str,
+  content: String,
+) -> anyhow::Result<()> {
+  // pushover needs all information to be encoded in the URL. At minimum they need
+  // the user key, the application token, and the message (url encoded).
+  // other optional params here: https://pushover.net/api (just add them to the
+  // webhook url along with the application token and the user key).
+  let content = [("message", content)];
+
+  let response = http_client()
+    .post(url)
+    .form(&content)
+    .send()
+    .await
+    .context("Failed to send message")?;
+
+  let status = response.status();
+  if status.is_success() {
+    debug!("pushover alert sent successfully: {}", status);
+    Ok(())
+  } else {
+    let text = response.text().await.with_context(|| {
+      format!(
+        "Failed to send message to pushover | {status} | failed to get response text"
+      )
+    })?;
+    Err(anyhow!(
+      "Failed to send message to pushover | {} | {}",
+      status,
+      text
+    ))
+  }
+}
+
+fn http_client() -> &'static reqwest::Client {
+  static CLIENT: OnceLock<reqwest::Client> = OnceLock::new();
+  CLIENT.get_or_init(reqwest::Client::new)
+}

bin/core/src/alert/slack.rs

@@ -1,136 +1,57 @@
-use anyhow::{anyhow, Context};
-use derive_variants::ExtractVariant;
-use futures::future::join_all;
-use monitor_client::entities::{
-  alert::{Alert, AlertData},
-  alerter::*,
-  deployment::DeploymentState,
-  server::stats::SeverityLevel,
-  stack::StackState,
-  update::ResourceTargetVariant,
-};
-use mungos::{find::find_collect, mongodb::bson::doc};
-use slack::types::Block;
-
-use crate::{config::core_config, state::db_client};
-
-#[instrument]
-pub async fn send_alerts(alerts: &[Alert]) {
-  if alerts.is_empty() {
-    return;
-  }
-
-  let Ok(alerters) = find_collect(
-    &db_client().await.alerters,
-    doc! { "config.enabled": true },
-    None,
-  )
-  .await
-  .inspect_err(|e| {
-    error!(
-      "ERROR sending alerts | failed to get alerters from db | {e:#}"
-    )
-  }) else {
-    return;
-  };
-
-  let handles =
-    alerts.iter().map(|alert| send_alert(&alerters, alert));
-
-  join_all(handles).await;
-}
+use super::*;
 
 #[instrument(level = "debug")]
-async fn send_alert(alerters: &[Alerter], alert: &Alert) {
-  if alerters.is_empty() {
-    return;
-  }
-
-  let alert_type = alert.data.extract_variant();
-
-  let handles = alerters.iter().map(|alerter| async {
-    // Don't send if not enabled
-    if !alerter.config.enabled {
-      return Ok(());
-    }
-
-    // Don't send if alert type not configured on the alerter
-    if !alerter.config.alert_types.is_empty()
-      && !alerter.config.alert_types.contains(&alert_type)
-    {
-      return Ok(());
-    }
-
-    // Don't send if resource is in the blacklist
-    if alerter.config.except_resources.contains(&alert.target) {
-      return Ok(());
-    }
-
-    // Don't send if whitelist configured and target is not included
-    if !alerter.config.resources.is_empty()
-      && !alerter.config.resources.contains(&alert.target)
-    {
-      return Ok(());
-    }
-
-    match &alerter.config.endpoint {
-      AlerterEndpoint::Slack(SlackAlerterEndpoint { url }) => {
-        send_slack_alert(url, alert).await.with_context(|| {
-          format!(
-            "failed to send alert to slack alerter {}",
-            alerter.name
-          )
-        })
-      }
-      AlerterEndpoint::Custom(CustomAlerterEndpoint { url }) => {
-        send_custom_alert(url, alert).await.with_context(|| {
-          format!(
-            "failed to send alert to custom alerter {}",
-            alerter.name
-          )
-        })
-      }
-    }
-  });
-
-  join_all(handles)
-    .await
-    .into_iter()
-    .filter_map(|res| res.err())
-    .for_each(|e| error!("{e:#}"));
-}
-
-#[instrument(level = "debug")]
-async fn send_custom_alert(
-  url: &str,
-  alert: &Alert,
-) -> anyhow::Result<()> {
-  let res = reqwest::Client::new()
-    .post(url)
-    .json(alert)
-    .send()
-    .await
-    .context("failed at post request to alerter")?;
-  let status = res.status();
-  if !status.is_success() {
-    let text = res
-      .text()
-      .await
-      .context("failed to get response text on alerter response")?;
-    return Err(anyhow!(
-      "post to alerter failed | {status} | {text}"
-    ));
-  }
-  Ok(())
-}
-
-#[instrument(level = "debug")]
-async fn send_slack_alert(
+pub async fn send_alert(
   url: &str,
   alert: &Alert,
 ) -> anyhow::Result<()> {
   let level = fmt_level(alert.level);
   let (text, blocks): (_, Option<_>) = match &alert.data {
+    AlertData::Test { id, name } => {
+      let text = format!(
+        "{level} | If you see this message, then Alerter *{name}* is *working*"
+      );
+      let blocks = vec![
+        Block::header(level),
+        Block::section(format!(
+          "If you see this message, then Alerter *{name}* is *working*"
+        )),
+        Block::section(resource_link(
+          ResourceTargetVariant::Alerter,
+          id,
+        )),
+      ];
+      (text, blocks.into())
+    }
+    AlertData::ServerVersionMismatch {
+      id,
+      name,
+      region,
+      server_version,
+      core_version,
+    } => {
+      let region = fmt_region(region);
+      let text = match alert.level {
+        SeverityLevel::Ok => {
+          format!(
+            "{level} | *{name}*{region} | Periphery version now matches Core version ✅"
+          )
+        }
+        _ => {
+          format!(
+            "{level} | *{name}*{region} | Version mismatch detected ⚠️\nPeriphery: {server_version} | Core: {core_version}"
+          )
+        }
+      };
+      let blocks = vec![
+        Block::header(text.clone()),
+        Block::section(resource_link(
+          ResourceTargetVariant::Server,
+          id,
+        )),
+      ];
+      (text, blocks.into())
+    }
     AlertData::ServerUnreachable {
       id,
       name,
@@ -181,7 +102,9 @@ async fn send_slack_alert(
       let region = fmt_region(region);
       match alert.level {
         SeverityLevel::Ok => {
-          let text = format!("{level} | *{name}*{region} cpu usage at *{percentage:.1}%*");
+          let text = format!(
+            "{level} | *{name}*{region} cpu usage at *{percentage:.1}%*"
+          );
           let blocks = vec![
             Block::header(level),
             Block::section(format!(
@@ -195,7 +118,9 @@ async fn send_slack_alert(
           (text, blocks.into())
         }
         _ => {
-          let text = format!("{level} | *{name}*{region} cpu usage at *{percentage:.1}%* 📈");
+          let text = format!(
+            "{level} | *{name}*{region} cpu usage at *{percentage:.1}%* 📈"
+          );
           let blocks = vec![
             Block::header(level),
             Block::section(format!(
@@ -221,7 +146,9 @@ async fn send_slack_alert(
       let percentage = 100.0 * used_gb / total_gb;
       match alert.level {
         SeverityLevel::Ok => {
-          let text = format!("{level} | *{name}*{region} memory usage at *{percentage:.1}%* 💾");
+          let text = format!(
+            "{level} | *{name}*{region} memory usage at *{percentage:.1}%* 💾"
+          );
           let blocks = vec![
             Block::header(level),
             Block::section(format!(
@@ -238,7 +165,9 @@ async fn send_slack_alert(
           (text, blocks.into())
         }
         _ => {
-          let text = format!("{level} | *{name}*{region} memory usage at *{percentage:.1}%* 💾");
+          let text = format!(
+            "{level} | *{name}*{region} memory usage at *{percentage:.1}%* 💾"
+          );
           let blocks = vec![
             Block::header(level),
             Block::section(format!(
@@ -268,7 +197,9 @@ async fn send_slack_alert(
       let percentage = 100.0 * used_gb / total_gb;
       match alert.level {
         SeverityLevel::Ok => {
-          let text = format!("{level} | *{name}*{region} disk usage at *{percentage:.1}%* | mount point: *{path:?}* 💿");
+          let text = format!(
+            "{level} | *{name}*{region} disk usage at *{percentage:.1}%* | mount point: *{path:?}* 💿"
+          );
           let blocks = vec![
             Block::header(level),
             Block::section(format!(
@@ -277,12 +208,17 @@ async fn send_slack_alert(
             Block::section(format!(
               "mount point: {path:?} | using *{used_gb:.1} GiB* / *{total_gb:.1} GiB*"
             )),
-            Block::section(resource_link(ResourceTargetVariant::Server, id)),
+            Block::section(resource_link(
+              ResourceTargetVariant::Server,
+              id,
+            )),
           ];
           (text, blocks.into())
         }
         _ => {
-          let text = format!("{level} | *{name}*{region} disk usage at *{percentage:.1}%* | mount point: *{path:?}* 💿");
+          let text = format!(
+            "{level} | *{name}*{region} disk usage at *{percentage:.1}%* | mount point: *{path:?}* 💿"
+          );
           let blocks = vec![
             Block::header(level),
             Block::section(format!(
@@ -291,7 +227,10 @@ async fn send_slack_alert(
             Block::section(format!(
               "mount point: {path:?} | using *{used_gb:.1} GiB* / *{total_gb:.1} GiB*"
             )),
-            Block::section(resource_link(ResourceTargetVariant::Server, id)),
+            Block::section(resource_link(
+              ResourceTargetVariant::Server,
+              id,
+            )),
           ];
           (text, blocks.into())
         }
@@ -306,7 +245,7 @@ async fn send_slack_alert(
       ..
     } => {
       let to = fmt_docker_container_state(to);
-      let text = format!("📦 Container *{name}* is now {to}");
+      let text = format!("📦 Container *{name}* is now *{to}*");
       let blocks = vec![
         Block::header(text.clone()),
         Block::section(format!(
@@ -319,6 +258,48 @@ async fn send_slack_alert(
       ];
       (text, blocks.into())
     }
+    AlertData::DeploymentImageUpdateAvailable {
+      id,
+      name,
+      server_name,
+      server_id: _server_id,
+      image,
+    } => {
+      let text =
+        format!("⬆ Deployment *{name}* has an update available");
+      let blocks = vec![
+        Block::header(text.clone()),
+        Block::section(format!(
+          "server: *{server_name}*\nimage: *{image}*",
+        )),
+        Block::section(resource_link(
+          ResourceTargetVariant::Deployment,
+          id,
+        )),
+      ];
+      (text, blocks.into())
+    }
+    AlertData::DeploymentAutoUpdated {
+      id,
+      name,
+      server_name,
+      server_id: _server_id,
+      image,
+    } => {
+      let text =
+        format!("⬆ Deployment *{name}* was updated automatically ⏫");
+      let blocks = vec![
+        Block::header(text.clone()),
+        Block::section(format!(
+          "server: *{server_name}*\nimage: *{image}*",
+        )),
+        Block::section(resource_link(
+          ResourceTargetVariant::Deployment,
+          id,
+        )),
+      ];
+      (text, blocks.into())
+    }
     AlertData::StackStateChange {
       name,
       server_name,
@@ -328,11 +309,56 @@ async fn send_slack_alert(
       ..
     } => {
       let to = fmt_stack_state(to);
-      let text = format!("🥞 Stack *{name}* is now {to}");
+      let text = format!("🥞 Stack *{name}* is now *{to}*");
       let blocks = vec![
         Block::header(text.clone()),
         Block::section(format!(
-          "server: {server_name}\nprevious: {from}",
+          "server: *{server_name}*\nprevious: *{from}*",
+        )),
+        Block::section(resource_link(
+          ResourceTargetVariant::Stack,
+          id,
+        )),
+      ];
+      (text, blocks.into())
+    }
+    AlertData::StackImageUpdateAvailable {
+      id,
+      name,
+      server_name,
+      server_id: _server_id,
+      service,
+      image,
+    } => {
+      let text = format!("⬆ Stack *{name}* has an update available");
+      let blocks = vec![
+        Block::header(text.clone()),
+        Block::section(format!(
+          "server: *{server_name}*\nservice: *{service}*\nimage: *{image}*",
+        )),
+        Block::section(resource_link(
+          ResourceTargetVariant::Stack,
+          id,
+        )),
+      ];
+      (text, blocks.into())
+    }
+    AlertData::StackAutoUpdated {
+      id,
+      name,
+      server_name,
+      server_id: _server_id,
+      images,
+    } => {
+      let text =
+        format!("⬆ Stack *{name}* was updated automatically ⏫");
+      let images_label =
+        if images.len() > 1 { "images" } else { "image" };
+      let images = images.join(", ");
+      let blocks = vec![
+        Block::header(text.clone()),
+        Block::section(format!(
+          "server: *{server_name}*\n{images_label}: *{images}*",
         )),
         Block::section(resource_link(
           ResourceTargetVariant::Stack,
@@ -357,8 +383,9 @@ async fn send_slack_alert(
       (text, blocks.into())
     }
     AlertData::ResourceSyncPendingUpdates { id, name } => {
-      let text =
-        format!("{level} | Pending resource sync updates on {name}");
+      let text = format!(
+        "{level} | Pending resource sync updates on *{name}*"
+      );
       let blocks = vec![
         Block::header(text.clone()),
         Block::section(format!(
@@ -375,21 +402,19 @@ async fn send_slack_alert(
       let text = format!("{level} | Build {name} has failed");
       let blocks = vec![
         Block::header(text.clone()),
-        Block::section(format!(
-          "build id: *{id}*\nbuild name: *{name}*\nversion: v{version}",
+        Block::section(format!("version: *v{version}*",)),
+        Block::section(resource_link(
+          ResourceTargetVariant::Build,
+          id,
         )),
-        Block::section(resource_link(ResourceTargetVariant::Build, id))
       ];
       (text, blocks.into())
     }
     AlertData::RepoBuildFailed { id, name } => {
       let text =
-        format!("{level} | Repo build for {name} has failed");
+        format!("{level} | Repo build for *{name}* has *failed*");
       let blocks = vec![
         Block::header(text.clone()),
-        Block::section(format!(
-          "repo id: *{id}*\nrepo name: *{name}*",
-        )),
         Block::section(resource_link(
           ResourceTargetVariant::Repo,
           id,
@@ -397,83 +422,72 @@ async fn send_slack_alert(
       ];
       (text, blocks.into())
     }
+    AlertData::ProcedureFailed { id, name } => {
+      let text = format!("{level} | Procedure *{name}* has *failed*");
+      let blocks = vec![
+        Block::header(text.clone()),
+        Block::section(resource_link(
+          ResourceTargetVariant::Procedure,
+          id,
+        )),
+      ];
+      (text, blocks.into())
+    }
+    AlertData::ActionFailed { id, name } => {
+      let text = format!("{level} | Action *{name}* has *failed*");
+      let blocks = vec![
+        Block::header(text.clone()),
+        Block::section(resource_link(
+          ResourceTargetVariant::Action,
+          id,
+        )),
+      ];
+      (text, blocks.into())
+    }
+    AlertData::ScheduleRun {
+      resource_type,
+      id,
+      name,
+    } => {
+      let text = format!(
+        "{level} | *{name}* ({resource_type}) | Scheduled run started 🕝"
+      );
+      let blocks = vec![
+        Block::header(text.clone()),
+        Block::section(resource_link(*resource_type, id)),
+      ];
+      (text, blocks.into())
+    }
+    AlertData::Custom { message, details } => {
+      let text = format!("{level} | {message}");
+      let blocks =
+        vec![Block::header(text.clone()), Block::section(details)];
+      (text, blocks.into())
+    }
     AlertData::None {} => Default::default(),
   };
   if !text.is_empty() {
-    let slack = slack::Client::new(url);
-    slack.send_message(text, blocks).await?;
+    let VariablesAndSecrets { variables, secrets } =
+      get_variables_and_secrets().await?;
+    let mut url_interpolated = url.to_string();
+
+    let mut interpolator =
+      Interpolator::new(Some(&variables), &secrets);
+
+    interpolator.interpolate_string(&mut url_interpolated)?;
+
+    let slack = ::slack::Client::new(url_interpolated);
+    slack.send_message(text, blocks).await.map_err(|e| {
+      let replacers = interpolator
+        .secret_replacers
+        .into_iter()
+        .collect::<Vec<_>>();
+      let sanitized_error =
+        svi::replace_in_string(&format!("{e:?}"), &replacers);
+      anyhow::Error::msg(format!(
+        "Error with slack request: {sanitized_error}"
+      ))
+    })?;
   }
   Ok(())
 }
-
-fn fmt_region(region: &Option<String>) -> String {
-  match region {
-    Some(region) => format!(" ({region})"),
-    None => String::new(),
-  }
-}
-
-fn fmt_docker_container_state(state: &DeploymentState) -> String {
-  match state {
-    DeploymentState::Running => String::from("Running ▶️"),
-    DeploymentState::Exited => String::from("Exited 🛑"),
-    DeploymentState::Restarting => String::from("Restarting 🔄"),
-    DeploymentState::NotDeployed => String::from("Not Deployed"),
-    _ => state.to_string(),
-  }
-}
-
-fn fmt_stack_state(state: &StackState) -> String {
-  match state {
-    StackState::Running => String::from("Running ▶️"),
-    StackState::Stopped => String::from("Stopped 🛑"),
-    StackState::Restarting => String::from("Restarting 🔄"),
-    StackState::Down => String::from("Down ⬇️"),
-    _ => state.to_string(),
-  }
-}
-
-fn fmt_level(level: SeverityLevel) -> &'static str {
-  match level {
-    SeverityLevel::Critical => "CRITICAL 🚨",
-    SeverityLevel::Warning => "WARNING ‼️",
-    SeverityLevel::Ok => "OK ✅",
-  }
-}
-
-fn resource_link(
-  resource_type: ResourceTargetVariant,
-  id: &str,
-) -> String {
-  let path = match resource_type {
-    ResourceTargetVariant::System => unreachable!(),
-    ResourceTargetVariant::Build => format!("/builds/{id}"),
-    ResourceTargetVariant::Builder => {
-      format!("/builders/{id}")
-    }
-    ResourceTargetVariant::Deployment => {
-      format!("/deployments/{id}")
-    }
-    ResourceTargetVariant::Stack => {
-      format!("/stacks/{id}")
-    }
-    ResourceTargetVariant::Server => {
-      format!("/servers/{id}")
-    }
-    ResourceTargetVariant::Repo => format!("/repos/{id}"),
-    ResourceTargetVariant::Alerter => {
-      format!("/alerters/{id}")
-    }
-    ResourceTargetVariant::Procedure => {
-      format!("/procedures/{id}")
-    }
-    ResourceTargetVariant::ServerTemplate => {
-      format!("/server-templates/{id}")
-    }
-    ResourceTargetVariant::ResourceSync => {
-      format!("/resource-syncs/{id}")
-    }
-  };
-
-  format!("{}{path}", core_config().host)
-}

bin/core/src/api/auth.rs

@@ -1,12 +1,14 @@
-use std::{sync::OnceLock, time::Instant};
+use std::time::Instant;
 
-use anyhow::anyhow;
-use axum::{http::HeaderMap, routing::post, Router};
-use axum_extra::{headers::ContentType, TypedHeader};
-use monitor_client::{api::auth::*, entities::user::User};
-use resolver_api::{derive::Resolver, Resolve, Resolver};
+use axum::{Router, extract::Path, http::HeaderMap, routing::post};
+use derive_variants::{EnumVariants, ExtractVariant};
+use komodo_client::{api::auth::*, entities::user::User};
+use reqwest::StatusCode;
+use resolver_api::Resolve;
+use response::Response;
 use serde::{Deserialize, Serialize};
-use serror::Json;
+use serde_json::json;
+use serror::{AddStatusCode, Json};
 use typeshare::typeshare;
 use uuid::Uuid;
 
@@ -15,114 +17,142 @@ use crate::{
     get_user_id_from_headers,
     github::{self, client::github_oauth_client},
     google::{self, client::google_oauth_client},
+    oidc::{self, client::oidc_client},
   },
   config::core_config,
   helpers::query::get_user,
-  state::{jwt_client, State},
+  state::jwt_client,
 };
 
+use super::Variant;
+
+#[derive(Default)]
+pub struct AuthArgs {
+  pub headers: HeaderMap,
+}
+
 #[typeshare]
-#[derive(Serialize, Deserialize, Debug, Clone, Resolver)]
-#[resolver_target(State)]
-#[resolver_args(HeaderMap)]
+#[derive(
+  Serialize, Deserialize, Debug, Clone, Resolve, EnumVariants,
+)]
+#[args(AuthArgs)]
+#[response(Response)]
+#[error(serror::Error)]
+#[variant_derive(Debug)]
 #[serde(tag = "type", content = "params")]
 #[allow(clippy::enum_variant_names, clippy::large_enum_variant)]
 pub enum AuthRequest {
   GetLoginOptions(GetLoginOptions),
-  CreateLocalUser(CreateLocalUser),
+  SignUpLocalUser(SignUpLocalUser),
   LoginLocalUser(LoginLocalUser),
   ExchangeForJwt(ExchangeForJwt),
   GetUser(GetUser),
 }
 
 pub fn router() -> Router {
-  let mut router = Router::new().route("/", post(handler));
+  let mut router = Router::new()
+    .route("/", post(handler))
+    .route("/{variant}", post(variant_handler));
+
+  if core_config().local_auth {
+    info!("🔑 Local Login Enabled");
+  }
 
   if github_oauth_client().is_some() {
+    info!("🔑 Github Login Enabled");
     router = router.nest("/github", github::router())
   }
 
   if google_oauth_client().is_some() {
+    info!("🔑 Google Login Enabled");
     router = router.nest("/google", google::router())
   }
 
+  if core_config().oidc_enabled {
+    info!("🔑 OIDC Login Enabled");
+    router = router.nest("/oidc", oidc::router())
+  }
+
   router
 }
 
+async fn variant_handler(
+  headers: HeaderMap,
+  Path(Variant { variant }): Path<Variant>,
+  Json(params): Json<serde_json::Value>,
+) -> serror::Result<axum::response::Response> {
+  let req: AuthRequest = serde_json::from_value(json!({
+    "type": variant,
+    "params": params,
+  }))?;
+  handler(headers, Json(req)).await
+}
+
 #[instrument(name = "AuthHandler", level = "debug", skip(headers))]
 async fn handler(
   headers: HeaderMap,
   Json(request): Json<AuthRequest>,
-) -> serror::Result<(TypedHeader<ContentType>, String)> {
+) -> serror::Result<axum::response::Response> {
   let timer = Instant::now();
   let req_id = Uuid::new_v4();
-  debug!("/auth request {req_id} | METHOD: {}", request.req_type());
-  let res = State.resolve_request(request, headers).await.map_err(
-    |e| match e {
-      resolver_api::Error::Serialization(e) => {
-        anyhow!("{e:?}").context("response serialization error")
-      }
-      resolver_api::Error::Inner(e) => e,
-    },
+  debug!(
+    "/auth request {req_id} | METHOD: {:?}",
+    request.extract_variant()
   );
+  let res = request.resolve(&AuthArgs { headers }).await;
   if let Err(e) = &res {
-    debug!("/auth request {req_id} | error: {e:#}");
+    debug!("/auth request {req_id} | error: {:#}", e.error);
   }
   let elapsed = timer.elapsed();
   debug!("/auth request {req_id} | resolve time: {elapsed:?}");
-  Ok((TypedHeader(ContentType::json()), res?))
+  res.map(|res| res.0)
 }
 
-fn login_options_reponse() -> &'static GetLoginOptionsResponse {
-  static GET_LOGIN_OPTIONS_RESPONSE: OnceLock<
-    GetLoginOptionsResponse,
-  > = OnceLock::new();
-  GET_LOGIN_OPTIONS_RESPONSE.get_or_init(|| {
-    let config = core_config();
-    GetLoginOptionsResponse {
-      local: config.local_auth,
-      github: config.github_oauth.enabled
-        && !config.github_oauth.id.is_empty()
-        && !config.github_oauth.secret.is_empty(),
-      google: config.google_oauth.enabled
-        && !config.google_oauth.id.is_empty()
-        && !config.google_oauth.secret.is_empty(),
-    }
-  })
+fn login_options_response() -> GetLoginOptionsResponse {
+  let config = core_config();
+  GetLoginOptionsResponse {
+    local: config.local_auth,
+    github: github_oauth_client().is_some(),
+    google: google_oauth_client().is_some(),
+    oidc: oidc_client().load().is_some(),
+    registration_disabled: config.disable_user_registration,
+  }
 }
 
-impl Resolve<GetLoginOptions, HeaderMap> for State {
+impl Resolve<AuthArgs> for GetLoginOptions {
   #[instrument(name = "GetLoginOptions", level = "debug", skip(self))]
   async fn resolve(
-    &self,
-    _: GetLoginOptions,
-    _: HeaderMap,
-  ) -> anyhow::Result<GetLoginOptionsResponse> {
-    Ok(*login_options_reponse())
+    self,
+    _: &AuthArgs,
+  ) -> serror::Result<GetLoginOptionsResponse> {
+    Ok(login_options_response())
   }
 }
 
-impl Resolve<ExchangeForJwt, HeaderMap> for State {
+impl Resolve<AuthArgs> for ExchangeForJwt {
   #[instrument(name = "ExchangeForJwt", level = "debug", skip(self))]
   async fn resolve(
-    &self,
-    ExchangeForJwt { token }: ExchangeForJwt,
-    _: HeaderMap,
-  ) -> anyhow::Result<ExchangeForJwtResponse> {
-    let jwt = jwt_client().redeem_exchange_token(&token).await?;
-    let res = ExchangeForJwtResponse { jwt };
-    Ok(res)
+    self,
+    _: &AuthArgs,
+  ) -> serror::Result<ExchangeForJwtResponse> {
+    jwt_client()
+      .redeem_exchange_token(&self.token)
+      .await
+      .map_err(Into::into)
   }
 }
 
-impl Resolve<GetUser, HeaderMap> for State {
+impl Resolve<AuthArgs> for GetUser {
   #[instrument(name = "GetUser", level = "debug", skip(self))]
   async fn resolve(
-    &self,
-    GetUser {}: GetUser,
-    headers: HeaderMap,
-  ) -> anyhow::Result<User> {
-    let user_id = get_user_id_from_headers(&headers).await?;
-    get_user(&user_id).await
+    self,
+    AuthArgs { headers }: &AuthArgs,
+  ) -> serror::Result<User> {
+    let user_id = get_user_id_from_headers(headers)
+      .await
+      .status_code(StatusCode::UNAUTHORIZED)?;
+    get_user(&user_id)
+      .await
+      .status_code(StatusCode::UNAUTHORIZED)
   }
 }

bin/core/src/api/execute/action.rs

@@ -0,0 +1,426 @@
+use std::{
+  collections::HashSet,
+  path::{Path, PathBuf},
+  str::FromStr,
+  sync::OnceLock,
+};
+
+use anyhow::Context;
+use command::run_komodo_command;
+use config::merge_objects;
+use database::mungos::{
+  by_id::update_one_by_id, mongodb::bson::to_document,
+};
+use interpolate::Interpolator;
+use komodo_client::{
+  api::{
+    execute::{BatchExecutionResponse, BatchRunAction, RunAction},
+    user::{CreateApiKey, CreateApiKeyResponse, DeleteApiKey},
+  },
+  entities::{
+    FileFormat, JsonObject,
+    action::Action,
+    alert::{Alert, AlertData, SeverityLevel},
+    config::core::CoreConfig,
+    komodo_timestamp,
+    permission::PermissionLevel,
+    update::Update,
+    user::action_user,
+  },
+  parsers::parse_key_value_list,
+};
+use resolver_api::Resolve;
+use tokio::fs;
+
+use crate::{
+  alert::send_alerts,
+  api::{execute::ExecuteRequest, user::UserArgs},
+  config::core_config,
+  helpers::{
+    query::{VariablesAndSecrets, get_variables_and_secrets},
+    random_string,
+    update::update_update,
+  },
+  permission::get_check_permissions,
+  resource::refresh_action_state_cache,
+  state::{action_states, db_client},
+};
+
+use super::ExecuteArgs;
+
+impl super::BatchExecute for BatchRunAction {
+  type Resource = Action;
+  fn single_request(action: String) -> ExecuteRequest {
+    ExecuteRequest::RunAction(RunAction {
+      action,
+      args: Default::default(),
+    })
+  }
+}
+
+impl Resolve<ExecuteArgs> for BatchRunAction {
+  #[instrument(name = "BatchRunAction", skip(self, user), fields(user_id = user.id))]
+  async fn resolve(
+    self,
+    ExecuteArgs { user, .. }: &ExecuteArgs,
+  ) -> serror::Result<BatchExecutionResponse> {
+    Ok(
+      super::batch_execute::<BatchRunAction>(&self.pattern, user)
+        .await?,
+    )
+  }
+}
+
+impl Resolve<ExecuteArgs> for RunAction {
+  #[instrument(name = "RunAction", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+  async fn resolve(
+    self,
+    ExecuteArgs { user, update }: &ExecuteArgs,
+  ) -> serror::Result<Update> {
+    let mut action = get_check_permissions::<Action>(
+      &self.action,
+      user,
+      PermissionLevel::Execute.into(),
+    )
+    .await?;
+
+    // get the action state for the action (or insert default).
+    let action_state = action_states()
+      .action
+      .get_or_insert_default(&action.id)
+      .await;
+
+    // This will set action state back to default when dropped.
+    // Will also check to ensure action not already busy before updating.
+    let _action_guard = action_state.update_custom(
+      |state| state.running += 1,
+      |state| state.running -= 1,
+      false,
+    )?;
+
+    let mut update = update.clone();
+
+    update_update(update.clone()).await?;
+
+    let default_args = parse_action_arguments(
+      &action.config.arguments,
+      action.config.arguments_format,
+    )
+    .context("Failed to parse default Action arguments")?;
+
+    let args = merge_objects(
+      default_args,
+      self.args.unwrap_or_default(),
+      true,
+      true,
+    )
+    .context("Failed to merge request args with default args")?;
+
+    let args = serde_json::to_string(&args)
+      .context("Failed to serialize action run arguments")?;
+
+    let CreateApiKeyResponse { key, secret } = CreateApiKey {
+      name: update.id.clone(),
+      expires: 0,
+    }
+    .resolve(&UserArgs {
+      user: action_user().to_owned(),
+    })
+    .await?;
+
+    let contents = &mut action.config.file_contents;
+
+    // Wrap the file contents in the execution context.
+    *contents = full_contents(contents, &args, &key, &secret);
+
+    let replacers =
+      interpolate(contents, &mut update, key.clone(), secret.clone())
+        .await?
+        .into_iter()
+        .collect::<Vec<_>>();
+
+    let file = format!("{}.ts", random_string(10));
+    let path = core_config().action_directory.join(&file);
+
+    if let Some(parent) = path.parent() {
+      fs::create_dir_all(parent)
+        .await
+        .with_context(|| format!("Failed to initialize Action file parent directory {parent:?}"))?;
+    }
+
+    fs::write(&path, contents).await.with_context(|| {
+      format!("Failed to write action file to {path:?}")
+    })?;
+
+    let CoreConfig { ssl_enabled, .. } = core_config();
+
+    let https_cert_flag = if *ssl_enabled {
+      " --unsafely-ignore-certificate-errors=localhost"
+    } else {
+      ""
+    };
+
+    let reload = if action.config.reload_deno_deps {
+      " --reload"
+    } else {
+      ""
+    };
+
+    let mut res = run_komodo_command(
+      // Keep this stage name as is, the UI will find the latest update log by matching the stage name
+      "Execute Action",
+      None,
+      format!(
+        "deno run --allow-all{https_cert_flag}{reload} {}",
+        path.display()
+      ),
+    )
+    .await;
+
+    res.stdout = svi::replace_in_string(&res.stdout, &replacers)
+      .replace(&key, "<ACTION_API_KEY>");
+    res.stderr = svi::replace_in_string(&res.stderr, &replacers)
+      .replace(&secret, "<ACTION_API_SECRET>");
+
+    cleanup_run(file + ".js", &path).await;
+
+    if let Err(e) = (DeleteApiKey { key })
+      .resolve(&UserArgs {
+        user: action_user().to_owned(),
+      })
+      .await
+    {
+      warn!(
+        "Failed to delete API key after action execution | {:#}",
+        e.error
+      );
+    };
+
+    update.logs.push(res);
+    update.finalize();
+
+    // Need to manually update the update before cache refresh,
+    // and before broadcast with update_update.
+    // The Err case of to_document should be unreachable,
+    // but will fail to update cache in that case.
+    if let Ok(update_doc) = to_document(&update) {
+      let _ = update_one_by_id(
+        &db_client().updates,
+        &update.id,
+        database::mungos::update::Update::Set(update_doc),
+        None,
+      )
+      .await;
+      refresh_action_state_cache().await;
+    }
+
+    update_update(update.clone()).await?;
+
+    if !update.success && action.config.failure_alert {
+      warn!("action unsuccessful, alerting...");
+      let target = update.target.clone();
+      tokio::spawn(async move {
+        let alert = Alert {
+          id: Default::default(),
+          target,
+          ts: komodo_timestamp(),
+          resolved_ts: Some(komodo_timestamp()),
+          resolved: true,
+          level: SeverityLevel::Warning,
+          data: AlertData::ActionFailed {
+            id: action.id,
+            name: action.name,
+          },
+        };
+        send_alerts(&[alert]).await
+      });
+    }
+
+    Ok(update)
+  }
+}
+
+async fn interpolate(
+  contents: &mut String,
+  update: &mut Update,
+  key: String,
+  secret: String,
+) -> serror::Result<HashSet<(String, String)>> {
+  let VariablesAndSecrets {
+    variables,
+    mut secrets,
+  } = get_variables_and_secrets().await?;
+
+  secrets.insert(String::from("ACTION_API_KEY"), key);
+  secrets.insert(String::from("ACTION_API_SECRET"), secret);
+
+  let mut interpolator =
+    Interpolator::new(Some(&variables), &secrets);
+
+  interpolator
+    .interpolate_string(contents)?
+    .push_logs(&mut update.logs);
+
+  Ok(interpolator.secret_replacers)
+}
+
+fn full_contents(
+  contents: &str,
+  // Pre-serialized to JSON string.
+  args: &str,
+  key: &str,
+  secret: &str,
+) -> String {
+  let CoreConfig {
+    port, ssl_enabled, ..
+  } = core_config();
+  let protocol = if *ssl_enabled { "https" } else { "http" };
+  let base_url = format!("{protocol}://localhost:{port}");
+  format!(
+    "import {{ KomodoClient, Types }} from '{base_url}/client/lib.js';
+import * as __YAML__ from 'jsr:@std/yaml';
+import * as __TOML__ from 'jsr:@std/toml';
+
+const YAML = {{
+  stringify: __YAML__.stringify,
+  parse: __YAML__.parse,
+  parseAll: __YAML__.parseAll,
+  parseDockerCompose: __YAML__.parse,
+}}
+
+const TOML = {{
+  stringify: __TOML__.stringify,
+  parse: __TOML__.parse,
+  parseResourceToml: __TOML__.parse,
+  parseCargoToml: __TOML__.parse,
+}}
+
+const ARGS = {args};
+
+const komodo = KomodoClient('{base_url}', {{
+  type: 'api-key',
+  params: {{ key: '{key}', secret: '{secret}' }}
+}});
+
+async function main() {{
+{contents}
+
+console.log('🦎 Action completed successfully 🦎');
+}}
+
+main()
+.catch(error => {{
+  console.error('🚨 Action exited early with errors 🚨')
+  if (error.status !== undefined && error.result !== undefined) {{
+    console.error('Status:', error.status);
+    console.error(JSON.stringify(error.result, null, 2));
+  }} else {{
+    console.error(error);
+  }}
+  Deno.exit(1)
+}});"
+  )
+}
+
+/// Cleans up file at given path.
+/// ALSO if $DENO_DIR is set,
+/// will clean up the generated file matching "file"
+async fn cleanup_run(file: String, path: &Path) {
+  if let Err(e) = fs::remove_file(path).await {
+    warn!(
+      "Failed to delete action file after action execution | {e:#}"
+    );
+  }
+  // If $DENO_DIR is set (will be in container),
+  // will clean up the generated file matching "file" (NOT under path)
+  let Some(deno_dir) = deno_dir() else {
+    return;
+  };
+  delete_file(deno_dir.join("gen/file"), file).await;
+}
+
+fn deno_dir() -> Option<&'static Path> {
+  static DENO_DIR: OnceLock<Option<PathBuf>> = OnceLock::new();
+  DENO_DIR
+    .get_or_init(|| {
+      let deno_dir = std::env::var("DENO_DIR").ok()?;
+      PathBuf::from_str(&deno_dir).ok()
+    })
+    .as_deref()
+}
+
+/// file is just the terminating file path,
+/// it may be nested multiple folder under path,
+/// this will find the nested file and delete it.
+/// Assumes the file is only there once.
+fn delete_file(
+  dir: PathBuf,
+  file: String,
+) -> std::pin::Pin<Box<dyn std::future::Future<Output = bool> + Send>>
+{
+  Box::pin(async move {
+    let Ok(mut dir) = fs::read_dir(dir).await else {
+      return false;
+    };
+    // Collect the nested folders for recursing
+    // only after checking all the files in directory.
+    let mut folders = Vec::<PathBuf>::new();
+
+    while let Ok(Some(entry)) = dir.next_entry().await {
+      let Ok(meta) = entry.metadata().await else {
+        continue;
+      };
+      if meta.is_file() {
+        let Ok(name) = entry.file_name().into_string() else {
+          continue;
+        };
+        if name == file {
+          if let Err(e) = fs::remove_file(entry.path()).await {
+            warn!(
+              "Failed to clean up generated file after action execution | {e:#}"
+            );
+          };
+          return true;
+        }
+      } else {
+        folders.push(entry.path());
+      }
+    }
+
+    if folders.len() == 1 {
+      // unwrap ok, folders definitely is not empty
+      let folder = folders.pop().unwrap();
+      delete_file(folder, file).await
+    } else {
+      // Check folders with file.clone
+      for folder in folders {
+        if delete_file(folder, file.clone()).await {
+          return true;
+        }
+      }
+      false
+    }
+  })
+}
+
+fn parse_action_arguments(
+  args: &str,
+  format: FileFormat,
+) -> anyhow::Result<JsonObject> {
+  match format {
+    FileFormat::KeyValue => {
+      let args = parse_key_value_list(args)
+        .context("Failed to parse args as key value list")?
+        .into_iter()
+        .map(|(k, v)| (k, serde_json::Value::String(v)))
+        .collect();
+      Ok(args)
+    }
+    FileFormat::Toml => toml::from_str(args)
+      .context("Failed to parse Toml to Action args"),
+    FileFormat::Yaml => serde_yaml_ng::from_str(args)
+      .context("Failed to parse Yaml to action args"),
+    FileFormat::Json => serde_json::from_str(args)
+      .context("Failed to parse Json to action args"),
+  }
+}

bin/core/src/api/execute/alerter.rs

@@ -0,0 +1,149 @@
+use anyhow::{Context, anyhow};
+use formatting::format_serror;
+use futures::{TryStreamExt, stream::FuturesUnordered};
+use komodo_client::{
+  api::execute::{SendAlert, TestAlerter},
+  entities::{
+    alert::{Alert, AlertData, AlertDataVariant, SeverityLevel},
+    alerter::Alerter,
+    komodo_timestamp,
+    permission::PermissionLevel,
+  },
+};
+use reqwest::StatusCode;
+use resolver_api::Resolve;
+use serror::AddStatusCodeError;
+
+use crate::{
+  alert::send_alert_to_alerter, helpers::update::update_update,
+  permission::get_check_permissions, resource::list_full_for_user,
+};
+
+use super::ExecuteArgs;
+
+impl Resolve<ExecuteArgs> for TestAlerter {
+  #[instrument(name = "TestAlerter", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+  async fn resolve(
+    self,
+    ExecuteArgs { user, update }: &ExecuteArgs,
+  ) -> Result<Self::Response, Self::Error> {
+    let alerter = get_check_permissions::<Alerter>(
+      &self.alerter,
+      user,
+      PermissionLevel::Execute.into(),
+    )
+    .await?;
+
+    let mut update = update.clone();
+
+    if !alerter.config.enabled {
+      update.push_error_log(
+        "Test Alerter",
+        String::from(
+          "Alerter is disabled. Enable the Alerter to send alerts.",
+        ),
+      );
+      update.finalize();
+      update_update(update.clone()).await?;
+      return Ok(update);
+    }
+
+    let ts = komodo_timestamp();
+
+    let alert = Alert {
+      id: Default::default(),
+      ts,
+      resolved: true,
+      level: SeverityLevel::Ok,
+      target: update.target.clone(),
+      data: AlertData::Test {
+        id: alerter.id.clone(),
+        name: alerter.name.clone(),
+      },
+      resolved_ts: Some(ts),
+    };
+
+    if let Err(e) = send_alert_to_alerter(&alerter, &alert).await {
+      update.push_error_log("Test Alerter", format_serror(&e.into()));
+    } else {
+      update.push_simple_log("Test Alerter", String::from("Alert sent successfully. It should be visible at your alerting destination."));
+    };
+
+    update.finalize();
+    update_update(update.clone()).await?;
+
+    Ok(update)
+  }
+}
+
+//
+
+impl Resolve<ExecuteArgs> for SendAlert {
+  #[instrument(name = "SendAlert", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+  async fn resolve(
+    self,
+    ExecuteArgs { user, update }: &ExecuteArgs,
+  ) -> Result<Self::Response, Self::Error> {
+    let alerters = list_full_for_user::<Alerter>(
+      Default::default(),
+      user,
+      PermissionLevel::Execute.into(),
+      &[],
+    )
+    .await?
+    .into_iter()
+    .filter(|a| {
+      a.config.enabled
+        && (self.alerters.is_empty()
+          || self.alerters.contains(&a.name)
+          || self.alerters.contains(&a.id))
+        && (a.config.alert_types.is_empty()
+          || a.config.alert_types.contains(&AlertDataVariant::Custom))
+    })
+    .collect::<Vec<_>>();
+
+    if alerters.is_empty() {
+      return Err(anyhow!(
+        "Could not find any valid alerters to send to, this required Execute permissions on the Alerter"
+      ).status_code(StatusCode::BAD_REQUEST));
+    }
+
+    let mut update = update.clone();
+
+    let ts = komodo_timestamp();
+
+    let alert = Alert {
+      id: Default::default(),
+      ts,
+      resolved: true,
+      level: self.level,
+      target: update.target.clone(),
+      data: AlertData::Custom {
+        message: self.message,
+        details: self.details,
+      },
+      resolved_ts: Some(ts),
+    };
+
+    update.push_simple_log(
+      "Send alert",
+      serde_json::to_string_pretty(&alert)
+        .context("Failed to serialize alert to JSON")?,
+    );
+
+    if let Err(e) = alerters
+      .iter()
+      .map(|alerter| send_alert_to_alerter(alerter, &alert))
+      .collect::<FuturesUnordered<_>>()
+      .try_collect::<Vec<_>>()
+      .await
+    {
+      update.push_error_log("Send Error", format_serror(&e.into()));
+    };
+
+    update.finalize();
+    update_update(update.clone()).await?;
+
+    Ok(update)
+  }
+}

bin/core/src/api/execute/build.rs

@@ -1,26 +1,11 @@
-use std::{collections::HashSet, future::IntoFuture, time::Duration};
-
-use anyhow::{anyhow, Context};
-use formatting::format_serror;
-use futures::future::join_all;
-use monitor_client::{
-  api::execute::{CancelBuild, Deploy, RunBuild},
-  entities::{
-    alert::{Alert, AlertData},
-    all_logs_success,
-    build::{Build, ImageRegistry, StandardRegistryConfig},
-    builder::{Builder, BuilderConfig},
-    config::core::{AwsEcrConfig, AwsEcrConfigWithCredentials},
-    deployment::DeploymentState,
-    monitor_timestamp,
-    permission::PermissionLevel,
-    server::stats::SeverityLevel,
-    to_monitor_name,
-    update::{Log, Update},
-    user::{auto_redeploy_user, User},
-  },
+use std::{
+  collections::{HashMap, HashSet},
+  future::IntoFuture,
+  time::Duration,
 };
-use mungos::{
+
+use anyhow::{Context, anyhow};
+use database::mungos::{
   by_id::update_one_by_id,
   find::find_collect,
   mongodb::{
@@ -28,46 +13,107 @@ use mungos::{
     options::FindOneOptions,
   },
 };
-use periphery_client::api::{self, git::RepoActionResponseV1_13};
+use formatting::format_serror;
+use futures::future::join_all;
+use interpolate::Interpolator;
+use komodo_client::{
+  api::execute::{
+    BatchExecutionResponse, BatchRunBuild, CancelBuild, Deploy,
+    RunBuild,
+  },
+  entities::{
+    alert::{Alert, AlertData, SeverityLevel},
+    all_logs_success,
+    build::{Build, BuildConfig},
+    builder::{Builder, BuilderConfig},
+    deployment::DeploymentState,
+    komodo_timestamp, optional_string,
+    permission::PermissionLevel,
+    repo::Repo,
+    update::{Log, Update},
+    user::auto_redeploy_user,
+  },
+};
+use periphery_client::api;
 use resolver_api::Resolve;
 use tokio_util::sync::CancellationToken;
 
 use crate::{
-  cloud::aws::ecr,
-  config::core_config,
+  alert::send_alerts,
   helpers::{
-    alert::send_alerts,
+    build_git_token,
     builder::{cleanup_builder_instance, get_builder_periphery},
     channel::build_cancel_channel,
-    git_token,
-    query::{get_deployment_state, get_global_variables},
+    query::{
+      VariablesAndSecrets, get_deployment_state,
+      get_variables_and_secrets,
+    },
     registry_token,
-    update::update_update,
+    update::{init_execution_update, update_update},
   },
+  permission::get_check_permissions,
   resource::{self, refresh_build_state_cache},
-  state::{action_states, db_client, State},
+  state::{action_states, db_client},
 };
 
-use crate::helpers::update::init_execution_update;
+use super::{ExecuteArgs, ExecuteRequest};
 
-use super::ExecuteRequest;
+impl super::BatchExecute for BatchRunBuild {
+  type Resource = Build;
+  fn single_request(build: String) -> ExecuteRequest {
+    ExecuteRequest::RunBuild(RunBuild { build })
+  }
+}
 
-impl Resolve<RunBuild, (User, Update)> for State {
-  #[instrument(name = "RunBuild", skip(self, user, update), fields(user_id = user.id, update_id = update.id))]
+impl Resolve<ExecuteArgs> for BatchRunBuild {
+  #[instrument(name = "BatchRunBuild", skip(user), fields(user_id = user.id))]
   async fn resolve(
-    &self,
-    RunBuild { build }: RunBuild,
-    (user, mut update): (User, Update),
-  ) -> anyhow::Result<Update> {
-    let mut build = resource::get_check_permissions::<Build>(
-      &build,
-      &user,
-      PermissionLevel::Execute,
+    self,
+    ExecuteArgs { user, .. }: &ExecuteArgs,
+  ) -> serror::Result<BatchExecutionResponse> {
+    Ok(
+      super::batch_execute::<BatchRunBuild>(&self.pattern, user)
+        .await?,
+    )
+  }
+}
+
+impl Resolve<ExecuteArgs> for RunBuild {
+  #[instrument(name = "RunBuild", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+  async fn resolve(
+    self,
+    ExecuteArgs { user, update }: &ExecuteArgs,
+  ) -> serror::Result<Update> {
+    let mut build = get_check_permissions::<Build>(
+      &self.build,
+      user,
+      PermissionLevel::Execute.into(),
     )
     .await?;
 
+    let mut repo = if !build.config.files_on_host
+      && !build.config.linked_repo.is_empty()
+    {
+      crate::resource::get::<Repo>(&build.config.linked_repo)
+        .await?
+        .into()
+    } else {
+      None
+    };
+
+    let VariablesAndSecrets {
+      mut variables,
+      secrets,
+    } = get_variables_and_secrets().await?;
+
+    // Add the $VERSION to variables. Use with [[$VERSION]]
+    variables.insert(
+      String::from("$VERSION"),
+      build.config.version.to_string(),
+    );
+
     if build.config.builder_id.is_empty() {
-      return Err(anyhow!("Must attach builder to RunBuild"));
+      return Err(anyhow!("Must attach builder to RunBuild").into());
     }
 
     // get the action state for the build (or insert default).
@@ -82,21 +128,17 @@ impl Resolve<RunBuild, (User, Update)> for State {
     if build.config.auto_increment_version {
       build.config.version.increment();
     }
+
+    let mut update = update.clone();
+
     update.version = build.config.version;
     update_update(update.clone()).await?;
 
-    let git_token = git_token(
-      &build.config.git_provider,
-      &build.config.git_account,
-      |https| build.config.git_https = https,
-    )
-    .await
-    .with_context(
-      || format!("Failed to get git token in call to db. This is a database error, not a token exisitence error. Stopping run. | {} | {}", build.config.git_provider, build.config.git_account),
-    )?;
+    let git_token =
+      build_git_token(&mut build, repo.as_mut()).await?;
 
-    let (registry_token, aws_ecr) =
-      validate_account_extract_registry_token_aws_ecr(&build).await?;
+    let registry_tokens =
+      validate_account_extract_registry_tokens(&build).await?;
 
     let cancel = CancellationToken::new();
     let cancel_clone = cancel.clone();
@@ -144,7 +186,6 @@ impl Resolve<RunBuild, (User, Update)> for State {
     });
 
     // GET BUILDER PERIPHERY
-
     let (periphery, cleanup_data) = match get_builder_periphery(
       build.name.clone(),
       Some(build.config.version),
@@ -170,158 +211,94 @@ impl Resolve<RunBuild, (User, Update)> for State {
       }
     };
 
-    // CLONE REPO
+    // INTERPOLATE VARIABLES
+    let secret_replacers = if !build.config.skip_secret_interp {
+      let mut interpolator =
+        Interpolator::new(Some(&variables), &secrets);
 
-    let res = tokio::select! {
-      res = periphery
-        .request(api::git::CloneRepo {
-          args: (&build).into(),
-          git_token,
-          environment: Default::default(),
-          env_file_path: Default::default(),
-          skip_secret_interp: Default::default(),
-        }) => res,
-      _ = cancel.cancelled() => {
-        debug!("build cancelled during clone, cleaning up builder");
-        update.push_error_log("build cancelled", String::from("user cancelled build during repo clone"));
-        cleanup_builder_instance(periphery, cleanup_data, &mut update)
-          .await;
-        info!("builder cleaned up");
-        return handle_early_return(update, build.id, build.name, true).await
-      },
+      interpolator.interpolate_build(&mut build)?;
+
+      if let Some(repo) = repo.as_mut() {
+        interpolator.interpolate_repo(repo)?;
+      }
+
+      interpolator.push_logs(&mut update.logs);
+
+      interpolator.secret_replacers
+    } else {
+      Default::default()
     };
 
-    let commit_message = match res {
-      Ok(res) => {
-        debug!("finished repo clone");
-        let res: RepoActionResponseV1_13 = res.into();
-        update.logs.extend(res.logs);
-        update.commit_hash =
-          res.commit_hash.unwrap_or_default().to_string();
-        res.commit_message.unwrap_or_default()
-      }
-      Err(e) => {
-        warn!("failed build at clone repo | {e:#}");
-        update.push_error_log(
-          "clone repo",
-          format_serror(&e.context("failed to clone repo").into()),
-        );
-        Default::default()
-      }
-    };
-
-    update_update(update.clone()).await?;
-
-    if all_logs_success(&update.logs) {
-      let secret_replacers = if !build.config.skip_secret_interp {
-        let core_config = core_config();
-        let variables = get_global_variables().await?;
-        // Interpolate variables / secrets into build args
-        let mut global_replacers = HashSet::new();
-        let mut secret_replacers = HashSet::new();
-        let mut secret_replacers_for_log = HashSet::new();
-
-        // Interpolate into build args
-        for arg in &mut build.config.build_args {
-          // first pass - global variables
-          let (res, more_replacers) = svi::interpolate_variables(
-            &arg.value,
-            &variables,
-            svi::Interpolator::DoubleBrackets,
-            false,
-          )
-          .context("failed to interpolate global variables")?;
-          global_replacers.extend(more_replacers);
-          // second pass - core secrets
-          let (res, more_replacers) = svi::interpolate_variables(
-            &res,
-            &core_config.secrets,
-            svi::Interpolator::DoubleBrackets,
-            false,
-          )
-          .context("failed to interpolate core secrets")?;
-          secret_replacers_for_log.extend(
-            more_replacers
-              .iter()
-              .map(|(_, variable)| variable.clone()),
-          );
-          secret_replacers.extend(more_replacers);
-          arg.value = res;
-        }
-
-        // Interpolate into secret args
-        for arg in &mut build.config.secret_args {
-          // first pass - global variables
-          let (res, more_replacers) = svi::interpolate_variables(
-            &arg.value,
-            &variables,
-            svi::Interpolator::DoubleBrackets,
-            false,
-          )
-          .context("failed to interpolate global variables")?;
-          global_replacers.extend(more_replacers);
-          // second pass - core secrets
-          let (res, more_replacers) = svi::interpolate_variables(
-            &res,
-            &core_config.secrets,
-            svi::Interpolator::DoubleBrackets,
-            false,
-          )
-          .context("failed to interpolate core secrets")?;
-          secret_replacers_for_log.extend(
-            more_replacers.into_iter().map(|(_, variable)| variable),
-          );
-          // Secret args don't need to be in replacers sent to periphery.
-          // The secret args don't end up in the command like build args do.
-          arg.value = res;
-        }
-
-        // Show which variables were interpolated
-        if !global_replacers.is_empty() {
-          update.push_simple_log(
-            "interpolate global variables",
-            global_replacers
-              .into_iter()
-              .map(|(value, variable)| format!("<span class=\"text-muted-foreground\">{variable} =></span> {value}"))
-              .collect::<Vec<_>>()
-              .join("\n"),
-          );
-        }
-
-        if !secret_replacers_for_log.is_empty() {
-          update.push_simple_log(
-            "interpolate core secrets",
-            secret_replacers_for_log
-              .into_iter()
-              .map(|variable| format!("<span class=\"text-muted-foreground\">replaced:</span> {variable}"))
-              .collect::<Vec<_>>()
-              .join("\n"),
-          );
-        }
-
-        secret_replacers
-      } else {
-        Default::default()
+    let commit_message = if !build.config.files_on_host
+      && (!build.config.repo.is_empty()
+        || !build.config.linked_repo.is_empty())
+    {
+      // PULL OR CLONE REPO
+      let res = tokio::select! {
+        res = periphery
+          .request(api::git::PullOrCloneRepo {
+            args: repo.as_ref().map(Into::into).unwrap_or((&build).into()),
+            git_token,
+            environment: Default::default(),
+            env_file_path: Default::default(),
+            on_clone: None,
+            on_pull: None,
+            skip_secret_interp: Default::default(),
+            replacers: Default::default(),
+          }) => res,
+        _ = cancel.cancelled() => {
+          debug!("build cancelled during clone, cleaning up builder");
+          update.push_error_log("build cancelled", String::from("user cancelled build during repo clone"));
+          cleanup_builder_instance(cleanup_data, &mut update)
+            .await;
+          info!("builder cleaned up");
+          return handle_early_return(update, build.id, build.name, true).await
+        },
       };
 
+      let commit_message = match res {
+        Ok(res) => {
+          debug!("finished repo clone");
+          update.logs.extend(res.res.logs);
+          update.commit_hash =
+            res.res.commit_hash.unwrap_or_default().to_string();
+          res.res.commit_message.unwrap_or_default()
+        }
+        Err(e) => {
+          warn!("Failed build at clone repo | {e:#}");
+          update.push_error_log(
+            "Clone Repo",
+            format_serror(&e.context("Failed to clone repo").into()),
+          );
+          Default::default()
+        }
+      };
+
+      update_update(update.clone()).await?;
+
+      Some(commit_message)
+    } else {
+      None
+    };
+
+    if all_logs_success(&update.logs) {
+      // RUN BUILD
       let res = tokio::select! {
         res = periphery
           .request(api::build::Build {
             build: build.clone(),
-            registry_token,
-            aws_ecr,
+            repo,
+            registry_tokens,
             replacers: secret_replacers.into_iter().collect(),
-            // Push a commit hash tagged image
-            additional_tags: if update.commit_hash.is_empty() {
-              Default::default()
-            } else {
-              vec![update.commit_hash.clone()]
-            },
+            // To push a commit hash tagged image
+            commit_hash: optional_string(&update.commit_hash),
+            // Unused for now
+            additional_tags: Default::default(),
           }) => res.context("failed at call to periphery to build"),
         _ = cancel.cancelled() => {
           info!("build cancelled during build, cleaning up builder");
           update.push_error_log("build cancelled", String::from("user cancelled build during docker build"));
-          cleanup_builder_instance(periphery, cleanup_data, &mut update)
+          cleanup_builder_instance(cleanup_data, &mut update)
             .await;
           return handle_early_return(update, build.id, build.name, true).await
         },
@@ -344,7 +321,7 @@ impl Resolve<RunBuild, (User, Update)> for State {
 
     update.finalize();
 
-    let db = db_client().await;
+    let db = db_client();
 
     if update.success {
       let _ = db
@@ -354,7 +331,7 @@ impl Resolve<RunBuild, (User, Update)> for State {
           doc! { "$set": {
             "config.version": to_bson(&build.config.version)
               .context("failed at converting version to bson")?,
-            "info.last_built_at": monitor_timestamp(),
+            "info.last_built_at": komodo_timestamp(),
             "info.built_hash": &update.commit_hash,
             "info.built_message": commit_message
           }},
@@ -365,8 +342,9 @@ impl Resolve<RunBuild, (User, Update)> for State {
     // stop the cancel listening task from going forever
     cancel.cancel();
 
-    cleanup_builder_instance(periphery, cleanup_data, &mut update)
-      .await;
+    // If building on temporary cloud server (AWS),
+    // this will terminate the server.
+    cleanup_builder_instance(cleanup_data, &mut update).await;
 
     // Need to manually update the update before cache refresh,
     // and before broadcast with add_update.
@@ -376,7 +354,7 @@ impl Resolve<RunBuild, (User, Update)> for State {
       let _ = update_one_by_id(
         &db.updates,
         &update.id,
-        mungos::update::Update::Set(update_doc),
+        database::mungos::update::Update::Set(update_doc),
         None,
       )
       .await;
@@ -398,8 +376,8 @@ impl Resolve<RunBuild, (User, Update)> for State {
         let alert = Alert {
           id: Default::default(),
           target,
-          ts: monitor_timestamp(),
-          resolved_ts: Some(monitor_timestamp()),
+          ts: komodo_timestamp(),
+          resolved_ts: Some(komodo_timestamp()),
           resolved: true,
           level: SeverityLevel::Warning,
           data: AlertData::BuildFailed {
@@ -412,7 +390,7 @@ impl Resolve<RunBuild, (User, Update)> for State {
       });
     }
 
-    Ok(update)
+    Ok(update.clone())
   }
 }
 
@@ -422,7 +400,7 @@ async fn handle_early_return(
   build_id: String,
   build_name: String,
   is_cancel: bool,
-) -> anyhow::Result<Update> {
+) -> serror::Result<Update> {
   update.finalize();
   // Need to manually update the update before cache refresh,
   // and before broadcast with add_update.
@@ -430,9 +408,9 @@ async fn handle_early_return(
   // but will fail to update cache in that case.
   if let Ok(update_doc) = to_document(&update) {
     let _ = update_one_by_id(
-      &db_client().await.updates,
+      &db_client().updates,
       &update.id,
-      mungos::update::Update::Set(update_doc),
+      database::mungos::update::Update::Set(update_doc),
       None,
     )
     .await;
@@ -447,8 +425,8 @@ async fn handle_early_return(
       let alert = Alert {
         id: Default::default(),
         target,
-        ts: monitor_timestamp(),
-        resolved_ts: Some(monitor_timestamp()),
+        ts: komodo_timestamp(),
+        resolved_ts: Some(komodo_timestamp()),
         resolved: true,
         level: SeverityLevel::Warning,
         data: AlertData::BuildFailed {
@@ -460,17 +438,16 @@ async fn handle_early_return(
       send_alerts(&[alert]).await
     });
   }
-  Ok(update)
+  Ok(update.clone())
 }
 
-#[instrument(skip_all)]
 pub async fn validate_cancel_build(
   request: &ExecuteRequest,
 ) -> anyhow::Result<()> {
   if let ExecuteRequest::CancelBuild(req) = request {
     let build = resource::get::<Build>(&req.build).await?;
 
-    let db = db_client().await;
+    let db = db_client();
 
     let (latest_build, latest_cancel) = tokio::try_join!(
       db.updates
@@ -510,17 +487,16 @@ pub async fn validate_cancel_build(
   Ok(())
 }
 
-impl Resolve<CancelBuild, (User, Update)> for State {
-  #[instrument(name = "CancelBuild", skip(self, user, update), fields(user_id = user.id, update_id = update.id))]
+impl Resolve<ExecuteArgs> for CancelBuild {
+  #[instrument(name = "CancelBuild", skip(user, update), fields(user_id = user.id, update_id = update.id))]
   async fn resolve(
-    &self,
-    CancelBuild { build }: CancelBuild,
-    (user, mut update): (User, Update),
-  ) -> anyhow::Result<Update> {
-    let build = resource::get_check_permissions::<Build>(
-      &build,
-      &user,
-      PermissionLevel::Execute,
+    self,
+    ExecuteArgs { user, update }: &ExecuteArgs,
+  ) -> serror::Result<Update> {
+    let build = get_check_permissions::<Build>(
+      &self.build,
+      user,
+      PermissionLevel::Execute.into(),
     )
     .await?;
 
@@ -532,9 +508,11 @@ impl Resolve<CancelBuild, (User, Update)> for State {
       .and_then(|s| s.get().ok().map(|s| s.building))
       .unwrap_or_default()
     {
-      return Err(anyhow!("Build is not building."));
+      return Err(anyhow!("Build is not building.").into());
     }
 
+    let mut update = update.clone();
+
     update.push_simple_log(
       "cancel triggered",
       "the build cancel has been triggered",
@@ -553,14 +531,16 @@ impl Resolve<CancelBuild, (User, Update)> for State {
     tokio::spawn(async move {
       tokio::time::sleep(Duration::from_secs(60)).await;
       if let Err(e) = update_one_by_id(
-        &db_client().await.updates,
+        &db_client().updates,
         &update_id,
         doc! { "$set": { "status": "Complete" } },
         None,
       )
       .await
       {
-        warn!("failed to set CancelBuild Update status Complete after timeout | {e:#}")
+        warn!(
+          "failed to set CancelBuild Update status Complete after timeout | {e:#}"
+        )
       }
     });
 
@@ -571,7 +551,7 @@ impl Resolve<CancelBuild, (User, Update)> for State {
 #[instrument]
 async fn handle_post_build_redeploy(build_id: &str) {
   let Ok(redeploy_deployments) = find_collect(
-    &db_client().await.deployments,
+    &db_client().deployments,
     doc! {
       "config.image.params.build_id": build_id,
       "config.redeploy_on_build": true
@@ -587,8 +567,9 @@ async fn handle_post_build_redeploy(build_id: &str) {
     redeploy_deployments
       .into_iter()
       .map(|deployment| async move {
-        let state =
-          get_deployment_state(&deployment).await.unwrap_or_default();
+        let state = get_deployment_state(&deployment.id)
+          .await
+          .unwrap_or_default();
         if state == DeploymentState::Running {
           let req = super::ExecuteRequest::Deploy(Deploy {
             deployment: deployment.id.clone(),
@@ -598,16 +579,13 @@ async fn handle_post_build_redeploy(build_id: &str) {
           let user = auto_redeploy_user().to_owned();
           let res = async {
             let update = init_execution_update(&req, &user).await?;
-            State
-              .resolve(
-                Deploy {
-                  deployment: deployment.id.clone(),
-                  stop_signal: None,
-                  stop_time: None,
-                },
-                (user, update),
-              )
-              .await
+            Deploy {
+              deployment: deployment.id.clone(),
+              stop_signal: None,
+              stop_time: None,
+            }
+            .resolve(&ExecuteArgs { user, update })
+            .await
           }
           .await;
           Some((deployment.id.clone(), res))
@@ -621,71 +599,59 @@ async fn handle_post_build_redeploy(build_id: &str) {
       continue;
     };
     if let Err(e) = res {
-      warn!("failed post build redeploy for deployment {id}: {e:#}");
+      warn!(
+        "failed post build redeploy for deployment {id}: {:#}",
+        e.error
+      );
     }
   }
 }
 
 /// This will make sure that a build with non-none image registry has an account attached,
-/// and will check the core config for a token / aws ecr config matching requirements.
+/// and will check the core config for a token matching requirements.
 /// Otherwise it is left to periphery.
-async fn validate_account_extract_registry_token_aws_ecr(
-  build: &Build,
-) -> anyhow::Result<(Option<String>, Option<AwsEcrConfig>)> {
-  let (domain, account) = match &build.config.image_registry {
-    // Early return for None
-    ImageRegistry::None(_) => return Ok((None, None)),
-    // Early return for AwsEcr
-    ImageRegistry::AwsEcr(label) => {
-      // Note that aws ecr config still only lives in config file
-      let config = core_config()
-        .aws_ecr_registries
-        .iter()
-        .find(|reg| &reg.label == label);
-      let token = match config {
-        Some(AwsEcrConfigWithCredentials {
-          region,
-          access_key_id,
-          secret_access_key,
-          ..
-        }) => {
-          let token = ecr::get_ecr_token(
-            region,
-            access_key_id,
-            secret_access_key,
-          )
-          .await
-          .context("failed to get aws ecr token")?;
-          ecr::maybe_create_repo(
-            &to_monitor_name(&build.name),
-            region.to_string(),
-            access_key_id,
-            secret_access_key,
-          )
-          .await
-          .context("failed to create aws ecr repo")?;
-          Some(token)
-        }
-        None => None,
-      };
-      return Ok((token, config.map(AwsEcrConfig::from)));
-    }
-    ImageRegistry::Standard(StandardRegistryConfig {
-      domain,
-      account,
-      ..
-    }) => (domain.as_str(), account),
-  };
+async fn validate_account_extract_registry_tokens(
+  Build {
+    config: BuildConfig { image_registry, .. },
+    ..
+  }: &Build,
+  // Maps (domain, account) -> token
+) -> serror::Result<Vec<(String, String, String)>> {
+  let mut res = HashMap::with_capacity(image_registry.capacity());
 
-  if account.is_empty() {
-    return Err(anyhow!(
-      "Must attach account to use registry provider {domain}"
-    ));
+  for (domain, account) in image_registry
+    .iter()
+    .map(|r| (r.domain.as_str(), r.account.as_str()))
+    // This ensures uniqueness / prevents redundant logins
+    .collect::<HashSet<_>>()
+  {
+    if domain.is_empty() {
+      continue;
+    }
+    if account.is_empty() {
+      return Err(
+        anyhow!(
+          "Must attach account to use registry provider {domain}"
+        )
+        .into(),
+      );
+    }
+    let Some(registry_token) = registry_token(domain, account).await.with_context(
+      || format!("Failed to get registry token in call to db. Stopping run. | {domain} | {account}"),
+    )? else {
+      continue;
+    };
+
+    res.insert(
+      (domain.to_string(), account.to_string()),
+      registry_token,
+    );
   }
 
-  let registry_token = registry_token(domain, account).await.with_context(
-    || format!("Failed to get registry token in call to db. Stopping run. | {domain} | {account}"),
-  )?;
-
-  Ok((registry_token, None))
+  Ok(
+    res
+      .into_iter()
+      .map(|((domain, account), token)| (domain, account, token))
+      .collect(),
+  )
 }

bin/core/src/api/execute/deployment.rs

@@ -1,76 +1,99 @@
-use anyhow::{anyhow, Context};
+use std::sync::OnceLock;
+
+use anyhow::{Context, anyhow};
+use cache::TimeoutCache;
 use formatting::format_serror;
-use monitor_client::{
+use interpolate::Interpolator;
+use komodo_client::{
   api::execute::*,
   entities::{
-    build::{Build, ImageRegistry},
-    config::core::AwsEcrConfig,
+    Version,
+    build::{Build, ImageRegistryConfig},
     deployment::{
-      extract_registry_domain, Deployment, DeploymentImage,
+      Deployment, DeploymentImage, extract_registry_domain,
     },
-    get_image_name,
+    komodo_timestamp, optional_string,
     permission::PermissionLevel,
-    server::{Server, ServerState},
+    server::Server,
     update::{Log, Update},
     user::User,
-    Version,
   },
 };
 use periphery_client::api;
 use resolver_api::Resolve;
 
 use crate::{
-  cloud::aws::ecr,
-  config::core_config,
   helpers::{
-    interpolate_variables_secrets_into_environment, periphery_client,
-    query::get_server_with_status, registry_token,
+    periphery_client,
+    query::{VariablesAndSecrets, get_variables_and_secrets},
+    registry_token,
     update::update_update,
   },
   monitor::update_cache_for_server,
+  permission::get_check_permissions,
   resource,
-  state::{action_states, State},
+  state::action_states,
 };
 
+use super::{ExecuteArgs, ExecuteRequest};
+
+impl super::BatchExecute for BatchDeploy {
+  type Resource = Deployment;
+  fn single_request(deployment: String) -> ExecuteRequest {
+    ExecuteRequest::Deploy(Deploy {
+      deployment,
+      stop_signal: None,
+      stop_time: None,
+    })
+  }
+}
+
+impl Resolve<ExecuteArgs> for BatchDeploy {
+  #[instrument(name = "BatchDeploy", skip(user), fields(user_id = user.id))]
+  async fn resolve(
+    self,
+    ExecuteArgs { user, .. }: &ExecuteArgs,
+  ) -> serror::Result<BatchExecutionResponse> {
+    Ok(
+      super::batch_execute::<BatchDeploy>(&self.pattern, user)
+        .await?,
+    )
+  }
+}
+
 async fn setup_deployment_execution(
   deployment: &str,
   user: &User,
 ) -> anyhow::Result<(Deployment, Server)> {
-  let deployment = resource::get_check_permissions::<Deployment>(
+  let deployment = get_check_permissions::<Deployment>(
     deployment,
     user,
-    PermissionLevel::Execute,
+    PermissionLevel::Execute.into(),
   )
   .await?;
 
   if deployment.config.server_id.is_empty() {
-    return Err(anyhow!("deployment has no server configured"));
+    return Err(anyhow!("Deployment has no Server configured"));
   }
 
-  let (server, status) =
-    get_server_with_status(&deployment.config.server_id).await?;
-  if status != ServerState::Ok {
-    return Err(anyhow!(
-      "cannot send action when server is unreachable or disabled"
-    ));
+  let server =
+    resource::get::<Server>(&deployment.config.server_id).await?;
+
+  if !server.config.enabled {
+    return Err(anyhow!("Attached Server is not enabled"));
   }
 
   Ok((deployment, server))
 }
 
-impl Resolve<Deploy, (User, Update)> for State {
-  #[instrument(name = "Deploy", skip(self, user, update), fields(user_id = user.id, update_id = update.id))]
+impl Resolve<ExecuteArgs> for Deploy {
+  #[instrument(name = "Deploy", skip(user, update), fields(user_id = user.id, update_id = update.id))]
   async fn resolve(
-    &self,
-    Deploy {
-      deployment,
-      stop_signal,
-      stop_time,
-    }: Deploy,
-    (user, mut update): (User, Update),
-  ) -> anyhow::Result<Update> {
+    self,
+    ExecuteArgs { user, update }: &ExecuteArgs,
+  ) -> serror::Result<Update> {
     let (mut deployment, server) =
-      setup_deployment_execution(&deployment, &user).await?;
+      setup_deployment_execution(&self.deployment, user).await?;
 
     // get the action state for the deployment (or insert default).
     let action_state = action_states()
@@ -83,36 +106,26 @@ impl Resolve<Deploy, (User, Update)> for State {
     let _action_guard =
       action_state.update(|state| state.deploying = true)?;
 
+    let mut update = update.clone();
+
     // Send update after setting action state, this way frontend gets correct state.
     update_update(update.clone()).await?;
 
-    let periphery = periphery_client(&server)?;
-
-    let (version, registry_token, aws_ecr) = match &deployment
-      .config
-      .image
-    {
+    // This block resolves the attached Build to an actual versioned image
+    let (version, registry_token) = match &deployment.config.image {
       DeploymentImage::Build { build_id, version } => {
         let build = resource::get::<Build>(build_id).await?;
-        let image_name = get_image_name(&build, |label| {
-          core_config()
-            .aws_ecr_registries
-            .iter()
-            .find(|reg| &reg.label == label)
-            .map(AwsEcrConfig::from)
-        })
-        .context("failed to create image name")?;
+        let image_names = build.get_image_names();
+        let image_name = image_names
+          .first()
+          .context("No image name could be created")
+          .context("Failed to create image name")?;
         let version = if version.is_none() {
           build.config.version
         } else {
           *version
         };
-        // Remove ending patch if it is 0, this means use latest patch.
-        let version_str = if version.patch == 0 {
-          format!("{}.{}", version.major, version.minor)
-        } else {
-          version.to_string()
-        };
+        let version_str = version.to_string();
         // Potentially add the build image_tag postfix
         let version_str = if build.config.image_tag.is_empty() {
           version_str
@@ -123,45 +136,33 @@ impl Resolve<Deploy, (User, Update)> for State {
         deployment.config.image = DeploymentImage::Image {
           image: format!("{image_name}:{version_str}"),
         };
-        match build.config.image_registry {
-          ImageRegistry::None(_) => (version, None, None),
-          ImageRegistry::AwsEcr(label) => {
-            let config = core_config()
-              .aws_ecr_registries
-              .iter()
-              .find(|reg| reg.label == label)
-              .with_context(|| {
-                format!(
-                  "did not find config for aws ecr registry {label}"
-                )
-              })?;
-            let token = ecr::get_ecr_token(
-              &config.region,
-              &config.access_key_id,
-              &config.secret_access_key,
-            )
-            .await
-            .context("failed to create aws ecr login token")?;
-            (version, Some(token), Some(AwsEcrConfig::from(config)))
-          }
-          ImageRegistry::Standard(params) => {
-            if deployment.config.image_registry_account.is_empty() {
-              deployment.config.image_registry_account =
-                params.account
-            }
-            let token = if !deployment
-              .config
-              .image_registry_account
-              .is_empty()
-            {
-              registry_token(&params.domain, &deployment.config.image_registry_account).await.with_context(
-                || format!("Failed to get git token in call to db. Stopping run. | {} | {}", params.domain, deployment.config.image_registry_account),
-              )?
-            } else {
-              None
-            };
-            (version, token, None)
+        let first_registry = build
+          .config
+          .image_registry
+          .first()
+          .unwrap_or(ImageRegistryConfig::static_default());
+        if first_registry.domain.is_empty() {
+          (version, None)
+        } else {
+          let ImageRegistryConfig {
+            domain, account, ..
+          } = first_registry;
+          if deployment.config.image_registry_account.is_empty() {
+            deployment.config.image_registry_account =
+              account.to_string();
           }
+          let token = if !deployment
+            .config
+            .image_registry_account
+            .is_empty()
+          {
+            registry_token(domain, &deployment.config.image_registry_account).await.with_context(
+              || format!("Failed to get git token in call to db. Stopping run. | {domain} | {}", deployment.config.image_registry_account),
+            )?
+          } else {
+            None
+          };
+          (version, token)
         }
       }
       DeploymentImage::Image { image } => {
@@ -177,16 +178,24 @@ impl Resolve<Deploy, (User, Update)> for State {
         } else {
           None
         };
-        (Version::default(), token, None)
+        (Version::default(), token)
       }
     };
 
+    // interpolate variables / secrets, returning the sanitizing replacers to send to
+    // periphery so it may sanitize the final command for safe logging (avoids exposing secret values)
     let secret_replacers = if !deployment.config.skip_secret_interp {
-      interpolate_variables_secrets_into_environment(
-        &mut deployment.config.environment,
-        &mut update,
-      )
-      .await?
+      let VariablesAndSecrets { variables, secrets } =
+        get_variables_and_secrets().await?;
+
+      let mut interpolator =
+        Interpolator::new(Some(&variables), &secrets);
+
+      interpolator
+        .interpolate_deployment(&mut deployment)?
+        .push_logs(&mut update.logs);
+
+      interpolator.secret_replacers
     } else {
       Default::default()
     };
@@ -194,13 +203,12 @@ impl Resolve<Deploy, (User, Update)> for State {
     update.version = version;
     update_update(update.clone()).await?;
 
-    match periphery
+    match periphery_client(&server)?
       .request(api::container::Deploy {
         deployment,
-        stop_signal,
-        stop_time,
+        stop_signal: self.stop_signal,
+        stop_time: self.stop_time,
         registry_token,
-        aws_ecr,
         replacers: secret_replacers.into_iter().collect(),
       })
       .await
@@ -208,15 +216,13 @@ impl Resolve<Deploy, (User, Update)> for State {
       Ok(log) => update.logs.push(log),
       Err(e) => {
         update.push_error_log(
-          "deploy container",
-          format_serror(
-            &e.context("failed to deploy container").into(),
-          ),
+          "Deploy Container",
+          format_serror(&e.into()),
         );
       }
     };
 
-    update_cache_for_server(&server).await;
+    update_cache_for_server(&server, true).await;
 
     update.finalize();
     update_update(update.clone()).await?;
@@ -225,15 +231,172 @@ impl Resolve<Deploy, (User, Update)> for State {
   }
 }
 
-impl Resolve<StartContainer, (User, Update)> for State {
-  #[instrument(name = "StartContainer", skip(self, user, update), fields(user_id = user.id, update_id = update.id))]
+/// Wait this long after a pull to allow another pull through
+const PULL_TIMEOUT: i64 = 5_000;
+type ServerId = String;
+type Image = String;
+type PullCache = TimeoutCache<(ServerId, Image), Log>;
+
+fn pull_cache() -> &'static PullCache {
+  static PULL_CACHE: OnceLock<PullCache> = OnceLock::new();
+  PULL_CACHE.get_or_init(Default::default)
+}
+
+pub async fn pull_deployment_inner(
+  deployment: Deployment,
+  server: &Server,
+) -> anyhow::Result<Log> {
+  let (image, account, token) = match deployment.config.image {
+    DeploymentImage::Build { build_id, version } => {
+      let build = resource::get::<Build>(&build_id).await?;
+      let image_names = build.get_image_names();
+      let image_name = image_names
+        .first()
+        .context("No image name could be created")
+        .context("Failed to create image name")?;
+      let version = if version.is_none() {
+        build.config.version.to_string()
+      } else {
+        version.to_string()
+      };
+      // Potentially add the build image_tag postfix
+      let version = if build.config.image_tag.is_empty() {
+        version
+      } else {
+        format!("{version}-{}", build.config.image_tag)
+      };
+      // replace image with corresponding build image.
+      let image = format!("{image_name}:{version}");
+      let first_registry = build
+        .config
+        .image_registry
+        .first()
+        .unwrap_or(ImageRegistryConfig::static_default());
+      if first_registry.domain.is_empty() {
+        (image, None, None)
+      } else {
+        let ImageRegistryConfig {
+          domain, account, ..
+        } = first_registry;
+        let account =
+          if deployment.config.image_registry_account.is_empty() {
+            account
+          } else {
+            &deployment.config.image_registry_account
+          };
+        let token = if !account.is_empty() {
+          registry_token(domain, account).await.with_context(
+              || format!("Failed to get git token in call to db. Stopping run. | {domain} | {account}"),
+            )?
+        } else {
+          None
+        };
+        (image, optional_string(account), token)
+      }
+    }
+    DeploymentImage::Image { image } => {
+      let domain = extract_registry_domain(&image)?;
+      let token = if !deployment
+        .config
+        .image_registry_account
+        .is_empty()
+      {
+        registry_token(&domain, &deployment.config.image_registry_account).await.with_context(
+            || format!("Failed to get git token in call to db. Stopping run. | {domain} | {}", deployment.config.image_registry_account),
+          )?
+      } else {
+        None
+      };
+      (
+        image,
+        optional_string(&deployment.config.image_registry_account),
+        token,
+      )
+    }
+  };
+
+  // Acquire the pull lock for this image on the server
+  let lock = pull_cache()
+    .get_lock((server.id.clone(), image.clone()))
+    .await;
+
+  // Lock the path lock, prevents simultaneous pulls by
+  // ensuring simultaneous pulls will wait for first to finish
+  // and checking cached results.
+  let mut locked = lock.lock().await;
+
+  // Early return from cache if lasted pulled with PULL_TIMEOUT
+  if locked.last_ts + PULL_TIMEOUT > komodo_timestamp() {
+    return locked.clone_res();
+  }
+
+  let res = async {
+    let log = match periphery_client(server)?
+      .request(api::image::PullImage {
+        name: image,
+        account,
+        token,
+      })
+      .await
+    {
+      Ok(log) => log,
+      Err(e) => Log::error("Pull image", format_serror(&e.into())),
+    };
+
+    update_cache_for_server(server, true).await;
+    anyhow::Ok(log)
+  }
+  .await;
+
+  // Set the cache with results. Any other calls waiting on the lock will
+  // then immediately also use this same result.
+  locked.set(&res, komodo_timestamp());
+
+  res
+}
+
+impl Resolve<ExecuteArgs> for PullDeployment {
+  #[instrument(name = "PullDeployment", skip(user, update), fields(user_id = user.id, update_id = update.id))]
   async fn resolve(
-    &self,
-    StartContainer { deployment }: StartContainer,
-    (user, mut update): (User, Update),
-  ) -> anyhow::Result<Update> {
+    self,
+    ExecuteArgs { user, update }: &ExecuteArgs,
+  ) -> serror::Result<Update> {
     let (deployment, server) =
-      setup_deployment_execution(&deployment, &user).await?;
+      setup_deployment_execution(&self.deployment, user).await?;
+
+    // get the action state for the deployment (or insert default).
+    let action_state = action_states()
+      .deployment
+      .get_or_insert_default(&deployment.id)
+      .await;
+
+    // Will check to ensure deployment not already busy before updating, and return Err if so.
+    // The returned guard will set the action state back to default when dropped.
+    let _action_guard =
+      action_state.update(|state| state.pulling = true)?;
+
+    let mut update = update.clone();
+    // Send update after setting action state, this way frontend gets correct state.
+    update_update(update.clone()).await?;
+
+    let log = pull_deployment_inner(deployment, &server).await?;
+
+    update.logs.push(log);
+    update.finalize();
+    update_update(update.clone()).await?;
+
+    Ok(update)
+  }
+}
+
+impl Resolve<ExecuteArgs> for StartDeployment {
+  #[instrument(name = "StartDeployment", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+  async fn resolve(
+    self,
+    ExecuteArgs { user, update }: &ExecuteArgs,
+  ) -> serror::Result<Update> {
+    let (deployment, server) =
+      setup_deployment_execution(&self.deployment, user).await?;
 
     // get the action state for the deployment (or insert default).
     let action_state = action_states()
@@ -246,14 +409,14 @@ impl Resolve<StartContainer, (User, Update)> for State {
     let _action_guard =
       action_state.update(|state| state.starting = true)?;
 
+    let mut update = update.clone();
+
     // Send update after setting action state, this way frontend gets correct state.
     update_update(update.clone()).await?;
 
-    let periphery = periphery_client(&server)?;
-
-    let log = match periphery
+    let log = match periphery_client(&server)?
       .request(api::container::StartContainer {
-        name: deployment.name.clone(),
+        name: deployment.name,
       })
       .await
     {
@@ -265,7 +428,7 @@ impl Resolve<StartContainer, (User, Update)> for State {
     };
 
     update.logs.push(log);
-    update_cache_for_server(&server).await;
+    update_cache_for_server(&server, true).await;
     update.finalize();
     update_update(update.clone()).await?;
 
@@ -273,15 +436,14 @@ impl Resolve<StartContainer, (User, Update)> for State {
   }
 }
 
-impl Resolve<RestartContainer, (User, Update)> for State {
-  #[instrument(name = "RestartContainer", skip(self, user, update), fields(user_id = user.id, update_id = update.id))]
+impl Resolve<ExecuteArgs> for RestartDeployment {
+  #[instrument(name = "RestartDeployment", skip(user, update), fields(user_id = user.id, update_id = update.id))]
   async fn resolve(
-    &self,
-    RestartContainer { deployment }: RestartContainer,
-    (user, mut update): (User, Update),
-  ) -> anyhow::Result<Update> {
+    self,
+    ExecuteArgs { user, update }: &ExecuteArgs,
+  ) -> serror::Result<Update> {
     let (deployment, server) =
-      setup_deployment_execution(&deployment, &user).await?;
+      setup_deployment_execution(&self.deployment, user).await?;
 
     // get the action state for the deployment (or insert default).
     let action_state = action_states()
@@ -294,14 +456,14 @@ impl Resolve<RestartContainer, (User, Update)> for State {
     let _action_guard =
       action_state.update(|state| state.restarting = true)?;
 
+    let mut update = update.clone();
+
     // Send update after setting action state, this way frontend gets correct state.
     update_update(update.clone()).await?;
 
-    let periphery = periphery_client(&server)?;
-
-    let log = match periphery
+    let log = match periphery_client(&server)?
       .request(api::container::RestartContainer {
-        name: deployment.name.clone(),
+        name: deployment.name,
       })
       .await
     {
@@ -315,7 +477,7 @@ impl Resolve<RestartContainer, (User, Update)> for State {
     };
 
     update.logs.push(log);
-    update_cache_for_server(&server).await;
+    update_cache_for_server(&server, true).await;
     update.finalize();
     update_update(update.clone()).await?;
 
@@ -323,15 +485,14 @@ impl Resolve<RestartContainer, (User, Update)> for State {
   }
 }
 
-impl Resolve<PauseContainer, (User, Update)> for State {
-  #[instrument(name = "PauseContainer", skip(self, user, update), fields(user_id = user.id, update_id = update.id))]
+impl Resolve<ExecuteArgs> for PauseDeployment {
+  #[instrument(name = "PauseDeployment", skip(user, update), fields(user_id = user.id, update_id = update.id))]
   async fn resolve(
-    &self,
-    PauseContainer { deployment }: PauseContainer,
-    (user, mut update): (User, Update),
-  ) -> anyhow::Result<Update> {
+    self,
+    ExecuteArgs { user, update }: &ExecuteArgs,
+  ) -> serror::Result<Update> {
     let (deployment, server) =
-      setup_deployment_execution(&deployment, &user).await?;
+      setup_deployment_execution(&self.deployment, user).await?;
 
     // get the action state for the deployment (or insert default).
     let action_state = action_states()
@@ -344,14 +505,14 @@ impl Resolve<PauseContainer, (User, Update)> for State {
     let _action_guard =
       action_state.update(|state| state.pausing = true)?;
 
+    let mut update = update.clone();
+
     // Send update after setting action state, this way frontend gets correct state.
     update_update(update.clone()).await?;
 
-    let periphery = periphery_client(&server)?;
-
-    let log = match periphery
+    let log = match periphery_client(&server)?
       .request(api::container::PauseContainer {
-        name: deployment.name.clone(),
+        name: deployment.name,
       })
       .await
     {
@@ -363,7 +524,7 @@ impl Resolve<PauseContainer, (User, Update)> for State {
     };
 
     update.logs.push(log);
-    update_cache_for_server(&server).await;
+    update_cache_for_server(&server, true).await;
     update.finalize();
     update_update(update.clone()).await?;
 
@@ -371,15 +532,14 @@ impl Resolve<PauseContainer, (User, Update)> for State {
   }
 }
 
-impl Resolve<UnpauseContainer, (User, Update)> for State {
-  #[instrument(name = "UnpauseContainer", skip(self, user, update), fields(user_id = user.id, update_id = update.id))]
+impl Resolve<ExecuteArgs> for UnpauseDeployment {
+  #[instrument(name = "UnpauseDeployment", skip(user, update), fields(user_id = user.id, update_id = update.id))]
   async fn resolve(
-    &self,
-    UnpauseContainer { deployment }: UnpauseContainer,
-    (user, mut update): (User, Update),
-  ) -> anyhow::Result<Update> {
+    self,
+    ExecuteArgs { user, update }: &ExecuteArgs,
+  ) -> serror::Result<Update> {
     let (deployment, server) =
-      setup_deployment_execution(&deployment, &user).await?;
+      setup_deployment_execution(&self.deployment, user).await?;
 
     // get the action state for the deployment (or insert default).
     let action_state = action_states()
@@ -392,14 +552,14 @@ impl Resolve<UnpauseContainer, (User, Update)> for State {
     let _action_guard =
       action_state.update(|state| state.unpausing = true)?;
 
+    let mut update = update.clone();
+
     // Send update after setting action state, this way frontend gets correct state.
     update_update(update.clone()).await?;
 
-    let periphery = periphery_client(&server)?;
-
-    let log = match periphery
+    let log = match periphery_client(&server)?
       .request(api::container::UnpauseContainer {
-        name: deployment.name.clone(),
+        name: deployment.name,
       })
       .await
     {
@@ -413,7 +573,7 @@ impl Resolve<UnpauseContainer, (User, Update)> for State {
     };
 
     update.logs.push(log);
-    update_cache_for_server(&server).await;
+    update_cache_for_server(&server, true).await;
     update.finalize();
     update_update(update.clone()).await?;
 
@@ -421,19 +581,14 @@ impl Resolve<UnpauseContainer, (User, Update)> for State {
   }
 }
 
-impl Resolve<StopContainer, (User, Update)> for State {
-  #[instrument(name = "StopContainer", skip(self, user, update), fields(user_id = user.id, update_id = update.id))]
+impl Resolve<ExecuteArgs> for StopDeployment {
+  #[instrument(name = "StopDeployment", skip(user, update), fields(user_id = user.id, update_id = update.id))]
   async fn resolve(
-    &self,
-    StopContainer {
-      deployment,
-      signal,
-      time,
-    }: StopContainer,
-    (user, mut update): (User, Update),
-  ) -> anyhow::Result<Update> {
+    self,
+    ExecuteArgs { user, update }: &ExecuteArgs,
+  ) -> serror::Result<Update> {
     let (deployment, server) =
-      setup_deployment_execution(&deployment, &user).await?;
+      setup_deployment_execution(&self.deployment, user).await?;
 
     // get the action state for the deployment (or insert default).
     let action_state = action_states()
@@ -446,18 +601,20 @@ impl Resolve<StopContainer, (User, Update)> for State {
     let _action_guard =
       action_state.update(|state| state.stopping = true)?;
 
+    let mut update = update.clone();
+
     // Send update after setting action state, this way frontend gets correct state.
     update_update(update.clone()).await?;
 
-    let periphery = periphery_client(&server)?;
-
-    let log = match periphery
+    let log = match periphery_client(&server)?
       .request(api::container::StopContainer {
-        name: deployment.name.clone(),
-        signal: signal
+        name: deployment.name,
+        signal: self
+          .signal
           .unwrap_or(deployment.config.termination_signal)
           .into(),
-        time: time
+        time: self
+          .time
           .unwrap_or(deployment.config.termination_timeout)
           .into(),
       })
@@ -471,7 +628,7 @@ impl Resolve<StopContainer, (User, Update)> for State {
     };
 
     update.logs.push(log);
-    update_cache_for_server(&server).await;
+    update_cache_for_server(&server, true).await;
     update.finalize();
     update_update(update.clone()).await?;
 
@@ -479,19 +636,41 @@ impl Resolve<StopContainer, (User, Update)> for State {
   }
 }
 
-impl Resolve<RemoveContainer, (User, Update)> for State {
-  #[instrument(name = "RemoveContainer", skip(self, user, update), fields(user_id = user.id, update_id = update.id))]
-  async fn resolve(
-    &self,
-    RemoveContainer {
+impl super::BatchExecute for BatchDestroyDeployment {
+  type Resource = Deployment;
+  fn single_request(deployment: String) -> ExecuteRequest {
+    ExecuteRequest::DestroyDeployment(DestroyDeployment {
       deployment,
-      signal,
-      time,
-    }: RemoveContainer,
-    (user, mut update): (User, Update),
-  ) -> anyhow::Result<Update> {
+      signal: None,
+      time: None,
+    })
+  }
+}
+
+impl Resolve<ExecuteArgs> for BatchDestroyDeployment {
+  #[instrument(name = "BatchDestroyDeployment", skip(user), fields(user_id = user.id))]
+  async fn resolve(
+    self,
+    ExecuteArgs { user, .. }: &ExecuteArgs,
+  ) -> serror::Result<BatchExecutionResponse> {
+    Ok(
+      super::batch_execute::<BatchDestroyDeployment>(
+        &self.pattern,
+        user,
+      )
+      .await?,
+    )
+  }
+}
+
+impl Resolve<ExecuteArgs> for DestroyDeployment {
+  #[instrument(name = "DestroyDeployment", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+  async fn resolve(
+    self,
+    ExecuteArgs { user, update }: &ExecuteArgs,
+  ) -> serror::Result<Update> {
     let (deployment, server) =
-      setup_deployment_execution(&deployment, &user).await?;
+      setup_deployment_execution(&self.deployment, user).await?;
 
     // get the action state for the deployment (or insert default).
     let action_state = action_states()
@@ -502,20 +681,22 @@ impl Resolve<RemoveContainer, (User, Update)> for State {
     // Will check to ensure deployment not already busy before updating, and return Err if so.
     // The returned guard will set the action state back to default when dropped.
     let _action_guard =
-      action_state.update(|state| state.removing = true)?;
+      action_state.update(|state| state.destroying = true)?;
+
+    let mut update = update.clone();
 
     // Send update after setting action state, this way frontend gets correct state.
     update_update(update.clone()).await?;
 
-    let periphery = periphery_client(&server)?;
-
-    let log = match periphery
+    let log = match periphery_client(&server)?
       .request(api::container::RemoveContainer {
-        name: deployment.name.clone(),
-        signal: signal
+        name: deployment.name,
+        signal: self
+          .signal
           .unwrap_or(deployment.config.termination_signal)
           .into(),
-        time: time
+        time: self
+          .time
           .unwrap_or(deployment.config.termination_timeout)
           .into(),
       })
@@ -530,7 +711,7 @@ impl Resolve<RemoveContainer, (User, Update)> for State {
 
     update.logs.push(log);
     update.finalize();
-    update_cache_for_server(&server).await;
+    update_cache_for_server(&server, true).await;
     update_update(update.clone()).await?;
 
     Ok(update)

bin/core/src/api/execute/maintenance.rs

@@ -0,0 +1,319 @@
+use std::sync::OnceLock;
+
+use anyhow::{Context, anyhow};
+use command::run_komodo_command;
+use database::mungos::{find::find_collect, mongodb::bson::doc};
+use formatting::{bold, format_serror};
+use komodo_client::{
+  api::execute::{
+    BackupCoreDatabase, ClearRepoCache, GlobalAutoUpdate,
+  },
+  entities::{
+    deployment::DeploymentState, server::ServerState,
+    stack::StackState,
+  },
+};
+use reqwest::StatusCode;
+use resolver_api::Resolve;
+use serror::AddStatusCodeError;
+use tokio::sync::Mutex;
+
+use crate::{
+  api::execute::{
+    ExecuteArgs, pull_deployment_inner, pull_stack_inner,
+  },
+  config::core_config,
+  helpers::update::update_update,
+  state::{
+    db_client, deployment_status_cache, server_status_cache,
+    stack_status_cache,
+  },
+};
+
+/// Makes sure the method can only be called once at a time
+fn clear_repo_cache_lock() -> &'static Mutex<()> {
+  static LOCK: OnceLock<Mutex<()>> = OnceLock::new();
+  LOCK.get_or_init(Default::default)
+}
+
+impl Resolve<ExecuteArgs> for ClearRepoCache {
+  #[instrument(
+    name = "ClearRepoCache",
+    skip(user, update),
+    fields(user_id = user.id, update_id = update.id)
+  )]
+  async fn resolve(
+    self,
+    ExecuteArgs { user, update }: &ExecuteArgs,
+  ) -> Result<Self::Response, Self::Error> {
+    if !user.admin {
+      return Err(
+        anyhow!("This method is admin only.")
+          .status_code(StatusCode::FORBIDDEN),
+      );
+    }
+
+    let _lock = clear_repo_cache_lock()
+      .try_lock()
+      .context("Clear already in progress...")?;
+
+    let mut update = update.clone();
+
+    let mut contents =
+      tokio::fs::read_dir(&core_config().repo_directory)
+        .await
+        .context("Failed to read repo cache directory")?;
+
+    loop {
+      let path = match contents
+        .next_entry()
+        .await
+        .context("Failed to read contents at path")
+      {
+        Ok(Some(contents)) => contents.path(),
+        Ok(None) => break,
+        Err(e) => {
+          update.push_error_log(
+            "Read Directory",
+            format_serror(&e.into()),
+          );
+          continue;
+        }
+      };
+      if path.is_dir() {
+        match tokio::fs::remove_dir_all(&path)
+          .await
+          .context("Failed to clear contents at path")
+        {
+          Ok(_) => {}
+          Err(e) => {
+            update.push_error_log(
+              "Clear Directory",
+              format_serror(&e.into()),
+            );
+          }
+        };
+      }
+    }
+
+    update.finalize();
+    update_update(update.clone()).await?;
+
+    Ok(update)
+  }
+}
+
+//
+
+/// Makes sure the method can only be called once at a time
+fn backup_database_lock() -> &'static Mutex<()> {
+  static LOCK: OnceLock<Mutex<()>> = OnceLock::new();
+  LOCK.get_or_init(Default::default)
+}
+
+impl Resolve<ExecuteArgs> for BackupCoreDatabase {
+  #[instrument(
+    name = "BackupCoreDatabase",
+    skip(user, update),
+    fields(user_id = user.id, update_id = update.id)
+  )]
+  async fn resolve(
+    self,
+    ExecuteArgs { user, update }: &ExecuteArgs,
+  ) -> Result<Self::Response, Self::Error> {
+    if !user.admin {
+      return Err(
+        anyhow!("This method is admin only.")
+          .status_code(StatusCode::FORBIDDEN),
+      );
+    }
+
+    let _lock = backup_database_lock()
+      .try_lock()
+      .context("Backup already in progress...")?;
+
+    let mut update = update.clone();
+
+    update_update(update.clone()).await?;
+
+    let res = run_komodo_command(
+      "Backup Core Database",
+      None,
+      "km database backup --yes",
+    )
+    .await;
+
+    update.logs.push(res);
+    update.finalize();
+
+    update_update(update.clone()).await?;
+
+    Ok(update)
+  }
+}
+
+//
+
+/// Makes sure the method can only be called once at a time
+fn global_update_lock() -> &'static Mutex<()> {
+  static LOCK: OnceLock<Mutex<()>> = OnceLock::new();
+  LOCK.get_or_init(Default::default)
+}
+
+impl Resolve<ExecuteArgs> for GlobalAutoUpdate {
+  #[instrument(
+    name = "GlobalAutoUpdate",
+    skip(user, update),
+    fields(user_id = user.id, update_id = update.id)
+  )]
+  async fn resolve(
+    self,
+    ExecuteArgs { user, update }: &ExecuteArgs,
+  ) -> Result<Self::Response, Self::Error> {
+    if !user.admin {
+      return Err(
+        anyhow!("This method is admin only.")
+          .status_code(StatusCode::FORBIDDEN),
+      );
+    }
+
+    let _lock = global_update_lock()
+      .try_lock()
+      .context("Global update already in progress...")?;
+
+    let mut update = update.clone();
+
+    update_update(update.clone()).await?;
+
+    // This is all done in sequence because there is no rush,
+    // the pulls / deploys happen spaced out to ease the load on system.
+    let servers = find_collect(&db_client().servers, None, None)
+      .await
+      .context("Failed to query for servers from database")?;
+
+    let query = doc! {
+      "$or": [
+        { "config.poll_for_updates": true },
+        { "config.auto_update": true }
+      ]
+    };
+
+    let (stacks, repos) = tokio::try_join!(
+      find_collect(&db_client().stacks, query.clone(), None),
+      find_collect(&db_client().repos, None, None)
+    )
+    .context("Failed to query for resources from database")?;
+
+    let server_status_cache = server_status_cache();
+    let stack_status_cache = stack_status_cache();
+
+    // Will be edited later at update.logs[0]
+    update.push_simple_log("Auto Pull", String::new());
+
+    for stack in stacks {
+      let Some(status) = stack_status_cache.get(&stack.id).await
+      else {
+        continue;
+      };
+      // Only pull running stacks.
+      if !matches!(status.curr.state, StackState::Running) {
+        continue;
+      }
+      if let Some(server) =
+        servers.iter().find(|s| s.id == stack.config.server_id)
+        // This check is probably redundant along with running check
+        // but shouldn't hurt
+        && server_status_cache
+          .get(&server.id)
+          .await
+          .map(|s| matches!(s.state, ServerState::Ok))
+          .unwrap_or_default()
+      {
+        let name = stack.name.clone();
+        let repo = if stack.config.linked_repo.is_empty() {
+          None
+        } else {
+          let Some(repo) =
+            repos.iter().find(|r| r.id == stack.config.linked_repo)
+          else {
+            update.push_error_log(
+              &format!("Pull Stack {name}"),
+              format!(
+                "Did not find any Repo matching {}",
+                stack.config.linked_repo
+              ),
+            );
+            continue;
+          };
+          Some(repo.clone())
+        };
+        if let Err(e) =
+          pull_stack_inner(stack, Vec::new(), server, repo, None)
+            .await
+        {
+          update.push_error_log(
+            &format!("Pull Stack {name}"),
+            format_serror(&e.into()),
+          );
+        } else {
+          if !update.logs[0].stdout.is_empty() {
+            update.logs[0].stdout.push('\n');
+          }
+          update.logs[0]
+            .stdout
+            .push_str(&format!("Pulled Stack {} ✅", bold(name)));
+        }
+      }
+    }
+
+    let deployment_status_cache = deployment_status_cache();
+    let deployments =
+      find_collect(&db_client().deployments, query, None)
+        .await
+        .context("Failed to query for deployments from database")?;
+    for deployment in deployments {
+      let Some(status) =
+        deployment_status_cache.get(&deployment.id).await
+      else {
+        continue;
+      };
+      // Only pull running deployments.
+      if !matches!(status.curr.state, DeploymentState::Running) {
+        continue;
+      }
+      if let Some(server) =
+        servers.iter().find(|s| s.id == deployment.config.server_id)
+        // This check is probably redundant along with running check
+        // but shouldn't hurt
+        && server_status_cache
+          .get(&server.id)
+          .await
+          .map(|s| matches!(s.state, ServerState::Ok))
+          .unwrap_or_default()
+      {
+        let name = deployment.name.clone();
+        if let Err(e) =
+          pull_deployment_inner(deployment, server).await
+        {
+          update.push_error_log(
+            &format!("Pull Deployment {name}"),
+            format_serror(&e.into()),
+          );
+        } else {
+          if !update.logs[0].stdout.is_empty() {
+            update.logs[0].stdout.push('\n');
+          }
+          update.logs[0].stdout.push_str(&format!(
+            "Pulled Deployment {} ✅",
+            bold(name)
+          ));
+        }
+      }
+    }
+
+    update.finalize();
+    update_update(update.clone()).await?;
+
+    Ok(update)
+  }
+}

bin/core/src/api/execute/mod.rs

@@ -1,18 +1,27 @@
-use std::time::Instant;
+use std::{pin::Pin, time::Instant};
 
-use anyhow::{anyhow, Context};
-use axum::{middleware, routing::post, Extension, Router};
+use anyhow::Context;
+use axum::{
+  Extension, Router, extract::Path, middleware, routing::post,
+};
+use axum_extra::{TypedHeader, headers::ContentType};
+use database::mungos::by_id::find_one_by_id;
+use derive_variants::{EnumVariants, ExtractVariant};
 use formatting::format_serror;
-use monitor_client::{
+use futures::future::join_all;
+use komodo_client::{
   api::execute::*,
   entities::{
+    Operation,
+    permission::PermissionLevel,
     update::{Log, Update},
     user::User,
   },
 };
-use mungos::by_id::find_one_by_id;
-use resolver_api::{derive::Resolver, Resolver};
+use resolver_api::Resolve;
+use response::JsonString;
 use serde::{Deserialize, Serialize};
+use serde_json::json;
 use serror::Json;
 use typeshare::typeshare;
 use uuid::Uuid;
@@ -20,145 +29,274 @@ use uuid::Uuid;
 use crate::{
   auth::auth_request,
   helpers::update::{init_execution_update, update_update},
-  state::{db_client, State},
+  resource::{KomodoResource, list_full_for_user_using_pattern},
+  state::db_client,
 };
 
+mod action;
+mod alerter;
 mod build;
 mod deployment;
+mod maintenance;
 mod procedure;
 mod repo;
 mod server;
-mod server_template;
 mod stack;
 mod sync;
 
+use super::Variant;
+
+pub use {
+  deployment::pull_deployment_inner, stack::pull_stack_inner,
+};
+
+pub struct ExecuteArgs {
+  pub user: User,
+  pub update: Update,
+}
+
 #[typeshare]
-#[derive(Serialize, Deserialize, Debug, Clone, Resolver)]
-#[resolver_target(State)]
-#[resolver_args((User, Update))]
+#[derive(
+  Serialize, Deserialize, Debug, Clone, Resolve, EnumVariants,
+)]
+#[variant_derive(Debug)]
+#[args(ExecuteArgs)]
+#[response(JsonString)]
+#[error(serror::Error)]
 #[serde(tag = "type", content = "params")]
 pub enum ExecuteRequest {
   // ==== SERVER ====
-  StopAllContainers(StopAllContainers),
-  PruneContainers(PruneContainers),
-  PruneImages(PruneImages),
-  PruneNetworks(PruneNetworks),
-
-  // ==== DEPLOYMENT ====
-  Deploy(Deploy),
   StartContainer(StartContainer),
   RestartContainer(RestartContainer),
   PauseContainer(PauseContainer),
   UnpauseContainer(UnpauseContainer),
   StopContainer(StopContainer),
-  RemoveContainer(RemoveContainer),
+  DestroyContainer(DestroyContainer),
+  StartAllContainers(StartAllContainers),
+  RestartAllContainers(RestartAllContainers),
+  PauseAllContainers(PauseAllContainers),
+  UnpauseAllContainers(UnpauseAllContainers),
+  StopAllContainers(StopAllContainers),
+  PruneContainers(PruneContainers),
+  DeleteNetwork(DeleteNetwork),
+  PruneNetworks(PruneNetworks),
+  DeleteImage(DeleteImage),
+  PruneImages(PruneImages),
+  DeleteVolume(DeleteVolume),
+  PruneVolumes(PruneVolumes),
+  PruneDockerBuilders(PruneDockerBuilders),
+  PruneBuildx(PruneBuildx),
+  PruneSystem(PruneSystem),
 
   // ==== STACK ====
   DeployStack(DeployStack),
+  BatchDeployStack(BatchDeployStack),
+  DeployStackIfChanged(DeployStackIfChanged),
+  BatchDeployStackIfChanged(BatchDeployStackIfChanged),
+  PullStack(PullStack),
+  BatchPullStack(BatchPullStack),
   StartStack(StartStack),
   RestartStack(RestartStack),
   StopStack(StopStack),
   PauseStack(PauseStack),
   UnpauseStack(UnpauseStack),
   DestroyStack(DestroyStack),
+  BatchDestroyStack(BatchDestroyStack),
+  RunStackService(RunStackService),
+
+  // ==== DEPLOYMENT ====
+  Deploy(Deploy),
+  BatchDeploy(BatchDeploy),
+  PullDeployment(PullDeployment),
+  StartDeployment(StartDeployment),
+  RestartDeployment(RestartDeployment),
+  PauseDeployment(PauseDeployment),
+  UnpauseDeployment(UnpauseDeployment),
+  StopDeployment(StopDeployment),
+  DestroyDeployment(DestroyDeployment),
+  BatchDestroyDeployment(BatchDestroyDeployment),
 
   // ==== BUILD ====
   RunBuild(RunBuild),
+  BatchRunBuild(BatchRunBuild),
   CancelBuild(CancelBuild),
 
   // ==== REPO ====
   CloneRepo(CloneRepo),
+  BatchCloneRepo(BatchCloneRepo),
   PullRepo(PullRepo),
+  BatchPullRepo(BatchPullRepo),
   BuildRepo(BuildRepo),
+  BatchBuildRepo(BatchBuildRepo),
   CancelRepoBuild(CancelRepoBuild),
 
   // ==== PROCEDURE ====
   RunProcedure(RunProcedure),
+  BatchRunProcedure(BatchRunProcedure),
 
-  // ==== SERVER TEMPLATE ====
-  LaunchServer(LaunchServer),
+  // ==== ACTION ====
+  RunAction(RunAction),
+  BatchRunAction(BatchRunAction),
+
+  // ==== ALERTER ====
+  TestAlerter(TestAlerter),
+  SendAlert(SendAlert),
 
   // ==== SYNC ====
   RunSync(RunSync),
+
+  // ==== MAINTENANCE ====
+  ClearRepoCache(ClearRepoCache),
+  BackupCoreDatabase(BackupCoreDatabase),
+  GlobalAutoUpdate(GlobalAutoUpdate),
 }
 
 pub fn router() -> Router {
   Router::new()
     .route("/", post(handler))
+    .route("/{variant}", post(variant_handler))
     .layer(middleware::from_fn(auth_request))
 }
 
+async fn variant_handler(
+  user: Extension<User>,
+  Path(Variant { variant }): Path<Variant>,
+  Json(params): Json<serde_json::Value>,
+) -> serror::Result<(TypedHeader<ContentType>, String)> {
+  let req: ExecuteRequest = serde_json::from_value(json!({
+    "type": variant,
+    "params": params,
+  }))?;
+  handler(user, Json(req)).await
+}
+
 async fn handler(
   Extension(user): Extension<User>,
   Json(request): Json<ExecuteRequest>,
-) -> serror::Result<Json<Update>> {
-  let req_id = Uuid::new_v4();
-
-  // need to validate no cancel is active before any update is created.
-  build::validate_cancel_build(&request).await?;
-
-  let update = init_execution_update(&request, &user).await?;
-
-  let handle =
-    tokio::spawn(task(req_id, request, user, update.clone()));
-
-  tokio::spawn({
-    let update_id = update.id.clone();
-    async move {
-      let log = match handle.await {
-        Ok(Err(e)) => {
-          warn!("/execute request {req_id} task error: {e:#}",);
-          Log::error("task error", format_serror(&e.into()))
-        }
-        Err(e) => {
-          warn!("/execute request {req_id} spawn error: {e:?}",);
-          Log::error("spawn error", format!("{e:#?}"))
-        }
-        _ => return,
-      };
-      let res = async {
-        let mut update =
-          find_one_by_id(&db_client().await.updates, &update_id)
-            .await
-            .context("failed to query to db")?
-            .context("no update exists with given id")?;
-        update.logs.push(log);
-        update.finalize();
-        update_update(update).await
-      }
-      .await;
-
-      if let Err(e) = res {
-        warn!("failed to update update with task error log | {e:#}");
-      }
-    }
-  });
-
-  Ok(Json(update))
+) -> serror::Result<(TypedHeader<ContentType>, String)> {
+  let res = match inner_handler(request, user).await? {
+    ExecutionResult::Single(update) => serde_json::to_string(&update)
+      .context("Failed to serialize Update")?,
+    ExecutionResult::Batch(res) => res,
+  };
+  Ok((TypedHeader(ContentType::json()), res))
 }
 
-#[instrument(name = "ExecuteRequest", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+#[typeshare(serialized_as = "Update")]
+type BoxUpdate = Box<Update>;
+
+pub enum ExecutionResult {
+  Single(BoxUpdate),
+  /// The batch contents will be pre serialized here
+  Batch(String),
+}
+
+pub fn inner_handler(
+  request: ExecuteRequest,
+  user: User,
+) -> Pin<
+  Box<
+    dyn std::future::Future<Output = anyhow::Result<ExecutionResult>>
+      + Send,
+  >,
+> {
+  Box::pin(async move {
+    let req_id = Uuid::new_v4();
+
+    // Need to validate no cancel is active before any update is created.
+    // This ensures no double update created if Cancel is called more than once for the same request.
+    build::validate_cancel_build(&request).await?;
+    repo::validate_cancel_repo_build(&request).await?;
+
+    let update = init_execution_update(&request, &user).await?;
+
+    // This will be the case for the Batch exections,
+    // they don't have their own updates.
+    // The batch calls also call "inner_handler" themselves,
+    // and in their case will spawn tasks, so that isn't necessary
+    // here either.
+    if update.operation == Operation::None {
+      return Ok(ExecutionResult::Batch(
+        task(req_id, request, user, update).await?,
+      ));
+    }
+
+    // Spawn a task for the execution which continues
+    // running after this method returns.
+    let handle =
+      tokio::spawn(task(req_id, request, user, update.clone()));
+
+    // Spawns another task to monitor the first for failures,
+    // and add the log to Update about it (which primary task can't do because it errored out)
+    tokio::spawn({
+      let update_id = update.id.clone();
+      async move {
+        let log = match handle.await {
+          Ok(Err(e)) => {
+            warn!("/execute request {req_id} task error: {e:#}",);
+            Log::error("Task Error", format_serror(&e.into()))
+          }
+          Err(e) => {
+            warn!("/execute request {req_id} spawn error: {e:?}",);
+            Log::error("Spawn Error", format!("{e:#?}"))
+          }
+          _ => return,
+        };
+        let res = async {
+          // Nothing to do if update was never actually created,
+          // which is the case when the id is empty.
+          if update_id.is_empty() {
+            return Ok(());
+          }
+          let mut update =
+            find_one_by_id(&db_client().updates, &update_id)
+              .await
+              .context("failed to query to db")?
+              .context("no update exists with given id")?;
+          update.logs.push(log);
+          update.finalize();
+          update_update(update).await
+        }
+        .await;
+
+        if let Err(e) = res {
+          warn!(
+            "failed to update update with task error log | {e:#}"
+          );
+        }
+      }
+    });
+
+    Ok(ExecutionResult::Single(update.into()))
+  })
+}
+
+#[instrument(
+  name = "ExecuteRequest",
+  skip(user, update),
+  fields(
+    user_id = user.id,
+    update_id = update.id,
+    request = format!("{:?}", request.extract_variant()))
+  )
+]
 async fn task(
   req_id: Uuid,
   request: ExecuteRequest,
   user: User,
   update: Update,
 ) -> anyhow::Result<String> {
-  info!(
-    "/execute request {req_id} | user: {} ({})",
-    user.username, user.id
-  );
+  info!("/execute request {req_id} | user: {}", user.username);
   let timer = Instant::now();
 
-  let res = State
-    .resolve_request(request, (user, update))
-    .await
-    .map_err(|e| match e {
-      resolver_api::Error::Serialization(e) => {
-        anyhow!("{e:?}").context("response serialization error")
-      }
-      resolver_api::Error::Inner(e) => e,
-    });
+  let res = match request.resolve(&ExecuteArgs { user, update }).await
+  {
+    Err(e) => Err(e.error),
+    Ok(JsonString::Err(e)) => Err(
+      anyhow::Error::from(e).context("failed to serialize response"),
+    ),
+    Ok(JsonString::Ok(res)) => Ok(res),
+  };
 
   if let Err(e) = &res {
     warn!("/execute request {req_id} error: {e:#}");
@@ -169,3 +307,41 @@ async fn task(
 
   res
 }
+
+trait BatchExecute {
+  type Resource: KomodoResource;
+  fn single_request(name: String) -> ExecuteRequest;
+}
+
+async fn batch_execute<E: BatchExecute>(
+  pattern: &str,
+  user: &User,
+) -> anyhow::Result<BatchExecutionResponse> {
+  let resources = list_full_for_user_using_pattern::<E::Resource>(
+    pattern,
+    Default::default(),
+    user,
+    PermissionLevel::Execute.into(),
+    &[],
+  )
+  .await?;
+  let futures = resources.into_iter().map(|resource| {
+    let user = user.clone();
+    async move {
+      inner_handler(E::single_request(resource.name.clone()), user)
+        .await
+        .map(|r| {
+          let ExecutionResult::Single(update) = r else {
+            unreachable!()
+          };
+          update
+        })
+        .map_err(|e| BatchExecutionResponseItemErr {
+          name: resource.name,
+          error: e.into(),
+        })
+        .into()
+    }
+  });
+  Ok(join_all(futures).await)
+}

bin/core/src/api/execute/procedure.rs

@@ -1,31 +1,65 @@
 use std::pin::Pin;
 
-use formatting::{bold, colored, format_serror, muted, Color};
-use monitor_client::{
-  api::execute::RunProcedure,
+use database::mungos::{
+  by_id::update_one_by_id, mongodb::bson::to_document,
+};
+use formatting::{Color, bold, colored, format_serror, muted};
+use komodo_client::{
+  api::execute::{
+    BatchExecutionResponse, BatchRunProcedure, RunProcedure,
+  },
   entities::{
-    permission::PermissionLevel, procedure::Procedure,
-    update::Update, user::User,
+    alert::{Alert, AlertData, SeverityLevel},
+    komodo_timestamp,
+    permission::PermissionLevel,
+    procedure::Procedure,
+    update::Update,
+    user::User,
   },
 };
-use mungos::{by_id::update_one_by_id, mongodb::bson::to_document};
 use resolver_api::Resolve;
 use tokio::sync::Mutex;
 
 use crate::{
+  alert::send_alerts,
   helpers::{procedure::execute_procedure, update::update_update},
-  resource::{self, refresh_procedure_state_cache},
-  state::{action_states, db_client, State},
+  permission::get_check_permissions,
+  resource::refresh_procedure_state_cache,
+  state::{action_states, db_client},
 };
 
-impl Resolve<RunProcedure, (User, Update)> for State {
-  #[instrument(name = "RunProcedure", skip(self, user, update), fields(user_id = user.id, update_id = update.id))]
+use super::{ExecuteArgs, ExecuteRequest};
+
+impl super::BatchExecute for BatchRunProcedure {
+  type Resource = Procedure;
+  fn single_request(procedure: String) -> ExecuteRequest {
+    ExecuteRequest::RunProcedure(RunProcedure { procedure })
+  }
+}
+
+impl Resolve<ExecuteArgs> for BatchRunProcedure {
+  #[instrument(name = "BatchRunProcedure", skip(user), fields(user_id = user.id))]
   async fn resolve(
-    &self,
-    RunProcedure { procedure }: RunProcedure,
-    (user, update): (User, Update),
-  ) -> anyhow::Result<Update> {
-    resolve_inner(procedure, user, update).await
+    self,
+    ExecuteArgs { user, .. }: &ExecuteArgs,
+  ) -> serror::Result<BatchExecutionResponse> {
+    Ok(
+      super::batch_execute::<BatchRunProcedure>(&self.pattern, user)
+        .await?,
+    )
+  }
+}
+
+impl Resolve<ExecuteArgs> for RunProcedure {
+  #[instrument(name = "RunProcedure", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+  async fn resolve(
+    self,
+    ExecuteArgs { user, update }: &ExecuteArgs,
+  ) -> serror::Result<Update> {
+    Ok(
+      resolve_inner(self.procedure, user.clone(), update.clone())
+        .await?,
+    )
   }
 }
 
@@ -39,10 +73,10 @@ fn resolve_inner(
   >,
 > {
   Box::pin(async move {
-    let procedure = resource::get_check_permissions::<Procedure>(
+    let procedure = get_check_permissions::<Procedure>(
       &procedure,
       &user,
-      PermissionLevel::Execute,
+      PermissionLevel::Execute.into(),
     )
     .await?;
 
@@ -50,7 +84,7 @@ fn resolve_inner(
     // assumes first log is already created
     // and will panic otherwise.
     update.push_simple_log(
-      "execute_procedure",
+      "Execute procedure",
       format!(
         "{}: executing procedure '{}'",
         muted("INFO"),
@@ -80,9 +114,9 @@ fn resolve_inner(
     match res {
       Ok(_) => {
         update.push_simple_log(
-          "execution ok",
+          "Execution ok",
           format!(
-            "{}: the procedure has {} with no errors",
+            "{}: The procedure has {} with no errors",
             muted("INFO"),
             colored("completed", Color::Green)
           ),
@@ -100,9 +134,9 @@ fn resolve_inner(
     // but will fail to update cache in that case.
     if let Ok(update_doc) = to_document(&update) {
       let _ = update_one_by_id(
-        &db_client().await.updates,
+        &db_client().updates,
         &update.id,
-        mungos::update::Update::Set(update_doc),
+        database::mungos::update::Update::Set(update_doc),
         None,
       )
       .await;
@@ -111,6 +145,26 @@ fn resolve_inner(
 
     update_update(update.clone()).await?;
 
+    if !update.success && procedure.config.failure_alert {
+      warn!("procedure unsuccessful, alerting...");
+      let target = update.target.clone();
+      tokio::spawn(async move {
+        let alert = Alert {
+          id: Default::default(),
+          target,
+          ts: komodo_timestamp(),
+          resolved_ts: Some(komodo_timestamp()),
+          resolved: true,
+          level: SeverityLevel::Warning,
+          data: AlertData::ProcedureFailed {
+            id: procedure.id,
+            name: procedure.name,
+          },
+        };
+        send_alerts(&[alert]).await
+      });
+    }
+
     Ok(update)
   })
 }

bin/core/src/api/execute/repo.rs

@@ -1,56 +1,78 @@
-use std::{future::IntoFuture, time::Duration};
+use std::{collections::HashSet, future::IntoFuture, time::Duration};
 
-use anyhow::{anyhow, Context};
-use formatting::format_serror;
-use monitor_client::{
-  api::execute::*,
-  entities::{
-    alert::{Alert, AlertData},
-    builder::{Builder, BuilderConfig},
-    monitor_timestamp, optional_string,
-    permission::PermissionLevel,
-    repo::Repo,
-    server::{stats::SeverityLevel, Server},
-    update::{Log, Update},
-    user::User,
-  },
-};
-use mungos::{
+use anyhow::{Context, anyhow};
+use database::mungos::{
   by_id::update_one_by_id,
   mongodb::{
     bson::{doc, to_document},
     options::FindOneOptions,
   },
 };
-use periphery_client::api::{self, git::RepoActionResponseV1_13};
+use formatting::format_serror;
+use interpolate::Interpolator;
+use komodo_client::{
+  api::{execute::*, write::RefreshRepoCache},
+  entities::{
+    alert::{Alert, AlertData, SeverityLevel},
+    builder::{Builder, BuilderConfig},
+    komodo_timestamp,
+    permission::PermissionLevel,
+    repo::Repo,
+    server::Server,
+    update::{Log, Update},
+  },
+};
+use periphery_client::api;
 use resolver_api::Resolve;
 use tokio_util::sync::CancellationToken;
 
 use crate::{
+  alert::send_alerts,
+  api::write::WriteArgs,
   helpers::{
-    alert::send_alerts,
     builder::{cleanup_builder_instance, get_builder_periphery},
     channel::repo_cancel_channel,
     git_token, periphery_client,
+    query::{VariablesAndSecrets, get_variables_and_secrets},
     update::update_update,
   },
+  permission::get_check_permissions,
   resource::{self, refresh_repo_state_cache},
-  state::{action_states, db_client, State},
+  state::{action_states, db_client},
 };
 
-use super::ExecuteRequest;
+use super::{ExecuteArgs, ExecuteRequest};
 
-impl Resolve<CloneRepo, (User, Update)> for State {
-  #[instrument(name = "CloneRepo", skip(self, user, update), fields(user_id = user.id, update_id = update.id))]
+impl super::BatchExecute for BatchCloneRepo {
+  type Resource = Repo;
+  fn single_request(repo: String) -> ExecuteRequest {
+    ExecuteRequest::CloneRepo(CloneRepo { repo })
+  }
+}
+
+impl Resolve<ExecuteArgs> for BatchCloneRepo {
+  #[instrument(name = "BatchCloneRepo", skip( user), fields(user_id = user.id))]
   async fn resolve(
-    &self,
-    CloneRepo { repo }: CloneRepo,
-    (user, mut update): (User, Update),
-  ) -> anyhow::Result<Update> {
-    let mut repo = resource::get_check_permissions::<Repo>(
-      &repo,
-      &user,
-      PermissionLevel::Execute,
+    self,
+    ExecuteArgs { user, update }: &ExecuteArgs,
+  ) -> serror::Result<BatchExecutionResponse> {
+    Ok(
+      super::batch_execute::<BatchCloneRepo>(&self.pattern, user)
+        .await?,
+    )
+  }
+}
+
+impl Resolve<ExecuteArgs> for CloneRepo {
+  #[instrument(name = "CloneRepo", skip( user, update), fields(user_id = user.id, update_id = update.id))]
+  async fn resolve(
+    self,
+    ExecuteArgs { user, update }: &ExecuteArgs,
+  ) -> serror::Result<Update> {
+    let mut repo = get_check_permissions::<Repo>(
+      &self.repo,
+      user,
+      PermissionLevel::Execute.into(),
     )
     .await?;
 
@@ -63,8 +85,13 @@ impl Resolve<CloneRepo, (User, Update)> for State {
     let _action_guard =
       action_state.update(|state| state.cloning = true)?;
 
+    let mut update = update.clone();
     update_update(update.clone()).await?;
 
+    if repo.config.server_id.is_empty() {
+      return Err(anyhow!("repo has no server attached").into());
+    }
+
     let git_token = git_token(
       &repo.config.git_provider,
       &repo.config.git_account,
@@ -75,33 +102,34 @@ impl Resolve<CloneRepo, (User, Update)> for State {
       || format!("Failed to get git token in call to db. This is a database error, not a token exisitence error. Stopping run. | {} | {}", repo.config.git_provider, repo.config.git_account),
     )?;
 
-    if repo.config.server_id.is_empty() {
-      return Err(anyhow!("repo has no server attached"));
-    }
-
     let server =
       resource::get::<Server>(&repo.config.server_id).await?;
 
     let periphery = periphery_client(&server)?;
 
+    // interpolate variables / secrets, returning the sanitizing replacers to send to
+    // periphery so it may sanitize the final command for safe logging (avoids exposing secret values)
+    let secret_replacers =
+      interpolate(&mut repo, &mut update).await?;
+
     let logs = match periphery
       .request(api::git::CloneRepo {
         args: (&repo).into(),
         git_token,
-        environment: repo.config.environment,
+        environment: repo.config.env_vars()?,
         env_file_path: repo.config.env_file_path,
+        on_clone: repo.config.on_clone.into(),
+        on_pull: repo.config.on_pull.into(),
         skip_secret_interp: repo.config.skip_secret_interp,
+        replacers: secret_replacers.into_iter().collect(),
       })
       .await
     {
-      Ok(res) => {
-        let res: RepoActionResponseV1_13 = res.into();
-        res.logs
-      }
+      Ok(res) => res.res.logs,
       Err(e) => {
         vec![Log::error(
-          "clone repo",
-          format_serror(&e.context("failed to clone repo").into()),
+          "Clone Repo",
+          format_serror(&e.context("Failed to clone repo").into()),
         )]
       }
     };
@@ -113,21 +141,52 @@ impl Resolve<CloneRepo, (User, Update)> for State {
       update_last_pulled_time(&repo.name).await;
     }
 
-    handle_server_update_return(update).await
+    if let Err(e) = (RefreshRepoCache { repo: repo.id })
+      .resolve(&WriteArgs { user: user.clone() })
+      .await
+      .map_err(|e| e.error)
+      .context("Failed to refresh repo cache")
+    {
+      update.push_error_log(
+        "Refresh Repo cache",
+        format_serror(&e.into()),
+      );
+    };
+
+    handle_repo_update_return(update).await
   }
 }
 
-impl Resolve<PullRepo, (User, Update)> for State {
-  #[instrument(name = "PullRepo", skip(self, user, update), fields(user_id = user.id, update_id = update.id))]
+impl super::BatchExecute for BatchPullRepo {
+  type Resource = Repo;
+  fn single_request(repo: String) -> ExecuteRequest {
+    ExecuteRequest::PullRepo(PullRepo { repo })
+  }
+}
+
+impl Resolve<ExecuteArgs> for BatchPullRepo {
+  #[instrument(name = "BatchPullRepo", skip(user), fields(user_id = user.id))]
   async fn resolve(
-    &self,
-    PullRepo { repo }: PullRepo,
-    (user, mut update): (User, Update),
-  ) -> anyhow::Result<Update> {
-    let repo = resource::get_check_permissions::<Repo>(
-      &repo,
-      &user,
-      PermissionLevel::Execute,
+    self,
+    ExecuteArgs { user, .. }: &ExecuteArgs,
+  ) -> serror::Result<BatchExecutionResponse> {
+    Ok(
+      super::batch_execute::<BatchPullRepo>(&self.pattern, user)
+        .await?,
+    )
+  }
+}
+
+impl Resolve<ExecuteArgs> for PullRepo {
+  #[instrument(name = "PullRepo", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+  async fn resolve(
+    self,
+    ExecuteArgs { user, update }: &ExecuteArgs,
+  ) -> serror::Result<Update> {
+    let mut repo = get_check_permissions::<Repo>(
+      &self.repo,
+      user,
+      PermissionLevel::Execute.into(),
     )
     .await?;
 
@@ -140,33 +199,49 @@ impl Resolve<PullRepo, (User, Update)> for State {
     let _action_guard =
       action_state.update(|state| state.pulling = true)?;
 
+    let mut update = update.clone();
+
     update_update(update.clone()).await?;
 
     if repo.config.server_id.is_empty() {
-      return Err(anyhow!("repo has no server attached"));
+      return Err(anyhow!("repo has no server attached").into());
     }
 
+    let git_token = git_token(
+      &repo.config.git_provider,
+      &repo.config.git_account,
+      |https| repo.config.git_https = https,
+    )
+    .await
+    .with_context(
+      || format!("Failed to get git token in call to db. This is a database error, not a token exisitence error. Stopping run. | {} | {}", repo.config.git_provider, repo.config.git_account),
+    )?;
+
     let server =
       resource::get::<Server>(&repo.config.server_id).await?;
 
     let periphery = periphery_client(&server)?;
 
+    // interpolate variables / secrets, returning the sanitizing replacers to send to
+    // periphery so it may sanitize the final command for safe logging (avoids exposing secret values)
+    let secret_replacers =
+      interpolate(&mut repo, &mut update).await?;
+
     let logs = match periphery
       .request(api::git::PullRepo {
-        name: repo.name.clone(),
-        branch: optional_string(&repo.config.branch),
-        commit: optional_string(&repo.config.commit),
-        on_pull: repo.config.on_pull.into_option(),
-        environment: repo.config.environment,
+        args: (&repo).into(),
+        git_token,
+        environment: repo.config.env_vars()?,
         env_file_path: repo.config.env_file_path,
+        on_pull: repo.config.on_pull.into(),
         skip_secret_interp: repo.config.skip_secret_interp,
+        replacers: secret_replacers.into_iter().collect(),
       })
       .await
     {
       Ok(res) => {
-        let res: RepoActionResponseV1_13 = res.into();
-        update.commit_hash = res.commit_hash.unwrap_or_default();
-        res.logs
+        update.commit_hash = res.res.commit_hash.unwrap_or_default();
+        res.res.logs
       }
       Err(e) => {
         vec![Log::error(
@@ -184,23 +259,35 @@ impl Resolve<PullRepo, (User, Update)> for State {
       update_last_pulled_time(&repo.name).await;
     }
 
-    handle_server_update_return(update).await
+    if let Err(e) = (RefreshRepoCache { repo: repo.id })
+      .resolve(&WriteArgs { user: user.clone() })
+      .await
+      .map_err(|e| e.error)
+      .context("Failed to refresh repo cache")
+    {
+      update.push_error_log(
+        "Refresh Repo cache",
+        format_serror(&e.into()),
+      );
+    };
+
+    handle_repo_update_return(update).await
   }
 }
 
 #[instrument(skip_all, fields(update_id = update.id))]
-async fn handle_server_update_return(
+async fn handle_repo_update_return(
   update: Update,
-) -> anyhow::Result<Update> {
+) -> serror::Result<Update> {
   // Need to manually update the update before cache refresh,
   // and before broadcast with add_update.
   // The Err case of to_document should be unreachable,
   // but will fail to update cache in that case.
   if let Ok(update_doc) = to_document(&update) {
     let _ = update_one_by_id(
-      &db_client().await.updates,
+      &db_client().updates,
       &update.id,
-      mungos::update::Update::Set(update_doc),
+      database::mungos::update::Update::Set(update_doc),
       None,
     )
     .await;
@@ -213,11 +300,10 @@ async fn handle_server_update_return(
 #[instrument]
 async fn update_last_pulled_time(repo_name: &str) {
   let res = db_client()
-    .await
     .repos
     .update_one(
       doc! { "name": repo_name },
-      doc! { "$set": { "info.last_pulled_at": monitor_timestamp() } },
+      doc! { "$set": { "info.last_pulled_at": komodo_timestamp() } },
     )
     .await;
   if let Err(e) = res {
@@ -227,22 +313,41 @@ async fn update_last_pulled_time(repo_name: &str) {
   }
 }
 
-impl Resolve<BuildRepo, (User, Update)> for State {
-  #[instrument(name = "BuildRepo", skip(self, user, update), fields(user_id = user.id, update_id = update.id))]
+impl super::BatchExecute for BatchBuildRepo {
+  type Resource = Repo;
+  fn single_request(repo: String) -> ExecuteRequest {
+    ExecuteRequest::CloneRepo(CloneRepo { repo })
+  }
+}
+
+impl Resolve<ExecuteArgs> for BatchBuildRepo {
+  #[instrument(name = "BatchBuildRepo", skip(user), fields(user_id = user.id))]
   async fn resolve(
-    &self,
-    BuildRepo { repo }: BuildRepo,
-    (user, mut update): (User, Update),
-  ) -> anyhow::Result<Update> {
-    let mut repo = resource::get_check_permissions::<Repo>(
-      &repo,
-      &user,
-      PermissionLevel::Execute,
+    self,
+    ExecuteArgs { user, .. }: &ExecuteArgs,
+  ) -> serror::Result<BatchExecutionResponse> {
+    Ok(
+      super::batch_execute::<BatchBuildRepo>(&self.pattern, user)
+        .await?,
+    )
+  }
+}
+
+impl Resolve<ExecuteArgs> for BuildRepo {
+  #[instrument(name = "BuildRepo", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+  async fn resolve(
+    self,
+    ExecuteArgs { user, update }: &ExecuteArgs,
+  ) -> serror::Result<Update> {
+    let mut repo = get_check_permissions::<Repo>(
+      &self.repo,
+      user,
+      PermissionLevel::Execute.into(),
     )
     .await?;
 
     if repo.config.builder_id.is_empty() {
-      return Err(anyhow!("Must attach builder to BuildRepo"));
+      return Err(anyhow!("Must attach builder to BuildRepo").into());
     }
 
     // get the action state for the repo (or insert default).
@@ -254,6 +359,7 @@ impl Resolve<BuildRepo, (User, Update)> for State {
     let _action_guard =
       action_state.update(|state| state.building = true)?;
 
+    let mut update = update.clone();
     update_update(update.clone()).await?;
 
     let git_token = git_token(
@@ -337,19 +443,27 @@ impl Resolve<BuildRepo, (User, Update)> for State {
 
     // CLONE REPO
 
+    // interpolate variables / secrets, returning the sanitizing replacers to send to
+    // periphery so it may sanitize the final command for safe logging (avoids exposing secret values)
+    let secret_replacers =
+      interpolate(&mut repo, &mut update).await?;
+
     let res = tokio::select! {
       res = periphery
         .request(api::git::CloneRepo {
           args: (&repo).into(),
           git_token,
-          environment: Default::default(),
-          env_file_path: Default::default(),
-          skip_secret_interp: Default::default(),
+          environment: repo.config.env_vars()?,
+          env_file_path: repo.config.env_file_path,
+          on_clone: repo.config.on_clone.into(),
+          on_pull: repo.config.on_pull.into(),
+          skip_secret_interp: repo.config.skip_secret_interp,
+          replacers: secret_replacers.into_iter().collect()
         }) => res,
       _ = cancel.cancelled() => {
         debug!("build cancelled during clone, cleaning up builder");
         update.push_error_log("build cancelled", String::from("user cancelled build during repo clone"));
-        cleanup_builder_instance(periphery, cleanup_data, &mut update)
+        cleanup_builder_instance(cleanup_data, &mut update)
           .await;
         info!("builder cleaned up");
         return handle_builder_early_return(update, repo.id, repo.name, true).await
@@ -359,15 +473,15 @@ impl Resolve<BuildRepo, (User, Update)> for State {
     let commit_message = match res {
       Ok(res) => {
         debug!("finished repo clone");
-        let res: RepoActionResponseV1_13 = res.into();
-        update.logs.extend(res.logs);
-        update.commit_hash = res.commit_hash.unwrap_or_default();
-        res.commit_message.unwrap_or_default()
+        update.logs.extend(res.res.logs);
+        update.commit_hash = res.res.commit_hash.unwrap_or_default();
+
+        res.res.commit_message.unwrap_or_default()
       }
       Err(e) => {
         update.push_error_log(
-          "clone repo",
-          format_serror(&e.context("failed to clone repo").into()),
+          "Clone Repo",
+          format_serror(&e.context("Failed to clone repo").into()),
         );
         Default::default()
       }
@@ -375,7 +489,7 @@ impl Resolve<BuildRepo, (User, Update)> for State {
 
     update.finalize();
 
-    let db = db_client().await;
+    let db = db_client();
 
     if update.success {
       let _ = db
@@ -383,7 +497,7 @@ impl Resolve<BuildRepo, (User, Update)> for State {
         .update_one(
           doc! { "name": &repo.name },
           doc! { "$set": {
-            "info.last_built_at": monitor_timestamp(),
+            "info.last_built_at": komodo_timestamp(),
             "info.built_hash": &update.commit_hash,
             "info.built_message": commit_message
           }},
@@ -394,8 +508,9 @@ impl Resolve<BuildRepo, (User, Update)> for State {
     // stop the cancel listening task from going forever
     cancel.cancel();
 
-    cleanup_builder_instance(periphery, cleanup_data, &mut update)
-      .await;
+    // If building on temporary cloud server (AWS),
+    // this will terminate the server.
+    cleanup_builder_instance(cleanup_data, &mut update).await;
 
     // Need to manually update the update before cache refresh,
     // and before broadcast with add_update.
@@ -405,7 +520,7 @@ impl Resolve<BuildRepo, (User, Update)> for State {
       let _ = update_one_by_id(
         &db.updates,
         &update.id,
-        mungos::update::Update::Set(update_doc),
+        database::mungos::update::Update::Set(update_doc),
         None,
       )
       .await;
@@ -421,8 +536,8 @@ impl Resolve<BuildRepo, (User, Update)> for State {
         let alert = Alert {
           id: Default::default(),
           target,
-          ts: monitor_timestamp(),
-          resolved_ts: Some(monitor_timestamp()),
+          ts: komodo_timestamp(),
+          resolved_ts: Some(komodo_timestamp()),
           resolved: true,
           level: SeverityLevel::Warning,
           data: AlertData::RepoBuildFailed {
@@ -444,7 +559,7 @@ async fn handle_builder_early_return(
   repo_id: String,
   repo_name: String,
   is_cancel: bool,
-) -> anyhow::Result<Update> {
+) -> serror::Result<Update> {
   update.finalize();
   // Need to manually update the update before cache refresh,
   // and before broadcast with add_update.
@@ -452,9 +567,9 @@ async fn handle_builder_early_return(
   // but will fail to update cache in that case.
   if let Ok(update_doc) = to_document(&update) {
     let _ = update_one_by_id(
-      &db_client().await.updates,
+      &db_client().updates,
       &update.id,
-      mungos::update::Update::Set(update_doc),
+      database::mungos::update::Update::Set(update_doc),
       None,
     )
     .await;
@@ -468,8 +583,8 @@ async fn handle_builder_early_return(
       let alert = Alert {
         id: Default::default(),
         target,
-        ts: monitor_timestamp(),
-        resolved_ts: Some(monitor_timestamp()),
+        ts: komodo_timestamp(),
+        resolved_ts: Some(komodo_timestamp()),
         resolved: true,
         level: SeverityLevel::Warning,
         data: AlertData::RepoBuildFailed {
@@ -490,7 +605,7 @@ pub async fn validate_cancel_repo_build(
   if let ExecuteRequest::CancelRepoBuild(req) = request {
     let repo = resource::get::<Repo>(&req.repo).await?;
 
-    let db = db_client().await;
+    let db = db_client();
 
     let (latest_build, latest_cancel) = tokio::try_join!(
       db.updates
@@ -532,17 +647,16 @@ pub async fn validate_cancel_repo_build(
   Ok(())
 }
 
-impl Resolve<CancelRepoBuild, (User, Update)> for State {
-  #[instrument(name = "CancelRepoBuild", skip(self, user, update), fields(user_id = user.id, update_id = update.id))]
+impl Resolve<ExecuteArgs> for CancelRepoBuild {
+  #[instrument(name = "CancelRepoBuild", skip(user, update), fields(user_id = user.id, update_id = update.id))]
   async fn resolve(
-    &self,
-    CancelRepoBuild { repo }: CancelRepoBuild,
-    (user, mut update): (User, Update),
-  ) -> anyhow::Result<Update> {
-    let repo = resource::get_check_permissions::<Repo>(
-      &repo,
-      &user,
-      PermissionLevel::Execute,
+    self,
+    ExecuteArgs { user, update }: &ExecuteArgs,
+  ) -> serror::Result<Update> {
+    let repo = get_check_permissions::<Repo>(
+      &self.repo,
+      user,
+      PermissionLevel::Execute.into(),
     )
     .await?;
 
@@ -554,9 +668,11 @@ impl Resolve<CancelRepoBuild, (User, Update)> for State {
       .and_then(|s| s.get().ok().map(|s| s.building))
       .unwrap_or_default()
     {
-      return Err(anyhow!("Repo is not building."));
+      return Err(anyhow!("Repo is not building.").into());
     }
 
+    let mut update = update.clone();
+
     update.push_simple_log(
       "cancel triggered",
       "the repo build cancel has been triggered",
@@ -575,17 +691,40 @@ impl Resolve<CancelRepoBuild, (User, Update)> for State {
     tokio::spawn(async move {
       tokio::time::sleep(Duration::from_secs(60)).await;
       if let Err(e) = update_one_by_id(
-        &db_client().await.updates,
+        &db_client().updates,
         &update_id,
         doc! { "$set": { "status": "Complete" } },
         None,
       )
       .await
       {
-        warn!("failed to set CancelRepoBuild Update status Complete after timeout | {e:#}")
+        warn!(
+          "failed to set CancelRepoBuild Update status Complete after timeout | {e:#}"
+        )
       }
     });
 
     Ok(update)
   }
 }
+
+async fn interpolate(
+  repo: &mut Repo,
+  update: &mut Update,
+) -> anyhow::Result<HashSet<(String, String)>> {
+  if !repo.config.skip_secret_interp {
+    let VariablesAndSecrets { variables, secrets } =
+      get_variables_and_secrets().await?;
+
+    let mut interpolator =
+      Interpolator::new(Some(&variables), &secrets);
+
+    interpolator
+      .interpolate_repo(repo)?
+      .push_logs(&mut update.logs);
+
+    Ok(interpolator.secret_replacers)
+  } else {
+    Ok(Default::default())
+  }
+}

bin/core/src/api/execute/server_template.rs

@@ -1,141 +0,0 @@
-use anyhow::{anyhow, Context};
-use formatting::format_serror;
-use monitor_client::{
-  api::{execute::LaunchServer, write::CreateServer},
-  entities::{
-    permission::PermissionLevel,
-    server::PartialServerConfig,
-    server_template::{ServerTemplate, ServerTemplateConfig},
-    update::Update,
-    user::User,
-  },
-};
-use mungos::mongodb::bson::doc;
-use resolver_api::Resolve;
-
-use crate::{
-  cloud::{
-    aws::ec2::launch_ec2_instance, hetzner::launch_hetzner_server,
-  },
-  helpers::update::update_update,
-  resource,
-  state::{db_client, State},
-};
-
-impl Resolve<LaunchServer, (User, Update)> for State {
-  #[instrument(name = "LaunchServer", skip(self, user, update), fields(user_id = user.id, update_id = update.id))]
-  async fn resolve(
-    &self,
-    LaunchServer {
-      name,
-      server_template,
-    }: LaunchServer,
-    (user, mut update): (User, Update),
-  ) -> anyhow::Result<Update> {
-    // validate name isn't already taken by another server
-    if db_client()
-      .await
-      .servers
-      .find_one(doc! {
-        "name": &name
-      })
-      .await
-      .context("failed to query db for servers")?
-      .is_some()
-    {
-      return Err(anyhow!("name is already taken"));
-    }
-
-    let template = resource::get_check_permissions::<ServerTemplate>(
-      &server_template,
-      &user,
-      PermissionLevel::Execute,
-    )
-    .await?;
-
-    update.push_simple_log(
-      "launching server",
-      format!("{:#?}", template.config),
-    );
-    update_update(update.clone()).await?;
-
-    let config = match template.config {
-      ServerTemplateConfig::Aws(config) => {
-        let region = config.region.clone();
-        let instance = match launch_ec2_instance(&name, config).await
-        {
-          Ok(instance) => instance,
-          Err(e) => {
-            update.push_error_log(
-              "launch server",
-              format!("failed to launch aws instance\n\n{e:#?}"),
-            );
-            update.finalize();
-            update_update(update.clone()).await?;
-            return Ok(update);
-          }
-        };
-        update.push_simple_log(
-          "launch server",
-          format!(
-            "successfully launched server {name} on ip {}",
-            instance.ip
-          ),
-        );
-        PartialServerConfig {
-          address: format!("http://{}:8120", instance.ip).into(),
-          region: region.into(),
-          ..Default::default()
-        }
-      }
-      ServerTemplateConfig::Hetzner(config) => {
-        let datacenter = config.datacenter;
-        let server = match launch_hetzner_server(&name, config).await
-        {
-          Ok(server) => server,
-          Err(e) => {
-            update.push_error_log(
-              "launch server",
-              format!("failed to launch hetzner server\n\n{e:#?}"),
-            );
-            update.finalize();
-            update_update(update.clone()).await?;
-            return Ok(update);
-          }
-        };
-        update.push_simple_log(
-          "launch server",
-          format!(
-            "successfully launched server {name} on ip {}",
-            server.ip
-          ),
-        );
-        PartialServerConfig {
-          address: format!("http://{}:8120", server.ip).into(),
-          region: datacenter.as_ref().to_string().into(),
-          ..Default::default()
-        }
-      }
-    };
-
-    match self.resolve(CreateServer { name, config }, user).await {
-      Ok(server) => {
-        update.push_simple_log(
-          "create server",
-          format!("created server {} ({})", server.name, server.id),
-        );
-        update.other_data = server.id;
-      }
-      Err(e) => {
-        update.push_error_log(
-          "create server",
-          format_serror(&e.context("failed to create server").into()),
-        );
-      }
-    };
-
-    update.finalize();
-    update_update(update.clone()).await?;
-    Ok(update)
-  }
-}

bin/core/src/api/execute/sync.rs

@@ -1,87 +1,208 @@
-use std::collections::HashMap;
+use std::{collections::HashMap, str::FromStr};
 
-use anyhow::{anyhow, Context};
-use formatting::{colored, format_serror, Color};
-use mongo_indexed::doc;
-use monitor_client::{
+use anyhow::{Context, anyhow};
+use database::mungos::{
+  by_id::update_one_by_id,
+  mongodb::bson::{doc, oid::ObjectId},
+};
+use formatting::{Color, colored, format_serror};
+use komodo_client::{
   api::{execute::RunSync, write::RefreshResourceSyncPending},
   entities::{
-    self,
+    self, ResourceTargetVariant,
+    action::Action,
     alerter::Alerter,
     build::Build,
     builder::Builder,
     deployment::Deployment,
-    monitor_timestamp,
+    komodo_timestamp,
     permission::PermissionLevel,
     procedure::Procedure,
     repo::Repo,
     server::Server,
-    server_template::ServerTemplate,
     stack::Stack,
+    sync::ResourceSync,
     update::{Log, Update},
-    user::{sync_user, User},
+    user::sync_user,
   },
 };
-use mungos::{by_id::update_one_by_id, mongodb::bson::to_document};
 use resolver_api::Resolve;
 
 use crate::{
+  api::write::WriteArgs,
   helpers::{
-    query::get_id_to_tags,
-    sync::{
-      deploy::{
-        build_deploy_cache, deploy_from_cache, SyncDeployParams,
-      },
-      resource::{
-        get_updates_for_execution, AllResourcesById, ResourceSync,
-      },
-    },
+    all_resources::AllResourcesById, query::get_id_to_tags,
     update::update_update,
   },
-  resource::{self, refresh_resource_sync_state_cache},
-  state::{db_client, State},
+  permission::get_check_permissions,
+  state::{action_states, db_client},
+  sync::{
+    ResourceSyncTrait,
+    deploy::{
+      SyncDeployParams, build_deploy_cache, deploy_from_cache,
+    },
+    execute::{ExecuteResourceSync, get_updates_for_execution},
+    remote::RemoteResources,
+  },
 };
 
-impl Resolve<RunSync, (User, Update)> for State {
-  #[instrument(name = "RunSync", skip(self, user, update), fields(user_id = user.id, update_id = update.id))]
+use super::ExecuteArgs;
+
+impl Resolve<ExecuteArgs> for RunSync {
+  #[instrument(name = "RunSync", skip(user, update), fields(user_id = user.id, update_id = update.id))]
   async fn resolve(
-    &self,
-    RunSync { sync }: RunSync,
-    (user, mut update): (User, Update),
-  ) -> anyhow::Result<Update> {
-    let sync = resource::get_check_permissions::<
-      entities::sync::ResourceSync,
-    >(&sync, &user, PermissionLevel::Execute)
+    self,
+    ExecuteArgs { user, update }: &ExecuteArgs,
+  ) -> serror::Result<Update> {
+    let RunSync {
+      sync,
+      resource_type: match_resource_type,
+      resources: match_resources,
+    } = self;
+    let sync = get_check_permissions::<entities::sync::ResourceSync>(
+      &sync,
+      user,
+      PermissionLevel::Execute.into(),
+    )
     .await?;
 
-    if sync.config.repo.is_empty() {
-      return Err(anyhow!("resource sync repo not configured"));
-    }
+    let repo = if !sync.config.files_on_host
+      && !sync.config.linked_repo.is_empty()
+    {
+      crate::resource::get::<Repo>(&sync.config.linked_repo)
+        .await?
+        .into()
+    } else {
+      None
+    };
+
+    // get the action state for the sync (or insert default).
+    let action_state =
+      action_states().sync.get_or_insert_default(&sync.id).await;
+
+    // This will set action state back to default when dropped.
+    // Will also check to ensure sync not already busy before updating.
+    let _action_guard =
+      action_state.update(|state| state.syncing = true)?;
+
+    let mut update = update.clone();
 
     // Send update here for FE to recheck action state
     update_update(update.clone()).await?;
 
-    let (res, logs, hash, message) =
-      crate::helpers::sync::remote::get_remote_resources(&sync)
+    let RemoteResources {
+      resources,
+      logs,
+      hash,
+      message,
+      file_errors,
+      ..
+    } =
+      crate::sync::remote::get_remote_resources(&sync, repo.as_ref())
         .await
         .context("failed to get remote resources")?;
 
     update.logs.extend(logs);
     update_update(update.clone()).await?;
 
-    let resources = res?;
+    if !file_errors.is_empty() {
+      return Err(
+        anyhow!("Found file errors. Cannot execute sync.").into(),
+      );
+    }
+
+    let resources = resources?;
 
     let id_to_tags = get_id_to_tags(None).await?;
     let all_resources = AllResourcesById::load().await?;
+    // Convert all match_resources to names
+    let match_resources = match_resources.map(|resources| {
+      resources
+        .into_iter()
+        .filter_map(|name_or_id| {
+          let Some(resource_type) = match_resource_type else {
+            return Some(name_or_id);
+          };
+          match ObjectId::from_str(&name_or_id) {
+            Ok(_) => match resource_type {
+              ResourceTargetVariant::Alerter => all_resources
+                .alerters
+                .get(&name_or_id)
+                .map(|a| a.name.clone()),
+              ResourceTargetVariant::Build => all_resources
+                .builds
+                .get(&name_or_id)
+                .map(|b| b.name.clone()),
+              ResourceTargetVariant::Builder => all_resources
+                .builders
+                .get(&name_or_id)
+                .map(|b| b.name.clone()),
+              ResourceTargetVariant::Deployment => all_resources
+                .deployments
+                .get(&name_or_id)
+                .map(|d| d.name.clone()),
+              ResourceTargetVariant::Procedure => all_resources
+                .procedures
+                .get(&name_or_id)
+                .map(|p| p.name.clone()),
+              ResourceTargetVariant::Action => all_resources
+                .actions
+                .get(&name_or_id)
+                .map(|p| p.name.clone()),
+              ResourceTargetVariant::Repo => all_resources
+                .repos
+                .get(&name_or_id)
+                .map(|r| r.name.clone()),
+              ResourceTargetVariant::Server => all_resources
+                .servers
+                .get(&name_or_id)
+                .map(|s| s.name.clone()),
+              ResourceTargetVariant::Stack => all_resources
+                .stacks
+                .get(&name_or_id)
+                .map(|s| s.name.clone()),
+              ResourceTargetVariant::ResourceSync => all_resources
+                .syncs
+                .get(&name_or_id)
+                .map(|s| s.name.clone()),
+              ResourceTargetVariant::System => None,
+            },
+            Err(_) => Some(name_or_id),
+          }
+        })
+        .collect::<Vec<_>>()
+    });
 
     let deployments_by_name = all_resources
       .deployments
       .values()
+      .filter(|deployment| {
+        Deployment::include_resource(
+          &deployment.name,
+          &deployment.config,
+          match_resource_type,
+          match_resources.as_deref(),
+          &deployment.tags,
+          &id_to_tags,
+          &sync.config.match_tags,
+        )
+      })
       .map(|deployment| (deployment.name.clone(), deployment.clone()))
       .collect::<HashMap<_, _>>();
     let stacks_by_name = all_resources
       .stacks
       .values()
+      .filter(|stack| {
+        Stack::include_resource(
+          &stack.name,
+          &stack.config,
+          match_resource_type,
+          match_resources.as_deref(),
+          &stack.tags,
+          &id_to_tags,
+          &sync.config.match_tags,
+        )
+      })
       .map(|stack| (stack.name.clone(), stack.clone()))
       .collect::<HashMap<_, _>>();
 
@@ -90,153 +211,186 @@ impl Resolve<RunSync, (User, Update)> for State {
       deployment_map: &deployments_by_name,
       stacks: &resources.stacks,
       stack_map: &stacks_by_name,
-      all_resources: &all_resources,
     })
     .await?;
 
-    let (servers_to_create, servers_to_update, servers_to_delete) =
+    let delete = sync.config.managed || sync.config.delete;
+
+    let server_deltas = if sync.config.include_resources {
       get_updates_for_execution::<Server>(
         resources.servers,
-        sync.config.delete,
-        &all_resources,
+        delete,
+        match_resource_type,
+        match_resources.as_deref(),
         &id_to_tags,
+        &sync.config.match_tags,
       )
-      .await?;
-    let (
-      deployments_to_create,
-      deployments_to_update,
-      deployments_to_delete,
-    ) = get_updates_for_execution::<Deployment>(
-      resources.deployments,
-      sync.config.delete,
-      &all_resources,
-      &id_to_tags,
-    )
-    .await?;
-    let (stacks_to_create, stacks_to_update, stacks_to_delete) =
+      .await?
+    } else {
+      Default::default()
+    };
+    let stack_deltas = if sync.config.include_resources {
       get_updates_for_execution::<Stack>(
         resources.stacks,
-        sync.config.delete,
-        &all_resources,
+        delete,
+        match_resource_type,
+        match_resources.as_deref(),
         &id_to_tags,
+        &sync.config.match_tags,
       )
-      .await?;
-    let (builds_to_create, builds_to_update, builds_to_delete) =
+      .await?
+    } else {
+      Default::default()
+    };
+    let deployment_deltas = if sync.config.include_resources {
+      get_updates_for_execution::<Deployment>(
+        resources.deployments,
+        delete,
+        match_resource_type,
+        match_resources.as_deref(),
+        &id_to_tags,
+        &sync.config.match_tags,
+      )
+      .await?
+    } else {
+      Default::default()
+    };
+    let build_deltas = if sync.config.include_resources {
       get_updates_for_execution::<Build>(
         resources.builds,
-        sync.config.delete,
-        &all_resources,
+        delete,
+        match_resource_type,
+        match_resources.as_deref(),
         &id_to_tags,
+        &sync.config.match_tags,
       )
-      .await?;
-    let (repos_to_create, repos_to_update, repos_to_delete) =
+      .await?
+    } else {
+      Default::default()
+    };
+    let repo_deltas = if sync.config.include_resources {
       get_updates_for_execution::<Repo>(
         resources.repos,
-        sync.config.delete,
-        &all_resources,
+        delete,
+        match_resource_type,
+        match_resources.as_deref(),
         &id_to_tags,
+        &sync.config.match_tags,
       )
-      .await?;
-    let (
-      procedures_to_create,
-      procedures_to_update,
-      procedures_to_delete,
-    ) = get_updates_for_execution::<Procedure>(
-      resources.procedures,
-      sync.config.delete,
-      &all_resources,
-      &id_to_tags,
-    )
-    .await?;
-    let (builders_to_create, builders_to_update, builders_to_delete) =
+      .await?
+    } else {
+      Default::default()
+    };
+    let procedure_deltas = if sync.config.include_resources {
+      get_updates_for_execution::<Procedure>(
+        resources.procedures,
+        delete,
+        match_resource_type,
+        match_resources.as_deref(),
+        &id_to_tags,
+        &sync.config.match_tags,
+      )
+      .await?
+    } else {
+      Default::default()
+    };
+    let action_deltas = if sync.config.include_resources {
+      get_updates_for_execution::<Action>(
+        resources.actions,
+        delete,
+        match_resource_type,
+        match_resources.as_deref(),
+        &id_to_tags,
+        &sync.config.match_tags,
+      )
+      .await?
+    } else {
+      Default::default()
+    };
+    let builder_deltas = if sync.config.include_resources {
       get_updates_for_execution::<Builder>(
         resources.builders,
-        sync.config.delete,
-        &all_resources,
+        delete,
+        match_resource_type,
+        match_resources.as_deref(),
         &id_to_tags,
+        &sync.config.match_tags,
       )
-      .await?;
-    let (alerters_to_create, alerters_to_update, alerters_to_delete) =
+      .await?
+    } else {
+      Default::default()
+    };
+    let alerter_deltas = if sync.config.include_resources {
       get_updates_for_execution::<Alerter>(
         resources.alerters,
-        sync.config.delete,
-        &all_resources,
+        delete,
+        match_resource_type,
+        match_resources.as_deref(),
         &id_to_tags,
+        &sync.config.match_tags,
       )
-      .await?;
-    let (
-      server_templates_to_create,
-      server_templates_to_update,
-      server_templates_to_delete,
-    ) = get_updates_for_execution::<ServerTemplate>(
-      resources.server_templates,
-      sync.config.delete,
-      &all_resources,
-      &id_to_tags,
-    )
-    .await?;
-    let (
-      resource_syncs_to_create,
-      resource_syncs_to_update,
-      resource_syncs_to_delete,
-    ) = get_updates_for_execution::<entities::sync::ResourceSync>(
-      resources.resource_syncs,
-      sync.config.delete,
-      &all_resources,
-      &id_to_tags,
-    )
-    .await?;
+      .await?
+    } else {
+      Default::default()
+    };
+    let resource_sync_deltas = if sync.config.include_resources {
+      get_updates_for_execution::<entities::sync::ResourceSync>(
+        resources.resource_syncs,
+        delete,
+        match_resource_type,
+        match_resources.as_deref(),
+        &id_to_tags,
+        &sync.config.match_tags,
+      )
+      .await?
+    } else {
+      Default::default()
+    };
+
     let (
       variables_to_create,
       variables_to_update,
       variables_to_delete,
-    ) = crate::helpers::sync::variables::get_updates_for_execution(
-      resources.variables,
-      sync.config.delete,
-    )
-    .await?;
+    ) = if match_resource_type.is_none()
+      && match_resources.is_none()
+      && sync.config.include_variables
+    {
+      crate::sync::variables::get_updates_for_execution(
+        resources.variables,
+        delete,
+      )
+      .await?
+    } else {
+      Default::default()
+    };
     let (
       user_groups_to_create,
       user_groups_to_update,
       user_groups_to_delete,
-    ) = crate::helpers::sync::user_groups::get_updates_for_execution(
-      resources.user_groups,
-      sync.config.delete,
-      &all_resources,
-    )
-    .await?;
+    ) = if match_resource_type.is_none()
+      && match_resources.is_none()
+      && sync.config.include_user_groups
+    {
+      crate::sync::user_groups::get_updates_for_execution(
+        resources.user_groups,
+        delete,
+      )
+      .await?
+    } else {
+      Default::default()
+    };
 
     if deploy_cache.is_empty()
-      && resource_syncs_to_create.is_empty()
-      && resource_syncs_to_update.is_empty()
-      && resource_syncs_to_delete.is_empty()
-      && server_templates_to_create.is_empty()
-      && server_templates_to_update.is_empty()
-      && server_templates_to_delete.is_empty()
-      && servers_to_create.is_empty()
-      && servers_to_update.is_empty()
-      && servers_to_delete.is_empty()
-      && deployments_to_create.is_empty()
-      && deployments_to_update.is_empty()
-      && deployments_to_delete.is_empty()
-      && stacks_to_create.is_empty()
-      && stacks_to_update.is_empty()
-      && stacks_to_delete.is_empty()
-      && builds_to_create.is_empty()
-      && builds_to_update.is_empty()
-      && builds_to_delete.is_empty()
-      && builders_to_create.is_empty()
-      && builders_to_update.is_empty()
-      && builders_to_delete.is_empty()
-      && alerters_to_create.is_empty()
-      && alerters_to_update.is_empty()
-      && alerters_to_delete.is_empty()
-      && repos_to_create.is_empty()
-      && repos_to_update.is_empty()
-      && repos_to_delete.is_empty()
-      && procedures_to_create.is_empty()
-      && procedures_to_update.is_empty()
-      && procedures_to_delete.is_empty()
+      && resource_sync_deltas.no_changes()
+      && server_deltas.no_changes()
+      && deployment_deltas.no_changes()
+      && stack_deltas.no_changes()
+      && build_deltas.no_changes()
+      && builder_deltas.no_changes()
+      && alerter_deltas.no_changes()
+      && repo_deltas.no_changes()
+      && procedure_deltas.no_changes()
+      && action_deltas.no_changes()
       && user_groups_to_create.is_empty()
       && user_groups_to_update.is_empty()
       && user_groups_to_delete.is_empty()
@@ -261,7 +415,7 @@ impl Resolve<RunSync, (User, Update)> for State {
     // No deps
     maybe_extend(
       &mut update.logs,
-      crate::helpers::sync::variables::run_updates(
+      crate::sync::variables::run_updates(
         variables_to_create,
         variables_to_update,
         variables_to_delete,
@@ -270,7 +424,7 @@ impl Resolve<RunSync, (User, Update)> for State {
     );
     maybe_extend(
       &mut update.logs,
-      crate::helpers::sync::user_groups::run_updates(
+      crate::sync::user_groups::run_updates(
         user_groups_to_create,
         user_groups_to_update,
         user_groups_to_delete,
@@ -279,115 +433,65 @@ impl Resolve<RunSync, (User, Update)> for State {
     );
     maybe_extend(
       &mut update.logs,
-      entities::sync::ResourceSync::run_updates(
-        resource_syncs_to_create,
-        resource_syncs_to_update,
-        resource_syncs_to_delete,
-      )
-      .await,
+      ResourceSync::execute_sync_updates(resource_sync_deltas).await,
     );
     maybe_extend(
       &mut update.logs,
-      ServerTemplate::run_updates(
-        server_templates_to_create,
-        server_templates_to_update,
-        server_templates_to_delete,
-      )
-      .await,
+      Server::execute_sync_updates(server_deltas).await,
     );
     maybe_extend(
       &mut update.logs,
-      Server::run_updates(
-        servers_to_create,
-        servers_to_update,
-        servers_to_delete,
-      )
-      .await,
+      Alerter::execute_sync_updates(alerter_deltas).await,
     );
     maybe_extend(
       &mut update.logs,
-      Alerter::run_updates(
-        alerters_to_create,
-        alerters_to_update,
-        alerters_to_delete,
-      )
-      .await,
+      Action::execute_sync_updates(action_deltas).await,
     );
 
     // Dependent on server
     maybe_extend(
       &mut update.logs,
-      Builder::run_updates(
-        builders_to_create,
-        builders_to_update,
-        builders_to_delete,
-      )
-      .await,
+      Builder::execute_sync_updates(builder_deltas).await,
     );
     maybe_extend(
       &mut update.logs,
-      Repo::run_updates(
-        repos_to_create,
-        repos_to_update,
-        repos_to_delete,
-      )
-      .await,
+      Repo::execute_sync_updates(repo_deltas).await,
     );
 
     // Dependant on builder
     maybe_extend(
       &mut update.logs,
-      Build::run_updates(
-        builds_to_create,
-        builds_to_update,
-        builds_to_delete,
-      )
-      .await,
+      Build::execute_sync_updates(build_deltas).await,
     );
 
     // Dependant on server / build
     maybe_extend(
       &mut update.logs,
-      Deployment::run_updates(
-        deployments_to_create,
-        deployments_to_update,
-        deployments_to_delete,
-      )
-      .await,
+      Deployment::execute_sync_updates(deployment_deltas).await,
     );
     // stack only depends on server, but maybe will depend on build later.
     maybe_extend(
       &mut update.logs,
-      Stack::run_updates(
-        stacks_to_create,
-        stacks_to_update,
-        stacks_to_delete,
-      )
-      .await,
+      Stack::execute_sync_updates(stack_deltas).await,
     );
 
     // Dependant on everything
     maybe_extend(
       &mut update.logs,
-      Procedure::run_updates(
-        procedures_to_create,
-        procedures_to_update,
-        procedures_to_delete,
-      )
-      .await,
+      Procedure::execute_sync_updates(procedure_deltas).await,
     );
 
     // Execute the deploy cache
     deploy_from_cache(deploy_cache, &mut update.logs).await;
 
-    let db = db_client().await;
+    let db = db_client();
 
     if let Err(e) = update_one_by_id(
       &db.resource_syncs,
       &sync.id,
       doc! {
         "$set": {
-          "info.last_sync_ts": monitor_timestamp(),
+          "info.last_sync_ts": komodo_timestamp(),
           "info.last_sync_hash": hash,
           "info.last_sync_message": message,
         }
@@ -402,39 +506,27 @@ impl Resolve<RunSync, (User, Update)> for State {
       )
     }
 
-    if let Err(e) = State
-      .resolve(
-        RefreshResourceSyncPending { sync: sync.id },
-        sync_user().to_owned(),
-      )
+    if let Err(e) = (RefreshResourceSyncPending { sync: sync.id })
+      .resolve(&WriteArgs {
+        user: sync_user().to_owned(),
+      })
       .await
     {
-      warn!("failed to refresh sync {} after run | {e:#}", sync.name);
+      warn!(
+        "failed to refresh sync {} after run | {:#}",
+        sync.name, e.error
+      );
       update.push_error_log(
         "refresh sync",
         format_serror(
-          &e.context("failed to refresh sync pending after run")
+          &e.error
+            .context("failed to refresh sync pending after run")
             .into(),
         ),
       );
     }
 
     update.finalize();
-
-    // Need to manually update the update before cache refresh,
-    // and before broadcast with add_update.
-    // The Err case of to_document should be unreachable,
-    // but will fail to update cache in that case.
-    if let Ok(update_doc) = to_document(&update) {
-      let _ = update_one_by_id(
-        &db.updates,
-        &update.id,
-        mungos::update::Update::Set(update_doc),
-        None,
-      )
-      .await;
-      refresh_resource_sync_state_cache().await;
-    }
     update_update(update.clone()).await?;
 
     Ok(update)

bin/core/src/api/mod.rs

@@ -1,5 +1,11 @@
 pub mod auth;
 pub mod execute;
 pub mod read;
+pub mod terminal;
 pub mod user;
 pub mod write;
+
+#[derive(serde::Deserialize)]
+struct Variant {
+  variant: String,
+}

bin/core/src/api/read/action.rs

@@ -0,0 +1,147 @@
+use anyhow::Context;
+use komodo_client::{
+  api::read::*,
+  entities::{
+    action::{
+      Action, ActionActionState, ActionListItem, ActionState,
+    },
+    permission::PermissionLevel,
+  },
+};
+use resolver_api::Resolve;
+
+use crate::{
+  helpers::query::get_all_tags,
+  permission::get_check_permissions,
+  resource,
+  state::{action_state_cache, action_states},
+};
+
+use super::ReadArgs;
+
+impl Resolve<ReadArgs> for GetAction {
+  async fn resolve(
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<Action> {
+    Ok(
+      get_check_permissions::<Action>(
+        &self.action,
+        user,
+        PermissionLevel::Read.into(),
+      )
+      .await?,
+    )
+  }
+}
+
+impl Resolve<ReadArgs> for ListActions {
+  async fn resolve(
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<Vec<ActionListItem>> {
+    let all_tags = if self.query.tags.is_empty() {
+      vec![]
+    } else {
+      get_all_tags(None).await?
+    };
+    Ok(
+      resource::list_for_user::<Action>(
+        self.query,
+        user,
+        PermissionLevel::Read.into(),
+        &all_tags,
+      )
+      .await?,
+    )
+  }
+}
+
+impl Resolve<ReadArgs> for ListFullActions {
+  async fn resolve(
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<ListFullActionsResponse> {
+    let all_tags = if self.query.tags.is_empty() {
+      vec![]
+    } else {
+      get_all_tags(None).await?
+    };
+    Ok(
+      resource::list_full_for_user::<Action>(
+        self.query,
+        user,
+        PermissionLevel::Read.into(),
+        &all_tags,
+      )
+      .await?,
+    )
+  }
+}
+
+impl Resolve<ReadArgs> for GetActionActionState {
+  async fn resolve(
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<ActionActionState> {
+    let action = get_check_permissions::<Action>(
+      &self.action,
+      user,
+      PermissionLevel::Read.into(),
+    )
+    .await?;
+    let action_state = action_states()
+      .action
+      .get(&action.id)
+      .await
+      .unwrap_or_default()
+      .get()?;
+    Ok(action_state)
+  }
+}
+
+impl Resolve<ReadArgs> for GetActionsSummary {
+  async fn resolve(
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<GetActionsSummaryResponse> {
+    let actions = resource::list_full_for_user::<Action>(
+      Default::default(),
+      user,
+      PermissionLevel::Read.into(),
+      &[],
+    )
+    .await
+    .context("failed to get actions from db")?;
+
+    let mut res = GetActionsSummaryResponse::default();
+
+    let cache = action_state_cache();
+    let action_states = action_states();
+
+    for action in actions {
+      res.total += 1;
+
+      match (
+        cache.get(&action.id).await.unwrap_or_default(),
+        action_states
+          .action
+          .get(&action.id)
+          .await
+          .unwrap_or_default()
+          .get()?,
+      ) {
+        (_, action_states) if action_states.running > 0 => {
+          res.running += action_states.running;
+        }
+        (ActionState::Ok, _) => res.ok += 1,
+        (ActionState::Failed, _) => res.failed += 1,
+        (ActionState::Unknown, _) => res.unknown += 1,
+        // will never come off the cache in the running state, since that comes from action states
+        (ActionState::Running, _) => unreachable!(),
+      }
+    }
+
+    Ok(res)
+  }
+}

bin/core/src/api/read/alert.rs

@@ -1,52 +1,61 @@
 use anyhow::Context;
-use monitor_client::{
-  api::read::{
-    GetAlert, GetAlertResponse, ListAlerts, ListAlertsResponse,
-  },
-  entities::{deployment::Deployment, server::Server, user::User},
-};
-use mungos::{
+use database::mungos::{
   by_id::find_one_by_id,
   find::find_collect,
   mongodb::{bson::doc, options::FindOptions},
 };
+use komodo_client::{
+  api::read::{
+    GetAlert, GetAlertResponse, ListAlerts, ListAlertsResponse,
+  },
+  entities::{
+    deployment::Deployment, server::Server, stack::Stack,
+    sync::ResourceSync,
+  },
+};
 use resolver_api::Resolve;
 
 use crate::{
-  config::core_config,
-  resource::get_resource_ids_for_user,
-  state::{db_client, State},
+  config::core_config, permission::get_resource_ids_for_user,
+  state::db_client,
 };
 
+use super::ReadArgs;
+
 const NUM_ALERTS_PER_PAGE: u64 = 100;
 
-impl Resolve<ListAlerts, User> for State {
+impl Resolve<ReadArgs> for ListAlerts {
   async fn resolve(
-    &self,
-    ListAlerts { query, page }: ListAlerts,
-    user: User,
-  ) -> anyhow::Result<ListAlertsResponse> {
-    let mut query = query.unwrap_or_default();
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<ListAlertsResponse> {
+    let mut query = self.query.unwrap_or_default();
     if !user.admin && !core_config().transparent_mode {
       let server_ids =
-        get_resource_ids_for_user::<Server>(&user).await?;
+        get_resource_ids_for_user::<Server>(user).await?;
+      let stack_ids =
+        get_resource_ids_for_user::<Stack>(user).await?;
       let deployment_ids =
-        get_resource_ids_for_user::<Deployment>(&user).await?;
+        get_resource_ids_for_user::<Deployment>(user).await?;
+      let sync_ids =
+        get_resource_ids_for_user::<ResourceSync>(user).await?;
       query.extend(doc! {
         "$or": [
           { "target.type": "Server", "target.id": { "$in": &server_ids } },
+          { "target.type": "Stack", "target.id": { "$in": &stack_ids } },
           { "target.type": "Deployment", "target.id": { "$in": &deployment_ids } },
+          { "target.type": "ResourceSync", "target.id": { "$in": &sync_ids } },
         ]
       });
     }
 
     let alerts = find_collect(
-      &db_client().await.alerts,
+      &db_client().alerts,
       query,
       FindOptions::builder()
         .sort(doc! { "ts": -1 })
         .limit(NUM_ALERTS_PER_PAGE as i64)
-        .skip(page * NUM_ALERTS_PER_PAGE)
+        .skip(self.page * NUM_ALERTS_PER_PAGE)
         .build(),
     )
     .await
@@ -55,7 +64,7 @@ impl Resolve<ListAlerts, User> for State {
     let next_page = if alerts.len() < NUM_ALERTS_PER_PAGE as usize {
       None
     } else {
-      Some((page + 1) as i64)
+      Some((self.page + 1) as i64)
     };
 
     let res = ListAlertsResponse { next_page, alerts };
@@ -64,15 +73,16 @@ impl Resolve<ListAlerts, User> for State {
   }
 }
 
-impl Resolve<GetAlert, User> for State {
+impl Resolve<ReadArgs> for GetAlert {
   async fn resolve(
-    &self,
-    GetAlert { id }: GetAlert,
-    _: User,
-  ) -> anyhow::Result<GetAlertResponse> {
-    find_one_by_id(&db_client().await.alerts, &id)
-      .await
-      .context("failed to query db for alert")?
-      .context("no alert found with given id")
+    self,
+    _: &ReadArgs,
+  ) -> serror::Result<GetAlertResponse> {
+    Ok(
+      find_one_by_id(&db_client().alerts, &self.id)
+        .await
+        .context("failed to query db for alert")?
+        .context("no alert found with given id")?,
+    )
   }
 }

bin/core/src/api/read/alerter.rs

@@ -1,73 +1,98 @@
 use anyhow::Context;
-use mongo_indexed::Document;
-use monitor_client::{
+use database::mongo_indexed::Document;
+use database::mungos::mongodb::bson::doc;
+use komodo_client::{
   api::read::*,
   entities::{
     alerter::{Alerter, AlerterListItem},
     permission::PermissionLevel,
-    user::User,
   },
 };
-use mungos::mongodb::bson::doc;
 use resolver_api::Resolve;
 
 use crate::{
-  resource,
-  state::{db_client, State},
+  helpers::query::get_all_tags, permission::get_check_permissions,
+  resource, state::db_client,
 };
 
-impl Resolve<GetAlerter, User> for State {
+use super::ReadArgs;
+
+impl Resolve<ReadArgs> for GetAlerter {
   async fn resolve(
-    &self,
-    GetAlerter { alerter }: GetAlerter,
-    user: User,
-  ) -> anyhow::Result<Alerter> {
-    resource::get_check_permissions::<Alerter>(
-      &alerter,
-      &user,
-      PermissionLevel::Read,
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<Alerter> {
+    Ok(
+      get_check_permissions::<Alerter>(
+        &self.alerter,
+        user,
+        PermissionLevel::Read.into(),
+      )
+      .await?,
     )
-    .await
   }
 }
 
-impl Resolve<ListAlerters, User> for State {
+impl Resolve<ReadArgs> for ListAlerters {
   async fn resolve(
-    &self,
-    ListAlerters { query }: ListAlerters,
-    user: User,
-  ) -> anyhow::Result<Vec<AlerterListItem>> {
-    resource::list_for_user::<Alerter>(query, &user).await
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<Vec<AlerterListItem>> {
+    let all_tags = if self.query.tags.is_empty() {
+      vec![]
+    } else {
+      get_all_tags(None).await?
+    };
+    Ok(
+      resource::list_for_user::<Alerter>(
+        self.query,
+        user,
+        PermissionLevel::Read.into(),
+        &all_tags,
+      )
+      .await?,
+    )
   }
 }
 
-impl Resolve<ListFullAlerters, User> for State {
+impl Resolve<ReadArgs> for ListFullAlerters {
   async fn resolve(
-    &self,
-    ListFullAlerters { query }: ListFullAlerters,
-    user: User,
-  ) -> anyhow::Result<ListFullAlertersResponse> {
-    resource::list_full_for_user::<Alerter>(query, &user).await
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<ListFullAlertersResponse> {
+    let all_tags = if self.query.tags.is_empty() {
+      vec![]
+    } else {
+      get_all_tags(None).await?
+    };
+    Ok(
+      resource::list_full_for_user::<Alerter>(
+        self.query,
+        user,
+        PermissionLevel::Read.into(),
+        &all_tags,
+      )
+      .await?,
+    )
   }
 }
 
-impl Resolve<GetAlertersSummary, User> for State {
+impl Resolve<ReadArgs> for GetAlertersSummary {
   async fn resolve(
-    &self,
-    GetAlertersSummary {}: GetAlertersSummary,
-    user: User,
-  ) -> anyhow::Result<GetAlertersSummaryResponse> {
-    let query =
-      match resource::get_resource_ids_for_user::<Alerter>(&user)
-        .await?
-      {
-        Some(ids) => doc! {
-          "_id": { "$in": ids }
-        },
-        None => Document::new(),
-      };
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<GetAlertersSummaryResponse> {
+    let query = match resource::get_resource_object_ids_for_user::<
+      Alerter,
+    >(user)
+    .await?
+    {
+      Some(ids) => doc! {
+        "_id": { "$in": ids }
+      },
+      None => Document::new(),
+    };
     let total = db_client()
-      .await
       .alerters
       .count_documents(query)
       .await

bin/core/src/api/read/build.rs

@@ -2,77 +2,104 @@ use std::collections::{HashMap, HashSet};
 
 use anyhow::Context;
 use async_timing_util::unix_timestamp_ms;
+use database::mungos::{
+  find::find_collect,
+  mongodb::{bson::doc, options::FindOptions},
+};
 use futures::TryStreamExt;
-use monitor_client::{
+use komodo_client::{
   api::read::*,
   entities::{
+    Operation,
     build::{Build, BuildActionState, BuildListItem, BuildState},
     config::core::CoreConfig,
     permission::PermissionLevel,
     update::UpdateStatus,
-    user::User,
-    Operation,
   },
 };
-use mungos::{
-  find::find_collect,
-  mongodb::{bson::doc, options::FindOptions},
-};
 use resolver_api::Resolve;
 
 use crate::{
   config::core_config,
+  helpers::query::get_all_tags,
+  permission::get_check_permissions,
   resource,
   state::{
-    action_states, build_state_cache, db_client, github_client, State,
+    action_states, build_state_cache, db_client, github_client,
   },
 };
 
-impl Resolve<GetBuild, User> for State {
+use super::ReadArgs;
+
+impl Resolve<ReadArgs> for GetBuild {
   async fn resolve(
-    &self,
-    GetBuild { build }: GetBuild,
-    user: User,
-  ) -> anyhow::Result<Build> {
-    resource::get_check_permissions::<Build>(
-      &build,
-      &user,
-      PermissionLevel::Read,
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<Build> {
+    Ok(
+      get_check_permissions::<Build>(
+        &self.build,
+        user,
+        PermissionLevel::Read.into(),
+      )
+      .await?,
     )
-    .await
   }
 }
 
-impl Resolve<ListBuilds, User> for State {
+impl Resolve<ReadArgs> for ListBuilds {
   async fn resolve(
-    &self,
-    ListBuilds { query }: ListBuilds,
-    user: User,
-  ) -> anyhow::Result<Vec<BuildListItem>> {
-    resource::list_for_user::<Build>(query, &user).await
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<Vec<BuildListItem>> {
+    let all_tags = if self.query.tags.is_empty() {
+      vec![]
+    } else {
+      get_all_tags(None).await?
+    };
+    Ok(
+      resource::list_for_user::<Build>(
+        self.query,
+        user,
+        PermissionLevel::Read.into(),
+        &all_tags,
+      )
+      .await?,
+    )
   }
 }
 
-impl Resolve<ListFullBuilds, User> for State {
+impl Resolve<ReadArgs> for ListFullBuilds {
   async fn resolve(
-    &self,
-    ListFullBuilds { query }: ListFullBuilds,
-    user: User,
-  ) -> anyhow::Result<ListFullBuildsResponse> {
-    resource::list_full_for_user::<Build>(query, &user).await
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<ListFullBuildsResponse> {
+    let all_tags = if self.query.tags.is_empty() {
+      vec![]
+    } else {
+      get_all_tags(None).await?
+    };
+    Ok(
+      resource::list_full_for_user::<Build>(
+        self.query,
+        user,
+        PermissionLevel::Read.into(),
+        &all_tags,
+      )
+      .await?,
+    )
   }
 }
 
-impl Resolve<GetBuildActionState, User> for State {
+impl Resolve<ReadArgs> for GetBuildActionState {
   async fn resolve(
-    &self,
-    GetBuildActionState { build }: GetBuildActionState,
-    user: User,
-  ) -> anyhow::Result<BuildActionState> {
-    let build = resource::get_check_permissions::<Build>(
-      &build,
-      &user,
-      PermissionLevel::Read,
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<BuildActionState> {
+    let build = get_check_permissions::<Build>(
+      &self.build,
+      user,
+      PermissionLevel::Read.into(),
     )
     .await?;
     let action_state = action_states()
@@ -85,15 +112,16 @@ impl Resolve<GetBuildActionState, User> for State {
   }
 }
 
-impl Resolve<GetBuildsSummary, User> for State {
+impl Resolve<ReadArgs> for GetBuildsSummary {
   async fn resolve(
-    &self,
-    GetBuildsSummary {}: GetBuildsSummary,
-    user: User,
-  ) -> anyhow::Result<GetBuildsSummaryResponse> {
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<GetBuildsSummaryResponse> {
     let builds = resource::list_full_for_user::<Build>(
       Default::default(),
-      &user,
+      user,
+      PermissionLevel::Read.into(),
+      &[],
     )
     .await
     .context("failed to get all builds")?;
@@ -132,20 +160,18 @@ impl Resolve<GetBuildsSummary, User> for State {
 
 const ONE_DAY_MS: i64 = 86400000;
 
-impl Resolve<GetBuildMonthlyStats, User> for State {
+impl Resolve<ReadArgs> for GetBuildMonthlyStats {
   async fn resolve(
-    &self,
-    GetBuildMonthlyStats { page }: GetBuildMonthlyStats,
-    _: User,
-  ) -> anyhow::Result<GetBuildMonthlyStatsResponse> {
+    self,
+    _: &ReadArgs,
+  ) -> serror::Result<GetBuildMonthlyStatsResponse> {
     let curr_ts = unix_timestamp_ms() as i64;
     let next_day = curr_ts - curr_ts % ONE_DAY_MS + ONE_DAY_MS;
 
-    let close_ts = next_day - page as i64 * 30 * ONE_DAY_MS;
+    let close_ts = next_day - self.page as i64 * 30 * ONE_DAY_MS;
     let open_ts = close_ts - 30 * ONE_DAY_MS;
 
     let mut build_updates = db_client()
-      .await
       .updates
       .find(doc! {
         "start_ts": {
@@ -190,22 +216,22 @@ fn ms_to_hour(duration: i64) -> f64 {
   duration as f64 / MS_TO_HOUR_DIVISOR
 }
 
-impl Resolve<ListBuildVersions, User> for State {
+impl Resolve<ReadArgs> for ListBuildVersions {
   async fn resolve(
-    &self,
-    ListBuildVersions {
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<Vec<BuildVersionResponseItem>> {
+    let ListBuildVersions {
       build,
       major,
       minor,
       patch,
       limit,
-    }: ListBuildVersions,
-    user: User,
-  ) -> anyhow::Result<Vec<BuildVersionResponseItem>> {
-    let build = resource::get_check_permissions::<Build>(
+    } = self;
+    let build = get_check_permissions::<Build>(
       &build,
-      &user,
-      PermissionLevel::Read,
+      user,
+      PermissionLevel::Read.into(),
     )
     .await?;
 
@@ -229,7 +255,7 @@ impl Resolve<ListBuildVersions, User> for State {
     }
 
     let versions = find_collect(
-      &db_client().await.updates,
+      &db_client().updates,
       filter,
       FindOptions::builder()
         .sort(doc! { "_id": -1 })
@@ -247,15 +273,24 @@ impl Resolve<ListBuildVersions, User> for State {
   }
 }
 
-impl Resolve<ListCommonBuildExtraArgs, User> for State {
+impl Resolve<ReadArgs> for ListCommonBuildExtraArgs {
   async fn resolve(
-    &self,
-    ListCommonBuildExtraArgs { query }: ListCommonBuildExtraArgs,
-    user: User,
-  ) -> anyhow::Result<ListCommonBuildExtraArgsResponse> {
-    let builds = resource::list_full_for_user::<Build>(query, &user)
-      .await
-      .context("failed to get resources matching query")?;
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<ListCommonBuildExtraArgsResponse> {
+    let all_tags = if self.query.tags.is_empty() {
+      vec![]
+    } else {
+      get_all_tags(None).await?
+    };
+    let builds = resource::list_full_for_user::<Build>(
+      self.query,
+      user,
+      PermissionLevel::Read.into(),
+      &all_tags,
+    )
+    .await
+    .context("failed to get resources matching query")?;
 
     // first collect with guaranteed uniqueness
     let mut res = HashSet::<String>::new();
@@ -272,12 +307,11 @@ impl Resolve<ListCommonBuildExtraArgs, User> for State {
   }
 }
 
-impl Resolve<GetBuildWebhookEnabled, User> for State {
+impl Resolve<ReadArgs> for GetBuildWebhookEnabled {
   async fn resolve(
-    &self,
-    GetBuildWebhookEnabled { build }: GetBuildWebhookEnabled,
-    user: User,
-  ) -> anyhow::Result<GetBuildWebhookEnabledResponse> {
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<GetBuildWebhookEnabledResponse> {
     let Some(github) = github_client() else {
       return Ok(GetBuildWebhookEnabledResponse {
         managed: false,
@@ -285,10 +319,10 @@ impl Resolve<GetBuildWebhookEnabled, User> for State {
       });
     };
 
-    let build = resource::get_check_permissions::<Build>(
-      &build,
-      &user,
-      PermissionLevel::Read,
+    let build = get_check_permissions::<Build>(
+      &self.build,
+      user,
+      PermissionLevel::Read.into(),
     )
     .await?;
 
@@ -328,7 +362,11 @@ impl Resolve<GetBuildWebhookEnabled, User> for State {
       ..
     } = core_config();
 
-    let host = webhook_base_url.as_ref().unwrap_or(host);
+    let host = if webhook_base_url.is_empty() {
+      host
+    } else {
+      webhook_base_url
+    };
     let url = format!("{host}/listener/github/build/{}", build.id);
 
     for webhook in webhooks {

bin/core/src/api/read/builder.rs

@@ -1,73 +1,98 @@
 use anyhow::Context;
-use mongo_indexed::Document;
-use monitor_client::{
+use database::mongo_indexed::Document;
+use database::mungos::mongodb::bson::doc;
+use komodo_client::{
   api::read::*,
   entities::{
     builder::{Builder, BuilderListItem},
     permission::PermissionLevel,
-    user::User,
   },
 };
-use mungos::mongodb::bson::doc;
 use resolver_api::Resolve;
 
 use crate::{
-  resource,
-  state::{db_client, State},
+  helpers::query::get_all_tags, permission::get_check_permissions,
+  resource, state::db_client,
 };
 
-impl Resolve<GetBuilder, User> for State {
+use super::ReadArgs;
+
+impl Resolve<ReadArgs> for GetBuilder {
   async fn resolve(
-    &self,
-    GetBuilder { builder }: GetBuilder,
-    user: User,
-  ) -> anyhow::Result<Builder> {
-    resource::get_check_permissions::<Builder>(
-      &builder,
-      &user,
-      PermissionLevel::Read,
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<Builder> {
+    Ok(
+      get_check_permissions::<Builder>(
+        &self.builder,
+        user,
+        PermissionLevel::Read.into(),
+      )
+      .await?,
     )
-    .await
   }
 }
 
-impl Resolve<ListBuilders, User> for State {
+impl Resolve<ReadArgs> for ListBuilders {
   async fn resolve(
-    &self,
-    ListBuilders { query }: ListBuilders,
-    user: User,
-  ) -> anyhow::Result<Vec<BuilderListItem>> {
-    resource::list_for_user::<Builder>(query, &user).await
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<Vec<BuilderListItem>> {
+    let all_tags = if self.query.tags.is_empty() {
+      vec![]
+    } else {
+      get_all_tags(None).await?
+    };
+    Ok(
+      resource::list_for_user::<Builder>(
+        self.query,
+        user,
+        PermissionLevel::Read.into(),
+        &all_tags,
+      )
+      .await?,
+    )
   }
 }
 
-impl Resolve<ListFullBuilders, User> for State {
+impl Resolve<ReadArgs> for ListFullBuilders {
   async fn resolve(
-    &self,
-    ListFullBuilders { query }: ListFullBuilders,
-    user: User,
-  ) -> anyhow::Result<ListFullBuildersResponse> {
-    resource::list_full_for_user::<Builder>(query, &user).await
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<ListFullBuildersResponse> {
+    let all_tags = if self.query.tags.is_empty() {
+      vec![]
+    } else {
+      get_all_tags(None).await?
+    };
+    Ok(
+      resource::list_full_for_user::<Builder>(
+        self.query,
+        user,
+        PermissionLevel::Read.into(),
+        &all_tags,
+      )
+      .await?,
+    )
   }
 }
 
-impl Resolve<GetBuildersSummary, User> for State {
+impl Resolve<ReadArgs> for GetBuildersSummary {
   async fn resolve(
-    &self,
-    GetBuildersSummary {}: GetBuildersSummary,
-    user: User,
-  ) -> anyhow::Result<GetBuildersSummaryResponse> {
-    let query =
-      match resource::get_resource_ids_for_user::<Builder>(&user)
-        .await?
-      {
-        Some(ids) => doc! {
-          "_id": { "$in": ids }
-        },
-        None => Document::new(),
-      };
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<GetBuildersSummaryResponse> {
+    let query = match resource::get_resource_object_ids_for_user::<
+      Builder,
+    >(user)
+    .await?
+    {
+      Some(ids) => doc! {
+        "_id": { "$in": ids }
+      },
+      None => Document::new(),
+    };
     let total = db_client()
-      .await
       .builders
       .count_documents(query)
       .await

bin/core/src/api/read/deployment.rs

@@ -1,73 +1,110 @@
 use std::{cmp, collections::HashSet};
 
-use anyhow::{anyhow, Context};
-use monitor_client::{
+use anyhow::{Context, anyhow};
+use komodo_client::{
   api::read::*,
   entities::{
     deployment::{
       Deployment, DeploymentActionState, DeploymentConfig,
-      DeploymentListItem, DeploymentState, DockerContainerStats,
+      DeploymentListItem, DeploymentState,
     },
+    docker::container::{Container, ContainerStats},
     permission::PermissionLevel,
-    server::Server,
+    server::{Server, ServerState},
     update::Log,
-    user::User,
   },
 };
-use periphery_client::api;
+use periphery_client::api::{self, container::InspectContainer};
 use resolver_api::Resolve;
 
 use crate::{
-  helpers::periphery_client,
+  helpers::{periphery_client, query::get_all_tags},
+  permission::get_check_permissions,
   resource,
-  state::{action_states, deployment_status_cache, State},
+  state::{
+    action_states, deployment_status_cache, server_status_cache,
+  },
 };
 
-impl Resolve<GetDeployment, User> for State {
+use super::ReadArgs;
+
+impl Resolve<ReadArgs> for GetDeployment {
   async fn resolve(
-    &self,
-    GetDeployment { deployment }: GetDeployment,
-    user: User,
-  ) -> anyhow::Result<Deployment> {
-    resource::get_check_permissions::<Deployment>(
-      &deployment,
-      &user,
-      PermissionLevel::Read,
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<Deployment> {
+    Ok(
+      get_check_permissions::<Deployment>(
+        &self.deployment,
+        user,
+        PermissionLevel::Read.into(),
+      )
+      .await?,
     )
-    .await
   }
 }
 
-impl Resolve<ListDeployments, User> for State {
+impl Resolve<ReadArgs> for ListDeployments {
   async fn resolve(
-    &self,
-    ListDeployments { query }: ListDeployments,
-    user: User,
-  ) -> anyhow::Result<Vec<DeploymentListItem>> {
-    resource::list_for_user::<Deployment>(query, &user).await
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<Vec<DeploymentListItem>> {
+    let all_tags = if self.query.tags.is_empty() {
+      vec![]
+    } else {
+      get_all_tags(None).await?
+    };
+    let only_update_available = self.query.specific.update_available;
+    let deployments = resource::list_for_user::<Deployment>(
+      self.query,
+      user,
+      PermissionLevel::Read.into(),
+      &all_tags,
+    )
+    .await?;
+    let deployments = if only_update_available {
+      deployments
+        .into_iter()
+        .filter(|deployment| deployment.info.update_available)
+        .collect()
+    } else {
+      deployments
+    };
+    Ok(deployments)
   }
 }
 
-impl Resolve<ListFullDeployments, User> for State {
+impl Resolve<ReadArgs> for ListFullDeployments {
   async fn resolve(
-    &self,
-    ListFullDeployments { query }: ListFullDeployments,
-    user: User,
-  ) -> anyhow::Result<ListFullDeploymentsResponse> {
-    resource::list_full_for_user::<Deployment>(query, &user).await
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<ListFullDeploymentsResponse> {
+    let all_tags = if self.query.tags.is_empty() {
+      vec![]
+    } else {
+      get_all_tags(None).await?
+    };
+    Ok(
+      resource::list_full_for_user::<Deployment>(
+        self.query,
+        user,
+        PermissionLevel::Read.into(),
+        &all_tags,
+      )
+      .await?,
+    )
   }
 }
 
-impl Resolve<GetDeploymentContainer, User> for State {
+impl Resolve<ReadArgs> for GetDeploymentContainer {
   async fn resolve(
-    &self,
-    GetDeploymentContainer { deployment }: GetDeploymentContainer,
-    user: User,
-  ) -> anyhow::Result<GetDeploymentContainerResponse> {
-    let deployment = resource::get_check_permissions::<Deployment>(
-      &deployment,
-      &user,
-      PermissionLevel::Read,
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<GetDeploymentContainerResponse> {
+    let deployment = get_check_permissions::<Deployment>(
+      &self.deployment,
+      user,
+      PermissionLevel::Read.into(),
     )
     .await?;
     let status = deployment_status_cache()
@@ -84,110 +121,164 @@ impl Resolve<GetDeploymentContainer, User> for State {
 
 const MAX_LOG_LENGTH: u64 = 5000;
 
-impl Resolve<GetLog, User> for State {
+impl Resolve<ReadArgs> for GetDeploymentLog {
   async fn resolve(
-    &self,
-    GetLog { deployment, tail }: GetLog,
-    user: User,
-  ) -> anyhow::Result<Log> {
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<Log> {
+    let GetDeploymentLog {
+      deployment,
+      tail,
+      timestamps,
+    } = self;
     let Deployment {
       name,
       config: DeploymentConfig { server_id, .. },
       ..
-    } = resource::get_check_permissions::<Deployment>(
+    } = get_check_permissions::<Deployment>(
       &deployment,
-      &user,
-      PermissionLevel::Read,
+      user,
+      PermissionLevel::Read.logs(),
     )
     .await?;
     if server_id.is_empty() {
       return Ok(Log::default());
     }
     let server = resource::get::<Server>(&server_id).await?;
-    periphery_client(&server)?
+    let res = periphery_client(&server)?
       .request(api::container::GetContainerLog {
         name,
         tail: cmp::min(tail, MAX_LOG_LENGTH),
+        timestamps,
       })
       .await
-      .context("failed at call to periphery")
+      .context("failed at call to periphery")?;
+    Ok(res)
   }
 }
 
-impl Resolve<SearchLog, User> for State {
+impl Resolve<ReadArgs> for SearchDeploymentLog {
   async fn resolve(
-    &self,
-    SearchLog {
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<Log> {
+    let SearchDeploymentLog {
       deployment,
       terms,
       combinator,
       invert,
-    }: SearchLog,
-    user: User,
-  ) -> anyhow::Result<Log> {
+      timestamps,
+    } = self;
     let Deployment {
       name,
       config: DeploymentConfig { server_id, .. },
       ..
-    } = resource::get_check_permissions::<Deployment>(
+    } = get_check_permissions::<Deployment>(
       &deployment,
-      &user,
-      PermissionLevel::Read,
+      user,
+      PermissionLevel::Read.logs(),
     )
     .await?;
     if server_id.is_empty() {
       return Ok(Log::default());
     }
     let server = resource::get::<Server>(&server_id).await?;
-    periphery_client(&server)?
+    let res = periphery_client(&server)?
       .request(api::container::GetContainerLogSearch {
         name,
         terms,
         combinator,
         invert,
+        timestamps,
       })
       .await
-      .context("failed at call to periphery")
+      .context("failed at call to periphery")?;
+    Ok(res)
   }
 }
 
-impl Resolve<GetDeploymentStats, User> for State {
+impl Resolve<ReadArgs> for InspectDeploymentContainer {
   async fn resolve(
-    &self,
-    GetDeploymentStats { deployment }: GetDeploymentStats,
-    user: User,
-  ) -> anyhow::Result<DockerContainerStats> {
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<Container> {
+    let InspectDeploymentContainer { deployment } = self;
     let Deployment {
       name,
       config: DeploymentConfig { server_id, .. },
       ..
-    } = resource::get_check_permissions::<Deployment>(
+    } = get_check_permissions::<Deployment>(
       &deployment,
-      &user,
-      PermissionLevel::Read,
+      user,
+      PermissionLevel::Read.inspect(),
     )
     .await?;
     if server_id.is_empty() {
-      return Err(anyhow!("deployment has no server attached"));
+      return Err(
+        anyhow!(
+          "Cannot inspect deployment, not attached to any server"
+        )
+        .into(),
+      );
     }
     let server = resource::get::<Server>(&server_id).await?;
-    periphery_client(&server)?
-      .request(api::container::GetContainerStats { name })
-      .await
-      .context("failed to get stats from periphery")
+    let cache = server_status_cache()
+      .get_or_insert_default(&server.id)
+      .await;
+    if cache.state != ServerState::Ok {
+      return Err(
+        anyhow!(
+          "Cannot inspect container: server is {:?}",
+          cache.state
+        )
+        .into(),
+      );
+    }
+    let res = periphery_client(&server)?
+      .request(InspectContainer { name })
+      .await?;
+    Ok(res)
   }
 }
 
-impl Resolve<GetDeploymentActionState, User> for State {
+impl Resolve<ReadArgs> for GetDeploymentStats {
   async fn resolve(
-    &self,
-    GetDeploymentActionState { deployment }: GetDeploymentActionState,
-    user: User,
-  ) -> anyhow::Result<DeploymentActionState> {
-    let deployment = resource::get_check_permissions::<Deployment>(
-      &deployment,
-      &user,
-      PermissionLevel::Read,
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<ContainerStats> {
+    let Deployment {
+      name,
+      config: DeploymentConfig { server_id, .. },
+      ..
+    } = get_check_permissions::<Deployment>(
+      &self.deployment,
+      user,
+      PermissionLevel::Read.into(),
+    )
+    .await?;
+    if server_id.is_empty() {
+      return Err(
+        anyhow!("deployment has no server attached").into(),
+      );
+    }
+    let server = resource::get::<Server>(&server_id).await?;
+    let res = periphery_client(&server)?
+      .request(api::container::GetContainerStats { name })
+      .await
+      .context("failed to get stats from periphery")?;
+    Ok(res)
+  }
+}
+
+impl Resolve<ReadArgs> for GetDeploymentActionState {
+  async fn resolve(
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<DeploymentActionState> {
+    let deployment = get_check_permissions::<Deployment>(
+      &self.deployment,
+      user,
+      PermissionLevel::Read.into(),
     )
     .await?;
     let action_state = action_states()
@@ -200,15 +291,16 @@ impl Resolve<GetDeploymentActionState, User> for State {
   }
 }
 
-impl Resolve<GetDeploymentsSummary, User> for State {
+impl Resolve<ReadArgs> for GetDeploymentsSummary {
   async fn resolve(
-    &self,
-    GetDeploymentsSummary {}: GetDeploymentsSummary,
-    user: User,
-  ) -> anyhow::Result<GetDeploymentsSummaryResponse> {
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<GetDeploymentsSummaryResponse> {
     let deployments = resource::list_full_for_user::<Deployment>(
       Default::default(),
-      &user,
+      user,
+      PermissionLevel::Read.into(),
+      &[],
     )
     .await
     .context("failed to get deployments from db")?;
@@ -222,14 +314,17 @@ impl Resolve<GetDeploymentsSummary, User> for State {
         DeploymentState::Running => {
           res.running += 1;
         }
-        DeploymentState::Unknown => {
-          res.unknown += 1;
+        DeploymentState::Exited | DeploymentState::Paused => {
+          res.stopped += 1;
         }
         DeploymentState::NotDeployed => {
           res.not_deployed += 1;
         }
+        DeploymentState::Unknown => {
+          res.unknown += 1;
+        }
         _ => {
-          res.stopped += 1;
+          res.unhealthy += 1;
         }
       }
     }
@@ -237,16 +332,24 @@ impl Resolve<GetDeploymentsSummary, User> for State {
   }
 }
 
-impl Resolve<ListCommonDeploymentExtraArgs, User> for State {
+impl Resolve<ReadArgs> for ListCommonDeploymentExtraArgs {
   async fn resolve(
-    &self,
-    ListCommonDeploymentExtraArgs { query }: ListCommonDeploymentExtraArgs,
-    user: User,
-  ) -> anyhow::Result<ListCommonDeploymentExtraArgsResponse> {
-    let deployments =
-      resource::list_full_for_user::<Deployment>(query, &user)
-        .await
-        .context("failed to get resources matching query")?;
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<ListCommonDeploymentExtraArgsResponse> {
+    let all_tags = if self.query.tags.is_empty() {
+      vec![]
+    } else {
+      get_all_tags(None).await?
+    };
+    let deployments = resource::list_full_for_user::<Deployment>(
+      self.query,
+      user,
+      PermissionLevel::Read.into(),
+      &all_tags,
+    )
+    .await
+    .context("failed to get resources matching query")?;
 
     // first collect with guaranteed uniqueness
     let mut res = HashSet::<String>::new();

bin/core/src/api/read/mod.rs

@@ -1,34 +1,39 @@
 use std::{collections::HashSet, sync::OnceLock, time::Instant};
 
-use anyhow::{anyhow, Context};
-use axum::{middleware, routing::post, Extension, Router};
-use axum_extra::{headers::ContentType, TypedHeader};
-use monitor_client::{
+use anyhow::{Context, anyhow};
+use axum::{
+  Extension, Router, extract::Path, middleware, routing::post,
+};
+use komodo_client::{
   api::read::*,
   entities::{
+    ResourceTarget,
     build::Build,
     builder::{Builder, BuilderConfig},
     config::{DockerRegistry, GitProvider},
+    permission::PermissionLevel,
     repo::Repo,
     server::Server,
     sync::ResourceSync,
-    update::ResourceTarget,
     user::User,
   },
 };
-use resolver_api::{
-  derive::Resolver, Resolve, ResolveToString, Resolver,
-};
+use resolver_api::Resolve;
+use response::Response;
 use serde::{Deserialize, Serialize};
+use serde_json::json;
 use serror::Json;
 use typeshare::typeshare;
 use uuid::Uuid;
 
 use crate::{
   auth::auth_request, config::core_config, helpers::periphery_client,
-  resource, state::State,
+  resource,
 };
 
+use super::Variant;
+
+mod action;
 mod alert;
 mod alerter;
 mod build;
@@ -38,9 +43,8 @@ mod permission;
 mod procedure;
 mod provider;
 mod repo;
-mod search;
+mod schedule;
 mod server;
-mod server_template;
 mod stack;
 mod sync;
 mod tag;
@@ -50,25 +54,26 @@ mod user;
 mod user_group;
 mod variable;
 
+pub struct ReadArgs {
+  pub user: User,
+}
+
 #[typeshare]
-#[derive(Serialize, Deserialize, Debug, Clone, Resolver)]
-#[resolver_target(State)]
-#[resolver_args(User)]
+#[derive(Serialize, Deserialize, Debug, Clone, Resolve)]
+#[args(ReadArgs)]
+#[response(Response)]
+#[error(serror::Error)]
 #[serde(tag = "type", content = "params")]
 enum ReadRequest {
-  #[to_string_resolver]
   GetVersion(GetVersion),
-  #[to_string_resolver]
   GetCoreInfo(GetCoreInfo),
-  #[to_string_resolver]
-  ListAwsEcrLabels(ListAwsEcrLabels),
   ListSecrets(ListSecrets),
   ListGitProvidersFromConfig(ListGitProvidersFromConfig),
   ListDockerRegistriesFromConfig(ListDockerRegistriesFromConfig),
 
   // ==== USER ====
   GetUsername(GetUsername),
-  GetPermissionLevel(GetPermissionLevel),
+  GetPermission(GetPermission),
   FindUser(FindUser),
   ListUsers(ListUsers),
   ListApiKeys(ListApiKeys),
@@ -80,9 +85,6 @@ enum ReadRequest {
   GetUserGroup(GetUserGroup),
   ListUserGroups(ListUserGroups),
 
-  // ==== SEARCH ====
-  FindResources(FindResources),
-
   // ==== PROCEDURE ====
   GetProceduresSummary(GetProceduresSummary),
   GetProcedure(GetProcedure),
@@ -90,11 +92,15 @@ enum ReadRequest {
   ListProcedures(ListProcedures),
   ListFullProcedures(ListFullProcedures),
 
-  // ==== SERVER TEMPLATE ====
-  GetServerTemplate(GetServerTemplate),
-  GetServerTemplatesSummary(GetServerTemplatesSummary),
-  ListServerTemplates(ListServerTemplates),
-  ListFullServerTemplates(ListFullServerTemplates),
+  // ==== ACTION ====
+  GetActionsSummary(GetActionsSummary),
+  GetAction(GetAction),
+  GetActionActionState(GetActionActionState),
+  ListActions(ListActions),
+  ListFullActions(ListFullActions),
+
+  // ==== SCHEDULE ====
+  ListSchedules(ListSchedules),
 
   // ==== SERVER ====
   GetServersSummary(GetServersSummary),
@@ -105,14 +111,41 @@ enum ReadRequest {
   GetHistoricalServerStats(GetHistoricalServerStats),
   ListServers(ListServers),
   ListFullServers(ListFullServers),
-  #[to_string_resolver]
+  InspectDockerContainer(InspectDockerContainer),
+  GetResourceMatchingContainer(GetResourceMatchingContainer),
+  GetContainerLog(GetContainerLog),
+  SearchContainerLog(SearchContainerLog),
+  InspectDockerNetwork(InspectDockerNetwork),
+  InspectDockerImage(InspectDockerImage),
+  ListDockerImageHistory(ListDockerImageHistory),
+  InspectDockerVolume(InspectDockerVolume),
+  GetDockerContainersSummary(GetDockerContainersSummary),
+  ListAllDockerContainers(ListAllDockerContainers),
   ListDockerContainers(ListDockerContainers),
-  #[to_string_resolver]
   ListDockerNetworks(ListDockerNetworks),
-  #[to_string_resolver]
   ListDockerImages(ListDockerImages),
-  #[to_string_resolver]
+  ListDockerVolumes(ListDockerVolumes),
   ListComposeProjects(ListComposeProjects),
+  ListTerminals(ListTerminals),
+
+  // ==== SERVER STATS ====
+  GetSystemInformation(GetSystemInformation),
+  GetSystemStats(GetSystemStats),
+  ListSystemProcesses(ListSystemProcesses),
+
+  // ==== STACK ====
+  GetStacksSummary(GetStacksSummary),
+  GetStack(GetStack),
+  GetStackActionState(GetStackActionState),
+  GetStackWebhooksEnabled(GetStackWebhooksEnabled),
+  GetStackLog(GetStackLog),
+  SearchStackLog(SearchStackLog),
+  InspectStackContainer(InspectStackContainer),
+  ListStacks(ListStacks),
+  ListFullStacks(ListFullStacks),
+  ListStackServices(ListStackServices),
+  ListCommonStackExtraArgs(ListCommonStackExtraArgs),
+  ListCommonStackBuildExtraArgs(ListCommonStackBuildExtraArgs),
 
   // ==== DEPLOYMENT ====
   GetDeploymentsSummary(GetDeploymentsSummary),
@@ -120,8 +153,9 @@ enum ReadRequest {
   GetDeploymentContainer(GetDeploymentContainer),
   GetDeploymentActionState(GetDeploymentActionState),
   GetDeploymentStats(GetDeploymentStats),
-  GetLog(GetLog),
-  SearchLog(SearchLog),
+  GetDeploymentLog(GetDeploymentLog),
+  SearchDeploymentLog(SearchDeploymentLog),
+  InspectDeploymentContainer(InspectDeploymentContainer),
   ListDeployments(ListDeployments),
   ListFullDeployments(ListFullDeployments),
   ListCommonDeploymentExtraArgs(ListCommonDeploymentExtraArgs),
@@ -153,18 +187,6 @@ enum ReadRequest {
   ListResourceSyncs(ListResourceSyncs),
   ListFullResourceSyncs(ListFullResourceSyncs),
 
-  // ==== STACK ====
-  GetStacksSummary(GetStacksSummary),
-  GetStack(GetStack),
-  GetStackActionState(GetStackActionState),
-  GetStackWebhooksEnabled(GetStackWebhooksEnabled),
-  GetStackServiceLog(GetStackServiceLog),
-  SearchStackServiceLog(SearchStackServiceLog),
-  ListStacks(ListStacks),
-  ListFullStacks(ListFullStacks),
-  ListStackServices(ListStackServices),
-  ListCommonStackExtraArgs(ListCommonStackExtraArgs),
-
   // ==== BUILDER ====
   GetBuildersSummary(GetBuildersSummary),
   GetBuilder(GetBuilder),
@@ -193,14 +215,6 @@ enum ReadRequest {
   ListAlerts(ListAlerts),
   GetAlert(GetAlert),
 
-  // ==== SERVER STATS ====
-  #[to_string_resolver]
-  GetSystemInformation(GetSystemInformation),
-  #[to_string_resolver]
-  GetSystemStats(GetSystemStats),
-  #[to_string_resolver]
-  ListSystemProcesses(ListSystemProcesses),
-
   // ==== VARIABLE ====
   GetVariable(GetVariable),
   ListVariables(ListVariables),
@@ -215,134 +229,105 @@ enum ReadRequest {
 pub fn router() -> Router {
   Router::new()
     .route("/", post(handler))
+    .route("/{variant}", post(variant_handler))
     .layer(middleware::from_fn(auth_request))
 }
 
+async fn variant_handler(
+  user: Extension<User>,
+  Path(Variant { variant }): Path<Variant>,
+  Json(params): Json<serde_json::Value>,
+) -> serror::Result<axum::response::Response> {
+  let req: ReadRequest = serde_json::from_value(json!({
+    "type": variant,
+    "params": params,
+  }))?;
+  handler(user, Json(req)).await
+}
+
 #[instrument(name = "ReadHandler", level = "debug", skip(user), fields(user_id = user.id))]
 async fn handler(
   Extension(user): Extension<User>,
   Json(request): Json<ReadRequest>,
-) -> serror::Result<(TypedHeader<ContentType>, String)> {
+) -> serror::Result<axum::response::Response> {
   let timer = Instant::now();
   let req_id = Uuid::new_v4();
   debug!("/read request | user: {}", user.username);
-  let res =
-    State
-      .resolve_request(request, user)
-      .await
-      .map_err(|e| match e {
-        resolver_api::Error::Serialization(e) => {
-          anyhow!("{e:?}").context("response serialization error")
-        }
-        resolver_api::Error::Inner(e) => e,
-      });
+  let res = request.resolve(&ReadArgs { user }).await;
   if let Err(e) = &res {
-    debug!("/read request {req_id} error: {e:#}");
+    debug!("/read request {req_id} error: {:#}", e.error);
   }
   let elapsed = timer.elapsed();
   debug!("/read request {req_id} | resolve time: {elapsed:?}");
-  Ok((TypedHeader(ContentType::json()), res?))
+  res.map(|res| res.0)
 }
 
-fn version() -> &'static String {
-  static VERSION: OnceLock<String> = OnceLock::new();
-  VERSION.get_or_init(|| {
-    serde_json::to_string(&GetVersionResponse {
+impl Resolve<ReadArgs> for GetVersion {
+  async fn resolve(
+    self,
+    _: &ReadArgs,
+  ) -> serror::Result<GetVersionResponse> {
+    Ok(GetVersionResponse {
       version: env!("CARGO_PKG_VERSION").to_string(),
     })
-    .context("failed to serialize GetVersionResponse")
-    .unwrap()
-  })
-}
-
-impl ResolveToString<GetVersion, User> for State {
-  async fn resolve_to_string(
-    &self,
-    GetVersion {}: GetVersion,
-    _: User,
-  ) -> anyhow::Result<String> {
-    Ok(version().to_string())
   }
 }
 
-fn core_info() -> &'static String {
-  static CORE_INFO: OnceLock<String> = OnceLock::new();
+fn core_info() -> &'static GetCoreInfoResponse {
+  static CORE_INFO: OnceLock<GetCoreInfoResponse> = OnceLock::new();
   CORE_INFO.get_or_init(|| {
     let config = core_config();
-    let info = GetCoreInfoResponse {
+    GetCoreInfoResponse {
       title: config.title.clone(),
       monitoring_interval: config.monitoring_interval,
-      webhook_base_url: config
-        .webhook_base_url
-        .clone()
-        .unwrap_or_else(|| config.host.clone()),
+      webhook_base_url: if config.webhook_base_url.is_empty() {
+        config.host.clone()
+      } else {
+        config.webhook_base_url.clone()
+      },
       transparent_mode: config.transparent_mode,
       ui_write_disabled: config.ui_write_disabled,
+      disable_confirm_dialog: config.disable_confirm_dialog,
+      disable_non_admin_create: config.disable_non_admin_create,
+      disable_websocket_reconnect: config.disable_websocket_reconnect,
+      enable_fancy_toml: config.enable_fancy_toml,
       github_webhook_owners: config
         .github_webhook_app
         .installations
         .iter()
         .map(|i| i.namespace.to_string())
         .collect(),
-    };
-    serde_json::to_string(&info)
-      .context("failed to serialize GetCoreInfoResponse")
-      .unwrap()
+      timezone: config.timezone.clone(),
+    }
   })
 }
 
-impl ResolveToString<GetCoreInfo, User> for State {
-  async fn resolve_to_string(
-    &self,
-    GetCoreInfo {}: GetCoreInfo,
-    _: User,
-  ) -> anyhow::Result<String> {
-    Ok(core_info().to_string())
-  }
-}
-
-fn ecr_labels() -> &'static String {
-  static ECR_LABELS: OnceLock<String> = OnceLock::new();
-  ECR_LABELS.get_or_init(|| {
-    serde_json::to_string(
-      &core_config()
-        .aws_ecr_registries
-        .iter()
-        .map(|reg| reg.label.clone())
-        .collect::<Vec<_>>(),
-    )
-    .context("failed to serialize ecr registries")
-    .unwrap()
-  })
-}
-
-impl ResolveToString<ListAwsEcrLabels, User> for State {
-  async fn resolve_to_string(
-    &self,
-    ListAwsEcrLabels {}: ListAwsEcrLabels,
-    _: User,
-  ) -> anyhow::Result<String> {
-    Ok(ecr_labels().to_string())
-  }
-}
-
-impl Resolve<ListSecrets, User> for State {
+impl Resolve<ReadArgs> for GetCoreInfo {
   async fn resolve(
-    &self,
-    ListSecrets { target }: ListSecrets,
-    _: User,
-  ) -> anyhow::Result<ListSecretsResponse> {
+    self,
+    _: &ReadArgs,
+  ) -> serror::Result<GetCoreInfoResponse> {
+    Ok(core_info().clone())
+  }
+}
+
+impl Resolve<ReadArgs> for ListSecrets {
+  async fn resolve(
+    self,
+    _: &ReadArgs,
+  ) -> serror::Result<ListSecretsResponse> {
     let mut secrets = core_config()
       .secrets
       .keys()
       .cloned()
       .collect::<HashSet<_>>();
 
-    if let Some(target) = target {
+    if let Some(target) = self.target {
       let server_id = match target {
         ResourceTarget::Server(id) => Some(id),
         ResourceTarget::Builder(id) => {
           match resource::get::<Builder>(&id).await?.config {
+            BuilderConfig::Url(_) => None,
             BuilderConfig::Server(config) => Some(config.server_id),
             BuilderConfig::Aws(config) => {
               secrets.extend(config.secrets);
@@ -351,7 +336,9 @@ impl Resolve<ListSecrets, User> for State {
           }
         }
         _ => {
-          return Err(anyhow!("target must be `Server` or `Builder`"))
+          return Err(
+            anyhow!("target must be `Server` or `Builder`").into(),
+          );
         }
       };
       if let Some(id) = server_id {
@@ -376,21 +363,21 @@ impl Resolve<ListSecrets, User> for State {
   }
 }
 
-impl Resolve<ListGitProvidersFromConfig, User> for State {
+impl Resolve<ReadArgs> for ListGitProvidersFromConfig {
   async fn resolve(
-    &self,
-    ListGitProvidersFromConfig { target }: ListGitProvidersFromConfig,
-    user: User,
-  ) -> anyhow::Result<ListGitProvidersFromConfigResponse> {
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<ListGitProvidersFromConfigResponse> {
     let mut providers = core_config().git_providers.clone();
 
-    if let Some(target) = target {
+    if let Some(target) = self.target {
       match target {
         ResourceTarget::Server(id) => {
           merge_git_providers_for_server(&mut providers, &id).await?;
         }
         ResourceTarget::Builder(id) => {
           match resource::get::<Builder>(&id).await?.config {
+            BuilderConfig::Url(_) => {}
             BuilderConfig::Server(config) => {
               merge_git_providers_for_server(
                 &mut providers,
@@ -407,7 +394,9 @@ impl Resolve<ListGitProvidersFromConfig, User> for State {
           }
         }
         _ => {
-          return Err(anyhow!("target must be `Server` or `Builder`"))
+          return Err(
+            anyhow!("target must be `Server` or `Builder`").into(),
+          );
         }
       }
     }
@@ -415,12 +404,21 @@ impl Resolve<ListGitProvidersFromConfig, User> for State {
     let (builds, repos, syncs) = tokio::try_join!(
       resource::list_full_for_user::<Build>(
         Default::default(),
-        &user
+        user,
+        PermissionLevel::Read.into(),
+        &[]
+      ),
+      resource::list_full_for_user::<Repo>(
+        Default::default(),
+        user,
+        PermissionLevel::Read.into(),
+        &[]
       ),
-      resource::list_full_for_user::<Repo>(Default::default(), &user),
       resource::list_full_for_user::<ResourceSync>(
         Default::default(),
-        &user
+        user,
+        PermissionLevel::Read.into(),
+        &[]
       ),
     )?;
 
@@ -467,15 +465,14 @@ impl Resolve<ListGitProvidersFromConfig, User> for State {
   }
 }
 
-impl Resolve<ListDockerRegistriesFromConfig, User> for State {
+impl Resolve<ReadArgs> for ListDockerRegistriesFromConfig {
   async fn resolve(
-    &self,
-    ListDockerRegistriesFromConfig { target }: ListDockerRegistriesFromConfig,
-    _: User,
-  ) -> anyhow::Result<ListDockerRegistriesFromConfigResponse> {
+    self,
+    _: &ReadArgs,
+  ) -> serror::Result<ListDockerRegistriesFromConfigResponse> {
     let mut registries = core_config().docker_registries.clone();
 
-    if let Some(target) = target {
+    if let Some(target) = self.target {
       match target {
         ResourceTarget::Server(id) => {
           merge_docker_registries_for_server(&mut registries, &id)
@@ -483,6 +480,7 @@ impl Resolve<ListDockerRegistriesFromConfig, User> for State {
         }
         ResourceTarget::Builder(id) => {
           match resource::get::<Builder>(&id).await?.config {
+            BuilderConfig::Url(_) => {}
             BuilderConfig::Server(config) => {
               merge_docker_registries_for_server(
                 &mut registries,
@@ -499,7 +497,9 @@ impl Resolve<ListDockerRegistriesFromConfig, User> for State {
           }
         }
         _ => {
-          return Err(anyhow!("target must be `Server` or `Builder`"))
+          return Err(
+            anyhow!("target must be `Server` or `Builder`").into(),
+          );
         }
       }
     }
@@ -513,7 +513,7 @@ impl Resolve<ListDockerRegistriesFromConfig, User> for State {
 async fn merge_git_providers_for_server(
   providers: &mut Vec<GitProvider>,
   server_id: &str,
-) -> anyhow::Result<()> {
+) -> serror::Result<()> {
   let server = resource::get::<Server>(server_id).await?;
   let more = periphery_client(&server)?
     .request(periphery_client::api::ListGitProviders {})
@@ -551,7 +551,7 @@ fn merge_git_providers(
 async fn merge_docker_registries_for_server(
   registries: &mut Vec<DockerRegistry>,
   server_id: &str,
-) -> anyhow::Result<()> {
+) -> serror::Result<()> {
   let server = resource::get::<Server>(server_id).await?;
   let more = periphery_client(&server)?
     .request(periphery_client::api::ListDockerRegistries {})

bin/core/src/api/read/permission.rs

@@ -1,28 +1,28 @@
-use anyhow::{anyhow, Context};
-use monitor_client::{
+use anyhow::{Context, anyhow};
+use database::mungos::{find::find_collect, mongodb::bson::doc};
+use komodo_client::{
   api::read::{
-    GetPermissionLevel, GetPermissionLevelResponse, ListPermissions,
+    GetPermission, GetPermissionResponse, ListPermissions,
     ListPermissionsResponse, ListUserTargetPermissions,
     ListUserTargetPermissionsResponse,
   },
-  entities::{permission::PermissionLevel, user::User},
+  entities::permission::PermissionLevel,
 };
-use mungos::{find::find_collect, mongodb::bson::doc};
 use resolver_api::Resolve;
 
 use crate::{
-  helpers::query::get_user_permission_on_target,
-  state::{db_client, State},
+  helpers::query::get_user_permission_on_target, state::db_client,
 };
 
-impl Resolve<ListPermissions, User> for State {
+use super::ReadArgs;
+
+impl Resolve<ReadArgs> for ListPermissions {
   async fn resolve(
-    &self,
-    ListPermissions {}: ListPermissions,
-    user: User,
-  ) -> anyhow::Result<ListPermissionsResponse> {
-    find_collect(
-      &db_client().await.permissions,
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<ListPermissionsResponse> {
+    let res = find_collect(
+      &db_client().permissions,
       doc! {
         "user_target.type": "User",
         "user_target.id": &user.id
@@ -30,35 +30,34 @@ impl Resolve<ListPermissions, User> for State {
       None,
     )
     .await
-    .context("failed to query db for permissions")
+    .context("failed to query db for permissions")?;
+    Ok(res)
   }
 }
 
-impl Resolve<GetPermissionLevel, User> for State {
+impl Resolve<ReadArgs> for GetPermission {
   async fn resolve(
-    &self,
-    GetPermissionLevel { target }: GetPermissionLevel,
-    user: User,
-  ) -> anyhow::Result<GetPermissionLevelResponse> {
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<GetPermissionResponse> {
     if user.admin {
-      return Ok(PermissionLevel::Write);
+      return Ok(PermissionLevel::Write.all());
     }
-    get_user_permission_on_target(&user, &target).await
+    Ok(get_user_permission_on_target(user, &self.target).await?)
   }
 }
 
-impl Resolve<ListUserTargetPermissions, User> for State {
+impl Resolve<ReadArgs> for ListUserTargetPermissions {
   async fn resolve(
-    &self,
-    ListUserTargetPermissions { user_target }: ListUserTargetPermissions,
-    user: User,
-  ) -> anyhow::Result<ListUserTargetPermissionsResponse> {
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<ListUserTargetPermissionsResponse> {
     if !user.admin {
-      return Err(anyhow!("this method is admin only"));
+      return Err(anyhow!("this method is admin only").into());
     }
-    let (variant, id) = user_target.extract_variant_id();
-    find_collect(
-      &db_client().await.permissions,
+    let (variant, id) = self.user_target.extract_variant_id();
+    let res = find_collect(
+      &db_client().permissions,
       doc! {
         "user_target.type": variant.as_ref(),
         "user_target.id": id
@@ -66,6 +65,7 @@ impl Resolve<ListUserTargetPermissions, User> for State {
       None,
     )
     .await
-    .context("failed to query db for permissions")
+    .context("failed to query db for permissions")?;
+    Ok(res)
   }
 }

bin/core/src/api/read/procedure.rs

@@ -1,63 +1,92 @@
 use anyhow::Context;
-use monitor_client::{
+use komodo_client::{
   api::read::*,
   entities::{
     permission::PermissionLevel,
     procedure::{Procedure, ProcedureState},
-    user::User,
   },
 };
 use resolver_api::Resolve;
 
 use crate::{
+  helpers::query::get_all_tags,
+  permission::get_check_permissions,
   resource,
-  state::{action_states, procedure_state_cache, State},
+  state::{action_states, procedure_state_cache},
 };
 
-impl Resolve<GetProcedure, User> for State {
+use super::ReadArgs;
+
+impl Resolve<ReadArgs> for GetProcedure {
   async fn resolve(
-    &self,
-    GetProcedure { procedure }: GetProcedure,
-    user: User,
-  ) -> anyhow::Result<GetProcedureResponse> {
-    resource::get_check_permissions::<Procedure>(
-      &procedure,
-      &user,
-      PermissionLevel::Read,
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<GetProcedureResponse> {
+    Ok(
+      get_check_permissions::<Procedure>(
+        &self.procedure,
+        user,
+        PermissionLevel::Read.into(),
+      )
+      .await?,
     )
-    .await
   }
 }
 
-impl Resolve<ListProcedures, User> for State {
+impl Resolve<ReadArgs> for ListProcedures {
   async fn resolve(
-    &self,
-    ListProcedures { query }: ListProcedures,
-    user: User,
-  ) -> anyhow::Result<ListProceduresResponse> {
-    resource::list_for_user::<Procedure>(query, &user).await
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<ListProceduresResponse> {
+    let all_tags = if self.query.tags.is_empty() {
+      vec![]
+    } else {
+      get_all_tags(None).await?
+    };
+    Ok(
+      resource::list_for_user::<Procedure>(
+        self.query,
+        user,
+        PermissionLevel::Read.into(),
+        &all_tags,
+      )
+      .await?,
+    )
   }
 }
 
-impl Resolve<ListFullProcedures, User> for State {
+impl Resolve<ReadArgs> for ListFullProcedures {
   async fn resolve(
-    &self,
-    ListFullProcedures { query }: ListFullProcedures,
-    user: User,
-  ) -> anyhow::Result<ListFullProceduresResponse> {
-    resource::list_full_for_user::<Procedure>(query, &user).await
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<ListFullProceduresResponse> {
+    let all_tags = if self.query.tags.is_empty() {
+      vec![]
+    } else {
+      get_all_tags(None).await?
+    };
+    Ok(
+      resource::list_full_for_user::<Procedure>(
+        self.query,
+        user,
+        PermissionLevel::Read.into(),
+        &all_tags,
+      )
+      .await?,
+    )
   }
 }
 
-impl Resolve<GetProceduresSummary, User> for State {
+impl Resolve<ReadArgs> for GetProceduresSummary {
   async fn resolve(
-    &self,
-    GetProceduresSummary {}: GetProceduresSummary,
-    user: User,
-  ) -> anyhow::Result<GetProceduresSummaryResponse> {
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<GetProceduresSummaryResponse> {
     let procedures = resource::list_full_for_user::<Procedure>(
       Default::default(),
-      &user,
+      user,
+      PermissionLevel::Read.into(),
+      &[],
     )
     .await
     .context("failed to get procedures from db")?;
@@ -94,16 +123,15 @@ impl Resolve<GetProceduresSummary, User> for State {
   }
 }
 
-impl Resolve<GetProcedureActionState, User> for State {
+impl Resolve<ReadArgs> for GetProcedureActionState {
   async fn resolve(
-    &self,
-    GetProcedureActionState { procedure }: GetProcedureActionState,
-    user: User,
-  ) -> anyhow::Result<GetProcedureActionStateResponse> {
-    let procedure = resource::get_check_permissions::<Procedure>(
-      &procedure,
-      &user,
-      PermissionLevel::Read,
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<GetProcedureActionStateResponse> {
+    let procedure = get_check_permissions::<Procedure>(
+      &self.procedure,
+      user,
+      PermissionLevel::Read.into(),
     )
     .await?;
     let action_state = action_states()

bin/core/src/api/read/provider.rs

@@ -1,116 +1,115 @@
-use anyhow::{anyhow, Context};
-use mongo_indexed::{doc, Document};
-use monitor_client::{
-  api::read::{
-    GetDockerRegistryAccount, GetDockerRegistryAccountResponse,
-    GetGitProviderAccount, GetGitProviderAccountResponse,
-    ListDockerRegistryAccounts, ListDockerRegistryAccountsResponse,
-    ListGitProviderAccounts, ListGitProviderAccountsResponse,
-  },
-  entities::user::User,
-};
-use mungos::{
+use anyhow::{Context, anyhow};
+use database::mongo_indexed::{Document, doc};
+use database::mungos::{
   by_id::find_one_by_id, find::find_collect,
   mongodb::options::FindOptions,
 };
+use komodo_client::api::read::*;
 use resolver_api::Resolve;
 
-use crate::state::{db_client, State};
+use crate::state::db_client;
 
-impl Resolve<GetGitProviderAccount, User> for State {
+use super::ReadArgs;
+
+impl Resolve<ReadArgs> for GetGitProviderAccount {
   async fn resolve(
-    &self,
-    GetGitProviderAccount { id }: GetGitProviderAccount,
-    user: User,
-  ) -> anyhow::Result<GetGitProviderAccountResponse> {
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<GetGitProviderAccountResponse> {
     if !user.admin {
-      return Err(anyhow!(
-        "Only admins can read git provider accounts"
-      ));
+      return Err(
+        anyhow!("Only admins can read git provider accounts").into(),
+      );
     }
-    find_one_by_id(&db_client().await.git_accounts, &id)
+    let res = find_one_by_id(&db_client().git_accounts, &self.id)
       .await
       .context("failed to query db for git provider accounts")?
-      .context("did not find git provider account with the given id")
-  }
-}
-
-impl Resolve<ListGitProviderAccounts, User> for State {
-  async fn resolve(
-    &self,
-    ListGitProviderAccounts { domain, username }: ListGitProviderAccounts,
-    user: User,
-  ) -> anyhow::Result<ListGitProviderAccountsResponse> {
-    if !user.admin {
-      return Err(anyhow!(
-        "Only admins can read git provider accounts"
-      ));
-    }
-    let mut filter = Document::new();
-    if let Some(domain) = domain {
-      filter.insert("domain", domain);
-    }
-    if let Some(username) = username {
-      filter.insert("username", username);
-    }
-    find_collect(
-      &db_client().await.git_accounts,
-      filter,
-      FindOptions::builder()
-        .sort(doc! { "domain": 1, "username": 1 })
-        .build(),
-    )
-    .await
-    .context("failed to query db for git provider accounts")
-  }
-}
-
-impl Resolve<GetDockerRegistryAccount, User> for State {
-  async fn resolve(
-    &self,
-    GetDockerRegistryAccount { id }: GetDockerRegistryAccount,
-    user: User,
-  ) -> anyhow::Result<GetDockerRegistryAccountResponse> {
-    if !user.admin {
-      return Err(anyhow!(
-        "Only admins can read docker registry accounts"
-      ));
-    }
-    find_one_by_id(&db_client().await.registry_accounts, &id)
-      .await
-      .context("failed to query db for docker registry accounts")?
       .context(
-        "did not find docker registry account with the given id",
-      )
+        "did not find git provider account with the given id",
+      )?;
+    Ok(res)
   }
 }
 
-impl Resolve<ListDockerRegistryAccounts, User> for State {
+impl Resolve<ReadArgs> for ListGitProviderAccounts {
   async fn resolve(
-    &self,
-    ListDockerRegistryAccounts { domain, username }: ListDockerRegistryAccounts,
-    user: User,
-  ) -> anyhow::Result<ListDockerRegistryAccountsResponse> {
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<ListGitProviderAccountsResponse> {
     if !user.admin {
-      return Err(anyhow!(
-        "Only admins can read docker registry accounts"
-      ));
+      return Err(
+        anyhow!("Only admins can read git provider accounts").into(),
+      );
     }
     let mut filter = Document::new();
-    if let Some(domain) = domain {
+    if let Some(domain) = self.domain {
       filter.insert("domain", domain);
     }
-    if let Some(username) = username {
+    if let Some(username) = self.username {
       filter.insert("username", username);
     }
-    find_collect(
-      &db_client().await.registry_accounts,
+    let res = find_collect(
+      &db_client().git_accounts,
       filter,
       FindOptions::builder()
         .sort(doc! { "domain": 1, "username": 1 })
         .build(),
     )
     .await
-    .context("failed to query db for docker registry accounts")
+    .context("failed to query db for git provider accounts")?;
+    Ok(res)
+  }
+}
+
+impl Resolve<ReadArgs> for GetDockerRegistryAccount {
+  async fn resolve(
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<GetDockerRegistryAccountResponse> {
+    if !user.admin {
+      return Err(
+        anyhow!("Only admins can read docker registry accounts")
+          .into(),
+      );
+    }
+    let res =
+      find_one_by_id(&db_client().registry_accounts, &self.id)
+        .await
+        .context("failed to query db for docker registry accounts")?
+        .context(
+          "did not find docker registry account with the given id",
+        )?;
+    Ok(res)
+  }
+}
+
+impl Resolve<ReadArgs> for ListDockerRegistryAccounts {
+  async fn resolve(
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<ListDockerRegistryAccountsResponse> {
+    if !user.admin {
+      return Err(
+        anyhow!("Only admins can read docker registry accounts")
+          .into(),
+      );
+    }
+    let mut filter = Document::new();
+    if let Some(domain) = self.domain {
+      filter.insert("domain", domain);
+    }
+    if let Some(username) = self.username {
+      filter.insert("username", username);
+    }
+    let res = find_collect(
+      &db_client().registry_accounts,
+      filter,
+      FindOptions::builder()
+        .sort(doc! { "domain": 1, "username": 1 })
+        .build(),
+    )
+    .await
+    .context("failed to query db for docker registry accounts")?;
+    Ok(res)
   }
 }

bin/core/src/api/read/repo.rs

@@ -1,66 +1,93 @@
 use anyhow::Context;
-use monitor_client::{
+use komodo_client::{
   api::read::*,
   entities::{
     config::core::CoreConfig,
     permission::PermissionLevel,
     repo::{Repo, RepoActionState, RepoListItem, RepoState},
-    user::User,
   },
 };
 use resolver_api::Resolve;
 
 use crate::{
   config::core_config,
+  helpers::query::get_all_tags,
+  permission::get_check_permissions,
   resource,
-  state::{action_states, github_client, repo_state_cache, State},
+  state::{action_states, github_client, repo_state_cache},
 };
 
-impl Resolve<GetRepo, User> for State {
+use super::ReadArgs;
+
+impl Resolve<ReadArgs> for GetRepo {
   async fn resolve(
-    &self,
-    GetRepo { repo }: GetRepo,
-    user: User,
-  ) -> anyhow::Result<Repo> {
-    resource::get_check_permissions::<Repo>(
-      &repo,
-      &user,
-      PermissionLevel::Read,
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<Repo> {
+    Ok(
+      get_check_permissions::<Repo>(
+        &self.repo,
+        user,
+        PermissionLevel::Read.into(),
+      )
+      .await?,
     )
-    .await
   }
 }
 
-impl Resolve<ListRepos, User> for State {
+impl Resolve<ReadArgs> for ListRepos {
   async fn resolve(
-    &self,
-    ListRepos { query }: ListRepos,
-    user: User,
-  ) -> anyhow::Result<Vec<RepoListItem>> {
-    resource::list_for_user::<Repo>(query, &user).await
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<Vec<RepoListItem>> {
+    let all_tags = if self.query.tags.is_empty() {
+      vec![]
+    } else {
+      get_all_tags(None).await?
+    };
+    Ok(
+      resource::list_for_user::<Repo>(
+        self.query,
+        user,
+        PermissionLevel::Read.into(),
+        &all_tags,
+      )
+      .await?,
+    )
   }
 }
 
-impl Resolve<ListFullRepos, User> for State {
+impl Resolve<ReadArgs> for ListFullRepos {
   async fn resolve(
-    &self,
-    ListFullRepos { query }: ListFullRepos,
-    user: User,
-  ) -> anyhow::Result<ListFullReposResponse> {
-    resource::list_full_for_user::<Repo>(query, &user).await
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<ListFullReposResponse> {
+    let all_tags = if self.query.tags.is_empty() {
+      vec![]
+    } else {
+      get_all_tags(None).await?
+    };
+    Ok(
+      resource::list_full_for_user::<Repo>(
+        self.query,
+        user,
+        PermissionLevel::Read.into(),
+        &all_tags,
+      )
+      .await?,
+    )
   }
 }
 
-impl Resolve<GetRepoActionState, User> for State {
+impl Resolve<ReadArgs> for GetRepoActionState {
   async fn resolve(
-    &self,
-    GetRepoActionState { repo }: GetRepoActionState,
-    user: User,
-  ) -> anyhow::Result<RepoActionState> {
-    let repo = resource::get_check_permissions::<Repo>(
-      &repo,
-      &user,
-      PermissionLevel::Read,
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<RepoActionState> {
+    let repo = get_check_permissions::<Repo>(
+      &self.repo,
+      user,
+      PermissionLevel::Read.into(),
     )
     .await?;
     let action_state = action_states()
@@ -73,16 +100,19 @@ impl Resolve<GetRepoActionState, User> for State {
   }
 }
 
-impl Resolve<GetReposSummary, User> for State {
+impl Resolve<ReadArgs> for GetReposSummary {
   async fn resolve(
-    &self,
-    GetReposSummary {}: GetReposSummary,
-    user: User,
-  ) -> anyhow::Result<GetReposSummaryResponse> {
-    let repos =
-      resource::list_full_for_user::<Repo>(Default::default(), &user)
-        .await
-        .context("failed to get repos from db")?;
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<GetReposSummaryResponse> {
+    let repos = resource::list_full_for_user::<Repo>(
+      Default::default(),
+      user,
+      PermissionLevel::Read.into(),
+      &[],
+    )
+    .await
+    .context("failed to get repos from db")?;
 
     let mut res = GetReposSummaryResponse::default();
 
@@ -126,12 +156,11 @@ impl Resolve<GetReposSummary, User> for State {
   }
 }
 
-impl Resolve<GetRepoWebhooksEnabled, User> for State {
+impl Resolve<ReadArgs> for GetRepoWebhooksEnabled {
   async fn resolve(
-    &self,
-    GetRepoWebhooksEnabled { repo }: GetRepoWebhooksEnabled,
-    user: User,
-  ) -> anyhow::Result<GetRepoWebhooksEnabledResponse> {
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<GetRepoWebhooksEnabledResponse> {
     let Some(github) = github_client() else {
       return Ok(GetRepoWebhooksEnabledResponse {
         managed: false,
@@ -141,10 +170,10 @@ impl Resolve<GetRepoWebhooksEnabled, User> for State {
       });
     };
 
-    let repo = resource::get_check_permissions::<Repo>(
-      &repo,
-      &user,
-      PermissionLevel::Read,
+    let repo = get_check_permissions::<Repo>(
+      &self.repo,
+      user,
+      PermissionLevel::Read.into(),
     )
     .await?;
 
@@ -188,7 +217,11 @@ impl Resolve<GetRepoWebhooksEnabled, User> for State {
       ..
     } = core_config();
 
-    let host = webhook_base_url.as_ref().unwrap_or(host);
+    let host = if webhook_base_url.is_empty() {
+      host
+    } else {
+      webhook_base_url
+    };
     let clone_url =
       format!("{host}/listener/github/repo/{}/clone", repo.id);
     let pull_url =

bin/core/src/api/read/schedule.rs

@@ -0,0 +1,107 @@
+use futures::future::join_all;
+use komodo_client::{
+  api::read::*,
+  entities::{
+    ResourceTarget,
+    action::Action,
+    permission::PermissionLevel,
+    procedure::Procedure,
+    resource::{ResourceQuery, TemplatesQueryBehavior},
+    schedule::Schedule,
+  },
+};
+use resolver_api::Resolve;
+
+use crate::{
+  helpers::query::{get_all_tags, get_last_run_at},
+  resource::list_full_for_user,
+  schedule::get_schedule_item_info,
+};
+
+use super::ReadArgs;
+
+impl Resolve<ReadArgs> for ListSchedules {
+  async fn resolve(
+    self,
+    args: &ReadArgs,
+  ) -> serror::Result<Vec<Schedule>> {
+    let all_tags = get_all_tags(None).await?;
+    let (actions, procedures) = tokio::try_join!(
+      list_full_for_user::<Action>(
+        ResourceQuery {
+          names: Default::default(),
+          templates: TemplatesQueryBehavior::Include,
+          tag_behavior: self.tag_behavior,
+          tags: self.tags.clone(),
+          specific: Default::default(),
+        },
+        &args.user,
+        PermissionLevel::Read.into(),
+        &all_tags,
+      ),
+      list_full_for_user::<Procedure>(
+        ResourceQuery {
+          names: Default::default(),
+          templates: TemplatesQueryBehavior::Include,
+          tag_behavior: self.tag_behavior,
+          tags: self.tags.clone(),
+          specific: Default::default(),
+        },
+        &args.user,
+        PermissionLevel::Read.into(),
+        &all_tags,
+      )
+    )?;
+    let actions = actions.into_iter().map(async |action| {
+      let (next_scheduled_run, schedule_error) =
+        get_schedule_item_info(&ResourceTarget::Action(
+          action.id.clone(),
+        ));
+      let last_run_at =
+        get_last_run_at::<Action>(&action.id).await.unwrap_or(None);
+      Schedule {
+        target: ResourceTarget::Action(action.id),
+        name: action.name,
+        enabled: action.config.schedule_enabled,
+        schedule_format: action.config.schedule_format,
+        schedule: action.config.schedule,
+        schedule_timezone: action.config.schedule_timezone,
+        tags: action.tags,
+        last_run_at,
+        next_scheduled_run,
+        schedule_error,
+      }
+    });
+    let procedures = procedures.into_iter().map(async |procedure| {
+      let (next_scheduled_run, schedule_error) =
+        get_schedule_item_info(&ResourceTarget::Procedure(
+          procedure.id.clone(),
+        ));
+      let last_run_at = get_last_run_at::<Procedure>(&procedure.id)
+        .await
+        .unwrap_or(None);
+      Schedule {
+        target: ResourceTarget::Procedure(procedure.id),
+        name: procedure.name,
+        enabled: procedure.config.schedule_enabled,
+        schedule_format: procedure.config.schedule_format,
+        schedule: procedure.config.schedule,
+        schedule_timezone: procedure.config.schedule_timezone,
+        tags: procedure.tags,
+        last_run_at,
+        next_scheduled_run,
+        schedule_error,
+      }
+    });
+    let (actions, procedures) =
+      tokio::join!(join_all(actions), join_all(procedures));
+
+    Ok(
+      actions
+        .into_iter()
+        .chain(procedures)
+        .filter(|s| !s.schedule.is_empty())
+        .collect(),
+    )
+  }
+}

bin/core/src/api/read/search.rs

@@ -1,83 +0,0 @@
-use monitor_client::{
-  api::read::{FindResources, FindResourcesResponse},
-  entities::{
-    build::Build, deployment::Deployment, procedure::Procedure,
-    repo::Repo, server::Server, update::ResourceTargetVariant,
-    user::User,
-  },
-};
-use resolver_api::Resolve;
-
-use crate::{resource, state::State};
-
-const FIND_RESOURCE_TYPES: [ResourceTargetVariant; 5] = [
-  ResourceTargetVariant::Server,
-  ResourceTargetVariant::Build,
-  ResourceTargetVariant::Deployment,
-  ResourceTargetVariant::Repo,
-  ResourceTargetVariant::Procedure,
-];
-
-impl Resolve<FindResources, User> for State {
-  async fn resolve(
-    &self,
-    FindResources { query, resources }: FindResources,
-    user: User,
-  ) -> anyhow::Result<FindResourcesResponse> {
-    let mut res = FindResourcesResponse::default();
-    let resource_types = if resources.is_empty() {
-      FIND_RESOURCE_TYPES.to_vec()
-    } else {
-      resources
-        .into_iter()
-        .filter(|r| {
-          !matches!(
-            r,
-            ResourceTargetVariant::System
-              | ResourceTargetVariant::Builder
-              | ResourceTargetVariant::Alerter
-          )
-        })
-        .collect()
-    };
-    for resource_type in resource_types {
-      match resource_type {
-        ResourceTargetVariant::Server => {
-          res.servers = resource::list_for_user_using_document::<
-            Server,
-          >(query.clone(), &user)
-          .await?;
-        }
-        ResourceTargetVariant::Deployment => {
-          res.deployments = resource::list_for_user_using_document::<
-            Deployment,
-          >(query.clone(), &user)
-          .await?;
-        }
-        ResourceTargetVariant::Build => {
-          res.builds =
-            resource::list_for_user_using_document::<Build>(
-              query.clone(),
-              &user,
-            )
-            .await?;
-        }
-        ResourceTargetVariant::Repo => {
-          res.repos = resource::list_for_user_using_document::<Repo>(
-            query.clone(),
-            &user,
-          )
-          .await?;
-        }
-        ResourceTargetVariant::Procedure => {
-          res.procedures = resource::list_for_user_using_document::<
-            Procedure,
-          >(query.clone(), &user)
-          .await?;
-        }
-        _ => {}
-      }
-    }
-    Ok(res)
-  }
-}

bin/core/src/api/read/server_template.rs

@@ -1,80 +0,0 @@
-use anyhow::Context;
-use mongo_indexed::Document;
-use monitor_client::{
-  api::read::*,
-  entities::{
-    permission::PermissionLevel, server_template::ServerTemplate,
-    user::User,
-  },
-};
-use mungos::mongodb::bson::doc;
-use resolver_api::Resolve;
-
-use crate::{
-  resource,
-  state::{db_client, State},
-};
-
-impl Resolve<GetServerTemplate, User> for State {
-  async fn resolve(
-    &self,
-    GetServerTemplate { server_template }: GetServerTemplate,
-    user: User,
-  ) -> anyhow::Result<GetServerTemplateResponse> {
-    resource::get_check_permissions::<ServerTemplate>(
-      &server_template,
-      &user,
-      PermissionLevel::Read,
-    )
-    .await
-  }
-}
-
-impl Resolve<ListServerTemplates, User> for State {
-  async fn resolve(
-    &self,
-    ListServerTemplates { query }: ListServerTemplates,
-    user: User,
-  ) -> anyhow::Result<ListServerTemplatesResponse> {
-    resource::list_for_user::<ServerTemplate>(query, &user).await
-  }
-}
-
-impl Resolve<ListFullServerTemplates, User> for State {
-  async fn resolve(
-    &self,
-    ListFullServerTemplates { query }: ListFullServerTemplates,
-    user: User,
-  ) -> anyhow::Result<ListFullServerTemplatesResponse> {
-    resource::list_full_for_user::<ServerTemplate>(query, &user).await
-  }
-}
-
-impl Resolve<GetServerTemplatesSummary, User> for State {
-  async fn resolve(
-    &self,
-    GetServerTemplatesSummary {}: GetServerTemplatesSummary,
-    user: User,
-  ) -> anyhow::Result<GetServerTemplatesSummaryResponse> {
-    let query = match resource::get_resource_ids_for_user::<
-      ServerTemplate,
-    >(&user)
-    .await?
-    {
-      Some(ids) => doc! {
-        "_id": { "$in": ids }
-      },
-      None => Document::new(),
-    };
-    let total = db_client()
-      .await
-      .server_templates
-      .count_documents(query)
-      .await
-      .context("failed to count all server template documents")?;
-    let res = GetServerTemplatesSummaryResponse {
-      total: total as u32,
-    };
-    Ok(res)
-  }
-}

bin/core/src/api/read/stack.rs

@@ -1,52 +1,61 @@
 use std::collections::HashSet;
 
-use anyhow::Context;
-use monitor_client::{
+use anyhow::{Context, anyhow};
+use komodo_client::{
   api::read::*,
   entities::{
     config::core::CoreConfig,
+    docker::container::Container,
     permission::PermissionLevel,
+    server::{Server, ServerState},
     stack::{Stack, StackActionState, StackListItem, StackState},
-    user::User,
   },
 };
-use periphery_client::api::compose::{
-  GetComposeServiceLog, GetComposeServiceLogSearch,
+use periphery_client::api::{
+  compose::{GetComposeLog, GetComposeLogSearch},
+  container::InspectContainer,
 };
 use resolver_api::Resolve;
 
 use crate::{
   config::core_config,
-  helpers::{periphery_client, stack::get_stack_and_server},
+  helpers::{periphery_client, query::get_all_tags},
+  permission::get_check_permissions,
   resource,
-  state::{action_states, github_client, stack_status_cache, State},
+  stack::get_stack_and_server,
+  state::{
+    action_states, github_client, server_status_cache,
+    stack_status_cache,
+  },
 };
 
-impl Resolve<GetStack, User> for State {
+use super::ReadArgs;
+
+impl Resolve<ReadArgs> for GetStack {
   async fn resolve(
-    &self,
-    GetStack { stack }: GetStack,
-    user: User,
-  ) -> anyhow::Result<Stack> {
-    resource::get_check_permissions::<Stack>(
-      &stack,
-      &user,
-      PermissionLevel::Read,
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<Stack> {
+    Ok(
+      get_check_permissions::<Stack>(
+        &self.stack,
+        user,
+        PermissionLevel::Read.into(),
+      )
+      .await?,
     )
-    .await
   }
 }
 
-impl Resolve<ListStackServices, User> for State {
+impl Resolve<ReadArgs> for ListStackServices {
   async fn resolve(
-    &self,
-    ListStackServices { stack }: ListStackServices,
-    user: User,
-  ) -> anyhow::Result<ListStackServicesResponse> {
-    let stack = resource::get_check_permissions::<Stack>(
-      &stack,
-      &user,
-      PermissionLevel::Read,
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<ListStackServicesResponse> {
+    let stack = get_check_permissions::<Stack>(
+      &self.stack,
+      user,
+      PermissionLevel::Read.into(),
     )
     .await?;
 
@@ -62,75 +71,144 @@ impl Resolve<ListStackServices, User> for State {
   }
 }
 
-impl Resolve<GetStackServiceLog, User> for State {
+impl Resolve<ReadArgs> for GetStackLog {
   async fn resolve(
-    &self,
-    GetStackServiceLog {
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<GetStackLogResponse> {
+    let GetStackLog {
       stack,
-      service,
+      services,
       tail,
-    }: GetStackServiceLog,
-    user: User,
-  ) -> anyhow::Result<GetStackServiceLogResponse> {
+      timestamps,
+    } = self;
     let (stack, server) = get_stack_and_server(
       &stack,
-      &user,
-      PermissionLevel::Read,
+      user,
+      PermissionLevel::Read.logs(),
       true,
     )
     .await?;
-    periphery_client(&server)?
-      .request(GetComposeServiceLog {
+    let res = periphery_client(&server)?
+      .request(GetComposeLog {
         project: stack.project_name(false),
-        service,
+        services,
         tail,
+        timestamps,
       })
       .await
-      .context("failed to get stack service log from periphery")
+      .context("Failed to get stack log from periphery")?;
+    Ok(res)
   }
 }
 
-impl Resolve<SearchStackServiceLog, User> for State {
+impl Resolve<ReadArgs> for SearchStackLog {
   async fn resolve(
-    &self,
-    SearchStackServiceLog {
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<SearchStackLogResponse> {
+    let SearchStackLog {
       stack,
-      service,
+      services,
       terms,
       combinator,
       invert,
-    }: SearchStackServiceLog,
-    user: User,
-  ) -> anyhow::Result<SearchStackServiceLogResponse> {
+      timestamps,
+    } = self;
     let (stack, server) = get_stack_and_server(
       &stack,
-      &user,
-      PermissionLevel::Read,
+      user,
+      PermissionLevel::Read.logs(),
       true,
     )
     .await?;
-    periphery_client(&server)?
-      .request(GetComposeServiceLogSearch {
+    let res = periphery_client(&server)?
+      .request(GetComposeLogSearch {
         project: stack.project_name(false),
-        service,
+        services,
         terms,
         combinator,
         invert,
+        timestamps,
       })
       .await
-      .context("failed to get stack service log from periphery")
+      .context("Failed to search stack log from periphery")?;
+    Ok(res)
   }
 }
 
-impl Resolve<ListCommonStackExtraArgs, User> for State {
+impl Resolve<ReadArgs> for InspectStackContainer {
   async fn resolve(
-    &self,
-    ListCommonStackExtraArgs { query }: ListCommonStackExtraArgs,
-    user: User,
-  ) -> anyhow::Result<ListCommonStackExtraArgsResponse> {
-    let stacks = resource::list_full_for_user::<Stack>(query, &user)
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<Container> {
+    let InspectStackContainer { stack, service } = self;
+    let stack = get_check_permissions::<Stack>(
+      &stack,
+      user,
+      PermissionLevel::Read.inspect(),
+    )
+    .await?;
+    if stack.config.server_id.is_empty() {
+      return Err(
+        anyhow!("Cannot inspect stack, not attached to any server")
+          .into(),
+      );
+    }
+    let server =
+      resource::get::<Server>(&stack.config.server_id).await?;
+    let cache = server_status_cache()
+      .get_or_insert_default(&server.id)
+      .await;
+    if cache.state != ServerState::Ok {
+      return Err(
+        anyhow!(
+          "Cannot inspect container: server is {:?}",
+          cache.state
+        )
+        .into(),
+      );
+    }
+    let services = &stack_status_cache()
+      .get(&stack.id)
       .await
-      .context("failed to get resources matching query")?;
+      .unwrap_or_default()
+      .curr
+      .services;
+    let Some(name) = services
+      .iter()
+      .find(|s| s.service == service)
+      .and_then(|s| s.container.as_ref().map(|c| c.name.clone()))
+    else {
+      return Err(anyhow!(
+        "No service found matching '{service}'. Was the stack last deployed manually?"
+      ).into());
+    };
+    let res = periphery_client(&server)?
+      .request(InspectContainer { name })
+      .await?;
+    Ok(res)
+  }
+}
+
+impl Resolve<ReadArgs> for ListCommonStackExtraArgs {
+  async fn resolve(
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<ListCommonStackExtraArgsResponse> {
+    let all_tags = if self.query.tags.is_empty() {
+      vec![]
+    } else {
+      get_all_tags(None).await?
+    };
+    let stacks = resource::list_full_for_user::<Stack>(
+      self.query,
+      user,
+      PermissionLevel::Read.into(),
+      &all_tags,
+    )
+    .await
+    .context("failed to get resources matching query")?;
 
     // first collect with guaranteed uniqueness
     let mut res = HashSet::<String>::new();
@@ -147,36 +225,107 @@ impl Resolve<ListCommonStackExtraArgs, User> for State {
   }
 }
 
-impl Resolve<ListStacks, User> for State {
+impl Resolve<ReadArgs> for ListCommonStackBuildExtraArgs {
   async fn resolve(
-    &self,
-    ListStacks { query }: ListStacks,
-    user: User,
-  ) -> anyhow::Result<Vec<StackListItem>> {
-    resource::list_for_user::<Stack>(query, &user).await
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<ListCommonStackBuildExtraArgsResponse> {
+    let all_tags = if self.query.tags.is_empty() {
+      vec![]
+    } else {
+      get_all_tags(None).await?
+    };
+    let stacks = resource::list_full_for_user::<Stack>(
+      self.query,
+      user,
+      PermissionLevel::Read.into(),
+      &all_tags,
+    )
+    .await
+    .context("failed to get resources matching query")?;
+
+    // first collect with guaranteed uniqueness
+    let mut res = HashSet::<String>::new();
+
+    for stack in stacks {
+      for extra_arg in stack.config.build_extra_args {
+        res.insert(extra_arg);
+      }
+    }
+
+    let mut res = res.into_iter().collect::<Vec<_>>();
+    res.sort();
+    Ok(res)
   }
 }
 
-impl Resolve<ListFullStacks, User> for State {
+impl Resolve<ReadArgs> for ListStacks {
   async fn resolve(
-    &self,
-    ListFullStacks { query }: ListFullStacks,
-    user: User,
-  ) -> anyhow::Result<ListFullStacksResponse> {
-    resource::list_full_for_user::<Stack>(query, &user).await
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<Vec<StackListItem>> {
+    let all_tags = if self.query.tags.is_empty() {
+      vec![]
+    } else {
+      get_all_tags(None).await?
+    };
+    let only_update_available = self.query.specific.update_available;
+    let stacks = resource::list_for_user::<Stack>(
+      self.query,
+      user,
+      PermissionLevel::Read.into(),
+      &all_tags,
+    )
+    .await?;
+    let stacks = if only_update_available {
+      stacks
+        .into_iter()
+        .filter(|stack| {
+          stack
+            .info
+            .services
+            .iter()
+            .any(|service| service.update_available)
+        })
+        .collect()
+    } else {
+      stacks
+    };
+    Ok(stacks)
   }
 }
 
-impl Resolve<GetStackActionState, User> for State {
+impl Resolve<ReadArgs> for ListFullStacks {
   async fn resolve(
-    &self,
-    GetStackActionState { stack }: GetStackActionState,
-    user: User,
-  ) -> anyhow::Result<StackActionState> {
-    let stack = resource::get_check_permissions::<Stack>(
-      &stack,
-      &user,
-      PermissionLevel::Read,
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<ListFullStacksResponse> {
+    let all_tags = if self.query.tags.is_empty() {
+      vec![]
+    } else {
+      get_all_tags(None).await?
+    };
+    Ok(
+      resource::list_full_for_user::<Stack>(
+        self.query,
+        user,
+        PermissionLevel::Read.into(),
+        &all_tags,
+      )
+      .await?,
+    )
+  }
+}
+
+impl Resolve<ReadArgs> for GetStackActionState {
+  async fn resolve(
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<StackActionState> {
+    let stack = get_check_permissions::<Stack>(
+      &self.stack,
+      user,
+      PermissionLevel::Read.into(),
     )
     .await?;
     let action_state = action_states()
@@ -189,15 +338,16 @@ impl Resolve<GetStackActionState, User> for State {
   }
 }
 
-impl Resolve<GetStacksSummary, User> for State {
+impl Resolve<ReadArgs> for GetStacksSummary {
   async fn resolve(
-    &self,
-    GetStacksSummary {}: GetStacksSummary,
-    user: User,
-  ) -> anyhow::Result<GetStacksSummaryResponse> {
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<GetStacksSummaryResponse> {
     let stacks = resource::list_full_for_user::<Stack>(
       Default::default(),
-      &user,
+      user,
+      PermissionLevel::Read.into(),
+      &[],
     )
     .await
     .context("failed to get stacks from db")?;
@@ -211,15 +361,10 @@ impl Resolve<GetStacksSummary, User> for State {
       match cache.get(&stack.id).await.unwrap_or_default().curr.state
       {
         StackState::Running => res.running += 1,
-        StackState::Paused => res.paused += 1,
-        StackState::Stopped => res.stopped += 1,
-        StackState::Restarting => res.restarting += 1,
-        StackState::Created => res.created += 1,
-        StackState::Removing => res.removing += 1,
-        StackState::Dead => res.dead += 1,
-        StackState::Unhealthy => res.unhealthy += 1,
+        StackState::Stopped | StackState::Paused => res.stopped += 1,
         StackState::Down => res.down += 1,
         StackState::Unknown => res.unknown += 1,
+        _ => res.unhealthy += 1,
       }
     }
 
@@ -227,12 +372,11 @@ impl Resolve<GetStacksSummary, User> for State {
   }
 }
 
-impl Resolve<GetStackWebhooksEnabled, User> for State {
+impl Resolve<ReadArgs> for GetStackWebhooksEnabled {
   async fn resolve(
-    &self,
-    GetStackWebhooksEnabled { stack }: GetStackWebhooksEnabled,
-    user: User,
-  ) -> anyhow::Result<GetStackWebhooksEnabledResponse> {
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<GetStackWebhooksEnabledResponse> {
     let Some(github) = github_client() else {
       return Ok(GetStackWebhooksEnabledResponse {
         managed: false,
@@ -241,10 +385,10 @@ impl Resolve<GetStackWebhooksEnabled, User> for State {
       });
     };
 
-    let stack = resource::get_check_permissions::<Stack>(
-      &stack,
-      &user,
-      PermissionLevel::Read,
+    let stack = get_check_permissions::<Stack>(
+      &self.stack,
+      user,
+      PermissionLevel::Read.into(),
     )
     .await?;
 
@@ -286,7 +430,11 @@ impl Resolve<GetStackWebhooksEnabled, User> for State {
       ..
     } = core_config();
 
-    let host = webhook_base_url.as_ref().unwrap_or(host);
+    let host = if webhook_base_url.is_empty() {
+      host
+    } else {
+      webhook_base_url
+    };
     let refresh_url =
       format!("{host}/listener/github/stack/{}/refresh", stack.id);
     let deploy_url =

bin/core/src/api/read/sync.rs

@@ -1,75 +1,99 @@
 use anyhow::Context;
-use monitor_client::{
+use komodo_client::{
   api::read::*,
   entities::{
     config::core::CoreConfig,
     permission::PermissionLevel,
     sync::{
-      PendingSyncUpdatesData, ResourceSync, ResourceSyncActionState,
-      ResourceSyncListItem, ResourceSyncState,
+      ResourceSync, ResourceSyncActionState, ResourceSyncListItem,
     },
-    user::User,
   },
 };
 use resolver_api::Resolve;
 
 use crate::{
   config::core_config,
+  helpers::query::get_all_tags,
+  permission::get_check_permissions,
   resource,
-  state::{
-    action_states, github_client, resource_sync_state_cache, State,
-  },
+  state::{action_states, github_client},
 };
 
-impl Resolve<GetResourceSync, User> for State {
+use super::ReadArgs;
+
+impl Resolve<ReadArgs> for GetResourceSync {
   async fn resolve(
-    &self,
-    GetResourceSync { sync }: GetResourceSync,
-    user: User,
-  ) -> anyhow::Result<ResourceSync> {
-    resource::get_check_permissions::<ResourceSync>(
-      &sync,
-      &user,
-      PermissionLevel::Read,
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<ResourceSync> {
+    Ok(
+      get_check_permissions::<ResourceSync>(
+        &self.sync,
+        user,
+        PermissionLevel::Read.into(),
+      )
+      .await?,
     )
-    .await
   }
 }
 
-impl Resolve<ListResourceSyncs, User> for State {
+impl Resolve<ReadArgs> for ListResourceSyncs {
   async fn resolve(
-    &self,
-    ListResourceSyncs { query }: ListResourceSyncs,
-    user: User,
-  ) -> anyhow::Result<Vec<ResourceSyncListItem>> {
-    resource::list_for_user::<ResourceSync>(query, &user).await
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<Vec<ResourceSyncListItem>> {
+    let all_tags = if self.query.tags.is_empty() {
+      vec![]
+    } else {
+      get_all_tags(None).await?
+    };
+    Ok(
+      resource::list_for_user::<ResourceSync>(
+        self.query,
+        user,
+        PermissionLevel::Read.into(),
+        &all_tags,
+      )
+      .await?,
+    )
   }
 }
 
-impl Resolve<ListFullResourceSyncs, User> for State {
+impl Resolve<ReadArgs> for ListFullResourceSyncs {
   async fn resolve(
-    &self,
-    ListFullResourceSyncs { query }: ListFullResourceSyncs,
-    user: User,
-  ) -> anyhow::Result<ListFullResourceSyncsResponse> {
-    resource::list_full_for_user::<ResourceSync>(query, &user).await
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<ListFullResourceSyncsResponse> {
+    let all_tags = if self.query.tags.is_empty() {
+      vec![]
+    } else {
+      get_all_tags(None).await?
+    };
+    Ok(
+      resource::list_full_for_user::<ResourceSync>(
+        self.query,
+        user,
+        PermissionLevel::Read.into(),
+        &all_tags,
+      )
+      .await?,
+    )
   }
 }
 
-impl Resolve<GetResourceSyncActionState, User> for State {
+impl Resolve<ReadArgs> for GetResourceSyncActionState {
   async fn resolve(
-    &self,
-    GetResourceSyncActionState { sync }: GetResourceSyncActionState,
-    user: User,
-  ) -> anyhow::Result<ResourceSyncActionState> {
-    let sync = resource::get_check_permissions::<ResourceSync>(
-      &sync,
-      &user,
-      PermissionLevel::Read,
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<ResourceSyncActionState> {
+    let sync = get_check_permissions::<ResourceSync>(
+      &self.sync,
+      user,
+      PermissionLevel::Read.into(),
     )
     .await?;
     let action_state = action_states()
-      .resource_sync
+      .sync
       .get(&sync.id)
       .await
       .unwrap_or_default()
@@ -78,76 +102,64 @@ impl Resolve<GetResourceSyncActionState, User> for State {
   }
 }
 
-impl Resolve<GetResourceSyncsSummary, User> for State {
+impl Resolve<ReadArgs> for GetResourceSyncsSummary {
   async fn resolve(
-    &self,
-    GetResourceSyncsSummary {}: GetResourceSyncsSummary,
-    user: User,
-  ) -> anyhow::Result<GetResourceSyncsSummaryResponse> {
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<GetResourceSyncsSummaryResponse> {
     let resource_syncs =
       resource::list_full_for_user::<ResourceSync>(
         Default::default(),
-        &user,
+        user,
+        PermissionLevel::Read.into(),
+        &[],
       )
       .await
       .context("failed to get resource_syncs from db")?;
 
     let mut res = GetResourceSyncsSummaryResponse::default();
 
-    let cache = resource_sync_state_cache();
     let action_states = action_states();
 
     for resource_sync in resource_syncs {
       res.total += 1;
 
-      match resource_sync.info.pending.data {
-        PendingSyncUpdatesData::Ok(data) => {
-          if !data.no_updates() {
-            res.pending += 1;
-            continue;
-          }
-        }
-        PendingSyncUpdatesData::Err(_) => {
-          res.failed += 1;
-          continue;
-        }
+      if !(resource_sync.info.pending_deploy.to_deploy == 0
+        && resource_sync.info.resource_updates.is_empty()
+        && resource_sync.info.variable_updates.is_empty()
+        && resource_sync.info.user_group_updates.is_empty())
+      {
+        res.pending += 1;
+        continue;
+      } else if resource_sync.info.pending_error.is_some()
+        || !resource_sync.info.remote_errors.is_empty()
+      {
+        res.failed += 1;
+        continue;
       }
-
-      match (
-        cache.get(&resource_sync.id).await.unwrap_or_default(),
-        action_states
-          .resource_sync
-          .get(&resource_sync.id)
-          .await
-          .unwrap_or_default()
-          .get()?,
-      ) {
-        (_, action_states) if action_states.syncing => {
-          res.syncing += 1;
-        }
-        (ResourceSyncState::Ok, _) => res.ok += 1,
-        (ResourceSyncState::Failed, _) => res.failed += 1,
-        (ResourceSyncState::Unknown, _) => res.unknown += 1,
-        // will never come off the cache in the building state, since that comes from action states
-        (ResourceSyncState::Syncing, _) => {
-          unreachable!()
-        }
-        (ResourceSyncState::Pending, _) => {
-          unreachable!()
-        }
+      if action_states
+        .sync
+        .get(&resource_sync.id)
+        .await
+        .unwrap_or_default()
+        .get()?
+        .syncing
+      {
+        res.syncing += 1;
+        continue;
       }
+      res.ok += 1;
     }
 
     Ok(res)
   }
 }
 
-impl Resolve<GetSyncWebhooksEnabled, User> for State {
+impl Resolve<ReadArgs> for GetSyncWebhooksEnabled {
   async fn resolve(
-    &self,
-    GetSyncWebhooksEnabled { sync }: GetSyncWebhooksEnabled,
-    user: User,
-  ) -> anyhow::Result<GetSyncWebhooksEnabledResponse> {
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<GetSyncWebhooksEnabledResponse> {
     let Some(github) = github_client() else {
       return Ok(GetSyncWebhooksEnabledResponse {
         managed: false,
@@ -156,10 +168,10 @@ impl Resolve<GetSyncWebhooksEnabled, User> for State {
       });
     };
 
-    let sync = resource::get_check_permissions::<ResourceSync>(
-      &sync,
-      &user,
-      PermissionLevel::Read,
+    let sync = get_check_permissions::<ResourceSync>(
+      &self.sync,
+      user,
+      PermissionLevel::Read.into(),
     )
     .await?;
 
@@ -201,7 +213,11 @@ impl Resolve<GetSyncWebhooksEnabled, User> for State {
       ..
     } = core_config();
 
-    let host = webhook_base_url.as_ref().unwrap_or(host);
+    let host = if webhook_base_url.is_empty() {
+      host
+    } else {
+      webhook_base_url
+    };
     let refresh_url =
       format!("{host}/listener/github/sync/{}/refresh", sync.id);
     let sync_url =

bin/core/src/api/read/tag.rs

@@ -1,39 +1,33 @@
 use anyhow::Context;
-use mongo_indexed::doc;
-use monitor_client::{
-  api::read::{GetTag, ListTags},
-  entities::{tag::Tag, user::User},
+use database::mongo_indexed::doc;
+use database::mungos::{
+  find::find_collect, mongodb::options::FindOptions,
+};
+use komodo_client::{
+  api::read::{GetTag, ListTags},
+  entities::tag::Tag,
 };
-use mungos::{find::find_collect, mongodb::options::FindOptions};
 use resolver_api::Resolve;
 
-use crate::{
-  helpers::query::get_tag,
-  state::{db_client, State},
-};
+use crate::{helpers::query::get_tag, state::db_client};
 
-impl Resolve<GetTag, User> for State {
-  async fn resolve(
-    &self,
-    GetTag { tag }: GetTag,
-    _: User,
-  ) -> anyhow::Result<Tag> {
-    get_tag(&tag).await
+use super::ReadArgs;
+
+impl Resolve<ReadArgs> for GetTag {
+  async fn resolve(self, _: &ReadArgs) -> serror::Result<Tag> {
+    Ok(get_tag(&self.tag).await?)
   }
 }
 
-impl Resolve<ListTags, User> for State {
-  async fn resolve(
-    &self,
-    ListTags { query }: ListTags,
-    _: User,
-  ) -> anyhow::Result<Vec<Tag>> {
-    find_collect(
-      &db_client().await.tags,
-      query,
+impl Resolve<ReadArgs> for ListTags {
+  async fn resolve(self, _: &ReadArgs) -> serror::Result<Vec<Tag>> {
+    let res = find_collect(
+      &db_client().tags,
+      self.query,
       FindOptions::builder().sort(doc! { "name": 1 }).build(),
     )
     .await
-    .context("failed to get tags from db")
+    .context("failed to get tags from db")?;
+    Ok(res)
   }
 }

bin/core/src/api/read/update.rs

@@ -1,9 +1,16 @@
 use std::collections::HashMap;
 
-use anyhow::{anyhow, Context};
-use monitor_client::{
+use anyhow::{Context, anyhow};
+use database::mungos::{
+  by_id::find_one_by_id,
+  find::find_collect,
+  mongodb::{bson::doc, options::FindOptions},
+};
+use komodo_client::{
   api::read::{GetUpdate, ListUpdates, ListUpdatesResponse},
   entities::{
+    ResourceTarget,
+    action::Action,
     alerter::Alerter,
     build::Build,
     builder::Builder,
@@ -12,49 +19,43 @@ use monitor_client::{
     procedure::Procedure,
     repo::Repo,
     server::Server,
-    server_template::ServerTemplate,
     stack::Stack,
     sync::ResourceSync,
-    update::{ResourceTarget, Update, UpdateListItem},
+    update::{Update, UpdateListItem},
     user::User,
   },
 };
-use mungos::{
-  by_id::find_one_by_id,
-  find::find_collect,
-  mongodb::{bson::doc, options::FindOptions},
-};
 use resolver_api::Resolve;
 
 use crate::{
   config::core_config,
-  resource,
-  state::{db_client, State},
+  permission::{get_check_permissions, get_resource_ids_for_user},
+  state::db_client,
 };
 
+use super::ReadArgs;
+
 const UPDATES_PER_PAGE: i64 = 100;
 
-impl Resolve<ListUpdates, User> for State {
+impl Resolve<ReadArgs> for ListUpdates {
   async fn resolve(
-    &self,
-    ListUpdates { query, page }: ListUpdates,
-    user: User,
-  ) -> anyhow::Result<ListUpdatesResponse> {
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<ListUpdatesResponse> {
     let query = if user.admin || core_config().transparent_mode {
-      query
+      self.query
     } else {
-      let server_query =
-        resource::get_resource_ids_for_user::<Server>(&user)
-          .await?
-          .map(|ids| {
-            doc! {
-              "target.type": "Server", "target.id": { "$in": ids }
-            }
-          })
-          .unwrap_or_else(|| doc! { "target.type": "Server" });
+      let server_query = get_resource_ids_for_user::<Server>(user)
+        .await?
+        .map(|ids| {
+          doc! {
+            "target.type": "Server", "target.id": { "$in": ids }
+          }
+        })
+        .unwrap_or_else(|| doc! { "target.type": "Server" });
 
       let deployment_query =
-        resource::get_resource_ids_for_user::<Deployment>(&user)
+        get_resource_ids_for_user::<Deployment>(user)
           .await?
           .map(|ids| {
             doc! {
@@ -63,38 +64,35 @@ impl Resolve<ListUpdates, User> for State {
           })
           .unwrap_or_else(|| doc! { "target.type": "Deployment" });
 
-      let stack_query =
-        resource::get_resource_ids_for_user::<Stack>(&user)
-          .await?
-          .map(|ids| {
-            doc! {
-              "target.type": "Stack", "target.id": { "$in": ids }
-            }
-          })
-          .unwrap_or_else(|| doc! { "target.type": "Stack" });
+      let stack_query = get_resource_ids_for_user::<Stack>(user)
+        .await?
+        .map(|ids| {
+          doc! {
+            "target.type": "Stack", "target.id": { "$in": ids }
+          }
+        })
+        .unwrap_or_else(|| doc! { "target.type": "Stack" });
 
-      let build_query =
-        resource::get_resource_ids_for_user::<Build>(&user)
-          .await?
-          .map(|ids| {
-            doc! {
-              "target.type": "Build", "target.id": { "$in": ids }
-            }
-          })
-          .unwrap_or_else(|| doc! { "target.type": "Build" });
+      let build_query = get_resource_ids_for_user::<Build>(user)
+        .await?
+        .map(|ids| {
+          doc! {
+            "target.type": "Build", "target.id": { "$in": ids }
+          }
+        })
+        .unwrap_or_else(|| doc! { "target.type": "Build" });
 
-      let repo_query =
-        resource::get_resource_ids_for_user::<Repo>(&user)
-          .await?
-          .map(|ids| {
-            doc! {
-              "target.type": "Repo", "target.id": { "$in": ids }
-            }
-          })
-          .unwrap_or_else(|| doc! { "target.type": "Repo" });
+      let repo_query = get_resource_ids_for_user::<Repo>(user)
+        .await?
+        .map(|ids| {
+          doc! {
+            "target.type": "Repo", "target.id": { "$in": ids }
+          }
+        })
+        .unwrap_or_else(|| doc! { "target.type": "Repo" });
 
       let procedure_query =
-        resource::get_resource_ids_for_user::<Procedure>(&user)
+        get_resource_ids_for_user::<Procedure>(user)
           .await?
           .map(|ids| {
             doc! {
@@ -103,40 +101,36 @@ impl Resolve<ListUpdates, User> for State {
           })
           .unwrap_or_else(|| doc! { "target.type": "Procedure" });
 
-      let builder_query =
-        resource::get_resource_ids_for_user::<Builder>(&user)
-          .await?
-          .map(|ids| {
-            doc! {
-              "target.type": "Builder", "target.id": { "$in": ids }
-            }
-          })
-          .unwrap_or_else(|| doc! { "target.type": "Builder" });
+      let action_query = get_resource_ids_for_user::<Action>(user)
+        .await?
+        .map(|ids| {
+          doc! {
+            "target.type": "Action", "target.id": { "$in": ids }
+          }
+        })
+        .unwrap_or_else(|| doc! { "target.type": "Action" });
 
-      let alerter_query =
-        resource::get_resource_ids_for_user::<Alerter>(&user)
-          .await?
-          .map(|ids| {
-            doc! {
-              "target.type": "Alerter", "target.id": { "$in": ids }
-            }
-          })
-          .unwrap_or_else(|| doc! { "target.type": "Alerter" });
+      let builder_query = get_resource_ids_for_user::<Builder>(user)
+        .await?
+        .map(|ids| {
+          doc! {
+            "target.type": "Builder", "target.id": { "$in": ids }
+          }
+        })
+        .unwrap_or_else(|| doc! { "target.type": "Builder" });
 
-      let server_template_query = resource::get_resource_ids_for_user::<ServerTemplate>(
-        &user,
-      )
-      .await?
-      .map(|ids| {
-        doc! {
-          "target.type": "ServerTemplate", "target.id": { "$in": ids }
-        }
-      })
-      .unwrap_or_else(|| doc! { "target.type": "ServerTemplate" });
+      let alerter_query = get_resource_ids_for_user::<Alerter>(user)
+        .await?
+        .map(|ids| {
+          doc! {
+            "target.type": "Alerter", "target.id": { "$in": ids }
+          }
+        })
+        .unwrap_or_else(|| doc! { "target.type": "Alerter" });
 
-      let resource_sync_query = resource::get_resource_ids_for_user::<ResourceSync>(
-        &user,
-      )
+      let resource_sync_query = get_resource_ids_for_user::<
+        ResourceSync,
+      >(user)
       .await?
       .map(|ids| {
         doc! {
@@ -145,7 +139,7 @@ impl Resolve<ListUpdates, User> for State {
       })
       .unwrap_or_else(|| doc! { "target.type": "ResourceSync" });
 
-      let mut query = query.unwrap_or_default();
+      let mut query = self.query.unwrap_or_default();
       query.extend(doc! {
         "$or": [
           server_query,
@@ -154,29 +148,28 @@ impl Resolve<ListUpdates, User> for State {
           build_query,
           repo_query,
           procedure_query,
+          action_query,
           alerter_query,
           builder_query,
-          server_template_query,
           resource_sync_query,
         ]
       });
       query.into()
     };
 
-    let usernames =
-      find_collect(&db_client().await.users, None, None)
-        .await
-        .context("failed to pull users from db")?
-        .into_iter()
-        .map(|u| (u.id, u.username))
-        .collect::<HashMap<_, _>>();
+    let usernames = find_collect(&db_client().users, None, None)
+      .await
+      .context("failed to pull users from db")?
+      .into_iter()
+      .map(|u| (u.id, u.username))
+      .collect::<HashMap<_, _>>();
 
     let updates = find_collect(
-      &db_client().await.updates,
+      &db_client().updates,
       query,
       FindOptions::builder()
         .sort(doc! { "start_ts": -1 })
-        .skip(page as u64 * UPDATES_PER_PAGE as u64)
+        .skip(self.page as u64 * UPDATES_PER_PAGE as u64)
         .limit(UPDATES_PER_PAGE)
         .build(),
     )
@@ -208,7 +201,7 @@ impl Resolve<ListUpdates, User> for State {
     .collect::<Vec<_>>();
 
     let next_page = if updates.len() == UPDATES_PER_PAGE as usize {
-      Some(page + 1)
+      Some(self.page + 1)
     } else {
       None
     };
@@ -217,13 +210,12 @@ impl Resolve<ListUpdates, User> for State {
   }
 }
 
-impl Resolve<GetUpdate, User> for State {
+impl Resolve<ReadArgs> for GetUpdate {
   async fn resolve(
-    &self,
-    GetUpdate { id }: GetUpdate,
-    user: User,
-  ) -> anyhow::Result<Update> {
-    let update = find_one_by_id(&db_client().await.updates, &id)
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<Update> {
+    let update = find_one_by_id(&db_client().updates, &self.id)
       .await
       .context("failed to query to db")?
       .context("no update exists with given id")?;
@@ -232,87 +224,87 @@ impl Resolve<GetUpdate, User> for State {
     }
     match &update.target {
       ResourceTarget::System(_) => {
-        return Err(anyhow!(
-          "user must be admin to view system updates"
-        ))
+        return Err(
+          anyhow!("user must be admin to view system updates").into(),
+        );
       }
       ResourceTarget::Server(id) => {
-        resource::get_check_permissions::<Server>(
+        get_check_permissions::<Server>(
           id,
-          &user,
-          PermissionLevel::Read,
+          user,
+          PermissionLevel::Read.into(),
         )
         .await?;
       }
       ResourceTarget::Deployment(id) => {
-        resource::get_check_permissions::<Deployment>(
+        get_check_permissions::<Deployment>(
           id,
-          &user,
-          PermissionLevel::Read,
+          user,
+          PermissionLevel::Read.into(),
         )
         .await?;
       }
       ResourceTarget::Build(id) => {
-        resource::get_check_permissions::<Build>(
+        get_check_permissions::<Build>(
           id,
-          &user,
-          PermissionLevel::Read,
+          user,
+          PermissionLevel::Read.into(),
         )
         .await?;
       }
       ResourceTarget::Repo(id) => {
-        resource::get_check_permissions::<Repo>(
+        get_check_permissions::<Repo>(
           id,
-          &user,
-          PermissionLevel::Read,
+          user,
+          PermissionLevel::Read.into(),
         )
         .await?;
       }
       ResourceTarget::Builder(id) => {
-        resource::get_check_permissions::<Builder>(
+        get_check_permissions::<Builder>(
           id,
-          &user,
-          PermissionLevel::Read,
+          user,
+          PermissionLevel::Read.into(),
         )
         .await?;
       }
       ResourceTarget::Alerter(id) => {
-        resource::get_check_permissions::<Alerter>(
+        get_check_permissions::<Alerter>(
           id,
-          &user,
-          PermissionLevel::Read,
+          user,
+          PermissionLevel::Read.into(),
         )
         .await?;
       }
       ResourceTarget::Procedure(id) => {
-        resource::get_check_permissions::<Procedure>(
+        get_check_permissions::<Procedure>(
           id,
-          &user,
-          PermissionLevel::Read,
+          user,
+          PermissionLevel::Read.into(),
         )
         .await?;
       }
-      ResourceTarget::ServerTemplate(id) => {
-        resource::get_check_permissions::<ServerTemplate>(
+      ResourceTarget::Action(id) => {
+        get_check_permissions::<Action>(
           id,
-          &user,
-          PermissionLevel::Read,
+          user,
+          PermissionLevel::Read.into(),
         )
         .await?;
       }
       ResourceTarget::ResourceSync(id) => {
-        resource::get_check_permissions::<ResourceSync>(
+        get_check_permissions::<ResourceSync>(
           id,
-          &user,
-          PermissionLevel::Read,
+          user,
+          PermissionLevel::Read.into(),
         )
         .await?;
       }
       ResourceTarget::Stack(id) => {
-        resource::get_check_permissions::<Stack>(
+        get_check_permissions::<Stack>(
           id,
-          &user,
-          PermissionLevel::Read,
+          user,
+          PermissionLevel::Read.into(),
         )
         .await?;
       }

bin/core/src/api/read/user.rs

@@ -1,32 +1,37 @@
-use anyhow::{anyhow, Context};
-use monitor_client::{
+use anyhow::{Context, anyhow};
+use database::mungos::{
+  by_id::find_one_by_id,
+  find::find_collect,
+  mongodb::{bson::doc, options::FindOptions},
+};
+use komodo_client::{
   api::read::{
     FindUser, FindUserResponse, GetUsername, GetUsernameResponse,
     ListApiKeys, ListApiKeysForServiceUser,
     ListApiKeysForServiceUserResponse, ListApiKeysResponse,
     ListUsers, ListUsersResponse,
   },
-  entities::user::{User, UserConfig},
-};
-use mungos::{
-  by_id::find_one_by_id,
-  find::find_collect,
-  mongodb::{bson::doc, options::FindOptions},
+  entities::user::{UserConfig, admin_service_user},
 };
 use resolver_api::Resolve;
 
-use crate::{
-  helpers::query::get_user,
-  state::{db_client, State},
-};
+use crate::{helpers::query::get_user, state::db_client};
 
-impl Resolve<GetUsername, User> for State {
+use super::ReadArgs;
+
+impl Resolve<ReadArgs> for GetUsername {
   async fn resolve(
-    &self,
-    GetUsername { user_id }: GetUsername,
-    _: User,
-  ) -> anyhow::Result<GetUsernameResponse> {
-    let user = find_one_by_id(&db_client().await.users, &user_id)
+    self,
+    _: &ReadArgs,
+  ) -> serror::Result<GetUsernameResponse> {
+    if let Some(user) = admin_service_user(&self.user_id) {
+      return Ok(GetUsernameResponse {
+        username: user.username,
+        avatar: None,
+      });
+    }
+
+    let user = find_one_by_id(&db_client().users, &self.user_id)
       .await
       .context("failed at mongo query for user")?
       .context("no user found with id")?;
@@ -44,30 +49,30 @@ impl Resolve<GetUsername, User> for State {
   }
 }
 
-impl Resolve<FindUser, User> for State {
+impl Resolve<ReadArgs> for FindUser {
   async fn resolve(
-    &self,
-    FindUser { user }: FindUser,
-    admin: User,
-  ) -> anyhow::Result<FindUserResponse> {
+    self,
+    ReadArgs { user: admin }: &ReadArgs,
+  ) -> serror::Result<FindUserResponse> {
     if !admin.admin {
-      return Err(anyhow!("This method is admin only."));
+      return Err(anyhow!("This method is admin only.").into());
     }
-    get_user(&user).await
+    Ok(get_user(&self.user).await?)
   }
 }
 
-impl Resolve<ListUsers, User> for State {
+impl Resolve<ReadArgs> for ListUsers {
   async fn resolve(
-    &self,
-    ListUsers {}: ListUsers,
-    user: User,
-  ) -> anyhow::Result<ListUsersResponse> {
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<ListUsersResponse> {
     if !user.admin {
-      return Err(anyhow!("this route is only accessable by admins"));
+      return Err(
+        anyhow!("this route is only accessable by admins").into(),
+      );
     }
     let mut users = find_collect(
-      &db_client().await.users,
+      &db_client().users,
       None,
       FindOptions::builder().sort(doc! { "username": 1 }).build(),
     )
@@ -78,14 +83,13 @@ impl Resolve<ListUsers, User> for State {
   }
 }
 
-impl Resolve<ListApiKeys, User> for State {
+impl Resolve<ReadArgs> for ListApiKeys {
   async fn resolve(
-    &self,
-    ListApiKeys {}: ListApiKeys,
-    user: User,
-  ) -> anyhow::Result<ListApiKeysResponse> {
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<ListApiKeysResponse> {
     let api_keys = find_collect(
-      &db_client().await.api_keys,
+      &db_client().api_keys,
       doc! { "user_id": &user.id },
       FindOptions::builder().sort(doc! { "name": 1 }).build(),
     )
@@ -101,23 +105,22 @@ impl Resolve<ListApiKeys, User> for State {
   }
 }
 
-impl Resolve<ListApiKeysForServiceUser, User> for State {
+impl Resolve<ReadArgs> for ListApiKeysForServiceUser {
   async fn resolve(
-    &self,
-    ListApiKeysForServiceUser { user }: ListApiKeysForServiceUser,
-    admin: User,
-  ) -> anyhow::Result<ListApiKeysForServiceUserResponse> {
+    self,
+    ReadArgs { user: admin }: &ReadArgs,
+  ) -> serror::Result<ListApiKeysForServiceUserResponse> {
     if !admin.admin {
-      return Err(anyhow!("This method is admin only."));
+      return Err(anyhow!("This method is admin only.").into());
     }
 
-    let user = get_user(&user).await?;
+    let user = get_user(&self.user).await?;
 
     let UserConfig::Service { .. } = user.config else {
-      return Err(anyhow!("Given user is not service user"));
+      return Err(anyhow!("Given user is not service user").into());
     };
     let api_keys = find_collect(
-      &db_client().await.api_keys,
+      &db_client().api_keys,
       doc! { "user_id": &user.id },
       None,
     )

bin/core/src/api/read/user_group.rs

@@ -1,65 +1,60 @@
 use std::str::FromStr;
 
 use anyhow::Context;
-use monitor_client::{
-  api::read::{
-    GetUserGroup, GetUserGroupResponse, ListUserGroups,
-    ListUserGroupsResponse,
-  },
-  entities::user::User,
-};
-use mungos::{
+use database::mungos::{
   find::find_collect,
   mongodb::{
-    bson::{doc, oid::ObjectId, Document},
+    bson::{Document, doc, oid::ObjectId},
     options::FindOptions,
   },
 };
+use komodo_client::api::read::*;
 use resolver_api::Resolve;
 
-use crate::state::{db_client, State};
+use crate::state::db_client;
 
-impl Resolve<GetUserGroup, User> for State {
+use super::ReadArgs;
+
+impl Resolve<ReadArgs> for GetUserGroup {
   async fn resolve(
-    &self,
-    GetUserGroup { user_group }: GetUserGroup,
-    user: User,
-  ) -> anyhow::Result<GetUserGroupResponse> {
-    let mut filter = match ObjectId::from_str(&user_group) {
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<GetUserGroupResponse> {
+    let mut filter = match ObjectId::from_str(&self.user_group) {
       Ok(id) => doc! { "_id": id },
-      Err(_) => doc! { "name": &user_group },
+      Err(_) => doc! { "name": &self.user_group },
     };
     // Don't allow non admin users to get UserGroups they aren't a part of.
     if !user.admin {
       // Filter for only UserGroups which contain the users id
       filter.insert("users", &user.id);
     }
-    db_client()
-      .await
+    let res = db_client()
       .user_groups
       .find_one(filter)
       .await
       .context("failed to query db for user groups")?
-      .context("no UserGroup found with given name or id")
+      .context("no UserGroup found with given name or id")?;
+    Ok(res)
   }
 }
 
-impl Resolve<ListUserGroups, User> for State {
+impl Resolve<ReadArgs> for ListUserGroups {
   async fn resolve(
-    &self,
-    ListUserGroups {}: ListUserGroups,
-    user: User,
-  ) -> anyhow::Result<ListUserGroupsResponse> {
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<ListUserGroupsResponse> {
     let mut filter = Document::new();
     if !user.admin {
       filter.insert("users", &user.id);
     }
-    find_collect(
-      &db_client().await.user_groups,
+    let res = find_collect(
+      &db_client().user_groups,
       filter,
       FindOptions::builder().sort(doc! { "name": 1 }).build(),
     )
     .await
-    .context("failed to query db for UserGroups")
+    .context("failed to query db for UserGroups")?;
+    Ok(res)
   }
 }

bin/core/src/api/read/variable.rs

@@ -1,42 +1,53 @@
 use anyhow::Context;
-use mongo_indexed::doc;
-use monitor_client::{
-  api::read::{
-    GetVariable, GetVariableResponse, ListVariables,
-    ListVariablesResponse,
-  },
-  entities::user::User,
+use database::mongo_indexed::doc;
+use database::mungos::{
+  find::find_collect, mongodb::options::FindOptions,
 };
-use mungos::{find::find_collect, mongodb::options::FindOptions};
+use komodo_client::api::read::*;
 use resolver_api::Resolve;
 
-use crate::{
-  helpers::query::get_variable,
-  state::{db_client, State},
-};
+use crate::{helpers::query::get_variable, state::db_client};
 
-impl Resolve<GetVariable, User> for State {
+use super::ReadArgs;
+
+impl Resolve<ReadArgs> for GetVariable {
   async fn resolve(
-    &self,
-    GetVariable { name }: GetVariable,
-    _: User,
-  ) -> anyhow::Result<GetVariableResponse> {
-    get_variable(&name).await
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<GetVariableResponse> {
+    let mut variable = get_variable(&self.name).await?;
+    if !variable.is_secret || user.admin {
+      return Ok(variable);
+    }
+    variable.value = "#".repeat(variable.value.len());
+    Ok(variable)
   }
 }
 
-impl Resolve<ListVariables, User> for State {
+impl Resolve<ReadArgs> for ListVariables {
   async fn resolve(
-    &self,
-    ListVariables {}: ListVariables,
-    _: User,
-  ) -> anyhow::Result<ListVariablesResponse> {
-    find_collect(
-      &db_client().await.variables,
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<ListVariablesResponse> {
+    let variables = find_collect(
+      &db_client().variables,
       None,
       FindOptions::builder().sort(doc! { "name": 1 }).build(),
     )
     .await
-    .context("failed to query db for variables")
+    .context("failed to query db for variables")?;
+    if user.admin {
+      return Ok(variables);
+    }
+    let variables = variables
+      .into_iter()
+      .map(|mut variable| {
+        if variable.is_secret {
+          variable.value = "#".repeat(variable.value.len());
+        }
+        variable
+      })
+      .collect();
+    Ok(variables)
   }
 }

bin/core/src/api/terminal.rs

@@ -0,0 +1,299 @@
+use anyhow::Context;
+use axum::{Extension, Router, middleware, routing::post};
+use komodo_client::{
+  api::terminal::*,
+  entities::{
+    deployment::Deployment, permission::PermissionLevel,
+    server::Server, stack::Stack, user::User,
+  },
+};
+use serror::Json;
+use uuid::Uuid;
+
+use crate::{
+  auth::auth_request, helpers::periphery_client,
+  permission::get_check_permissions, resource::get,
+  state::stack_status_cache,
+};
+
+pub fn router() -> Router {
+  Router::new()
+    .route("/execute", post(execute_terminal))
+    .route("/execute/container", post(execute_container_exec))
+    .route("/execute/deployment", post(execute_deployment_exec))
+    .route("/execute/stack", post(execute_stack_exec))
+    .layer(middleware::from_fn(auth_request))
+}
+
+// =================
+//  ExecuteTerminal
+// =================
+
+async fn execute_terminal(
+  Extension(user): Extension<User>,
+  Json(request): Json<ExecuteTerminalBody>,
+) -> serror::Result<axum::body::Body> {
+  execute_terminal_inner(Uuid::new_v4(), request, user).await
+}
+
+#[instrument(
+  name = "ExecuteTerminal",
+  skip(user),
+  fields(
+    user_id = user.id,
+  )
+)]
+async fn execute_terminal_inner(
+  req_id: Uuid,
+  ExecuteTerminalBody {
+    server,
+    terminal,
+    command,
+  }: ExecuteTerminalBody,
+  user: User,
+) -> serror::Result<axum::body::Body> {
+  info!("/terminal/execute request | user: {}", user.username);
+
+  let res = async {
+    let server = get_check_permissions::<Server>(
+      &server,
+      &user,
+      PermissionLevel::Read.terminal(),
+    )
+    .await?;
+
+    let periphery = periphery_client(&server)?;
+
+    let stream = periphery
+      .execute_terminal(terminal, command)
+      .await
+      .context("Failed to execute command on periphery")?;
+
+    anyhow::Ok(stream)
+  }
+  .await;
+
+  let stream = match res {
+    Ok(stream) => stream,
+    Err(e) => {
+      warn!("/terminal/execute request {req_id} error: {e:#}");
+      return Err(e.into());
+    }
+  };
+
+  Ok(axum::body::Body::from_stream(stream.into_line_stream()))
+}
+
+// ======================
+//  ExecuteContainerExec
+// ======================
+
+async fn execute_container_exec(
+  Extension(user): Extension<User>,
+  Json(request): Json<ExecuteContainerExecBody>,
+) -> serror::Result<axum::body::Body> {
+  execute_container_exec_inner(Uuid::new_v4(), request, user).await
+}
+
+#[instrument(
+  name = "ExecuteContainerExec",
+  skip(user),
+  fields(
+    user_id = user.id,
+  )
+)]
+async fn execute_container_exec_inner(
+  req_id: Uuid,
+  ExecuteContainerExecBody {
+    server,
+    container,
+    shell,
+    command,
+  }: ExecuteContainerExecBody,
+  user: User,
+) -> serror::Result<axum::body::Body> {
+  info!(
+    "/terminal/execute/container request | user: {}",
+    user.username
+  );
+
+  let res = async {
+    let server = get_check_permissions::<Server>(
+      &server,
+      &user,
+      PermissionLevel::Read.terminal(),
+    )
+    .await?;
+
+    let periphery = periphery_client(&server)?;
+
+    let stream = periphery
+      .execute_container_exec(container, shell, command)
+      .await
+      .context(
+        "Failed to execute container exec command on periphery",
+      )?;
+
+    anyhow::Ok(stream)
+  }
+  .await;
+
+  let stream = match res {
+    Ok(stream) => stream,
+    Err(e) => {
+      warn!(
+        "/terminal/execute/container request {req_id} error: {e:#}"
+      );
+      return Err(e.into());
+    }
+  };
+
+  Ok(axum::body::Body::from_stream(stream.into_line_stream()))
+}
+
+// =======================
+//  ExecuteDeploymentExec
+// =======================
+
+async fn execute_deployment_exec(
+  Extension(user): Extension<User>,
+  Json(request): Json<ExecuteDeploymentExecBody>,
+) -> serror::Result<axum::body::Body> {
+  execute_deployment_exec_inner(Uuid::new_v4(), request, user).await
+}
+
+#[instrument(
+  name = "ExecuteDeploymentExec",
+  skip(user),
+  fields(
+    user_id = user.id,
+  )
+)]
+async fn execute_deployment_exec_inner(
+  req_id: Uuid,
+  ExecuteDeploymentExecBody {
+    deployment,
+    shell,
+    command,
+  }: ExecuteDeploymentExecBody,
+  user: User,
+) -> serror::Result<axum::body::Body> {
+  info!(
+    "/terminal/execute/deployment request | user: {}",
+    user.username
+  );
+
+  let res = async {
+    let deployment = get_check_permissions::<Deployment>(
+      &deployment,
+      &user,
+      PermissionLevel::Read.terminal(),
+    )
+    .await?;
+
+    let server = get::<Server>(&deployment.config.server_id).await?;
+
+    let periphery = periphery_client(&server)?;
+
+    let stream = periphery
+      .execute_container_exec(deployment.name, shell, command)
+      .await
+      .context(
+        "Failed to execute container exec command on periphery",
+      )?;
+
+    anyhow::Ok(stream)
+  }
+  .await;
+
+  let stream = match res {
+    Ok(stream) => stream,
+    Err(e) => {
+      warn!(
+        "/terminal/execute/deployment request {req_id} error: {e:#}"
+      );
+      return Err(e.into());
+    }
+  };
+
+  Ok(axum::body::Body::from_stream(stream.into_line_stream()))
+}
+
+// ==================
+//  ExecuteStackExec
+// ==================
+
+async fn execute_stack_exec(
+  Extension(user): Extension<User>,
+  Json(request): Json<ExecuteStackExecBody>,
+) -> serror::Result<axum::body::Body> {
+  execute_stack_exec_inner(Uuid::new_v4(), request, user).await
+}
+
+#[instrument(
+  name = "ExecuteStackExec",
+  skip(user),
+  fields(
+    user_id = user.id,
+  )
+)]
+async fn execute_stack_exec_inner(
+  req_id: Uuid,
+  ExecuteStackExecBody {
+    stack,
+    service,
+    shell,
+    command,
+  }: ExecuteStackExecBody,
+  user: User,
+) -> serror::Result<axum::body::Body> {
+  info!("/terminal/execute/stack request | user: {}", user.username);
+
+  let res = async {
+    let stack = get_check_permissions::<Stack>(
+      &stack,
+      &user,
+      PermissionLevel::Read.terminal(),
+    )
+    .await?;
+
+    let server = get::<Server>(&stack.config.server_id).await?;
+
+    let container = stack_status_cache()
+      .get(&stack.id)
+      .await
+      .context("could not get stack status")?
+      .curr
+      .services
+      .iter()
+      .find(|s| s.service == service)
+      .context("could not find service")?
+      .container
+      .as_ref()
+      .context("could not find service container")?
+      .name
+      .clone();
+
+    let periphery = periphery_client(&server)?;
+
+    let stream = periphery
+      .execute_container_exec(container, shell, command)
+      .await
+      .context(
+        "Failed to execute container exec command on periphery",
+      )?;
+
+    anyhow::Ok(stream)
+  }
+  .await;
+
+  let stream = match res {
+    Ok(stream) => stream,
+    Err(e) => {
+      warn!("/terminal/execute/stack request {req_id} error: {e:#}");
+      return Err(e.into());
+    }
+  };
+
+  Ok(axum::body::Body::from_stream(stream.into_line_stream()))
+}

bin/core/src/api/user.rs

@@ -1,34 +1,44 @@
 use std::{collections::VecDeque, time::Instant};
 
-use anyhow::{anyhow, Context};
-use axum::{middleware, routing::post, Extension, Json, Router};
-use axum_extra::{headers::ContentType, TypedHeader};
-use mongo_indexed::doc;
-use monitor_client::{
-  api::user::{
-    CreateApiKey, CreateApiKeyResponse, DeleteApiKey,
-    DeleteApiKeyResponse, PushRecentlyViewed,
-    PushRecentlyViewedResponse, SetLastSeenUpdate,
-    SetLastSeenUpdateResponse,
-  },
-  entities::{api_key::ApiKey, monitor_timestamp, user::User},
+use anyhow::{Context, anyhow};
+use axum::{
+  Extension, Json, Router, extract::Path, middleware, routing::post,
 };
-use mungos::{by_id::update_one_by_id, mongodb::bson::to_bson};
-use resolver_api::{derive::Resolver, Resolve, Resolver};
+use database::mongo_indexed::doc;
+use database::mungos::{
+  by_id::update_one_by_id, mongodb::bson::to_bson,
+};
+use derive_variants::EnumVariants;
+use komodo_client::{
+  api::user::*,
+  entities::{api_key::ApiKey, komodo_timestamp, user::User},
+};
+use resolver_api::Resolve;
+use response::Response;
 use serde::{Deserialize, Serialize};
+use serde_json::json;
 use typeshare::typeshare;
 use uuid::Uuid;
 
 use crate::{
   auth::auth_request,
   helpers::{query::get_user, random_string},
-  state::{db_client, State},
+  state::db_client,
 };
 
+use super::Variant;
+
+pub struct UserArgs {
+  pub user: User,
+}
+
 #[typeshare]
-#[derive(Serialize, Deserialize, Debug, Clone, Resolver)]
-#[resolver_target(State)]
-#[resolver_args(User)]
+#[derive(
+  Serialize, Deserialize, Debug, Clone, Resolve, EnumVariants,
+)]
+#[args(UserArgs)]
+#[response(Response)]
+#[error(serror::Error)]
 #[serde(tag = "type", content = "params")]
 enum UserRequest {
   PushRecentlyViewed(PushRecentlyViewed),
@@ -40,54 +50,57 @@ enum UserRequest {
 pub fn router() -> Router {
   Router::new()
     .route("/", post(handler))
+    .route("/{variant}", post(variant_handler))
     .layer(middleware::from_fn(auth_request))
 }
 
+async fn variant_handler(
+  user: Extension<User>,
+  Path(Variant { variant }): Path<Variant>,
+  Json(params): Json<serde_json::Value>,
+) -> serror::Result<axum::response::Response> {
+  let req: UserRequest = serde_json::from_value(json!({
+    "type": variant,
+    "params": params,
+  }))?;
+  handler(user, Json(req)).await
+}
+
 #[instrument(name = "UserHandler", level = "debug", skip(user))]
 async fn handler(
   Extension(user): Extension<User>,
   Json(request): Json<UserRequest>,
-) -> serror::Result<(TypedHeader<ContentType>, String)> {
+) -> serror::Result<axum::response::Response> {
   let timer = Instant::now();
   let req_id = Uuid::new_v4();
   debug!(
     "/user request {req_id} | user: {} ({})",
     user.username, user.id
   );
-  let res =
-    State
-      .resolve_request(request, user)
-      .await
-      .map_err(|e| match e {
-        resolver_api::Error::Serialization(e) => {
-          anyhow!("{e:?}").context("response serialization error")
-        }
-        resolver_api::Error::Inner(e) => e,
-      });
+  let res = request.resolve(&UserArgs { user }).await;
   if let Err(e) = &res {
-    warn!("/user request {req_id} error: {e:#}");
+    warn!("/user request {req_id} error: {:#}", e.error);
   }
   let elapsed = timer.elapsed();
   debug!("/user request {req_id} | resolve time: {elapsed:?}");
-  Ok((TypedHeader(ContentType::json()), res?))
+  res.map(|res| res.0)
 }
 
 const RECENTLY_VIEWED_MAX: usize = 10;
 
-impl Resolve<PushRecentlyViewed, User> for State {
+impl Resolve<UserArgs> for PushRecentlyViewed {
   #[instrument(
     name = "PushRecentlyViewed",
     level = "debug",
-    skip(self, user)
+    skip(user)
   )]
   async fn resolve(
-    &self,
-    PushRecentlyViewed { resource }: PushRecentlyViewed,
-    user: User,
-  ) -> anyhow::Result<PushRecentlyViewedResponse> {
+    self,
+    UserArgs { user }: &UserArgs,
+  ) -> serror::Result<PushRecentlyViewedResponse> {
     let user = get_user(&user.id).await?;
 
-    let (resource_type, id) = resource.extract_variant_id();
+    let (resource_type, id) = self.resource.extract_variant_id();
     let update = match user.recents.get(&resource_type) {
       Some(recents) => {
         let mut recents = recents
@@ -103,9 +116,9 @@ impl Resolve<PushRecentlyViewed, User> for State {
       }
     };
     update_one_by_id(
-      &db_client().await.users,
+      &db_client().users,
       &user.id,
-      mungos::update::Update::Set(update),
+      database::mungos::update::Update::Set(update),
       None,
     )
     .await
@@ -117,22 +130,21 @@ impl Resolve<PushRecentlyViewed, User> for State {
   }
 }
 
-impl Resolve<SetLastSeenUpdate, User> for State {
+impl Resolve<UserArgs> for SetLastSeenUpdate {
   #[instrument(
     name = "SetLastSeenUpdate",
     level = "debug",
-    skip(self, user)
+    skip(user)
   )]
   async fn resolve(
-    &self,
-    SetLastSeenUpdate {}: SetLastSeenUpdate,
-    user: User,
-  ) -> anyhow::Result<SetLastSeenUpdateResponse> {
+    self,
+    UserArgs { user }: &UserArgs,
+  ) -> serror::Result<SetLastSeenUpdateResponse> {
     update_one_by_id(
-      &db_client().await.users,
+      &db_client().users,
       &user.id,
-      mungos::update::Update::Set(doc! {
-        "last_update_view": monitor_timestamp()
+      database::mungos::update::Update::Set(doc! {
+        "last_update_view": komodo_timestamp()
       }),
       None,
     )
@@ -145,17 +157,12 @@ impl Resolve<SetLastSeenUpdate, User> for State {
 const SECRET_LENGTH: usize = 40;
 const BCRYPT_COST: u32 = 10;
 
-impl Resolve<CreateApiKey, User> for State {
-  #[instrument(
-    name = "CreateApiKey",
-    level = "debug",
-    skip(self, user)
-  )]
+impl Resolve<UserArgs> for CreateApiKey {
+  #[instrument(name = "CreateApiKey", level = "debug", skip(user))]
   async fn resolve(
-    &self,
-    CreateApiKey { name, expires }: CreateApiKey,
-    user: User,
-  ) -> anyhow::Result<CreateApiKeyResponse> {
+    self,
+    UserArgs { user }: &UserArgs,
+  ) -> serror::Result<CreateApiKeyResponse> {
     let user = get_user(&user.id).await?;
 
     let key = format!("K-{}", random_string(SECRET_LENGTH));
@@ -164,15 +171,14 @@ impl Resolve<CreateApiKey, User> for State {
       .context("failed at hashing secret string")?;
 
     let api_key = ApiKey {
-      name,
+      name: self.name,
       key: key.clone(),
       secret: secret_hash,
       user_id: user.id.clone(),
-      created_at: monitor_timestamp(),
-      expires,
+      created_at: komodo_timestamp(),
+      expires: self.expires,
     };
     db_client()
-      .await
       .api_keys
       .insert_one(api_key)
       .await
@@ -181,26 +187,21 @@ impl Resolve<CreateApiKey, User> for State {
   }
 }
 
-impl Resolve<DeleteApiKey, User> for State {
-  #[instrument(
-    name = "DeleteApiKey",
-    level = "debug",
-    skip(self, user)
-  )]
+impl Resolve<UserArgs> for DeleteApiKey {
+  #[instrument(name = "DeleteApiKey", level = "debug", skip(user))]
   async fn resolve(
-    &self,
-    DeleteApiKey { key }: DeleteApiKey,
-    user: User,
-  ) -> anyhow::Result<DeleteApiKeyResponse> {
-    let client = db_client().await;
+    self,
+    UserArgs { user }: &UserArgs,
+  ) -> serror::Result<DeleteApiKeyResponse> {
+    let client = db_client();
     let key = client
       .api_keys
-      .find_one(doc! { "key": &key })
+      .find_one(doc! { "key": &self.key })
       .await
       .context("failed at db query")?
       .context("no api key with key found")?;
     if user.id != key.user_id {
-      return Err(anyhow!("api key does not belong to user"));
+      return Err(anyhow!("api key does not belong to user").into());
     }
     client
       .api_keys

bin/core/src/api/write/action.rs

@@ -0,0 +1,64 @@
+use komodo_client::{
+  api::write::*,
+  entities::{
+    action::Action, permission::PermissionLevel, update::Update,
+  },
+};
+use resolver_api::Resolve;
+
+use crate::{permission::get_check_permissions, resource};
+
+use super::WriteArgs;
+
+impl Resolve<WriteArgs> for CreateAction {
+  #[instrument(name = "CreateAction", skip(user))]
+  async fn resolve(
+    self,
+    WriteArgs { user }: &WriteArgs,
+  ) -> serror::Result<Action> {
+    resource::create::<Action>(&self.name, self.config, user).await
+  }
+}
+
+impl Resolve<WriteArgs> for CopyAction {
+  #[instrument(name = "CopyAction", skip(user))]
+  async fn resolve(
+    self,
+    WriteArgs { user }: &WriteArgs,
+  ) -> serror::Result<Action> {
+    let Action { config, .. } = get_check_permissions::<Action>(
+      &self.id,
+      user,
+      PermissionLevel::Write.into(),
+    )
+    .await?;
+    resource::create::<Action>(&self.name, config.into(), user).await
+  }
+}
+
+impl Resolve<WriteArgs> for UpdateAction {
+  #[instrument(name = "UpdateAction", skip(user))]
+  async fn resolve(
+    self,
+    WriteArgs { user }: &WriteArgs,
+  ) -> serror::Result<Action> {
+    Ok(resource::update::<Action>(&self.id, self.config, user).await?)
+  }
+}
+
+impl Resolve<WriteArgs> for RenameAction {
+  #[instrument(name = "RenameAction", skip(user))]
+  async fn resolve(
+    self,
+    WriteArgs { user }: &WriteArgs,
+  ) -> serror::Result<Update> {
+    Ok(resource::rename::<Action>(&self.id, &self.name, user).await?)
+  }
+}
+
+impl Resolve<WriteArgs> for DeleteAction {
+  #[instrument(name = "DeleteAction", skip(args))]
+  async fn resolve(self, args: &WriteArgs) -> serror::Result<Action> {
+    Ok(resource::delete::<Action>(&self.id, args).await?)
+  }
+}

bin/core/src/api/write/alerter.rs

@@ -1,61 +1,70 @@
-use monitor_client::{
-  api::write::{
-    CopyAlerter, CreateAlerter, DeleteAlerter, UpdateAlerter,
-  },
+use komodo_client::{
+  api::write::*,
   entities::{
-    alerter::Alerter, permission::PermissionLevel, user::User,
+    alerter::Alerter, permission::PermissionLevel, update::Update,
   },
 };
 use resolver_api::Resolve;
 
-use crate::{resource, state::State};
+use crate::{permission::get_check_permissions, resource};
 
-impl Resolve<CreateAlerter, User> for State {
-  #[instrument(name = "CreateAlerter", skip(self, user))]
+use super::WriteArgs;
+
+impl Resolve<WriteArgs> for CreateAlerter {
+  #[instrument(name = "CreateAlerter", skip(user))]
   async fn resolve(
-    &self,
-    CreateAlerter { name, config }: CreateAlerter,
-    user: User,
-  ) -> anyhow::Result<Alerter> {
-    resource::create::<Alerter>(&name, config, &user).await
+    self,
+    WriteArgs { user }: &WriteArgs,
+  ) -> serror::Result<Alerter> {
+    resource::create::<Alerter>(&self.name, self.config, user).await
   }
 }
 
-impl Resolve<CopyAlerter, User> for State {
-  #[instrument(name = "CopyAlerter", skip(self, user))]
+impl Resolve<WriteArgs> for CopyAlerter {
+  #[instrument(name = "CopyAlerter", skip(user))]
   async fn resolve(
-    &self,
-    CopyAlerter { name, id }: CopyAlerter,
-    user: User,
-  ) -> anyhow::Result<Alerter> {
-    let Alerter { config, .. } = resource::get_check_permissions::<
-      Alerter,
-    >(
-      &id, &user, PermissionLevel::Write
+    self,
+    WriteArgs { user }: &WriteArgs,
+  ) -> serror::Result<Alerter> {
+    let Alerter { config, .. } = get_check_permissions::<Alerter>(
+      &self.id,
+      user,
+      PermissionLevel::Write.into(),
     )
     .await?;
-    resource::create::<Alerter>(&name, config.into(), &user).await
+    resource::create::<Alerter>(&self.name, config.into(), user).await
   }
 }
 
-impl Resolve<DeleteAlerter, User> for State {
-  #[instrument(name = "DeleteAlerter", skip(self, user))]
+impl Resolve<WriteArgs> for DeleteAlerter {
+  #[instrument(name = "DeleteAlerter", skip(args))]
   async fn resolve(
-    &self,
-    DeleteAlerter { id }: DeleteAlerter,
-    user: User,
-  ) -> anyhow::Result<Alerter> {
-    resource::delete::<Alerter>(&id, &user).await
+    self,
+    args: &WriteArgs,
+  ) -> serror::Result<Alerter> {
+    Ok(resource::delete::<Alerter>(&self.id, args).await?)
   }
 }
 
-impl Resolve<UpdateAlerter, User> for State {
-  #[instrument(name = "UpdateAlerter", skip(self, user))]
+impl Resolve<WriteArgs> for UpdateAlerter {
+  #[instrument(name = "UpdateAlerter", skip(user))]
   async fn resolve(
-    &self,
-    UpdateAlerter { id, config }: UpdateAlerter,
-    user: User,
-  ) -> anyhow::Result<Alerter> {
-    resource::update::<Alerter>(&id, config, &user).await
+    self,
+    WriteArgs { user }: &WriteArgs,
+  ) -> serror::Result<Alerter> {
+    Ok(
+      resource::update::<Alerter>(&self.id, self.config, user)
+        .await?,
+    )
+  }
+}
+
+impl Resolve<WriteArgs> for RenameAlerter {
+  #[instrument(name = "RenameAlerter", skip(user))]
+  async fn resolve(
+    self,
+    WriteArgs { user }: &WriteArgs,
+  ) -> serror::Result<Update> {
+    Ok(resource::rename::<Alerter>(&self.id, &self.name, user).await?)
   }
 }

bin/core/src/api/write/build.rs

@@ -1,144 +1,397 @@
-use anyhow::{anyhow, Context};
-use mongo_indexed::doc;
-use monitor_client::{
+use std::{path::PathBuf, str::FromStr, time::Duration};
+
+use anyhow::{Context, anyhow};
+use database::mongo_indexed::doc;
+use database::mungos::mongodb::bson::to_document;
+use formatting::format_serror;
+use komodo_client::{
   api::write::*,
   entities::{
+    FileContents, NoData, Operation, RepoExecutionArgs,
+    all_logs_success,
     build::{Build, BuildInfo, PartialBuildConfig},
+    builder::{Builder, BuilderConfig},
     config::core::CoreConfig,
     permission::PermissionLevel,
-    user::User,
-    CloneArgs, NoData,
+    repo::Repo,
+    server::ServerState,
+    update::Update,
   },
 };
-use mungos::mongodb::bson::to_document;
 use octorust::types::{
   ReposCreateWebhookRequest, ReposCreateWebhookRequestConfig,
 };
+use periphery_client::{
+  PeripheryClient,
+  api::build::{
+    GetDockerfileContentsOnHost, WriteDockerfileContentsToHost,
+  },
+};
 use resolver_api::Resolve;
+use tokio::fs;
 
 use crate::{
   config::core_config,
-  helpers::{git_token, random_string},
+  helpers::{
+    git_token, periphery_client,
+    query::get_server_with_state,
+    update::{add_update, make_update},
+  },
+  permission::get_check_permissions,
   resource,
-  state::{db_client, github_client, State},
+  state::{db_client, github_client},
 };
 
-impl Resolve<CreateBuild, User> for State {
-  #[instrument(name = "CreateBuild", skip(self, user))]
+use super::WriteArgs;
+
+impl Resolve<WriteArgs> for CreateBuild {
+  #[instrument(name = "CreateBuild", skip(user))]
   async fn resolve(
-    &self,
-    CreateBuild { name, config }: CreateBuild,
-    user: User,
-  ) -> anyhow::Result<Build> {
-    resource::create::<Build>(&name, config, &user).await
+    self,
+    WriteArgs { user }: &WriteArgs,
+  ) -> serror::Result<Build> {
+    resource::create::<Build>(&self.name, self.config, user).await
   }
 }
 
-impl Resolve<CopyBuild, User> for State {
-  #[instrument(name = "CopyBuild", skip(self, user))]
+impl Resolve<WriteArgs> for CopyBuild {
+  #[instrument(name = "CopyBuild", skip(user))]
   async fn resolve(
-    &self,
-    CopyBuild { name, id }: CopyBuild,
-    user: User,
-  ) -> anyhow::Result<Build> {
-    let Build { mut config, .. } =
-      resource::get_check_permissions::<Build>(
-        &id,
-        &user,
-        PermissionLevel::Write,
-      )
-      .await?;
+    self,
+    WriteArgs { user }: &WriteArgs,
+  ) -> serror::Result<Build> {
+    let Build { mut config, .. } = get_check_permissions::<Build>(
+      &self.id,
+      user,
+      PermissionLevel::Read.into(),
+    )
+    .await?;
     // reset version to 0.0.0
     config.version = Default::default();
-    resource::create::<Build>(&name, config.into(), &user).await
+    resource::create::<Build>(&self.name, config.into(), user).await
   }
 }
 
-impl Resolve<DeleteBuild, User> for State {
-  #[instrument(name = "DeleteBuild", skip(self, user))]
-  async fn resolve(
-    &self,
-    DeleteBuild { id }: DeleteBuild,
-    user: User,
-  ) -> anyhow::Result<Build> {
-    resource::delete::<Build>(&id, &user).await
+impl Resolve<WriteArgs> for DeleteBuild {
+  #[instrument(name = "DeleteBuild", skip(args))]
+  async fn resolve(self, args: &WriteArgs) -> serror::Result<Build> {
+    Ok(resource::delete::<Build>(&self.id, args).await?)
   }
 }
 
-impl Resolve<UpdateBuild, User> for State {
-  #[instrument(name = "UpdateBuild", skip(self, user))]
+impl Resolve<WriteArgs> for UpdateBuild {
+  #[instrument(name = "UpdateBuild", skip(user))]
   async fn resolve(
-    &self,
-    UpdateBuild { id, config }: UpdateBuild,
-    user: User,
-  ) -> anyhow::Result<Build> {
-    resource::update::<Build>(&id, config, &user).await
+    self,
+    WriteArgs { user }: &WriteArgs,
+  ) -> serror::Result<Build> {
+    Ok(resource::update::<Build>(&self.id, self.config, user).await?)
   }
 }
 
-impl Resolve<RefreshBuildCache, User> for State {
-  #[instrument(
-    name = "RefreshBuildCache",
-    level = "debug",
-    skip(self, user)
-  )]
+impl Resolve<WriteArgs> for RenameBuild {
+  #[instrument(name = "RenameBuild", skip(user))]
   async fn resolve(
-    &self,
-    RefreshBuildCache { build }: RefreshBuildCache,
-    user: User,
-  ) -> anyhow::Result<NoData> {
-    // Even though this is a write request, this doesn't change any config. Anyone that can execute the
-    // build should be able to do this.
-    let build = resource::get_check_permissions::<Build>(
-      &build,
-      &user,
-      PermissionLevel::Execute,
+    self,
+    WriteArgs { user }: &WriteArgs,
+  ) -> serror::Result<Update> {
+    Ok(resource::rename::<Build>(&self.id, &self.name, user).await?)
+  }
+}
+
+impl Resolve<WriteArgs> for WriteBuildFileContents {
+  #[instrument(name = "WriteBuildFileContents", skip(args))]
+  async fn resolve(self, args: &WriteArgs) -> serror::Result<Update> {
+    let build = get_check_permissions::<Build>(
+      &self.build,
+      &args.user,
+      PermissionLevel::Write.into(),
     )
     .await?;
 
-    let config = core_config();
-
-    let repo_dir = config.repo_directory.join(random_string(10));
-    let mut clone_args: CloneArgs = (&build).into();
-    // Don't want to run these on core.
-    clone_args.on_clone = None;
-    clone_args.on_pull = None;
-    clone_args.destination = Some(repo_dir.display().to_string());
-
-    let access_token = match (&clone_args.account, &clone_args.provider)
+    if !build.config.files_on_host
+      && build.config.repo.is_empty()
+      && build.config.linked_repo.is_empty()
     {
-      (None, _) => None,
-      (Some(_), None) => {
-        return Err(anyhow!(
-          "Account is configured, but provider is empty"
-        ))
-      }
-      (Some(username), Some(provider)) => {
-        git_token(provider, username, |https| {
-          clone_args.https = https
+      return Err(anyhow!(
+        "Build is not configured to use Files on Host or Git Repo, can't write dockerfile contents"
+      ).into());
+    }
+
+    let mut update =
+      make_update(&build, Operation::WriteDockerfile, &args.user);
+
+    update.push_simple_log("Dockerfile to write", &self.contents);
+
+    if build.config.files_on_host {
+      match get_on_host_periphery(&build)
+        .await?
+        .request(WriteDockerfileContentsToHost {
+          name: build.name,
+          build_path: build.config.build_path,
+          dockerfile_path: build.config.dockerfile_path,
+          contents: self.contents,
         })
         .await
-        .with_context(
-          || format!("Failed to get git token in call to db. Stopping run. | {provider} | {username}"),
-        )?
+        .context("Failed to write dockerfile contents to host")
+      {
+        Ok(log) => {
+          update.logs.push(log);
+        }
+        Err(e) => {
+          update.push_error_log(
+            "Write Dockerfile Contents",
+            format_serror(&e.into()),
+          );
+        }
+      };
+
+      if !all_logs_success(&update.logs) {
+        update.finalize();
+        update.id = add_update(update.clone()).await?;
+
+        return Ok(update);
       }
+
+      if let Err(e) =
+        (RefreshBuildCache { build: build.id }).resolve(args).await
+      {
+        update.push_error_log(
+          "Refresh build cache",
+          format_serror(&e.error.into()),
+        );
+      }
+
+      update.finalize();
+      update.id = add_update(update.clone()).await?;
+
+      Ok(update)
+    } else {
+      write_dockerfile_contents_git(self, args, build, update).await
+    }
+  }
+}
+
+async fn write_dockerfile_contents_git(
+  req: WriteBuildFileContents,
+  args: &WriteArgs,
+  build: Build,
+  mut update: Update,
+) -> serror::Result<Update> {
+  let WriteBuildFileContents { build: _, contents } = req;
+
+  let mut repo_args: RepoExecutionArgs = if !build
+    .config
+    .files_on_host
+    && !build.config.linked_repo.is_empty()
+  {
+    (&crate::resource::get::<Repo>(&build.config.linked_repo).await?)
+      .into()
+  } else {
+    (&build).into()
+  };
+  let root = repo_args.unique_path(&core_config().repo_directory)?;
+  repo_args.destination = Some(root.display().to_string());
+
+  let build_path = build
+    .config
+    .build_path
+    .parse::<PathBuf>()
+    .context("Invalid build path")?;
+  let dockerfile_path = build
+    .config
+    .dockerfile_path
+    .parse::<PathBuf>()
+    .context("Invalid dockerfile path")?;
+
+  let full_path = root.join(&build_path).join(&dockerfile_path);
+
+  if let Some(parent) = full_path.parent() {
+    fs::create_dir_all(parent).await.with_context(|| {
+      format!(
+        "Failed to initialize dockerfile parent directory {parent:?}"
+      )
+    })?;
+  }
+
+  let access_token = if let Some(account) = &repo_args.account {
+    git_token(&repo_args.provider, account, |https| repo_args.https = https)
+    .await
+    .with_context(
+      || format!("Failed to get git token in call to db. Stopping run. | {} | {account}", repo_args.provider),
+    )?
+  } else {
+    None
+  };
+
+  // Ensure the folder is initialized as git repo.
+  // This allows a new file to be committed on a branch that may not exist.
+  if !root.join(".git").exists() {
+    git::init_folder_as_repo(
+      &root,
+      &repo_args,
+      access_token.as_deref(),
+      &mut update.logs,
+    )
+    .await;
+
+    if !all_logs_success(&update.logs) {
+      update.finalize();
+      update.id = add_update(update.clone()).await?;
+
+      return Ok(update);
+    }
+  }
+
+  // Save this for later -- repo_args moved next.
+  let branch = repo_args.branch.clone();
+  // Pull latest changes to repo to ensure linear commit history
+  match git::pull_or_clone(
+    repo_args,
+    &core_config().repo_directory,
+    access_token,
+  )
+  .await
+  .context("Failed to pull latest changes before commit")
+  {
+    Ok((res, _)) => update.logs.extend(res.logs),
+    Err(e) => {
+      update.push_error_log("Pull Repo", format_serror(&e.into()));
+      update.finalize();
+      return Ok(update);
+    }
+  };
+
+  if !all_logs_success(&update.logs) {
+    update.finalize();
+    update.id = add_update(update.clone()).await?;
+
+    return Ok(update);
+  }
+
+  if let Err(e) =
+    fs::write(&full_path, &contents).await.with_context(|| {
+      format!("Failed to write dockerfile contents to {full_path:?}")
+    })
+  {
+    update
+      .push_error_log("Write Dockerfile", format_serror(&e.into()));
+  } else {
+    update.push_simple_log(
+      "Write Dockerfile",
+      format!("File written to {full_path:?}"),
+    );
+  };
+
+  if !all_logs_success(&update.logs) {
+    update.finalize();
+    update.id = add_update(update.clone()).await?;
+
+    return Ok(update);
+  }
+
+  let commit_res = git::commit_file(
+    &format!("{}: Commit Dockerfile", args.user.username),
+    &root,
+    &build_path.join(&dockerfile_path),
+    &branch,
+  )
+  .await;
+
+  update.logs.extend(commit_res.logs);
+
+  if let Err(e) = (RefreshBuildCache { build: build.name })
+    .resolve(args)
+    .await
+  {
+    update.push_error_log(
+      "Refresh build cache",
+      format_serror(&e.error.into()),
+    );
+  }
+
+  update.finalize();
+  update.id = add_update(update.clone()).await?;
+
+  Ok(update)
+}
+
+impl Resolve<WriteArgs> for RefreshBuildCache {
+  #[instrument(
+    name = "RefreshBuildCache",
+    level = "debug",
+    skip(user)
+  )]
+  async fn resolve(
+    self,
+    WriteArgs { user }: &WriteArgs,
+  ) -> serror::Result<NoData> {
+    // Even though this is a write request, this doesn't change any config. Anyone that can execute the
+    // build should be able to do this.
+    let build = get_check_permissions::<Build>(
+      &self.build,
+      user,
+      PermissionLevel::Execute.into(),
+    )
+    .await?;
+
+    let repo = if !build.config.files_on_host
+      && !build.config.linked_repo.is_empty()
+    {
+      crate::resource::get::<Repo>(&build.config.linked_repo)
+        .await?
+        .into()
+    } else {
+      None
     };
 
-    let (_, latest_hash, latest_message, _) = git::clone(
-      clone_args,
-      &config.repo_directory,
-      access_token,
-      &[],
-      "",
-      None,
-    )
-    .await
-    .context("failed to clone build repo")?;
+    let (
+      remote_path,
+      remote_contents,
+      remote_error,
+      latest_hash,
+      latest_message,
+    ) = if build.config.files_on_host {
+      // =============
+      // FILES ON HOST
+      // =============
+      match get_on_host_dockerfile(&build).await {
+        Ok(FileContents { path, contents }) => {
+          (Some(path), Some(contents), None, None, None)
+        }
+        Err(e) => {
+          (None, None, Some(format_serror(&e.into())), None, None)
+        }
+      }
+    } else if let Some(repo) = &repo {
+      let Some(res) = get_git_remote(&build, repo.into()).await?
+      else {
+        // Nothing to do here
+        return Ok(NoData {});
+      };
+      res
+    } else if !build.config.repo.is_empty() {
+      let Some(res) = get_git_remote(&build, (&build).into()).await?
+      else {
+        // Nothing to do here
+        return Ok(NoData {});
+      };
+      res
+    } else {
+      // =============
+      // UI BASED FILE
+      // =============
+      (None, None, None, None, None)
+    };
 
     let info = BuildInfo {
       last_built_at: build.info.last_built_at,
       built_hash: build.info.built_hash,
       built_message: build.info.built_message,
+      built_contents: build.info.built_contents,
+      remote_path,
+      remote_contents,
+      remote_error,
       latest_hash,
       latest_message,
     };
@@ -147,7 +400,6 @@ impl Resolve<RefreshBuildCache, User> for State {
       .context("failed to serialize build info to bson")?;
 
     db_client()
-      .await
       .builds
       .update_one(
         doc! { "name": &build.name },
@@ -156,49 +408,166 @@ impl Resolve<RefreshBuildCache, User> for State {
       .await
       .context("failed to update build info on db")?;
 
-    if repo_dir.exists() {
-      if let Err(e) = std::fs::remove_dir_all(&repo_dir) {
-        warn!("failed to remove build cache update repo directory | {e:?}")
-      }
-    }
-
     Ok(NoData {})
   }
 }
 
-impl Resolve<CreateBuildWebhook, User> for State {
-  #[instrument(name = "CreateBuildWebhook", skip(self, user))]
+async fn get_on_host_periphery(
+  build: &Build,
+) -> anyhow::Result<PeripheryClient> {
+  if build.config.builder_id.is_empty() {
+    return Err(anyhow!("No builder associated with build"));
+  }
+
+  let builder = resource::get::<Builder>(&build.config.builder_id)
+    .await
+    .context("Failed to get builder")?;
+
+  match builder.config {
+    BuilderConfig::Aws(_) => {
+      Err(anyhow!("Files on host doesn't work with AWS builder"))
+    }
+    BuilderConfig::Url(config) => {
+      let periphery = PeripheryClient::new(
+        config.address,
+        config.passkey,
+        Duration::from_secs(3),
+      );
+      periphery.health_check().await?;
+      Ok(periphery)
+    }
+    BuilderConfig::Server(config) => {
+      if config.server_id.is_empty() {
+        return Err(anyhow!(
+          "Builder is type server, but has no server attached"
+        ));
+      }
+      let (server, state) =
+        get_server_with_state(&config.server_id).await?;
+      if state != ServerState::Ok {
+        return Err(anyhow!(
+          "Builder server is disabled or not reachable"
+        ));
+      };
+      periphery_client(&server)
+    }
+  }
+}
+
+/// The successful case will be included as Some(remote_contents).
+/// The error case will be included as Some(remote_error)
+async fn get_on_host_dockerfile(
+  build: &Build,
+) -> anyhow::Result<FileContents> {
+  get_on_host_periphery(build)
+    .await?
+    .request(GetDockerfileContentsOnHost {
+      name: build.name.clone(),
+      build_path: build.config.build_path.clone(),
+      dockerfile_path: build.config.dockerfile_path.clone(),
+    })
+    .await
+}
+
+async fn get_git_remote(
+  build: &Build,
+  mut clone_args: RepoExecutionArgs,
+) -> anyhow::Result<
+  Option<(
+    Option<String>,
+    Option<String>,
+    Option<String>,
+    Option<String>,
+    Option<String>,
+  )>,
+> {
+  if clone_args.provider.is_empty() {
+    // Nothing to do here
+    return Ok(None);
+  }
+  let config = core_config();
+  let repo_path = clone_args.unique_path(&config.repo_directory)?;
+  clone_args.destination = Some(repo_path.display().to_string());
+
+  let access_token = if let Some(username) = &clone_args.account {
+    git_token(&clone_args.provider, username, |https| {
+          clone_args.https = https
+        })
+        .await
+        .with_context(
+          || format!("Failed to get git token in call to db. Stopping run. | {} | {username}", clone_args.provider),
+        )?
+  } else {
+    None
+  };
+
+  let (res, _) = git::pull_or_clone(
+    clone_args,
+    &config.repo_directory,
+    access_token,
+  )
+  .await
+  .context("failed to clone build repo")?;
+
+  let relative_path = PathBuf::from_str(&build.config.build_path)
+    .context("Invalid build path")?
+    .join(&build.config.dockerfile_path);
+
+  let full_path = repo_path.join(&relative_path);
+  let (contents, error) =
+    match fs::read_to_string(&full_path).await.with_context(|| {
+      format!("Failed to read dockerfile contents at {full_path:?}")
+    }) {
+      Ok(contents) => (Some(contents), None),
+      Err(e) => (None, Some(format_serror(&e.into()))),
+    };
+  Ok(Some((
+    Some(relative_path.display().to_string()),
+    contents,
+    error,
+    res.commit_hash,
+    res.commit_message,
+  )))
+}
+
+impl Resolve<WriteArgs> for CreateBuildWebhook {
+  #[instrument(name = "CreateBuildWebhook", skip(args))]
   async fn resolve(
-    &self,
-    CreateBuildWebhook { build }: CreateBuildWebhook,
-    user: User,
-  ) -> anyhow::Result<CreateBuildWebhookResponse> {
+    self,
+    args: &WriteArgs,
+  ) -> serror::Result<CreateBuildWebhookResponse> {
     let Some(github) = github_client() else {
-      return Err(anyhow!(
-        "github_webhook_app is not configured in core config toml"
-      ));
+      return Err(
+        anyhow!(
+          "github_webhook_app is not configured in core config toml"
+        )
+        .into(),
+      );
     };
 
-    let build = resource::get_check_permissions::<Build>(
-      &build,
-      &user,
-      PermissionLevel::Write,
+    let WriteArgs { user } = args;
+
+    let build = get_check_permissions::<Build>(
+      &self.build,
+      user,
+      PermissionLevel::Write.into(),
     )
     .await?;
 
     if build.config.repo.is_empty() {
-      return Err(anyhow!(
-        "No repo configured, can't create webhook"
-      ));
+      return Err(
+        anyhow!("No repo configured, can't create webhook").into(),
+      );
     }
 
     let mut split = build.config.repo.split('/');
     let owner = split.next().context("Build repo has no owner")?;
 
     let Some(github) = github.get(owner) else {
-      return Err(anyhow!(
-        "Cannot manage repo webhooks under owner {owner}"
-      ));
+      return Err(
+        anyhow!("Cannot manage repo webhooks under owner {owner}")
+          .into(),
+      );
     };
 
     let repo =
@@ -226,7 +595,11 @@ impl Resolve<CreateBuildWebhook, User> for State {
       &build.config.webhook_secret
     };
 
-    let host = webhook_base_url.as_ref().unwrap_or(host);
+    let host = if webhook_base_url.is_empty() {
+      host
+    } else {
+      webhook_base_url
+    };
     let url = format!("{host}/listener/github/build/{}", build.id);
 
     for webhook in webhooks {
@@ -255,64 +628,65 @@ impl Resolve<CreateBuildWebhook, User> for State {
       .context("failed to create webhook")?;
 
     if !build.config.webhook_enabled {
-      self
-        .resolve(
-          UpdateBuild {
-            id: build.id,
-            config: PartialBuildConfig {
-              webhook_enabled: Some(true),
-              ..Default::default()
-            },
-          },
-          user,
-        )
-        .await
-        .context("failed to update build to enable webhook")?;
+      UpdateBuild {
+        id: build.id,
+        config: PartialBuildConfig {
+          webhook_enabled: Some(true),
+          ..Default::default()
+        },
+      }
+      .resolve(args)
+      .await
+      .map_err(|e| e.error)
+      .context("failed to update build to enable webhook")?;
     }
 
     Ok(NoData {})
   }
 }
 
-impl Resolve<DeleteBuildWebhook, User> for State {
-  #[instrument(name = "DeleteBuildWebhook", skip(self, user))]
+impl Resolve<WriteArgs> for DeleteBuildWebhook {
+  #[instrument(name = "DeleteBuildWebhook", skip(user))]
   async fn resolve(
-    &self,
-    DeleteBuildWebhook { build }: DeleteBuildWebhook,
-    user: User,
-  ) -> anyhow::Result<DeleteBuildWebhookResponse> {
+    self,
+    WriteArgs { user }: &WriteArgs,
+  ) -> serror::Result<DeleteBuildWebhookResponse> {
     let Some(github) = github_client() else {
-      return Err(anyhow!(
-        "github_webhook_app is not configured in core config toml"
-      ));
+      return Err(
+        anyhow!(
+          "github_webhook_app is not configured in core config toml"
+        )
+        .into(),
+      );
     };
 
-    let build = resource::get_check_permissions::<Build>(
-      &build,
-      &user,
-      PermissionLevel::Write,
+    let build = get_check_permissions::<Build>(
+      &self.build,
+      user,
+      PermissionLevel::Write.into(),
     )
     .await?;
 
     if build.config.git_provider != "github.com" {
-      return Err(anyhow!(
-        "Can only manage github.com repo webhooks"
-      ));
+      return Err(
+        anyhow!("Can only manage github.com repo webhooks").into(),
+      );
     }
 
     if build.config.repo.is_empty() {
-      return Err(anyhow!(
-        "No repo configured, can't delete webhook"
-      ));
+      return Err(
+        anyhow!("No repo configured, can't delete webhook").into(),
+      );
     }
 
     let mut split = build.config.repo.split('/');
     let owner = split.next().context("Build repo has no owner")?;
 
     let Some(github) = github.get(owner) else {
-      return Err(anyhow!(
-        "Cannot manage repo webhooks under owner {owner}"
-      ));
+      return Err(
+        anyhow!("Cannot manage repo webhooks under owner {owner}")
+          .into(),
+      );
     };
 
     let repo =
@@ -332,7 +706,11 @@ impl Resolve<DeleteBuildWebhook, User> for State {
       ..
     } = core_config();
 
-    let host = webhook_base_url.as_ref().unwrap_or(host);
+    let host = if webhook_base_url.is_empty() {
+      host
+    } else {
+      webhook_base_url
+    };
     let url = format!("{host}/listener/github/build/{}", build.id);
 
     for webhook in webhooks {

bin/core/src/api/write/builder.rs

@@ -1,59 +1,70 @@
-use monitor_client::{
+use komodo_client::{
   api::write::*,
   entities::{
-    builder::Builder, permission::PermissionLevel, user::User,
+    builder::Builder, permission::PermissionLevel, update::Update,
   },
 };
 use resolver_api::Resolve;
 
-use crate::{resource, state::State};
+use crate::{permission::get_check_permissions, resource};
 
-impl Resolve<CreateBuilder, User> for State {
-  #[instrument(name = "CreateBuilder", skip(self, user))]
+use super::WriteArgs;
+
+impl Resolve<WriteArgs> for CreateBuilder {
+  #[instrument(name = "CreateBuilder", skip(user))]
   async fn resolve(
-    &self,
-    CreateBuilder { name, config }: CreateBuilder,
-    user: User,
-  ) -> anyhow::Result<Builder> {
-    resource::create::<Builder>(&name, config, &user).await
+    self,
+    WriteArgs { user }: &WriteArgs,
+  ) -> serror::Result<Builder> {
+    resource::create::<Builder>(&self.name, self.config, user).await
   }
 }
 
-impl Resolve<CopyBuilder, User> for State {
-  #[instrument(name = "CopyBuilder", skip(self, user))]
+impl Resolve<WriteArgs> for CopyBuilder {
+  #[instrument(name = "CopyBuilder", skip(user))]
   async fn resolve(
-    &self,
-    CopyBuilder { name, id }: CopyBuilder,
-    user: User,
-  ) -> anyhow::Result<Builder> {
-    let Builder { config, .. } = resource::get_check_permissions::<
-      Builder,
-    >(
-      &id, &user, PermissionLevel::Write
+    self,
+    WriteArgs { user }: &WriteArgs,
+  ) -> serror::Result<Builder> {
+    let Builder { config, .. } = get_check_permissions::<Builder>(
+      &self.id,
+      user,
+      PermissionLevel::Write.into(),
     )
     .await?;
-    resource::create::<Builder>(&name, config.into(), &user).await
+    resource::create::<Builder>(&self.name, config.into(), user).await
   }
 }
 
-impl Resolve<DeleteBuilder, User> for State {
-  #[instrument(name = "DeleteBuilder", skip(self, user))]
+impl Resolve<WriteArgs> for DeleteBuilder {
+  #[instrument(name = "DeleteBuilder", skip(args))]
   async fn resolve(
-    &self,
-    DeleteBuilder { id }: DeleteBuilder,
-    user: User,
-  ) -> anyhow::Result<Builder> {
-    resource::delete::<Builder>(&id, &user).await
+    self,
+    args: &WriteArgs,
+  ) -> serror::Result<Builder> {
+    Ok(resource::delete::<Builder>(&self.id, args).await?)
   }
 }
 
-impl Resolve<UpdateBuilder, User> for State {
-  #[instrument(name = "UpdateBuilder", skip(self, user))]
+impl Resolve<WriteArgs> for UpdateBuilder {
+  #[instrument(name = "UpdateBuilder", skip(user))]
   async fn resolve(
-    &self,
-    UpdateBuilder { id, config }: UpdateBuilder,
-    user: User,
-  ) -> anyhow::Result<Builder> {
-    resource::update::<Builder>(&id, config, &user).await
+    self,
+    WriteArgs { user }: &WriteArgs,
+  ) -> serror::Result<Builder> {
+    Ok(
+      resource::update::<Builder>(&self.id, self.config, user)
+        .await?,
+    )
+  }
+}
+
+impl Resolve<WriteArgs> for RenameBuilder {
+  #[instrument(name = "RenameBuilder", skip(user))]
+  async fn resolve(
+    self,
+    WriteArgs { user }: &WriteArgs,
+  ) -> serror::Result<Update> {
+    Ok(resource::rename::<Builder>(&self.id, &self.name, user).await?)
   }
 }

bin/core/src/api/write/deployment.rs

@@ -1,19 +1,22 @@
-use anyhow::{anyhow, Context};
-use monitor_client::{
+use anyhow::{Context, anyhow};
+use database::mungos::{by_id::update_one_by_id, mongodb::bson::doc};
+use komodo_client::{
   api::write::*,
   entities::{
-    deployment::{Deployment, DeploymentState},
-    monitor_timestamp,
-    permission::PermissionLevel,
-    server::Server,
-    to_monitor_name,
-    update::Update,
-    user::User,
     Operation,
+    deployment::{
+      Deployment, DeploymentImage, DeploymentState,
+      PartialDeploymentConfig, RestartMode,
+    },
+    docker::container::RestartPolicyNameEnum,
+    komodo_timestamp,
+    permission::PermissionLevel,
+    server::{Server, ServerState},
+    to_container_compatible_name,
+    update::Update,
   },
 };
-use mungos::{by_id::update_one_by_id, mongodb::bson::doc};
-use periphery_client::api;
+use periphery_client::api::{self, container::InspectContainer};
 use resolver_api::Resolve;
 
 use crate::{
@@ -22,72 +25,167 @@ use crate::{
     query::get_deployment_state,
     update::{add_update, make_update},
   },
+  permission::get_check_permissions,
   resource,
-  state::{action_states, db_client, State},
+  state::{action_states, db_client, server_status_cache},
 };
 
-impl Resolve<CreateDeployment, User> for State {
-  #[instrument(name = "CreateDeployment", skip(self, user))]
+use super::WriteArgs;
+
+impl Resolve<WriteArgs> for CreateDeployment {
+  #[instrument(name = "CreateDeployment", skip(user))]
   async fn resolve(
-    &self,
-    CreateDeployment { name, config }: CreateDeployment,
-    user: User,
-  ) -> anyhow::Result<Deployment> {
-    resource::create::<Deployment>(&name, config, &user).await
+    self,
+    WriteArgs { user }: &WriteArgs,
+  ) -> serror::Result<Deployment> {
+    resource::create::<Deployment>(&self.name, self.config, user)
+      .await
   }
 }
 
-impl Resolve<CopyDeployment, User> for State {
-  #[instrument(name = "CopyDeployment", skip(self, user))]
+impl Resolve<WriteArgs> for CopyDeployment {
+  #[instrument(name = "CopyDeployment", skip(user))]
   async fn resolve(
-    &self,
-    CopyDeployment { name, id }: CopyDeployment,
-    user: User,
-  ) -> anyhow::Result<Deployment> {
+    self,
+    WriteArgs { user }: &WriteArgs,
+  ) -> serror::Result<Deployment> {
     let Deployment { config, .. } =
-      resource::get_check_permissions::<Deployment>(
-        &id,
-        &user,
-        PermissionLevel::Write,
+      get_check_permissions::<Deployment>(
+        &self.id,
+        user,
+        PermissionLevel::Read.into(),
       )
       .await?;
-    resource::create::<Deployment>(&name, config.into(), &user).await
+    resource::create::<Deployment>(&self.name, config.into(), user)
+      .await
   }
 }
 
-impl Resolve<DeleteDeployment, User> for State {
-  #[instrument(name = "DeleteDeployment", skip(self, user))]
+impl Resolve<WriteArgs> for CreateDeploymentFromContainer {
+  #[instrument(name = "CreateDeploymentFromContainer", skip(user))]
   async fn resolve(
-    &self,
-    DeleteDeployment { id }: DeleteDeployment,
-    user: User,
-  ) -> anyhow::Result<Deployment> {
-    resource::delete::<Deployment>(&id, &user).await
+    self,
+    WriteArgs { user }: &WriteArgs,
+  ) -> serror::Result<Deployment> {
+    let server = get_check_permissions::<Server>(
+      &self.server,
+      user,
+      PermissionLevel::Read.inspect().attach(),
+    )
+    .await?;
+    let cache = server_status_cache()
+      .get_or_insert_default(&server.id)
+      .await;
+    if cache.state != ServerState::Ok {
+      return Err(
+        anyhow!(
+          "Cannot inspect container: server is {:?}",
+          cache.state
+        )
+        .into(),
+      );
+    }
+    let container = periphery_client(&server)?
+      .request(InspectContainer {
+        name: self.name.clone(),
+      })
+      .await
+      .context("Failed to inspect container")?;
+
+    let mut config = PartialDeploymentConfig {
+      server_id: server.id.into(),
+      ..Default::default()
+    };
+
+    if let Some(container_config) = container.config {
+      config.image = container_config
+        .image
+        .map(|image| DeploymentImage::Image { image });
+      config.command = container_config.cmd.join(" ").into();
+      config.environment = container_config
+        .env
+        .into_iter()
+        .map(|env| format!("  {env}"))
+        .collect::<Vec<_>>()
+        .join("\n")
+        .into();
+      config.labels = container_config
+        .labels
+        .into_iter()
+        .map(|(key, val)| format!("  {key}: {val}"))
+        .collect::<Vec<_>>()
+        .join("\n")
+        .into();
+    }
+    if let Some(host_config) = container.host_config {
+      config.volumes = host_config
+        .binds
+        .into_iter()
+        .map(|bind| format!("  {bind}"))
+        .collect::<Vec<_>>()
+        .join("\n")
+        .into();
+      config.network = host_config.network_mode;
+      config.ports = host_config
+        .port_bindings
+        .into_iter()
+        .filter_map(|(container, mut host)| {
+          let host = host.pop()?.host_port?;
+          Some(format!("  {host}:{}", container.replace("/tcp", "")))
+        })
+        .collect::<Vec<_>>()
+        .join("\n")
+        .into();
+      config.restart = host_config.restart_policy.map(|restart| {
+        match restart.name {
+          RestartPolicyNameEnum::Always => RestartMode::Always,
+          RestartPolicyNameEnum::No
+          | RestartPolicyNameEnum::Empty => RestartMode::NoRestart,
+          RestartPolicyNameEnum::UnlessStopped => {
+            RestartMode::UnlessStopped
+          }
+          RestartPolicyNameEnum::OnFailure => RestartMode::OnFailure,
+        }
+      });
+    }
+
+    resource::create::<Deployment>(&self.name, config, user).await
   }
 }
 
-impl Resolve<UpdateDeployment, User> for State {
-  #[instrument(name = "UpdateDeployment", skip(self, user))]
+impl Resolve<WriteArgs> for DeleteDeployment {
+  #[instrument(name = "DeleteDeployment", skip(args))]
   async fn resolve(
-    &self,
-    UpdateDeployment { id, config }: UpdateDeployment,
-    user: User,
-  ) -> anyhow::Result<Deployment> {
-    resource::update::<Deployment>(&id, config, &user).await
+    self,
+    args: &WriteArgs,
+  ) -> serror::Result<Deployment> {
+    Ok(resource::delete::<Deployment>(&self.id, args).await?)
   }
 }
 
-impl Resolve<RenameDeployment, User> for State {
-  #[instrument(name = "RenameDeployment", skip(self, user))]
+impl Resolve<WriteArgs> for UpdateDeployment {
+  #[instrument(name = "UpdateDeployment", skip(user))]
   async fn resolve(
-    &self,
-    RenameDeployment { id, name }: RenameDeployment,
-    user: User,
-  ) -> anyhow::Result<Update> {
-    let deployment = resource::get_check_permissions::<Deployment>(
-      &id,
-      &user,
-      PermissionLevel::Write,
+    self,
+    WriteArgs { user }: &WriteArgs,
+  ) -> serror::Result<Deployment> {
+    Ok(
+      resource::update::<Deployment>(&self.id, self.config, user)
+        .await?,
+    )
+  }
+}
+
+impl Resolve<WriteArgs> for RenameDeployment {
+  #[instrument(name = "RenameDeployment", skip(user))]
+  async fn resolve(
+    self,
+    WriteArgs { user }: &WriteArgs,
+  ) -> serror::Result<Update> {
+    let deployment = get_check_permissions::<Deployment>(
+      &self.id,
+      user,
+      PermissionLevel::Write.into(),
     )
     .await?;
 
@@ -102,29 +200,33 @@ impl Resolve<RenameDeployment, User> for State {
     let _action_guard =
       action_state.update(|state| state.renaming = true)?;
 
-    let name = to_monitor_name(&name);
+    let name = to_container_compatible_name(&self.name);
 
-    let container_state = get_deployment_state(&deployment).await?;
+    let container_state =
+      get_deployment_state(&deployment.id).await?;
 
     if container_state == DeploymentState::Unknown {
-      return Err(anyhow!(
-        "cannot rename deployment when container status is unknown"
-      ));
+      return Err(
+        anyhow!(
+          "Cannot rename Deployment when container status is unknown"
+        )
+        .into(),
+      );
     }
 
     let mut update =
-      make_update(&deployment, Operation::RenameDeployment, &user);
+      make_update(&deployment, Operation::RenameDeployment, user);
 
     update_one_by_id(
-      &db_client().await.deployments,
+      &db_client().deployments,
       &deployment.id,
-      mungos::update::Update::Set(
-        doc! { "name": &name, "updated_at": monitor_timestamp() },
+      database::mungos::update::Update::Set(
+        doc! { "name": &name, "updated_at": komodo_timestamp() },
       ),
       None,
     )
     .await
-    .context("failed to update deployment name on db")?;
+    .context("Failed to update Deployment name on db")?;
 
     if container_state != DeploymentState::NotDeployed {
       let server =
@@ -135,20 +237,19 @@ impl Resolve<RenameDeployment, User> for State {
           new_name: name.clone(),
         })
         .await
-        .context("failed to rename container on server")?;
+        .context("Failed to rename container on server")?;
       update.logs.push(log);
     }
 
     update.push_simple_log(
-      "rename deployment",
+      "Rename Deployment",
       format!(
-        "renamed deployment from {} to {}",
+        "Renamed Deployment from {} to {}",
         deployment.name, name
       ),
     );
     update.finalize();
-
-    add_update(update.clone()).await?;
+    update.id = add_update(update.clone()).await?;
 
     Ok(update)
   }

bin/core/src/api/write/description.rs

@@ -1,114 +0,0 @@
-use anyhow::anyhow;
-use monitor_client::{
-  api::write::{UpdateDescription, UpdateDescriptionResponse},
-  entities::{
-    alerter::Alerter, build::Build, builder::Builder,
-    deployment::Deployment, procedure::Procedure, repo::Repo,
-    server::Server, server_template::ServerTemplate, stack::Stack,
-    sync::ResourceSync, update::ResourceTarget, user::User,
-  },
-};
-use resolver_api::Resolve;
-
-use crate::{resource, state::State};
-
-impl Resolve<UpdateDescription, User> for State {
-  #[instrument(name = "UpdateDescription", skip(self, user))]
-  async fn resolve(
-    &self,
-    UpdateDescription {
-      target,
-      description,
-    }: UpdateDescription,
-    user: User,
-  ) -> anyhow::Result<UpdateDescriptionResponse> {
-    match target {
-      ResourceTarget::System(_) => {
-        return Err(anyhow!(
-          "cannot update description of System resource target"
-        ))
-      }
-      ResourceTarget::Server(id) => {
-        resource::update_description::<Server>(
-          &id,
-          &description,
-          &user,
-        )
-        .await?;
-      }
-      ResourceTarget::Deployment(id) => {
-        resource::update_description::<Deployment>(
-          &id,
-          &description,
-          &user,
-        )
-        .await?;
-      }
-      ResourceTarget::Build(id) => {
-        resource::update_description::<Build>(
-          &id,
-          &description,
-          &user,
-        )
-        .await?;
-      }
-      ResourceTarget::Repo(id) => {
-        resource::update_description::<Repo>(
-          &id,
-          &description,
-          &user,
-        )
-        .await?;
-      }
-      ResourceTarget::Builder(id) => {
-        resource::update_description::<Builder>(
-          &id,
-          &description,
-          &user,
-        )
-        .await?;
-      }
-      ResourceTarget::Alerter(id) => {
-        resource::update_description::<Alerter>(
-          &id,
-          &description,
-          &user,
-        )
-        .await?;
-      }
-      ResourceTarget::Procedure(id) => {
-        resource::update_description::<Procedure>(
-          &id,
-          &description,
-          &user,
-        )
-        .await?;
-      }
-      ResourceTarget::ServerTemplate(id) => {
-        resource::update_description::<ServerTemplate>(
-          &id,
-          &description,
-          &user,
-        )
-        .await?;
-      }
-      ResourceTarget::ResourceSync(id) => {
-        resource::update_description::<ResourceSync>(
-          &id,
-          &description,
-          &user,
-        )
-        .await?;
-      }
-      ResourceTarget::Stack(id) => {
-        resource::update_description::<Stack>(
-          &id,
-          &description,
-          &user,
-        )
-        .await?;
-      }
-    }
-    Ok(UpdateDescriptionResponse {})
-  }
-}

bin/core/src/api/write/mod.rs

@@ -1,45 +1,62 @@
 use std::time::Instant;
 
-use anyhow::{anyhow, Context};
-use axum::{middleware, routing::post, Extension, Router};
-use axum_extra::{headers::ContentType, TypedHeader};
+use anyhow::Context;
+use axum::{
+  Extension, Router, extract::Path, middleware, routing::post,
+};
 use derive_variants::{EnumVariants, ExtractVariant};
-use monitor_client::{api::write::*, entities::user::User};
-use resolver_api::{derive::Resolver, Resolver};
+use komodo_client::{api::write::*, entities::user::User};
+use resolver_api::Resolve;
+use response::Response;
 use serde::{Deserialize, Serialize};
+use serde_json::json;
 use serror::Json;
 use typeshare::typeshare;
 use uuid::Uuid;
 
-use crate::{auth::auth_request, state::State};
+use crate::auth::auth_request;
 
+use super::Variant;
+
+mod action;
 mod alerter;
 mod build;
 mod builder;
 mod deployment;
-mod description;
 mod permissions;
 mod procedure;
 mod provider;
 mod repo;
+mod resource;
 mod server;
-mod server_template;
 mod service_user;
 mod stack;
 mod sync;
 mod tag;
+mod user;
 mod user_group;
 mod variable;
 
+pub struct WriteArgs {
+  pub user: User,
+}
+
 #[typeshare]
 #[derive(
-  Serialize, Deserialize, Debug, Clone, Resolver, EnumVariants,
+  Serialize, Deserialize, Debug, Clone, Resolve, EnumVariants,
 )]
 #[variant_derive(Debug)]
-#[resolver_target(State)]
-#[resolver_args(User)]
+#[args(WriteArgs)]
+#[response(Response)]
+#[error(serror::Error)]
 #[serde(tag = "type", content = "params")]
 pub enum WriteRequest {
+  // ==== USER ====
+  CreateLocalUser(CreateLocalUser),
+  UpdateUserUsername(UpdateUserUsername),
+  UpdateUserPassword(UpdateUserPassword),
+  DeleteUser(DeleteUser),
+
   // ==== SERVICE USER ====
   CreateServiceUser(CreateServiceUser),
   UpdateServiceUserDescription(UpdateServiceUserDescription),
@@ -53,26 +70,43 @@ pub enum WriteRequest {
   AddUserToUserGroup(AddUserToUserGroup),
   RemoveUserFromUserGroup(RemoveUserFromUserGroup),
   SetUsersInUserGroup(SetUsersInUserGroup),
+  SetEveryoneUserGroup(SetEveryoneUserGroup),
 
   // ==== PERMISSIONS ====
+  UpdateUserAdmin(UpdateUserAdmin),
   UpdateUserBasePermissions(UpdateUserBasePermissions),
   UpdatePermissionOnResourceType(UpdatePermissionOnResourceType),
   UpdatePermissionOnTarget(UpdatePermissionOnTarget),
 
-  // ==== DESCRIPTION ====
-  UpdateDescription(UpdateDescription),
+  // ==== RESOURCE ====
+  UpdateResourceMeta(UpdateResourceMeta),
 
   // ==== SERVER ====
   CreateServer(CreateServer),
+  CopyServer(CopyServer),
   DeleteServer(DeleteServer),
   UpdateServer(UpdateServer),
   RenameServer(RenameServer),
   CreateNetwork(CreateNetwork),
-  DeleteNetwork(DeleteNetwork),
+  CreateTerminal(CreateTerminal),
+  DeleteTerminal(DeleteTerminal),
+  DeleteAllTerminals(DeleteAllTerminals),
+
+  // ==== STACK ====
+  CreateStack(CreateStack),
+  CopyStack(CopyStack),
+  DeleteStack(DeleteStack),
+  UpdateStack(UpdateStack),
+  RenameStack(RenameStack),
+  WriteStackFileContents(WriteStackFileContents),
+  RefreshStackCache(RefreshStackCache),
+  CreateStackWebhook(CreateStackWebhook),
+  DeleteStackWebhook(DeleteStackWebhook),
 
   // ==== DEPLOYMENT ====
   CreateDeployment(CreateDeployment),
   CopyDeployment(CopyDeployment),
+  CreateDeploymentFromContainer(CreateDeploymentFromContainer),
   DeleteDeployment(DeleteDeployment),
   UpdateDeployment(UpdateDeployment),
   RenameDeployment(RenameDeployment),
@@ -82,6 +116,8 @@ pub enum WriteRequest {
   CopyBuild(CopyBuild),
   DeleteBuild(DeleteBuild),
   UpdateBuild(UpdateBuild),
+  RenameBuild(RenameBuild),
+  WriteBuildFileContents(WriteBuildFileContents),
   RefreshBuildCache(RefreshBuildCache),
   CreateBuildWebhook(CreateBuildWebhook),
   DeleteBuildWebhook(DeleteBuildWebhook),
@@ -91,18 +127,14 @@ pub enum WriteRequest {
   CopyBuilder(CopyBuilder),
   DeleteBuilder(DeleteBuilder),
   UpdateBuilder(UpdateBuilder),
-
-  // ==== SERVER TEMPLATE ====
-  CreateServerTemplate(CreateServerTemplate),
-  CopyServerTemplate(CopyServerTemplate),
-  DeleteServerTemplate(DeleteServerTemplate),
-  UpdateServerTemplate(UpdateServerTemplate),
+  RenameBuilder(RenameBuilder),
 
   // ==== REPO ====
   CreateRepo(CreateRepo),
   CopyRepo(CopyRepo),
   DeleteRepo(DeleteRepo),
   UpdateRepo(UpdateRepo),
+  RenameRepo(RenameRepo),
   RefreshRepoCache(RefreshRepoCache),
   CreateRepoWebhook(CreateRepoWebhook),
   DeleteRepoWebhook(DeleteRepoWebhook),
@@ -112,42 +144,45 @@ pub enum WriteRequest {
   CopyAlerter(CopyAlerter),
   DeleteAlerter(DeleteAlerter),
   UpdateAlerter(UpdateAlerter),
+  RenameAlerter(RenameAlerter),
 
   // ==== PROCEDURE ====
   CreateProcedure(CreateProcedure),
   CopyProcedure(CopyProcedure),
   DeleteProcedure(DeleteProcedure),
   UpdateProcedure(UpdateProcedure),
+  RenameProcedure(RenameProcedure),
+
+  // ==== ACTION ====
+  CreateAction(CreateAction),
+  CopyAction(CopyAction),
+  DeleteAction(DeleteAction),
+  UpdateAction(UpdateAction),
+  RenameAction(RenameAction),
 
   // ==== SYNC ====
   CreateResourceSync(CreateResourceSync),
   CopyResourceSync(CopyResourceSync),
   DeleteResourceSync(DeleteResourceSync),
   UpdateResourceSync(UpdateResourceSync),
+  RenameResourceSync(RenameResourceSync),
+  WriteSyncFileContents(WriteSyncFileContents),
+  CommitSync(CommitSync),
   RefreshResourceSyncPending(RefreshResourceSyncPending),
   CreateSyncWebhook(CreateSyncWebhook),
   DeleteSyncWebhook(DeleteSyncWebhook),
 
-  // ==== STACK ====
-  CreateStack(CreateStack),
-  CopyStack(CopyStack),
-  DeleteStack(DeleteStack),
-  UpdateStack(UpdateStack),
-  RenameStack(RenameStack),
-  RefreshStackCache(RefreshStackCache),
-  CreateStackWebhook(CreateStackWebhook),
-  DeleteStackWebhook(DeleteStackWebhook),
-
   // ==== TAG ====
   CreateTag(CreateTag),
   DeleteTag(DeleteTag),
   RenameTag(RenameTag),
-  UpdateTagsOnResource(UpdateTagsOnResource),
+  UpdateTagColor(UpdateTagColor),
 
   // ==== VARIABLE ====
   CreateVariable(CreateVariable),
   UpdateVariableValue(UpdateVariableValue),
   UpdateVariableDescription(UpdateVariableDescription),
+  UpdateVariableIsSecret(UpdateVariableIsSecret),
   DeleteVariable(DeleteVariable),
 
   // ==== PROVIDERS ====
@@ -162,57 +197,60 @@ pub enum WriteRequest {
 pub fn router() -> Router {
   Router::new()
     .route("/", post(handler))
+    .route("/{variant}", post(variant_handler))
     .layer(middleware::from_fn(auth_request))
 }
 
+async fn variant_handler(
+  user: Extension<User>,
+  Path(Variant { variant }): Path<Variant>,
+  Json(params): Json<serde_json::Value>,
+) -> serror::Result<axum::response::Response> {
+  let req: WriteRequest = serde_json::from_value(json!({
+    "type": variant,
+    "params": params,
+  }))?;
+  handler(user, Json(req)).await
+}
+
 async fn handler(
   Extension(user): Extension<User>,
   Json(request): Json<WriteRequest>,
-) -> serror::Result<(TypedHeader<ContentType>, String)> {
+) -> serror::Result<axum::response::Response> {
   let req_id = Uuid::new_v4();
 
   let res = tokio::spawn(task(req_id, request, user))
     .await
     .context("failure in spawned task");
 
-  if let Err(e) = &res {
-    warn!("/write request {req_id} spawn error: {e:#}");
-  }
-
-  Ok((TypedHeader(ContentType::json()), res??))
+  res?
 }
 
 #[instrument(
   name = "WriteRequest",
   skip(user, request),
-  fields(user_id = user.id, request = format!("{:?}", request.extract_variant()))
+  fields(
+    user_id = user.id,
+    request = format!("{:?}", request.extract_variant())
+  )
 )]
 async fn task(
   req_id: Uuid,
   request: WriteRequest,
   user: User,
-) -> anyhow::Result<String> {
+) -> serror::Result<axum::response::Response> {
   info!("/write request | user: {}", user.username);
 
   let timer = Instant::now();
 
-  let res =
-    State
-      .resolve_request(request, user)
-      .await
-      .map_err(|e| match e {
-        resolver_api::Error::Serialization(e) => {
-          anyhow!("{e:?}").context("response serialization error")
-        }
-        resolver_api::Error::Inner(e) => e,
-      });
+  let res = request.resolve(&WriteArgs { user }).await;
 
   if let Err(e) = &res {
-    warn!("/write request {req_id} error: {e:#}");
+    warn!("/write request {req_id} error: {:#}", e.error);
   }
 
   let elapsed = timer.elapsed();
   debug!("/write request {req_id} | resolve time: {elapsed:?}");
 
-  res
+  res.map(|res| res.0)
 }

bin/core/src/api/write/permissions.rs

@@ -1,57 +1,97 @@
 use std::str::FromStr;
 
-use anyhow::{anyhow, Context};
-use monitor_client::{
-  api::write::{
-    UpdatePermissionOnResourceType,
-    UpdatePermissionOnResourceTypeResponse, UpdatePermissionOnTarget,
-    UpdatePermissionOnTargetResponse, UpdateUserBasePermissions,
-    UpdateUserBasePermissionsResponse,
-  },
-  entities::{
-    permission::{UserTarget, UserTargetVariant},
-    update::{ResourceTarget, ResourceTargetVariant},
-    user::User,
-  },
-};
-use mungos::{
+use anyhow::{Context, anyhow};
+use database::mungos::{
   by_id::{find_one_by_id, update_one_by_id},
   mongodb::{
-    bson::{doc, oid::ObjectId, Document},
+    bson::{Document, doc, oid::ObjectId, to_bson},
     options::UpdateOptions,
   },
 };
+use komodo_client::{
+  api::write::*,
+  entities::{
+    ResourceTarget, ResourceTargetVariant,
+    permission::{UserTarget, UserTargetVariant},
+  },
+};
 use resolver_api::Resolve;
 
-use crate::{
-  helpers::query::get_user,
-  state::{db_client, State},
-};
+use crate::{helpers::query::get_user, state::db_client};
 
-impl Resolve<UpdateUserBasePermissions, User> for State {
-  #[instrument(name = "UpdateUserBasePermissions", skip(self, admin))]
+use super::WriteArgs;
+
+impl Resolve<WriteArgs> for UpdateUserAdmin {
+  #[instrument(name = "UpdateUserAdmin", skip(super_admin))]
   async fn resolve(
-    &self,
-    UpdateUserBasePermissions {
+    self,
+    WriteArgs { user: super_admin }: &WriteArgs,
+  ) -> serror::Result<UpdateUserAdminResponse> {
+    if !super_admin.super_admin {
+      return Err(
+        anyhow!("Only super admins can call this method.").into(),
+      );
+    }
+    let user = find_one_by_id(&db_client().users, &self.user_id)
+      .await
+      .context("failed to query mongo for user")?
+      .context("did not find user with given id")?;
+
+    if !user.enabled {
+      return Err(
+        anyhow!("User is disabled. Enable user first.").into(),
+      );
+    }
+
+    if user.super_admin {
+      return Err(anyhow!("Cannot update other super admins").into());
+    }
+
+    update_one_by_id(
+      &db_client().users,
+      &self.user_id,
+      doc! { "$set": { "admin": self.admin } },
+      None,
+    )
+    .await?;
+
+    Ok(UpdateUserAdminResponse {})
+  }
+}
+
+impl Resolve<WriteArgs> for UpdateUserBasePermissions {
+  #[instrument(name = "UpdateUserBasePermissions", skip(admin))]
+  async fn resolve(
+    self,
+    WriteArgs { user: admin }: &WriteArgs,
+  ) -> serror::Result<UpdateUserBasePermissionsResponse> {
+    if !admin.admin {
+      return Err(anyhow!("this method is admin only").into());
+    }
+
+    let UpdateUserBasePermissions {
       user_id,
       enabled,
       create_servers,
       create_builds,
-    }: UpdateUserBasePermissions,
-    admin: User,
-  ) -> anyhow::Result<UpdateUserBasePermissionsResponse> {
-    if !admin.admin {
-      return Err(anyhow!("this method is admin only"));
-    }
+    } = self;
 
-    let user = find_one_by_id(&db_client().await.users, &user_id)
+    let user = find_one_by_id(&db_client().users, &user_id)
       .await
       .context("failed to query mongo for user")?
       .context("did not find user with given id")?;
-    if user.admin {
+    if user.super_admin {
+      return Err(
+        anyhow!(
+          "Cannot use this method to update super admins permissions"
+        )
+        .into(),
+      );
+    }
+    if user.admin && !admin.super_admin {
       return Err(anyhow!(
-        "cannot use this method to update other admins permissions"
-      ));
+        "Only super admins can use this method to update other admins permissions"
+      ).into());
     }
     let mut update_doc = Document::new();
     if let Some(enabled) = enabled {
@@ -65,9 +105,9 @@ impl Resolve<UpdateUserBasePermissions, User> for State {
     }
 
     update_one_by_id(
-      &db_client().await.users,
+      &db_client().users,
       &user_id,
-      mungos::update::Update::Set(update_doc),
+      database::mungos::update::Update::Set(update_doc),
       None,
     )
     .await?;
@@ -76,34 +116,35 @@ impl Resolve<UpdateUserBasePermissions, User> for State {
   }
 }
 
-impl Resolve<UpdatePermissionOnResourceType, User> for State {
-  #[instrument(
-    name = "UpdatePermissionOnResourceType",
-    skip(self, admin)
-  )]
+impl Resolve<WriteArgs> for UpdatePermissionOnResourceType {
+  #[instrument(name = "UpdatePermissionOnResourceType", skip(admin))]
   async fn resolve(
-    &self,
-    UpdatePermissionOnResourceType {
+    self,
+    WriteArgs { user: admin }: &WriteArgs,
+  ) -> serror::Result<UpdatePermissionOnResourceTypeResponse> {
+    if !admin.admin {
+      return Err(anyhow!("this method is admin only").into());
+    }
+
+    let Self {
       user_target,
       resource_type,
       permission,
-    }: UpdatePermissionOnResourceType,
-    admin: User,
-  ) -> anyhow::Result<UpdatePermissionOnResourceTypeResponse> {
-    if !admin.admin {
-      return Err(anyhow!("this method is admin only"));
-    }
+    } = self;
 
     // Some extra checks if user target is an actual User
     if let UserTarget::User(user_id) = &user_target {
       let user = get_user(user_id).await?;
       if user.admin {
-        return Err(anyhow!(
+        return Err(
+          anyhow!(
           "cannot use this method to update other admins permissions"
-        ));
+        )
+          .into(),
+        );
       }
       if !user.enabled {
-        return Err(anyhow!("user not enabled"));
+        return Err(anyhow!("user not enabled").into());
       }
     }
 
@@ -112,29 +153,29 @@ impl Resolve<UpdatePermissionOnResourceType, User> for State {
 
     let id = ObjectId::from_str(&user_target_id)
       .context("id is not ObjectId")?;
-    let field = format!("all.{resource_type}");
     let filter = doc! { "_id": id };
-    let update = doc! { "$set": { &field: permission.as_ref() } };
+    let field = format!("all.{resource_type}");
+    let set =
+      to_bson(&permission).context("permission is not Bson")?;
+    let update = doc! { "$set": { &field: &set } };
 
     match user_target_variant {
       UserTargetVariant::User => {
         db_client()
-          .await
           .users
           .update_one(filter, update)
           .await
           .with_context(|| {
-            format!("failed to set {field}: {permission} on db")
+            format!("failed to set {field}: {set} on db")
           })?;
       }
       UserTargetVariant::UserGroup => {
         db_client()
-          .await
           .user_groups
           .update_one(filter, update)
           .await
           .with_context(|| {
-            format!("failed to set {field}: {permission} on db")
+            format!("failed to set {field}: {set} on db")
           })?;
       }
     }
@@ -143,31 +184,35 @@ impl Resolve<UpdatePermissionOnResourceType, User> for State {
   }
 }
 
-impl Resolve<UpdatePermissionOnTarget, User> for State {
-  #[instrument(name = "UpdatePermissionOnTarget", skip(self, admin))]
+impl Resolve<WriteArgs> for UpdatePermissionOnTarget {
+  #[instrument(name = "UpdatePermissionOnTarget", skip(admin))]
   async fn resolve(
-    &self,
-    UpdatePermissionOnTarget {
+    self,
+    WriteArgs { user: admin }: &WriteArgs,
+  ) -> serror::Result<UpdatePermissionOnTargetResponse> {
+    if !admin.admin {
+      return Err(anyhow!("this method is admin only").into());
+    }
+
+    let UpdatePermissionOnTarget {
       user_target,
       resource_target,
       permission,
-    }: UpdatePermissionOnTarget,
-    admin: User,
-  ) -> anyhow::Result<UpdatePermissionOnTargetResponse> {
-    if !admin.admin {
-      return Err(anyhow!("this method is admin only"));
-    }
+    } = self;
 
-    // Some extra checks if user target is an actual User
+    // Some extra checks relevant if user target is an actual User
     if let UserTarget::User(user_id) = &user_target {
       let user = get_user(user_id).await?;
-      if user.admin {
-        return Err(anyhow!(
-          "cannot use this method to update other admins permissions"
-        ));
-      }
       if !user.enabled {
-        return Err(anyhow!("user not enabled"));
+        return Err(anyhow!("user not enabled").into());
+      }
+      if user.admin {
+        return Err(
+          anyhow!(
+          "cannot use this method to update other admins permissions"
+        )
+          .into(),
+        );
       }
     }
 
@@ -180,8 +225,10 @@ impl Resolve<UpdatePermissionOnTarget, User> for State {
     let (user_target_variant, resource_variant) =
       (user_target_variant.as_ref(), resource_variant.as_ref());
 
+    let specific = to_bson(&permission.specific)
+      .context("permission.specific is not valid Bson")?;
+
     db_client()
-      .await
       .permissions
       .update_one(
         doc! {
@@ -196,7 +243,8 @@ impl Resolve<UpdatePermissionOnTarget, User> for State {
             "user_target.id": user_target_id,
             "resource_target.type": resource_variant,
             "resource_target.id": resource_id,
-            "level": permission.as_ref(),
+            "level": permission.level.as_ref(),
+            "specific": specific
           }
         },
       )
@@ -210,7 +258,7 @@ impl Resolve<UpdatePermissionOnTarget, User> for State {
 /// checks if inner id is actually a `name`, and replaces it with id if so.
 async fn extract_user_target_with_validation(
   user_target: &UserTarget,
-) -> anyhow::Result<(UserTargetVariant, String)> {
+) -> serror::Result<(UserTargetVariant, String)> {
   match user_target {
     UserTarget::User(ident) => {
       let filter = match ObjectId::from_str(ident) {
@@ -218,7 +266,6 @@ async fn extract_user_target_with_validation(
         Err(_) => doc! { "username": ident },
       };
       let id = db_client()
-        .await
         .users
         .find_one(filter)
         .await
@@ -233,7 +280,6 @@ async fn extract_user_target_with_validation(
         Err(_) => doc! { "name": ident },
       };
       let id = db_client()
-        .await
         .user_groups
         .find_one(filter)
         .await
@@ -248,7 +294,7 @@ async fn extract_user_target_with_validation(
 /// checks if inner id is actually a `name`, and replaces it with id if so.
 async fn extract_resource_target_with_validation(
   resource_target: &ResourceTarget,
-) -> anyhow::Result<(ResourceTargetVariant, String)> {
+) -> serror::Result<(ResourceTargetVariant, String)> {
   match resource_target {
     ResourceTarget::System(_) => {
       let res = resource_target.extract_variant_id();
@@ -260,7 +306,6 @@ async fn extract_resource_target_with_validation(
         Err(_) => doc! { "name": ident },
       };
       let id = db_client()
-        .await
         .builds
         .find_one(filter)
         .await
@@ -275,7 +320,6 @@ async fn extract_resource_target_with_validation(
         Err(_) => doc! { "name": ident },
       };
       let id = db_client()
-        .await
         .builders
         .find_one(filter)
         .await
@@ -290,7 +334,6 @@ async fn extract_resource_target_with_validation(
         Err(_) => doc! { "name": ident },
       };
       let id = db_client()
-        .await
         .deployments
         .find_one(filter)
         .await
@@ -305,7 +348,6 @@ async fn extract_resource_target_with_validation(
         Err(_) => doc! { "name": ident },
       };
       let id = db_client()
-        .await
         .servers
         .find_one(filter)
         .await
@@ -320,7 +362,6 @@ async fn extract_resource_target_with_validation(
         Err(_) => doc! { "name": ident },
       };
       let id = db_client()
-        .await
         .repos
         .find_one(filter)
         .await
@@ -335,7 +376,6 @@ async fn extract_resource_target_with_validation(
         Err(_) => doc! { "name": ident },
       };
       let id = db_client()
-        .await
         .alerters
         .find_one(filter)
         .await
@@ -350,7 +390,6 @@ async fn extract_resource_target_with_validation(
         Err(_) => doc! { "name": ident },
       };
       let id = db_client()
-        .await
         .procedures
         .find_one(filter)
         .await
@@ -359,20 +398,19 @@ async fn extract_resource_target_with_validation(
         .id;
       Ok((ResourceTargetVariant::Procedure, id))
     }
-    ResourceTarget::ServerTemplate(ident) => {
+    ResourceTarget::Action(ident) => {
       let filter = match ObjectId::from_str(ident) {
         Ok(id) => doc! { "_id": id },
         Err(_) => doc! { "name": ident },
       };
       let id = db_client()
-        .await
-        .server_templates
+        .actions
         .find_one(filter)
         .await
-        .context("failed to query db for server templates")?
-        .context("no matching server template found")?
+        .context("failed to query db for actions")?
+        .context("no matching action found")?
         .id;
-      Ok((ResourceTargetVariant::ServerTemplate, id))
+      Ok((ResourceTargetVariant::Action, id))
     }
     ResourceTarget::ResourceSync(ident) => {
       let filter = match ObjectId::from_str(ident) {
@@ -380,7 +418,6 @@ async fn extract_resource_target_with_validation(
         Err(_) => doc! { "name": ident },
       };
       let id = db_client()
-        .await
         .resource_syncs
         .find_one(filter)
         .await
@@ -395,7 +432,6 @@ async fn extract_resource_target_with_validation(
         Err(_) => doc! { "name": ident },
       };
       let id = db_client()
-        .await
         .stacks
         .find_one(filter)
         .await

bin/core/src/api/write/procedure.rs

@@ -1,60 +1,75 @@
-use monitor_client::{
+use komodo_client::{
   api::write::*,
   entities::{
-    permission::PermissionLevel, procedure::Procedure, user::User,
+    permission::PermissionLevel, procedure::Procedure, update::Update,
   },
 };
 use resolver_api::Resolve;
 
-use crate::{resource, state::State};
+use crate::{permission::get_check_permissions, resource};
 
-impl Resolve<CreateProcedure, User> for State {
-  #[instrument(name = "CreateProcedure", skip(self, user))]
+use super::WriteArgs;
+
+impl Resolve<WriteArgs> for CreateProcedure {
+  #[instrument(name = "CreateProcedure", skip(user))]
   async fn resolve(
-    &self,
-    CreateProcedure { name, config }: CreateProcedure,
-    user: User,
-  ) -> anyhow::Result<CreateProcedureResponse> {
-    resource::create::<Procedure>(&name, config, &user).await
+    self,
+    WriteArgs { user }: &WriteArgs,
+  ) -> serror::Result<CreateProcedureResponse> {
+    resource::create::<Procedure>(&self.name, self.config, user).await
   }
 }
 
-impl Resolve<CopyProcedure, User> for State {
-  #[instrument(name = "CopyProcedure", skip(self, user))]
+impl Resolve<WriteArgs> for CopyProcedure {
+  #[instrument(name = "CopyProcedure", skip(user))]
   async fn resolve(
-    &self,
-    CopyProcedure { name, id }: CopyProcedure,
-    user: User,
-  ) -> anyhow::Result<CopyProcedureResponse> {
+    self,
+    WriteArgs { user }: &WriteArgs,
+  ) -> serror::Result<CopyProcedureResponse> {
     let Procedure { config, .. } =
-      resource::get_check_permissions::<Procedure>(
-        &id,
-        &user,
-        PermissionLevel::Write,
+      get_check_permissions::<Procedure>(
+        &self.id,
+        user,
+        PermissionLevel::Write.into(),
       )
       .await?;
-    resource::create::<Procedure>(&name, config.into(), &user).await
+    resource::create::<Procedure>(&self.name, config.into(), user)
+      .await
   }
 }
 
-impl Resolve<UpdateProcedure, User> for State {
-  #[instrument(name = "UpdateProcedure", skip(self, user))]
+impl Resolve<WriteArgs> for UpdateProcedure {
+  #[instrument(name = "UpdateProcedure", skip(user))]
   async fn resolve(
-    &self,
-    UpdateProcedure { id, config }: UpdateProcedure,
-    user: User,
-  ) -> anyhow::Result<UpdateProcedureResponse> {
-    resource::update::<Procedure>(&id, config, &user).await
+    self,
+    WriteArgs { user }: &WriteArgs,
+  ) -> serror::Result<UpdateProcedureResponse> {
+    Ok(
+      resource::update::<Procedure>(&self.id, self.config, user)
+        .await?,
+    )
   }
 }
 
-impl Resolve<DeleteProcedure, User> for State {
-  #[instrument(name = "DeleteProcedure", skip(self, user))]
+impl Resolve<WriteArgs> for RenameProcedure {
+  #[instrument(name = "RenameProcedure", skip(user))]
   async fn resolve(
-    &self,
-    DeleteProcedure { id }: DeleteProcedure,
-    user: User,
-  ) -> anyhow::Result<DeleteProcedureResponse> {
-    resource::delete::<Procedure>(&id, &user).await
+    self,
+    WriteArgs { user }: &WriteArgs,
+  ) -> serror::Result<Update> {
+    Ok(
+      resource::rename::<Procedure>(&self.id, &self.name, user)
+        .await?,
+    )
+  }
+}
+
+impl Resolve<WriteArgs> for DeleteProcedure {
+  #[instrument(name = "DeleteProcedure", skip(args))]
+  async fn resolve(
+    self,
+    args: &WriteArgs,
+  ) -> serror::Result<DeleteProcedureResponse> {
+    Ok(resource::delete::<Procedure>(&self.id, args).await?)
   }
 }

bin/core/src/api/write/provider.rs

@@ -1,54 +1,53 @@
-use anyhow::{anyhow, Context};
-use monitor_client::{
-  api::write::*,
-  entities::{
-    provider::{DockerRegistryAccount, GitProviderAccount},
-    update::ResourceTarget,
-    user::User,
-    Operation,
-  },
-};
-use mungos::{
+use anyhow::{Context, anyhow};
+use database::mungos::{
   by_id::{delete_one_by_id, find_one_by_id, update_one_by_id},
   mongodb::bson::{doc, to_document},
 };
+use komodo_client::{
+  api::write::*,
+  entities::{
+    Operation, ResourceTarget,
+    provider::{DockerRegistryAccount, GitProviderAccount},
+  },
+};
 use resolver_api::Resolve;
 
 use crate::{
   helpers::update::{add_update, make_update},
-  state::{db_client, State},
+  state::db_client,
 };
 
-impl Resolve<CreateGitProviderAccount, User> for State {
+use super::WriteArgs;
+
+impl Resolve<WriteArgs> for CreateGitProviderAccount {
   async fn resolve(
-    &self,
-    CreateGitProviderAccount { account }: CreateGitProviderAccount,
-    user: User,
-  ) -> anyhow::Result<CreateGitProviderAccountResponse> {
+    self,
+    WriteArgs { user }: &WriteArgs,
+  ) -> serror::Result<CreateGitProviderAccountResponse> {
     if !user.admin {
-      return Err(anyhow!(
-        "only admins can create git provider accounts"
-      ));
+      return Err(
+        anyhow!("only admins can create git provider accounts")
+          .into(),
+      );
     }
 
-    let mut account: GitProviderAccount = account.into();
+    let mut account: GitProviderAccount = self.account.into();
 
     if account.domain.is_empty() {
-      return Err(anyhow!("domain cannot be empty string."));
+      return Err(anyhow!("domain cannot be empty string.").into());
     }
 
     if account.username.is_empty() {
-      return Err(anyhow!("username cannot be empty string."));
+      return Err(anyhow!("username cannot be empty string.").into());
     }
 
     let mut update = make_update(
       ResourceTarget::system(),
       Operation::CreateGitProviderAccount,
-      &user,
+      user,
     );
 
     account.id = db_client()
-      .await
       .git_accounts
       .insert_one(&account)
       .await
@@ -79,62 +78,63 @@ impl Resolve<CreateGitProviderAccount, User> for State {
   }
 }
 
-impl Resolve<UpdateGitProviderAccount, User> for State {
+impl Resolve<WriteArgs> for UpdateGitProviderAccount {
   async fn resolve(
-    &self,
-    UpdateGitProviderAccount { id, mut account }: UpdateGitProviderAccount,
-    user: User,
-  ) -> anyhow::Result<UpdateGitProviderAccountResponse> {
+    mut self,
+    WriteArgs { user }: &WriteArgs,
+  ) -> serror::Result<UpdateGitProviderAccountResponse> {
     if !user.admin {
-      return Err(anyhow!(
-        "only admins can update git provider accounts"
-      ));
+      return Err(
+        anyhow!("only admins can update git provider accounts")
+          .into(),
+      );
     }
 
-    if let Some(domain) = &account.domain {
-      if domain.is_empty() {
-        return Err(anyhow!(
-          "cannot update git provider with empty domain"
-        ));
-      }
+    if let Some(domain) = &self.account.domain
+      && domain.is_empty()
+    {
+      return Err(
+        anyhow!("cannot update git provider with empty domain")
+          .into(),
+      );
     }
 
-    if let Some(username) = &account.username {
-      if username.is_empty() {
-        return Err(anyhow!(
-          "cannot update git provider with empty username"
-        ));
-      }
+    if let Some(username) = &self.account.username
+      && username.is_empty()
+    {
+      return Err(
+        anyhow!("cannot update git provider with empty username")
+          .into(),
+      );
     }
 
     // Ensure update does not change id
-    account.id = None;
+    self.account.id = None;
 
     let mut update = make_update(
       ResourceTarget::system(),
       Operation::UpdateGitProviderAccount,
-      &user,
+      user,
     );
 
-    let account = to_document(&account).context(
+    let account = to_document(&self.account).context(
       "failed to serialize partial git provider account to bson",
     )?;
-    let db = db_client().await;
+    let db = db_client();
     update_one_by_id(
       &db.git_accounts,
-      &id,
+      &self.id,
       doc! { "$set": account },
       None,
     )
     .await
     .context("failed to update git provider account on db")?;
 
-    let Some(account) =
-      find_one_by_id(&db.git_accounts, &id)
-        .await
-        .context("failed to query db for git accounts")?
+    let Some(account) = find_one_by_id(&db.git_accounts, &self.id)
+      .await
+      .context("failed to query db for git accounts")?
     else {
-      return Err(anyhow!("no account found with given id"));
+      return Err(anyhow!("no account found with given id").into());
     };
 
     update.push_simple_log(
@@ -158,33 +158,32 @@ impl Resolve<UpdateGitProviderAccount, User> for State {
   }
 }
 
-impl Resolve<DeleteGitProviderAccount, User> for State {
+impl Resolve<WriteArgs> for DeleteGitProviderAccount {
   async fn resolve(
-    &self,
-    DeleteGitProviderAccount { id }: DeleteGitProviderAccount,
-    user: User,
-  ) -> anyhow::Result<DeleteGitProviderAccountResponse> {
+    self,
+    WriteArgs { user }: &WriteArgs,
+  ) -> serror::Result<DeleteGitProviderAccountResponse> {
     if !user.admin {
-      return Err(anyhow!(
-        "only admins can delete git provider accounts"
-      ));
+      return Err(
+        anyhow!("only admins can delete git provider accounts")
+          .into(),
+      );
     }
 
     let mut update = make_update(
       ResourceTarget::system(),
       Operation::UpdateGitProviderAccount,
-      &user,
+      user,
     );
 
-    let db = db_client().await;
-    let Some(account) =
-      find_one_by_id(&db.git_accounts, &id)
-        .await
-        .context("failed to query db for git accounts")?
+    let db = db_client();
+    let Some(account) = find_one_by_id(&db.git_accounts, &self.id)
+      .await
+      .context("failed to query db for git accounts")?
     else {
-      return Err(anyhow!("no account found with given id"));
+      return Err(anyhow!("no account found with given id").into());
     };
-    delete_one_by_id(&db.git_accounts, &id, None)
+    delete_one_by_id(&db.git_accounts, &self.id, None)
       .await
       .context("failed to delete git account on db")?;
 
@@ -209,36 +208,37 @@ impl Resolve<DeleteGitProviderAccount, User> for State {
   }
 }
 
-impl Resolve<CreateDockerRegistryAccount, User> for State {
+impl Resolve<WriteArgs> for CreateDockerRegistryAccount {
   async fn resolve(
-    &self,
-    CreateDockerRegistryAccount { account }: CreateDockerRegistryAccount,
-    user: User,
-  ) -> anyhow::Result<CreateDockerRegistryAccountResponse> {
+    self,
+    WriteArgs { user }: &WriteArgs,
+  ) -> serror::Result<CreateDockerRegistryAccountResponse> {
     if !user.admin {
-      return Err(anyhow!(
-        "only admins can create docker registry account accounts"
-      ));
+      return Err(
+        anyhow!(
+          "only admins can create docker registry account accounts"
+        )
+        .into(),
+      );
     }
 
-    let mut account: DockerRegistryAccount = account.into();
+    let mut account: DockerRegistryAccount = self.account.into();
 
     if account.domain.is_empty() {
-      return Err(anyhow!("domain cannot be empty string."));
+      return Err(anyhow!("domain cannot be empty string.").into());
     }
 
     if account.username.is_empty() {
-      return Err(anyhow!("username cannot be empty string."));
+      return Err(anyhow!("username cannot be empty string.").into());
     }
 
     let mut update = make_update(
       ResourceTarget::system(),
       Operation::CreateDockerRegistryAccount,
-      &user,
+      user,
     );
 
     account.id = db_client()
-      .await
       .registry_accounts
       .insert_one(&account)
       .await
@@ -271,50 +271,56 @@ impl Resolve<CreateDockerRegistryAccount, User> for State {
   }
 }
 
-impl Resolve<UpdateDockerRegistryAccount, User> for State {
+impl Resolve<WriteArgs> for UpdateDockerRegistryAccount {
   async fn resolve(
-    &self,
-    UpdateDockerRegistryAccount { id, mut account }: UpdateDockerRegistryAccount,
-    user: User,
-  ) -> anyhow::Result<UpdateDockerRegistryAccountResponse> {
+    mut self,
+    WriteArgs { user }: &WriteArgs,
+  ) -> serror::Result<UpdateDockerRegistryAccountResponse> {
     if !user.admin {
-      return Err(anyhow!(
-        "only admins can update docker registry accounts"
-      ));
+      return Err(
+        anyhow!("only admins can update docker registry accounts")
+          .into(),
+      );
     }
 
-    if let Some(domain) = &account.domain {
-      if domain.is_empty() {
-        return Err(anyhow!(
+    if let Some(domain) = &self.account.domain
+      && domain.is_empty()
+    {
+      return Err(
+        anyhow!(
           "cannot update docker registry account with empty domain"
-        ));
-      }
+        )
+        .into(),
+      );
     }
 
-    if let Some(username) = &account.username {
-      if username.is_empty() {
-        return Err(anyhow!(
+    if let Some(username) = &self.account.username
+      && username.is_empty()
+    {
+      return Err(
+        anyhow!(
           "cannot update docker registry account with empty username"
-        ));
-      }
+        )
+        .into(),
+      );
     }
 
-    account.id = None;
+    self.account.id = None;
 
     let mut update = make_update(
       ResourceTarget::system(),
       Operation::UpdateDockerRegistryAccount,
-      &user,
+      user,
     );
 
-    let account = to_document(&account).context(
+    let account = to_document(&self.account).context(
       "failed to serialize partial docker registry account account to bson",
     )?;
 
-    let db = db_client().await;
+    let db = db_client();
     update_one_by_id(
       &db.registry_accounts,
-      &id,
+      &self.id,
       doc! { "$set": account },
       None,
     )
@@ -323,11 +329,12 @@ impl Resolve<UpdateDockerRegistryAccount, User> for State {
       "failed to update docker registry account account on db",
     )?;
 
-    let Some(account) = find_one_by_id(&db.registry_accounts, &id)
-      .await
-      .context("failed to query db for registry accounts")?
+    let Some(account) =
+      find_one_by_id(&db.registry_accounts, &self.id)
+        .await
+        .context("failed to query db for registry accounts")?
     else {
-      return Err(anyhow!("no account found with given id"));
+      return Err(anyhow!("no account found with given id").into());
     };
 
     update.push_simple_log(
@@ -351,32 +358,33 @@ impl Resolve<UpdateDockerRegistryAccount, User> for State {
   }
 }
 
-impl Resolve<DeleteDockerRegistryAccount, User> for State {
+impl Resolve<WriteArgs> for DeleteDockerRegistryAccount {
   async fn resolve(
-    &self,
-    DeleteDockerRegistryAccount { id }: DeleteDockerRegistryAccount,
-    user: User,
-  ) -> anyhow::Result<DeleteDockerRegistryAccountResponse> {
+    self,
+    WriteArgs { user }: &WriteArgs,
+  ) -> serror::Result<DeleteDockerRegistryAccountResponse> {
     if !user.admin {
-      return Err(anyhow!(
-        "only admins can delete docker registry accounts"
-      ));
+      return Err(
+        anyhow!("only admins can delete docker registry accounts")
+          .into(),
+      );
     }
 
     let mut update = make_update(
       ResourceTarget::system(),
       Operation::UpdateDockerRegistryAccount,
-      &user,
+      user,
     );
 
-    let db = db_client().await;
-    let Some(account) = find_one_by_id(&db.registry_accounts, &id)
-      .await
-      .context("failed to query db for git accounts")?
+    let db = db_client();
+    let Some(account) =
+      find_one_by_id(&db.registry_accounts, &self.id)
+        .await
+        .context("failed to query db for git accounts")?
     else {
-      return Err(anyhow!("no account found with given id"));
+      return Err(anyhow!("no account found with given id").into());
     };
-    delete_one_by_id(&db.registry_accounts, &id, None)
+    delete_one_by_id(&db.registry_accounts, &self.id, None)
       .await
       .context("failed to delete registry account on db")?;
 

bin/core/src/api/write/repo.rs

@@ -1,157 +1,226 @@
-use anyhow::{anyhow, Context};
-use mongo_indexed::doc;
-use monitor_client::{
+use anyhow::{Context, anyhow};
+use database::mongo_indexed::doc;
+use database::mungos::{
+  by_id::update_one_by_id, mongodb::bson::to_document,
+};
+use formatting::format_serror;
+use komodo_client::{
   api::write::*,
   entities::{
+    NoData, Operation, RepoExecutionArgs,
     config::core::CoreConfig,
+    komodo_timestamp,
     permission::PermissionLevel,
     repo::{PartialRepoConfig, Repo, RepoInfo},
-    user::User,
-    CloneArgs, NoData,
+    server::Server,
+    to_path_compatible_name,
+    update::{Log, Update},
   },
 };
-use mungos::mongodb::bson::to_document;
 use octorust::types::{
   ReposCreateWebhookRequest, ReposCreateWebhookRequestConfig,
 };
+use periphery_client::api;
 use resolver_api::Resolve;
 
 use crate::{
   config::core_config,
-  helpers::{git_token, random_string},
+  helpers::{
+    git_token, periphery_client,
+    update::{add_update, make_update},
+  },
+  permission::get_check_permissions,
   resource,
-  state::{db_client, github_client, State},
+  state::{action_states, db_client, github_client},
 };
 
-impl Resolve<CreateRepo, User> for State {
-  #[instrument(name = "CreateRepo", skip(self, user))]
+use super::WriteArgs;
+
+impl Resolve<WriteArgs> for CreateRepo {
+  #[instrument(name = "CreateRepo", skip(user))]
   async fn resolve(
-    &self,
-    CreateRepo { name, config }: CreateRepo,
-    user: User,
-  ) -> anyhow::Result<Repo> {
-    resource::create::<Repo>(&name, config, &user).await
+    self,
+    WriteArgs { user }: &WriteArgs,
+  ) -> serror::Result<Repo> {
+    resource::create::<Repo>(&self.name, self.config, user).await
   }
 }
 
-impl Resolve<CopyRepo, User> for State {
-  #[instrument(name = "CopyRepo", skip(self, user))]
+impl Resolve<WriteArgs> for CopyRepo {
+  #[instrument(name = "CopyRepo", skip(user))]
   async fn resolve(
-    &self,
-    CopyRepo { name, id }: CopyRepo,
-    user: User,
-  ) -> anyhow::Result<Repo> {
-    let Repo { config, .. } =
-      resource::get_check_permissions::<Repo>(
-        &id,
-        &user,
-        PermissionLevel::Write,
-      )
-      .await?;
-    resource::create::<Repo>(&name, config.into(), &user).await
+    self,
+    WriteArgs { user }: &WriteArgs,
+  ) -> serror::Result<Repo> {
+    let Repo { config, .. } = get_check_permissions::<Repo>(
+      &self.id,
+      user,
+      PermissionLevel::Read.into(),
+    )
+    .await?;
+    resource::create::<Repo>(&self.name, config.into(), user).await
   }
 }
 
-impl Resolve<DeleteRepo, User> for State {
-  #[instrument(name = "DeleteRepo", skip(self, user))]
-  async fn resolve(
-    &self,
-    DeleteRepo { id }: DeleteRepo,
-    user: User,
-  ) -> anyhow::Result<Repo> {
-    resource::delete::<Repo>(&id, &user).await
+impl Resolve<WriteArgs> for DeleteRepo {
+  #[instrument(name = "DeleteRepo", skip(args))]
+  async fn resolve(self, args: &WriteArgs) -> serror::Result<Repo> {
+    Ok(resource::delete::<Repo>(&self.id, args).await?)
   }
 }
 
-impl Resolve<UpdateRepo, User> for State {
-  #[instrument(name = "UpdateRepo", skip(self, user))]
+impl Resolve<WriteArgs> for UpdateRepo {
+  #[instrument(name = "UpdateRepo", skip(user))]
   async fn resolve(
-    &self,
-    UpdateRepo { id, config }: UpdateRepo,
-    user: User,
-  ) -> anyhow::Result<Repo> {
-    resource::update::<Repo>(&id, config, &user).await
+    self,
+    WriteArgs { user }: &WriteArgs,
+  ) -> serror::Result<Repo> {
+    Ok(resource::update::<Repo>(&self.id, self.config, user).await?)
   }
 }
 
-impl Resolve<RefreshRepoCache, User> for State {
-  #[instrument(
-    name = "RefreshRepoCache",
-    level = "debug",
-    skip(self, user)
-  )]
+impl Resolve<WriteArgs> for RenameRepo {
+  #[instrument(name = "RenameRepo", skip(user))]
   async fn resolve(
-    &self,
-    RefreshRepoCache { repo }: RefreshRepoCache,
-    user: User,
-  ) -> anyhow::Result<NoData> {
-    // Even though this is a write request, this doesn't change any config. Anyone that can execute the
-    // repo should be able to do this.
-    let repo = resource::get_check_permissions::<Repo>(
-      &repo,
-      &user,
-      PermissionLevel::Execute,
+    self,
+    WriteArgs { user }: &WriteArgs,
+  ) -> serror::Result<Update> {
+    let repo = get_check_permissions::<Repo>(
+      &self.id,
+      user,
+      PermissionLevel::Write.into(),
     )
     .await?;
 
-    if repo.config.repo.is_empty() {
+    if repo.config.server_id.is_empty()
+      || !repo.config.path.is_empty()
+    {
+      return Ok(
+        resource::rename::<Repo>(&repo.id, &self.name, user).await?,
+      );
+    }
+
+    // get the action state for the repo (or insert default).
+    let action_state =
+      action_states().repo.get_or_insert_default(&repo.id).await;
+
+    // Will check to ensure repo not already busy before updating, and return Err if so.
+    // The returned guard will set the action state back to default when dropped.
+    let _action_guard =
+      action_state.update(|state| state.renaming = true)?;
+
+    let name = to_path_compatible_name(&self.name);
+
+    let mut update = make_update(&repo, Operation::RenameRepo, user);
+
+    update_one_by_id(
+      &db_client().repos,
+      &repo.id,
+      database::mungos::update::Update::Set(
+        doc! { "name": &name, "updated_at": komodo_timestamp() },
+      ),
+      None,
+    )
+    .await
+    .context("Failed to update Repo name on db")?;
+
+    let server =
+      resource::get::<Server>(&repo.config.server_id).await?;
+
+    let log = match periphery_client(&server)?
+      .request(api::git::RenameRepo {
+        curr_name: to_path_compatible_name(&repo.name),
+        new_name: name.clone(),
+      })
+      .await
+      .context("Failed to rename Repo directory on Server")
+    {
+      Ok(log) => log,
+      Err(e) => Log::error(
+        "Rename Repo directory failure",
+        format_serror(&e.into()),
+      ),
+    };
+
+    update.logs.push(log);
+
+    update.push_simple_log(
+      "Rename Repo",
+      format!("Renamed Repo from {} to {}", repo.name, name),
+    );
+    update.finalize();
+    update.id = add_update(update.clone()).await?;
+
+    Ok(update)
+  }
+}
+
+impl Resolve<WriteArgs> for RefreshRepoCache {
+  #[instrument(
+    name = "RefreshRepoCache",
+    level = "debug",
+    skip(user)
+  )]
+  async fn resolve(
+    self,
+    WriteArgs { user }: &WriteArgs,
+  ) -> serror::Result<NoData> {
+    // Even though this is a write request, this doesn't change any config. Anyone that can execute the
+    // repo should be able to do this.
+    let repo = get_check_permissions::<Repo>(
+      &self.repo,
+      user,
+      PermissionLevel::Execute.into(),
+    )
+    .await?;
+
+    if repo.config.git_provider.is_empty()
+      || repo.config.repo.is_empty()
+    {
       // Nothing to do
       return Ok(NoData {});
     }
 
-    let config = core_config();
+    let mut clone_args: RepoExecutionArgs = (&repo).into();
+    let repo_path =
+      clone_args.unique_path(&core_config().repo_directory)?;
+    clone_args.destination = Some(repo_path.display().to_string());
 
-    let repo_dir = config.repo_directory.join(random_string(10));
-    let mut clone_args: CloneArgs = (&repo).into();
-    // No reason to to the commands here.
-    clone_args.on_clone = None;
-    clone_args.on_pull = None;
-    clone_args.destination = Some(repo_dir.display().to_string());
-
-    let access_token = match (&clone_args.account, &clone_args.provider)
-    {
-      (None, _) => None,
-      (Some(_), None) => {
-        return Err(anyhow!(
-          "Account is configured, but provider is empty"
-        ))
-      }
-      (Some(username), Some(provider)) => {
-        git_token(provider, username, |https| {
+    let access_token = if let Some(username) = &clone_args.account {
+      git_token(&clone_args.provider, username, |https| {
           clone_args.https = https
         })
         .await
         .with_context(
-          || format!("Failed to get git token in call to db. Stopping run. | {provider} | {username}"),
+          || format!("Failed to get git token in call to db. Stopping run. | {} | {username}", clone_args.provider),
         )?
-      }
+    } else {
+      None
     };
 
-    let (_, latest_hash, latest_message, _) = git::clone(
+    let (res, _) = git::pull_or_clone(
       clone_args,
-      &config.repo_directory,
+      &core_config().repo_directory,
       access_token,
-      &[],
-      "",
-      None,
     )
     .await
-    .context("failed to clone repo (the resource) repo")?;
+    .with_context(|| {
+      format!("Failed to update repo at {repo_path:?}")
+    })?;
 
     let info = RepoInfo {
       last_pulled_at: repo.info.last_pulled_at,
       last_built_at: repo.info.last_built_at,
       built_hash: repo.info.built_hash,
       built_message: repo.info.built_message,
-      latest_hash,
-      latest_message,
+      latest_hash: res.commit_hash,
+      latest_message: res.commit_message,
     };
 
     let info = to_document(&info)
       .context("failed to serialize repo info to bson")?;
 
     db_client()
-      .await
       .repos
       .update_one(
         doc! { "name": &repo.name },
@@ -160,51 +229,46 @@ impl Resolve<RefreshRepoCache, User> for State {
       .await
       .context("failed to update repo info on db")?;
 
-    if repo_dir.exists() {
-      if let Err(e) = std::fs::remove_dir_all(&repo_dir) {
-        warn!(
-          "failed to remove repo (resource) cache update repo directory | {e:?}"
-        )
-      }
-    }
-
     Ok(NoData {})
   }
 }
 
-impl Resolve<CreateRepoWebhook, User> for State {
-  #[instrument(name = "CreateRepoWebhook", skip(self, user))]
+impl Resolve<WriteArgs> for CreateRepoWebhook {
+  #[instrument(name = "CreateRepoWebhook", skip(args))]
   async fn resolve(
-    &self,
-    CreateRepoWebhook { repo, action }: CreateRepoWebhook,
-    user: User,
-  ) -> anyhow::Result<CreateRepoWebhookResponse> {
+    self,
+    args: &WriteArgs,
+  ) -> serror::Result<CreateRepoWebhookResponse> {
     let Some(github) = github_client() else {
-      return Err(anyhow!(
-        "github_webhook_app is not configured in core config toml"
-      ));
+      return Err(
+        anyhow!(
+          "github_webhook_app is not configured in core config toml"
+        )
+        .into(),
+      );
     };
 
-    let repo = resource::get_check_permissions::<Repo>(
-      &repo,
-      &user,
-      PermissionLevel::Write,
+    let repo = get_check_permissions::<Repo>(
+      &self.repo,
+      &args.user,
+      PermissionLevel::Write.into(),
     )
     .await?;
 
     if repo.config.repo.is_empty() {
-      return Err(anyhow!(
-        "No repo configured, can't create webhook"
-      ));
+      return Err(
+        anyhow!("No repo configured, can't create webhook").into(),
+      );
     }
 
     let mut split = repo.config.repo.split('/');
     let owner = split.next().context("Repo repo has no owner")?;
 
     let Some(github) = github.get(owner) else {
-      return Err(anyhow!(
-        "Cannot manage repo webhooks under owner {owner}"
-      ));
+      return Err(
+        anyhow!("Cannot manage repo webhooks under owner {owner}")
+          .into(),
+      );
     };
 
     let repo_name =
@@ -232,8 +296,12 @@ impl Resolve<CreateRepoWebhook, User> for State {
       &repo.config.webhook_secret
     };
 
-    let host = webhook_base_url.as_ref().unwrap_or(host);
-    let url = match action {
+    let host = if webhook_base_url.is_empty() {
+      host
+    } else {
+      webhook_base_url
+    };
+    let url = match self.action {
       RepoWebhookAction::Clone => {
         format!("{host}/listener/github/repo/{}/clone", repo.id)
       }
@@ -271,64 +339,65 @@ impl Resolve<CreateRepoWebhook, User> for State {
       .context("failed to create webhook")?;
 
     if !repo.config.webhook_enabled {
-      self
-        .resolve(
-          UpdateRepo {
-            id: repo.id,
-            config: PartialRepoConfig {
-              webhook_enabled: Some(true),
-              ..Default::default()
-            },
-          },
-          user,
-        )
-        .await
-        .context("failed to update repo to enable webhook")?;
+      UpdateRepo {
+        id: repo.id,
+        config: PartialRepoConfig {
+          webhook_enabled: Some(true),
+          ..Default::default()
+        },
+      }
+      .resolve(args)
+      .await
+      .map_err(|e| e.error)
+      .context("failed to update repo to enable webhook")?;
     }
 
     Ok(NoData {})
   }
 }
 
-impl Resolve<DeleteRepoWebhook, User> for State {
-  #[instrument(name = "DeleteRepoWebhook", skip(self, user))]
+impl Resolve<WriteArgs> for DeleteRepoWebhook {
+  #[instrument(name = "DeleteRepoWebhook", skip(user))]
   async fn resolve(
-    &self,
-    DeleteRepoWebhook { repo, action }: DeleteRepoWebhook,
-    user: User,
-  ) -> anyhow::Result<DeleteRepoWebhookResponse> {
+    self,
+    WriteArgs { user }: &WriteArgs,
+  ) -> serror::Result<DeleteRepoWebhookResponse> {
     let Some(github) = github_client() else {
-      return Err(anyhow!(
-        "github_webhook_app is not configured in core config toml"
-      ));
+      return Err(
+        anyhow!(
+          "github_webhook_app is not configured in core config toml"
+        )
+        .into(),
+      );
     };
 
-    let repo = resource::get_check_permissions::<Repo>(
-      &repo,
-      &user,
-      PermissionLevel::Write,
+    let repo = get_check_permissions::<Repo>(
+      &self.repo,
+      user,
+      PermissionLevel::Write.into(),
     )
     .await?;
 
     if repo.config.git_provider != "github.com" {
-      return Err(anyhow!(
-        "Can only manage github.com repo webhooks"
-      ));
+      return Err(
+        anyhow!("Can only manage github.com repo webhooks").into(),
+      );
     }
 
     if repo.config.repo.is_empty() {
-      return Err(anyhow!(
-        "No repo configured, can't create webhook"
-      ));
+      return Err(
+        anyhow!("No repo configured, can't create webhook").into(),
+      );
     }
 
     let mut split = repo.config.repo.split('/');
     let owner = split.next().context("Repo repo has no owner")?;
 
     let Some(github) = github.get(owner) else {
-      return Err(anyhow!(
-        "Cannot manage repo webhooks under owner {owner}"
-      ));
+      return Err(
+        anyhow!("Cannot manage repo webhooks under owner {owner}")
+          .into(),
+      );
     };
 
     let repo_name =
@@ -349,8 +418,12 @@ impl Resolve<DeleteRepoWebhook, User> for State {
       ..
     } = core_config();
 
-    let host = webhook_base_url.as_ref().unwrap_or(host);
-    let url = match action {
+    let host = if webhook_base_url.is_empty() {
+      host
+    } else {
+      webhook_base_url
+    };
+    let url = match self.action {
       RepoWebhookAction::Clone => {
         format!("{host}/listener/github/repo/{}/clone", repo.id)
       }

bin/core/src/api/write/resource.rs

@@ -0,0 +1,68 @@
+use anyhow::anyhow;
+use komodo_client::{
+  api::write::{UpdateResourceMeta, UpdateResourceMetaResponse},
+  entities::{
+    ResourceTarget, action::Action, alerter::Alerter, build::Build,
+    builder::Builder, deployment::Deployment, procedure::Procedure,
+    repo::Repo, server::Server, stack::Stack, sync::ResourceSync,
+  },
+};
+use resolver_api::Resolve;
+
+use crate::resource::{self, ResourceMetaUpdate};
+
+use super::WriteArgs;
+
+impl Resolve<WriteArgs> for UpdateResourceMeta {
+  #[instrument(name = "UpdateResourceMeta", skip(args))]
+  async fn resolve(
+    self,
+    args: &WriteArgs,
+  ) -> serror::Result<UpdateResourceMetaResponse> {
+    let meta = ResourceMetaUpdate {
+      description: self.description,
+      template: self.template,
+      tags: self.tags,
+    };
+    match self.target {
+      ResourceTarget::System(_) => {
+        return Err(
+          anyhow!("cannot update meta of System resource target")
+            .into(),
+        );
+      }
+      ResourceTarget::Server(id) => {
+        resource::update_meta::<Server>(&id, meta, args).await?;
+      }
+      ResourceTarget::Deployment(id) => {
+        resource::update_meta::<Deployment>(&id, meta, args).await?;
+      }
+      ResourceTarget::Build(id) => {
+        resource::update_meta::<Build>(&id, meta, args).await?;
+      }
+      ResourceTarget::Repo(id) => {
+        resource::update_meta::<Repo>(&id, meta, args).await?;
+      }
+      ResourceTarget::Builder(id) => {
+        resource::update_meta::<Builder>(&id, meta, args).await?;
+      }
+      ResourceTarget::Alerter(id) => {
+        resource::update_meta::<Alerter>(&id, meta, args).await?;
+      }
+      ResourceTarget::Procedure(id) => {
+        resource::update_meta::<Procedure>(&id, meta, args).await?;
+      }
+      ResourceTarget::Action(id) => {
+        resource::update_meta::<Action>(&id, meta, args).await?;
+      }
+      ResourceTarget::ResourceSync(id) => {
+        resource::update_meta::<ResourceSync>(&id, meta, args)
+          .await?;
+      }
+      ResourceTarget::Stack(id) => {
+        resource::update_meta::<Stack>(&id, meta, args).await?;
+      }
+    }
+    Ok(UpdateResourceMetaResponse {})
+  }
+}

bin/core/src/api/write/server.rs

@@ -1,17 +1,15 @@
 use anyhow::Context;
 use formatting::format_serror;
-use monitor_client::{
+use komodo_client::{
   api::write::*,
   entities::{
-    monitor_timestamp,
+    NoData, Operation,
     permission::PermissionLevel,
     server::Server,
+    to_docker_compatible_name,
     update::{Update, UpdateStatus},
-    user::User,
-    Operation,
   },
 };
-use mungos::{by_id::update_one_by_id, mongodb::bson::doc};
 use periphery_client::api;
 use resolver_api::Resolve;
 
@@ -20,95 +18,91 @@ use crate::{
     periphery_client,
     update::{add_update, make_update, update_update},
   },
+  permission::get_check_permissions,
   resource,
-  state::{db_client, State},
 };
 
-impl Resolve<CreateServer, User> for State {
-  #[instrument(name = "CreateServer", skip(self, user))]
+use super::WriteArgs;
+
+impl Resolve<WriteArgs> for CreateServer {
+  #[instrument(name = "CreateServer", skip(user))]
   async fn resolve(
-    &self,
-    CreateServer { name, config }: CreateServer,
-    user: User,
-  ) -> anyhow::Result<Server> {
-    resource::create::<Server>(&name, config, &user).await
+    self,
+    WriteArgs { user }: &WriteArgs,
+  ) -> serror::Result<Server> {
+    resource::create::<Server>(&self.name, self.config, user).await
   }
 }
 
-impl Resolve<DeleteServer, User> for State {
-  #[instrument(name = "DeleteServer", skip(self, user))]
+impl Resolve<WriteArgs> for CopyServer {
+  #[instrument(name = "CopyServer", skip(user))]
   async fn resolve(
-    &self,
-    DeleteServer { id }: DeleteServer,
-    user: User,
-  ) -> anyhow::Result<Server> {
-    resource::delete::<Server>(&id, &user).await
-  }
-}
-
-impl Resolve<UpdateServer, User> for State {
-  #[instrument(name = "UpdateServer", skip(self, user))]
-  async fn resolve(
-    &self,
-    UpdateServer { id, config }: UpdateServer,
-    user: User,
-  ) -> anyhow::Result<Server> {
-    resource::update::<Server>(&id, config, &user).await
-  }
-}
-
-impl Resolve<RenameServer, User> for State {
-  #[instrument(name = "RenameServer", skip(self, user))]
-  async fn resolve(
-    &self,
-    RenameServer { id, name }: RenameServer,
-    user: User,
-  ) -> anyhow::Result<Update> {
-    let server = resource::get_check_permissions::<Server>(
-      &id,
-      &user,
-      PermissionLevel::Write,
+    self,
+    WriteArgs { user }: &WriteArgs,
+  ) -> serror::Result<Server> {
+    let Server { config, .. } = get_check_permissions::<Server>(
+      &self.id,
+      user,
+      PermissionLevel::Read.into(),
     )
     .await?;
-    let mut update =
-      make_update(&server, Operation::RenameServer, &user);
 
-    update_one_by_id(&db_client().await.servers, &id, mungos::update::Update::Set(doc! { "name": &name, "updated_at": monitor_timestamp() }), None)
-      .await
-      .context("failed to update server on db. this name may already be taken.")?;
-    update.push_simple_log(
-      "rename server",
-      format!("renamed server {id} from {} to {name}", server.name),
-    );
-    update.finalize();
-    update.id = add_update(update.clone()).await?;
-    Ok(update)
+    resource::create::<Server>(&self.name, config.into(), user).await
   }
 }
 
-impl Resolve<CreateNetwork, User> for State {
-  #[instrument(name = "CreateNetwork", skip(self, user))]
+impl Resolve<WriteArgs> for DeleteServer {
+  #[instrument(name = "DeleteServer", skip(args))]
+  async fn resolve(self, args: &WriteArgs) -> serror::Result<Server> {
+    Ok(resource::delete::<Server>(&self.id, args).await?)
+  }
+}
+
+impl Resolve<WriteArgs> for UpdateServer {
+  #[instrument(name = "UpdateServer", skip(user))]
   async fn resolve(
-    &self,
-    CreateNetwork { server, name }: CreateNetwork,
-    user: User,
-  ) -> anyhow::Result<Update> {
-    let server = resource::get_check_permissions::<Server>(
-      &server,
-      &user,
-      PermissionLevel::Write,
+    self,
+    WriteArgs { user }: &WriteArgs,
+  ) -> serror::Result<Server> {
+    Ok(resource::update::<Server>(&self.id, self.config, user).await?)
+  }
+}
+
+impl Resolve<WriteArgs> for RenameServer {
+  #[instrument(name = "RenameServer", skip(user))]
+  async fn resolve(
+    self,
+    WriteArgs { user }: &WriteArgs,
+  ) -> serror::Result<Update> {
+    Ok(resource::rename::<Server>(&self.id, &self.name, user).await?)
+  }
+}
+
+impl Resolve<WriteArgs> for CreateNetwork {
+  #[instrument(name = "CreateNetwork", skip(user))]
+  async fn resolve(
+    self,
+    WriteArgs { user }: &WriteArgs,
+  ) -> serror::Result<Update> {
+    let server = get_check_permissions::<Server>(
+      &self.server,
+      user,
+      PermissionLevel::Write.into(),
     )
     .await?;
 
     let periphery = periphery_client(&server)?;
 
     let mut update =
-      make_update(&server, Operation::CreateNetwork, &user);
+      make_update(&server, Operation::CreateNetwork, user);
     update.status = UpdateStatus::InProgress;
     update.id = add_update(update.clone()).await?;
 
     match periphery
-      .request(api::network::CreateNetwork { name, driver: None })
+      .request(api::network::CreateNetwork {
+        name: to_docker_compatible_name(&self.name),
+        driver: None,
+      })
       .await
     {
       Ok(log) => update.logs.push(log),
@@ -125,41 +119,80 @@ impl Resolve<CreateNetwork, User> for State {
   }
 }
 
-impl Resolve<DeleteNetwork, User> for State {
-  #[instrument(name = "DeleteNetwork", skip(self, user))]
+impl Resolve<WriteArgs> for CreateTerminal {
+  #[instrument(name = "CreateTerminal", skip(user))]
   async fn resolve(
-    &self,
-    DeleteNetwork { server, name }: DeleteNetwork,
-    user: User,
-  ) -> anyhow::Result<Update> {
-    let server = resource::get_check_permissions::<Server>(
-      &server,
-      &user,
-      PermissionLevel::Write,
+    self,
+    WriteArgs { user }: &WriteArgs,
+  ) -> serror::Result<NoData> {
+    let server = get_check_permissions::<Server>(
+      &self.server,
+      user,
+      PermissionLevel::Write.terminal(),
     )
     .await?;
 
     let periphery = periphery_client(&server)?;
 
-    let mut update =
-      make_update(&server, Operation::DeleteNetwork, &user);
-    update.status = UpdateStatus::InProgress;
-    update.id = add_update(update.clone()).await?;
-
-    match periphery
-      .request(api::network::DeleteNetwork { name })
+    periphery
+      .request(api::terminal::CreateTerminal {
+        name: self.name,
+        command: self.command,
+        recreate: self.recreate,
+      })
       .await
-    {
-      Ok(log) => update.logs.push(log),
-      Err(e) => update.push_error_log(
-        "delete network",
-        format_serror(&e.context("failed to delete network").into()),
-      ),
-    };
+      .context("Failed to create terminal on periphery")?;
 
-    update.finalize();
-    update_update(update.clone()).await?;
-
-    Ok(update)
+    Ok(NoData {})
+  }
+}
+
+impl Resolve<WriteArgs> for DeleteTerminal {
+  #[instrument(name = "DeleteTerminal", skip(user))]
+  async fn resolve(
+    self,
+    WriteArgs { user }: &WriteArgs,
+  ) -> serror::Result<NoData> {
+    let server = get_check_permissions::<Server>(
+      &self.server,
+      user,
+      PermissionLevel::Write.terminal(),
+    )
+    .await?;
+
+    let periphery = periphery_client(&server)?;
+
+    periphery
+      .request(api::terminal::DeleteTerminal {
+        terminal: self.terminal,
+      })
+      .await
+      .context("Failed to delete terminal on periphery")?;
+
+    Ok(NoData {})
+  }
+}
+
+impl Resolve<WriteArgs> for DeleteAllTerminals {
+  #[instrument(name = "DeleteAllTerminals", skip(user))]
+  async fn resolve(
+    self,
+    WriteArgs { user }: &WriteArgs,
+  ) -> serror::Result<NoData> {
+    let server = get_check_permissions::<Server>(
+      &self.server,
+      user,
+      PermissionLevel::Write.terminal(),
+    )
+    .await?;
+
+    let periphery = periphery_client(&server)?;
+
+    periphery
+      .request(api::terminal::DeleteAllTerminals {})
+      .await
+      .context("Failed to delete all terminals on periphery")?;
+
+    Ok(NoData {})
   }
 }

bin/core/src/api/write/server_template.rs

@@ -1,65 +0,0 @@
-use monitor_client::{
-  api::write::{
-    CopyServerTemplate, CreateServerTemplate, DeleteServerTemplate,
-    UpdateServerTemplate,
-  },
-  entities::{
-    permission::PermissionLevel, server_template::ServerTemplate,
-    user::User,
-  },
-};
-use resolver_api::Resolve;
-
-use crate::{resource, state::State};
-
-impl Resolve<CreateServerTemplate, User> for State {
-  #[instrument(name = "CreateServerTemplate", skip(self, user))]
-  async fn resolve(
-    &self,
-    CreateServerTemplate { name, config }: CreateServerTemplate,
-    user: User,
-  ) -> anyhow::Result<ServerTemplate> {
-    resource::create::<ServerTemplate>(&name, config, &user).await
-  }
-}
-
-impl Resolve<CopyServerTemplate, User> for State {
-  #[instrument(name = "CopyServerTemplate", skip(self, user))]
-  async fn resolve(
-    &self,
-    CopyServerTemplate { name, id }: CopyServerTemplate,
-    user: User,
-  ) -> anyhow::Result<ServerTemplate> {
-    let ServerTemplate { config, .. } =
-      resource::get_check_permissions::<ServerTemplate>(
-        &id,
-        &user,
-        PermissionLevel::Write,
-      )
-      .await?;
-    resource::create::<ServerTemplate>(&name, config.into(), &user)
-      .await
-  }
-}
-
-impl Resolve<DeleteServerTemplate, User> for State {
-  #[instrument(name = "DeleteServerTemplate", skip(self, user))]
-  async fn resolve(
-    &self,
-    DeleteServerTemplate { id }: DeleteServerTemplate,
-    user: User,
-  ) -> anyhow::Result<ServerTemplate> {
-    resource::delete::<ServerTemplate>(&id, &user).await
-  }
-}
-
-impl Resolve<UpdateServerTemplate, User> for State {
-  #[instrument(name = "UpdateServerTemplate", skip(self, user))]
-  async fn resolve(
-    &self,
-    UpdateServerTemplate { id, config }: UpdateServerTemplate,
-    user: User,
-  ) -> anyhow::Result<ServerTemplate> {
-    resource::update::<ServerTemplate>(&id, config, &user).await
-  }
-}