update deps

.vscode/resolver.code-snippets

@@ -3,7 +3,6 @@
 		"scope": "rust",
 		"prefix": "resolve",
 		"body": [
-			"#[async_trait]",
 			"impl Resolve<${1}, User> for State {",
 			"\tasync fn resolve(&self, ${1} { ${0} }: ${1}, _: User) -> anyhow::Result<${2}> {",
 			"\t\ttodo!()",

Cargo.toml

@@ -3,33 +3,38 @@ resolver = "2"
 members = ["bin/*", "lib/*", "client/core/rs", "client/periphery/rs"]
 
 [workspace.package]
-version = "1.1.0"
+version = "1.5.0"
 edition = "2021"
 authors = ["mbecker20 <becker.maxh@gmail.com>"]
 license = "GPL-3.0-or-later"
+repository = "https://github.com/mbecker20/monitor"
+homepage = "https://docs.monitor.mogh.tech"
+
+[patch.crates-io]
+monitor_client = { path = "client/core/rs" }
 
 [workspace.dependencies]
 # LOCAL
-monitor_client = { path = "client/core/rs" }
+monitor_client = "1.5.0"
 periphery_client = { path = "client/periphery/rs" }
 logger = { path = "lib/logger" }
 
 # MOGH
 run_command = { version = "0.0.6", features = ["async_tokio"] }
-serror = { version = "0.3.4", features = ["axum"] }
+serror = { version = "0.3.4", default-features = false }
 slack = { version = "0.1.0", package = "slack_client_rs" }
 derive_default_builder = "0.1.8"
 derive_empty_traits = "0.1.0"
 merge_config_files = "0.1.5"
 termination_signal = "0.1.3"
 async_timing_util = "0.1.14"
-partial_derive2 = "0.4.1"
+partial_derive2 = "0.4.2"
 derive_variants = "0.1.3"
-mongo_indexed = "0.2.2"
-resolver_api = "0.1.9"
+mongo_indexed = "0.3.0"
+resolver_api = "1.1.0"
 parse_csl = "0.1.0"
-mungos = "0.5.4"
-svi = "0.1.4"
+mungos = "0.5.6"
+svi = "1.0.1"
 
 # ASYNC
 tokio = { version = "1.37.0", features = ["full"] }
@@ -37,8 +42,6 @@ reqwest = { version = "0.12.4", features = ["json"] }
 tokio-util = "0.7.11"
 futures = "0.3.30"
 futures-util = "0.3.30"
-async-trait = "0.1.80"
-async-recursion = "1.1.1"
 
 # SERVER
 axum = { version = "0.7.5", features = ["ws", "json"] }
@@ -48,14 +51,14 @@ tower-http = { version = "0.5.2", features = ["fs", "cors"] }
 tokio-tungstenite = "0.21.0"
 
 # SER/DE
-serde = { version = "1.0.201", features = ["derive"] }
+serde = { version = "1.0.202", features = ["derive"] }
 strum = { version = "0.26.2", features = ["derive"] }
-serde_json = "1.0.116"
-toml = "0.8.12"
+serde_json = "1.0.117"
+toml = "0.8.13"
 
 # ERROR
-anyhow = "1.0.83"
-thiserror = "1.0.60"
+anyhow = "1.0.86"
+thiserror = "1.0.61"
 
 # LOGGING
 opentelemetry_sdk = { version = "0.22.1", features = ["rt-tokio"] }
@@ -85,10 +88,11 @@ bollard = "0.16.1"
 sysinfo = "0.30.12"
 
 # CLOUD
-aws-config = "1.3.0"
-aws-sdk-ec2 = "1.40.0"
+aws-config = "1.4.0"
+aws-sdk-ec2 = "1.42.0"
 
 # MISC
 derive_builder = "0.20.0"
-colored = "2.1.0"
 typeshare = "1.0.3"
+colored = "2.1.0"
+bson = "2.10.0"

bin/alerter/Cargo.toml

@@ -1,9 +1,11 @@
 [package]
-name = "alert_logger"
+name = "alerter"
 version.workspace = true
 edition.workspace = true
 authors.workspace = true
 license.workspace = true
+homepage.workspace = true
+repository.workspace = true
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 

bin/alerter/README.md

@@ -0,0 +1,4 @@
+# Alerter
+
+This crate sets up a basic axum server that listens for incoming alert POSTs.
+It can be used as a monitor alerting endpoint, and serves as a template for other custom alerter implementations.

bin/cli/Cargo.toml

@@ -5,6 +5,8 @@ version.workspace = true
 edition.workspace = true
 authors.workspace = true
 license.workspace = true
+homepage.workspace = true
+repository.workspace = true
 
 [[bin]]
 name = "monitor"
@@ -18,6 +20,10 @@ monitor_client.workspace = true
 # mogh
 partial_derive2.workspace = true
 # external
+tracing-subscriber.workspace = true
+serde_json.workspace = true
+futures.workspace = true
+tracing.workspace = true
 colored.workspace = true
 anyhow.workspace = true
 tokio.workspace = true
@@ -25,6 +31,3 @@ serde.workspace = true
 strum.workspace = true
 toml.workspace = true
 clap.workspace = true
-futures.workspace = true
-tracing.workspace = true
-tracing-subscriber.workspace = true

bin/cli/README.md

@@ -1,8 +1,25 @@
-# monitor CLI
+# Monitor CLI
 
-Monitor CLI is a command line tool to sync monitor resources and execute file defined procedures.
+Monitor CLI is a tool to sync monitor resources and execute file defined procedures.
 
-## Examples
+## Install
+
+```sh
+cargo install monitor_cli
+```
+
+## Usage
+
+Configure a file `~/.config/monitor/creds.toml` file with contents:
+```toml
+url = "https://your.monitor.address"
+key = "YOUR-API-KEY"
+secret = "YOUR-API-SECRET"
+```
+
+Note. You can specify a different creds file by using `--creds ./other/path.toml`.
+
+With your creds in place, you can run syncs:
 
 ```sh
 ## Sync resources in a single file
@@ -15,6 +32,8 @@ monitor sync ./resources
 monitor sync
 ```
 
+And executions:
+
 ```sh
 ## Execute a TOML defined procedure
 monitor exec ./execution/execution.toml

bin/cli/resources/builds.toml

@@ -24,6 +24,40 @@ docker_account = "mbecker2020"
 build_path = "."
 dockerfile_path = "bin/core/Dockerfile"
 
+[[build]]
+name = "monitor_frontend"
+description = "standalone hosted frontend for monitor.mogh.tech"
+tags = ["monitor", "frontend"]
+
+[build.config]
+builder_id = "mogh-builder"
+repo = "mbecker20/monitor"
+branch = "main"
+docker_account = "mbecker2020"
+build_path = "."
+dockerfile_path = "frontend/Dockerfile"
+
+[[build.config.build_args]]
+variable = "VITE_MONITOR_HOST"
+value = "https://monitor.api.mogh.tech"
+
+[[build]]
+name = "monitor_frontend_dev"
+description = "standalone hosted frontend for monitor-dev.mogh.tech"
+tags = ["monitor", "frontend"]
+
+[build.config]
+builder_id = "mogh-builder"
+repo = "mbecker20/monitor"
+branch = "main"
+docker_account = "mbecker2020"
+build_path = "."
+dockerfile_path = "frontend/Dockerfile"
+
+[[build.config.build_args]]
+variable = "VITE_MONITOR_HOST"
+value = "https://monitor-dev.api.mogh.tech"
+
 ## BUILDER
 
 [[builder]]
@@ -39,11 +73,9 @@ region = "us-east-2"
 instance_type = "c5.2xlarge"
 volume_gb = 20
 port = 8120
-ami_id = "ami-0005a05fa63a080ab"
+ami_id = "ami-0c5d5efd79075c0e3"
 subnet_id = "subnet-02ae5ad480eacc4bc"
 security_group_ids = ["sg-049d98c819f9ace58", "sg-006c0ca638af8eb44"]
 key_pair_name = "mogh-key"
 assign_public_ip = true
-use_public_ip = false
-github_accounts = []
-docker_accounts = []
+use_public_ip = true

bin/cli/resources/deployments.toml

@@ -5,7 +5,7 @@ description = "An NGINX proxy for mogh.tech"
 tags = ["monitor"]
 config.server_id = "monitor-01"
 config.network = "host"
-config.restart = "on-failure"
+config.restart = "unless-stopped"
 config.image.type = "Image"
 config.image.params.image = "jc21/nginx-proxy-manager"
 
@@ -20,7 +20,7 @@ container = "/etc/letsencrypt"
 ## MONITOR MONGO
 [[deployment]]
 name = "monitor-mongo"
-description = ""
+description = "shared mongo deployment"
 tags = ["monitor"]
 
 [deployment.config]
@@ -50,7 +50,7 @@ params.image = "mbecker2020/monitor_core"
 ## GRAFANA
 [[deployment]]
 name = "grafana"
-description = ""
+description = "runs on port 3080"
 tags = ["logging"]
 
 [deployment.config]
@@ -78,7 +78,7 @@ value = "key-value"
 ## LOKI
 [[deployment]]
 name = "loki"
-description = ""
+description = "runs on 3100"
 tags = ["logging"]
 
 [deployment.config]
@@ -97,7 +97,7 @@ container = "/loki"
 
 [[deployment]]
 name = "tempo"
-description = ""
+description = "OTLP: port 4317. HTTP: 3200"
 tags = ["logging"]
 
 [deployment.config]
@@ -122,7 +122,7 @@ value = "key-value"
 ## VECTOR
 [[deployment]]
 name = "vector"
-description = ""
+description = "collects the docker logs (and alert endpoint)"
 tags = ["logging"]
 
 [deployment.config]
@@ -137,7 +137,7 @@ type = "Image"
 params.image = "timberio/vector:latest-debian"
 
 [[deployment.config.volumes]]
-local = "/home/ubuntu/.config/vector"
+local = "/root/.config/vector"
 container = "/etc/vector"
 
 [[deployment.config.volumes]]
@@ -155,7 +155,7 @@ value = "key-value"
 ## MONITOR CORE DEV
 [[deployment]]
 name = "monitor-core-dev"
-description = ""
+description = "runs on 9001"
 tags = ["monitor", "dev"]
 
 [deployment.config]
@@ -169,13 +169,45 @@ type = "Build"
 params.build_id = "monitor_core"
 
 [[deployment.config.volumes]]
-local = "/home/ubuntu/.config/monitor/dev.core.config.toml"
+local = "/root/.config/monitor/dev.core.config.toml"
 container = "/config/config.toml"
 
 [[deployment.config.volumes]]
-local = "/etc/monitor/repos/monitor-dev-frontend/frontend/dist"
+local = "/data/repos/monitor-dev-frontend/frontend/dist"
 container = "/frontend"
 
 [[deployment.config.labels]]
 variable = "vector"
-value = "rust"
+value = "rust"
+
+## MONITOR FRONTEND
+[[deployment]]
+name = "monitor-frontend"
+description = ""
+tags = ["monitor", "frontend"]
+
+[deployment.config]
+server_id = "monitor-01"
+redeploy_on_build = true
+network = "host"
+restart = "unless-stopped"
+image.type = "Build"
+image.params.build = "monitor_frontend"
+
+## MONITOR DEV FRONTEND
+[[deployment]]
+name = "monitor-dev-frontend"
+description = ""
+tags = ["monitor", "dev", "frontend"]
+
+[deployment.config]
+server_id = "monitor-01"
+redeploy_on_build = true
+network = "host"
+restart = "unless-stopped"
+image.type = "Build"
+image.params.build = "monitor_frontend_dev"
+
+[[deployment.config.environment]]
+variable = "PORT"
+value = "4175"

bin/cli/resources/procedures.toml

@@ -1,8 +1,35 @@
 [[procedure]]
-name = "test-procedure"
+name = "build-monitor"
 description = ""
-tags = []
+tags = ["monitor"]
 
 [procedure.config]
-procedure_type = "Sequence"
-executions = []
+procedure_type = "Parallel"
+webhook_enabled = true
+
+[[procedure.config.executions]]
+enabled = true
+
+[procedure.config.executions.execution]
+type = "RunBuild"
+
+[procedure.config.executions.execution.params]
+build = "monitor_core"
+
+[[procedure.config.executions]]
+enabled = true
+
+[procedure.config.executions.execution]
+type = "RunBuild"
+
+[procedure.config.executions.execution.params]
+build = "monitor_frontend"
+
+[[procedure.config.executions]]
+enabled = false
+
+[procedure.config.executions.execution]
+type = "RunBuild"
+
+[procedure.config.executions.execution.params]
+build = "monitor_frontend_dev"

bin/cli/resources/repos.toml

@@ -1,7 +1,26 @@
+# [[repo]]
+# name = "monitor-dev-frontend"
+# description = "Used as frontend for monitor-core-dev"
+# tags = ["monitor", "dev"]
+
+# [repo.config]
+# server_id = "monitor-01"
+# repo = "mbecker20/monitor"
+# branch = "main"
+# github_account = ""
+
+# [repo.config.on_clone]
+# path = ""
+# command = ""
+
+# [repo.config.on_pull]
+# path = "frontend"
+# command = "sh on_pull.sh"
+
 [[repo]]
-name = "monitor-dev-frontend"
-description = "Used as frontend for monitor-core-dev"
-tags = ["monitor", "dev"]
+name = "monitor-periphery"
+description = ""
+tags = ["monitor"]
 
 [repo.config]
 server_id = "monitor-01"
@@ -14,5 +33,5 @@ path = ""
 command = ""
 
 [repo.config.on_pull]
-path = "frontend"
-command = "sh on_pull.sh"
+path = "."
+command = "/root/.cargo/bin/cargo build -p monitor_periphery --release && cp ./target/release/periphery /home/ubuntu/periphery"

bin/cli/src/main.rs

@@ -25,13 +25,19 @@ struct CliArgs {
   #[command(subcommand)]
   command: Command,
   /// The path to a creds file.
-  #[arg(long, default_value_t = String::from("./creds.toml"))]
+  #[arg(long, default_value_t = default_creds())]
   creds: String,
   /// Log less (just resource names).
   #[arg(long, default_value_t = false)]
   quiet: bool,
 }
 
+fn default_creds() -> String {
+  let home = std::env::var("HOME")
+    .expect("no HOME env var. cannot get default config path.");
+  format!("{home}/.config/monitor/creds.toml")
+}
+
 #[derive(Debug, Clone, Subcommand)]
 enum Command {
   /// Runs syncs on resource files
@@ -69,11 +75,11 @@ fn monitor_client() -> &'static MonitorClient {
 
 #[tokio::main]
 async fn main() -> anyhow::Result<()> {
-  tracing_subscriber::fmt::init();
+  tracing_subscriber::fmt().with_target(false).init();
 
   let version =
     monitor_client().read(read::GetVersion {}).await?.version;
-  info!("monitor version: {version}");
+  info!("monitor version: {}", version.to_string().blue().bold());
 
   match &cli_args().command {
     Command::Exec { path } => execution::run_execution(path).await?,
@@ -94,7 +100,11 @@ fn parse_toml_file<T: DeserializeOwned>(
 }
 
 fn wait_for_enter(press_enter_to: &str) -> anyhow::Result<()> {
-  println!("\nPress {} to {}\n", "ENTER".green(), press_enter_to.bold());
+  println!(
+    "\nPress {} to {}\n",
+    "ENTER".green(),
+    press_enter_to.bold()
+  );
   let buffer = &mut [0u8];
   std::io::stdin()
     .read_exact(buffer)

bin/cli/src/sync/file.rs

@@ -1,6 +1,7 @@
 use std::{fs, path::Path};
 
 use anyhow::{anyhow, Context};
+use colored::Colorize;
 use monitor_client::entities::toml::ResourcesToml;
 
 pub fn read_resources(path: &Path) -> anyhow::Result<ResourcesToml> {
@@ -30,7 +31,11 @@ fn read_resources_recursive(
         return Ok(());
       }
     };
-    info!("adding resources from {path:?}");
+    info!(
+      "{} from {}",
+      "adding resources".green().bold(),
+      path.display().to_string().blue().bold()
+    );
     resources.server_templates.extend(more.server_templates);
     resources.servers.extend(more.servers);
     resources.builds.extend(more.builds);

bin/cli/src/sync/mod.rs

@@ -1,5 +1,6 @@
 use std::path::Path;
 
+use colored::Colorize;
 use monitor_client::entities::{
   alerter::Alerter, build::Build, builder::Builder,
   deployment::Deployment, procedure::Procedure, repo::Repo,
@@ -8,14 +9,17 @@ use monitor_client::entities::{
 
 use crate::{sync::resources::ResourceSync, wait_for_enter};
 
-mod resource_file;
+mod file;
 mod resources;
 mod user_group;
 
 pub async fn run_sync(path: &Path) -> anyhow::Result<()> {
-  info!("path: {path:?}");
+  info!(
+    "resources path: {}",
+    path.display().to_string().blue().bold()
+  );
 
-  let resources = resource_file::read_resources(path)?;
+  let resources = file::read_resources(path)?;
 
   info!("computing sync actions...");
 
@@ -57,7 +61,7 @@ pub async fn run_sync(path: &Path) -> anyhow::Result<()> {
     && user_group_creates.is_empty()
     && user_group_updates.is_empty()
   {
-    info!("nothing to do. exiting.");
+    info!("{}. exiting.", "nothing to do".green().bold());
     return Ok(());
   }
 

bin/cli/src/sync/resources/mod.rs

@@ -10,6 +10,7 @@ use monitor_client::{
   },
 };
 use partial_derive2::{Diff, FieldDiff, MaybeNone, PartialDiff};
+use serde::Serialize;
 
 use crate::{cli_args, maps::id_to_tag, monitor_client};
 
@@ -43,6 +44,8 @@ pub trait ResourceSync {
     + Clone
     + Send
     + From<Self::ConfigDiff>
+    + Serialize
+    + MaybeNone
     + 'static;
   type ConfigDiff: Diff + MaybeNone;
   type ListItemInfo: 'static;
@@ -171,10 +174,16 @@ pub trait ResourceSync {
         None => {
           if !quiet {
             println!(
-              "{}: {}: {}: {resource:#?}",
+              "\n{}: {}: {}\n{}: {}\n{}: {:?}\n{}: {}",
               "CREATE".green(),
               Self::display(),
               resource.name.bold().green(),
+              "description".dimmed(),
+              resource.description,
+              "tags".dimmed(),
+              resource.tags,
+              "config".dimmed(),
+              serde_json::to_string_pretty(&resource.config)?
             )
           }
           to_create.push(resource);
@@ -210,8 +219,6 @@ pub trait ResourceSync {
     to_create: ToCreate<Self::PartialConfig>,
     to_update: ToUpdate<Self::PartialConfig>,
   ) {
-    let log_after = !to_update.is_empty() || !to_create.is_empty();
-
     for resource in to_create {
       let name = resource.name.clone();
       let tags = resource.tags.clone();
@@ -228,7 +235,12 @@ pub trait ResourceSync {
       };
       Self::update_tags(id.clone(), &name, tags).await;
       Self::update_description(id, &name, description).await;
-      info!("{} {name} created", Self::display());
+      info!(
+        "{} {} '{}'",
+        "created".green().bold(),
+        Self::display(),
+        name.bold(),
+      );
     }
 
     for ToUpdateItem {
@@ -252,23 +264,21 @@ pub trait ResourceSync {
         Self::update_tags(id.clone(), &name, tags).await;
       }
 
-      if let Err(e) = Self::update(id, resource).await {
-        warn!(
-          "failed to update config on {} {name} | {e:#}",
-          Self::display()
-        );
-      } else {
-        info!("updated {} {name} config", Self::display());
+      if !resource.config.is_none() {
+        if let Err(e) = Self::update(id, resource).await {
+          warn!(
+            "failed to update config on {} {name} | {e:#}",
+            Self::display()
+          );
+        } else {
+          info!(
+            "{} {} '{}' configuration",
+            "updated".blue().bold(),
+            Self::display(),
+            name.bold(),
+          );
+        }
       }
-
-      info!("{} {name} updated", Self::display());
-    }
-
-    if log_after {
-      info!(
-        "============ {}s synced ✅ ============",
-        Self::display()
-      );
     }
   }
 
@@ -286,7 +296,12 @@ pub trait ResourceSync {
         Self::display(),
       );
     } else {
-      info!("updated {} {name} tags", Self::display());
+      info!(
+        "{} {} '{}' tags",
+        "updated".blue().bold(),
+        Self::display(),
+        name.bold(),
+      );
     }
   }
 
@@ -304,7 +319,12 @@ pub trait ResourceSync {
     {
       warn!("failed to update resource {id} description | {e:#}");
     } else {
-      info!("updated {} {name} description", Self::display());
+      info!(
+        "{} {} '{}' description",
+        "updated".blue().bold(),
+        Self::display(),
+        name.bold(),
+      );
     }
   }
 }

bin/core/Cargo.toml

@@ -4,6 +4,8 @@ version.workspace = true
 edition.workspace = true
 authors.workspace = true
 license.workspace = true
+homepage.workspace = true
+repository.workspace = true
 
 [[bin]]
 name = "core"
@@ -13,10 +15,11 @@ path = "src/main.rs"
 
 [dependencies]
 # local
+monitor_client = { workspace = true, features = ["mongo"] }
 periphery_client.workspace = true
-monitor_client.workspace = true
 logger.workspace = true
 # mogh
+serror = { workspace = true, features = ["axum"] }
 merge_config_files.workspace = true
 termination_signal.workspace = true
 async_timing_util.workspace = true
@@ -25,34 +28,32 @@ mongo_indexed.workspace = true
 resolver_api.workspace = true
 parse_csl.workspace = true
 mungos.workspace = true
-serror.workspace = true
 slack.workspace = true
+svi.workspace = true
 # external
-tokio.workspace = true
+urlencoding.workspace = true
+aws-sdk-ec2.workspace = true
+aws-config.workspace = true
 tokio-util.workspace = true
-axum.workspace = true
 axum-extra.workspace = true
-tower.workspace = true
 tower-http.workspace = true
-serde.workspace = true
 serde_json.workspace = true
+typeshare.workspace = true
+tracing.workspace = true
+reqwest.workspace = true
+futures.workspace = true
+anyhow.workspace = true
+dotenv.workspace = true
+bcrypt.workspace = true
+tokio.workspace = true
+tower.workspace = true
+serde.workspace = true
+axum.workspace = true
 toml.workspace = true
 uuid.workspace = true
-anyhow.workspace = true
-tracing.workspace = true
-dotenv.workspace = true
 envy.workspace = true
-reqwest.workspace = true
-urlencoding.workspace = true
 rand.workspace = true
-jwt.workspace = true
 hmac.workspace = true
 sha2.workspace = true
-bcrypt.workspace = true
+jwt.workspace = true
 hex.workspace = true
-async-trait.workspace = true
-async-recursion.workspace = true
-futures.workspace = true
-aws-config.workspace = true
-aws-sdk-ec2.workspace = true
-typeshare.workspace = true

bin/core/Dockerfile

@@ -1,5 +1,5 @@
 # Build Core
-FROM rust:1.77.2-bullseye as core-builder
+FROM rust:1.78.0-bullseye as core-builder
 WORKDIR /builder
 COPY . .
 RUN cargo build -p monitor_core --release

bin/core/src/api/auth.rs

@@ -1,7 +1,6 @@
 use std::{sync::OnceLock, time::Instant};
 
 use anyhow::anyhow;
-use async_trait::async_trait;
 use axum::{http::HeaderMap, routing::post, Router};
 use axum_extra::{headers::ContentType, TypedHeader};
 use monitor_client::{api::auth::*, entities::user::User};
@@ -92,7 +91,6 @@ fn login_options_reponse() -> &'static GetLoginOptionsResponse {
   })
 }
 
-#[async_trait]
 impl Resolve<GetLoginOptions, HeaderMap> for State {
   #[instrument(name = "GetLoginOptions", level = "debug", skip(self))]
   async fn resolve(
@@ -104,7 +102,6 @@ impl Resolve<GetLoginOptions, HeaderMap> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<ExchangeForJwt, HeaderMap> for State {
   #[instrument(name = "ExchangeForJwt", level = "debug", skip(self))]
   async fn resolve(
@@ -118,7 +115,6 @@ impl Resolve<ExchangeForJwt, HeaderMap> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<GetUser, HeaderMap> for State {
   #[instrument(name = "GetUser", level = "debug", skip(self))]
   async fn resolve(

bin/core/src/api/execute/build.rs

@@ -1,7 +1,6 @@
-use std::{str::FromStr, time::Duration};
+use std::{collections::HashSet, time::Duration};
 
 use anyhow::{anyhow, Context};
-use async_trait::async_trait;
 use futures::future::join_all;
 use monitor_client::{
   api::execute::{
@@ -11,19 +10,20 @@ use monitor_client::{
     all_logs_success,
     build::Build,
     builder::{AwsBuilderConfig, Builder, BuilderConfig},
-    deployment::DockerContainerState,
+    deployment::DeploymentState,
     monitor_timestamp,
     permission::PermissionLevel,
     server::Server,
-    server_template::AwsServerTemplateConfig,
+    server_template::aws::AwsServerTemplateConfig,
     update::{Log, Update},
     user::{auto_redeploy_user, User},
     Operation,
   },
 };
 use mungos::{
+  by_id::update_one_by_id,
   find::find_collect,
-  mongodb::bson::{doc, oid::ObjectId, to_bson},
+  mongodb::bson::{doc, to_bson, to_document},
 };
 use periphery_client::{
   api::{self, GetVersionResponse},
@@ -45,14 +45,13 @@ use crate::{
   helpers::{
     channel::build_cancel_channel,
     periphery_client,
-    query::get_deployment_state,
+    query::{get_deployment_state, get_global_variables},
     update::{add_update, make_update, update_update},
   },
-  resource,
+  resource::{self, refresh_build_state_cache},
   state::{action_states, db_client, State},
 };
 
-#[async_trait]
 impl Resolve<RunBuild, User> for State {
   #[instrument(name = "RunBuild", skip(self, user))]
   async fn resolve(
@@ -135,13 +134,12 @@ impl Resolve<RunBuild, User> for State {
             "get builder",
             serialize_error_pretty(&e),
           ));
-          update.finalize();
-          update_update(update.clone()).await?;
-          return Ok(update);
+          return handle_early_return(update).await;
         }
       };
 
     let core_config = core_config();
+    let variables = get_global_variables().await?;
 
     // CLONE REPO
 
@@ -162,9 +160,7 @@ impl Resolve<RunBuild, User> for State {
         cleanup_builder_instance(periphery, cleanup_data, &mut update)
           .await;
         info!("builder cleaned up");
-        update.finalize();
-        update_update(update.clone()).await?;
-        return Ok(update)
+        return handle_early_return(update).await
       },
     };
 
@@ -187,20 +183,66 @@ impl Resolve<RunBuild, User> for State {
         .get(&build.config.docker_account)
         .cloned();
 
+      // Interpolate variables / secrets into build args
+      let mut global_replacers = HashSet::new();
+      let mut secret_replacers = HashSet::new();
+      for arg in &mut build.config.build_args {
+        // first pass - global variables
+        let (res, more_replacers) = svi::interpolate_variables(
+          &arg.value,
+          &variables,
+          svi::Interpolator::DoubleBrackets,
+          false,
+        )
+        .context("failed to interpolate global variables")?;
+        global_replacers.extend(more_replacers);
+        // second pass - core secrets
+        let (res, more_replacers) = svi::interpolate_variables(
+          &res,
+          &core_config.secrets,
+          svi::Interpolator::DoubleBrackets,
+          false,
+        )
+        .context("failed to interpolate core secrets")?;
+        secret_replacers.extend(more_replacers);
+        arg.value = res;
+      }
+
+      // Show which variables were interpolated
+      if !global_replacers.is_empty() {
+        update.push_simple_log(
+          "interpolate global variables",
+          global_replacers
+            .into_iter()
+            .map(|(value, variable)| format!("<span class=\"text-muted-foreground\">{variable} =></span> {value}"))
+            .collect::<Vec<_>>()
+            .join("\n"),
+        );
+      }
+      if !secret_replacers.is_empty() {
+        update.push_simple_log(
+          "interpolate core secrets",
+          secret_replacers
+            .iter()
+            .map(|(_, variable)| format!("<span class=\"text-muted-foreground\">replaced:</span> {variable}"))
+            .collect::<Vec<_>>()
+            .join("\n"),
+        );
+      }
+
       let res = tokio::select! {
         res = periphery
           .request(api::build::Build {
             build: build.clone(),
             docker_token,
+            replacers: secret_replacers.into_iter().collect(),
           }) => res.context("failed at call to periphery to build"),
         _ = cancel.cancelled() => {
           info!("build cancelled during build, cleaning up builder");
           update.push_error_log("build cancelled", String::from("user cancelled build during docker build"));
           cleanup_builder_instance(periphery, cleanup_data, &mut update)
             .await;
-          update.finalize();
-          update_update(update.clone()).await?;
-          return Ok(update)
+          return handle_early_return(update).await
         },
       };
 
@@ -218,12 +260,13 @@ impl Resolve<RunBuild, User> for State {
 
     update.finalize();
 
+    let db = db_client().await;
+
     if update.success {
-      let _ = db_client()
-        .await
+      let _ = db
         .builds
         .update_one(
-          doc! { "_id": ObjectId::from_str(&build.id)? },
+          doc! { "name": &build.name },
           doc! {
             "$set": {
               "config.version": to_bson(&build.config.version)
@@ -241,20 +284,57 @@ impl Resolve<RunBuild, User> for State {
     cleanup_builder_instance(periphery, cleanup_data, &mut update)
       .await;
 
-    info!("builder instance cleaned up");
+    // Need to manually update the update before cache refresh,
+    // and before broadcast with add_update.
+    // The Err case of to_document should be unreachable,
+    // but will fail to update cache in that case.
+    if let Ok(update_doc) = to_document(&update) {
+      let _ = update_one_by_id(
+        &db.updates,
+        &update.id,
+        mungos::update::Update::Set(update_doc),
+        None,
+      )
+      .await;
+      refresh_build_state_cache().await;
+    }
 
     update_update(update.clone()).await?;
 
     if update.success {
-      handle_post_build_redeploy(&build.id).await;
-      info!("post build redeploy handled");
+      // don't hold response up for user
+      tokio::spawn(async move {
+        handle_post_build_redeploy(&build.id).await;
+        info!("post build redeploy handled");
+      });
     }
 
     Ok(update)
   }
 }
 
-#[async_trait]
+async fn handle_early_return(
+  mut update: Update,
+) -> anyhow::Result<Update> {
+  update.finalize();
+  // Need to manually update the update before cache refresh,
+  // and before broadcast with add_update.
+  // The Err case of to_document should be unreachable,
+  // but will fail to update cache in that case.
+  if let Ok(update_doc) = to_document(&update) {
+    let _ = update_one_by_id(
+      &db_client().await.updates,
+      &update.id,
+      mungos::update::Update::Set(update_doc),
+      None,
+    )
+    .await;
+    refresh_build_state_cache().await;
+  }
+  update_update(update.clone()).await?;
+  Ok(update)
+}
+
 impl Resolve<CancelBuild, User> for State {
   #[instrument(name = "CancelBuild", skip(self, user))]
   async fn resolve(
@@ -352,11 +432,8 @@ async fn get_aws_builder(
 ) -> anyhow::Result<(PeripheryClient, BuildCleanupData)> {
   let start_create_ts = monitor_timestamp();
 
-  let instance_name = format!(
-    "BUILDER-{}-v{}",
-    build.name,
-    build.config.version
-  );
+  let instance_name =
+    format!("BUILDER-{}-v{}", build.name, build.config.version);
   let Ec2Instance { instance_id, ip } = launch_ec2_instance(
     &instance_name,
     AwsServerTemplateConfig::from_builder_config(&config),
@@ -478,7 +555,7 @@ async fn handle_post_build_redeploy(build_id: &str) {
       .map(|deployment| async move {
         let state =
           get_deployment_state(&deployment).await.unwrap_or_default();
-        if state == DockerContainerState::Running {
+        if state == DeploymentState::Running {
           let res = State
             .resolve(
               Deploy {

bin/core/src/api/execute/deployment.rs

@@ -1,5 +1,6 @@
+use std::collections::HashSet;
+
 use anyhow::{anyhow, Context};
-use async_trait::async_trait;
 use futures::future::join_all;
 use monitor_client::{
   api::execute::*,
@@ -8,7 +9,7 @@ use monitor_client::{
     deployment::{Deployment, DeploymentImage},
     get_image_name, monitor_timestamp,
     permission::PermissionLevel,
-    server::ServerStatus,
+    server::ServerState,
     update::{Log, ResourceTarget, Update, UpdateStatus},
     user::User,
     Operation, Version,
@@ -23,7 +24,7 @@ use crate::{
   config::core_config,
   helpers::{
     periphery_client,
-    query::get_server_with_status,
+    query::{get_global_variables, get_server_with_status},
     update::{add_update, make_update, update_update},
   },
   monitor::update_cache_for_server,
@@ -31,7 +32,6 @@ use crate::{
   state::{action_states, db_client, State},
 };
 
-#[async_trait]
 impl Resolve<Deploy, User> for State {
   #[instrument(name = "Deploy", skip(self, user))]
   async fn resolve(
@@ -51,6 +51,10 @@ impl Resolve<Deploy, User> for State {
       )
       .await?;
 
+    if deployment.config.server_id.is_empty() {
+      return Err(anyhow!("deployment has no server configured"));
+    }
+
     // get the action state for the deployment (or insert default).
     let action_state = action_states()
       .deployment
@@ -62,13 +66,9 @@ impl Resolve<Deploy, User> for State {
     let _action_guard =
       action_state.update(|state| state.deploying = true)?;
 
-    if deployment.config.server_id.is_empty() {
-      return Err(anyhow!("deployment has no server configured"));
-    }
-
     let (server, status) =
       get_server_with_status(&deployment.config.server_id).await?;
-    if status != ServerStatus::Ok {
+    if status != ServerState::Ok {
       return Err(anyhow!(
         "cannot send action when server is unreachable or disabled"
       ));
@@ -85,9 +85,11 @@ impl Resolve<Deploy, User> for State {
         } else {
           version
         };
+        // replace image with corresponding build image.
         deployment.config.image = DeploymentImage::Image {
           image: format!("{image_name}:{version}"),
         };
+        // set docker account to match build docker account if it's not overridden by deployment
         if deployment.config.docker_account.is_empty() {
           deployment.config.docker_account =
             build.config.docker_account;
@@ -97,14 +99,66 @@ impl Resolve<Deploy, User> for State {
       DeploymentImage::Image { .. } => Version::default(),
     };
 
+    let variables = get_global_variables().await?;
+    let core_config = core_config();
+
+    // Interpolate variables into environment
+    let mut global_replacers = HashSet::new();
+    let mut secret_replacers = HashSet::new();
+    for env in &mut deployment.config.environment {
+      // first pass - global variables
+      let (res, more_replacers) = svi::interpolate_variables(
+        &env.value,
+        &variables,
+        svi::Interpolator::DoubleBrackets,
+        false,
+      )
+      .context("failed to interpolate global variables")?;
+      global_replacers.extend(more_replacers);
+      // second pass - core secrets
+      let (res, more_replacers) = svi::interpolate_variables(
+        &res,
+        &core_config.secrets,
+        svi::Interpolator::DoubleBrackets,
+        false,
+      )
+      .context("failed to interpolate core secrets")?;
+      secret_replacers.extend(more_replacers);
+
+      // set env value with the result
+      env.value = res;
+    }
+
     let mut update =
       make_update(&deployment, Operation::DeployContainer, &user);
     update.in_progress();
     update.version = version;
 
+    // Show which variables were interpolated
+    if !global_replacers.is_empty() {
+      update.push_simple_log(
+        "interpolate global variables",
+        global_replacers
+          .into_iter()
+          .map(|(value, variable)| format!("<span class=\"text-muted-foreground\">{variable} =></span> {value}"))
+          .collect::<Vec<_>>()
+          .join("\n"),
+      );
+    }
+    if !secret_replacers.is_empty() {
+      update.push_simple_log(
+        "interpolate core secrets",
+        secret_replacers
+          .iter()
+          .map(|(_, variable)| format!("<span class=\"text-muted-foreground\">replaced:</span> {variable}"))
+          .collect::<Vec<_>>()
+          .join("\n"),
+      );
+    }
+
     update.id = add_update(update.clone()).await?;
 
-    let docker_token = core_config()
+    let docker_token = core_config
       .docker_accounts
       .get(&deployment.config.docker_account)
       .cloned();
@@ -115,6 +169,7 @@ impl Resolve<Deploy, User> for State {
         stop_signal,
         stop_time,
         docker_token,
+        replacers: secret_replacers.into_iter().collect(),
       })
       .await
     {
@@ -136,7 +191,6 @@ impl Resolve<Deploy, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<StartContainer, User> for State {
   #[instrument(name = "StartContainer", skip(self, user))]
   async fn resolve(
@@ -168,7 +222,7 @@ impl Resolve<StartContainer, User> for State {
 
     let (server, status) =
       get_server_with_status(&deployment.config.server_id).await?;
-    if status != ServerStatus::Ok {
+    if status != ServerState::Ok {
       return Err(anyhow!(
         "cannot send action when server is unreachable or disabled"
       ));
@@ -211,7 +265,6 @@ impl Resolve<StartContainer, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<StopContainer, User> for State {
   #[instrument(name = "StopContainer", skip(self, user))]
   async fn resolve(
@@ -247,7 +300,7 @@ impl Resolve<StopContainer, User> for State {
 
     let (server, status) =
       get_server_with_status(&deployment.config.server_id).await?;
-    if status != ServerStatus::Ok {
+    if status != ServerState::Ok {
       return Err(anyhow!(
         "cannot send action when server is unreachable or disabled"
       ));
@@ -287,7 +340,6 @@ impl Resolve<StopContainer, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<StopAllContainers, User> for State {
   #[instrument(name = "StopAllContainers", skip(self, user))]
   async fn resolve(
@@ -296,7 +348,7 @@ impl Resolve<StopAllContainers, User> for State {
     user: User,
   ) -> anyhow::Result<Update> {
     let (server, status) = get_server_with_status(&server).await?;
-    if status != ServerStatus::Ok {
+    if status != ServerState::Ok {
       return Err(anyhow!(
         "cannot send action when server is unreachable or disabled"
       ));
@@ -370,7 +422,6 @@ impl Resolve<StopAllContainers, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<RemoveContainer, User> for State {
   #[instrument(name = "RemoveContainer", skip(self, user))]
   async fn resolve(
@@ -406,7 +457,7 @@ impl Resolve<RemoveContainer, User> for State {
 
     let (server, status) =
       get_server_with_status(&deployment.config.server_id).await?;
-    if status != ServerStatus::Ok {
+    if status != ServerState::Ok {
       return Err(anyhow!(
         "cannot send action when server is unreachable or disabled"
       ));

bin/core/src/api/execute/mod.rs

@@ -4,7 +4,7 @@ use anyhow::{anyhow, Context};
 use axum::{middleware, routing::post, Extension, Router};
 use axum_extra::{headers::ContentType, TypedHeader};
 use monitor_client::{api::execute::*, entities::user::User};
-use resolver_api::{derive::Resolver, Resolve, Resolver};
+use resolver_api::{derive::Resolver, Resolver};
 use serde::{Deserialize, Serialize};
 use serror::Json;
 use typeshare::typeshare;

bin/core/src/api/execute/procedure.rs

@@ -1,4 +1,5 @@
-use async_trait::async_trait;
+use std::pin::Pin;
+
 use monitor_client::{
   api::execute::RunProcedure,
   entities::{
@@ -6,6 +7,7 @@ use monitor_client::{
     update::Update, user::User, Operation,
   },
 };
+use mungos::{by_id::update_one_by_id, mongodb::bson::to_document};
 use resolver_api::Resolve;
 use serror::serialize_error_pretty;
 use tokio::sync::Mutex;
@@ -14,10 +16,11 @@ use crate::{
   helpers::{
     procedure::execute_procedure,
     update::{add_update, make_update, update_update},
-  }, resource, state::{action_states, State}
+  },
+  resource::{self, refresh_procedure_state_cache},
+  state::{action_states, db_client, State},
 };
 
-#[async_trait]
 impl Resolve<RunProcedure, User> for State {
   #[instrument(name = "RunProcedure", skip(self, user))]
   async fn resolve(
@@ -25,6 +28,19 @@ impl Resolve<RunProcedure, User> for State {
     RunProcedure { procedure }: RunProcedure,
     user: User,
   ) -> anyhow::Result<Update> {
+    resolve_inner(procedure, user).await
+  }
+}
+
+fn resolve_inner(
+  procedure: String,
+  user: User,
+) -> Pin<
+  Box<
+    dyn std::future::Future<Output = anyhow::Result<Update>> + Send,
+  >,
+> {
+  Box::pin(async move {
     let procedure = resource::get_check_permissions::<Procedure>(
       &procedure,
       &user,
@@ -74,8 +90,23 @@ impl Resolve<RunProcedure, User> for State {
 
     update.finalize();
 
+    // Need to manually update the update before cache refresh,
+    // and before broadcast with add_update.
+    // The Err case of to_document should be unreachable,
+    // but will fail to update cache in that case.
+    if let Ok(update_doc) = to_document(&update) {
+      let _ = update_one_by_id(
+        &db_client().await.updates,
+        &update.id,
+        mungos::update::Update::Set(update_doc),
+        None,
+      )
+      .await;
+      refresh_procedure_state_cache().await;
+    }
+
     update_update(update.clone()).await?;
 
     Ok(update)
-  }
+  })
 }

bin/core/src/api/execute/repo.rs

@@ -1,7 +1,4 @@
-use std::str::FromStr;
-
 use anyhow::anyhow;
-use async_trait::async_trait;
 use monitor_client::{
   api::execute::*,
   entities::{
@@ -14,7 +11,10 @@ use monitor_client::{
     Operation,
   },
 };
-use mungos::mongodb::bson::{doc, oid::ObjectId};
+use mungos::{
+  by_id::update_one_by_id,
+  mongodb::bson::{doc, to_document},
+};
 use periphery_client::api;
 use resolver_api::Resolve;
 use serror::serialize_error_pretty;
@@ -25,11 +25,10 @@ use crate::{
     periphery_client,
     update::{add_update, update_update},
   },
-  resource,
+  resource::{self, refresh_repo_state_cache},
   state::{action_states, db_client, State},
 };
 
-#[async_trait]
 impl Resolve<CloneRepo, User> for State {
   #[instrument(name = "CloneRepo", skip(self, user))]
   async fn resolve(
@@ -98,28 +97,13 @@ impl Resolve<CloneRepo, User> for State {
     update.finalize();
 
     if update.success {
-      let res = db_client().await
-          .repos
-          .update_one(
-            doc! { "_id": ObjectId::from_str(&repo.id)? },
-            doc! { "$set": { "info.last_pulled_at": monitor_timestamp() } },
-            None,
-          )
-          .await;
-      if let Err(e) = res {
-        warn!(
-            "failed to update repo last_pulled_at | repo id: {} | {e:#}",
-            repo.id
-          );
-      }
+      update_last_pulled(&repo.name).await;
     }
 
-    update_update(update.clone()).await?;
-    Ok(update)
+    handle_update_return(update).await
   }
 }
 
-#[async_trait]
 impl Resolve<PullRepo, User> for State {
   #[instrument(name = "PullRepo", skip(self, user))]
   async fn resolve(
@@ -168,8 +152,9 @@ impl Resolve<PullRepo, User> for State {
 
     let logs = match periphery
       .request(api::git::PullRepo {
-        name: repo.name,
+        name: repo.name.clone(),
         branch: optional_string(&repo.config.branch),
+        commit: optional_string(&repo.config.commit),
         on_pull: repo.config.on_pull.into_option(),
       })
       .await
@@ -185,23 +170,47 @@ impl Resolve<PullRepo, User> for State {
     update.finalize();
 
     if update.success {
-      let res = db_client().await
-          .repos
-          .update_one(
-            doc! { "_id": ObjectId::from_str(&repo.id)? },
-            doc! { "$set": { "info.last_pulled_at": monitor_timestamp() } },
-            None,
-          )
-          .await;
-      if let Err(e) = res {
-        warn!(
-            "failed to update repo last_pulled_at | repo id: {} | {e:#}",
-            repo.id
-          );
-      }
+      update_last_pulled(&repo.name).await;
     }
 
-    update_update(update.clone()).await?;
-    Ok(update)
+    handle_update_return(update).await
+  }
+}
+
+async fn handle_update_return(
+  update: Update,
+) -> anyhow::Result<Update> {
+  // Need to manually update the update before cache refresh,
+  // and before broadcast with add_update.
+  // The Err case of to_document should be unreachable,
+  // but will fail to update cache in that case.
+  if let Ok(update_doc) = to_document(&update) {
+    let _ = update_one_by_id(
+      &db_client().await.updates,
+      &update.id,
+      mungos::update::Update::Set(update_doc),
+      None,
+    )
+    .await;
+    refresh_repo_state_cache().await;
+  }
+  update_update(update.clone()).await?;
+  Ok(update)
+}
+
+async fn update_last_pulled(repo_name: &str) {
+  let res = db_client()
+    .await
+    .repos
+    .update_one(
+      doc! { "name": repo_name },
+      doc! { "$set": { "info.last_pulled_at": monitor_timestamp() } },
+      None,
+    )
+    .await;
+  if let Err(e) = res {
+    warn!(
+      "failed to update repo last_pulled_at | repo: {repo_name} | {e:#}",
+    );
   }
 }

bin/core/src/api/execute/server.rs

@@ -1,5 +1,4 @@
 use anyhow::Context;
-use async_trait::async_trait;
 use monitor_client::{
   api::execute::*,
   entities::{
@@ -24,7 +23,6 @@ use crate::{
   state::{action_states, State},
 };
 
-#[async_trait]
 impl Resolve<PruneDockerContainers, User> for State {
   #[instrument(name = "PruneDockerContainers", skip(self, user))]
   async fn resolve(
@@ -81,7 +79,6 @@ impl Resolve<PruneDockerContainers, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<PruneDockerNetworks, User> for State {
   #[instrument(name = "PruneDockerNetworks", skip(self, user))]
   async fn resolve(
@@ -138,7 +135,6 @@ impl Resolve<PruneDockerNetworks, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<PruneDockerImages, User> for State {
   #[instrument(name = "PruneDockerImages", skip(self, user))]
   async fn resolve(

bin/core/src/api/execute/server_template.rs

@@ -1,5 +1,4 @@
 use anyhow::{anyhow, Context};
-use axum::async_trait;
 use monitor_client::{
   api::{execute::LaunchServer, write::CreateServer},
   entities::{
@@ -16,10 +15,12 @@ use resolver_api::Resolve;
 use serror::serialize_error_pretty;
 
 use crate::{
-  cloud::aws::launch_ec2_instance, helpers::update::{add_update, make_update, update_update}, resource, state::{db_client, State}
+  cloud::{aws::launch_ec2_instance, hetzner::launch_hetzner_server},
+  helpers::update::{add_update, make_update, update_update},
+  resource,
+  state::{db_client, State},
 };
 
-#[async_trait]
 impl Resolve<LaunchServer, User> for State {
   #[instrument(name = "LaunchServer", skip(self, user))]
   async fn resolve(
@@ -66,17 +67,19 @@ impl Resolve<LaunchServer, User> for State {
     let config = match template.config {
       ServerTemplateConfig::Aws(config) => {
         let region = config.region.clone();
-        let instance = launch_ec2_instance(&name, config).await;
-        if let Err(e) = &instance {
-          update.push_error_log(
-            "launch server",
-            format!("failed to launch aws instance\n\n{e:#?}"),
-          );
-          update.finalize();
-          update_update(update.clone()).await?;
-          return Ok(update);
-        }
-        let instance = instance.unwrap();
+        let instance = match launch_ec2_instance(&name, config).await
+        {
+          Ok(instance) => instance,
+          Err(e) => {
+            update.push_error_log(
+              "launch server",
+              format!("failed to launch aws instance\n\n{e:#?}"),
+            );
+            update.finalize();
+            update_update(update.clone()).await?;
+            return Ok(update);
+          }
+        };
         update.push_simple_log(
           "launch server",
           format!(
@@ -90,6 +93,34 @@ impl Resolve<LaunchServer, User> for State {
           ..Default::default()
         }
       }
+      ServerTemplateConfig::Hetzner(config) => {
+        let datacenter = config.datacenter;
+        let server = match launch_hetzner_server(&name, config).await
+        {
+          Ok(server) => server,
+          Err(e) => {
+            update.push_error_log(
+              "launch server",
+              format!("failed to launch hetzner server\n\n{e:#?}"),
+            );
+            update.finalize();
+            update_update(update.clone()).await?;
+            return Ok(update);
+          }
+        };
+        update.push_simple_log(
+          "launch server",
+          format!(
+            "successfully launched server {name} on ip {}",
+            server.ip
+          ),
+        );
+        PartialServerConfig {
+          address: format!("http://{}:8120", server.ip).into(),
+          region: datacenter.as_ref().to_string().into(),
+          ..Default::default()
+        }
+      }
     };
 
     match self.resolve(CreateServer { name, config }, user).await {
@@ -98,6 +129,7 @@ impl Resolve<LaunchServer, User> for State {
           "create server",
           format!("created server {} ({})", server.name, server.id),
         );
+        update.other_data = server.id;
       }
       Err(e) => {
         update.push_error_log(

bin/core/src/api/read/alert.rs

@@ -1,5 +1,4 @@
 use anyhow::Context;
-use async_trait::async_trait;
 use monitor_client::{
   api::read::{
     GetAlert, GetAlertResponse, ListAlerts, ListAlertsResponse,
@@ -14,13 +13,11 @@ use mungos::{
 use resolver_api::Resolve;
 
 use crate::{
-  helpers::query::get_resource_ids_for_non_admin,
-  state::{db_client, State},
+  config::core_config, helpers::query::get_resource_ids_for_non_admin, state::{db_client, State}
 };
 
-const NUM_ALERTS_PER_PAGE: u64 = 20;
+const NUM_ALERTS_PER_PAGE: u64 = 100;
 
-#[async_trait]
 impl Resolve<ListAlerts, User> for State {
   async fn resolve(
     &self,
@@ -28,7 +25,7 @@ impl Resolve<ListAlerts, User> for State {
     user: User,
   ) -> anyhow::Result<ListAlertsResponse> {
     let mut query = query.unwrap_or_default();
-    if !user.admin {
+    if !user.admin && !core_config().transparent_mode {
       let server_ids = get_resource_ids_for_non_admin(
         &user.id,
         ResourceTargetVariant::Server,
@@ -71,7 +68,6 @@ impl Resolve<ListAlerts, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<GetAlert, User> for State {
   async fn resolve(
     &self,

bin/core/src/api/read/alerter.rs

@@ -1,7 +1,6 @@
 use std::str::FromStr;
 
 use anyhow::Context;
-use async_trait::async_trait;
 use monitor_client::{
   api::read::*,
   entities::{
@@ -15,12 +14,9 @@ use mungos::mongodb::bson::{doc, oid::ObjectId};
 use resolver_api::Resolve;
 
 use crate::{
-  helpers::query::get_resource_ids_for_non_admin,
-  resource,
-  state::{db_client, State},
+  config::core_config, helpers::query::get_resource_ids_for_non_admin, resource, state::{db_client, State}
 };
 
-#[async_trait]
 impl Resolve<GetAlerter, User> for State {
   async fn resolve(
     &self,
@@ -36,7 +32,6 @@ impl Resolve<GetAlerter, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<ListAlerters, User> for State {
   async fn resolve(
     &self,
@@ -47,14 +42,13 @@ impl Resolve<ListAlerters, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<GetAlertersSummary, User> for State {
   async fn resolve(
     &self,
     GetAlertersSummary {}: GetAlertersSummary,
     user: User,
   ) -> anyhow::Result<GetAlertersSummaryResponse> {
-    let query = if user.admin {
+    let query = if user.admin || core_config().transparent_mode {
       None
     } else {
       let ids = get_resource_ids_for_non_admin(

bin/core/src/api/read/build.rs

@@ -1,36 +1,33 @@
-use std::{collections::HashMap, str::FromStr, sync::OnceLock};
+use std::{
+  collections::{HashMap, HashSet},
+  sync::OnceLock,
+};
 
 use anyhow::Context;
 use async_timing_util::unix_timestamp_ms;
-use async_trait::async_trait;
 use futures::TryStreamExt;
 use monitor_client::{
   api::read::*,
   entities::{
-    build::{Build, BuildActionState, BuildListItem},
+    build::{Build, BuildActionState, BuildListItem, BuildState},
     permission::PermissionLevel,
-    update::{ResourceTargetVariant, UpdateStatus},
+    update::UpdateStatus,
     user::User,
     Operation,
   },
 };
 use mungos::{
   find::find_collect,
-  mongodb::{
-    bson::{doc, oid::ObjectId},
-    options::FindOptions,
-  },
+  mongodb::{bson::doc, options::FindOptions},
 };
 use resolver_api::{Resolve, ResolveToString};
 
 use crate::{
   config::core_config,
-  helpers::query::get_resource_ids_for_non_admin,
   resource,
-  state::{action_states, db_client, State},
+  state::{action_states, build_state_cache, db_client, State},
 };
 
-#[async_trait]
 impl Resolve<GetBuild, User> for State {
   async fn resolve(
     &self,
@@ -46,7 +43,6 @@ impl Resolve<GetBuild, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<ListBuilds, User> for State {
   async fn resolve(
     &self,
@@ -57,7 +53,6 @@ impl Resolve<ListBuilds, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<GetBuildActionState, User> for State {
   async fn resolve(
     &self,
@@ -80,45 +75,53 @@ impl Resolve<GetBuildActionState, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<GetBuildsSummary, User> for State {
   async fn resolve(
     &self,
     GetBuildsSummary {}: GetBuildsSummary,
     user: User,
   ) -> anyhow::Result<GetBuildsSummaryResponse> {
-    let query = if user.admin {
-      None
-    } else {
-      let ids = get_resource_ids_for_non_admin(
-        &user.id,
-        ResourceTargetVariant::Build,
-      )
-      .await?
-      .into_iter()
-      .flat_map(|id| ObjectId::from_str(&id))
-      .collect::<Vec<_>>();
-      let query = doc! {
-        "_id": { "$in": ids }
-      };
-      Some(query)
-    };
-    let total = db_client()
-      .await
-      .builds
-      .count_documents(query, None)
-      .await
-      .context("failed to count all build documents")?;
-    let res = GetBuildsSummaryResponse {
-      total: total as u32,
-    };
+    let builds = resource::list_full_for_user::<Build>(
+      Default::default(),
+      &user,
+    )
+    .await
+    .context("failed to get all builds")?;
+
+    let mut res = GetBuildsSummaryResponse::default();
+
+    let cache = build_state_cache();
+    let action_states = action_states();
+
+    for build in builds {
+      res.total += 1;
+
+      match (
+        cache.get(&build.id).await.unwrap_or_default(),
+        action_states
+          .build
+          .get(&build.id)
+          .await
+          .unwrap_or_default()
+          .get()?,
+      ) {
+        (_, action_states) if action_states.building => {
+          res.building += 1;
+        }
+        (BuildState::Ok, _) => res.ok += 1,
+        (BuildState::Failed, _) => res.failed += 1,
+        (BuildState::Unknown, _) => res.unknown += 1,
+        // will never come off the cache in the building state, since that comes from action states
+        (BuildState::Building, _) => unreachable!(),
+      }
+    }
+
     Ok(res)
   }
 }
 
 const ONE_DAY_MS: i64 = 86400000;
 
-#[async_trait]
 impl Resolve<GetBuildMonthlyStats, User> for State {
   async fn resolve(
     &self,
@@ -180,15 +183,11 @@ fn ms_to_hour(duration: i64) -> f64 {
   duration as f64 / MS_TO_HOUR_DIVISOR
 }
 
-const NUM_VERSIONS_PER_PAGE: u64 = 10;
-
-#[async_trait]
 impl Resolve<GetBuildVersions, User> for State {
   async fn resolve(
     &self,
     GetBuildVersions {
       build,
-      page,
       major,
       minor,
       patch,
@@ -224,11 +223,7 @@ impl Resolve<GetBuildVersions, User> for State {
     let versions = find_collect(
       &db_client().await.updates,
       filter,
-      FindOptions::builder()
-        .sort(doc! { "_id": -1 })
-        .limit(NUM_VERSIONS_PER_PAGE as i64)
-        .skip(page as u64 * NUM_VERSIONS_PER_PAGE)
-        .build(),
+      FindOptions::builder().sort(doc! { "_id": -1 }).build(),
     )
     .await
     .context("failed to pull versions from mongo")?
@@ -249,7 +244,6 @@ fn docker_organizations() -> &'static String {
   })
 }
 
-#[async_trait]
 impl ResolveToString<ListDockerOrganizations, User> for State {
   async fn resolve_to_string(
     &self,
@@ -259,3 +253,26 @@ impl ResolveToString<ListDockerOrganizations, User> for State {
     Ok(docker_organizations().clone())
   }
 }
+
+impl Resolve<ListCommonBuildExtraArgs, User> for State {
+  async fn resolve(
+    &self,
+    ListCommonBuildExtraArgs { query }: ListCommonBuildExtraArgs,
+    user: User,
+  ) -> anyhow::Result<ListCommonBuildExtraArgsResponse> {
+    let builds = resource::list_full_for_user::<Build>(query, &user)
+      .await
+      .context("failed to get resources matching query")?;
+
+    // first collect with guaranteed uniqueness
+    let mut res = HashSet::<String>::new();
+
+    for build in builds {
+      for extra_arg in build.config.extra_args {
+        res.insert(extra_arg);
+      }
+    }
+
+    Ok(res.into_iter().collect())
+  }
+}

bin/core/src/api/read/builder.rs

@@ -1,7 +1,6 @@
 use std::{collections::HashSet, str::FromStr};
 
 use anyhow::Context;
-use async_trait::async_trait;
 use monitor_client::{
   api::read::{self, *},
   entities::{
@@ -21,7 +20,6 @@ use crate::{
   state::{db_client, State},
 };
 
-#[async_trait]
 impl Resolve<GetBuilder, User> for State {
   async fn resolve(
     &self,
@@ -37,7 +35,6 @@ impl Resolve<GetBuilder, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<ListBuilders, User> for State {
   async fn resolve(
     &self,
@@ -48,14 +45,13 @@ impl Resolve<ListBuilders, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<GetBuildersSummary, User> for State {
   async fn resolve(
     &self,
     GetBuildersSummary {}: GetBuildersSummary,
     user: User,
   ) -> anyhow::Result<GetBuildersSummaryResponse> {
-    let query = if user.admin {
+    let query = if user.admin || core_config().transparent_mode {
       None
     } else {
       let ids = get_resource_ids_for_non_admin(
@@ -84,7 +80,6 @@ impl Resolve<GetBuildersSummary, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<GetBuilderAvailableAccounts, User> for State {
   async fn resolve(
     &self,

bin/core/src/api/read/deployment.rs

@@ -1,36 +1,28 @@
-use std::{cmp, collections::HashSet, str::FromStr};
+use std::{cmp, collections::HashSet};
 
 use anyhow::{anyhow, Context};
-use async_trait::async_trait;
 use monitor_client::{
   api::read::*,
   entities::{
     deployment::{
       Deployment, DeploymentActionState, DeploymentConfig,
-      DeploymentListItem, DockerContainerState, DockerContainerStats,
+      DeploymentListItem, DeploymentState, DockerContainerStats,
     },
     permission::PermissionLevel,
     server::Server,
-    update::{Log, ResourceTargetVariant},
+    update::Log,
     user::User,
   },
 };
-use mungos::{
-  find::find_collect,
-  mongodb::bson::{doc, oid::ObjectId},
-};
 use periphery_client::api;
 use resolver_api::Resolve;
 
 use crate::{
-  helpers::{
-    periphery_client, query::get_resource_ids_for_non_admin,
-  },
+  helpers::periphery_client,
   resource,
-  state::{action_states, db_client, deployment_status_cache, State},
+  state::{action_states, deployment_status_cache, State},
 };
 
-#[async_trait]
 impl Resolve<GetDeployment, User> for State {
   async fn resolve(
     &self,
@@ -46,7 +38,6 @@ impl Resolve<GetDeployment, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<ListDeployments, User> for State {
   async fn resolve(
     &self,
@@ -57,7 +48,6 @@ impl Resolve<ListDeployments, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<GetDeploymentContainer, User> for State {
   async fn resolve(
     &self,
@@ -84,7 +74,6 @@ impl Resolve<GetDeploymentContainer, User> for State {
 
 const MAX_LOG_LENGTH: u64 = 5000;
 
-#[async_trait]
 impl Resolve<GetLog, User> for State {
   async fn resolve(
     &self,
@@ -115,7 +104,6 @@ impl Resolve<GetLog, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<SearchLog, User> for State {
   async fn resolve(
     &self,
@@ -151,7 +139,6 @@ impl Resolve<SearchLog, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<GetDeploymentStats, User> for State {
   async fn resolve(
     &self,
@@ -179,7 +166,6 @@ impl Resolve<GetDeploymentStats, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<GetDeploymentActionState, User> for State {
   async fn resolve(
     &self,
@@ -202,34 +188,18 @@ impl Resolve<GetDeploymentActionState, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<GetDeploymentsSummary, User> for State {
   async fn resolve(
     &self,
     GetDeploymentsSummary {}: GetDeploymentsSummary,
     user: User,
   ) -> anyhow::Result<GetDeploymentsSummaryResponse> {
-    let query = if user.admin {
-      None
-    } else {
-      let ids = get_resource_ids_for_non_admin(
-        &user.id,
-        ResourceTargetVariant::Deployment,
-      )
-      .await?
-      .into_iter()
-      .flat_map(|id| ObjectId::from_str(&id))
-      .collect::<Vec<_>>();
-      let query = doc! {
-        "_id": { "$in": ids }
-      };
-      Some(query)
-    };
-
-    let deployments =
-      find_collect(&db_client().await.deployments, query, None)
-        .await
-        .context("failed to count all deployment documents")?;
+    let deployments = resource::list_full_for_user::<Deployment>(
+      Default::default(),
+      &user,
+    )
+    .await
+    .context("failed to get deployments from db")?;
     let mut res = GetDeploymentsSummaryResponse::default();
     let status_cache = deployment_status_cache();
     for deployment in deployments {
@@ -237,13 +207,13 @@ impl Resolve<GetDeploymentsSummary, User> for State {
       let status =
         status_cache.get(&deployment.id).await.unwrap_or_default();
       match status.curr.state {
-        DockerContainerState::Running => {
+        DeploymentState::Running => {
           res.running += 1;
         }
-        DockerContainerState::Unknown => {
+        DeploymentState::Unknown => {
           res.unknown += 1;
         }
-        DockerContainerState::NotDeployed => {
+        DeploymentState::NotDeployed => {
           res.not_deployed += 1;
         }
         _ => {
@@ -255,13 +225,12 @@ impl Resolve<GetDeploymentsSummary, User> for State {
   }
 }
 
-#[async_trait]
-impl Resolve<ListCommonExtraArgs, User> for State {
+impl Resolve<ListCommonDeploymentExtraArgs, User> for State {
   async fn resolve(
     &self,
-    ListCommonExtraArgs { query }: ListCommonExtraArgs,
+    ListCommonDeploymentExtraArgs { query }: ListCommonDeploymentExtraArgs,
     user: User,
-  ) -> anyhow::Result<ListCommonExtraArgsResponse> {
+  ) -> anyhow::Result<ListCommonDeploymentExtraArgsResponse> {
     let deployments =
       resource::list_full_for_user::<Deployment>(query, &user)
         .await

bin/core/src/api/read/mod.rs

@@ -1,13 +1,10 @@
 use std::time::Instant;
 
 use anyhow::anyhow;
-use async_trait::async_trait;
 use axum::{middleware, routing::post, Extension, Router};
 use axum_extra::{headers::ContentType, TypedHeader};
 use monitor_client::{api::read::*, entities::user::User};
-use resolver_api::{
-  derive::Resolver, Resolve, ResolveToString, Resolver,
-};
+use resolver_api::{derive::Resolver, Resolve, Resolver};
 use serde::{Deserialize, Serialize};
 use serror::Json;
 use typeshare::typeshare;
@@ -31,6 +28,7 @@ mod toml;
 mod update;
 mod user;
 mod user_group;
+mod variable;
 
 #[typeshare]
 #[derive(Serialize, Deserialize, Debug, Clone, Resolver)]
@@ -72,7 +70,7 @@ enum ReadRequest {
   GetServersSummary(GetServersSummary),
   GetServer(GetServer),
   ListServers(ListServers),
-  GetServerStatus(GetServerStatus),
+  GetServerState(GetServerState),
   GetPeripheryVersion(GetPeripheryVersion),
   GetDockerContainers(GetDockerContainers),
   GetDockerImages(GetDockerImages),
@@ -91,7 +89,7 @@ enum ReadRequest {
   GetDeploymentStats(GetDeploymentStats),
   GetLog(GetLog),
   SearchLog(SearchLog),
-  ListCommonExtraArgs(ListCommonExtraArgs),
+  ListCommonDeploymentExtraArgs(ListCommonDeploymentExtraArgs),
 
   // ==== BUILD ====
   GetBuildsSummary(GetBuildsSummary),
@@ -100,6 +98,7 @@ enum ReadRequest {
   GetBuildActionState(GetBuildActionState),
   GetBuildMonthlyStats(GetBuildMonthlyStats),
   GetBuildVersions(GetBuildVersions),
+  ListCommonBuildExtraArgs(ListCommonBuildExtraArgs),
   #[to_string_resolver]
   ListDockerOrganizations(ListDockerOrganizations),
 
@@ -143,6 +142,10 @@ enum ReadRequest {
   GetSystemStats(GetSystemStats),
   #[to_string_resolver]
   GetSystemProcesses(GetSystemProcesses),
+
+  // ==== VARIABLE ====
+  GetVariable(GetVariable),
+  ListVariables(ListVariables),
 }
 
 pub fn router() -> Router {
@@ -180,7 +183,6 @@ async fn handler(
   Ok((TypedHeader(ContentType::json()), res?))
 }
 
-#[async_trait]
 impl Resolve<GetVersion, User> for State {
   #[instrument(name = "GetVersion", level = "debug", skip(self))]
   async fn resolve(
@@ -194,7 +196,6 @@ impl Resolve<GetVersion, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<GetCoreInfo, User> for State {
   #[instrument(name = "GetCoreInfo", level = "debug", skip(self))]
   async fn resolve(
@@ -210,6 +211,7 @@ impl Resolve<GetCoreInfo, User> for State {
         .github_webhook_base_url
         .clone()
         .unwrap_or_else(|| config.host.clone()),
+      transparent_mode: config.transparent_mode,
     })
   }
 }

bin/core/src/api/read/permission.rs

@@ -1,5 +1,4 @@
 use anyhow::{anyhow, Context};
-use axum::async_trait;
 use monitor_client::{
   api::read::{
     GetPermissionLevel, GetPermissionLevelResponse, ListPermissions,
@@ -16,7 +15,6 @@ use crate::{
   state::{db_client, State},
 };
 
-#[async_trait]
 impl Resolve<ListPermissions, User> for State {
   async fn resolve(
     &self,
@@ -36,7 +34,6 @@ impl Resolve<ListPermissions, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<GetPermissionLevel, User> for State {
   async fn resolve(
     &self,
@@ -51,7 +48,6 @@ impl Resolve<GetPermissionLevel, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<ListUserTargetPermissions, User> for State {
   async fn resolve(
     &self,

bin/core/src/api/read/procedure.rs

@@ -1,7 +1,4 @@
-use std::str::FromStr;
-
 use anyhow::Context;
-use async_trait::async_trait;
 use monitor_client::{
   api::read::{
     GetProcedure, GetProcedureActionState,
@@ -10,20 +7,18 @@ use monitor_client::{
     ListProcedures, ListProceduresResponse,
   },
   entities::{
-    permission::PermissionLevel, procedure::Procedure,
-    update::ResourceTargetVariant, user::User,
+    permission::PermissionLevel,
+    procedure::{Procedure, ProcedureState},
+    user::User,
   },
 };
-use mungos::mongodb::bson::{doc, oid::ObjectId};
 use resolver_api::Resolve;
 
 use crate::{
-  helpers::query::get_resource_ids_for_non_admin,
   resource,
-  state::{action_states, db_client, State},
+  state::{action_states, procedure_state_cache, State},
 };
 
-#[async_trait]
 impl Resolve<GetProcedure, User> for State {
   async fn resolve(
     &self,
@@ -39,7 +34,6 @@ impl Resolve<GetProcedure, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<ListProcedures, User> for State {
   async fn resolve(
     &self,
@@ -50,43 +44,51 @@ impl Resolve<ListProcedures, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<GetProceduresSummary, User> for State {
   async fn resolve(
     &self,
     GetProceduresSummary {}: GetProceduresSummary,
     user: User,
   ) -> anyhow::Result<GetProceduresSummaryResponse> {
-    let query = if user.admin {
-      None
-    } else {
-      let ids = get_resource_ids_for_non_admin(
-        &user.id,
-        ResourceTargetVariant::Procedure,
-      )
-      .await?
-      .into_iter()
-      .flat_map(|id| ObjectId::from_str(&id))
-      .collect::<Vec<_>>();
-      let query = doc! {
-        "_id": { "$in": ids }
-      };
-      Some(query)
-    };
-    let total = db_client()
-      .await
-      .procedures
-      .count_documents(query, None)
-      .await
-      .context("failed to count all procedure documents")?;
-    let res = GetProceduresSummaryResponse {
-      total: total as u32,
-    };
+    let procedures = resource::list_full_for_user::<Procedure>(
+      Default::default(),
+      &user,
+    )
+    .await
+    .context("failed to get procedures from db")?;
+
+    let mut res = GetProceduresSummaryResponse::default();
+
+    let cache = procedure_state_cache();
+    let action_states = action_states();
+
+    for procedure in procedures {
+      res.total += 1;
+
+      match (
+        cache.get(&procedure.id).await.unwrap_or_default(),
+        action_states
+          .procedure
+          .get(&procedure.id)
+          .await
+          .unwrap_or_default()
+          .get()?,
+      ) {
+        (_, action_states) if action_states.running => {
+          res.running += 1;
+        }
+        (ProcedureState::Ok, _) => res.ok += 1,
+        (ProcedureState::Failed, _) => res.failed += 1,
+        (ProcedureState::Unknown, _) => res.unknown += 1,
+        // will never come off the cache in the running state, since that comes from action states
+        (ProcedureState::Running, _) => unreachable!(),
+      }
+    }
+
     Ok(res)
   }
 }
 
-#[async_trait]
 impl Resolve<GetProcedureActionState, User> for State {
   async fn resolve(
     &self,

bin/core/src/api/read/repo.rs

@@ -1,26 +1,19 @@
-use std::str::FromStr;
-
 use anyhow::Context;
-use async_trait::async_trait;
 use monitor_client::{
   api::read::*,
   entities::{
     permission::PermissionLevel,
-    repo::{Repo, RepoActionState, RepoListItem},
-    update::ResourceTargetVariant,
+    repo::{Repo, RepoActionState, RepoListItem, RepoState},
     user::User,
   },
 };
-use mungos::mongodb::bson::{doc, oid::ObjectId};
 use resolver_api::Resolve;
 
 use crate::{
-  helpers::query::get_resource_ids_for_non_admin,
   resource,
-  state::{action_states, db_client, State},
+  state::{action_states, repo_state_cache, State},
 };
 
-#[async_trait]
 impl Resolve<GetRepo, User> for State {
   async fn resolve(
     &self,
@@ -36,7 +29,6 @@ impl Resolve<GetRepo, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<ListRepos, User> for State {
   async fn resolve(
     &self,
@@ -47,7 +39,6 @@ impl Resolve<ListRepos, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<GetRepoActionState, User> for State {
   async fn resolve(
     &self,
@@ -70,38 +61,50 @@ impl Resolve<GetRepoActionState, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<GetReposSummary, User> for State {
   async fn resolve(
     &self,
     GetReposSummary {}: GetReposSummary,
     user: User,
   ) -> anyhow::Result<GetReposSummaryResponse> {
-    let query = if user.admin {
-      None
-    } else {
-      let ids = get_resource_ids_for_non_admin(
-        &user.id,
-        ResourceTargetVariant::Alerter,
-      )
-      .await?
-      .into_iter()
-      .flat_map(|id| ObjectId::from_str(&id))
-      .collect::<Vec<_>>();
-      let query = doc! {
-        "_id": { "$in": ids }
-      };
-      Some(query)
-    };
-    let total = db_client()
-      .await
-      .repos
-      .count_documents(query, None)
-      .await
-      .context("failed to count all build documents")?;
-    let res = GetReposSummaryResponse {
-      total: total as u32,
-    };
+    let repos =
+      resource::list_full_for_user::<Repo>(Default::default(), &user)
+        .await
+        .context("failed to get repos from db")?;
+
+    let mut res = GetReposSummaryResponse::default();
+
+    let cache = repo_state_cache();
+    let action_states = action_states();
+
+    for repo in repos {
+      res.total += 1;
+
+      match (
+        cache.get(&repo.id).await.unwrap_or_default(),
+        action_states
+          .repo
+          .get(&repo.id)
+          .await
+          .unwrap_or_default()
+          .get()?,
+      ) {
+        (_, action_states) if action_states.cloning => {
+          res.cloning += 1;
+        }
+        (_, action_states) if action_states.pulling => {
+          res.pulling += 1;
+        }
+        (RepoState::Ok, _) => res.ok += 1,
+        (RepoState::Failed, _) => res.failed += 1,
+        (RepoState::Unknown, _) => res.unknown += 1,
+        // will never come off the cache in the building state, since that comes from action states
+        (RepoState::Cloning, _) | (RepoState::Pulling, _) => {
+          unreachable!()
+        }
+      }
+    }
+
     Ok(res)
   }
 }

bin/core/src/api/read/search.rs

@@ -1,4 +1,3 @@
-use async_trait::async_trait;
 use monitor_client::{
   api::read::{FindResources, FindResourcesResponse},
   entities::{
@@ -19,7 +18,6 @@ const FIND_RESOURCE_TYPES: [ResourceTargetVariant; 5] = [
   ResourceTargetVariant::Procedure,
 ];
 
-#[async_trait]
 impl Resolve<FindResources, User> for State {
   async fn resolve(
     &self,

bin/core/src/api/read/server.rs

@@ -7,7 +7,6 @@ use anyhow::{anyhow, Context};
 use async_timing_util::{
   get_timelength_in_ms, unix_timestamp_ms, FIFTEEN_SECONDS_MS,
 };
-use async_trait::async_trait;
 use monitor_client::{
   api::read::*,
   entities::{
@@ -15,7 +14,7 @@ use monitor_client::{
     permission::PermissionLevel,
     server::{
       docker_image::ImageSummary, docker_network::DockerNetwork,
-      Server, ServerActionState, ServerListItem, ServerStatus,
+      Server, ServerActionState, ServerListItem, ServerState,
     },
     user::User,
   },
@@ -35,7 +34,6 @@ use crate::{
   state::{action_states, db_client, server_status_cache, State},
 };
 
-#[async_trait]
 impl Resolve<GetServersSummary, User> for State {
   async fn resolve(
     &self,
@@ -48,14 +46,14 @@ impl Resolve<GetServersSummary, User> for State {
     let mut res = GetServersSummaryResponse::default();
     for server in servers {
       res.total += 1;
-      match server.info.status {
-        ServerStatus::Ok => {
+      match server.info.state {
+        ServerState::Ok => {
           res.healthy += 1;
         }
-        ServerStatus::NotOk => {
+        ServerState::NotOk => {
           res.unhealthy += 1;
         }
-        ServerStatus::Disabled => {
+        ServerState::Disabled => {
           res.disabled += 1;
         }
       }
@@ -64,7 +62,6 @@ impl Resolve<GetServersSummary, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<GetPeripheryVersion, User> for State {
   async fn resolve(
     &self,
@@ -86,7 +83,6 @@ impl Resolve<GetPeripheryVersion, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<GetServer, User> for State {
   async fn resolve(
     &self,
@@ -102,7 +98,6 @@ impl Resolve<GetServer, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<ListServers, User> for State {
   async fn resolve(
     &self,
@@ -113,13 +108,12 @@ impl Resolve<ListServers, User> for State {
   }
 }
 
-#[async_trait]
-impl Resolve<GetServerStatus, User> for State {
+impl Resolve<GetServerState, User> for State {
   async fn resolve(
     &self,
-    GetServerStatus { server }: GetServerStatus,
+    GetServerState { server }: GetServerState,
     user: User,
-  ) -> anyhow::Result<GetServerStatusResponse> {
+  ) -> anyhow::Result<GetServerStateResponse> {
     let server = resource::get_check_permissions::<Server>(
       &server,
       &user,
@@ -130,14 +124,13 @@ impl Resolve<GetServerStatus, User> for State {
       .get(&server.id)
       .await
       .ok_or(anyhow!("did not find cached status for server"))?;
-    let response = GetServerStatusResponse {
-      status: status.status,
+    let response = GetServerStateResponse {
+      status: status.state,
     };
     Ok(response)
   }
 }
 
-#[async_trait]
 impl Resolve<GetServerActionState, User> for State {
   async fn resolve(
     &self,
@@ -169,7 +162,6 @@ fn system_info_cache() -> &'static SystemInfoCache {
   SYSTEM_INFO_CACHE.get_or_init(Default::default)
 }
 
-#[async_trait]
 impl ResolveToString<GetSystemInformation, User> for State {
   async fn resolve_to_string(
     &self,
@@ -205,7 +197,6 @@ impl ResolveToString<GetSystemInformation, User> for State {
   }
 }
 
-#[async_trait]
 impl ResolveToString<GetSystemStats, User> for State {
   async fn resolve_to_string(
     &self,
@@ -239,7 +230,6 @@ fn processes_cache() -> &'static ProcessesCache {
   PROCESSES_CACHE.get_or_init(Default::default)
 }
 
-#[async_trait]
 impl ResolveToString<GetSystemProcesses, User> for State {
   async fn resolve_to_string(
     &self,
@@ -276,7 +266,6 @@ impl ResolveToString<GetSystemProcesses, User> for State {
 
 const STATS_PER_PAGE: i64 = 500;
 
-#[async_trait]
 impl Resolve<GetHistoricalServerStats, User> for State {
   async fn resolve(
     &self,
@@ -330,7 +319,6 @@ impl Resolve<GetHistoricalServerStats, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<GetDockerImages, User> for State {
   async fn resolve(
     &self,
@@ -349,7 +337,6 @@ impl Resolve<GetDockerImages, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<GetDockerNetworks, User> for State {
   async fn resolve(
     &self,
@@ -368,7 +355,6 @@ impl Resolve<GetDockerNetworks, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<GetDockerContainers, User> for State {
   async fn resolve(
     &self,
@@ -387,7 +373,6 @@ impl Resolve<GetDockerContainers, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<GetAvailableAccounts, User> for State {
   async fn resolve(
     &self,
@@ -428,7 +413,6 @@ impl Resolve<GetAvailableAccounts, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<GetAvailableSecrets, User> for State {
   async fn resolve(
     &self,

bin/core/src/api/read/server_template.rs

@@ -1,7 +1,6 @@
 use std::str::FromStr;
 
 use anyhow::Context;
-use axum::async_trait;
 use monitor_client::{
   api::read::{
     GetServerTemplate, GetServerTemplateResponse,
@@ -20,7 +19,6 @@ use crate::{
   helpers::query::get_resource_ids_for_non_admin, resource, state::{db_client, State}
 };
 
-#[async_trait]
 impl Resolve<GetServerTemplate, User> for State {
   async fn resolve(
     &self,
@@ -36,7 +34,6 @@ impl Resolve<GetServerTemplate, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<ListServerTemplates, User> for State {
   async fn resolve(
     &self,
@@ -47,7 +44,6 @@ impl Resolve<ListServerTemplates, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<GetServerTemplatesSummary, User> for State {
   async fn resolve(
     &self,
@@ -72,10 +68,10 @@ impl Resolve<GetServerTemplatesSummary, User> for State {
     };
     let total = db_client()
       .await
-      .builders
+      .server_templates
       .count_documents(query, None)
       .await
-      .context("failed to count all builder documents")?;
+      .context("failed to count all server template documents")?;
     let res = GetServerTemplatesSummaryResponse {
       total: total as u32,
     };

bin/core/src/api/read/tag.rs

@@ -1,5 +1,4 @@
 use anyhow::Context;
-use async_trait::async_trait;
 use monitor_client::{
   api::read::{GetTag, ListTags},
   entities::{tag::Tag, user::User},
@@ -12,7 +11,6 @@ use crate::{
   state::{db_client, State},
 };
 
-#[async_trait]
 impl Resolve<GetTag, User> for State {
   async fn resolve(
     &self,
@@ -23,7 +21,6 @@ impl Resolve<GetTag, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<ListTags, User> for State {
   async fn resolve(
     &self,

bin/core/src/api/read/toml.rs

@@ -1,7 +1,6 @@
 use std::collections::HashMap;
 
 use anyhow::Context;
-use axum::async_trait;
 use monitor_client::{
   api::{
     execute::Execution,
@@ -38,7 +37,6 @@ use crate::{
   state::{db_client, State},
 };
 
-#[async_trait]
 impl Resolve<ExportAllResourcesToToml, User> for State {
   async fn resolve(
     &self,
@@ -125,7 +123,6 @@ impl Resolve<ExportAllResourcesToToml, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<ExportResourcesToToml, User> for State {
   async fn resolve(
     &self,

bin/core/src/api/read/update.rs

@@ -1,7 +1,6 @@
 use std::collections::HashMap;
 
 use anyhow::{anyhow, Context};
-use async_trait::async_trait;
 use monitor_client::{
   api::read::{GetUpdate, ListUpdates, ListUpdatesResponse},
   entities::{
@@ -28,21 +27,21 @@ use mungos::{
 use resolver_api::Resolve;
 
 use crate::{
+  config::core_config,
   helpers::query::get_resource_ids_for_non_admin,
   resource,
   state::{db_client, State},
 };
 
-const UPDATES_PER_PAGE: i64 = 20;
+const UPDATES_PER_PAGE: i64 = 100;
 
-#[async_trait]
 impl Resolve<ListUpdates, User> for State {
   async fn resolve(
     &self,
     ListUpdates { query, page }: ListUpdates,
     user: User,
   ) -> anyhow::Result<ListUpdatesResponse> {
-    let query = if user.admin {
+    let query = if user.admin || core_config().transparent_mode {
       query
     } else {
       let server_ids = get_resource_ids_for_non_admin(
@@ -141,6 +140,7 @@ impl Resolve<ListUpdates, User> for State {
         target: u.target,
         status: u.status,
         version: u.version,
+        other_data: u.other_data,
       }
     })
     .collect::<Vec<_>>();
@@ -155,7 +155,6 @@ impl Resolve<ListUpdates, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<GetUpdate, User> for State {
   async fn resolve(
     &self,
@@ -166,7 +165,7 @@ impl Resolve<GetUpdate, User> for State {
       .await
       .context("failed to query to db")?
       .context("no update exists with given id")?;
-    if user.admin {
+    if user.admin || core_config().transparent_mode {
       return Ok(update);
     }
     match &update.target {

bin/core/src/api/read/user.rs

@@ -1,5 +1,4 @@
 use anyhow::{anyhow, Context};
-use async_trait::async_trait;
 use monitor_client::{
   api::read::{
     GetUsername, GetUsernameResponse, ListApiKeys,
@@ -15,7 +14,6 @@ use resolver_api::Resolve;
 
 use crate::state::{db_client, State};
 
-#[async_trait]
 impl Resolve<GetUsername, User> for State {
   async fn resolve(
     &self,
@@ -27,13 +25,19 @@ impl Resolve<GetUsername, User> for State {
       .context("failed at mongo query for user")?
       .context("no user found with id")?;
 
+    let avatar = match user.config {
+      UserConfig::Github { avatar, .. } => Some(avatar),
+      UserConfig::Google { avatar, .. } => Some(avatar),
+      _ => None,
+    };
+
     Ok(GetUsernameResponse {
       username: user.username,
+      avatar,
     })
   }
 }
 
-#[async_trait]
 impl Resolve<ListUsers, User> for State {
   async fn resolve(
     &self,
@@ -52,7 +56,6 @@ impl Resolve<ListUsers, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<ListApiKeys, User> for State {
   async fn resolve(
     &self,
@@ -76,7 +79,6 @@ impl Resolve<ListApiKeys, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<ListApiKeysForServiceUser, User> for State {
   async fn resolve(
     &self,

bin/core/src/api/read/user_group.rs

@@ -1,7 +1,6 @@
 use std::str::FromStr;
 
 use anyhow::Context;
-use async_trait::async_trait;
 use monitor_client::{
   api::read::{
     GetUserGroup, GetUserGroupResponse, ListUserGroups,
@@ -17,7 +16,6 @@ use resolver_api::Resolve;
 
 use crate::state::{db_client, State};
 
-#[async_trait]
 impl Resolve<GetUserGroup, User> for State {
   async fn resolve(
     &self,
@@ -43,7 +41,6 @@ impl Resolve<GetUserGroup, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<ListUserGroups, User> for State {
   async fn resolve(
     &self,

bin/core/src/api/read/variable.rs

@@ -0,0 +1,43 @@
+use anyhow::Context;
+use monitor_client::{
+  api::read::{
+    GetVariable, GetVariableResponse, ListVariables,
+    ListVariablesResponse,
+  },
+  entities::user::User,
+};
+use mungos::find::find_collect;
+use resolver_api::Resolve;
+
+use crate::{
+  config::core_config,
+  helpers::query::get_variable,
+  state::{db_client, State},
+};
+
+impl Resolve<GetVariable, User> for State {
+  async fn resolve(
+    &self,
+    GetVariable { name }: GetVariable,
+    _: User,
+  ) -> anyhow::Result<GetVariableResponse> {
+    get_variable(&name).await
+  }
+}
+
+impl Resolve<ListVariables, User> for State {
+  async fn resolve(
+    &self,
+    ListVariables {}: ListVariables,
+    _: User,
+  ) -> anyhow::Result<ListVariablesResponse> {
+    let variables =
+      find_collect(&db_client().await.variables, None, None)
+        .await
+        .context("failed to query db for variables")?;
+    Ok(ListVariablesResponse {
+      variables,
+      secrets: core_config().secrets.keys().cloned().collect(),
+    })
+  }
+}

bin/core/src/api/write/alerter.rs

@@ -1,4 +1,3 @@
-use async_trait::async_trait;
 use monitor_client::{
   api::write::{
     CopyAlerter, CreateAlerter, DeleteAlerter, UpdateAlerter,
@@ -11,7 +10,6 @@ use resolver_api::Resolve;
 
 use crate::{resource, state::State};
 
-#[async_trait]
 impl Resolve<CreateAlerter, User> for State {
   #[instrument(name = "CreateAlerter", skip(self, user))]
   async fn resolve(
@@ -23,7 +21,6 @@ impl Resolve<CreateAlerter, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<CopyAlerter, User> for State {
   #[instrument(name = "CopyAlerter", skip(self, user))]
   async fn resolve(
@@ -44,7 +41,6 @@ impl Resolve<CopyAlerter, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<DeleteAlerter, User> for State {
   #[instrument(name = "DeleteAlerter", skip(self, user))]
   async fn resolve(
@@ -56,7 +52,6 @@ impl Resolve<DeleteAlerter, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<UpdateAlerter, User> for State {
   #[instrument(name = "UpdateAlerter", skip(self, user))]
   async fn resolve(

bin/core/src/api/write/api_key.rs

@@ -1,5 +1,4 @@
 use anyhow::{anyhow, Context};
-use async_trait::async_trait;
 use monitor_client::{
   api::write::*,
   entities::{
@@ -20,7 +19,6 @@ use crate::{
 const SECRET_LENGTH: usize = 40;
 const BCRYPT_COST: u32 = 10;
 
-#[async_trait]
 impl Resolve<CreateApiKey, User> for State {
   #[instrument(
     name = "CreateApiKey",
@@ -57,7 +55,6 @@ impl Resolve<CreateApiKey, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<DeleteApiKey, User> for State {
   #[instrument(
     name = "DeleteApiKey",
@@ -88,7 +85,6 @@ impl Resolve<DeleteApiKey, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<CreateApiKeyForServiceUser, User> for State {
   #[instrument(name = "CreateApiKeyForServiceUser", skip(self, user))]
   async fn resolve(
@@ -117,7 +113,6 @@ impl Resolve<CreateApiKeyForServiceUser, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<DeleteApiKeyForServiceUser, User> for State {
   #[instrument(name = "DeleteApiKeyForServiceUser", skip(self, user))]
   async fn resolve(

bin/core/src/api/write/build.rs

@@ -1,4 +1,3 @@
-use async_trait::async_trait;
 use monitor_client::{
   api::write::*,
   entities::{build::Build, permission::PermissionLevel, user::User},
@@ -7,7 +6,6 @@ use resolver_api::Resolve;
 
 use crate::{resource, state::State};
 
-#[async_trait]
 impl Resolve<CreateBuild, User> for State {
   #[instrument(name = "CreateBuild", skip(self, user))]
   async fn resolve(
@@ -19,7 +17,6 @@ impl Resolve<CreateBuild, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<CopyBuild, User> for State {
   #[instrument(name = "CopyBuild", skip(self, user))]
   async fn resolve(
@@ -40,7 +37,6 @@ impl Resolve<CopyBuild, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<DeleteBuild, User> for State {
   #[instrument(name = "DeleteBuild", skip(self, user))]
   async fn resolve(
@@ -52,7 +48,6 @@ impl Resolve<DeleteBuild, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<UpdateBuild, User> for State {
   #[instrument(name = "UpdateBuild", skip(self, user))]
   async fn resolve(

bin/core/src/api/write/builder.rs

@@ -1,4 +1,3 @@
-use async_trait::async_trait;
 use monitor_client::{
   api::write::*,
   entities::{
@@ -9,7 +8,6 @@ use resolver_api::Resolve;
 
 use crate::{resource, state::State};
 
-#[async_trait]
 impl Resolve<CreateBuilder, User> for State {
   #[instrument(name = "CreateBuilder", skip(self, user))]
   async fn resolve(
@@ -21,7 +19,6 @@ impl Resolve<CreateBuilder, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<CopyBuilder, User> for State {
   #[instrument(name = "CopyBuilder", skip(self, user))]
   async fn resolve(
@@ -39,7 +36,6 @@ impl Resolve<CopyBuilder, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<DeleteBuilder, User> for State {
   #[instrument(name = "DeleteBuilder", skip(self, user))]
   async fn resolve(
@@ -51,7 +47,6 @@ impl Resolve<DeleteBuilder, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<UpdateBuilder, User> for State {
   #[instrument(name = "UpdateBuilder", skip(self, user))]
   async fn resolve(

bin/core/src/api/write/deployment.rs

@@ -1,9 +1,8 @@
 use anyhow::{anyhow, Context};
-use async_trait::async_trait;
 use monitor_client::{
   api::write::*,
   entities::{
-    deployment::{Deployment, DockerContainerState},
+    deployment::{Deployment, DeploymentState},
     monitor_timestamp,
     permission::PermissionLevel,
     server::Server,
@@ -27,7 +26,6 @@ use crate::{
   state::{action_states, db_client, State},
 };
 
-#[async_trait]
 impl Resolve<CreateDeployment, User> for State {
   #[instrument(name = "CreateDeployment", skip(self, user))]
   async fn resolve(
@@ -39,7 +37,6 @@ impl Resolve<CreateDeployment, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<CopyDeployment, User> for State {
   #[instrument(name = "CopyDeployment", skip(self, user))]
   async fn resolve(
@@ -58,7 +55,6 @@ impl Resolve<CopyDeployment, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<DeleteDeployment, User> for State {
   #[instrument(name = "DeleteDeployment", skip(self, user))]
   async fn resolve(
@@ -70,7 +66,6 @@ impl Resolve<DeleteDeployment, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<UpdateDeployment, User> for State {
   #[instrument(name = "UpdateDeployment", skip(self, user))]
   async fn resolve(
@@ -82,7 +77,6 @@ impl Resolve<UpdateDeployment, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<RenameDeployment, User> for State {
   #[instrument(name = "RenameDeployment", skip(self, user))]
   async fn resolve(
@@ -112,7 +106,7 @@ impl Resolve<RenameDeployment, User> for State {
 
     let container_state = get_deployment_state(&deployment).await?;
 
-    if container_state == DockerContainerState::Unknown {
+    if container_state == DeploymentState::Unknown {
       return Err(anyhow!(
         "cannot rename deployment when container status is unknown"
       ));
@@ -132,7 +126,7 @@ impl Resolve<RenameDeployment, User> for State {
     .await
     .context("failed to update deployment name on db")?;
 
-    if container_state != DockerContainerState::NotDeployed {
+    if container_state != DeploymentState::NotDeployed {
       let server =
         resource::get::<Server>(&deployment.config.server_id).await?;
       let log = periphery_client(&server)?

bin/core/src/api/write/description.rs

@@ -1,5 +1,4 @@
 use anyhow::anyhow;
-use async_trait::async_trait;
 use monitor_client::{
   api::write::{UpdateDescription, UpdateDescriptionResponse},
   entities::{
@@ -13,7 +12,6 @@ use resolver_api::Resolve;
 
 use crate::{resource, state::State};
 
-#[async_trait]
 impl Resolve<UpdateDescription, User> for State {
   #[instrument(name = "UpdateDescription", skip(self, user))]
   async fn resolve(

bin/core/src/api/write/mod.rs

@@ -4,7 +4,7 @@ use anyhow::{anyhow, Context};
 use axum::{middleware, routing::post, Extension, Router};
 use axum_extra::{headers::ContentType, TypedHeader};
 use monitor_client::{api::write::*, entities::user::User};
-use resolver_api::{derive::Resolver, Resolve, Resolver};
+use resolver_api::{derive::Resolver, Resolver};
 use serde::{Deserialize, Serialize};
 use serror::Json;
 use typeshare::typeshare;
@@ -26,6 +26,7 @@ mod server_template;
 mod tag;
 mod user;
 mod user_group;
+mod variable;
 
 #[typeshare]
 #[derive(Serialize, Deserialize, Debug, Clone, Resolver)]
@@ -116,6 +117,12 @@ enum WriteRequest {
   DeleteTag(DeleteTag),
   RenameTag(RenameTag),
   UpdateTagsOnResource(UpdateTagsOnResource),
+
+  // ==== VARIABLE ====
+  CreateVariable(CreateVariable),
+  UpdateVariableValue(UpdateVariableValue),
+  UpdateVariableDescription(UpdateVariableDescription),
+  DeleteVariable(DeleteVariable),
 }
 
 pub fn router() -> Router {

bin/core/src/api/write/permissions.rs

@@ -1,7 +1,6 @@
 use std::str::FromStr;
 
 use anyhow::{anyhow, Context};
-use async_trait::async_trait;
 use monitor_client::{
   api::write::{
     UpdatePermissionOnTarget, UpdatePermissionOnTargetResponse,
@@ -27,7 +26,6 @@ use crate::{
   state::{db_client, State},
 };
 
-#[async_trait]
 impl Resolve<UpdateUserBasePermissions, User> for State {
   #[instrument(name = "UpdateUserBasePermissions", skip(self, admin))]
   async fn resolve(
@@ -75,7 +73,6 @@ impl Resolve<UpdateUserBasePermissions, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<UpdatePermissionOnTarget, User> for State {
   #[instrument(name = "UpdatePermissionOnTarget", skip(self, admin))]
   async fn resolve(

bin/core/src/api/write/procedure.rs

@@ -1,4 +1,3 @@
-use async_trait::async_trait;
 use monitor_client::{
   api::write::*,
   entities::{
@@ -9,7 +8,6 @@ use resolver_api::Resolve;
 
 use crate::{resource, state::State};
 
-#[async_trait]
 impl Resolve<CreateProcedure, User> for State {
   #[instrument(name = "CreateProcedure", skip(self, user))]
   async fn resolve(
@@ -21,7 +19,6 @@ impl Resolve<CreateProcedure, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<CopyProcedure, User> for State {
   #[instrument(name = "CopyProcedure", skip(self, user))]
   async fn resolve(
@@ -40,7 +37,6 @@ impl Resolve<CopyProcedure, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<UpdateProcedure, User> for State {
   #[instrument(name = "UpdateProcedure", skip(self, user))]
   async fn resolve(
@@ -52,7 +48,6 @@ impl Resolve<UpdateProcedure, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<DeleteProcedure, User> for State {
   #[instrument(name = "DeleteProcedure", skip(self, user))]
   async fn resolve(

bin/core/src/api/write/repo.rs

@@ -1,4 +1,3 @@
-use async_trait::async_trait;
 use monitor_client::{
   api::write::*,
   entities::{permission::PermissionLevel, repo::Repo, user::User},
@@ -7,7 +6,6 @@ use resolver_api::Resolve;
 
 use crate::{resource, state::State};
 
-#[async_trait]
 impl Resolve<CreateRepo, User> for State {
   #[instrument(name = "CreateRepo", skip(self, user))]
   async fn resolve(
@@ -19,7 +17,6 @@ impl Resolve<CreateRepo, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<CopyRepo, User> for State {
   #[instrument(name = "CopyRepo", skip(self, user))]
   async fn resolve(
@@ -38,7 +35,6 @@ impl Resolve<CopyRepo, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<DeleteRepo, User> for State {
   #[instrument(name = "DeleteRepo", skip(self, user))]
   async fn resolve(
@@ -50,7 +46,6 @@ impl Resolve<DeleteRepo, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<UpdateRepo, User> for State {
   #[instrument(name = "UpdateRepo", skip(self, user))]
   async fn resolve(

bin/core/src/api/write/server.rs

@@ -1,5 +1,4 @@
 use anyhow::Context;
-use async_trait::async_trait;
 use monitor_client::{
   api::write::*,
   entities::{
@@ -25,7 +24,6 @@ use crate::{
   state::{db_client, State},
 };
 
-#[async_trait]
 impl Resolve<CreateServer, User> for State {
   #[instrument(name = "CreateServer", skip(self, user))]
   async fn resolve(
@@ -37,7 +35,6 @@ impl Resolve<CreateServer, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<DeleteServer, User> for State {
   #[instrument(name = "DeleteServer", skip(self, user))]
   async fn resolve(
@@ -49,7 +46,6 @@ impl Resolve<DeleteServer, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<UpdateServer, User> for State {
   #[instrument(name = "UpdateServer", skip(self, user))]
   async fn resolve(
@@ -61,7 +57,6 @@ impl Resolve<UpdateServer, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<RenameServer, User> for State {
   #[instrument(name = "RenameServer", skip(self, user))]
   async fn resolve(
@@ -91,7 +86,6 @@ impl Resolve<RenameServer, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<CreateNetwork, User> for State {
   #[instrument(name = "CreateNetwork", skip(self, user))]
   async fn resolve(
@@ -129,7 +123,6 @@ impl Resolve<CreateNetwork, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<DeleteNetwork, User> for State {
   #[instrument(name = "DeleteNetwork", skip(self, user))]
   async fn resolve(

bin/core/src/api/write/server_template.rs

@@ -1,4 +1,3 @@
-use axum::async_trait;
 use monitor_client::{
   api::write::{
     CopyServerTemplate, CreateServerTemplate, DeleteServerTemplate,
@@ -13,7 +12,6 @@ use resolver_api::Resolve;
 
 use crate::{resource, state::State};
 
-#[async_trait]
 impl Resolve<CreateServerTemplate, User> for State {
   async fn resolve(
     &self,
@@ -24,7 +22,6 @@ impl Resolve<CreateServerTemplate, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<CopyServerTemplate, User> for State {
   async fn resolve(
     &self,
@@ -43,7 +40,6 @@ impl Resolve<CopyServerTemplate, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<DeleteServerTemplate, User> for State {
   async fn resolve(
     &self,
@@ -54,7 +50,6 @@ impl Resolve<DeleteServerTemplate, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<UpdateServerTemplate, User> for State {
   async fn resolve(
     &self,

bin/core/src/api/write/tag.rs

@@ -1,7 +1,6 @@
 use std::str::FromStr;
 
 use anyhow::{anyhow, Context};
-use async_trait::async_trait;
 use monitor_client::{
   api::write::{
     CreateTag, DeleteTag, RenameTag, UpdateTagsOnResource,
@@ -27,7 +26,6 @@ use crate::{
   state::{db_client, State},
 };
 
-#[async_trait]
 impl Resolve<CreateTag, User> for State {
   #[instrument(name = "CreateTag", skip(self, user))]
   async fn resolve(
@@ -60,7 +58,6 @@ impl Resolve<CreateTag, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<RenameTag, User> for State {
   async fn resolve(
     &self,
@@ -86,7 +83,6 @@ impl Resolve<RenameTag, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<DeleteTag, User> for State {
   #[instrument(name = "DeleteTag", skip(self, user))]
   async fn resolve(
@@ -113,7 +109,6 @@ impl Resolve<DeleteTag, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<UpdateTagsOnResource, User> for State {
   #[instrument(name = "UpdateTagsOnResource", skip(self, user))]
   async fn resolve(

bin/core/src/api/write/user.rs

@@ -1,7 +1,6 @@
 use std::{collections::VecDeque, str::FromStr};
 
 use anyhow::{anyhow, Context};
-use async_trait::async_trait;
 use monitor_client::{
   api::write::{
     CreateServiceUser, CreateServiceUserResponse, PushRecentlyViewed,
@@ -11,6 +10,7 @@ use monitor_client::{
   },
   entities::{
     monitor_timestamp,
+    update::ResourceTarget,
     user::{User, UserConfig},
   },
 };
@@ -27,7 +27,6 @@ use crate::{
 
 const RECENTLY_VIEWED_MAX: usize = 10;
 
-#[async_trait]
 impl Resolve<PushRecentlyViewed, User> for State {
   #[instrument(name = "PushRecentlyViewed", skip(self, user))]
   async fn resolve(
@@ -35,35 +34,48 @@ impl Resolve<PushRecentlyViewed, User> for State {
     PushRecentlyViewed { resource }: PushRecentlyViewed,
     user: User,
   ) -> anyhow::Result<PushRecentlyViewedResponse> {
-    let mut recently_viewed = get_user(&user.id)
-      .await?
-      .recently_viewed
+    let user = get_user(&user.id).await?;
+
+    let (recents, id, field) = match resource {
+      ResourceTarget::Server(id) => {
+        (user.recent_servers, id, "recent_servers")
+      }
+      ResourceTarget::Deployment(id) => {
+        (user.recent_deployments, id, "recent_deployments")
+      }
+      ResourceTarget::Build(id) => {
+        (user.recent_builds, id, "recent_builds")
+      }
+      ResourceTarget::Repo(id) => {
+        (user.recent_repos, id, "recent_repos")
+      }
+      ResourceTarget::Procedure(id) => {
+        (user.recent_procedures, id, "recent_procedures")
+      }
+      _ => return Ok(PushRecentlyViewedResponse {}),
+    };
+
+    let mut recents = recents
       .into_iter()
-      .filter(|r| !resource.eq(r))
+      .filter(|_id| !id.eq(_id))
       .take(RECENTLY_VIEWED_MAX - 1)
       .collect::<VecDeque<_>>();
-
-    recently_viewed.push_front(resource);
-
-    let recently_viewed = to_bson(&recently_viewed)
-      .context("failed to convert recently views to bson")?;
+    recents.push_front(id);
+    let update = doc! { field: to_bson(&recents)? };
 
     update_one_by_id(
       &db_client().await.users,
       &user.id,
-      mungos::update::Update::Set(doc! {
-        "recently_viewed": recently_viewed
-      }),
+      mungos::update::Update::Set(update),
       None,
     )
     .await
-    .context("context")?;
+    .with_context(|| format!("failed to update {field}"))?;
 
     Ok(PushRecentlyViewedResponse {})
   }
 }
 
-#[async_trait]
 impl Resolve<SetLastSeenUpdate, User> for State {
   #[instrument(name = "SetLastSeenUpdate", skip(self, user))]
   async fn resolve(
@@ -85,7 +97,6 @@ impl Resolve<SetLastSeenUpdate, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<CreateServiceUser, User> for State {
   #[instrument(name = "CreateServiceUser", skip(self, user))]
   async fn resolve(
@@ -112,7 +123,11 @@ impl Resolve<CreateServiceUser, User> for State {
       create_server_permissions: false,
       create_build_permissions: false,
       last_update_view: 0,
-      recently_viewed: Vec::new(),
+      recent_servers: Vec::new(),
+      recent_deployments: Vec::new(),
+      recent_builds: Vec::new(),
+      recent_repos: Vec::new(),
+      recent_procedures: Vec::new(),
       updated_at: monitor_timestamp(),
     };
     user.id = db_client()
@@ -129,7 +144,6 @@ impl Resolve<CreateServiceUser, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<UpdateServiceUserDescription, User> for State {
   #[instrument(
     name = "UpdateServiceUserDescription",

bin/core/src/api/write/user_group.rs

@@ -1,7 +1,6 @@
 use std::{collections::HashMap, str::FromStr};
 
 use anyhow::{anyhow, Context};
-use axum::async_trait;
 use monitor_client::{
   api::write::{
     AddUserToUserGroup, CreateUserGroup, DeleteUserGroup,
@@ -18,7 +17,6 @@ use resolver_api::Resolve;
 
 use crate::state::{db_client, State};
 
-#[async_trait]
 impl Resolve<CreateUserGroup, User> for State {
   async fn resolve(
     &self,
@@ -51,7 +49,6 @@ impl Resolve<CreateUserGroup, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<RenameUserGroup, User> for State {
   async fn resolve(
     &self,
@@ -77,7 +74,6 @@ impl Resolve<RenameUserGroup, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<DeleteUserGroup, User> for State {
   async fn resolve(
     &self,
@@ -111,7 +107,6 @@ impl Resolve<DeleteUserGroup, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<AddUserToUserGroup, User> for State {
   async fn resolve(
     &self,
@@ -155,7 +150,6 @@ impl Resolve<AddUserToUserGroup, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<RemoveUserFromUserGroup, User> for State {
   async fn resolve(
     &self,
@@ -202,7 +196,6 @@ impl Resolve<RemoveUserFromUserGroup, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<SetUsersInUserGroup, User> for State {
   async fn resolve(
     &self,

bin/core/src/api/write/variable.rs

@@ -0,0 +1,169 @@
+use anyhow::{anyhow, Context};
+use monitor_client::{
+  api::write::{
+    CreateVariable, CreateVariableResponse, DeleteVariable,
+    DeleteVariableResponse, UpdateVariableDescription,
+    UpdateVariableDescriptionResponse, UpdateVariableValue,
+    UpdateVariableValueResponse,
+  },
+  entities::{
+    update::ResourceTarget, user::User, variable::Variable, Operation,
+  },
+};
+use mungos::mongodb::bson::doc;
+use resolver_api::Resolve;
+
+use crate::{
+  helpers::{
+    query::get_variable,
+    update::{add_update, make_update},
+  },
+  state::{db_client, State},
+};
+
+impl Resolve<CreateVariable, User> for State {
+  async fn resolve(
+    &self,
+    CreateVariable {
+      name,
+      value,
+      description,
+    }: CreateVariable,
+    user: User,
+  ) -> anyhow::Result<CreateVariableResponse> {
+    if !user.admin {
+      return Err(anyhow!("only admins can create variables"));
+    }
+
+    let variable = Variable {
+      name,
+      value,
+      description,
+    };
+
+    db_client()
+      .await
+      .variables
+      .insert_one(&variable, None)
+      .await
+      .context("failed to create variable on db")?;
+
+    let mut update = make_update(
+      ResourceTarget::system(),
+      Operation::CreateVariable,
+      &user,
+    );
+
+    update
+      .push_simple_log("create variable", format!("{variable:#?}"));
+    update.finalize();
+
+    add_update(update).await?;
+
+    get_variable(&variable.name).await
+  }
+}
+
+impl Resolve<UpdateVariableValue, User> for State {
+  async fn resolve(
+    &self,
+    UpdateVariableValue { name, value }: UpdateVariableValue,
+    user: User,
+  ) -> anyhow::Result<UpdateVariableValueResponse> {
+    if !user.admin {
+      return Err(anyhow!("only admins can create variables"));
+    }
+
+    let variable = get_variable(&name).await?;
+
+    if value == variable.value {
+      return Err(anyhow!("no change"));
+    }
+
+    db_client()
+      .await
+      .variables
+      .update_one(
+        doc! { "name": &name },
+        doc! { "$set": { "value": &value } },
+        None,
+      )
+      .await
+      .context("failed to update variable value on db")?;
+
+    let mut update = make_update(
+      ResourceTarget::system(),
+      Operation::UpdateVariableValue,
+      &user,
+    );
+
+    update.push_simple_log(
+      "update variable value",
+      format!(
+        "<span class=\"text-muted-foreground\">variable</span>: '{name}'\n<span class=\"text-muted-foreground\">from</span>: <span class=\"text-red-500\">{}</span>\n<span class=\"text-muted-foreground\">to</span>:   <span class=\"text-green-500\">{value}</span>",
+        variable.value
+      ),
+    );
+    update.finalize();
+
+    add_update(update).await?;
+
+    get_variable(&name).await
+  }
+}
+
+impl Resolve<UpdateVariableDescription, User> for State {
+  async fn resolve(
+    &self,
+    UpdateVariableDescription { name, description }: UpdateVariableDescription,
+    user: User,
+  ) -> anyhow::Result<UpdateVariableDescriptionResponse> {
+    if !user.admin {
+      return Err(anyhow!("only admins can create variables"));
+    }
+    db_client()
+      .await
+      .variables
+      .update_one(
+        doc! { "name": &name },
+        doc! { "$set": { "description": &description } },
+        None,
+      )
+      .await
+      .context("failed to update variable description on db")?;
+    get_variable(&name).await
+  }
+}
+
+impl Resolve<DeleteVariable, User> for State {
+  async fn resolve(
+    &self,
+    DeleteVariable { name }: DeleteVariable,
+    user: User,
+  ) -> anyhow::Result<DeleteVariableResponse> {
+    if !user.admin {
+      return Err(anyhow!("only admins can create variables"));
+    }
+    let variable = get_variable(&name).await?;
+    db_client()
+      .await
+      .variables
+      .delete_one(doc! { "name": &name }, None)
+      .await
+      .context("failed to delete variable on db")?;
+
+    let mut update = make_update(
+      ResourceTarget::system(),
+      Operation::DeleteVariable,
+      &user,
+    );
+
+    update
+      .push_simple_log("delete variable", format!("{variable:#?}"));
+    update.finalize();
+
+    add_update(update).await?;
+
+    Ok(variable)
+  }
+}

bin/core/src/auth/github/mod.rs

@@ -86,7 +86,11 @@ async fn callback(
         create_build_permissions: no_users_exist,
         updated_at: ts,
         last_update_view: 0,
-        recently_viewed: Vec::new(),
+        recent_servers: Vec::new(),
+        recent_deployments: Vec::new(),
+        recent_builds: Vec::new(),
+        recent_repos: Vec::new(),
+        recent_procedures: Vec::new(),
         config: UserConfig::Github {
           github_id,
           avatar: github_user.avatar_url,

bin/core/src/auth/google/mod.rs

@@ -101,7 +101,11 @@ async fn callback(
         create_build_permissions: no_users_exist,
         updated_at: ts,
         last_update_view: 0,
-        recently_viewed: Vec::new(),
+        recent_servers: Vec::new(),
+        recent_deployments: Vec::new(),
+        recent_builds: Vec::new(),
+        recent_repos: Vec::new(),
+        recent_procedures: Vec::new(),
         config: UserConfig::Google {
           google_id,
           avatar: google_user.picture,

bin/core/src/auth/local.rs

@@ -2,7 +2,6 @@ use std::str::FromStr;
 
 use anyhow::{anyhow, Context};
 use async_timing_util::unix_timestamp_ms;
-use async_trait::async_trait;
 use axum::http::HeaderMap;
 use monitor_client::{
   api::auth::{
@@ -22,7 +21,6 @@ use crate::{
 
 const BCRYPT_COST: u32 = 10;
 
-#[async_trait]
 impl Resolve<CreateLocalUser, HeaderMap> for State {
   #[instrument(name = "CreateLocalUser", skip(self))]
   async fn resolve(
@@ -63,7 +61,11 @@ impl Resolve<CreateLocalUser, HeaderMap> for State {
       create_build_permissions: no_users_exist,
       updated_at: ts,
       last_update_view: 0,
-      recently_viewed: Vec::new(),
+      recent_servers: Vec::new(),
+      recent_deployments: Vec::new(),
+      recent_builds: Vec::new(),
+      recent_repos: Vec::new(),
+      recent_procedures: Vec::new(),
       config: UserConfig::Local { password },
     };
 
@@ -86,7 +88,6 @@ impl Resolve<CreateLocalUser, HeaderMap> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<LoginLocalUser, HeaderMap> for State {
   #[instrument(name = "LoginLocalUser", level = "debug", skip(self))]
   async fn resolve(

bin/core/src/auth/mod.rs

@@ -8,7 +8,9 @@ use axum::{
 use monitor_client::entities::{monitor_timestamp, user::User};
 use mungos::mongodb::bson::doc;
 use rand::{distributions::Alphanumeric, thread_rng, Rng};
+use reqwest::StatusCode;
 use serde::Deserialize;
+use serror::AddStatusCode;
 
 use crate::{
   helpers::query::get_user,
@@ -36,7 +38,9 @@ pub async fn auth_request(
   mut req: Request,
   next: Next,
 ) -> serror::Result<Response> {
-  let user = authenticate_check_enabled(&headers).await?;
+  let user = authenticate_check_enabled(&headers)
+    .await
+    .status_code(StatusCode::UNAUTHORIZED)?;
   req.extensions_mut().insert(user);
   Ok(next.run(req).await)
 }

bin/core/src/cloud/aws.rs

@@ -16,7 +16,7 @@ use monitor_client::entities::{
   alert::{Alert, AlertData, AlertDataVariant},
   monitor_timestamp,
   server::stats::SeverityLevel,
-  server_template::AwsServerTemplateConfig,
+  server_template::aws::AwsServerTemplateConfig,
   update::ResourceTarget,
 };
 
@@ -165,7 +165,7 @@ pub async fn terminate_ec2_instance_with_retry(
       }
       Err(e) => {
         if i == MAX_TERMINATION_TRIES - 1 {
-          error!("failed to terminate instance {instance_id}.");
+          error!("failed to terminate aws instance {instance_id}.");
           let alert = Alert {
             id: Default::default(),
             ts: monitor_timestamp(),
@@ -175,6 +175,7 @@ pub async fn terminate_ec2_instance_with_retry(
             variant: AlertDataVariant::AwsBuilderTerminationFailed,
             data: AlertData::AwsBuilderTerminationFailed {
               instance_id: instance_id.to_string(),
+              message: format!("{e:#}"),
             },
             resolved_ts: None,
           };
@@ -191,7 +192,7 @@ pub async fn terminate_ec2_instance_with_retry(
   unreachable!()
 }
 
-#[instrument]
+#[instrument(skip(client))]
 async fn terminate_ec2_instance_inner(
   client: &Client,
   instance_id: &str,

bin/core/src/cloud/hetzner/client.rs

@@ -0,0 +1,156 @@
+use anyhow::{anyhow, Context};
+use axum::http::{HeaderName, HeaderValue};
+use reqwest::{RequestBuilder, StatusCode};
+use serde::{de::DeserializeOwned, Serialize};
+
+use super::{
+  common::{
+    HetznerActionResponse, HetznerDatacenterResponse,
+    HetznerServerResponse, HetznerVolumeResponse,
+  },
+  create_server::{CreateServerBody, CreateServerResponse},
+  create_volume::{CreateVolumeBody, CreateVolumeResponse},
+};
+
+const BASE_URL: &str = "https://api.hetzner.cloud/v1";
+
+pub struct HetznerClient(reqwest::Client);
+
+impl HetznerClient {
+  pub fn new(token: &str) -> HetznerClient {
+    HetznerClient(
+      reqwest::ClientBuilder::new()
+        .default_headers(
+          [(
+            HeaderName::from_static("authorization"),
+            HeaderValue::from_str(&format!("Bearer {token}"))
+              .unwrap(),
+          )]
+          .into_iter()
+          .collect(),
+        )
+        .build()
+        .context("failed to build Hetzner request client")
+        .unwrap(),
+    )
+  }
+
+  pub async fn get_server(
+    &self,
+    id: i64,
+  ) -> anyhow::Result<HetznerServerResponse> {
+    self.get(&format!("/servers/{id}")).await
+  }
+
+  pub async fn create_server(
+    &self,
+    body: &CreateServerBody,
+  ) -> anyhow::Result<CreateServerResponse> {
+    self.post("/servers", body).await
+  }
+
+  pub async fn delete_server(
+    &self,
+    id: i64,
+  ) -> anyhow::Result<HetznerActionResponse> {
+    self.delete(&format!("/servers/{id}")).await
+  }
+
+  #[allow(unused)]
+  pub async fn get_volume(
+    &self,
+    id: i64,
+  ) -> anyhow::Result<HetznerVolumeResponse> {
+    self.get(&format!("/volumes/{id}")).await
+  }
+
+  pub async fn create_volume(
+    &self,
+    body: &CreateVolumeBody,
+  ) -> anyhow::Result<CreateVolumeResponse> {
+    self.post("/volumes", body).await
+  }
+
+  #[allow(unused)]
+  pub async fn delete_volume(&self, id: i64) -> anyhow::Result<()> {
+    let res = self
+      .0
+      .delete(format!("{BASE_URL}/volumes/{id}"))
+      .send()
+      .await
+      .context("failed at request to delete volume")?;
+
+    let status = res.status();
+
+    if status == StatusCode::NO_CONTENT {
+      Ok(())
+    } else {
+      let text = res
+        .text()
+        .await
+        .context("failed to get response body as text")?;
+      Err(anyhow!("{status} | {text}"))
+    }
+  }
+
+  #[allow(unused)]
+  pub async fn list_datacenters(
+    &self,
+  ) -> anyhow::Result<HetznerDatacenterResponse> {
+    self.get("/datacenters").await
+  }
+
+  async fn get<Res: DeserializeOwned>(
+    &self,
+    path: &str,
+  ) -> anyhow::Result<Res> {
+    let req = self.0.get(format!("{BASE_URL}{path}"));
+    handle_req(req).await.with_context(|| {
+      format!("failed at GET request to Hetzner | path: {path}")
+    })
+  }
+
+  async fn post<Body: Serialize, Res: DeserializeOwned>(
+    &self,
+    path: &str,
+    body: &Body,
+  ) -> anyhow::Result<Res> {
+    let req = self.0.post(format!("{BASE_URL}{path}")).json(&body);
+    handle_req(req).await.with_context(|| {
+      format!("failed at POST request to Hetzner | path: {path}")
+    })
+  }
+
+  async fn delete<Res: DeserializeOwned>(
+    &self,
+    path: &str,
+  ) -> anyhow::Result<Res> {
+    let req = self.0.delete(format!("{BASE_URL}{path}"));
+    handle_req(req).await.with_context(|| {
+      format!("failed at DELETE request to Hetzner | path: {path}")
+    })
+  }
+}
+
+async fn handle_req<Res: DeserializeOwned>(
+  req: RequestBuilder,
+) -> anyhow::Result<Res> {
+  let res = req.send().await?;
+
+  let status = res.status();
+
+  if status.is_success() {
+    res.json().await.context("failed to parse response to json")
+  } else {
+    let text = res
+      .text()
+      .await
+      .context("failed to get response body as text")?;
+    if let Ok(json_error) =
+      serde_json::from_str::<serde_json::Value>(&text)
+    {
+      return Err(anyhow!("{status} | {json_error:?}"));
+    }
+    Err(anyhow!("{status} | {text}"))
+  }
+}

bin/core/src/cloud/hetzner/common.rs

@@ -0,0 +1,277 @@
+use std::collections::HashMap;
+
+use serde::{Deserialize, Serialize};
+
+#[derive(Debug, Clone, Deserialize)]
+pub struct HetznerServerResponse {
+  pub server: HetznerServer,
+}
+
+#[derive(Debug, Clone, Deserialize)]
+pub struct HetznerServer {
+  pub id: i64,
+  pub name: String,
+  pub primary_disk_size: f64,
+  pub image: Option<HetznerImage>,
+  pub private_net: Vec<HetznerPrivateNet>,
+  pub public_net: HetznerPublicNet,
+  pub server_type: HetznerServerTypeDetails,
+  pub status: HetznerServerStatus,
+  #[serde(default)]
+  pub volumes: Vec<i64>,
+}
+
+#[derive(Debug, Clone, Deserialize)]
+pub struct HetznerServerTypeDetails {
+  pub architecture: String,
+  pub cores: i64,
+  pub cpu_type: String,
+  pub description: String,
+  pub disk: f64,
+  pub id: i64,
+  pub memory: f64,
+  pub name: String,
+}
+
+#[derive(Debug, Clone, Deserialize)]
+pub struct HetznerPrivateNet {
+  pub alias_ips: Vec<String>,
+  pub ip: String,
+  pub mac_address: String,
+  pub network: i64,
+}
+
+#[derive(Debug, Clone, Deserialize)]
+pub struct HetznerPublicNet {
+  #[serde(default)]
+  pub firewalls: Vec<HetznerFirewall>,
+  pub floating_ips: Vec<i64>,
+  pub ipv4: Option<HetznerIpv4>,
+}
+
+#[derive(Debug, Clone, Deserialize)]
+pub struct HetznerFirewall {
+  pub id: i64,
+  pub status: String,
+}
+
+#[derive(Debug, Clone, Deserialize)]
+pub struct HetznerIpv4 {
+  pub id: Option<i64>,
+  pub blocked: bool,
+  pub dns_ptr: String,
+  pub ip: String,
+}
+
+#[derive(Debug, Clone, Deserialize)]
+pub struct HetznerImage {
+  pub id: i64,
+  pub description: String,
+  pub name: Option<String>,
+  pub os_flavor: String,
+  pub os_version: Option<String>,
+  pub rapid_deploy: Option<bool>,
+}
+
+#[derive(Debug, Clone, Deserialize)]
+pub struct HetznerActionResponse {
+  pub action: HetznerAction,
+}
+
+#[derive(Debug, Clone, Deserialize)]
+pub struct HetznerAction {
+  pub command: String,
+  pub error: Option<HetznerError>,
+  pub finished: Option<String>,
+  pub id: i64,
+  pub progress: i32,
+  pub resources: Vec<HetznerResource>,
+  pub started: String,
+  pub status: HetznerActionStatus,
+}
+
+#[derive(Debug, Clone, Deserialize)]
+pub struct HetznerError {
+  pub code: String,
+  pub message: String,
+}
+
+#[derive(Debug, Clone, Deserialize)]
+pub struct HetznerResource {
+  pub id: i64,
+  #[serde(rename = "type")]
+  pub ty: String,
+}
+
+#[derive(Debug, Clone, Deserialize)]
+pub struct HetznerVolumeResponse {
+  pub volume: HetznerVolume,
+}
+
+#[derive(Debug, Clone, Deserialize)]
+pub struct HetznerVolume {
+  /// Name of the Resource. Must be unique per Project.
+  pub name: String,
+  /// Point in time when the Resource was created (in ISO-8601 format).
+  pub created: String,
+  /// Filesystem of the Volume if formatted on creation, null if not formatted on creation
+  pub format: Option<HetznerVolumeFormat>,
+  /// ID of the Volume.
+  pub id: i64,
+  /// User-defined labels ( key/value pairs) for the Resource
+  pub labels: HashMap<String, String>,
+  /// Device path on the file system for the Volume
+  pub linux_device: String,
+  /// Protection configuration for the Resource.
+  pub protection: HetznerProtection,
+  /// ID of the Server the Volume is attached to, null if it is not attached at all
+  pub server: Option<i64>,
+  /// Size in GB of the  Volume
+  pub size: i64,
+  /// Current status of the Volume. Allowed: `creating`, `available`
+  pub status: HetznerVolumeStatus,
+}
+
+#[derive(Debug, Clone, Deserialize)]
+pub struct HetznerProtection {
+  /// Prevent the Resource from being deleted.
+  pub delete: bool,
+}
+
+#[derive(Debug, Clone, Deserialize)]
+pub struct HetznerDatacenterResponse {
+  pub datacenters: Vec<HetznerDatacenterDetails>,
+}
+
+#[derive(Debug, Clone, Deserialize)]
+pub struct HetznerDatacenterDetails {
+  pub id: i64,
+  pub name: String,
+  pub location: serde_json::Map<String, serde_json::Value>,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub enum HetznerLocation {
+  #[serde(rename = "nbg1")]
+  Nuremberg1,
+  #[serde(rename = "hel1")]
+  Helsinki1,
+  #[serde(rename = "fsn1")]
+  Falkenstein1,
+  #[serde(rename = "ash")]
+  Ashburn,
+  #[serde(rename = "hil")]
+  Hillsboro,
+}
+
+#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
+pub enum HetznerDatacenter {
+  #[serde(rename = "nbg1-dc3")]
+  Nuremberg1Dc3,
+  #[serde(rename = "hel1-dc2")]
+  Helsinki1Dc2,
+  #[serde(rename = "fsn1-dc14")]
+  Falkenstein1Dc14,
+  #[serde(rename = "ash-dc1")]
+  AshburnDc1,
+  #[serde(rename = "hil-dc1")]
+  HillsboroDc1,
+}
+
+impl From<HetznerDatacenter> for HetznerLocation {
+  fn from(value: HetznerDatacenter) -> Self {
+    match value {
+      HetznerDatacenter::Nuremberg1Dc3 => HetznerLocation::Nuremberg1,
+      HetznerDatacenter::Helsinki1Dc2 => HetznerLocation::Helsinki1,
+      HetznerDatacenter::Falkenstein1Dc14 => {
+        HetznerLocation::Falkenstein1
+      }
+      HetznerDatacenter::AshburnDc1 => HetznerLocation::Ashburn,
+      HetznerDatacenter::HillsboroDc1 => HetznerLocation::Hillsboro,
+    }
+  }
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(rename_all = "lowercase")]
+pub enum HetznerVolumeFormat {
+  Xfs,
+  Ext4,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(rename_all = "lowercase")]
+pub enum HetznerVolumeStatus {
+  Creating,
+  Available,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(rename_all = "lowercase")]
+pub enum HetznerServerStatus {
+  Running,
+  Initializing,
+  Starting,
+  Stopping,
+  Off,
+  Deleting,
+  Migrating,
+  Rebuilding,
+  Unknown,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(rename_all = "lowercase")]
+pub enum HetznerActionStatus {
+  Running,
+  Success,
+  Error,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(rename_all = "UPPERCASE")]
+#[allow(clippy::enum_variant_names)]
+pub enum HetznerServerType {
+  // Shared
+  #[serde(rename = "cx11")]
+  SharedIntel1Core2Ram20Disk,
+  #[serde(rename = "cpx11")]
+  SharedAmd2Core2Ram40Disk,
+  #[serde(rename = "cax11")]
+  SharedArm2Core4Ram40Disk,
+  #[serde(rename = "cx21")]
+  SharedIntel2Core4Ram40Disk,
+  #[serde(rename = "cpx21")]
+  SharedAmd3Core4Ram80Disk,
+  #[serde(rename = "cax21")]
+  SharedArm4Core8Ram80Disk,
+  #[serde(rename = "cx31")]
+  SharedIntel2Core8Ram80Disk,
+  #[serde(rename = "cpx31")]
+  SharedAmd4Core8Ram160Disk,
+  #[serde(rename = "cax31")]
+  SharedArm8Core16Ram160Disk,
+  #[serde(rename = "cx41")]
+  SharedIntel4Core16Ram160Disk,
+  #[serde(rename = "cpx41")]
+  SharedAmd8Core16Ram240Disk,
+  #[serde(rename = "cax41")]
+  SharedArm16Core32Ram320Disk,
+  #[serde(rename = "cx51")]
+  SharedIntel8Core32Ram240Disk,
+  #[serde(rename = "cpx51")]
+  SharedAmd16Core32Ram360Disk,
+  // Dedicated
+  #[serde(rename = "ccx13")]
+  DedicatedAmd2Core8Ram80Disk,
+  #[serde(rename = "ccx23")]
+  DedicatedAmd4Core16Ram160Disk,
+  #[serde(rename = "ccx33")]
+  DedicatedAmd8Core32Ram240Disk,
+  #[serde(rename = "ccx43")]
+  DedicatedAmd16Core64Ram360Disk,
+  #[serde(rename = "ccx53")]
+  DedicatedAmd32Core128Ram600Disk,
+  #[serde(rename = "ccx63")]
+  DedicatedAmd48Core192Ram960Disk,
+}

bin/core/src/cloud/hetzner/create_server.rs

@@ -0,0 +1,76 @@
+use std::collections::HashMap;
+
+use serde::{Deserialize, Serialize};
+
+use super::common::{
+  HetznerAction, HetznerDatacenter, HetznerLocation, HetznerServer,
+  HetznerServerType,
+};
+
+#[derive(Debug, Clone, Serialize)]
+pub struct CreateServerBody {
+  /// Name of the Server to create (must be unique per Project and a valid hostname as per RFC 1123)
+  pub name: String,
+  /// Auto-mount Volumes after attach
+  #[serde(skip_serializing_if = "Option::is_none")]
+  pub automount: Option<bool>,
+  /// ID or name of Datacenter to create Server in (must not be used together with location)
+  #[serde(skip_serializing_if = "Option::is_none")]
+  pub datacenter: Option<HetznerDatacenter>,
+  /// ID or name of Location to create Server in (must not be used together with datacenter)
+  #[serde(skip_serializing_if = "Option::is_none")]
+  pub location: Option<HetznerLocation>,
+  /// Firewalls which should be applied on the Server's public network interface at creation time
+  pub firewalls: Vec<Firewall>,
+  /// ID or name of the Image the Server is created from
+  pub image: String,
+  /// User-defined labels (key-value pairs) for the Resource
+  pub labels: HashMap<String, String>,
+  /// Network IDs which should be attached to the Server private network interface at the creation time
+  pub networks: Vec<i64>,
+  /// ID of the Placement Group the server should be in
+  #[serde(skip_serializing_if = "Option::is_none")]
+  pub placement_group: Option<i64>,
+  /// Public Network options
+  #[serde(skip_serializing_if = "Option::is_none")]
+  pub public_net: Option<PublicNet>,
+  /// ID or name of the Server type this Server should be created with
+  pub server_type: HetznerServerType,
+  /// SSH key IDs ( integer ) or names ( string ) which should be injected into the Server at creation time
+  pub ssh_keys: Vec<String>,
+  /// This automatically triggers a Power on a Server-Server Action after the creation is finished and is returned in the next_actions response object.
+  pub start_after_create: bool,
+  /// Cloud-Init user data to use during Server creation. This field is limited to 32KiB.
+  #[serde(skip_serializing_if = "Option::is_none")]
+  pub user_data: Option<String>,
+  /// Volume IDs which should be attached to the Server at the creation time. Volumes must be in the same Location.
+  pub volumes: Vec<i64>,
+}
+
+#[derive(Debug, Clone, Copy, Serialize)]
+pub struct Firewall {
+  /// ID of the Firewall
+  pub firewall: i64,
+}
+
+#[derive(Debug, Clone, Copy, Serialize)]
+pub struct PublicNet {
+  /// Attach an IPv4 on the public NIC. If false, no IPv4 address will be attached.
+  pub enable_ipv4: bool,
+  /// Attach an IPv6 on the public NIC. If false, no IPv6 address will be attached.
+  pub enable_ipv6: bool,
+  /// ID of the ipv4 Primary IP to use. If omitted and enable_ipv4 is true, a new ipv4 Primary IP will automatically be created.
+  #[serde(skip_serializing_if = "Option::is_none")]
+  pub ipv4: Option<i64>,
+  /// ID of the ipv6 Primary IP to use. If omitted and enable_ipv6 is true, a new ipv6 Primary IP will automatically be created.
+  #[serde(skip_serializing_if = "Option::is_none")]
+  pub ipv6: Option<i64>,
+}
+
+#[derive(Debug, Clone, Deserialize)]
+pub struct CreateServerResponse {
+  pub action: HetznerAction,
+  pub next_actions: Vec<HetznerAction>,
+  pub root_password: Option<String>,
+  pub server: HetznerServer,
+}

bin/core/src/cloud/hetzner/create_volume.rs

@@ -0,0 +1,36 @@
+use std::collections::HashMap;
+
+use serde::{Deserialize, Serialize};
+
+use super::common::{
+  HetznerAction, HetznerLocation, HetznerVolume, HetznerVolumeFormat,
+};
+
+#[derive(Debug, Clone, Serialize)]
+pub struct CreateVolumeBody {
+  /// Name of the volume
+  pub name: String,
+  /// Auto-mount Volume after attach. server must be provided.
+  #[serde(skip_serializing_if = "Option::is_none")]
+  pub automount: Option<bool>,
+  /// Format Volume after creation. One of: xfs, ext4
+  #[serde(skip_serializing_if = "Option::is_none")]
+  pub format: Option<HetznerVolumeFormat>,
+  /// User-defined labels (key-value pairs) for the Resource
+  pub labels: HashMap<String, String>,
+  /// Location to create the Volume in (can be omitted if Server is specified)
+  #[serde(skip_serializing_if = "Option::is_none")]
+  pub location: Option<HetznerLocation>,
+  /// Server to which to attach the Volume once it's created (Volume will be created in the same Location as the server)
+  #[serde(skip_serializing_if = "Option::is_none")]
+  pub server: Option<i64>,
+  /// Size of the Volume in GB
+  pub size: i64,
+}
+
+#[derive(Debug, Clone, Deserialize)]
+pub struct CreateVolumeResponse {
+  pub action: HetznerAction,
+  pub next_actions: Vec<HetznerAction>,
+  pub volume: HetznerVolume,
+}

bin/core/src/cloud/hetzner/mod.rs

@@ -0,0 +1,318 @@
+use std::{sync::OnceLock, time::Duration};
+
+use anyhow::{anyhow, Context};
+use monitor_client::entities::{
+  alert::{Alert, AlertData, AlertDataVariant},
+  monitor_timestamp,
+  server::stats::SeverityLevel,
+  server_template::hetzner::{
+    HetznerDatacenter, HetznerServerTemplateConfig,
+    HetznerServerType, HetznerVolumeFormat,
+  },
+  update::ResourceTarget,
+};
+
+use crate::{
+  cloud::hetzner::{
+    common::HetznerServerStatus, create_server::CreateServerBody,
+    create_volume::CreateVolumeBody,
+  },
+  config::core_config,
+  helpers::alert::send_alerts,
+};
+
+use self::{
+  client::HetznerClient,
+  common::{HetznerAction, HetznerActionResponse},
+};
+
+mod client;
+mod common;
+mod create_server;
+mod create_volume;
+
+fn hetzner() -> Option<&'static HetznerClient> {
+  static HETZNER_CLIENT: OnceLock<Option<HetznerClient>> =
+    OnceLock::new();
+  HETZNER_CLIENT
+    .get_or_init(|| {
+      let token = &core_config().hetzner.token;
+      (!token.is_empty()).then(|| HetznerClient::new(token))
+    })
+    .as_ref()
+}
+
+pub struct HetznerServerMinimal {
+  pub id: i64,
+  pub ip: String,
+}
+
+const POLL_RATE_SECS: u64 = 3;
+const MAX_POLL_TRIES: usize = 100;
+
+#[instrument]
+pub async fn launch_hetzner_server(
+  name: &str,
+  config: HetznerServerTemplateConfig,
+) -> anyhow::Result<HetznerServerMinimal> {
+  let hetzner =
+    *hetzner().as_ref().context("Hetzner token not configured")?;
+  let HetznerServerTemplateConfig {
+    image,
+    automount,
+    datacenter,
+    private_network_ids,
+    placement_group,
+    enable_public_ipv4,
+    enable_public_ipv6,
+    firewall_ids,
+    server_type,
+    ssh_keys,
+    user_data,
+    use_public_ip,
+    labels,
+    volumes,
+    port: _,
+  } = config;
+  let datacenter = hetzner_datacenter(datacenter);
+
+  // Create volumes and get their ids
+  let mut volume_ids = Vec::new();
+  for volume in volumes {
+    let body = CreateVolumeBody {
+      name: volume.name,
+      format: Some(hetzner_format(volume.format)),
+      location: Some(datacenter.into()),
+      labels: volume.labels,
+      size: volume.size_gb,
+      automount: None,
+      server: None,
+    };
+    let id = hetzner
+      .create_volume(&body)
+      .await
+      .context("failed to create hetzner volume")?
+      .volume
+      .id;
+    volume_ids.push(id);
+  }
+
+  let body = CreateServerBody {
+    name: name.to_string(),
+    // only actually add automount if volumes nonempty
+    automount: (!volume_ids.is_empty()).then_some(automount),
+    datacenter: Some(datacenter),
+    location: None,
+    firewalls: firewall_ids
+      .into_iter()
+      .map(|firewall| create_server::Firewall { firewall })
+      .collect(),
+    image,
+    labels,
+    networks: private_network_ids,
+    placement_group: (placement_group > 0).then_some(placement_group),
+    public_net: (enable_public_ipv4 || enable_public_ipv6).then_some(
+      create_server::PublicNet {
+        enable_ipv4: enable_public_ipv4,
+        enable_ipv6: enable_public_ipv6,
+        ipv4: None,
+        ipv6: None,
+      },
+    ),
+    server_type: hetzner_server_type(server_type),
+    ssh_keys,
+    start_after_create: true,
+    user_data: (!user_data.is_empty()).then_some(user_data),
+    volumes: volume_ids,
+  };
+
+  let server_id = hetzner
+    .create_server(&body)
+    .await
+    .context("failed to create hetnzer server")?
+    .server
+    .id;
+
+  for _ in 0..MAX_POLL_TRIES {
+    tokio::time::sleep(Duration::from_secs(POLL_RATE_SECS)).await;
+    let Ok(res) = hetzner.get_server(server_id).await else {
+      continue;
+    };
+    if matches!(res.server.status, HetznerServerStatus::Running) {
+      let ip = if use_public_ip {
+        res
+          .server
+          .public_net
+          .ipv4
+          .context("instance does not have public ipv4 attached")?
+          .ip
+      } else {
+        res
+          .server
+          .private_net
+          .first()
+          .context("no private networks attached")?
+          .ip
+          .to_string()
+      };
+      let server = HetznerServerMinimal { id: server_id, ip };
+      return Ok(server);
+    }
+  }
+
+  Err(anyhow!(
+    "failed to verify server running after polling status"
+  ))
+}
+
+#[allow(unused)]
+const MAX_TERMINATION_TRIES: usize = 5;
+#[allow(unused)]
+const TERMINATION_WAIT_SECS: u64 = 15;
+
+#[allow(unused)]
+pub async fn terminate_hetzner_server_with_retry(
+  id: i64,
+) -> anyhow::Result<()> {
+  let hetzner =
+    *hetzner().as_ref().context("Hetzner token not configured")?;
+
+  for i in 0..MAX_TERMINATION_TRIES {
+    let message = match hetzner.delete_server(id).await {
+      Ok(HetznerActionResponse {
+        action: HetznerAction { error: None, .. },
+      }) => return Ok(()),
+      Ok(HetznerActionResponse {
+        action: HetznerAction { error: Some(e), .. },
+      }) => (i == MAX_TERMINATION_TRIES - 1).then(|| {
+        format!(
+          "failed to terminate instance | code: {} | {}",
+          e.code, e.message
+        )
+      }),
+      Err(e) => {
+        (i == MAX_TERMINATION_TRIES - 1).then(|| format!("{e:#}"))
+      }
+    };
+    if let Some(message) = message {
+      error!("failed to terminate hetzner server {id} | {message}");
+      let alert = Alert {
+        id: Default::default(),
+        ts: monitor_timestamp(),
+        resolved: false,
+        level: SeverityLevel::Critical,
+        target: ResourceTarget::system(),
+        variant: AlertDataVariant::HetznerBuilderTerminationFailed,
+        data: AlertData::HetznerBuilderTerminationFailed {
+          server_id: id,
+          message: message.clone(),
+        },
+        resolved_ts: None,
+      };
+      send_alerts(&[alert]).await;
+      return Err(anyhow::Error::msg(message));
+    }
+    tokio::time::sleep(Duration::from_secs(TERMINATION_WAIT_SECS))
+      .await;
+  }
+
+  Ok(())
+}
+
+fn hetzner_format(
+  format: HetznerVolumeFormat,
+) -> common::HetznerVolumeFormat {
+  match format {
+    HetznerVolumeFormat::Xfs => common::HetznerVolumeFormat::Xfs,
+    HetznerVolumeFormat::Ext4 => common::HetznerVolumeFormat::Ext4,
+  }
+}
+
+fn hetzner_datacenter(
+  datacenter: HetznerDatacenter,
+) -> common::HetznerDatacenter {
+  match datacenter {
+    HetznerDatacenter::Nuremberg1Dc3 => {
+      common::HetznerDatacenter::Nuremberg1Dc3
+    }
+    HetznerDatacenter::Helsinki1Dc2 => {
+      common::HetznerDatacenter::Helsinki1Dc2
+    }
+    HetznerDatacenter::Falkenstein1Dc14 => {
+      common::HetznerDatacenter::Falkenstein1Dc14
+    }
+    HetznerDatacenter::AshburnDc1 => {
+      common::HetznerDatacenter::AshburnDc1
+    }
+    HetznerDatacenter::HillsboroDc1 => {
+      common::HetznerDatacenter::HillsboroDc1
+    }
+  }
+}
+
+fn hetzner_server_type(
+  server_type: HetznerServerType,
+) -> common::HetznerServerType {
+  match server_type {
+    HetznerServerType::SharedIntel1Core2Ram20Disk => {
+      common::HetznerServerType::SharedIntel1Core2Ram20Disk
+    }
+    HetznerServerType::SharedAmd2Core2Ram40Disk => {
+      common::HetznerServerType::SharedAmd2Core2Ram40Disk
+    }
+    HetznerServerType::SharedArm2Core4Ram40Disk => {
+      common::HetznerServerType::SharedArm2Core4Ram40Disk
+    }
+    HetznerServerType::SharedIntel2Core4Ram40Disk => {
+      common::HetznerServerType::SharedIntel2Core4Ram40Disk
+    }
+    HetznerServerType::SharedAmd3Core4Ram80Disk => {
+      common::HetznerServerType::SharedAmd3Core4Ram80Disk
+    }
+    HetznerServerType::SharedArm4Core8Ram80Disk => {
+      common::HetznerServerType::SharedArm4Core8Ram80Disk
+    }
+    HetznerServerType::SharedIntel2Core8Ram80Disk => {
+      common::HetznerServerType::SharedIntel2Core8Ram80Disk
+    }
+    HetznerServerType::SharedAmd4Core8Ram160Disk => {
+      common::HetznerServerType::SharedAmd4Core8Ram160Disk
+    }
+    HetznerServerType::SharedArm8Core16Ram160Disk => {
+      common::HetznerServerType::SharedArm8Core16Ram160Disk
+    }
+    HetznerServerType::SharedIntel4Core16Ram160Disk => {
+      common::HetznerServerType::SharedIntel4Core16Ram160Disk
+    }
+    HetznerServerType::SharedAmd8Core16Ram240Disk => {
+      common::HetznerServerType::SharedAmd8Core16Ram240Disk
+    }
+    HetznerServerType::SharedArm16Core32Ram320Disk => {
+      common::HetznerServerType::SharedArm16Core32Ram320Disk
+    }
+    HetznerServerType::SharedIntel8Core32Ram240Disk => {
+      common::HetznerServerType::SharedIntel8Core32Ram240Disk
+    }
+    HetznerServerType::SharedAmd16Core32Ram360Disk => {
+      common::HetznerServerType::SharedAmd16Core32Ram360Disk
+    }
+    HetznerServerType::DedicatedAmd2Core8Ram80Disk => {
+      common::HetznerServerType::DedicatedAmd2Core8Ram80Disk
+    }
+    HetznerServerType::DedicatedAmd4Core16Ram160Disk => {
+      common::HetznerServerType::DedicatedAmd4Core16Ram160Disk
+    }
+    HetznerServerType::DedicatedAmd8Core32Ram240Disk => {
+      common::HetznerServerType::DedicatedAmd8Core32Ram240Disk
+    }
+    HetznerServerType::DedicatedAmd16Core64Ram360Disk => {
+      common::HetznerServerType::DedicatedAmd16Core64Ram360Disk
+    }
+    HetznerServerType::DedicatedAmd32Core128Ram600Disk => {
+      common::HetznerServerType::DedicatedAmd32Core128Ram600Disk
+    }
+    HetznerServerType::DedicatedAmd48Core192Ram960Disk => {
+      common::HetznerServerType::DedicatedAmd48Core192Ram960Disk
+    }
+  }
+}

bin/core/src/cloud/mod.rs

@@ -1,4 +1,5 @@
 pub mod aws;
+pub mod hetzner;
 
 #[derive(Debug)]
 pub enum BuildCleanupData {

bin/core/src/config.rs

@@ -2,7 +2,13 @@ use std::sync::OnceLock;
 
 use anyhow::Context;
 use merge_config_files::parse_config_file;
-use monitor_client::entities::config::core::{CoreConfig, Env};
+use monitor_client::entities::{
+  config::core::{
+    AwsCredentials, CoreConfig, Env, HetznerCredentials, MongoConfig,
+    OauthCredentials,
+  },
+  logger::LogConfig,
+};
 use serde::Deserialize;
 
 pub fn frontend_path() -> &'static String {
@@ -34,91 +40,112 @@ pub fn core_config() -> &'static CoreConfig {
       .context("failed to parse core Env")
       .unwrap();
     let config_path = &env.monitor_config_path;
-    let mut config =
+    let config =
       parse_config_file::<CoreConfig>(config_path.as_str())
         .unwrap_or_else(|e| {
           panic!("failed at parsing config at {config_path} | {e:#}")
         });
+    // recreating CoreConfig here makes sure we apply all env overrides.
+    CoreConfig {
+      title: env.monitor_title.unwrap_or(config.title),
+      host: env.monitor_host.unwrap_or(config.host),
+      port: env.monitor_port.unwrap_or(config.port),
+      passkey: env.monitor_passkey.unwrap_or(config.passkey),
+      jwt_valid_for: env
+        .monitor_jwt_valid_for
+        .unwrap_or(config.jwt_valid_for),
+      monitoring_interval: env
+        .monitor_monitoring_interval
+        .unwrap_or(config.monitoring_interval),
+      keep_stats_for_days: env
+        .monitor_keep_stats_for_days
+        .unwrap_or(config.keep_stats_for_days),
+      keep_alerts_for_days: env
+        .monitor_keep_alerts_for_days
+        .unwrap_or(config.keep_alerts_for_days),
+      github_webhook_secret: env
+        .monitor_github_webhook_secret
+        .unwrap_or(config.github_webhook_secret),
+      github_webhook_base_url: env
+        .monitor_github_webhook_base_url
+        .or(config.github_webhook_base_url),
+      docker_organizations: env
+        .monitor_docker_organizations
+        .unwrap_or(config.docker_organizations),
+      transparent_mode: env
+        .monitor_transparent_mode
+        .unwrap_or(config.transparent_mode),
+      local_auth: env.monitor_local_auth.unwrap_or(config.local_auth),
+      google_oauth: OauthCredentials {
+        enabled: env
+          .monitor_google_oauth_enabled
+          .unwrap_or(config.google_oauth.enabled),
+        id: env
+          .monitor_google_oauth_id
+          .unwrap_or(config.google_oauth.id),
+        secret: env
+          .monitor_google_oauth_secret
+          .unwrap_or(config.google_oauth.secret),
+      },
+      github_oauth: OauthCredentials {
+        enabled: env
+          .monitor_github_oauth_enabled
+          .unwrap_or(config.github_oauth.enabled),
+        id: env
+          .monitor_github_oauth_id
+          .unwrap_or(config.github_oauth.id),
+        secret: env
+          .monitor_github_oauth_secret
+          .unwrap_or(config.github_oauth.secret),
+      },
+      aws: AwsCredentials {
+        access_key_id: env
+          .monitor_aws_access_key_id
+          .unwrap_or(config.aws.access_key_id),
+        secret_access_key: env
+          .monitor_aws_secret_access_key
+          .unwrap_or(config.aws.secret_access_key),
+      },
+      hetzner: HetznerCredentials {
+        token: env
+          .monitor_hetzner_token
+          .unwrap_or(config.hetzner.token),
+      },
+      mongo: MongoConfig {
+        uri: env.monitor_mongo_uri.or(config.mongo.uri),
+        address: env.monitor_mongo_address.or(config.mongo.address),
+        username: env
+          .monitor_mongo_username
+          .or(config.mongo.username),
+        password: env
+          .monitor_mongo_password
+          .or(config.mongo.password),
+        app_name: env
+          .monitor_mongo_app_name
+          .unwrap_or(config.mongo.app_name),
+        db_name: env
+          .monitor_mongo_db_name
+          .unwrap_or(config.mongo.db_name),
+      },
+      logging: LogConfig {
+        level: env
+          .monitor_logging_level
+          .unwrap_or(config.logging.level),
+        stdio: env
+          .monitor_logging_stdio
+          .unwrap_or(config.logging.stdio),
+        otlp_endpoint: env
+          .monitor_logging_otlp_endpoint
+          .or(config.logging.otlp_endpoint),
+        opentelemetry_service_name: env
+          .monitor_logging_opentelemetry_service_name
+          .unwrap_or(config.logging.opentelemetry_service_name),
+      },
 
-    // Overrides
-    config.title = env.monitor_title.unwrap_or(config.title);
-    config.host = env.monitor_host.unwrap_or(config.host);
-    config.port = env.monitor_port.unwrap_or(config.port);
-    config.passkey = env.monitor_passkey.unwrap_or(config.passkey);
-    config.jwt_valid_for =
-      env.monitor_jwt_valid_for.unwrap_or(config.jwt_valid_for);
-    config.monitoring_interval = env
-      .monitor_monitoring_interval
-      .unwrap_or(config.monitoring_interval);
-    config.keep_stats_for_days = env
-      .monitor_keep_stats_for_days
-      .unwrap_or(config.keep_stats_for_days);
-    config.keep_alerts_for_days = env
-      .monitor_keep_alerts_for_days
-      .unwrap_or(config.keep_alerts_for_days);
-    config.github_webhook_secret = env
-      .monitor_github_webhook_secret
-      .unwrap_or(config.github_webhook_secret);
-    config.github_webhook_base_url = env
-      .monitor_github_webhook_base_url
-      .or(config.github_webhook_base_url);
-    config.docker_organizations = env
-      .monitor_docker_organizations
-      .unwrap_or(config.docker_organizations);
-
-    config.logging.level =
-      env.monitor_logging_level.unwrap_or(config.logging.level);
-    config.logging.stdio =
-      env.monitor_logging_stdio.unwrap_or(config.logging.stdio);
-    config.logging.otlp_endpoint = env
-      .monitor_logging_otlp_endpoint
-      .or(config.logging.otlp_endpoint);
-    config.logging.opentelemetry_service_name = env
-      .monitor_logging_opentelemetry_service_name
-      .unwrap_or(config.logging.opentelemetry_service_name);
-
-    config.local_auth =
-      env.monitor_local_auth.unwrap_or(config.local_auth);
-
-    config.github_oauth.enabled = env
-      .monitor_github_oauth_enabled
-      .unwrap_or(config.github_oauth.enabled);
-    config.github_oauth.id = env
-      .monitor_github_oauth_id
-      .unwrap_or(config.github_oauth.id);
-    config.github_oauth.secret = env
-      .monitor_github_oauth_secret
-      .unwrap_or(config.github_oauth.secret);
-
-    config.google_oauth.enabled = env
-      .monitor_google_oauth_enabled
-      .unwrap_or(config.google_oauth.enabled);
-    config.google_oauth.id = env
-      .monitor_google_oauth_id
-      .unwrap_or(config.google_oauth.id);
-    config.google_oauth.secret = env
-      .monitor_google_oauth_secret
-      .unwrap_or(config.google_oauth.secret);
-
-    config.mongo.uri = env.monitor_mongo_uri.or(config.mongo.uri);
-    config.mongo.address =
-      env.monitor_mongo_address.or(config.mongo.address);
-    config.mongo.username =
-      env.monitor_mongo_username.or(config.mongo.username);
-    config.mongo.password =
-      env.monitor_mongo_password.or(config.mongo.password);
-    config.mongo.app_name =
-      env.monitor_mongo_app_name.unwrap_or(config.mongo.app_name);
-    config.mongo.db_name =
-      env.monitor_mongo_db_name.unwrap_or(config.mongo.db_name);
-
-    config.aws.access_key_id = env
-      .monitor_aws_access_key_id
-      .unwrap_or(config.aws.access_key_id);
-    config.aws.secret_access_key = env
-      .monitor_aws_secret_access_key
-      .unwrap_or(config.aws.secret_access_key);
-
-    config
+      // These can't be overridden on env
+      secrets: config.secrets,
+      github_accounts: config.github_accounts,
+      docker_accounts: config.docker_accounts
+    }
   })
 }

bin/core/src/db.rs

@@ -1,4 +1,4 @@
-use mongo_indexed::{create_index, create_unique_index, Indexed};
+use mongo_indexed::{create_index, create_unique_index};
 use monitor_client::entities::{
   alert::Alert,
   alerter::Alerter,
@@ -16,6 +16,7 @@ use monitor_client::entities::{
   update::Update,
   user::User,
   user_group::UserGroup,
+  variable::Variable,
 };
 use mungos::{
   init::MongoBuilder,
@@ -28,6 +29,7 @@ pub struct DbClient {
   pub permissions: Collection<Permission>,
   pub api_keys: Collection<ApiKey>,
   pub tags: Collection<Tag>,
+  pub variables: Collection<Variable>,
   pub updates: Collection<Update>,
   pub alerts: Collection<Alert>,
   pub stats: Collection<SystemStatsRecord>,
@@ -80,14 +82,16 @@ impl DbClient {
     let db = client.database(db_name);
 
     let client = DbClient {
-      users: User::collection(&db, true).await?,
-      user_groups: UserGroup::collection(&db, true).await?,
-      permissions: Permission::collection(&db, true).await?,
-      api_keys: ApiKey::collection(&db, true).await?,
-      tags: Tag::collection(&db, true).await?,
-      updates: Update::collection(&db, true).await?,
-      alerts: Alert::collection(&db, true).await?,
-      stats: SystemStatsRecord::collection(&db, true).await?,
+      users: mongo_indexed::collection(&db, true).await?,
+      user_groups: mongo_indexed::collection(&db, true).await?,
+      permissions: mongo_indexed::collection(&db, true).await?,
+      api_keys: mongo_indexed::collection(&db, true).await?,
+      tags: mongo_indexed::collection(&db, true).await?,
+      variables: mongo_indexed::collection(&db, true).await?,
+      updates: mongo_indexed::collection(&db, true).await?,
+      alerts: mongo_indexed::collection(&db, true).await?,
+      stats: mongo_indexed::collection(&db, true).await?,
+      // RESOURCES
       servers: resource_collection(&db, "Server").await?,
       deployments: resource_collection(&db, "Deployment").await?,
       builds: resource_collection(&db, "Build").await?,
@@ -97,6 +101,7 @@ impl DbClient {
       procedures: resource_collection(&db, "Procedure").await?,
       server_templates: resource_collection(&db, "ServerTemplate")
         .await?,
+      //
       db,
     };
     Ok(client)

bin/core/src/helpers/alert.rs

@@ -3,7 +3,7 @@ use futures::future::join_all;
 use monitor_client::entities::{
   alert::{Alert, AlertData},
   alerter::*,
-  deployment::DockerContainerState,
+  deployment::DeploymentState,
   server::stats::SeverityLevel,
 };
 use mungos::{find::find_collect, mongodb::bson::doc};
@@ -210,13 +210,33 @@ async fn send_slack_alert(
       ];
       (text, blocks.into())
     }
-    AlertData::AwsBuilderTerminationFailed { instance_id } => {
+    AlertData::AwsBuilderTerminationFailed {
+      instance_id,
+      message,
+    } => {
       let text = format!(
         "{level} | Failed to terminated AWS builder instance"
       );
       let blocks = vec![
         Block::header(text.clone()),
-        Block::section(format!("instance id: {instance_id}")),
+        Block::section(format!(
+          "instance id: **{instance_id}**\n{message}"
+        )),
+      ];
+      (text, blocks.into())
+    }
+    AlertData::HetznerBuilderTerminationFailed {
+      server_id,
+      message,
+    } => {
+      let text = format!(
+        "{level} | Failed to terminated Hetzner builder instance"
+      );
+      let blocks = vec![
+        Block::header(text.clone()),
+        Block::section(format!(
+          "server id: **{server_id}**\n{message}"
+        )),
       ];
       (text, blocks.into())
     }
@@ -236,14 +256,12 @@ fn fmt_region(region: &Option<String>) -> String {
   }
 }
 
-fn fmt_docker_container_state(
-  state: &DockerContainerState,
-) -> String {
+fn fmt_docker_container_state(state: &DeploymentState) -> String {
   match state {
-    DockerContainerState::Running => String::from("Running ▶️"),
-    DockerContainerState::Exited => String::from("Exited 🛑"),
-    DockerContainerState::Restarting => String::from("Restarting 🔄"),
-    DockerContainerState::NotDeployed => String::from("Not Deployed"),
+    DeploymentState::Running => String::from("Running ▶️"),
+    DeploymentState::Exited => String::from("Exited 🛑"),
+    DeploymentState::Restarting => String::from("Restarting 🔄"),
+    DeploymentState::NotDeployed => String::from("Not Deployed"),
     _ => state.to_string(),
   }
 }

bin/core/src/helpers/prune.rs

@@ -1,16 +1,24 @@
+use anyhow::Context;
 use async_timing_util::{
   unix_timestamp_ms, wait_until_timelength, Timelength, ONE_DAY_MS,
 };
-use mungos::mongodb::bson::doc;
+use futures::future::join_all;
+use mungos::{find::find_collect, mongodb::bson::doc};
+use periphery_client::api::build::PruneImages;
 
 use crate::{config::core_config, state::db_client};
 
+use super::periphery_client;
+
 pub fn spawn_prune_loop() {
   tokio::spawn(async move {
     loop {
       wait_until_timelength(Timelength::OneDay, 5000).await;
-      let (stats_res, alerts_res) =
-        tokio::join!(prune_stats(), prune_alerts());
+      let (images_res, stats_res, alerts_res) =
+        tokio::join!(prune_images(), prune_stats(), prune_alerts());
+      if let Err(e) = images_res {
+        error!("error in pruning images | {e:#}");
+      }
       if let Err(e) = stats_res {
         error!("error in pruning stats | {e:#}");
       }
@@ -21,6 +29,35 @@ pub fn spawn_prune_loop() {
   });
 }
 
+async fn prune_images() -> anyhow::Result<()> {
+  let futures = find_collect(&db_client().await.servers, None, None)
+    .await
+    .context("failed to get servers from db")?
+    .into_iter()
+    // This could be done in the mongo query, but rather have rust type system guarantee this.
+    .filter(|server| server.config.auto_prune)
+    .map(|server| async move {
+      (
+        async {
+          periphery_client(&server)?.request(PruneImages {}).await
+        }
+        .await,
+        server,
+      )
+    });
+
+  for (res, server) in join_all(futures).await {
+    if let Err(e) = res {
+      error!(
+        "failed to prune images on server {} ({}) | {e:#}",
+        server.name, server.id
+      )
+    }
+  }
+
+  Ok(())
+}
+
 async fn prune_stats() -> anyhow::Result<()> {
   if core_config().keep_stats_for_days == 0 {
     return Ok(());

bin/core/src/helpers/query.rs

@@ -1,13 +1,17 @@
-use std::{collections::HashSet, str::FromStr};
+use std::{
+  collections::{HashMap, HashSet},
+  str::FromStr,
+};
 
 use anyhow::{anyhow, Context};
 use monitor_client::entities::{
-  deployment::{Deployment, DockerContainerState},
+  deployment::{Deployment, DeploymentState},
   permission::PermissionLevel,
-  server::{Server, ServerStatus},
+  server::{Server, ServerState},
   tag::Tag,
   update::ResourceTargetVariant,
   user::{admin_service_user, User},
+  variable::Variable,
 };
 use mungos::{
   by_id::find_one_by_id,
@@ -15,7 +19,7 @@ use mungos::{
   mongodb::bson::{doc, oid::ObjectId, Document},
 };
 
-use crate::{resource, state::db_client};
+use crate::{config::core_config, resource, state::db_client};
 
 #[instrument(level = "debug")]
 pub async fn get_user(user_id: &str) -> anyhow::Result<User> {
@@ -31,17 +35,17 @@ pub async fn get_user(user_id: &str) -> anyhow::Result<User> {
 #[instrument(level = "debug")]
 pub async fn get_server_with_status(
   server_id_or_name: &str,
-) -> anyhow::Result<(Server, ServerStatus)> {
+) -> anyhow::Result<(Server, ServerState)> {
   let server = resource::get::<Server>(server_id_or_name).await?;
   if !server.config.enabled {
-    return Ok((server, ServerStatus::Disabled));
+    return Ok((server, ServerState::Disabled));
   }
   let status = match super::periphery_client(&server)?
     .request(periphery_client::api::GetHealth {})
     .await
   {
-    Ok(_) => ServerStatus::Ok,
-    Err(_) => ServerStatus::NotOk,
+    Ok(_) => ServerState::Ok,
+    Err(_) => ServerState::NotOk,
   };
   Ok((server, status))
 }
@@ -49,14 +53,14 @@ pub async fn get_server_with_status(
 #[instrument(level = "debug")]
 pub async fn get_deployment_state(
   deployment: &Deployment,
-) -> anyhow::Result<DockerContainerState> {
+) -> anyhow::Result<DeploymentState> {
   if deployment.config.server_id.is_empty() {
-    return Ok(DockerContainerState::NotDeployed);
+    return Ok(DeploymentState::NotDeployed);
   }
   let (server, status) =
     get_server_with_status(&deployment.config.server_id).await?;
-  if status != ServerStatus::Ok {
-    return Ok(DockerContainerState::Unknown);
+  if status != ServerState::Ok {
+    return Ok(DeploymentState::Unknown);
   }
   let container = super::periphery_client(&server)?
     .request(periphery_client::api::container::GetContainerList {})
@@ -66,7 +70,7 @@ pub async fn get_deployment_state(
 
   let state = match container {
     Some(container) => container.state,
-    None => DockerContainerState::NotDeployed,
+    None => DeploymentState::NotDeployed,
   };
 
   Ok(state)
@@ -146,6 +150,11 @@ pub async fn get_user_permission_on_resource(
   resource_variant: ResourceTargetVariant,
   resource_id: &str,
 ) -> anyhow::Result<PermissionLevel> {
+  let lowest_permission = if core_config().transparent_mode {
+    PermissionLevel::Read
+  } else {
+    PermissionLevel::None
+  };
   let permission = find_collect(
     &db_client().await.permissions,
     doc! {
@@ -159,7 +168,7 @@ pub async fn get_user_permission_on_resource(
   .context("failed to query db for permissions")?
   .into_iter()
   // get the max permission user has between personal / any user groups
-  .fold(PermissionLevel::None, |level, permission| {
+  .fold(lowest_permission, |level, permission| {
     if permission.level > level {
       permission.level
     } else {
@@ -198,3 +207,27 @@ pub fn id_or_name_filter(id_or_name: &str) -> Document {
     Err(_) => doc! { "name": id_or_name },
   }
 }
+
+pub async fn get_global_variables(
+) -> anyhow::Result<HashMap<String, String>> {
+  Ok(
+    find_collect(&db_client().await.variables, None, None)
+      .await
+      .context("failed to get all variables from db")?
+      .into_iter()
+      .map(|variable| (variable.name, variable.value))
+      .collect(),
+  )
+}
+
+pub async fn get_variable(name: &str) -> anyhow::Result<Variable> {
+  db_client()
+    .await
+    .variables
+    .find_one(doc! { "name": &name }, None)
+    .await
+    .context("failed at call to db")?
+    .with_context(|| {
+      format!("no variable found with given name: {name}")
+    })
+}

bin/core/src/helpers/update.rs

@@ -83,6 +83,7 @@ async fn update_list_item(
     target: update.target,
     status: update.status,
     version: update.version,
+    other_data: update.other_data,
     username,
   };
   Ok(update)

bin/core/src/listener/github.rs

@@ -6,7 +6,9 @@ use hex::ToHex;
 use hmac::{Hmac, Mac};
 use monitor_client::{
   api::execute,
-  entities::{build::Build, repo::Repo, user::github_user},
+  entities::{
+    build::Build, procedure::Procedure, repo::Repo, user::github_user,
+  },
 };
 use resolver_api::Resolve;
 use serde::Deserialize;
@@ -129,6 +131,9 @@ async fn handle_build_webhook(
   verify_gh_signature(headers, &body).await?;
   let request_branch = extract_branch(&body)?;
   let build = resource::get::<Build>(&build_id).await?;
+  if !build.config.webhook_enabled {
+    return Err(anyhow!("build does not have webhook enabled"));
+  }
   if request_branch != build.config.branch {
     return Err(anyhow!("request branch does not match expected"));
   }
@@ -155,6 +160,9 @@ async fn handle_repo_clone_webhook(
   verify_gh_signature(headers, &body).await?;
   let request_branch = extract_branch(&body)?;
   let repo = resource::get::<Repo>(&repo_id).await?;
+  if !repo.config.webhook_enabled {
+    return Err(anyhow!("repo does not have webhook enabled"));
+  }
   if request_branch != repo.config.branch {
     return Err(anyhow!("request branch does not match expected"));
   }
@@ -181,6 +189,9 @@ async fn handle_repo_pull_webhook(
   verify_gh_signature(headers, &body).await?;
   let request_branch = extract_branch(&body)?;
   let repo = resource::get::<Repo>(&repo_id).await?;
+  if !repo.config.webhook_enabled {
+    return Err(anyhow!("repo does not have webhook enabled"));
+  }
   if request_branch != repo.config.branch {
     return Err(anyhow!("request branch does not match expected"));
   }
@@ -211,6 +222,10 @@ async fn handle_procedure_webhook(
   if request_branch != target_branch {
     return Err(anyhow!("request branch does not match expected"));
   }
+  let procedure = resource::get::<Procedure>(&procedure_id).await?;
+  if !procedure.config.webhook_enabled {
+    return Err(anyhow!("procedure does not have webhook enabled"));
+  }
   State
     .resolve(
       execute::RunProcedure {

bin/core/src/main.rs

@@ -33,8 +33,11 @@ async fn app() -> anyhow::Result<()> {
   info!("config: {:?}", config.sanitized());
 
   // Spawn monitoring loops
-  monitor::spawn_monitor_loop();
+  monitor::spawn_monitor_loop()?;
   helpers::prune::spawn_prune_loop();
+  resource::spawn_build_state_refresh_loop();
+  resource::spawn_repo_state_refresh_loop();
+  resource::spawn_procedure_state_refresh_loop();
 
   // Setup static frontend services
   let frontend_path = frontend_path();

bin/core/src/monitor/alert/server.rs

@@ -5,7 +5,7 @@ use mongo_indexed::Indexed;
 use monitor_client::entities::{
   alert::{Alert, AlertData, AlertDataVariant},
   monitor_timestamp, optional_string,
-  server::{stats::SeverityLevel, ServerListItem, ServerStatus},
+  server::{stats::SeverityLevel, ServerListItem, ServerState},
   update::ResourceTarget,
 };
 use mungos::{
@@ -56,8 +56,8 @@ pub async fn alert_servers(
     let health_alert = server_alerts.as_ref().and_then(|alerts| {
       alerts.get(&AlertDataVariant::ServerUnreachable)
     });
-    match (server_status.status, health_alert) {
-      (ServerStatus::NotOk, None) => {
+    match (server_status.state, health_alert) {
+      (ServerState::NotOk, None) => {
         // open unreachable alert
         let alert = Alert {
           id: Default::default(),
@@ -77,7 +77,7 @@ pub async fn alert_servers(
         alerts_to_open
           .push((alert, server.info.send_unreachable_alerts))
       }
-      (ServerStatus::NotOk, Some(alert)) => {
+      (ServerState::NotOk, Some(alert)) => {
         // update alert err
         let mut alert = alert.clone();
         let (id, name, region) = match alert.data {
@@ -102,7 +102,7 @@ pub async fn alert_servers(
 
       // Close an open alert
       (
-        ServerStatus::Ok | ServerStatus::Disabled,
+        ServerState::Ok | ServerState::Disabled,
         Some(health_alert),
       ) => alert_ids_to_close.push((
         health_alert.id.clone(),

bin/core/src/monitor/helpers.rs

@@ -1,17 +1,23 @@
 use monitor_client::entities::{
-  deployment::{Deployment, DockerContainerState},
+  deployment::{Deployment, DeploymentState},
+  repo::Repo,
   server::{
     stats::{
       ServerHealth, SeverityLevel, SingleDiskUsage, SystemStats,
     },
-    Server, ServerConfig, ServerStatus,
+    Server, ServerConfig, ServerState,
   },
 };
 use serror::Serror;
 
-use crate::state::{deployment_status_cache, server_status_cache};
+use crate::state::{
+  deployment_status_cache, repo_status_cache, server_status_cache,
+};
 
-use super::{CachedDeploymentStatus, CachedServerStatus, History};
+use super::{
+  CachedDeploymentStatus, CachedRepoStatus, CachedServerStatus,
+  History,
+};
 
 #[instrument(level = "debug", skip_all)]
 pub async fn insert_deployments_status_unknown(
@@ -27,7 +33,7 @@ pub async fn insert_deployments_status_unknown(
         History {
           curr: CachedDeploymentStatus {
             id: deployment.id,
-            state: DockerContainerState::Unknown,
+            state: DeploymentState::Unknown,
             container: None,
           },
           prev,
@@ -38,10 +44,28 @@ pub async fn insert_deployments_status_unknown(
   }
 }
 
+#[instrument(level = "debug", skip_all)]
+pub async fn insert_repos_status_unknown(repos: Vec<Repo>) {
+  let status_cache = repo_status_cache();
+  for repo in repos {
+    status_cache
+      .insert(
+        repo.id.clone(),
+        CachedRepoStatus {
+          id: repo.id,
+          latest_hash: None,
+          latest_message: None,
+        }
+        .into(),
+      )
+      .await;
+  }
+}
+
 #[instrument(level = "debug", skip_all)]
 pub async fn insert_server_status(
   server: &Server,
-  status: ServerStatus,
+  state: ServerState,
   version: String,
   stats: Option<SystemStats>,
   err: impl Into<Option<Serror>>,
@@ -52,7 +76,7 @@ pub async fn insert_server_status(
       server.id.clone(),
       CachedServerStatus {
         id: server.id.clone(),
-        status,
+        state,
         version,
         stats,
         health,
@@ -101,6 +125,7 @@ fn get_server_health(
     mount,
     used_gb,
     total_gb,
+    ..
   } in disks
   {
     let perc = 100.0 * used_gb / total_gb;

bin/core/src/monitor/mod.rs

@@ -1,24 +1,26 @@
-use async_timing_util::{wait_until_timelength, Timelength};
+use async_timing_util::wait_until_timelength;
 use futures::future::join_all;
 use monitor_client::entities::{
-  deployment::{ContainerSummary, DockerContainerState},
+  deployment::{ContainerSummary, DeploymentState},
   server::{
     stats::{ServerHealth, SystemStats},
-    Server, ServerStatus,
+    Server, ServerState,
   },
 };
 use mungos::{find::find_collect, mongodb::bson::doc};
-use periphery_client::api;
+use periphery_client::api::{self, git::GetLatestCommit};
 use serror::Serror;
 
 use crate::{
+  config::core_config,
   helpers::periphery_client,
   monitor::{alert::check_alerts, record::record_server_stats},
-  state::{db_client, deployment_status_cache},
+  state::{db_client, deployment_status_cache, repo_status_cache},
 };
 
 use self::helpers::{
-  insert_deployments_status_unknown, insert_server_status,
+  insert_deployments_status_unknown, insert_repos_status_unknown,
+  insert_server_status,
 };
 
 mod alert;
@@ -34,7 +36,7 @@ pub struct History<Curr: Default, Prev> {
 #[derive(Default, Clone, Debug)]
 pub struct CachedServerStatus {
   pub id: String,
-  pub status: ServerStatus,
+  pub state: ServerState,
   pub version: String,
   pub stats: Option<SystemStats>,
   pub health: Option<ServerHealth>,
@@ -44,16 +46,24 @@ pub struct CachedServerStatus {
 #[derive(Default, Clone, Debug)]
 pub struct CachedDeploymentStatus {
   pub id: String,
-  pub state: DockerContainerState,
+  pub state: DeploymentState,
   pub container: Option<ContainerSummary>,
 }
 
-pub fn spawn_monitor_loop() {
+#[derive(Default, Clone, Debug)]
+pub struct CachedRepoStatus {
+  pub id: String,
+  pub latest_hash: Option<String>,
+  pub latest_message: Option<String>,
+}
+
+pub fn spawn_monitor_loop() -> anyhow::Result<()> {
+  let interval: async_timing_util::Timelength =
+    core_config().monitoring_interval.try_into()?;
   tokio::spawn(async move {
     loop {
-      let ts = (wait_until_timelength(Timelength::FiveSeconds, 500)
-        .await
-        - 500) as i64;
+      let ts =
+        (wait_until_timelength(interval, 500).await - 500) as i64;
       let servers =
         match find_collect(&db_client().await.servers, None, None)
           .await
@@ -73,6 +83,7 @@ pub fn spawn_monitor_loop() {
       tokio::join!(check_alerts(ts), record_server_stats(ts));
     }
   });
+  Ok(())
 }
 
 #[instrument(level = "debug")]
@@ -87,15 +98,31 @@ pub async fn update_cache_for_server(server: &Server) {
     Ok(deployments) => deployments,
     Err(e) => {
       error!("failed to get deployments list from mongo (update status cache) | server id: {} | {e:#}", server.id);
-      return;
+      Vec::new()
     }
   };
 
+  let repos = match find_collect(
+    &db_client().await.repos,
+    doc! { "config.server_id": &server.id },
+    None,
+  )
+  .await
+  {
+    Ok(repos) => repos,
+    Err(e) => {
+      error!("failed to get repos list from mongo (update status cache) | server id: {} | {e:#}", server.id);
+      Vec::new()
+    }
+  };
+
+  // Handle server disabled
   if !server.config.enabled {
     insert_deployments_status_unknown(deployments).await;
+    insert_repos_status_unknown(repos).await;
     insert_server_status(
       server,
-      ServerStatus::Disabled,
+      ServerState::Disabled,
       String::from("unknown"),
       None,
       None,
@@ -103,16 +130,22 @@ pub async fn update_cache_for_server(server: &Server) {
     .await;
     return;
   }
-  // already handle server disabled case above, so using unwrap here
-  let periphery = periphery_client(server).unwrap();
+
+  let Ok(periphery) = periphery_client(server) else {
+    error!(
+      "somehow periphery not ok to create. should not be reached."
+    );
+    return;
+  };
 
   let version = match periphery.request(api::GetVersion {}).await {
     Ok(version) => version.version,
     Err(e) => {
       insert_deployments_status_unknown(deployments).await;
+      insert_repos_status_unknown(repos).await;
       insert_server_status(
         server,
-        ServerStatus::NotOk,
+        ServerState::NotOk,
         String::from("unknown"),
         None,
         Serror::from(&e),
@@ -127,9 +160,10 @@ pub async fn update_cache_for_server(server: &Server) {
       Ok(stats) => Some(stats),
       Err(e) => {
         insert_deployments_status_unknown(deployments).await;
+        insert_repos_status_unknown(repos).await;
         insert_server_status(
           server,
-          ServerStatus::NotOk,
+          ServerState::NotOk,
           String::from("unknown"),
           None,
           Serror::from(&e),
@@ -142,45 +176,64 @@ pub async fn update_cache_for_server(server: &Server) {
     None
   };
 
-  insert_server_status(
-    server,
-    ServerStatus::Ok,
-    version,
-    stats,
-    None,
-  )
-  .await;
+  insert_server_status(server, ServerState::Ok, version, stats, None)
+    .await;
 
-  let containers =
-    periphery.request(api::container::GetContainerList {}).await;
-  if containers.is_err() {
-    insert_deployments_status_unknown(deployments).await;
-    return;
-  }
+  match periphery.request(api::container::GetContainerList {}).await {
+    Ok(containers) => {
+      let status_cache = deployment_status_cache();
+      for deployment in deployments {
+        let container = containers
+          .iter()
+          .find(|c| c.name == deployment.name)
+          .cloned();
+        let prev = status_cache
+          .get(&deployment.id)
+          .await
+          .map(|s| s.curr.state);
+        let state = container
+          .as_ref()
+          .map(|c| c.state)
+          .unwrap_or(DeploymentState::NotDeployed);
+        status_cache
+          .insert(
+            deployment.id.clone(),
+            History {
+              curr: CachedDeploymentStatus {
+                id: deployment.id,
+                state,
+                container,
+              },
+              prev,
+            }
+            .into(),
+          )
+          .await;
+      }
+    }
+    Err(e) => {
+      warn!("could not get containers list | {e:#}");
+      insert_deployments_status_unknown(deployments).await;
+    }
+  };
 
-  let containers = containers.unwrap();
-  let status_cache = deployment_status_cache();
-  for deployment in deployments {
-    let container = containers
-      .iter()
-      .find(|c| c.name == deployment.name)
-      .cloned();
-    let prev =
-      status_cache.get(&deployment.id).await.map(|s| s.curr.state);
-    let state = container
-      .as_ref()
-      .map(|c| c.state)
-      .unwrap_or(DockerContainerState::NotDeployed);
+  let status_cache = repo_status_cache();
+  for repo in repos {
+    let (latest_hash, latest_message) = periphery
+      .request(GetLatestCommit {
+        name: repo.name.clone(),
+      })
+      .await
+      .map(|r| (r.hash, r.message))
+      .ok()
+      .unzip();
     status_cache
       .insert(
-        deployment.id.clone(),
-        History {
-          curr: CachedDeploymentStatus {
-            id: deployment.id,
-            state,
-            container,
-          },
-          prev,
+        repo.id.clone(),
+        CachedRepoStatus {
+          id: repo.id,
+          latest_hash,
+          latest_message,
         }
         .into(),
       )

bin/core/src/resource/alerter.rs

@@ -35,7 +35,7 @@ impl super::MonitorResource for Alerter {
 
   async fn to_list_item(
     alerter: Resource<Self::Config, Self::Info>,
-  ) -> anyhow::Result<Self::ListItem> {
+  ) -> Self::ListItem {
     let (alerter_type, enabled) = match alerter.config {
       AlerterConfig::Custom(config) => {
         (AlerterConfigVariant::Custom.to_string(), config.enabled)
@@ -44,7 +44,7 @@ impl super::MonitorResource for Alerter {
         (AlerterConfigVariant::Slack.to_string(), config.enabled)
       }
     };
-    Ok(AlerterListItem {
+    AlerterListItem {
       name: alerter.name,
       id: alerter.id,
       tags: alerter.tags,
@@ -54,7 +54,7 @@ impl super::MonitorResource for Alerter {
         is_default: alerter.info.is_default,
         enabled,
       },
-    })
+    }
   }
 
   async fn busy(_id: &String) -> anyhow::Result<bool> {

bin/core/src/resource/build.rs

@@ -1,8 +1,11 @@
+use std::time::Duration;
+
 use anyhow::Context;
 use monitor_client::entities::{
   build::{
     Build, BuildConfig, BuildConfigDiff, BuildInfo, BuildListItem,
-    BuildListItemInfo, BuildQuerySpecifics, PartialBuildConfig,
+    BuildListItemInfo, BuildQuerySpecifics, BuildState,
+    PartialBuildConfig,
   },
   builder::Builder,
   permission::PermissionLevel,
@@ -11,11 +14,14 @@ use monitor_client::entities::{
   user::User,
   Operation,
 };
-use mungos::mongodb::Collection;
+use mungos::{
+  find::find_collect,
+  mongodb::{bson::doc, options::FindOneOptions, Collection},
+};
 
 use crate::{
   helpers::empty_or_only_spaces,
-  state::{action_states, db_client},
+  state::{action_states, build_state_cache, db_client},
 };
 
 impl super::MonitorResource for Build {
@@ -37,8 +43,9 @@ impl super::MonitorResource for Build {
 
   async fn to_list_item(
     build: Resource<Self::Config, Self::Info>,
-  ) -> anyhow::Result<Self::ListItem> {
-    Ok(BuildListItem {
+  ) -> Self::ListItem {
+    let state = get_build_state(&build.id).await;
+    BuildListItem {
       name: build.name,
       id: build.id,
       tags: build.tags,
@@ -48,8 +55,9 @@ impl super::MonitorResource for Build {
         version: build.config.version,
         repo: build.config.repo,
         branch: build.config.branch,
+        state,
       },
-    })
+    }
   }
 
   async fn busy(id: &String) -> anyhow::Result<bool> {
@@ -127,6 +135,33 @@ impl super::MonitorResource for Build {
   }
 }
 
+pub fn spawn_build_state_refresh_loop() {
+  tokio::spawn(async move {
+    loop {
+      refresh_build_state_cache().await;
+      tokio::time::sleep(Duration::from_secs(60)).await;
+    }
+  });
+}
+
+pub async fn refresh_build_state_cache() {
+  let _ = async {
+    let builds = find_collect(&db_client().await.builds, None, None)
+      .await
+      .context("failed to get builds from db")?;
+    let cache = build_state_cache();
+    for build in builds {
+      let state = get_build_state_from_db(&build.id).await;
+      cache.insert(build.id, state).await;
+    }
+    anyhow::Ok(())
+  }
+  .await
+  .inspect_err(|e| {
+    error!("failed to refresh build state cache | {e:#}")
+  });
+}
+
 #[instrument(skip(user))]
 async fn validate_config(
   config: &mut PartialBuildConfig,
@@ -149,3 +184,52 @@ async fn validate_config(
   }
   Ok(())
 }
+
+async fn get_build_state(id: &String) -> BuildState {
+  if action_states()
+    .build
+    .get(id)
+    .await
+    .map(|s| s.get().map(|s| s.building))
+    .transpose()
+    .ok()
+    .flatten()
+    .unwrap_or_default()
+  {
+    return BuildState::Building;
+  }
+  build_state_cache().get(id).await.unwrap_or_default()
+}
+
+async fn get_build_state_from_db(id: &str) -> BuildState {
+  async {
+    let state = db_client()
+      .await
+      .updates
+      .find_one(
+        doc! {
+          "target.type": "Build",
+          "target.id": id,
+          "operation": "RunBuild"
+        },
+        FindOneOptions::builder()
+          .sort(doc! { "start_ts": -1 })
+          .build(),
+      )
+      .await?
+      .map(|u| {
+        if u.success {
+          BuildState::Ok
+        } else {
+          BuildState::Failed
+        }
+      })
+      .unwrap_or(BuildState::Ok);
+    anyhow::Ok(state)
+  }
+  .await
+  .inspect_err(|e| {
+    warn!("failed to get build state for {id} | {e:#}")
+  })
+  .unwrap_or(BuildState::Unknown)
+}

bin/core/src/resource/builder.rs

@@ -38,7 +38,7 @@ impl super::MonitorResource for Builder {
 
   async fn to_list_item(
     builder: Resource<Self::Config, Self::Info>,
-  ) -> anyhow::Result<Self::ListItem> {
+  ) -> Self::ListItem {
     let (builder_type, instance_type) = match builder.config {
       BuilderConfig::Server(config) => (
         BuilderConfigVariant::Server.to_string(),
@@ -49,8 +49,7 @@ impl super::MonitorResource for Builder {
         Some(config.instance_type),
       ),
     };
-
-    Ok(BuilderListItem {
+    BuilderListItem {
       name: builder.name,
       id: builder.id,
       tags: builder.tags,
@@ -59,7 +58,7 @@ impl super::MonitorResource for Builder {
         builder_type,
         instance_type,
       },
-    })
+    }
   }
 
   async fn busy(_id: &String) -> anyhow::Result<bool> {

bin/core/src/resource/deployment.rs

@@ -4,7 +4,7 @@ use monitor_client::entities::{
   deployment::{
     Deployment, DeploymentConfig, DeploymentConfigDiff,
     DeploymentImage, DeploymentListItem, DeploymentListItemInfo,
-    DeploymentQuerySpecifics, DockerContainerState,
+    DeploymentQuerySpecifics, DeploymentState,
     PartialDeploymentConfig,
   },
   permission::PermissionLevel,
@@ -46,21 +46,29 @@ impl super::MonitorResource for Deployment {
 
   async fn to_list_item(
     deployment: Resource<Self::Config, Self::Info>,
-  ) -> anyhow::Result<Self::ListItem> {
+  ) -> Self::ListItem {
     let status = deployment_status_cache().get(&deployment.id).await;
-    let (image, build_id) = match deployment.config.image {
+    let (build_image, build_id) = match deployment.config.image {
       DeploymentImage::Build { build_id, version } => {
-        let build = super::get::<Build>(&build_id).await?;
+        let (build_name, build_id, build_version) =
+          super::get::<Build>(&build_id)
+            .await
+            .map(|b| (b.name, b.id, b.config.version))
+            .unwrap_or((
+              String::from("unknown"),
+              String::new(),
+              Default::default(),
+            ));
         let version = if version.is_none() {
-          build.config.version.to_string()
+          build_version.to_string()
         } else {
           version.to_string()
         };
-        (format!("{}:{version}", build.name), Some(build_id))
+        (format!("{build_name}:{version}"), Some(build_id))
       }
       DeploymentImage::Image { image } => (image, None),
     };
-    Ok(DeploymentListItem {
+    DeploymentListItem {
       name: deployment.name,
       id: deployment.id,
       tags: deployment.tags,
@@ -73,11 +81,16 @@ impl super::MonitorResource for Deployment {
         status: status.as_ref().and_then(|s| {
           s.curr.container.as_ref().and_then(|c| c.status.to_owned())
         }),
-        image,
+        image: status
+          .as_ref()
+          .and_then(|s| {
+            s.curr.container.as_ref().map(|c| c.image.clone())
+          })
+          .unwrap_or(build_image),
         server_id: deployment.config.server_id,
         build_id,
       },
-    })
+    }
   }
 
   async fn busy(id: &String) -> anyhow::Result<bool> {
@@ -149,8 +162,7 @@ impl super::MonitorResource for Deployment {
       .context("failed to get container state")?;
     if !matches!(
       state,
-      DockerContainerState::NotDeployed
-        | DockerContainerState::Unknown
+      DeploymentState::NotDeployed | DeploymentState::Unknown
     ) {
       // container needs to be destroyed
       let server =

bin/core/src/resource/mod.rs

@@ -28,6 +28,7 @@ use serde::{de::DeserializeOwned, Serialize};
 use serror::serialize_error_pretty;
 
 use crate::{
+  config::core_config,
   helpers::{
     create_permission,
     query::{
@@ -48,6 +49,16 @@ mod repo;
 mod server;
 mod server_template;
 
+pub use build::{
+  refresh_build_state_cache, spawn_build_state_refresh_loop,
+};
+pub use procedure::{
+  refresh_procedure_state_cache, spawn_procedure_state_refresh_loop,
+};
+pub use repo::{
+  refresh_repo_state_cache, spawn_repo_state_refresh_loop,
+};
+
 /// Implement on each monitor resource for common methods
 pub trait MonitorResource {
   type ListItem: Serialize + Send;
@@ -79,7 +90,7 @@ pub trait MonitorResource {
 
   async fn to_list_item(
     resource: Resource<Self::Config, Self::Info>,
-  ) -> anyhow::Result<Self::ListItem>;
+  ) -> Self::ListItem;
 
   #[allow(clippy::ptr_arg)]
   async fn busy(id: &String) -> anyhow::Result<bool>;
@@ -181,7 +192,10 @@ pub async fn get_check_permissions<T: MonitorResource>(
   permission_level: PermissionLevel,
 ) -> anyhow::Result<Resource<T::Config, T::Info>> {
   let resource = get::<T>(id_or_name).await?;
-  if user.admin {
+  if user.admin
+    || (permission_level <= PermissionLevel::Read
+      && core_config().transparent_mode)
+  {
     return Ok(resource);
   }
   let permissions = get_user_permission_on_resource(
@@ -222,17 +236,7 @@ pub async fn list_for_user_using_document<T: MonitorResource>(
     .await?
     .into_iter()
     .map(|resource| T::to_list_item(resource));
-
-  let list = join_all(list)
-    .await
-    .into_iter()
-    .collect::<anyhow::Result<Vec<_>>>()
-    .context(format!(
-      "failed to convert {} list item",
-      T::resource_type()
-    ))?;
-
-  Ok(list)
+  Ok(join_all(list).await)
 }
 
 pub async fn list_full_for_user<T: MonitorResource>(
@@ -249,7 +253,7 @@ async fn list_full_for_user_using_document<T: MonitorResource>(
   mut filters: Document,
   user: &User,
 ) -> anyhow::Result<Vec<Resource<T::Config, T::Info>>> {
-  if !user.admin {
+  if !user.admin && !core_config().transparent_mode {
     let ids =
       get_resource_ids_for_non_admin(&user.id, T::resource_type())
         .await?
@@ -377,7 +381,7 @@ pub async fn update<T: MonitorResource>(
 
   for FieldDiff { field, from, to } in diff.iter_field_diffs() {
     diff_log.push_str(&format!(
-      "\n\nfield: '{field}'\nfrom:  {from}\nto:    {to}"
+      "\n\n<span class=\"text-muted-foreground\">field</span>: '{field}'\n<span class=\"text-muted-foreground\">from</span>:  <span class=\"text-red-500\">{from}</span>\n<span class=\"text-muted-foreground\">to</span>:    <span class=\"text-green-500\">{to}</span>",
     ));
   }
 

bin/core/src/resource/procedure.rs

@@ -1,4 +1,6 @@
-use anyhow::anyhow;
+use std::time::Duration;
+
+use anyhow::{anyhow, Context};
 use monitor_client::{
   api::execute::Execution,
   entities::{
@@ -8,7 +10,7 @@ use monitor_client::{
     procedure::{
       PartialProcedureConfig, Procedure, ProcedureConfig,
       ProcedureConfigDiff, ProcedureListItem, ProcedureListItemInfo,
-      ProcedureQuerySpecifics,
+      ProcedureQuerySpecifics, ProcedureState,
     },
     repo::Repo,
     resource::Resource,
@@ -18,9 +20,12 @@ use monitor_client::{
     Operation,
   },
 };
-use mungos::mongodb::Collection;
+use mungos::{
+  find::find_collect,
+  mongodb::{bson::doc, options::FindOneOptions, Collection},
+};
 
-use crate::state::{action_states, db_client};
+use crate::state::{action_states, db_client, procedure_state_cache};
 
 impl super::MonitorResource for Procedure {
   type Config = ProcedureConfig;
@@ -41,16 +46,18 @@ impl super::MonitorResource for Procedure {
 
   async fn to_list_item(
     procedure: Resource<Self::Config, Self::Info>,
-  ) -> anyhow::Result<Self::ListItem> {
-    Ok(ProcedureListItem {
+  ) -> Self::ListItem {
+    let state = get_procedure_state(&procedure.id).await;
+    ProcedureListItem {
       name: procedure.name,
       id: procedure.id,
       tags: procedure.tags,
       resource_type: ResourceTargetVariant::Procedure,
       info: ProcedureListItemInfo {
         procedure_type: procedure.config.procedure_type,
+        state,
       },
-    })
+    }
   }
 
   async fn busy(id: &String) -> anyhow::Result<bool> {
@@ -260,3 +267,80 @@ async fn validate_config(
   }
   Ok(())
 }
+
+pub fn spawn_procedure_state_refresh_loop() {
+  tokio::spawn(async move {
+    loop {
+      refresh_procedure_state_cache().await;
+      tokio::time::sleep(Duration::from_secs(60)).await;
+    }
+  });
+}
+
+pub async fn refresh_procedure_state_cache() {
+  let _ = async {
+    let procedures =
+      find_collect(&db_client().await.procedures, None, None)
+        .await
+        .context("failed to get procedures from db")?;
+    let cache = procedure_state_cache();
+    for procedure in procedures {
+      let state = get_procedure_state_from_db(&procedure.id).await;
+      cache.insert(procedure.id, state).await;
+    }
+    anyhow::Ok(())
+  }
+  .await
+  .inspect_err(|e| {
+    error!("failed to refresh build state cache | {e:#}")
+  });
+}
+
+async fn get_procedure_state(id: &String) -> ProcedureState {
+  if action_states()
+    .procedure
+    .get(id)
+    .await
+    .map(|s| s.get().map(|s| s.running))
+    .transpose()
+    .ok()
+    .flatten()
+    .unwrap_or_default()
+  {
+    return ProcedureState::Running;
+  }
+  procedure_state_cache().get(id).await.unwrap_or_default()
+}
+
+async fn get_procedure_state_from_db(id: &str) -> ProcedureState {
+  async {
+    let state = db_client()
+      .await
+      .updates
+      .find_one(
+        doc! {
+          "target.type": "Procedure",
+          "target.id": id,
+          "operation": "RunProcedure"
+        },
+        FindOneOptions::builder()
+          .sort(doc! { "start_ts": -1 })
+          .build(),
+      )
+      .await?
+      .map(|u| {
+        if u.success {
+          ProcedureState::Ok
+        } else {
+          ProcedureState::Failed
+        }
+      })
+      .unwrap_or(ProcedureState::Ok);
+    anyhow::Ok(state)
+  }
+  .await
+  .inspect_err(|e| {
+    warn!("failed to get procedure state for {id} | {e:#}")
+  })
+  .unwrap_or(ProcedureState::Unknown)
+}

bin/core/src/resource/repo.rs

@@ -1,9 +1,11 @@
+use std::time::Duration;
+
 use anyhow::Context;
 use monitor_client::entities::{
   permission::PermissionLevel,
   repo::{
     PartialRepoConfig, Repo, RepoConfig, RepoConfigDiff, RepoInfo,
-    RepoListItem, RepoListItemInfo, RepoQuerySpecifics,
+    RepoListItem, RepoListItemInfo, RepoQuerySpecifics, RepoState,
   },
   resource::Resource,
   server::Server,
@@ -11,13 +13,18 @@ use monitor_client::entities::{
   user::User,
   Operation,
 };
-use mungos::mongodb::Collection;
+use mungos::{
+  find::find_collect,
+  mongodb::{bson::doc, options::FindOneOptions, Collection},
+};
 use periphery_client::api::git::DeleteRepo;
 use serror::serialize_error_pretty;
 
 use crate::{
   helpers::periphery_client,
-  state::{action_states, db_client},
+  state::{
+    action_states, db_client, repo_state_cache, repo_status_cache,
+  },
 };
 
 use super::get_check_permissions;
@@ -41,18 +48,25 @@ impl super::MonitorResource for Repo {
 
   async fn to_list_item(
     repo: Resource<Self::Config, Self::Info>,
-  ) -> anyhow::Result<Self::ListItem> {
-    Ok(RepoListItem {
+  ) -> Self::ListItem {
+    let state = get_repo_state(&repo.id).await;
+    let status =
+      repo_status_cache().get(&repo.id).await.unwrap_or_default();
+    RepoListItem {
       name: repo.name,
       id: repo.id,
       tags: repo.tags,
       resource_type: ResourceTargetVariant::Repo,
       info: RepoListItemInfo {
+        server_id: repo.config.server_id,
         last_pulled_at: repo.info.last_pulled_at,
         repo: repo.config.repo,
         branch: repo.config.branch,
+        state,
+        latest_hash: status.latest_hash.clone(),
+        latest_message: status.latest_message.clone(),
       },
-    })
+    }
   }
 
   async fn busy(id: &String) -> anyhow::Result<bool> {
@@ -150,6 +164,33 @@ impl super::MonitorResource for Repo {
   }
 }
 
+pub fn spawn_repo_state_refresh_loop() {
+  tokio::spawn(async move {
+    loop {
+      refresh_repo_state_cache().await;
+      tokio::time::sleep(Duration::from_secs(60)).await;
+    }
+  });
+}
+
+pub async fn refresh_repo_state_cache() {
+  let _ = async {
+    let repos = find_collect(&db_client().await.repos, None, None)
+      .await
+      .context("failed to get repos from db")?;
+    let cache = repo_state_cache();
+    for repo in repos {
+      let state = get_repo_state_from_db(&repo.id).await;
+      cache.insert(repo.id, state).await;
+    }
+    anyhow::Ok(())
+  }
+  .await
+  .inspect_err(|e| {
+    error!("failed to refresh repo state cache | {e:#}")
+  });
+}
+
 #[instrument(skip(user))]
 async fn validate_config(
   config: &mut PartialRepoConfig,
@@ -170,3 +211,62 @@ async fn validate_config(
   }
   Ok(())
 }
+
+async fn get_repo_state(id: &String) -> RepoState {
+  if let Some(state) = action_states()
+    .repo
+    .get(id)
+    .await
+    .and_then(|s| {
+      s.get()
+        .map(|s| {
+          if s.cloning {
+            Some(RepoState::Cloning)
+          } else if s.pulling {
+            Some(RepoState::Pulling)
+          } else {
+            None
+          }
+        })
+        .ok()
+    })
+    .flatten()
+  {
+    return state;
+  }
+  repo_state_cache().get(id).await.unwrap_or_default()
+}
+
+async fn get_repo_state_from_db(id: &str) -> RepoState {
+  async {
+    let state = db_client()
+      .await
+      .updates
+      .find_one(
+        doc! {
+          "target.type": "Repo",
+          "target.id": id,
+          "$or": [
+            { "operation": "CloneRepo" },
+            { "operation": "PullRepo" },
+          ],
+        },
+        FindOneOptions::builder()
+          .sort(doc! { "start_ts": -1 })
+          .build(),
+      )
+      .await?
+      .map(|u| {
+        if u.success {
+          RepoState::Ok
+        } else {
+          RepoState::Failed
+        }
+      })
+      .unwrap_or(RepoState::Ok);
+    anyhow::Ok(state)
+  }
+  .await
+  .inspect_err(|e| warn!("failed to get repo state for {id} | {e:#}"))
+  .unwrap_or(RepoState::Unknown)
+}

bin/core/src/resource/server.rs

@@ -36,15 +36,15 @@ impl super::MonitorResource for Server {
 
   async fn to_list_item(
     server: Resource<Self::Config, Self::Info>,
-  ) -> anyhow::Result<Self::ListItem> {
+  ) -> Self::ListItem {
     let status = server_status_cache().get(&server.id).await;
-    Ok(ServerListItem {
+    ServerListItem {
       name: server.name,
       id: server.id,
       tags: server.tags,
       resource_type: ResourceTargetVariant::Server,
       info: ServerListItemInfo {
-        status: status.map(|s| s.status).unwrap_or_default(),
+        state: status.map(|s| s.state).unwrap_or_default(),
         region: server.config.region,
         send_unreachable_alerts: server
           .config
@@ -53,7 +53,7 @@ impl super::MonitorResource for Server {
         send_mem_alerts: server.config.send_mem_alerts,
         send_disk_alerts: server.config.send_disk_alerts,
       },
-    })
+    }
   }
 
   async fn busy(id: &String) -> anyhow::Result<bool> {

bin/core/src/resource/server_template.rs

@@ -36,15 +36,19 @@ impl super::MonitorResource for ServerTemplate {
 
   async fn to_list_item(
     server_template: Resource<Self::Config, Self::Info>,
-  ) -> anyhow::Result<Self::ListItem> {
+  ) -> Self::ListItem {
     let (template_type, instance_type) = match server_template.config
     {
       ServerTemplateConfig::Aws(config) => (
         ServerTemplateConfigVariant::Aws.to_string(),
         Some(config.instance_type),
       ),
+      ServerTemplateConfig::Hetzner(config) => (
+        ServerTemplateConfigVariant::Hetzner.to_string(),
+        Some(config.server_type.as_ref().to_string()),
+      ),
     };
-    Ok(ServerTemplateListItem {
+    ServerTemplateListItem {
       name: server_template.name,
       id: server_template.id,
       tags: server_template.tags,
@@ -53,7 +57,7 @@ impl super::MonitorResource for ServerTemplate {
         provider: template_type.to_string(),
         instance_type,
       },
-    })
+    }
   }
 
   async fn busy(_id: &String) -> anyhow::Result<bool> {

bin/core/src/state.rs

@@ -1,6 +1,9 @@
 use std::sync::{Arc, OnceLock};
 
-use monitor_client::entities::deployment::DockerContainerState;
+use monitor_client::entities::{
+  build::BuildState, deployment::DeploymentState,
+  procedure::ProcedureState, repo::RepoState,
+};
 use tokio::sync::OnceCell;
 
 use crate::{
@@ -8,7 +11,10 @@ use crate::{
   config::core_config,
   db::DbClient,
   helpers::{action_state::ActionStates, cache::Cache},
-  monitor::{CachedDeploymentStatus, CachedServerStatus, History},
+  monitor::{
+    CachedDeploymentStatus, CachedRepoStatus, CachedServerStatus,
+    History,
+  },
 };
 
 pub struct State;
@@ -36,7 +42,7 @@ pub fn action_states() -> &'static ActionStates {
 
 pub type DeploymentStatusCache = Cache<
   String,
-  Arc<History<CachedDeploymentStatus, DockerContainerState>>,
+  Arc<History<CachedDeploymentStatus, DeploymentState>>,
 >;
 
 pub fn deployment_status_cache() -> &'static DeploymentStatusCache {
@@ -52,3 +58,34 @@ pub fn server_status_cache() -> &'static ServerStatusCache {
     OnceLock::new();
   SERVER_STATUS_CACHE.get_or_init(Default::default)
 }
+
+pub type RepoStatusCache = Cache<String, Arc<CachedRepoStatus>>;
+
+pub fn repo_status_cache() -> &'static RepoStatusCache {
+  static REPO_STATUS_CACHE: OnceLock<RepoStatusCache> =
+    OnceLock::new();
+  REPO_STATUS_CACHE.get_or_init(Default::default)
+}
+
+pub type BuildStateCache = Cache<String, BuildState>;
+
+pub fn build_state_cache() -> &'static BuildStateCache {
+  static BUILD_STATE_CACHE: OnceLock<BuildStateCache> =
+    OnceLock::new();
+  BUILD_STATE_CACHE.get_or_init(Default::default)
+}
+
+pub type RepoStateCache = Cache<String, RepoState>;
+
+pub fn repo_state_cache() -> &'static RepoStateCache {
+  static REPO_STATE_CACHE: OnceLock<RepoStateCache> = OnceLock::new();
+  REPO_STATE_CACHE.get_or_init(Default::default)
+}
+
+pub type ProcedureStateCache = Cache<String, ProcedureState>;
+
+pub fn procedure_state_cache() -> &'static ProcedureStateCache {
+  static PROCEDURE_STATE_CACHE: OnceLock<ProcedureStateCache> =
+    OnceLock::new();
+  PROCEDURE_STATE_CACHE.get_or_init(Default::default)
+}

bin/migrator/Cargo.toml

@@ -4,17 +4,17 @@ version.workspace = true
 edition.workspace = true
 authors.workspace = true
 license.workspace = true
+homepage.workspace = true
+repository.workspace = true
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
 monitor_client.workspace = true
-# db_client.workspace = true
 logger.workspace = true
 #
 termination_signal.workspace = true
 mungos.workspace = true
-mongo_indexed.workspace = true
 #
 tokio.workspace = true
 anyhow.workspace = true

bin/migrator/README.md

@@ -0,0 +1,5 @@
+# Migrator
+
+Upgrade data between periphery versions.
+
+Supports v0.x -> v1.x migration.

bin/migrator/src/legacy/v0/action.rs

@@ -1,53 +0,0 @@
-use mungos::mongodb::bson::serde_helpers::hex_string_as_object_id;
-use serde::{Deserialize, Serialize};
-
-use super::{Command, PermissionsMap};
-
-#[derive(Serialize, Deserialize, Debug, Clone, Default)]
-pub struct Action {
-  #[serde(
-    default,
-    rename = "_id",
-    skip_serializing_if = "String::is_empty",
-    with = "hex_string_as_object_id"
-  )]
-  pub id: String,
-
-  pub name: String,
-
-  #[serde(default)]
-  pub description: String,
-
-  pub path: String,
-
-  pub command: String,
-
-  // run action on all servers in this array
-  #[serde(default)]
-  pub server_ids: Vec<String>,
-
-  // run action on all servers in these groups
-  #[serde(default)]
-  pub group_ids: Vec<String>,
-
-  // run action on all servers
-  #[serde(default)]
-  pub run_on_all: bool,
-
-  #[serde(default)]
-  pub permissions: PermissionsMap,
-
-  #[serde(default, skip_serializing_if = "String::is_empty")]
-  pub created_at: String,
-  #[serde(default)]
-  pub updated_at: String,
-}
-
-impl From<Action> for Command {
-  fn from(value: Action) -> Command {
-    Command {
-      path: value.path,
-      command: value.command,
-    }
-  }
-}

bin/migrator/src/legacy/v0/alert.rs

@@ -1,13 +0,0 @@
-use mungos::mongodb::bson::serde_helpers::hex_string_as_object_id;
-use serde::{Deserialize, Serialize};
-
-#[derive(Serialize, Deserialize, Debug, Clone, Default)]
-pub struct Alert {
-  #[serde(
-    default,
-    rename = "_id",
-    skip_serializing_if = "String::is_empty",
-    with = "hex_string_as_object_id"
-  )]
-  pub id: String,
-}

bin/migrator/src/legacy/v0/build.rs

@@ -230,6 +230,8 @@ impl TryFrom<Build> for monitor_client::entities::build::Build {
         extra_args,
         use_buildx,
         labels: Default::default(),
+        webhook_enabled: true,
+        commit: Default::default(),
       },
     };
     Ok(build)

bin/migrator/src/legacy/v0/group.rs

@@ -1,38 +0,0 @@
-use mungos::mongodb::bson::serde_helpers::hex_string_as_object_id;
-use serde::{Deserialize, Serialize};
-
-use super::PermissionsMap;
-
-#[derive(Serialize, Deserialize, Debug, Clone, Default)]
-pub struct Group {
-  #[serde(
-    default,
-    rename = "_id",
-    skip_serializing_if = "String::is_empty",
-    with = "hex_string_as_object_id"
-  )]
-  pub id: String,
-
-  pub name: String,
-
-  #[serde(default)]
-  pub description: String,
-
-  #[serde(default)]
-  pub permissions: PermissionsMap,
-
-  pub builds: Vec<String>,
-
-  pub deployments: Vec<String>,
-
-  pub servers: Vec<String>,
-
-  pub procedures: Vec<String>,
-
-  pub groups: Vec<String>,
-
-  #[serde(default, skip_serializing_if = "String::is_empty")]
-  pub created_at: String,
-  #[serde(default)]
-  pub updated_at: String,
-}

bin/migrator/src/legacy/v0/mod.rs

@@ -1,35 +1,53 @@
 use std::collections::HashMap;
 
-use anyhow::{anyhow, Context};
-use chrono::{DateTime, LocalResult, SecondsFormat, TimeZone, Utc};
+use anyhow::Context;
+use chrono::DateTime;
+use mungos::{init::MongoBuilder, mongodb::Collection};
 use serde::{Deserialize, Serialize};
 
-pub mod traits;
-
-mod action;
-mod alert;
 mod build;
 mod config;
 mod deployment;
-mod group;
-mod periphery_command;
-mod procedure;
 mod server;
 mod update;
 mod user;
 
-pub use action::*;
-pub use alert::*;
 pub use build::*;
 pub use config::*;
 pub use deployment::*;
-pub use group::*;
-pub use periphery_command::*;
-pub use procedure::*;
 pub use server::*;
 pub use update::*;
 pub use user::*;
 
+pub struct DbClient {
+  pub users: Collection<User>,
+  pub servers: Collection<Server>,
+  pub deployments: Collection<Deployment>,
+  pub builds: Collection<Build>,
+  pub updates: Collection<Update>,
+}
+
+impl DbClient {
+  pub async fn new(
+    legacy_uri: &str,
+    legacy_db_name: &str,
+  ) -> DbClient {
+    let client = MongoBuilder::default()
+      .uri(legacy_uri)
+      .build()
+      .await
+      .expect("failed to init legacy mongo client");
+    let db = client.database(legacy_db_name);
+    DbClient {
+      users: db.collection("users"),
+      servers: db.collection("servers"),
+      deployments: db.collection("deployments"),
+      builds: db.collection("builds"),
+      updates: db.collection("updates"),
+    }
+  }
+}
+
 pub type PermissionsMap = HashMap<String, PermissionLevel>;
 
 #[derive(Serialize, Deserialize, Clone, Debug)]
@@ -322,15 +340,3 @@ pub fn unix_from_monitor_ts(ts: &str) -> anyhow::Result<i64> {
       .timestamp_millis(),
   )
 }
-
-pub fn monitor_ts_from_unix(ts: i64) -> anyhow::Result<String> {
-  match Utc.timestamp_millis_opt(ts) {
-    LocalResult::Single(dt) => {
-      Ok(dt.to_rfc3339_opts(SecondsFormat::Millis, false))
-    }
-    LocalResult::None => {
-      Err(anyhow!("out of bounds timestamp passed"))
-    }
-    _ => unreachable!(),
-  }
-}