update deps

.vscode/resolver.code-snippets

@@ -3,7 +3,6 @@
 		"scope": "rust",
 		"prefix": "resolve",
 		"body": [
-			"#[async_trait]",
 			"impl Resolve<${1}, User> for State {",
 			"\tasync fn resolve(&self, ${1} { ${0} }: ${1}, _: User) -> anyhow::Result<${2}> {",
 			"\t\ttodo!()",

Cargo.toml

@@ -3,20 +3,25 @@ resolver = "2"
 members = ["bin/*", "lib/*", "client/core/rs", "client/periphery/rs"]
 
 [workspace.package]
-version = "1.2.0"
+version = "1.5.0"
 edition = "2021"
 authors = ["mbecker20 <becker.maxh@gmail.com>"]
 license = "GPL-3.0-or-later"
+repository = "https://github.com/mbecker20/monitor"
+homepage = "https://docs.monitor.mogh.tech"
+
+[patch.crates-io]
+monitor_client = { path = "client/core/rs" }
 
 [workspace.dependencies]
 # LOCAL
-monitor_client = { path = "client/core/rs" }
+monitor_client = "1.5.0"
 periphery_client = { path = "client/periphery/rs" }
 logger = { path = "lib/logger" }
 
 # MOGH
 run_command = { version = "0.0.6", features = ["async_tokio"] }
-serror = { version = "0.3.4", features = ["axum"] }
+serror = { version = "0.3.4", default-features = false }
 slack = { version = "0.1.0", package = "slack_client_rs" }
 derive_default_builder = "0.1.8"
 derive_empty_traits = "0.1.0"
@@ -25,11 +30,11 @@ termination_signal = "0.1.3"
 async_timing_util = "0.1.14"
 partial_derive2 = "0.4.2"
 derive_variants = "0.1.3"
-mongo_indexed = "0.2.2"
-resolver_api = "0.1.9"
+mongo_indexed = "0.3.0"
+resolver_api = "1.1.0"
 parse_csl = "0.1.0"
-mungos = "0.5.4"
-svi = "0.1.4"
+mungos = "0.5.6"
+svi = "1.0.1"
 
 # ASYNC
 tokio = { version = "1.37.0", features = ["full"] }
@@ -37,7 +42,6 @@ reqwest = { version = "0.12.4", features = ["json"] }
 tokio-util = "0.7.11"
 futures = "0.3.30"
 futures-util = "0.3.30"
-async-trait = "0.1.80"
 
 # SERVER
 axum = { version = "0.7.5", features = ["ws", "json"] }
@@ -47,14 +51,14 @@ tower-http = { version = "0.5.2", features = ["fs", "cors"] }
 tokio-tungstenite = "0.21.0"
 
 # SER/DE
-serde = { version = "1.0.201", features = ["derive"] }
+serde = { version = "1.0.202", features = ["derive"] }
 strum = { version = "0.26.2", features = ["derive"] }
-serde_json = "1.0.116"
-toml = "0.8.12"
+serde_json = "1.0.117"
+toml = "0.8.13"
 
 # ERROR
-anyhow = "1.0.83"
-thiserror = "1.0.60"
+anyhow = "1.0.86"
+thiserror = "1.0.61"
 
 # LOGGING
 opentelemetry_sdk = { version = "0.22.1", features = ["rt-tokio"] }
@@ -84,10 +88,11 @@ bollard = "0.16.1"
 sysinfo = "0.30.12"
 
 # CLOUD
-aws-config = "1.3.0"
-aws-sdk-ec2 = "1.40.0"
+aws-config = "1.4.0"
+aws-sdk-ec2 = "1.42.0"
 
 # MISC
 derive_builder = "0.20.0"
 typeshare = "1.0.3"
 colored = "2.1.0"
+bson = "2.10.0"

bin/alerter/Cargo.toml

@@ -1,9 +1,11 @@
 [package]
-name = "alert_logger"
+name = "alerter"
 version.workspace = true
 edition.workspace = true
 authors.workspace = true
 license.workspace = true
+homepage.workspace = true
+repository.workspace = true
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 

bin/alerter/README.md

@@ -0,0 +1,4 @@
+# Alerter
+
+This crate sets up a basic axum server that listens for incoming alert POSTs.
+It can be used as a monitor alerting endpoint, and serves as a template for other custom alerter implementations.

bin/cli/Cargo.toml

@@ -5,6 +5,8 @@ version.workspace = true
 edition.workspace = true
 authors.workspace = true
 license.workspace = true
+homepage.workspace = true
+repository.workspace = true
 
 [[bin]]
 name = "monitor"

bin/cli/README.md

@@ -1,6 +1,12 @@
 # Monitor CLI
 
-Monitor CLI is a command line tool to sync monitor resources and execute file defined procedures.
+Monitor CLI is a tool to sync monitor resources and execute file defined procedures.
+
+## Install
+
+```sh
+cargo install monitor_cli
+```
 
 ## Usage
 

bin/cli/resources/builds.toml

@@ -73,11 +73,9 @@ region = "us-east-2"
 instance_type = "c5.2xlarge"
 volume_gb = 20
 port = 8120
-ami_id = "ami-0005a05fa63a080ab"
+ami_id = "ami-0c5d5efd79075c0e3"
 subnet_id = "subnet-02ae5ad480eacc4bc"
 security_group_ids = ["sg-049d98c819f9ace58", "sg-006c0ca638af8eb44"]
 key_pair_name = "mogh-key"
 assign_public_ip = true
-use_public_ip = false
-github_accounts = []
-docker_accounts = []
+use_public_ip = true

bin/cli/resources/deployments.toml

@@ -5,7 +5,7 @@ description = "An NGINX proxy for mogh.tech"
 tags = ["monitor"]
 config.server_id = "monitor-01"
 config.network = "host"
-config.restart = "on-failure"
+config.restart = "unless-stopped"
 config.image.type = "Image"
 config.image.params.image = "jc21/nginx-proxy-manager"
 
@@ -20,7 +20,7 @@ container = "/etc/letsencrypt"
 ## MONITOR MONGO
 [[deployment]]
 name = "monitor-mongo"
-description = ""
+description = "shared mongo deployment"
 tags = ["monitor"]
 
 [deployment.config]
@@ -50,7 +50,7 @@ params.image = "mbecker2020/monitor_core"
 ## GRAFANA
 [[deployment]]
 name = "grafana"
-description = ""
+description = "runs on port 3080"
 tags = ["logging"]
 
 [deployment.config]
@@ -78,7 +78,7 @@ value = "key-value"
 ## LOKI
 [[deployment]]
 name = "loki"
-description = ""
+description = "runs on 3100"
 tags = ["logging"]
 
 [deployment.config]
@@ -97,7 +97,7 @@ container = "/loki"
 
 [[deployment]]
 name = "tempo"
-description = ""
+description = "OTLP: port 4317. HTTP: 3200"
 tags = ["logging"]
 
 [deployment.config]
@@ -122,7 +122,7 @@ value = "key-value"
 ## VECTOR
 [[deployment]]
 name = "vector"
-description = ""
+description = "collects the docker logs (and alert endpoint)"
 tags = ["logging"]
 
 [deployment.config]
@@ -137,7 +137,7 @@ type = "Image"
 params.image = "timberio/vector:latest-debian"
 
 [[deployment.config.volumes]]
-local = "/home/ubuntu/.config/vector"
+local = "/root/.config/vector"
 container = "/etc/vector"
 
 [[deployment.config.volumes]]
@@ -155,7 +155,7 @@ value = "key-value"
 ## MONITOR CORE DEV
 [[deployment]]
 name = "monitor-core-dev"
-description = ""
+description = "runs on 9001"
 tags = ["monitor", "dev"]
 
 [deployment.config]
@@ -169,7 +169,7 @@ type = "Build"
 params.build_id = "monitor_core"
 
 [[deployment.config.volumes]]
-local = "/home/ubuntu/.config/monitor/dev.core.config.toml"
+local = "/root/.config/monitor/dev.core.config.toml"
 container = "/config/config.toml"
 
 [[deployment.config.volumes]]

bin/cli/resources/procedures.toml

@@ -1,8 +1,35 @@
 [[procedure]]
-name = "test-procedure"
+name = "build-monitor"
 description = ""
-tags = []
+tags = ["monitor"]
 
 [procedure.config]
-procedure_type = "Sequence"
-executions = []
+procedure_type = "Parallel"
+webhook_enabled = true
+
+[[procedure.config.executions]]
+enabled = true
+
+[procedure.config.executions.execution]
+type = "RunBuild"
+
+[procedure.config.executions.execution.params]
+build = "monitor_core"
+
+[[procedure.config.executions]]
+enabled = true
+
+[procedure.config.executions.execution]
+type = "RunBuild"
+
+[procedure.config.executions.execution.params]
+build = "monitor_frontend"
+
+[[procedure.config.executions]]
+enabled = false
+
+[procedure.config.executions.execution]
+type = "RunBuild"
+
+[procedure.config.executions.execution.params]
+build = "monitor_frontend_dev"

bin/cli/resources/repos.toml

@@ -34,4 +34,4 @@ command = ""
 
 [repo.config.on_pull]
 path = "."
-command = "/root/.cargo/bin/cargo build -p monitor_periphery --release"
+command = "/root/.cargo/bin/cargo build -p monitor_periphery --release && cp ./target/release/periphery /home/ubuntu/periphery"

bin/cli/src/main.rs

@@ -75,11 +75,11 @@ fn monitor_client() -> &'static MonitorClient {
 
 #[tokio::main]
 async fn main() -> anyhow::Result<()> {
-  tracing_subscriber::fmt::init();
+  tracing_subscriber::fmt().with_target(false).init();
 
   let version =
     monitor_client().read(read::GetVersion {}).await?.version;
-  info!("monitor version: {version}");
+  info!("monitor version: {}", version.to_string().blue().bold());
 
   match &cli_args().command {
     Command::Exec { path } => execution::run_execution(path).await?,

bin/cli/src/sync/file.rs

@@ -1,6 +1,7 @@
 use std::{fs, path::Path};
 
 use anyhow::{anyhow, Context};
+use colored::Colorize;
 use monitor_client::entities::toml::ResourcesToml;
 
 pub fn read_resources(path: &Path) -> anyhow::Result<ResourcesToml> {
@@ -30,7 +31,11 @@ fn read_resources_recursive(
         return Ok(());
       }
     };
-    info!("adding resources from {path:?}");
+    info!(
+      "{} from {}",
+      "adding resources".green().bold(),
+      path.display().to_string().blue().bold()
+    );
     resources.server_templates.extend(more.server_templates);
     resources.servers.extend(more.servers);
     resources.builds.extend(more.builds);

bin/cli/src/sync/mod.rs

@@ -1,5 +1,6 @@
 use std::path::Path;
 
+use colored::Colorize;
 use monitor_client::entities::{
   alerter::Alerter, build::Build, builder::Builder,
   deployment::Deployment, procedure::Procedure, repo::Repo,
@@ -8,14 +9,17 @@ use monitor_client::entities::{
 
 use crate::{sync::resources::ResourceSync, wait_for_enter};
 
-mod resource_file;
+mod file;
 mod resources;
 mod user_group;
 
 pub async fn run_sync(path: &Path) -> anyhow::Result<()> {
-  info!("path: {path:?}");
+  info!(
+    "resources path: {}",
+    path.display().to_string().blue().bold()
+  );
 
-  let resources = resource_file::read_resources(path)?;
+  let resources = file::read_resources(path)?;
 
   info!("computing sync actions...");
 
@@ -57,7 +61,7 @@ pub async fn run_sync(path: &Path) -> anyhow::Result<()> {
     && user_group_creates.is_empty()
     && user_group_updates.is_empty()
   {
-    info!("nothing to do. exiting.");
+    info!("{}. exiting.", "nothing to do".green().bold());
     return Ok(());
   }
 

bin/cli/src/sync/resources/mod.rs

@@ -45,6 +45,7 @@ pub trait ResourceSync {
     + Send
     + From<Self::ConfigDiff>
     + Serialize
+    + MaybeNone
     + 'static;
   type ConfigDiff: Diff + MaybeNone;
   type ListItemInfo: 'static;
@@ -218,8 +219,6 @@ pub trait ResourceSync {
     to_create: ToCreate<Self::PartialConfig>,
     to_update: ToUpdate<Self::PartialConfig>,
   ) {
-    let log_after = !to_update.is_empty() || !to_create.is_empty();
-
     for resource in to_create {
       let name = resource.name.clone();
       let tags = resource.tags.clone();
@@ -236,7 +235,12 @@ pub trait ResourceSync {
       };
       Self::update_tags(id.clone(), &name, tags).await;
       Self::update_description(id, &name, description).await;
-      info!("{} {name} created", Self::display());
+      info!(
+        "{} {} '{}'",
+        "created".green().bold(),
+        Self::display(),
+        name.bold(),
+      );
     }
 
     for ToUpdateItem {
@@ -260,23 +264,21 @@ pub trait ResourceSync {
         Self::update_tags(id.clone(), &name, tags).await;
       }
 
-      if let Err(e) = Self::update(id, resource).await {
-        warn!(
-          "failed to update config on {} {name} | {e:#}",
-          Self::display()
-        );
-      } else {
-        info!("updated {} {name} config", Self::display());
+      if !resource.config.is_none() {
+        if let Err(e) = Self::update(id, resource).await {
+          warn!(
+            "failed to update config on {} {name} | {e:#}",
+            Self::display()
+          );
+        } else {
+          info!(
+            "{} {} '{}' configuration",
+            "updated".blue().bold(),
+            Self::display(),
+            name.bold(),
+          );
+        }
       }
-
-      info!("{} {name} updated", Self::display());
-    }
-
-    if log_after {
-      info!(
-        "============ {}s synced ✅ ============",
-        Self::display()
-      );
     }
   }
 
@@ -294,7 +296,12 @@ pub trait ResourceSync {
         Self::display(),
       );
     } else {
-      info!("updated {} {name} tags", Self::display());
+      info!(
+        "{} {} '{}' tags",
+        "updated".blue().bold(),
+        Self::display(),
+        name.bold(),
+      );
     }
   }
 
@@ -312,7 +319,12 @@ pub trait ResourceSync {
     {
       warn!("failed to update resource {id} description | {e:#}");
     } else {
-      info!("updated {} {name} description", Self::display());
+      info!(
+        "{} {} '{}' description",
+        "updated".blue().bold(),
+        Self::display(),
+        name.bold(),
+      );
     }
   }
 }

bin/core/Cargo.toml

@@ -4,6 +4,8 @@ version.workspace = true
 edition.workspace = true
 authors.workspace = true
 license.workspace = true
+homepage.workspace = true
+repository.workspace = true
 
 [[bin]]
 name = "core"
@@ -13,10 +15,11 @@ path = "src/main.rs"
 
 [dependencies]
 # local
+monitor_client = { workspace = true, features = ["mongo"] }
 periphery_client.workspace = true
-monitor_client.workspace = true
 logger.workspace = true
 # mogh
+serror = { workspace = true, features = ["axum"] }
 merge_config_files.workspace = true
 termination_signal.workspace = true
 async_timing_util.workspace = true
@@ -25,11 +28,10 @@ mongo_indexed.workspace = true
 resolver_api.workspace = true
 parse_csl.workspace = true
 mungos.workspace = true
-serror.workspace = true
 slack.workspace = true
+svi.workspace = true
 # external
 urlencoding.workspace = true
-async-trait.workspace = true
 aws-sdk-ec2.workspace = true
 aws-config.workspace = true
 tokio-util.workspace = true

bin/core/src/api/auth.rs

@@ -1,7 +1,6 @@
 use std::{sync::OnceLock, time::Instant};
 
 use anyhow::anyhow;
-use async_trait::async_trait;
 use axum::{http::HeaderMap, routing::post, Router};
 use axum_extra::{headers::ContentType, TypedHeader};
 use monitor_client::{api::auth::*, entities::user::User};
@@ -92,7 +91,6 @@ fn login_options_reponse() -> &'static GetLoginOptionsResponse {
   })
 }
 
-#[async_trait]
 impl Resolve<GetLoginOptions, HeaderMap> for State {
   #[instrument(name = "GetLoginOptions", level = "debug", skip(self))]
   async fn resolve(
@@ -104,7 +102,6 @@ impl Resolve<GetLoginOptions, HeaderMap> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<ExchangeForJwt, HeaderMap> for State {
   #[instrument(name = "ExchangeForJwt", level = "debug", skip(self))]
   async fn resolve(
@@ -118,7 +115,6 @@ impl Resolve<ExchangeForJwt, HeaderMap> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<GetUser, HeaderMap> for State {
   #[instrument(name = "GetUser", level = "debug", skip(self))]
   async fn resolve(

bin/core/src/api/execute/build.rs

@@ -1,7 +1,6 @@
-use std::time::Duration;
+use std::{collections::HashSet, time::Duration};
 
 use anyhow::{anyhow, Context};
-use async_trait::async_trait;
 use futures::future::join_all;
 use monitor_client::{
   api::execute::{
@@ -15,7 +14,7 @@ use monitor_client::{
     monitor_timestamp,
     permission::PermissionLevel,
     server::Server,
-    server_template::AwsServerTemplateConfig,
+    server_template::aws::AwsServerTemplateConfig,
     update::{Log, Update},
     user::{auto_redeploy_user, User},
     Operation,
@@ -46,14 +45,13 @@ use crate::{
   helpers::{
     channel::build_cancel_channel,
     periphery_client,
-    query::get_deployment_state,
+    query::{get_deployment_state, get_global_variables},
     update::{add_update, make_update, update_update},
   },
   resource::{self, refresh_build_state_cache},
   state::{action_states, db_client, State},
 };
 
-#[async_trait]
 impl Resolve<RunBuild, User> for State {
   #[instrument(name = "RunBuild", skip(self, user))]
   async fn resolve(
@@ -141,6 +139,7 @@ impl Resolve<RunBuild, User> for State {
       };
 
     let core_config = core_config();
+    let variables = get_global_variables().await?;
 
     // CLONE REPO
 
@@ -184,11 +183,59 @@ impl Resolve<RunBuild, User> for State {
         .get(&build.config.docker_account)
         .cloned();
 
+      // Interpolate variables / secrets into build args
+      let mut global_replacers = HashSet::new();
+      let mut secret_replacers = HashSet::new();
+      for arg in &mut build.config.build_args {
+        // first pass - global variables
+        let (res, more_replacers) = svi::interpolate_variables(
+          &arg.value,
+          &variables,
+          svi::Interpolator::DoubleBrackets,
+          false,
+        )
+        .context("failed to interpolate global variables")?;
+        global_replacers.extend(more_replacers);
+        // second pass - core secrets
+        let (res, more_replacers) = svi::interpolate_variables(
+          &res,
+          &core_config.secrets,
+          svi::Interpolator::DoubleBrackets,
+          false,
+        )
+        .context("failed to interpolate core secrets")?;
+        secret_replacers.extend(more_replacers);
+        arg.value = res;
+      }
+
+      // Show which variables were interpolated
+      if !global_replacers.is_empty() {
+        update.push_simple_log(
+          "interpolate global variables",
+          global_replacers
+            .into_iter()
+            .map(|(value, variable)| format!("<span class=\"text-muted-foreground\">{variable} =></span> {value}"))
+            .collect::<Vec<_>>()
+            .join("\n"),
+        );
+      }
+      if !secret_replacers.is_empty() {
+        update.push_simple_log(
+          "interpolate core secrets",
+          secret_replacers
+            .iter()
+            .map(|(_, variable)| format!("<span class=\"text-muted-foreground\">replaced:</span> {variable}"))
+            .collect::<Vec<_>>()
+            .join("\n"),
+        );
+      }
+
       let res = tokio::select! {
         res = periphery
           .request(api::build::Build {
             build: build.clone(),
             docker_token,
+            replacers: secret_replacers.into_iter().collect(),
           }) => res.context("failed at call to periphery to build"),
         _ = cancel.cancelled() => {
           info!("build cancelled during build, cleaning up builder");
@@ -288,7 +335,6 @@ async fn handle_early_return(
   Ok(update)
 }
 
-#[async_trait]
 impl Resolve<CancelBuild, User> for State {
   #[instrument(name = "CancelBuild", skip(self, user))]
   async fn resolve(

bin/core/src/api/execute/deployment.rs

@@ -1,5 +1,6 @@
+use std::collections::HashSet;
+
 use anyhow::{anyhow, Context};
-use async_trait::async_trait;
 use futures::future::join_all;
 use monitor_client::{
   api::execute::*,
@@ -23,7 +24,7 @@ use crate::{
   config::core_config,
   helpers::{
     periphery_client,
-    query::get_server_with_status,
+    query::{get_global_variables, get_server_with_status},
     update::{add_update, make_update, update_update},
   },
   monitor::update_cache_for_server,
@@ -31,7 +32,6 @@ use crate::{
   state::{action_states, db_client, State},
 };
 
-#[async_trait]
 impl Resolve<Deploy, User> for State {
   #[instrument(name = "Deploy", skip(self, user))]
   async fn resolve(
@@ -51,6 +51,10 @@ impl Resolve<Deploy, User> for State {
       )
       .await?;
 
+    if deployment.config.server_id.is_empty() {
+      return Err(anyhow!("deployment has no server configured"));
+    }
+
     // get the action state for the deployment (or insert default).
     let action_state = action_states()
       .deployment
@@ -62,10 +66,6 @@ impl Resolve<Deploy, User> for State {
     let _action_guard =
       action_state.update(|state| state.deploying = true)?;
 
-    if deployment.config.server_id.is_empty() {
-      return Err(anyhow!("deployment has no server configured"));
-    }
-
     let (server, status) =
       get_server_with_status(&deployment.config.server_id).await?;
     if status != ServerState::Ok {
@@ -85,9 +85,11 @@ impl Resolve<Deploy, User> for State {
         } else {
           version
         };
+        // replace image with corresponding build image.
         deployment.config.image = DeploymentImage::Image {
           image: format!("{image_name}:{version}"),
         };
+        // set docker account to match build docker account if it's not overridden by deployment
         if deployment.config.docker_account.is_empty() {
           deployment.config.docker_account =
             build.config.docker_account;
@@ -97,14 +99,66 @@ impl Resolve<Deploy, User> for State {
       DeploymentImage::Image { .. } => Version::default(),
     };
 
+    let variables = get_global_variables().await?;
+    let core_config = core_config();
+
+    // Interpolate variables into environment
+    let mut global_replacers = HashSet::new();
+    let mut secret_replacers = HashSet::new();
+    for env in &mut deployment.config.environment {
+      // first pass - global variables
+      let (res, more_replacers) = svi::interpolate_variables(
+        &env.value,
+        &variables,
+        svi::Interpolator::DoubleBrackets,
+        false,
+      )
+      .context("failed to interpolate global variables")?;
+      global_replacers.extend(more_replacers);
+      // second pass - core secrets
+      let (res, more_replacers) = svi::interpolate_variables(
+        &res,
+        &core_config.secrets,
+        svi::Interpolator::DoubleBrackets,
+        false,
+      )
+      .context("failed to interpolate core secrets")?;
+      secret_replacers.extend(more_replacers);
+
+      // set env value with the result
+      env.value = res;
+    }
+
     let mut update =
       make_update(&deployment, Operation::DeployContainer, &user);
     update.in_progress();
     update.version = version;
 
+    // Show which variables were interpolated
+    if !global_replacers.is_empty() {
+      update.push_simple_log(
+        "interpolate global variables",
+        global_replacers
+          .into_iter()
+          .map(|(value, variable)| format!("<span class=\"text-muted-foreground\">{variable} =></span> {value}"))
+          .collect::<Vec<_>>()
+          .join("\n"),
+      );
+    }
+    if !secret_replacers.is_empty() {
+      update.push_simple_log(
+        "interpolate core secrets",
+        secret_replacers
+          .iter()
+          .map(|(_, variable)| format!("<span class=\"text-muted-foreground\">replaced:</span> {variable}"))
+          .collect::<Vec<_>>()
+          .join("\n"),
+      );
+    }
+
     update.id = add_update(update.clone()).await?;
 
-    let docker_token = core_config()
+    let docker_token = core_config
       .docker_accounts
       .get(&deployment.config.docker_account)
       .cloned();
@@ -115,6 +169,7 @@ impl Resolve<Deploy, User> for State {
         stop_signal,
         stop_time,
         docker_token,
+        replacers: secret_replacers.into_iter().collect(),
       })
       .await
     {
@@ -136,7 +191,6 @@ impl Resolve<Deploy, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<StartContainer, User> for State {
   #[instrument(name = "StartContainer", skip(self, user))]
   async fn resolve(
@@ -211,7 +265,6 @@ impl Resolve<StartContainer, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<StopContainer, User> for State {
   #[instrument(name = "StopContainer", skip(self, user))]
   async fn resolve(
@@ -287,7 +340,6 @@ impl Resolve<StopContainer, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<StopAllContainers, User> for State {
   #[instrument(name = "StopAllContainers", skip(self, user))]
   async fn resolve(
@@ -370,7 +422,6 @@ impl Resolve<StopAllContainers, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<RemoveContainer, User> for State {
   #[instrument(name = "RemoveContainer", skip(self, user))]
   async fn resolve(

bin/core/src/api/execute/mod.rs

@@ -4,7 +4,7 @@ use anyhow::{anyhow, Context};
 use axum::{middleware, routing::post, Extension, Router};
 use axum_extra::{headers::ContentType, TypedHeader};
 use monitor_client::{api::execute::*, entities::user::User};
-use resolver_api::{derive::Resolver, Resolve, Resolver};
+use resolver_api::{derive::Resolver, Resolver};
 use serde::{Deserialize, Serialize};
 use serror::Json;
 use typeshare::typeshare;

bin/core/src/api/execute/procedure.rs

@@ -1,4 +1,5 @@
-use async_trait::async_trait;
+use std::pin::Pin;
+
 use monitor_client::{
   api::execute::RunProcedure,
   entities::{
@@ -6,6 +7,7 @@ use monitor_client::{
     update::Update, user::User, Operation,
   },
 };
+use mungos::{by_id::update_one_by_id, mongodb::bson::to_document};
 use resolver_api::Resolve;
 use serror::serialize_error_pretty;
 use tokio::sync::Mutex;
@@ -14,10 +16,11 @@ use crate::{
   helpers::{
     procedure::execute_procedure,
     update::{add_update, make_update, update_update},
-  }, resource, state::{action_states, State}
+  },
+  resource::{self, refresh_procedure_state_cache},
+  state::{action_states, db_client, State},
 };
 
-#[async_trait]
 impl Resolve<RunProcedure, User> for State {
   #[instrument(name = "RunProcedure", skip(self, user))]
   async fn resolve(
@@ -25,6 +28,19 @@ impl Resolve<RunProcedure, User> for State {
     RunProcedure { procedure }: RunProcedure,
     user: User,
   ) -> anyhow::Result<Update> {
+    resolve_inner(procedure, user).await
+  }
+}
+
+fn resolve_inner(
+  procedure: String,
+  user: User,
+) -> Pin<
+  Box<
+    dyn std::future::Future<Output = anyhow::Result<Update>> + Send,
+  >,
+> {
+  Box::pin(async move {
     let procedure = resource::get_check_permissions::<Procedure>(
       &procedure,
       &user,
@@ -74,8 +90,23 @@ impl Resolve<RunProcedure, User> for State {
 
     update.finalize();
 
+    // Need to manually update the update before cache refresh,
+    // and before broadcast with add_update.
+    // The Err case of to_document should be unreachable,
+    // but will fail to update cache in that case.
+    if let Ok(update_doc) = to_document(&update) {
+      let _ = update_one_by_id(
+        &db_client().await.updates,
+        &update.id,
+        mungos::update::Update::Set(update_doc),
+        None,
+      )
+      .await;
+      refresh_procedure_state_cache().await;
+    }
+
     update_update(update.clone()).await?;
 
     Ok(update)
-  }
+  })
 }

bin/core/src/api/execute/repo.rs

@@ -1,5 +1,4 @@
 use anyhow::anyhow;
-use async_trait::async_trait;
 use monitor_client::{
   api::execute::*,
   entities::{
@@ -30,7 +29,6 @@ use crate::{
   state::{action_states, db_client, State},
 };
 
-#[async_trait]
 impl Resolve<CloneRepo, User> for State {
   #[instrument(name = "CloneRepo", skip(self, user))]
   async fn resolve(
@@ -106,7 +104,6 @@ impl Resolve<CloneRepo, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<PullRepo, User> for State {
   #[instrument(name = "PullRepo", skip(self, user))]
   async fn resolve(

bin/core/src/api/execute/server.rs

@@ -1,5 +1,4 @@
 use anyhow::Context;
-use async_trait::async_trait;
 use monitor_client::{
   api::execute::*,
   entities::{
@@ -24,7 +23,6 @@ use crate::{
   state::{action_states, State},
 };
 
-#[async_trait]
 impl Resolve<PruneDockerContainers, User> for State {
   #[instrument(name = "PruneDockerContainers", skip(self, user))]
   async fn resolve(
@@ -81,7 +79,6 @@ impl Resolve<PruneDockerContainers, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<PruneDockerNetworks, User> for State {
   #[instrument(name = "PruneDockerNetworks", skip(self, user))]
   async fn resolve(
@@ -138,7 +135,6 @@ impl Resolve<PruneDockerNetworks, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<PruneDockerImages, User> for State {
   #[instrument(name = "PruneDockerImages", skip(self, user))]
   async fn resolve(

bin/core/src/api/execute/server_template.rs

@@ -1,5 +1,4 @@
 use anyhow::{anyhow, Context};
-use axum::async_trait;
 use monitor_client::{
   api::{execute::LaunchServer, write::CreateServer},
   entities::{
@@ -16,10 +15,12 @@ use resolver_api::Resolve;
 use serror::serialize_error_pretty;
 
 use crate::{
-  cloud::aws::launch_ec2_instance, helpers::update::{add_update, make_update, update_update}, resource, state::{db_client, State}
+  cloud::{aws::launch_ec2_instance, hetzner::launch_hetzner_server},
+  helpers::update::{add_update, make_update, update_update},
+  resource,
+  state::{db_client, State},
 };
 
-#[async_trait]
 impl Resolve<LaunchServer, User> for State {
   #[instrument(name = "LaunchServer", skip(self, user))]
   async fn resolve(
@@ -66,17 +67,19 @@ impl Resolve<LaunchServer, User> for State {
     let config = match template.config {
       ServerTemplateConfig::Aws(config) => {
         let region = config.region.clone();
-        let instance = launch_ec2_instance(&name, config).await;
-        if let Err(e) = &instance {
-          update.push_error_log(
-            "launch server",
-            format!("failed to launch aws instance\n\n{e:#?}"),
-          );
-          update.finalize();
-          update_update(update.clone()).await?;
-          return Ok(update);
-        }
-        let instance = instance.unwrap();
+        let instance = match launch_ec2_instance(&name, config).await
+        {
+          Ok(instance) => instance,
+          Err(e) => {
+            update.push_error_log(
+              "launch server",
+              format!("failed to launch aws instance\n\n{e:#?}"),
+            );
+            update.finalize();
+            update_update(update.clone()).await?;
+            return Ok(update);
+          }
+        };
         update.push_simple_log(
           "launch server",
           format!(
@@ -90,6 +93,34 @@ impl Resolve<LaunchServer, User> for State {
           ..Default::default()
         }
       }
+      ServerTemplateConfig::Hetzner(config) => {
+        let datacenter = config.datacenter;
+        let server = match launch_hetzner_server(&name, config).await
+        {
+          Ok(server) => server,
+          Err(e) => {
+            update.push_error_log(
+              "launch server",
+              format!("failed to launch hetzner server\n\n{e:#?}"),
+            );
+            update.finalize();
+            update_update(update.clone()).await?;
+            return Ok(update);
+          }
+        };
+        update.push_simple_log(
+          "launch server",
+          format!(
+            "successfully launched server {name} on ip {}",
+            server.ip
+          ),
+        );
+        PartialServerConfig {
+          address: format!("http://{}:8120", server.ip).into(),
+          region: datacenter.as_ref().to_string().into(),
+          ..Default::default()
+        }
+      }
     };
 
     match self.resolve(CreateServer { name, config }, user).await {
@@ -98,6 +129,7 @@ impl Resolve<LaunchServer, User> for State {
           "create server",
           format!("created server {} ({})", server.name, server.id),
         );
+        update.other_data = server.id;
       }
       Err(e) => {
         update.push_error_log(

bin/core/src/api/read/alert.rs

@@ -1,5 +1,4 @@
 use anyhow::Context;
-use async_trait::async_trait;
 use monitor_client::{
   api::read::{
     GetAlert, GetAlertResponse, ListAlerts, ListAlertsResponse,
@@ -14,13 +13,11 @@ use mungos::{
 use resolver_api::Resolve;
 
 use crate::{
-  helpers::query::get_resource_ids_for_non_admin,
-  state::{db_client, State},
+  config::core_config, helpers::query::get_resource_ids_for_non_admin, state::{db_client, State}
 };
 
-const NUM_ALERTS_PER_PAGE: u64 = 20;
+const NUM_ALERTS_PER_PAGE: u64 = 100;
 
-#[async_trait]
 impl Resolve<ListAlerts, User> for State {
   async fn resolve(
     &self,
@@ -28,7 +25,7 @@ impl Resolve<ListAlerts, User> for State {
     user: User,
   ) -> anyhow::Result<ListAlertsResponse> {
     let mut query = query.unwrap_or_default();
-    if !user.admin {
+    if !user.admin && !core_config().transparent_mode {
       let server_ids = get_resource_ids_for_non_admin(
         &user.id,
         ResourceTargetVariant::Server,
@@ -71,7 +68,6 @@ impl Resolve<ListAlerts, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<GetAlert, User> for State {
   async fn resolve(
     &self,

bin/core/src/api/read/alerter.rs

@@ -1,7 +1,6 @@
 use std::str::FromStr;
 
 use anyhow::Context;
-use async_trait::async_trait;
 use monitor_client::{
   api::read::*,
   entities::{
@@ -15,12 +14,9 @@ use mungos::mongodb::bson::{doc, oid::ObjectId};
 use resolver_api::Resolve;
 
 use crate::{
-  helpers::query::get_resource_ids_for_non_admin,
-  resource,
-  state::{db_client, State},
+  config::core_config, helpers::query::get_resource_ids_for_non_admin, resource, state::{db_client, State}
 };
 
-#[async_trait]
 impl Resolve<GetAlerter, User> for State {
   async fn resolve(
     &self,
@@ -36,7 +32,6 @@ impl Resolve<GetAlerter, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<ListAlerters, User> for State {
   async fn resolve(
     &self,
@@ -47,14 +42,13 @@ impl Resolve<ListAlerters, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<GetAlertersSummary, User> for State {
   async fn resolve(
     &self,
     GetAlertersSummary {}: GetAlertersSummary,
     user: User,
   ) -> anyhow::Result<GetAlertersSummaryResponse> {
-    let query = if user.admin {
+    let query = if user.admin || core_config().transparent_mode {
       None
     } else {
       let ids = get_resource_ids_for_non_admin(

bin/core/src/api/read/build.rs

@@ -1,36 +1,33 @@
-use std::{collections::HashMap, str::FromStr, sync::OnceLock};
+use std::{
+  collections::{HashMap, HashSet},
+  sync::OnceLock,
+};
 
 use anyhow::Context;
 use async_timing_util::unix_timestamp_ms;
-use async_trait::async_trait;
 use futures::TryStreamExt;
 use monitor_client::{
   api::read::*,
   entities::{
-    build::{Build, BuildActionState, BuildListItem},
+    build::{Build, BuildActionState, BuildListItem, BuildState},
     permission::PermissionLevel,
-    update::{ResourceTargetVariant, UpdateStatus},
+    update::UpdateStatus,
     user::User,
     Operation,
   },
 };
 use mungos::{
   find::find_collect,
-  mongodb::{
-    bson::{doc, oid::ObjectId},
-    options::FindOptions,
-  },
+  mongodb::{bson::doc, options::FindOptions},
 };
 use resolver_api::{Resolve, ResolveToString};
 
 use crate::{
   config::core_config,
-  helpers::query::get_resource_ids_for_non_admin,
   resource,
-  state::{action_states, db_client, State},
+  state::{action_states, build_state_cache, db_client, State},
 };
 
-#[async_trait]
 impl Resolve<GetBuild, User> for State {
   async fn resolve(
     &self,
@@ -46,7 +43,6 @@ impl Resolve<GetBuild, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<ListBuilds, User> for State {
   async fn resolve(
     &self,
@@ -57,7 +53,6 @@ impl Resolve<ListBuilds, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<GetBuildActionState, User> for State {
   async fn resolve(
     &self,
@@ -80,45 +75,53 @@ impl Resolve<GetBuildActionState, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<GetBuildsSummary, User> for State {
   async fn resolve(
     &self,
     GetBuildsSummary {}: GetBuildsSummary,
     user: User,
   ) -> anyhow::Result<GetBuildsSummaryResponse> {
-    let query = if user.admin {
-      None
-    } else {
-      let ids = get_resource_ids_for_non_admin(
-        &user.id,
-        ResourceTargetVariant::Build,
-      )
-      .await?
-      .into_iter()
-      .flat_map(|id| ObjectId::from_str(&id))
-      .collect::<Vec<_>>();
-      let query = doc! {
-        "_id": { "$in": ids }
-      };
-      Some(query)
-    };
-    let total = db_client()
-      .await
-      .builds
-      .count_documents(query, None)
-      .await
-      .context("failed to count all build documents")?;
-    let res = GetBuildsSummaryResponse {
-      total: total as u32,
-    };
+    let builds = resource::list_full_for_user::<Build>(
+      Default::default(),
+      &user,
+    )
+    .await
+    .context("failed to get all builds")?;
+
+    let mut res = GetBuildsSummaryResponse::default();
+
+    let cache = build_state_cache();
+    let action_states = action_states();
+
+    for build in builds {
+      res.total += 1;
+
+      match (
+        cache.get(&build.id).await.unwrap_or_default(),
+        action_states
+          .build
+          .get(&build.id)
+          .await
+          .unwrap_or_default()
+          .get()?,
+      ) {
+        (_, action_states) if action_states.building => {
+          res.building += 1;
+        }
+        (BuildState::Ok, _) => res.ok += 1,
+        (BuildState::Failed, _) => res.failed += 1,
+        (BuildState::Unknown, _) => res.unknown += 1,
+        // will never come off the cache in the building state, since that comes from action states
+        (BuildState::Building, _) => unreachable!(),
+      }
+    }
+
     Ok(res)
   }
 }
 
 const ONE_DAY_MS: i64 = 86400000;
 
-#[async_trait]
 impl Resolve<GetBuildMonthlyStats, User> for State {
   async fn resolve(
     &self,
@@ -180,15 +183,11 @@ fn ms_to_hour(duration: i64) -> f64 {
   duration as f64 / MS_TO_HOUR_DIVISOR
 }
 
-const NUM_VERSIONS_PER_PAGE: u64 = 10;
-
-#[async_trait]
 impl Resolve<GetBuildVersions, User> for State {
   async fn resolve(
     &self,
     GetBuildVersions {
       build,
-      page,
       major,
       minor,
       patch,
@@ -224,11 +223,7 @@ impl Resolve<GetBuildVersions, User> for State {
     let versions = find_collect(
       &db_client().await.updates,
       filter,
-      FindOptions::builder()
-        .sort(doc! { "_id": -1 })
-        .limit(NUM_VERSIONS_PER_PAGE as i64)
-        .skip(page as u64 * NUM_VERSIONS_PER_PAGE)
-        .build(),
+      FindOptions::builder().sort(doc! { "_id": -1 }).build(),
     )
     .await
     .context("failed to pull versions from mongo")?
@@ -249,7 +244,6 @@ fn docker_organizations() -> &'static String {
   })
 }
 
-#[async_trait]
 impl ResolveToString<ListDockerOrganizations, User> for State {
   async fn resolve_to_string(
     &self,
@@ -259,3 +253,26 @@ impl ResolveToString<ListDockerOrganizations, User> for State {
     Ok(docker_organizations().clone())
   }
 }
+
+impl Resolve<ListCommonBuildExtraArgs, User> for State {
+  async fn resolve(
+    &self,
+    ListCommonBuildExtraArgs { query }: ListCommonBuildExtraArgs,
+    user: User,
+  ) -> anyhow::Result<ListCommonBuildExtraArgsResponse> {
+    let builds = resource::list_full_for_user::<Build>(query, &user)
+      .await
+      .context("failed to get resources matching query")?;
+
+    // first collect with guaranteed uniqueness
+    let mut res = HashSet::<String>::new();
+
+    for build in builds {
+      for extra_arg in build.config.extra_args {
+        res.insert(extra_arg);
+      }
+    }
+
+    Ok(res.into_iter().collect())
+  }
+}

bin/core/src/api/read/builder.rs

@@ -1,7 +1,6 @@
 use std::{collections::HashSet, str::FromStr};
 
 use anyhow::Context;
-use async_trait::async_trait;
 use monitor_client::{
   api::read::{self, *},
   entities::{
@@ -21,7 +20,6 @@ use crate::{
   state::{db_client, State},
 };
 
-#[async_trait]
 impl Resolve<GetBuilder, User> for State {
   async fn resolve(
     &self,
@@ -37,7 +35,6 @@ impl Resolve<GetBuilder, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<ListBuilders, User> for State {
   async fn resolve(
     &self,
@@ -48,14 +45,13 @@ impl Resolve<ListBuilders, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<GetBuildersSummary, User> for State {
   async fn resolve(
     &self,
     GetBuildersSummary {}: GetBuildersSummary,
     user: User,
   ) -> anyhow::Result<GetBuildersSummaryResponse> {
-    let query = if user.admin {
+    let query = if user.admin || core_config().transparent_mode {
       None
     } else {
       let ids = get_resource_ids_for_non_admin(
@@ -84,7 +80,6 @@ impl Resolve<GetBuildersSummary, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<GetBuilderAvailableAccounts, User> for State {
   async fn resolve(
     &self,

bin/core/src/api/read/deployment.rs

@@ -1,7 +1,6 @@
-use std::{cmp, collections::HashSet, str::FromStr};
+use std::{cmp, collections::HashSet};
 
 use anyhow::{anyhow, Context};
-use async_trait::async_trait;
 use monitor_client::{
   api::read::*,
   entities::{
@@ -11,26 +10,19 @@ use monitor_client::{
     },
     permission::PermissionLevel,
     server::Server,
-    update::{Log, ResourceTargetVariant},
+    update::Log,
     user::User,
   },
 };
-use mungos::{
-  find::find_collect,
-  mongodb::bson::{doc, oid::ObjectId},
-};
 use periphery_client::api;
 use resolver_api::Resolve;
 
 use crate::{
-  helpers::{
-    periphery_client, query::get_resource_ids_for_non_admin,
-  },
+  helpers::periphery_client,
   resource,
-  state::{action_states, db_client, deployment_status_cache, State},
+  state::{action_states, deployment_status_cache, State},
 };
 
-#[async_trait]
 impl Resolve<GetDeployment, User> for State {
   async fn resolve(
     &self,
@@ -46,7 +38,6 @@ impl Resolve<GetDeployment, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<ListDeployments, User> for State {
   async fn resolve(
     &self,
@@ -57,7 +48,6 @@ impl Resolve<ListDeployments, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<GetDeploymentContainer, User> for State {
   async fn resolve(
     &self,
@@ -84,7 +74,6 @@ impl Resolve<GetDeploymentContainer, User> for State {
 
 const MAX_LOG_LENGTH: u64 = 5000;
 
-#[async_trait]
 impl Resolve<GetLog, User> for State {
   async fn resolve(
     &self,
@@ -115,7 +104,6 @@ impl Resolve<GetLog, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<SearchLog, User> for State {
   async fn resolve(
     &self,
@@ -151,7 +139,6 @@ impl Resolve<SearchLog, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<GetDeploymentStats, User> for State {
   async fn resolve(
     &self,
@@ -179,7 +166,6 @@ impl Resolve<GetDeploymentStats, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<GetDeploymentActionState, User> for State {
   async fn resolve(
     &self,
@@ -202,34 +188,18 @@ impl Resolve<GetDeploymentActionState, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<GetDeploymentsSummary, User> for State {
   async fn resolve(
     &self,
     GetDeploymentsSummary {}: GetDeploymentsSummary,
     user: User,
   ) -> anyhow::Result<GetDeploymentsSummaryResponse> {
-    let query = if user.admin {
-      None
-    } else {
-      let ids = get_resource_ids_for_non_admin(
-        &user.id,
-        ResourceTargetVariant::Deployment,
-      )
-      .await?
-      .into_iter()
-      .flat_map(|id| ObjectId::from_str(&id))
-      .collect::<Vec<_>>();
-      let query = doc! {
-        "_id": { "$in": ids }
-      };
-      Some(query)
-    };
-
-    let deployments =
-      find_collect(&db_client().await.deployments, query, None)
-        .await
-        .context("failed to count all deployment documents")?;
+    let deployments = resource::list_full_for_user::<Deployment>(
+      Default::default(),
+      &user,
+    )
+    .await
+    .context("failed to get deployments from db")?;
     let mut res = GetDeploymentsSummaryResponse::default();
     let status_cache = deployment_status_cache();
     for deployment in deployments {
@@ -255,13 +225,12 @@ impl Resolve<GetDeploymentsSummary, User> for State {
   }
 }
 
-#[async_trait]
-impl Resolve<ListCommonExtraArgs, User> for State {
+impl Resolve<ListCommonDeploymentExtraArgs, User> for State {
   async fn resolve(
     &self,
-    ListCommonExtraArgs { query }: ListCommonExtraArgs,
+    ListCommonDeploymentExtraArgs { query }: ListCommonDeploymentExtraArgs,
     user: User,
-  ) -> anyhow::Result<ListCommonExtraArgsResponse> {
+  ) -> anyhow::Result<ListCommonDeploymentExtraArgsResponse> {
     let deployments =
       resource::list_full_for_user::<Deployment>(query, &user)
         .await

bin/core/src/api/read/mod.rs

@@ -1,13 +1,10 @@
 use std::time::Instant;
 
 use anyhow::anyhow;
-use async_trait::async_trait;
 use axum::{middleware, routing::post, Extension, Router};
 use axum_extra::{headers::ContentType, TypedHeader};
 use monitor_client::{api::read::*, entities::user::User};
-use resolver_api::{
-  derive::Resolver, Resolve, ResolveToString, Resolver,
-};
+use resolver_api::{derive::Resolver, Resolve, Resolver};
 use serde::{Deserialize, Serialize};
 use serror::Json;
 use typeshare::typeshare;
@@ -31,6 +28,7 @@ mod toml;
 mod update;
 mod user;
 mod user_group;
+mod variable;
 
 #[typeshare]
 #[derive(Serialize, Deserialize, Debug, Clone, Resolver)]
@@ -91,7 +89,7 @@ enum ReadRequest {
   GetDeploymentStats(GetDeploymentStats),
   GetLog(GetLog),
   SearchLog(SearchLog),
-  ListCommonExtraArgs(ListCommonExtraArgs),
+  ListCommonDeploymentExtraArgs(ListCommonDeploymentExtraArgs),
 
   // ==== BUILD ====
   GetBuildsSummary(GetBuildsSummary),
@@ -100,6 +98,7 @@ enum ReadRequest {
   GetBuildActionState(GetBuildActionState),
   GetBuildMonthlyStats(GetBuildMonthlyStats),
   GetBuildVersions(GetBuildVersions),
+  ListCommonBuildExtraArgs(ListCommonBuildExtraArgs),
   #[to_string_resolver]
   ListDockerOrganizations(ListDockerOrganizations),
 
@@ -143,6 +142,10 @@ enum ReadRequest {
   GetSystemStats(GetSystemStats),
   #[to_string_resolver]
   GetSystemProcesses(GetSystemProcesses),
+
+  // ==== VARIABLE ====
+  GetVariable(GetVariable),
+  ListVariables(ListVariables),
 }
 
 pub fn router() -> Router {
@@ -180,7 +183,6 @@ async fn handler(
   Ok((TypedHeader(ContentType::json()), res?))
 }
 
-#[async_trait]
 impl Resolve<GetVersion, User> for State {
   #[instrument(name = "GetVersion", level = "debug", skip(self))]
   async fn resolve(
@@ -194,7 +196,6 @@ impl Resolve<GetVersion, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<GetCoreInfo, User> for State {
   #[instrument(name = "GetCoreInfo", level = "debug", skip(self))]
   async fn resolve(
@@ -210,6 +211,7 @@ impl Resolve<GetCoreInfo, User> for State {
         .github_webhook_base_url
         .clone()
         .unwrap_or_else(|| config.host.clone()),
+      transparent_mode: config.transparent_mode,
     })
   }
 }

bin/core/src/api/read/permission.rs

@@ -1,5 +1,4 @@
 use anyhow::{anyhow, Context};
-use axum::async_trait;
 use monitor_client::{
   api::read::{
     GetPermissionLevel, GetPermissionLevelResponse, ListPermissions,
@@ -16,7 +15,6 @@ use crate::{
   state::{db_client, State},
 };
 
-#[async_trait]
 impl Resolve<ListPermissions, User> for State {
   async fn resolve(
     &self,
@@ -36,7 +34,6 @@ impl Resolve<ListPermissions, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<GetPermissionLevel, User> for State {
   async fn resolve(
     &self,
@@ -51,7 +48,6 @@ impl Resolve<GetPermissionLevel, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<ListUserTargetPermissions, User> for State {
   async fn resolve(
     &self,

bin/core/src/api/read/procedure.rs

@@ -1,7 +1,4 @@
-use std::str::FromStr;
-
 use anyhow::Context;
-use async_trait::async_trait;
 use monitor_client::{
   api::read::{
     GetProcedure, GetProcedureActionState,
@@ -10,20 +7,18 @@ use monitor_client::{
     ListProcedures, ListProceduresResponse,
   },
   entities::{
-    permission::PermissionLevel, procedure::Procedure,
-    update::ResourceTargetVariant, user::User,
+    permission::PermissionLevel,
+    procedure::{Procedure, ProcedureState},
+    user::User,
   },
 };
-use mungos::mongodb::bson::{doc, oid::ObjectId};
 use resolver_api::Resolve;
 
 use crate::{
-  helpers::query::get_resource_ids_for_non_admin,
   resource,
-  state::{action_states, db_client, State},
+  state::{action_states, procedure_state_cache, State},
 };
 
-#[async_trait]
 impl Resolve<GetProcedure, User> for State {
   async fn resolve(
     &self,
@@ -39,7 +34,6 @@ impl Resolve<GetProcedure, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<ListProcedures, User> for State {
   async fn resolve(
     &self,
@@ -50,43 +44,51 @@ impl Resolve<ListProcedures, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<GetProceduresSummary, User> for State {
   async fn resolve(
     &self,
     GetProceduresSummary {}: GetProceduresSummary,
     user: User,
   ) -> anyhow::Result<GetProceduresSummaryResponse> {
-    let query = if user.admin {
-      None
-    } else {
-      let ids = get_resource_ids_for_non_admin(
-        &user.id,
-        ResourceTargetVariant::Procedure,
-      )
-      .await?
-      .into_iter()
-      .flat_map(|id| ObjectId::from_str(&id))
-      .collect::<Vec<_>>();
-      let query = doc! {
-        "_id": { "$in": ids }
-      };
-      Some(query)
-    };
-    let total = db_client()
-      .await
-      .procedures
-      .count_documents(query, None)
-      .await
-      .context("failed to count all procedure documents")?;
-    let res = GetProceduresSummaryResponse {
-      total: total as u32,
-    };
+    let procedures = resource::list_full_for_user::<Procedure>(
+      Default::default(),
+      &user,
+    )
+    .await
+    .context("failed to get procedures from db")?;
+
+    let mut res = GetProceduresSummaryResponse::default();
+
+    let cache = procedure_state_cache();
+    let action_states = action_states();
+
+    for procedure in procedures {
+      res.total += 1;
+
+      match (
+        cache.get(&procedure.id).await.unwrap_or_default(),
+        action_states
+          .procedure
+          .get(&procedure.id)
+          .await
+          .unwrap_or_default()
+          .get()?,
+      ) {
+        (_, action_states) if action_states.running => {
+          res.running += 1;
+        }
+        (ProcedureState::Ok, _) => res.ok += 1,
+        (ProcedureState::Failed, _) => res.failed += 1,
+        (ProcedureState::Unknown, _) => res.unknown += 1,
+        // will never come off the cache in the running state, since that comes from action states
+        (ProcedureState::Running, _) => unreachable!(),
+      }
+    }
+
     Ok(res)
   }
 }
 
-#[async_trait]
 impl Resolve<GetProcedureActionState, User> for State {
   async fn resolve(
     &self,

bin/core/src/api/read/repo.rs

@@ -1,26 +1,19 @@
-use std::str::FromStr;
-
 use anyhow::Context;
-use async_trait::async_trait;
 use monitor_client::{
   api::read::*,
   entities::{
     permission::PermissionLevel,
-    repo::{Repo, RepoActionState, RepoListItem},
-    update::ResourceTargetVariant,
+    repo::{Repo, RepoActionState, RepoListItem, RepoState},
     user::User,
   },
 };
-use mungos::mongodb::bson::{doc, oid::ObjectId};
 use resolver_api::Resolve;
 
 use crate::{
-  helpers::query::get_resource_ids_for_non_admin,
   resource,
-  state::{action_states, db_client, State},
+  state::{action_states, repo_state_cache, State},
 };
 
-#[async_trait]
 impl Resolve<GetRepo, User> for State {
   async fn resolve(
     &self,
@@ -36,7 +29,6 @@ impl Resolve<GetRepo, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<ListRepos, User> for State {
   async fn resolve(
     &self,
@@ -47,7 +39,6 @@ impl Resolve<ListRepos, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<GetRepoActionState, User> for State {
   async fn resolve(
     &self,
@@ -70,38 +61,50 @@ impl Resolve<GetRepoActionState, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<GetReposSummary, User> for State {
   async fn resolve(
     &self,
     GetReposSummary {}: GetReposSummary,
     user: User,
   ) -> anyhow::Result<GetReposSummaryResponse> {
-    let query = if user.admin {
-      None
-    } else {
-      let ids = get_resource_ids_for_non_admin(
-        &user.id,
-        ResourceTargetVariant::Alerter,
-      )
-      .await?
-      .into_iter()
-      .flat_map(|id| ObjectId::from_str(&id))
-      .collect::<Vec<_>>();
-      let query = doc! {
-        "_id": { "$in": ids }
-      };
-      Some(query)
-    };
-    let total = db_client()
-      .await
-      .repos
-      .count_documents(query, None)
-      .await
-      .context("failed to count all build documents")?;
-    let res = GetReposSummaryResponse {
-      total: total as u32,
-    };
+    let repos =
+      resource::list_full_for_user::<Repo>(Default::default(), &user)
+        .await
+        .context("failed to get repos from db")?;
+
+    let mut res = GetReposSummaryResponse::default();
+
+    let cache = repo_state_cache();
+    let action_states = action_states();
+
+    for repo in repos {
+      res.total += 1;
+
+      match (
+        cache.get(&repo.id).await.unwrap_or_default(),
+        action_states
+          .repo
+          .get(&repo.id)
+          .await
+          .unwrap_or_default()
+          .get()?,
+      ) {
+        (_, action_states) if action_states.cloning => {
+          res.cloning += 1;
+        }
+        (_, action_states) if action_states.pulling => {
+          res.pulling += 1;
+        }
+        (RepoState::Ok, _) => res.ok += 1,
+        (RepoState::Failed, _) => res.failed += 1,
+        (RepoState::Unknown, _) => res.unknown += 1,
+        // will never come off the cache in the building state, since that comes from action states
+        (RepoState::Cloning, _) | (RepoState::Pulling, _) => {
+          unreachable!()
+        }
+      }
+    }
+
     Ok(res)
   }
 }

bin/core/src/api/read/search.rs

@@ -1,4 +1,3 @@
-use async_trait::async_trait;
 use monitor_client::{
   api::read::{FindResources, FindResourcesResponse},
   entities::{
@@ -19,7 +18,6 @@ const FIND_RESOURCE_TYPES: [ResourceTargetVariant; 5] = [
   ResourceTargetVariant::Procedure,
 ];
 
-#[async_trait]
 impl Resolve<FindResources, User> for State {
   async fn resolve(
     &self,

bin/core/src/api/read/server.rs

@@ -7,7 +7,6 @@ use anyhow::{anyhow, Context};
 use async_timing_util::{
   get_timelength_in_ms, unix_timestamp_ms, FIFTEEN_SECONDS_MS,
 };
-use async_trait::async_trait;
 use monitor_client::{
   api::read::*,
   entities::{
@@ -35,7 +34,6 @@ use crate::{
   state::{action_states, db_client, server_status_cache, State},
 };
 
-#[async_trait]
 impl Resolve<GetServersSummary, User> for State {
   async fn resolve(
     &self,
@@ -64,7 +62,6 @@ impl Resolve<GetServersSummary, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<GetPeripheryVersion, User> for State {
   async fn resolve(
     &self,
@@ -86,7 +83,6 @@ impl Resolve<GetPeripheryVersion, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<GetServer, User> for State {
   async fn resolve(
     &self,
@@ -102,7 +98,6 @@ impl Resolve<GetServer, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<ListServers, User> for State {
   async fn resolve(
     &self,
@@ -113,7 +108,6 @@ impl Resolve<ListServers, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<GetServerState, User> for State {
   async fn resolve(
     &self,
@@ -137,7 +131,6 @@ impl Resolve<GetServerState, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<GetServerActionState, User> for State {
   async fn resolve(
     &self,
@@ -169,7 +162,6 @@ fn system_info_cache() -> &'static SystemInfoCache {
   SYSTEM_INFO_CACHE.get_or_init(Default::default)
 }
 
-#[async_trait]
 impl ResolveToString<GetSystemInformation, User> for State {
   async fn resolve_to_string(
     &self,
@@ -205,7 +197,6 @@ impl ResolveToString<GetSystemInformation, User> for State {
   }
 }
 
-#[async_trait]
 impl ResolveToString<GetSystemStats, User> for State {
   async fn resolve_to_string(
     &self,
@@ -239,7 +230,6 @@ fn processes_cache() -> &'static ProcessesCache {
   PROCESSES_CACHE.get_or_init(Default::default)
 }
 
-#[async_trait]
 impl ResolveToString<GetSystemProcesses, User> for State {
   async fn resolve_to_string(
     &self,
@@ -276,7 +266,6 @@ impl ResolveToString<GetSystemProcesses, User> for State {
 
 const STATS_PER_PAGE: i64 = 500;
 
-#[async_trait]
 impl Resolve<GetHistoricalServerStats, User> for State {
   async fn resolve(
     &self,
@@ -330,7 +319,6 @@ impl Resolve<GetHistoricalServerStats, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<GetDockerImages, User> for State {
   async fn resolve(
     &self,
@@ -349,7 +337,6 @@ impl Resolve<GetDockerImages, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<GetDockerNetworks, User> for State {
   async fn resolve(
     &self,
@@ -368,7 +355,6 @@ impl Resolve<GetDockerNetworks, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<GetDockerContainers, User> for State {
   async fn resolve(
     &self,
@@ -387,7 +373,6 @@ impl Resolve<GetDockerContainers, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<GetAvailableAccounts, User> for State {
   async fn resolve(
     &self,
@@ -428,7 +413,6 @@ impl Resolve<GetAvailableAccounts, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<GetAvailableSecrets, User> for State {
   async fn resolve(
     &self,

bin/core/src/api/read/server_template.rs

@@ -1,7 +1,6 @@
 use std::str::FromStr;
 
 use anyhow::Context;
-use axum::async_trait;
 use monitor_client::{
   api::read::{
     GetServerTemplate, GetServerTemplateResponse,
@@ -20,7 +19,6 @@ use crate::{
   helpers::query::get_resource_ids_for_non_admin, resource, state::{db_client, State}
 };
 
-#[async_trait]
 impl Resolve<GetServerTemplate, User> for State {
   async fn resolve(
     &self,
@@ -36,7 +34,6 @@ impl Resolve<GetServerTemplate, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<ListServerTemplates, User> for State {
   async fn resolve(
     &self,
@@ -47,7 +44,6 @@ impl Resolve<ListServerTemplates, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<GetServerTemplatesSummary, User> for State {
   async fn resolve(
     &self,
@@ -72,10 +68,10 @@ impl Resolve<GetServerTemplatesSummary, User> for State {
     };
     let total = db_client()
       .await
-      .builders
+      .server_templates
       .count_documents(query, None)
       .await
-      .context("failed to count all builder documents")?;
+      .context("failed to count all server template documents")?;
     let res = GetServerTemplatesSummaryResponse {
       total: total as u32,
     };

bin/core/src/api/read/tag.rs

@@ -1,5 +1,4 @@
 use anyhow::Context;
-use async_trait::async_trait;
 use monitor_client::{
   api::read::{GetTag, ListTags},
   entities::{tag::Tag, user::User},
@@ -12,7 +11,6 @@ use crate::{
   state::{db_client, State},
 };
 
-#[async_trait]
 impl Resolve<GetTag, User> for State {
   async fn resolve(
     &self,
@@ -23,7 +21,6 @@ impl Resolve<GetTag, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<ListTags, User> for State {
   async fn resolve(
     &self,

bin/core/src/api/read/toml.rs

@@ -1,7 +1,6 @@
 use std::collections::HashMap;
 
 use anyhow::Context;
-use axum::async_trait;
 use monitor_client::{
   api::{
     execute::Execution,
@@ -38,7 +37,6 @@ use crate::{
   state::{db_client, State},
 };
 
-#[async_trait]
 impl Resolve<ExportAllResourcesToToml, User> for State {
   async fn resolve(
     &self,
@@ -125,7 +123,6 @@ impl Resolve<ExportAllResourcesToToml, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<ExportResourcesToToml, User> for State {
   async fn resolve(
     &self,

bin/core/src/api/read/update.rs

@@ -1,7 +1,6 @@
 use std::collections::HashMap;
 
 use anyhow::{anyhow, Context};
-use async_trait::async_trait;
 use monitor_client::{
   api::read::{GetUpdate, ListUpdates, ListUpdatesResponse},
   entities::{
@@ -28,21 +27,21 @@ use mungos::{
 use resolver_api::Resolve;
 
 use crate::{
+  config::core_config,
   helpers::query::get_resource_ids_for_non_admin,
   resource,
   state::{db_client, State},
 };
 
-const UPDATES_PER_PAGE: i64 = 20;
+const UPDATES_PER_PAGE: i64 = 100;
 
-#[async_trait]
 impl Resolve<ListUpdates, User> for State {
   async fn resolve(
     &self,
     ListUpdates { query, page }: ListUpdates,
     user: User,
   ) -> anyhow::Result<ListUpdatesResponse> {
-    let query = if user.admin {
+    let query = if user.admin || core_config().transparent_mode {
       query
     } else {
       let server_ids = get_resource_ids_for_non_admin(
@@ -141,6 +140,7 @@ impl Resolve<ListUpdates, User> for State {
         target: u.target,
         status: u.status,
         version: u.version,
+        other_data: u.other_data,
       }
     })
     .collect::<Vec<_>>();
@@ -155,7 +155,6 @@ impl Resolve<ListUpdates, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<GetUpdate, User> for State {
   async fn resolve(
     &self,
@@ -166,7 +165,7 @@ impl Resolve<GetUpdate, User> for State {
       .await
       .context("failed to query to db")?
       .context("no update exists with given id")?;
-    if user.admin {
+    if user.admin || core_config().transparent_mode {
       return Ok(update);
     }
     match &update.target {

bin/core/src/api/read/user.rs

@@ -1,5 +1,4 @@
 use anyhow::{anyhow, Context};
-use async_trait::async_trait;
 use monitor_client::{
   api::read::{
     GetUsername, GetUsernameResponse, ListApiKeys,
@@ -15,7 +14,6 @@ use resolver_api::Resolve;
 
 use crate::state::{db_client, State};
 
-#[async_trait]
 impl Resolve<GetUsername, User> for State {
   async fn resolve(
     &self,
@@ -27,13 +25,19 @@ impl Resolve<GetUsername, User> for State {
       .context("failed at mongo query for user")?
       .context("no user found with id")?;
 
+    let avatar = match user.config {
+      UserConfig::Github { avatar, .. } => Some(avatar),
+      UserConfig::Google { avatar, .. } => Some(avatar),
+      _ => None,
+    };
+
     Ok(GetUsernameResponse {
       username: user.username,
+      avatar,
     })
   }
 }
 
-#[async_trait]
 impl Resolve<ListUsers, User> for State {
   async fn resolve(
     &self,
@@ -52,7 +56,6 @@ impl Resolve<ListUsers, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<ListApiKeys, User> for State {
   async fn resolve(
     &self,
@@ -76,7 +79,6 @@ impl Resolve<ListApiKeys, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<ListApiKeysForServiceUser, User> for State {
   async fn resolve(
     &self,

bin/core/src/api/read/user_group.rs

@@ -1,7 +1,6 @@
 use std::str::FromStr;
 
 use anyhow::Context;
-use async_trait::async_trait;
 use monitor_client::{
   api::read::{
     GetUserGroup, GetUserGroupResponse, ListUserGroups,
@@ -17,7 +16,6 @@ use resolver_api::Resolve;
 
 use crate::state::{db_client, State};
 
-#[async_trait]
 impl Resolve<GetUserGroup, User> for State {
   async fn resolve(
     &self,
@@ -43,7 +41,6 @@ impl Resolve<GetUserGroup, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<ListUserGroups, User> for State {
   async fn resolve(
     &self,

bin/core/src/api/read/variable.rs

@@ -0,0 +1,43 @@
+use anyhow::Context;
+use monitor_client::{
+  api::read::{
+    GetVariable, GetVariableResponse, ListVariables,
+    ListVariablesResponse,
+  },
+  entities::user::User,
+};
+use mungos::find::find_collect;
+use resolver_api::Resolve;
+
+use crate::{
+  config::core_config,
+  helpers::query::get_variable,
+  state::{db_client, State},
+};
+
+impl Resolve<GetVariable, User> for State {
+  async fn resolve(
+    &self,
+    GetVariable { name }: GetVariable,
+    _: User,
+  ) -> anyhow::Result<GetVariableResponse> {
+    get_variable(&name).await
+  }
+}
+
+impl Resolve<ListVariables, User> for State {
+  async fn resolve(
+    &self,
+    ListVariables {}: ListVariables,
+    _: User,
+  ) -> anyhow::Result<ListVariablesResponse> {
+    let variables =
+      find_collect(&db_client().await.variables, None, None)
+        .await
+        .context("failed to query db for variables")?;
+    Ok(ListVariablesResponse {
+      variables,
+      secrets: core_config().secrets.keys().cloned().collect(),
+    })
+  }
+}

bin/core/src/api/write/alerter.rs

@@ -1,4 +1,3 @@
-use async_trait::async_trait;
 use monitor_client::{
   api::write::{
     CopyAlerter, CreateAlerter, DeleteAlerter, UpdateAlerter,
@@ -11,7 +10,6 @@ use resolver_api::Resolve;
 
 use crate::{resource, state::State};
 
-#[async_trait]
 impl Resolve<CreateAlerter, User> for State {
   #[instrument(name = "CreateAlerter", skip(self, user))]
   async fn resolve(
@@ -23,7 +21,6 @@ impl Resolve<CreateAlerter, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<CopyAlerter, User> for State {
   #[instrument(name = "CopyAlerter", skip(self, user))]
   async fn resolve(
@@ -44,7 +41,6 @@ impl Resolve<CopyAlerter, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<DeleteAlerter, User> for State {
   #[instrument(name = "DeleteAlerter", skip(self, user))]
   async fn resolve(
@@ -56,7 +52,6 @@ impl Resolve<DeleteAlerter, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<UpdateAlerter, User> for State {
   #[instrument(name = "UpdateAlerter", skip(self, user))]
   async fn resolve(

bin/core/src/api/write/api_key.rs

@@ -1,5 +1,4 @@
 use anyhow::{anyhow, Context};
-use async_trait::async_trait;
 use monitor_client::{
   api::write::*,
   entities::{
@@ -20,7 +19,6 @@ use crate::{
 const SECRET_LENGTH: usize = 40;
 const BCRYPT_COST: u32 = 10;
 
-#[async_trait]
 impl Resolve<CreateApiKey, User> for State {
   #[instrument(
     name = "CreateApiKey",
@@ -57,7 +55,6 @@ impl Resolve<CreateApiKey, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<DeleteApiKey, User> for State {
   #[instrument(
     name = "DeleteApiKey",
@@ -88,7 +85,6 @@ impl Resolve<DeleteApiKey, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<CreateApiKeyForServiceUser, User> for State {
   #[instrument(name = "CreateApiKeyForServiceUser", skip(self, user))]
   async fn resolve(
@@ -117,7 +113,6 @@ impl Resolve<CreateApiKeyForServiceUser, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<DeleteApiKeyForServiceUser, User> for State {
   #[instrument(name = "DeleteApiKeyForServiceUser", skip(self, user))]
   async fn resolve(

bin/core/src/api/write/build.rs

@@ -1,4 +1,3 @@
-use async_trait::async_trait;
 use monitor_client::{
   api::write::*,
   entities::{build::Build, permission::PermissionLevel, user::User},
@@ -7,7 +6,6 @@ use resolver_api::Resolve;
 
 use crate::{resource, state::State};
 
-#[async_trait]
 impl Resolve<CreateBuild, User> for State {
   #[instrument(name = "CreateBuild", skip(self, user))]
   async fn resolve(
@@ -19,7 +17,6 @@ impl Resolve<CreateBuild, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<CopyBuild, User> for State {
   #[instrument(name = "CopyBuild", skip(self, user))]
   async fn resolve(
@@ -40,7 +37,6 @@ impl Resolve<CopyBuild, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<DeleteBuild, User> for State {
   #[instrument(name = "DeleteBuild", skip(self, user))]
   async fn resolve(
@@ -52,7 +48,6 @@ impl Resolve<DeleteBuild, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<UpdateBuild, User> for State {
   #[instrument(name = "UpdateBuild", skip(self, user))]
   async fn resolve(

bin/core/src/api/write/builder.rs

@@ -1,4 +1,3 @@
-use async_trait::async_trait;
 use monitor_client::{
   api::write::*,
   entities::{
@@ -9,7 +8,6 @@ use resolver_api::Resolve;
 
 use crate::{resource, state::State};
 
-#[async_trait]
 impl Resolve<CreateBuilder, User> for State {
   #[instrument(name = "CreateBuilder", skip(self, user))]
   async fn resolve(
@@ -21,7 +19,6 @@ impl Resolve<CreateBuilder, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<CopyBuilder, User> for State {
   #[instrument(name = "CopyBuilder", skip(self, user))]
   async fn resolve(
@@ -39,7 +36,6 @@ impl Resolve<CopyBuilder, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<DeleteBuilder, User> for State {
   #[instrument(name = "DeleteBuilder", skip(self, user))]
   async fn resolve(
@@ -51,7 +47,6 @@ impl Resolve<DeleteBuilder, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<UpdateBuilder, User> for State {
   #[instrument(name = "UpdateBuilder", skip(self, user))]
   async fn resolve(

bin/core/src/api/write/deployment.rs

@@ -1,5 +1,4 @@
 use anyhow::{anyhow, Context};
-use async_trait::async_trait;
 use monitor_client::{
   api::write::*,
   entities::{
@@ -27,7 +26,6 @@ use crate::{
   state::{action_states, db_client, State},
 };
 
-#[async_trait]
 impl Resolve<CreateDeployment, User> for State {
   #[instrument(name = "CreateDeployment", skip(self, user))]
   async fn resolve(
@@ -39,7 +37,6 @@ impl Resolve<CreateDeployment, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<CopyDeployment, User> for State {
   #[instrument(name = "CopyDeployment", skip(self, user))]
   async fn resolve(
@@ -58,7 +55,6 @@ impl Resolve<CopyDeployment, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<DeleteDeployment, User> for State {
   #[instrument(name = "DeleteDeployment", skip(self, user))]
   async fn resolve(
@@ -70,7 +66,6 @@ impl Resolve<DeleteDeployment, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<UpdateDeployment, User> for State {
   #[instrument(name = "UpdateDeployment", skip(self, user))]
   async fn resolve(
@@ -82,7 +77,6 @@ impl Resolve<UpdateDeployment, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<RenameDeployment, User> for State {
   #[instrument(name = "RenameDeployment", skip(self, user))]
   async fn resolve(

bin/core/src/api/write/description.rs

@@ -1,5 +1,4 @@
 use anyhow::anyhow;
-use async_trait::async_trait;
 use monitor_client::{
   api::write::{UpdateDescription, UpdateDescriptionResponse},
   entities::{
@@ -13,7 +12,6 @@ use resolver_api::Resolve;
 
 use crate::{resource, state::State};
 
-#[async_trait]
 impl Resolve<UpdateDescription, User> for State {
   #[instrument(name = "UpdateDescription", skip(self, user))]
   async fn resolve(

bin/core/src/api/write/mod.rs

@@ -4,7 +4,7 @@ use anyhow::{anyhow, Context};
 use axum::{middleware, routing::post, Extension, Router};
 use axum_extra::{headers::ContentType, TypedHeader};
 use monitor_client::{api::write::*, entities::user::User};
-use resolver_api::{derive::Resolver, Resolve, Resolver};
+use resolver_api::{derive::Resolver, Resolver};
 use serde::{Deserialize, Serialize};
 use serror::Json;
 use typeshare::typeshare;
@@ -26,6 +26,7 @@ mod server_template;
 mod tag;
 mod user;
 mod user_group;
+mod variable;
 
 #[typeshare]
 #[derive(Serialize, Deserialize, Debug, Clone, Resolver)]
@@ -116,6 +117,12 @@ enum WriteRequest {
   DeleteTag(DeleteTag),
   RenameTag(RenameTag),
   UpdateTagsOnResource(UpdateTagsOnResource),
+
+  // ==== VARIABLE ====
+  CreateVariable(CreateVariable),
+  UpdateVariableValue(UpdateVariableValue),
+  UpdateVariableDescription(UpdateVariableDescription),
+  DeleteVariable(DeleteVariable),
 }
 
 pub fn router() -> Router {

bin/core/src/api/write/permissions.rs

@@ -1,7 +1,6 @@
 use std::str::FromStr;
 
 use anyhow::{anyhow, Context};
-use async_trait::async_trait;
 use monitor_client::{
   api::write::{
     UpdatePermissionOnTarget, UpdatePermissionOnTargetResponse,
@@ -27,7 +26,6 @@ use crate::{
   state::{db_client, State},
 };
 
-#[async_trait]
 impl Resolve<UpdateUserBasePermissions, User> for State {
   #[instrument(name = "UpdateUserBasePermissions", skip(self, admin))]
   async fn resolve(
@@ -75,7 +73,6 @@ impl Resolve<UpdateUserBasePermissions, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<UpdatePermissionOnTarget, User> for State {
   #[instrument(name = "UpdatePermissionOnTarget", skip(self, admin))]
   async fn resolve(

bin/core/src/api/write/procedure.rs

@@ -1,4 +1,3 @@
-use async_trait::async_trait;
 use monitor_client::{
   api::write::*,
   entities::{
@@ -9,7 +8,6 @@ use resolver_api::Resolve;
 
 use crate::{resource, state::State};
 
-#[async_trait]
 impl Resolve<CreateProcedure, User> for State {
   #[instrument(name = "CreateProcedure", skip(self, user))]
   async fn resolve(
@@ -21,7 +19,6 @@ impl Resolve<CreateProcedure, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<CopyProcedure, User> for State {
   #[instrument(name = "CopyProcedure", skip(self, user))]
   async fn resolve(
@@ -40,7 +37,6 @@ impl Resolve<CopyProcedure, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<UpdateProcedure, User> for State {
   #[instrument(name = "UpdateProcedure", skip(self, user))]
   async fn resolve(
@@ -52,7 +48,6 @@ impl Resolve<UpdateProcedure, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<DeleteProcedure, User> for State {
   #[instrument(name = "DeleteProcedure", skip(self, user))]
   async fn resolve(

bin/core/src/api/write/repo.rs

@@ -1,4 +1,3 @@
-use async_trait::async_trait;
 use monitor_client::{
   api::write::*,
   entities::{permission::PermissionLevel, repo::Repo, user::User},
@@ -7,7 +6,6 @@ use resolver_api::Resolve;
 
 use crate::{resource, state::State};
 
-#[async_trait]
 impl Resolve<CreateRepo, User> for State {
   #[instrument(name = "CreateRepo", skip(self, user))]
   async fn resolve(
@@ -19,7 +17,6 @@ impl Resolve<CreateRepo, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<CopyRepo, User> for State {
   #[instrument(name = "CopyRepo", skip(self, user))]
   async fn resolve(
@@ -38,7 +35,6 @@ impl Resolve<CopyRepo, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<DeleteRepo, User> for State {
   #[instrument(name = "DeleteRepo", skip(self, user))]
   async fn resolve(
@@ -50,7 +46,6 @@ impl Resolve<DeleteRepo, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<UpdateRepo, User> for State {
   #[instrument(name = "UpdateRepo", skip(self, user))]
   async fn resolve(

bin/core/src/api/write/server.rs

@@ -1,5 +1,4 @@
 use anyhow::Context;
-use async_trait::async_trait;
 use monitor_client::{
   api::write::*,
   entities::{
@@ -25,7 +24,6 @@ use crate::{
   state::{db_client, State},
 };
 
-#[async_trait]
 impl Resolve<CreateServer, User> for State {
   #[instrument(name = "CreateServer", skip(self, user))]
   async fn resolve(
@@ -37,7 +35,6 @@ impl Resolve<CreateServer, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<DeleteServer, User> for State {
   #[instrument(name = "DeleteServer", skip(self, user))]
   async fn resolve(
@@ -49,7 +46,6 @@ impl Resolve<DeleteServer, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<UpdateServer, User> for State {
   #[instrument(name = "UpdateServer", skip(self, user))]
   async fn resolve(
@@ -61,7 +57,6 @@ impl Resolve<UpdateServer, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<RenameServer, User> for State {
   #[instrument(name = "RenameServer", skip(self, user))]
   async fn resolve(
@@ -91,7 +86,6 @@ impl Resolve<RenameServer, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<CreateNetwork, User> for State {
   #[instrument(name = "CreateNetwork", skip(self, user))]
   async fn resolve(
@@ -129,7 +123,6 @@ impl Resolve<CreateNetwork, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<DeleteNetwork, User> for State {
   #[instrument(name = "DeleteNetwork", skip(self, user))]
   async fn resolve(

bin/core/src/api/write/server_template.rs

@@ -1,4 +1,3 @@
-use axum::async_trait;
 use monitor_client::{
   api::write::{
     CopyServerTemplate, CreateServerTemplate, DeleteServerTemplate,
@@ -13,7 +12,6 @@ use resolver_api::Resolve;
 
 use crate::{resource, state::State};
 
-#[async_trait]
 impl Resolve<CreateServerTemplate, User> for State {
   async fn resolve(
     &self,
@@ -24,7 +22,6 @@ impl Resolve<CreateServerTemplate, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<CopyServerTemplate, User> for State {
   async fn resolve(
     &self,
@@ -43,7 +40,6 @@ impl Resolve<CopyServerTemplate, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<DeleteServerTemplate, User> for State {
   async fn resolve(
     &self,
@@ -54,7 +50,6 @@ impl Resolve<DeleteServerTemplate, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<UpdateServerTemplate, User> for State {
   async fn resolve(
     &self,

bin/core/src/api/write/tag.rs

@@ -1,7 +1,6 @@
 use std::str::FromStr;
 
 use anyhow::{anyhow, Context};
-use async_trait::async_trait;
 use monitor_client::{
   api::write::{
     CreateTag, DeleteTag, RenameTag, UpdateTagsOnResource,
@@ -27,7 +26,6 @@ use crate::{
   state::{db_client, State},
 };
 
-#[async_trait]
 impl Resolve<CreateTag, User> for State {
   #[instrument(name = "CreateTag", skip(self, user))]
   async fn resolve(
@@ -60,7 +58,6 @@ impl Resolve<CreateTag, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<RenameTag, User> for State {
   async fn resolve(
     &self,
@@ -86,7 +83,6 @@ impl Resolve<RenameTag, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<DeleteTag, User> for State {
   #[instrument(name = "DeleteTag", skip(self, user))]
   async fn resolve(
@@ -113,7 +109,6 @@ impl Resolve<DeleteTag, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<UpdateTagsOnResource, User> for State {
   #[instrument(name = "UpdateTagsOnResource", skip(self, user))]
   async fn resolve(

bin/core/src/api/write/user.rs

@@ -1,7 +1,6 @@
 use std::{collections::VecDeque, str::FromStr};
 
 use anyhow::{anyhow, Context};
-use async_trait::async_trait;
 use monitor_client::{
   api::write::{
     CreateServiceUser, CreateServiceUserResponse, PushRecentlyViewed,
@@ -11,6 +10,7 @@ use monitor_client::{
   },
   entities::{
     monitor_timestamp,
+    update::ResourceTarget,
     user::{User, UserConfig},
   },
 };
@@ -27,7 +27,6 @@ use crate::{
 
 const RECENTLY_VIEWED_MAX: usize = 10;
 
-#[async_trait]
 impl Resolve<PushRecentlyViewed, User> for State {
   #[instrument(name = "PushRecentlyViewed", skip(self, user))]
   async fn resolve(
@@ -35,35 +34,48 @@ impl Resolve<PushRecentlyViewed, User> for State {
     PushRecentlyViewed { resource }: PushRecentlyViewed,
     user: User,
   ) -> anyhow::Result<PushRecentlyViewedResponse> {
-    let mut recently_viewed = get_user(&user.id)
-      .await?
-      .recently_viewed
+    let user = get_user(&user.id).await?;
+
+    let (recents, id, field) = match resource {
+      ResourceTarget::Server(id) => {
+        (user.recent_servers, id, "recent_servers")
+      }
+      ResourceTarget::Deployment(id) => {
+        (user.recent_deployments, id, "recent_deployments")
+      }
+      ResourceTarget::Build(id) => {
+        (user.recent_builds, id, "recent_builds")
+      }
+      ResourceTarget::Repo(id) => {
+        (user.recent_repos, id, "recent_repos")
+      }
+      ResourceTarget::Procedure(id) => {
+        (user.recent_procedures, id, "recent_procedures")
+      }
+      _ => return Ok(PushRecentlyViewedResponse {}),
+    };
+
+    let mut recents = recents
       .into_iter()
-      .filter(|r| !resource.eq(r))
+      .filter(|_id| !id.eq(_id))
       .take(RECENTLY_VIEWED_MAX - 1)
       .collect::<VecDeque<_>>();
-
-    recently_viewed.push_front(resource);
-
-    let recently_viewed = to_bson(&recently_viewed)
-      .context("failed to convert recently views to bson")?;
+    recents.push_front(id);
+    let update = doc! { field: to_bson(&recents)? };
 
     update_one_by_id(
       &db_client().await.users,
       &user.id,
-      mungos::update::Update::Set(doc! {
-        "recently_viewed": recently_viewed
-      }),
+      mungos::update::Update::Set(update),
       None,
     )
     .await
-    .context("context")?;
+    .with_context(|| format!("failed to update {field}"))?;
 
     Ok(PushRecentlyViewedResponse {})
   }
 }
 
-#[async_trait]
 impl Resolve<SetLastSeenUpdate, User> for State {
   #[instrument(name = "SetLastSeenUpdate", skip(self, user))]
   async fn resolve(
@@ -85,7 +97,6 @@ impl Resolve<SetLastSeenUpdate, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<CreateServiceUser, User> for State {
   #[instrument(name = "CreateServiceUser", skip(self, user))]
   async fn resolve(
@@ -112,7 +123,11 @@ impl Resolve<CreateServiceUser, User> for State {
       create_server_permissions: false,
       create_build_permissions: false,
       last_update_view: 0,
-      recently_viewed: Vec::new(),
+      recent_servers: Vec::new(),
+      recent_deployments: Vec::new(),
+      recent_builds: Vec::new(),
+      recent_repos: Vec::new(),
+      recent_procedures: Vec::new(),
       updated_at: monitor_timestamp(),
     };
     user.id = db_client()
@@ -129,7 +144,6 @@ impl Resolve<CreateServiceUser, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<UpdateServiceUserDescription, User> for State {
   #[instrument(
     name = "UpdateServiceUserDescription",

bin/core/src/api/write/user_group.rs

@@ -1,7 +1,6 @@
 use std::{collections::HashMap, str::FromStr};
 
 use anyhow::{anyhow, Context};
-use axum::async_trait;
 use monitor_client::{
   api::write::{
     AddUserToUserGroup, CreateUserGroup, DeleteUserGroup,
@@ -18,7 +17,6 @@ use resolver_api::Resolve;
 
 use crate::state::{db_client, State};
 
-#[async_trait]
 impl Resolve<CreateUserGroup, User> for State {
   async fn resolve(
     &self,
@@ -51,7 +49,6 @@ impl Resolve<CreateUserGroup, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<RenameUserGroup, User> for State {
   async fn resolve(
     &self,
@@ -77,7 +74,6 @@ impl Resolve<RenameUserGroup, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<DeleteUserGroup, User> for State {
   async fn resolve(
     &self,
@@ -111,7 +107,6 @@ impl Resolve<DeleteUserGroup, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<AddUserToUserGroup, User> for State {
   async fn resolve(
     &self,
@@ -155,7 +150,6 @@ impl Resolve<AddUserToUserGroup, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<RemoveUserFromUserGroup, User> for State {
   async fn resolve(
     &self,
@@ -202,7 +196,6 @@ impl Resolve<RemoveUserFromUserGroup, User> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<SetUsersInUserGroup, User> for State {
   async fn resolve(
     &self,

bin/core/src/api/write/variable.rs

@@ -0,0 +1,169 @@
+use anyhow::{anyhow, Context};
+use monitor_client::{
+  api::write::{
+    CreateVariable, CreateVariableResponse, DeleteVariable,
+    DeleteVariableResponse, UpdateVariableDescription,
+    UpdateVariableDescriptionResponse, UpdateVariableValue,
+    UpdateVariableValueResponse,
+  },
+  entities::{
+    update::ResourceTarget, user::User, variable::Variable, Operation,
+  },
+};
+use mungos::mongodb::bson::doc;
+use resolver_api::Resolve;
+
+use crate::{
+  helpers::{
+    query::get_variable,
+    update::{add_update, make_update},
+  },
+  state::{db_client, State},
+};
+
+impl Resolve<CreateVariable, User> for State {
+  async fn resolve(
+    &self,
+    CreateVariable {
+      name,
+      value,
+      description,
+    }: CreateVariable,
+    user: User,
+  ) -> anyhow::Result<CreateVariableResponse> {
+    if !user.admin {
+      return Err(anyhow!("only admins can create variables"));
+    }
+
+    let variable = Variable {
+      name,
+      value,
+      description,
+    };
+
+    db_client()
+      .await
+      .variables
+      .insert_one(&variable, None)
+      .await
+      .context("failed to create variable on db")?;
+
+    let mut update = make_update(
+      ResourceTarget::system(),
+      Operation::CreateVariable,
+      &user,
+    );
+
+    update
+      .push_simple_log("create variable", format!("{variable:#?}"));
+    update.finalize();
+
+    add_update(update).await?;
+
+    get_variable(&variable.name).await
+  }
+}
+
+impl Resolve<UpdateVariableValue, User> for State {
+  async fn resolve(
+    &self,
+    UpdateVariableValue { name, value }: UpdateVariableValue,
+    user: User,
+  ) -> anyhow::Result<UpdateVariableValueResponse> {
+    if !user.admin {
+      return Err(anyhow!("only admins can create variables"));
+    }
+
+    let variable = get_variable(&name).await?;
+
+    if value == variable.value {
+      return Err(anyhow!("no change"));
+    }
+
+    db_client()
+      .await
+      .variables
+      .update_one(
+        doc! { "name": &name },
+        doc! { "$set": { "value": &value } },
+        None,
+      )
+      .await
+      .context("failed to update variable value on db")?;
+
+    let mut update = make_update(
+      ResourceTarget::system(),
+      Operation::UpdateVariableValue,
+      &user,
+    );
+
+    update.push_simple_log(
+      "update variable value",
+      format!(
+        "<span class=\"text-muted-foreground\">variable</span>: '{name}'\n<span class=\"text-muted-foreground\">from</span>: <span class=\"text-red-500\">{}</span>\n<span class=\"text-muted-foreground\">to</span>:   <span class=\"text-green-500\">{value}</span>",
+        variable.value
+      ),
+    );
+    update.finalize();
+
+    add_update(update).await?;
+
+    get_variable(&name).await
+  }
+}
+
+impl Resolve<UpdateVariableDescription, User> for State {
+  async fn resolve(
+    &self,
+    UpdateVariableDescription { name, description }: UpdateVariableDescription,
+    user: User,
+  ) -> anyhow::Result<UpdateVariableDescriptionResponse> {
+    if !user.admin {
+      return Err(anyhow!("only admins can create variables"));
+    }
+    db_client()
+      .await
+      .variables
+      .update_one(
+        doc! { "name": &name },
+        doc! { "$set": { "description": &description } },
+        None,
+      )
+      .await
+      .context("failed to update variable description on db")?;
+    get_variable(&name).await
+  }
+}
+
+impl Resolve<DeleteVariable, User> for State {
+  async fn resolve(
+    &self,
+    DeleteVariable { name }: DeleteVariable,
+    user: User,
+  ) -> anyhow::Result<DeleteVariableResponse> {
+    if !user.admin {
+      return Err(anyhow!("only admins can create variables"));
+    }
+    let variable = get_variable(&name).await?;
+    db_client()
+      .await
+      .variables
+      .delete_one(doc! { "name": &name }, None)
+      .await
+      .context("failed to delete variable on db")?;
+
+    let mut update = make_update(
+      ResourceTarget::system(),
+      Operation::DeleteVariable,
+      &user,
+    );
+
+    update
+      .push_simple_log("delete variable", format!("{variable:#?}"));
+    update.finalize();
+
+    add_update(update).await?;
+
+    Ok(variable)
+  }
+}

bin/core/src/auth/github/mod.rs

@@ -86,7 +86,11 @@ async fn callback(
         create_build_permissions: no_users_exist,
         updated_at: ts,
         last_update_view: 0,
-        recently_viewed: Vec::new(),
+        recent_servers: Vec::new(),
+        recent_deployments: Vec::new(),
+        recent_builds: Vec::new(),
+        recent_repos: Vec::new(),
+        recent_procedures: Vec::new(),
         config: UserConfig::Github {
           github_id,
           avatar: github_user.avatar_url,

bin/core/src/auth/google/mod.rs

@@ -101,7 +101,11 @@ async fn callback(
         create_build_permissions: no_users_exist,
         updated_at: ts,
         last_update_view: 0,
-        recently_viewed: Vec::new(),
+        recent_servers: Vec::new(),
+        recent_deployments: Vec::new(),
+        recent_builds: Vec::new(),
+        recent_repos: Vec::new(),
+        recent_procedures: Vec::new(),
         config: UserConfig::Google {
           google_id,
           avatar: google_user.picture,

bin/core/src/auth/local.rs

@@ -2,7 +2,6 @@ use std::str::FromStr;
 
 use anyhow::{anyhow, Context};
 use async_timing_util::unix_timestamp_ms;
-use async_trait::async_trait;
 use axum::http::HeaderMap;
 use monitor_client::{
   api::auth::{
@@ -22,7 +21,6 @@ use crate::{
 
 const BCRYPT_COST: u32 = 10;
 
-#[async_trait]
 impl Resolve<CreateLocalUser, HeaderMap> for State {
   #[instrument(name = "CreateLocalUser", skip(self))]
   async fn resolve(
@@ -63,7 +61,11 @@ impl Resolve<CreateLocalUser, HeaderMap> for State {
       create_build_permissions: no_users_exist,
       updated_at: ts,
       last_update_view: 0,
-      recently_viewed: Vec::new(),
+      recent_servers: Vec::new(),
+      recent_deployments: Vec::new(),
+      recent_builds: Vec::new(),
+      recent_repos: Vec::new(),
+      recent_procedures: Vec::new(),
       config: UserConfig::Local { password },
     };
 
@@ -86,7 +88,6 @@ impl Resolve<CreateLocalUser, HeaderMap> for State {
   }
 }
 
-#[async_trait]
 impl Resolve<LoginLocalUser, HeaderMap> for State {
   #[instrument(name = "LoginLocalUser", level = "debug", skip(self))]
   async fn resolve(

bin/core/src/auth/mod.rs

@@ -8,7 +8,9 @@ use axum::{
 use monitor_client::entities::{monitor_timestamp, user::User};
 use mungos::mongodb::bson::doc;
 use rand::{distributions::Alphanumeric, thread_rng, Rng};
+use reqwest::StatusCode;
 use serde::Deserialize;
+use serror::AddStatusCode;
 
 use crate::{
   helpers::query::get_user,
@@ -36,7 +38,9 @@ pub async fn auth_request(
   mut req: Request,
   next: Next,
 ) -> serror::Result<Response> {
-  let user = authenticate_check_enabled(&headers).await?;
+  let user = authenticate_check_enabled(&headers)
+    .await
+    .status_code(StatusCode::UNAUTHORIZED)?;
   req.extensions_mut().insert(user);
   Ok(next.run(req).await)
 }

bin/core/src/cloud/aws.rs

@@ -16,7 +16,7 @@ use monitor_client::entities::{
   alert::{Alert, AlertData, AlertDataVariant},
   monitor_timestamp,
   server::stats::SeverityLevel,
-  server_template::AwsServerTemplateConfig,
+  server_template::aws::AwsServerTemplateConfig,
   update::ResourceTarget,
 };
 
@@ -165,7 +165,7 @@ pub async fn terminate_ec2_instance_with_retry(
       }
       Err(e) => {
         if i == MAX_TERMINATION_TRIES - 1 {
-          error!("failed to terminate instance {instance_id}.");
+          error!("failed to terminate aws instance {instance_id}.");
           let alert = Alert {
             id: Default::default(),
             ts: monitor_timestamp(),
@@ -175,6 +175,7 @@ pub async fn terminate_ec2_instance_with_retry(
             variant: AlertDataVariant::AwsBuilderTerminationFailed,
             data: AlertData::AwsBuilderTerminationFailed {
               instance_id: instance_id.to_string(),
+              message: format!("{e:#}"),
             },
             resolved_ts: None,
           };
@@ -191,7 +192,7 @@ pub async fn terminate_ec2_instance_with_retry(
   unreachable!()
 }
 
-#[instrument]
+#[instrument(skip(client))]
 async fn terminate_ec2_instance_inner(
   client: &Client,
   instance_id: &str,

bin/core/src/cloud/hetzner/client.rs

@@ -0,0 +1,156 @@
+use anyhow::{anyhow, Context};
+use axum::http::{HeaderName, HeaderValue};
+use reqwest::{RequestBuilder, StatusCode};
+use serde::{de::DeserializeOwned, Serialize};
+
+use super::{
+  common::{
+    HetznerActionResponse, HetznerDatacenterResponse,
+    HetznerServerResponse, HetznerVolumeResponse,
+  },
+  create_server::{CreateServerBody, CreateServerResponse},
+  create_volume::{CreateVolumeBody, CreateVolumeResponse},
+};
+
+const BASE_URL: &str = "https://api.hetzner.cloud/v1";
+
+pub struct HetznerClient(reqwest::Client);
+
+impl HetznerClient {
+  pub fn new(token: &str) -> HetznerClient {
+    HetznerClient(
+      reqwest::ClientBuilder::new()
+        .default_headers(
+          [(
+            HeaderName::from_static("authorization"),
+            HeaderValue::from_str(&format!("Bearer {token}"))
+              .unwrap(),
+          )]
+          .into_iter()
+          .collect(),
+        )
+        .build()
+        .context("failed to build Hetzner request client")
+        .unwrap(),
+    )
+  }
+
+  pub async fn get_server(
+    &self,
+    id: i64,
+  ) -> anyhow::Result<HetznerServerResponse> {
+    self.get(&format!("/servers/{id}")).await
+  }
+
+  pub async fn create_server(
+    &self,
+    body: &CreateServerBody,
+  ) -> anyhow::Result<CreateServerResponse> {
+    self.post("/servers", body).await
+  }
+
+  pub async fn delete_server(
+    &self,
+    id: i64,
+  ) -> anyhow::Result<HetznerActionResponse> {
+    self.delete(&format!("/servers/{id}")).await
+  }
+
+  #[allow(unused)]
+  pub async fn get_volume(
+    &self,
+    id: i64,
+  ) -> anyhow::Result<HetznerVolumeResponse> {
+    self.get(&format!("/volumes/{id}")).await
+  }
+
+  pub async fn create_volume(
+    &self,
+    body: &CreateVolumeBody,
+  ) -> anyhow::Result<CreateVolumeResponse> {
+    self.post("/volumes", body).await
+  }
+
+  #[allow(unused)]
+  pub async fn delete_volume(&self, id: i64) -> anyhow::Result<()> {
+    let res = self
+      .0
+      .delete(format!("{BASE_URL}/volumes/{id}"))
+      .send()
+      .await
+      .context("failed at request to delete volume")?;
+
+    let status = res.status();
+
+    if status == StatusCode::NO_CONTENT {
+      Ok(())
+    } else {
+      let text = res
+        .text()
+        .await
+        .context("failed to get response body as text")?;
+      Err(anyhow!("{status} | {text}"))
+    }
+  }
+
+  #[allow(unused)]
+  pub async fn list_datacenters(
+    &self,
+  ) -> anyhow::Result<HetznerDatacenterResponse> {
+    self.get("/datacenters").await
+  }
+
+  async fn get<Res: DeserializeOwned>(
+    &self,
+    path: &str,
+  ) -> anyhow::Result<Res> {
+    let req = self.0.get(format!("{BASE_URL}{path}"));
+    handle_req(req).await.with_context(|| {
+      format!("failed at GET request to Hetzner | path: {path}")
+    })
+  }
+
+  async fn post<Body: Serialize, Res: DeserializeOwned>(
+    &self,
+    path: &str,
+    body: &Body,
+  ) -> anyhow::Result<Res> {
+    let req = self.0.post(format!("{BASE_URL}{path}")).json(&body);
+    handle_req(req).await.with_context(|| {
+      format!("failed at POST request to Hetzner | path: {path}")
+    })
+  }
+
+  async fn delete<Res: DeserializeOwned>(
+    &self,
+    path: &str,
+  ) -> anyhow::Result<Res> {
+    let req = self.0.delete(format!("{BASE_URL}{path}"));
+    handle_req(req).await.with_context(|| {
+      format!("failed at DELETE request to Hetzner | path: {path}")
+    })
+  }
+}
+
+async fn handle_req<Res: DeserializeOwned>(
+  req: RequestBuilder,
+) -> anyhow::Result<Res> {
+  let res = req.send().await?;
+
+  let status = res.status();
+
+  if status.is_success() {
+    res.json().await.context("failed to parse response to json")
+  } else {
+    let text = res
+      .text()
+      .await
+      .context("failed to get response body as text")?;
+    if let Ok(json_error) =
+      serde_json::from_str::<serde_json::Value>(&text)
+    {
+      return Err(anyhow!("{status} | {json_error:?}"));
+    }
+    Err(anyhow!("{status} | {text}"))
+  }
+}

bin/core/src/cloud/hetzner/common.rs

@@ -0,0 +1,277 @@
+use std::collections::HashMap;
+
+use serde::{Deserialize, Serialize};
+
+#[derive(Debug, Clone, Deserialize)]
+pub struct HetznerServerResponse {
+  pub server: HetznerServer,
+}
+
+#[derive(Debug, Clone, Deserialize)]
+pub struct HetznerServer {
+  pub id: i64,
+  pub name: String,
+  pub primary_disk_size: f64,
+  pub image: Option<HetznerImage>,
+  pub private_net: Vec<HetznerPrivateNet>,
+  pub public_net: HetznerPublicNet,
+  pub server_type: HetznerServerTypeDetails,
+  pub status: HetznerServerStatus,
+  #[serde(default)]
+  pub volumes: Vec<i64>,
+}
+
+#[derive(Debug, Clone, Deserialize)]
+pub struct HetznerServerTypeDetails {
+  pub architecture: String,
+  pub cores: i64,
+  pub cpu_type: String,
+  pub description: String,
+  pub disk: f64,
+  pub id: i64,
+  pub memory: f64,
+  pub name: String,
+}
+
+#[derive(Debug, Clone, Deserialize)]
+pub struct HetznerPrivateNet {
+  pub alias_ips: Vec<String>,
+  pub ip: String,
+  pub mac_address: String,
+  pub network: i64,
+}
+
+#[derive(Debug, Clone, Deserialize)]
+pub struct HetznerPublicNet {
+  #[serde(default)]
+  pub firewalls: Vec<HetznerFirewall>,
+  pub floating_ips: Vec<i64>,
+  pub ipv4: Option<HetznerIpv4>,
+}
+
+#[derive(Debug, Clone, Deserialize)]
+pub struct HetznerFirewall {
+  pub id: i64,
+  pub status: String,
+}
+
+#[derive(Debug, Clone, Deserialize)]
+pub struct HetznerIpv4 {
+  pub id: Option<i64>,
+  pub blocked: bool,
+  pub dns_ptr: String,
+  pub ip: String,
+}
+
+#[derive(Debug, Clone, Deserialize)]
+pub struct HetznerImage {
+  pub id: i64,
+  pub description: String,
+  pub name: Option<String>,
+  pub os_flavor: String,
+  pub os_version: Option<String>,
+  pub rapid_deploy: Option<bool>,
+}
+
+#[derive(Debug, Clone, Deserialize)]
+pub struct HetznerActionResponse {
+  pub action: HetznerAction,
+}
+
+#[derive(Debug, Clone, Deserialize)]
+pub struct HetznerAction {
+  pub command: String,
+  pub error: Option<HetznerError>,
+  pub finished: Option<String>,
+  pub id: i64,
+  pub progress: i32,
+  pub resources: Vec<HetznerResource>,
+  pub started: String,
+  pub status: HetznerActionStatus,
+}
+
+#[derive(Debug, Clone, Deserialize)]
+pub struct HetznerError {
+  pub code: String,
+  pub message: String,
+}
+
+#[derive(Debug, Clone, Deserialize)]
+pub struct HetznerResource {
+  pub id: i64,
+  #[serde(rename = "type")]
+  pub ty: String,
+}
+
+#[derive(Debug, Clone, Deserialize)]
+pub struct HetznerVolumeResponse {
+  pub volume: HetznerVolume,
+}
+
+#[derive(Debug, Clone, Deserialize)]
+pub struct HetznerVolume {
+  /// Name of the Resource. Must be unique per Project.
+  pub name: String,
+  /// Point in time when the Resource was created (in ISO-8601 format).
+  pub created: String,
+  /// Filesystem of the Volume if formatted on creation, null if not formatted on creation
+  pub format: Option<HetznerVolumeFormat>,
+  /// ID of the Volume.
+  pub id: i64,
+  /// User-defined labels ( key/value pairs) for the Resource
+  pub labels: HashMap<String, String>,
+  /// Device path on the file system for the Volume
+  pub linux_device: String,
+  /// Protection configuration for the Resource.
+  pub protection: HetznerProtection,
+  /// ID of the Server the Volume is attached to, null if it is not attached at all
+  pub server: Option<i64>,
+  /// Size in GB of the  Volume
+  pub size: i64,
+  /// Current status of the Volume. Allowed: `creating`, `available`
+  pub status: HetznerVolumeStatus,
+}
+
+#[derive(Debug, Clone, Deserialize)]
+pub struct HetznerProtection {
+  /// Prevent the Resource from being deleted.
+  pub delete: bool,
+}
+
+#[derive(Debug, Clone, Deserialize)]
+pub struct HetznerDatacenterResponse {
+  pub datacenters: Vec<HetznerDatacenterDetails>,
+}
+
+#[derive(Debug, Clone, Deserialize)]
+pub struct HetznerDatacenterDetails {
+  pub id: i64,
+  pub name: String,
+  pub location: serde_json::Map<String, serde_json::Value>,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub enum HetznerLocation {
+  #[serde(rename = "nbg1")]
+  Nuremberg1,
+  #[serde(rename = "hel1")]
+  Helsinki1,
+  #[serde(rename = "fsn1")]
+  Falkenstein1,
+  #[serde(rename = "ash")]
+  Ashburn,
+  #[serde(rename = "hil")]
+  Hillsboro,
+}
+
+#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
+pub enum HetznerDatacenter {
+  #[serde(rename = "nbg1-dc3")]
+  Nuremberg1Dc3,
+  #[serde(rename = "hel1-dc2")]
+  Helsinki1Dc2,
+  #[serde(rename = "fsn1-dc14")]
+  Falkenstein1Dc14,
+  #[serde(rename = "ash-dc1")]
+  AshburnDc1,
+  #[serde(rename = "hil-dc1")]
+  HillsboroDc1,
+}
+
+impl From<HetznerDatacenter> for HetznerLocation {
+  fn from(value: HetznerDatacenter) -> Self {
+    match value {
+      HetznerDatacenter::Nuremberg1Dc3 => HetznerLocation::Nuremberg1,
+      HetznerDatacenter::Helsinki1Dc2 => HetznerLocation::Helsinki1,
+      HetznerDatacenter::Falkenstein1Dc14 => {
+        HetznerLocation::Falkenstein1
+      }
+      HetznerDatacenter::AshburnDc1 => HetznerLocation::Ashburn,
+      HetznerDatacenter::HillsboroDc1 => HetznerLocation::Hillsboro,
+    }
+  }
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(rename_all = "lowercase")]
+pub enum HetznerVolumeFormat {
+  Xfs,
+  Ext4,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(rename_all = "lowercase")]
+pub enum HetznerVolumeStatus {
+  Creating,
+  Available,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(rename_all = "lowercase")]
+pub enum HetznerServerStatus {
+  Running,
+  Initializing,
+  Starting,
+  Stopping,
+  Off,
+  Deleting,
+  Migrating,
+  Rebuilding,
+  Unknown,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(rename_all = "lowercase")]
+pub enum HetznerActionStatus {
+  Running,
+  Success,
+  Error,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(rename_all = "UPPERCASE")]
+#[allow(clippy::enum_variant_names)]
+pub enum HetznerServerType {
+  // Shared
+  #[serde(rename = "cx11")]
+  SharedIntel1Core2Ram20Disk,
+  #[serde(rename = "cpx11")]
+  SharedAmd2Core2Ram40Disk,
+  #[serde(rename = "cax11")]
+  SharedArm2Core4Ram40Disk,
+  #[serde(rename = "cx21")]
+  SharedIntel2Core4Ram40Disk,
+  #[serde(rename = "cpx21")]
+  SharedAmd3Core4Ram80Disk,
+  #[serde(rename = "cax21")]
+  SharedArm4Core8Ram80Disk,
+  #[serde(rename = "cx31")]
+  SharedIntel2Core8Ram80Disk,
+  #[serde(rename = "cpx31")]
+  SharedAmd4Core8Ram160Disk,
+  #[serde(rename = "cax31")]
+  SharedArm8Core16Ram160Disk,
+  #[serde(rename = "cx41")]
+  SharedIntel4Core16Ram160Disk,
+  #[serde(rename = "cpx41")]
+  SharedAmd8Core16Ram240Disk,
+  #[serde(rename = "cax41")]
+  SharedArm16Core32Ram320Disk,
+  #[serde(rename = "cx51")]
+  SharedIntel8Core32Ram240Disk,
+  #[serde(rename = "cpx51")]
+  SharedAmd16Core32Ram360Disk,
+  // Dedicated
+  #[serde(rename = "ccx13")]
+  DedicatedAmd2Core8Ram80Disk,
+  #[serde(rename = "ccx23")]
+  DedicatedAmd4Core16Ram160Disk,
+  #[serde(rename = "ccx33")]
+  DedicatedAmd8Core32Ram240Disk,
+  #[serde(rename = "ccx43")]
+  DedicatedAmd16Core64Ram360Disk,
+  #[serde(rename = "ccx53")]
+  DedicatedAmd32Core128Ram600Disk,
+  #[serde(rename = "ccx63")]
+  DedicatedAmd48Core192Ram960Disk,
+}

bin/core/src/cloud/hetzner/create_server.rs

@@ -0,0 +1,76 @@
+use std::collections::HashMap;
+
+use serde::{Deserialize, Serialize};
+
+use super::common::{
+  HetznerAction, HetznerDatacenter, HetznerLocation, HetznerServer,
+  HetznerServerType,
+};
+
+#[derive(Debug, Clone, Serialize)]
+pub struct CreateServerBody {
+  /// Name of the Server to create (must be unique per Project and a valid hostname as per RFC 1123)
+  pub name: String,
+  /// Auto-mount Volumes after attach
+  #[serde(skip_serializing_if = "Option::is_none")]
+  pub automount: Option<bool>,
+  /// ID or name of Datacenter to create Server in (must not be used together with location)
+  #[serde(skip_serializing_if = "Option::is_none")]
+  pub datacenter: Option<HetznerDatacenter>,
+  /// ID or name of Location to create Server in (must not be used together with datacenter)
+  #[serde(skip_serializing_if = "Option::is_none")]
+  pub location: Option<HetznerLocation>,
+  /// Firewalls which should be applied on the Server's public network interface at creation time
+  pub firewalls: Vec<Firewall>,
+  /// ID or name of the Image the Server is created from
+  pub image: String,
+  /// User-defined labels (key-value pairs) for the Resource
+  pub labels: HashMap<String, String>,
+  /// Network IDs which should be attached to the Server private network interface at the creation time
+  pub networks: Vec<i64>,
+  /// ID of the Placement Group the server should be in
+  #[serde(skip_serializing_if = "Option::is_none")]
+  pub placement_group: Option<i64>,
+  /// Public Network options
+  #[serde(skip_serializing_if = "Option::is_none")]
+  pub public_net: Option<PublicNet>,
+  /// ID or name of the Server type this Server should be created with
+  pub server_type: HetznerServerType,
+  /// SSH key IDs ( integer ) or names ( string ) which should be injected into the Server at creation time
+  pub ssh_keys: Vec<String>,
+  /// This automatically triggers a Power on a Server-Server Action after the creation is finished and is returned in the next_actions response object.
+  pub start_after_create: bool,
+  /// Cloud-Init user data to use during Server creation. This field is limited to 32KiB.
+  #[serde(skip_serializing_if = "Option::is_none")]
+  pub user_data: Option<String>,
+  /// Volume IDs which should be attached to the Server at the creation time. Volumes must be in the same Location.
+  pub volumes: Vec<i64>,
+}
+
+#[derive(Debug, Clone, Copy, Serialize)]
+pub struct Firewall {
+  /// ID of the Firewall
+  pub firewall: i64,
+}
+
+#[derive(Debug, Clone, Copy, Serialize)]
+pub struct PublicNet {
+  /// Attach an IPv4 on the public NIC. If false, no IPv4 address will be attached.
+  pub enable_ipv4: bool,
+  /// Attach an IPv6 on the public NIC. If false, no IPv6 address will be attached.
+  pub enable_ipv6: bool,
+  /// ID of the ipv4 Primary IP to use. If omitted and enable_ipv4 is true, a new ipv4 Primary IP will automatically be created.
+  #[serde(skip_serializing_if = "Option::is_none")]
+  pub ipv4: Option<i64>,
+  /// ID of the ipv6 Primary IP to use. If omitted and enable_ipv6 is true, a new ipv6 Primary IP will automatically be created.
+  #[serde(skip_serializing_if = "Option::is_none")]
+  pub ipv6: Option<i64>,
+}
+
+#[derive(Debug, Clone, Deserialize)]
+pub struct CreateServerResponse {
+  pub action: HetznerAction,
+  pub next_actions: Vec<HetznerAction>,
+  pub root_password: Option<String>,
+  pub server: HetznerServer,
+}

bin/core/src/cloud/hetzner/create_volume.rs

@@ -0,0 +1,36 @@
+use std::collections::HashMap;
+
+use serde::{Deserialize, Serialize};
+
+use super::common::{
+  HetznerAction, HetznerLocation, HetznerVolume, HetznerVolumeFormat,
+};
+
+#[derive(Debug, Clone, Serialize)]
+pub struct CreateVolumeBody {
+  /// Name of the volume
+  pub name: String,
+  /// Auto-mount Volume after attach. server must be provided.
+  #[serde(skip_serializing_if = "Option::is_none")]
+  pub automount: Option<bool>,
+  /// Format Volume after creation. One of: xfs, ext4
+  #[serde(skip_serializing_if = "Option::is_none")]
+  pub format: Option<HetznerVolumeFormat>,
+  /// User-defined labels (key-value pairs) for the Resource
+  pub labels: HashMap<String, String>,
+  /// Location to create the Volume in (can be omitted if Server is specified)
+  #[serde(skip_serializing_if = "Option::is_none")]
+  pub location: Option<HetznerLocation>,
+  /// Server to which to attach the Volume once it's created (Volume will be created in the same Location as the server)
+  #[serde(skip_serializing_if = "Option::is_none")]
+  pub server: Option<i64>,
+  /// Size of the Volume in GB
+  pub size: i64,
+}
+
+#[derive(Debug, Clone, Deserialize)]
+pub struct CreateVolumeResponse {
+  pub action: HetznerAction,
+  pub next_actions: Vec<HetznerAction>,
+  pub volume: HetznerVolume,
+}

bin/core/src/cloud/hetzner/mod.rs

@@ -0,0 +1,318 @@
+use std::{sync::OnceLock, time::Duration};
+
+use anyhow::{anyhow, Context};
+use monitor_client::entities::{
+  alert::{Alert, AlertData, AlertDataVariant},
+  monitor_timestamp,
+  server::stats::SeverityLevel,
+  server_template::hetzner::{
+    HetznerDatacenter, HetznerServerTemplateConfig,
+    HetznerServerType, HetznerVolumeFormat,
+  },
+  update::ResourceTarget,
+};
+
+use crate::{
+  cloud::hetzner::{
+    common::HetznerServerStatus, create_server::CreateServerBody,
+    create_volume::CreateVolumeBody,
+  },
+  config::core_config,
+  helpers::alert::send_alerts,
+};
+
+use self::{
+  client::HetznerClient,
+  common::{HetznerAction, HetznerActionResponse},
+};
+
+mod client;
+mod common;
+mod create_server;
+mod create_volume;
+
+fn hetzner() -> Option<&'static HetznerClient> {
+  static HETZNER_CLIENT: OnceLock<Option<HetznerClient>> =
+    OnceLock::new();
+  HETZNER_CLIENT
+    .get_or_init(|| {
+      let token = &core_config().hetzner.token;
+      (!token.is_empty()).then(|| HetznerClient::new(token))
+    })
+    .as_ref()
+}
+
+pub struct HetznerServerMinimal {
+  pub id: i64,
+  pub ip: String,
+}
+
+const POLL_RATE_SECS: u64 = 3;
+const MAX_POLL_TRIES: usize = 100;
+
+#[instrument]
+pub async fn launch_hetzner_server(
+  name: &str,
+  config: HetznerServerTemplateConfig,
+) -> anyhow::Result<HetznerServerMinimal> {
+  let hetzner =
+    *hetzner().as_ref().context("Hetzner token not configured")?;
+  let HetznerServerTemplateConfig {
+    image,
+    automount,
+    datacenter,
+    private_network_ids,
+    placement_group,
+    enable_public_ipv4,
+    enable_public_ipv6,
+    firewall_ids,
+    server_type,
+    ssh_keys,
+    user_data,
+    use_public_ip,
+    labels,
+    volumes,
+    port: _,
+  } = config;
+  let datacenter = hetzner_datacenter(datacenter);
+
+  // Create volumes and get their ids
+  let mut volume_ids = Vec::new();
+  for volume in volumes {
+    let body = CreateVolumeBody {
+      name: volume.name,
+      format: Some(hetzner_format(volume.format)),
+      location: Some(datacenter.into()),
+      labels: volume.labels,
+      size: volume.size_gb,
+      automount: None,
+      server: None,
+    };
+    let id = hetzner
+      .create_volume(&body)
+      .await
+      .context("failed to create hetzner volume")?
+      .volume
+      .id;
+    volume_ids.push(id);
+  }
+
+  let body = CreateServerBody {
+    name: name.to_string(),
+    // only actually add automount if volumes nonempty
+    automount: (!volume_ids.is_empty()).then_some(automount),
+    datacenter: Some(datacenter),
+    location: None,
+    firewalls: firewall_ids
+      .into_iter()
+      .map(|firewall| create_server::Firewall { firewall })
+      .collect(),
+    image,
+    labels,
+    networks: private_network_ids,
+    placement_group: (placement_group > 0).then_some(placement_group),
+    public_net: (enable_public_ipv4 || enable_public_ipv6).then_some(
+      create_server::PublicNet {
+        enable_ipv4: enable_public_ipv4,
+        enable_ipv6: enable_public_ipv6,
+        ipv4: None,
+        ipv6: None,
+      },
+    ),
+    server_type: hetzner_server_type(server_type),
+    ssh_keys,
+    start_after_create: true,
+    user_data: (!user_data.is_empty()).then_some(user_data),
+    volumes: volume_ids,
+  };
+
+  let server_id = hetzner
+    .create_server(&body)
+    .await
+    .context("failed to create hetnzer server")?
+    .server
+    .id;
+
+  for _ in 0..MAX_POLL_TRIES {
+    tokio::time::sleep(Duration::from_secs(POLL_RATE_SECS)).await;
+    let Ok(res) = hetzner.get_server(server_id).await else {
+      continue;
+    };
+    if matches!(res.server.status, HetznerServerStatus::Running) {
+      let ip = if use_public_ip {
+        res
+          .server
+          .public_net
+          .ipv4
+          .context("instance does not have public ipv4 attached")?
+          .ip
+      } else {
+        res
+          .server
+          .private_net
+          .first()
+          .context("no private networks attached")?
+          .ip
+          .to_string()
+      };
+      let server = HetznerServerMinimal { id: server_id, ip };
+      return Ok(server);
+    }
+  }
+
+  Err(anyhow!(
+    "failed to verify server running after polling status"
+  ))
+}
+
+#[allow(unused)]
+const MAX_TERMINATION_TRIES: usize = 5;
+#[allow(unused)]
+const TERMINATION_WAIT_SECS: u64 = 15;
+
+#[allow(unused)]
+pub async fn terminate_hetzner_server_with_retry(
+  id: i64,
+) -> anyhow::Result<()> {
+  let hetzner =
+    *hetzner().as_ref().context("Hetzner token not configured")?;
+
+  for i in 0..MAX_TERMINATION_TRIES {
+    let message = match hetzner.delete_server(id).await {
+      Ok(HetznerActionResponse {
+        action: HetznerAction { error: None, .. },
+      }) => return Ok(()),
+      Ok(HetznerActionResponse {
+        action: HetznerAction { error: Some(e), .. },
+      }) => (i == MAX_TERMINATION_TRIES - 1).then(|| {
+        format!(
+          "failed to terminate instance | code: {} | {}",
+          e.code, e.message
+        )
+      }),
+      Err(e) => {
+        (i == MAX_TERMINATION_TRIES - 1).then(|| format!("{e:#}"))
+      }
+    };
+    if let Some(message) = message {
+      error!("failed to terminate hetzner server {id} | {message}");
+      let alert = Alert {
+        id: Default::default(),
+        ts: monitor_timestamp(),
+        resolved: false,
+        level: SeverityLevel::Critical,
+        target: ResourceTarget::system(),
+        variant: AlertDataVariant::HetznerBuilderTerminationFailed,
+        data: AlertData::HetznerBuilderTerminationFailed {
+          server_id: id,
+          message: message.clone(),
+        },
+        resolved_ts: None,
+      };
+      send_alerts(&[alert]).await;
+      return Err(anyhow::Error::msg(message));
+    }
+    tokio::time::sleep(Duration::from_secs(TERMINATION_WAIT_SECS))
+      .await;
+  }
+
+  Ok(())
+}
+
+fn hetzner_format(
+  format: HetznerVolumeFormat,
+) -> common::HetznerVolumeFormat {
+  match format {
+    HetznerVolumeFormat::Xfs => common::HetznerVolumeFormat::Xfs,
+    HetznerVolumeFormat::Ext4 => common::HetznerVolumeFormat::Ext4,
+  }
+}
+
+fn hetzner_datacenter(
+  datacenter: HetznerDatacenter,
+) -> common::HetznerDatacenter {
+  match datacenter {
+    HetznerDatacenter::Nuremberg1Dc3 => {
+      common::HetznerDatacenter::Nuremberg1Dc3
+    }
+    HetznerDatacenter::Helsinki1Dc2 => {
+      common::HetznerDatacenter::Helsinki1Dc2
+    }
+    HetznerDatacenter::Falkenstein1Dc14 => {
+      common::HetznerDatacenter::Falkenstein1Dc14
+    }
+    HetznerDatacenter::AshburnDc1 => {
+      common::HetznerDatacenter::AshburnDc1
+    }
+    HetznerDatacenter::HillsboroDc1 => {
+      common::HetznerDatacenter::HillsboroDc1
+    }
+  }
+}
+
+fn hetzner_server_type(
+  server_type: HetznerServerType,
+) -> common::HetznerServerType {
+  match server_type {
+    HetznerServerType::SharedIntel1Core2Ram20Disk => {
+      common::HetznerServerType::SharedIntel1Core2Ram20Disk
+    }
+    HetznerServerType::SharedAmd2Core2Ram40Disk => {
+      common::HetznerServerType::SharedAmd2Core2Ram40Disk
+    }
+    HetznerServerType::SharedArm2Core4Ram40Disk => {
+      common::HetznerServerType::SharedArm2Core4Ram40Disk
+    }
+    HetznerServerType::SharedIntel2Core4Ram40Disk => {
+      common::HetznerServerType::SharedIntel2Core4Ram40Disk
+    }
+    HetznerServerType::SharedAmd3Core4Ram80Disk => {
+      common::HetznerServerType::SharedAmd3Core4Ram80Disk
+    }
+    HetznerServerType::SharedArm4Core8Ram80Disk => {
+      common::HetznerServerType::SharedArm4Core8Ram80Disk
+    }
+    HetznerServerType::SharedIntel2Core8Ram80Disk => {
+      common::HetznerServerType::SharedIntel2Core8Ram80Disk
+    }
+    HetznerServerType::SharedAmd4Core8Ram160Disk => {
+      common::HetznerServerType::SharedAmd4Core8Ram160Disk
+    }
+    HetznerServerType::SharedArm8Core16Ram160Disk => {
+      common::HetznerServerType::SharedArm8Core16Ram160Disk
+    }
+    HetznerServerType::SharedIntel4Core16Ram160Disk => {
+      common::HetznerServerType::SharedIntel4Core16Ram160Disk
+    }
+    HetznerServerType::SharedAmd8Core16Ram240Disk => {
+      common::HetznerServerType::SharedAmd8Core16Ram240Disk
+    }
+    HetznerServerType::SharedArm16Core32Ram320Disk => {
+      common::HetznerServerType::SharedArm16Core32Ram320Disk
+    }
+    HetznerServerType::SharedIntel8Core32Ram240Disk => {
+      common::HetznerServerType::SharedIntel8Core32Ram240Disk
+    }
+    HetznerServerType::SharedAmd16Core32Ram360Disk => {
+      common::HetznerServerType::SharedAmd16Core32Ram360Disk
+    }
+    HetznerServerType::DedicatedAmd2Core8Ram80Disk => {
+      common::HetznerServerType::DedicatedAmd2Core8Ram80Disk
+    }
+    HetznerServerType::DedicatedAmd4Core16Ram160Disk => {
+      common::HetznerServerType::DedicatedAmd4Core16Ram160Disk
+    }
+    HetznerServerType::DedicatedAmd8Core32Ram240Disk => {
+      common::HetznerServerType::DedicatedAmd8Core32Ram240Disk
+    }
+    HetznerServerType::DedicatedAmd16Core64Ram360Disk => {
+      common::HetznerServerType::DedicatedAmd16Core64Ram360Disk
+    }
+    HetznerServerType::DedicatedAmd32Core128Ram600Disk => {
+      common::HetznerServerType::DedicatedAmd32Core128Ram600Disk
+    }
+    HetznerServerType::DedicatedAmd48Core192Ram960Disk => {
+      common::HetznerServerType::DedicatedAmd48Core192Ram960Disk
+    }
+  }
+}

bin/core/src/cloud/mod.rs

@@ -1,4 +1,5 @@
 pub mod aws;
+pub mod hetzner;
 
 #[derive(Debug)]
 pub enum BuildCleanupData {

bin/core/src/config.rs

@@ -2,7 +2,13 @@ use std::sync::OnceLock;
 
 use anyhow::Context;
 use merge_config_files::parse_config_file;
-use monitor_client::entities::config::core::{CoreConfig, Env};
+use monitor_client::entities::{
+  config::core::{
+    AwsCredentials, CoreConfig, Env, HetznerCredentials, MongoConfig,
+    OauthCredentials,
+  },
+  logger::LogConfig,
+};
 use serde::Deserialize;
 
 pub fn frontend_path() -> &'static String {
@@ -34,91 +40,112 @@ pub fn core_config() -> &'static CoreConfig {
       .context("failed to parse core Env")
       .unwrap();
     let config_path = &env.monitor_config_path;
-    let mut config =
+    let config =
       parse_config_file::<CoreConfig>(config_path.as_str())
         .unwrap_or_else(|e| {
           panic!("failed at parsing config at {config_path} | {e:#}")
         });
+    // recreating CoreConfig here makes sure we apply all env overrides.
+    CoreConfig {
+      title: env.monitor_title.unwrap_or(config.title),
+      host: env.monitor_host.unwrap_or(config.host),
+      port: env.monitor_port.unwrap_or(config.port),
+      passkey: env.monitor_passkey.unwrap_or(config.passkey),
+      jwt_valid_for: env
+        .monitor_jwt_valid_for
+        .unwrap_or(config.jwt_valid_for),
+      monitoring_interval: env
+        .monitor_monitoring_interval
+        .unwrap_or(config.monitoring_interval),
+      keep_stats_for_days: env
+        .monitor_keep_stats_for_days
+        .unwrap_or(config.keep_stats_for_days),
+      keep_alerts_for_days: env
+        .monitor_keep_alerts_for_days
+        .unwrap_or(config.keep_alerts_for_days),
+      github_webhook_secret: env
+        .monitor_github_webhook_secret
+        .unwrap_or(config.github_webhook_secret),
+      github_webhook_base_url: env
+        .monitor_github_webhook_base_url
+        .or(config.github_webhook_base_url),
+      docker_organizations: env
+        .monitor_docker_organizations
+        .unwrap_or(config.docker_organizations),
+      transparent_mode: env
+        .monitor_transparent_mode
+        .unwrap_or(config.transparent_mode),
+      local_auth: env.monitor_local_auth.unwrap_or(config.local_auth),
+      google_oauth: OauthCredentials {
+        enabled: env
+          .monitor_google_oauth_enabled
+          .unwrap_or(config.google_oauth.enabled),
+        id: env
+          .monitor_google_oauth_id
+          .unwrap_or(config.google_oauth.id),
+        secret: env
+          .monitor_google_oauth_secret
+          .unwrap_or(config.google_oauth.secret),
+      },
+      github_oauth: OauthCredentials {
+        enabled: env
+          .monitor_github_oauth_enabled
+          .unwrap_or(config.github_oauth.enabled),
+        id: env
+          .monitor_github_oauth_id
+          .unwrap_or(config.github_oauth.id),
+        secret: env
+          .monitor_github_oauth_secret
+          .unwrap_or(config.github_oauth.secret),
+      },
+      aws: AwsCredentials {
+        access_key_id: env
+          .monitor_aws_access_key_id
+          .unwrap_or(config.aws.access_key_id),
+        secret_access_key: env
+          .monitor_aws_secret_access_key
+          .unwrap_or(config.aws.secret_access_key),
+      },
+      hetzner: HetznerCredentials {
+        token: env
+          .monitor_hetzner_token
+          .unwrap_or(config.hetzner.token),
+      },
+      mongo: MongoConfig {
+        uri: env.monitor_mongo_uri.or(config.mongo.uri),
+        address: env.monitor_mongo_address.or(config.mongo.address),
+        username: env
+          .monitor_mongo_username
+          .or(config.mongo.username),
+        password: env
+          .monitor_mongo_password
+          .or(config.mongo.password),
+        app_name: env
+          .monitor_mongo_app_name
+          .unwrap_or(config.mongo.app_name),
+        db_name: env
+          .monitor_mongo_db_name
+          .unwrap_or(config.mongo.db_name),
+      },
+      logging: LogConfig {
+        level: env
+          .monitor_logging_level
+          .unwrap_or(config.logging.level),
+        stdio: env
+          .monitor_logging_stdio
+          .unwrap_or(config.logging.stdio),
+        otlp_endpoint: env
+          .monitor_logging_otlp_endpoint
+          .or(config.logging.otlp_endpoint),
+        opentelemetry_service_name: env
+          .monitor_logging_opentelemetry_service_name
+          .unwrap_or(config.logging.opentelemetry_service_name),
+      },
 
-    // Overrides
-    config.title = env.monitor_title.unwrap_or(config.title);
-    config.host = env.monitor_host.unwrap_or(config.host);
-    config.port = env.monitor_port.unwrap_or(config.port);
-    config.passkey = env.monitor_passkey.unwrap_or(config.passkey);
-    config.jwt_valid_for =
-      env.monitor_jwt_valid_for.unwrap_or(config.jwt_valid_for);
-    config.monitoring_interval = env
-      .monitor_monitoring_interval
-      .unwrap_or(config.monitoring_interval);
-    config.keep_stats_for_days = env
-      .monitor_keep_stats_for_days
-      .unwrap_or(config.keep_stats_for_days);
-    config.keep_alerts_for_days = env
-      .monitor_keep_alerts_for_days
-      .unwrap_or(config.keep_alerts_for_days);
-    config.github_webhook_secret = env
-      .monitor_github_webhook_secret
-      .unwrap_or(config.github_webhook_secret);
-    config.github_webhook_base_url = env
-      .monitor_github_webhook_base_url
-      .or(config.github_webhook_base_url);
-    config.docker_organizations = env
-      .monitor_docker_organizations
-      .unwrap_or(config.docker_organizations);
-
-    config.logging.level =
-      env.monitor_logging_level.unwrap_or(config.logging.level);
-    config.logging.stdio =
-      env.monitor_logging_stdio.unwrap_or(config.logging.stdio);
-    config.logging.otlp_endpoint = env
-      .monitor_logging_otlp_endpoint
-      .or(config.logging.otlp_endpoint);
-    config.logging.opentelemetry_service_name = env
-      .monitor_logging_opentelemetry_service_name
-      .unwrap_or(config.logging.opentelemetry_service_name);
-
-    config.local_auth =
-      env.monitor_local_auth.unwrap_or(config.local_auth);
-
-    config.github_oauth.enabled = env
-      .monitor_github_oauth_enabled
-      .unwrap_or(config.github_oauth.enabled);
-    config.github_oauth.id = env
-      .monitor_github_oauth_id
-      .unwrap_or(config.github_oauth.id);
-    config.github_oauth.secret = env
-      .monitor_github_oauth_secret
-      .unwrap_or(config.github_oauth.secret);
-
-    config.google_oauth.enabled = env
-      .monitor_google_oauth_enabled
-      .unwrap_or(config.google_oauth.enabled);
-    config.google_oauth.id = env
-      .monitor_google_oauth_id
-      .unwrap_or(config.google_oauth.id);
-    config.google_oauth.secret = env
-      .monitor_google_oauth_secret
-      .unwrap_or(config.google_oauth.secret);
-
-    config.mongo.uri = env.monitor_mongo_uri.or(config.mongo.uri);
-    config.mongo.address =
-      env.monitor_mongo_address.or(config.mongo.address);
-    config.mongo.username =
-      env.monitor_mongo_username.or(config.mongo.username);
-    config.mongo.password =
-      env.monitor_mongo_password.or(config.mongo.password);
-    config.mongo.app_name =
-      env.monitor_mongo_app_name.unwrap_or(config.mongo.app_name);
-    config.mongo.db_name =
-      env.monitor_mongo_db_name.unwrap_or(config.mongo.db_name);
-
-    config.aws.access_key_id = env
-      .monitor_aws_access_key_id
-      .unwrap_or(config.aws.access_key_id);
-    config.aws.secret_access_key = env
-      .monitor_aws_secret_access_key
-      .unwrap_or(config.aws.secret_access_key);
-
-    config
+      // These can't be overridden on env
+      secrets: config.secrets,
+      github_accounts: config.github_accounts,
+      docker_accounts: config.docker_accounts
+    }
   })
 }

bin/core/src/db.rs

@@ -1,4 +1,4 @@
-use mongo_indexed::{create_index, create_unique_index, Indexed};
+use mongo_indexed::{create_index, create_unique_index};
 use monitor_client::entities::{
   alert::Alert,
   alerter::Alerter,
@@ -16,6 +16,7 @@ use monitor_client::entities::{
   update::Update,
   user::User,
   user_group::UserGroup,
+  variable::Variable,
 };
 use mungos::{
   init::MongoBuilder,
@@ -28,6 +29,7 @@ pub struct DbClient {
   pub permissions: Collection<Permission>,
   pub api_keys: Collection<ApiKey>,
   pub tags: Collection<Tag>,
+  pub variables: Collection<Variable>,
   pub updates: Collection<Update>,
   pub alerts: Collection<Alert>,
   pub stats: Collection<SystemStatsRecord>,
@@ -80,14 +82,16 @@ impl DbClient {
     let db = client.database(db_name);
 
     let client = DbClient {
-      users: User::collection(&db, true).await?,
-      user_groups: UserGroup::collection(&db, true).await?,
-      permissions: Permission::collection(&db, true).await?,
-      api_keys: ApiKey::collection(&db, true).await?,
-      tags: Tag::collection(&db, true).await?,
-      updates: Update::collection(&db, true).await?,
-      alerts: Alert::collection(&db, true).await?,
-      stats: SystemStatsRecord::collection(&db, true).await?,
+      users: mongo_indexed::collection(&db, true).await?,
+      user_groups: mongo_indexed::collection(&db, true).await?,
+      permissions: mongo_indexed::collection(&db, true).await?,
+      api_keys: mongo_indexed::collection(&db, true).await?,
+      tags: mongo_indexed::collection(&db, true).await?,
+      variables: mongo_indexed::collection(&db, true).await?,
+      updates: mongo_indexed::collection(&db, true).await?,
+      alerts: mongo_indexed::collection(&db, true).await?,
+      stats: mongo_indexed::collection(&db, true).await?,
+      // RESOURCES
       servers: resource_collection(&db, "Server").await?,
       deployments: resource_collection(&db, "Deployment").await?,
       builds: resource_collection(&db, "Build").await?,
@@ -97,6 +101,7 @@ impl DbClient {
       procedures: resource_collection(&db, "Procedure").await?,
       server_templates: resource_collection(&db, "ServerTemplate")
         .await?,
+      //
       db,
     };
     Ok(client)

bin/core/src/helpers/alert.rs

@@ -210,13 +210,33 @@ async fn send_slack_alert(
       ];
       (text, blocks.into())
     }
-    AlertData::AwsBuilderTerminationFailed { instance_id } => {
+    AlertData::AwsBuilderTerminationFailed {
+      instance_id,
+      message,
+    } => {
       let text = format!(
         "{level} | Failed to terminated AWS builder instance"
       );
       let blocks = vec![
         Block::header(text.clone()),
-        Block::section(format!("instance id: {instance_id}")),
+        Block::section(format!(
+          "instance id: **{instance_id}**\n{message}"
+        )),
+      ];
+      (text, blocks.into())
+    }
+    AlertData::HetznerBuilderTerminationFailed {
+      server_id,
+      message,
+    } => {
+      let text = format!(
+        "{level} | Failed to terminated Hetzner builder instance"
+      );
+      let blocks = vec![
+        Block::header(text.clone()),
+        Block::section(format!(
+          "server id: **{server_id}**\n{message}"
+        )),
       ];
       (text, blocks.into())
     }

bin/core/src/helpers/prune.rs

@@ -1,16 +1,24 @@
+use anyhow::Context;
 use async_timing_util::{
   unix_timestamp_ms, wait_until_timelength, Timelength, ONE_DAY_MS,
 };
-use mungos::mongodb::bson::doc;
+use futures::future::join_all;
+use mungos::{find::find_collect, mongodb::bson::doc};
+use periphery_client::api::build::PruneImages;
 
 use crate::{config::core_config, state::db_client};
 
+use super::periphery_client;
+
 pub fn spawn_prune_loop() {
   tokio::spawn(async move {
     loop {
       wait_until_timelength(Timelength::OneDay, 5000).await;
-      let (stats_res, alerts_res) =
-        tokio::join!(prune_stats(), prune_alerts());
+      let (images_res, stats_res, alerts_res) =
+        tokio::join!(prune_images(), prune_stats(), prune_alerts());
+      if let Err(e) = images_res {
+        error!("error in pruning images | {e:#}");
+      }
       if let Err(e) = stats_res {
         error!("error in pruning stats | {e:#}");
       }
@@ -21,6 +29,35 @@ pub fn spawn_prune_loop() {
   });
 }
 
+async fn prune_images() -> anyhow::Result<()> {
+  let futures = find_collect(&db_client().await.servers, None, None)
+    .await
+    .context("failed to get servers from db")?
+    .into_iter()
+    // This could be done in the mongo query, but rather have rust type system guarantee this.
+    .filter(|server| server.config.auto_prune)
+    .map(|server| async move {
+      (
+        async {
+          periphery_client(&server)?.request(PruneImages {}).await
+        }
+        .await,
+        server,
+      )
+    });
+
+  for (res, server) in join_all(futures).await {
+    if let Err(e) = res {
+      error!(
+        "failed to prune images on server {} ({}) | {e:#}",
+        server.name, server.id
+      )
+    }
+  }
+
+  Ok(())
+}
+
 async fn prune_stats() -> anyhow::Result<()> {
   if core_config().keep_stats_for_days == 0 {
     return Ok(());

bin/core/src/helpers/query.rs

@@ -1,4 +1,7 @@
-use std::{collections::HashSet, str::FromStr};
+use std::{
+  collections::{HashMap, HashSet},
+  str::FromStr,
+};
 
 use anyhow::{anyhow, Context};
 use monitor_client::entities::{
@@ -8,6 +11,7 @@ use monitor_client::entities::{
   tag::Tag,
   update::ResourceTargetVariant,
   user::{admin_service_user, User},
+  variable::Variable,
 };
 use mungos::{
   by_id::find_one_by_id,
@@ -15,7 +19,7 @@ use mungos::{
   mongodb::bson::{doc, oid::ObjectId, Document},
 };
 
-use crate::{resource, state::db_client};
+use crate::{config::core_config, resource, state::db_client};
 
 #[instrument(level = "debug")]
 pub async fn get_user(user_id: &str) -> anyhow::Result<User> {
@@ -146,6 +150,11 @@ pub async fn get_user_permission_on_resource(
   resource_variant: ResourceTargetVariant,
   resource_id: &str,
 ) -> anyhow::Result<PermissionLevel> {
+  let lowest_permission = if core_config().transparent_mode {
+    PermissionLevel::Read
+  } else {
+    PermissionLevel::None
+  };
   let permission = find_collect(
     &db_client().await.permissions,
     doc! {
@@ -159,7 +168,7 @@ pub async fn get_user_permission_on_resource(
   .context("failed to query db for permissions")?
   .into_iter()
   // get the max permission user has between personal / any user groups
-  .fold(PermissionLevel::None, |level, permission| {
+  .fold(lowest_permission, |level, permission| {
     if permission.level > level {
       permission.level
     } else {
@@ -198,3 +207,27 @@ pub fn id_or_name_filter(id_or_name: &str) -> Document {
     Err(_) => doc! { "name": id_or_name },
   }
 }
+
+pub async fn get_global_variables(
+) -> anyhow::Result<HashMap<String, String>> {
+  Ok(
+    find_collect(&db_client().await.variables, None, None)
+      .await
+      .context("failed to get all variables from db")?
+      .into_iter()
+      .map(|variable| (variable.name, variable.value))
+      .collect(),
+  )
+}
+
+pub async fn get_variable(name: &str) -> anyhow::Result<Variable> {
+  db_client()
+    .await
+    .variables
+    .find_one(doc! { "name": &name }, None)
+    .await
+    .context("failed at call to db")?
+    .with_context(|| {
+      format!("no variable found with given name: {name}")
+    })
+}

bin/core/src/helpers/update.rs

@@ -83,6 +83,7 @@ async fn update_list_item(
     target: update.target,
     status: update.status,
     version: update.version,
+    other_data: update.other_data,
     username,
   };
   Ok(update)

bin/core/src/main.rs

@@ -37,6 +37,7 @@ async fn app() -> anyhow::Result<()> {
   helpers::prune::spawn_prune_loop();
   resource::spawn_build_state_refresh_loop();
   resource::spawn_repo_state_refresh_loop();
+  resource::spawn_procedure_state_refresh_loop();
 
   // Setup static frontend services
   let frontend_path = frontend_path();

bin/core/src/monitor/helpers.rs

@@ -125,6 +125,7 @@ fn get_server_health(
     mount,
     used_gb,
     total_gb,
+    ..
   } in disks
   {
     let perc = 100.0 * used_gb / total_gb;

bin/core/src/resource/alerter.rs

@@ -35,7 +35,7 @@ impl super::MonitorResource for Alerter {
 
   async fn to_list_item(
     alerter: Resource<Self::Config, Self::Info>,
-  ) -> anyhow::Result<Self::ListItem> {
+  ) -> Self::ListItem {
     let (alerter_type, enabled) = match alerter.config {
       AlerterConfig::Custom(config) => {
         (AlerterConfigVariant::Custom.to_string(), config.enabled)
@@ -44,7 +44,7 @@ impl super::MonitorResource for Alerter {
         (AlerterConfigVariant::Slack.to_string(), config.enabled)
       }
     };
-    Ok(AlerterListItem {
+    AlerterListItem {
       name: alerter.name,
       id: alerter.id,
       tags: alerter.tags,
@@ -54,7 +54,7 @@ impl super::MonitorResource for Alerter {
         is_default: alerter.info.is_default,
         enabled,
       },
-    })
+    }
   }
 
   async fn busy(_id: &String) -> anyhow::Result<bool> {

bin/core/src/resource/build.rs

@@ -43,9 +43,9 @@ impl super::MonitorResource for Build {
 
   async fn to_list_item(
     build: Resource<Self::Config, Self::Info>,
-  ) -> anyhow::Result<Self::ListItem> {
+  ) -> Self::ListItem {
     let state = get_build_state(&build.id).await;
-    Ok(BuildListItem {
+    BuildListItem {
       name: build.name,
       id: build.id,
       tags: build.tags,
@@ -57,7 +57,7 @@ impl super::MonitorResource for Build {
         branch: build.config.branch,
         state,
       },
-    })
+    }
   }
 
   async fn busy(id: &String) -> anyhow::Result<bool> {

bin/core/src/resource/builder.rs

@@ -38,7 +38,7 @@ impl super::MonitorResource for Builder {
 
   async fn to_list_item(
     builder: Resource<Self::Config, Self::Info>,
-  ) -> anyhow::Result<Self::ListItem> {
+  ) -> Self::ListItem {
     let (builder_type, instance_type) = match builder.config {
       BuilderConfig::Server(config) => (
         BuilderConfigVariant::Server.to_string(),
@@ -49,8 +49,7 @@ impl super::MonitorResource for Builder {
         Some(config.instance_type),
       ),
     };
-
-    Ok(BuilderListItem {
+    BuilderListItem {
       name: builder.name,
       id: builder.id,
       tags: builder.tags,
@@ -59,7 +58,7 @@ impl super::MonitorResource for Builder {
         builder_type,
         instance_type,
       },
-    })
+    }
   }
 
   async fn busy(_id: &String) -> anyhow::Result<bool> {

bin/core/src/resource/deployment.rs

@@ -46,21 +46,29 @@ impl super::MonitorResource for Deployment {
 
   async fn to_list_item(
     deployment: Resource<Self::Config, Self::Info>,
-  ) -> anyhow::Result<Self::ListItem> {
+  ) -> Self::ListItem {
     let status = deployment_status_cache().get(&deployment.id).await;
-    let (image, build_id) = match deployment.config.image {
+    let (build_image, build_id) = match deployment.config.image {
       DeploymentImage::Build { build_id, version } => {
-        let build = super::get::<Build>(&build_id).await?;
+        let (build_name, build_id, build_version) =
+          super::get::<Build>(&build_id)
+            .await
+            .map(|b| (b.name, b.id, b.config.version))
+            .unwrap_or((
+              String::from("unknown"),
+              String::new(),
+              Default::default(),
+            ));
         let version = if version.is_none() {
-          build.config.version.to_string()
+          build_version.to_string()
         } else {
           version.to_string()
         };
-        (format!("{}:{version}", build.name), Some(build_id))
+        (format!("{build_name}:{version}"), Some(build_id))
       }
       DeploymentImage::Image { image } => (image, None),
     };
-    Ok(DeploymentListItem {
+    DeploymentListItem {
       name: deployment.name,
       id: deployment.id,
       tags: deployment.tags,
@@ -73,11 +81,16 @@ impl super::MonitorResource for Deployment {
         status: status.as_ref().and_then(|s| {
           s.curr.container.as_ref().and_then(|c| c.status.to_owned())
         }),
-        image,
+        image: status
+          .as_ref()
+          .and_then(|s| {
+            s.curr.container.as_ref().map(|c| c.image.clone())
+          })
+          .unwrap_or(build_image),
         server_id: deployment.config.server_id,
         build_id,
       },
-    })
+    }
   }
 
   async fn busy(id: &String) -> anyhow::Result<bool> {
@@ -149,8 +162,7 @@ impl super::MonitorResource for Deployment {
       .context("failed to get container state")?;
     if !matches!(
       state,
-      DeploymentState::NotDeployed
-        | DeploymentState::Unknown
+      DeploymentState::NotDeployed | DeploymentState::Unknown
     ) {
       // container needs to be destroyed
       let server =

bin/core/src/resource/mod.rs

@@ -28,6 +28,7 @@ use serde::{de::DeserializeOwned, Serialize};
 use serror::serialize_error_pretty;
 
 use crate::{
+  config::core_config,
   helpers::{
     create_permission,
     query::{
@@ -51,6 +52,9 @@ mod server_template;
 pub use build::{
   refresh_build_state_cache, spawn_build_state_refresh_loop,
 };
+pub use procedure::{
+  refresh_procedure_state_cache, spawn_procedure_state_refresh_loop,
+};
 pub use repo::{
   refresh_repo_state_cache, spawn_repo_state_refresh_loop,
 };
@@ -86,7 +90,7 @@ pub trait MonitorResource {
 
   async fn to_list_item(
     resource: Resource<Self::Config, Self::Info>,
-  ) -> anyhow::Result<Self::ListItem>;
+  ) -> Self::ListItem;
 
   #[allow(clippy::ptr_arg)]
   async fn busy(id: &String) -> anyhow::Result<bool>;
@@ -188,7 +192,10 @@ pub async fn get_check_permissions<T: MonitorResource>(
   permission_level: PermissionLevel,
 ) -> anyhow::Result<Resource<T::Config, T::Info>> {
   let resource = get::<T>(id_or_name).await?;
-  if user.admin {
+  if user.admin
+    || (permission_level <= PermissionLevel::Read
+      && core_config().transparent_mode)
+  {
     return Ok(resource);
   }
   let permissions = get_user_permission_on_resource(
@@ -229,17 +236,7 @@ pub async fn list_for_user_using_document<T: MonitorResource>(
     .await?
     .into_iter()
     .map(|resource| T::to_list_item(resource));
-
-  let list = join_all(list)
-    .await
-    .into_iter()
-    .collect::<anyhow::Result<Vec<_>>>()
-    .context(format!(
-      "failed to convert {} list item",
-      T::resource_type()
-    ))?;
-
-  Ok(list)
+  Ok(join_all(list).await)
 }
 
 pub async fn list_full_for_user<T: MonitorResource>(
@@ -256,7 +253,7 @@ async fn list_full_for_user_using_document<T: MonitorResource>(
   mut filters: Document,
   user: &User,
 ) -> anyhow::Result<Vec<Resource<T::Config, T::Info>>> {
-  if !user.admin {
+  if !user.admin && !core_config().transparent_mode {
     let ids =
       get_resource_ids_for_non_admin(&user.id, T::resource_type())
         .await?

bin/core/src/resource/procedure.rs

@@ -1,4 +1,6 @@
-use anyhow::anyhow;
+use std::time::Duration;
+
+use anyhow::{anyhow, Context};
 use monitor_client::{
   api::execute::Execution,
   entities::{
@@ -8,7 +10,7 @@ use monitor_client::{
     procedure::{
       PartialProcedureConfig, Procedure, ProcedureConfig,
       ProcedureConfigDiff, ProcedureListItem, ProcedureListItemInfo,
-      ProcedureQuerySpecifics,
+      ProcedureQuerySpecifics, ProcedureState,
     },
     repo::Repo,
     resource::Resource,
@@ -18,9 +20,12 @@ use monitor_client::{
     Operation,
   },
 };
-use mungos::mongodb::Collection;
+use mungos::{
+  find::find_collect,
+  mongodb::{bson::doc, options::FindOneOptions, Collection},
+};
 
-use crate::state::{action_states, db_client};
+use crate::state::{action_states, db_client, procedure_state_cache};
 
 impl super::MonitorResource for Procedure {
   type Config = ProcedureConfig;
@@ -41,16 +46,18 @@ impl super::MonitorResource for Procedure {
 
   async fn to_list_item(
     procedure: Resource<Self::Config, Self::Info>,
-  ) -> anyhow::Result<Self::ListItem> {
-    Ok(ProcedureListItem {
+  ) -> Self::ListItem {
+    let state = get_procedure_state(&procedure.id).await;
+    ProcedureListItem {
       name: procedure.name,
       id: procedure.id,
       tags: procedure.tags,
       resource_type: ResourceTargetVariant::Procedure,
       info: ProcedureListItemInfo {
         procedure_type: procedure.config.procedure_type,
+        state,
       },
-    })
+    }
   }
 
   async fn busy(id: &String) -> anyhow::Result<bool> {
@@ -260,3 +267,80 @@ async fn validate_config(
   }
   Ok(())
 }
+
+pub fn spawn_procedure_state_refresh_loop() {
+  tokio::spawn(async move {
+    loop {
+      refresh_procedure_state_cache().await;
+      tokio::time::sleep(Duration::from_secs(60)).await;
+    }
+  });
+}
+
+pub async fn refresh_procedure_state_cache() {
+  let _ = async {
+    let procedures =
+      find_collect(&db_client().await.procedures, None, None)
+        .await
+        .context("failed to get procedures from db")?;
+    let cache = procedure_state_cache();
+    for procedure in procedures {
+      let state = get_procedure_state_from_db(&procedure.id).await;
+      cache.insert(procedure.id, state).await;
+    }
+    anyhow::Ok(())
+  }
+  .await
+  .inspect_err(|e| {
+    error!("failed to refresh build state cache | {e:#}")
+  });
+}
+
+async fn get_procedure_state(id: &String) -> ProcedureState {
+  if action_states()
+    .procedure
+    .get(id)
+    .await
+    .map(|s| s.get().map(|s| s.running))
+    .transpose()
+    .ok()
+    .flatten()
+    .unwrap_or_default()
+  {
+    return ProcedureState::Running;
+  }
+  procedure_state_cache().get(id).await.unwrap_or_default()
+}
+
+async fn get_procedure_state_from_db(id: &str) -> ProcedureState {
+  async {
+    let state = db_client()
+      .await
+      .updates
+      .find_one(
+        doc! {
+          "target.type": "Procedure",
+          "target.id": id,
+          "operation": "RunProcedure"
+        },
+        FindOneOptions::builder()
+          .sort(doc! { "start_ts": -1 })
+          .build(),
+      )
+      .await?
+      .map(|u| {
+        if u.success {
+          ProcedureState::Ok
+        } else {
+          ProcedureState::Failed
+        }
+      })
+      .unwrap_or(ProcedureState::Ok);
+    anyhow::Ok(state)
+  }
+  .await
+  .inspect_err(|e| {
+    warn!("failed to get procedure state for {id} | {e:#}")
+  })
+  .unwrap_or(ProcedureState::Unknown)
+}

bin/core/src/resource/repo.rs

@@ -48,16 +48,17 @@ impl super::MonitorResource for Repo {
 
   async fn to_list_item(
     repo: Resource<Self::Config, Self::Info>,
-  ) -> anyhow::Result<Self::ListItem> {
+  ) -> Self::ListItem {
     let state = get_repo_state(&repo.id).await;
     let status =
       repo_status_cache().get(&repo.id).await.unwrap_or_default();
-    Ok(RepoListItem {
+    RepoListItem {
       name: repo.name,
       id: repo.id,
       tags: repo.tags,
       resource_type: ResourceTargetVariant::Repo,
       info: RepoListItemInfo {
+        server_id: repo.config.server_id,
         last_pulled_at: repo.info.last_pulled_at,
         repo: repo.config.repo,
         branch: repo.config.branch,
@@ -65,7 +66,7 @@ impl super::MonitorResource for Repo {
         latest_hash: status.latest_hash.clone(),
         latest_message: status.latest_message.clone(),
       },
-    })
+    }
   }
 
   async fn busy(id: &String) -> anyhow::Result<bool> {

bin/core/src/resource/server.rs

@@ -36,9 +36,9 @@ impl super::MonitorResource for Server {
 
   async fn to_list_item(
     server: Resource<Self::Config, Self::Info>,
-  ) -> anyhow::Result<Self::ListItem> {
+  ) -> Self::ListItem {
     let status = server_status_cache().get(&server.id).await;
-    Ok(ServerListItem {
+    ServerListItem {
       name: server.name,
       id: server.id,
       tags: server.tags,
@@ -53,7 +53,7 @@ impl super::MonitorResource for Server {
         send_mem_alerts: server.config.send_mem_alerts,
         send_disk_alerts: server.config.send_disk_alerts,
       },
-    })
+    }
   }
 
   async fn busy(id: &String) -> anyhow::Result<bool> {

bin/core/src/resource/server_template.rs

@@ -36,15 +36,19 @@ impl super::MonitorResource for ServerTemplate {
 
   async fn to_list_item(
     server_template: Resource<Self::Config, Self::Info>,
-  ) -> anyhow::Result<Self::ListItem> {
+  ) -> Self::ListItem {
     let (template_type, instance_type) = match server_template.config
     {
       ServerTemplateConfig::Aws(config) => (
         ServerTemplateConfigVariant::Aws.to_string(),
         Some(config.instance_type),
       ),
+      ServerTemplateConfig::Hetzner(config) => (
+        ServerTemplateConfigVariant::Hetzner.to_string(),
+        Some(config.server_type.as_ref().to_string()),
+      ),
     };
-    Ok(ServerTemplateListItem {
+    ServerTemplateListItem {
       name: server_template.name,
       id: server_template.id,
       tags: server_template.tags,
@@ -53,7 +57,7 @@ impl super::MonitorResource for ServerTemplate {
         provider: template_type.to_string(),
         instance_type,
       },
-    })
+    }
   }
 
   async fn busy(_id: &String) -> anyhow::Result<bool> {

bin/core/src/state.rs

@@ -1,7 +1,8 @@
 use std::sync::{Arc, OnceLock};
 
 use monitor_client::entities::{
-  build::BuildState, deployment::DeploymentState, repo::RepoState,
+  build::BuildState, deployment::DeploymentState,
+  procedure::ProcedureState, repo::RepoState,
 };
 use tokio::sync::OnceCell;
 
@@ -80,3 +81,11 @@ pub fn repo_state_cache() -> &'static RepoStateCache {
   static REPO_STATE_CACHE: OnceLock<RepoStateCache> = OnceLock::new();
   REPO_STATE_CACHE.get_or_init(Default::default)
 }
+
+pub type ProcedureStateCache = Cache<String, ProcedureState>;
+
+pub fn procedure_state_cache() -> &'static ProcedureStateCache {
+  static PROCEDURE_STATE_CACHE: OnceLock<ProcedureStateCache> =
+    OnceLock::new();
+  PROCEDURE_STATE_CACHE.get_or_init(Default::default)
+}

bin/migrator/Cargo.toml

@@ -4,17 +4,17 @@ version.workspace = true
 edition.workspace = true
 authors.workspace = true
 license.workspace = true
+homepage.workspace = true
+repository.workspace = true
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
 monitor_client.workspace = true
-# db_client.workspace = true
 logger.workspace = true
 #
 termination_signal.workspace = true
 mungos.workspace = true
-mongo_indexed.workspace = true
 #
 tokio.workspace = true
 anyhow.workspace = true

bin/migrator/README.md

@@ -0,0 +1,5 @@
+# Migrator
+
+Upgrade data between periphery versions.
+
+Supports v0.x -> v1.x migration.

bin/migrator/src/legacy/v0/action.rs

@@ -1,53 +0,0 @@
-use mungos::mongodb::bson::serde_helpers::hex_string_as_object_id;
-use serde::{Deserialize, Serialize};
-
-use super::{Command, PermissionsMap};
-
-#[derive(Serialize, Deserialize, Debug, Clone, Default)]
-pub struct Action {
-  #[serde(
-    default,
-    rename = "_id",
-    skip_serializing_if = "String::is_empty",
-    with = "hex_string_as_object_id"
-  )]
-  pub id: String,
-
-  pub name: String,
-
-  #[serde(default)]
-  pub description: String,
-
-  pub path: String,
-
-  pub command: String,
-
-  // run action on all servers in this array
-  #[serde(default)]
-  pub server_ids: Vec<String>,
-
-  // run action on all servers in these groups
-  #[serde(default)]
-  pub group_ids: Vec<String>,
-
-  // run action on all servers
-  #[serde(default)]
-  pub run_on_all: bool,
-
-  #[serde(default)]
-  pub permissions: PermissionsMap,
-
-  #[serde(default, skip_serializing_if = "String::is_empty")]
-  pub created_at: String,
-  #[serde(default)]
-  pub updated_at: String,
-}
-
-impl From<Action> for Command {
-  fn from(value: Action) -> Command {
-    Command {
-      path: value.path,
-      command: value.command,
-    }
-  }
-}

bin/migrator/src/legacy/v0/alert.rs

@@ -1,13 +0,0 @@
-use mungos::mongodb::bson::serde_helpers::hex_string_as_object_id;
-use serde::{Deserialize, Serialize};
-
-#[derive(Serialize, Deserialize, Debug, Clone, Default)]
-pub struct Alert {
-  #[serde(
-    default,
-    rename = "_id",
-    skip_serializing_if = "String::is_empty",
-    with = "hex_string_as_object_id"
-  )]
-  pub id: String,
-}

bin/migrator/src/legacy/v0/group.rs

@@ -1,38 +0,0 @@
-use mungos::mongodb::bson::serde_helpers::hex_string_as_object_id;
-use serde::{Deserialize, Serialize};
-
-use super::PermissionsMap;
-
-#[derive(Serialize, Deserialize, Debug, Clone, Default)]
-pub struct Group {
-  #[serde(
-    default,
-    rename = "_id",
-    skip_serializing_if = "String::is_empty",
-    with = "hex_string_as_object_id"
-  )]
-  pub id: String,
-
-  pub name: String,
-
-  #[serde(default)]
-  pub description: String,
-
-  #[serde(default)]
-  pub permissions: PermissionsMap,
-
-  pub builds: Vec<String>,
-
-  pub deployments: Vec<String>,
-
-  pub servers: Vec<String>,
-
-  pub procedures: Vec<String>,
-
-  pub groups: Vec<String>,
-
-  #[serde(default, skip_serializing_if = "String::is_empty")]
-  pub created_at: String,
-  #[serde(default)]
-  pub updated_at: String,
-}

bin/migrator/src/legacy/v0/mod.rs

@@ -1,35 +1,53 @@
 use std::collections::HashMap;
 
-use anyhow::{anyhow, Context};
-use chrono::{DateTime, LocalResult, SecondsFormat, TimeZone, Utc};
+use anyhow::Context;
+use chrono::DateTime;
+use mungos::{init::MongoBuilder, mongodb::Collection};
 use serde::{Deserialize, Serialize};
 
-pub mod traits;
-
-mod action;
-mod alert;
 mod build;
 mod config;
 mod deployment;
-mod group;
-mod periphery_command;
-mod procedure;
 mod server;
 mod update;
 mod user;
 
-pub use action::*;
-pub use alert::*;
 pub use build::*;
 pub use config::*;
 pub use deployment::*;
-pub use group::*;
-pub use periphery_command::*;
-pub use procedure::*;
 pub use server::*;
 pub use update::*;
 pub use user::*;
 
+pub struct DbClient {
+  pub users: Collection<User>,
+  pub servers: Collection<Server>,
+  pub deployments: Collection<Deployment>,
+  pub builds: Collection<Build>,
+  pub updates: Collection<Update>,
+}
+
+impl DbClient {
+  pub async fn new(
+    legacy_uri: &str,
+    legacy_db_name: &str,
+  ) -> DbClient {
+    let client = MongoBuilder::default()
+      .uri(legacy_uri)
+      .build()
+      .await
+      .expect("failed to init legacy mongo client");
+    let db = client.database(legacy_db_name);
+    DbClient {
+      users: db.collection("users"),
+      servers: db.collection("servers"),
+      deployments: db.collection("deployments"),
+      builds: db.collection("builds"),
+      updates: db.collection("updates"),
+    }
+  }
+}
+
 pub type PermissionsMap = HashMap<String, PermissionLevel>;
 
 #[derive(Serialize, Deserialize, Clone, Debug)]
@@ -322,15 +340,3 @@ pub fn unix_from_monitor_ts(ts: &str) -> anyhow::Result<i64> {
       .timestamp_millis(),
   )
 }
-
-pub fn monitor_ts_from_unix(ts: i64) -> anyhow::Result<String> {
-  match Utc.timestamp_millis_opt(ts) {
-    LocalResult::Single(dt) => {
-      Ok(dt.to_rfc3339_opts(SecondsFormat::Millis, false))
-    }
-    LocalResult::None => {
-      Err(anyhow!("out of bounds timestamp passed"))
-    }
-    _ => unreachable!(),
-  }
-}

bin/migrator/src/legacy/v0/periphery_command.rs

@@ -1,38 +0,0 @@
-use mungos::mongodb::bson::serde_helpers::hex_string_as_object_id;
-use serde::{Deserialize, Serialize};
-
-use super::{Command, PermissionsMap};
-
-#[derive(Serialize, Deserialize, Debug, Clone, Default)]
-pub struct PeripheryCommand {
-  #[serde(
-    default,
-    rename = "_id",
-    skip_serializing_if = "String::is_empty",
-    with = "hex_string_as_object_id"
-  )]
-  pub id: String,
-
-  pub name: String, // must be formatted to be compat with docker
-
-  #[serde(default)]
-  pub description: String,
-
-  pub server_id: String,
-
-  #[serde(default)]
-  pub permissions: PermissionsMap,
-
-  #[serde(default)]
-  pub command: Command,
-
-  #[serde(default, skip_serializing_if = "String::is_empty")]
-  pub created_at: String,
-  #[serde(default)]
-  pub updated_at: String,
-}
-
-#[derive(Serialize, Deserialize, Debug, Clone, Default)]
-pub struct CommandActionState {
-  pub running: bool,
-}

bin/migrator/src/legacy/v0/procedure.rs

@@ -1,79 +0,0 @@
-use mungos::mongodb::bson::serde_helpers::hex_string_as_object_id;
-use serde::{Deserialize, Serialize};
-
-use super::PermissionsMap;
-
-#[derive(Serialize, Deserialize, Debug, Clone, Default)]
-pub struct Procedure {
-  #[serde(
-    default,
-    rename = "_id",
-    skip_serializing_if = "String::is_empty",
-    with = "hex_string_as_object_id"
-  )]
-  pub id: String,
-
-  pub name: String,
-
-  #[serde(default)]
-  pub description: String,
-
-  #[serde(default)]
-  pub stages: Vec<ProcedureStage>,
-
-  #[serde(default)]
-  pub webhook_branches: Vec<String>,
-
-  #[serde(default)]
-  pub permissions: PermissionsMap,
-
-  #[serde(default, skip_serializing_if = "String::is_empty")]
-  pub created_at: String,
-  #[serde(default)]
-  pub updated_at: String,
-}
-
-#[derive(
-  Serialize, Deserialize, Debug, Clone, Default, PartialEq,
-)]
-pub struct ProcedureStage {
-  pub operation: ProcedureOperation,
-  pub target_id: String,
-}
-
-#[derive(
-  Serialize,
-  Deserialize,
-  Debug,
-  PartialEq,
-  Hash,
-  Eq,
-  Clone,
-  Copy,
-  Default,
-)]
-#[serde(rename_all = "snake_case")]
-pub enum ProcedureOperation {
-  // do nothing
-  #[default]
-  None,
-
-  // server
-  PruneImagesServer,
-  PruneContainersServer,
-  PruneNetworksServer,
-
-  // build
-  BuildBuild,
-
-  // deployment
-  DeployContainer,
-  StopContainer,
-  StartContainer,
-  RemoveContainer,
-  PullDeployment,
-  RecloneDeployment,
-
-  // procedure
-  RunProcedure,
-}

bin/migrator/src/legacy/v0/traits.rs

@@ -1,112 +0,0 @@
-use super::{
-  Build, BuildActionState, CloneArgs, CommandActionState, Deployment,
-  DeploymentActionState, Group, PeripheryCommand, PermissionLevel,
-  PermissionsMap, Procedure, Server, ServerActionState,
-};
-
-pub trait Permissioned {
-  fn permissions_map(&self) -> &PermissionsMap;
-
-  fn get_user_permissions(&self, user_id: &str) -> PermissionLevel {
-    *self.permissions_map().get(user_id).unwrap_or_default()
-  }
-}
-
-impl Permissioned for Deployment {
-  fn permissions_map(&self) -> &PermissionsMap {
-    &self.permissions
-  }
-}
-
-impl Permissioned for Build {
-  fn permissions_map(&self) -> &PermissionsMap {
-    &self.permissions
-  }
-}
-
-impl Permissioned for Server {
-  fn permissions_map(&self) -> &PermissionsMap {
-    &self.permissions
-  }
-}
-
-impl Permissioned for Procedure {
-  fn permissions_map(&self) -> &PermissionsMap {
-    &self.permissions
-  }
-}
-
-impl Permissioned for Group {
-  fn permissions_map(&self) -> &PermissionsMap {
-    &self.permissions
-  }
-}
-
-impl Permissioned for PeripheryCommand {
-  fn permissions_map(&self) -> &PermissionsMap {
-    &self.permissions
-  }
-}
-
-pub trait Busy {
-  fn busy(&self) -> bool;
-}
-
-impl Busy for ServerActionState {
-  fn busy(&self) -> bool {
-    self.pruning_containers
-      || self.pruning_images
-      || self.pruning_networks
-  }
-}
-
-impl Busy for DeploymentActionState {
-  fn busy(&self) -> bool {
-    self.deploying
-      || self.pulling
-      || self.recloning
-      || self.removing
-      || self.starting
-      || self.stopping
-      || self.updating
-      || self.renaming
-  }
-}
-
-impl Busy for BuildActionState {
-  fn busy(&self) -> bool {
-    self.building || self.updating
-  }
-}
-
-impl Busy for CommandActionState {
-  fn busy(&self) -> bool {
-    self.running
-  }
-}
-
-impl From<&Deployment> for CloneArgs {
-  fn from(d: &Deployment) -> Self {
-    CloneArgs {
-      name: d.name.clone(),
-      repo: d.repo.clone(),
-      branch: d.branch.clone(),
-      on_clone: d.on_clone.clone(),
-      on_pull: d.on_pull.clone(),
-      github_account: d.github_account.clone(),
-    }
-  }
-}
-
-impl From<&Build> for CloneArgs {
-  fn from(b: &Build) -> Self {
-    CloneArgs {
-      name: b.name.clone(),
-      repo: b.repo.clone(),
-      branch: b.branch.clone(),
-      on_clone: b.pre_build.clone(),
-      on_pull: None,
-      github_account: b.github_account.clone(),
-    }
-  }
-}

bin/migrator/src/legacy/v0/update.rs

@@ -2,10 +2,7 @@ use monitor_client::entities::update::ResourceTarget;
 use mungos::mongodb::bson::serde_helpers::hex_string_as_object_id;
 use serde::{Deserialize, Serialize};
 
-use super::{
-  unix_from_monitor_ts, Build, Deployment, Group, Operation,
-  Procedure, Server, Version,
-};
+use super::{unix_from_monitor_ts, Operation, Version};
 
 #[derive(Serialize, Deserialize, Debug, Clone, Default)]
 pub struct Update {
@@ -51,6 +48,7 @@ impl TryFrom<Update> for monitor_client::entities::update::Update {
         .and_then(|ts| unix_from_monitor_ts(&ts).ok()),
       status: value.status.into(),
       version: value.version.map(|v| v.into()).unwrap_or_default(),
+      other_data: Default::default(),
     };
     Ok(update)
   }
@@ -107,7 +105,7 @@ impl From<UpdateTarget>
       UpdateTarget::Server(id) => Some(Server(id)),
       UpdateTarget::Procedure(_) => None,
       UpdateTarget::Group(_) => None,
-      UpdateTarget::Command(id) => None,
+      UpdateTarget::Command(_) => None,
     }
   }
 }

bin/migrator/src/legacy/v0/user.rs

@@ -106,7 +106,11 @@ impl TryFrom<User> for monitor_client::entities::user::User {
       create_server_permissions: value.create_server_permissions,
       create_build_permissions: value.create_build_permissions,
       last_update_view: Default::default(),
-      recently_viewed: Default::default(),
+      recent_servers: Vec::new(),
+      recent_deployments: Vec::new(),
+      recent_builds: Vec::new(),
+      recent_repos: Vec::new(),
+      recent_procedures: Vec::new(),
       updated_at: unix_from_monitor_ts(&value.updated_at)?,
     };
     Ok(user)