deploy 2.0.0-dev-59

Cargo.toml

@@ -8,7 +8,7 @@ members = [
 ]
 
 [workspace.package]
-version = "2.0.0-dev-36"
+version = "2.0.0-dev-59"
 edition = "2024"
 authors = ["mbecker20 <becker.maxh@gmail.com>"]
 license = "GPL-3.0-or-later"
@@ -25,9 +25,11 @@ periphery_client = { path = "client/periphery/rs" }
 environment_file = { path = "lib/environment_file" }
 environment = { path = "lib/environment" }
 interpolate = { path = "lib/interpolate" }
+secret_file = { path = "lib/secret_file" }
 formatting = { path = "lib/formatting" }
 transport = { path = "lib/transport" }
 database = { path = "lib/database" }
+encoding = { path = "lib/encoding" }
 response = { path = "lib/response" }
 command = { path = "lib/command" }
 config = { path = "lib/config" }
@@ -39,7 +41,7 @@ git = { path = "lib/git" }
 # MOGH
 run_command = { version = "0.0.6", features = ["async_tokio"] }
 serror = { version = "0.5.3", default-features = false }
-slack = { version = "1.1.0", package = "slack_client_rs", default-features = false, features = ["rustls"] }
+slack = { version = "1.2.1", package = "slack_client_rs", default-features = false, features = ["rustls"] }
 derive_default_builder = "0.1.8"
 derive_empty_traits = "0.1.0"
 async_timing_util = "1.1.0"
@@ -52,7 +54,7 @@ mungos = "3.2.2"
 svi = "1.2.0"
 
 # ASYNC
-reqwest = { version = "0.12.23", default-features = false, features = ["json", "stream", "rustls-tls-native-roots"] }
+reqwest = { version = "0.12.24", default-features = false, features = ["json", "stream", "rustls-tls-native-roots"] }
 tokio = { version = "1.47.1", features = ["full"] }
 tokio-util = { version = "0.7.16", features = ["io", "codec"] }
 tokio-stream = { version = "0.1.17", features = ["sync"] }
@@ -77,7 +79,7 @@ bson = { version = "2.15.0" } # must keep in sync with mongodb version
 serde_yaml_ng = "0.10.0"
 serde_json = "1.0.145"
 serde_qs = "0.15.0"
-toml = "0.9.7"
+toml = "0.9.8"
 url = "2.5.7"
 
 # ERROR
@@ -94,18 +96,17 @@ opentelemetry = "0.31.0"
 tracing = "0.1.41"
 
 # CONFIG
-clap = { version = "4.5.48", features = ["derive"] }
+clap = { version = "4.5.49", features = ["derive"] }
 dotenvy = "0.15.7"
 envy = "0.4.2"
 
 # CRYPTO / AUTH
 uuid = { version = "1.18.1", features = ["v4", "fast-rng", "serde"] }
-jsonwebtoken = { version = "9.3.1", default-features = false } # locked back with octorust
+jsonwebtoken = { version = "10.0.0", features = ["aws_lc_rs"] } # locked back with octorust
 rustls = { version = "0.23.32", features = ["aws-lc-rs"] }
 pem-rfc7468 = { version = "0.7.0", features = ["alloc"] }
 openidconnect = "4.0.1"
 urlencoding = "2.1.3"
-nom_pem = "4.0.0"
 bcrypt = "0.17.1"
 base64 = "0.22.1"
 pkcs8 = "0.10.2"
@@ -124,9 +125,9 @@ bollard = "0.19.3"
 sysinfo = "0.37.1"
 
 # CLOUD
-aws-config = "1.8.6"
-aws-sdk-ec2 = "1.170.1"
-aws-credential-types = "1.2.6"
+aws-config = "1.8.8"
+aws-sdk-ec2 = "1.173.0"
+aws-credential-types = "1.2.8"
 
 ## CRON
 english-to-cron = "0.1.6"
@@ -140,9 +141,8 @@ derive_builder = "0.20.2"
 shell-escape = "0.1.5"
 comfy-table = "7.2.1"
 typeshare = "1.0.4"
-octorust = "0.10.0"
 dashmap = "6.1.0"
 wildcard = "0.3.0"
 colored = "3.0.0"
-regex = "1.11.3"
+regex = "1.12.2"
 bytes = "1.10.1"

bin/cli/src/command/database.rs

@@ -358,7 +358,9 @@ async fn v1_downgrade(yes: bool) -> anyhow::Result<()> {
     .await
     .context("Failed to downgrade Server schema")?;
 
-  info!("V1 Downgrade complete. Ready to downgrade to komodo-core:1 ✅");
+  info!(
+    "V1 Downgrade complete. Ready to downgrade to komodo-core:1 ✅"
+  );
 
   Ok(())
 }

bin/cli/src/command/execute.rs

@@ -233,6 +233,9 @@ pub async fn handle(
     Execution::RotateAllServerKeys(data) => {
       println!("{}: {data:?}", "Data".dimmed())
     }
+    Execution::RotateCoreKeys(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
     Execution::Sleep(data) => {
       println!("{}: {data:?}", "Data".dimmed())
     }
@@ -501,6 +504,10 @@ pub async fn handle(
       .execute(request)
       .await
       .map(|u| ExecutionResult::Single(u.into())),
+    Execution::RotateCoreKeys(request) => client
+      .execute(request)
+      .await
+      .map(|u| ExecutionResult::Single(u.into())),
     Execution::Sleep(request) => {
       let duration =
         Duration::from_millis(request.duration_ms as u64);

bin/cli/src/config.rs

@@ -28,7 +28,7 @@ pub fn cli_env() -> &'static Env {
     {
       Ok(env) => env,
       Err(e) => {
-        panic!("{e:?}");
+        panic!("{e:?}")
       }
     }
   })
@@ -261,12 +261,18 @@ pub fn cli_config() -> &'static CliConfig {
           .komodo_cli_logging_pretty
           .unwrap_or(config.cli_logging.pretty),
         location: false,
+        ansi: env
+          .komodo_cli_logging_ansi
+          .unwrap_or(config.cli_logging.ansi),
         otlp_endpoint: env
           .komodo_cli_logging_otlp_endpoint
           .unwrap_or(config.cli_logging.otlp_endpoint),
         opentelemetry_service_name: env
           .komodo_cli_logging_opentelemetry_service_name
           .unwrap_or(config.cli_logging.opentelemetry_service_name),
+        opentelemetry_scope_name: env
+          .komodo_cli_logging_opentelemetry_scope_name
+          .unwrap_or(config.cli_logging.opentelemetry_scope_name),
       },
       profile: config.profile,
     }

bin/core/Cargo.toml

@@ -19,9 +19,11 @@ komodo_client = { workspace = true, features = ["mongo"] }
 periphery_client.workspace = true
 environment_file.workspace = true
 interpolate.workspace = true
+secret_file.workspace = true
 formatting.workspace = true
 transport.workspace = true
 database.workspace = true
+encoding.workspace = true
 response.workspace = true
 command.workspace = true
 config.workspace = true
@@ -55,7 +57,6 @@ serde_yaml_ng.workspace = true
 typeshare.workspace = true
 chrono-tz.workspace = true
 indexmap.workspace = true
-octorust.workspace = true
 wildcard.workspace = true
 arc-swap.workspace = true
 colored.workspace = true
@@ -63,7 +64,6 @@ dashmap.workspace = true
 tracing.workspace = true
 reqwest.workspace = true
 futures.workspace = true
-nom_pem.workspace = true
 dotenvy.workspace = true
 anyhow.workspace = true
 croner.workspace = true
@@ -74,12 +74,12 @@ rustls.workspace = true
 bytes.workspace = true
 tokio.workspace = true
 serde.workspace = true
+strum.workspace = true
 regex.workspace = true
 axum.workspace = true
 toml.workspace = true
 uuid.workspace = true
 envy.workspace = true
-rand.workspace = true
 hmac.workspace = true
 sha2.workspace = true
 hex.workspace = true

bin/core/aio.Dockerfile

@@ -55,8 +55,10 @@ ENV KOMODO_CLI_CONFIG_PATHS="/config"
 # This ensures any `komodo.cli.*` takes precedence over the Core `/config/*config.*`
 ENV KOMODO_CLI_CONFIG_KEYWORDS="*config.*,*komodo.cli*.*"
 
-CMD [ "core" ]
+CMD [ "/bin/bash", "-c", "update-ca-certificates && core" ]
 
+# Label to prevent Komodo from stopping with StopAllContainers
+LABEL komodo.skip="true"
 # Label for Ghcr
 LABEL org.opencontainers.image.source="https://github.com/moghtech/komodo"
 LABEL org.opencontainers.image.description="Komodo Core"

bin/core/multi-arch.Dockerfile

@@ -51,8 +51,10 @@ ENV KOMODO_CLI_CONFIG_PATHS="/config"
 # This ensures any `komodo.cli.*` takes precedence over the Core `/config/*config.*`
 ENV KOMODO_CLI_CONFIG_KEYWORDS="*config.*,*komodo.cli*.*"
 
-CMD [ "core" ]
+CMD [ "/bin/bash", "-c", "update-ca-certificates && core" ]
 
+# Label to prevent Komodo from stopping with StopAllContainers
+LABEL komodo.skip="true"
 # Label for Ghcr
 LABEL org.opencontainers.image.source="https://github.com/moghtech/komodo"
 LABEL org.opencontainers.image.description="Komodo Core"

bin/core/single-arch.Dockerfile

@@ -40,8 +40,10 @@ ENV KOMODO_CLI_CONFIG_PATHS="/config"
 # This ensures any `komodo.cli.*` takes precedence over the Core `/config/*config.*`
 ENV KOMODO_CLI_CONFIG_KEYWORDS="*config.*,*komodo.cli*.*"
 
-CMD [ "core" ]
+CMD [ "/bin/bash", "-c", "update-ca-certificates && core" ]
 
+# Label to prevent Komodo from stopping with StopAllContainers
+LABEL komodo.skip="true"
 # Label for Ghcr
 LABEL org.opencontainers.image.source="https://github.com/moghtech/komodo"
 LABEL org.opencontainers.image.description="Komodo Core"

bin/core/src/alert/discord.rs

@@ -4,7 +4,6 @@ use serde::Serialize;
 
 use super::*;
 
-#[instrument(level = "debug")]
 pub async fn send_alert(
   url: &str,
   alert: &Alert,
@@ -50,7 +49,7 @@ pub async fn send_alert(
       match alert.level {
         SeverityLevel::Ok => {
           format!(
-            "{level} | **{name}**{region} is now **reachable**\n{link}"
+            "{level} | **{name}**{region} is now **connected**\n{link}"
           )
         }
         SeverityLevel::Critical => {
@@ -241,31 +240,33 @@ pub async fn send_alert(
     }
     AlertData::None {} => Default::default(),
   };
-  if !content.is_empty() {
-    let VariablesAndSecrets { variables, secrets } =
-      get_variables_and_secrets().await?;
-    let mut url_interpolated = url.to_string();
 
-    let mut interpolator =
-      Interpolator::new(Some(&variables), &secrets);
-
-    interpolator.interpolate_string(&mut url_interpolated)?;
-
-    send_message(&url_interpolated, &content)
-      .await
-      .map_err(|e| {
-        let replacers = interpolator
-          .secret_replacers
-          .into_iter()
-          .collect::<Vec<_>>();
-        let sanitized_error =
-          svi::replace_in_string(&format!("{e:?}"), &replacers);
-        anyhow::Error::msg(format!(
-          "Error with slack request: {sanitized_error}"
-        ))
-      })?;
+  if content.is_empty() {
+    return Ok(());
   }
-  Ok(())
+
+  let VariablesAndSecrets { variables, secrets } =
+    get_variables_and_secrets().await?;
+  let mut url_interpolated = url.to_string();
+
+  let mut interpolator =
+    Interpolator::new(Some(&variables), &secrets);
+
+  interpolator.interpolate_string(&mut url_interpolated)?;
+
+  send_message(&url_interpolated, &content)
+    .await
+    .map_err(|e| {
+      let replacers = interpolator
+        .secret_replacers
+        .into_iter()
+        .collect::<Vec<_>>();
+      let sanitized_error =
+        svi::replace_in_string(&format!("{e:?}"), &replacers);
+      anyhow::Error::msg(format!(
+        "Error with slack request: {sanitized_error}"
+      ))
+    })
 }
 
 async fn send_message(

bin/core/src/alert/mod.rs

@@ -11,7 +11,6 @@ use komodo_client::entities::{
   komodo_timestamp,
   stack::StackState,
 };
-use tracing::Instrument;
 
 use crate::helpers::query::get_variables_and_secrets;
 use crate::helpers::{
@@ -24,40 +23,32 @@ mod ntfy;
 mod pushover;
 mod slack;
 
-#[instrument(level = "debug")]
 pub async fn send_alerts(alerts: &[Alert]) {
   if alerts.is_empty() {
     return;
   }
 
-  let span =
-    info_span!("send_alerts", alerts = format!("{alerts:?}"));
-  async {
-    let Ok(alerters) = find_collect(
-      &db_client().alerters,
-      doc! { "config.enabled": true },
-      None,
-    )
-    .await
-    .inspect_err(|e| {
-      error!(
+  let Ok(alerters) = find_collect(
+    &db_client().alerters,
+    doc! { "config.enabled": true },
+    None,
+  )
+  .await
+  .inspect_err(|e| {
+    error!(
       "ERROR sending alerts | failed to get alerters from db | {e:#}"
     )
-    }) else {
-      return;
-    };
+  }) else {
+    return;
+  };
 
-    let handles = alerts
-      .iter()
-      .map(|alert| send_alert_to_alerters(&alerters, alert));
+  let handles = alerts
+    .iter()
+    .map(|alert| send_alert_to_alerters(&alerters, alert));
 
-    join_all(handles).await;
-  }
-  .instrument(span)
-  .await
+  join_all(handles).await;
 }
 
-#[instrument(level = "debug")]
 async fn send_alert_to_alerters(alerters: &[Alerter], alert: &Alert) {
   if alerters.is_empty() {
     return;
@@ -161,7 +152,6 @@ pub async fn send_alert_to_alerter(
   }
 }
 
-#[instrument(level = "debug")]
 async fn send_custom_alert(
   url: &str,
   alert: &Alert,
@@ -294,7 +284,7 @@ fn standard_alert_content(alert: &Alert) -> String {
       let link = resource_link(ResourceTargetVariant::Server, id);
       match alert.level {
         SeverityLevel::Ok => {
-          format!("{level} | {name}{region} is now reachable\n{link}")
+          format!("{level} | {name}{region} is now connected\n{link}")
         }
         SeverityLevel::Critical => {
           let err = err

bin/core/src/alert/ntfy.rs

@@ -2,17 +2,38 @@ use std::sync::OnceLock;
 
 use super::*;
 
-#[instrument(level = "debug")]
 pub async fn send_alert(
   url: &str,
   email: Option<&str>,
   alert: &Alert,
 ) -> anyhow::Result<()> {
   let content = standard_alert_content(alert);
-  if !content.is_empty() {
-    send_message(url, email, content).await?;
+  if content.is_empty() {
+    return Ok(());
   }
-  Ok(())
+
+  let VariablesAndSecrets { variables, secrets } =
+    get_variables_and_secrets().await?;
+  let mut url_interpolated = url.to_string();
+
+  let mut interpolator =
+    Interpolator::new(Some(&variables), &secrets);
+
+  interpolator.interpolate_string(&mut url_interpolated)?;
+
+  send_message(&url_interpolated, email, content)
+    .await
+    .map_err(|e| {
+      let replacers = interpolator
+        .secret_replacers
+        .into_iter()
+        .collect::<Vec<_>>();
+      let sanitized_error =
+        svi::replace_in_string(&format!("{e:?}"), &replacers);
+      anyhow::Error::msg(format!(
+        "Error with slack request: {sanitized_error}"
+      ))
+    })
 }
 
 async fn send_message(
@@ -22,7 +43,7 @@ async fn send_message(
 ) -> anyhow::Result<()> {
   let mut request = http_client()
     .post(url)
-    .header("Title", "ntfy Alert")
+    .header("Title", "Komodo Alert")
     .body(content);
 
   if let Some(email) = email {
@@ -43,9 +64,7 @@ async fn send_message(
       )
     })?;
     Err(anyhow!(
-      "Failed to send message to ntfy | {} | {}",
-      status,
-      text
+      "Failed to send message to ntfy | {status} | {text}",
     ))
   }
 }

bin/core/src/alert/pushover.rs

@@ -2,16 +2,35 @@ use std::sync::OnceLock;
 
 use super::*;
 
-#[instrument(level = "debug")]
 pub async fn send_alert(
   url: &str,
   alert: &Alert,
 ) -> anyhow::Result<()> {
   let content = standard_alert_content(alert);
-  if !content.is_empty() {
-    send_message(url, content).await?;
+  if content.is_empty() {
+    return Ok(());
   }
-  Ok(())
+
+  let VariablesAndSecrets { variables, secrets } =
+    get_variables_and_secrets().await?;
+  let mut url_interpolated = url.to_string();
+
+  let mut interpolator =
+    Interpolator::new(Some(&variables), &secrets);
+
+  interpolator.interpolate_string(&mut url_interpolated)?;
+
+  send_message(&url_interpolated, content).await.map_err(|e| {
+    let replacers = interpolator
+      .secret_replacers
+      .into_iter()
+      .collect::<Vec<_>>();
+    let sanitized_error =
+      svi::replace_in_string(&format!("{e:?}"), &replacers);
+    anyhow::Error::msg(format!(
+      "Error with slack request: {sanitized_error}"
+    ))
+  })
 }
 
 async fn send_message(

bin/core/src/alert/slack.rs

@@ -2,7 +2,6 @@ use ::slack::types::OwnedBlock as Block;
 
 use super::*;
 
-#[instrument(level = "debug")]
 pub async fn send_alert(
   url: &str,
   alert: &Alert,
@@ -64,11 +63,11 @@ pub async fn send_alert(
       match alert.level {
         SeverityLevel::Ok => {
           let text =
-            format!("{level} | *{name}*{region} is now *reachable*");
+            format!("{level} | *{name}*{region} is now *connected*");
           let blocks = vec![
             Block::header(level),
             Block::section(format!(
-              "*{name}*{region} is now *reachable*"
+              "*{name}*{region} is now *connnected*"
             )),
           ];
           (text, blocks.into())
@@ -468,31 +467,32 @@ pub async fn send_alert(
     }
     AlertData::None {} => Default::default(),
   };
-  if !text.is_empty() {
-    let VariablesAndSecrets { variables, secrets } =
-      get_variables_and_secrets().await?;
-    let mut url_interpolated = url.to_string();
-
-    let mut interpolator =
-      Interpolator::new(Some(&variables), &secrets);
-
-    interpolator.interpolate_string(&mut url_interpolated)?;
-
-    let slack = ::slack::Client::new(url_interpolated);
-    slack
-      .send_owned_message_single(&text, blocks.as_deref())
-      .await
-      .map_err(|e| {
-        let replacers = interpolator
-          .secret_replacers
-          .into_iter()
-          .collect::<Vec<_>>();
-        let sanitized_error =
-          svi::replace_in_string(&format!("{e:?}"), &replacers);
-        anyhow::Error::msg(format!(
-          "Error with slack request: {sanitized_error}"
-        ))
-      })?;
+  if text.is_empty() {
+    return Ok(());
   }
+  let VariablesAndSecrets { variables, secrets } =
+    get_variables_and_secrets().await?;
+  let mut url_interpolated = url.to_string();
+
+  let mut interpolator =
+    Interpolator::new(Some(&variables), &secrets);
+
+  interpolator.interpolate_string(&mut url_interpolated)?;
+
+  let slack = ::slack::Client::new(url_interpolated);
+  slack
+    .send_owned_message_single(&text, None, blocks.as_deref())
+    .await
+    .map_err(|e| {
+      let replacers = interpolator
+        .secret_replacers
+        .into_iter()
+        .collect::<Vec<_>>();
+      let sanitized_error =
+        svi::replace_in_string(&format!("{e:?}"), &replacers);
+      anyhow::Error::msg(format!(
+        "Error with slack request: {sanitized_error}"
+      ))
+    })?;
   Ok(())
 }

bin/core/src/api/auth.rs

@@ -88,7 +88,6 @@ async fn variant_handler(
   handler(headers, Json(req)).await
 }
 
-#[instrument(name = "AuthHandler", level = "debug", skip(headers))]
 async fn handler(
   headers: HeaderMap,
   Json(request): Json<AuthRequest>,
@@ -125,7 +124,6 @@ fn login_options_reponse() -> &'static GetLoginOptionsResponse {
 }
 
 impl Resolve<AuthArgs> for GetLoginOptions {
-  #[instrument(name = "GetLoginOptions", level = "debug", skip(self))]
   async fn resolve(
     self,
     _: &AuthArgs,
@@ -135,7 +133,6 @@ impl Resolve<AuthArgs> for GetLoginOptions {
 }
 
 impl Resolve<AuthArgs> for ExchangeForJwt {
-  #[instrument(name = "ExchangeForJwt", level = "debug", skip(self))]
   async fn resolve(
     self,
     _: &AuthArgs,
@@ -148,7 +145,6 @@ impl Resolve<AuthArgs> for ExchangeForJwt {
 }
 
 impl Resolve<AuthArgs> for GetUser {
-  #[instrument(name = "GetUser", level = "debug", skip(self))]
   async fn resolve(
     self,
     AuthArgs { headers }: &AuthArgs,

bin/core/src/api/execute/action.rs

@@ -1,7 +1,6 @@
 use std::{
   collections::HashSet,
   path::{Path, PathBuf},
-  str::FromStr,
   sync::OnceLock,
 };
 
@@ -24,6 +23,7 @@ use komodo_client::{
     config::core::CoreConfig,
     komodo_timestamp,
     permission::PermissionLevel,
+    random_string,
     update::Update,
     user::action_user,
   },
@@ -38,7 +38,6 @@ use crate::{
   config::core_config,
   helpers::{
     query::{VariablesAndSecrets, get_variables_and_secrets},
-    random_string,
     update::update_update,
   },
   permission::get_check_permissions,
@@ -59,10 +58,18 @@ impl super::BatchExecute for BatchRunAction {
 }
 
 impl Resolve<ExecuteArgs> for BatchRunAction {
-  #[instrument(name = "BatchRunAction", skip(self, user), fields(user_id = user.id))]
+  #[instrument(
+    "BatchRunAction",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      pattern = self.pattern,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, .. }: &ExecuteArgs,
+    ExecuteArgs { user, id, .. }: &ExecuteArgs,
   ) -> serror::Result<BatchExecutionResponse> {
     Ok(
       super::batch_execute::<BatchRunAction>(&self.pattern, user)
@@ -72,10 +79,19 @@ impl Resolve<ExecuteArgs> for BatchRunAction {
 }
 
 impl Resolve<ExecuteArgs> for RunAction {
-  #[instrument(name = "RunAction", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+  #[instrument(
+    "RunAction",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      action = self.action,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
     let mut action = get_check_permissions::<Action>(
       &self.action,
@@ -142,15 +158,11 @@ impl Resolve<ExecuteArgs> for RunAction {
     let file = format!("{}.ts", random_string(10));
     let path = core_config().action_directory.join(&file);
 
-    if let Some(parent) = path.parent() {
-      fs::create_dir_all(parent)
-        .await
-        .with_context(|| format!("Failed to initialize Action file parent directory {parent:?}"))?;
-    }
-
-    fs::write(&path, contents).await.with_context(|| {
-      format!("Failed to write action file to {path:?}")
-    })?;
+    secret_file::write_async(&path, contents)
+      .await
+      .with_context(|| {
+        format!("Failed to write action file to {path:?}")
+      })?;
 
     let CoreConfig { ssl_enabled, .. } = core_config();
 
@@ -217,7 +229,6 @@ impl Resolve<ExecuteArgs> for RunAction {
     update_update(update.clone()).await?;
 
     if !update.success && action.config.failure_alert {
-      warn!("action unsuccessful, alerting...");
       let target = update.target.clone();
       tokio::spawn(async move {
         let alert = Alert {
@@ -240,6 +251,7 @@ impl Resolve<ExecuteArgs> for RunAction {
   }
 }
 
+#[instrument("Interpolate", skip(contents, update, secret))]
 async fn interpolate(
   contents: &mut String,
   update: &mut Update,
@@ -325,6 +337,7 @@ main()
 /// Cleans up file at given path.
 /// ALSO if $DENO_DIR is set,
 /// will clean up the generated file matching "file"
+#[instrument("CleanupRun")]
 async fn cleanup_run(file: String, path: &Path) {
   if let Err(e) = fs::remove_file(path).await {
     warn!(
@@ -344,7 +357,7 @@ fn deno_dir() -> Option<&'static Path> {
   DENO_DIR
     .get_or_init(|| {
       let deno_dir = std::env::var("DENO_DIR").ok()?;
-      PathBuf::from_str(&deno_dir).ok()
+      Some(PathBuf::from(&deno_dir))
     })
     .as_deref()
 }

bin/core/src/api/execute/alerter.rs

@@ -1,6 +1,6 @@
 use anyhow::{Context, anyhow};
 use formatting::format_serror;
-use futures::{TryStreamExt, stream::FuturesUnordered};
+use futures::{StreamExt, TryStreamExt, stream::FuturesUnordered};
 use komodo_client::{
   api::execute::{SendAlert, TestAlerter},
   entities::{
@@ -22,10 +22,19 @@ use crate::{
 use super::ExecuteArgs;
 
 impl Resolve<ExecuteArgs> for TestAlerter {
-  #[instrument(name = "TestAlerter", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+  #[instrument(
+    "TestAlerter",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      alerter = self.alerter,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> Result<Self::Response, Self::Error> {
     let alerter = get_check_permissions::<Alerter>(
       &self.alerter,
@@ -79,28 +88,58 @@ impl Resolve<ExecuteArgs> for TestAlerter {
 //
 
 impl Resolve<ExecuteArgs> for SendAlert {
-  #[instrument(name = "SendAlert", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+  #[instrument(
+    "SendAlert",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      request = format!("{self:?}"),
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> Result<Self::Response, Self::Error> {
-    let alerters = list_full_for_user::<Alerter>(
-      Default::default(),
-      user,
-      PermissionLevel::Execute.into(),
-      &[],
-    )
-    .await?
-    .into_iter()
-    .filter(|a| {
-      a.config.enabled
-        && (self.alerters.is_empty()
-          || self.alerters.contains(&a.name)
-          || self.alerters.contains(&a.id))
-        && (a.config.alert_types.is_empty()
-          || a.config.alert_types.contains(&AlertDataVariant::Custom))
-    })
-    .collect::<Vec<_>>();
+    let alerters =
+      list_full_for_user::<Alerter>(Default::default(), user, &[])
+        .await?
+        .into_iter()
+        .filter(|a| {
+          a.config.enabled
+            && (self.alerters.is_empty()
+              || self.alerters.contains(&a.name)
+              || self.alerters.contains(&a.id))
+            && (a.config.alert_types.is_empty()
+              || a
+                .config
+                .alert_types
+                .contains(&AlertDataVariant::Custom))
+        })
+        .collect::<Vec<_>>();
+
+    let alerters = if user.admin {
+      alerters
+    } else {
+      // Only keep alerters with execute permissions
+      alerters
+        .into_iter()
+        .map(|alerter| async move {
+          get_check_permissions::<Alerter>(
+            &alerter.id,
+            user,
+            PermissionLevel::Execute.into(),
+          )
+          .await
+        })
+        .collect::<FuturesUnordered<_>>()
+        .collect::<Vec<_>>()
+        .await
+        .into_iter()
+        .flatten()
+        .collect()
+    };
 
     if alerters.is_empty() {
       return Err(anyhow!(

bin/core/src/api/execute/build.rs

@@ -37,12 +37,13 @@ use komodo_client::{
 use periphery_client::api;
 use resolver_api::Resolve;
 use tokio_util::sync::CancellationToken;
+use uuid::Uuid;
 
 use crate::{
   alert::send_alerts,
   helpers::{
     build_git_token,
-    builder::{cleanup_builder_instance, get_builder_periphery},
+    builder::{cleanup_builder_instance, connect_builder_periphery},
     channel::build_cancel_channel,
     query::{
       VariablesAndSecrets, get_deployment_state,
@@ -66,10 +67,18 @@ impl super::BatchExecute for BatchRunBuild {
 }
 
 impl Resolve<ExecuteArgs> for BatchRunBuild {
-  #[instrument(name = "BatchRunBuild", skip(user), fields(user_id = user.id))]
+  #[instrument(
+    "BatchRunBuild",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      pattern = self.pattern,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, .. }: &ExecuteArgs,
+    ExecuteArgs { user, id, .. }: &ExecuteArgs,
   ) -> serror::Result<BatchExecutionResponse> {
     Ok(
       super::batch_execute::<BatchRunBuild>(&self.pattern, user)
@@ -79,10 +88,19 @@ impl Resolve<ExecuteArgs> for BatchRunBuild {
 }
 
 impl Resolve<ExecuteArgs> for RunBuild {
-  #[instrument(name = "RunBuild", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+  #[instrument(
+    "RunBuild",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      build = self.build,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
     let mut build = get_check_permissions::<Build>(
       &self.build,
@@ -186,7 +204,7 @@ impl Resolve<ExecuteArgs> for RunBuild {
     });
 
     // GET BUILDER PERIPHERY
-    let (periphery, cleanup_data) = match get_builder_periphery(
+    let (periphery, cleanup_data) = match connect_builder_periphery(
       build.name.clone(),
       Some(build.config.version),
       builder,
@@ -247,11 +265,11 @@ impl Resolve<ExecuteArgs> for RunBuild {
             replacers: Default::default(),
           }) => res,
         _ = cancel.cancelled() => {
-          debug!("build cancelled during clone, cleaning up builder");
-          update.push_error_log("build cancelled", String::from("user cancelled build during repo clone"));
+          debug!("Build cancelled during clone, cleaning up builder");
+          update.push_error_log("Build cancelled", String::from("user cancelled build during repo clone"));
           cleanup_builder_instance(periphery, cleanup_data, &mut update)
             .await;
-          info!("builder cleaned up");
+          info!("Builder cleaned up");
           return handle_early_return(update, build.id, build.name, true).await
         },
       };
@@ -296,8 +314,8 @@ impl Resolve<ExecuteArgs> for RunBuild {
             additional_tags: Default::default(),
           }) => res.context("failed at call to periphery to build"),
         _ = cancel.cancelled() => {
-          info!("build cancelled during build, cleaning up builder");
-          update.push_error_log("build cancelled", String::from("user cancelled build during docker build"));
+          info!("Build cancelled during build, cleaning up builder");
+          update.push_error_log("Build cancelled", String::from("user cancelled build during docker build"));
           cleanup_builder_instance(periphery, cleanup_data, &mut update)
             .await;
           return handle_early_return(update, build.id, build.name, true).await
@@ -312,7 +330,7 @@ impl Resolve<ExecuteArgs> for RunBuild {
         Err(e) => {
           warn!("error in build | {e:#}");
           update.push_error_log(
-            "build",
+            "Build Error",
             format_serror(&e.context("failed to build").into()),
           )
         }
@@ -370,7 +388,6 @@ impl Resolve<ExecuteArgs> for RunBuild {
         handle_post_build_redeploy(&build.id).await;
       });
     } else {
-      warn!("build unsuccessful, alerting...");
       let target = update.target.clone();
       let version = update.version;
       tokio::spawn(async move {
@@ -395,7 +412,7 @@ impl Resolve<ExecuteArgs> for RunBuild {
   }
 }
 
-#[instrument(skip(update))]
+#[instrument("HandleEarlyReturn", skip(update))]
 async fn handle_early_return(
   mut update: Update,
   build_id: String,
@@ -419,7 +436,6 @@ async fn handle_early_return(
   }
   update_update(update.clone()).await?;
   if !update.success && !is_cancel {
-    warn!("build unsuccessful, alerting...");
     let target = update.target.clone();
     let version = update.version;
     tokio::spawn(async move {
@@ -489,10 +505,19 @@ pub async fn validate_cancel_build(
 }
 
 impl Resolve<ExecuteArgs> for CancelBuild {
-  #[instrument(name = "CancelBuild", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+  #[instrument(
+    "CancelBuild",
+    skip(user, update),
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      build = self.build,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
     let build = get_check_permissions::<Build>(
       &self.build,
@@ -549,7 +574,7 @@ impl Resolve<ExecuteArgs> for CancelBuild {
   }
 }
 
-#[instrument]
+#[instrument("PostBuildRedeploy")]
 async fn handle_post_build_redeploy(build_id: &str) {
   let Ok(redeploy_deployments) = find_collect(
     &db_client().deployments,
@@ -585,7 +610,11 @@ async fn handle_post_build_redeploy(build_id: &str) {
               stop_signal: None,
               stop_time: None,
             }
-            .resolve(&ExecuteArgs { user, update })
+            .resolve(&ExecuteArgs {
+              user,
+              update,
+              id: Uuid::new_v4(),
+            })
             .await
           }
           .await;
@@ -611,6 +640,7 @@ async fn handle_post_build_redeploy(build_id: &str) {
 /// This will make sure that a build with non-none image registry has an account attached,
 /// and will check the core config for a token matching requirements.
 /// Otherwise it is left to periphery.
+#[instrument("ValidateRegistryTokens")]
 async fn validate_account_extract_registry_tokens(
   Build {
     config: BuildConfig { image_registry, .. },

bin/core/src/api/execute/deployment.rs

@@ -49,10 +49,18 @@ impl super::BatchExecute for BatchDeploy {
 }
 
 impl Resolve<ExecuteArgs> for BatchDeploy {
-  #[instrument(name = "BatchDeploy", skip(user), fields(user_id = user.id))]
+  #[instrument(
+    "BatchDeploy",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      pattern = self.pattern,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, .. }: &ExecuteArgs,
+    ExecuteArgs { user, id, .. }: &ExecuteArgs,
   ) -> serror::Result<BatchExecutionResponse> {
     Ok(
       super::batch_execute::<BatchDeploy>(&self.pattern, user)
@@ -61,6 +69,7 @@ impl Resolve<ExecuteArgs> for BatchDeploy {
   }
 }
 
+#[instrument("SetupDeploy", skip_all)]
 async fn setup_deployment_execution(
   deployment: &str,
   user: &User,
@@ -87,10 +96,21 @@ async fn setup_deployment_execution(
 }
 
 impl Resolve<ExecuteArgs> for Deploy {
-  #[instrument(name = "Deploy", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+  #[instrument(
+    "Deploy",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      deployment = self.deployment,
+      stop_signal = format!("{:?}", self.stop_signal),
+      stop_time = self.stop_time,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
     let (mut deployment, server) =
       setup_deployment_execution(&self.deployment, user).await?;
@@ -243,6 +263,14 @@ fn pull_cache() -> &'static PullCache {
   PULL_CACHE.get_or_init(Default::default)
 }
 
+#[instrument(
+  "PullDeploymentInner",
+  skip_all,
+  fields(
+    deployment = deployment.id,
+    server = server.id
+  )
+)]
 pub async fn pull_deployment_inner(
   deployment: Deployment,
   server: &Server,
@@ -358,10 +386,19 @@ pub async fn pull_deployment_inner(
 }
 
 impl Resolve<ExecuteArgs> for PullDeployment {
-  #[instrument(name = "PullDeployment", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+  #[instrument(
+    "PullDeployment",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      deployment = self.deployment,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
     let (deployment, server) =
       setup_deployment_execution(&self.deployment, user).await?;
@@ -392,10 +429,19 @@ impl Resolve<ExecuteArgs> for PullDeployment {
 }
 
 impl Resolve<ExecuteArgs> for StartDeployment {
-  #[instrument(name = "StartDeployment", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+  #[instrument(
+    "StartDeployment",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      deployment = self.deployment,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
     let (deployment, server) =
       setup_deployment_execution(&self.deployment, user).await?;
@@ -440,10 +486,19 @@ impl Resolve<ExecuteArgs> for StartDeployment {
 }
 
 impl Resolve<ExecuteArgs> for RestartDeployment {
-  #[instrument(name = "RestartDeployment", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+  #[instrument(
+    "RestartDeployment",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      deployment = self.deployment,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
     let (deployment, server) =
       setup_deployment_execution(&self.deployment, user).await?;
@@ -490,10 +545,19 @@ impl Resolve<ExecuteArgs> for RestartDeployment {
 }
 
 impl Resolve<ExecuteArgs> for PauseDeployment {
-  #[instrument(name = "PauseDeployment", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+  #[instrument(
+    "PauseDeployment",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      deployment = self.deployment,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
     let (deployment, server) =
       setup_deployment_execution(&self.deployment, user).await?;
@@ -538,10 +602,19 @@ impl Resolve<ExecuteArgs> for PauseDeployment {
 }
 
 impl Resolve<ExecuteArgs> for UnpauseDeployment {
-  #[instrument(name = "UnpauseDeployment", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+  #[instrument(
+    "UnpauseDeployment",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      deployment = self.deployment,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
     let (deployment, server) =
       setup_deployment_execution(&self.deployment, user).await?;
@@ -588,10 +661,21 @@ impl Resolve<ExecuteArgs> for UnpauseDeployment {
 }
 
 impl Resolve<ExecuteArgs> for StopDeployment {
-  #[instrument(name = "StopDeployment", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+  #[instrument(
+    "StopDeployment",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      deployment = self.deployment,
+      signal = format!("{:?}", self.signal),
+      time = self.time,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
     let (deployment, server) =
       setup_deployment_execution(&self.deployment, user).await?;
@@ -655,10 +739,18 @@ impl super::BatchExecute for BatchDestroyDeployment {
 }
 
 impl Resolve<ExecuteArgs> for BatchDestroyDeployment {
-  #[instrument(name = "BatchDestroyDeployment", skip(user), fields(user_id = user.id))]
+  #[instrument(
+    "BatchDestroyDeployment",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      pattern = self.pattern,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, .. }: &ExecuteArgs,
+    ExecuteArgs { user, id, .. }: &ExecuteArgs,
   ) -> serror::Result<BatchExecutionResponse> {
     Ok(
       super::batch_execute::<BatchDestroyDeployment>(
@@ -671,10 +763,21 @@ impl Resolve<ExecuteArgs> for BatchDestroyDeployment {
 }
 
 impl Resolve<ExecuteArgs> for DestroyDeployment {
-  #[instrument(name = "DestroyDeployment", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+  #[instrument(
+    "DestroyDeployment",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      deployment = self.deployment,
+      signal = format!("{:?}", self.signal),
+      time = self.time,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
     let (deployment, server) =
       setup_deployment_execution(&self.deployment, user).await?;

bin/core/src/api/execute/maintenance.rs

@@ -2,19 +2,23 @@ use std::{fmt::Write as _, sync::OnceLock};
 
 use anyhow::{Context, anyhow};
 use command::run_komodo_command;
-use database::mungos::{find::find_collect, mongodb::bson::doc};
+use database::{
+  bson::{Document, doc},
+  mungos::find::find_collect,
+};
 use formatting::{bold, format_serror};
-use futures::StreamExt;
+use futures::{StreamExt, stream::FuturesOrdered};
 use komodo_client::{
   api::execute::{
     BackupCoreDatabase, ClearRepoCache, GlobalAutoUpdate,
-    RotateAllServerKeys,
+    RotateAllServerKeys, RotateCoreKeys,
   },
   entities::{
     deployment::DeploymentState, server::ServerState,
     stack::StackState,
   },
 };
+use periphery_client::api;
 use reqwest::StatusCode;
 use resolver_api::Resolve;
 use serror::AddStatusCodeError;
@@ -24,8 +28,8 @@ use crate::{
   api::execute::{
     ExecuteArgs, pull_deployment_inner, pull_stack_inner,
   },
-  config::core_config,
-  helpers::update::update_update,
+  config::{core_config, core_keys},
+  helpers::{periphery_client, update::update_update},
   resource::rotate_server_keys,
   state::{
     db_client, deployment_status_cache, server_status_cache,
@@ -41,13 +45,17 @@ fn clear_repo_cache_lock() -> &'static Mutex<()> {
 
 impl Resolve<ExecuteArgs> for ClearRepoCache {
   #[instrument(
-    name = "ClearRepoCache",
-    skip(user, update),
-    fields(user_id = user.id, update_id = update.id)
+    "ClearRepoCache",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id
+    )
   )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> Result<Self::Response, Self::Error> {
     if !user.admin {
       return Err(
@@ -116,13 +124,17 @@ fn backup_database_lock() -> &'static Mutex<()> {
 
 impl Resolve<ExecuteArgs> for BackupCoreDatabase {
   #[instrument(
-    name = "BackupCoreDatabase",
-    skip(user, update),
-    fields(user_id = user.id, update_id = update.id)
+    "BackupCoreDatabase",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+    )
   )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> Result<Self::Response, Self::Error> {
     if !user.admin {
       return Err(
@@ -165,13 +177,17 @@ fn global_update_lock() -> &'static Mutex<()> {
 
 impl Resolve<ExecuteArgs> for GlobalAutoUpdate {
   #[instrument(
-    name = "GlobalAutoUpdate",
-    skip(user, update),
-    fields(user_id = user.id, update_id = update.id)
+    "GlobalAutoUpdate",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+    )
   )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> Result<Self::Response, Self::Error> {
     if !user.admin {
       return Err(
@@ -330,9 +346,18 @@ fn global_rotate_lock() -> &'static Mutex<()> {
 }
 
 impl Resolve<ExecuteArgs> for RotateAllServerKeys {
+  #[instrument(
+    "RotateAllServerKeys",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> Result<Self::Response, Self::Error> {
     if !user.admin {
       return Err(
@@ -343,7 +368,7 @@ impl Resolve<ExecuteArgs> for RotateAllServerKeys {
 
     let _lock = global_rotate_lock()
       .try_lock()
-      .context("Rotate All Server Keys already in progress...")?;
+      .context("Key rotation already in progress...")?;
 
     let mut update = update.clone();
 
@@ -351,7 +376,7 @@ impl Resolve<ExecuteArgs> for RotateAllServerKeys {
 
     let mut servers = db_client()
       .servers
-      .find(doc! { "config.enabled": true })
+      .find(Document::new())
       .await
       .context("Failed to query servers from database")?;
 
@@ -367,6 +392,14 @@ impl Resolve<ExecuteArgs> for RotateAllServerKeys {
           continue;
         }
       };
+      if !server.config.auto_rotate_keys {
+        let _ = write!(
+          &mut log,
+          "\nSkipping {}: Key Rotation Disabled ⚙️",
+          bold(&server.name)
+        );
+        continue;
+      }
       let Some(status) = server_status_cache.get(&server.id).await
       else {
         let _ = write!(
@@ -376,14 +409,24 @@ impl Resolve<ExecuteArgs> for RotateAllServerKeys {
         );
         continue;
       };
-      if !matches!(status.state, ServerState::Ok) {
-        let _ = write!(
-          &mut log,
-          "\nSkipping {}: {} ⚠️",
-          bold(&server.name),
-          status.state
-        );
-        continue;
+      match status.state {
+        ServerState::Disabled => {
+          let _ = write!(
+            &mut log,
+            "\nSkipping {}: Server Disabled ⚙️",
+            bold(&server.name)
+          );
+          continue;
+        }
+        ServerState::NotOk => {
+          let _ = write!(
+            &mut log,
+            "\nSkipping {}: Server Not Ok ⚠️",
+            bold(&server.name)
+          );
+          continue;
+        }
+        _ => {}
       }
       match rotate_server_keys(&server).await {
         Ok(_) => {
@@ -415,3 +458,131 @@ impl Resolve<ExecuteArgs> for RotateAllServerKeys {
     Ok(update)
   }
 }
+
+impl Resolve<ExecuteArgs> for RotateCoreKeys {
+  #[instrument(
+    "RotateCoreKeys",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      force = self.force,
+    )
+  )]
+  async fn resolve(
+    self,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
+  ) -> Result<Self::Response, Self::Error> {
+    if !user.admin {
+      return Err(
+        anyhow!("This method is admin only.")
+          .status_code(StatusCode::FORBIDDEN),
+      );
+    }
+
+    let _lock = global_rotate_lock()
+      .try_lock()
+      .context("Key rotation already in progress...")?;
+
+    let mut update = update.clone();
+
+    update_update(update.clone()).await?;
+
+    let core_keys = core_keys();
+
+    if !core_keys.rotatable() {
+      return Err(anyhow!("Core `private_key` must be pointing to file, for example 'file:/config/keys/core.key'").into());
+    };
+
+    let server_status_cache = server_status_cache();
+    let servers =
+      find_collect(&db_client().servers, Document::new(), None)
+        .await
+        .context("Failed to query servers from database")?
+        .into_iter()
+        .map(|server| async move {
+          let state = server_status_cache
+            .get(&server.id)
+            .await
+            .map(|s| s.state)
+            .unwrap_or(ServerState::NotOk);
+          (server, state)
+        })
+        .collect::<FuturesOrdered<_>>()
+        .collect::<Vec<_>>()
+        .await;
+
+    if !self.force
+      && let Some((server, _)) = servers
+        .iter()
+        .find(|(_, state)| matches!(state, ServerState::NotOk))
+    {
+      return Err(
+        anyhow!("Server {} is NotOk, stopping key rotation. Pass `force: true` to continue anyways.", server.name).into(),
+      );
+    }
+
+    let public_key = core_keys.rotate().await?.into_inner();
+
+    info!("New Public Key: {public_key}");
+
+    let mut log = format!("New Public Key: {public_key}\n");
+
+    for (server, state) in servers {
+      match state {
+        ServerState::Disabled => {
+          let _ = write!(
+            &mut log,
+            "\nSkipping {}: Server Disabled ⚙️",
+            bold(&server.name)
+          );
+          continue;
+        }
+        ServerState::NotOk => {
+          // Shouldn't be reached unless 'force: true'
+          let _ = write!(
+            &mut log,
+            "\nSkipping {}: Server Not Ok ⚠️",
+            bold(&server.name)
+          );
+          continue;
+        }
+        _ => {}
+      }
+      let periphery = periphery_client(&server).await?;
+      let res = periphery
+        .request(api::keys::RotateCorePublicKey {
+          public_key: public_key.clone(),
+        })
+        .await;
+      match res {
+        Ok(_) => {
+          let _ = write!(
+            &mut log,
+            "\nRotated key for {} ✅",
+            bold(&server.name)
+          );
+        }
+        Err(e) => {
+          update.push_error_log(
+            "Key Rotation Failure",
+            format_serror(
+              &e.context(format!(
+                "Failed to rotate for {}. The new Core public key will have to be added manually.",
+                bold(&server.name)
+              ))
+              .into(),
+            ),
+          );
+        }
+      }
+    }
+
+    update.push_simple_log("Rotate Core Keys", log);
+    update.finalize();
+    update_update(update.clone()).await?;
+
+    Ok(update)
+  }
+}

bin/core/src/api/execute/mod.rs

@@ -1,4 +1,4 @@
-use std::{pin::Pin, time::Instant};
+use std::pin::Pin;
 
 use anyhow::Context;
 use axum::{
@@ -8,7 +8,9 @@ use axum_extra::{TypedHeader, headers::ContentType};
 use database::mungos::by_id::find_one_by_id;
 use derive_variants::{EnumVariants, ExtractVariant};
 use formatting::format_serror;
-use futures::future::join_all;
+use futures::{
+  StreamExt as _, future::join_all, stream::FuturesUnordered,
+};
 use komodo_client::{
   api::execute::*,
   entities::{
@@ -23,12 +25,14 @@ use response::JsonString;
 use serde::{Deserialize, Serialize};
 use serde_json::json;
 use serror::Json;
+use strum::Display;
 use typeshare::typeshare;
 use uuid::Uuid;
 
 use crate::{
   auth::auth_request,
   helpers::update::{init_execution_update, update_update},
+  permission::get_check_permissions,
   resource::{KomodoResource, list_full_for_user_using_pattern},
   state::db_client,
 };
@@ -51,6 +55,9 @@ pub use {
 };
 
 pub struct ExecuteArgs {
+  /// The execution id.
+  /// Unique for every /execute call.
+  pub id: Uuid,
   pub user: User,
   pub update: Update,
 }
@@ -59,7 +66,7 @@ pub struct ExecuteArgs {
 #[derive(
   Serialize, Deserialize, Debug, Clone, Resolve, EnumVariants,
 )]
-#[variant_derive(Debug)]
+#[variant_derive(Debug, Display)]
 #[args(ExecuteArgs)]
 #[response(JsonString)]
 #[error(serror::Error)]
@@ -150,6 +157,7 @@ pub enum ExecuteRequest {
   BackupCoreDatabase(BackupCoreDatabase),
   GlobalAutoUpdate(GlobalAutoUpdate),
   RotateAllServerKeys(RotateAllServerKeys),
+  RotateCoreKeys(RotateCoreKeys),
 }
 
 pub fn router() -> Router {
@@ -202,7 +210,7 @@ pub fn inner_handler(
   >,
 > {
   Box::pin(async move {
-    let req_id = Uuid::new_v4();
+    let task_id = Uuid::new_v4();
 
     // Need to validate no cancel is active before any update is created.
     // This ensures no double update created if Cancel is called more than once for the same request.
@@ -218,14 +226,14 @@ pub fn inner_handler(
     // here either.
     if update.operation == Operation::None {
       return Ok(ExecutionResult::Batch(
-        task(req_id, request, user, update).await?,
+        task(task_id, request, user, update).await?,
       ));
     }
 
     // Spawn a task for the execution which continues
     // running after this method returns.
     let handle =
-      tokio::spawn(task(req_id, request, user, update.clone()));
+      tokio::spawn(task(task_id, request, user, update.clone()));
 
     // Spawns another task to monitor the first for failures,
     // and add the log to Update about it (which primary task can't do because it errored out)
@@ -234,11 +242,11 @@ pub fn inner_handler(
       async move {
         let log = match handle.await {
           Ok(Err(e)) => {
-            warn!("/execute request {req_id} task error: {e:#}",);
+            warn!("/execute request {task_id} task error: {e:#}",);
             Log::error("Task Error", format_serror(&e.into()))
           }
           Err(e) => {
-            warn!("/execute request {req_id} spawn error: {e:?}",);
+            warn!("/execute request {task_id} spawn error: {e:?}",);
             Log::error("Spawn Error", format!("{e:#?}"))
           }
           _ => return,
@@ -272,40 +280,33 @@ pub fn inner_handler(
   })
 }
 
-#[instrument(
-  name = "ExecuteRequest",
-  skip(user, update),
-  fields(
-    user_id = user.id,
-    update_id = update.id,
-    request = format!("{:?}", request.extract_variant()))
-  )
-]
 async fn task(
-  req_id: Uuid,
+  id: Uuid,
   request: ExecuteRequest,
   user: User,
   update: Update,
 ) -> anyhow::Result<String> {
-  info!("/execute request {req_id} | user: {}", user.username);
-  let timer = Instant::now();
+  let variant = request.extract_variant();
 
-  let res = match request.resolve(&ExecuteArgs { user, update }).await
-  {
-    Err(e) => Err(e.error),
-    Ok(JsonString::Err(e)) => Err(
-      anyhow::Error::from(e).context("failed to serialize response"),
-    ),
-    Ok(JsonString::Ok(res)) => Ok(res),
-  };
+  info!(
+    "/execute request {id} | {variant} | user: {}",
+    user.username
+  );
+
+  let res =
+    match request.resolve(&ExecuteArgs { user, update, id }).await {
+      Err(e) => Err(e.error),
+      Ok(JsonString::Err(e)) => Err(
+        anyhow::Error::from(e)
+          .context("failed to serialize response"),
+      ),
+      Ok(JsonString::Ok(res)) => Ok(res),
+    };
 
   if let Err(e) = &res {
-    warn!("/execute request {req_id} error: {e:#}");
+    warn!("/execute request {id} error: {e:#}");
   }
 
-  let elapsed = timer.elapsed();
-  debug!("/execute request {req_id} | resolve time: {elapsed:?}");
-
   res
 }
 
@@ -314,6 +315,7 @@ trait BatchExecute {
   fn single_request(name: String) -> ExecuteRequest;
 }
 
+#[instrument("BatchExecute", skip(user))]
 async fn batch_execute<E: BatchExecute>(
   pattern: &str,
   user: &User,
@@ -322,10 +324,32 @@ async fn batch_execute<E: BatchExecute>(
     pattern,
     Default::default(),
     user,
-    PermissionLevel::Execute.into(),
     &[],
   )
   .await?;
+
+  let resources = if user.admin {
+    resources
+  } else {
+    // Only keep resources with execute permissions
+    resources
+      .into_iter()
+      .map(|resource| async move {
+        get_check_permissions::<E::Resource>(
+          &resource.id,
+          user,
+          PermissionLevel::Execute.into(),
+        )
+        .await
+      })
+      .collect::<FuturesUnordered<_>>()
+      .collect::<Vec<_>>()
+      .await
+      .into_iter()
+      .flatten()
+      .collect()
+  };
+
   let futures = resources.into_iter().map(|resource| {
     let user = user.clone();
     async move {

bin/core/src/api/execute/procedure.rs

@@ -38,7 +38,11 @@ impl super::BatchExecute for BatchRunProcedure {
 }
 
 impl Resolve<ExecuteArgs> for BatchRunProcedure {
-  #[instrument(name = "BatchRunProcedure", skip(user), fields(user_id = user.id))]
+  #[instrument(
+    "BatchRunProcedure",
+    skip_all,
+    fields(operator = user.id)
+  )]
   async fn resolve(
     self,
     ExecuteArgs { user, .. }: &ExecuteArgs,
@@ -51,10 +55,19 @@ impl Resolve<ExecuteArgs> for BatchRunProcedure {
 }
 
 impl Resolve<ExecuteArgs> for RunProcedure {
-  #[instrument(name = "RunProcedure", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+  #[instrument(
+    "RunProcedure",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      procedure = self.procedure,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
     Ok(
       resolve_inner(self.procedure, user.clone(), update.clone())
@@ -146,7 +159,6 @@ fn resolve_inner(
     update_update(update.clone()).await?;
 
     if !update.success && procedure.config.failure_alert {
-      warn!("procedure unsuccessful, alerting...");
       let target = update.target.clone();
       tokio::spawn(async move {
         let alert = Alert {

bin/core/src/api/execute/repo.rs

@@ -30,7 +30,7 @@ use crate::{
   alert::send_alerts,
   api::write::WriteArgs,
   helpers::{
-    builder::{cleanup_builder_instance, get_builder_periphery},
+    builder::{cleanup_builder_instance, connect_builder_periphery},
     channel::repo_cancel_channel,
     git_token, periphery_client,
     query::{VariablesAndSecrets, get_variables_and_secrets},
@@ -51,10 +51,18 @@ impl super::BatchExecute for BatchCloneRepo {
 }
 
 impl Resolve<ExecuteArgs> for BatchCloneRepo {
-  #[instrument(name = "BatchCloneRepo", skip( user), fields(user_id = user.id))]
+  #[instrument(
+    "BatchCloneRepo",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      pattern = self.pattern,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, id, .. }: &ExecuteArgs,
   ) -> serror::Result<BatchExecutionResponse> {
     Ok(
       super::batch_execute::<BatchCloneRepo>(&self.pattern, user)
@@ -64,10 +72,19 @@ impl Resolve<ExecuteArgs> for BatchCloneRepo {
 }
 
 impl Resolve<ExecuteArgs> for CloneRepo {
-  #[instrument(name = "CloneRepo", skip( user, update), fields(user_id = user.id, update_id = update.id))]
+  #[instrument(
+    "CloneRepo",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      repo = self.repo,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
     let mut repo = get_check_permissions::<Repo>(
       &self.repo,
@@ -165,10 +182,18 @@ impl super::BatchExecute for BatchPullRepo {
 }
 
 impl Resolve<ExecuteArgs> for BatchPullRepo {
-  #[instrument(name = "BatchPullRepo", skip(user), fields(user_id = user.id))]
+  #[instrument(
+    "BatchPullRepo",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      pattern = self.pattern
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, .. }: &ExecuteArgs,
+    ExecuteArgs { user, id, .. }: &ExecuteArgs,
   ) -> serror::Result<BatchExecutionResponse> {
     Ok(
       super::batch_execute::<BatchPullRepo>(&self.pattern, user)
@@ -178,10 +203,19 @@ impl Resolve<ExecuteArgs> for BatchPullRepo {
 }
 
 impl Resolve<ExecuteArgs> for PullRepo {
-  #[instrument(name = "PullRepo", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+  #[instrument(
+    "PullRepo",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      repo = self.repo,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
     let mut repo = get_check_permissions::<Repo>(
       &self.repo,
@@ -275,7 +309,11 @@ impl Resolve<ExecuteArgs> for PullRepo {
   }
 }
 
-#[instrument(skip_all, fields(update_id = update.id))]
+#[instrument(
+  "HandleRepoEarlyReturn",
+  skip_all,
+  fields(update_id = update.id)
+)]
 async fn handle_repo_update_return(
   update: Update,
 ) -> serror::Result<Update> {
@@ -297,7 +335,7 @@ async fn handle_repo_update_return(
   Ok(update)
 }
 
-#[instrument]
+#[instrument("UpdateLastPulledTime")]
 async fn update_last_pulled_time(repo_name: &str) {
   let res = db_client()
     .repos
@@ -321,10 +359,18 @@ impl super::BatchExecute for BatchBuildRepo {
 }
 
 impl Resolve<ExecuteArgs> for BatchBuildRepo {
-  #[instrument(name = "BatchBuildRepo", skip(user), fields(user_id = user.id))]
+  #[instrument(
+    "BatchBuildRepo",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      pattern = self.pattern,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, .. }: &ExecuteArgs,
+    ExecuteArgs { user, id, .. }: &ExecuteArgs,
   ) -> serror::Result<BatchExecutionResponse> {
     Ok(
       super::batch_execute::<BatchBuildRepo>(&self.pattern, user)
@@ -334,10 +380,19 @@ impl Resolve<ExecuteArgs> for BatchBuildRepo {
 }
 
 impl Resolve<ExecuteArgs> for BuildRepo {
-  #[instrument(name = "BuildRepo", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+  #[instrument(
+    "BuildRepo",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      repo = self.repo,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
     let mut repo = get_check_permissions::<Repo>(
       &self.repo,
@@ -419,7 +474,7 @@ impl Resolve<ExecuteArgs> for BuildRepo {
 
     // GET BUILDER PERIPHERY
 
-    let (periphery, cleanup_data) = match get_builder_periphery(
+    let (periphery, cleanup_data) = match connect_builder_periphery(
       repo.name.clone(),
       None,
       builder,
@@ -531,7 +586,6 @@ impl Resolve<ExecuteArgs> for BuildRepo {
     update_update(update.clone()).await?;
 
     if !update.success {
-      warn!("repo build unsuccessful, alerting...");
       let target = update.target.clone();
       tokio::spawn(async move {
         let alert = Alert {
@@ -554,7 +608,7 @@ impl Resolve<ExecuteArgs> for BuildRepo {
   }
 }
 
-#[instrument(skip(update))]
+#[instrument("HandleRepoBuildEarlyReturn", skip(update))]
 async fn handle_builder_early_return(
   mut update: Update,
   repo_id: String,
@@ -578,7 +632,6 @@ async fn handle_builder_early_return(
   }
   update_update(update.clone()).await?;
   if !update.success && !is_cancel {
-    warn!("repo build unsuccessful, alerting...");
     let target = update.target.clone();
     tokio::spawn(async move {
       let alert = Alert {
@@ -599,7 +652,6 @@ async fn handle_builder_early_return(
   Ok(update)
 }
 
-#[instrument(skip_all)]
 pub async fn validate_cancel_repo_build(
   request: &ExecuteRequest,
 ) -> anyhow::Result<()> {
@@ -649,10 +701,19 @@ pub async fn validate_cancel_repo_build(
 }
 
 impl Resolve<ExecuteArgs> for CancelRepoBuild {
-  #[instrument(name = "CancelRepoBuild", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+  #[instrument(
+    "CancelRepoBuild",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      repo = self.repo,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
     let repo = get_check_permissions::<Repo>(
       &self.repo,
@@ -709,6 +770,13 @@ impl Resolve<ExecuteArgs> for CancelRepoBuild {
   }
 }
 
+#[instrument(
+  "Interpolate",
+  skip_all,
+  fields(
+    skip_secret_interp = repo.config.skip_secret_interp
+  )
+)]
 async fn interpolate(
   repo: &mut Repo,
   update: &mut Update,

bin/core/src/api/execute/server.rs

@@ -22,10 +22,20 @@ use crate::{
 use super::ExecuteArgs;
 
 impl Resolve<ExecuteArgs> for StartContainer {
-  #[instrument(name = "StartContainer", skip(self, user, update), fields(user_id = user.id, update_id = update.id))]
+  #[instrument(
+    "StartContainer",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      server = self.server,
+      container = self.container,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
     let server = get_check_permissions::<Server>(
       &self.server,
@@ -76,10 +86,20 @@ impl Resolve<ExecuteArgs> for StartContainer {
 }
 
 impl Resolve<ExecuteArgs> for RestartContainer {
-  #[instrument(name = "RestartContainer", skip(self, user, update), fields(user_id = user.id, update_id = update.id))]
+  #[instrument(
+    "RestartContainer",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      server = self.server,
+      container = self.container,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
     let server = get_check_permissions::<Server>(
       &self.server,
@@ -132,10 +152,20 @@ impl Resolve<ExecuteArgs> for RestartContainer {
 }
 
 impl Resolve<ExecuteArgs> for PauseContainer {
-  #[instrument(name = "PauseContainer", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+  #[instrument(
+    "PauseContainer",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      server = self.server,
+      container = self.container,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
     let server = get_check_permissions::<Server>(
       &self.server,
@@ -186,10 +216,20 @@ impl Resolve<ExecuteArgs> for PauseContainer {
 }
 
 impl Resolve<ExecuteArgs> for UnpauseContainer {
-  #[instrument(name = "UnpauseContainer", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+  #[instrument(
+    "UnpauseContainer",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      server = self.server,
+      container = self.container,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
     let server = get_check_permissions::<Server>(
       &self.server,
@@ -242,10 +282,22 @@ impl Resolve<ExecuteArgs> for UnpauseContainer {
 }
 
 impl Resolve<ExecuteArgs> for StopContainer {
-  #[instrument(name = "StopContainer", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+  #[instrument(
+    "StopContainer",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      server = self.server,
+      container = self.container,
+      signal = format!("{:?}", self.signal),
+      time = self.time,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
     let server = get_check_permissions::<Server>(
       &self.server,
@@ -298,10 +350,22 @@ impl Resolve<ExecuteArgs> for StopContainer {
 }
 
 impl Resolve<ExecuteArgs> for DestroyContainer {
-  #[instrument(name = "DestroyContainer", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+  #[instrument(
+    "DestroyContainer",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      server = self.server,
+      container = self.container,
+      signal = format!("{:?}", self.signal),
+      time = self.time,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
     let DestroyContainer {
       server,
@@ -360,10 +424,19 @@ impl Resolve<ExecuteArgs> for DestroyContainer {
 }
 
 impl Resolve<ExecuteArgs> for StartAllContainers {
-  #[instrument(name = "StartAllContainers", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+  #[instrument(
+    "StartAllContainers",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      server = self.server,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
     let server = get_check_permissions::<Server>(
       &self.server,
@@ -411,10 +484,19 @@ impl Resolve<ExecuteArgs> for StartAllContainers {
 }
 
 impl Resolve<ExecuteArgs> for RestartAllContainers {
-  #[instrument(name = "RestartAllContainers", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+  #[instrument(
+    "RestartAllContainers",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      server = self.server,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
     let server = get_check_permissions::<Server>(
       &self.server,
@@ -464,10 +546,19 @@ impl Resolve<ExecuteArgs> for RestartAllContainers {
 }
 
 impl Resolve<ExecuteArgs> for PauseAllContainers {
-  #[instrument(name = "PauseAllContainers", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+  #[instrument(
+    "PauseAllContainers",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      server = self.server,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
     let server = get_check_permissions::<Server>(
       &self.server,
@@ -515,10 +606,19 @@ impl Resolve<ExecuteArgs> for PauseAllContainers {
 }
 
 impl Resolve<ExecuteArgs> for UnpauseAllContainers {
-  #[instrument(name = "UnpauseAllContainers", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+  #[instrument(
+    "UnpauseAllContainers",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      server = self.server,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
     let server = get_check_permissions::<Server>(
       &self.server,
@@ -568,10 +668,19 @@ impl Resolve<ExecuteArgs> for UnpauseAllContainers {
 }
 
 impl Resolve<ExecuteArgs> for StopAllContainers {
-  #[instrument(name = "StopAllContainers", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+  #[instrument(
+    "StopAllContainers",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      server = self.server,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
     let server = get_check_permissions::<Server>(
       &self.server,
@@ -619,10 +728,19 @@ impl Resolve<ExecuteArgs> for StopAllContainers {
 }
 
 impl Resolve<ExecuteArgs> for PruneContainers {
-  #[instrument(name = "PruneContainers", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+  #[instrument(
+    "PruneContainers",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      server = self.server,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
     let server = get_check_permissions::<Server>(
       &self.server,
@@ -675,10 +793,20 @@ impl Resolve<ExecuteArgs> for PruneContainers {
 }
 
 impl Resolve<ExecuteArgs> for DeleteNetwork {
-  #[instrument(name = "DeleteNetwork", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+  #[instrument(
+    "DeleteNetwork",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      server = self.server,
+      network = self.name
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
     let server = get_check_permissions::<Server>(
       &self.server,
@@ -726,10 +854,19 @@ impl Resolve<ExecuteArgs> for DeleteNetwork {
 }
 
 impl Resolve<ExecuteArgs> for PruneNetworks {
-  #[instrument(name = "PruneNetworks", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+  #[instrument(
+    "PruneNetworks",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      server = self.server,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
     let server = get_check_permissions::<Server>(
       &self.server,
@@ -780,10 +917,20 @@ impl Resolve<ExecuteArgs> for PruneNetworks {
 }
 
 impl Resolve<ExecuteArgs> for DeleteImage {
-  #[instrument(name = "DeleteImage", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+  #[instrument(
+    "DeleteImage",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      server = self.server,
+      image = self.name,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
     let server = get_check_permissions::<Server>(
       &self.server,
@@ -828,10 +975,19 @@ impl Resolve<ExecuteArgs> for DeleteImage {
 }
 
 impl Resolve<ExecuteArgs> for PruneImages {
-  #[instrument(name = "PruneImages", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+  #[instrument(
+    "PruneImages",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      server = self.server,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
     let server = get_check_permissions::<Server>(
       &self.server,
@@ -880,10 +1036,20 @@ impl Resolve<ExecuteArgs> for PruneImages {
 }
 
 impl Resolve<ExecuteArgs> for DeleteVolume {
-  #[instrument(name = "DeleteVolume", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+  #[instrument(
+    "DeleteVolume",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      server = self.server,
+      volume = self.name,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
     let server = get_check_permissions::<Server>(
       &self.server,
@@ -931,10 +1097,19 @@ impl Resolve<ExecuteArgs> for DeleteVolume {
 }
 
 impl Resolve<ExecuteArgs> for PruneVolumes {
-  #[instrument(name = "PruneVolumes", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+  #[instrument(
+    "PruneVolumes",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      server = self.server,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
     let server = get_check_permissions::<Server>(
       &self.server,
@@ -983,10 +1158,19 @@ impl Resolve<ExecuteArgs> for PruneVolumes {
 }
 
 impl Resolve<ExecuteArgs> for PruneDockerBuilders {
-  #[instrument(name = "PruneDockerBuilders", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+  #[instrument(
+    "PruneDockerBuilders",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      server = self.server,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
     let server = get_check_permissions::<Server>(
       &self.server,
@@ -1035,10 +1219,19 @@ impl Resolve<ExecuteArgs> for PruneDockerBuilders {
 }
 
 impl Resolve<ExecuteArgs> for PruneBuildx {
-  #[instrument(name = "PruneBuildx", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+  #[instrument(
+    "PruneBuildx",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      server = self.server,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
     let server = get_check_permissions::<Server>(
       &self.server,
@@ -1087,10 +1280,19 @@ impl Resolve<ExecuteArgs> for PruneBuildx {
 }
 
 impl Resolve<ExecuteArgs> for PruneSystem {
-  #[instrument(name = "PruneSystem", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+  #[instrument(
+    "PruneSystem",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      server = self.server,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
     let server = get_check_permissions::<Server>(
       &self.server,

bin/core/src/api/execute/stack.rs

@@ -22,6 +22,7 @@ use komodo_client::{
 };
 use periphery_client::api::compose::*;
 use resolver_api::Resolve;
+use uuid::Uuid;
 
 use crate::{
   api::write::WriteArgs,
@@ -54,10 +55,18 @@ impl super::BatchExecute for BatchDeployStack {
 }
 
 impl Resolve<ExecuteArgs> for BatchDeployStack {
-  #[instrument(name = "BatchDeployStack", skip(user), fields(user_id = user.id))]
+  #[instrument(
+    "BatchDeployStack",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      pattern = self.pattern,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, .. }: &ExecuteArgs,
+    ExecuteArgs { user, id, .. }: &ExecuteArgs,
   ) -> serror::Result<BatchExecutionResponse> {
     Ok(
       super::batch_execute::<BatchDeployStack>(&self.pattern, user)
@@ -67,10 +76,21 @@ impl Resolve<ExecuteArgs> for BatchDeployStack {
 }
 
 impl Resolve<ExecuteArgs> for DeployStack {
-  #[instrument(name = "DeployStack", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+  #[instrument(
+    "DeployStack",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      stack = self.stack,
+      services = format!("{:?}", self.services),
+      stop_time = self.stop_time,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
     let (mut stack, server) = get_stack_and_server(
       &self.stack,
@@ -281,10 +301,18 @@ impl super::BatchExecute for BatchDeployStackIfChanged {
 }
 
 impl Resolve<ExecuteArgs> for BatchDeployStackIfChanged {
-  #[instrument(name = "BatchDeployStackIfChanged", skip(user), fields(user_id = user.id))]
+  #[instrument(
+    "BatchDeployStackIfChanged",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      pattern = self.pattern,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, .. }: &ExecuteArgs,
+    ExecuteArgs { user, id, .. }: &ExecuteArgs,
   ) -> serror::Result<BatchExecutionResponse> {
     Ok(
       super::batch_execute::<BatchDeployStackIfChanged>(
@@ -297,10 +325,20 @@ impl Resolve<ExecuteArgs> for BatchDeployStackIfChanged {
 }
 
 impl Resolve<ExecuteArgs> for DeployStackIfChanged {
-  #[instrument(name = "DeployStackIfChanged", skip(user, update), fields(user_id = user.id))]
+  #[instrument(
+    "DeployStackIfChanged",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      stack = self.stack,
+      stop_time = self.stop_time,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
     let stack = get_check_permissions::<Stack>(
       &self.stack,
@@ -358,6 +396,7 @@ impl Resolve<ExecuteArgs> for DeployStackIfChanged {
         .resolve(&ExecuteArgs {
           user: user.clone(),
           update,
+          id: *id,
         })
         .await
       }
@@ -467,6 +506,14 @@ impl Resolve<ExecuteArgs> for DeployStackIfChanged {
   }
 }
 
+#[instrument(
+  "DeployStackServices",
+  skip_all,
+  fields(
+    stack = stack,
+    services = format!("{services:?}")
+  )
+)]
 async fn deploy_services(
   stack: String,
   services: Vec<String>,
@@ -488,10 +535,19 @@ async fn deploy_services(
     .resolve(&ExecuteArgs {
       user: user.clone(),
       update,
+      id: Uuid::new_v4(),
     })
     .await
 }
 
+#[instrument(
+  "RestartStackServices",
+  skip_all,
+  fields(
+    stack = stack,
+    services = format!("{services:?}")
+  )
+)]
 async fn restart_services(
   stack: String,
   services: Vec<String>,
@@ -510,6 +566,7 @@ async fn restart_services(
     .resolve(&ExecuteArgs {
       user: user.clone(),
       update,
+      id: Uuid::new_v4(),
     })
     .await
 }
@@ -526,6 +583,11 @@ async fn restart_services(
 /// Changes to config files after restart is applied should
 /// be taken as the deployed contents, otherwise next changed check
 /// will restart service again for no reason.
+#[instrument(
+  "UpdateStackDeployedContents",
+  skip_all,
+  fields(stack = id)
+)]
 async fn update_deployed_contents_with_latest(
   id: &str,
   contents: Option<Vec<StackRemoteFileContents>>,
@@ -663,10 +725,18 @@ impl super::BatchExecute for BatchPullStack {
 }
 
 impl Resolve<ExecuteArgs> for BatchPullStack {
-  #[instrument(name = "BatchPullStack", skip(user), fields(user_id = user.id))]
+  #[instrument(
+    "BatchPullStack",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      pattern = self.pattern,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, .. }: &ExecuteArgs,
+    ExecuteArgs { user, id, .. }: &ExecuteArgs,
   ) -> serror::Result<BatchExecutionResponse> {
     Ok(
       super::batch_execute::<BatchPullStack>(&self.pattern, user)
@@ -700,6 +770,14 @@ async fn maybe_pull_stack(
   Ok(())
 }
 
+#[instrument(
+  "PullStackInner",
+  skip_all,
+  fields(
+    stack = stack.id,
+    services = format!("{services:?}"),
+  )
+)]
 pub async fn pull_stack_inner(
   mut stack: Stack,
   services: Vec<String>,
@@ -769,10 +847,20 @@ pub async fn pull_stack_inner(
 }
 
 impl Resolve<ExecuteArgs> for PullStack {
-  #[instrument(name = "PullStack", skip(user, update), fields(user_id = user.id))]
+  #[instrument(
+    "PullStack",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      stack = self.stack,
+      services = format!("{:?}", self.services),
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
     let (stack, server) = get_stack_and_server(
       &self.stack,
@@ -822,10 +910,20 @@ impl Resolve<ExecuteArgs> for PullStack {
 }
 
 impl Resolve<ExecuteArgs> for StartStack {
-  #[instrument(name = "StartStack", skip(user, update), fields(user_id = user.id))]
+  #[instrument(
+    "StartStack",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      stack = self.stack,
+      services = format!("{:?}", self.services),
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
     execute_compose::<StartStack>(
       &self.stack,
@@ -841,10 +939,20 @@ impl Resolve<ExecuteArgs> for StartStack {
 }
 
 impl Resolve<ExecuteArgs> for RestartStack {
-  #[instrument(name = "RestartStack", skip(user, update), fields(user_id = user.id))]
+  #[instrument(
+    "RestartStack",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      stack = self.stack,
+      services = format!("{:?}", self.services),
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
     execute_compose::<RestartStack>(
       &self.stack,
@@ -862,10 +970,20 @@ impl Resolve<ExecuteArgs> for RestartStack {
 }
 
 impl Resolve<ExecuteArgs> for PauseStack {
-  #[instrument(name = "PauseStack", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+  #[instrument(
+    "PauseStack",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      stack = self.stack,
+      services = format!("{:?}", self.services),
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
     execute_compose::<PauseStack>(
       &self.stack,
@@ -881,10 +999,20 @@ impl Resolve<ExecuteArgs> for PauseStack {
 }
 
 impl Resolve<ExecuteArgs> for UnpauseStack {
-  #[instrument(name = "UnpauseStack", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+  #[instrument(
+    "UnpauseStack",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      stack = self.stack,
+      services = format!("{:?}", self.services),
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
     execute_compose::<UnpauseStack>(
       &self.stack,
@@ -900,10 +1028,20 @@ impl Resolve<ExecuteArgs> for UnpauseStack {
 }
 
 impl Resolve<ExecuteArgs> for StopStack {
-  #[instrument(name = "StopStack", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+  #[instrument(
+    "StopStack",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      stack = self.stack,
+      services = format!("{:?}", self.services),
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
     execute_compose::<StopStack>(
       &self.stack,
@@ -931,10 +1069,18 @@ impl super::BatchExecute for BatchDestroyStack {
 }
 
 impl Resolve<ExecuteArgs> for BatchDestroyStack {
-  #[instrument(name = "BatchDestroyStack", skip(user), fields(user_id = user.id))]
+  #[instrument(
+    "BatchDestroyStack",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      pattern = self.pattern,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, .. }: &ExecuteArgs,
+    ExecuteArgs { user, id, .. }: &ExecuteArgs,
   ) -> serror::Result<BatchExecutionResponse> {
     super::batch_execute::<BatchDestroyStack>(&self.pattern, user)
       .await
@@ -943,10 +1089,22 @@ impl Resolve<ExecuteArgs> for BatchDestroyStack {
 }
 
 impl Resolve<ExecuteArgs> for DestroyStack {
-  #[instrument(name = "DestroyStack", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+  #[instrument(
+    "DestroyStack",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      stack = self.stack,
+      services = format!("{:?}", self.services),
+      remove_orphans = self.remove_orphans,
+      stop_time = self.stop_time,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
     execute_compose::<DestroyStack>(
       &self.stack,
@@ -962,10 +1120,21 @@ impl Resolve<ExecuteArgs> for DestroyStack {
 }
 
 impl Resolve<ExecuteArgs> for RunStackService {
-  #[instrument(name = "RunStackService", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+  #[instrument(
+    "RunStackService",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      stack = self.stack,
+      service = self.service,
+      request = format!("{self:?}"),
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
     let (mut stack, server) = get_stack_and_server(
       &self.stack,

bin/core/src/api/execute/sync.rs

@@ -49,10 +49,21 @@ use crate::{
 use super::ExecuteArgs;
 
 impl Resolve<ExecuteArgs> for RunSync {
-  #[instrument(name = "RunSync", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+  #[instrument(
+    "RunSync",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      sync = self.sync,
+      resource_type = format!("{:?}", self.resource_type),
+      resources = format!("{:?}", self.resources),
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
     let RunSync {
       sync,

bin/core/src/api/read/action.rs

@@ -46,13 +46,8 @@ impl Resolve<ReadArgs> for ListActions {
       get_all_tags(None).await?
     };
     Ok(
-      resource::list_for_user::<Action>(
-        self.query,
-        user,
-        PermissionLevel::Read.into(),
-        &all_tags,
-      )
-      .await?,
+      resource::list_for_user::<Action>(self.query, user, &all_tags)
+        .await?,
     )
   }
 }
@@ -69,10 +64,7 @@ impl Resolve<ReadArgs> for ListFullActions {
     };
     Ok(
       resource::list_full_for_user::<Action>(
-        self.query,
-        user,
-        PermissionLevel::Read.into(),
-        &all_tags,
+        self.query, user, &all_tags,
       )
       .await?,
     )
@@ -108,7 +100,6 @@ impl Resolve<ReadArgs> for GetActionsSummary {
     let actions = resource::list_full_for_user::<Action>(
       Default::default(),
       user,
-      PermissionLevel::Read.into(),
       &[],
     )
     .await

bin/core/src/api/read/alerter.rs

@@ -44,13 +44,8 @@ impl Resolve<ReadArgs> for ListAlerters {
       get_all_tags(None).await?
     };
     Ok(
-      resource::list_for_user::<Alerter>(
-        self.query,
-        user,
-        PermissionLevel::Read.into(),
-        &all_tags,
-      )
-      .await?,
+      resource::list_for_user::<Alerter>(self.query, user, &all_tags)
+        .await?,
     )
   }
 }
@@ -67,10 +62,7 @@ impl Resolve<ReadArgs> for ListFullAlerters {
     };
     Ok(
       resource::list_full_for_user::<Alerter>(
-        self.query,
-        user,
-        PermissionLevel::Read.into(),
-        &all_tags,
+        self.query, user, &all_tags,
       )
       .await?,
     )

bin/core/src/api/read/build.rs

@@ -12,7 +12,6 @@ use komodo_client::{
   entities::{
     Operation,
     build::{Build, BuildActionState, BuildListItem, BuildState},
-    config::core::CoreConfig,
     permission::PermissionLevel,
     update::UpdateStatus,
   },
@@ -20,13 +19,10 @@ use komodo_client::{
 use resolver_api::Resolve;
 
 use crate::{
-  config::core_config,
   helpers::query::get_all_tags,
   permission::get_check_permissions,
   resource,
-  state::{
-    action_states, build_state_cache, db_client, github_client,
-  },
+  state::{action_states, build_state_cache, db_client},
 };
 
 use super::ReadArgs;
@@ -58,13 +54,8 @@ impl Resolve<ReadArgs> for ListBuilds {
       get_all_tags(None).await?
     };
     Ok(
-      resource::list_for_user::<Build>(
-        self.query,
-        user,
-        PermissionLevel::Read.into(),
-        &all_tags,
-      )
-      .await?,
+      resource::list_for_user::<Build>(self.query, user, &all_tags)
+        .await?,
     )
   }
 }
@@ -81,10 +72,7 @@ impl Resolve<ReadArgs> for ListFullBuilds {
     };
     Ok(
       resource::list_full_for_user::<Build>(
-        self.query,
-        user,
-        PermissionLevel::Read.into(),
-        &all_tags,
+        self.query, user, &all_tags,
       )
       .await?,
     )
@@ -120,7 +108,6 @@ impl Resolve<ReadArgs> for GetBuildsSummary {
     let builds = resource::list_full_for_user::<Build>(
       Default::default(),
       user,
-      PermissionLevel::Read.into(),
       &[],
     )
     .await
@@ -284,10 +271,7 @@ impl Resolve<ReadArgs> for ListCommonBuildExtraArgs {
       get_all_tags(None).await?
     };
     let builds = resource::list_full_for_user::<Build>(
-      self.query,
-      user,
-      PermissionLevel::Read.into(),
-      &all_tags,
+      self.query, user, &all_tags,
     )
     .await
     .context("failed to get resources matching query")?;
@@ -306,81 +290,3 @@ impl Resolve<ReadArgs> for ListCommonBuildExtraArgs {
     Ok(res)
   }
 }
-
-impl Resolve<ReadArgs> for GetBuildWebhookEnabled {
-  async fn resolve(
-    self,
-    ReadArgs { user }: &ReadArgs,
-  ) -> serror::Result<GetBuildWebhookEnabledResponse> {
-    let Some(github) = github_client() else {
-      return Ok(GetBuildWebhookEnabledResponse {
-        managed: false,
-        enabled: false,
-      });
-    };
-
-    let build = get_check_permissions::<Build>(
-      &self.build,
-      user,
-      PermissionLevel::Read.into(),
-    )
-    .await?;
-
-    if build.config.git_provider != "github.com"
-      || build.config.repo.is_empty()
-    {
-      return Ok(GetBuildWebhookEnabledResponse {
-        managed: false,
-        enabled: false,
-      });
-    }
-
-    let mut split = build.config.repo.split('/');
-    let owner = split.next().context("Build repo has no owner")?;
-
-    let Some(github) = github.get(owner) else {
-      return Ok(GetBuildWebhookEnabledResponse {
-        managed: false,
-        enabled: false,
-      });
-    };
-
-    let repo =
-      split.next().context("Build repo has no repo after the /")?;
-
-    let github_repos = github.repos();
-
-    let webhooks = github_repos
-      .list_all_webhooks(owner, repo)
-      .await
-      .context("failed to list all webhooks on repo")?
-      .body;
-
-    let CoreConfig {
-      host,
-      webhook_base_url,
-      ..
-    } = core_config();
-
-    let host = if webhook_base_url.is_empty() {
-      host
-    } else {
-      webhook_base_url
-    };
-    let url = format!("{host}/listener/github/build/{}", build.id);
-
-    for webhook in webhooks {
-      if webhook.active && webhook.config.url == url {
-        return Ok(GetBuildWebhookEnabledResponse {
-          managed: true,
-          enabled: true,
-        });
-      }
-    }
-
-    Ok(GetBuildWebhookEnabledResponse {
-      managed: true,
-      enabled: false,
-    })
-  }
-}

bin/core/src/api/read/builder.rs

@@ -44,13 +44,8 @@ impl Resolve<ReadArgs> for ListBuilders {
       get_all_tags(None).await?
     };
     Ok(
-      resource::list_for_user::<Builder>(
-        self.query,
-        user,
-        PermissionLevel::Read.into(),
-        &all_tags,
-      )
-      .await?,
+      resource::list_for_user::<Builder>(self.query, user, &all_tags)
+        .await?,
     )
   }
 }
@@ -67,10 +62,7 @@ impl Resolve<ReadArgs> for ListFullBuilders {
     };
     Ok(
       resource::list_full_for_user::<Builder>(
-        self.query,
-        user,
-        PermissionLevel::Read.into(),
-        &all_tags,
+        self.query, user, &all_tags,
       )
       .await?,
     )

bin/core/src/api/read/deployment.rs

@@ -56,10 +56,7 @@ impl Resolve<ReadArgs> for ListDeployments {
     };
     let only_update_available = self.query.specific.update_available;
     let deployments = resource::list_for_user::<Deployment>(
-      self.query,
-      user,
-      PermissionLevel::Read.into(),
-      &all_tags,
+      self.query, user, &all_tags,
     )
     .await?;
     let deployments = if only_update_available {
@@ -86,10 +83,7 @@ impl Resolve<ReadArgs> for ListFullDeployments {
     };
     Ok(
       resource::list_full_for_user::<Deployment>(
-        self.query,
-        user,
-        PermissionLevel::Read.into(),
-        &all_tags,
+        self.query, user, &all_tags,
       )
       .await?,
     )
@@ -303,7 +297,6 @@ impl Resolve<ReadArgs> for GetDeploymentsSummary {
     let deployments = resource::list_full_for_user::<Deployment>(
       Default::default(),
       user,
-      PermissionLevel::Read.into(),
       &[],
     )
     .await
@@ -349,10 +342,7 @@ impl Resolve<ReadArgs> for ListCommonDeploymentExtraArgs {
       get_all_tags(None).await?
     };
     let deployments = resource::list_full_for_user::<Deployment>(
-      self.query,
-      user,
-      PermissionLevel::Read.into(),
-      &all_tags,
+      self.query, user, &all_tags,
     )
     .await
     .context("failed to get resources matching query")?;

bin/core/src/api/read/mod.rs

@@ -1,4 +1,4 @@
-use std::{collections::HashSet, sync::OnceLock, time::Instant};
+use std::{collections::HashSet, time::Instant};
 
 use anyhow::{Context, anyhow};
 use axum::{
@@ -11,7 +11,6 @@ use komodo_client::{
     build::Build,
     builder::{Builder, BuilderConfig},
     config::{DockerRegistry, GitProvider},
-    permission::PermissionLevel,
     repo::Repo,
     server::Server,
     sync::ResourceSync,
@@ -28,7 +27,7 @@ use uuid::Uuid;
 
 use crate::{
   auth::auth_request,
-  config::{core_config, core_public_key},
+  config::{core_config, core_keys},
   helpers::periphery_client,
   resource,
 };
@@ -142,7 +141,6 @@ enum ReadRequest {
   GetStacksSummary(GetStacksSummary),
   GetStack(GetStack),
   GetStackActionState(GetStackActionState),
-  GetStackWebhooksEnabled(GetStackWebhooksEnabled),
   GetStackLog(GetStackLog),
   SearchStackLog(SearchStackLog),
   InspectStackContainer(InspectStackContainer),
@@ -171,7 +169,6 @@ enum ReadRequest {
   GetBuildActionState(GetBuildActionState),
   GetBuildMonthlyStats(GetBuildMonthlyStats),
   ListBuildVersions(ListBuildVersions),
-  GetBuildWebhookEnabled(GetBuildWebhookEnabled),
   ListBuilds(ListBuilds),
   ListFullBuilds(ListFullBuilds),
   ListCommonBuildExtraArgs(ListCommonBuildExtraArgs),
@@ -180,7 +177,6 @@ enum ReadRequest {
   GetReposSummary(GetReposSummary),
   GetRepo(GetRepo),
   GetRepoActionState(GetRepoActionState),
-  GetRepoWebhooksEnabled(GetRepoWebhooksEnabled),
   ListRepos(ListRepos),
   ListFullRepos(ListFullRepos),
 
@@ -188,7 +184,6 @@ enum ReadRequest {
   GetResourceSyncsSummary(GetResourceSyncsSummary),
   GetResourceSync(GetResourceSync),
   GetResourceSyncActionState(GetResourceSyncActionState),
-  GetSyncWebhooksEnabled(GetSyncWebhooksEnabled),
   ListResourceSyncs(ListResourceSyncs),
   ListFullResourceSyncs(ListFullResourceSyncs),
 
@@ -253,7 +248,6 @@ async fn variant_handler(
   handler(user, Json(req)).await
 }
 
-#[instrument(name = "ReadHandler", level = "debug", skip(user), fields(user_id = user.id))]
 async fn handler(
   Extension(user): Extension<User>,
   Json(request): Json<ReadRequest>,
@@ -281,11 +275,13 @@ impl Resolve<ReadArgs> for GetVersion {
   }
 }
 
-fn core_info() -> &'static GetCoreInfoResponse {
-  static CORE_INFO: OnceLock<GetCoreInfoResponse> = OnceLock::new();
-  CORE_INFO.get_or_init(|| {
+impl Resolve<ReadArgs> for GetCoreInfo {
+  async fn resolve(
+    self,
+    _: &ReadArgs,
+  ) -> serror::Result<GetCoreInfoResponse> {
     let config = core_config();
-    GetCoreInfoResponse {
+    let info = GetCoreInfoResponse {
       title: config.title.clone(),
       monitoring_interval: config.monitoring_interval,
       webhook_base_url: if config.webhook_base_url.is_empty() {
@@ -299,24 +295,10 @@ fn core_info() -> &'static GetCoreInfoResponse {
       disable_non_admin_create: config.disable_non_admin_create,
       disable_websocket_reconnect: config.disable_websocket_reconnect,
       enable_fancy_toml: config.enable_fancy_toml,
-      github_webhook_owners: config
-        .github_webhook_app
-        .installations
-        .iter()
-        .map(|i| i.namespace.to_string())
-        .collect(),
       timezone: config.timezone.clone(),
-      public_key: core_public_key().to_string(),
-    }
-  })
-}
-
-impl Resolve<ReadArgs> for GetCoreInfo {
-  async fn resolve(
-    self,
-    _: &ReadArgs,
-  ) -> serror::Result<GetCoreInfoResponse> {
-    Ok(core_info().clone())
+      public_key: core_keys().load().public.to_string(),
+    };
+    Ok(info)
   }
 }
 
@@ -415,19 +397,16 @@ impl Resolve<ReadArgs> for ListGitProvidersFromConfig {
       resource::list_full_for_user::<Build>(
         Default::default(),
         user,
-        PermissionLevel::Read.into(),
         &[]
       ),
       resource::list_full_for_user::<Repo>(
         Default::default(),
         user,
-        PermissionLevel::Read.into(),
         &[]
       ),
       resource::list_full_for_user::<ResourceSync>(
         Default::default(),
         user,
-        PermissionLevel::Read.into(),
         &[]
       ),
     )?;

bin/core/src/api/read/onboarding_key.rs

@@ -1,3 +1,5 @@
+use std::cmp::Ordering;
+
 use anyhow::{Context, anyhow};
 use database::mungos::find::find_collect;
 use komodo_client::api::read::{
@@ -22,9 +24,28 @@ impl Resolve<ReadArgs> for ListOnboardingKeys {
           .status_code(StatusCode::FORBIDDEN),
       );
     }
-    find_collect(&db_client().onboarding_keys, None, None)
-      .await
-      .context("Failed to query database for Server onboarding keys")
-      .map_err(Into::into)
+
+    let mut keys =
+      find_collect(&db_client().onboarding_keys, None, None)
+        .await
+        .context(
+          "Failed to query database for Server onboarding keys",
+        )?;
+
+    // No expiry keys first, followed
+    keys.sort_by(|a, b| {
+      if a.expires == b.expires {
+        Ordering::Equal
+      } else if a.expires == 0 {
+        Ordering::Less
+      } else if b.expires == 0 {
+        Ordering::Greater
+      } else {
+        // Descending
+        b.expires.cmp(&a.expires)
+      }
+    });
+
+    Ok(keys)
   }
 }

bin/core/src/api/read/procedure.rs

@@ -45,10 +45,7 @@ impl Resolve<ReadArgs> for ListProcedures {
     };
     Ok(
       resource::list_for_user::<Procedure>(
-        self.query,
-        user,
-        PermissionLevel::Read.into(),
-        &all_tags,
+        self.query, user, &all_tags,
       )
       .await?,
     )
@@ -67,10 +64,7 @@ impl Resolve<ReadArgs> for ListFullProcedures {
     };
     Ok(
       resource::list_full_for_user::<Procedure>(
-        self.query,
-        user,
-        PermissionLevel::Read.into(),
-        &all_tags,
+        self.query, user, &all_tags,
       )
       .await?,
     )
@@ -85,7 +79,6 @@ impl Resolve<ReadArgs> for GetProceduresSummary {
     let procedures = resource::list_full_for_user::<Procedure>(
       Default::default(),
       user,
-      PermissionLevel::Read.into(),
       &[],
     )
     .await

bin/core/src/api/read/repo.rs

@@ -2,7 +2,6 @@ use anyhow::Context;
 use komodo_client::{
   api::read::*,
   entities::{
-    config::core::CoreConfig,
     permission::PermissionLevel,
     repo::{Repo, RepoActionState, RepoListItem, RepoState},
   },
@@ -10,11 +9,10 @@ use komodo_client::{
 use resolver_api::Resolve;
 
 use crate::{
-  config::core_config,
   helpers::query::get_all_tags,
   permission::get_check_permissions,
   resource,
-  state::{action_states, github_client, repo_state_cache},
+  state::{action_states, repo_state_cache},
 };
 
 use super::ReadArgs;
@@ -46,13 +44,8 @@ impl Resolve<ReadArgs> for ListRepos {
       get_all_tags(None).await?
     };
     Ok(
-      resource::list_for_user::<Repo>(
-        self.query,
-        user,
-        PermissionLevel::Read.into(),
-        &all_tags,
-      )
-      .await?,
+      resource::list_for_user::<Repo>(self.query, user, &all_tags)
+        .await?,
     )
   }
 }
@@ -69,10 +62,7 @@ impl Resolve<ReadArgs> for ListFullRepos {
     };
     Ok(
       resource::list_full_for_user::<Repo>(
-        self.query,
-        user,
-        PermissionLevel::Read.into(),
-        &all_tags,
+        self.query, user, &all_tags,
       )
       .await?,
     )
@@ -108,7 +98,6 @@ impl Resolve<ReadArgs> for GetReposSummary {
     let repos = resource::list_full_for_user::<Repo>(
       Default::default(),
       user,
-      PermissionLevel::Read.into(),
       &[],
     )
     .await
@@ -159,104 +148,3 @@ impl Resolve<ReadArgs> for GetReposSummary {
     Ok(res)
   }
 }
-
-impl Resolve<ReadArgs> for GetRepoWebhooksEnabled {
-  async fn resolve(
-    self,
-    ReadArgs { user }: &ReadArgs,
-  ) -> serror::Result<GetRepoWebhooksEnabledResponse> {
-    let Some(github) = github_client() else {
-      return Ok(GetRepoWebhooksEnabledResponse {
-        managed: false,
-        clone_enabled: false,
-        pull_enabled: false,
-        build_enabled: false,
-      });
-    };
-
-    let repo = get_check_permissions::<Repo>(
-      &self.repo,
-      user,
-      PermissionLevel::Read.into(),
-    )
-    .await?;
-
-    if repo.config.git_provider != "github.com"
-      || repo.config.repo.is_empty()
-    {
-      return Ok(GetRepoWebhooksEnabledResponse {
-        managed: false,
-        clone_enabled: false,
-        pull_enabled: false,
-        build_enabled: false,
-      });
-    }
-
-    let mut split = repo.config.repo.split('/');
-    let owner = split.next().context("Repo repo has no owner")?;
-
-    let Some(github) = github.get(owner) else {
-      return Ok(GetRepoWebhooksEnabledResponse {
-        managed: false,
-        clone_enabled: false,
-        pull_enabled: false,
-        build_enabled: false,
-      });
-    };
-
-    let repo_name =
-      split.next().context("Repo repo has no repo after the /")?;
-
-    let github_repos = github.repos();
-
-    let webhooks = github_repos
-      .list_all_webhooks(owner, repo_name)
-      .await
-      .context("failed to list all webhooks on repo")?
-      .body;
-
-    let CoreConfig {
-      host,
-      webhook_base_url,
-      ..
-    } = core_config();
-
-    let host = if webhook_base_url.is_empty() {
-      host
-    } else {
-      webhook_base_url
-    };
-    let clone_url =
-      format!("{host}/listener/github/repo/{}/clone", repo.id);
-    let pull_url =
-      format!("{host}/listener/github/repo/{}/pull", repo.id);
-    let build_url =
-      format!("{host}/listener/github/repo/{}/build", repo.id);
-
-    let mut clone_enabled = false;
-    let mut pull_enabled = false;
-    let mut build_enabled = false;
-
-    for webhook in webhooks {
-      if !webhook.active {
-        continue;
-      }
-      if webhook.config.url == clone_url {
-        clone_enabled = true
-      }
-      if webhook.config.url == pull_url {
-        pull_enabled = true
-      }
-      if webhook.config.url == build_url {
-        build_enabled = true
-      }
-    }
-
-    Ok(GetRepoWebhooksEnabledResponse {
-      managed: true,
-      clone_enabled,
-      pull_enabled,
-      build_enabled,
-    })
-  }
-}

bin/core/src/api/read/schedule.rs

@@ -4,7 +4,6 @@ use komodo_client::{
   entities::{
     ResourceTarget,
     action::Action,
-    permission::PermissionLevel,
     procedure::Procedure,
     resource::{ResourceQuery, TemplatesQueryBehavior},
     schedule::Schedule,
@@ -36,7 +35,6 @@ impl Resolve<ReadArgs> for ListSchedules {
           specific: Default::default(),
         },
         &args.user,
-        PermissionLevel::Read.into(),
         &all_tags,
       ),
       list_full_for_user::<Procedure>(
@@ -48,7 +46,6 @@ impl Resolve<ReadArgs> for ListSchedules {
           specific: Default::default(),
         },
         &args.user,
-        PermissionLevel::Read.into(),
         &all_tags,
       )
     )?;

bin/core/src/api/read/server.rs

@@ -66,7 +66,6 @@ impl Resolve<ReadArgs> for GetServersSummary {
     let servers = resource::list_for_user::<Server>(
       Default::default(),
       user,
-      PermissionLevel::Read.into(),
       &[],
     )
     .await?;
@@ -127,13 +126,8 @@ impl Resolve<ReadArgs> for ListServers {
       get_all_tags(None).await?
     };
     Ok(
-      resource::list_for_user::<Server>(
-        self.query,
-        user,
-        PermissionLevel::Read.into(),
-        &all_tags,
-      )
-      .await?,
+      resource::list_for_user::<Server>(self.query, user, &all_tags)
+        .await?,
     )
   }
 }
@@ -150,10 +144,7 @@ impl Resolve<ReadArgs> for ListFullServers {
     };
     Ok(
       resource::list_full_for_user::<Server>(
-        self.query,
-        user,
-        PermissionLevel::Read.into(),
-        &all_tags,
+        self.query, user, &all_tags,
       )
       .await?,
     )
@@ -400,7 +391,6 @@ impl Resolve<ReadArgs> for ListAllDockerContainers {
     let servers = resource::list_for_user::<Server>(
       Default::default(),
       user,
-      PermissionLevel::Read.into(),
       &[],
     )
     .await?
@@ -434,7 +424,6 @@ impl Resolve<ReadArgs> for GetDockerContainersSummary {
     let servers = resource::list_full_for_user::<Server>(
       Default::default(),
       user,
-      PermissionLevel::Read.into(),
       &[],
     )
     .await
@@ -885,7 +874,9 @@ impl Resolve<ReadArgs> for ListTerminals {
     if self.fresh || komodo_timestamp() > cache.ttl {
       cache.list = periphery_client(&server)
         .await?
-        .request(periphery_client::api::terminal::ListTerminals {})
+        .request(periphery_client::api::terminal::ListTerminals {
+          container: None,
+        })
         .await
         .context("Failed to get fresh terminal list")?;
       cache.ttl = komodo_timestamp() + TERMINAL_CACHE_TIMEOUT;

bin/core/src/api/read/stack.rs

@@ -4,7 +4,6 @@ use anyhow::{Context, anyhow};
 use komodo_client::{
   api::read::*,
   entities::{
-    config::core::CoreConfig,
     docker::container::Container,
     permission::PermissionLevel,
     server::{Server, ServerState},
@@ -18,15 +17,11 @@ use periphery_client::api::{
 use resolver_api::Resolve;
 
 use crate::{
-  config::core_config,
   helpers::{periphery_client, query::get_all_tags},
   permission::get_check_permissions,
   resource,
   stack::get_stack_and_server,
-  state::{
-    action_states, github_client, server_status_cache,
-    stack_status_cache,
-  },
+  state::{action_states, server_status_cache, stack_status_cache},
 };
 
 use super::ReadArgs;
@@ -205,10 +200,7 @@ impl Resolve<ReadArgs> for ListCommonStackExtraArgs {
       get_all_tags(None).await?
     };
     let stacks = resource::list_full_for_user::<Stack>(
-      self.query,
-      user,
-      PermissionLevel::Read.into(),
-      &all_tags,
+      self.query, user, &all_tags,
     )
     .await
     .context("failed to get resources matching query")?;
@@ -239,10 +231,7 @@ impl Resolve<ReadArgs> for ListCommonStackBuildExtraArgs {
       get_all_tags(None).await?
     };
     let stacks = resource::list_full_for_user::<Stack>(
-      self.query,
-      user,
-      PermissionLevel::Read.into(),
-      &all_tags,
+      self.query, user, &all_tags,
     )
     .await
     .context("failed to get resources matching query")?;
@@ -273,13 +262,9 @@ impl Resolve<ReadArgs> for ListStacks {
       get_all_tags(None).await?
     };
     let only_update_available = self.query.specific.update_available;
-    let stacks = resource::list_for_user::<Stack>(
-      self.query,
-      user,
-      PermissionLevel::Read.into(),
-      &all_tags,
-    )
-    .await?;
+    let stacks =
+      resource::list_for_user::<Stack>(self.query, user, &all_tags)
+        .await?;
     let stacks = if only_update_available {
       stacks
         .into_iter()
@@ -310,10 +295,7 @@ impl Resolve<ReadArgs> for ListFullStacks {
     };
     Ok(
       resource::list_full_for_user::<Stack>(
-        self.query,
-        user,
-        PermissionLevel::Read.into(),
-        &all_tags,
+        self.query, user, &all_tags,
       )
       .await?,
     )
@@ -349,7 +331,6 @@ impl Resolve<ReadArgs> for GetStacksSummary {
     let stacks = resource::list_full_for_user::<Stack>(
       Default::default(),
       user,
-      PermissionLevel::Read.into(),
       &[],
     )
     .await
@@ -378,91 +359,3 @@ impl Resolve<ReadArgs> for GetStacksSummary {
     Ok(res)
   }
 }
-
-impl Resolve<ReadArgs> for GetStackWebhooksEnabled {
-  async fn resolve(
-    self,
-    ReadArgs { user }: &ReadArgs,
-  ) -> serror::Result<GetStackWebhooksEnabledResponse> {
-    let Some(github) = github_client() else {
-      return Ok(GetStackWebhooksEnabledResponse {
-        managed: false,
-        refresh_enabled: false,
-        deploy_enabled: false,
-      });
-    };
-
-    let stack = get_check_permissions::<Stack>(
-      &self.stack,
-      user,
-      PermissionLevel::Read.into(),
-    )
-    .await?;
-
-    if stack.config.git_provider != "github.com"
-      || stack.config.repo.is_empty()
-    {
-      return Ok(GetStackWebhooksEnabledResponse {
-        managed: false,
-        refresh_enabled: false,
-        deploy_enabled: false,
-      });
-    }
-
-    let mut split = stack.config.repo.split('/');
-    let owner = split.next().context("Sync repo has no owner")?;
-
-    let Some(github) = github.get(owner) else {
-      return Ok(GetStackWebhooksEnabledResponse {
-        managed: false,
-        refresh_enabled: false,
-        deploy_enabled: false,
-      });
-    };
-
-    let repo_name =
-      split.next().context("Repo repo has no repo after the /")?;
-
-    let github_repos = github.repos();
-
-    let webhooks = github_repos
-      .list_all_webhooks(owner, repo_name)
-      .await
-      .context("failed to list all webhooks on repo")?
-      .body;
-
-    let CoreConfig {
-      host,
-      webhook_base_url,
-      ..
-    } = core_config();
-
-    let host = if webhook_base_url.is_empty() {
-      host
-    } else {
-      webhook_base_url
-    };
-    let refresh_url =
-      format!("{host}/listener/github/stack/{}/refresh", stack.id);
-    let deploy_url =
-      format!("{host}/listener/github/stack/{}/deploy", stack.id);
-
-    let mut refresh_enabled = false;
-    let mut deploy_enabled = false;
-
-    for webhook in webhooks {
-      if webhook.active && webhook.config.url == refresh_url {
-        refresh_enabled = true
-      }
-      if webhook.active && webhook.config.url == deploy_url {
-        deploy_enabled = true
-      }
-    }
-
-    Ok(GetStackWebhooksEnabledResponse {
-      managed: true,
-      refresh_enabled,
-      deploy_enabled,
-    })
-  }
-}

bin/core/src/api/read/sync.rs

@@ -2,7 +2,6 @@ use anyhow::Context;
 use komodo_client::{
   api::read::*,
   entities::{
-    config::core::CoreConfig,
     permission::PermissionLevel,
     sync::{
       ResourceSync, ResourceSyncActionState, ResourceSyncListItem,
@@ -12,11 +11,8 @@ use komodo_client::{
 use resolver_api::Resolve;
 
 use crate::{
-  config::core_config,
-  helpers::query::get_all_tags,
-  permission::get_check_permissions,
-  resource,
-  state::{action_states, github_client},
+  helpers::query::get_all_tags, permission::get_check_permissions,
+  resource, state::action_states,
 };
 
 use super::ReadArgs;
@@ -49,10 +45,7 @@ impl Resolve<ReadArgs> for ListResourceSyncs {
     };
     Ok(
       resource::list_for_user::<ResourceSync>(
-        self.query,
-        user,
-        PermissionLevel::Read.into(),
-        &all_tags,
+        self.query, user, &all_tags,
       )
       .await?,
     )
@@ -71,10 +64,7 @@ impl Resolve<ReadArgs> for ListFullResourceSyncs {
     };
     Ok(
       resource::list_full_for_user::<ResourceSync>(
-        self.query,
-        user,
-        PermissionLevel::Read.into(),
-        &all_tags,
+        self.query, user, &all_tags,
       )
       .await?,
     )
@@ -111,7 +101,6 @@ impl Resolve<ReadArgs> for GetResourceSyncsSummary {
       resource::list_full_for_user::<ResourceSync>(
         Default::default(),
         user,
-        PermissionLevel::Read.into(),
         &[],
       )
       .await
@@ -154,91 +143,3 @@ impl Resolve<ReadArgs> for GetResourceSyncsSummary {
     Ok(res)
   }
 }
-
-impl Resolve<ReadArgs> for GetSyncWebhooksEnabled {
-  async fn resolve(
-    self,
-    ReadArgs { user }: &ReadArgs,
-  ) -> serror::Result<GetSyncWebhooksEnabledResponse> {
-    let Some(github) = github_client() else {
-      return Ok(GetSyncWebhooksEnabledResponse {
-        managed: false,
-        refresh_enabled: false,
-        sync_enabled: false,
-      });
-    };
-
-    let sync = get_check_permissions::<ResourceSync>(
-      &self.sync,
-      user,
-      PermissionLevel::Read.into(),
-    )
-    .await?;
-
-    if sync.config.git_provider != "github.com"
-      || sync.config.repo.is_empty()
-    {
-      return Ok(GetSyncWebhooksEnabledResponse {
-        managed: false,
-        refresh_enabled: false,
-        sync_enabled: false,
-      });
-    }
-
-    let mut split = sync.config.repo.split('/');
-    let owner = split.next().context("Sync repo has no owner")?;
-
-    let Some(github) = github.get(owner) else {
-      return Ok(GetSyncWebhooksEnabledResponse {
-        managed: false,
-        refresh_enabled: false,
-        sync_enabled: false,
-      });
-    };
-
-    let repo_name =
-      split.next().context("Repo repo has no repo after the /")?;
-
-    let github_repos = github.repos();
-
-    let webhooks = github_repos
-      .list_all_webhooks(owner, repo_name)
-      .await
-      .context("failed to list all webhooks on repo")?
-      .body;
-
-    let CoreConfig {
-      host,
-      webhook_base_url,
-      ..
-    } = core_config();
-
-    let host = if webhook_base_url.is_empty() {
-      host
-    } else {
-      webhook_base_url
-    };
-    let refresh_url =
-      format!("{host}/listener/github/sync/{}/refresh", sync.id);
-    let sync_url =
-      format!("{host}/listener/github/sync/{}/sync", sync.id);
-
-    let mut refresh_enabled = false;
-    let mut sync_enabled = false;
-
-    for webhook in webhooks {
-      if webhook.active && webhook.config.url == refresh_url {
-        refresh_enabled = true
-      }
-      if webhook.active && webhook.config.url == sync_url {
-        sync_enabled = true
-      }
-    }
-
-    Ok(GetSyncWebhooksEnabledResponse {
-      managed: true,
-      refresh_enabled,
-      sync_enabled,
-    })
-  }
-}

bin/core/src/api/read/toml.rs

@@ -46,7 +46,6 @@ async fn get_all_targets(
     resource::list_full_for_user::<Alerter>(
       ResourceQuery::builder().tags(tags).build(),
       user,
-      PermissionLevel::Read.into(),
       &all_tags,
     )
     .await?
@@ -57,7 +56,6 @@ async fn get_all_targets(
     resource::list_full_for_user::<Builder>(
       ResourceQuery::builder().tags(tags).build(),
       user,
-      PermissionLevel::Read.into(),
       &all_tags,
     )
     .await?
@@ -68,7 +66,6 @@ async fn get_all_targets(
     resource::list_full_for_user::<Server>(
       ResourceQuery::builder().tags(tags).build(),
       user,
-      PermissionLevel::Read.into(),
       &all_tags,
     )
     .await?
@@ -79,7 +76,6 @@ async fn get_all_targets(
     resource::list_full_for_user::<Stack>(
       ResourceQuery::builder().tags(tags).build(),
       user,
-      PermissionLevel::Read.into(),
       &all_tags,
     )
     .await?
@@ -90,7 +86,6 @@ async fn get_all_targets(
     resource::list_full_for_user::<Deployment>(
       ResourceQuery::builder().tags(tags).build(),
       user,
-      PermissionLevel::Read.into(),
       &all_tags,
     )
     .await?
@@ -101,7 +96,6 @@ async fn get_all_targets(
     resource::list_full_for_user::<Build>(
       ResourceQuery::builder().tags(tags).build(),
       user,
-      PermissionLevel::Read.into(),
       &all_tags,
     )
     .await?
@@ -112,7 +106,6 @@ async fn get_all_targets(
     resource::list_full_for_user::<Repo>(
       ResourceQuery::builder().tags(tags).build(),
       user,
-      PermissionLevel::Read.into(),
       &all_tags,
     )
     .await?
@@ -123,7 +116,6 @@ async fn get_all_targets(
     resource::list_full_for_user::<Procedure>(
       ResourceQuery::builder().tags(tags).build(),
       user,
-      PermissionLevel::Read.into(),
       &all_tags,
     )
     .await?
@@ -134,7 +126,6 @@ async fn get_all_targets(
     resource::list_full_for_user::<Action>(
       ResourceQuery::builder().tags(tags).build(),
       user,
-      PermissionLevel::Read.into(),
       &all_tags,
     )
     .await?
@@ -145,7 +136,6 @@ async fn get_all_targets(
     resource::list_full_for_user::<ResourceSync>(
       ResourceQuery::builder().tags(tags).build(),
       user,
-      PermissionLevel::Read.into(),
       &all_tags,
     )
     .await?

bin/core/src/api/terminal.rs

@@ -8,7 +8,6 @@ use komodo_client::{
   },
 };
 use serror::Json;
-use uuid::Uuid;
 
 use crate::{
   auth::auth_request, helpers::periphery_client,
@@ -29,28 +28,22 @@ pub fn router() -> Router {
 //  ExecuteTerminal
 // =================
 
-async fn execute_terminal(
-  Extension(user): Extension<User>,
-  Json(request): Json<ExecuteTerminalBody>,
-) -> serror::Result<axum::body::Body> {
-  execute_terminal_inner(Uuid::new_v4(), request, user).await
-}
-
 #[instrument(
   name = "ExecuteTerminal",
-  skip(user),
+  skip_all,
   fields(
-    user_id = user.id,
+    operator = user.id,
+    server,
+    terminal,
   )
 )]
-async fn execute_terminal_inner(
-  req_id: Uuid,
-  ExecuteTerminalBody {
+async fn execute_terminal(
+  Extension(user): Extension<User>,
+  Json(ExecuteTerminalBody {
     server,
     terminal,
     command,
-  }: ExecuteTerminalBody,
-  user: User,
+  }): Json<ExecuteTerminalBody>,
 ) -> serror::Result<axum::body::Body> {
   info!("/terminal/execute request | user: {}", user.username);
 
@@ -74,29 +67,26 @@ async fn execute_terminal_inner(
 //  ExecuteContainerExec
 // ======================
 
-async fn execute_container_exec(
-  Extension(user): Extension<User>,
-  Json(request): Json<ExecuteContainerExecBody>,
-) -> serror::Result<axum::body::Body> {
-  execute_container_exec_inner(Uuid::new_v4(), request, user).await
-}
-
 #[instrument(
   name = "ExecuteContainerExec",
-  skip(user),
+  skip_all,
   fields(
-    user_id = user.id,
+    operator = user.id,
+    server,
+    container,
+    shell,
+    recreate = format!("{recreate:?}"),
   )
 )]
-async fn execute_container_exec_inner(
-  req_id: Uuid,
-  ExecuteContainerExecBody {
+async fn execute_container_exec(
+  Extension(user): Extension<User>,
+  Json(ExecuteContainerExecBody {
     server,
     container,
     shell,
     command,
-  }: ExecuteContainerExecBody,
-  user: User,
+    recreate,
+  }): Json<ExecuteContainerExecBody>,
 ) -> serror::Result<axum::body::Body> {
   info!("ExecuteContainerExec request | user: {}", user.username);
 
@@ -110,7 +100,7 @@ async fn execute_container_exec_inner(
   let periphery = periphery_client(&server).await?;
 
   let stream = periphery
-    .execute_container_exec(container, shell, command)
+    .execute_container_exec(container, shell, command, recreate)
     .await
     .context(
       "Failed to execute container exec command on periphery",
@@ -123,28 +113,24 @@ async fn execute_container_exec_inner(
 //  ExecuteDeploymentExec
 // =======================
 
-async fn execute_deployment_exec(
-  Extension(user): Extension<User>,
-  Json(request): Json<ExecuteDeploymentExecBody>,
-) -> serror::Result<axum::body::Body> {
-  execute_deployment_exec_inner(Uuid::new_v4(), request, user).await
-}
-
 #[instrument(
   name = "ExecuteDeploymentExec",
-  skip(user),
+  skip_all,
   fields(
-    user_id = user.id,
+    operator = user.id,
+    deployment,
+    shell,
+    recreate = format!("{recreate:?}"),
   )
 )]
-async fn execute_deployment_exec_inner(
-  req_id: Uuid,
-  ExecuteDeploymentExecBody {
+async fn execute_deployment_exec(
+  Extension(user): Extension<User>,
+  Json(ExecuteDeploymentExecBody {
     deployment,
     shell,
     command,
-  }: ExecuteDeploymentExecBody,
-  user: User,
+    recreate,
+  }): Json<ExecuteDeploymentExecBody>,
 ) -> serror::Result<axum::body::Body> {
   info!("ExecuteDeploymentExec request | user: {}", user.username);
 
@@ -160,7 +146,7 @@ async fn execute_deployment_exec_inner(
   let periphery = periphery_client(&server).await?;
 
   let stream = periphery
-    .execute_container_exec(deployment.name, shell, command)
+    .execute_container_exec(deployment.name, shell, command, recreate)
     .await
     .context(
       "Failed to execute container exec command on periphery",
@@ -173,29 +159,26 @@ async fn execute_deployment_exec_inner(
 //  ExecuteStackExec
 // ==================
 
-async fn execute_stack_exec(
-  Extension(user): Extension<User>,
-  Json(request): Json<ExecuteStackExecBody>,
-) -> serror::Result<axum::body::Body> {
-  execute_stack_exec_inner(Uuid::new_v4(), request, user).await
-}
-
 #[instrument(
   name = "ExecuteStackExec",
-  skip(user),
+  skip_all,
   fields(
-    user_id = user.id,
+    operator = user.id,
+    stack,
+    service,
+    shell,
+    recreate = format!("{recreate:?}"),
   )
 )]
-async fn execute_stack_exec_inner(
-  req_id: Uuid,
-  ExecuteStackExecBody {
+async fn execute_stack_exec(
+  Extension(user): Extension<User>,
+  Json(ExecuteStackExecBody {
     stack,
     service,
     shell,
     command,
-  }: ExecuteStackExecBody,
-  user: User,
+    recreate,
+  }): Json<ExecuteStackExecBody>,
 ) -> serror::Result<axum::body::Body> {
   info!("ExecuteStackExec request | user: {}", user.username);
 
@@ -226,7 +209,7 @@ async fn execute_stack_exec_inner(
   let periphery = periphery_client(&server).await?;
 
   let stream = periphery
-    .execute_container_exec(container, shell, command)
+    .execute_container_exec(container, shell, command, recreate)
     .await
     .context(
       "Failed to execute container exec command on periphery",

bin/core/src/api/user.rs

@@ -9,6 +9,7 @@ use database::mungos::{
   by_id::update_one_by_id, mongodb::bson::to_bson,
 };
 use derive_variants::EnumVariants;
+use komodo_client::entities::random_string;
 use komodo_client::{
   api::user::*,
   entities::{api_key::ApiKey, komodo_timestamp, user::User},
@@ -21,9 +22,7 @@ use typeshare::typeshare;
 use uuid::Uuid;
 
 use crate::{
-  auth::auth_request,
-  helpers::{query::get_user, random_string},
-  state::db_client,
+  auth::auth_request, helpers::query::get_user, state::db_client,
 };
 
 use super::Variant;
@@ -66,7 +65,6 @@ async fn variant_handler(
   handler(user, Json(req)).await
 }
 
-#[instrument(name = "UserHandler", level = "debug", skip(user))]
 async fn handler(
   Extension(user): Extension<User>,
   Json(request): Json<UserRequest>,
@@ -89,11 +87,6 @@ async fn handler(
 const RECENTLY_VIEWED_MAX: usize = 10;
 
 impl Resolve<UserArgs> for PushRecentlyViewed {
-  #[instrument(
-    name = "PushRecentlyViewed",
-    level = "debug",
-    skip(user)
-  )]
   async fn resolve(
     self,
     UserArgs { user }: &UserArgs,
@@ -131,11 +124,6 @@ impl Resolve<UserArgs> for PushRecentlyViewed {
 }
 
 impl Resolve<UserArgs> for SetLastSeenUpdate {
-  #[instrument(
-    name = "SetLastSeenUpdate",
-    level = "debug",
-    skip(user)
-  )]
   async fn resolve(
     self,
     UserArgs { user }: &UserArgs,
@@ -158,7 +146,11 @@ const SECRET_LENGTH: usize = 40;
 const BCRYPT_COST: u32 = 10;
 
 impl Resolve<UserArgs> for CreateApiKey {
-  #[instrument(name = "CreateApiKey", level = "debug", skip(user))]
+  #[instrument(
+    "CreateApiKey",
+    skip_all,
+    fields(operator = user.id)
+  )]
   async fn resolve(
     self,
     UserArgs { user }: &UserArgs,
@@ -188,7 +180,11 @@ impl Resolve<UserArgs> for CreateApiKey {
 }
 
 impl Resolve<UserArgs> for DeleteApiKey {
-  #[instrument(name = "DeleteApiKey", level = "debug", skip(user))]
+  #[instrument(
+    "DeleteApiKey",
+    skip_all,
+    fields(operator = user.id)
+  )]
   async fn resolve(
     self,
     UserArgs { user }: &UserArgs,

bin/core/src/api/write/action.rs

@@ -11,7 +11,15 @@ use crate::{permission::get_check_permissions, resource};
 use super::WriteArgs;
 
 impl Resolve<WriteArgs> for CreateAction {
-  #[instrument(name = "CreateAction", skip(user))]
+  #[instrument(
+    "CreateAction",
+    skip_all,
+    fields(
+      operator = user.id,
+      action = self.name,
+      config = serde_json::to_string(&self.config).unwrap(),
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -22,7 +30,15 @@ impl Resolve<WriteArgs> for CreateAction {
 }
 
 impl Resolve<WriteArgs> for CopyAction {
-  #[instrument(name = "CopyAction", skip(user))]
+  #[instrument(
+    "CopyAction",
+    skip_all,
+    fields(
+      operator = user.id,
+      action = self.name,
+      copy_action = self.id,
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -39,7 +55,15 @@ impl Resolve<WriteArgs> for CopyAction {
 }
 
 impl Resolve<WriteArgs> for UpdateAction {
-  #[instrument(name = "UpdateAction", skip(user))]
+  #[instrument(
+    "UpdateAction",
+    skip_all,
+    fields(
+      operator = user.id,
+      action = self.id,
+      update = serde_json::to_string(&self.config).unwrap(),
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -49,7 +73,15 @@ impl Resolve<WriteArgs> for UpdateAction {
 }
 
 impl Resolve<WriteArgs> for RenameAction {
-  #[instrument(name = "RenameAction", skip(user))]
+  #[instrument(
+    "RenameAction",
+    skip_all,
+    fields(
+      operator = user.id,
+      action = self.id,
+      new_name = self.name,
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -59,8 +91,18 @@ impl Resolve<WriteArgs> for RenameAction {
 }
 
 impl Resolve<WriteArgs> for DeleteAction {
-  #[instrument(name = "DeleteAction", skip(args))]
-  async fn resolve(self, args: &WriteArgs) -> serror::Result<Action> {
-    Ok(resource::delete::<Action>(&self.id, args).await?)
+  #[instrument(
+    "DeleteAction",
+    skip_all,
+    fields(
+      operator = user.id,
+      action = self.id
+    )
+  )]
+  async fn resolve(
+    self,
+    WriteArgs { user }: &WriteArgs,
+  ) -> serror::Result<Action> {
+    Ok(resource::delete::<Action>(&self.id, user).await?)
   }
 }

bin/core/src/api/write/alert.rs

@@ -0,0 +1,41 @@
+use std::str::FromStr;
+
+use anyhow::{Context, anyhow};
+use database::mungos::mongodb::bson::{doc, oid::ObjectId};
+use komodo_client::{api::write::CloseAlert, entities::NoData};
+use reqwest::StatusCode;
+use resolver_api::Resolve;
+use serror::AddStatusCodeError;
+
+use crate::{api::write::WriteArgs, state::db_client};
+
+impl Resolve<WriteArgs> for CloseAlert {
+  #[instrument(
+    "CloseAlert",
+    skip_all,
+    fields(
+      operator = admin.id,
+      alert_id = self.id,
+    )
+  )]
+  async fn resolve(
+    self,
+    WriteArgs { user: admin }: &WriteArgs,
+  ) -> Result<Self::Response, Self::Error> {
+    if !admin.admin {
+      return Err(
+        anyhow!("This call is admin only")
+          .status_code(StatusCode::FORBIDDEN),
+      );
+    }
+    db_client()
+      .alerts
+      .update_one(
+        doc! { "_id": ObjectId::from_str(&self.id)? },
+        doc! { "$set": { "resolved": true } },
+      )
+      .await
+      .context("Failed to close Alert on database")?;
+    Ok(NoData {})
+  }
+}

bin/core/src/api/write/alerter.rs

@@ -11,7 +11,15 @@ use crate::{permission::get_check_permissions, resource};
 use super::WriteArgs;
 
 impl Resolve<WriteArgs> for CreateAlerter {
-  #[instrument(name = "CreateAlerter", skip(user))]
+  #[instrument(
+    "CreateAlerter",
+    skip_all,
+    fields(
+      operator = user.id,
+      alerter = self.name,
+      config = serde_json::to_string(&self.config).unwrap(),
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -22,7 +30,15 @@ impl Resolve<WriteArgs> for CreateAlerter {
 }
 
 impl Resolve<WriteArgs> for CopyAlerter {
-  #[instrument(name = "CopyAlerter", skip(user))]
+  #[instrument(
+    "CopyAlerter",
+    skip_all,
+    fields(
+      operator = user.id,
+      alerter = self.name,
+      copy_alerter = self.id,
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -39,17 +55,32 @@ impl Resolve<WriteArgs> for CopyAlerter {
 }
 
 impl Resolve<WriteArgs> for DeleteAlerter {
-  #[instrument(name = "DeleteAlerter", skip(args))]
+  #[instrument(
+    "DeleteAlerter",
+    skip_all,
+    fields(
+      operator = user.id,
+      alerter = self.id,
+    )
+  )]
   async fn resolve(
     self,
-    args: &WriteArgs,
+    WriteArgs { user }: &WriteArgs,
   ) -> serror::Result<Alerter> {
-    Ok(resource::delete::<Alerter>(&self.id, args).await?)
+    Ok(resource::delete::<Alerter>(&self.id, user).await?)
   }
 }
 
 impl Resolve<WriteArgs> for UpdateAlerter {
-  #[instrument(name = "UpdateAlerter", skip(user))]
+  #[instrument(
+    "UpdateAlerter",
+    skip_all,
+    fields(
+      operator = user.id,
+      alerter = self.id,
+      update = serde_json::to_string(&self.config).unwrap()
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -62,7 +93,15 @@ impl Resolve<WriteArgs> for UpdateAlerter {
 }
 
 impl Resolve<WriteArgs> for RenameAlerter {
-  #[instrument(name = "RenameAlerter", skip(user))]
+  #[instrument(
+    "RenameAlerter",
+    skip_all,
+    fields(
+      operator = user.id,
+      alerter = self.id,
+      new_name = self.name,
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,

bin/core/src/api/write/build.rs

@@ -1,4 +1,4 @@
-use std::{path::PathBuf, str::FromStr, time::Duration};
+use std::{path::PathBuf, time::Duration};
 
 use anyhow::{Context, anyhow};
 use database::mungos::mongodb::bson::to_document;
@@ -11,18 +11,14 @@ use komodo_client::{
   entities::{
     FileContents, NoData, Operation, RepoExecutionArgs,
     all_logs_success,
-    build::{Build, BuildInfo, PartialBuildConfig},
+    build::{Build, BuildInfo},
     builder::{Builder, BuilderConfig},
-    config::core::CoreConfig,
     permission::PermissionLevel,
     repo::Repo,
     server::ServerState,
     update::Update,
   },
 };
-use octorust::types::{
-  ReposCreateWebhookRequest, ReposCreateWebhookRequestConfig,
-};
 use periphery_client::api::build::{
   GetDockerfileContentsOnHost, WriteDockerfileContentsToHost,
 };
@@ -40,13 +36,21 @@ use crate::{
   periphery::PeripheryClient,
   permission::get_check_permissions,
   resource,
-  state::{db_client, github_client},
+  state::db_client,
 };
 
 use super::WriteArgs;
 
 impl Resolve<WriteArgs> for CreateBuild {
-  #[instrument(name = "CreateBuild", skip(user))]
+  #[instrument(
+    "CreateBuild",
+    skip_all,
+    fields(
+      operator = user.id,
+      build = self.name,
+      config = serde_json::to_string(&self.config).unwrap(),
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -57,7 +61,15 @@ impl Resolve<WriteArgs> for CreateBuild {
 }
 
 impl Resolve<WriteArgs> for CopyBuild {
-  #[instrument(name = "CopyBuild", skip(user))]
+  #[instrument(
+    "CopyBuild",
+    skip_all,
+    fields(
+      operator = user.id,
+      build = self.name,
+      copy_build = self.id,
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -76,14 +88,32 @@ impl Resolve<WriteArgs> for CopyBuild {
 }
 
 impl Resolve<WriteArgs> for DeleteBuild {
-  #[instrument(name = "DeleteBuild", skip(args))]
-  async fn resolve(self, args: &WriteArgs) -> serror::Result<Build> {
-    Ok(resource::delete::<Build>(&self.id, args).await?)
+  #[instrument(
+    "DeleteBuild",
+    skip_all,
+    fields(
+      operator = user.id,
+      build = self.id,
+    )
+  )]
+  async fn resolve(
+    self,
+    WriteArgs { user }: &WriteArgs,
+  ) -> serror::Result<Build> {
+    Ok(resource::delete::<Build>(&self.id, user).await?)
   }
 }
 
 impl Resolve<WriteArgs> for UpdateBuild {
-  #[instrument(name = "UpdateBuild", skip(user))]
+  #[instrument(
+    "UpdateBuild",
+    skip_all,
+    fields(
+      operator = user.id,
+      build = self.id,
+      update = serde_json::to_string(&self.config).unwrap(),
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -93,7 +123,15 @@ impl Resolve<WriteArgs> for UpdateBuild {
 }
 
 impl Resolve<WriteArgs> for RenameBuild {
-  #[instrument(name = "RenameBuild", skip(user))]
+  #[instrument(
+    "RenameBuild",
+    skip_all,
+    fields(
+      operator = user.id,
+      build = self.id,
+      new_name = self.name,
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -103,7 +141,14 @@ impl Resolve<WriteArgs> for RenameBuild {
 }
 
 impl Resolve<WriteArgs> for WriteBuildFileContents {
-  #[instrument(name = "WriteBuildFileContents", skip(args))]
+  #[instrument(
+    "WriteBuildFileContents",
+    skip_all,
+    fields(
+      operator = args.user.id,
+      build = self.build,
+    )
+  )]
   async fn resolve(self, args: &WriteArgs) -> serror::Result<Update> {
     let build = get_check_permissions::<Build>(
       &self.build,
@@ -175,6 +220,7 @@ impl Resolve<WriteArgs> for WriteBuildFileContents {
   }
 }
 
+#[instrument("WriteDockerfileContentsGit", skip_all)]
 async fn write_dockerfile_contents_git(
   req: WriteBuildFileContents,
   args: &WriteArgs,
@@ -272,8 +318,9 @@ async fn write_dockerfile_contents_git(
     return Ok(update);
   }
 
-  if let Err(e) =
-    fs::write(&full_path, &contents).await.with_context(|| {
+  if let Err(e) = secret_file::write_async(&full_path, &contents)
+    .await
+    .with_context(|| {
       format!("Failed to write dockerfile contents to {full_path:?}")
     })
   {
@@ -320,11 +367,6 @@ async fn write_dockerfile_contents_git(
 }
 
 impl Resolve<WriteArgs> for RefreshBuildCache {
-  #[instrument(
-    name = "RefreshBuildCache",
-    level = "debug",
-    skip(user)
-  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -439,7 +481,6 @@ async fn get_on_host_periphery(
           &config,
         ),
         config.insecure_tls,
-        &config.passkey,
       )
       .await?;
       // Poll for connection to be estalished
@@ -526,8 +567,7 @@ async fn get_git_remote(
   .await
   .context("failed to clone build repo")?;
 
-  let relative_path = PathBuf::from_str(&build.config.build_path)
-    .context("Invalid build path")?
+  let relative_path = PathBuf::from(&build.config.build_path)
     .join(&build.config.dockerfile_path);
 
   let full_path = repo_path.join(&relative_path);
@@ -546,201 +586,3 @@ async fn get_git_remote(
     res.commit_message,
   )))
 }
-
-impl Resolve<WriteArgs> for CreateBuildWebhook {
-  #[instrument(name = "CreateBuildWebhook", skip(args))]
-  async fn resolve(
-    self,
-    args: &WriteArgs,
-  ) -> serror::Result<CreateBuildWebhookResponse> {
-    let Some(github) = github_client() else {
-      return Err(
-        anyhow!(
-          "github_webhook_app is not configured in core config toml"
-        )
-        .into(),
-      );
-    };
-
-    let WriteArgs { user } = args;
-
-    let build = get_check_permissions::<Build>(
-      &self.build,
-      user,
-      PermissionLevel::Write.into(),
-    )
-    .await?;
-
-    if build.config.repo.is_empty() {
-      return Err(
-        anyhow!("No repo configured, can't create webhook").into(),
-      );
-    }
-
-    let mut split = build.config.repo.split('/');
-    let owner = split.next().context("Build repo has no owner")?;
-
-    let Some(github) = github.get(owner) else {
-      return Err(
-        anyhow!("Cannot manage repo webhooks under owner {owner}")
-          .into(),
-      );
-    };
-
-    let repo =
-      split.next().context("Build repo has no repo after the /")?;
-
-    let github_repos = github.repos();
-
-    // First make sure the webhook isn't already created (inactive ones are ignored)
-    let webhooks = github_repos
-      .list_all_webhooks(owner, repo)
-      .await
-      .context("failed to list all webhooks on repo")?
-      .body;
-
-    let CoreConfig {
-      host,
-      webhook_base_url,
-      webhook_secret,
-      ..
-    } = core_config();
-
-    let webhook_secret = if build.config.webhook_secret.is_empty() {
-      webhook_secret
-    } else {
-      &build.config.webhook_secret
-    };
-
-    let host = if webhook_base_url.is_empty() {
-      host
-    } else {
-      webhook_base_url
-    };
-    let url = format!("{host}/listener/github/build/{}", build.id);
-
-    for webhook in webhooks {
-      if webhook.active && webhook.config.url == url {
-        return Ok(NoData {});
-      }
-    }
-
-    // Now good to create the webhook
-    let request = ReposCreateWebhookRequest {
-      active: Some(true),
-      config: Some(ReposCreateWebhookRequestConfig {
-        url,
-        secret: webhook_secret.to_string(),
-        content_type: String::from("json"),
-        insecure_ssl: None,
-        digest: Default::default(),
-        token: Default::default(),
-      }),
-      events: vec![String::from("push")],
-      name: String::from("web"),
-    };
-    github_repos
-      .create_webhook(owner, repo, &request)
-      .await
-      .context("failed to create webhook")?;
-
-    if !build.config.webhook_enabled {
-      UpdateBuild {
-        id: build.id,
-        config: PartialBuildConfig {
-          webhook_enabled: Some(true),
-          ..Default::default()
-        },
-      }
-      .resolve(args)
-      .await
-      .map_err(|e| e.error)
-      .context("failed to update build to enable webhook")?;
-    }
-
-    Ok(NoData {})
-  }
-}
-
-impl Resolve<WriteArgs> for DeleteBuildWebhook {
-  #[instrument(name = "DeleteBuildWebhook", skip(user))]
-  async fn resolve(
-    self,
-    WriteArgs { user }: &WriteArgs,
-  ) -> serror::Result<DeleteBuildWebhookResponse> {
-    let Some(github) = github_client() else {
-      return Err(
-        anyhow!(
-          "github_webhook_app is not configured in core config toml"
-        )
-        .into(),
-      );
-    };
-
-    let build = get_check_permissions::<Build>(
-      &self.build,
-      user,
-      PermissionLevel::Write.into(),
-    )
-    .await?;
-
-    if build.config.git_provider != "github.com" {
-      return Err(
-        anyhow!("Can only manage github.com repo webhooks").into(),
-      );
-    }
-
-    if build.config.repo.is_empty() {
-      return Err(
-        anyhow!("No repo configured, can't delete webhook").into(),
-      );
-    }
-
-    let mut split = build.config.repo.split('/');
-    let owner = split.next().context("Build repo has no owner")?;
-
-    let Some(github) = github.get(owner) else {
-      return Err(
-        anyhow!("Cannot manage repo webhooks under owner {owner}")
-          .into(),
-      );
-    };
-
-    let repo =
-      split.next().context("Build repo has no repo after the /")?;
-
-    let github_repos = github.repos();
-
-    let webhooks = github_repos
-      .list_all_webhooks(owner, repo)
-      .await
-      .context("failed to list all webhooks on repo")?
-      .body;
-
-    let CoreConfig {
-      host,
-      webhook_base_url,
-      ..
-    } = core_config();
-
-    let host = if webhook_base_url.is_empty() {
-      host
-    } else {
-      webhook_base_url
-    };
-    let url = format!("{host}/listener/github/build/{}", build.id);
-
-    for webhook in webhooks {
-      if webhook.active && webhook.config.url == url {
-        github_repos
-          .delete_webhook(owner, repo, webhook.id)
-          .await
-          .context("failed to delete webhook")?;
-        return Ok(NoData {});
-      }
-    }
-
-    // No webhook to delete, all good
-    Ok(NoData {})
-  }
-}

bin/core/src/api/write/builder.rs

@@ -11,7 +11,15 @@ use crate::{permission::get_check_permissions, resource};
 use super::WriteArgs;
 
 impl Resolve<WriteArgs> for CreateBuilder {
-  #[instrument(name = "CreateBuilder", skip(user))]
+  #[instrument(
+    "CreateBuilder",
+    skip_all,
+    fields(
+      operator = user.id,
+      builder = self.name,
+      config = serde_json::to_string(&self.config).unwrap(),
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -22,7 +30,15 @@ impl Resolve<WriteArgs> for CreateBuilder {
 }
 
 impl Resolve<WriteArgs> for CopyBuilder {
-  #[instrument(name = "CopyBuilder", skip(user))]
+  #[instrument(
+    "CopyBuilder",
+    skip_all,
+    fields(
+      operator = user.id,
+      builder = self.name,
+      copy_builder = self.id,
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -39,17 +55,32 @@ impl Resolve<WriteArgs> for CopyBuilder {
 }
 
 impl Resolve<WriteArgs> for DeleteBuilder {
-  #[instrument(name = "DeleteBuilder", skip(args))]
+  #[instrument(
+    "DeleteBuilder",
+    skip_all,
+    fields(
+      operator = user.id,
+      builder = self.id,
+    )
+  )]
   async fn resolve(
     self,
-    args: &WriteArgs,
+    WriteArgs { user }: &WriteArgs,
   ) -> serror::Result<Builder> {
-    Ok(resource::delete::<Builder>(&self.id, args).await?)
+    Ok(resource::delete::<Builder>(&self.id, user).await?)
   }
 }
 
 impl Resolve<WriteArgs> for UpdateBuilder {
-  #[instrument(name = "UpdateBuilder", skip(user))]
+  #[instrument(
+    "UpdateBuilder",
+    skip_all,
+    fields(
+      operator = user.id,
+      builder = self.id,
+      update = serde_json::to_string(&self.config).unwrap(),
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -62,7 +93,15 @@ impl Resolve<WriteArgs> for UpdateBuilder {
 }
 
 impl Resolve<WriteArgs> for RenameBuilder {
-  #[instrument(name = "RenameBuilder", skip(user))]
+  #[instrument(
+    "RenameBuilder",
+    skip_all,
+    fields(
+      operator = user.id,
+      builder = self.id,
+      new_name = self.name
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,

bin/core/src/api/write/deployment.rs

@@ -33,7 +33,15 @@ use crate::{
 use super::WriteArgs;
 
 impl Resolve<WriteArgs> for CreateDeployment {
-  #[instrument(name = "CreateDeployment", skip(user))]
+  #[instrument(
+    "CreateDeployment",
+    skip_all,
+    fields(
+      operator = user.id,
+      deployment = self.name,
+      config = serde_json::to_string(&self.config).unwrap(),
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -49,7 +57,15 @@ impl Resolve<WriteArgs> for CreateDeployment {
 }
 
 impl Resolve<WriteArgs> for CopyDeployment {
-  #[instrument(name = "CopyDeployment", skip(user))]
+  #[instrument(
+    "CopyDeployment",
+    skip_all,
+    fields(
+      operator = user.id,
+      deployment = self.name,
+      copy_deployment = self.id,
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -72,7 +88,15 @@ impl Resolve<WriteArgs> for CopyDeployment {
 }
 
 impl Resolve<WriteArgs> for CreateDeploymentFromContainer {
-  #[instrument(name = "CreateDeploymentFromContainer", skip(user))]
+  #[instrument(
+    "CreateDeploymentFromContainer",
+    skip_all,
+    fields(
+      operator = user.id,
+      server = self.server,
+      deployment = self.name,
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -166,17 +190,32 @@ impl Resolve<WriteArgs> for CreateDeploymentFromContainer {
 }
 
 impl Resolve<WriteArgs> for DeleteDeployment {
-  #[instrument(name = "DeleteDeployment", skip(args))]
+  #[instrument(
+    "DeleteDeployment",
+    skip_all,
+    fields(
+      operator = user.id,
+      deployment = self.id
+    )
+  )]
   async fn resolve(
     self,
-    args: &WriteArgs,
+    WriteArgs { user }: &WriteArgs,
   ) -> serror::Result<Deployment> {
-    Ok(resource::delete::<Deployment>(&self.id, args).await?)
+    Ok(resource::delete::<Deployment>(&self.id, user).await?)
   }
 }
 
 impl Resolve<WriteArgs> for UpdateDeployment {
-  #[instrument(name = "UpdateDeployment", skip(user))]
+  #[instrument(
+    "UpdateDeployment",
+    skip_all,
+    fields(
+      operator = user.id,
+      deployment = self.id,
+      update = serde_json::to_string(&self.config).unwrap(),
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -189,7 +228,15 @@ impl Resolve<WriteArgs> for UpdateDeployment {
 }
 
 impl Resolve<WriteArgs> for RenameDeployment {
-  #[instrument(name = "RenameDeployment", skip(user))]
+  #[instrument(
+    "RenameDeployment",
+    skip_all,
+    fields(
+      operator = user.id,
+      deployment = self.id,
+      new_name = self.name,
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,

bin/core/src/api/write/mod.rs

@@ -1,5 +1,3 @@
-use std::time::Instant;
-
 use anyhow::Context;
 use axum::{
   Extension, Router, extract::Path, middleware, routing::post,
@@ -11,6 +9,7 @@ use response::Response;
 use serde::{Deserialize, Serialize};
 use serde_json::json;
 use serror::Json;
+use strum::Display;
 use typeshare::typeshare;
 use uuid::Uuid;
 
@@ -19,6 +18,7 @@ use crate::auth::auth_request;
 use super::Variant;
 
 mod action;
+mod alert;
 mod alerter;
 mod build;
 mod builder;
@@ -46,7 +46,7 @@ pub struct WriteArgs {
 #[derive(
   Serialize, Deserialize, Debug, Clone, Resolve, EnumVariants,
 )]
-#[variant_derive(Debug)]
+#[variant_derive(Debug, Display)]
 #[args(WriteArgs)]
 #[response(Response)]
 #[error(serror::Error)]
@@ -103,8 +103,6 @@ pub enum WriteRequest {
   RenameStack(RenameStack),
   WriteStackFileContents(WriteStackFileContents),
   RefreshStackCache(RefreshStackCache),
-  CreateStackWebhook(CreateStackWebhook),
-  DeleteStackWebhook(DeleteStackWebhook),
 
   // ==== DEPLOYMENT ====
   CreateDeployment(CreateDeployment),
@@ -122,8 +120,6 @@ pub enum WriteRequest {
   RenameBuild(RenameBuild),
   WriteBuildFileContents(WriteBuildFileContents),
   RefreshBuildCache(RefreshBuildCache),
-  CreateBuildWebhook(CreateBuildWebhook),
-  DeleteBuildWebhook(DeleteBuildWebhook),
 
   // ==== BUILDER ====
   CreateBuilder(CreateBuilder),
@@ -139,8 +135,6 @@ pub enum WriteRequest {
   UpdateRepo(UpdateRepo),
   RenameRepo(RenameRepo),
   RefreshRepoCache(RefreshRepoCache),
-  CreateRepoWebhook(CreateRepoWebhook),
-  DeleteRepoWebhook(DeleteRepoWebhook),
 
   // ==== ALERTER ====
   CreateAlerter(CreateAlerter),
@@ -172,8 +166,6 @@ pub enum WriteRequest {
   WriteSyncFileContents(WriteSyncFileContents),
   CommitSync(CommitSync),
   RefreshResourceSyncPending(RefreshResourceSyncPending),
-  CreateSyncWebhook(CreateSyncWebhook),
-  DeleteSyncWebhook(DeleteSyncWebhook),
 
   // ==== TAG ====
   CreateTag(CreateTag),
@@ -200,6 +192,9 @@ pub enum WriteRequest {
   CreateOnboardingKey(CreateOnboardingKey),
   UpdateOnboardingKey(UpdateOnboardingKey),
   DeleteOnboardingKey(DeleteOnboardingKey),
+
+  // ==== ALERT ====
+  CloseAlert(CloseAlert),
 }
 
 pub fn router() -> Router {
@@ -234,31 +229,22 @@ async fn handler(
   res?
 }
 
-#[instrument(
-  name = "WriteRequest",
-  skip(user, request),
-  fields(
-    user_id = user.id,
-    request = format!("{:?}", request.extract_variant())
-  )
-)]
 async fn task(
   req_id: Uuid,
   request: WriteRequest,
   user: User,
 ) -> serror::Result<axum::response::Response> {
-  info!("/write request | user: {}", user.username);
-
-  let timer = Instant::now();
+  let variant = request.extract_variant();
+  info!("/write request | {variant} | user: {}", user.username);
 
   let res = request.resolve(&WriteArgs { user }).await;
 
   if let Err(e) = &res {
-    warn!("/write request {req_id} error: {:#}", e.error);
+    warn!(
+      "/write request {req_id} | {variant} | error: {:#}",
+      e.error
+    );
   }
 
-  let elapsed = timer.elapsed();
-  debug!("/write request {req_id} | resolve time: {elapsed:?}");
-
   res.map(|res| res.0)
 }

bin/core/src/api/write/onboarding_key.rs

@@ -6,7 +6,9 @@ use komodo_client::{
     DeleteOnboardingKey, DeleteOnboardingKeyResponse,
     UpdateOnboardingKey, UpdateOnboardingKeyResponse,
   },
-  entities::{komodo_timestamp, onboarding_key::OnboardingKey},
+  entities::{
+    komodo_timestamp, onboarding_key::OnboardingKey, random_string,
+  },
 };
 use noise::key::EncodedKeyPair;
 use reqwest::StatusCode;
@@ -18,7 +20,18 @@ use crate::{api::write::WriteArgs, state::db_client};
 //
 
 impl Resolve<WriteArgs> for CreateOnboardingKey {
-  #[instrument(name = "CreateServerOnboardingKey", skip(self, admin))]
+  #[instrument(
+    "CreateOnboardingKey",
+    skip_all,
+    fields(
+      operator = admin.id,
+      name = self.name,
+      expires = self.expires,
+      tags = format!("{:?}", self.tags),
+      copy_server = self.copy_server,
+      create_builder = self.create_builder,
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user: admin }: &WriteArgs,
@@ -29,13 +42,16 @@ impl Resolve<WriteArgs> for CreateOnboardingKey {
           .status_code(StatusCode::FORBIDDEN),
       );
     }
-    let keys = if let Some(private_key) = self.private_key {
-      EncodedKeyPair::from_private_key(&private_key)?
+    let private_key = if let Some(private_key) = self.private_key {
+      private_key
     } else {
-      EncodedKeyPair::generate()?
+      format!("O-{}", random_string(30))
     };
+    let public_key = EncodedKeyPair::from_private_key(&private_key)?
+      .public
+      .into_inner();
     let onboarding_key = OnboardingKey {
-      public_key: keys.public.into_inner(),
+      public_key,
       name: self.name,
       enabled: true,
       onboarded: Default::default(),
@@ -62,7 +78,7 @@ impl Resolve<WriteArgs> for CreateOnboardingKey {
         "No Server onboarding key found on database after create",
       )?;
     Ok(CreateOnboardingKeyResponse {
-      private_key: keys.private.into_inner(),
+      private_key,
       created,
     })
   }
@@ -71,6 +87,15 @@ impl Resolve<WriteArgs> for CreateOnboardingKey {
 //
 
 impl Resolve<WriteArgs> for UpdateOnboardingKey {
+  #[instrument(
+    "UpdateOnboardingKey",
+    skip_all,
+    fields(
+      operator = admin.id,
+      public_key = self.public_key,
+      update = format!("{:?}", self),
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user: admin }: &WriteArgs,
@@ -140,7 +165,14 @@ impl Resolve<WriteArgs> for UpdateOnboardingKey {
 //
 
 impl Resolve<WriteArgs> for DeleteOnboardingKey {
-  #[instrument(name = "DeleteServerOnboardingKey", skip(admin))]
+  #[instrument(
+    "DeleteOnboardingKey",
+    skip_all,
+    fields(
+      operator = admin.id,
+      public_key = self.public_key,
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user: admin }: &WriteArgs,

bin/core/src/api/write/permissions.rs

@@ -8,6 +8,7 @@ use database::mungos::{
     options::UpdateOptions,
   },
 };
+use derive_variants::ExtractVariant as _;
 use komodo_client::{
   api::write::*,
   entities::{
@@ -22,7 +23,15 @@ use crate::{helpers::query::get_user, state::db_client};
 use super::WriteArgs;
 
 impl Resolve<WriteArgs> for UpdateUserAdmin {
-  #[instrument(name = "UpdateUserAdmin", skip(super_admin))]
+  #[instrument(
+    "UpdateUserAdmin",
+    skip_all,
+    fields(
+      operator = super_admin.id,
+      target_user = self.user_id,
+      admin = self.admin,
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user: super_admin }: &WriteArgs,
@@ -60,7 +69,17 @@ impl Resolve<WriteArgs> for UpdateUserAdmin {
 }
 
 impl Resolve<WriteArgs> for UpdateUserBasePermissions {
-  #[instrument(name = "UpdateUserBasePermissions", skip(admin))]
+  #[instrument(
+    "UpdateUserBasePermissions",
+    skip_all,
+    fields(
+      operator = admin.id,
+      target_user = self.user_id,
+      enabled = self.enabled,
+      create_servers = self.create_servers,
+      create_builds = self.create_builds,
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user: admin }: &WriteArgs,
@@ -117,7 +136,16 @@ impl Resolve<WriteArgs> for UpdateUserBasePermissions {
 }
 
 impl Resolve<WriteArgs> for UpdatePermissionOnResourceType {
-  #[instrument(name = "UpdatePermissionOnResourceType", skip(admin))]
+  #[instrument(
+    "UpdatePermissionOnResourceType",
+    skip_all,
+    fields(
+      operator = admin.id,
+      user_target = format!("{:?}", self.user_target),
+      resource_type = self.resource_type.to_string(),
+      permission = format!("{:?}", self.permission),
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user: admin }: &WriteArgs,
@@ -185,7 +213,17 @@ impl Resolve<WriteArgs> for UpdatePermissionOnResourceType {
 }
 
 impl Resolve<WriteArgs> for UpdatePermissionOnTarget {
-  #[instrument(name = "UpdatePermissionOnTarget", skip(admin))]
+  #[instrument(
+    "UpdatePermissionOnTarget",
+    skip_all,
+    fields(
+      operator = admin.id,
+      user_target = format!("{:?}", self.user_target),
+      resource_type = self.resource_target.extract_variant().to_string(),
+      resource_id = self.resource_target.extract_variant_id().1,
+      permission = format!("{:?}", self.permission),
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user: admin }: &WriteArgs,

bin/core/src/api/write/procedure.rs

@@ -11,7 +11,15 @@ use crate::{permission::get_check_permissions, resource};
 use super::WriteArgs;
 
 impl Resolve<WriteArgs> for CreateProcedure {
-  #[instrument(name = "CreateProcedure", skip(user))]
+  #[instrument(
+    "CreateProcedure",
+    skip_all,
+    fields(
+      operator = user.id,
+      procedure = self.name,
+      config = serde_json::to_string(&self.config).unwrap()
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -22,7 +30,15 @@ impl Resolve<WriteArgs> for CreateProcedure {
 }
 
 impl Resolve<WriteArgs> for CopyProcedure {
-  #[instrument(name = "CopyProcedure", skip(user))]
+  #[instrument(
+    "CopyProcedure",
+    skip_all,
+    fields(
+      operator = user.id,
+      procedure = self.name,
+      copy_procedure = self.id,
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -45,7 +61,15 @@ impl Resolve<WriteArgs> for CopyProcedure {
 }
 
 impl Resolve<WriteArgs> for UpdateProcedure {
-  #[instrument(name = "UpdateProcedure", skip(user))]
+  #[instrument(
+    "UpdateProcedure",
+    skip_all,
+    fields(
+      operator = user.id,
+      procedure = self.id,
+      update = serde_json::to_string(&self.config).unwrap(),
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -58,7 +82,15 @@ impl Resolve<WriteArgs> for UpdateProcedure {
 }
 
 impl Resolve<WriteArgs> for RenameProcedure {
-  #[instrument(name = "RenameProcedure", skip(user))]
+  #[instrument(
+    "RenameProcedure",
+    skip_all,
+    fields(
+      operator = user.id,
+      procedure = self.id,
+      new_name = self.name,
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -71,11 +103,18 @@ impl Resolve<WriteArgs> for RenameProcedure {
 }
 
 impl Resolve<WriteArgs> for DeleteProcedure {
-  #[instrument(name = "DeleteProcedure", skip(args))]
+  #[instrument(
+    "DeleteProcedure",
+    skip_all,
+    fields(
+      operator = user.id,
+      procedure = self.id
+    )
+  )]
   async fn resolve(
     self,
-    args: &WriteArgs,
+    WriteArgs { user }: &WriteArgs,
   ) -> serror::Result<DeleteProcedureResponse> {
-    Ok(resource::delete::<Procedure>(&self.id, args).await?)
+    Ok(resource::delete::<Procedure>(&self.id, user).await?)
   }
 }

bin/core/src/api/write/provider.rs

@@ -10,7 +10,9 @@ use komodo_client::{
     provider::{DockerRegistryAccount, GitProviderAccount},
   },
 };
+use reqwest::StatusCode;
 use resolver_api::Resolve;
+use serror::AddStatusCodeError;
 
 use crate::{
   helpers::update::{add_update, make_update},
@@ -20,25 +22,41 @@ use crate::{
 use super::WriteArgs;
 
 impl Resolve<WriteArgs> for CreateGitProviderAccount {
+  #[instrument(
+    "CreateGitProviderAccount",
+    skip_all,
+    fields(
+      operator = user.id,
+      domain = self.account.domain,
+      username = self.account.username,
+      https = self.account.https.unwrap_or(true),
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
   ) -> serror::Result<CreateGitProviderAccountResponse> {
     if !user.admin {
       return Err(
-        anyhow!("only admins can create git provider accounts")
-          .into(),
+        anyhow!("Only admins can create git provider accounts")
+          .status_code(StatusCode::FORBIDDEN),
       );
     }
 
     let mut account: GitProviderAccount = self.account.into();
 
     if account.domain.is_empty() {
-      return Err(anyhow!("domain cannot be empty string.").into());
+      return Err(
+        anyhow!("Domain cannot be empty string.")
+          .status_code(StatusCode::BAD_REQUEST),
+      );
     }
 
     if account.username.is_empty() {
-      return Err(anyhow!("username cannot be empty string.").into());
+      return Err(
+        anyhow!("Username cannot be empty string.")
+          .status_code(StatusCode::BAD_REQUEST),
+      );
     }
 
     let mut update = make_update(
@@ -51,14 +69,14 @@ impl Resolve<WriteArgs> for CreateGitProviderAccount {
       .git_accounts
       .insert_one(&account)
       .await
-      .context("failed to create git provider account on db")?
+      .context("Failed to create git provider account on db")?
       .inserted_id
       .as_object_id()
-      .context("inserted id is not ObjectId")?
+      .context("Inserted id is not ObjectId")?
       .to_string();
 
     update.push_simple_log(
-      "create git provider account",
+      "Create git provider account",
       format!(
         "Created git provider account for {} with username {}",
         account.domain, account.username
@@ -70,7 +88,7 @@ impl Resolve<WriteArgs> for CreateGitProviderAccount {
     add_update(update)
       .await
       .inspect_err(|e| {
-        error!("failed to add update for create git provider account | {e:#}")
+        error!("Failed to add update for create git provider account | {e:#}")
       })
       .ok();
 
@@ -79,14 +97,25 @@ impl Resolve<WriteArgs> for CreateGitProviderAccount {
 }
 
 impl Resolve<WriteArgs> for UpdateGitProviderAccount {
+  #[instrument(
+    "UpdateGitProviderAccount",
+    skip_all,
+    fields(
+      operator = user.id,
+      id = self.id,
+      domain = self.account.domain,
+      username = self.account.username,
+      https = self.account.https.unwrap_or(true),
+    )
+  )]
   async fn resolve(
     mut self,
     WriteArgs { user }: &WriteArgs,
   ) -> serror::Result<UpdateGitProviderAccountResponse> {
     if !user.admin {
       return Err(
-        anyhow!("only admins can update git provider accounts")
-          .into(),
+        anyhow!("Only admins can update git provider accounts")
+          .status_code(StatusCode::FORBIDDEN),
       );
     }
 
@@ -94,8 +123,8 @@ impl Resolve<WriteArgs> for UpdateGitProviderAccount {
       && domain.is_empty()
     {
       return Err(
-        anyhow!("cannot update git provider with empty domain")
-          .into(),
+        anyhow!("Cannot update git provider with empty domain")
+          .status_code(StatusCode::BAD_REQUEST),
       );
     }
 
@@ -103,8 +132,8 @@ impl Resolve<WriteArgs> for UpdateGitProviderAccount {
       && username.is_empty()
     {
       return Err(
-        anyhow!("cannot update git provider with empty username")
-          .into(),
+        anyhow!("Cannot update git provider with empty username")
+          .status_code(StatusCode::BAD_REQUEST),
       );
     }
 
@@ -118,7 +147,7 @@ impl Resolve<WriteArgs> for UpdateGitProviderAccount {
     );
 
     let account = to_document(&self.account).context(
-      "failed to serialize partial git provider account to bson",
+      "Failed to serialize partial git provider account to bson",
     )?;
     let db = db_client();
     update_one_by_id(
@@ -128,17 +157,17 @@ impl Resolve<WriteArgs> for UpdateGitProviderAccount {
       None,
     )
     .await
-    .context("failed to update git provider account on db")?;
+    .context("Failed to update git provider account on db")?;
 
     let Some(account) = find_one_by_id(&db.git_accounts, &self.id)
       .await
-      .context("failed to query db for git accounts")?
+      .context("Failed to query db for git accounts")?
     else {
-      return Err(anyhow!("no account found with given id").into());
+      return Err(anyhow!("No account found with given id").into());
     };
 
     update.push_simple_log(
-      "update git provider account",
+      "Update git provider account",
       format!(
         "Updated git provider account for {} with username {}",
         account.domain, account.username
@@ -150,7 +179,7 @@ impl Resolve<WriteArgs> for UpdateGitProviderAccount {
     add_update(update)
       .await
       .inspect_err(|e| {
-        error!("failed to add update for update git provider account | {e:#}")
+        error!("Failed to add update for update git provider account | {e:#}")
       })
       .ok();
 
@@ -159,14 +188,22 @@ impl Resolve<WriteArgs> for UpdateGitProviderAccount {
 }
 
 impl Resolve<WriteArgs> for DeleteGitProviderAccount {
+  #[instrument(
+    "DeleteGitProviderAccount",
+    skip_all,
+    fields(
+      operator = user.id,
+      id = self.id,
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
   ) -> serror::Result<DeleteGitProviderAccountResponse> {
     if !user.admin {
       return Err(
-        anyhow!("only admins can delete git provider accounts")
-          .into(),
+        anyhow!("Only admins can delete git provider accounts")
+          .status_code(StatusCode::FORBIDDEN),
       );
     }
 
@@ -179,16 +216,19 @@ impl Resolve<WriteArgs> for DeleteGitProviderAccount {
     let db = db_client();
     let Some(account) = find_one_by_id(&db.git_accounts, &self.id)
       .await
-      .context("failed to query db for git accounts")?
+      .context("Failed to query db for git accounts")?
     else {
-      return Err(anyhow!("no account found with given id").into());
+      return Err(
+        anyhow!("No account found with given id")
+          .status_code(StatusCode::BAD_REQUEST),
+      );
     };
     delete_one_by_id(&db.git_accounts, &self.id, None)
       .await
       .context("failed to delete git account on db")?;
 
     update.push_simple_log(
-      "delete git provider account",
+      "Delete git provider account",
       format!(
         "Deleted git provider account for {} with username {}",
         account.domain, account.username
@@ -200,7 +240,7 @@ impl Resolve<WriteArgs> for DeleteGitProviderAccount {
     add_update(update)
       .await
       .inspect_err(|e| {
-        error!("failed to add update for delete git provider account | {e:#}")
+        error!("Failed to add update for delete git provider account | {e:#}")
       })
       .ok();
 
@@ -209,6 +249,15 @@ impl Resolve<WriteArgs> for DeleteGitProviderAccount {
 }
 
 impl Resolve<WriteArgs> for CreateDockerRegistryAccount {
+  #[instrument(
+    "CreateDockerRegistryAccount",
+    skip_all,
+    fields(
+      operator = user.id,
+      domain = self.account.domain,
+      username = self.account.username,
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -216,20 +265,26 @@ impl Resolve<WriteArgs> for CreateDockerRegistryAccount {
     if !user.admin {
       return Err(
         anyhow!(
-          "only admins can create docker registry account accounts"
+          "Only admins can create docker registry account accounts"
         )
-        .into(),
+        .status_code(StatusCode::FORBIDDEN),
       );
     }
 
     let mut account: DockerRegistryAccount = self.account.into();
 
     if account.domain.is_empty() {
-      return Err(anyhow!("domain cannot be empty string.").into());
+      return Err(
+        anyhow!("Domain cannot be empty string.")
+          .status_code(StatusCode::BAD_REQUEST),
+      );
     }
 
     if account.username.is_empty() {
-      return Err(anyhow!("username cannot be empty string.").into());
+      return Err(
+        anyhow!("Username cannot be empty string.")
+          .status_code(StatusCode::BAD_REQUEST),
+      );
     }
 
     let mut update = make_update(
@@ -243,15 +298,15 @@ impl Resolve<WriteArgs> for CreateDockerRegistryAccount {
       .insert_one(&account)
       .await
       .context(
-        "failed to create docker registry account account on db",
+        "Failed to create docker registry account account on db",
       )?
       .inserted_id
       .as_object_id()
-      .context("inserted id is not ObjectId")?
+      .context("Inserted id is not ObjectId")?
       .to_string();
 
     update.push_simple_log(
-      "create docker registry account",
+      "Create docker registry account",
       format!(
         "Created docker registry account account for {} with username {}",
         account.domain, account.username
@@ -263,7 +318,7 @@ impl Resolve<WriteArgs> for CreateDockerRegistryAccount {
     add_update(update)
       .await
       .inspect_err(|e| {
-        error!("failed to add update for create docker registry account | {e:#}")
+        error!("Failed to add update for create docker registry account | {e:#}")
       })
       .ok();
 
@@ -272,14 +327,24 @@ impl Resolve<WriteArgs> for CreateDockerRegistryAccount {
 }
 
 impl Resolve<WriteArgs> for UpdateDockerRegistryAccount {
+  #[instrument(
+    "UpdateDockerRegistryAccount",
+    skip_all,
+    fields(
+      operator = user.id,
+      id = self.id,
+      domain = self.account.domain,
+      username = self.account.username,
+    )
+  )]
   async fn resolve(
     mut self,
     WriteArgs { user }: &WriteArgs,
   ) -> serror::Result<UpdateDockerRegistryAccountResponse> {
     if !user.admin {
       return Err(
-        anyhow!("only admins can update docker registry accounts")
-          .into(),
+        anyhow!("Only admins can update docker registry accounts")
+          .status_code(StatusCode::FORBIDDEN),
       );
     }
 
@@ -288,9 +353,9 @@ impl Resolve<WriteArgs> for UpdateDockerRegistryAccount {
     {
       return Err(
         anyhow!(
-          "cannot update docker registry account with empty domain"
+          "Cannot update docker registry account with empty domain"
         )
-        .into(),
+        .status_code(StatusCode::BAD_REQUEST),
       );
     }
 
@@ -299,9 +364,9 @@ impl Resolve<WriteArgs> for UpdateDockerRegistryAccount {
     {
       return Err(
         anyhow!(
-          "cannot update docker registry account with empty username"
+          "Cannot update docker registry account with empty username"
         )
-        .into(),
+        .status_code(StatusCode::BAD_REQUEST),
       );
     }
 
@@ -314,7 +379,7 @@ impl Resolve<WriteArgs> for UpdateDockerRegistryAccount {
     );
 
     let account = to_document(&self.account).context(
-      "failed to serialize partial docker registry account account to bson",
+      "Failed to serialize partial docker registry account account to bson",
     )?;
 
     let db = db_client();
@@ -326,19 +391,19 @@ impl Resolve<WriteArgs> for UpdateDockerRegistryAccount {
     )
     .await
     .context(
-      "failed to update docker registry account account on db",
+      "Failed to update docker registry account account on db",
     )?;
 
     let Some(account) =
       find_one_by_id(&db.registry_accounts, &self.id)
         .await
-        .context("failed to query db for registry accounts")?
+        .context("Failed to query db for registry accounts")?
     else {
-      return Err(anyhow!("no account found with given id").into());
+      return Err(anyhow!("No account found with given id").into());
     };
 
     update.push_simple_log(
-      "update docker registry account",
+      "Update docker registry account",
       format!(
         "Updated docker registry account account for {} with username {}",
         account.domain, account.username
@@ -350,7 +415,7 @@ impl Resolve<WriteArgs> for UpdateDockerRegistryAccount {
     add_update(update)
       .await
       .inspect_err(|e| {
-        error!("failed to add update for update docker registry account | {e:#}")
+        error!("Failed to add update for update docker registry account | {e:#}")
       })
       .ok();
 
@@ -359,14 +424,22 @@ impl Resolve<WriteArgs> for UpdateDockerRegistryAccount {
 }
 
 impl Resolve<WriteArgs> for DeleteDockerRegistryAccount {
+  #[instrument(
+    "DeleteDockerRegistryAccount",
+    skip_all,
+    fields(
+      operator = user.id,
+      id = self.id,
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
   ) -> serror::Result<DeleteDockerRegistryAccountResponse> {
     if !user.admin {
       return Err(
-        anyhow!("only admins can delete docker registry accounts")
-          .into(),
+        anyhow!("Only admins can delete docker registry accounts")
+          .status_code(StatusCode::FORBIDDEN),
       );
     }
 
@@ -380,16 +453,19 @@ impl Resolve<WriteArgs> for DeleteDockerRegistryAccount {
     let Some(account) =
       find_one_by_id(&db.registry_accounts, &self.id)
         .await
-        .context("failed to query db for git accounts")?
+        .context("Failed to query db for git accounts")?
     else {
-      return Err(anyhow!("no account found with given id").into());
+      return Err(
+        anyhow!("No account found with given id")
+          .status_code(StatusCode::BAD_REQUEST),
+      );
     };
     delete_one_by_id(&db.registry_accounts, &self.id, None)
       .await
-      .context("failed to delete registry account on db")?;
+      .context("Failed to delete registry account on db")?;
 
     update.push_simple_log(
-      "delete registry account",
+      "Delete registry account",
       format!(
         "Deleted registry account for {} with username {}",
         account.domain, account.username
@@ -401,7 +477,7 @@ impl Resolve<WriteArgs> for DeleteDockerRegistryAccount {
     add_update(update)
       .await
       .inspect_err(|e| {
-        error!("failed to add update for delete docker registry account | {e:#}")
+        error!("Failed to add update for delete docker registry account | {e:#}")
       })
       .ok();
 

bin/core/src/api/write/repo.rs

@@ -1,4 +1,4 @@
-use anyhow::{Context, anyhow};
+use anyhow::Context;
 use database::mongo_indexed::doc;
 use database::mungos::{
   by_id::update_one_by_id, mongodb::bson::to_document,
@@ -7,19 +7,14 @@ use formatting::format_serror;
 use komodo_client::{
   api::write::*,
   entities::{
-    NoData, Operation, RepoExecutionArgs,
-    config::core::CoreConfig,
-    komodo_timestamp,
+    NoData, Operation, RepoExecutionArgs, komodo_timestamp,
     permission::PermissionLevel,
-    repo::{PartialRepoConfig, Repo, RepoInfo},
+    repo::{Repo, RepoInfo},
     server::Server,
     to_path_compatible_name,
     update::{Log, Update},
   },
 };
-use octorust::types::{
-  ReposCreateWebhookRequest, ReposCreateWebhookRequestConfig,
-};
 use periphery_client::api;
 use resolver_api::Resolve;
 
@@ -31,13 +26,21 @@ use crate::{
   },
   permission::get_check_permissions,
   resource,
-  state::{action_states, db_client, github_client},
+  state::{action_states, db_client},
 };
 
 use super::WriteArgs;
 
 impl Resolve<WriteArgs> for CreateRepo {
-  #[instrument(name = "CreateRepo", skip(user))]
+  #[instrument(
+    "CreateRepo",
+    skip_all,
+    fields(
+      operator = user.id,
+      repo = self.name,
+      config = serde_json::to_string(&self.config).unwrap(),
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -48,7 +51,15 @@ impl Resolve<WriteArgs> for CreateRepo {
 }
 
 impl Resolve<WriteArgs> for CopyRepo {
-  #[instrument(name = "CopyRepo", skip(user))]
+  #[instrument(
+    "CopyRepo",
+    skip_all,
+    fields(
+      operator = user.id,
+      repo = self.name,
+      copy_repo = self.id,
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -65,14 +76,32 @@ impl Resolve<WriteArgs> for CopyRepo {
 }
 
 impl Resolve<WriteArgs> for DeleteRepo {
-  #[instrument(name = "DeleteRepo", skip(args))]
-  async fn resolve(self, args: &WriteArgs) -> serror::Result<Repo> {
-    Ok(resource::delete::<Repo>(&self.id, args).await?)
+  #[instrument(
+    "DeleteRepo",
+    skip_all,
+    fields(
+      operator = user.id,
+      repo = self.id,
+    )
+  )]
+  async fn resolve(
+    self,
+    WriteArgs { user }: &WriteArgs,
+  ) -> serror::Result<Repo> {
+    Ok(resource::delete::<Repo>(&self.id, user).await?)
   }
 }
 
 impl Resolve<WriteArgs> for UpdateRepo {
-  #[instrument(name = "UpdateRepo", skip(user))]
+  #[instrument(
+    "UpdateRepo",
+    skip_all,
+    fields(
+      operator = user.id,
+      repo = self.id,
+      update = serde_json::to_string(&self.config).unwrap()
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -82,7 +111,15 @@ impl Resolve<WriteArgs> for UpdateRepo {
 }
 
 impl Resolve<WriteArgs> for RenameRepo {
-  #[instrument(name = "RenameRepo", skip(user))]
+  #[instrument(
+    "RenameRepo",
+    skip_all,
+    fields(
+      operator = user.id,
+      repo = self.id,
+      new_name = self.name
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -159,11 +196,6 @@ impl Resolve<WriteArgs> for RenameRepo {
 }
 
 impl Resolve<WriteArgs> for RefreshRepoCache {
-  #[instrument(
-    name = "RefreshRepoCache",
-    level = "debug",
-    skip(user)
-  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -235,220 +267,3 @@ impl Resolve<WriteArgs> for RefreshRepoCache {
     Ok(NoData {})
   }
 }
-
-impl Resolve<WriteArgs> for CreateRepoWebhook {
-  #[instrument(name = "CreateRepoWebhook", skip(args))]
-  async fn resolve(
-    self,
-    args: &WriteArgs,
-  ) -> serror::Result<CreateRepoWebhookResponse> {
-    let Some(github) = github_client() else {
-      return Err(
-        anyhow!(
-          "github_webhook_app is not configured in core config toml"
-        )
-        .into(),
-      );
-    };
-
-    let repo = get_check_permissions::<Repo>(
-      &self.repo,
-      &args.user,
-      PermissionLevel::Write.into(),
-    )
-    .await?;
-
-    if repo.config.repo.is_empty() {
-      return Err(
-        anyhow!("No repo configured, can't create webhook").into(),
-      );
-    }
-
-    let mut split = repo.config.repo.split('/');
-    let owner = split.next().context("Repo repo has no owner")?;
-
-    let Some(github) = github.get(owner) else {
-      return Err(
-        anyhow!("Cannot manage repo webhooks under owner {owner}")
-          .into(),
-      );
-    };
-
-    let repo_name =
-      split.next().context("Repo repo has no repo after the /")?;
-
-    let github_repos = github.repos();
-
-    // First make sure the webhook isn't already created (inactive ones are ignored)
-    let webhooks = github_repos
-      .list_all_webhooks(owner, repo_name)
-      .await
-      .context("failed to list all webhooks on repo")?
-      .body;
-
-    let CoreConfig {
-      host,
-      webhook_base_url,
-      webhook_secret,
-      ..
-    } = core_config();
-
-    let webhook_secret = if repo.config.webhook_secret.is_empty() {
-      webhook_secret
-    } else {
-      &repo.config.webhook_secret
-    };
-
-    let host = if webhook_base_url.is_empty() {
-      host
-    } else {
-      webhook_base_url
-    };
-    let url = match self.action {
-      RepoWebhookAction::Clone => {
-        format!("{host}/listener/github/repo/{}/clone", repo.id)
-      }
-      RepoWebhookAction::Pull => {
-        format!("{host}/listener/github/repo/{}/pull", repo.id)
-      }
-      RepoWebhookAction::Build => {
-        format!("{host}/listener/github/repo/{}/build", repo.id)
-      }
-    };
-
-    for webhook in webhooks {
-      if webhook.active && webhook.config.url == url {
-        return Ok(NoData {});
-      }
-    }
-
-    // Now good to create the webhook
-    let request = ReposCreateWebhookRequest {
-      active: Some(true),
-      config: Some(ReposCreateWebhookRequestConfig {
-        url,
-        secret: webhook_secret.to_string(),
-        content_type: String::from("json"),
-        insecure_ssl: None,
-        digest: Default::default(),
-        token: Default::default(),
-      }),
-      events: vec![String::from("push")],
-      name: String::from("web"),
-    };
-    github_repos
-      .create_webhook(owner, repo_name, &request)
-      .await
-      .context("failed to create webhook")?;
-
-    if !repo.config.webhook_enabled {
-      UpdateRepo {
-        id: repo.id,
-        config: PartialRepoConfig {
-          webhook_enabled: Some(true),
-          ..Default::default()
-        },
-      }
-      .resolve(args)
-      .await
-      .map_err(|e| e.error)
-      .context("failed to update repo to enable webhook")?;
-    }
-
-    Ok(NoData {})
-  }
-}
-
-impl Resolve<WriteArgs> for DeleteRepoWebhook {
-  #[instrument(name = "DeleteRepoWebhook", skip(user))]
-  async fn resolve(
-    self,
-    WriteArgs { user }: &WriteArgs,
-  ) -> serror::Result<DeleteRepoWebhookResponse> {
-    let Some(github) = github_client() else {
-      return Err(
-        anyhow!(
-          "github_webhook_app is not configured in core config toml"
-        )
-        .into(),
-      );
-    };
-
-    let repo = get_check_permissions::<Repo>(
-      &self.repo,
-      user,
-      PermissionLevel::Write.into(),
-    )
-    .await?;
-
-    if repo.config.git_provider != "github.com" {
-      return Err(
-        anyhow!("Can only manage github.com repo webhooks").into(),
-      );
-    }
-
-    if repo.config.repo.is_empty() {
-      return Err(
-        anyhow!("No repo configured, can't create webhook").into(),
-      );
-    }
-
-    let mut split = repo.config.repo.split('/');
-    let owner = split.next().context("Repo repo has no owner")?;
-
-    let Some(github) = github.get(owner) else {
-      return Err(
-        anyhow!("Cannot manage repo webhooks under owner {owner}")
-          .into(),
-      );
-    };
-
-    let repo_name =
-      split.next().context("Repo repo has no repo after the /")?;
-
-    let github_repos = github.repos();
-
-    // First make sure the webhook isn't already created (inactive ones are ignored)
-    let webhooks = github_repos
-      .list_all_webhooks(owner, repo_name)
-      .await
-      .context("failed to list all webhooks on repo")?
-      .body;
-
-    let CoreConfig {
-      host,
-      webhook_base_url,
-      ..
-    } = core_config();
-
-    let host = if webhook_base_url.is_empty() {
-      host
-    } else {
-      webhook_base_url
-    };
-    let url = match self.action {
-      RepoWebhookAction::Clone => {
-        format!("{host}/listener/github/repo/{}/clone", repo.id)
-      }
-      RepoWebhookAction::Pull => {
-        format!("{host}/listener/github/repo/{}/pull", repo.id)
-      }
-      RepoWebhookAction::Build => {
-        format!("{host}/listener/github/repo/{}/build", repo.id)
-      }
-    };
-
-    for webhook in webhooks {
-      if webhook.active && webhook.config.url == url {
-        github_repos
-          .delete_webhook(owner, repo_name, webhook.id)
-          .await
-          .context("failed to delete webhook")?;
-        return Ok(NoData {});
-      }
-    }
-
-    // No webhook to delete, all good
-    Ok(NoData {})
-  }
-}

bin/core/src/api/write/resource.rs

@@ -1,4 +1,5 @@
 use anyhow::anyhow;
+use derive_variants::ExtractVariant as _;
 use komodo_client::{
   api::write::{UpdateResourceMeta, UpdateResourceMetaResponse},
   entities::{
@@ -7,14 +8,27 @@ use komodo_client::{
     repo::Repo, server::Server, stack::Stack, sync::ResourceSync,
   },
 };
+use reqwest::StatusCode;
 use resolver_api::Resolve;
+use serror::AddStatusCodeError;
 
 use crate::resource::{self, ResourceMetaUpdate};
 
 use super::WriteArgs;
 
 impl Resolve<WriteArgs> for UpdateResourceMeta {
-  #[instrument(name = "UpdateResourceMeta", skip(args))]
+  #[instrument(
+    "UpdateResourceMeta",
+    skip_all,
+    fields(
+      operator = args.user.id,
+      resource_type = self.target.extract_variant().to_string(),
+      resource_id = self.target.extract_variant_id().1,
+      description = self.description,
+      template = self.template,
+      tags = format!("{:?}", self.tags),
+    )
+  )]
   async fn resolve(
     self,
     args: &WriteArgs,
@@ -28,7 +42,7 @@ impl Resolve<WriteArgs> for UpdateResourceMeta {
       ResourceTarget::System(_) => {
         return Err(
           anyhow!("cannot update meta of System resource target")
-            .into(),
+            .status_code(StatusCode::BAD_REQUEST),
         );
       }
       ResourceTarget::Server(id) => {

bin/core/src/api/write/server.rs

@@ -25,7 +25,15 @@ use crate::{
 use super::WriteArgs;
 
 impl Resolve<WriteArgs> for CreateServer {
-  #[instrument(name = "CreateServer", skip(user))]
+  #[instrument(
+    "CreateServer",
+    skip_all,
+    fields(
+      operator = user.id,
+      server = self.name,
+      config = serde_json::to_string(&self.config).unwrap()
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -44,7 +52,15 @@ impl Resolve<WriteArgs> for CreateServer {
 }
 
 impl Resolve<WriteArgs> for CopyServer {
-  #[instrument(name = "CopyServer", skip(user))]
+  #[instrument(
+    "CopyServer",
+    skip_all,
+    fields(
+      operator = user.id,
+      server = self.name,
+      copy_server = self.id,
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -70,14 +86,32 @@ impl Resolve<WriteArgs> for CopyServer {
 }
 
 impl Resolve<WriteArgs> for DeleteServer {
-  #[instrument(name = "DeleteServer", skip(args))]
-  async fn resolve(self, args: &WriteArgs) -> serror::Result<Server> {
-    Ok(resource::delete::<Server>(&self.id, args).await?)
+  #[instrument(
+    "DeleteServer",
+    skip_all,
+    fields(
+      operator = user.id,
+      server = self.id,
+    )
+  )]
+  async fn resolve(
+    self,
+    WriteArgs { user }: &WriteArgs,
+  ) -> serror::Result<Server> {
+    Ok(resource::delete::<Server>(&self.id, user).await?)
   }
 }
 
 impl Resolve<WriteArgs> for UpdateServer {
-  #[instrument(name = "UpdateServer", skip(user))]
+  #[instrument(
+    "UpdateServer",
+    skip_all,
+    fields(
+      operator = user.id,
+      server = self.id,
+      update = serde_json::to_string(&self.config).unwrap(),
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -87,7 +121,15 @@ impl Resolve<WriteArgs> for UpdateServer {
 }
 
 impl Resolve<WriteArgs> for RenameServer {
-  #[instrument(name = "RenameServer", skip(user))]
+  #[instrument(
+    "RenameServer",
+    skip_all,
+    fields(
+      operator = user.id,
+      server = self.id,
+      new_name = self.name,
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -97,7 +139,15 @@ impl Resolve<WriteArgs> for RenameServer {
 }
 
 impl Resolve<WriteArgs> for CreateNetwork {
-  #[instrument(name = "CreateNetwork", skip(user))]
+  #[instrument(
+    "CreateNetwork",
+    skip_all,
+    fields(
+      operator = user.id,
+      server = self.server,
+      network = self.name
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -138,7 +188,17 @@ impl Resolve<WriteArgs> for CreateNetwork {
 }
 
 impl Resolve<WriteArgs> for CreateTerminal {
-  #[instrument(name = "CreateTerminal", skip(user))]
+  #[instrument(
+    "CreateTerminal",
+    skip_all,
+    fields(
+      operator = user.id,
+      server = self.server,
+      terminal = self.name,
+      command = self.command,
+      recreate = format!("{:?}", self.recreate),
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -166,7 +226,15 @@ impl Resolve<WriteArgs> for CreateTerminal {
 }
 
 impl Resolve<WriteArgs> for DeleteTerminal {
-  #[instrument(name = "DeleteTerminal", skip(user))]
+  #[instrument(
+    "DeleteTerminal",
+    skip_all,
+    fields(
+      operator = user.id,
+      server = self.server,
+      terminal = self.terminal,
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -192,7 +260,14 @@ impl Resolve<WriteArgs> for DeleteTerminal {
 }
 
 impl Resolve<WriteArgs> for DeleteAllTerminals {
-  #[instrument(name = "DeleteAllTerminals", skip(user))]
+  #[instrument(
+    "DeleteAllTerminals",
+    skip_all,
+    fields(
+      operator = user.id,
+      server = self.server,
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -218,7 +293,15 @@ impl Resolve<WriteArgs> for DeleteAllTerminals {
 //
 
 impl Resolve<WriteArgs> for UpdateServerPublicKey {
-  #[instrument(name = "UpdateServerPublicKey", skip(args))]
+  #[instrument(
+    "UpdateServerPublicKey",
+    skip_all,
+    fields(
+      operator = args.user.id,
+      server = self.server,
+      public_key = self.public_key,
+    )
+  )]
   async fn resolve(
     self,
     args: &WriteArgs,
@@ -249,7 +332,14 @@ impl Resolve<WriteArgs> for UpdateServerPublicKey {
 //
 
 impl Resolve<WriteArgs> for RotateServerKeys {
-  #[instrument(name = "RotateServerPrivateKey", skip(args))]
+  #[instrument(
+    "RotateServerKeys",
+    skip_all,
+    fields(
+      operator = args.user.id,
+      server = self.server,
+    )
+  )]
   async fn resolve(
     self,
     args: &WriteArgs,

bin/core/src/api/write/service_user.rs

@@ -19,7 +19,15 @@ use crate::{api::user::UserArgs, state::db_client};
 use super::WriteArgs;
 
 impl Resolve<WriteArgs> for CreateServiceUser {
-  #[instrument(name = "CreateServiceUser", skip(user))]
+  #[instrument(
+    "CreateServiceUser",
+    skip_all,
+    fields(
+      operator = user.id,
+      username = self.username,
+      description = self.description,
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -63,7 +71,15 @@ impl Resolve<WriteArgs> for CreateServiceUser {
 }
 
 impl Resolve<WriteArgs> for UpdateServiceUserDescription {
-  #[instrument(name = "UpdateServiceUserDescription", skip(user))]
+  #[instrument(
+    "UpdateServiceUserDescription",
+    skip_all,
+    fields(
+      operator = user.id,
+      username = self.username,
+      description = self.description,
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -99,7 +115,16 @@ impl Resolve<WriteArgs> for UpdateServiceUserDescription {
 }
 
 impl Resolve<WriteArgs> for CreateApiKeyForServiceUser {
-  #[instrument(name = "CreateApiKeyForServiceUser", skip(user))]
+  #[instrument(
+    "CreateApiKeyForServiceUser",
+    skip_all,
+    fields(
+      operator = user.id,
+      service_user = self.user_id,
+      name = self.name,
+      expires = self.expires,
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -125,7 +150,14 @@ impl Resolve<WriteArgs> for CreateApiKeyForServiceUser {
 }
 
 impl Resolve<WriteArgs> for DeleteApiKeyForServiceUser {
-  #[instrument(name = "DeleteApiKeyForServiceUser", skip(user))]
+  #[instrument(
+    "DeleteApiKeyForServiceUser",
+    skip_all,
+    fields(
+      operator = user.id,
+      key = self.key,
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,

bin/core/src/api/write/stack.rs

@@ -8,18 +8,14 @@ use komodo_client::{
   entities::{
     FileContents, NoData, Operation, RepoExecutionArgs,
     all_logs_success,
-    config::core::CoreConfig,
     permission::PermissionLevel,
     repo::Repo,
     server::ServerState,
-    stack::{PartialStackConfig, Stack, StackInfo},
+    stack::{Stack, StackInfo},
     update::Update,
     user::stack_user,
   },
 };
-use octorust::types::{
-  ReposCreateWebhookRequest, ReposCreateWebhookRequestConfig,
-};
 use periphery_client::api::compose::{
   GetComposeContentsOnHost, GetComposeContentsOnHostResponse,
   WriteComposeContentsToHost,
@@ -40,13 +36,21 @@ use crate::{
     remote::{RemoteComposeContents, get_repo_compose_contents},
     services::extract_services_into_res,
   },
-  state::{db_client, github_client},
+  state::db_client,
 };
 
 use super::WriteArgs;
 
 impl Resolve<WriteArgs> for CreateStack {
-  #[instrument(name = "CreateStack", skip(user))]
+  #[instrument(
+    "CreateStack",
+    skip_all,
+    fields(
+      operator = user.id,
+      stack = self.name,
+      config = serde_json::to_string(&self.config).unwrap(),
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -57,7 +61,15 @@ impl Resolve<WriteArgs> for CreateStack {
 }
 
 impl Resolve<WriteArgs> for CopyStack {
-  #[instrument(name = "CopyStack", skip(user))]
+  #[instrument(
+    "CopyStack",
+    skip_all,
+    fields(
+      operator = user.id,
+      stack = self.name,
+      copy_stack = self.id,
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -75,14 +87,32 @@ impl Resolve<WriteArgs> for CopyStack {
 }
 
 impl Resolve<WriteArgs> for DeleteStack {
-  #[instrument(name = "DeleteStack", skip(args))]
-  async fn resolve(self, args: &WriteArgs) -> serror::Result<Stack> {
-    Ok(resource::delete::<Stack>(&self.id, args).await?)
+  #[instrument(
+    "DeleteStack",
+    skip_all,
+    fields(
+      operator = user.id,
+      stack = self.id,
+    )
+  )]
+  async fn resolve(
+    self,
+    WriteArgs { user }: &WriteArgs,
+  ) -> serror::Result<Stack> {
+    Ok(resource::delete::<Stack>(&self.id, user).await?)
   }
 }
 
 impl Resolve<WriteArgs> for UpdateStack {
-  #[instrument(name = "UpdateStack", skip(user))]
+  #[instrument(
+    "UpdateStack",
+    skip_all,
+    fields(
+      operator = user.id,
+      stack = self.id,
+      update = serde_json::to_string(&self.config).unwrap(),
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -92,7 +122,15 @@ impl Resolve<WriteArgs> for UpdateStack {
 }
 
 impl Resolve<WriteArgs> for RenameStack {
-  #[instrument(name = "RenameStack", skip(user))]
+  #[instrument(
+    "RenameStack",
+    skip_all,
+    fields(
+      operator = user.id,
+      stack = self.id,
+      new_name = self.name
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -102,7 +140,15 @@ impl Resolve<WriteArgs> for RenameStack {
 }
 
 impl Resolve<WriteArgs> for WriteStackFileContents {
-  #[instrument(name = "WriteStackFileContents", skip(user))]
+  #[instrument(
+    "WriteStackFileContents",
+    skip_all,
+    fields(
+      operator = user.id,
+      stack = self.stack,
+      path = self.file_path,
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -151,6 +197,7 @@ impl Resolve<WriteArgs> for WriteStackFileContents {
   }
 }
 
+#[instrument("WriteStackFileContentsOnHost", skip_all)]
 async fn write_stack_file_contents_on_host(
   stack: Stack,
   file_path: String,
@@ -223,6 +270,7 @@ async fn write_stack_file_contents_on_host(
   Ok(update)
 }
 
+#[instrument("WriteStackFileContentsGit", skip_all)]
 async fn write_stack_file_contents_git(
   mut stack: Stack,
   file_path: &str,
@@ -364,11 +412,6 @@ async fn write_stack_file_contents_git(
 }
 
 impl Resolve<WriteArgs> for RefreshStackCache {
-  #[instrument(
-    name = "RefreshStackCache",
-    level = "debug",
-    skip(user)
-  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -567,216 +610,3 @@ impl Resolve<WriteArgs> for RefreshStackCache {
     Ok(NoData {})
   }
 }
-
-impl Resolve<WriteArgs> for CreateStackWebhook {
-  #[instrument(name = "CreateStackWebhook", skip(args))]
-  async fn resolve(
-    self,
-    args: &WriteArgs,
-  ) -> serror::Result<CreateStackWebhookResponse> {
-    let WriteArgs { user } = args;
-
-    let Some(github) = github_client() else {
-      return Err(
-        anyhow!(
-          "github_webhook_app is not configured in core config toml"
-        )
-        .into(),
-      );
-    };
-
-    let stack = get_check_permissions::<Stack>(
-      &self.stack,
-      user,
-      PermissionLevel::Write.into(),
-    )
-    .await?;
-
-    if stack.config.repo.is_empty() {
-      return Err(
-        anyhow!("No repo configured, can't create webhook").into(),
-      );
-    }
-
-    let mut split = stack.config.repo.split('/');
-    let owner = split.next().context("Stack repo has no owner")?;
-
-    let Some(github) = github.get(owner) else {
-      return Err(
-        anyhow!("Cannot manage repo webhooks under owner {owner}")
-          .into(),
-      );
-    };
-
-    let repo =
-      split.next().context("Stack repo has no repo after the /")?;
-
-    let github_repos = github.repos();
-
-    // First make sure the webhook isn't already created (inactive ones are ignored)
-    let webhooks = github_repos
-      .list_all_webhooks(owner, repo)
-      .await
-      .context("failed to list all webhooks on repo")?
-      .body;
-
-    let CoreConfig {
-      host,
-      webhook_base_url,
-      webhook_secret,
-      ..
-    } = core_config();
-
-    let webhook_secret = if stack.config.webhook_secret.is_empty() {
-      webhook_secret
-    } else {
-      &stack.config.webhook_secret
-    };
-
-    let host = if webhook_base_url.is_empty() {
-      host
-    } else {
-      webhook_base_url
-    };
-    let url = match self.action {
-      StackWebhookAction::Refresh => {
-        format!("{host}/listener/github/stack/{}/refresh", stack.id)
-      }
-      StackWebhookAction::Deploy => {
-        format!("{host}/listener/github/stack/{}/deploy", stack.id)
-      }
-    };
-
-    for webhook in webhooks {
-      if webhook.active && webhook.config.url == url {
-        return Ok(NoData {});
-      }
-    }
-
-    // Now good to create the webhook
-    let request = ReposCreateWebhookRequest {
-      active: Some(true),
-      config: Some(ReposCreateWebhookRequestConfig {
-        url,
-        secret: webhook_secret.to_string(),
-        content_type: String::from("json"),
-        insecure_ssl: None,
-        digest: Default::default(),
-        token: Default::default(),
-      }),
-      events: vec![String::from("push")],
-      name: String::from("web"),
-    };
-    github_repos
-      .create_webhook(owner, repo, &request)
-      .await
-      .context("failed to create webhook")?;
-
-    if !stack.config.webhook_enabled {
-      UpdateStack {
-        id: stack.id,
-        config: PartialStackConfig {
-          webhook_enabled: Some(true),
-          ..Default::default()
-        },
-      }
-      .resolve(args)
-      .await
-      .map_err(|e| e.error)
-      .context("failed to update stack to enable webhook")?;
-    }
-
-    Ok(NoData {})
-  }
-}
-
-impl Resolve<WriteArgs> for DeleteStackWebhook {
-  #[instrument(name = "DeleteStackWebhook", skip(user))]
-  async fn resolve(
-    self,
-    WriteArgs { user }: &WriteArgs,
-  ) -> serror::Result<DeleteStackWebhookResponse> {
-    let Some(github) = github_client() else {
-      return Err(
-        anyhow!(
-          "github_webhook_app is not configured in core config toml"
-        )
-        .into(),
-      );
-    };
-
-    let stack = get_check_permissions::<Stack>(
-      &self.stack,
-      user,
-      PermissionLevel::Write.into(),
-    )
-    .await?;
-
-    if stack.config.git_provider != "github.com" {
-      return Err(
-        anyhow!("Can only manage github.com repo webhooks").into(),
-      );
-    }
-
-    if stack.config.repo.is_empty() {
-      return Err(
-        anyhow!("No repo configured, can't create webhook").into(),
-      );
-    }
-
-    let mut split = stack.config.repo.split('/');
-    let owner = split.next().context("Stack repo has no owner")?;
-
-    let Some(github) = github.get(owner) else {
-      return Err(
-        anyhow!("Cannot manage repo webhooks under owner {owner}")
-          .into(),
-      );
-    };
-
-    let repo =
-      split.next().context("Sync repo has no repo after the /")?;
-
-    let github_repos = github.repos();
-
-    // First make sure the webhook isn't already created (inactive ones are ignored)
-    let webhooks = github_repos
-      .list_all_webhooks(owner, repo)
-      .await
-      .context("failed to list all webhooks on repo")?
-      .body;
-
-    let CoreConfig {
-      host,
-      webhook_base_url,
-      ..
-    } = core_config();
-
-    let host = if webhook_base_url.is_empty() {
-      host
-    } else {
-      webhook_base_url
-    };
-    let url = match self.action {
-      StackWebhookAction::Refresh => {
-        format!("{host}/listener/github/stack/{}/refresh", stack.id)
-      }
-      StackWebhookAction::Deploy => {
-        format!("{host}/listener/github/stack/{}/deploy", stack.id)
-      }
-    };
-
-    for webhook in webhooks {
-      if webhook.active && webhook.config.url == url {
-        github_repos
-          .delete_webhook(owner, repo, webhook.id)
-          .await
-          .context("failed to delete webhook")?;
-        return Ok(NoData {});
-      }
-    }
-
-    // No webhook to delete, all good
-    Ok(NoData {})
-  }
-}

bin/core/src/api/write/sync.rs

@@ -12,14 +12,13 @@ use formatting::format_serror;
 use komodo_client::{
   api::{read::ExportAllResourcesToToml, write::*},
   entities::{
-    self, NoData, Operation, RepoExecutionArgs, ResourceTarget,
+    self, Operation, RepoExecutionArgs, ResourceTarget,
     action::Action,
     alert::{Alert, AlertData, SeverityLevel},
     alerter::Alerter,
     all_logs_success,
     build::Build,
     builder::Builder,
-    config::core::CoreConfig,
     deployment::Deployment,
     komodo_timestamp,
     permission::PermissionLevel,
@@ -27,19 +26,14 @@ use komodo_client::{
     repo::Repo,
     server::Server,
     stack::Stack,
-    sync::{
-      PartialResourceSyncConfig, ResourceSync, ResourceSyncInfo,
-      SyncDeployUpdate,
-    },
+    sync::{ResourceSync, ResourceSyncInfo, SyncDeployUpdate},
     to_path_compatible_name,
     update::{Log, Update},
     user::sync_user,
   },
 };
-use octorust::types::{
-  ReposCreateWebhookRequest, ReposCreateWebhookRequestConfig,
-};
 use resolver_api::Resolve;
+use tracing::Instrument;
 
 use crate::{
   alert::send_alerts,
@@ -53,7 +47,7 @@ use crate::{
   },
   permission::get_check_permissions,
   resource,
-  state::{db_client, github_client},
+  state::db_client,
   sync::{
     deploy::SyncDeployParams, remote::RemoteResources,
     view::push_updates_for_view,
@@ -63,7 +57,15 @@ use crate::{
 use super::WriteArgs;
 
 impl Resolve<WriteArgs> for CreateResourceSync {
-  #[instrument(name = "CreateResourceSync", skip(user))]
+  #[instrument(
+    "CreateResourceSync",
+    skip_all,
+    fields(
+      operator = user.id,
+      sync = self.name,
+      config = serde_json::to_string(&self.config).unwrap(),
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -79,7 +81,15 @@ impl Resolve<WriteArgs> for CreateResourceSync {
 }
 
 impl Resolve<WriteArgs> for CopyResourceSync {
-  #[instrument(name = "CopyResourceSync", skip(user))]
+  #[instrument(
+    "CopyResourceSync",
+    skip_all,
+    fields(
+      operator = user.id,
+      sync = self.name,
+      copy_sync = self.id,
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -102,17 +112,32 @@ impl Resolve<WriteArgs> for CopyResourceSync {
 }
 
 impl Resolve<WriteArgs> for DeleteResourceSync {
-  #[instrument(name = "DeleteResourceSync", skip(args))]
+  #[instrument(
+    "DeleteResourceSync",
+    skip_all,
+    fields(
+      operator = user.id,
+      sync = self.id,
+    )
+  )]
   async fn resolve(
     self,
-    args: &WriteArgs,
+    WriteArgs { user }: &WriteArgs,
   ) -> serror::Result<ResourceSync> {
-    Ok(resource::delete::<ResourceSync>(&self.id, args).await?)
+    Ok(resource::delete::<ResourceSync>(&self.id, user).await?)
   }
 }
 
 impl Resolve<WriteArgs> for UpdateResourceSync {
-  #[instrument(name = "UpdateResourceSync", skip(user))]
+  #[instrument(
+    "UpdateResourceSync",
+    skip_all,
+    fields(
+      operator = user.id,
+      sync = self.id,
+      update = serde_json::to_string(&self.config).unwrap(),
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -125,7 +150,15 @@ impl Resolve<WriteArgs> for UpdateResourceSync {
 }
 
 impl Resolve<WriteArgs> for RenameResourceSync {
-  #[instrument(name = "RenameResourceSync", skip(user))]
+  #[instrument(
+    "RenameResourceSync",
+    skip_all,
+    fields(
+      operator = user.id,
+      sync = self.id,
+      new_name = self.name
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -138,7 +171,16 @@ impl Resolve<WriteArgs> for RenameResourceSync {
 }
 
 impl Resolve<WriteArgs> for WriteSyncFileContents {
-  #[instrument(name = "WriteSyncFileContents", skip(args))]
+  #[instrument(
+    "WriteSyncFileContents",
+    skip_all,
+    fields(
+      operator = args.user.id,
+      sync = self.sync,
+      resource_path = self.resource_path,
+      file_path = self.file_path,
+    )
+  )]
   async fn resolve(self, args: &WriteArgs) -> serror::Result<Update> {
     let sync = get_check_permissions::<ResourceSync>(
       &self.sync,
@@ -183,6 +225,7 @@ impl Resolve<WriteArgs> for WriteSyncFileContents {
   }
 }
 
+#[instrument("WriteSyncFileContentsOnHost", skip_all)]
 async fn write_sync_file_contents_on_host(
   req: WriteSyncFileContents,
   args: &WriteArgs,
@@ -206,15 +249,7 @@ async fn write_sync_file_contents_on_host(
     .context("Invalid resource path")?;
   let full_path = root.join(&resource_path).join(&file_path);
 
-  if let Some(parent) = full_path.parent() {
-    tokio::fs::create_dir_all(parent).await.with_context(|| {
-      format!(
-        "Failed to initialize resource file parent directory {parent:?}"
-      )
-    })?;
-  }
-
-  if let Err(e) = tokio::fs::write(&full_path, &contents)
+  if let Err(e) = secret_file::write_async(&full_path, &contents)
     .await
     .with_context(|| {
       format!(
@@ -253,6 +288,7 @@ async fn write_sync_file_contents_on_host(
   Ok(update)
 }
 
+#[instrument("WriteSyncFileContentsGit", skip_all)]
 async fn write_sync_file_contents_git(
   req: WriteSyncFileContents,
   args: &WriteArgs,
@@ -404,7 +440,14 @@ async fn write_sync_file_contents_git(
 }
 
 impl Resolve<WriteArgs> for CommitSync {
-  #[instrument(name = "CommitSync", skip(args))]
+  #[instrument(
+    "CommitSync",
+    skip_all,
+    fields(
+      operator = args.user.id,
+      sync = self.sync,
+    )
+  )]
   async fn resolve(self, args: &WriteArgs) -> serror::Result<Update> {
     let WriteArgs { user } = args;
 
@@ -491,12 +534,9 @@ impl Resolve<WriteArgs> for CommitSync {
         .sync_directory
         .join(to_path_compatible_name(&sync.name))
         .join(&resource_path);
-      if let Some(parent) = file_path.parent() {
-        tokio::fs::create_dir_all(parent)
-          .await
-          .with_context(|| format!("Failed to initialize resource file parent directory {parent:?}"))?;
-      };
-      if let Err(e) = tokio::fs::write(&file_path, &res.toml)
+      let span = info_span!("CommitSyncOnHost");
+      if let Err(e) = secret_file::write_async(&file_path, &res.toml)
+        .instrument(span)
         .await
         .with_context(|| {
           format!("Failed to write resource file to {file_path:?}",)
@@ -589,6 +629,7 @@ impl Resolve<WriteArgs> for CommitSync {
   }
 }
 
+#[instrument("CommitSyncGit", skip_all)]
 async fn commit_git_sync(
   mut args: RepoExecutionArgs,
   resource_path: &Path,
@@ -633,11 +674,6 @@ async fn commit_git_sync(
 }
 
 impl Resolve<WriteArgs> for RefreshResourceSyncPending {
-  #[instrument(
-    name = "RefreshResourceSyncPending",
-    level = "debug",
-    skip(user)
-  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -982,215 +1018,3 @@ impl Resolve<WriteArgs> for RefreshResourceSyncPending {
     Ok(crate::resource::get::<ResourceSync>(&sync.id).await?)
   }
 }
-
-impl Resolve<WriteArgs> for CreateSyncWebhook {
-  #[instrument(name = "CreateSyncWebhook", skip(args))]
-  async fn resolve(
-    self,
-    args: &WriteArgs,
-  ) -> serror::Result<CreateSyncWebhookResponse> {
-    let WriteArgs { user } = args;
-    let Some(github) = github_client() else {
-      return Err(
-        anyhow!(
-          "github_webhook_app is not configured in core config toml"
-        )
-        .into(),
-      );
-    };
-
-    let sync = get_check_permissions::<ResourceSync>(
-      &self.sync,
-      user,
-      PermissionLevel::Write.into(),
-    )
-    .await?;
-
-    if sync.config.repo.is_empty() {
-      return Err(
-        anyhow!("No repo configured, can't create webhook").into(),
-      );
-    }
-
-    let mut split = sync.config.repo.split('/');
-    let owner = split.next().context("Sync repo has no owner")?;
-
-    let Some(github) = github.get(owner) else {
-      return Err(
-        anyhow!("Cannot manage repo webhooks under owner {owner}")
-          .into(),
-      );
-    };
-
-    let repo =
-      split.next().context("Repo repo has no repo after the /")?;
-
-    let github_repos = github.repos();
-
-    // First make sure the webhook isn't already created (inactive ones are ignored)
-    let webhooks = github_repos
-      .list_all_webhooks(owner, repo)
-      .await
-      .context("failed to list all webhooks on repo")?
-      .body;
-
-    let CoreConfig {
-      host,
-      webhook_base_url,
-      webhook_secret,
-      ..
-    } = core_config();
-
-    let webhook_secret = if sync.config.webhook_secret.is_empty() {
-      webhook_secret
-    } else {
-      &sync.config.webhook_secret
-    };
-
-    let host = if webhook_base_url.is_empty() {
-      host
-    } else {
-      webhook_base_url
-    };
-    let url = match self.action {
-      SyncWebhookAction::Refresh => {
-        format!("{host}/listener/github/sync/{}/refresh", sync.id)
-      }
-      SyncWebhookAction::Sync => {
-        format!("{host}/listener/github/sync/{}/sync", sync.id)
-      }
-    };
-
-    for webhook in webhooks {
-      if webhook.active && webhook.config.url == url {
-        return Ok(NoData {});
-      }
-    }
-
-    // Now good to create the webhook
-    let request = ReposCreateWebhookRequest {
-      active: Some(true),
-      config: Some(ReposCreateWebhookRequestConfig {
-        url,
-        secret: webhook_secret.to_string(),
-        content_type: String::from("json"),
-        insecure_ssl: None,
-        digest: Default::default(),
-        token: Default::default(),
-      }),
-      events: vec![String::from("push")],
-      name: String::from("web"),
-    };
-    github_repos
-      .create_webhook(owner, repo, &request)
-      .await
-      .context("failed to create webhook")?;
-
-    if !sync.config.webhook_enabled {
-      UpdateResourceSync {
-        id: sync.id,
-        config: PartialResourceSyncConfig {
-          webhook_enabled: Some(true),
-          ..Default::default()
-        },
-      }
-      .resolve(args)
-      .await
-      .map_err(|e| e.error)
-      .context("failed to update sync to enable webhook")?;
-    }
-
-    Ok(NoData {})
-  }
-}
-
-impl Resolve<WriteArgs> for DeleteSyncWebhook {
-  #[instrument(name = "DeleteSyncWebhook", skip(user))]
-  async fn resolve(
-    self,
-    WriteArgs { user }: &WriteArgs,
-  ) -> serror::Result<DeleteSyncWebhookResponse> {
-    let Some(github) = github_client() else {
-      return Err(
-        anyhow!(
-          "github_webhook_app is not configured in core config toml"
-        )
-        .into(),
-      );
-    };
-
-    let sync = get_check_permissions::<ResourceSync>(
-      &self.sync,
-      user,
-      PermissionLevel::Write.into(),
-    )
-    .await?;
-
-    if sync.config.git_provider != "github.com" {
-      return Err(
-        anyhow!("Can only manage github.com repo webhooks").into(),
-      );
-    }
-
-    if sync.config.repo.is_empty() {
-      return Err(
-        anyhow!("No repo configured, can't create webhook").into(),
-      );
-    }
-
-    let mut split = sync.config.repo.split('/');
-    let owner = split.next().context("Sync repo has no owner")?;
-
-    let Some(github) = github.get(owner) else {
-      return Err(
-        anyhow!("Cannot manage repo webhooks under owner {owner}")
-          .into(),
-      );
-    };
-
-    let repo =
-      split.next().context("Sync repo has no repo after the /")?;
-
-    let github_repos = github.repos();
-
-    // First make sure the webhook isn't already created (inactive ones are ignored)
-    let webhooks = github_repos
-      .list_all_webhooks(owner, repo)
-      .await
-      .context("failed to list all webhooks on repo")?
-      .body;
-
-    let CoreConfig {
-      host,
-      webhook_base_url,
-      ..
-    } = core_config();
-
-    let host = if webhook_base_url.is_empty() {
-      host
-    } else {
-      webhook_base_url
-    };
-    let url = match self.action {
-      SyncWebhookAction::Refresh => {
-        format!("{host}/listener/github/sync/{}/refresh", sync.id)
-      }
-      SyncWebhookAction::Sync => {
-        format!("{host}/listener/github/sync/{}/sync", sync.id)
-      }
-    };
-
-    for webhook in webhooks {
-      if webhook.active && webhook.config.url == url {
-        github_repos
-          .delete_webhook(owner, repo, webhook.id)
-          .await
-          .context("failed to delete webhook")?;
-        return Ok(NoData {});
-      }
-    }
-
-    // No webhook to delete, all good
-    Ok(NoData {})
-  }
-}

bin/core/src/api/write/tag.rs

@@ -27,7 +27,15 @@ use crate::{
 use super::WriteArgs;
 
 impl Resolve<WriteArgs> for CreateTag {
-  #[instrument(name = "CreateTag", skip(user))]
+  #[instrument(
+    "CreateTag",
+    skip_all,
+    fields(
+      operator = user.id,
+      tag = self.name,
+      color = format!("{:?}", self.color),
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -68,7 +76,15 @@ impl Resolve<WriteArgs> for CreateTag {
 }
 
 impl Resolve<WriteArgs> for RenameTag {
-  #[instrument(name = "RenameTag", skip(user))]
+  #[instrument(
+    "RenameTag",
+    skip_all,
+    fields(
+      operator = user.id,
+      tag = self.id,
+      new_name = self.name,
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -93,7 +109,15 @@ impl Resolve<WriteArgs> for RenameTag {
 }
 
 impl Resolve<WriteArgs> for UpdateTagColor {
-  #[instrument(name = "UpdateTagColor", skip(user))]
+  #[instrument(
+    "UpdateTagColor",
+    skip_all,
+    fields(
+      operator = user.id,
+      tag = self.tag,
+      color = format!("{:?}", self.color),
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -114,7 +138,14 @@ impl Resolve<WriteArgs> for UpdateTagColor {
 }
 
 impl Resolve<WriteArgs> for DeleteTag {
-  #[instrument(name = "DeleteTag", skip(user))]
+  #[instrument(
+    "DeleteTag",
+    skip_all,
+    fields(
+      operator = user.id,
+      tag_id = self.id,
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,

bin/core/src/api/write/user.rs

@@ -24,7 +24,14 @@ use super::WriteArgs;
 //
 
 impl Resolve<WriteArgs> for CreateLocalUser {
-  #[instrument(name = "CreateLocalUser", skip(admin, self), fields(admin_id = admin.id, username = self.username))]
+  #[instrument(
+    "CreateLocalUser",
+    skip_all,
+    fields(
+      admin_id = admin.id,
+      username = self.username
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user: admin }: &WriteArgs,
@@ -101,7 +108,14 @@ impl Resolve<WriteArgs> for CreateLocalUser {
 //
 
 impl Resolve<WriteArgs> for UpdateUserUsername {
-  #[instrument(name = "UpdateUserUsername", skip(user), fields(user_id = user.id))]
+  #[instrument(
+    "UpdateUserUsername",
+    skip_all,
+    fields(
+      operator = user.id,
+      new_username = self.username,
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -152,7 +166,11 @@ impl Resolve<WriteArgs> for UpdateUserUsername {
 //
 
 impl Resolve<WriteArgs> for UpdateUserPassword {
-  #[instrument(name = "UpdateUserPassword", skip(user, self), fields(user_id = user.id))]
+  #[instrument(
+    "UpdateUserPassword",
+    skip_all,
+    fields(operator = user.id)
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -175,7 +193,14 @@ impl Resolve<WriteArgs> for UpdateUserPassword {
 //
 
 impl Resolve<WriteArgs> for DeleteUser {
-  #[instrument(name = "DeleteUser", skip(admin), fields(user = self.user))]
+  #[instrument(
+    "DeleteUser",
+    skip_all,
+    fields(
+      admin_id = admin.id,
+      user_to_delete = self.user
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user: admin }: &WriteArgs,

bin/core/src/api/write/user_group.rs

@@ -19,7 +19,14 @@ use crate::state::db_client;
 use super::WriteArgs;
 
 impl Resolve<WriteArgs> for CreateUserGroup {
-  #[instrument(name = "CreateUserGroup", skip(admin), fields(admin = admin.username))]
+  #[instrument(
+    "CreateUserGroup",
+    skip_all,
+    fields(
+      operator = admin.id,
+      group = self.name,
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user: admin }: &WriteArgs,
@@ -57,7 +64,15 @@ impl Resolve<WriteArgs> for CreateUserGroup {
 }
 
 impl Resolve<WriteArgs> for RenameUserGroup {
-  #[instrument(name = "RenameUserGroup", skip(admin), fields(admin = admin.username))]
+  #[instrument(
+    "RenameUserGroup",
+    skip_all,
+    fields(
+      operator = admin.id,
+      group = self.id,
+      new_name = self.name,
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user: admin }: &WriteArgs,
@@ -86,7 +101,14 @@ impl Resolve<WriteArgs> for RenameUserGroup {
 }
 
 impl Resolve<WriteArgs> for DeleteUserGroup {
-  #[instrument(name = "DeleteUserGroup", skip(admin), fields(admin = admin.username))]
+  #[instrument(
+    "DeleteUserGroup",
+    skip_all,
+    fields(
+      operator = admin.id,
+      group = self.id,
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user: admin }: &WriteArgs,
@@ -122,7 +144,15 @@ impl Resolve<WriteArgs> for DeleteUserGroup {
 }
 
 impl Resolve<WriteArgs> for AddUserToUserGroup {
-  #[instrument(name = "AddUserToUserGroup", skip(admin), fields(admin = admin.username))]
+  #[instrument(
+    "AddUserToUserGroup",
+    skip_all,
+    fields(
+      operator = admin.id,
+      group = self.user_group,
+      user = self.user,
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user: admin }: &WriteArgs,
@@ -169,7 +199,15 @@ impl Resolve<WriteArgs> for AddUserToUserGroup {
 }
 
 impl Resolve<WriteArgs> for RemoveUserFromUserGroup {
-  #[instrument(name = "RemoveUserFromUserGroup", skip(admin), fields(admin = admin.username))]
+  #[instrument(
+    "RemoveUserFromUserGroup",
+    skip_all,
+    fields(
+      operator = admin.id,
+      group = self.user_group,
+      user = self.user,
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user: admin }: &WriteArgs,
@@ -216,7 +254,15 @@ impl Resolve<WriteArgs> for RemoveUserFromUserGroup {
 }
 
 impl Resolve<WriteArgs> for SetUsersInUserGroup {
-  #[instrument(name = "SetUsersInUserGroup", skip(admin), fields(admin = admin.username))]
+  #[instrument(
+    "SetUsersInUserGroup",
+    skip_all,
+    fields(
+      operator = admin.id,
+      group = self.user_group,
+      users = format!("{:?}", self.users)
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user: admin }: &WriteArgs,
@@ -266,7 +312,15 @@ impl Resolve<WriteArgs> for SetUsersInUserGroup {
 }
 
 impl Resolve<WriteArgs> for SetEveryoneUserGroup {
-  #[instrument(name = "SetEveryoneUserGroup", skip(admin), fields(admin = admin.username))]
+  #[instrument(
+    "SetEveryoneUserGroup",
+    skip_all,
+    fields(
+      operator = admin.id,
+      group = self.user_group,
+      everyone = self.everyone,
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user: admin }: &WriteArgs,

bin/core/src/api/write/variable.rs

@@ -19,7 +19,16 @@ use crate::{
 use super::WriteArgs;
 
 impl Resolve<WriteArgs> for CreateVariable {
-  #[instrument(name = "CreateVariable", skip(user, self), fields(name = &self.name))]
+  #[instrument(
+    "CreateVariable",
+    skip_all,
+    fields(
+      operator = user.id,
+      variable = self.name,
+      description = self.description,
+      is_secret = self.is_secret,
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -68,7 +77,14 @@ impl Resolve<WriteArgs> for CreateVariable {
 }
 
 impl Resolve<WriteArgs> for UpdateVariableValue {
-  #[instrument(name = "UpdateVariableValue", skip(user, self), fields(name = &self.name))]
+  #[instrument(
+    "UpdateVariableValue",
+    skip_all,
+    fields(
+      operator = user.id,
+      variable = self.name,
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -125,7 +141,15 @@ impl Resolve<WriteArgs> for UpdateVariableValue {
 }
 
 impl Resolve<WriteArgs> for UpdateVariableDescription {
-  #[instrument(name = "UpdateVariableDescription", skip(user))]
+  #[instrument(
+    "UpdateVariableDescription",
+    skip_all,
+    fields(
+      operator = user.id,
+      variable = self.name,
+      description = self.description,
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -149,7 +173,15 @@ impl Resolve<WriteArgs> for UpdateVariableDescription {
 }
 
 impl Resolve<WriteArgs> for UpdateVariableIsSecret {
-  #[instrument(name = "UpdateVariableIsSecret", skip(user))]
+  #[instrument(
+    "UpdateVariableIsSecret",
+    skip_all,
+    fields(
+      operator = user.id,
+      variable = self.name,
+      is_secret = self.is_secret,
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -173,6 +205,14 @@ impl Resolve<WriteArgs> for UpdateVariableIsSecret {
 }
 
 impl Resolve<WriteArgs> for DeleteVariable {
+  #[instrument(
+    "DeleteVariable",
+    skip_all,
+    fields(
+      operator = user.id,
+      variable = self.name,
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,

bin/core/src/auth/github/client.rs

@@ -1,17 +1,15 @@
 use std::sync::OnceLock;
 
 use anyhow::{Context, anyhow};
-use komodo_client::entities::config::core::{
-  CoreConfig, OauthCredentials,
+use komodo_client::entities::{
+  config::core::{CoreConfig, OauthCredentials},
+  random_string,
 };
 use reqwest::StatusCode;
 use serde::{Deserialize, Serialize, de::DeserializeOwned};
 use tokio::sync::Mutex;
 
-use crate::{
-  auth::STATE_PREFIX_LENGTH, config::core_config,
-  helpers::random_string,
-};
+use crate::{auth::STATE_PREFIX_LENGTH, config::core_config};
 
 pub fn github_oauth_client() -> &'static Option<GithubOauthClient> {
   static GITHUB_OAUTH_CLIENT: OnceLock<Option<GithubOauthClient>> =
@@ -76,7 +74,6 @@ impl GithubOauthClient {
     .into()
   }
 
-  #[instrument(level = "debug", skip(self))]
   pub async fn get_login_redirect_url(
     &self,
     redirect: Option<String>,
@@ -95,7 +92,6 @@ impl GithubOauthClient {
     redirect_url
   }
 
-  #[instrument(level = "debug", skip(self))]
   pub async fn check_state(&self, state: &str) -> bool {
     let mut contained = false;
     self.states.lock().await.retain(|s| {
@@ -109,7 +105,6 @@ impl GithubOauthClient {
     contained
   }
 
-  #[instrument(level = "debug", skip(self))]
   pub async fn get_access_token(
     &self,
     code: &str,
@@ -130,7 +125,6 @@ impl GithubOauthClient {
       .context("failed to get github access token using code")
   }
 
-  #[instrument(level = "debug", skip(self))]
   pub async fn get_github_user(
     &self,
     token: &str,
@@ -141,7 +135,6 @@ impl GithubOauthClient {
       .context("failed to get github user using access token")
   }
 
-  #[instrument(level = "debug", skip(self))]
   async fn get<R: DeserializeOwned>(
     &self,
     endpoint: &str,

bin/core/src/auth/github/mod.rs

@@ -5,7 +5,7 @@ use axum::{
 use database::mongo_indexed::Document;
 use database::mungos::mongodb::bson::doc;
 use komodo_client::entities::{
-  komodo_timestamp,
+  komodo_timestamp, random_string,
   user::{User, UserConfig},
 };
 use reqwest::StatusCode;
@@ -14,7 +14,6 @@ use serror::AddStatusCode;
 
 use crate::{
   config::core_config,
-  helpers::random_string,
   state::{db_client, jwt_client},
 };
 
@@ -53,7 +52,6 @@ struct CallbackQuery {
   code: String,
 }
 
-#[instrument(name = "GithubCallback", level = "debug")]
 async fn callback(
   Query(query): Query<CallbackQuery>,
 ) -> anyhow::Result<Redirect> {

bin/core/src/auth/google/client.rs

@@ -2,17 +2,15 @@ use std::sync::OnceLock;
 
 use anyhow::{Context, anyhow};
 use jsonwebtoken::{DecodingKey, Validation, decode};
-use komodo_client::entities::config::core::{
-  CoreConfig, OauthCredentials,
+use komodo_client::entities::{
+  config::core::{CoreConfig, OauthCredentials},
+  random_string,
 };
 use reqwest::StatusCode;
 use serde::{Deserialize, de::DeserializeOwned};
 use tokio::sync::Mutex;
 
-use crate::{
-  auth::STATE_PREFIX_LENGTH, config::core_config,
-  helpers::random_string,
-};
+use crate::{auth::STATE_PREFIX_LENGTH, config::core_config};
 
 pub fn google_oauth_client() -> &'static Option<GoogleOauthClient> {
   static GOOGLE_OAUTH_CLIENT: OnceLock<Option<GoogleOauthClient>> =
@@ -85,7 +83,6 @@ impl GoogleOauthClient {
     .into()
   }
 
-  #[instrument(level = "debug", skip(self))]
   pub async fn get_login_redirect_url(
     &self,
     redirect: Option<String>,
@@ -104,7 +101,6 @@ impl GoogleOauthClient {
     redirect_url
   }
 
-  #[instrument(level = "debug", skip(self))]
   pub async fn check_state(&self, state: &str) -> bool {
     let mut contained = false;
     self.states.lock().await.retain(|s| {
@@ -118,7 +114,6 @@ impl GoogleOauthClient {
     contained
   }
 
-  #[instrument(level = "debug", skip(self))]
   pub async fn get_access_token(
     &self,
     code: &str,
@@ -139,7 +134,6 @@ impl GoogleOauthClient {
       .context("failed to get google access token using code")
   }
 
-  #[instrument(level = "debug", skip(self))]
   pub fn get_google_user(
     &self,
     id_token: &str,
@@ -156,7 +150,6 @@ impl GoogleOauthClient {
     Ok(res.claims)
   }
 
-  #[instrument(level = "debug", skip(self))]
   async fn post<R: DeserializeOwned>(
     &self,
     endpoint: &str,

bin/core/src/auth/google/mod.rs

@@ -5,14 +5,16 @@ use axum::{
 };
 use database::mongo_indexed::Document;
 use database::mungos::mongodb::bson::doc;
-use komodo_client::entities::user::{User, UserConfig};
+use komodo_client::entities::{
+  random_string,
+  user::{User, UserConfig},
+};
 use reqwest::StatusCode;
 use serde::Deserialize;
 use serror::AddStatusCode;
 
 use crate::{
   config::core_config,
-  helpers::random_string,
   state::{db_client, jwt_client},
 };
 
@@ -52,7 +54,6 @@ struct CallbackQuery {
   error: Option<String>,
 }
 
-#[instrument(name = "GoogleCallback", level = "debug")]
 async fn callback(
   Query(query): Query<CallbackQuery>,
 ) -> anyhow::Result<Redirect> {

bin/core/src/auth/jwt.rs

@@ -9,16 +9,15 @@ use jsonwebtoken::{
   DecodingKey, EncodingKey, Header, Validation, decode, encode,
 };
 use komodo_client::{
-  api::auth::JwtResponse, entities::config::core::CoreConfig,
+  api::auth::JwtResponse,
+  entities::{config::core::CoreConfig, random_string},
 };
 use serde::{Deserialize, Serialize};
 use tokio::sync::Mutex;
 
-use crate::helpers::random_string;
-
 type ExchangeTokenMap = Mutex<HashMap<String, (JwtResponse, u128)>>;
 
-#[derive(Serialize, Deserialize)]
+#[derive(Serialize, Deserialize, Clone)]
 pub struct JwtClaims {
   pub id: String,
   pub iat: u128,
@@ -75,7 +74,6 @@ impl JwtClient {
       .context("failed to decode token claims")
   }
 
-  #[instrument(level = "debug", skip_all)]
   pub async fn create_exchange_token(
     &self,
     jwt: JwtResponse,
@@ -91,7 +89,7 @@ impl JwtClient {
     );
     exchange_token
   }
-  #[instrument(level = "debug", skip(self))]
+
   pub async fn redeem_exchange_token(
     &self,
     exchange_token: &str,

bin/core/src/auth/local.rs

@@ -22,7 +22,7 @@ use crate::{
 };
 
 impl Resolve<AuthArgs> for SignUpLocalUser {
-  #[instrument(name = "SignUpLocalUser", skip(self))]
+  #[instrument("SignUpLocalUser", skip(self))]
   async fn resolve(
     self,
     _: &AuthArgs,
@@ -104,7 +104,6 @@ impl Resolve<AuthArgs> for SignUpLocalUser {
 }
 
 impl Resolve<AuthArgs> for LoginLocalUser {
-  #[instrument(name = "LoginLocalUser", level = "debug", skip(self))]
   async fn resolve(
     self,
     _: &AuthArgs,

bin/core/src/auth/mod.rs

@@ -31,7 +31,6 @@ struct RedirectQuery {
   redirect: Option<String>,
 }
 
-#[instrument(level = "debug")]
 pub async fn auth_request(
   headers: HeaderMap,
   mut req: Request,
@@ -44,7 +43,6 @@ pub async fn auth_request(
   Ok(next.run(req).await)
 }
 
-#[instrument(level = "debug")]
 pub async fn get_user_id_from_headers(
   headers: &HeaderMap,
 ) -> anyhow::Result<String> {
@@ -77,7 +75,6 @@ pub async fn get_user_id_from_headers(
   }
 }
 
-#[instrument(level = "debug")]
 pub async fn authenticate_check_enabled(
   headers: &HeaderMap,
 ) -> anyhow::Result<User> {
@@ -90,7 +87,6 @@ pub async fn authenticate_check_enabled(
   }
 }
 
-#[instrument(level = "debug")]
 pub async fn auth_jwt_get_user_id(
   jwt: &str,
 ) -> anyhow::Result<String> {
@@ -102,7 +98,6 @@ pub async fn auth_jwt_get_user_id(
   }
 }
 
-#[instrument(level = "debug")]
 pub async fn auth_jwt_check_enabled(
   jwt: &str,
 ) -> anyhow::Result<User> {
@@ -110,7 +105,6 @@ pub async fn auth_jwt_check_enabled(
   check_enabled(user_id).await
 }
 
-#[instrument(level = "debug")]
 pub async fn auth_api_key_get_user_id(
   key: &str,
   secret: &str,
@@ -135,7 +129,6 @@ pub async fn auth_api_key_get_user_id(
   }
 }
 
-#[instrument(level = "debug")]
 pub async fn auth_api_key_check_enabled(
   key: &str,
   secret: &str,
@@ -144,7 +137,6 @@ pub async fn auth_api_key_check_enabled(
   check_enabled(user_id).await
 }
 
-#[instrument(level = "debug")]
 async fn check_enabled(user_id: String) -> anyhow::Result<User> {
   let user = get_user(&user_id).await?;
   if user.enabled {

bin/core/src/auth/oidc/mod.rs

@@ -8,7 +8,7 @@ use client::oidc_client;
 use dashmap::DashMap;
 use database::mungos::mongodb::bson::{Document, doc};
 use komodo_client::entities::{
-  komodo_timestamp,
+  komodo_timestamp, random_string,
   user::{User, UserConfig},
 };
 use openidconnect::{
@@ -23,7 +23,6 @@ use serror::AddStatusCode;
 
 use crate::{
   config::core_config,
-  helpers::random_string,
   state::{db_client, jwt_client},
 };
 
@@ -75,7 +74,6 @@ pub fn router() -> Router {
     )
 }
 
-#[instrument(name = "OidcRedirect", level = "debug")]
 async fn login(
   Query(RedirectQuery { redirect }): Query<RedirectQuery>,
 ) -> anyhow::Result<Redirect> {
@@ -138,7 +136,6 @@ struct CallbackQuery {
   error: Option<String>,
 }
 
-#[instrument(name = "OidcCallback", level = "debug")]
 async fn callback(
   Query(query): Query<CallbackQuery>,
 ) -> anyhow::Result<Redirect> {

bin/core/src/cloud/aws/ec2.rs

@@ -57,7 +57,6 @@ impl aws_credential_types::provider::ProvideCredentials
   }
 }
 
-#[instrument]
 async fn create_ec2_client(region: String) -> Client {
   let region = Region::new(region);
   let config = aws_config::defaults(BehaviorVersion::latest())
@@ -68,7 +67,7 @@ async fn create_ec2_client(region: String) -> Client {
   Client::new(&config)
 }
 
-#[instrument]
+#[instrument("LaunchEc2Instance")]
 pub async fn launch_ec2_instance(
   name: &str,
   config: &AwsBuilderConfig,
@@ -170,7 +169,7 @@ pub async fn launch_ec2_instance(
 const MAX_TERMINATION_TRIES: usize = 5;
 const TERMINATION_WAIT_SECS: u64 = 15;
 
-#[instrument]
+#[instrument("TerminateEc2Instance")]
 pub async fn terminate_ec2_instance_with_retry(
   region: String,
   instance_id: &str,
@@ -210,7 +209,7 @@ pub async fn terminate_ec2_instance_with_retry(
   unreachable!()
 }
 
-#[instrument(skip(client))]
+#[instrument("TerminateEc2InstanceInner", skip_all)]
 async fn terminate_ec2_instance_inner(
   client: &Client,
   instance_id: &str,
@@ -229,7 +228,6 @@ async fn terminate_ec2_instance_inner(
 }
 
 /// Automatically retries 5 times, waiting 2 sec in between
-#[instrument(level = "debug")]
 async fn get_ec2_instance_status(
   client: &Client,
   instance_id: &str,
@@ -261,7 +259,6 @@ async fn get_ec2_instance_status(
   }
 }
 
-#[instrument(level = "debug")]
 async fn get_ec2_instance_state_name(
   client: &Client,
   instance_id: &str,
@@ -281,7 +278,6 @@ async fn get_ec2_instance_state_name(
 }
 
 /// Automatically retries 5 times, waiting 2 sec in between
-#[instrument(level = "debug")]
 async fn get_ec2_instance_public_ip(
   client: &Client,
   instance_id: &str,

bin/core/src/config.rs

@@ -9,36 +9,20 @@ use environment_file::{
 use komodo_client::entities::{
   config::{
     DatabaseConfig,
-    core::{
-      AwsCredentials, CoreConfig, Env, GithubWebhookAppConfig,
-      GithubWebhookAppInstallationConfig, OauthCredentials,
-    },
+    core::{AwsCredentials, CoreConfig, Env, OauthCredentials},
   },
   logger::LogConfig,
 };
-use noise::key::{SpkiPublicKey, load_maybe_generate_private_key};
+use noise::key::{RotatableKeyPair, SpkiPublicKey};
 
 /// Should call in startup to ensure Core errors without valid private key.
-pub fn core_private_key() -> &'static String {
-  static CORE_PRIVATE_KEY: OnceLock<String> = OnceLock::new();
-  CORE_PRIVATE_KEY.get_or_init(|| {
-    let config = core_config();
-    if let Some(path) = config.private_key.strip_prefix("file:") {
-      load_maybe_generate_private_key(path).unwrap()
-    } else {
-      config.private_key.clone()
-    }
-  })
-}
-
-/// Should call in startup to ensure Core errors without valid private key.
-pub fn core_public_key() -> &'static String {
-  static CORE_PUBLIC_KEY: OnceLock<String> = OnceLock::new();
-  CORE_PUBLIC_KEY.get_or_init(|| {
-    SpkiPublicKey::from_private_key(core_private_key())
-      .context("Got invalid private key")
-      .unwrap()
-      .into_inner()
+pub fn core_keys() -> &'static RotatableKeyPair {
+  static CORE_KEYS: OnceLock<RotatableKeyPair> = OnceLock::new();
+  CORE_KEYS.get_or_init(|| {
+    RotatableKeyPair::from_private_key_spec(
+      &core_config().private_key,
+    )
+    .unwrap()
   })
 }
 
@@ -109,12 +93,13 @@ pub fn core_config() -> &'static CoreConfig {
   static CORE_CONFIG: OnceLock<CoreConfig> = OnceLock::new();
   CORE_CONFIG.get_or_init(|| {
     let env: Env = match envy::from_env()
-      .context("Failed to parse Komodo Core environment") {
-        Ok(env) => env,
-        Err(e) => {
-          panic!("{e:?}");
-        }
-      };
+      .context("Failed to parse Komodo Core environment")
+    {
+      Ok(env) => env,
+      Err(e) => {
+        panic!("{e:?}");
+      }
+    };
     let config = if env.komodo_config_paths.is_empty() {
       println!(
         "{}: No config paths found, using default config",
@@ -122,7 +107,8 @@ pub fn core_config() -> &'static CoreConfig {
       );
       CoreConfig::default()
     } else {
-      let config_keywords = env.komodo_config_keywords
+      let config_keywords = env
+        .komodo_config_keywords
         .iter()
         .map(String::as_str)
         .collect::<Vec<_>>();
@@ -132,7 +118,8 @@ pub fn core_config() -> &'static CoreConfig {
         "Config File Keywords".dimmed(),
       );
       (ConfigLoader {
-        paths: &env.komodo_config_paths
+        paths: &env
+          .komodo_config_paths
           .iter()
           .map(PathBuf::as_path)
           .collect::<Vec<_>>(),
@@ -141,57 +128,53 @@ pub fn core_config() -> &'static CoreConfig {
         merge_nested: env.komodo_merge_nested_config,
         extend_array: env.komodo_extend_config_arrays,
         debug_print: env.komodo_config_debug,
-      }).load::<CoreConfig>()
+      })
+      .load::<CoreConfig>()
       .expect("Failed at parsing config from paths")
     };
 
-    let installations = match (
-      maybe_read_list_from_file(
-        env.komodo_github_webhook_app_installations_ids_file,
-        env.komodo_github_webhook_app_installations_ids
-      ),
-      env.komodo_github_webhook_app_installations_namespaces
-    ) {
-      (Some(ids), Some(namespaces)) => {
-        if ids.len() != namespaces.len() {
-          panic!("KOMODO_GITHUB_WEBHOOK_APP_INSTALLATIONS_IDS length and KOMODO_GITHUB_WEBHOOK_APP_INSTALLATIONS_NAMESPACES length mismatch. Got {ids:?} and {namespaces:?}")
-        }
-        ids
-          .into_iter()
-          .zip(namespaces)
-          .map(|(id, namespace)| GithubWebhookAppInstallationConfig {
-            id,
-            namespace
-          })
-          .collect()
-      },
-      (Some(_), None) | (None, Some(_)) => {
-        panic!("Got only one of KOMODO_GITHUB_WEBHOOK_APP_INSTALLATIONS_IDS or KOMODO_GITHUB_WEBHOOK_APP_INSTALLATIONS_NAMESPACES, both MUST be provided");
-      }
-      (None, None) => {
-        config.github_webhook_app.installations
-      }
-    };
-
     // recreating CoreConfig here makes sure apply all env overrides applied.
     CoreConfig {
       // Secret things overridden with file
-      private_key: maybe_read_item_from_file(env.komodo_private_key_file, env.komodo_private_key)
-        .unwrap_or(config.private_key),
-      passkey: maybe_read_item_from_file(env.komodo_passkey_file, env.komodo_passkey)
-        .or(config.passkey),
-      jwt_secret: maybe_read_item_from_file(env.komodo_jwt_secret_file, env.komodo_jwt_secret).unwrap_or(config.jwt_secret),
-      webhook_secret: maybe_read_item_from_file(env.komodo_webhook_secret_file, env.komodo_webhook_secret)
-        .unwrap_or(config.webhook_secret),
+      private_key: maybe_read_item_from_file(
+        env.komodo_private_key_file,
+        env.komodo_private_key,
+      )
+      .unwrap_or(config.private_key),
+      passkey: maybe_read_item_from_file(
+        env.komodo_passkey_file,
+        env.komodo_passkey,
+      )
+      .or(config.passkey),
+      jwt_secret: maybe_read_item_from_file(
+        env.komodo_jwt_secret_file,
+        env.komodo_jwt_secret,
+      )
+      .unwrap_or(config.jwt_secret),
+      webhook_secret: maybe_read_item_from_file(
+        env.komodo_webhook_secret_file,
+        env.komodo_webhook_secret,
+      )
+      .unwrap_or(config.webhook_secret),
       database: DatabaseConfig {
-        uri: maybe_read_item_from_file(env.komodo_database_uri_file,env.komodo_database_uri).unwrap_or(config.database.uri),
-        address: env.komodo_database_address.unwrap_or(config.database.address),
-        username: maybe_read_item_from_file(env.komodo_database_username_file,env
-          .komodo_database_username)
-          .unwrap_or(config.database.username),
-        password: maybe_read_item_from_file(env.komodo_database_password_file,env
-          .komodo_database_password)
-          .unwrap_or(config.database.password),
+        uri: maybe_read_item_from_file(
+          env.komodo_database_uri_file,
+          env.komodo_database_uri,
+        )
+        .unwrap_or(config.database.uri),
+        address: env
+          .komodo_database_address
+          .unwrap_or(config.database.address),
+        username: maybe_read_item_from_file(
+          env.komodo_database_username_file,
+          env.komodo_database_username,
+        )
+        .unwrap_or(config.database.username),
+        password: maybe_read_item_from_file(
+          env.komodo_database_password_file,
+          env.komodo_database_password,
+        )
+        .unwrap_or(config.database.password),
         app_name: env
           .komodo_database_app_name
           .unwrap_or(config.database.app_name),
@@ -201,64 +184,82 @@ pub fn core_config() -> &'static CoreConfig {
       },
       init_admin_username: maybe_read_item_from_file(
         env.komodo_init_admin_username_file,
-        env.komodo_init_admin_username
-      ).or(config.init_admin_username),
+        env.komodo_init_admin_username,
+      )
+      .or(config.init_admin_username),
       init_admin_password: maybe_read_item_from_file(
         env.komodo_init_admin_password_file,
-        env.komodo_init_admin_password
-      ).unwrap_or(config.init_admin_password),
-      oidc_enabled: env.komodo_oidc_enabled.unwrap_or(config.oidc_enabled),
-      oidc_provider: env.komodo_oidc_provider.unwrap_or(config.oidc_provider),
-      oidc_redirect_host: env.komodo_oidc_redirect_host.unwrap_or(config.oidc_redirect_host),
-      oidc_client_id: maybe_read_item_from_file(env.komodo_oidc_client_id_file,env
-        .komodo_oidc_client_id)
-        .unwrap_or(config.oidc_client_id),
-      oidc_client_secret: maybe_read_item_from_file(env.komodo_oidc_client_secret_file,env
-        .komodo_oidc_client_secret)
-        .unwrap_or(config.oidc_client_secret),
-      oidc_use_full_email: env.komodo_oidc_use_full_email
+        env.komodo_init_admin_password,
+      )
+      .unwrap_or(config.init_admin_password),
+      oidc_enabled: env
+        .komodo_oidc_enabled
+        .unwrap_or(config.oidc_enabled),
+      oidc_provider: env
+        .komodo_oidc_provider
+        .unwrap_or(config.oidc_provider),
+      oidc_redirect_host: env
+        .komodo_oidc_redirect_host
+        .unwrap_or(config.oidc_redirect_host),
+      oidc_client_id: maybe_read_item_from_file(
+        env.komodo_oidc_client_id_file,
+        env.komodo_oidc_client_id,
+      )
+      .unwrap_or(config.oidc_client_id),
+      oidc_client_secret: maybe_read_item_from_file(
+        env.komodo_oidc_client_secret_file,
+        env.komodo_oidc_client_secret,
+      )
+      .unwrap_or(config.oidc_client_secret),
+      oidc_use_full_email: env
+        .komodo_oidc_use_full_email
         .unwrap_or(config.oidc_use_full_email),
-      oidc_additional_audiences: maybe_read_list_from_file(env.komodo_oidc_additional_audiences_file,env
-        .komodo_oidc_additional_audiences)
-        .unwrap_or(config.oidc_additional_audiences),
+      oidc_additional_audiences: maybe_read_list_from_file(
+        env.komodo_oidc_additional_audiences_file,
+        env.komodo_oidc_additional_audiences,
+      )
+      .unwrap_or(config.oidc_additional_audiences),
       google_oauth: OauthCredentials {
         enabled: env
           .komodo_google_oauth_enabled
           .unwrap_or(config.google_oauth.enabled),
-        id: maybe_read_item_from_file(env.komodo_google_oauth_id_file,env
-          .komodo_google_oauth_id)
-          .unwrap_or(config.google_oauth.id),
-        secret: maybe_read_item_from_file(env.komodo_google_oauth_secret_file,env
-          .komodo_google_oauth_secret)
-          .unwrap_or(config.google_oauth.secret),
+        id: maybe_read_item_from_file(
+          env.komodo_google_oauth_id_file,
+          env.komodo_google_oauth_id,
+        )
+        .unwrap_or(config.google_oauth.id),
+        secret: maybe_read_item_from_file(
+          env.komodo_google_oauth_secret_file,
+          env.komodo_google_oauth_secret,
+        )
+        .unwrap_or(config.google_oauth.secret),
       },
       github_oauth: OauthCredentials {
         enabled: env
           .komodo_github_oauth_enabled
           .unwrap_or(config.github_oauth.enabled),
-        id: maybe_read_item_from_file(env.komodo_github_oauth_id_file,env
-          .komodo_github_oauth_id)
-          .unwrap_or(config.github_oauth.id),
-        secret: maybe_read_item_from_file(env.komodo_github_oauth_secret_file,env
-          .komodo_github_oauth_secret)
-          .unwrap_or(config.github_oauth.secret),
+        id: maybe_read_item_from_file(
+          env.komodo_github_oauth_id_file,
+          env.komodo_github_oauth_id,
+        )
+        .unwrap_or(config.github_oauth.id),
+        secret: maybe_read_item_from_file(
+          env.komodo_github_oauth_secret_file,
+          env.komodo_github_oauth_secret,
+        )
+        .unwrap_or(config.github_oauth.secret),
       },
       aws: AwsCredentials {
-        access_key_id: maybe_read_item_from_file(env.komodo_aws_access_key_id_file, env
-          .komodo_aws_access_key_id)
-          .unwrap_or(config.aws.access_key_id),
-        secret_access_key: maybe_read_item_from_file(env.komodo_aws_secret_access_key_file, env
-          .komodo_aws_secret_access_key)
-          .unwrap_or(config.aws.secret_access_key),
-      },
-      github_webhook_app: GithubWebhookAppConfig {
-        app_id: maybe_read_item_from_file(env.komodo_github_webhook_app_app_id_file, env
-          .komodo_github_webhook_app_app_id)
-          .unwrap_or(config.github_webhook_app.app_id),
-        pk_path: env
-          .komodo_github_webhook_app_pk_path
-          .unwrap_or(config.github_webhook_app.pk_path),
-        installations,
+        access_key_id: maybe_read_item_from_file(
+          env.komodo_aws_access_key_id_file,
+          env.komodo_aws_access_key_id,
+        )
+        .unwrap_or(config.aws.access_key_id),
+        secret_access_key: maybe_read_item_from_file(
+          env.komodo_aws_secret_access_key_file,
+          env.komodo_aws_secret_access_key,
+        )
+        .unwrap_or(config.aws.secret_access_key),
       },
 
       // Non secrets
@@ -267,13 +268,19 @@ pub fn core_config() -> &'static CoreConfig {
       port: env.komodo_port.unwrap_or(config.port),
       bind_ip: env.komodo_bind_ip.unwrap_or(config.bind_ip),
       timezone: env.komodo_timezone.unwrap_or(config.timezone),
-      periphery_public_keys: env.komodo_periphery_public_keys.or(config.periphery_public_keys),
-      first_server_address: env.komodo_first_server_address.or(config.first_server_address),
-      first_server_name: env.komodo_first_server_name.or(config.first_server_name),
-      frontend_path: env.komodo_frontend_path.unwrap_or(config.frontend_path),
-      jwt_ttl: env
-        .komodo_jwt_ttl
-        .unwrap_or(config.jwt_ttl),
+      periphery_public_keys: env
+        .komodo_periphery_public_keys
+        .or(config.periphery_public_keys),
+      first_server_address: env
+        .komodo_first_server_address
+        .or(config.first_server_address),
+      first_server_name: env
+        .komodo_first_server_name
+        .or(config.first_server_name),
+      frontend_path: env
+        .komodo_frontend_path
+        .unwrap_or(config.frontend_path),
+      jwt_ttl: env.komodo_jwt_ttl.unwrap_or(config.jwt_ttl),
       sync_directory: env
         .komodo_sync_directory
         .unwrap_or(config.sync_directory),
@@ -304,24 +311,31 @@ pub fn core_config() -> &'static CoreConfig {
       ui_write_disabled: env
         .komodo_ui_write_disabled
         .unwrap_or(config.ui_write_disabled),
-      disable_confirm_dialog: env.komodo_disable_confirm_dialog
+      disable_confirm_dialog: env
+        .komodo_disable_confirm_dialog
         .unwrap_or(config.disable_confirm_dialog),
-      disable_websocket_reconnect: env.komodo_disable_websocket_reconnect
+      disable_websocket_reconnect: env
+        .komodo_disable_websocket_reconnect
         .unwrap_or(config.disable_websocket_reconnect),
-      enable_new_users: env.komodo_enable_new_users
+      enable_new_users: env
+        .komodo_enable_new_users
         .unwrap_or(config.enable_new_users),
-      disable_user_registration: env.komodo_disable_user_registration
+      disable_user_registration: env
+        .komodo_disable_user_registration
         .unwrap_or(config.disable_user_registration),
-      disable_non_admin_create: env.komodo_disable_non_admin_create
+      disable_non_admin_create: env
+        .komodo_disable_non_admin_create
         .unwrap_or(config.disable_non_admin_create),
-      disable_init_resources: env.komodo_disable_init_resources
+      disable_init_resources: env
+        .komodo_disable_init_resources
         .unwrap_or(config.disable_init_resources),
-      enable_fancy_toml: env.komodo_enable_fancy_toml
+      enable_fancy_toml: env
+        .komodo_enable_fancy_toml
         .unwrap_or(config.enable_fancy_toml),
-      lock_login_credentials_for: env.komodo_lock_login_credentials_for
+      lock_login_credentials_for: env
+        .komodo_lock_login_credentials_for
         .unwrap_or(config.lock_login_credentials_for),
-      local_auth: env.komodo_local_auth
-        .unwrap_or(config.local_auth),
+      local_auth: env.komodo_local_auth.unwrap_or(config.local_auth),
       logging: LogConfig {
         level: env
           .komodo_logging_level
@@ -329,23 +343,41 @@ pub fn core_config() -> &'static CoreConfig {
         stdio: env
           .komodo_logging_stdio
           .unwrap_or(config.logging.stdio),
-        pretty: env.komodo_logging_pretty
+        pretty: env
+          .komodo_logging_pretty
           .unwrap_or(config.logging.pretty),
-        location: env.komodo_logging_location
+        location: env
+          .komodo_logging_location
           .unwrap_or(config.logging.location),
+        ansi: env.komodo_logging_ansi.unwrap_or(config.logging.ansi),
         otlp_endpoint: env
           .komodo_logging_otlp_endpoint
           .unwrap_or(config.logging.otlp_endpoint),
         opentelemetry_service_name: env
           .komodo_logging_opentelemetry_service_name
           .unwrap_or(config.logging.opentelemetry_service_name),
+        opentelemetry_scope_name: env
+          .komodo_logging_opentelemetry_scope_name
+          .unwrap_or(config.logging.opentelemetry_scope_name),
       },
-      pretty_startup_config: env.komodo_pretty_startup_config.unwrap_or(config.pretty_startup_config),
-      unsafe_unsanitized_startup_config: env.komodo_unsafe_unsanitized_startup_config.unwrap_or(config.unsafe_unsanitized_startup_config),
-      internet_interface: env.komodo_internet_interface.unwrap_or(config.internet_interface),
-      ssl_enabled: env.komodo_ssl_enabled.unwrap_or(config.ssl_enabled),
-      ssl_key_file: env.komodo_ssl_key_file.unwrap_or(config.ssl_key_file),
-      ssl_cert_file: env.komodo_ssl_cert_file.unwrap_or(config.ssl_cert_file),
+      pretty_startup_config: env
+        .komodo_pretty_startup_config
+        .unwrap_or(config.pretty_startup_config),
+      unsafe_unsanitized_startup_config: env
+        .komodo_unsafe_unsanitized_startup_config
+        .unwrap_or(config.unsafe_unsanitized_startup_config),
+      internet_interface: env
+        .komodo_internet_interface
+        .unwrap_or(config.internet_interface),
+      ssl_enabled: env
+        .komodo_ssl_enabled
+        .unwrap_or(config.ssl_enabled),
+      ssl_key_file: env
+        .komodo_ssl_key_file
+        .unwrap_or(config.ssl_key_file),
+      ssl_cert_file: env
+        .komodo_ssl_cert_file
+        .unwrap_or(config.ssl_cert_file),
 
       // These can't be overridden on env
       secrets: config.secrets,

bin/core/src/connection/client.rs

@@ -1,21 +1,24 @@
-use std::{sync::Arc, time::Duration};
+use std::time::Duration;
 
 use anyhow::{Context, anyhow};
-use periphery_client::CONNECTION_RETRY_SECONDS;
-use serror::{deserialize_error_bytes, serialize_error_bytes};
+use periphery_client::{
+  CONNECTION_RETRY_SECONDS, transport::LoginMessage,
+};
 use transport::{
-  MessageState,
   auth::{
     AddressConnectionIdentifiers, ClientLoginFlow,
     ConnectionIdentifiers,
   },
   fix_ws_address,
-  websocket::{Websocket, tungstenite::TungsteniteWebsocket},
+  websocket::{
+    Websocket, WebsocketExt as _, login::LoginWebsocketExt,
+    tungstenite::TungsteniteWebsocket,
+  },
 };
 
 use crate::{
   config::{core_config, core_connection_query},
-  periphery::ConnectionChannels,
+  periphery::PeripheryClient,
   state::periphery_connections,
 };
 
@@ -26,8 +29,7 @@ impl PeripheryConnectionArgs<'_> {
     self,
     id: String,
     insecure: bool,
-    passkey: String,
-  ) -> anyhow::Result<Arc<ConnectionChannels>> {
+  ) -> anyhow::Result<PeripheryClient> {
     let Some(address) = self.address else {
       return Err(anyhow!(
         "Cannot spawn client connection with empty address"
@@ -42,7 +44,8 @@ impl PeripheryConnectionArgs<'_> {
     let (connection, mut receiver) =
       periphery_connections().insert(id.clone(), self).await;
 
-    let channels = connection.channels.clone();
+    let responses = connection.responses.clone();
+    let terminals = connection.terminals.clone();
 
     tokio::spawn(async move {
       loop {
@@ -73,9 +76,8 @@ impl PeripheryConnectionArgs<'_> {
           core_connection_query().as_bytes(),
         );
 
-        if let Err(e) = connection
-          .client_login(&mut socket, identifiers, &passkey)
-          .await
+        if let Err(e) =
+          connection.client_login(&mut socket, identifiers).await
         {
           connection.set_error(e).await;
           tokio::time::sleep(Duration::from_secs(
@@ -89,87 +91,83 @@ impl PeripheryConnectionArgs<'_> {
       }
     });
 
-    Ok(channels)
+    Ok(PeripheryClient {
+      id,
+      responses,
+      terminals,
+    })
   }
 }
 
 impl PeripheryConnection {
   /// Custom Core -> Periphery side only login wrapper
   /// to implement passkey support for backward compatibility
+  #[instrument(
+    "PeripheryLogin",
+    skip(self, socket, identifiers),
+    fields(
+      server_id = self.args.id,
+      address = self.args.address,
+      direction = "CoreToPeriphery"
+    )
+  )]
   async fn client_login(
     &self,
     socket: &mut TungsteniteWebsocket,
     identifiers: ConnectionIdentifiers<'_>,
-    // for legacy auth
-    passkey: &str,
   ) -> anyhow::Result<()> {
     // Get the required auth type
-    let bytes = socket
-      .recv_bytes()
-      .with_timeout(Duration::from_secs(2))
-      .await?
-      .context("Failed to receive login type indicator")?;
+    let v1_passkey_flow =
+      socket
+        .recv_login_v1_passkey_flow()
+        .await
+        .context("Failed to receive Login V1PasskeyFlow message")?;
 
-    match bytes.iter().as_slice() {
-      // Noise auth
-      &[0] => {
-        self
-          .handle_login::<_, ClientLoginFlow>(socket, identifiers)
-          .await
-      }
-      // Passkey auth
-      &[1] => handle_passkey_login(socket, passkey).await,
-      other => Err(anyhow!(
-        "Receieved invalid login type pattern: {other:?}"
-      )),
+    if v1_passkey_flow {
+      handle_passkey_login(socket, self.args.passkey.as_deref()).await
+    } else {
+      self
+        .handle_login::<_, ClientLoginFlow>(socket, identifiers)
+        .await
     }
   }
 }
 
+#[instrument("V1PasskeyPeripheryLoginFlow", skip(socket, passkey))]
 async fn handle_passkey_login(
   socket: &mut TungsteniteWebsocket,
   // for legacy auth
-  passkey: &str,
+  passkey: Option<&str>,
 ) -> anyhow::Result<()> {
   let res = async {
-    let mut passkey = if passkey.is_empty() {
+    let passkey = if let Some(passkey) = passkey {
+      passkey.as_bytes().to_vec()
+    } else {
       core_config()
         .passkey
         .as_deref()
         .context("Periphery requires passkey auth")?
         .as_bytes()
         .to_vec()
-    } else {
-      passkey.as_bytes().to_vec()
     };
-    passkey.push(MessageState::Successful.as_byte());
 
     socket
-      .send(passkey.into())
+      .send_message(LoginMessage::V1Passkey(passkey))
       .await
-      .context("Failed to send passkey")?;
+      .context("Failed to send Login V1Passkey message")?;
 
     // Receive login state message and return based on value
-    let state_msg = socket
-      .recv_bytes()
+    socket
+      .recv_login_success()
       .await
-      .context("Failed to receive authentication state message")?;
-    let state = state_msg.last().context(
-      "Authentication state message did not contain state byte",
-    )?;
-    match MessageState::from_byte(*state) {
-      MessageState::Successful => anyhow::Ok(()),
-      _ => Err(deserialize_error_bytes(
-        &state_msg[..(state_msg.len() - 1)],
-      )),
-    }
+      .context("Failed to receive Login Success message")?;
+
+    anyhow::Ok(())
   }
   .await;
   if let Err(e) = res {
-    let mut bytes = serialize_error_bytes(&e);
-    bytes.push(MessageState::Failed.as_byte());
     if let Err(e) = socket
-      .send(bytes.into())
+      .send_login_error(&e)
       .await
       .context("Failed to send login failed to client")
     {
@@ -177,7 +175,7 @@ async fn handle_passkey_login(
       warn!("{e:#}");
     }
     // Close socket
-    let _ = socket.close(None).await;
+    let _ = socket.close().await;
     // Return the original error
     Err(e)
   } else {

bin/core/src/connection/mod.rs

@@ -7,36 +7,38 @@ use std::{
 };
 
 use anyhow::anyhow;
-use bytes::Bytes;
 use cache::CloneCache;
 use database::mungos::{by_id::update_one_by_id, mongodb::bson::doc};
+use encoding::{
+  CastBytes as _, Decode as _, EncodedJsonMessage, EncodedResponse,
+  WithChannel,
+};
 use komodo_client::entities::{
   builder::{AwsBuilderConfig, UrlBuilderConfig},
   optional_str,
   server::Server,
 };
-use serror::serror_into_anyhow_error;
-use tokio::sync::{
-  RwLock,
-  mpsc::{Sender, error::SendError},
+use periphery_client::transport::{
+  EncodedTransportMessage, ResponseMessage, TransportMessage,
 };
+use serror::serror_into_anyhow_error;
+use tokio::sync::RwLock;
 use tokio_util::sync::CancellationToken;
 use transport::{
   auth::{
     ConnectionIdentifiers, LoginFlow, LoginFlowArgs,
     PublicKeyValidator,
   },
-  bytes::id_from_transport_bytes,
-  channel::{BufferedReceiver, buffered_channel},
+  channel::{BufferedReceiver, Sender, buffered_channel},
   websocket::{
     Websocket, WebsocketMessage, WebsocketReceiver as _,
     WebsocketSender as _,
   },
 };
+use uuid::Uuid;
 
 use crate::{
-  config::{core_private_key, periphery_public_keys},
-  periphery::ConnectionChannels,
+  config::{core_keys, periphery_public_keys},
   state::db_client,
 };
 
@@ -56,7 +58,10 @@ impl PeripheryConnections {
     &self,
     server_id: String,
     args: PeripheryConnectionArgs<'_>,
-  ) -> (Arc<PeripheryConnection>, BufferedReceiver<Bytes>) {
+  ) -> (
+    Arc<PeripheryConnection>,
+    BufferedReceiver<EncodedTransportMessage>,
+  ) {
     let (connection, receiver) = if let Some(existing_connection) =
       self.0.remove(&server_id).await
     {
@@ -97,10 +102,14 @@ pub struct PeripheryConnectionArgs<'a> {
   pub id: &'a str,
   pub address: Option<&'a str>,
   periphery_public_key: Option<&'a str>,
+  /// V1 legacy support.
+  /// Only possible for Core -> Periphery.
+  passkey: Option<&'a str>,
 }
 
 impl PublicKeyValidator for PeripheryConnectionArgs<'_> {
   type ValidationResult = String;
+  #[instrument("ValidatePeripheryPublicKey", skip(self))]
   async fn validate(
     &self,
     public_key: String,
@@ -110,12 +119,11 @@ impl PublicKeyValidator for PeripheryConnectionArgs<'_> {
         self.id.to_string(),
         Some(public_key.clone()),
       );
-      let e = anyhow!("{public_key} is invalid")
+      anyhow!("{public_key} is invalid")
         .context(
           "Ensure public key matches configured Periphery Public Key",
         )
-        .context("Core failed to validate Periphery public key");
-      e
+        .context("Core failed to validate Periphery public key")
     };
     let core_to_periphery = self.address.is_some();
     match (self.periphery_public_key, core_to_periphery) {
@@ -150,6 +158,7 @@ impl<'a> PeripheryConnectionArgs<'a> {
       id: &server.id,
       address: optional_str(&server.config.address),
       periphery_public_key: optional_str(&server.info.public_key),
+      passkey: optional_str(&server.config.passkey),
     }
   }
 
@@ -163,6 +172,7 @@ impl<'a> PeripheryConnectionArgs<'a> {
       periphery_public_key: optional_str(
         &config.periphery_public_key,
       ),
+      passkey: optional_str(&config.passkey),
     }
   }
 
@@ -177,6 +187,7 @@ impl<'a> PeripheryConnectionArgs<'a> {
       periphery_public_key: optional_str(
         &config.periphery_public_key,
       ),
+      passkey: None,
     }
   }
 
@@ -187,6 +198,7 @@ impl<'a> PeripheryConnectionArgs<'a> {
       periphery_public_key: self
         .periphery_public_key
         .map(str::to_string),
+      passkey: self.passkey.map(str::to_string),
     }
   }
 
@@ -209,6 +221,9 @@ pub struct OwnedPeripheryConnectionArgs {
   /// If None, must have 'periphery_public_keys' set
   /// in Core config, or will error
   pub periphery_public_key: Option<String>,
+  /// V1 legacy support.
+  /// Only possible for Core -> Periphery connection.
+  pub passkey: Option<String>,
 }
 
 impl OwnedPeripheryConnectionArgs {
@@ -217,6 +232,7 @@ impl OwnedPeripheryConnectionArgs {
       id: &self.id,
       address: self.address.as_deref(),
       periphery_public_key: self.periphery_public_key.as_deref(),
+      passkey: self.passkey.as_deref(),
     }
   }
 }
@@ -237,12 +253,18 @@ impl<'a> From<&'a OwnedPeripheryConnectionArgs>
   }
 }
 
+/// Sends None as InProgress ping.
+pub type ResponseChannels =
+  CloneCache<Uuid, Sender<EncodedResponse<EncodedJsonMessage>>>;
+
+pub type TerminalChannels = CloneCache<Uuid, Sender<Vec<u8>>>;
+
 #[derive(Debug)]
 pub struct PeripheryConnection {
   /// The connection args
   pub args: OwnedPeripheryConnectionArgs,
   /// Send and receive bytes over the connection socket.
-  pub sender: Sender<Bytes>,
+  pub sender: Sender<EncodedTransportMessage>,
   /// Cancel the connection
   pub cancel: CancellationToken,
   /// Whether Periphery is currently connected.
@@ -251,14 +273,19 @@ pub struct PeripheryConnection {
   // at the same server id.
   /// Stores latest connection error
   pub error: Arc<RwLock<Option<serror::Serror>>>,
-  /// Forward bytes from Periphery to specific channel handlers.
-  pub channels: Arc<ConnectionChannels>,
+  /// Forward bytes from Periphery to response channel handlers.
+  pub responses: Arc<ResponseChannels>,
+  /// Forward bytes from Periphery to terminal channel handlers.
+  pub terminals: Arc<TerminalChannels>,
 }
 
 impl PeripheryConnection {
   pub fn new(
     args: impl Into<OwnedPeripheryConnectionArgs>,
-  ) -> (Arc<PeripheryConnection>, BufferedReceiver<Bytes>) {
+  ) -> (
+    Arc<PeripheryConnection>,
+    BufferedReceiver<EncodedTransportMessage>,
+  ) {
     let (sender, receiever) = buffered_channel();
     (
       PeripheryConnection {
@@ -267,7 +294,8 @@ impl PeripheryConnection {
         cancel: CancellationToken::new(),
         connected: AtomicBool::new(false),
         error: Default::default(),
-        channels: Default::default(),
+        responses: Default::default(),
+        terminals: Default::default(),
       }
       .into(),
       receiever,
@@ -277,7 +305,10 @@ impl PeripheryConnection {
   pub fn with_new_args(
     &self,
     args: impl Into<OwnedPeripheryConnectionArgs>,
-  ) -> (Arc<PeripheryConnection>, BufferedReceiver<Bytes>) {
+  ) -> (
+    Arc<PeripheryConnection>,
+    BufferedReceiver<EncodedTransportMessage>,
+  ) {
     // Ensure this connection is cancelled.
     self.cancel();
     let (sender, receiever) = buffered_channel();
@@ -288,13 +319,19 @@ impl PeripheryConnection {
         cancel: CancellationToken::new(),
         connected: AtomicBool::new(false),
         error: self.error.clone(),
-        channels: self.channels.clone(),
+        responses: self.responses.clone(),
+        terminals: self.terminals.clone(),
       }
       .into(),
       receiever,
     )
   }
 
+  #[instrument(
+    "StandardPeripheryLoginFlow",
+    skip(self, socket, identifiers),
+    fields(expected_public_key = self.args.periphery_public_key)
+  )]
   pub async fn handle_login<W: Websocket, L: LoginFlow>(
     &self,
     socket: &mut W,
@@ -303,7 +340,7 @@ impl PeripheryConnection {
     L::login(LoginFlowArgs {
       socket,
       identifiers,
-      private_key: core_private_key(),
+      private_key: core_keys().load().private.as_str(),
       public_key_validator: self.args.borrow(),
     })
     .await?;
@@ -315,7 +352,7 @@ impl PeripheryConnection {
   pub async fn handle_socket<W: Websocket>(
     &self,
     socket: W,
-    receiver: &mut BufferedReceiver<Bytes>,
+    receiver: &mut BufferedReceiver<EncodedTransportMessage>,
   ) {
     let cancel = self.cancel.child_token();
 
@@ -324,42 +361,32 @@ impl PeripheryConnection {
 
     let (mut ws_write, mut ws_read) = socket.split();
 
+    ws_read.set_cancel(cancel.clone());
+    receiver.set_cancel(cancel.clone());
+
     let forward_writes = async {
       loop {
-        let next = tokio::select! {
-          next = receiver.recv() => next,
-          _ = cancel.cancelled() => break,
+        let Ok(message) = receiver.recv().await else {
+          break;
         };
-
-        let message = match next {
-          Some(request) => Bytes::copy_from_slice(request),
-          // Sender Dropped (shouldn't happen, a reference is held on 'connection').
-          None => break,
-        };
-
-        match ws_write.send(message).await {
+        match ws_write.send(message.into_bytes()).await {
           Ok(_) => receiver.clear_buffer(),
           Err(e) => {
-            self.set_error(e.into()).await;
+            self.set_error(e).await;
             break;
           }
         }
       }
       // Cancel again if not already
-      let _ = ws_write.close(None).await;
+      let _ = ws_write.close().await;
       cancel.cancel();
     };
 
     let handle_reads = async {
       loop {
-        let next = tokio::select! {
-          next = ws_read.recv() => next,
-          _ = cancel.cancelled() => break,
-        };
-
-        match next {
-          Ok(WebsocketMessage::Binary(bytes)) => {
-            self.handle_incoming_bytes(bytes).await
+        match ws_read.recv().await {
+          Ok(WebsocketMessage::Message(message)) => {
+            self.handle_incoming_message(message).await
           }
           Ok(WebsocketMessage::Close(_))
           | Ok(WebsocketMessage::Closed) => {
@@ -367,7 +394,7 @@ impl PeripheryConnection {
             break;
           }
           Err(e) => {
-            self.set_error(e.into()).await;
+            self.set_error(e).await;
           }
         };
       }
@@ -380,33 +407,69 @@ impl PeripheryConnection {
     self.set_connected(false);
   }
 
-  pub async fn handle_incoming_bytes(&self, bytes: Bytes) {
-    let id = match id_from_transport_bytes(&bytes) {
+  pub async fn handle_incoming_message(
+    &self,
+    message: EncodedTransportMessage,
+  ) {
+    let message: TransportMessage = match message.decode() {
       Ok(res) => res,
       Err(e) => {
-        // TODO: handle better
-        warn!("Failed to read id | {e:#}");
+        warn!("Failed to parse Message bytes | {e:#}");
         return;
       }
     };
-    let Some(channel) = self.channels.get(&id).await else {
-      // TODO: handle better
-      debug!("Failed to send response | No response channel found");
-      return;
-    };
-    if let Err(e) = channel.send(bytes).await {
-      // TODO: handle better
-      warn!("Failed to send response | Channel failure | {e:#}");
+    match message {
+      TransportMessage::Response(data) => {
+        match data.decode().map(ResponseMessage::into_inner) {
+          Ok(WithChannel { channel, data }) => {
+            let Some(response_channel) =
+              self.responses.get(&channel).await
+            else {
+              warn!(
+                "Failed to forward Response message | No response channel found at {channel}"
+              );
+              return;
+            };
+            if let Err(e) = response_channel.send(data).await {
+              warn!(
+                "Failed to forward Response | Response channel failure at {channel} | {e:#}"
+              );
+            }
+          }
+          Err(e) => {
+            warn!("Failed to read Response message | {e:#}");
+          }
+        }
+      }
+      TransportMessage::Terminal(data) => match data.decode() {
+        Ok(WithChannel {
+          channel: channel_id,
+          data,
+        }) => {
+          let Some(channel) = self.terminals.get(&channel_id).await
+          else {
+            warn!(
+              "Failed to forward Terminal message | No terminal channel found at {channel_id}"
+            );
+            return;
+          };
+          if let Err(e) = channel.send(data).await {
+            warn!(
+              "Failed to forward Terminal message | Channel failure at {channel_id} | {e:#}"
+            );
+          }
+        }
+        Err(e) => {
+          warn!("Failed to read Terminal message | {e:#}");
+        }
+      },
+      //
+      other => {
+        warn!("Received unexpected transport message | {other:?}");
+      }
     }
   }
 
-  pub async fn send(
-    &self,
-    value: Bytes,
-  ) -> Result<(), SendError<Bytes>> {
-    self.sender.send(value).await
-  }
-
   pub fn set_connected(&self, connected: bool) {
     self.connected.store(connected, atomic::Ordering::Relaxed);
   }

bin/core/src/connection/server.rs

@@ -1,4 +1,4 @@
-use std::{str::FromStr, time::Duration};
+use std::str::FromStr;
 
 use anyhow::{Context, anyhow};
 use axum::{
@@ -6,31 +6,37 @@ use axum::{
   http::{HeaderMap, StatusCode},
   response::Response,
 };
-use bytes::Bytes;
 use database::mungos::mongodb::bson::{doc, oid::ObjectId};
 use komodo_client::{
   api::write::{CreateBuilder, CreateServer, UpdateResourceMeta},
   entities::{
     builder::{PartialBuilderConfig, PartialServerBuilderConfig},
+    komodo_timestamp,
     onboarding_key::OnboardingKey,
     server::{PartialServerConfig, Server},
     user::system_user,
   },
 };
+use periphery_client::{
+  api::PeripheryConnectionQuery, transport::LoginMessage,
+};
 use resolver_api::Resolve;
 use serror::{AddStatusCode, AddStatusCodeError};
+use tracing::Instrument;
 use transport::{
-  MessageState, PeripheryConnectionQuery,
   auth::{
     HeaderConnectionIdentifiers, LoginFlow, LoginFlowArgs,
     PublicKeyValidator, ServerLoginFlow,
   },
-  websocket::{Websocket, axum::AxumWebsocket},
+  websocket::{
+    Websocket, WebsocketExt as _, axum::AxumWebsocket,
+    login::LoginWebsocketExt,
+  },
 };
 
 use crate::{
   api::write::WriteArgs,
-  config::core_private_key,
+  config::core_keys,
   helpers::query::id_or_name_filter,
   resource::KomodoResource,
   state::{db_client, periphery_connections},
@@ -119,20 +125,32 @@ async fn existing_server_handler(
       format!("server={}", urlencoding::encode(&server_query));
     let mut socket = AxumWebsocket(socket);
 
-    if let Err(e) = socket.send(Bytes::from_owner([0])).await.context(
-      "Failed to send the login flow indicator over connnection",
-    ) {
+    if let Err(e) = socket
+      .send_message(LoginMessage::OnboardingFlow(false))
+      .await
+      .context("Failed to send Login OnboardingFlow false message")
+    {
       connection.set_error(e).await;
       return;
     };
 
-    if let Err(e) = connection
-      .handle_login::<_, ServerLoginFlow>(
-        &mut socket,
-        identifiers.build(query.as_bytes()),
-      )
-      .await
-    {
+    let span = info_span!(
+      "PeripheryLogin",
+      server_id = server.id,
+      direction = "PeripheryToCore"
+    );
+    let login = async {
+      connection
+        .handle_login::<_, ServerLoginFlow>(
+          &mut socket,
+          identifiers.build(query.as_bytes()),
+        )
+        .await
+    }
+    .instrument(span)
+    .await;
+
+    if let Err(e) = login {
       connection.set_error(e).await;
       return;
     }
@@ -151,8 +169,8 @@ async fn onboard_server_handler(
       format!("server={}", urlencoding::encode(&server_query));
     let mut socket = AxumWebsocket(socket);
 
-    if let Err(e) = socket.send(Bytes::from_owner([1])).await.context(
-      "Failed to send the login flow indicator over connnection",
+    if let Err(e) = socket.send_message(LoginMessage::OnboardingFlow(true)).await.context(
+      "Failed to send Login OnboardingFlow true message",
     ).context("Server onboarding error") {
       warn!("{e:#}");
       return;
@@ -161,7 +179,7 @@ async fn onboard_server_handler(
     let onboarding_key = match ServerLoginFlow::login(LoginFlowArgs {
       socket: &mut socket,
       identifiers: identifiers.build(query.as_bytes()),
-      private_key: core_private_key(),
+      private_key: core_keys().load().private.as_str(),
       public_key_validator: CreationKeyValidator,
     })
     .await
@@ -173,19 +191,10 @@ async fn onboard_server_handler(
       }
     };
 
-    let res = socket
-      .recv_bytes()
-      .with_timeout(Duration::from_secs(2))
-      .await
-      .and_then(|res| {
-        res.and_then(|public_key_bytes| {
-          String::from_utf8(public_key_bytes.into())
-            .context("Public key bytes are not valid utf8")
-        })
-      });
-
     // Post onboarding login 1: Receive public key
-    let public_key = match res
+    let public_key = match socket
+      .recv_login_public_key()
+      .await
     {
       Ok(public_key) => public_key,
       Err(e) => {
@@ -194,11 +203,9 @@ async fn onboard_server_handler(
       }
     };
 
-    
-
     let server_id = match create_server_maybe_builder(
       server_query,
-      public_key,
+      public_key.into_inner(),
       onboarding_key.copy_server,
       onboarding_key.tags,
       onboarding_key.create_builder
@@ -207,7 +214,7 @@ async fn onboard_server_handler(
       Err(e) => {
         warn!("{e:#}");
         if let Err(e) = socket
-          .send_error(&e)
+          .send_login_error(&e)
           .await
           .context("Failed to send Server creation failed to client")
         {
@@ -219,9 +226,9 @@ async fn onboard_server_handler(
     };
 
     if let Err(e) = socket
-      .send(MessageState::Successful.into())
+      .send_message(LoginMessage::Success)
       .await
-      .context("Failed to send Server creation successful to client")
+      .context("Failed to send Login Onboarding Successful message")
     {
       // Log additional error
       warn!("{e:#}");
@@ -229,7 +236,7 @@ async fn onboard_server_handler(
 
     // Server created, close and trigger reconnect
     // and handling using existing server handler.
-    let _ = socket.close(None).await;
+    let _ = socket.close().await;
 
     // Add the server to onboarding key "Onboarded"
     let res = db_client()
@@ -257,14 +264,23 @@ async fn create_server_maybe_builder(
       ..Default::default()
     }
   } else {
-    let config = match db_client().servers.find_one(id_or_name_filter(&copy_server)).await {
+    let config = match db_client()
+      .servers
+      .find_one(id_or_name_filter(&copy_server))
+      .await
+    {
       Ok(Some(server)) => server.config,
       Ok(None) => {
-        warn!("Server onboarding: Failed to find Server {}", copy_server);
+        warn!(
+          "Server onboarding: Failed to find Server {}",
+          copy_server
+        );
         Default::default()
       }
       Err(e) => {
-        warn!("Failed to query database for onboarding key 'copy_server' | {e:?}");
+        warn!(
+          "Failed to query database for onboarding key 'copy_server' | {e:?}"
+        );
         Default::default()
       }
     };
@@ -340,10 +356,14 @@ impl PublicKeyValidator for CreationKeyValidator {
       .await
       .context("Failed to query database for Server onboarding keys")?
       .context("Matching Server onboarding key not found")?;
-    if onboarding_key.enabled {
+    // Check enabled and not expired.
+    if onboarding_key.enabled
+      && (onboarding_key.expires == 0
+        || onboarding_key.expires > komodo_timestamp())
+    {
       Ok(onboarding_key)
     } else {
-      Err(anyhow!("Onboarding key is disabled"))
+      Err(anyhow!("Onboarding key is invalid"))
     }
   }
 }

bin/core/src/helpers/builder.rs

@@ -31,8 +31,16 @@ use super::periphery_client;
 const BUILDER_POLL_RATE_SECS: u64 = 2;
 const BUILDER_POLL_MAX_TRIES: usize = 60;
 
-#[instrument(skip_all, fields(builder_id = builder.id, update_id = update.id))]
-pub async fn get_builder_periphery(
+#[instrument(
+  "ConnectBuilderPeriphery",
+  skip_all,
+  fields(
+    resource_name,
+    builder_id = builder.id,
+    update_id = update.id
+  )
+)]
+pub async fn connect_builder_periphery(
   // build: &Build,
   resource_name: String,
   version: Option<Version>,
@@ -54,7 +62,6 @@ pub async fn get_builder_periphery(
           &config,
         ),
         config.insecure_tls,
-        &config.passkey,
       )
       .await?;
       periphery
@@ -77,7 +84,14 @@ pub async fn get_builder_periphery(
   }
 }
 
-#[instrument(skip_all, fields(resource_name, update_id = update.id))]
+#[instrument(
+  "GetAwsBuilder",
+  skip_all,
+  fields(
+    resource_name,
+    update_id = update.id,
+  )
+)]
 async fn get_aws_builder(
   resource_name: &str,
   version: Option<Version>,
@@ -116,7 +130,6 @@ async fn get_aws_builder(
       &config,
     ),
     config.insecure_tls,
-    "",
   )
   .await?;
 
@@ -170,7 +183,11 @@ async fn get_aws_builder(
   )
 }
 
-#[instrument(skip(update))]
+#[instrument(
+  "CleanupBuilderInstance",
+  skip_all,
+  fields(update_id = update.id)
+)]
 pub async fn cleanup_builder_instance(
   periphery: PeripheryClient,
   cleanup_data: BuildCleanupData,

bin/core/src/helpers/mod.rs

@@ -15,7 +15,6 @@ use komodo_client::entities::{
   stack::Stack,
   user::User,
 };
-use rand::Rng;
 
 use crate::{
   config::core_config, connection::PeripheryConnectionArgs,
@@ -47,14 +46,6 @@ pub fn empty_or_only_spaces(word: &str) -> bool {
   true
 }
 
-pub fn random_string(length: usize) -> String {
-  rand::rng()
-    .sample_iter(&rand::distr::Alphanumeric)
-    .take(length)
-    .map(char::from)
-    .collect()
-}
-
 /// First checks db for token, then checks core config.
 /// Only errors if db call errors.
 /// Returns (token, use_https)
@@ -195,12 +186,18 @@ pub async fn periphery_client(
   PeripheryClient::new(
     PeripheryConnectionArgs::from_server(server),
     server.config.insecure_tls,
-    &server.config.passkey,
   )
   .await
 }
 
-#[instrument]
+#[instrument(
+  "CreatePermission",
+  skip(user),
+  fields(
+    operator = user.id,
+    username = user.username
+  )
+)]
 pub async fn create_permission<T>(
   user: &User,
   target: T,

bin/core/src/helpers/procedure.rs

@@ -10,7 +10,6 @@ use komodo_client::{
     action::Action,
     build::Build,
     deployment::Deployment,
-    permission::PermissionLevel,
     procedure::Procedure,
     repo::Repo,
     stack::Stack,
@@ -20,6 +19,7 @@ use komodo_client::{
 };
 use resolver_api::Resolve;
 use tokio::sync::Mutex;
+use uuid::Uuid;
 
 use crate::{
   api::{
@@ -32,7 +32,6 @@ use crate::{
 
 use super::update::{init_execution_update, update_update};
 
-#[instrument(skip_all)]
 pub async fn execute_procedure(
   procedure: &Procedure,
   update: &Mutex<Update>,
@@ -51,7 +50,7 @@ pub async fn execute_procedure(
     )
     .await;
     let timer = Instant::now();
-    execute_stage(
+    execute_procedure_stage(
       stage
         .executions
         .iter()
@@ -86,9 +85,8 @@ pub async fn execute_procedure(
   Ok(())
 }
 
-#[allow(dependency_on_unit_never_type_fallback)]
-#[instrument(skip(update))]
-async fn execute_stage(
+#[instrument("ExecuteProcedureStage", skip_all)]
+async fn execute_procedure_stage(
   _executions: Vec<Execution>,
   parent_id: &str,
   parent_name: &str,
@@ -218,6 +216,10 @@ async fn execute_stage(
   Ok(())
 }
 
+#[instrument(
+  "ExecuteProcedureExecution",
+  skip(parent_id, parent_name)
+)]
 async fn execute_execution(
   execution: Execution,
   // used to prevent recursive procedure
@@ -225,6 +227,7 @@ async fn execute_execution(
   parent_name: &str,
 ) -> anyhow::Result<()> {
   let user = procedure_user().to_owned();
+  let id = Uuid::new_v4();
   let update = match execution {
     Execution::None(_) => return Ok(()),
     Execution::RunProcedure(req) => {
@@ -239,7 +242,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at RunProcedure"),
@@ -262,7 +265,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at RunAction"),
@@ -285,7 +288,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at RunBuild"),
@@ -308,7 +311,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at CancelBuild"),
@@ -325,7 +328,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at Deploy"),
@@ -348,7 +351,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at PullDeployment"),
@@ -365,7 +368,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at StartDeployment"),
@@ -382,7 +385,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at RestartDeployment"),
@@ -399,7 +402,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at PauseDeployment"),
@@ -416,7 +419,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at UnpauseDeployment"),
@@ -433,7 +436,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at StopDeployment"),
@@ -450,7 +453,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at RemoveDeployment"),
@@ -473,7 +476,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at CloneRepo"),
@@ -496,7 +499,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at PullRepo"),
@@ -519,7 +522,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at BuildRepo"),
@@ -542,7 +545,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at CancelRepoBuild"),
@@ -559,7 +562,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at StartContainer"),
@@ -576,7 +579,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at RestartContainer"),
@@ -593,7 +596,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at PauseContainer"),
@@ -610,7 +613,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at UnpauseContainer"),
@@ -627,7 +630,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at StopContainer"),
@@ -644,7 +647,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at RemoveContainer"),
@@ -661,7 +664,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at StartAllContainers"),
@@ -678,7 +681,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at RestartAllContainers"),
@@ -695,7 +698,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at PauseAllContainers"),
@@ -712,7 +715,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at UnpauseAllContainers"),
@@ -729,7 +732,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at StopAllContainers"),
@@ -746,7 +749,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at PruneContainers"),
@@ -763,7 +766,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at DeleteNetwork"),
@@ -780,7 +783,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at PruneNetworks"),
@@ -797,7 +800,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at DeleteImage"),
@@ -814,7 +817,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at PruneImages"),
@@ -831,7 +834,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at DeleteVolume"),
@@ -848,7 +851,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at PruneVolumes"),
@@ -865,7 +868,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at PruneDockerBuilders"),
@@ -882,7 +885,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at PruneBuildx"),
@@ -899,7 +902,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at PruneSystem"),
@@ -916,7 +919,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at RunSync"),
@@ -939,7 +942,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at DeployStack"),
@@ -962,7 +965,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at DeployStackIfChanged"),
@@ -985,7 +988,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at PullStack"),
@@ -1008,7 +1011,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at StartStack"),
@@ -1025,7 +1028,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at RestartStack"),
@@ -1042,7 +1045,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at PauseStack"),
@@ -1059,7 +1062,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at UnpauseStack"),
@@ -1076,7 +1079,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at StopStack"),
@@ -1093,7 +1096,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at DestroyStack"),
@@ -1110,7 +1113,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at RunStackService"),
@@ -1133,7 +1136,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at TestAlerter"),
@@ -1150,7 +1153,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at SendAlert"),
@@ -1167,7 +1170,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at ClearRepoCache"),
@@ -1184,7 +1187,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at BackupCoreDatabase"),
@@ -1201,7 +1204,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at GlobalAutoUpdate"),
@@ -1218,7 +1221,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at RotateAllServerKeys"),
@@ -1226,6 +1229,23 @@ async fn execute_execution(
       )
       .await?
     }
+    Execution::RotateCoreKeys(req) => {
+      let req = ExecuteRequest::RotateCoreKeys(req);
+      let update = init_execution_update(&req, &user).await?;
+      let ExecuteRequest::RotateCoreKeys(req) = req else {
+        unreachable!()
+      };
+      let update_id = update.id.clone();
+      handle_resolve_result(
+        req
+          .resolve(&ExecuteArgs { user, update, id })
+          .await
+          .map_err(|e| e.error)
+          .context("Failed at RotateCoreKeys"),
+        &update_id,
+      )
+      .await?
+    }
     Execution::Sleep(req) => {
       let duration = Duration::from_millis(req.duration_ms as u64);
       tokio::time::sleep(duration).await;
@@ -1271,7 +1291,6 @@ async fn handle_resolve_result(
 }
 
 /// ASSUMES FIRST LOG IS ALREADY CREATED
-#[instrument(level = "debug")]
 async fn add_line_to_update(update: &Mutex<Update>, line: &str) {
   let mut lock = update.lock().await;
   let log = &mut lock.logs[0];
@@ -1292,7 +1311,6 @@ async fn extend_batch_exection<E: ExtendBatch>(
     pattern,
     Default::default(),
     procedure_user(),
-    PermissionLevel::Read.into(),
     &[],
   )
   .await?

bin/core/src/helpers/query.rs

@@ -47,7 +47,6 @@ use crate::{
 };
 
 // user: Id or username
-#[instrument(level = "debug")]
 pub async fn get_user(user: &str) -> anyhow::Result<User> {
   if let Some(user) = admin_service_user(user) {
     return Ok(user);
@@ -60,7 +59,6 @@ pub async fn get_user(user: &str) -> anyhow::Result<User> {
     .with_context(|| format!("no user found with {user}"))
 }
 
-#[instrument(level = "debug")]
 pub async fn get_server_with_state(
   server_id_or_name: &str,
 ) -> anyhow::Result<(Server, ServerState)> {
@@ -69,7 +67,6 @@ pub async fn get_server_with_state(
   Ok((server, state))
 }
 
-#[instrument(level = "debug")]
 pub async fn get_server_state(server: &Server) -> ServerState {
   if !server.config.enabled {
     return ServerState::Disabled;
@@ -86,7 +83,6 @@ pub async fn get_server_state(server: &Server) -> ServerState {
   }
 }
 
-#[instrument(level = "debug")]
 pub async fn get_deployment_state(
   id: &String,
 ) -> anyhow::Result<DeploymentState> {
@@ -182,7 +178,6 @@ pub fn get_stack_state_from_containers(
   StackState::Unhealthy
 }
 
-#[instrument(level = "debug")]
 pub async fn get_stack_state(
   stack: &Stack,
 ) -> anyhow::Result<StackState> {
@@ -198,7 +193,6 @@ pub async fn get_stack_state(
   Ok(state)
 }
 
-#[instrument(level = "debug")]
 pub async fn get_tag(id_or_name: &str) -> anyhow::Result<Tag> {
   let query = match ObjectId::from_str(id_or_name) {
     Ok(id) => doc! { "_id": id },
@@ -212,7 +206,6 @@ pub async fn get_tag(id_or_name: &str) -> anyhow::Result<Tag> {
     .with_context(|| format!("no tag found matching {id_or_name}"))
 }
 
-#[instrument(level = "debug")]
 pub async fn get_tag_check_owner(
   id_or_name: &str,
   user: &User,
@@ -244,7 +237,6 @@ pub async fn get_id_to_tags(
   Ok(res)
 }
 
-#[instrument(level = "debug")]
 pub async fn get_user_user_groups(
   user_id: &str,
 ) -> anyhow::Result<Vec<UserGroup>> {
@@ -262,7 +254,6 @@ pub async fn get_user_user_groups(
   .context("failed to query db for user groups")
 }
 
-#[instrument(level = "debug")]
 pub async fn get_user_user_group_ids(
   user_id: &str,
 ) -> anyhow::Result<Vec<String>> {

bin/core/src/helpers/update.rs

@@ -40,7 +40,6 @@ pub fn make_update(
   }
 }
 
-#[instrument(level = "debug")]
 pub async fn add_update(
   mut update: Update,
 ) -> anyhow::Result<String> {
@@ -59,7 +58,6 @@ pub async fn add_update(
   Ok(id)
 }
 
-#[instrument(level = "debug")]
 pub async fn add_update_without_send(
   update: &Update,
 ) -> anyhow::Result<String> {
@@ -75,7 +73,6 @@ pub async fn add_update_without_send(
   Ok(id)
 }
 
-#[instrument(level = "debug")]
 pub async fn update_update(update: Update) -> anyhow::Result<()> {
   update_one_by_id(&db_client().updates, &update.id, database::mungos::update::Update::Set(to_document(&update)?), None)
     .await
@@ -85,7 +82,6 @@ pub async fn update_update(update: Update) -> anyhow::Result<()> {
   Ok(())
 }
 
-#[instrument(level = "debug")]
 async fn update_list_item(
   update: Update,
 ) -> anyhow::Result<UpdateListItem> {
@@ -115,7 +111,6 @@ async fn update_list_item(
   Ok(update)
 }
 
-#[instrument(level = "debug")]
 async fn send_update(update: UpdateListItem) -> anyhow::Result<()> {
   update_channel().sender.lock().await.send(update)?;
   Ok(())
@@ -523,6 +518,9 @@ pub async fn init_execution_update(
     ExecuteRequest::RotateAllServerKeys(_data) => {
       (Operation::RotateAllServerKeys, ResourceTarget::system())
     }
+    ExecuteRequest::RotateCoreKeys(_data) => {
+      (Operation::RotateCoreKeys, ResourceTarget::system())
+    }
   };
 
   let mut update = make_update(target, operation, user);

bin/core/src/listener/resources.rs

@@ -14,6 +14,7 @@ use komodo_client::{
 use resolver_api::Resolve;
 use serde::Deserialize;
 use serde_json::json;
+use uuid::Uuid;
 
 use crate::{
   api::{
@@ -21,6 +22,7 @@ use crate::{
     write::WriteArgs,
   },
   helpers::update::init_execution_update,
+  resource,
 };
 
 use super::{ANY_BRANCH, ListenerLockCache};
@@ -54,7 +56,18 @@ pub async fn handle_build_webhook<B: super::ExtractBranch>(
   let lock = build_locks().get_or_insert_default(&build.id).await;
   let _lock = lock.lock().await;
 
-  B::verify_branch(&body, &build.config.branch)?;
+  // Use the correct target branch when using linked repo.
+  let branch = if build.config.linked_repo.is_empty() {
+    build.config.branch
+  } else {
+    resource::get::<Repo>(&build.config.linked_repo)
+      .await
+      .context("Failed to find 'linked_repo'")?
+      .config
+      .branch
+  };
+
+  B::verify_branch(&body, &branch)?;
 
   let user = git_webhook_user().to_owned();
   let req = ExecuteRequest::RunBuild(RunBuild { build: build.id });
@@ -63,7 +76,11 @@ pub async fn handle_build_webhook<B: super::ExtractBranch>(
     unreachable!()
   };
   req
-    .resolve(&ExecuteArgs { user, update })
+    .resolve(&ExecuteArgs {
+      user,
+      update,
+      id: Uuid::new_v4(),
+    })
     .await
     .map_err(|e| e.error)?;
   Ok(())
@@ -101,7 +118,11 @@ impl RepoExecution for CloneRepo {
       unreachable!()
     };
     req
-      .resolve(&ExecuteArgs { user, update })
+      .resolve(&ExecuteArgs {
+        user,
+        update,
+        id: Uuid::new_v4(),
+      })
       .await
       .map_err(|e| e.error)?;
     Ok(())
@@ -121,7 +142,11 @@ impl RepoExecution for PullRepo {
       unreachable!()
     };
     req
-      .resolve(&ExecuteArgs { user, update })
+      .resolve(&ExecuteArgs {
+        user,
+        update,
+        id: Uuid::new_v4(),
+      })
       .await
       .map_err(|e| e.error)?;
     Ok(())
@@ -141,7 +166,11 @@ impl RepoExecution for BuildRepo {
       unreachable!()
     };
     req
-      .resolve(&ExecuteArgs { user, update })
+      .resolve(&ExecuteArgs {
+        user,
+        update,
+        id: Uuid::new_v4(),
+      })
       .await
       .map_err(|e| e.error)?;
     Ok(())
@@ -240,7 +269,11 @@ impl StackExecution for DeployStack {
         unreachable!()
       };
       req
-        .resolve(&ExecuteArgs { user, update })
+        .resolve(&ExecuteArgs {
+          user,
+          update,
+          id: Uuid::new_v4(),
+        })
         .await
         .map_err(|e| e.error)?;
     } else {
@@ -254,7 +287,11 @@ impl StackExecution for DeployStack {
         unreachable!()
       };
       req
-        .resolve(&ExecuteArgs { user, update })
+        .resolve(&ExecuteArgs {
+          user,
+          update,
+          id: Uuid::new_v4(),
+        })
         .await
         .map_err(|e| e.error)?;
     }
@@ -303,7 +340,18 @@ pub async fn handle_stack_webhook_inner<
   let lock = stack_locks().get_or_insert_default(&stack.id).await;
   let _lock = lock.lock().await;
 
-  B::verify_branch(&body, &stack.config.branch)?;
+  // Use the correct target branch when using linked repo.
+  let branch = if stack.config.linked_repo.is_empty() {
+    stack.config.branch.clone()
+  } else {
+    resource::get::<Repo>(&stack.config.linked_repo)
+      .await
+      .context("Failed to find 'linked_repo'")?
+      .config
+      .branch
+  };
+
+  B::verify_branch(&body, &branch)?;
 
   E::resolve(stack).await.map_err(|e| e.error)
 }
@@ -352,7 +400,11 @@ impl SyncExecution for RunSync {
       unreachable!()
     };
     req
-      .resolve(&ExecuteArgs { user, update })
+      .resolve(&ExecuteArgs {
+        user,
+        update,
+        id: Uuid::new_v4(),
+      })
       .await
       .map_err(|e| e.error)?;
     Ok(())
@@ -401,7 +453,18 @@ async fn handle_sync_webhook_inner<
   let lock = sync_locks().get_or_insert_default(&sync.id).await;
   let _lock = lock.lock().await;
 
-  B::verify_branch(&body, &sync.config.branch)?;
+  // Use the correct target branch when using linked repo.
+  let branch = if sync.config.linked_repo.is_empty() {
+    sync.config.branch.clone()
+  } else {
+    resource::get::<Repo>(&sync.config.linked_repo)
+      .await
+      .context("Failed to find 'linked_repo'")?
+      .config
+      .branch
+  };
+
+  B::verify_branch(&body, &branch)?;
 
   E::resolve(sync).await
 }
@@ -451,7 +514,11 @@ pub async fn handle_procedure_webhook<B: super::ExtractBranch>(
     unreachable!()
   };
   req
-    .resolve(&ExecuteArgs { user, update })
+    .resolve(&ExecuteArgs {
+      user,
+      update,
+      id: Uuid::new_v4(),
+    })
     .await
     .map_err(|e| e.error)?;
   Ok(())
@@ -513,7 +580,11 @@ pub async fn handle_action_webhook<B: super::ExtractBranch>(
     unreachable!()
   };
   req
-    .resolve(&ExecuteArgs { user, update })
+    .resolve(&ExecuteArgs {
+      user,
+      update,
+      id: Uuid::new_v4(),
+    })
     .await
     .map_err(|e| e.error)?;
   Ok(())

bin/core/src/main.rs

@@ -6,14 +6,15 @@ extern crate tracing;
 use std::{net::SocketAddr, str::FromStr};
 
 use anyhow::Context;
-use axum::Router;
+use axum::{Router, routing::get};
 use axum_server::{Handle, tls_rustls::RustlsConfig};
 use tower_http::{
   cors::{Any, CorsLayer},
   services::{ServeDir, ServeFile},
 };
+use tracing::Instrument;
 
-use crate::config::{core_config, core_public_key};
+use crate::config::{core_config, core_keys};
 
 mod alert;
 mod api;
@@ -41,46 +42,51 @@ async fn app() -> anyhow::Result<()> {
   let config = core_config();
   logger::init(&config.logging)?;
 
-  info!("Komodo Core version: v{}", env!("CARGO_PKG_VERSION"));
+  let startup_span = info_span!("CoreStartup");
+  async {
+    info!("Komodo Core version: v{}", env!("CARGO_PKG_VERSION"));
 
-  match (
-    config.pretty_startup_config,
-    config.unsafe_unsanitized_startup_config,
-  ) {
-    (true, true) => info!("{:#?}", config),
-    (true, false) => info!("{:#?}", config.sanitized()),
-    (false, true) => info!("{:?}", config),
-    (false, false) => info!("{:?}", config.sanitized()),
+    match (
+      config.pretty_startup_config,
+      config.unsafe_unsanitized_startup_config,
+    ) {
+      (true, true) => info!("{:#?}", config),
+      (true, false) => info!("{:#?}", config.sanitized()),
+      (false, true) => info!("{:?}", config),
+      (false, false) => info!("{:?}", config.sanitized()),
+    }
+
+    // Init + log public key. Will crash if invalid private key here.
+    info!("Public Key: {}", core_keys().load().public);
+
+    rustls::crypto::aws_lc_rs::default_provider()
+      .install_default()
+      .expect("Failed to install default crypto provider");
+
+    // Init jwt client to crash on failure
+    state::jwt_client();
+    tokio::join!(
+      // Init db_client check to crash on db init failure
+      state::init_db_client(),
+      // Manage OIDC client (defined in config / env vars / compose secret file)
+      auth::oidc::client::spawn_oidc_client_management()
+    );
+    // Run after db connection.
+    startup::on_startup().await;
+
+    // Spawn background tasks
+    monitor::spawn_monitor_loop();
+    resource::spawn_resource_refresh_loop();
+    resource::spawn_all_resources_cache_refresh_loop();
+    resource::spawn_build_state_refresh_loop();
+    resource::spawn_repo_state_refresh_loop();
+    resource::spawn_procedure_state_refresh_loop();
+    resource::spawn_action_state_refresh_loop();
+    schedule::spawn_schedule_executor();
+    helpers::prune::spawn_prune_loop();
   }
-
-  // Init + log public key. Will crash if invalid private key here.
-  info!("Public Key: {}", core_public_key());
-
-  rustls::crypto::aws_lc_rs::default_provider()
-    .install_default()
-    .expect("Failed to install default crypto provider");
-
-  // Init jwt client to crash on failure
-  state::jwt_client();
-  tokio::join!(
-    // Init db_client check to crash on db init failure
-    state::init_db_client(),
-    // Manage OIDC client (defined in config / env vars / compose secret file)
-    auth::oidc::client::spawn_oidc_client_management()
-  );
-  // Run after db connection.
-  startup::on_startup().await;
-
-  // Spawn background tasks
-  monitor::spawn_monitor_loop();
-  resource::spawn_resource_refresh_loop();
-  resource::spawn_all_resources_cache_refresh_loop();
-  resource::spawn_build_state_refresh_loop();
-  resource::spawn_repo_state_refresh_loop();
-  resource::spawn_procedure_state_refresh_loop();
-  resource::spawn_action_state_refresh_loop();
-  schedule::spawn_schedule_executor();
-  helpers::prune::spawn_prune_loop();
+  .instrument(startup_span)
+  .await;
 
   // Setup static frontend services
   let frontend_path = &config.frontend_path;
@@ -90,6 +96,7 @@ async fn app() -> anyhow::Result<()> {
     .not_found_service(frontend_index.clone());
 
   let app = Router::new()
+    .route("/version", get(|| async { env!("CARGO_PKG_VERSION") }))
     .nest("/auth", api::auth::router())
     .nest("/user", api::user::router())
     .nest("/read", api::read::router())

bin/core/src/monitor/alert/deployment.rs

@@ -13,7 +13,6 @@ use crate::{
   state::{action_states, db_client},
 };
 
-#[instrument(level = "debug")]
 pub async fn alert_deployments(
   ts: i64,
   server_names: &HashMap<String, String>,

bin/core/src/monitor/alert/mod.rs

@@ -2,8 +2,7 @@ use std::collections::HashMap;
 
 use anyhow::Context;
 use komodo_client::entities::{
-  permission::PermissionLevel, resource::ResourceQuery,
-  server::Server, user::User,
+  resource::ResourceQuery, server::Server, user::User,
 };
 
 use crate::resource;
@@ -13,7 +12,6 @@ mod server;
 mod stack;
 
 // called after cache update
-#[instrument(level = "debug")]
 pub async fn check_alerts(ts: i64) {
   let (servers, server_names) = match get_all_servers_map().await {
     Ok(res) => res,
@@ -30,7 +28,6 @@ pub async fn check_alerts(ts: i64) {
   );
 }
 
-#[instrument(level = "debug")]
 async fn get_all_servers_map()
 -> anyhow::Result<(HashMap<String, Server>, HashMap<String, String>)>
 {
@@ -40,7 +37,6 @@ async fn get_all_servers_map()
       admin: true,
       ..Default::default()
     },
-    PermissionLevel::Read.into(),
     &[],
   )
   .await

bin/core/src/monitor/alert/server.rs

@@ -73,7 +73,6 @@ fn alert_buffer() -> &'static AlertBuffer {
   BUFFER.get_or_init(AlertBuffer::new)
 }
 
-#[instrument(level = "debug")]
 pub async fn alert_servers(
   ts: i64,
   mut servers: HashMap<String, Server>,
@@ -555,7 +554,6 @@ pub async fn alert_servers(
   );
 }
 
-#[instrument(level = "debug")]
 async fn open_new_alerts(alerts: &[(Alert, SendAlerts)]) {
   if alerts.is_empty() {
     return;
@@ -603,7 +601,6 @@ async fn open_new_alerts(alerts: &[(Alert, SendAlerts)]) {
   send_alerts(&alerts).await
 }
 
-#[instrument(level = "debug")]
 async fn update_alerts(alerts: &[(Alert, SendAlerts)]) {
   if alerts.is_empty() {
     return;
@@ -651,7 +648,6 @@ async fn update_alerts(alerts: &[(Alert, SendAlerts)]) {
   }
 }
 
-#[instrument(level = "debug")]
 async fn resolve_alerts(alerts: &[(Alert, SendAlerts)]) {
   if alerts.is_empty() {
     return;
@@ -708,7 +704,6 @@ async fn resolve_alerts(alerts: &[(Alert, SendAlerts)]) {
   }
 }
 
-#[instrument(level = "debug")]
 async fn get_open_alerts()
 -> anyhow::Result<(OpenAlertMap, OpenDiskAlertMap)> {
   let alerts = find_collect(

bin/core/src/monitor/alert/stack.rs

@@ -12,7 +12,6 @@ use crate::{
   state::{action_states, db_client, stack_status_cache},
 };
 
-#[instrument(level = "debug")]
 pub async fn alert_stacks(
   ts: i64,
   server_names: &HashMap<String, String>,

bin/core/src/monitor/helpers.rs

@@ -25,7 +25,6 @@ use super::{
   CachedStackStatus, History,
 };
 
-#[instrument(level = "debug", skip_all)]
 pub async fn insert_deployments_status_unknown(
   deployments: Vec<Deployment>,
 ) {
@@ -51,7 +50,6 @@ pub async fn insert_deployments_status_unknown(
   }
 }
 
-#[instrument(level = "debug", skip_all)]
 pub async fn insert_repos_status_unknown(repos: Vec<Repo>) {
   let status_cache = repo_status_cache();
   for repo in repos {
@@ -68,7 +66,6 @@ pub async fn insert_repos_status_unknown(repos: Vec<Repo>) {
   }
 }
 
-#[instrument(level = "debug", skip_all)]
 pub async fn insert_stacks_status_unknown(stacks: Vec<Stack>) {
   let status_cache = stack_status_cache();
   for stack in stacks {
@@ -99,7 +96,6 @@ type DockerLists = (
   Option<Vec<ComposeProject>>,
 );
 
-#[instrument(level = "debug", skip_all)]
 pub async fn insert_server_status(
   server: &Server,
   state: ServerState,

bin/core/src/monitor/mod.rs

@@ -140,7 +140,6 @@ fn update_cache_for_server_controller()
 /// which exits early if the lock is busy or it was completed too recently.
 /// If force is true, it will wait on simultaneous calls, and will
 /// ignore the restriction on being completed too recently.
-#[instrument(level = "debug")]
 pub async fn update_cache_for_server(server: &Server, force: bool) {
   // Concurrency controller to ensure it isn't done too often
   // when it happens in other contexts.

bin/core/src/monitor/record.rs

@@ -4,7 +4,6 @@ use komodo_client::entities::stats::{
 
 use crate::state::{db_client, server_status_cache};
 
-#[instrument(level = "debug")]
 pub async fn record_server_stats(ts: i64) {
   let status = server_status_cache().get_values().await;
   let records = status

bin/core/src/periphery/mod.rs

@@ -1,42 +1,35 @@
 use std::{sync::Arc, time::Duration};
 
 use anyhow::{Context, anyhow};
-use bytes::Bytes;
-use cache::CloneCache;
+use encoding::Decode as _;
 use periphery_client::api;
 use resolver_api::HasResponse;
 use serde::{Serialize, de::DeserializeOwned};
 use serde_json::json;
-use serror::deserialize_error_bytes;
-use tokio::sync::mpsc::{self, Sender};
-use tracing::warn;
-use transport::{
-  MessageState,
-  bytes::{from_transport_bytes, to_transport_bytes},
-};
+use transport::channel::channel;
 use uuid::Uuid;
 
 use crate::{
-  connection::{PeripheryConnection, PeripheryConnectionArgs},
+  connection::{
+    PeripheryConnection, PeripheryConnectionArgs, ResponseChannels,
+    TerminalChannels,
+  },
   state::periphery_connections,
 };
 
 pub mod terminal;
 
-pub type ConnectionChannels = CloneCache<Uuid, Sender<Bytes>>;
-
 #[derive(Debug)]
 pub struct PeripheryClient {
   pub id: String,
-  channels: Arc<ConnectionChannels>,
+  pub responses: Arc<ResponseChannels>,
+  pub terminals: Arc<TerminalChannels>,
 }
 
 impl PeripheryClient {
   pub async fn new(
     args: PeripheryConnectionArgs<'_>,
     insecure_tls: bool,
-    // deprecated.
-    passkey: &str,
   ) -> anyhow::Result<PeripheryClient> {
     let connections = periphery_connections();
 
@@ -47,21 +40,17 @@ impl PeripheryClient {
       if args.address.is_none() {
         return Err(anyhow!("Server {id} is not connected"));
       }
-      let channels = args
-        .spawn_client_connection(
-          id.clone(),
-          insecure_tls,
-          passkey.to_string(),
-        )
-        .await?;
-      return Ok(PeripheryClient { id, channels });
+      return args
+        .spawn_client_connection(id.clone(), insecure_tls)
+        .await;
     };
 
     // Ensure the connection args are unchanged.
     if args.matches(&connection.args) {
       return Ok(PeripheryClient {
         id,
-        channels: connection.channels.clone(),
+        responses: connection.responses.clone(),
+        terminals: connection.terminals.clone(),
       });
     }
 
@@ -77,18 +66,12 @@ impl PeripheryClient {
         .with_context(|| format!("Server {id} is not connected"))?;
       Ok(PeripheryClient {
         id,
-        channels: connection.channels.clone(),
+        responses: connection.responses.clone(),
+        terminals: connection.terminals.clone(),
       })
     } else {
       // Core -> Periphery connection
-      let channels = args
-        .spawn_client_connection(
-          id.clone(),
-          insecure_tls,
-          passkey.to_string(),
-        )
-        .await?;
-      Ok(PeripheryClient { id, channels })
+      args.spawn_client_connection(id.clone(), insecure_tls).await
     }
   }
 
@@ -96,17 +79,11 @@ impl PeripheryClient {
     periphery_connections().remove(&self.id).await
   }
 
-  #[tracing::instrument(level = "debug", skip(self))]
   pub async fn health_check(&self) -> anyhow::Result<()> {
     self.request(api::GetHealth {}).await?;
     Ok(())
   }
 
-  #[tracing::instrument(
-    name = "PeripheryRequest",
-    skip(self),
-    level = "debug"
-  )]
   pub async fn request<T>(
     &self,
     request: T,
@@ -123,83 +100,40 @@ impl PeripheryClient {
     // Polls connected 3 times before bailing
     connection.bail_if_not_connected().await?;
 
-    let id = Uuid::new_v4();
-    let (response_sender, mut response_receiever) =
-      mpsc::channel(1000);
-    self.channels.insert(id, response_sender).await;
-
-    let req_type = T::req_type();
-    let data = serde_json::to_vec(&json!({
-      "type": req_type,
-      "params": request
-    }))
-    .context("Failed to serialize request to bytes")?;
+    let channel_id = Uuid::new_v4();
+    let (response_sender, mut response_receiever) = channel();
+    self.responses.insert(channel_id, response_sender).await;
 
     if let Err(e) = connection
-      .send(to_transport_bytes(data, id, MessageState::Request))
+      .sender
+      .send_request(
+        channel_id,
+        &json!({
+          "type": T::req_type(),
+          "params": request
+        }),
+      )
       .await
       .context("Failed to send request over channel")
     {
       // cleanup
-      self.channels.remove(&id).await;
+      self.terminals.remove(&channel_id).await;
       return Err(e);
     }
 
     // Poll for the associated response
     loop {
-      let next = tokio::select! {
-        msg = response_receiever.recv() => msg,
-        // Periphery will send InProgress every 5s to avoid timeout
-        _ = tokio::time::sleep(Duration::from_secs(10)) => {
-          return Err(anyhow!("Response timed out"));
-        }
+      let message = response_receiever
+        .recv()
+        .with_timeout(Duration::from_secs(10))
+        .await?;
+
+      // Still in progress, sent to avoid timeout.
+      let Some(message) = message.decode()? else {
+        continue;
       };
 
-      let bytes = match next {
-        Some(bytes) => bytes,
-        None => {
-          return Err(anyhow!(
-            "Sender dropped before response was recieved"
-          ));
-        }
-      };
-
-      let (state, data) = match from_transport_bytes(bytes) {
-        Ok((data, _, state)) if !data.is_empty() => (state, data),
-        // Ignore no data cases
-        Ok(_) => continue,
-        Err(e) => {
-          warn!(
-            "Server {} | Received invalid message | {e:#}",
-            self.id
-          );
-          continue;
-        }
-      };
-      match state {
-        // TODO: improve the allocation in .to_vec
-        MessageState::Successful => {
-          // cleanup
-          self.channels.remove(&id).await;
-          return serde_json::from_slice(&data)
-            .context("Failed to parse successful response");
-        }
-        MessageState::Failed => {
-          // cleanup
-          self.channels.remove(&id).await;
-          return Err(deserialize_error_bytes(&data));
-        }
-        MessageState::InProgress => continue,
-        // Shouldn't be received by this receiver
-        other => {
-          // TODO: delete log
-          warn!(
-            "Server {} | Got other message over over response channel: {other:?}",
-            self.id
-          );
-          continue;
-        }
-      }
+      return message.decode();
     }
   }
 }

bin/core/src/periphery/terminal.rs

@@ -8,12 +8,15 @@ use anyhow::Context;
 use bytes::Bytes;
 use cache::CloneCache;
 use futures::Stream;
-use periphery_client::api::terminal::{
-  ConnectContainerExec, ConnectTerminal, END_OF_OUTPUT,
-  ExecuteContainerExec, ExecuteTerminal,
+use komodo_client::api::write::TerminalRecreateMode;
+use periphery_client::{
+  api::terminal::{
+    ConnectContainerAttach, ConnectContainerExec, ConnectTerminal,
+    END_OF_OUTPUT, ExecuteContainerExec, ExecuteTerminal,
+  },
+  transport::EncodedTransportMessage,
 };
-use tokio::sync::mpsc::{Receiver, Sender, channel};
-use transport::bytes::data_from_transport_bytes;
+use transport::channel::{Receiver, Sender, channel};
 use uuid::Uuid;
 
 use crate::{
@@ -21,10 +24,15 @@ use crate::{
 };
 
 impl PeripheryClient {
+  #[instrument("ConnectTerminal", skip(self), fields(server_id = self.id))]
   pub async fn connect_terminal(
     &self,
     terminal: String,
-  ) -> anyhow::Result<(Uuid, Sender<Bytes>, Receiver<Bytes>)> {
+  ) -> anyhow::Result<(
+    Uuid,
+    Sender<EncodedTransportMessage>,
+    Receiver<Vec<u8>>,
+  )> {
     tracing::trace!(
       "request | type: ConnectTerminal | terminal name: {terminal}",
     );
@@ -34,22 +42,36 @@ impl PeripheryClient {
         || format!("No connection found for server {}", self.id),
       )?;
 
-    let id = self
+    let channel_id = self
       .request(ConnectTerminal { terminal })
       .await
       .context("Failed to create terminal connection")?;
 
-    let (sender, receiever) = channel(1024);
-    connection.channels.insert(id, sender).await;
+    let (sender, receiever) = channel();
+    connection.terminals.insert(channel_id, sender).await;
 
-    Ok((id, connection.sender.clone(), receiever))
+    connection
+      .sender
+      .send_terminal(channel_id, Bytes::new())
+      .await
+      .context(
+        "Failed to send TerminalTrigger to begin forwarding.",
+      )?;
+
+    Ok((channel_id, connection.sender.clone(), receiever))
   }
 
+  #[instrument("ConnectContainerExec", skip(self), fields(server_id = self.id))]
   pub async fn connect_container_exec(
     &self,
     container: String,
     shell: String,
-  ) -> anyhow::Result<(Uuid, Sender<Bytes>, Receiver<Bytes>)> {
+    recreate: TerminalRecreateMode,
+  ) -> anyhow::Result<(
+    Uuid,
+    Sender<EncodedTransportMessage>,
+    Receiver<Vec<u8>>,
+  )> {
     tracing::trace!(
       "request | type: ConnectContainerExec | container name: {container} | shell: {shell}",
     );
@@ -59,15 +81,68 @@ impl PeripheryClient {
         || format!("No connection found for server {}", self.id),
       )?;
 
-    let id = self
-      .request(ConnectContainerExec { container, shell })
+    let channel_id = self
+      .request(ConnectContainerExec {
+        container,
+        shell,
+        recreate,
+      })
       .await
       .context("Failed to create container exec connection")?;
 
-    let (sender, receiever) = channel(1000);
-    connection.channels.insert(id, sender).await;
+    let (sender, receiever) = channel();
+    connection.terminals.insert(channel_id, sender).await;
 
-    Ok((id, connection.sender.clone(), receiever))
+    connection
+      .sender
+      .send_terminal(channel_id, Bytes::new())
+      .await
+      .context(
+        "Failed to send TerminalTrigger to begin forwarding.",
+      )?;
+
+    Ok((channel_id, connection.sender.clone(), receiever))
+  }
+
+  #[instrument("ConnectContainerAttach", skip(self), fields(server_id = self.id))]
+  pub async fn connect_container_attach(
+    &self,
+    container: String,
+    recreate: TerminalRecreateMode,
+  ) -> anyhow::Result<(
+    Uuid,
+    Sender<EncodedTransportMessage>,
+    Receiver<Vec<u8>>,
+  )> {
+    tracing::trace!(
+      "request | type: ConnectContainerAttach | container name: {container}",
+    );
+
+    let connection =
+      periphery_connections().get(&self.id).await.with_context(
+        || format!("No connection found for server {}", self.id),
+      )?;
+
+    let channel = self
+      .request(ConnectContainerAttach {
+        container,
+        recreate,
+      })
+      .await
+      .context("Failed to create container attach connection")?;
+
+    let (sender, receiever) = transport::channel::channel();
+    connection.terminals.insert(channel, sender).await;
+
+    connection
+      .sender
+      .send_terminal(channel, Bytes::new())
+      .await
+      .context(
+        "Failed to send TerminalTrigger to begin forwarding.",
+      )?;
+
+    Ok((channel, connection.sender.clone(), receiever))
   }
 
   /// Executes command on specified terminal,
@@ -83,15 +158,15 @@ impl PeripheryClient {
   ///
   /// If this value is NOT the final item before stream closes, it means
   /// the terminal exited mid command, before giving status. Example: running `exit`.
-  #[tracing::instrument(level = "debug", skip(self))]
+  #[instrument("ExecuteTerminal", skip(self), fields(server_id = self.id))]
   pub async fn execute_terminal(
     &self,
     terminal: String,
     command: String,
   ) -> anyhow::Result<
-    impl Stream<Item = anyhow::Result<Bytes>> + 'static,
+    impl Stream<Item = anyhow::Result<Vec<u8>>> + 'static,
   > {
-    tracing::trace!(
+    trace!(
       "sending request | type: ExecuteTerminal | terminal name: {terminal} | command: {command}",
     );
 
@@ -100,19 +175,29 @@ impl PeripheryClient {
         || format!("No connection found for server {}", self.id),
       )?;
 
-    let id = self
+    let channel_id = self
       .request(ExecuteTerminal { terminal, command })
       .await
       .context("Failed to create execute terminal connection")?;
 
-    let (sender, receiver) = channel(1000);
+    let (terminal_sender, terminal_receiver) = channel();
+    connection
+      .terminals
+      .insert(channel_id, terminal_sender)
+      .await;
 
-    connection.channels.insert(id, sender).await;
+    connection
+      .sender
+      .send_terminal(channel_id, Bytes::new())
+      .await
+      .context(
+        "Failed to send TerminalTrigger to begin forwarding.",
+      )?;
 
     Ok(ReceiverStream {
-      id,
-      receiver,
-      channels: connection.channels.clone(),
+      channel_id,
+      receiver: terminal_receiver,
+      channels: connection.terminals.clone(),
     })
   }
 
@@ -129,12 +214,13 @@ impl PeripheryClient {
   ///
   /// If this value is NOT the final item before stream closes, it means
   /// the container shell exited mid command, before giving status. Example: running `exit`.
-  #[tracing::instrument(level = "debug", skip(self))]
+  #[instrument("ExecuteContainerExec", skip(self), fields(server_id = self.id))]
   pub async fn execute_container_exec(
     &self,
     container: String,
     shell: String,
     command: String,
+    recreate: TerminalRecreateMode,
   ) -> anyhow::Result<ReceiverStream> {
     tracing::trace!(
       "sending request | type: ExecuteContainerExec | container: {container} | shell: {shell} | command: {command}",
@@ -145,50 +231,58 @@ impl PeripheryClient {
         || format!("No connection found for server {}", self.id),
       )?;
 
-    let id = self
+    let channel_id = self
       .request(ExecuteContainerExec {
         container,
         shell,
         command,
+        recreate,
       })
       .await
       .context("Failed to create execute terminal connection")?;
 
-    let (sender, receiver) = channel(1000);
+    let (terminal_sender, terminal_receiver) = channel();
+    connection
+      .terminals
+      .insert(channel_id, terminal_sender)
+      .await;
 
-    connection.channels.insert(id, sender).await;
+    // Trigger forwarding to begin now that forwarding channel is ready.
+    // This is required to not miss messages.
+    connection
+      .sender
+      .send_terminal(channel_id, Bytes::new())
+      .await?;
 
     Ok(ReceiverStream {
-      id,
-      receiver,
-      channels: connection.channels.clone(),
+      channel_id,
+      receiver: terminal_receiver,
+      channels: connection.terminals.clone(),
     })
   }
 }
 
 pub struct ReceiverStream {
-  id: Uuid,
-  channels: Arc<CloneCache<Uuid, Sender<Bytes>>>,
-  receiver: Receiver<Bytes>,
+  channel_id: Uuid,
+  channels: Arc<CloneCache<Uuid, Sender<Vec<u8>>>>,
+  receiver: Receiver<Vec<u8>>,
 }
 
 impl Stream for ReceiverStream {
-  type Item = anyhow::Result<Bytes>;
+  type Item = anyhow::Result<Vec<u8>>;
   fn poll_next(
     mut self: Pin<&mut Self>,
     cx: &mut task::Context<'_>,
   ) -> Poll<Option<Self::Item>> {
-    match self
-      .receiver
-      .poll_recv(cx)
-      .map(|bytes| bytes.map(data_from_transport_bytes))
-    {
-      Poll::Ready(Some(Ok(bytes))) if bytes == END_OF_OUTPUT => {
+    match self.receiver.poll_recv(cx) {
+      Poll::Ready(Some(bytes))
+        if bytes == END_OF_OUTPUT.as_bytes() =>
+      {
         self.cleanup();
         Poll::Ready(None)
       }
-      Poll::Ready(Some(Ok(bytes))) => Poll::Ready(Some(Ok(bytes))),
-      Poll::Ready(Some(Err(e))) => Poll::Ready(Some(Err(e))),
+      Poll::Ready(Some(bytes)) => Poll::Ready(Some(Ok(bytes))),
+      // Poll::Ready(Some(Err(e))) => Poll::Ready(Some(Err(e))),
       Poll::Ready(None) => {
         self.cleanup();
         Poll::Ready(None)
@@ -202,7 +296,7 @@ impl ReceiverStream {
   fn cleanup(&self) {
     // Not the prettiest but it should be fine
     let channels = self.channels.clone();
-    let id = self.id;
+    let id = self.channel_id;
     tokio::spawn(async move {
       channels.remove(&id).await;
     });

bin/core/src/permission.rs

@@ -68,12 +68,11 @@ pub async fn get_check_permissions<T: KomodoResource>(
   }
 }
 
-#[instrument(level = "debug")]
 pub fn get_user_permission_on_resource<'a, T: KomodoResource>(
   user: &'a User,
   resource_id: &'a str,
 ) -> BoxFuture<'a, anyhow::Result<PermissionLevelAndSpecifics>> {
-  Box::pin(async {
+  Box::pin(async move {
     // Admin returns early with max permissions
     if user.admin {
       return Ok(PermissionLevel::Write.all());
@@ -164,7 +163,6 @@ pub fn get_user_permission_on_resource<'a, T: KomodoResource>(
 }
 
 /// Returns None if still no need to filter by resource id (eg transparent mode, group membership with all access).
-#[instrument(level = "debug")]
 pub async fn get_resource_ids_for_user<T: KomodoResource>(
   user: &User,
 ) -> anyhow::Result<Option<Vec<String>>> {

bin/core/src/resource/build.rs

@@ -253,7 +253,7 @@ pub async fn refresh_build_state_cache() {
   });
 }
 
-#[instrument(skip(user))]
+#[instrument("ValidateBuildConfig", skip_all)]
 async fn validate_config(
   config: &mut PartialBuildConfig,
   user: &User,

bin/core/src/resource/builder.rs

@@ -179,7 +179,7 @@ impl super::KomodoResource for Builder {
   }
 }
 
-#[instrument(skip(user))]
+#[instrument("ValidateBuilderConfig", skip_all)]
 async fn validate_config(
   config: &mut PartialBuilderConfig,
   user: &User,

bin/core/src/resource/deployment.rs

@@ -309,7 +309,7 @@ impl super::KomodoResource for Deployment {
   }
 }
 
-#[instrument(skip(user))]
+#[instrument("ValidateDeploymentConfig", skip_all)]
 async fn validate_config(
   config: &mut PartialDeploymentConfig,
   user: &User,

bin/core/src/resource/mod.rs

@@ -21,10 +21,7 @@ use komodo_client::{
   entities::{
     Operation, ResourceTarget, ResourceTargetVariant,
     komodo_timestamp,
-    permission::{
-      PermissionLevel, PermissionLevelAndSpecifics,
-      SpecificPermission,
-    },
+    permission::{PermissionLevel, SpecificPermission},
     resource::{AddFilters, Resource, ResourceQuery},
     tag::Tag,
     to_general_name,
@@ -255,7 +252,6 @@ pub async fn get<T: KomodoResource>(
 // ======
 
 /// Returns None if still no need to filter by resource id (eg transparent mode, group membership with all access).
-#[instrument(level = "debug")]
 pub async fn get_resource_object_ids_for_user<T: KomodoResource>(
   user: &User,
 ) -> anyhow::Result<Option<Vec<ObjectId>>> {
@@ -269,21 +265,19 @@ pub async fn get_resource_object_ids_for_user<T: KomodoResource>(
   })
 }
 
-#[instrument(level = "debug")]
 pub async fn list_for_user<T: KomodoResource>(
   mut query: ResourceQuery<T::QuerySpecifics>,
   user: &User,
-  permissions: PermissionLevelAndSpecifics,
+  // permissions: PermissionLevelAndSpecifics,
   all_tags: &[Tag],
 ) -> anyhow::Result<Vec<T::ListItem>> {
   validate_resource_query_tags(&mut query, all_tags)?;
   let mut filters = Document::new();
   query.add_filters(&mut filters);
-  list_for_user_using_document::<T>(filters, user, permissions).await
+  list_for_user_using_document::<T>(filters, user).await
 }
 
-// #[instrument(level = "debug")]
-// pub async fn list_for_user_using_pattern<T: KomodoResource>(
+// // pub async fn list_for_user_using_pattern<T: KomodoResource>(
 //   pattern: &str,
 //   query: ResourceQuery<T::QuerySpecifics>,
 //   user: &User,
@@ -303,11 +297,10 @@ pub async fn list_for_user<T: KomodoResource>(
 //   Ok(join_all(list).await)
 // }
 
-#[instrument(level = "debug")]
 pub async fn list_for_user_using_document<T: KomodoResource>(
   filters: Document,
   user: &User,
-  permissions: PermissionLevelAndSpecifics,
+  // permissions: PermissionLevelAndSpecifics,
 ) -> anyhow::Result<Vec<T::ListItem>> {
   let list = list_full_for_user_using_document::<T>(filters, user)
     .await?
@@ -324,17 +317,14 @@ pub async fn list_for_user_using_document<T: KomodoResource>(
 /// let items = list_full_for_user_using_match_string::<Build>("foo-*", Default::default(), user, all_tags).await?;
 /// let items = list_full_for_user_using_match_string::<Build>("\\^foo-.*$\\", Default::default(), user, all_tags).await?;
 /// ```
-#[instrument(level = "debug")]
 pub async fn list_full_for_user_using_pattern<T: KomodoResource>(
   pattern: &str,
   query: ResourceQuery<T::QuerySpecifics>,
   user: &User,
-  permissions: PermissionLevelAndSpecifics,
   all_tags: &[Tag],
 ) -> anyhow::Result<Vec<Resource<T::Config, T::Info>>> {
   let resources =
-    list_full_for_user::<T>(query, user, permissions, all_tags)
-      .await?;
+    list_full_for_user::<T>(query, user, all_tags).await?;
 
   let patterns = parse_string_list(pattern);
   let mut names = HashSet::<String>::new();
@@ -367,11 +357,9 @@ pub async fn list_full_for_user_using_pattern<T: KomodoResource>(
   )
 }
 
-#[instrument(level = "debug")]
 pub async fn list_full_for_user<T: KomodoResource>(
   mut query: ResourceQuery<T::QuerySpecifics>,
   user: &User,
-  permissions: PermissionLevelAndSpecifics,
   all_tags: &[Tag],
 ) -> anyhow::Result<Vec<Resource<T::Config, T::Info>>> {
   validate_resource_query_tags(&mut query, all_tags)?;
@@ -380,7 +368,6 @@ pub async fn list_full_for_user<T: KomodoResource>(
   list_full_for_user_using_document::<T>(filters, user).await
 }
 
-#[instrument(level = "debug")]
 pub async fn list_full_for_user_using_document<T: KomodoResource>(
   mut filters: Document,
   user: &User,
@@ -409,7 +396,6 @@ pub type IdResourceMap<T> = HashMap<
   >,
 >;
 
-#[instrument(level = "debug")]
 pub async fn get_id_to_resource_map<T: KomodoResource>(
   id_to_tags: &HashMap<String, Tag>,
   match_tags: &[String],
@@ -491,16 +477,11 @@ pub async fn create<T: KomodoResource>(
 
   // Ensure an existing resource with same name doesn't already exist
   // The database indexing also ensures this but doesn't give a good error message.
-  if list_full_for_user::<T>(
-    Default::default(),
-    system_user(),
-    PermissionLevel::Read.into(),
-    &[],
-  )
-  .await
-  .context("Failed to list all resources for duplicate name check")?
-  .into_iter()
-  .any(|r| r.name == name)
+  if list_full_for_user::<T>(Default::default(), system_user(), &[])
+    .await
+    .context("Failed to list all resources for duplicate name check")?
+    .into_iter()
+    .any(|r| r.name == name)
   {
     return Err(
       anyhow!("Resource with name '{}' already exists", name)
@@ -828,11 +809,11 @@ pub async fn rename<T: KomodoResource>(
 
 pub async fn delete<T: KomodoResource>(
   id_or_name: &str,
-  args: &WriteArgs,
+  user: &User,
 ) -> anyhow::Result<Resource<T::Config, T::Info>> {
   let resource = get_check_permissions::<T>(
     id_or_name,
-    &args.user,
+    user,
     PermissionLevel::Write.into(),
   )
   .await?;
@@ -846,15 +827,13 @@ pub async fn delete<T: KomodoResource>(
     targets: vec![target.clone()],
     ..Default::default()
   }
-  .resolve(&ReadArgs {
-    user: args.user.clone(),
-  })
+  .resolve(&ReadArgs { user: user.clone() })
   .await
   .map_err(|e| e.error)?
   .toml;
 
   let mut update =
-    make_update(target.clone(), T::delete_operation(), &args.user);
+    make_update(target.clone(), T::delete_operation(), user);
 
   T::pre_delete(&resource, &mut update).await?;
 
@@ -913,7 +892,6 @@ async fn delete_from_alerters<T: KomodoResource>(id: &str) {
 
 // =======
 
-#[instrument(level = "debug")]
 pub fn validate_resource_query_tags<T: Default + std::fmt::Debug>(
   query: &mut ResourceQuery<T>,
   all_tags: &[Tag],
@@ -934,7 +912,7 @@ pub fn validate_resource_query_tags<T: Default + std::fmt::Debug>(
   Ok(())
 }
 
-#[instrument]
+#[instrument("DeleteAllPermissionsOnResource")]
 pub async fn delete_all_permissions_on_resource<T>(target: T)
 where
   T: Into<ResourceTarget> + std::fmt::Debug,
@@ -955,7 +933,7 @@ where
   }
 }
 
-#[instrument]
+#[instrument("RemoveFromRecentlyViewed")]
 pub async fn remove_from_recently_viewed<T>(resource: T)
 where
   T: Into<ResourceTarget> + std::fmt::Debug,

bin/core/src/resource/procedure.rs

@@ -171,7 +171,7 @@ impl super::KomodoResource for Procedure {
   }
 }
 
-#[instrument(skip(user))]
+#[instrument("ValidateProcedureConfig", skip_all)]
 async fn validate_config(
   config: &mut PartialProcedureConfig,
   user: &User,
@@ -781,6 +781,13 @@ async fn validate_config(
             ));
           }
         }
+        Execution::RotateCoreKeys(_params) => {
+          if !user.admin {
+            return Err(anyhow!(
+              "Non admin user cannot trigger rotate core keys"
+            ));
+          }
+        }
         Execution::Sleep(_) => {}
       }
     }

bin/core/src/resource/repo.rs

@@ -231,7 +231,7 @@ pub async fn refresh_repo_state_cache() {
   });
 }
 
-#[instrument(skip(user))]
+#[instrument("ValidateRepoConfig", skip_all)]
 async fn validate_config(
   config: &mut PartialRepoConfig,
   user: &User,

bin/core/src/resource/server.rs

@@ -72,14 +72,14 @@ impl super::KomodoResource for Server {
       version,
       public_key,
       terminals_disabled,
-      container_exec_disabled,
+      container_terminals_disabled,
     ) = match status.as_ref().and_then(|s| s.periphery_info.as_ref())
     {
       Some(info) => (
         Some(info.version.clone()),
         Some(info.public_key.clone()),
         info.terminals_disabled,
-        info.container_exec_disabled,
+        info.container_terminals_disabled,
       ),
       None => (None, None, true, true),
     };
@@ -111,7 +111,7 @@ impl super::KomodoResource for Server {
           server.info.attempted_public_key,
         ),
         terminals_disabled,
-        container_exec_disabled,
+        container_terminals_disabled,
       },
     }
   }
@@ -245,6 +245,14 @@ impl super::KomodoResource for Server {
       )
       .await;
 
+    let _ = db_client()
+      .onboarding_keys
+      .update_many(
+        doc! { "copy_server": &id },
+        doc! { "$set": { "copy_server": "" } },
+      )
+      .await;
+
     Ok(())
   }
 
@@ -281,7 +289,7 @@ pub async fn update_server_public_key(
 pub async fn rotate_server_keys(
   server: &Server,
 ) -> anyhow::Result<()> {
-  let periphery = periphery_client(&server).await?;
+  let periphery = periphery_client(server).await?;
   let public_key = periphery
     .request(api::keys::RotatePrivateKey {})
     .await