alert responsive, better Server terminal disabled

Cargo.toml

@@ -8,7 +8,7 @@ members = [
 ]
 
 [workspace.package]
-version = "2.0.0-dev-50"
+version = "2.0.0-dev-72"
 edition = "2024"
 authors = ["mbecker20 <becker.maxh@gmail.com>"]
 license = "GPL-3.0-or-later"
@@ -41,7 +41,7 @@ git = { path = "lib/git" }
 # MOGH
 run_command = { version = "0.0.6", features = ["async_tokio"] }
 serror = { version = "0.5.3", default-features = false }
-slack = { version = "1.1.0", package = "slack_client_rs", default-features = false, features = ["rustls"] }
+slack = { version = "2.0.0", package = "slack_client_rs", default-features = false, features = ["rustls"] }
 derive_default_builder = "0.1.8"
 derive_empty_traits = "0.1.0"
 async_timing_util = "1.1.0"
@@ -54,12 +54,11 @@ mungos = "3.2.2"
 svi = "1.2.0"
 
 # ASYNC
-reqwest = { version = "0.12.23", default-features = false, features = ["json", "stream", "rustls-tls-native-roots"] }
-tokio = { version = "1.47.1", features = ["full"] }
+reqwest = { version = "0.12.24", default-features = false, features = ["json", "stream", "rustls-tls-native-roots"] }
+tokio = { version = "1.48.0", features = ["full"] }
 tokio-util = { version = "0.7.16", features = ["io", "codec"] }
 tokio-stream = { version = "0.1.17", features = ["sync"] }
 pin-project-lite = "0.2.16"
-futures = "0.3.31"
 futures-util = "0.3.31"
 arc-swap = "1.7.1"
 
@@ -72,7 +71,7 @@ axum = { version = "0.8.6", features = ["ws", "json", "macros"] }
 
 # SER/DE
 ipnetwork = { version = "0.21.1", features = ["serde"] }
-indexmap = { version = "2.11.4", features = ["serde"] }
+indexmap = { version = "2.12.0", features = ["serde"] }
 serde = { version = "1.0.227", features = ["derive"] }
 strum = { version = "0.27.2", features = ["derive"] }
 bson = { version = "2.15.0" } # must keep in sync with mongodb version
@@ -96,14 +95,14 @@ opentelemetry = "0.31.0"
 tracing = "0.1.41"
 
 # CONFIG
-clap = { version = "4.5.48", features = ["derive"] }
+clap = { version = "4.5.49", features = ["derive"] }
 dotenvy = "0.15.7"
 envy = "0.4.2"
 
 # CRYPTO / AUTH
 uuid = { version = "1.18.1", features = ["v4", "fast-rng", "serde"] }
 jsonwebtoken = { version = "10.0.0", features = ["aws_lc_rs"] } # locked back with octorust
-rustls = { version = "0.23.32", features = ["aws-lc-rs"] }
+rustls = { version = "0.23.33", features = ["aws-lc-rs"] }
 pem-rfc7468 = { version = "0.7.0", features = ["alloc"] }
 openidconnect = "4.0.1"
 urlencoding = "2.1.3"
@@ -120,13 +119,16 @@ spki = "0.7.3"
 der = "0.7.10"
 
 # SYSTEM
+hickory-resolver = "0.25.2"
 portable-pty = "0.9.0"
+shell-escape = "0.1.5"
+crossterm = "0.29.0"
 bollard = "0.19.3"
 sysinfo = "0.37.1"
 
 # CLOUD
 aws-config = "1.8.8"
-aws-sdk-ec2 = "1.172.0"
+aws-sdk-ec2 = "1.175.1"
 aws-credential-types = "1.2.8"
 
 ## CRON
@@ -138,11 +140,10 @@ croner = "3.0.0"
 # MISC
 async-compression = { version = "0.4.32", features = ["tokio", "gzip"] }
 derive_builder = "0.20.2"
-shell-escape = "0.1.5"
 comfy-table = "7.2.1"
 typeshare = "1.0.4"
 dashmap = "6.1.0"
 wildcard = "0.3.0"
 colored = "3.0.0"
-regex = "1.11.3"
+regex = "1.12.2"
 bytes = "1.10.1"

action/deploy-fe.ts

@@ -0,0 +1,4 @@
+const cmd = "km run -y action deploy-komodo-fe-change";
+new Deno.Command("bash", {
+  args: ["-c", cmd],
+}).spawn();

bin/cli/Cargo.toml

@@ -21,9 +21,11 @@ config.workspace = true
 logger.workspace = true
 noise.workspace = true
 # external
+tokio-tungstenite.workspace = true
 futures-util.workspace = true
 comfy-table.workspace = true
 serde_json.workspace = true
+crossterm.workspace = true
 serde_qs.workspace = true
 wildcard.workspace = true
 tracing.workspace = true
@@ -31,6 +33,7 @@ colored.workspace = true
 dotenvy.workspace = true
 anyhow.workspace = true
 chrono.workspace = true
+bytes.workspace = true
 tokio.workspace = true
 serde.workspace = true
 clap.workspace = true

bin/cli/src/command/list.rs

@@ -5,9 +5,10 @@ use futures_util::{FutureExt, try_join};
 use komodo_client::{
   KomodoClient,
   api::read::{
-    ListActions, ListAlerters, ListBuilders, ListBuilds,
-    ListDeployments, ListProcedures, ListRepos, ListResourceSyncs,
-    ListSchedules, ListServers, ListStacks, ListTags,
+    ListActions, ListAlerters, ListAllTerminals, ListBuilders,
+    ListBuilds, ListDeployments, ListProcedures, ListRepos,
+    ListResourceSyncs, ListSchedules, ListServers, ListStacks,
+    ListTags,
   },
   entities::{
     ResourceTargetVariant,
@@ -26,10 +27,15 @@ use komodo_client::{
       ProcedureListItem, ProcedureListItemInfo, ProcedureState,
     },
     repo::{RepoListItem, RepoListItemInfo, RepoState},
-    resource::{ResourceListItem, ResourceQuery},
+    resource::{
+      ResourceListItem, ResourceQuery, TemplatesQueryBehavior,
+    },
     resource_link,
     schedule::Schedule,
-    server::{ServerListItem, ServerListItemInfo, ServerState},
+    server::{
+      ServerListItem, ServerListItemInfo, ServerState,
+      TerminalInfoWithServer,
+    },
     stack::{StackListItem, StackListItemInfo, StackState},
     sync::{
       ResourceSyncListItem, ResourceSyncListItemInfo,
@@ -74,15 +80,18 @@ pub async fn handle(list: &args::list::List) -> anyhow::Result<()> {
     Some(ListCommand::Syncs(filters)) => {
       list_resources::<ResourceSyncListItem>(filters, false).await
     }
+    Some(ListCommand::Terminals(filters)) => {
+      list_terminals(filters).await
+    }
+    Some(ListCommand::Schedules(filters)) => {
+      list_schedules(filters).await
+    }
     Some(ListCommand::Builders(filters)) => {
       list_resources::<BuilderListItem>(filters, false).await
     }
     Some(ListCommand::Alerters(filters)) => {
       list_resources::<AlerterListItem>(filters, false).await
     }
-    Some(ListCommand::Schedules(filters)) => {
-      list_schedules(filters).await
-    }
   }
 }
 
@@ -189,6 +198,40 @@ where
   Ok(())
 }
 
+async fn list_terminals(
+  filters: &ResourceFilters,
+) -> anyhow::Result<()> {
+  let client = crate::command::komodo_client().await?;
+  let query = ResourceQuery::builder()
+    .tags(filters.tags.clone())
+    .templates(TemplatesQueryBehavior::Exclude)
+    .build();
+  let (mut terminals, servers) = tokio::try_join!(
+    client.read(ListAllTerminals {
+      query: query.clone(),
+      fresh: true,
+    }),
+    client
+      .read(ListServers { query })
+      .map(|res| res.map(|res| res
+        .into_iter()
+        .map(|t| (t.id, t.name))
+        .collect::<HashMap<_, _>>()))
+  )?;
+  // Fix server ids -> names
+  terminals.iter_mut().for_each(|terminal| {
+    let Some(name) = servers.get(&terminal.server_id) else {
+      terminal.server_id = String::new();
+      return;
+    };
+    terminal.server_id.clone_from(name);
+  });
+  if !terminals.is_empty() {
+    print_items(terminals, filters.format, filters.links)?;
+  }
+  Ok(())
+}
+
 async fn list_schedules(
   filters: &ResourceFilters,
 ) -> anyhow::Result<()> {
@@ -1134,6 +1177,28 @@ impl PrintTable for ResourceListItem<AlerterListItemInfo> {
   }
 }
 
+impl PrintTable for TerminalInfoWithServer {
+  fn header(_links: bool) -> &'static [&'static str] {
+    &["Terminal", "Server", "Command", "Size", "Created"]
+  }
+  fn row(self, _links: bool) -> Vec<comfy_table::Cell> {
+    vec![
+      Cell::new(self.name).add_attribute(Attribute::Bold),
+      Cell::new(self.server_id),
+      Cell::new(self.command),
+      Cell::new(if self.stored_size_kb < 1.0 {
+        format!("{:.1} KiB", self.stored_size_kb)
+      } else {
+        format!("{:.} KiB", self.stored_size_kb)
+      }),
+      Cell::new(
+        format_timetamp(self.created_at)
+          .unwrap_or_else(|_| String::from("Invalid created at")),
+      ),
+    ]
+  }
+}
+
 impl PrintTable for Schedule {
   fn header(links: bool) -> &'static [&'static str] {
     if links {
@@ -1146,7 +1211,7 @@ impl PrintTable for Schedule {
     let next_run = if let Some(ts) = self.next_scheduled_run {
       Cell::new(
         format_timetamp(ts)
-          .unwrap_or(String::from("Invalid next ts")),
+          .unwrap_or_else(|_| String::from("Invalid next ts")),
       )
       .add_attribute(Attribute::Bold)
     } else {

bin/cli/src/command/mod.rs

@@ -18,6 +18,7 @@ pub mod container;
 pub mod database;
 pub mod execute;
 pub mod list;
+pub mod ssh;
 pub mod update;
 
 async fn komodo_client() -> anyhow::Result<&'static KomodoClient> {

bin/cli/src/command/ssh.rs

@@ -0,0 +1,193 @@
+use anyhow::Context;
+use bytes::Bytes;
+use colored::Colorize;
+use futures_util::{SinkExt, StreamExt};
+use komodo_client::{
+  api::write::{CreateTerminal, TerminalRecreateMode},
+  entities::config::cli::args::ssh::Ssh,
+};
+use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _};
+use tokio_tungstenite::tungstenite;
+
+pub async fn handle(
+  Ssh {
+    server,
+    name,
+    command,
+    recreate,
+  }: &Ssh,
+) -> anyhow::Result<()> {
+  // Need to forward multiple sources into ws write
+  let (write_tx, mut write_rx) =
+    tokio::sync::mpsc::channel::<Bytes>(1024);
+
+  // ================
+  //  SETUP RESIZING
+  // ================
+
+  // Subscribe to SIGWINCH for resize messages
+  let mut sigwinch = tokio::signal::unix::signal(
+    tokio::signal::unix::SignalKind::window_change(),
+  )
+  .context("failed to register SIGWINCH handler")?;
+
+  // Send first resize messsage, bailing if it fails to get the size.
+  write_tx.send(resize_message()?).await?;
+
+  let forward_resize = async {
+    while sigwinch.recv().await.is_some() {
+      if let Ok(resize_message) = resize_message()
+        && write_tx.send(resize_message).await.is_err()
+      {
+        break;
+      }
+    }
+  };
+
+  let forward_stdin = async {
+    let mut stdin = tokio::io::stdin();
+    let mut buf = [0u8; 8192];
+    loop {
+      // Read into buffer starting from index 1,
+      // leaving first byte to represent 'data' message.
+      let n = match stdin.read(&mut buf[1..]).await {
+        Ok(0) => break, // EOF
+        Ok(n) => n,
+        Err(_) => break,
+      };
+      // Check for disconnect sequence (alt + q)
+      if buf[1..(n + 1)] == [197, 147] {
+        break;
+      }
+      let bytes = Bytes::copy_from_slice(&buf[..(n + 1)]);
+      if write_tx.send(bytes).await.is_err() {
+        break;
+      };
+    }
+  };
+
+  // =====================
+  //  CONNECT AND FORWARD
+  // =====================
+
+  let client = super::komodo_client().await?;
+
+  // Init the terminal if it doesn't exist already.
+  client
+    .write(CreateTerminal {
+      server: server.to_string(),
+      name: name.to_string(),
+      command: command.clone(),
+      recreate: if *recreate {
+        TerminalRecreateMode::Always
+      } else {
+        TerminalRecreateMode::DifferentCommand
+      },
+    })
+    .await?;
+
+  let (mut ws_write, mut ws_read) = client
+    .connect_terminal_websocket(server, name)
+    .await?
+    .split();
+
+  let forward_write = async {
+    while let Some(bytes) = write_rx.recv().await {
+      if let Err(e) =
+        ws_write.send(tungstenite::Message::Binary(bytes)).await
+      {
+        return Some(e);
+      };
+    }
+    None
+  };
+
+  let forward_read = async {
+    let mut stdout = tokio::io::stdout();
+    loop {
+      match ws_read.next().await {
+        Some(Ok(tungstenite::Message::Binary(bytes))) => {
+          if let Err(e) =
+            tokio::io::copy(&mut bytes.as_ref(), &mut stdout)
+              .await
+              .context("Failed to copy bytes to stdout")
+          {
+            return Some(e);
+          }
+          let _ = stdout.flush().await;
+        }
+        Some(Ok(tungstenite::Message::Text(text))) => {
+          if let Err(e) =
+            tokio::io::copy(&mut text.as_ref(), &mut stdout)
+              .await
+              .context("Failed to copy text to stdout")
+          {
+            return Some(e);
+          }
+          let _ = stdout.flush().await;
+        }
+        Some(Ok(tungstenite::Message::Close(_))) => break,
+        Some(Err(e)) => {
+          return Some(
+            anyhow::Error::from(e).context("Websocket read error"),
+          );
+        }
+        None => break,
+        _ => {}
+      }
+    }
+    None
+  };
+
+  let guard = RawModeGuard::enable_raw_mode()?;
+
+  tokio::select! {
+    _ = forward_resize => drop(guard),
+    _ = forward_stdin => drop(guard),
+    e = forward_write => {
+      drop(guard);
+      if let Some(e) = e {
+        eprintln!("\nFailed to forward stdin | {e:#}");
+      }
+    },
+    e = forward_read => {
+      drop(guard);
+      if let Some(e) = e {
+        eprintln!("\nFailed to forward stdout | {e:#}");
+      }
+    },
+  };
+
+  println!("\n\n{} {}", "connection".bold(), "closed".red().bold());
+
+  // It doesn't seem to exit by itself after the raw mode stuff.
+  std::process::exit(0)
+}
+
+fn resize_message() -> anyhow::Result<Bytes> {
+  let (cols, rows) = crossterm::terminal::size()
+    .context("Failed to get terminal size")?;
+  let bytes: Vec<u8> =
+    format!(r#"{{"rows":{rows},"cols":{cols}}}"#).into();
+  let mut msg = Vec::with_capacity(bytes.len() + 1);
+  msg.push(0xff); // resize prefix
+  msg.extend(bytes);
+  Ok(msg.into())
+}
+
+struct RawModeGuard;
+
+impl RawModeGuard {
+  fn enable_raw_mode() -> anyhow::Result<Self> {
+    crossterm::terminal::enable_raw_mode()
+      .context("Failed to enable terminal raw mode")?;
+    Ok(Self)
+  }
+}
+impl Drop for RawModeGuard {
+  fn drop(&mut self) {
+    if let Err(e) = crossterm::terminal::disable_raw_mode() {
+      eprintln!("Failed to disable terminal raw mode | {e:?}");
+    }
+  }
+}

bin/cli/src/config.rs

@@ -28,7 +28,7 @@ pub fn cli_env() -> &'static Env {
     {
       Ok(env) => env,
       Err(e) => {
-        panic!("{e:?}");
+        panic!("{e:?}")
       }
     }
   })
@@ -261,12 +261,18 @@ pub fn cli_config() -> &'static CliConfig {
           .komodo_cli_logging_pretty
           .unwrap_or(config.cli_logging.pretty),
         location: false,
+        ansi: env
+          .komodo_cli_logging_ansi
+          .unwrap_or(config.cli_logging.ansi),
         otlp_endpoint: env
           .komodo_cli_logging_otlp_endpoint
           .unwrap_or(config.cli_logging.otlp_endpoint),
         opentelemetry_service_name: env
           .komodo_cli_logging_opentelemetry_service_name
           .unwrap_or(config.cli_logging.opentelemetry_service_name),
+        opentelemetry_scope_name: env
+          .komodo_cli_logging_opentelemetry_scope_name
+          .unwrap_or(config.cli_logging.opentelemetry_scope_name),
       },
       profile: config.profile,
     }

bin/cli/src/main.rs

@@ -41,12 +41,6 @@ async fn app() -> anyhow::Result<()> {
       }
       Ok(())
     }
-    args::Command::Key { command } => {
-      noise::key::command::handle(command).await
-    }
-    args::Command::Database { command } => {
-      command::database::handle(command).await
-    }
     args::Command::Container(container) => {
       command::container::handle(container).await
     }
@@ -60,6 +54,13 @@ async fn app() -> anyhow::Result<()> {
     args::Command::Update { command } => {
       command::update::handle(command).await
     }
+    args::Command::Ssh(ssh) => command::ssh::handle(ssh).await,
+    args::Command::Key { command } => {
+      noise::key::command::handle(command).await
+    }
+    args::Command::Database { command } => {
+      command::database::handle(command).await
+    }
   }
 }
 

bin/core/Cargo.toml

@@ -45,6 +45,7 @@ aws-credential-types.workspace = true
 english-to-cron.workspace = true
 openidconnect.workspace = true
 jsonwebtoken.workspace = true
+futures-util.workspace = true
 axum-server.workspace = true
 urlencoding.workspace = true
 aws-sdk-ec2.workspace = true
@@ -63,7 +64,6 @@ colored.workspace = true
 dashmap.workspace = true
 tracing.workspace = true
 reqwest.workspace = true
-futures.workspace = true
 dotenvy.workspace = true
 anyhow.workspace = true
 croner.workspace = true
@@ -74,12 +74,12 @@ rustls.workspace = true
 bytes.workspace = true
 tokio.workspace = true
 serde.workspace = true
+strum.workspace = true
 regex.workspace = true
 axum.workspace = true
 toml.workspace = true
 uuid.workspace = true
 envy.workspace = true
-rand.workspace = true
 hmac.workspace = true
 sha2.workspace = true
 hex.workspace = true

bin/core/aio.Dockerfile

@@ -48,6 +48,9 @@ RUN mkdir /action-cache && \
   cd /action-cache && \
   deno install jsr:@std/yaml jsr:@std/toml
 
+COPY ./bin/entrypoint.sh /usr/local/bin/entrypoint.sh
+RUN chmod +x /usr/local/bin/entrypoint.sh
+
 # Hint at the port
 EXPOSE 9120
 
@@ -55,7 +58,7 @@ ENV KOMODO_CLI_CONFIG_PATHS="/config"
 # This ensures any `komodo.cli.*` takes precedence over the Core `/config/*config.*`
 ENV KOMODO_CLI_CONFIG_KEYWORDS="*config.*,*komodo.cli*.*"
 
-CMD [ "core" ]
+CMD [ "/bin/bash", "-c", "update-ca-certificates && core" ]
 
 # Label to prevent Komodo from stopping with StopAllContainers
 LABEL komodo.skip="true"

bin/core/multi-arch.Dockerfile

@@ -28,7 +28,7 @@ COPY --from=x86_64 /core /app/core/linux/amd64
 COPY --from=aarch64 /core /app/core/linux/arm64
 RUN mv /app/core/${TARGETPLATFORM} /usr/local/bin/core && rm -r /app/core
 
-# Same for util
+# Same for km
 COPY --from=x86_64 /km /app/km/linux/amd64
 COPY --from=aarch64 /km /app/km/linux/arm64
 RUN mv /app/km/${TARGETPLATFORM} /usr/local/bin/km && rm -r /app/km
@@ -44,6 +44,9 @@ RUN mkdir /action-cache && \
   cd /action-cache && \
   deno install jsr:@std/yaml jsr:@std/toml
 
+COPY ./bin/entrypoint.sh /usr/local/bin/entrypoint.sh
+RUN chmod +x /usr/local/bin/entrypoint.sh
+
 # Hint at the port
 EXPOSE 9120
 
@@ -51,6 +54,7 @@ ENV KOMODO_CLI_CONFIG_PATHS="/config"
 # This ensures any `komodo.cli.*` takes precedence over the Core `/config/*config.*`
 ENV KOMODO_CLI_CONFIG_KEYWORDS="*config.*,*komodo.cli*.*"
 
+ENTRYPOINT [ "entrypoint.sh" ]
 CMD [ "core" ]
 
 # Label to prevent Komodo from stopping with StopAllContainers

bin/core/single-arch.Dockerfile

@@ -33,6 +33,9 @@ RUN mkdir /action-cache && \
 	cd /action-cache && \
 	deno install jsr:@std/yaml jsr:@std/toml
 
+COPY ./bin/entrypoint.sh /usr/local/bin/entrypoint.sh
+RUN chmod +x /usr/local/bin/entrypoint.sh
+
 # Hint at the port
 EXPOSE 9120
 
@@ -40,6 +43,7 @@ ENV KOMODO_CLI_CONFIG_PATHS="/config"
 # This ensures any `komodo.cli.*` takes precedence over the Core `/config/*config.*`
 ENV KOMODO_CLI_CONFIG_KEYWORDS="*config.*,*komodo.cli*.*"
 
+ENTRYPOINT [ "entrypoint.sh" ]
 CMD [ "core" ]
 
 # Label to prevent Komodo from stopping with StopAllContainers

bin/core/src/alert/discord.rs

@@ -4,7 +4,6 @@ use serde::Serialize;
 
 use super::*;
 
-#[instrument(level = "debug")]
 pub async fn send_alert(
   url: &str,
   alert: &Alert,
@@ -50,7 +49,7 @@ pub async fn send_alert(
       match alert.level {
         SeverityLevel::Ok => {
           format!(
-            "{level} | **{name}**{region} is now **reachable**\n{link}"
+            "{level} | **{name}**{region} is now **connected**\n{link}"
           )
         }
         SeverityLevel::Critical => {
@@ -241,31 +240,33 @@ pub async fn send_alert(
     }
     AlertData::None {} => Default::default(),
   };
-  if !content.is_empty() {
-    let VariablesAndSecrets { variables, secrets } =
-      get_variables_and_secrets().await?;
-    let mut url_interpolated = url.to_string();
 
-    let mut interpolator =
-      Interpolator::new(Some(&variables), &secrets);
-
-    interpolator.interpolate_string(&mut url_interpolated)?;
-
-    send_message(&url_interpolated, &content)
-      .await
-      .map_err(|e| {
-        let replacers = interpolator
-          .secret_replacers
-          .into_iter()
-          .collect::<Vec<_>>();
-        let sanitized_error =
-          svi::replace_in_string(&format!("{e:?}"), &replacers);
-        anyhow::Error::msg(format!(
-          "Error with slack request: {sanitized_error}"
-        ))
-      })?;
+  if content.is_empty() {
+    return Ok(());
   }
-  Ok(())
+
+  let VariablesAndSecrets { variables, secrets } =
+    get_variables_and_secrets().await?;
+  let mut url_interpolated = url.to_string();
+
+  let mut interpolator =
+    Interpolator::new(Some(&variables), &secrets);
+
+  interpolator.interpolate_string(&mut url_interpolated)?;
+
+  send_message(&url_interpolated, &content)
+    .await
+    .map_err(|e| {
+      let replacers = interpolator
+        .secret_replacers
+        .into_iter()
+        .collect::<Vec<_>>();
+      let sanitized_error =
+        svi::replace_in_string(&format!("{e:?}"), &replacers);
+      anyhow::Error::msg(format!(
+        "Error with slack request: {sanitized_error}"
+      ))
+    })
 }
 
 async fn send_message(

bin/core/src/alert/mod.rs

@@ -1,7 +1,7 @@
 use anyhow::{Context, anyhow};
 use database::mungos::{find::find_collect, mongodb::bson::doc};
 use derive_variants::ExtractVariant;
-use futures::future::join_all;
+use futures_util::future::join_all;
 use interpolate::Interpolator;
 use komodo_client::entities::{
   ResourceTargetVariant,
@@ -11,7 +11,6 @@ use komodo_client::entities::{
   komodo_timestamp,
   stack::StackState,
 };
-use tracing::Instrument;
 
 use crate::helpers::query::get_variables_and_secrets;
 use crate::helpers::{
@@ -24,40 +23,32 @@ mod ntfy;
 mod pushover;
 mod slack;
 
-#[instrument(level = "debug")]
 pub async fn send_alerts(alerts: &[Alert]) {
   if alerts.is_empty() {
     return;
   }
 
-  let span =
-    info_span!("send_alerts", alerts = format!("{alerts:?}"));
-  async {
-    let Ok(alerters) = find_collect(
-      &db_client().alerters,
-      doc! { "config.enabled": true },
-      None,
-    )
-    .await
-    .inspect_err(|e| {
-      error!(
+  let Ok(alerters) = find_collect(
+    &db_client().alerters,
+    doc! { "config.enabled": true },
+    None,
+  )
+  .await
+  .inspect_err(|e| {
+    error!(
       "ERROR sending alerts | failed to get alerters from db | {e:#}"
     )
-    }) else {
-      return;
-    };
+  }) else {
+    return;
+  };
 
-    let handles = alerts
-      .iter()
-      .map(|alert| send_alert_to_alerters(&alerters, alert));
+  let handles = alerts
+    .iter()
+    .map(|alert| send_alert_to_alerters(&alerters, alert));
 
-    join_all(handles).await;
-  }
-  .instrument(span)
-  .await
+  join_all(handles).await;
 }
 
-#[instrument(level = "debug")]
 async fn send_alert_to_alerters(alerters: &[Alerter], alert: &Alert) {
   if alerters.is_empty() {
     return;
@@ -161,7 +152,6 @@ pub async fn send_alert_to_alerter(
   }
 }
 
-#[instrument(level = "debug")]
 async fn send_custom_alert(
   url: &str,
   alert: &Alert,
@@ -294,7 +284,7 @@ fn standard_alert_content(alert: &Alert) -> String {
       let link = resource_link(ResourceTargetVariant::Server, id);
       match alert.level {
         SeverityLevel::Ok => {
-          format!("{level} | {name}{region} is now reachable\n{link}")
+          format!("{level} | {name}{region} is now connected\n{link}")
         }
         SeverityLevel::Critical => {
           let err = err

bin/core/src/alert/ntfy.rs

@@ -2,17 +2,38 @@ use std::sync::OnceLock;
 
 use super::*;
 
-#[instrument(level = "debug")]
 pub async fn send_alert(
   url: &str,
   email: Option<&str>,
   alert: &Alert,
 ) -> anyhow::Result<()> {
   let content = standard_alert_content(alert);
-  if !content.is_empty() {
-    send_message(url, email, content).await?;
+  if content.is_empty() {
+    return Ok(());
   }
-  Ok(())
+
+  let VariablesAndSecrets { variables, secrets } =
+    get_variables_and_secrets().await?;
+  let mut url_interpolated = url.to_string();
+
+  let mut interpolator =
+    Interpolator::new(Some(&variables), &secrets);
+
+  interpolator.interpolate_string(&mut url_interpolated)?;
+
+  send_message(&url_interpolated, email, content)
+    .await
+    .map_err(|e| {
+      let replacers = interpolator
+        .secret_replacers
+        .into_iter()
+        .collect::<Vec<_>>();
+      let sanitized_error =
+        svi::replace_in_string(&format!("{e:?}"), &replacers);
+      anyhow::Error::msg(format!(
+        "Error with slack request: {sanitized_error}"
+      ))
+    })
 }
 
 async fn send_message(
@@ -22,7 +43,7 @@ async fn send_message(
 ) -> anyhow::Result<()> {
   let mut request = http_client()
     .post(url)
-    .header("Title", "ntfy Alert")
+    .header("Title", "Komodo Alert")
     .body(content);
 
   if let Some(email) = email {
@@ -43,9 +64,7 @@ async fn send_message(
       )
     })?;
     Err(anyhow!(
-      "Failed to send message to ntfy | {} | {}",
-      status,
-      text
+      "Failed to send message to ntfy | {status} | {text}",
     ))
   }
 }

bin/core/src/alert/pushover.rs

@@ -2,16 +2,35 @@ use std::sync::OnceLock;
 
 use super::*;
 
-#[instrument(level = "debug")]
 pub async fn send_alert(
   url: &str,
   alert: &Alert,
 ) -> anyhow::Result<()> {
   let content = standard_alert_content(alert);
-  if !content.is_empty() {
-    send_message(url, content).await?;
+  if content.is_empty() {
+    return Ok(());
   }
-  Ok(())
+
+  let VariablesAndSecrets { variables, secrets } =
+    get_variables_and_secrets().await?;
+  let mut url_interpolated = url.to_string();
+
+  let mut interpolator =
+    Interpolator::new(Some(&variables), &secrets);
+
+  interpolator.interpolate_string(&mut url_interpolated)?;
+
+  send_message(&url_interpolated, content).await.map_err(|e| {
+    let replacers = interpolator
+      .secret_replacers
+      .into_iter()
+      .collect::<Vec<_>>();
+    let sanitized_error =
+      svi::replace_in_string(&format!("{e:?}"), &replacers);
+    anyhow::Error::msg(format!(
+      "Error with slack request: {sanitized_error}"
+    ))
+  })
 }
 
 async fn send_message(

bin/core/src/alert/slack.rs

@@ -2,7 +2,6 @@ use ::slack::types::OwnedBlock as Block;
 
 use super::*;
 
-#[instrument(level = "debug")]
 pub async fn send_alert(
   url: &str,
   alert: &Alert,
@@ -64,11 +63,11 @@ pub async fn send_alert(
       match alert.level {
         SeverityLevel::Ok => {
           let text =
-            format!("{level} | *{name}*{region} is now *reachable*");
+            format!("{level} | *{name}*{region} is now *connected*");
           let blocks = vec![
             Block::header(level),
             Block::section(format!(
-              "*{name}*{region} is now *reachable*"
+              "*{name}*{region} is now *connnected*"
             )),
           ];
           (text, blocks.into())
@@ -468,31 +467,32 @@ pub async fn send_alert(
     }
     AlertData::None {} => Default::default(),
   };
-  if !text.is_empty() {
-    let VariablesAndSecrets { variables, secrets } =
-      get_variables_and_secrets().await?;
-    let mut url_interpolated = url.to_string();
-
-    let mut interpolator =
-      Interpolator::new(Some(&variables), &secrets);
-
-    interpolator.interpolate_string(&mut url_interpolated)?;
-
-    let slack = ::slack::Client::new(url_interpolated);
-    slack
-      .send_owned_message_single(&text, blocks.as_deref())
-      .await
-      .map_err(|e| {
-        let replacers = interpolator
-          .secret_replacers
-          .into_iter()
-          .collect::<Vec<_>>();
-        let sanitized_error =
-          svi::replace_in_string(&format!("{e:?}"), &replacers);
-        anyhow::Error::msg(format!(
-          "Error with slack request: {sanitized_error}"
-        ))
-      })?;
+  if text.is_empty() {
+    return Ok(());
   }
+  let VariablesAndSecrets { variables, secrets } =
+    get_variables_and_secrets().await?;
+  let mut url_interpolated = url.to_string();
+
+  let mut interpolator =
+    Interpolator::new(Some(&variables), &secrets);
+
+  interpolator.interpolate_string(&mut url_interpolated)?;
+
+  let slack = ::slack::Client::new(url_interpolated);
+  slack
+    .send_owned_message_single(&text, None, blocks.as_deref())
+    .await
+    .map_err(|e| {
+      let replacers = interpolator
+        .secret_replacers
+        .into_iter()
+        .collect::<Vec<_>>();
+      let sanitized_error =
+        svi::replace_in_string(&format!("{e:?}"), &replacers);
+      anyhow::Error::msg(format!(
+        "Error with slack request: {sanitized_error}"
+      ))
+    })?;
   Ok(())
 }

bin/core/src/api/auth.rs

@@ -88,7 +88,6 @@ async fn variant_handler(
   handler(headers, Json(req)).await
 }
 
-#[instrument(name = "AuthHandler", level = "debug", skip(headers))]
 async fn handler(
   headers: HeaderMap,
   Json(request): Json<AuthRequest>,
@@ -125,7 +124,6 @@ fn login_options_reponse() -> &'static GetLoginOptionsResponse {
 }
 
 impl Resolve<AuthArgs> for GetLoginOptions {
-  #[instrument(name = "GetLoginOptions", level = "debug", skip(self))]
   async fn resolve(
     self,
     _: &AuthArgs,
@@ -135,7 +133,6 @@ impl Resolve<AuthArgs> for GetLoginOptions {
 }
 
 impl Resolve<AuthArgs> for ExchangeForJwt {
-  #[instrument(name = "ExchangeForJwt", level = "debug", skip(self))]
   async fn resolve(
     self,
     _: &AuthArgs,
@@ -148,7 +145,6 @@ impl Resolve<AuthArgs> for ExchangeForJwt {
 }
 
 impl Resolve<AuthArgs> for GetUser {
-  #[instrument(name = "GetUser", level = "debug", skip(self))]
   async fn resolve(
     self,
     AuthArgs { headers }: &AuthArgs,

bin/core/src/api/execute/action.rs

@@ -1,7 +1,6 @@
 use std::{
   collections::HashSet,
   path::{Path, PathBuf},
-  str::FromStr,
   sync::OnceLock,
 };
 
@@ -24,6 +23,7 @@ use komodo_client::{
     config::core::CoreConfig,
     komodo_timestamp,
     permission::PermissionLevel,
+    random_string,
     update::Update,
     user::action_user,
   },
@@ -38,7 +38,6 @@ use crate::{
   config::core_config,
   helpers::{
     query::{VariablesAndSecrets, get_variables_and_secrets},
-    random_string,
     update::update_update,
   },
   permission::get_check_permissions,
@@ -59,10 +58,18 @@ impl super::BatchExecute for BatchRunAction {
 }
 
 impl Resolve<ExecuteArgs> for BatchRunAction {
-  #[instrument(name = "BatchRunAction", skip(self, user), fields(user_id = user.id))]
+  #[instrument(
+    "BatchRunAction",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      pattern = self.pattern,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, .. }: &ExecuteArgs,
+    ExecuteArgs { user, id, .. }: &ExecuteArgs,
   ) -> serror::Result<BatchExecutionResponse> {
     Ok(
       super::batch_execute::<BatchRunAction>(&self.pattern, user)
@@ -72,10 +79,19 @@ impl Resolve<ExecuteArgs> for BatchRunAction {
 }
 
 impl Resolve<ExecuteArgs> for RunAction {
-  #[instrument(name = "RunAction", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+  #[instrument(
+    "RunAction",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      action = self.action,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
     let mut action = get_check_permissions::<Action>(
       &self.action,
@@ -213,7 +229,6 @@ impl Resolve<ExecuteArgs> for RunAction {
     update_update(update.clone()).await?;
 
     if !update.success && action.config.failure_alert {
-      warn!("action unsuccessful, alerting...");
       let target = update.target.clone();
       tokio::spawn(async move {
         let alert = Alert {
@@ -236,6 +251,7 @@ impl Resolve<ExecuteArgs> for RunAction {
   }
 }
 
+#[instrument("Interpolate", skip(contents, update, secret))]
 async fn interpolate(
   contents: &mut String,
   update: &mut Update,
@@ -321,6 +337,7 @@ main()
 /// Cleans up file at given path.
 /// ALSO if $DENO_DIR is set,
 /// will clean up the generated file matching "file"
+#[instrument("CleanupRun")]
 async fn cleanup_run(file: String, path: &Path) {
   if let Err(e) = fs::remove_file(path).await {
     warn!(
@@ -340,7 +357,7 @@ fn deno_dir() -> Option<&'static Path> {
   DENO_DIR
     .get_or_init(|| {
       let deno_dir = std::env::var("DENO_DIR").ok()?;
-      PathBuf::from_str(&deno_dir).ok()
+      Some(PathBuf::from(&deno_dir))
     })
     .as_deref()
 }

bin/core/src/api/execute/alerter.rs

@@ -1,6 +1,8 @@
 use anyhow::{Context, anyhow};
 use formatting::format_serror;
-use futures::{TryStreamExt, stream::FuturesUnordered};
+use futures_util::{
+  StreamExt, TryStreamExt, stream::FuturesUnordered,
+};
 use komodo_client::{
   api::execute::{SendAlert, TestAlerter},
   entities::{
@@ -22,10 +24,19 @@ use crate::{
 use super::ExecuteArgs;
 
 impl Resolve<ExecuteArgs> for TestAlerter {
-  #[instrument(name = "TestAlerter", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+  #[instrument(
+    "TestAlerter",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      alerter = self.alerter,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> Result<Self::Response, Self::Error> {
     let alerter = get_check_permissions::<Alerter>(
       &self.alerter,
@@ -79,28 +90,58 @@ impl Resolve<ExecuteArgs> for TestAlerter {
 //
 
 impl Resolve<ExecuteArgs> for SendAlert {
-  #[instrument(name = "SendAlert", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+  #[instrument(
+    "SendAlert",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      request = format!("{self:?}"),
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> Result<Self::Response, Self::Error> {
-    let alerters = list_full_for_user::<Alerter>(
-      Default::default(),
-      user,
-      PermissionLevel::Execute.into(),
-      &[],
-    )
-    .await?
-    .into_iter()
-    .filter(|a| {
-      a.config.enabled
-        && (self.alerters.is_empty()
-          || self.alerters.contains(&a.name)
-          || self.alerters.contains(&a.id))
-        && (a.config.alert_types.is_empty()
-          || a.config.alert_types.contains(&AlertDataVariant::Custom))
-    })
-    .collect::<Vec<_>>();
+    let alerters =
+      list_full_for_user::<Alerter>(Default::default(), user, &[])
+        .await?
+        .into_iter()
+        .filter(|a| {
+          a.config.enabled
+            && (self.alerters.is_empty()
+              || self.alerters.contains(&a.name)
+              || self.alerters.contains(&a.id))
+            && (a.config.alert_types.is_empty()
+              || a
+                .config
+                .alert_types
+                .contains(&AlertDataVariant::Custom))
+        })
+        .collect::<Vec<_>>();
+
+    let alerters = if user.admin {
+      alerters
+    } else {
+      // Only keep alerters with execute permissions
+      alerters
+        .into_iter()
+        .map(|alerter| async move {
+          get_check_permissions::<Alerter>(
+            &alerter.id,
+            user,
+            PermissionLevel::Execute.into(),
+          )
+          .await
+        })
+        .collect::<FuturesUnordered<_>>()
+        .collect::<Vec<_>>()
+        .await
+        .into_iter()
+        .flatten()
+        .collect()
+    };
 
     if alerters.is_empty() {
       return Err(anyhow!(

bin/core/src/api/execute/build.rs

@@ -14,7 +14,7 @@ use database::mungos::{
   },
 };
 use formatting::format_serror;
-use futures::future::join_all;
+use futures_util::future::join_all;
 use interpolate::Interpolator;
 use komodo_client::{
   api::execute::{
@@ -37,12 +37,13 @@ use komodo_client::{
 use periphery_client::api;
 use resolver_api::Resolve;
 use tokio_util::sync::CancellationToken;
+use uuid::Uuid;
 
 use crate::{
   alert::send_alerts,
   helpers::{
     build_git_token,
-    builder::{cleanup_builder_instance, get_builder_periphery},
+    builder::{cleanup_builder_instance, connect_builder_periphery},
     channel::build_cancel_channel,
     query::{
       VariablesAndSecrets, get_deployment_state,
@@ -66,10 +67,18 @@ impl super::BatchExecute for BatchRunBuild {
 }
 
 impl Resolve<ExecuteArgs> for BatchRunBuild {
-  #[instrument(name = "BatchRunBuild", skip(user), fields(user_id = user.id))]
+  #[instrument(
+    "BatchRunBuild",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      pattern = self.pattern,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, .. }: &ExecuteArgs,
+    ExecuteArgs { user, id, .. }: &ExecuteArgs,
   ) -> serror::Result<BatchExecutionResponse> {
     Ok(
       super::batch_execute::<BatchRunBuild>(&self.pattern, user)
@@ -79,10 +88,19 @@ impl Resolve<ExecuteArgs> for BatchRunBuild {
 }
 
 impl Resolve<ExecuteArgs> for RunBuild {
-  #[instrument(name = "RunBuild", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+  #[instrument(
+    "RunBuild",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      build = self.build,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
     let mut build = get_check_permissions::<Build>(
       &self.build,
@@ -186,7 +204,7 @@ impl Resolve<ExecuteArgs> for RunBuild {
     });
 
     // GET BUILDER PERIPHERY
-    let (periphery, cleanup_data) = match get_builder_periphery(
+    let (periphery, cleanup_data) = match connect_builder_periphery(
       build.name.clone(),
       Some(build.config.version),
       builder,
@@ -247,11 +265,11 @@ impl Resolve<ExecuteArgs> for RunBuild {
             replacers: Default::default(),
           }) => res,
         _ = cancel.cancelled() => {
-          debug!("build cancelled during clone, cleaning up builder");
-          update.push_error_log("build cancelled", String::from("user cancelled build during repo clone"));
+          debug!("Build cancelled during clone, cleaning up builder");
+          update.push_error_log("Build cancelled", String::from("user cancelled build during repo clone"));
           cleanup_builder_instance(periphery, cleanup_data, &mut update)
             .await;
-          info!("builder cleaned up");
+          info!("Builder cleaned up");
           return handle_early_return(update, build.id, build.name, true).await
         },
       };
@@ -296,8 +314,8 @@ impl Resolve<ExecuteArgs> for RunBuild {
             additional_tags: Default::default(),
           }) => res.context("failed at call to periphery to build"),
         _ = cancel.cancelled() => {
-          info!("build cancelled during build, cleaning up builder");
-          update.push_error_log("build cancelled", String::from("user cancelled build during docker build"));
+          info!("Build cancelled during build, cleaning up builder");
+          update.push_error_log("Build cancelled", String::from("user cancelled build during docker build"));
           cleanup_builder_instance(periphery, cleanup_data, &mut update)
             .await;
           return handle_early_return(update, build.id, build.name, true).await
@@ -312,7 +330,7 @@ impl Resolve<ExecuteArgs> for RunBuild {
         Err(e) => {
           warn!("error in build | {e:#}");
           update.push_error_log(
-            "build",
+            "Build Error",
             format_serror(&e.context("failed to build").into()),
           )
         }
@@ -370,7 +388,6 @@ impl Resolve<ExecuteArgs> for RunBuild {
         handle_post_build_redeploy(&build.id).await;
       });
     } else {
-      warn!("build unsuccessful, alerting...");
       let target = update.target.clone();
       let version = update.version;
       tokio::spawn(async move {
@@ -395,7 +412,7 @@ impl Resolve<ExecuteArgs> for RunBuild {
   }
 }
 
-#[instrument(skip(update))]
+#[instrument("HandleEarlyReturn", skip(update))]
 async fn handle_early_return(
   mut update: Update,
   build_id: String,
@@ -419,7 +436,6 @@ async fn handle_early_return(
   }
   update_update(update.clone()).await?;
   if !update.success && !is_cancel {
-    warn!("build unsuccessful, alerting...");
     let target = update.target.clone();
     let version = update.version;
     tokio::spawn(async move {
@@ -489,10 +505,19 @@ pub async fn validate_cancel_build(
 }
 
 impl Resolve<ExecuteArgs> for CancelBuild {
-  #[instrument(name = "CancelBuild", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+  #[instrument(
+    "CancelBuild",
+    skip(user, update),
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      build = self.build,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
     let build = get_check_permissions::<Build>(
       &self.build,
@@ -549,7 +574,7 @@ impl Resolve<ExecuteArgs> for CancelBuild {
   }
 }
 
-#[instrument]
+#[instrument("PostBuildRedeploy")]
 async fn handle_post_build_redeploy(build_id: &str) {
   let Ok(redeploy_deployments) = find_collect(
     &db_client().deployments,
@@ -585,7 +610,11 @@ async fn handle_post_build_redeploy(build_id: &str) {
               stop_signal: None,
               stop_time: None,
             }
-            .resolve(&ExecuteArgs { user, update })
+            .resolve(&ExecuteArgs {
+              user,
+              update,
+              id: Uuid::new_v4(),
+            })
             .await
           }
           .await;
@@ -611,6 +640,7 @@ async fn handle_post_build_redeploy(build_id: &str) {
 /// This will make sure that a build with non-none image registry has an account attached,
 /// and will check the core config for a token matching requirements.
 /// Otherwise it is left to periphery.
+#[instrument("ValidateRegistryTokens")]
 async fn validate_account_extract_registry_tokens(
   Build {
     config: BuildConfig { image_registry, .. },

bin/core/src/api/execute/deployment.rs

@@ -49,10 +49,18 @@ impl super::BatchExecute for BatchDeploy {
 }
 
 impl Resolve<ExecuteArgs> for BatchDeploy {
-  #[instrument(name = "BatchDeploy", skip(user), fields(user_id = user.id))]
+  #[instrument(
+    "BatchDeploy",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      pattern = self.pattern,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, .. }: &ExecuteArgs,
+    ExecuteArgs { user, id, .. }: &ExecuteArgs,
   ) -> serror::Result<BatchExecutionResponse> {
     Ok(
       super::batch_execute::<BatchDeploy>(&self.pattern, user)
@@ -61,6 +69,7 @@ impl Resolve<ExecuteArgs> for BatchDeploy {
   }
 }
 
+#[instrument("SetupDeploy", skip_all)]
 async fn setup_deployment_execution(
   deployment: &str,
   user: &User,
@@ -87,10 +96,21 @@ async fn setup_deployment_execution(
 }
 
 impl Resolve<ExecuteArgs> for Deploy {
-  #[instrument(name = "Deploy", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+  #[instrument(
+    "Deploy",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      deployment = self.deployment,
+      stop_signal = format!("{:?}", self.stop_signal),
+      stop_time = self.stop_time,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
     let (mut deployment, server) =
       setup_deployment_execution(&self.deployment, user).await?;
@@ -243,6 +263,14 @@ fn pull_cache() -> &'static PullCache {
   PULL_CACHE.get_or_init(Default::default)
 }
 
+#[instrument(
+  "PullDeploymentInner",
+  skip_all,
+  fields(
+    deployment = deployment.id,
+    server = server.id
+  )
+)]
 pub async fn pull_deployment_inner(
   deployment: Deployment,
   server: &Server,
@@ -358,10 +386,19 @@ pub async fn pull_deployment_inner(
 }
 
 impl Resolve<ExecuteArgs> for PullDeployment {
-  #[instrument(name = "PullDeployment", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+  #[instrument(
+    "PullDeployment",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      deployment = self.deployment,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
     let (deployment, server) =
       setup_deployment_execution(&self.deployment, user).await?;
@@ -392,10 +429,19 @@ impl Resolve<ExecuteArgs> for PullDeployment {
 }
 
 impl Resolve<ExecuteArgs> for StartDeployment {
-  #[instrument(name = "StartDeployment", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+  #[instrument(
+    "StartDeployment",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      deployment = self.deployment,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
     let (deployment, server) =
       setup_deployment_execution(&self.deployment, user).await?;
@@ -440,10 +486,19 @@ impl Resolve<ExecuteArgs> for StartDeployment {
 }
 
 impl Resolve<ExecuteArgs> for RestartDeployment {
-  #[instrument(name = "RestartDeployment", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+  #[instrument(
+    "RestartDeployment",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      deployment = self.deployment,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
     let (deployment, server) =
       setup_deployment_execution(&self.deployment, user).await?;
@@ -490,10 +545,19 @@ impl Resolve<ExecuteArgs> for RestartDeployment {
 }
 
 impl Resolve<ExecuteArgs> for PauseDeployment {
-  #[instrument(name = "PauseDeployment", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+  #[instrument(
+    "PauseDeployment",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      deployment = self.deployment,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
     let (deployment, server) =
       setup_deployment_execution(&self.deployment, user).await?;
@@ -538,10 +602,19 @@ impl Resolve<ExecuteArgs> for PauseDeployment {
 }
 
 impl Resolve<ExecuteArgs> for UnpauseDeployment {
-  #[instrument(name = "UnpauseDeployment", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+  #[instrument(
+    "UnpauseDeployment",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      deployment = self.deployment,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
     let (deployment, server) =
       setup_deployment_execution(&self.deployment, user).await?;
@@ -588,10 +661,21 @@ impl Resolve<ExecuteArgs> for UnpauseDeployment {
 }
 
 impl Resolve<ExecuteArgs> for StopDeployment {
-  #[instrument(name = "StopDeployment", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+  #[instrument(
+    "StopDeployment",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      deployment = self.deployment,
+      signal = format!("{:?}", self.signal),
+      time = self.time,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
     let (deployment, server) =
       setup_deployment_execution(&self.deployment, user).await?;
@@ -655,10 +739,18 @@ impl super::BatchExecute for BatchDestroyDeployment {
 }
 
 impl Resolve<ExecuteArgs> for BatchDestroyDeployment {
-  #[instrument(name = "BatchDestroyDeployment", skip(user), fields(user_id = user.id))]
+  #[instrument(
+    "BatchDestroyDeployment",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      pattern = self.pattern,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, .. }: &ExecuteArgs,
+    ExecuteArgs { user, id, .. }: &ExecuteArgs,
   ) -> serror::Result<BatchExecutionResponse> {
     Ok(
       super::batch_execute::<BatchDestroyDeployment>(
@@ -671,10 +763,21 @@ impl Resolve<ExecuteArgs> for BatchDestroyDeployment {
 }
 
 impl Resolve<ExecuteArgs> for DestroyDeployment {
-  #[instrument(name = "DestroyDeployment", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+  #[instrument(
+    "DestroyDeployment",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      deployment = self.deployment,
+      signal = format!("{:?}", self.signal),
+      time = self.time,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
     let (deployment, server) =
       setup_deployment_execution(&self.deployment, user).await?;

bin/core/src/api/execute/maintenance.rs

@@ -7,7 +7,7 @@ use database::{
   mungos::find::find_collect,
 };
 use formatting::{bold, format_serror};
-use futures::{StreamExt, stream::FuturesOrdered};
+use futures_util::{StreamExt, stream::FuturesOrdered};
 use komodo_client::{
   api::execute::{
     BackupCoreDatabase, ClearRepoCache, GlobalAutoUpdate,
@@ -45,13 +45,17 @@ fn clear_repo_cache_lock() -> &'static Mutex<()> {
 
 impl Resolve<ExecuteArgs> for ClearRepoCache {
   #[instrument(
-    name = "ClearRepoCache",
-    skip(user, update),
-    fields(user_id = user.id, update_id = update.id)
+    "ClearRepoCache",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id
+    )
   )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> Result<Self::Response, Self::Error> {
     if !user.admin {
       return Err(
@@ -120,13 +124,17 @@ fn backup_database_lock() -> &'static Mutex<()> {
 
 impl Resolve<ExecuteArgs> for BackupCoreDatabase {
   #[instrument(
-    name = "BackupCoreDatabase",
-    skip(user, update),
-    fields(user_id = user.id, update_id = update.id)
+    "BackupCoreDatabase",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+    )
   )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> Result<Self::Response, Self::Error> {
     if !user.admin {
       return Err(
@@ -169,13 +177,17 @@ fn global_update_lock() -> &'static Mutex<()> {
 
 impl Resolve<ExecuteArgs> for GlobalAutoUpdate {
   #[instrument(
-    name = "GlobalAutoUpdate",
-    skip(user, update),
-    fields(user_id = user.id, update_id = update.id)
+    "GlobalAutoUpdate",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+    )
   )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> Result<Self::Response, Self::Error> {
     if !user.admin {
       return Err(
@@ -335,13 +347,17 @@ fn global_rotate_lock() -> &'static Mutex<()> {
 
 impl Resolve<ExecuteArgs> for RotateAllServerKeys {
   #[instrument(
-    name = "RotateAllServerKeys",
-    skip(user, update),
-    fields(user_id = user.id, update_id = update.id)
+    "RotateAllServerKeys",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+    )
   )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> Result<Self::Response, Self::Error> {
     if !user.admin {
       return Err(
@@ -445,13 +461,18 @@ impl Resolve<ExecuteArgs> for RotateAllServerKeys {
 
 impl Resolve<ExecuteArgs> for RotateCoreKeys {
   #[instrument(
-    name = "RotateCoreKeys",
-    skip(user, update),
-    fields(user_id = user.id, update_id = update.id)
+    "RotateCoreKeys",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      force = self.force,
+    )
   )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> Result<Self::Response, Self::Error> {
     if !user.admin {
       return Err(

bin/core/src/api/execute/mod.rs

@@ -1,4 +1,4 @@
-use std::{pin::Pin, time::Instant};
+use std::pin::Pin;
 
 use anyhow::Context;
 use axum::{
@@ -8,7 +8,9 @@ use axum_extra::{TypedHeader, headers::ContentType};
 use database::mungos::by_id::find_one_by_id;
 use derive_variants::{EnumVariants, ExtractVariant};
 use formatting::format_serror;
-use futures::future::join_all;
+use futures_util::{
+  StreamExt as _, future::join_all, stream::FuturesUnordered,
+};
 use komodo_client::{
   api::execute::*,
   entities::{
@@ -23,12 +25,14 @@ use response::JsonString;
 use serde::{Deserialize, Serialize};
 use serde_json::json;
 use serror::Json;
+use strum::Display;
 use typeshare::typeshare;
 use uuid::Uuid;
 
 use crate::{
   auth::auth_request,
   helpers::update::{init_execution_update, update_update},
+  permission::get_check_permissions,
   resource::{KomodoResource, list_full_for_user_using_pattern},
   state::db_client,
 };
@@ -51,6 +55,9 @@ pub use {
 };
 
 pub struct ExecuteArgs {
+  /// The execution id.
+  /// Unique for every /execute call.
+  pub id: Uuid,
   pub user: User,
   pub update: Update,
 }
@@ -59,7 +66,7 @@ pub struct ExecuteArgs {
 #[derive(
   Serialize, Deserialize, Debug, Clone, Resolve, EnumVariants,
 )]
-#[variant_derive(Debug)]
+#[variant_derive(Debug, Display)]
 #[args(ExecuteArgs)]
 #[response(JsonString)]
 #[error(serror::Error)]
@@ -203,7 +210,7 @@ pub fn inner_handler(
   >,
 > {
   Box::pin(async move {
-    let req_id = Uuid::new_v4();
+    let task_id = Uuid::new_v4();
 
     // Need to validate no cancel is active before any update is created.
     // This ensures no double update created if Cancel is called more than once for the same request.
@@ -219,14 +226,14 @@ pub fn inner_handler(
     // here either.
     if update.operation == Operation::None {
       return Ok(ExecutionResult::Batch(
-        task(req_id, request, user, update).await?,
+        task(task_id, request, user, update).await?,
       ));
     }
 
     // Spawn a task for the execution which continues
     // running after this method returns.
     let handle =
-      tokio::spawn(task(req_id, request, user, update.clone()));
+      tokio::spawn(task(task_id, request, user, update.clone()));
 
     // Spawns another task to monitor the first for failures,
     // and add the log to Update about it (which primary task can't do because it errored out)
@@ -235,11 +242,11 @@ pub fn inner_handler(
       async move {
         let log = match handle.await {
           Ok(Err(e)) => {
-            warn!("/execute request {req_id} task error: {e:#}",);
+            warn!("/execute request {task_id} task error: {e:#}",);
             Log::error("Task Error", format_serror(&e.into()))
           }
           Err(e) => {
-            warn!("/execute request {req_id} spawn error: {e:?}",);
+            warn!("/execute request {task_id} spawn error: {e:?}",);
             Log::error("Spawn Error", format!("{e:#?}"))
           }
           _ => return,
@@ -273,40 +280,33 @@ pub fn inner_handler(
   })
 }
 
-#[instrument(
-  name = "ExecuteRequest",
-  skip(user, update),
-  fields(
-    user_id = user.id,
-    update_id = update.id,
-    request = format!("{:?}", request.extract_variant()))
-  )
-]
 async fn task(
-  req_id: Uuid,
+  id: Uuid,
   request: ExecuteRequest,
   user: User,
   update: Update,
 ) -> anyhow::Result<String> {
-  info!("/execute request {req_id} | user: {}", user.username);
-  let timer = Instant::now();
+  let variant = request.extract_variant();
 
-  let res = match request.resolve(&ExecuteArgs { user, update }).await
-  {
-    Err(e) => Err(e.error),
-    Ok(JsonString::Err(e)) => Err(
-      anyhow::Error::from(e).context("failed to serialize response"),
-    ),
-    Ok(JsonString::Ok(res)) => Ok(res),
-  };
+  info!(
+    "/execute request {id} | {variant} | user: {}",
+    user.username
+  );
+
+  let res =
+    match request.resolve(&ExecuteArgs { user, update, id }).await {
+      Err(e) => Err(e.error),
+      Ok(JsonString::Err(e)) => Err(
+        anyhow::Error::from(e)
+          .context("failed to serialize response"),
+      ),
+      Ok(JsonString::Ok(res)) => Ok(res),
+    };
 
   if let Err(e) = &res {
-    warn!("/execute request {req_id} error: {e:#}");
+    warn!("/execute request {id} error: {e:#}");
   }
 
-  let elapsed = timer.elapsed();
-  debug!("/execute request {req_id} | resolve time: {elapsed:?}");
-
   res
 }
 
@@ -315,6 +315,7 @@ trait BatchExecute {
   fn single_request(name: String) -> ExecuteRequest;
 }
 
+#[instrument("BatchExecute", skip(user))]
 async fn batch_execute<E: BatchExecute>(
   pattern: &str,
   user: &User,
@@ -323,10 +324,32 @@ async fn batch_execute<E: BatchExecute>(
     pattern,
     Default::default(),
     user,
-    PermissionLevel::Execute.into(),
     &[],
   )
   .await?;
+
+  let resources = if user.admin {
+    resources
+  } else {
+    // Only keep resources with execute permissions
+    resources
+      .into_iter()
+      .map(|resource| async move {
+        get_check_permissions::<E::Resource>(
+          &resource.id,
+          user,
+          PermissionLevel::Execute.into(),
+        )
+        .await
+      })
+      .collect::<FuturesUnordered<_>>()
+      .collect::<Vec<_>>()
+      .await
+      .into_iter()
+      .flatten()
+      .collect()
+  };
+
   let futures = resources.into_iter().map(|resource| {
     let user = user.clone();
     async move {

bin/core/src/api/execute/procedure.rs

@@ -38,7 +38,11 @@ impl super::BatchExecute for BatchRunProcedure {
 }
 
 impl Resolve<ExecuteArgs> for BatchRunProcedure {
-  #[instrument(name = "BatchRunProcedure", skip(user), fields(user_id = user.id))]
+  #[instrument(
+    "BatchRunProcedure",
+    skip_all,
+    fields(operator = user.id)
+  )]
   async fn resolve(
     self,
     ExecuteArgs { user, .. }: &ExecuteArgs,
@@ -51,10 +55,19 @@ impl Resolve<ExecuteArgs> for BatchRunProcedure {
 }
 
 impl Resolve<ExecuteArgs> for RunProcedure {
-  #[instrument(name = "RunProcedure", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+  #[instrument(
+    "RunProcedure",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      procedure = self.procedure,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
     Ok(
       resolve_inner(self.procedure, user.clone(), update.clone())
@@ -146,7 +159,6 @@ fn resolve_inner(
     update_update(update.clone()).await?;
 
     if !update.success && procedure.config.failure_alert {
-      warn!("procedure unsuccessful, alerting...");
       let target = update.target.clone();
       tokio::spawn(async move {
         let alert = Alert {

bin/core/src/api/execute/repo.rs

@@ -30,7 +30,7 @@ use crate::{
   alert::send_alerts,
   api::write::WriteArgs,
   helpers::{
-    builder::{cleanup_builder_instance, get_builder_periphery},
+    builder::{cleanup_builder_instance, connect_builder_periphery},
     channel::repo_cancel_channel,
     git_token, periphery_client,
     query::{VariablesAndSecrets, get_variables_and_secrets},
@@ -51,10 +51,18 @@ impl super::BatchExecute for BatchCloneRepo {
 }
 
 impl Resolve<ExecuteArgs> for BatchCloneRepo {
-  #[instrument(name = "BatchCloneRepo", skip( user), fields(user_id = user.id))]
+  #[instrument(
+    "BatchCloneRepo",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      pattern = self.pattern,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, id, .. }: &ExecuteArgs,
   ) -> serror::Result<BatchExecutionResponse> {
     Ok(
       super::batch_execute::<BatchCloneRepo>(&self.pattern, user)
@@ -64,10 +72,19 @@ impl Resolve<ExecuteArgs> for BatchCloneRepo {
 }
 
 impl Resolve<ExecuteArgs> for CloneRepo {
-  #[instrument(name = "CloneRepo", skip( user, update), fields(user_id = user.id, update_id = update.id))]
+  #[instrument(
+    "CloneRepo",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      repo = self.repo,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
     let mut repo = get_check_permissions::<Repo>(
       &self.repo,
@@ -165,10 +182,18 @@ impl super::BatchExecute for BatchPullRepo {
 }
 
 impl Resolve<ExecuteArgs> for BatchPullRepo {
-  #[instrument(name = "BatchPullRepo", skip(user), fields(user_id = user.id))]
+  #[instrument(
+    "BatchPullRepo",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      pattern = self.pattern
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, .. }: &ExecuteArgs,
+    ExecuteArgs { user, id, .. }: &ExecuteArgs,
   ) -> serror::Result<BatchExecutionResponse> {
     Ok(
       super::batch_execute::<BatchPullRepo>(&self.pattern, user)
@@ -178,10 +203,19 @@ impl Resolve<ExecuteArgs> for BatchPullRepo {
 }
 
 impl Resolve<ExecuteArgs> for PullRepo {
-  #[instrument(name = "PullRepo", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+  #[instrument(
+    "PullRepo",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      repo = self.repo,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
     let mut repo = get_check_permissions::<Repo>(
       &self.repo,
@@ -275,7 +309,11 @@ impl Resolve<ExecuteArgs> for PullRepo {
   }
 }
 
-#[instrument(skip_all, fields(update_id = update.id))]
+#[instrument(
+  "HandleRepoEarlyReturn",
+  skip_all,
+  fields(update_id = update.id)
+)]
 async fn handle_repo_update_return(
   update: Update,
 ) -> serror::Result<Update> {
@@ -297,7 +335,7 @@ async fn handle_repo_update_return(
   Ok(update)
 }
 
-#[instrument]
+#[instrument("UpdateLastPulledTime")]
 async fn update_last_pulled_time(repo_name: &str) {
   let res = db_client()
     .repos
@@ -321,10 +359,18 @@ impl super::BatchExecute for BatchBuildRepo {
 }
 
 impl Resolve<ExecuteArgs> for BatchBuildRepo {
-  #[instrument(name = "BatchBuildRepo", skip(user), fields(user_id = user.id))]
+  #[instrument(
+    "BatchBuildRepo",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      pattern = self.pattern,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, .. }: &ExecuteArgs,
+    ExecuteArgs { user, id, .. }: &ExecuteArgs,
   ) -> serror::Result<BatchExecutionResponse> {
     Ok(
       super::batch_execute::<BatchBuildRepo>(&self.pattern, user)
@@ -334,10 +380,19 @@ impl Resolve<ExecuteArgs> for BatchBuildRepo {
 }
 
 impl Resolve<ExecuteArgs> for BuildRepo {
-  #[instrument(name = "BuildRepo", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+  #[instrument(
+    "BuildRepo",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      repo = self.repo,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
     let mut repo = get_check_permissions::<Repo>(
       &self.repo,
@@ -419,7 +474,7 @@ impl Resolve<ExecuteArgs> for BuildRepo {
 
     // GET BUILDER PERIPHERY
 
-    let (periphery, cleanup_data) = match get_builder_periphery(
+    let (periphery, cleanup_data) = match connect_builder_periphery(
       repo.name.clone(),
       None,
       builder,
@@ -531,7 +586,6 @@ impl Resolve<ExecuteArgs> for BuildRepo {
     update_update(update.clone()).await?;
 
     if !update.success {
-      warn!("repo build unsuccessful, alerting...");
       let target = update.target.clone();
       tokio::spawn(async move {
         let alert = Alert {
@@ -554,7 +608,7 @@ impl Resolve<ExecuteArgs> for BuildRepo {
   }
 }
 
-#[instrument(skip(update))]
+#[instrument("HandleRepoBuildEarlyReturn", skip(update))]
 async fn handle_builder_early_return(
   mut update: Update,
   repo_id: String,
@@ -578,7 +632,6 @@ async fn handle_builder_early_return(
   }
   update_update(update.clone()).await?;
   if !update.success && !is_cancel {
-    warn!("repo build unsuccessful, alerting...");
     let target = update.target.clone();
     tokio::spawn(async move {
       let alert = Alert {
@@ -599,7 +652,6 @@ async fn handle_builder_early_return(
   Ok(update)
 }
 
-#[instrument(skip_all)]
 pub async fn validate_cancel_repo_build(
   request: &ExecuteRequest,
 ) -> anyhow::Result<()> {
@@ -649,10 +701,19 @@ pub async fn validate_cancel_repo_build(
 }
 
 impl Resolve<ExecuteArgs> for CancelRepoBuild {
-  #[instrument(name = "CancelRepoBuild", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+  #[instrument(
+    "CancelRepoBuild",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      repo = self.repo,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
     let repo = get_check_permissions::<Repo>(
       &self.repo,
@@ -709,6 +770,13 @@ impl Resolve<ExecuteArgs> for CancelRepoBuild {
   }
 }
 
+#[instrument(
+  "Interpolate",
+  skip_all,
+  fields(
+    skip_secret_interp = repo.config.skip_secret_interp
+  )
+)]
 async fn interpolate(
   repo: &mut Repo,
   update: &mut Update,

bin/core/src/api/execute/server.rs

@@ -22,10 +22,20 @@ use crate::{
 use super::ExecuteArgs;
 
 impl Resolve<ExecuteArgs> for StartContainer {
-  #[instrument(name = "StartContainer", skip(self, user, update), fields(user_id = user.id, update_id = update.id))]
+  #[instrument(
+    "StartContainer",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      server = self.server,
+      container = self.container,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
     let server = get_check_permissions::<Server>(
       &self.server,
@@ -76,10 +86,20 @@ impl Resolve<ExecuteArgs> for StartContainer {
 }
 
 impl Resolve<ExecuteArgs> for RestartContainer {
-  #[instrument(name = "RestartContainer", skip(self, user, update), fields(user_id = user.id, update_id = update.id))]
+  #[instrument(
+    "RestartContainer",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      server = self.server,
+      container = self.container,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
     let server = get_check_permissions::<Server>(
       &self.server,
@@ -132,10 +152,20 @@ impl Resolve<ExecuteArgs> for RestartContainer {
 }
 
 impl Resolve<ExecuteArgs> for PauseContainer {
-  #[instrument(name = "PauseContainer", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+  #[instrument(
+    "PauseContainer",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      server = self.server,
+      container = self.container,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
     let server = get_check_permissions::<Server>(
       &self.server,
@@ -186,10 +216,20 @@ impl Resolve<ExecuteArgs> for PauseContainer {
 }
 
 impl Resolve<ExecuteArgs> for UnpauseContainer {
-  #[instrument(name = "UnpauseContainer", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+  #[instrument(
+    "UnpauseContainer",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      server = self.server,
+      container = self.container,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
     let server = get_check_permissions::<Server>(
       &self.server,
@@ -242,10 +282,22 @@ impl Resolve<ExecuteArgs> for UnpauseContainer {
 }
 
 impl Resolve<ExecuteArgs> for StopContainer {
-  #[instrument(name = "StopContainer", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+  #[instrument(
+    "StopContainer",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      server = self.server,
+      container = self.container,
+      signal = format!("{:?}", self.signal),
+      time = self.time,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
     let server = get_check_permissions::<Server>(
       &self.server,
@@ -298,10 +350,22 @@ impl Resolve<ExecuteArgs> for StopContainer {
 }
 
 impl Resolve<ExecuteArgs> for DestroyContainer {
-  #[instrument(name = "DestroyContainer", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+  #[instrument(
+    "DestroyContainer",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      server = self.server,
+      container = self.container,
+      signal = format!("{:?}", self.signal),
+      time = self.time,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
     let DestroyContainer {
       server,
@@ -360,10 +424,19 @@ impl Resolve<ExecuteArgs> for DestroyContainer {
 }
 
 impl Resolve<ExecuteArgs> for StartAllContainers {
-  #[instrument(name = "StartAllContainers", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+  #[instrument(
+    "StartAllContainers",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      server = self.server,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
     let server = get_check_permissions::<Server>(
       &self.server,
@@ -411,10 +484,19 @@ impl Resolve<ExecuteArgs> for StartAllContainers {
 }
 
 impl Resolve<ExecuteArgs> for RestartAllContainers {
-  #[instrument(name = "RestartAllContainers", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+  #[instrument(
+    "RestartAllContainers",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      server = self.server,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
     let server = get_check_permissions::<Server>(
       &self.server,
@@ -464,10 +546,19 @@ impl Resolve<ExecuteArgs> for RestartAllContainers {
 }
 
 impl Resolve<ExecuteArgs> for PauseAllContainers {
-  #[instrument(name = "PauseAllContainers", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+  #[instrument(
+    "PauseAllContainers",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      server = self.server,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
     let server = get_check_permissions::<Server>(
       &self.server,
@@ -515,10 +606,19 @@ impl Resolve<ExecuteArgs> for PauseAllContainers {
 }
 
 impl Resolve<ExecuteArgs> for UnpauseAllContainers {
-  #[instrument(name = "UnpauseAllContainers", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+  #[instrument(
+    "UnpauseAllContainers",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      server = self.server,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
     let server = get_check_permissions::<Server>(
       &self.server,
@@ -568,10 +668,19 @@ impl Resolve<ExecuteArgs> for UnpauseAllContainers {
 }
 
 impl Resolve<ExecuteArgs> for StopAllContainers {
-  #[instrument(name = "StopAllContainers", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+  #[instrument(
+    "StopAllContainers",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      server = self.server,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
     let server = get_check_permissions::<Server>(
       &self.server,
@@ -619,10 +728,19 @@ impl Resolve<ExecuteArgs> for StopAllContainers {
 }
 
 impl Resolve<ExecuteArgs> for PruneContainers {
-  #[instrument(name = "PruneContainers", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+  #[instrument(
+    "PruneContainers",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      server = self.server,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
     let server = get_check_permissions::<Server>(
       &self.server,
@@ -675,10 +793,20 @@ impl Resolve<ExecuteArgs> for PruneContainers {
 }
 
 impl Resolve<ExecuteArgs> for DeleteNetwork {
-  #[instrument(name = "DeleteNetwork", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+  #[instrument(
+    "DeleteNetwork",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      server = self.server,
+      network = self.name
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
     let server = get_check_permissions::<Server>(
       &self.server,
@@ -726,10 +854,19 @@ impl Resolve<ExecuteArgs> for DeleteNetwork {
 }
 
 impl Resolve<ExecuteArgs> for PruneNetworks {
-  #[instrument(name = "PruneNetworks", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+  #[instrument(
+    "PruneNetworks",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      server = self.server,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
     let server = get_check_permissions::<Server>(
       &self.server,
@@ -780,10 +917,20 @@ impl Resolve<ExecuteArgs> for PruneNetworks {
 }
 
 impl Resolve<ExecuteArgs> for DeleteImage {
-  #[instrument(name = "DeleteImage", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+  #[instrument(
+    "DeleteImage",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      server = self.server,
+      image = self.name,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
     let server = get_check_permissions::<Server>(
       &self.server,
@@ -828,10 +975,19 @@ impl Resolve<ExecuteArgs> for DeleteImage {
 }
 
 impl Resolve<ExecuteArgs> for PruneImages {
-  #[instrument(name = "PruneImages", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+  #[instrument(
+    "PruneImages",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      server = self.server,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
     let server = get_check_permissions::<Server>(
       &self.server,
@@ -880,10 +1036,20 @@ impl Resolve<ExecuteArgs> for PruneImages {
 }
 
 impl Resolve<ExecuteArgs> for DeleteVolume {
-  #[instrument(name = "DeleteVolume", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+  #[instrument(
+    "DeleteVolume",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      server = self.server,
+      volume = self.name,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
     let server = get_check_permissions::<Server>(
       &self.server,
@@ -931,10 +1097,19 @@ impl Resolve<ExecuteArgs> for DeleteVolume {
 }
 
 impl Resolve<ExecuteArgs> for PruneVolumes {
-  #[instrument(name = "PruneVolumes", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+  #[instrument(
+    "PruneVolumes",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      server = self.server,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
     let server = get_check_permissions::<Server>(
       &self.server,
@@ -983,10 +1158,19 @@ impl Resolve<ExecuteArgs> for PruneVolumes {
 }
 
 impl Resolve<ExecuteArgs> for PruneDockerBuilders {
-  #[instrument(name = "PruneDockerBuilders", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+  #[instrument(
+    "PruneDockerBuilders",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      server = self.server,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
     let server = get_check_permissions::<Server>(
       &self.server,
@@ -1035,10 +1219,19 @@ impl Resolve<ExecuteArgs> for PruneDockerBuilders {
 }
 
 impl Resolve<ExecuteArgs> for PruneBuildx {
-  #[instrument(name = "PruneBuildx", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+  #[instrument(
+    "PruneBuildx",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      server = self.server,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
     let server = get_check_permissions::<Server>(
       &self.server,
@@ -1087,10 +1280,19 @@ impl Resolve<ExecuteArgs> for PruneBuildx {
 }
 
 impl Resolve<ExecuteArgs> for PruneSystem {
-  #[instrument(name = "PruneSystem", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+  #[instrument(
+    "PruneSystem",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      server = self.server,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
     let server = get_check_permissions::<Server>(
       &self.server,

bin/core/src/api/execute/stack.rs

@@ -22,6 +22,7 @@ use komodo_client::{
 };
 use periphery_client::api::compose::*;
 use resolver_api::Resolve;
+use uuid::Uuid;
 
 use crate::{
   api::write::WriteArgs,
@@ -54,10 +55,18 @@ impl super::BatchExecute for BatchDeployStack {
 }
 
 impl Resolve<ExecuteArgs> for BatchDeployStack {
-  #[instrument(name = "BatchDeployStack", skip(user), fields(user_id = user.id))]
+  #[instrument(
+    "BatchDeployStack",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      pattern = self.pattern,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, .. }: &ExecuteArgs,
+    ExecuteArgs { user, id, .. }: &ExecuteArgs,
   ) -> serror::Result<BatchExecutionResponse> {
     Ok(
       super::batch_execute::<BatchDeployStack>(&self.pattern, user)
@@ -67,10 +76,21 @@ impl Resolve<ExecuteArgs> for BatchDeployStack {
 }
 
 impl Resolve<ExecuteArgs> for DeployStack {
-  #[instrument(name = "DeployStack", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+  #[instrument(
+    "DeployStack",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      stack = self.stack,
+      services = format!("{:?}", self.services),
+      stop_time = self.stop_time,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
     let (mut stack, server) = get_stack_and_server(
       &self.stack,
@@ -281,10 +301,18 @@ impl super::BatchExecute for BatchDeployStackIfChanged {
 }
 
 impl Resolve<ExecuteArgs> for BatchDeployStackIfChanged {
-  #[instrument(name = "BatchDeployStackIfChanged", skip(user), fields(user_id = user.id))]
+  #[instrument(
+    "BatchDeployStackIfChanged",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      pattern = self.pattern,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, .. }: &ExecuteArgs,
+    ExecuteArgs { user, id, .. }: &ExecuteArgs,
   ) -> serror::Result<BatchExecutionResponse> {
     Ok(
       super::batch_execute::<BatchDeployStackIfChanged>(
@@ -297,10 +325,20 @@ impl Resolve<ExecuteArgs> for BatchDeployStackIfChanged {
 }
 
 impl Resolve<ExecuteArgs> for DeployStackIfChanged {
-  #[instrument(name = "DeployStackIfChanged", skip(user, update), fields(user_id = user.id))]
+  #[instrument(
+    "DeployStackIfChanged",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      stack = self.stack,
+      stop_time = self.stop_time,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
     let stack = get_check_permissions::<Stack>(
       &self.stack,
@@ -358,6 +396,7 @@ impl Resolve<ExecuteArgs> for DeployStackIfChanged {
         .resolve(&ExecuteArgs {
           user: user.clone(),
           update,
+          id: *id,
         })
         .await
       }
@@ -467,6 +506,14 @@ impl Resolve<ExecuteArgs> for DeployStackIfChanged {
   }
 }
 
+#[instrument(
+  "DeployStackServices",
+  skip_all,
+  fields(
+    stack = stack,
+    services = format!("{services:?}")
+  )
+)]
 async fn deploy_services(
   stack: String,
   services: Vec<String>,
@@ -488,10 +535,19 @@ async fn deploy_services(
     .resolve(&ExecuteArgs {
       user: user.clone(),
       update,
+      id: Uuid::new_v4(),
     })
     .await
 }
 
+#[instrument(
+  "RestartStackServices",
+  skip_all,
+  fields(
+    stack = stack,
+    services = format!("{services:?}")
+  )
+)]
 async fn restart_services(
   stack: String,
   services: Vec<String>,
@@ -510,6 +566,7 @@ async fn restart_services(
     .resolve(&ExecuteArgs {
       user: user.clone(),
       update,
+      id: Uuid::new_v4(),
     })
     .await
 }
@@ -526,6 +583,11 @@ async fn restart_services(
 /// Changes to config files after restart is applied should
 /// be taken as the deployed contents, otherwise next changed check
 /// will restart service again for no reason.
+#[instrument(
+  "UpdateStackDeployedContents",
+  skip_all,
+  fields(stack = id)
+)]
 async fn update_deployed_contents_with_latest(
   id: &str,
   contents: Option<Vec<StackRemoteFileContents>>,
@@ -663,10 +725,18 @@ impl super::BatchExecute for BatchPullStack {
 }
 
 impl Resolve<ExecuteArgs> for BatchPullStack {
-  #[instrument(name = "BatchPullStack", skip(user), fields(user_id = user.id))]
+  #[instrument(
+    "BatchPullStack",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      pattern = self.pattern,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, .. }: &ExecuteArgs,
+    ExecuteArgs { user, id, .. }: &ExecuteArgs,
   ) -> serror::Result<BatchExecutionResponse> {
     Ok(
       super::batch_execute::<BatchPullStack>(&self.pattern, user)
@@ -700,6 +770,14 @@ async fn maybe_pull_stack(
   Ok(())
 }
 
+#[instrument(
+  "PullStackInner",
+  skip_all,
+  fields(
+    stack = stack.id,
+    services = format!("{services:?}"),
+  )
+)]
 pub async fn pull_stack_inner(
   mut stack: Stack,
   services: Vec<String>,
@@ -769,10 +847,20 @@ pub async fn pull_stack_inner(
 }
 
 impl Resolve<ExecuteArgs> for PullStack {
-  #[instrument(name = "PullStack", skip(user, update), fields(user_id = user.id))]
+  #[instrument(
+    "PullStack",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      stack = self.stack,
+      services = format!("{:?}", self.services),
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
     let (stack, server) = get_stack_and_server(
       &self.stack,
@@ -822,10 +910,20 @@ impl Resolve<ExecuteArgs> for PullStack {
 }
 
 impl Resolve<ExecuteArgs> for StartStack {
-  #[instrument(name = "StartStack", skip(user, update), fields(user_id = user.id))]
+  #[instrument(
+    "StartStack",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      stack = self.stack,
+      services = format!("{:?}", self.services),
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
     execute_compose::<StartStack>(
       &self.stack,
@@ -841,10 +939,20 @@ impl Resolve<ExecuteArgs> for StartStack {
 }
 
 impl Resolve<ExecuteArgs> for RestartStack {
-  #[instrument(name = "RestartStack", skip(user, update), fields(user_id = user.id))]
+  #[instrument(
+    "RestartStack",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      stack = self.stack,
+      services = format!("{:?}", self.services),
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
     execute_compose::<RestartStack>(
       &self.stack,
@@ -862,10 +970,20 @@ impl Resolve<ExecuteArgs> for RestartStack {
 }
 
 impl Resolve<ExecuteArgs> for PauseStack {
-  #[instrument(name = "PauseStack", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+  #[instrument(
+    "PauseStack",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      stack = self.stack,
+      services = format!("{:?}", self.services),
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
     execute_compose::<PauseStack>(
       &self.stack,
@@ -881,10 +999,20 @@ impl Resolve<ExecuteArgs> for PauseStack {
 }
 
 impl Resolve<ExecuteArgs> for UnpauseStack {
-  #[instrument(name = "UnpauseStack", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+  #[instrument(
+    "UnpauseStack",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      stack = self.stack,
+      services = format!("{:?}", self.services),
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
     execute_compose::<UnpauseStack>(
       &self.stack,
@@ -900,10 +1028,20 @@ impl Resolve<ExecuteArgs> for UnpauseStack {
 }
 
 impl Resolve<ExecuteArgs> for StopStack {
-  #[instrument(name = "StopStack", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+  #[instrument(
+    "StopStack",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      stack = self.stack,
+      services = format!("{:?}", self.services),
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
     execute_compose::<StopStack>(
       &self.stack,
@@ -931,10 +1069,18 @@ impl super::BatchExecute for BatchDestroyStack {
 }
 
 impl Resolve<ExecuteArgs> for BatchDestroyStack {
-  #[instrument(name = "BatchDestroyStack", skip(user), fields(user_id = user.id))]
+  #[instrument(
+    "BatchDestroyStack",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      pattern = self.pattern,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, .. }: &ExecuteArgs,
+    ExecuteArgs { user, id, .. }: &ExecuteArgs,
   ) -> serror::Result<BatchExecutionResponse> {
     super::batch_execute::<BatchDestroyStack>(&self.pattern, user)
       .await
@@ -943,10 +1089,22 @@ impl Resolve<ExecuteArgs> for BatchDestroyStack {
 }
 
 impl Resolve<ExecuteArgs> for DestroyStack {
-  #[instrument(name = "DestroyStack", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+  #[instrument(
+    "DestroyStack",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      stack = self.stack,
+      services = format!("{:?}", self.services),
+      remove_orphans = self.remove_orphans,
+      stop_time = self.stop_time,
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
     execute_compose::<DestroyStack>(
       &self.stack,
@@ -962,10 +1120,21 @@ impl Resolve<ExecuteArgs> for DestroyStack {
 }
 
 impl Resolve<ExecuteArgs> for RunStackService {
-  #[instrument(name = "RunStackService", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+  #[instrument(
+    "RunStackService",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      stack = self.stack,
+      service = self.service,
+      request = format!("{self:?}"),
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
     let (mut stack, server) = get_stack_and_server(
       &self.stack,

bin/core/src/api/execute/sync.rs

@@ -49,10 +49,21 @@ use crate::{
 use super::ExecuteArgs;
 
 impl Resolve<ExecuteArgs> for RunSync {
-  #[instrument(name = "RunSync", skip(user, update), fields(user_id = user.id, update_id = update.id))]
+  #[instrument(
+    "RunSync",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      sync = self.sync,
+      resource_type = format!("{:?}", self.resource_type),
+      resources = format!("{:?}", self.resources),
+    )
+  )]
   async fn resolve(
     self,
-    ExecuteArgs { user, update }: &ExecuteArgs,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
     let RunSync {
       sync,

bin/core/src/api/read/action.rs

@@ -46,13 +46,8 @@ impl Resolve<ReadArgs> for ListActions {
       get_all_tags(None).await?
     };
     Ok(
-      resource::list_for_user::<Action>(
-        self.query,
-        user,
-        PermissionLevel::Read.into(),
-        &all_tags,
-      )
-      .await?,
+      resource::list_for_user::<Action>(self.query, user, &all_tags)
+        .await?,
     )
   }
 }
@@ -69,10 +64,7 @@ impl Resolve<ReadArgs> for ListFullActions {
     };
     Ok(
       resource::list_full_for_user::<Action>(
-        self.query,
-        user,
-        PermissionLevel::Read.into(),
-        &all_tags,
+        self.query, user, &all_tags,
       )
       .await?,
     )
@@ -108,7 +100,6 @@ impl Resolve<ReadArgs> for GetActionsSummary {
     let actions = resource::list_full_for_user::<Action>(
       Default::default(),
       user,
-      PermissionLevel::Read.into(),
       &[],
     )
     .await

bin/core/src/api/read/alerter.rs

@@ -44,13 +44,8 @@ impl Resolve<ReadArgs> for ListAlerters {
       get_all_tags(None).await?
     };
     Ok(
-      resource::list_for_user::<Alerter>(
-        self.query,
-        user,
-        PermissionLevel::Read.into(),
-        &all_tags,
-      )
-      .await?,
+      resource::list_for_user::<Alerter>(self.query, user, &all_tags)
+        .await?,
     )
   }
 }
@@ -67,10 +62,7 @@ impl Resolve<ReadArgs> for ListFullAlerters {
     };
     Ok(
       resource::list_full_for_user::<Alerter>(
-        self.query,
-        user,
-        PermissionLevel::Read.into(),
-        &all_tags,
+        self.query, user, &all_tags,
       )
       .await?,
     )

bin/core/src/api/read/build.rs

@@ -6,7 +6,7 @@ use database::mungos::{
   find::find_collect,
   mongodb::{bson::doc, options::FindOptions},
 };
-use futures::TryStreamExt;
+use futures_util::TryStreamExt;
 use komodo_client::{
   api::read::*,
   entities::{
@@ -54,13 +54,8 @@ impl Resolve<ReadArgs> for ListBuilds {
       get_all_tags(None).await?
     };
     Ok(
-      resource::list_for_user::<Build>(
-        self.query,
-        user,
-        PermissionLevel::Read.into(),
-        &all_tags,
-      )
-      .await?,
+      resource::list_for_user::<Build>(self.query, user, &all_tags)
+        .await?,
     )
   }
 }
@@ -77,10 +72,7 @@ impl Resolve<ReadArgs> for ListFullBuilds {
     };
     Ok(
       resource::list_full_for_user::<Build>(
-        self.query,
-        user,
-        PermissionLevel::Read.into(),
-        &all_tags,
+        self.query, user, &all_tags,
       )
       .await?,
     )
@@ -116,7 +108,6 @@ impl Resolve<ReadArgs> for GetBuildsSummary {
     let builds = resource::list_full_for_user::<Build>(
       Default::default(),
       user,
-      PermissionLevel::Read.into(),
       &[],
     )
     .await
@@ -280,10 +271,7 @@ impl Resolve<ReadArgs> for ListCommonBuildExtraArgs {
       get_all_tags(None).await?
     };
     let builds = resource::list_full_for_user::<Build>(
-      self.query,
-      user,
-      PermissionLevel::Read.into(),
-      &all_tags,
+      self.query, user, &all_tags,
     )
     .await
     .context("failed to get resources matching query")?;

bin/core/src/api/read/builder.rs

@@ -44,13 +44,8 @@ impl Resolve<ReadArgs> for ListBuilders {
       get_all_tags(None).await?
     };
     Ok(
-      resource::list_for_user::<Builder>(
-        self.query,
-        user,
-        PermissionLevel::Read.into(),
-        &all_tags,
-      )
-      .await?,
+      resource::list_for_user::<Builder>(self.query, user, &all_tags)
+        .await?,
     )
   }
 }
@@ -67,10 +62,7 @@ impl Resolve<ReadArgs> for ListFullBuilders {
     };
     Ok(
       resource::list_full_for_user::<Builder>(
-        self.query,
-        user,
-        PermissionLevel::Read.into(),
-        &all_tags,
+        self.query, user, &all_tags,
       )
       .await?,
     )

bin/core/src/api/read/deployment.rs

@@ -56,10 +56,7 @@ impl Resolve<ReadArgs> for ListDeployments {
     };
     let only_update_available = self.query.specific.update_available;
     let deployments = resource::list_for_user::<Deployment>(
-      self.query,
-      user,
-      PermissionLevel::Read.into(),
-      &all_tags,
+      self.query, user, &all_tags,
     )
     .await?;
     let deployments = if only_update_available {
@@ -86,10 +83,7 @@ impl Resolve<ReadArgs> for ListFullDeployments {
     };
     Ok(
       resource::list_full_for_user::<Deployment>(
-        self.query,
-        user,
-        PermissionLevel::Read.into(),
-        &all_tags,
+        self.query, user, &all_tags,
       )
       .await?,
     )
@@ -303,7 +297,6 @@ impl Resolve<ReadArgs> for GetDeploymentsSummary {
     let deployments = resource::list_full_for_user::<Deployment>(
       Default::default(),
       user,
-      PermissionLevel::Read.into(),
       &[],
     )
     .await
@@ -349,10 +342,7 @@ impl Resolve<ReadArgs> for ListCommonDeploymentExtraArgs {
       get_all_tags(None).await?
     };
     let deployments = resource::list_full_for_user::<Deployment>(
-      self.query,
-      user,
-      PermissionLevel::Read.into(),
-      &all_tags,
+      self.query, user, &all_tags,
     )
     .await
     .context("failed to get resources matching query")?;

bin/core/src/api/read/mod.rs

@@ -11,7 +11,6 @@ use komodo_client::{
     build::Build,
     builder::{Builder, BuilderConfig},
     config::{DockerRegistry, GitProvider},
-    permission::PermissionLevel,
     repo::Repo,
     server::Server,
     sync::ResourceSync,
@@ -115,6 +114,7 @@ enum ReadRequest {
   ListServers(ListServers),
   ListFullServers(ListFullServers),
   ListTerminals(ListTerminals),
+  ListAllTerminals(ListAllTerminals),
 
   // ==== DOCKER ====
   GetDockerContainersSummary(GetDockerContainersSummary),
@@ -249,7 +249,6 @@ async fn variant_handler(
   handler(user, Json(req)).await
 }
 
-#[instrument(name = "ReadHandler", level = "debug", skip(user), fields(user_id = user.id))]
 async fn handler(
   Extension(user): Extension<User>,
   Json(request): Json<ReadRequest>,
@@ -399,19 +398,16 @@ impl Resolve<ReadArgs> for ListGitProvidersFromConfig {
       resource::list_full_for_user::<Build>(
         Default::default(),
         user,
-        PermissionLevel::Read.into(),
         &[]
       ),
       resource::list_full_for_user::<Repo>(
         Default::default(),
         user,
-        PermissionLevel::Read.into(),
         &[]
       ),
       resource::list_full_for_user::<ResourceSync>(
         Default::default(),
         user,
-        PermissionLevel::Read.into(),
         &[]
       ),
     )?;

bin/core/src/api/read/procedure.rs

@@ -45,10 +45,7 @@ impl Resolve<ReadArgs> for ListProcedures {
     };
     Ok(
       resource::list_for_user::<Procedure>(
-        self.query,
-        user,
-        PermissionLevel::Read.into(),
-        &all_tags,
+        self.query, user, &all_tags,
       )
       .await?,
     )
@@ -67,10 +64,7 @@ impl Resolve<ReadArgs> for ListFullProcedures {
     };
     Ok(
       resource::list_full_for_user::<Procedure>(
-        self.query,
-        user,
-        PermissionLevel::Read.into(),
-        &all_tags,
+        self.query, user, &all_tags,
       )
       .await?,
     )
@@ -85,7 +79,6 @@ impl Resolve<ReadArgs> for GetProceduresSummary {
     let procedures = resource::list_full_for_user::<Procedure>(
       Default::default(),
       user,
-      PermissionLevel::Read.into(),
       &[],
     )
     .await

bin/core/src/api/read/repo.rs

@@ -44,13 +44,8 @@ impl Resolve<ReadArgs> for ListRepos {
       get_all_tags(None).await?
     };
     Ok(
-      resource::list_for_user::<Repo>(
-        self.query,
-        user,
-        PermissionLevel::Read.into(),
-        &all_tags,
-      )
-      .await?,
+      resource::list_for_user::<Repo>(self.query, user, &all_tags)
+        .await?,
     )
   }
 }
@@ -67,10 +62,7 @@ impl Resolve<ReadArgs> for ListFullRepos {
     };
     Ok(
       resource::list_full_for_user::<Repo>(
-        self.query,
-        user,
-        PermissionLevel::Read.into(),
-        &all_tags,
+        self.query, user, &all_tags,
       )
       .await?,
     )
@@ -106,7 +98,6 @@ impl Resolve<ReadArgs> for GetReposSummary {
     let repos = resource::list_full_for_user::<Repo>(
       Default::default(),
       user,
-      PermissionLevel::Read.into(),
       &[],
     )
     .await

bin/core/src/api/read/schedule.rs

@@ -1,10 +1,9 @@
-use futures::future::join_all;
+use futures_util::future::join_all;
 use komodo_client::{
   api::read::*,
   entities::{
     ResourceTarget,
     action::Action,
-    permission::PermissionLevel,
     procedure::Procedure,
     resource::{ResourceQuery, TemplatesQueryBehavior},
     schedule::Schedule,
@@ -36,7 +35,6 @@ impl Resolve<ReadArgs> for ListSchedules {
           specific: Default::default(),
         },
         &args.user,
-        PermissionLevel::Read.into(),
         &all_tags,
       ),
       list_full_for_user::<Procedure>(
@@ -48,7 +46,6 @@ impl Resolve<ReadArgs> for ListSchedules {
           specific: Default::default(),
         },
         &args.user,
-        PermissionLevel::Read.into(),
         &all_tags,
       )
     )?;

bin/core/src/api/read/server.rs

@@ -12,6 +12,7 @@ use database::mungos::{
   find::find_collect,
   mongodb::{bson::doc, options::FindOptions},
 };
+use futures_util::{StreamExt as _, stream::FuturesUnordered};
 use komodo_client::{
   api::read::*,
   entities::{
@@ -29,7 +30,7 @@ use komodo_client::{
     permission::PermissionLevel,
     server::{
       Server, ServerActionState, ServerListItem, ServerState,
-      TerminalInfo,
+      TerminalInfo, TerminalInfoWithServer,
     },
     stack::{Stack, StackServiceNames},
     stats::{SystemInformation, SystemProcess},
@@ -66,7 +67,6 @@ impl Resolve<ReadArgs> for GetServersSummary {
     let servers = resource::list_for_user::<Server>(
       Default::default(),
       user,
-      PermissionLevel::Read.into(),
       &[],
     )
     .await?;
@@ -127,13 +127,8 @@ impl Resolve<ReadArgs> for ListServers {
       get_all_tags(None).await?
     };
     Ok(
-      resource::list_for_user::<Server>(
-        self.query,
-        user,
-        PermissionLevel::Read.into(),
-        &all_tags,
-      )
-      .await?,
+      resource::list_for_user::<Server>(self.query, user, &all_tags)
+        .await?,
     )
   }
 }
@@ -150,10 +145,7 @@ impl Resolve<ReadArgs> for ListFullServers {
     };
     Ok(
       resource::list_full_for_user::<Server>(
-        self.query,
-        user,
-        PermissionLevel::Read.into(),
-        &all_tags,
+        self.query, user, &all_tags,
       )
       .await?,
     )
@@ -400,7 +392,6 @@ impl Resolve<ReadArgs> for ListAllDockerContainers {
     let servers = resource::list_for_user::<Server>(
       Default::default(),
       user,
-      PermissionLevel::Read.into(),
       &[],
     )
     .await?
@@ -434,7 +425,6 @@ impl Resolve<ReadArgs> for GetDockerContainersSummary {
     let servers = resource::list_full_for_user::<Server>(
       Default::default(),
       user,
-      PermissionLevel::Read.into(),
       &[],
     )
     .await
@@ -880,20 +870,78 @@ impl Resolve<ReadArgs> for ListTerminals {
       PermissionLevel::Read.terminal(),
     )
     .await?;
-    let cache = terminals_cache().get_or_insert(server.id.clone());
-    let mut cache = cache.lock().await;
-    if self.fresh || komodo_timestamp() > cache.ttl {
-      cache.list = periphery_client(&server)
-        .await?
-        .request(periphery_client::api::terminal::ListTerminals {
-          container: None,
-        })
-        .await
-        .context("Failed to get fresh terminal list")?;
-      cache.ttl = komodo_timestamp() + TERMINAL_CACHE_TIMEOUT;
-      Ok(cache.list.clone())
+    list_terminals_inner(&server, self.fresh)
+      .await
+      .map_err(Into::into)
+  }
+}
+
+impl Resolve<ReadArgs> for ListAllTerminals {
+  async fn resolve(
+    self,
+    args: &ReadArgs,
+  ) -> Result<Self::Response, Self::Error> {
+    let all_tags = if self.query.tags.is_empty() {
+      vec![]
     } else {
-      Ok(cache.list.clone())
-    }
+      get_all_tags(None).await?
+    };
+
+    let mut terminals = resource::list_full_for_user::<Server>(
+      self.query, &args.user, &all_tags,
+    )
+    .await?
+    .into_iter()
+    .map(|server| async move {
+      (
+        list_terminals_inner(&server, self.fresh).await,
+        (server.id, server.name),
+      )
+    })
+    .collect::<FuturesUnordered<_>>()
+    .collect::<Vec<_>>()
+    .await
+    .into_iter()
+    .flat_map(|(terminals, server)| {
+      let terminals = terminals.ok()?;
+      Some((terminals, server))
+    })
+    .flat_map(|(terminals, (server_id, server_name))| {
+      terminals.into_iter().map(move |info| {
+        TerminalInfoWithServer::from_terminal_info(
+          &server_id,
+          &server_name,
+          info,
+        )
+      })
+    })
+    .collect::<Vec<_>>();
+
+    terminals.sort_by(|a, b| {
+      a.server_name.cmp(&b.server_name).then(a.name.cmp(&b.name))
+    });
+
+    Ok(terminals)
+  }
+}
+
+async fn list_terminals_inner(
+  server: &Server,
+  fresh: bool,
+) -> anyhow::Result<Vec<TerminalInfo>> {
+  let cache = terminals_cache().get_or_insert(server.id.clone());
+  let mut cache = cache.lock().await;
+  if fresh || komodo_timestamp() > cache.ttl {
+    cache.list = periphery_client(server)
+      .await?
+      .request(periphery_client::api::terminal::ListTerminals {
+        container: None,
+      })
+      .await
+      .context("Failed to get fresh terminal list")?;
+    cache.ttl = komodo_timestamp() + TERMINAL_CACHE_TIMEOUT;
+    Ok(cache.list.clone())
+  } else {
+    Ok(cache.list.clone())
   }
 }

bin/core/src/api/read/stack.rs

@@ -200,10 +200,7 @@ impl Resolve<ReadArgs> for ListCommonStackExtraArgs {
       get_all_tags(None).await?
     };
     let stacks = resource::list_full_for_user::<Stack>(
-      self.query,
-      user,
-      PermissionLevel::Read.into(),
-      &all_tags,
+      self.query, user, &all_tags,
     )
     .await
     .context("failed to get resources matching query")?;
@@ -234,10 +231,7 @@ impl Resolve<ReadArgs> for ListCommonStackBuildExtraArgs {
       get_all_tags(None).await?
     };
     let stacks = resource::list_full_for_user::<Stack>(
-      self.query,
-      user,
-      PermissionLevel::Read.into(),
-      &all_tags,
+      self.query, user, &all_tags,
     )
     .await
     .context("failed to get resources matching query")?;
@@ -268,13 +262,9 @@ impl Resolve<ReadArgs> for ListStacks {
       get_all_tags(None).await?
     };
     let only_update_available = self.query.specific.update_available;
-    let stacks = resource::list_for_user::<Stack>(
-      self.query,
-      user,
-      PermissionLevel::Read.into(),
-      &all_tags,
-    )
-    .await?;
+    let stacks =
+      resource::list_for_user::<Stack>(self.query, user, &all_tags)
+        .await?;
     let stacks = if only_update_available {
       stacks
         .into_iter()
@@ -305,10 +295,7 @@ impl Resolve<ReadArgs> for ListFullStacks {
     };
     Ok(
       resource::list_full_for_user::<Stack>(
-        self.query,
-        user,
-        PermissionLevel::Read.into(),
-        &all_tags,
+        self.query, user, &all_tags,
       )
       .await?,
     )
@@ -344,7 +331,6 @@ impl Resolve<ReadArgs> for GetStacksSummary {
     let stacks = resource::list_full_for_user::<Stack>(
       Default::default(),
       user,
-      PermissionLevel::Read.into(),
       &[],
     )
     .await

bin/core/src/api/read/sync.rs

@@ -45,10 +45,7 @@ impl Resolve<ReadArgs> for ListResourceSyncs {
     };
     Ok(
       resource::list_for_user::<ResourceSync>(
-        self.query,
-        user,
-        PermissionLevel::Read.into(),
-        &all_tags,
+        self.query, user, &all_tags,
       )
       .await?,
     )
@@ -67,10 +64,7 @@ impl Resolve<ReadArgs> for ListFullResourceSyncs {
     };
     Ok(
       resource::list_full_for_user::<ResourceSync>(
-        self.query,
-        user,
-        PermissionLevel::Read.into(),
-        &all_tags,
+        self.query, user, &all_tags,
       )
       .await?,
     )
@@ -107,7 +101,6 @@ impl Resolve<ReadArgs> for GetResourceSyncsSummary {
       resource::list_full_for_user::<ResourceSync>(
         Default::default(),
         user,
-        PermissionLevel::Read.into(),
         &[],
       )
       .await

bin/core/src/api/read/toml.rs

@@ -46,7 +46,6 @@ async fn get_all_targets(
     resource::list_full_for_user::<Alerter>(
       ResourceQuery::builder().tags(tags).build(),
       user,
-      PermissionLevel::Read.into(),
       &all_tags,
     )
     .await?
@@ -57,7 +56,6 @@ async fn get_all_targets(
     resource::list_full_for_user::<Builder>(
       ResourceQuery::builder().tags(tags).build(),
       user,
-      PermissionLevel::Read.into(),
       &all_tags,
     )
     .await?
@@ -68,7 +66,6 @@ async fn get_all_targets(
     resource::list_full_for_user::<Server>(
       ResourceQuery::builder().tags(tags).build(),
       user,
-      PermissionLevel::Read.into(),
       &all_tags,
     )
     .await?
@@ -79,7 +76,6 @@ async fn get_all_targets(
     resource::list_full_for_user::<Stack>(
       ResourceQuery::builder().tags(tags).build(),
       user,
-      PermissionLevel::Read.into(),
       &all_tags,
     )
     .await?
@@ -90,7 +86,6 @@ async fn get_all_targets(
     resource::list_full_for_user::<Deployment>(
       ResourceQuery::builder().tags(tags).build(),
       user,
-      PermissionLevel::Read.into(),
       &all_tags,
     )
     .await?
@@ -101,7 +96,6 @@ async fn get_all_targets(
     resource::list_full_for_user::<Build>(
       ResourceQuery::builder().tags(tags).build(),
       user,
-      PermissionLevel::Read.into(),
       &all_tags,
     )
     .await?
@@ -112,7 +106,6 @@ async fn get_all_targets(
     resource::list_full_for_user::<Repo>(
       ResourceQuery::builder().tags(tags).build(),
       user,
-      PermissionLevel::Read.into(),
       &all_tags,
     )
     .await?
@@ -123,7 +116,6 @@ async fn get_all_targets(
     resource::list_full_for_user::<Procedure>(
       ResourceQuery::builder().tags(tags).build(),
       user,
-      PermissionLevel::Read.into(),
       &all_tags,
     )
     .await?
@@ -134,7 +126,6 @@ async fn get_all_targets(
     resource::list_full_for_user::<Action>(
       ResourceQuery::builder().tags(tags).build(),
       user,
-      PermissionLevel::Read.into(),
       &all_tags,
     )
     .await?
@@ -145,7 +136,6 @@ async fn get_all_targets(
     resource::list_full_for_user::<ResourceSync>(
       ResourceQuery::builder().tags(tags).build(),
       user,
-      PermissionLevel::Read.into(),
       &all_tags,
     )
     .await?

bin/core/src/api/terminal.rs

@@ -1,14 +1,14 @@
 use anyhow::Context;
 use axum::{Extension, Router, middleware, routing::post};
 use komodo_client::{
-  api::terminal::*,
+  api::{terminal::*, write::TerminalRecreateMode},
   entities::{
     deployment::Deployment, permission::PermissionLevel,
     server::Server, stack::Stack, user::User,
   },
 };
+use periphery_client::api::terminal::CreateTerminal;
 use serror::Json;
-use uuid::Uuid;
 
 use crate::{
   auth::auth_request, helpers::periphery_client,
@@ -29,28 +29,23 @@ pub fn router() -> Router {
 //  ExecuteTerminal
 // =================
 
-async fn execute_terminal(
-  Extension(user): Extension<User>,
-  Json(request): Json<ExecuteTerminalBody>,
-) -> serror::Result<axum::body::Body> {
-  execute_terminal_inner(Uuid::new_v4(), request, user).await
-}
-
 #[instrument(
   name = "ExecuteTerminal",
-  skip(user),
+  skip_all,
   fields(
-    user_id = user.id,
+    operator = user.id,
+    server,
+    terminal,
   )
 )]
-async fn execute_terminal_inner(
-  req_id: Uuid,
-  ExecuteTerminalBody {
+async fn execute_terminal(
+  Extension(user): Extension<User>,
+  Json(ExecuteTerminalBody {
     server,
     terminal,
     command,
-  }: ExecuteTerminalBody,
-  user: User,
+    init,
+  }): Json<ExecuteTerminalBody>,
 ) -> serror::Result<axum::body::Body> {
   info!("/terminal/execute request | user: {}", user.username);
 
@@ -61,8 +56,20 @@ async fn execute_terminal_inner(
   )
   .await?;
 
-  let stream = periphery_client(&server)
-    .await?
+  let periphery = periphery_client(&server).await?;
+
+  // Maybe init terminal.
+  if let Some(InitTerminal { command, recreate }) = init {
+    periphery
+      .request(CreateTerminal {
+        name: terminal.clone(),
+        command,
+        recreate,
+      })
+      .await?;
+  }
+
+  let stream = periphery
     .execute_terminal(terminal, command)
     .await
     .context("Failed to execute command on periphery")?;
@@ -74,30 +81,26 @@ async fn execute_terminal_inner(
 //  ExecuteContainerExec
 // ======================
 
-async fn execute_container_exec(
-  Extension(user): Extension<User>,
-  Json(request): Json<ExecuteContainerExecBody>,
-) -> serror::Result<axum::body::Body> {
-  execute_container_exec_inner(Uuid::new_v4(), request, user).await
-}
-
 #[instrument(
   name = "ExecuteContainerExec",
-  skip(user),
+  skip_all,
   fields(
-    user_id = user.id,
+    operator = user.id,
+    server,
+    container,
+    shell,
+    recreate = format!("{recreate:?}"),
   )
 )]
-async fn execute_container_exec_inner(
-  req_id: Uuid,
-  ExecuteContainerExecBody {
+async fn execute_container_exec(
+  Extension(user): Extension<User>,
+  Json(ExecuteContainerExecBody {
     server,
     container,
     shell,
     command,
     recreate,
-  }: ExecuteContainerExecBody,
-  user: User,
+  }): Json<ExecuteContainerExecBody>,
 ) -> serror::Result<axum::body::Body> {
   info!("ExecuteContainerExec request | user: {}", user.username);
 
@@ -111,7 +114,12 @@ async fn execute_container_exec_inner(
   let periphery = periphery_client(&server).await?;
 
   let stream = periphery
-    .execute_container_exec(container, shell, command, recreate)
+    .execute_container_exec(
+      container,
+      shell,
+      command,
+      recreate.unwrap_or(TerminalRecreateMode::DifferentCommand),
+    )
     .await
     .context(
       "Failed to execute container exec command on periphery",
@@ -124,29 +132,24 @@ async fn execute_container_exec_inner(
 //  ExecuteDeploymentExec
 // =======================
 
-async fn execute_deployment_exec(
-  Extension(user): Extension<User>,
-  Json(request): Json<ExecuteDeploymentExecBody>,
-) -> serror::Result<axum::body::Body> {
-  execute_deployment_exec_inner(Uuid::new_v4(), request, user).await
-}
-
 #[instrument(
   name = "ExecuteDeploymentExec",
-  skip(user),
+  skip_all,
   fields(
-    user_id = user.id,
+    operator = user.id,
+    deployment,
+    shell,
+    recreate = format!("{recreate:?}"),
   )
 )]
-async fn execute_deployment_exec_inner(
-  req_id: Uuid,
-  ExecuteDeploymentExecBody {
+async fn execute_deployment_exec(
+  Extension(user): Extension<User>,
+  Json(ExecuteDeploymentExecBody {
     deployment,
     shell,
     command,
     recreate,
-  }: ExecuteDeploymentExecBody,
-  user: User,
+  }): Json<ExecuteDeploymentExecBody>,
 ) -> serror::Result<axum::body::Body> {
   info!("ExecuteDeploymentExec request | user: {}", user.username);
 
@@ -162,7 +165,12 @@ async fn execute_deployment_exec_inner(
   let periphery = periphery_client(&server).await?;
 
   let stream = periphery
-    .execute_container_exec(deployment.name, shell, command, recreate)
+    .execute_container_exec(
+      deployment.name,
+      shell,
+      command,
+      recreate.unwrap_or(TerminalRecreateMode::DifferentCommand),
+    )
     .await
     .context(
       "Failed to execute container exec command on periphery",
@@ -175,30 +183,26 @@ async fn execute_deployment_exec_inner(
 //  ExecuteStackExec
 // ==================
 
-async fn execute_stack_exec(
-  Extension(user): Extension<User>,
-  Json(request): Json<ExecuteStackExecBody>,
-) -> serror::Result<axum::body::Body> {
-  execute_stack_exec_inner(Uuid::new_v4(), request, user).await
-}
-
 #[instrument(
   name = "ExecuteStackExec",
-  skip(user),
+  skip_all,
   fields(
-    user_id = user.id,
+    operator = user.id,
+    stack,
+    service,
+    shell,
+    recreate = format!("{recreate:?}"),
   )
 )]
-async fn execute_stack_exec_inner(
-  req_id: Uuid,
-  ExecuteStackExecBody {
+async fn execute_stack_exec(
+  Extension(user): Extension<User>,
+  Json(ExecuteStackExecBody {
     stack,
     service,
     shell,
     command,
     recreate,
-  }: ExecuteStackExecBody,
-  user: User,
+  }): Json<ExecuteStackExecBody>,
 ) -> serror::Result<axum::body::Body> {
   info!("ExecuteStackExec request | user: {}", user.username);
 
@@ -229,7 +233,12 @@ async fn execute_stack_exec_inner(
   let periphery = periphery_client(&server).await?;
 
   let stream = periphery
-    .execute_container_exec(container, shell, command, recreate)
+    .execute_container_exec(
+      container,
+      shell,
+      command,
+      recreate.unwrap_or(TerminalRecreateMode::DifferentCommand),
+    )
     .await
     .context(
       "Failed to execute container exec command on periphery",

bin/core/src/api/user.rs

@@ -9,6 +9,7 @@ use database::mungos::{
   by_id::update_one_by_id, mongodb::bson::to_bson,
 };
 use derive_variants::EnumVariants;
+use komodo_client::entities::random_string;
 use komodo_client::{
   api::user::*,
   entities::{api_key::ApiKey, komodo_timestamp, user::User},
@@ -21,9 +22,7 @@ use typeshare::typeshare;
 use uuid::Uuid;
 
 use crate::{
-  auth::auth_request,
-  helpers::{query::get_user, random_string},
-  state::db_client,
+  auth::auth_request, helpers::query::get_user, state::db_client,
 };
 
 use super::Variant;
@@ -66,7 +65,6 @@ async fn variant_handler(
   handler(user, Json(req)).await
 }
 
-#[instrument(name = "UserHandler", level = "debug", skip(user))]
 async fn handler(
   Extension(user): Extension<User>,
   Json(request): Json<UserRequest>,
@@ -89,11 +87,6 @@ async fn handler(
 const RECENTLY_VIEWED_MAX: usize = 10;
 
 impl Resolve<UserArgs> for PushRecentlyViewed {
-  #[instrument(
-    name = "PushRecentlyViewed",
-    level = "debug",
-    skip(user)
-  )]
   async fn resolve(
     self,
     UserArgs { user }: &UserArgs,
@@ -131,11 +124,6 @@ impl Resolve<UserArgs> for PushRecentlyViewed {
 }
 
 impl Resolve<UserArgs> for SetLastSeenUpdate {
-  #[instrument(
-    name = "SetLastSeenUpdate",
-    level = "debug",
-    skip(user)
-  )]
   async fn resolve(
     self,
     UserArgs { user }: &UserArgs,
@@ -158,7 +146,11 @@ const SECRET_LENGTH: usize = 40;
 const BCRYPT_COST: u32 = 10;
 
 impl Resolve<UserArgs> for CreateApiKey {
-  #[instrument(name = "CreateApiKey", level = "debug", skip(user))]
+  #[instrument(
+    "CreateApiKey",
+    skip_all,
+    fields(operator = user.id)
+  )]
   async fn resolve(
     self,
     UserArgs { user }: &UserArgs,
@@ -188,7 +180,11 @@ impl Resolve<UserArgs> for CreateApiKey {
 }
 
 impl Resolve<UserArgs> for DeleteApiKey {
-  #[instrument(name = "DeleteApiKey", level = "debug", skip(user))]
+  #[instrument(
+    "DeleteApiKey",
+    skip_all,
+    fields(operator = user.id)
+  )]
   async fn resolve(
     self,
     UserArgs { user }: &UserArgs,

bin/core/src/api/write/action.rs

@@ -11,7 +11,15 @@ use crate::{permission::get_check_permissions, resource};
 use super::WriteArgs;
 
 impl Resolve<WriteArgs> for CreateAction {
-  #[instrument(name = "CreateAction", skip(user))]
+  #[instrument(
+    "CreateAction",
+    skip_all,
+    fields(
+      operator = user.id,
+      action = self.name,
+      config = serde_json::to_string(&self.config).unwrap(),
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -22,7 +30,15 @@ impl Resolve<WriteArgs> for CreateAction {
 }
 
 impl Resolve<WriteArgs> for CopyAction {
-  #[instrument(name = "CopyAction", skip(user))]
+  #[instrument(
+    "CopyAction",
+    skip_all,
+    fields(
+      operator = user.id,
+      action = self.name,
+      copy_action = self.id,
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -39,7 +55,15 @@ impl Resolve<WriteArgs> for CopyAction {
 }
 
 impl Resolve<WriteArgs> for UpdateAction {
-  #[instrument(name = "UpdateAction", skip(user))]
+  #[instrument(
+    "UpdateAction",
+    skip_all,
+    fields(
+      operator = user.id,
+      action = self.id,
+      update = serde_json::to_string(&self.config).unwrap(),
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -49,7 +73,15 @@ impl Resolve<WriteArgs> for UpdateAction {
 }
 
 impl Resolve<WriteArgs> for RenameAction {
-  #[instrument(name = "RenameAction", skip(user))]
+  #[instrument(
+    "RenameAction",
+    skip_all,
+    fields(
+      operator = user.id,
+      action = self.id,
+      new_name = self.name,
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -59,8 +91,18 @@ impl Resolve<WriteArgs> for RenameAction {
 }
 
 impl Resolve<WriteArgs> for DeleteAction {
-  #[instrument(name = "DeleteAction", skip(args))]
-  async fn resolve(self, args: &WriteArgs) -> serror::Result<Action> {
-    Ok(resource::delete::<Action>(&self.id, args).await?)
+  #[instrument(
+    "DeleteAction",
+    skip_all,
+    fields(
+      operator = user.id,
+      action = self.id
+    )
+  )]
+  async fn resolve(
+    self,
+    WriteArgs { user }: &WriteArgs,
+  ) -> serror::Result<Action> {
+    Ok(resource::delete::<Action>(&self.id, user).await?)
   }
 }

bin/core/src/api/write/alert.rs

@@ -10,6 +10,14 @@ use serror::AddStatusCodeError;
 use crate::{api::write::WriteArgs, state::db_client};
 
 impl Resolve<WriteArgs> for CloseAlert {
+  #[instrument(
+    "CloseAlert",
+    skip_all,
+    fields(
+      operator = admin.id,
+      alert_id = self.id,
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user: admin }: &WriteArgs,

bin/core/src/api/write/alerter.rs

@@ -11,7 +11,15 @@ use crate::{permission::get_check_permissions, resource};
 use super::WriteArgs;
 
 impl Resolve<WriteArgs> for CreateAlerter {
-  #[instrument(name = "CreateAlerter", skip(user))]
+  #[instrument(
+    "CreateAlerter",
+    skip_all,
+    fields(
+      operator = user.id,
+      alerter = self.name,
+      config = serde_json::to_string(&self.config).unwrap(),
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -22,7 +30,15 @@ impl Resolve<WriteArgs> for CreateAlerter {
 }
 
 impl Resolve<WriteArgs> for CopyAlerter {
-  #[instrument(name = "CopyAlerter", skip(user))]
+  #[instrument(
+    "CopyAlerter",
+    skip_all,
+    fields(
+      operator = user.id,
+      alerter = self.name,
+      copy_alerter = self.id,
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -39,17 +55,32 @@ impl Resolve<WriteArgs> for CopyAlerter {
 }
 
 impl Resolve<WriteArgs> for DeleteAlerter {
-  #[instrument(name = "DeleteAlerter", skip(args))]
+  #[instrument(
+    "DeleteAlerter",
+    skip_all,
+    fields(
+      operator = user.id,
+      alerter = self.id,
+    )
+  )]
   async fn resolve(
     self,
-    args: &WriteArgs,
+    WriteArgs { user }: &WriteArgs,
   ) -> serror::Result<Alerter> {
-    Ok(resource::delete::<Alerter>(&self.id, args).await?)
+    Ok(resource::delete::<Alerter>(&self.id, user).await?)
   }
 }
 
 impl Resolve<WriteArgs> for UpdateAlerter {
-  #[instrument(name = "UpdateAlerter", skip(user))]
+  #[instrument(
+    "UpdateAlerter",
+    skip_all,
+    fields(
+      operator = user.id,
+      alerter = self.id,
+      update = serde_json::to_string(&self.config).unwrap()
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -62,7 +93,15 @@ impl Resolve<WriteArgs> for UpdateAlerter {
 }
 
 impl Resolve<WriteArgs> for RenameAlerter {
-  #[instrument(name = "RenameAlerter", skip(user))]
+  #[instrument(
+    "RenameAlerter",
+    skip_all,
+    fields(
+      operator = user.id,
+      alerter = self.id,
+      new_name = self.name,
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,

bin/core/src/api/write/build.rs

@@ -1,4 +1,4 @@
-use std::{path::PathBuf, str::FromStr, time::Duration};
+use std::{path::PathBuf, time::Duration};
 
 use anyhow::{Context, anyhow};
 use database::mungos::mongodb::bson::to_document;
@@ -42,7 +42,15 @@ use crate::{
 use super::WriteArgs;
 
 impl Resolve<WriteArgs> for CreateBuild {
-  #[instrument(name = "CreateBuild", skip(user))]
+  #[instrument(
+    "CreateBuild",
+    skip_all,
+    fields(
+      operator = user.id,
+      build = self.name,
+      config = serde_json::to_string(&self.config).unwrap(),
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -53,7 +61,15 @@ impl Resolve<WriteArgs> for CreateBuild {
 }
 
 impl Resolve<WriteArgs> for CopyBuild {
-  #[instrument(name = "CopyBuild", skip(user))]
+  #[instrument(
+    "CopyBuild",
+    skip_all,
+    fields(
+      operator = user.id,
+      build = self.name,
+      copy_build = self.id,
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -72,14 +88,32 @@ impl Resolve<WriteArgs> for CopyBuild {
 }
 
 impl Resolve<WriteArgs> for DeleteBuild {
-  #[instrument(name = "DeleteBuild", skip(args))]
-  async fn resolve(self, args: &WriteArgs) -> serror::Result<Build> {
-    Ok(resource::delete::<Build>(&self.id, args).await?)
+  #[instrument(
+    "DeleteBuild",
+    skip_all,
+    fields(
+      operator = user.id,
+      build = self.id,
+    )
+  )]
+  async fn resolve(
+    self,
+    WriteArgs { user }: &WriteArgs,
+  ) -> serror::Result<Build> {
+    Ok(resource::delete::<Build>(&self.id, user).await?)
   }
 }
 
 impl Resolve<WriteArgs> for UpdateBuild {
-  #[instrument(name = "UpdateBuild", skip(user))]
+  #[instrument(
+    "UpdateBuild",
+    skip_all,
+    fields(
+      operator = user.id,
+      build = self.id,
+      update = serde_json::to_string(&self.config).unwrap(),
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -89,7 +123,15 @@ impl Resolve<WriteArgs> for UpdateBuild {
 }
 
 impl Resolve<WriteArgs> for RenameBuild {
-  #[instrument(name = "RenameBuild", skip(user))]
+  #[instrument(
+    "RenameBuild",
+    skip_all,
+    fields(
+      operator = user.id,
+      build = self.id,
+      new_name = self.name,
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -99,7 +141,14 @@ impl Resolve<WriteArgs> for RenameBuild {
 }
 
 impl Resolve<WriteArgs> for WriteBuildFileContents {
-  #[instrument(name = "WriteBuildFileContents", skip(args))]
+  #[instrument(
+    "WriteBuildFileContents",
+    skip_all,
+    fields(
+      operator = args.user.id,
+      build = self.build,
+    )
+  )]
   async fn resolve(self, args: &WriteArgs) -> serror::Result<Update> {
     let build = get_check_permissions::<Build>(
       &self.build,
@@ -171,6 +220,7 @@ impl Resolve<WriteArgs> for WriteBuildFileContents {
   }
 }
 
+#[instrument("WriteDockerfileContentsGit", skip_all)]
 async fn write_dockerfile_contents_git(
   req: WriteBuildFileContents,
   args: &WriteArgs,
@@ -317,11 +367,6 @@ async fn write_dockerfile_contents_git(
 }
 
 impl Resolve<WriteArgs> for RefreshBuildCache {
-  #[instrument(
-    name = "RefreshBuildCache",
-    level = "debug",
-    skip(user)
-  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -345,23 +390,28 @@ impl Resolve<WriteArgs> for RefreshBuildCache {
       None
     };
 
-    let (
-      remote_path,
-      remote_contents,
-      remote_error,
-      latest_hash,
-      latest_message,
-    ) = if build.config.files_on_host {
+    let RemoteDockerfileContents {
+      path,
+      contents,
+      error,
+      hash,
+      message,
+    } = if build.config.files_on_host {
       // =============
       // FILES ON HOST
       // =============
       match get_on_host_dockerfile(&build).await {
         Ok(FileContents { path, contents }) => {
-          (Some(path), Some(contents), None, None, None)
-        }
-        Err(e) => {
-          (None, None, Some(format_serror(&e.into())), None, None)
+          RemoteDockerfileContents {
+            path: Some(path),
+            contents: Some(contents),
+            ..Default::default()
+          }
         }
+        Err(e) => RemoteDockerfileContents {
+          error: Some(format_serror(&e.into())),
+          ..Default::default()
+        },
       }
     } else if let Some(repo) = &repo {
       let Some(res) = get_git_remote(&build, repo.into()).await?
@@ -381,7 +431,7 @@ impl Resolve<WriteArgs> for RefreshBuildCache {
       // =============
       // UI BASED FILE
       // =============
-      (None, None, None, None, None)
+      RemoteDockerfileContents::default()
     };
 
     let info = BuildInfo {
@@ -389,11 +439,11 @@ impl Resolve<WriteArgs> for RefreshBuildCache {
       built_hash: build.info.built_hash,
       built_message: build.info.built_message,
       built_contents: build.info.built_contents,
-      remote_path,
-      remote_contents,
-      remote_error,
-      latest_hash,
-      latest_message,
+      remote_path: path,
+      remote_contents: contents,
+      remote_error: error,
+      latest_hash: hash,
+      latest_message: message,
     };
 
     let info = to_document(&info)
@@ -485,15 +535,7 @@ async fn get_on_host_dockerfile(
 async fn get_git_remote(
   build: &Build,
   mut clone_args: RepoExecutionArgs,
-) -> anyhow::Result<
-  Option<(
-    Option<String>,
-    Option<String>,
-    Option<String>,
-    Option<String>,
-    Option<String>,
-  )>,
-> {
+) -> anyhow::Result<Option<RemoteDockerfileContents>> {
   if clone_args.provider.is_empty() {
     // Nothing to do here
     return Ok(None);
@@ -520,10 +562,19 @@ async fn get_git_remote(
     access_token,
   )
   .await
-  .context("failed to clone build repo")?;
+  .context("Failed to clone Build repo")?;
 
-  let relative_path = PathBuf::from_str(&build.config.build_path)
-    .context("Invalid build path")?
+  // Ensure clone / pull successful,
+  // propogate error log -> 'errored' and return.
+  if let Some(failure) = res.logs.iter().find(|log| !log.success) {
+    return Ok(Some(RemoteDockerfileContents {
+      path: Some(format!("Failed at: {}", failure.stage)),
+      error: Some(failure.combined()),
+      ..Default::default()
+    }));
+  }
+
+  let relative_path = PathBuf::from(&build.config.build_path)
     .join(&build.config.dockerfile_path);
 
   let full_path = repo_path.join(&relative_path);
@@ -534,11 +585,20 @@ async fn get_git_remote(
       Ok(contents) => (Some(contents), None),
       Err(e) => (None, Some(format_serror(&e.into()))),
     };
-  Ok(Some((
-    Some(relative_path.display().to_string()),
+  Ok(Some(RemoteDockerfileContents {
+    path: Some(relative_path.display().to_string()),
     contents,
     error,
-    res.commit_hash,
-    res.commit_message,
-  )))
+    hash: res.commit_hash,
+    message: res.commit_message,
+  }))
+}
+
+#[derive(Default)]
+pub struct RemoteDockerfileContents {
+  pub path: Option<String>,
+  pub contents: Option<String>,
+  pub error: Option<String>,
+  pub hash: Option<String>,
+  pub message: Option<String>,
 }

bin/core/src/api/write/builder.rs

@@ -11,7 +11,15 @@ use crate::{permission::get_check_permissions, resource};
 use super::WriteArgs;
 
 impl Resolve<WriteArgs> for CreateBuilder {
-  #[instrument(name = "CreateBuilder", skip(user))]
+  #[instrument(
+    "CreateBuilder",
+    skip_all,
+    fields(
+      operator = user.id,
+      builder = self.name,
+      config = serde_json::to_string(&self.config).unwrap(),
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -22,7 +30,15 @@ impl Resolve<WriteArgs> for CreateBuilder {
 }
 
 impl Resolve<WriteArgs> for CopyBuilder {
-  #[instrument(name = "CopyBuilder", skip(user))]
+  #[instrument(
+    "CopyBuilder",
+    skip_all,
+    fields(
+      operator = user.id,
+      builder = self.name,
+      copy_builder = self.id,
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -39,17 +55,32 @@ impl Resolve<WriteArgs> for CopyBuilder {
 }
 
 impl Resolve<WriteArgs> for DeleteBuilder {
-  #[instrument(name = "DeleteBuilder", skip(args))]
+  #[instrument(
+    "DeleteBuilder",
+    skip_all,
+    fields(
+      operator = user.id,
+      builder = self.id,
+    )
+  )]
   async fn resolve(
     self,
-    args: &WriteArgs,
+    WriteArgs { user }: &WriteArgs,
   ) -> serror::Result<Builder> {
-    Ok(resource::delete::<Builder>(&self.id, args).await?)
+    Ok(resource::delete::<Builder>(&self.id, user).await?)
   }
 }
 
 impl Resolve<WriteArgs> for UpdateBuilder {
-  #[instrument(name = "UpdateBuilder", skip(user))]
+  #[instrument(
+    "UpdateBuilder",
+    skip_all,
+    fields(
+      operator = user.id,
+      builder = self.id,
+      update = serde_json::to_string(&self.config).unwrap(),
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -62,7 +93,15 @@ impl Resolve<WriteArgs> for UpdateBuilder {
 }
 
 impl Resolve<WriteArgs> for RenameBuilder {
-  #[instrument(name = "RenameBuilder", skip(user))]
+  #[instrument(
+    "RenameBuilder",
+    skip_all,
+    fields(
+      operator = user.id,
+      builder = self.id,
+      new_name = self.name
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,

bin/core/src/api/write/deployment.rs

@@ -33,7 +33,15 @@ use crate::{
 use super::WriteArgs;
 
 impl Resolve<WriteArgs> for CreateDeployment {
-  #[instrument(name = "CreateDeployment", skip(user))]
+  #[instrument(
+    "CreateDeployment",
+    skip_all,
+    fields(
+      operator = user.id,
+      deployment = self.name,
+      config = serde_json::to_string(&self.config).unwrap(),
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -49,7 +57,15 @@ impl Resolve<WriteArgs> for CreateDeployment {
 }
 
 impl Resolve<WriteArgs> for CopyDeployment {
-  #[instrument(name = "CopyDeployment", skip(user))]
+  #[instrument(
+    "CopyDeployment",
+    skip_all,
+    fields(
+      operator = user.id,
+      deployment = self.name,
+      copy_deployment = self.id,
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -72,7 +88,15 @@ impl Resolve<WriteArgs> for CopyDeployment {
 }
 
 impl Resolve<WriteArgs> for CreateDeploymentFromContainer {
-  #[instrument(name = "CreateDeploymentFromContainer", skip(user))]
+  #[instrument(
+    "CreateDeploymentFromContainer",
+    skip_all,
+    fields(
+      operator = user.id,
+      server = self.server,
+      deployment = self.name,
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -166,17 +190,32 @@ impl Resolve<WriteArgs> for CreateDeploymentFromContainer {
 }
 
 impl Resolve<WriteArgs> for DeleteDeployment {
-  #[instrument(name = "DeleteDeployment", skip(args))]
+  #[instrument(
+    "DeleteDeployment",
+    skip_all,
+    fields(
+      operator = user.id,
+      deployment = self.id
+    )
+  )]
   async fn resolve(
     self,
-    args: &WriteArgs,
+    WriteArgs { user }: &WriteArgs,
   ) -> serror::Result<Deployment> {
-    Ok(resource::delete::<Deployment>(&self.id, args).await?)
+    Ok(resource::delete::<Deployment>(&self.id, user).await?)
   }
 }
 
 impl Resolve<WriteArgs> for UpdateDeployment {
-  #[instrument(name = "UpdateDeployment", skip(user))]
+  #[instrument(
+    "UpdateDeployment",
+    skip_all,
+    fields(
+      operator = user.id,
+      deployment = self.id,
+      update = serde_json::to_string(&self.config).unwrap(),
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -189,7 +228,15 @@ impl Resolve<WriteArgs> for UpdateDeployment {
 }
 
 impl Resolve<WriteArgs> for RenameDeployment {
-  #[instrument(name = "RenameDeployment", skip(user))]
+  #[instrument(
+    "RenameDeployment",
+    skip_all,
+    fields(
+      operator = user.id,
+      deployment = self.id,
+      new_name = self.name,
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,

bin/core/src/api/write/mod.rs

@@ -1,5 +1,3 @@
-use std::time::Instant;
-
 use anyhow::Context;
 use axum::{
   Extension, Router, extract::Path, middleware, routing::post,
@@ -11,6 +9,7 @@ use response::Response;
 use serde::{Deserialize, Serialize};
 use serde_json::json;
 use serror::Json;
+use strum::Display;
 use typeshare::typeshare;
 use uuid::Uuid;
 
@@ -47,7 +46,7 @@ pub struct WriteArgs {
 #[derive(
   Serialize, Deserialize, Debug, Clone, Resolve, EnumVariants,
 )]
-#[variant_derive(Debug)]
+#[variant_derive(Debug, Display)]
 #[args(WriteArgs)]
 #[response(Response)]
 #[error(serror::Error)]
@@ -93,6 +92,7 @@ pub enum WriteRequest {
   CreateTerminal(CreateTerminal),
   DeleteTerminal(DeleteTerminal),
   DeleteAllTerminals(DeleteAllTerminals),
+  BatchDeleteAllTerminals(BatchDeleteAllTerminals),
   UpdateServerPublicKey(UpdateServerPublicKey),
   RotateServerKeys(RotateServerKeys),
 
@@ -230,31 +230,22 @@ async fn handler(
   res?
 }
 
-#[instrument(
-  name = "WriteRequest",
-  skip(user, request),
-  fields(
-    user_id = user.id,
-    request = format!("{:?}", request.extract_variant())
-  )
-)]
 async fn task(
   req_id: Uuid,
   request: WriteRequest,
   user: User,
 ) -> serror::Result<axum::response::Response> {
-  info!("/write request | user: {}", user.username);
-
-  let timer = Instant::now();
+  let variant = request.extract_variant();
+  info!("/write request | {variant} | user: {}", user.username);
 
   let res = request.resolve(&WriteArgs { user }).await;
 
   if let Err(e) = &res {
-    warn!("/write request {req_id} error: {:#}", e.error);
+    warn!(
+      "/write request {req_id} | {variant} | error: {:#}",
+      e.error
+    );
   }
 
-  let elapsed = timer.elapsed();
-  debug!("/write request {req_id} | resolve time: {elapsed:?}");
-
   res.map(|res| res.0)
 }

bin/core/src/api/write/onboarding_key.rs

@@ -6,7 +6,9 @@ use komodo_client::{
     DeleteOnboardingKey, DeleteOnboardingKeyResponse,
     UpdateOnboardingKey, UpdateOnboardingKeyResponse,
   },
-  entities::{komodo_timestamp, onboarding_key::OnboardingKey},
+  entities::{
+    komodo_timestamp, onboarding_key::OnboardingKey, random_string,
+  },
 };
 use noise::key::EncodedKeyPair;
 use reqwest::StatusCode;
@@ -18,7 +20,18 @@ use crate::{api::write::WriteArgs, state::db_client};
 //
 
 impl Resolve<WriteArgs> for CreateOnboardingKey {
-  #[instrument(name = "CreateServerOnboardingKey", skip(self, admin))]
+  #[instrument(
+    "CreateOnboardingKey",
+    skip_all,
+    fields(
+      operator = admin.id,
+      name = self.name,
+      expires = self.expires,
+      tags = format!("{:?}", self.tags),
+      copy_server = self.copy_server,
+      create_builder = self.create_builder,
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user: admin }: &WriteArgs,
@@ -29,13 +42,16 @@ impl Resolve<WriteArgs> for CreateOnboardingKey {
           .status_code(StatusCode::FORBIDDEN),
       );
     }
-    let keys = if let Some(private_key) = self.private_key {
-      EncodedKeyPair::from_private_key(&private_key)?
+    let private_key = if let Some(private_key) = self.private_key {
+      private_key
     } else {
-      EncodedKeyPair::generate()?
+      format!("O-{}", random_string(30))
     };
+    let public_key = EncodedKeyPair::from_private_key(&private_key)?
+      .public
+      .into_inner();
     let onboarding_key = OnboardingKey {
-      public_key: keys.public.into_inner(),
+      public_key,
       name: self.name,
       enabled: true,
       onboarded: Default::default(),
@@ -62,7 +78,7 @@ impl Resolve<WriteArgs> for CreateOnboardingKey {
         "No Server onboarding key found on database after create",
       )?;
     Ok(CreateOnboardingKeyResponse {
-      private_key: keys.private.into_inner(),
+      private_key,
       created,
     })
   }
@@ -71,6 +87,15 @@ impl Resolve<WriteArgs> for CreateOnboardingKey {
 //
 
 impl Resolve<WriteArgs> for UpdateOnboardingKey {
+  #[instrument(
+    "UpdateOnboardingKey",
+    skip_all,
+    fields(
+      operator = admin.id,
+      public_key = self.public_key,
+      update = format!("{:?}", self),
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user: admin }: &WriteArgs,
@@ -140,7 +165,14 @@ impl Resolve<WriteArgs> for UpdateOnboardingKey {
 //
 
 impl Resolve<WriteArgs> for DeleteOnboardingKey {
-  #[instrument(name = "DeleteServerOnboardingKey", skip(admin))]
+  #[instrument(
+    "DeleteOnboardingKey",
+    skip_all,
+    fields(
+      operator = admin.id,
+      public_key = self.public_key,
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user: admin }: &WriteArgs,

bin/core/src/api/write/permissions.rs

@@ -8,6 +8,7 @@ use database::mungos::{
     options::UpdateOptions,
   },
 };
+use derive_variants::ExtractVariant as _;
 use komodo_client::{
   api::write::*,
   entities::{
@@ -22,7 +23,15 @@ use crate::{helpers::query::get_user, state::db_client};
 use super::WriteArgs;
 
 impl Resolve<WriteArgs> for UpdateUserAdmin {
-  #[instrument(name = "UpdateUserAdmin", skip(super_admin))]
+  #[instrument(
+    "UpdateUserAdmin",
+    skip_all,
+    fields(
+      operator = super_admin.id,
+      target_user = self.user_id,
+      admin = self.admin,
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user: super_admin }: &WriteArgs,
@@ -60,7 +69,17 @@ impl Resolve<WriteArgs> for UpdateUserAdmin {
 }
 
 impl Resolve<WriteArgs> for UpdateUserBasePermissions {
-  #[instrument(name = "UpdateUserBasePermissions", skip(admin))]
+  #[instrument(
+    "UpdateUserBasePermissions",
+    skip_all,
+    fields(
+      operator = admin.id,
+      target_user = self.user_id,
+      enabled = self.enabled,
+      create_servers = self.create_servers,
+      create_builds = self.create_builds,
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user: admin }: &WriteArgs,
@@ -117,7 +136,16 @@ impl Resolve<WriteArgs> for UpdateUserBasePermissions {
 }
 
 impl Resolve<WriteArgs> for UpdatePermissionOnResourceType {
-  #[instrument(name = "UpdatePermissionOnResourceType", skip(admin))]
+  #[instrument(
+    "UpdatePermissionOnResourceType",
+    skip_all,
+    fields(
+      operator = admin.id,
+      user_target = format!("{:?}", self.user_target),
+      resource_type = self.resource_type.to_string(),
+      permission = format!("{:?}", self.permission),
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user: admin }: &WriteArgs,
@@ -185,7 +213,17 @@ impl Resolve<WriteArgs> for UpdatePermissionOnResourceType {
 }
 
 impl Resolve<WriteArgs> for UpdatePermissionOnTarget {
-  #[instrument(name = "UpdatePermissionOnTarget", skip(admin))]
+  #[instrument(
+    "UpdatePermissionOnTarget",
+    skip_all,
+    fields(
+      operator = admin.id,
+      user_target = format!("{:?}", self.user_target),
+      resource_type = self.resource_target.extract_variant().to_string(),
+      resource_id = self.resource_target.extract_variant_id().1,
+      permission = format!("{:?}", self.permission),
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user: admin }: &WriteArgs,

bin/core/src/api/write/procedure.rs

@@ -11,7 +11,15 @@ use crate::{permission::get_check_permissions, resource};
 use super::WriteArgs;
 
 impl Resolve<WriteArgs> for CreateProcedure {
-  #[instrument(name = "CreateProcedure", skip(user))]
+  #[instrument(
+    "CreateProcedure",
+    skip_all,
+    fields(
+      operator = user.id,
+      procedure = self.name,
+      config = serde_json::to_string(&self.config).unwrap()
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -22,7 +30,15 @@ impl Resolve<WriteArgs> for CreateProcedure {
 }
 
 impl Resolve<WriteArgs> for CopyProcedure {
-  #[instrument(name = "CopyProcedure", skip(user))]
+  #[instrument(
+    "CopyProcedure",
+    skip_all,
+    fields(
+      operator = user.id,
+      procedure = self.name,
+      copy_procedure = self.id,
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -45,7 +61,15 @@ impl Resolve<WriteArgs> for CopyProcedure {
 }
 
 impl Resolve<WriteArgs> for UpdateProcedure {
-  #[instrument(name = "UpdateProcedure", skip(user))]
+  #[instrument(
+    "UpdateProcedure",
+    skip_all,
+    fields(
+      operator = user.id,
+      procedure = self.id,
+      update = serde_json::to_string(&self.config).unwrap(),
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -58,7 +82,15 @@ impl Resolve<WriteArgs> for UpdateProcedure {
 }
 
 impl Resolve<WriteArgs> for RenameProcedure {
-  #[instrument(name = "RenameProcedure", skip(user))]
+  #[instrument(
+    "RenameProcedure",
+    skip_all,
+    fields(
+      operator = user.id,
+      procedure = self.id,
+      new_name = self.name,
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -71,11 +103,18 @@ impl Resolve<WriteArgs> for RenameProcedure {
 }
 
 impl Resolve<WriteArgs> for DeleteProcedure {
-  #[instrument(name = "DeleteProcedure", skip(args))]
+  #[instrument(
+    "DeleteProcedure",
+    skip_all,
+    fields(
+      operator = user.id,
+      procedure = self.id
+    )
+  )]
   async fn resolve(
     self,
-    args: &WriteArgs,
+    WriteArgs { user }: &WriteArgs,
   ) -> serror::Result<DeleteProcedureResponse> {
-    Ok(resource::delete::<Procedure>(&self.id, args).await?)
+    Ok(resource::delete::<Procedure>(&self.id, user).await?)
   }
 }

bin/core/src/api/write/provider.rs

@@ -10,7 +10,9 @@ use komodo_client::{
     provider::{DockerRegistryAccount, GitProviderAccount},
   },
 };
+use reqwest::StatusCode;
 use resolver_api::Resolve;
+use serror::AddStatusCodeError;
 
 use crate::{
   helpers::update::{add_update, make_update},
@@ -20,25 +22,41 @@ use crate::{
 use super::WriteArgs;
 
 impl Resolve<WriteArgs> for CreateGitProviderAccount {
+  #[instrument(
+    "CreateGitProviderAccount",
+    skip_all,
+    fields(
+      operator = user.id,
+      domain = self.account.domain,
+      username = self.account.username,
+      https = self.account.https.unwrap_or(true),
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
   ) -> serror::Result<CreateGitProviderAccountResponse> {
     if !user.admin {
       return Err(
-        anyhow!("only admins can create git provider accounts")
-          .into(),
+        anyhow!("Only admins can create git provider accounts")
+          .status_code(StatusCode::FORBIDDEN),
       );
     }
 
     let mut account: GitProviderAccount = self.account.into();
 
     if account.domain.is_empty() {
-      return Err(anyhow!("domain cannot be empty string.").into());
+      return Err(
+        anyhow!("Domain cannot be empty string.")
+          .status_code(StatusCode::BAD_REQUEST),
+      );
     }
 
     if account.username.is_empty() {
-      return Err(anyhow!("username cannot be empty string.").into());
+      return Err(
+        anyhow!("Username cannot be empty string.")
+          .status_code(StatusCode::BAD_REQUEST),
+      );
     }
 
     let mut update = make_update(
@@ -51,14 +69,14 @@ impl Resolve<WriteArgs> for CreateGitProviderAccount {
       .git_accounts
       .insert_one(&account)
       .await
-      .context("failed to create git provider account on db")?
+      .context("Failed to create git provider account on db")?
       .inserted_id
       .as_object_id()
-      .context("inserted id is not ObjectId")?
+      .context("Inserted id is not ObjectId")?
       .to_string();
 
     update.push_simple_log(
-      "create git provider account",
+      "Create git provider account",
       format!(
         "Created git provider account for {} with username {}",
         account.domain, account.username
@@ -70,7 +88,7 @@ impl Resolve<WriteArgs> for CreateGitProviderAccount {
     add_update(update)
       .await
       .inspect_err(|e| {
-        error!("failed to add update for create git provider account | {e:#}")
+        error!("Failed to add update for create git provider account | {e:#}")
       })
       .ok();
 
@@ -79,14 +97,25 @@ impl Resolve<WriteArgs> for CreateGitProviderAccount {
 }
 
 impl Resolve<WriteArgs> for UpdateGitProviderAccount {
+  #[instrument(
+    "UpdateGitProviderAccount",
+    skip_all,
+    fields(
+      operator = user.id,
+      id = self.id,
+      domain = self.account.domain,
+      username = self.account.username,
+      https = self.account.https.unwrap_or(true),
+    )
+  )]
   async fn resolve(
     mut self,
     WriteArgs { user }: &WriteArgs,
   ) -> serror::Result<UpdateGitProviderAccountResponse> {
     if !user.admin {
       return Err(
-        anyhow!("only admins can update git provider accounts")
-          .into(),
+        anyhow!("Only admins can update git provider accounts")
+          .status_code(StatusCode::FORBIDDEN),
       );
     }
 
@@ -94,8 +123,8 @@ impl Resolve<WriteArgs> for UpdateGitProviderAccount {
       && domain.is_empty()
     {
       return Err(
-        anyhow!("cannot update git provider with empty domain")
-          .into(),
+        anyhow!("Cannot update git provider with empty domain")
+          .status_code(StatusCode::BAD_REQUEST),
       );
     }
 
@@ -103,8 +132,8 @@ impl Resolve<WriteArgs> for UpdateGitProviderAccount {
       && username.is_empty()
     {
       return Err(
-        anyhow!("cannot update git provider with empty username")
-          .into(),
+        anyhow!("Cannot update git provider with empty username")
+          .status_code(StatusCode::BAD_REQUEST),
       );
     }
 
@@ -118,7 +147,7 @@ impl Resolve<WriteArgs> for UpdateGitProviderAccount {
     );
 
     let account = to_document(&self.account).context(
-      "failed to serialize partial git provider account to bson",
+      "Failed to serialize partial git provider account to bson",
     )?;
     let db = db_client();
     update_one_by_id(
@@ -128,17 +157,17 @@ impl Resolve<WriteArgs> for UpdateGitProviderAccount {
       None,
     )
     .await
-    .context("failed to update git provider account on db")?;
+    .context("Failed to update git provider account on db")?;
 
     let Some(account) = find_one_by_id(&db.git_accounts, &self.id)
       .await
-      .context("failed to query db for git accounts")?
+      .context("Failed to query db for git accounts")?
     else {
-      return Err(anyhow!("no account found with given id").into());
+      return Err(anyhow!("No account found with given id").into());
     };
 
     update.push_simple_log(
-      "update git provider account",
+      "Update git provider account",
       format!(
         "Updated git provider account for {} with username {}",
         account.domain, account.username
@@ -150,7 +179,7 @@ impl Resolve<WriteArgs> for UpdateGitProviderAccount {
     add_update(update)
       .await
       .inspect_err(|e| {
-        error!("failed to add update for update git provider account | {e:#}")
+        error!("Failed to add update for update git provider account | {e:#}")
       })
       .ok();
 
@@ -159,14 +188,22 @@ impl Resolve<WriteArgs> for UpdateGitProviderAccount {
 }
 
 impl Resolve<WriteArgs> for DeleteGitProviderAccount {
+  #[instrument(
+    "DeleteGitProviderAccount",
+    skip_all,
+    fields(
+      operator = user.id,
+      id = self.id,
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
   ) -> serror::Result<DeleteGitProviderAccountResponse> {
     if !user.admin {
       return Err(
-        anyhow!("only admins can delete git provider accounts")
-          .into(),
+        anyhow!("Only admins can delete git provider accounts")
+          .status_code(StatusCode::FORBIDDEN),
       );
     }
 
@@ -179,16 +216,19 @@ impl Resolve<WriteArgs> for DeleteGitProviderAccount {
     let db = db_client();
     let Some(account) = find_one_by_id(&db.git_accounts, &self.id)
       .await
-      .context("failed to query db for git accounts")?
+      .context("Failed to query db for git accounts")?
     else {
-      return Err(anyhow!("no account found with given id").into());
+      return Err(
+        anyhow!("No account found with given id")
+          .status_code(StatusCode::BAD_REQUEST),
+      );
     };
     delete_one_by_id(&db.git_accounts, &self.id, None)
       .await
       .context("failed to delete git account on db")?;
 
     update.push_simple_log(
-      "delete git provider account",
+      "Delete git provider account",
       format!(
         "Deleted git provider account for {} with username {}",
         account.domain, account.username
@@ -200,7 +240,7 @@ impl Resolve<WriteArgs> for DeleteGitProviderAccount {
     add_update(update)
       .await
       .inspect_err(|e| {
-        error!("failed to add update for delete git provider account | {e:#}")
+        error!("Failed to add update for delete git provider account | {e:#}")
       })
       .ok();
 
@@ -209,6 +249,15 @@ impl Resolve<WriteArgs> for DeleteGitProviderAccount {
 }
 
 impl Resolve<WriteArgs> for CreateDockerRegistryAccount {
+  #[instrument(
+    "CreateDockerRegistryAccount",
+    skip_all,
+    fields(
+      operator = user.id,
+      domain = self.account.domain,
+      username = self.account.username,
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -216,20 +265,26 @@ impl Resolve<WriteArgs> for CreateDockerRegistryAccount {
     if !user.admin {
       return Err(
         anyhow!(
-          "only admins can create docker registry account accounts"
+          "Only admins can create docker registry account accounts"
         )
-        .into(),
+        .status_code(StatusCode::FORBIDDEN),
       );
     }
 
     let mut account: DockerRegistryAccount = self.account.into();
 
     if account.domain.is_empty() {
-      return Err(anyhow!("domain cannot be empty string.").into());
+      return Err(
+        anyhow!("Domain cannot be empty string.")
+          .status_code(StatusCode::BAD_REQUEST),
+      );
     }
 
     if account.username.is_empty() {
-      return Err(anyhow!("username cannot be empty string.").into());
+      return Err(
+        anyhow!("Username cannot be empty string.")
+          .status_code(StatusCode::BAD_REQUEST),
+      );
     }
 
     let mut update = make_update(
@@ -243,15 +298,15 @@ impl Resolve<WriteArgs> for CreateDockerRegistryAccount {
       .insert_one(&account)
       .await
       .context(
-        "failed to create docker registry account account on db",
+        "Failed to create docker registry account account on db",
       )?
       .inserted_id
       .as_object_id()
-      .context("inserted id is not ObjectId")?
+      .context("Inserted id is not ObjectId")?
       .to_string();
 
     update.push_simple_log(
-      "create docker registry account",
+      "Create docker registry account",
       format!(
         "Created docker registry account account for {} with username {}",
         account.domain, account.username
@@ -263,7 +318,7 @@ impl Resolve<WriteArgs> for CreateDockerRegistryAccount {
     add_update(update)
       .await
       .inspect_err(|e| {
-        error!("failed to add update for create docker registry account | {e:#}")
+        error!("Failed to add update for create docker registry account | {e:#}")
       })
       .ok();
 
@@ -272,14 +327,24 @@ impl Resolve<WriteArgs> for CreateDockerRegistryAccount {
 }
 
 impl Resolve<WriteArgs> for UpdateDockerRegistryAccount {
+  #[instrument(
+    "UpdateDockerRegistryAccount",
+    skip_all,
+    fields(
+      operator = user.id,
+      id = self.id,
+      domain = self.account.domain,
+      username = self.account.username,
+    )
+  )]
   async fn resolve(
     mut self,
     WriteArgs { user }: &WriteArgs,
   ) -> serror::Result<UpdateDockerRegistryAccountResponse> {
     if !user.admin {
       return Err(
-        anyhow!("only admins can update docker registry accounts")
-          .into(),
+        anyhow!("Only admins can update docker registry accounts")
+          .status_code(StatusCode::FORBIDDEN),
       );
     }
 
@@ -288,9 +353,9 @@ impl Resolve<WriteArgs> for UpdateDockerRegistryAccount {
     {
       return Err(
         anyhow!(
-          "cannot update docker registry account with empty domain"
+          "Cannot update docker registry account with empty domain"
         )
-        .into(),
+        .status_code(StatusCode::BAD_REQUEST),
       );
     }
 
@@ -299,9 +364,9 @@ impl Resolve<WriteArgs> for UpdateDockerRegistryAccount {
     {
       return Err(
         anyhow!(
-          "cannot update docker registry account with empty username"
+          "Cannot update docker registry account with empty username"
         )
-        .into(),
+        .status_code(StatusCode::BAD_REQUEST),
       );
     }
 
@@ -314,7 +379,7 @@ impl Resolve<WriteArgs> for UpdateDockerRegistryAccount {
     );
 
     let account = to_document(&self.account).context(
-      "failed to serialize partial docker registry account account to bson",
+      "Failed to serialize partial docker registry account account to bson",
     )?;
 
     let db = db_client();
@@ -326,19 +391,19 @@ impl Resolve<WriteArgs> for UpdateDockerRegistryAccount {
     )
     .await
     .context(
-      "failed to update docker registry account account on db",
+      "Failed to update docker registry account account on db",
     )?;
 
     let Some(account) =
       find_one_by_id(&db.registry_accounts, &self.id)
         .await
-        .context("failed to query db for registry accounts")?
+        .context("Failed to query db for registry accounts")?
     else {
-      return Err(anyhow!("no account found with given id").into());
+      return Err(anyhow!("No account found with given id").into());
     };
 
     update.push_simple_log(
-      "update docker registry account",
+      "Update docker registry account",
       format!(
         "Updated docker registry account account for {} with username {}",
         account.domain, account.username
@@ -350,7 +415,7 @@ impl Resolve<WriteArgs> for UpdateDockerRegistryAccount {
     add_update(update)
       .await
       .inspect_err(|e| {
-        error!("failed to add update for update docker registry account | {e:#}")
+        error!("Failed to add update for update docker registry account | {e:#}")
       })
       .ok();
 
@@ -359,14 +424,22 @@ impl Resolve<WriteArgs> for UpdateDockerRegistryAccount {
 }
 
 impl Resolve<WriteArgs> for DeleteDockerRegistryAccount {
+  #[instrument(
+    "DeleteDockerRegistryAccount",
+    skip_all,
+    fields(
+      operator = user.id,
+      id = self.id,
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
   ) -> serror::Result<DeleteDockerRegistryAccountResponse> {
     if !user.admin {
       return Err(
-        anyhow!("only admins can delete docker registry accounts")
-          .into(),
+        anyhow!("Only admins can delete docker registry accounts")
+          .status_code(StatusCode::FORBIDDEN),
       );
     }
 
@@ -380,16 +453,19 @@ impl Resolve<WriteArgs> for DeleteDockerRegistryAccount {
     let Some(account) =
       find_one_by_id(&db.registry_accounts, &self.id)
         .await
-        .context("failed to query db for git accounts")?
+        .context("Failed to query db for git accounts")?
     else {
-      return Err(anyhow!("no account found with given id").into());
+      return Err(
+        anyhow!("No account found with given id")
+          .status_code(StatusCode::BAD_REQUEST),
+      );
     };
     delete_one_by_id(&db.registry_accounts, &self.id, None)
       .await
-      .context("failed to delete registry account on db")?;
+      .context("Failed to delete registry account on db")?;
 
     update.push_simple_log(
-      "delete registry account",
+      "Delete registry account",
       format!(
         "Deleted registry account for {} with username {}",
         account.domain, account.username
@@ -401,7 +477,7 @@ impl Resolve<WriteArgs> for DeleteDockerRegistryAccount {
     add_update(update)
       .await
       .inspect_err(|e| {
-        error!("failed to add update for delete docker registry account | {e:#}")
+        error!("Failed to add update for delete docker registry account | {e:#}")
       })
       .ok();
 

bin/core/src/api/write/repo.rs

@@ -32,7 +32,15 @@ use crate::{
 use super::WriteArgs;
 
 impl Resolve<WriteArgs> for CreateRepo {
-  #[instrument(name = "CreateRepo", skip(user))]
+  #[instrument(
+    "CreateRepo",
+    skip_all,
+    fields(
+      operator = user.id,
+      repo = self.name,
+      config = serde_json::to_string(&self.config).unwrap(),
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -43,7 +51,15 @@ impl Resolve<WriteArgs> for CreateRepo {
 }
 
 impl Resolve<WriteArgs> for CopyRepo {
-  #[instrument(name = "CopyRepo", skip(user))]
+  #[instrument(
+    "CopyRepo",
+    skip_all,
+    fields(
+      operator = user.id,
+      repo = self.name,
+      copy_repo = self.id,
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -60,14 +76,32 @@ impl Resolve<WriteArgs> for CopyRepo {
 }
 
 impl Resolve<WriteArgs> for DeleteRepo {
-  #[instrument(name = "DeleteRepo", skip(args))]
-  async fn resolve(self, args: &WriteArgs) -> serror::Result<Repo> {
-    Ok(resource::delete::<Repo>(&self.id, args).await?)
+  #[instrument(
+    "DeleteRepo",
+    skip_all,
+    fields(
+      operator = user.id,
+      repo = self.id,
+    )
+  )]
+  async fn resolve(
+    self,
+    WriteArgs { user }: &WriteArgs,
+  ) -> serror::Result<Repo> {
+    Ok(resource::delete::<Repo>(&self.id, user).await?)
   }
 }
 
 impl Resolve<WriteArgs> for UpdateRepo {
-  #[instrument(name = "UpdateRepo", skip(user))]
+  #[instrument(
+    "UpdateRepo",
+    skip_all,
+    fields(
+      operator = user.id,
+      repo = self.id,
+      update = serde_json::to_string(&self.config).unwrap()
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -77,7 +111,15 @@ impl Resolve<WriteArgs> for UpdateRepo {
 }
 
 impl Resolve<WriteArgs> for RenameRepo {
-  #[instrument(name = "RenameRepo", skip(user))]
+  #[instrument(
+    "RenameRepo",
+    skip_all,
+    fields(
+      operator = user.id,
+      repo = self.id,
+      new_name = self.name
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -154,11 +196,6 @@ impl Resolve<WriteArgs> for RenameRepo {
 }
 
 impl Resolve<WriteArgs> for RefreshRepoCache {
-  #[instrument(
-    name = "RefreshRepoCache",
-    level = "debug",
-    skip(user)
-  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,

bin/core/src/api/write/resource.rs

@@ -1,4 +1,5 @@
 use anyhow::anyhow;
+use derive_variants::ExtractVariant as _;
 use komodo_client::{
   api::write::{UpdateResourceMeta, UpdateResourceMetaResponse},
   entities::{
@@ -7,14 +8,27 @@ use komodo_client::{
     repo::Repo, server::Server, stack::Stack, sync::ResourceSync,
   },
 };
+use reqwest::StatusCode;
 use resolver_api::Resolve;
+use serror::AddStatusCodeError;
 
 use crate::resource::{self, ResourceMetaUpdate};
 
 use super::WriteArgs;
 
 impl Resolve<WriteArgs> for UpdateResourceMeta {
-  #[instrument(name = "UpdateResourceMeta", skip(args))]
+  #[instrument(
+    "UpdateResourceMeta",
+    skip_all,
+    fields(
+      operator = args.user.id,
+      resource_type = self.target.extract_variant().to_string(),
+      resource_id = self.target.extract_variant_id().1,
+      description = self.description,
+      template = self.template,
+      tags = format!("{:?}", self.tags),
+    )
+  )]
   async fn resolve(
     self,
     args: &WriteArgs,
@@ -28,7 +42,7 @@ impl Resolve<WriteArgs> for UpdateResourceMeta {
       ResourceTarget::System(_) => {
         return Err(
           anyhow!("cannot update meta of System resource target")
-            .into(),
+            .status_code(StatusCode::BAD_REQUEST),
         );
       }
       ResourceTarget::Server(id) => {

bin/core/src/api/write/server.rs

@@ -1,5 +1,6 @@
 use anyhow::Context;
 use formatting::{bold, format_serror};
+use futures_util::{StreamExt, stream::FuturesUnordered};
 use komodo_client::{
   api::write::*,
   entities::{
@@ -16,6 +17,7 @@ use resolver_api::Resolve;
 use crate::{
   helpers::{
     periphery_client,
+    query::get_all_tags,
     update::{add_update, make_update, update_update},
   },
   permission::get_check_permissions,
@@ -25,7 +27,15 @@ use crate::{
 use super::WriteArgs;
 
 impl Resolve<WriteArgs> for CreateServer {
-  #[instrument(name = "CreateServer", skip(user))]
+  #[instrument(
+    "CreateServer",
+    skip_all,
+    fields(
+      operator = user.id,
+      server = self.name,
+      config = serde_json::to_string(&self.config).unwrap()
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -44,7 +54,15 @@ impl Resolve<WriteArgs> for CreateServer {
 }
 
 impl Resolve<WriteArgs> for CopyServer {
-  #[instrument(name = "CopyServer", skip(user))]
+  #[instrument(
+    "CopyServer",
+    skip_all,
+    fields(
+      operator = user.id,
+      server = self.name,
+      copy_server = self.id,
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -70,14 +88,32 @@ impl Resolve<WriteArgs> for CopyServer {
 }
 
 impl Resolve<WriteArgs> for DeleteServer {
-  #[instrument(name = "DeleteServer", skip(args))]
-  async fn resolve(self, args: &WriteArgs) -> serror::Result<Server> {
-    Ok(resource::delete::<Server>(&self.id, args).await?)
+  #[instrument(
+    "DeleteServer",
+    skip_all,
+    fields(
+      operator = user.id,
+      server = self.id,
+    )
+  )]
+  async fn resolve(
+    self,
+    WriteArgs { user }: &WriteArgs,
+  ) -> serror::Result<Server> {
+    Ok(resource::delete::<Server>(&self.id, user).await?)
   }
 }
 
 impl Resolve<WriteArgs> for UpdateServer {
-  #[instrument(name = "UpdateServer", skip(user))]
+  #[instrument(
+    "UpdateServer",
+    skip_all,
+    fields(
+      operator = user.id,
+      server = self.id,
+      update = serde_json::to_string(&self.config).unwrap(),
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -87,7 +123,15 @@ impl Resolve<WriteArgs> for UpdateServer {
 }
 
 impl Resolve<WriteArgs> for RenameServer {
-  #[instrument(name = "RenameServer", skip(user))]
+  #[instrument(
+    "RenameServer",
+    skip_all,
+    fields(
+      operator = user.id,
+      server = self.id,
+      new_name = self.name,
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -97,7 +141,15 @@ impl Resolve<WriteArgs> for RenameServer {
 }
 
 impl Resolve<WriteArgs> for CreateNetwork {
-  #[instrument(name = "CreateNetwork", skip(user))]
+  #[instrument(
+    "CreateNetwork",
+    skip_all,
+    fields(
+      operator = user.id,
+      server = self.server,
+      network = self.name
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -138,7 +190,17 @@ impl Resolve<WriteArgs> for CreateNetwork {
 }
 
 impl Resolve<WriteArgs> for CreateTerminal {
-  #[instrument(name = "CreateTerminal", skip(user))]
+  #[instrument(
+    "CreateTerminal",
+    skip_all,
+    fields(
+      operator = user.id,
+      server = self.server,
+      terminal = self.name,
+      command = self.command,
+      recreate = format!("{:?}", self.recreate),
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -166,7 +228,15 @@ impl Resolve<WriteArgs> for CreateTerminal {
 }
 
 impl Resolve<WriteArgs> for DeleteTerminal {
-  #[instrument(name = "DeleteTerminal", skip(user))]
+  #[instrument(
+    "DeleteTerminal",
+    skip_all,
+    fields(
+      operator = user.id,
+      server = self.server,
+      terminal = self.terminal,
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -192,7 +262,14 @@ impl Resolve<WriteArgs> for DeleteTerminal {
 }
 
 impl Resolve<WriteArgs> for DeleteAllTerminals {
-  #[instrument(name = "DeleteAllTerminals", skip(user))]
+  #[instrument(
+    "DeleteAllTerminals",
+    skip_all,
+    fields(
+      operator = user.id,
+      server = self.server,
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -217,8 +294,69 @@ impl Resolve<WriteArgs> for DeleteAllTerminals {
 
 //
 
+impl Resolve<WriteArgs> for BatchDeleteAllTerminals {
+  #[instrument(
+    "BatchDeleteAllTerminals",
+    skip_all,
+    fields(
+      operator = user.id,
+      query = format!("{:?}", self.query),
+    )
+  )]
+  async fn resolve(
+    self,
+    WriteArgs { user }: &WriteArgs,
+  ) -> Result<Self::Response, Self::Error> {
+    let all_tags = if self.query.tags.is_empty() {
+      vec![]
+    } else {
+      get_all_tags(None).await?
+    };
+
+    resource::list_full_for_user::<Server>(
+      self.query, user, &all_tags,
+    )
+    .await?
+    .into_iter()
+    .map(|server| async move {
+      let res = async {
+        let periphery = periphery_client(&server).await?;
+
+        periphery
+          .request(api::terminal::DeleteAllTerminals {})
+          .await
+          .context("Failed to delete all terminals on Periphery")?;
+
+        anyhow::Ok(())
+      }
+      .await;
+      if let Err(e) = res {
+        warn!(
+          "Failed to delete all terminals on {} ({}) | {e:#}",
+          server.name, server.id
+        )
+      }
+    })
+    .collect::<FuturesUnordered<_>>()
+    .collect::<Vec<_>>()
+    .await;
+
+    Ok(NoData {})
+  }
+}
+
+//
+
 impl Resolve<WriteArgs> for UpdateServerPublicKey {
-  #[instrument(name = "UpdateServerPublicKey", skip(args))]
+  #[instrument(
+    "UpdateServerPublicKey",
+    skip_all,
+    fields(
+      operator = args.user.id,
+      server = self.server,
+      public_key = self.public_key,
+    )
+  )]
   async fn resolve(
     self,
     args: &WriteArgs,
@@ -249,7 +387,14 @@ impl Resolve<WriteArgs> for UpdateServerPublicKey {
 //
 
 impl Resolve<WriteArgs> for RotateServerKeys {
-  #[instrument(name = "RotateServerPrivateKey", skip(args))]
+  #[instrument(
+    "RotateServerKeys",
+    skip_all,
+    fields(
+      operator = args.user.id,
+      server = self.server,
+    )
+  )]
   async fn resolve(
     self,
     args: &WriteArgs,

bin/core/src/api/write/service_user.rs

@@ -19,7 +19,15 @@ use crate::{api::user::UserArgs, state::db_client};
 use super::WriteArgs;
 
 impl Resolve<WriteArgs> for CreateServiceUser {
-  #[instrument(name = "CreateServiceUser", skip(user))]
+  #[instrument(
+    "CreateServiceUser",
+    skip_all,
+    fields(
+      operator = user.id,
+      username = self.username,
+      description = self.description,
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -63,7 +71,15 @@ impl Resolve<WriteArgs> for CreateServiceUser {
 }
 
 impl Resolve<WriteArgs> for UpdateServiceUserDescription {
-  #[instrument(name = "UpdateServiceUserDescription", skip(user))]
+  #[instrument(
+    "UpdateServiceUserDescription",
+    skip_all,
+    fields(
+      operator = user.id,
+      username = self.username,
+      description = self.description,
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -99,7 +115,16 @@ impl Resolve<WriteArgs> for UpdateServiceUserDescription {
 }
 
 impl Resolve<WriteArgs> for CreateApiKeyForServiceUser {
-  #[instrument(name = "CreateApiKeyForServiceUser", skip(user))]
+  #[instrument(
+    "CreateApiKeyForServiceUser",
+    skip_all,
+    fields(
+      operator = user.id,
+      service_user = self.user_id,
+      name = self.name,
+      expires = self.expires,
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -125,7 +150,14 @@ impl Resolve<WriteArgs> for CreateApiKeyForServiceUser {
 }
 
 impl Resolve<WriteArgs> for DeleteApiKeyForServiceUser {
-  #[instrument(name = "DeleteApiKeyForServiceUser", skip(user))]
+  #[instrument(
+    "DeleteApiKeyForServiceUser",
+    skip_all,
+    fields(
+      operator = user.id,
+      key = self.key,
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,

bin/core/src/api/write/stack.rs

@@ -42,7 +42,15 @@ use crate::{
 use super::WriteArgs;
 
 impl Resolve<WriteArgs> for CreateStack {
-  #[instrument(name = "CreateStack", skip(user))]
+  #[instrument(
+    "CreateStack",
+    skip_all,
+    fields(
+      operator = user.id,
+      stack = self.name,
+      config = serde_json::to_string(&self.config).unwrap(),
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -53,7 +61,15 @@ impl Resolve<WriteArgs> for CreateStack {
 }
 
 impl Resolve<WriteArgs> for CopyStack {
-  #[instrument(name = "CopyStack", skip(user))]
+  #[instrument(
+    "CopyStack",
+    skip_all,
+    fields(
+      operator = user.id,
+      stack = self.name,
+      copy_stack = self.id,
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -71,14 +87,32 @@ impl Resolve<WriteArgs> for CopyStack {
 }
 
 impl Resolve<WriteArgs> for DeleteStack {
-  #[instrument(name = "DeleteStack", skip(args))]
-  async fn resolve(self, args: &WriteArgs) -> serror::Result<Stack> {
-    Ok(resource::delete::<Stack>(&self.id, args).await?)
+  #[instrument(
+    "DeleteStack",
+    skip_all,
+    fields(
+      operator = user.id,
+      stack = self.id,
+    )
+  )]
+  async fn resolve(
+    self,
+    WriteArgs { user }: &WriteArgs,
+  ) -> serror::Result<Stack> {
+    Ok(resource::delete::<Stack>(&self.id, user).await?)
   }
 }
 
 impl Resolve<WriteArgs> for UpdateStack {
-  #[instrument(name = "UpdateStack", skip(user))]
+  #[instrument(
+    "UpdateStack",
+    skip_all,
+    fields(
+      operator = user.id,
+      stack = self.id,
+      update = serde_json::to_string(&self.config).unwrap(),
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -88,7 +122,15 @@ impl Resolve<WriteArgs> for UpdateStack {
 }
 
 impl Resolve<WriteArgs> for RenameStack {
-  #[instrument(name = "RenameStack", skip(user))]
+  #[instrument(
+    "RenameStack",
+    skip_all,
+    fields(
+      operator = user.id,
+      stack = self.id,
+      new_name = self.name
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -98,7 +140,15 @@ impl Resolve<WriteArgs> for RenameStack {
 }
 
 impl Resolve<WriteArgs> for WriteStackFileContents {
-  #[instrument(name = "WriteStackFileContents", skip(user))]
+  #[instrument(
+    "WriteStackFileContents",
+    skip_all,
+    fields(
+      operator = user.id,
+      stack = self.stack,
+      path = self.file_path,
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -147,6 +197,7 @@ impl Resolve<WriteArgs> for WriteStackFileContents {
   }
 }
 
+#[instrument("WriteStackFileContentsOnHost", skip_all)]
 async fn write_stack_file_contents_on_host(
   stack: Stack,
   file_path: String,
@@ -219,6 +270,7 @@ async fn write_stack_file_contents_on_host(
   Ok(update)
 }
 
+#[instrument("WriteStackFileContentsGit", skip_all)]
 async fn write_stack_file_contents_git(
   mut stack: Stack,
   file_path: &str,
@@ -360,11 +412,6 @@ async fn write_stack_file_contents_git(
 }
 
 impl Resolve<WriteArgs> for RefreshStackCache {
-  #[instrument(
-    name = "RefreshStackCache",
-    level = "debug",
-    skip(user)
-  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,

bin/core/src/api/write/sync.rs

@@ -33,6 +33,7 @@ use komodo_client::{
   },
 };
 use resolver_api::Resolve;
+use tracing::Instrument;
 
 use crate::{
   alert::send_alerts,
@@ -56,7 +57,15 @@ use crate::{
 use super::WriteArgs;
 
 impl Resolve<WriteArgs> for CreateResourceSync {
-  #[instrument(name = "CreateResourceSync", skip(user))]
+  #[instrument(
+    "CreateResourceSync",
+    skip_all,
+    fields(
+      operator = user.id,
+      sync = self.name,
+      config = serde_json::to_string(&self.config).unwrap(),
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -72,7 +81,15 @@ impl Resolve<WriteArgs> for CreateResourceSync {
 }
 
 impl Resolve<WriteArgs> for CopyResourceSync {
-  #[instrument(name = "CopyResourceSync", skip(user))]
+  #[instrument(
+    "CopyResourceSync",
+    skip_all,
+    fields(
+      operator = user.id,
+      sync = self.name,
+      copy_sync = self.id,
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -95,17 +112,32 @@ impl Resolve<WriteArgs> for CopyResourceSync {
 }
 
 impl Resolve<WriteArgs> for DeleteResourceSync {
-  #[instrument(name = "DeleteResourceSync", skip(args))]
+  #[instrument(
+    "DeleteResourceSync",
+    skip_all,
+    fields(
+      operator = user.id,
+      sync = self.id,
+    )
+  )]
   async fn resolve(
     self,
-    args: &WriteArgs,
+    WriteArgs { user }: &WriteArgs,
   ) -> serror::Result<ResourceSync> {
-    Ok(resource::delete::<ResourceSync>(&self.id, args).await?)
+    Ok(resource::delete::<ResourceSync>(&self.id, user).await?)
   }
 }
 
 impl Resolve<WriteArgs> for UpdateResourceSync {
-  #[instrument(name = "UpdateResourceSync", skip(user))]
+  #[instrument(
+    "UpdateResourceSync",
+    skip_all,
+    fields(
+      operator = user.id,
+      sync = self.id,
+      update = serde_json::to_string(&self.config).unwrap(),
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -118,7 +150,15 @@ impl Resolve<WriteArgs> for UpdateResourceSync {
 }
 
 impl Resolve<WriteArgs> for RenameResourceSync {
-  #[instrument(name = "RenameResourceSync", skip(user))]
+  #[instrument(
+    "RenameResourceSync",
+    skip_all,
+    fields(
+      operator = user.id,
+      sync = self.id,
+      new_name = self.name
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -131,7 +171,16 @@ impl Resolve<WriteArgs> for RenameResourceSync {
 }
 
 impl Resolve<WriteArgs> for WriteSyncFileContents {
-  #[instrument(name = "WriteSyncFileContents", skip(args))]
+  #[instrument(
+    "WriteSyncFileContents",
+    skip_all,
+    fields(
+      operator = args.user.id,
+      sync = self.sync,
+      resource_path = self.resource_path,
+      file_path = self.file_path,
+    )
+  )]
   async fn resolve(self, args: &WriteArgs) -> serror::Result<Update> {
     let sync = get_check_permissions::<ResourceSync>(
       &self.sync,
@@ -176,6 +225,7 @@ impl Resolve<WriteArgs> for WriteSyncFileContents {
   }
 }
 
+#[instrument("WriteSyncFileContentsOnHost", skip_all)]
 async fn write_sync_file_contents_on_host(
   req: WriteSyncFileContents,
   args: &WriteArgs,
@@ -238,6 +288,7 @@ async fn write_sync_file_contents_on_host(
   Ok(update)
 }
 
+#[instrument("WriteSyncFileContentsGit", skip_all)]
 async fn write_sync_file_contents_git(
   req: WriteSyncFileContents,
   args: &WriteArgs,
@@ -389,7 +440,14 @@ async fn write_sync_file_contents_git(
 }
 
 impl Resolve<WriteArgs> for CommitSync {
-  #[instrument(name = "CommitSync", skip(args))]
+  #[instrument(
+    "CommitSync",
+    skip_all,
+    fields(
+      operator = args.user.id,
+      sync = self.sync,
+    )
+  )]
   async fn resolve(self, args: &WriteArgs) -> serror::Result<Update> {
     let WriteArgs { user } = args;
 
@@ -476,7 +534,9 @@ impl Resolve<WriteArgs> for CommitSync {
         .sync_directory
         .join(to_path_compatible_name(&sync.name))
         .join(&resource_path);
+      let span = info_span!("CommitSyncOnHost");
       if let Err(e) = secret_file::write_async(&file_path, &res.toml)
+        .instrument(span)
         .await
         .with_context(|| {
           format!("Failed to write resource file to {file_path:?}",)
@@ -569,6 +629,7 @@ impl Resolve<WriteArgs> for CommitSync {
   }
 }
 
+#[instrument("CommitSyncGit", skip_all)]
 async fn commit_git_sync(
   mut args: RepoExecutionArgs,
   resource_path: &Path,
@@ -613,11 +674,6 @@ async fn commit_git_sync(
 }
 
 impl Resolve<WriteArgs> for RefreshResourceSyncPending {
-  #[instrument(
-    name = "RefreshResourceSyncPending",
-    level = "debug",
-    skip(user)
-  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,

bin/core/src/api/write/tag.rs

@@ -27,7 +27,15 @@ use crate::{
 use super::WriteArgs;
 
 impl Resolve<WriteArgs> for CreateTag {
-  #[instrument(name = "CreateTag", skip(user))]
+  #[instrument(
+    "CreateTag",
+    skip_all,
+    fields(
+      operator = user.id,
+      tag = self.name,
+      color = format!("{:?}", self.color),
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -68,7 +76,15 @@ impl Resolve<WriteArgs> for CreateTag {
 }
 
 impl Resolve<WriteArgs> for RenameTag {
-  #[instrument(name = "RenameTag", skip(user))]
+  #[instrument(
+    "RenameTag",
+    skip_all,
+    fields(
+      operator = user.id,
+      tag = self.id,
+      new_name = self.name,
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -93,7 +109,15 @@ impl Resolve<WriteArgs> for RenameTag {
 }
 
 impl Resolve<WriteArgs> for UpdateTagColor {
-  #[instrument(name = "UpdateTagColor", skip(user))]
+  #[instrument(
+    "UpdateTagColor",
+    skip_all,
+    fields(
+      operator = user.id,
+      tag = self.tag,
+      color = format!("{:?}", self.color),
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -114,7 +138,14 @@ impl Resolve<WriteArgs> for UpdateTagColor {
 }
 
 impl Resolve<WriteArgs> for DeleteTag {
-  #[instrument(name = "DeleteTag", skip(user))]
+  #[instrument(
+    "DeleteTag",
+    skip_all,
+    fields(
+      operator = user.id,
+      tag_id = self.id,
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,

bin/core/src/api/write/user.rs

@@ -24,7 +24,14 @@ use super::WriteArgs;
 //
 
 impl Resolve<WriteArgs> for CreateLocalUser {
-  #[instrument(name = "CreateLocalUser", skip(admin, self), fields(admin_id = admin.id, username = self.username))]
+  #[instrument(
+    "CreateLocalUser",
+    skip_all,
+    fields(
+      admin_id = admin.id,
+      username = self.username
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user: admin }: &WriteArgs,
@@ -101,7 +108,14 @@ impl Resolve<WriteArgs> for CreateLocalUser {
 //
 
 impl Resolve<WriteArgs> for UpdateUserUsername {
-  #[instrument(name = "UpdateUserUsername", skip(user), fields(user_id = user.id))]
+  #[instrument(
+    "UpdateUserUsername",
+    skip_all,
+    fields(
+      operator = user.id,
+      new_username = self.username,
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -152,7 +166,11 @@ impl Resolve<WriteArgs> for UpdateUserUsername {
 //
 
 impl Resolve<WriteArgs> for UpdateUserPassword {
-  #[instrument(name = "UpdateUserPassword", skip(user, self), fields(user_id = user.id))]
+  #[instrument(
+    "UpdateUserPassword",
+    skip_all,
+    fields(operator = user.id)
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -175,7 +193,14 @@ impl Resolve<WriteArgs> for UpdateUserPassword {
 //
 
 impl Resolve<WriteArgs> for DeleteUser {
-  #[instrument(name = "DeleteUser", skip(admin), fields(user = self.user))]
+  #[instrument(
+    "DeleteUser",
+    skip_all,
+    fields(
+      admin_id = admin.id,
+      user_to_delete = self.user
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user: admin }: &WriteArgs,

bin/core/src/api/write/user_group.rs

@@ -19,7 +19,14 @@ use crate::state::db_client;
 use super::WriteArgs;
 
 impl Resolve<WriteArgs> for CreateUserGroup {
-  #[instrument(name = "CreateUserGroup", skip(admin), fields(admin = admin.username))]
+  #[instrument(
+    "CreateUserGroup",
+    skip_all,
+    fields(
+      operator = admin.id,
+      group = self.name,
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user: admin }: &WriteArgs,
@@ -57,7 +64,15 @@ impl Resolve<WriteArgs> for CreateUserGroup {
 }
 
 impl Resolve<WriteArgs> for RenameUserGroup {
-  #[instrument(name = "RenameUserGroup", skip(admin), fields(admin = admin.username))]
+  #[instrument(
+    "RenameUserGroup",
+    skip_all,
+    fields(
+      operator = admin.id,
+      group = self.id,
+      new_name = self.name,
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user: admin }: &WriteArgs,
@@ -86,7 +101,14 @@ impl Resolve<WriteArgs> for RenameUserGroup {
 }
 
 impl Resolve<WriteArgs> for DeleteUserGroup {
-  #[instrument(name = "DeleteUserGroup", skip(admin), fields(admin = admin.username))]
+  #[instrument(
+    "DeleteUserGroup",
+    skip_all,
+    fields(
+      operator = admin.id,
+      group = self.id,
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user: admin }: &WriteArgs,
@@ -122,7 +144,15 @@ impl Resolve<WriteArgs> for DeleteUserGroup {
 }
 
 impl Resolve<WriteArgs> for AddUserToUserGroup {
-  #[instrument(name = "AddUserToUserGroup", skip(admin), fields(admin = admin.username))]
+  #[instrument(
+    "AddUserToUserGroup",
+    skip_all,
+    fields(
+      operator = admin.id,
+      group = self.user_group,
+      user = self.user,
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user: admin }: &WriteArgs,
@@ -169,7 +199,15 @@ impl Resolve<WriteArgs> for AddUserToUserGroup {
 }
 
 impl Resolve<WriteArgs> for RemoveUserFromUserGroup {
-  #[instrument(name = "RemoveUserFromUserGroup", skip(admin), fields(admin = admin.username))]
+  #[instrument(
+    "RemoveUserFromUserGroup",
+    skip_all,
+    fields(
+      operator = admin.id,
+      group = self.user_group,
+      user = self.user,
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user: admin }: &WriteArgs,
@@ -216,7 +254,15 @@ impl Resolve<WriteArgs> for RemoveUserFromUserGroup {
 }
 
 impl Resolve<WriteArgs> for SetUsersInUserGroup {
-  #[instrument(name = "SetUsersInUserGroup", skip(admin), fields(admin = admin.username))]
+  #[instrument(
+    "SetUsersInUserGroup",
+    skip_all,
+    fields(
+      operator = admin.id,
+      group = self.user_group,
+      users = format!("{:?}", self.users)
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user: admin }: &WriteArgs,
@@ -266,7 +312,15 @@ impl Resolve<WriteArgs> for SetUsersInUserGroup {
 }
 
 impl Resolve<WriteArgs> for SetEveryoneUserGroup {
-  #[instrument(name = "SetEveryoneUserGroup", skip(admin), fields(admin = admin.username))]
+  #[instrument(
+    "SetEveryoneUserGroup",
+    skip_all,
+    fields(
+      operator = admin.id,
+      group = self.user_group,
+      everyone = self.everyone,
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user: admin }: &WriteArgs,

bin/core/src/api/write/variable.rs

@@ -19,7 +19,16 @@ use crate::{
 use super::WriteArgs;
 
 impl Resolve<WriteArgs> for CreateVariable {
-  #[instrument(name = "CreateVariable", skip(user, self), fields(name = &self.name))]
+  #[instrument(
+    "CreateVariable",
+    skip_all,
+    fields(
+      operator = user.id,
+      variable = self.name,
+      description = self.description,
+      is_secret = self.is_secret,
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -68,7 +77,14 @@ impl Resolve<WriteArgs> for CreateVariable {
 }
 
 impl Resolve<WriteArgs> for UpdateVariableValue {
-  #[instrument(name = "UpdateVariableValue", skip(user, self), fields(name = &self.name))]
+  #[instrument(
+    "UpdateVariableValue",
+    skip_all,
+    fields(
+      operator = user.id,
+      variable = self.name,
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -125,7 +141,15 @@ impl Resolve<WriteArgs> for UpdateVariableValue {
 }
 
 impl Resolve<WriteArgs> for UpdateVariableDescription {
-  #[instrument(name = "UpdateVariableDescription", skip(user))]
+  #[instrument(
+    "UpdateVariableDescription",
+    skip_all,
+    fields(
+      operator = user.id,
+      variable = self.name,
+      description = self.description,
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -149,7 +173,15 @@ impl Resolve<WriteArgs> for UpdateVariableDescription {
 }
 
 impl Resolve<WriteArgs> for UpdateVariableIsSecret {
-  #[instrument(name = "UpdateVariableIsSecret", skip(user))]
+  #[instrument(
+    "UpdateVariableIsSecret",
+    skip_all,
+    fields(
+      operator = user.id,
+      variable = self.name,
+      is_secret = self.is_secret,
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,
@@ -173,6 +205,14 @@ impl Resolve<WriteArgs> for UpdateVariableIsSecret {
 }
 
 impl Resolve<WriteArgs> for DeleteVariable {
+  #[instrument(
+    "DeleteVariable",
+    skip_all,
+    fields(
+      operator = user.id,
+      variable = self.name,
+    )
+  )]
   async fn resolve(
     self,
     WriteArgs { user }: &WriteArgs,

bin/core/src/auth/github/client.rs

@@ -1,17 +1,15 @@
 use std::sync::OnceLock;
 
 use anyhow::{Context, anyhow};
-use komodo_client::entities::config::core::{
-  CoreConfig, OauthCredentials,
+use komodo_client::entities::{
+  config::core::{CoreConfig, OauthCredentials},
+  random_string,
 };
 use reqwest::StatusCode;
 use serde::{Deserialize, Serialize, de::DeserializeOwned};
 use tokio::sync::Mutex;
 
-use crate::{
-  auth::STATE_PREFIX_LENGTH, config::core_config,
-  helpers::random_string,
-};
+use crate::{auth::STATE_PREFIX_LENGTH, config::core_config};
 
 pub fn github_oauth_client() -> &'static Option<GithubOauthClient> {
   static GITHUB_OAUTH_CLIENT: OnceLock<Option<GithubOauthClient>> =
@@ -76,7 +74,6 @@ impl GithubOauthClient {
     .into()
   }
 
-  #[instrument(level = "debug", skip(self))]
   pub async fn get_login_redirect_url(
     &self,
     redirect: Option<String>,
@@ -95,7 +92,6 @@ impl GithubOauthClient {
     redirect_url
   }
 
-  #[instrument(level = "debug", skip(self))]
   pub async fn check_state(&self, state: &str) -> bool {
     let mut contained = false;
     self.states.lock().await.retain(|s| {
@@ -109,7 +105,6 @@ impl GithubOauthClient {
     contained
   }
 
-  #[instrument(level = "debug", skip(self))]
   pub async fn get_access_token(
     &self,
     code: &str,
@@ -130,7 +125,6 @@ impl GithubOauthClient {
       .context("failed to get github access token using code")
   }
 
-  #[instrument(level = "debug", skip(self))]
   pub async fn get_github_user(
     &self,
     token: &str,
@@ -141,7 +135,6 @@ impl GithubOauthClient {
       .context("failed to get github user using access token")
   }
 
-  #[instrument(level = "debug", skip(self))]
   async fn get<R: DeserializeOwned>(
     &self,
     endpoint: &str,

bin/core/src/auth/github/mod.rs

@@ -5,7 +5,7 @@ use axum::{
 use database::mongo_indexed::Document;
 use database::mungos::mongodb::bson::doc;
 use komodo_client::entities::{
-  komodo_timestamp,
+  komodo_timestamp, random_string,
   user::{User, UserConfig},
 };
 use reqwest::StatusCode;
@@ -14,7 +14,6 @@ use serror::AddStatusCode;
 
 use crate::{
   config::core_config,
-  helpers::random_string,
   state::{db_client, jwt_client},
 };
 
@@ -53,7 +52,6 @@ struct CallbackQuery {
   code: String,
 }
 
-#[instrument(name = "GithubCallback", level = "debug")]
 async fn callback(
   Query(query): Query<CallbackQuery>,
 ) -> anyhow::Result<Redirect> {

bin/core/src/auth/google/client.rs

@@ -2,17 +2,15 @@ use std::sync::OnceLock;
 
 use anyhow::{Context, anyhow};
 use jsonwebtoken::{DecodingKey, Validation, decode};
-use komodo_client::entities::config::core::{
-  CoreConfig, OauthCredentials,
+use komodo_client::entities::{
+  config::core::{CoreConfig, OauthCredentials},
+  random_string,
 };
 use reqwest::StatusCode;
 use serde::{Deserialize, de::DeserializeOwned};
 use tokio::sync::Mutex;
 
-use crate::{
-  auth::STATE_PREFIX_LENGTH, config::core_config,
-  helpers::random_string,
-};
+use crate::{auth::STATE_PREFIX_LENGTH, config::core_config};
 
 pub fn google_oauth_client() -> &'static Option<GoogleOauthClient> {
   static GOOGLE_OAUTH_CLIENT: OnceLock<Option<GoogleOauthClient>> =
@@ -85,7 +83,6 @@ impl GoogleOauthClient {
     .into()
   }
 
-  #[instrument(level = "debug", skip(self))]
   pub async fn get_login_redirect_url(
     &self,
     redirect: Option<String>,
@@ -104,7 +101,6 @@ impl GoogleOauthClient {
     redirect_url
   }
 
-  #[instrument(level = "debug", skip(self))]
   pub async fn check_state(&self, state: &str) -> bool {
     let mut contained = false;
     self.states.lock().await.retain(|s| {
@@ -118,7 +114,6 @@ impl GoogleOauthClient {
     contained
   }
 
-  #[instrument(level = "debug", skip(self))]
   pub async fn get_access_token(
     &self,
     code: &str,
@@ -139,7 +134,6 @@ impl GoogleOauthClient {
       .context("failed to get google access token using code")
   }
 
-  #[instrument(level = "debug", skip(self))]
   pub fn get_google_user(
     &self,
     id_token: &str,
@@ -156,7 +150,6 @@ impl GoogleOauthClient {
     Ok(res.claims)
   }
 
-  #[instrument(level = "debug", skip(self))]
   async fn post<R: DeserializeOwned>(
     &self,
     endpoint: &str,

bin/core/src/auth/google/mod.rs

@@ -5,14 +5,16 @@ use axum::{
 };
 use database::mongo_indexed::Document;
 use database::mungos::mongodb::bson::doc;
-use komodo_client::entities::user::{User, UserConfig};
+use komodo_client::entities::{
+  random_string,
+  user::{User, UserConfig},
+};
 use reqwest::StatusCode;
 use serde::Deserialize;
 use serror::AddStatusCode;
 
 use crate::{
   config::core_config,
-  helpers::random_string,
   state::{db_client, jwt_client},
 };
 
@@ -52,7 +54,6 @@ struct CallbackQuery {
   error: Option<String>,
 }
 
-#[instrument(name = "GoogleCallback", level = "debug")]
 async fn callback(
   Query(query): Query<CallbackQuery>,
 ) -> anyhow::Result<Redirect> {

bin/core/src/auth/jwt.rs

@@ -9,13 +9,12 @@ use jsonwebtoken::{
   DecodingKey, EncodingKey, Header, Validation, decode, encode,
 };
 use komodo_client::{
-  api::auth::JwtResponse, entities::config::core::CoreConfig,
+  api::auth::JwtResponse,
+  entities::{config::core::CoreConfig, random_string},
 };
 use serde::{Deserialize, Serialize};
 use tokio::sync::Mutex;
 
-use crate::helpers::random_string;
-
 type ExchangeTokenMap = Mutex<HashMap<String, (JwtResponse, u128)>>;
 
 #[derive(Serialize, Deserialize, Clone)]
@@ -75,7 +74,6 @@ impl JwtClient {
       .context("failed to decode token claims")
   }
 
-  #[instrument(level = "debug", skip_all)]
   pub async fn create_exchange_token(
     &self,
     jwt: JwtResponse,
@@ -91,7 +89,7 @@ impl JwtClient {
     );
     exchange_token
   }
-  #[instrument(level = "debug", skip(self))]
+
   pub async fn redeem_exchange_token(
     &self,
     exchange_token: &str,

bin/core/src/auth/local.rs

@@ -22,7 +22,7 @@ use crate::{
 };
 
 impl Resolve<AuthArgs> for SignUpLocalUser {
-  #[instrument(name = "SignUpLocalUser", skip(self))]
+  #[instrument("SignUpLocalUser", skip(self))]
   async fn resolve(
     self,
     _: &AuthArgs,
@@ -104,7 +104,6 @@ impl Resolve<AuthArgs> for SignUpLocalUser {
 }
 
 impl Resolve<AuthArgs> for LoginLocalUser {
-  #[instrument(name = "LoginLocalUser", level = "debug", skip(self))]
   async fn resolve(
     self,
     _: &AuthArgs,

bin/core/src/auth/mod.rs

@@ -31,7 +31,6 @@ struct RedirectQuery {
   redirect: Option<String>,
 }
 
-#[instrument(level = "debug")]
 pub async fn auth_request(
   headers: HeaderMap,
   mut req: Request,
@@ -44,7 +43,6 @@ pub async fn auth_request(
   Ok(next.run(req).await)
 }
 
-#[instrument(level = "debug")]
 pub async fn get_user_id_from_headers(
   headers: &HeaderMap,
 ) -> anyhow::Result<String> {
@@ -77,7 +75,6 @@ pub async fn get_user_id_from_headers(
   }
 }
 
-#[instrument(level = "debug")]
 pub async fn authenticate_check_enabled(
   headers: &HeaderMap,
 ) -> anyhow::Result<User> {
@@ -90,7 +87,6 @@ pub async fn authenticate_check_enabled(
   }
 }
 
-#[instrument(level = "debug")]
 pub async fn auth_jwt_get_user_id(
   jwt: &str,
 ) -> anyhow::Result<String> {
@@ -102,7 +98,6 @@ pub async fn auth_jwt_get_user_id(
   }
 }
 
-#[instrument(level = "debug")]
 pub async fn auth_jwt_check_enabled(
   jwt: &str,
 ) -> anyhow::Result<User> {
@@ -110,7 +105,6 @@ pub async fn auth_jwt_check_enabled(
   check_enabled(user_id).await
 }
 
-#[instrument(level = "debug")]
 pub async fn auth_api_key_get_user_id(
   key: &str,
   secret: &str,
@@ -135,7 +129,6 @@ pub async fn auth_api_key_get_user_id(
   }
 }
 
-#[instrument(level = "debug")]
 pub async fn auth_api_key_check_enabled(
   key: &str,
   secret: &str,
@@ -144,7 +137,6 @@ pub async fn auth_api_key_check_enabled(
   check_enabled(user_id).await
 }
 
-#[instrument(level = "debug")]
 async fn check_enabled(user_id: String) -> anyhow::Result<User> {
   let user = get_user(&user_id).await?;
   if user.enabled {

bin/core/src/auth/oidc/mod.rs

@@ -8,7 +8,7 @@ use client::oidc_client;
 use dashmap::DashMap;
 use database::mungos::mongodb::bson::{Document, doc};
 use komodo_client::entities::{
-  komodo_timestamp,
+  komodo_timestamp, random_string,
   user::{User, UserConfig},
 };
 use openidconnect::{
@@ -23,7 +23,6 @@ use serror::AddStatusCode;
 
 use crate::{
   config::core_config,
-  helpers::random_string,
   state::{db_client, jwt_client},
 };
 
@@ -75,7 +74,6 @@ pub fn router() -> Router {
     )
 }
 
-#[instrument(name = "OidcRedirect", level = "debug")]
 async fn login(
   Query(RedirectQuery { redirect }): Query<RedirectQuery>,
 ) -> anyhow::Result<Redirect> {
@@ -138,7 +136,6 @@ struct CallbackQuery {
   error: Option<String>,
 }
 
-#[instrument(name = "OidcCallback", level = "debug")]
 async fn callback(
   Query(query): Query<CallbackQuery>,
 ) -> anyhow::Result<Redirect> {

bin/core/src/cloud/aws/ec2.rs

@@ -57,7 +57,6 @@ impl aws_credential_types::provider::ProvideCredentials
   }
 }
 
-#[instrument]
 async fn create_ec2_client(region: String) -> Client {
   let region = Region::new(region);
   let config = aws_config::defaults(BehaviorVersion::latest())
@@ -68,7 +67,7 @@ async fn create_ec2_client(region: String) -> Client {
   Client::new(&config)
 }
 
-#[instrument]
+#[instrument("LaunchEc2Instance")]
 pub async fn launch_ec2_instance(
   name: &str,
   config: &AwsBuilderConfig,
@@ -170,7 +169,7 @@ pub async fn launch_ec2_instance(
 const MAX_TERMINATION_TRIES: usize = 5;
 const TERMINATION_WAIT_SECS: u64 = 15;
 
-#[instrument]
+#[instrument("TerminateEc2Instance")]
 pub async fn terminate_ec2_instance_with_retry(
   region: String,
   instance_id: &str,
@@ -210,7 +209,7 @@ pub async fn terminate_ec2_instance_with_retry(
   unreachable!()
 }
 
-#[instrument(skip(client))]
+#[instrument("TerminateEc2InstanceInner", skip_all)]
 async fn terminate_ec2_instance_inner(
   client: &Client,
   instance_id: &str,
@@ -229,7 +228,6 @@ async fn terminate_ec2_instance_inner(
 }
 
 /// Automatically retries 5 times, waiting 2 sec in between
-#[instrument(level = "debug")]
 async fn get_ec2_instance_status(
   client: &Client,
   instance_id: &str,
@@ -261,7 +259,6 @@ async fn get_ec2_instance_status(
   }
 }
 
-#[instrument(level = "debug")]
 async fn get_ec2_instance_state_name(
   client: &Client,
   instance_id: &str,
@@ -281,7 +278,6 @@ async fn get_ec2_instance_state_name(
 }
 
 /// Automatically retries 5 times, waiting 2 sec in between
-#[instrument(level = "debug")]
 async fn get_ec2_instance_public_ip(
   client: &Client,
   instance_id: &str,

bin/core/src/config.rs

@@ -349,12 +349,16 @@ pub fn core_config() -> &'static CoreConfig {
         location: env
           .komodo_logging_location
           .unwrap_or(config.logging.location),
+        ansi: env.komodo_logging_ansi.unwrap_or(config.logging.ansi),
         otlp_endpoint: env
           .komodo_logging_otlp_endpoint
           .unwrap_or(config.logging.otlp_endpoint),
         opentelemetry_service_name: env
           .komodo_logging_opentelemetry_service_name
           .unwrap_or(config.logging.opentelemetry_service_name),
+        opentelemetry_scope_name: env
+          .komodo_logging_opentelemetry_scope_name
+          .unwrap_or(config.logging.opentelemetry_scope_name),
       },
       pretty_startup_config: env
         .komodo_pretty_startup_config

bin/core/src/connection/client.rs

@@ -102,6 +102,15 @@ impl PeripheryConnectionArgs<'_> {
 impl PeripheryConnection {
   /// Custom Core -> Periphery side only login wrapper
   /// to implement passkey support for backward compatibility
+  #[instrument(
+    "PeripheryLogin",
+    skip(self, socket, identifiers),
+    fields(
+      server_id = self.args.id,
+      address = self.args.address,
+      direction = "CoreToPeriphery"
+    )
+  )]
   async fn client_login(
     &self,
     socket: &mut TungsteniteWebsocket,
@@ -124,6 +133,7 @@ impl PeripheryConnection {
   }
 }
 
+#[instrument("V1PasskeyPeripheryLoginFlow", skip(socket, passkey))]
 async fn handle_passkey_login(
   socket: &mut TungsteniteWebsocket,
   // for legacy auth
@@ -142,7 +152,7 @@ async fn handle_passkey_login(
     };
 
     socket
-      .send(LoginMessage::V1Passkey(passkey))
+      .send_message(LoginMessage::V1Passkey(passkey))
       .await
       .context("Failed to send Login V1Passkey message")?;
 

bin/core/src/connection/mod.rs

@@ -10,8 +10,8 @@ use anyhow::anyhow;
 use cache::CloneCache;
 use database::mungos::{by_id::update_one_by_id, mongodb::bson::doc};
 use encoding::{
-  CastBytes as _, Decode as _, EncodedJsonMessage, EncodedOption,
-  EncodedResult, WithChannel,
+  CastBytes as _, Decode as _, EncodedJsonMessage, EncodedResponse,
+  WithChannel,
 };
 use komodo_client::entities::{
   builder::{AwsBuilderConfig, UrlBuilderConfig},
@@ -19,7 +19,7 @@ use komodo_client::entities::{
   server::Server,
 };
 use periphery_client::transport::{
-  EncodedTransportMessage, TransportMessage,
+  EncodedTransportMessage, ResponseMessage, TransportMessage,
 };
 use serror::serror_into_anyhow_error;
 use tokio::sync::RwLock;
@@ -109,6 +109,7 @@ pub struct PeripheryConnectionArgs<'a> {
 
 impl PublicKeyValidator for PeripheryConnectionArgs<'_> {
   type ValidationResult = String;
+  #[instrument("ValidatePeripheryPublicKey", skip(self))]
   async fn validate(
     &self,
     public_key: String,
@@ -253,12 +254,11 @@ impl<'a> From<&'a OwnedPeripheryConnectionArgs>
 }
 
 /// Sends None as InProgress ping.
-pub type ResponseChannels = CloneCache<
-  Uuid,
-  Sender<EncodedOption<EncodedResult<EncodedJsonMessage>>>,
->;
+pub type ResponseChannels =
+  CloneCache<Uuid, Sender<EncodedResponse<EncodedJsonMessage>>>;
 
-pub type TerminalChannels = CloneCache<Uuid, Sender<Vec<u8>>>;
+pub type TerminalChannels =
+  CloneCache<Uuid, Sender<anyhow::Result<Vec<u8>>>>;
 
 #[derive(Debug)]
 pub struct PeripheryConnection {
@@ -328,6 +328,11 @@ impl PeripheryConnection {
     )
   }
 
+  #[instrument(
+    "StandardPeripheryLoginFlow",
+    skip(self, socket, identifiers),
+    fields(expected_public_key = self.args.periphery_public_key)
+  )]
   pub async fn handle_login<W: Websocket, L: LoginFlow>(
     &self,
     socket: &mut W,
@@ -365,7 +370,7 @@ impl PeripheryConnection {
         let Ok(message) = receiver.recv().await else {
           break;
         };
-        match ws_write.send_inner(message.into_bytes()).await {
+        match ws_write.send(message.into_bytes()).await {
           Ok(_) => receiver.clear_buffer(),
           Err(e) => {
             self.set_error(e).await;
@@ -380,7 +385,7 @@ impl PeripheryConnection {
 
     let handle_reads = async {
       loop {
-        match ws_read.recv_inner().await {
+        match ws_read.recv().await {
           Ok(WebsocketMessage::Message(message)) => {
             self.handle_incoming_message(message).await
           }
@@ -415,28 +420,28 @@ impl PeripheryConnection {
       }
     };
     match message {
-      TransportMessage::Response(data) => match data.decode() {
-        Ok(WithChannel {
-          channel: channel_id,
-          data,
-        }) => {
-          let Some(channel) = self.responses.get(&channel_id).await
-          else {
-            warn!(
-              "Failed to forward Response message | No response channel found at {channel_id}"
-            );
-            return;
-          };
-          if let Err(e) = channel.send(data).await {
-            warn!(
-              "Failed to send response | Channel failure at {channel_id} | {e:#}"
-            );
+      TransportMessage::Response(data) => {
+        match data.decode().map(ResponseMessage::into_inner) {
+          Ok(WithChannel { channel, data }) => {
+            let Some(response_channel) =
+              self.responses.get(&channel).await
+            else {
+              warn!(
+                "Failed to forward Response message | No response channel found at {channel}"
+              );
+              return;
+            };
+            if let Err(e) = response_channel.send(data).await {
+              warn!(
+                "Failed to forward Response | Response channel failure at {channel} | {e:#}"
+              );
+            }
+          }
+          Err(e) => {
+            warn!("Failed to read Response message | {e:#}");
           }
         }
-        Err(e) => {
-          warn!("Failed to read Response message | {e:#}");
-        }
-      },
+      }
       TransportMessage::Terminal(data) => match data.decode() {
         Ok(WithChannel {
           channel: channel_id,

bin/core/src/connection/server.rs

@@ -17,11 +17,13 @@ use komodo_client::{
     user::system_user,
   },
 };
-use periphery_client::transport::LoginMessage;
+use periphery_client::{
+  api::PeripheryConnectionQuery, transport::LoginMessage,
+};
 use resolver_api::Resolve;
 use serror::{AddStatusCode, AddStatusCodeError};
+use tracing::Instrument;
 use transport::{
-  PeripheryConnectionQuery,
   auth::{
     HeaderConnectionIdentifiers, LoginFlow, LoginFlowArgs,
     PublicKeyValidator, ServerLoginFlow,
@@ -124,7 +126,7 @@ async fn existing_server_handler(
     let mut socket = AxumWebsocket(socket);
 
     if let Err(e) = socket
-      .send(LoginMessage::OnboardingFlow(false))
+      .send_message(LoginMessage::OnboardingFlow(false))
       .await
       .context("Failed to send Login OnboardingFlow false message")
     {
@@ -132,13 +134,23 @@ async fn existing_server_handler(
       return;
     };
 
-    if let Err(e) = connection
-      .handle_login::<_, ServerLoginFlow>(
-        &mut socket,
-        identifiers.build(query.as_bytes()),
-      )
-      .await
-    {
+    let span = info_span!(
+      "PeripheryLogin",
+      server_id = server.id,
+      direction = "PeripheryToCore"
+    );
+    let login = async {
+      connection
+        .handle_login::<_, ServerLoginFlow>(
+          &mut socket,
+          identifiers.build(query.as_bytes()),
+        )
+        .await
+    }
+    .instrument(span)
+    .await;
+
+    if let Err(e) = login {
       connection.set_error(e).await;
       return;
     }
@@ -157,7 +169,7 @@ async fn onboard_server_handler(
       format!("server={}", urlencoding::encode(&server_query));
     let mut socket = AxumWebsocket(socket);
 
-    if let Err(e) = socket.send(LoginMessage::OnboardingFlow(true)).await.context(
+    if let Err(e) = socket.send_message(LoginMessage::OnboardingFlow(true)).await.context(
       "Failed to send Login OnboardingFlow true message",
     ).context("Server onboarding error") {
       warn!("{e:#}");
@@ -214,7 +226,7 @@ async fn onboard_server_handler(
     };
 
     if let Err(e) = socket
-      .send(LoginMessage::Success)
+      .send_message(LoginMessage::Success)
       .await
       .context("Failed to send Login Onboarding Successful message")
     {

bin/core/src/helpers/builder.rs

@@ -31,8 +31,16 @@ use super::periphery_client;
 const BUILDER_POLL_RATE_SECS: u64 = 2;
 const BUILDER_POLL_MAX_TRIES: usize = 60;
 
-#[instrument(skip_all, fields(builder_id = builder.id, update_id = update.id))]
-pub async fn get_builder_periphery(
+#[instrument(
+  "ConnectBuilderPeriphery",
+  skip_all,
+  fields(
+    resource_name,
+    builder_id = builder.id,
+    update_id = update.id
+  )
+)]
+pub async fn connect_builder_periphery(
   // build: &Build,
   resource_name: String,
   version: Option<Version>,
@@ -76,7 +84,14 @@ pub async fn get_builder_periphery(
   }
 }
 
-#[instrument(skip_all, fields(resource_name, update_id = update.id))]
+#[instrument(
+  "GetAwsBuilder",
+  skip_all,
+  fields(
+    resource_name,
+    update_id = update.id,
+  )
+)]
 async fn get_aws_builder(
   resource_name: &str,
   version: Option<Version>,
@@ -168,7 +183,11 @@ async fn get_aws_builder(
   )
 }
 
-#[instrument(skip(update))]
+#[instrument(
+  "CleanupBuilderInstance",
+  skip_all,
+  fields(update_id = update.id)
+)]
 pub async fn cleanup_builder_instance(
   periphery: PeripheryClient,
   cleanup_data: BuildCleanupData,

bin/core/src/helpers/mod.rs

@@ -15,7 +15,6 @@ use komodo_client::entities::{
   stack::Stack,
   user::User,
 };
-use rand::Rng;
 
 use crate::{
   config::core_config, connection::PeripheryConnectionArgs,
@@ -47,14 +46,6 @@ pub fn empty_or_only_spaces(word: &str) -> bool {
   true
 }
 
-pub fn random_string(length: usize) -> String {
-  rand::rng()
-    .sample_iter(&rand::distr::Alphanumeric)
-    .take(length)
-    .map(char::from)
-    .collect()
-}
-
 /// First checks db for token, then checks core config.
 /// Only errors if db call errors.
 /// Returns (token, use_https)
@@ -199,7 +190,14 @@ pub async fn periphery_client(
   .await
 }
 
-#[instrument]
+#[instrument(
+  "CreatePermission",
+  skip(user),
+  fields(
+    operator = user.id,
+    username = user.username
+  )
+)]
 pub async fn create_permission<T>(
   user: &User,
   target: T,

bin/core/src/helpers/procedure.rs

@@ -3,14 +3,13 @@ use std::time::{Duration, Instant};
 use anyhow::{Context, anyhow};
 use database::mungos::by_id::find_one_by_id;
 use formatting::{Color, bold, colored, format_serror, muted};
-use futures::future::join_all;
+use futures_util::future::join_all;
 use komodo_client::{
   api::execute::*,
   entities::{
     action::Action,
     build::Build,
     deployment::Deployment,
-    permission::PermissionLevel,
     procedure::Procedure,
     repo::Repo,
     stack::Stack,
@@ -20,6 +19,7 @@ use komodo_client::{
 };
 use resolver_api::Resolve;
 use tokio::sync::Mutex;
+use uuid::Uuid;
 
 use crate::{
   api::{
@@ -32,7 +32,6 @@ use crate::{
 
 use super::update::{init_execution_update, update_update};
 
-#[instrument(skip_all)]
 pub async fn execute_procedure(
   procedure: &Procedure,
   update: &Mutex<Update>,
@@ -51,7 +50,7 @@ pub async fn execute_procedure(
     )
     .await;
     let timer = Instant::now();
-    execute_stage(
+    execute_procedure_stage(
       stage
         .executions
         .iter()
@@ -86,9 +85,8 @@ pub async fn execute_procedure(
   Ok(())
 }
 
-#[allow(dependency_on_unit_never_type_fallback)]
-#[instrument(skip(update))]
-async fn execute_stage(
+#[instrument("ExecuteProcedureStage", skip_all)]
+async fn execute_procedure_stage(
   _executions: Vec<Execution>,
   parent_id: &str,
   parent_name: &str,
@@ -218,6 +216,10 @@ async fn execute_stage(
   Ok(())
 }
 
+#[instrument(
+  "ExecuteProcedureExecution",
+  skip(parent_id, parent_name)
+)]
 async fn execute_execution(
   execution: Execution,
   // used to prevent recursive procedure
@@ -225,6 +227,7 @@ async fn execute_execution(
   parent_name: &str,
 ) -> anyhow::Result<()> {
   let user = procedure_user().to_owned();
+  let id = Uuid::new_v4();
   let update = match execution {
     Execution::None(_) => return Ok(()),
     Execution::RunProcedure(req) => {
@@ -239,7 +242,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at RunProcedure"),
@@ -262,7 +265,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at RunAction"),
@@ -285,7 +288,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at RunBuild"),
@@ -308,7 +311,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at CancelBuild"),
@@ -325,7 +328,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at Deploy"),
@@ -348,7 +351,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at PullDeployment"),
@@ -365,7 +368,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at StartDeployment"),
@@ -382,7 +385,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at RestartDeployment"),
@@ -399,7 +402,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at PauseDeployment"),
@@ -416,7 +419,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at UnpauseDeployment"),
@@ -433,7 +436,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at StopDeployment"),
@@ -450,7 +453,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at RemoveDeployment"),
@@ -473,7 +476,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at CloneRepo"),
@@ -496,7 +499,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at PullRepo"),
@@ -519,7 +522,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at BuildRepo"),
@@ -542,7 +545,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at CancelRepoBuild"),
@@ -559,7 +562,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at StartContainer"),
@@ -576,7 +579,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at RestartContainer"),
@@ -593,7 +596,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at PauseContainer"),
@@ -610,7 +613,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at UnpauseContainer"),
@@ -627,7 +630,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at StopContainer"),
@@ -644,7 +647,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at RemoveContainer"),
@@ -661,7 +664,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at StartAllContainers"),
@@ -678,7 +681,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at RestartAllContainers"),
@@ -695,7 +698,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at PauseAllContainers"),
@@ -712,7 +715,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at UnpauseAllContainers"),
@@ -729,7 +732,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at StopAllContainers"),
@@ -746,7 +749,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at PruneContainers"),
@@ -763,7 +766,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at DeleteNetwork"),
@@ -780,7 +783,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at PruneNetworks"),
@@ -797,7 +800,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at DeleteImage"),
@@ -814,7 +817,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at PruneImages"),
@@ -831,7 +834,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at DeleteVolume"),
@@ -848,7 +851,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at PruneVolumes"),
@@ -865,7 +868,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at PruneDockerBuilders"),
@@ -882,7 +885,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at PruneBuildx"),
@@ -899,7 +902,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at PruneSystem"),
@@ -916,7 +919,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at RunSync"),
@@ -939,7 +942,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at DeployStack"),
@@ -962,7 +965,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at DeployStackIfChanged"),
@@ -985,7 +988,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at PullStack"),
@@ -1008,7 +1011,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at StartStack"),
@@ -1025,7 +1028,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at RestartStack"),
@@ -1042,7 +1045,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at PauseStack"),
@@ -1059,7 +1062,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at UnpauseStack"),
@@ -1076,7 +1079,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at StopStack"),
@@ -1093,7 +1096,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at DestroyStack"),
@@ -1110,7 +1113,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at RunStackService"),
@@ -1133,7 +1136,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at TestAlerter"),
@@ -1150,7 +1153,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at SendAlert"),
@@ -1167,7 +1170,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at ClearRepoCache"),
@@ -1184,7 +1187,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at BackupCoreDatabase"),
@@ -1201,7 +1204,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at GlobalAutoUpdate"),
@@ -1218,7 +1221,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at RotateAllServerKeys"),
@@ -1235,7 +1238,7 @@ async fn execute_execution(
       let update_id = update.id.clone();
       handle_resolve_result(
         req
-          .resolve(&ExecuteArgs { user, update })
+          .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
           .context("Failed at RotateCoreKeys"),
@@ -1288,7 +1291,6 @@ async fn handle_resolve_result(
 }
 
 /// ASSUMES FIRST LOG IS ALREADY CREATED
-#[instrument(level = "debug")]
 async fn add_line_to_update(update: &Mutex<Update>, line: &str) {
   let mut lock = update.lock().await;
   let log = &mut lock.logs[0];
@@ -1309,7 +1311,6 @@ async fn extend_batch_exection<E: ExtendBatch>(
     pattern,
     Default::default(),
     procedure_user(),
-    PermissionLevel::Read.into(),
     &[],
   )
   .await?

bin/core/src/helpers/prune.rs

@@ -3,7 +3,7 @@ use async_timing_util::{
   ONE_DAY_MS, Timelength, unix_timestamp_ms, wait_until_timelength,
 };
 use database::mungos::{find::find_collect, mongodb::bson::doc};
-use futures::{StreamExt, stream::FuturesUnordered};
+use futures_util::{StreamExt, stream::FuturesUnordered};
 use periphery_client::api::docker::PruneImages;
 
 use crate::{config::core_config, state::db_client};

bin/core/src/helpers/query.rs

@@ -47,7 +47,6 @@ use crate::{
 };
 
 // user: Id or username
-#[instrument(level = "debug")]
 pub async fn get_user(user: &str) -> anyhow::Result<User> {
   if let Some(user) = admin_service_user(user) {
     return Ok(user);
@@ -60,7 +59,6 @@ pub async fn get_user(user: &str) -> anyhow::Result<User> {
     .with_context(|| format!("no user found with {user}"))
 }
 
-#[instrument(level = "debug")]
 pub async fn get_server_with_state(
   server_id_or_name: &str,
 ) -> anyhow::Result<(Server, ServerState)> {
@@ -69,7 +67,6 @@ pub async fn get_server_with_state(
   Ok((server, state))
 }
 
-#[instrument(level = "debug")]
 pub async fn get_server_state(server: &Server) -> ServerState {
   if !server.config.enabled {
     return ServerState::Disabled;
@@ -86,7 +83,6 @@ pub async fn get_server_state(server: &Server) -> ServerState {
   }
 }
 
-#[instrument(level = "debug")]
 pub async fn get_deployment_state(
   id: &String,
 ) -> anyhow::Result<DeploymentState> {
@@ -182,7 +178,6 @@ pub fn get_stack_state_from_containers(
   StackState::Unhealthy
 }
 
-#[instrument(level = "debug")]
 pub async fn get_stack_state(
   stack: &Stack,
 ) -> anyhow::Result<StackState> {
@@ -198,7 +193,6 @@ pub async fn get_stack_state(
   Ok(state)
 }
 
-#[instrument(level = "debug")]
 pub async fn get_tag(id_or_name: &str) -> anyhow::Result<Tag> {
   let query = match ObjectId::from_str(id_or_name) {
     Ok(id) => doc! { "_id": id },
@@ -212,7 +206,6 @@ pub async fn get_tag(id_or_name: &str) -> anyhow::Result<Tag> {
     .with_context(|| format!("no tag found matching {id_or_name}"))
 }
 
-#[instrument(level = "debug")]
 pub async fn get_tag_check_owner(
   id_or_name: &str,
   user: &User,
@@ -244,7 +237,6 @@ pub async fn get_id_to_tags(
   Ok(res)
 }
 
-#[instrument(level = "debug")]
 pub async fn get_user_user_groups(
   user_id: &str,
 ) -> anyhow::Result<Vec<UserGroup>> {
@@ -262,7 +254,6 @@ pub async fn get_user_user_groups(
   .context("failed to query db for user groups")
 }
 
-#[instrument(level = "debug")]
 pub async fn get_user_user_group_ids(
   user_id: &str,
 ) -> anyhow::Result<Vec<String>> {

bin/core/src/helpers/update.rs

@@ -40,7 +40,6 @@ pub fn make_update(
   }
 }
 
-#[instrument(level = "debug")]
 pub async fn add_update(
   mut update: Update,
 ) -> anyhow::Result<String> {
@@ -59,7 +58,6 @@ pub async fn add_update(
   Ok(id)
 }
 
-#[instrument(level = "debug")]
 pub async fn add_update_without_send(
   update: &Update,
 ) -> anyhow::Result<String> {
@@ -75,7 +73,6 @@ pub async fn add_update_without_send(
   Ok(id)
 }
 
-#[instrument(level = "debug")]
 pub async fn update_update(update: Update) -> anyhow::Result<()> {
   update_one_by_id(&db_client().updates, &update.id, database::mungos::update::Update::Set(to_document(&update)?), None)
     .await
@@ -85,7 +82,6 @@ pub async fn update_update(update: Update) -> anyhow::Result<()> {
   Ok(())
 }
 
-#[instrument(level = "debug")]
 async fn update_list_item(
   update: Update,
 ) -> anyhow::Result<UpdateListItem> {
@@ -115,7 +111,6 @@ async fn update_list_item(
   Ok(update)
 }
 
-#[instrument(level = "debug")]
 async fn send_update(update: UpdateListItem) -> anyhow::Result<()> {
   update_channel().sender.lock().await.send(update)?;
   Ok(())

bin/core/src/listener/resources.rs

@@ -14,6 +14,7 @@ use komodo_client::{
 use resolver_api::Resolve;
 use serde::Deserialize;
 use serde_json::json;
+use uuid::Uuid;
 
 use crate::{
   api::{
@@ -21,6 +22,7 @@ use crate::{
     write::WriteArgs,
   },
   helpers::update::init_execution_update,
+  resource,
 };
 
 use super::{ANY_BRANCH, ListenerLockCache};
@@ -54,7 +56,18 @@ pub async fn handle_build_webhook<B: super::ExtractBranch>(
   let lock = build_locks().get_or_insert_default(&build.id).await;
   let _lock = lock.lock().await;
 
-  B::verify_branch(&body, &build.config.branch)?;
+  // Use the correct target branch when using linked repo.
+  let branch = if build.config.linked_repo.is_empty() {
+    build.config.branch
+  } else {
+    resource::get::<Repo>(&build.config.linked_repo)
+      .await
+      .context("Failed to find 'linked_repo'")?
+      .config
+      .branch
+  };
+
+  B::verify_branch(&body, &branch)?;
 
   let user = git_webhook_user().to_owned();
   let req = ExecuteRequest::RunBuild(RunBuild { build: build.id });
@@ -63,7 +76,11 @@ pub async fn handle_build_webhook<B: super::ExtractBranch>(
     unreachable!()
   };
   req
-    .resolve(&ExecuteArgs { user, update })
+    .resolve(&ExecuteArgs {
+      user,
+      update,
+      id: Uuid::new_v4(),
+    })
     .await
     .map_err(|e| e.error)?;
   Ok(())
@@ -101,7 +118,11 @@ impl RepoExecution for CloneRepo {
       unreachable!()
     };
     req
-      .resolve(&ExecuteArgs { user, update })
+      .resolve(&ExecuteArgs {
+        user,
+        update,
+        id: Uuid::new_v4(),
+      })
       .await
       .map_err(|e| e.error)?;
     Ok(())
@@ -121,7 +142,11 @@ impl RepoExecution for PullRepo {
       unreachable!()
     };
     req
-      .resolve(&ExecuteArgs { user, update })
+      .resolve(&ExecuteArgs {
+        user,
+        update,
+        id: Uuid::new_v4(),
+      })
       .await
       .map_err(|e| e.error)?;
     Ok(())
@@ -141,7 +166,11 @@ impl RepoExecution for BuildRepo {
       unreachable!()
     };
     req
-      .resolve(&ExecuteArgs { user, update })
+      .resolve(&ExecuteArgs {
+        user,
+        update,
+        id: Uuid::new_v4(),
+      })
       .await
       .map_err(|e| e.error)?;
     Ok(())
@@ -240,7 +269,11 @@ impl StackExecution for DeployStack {
         unreachable!()
       };
       req
-        .resolve(&ExecuteArgs { user, update })
+        .resolve(&ExecuteArgs {
+          user,
+          update,
+          id: Uuid::new_v4(),
+        })
         .await
         .map_err(|e| e.error)?;
     } else {
@@ -254,7 +287,11 @@ impl StackExecution for DeployStack {
         unreachable!()
       };
       req
-        .resolve(&ExecuteArgs { user, update })
+        .resolve(&ExecuteArgs {
+          user,
+          update,
+          id: Uuid::new_v4(),
+        })
         .await
         .map_err(|e| e.error)?;
     }
@@ -303,7 +340,18 @@ pub async fn handle_stack_webhook_inner<
   let lock = stack_locks().get_or_insert_default(&stack.id).await;
   let _lock = lock.lock().await;
 
-  B::verify_branch(&body, &stack.config.branch)?;
+  // Use the correct target branch when using linked repo.
+  let branch = if stack.config.linked_repo.is_empty() {
+    stack.config.branch.clone()
+  } else {
+    resource::get::<Repo>(&stack.config.linked_repo)
+      .await
+      .context("Failed to find 'linked_repo'")?
+      .config
+      .branch
+  };
+
+  B::verify_branch(&body, &branch)?;
 
   E::resolve(stack).await.map_err(|e| e.error)
 }
@@ -352,7 +400,11 @@ impl SyncExecution for RunSync {
       unreachable!()
     };
     req
-      .resolve(&ExecuteArgs { user, update })
+      .resolve(&ExecuteArgs {
+        user,
+        update,
+        id: Uuid::new_v4(),
+      })
       .await
       .map_err(|e| e.error)?;
     Ok(())
@@ -401,7 +453,18 @@ async fn handle_sync_webhook_inner<
   let lock = sync_locks().get_or_insert_default(&sync.id).await;
   let _lock = lock.lock().await;
 
-  B::verify_branch(&body, &sync.config.branch)?;
+  // Use the correct target branch when using linked repo.
+  let branch = if sync.config.linked_repo.is_empty() {
+    sync.config.branch.clone()
+  } else {
+    resource::get::<Repo>(&sync.config.linked_repo)
+      .await
+      .context("Failed to find 'linked_repo'")?
+      .config
+      .branch
+  };
+
+  B::verify_branch(&body, &branch)?;
 
   E::resolve(sync).await
 }
@@ -451,7 +514,11 @@ pub async fn handle_procedure_webhook<B: super::ExtractBranch>(
     unreachable!()
   };
   req
-    .resolve(&ExecuteArgs { user, update })
+    .resolve(&ExecuteArgs {
+      user,
+      update,
+      id: Uuid::new_v4(),
+    })
     .await
     .map_err(|e| e.error)?;
   Ok(())
@@ -513,7 +580,11 @@ pub async fn handle_action_webhook<B: super::ExtractBranch>(
     unreachable!()
   };
   req
-    .resolve(&ExecuteArgs { user, update })
+    .resolve(&ExecuteArgs {
+      user,
+      update,
+      id: Uuid::new_v4(),
+    })
     .await
     .map_err(|e| e.error)?;
   Ok(())

bin/core/src/main.rs

@@ -6,12 +6,13 @@ extern crate tracing;
 use std::{net::SocketAddr, str::FromStr};
 
 use anyhow::Context;
-use axum::Router;
+use axum::{Router, routing::get};
 use axum_server::{Handle, tls_rustls::RustlsConfig};
 use tower_http::{
   cors::{Any, CorsLayer},
   services::{ServeDir, ServeFile},
 };
+use tracing::Instrument;
 
 use crate::config::{core_config, core_keys};
 
@@ -41,46 +42,51 @@ async fn app() -> anyhow::Result<()> {
   let config = core_config();
   logger::init(&config.logging)?;
 
-  info!("Komodo Core version: v{}", env!("CARGO_PKG_VERSION"));
+  let startup_span = info_span!("CoreStartup");
+  async {
+    info!("Komodo Core version: v{}", env!("CARGO_PKG_VERSION"));
 
-  match (
-    config.pretty_startup_config,
-    config.unsafe_unsanitized_startup_config,
-  ) {
-    (true, true) => info!("{:#?}", config),
-    (true, false) => info!("{:#?}", config.sanitized()),
-    (false, true) => info!("{:?}", config),
-    (false, false) => info!("{:?}", config.sanitized()),
+    match (
+      config.pretty_startup_config,
+      config.unsafe_unsanitized_startup_config,
+    ) {
+      (true, true) => info!("{:#?}", config),
+      (true, false) => info!("{:#?}", config.sanitized()),
+      (false, true) => info!("{:?}", config),
+      (false, false) => info!("{:?}", config.sanitized()),
+    }
+
+    // Init + log public key. Will crash if invalid private key here.
+    info!("Public Key: {}", core_keys().load().public);
+
+    rustls::crypto::aws_lc_rs::default_provider()
+      .install_default()
+      .expect("Failed to install default crypto provider");
+
+    // Init jwt client to crash on failure
+    state::jwt_client();
+    tokio::join!(
+      // Init db_client check to crash on db init failure
+      state::init_db_client(),
+      // Manage OIDC client (defined in config / env vars / compose secret file)
+      auth::oidc::client::spawn_oidc_client_management()
+    );
+    // Run after db connection.
+    startup::on_startup().await;
+
+    // Spawn background tasks
+    monitor::spawn_monitor_loop();
+    resource::spawn_resource_refresh_loop();
+    resource::spawn_all_resources_cache_refresh_loop();
+    resource::spawn_build_state_refresh_loop();
+    resource::spawn_repo_state_refresh_loop();
+    resource::spawn_procedure_state_refresh_loop();
+    resource::spawn_action_state_refresh_loop();
+    schedule::spawn_schedule_executor();
+    helpers::prune::spawn_prune_loop();
   }
-
-  // Init + log public key. Will crash if invalid private key here.
-  info!("Public Key: {}", core_keys().load().public);
-
-  rustls::crypto::aws_lc_rs::default_provider()
-    .install_default()
-    .expect("Failed to install default crypto provider");
-
-  // Init jwt client to crash on failure
-  state::jwt_client();
-  tokio::join!(
-    // Init db_client check to crash on db init failure
-    state::init_db_client(),
-    // Manage OIDC client (defined in config / env vars / compose secret file)
-    auth::oidc::client::spawn_oidc_client_management()
-  );
-  // Run after db connection.
-  startup::on_startup().await;
-
-  // Spawn background tasks
-  monitor::spawn_monitor_loop();
-  resource::spawn_resource_refresh_loop();
-  resource::spawn_all_resources_cache_refresh_loop();
-  resource::spawn_build_state_refresh_loop();
-  resource::spawn_repo_state_refresh_loop();
-  resource::spawn_procedure_state_refresh_loop();
-  resource::spawn_action_state_refresh_loop();
-  schedule::spawn_schedule_executor();
-  helpers::prune::spawn_prune_loop();
+  .instrument(startup_span)
+  .await;
 
   // Setup static frontend services
   let frontend_path = &config.frontend_path;
@@ -90,6 +96,7 @@ async fn app() -> anyhow::Result<()> {
     .not_found_service(frontend_index.clone());
 
   let app = Router::new()
+    .route("/version", get(|| async { env!("CARGO_PKG_VERSION") }))
     .nest("/auth", api::auth::router())
     .nest("/user", api::user::router())
     .nest("/read", api::read::router())

bin/core/src/monitor/alert/deployment.rs

@@ -13,7 +13,6 @@ use crate::{
   state::{action_states, db_client},
 };
 
-#[instrument(level = "debug")]
 pub async fn alert_deployments(
   ts: i64,
   server_names: &HashMap<String, String>,

bin/core/src/monitor/alert/mod.rs

@@ -2,8 +2,7 @@ use std::collections::HashMap;
 
 use anyhow::Context;
 use komodo_client::entities::{
-  permission::PermissionLevel, resource::ResourceQuery,
-  server::Server, user::User,
+  resource::ResourceQuery, server::Server, user::User,
 };
 
 use crate::resource;
@@ -13,7 +12,6 @@ mod server;
 mod stack;
 
 // called after cache update
-#[instrument(level = "debug")]
 pub async fn check_alerts(ts: i64) {
   let (servers, server_names) = match get_all_servers_map().await {
     Ok(res) => res,
@@ -30,7 +28,6 @@ pub async fn check_alerts(ts: i64) {
   );
 }
 
-#[instrument(level = "debug")]
 async fn get_all_servers_map()
 -> anyhow::Result<(HashMap<String, Server>, HashMap<String, String>)>
 {
@@ -40,7 +37,6 @@ async fn get_all_servers_map()
       admin: true,
       ..Default::default()
     },
-    PermissionLevel::Read.into(),
     &[],
   )
   .await

bin/core/src/monitor/alert/server.rs

@@ -73,7 +73,6 @@ fn alert_buffer() -> &'static AlertBuffer {
   BUFFER.get_or_init(AlertBuffer::new)
 }
 
-#[instrument(level = "debug")]
 pub async fn alert_servers(
   ts: i64,
   mut servers: HashMap<String, Server>,
@@ -555,7 +554,6 @@ pub async fn alert_servers(
   );
 }
 
-#[instrument(level = "debug")]
 async fn open_new_alerts(alerts: &[(Alert, SendAlerts)]) {
   if alerts.is_empty() {
     return;
@@ -603,7 +601,6 @@ async fn open_new_alerts(alerts: &[(Alert, SendAlerts)]) {
   send_alerts(&alerts).await
 }
 
-#[instrument(level = "debug")]
 async fn update_alerts(alerts: &[(Alert, SendAlerts)]) {
   if alerts.is_empty() {
     return;
@@ -651,7 +648,6 @@ async fn update_alerts(alerts: &[(Alert, SendAlerts)]) {
   }
 }
 
-#[instrument(level = "debug")]
 async fn resolve_alerts(alerts: &[(Alert, SendAlerts)]) {
   if alerts.is_empty() {
     return;
@@ -708,7 +704,6 @@ async fn resolve_alerts(alerts: &[(Alert, SendAlerts)]) {
   }
 }
 
-#[instrument(level = "debug")]
 async fn get_open_alerts()
 -> anyhow::Result<(OpenAlertMap, OpenDiskAlertMap)> {
   let alerts = find_collect(

bin/core/src/monitor/alert/stack.rs

@@ -12,7 +12,6 @@ use crate::{
   state::{action_states, db_client, stack_status_cache},
 };
 
-#[instrument(level = "debug")]
 pub async fn alert_stacks(
   ts: i64,
   server_names: &HashMap<String, String>,

bin/core/src/monitor/helpers.rs

@@ -25,7 +25,6 @@ use super::{
   CachedStackStatus, History,
 };
 
-#[instrument(level = "debug", skip_all)]
 pub async fn insert_deployments_status_unknown(
   deployments: Vec<Deployment>,
 ) {
@@ -51,7 +50,6 @@ pub async fn insert_deployments_status_unknown(
   }
 }
 
-#[instrument(level = "debug", skip_all)]
 pub async fn insert_repos_status_unknown(repos: Vec<Repo>) {
   let status_cache = repo_status_cache();
   for repo in repos {
@@ -68,7 +66,6 @@ pub async fn insert_repos_status_unknown(repos: Vec<Repo>) {
   }
 }
 
-#[instrument(level = "debug", skip_all)]
 pub async fn insert_stacks_status_unknown(stacks: Vec<Stack>) {
   let status_cache = stack_status_cache();
   for stack in stacks {
@@ -99,7 +96,6 @@ type DockerLists = (
   Option<Vec<ComposeProject>>,
 );
 
-#[instrument(level = "debug", skip_all)]
 pub async fn insert_server_status(
   server: &Server,
   state: ServerState,

bin/core/src/monitor/mod.rs

@@ -3,7 +3,7 @@ use std::sync::{Arc, OnceLock};
 use async_timing_util::wait_until_timelength;
 use cache::CloneCache;
 use database::mungos::{find::find_collect, mongodb::bson::doc};
-use futures::future::join_all;
+use futures_util::future::join_all;
 use helpers::insert_stacks_status_unknown;
 use komodo_client::entities::{
   build::Build,
@@ -140,7 +140,6 @@ fn update_cache_for_server_controller()
 /// which exits early if the lock is busy or it was completed too recently.
 /// If force is true, it will wait on simultaneous calls, and will
 /// ignore the restriction on being completed too recently.
-#[instrument(level = "debug")]
 pub async fn update_cache_for_server(server: &Server, force: bool) {
   // Concurrency controller to ensure it isn't done too often
   // when it happens in other contexts.

bin/core/src/monitor/record.rs

@@ -4,7 +4,6 @@ use komodo_client::entities::stats::{
 
 use crate::state::{db_client, server_status_cache};
 
-#[instrument(level = "debug")]
 pub async fn record_server_stats(ts: i64) {
   let status = server_status_cache().get_values().await;
   let records = status

bin/core/src/periphery/mod.rs

@@ -79,17 +79,11 @@ impl PeripheryClient {
     periphery_connections().remove(&self.id).await
   }
 
-  #[tracing::instrument(level = "debug", skip(self))]
   pub async fn health_check(&self) -> anyhow::Result<()> {
     self.request(api::GetHealth {}).await?;
     Ok(())
   }
 
-  #[tracing::instrument(
-    name = "PeripheryRequest",
-    skip(self),
-    level = "debug"
-  )]
   pub async fn request<T>(
     &self,
     request: T,
@@ -139,7 +133,7 @@ impl PeripheryClient {
         continue;
       };
 
-      return message.decode_into();
+      return message.decode();
     }
   }
 }

bin/core/src/periphery/terminal.rs

@@ -5,9 +5,8 @@ use std::{
 };
 
 use anyhow::Context;
-use bytes::Bytes;
 use cache::CloneCache;
-use futures::Stream;
+use futures_util::Stream;
 use komodo_client::api::write::TerminalRecreateMode;
 use periphery_client::{
   api::terminal::{
@@ -16,22 +15,25 @@ use periphery_client::{
   },
   transport::EncodedTransportMessage,
 };
-use transport::channel::{Receiver, Sender, channel};
+use transport::channel::{Receiver, Sender};
 use uuid::Uuid;
 
 use crate::{
   periphery::PeripheryClient, state::periphery_connections,
 };
 
+pub struct ConnectTerminalResponse {
+  pub channel: Uuid,
+  pub sender: Sender<EncodedTransportMessage>,
+  pub receiver: Receiver<anyhow::Result<Vec<u8>>>,
+}
+
 impl PeripheryClient {
+  #[instrument("ConnectTerminal", skip(self), fields(server_id = self.id))]
   pub async fn connect_terminal(
     &self,
     terminal: String,
-  ) -> anyhow::Result<(
-    Uuid,
-    Sender<EncodedTransportMessage>,
-    Receiver<Vec<u8>>,
-  )> {
+  ) -> anyhow::Result<ConnectTerminalResponse> {
     tracing::trace!(
       "request | type: ConnectTerminal | terminal name: {terminal}",
     );
@@ -41,35 +43,36 @@ impl PeripheryClient {
         || format!("No connection found for server {}", self.id),
       )?;
 
-    let channel_id = self
+    let channel = self
       .request(ConnectTerminal { terminal })
       .await
       .context("Failed to create terminal connection")?;
 
-    let (sender, receiever) = channel();
-    connection.terminals.insert(channel_id, sender).await;
+    let (sender, receiver) = transport::channel::channel();
+    connection.terminals.insert(channel, sender).await;
 
     connection
       .sender
-      .send_terminal(channel_id, Bytes::new())
+      .send_terminal(channel, Ok(Vec::with_capacity(17))) // 16 bytes uuid + 1 EncodedResponse
       .await
       .context(
         "Failed to send TerminalTrigger to begin forwarding.",
       )?;
 
-    Ok((channel_id, connection.sender.clone(), receiever))
+    Ok(ConnectTerminalResponse {
+      channel,
+      sender: connection.sender.clone(),
+      receiver,
+    })
   }
 
+  #[instrument("ConnectContainerExec", skip(self), fields(server_id = self.id))]
   pub async fn connect_container_exec(
     &self,
     container: String,
     shell: String,
     recreate: TerminalRecreateMode,
-  ) -> anyhow::Result<(
-    Uuid,
-    Sender<EncodedTransportMessage>,
-    Receiver<Vec<u8>>,
-  )> {
+  ) -> anyhow::Result<ConnectTerminalResponse> {
     tracing::trace!(
       "request | type: ConnectContainerExec | container name: {container} | shell: {shell}",
     );
@@ -79,7 +82,7 @@ impl PeripheryClient {
         || format!("No connection found for server {}", self.id),
       )?;
 
-    let channel_id = self
+    let channel = self
       .request(ConnectContainerExec {
         container,
         shell,
@@ -88,29 +91,30 @@ impl PeripheryClient {
       .await
       .context("Failed to create container exec connection")?;
 
-    let (sender, receiever) = channel();
-    connection.terminals.insert(channel_id, sender).await;
+    let (sender, receiver) = transport::channel::channel();
+    connection.terminals.insert(channel, sender).await;
 
     connection
       .sender
-      .send_terminal(channel_id, Bytes::new())
+      .send_terminal(channel, Ok(Vec::with_capacity(17)))
       .await
       .context(
         "Failed to send TerminalTrigger to begin forwarding.",
       )?;
 
-    Ok((channel_id, connection.sender.clone(), receiever))
+    Ok(ConnectTerminalResponse {
+      channel,
+      sender: connection.sender.clone(),
+      receiver,
+    })
   }
 
+  #[instrument("ConnectContainerAttach", skip(self), fields(server_id = self.id))]
   pub async fn connect_container_attach(
     &self,
     container: String,
     recreate: TerminalRecreateMode,
-  ) -> anyhow::Result<(
-    Uuid,
-    Sender<EncodedTransportMessage>,
-    Receiver<Vec<u8>>,
-  )> {
+  ) -> anyhow::Result<ConnectTerminalResponse> {
     tracing::trace!(
       "request | type: ConnectContainerAttach | container name: {container}",
     );
@@ -128,18 +132,22 @@ impl PeripheryClient {
       .await
       .context("Failed to create container attach connection")?;
 
-    let (sender, receiever) = transport::channel::channel();
+    let (sender, receiver) = transport::channel::channel();
     connection.terminals.insert(channel, sender).await;
 
     connection
       .sender
-      .send_terminal(channel, Bytes::new())
+      .send_terminal(channel, Ok(Vec::with_capacity(17)))
       .await
       .context(
         "Failed to send TerminalTrigger to begin forwarding.",
       )?;
 
-    Ok((channel, connection.sender.clone(), receiever))
+    Ok(ConnectTerminalResponse {
+      channel,
+      sender: connection.sender.clone(),
+      receiver,
+    })
   }
 
   /// Executes command on specified terminal,
@@ -155,7 +163,7 @@ impl PeripheryClient {
   ///
   /// If this value is NOT the final item before stream closes, it means
   /// the terminal exited mid command, before giving status. Example: running `exit`.
-  #[tracing::instrument(level = "debug", skip(self))]
+  #[instrument("ExecuteTerminal", skip(self), fields(server_id = self.id))]
   pub async fn execute_terminal(
     &self,
     terminal: String,
@@ -163,7 +171,7 @@ impl PeripheryClient {
   ) -> anyhow::Result<
     impl Stream<Item = anyhow::Result<Vec<u8>>> + 'static,
   > {
-    tracing::trace!(
+    trace!(
       "sending request | type: ExecuteTerminal | terminal name: {terminal} | command: {command}",
     );
 
@@ -172,27 +180,25 @@ impl PeripheryClient {
         || format!("No connection found for server {}", self.id),
       )?;
 
-    let channel_id = self
+    let channel = self
       .request(ExecuteTerminal { terminal, command })
       .await
       .context("Failed to create execute terminal connection")?;
 
-    let (terminal_sender, terminal_receiver) = channel();
-    connection
-      .terminals
-      .insert(channel_id, terminal_sender)
-      .await;
+    let (terminal_sender, terminal_receiver) =
+      transport::channel::channel();
+    connection.terminals.insert(channel, terminal_sender).await;
 
     connection
       .sender
-      .send_terminal(channel_id, Bytes::new())
+      .send_terminal(channel, Ok(Vec::with_capacity(17)))
       .await
       .context(
         "Failed to send TerminalTrigger to begin forwarding.",
       )?;
 
     Ok(ReceiverStream {
-      channel_id,
+      channel,
       receiver: terminal_receiver,
       channels: connection.terminals.clone(),
     })
@@ -211,7 +217,7 @@ impl PeripheryClient {
   ///
   /// If this value is NOT the final item before stream closes, it means
   /// the container shell exited mid command, before giving status. Example: running `exit`.
-  #[tracing::instrument(level = "debug", skip(self))]
+  #[instrument("ExecuteContainerExec", skip(self), fields(server_id = self.id))]
   pub async fn execute_container_exec(
     &self,
     container: String,
@@ -228,7 +234,7 @@ impl PeripheryClient {
         || format!("No connection found for server {}", self.id),
       )?;
 
-    let channel_id = self
+    let channel = self
       .request(ExecuteContainerExec {
         container,
         shell,
@@ -238,21 +244,19 @@ impl PeripheryClient {
       .await
       .context("Failed to create execute terminal connection")?;
 
-    let (terminal_sender, terminal_receiver) = channel();
-    connection
-      .terminals
-      .insert(channel_id, terminal_sender)
-      .await;
+    let (terminal_sender, terminal_receiver) =
+      transport::channel::channel();
+    connection.terminals.insert(channel, terminal_sender).await;
 
     // Trigger forwarding to begin now that forwarding channel is ready.
     // This is required to not miss messages.
     connection
       .sender
-      .send_terminal(channel_id, Bytes::new())
+      .send_terminal(channel, Ok(Vec::with_capacity(17)))
       .await?;
 
     Ok(ReceiverStream {
-      channel_id,
+      channel,
       receiver: terminal_receiver,
       channels: connection.terminals.clone(),
     })
@@ -260,9 +264,9 @@ impl PeripheryClient {
 }
 
 pub struct ReceiverStream {
-  channel_id: Uuid,
-  channels: Arc<CloneCache<Uuid, Sender<Vec<u8>>>>,
-  receiver: Receiver<Vec<u8>>,
+  channel: Uuid,
+  channels: Arc<CloneCache<Uuid, Sender<anyhow::Result<Vec<u8>>>>>,
+  receiver: Receiver<anyhow::Result<Vec<u8>>>,
 }
 
 impl Stream for ReceiverStream {
@@ -272,14 +276,14 @@ impl Stream for ReceiverStream {
     cx: &mut task::Context<'_>,
   ) -> Poll<Option<Self::Item>> {
     match self.receiver.poll_recv(cx) {
-      Poll::Ready(Some(bytes))
+      Poll::Ready(Some(Ok(bytes)))
         if bytes == END_OF_OUTPUT.as_bytes() =>
       {
         self.cleanup();
         Poll::Ready(None)
       }
-      Poll::Ready(Some(bytes)) => Poll::Ready(Some(Ok(bytes))),
-      // Poll::Ready(Some(Err(e))) => Poll::Ready(Some(Err(e))),
+      Poll::Ready(Some(Ok(bytes))) => Poll::Ready(Some(Ok(bytes))),
+      Poll::Ready(Some(Err(e))) => Poll::Ready(Some(Err(e))),
       Poll::Ready(None) => {
         self.cleanup();
         Poll::Ready(None)
@@ -293,9 +297,9 @@ impl ReceiverStream {
   fn cleanup(&self) {
     // Not the prettiest but it should be fine
     let channels = self.channels.clone();
-    let id = self.channel_id;
+    let channel = self.channel;
     tokio::spawn(async move {
-      channels.remove(&id).await;
+      channels.remove(&channel).await;
     });
   }
 }

bin/core/src/permission.rs

@@ -3,7 +3,7 @@ use std::collections::HashSet;
 use anyhow::{Context, anyhow};
 use database::mongo_indexed::doc;
 use database::mungos::find::find_collect;
-use futures::{FutureExt, future::BoxFuture};
+use futures_util::{FutureExt, future::BoxFuture};
 use indexmap::IndexSet;
 use komodo_client::{
   api::read::GetPermission,
@@ -68,12 +68,11 @@ pub async fn get_check_permissions<T: KomodoResource>(
   }
 }
 
-#[instrument(level = "debug")]
 pub fn get_user_permission_on_resource<'a, T: KomodoResource>(
   user: &'a User,
   resource_id: &'a str,
 ) -> BoxFuture<'a, anyhow::Result<PermissionLevelAndSpecifics>> {
-  Box::pin(async {
+  Box::pin(async move {
     // Admin returns early with max permissions
     if user.admin {
       return Ok(PermissionLevel::Write.all());
@@ -164,7 +163,6 @@ pub fn get_user_permission_on_resource<'a, T: KomodoResource>(
 }
 
 /// Returns None if still no need to filter by resource id (eg transparent mode, group membership with all access).
-#[instrument(level = "debug")]
 pub async fn get_resource_ids_for_user<T: KomodoResource>(
   user: &User,
 ) -> anyhow::Result<Option<Vec<String>>> {

bin/core/src/resource/build.rs

@@ -253,7 +253,7 @@ pub async fn refresh_build_state_cache() {
   });
 }
 
-#[instrument(skip(user))]
+#[instrument("ValidateBuildConfig", skip_all)]
 async fn validate_config(
   config: &mut PartialBuildConfig,
   user: &User,

bin/core/src/resource/builder.rs

@@ -179,7 +179,7 @@ impl super::KomodoResource for Builder {
   }
 }
 
-#[instrument(skip(user))]
+#[instrument("ValidateBuilderConfig", skip_all)]
 async fn validate_config(
   config: &mut PartialBuilderConfig,
   user: &User,

bin/core/src/resource/deployment.rs

@@ -309,7 +309,7 @@ impl super::KomodoResource for Deployment {
   }
 }
 
-#[instrument(skip(user))]
+#[instrument("ValidateDeploymentConfig", skip_all)]
 async fn validate_config(
   config: &mut PartialDeploymentConfig,
   user: &User,