deploy 2.0.0-dev-102

Cargo.toml

@@ -8,7 +8,7 @@ members = [
 ]
 
 [workspace.package]
-version = "2.0.0-dev-59"
+version = "2.0.0-dev-102"
 edition = "2024"
 authors = ["mbecker20 <becker.maxh@gmail.com>"]
 license = "GPL-3.0-or-later"
@@ -26,7 +26,9 @@ environment_file = { path = "lib/environment_file" }
 environment = { path = "lib/environment" }
 interpolate = { path = "lib/interpolate" }
 secret_file = { path = "lib/secret_file" }
+validations = { path = "lib/validations" }
 formatting = { path = "lib/formatting" }
+rate_limit = { path = "lib/rate_limit" }
 transport = { path = "lib/transport" }
 database = { path = "lib/database" }
 encoding = { path = "lib/encoding" }
@@ -39,9 +41,8 @@ noise = { path = "lib/noise" }
 git = { path = "lib/git" }
 
 # MOGH
-run_command = { version = "0.0.6", features = ["async_tokio"] }
 serror = { version = "0.5.3", default-features = false }
-slack = { version = "1.2.1", package = "slack_client_rs", default-features = false, features = ["rustls"] }
+slack = { version = "2.0.0", package = "slack_client_rs", default-features = false, features = ["rustls"] }
 derive_default_builder = "0.1.8"
 derive_empty_traits = "0.1.0"
 async_timing_util = "1.1.0"
@@ -49,30 +50,30 @@ partial_derive2 = "0.4.3"
 derive_variants = "1.0.0"
 mongo_indexed = "2.0.2"
 resolver_api = "3.0.0"
-toml_pretty = "1.2.0"
+toml_pretty = "2.0.0"
 mungos = "3.2.2"
 svi = "1.2.0"
 
 # ASYNC
 reqwest = { version = "0.12.24", default-features = false, features = ["json", "stream", "rustls-tls-native-roots"] }
-tokio = { version = "1.47.1", features = ["full"] }
-tokio-util = { version = "0.7.16", features = ["io", "codec"] }
+tokio = { version = "1.48.0", features = ["full"] }
+tokio-util = { version = "0.7.17", features = ["io", "codec"] }
 tokio-stream = { version = "0.1.17", features = ["sync"] }
 pin-project-lite = "0.2.16"
-futures = "0.3.31"
 futures-util = "0.3.31"
 arc-swap = "1.7.1"
 
 # SERVER
 tokio-tungstenite = { version = "0.28.0", features = ["rustls-tls-native-roots"] }
-axum-extra = { version = "0.10.3", features = ["typed-header"] }
-tower-http = { version = "0.6.6", features = ["fs", "cors"] }
-axum-server = { version = "0.7.2", features = ["tls-rustls"] }
-axum = { version = "0.8.6", features = ["ws", "json", "macros"] }
+tower-http = { version = "0.6.8", features = ["fs", "cors", "set-header"] }
+axum = { version = "0.8.7", features = ["ws", "json", "macros"] }
+axum-extra = { version = "0.12.2", features = ["typed-header"] }
+axum-server = { version = "0.8.0", features = ["tls-rustls"] }
+tower-sessions = "0.14.0"
 
 # SER/DE
 ipnetwork = { version = "0.21.1", features = ["serde"] }
-indexmap = { version = "2.11.4", features = ["serde"] }
+indexmap = { version = "2.12.1", features = ["serde"] }
 serde = { version = "1.0.227", features = ["derive"] }
 strum = { version = "0.27.2", features = ["derive"] }
 bson = { version = "2.15.0" } # must keep in sync with mongodb version
@@ -89,60 +90,66 @@ thiserror = "2.0.17"
 # LOGGING
 opentelemetry-otlp = { version = "0.31.0", features = ["tls-roots", "reqwest-rustls"] }
 opentelemetry_sdk = { version = "0.31.0", features = ["rt-tokio"] }
-tracing-subscriber = { version = "0.3.20", features = ["json"] }
+tracing-subscriber = { version = "0.3.22", features = ["json"] }
 opentelemetry-semantic-conventions = "0.31.0"
 tracing-opentelemetry = "0.32.0"
 opentelemetry = "0.31.0"
-tracing = "0.1.41"
+tracing = "0.1.43"
 
 # CONFIG
-clap = { version = "4.5.49", features = ["derive"] }
+clap = { version = "4.5.53", features = ["derive"] }
 dotenvy = "0.15.7"
 envy = "0.4.2"
 
 # CRYPTO / AUTH
-uuid = { version = "1.18.1", features = ["v4", "fast-rng", "serde"] }
-jsonwebtoken = { version = "10.0.0", features = ["aws_lc_rs"] } # locked back with octorust
-rustls = { version = "0.23.32", features = ["aws-lc-rs"] }
-pem-rfc7468 = { version = "0.7.0", features = ["alloc"] }
-openidconnect = "4.0.1"
+webauthn-rs = { version = "0.5.4", features = ["danger-allow-state-serialisation"] }
+openidconnect = { version = "4.0.1", features = ["accept-rfc3339-timestamps"] }
+uuid = { version = "1.19.0", features = ["v4", "fast-rng", "serde"] }
+jsonwebtoken = { version = "10.2.0", features = ["aws_lc_rs"] } # locked back with octorust
+rustls = { version = "0.23.35", features = ["aws-lc-rs"] }
+pem-rfc7468 = { version = "1.0.0", features = ["alloc"] }
+totp-rs = { version = "5.7.0", features = ["qr"] }
+webauthn-rs-proto = "0.5.4"
+base64urlsafedata = "0.5.4"
+data-encoding = "2.9.0"
 urlencoding = "2.1.3"
 bcrypt = "0.17.1"
-base64 = "0.22.1"
 pkcs8 = "0.10.2"
 snow = "0.10.0"
 hmac = "0.12.1"
 sha1 = "0.10.6"
 sha2 = "0.10.9"
 rand = "0.9.2"
-hex = "0.4.3"
 spki = "0.7.3"
 der = "0.7.10"
+hex = "0.4.3"
 
 # SYSTEM
+hickory-resolver = "0.25.2"
 portable-pty = "0.9.0"
-bollard = "0.19.3"
+shell-escape = "0.1.5"
+crossterm = "0.29.0"
+bollard = "0.19.4"
 sysinfo = "0.37.1"
+shlex = "1.3.0"
 
 # CLOUD
-aws-config = "1.8.8"
-aws-sdk-ec2 = "1.173.0"
-aws-credential-types = "1.2.8"
+aws-config = "1.8.12"
+aws-sdk-ec2 = "1.196.0"
+aws-credential-types = "1.2.11"
 
 ## CRON
-english-to-cron = "0.1.6"
+english-to-cron = "0.1.7"
 chrono-tz = "0.10.4"
 chrono = "0.4.42"
-croner = "3.0.0"
+croner = "3.0.1"
 
 # MISC
-async-compression = { version = "0.4.32", features = ["tokio", "gzip"] }
+async-compression = { version = "0.4.35", features = ["tokio", "gzip"] }
 derive_builder = "0.20.2"
-shell-escape = "0.1.5"
 comfy-table = "7.2.1"
 typeshare = "1.0.4"
-dashmap = "6.1.0"
 wildcard = "0.3.0"
 colored = "3.0.0"
+bytes = "1.11.0"
 regex = "1.12.2"
-bytes = "1.10.1"

action/deploy-fe.ts

@@ -0,0 +1,4 @@
+const cmd = "km run -y action deploy-komodo-fe-change";
+new Deno.Command("bash", {
+  args: ["-c", cmd],
+}).spawn();

bin/binaries.Dockerfile

@@ -1,7 +1,7 @@
 ## Builds the Komodo Core, Periphery, and Util binaries
-## for a specific architecture.
+## for a specific architecture. Requires OpenSSL 3 or later.
 
-FROM rust:1.90.0-bullseye AS builder
+FROM rust:1.91.1-bookworm AS builder
 RUN cargo install cargo-strip
 
 WORKDIR /builder

bin/chef.binaries.Dockerfile

@@ -1,9 +1,9 @@
 ## Builds the Komodo Core, Periphery, and Util binaries
-## for a specific architecture.
+## for a specific architecture. Requires OpenSSL 3 or later.
 
 ## Uses chef for dependency caching to help speed up back-to-back builds.
 
-FROM lukemathwalker/cargo-chef:latest-rust-1.90.0-bullseye AS chef
+FROM lukemathwalker/cargo-chef:latest-rust-1.90.0-bookworm AS chef
 WORKDIR /builder
 
 # Plan just the RECIPE to see if things have changed

bin/cli/Cargo.toml

@@ -23,7 +23,9 @@ noise.workspace = true
 # external
 futures-util.workspace = true
 comfy-table.workspace = true
+tokio-util.workspace = true
 serde_json.workspace = true
+crossterm.workspace = true
 serde_qs.workspace = true
 wildcard.workspace = true
 tracing.workspace = true

bin/cli/aio.Dockerfile

@@ -1,4 +1,4 @@
-FROM rust:1.90.0-bullseye AS builder
+FROM rust:1.91.1-bullseye AS builder
 RUN cargo install cargo-strip
 
 WORKDIR /builder

bin/cli/src/command/container.rs

@@ -61,7 +61,8 @@ async fn list_containers(
         .map(|s| (s.id.clone(), s))
         .collect::<HashMap<_, _>>())),
     client.read(ListAllDockerContainers {
-      servers: Default::default()
+      servers: Default::default(),
+      containers: Default::default(),
     }),
   )?;
 
@@ -145,7 +146,8 @@ pub async fn inspect_container(
         .map(|s| (s.id.clone(), s))
         .collect::<HashMap<_, _>>())),
     client.read(ListAllDockerContainers {
-      servers: Default::default()
+      servers: Default::default(),
+      containers: Default::default()
     }),
   )?;
 

bin/cli/src/command/execute.rs

@@ -221,6 +221,33 @@ pub async fn handle(
     Execution::SendAlert(data) => {
       println!("{}: {data:?}", "Data".dimmed())
     }
+    Execution::RemoveSwarmNodes(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
+    Execution::RemoveSwarmStacks(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
+    Execution::RemoveSwarmServices(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
+    Execution::CreateSwarmConfig(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
+    Execution::RotateSwarmConfig(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
+    Execution::RemoveSwarmConfigs(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
+    Execution::CreateSwarmSecret(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
+    Execution::RotateSwarmSecret(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
+    Execution::RemoveSwarmSecrets(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
     Execution::ClearRepoCache(data) => {
       println!("{}: {data:?}", "Data".dimmed())
     }
@@ -488,6 +515,42 @@ pub async fn handle(
       .execute(request)
       .await
       .map(|u| ExecutionResult::Single(u.into())),
+    Execution::RemoveSwarmNodes(request) => client
+      .execute(request)
+      .await
+      .map(|u| ExecutionResult::Single(u.into())),
+    Execution::RemoveSwarmStacks(request) => client
+      .execute(request)
+      .await
+      .map(|u| ExecutionResult::Single(u.into())),
+    Execution::RemoveSwarmServices(request) => client
+      .execute(request)
+      .await
+      .map(|u| ExecutionResult::Single(u.into())),
+    Execution::CreateSwarmConfig(request) => client
+      .execute(request)
+      .await
+      .map(|u| ExecutionResult::Single(u.into())),
+    Execution::RotateSwarmConfig(request) => client
+      .execute(request)
+      .await
+      .map(|u| ExecutionResult::Single(u.into())),
+    Execution::RemoveSwarmConfigs(request) => client
+      .execute(request)
+      .await
+      .map(|u| ExecutionResult::Single(u.into())),
+    Execution::CreateSwarmSecret(request) => client
+      .execute(request)
+      .await
+      .map(|u| ExecutionResult::Single(u.into())),
+    Execution::RotateSwarmSecret(request) => client
+      .execute(request)
+      .await
+      .map(|u| ExecutionResult::Single(u.into())),
+    Execution::RemoveSwarmSecrets(request) => client
+      .execute(request)
+      .await
+      .map(|u| ExecutionResult::Single(u.into())),
     Execution::ClearRepoCache(request) => client
       .execute(request)
       .await

bin/cli/src/command/list.rs

@@ -7,7 +7,7 @@ use komodo_client::{
   api::read::{
     ListActions, ListAlerters, ListBuilders, ListBuilds,
     ListDeployments, ListProcedures, ListRepos, ListResourceSyncs,
-    ListSchedules, ListServers, ListStacks, ListTags,
+    ListSchedules, ListServers, ListStacks, ListTags, ListTerminals,
   },
   entities::{
     ResourceTargetVariant,
@@ -35,6 +35,7 @@ use komodo_client::{
       ResourceSyncListItem, ResourceSyncListItemInfo,
       ResourceSyncState,
     },
+    terminal::Terminal,
   },
 };
 use serde::Serialize;
@@ -74,15 +75,18 @@ pub async fn handle(list: &args::list::List) -> anyhow::Result<()> {
     Some(ListCommand::Syncs(filters)) => {
       list_resources::<ResourceSyncListItem>(filters, false).await
     }
+    Some(ListCommand::Terminals(filters)) => {
+      list_terminals(filters).await
+    }
+    Some(ListCommand::Schedules(filters)) => {
+      list_schedules(filters).await
+    }
     Some(ListCommand::Builders(filters)) => {
       list_resources::<BuilderListItem>(filters, false).await
     }
     Some(ListCommand::Alerters(filters)) => {
       list_resources::<AlerterListItem>(filters, false).await
     }
-    Some(ListCommand::Schedules(filters)) => {
-      list_schedules(filters).await
-    }
   }
 }
 
@@ -189,6 +193,26 @@ where
   Ok(())
 }
 
+async fn list_terminals(
+  filters: &ResourceFilters,
+) -> anyhow::Result<()> {
+  let client = crate::command::komodo_client().await?;
+  // let query = ResourceQuery::builder()
+  //   .tags(filters.tags.clone())
+  //   .templates(TemplatesQueryBehavior::Exclude)
+  //   .build();
+  let terminals = client
+    .read(ListTerminals {
+      target: None,
+      use_names: true,
+    })
+    .await?;
+  if !terminals.is_empty() {
+    print_items(terminals, filters.format, filters.links)?;
+  }
+  Ok(())
+}
+
 async fn list_schedules(
   filters: &ResourceFilters,
 ) -> anyhow::Result<()> {
@@ -1134,6 +1158,28 @@ impl PrintTable for ResourceListItem<AlerterListItemInfo> {
   }
 }
 
+impl PrintTable for Terminal {
+  fn header(_links: bool) -> &'static [&'static str] {
+    &["Terminal", "Target", "Command", "Size", "Created"]
+  }
+  fn row(self, _links: bool) -> Vec<comfy_table::Cell> {
+    vec![
+      Cell::new(self.name).add_attribute(Attribute::Bold),
+      Cell::new(format!("{:?}", self.target)),
+      Cell::new(self.command),
+      Cell::new(if self.stored_size_kb < 1.0 {
+        format!("{:.1} KiB", self.stored_size_kb)
+      } else {
+        format!("{:.} KiB", self.stored_size_kb)
+      }),
+      Cell::new(
+        format_timetamp(self.created_at)
+          .unwrap_or_else(|_| String::from("Invalid created at")),
+      ),
+    ]
+  }
+}
+
 impl PrintTable for Schedule {
   fn header(links: bool) -> &'static [&'static str] {
     if links {
@@ -1146,7 +1192,7 @@ impl PrintTable for Schedule {
     let next_run = if let Some(ts) = self.next_scheduled_run {
       Cell::new(
         format_timetamp(ts)
-          .unwrap_or(String::from("Invalid next ts")),
+          .unwrap_or_else(|_| String::from("Invalid next ts")),
       )
       .add_attribute(Attribute::Bold)
     } else {

bin/cli/src/command/mod.rs

@@ -18,6 +18,7 @@ pub mod container;
 pub mod database;
 pub mod execute;
 pub mod list;
+pub mod terminal;
 pub mod update;
 
 async fn komodo_client() -> anyhow::Result<&'static KomodoClient> {

bin/cli/src/command/terminal.rs

@@ -0,0 +1,334 @@
+use anyhow::{Context, anyhow};
+use colored::Colorize;
+use komodo_client::{
+  api::{
+    read::{ListAllDockerContainers, ListServers},
+    terminal::InitTerminal,
+  },
+  entities::{
+    config::cli::args::terminal::{Attach, Connect, Exec},
+    server::ServerQuery,
+    terminal::{
+      ContainerTerminalMode, TerminalRecreateMode,
+      TerminalResizeMessage, TerminalStdinMessage,
+    },
+  },
+  ws::terminal::TerminalWebsocket,
+};
+use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _};
+use tokio_util::sync::CancellationToken;
+
+pub async fn handle_connect(
+  Connect {
+    server,
+    name,
+    command,
+    recreate,
+  }: &Connect,
+) -> anyhow::Result<()> {
+  handle_terminal_forwarding(async {
+    super::komodo_client()
+      .await?
+      .connect_server_terminal(
+        server.to_string(),
+        Some(name.to_string()),
+        Some(InitTerminal {
+          command: command.clone(),
+          recreate: if *recreate {
+            TerminalRecreateMode::Always
+          } else {
+            TerminalRecreateMode::DifferentCommand
+          },
+          mode: None,
+        }),
+      )
+      .await
+  })
+  .await
+}
+
+pub async fn handle_exec(
+  Exec {
+    server,
+    container,
+    shell,
+    recreate,
+  }: &Exec,
+) -> anyhow::Result<()> {
+  let server = get_server(server.clone(), container).await?;
+  handle_terminal_forwarding(async {
+    super::komodo_client()
+      .await?
+      .connect_container_terminal(
+        server,
+        container.to_string(),
+        None,
+        Some(InitTerminal {
+          command: Some(shell.to_string()),
+          recreate: if *recreate {
+            TerminalRecreateMode::Always
+          } else {
+            TerminalRecreateMode::DifferentCommand
+          },
+          mode: Some(ContainerTerminalMode::Exec),
+        }),
+      )
+      .await
+  })
+  .await
+}
+
+pub async fn handle_attach(
+  Attach {
+    server,
+    container,
+    recreate,
+  }: &Attach,
+) -> anyhow::Result<()> {
+  let server = get_server(server.clone(), container).await?;
+  handle_terminal_forwarding(async {
+    super::komodo_client()
+      .await?
+      .connect_container_terminal(
+        server,
+        container.to_string(),
+        None,
+        Some(InitTerminal {
+          command: None,
+          recreate: if *recreate {
+            TerminalRecreateMode::Always
+          } else {
+            TerminalRecreateMode::DifferentCommand
+          },
+          mode: Some(ContainerTerminalMode::Attach),
+        }),
+      )
+      .await
+  })
+  .await
+}
+
+async fn get_server(
+  server: Option<String>,
+  container: &str,
+) -> anyhow::Result<String> {
+  if let Some(server) = server {
+    return Ok(server);
+  }
+
+  let client = super::komodo_client().await?;
+
+  let mut containers = client
+    .read(ListAllDockerContainers {
+      servers: Default::default(),
+      containers: vec![container.to_string()],
+    })
+    .await?;
+
+  if containers.is_empty() {
+    return Err(anyhow!(
+      "Did not find any container matching {container}"
+    ));
+  }
+
+  if containers.len() == 1 {
+    return containers
+      .pop()
+      .context("Shouldn't happen")?
+      .server_id
+      .context("Container doesn't have server_id");
+  }
+
+  let servers = containers
+    .into_iter()
+    .flat_map(|container| container.server_id)
+    .collect::<Vec<_>>();
+
+  let servers = client
+    .read(ListServers {
+      query: ServerQuery::builder().names(servers).build(),
+    })
+    .await?
+    .into_iter()
+    .map(|server| format!("\t- {}", server.name.bold()))
+    .collect::<Vec<_>>()
+    .join("\n");
+
+  Err(anyhow!(
+    "Multiple containers matching '{}' on Servers:\n{servers}",
+    container.bold(),
+  ))
+}
+
+async fn handle_terminal_forwarding<
+  C: Future<Output = anyhow::Result<TerminalWebsocket>>,
+>(
+  connect: C,
+) -> anyhow::Result<()> {
+  // Need to forward multiple sources into ws write
+  let (write_tx, mut write_rx) =
+    tokio::sync::mpsc::channel::<TerminalStdinMessage>(1024);
+
+  // ================
+  //  SETUP RESIZING
+  // ================
+
+  // Subscribe to SIGWINCH for resize messages
+  let mut sigwinch = tokio::signal::unix::signal(
+    tokio::signal::unix::SignalKind::window_change(),
+  )
+  .context("failed to register SIGWINCH handler")?;
+
+  // Send first resize messsage, bailing if it fails to get the size.
+  write_tx.send(resize_message()?).await?;
+
+  let cancel = CancellationToken::new();
+
+  let forward_resize = async {
+    while future_or_cancel(sigwinch.recv(), &cancel)
+      .await
+      .flatten()
+      .is_some()
+    {
+      if let Ok(resize_message) = resize_message()
+        && write_tx.send(resize_message).await.is_err()
+      {
+        break;
+      }
+    }
+    cancel.cancel();
+  };
+
+  let forward_stdin = async {
+    let mut stdin = tokio::io::stdin();
+    let mut buf = [0u8; 8192];
+    while let Some(Ok(n)) =
+      future_or_cancel(stdin.read(&mut buf), &cancel).await
+    {
+      // EOF
+      if n == 0 {
+        break;
+      }
+      let bytes = &buf[..n];
+      // Check for disconnect sequence (alt + q)
+      if bytes == [197, 147] {
+        break;
+      }
+      // Forward bytes
+      if write_tx
+        .send(TerminalStdinMessage::Forward(bytes.to_vec()))
+        .await
+        .is_err()
+      {
+        break;
+      };
+    }
+    cancel.cancel();
+  };
+
+  // =====================
+  //  CONNECT AND FORWARD
+  // =====================
+
+  let (mut ws_write, mut ws_read) = connect.await?.split();
+
+  let forward_write = async {
+    while let Some(message) =
+      future_or_cancel(write_rx.recv(), &cancel).await.flatten()
+    {
+      if let Err(e) = ws_write.send_stdin_message(message).await {
+        cancel.cancel();
+        return Some(e);
+      };
+    }
+    cancel.cancel();
+    None
+  };
+
+  let forward_read = async {
+    let mut stdout = tokio::io::stdout();
+    while let Some(msg) =
+      future_or_cancel(ws_read.receive_stdout(), &cancel).await
+    {
+      let bytes = match msg {
+        Ok(Some(bytes)) => bytes,
+        Ok(None) => break,
+        Err(e) => {
+          cancel.cancel();
+          return Some(e.context("Websocket read error"));
+        }
+      };
+      if let Err(e) = stdout
+        .write_all(&bytes)
+        .await
+        .context("Failed to write text to stdout")
+      {
+        cancel.cancel();
+        return Some(e);
+      }
+      let _ = stdout.flush().await;
+    }
+    cancel.cancel();
+    None
+  };
+
+  let guard = RawModeGuard::enable_raw_mode()?;
+
+  let (_, _, write_error, read_error) = tokio::join!(
+    forward_resize,
+    forward_stdin,
+    forward_write,
+    forward_read
+  );
+
+  drop(guard);
+
+  if let Some(e) = write_error {
+    eprintln!("\nFailed to forward stdin | {e:#}");
+  }
+
+  if let Some(e) = read_error {
+    eprintln!("\nFailed to forward stdout | {e:#}");
+  }
+
+  println!("\n\n{} {}", "connection".bold(), "closed".red().bold());
+
+  // It doesn't seem to exit by itself after the raw mode stuff.
+  std::process::exit(0)
+}
+
+fn resize_message() -> anyhow::Result<TerminalStdinMessage> {
+  let (cols, rows) = crossterm::terminal::size()
+    .context("Failed to get terminal size")?;
+  Ok(TerminalStdinMessage::Resize(TerminalResizeMessage {
+    rows,
+    cols,
+  }))
+}
+
+struct RawModeGuard;
+
+impl RawModeGuard {
+  fn enable_raw_mode() -> anyhow::Result<Self> {
+    crossterm::terminal::enable_raw_mode()
+      .context("Failed to enable terminal raw mode")?;
+    Ok(Self)
+  }
+}
+impl Drop for RawModeGuard {
+  fn drop(&mut self) {
+    if let Err(e) = crossterm::terminal::disable_raw_mode() {
+      eprintln!("Failed to disable terminal raw mode | {e:?}");
+    }
+  }
+}
+
+async fn future_or_cancel<T, F: Future<Output = T>>(
+  fut: F,
+  cancel: &CancellationToken,
+) -> Option<T> {
+  tokio::select! {
+    res = fut => Some(res),
+    _ = cancel.cancelled() => None
+  }
+}

bin/cli/src/command/update/user.rs

@@ -26,6 +26,9 @@ pub async fn update(
     UpdateUserCommand::SuperAdmin { enabled, yes } => {
       update_super_admin(username, *enabled, *yes).await
     }
+    UpdateUserCommand::Clear2fa { yes } => {
+      clear_2fa(username, *yes).await
+    }
   }
 }
 
@@ -120,3 +123,20 @@ async fn update_super_admin(
 
   Ok(())
 }
+
+async fn clear_2fa(username: &str, yes: bool) -> anyhow::Result<()> {
+  println!("\n{}: Clear 2FA Methods\n", "Mode".dimmed());
+  println!(" - {}: {username}", "Username".dimmed());
+
+  crate::command::wait_for_enter("clear user 2FA methods", yes)?;
+
+  info!("Clearing 2FA methods...");
+
+  let db = database::Client::new(&cli_config().database).await?;
+
+  db.clear_user_2fa_methods(username).await?;
+
+  info!("2FA methods cleared ✅");
+
+  Ok(())
+}

bin/cli/src/main.rs

@@ -2,6 +2,7 @@
 extern crate tracing;
 
 use anyhow::Context;
+use colored::Colorize;
 use komodo_client::entities::config::cli::args;
 
 use crate::config::cli_config;
@@ -41,12 +42,6 @@ async fn app() -> anyhow::Result<()> {
       }
       Ok(())
     }
-    args::Command::Key { command } => {
-      noise::key::command::handle(command).await
-    }
-    args::Command::Database { command } => {
-      command::database::handle(command).await
-    }
     args::Command::Container(container) => {
       command::container::handle(container).await
     }
@@ -60,6 +55,21 @@ async fn app() -> anyhow::Result<()> {
     args::Command::Update { command } => {
       command::update::handle(command).await
     }
+    args::Command::Connect(connect) => {
+      command::terminal::handle_connect(connect).await
+    }
+    args::Command::Exec(exec) => {
+      command::terminal::handle_exec(exec).await
+    }
+    args::Command::Attach(attach) => {
+      command::terminal::handle_attach(attach).await
+    }
+    args::Command::Key { command } => {
+      noise::key::command::handle(command).await
+    }
+    args::Command::Database { command } => {
+      command::database::handle(command).await
+    }
   }
 }
 
@@ -69,7 +79,18 @@ async fn main() -> anyhow::Result<()> {
     tokio::signal::unix::SignalKind::terminate(),
   )?;
   tokio::select! {
-    res = tokio::spawn(app()) => res?,
-    _ = term_signal.recv() => Ok(()),
+    res = tokio::spawn(app()) => match res {
+      Ok(Err(e)) => {
+        eprintln!("{}: {e}", "ERROR".red());
+        std::process::exit(1)
+      }
+      Err(e) => {
+        eprintln!("{}: {e}", "ERROR".red());
+        std::process::exit(1)
+      },
+      Ok(_) => {}
+    },
+    _ = term_signal.recv() => {},
   }
+  Ok(())
 }

bin/core/Cargo.toml

@@ -20,7 +20,9 @@ periphery_client.workspace = true
 environment_file.workspace = true
 interpolate.workspace = true
 secret_file.workspace = true
+validations.workspace = true
 formatting.workspace = true
+rate_limit.workspace = true
 transport.workspace = true
 database.workspace = true
 encoding.workspace = true
@@ -43,33 +45,36 @@ svi.workspace = true
 # external
 aws-credential-types.workspace = true
 english-to-cron.workspace = true
+tower-sessions.workspace = true
 openidconnect.workspace = true
+data-encoding.workspace = true
+serde_yaml_ng.workspace = true
 jsonwebtoken.workspace = true
+futures-util.workspace = true
 axum-server.workspace = true
-urlencoding.workspace = true
 aws-sdk-ec2.workspace = true
+urlencoding.workspace = true
+webauthn-rs.workspace = true
 aws-config.workspace = true
 tokio-util.workspace = true
 axum-extra.workspace = true
 tower-http.workspace = true
 serde_json.workspace = true
-serde_yaml_ng.workspace = true
 typeshare.workspace = true
 chrono-tz.workspace = true
 indexmap.workspace = true
 wildcard.workspace = true
 arc-swap.workspace = true
+serde_qs.workspace = true
 colored.workspace = true
-dashmap.workspace = true
 tracing.workspace = true
 reqwest.workspace = true
-futures.workspace = true
 dotenvy.workspace = true
+totp-rs.workspace = true
 anyhow.workspace = true
 croner.workspace = true
 chrono.workspace = true
 bcrypt.workspace = true
-base64.workspace = true
 rustls.workspace = true
 bytes.workspace = true
 tokio.workspace = true

bin/core/aio.Dockerfile

@@ -1,7 +1,7 @@
 ## All in one, multi stage compile + runtime Docker build for your architecture.
 
 # Build Core
-FROM rust:1.90.0-bullseye AS core-builder
+FROM rust:1.91.1-trixie AS core-builder
 RUN cargo install cargo-strip
 
 WORKDIR /builder
@@ -26,7 +26,7 @@ RUN cd client && yarn && yarn build && yarn link
 RUN cd frontend && yarn link komodo_client && yarn && yarn build
 
 # Final Image
-FROM debian:bullseye-slim
+FROM debian:trixie-slim
 
 COPY ./bin/core/starship.toml /starship.toml
 COPY ./bin/core/debian-deps.sh .
@@ -48,6 +48,9 @@ RUN mkdir /action-cache && \
   cd /action-cache && \
   deno install jsr:@std/yaml jsr:@std/toml
 
+COPY ./bin/entrypoint.sh /usr/local/bin/entrypoint.sh
+RUN chmod +x /usr/local/bin/entrypoint.sh
+
 # Hint at the port
 EXPOSE 9120
 

bin/core/multi-arch.Dockerfile

@@ -13,7 +13,7 @@ FROM ${AARCH64_BINARIES} AS aarch64
 FROM ${FRONTEND_IMAGE} AS frontend
 
 # Final Image
-FROM debian:bullseye-slim
+FROM debian:trixie-slim
 
 COPY ./bin/core/starship.toml /starship.toml
 COPY ./bin/core/debian-deps.sh .
@@ -28,7 +28,7 @@ COPY --from=x86_64 /core /app/core/linux/amd64
 COPY --from=aarch64 /core /app/core/linux/arm64
 RUN mv /app/core/${TARGETPLATFORM} /usr/local/bin/core && rm -r /app/core
 
-# Same for util
+# Same for km
 COPY --from=x86_64 /km /app/km/linux/amd64
 COPY --from=aarch64 /km /app/km/linux/arm64
 RUN mv /app/km/${TARGETPLATFORM} /usr/local/bin/km && rm -r /app/km
@@ -44,6 +44,9 @@ RUN mkdir /action-cache && \
   cd /action-cache && \
   deno install jsr:@std/yaml jsr:@std/toml
 
+COPY ./bin/entrypoint.sh /usr/local/bin/entrypoint.sh
+RUN chmod +x /usr/local/bin/entrypoint.sh
+
 # Hint at the port
 EXPOSE 9120
 
@@ -51,7 +54,8 @@ ENV KOMODO_CLI_CONFIG_PATHS="/config"
 # This ensures any `komodo.cli.*` takes precedence over the Core `/config/*config.*`
 ENV KOMODO_CLI_CONFIG_KEYWORDS="*config.*,*komodo.cli*.*"
 
-CMD [ "/bin/bash", "-c", "update-ca-certificates && core" ]
+ENTRYPOINT [ "entrypoint.sh" ]
+CMD [ "core" ]
 
 # Label to prevent Komodo from stopping with StopAllContainers
 LABEL komodo.skip="true"

bin/core/single-arch.Dockerfile

@@ -14,7 +14,7 @@ COPY ./client/core/ts ./client
 RUN cd client && yarn && yarn build && yarn link
 RUN cd frontend && yarn link komodo_client && yarn && yarn build
 
-FROM debian:bullseye-slim
+FROM debian:trixie-slim
 
 COPY ./bin/core/starship.toml /starship.toml
 COPY ./bin/core/debian-deps.sh .
@@ -33,6 +33,9 @@ RUN mkdir /action-cache && \
 	cd /action-cache && \
 	deno install jsr:@std/yaml jsr:@std/toml
 
+COPY ./bin/entrypoint.sh /usr/local/bin/entrypoint.sh
+RUN chmod +x /usr/local/bin/entrypoint.sh
+
 # Hint at the port
 EXPOSE 9120
 
@@ -40,7 +43,8 @@ ENV KOMODO_CLI_CONFIG_PATHS="/config"
 # This ensures any `komodo.cli.*` takes precedence over the Core `/config/*config.*`
 ENV KOMODO_CLI_CONFIG_KEYWORDS="*config.*,*komodo.cli*.*"
 
-CMD [ "/bin/bash", "-c", "update-ca-certificates && core" ]
+ENTRYPOINT [ "entrypoint.sh" ]
+CMD [ "core" ]
 
 # Label to prevent Komodo from stopping with StopAllContainers
 LABEL komodo.skip="true"

bin/core/src/alert/discord.rs

@@ -232,9 +232,9 @@ pub async fn send_alert(
       format!(
         "{level} | {message}{}",
         if details.is_empty() {
-          format_args!("")
+          String::new()
         } else {
-          format_args!("\n{details}")
+          format!("\n{details}")
         }
       )
     }

bin/core/src/alert/mod.rs

@@ -1,7 +1,7 @@
 use anyhow::{Context, anyhow};
 use database::mungos::{find::find_collect, mongodb::bson::doc};
 use derive_variants::ExtractVariant;
-use futures::future::join_all;
+use futures_util::future::join_all;
 use interpolate::Interpolator;
 use komodo_client::entities::{
   ResourceTargetVariant,
@@ -466,9 +466,9 @@ fn standard_alert_content(alert: &Alert) -> String {
       format!(
         "{level} | {message}{}",
         if details.is_empty() {
-          format_args!("")
+          String::new()
         } else {
-          format_args!("\n{details}")
+          format!("\n{details}")
         }
       )
     }

bin/core/src/api/auth.rs

@@ -1,34 +1,57 @@
-use std::{sync::OnceLock, time::Instant};
+use std::{
+  net::{IpAddr, SocketAddr},
+  sync::OnceLock,
+  time::Instant,
+};
 
-use axum::{Router, extract::Path, http::HeaderMap, routing::post};
+use anyhow::{Context, anyhow};
+use axum::{
+  Router,
+  extract::{ConnectInfo, Path},
+  http::HeaderMap,
+  routing::post,
+};
+use data_encoding::BASE32_NOPAD;
+use database::{
+  bson::{doc, to_bson},
+  mungos::by_id::update_one_by_id,
+};
 use derive_variants::{EnumVariants, ExtractVariant};
 use komodo_client::{api::auth::*, entities::user::User};
+use rate_limit::WithFailureRateLimit;
 use reqwest::StatusCode;
 use resolver_api::Resolve;
 use response::Response;
 use serde::{Deserialize, Serialize};
 use serde_json::json;
-use serror::{AddStatusCode, Json};
+use serror::{AddStatusCode, AddStatusCodeError, Json};
+use tower_sessions::Session;
 use typeshare::typeshare;
 use uuid::Uuid;
 
 use crate::{
   auth::{
+    SessionPasskeyLogin, SessionTotpLogin, SessionUserId,
     get_user_id_from_headers,
     github::{self, client::github_oauth_client},
     google::{self, client::google_oauth_client},
     oidc::{self, client::oidc_client},
+    totp::make_totp,
   },
   config::core_config,
   helpers::query::get_user,
-  state::jwt_client,
+  state::{auth_rate_limiter, db_client, jwt_client, webauthn},
 };
 
 use super::Variant;
 
-#[derive(Default)]
 pub struct AuthArgs {
   pub headers: HeaderMap,
+  /// Prefer extracting IP from headers.
+  /// This IP will be the IP of reverse proxy itself.
+  pub ip: IpAddr,
+  /// Per-client session state
+  pub session: Option<Session>,
 }
 
 #[typeshare]
@@ -46,6 +69,8 @@ pub enum AuthRequest {
   SignUpLocalUser(SignUpLocalUser),
   LoginLocalUser(LoginLocalUser),
   ExchangeForJwt(ExchangeForJwt),
+  CompleteTotpLogin(CompleteTotpLogin),
+  CompletePasskeyLogin(CompletePasskeyLogin),
   GetUser(GetUser),
 }
 
@@ -78,6 +103,8 @@ pub fn router() -> Router {
 
 async fn variant_handler(
   headers: HeaderMap,
+  session: Session,
+  info: ConnectInfo<SocketAddr>,
   Path(Variant { variant }): Path<Variant>,
   Json(params): Json<serde_json::Value>,
 ) -> serror::Result<axum::response::Response> {
@@ -85,11 +112,13 @@ async fn variant_handler(
     "type": variant,
     "params": params,
   }))?;
-  handler(headers, Json(req)).await
+  handler(headers, session, info, Json(req)).await
 }
 
 async fn handler(
   headers: HeaderMap,
+  session: Session,
+  ConnectInfo(info): ConnectInfo<SocketAddr>,
   Json(request): Json<AuthRequest>,
 ) -> serror::Result<axum::response::Response> {
   let timer = Instant::now();
@@ -98,7 +127,13 @@ async fn handler(
     "/auth request {req_id} | METHOD: {:?}",
     request.extract_variant()
   );
-  let res = request.resolve(&AuthArgs { headers }).await;
+  let res = request
+    .resolve(&AuthArgs {
+      headers,
+      ip: info.ip(),
+      session: Some(session),
+    })
+    .await;
   if let Err(e) = &res {
     debug!("/auth request {req_id} | error: {:#}", e.error);
   }
@@ -135,25 +170,184 @@ impl Resolve<AuthArgs> for GetLoginOptions {
 impl Resolve<AuthArgs> for ExchangeForJwt {
   async fn resolve(
     self,
-    _: &AuthArgs,
+    AuthArgs {
+      headers,
+      ip,
+      session,
+    }: &AuthArgs,
   ) -> serror::Result<ExchangeForJwtResponse> {
-    jwt_client()
-      .redeem_exchange_token(&self.token)
-      .await
-      .map_err(Into::into)
+    async {
+      let session = session.as_ref().context(
+        "Method called in invalid context. This should not happen",
+      )?;
+
+      let SessionUserId(user_id) = session
+        .remove(SessionUserId::KEY)
+        .await
+        .context("Internal session type error")?
+        .context("Authentication steps must be completed before JWT can be retrieved")?;
+
+      jwt_client().encode(user_id).map_err(Into::into)
+    }
+    .with_failure_rate_limit_using_headers(
+      auth_rate_limiter(),
+      headers,
+      Some(*ip),
+    )
+    .await
+  }
+}
+
+impl Resolve<AuthArgs> for CompleteTotpLogin {
+  async fn resolve(
+    self,
+    AuthArgs {
+      headers,
+      ip,
+      session,
+    }: &AuthArgs,
+  ) -> serror::Result<CompleteTotpLoginResponse> {
+    async {
+      let session = session.as_ref().context(
+        "Method called in invalid context. This should not happen",
+      )?;
+
+      let SessionTotpLogin { user_id } = session
+        .get(SessionTotpLogin::KEY)
+        .await
+        .context("Internal session type error")?
+        .context(
+          "Totp login has not been initiated for this session",
+        )?;
+
+      let user = get_user(&user_id)
+        .await
+        .status_code(StatusCode::UNAUTHORIZED)?;
+
+      if user.totp.secret.is_empty() {
+        return Err(
+          anyhow!("User is not enrolled in totp")
+            .status_code(StatusCode::BAD_REQUEST),
+        );
+      }
+
+      let secret_bytes = BASE32_NOPAD
+        .decode(user.totp.secret.as_bytes())
+        .context("Failed to decode totp secret to bytes")?;
+
+      let totp = make_totp(secret_bytes, None)?;
+
+      let valid = totp
+        .check_current(&self.code)
+        .context("Failed to check TOTP code validity")?;
+
+      if !valid {
+        return Err(
+          anyhow!("Invalid totp code")
+            .status_code(StatusCode::UNAUTHORIZED),
+        );
+      }
+
+      jwt_client().encode(user_id).map_err(Into::into)
+    }
+    .with_failure_rate_limit_using_headers(
+      auth_rate_limiter(),
+      headers,
+      Some(*ip),
+    )
+    .await
+  }
+}
+
+impl Resolve<AuthArgs> for CompletePasskeyLogin {
+  async fn resolve(
+    self,
+    AuthArgs {
+      headers,
+      ip,
+      session,
+    }: &AuthArgs,
+  ) -> serror::Result<CompletePasskeyLoginResponse> {
+    async {
+      let session = session.as_ref().context(
+        "Method called in invalid context. This should not happen",
+      )?;
+
+      let webauthn = webauthn().context(
+        "No webauthn provider available, invalid KOMODO_HOST config",
+      )?;
+
+      let SessionPasskeyLogin { user_id, state } = session
+        .get(SessionPasskeyLogin::KEY)
+        .await
+        .context("Internal session type error")?
+        .context(
+          "Passkey login has not been initiated for this session",
+        )?;
+
+      // The result of this call must be used to
+      // update the stored passkey info on database.
+      let update = webauthn
+        .finish_passkey_authentication(&self.credential, &state)
+        .context("Failed to validate passkey")?;
+
+      let mut passkey = get_user(&user_id)
+        .await?
+        .passkey
+        .passkey
+        .context("Could not find passkey on database.")?;
+
+      passkey.update_credential(&update);
+
+      let passkey = to_bson(&passkey)
+        .context("Failed to serialize passkey to BSON")?;
+
+      let update = doc! { "$set": { "passkey.passkey": passkey } };
+
+      let _ =
+        update_one_by_id(&db_client().users, &user_id, update, None)
+          .await
+          .context(
+            "Failed to update user passkey on database after login",
+          )
+          .inspect_err(|e| warn!("{e:#}"));
+
+      jwt_client().encode(user_id).map_err(Into::into)
+    }
+    .with_failure_rate_limit_using_headers(
+      auth_rate_limiter(),
+      headers,
+      Some(*ip),
+    )
+    .await
   }
 }
 
 impl Resolve<AuthArgs> for GetUser {
   async fn resolve(
     self,
-    AuthArgs { headers }: &AuthArgs,
+    AuthArgs {
+      headers,
+      ip,
+      session: _,
+    }: &AuthArgs,
   ) -> serror::Result<User> {
-    let user_id = get_user_id_from_headers(headers)
-      .await
-      .status_code(StatusCode::UNAUTHORIZED)?;
-    get_user(&user_id)
-      .await
-      .status_code(StatusCode::UNAUTHORIZED)
+    async {
+      let user_id = get_user_id_from_headers(headers)
+        .await
+        .status_code(StatusCode::UNAUTHORIZED)?;
+      let mut user = get_user(&user_id)
+        .await
+        .status_code(StatusCode::UNAUTHORIZED)?;
+      // Sanitize before sending to client.
+      user.sanitize();
+      Ok(user)
+    }
+    .with_failure_rate_limit_using_headers(
+      auth_rate_limiter(),
+      headers,
+      Some(*ip),
+    )
+    .await
   }
 }

bin/core/src/api/execute/action.rs

@@ -5,7 +5,7 @@ use std::{
 };
 
 use anyhow::Context;
-use command::run_komodo_command;
+use command::run_komodo_standard_command;
 use config::merge_objects;
 use database::mungos::{
   by_id::update_one_by_id, mongodb::bson::to_document,
@@ -141,6 +141,7 @@ impl Resolve<ExecuteArgs> for RunAction {
     }
     .resolve(&UserArgs {
       user: action_user().to_owned(),
+      session: None,
     })
     .await?;
 
@@ -178,7 +179,7 @@ impl Resolve<ExecuteArgs> for RunAction {
       ""
     };
 
-    let mut res = run_komodo_command(
+    let mut res = run_komodo_standard_command(
       // Keep this stage name as is, the UI will find the latest update log by matching the stage name
       "Execute Action",
       None,
@@ -199,6 +200,7 @@ impl Resolve<ExecuteArgs> for RunAction {
     if let Err(e) = (DeleteApiKey { key })
       .resolve(&UserArgs {
         user: action_user().to_owned(),
+        session: None,
       })
       .await
     {

bin/core/src/api/execute/alerter.rs

@@ -1,6 +1,8 @@
 use anyhow::{Context, anyhow};
 use formatting::format_serror;
-use futures::{StreamExt, TryStreamExt, stream::FuturesUnordered};
+use futures_util::{
+  StreamExt, TryStreamExt, stream::FuturesUnordered,
+};
 use komodo_client::{
   api::execute::{SendAlert, TestAlerter},
   entities::{
@@ -102,22 +104,23 @@ impl Resolve<ExecuteArgs> for SendAlert {
     self,
     ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> Result<Self::Response, Self::Error> {
-    let alerters =
-      list_full_for_user::<Alerter>(Default::default(), user, &[])
-        .await?
-        .into_iter()
-        .filter(|a| {
-          a.config.enabled
-            && (self.alerters.is_empty()
-              || self.alerters.contains(&a.name)
-              || self.alerters.contains(&a.id))
-            && (a.config.alert_types.is_empty()
-              || a
-                .config
-                .alert_types
-                .contains(&AlertDataVariant::Custom))
-        })
-        .collect::<Vec<_>>();
+    let alerters = list_full_for_user::<Alerter>(
+      Default::default(),
+      user,
+      PermissionLevel::Read.into(),
+      &[],
+    )
+    .await?
+    .into_iter()
+    .filter(|a| {
+      a.config.enabled
+        && (self.alerters.is_empty()
+          || self.alerters.contains(&a.name)
+          || self.alerters.contains(&a.id))
+        && (a.config.alert_types.is_empty()
+          || a.config.alert_types.contains(&AlertDataVariant::Custom))
+    })
+    .collect::<Vec<_>>();
 
     let alerters = if user.admin {
       alerters

bin/core/src/api/execute/build.rs

@@ -14,12 +14,15 @@ use database::mungos::{
   },
 };
 use formatting::format_serror;
-use futures::future::join_all;
+use futures_util::future::join_all;
 use interpolate::Interpolator;
 use komodo_client::{
-  api::execute::{
-    BatchExecutionResponse, BatchRunBuild, CancelBuild, Deploy,
-    RunBuild,
+  api::{
+    execute::{
+      BatchExecutionResponse, BatchRunBuild, CancelBuild, Deploy,
+      RunBuild,
+    },
+    write::RefreshBuildCache,
   },
   entities::{
     alert::{Alert, AlertData, SeverityLevel},
@@ -41,6 +44,7 @@ use uuid::Uuid;
 
 use crate::{
   alert::send_alerts,
+  api::write::WriteArgs,
   helpers::{
     build_git_token,
     builder::{cleanup_builder_instance, connect_builder_periphery},
@@ -186,7 +190,7 @@ impl Resolve<ExecuteArgs> for RunBuild {
             update.finalize();
             let id = update.id.clone();
             if let Err(e) = update_update(update).await {
-              warn!("failed to modify Update {id} on db | {e:#}");
+              warn!("Failed to modify Update {id} on db | {e:#}");
             }
             if !is_server_builder {
               cancel_clone.cancel();
@@ -215,12 +219,12 @@ impl Resolve<ExecuteArgs> for RunBuild {
       Ok(builder) => builder,
       Err(e) => {
         warn!(
-          "failed to get builder for build {} | {e:#}",
+          "Failed to get Builder for Build {} | {e:#}",
           build.name
         );
         update.logs.push(Log::error(
-          "get builder",
-          format_serror(&e.context("failed to get builder").into()),
+          "Get Builder",
+          format_serror(&e.context("Failed to get Builder").into()),
         ));
         return handle_early_return(
           update, build.id, build.name, false,
@@ -276,7 +280,7 @@ impl Resolve<ExecuteArgs> for RunBuild {
 
       let commit_message = match res {
         Ok(res) => {
-          debug!("finished repo clone");
+          debug!("Finished repo clone");
           update.logs.extend(res.res.logs);
           update.commit_hash =
             res.res.commit_hash.unwrap_or_default().to_string();
@@ -312,10 +316,10 @@ impl Resolve<ExecuteArgs> for RunBuild {
             commit_hash: optional_string(&update.commit_hash),
             // Unused for now
             additional_tags: Default::default(),
-          }) => res.context("failed at call to periphery to build"),
+          }) => res.context("Failed at call to Periphery to build"),
         _ = cancel.cancelled() => {
           info!("Build cancelled during build, cleaning up builder");
-          update.push_error_log("Build cancelled", String::from("user cancelled build during docker build"));
+          update.push_error_log("Build cancelled", String::from("User cancelled build during docker build"));
           cleanup_builder_instance(periphery, cleanup_data, &mut update)
             .await;
           return handle_early_return(update, build.id, build.name, true).await
@@ -328,10 +332,10 @@ impl Resolve<ExecuteArgs> for RunBuild {
           update.logs.extend(logs);
         }
         Err(e) => {
-          warn!("error in build | {e:#}");
+          warn!("Error in build | {e:#}");
           update.push_error_log(
             "Build Error",
-            format_serror(&e.context("failed to build").into()),
+            format_serror(&e.context("Failed to build").into()),
           )
         }
       };
@@ -382,12 +386,15 @@ impl Resolve<ExecuteArgs> for RunBuild {
 
     update_update(update.clone()).await?;
 
+    let Build { id, name, .. } = build;
+
     if update.success {
       // don't hold response up for user
       tokio::spawn(async move {
-        handle_post_build_redeploy(&build.id).await;
+        handle_post_build_redeploy(&id).await;
       });
     } else {
+      let name = name.clone();
       let target = update.target.clone();
       let version = update.version;
       tokio::spawn(async move {
@@ -398,16 +405,22 @@ impl Resolve<ExecuteArgs> for RunBuild {
           resolved_ts: Some(komodo_timestamp()),
           resolved: true,
           level: SeverityLevel::Warning,
-          data: AlertData::BuildFailed {
-            id: build.id,
-            name: build.name,
-            version,
-          },
+          data: AlertData::BuildFailed { id, name, version },
         };
         send_alerts(&[alert]).await
       });
     }
 
+    if let Err(e) = (RefreshBuildCache { build: name })
+      .resolve(&WriteArgs { user: user.clone() })
+      .await
+    {
+      update.push_error_log(
+        "Refresh build cache",
+        format_serror(&e.error.into()),
+      );
+    }
+
     Ok(update.clone())
   }
 }
@@ -565,7 +578,7 @@ impl Resolve<ExecuteArgs> for CancelBuild {
       .await
       {
         warn!(
-          "failed to set CancelBuild Update status Complete after timeout | {e:#}"
+          "Failed to set CancelBuild Update status Complete after timeout | {e:#}"
         )
       }
     });

bin/core/src/api/execute/deployment.rs

@@ -7,7 +7,7 @@ use interpolate::Interpolator;
 use komodo_client::{
   api::execute::*,
   entities::{
-    Version,
+    SwarmOrServer, Version,
     build::{Build, ImageRegistryConfig},
     deployment::{
       Deployment, DeploymentImage, extract_registry_domain,
@@ -16,22 +16,23 @@ use komodo_client::{
     permission::PermissionLevel,
     server::Server,
     update::{Log, Update},
-    user::User,
   },
 };
 use periphery_client::api;
+use reqwest::StatusCode;
 use resolver_api::Resolve;
+use serror::AddStatusCodeError;
 
 use crate::{
   helpers::{
     periphery_client,
     query::{VariablesAndSecrets, get_variables_and_secrets},
     registry_token,
+    swarm::swarm_request,
     update::update_update,
   },
-  monitor::update_cache_for_server,
-  permission::get_check_permissions,
-  resource,
+  monitor::{update_cache_for_server, update_cache_for_swarm},
+  resource::{self, setup_deployment_execution},
   state::action_states,
 };
 
@@ -69,32 +70,6 @@ impl Resolve<ExecuteArgs> for BatchDeploy {
   }
 }
 
-#[instrument("SetupDeploy", skip_all)]
-async fn setup_deployment_execution(
-  deployment: &str,
-  user: &User,
-) -> anyhow::Result<(Deployment, Server)> {
-  let deployment = get_check_permissions::<Deployment>(
-    deployment,
-    user,
-    PermissionLevel::Execute.into(),
-  )
-  .await?;
-
-  if deployment.config.server_id.is_empty() {
-    return Err(anyhow!("Deployment has no Server configured"));
-  }
-
-  let server =
-    resource::get::<Server>(&deployment.config.server_id).await?;
-
-  if !server.config.enabled {
-    return Err(anyhow!("Attached Server is not enabled"));
-  }
-
-  Ok((deployment, server))
-}
-
 impl Resolve<ExecuteArgs> for Deploy {
   #[instrument(
     "Deploy",
@@ -112,8 +87,13 @@ impl Resolve<ExecuteArgs> for Deploy {
     self,
     ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
-    let (mut deployment, server) =
-      setup_deployment_execution(&self.deployment, user).await?;
+    let (mut deployment, swarm_or_server) =
+      setup_deployment_execution(
+        &self.deployment,
+        user,
+        PermissionLevel::Execute.into(),
+      )
+      .await?;
 
     // get the action state for the deployment (or insert default).
     let action_state = action_states()
@@ -223,27 +203,55 @@ impl Resolve<ExecuteArgs> for Deploy {
     update.version = version;
     update_update(update.clone()).await?;
 
-    match periphery_client(&server)
-      .await?
-      .request(api::container::Deploy {
-        deployment,
-        stop_signal: self.stop_signal,
-        stop_time: self.stop_time,
-        registry_token,
-        replacers: secret_replacers.into_iter().collect(),
-      })
-      .await
-    {
-      Ok(log) => update.logs.push(log),
-      Err(e) => {
-        update.push_error_log(
-          "Deploy Container",
-          format_serror(&e.into()),
-        );
+    match swarm_or_server {
+      SwarmOrServer::Swarm(swarm) => {
+        match swarm_request(
+          &swarm.config.server_ids,
+          api::swarm::CreateSwarmService {
+            deployment,
+            registry_token,
+            replacers: secret_replacers.into_iter().collect(),
+          },
+        )
+        .await
+        {
+          Ok(logs) => {
+            update_cache_for_swarm(&swarm, true).await;
+            update.logs.extend(logs)
+          }
+          Err(e) => {
+            update.push_error_log(
+              "Create Swarm Service",
+              format_serror(&e.into()),
+            );
+          }
+        };
       }
-    };
-
-    update_cache_for_server(&server, true).await;
+      SwarmOrServer::Server(server) => {
+        match periphery_client(&server)
+          .await?
+          .request(api::container::RunContainer {
+            deployment,
+            stop_signal: self.stop_signal,
+            stop_time: self.stop_time,
+            registry_token,
+            replacers: secret_replacers.into_iter().collect(),
+          })
+          .await
+        {
+          Ok(log) => {
+            update_cache_for_server(&server, true).await;
+            update.logs.push(log)
+          }
+          Err(e) => {
+            update.push_error_log(
+              "Deploy Container",
+              format_serror(&e.into()),
+            );
+          }
+        };
+      }
+    }
 
     update.finalize();
     update_update(update.clone()).await?;
@@ -400,8 +408,19 @@ impl Resolve<ExecuteArgs> for PullDeployment {
     self,
     ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
-    let (deployment, server) =
-      setup_deployment_execution(&self.deployment, user).await?;
+    let (deployment, swarm_or_server) = setup_deployment_execution(
+      &self.deployment,
+      user,
+      PermissionLevel::Execute.into(),
+    )
+    .await?;
+
+    let SwarmOrServer::Server(server) = swarm_or_server else {
+      return Err(
+        anyhow!("PullDeployment should not be called for Deployment in Swarm Mode")
+          .status_code(StatusCode::BAD_REQUEST),
+      );
+    };
 
     // get the action state for the deployment (or insert default).
     let action_state = action_states()
@@ -443,8 +462,19 @@ impl Resolve<ExecuteArgs> for StartDeployment {
     self,
     ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
-    let (deployment, server) =
-      setup_deployment_execution(&self.deployment, user).await?;
+    let (deployment, swarm_or_server) = setup_deployment_execution(
+      &self.deployment,
+      user,
+      PermissionLevel::Execute.into(),
+    )
+    .await?;
+
+    let SwarmOrServer::Server(server) = swarm_or_server else {
+      return Err(
+        anyhow!("StartDeployment should not be called for Deployment in Swarm Mode")
+          .status_code(StatusCode::BAD_REQUEST),
+      );
+    };
 
     // get the action state for the deployment (or insert default).
     let action_state = action_states()
@@ -500,8 +530,19 @@ impl Resolve<ExecuteArgs> for RestartDeployment {
     self,
     ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
-    let (deployment, server) =
-      setup_deployment_execution(&self.deployment, user).await?;
+    let (deployment, swarm_or_server) = setup_deployment_execution(
+      &self.deployment,
+      user,
+      PermissionLevel::Execute.into(),
+    )
+    .await?;
+
+    let SwarmOrServer::Server(server) = swarm_or_server else {
+      return Err(
+        anyhow!("RestartDeployment should not be called for Deployment in Swarm Mode")
+          .status_code(StatusCode::BAD_REQUEST),
+      );
+    };
 
     // get the action state for the deployment (or insert default).
     let action_state = action_states()
@@ -559,8 +600,19 @@ impl Resolve<ExecuteArgs> for PauseDeployment {
     self,
     ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
-    let (deployment, server) =
-      setup_deployment_execution(&self.deployment, user).await?;
+    let (deployment, swarm_or_server) = setup_deployment_execution(
+      &self.deployment,
+      user,
+      PermissionLevel::Execute.into(),
+    )
+    .await?;
+
+    let SwarmOrServer::Server(server) = swarm_or_server else {
+      return Err(
+        anyhow!("PauseDeployment should not be called for Deployment in Swarm Mode")
+          .status_code(StatusCode::BAD_REQUEST),
+      );
+    };
 
     // get the action state for the deployment (or insert default).
     let action_state = action_states()
@@ -616,8 +668,19 @@ impl Resolve<ExecuteArgs> for UnpauseDeployment {
     self,
     ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
-    let (deployment, server) =
-      setup_deployment_execution(&self.deployment, user).await?;
+    let (deployment, swarm_or_server) = setup_deployment_execution(
+      &self.deployment,
+      user,
+      PermissionLevel::Execute.into(),
+    )
+    .await?;
+
+    let SwarmOrServer::Server(server) = swarm_or_server else {
+      return Err(
+        anyhow!("UnpauseDeployment should not be called for Deployment in Swarm Mode")
+          .status_code(StatusCode::BAD_REQUEST),
+      );
+    };
 
     // get the action state for the deployment (or insert default).
     let action_state = action_states()
@@ -677,8 +740,19 @@ impl Resolve<ExecuteArgs> for StopDeployment {
     self,
     ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
-    let (deployment, server) =
-      setup_deployment_execution(&self.deployment, user).await?;
+    let (deployment, swarm_or_server) = setup_deployment_execution(
+      &self.deployment,
+      user,
+      PermissionLevel::Execute.into(),
+    )
+    .await?;
+
+    let SwarmOrServer::Server(server) = swarm_or_server else {
+      return Err(
+        anyhow!("StopDeployment should not be called for Deployment in Swarm Mode")
+          .status_code(StatusCode::BAD_REQUEST),
+      );
+    };
 
     // get the action state for the deployment (or insert default).
     let action_state = action_states()
@@ -779,8 +853,12 @@ impl Resolve<ExecuteArgs> for DestroyDeployment {
     self,
     ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
-    let (deployment, server) =
-      setup_deployment_execution(&self.deployment, user).await?;
+    let (deployment, swarm_or_server) = setup_deployment_execution(
+      &self.deployment,
+      user,
+      PermissionLevel::Execute.into(),
+    )
+    .await?;
 
     // get the action state for the deployment (or insert default).
     let action_state = action_states()
@@ -798,31 +876,61 @@ impl Resolve<ExecuteArgs> for DestroyDeployment {
     // Send update after setting action state, this way frontend gets correct state.
     update_update(update.clone()).await?;
 
-    let log = match periphery_client(&server)
-      .await?
-      .request(api::container::RemoveContainer {
-        name: deployment.name,
-        signal: self
-          .signal
-          .unwrap_or(deployment.config.termination_signal)
-          .into(),
-        time: self
-          .time
-          .unwrap_or(deployment.config.termination_timeout)
-          .into(),
-      })
-      .await
-    {
-      Ok(log) => log,
-      Err(e) => Log::error(
-        "stop container",
-        format_serror(&e.context("failed to stop container").into()),
-      ),
+    let log = match swarm_or_server {
+      SwarmOrServer::Swarm(swarm) => {
+        match swarm_request(
+          &swarm.config.server_ids,
+          api::swarm::RemoveSwarmServices {
+            services: vec![deployment.name],
+          },
+        )
+        .await
+        {
+          Ok(log) => {
+            update_cache_for_swarm(&swarm, true).await;
+            log
+          }
+          Err(e) => Log::error(
+            "Remove Swarm Service",
+            format_serror(
+              &e.context("Failed to remove swarm service").into(),
+            ),
+          ),
+        }
+      }
+      SwarmOrServer::Server(server) => {
+        match periphery_client(&server)
+          .await?
+          .request(api::container::RemoveContainer {
+            name: deployment.name,
+            signal: self
+              .signal
+              .unwrap_or(deployment.config.termination_signal)
+              .into(),
+            time: self
+              .time
+              .unwrap_or(deployment.config.termination_timeout)
+              .into(),
+          })
+          .await
+        {
+          Ok(log) => {
+            update_cache_for_server(&server, true).await;
+            log
+          }
+          Err(e) => Log::error(
+            "Destroy Container",
+            format_serror(
+              &e.context("Failed to destroy container").into(),
+            ),
+          ),
+        }
+      }
     };
 
     update.logs.push(log);
     update.finalize();
-    update_cache_for_server(&server, true).await;
+
     update_update(update.clone()).await?;
 
     Ok(update)

bin/core/src/api/execute/maintenance.rs

@@ -1,13 +1,13 @@
 use std::{fmt::Write as _, sync::OnceLock};
 
 use anyhow::{Context, anyhow};
-use command::run_komodo_command;
+use command::run_komodo_standard_command;
 use database::{
   bson::{Document, doc},
   mungos::find::find_collect,
 };
 use formatting::{bold, format_serror};
-use futures::{StreamExt, stream::FuturesOrdered};
+use futures_util::{StreamExt, stream::FuturesOrdered};
 use komodo_client::{
   api::execute::{
     BackupCoreDatabase, ClearRepoCache, GlobalAutoUpdate,
@@ -151,7 +151,7 @@ impl Resolve<ExecuteArgs> for BackupCoreDatabase {
 
     update_update(update.clone()).await?;
 
-    let res = run_komodo_command(
+    let res = run_komodo_standard_command(
       "Backup Core Database",
       None,
       "km database backup --yes",

bin/core/src/api/execute/mod.rs

@@ -8,9 +8,7 @@ use axum_extra::{TypedHeader, headers::ContentType};
 use database::mungos::by_id::find_one_by_id;
 use derive_variants::{EnumVariants, ExtractVariant};
 use formatting::format_serror;
-use futures::{
-  StreamExt as _, future::join_all, stream::FuturesUnordered,
-};
+use futures_util::future::join_all;
 use komodo_client::{
   api::execute::*,
   entities::{
@@ -32,7 +30,6 @@ use uuid::Uuid;
 use crate::{
   auth::auth_request,
   helpers::update::{init_execution_update, update_update},
-  permission::get_check_permissions,
   resource::{KomodoResource, list_full_for_user_using_pattern},
   state::db_client,
 };
@@ -46,6 +43,7 @@ mod procedure;
 mod repo;
 mod server;
 mod stack;
+mod swarm;
 mod sync;
 
 use super::Variant;
@@ -72,29 +70,6 @@ pub struct ExecuteArgs {
 #[error(serror::Error)]
 #[serde(tag = "type", content = "params")]
 pub enum ExecuteRequest {
-  // ==== SERVER ====
-  StartContainer(StartContainer),
-  RestartContainer(RestartContainer),
-  PauseContainer(PauseContainer),
-  UnpauseContainer(UnpauseContainer),
-  StopContainer(StopContainer),
-  DestroyContainer(DestroyContainer),
-  StartAllContainers(StartAllContainers),
-  RestartAllContainers(RestartAllContainers),
-  PauseAllContainers(PauseAllContainers),
-  UnpauseAllContainers(UnpauseAllContainers),
-  StopAllContainers(StopAllContainers),
-  PruneContainers(PruneContainers),
-  DeleteNetwork(DeleteNetwork),
-  PruneNetworks(PruneNetworks),
-  DeleteImage(DeleteImage),
-  PruneImages(PruneImages),
-  DeleteVolume(DeleteVolume),
-  PruneVolumes(PruneVolumes),
-  PruneDockerBuilders(PruneDockerBuilders),
-  PruneBuildx(PruneBuildx),
-  PruneSystem(PruneSystem),
-
   // ==== STACK ====
   DeployStack(DeployStack),
   BatchDeployStack(BatchDeployStack),
@@ -145,12 +120,46 @@ pub enum ExecuteRequest {
   RunAction(RunAction),
   BatchRunAction(BatchRunAction),
 
+  // ==== SYNC ====
+  RunSync(RunSync),
+
   // ==== ALERTER ====
   TestAlerter(TestAlerter),
   SendAlert(SendAlert),
 
-  // ==== SYNC ====
-  RunSync(RunSync),
+  // ==== SERVER ====
+  StartContainer(StartContainer),
+  RestartContainer(RestartContainer),
+  PauseContainer(PauseContainer),
+  UnpauseContainer(UnpauseContainer),
+  StopContainer(StopContainer),
+  DestroyContainer(DestroyContainer),
+  StartAllContainers(StartAllContainers),
+  RestartAllContainers(RestartAllContainers),
+  PauseAllContainers(PauseAllContainers),
+  UnpauseAllContainers(UnpauseAllContainers),
+  StopAllContainers(StopAllContainers),
+  PruneContainers(PruneContainers),
+  DeleteNetwork(DeleteNetwork),
+  PruneNetworks(PruneNetworks),
+  DeleteImage(DeleteImage),
+  PruneImages(PruneImages),
+  DeleteVolume(DeleteVolume),
+  PruneVolumes(PruneVolumes),
+  PruneDockerBuilders(PruneDockerBuilders),
+  PruneBuildx(PruneBuildx),
+  PruneSystem(PruneSystem),
+
+  // ==== SWARM ====
+  RemoveSwarmNodes(RemoveSwarmNodes),
+  RemoveSwarmStacks(RemoveSwarmStacks),
+  RemoveSwarmServices(RemoveSwarmServices),
+  CreateSwarmConfig(CreateSwarmConfig),
+  RotateSwarmConfig(RotateSwarmConfig),
+  RemoveSwarmConfigs(RemoveSwarmConfigs),
+  CreateSwarmSecret(CreateSwarmSecret),
+  RotateSwarmSecret(RotateSwarmSecret),
+  RemoveSwarmSecrets(RemoveSwarmSecrets),
 
   // ==== MAINTENANCE ====
   ClearRepoCache(ClearRepoCache),
@@ -324,32 +333,11 @@ async fn batch_execute<E: BatchExecute>(
     pattern,
     Default::default(),
     user,
+    PermissionLevel::Execute.into(),
     &[],
   )
   .await?;
 
-  let resources = if user.admin {
-    resources
-  } else {
-    // Only keep resources with execute permissions
-    resources
-      .into_iter()
-      .map(|resource| async move {
-        get_check_permissions::<E::Resource>(
-          &resource.id,
-          user,
-          PermissionLevel::Execute.into(),
-        )
-        .await
-      })
-      .collect::<FuturesUnordered<_>>()
-      .collect::<Vec<_>>()
-      .await
-      .into_iter()
-      .flatten()
-      .collect()
-  };
-
   let futures = resources.into_iter().map(|resource| {
     let user = user.clone();
     async move {

bin/core/src/api/execute/server.rs

@@ -70,8 +70,8 @@ impl Resolve<ExecuteArgs> for StartContainer {
     {
       Ok(log) => log,
       Err(e) => Log::error(
-        "start container",
-        format_serror(&e.context("failed to start container").into()),
+        "Start Container",
+        format_serror(&e.context("Failed to start container").into()),
       ),
     };
 
@@ -134,9 +134,9 @@ impl Resolve<ExecuteArgs> for RestartContainer {
     {
       Ok(log) => log,
       Err(e) => Log::error(
-        "restart container",
+        "Restart Container",
         format_serror(
-          &e.context("failed to restart container").into(),
+          &e.context("Failed to restart container").into(),
         ),
       ),
     };
@@ -200,8 +200,8 @@ impl Resolve<ExecuteArgs> for PauseContainer {
     {
       Ok(log) => log,
       Err(e) => Log::error(
-        "pause container",
-        format_serror(&e.context("failed to pause container").into()),
+        "Pause Container",
+        format_serror(&e.context("Failed to pause container").into()),
       ),
     };
 
@@ -264,9 +264,9 @@ impl Resolve<ExecuteArgs> for UnpauseContainer {
     {
       Ok(log) => log,
       Err(e) => Log::error(
-        "unpause container",
+        "Unpause Container",
         format_serror(
-          &e.context("failed to unpause container").into(),
+          &e.context("Failed to unpause container").into(),
         ),
       ),
     };
@@ -334,8 +334,8 @@ impl Resolve<ExecuteArgs> for StopContainer {
     {
       Ok(log) => log,
       Err(e) => Log::error(
-        "stop container",
-        format_serror(&e.context("failed to stop container").into()),
+        "Stop Container",
+        format_serror(&e.context("Failed to stop container").into()),
       ),
     };
 
@@ -408,8 +408,10 @@ impl Resolve<ExecuteArgs> for DestroyContainer {
     {
       Ok(log) => log,
       Err(e) => Log::error(
-        "stop container",
-        format_serror(&e.context("failed to stop container").into()),
+        "Remove Container",
+        format_serror(
+          &e.context("Failed to remove container").into(),
+        ),
       ),
     };
 
@@ -464,13 +466,13 @@ impl Resolve<ExecuteArgs> for StartAllContainers {
       .await?
       .request(api::container::StartAllContainers {})
       .await
-      .context("failed to start all containers on host")?;
+      .context("Failed to start all containers on host")?;
 
     update.logs.extend(logs);
 
     if all_logs_success(&update.logs) {
       update.push_simple_log(
-        "start all containers",
+        "Start All Containers",
         String::from("All containers have been started on the host."),
       );
     }
@@ -524,13 +526,13 @@ impl Resolve<ExecuteArgs> for RestartAllContainers {
       .await?
       .request(api::container::RestartAllContainers {})
       .await
-      .context("failed to restart all containers on host")?;
+      .context("Failed to restart all containers on host")?;
 
     update.logs.extend(logs);
 
     if all_logs_success(&update.logs) {
       update.push_simple_log(
-        "restart all containers",
+        "Restart All Containers",
         String::from(
           "All containers have been restarted on the host.",
         ),
@@ -586,13 +588,13 @@ impl Resolve<ExecuteArgs> for PauseAllContainers {
       .await?
       .request(api::container::PauseAllContainers {})
       .await
-      .context("failed to pause all containers on host")?;
+      .context("Failed to pause all containers on host")?;
 
     update.logs.extend(logs);
 
     if all_logs_success(&update.logs) {
       update.push_simple_log(
-        "pause all containers",
+        "Pause All Containers",
         String::from("All containers have been paused on the host."),
       );
     }
@@ -646,13 +648,13 @@ impl Resolve<ExecuteArgs> for UnpauseAllContainers {
       .await?
       .request(api::container::UnpauseAllContainers {})
       .await
-      .context("failed to unpause all containers on host")?;
+      .context("Failed to unpause all containers on host")?;
 
     update.logs.extend(logs);
 
     if all_logs_success(&update.logs) {
       update.push_simple_log(
-        "unpause all containers",
+        "Unpause All Containers",
         String::from(
           "All containers have been unpaused on the host.",
         ),
@@ -708,13 +710,13 @@ impl Resolve<ExecuteArgs> for StopAllContainers {
       .await?
       .request(api::container::StopAllContainers {})
       .await
-      .context("failed to stop all containers on host")?;
+      .context("Failed to stop all containers on host")?;
 
     update.logs.extend(logs);
 
     if all_logs_success(&update.logs) {
       update.push_simple_log(
-        "stop all containers",
+        "Stop All Containers",
         String::from("All containers have been stopped on the host."),
       );
     }
@@ -770,14 +772,14 @@ impl Resolve<ExecuteArgs> for PruneContainers {
       .request(api::container::PruneContainers {})
       .await
       .context(format!(
-        "failed to prune containers on server {}",
+        "Failed to prune containers on server {}",
         server.name
       )) {
       Ok(log) => log,
       Err(e) => Log::error(
-        "prune containers",
+        "Prune Containers",
         format_serror(
-          &e.context("failed to prune containers").into(),
+          &e.context("Failed to prune containers").into(),
         ),
       ),
     };
@@ -827,15 +829,15 @@ impl Resolve<ExecuteArgs> for DeleteNetwork {
       })
       .await
       .context(format!(
-        "failed to delete network {} on server {}",
+        "Failed to delete network {} on server {}",
         self.name, server.name
       )) {
       Ok(log) => log,
       Err(e) => Log::error(
-        "delete network",
+        "Delete Network",
         format_serror(
           &e.context(format!(
-            "failed to delete network {}",
+            "Failed to delete network {}",
             self.name
           ))
           .into(),
@@ -896,13 +898,13 @@ impl Resolve<ExecuteArgs> for PruneNetworks {
       .request(api::docker::PruneNetworks {})
       .await
       .context(format!(
-        "failed to prune networks on server {}",
+        "Failed to prune networks on server {}",
         server.name
       )) {
       Ok(log) => log,
       Err(e) => Log::error(
-        "prune networks",
-        format_serror(&e.context("failed to prune networks").into()),
+        "Prune Networks",
+        format_serror(&e.context("Failed to prune networks").into()),
       ),
     };
 
@@ -951,14 +953,14 @@ impl Resolve<ExecuteArgs> for DeleteImage {
       })
       .await
       .context(format!(
-        "failed to delete image {} on server {}",
+        "Failed to delete image {} on server {}",
         self.name, server.name
       )) {
       Ok(log) => log,
       Err(e) => Log::error(
         "delete image",
         format_serror(
-          &e.context(format!("failed to delete image {}", self.name))
+          &e.context(format!("Failed to delete image {}", self.name))
             .into(),
         ),
       ),
@@ -1017,9 +1019,9 @@ impl Resolve<ExecuteArgs> for PruneImages {
       match periphery.request(api::docker::PruneImages {}).await {
         Ok(log) => log,
         Err(e) => Log::error(
-          "prune images",
+          "Prune Images",
           format!(
-            "failed to prune images on server {} | {e:#?}",
+            "Failed to prune images on server {} | {e:#?}",
             server.name
           ),
         ),
@@ -1070,7 +1072,7 @@ impl Resolve<ExecuteArgs> for DeleteVolume {
       })
       .await
       .context(format!(
-        "failed to delete volume {} on server {}",
+        "Failed to delete volume {} on server {}",
         self.name, server.name
       )) {
       Ok(log) => log,
@@ -1078,7 +1080,7 @@ impl Resolve<ExecuteArgs> for DeleteVolume {
         "delete volume",
         format_serror(
           &e.context(format!(
-            "failed to delete volume {}",
+            "Failed to delete volume {}",
             self.name
           ))
           .into(),
@@ -1139,9 +1141,9 @@ impl Resolve<ExecuteArgs> for PruneVolumes {
       match periphery.request(api::docker::PruneVolumes {}).await {
         Ok(log) => log,
         Err(e) => Log::error(
-          "prune volumes",
+          "Prune Volumes",
           format!(
-            "failed to prune volumes on server {} | {e:#?}",
+            "Failed to prune volumes on server {} | {e:#?}",
             server.name
           ),
         ),
@@ -1200,9 +1202,9 @@ impl Resolve<ExecuteArgs> for PruneDockerBuilders {
       match periphery.request(api::build::PruneBuilders {}).await {
         Ok(log) => log,
         Err(e) => Log::error(
-          "prune builders",
+          "Prune Builders",
           format!(
-            "failed to docker builder prune on server {} | {e:#?}",
+            "Failed to docker builder prune on server {} | {e:#?}",
             server.name
           ),
         ),
@@ -1261,9 +1263,9 @@ impl Resolve<ExecuteArgs> for PruneBuildx {
       match periphery.request(api::build::PruneBuildx {}).await {
         Ok(log) => log,
         Err(e) => Log::error(
-          "prune buildx",
+          "Prune Buildx",
           format!(
-            "failed to docker buildx prune on server {} | {e:#?}",
+            "Failed to docker buildx prune on server {} | {e:#?}",
             server.name
           ),
         ),
@@ -1321,9 +1323,9 @@ impl Resolve<ExecuteArgs> for PruneSystem {
     let log = match periphery.request(api::PruneSystem {}).await {
       Ok(log) => log,
       Err(e) => Log::error(
-        "prune system",
+        "Prune System",
         format!(
-          "failed to docker system prune on server {} | {e:#?}",
+          "Failed to docker system prune on server {} | {e:#?}",
           server.name
         ),
       ),

bin/core/src/api/execute/stack.rs

@@ -1,6 +1,6 @@
 use std::{collections::HashSet, str::FromStr};
 
-use anyhow::Context;
+use anyhow::{Context, anyhow};
 use database::mungos::mongodb::bson::{
   doc, oid::ObjectId, to_bson, to_document,
 };
@@ -9,7 +9,7 @@ use interpolate::Interpolator;
 use komodo_client::{
   api::{execute::*, write::RefreshStackCache},
   entities::{
-    FileContents,
+    FileContents, SwarmOrServer,
     permission::PermissionLevel,
     repo::Repo,
     server::Server,
@@ -20,8 +20,12 @@ use komodo_client::{
     user::User,
   },
 };
-use periphery_client::api::compose::*;
+use periphery_client::api::{
+  DeployStackResponse, compose::*, swarm::DeploySwarmStack,
+};
+use reqwest::StatusCode;
 use resolver_api::Resolve;
+use serror::AddStatusCodeError as _;
 use uuid::Uuid;
 
 use crate::{
@@ -30,14 +34,20 @@ use crate::{
     periphery_client,
     query::{VariablesAndSecrets, get_variables_and_secrets},
     stack_git_token,
+    swarm::swarm_request,
     update::{
       add_update_without_send, init_execution_update, update_update,
     },
   },
-  monitor::update_cache_for_server,
+  monitor::{update_cache_for_server, update_cache_for_swarm},
   permission::get_check_permissions,
   resource,
-  stack::{execute::execute_compose, get_stack_and_server},
+  stack::{
+    execute::{
+      execute_compose, execute_compose_with_stack_and_server,
+    },
+    setup_stack_execution,
+  },
   state::{action_states, db_client},
 };
 
@@ -92,11 +102,10 @@ impl Resolve<ExecuteArgs> for DeployStack {
     self,
     ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
-    let (mut stack, server) = get_stack_and_server(
+    let (mut stack, swarm_or_server) = setup_stack_execution(
       &self.stack,
       user,
       PermissionLevel::Execute.into(),
-      true,
     )
     .await?;
 
@@ -165,27 +174,44 @@ impl Resolve<ExecuteArgs> for DeployStack {
       Default::default()
     };
 
-    let ComposeUpResponse {
+    let DeployStackResponse {
       logs,
       deployed,
       services,
       file_contents,
       missing_files,
       remote_errors,
-      compose_config,
+      merged_config,
       commit_hash,
       commit_message,
-    } = periphery_client(&server)
-      .await?
-      .request(ComposeUp {
-        stack: stack.clone(),
-        services: self.services,
-        repo,
-        git_token,
-        registry_token,
-        replacers: secret_replacers.into_iter().collect(),
-      })
-      .await?;
+    } = match &swarm_or_server {
+      SwarmOrServer::Swarm(swarm) => {
+        swarm_request(
+          &swarm.config.server_ids,
+          DeploySwarmStack {
+            stack: stack.clone(),
+            repo,
+            git_token,
+            registry_token,
+            replacers: secret_replacers.into_iter().collect(),
+          },
+        )
+        .await?
+      }
+      SwarmOrServer::Server(server) => {
+        periphery_client(server)
+          .await?
+          .request(ComposeUp {
+            stack: stack.clone(),
+            services: self.services,
+            repo,
+            git_token,
+            registry_token,
+            replacers: secret_replacers.into_iter().collect(),
+          })
+          .await?
+      }
+    };
 
     update.logs.extend(logs);
 
@@ -219,7 +245,7 @@ impl Resolve<ExecuteArgs> for DeployStack {
               })
               .collect(),
           ),
-          compose_config,
+          merged_config,
           commit_hash.clone(),
           commit_message.clone(),
         )
@@ -257,7 +283,7 @@ impl Resolve<ExecuteArgs> for DeployStack {
       };
 
       let info = to_document(&info)
-        .context("failed to serialize stack info to bson")?;
+        .context("Failed to serialize stack info to bson")?;
 
       db_client()
         .stacks
@@ -266,22 +292,29 @@ impl Resolve<ExecuteArgs> for DeployStack {
           doc! { "$set": { "info": info } },
         )
         .await
-        .context("failed to update stack info on db")?;
+        .context("Failed to update stack info on db")?;
       anyhow::Ok(())
     };
 
     // This will be weird with single service deploys. Come back to it.
     if let Err(e) = update_info.await {
       update.push_error_log(
-        "refresh stack info",
+        "Refresh Stack Info",
         format_serror(
-          &e.context("failed to refresh stack info on db").into(),
+          &e.context("Failed to refresh stack info on db").into(),
         ),
       )
     }
 
     // Ensure cached stack state up to date by updating server cache
-    update_cache_for_server(&server, true).await;
+    match swarm_or_server {
+      SwarmOrServer::Swarm(swarm) => {
+        update_cache_for_swarm(&swarm, true).await;
+      }
+      SwarmOrServer::Server(server) => {
+        update_cache_for_server(&server, true).await;
+      }
+    }
 
     update.finalize();
     update_update(update.clone()).await?;
@@ -862,14 +895,22 @@ impl Resolve<ExecuteArgs> for PullStack {
     self,
     ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
-    let (stack, server) = get_stack_and_server(
+    let (stack, swarm_or_server) = setup_stack_execution(
       &self.stack,
       user,
       PermissionLevel::Execute.into(),
-      true,
     )
     .await?;
 
+    let SwarmOrServer::Server(server) = swarm_or_server else {
+      return Err(
+        anyhow!(
+          "PullStack should not be called for Stack in Swarm Mode"
+        )
+        .status_code(StatusCode::BAD_REQUEST),
+      );
+    };
+
     let repo = if !stack.config.files_on_host
       && !stack.config.linked_repo.is_empty()
     {
@@ -1106,16 +1147,77 @@ impl Resolve<ExecuteArgs> for DestroyStack {
     self,
     ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
-    execute_compose::<DestroyStack>(
+    let (stack, swarm_or_server) = setup_stack_execution(
       &self.stack,
-      self.services,
       user,
-      |state| state.destroying = true,
-      update.clone(),
-      (self.stop_time, self.remove_orphans),
+      PermissionLevel::Execute.into(),
     )
-    .await
-    .map_err(Into::into)
+    .await?;
+
+    match swarm_or_server {
+      SwarmOrServer::Swarm(swarm) => {
+        if !self.services.is_empty() {
+          return Err(
+            anyhow!("Cannot destroy specific Stack services when in Swarm mode.")
+              .status_code(StatusCode::BAD_REQUEST)
+          );
+        }
+
+        // get the action state for the stack (or insert default).
+        let action_state = action_states()
+          .stack
+          .get_or_insert_default(&stack.id)
+          .await;
+
+        // Will check to ensure stack not already busy before updating, and return Err if so.
+        // The returned guard will set the action state back to default when dropped.
+        let _action_guard =
+          action_state.update(|state| state.destroying = true)?;
+
+        let mut update = update.clone();
+        // Send update here for frontend to recheck action state
+        update_update(update.clone()).await?;
+
+        match swarm_request(
+          &swarm.config.server_ids,
+          periphery_client::api::swarm::RemoveSwarmStacks {
+            stacks: vec![stack.project_name(false)],
+            detach: false,
+          },
+        )
+        .await
+        {
+          Ok(log) => update.logs.push(log),
+          Err(e) => update.push_error_log(
+            "Destroy Stack",
+            format_serror(
+              &e.context("Failed to 'docker stack rm' on swarm")
+                .into(),
+            ),
+          ),
+        }
+
+        // Ensure cached stack state up to date by updating swarm cache
+        update_cache_for_swarm(&swarm, true).await;
+
+        update.finalize();
+        update_update(update.clone()).await?;
+
+        Ok(update)
+      }
+      SwarmOrServer::Server(server) => {
+        execute_compose_with_stack_and_server::<DestroyStack>(
+          stack,
+          server,
+          self.services,
+          |state| state.destroying = true,
+          update.clone(),
+          (self.stop_time, self.remove_orphans),
+        )
+        .await
+        .map_err(Into::into)
+      }
+    }
   }
 }
 
@@ -1136,14 +1238,22 @@ impl Resolve<ExecuteArgs> for RunStackService {
     self,
     ExecuteArgs { user, update, id }: &ExecuteArgs,
   ) -> serror::Result<Update> {
-    let (mut stack, server) = get_stack_and_server(
+    let (mut stack, swarm_or_server) = setup_stack_execution(
       &self.stack,
       user,
       PermissionLevel::Execute.into(),
-      true,
     )
     .await?;
 
+    let SwarmOrServer::Server(server) = swarm_or_server else {
+      return Err(
+        anyhow!(
+          "RunStackService should not be called for Stack in Swarm Mode"
+        )
+        .status_code(StatusCode::BAD_REQUEST),
+      );
+    };
+
     let mut repo = if !stack.config.files_on_host
       && !stack.config.linked_repo.is_empty()
     {

bin/core/src/api/execute/swarm.rs

@@ -0,0 +1,516 @@
+use formatting::format_serror;
+use komodo_client::{
+  api::execute::{
+    CreateSwarmConfig, CreateSwarmSecret, RemoveSwarmConfigs,
+    RemoveSwarmNodes, RemoveSwarmSecrets, RemoveSwarmServices,
+    RemoveSwarmStacks, RotateSwarmConfig, RotateSwarmSecret,
+  },
+  entities::{permission::PermissionLevel, swarm::Swarm},
+};
+use resolver_api::Resolve;
+
+use crate::{
+  api::execute::ExecuteArgs,
+  helpers::{swarm::swarm_request, update::update_update},
+  monitor::update_cache_for_swarm,
+  permission::get_check_permissions,
+};
+
+impl Resolve<ExecuteArgs> for RemoveSwarmNodes {
+  #[instrument(
+    "RemoveSwarmNodes",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      swarm = self.swarm,
+      nodes = serde_json::to_string(&self.nodes).unwrap_or_else(|e| e.to_string()),
+      force = self.force,
+    )
+  )]
+  async fn resolve(
+    self,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
+  ) -> Result<Self::Response, Self::Error> {
+    let swarm = get_check_permissions::<Swarm>(
+      &self.swarm,
+      user,
+      PermissionLevel::Execute.into(),
+    )
+    .await?;
+
+    update_update(update.clone()).await?;
+
+    let mut update = update.clone();
+
+    match swarm_request(
+      &swarm.config.server_ids,
+      periphery_client::api::swarm::RemoveSwarmNodes {
+        nodes: self.nodes,
+        force: self.force,
+      },
+    )
+    .await
+    {
+      Ok(log) => {
+        update.logs.push(log);
+        update_cache_for_swarm(&swarm, true).await;
+      }
+      Err(e) => update.push_error_log(
+        "Remove Swarm Nodes",
+        format_serror(
+          &e.context("Failed to remove swarm nodes").into(),
+        ),
+      ),
+    };
+
+    update.finalize();
+    update_update(update.clone()).await?;
+
+    Ok(update)
+  }
+}
+
+impl Resolve<ExecuteArgs> for RemoveSwarmStacks {
+  #[instrument(
+    "RemoveSwarmStacks",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      swarm = self.swarm,
+      stacks = serde_json::to_string(&self.stacks).unwrap_or_else(|e| e.to_string()),
+      detach = self.detach,
+    )
+  )]
+  async fn resolve(
+    self,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
+  ) -> Result<Self::Response, Self::Error> {
+    let swarm = get_check_permissions::<Swarm>(
+      &self.swarm,
+      user,
+      PermissionLevel::Execute.into(),
+    )
+    .await?;
+
+    update_update(update.clone()).await?;
+
+    let mut update = update.clone();
+
+    match swarm_request(
+      &swarm.config.server_ids,
+      periphery_client::api::swarm::RemoveSwarmStacks {
+        stacks: self.stacks,
+        detach: self.detach,
+      },
+    )
+    .await
+    {
+      Ok(log) => {
+        update.logs.push(log);
+        update_cache_for_swarm(&swarm, true).await;
+      }
+      Err(e) => update.push_error_log(
+        "Remove Swarm Stacks",
+        format_serror(
+          &e.context("Failed to remove swarm stacks").into(),
+        ),
+      ),
+    };
+
+    update.finalize();
+    update_update(update.clone()).await?;
+
+    Ok(update)
+  }
+}
+
+impl Resolve<ExecuteArgs> for RemoveSwarmServices {
+  #[instrument(
+    "RemoveSwarmServices",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      swarm = self.swarm,
+      services = serde_json::to_string(&self.services).unwrap_or_else(|e| e.to_string()),
+    )
+  )]
+  async fn resolve(
+    self,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
+  ) -> Result<Self::Response, Self::Error> {
+    let swarm = get_check_permissions::<Swarm>(
+      &self.swarm,
+      user,
+      PermissionLevel::Execute.into(),
+    )
+    .await?;
+
+    update_update(update.clone()).await?;
+
+    let mut update = update.clone();
+
+    match swarm_request(
+      &swarm.config.server_ids,
+      periphery_client::api::swarm::RemoveSwarmServices {
+        services: self.services,
+      },
+    )
+    .await
+    {
+      Ok(log) => {
+        update.logs.push(log);
+        update_cache_for_swarm(&swarm, true).await;
+      }
+      Err(e) => update.push_error_log(
+        "Remove Swarm Services",
+        format_serror(
+          &e.context("Failed to remove swarm services").into(),
+        ),
+      ),
+    };
+
+    update.finalize();
+    update_update(update.clone()).await?;
+
+    Ok(update)
+  }
+}
+
+impl Resolve<ExecuteArgs> for CreateSwarmConfig {
+  #[instrument(
+    "CreateSwarmConfig",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      swarm = self.swarm,
+      config = self.name,
+    )
+  )]
+  async fn resolve(
+    self,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
+  ) -> Result<Self::Response, Self::Error> {
+    let swarm = get_check_permissions::<Swarm>(
+      &self.swarm,
+      user,
+      PermissionLevel::Execute.into(),
+    )
+    .await?;
+
+    update_update(update.clone()).await?;
+
+    let mut update = update.clone();
+
+    match swarm_request(
+      &swarm.config.server_ids,
+      periphery_client::api::swarm::CreateSwarmConfig {
+        name: self.name,
+        data: self.data,
+        labels: self.labels,
+        template_driver: self.template_driver,
+      },
+    )
+    .await
+    {
+      Ok(log) => {
+        update.logs.push(log);
+        update_cache_for_swarm(&swarm, true).await;
+      }
+      Err(e) => update.push_error_log(
+        "Create Swarm Config",
+        format_serror(
+          &e.context("Failed to create swarm config").into(),
+        ),
+      ),
+    };
+
+    update.finalize();
+    update_update(update.clone()).await?;
+
+    Ok(update)
+  }
+}
+
+impl Resolve<ExecuteArgs> for RotateSwarmConfig {
+  #[instrument(
+    "RotateSwarmConfig",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      swarm = self.swarm,
+      config = self.config,
+    )
+  )]
+  async fn resolve(
+    self,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
+  ) -> Result<Self::Response, Self::Error> {
+    let swarm = get_check_permissions::<Swarm>(
+      &self.swarm,
+      user,
+      PermissionLevel::Execute.into(),
+    )
+    .await?;
+
+    update_update(update.clone()).await?;
+
+    let mut update = update.clone();
+
+    match swarm_request(
+      &swarm.config.server_ids,
+      periphery_client::api::swarm::RotateSwarmConfig {
+        config: self.config,
+        data: self.data,
+      },
+    )
+    .await
+    {
+      Ok(logs) => {
+        update.logs.extend(logs);
+        update_cache_for_swarm(&swarm, true).await;
+      }
+      Err(e) => update.push_error_log(
+        "Rotate Swarm Config",
+        format_serror(
+          &e.context("Failed to rotate swarm config").into(),
+        ),
+      ),
+    };
+
+    update.finalize();
+    update_update(update.clone()).await?;
+
+    Ok(update)
+  }
+}
+
+impl Resolve<ExecuteArgs> for RemoveSwarmConfigs {
+  #[instrument(
+    "RemoveSwarmConfigs",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      swarm = self.swarm,
+      configs = serde_json::to_string(&self.configs).unwrap_or_else(|e| e.to_string()),
+    )
+  )]
+  async fn resolve(
+    self,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
+  ) -> Result<Self::Response, Self::Error> {
+    let swarm = get_check_permissions::<Swarm>(
+      &self.swarm,
+      user,
+      PermissionLevel::Execute.into(),
+    )
+    .await?;
+
+    update_update(update.clone()).await?;
+
+    let mut update = update.clone();
+
+    match swarm_request(
+      &swarm.config.server_ids,
+      periphery_client::api::swarm::RemoveSwarmConfigs {
+        configs: self.configs,
+      },
+    )
+    .await
+    {
+      Ok(log) => {
+        update.logs.push(log);
+        update_cache_for_swarm(&swarm, true).await;
+      }
+      Err(e) => update.push_error_log(
+        "Remove Swarm Configs",
+        format_serror(
+          &e.context("Failed to remove swarm configs").into(),
+        ),
+      ),
+    };
+
+    update.finalize();
+    update_update(update.clone()).await?;
+
+    Ok(update)
+  }
+}
+
+impl Resolve<ExecuteArgs> for CreateSwarmSecret {
+  #[instrument(
+    "CreateSwarmSecret",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      swarm = self.swarm,
+      secret = self.name,
+    )
+  )]
+  async fn resolve(
+    self,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
+  ) -> Result<Self::Response, Self::Error> {
+    let swarm = get_check_permissions::<Swarm>(
+      &self.swarm,
+      user,
+      PermissionLevel::Execute.into(),
+    )
+    .await?;
+
+    update_update(update.clone()).await?;
+
+    let mut update = update.clone();
+
+    match swarm_request(
+      &swarm.config.server_ids,
+      periphery_client::api::swarm::CreateSwarmSecret {
+        name: self.name,
+        data: self.data,
+        driver: self.driver,
+        labels: self.labels,
+        template_driver: self.template_driver,
+      },
+    )
+    .await
+    {
+      Ok(log) => {
+        update.logs.push(log);
+        update_cache_for_swarm(&swarm, true).await;
+      }
+      Err(e) => update.push_error_log(
+        "Create Swarm Secret",
+        format_serror(
+          &e.context("Failed to create swarm secret").into(),
+        ),
+      ),
+    };
+
+    update.finalize();
+    update_update(update.clone()).await?;
+
+    Ok(update)
+  }
+}
+
+impl Resolve<ExecuteArgs> for RotateSwarmSecret {
+  #[instrument(
+    "RotateSwarmSecret",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      swarm = self.swarm,
+      secret = self.secret,
+    )
+  )]
+  async fn resolve(
+    self,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
+  ) -> Result<Self::Response, Self::Error> {
+    let swarm = get_check_permissions::<Swarm>(
+      &self.swarm,
+      user,
+      PermissionLevel::Execute.into(),
+    )
+    .await?;
+
+    update_update(update.clone()).await?;
+
+    let mut update = update.clone();
+
+    match swarm_request(
+      &swarm.config.server_ids,
+      periphery_client::api::swarm::RotateSwarmSecret {
+        secret: self.secret,
+        data: self.data,
+      },
+    )
+    .await
+    {
+      Ok(logs) => {
+        update.logs.extend(logs);
+        update_cache_for_swarm(&swarm, true).await;
+      }
+      Err(e) => update.push_error_log(
+        "Rotate Swarm Secret",
+        format_serror(
+          &e.context("Failed to rotate swarm secret").into(),
+        ),
+      ),
+    };
+
+    update.finalize();
+    update_update(update.clone()).await?;
+
+    Ok(update)
+  }
+}
+
+impl Resolve<ExecuteArgs> for RemoveSwarmSecrets {
+  #[instrument(
+    "RemoveSwarmSecrets",
+    skip_all,
+    fields(
+      id = id.to_string(),
+      operator = user.id,
+      update_id = update.id,
+      swarm = self.swarm,
+      secrets = serde_json::to_string(&self.secrets).unwrap_or_else(|e| e.to_string()),
+    )
+  )]
+  async fn resolve(
+    self,
+    ExecuteArgs { user, update, id }: &ExecuteArgs,
+  ) -> Result<Self::Response, Self::Error> {
+    let swarm = get_check_permissions::<Swarm>(
+      &self.swarm,
+      user,
+      PermissionLevel::Execute.into(),
+    )
+    .await?;
+
+    update_update(update.clone()).await?;
+
+    let mut update = update.clone();
+
+    match swarm_request(
+      &swarm.config.server_ids,
+      periphery_client::api::swarm::RemoveSwarmSecrets {
+        secrets: self.secrets,
+      },
+    )
+    .await
+    {
+      Ok(log) => {
+        update.logs.push(log);
+        update_cache_for_swarm(&swarm, true).await;
+      }
+      Err(e) => update.push_error_log(
+        "Remove Swarm Secrets",
+        format_serror(
+          &e.context("Failed to remove swarm secrets").into(),
+        ),
+      ),
+    };
+
+    update.finalize();
+    update_update(update.clone()).await?;
+
+    Ok(update)
+  }
+}

bin/core/src/api/execute/sync.rs

@@ -136,34 +136,10 @@ impl Resolve<ExecuteArgs> for RunSync {
           };
           match ObjectId::from_str(&name_or_id) {
             Ok(_) => match resource_type {
-              ResourceTargetVariant::Alerter => all_resources
-                .alerters
+              ResourceTargetVariant::Swarm => all_resources
+                .swarms
                 .get(&name_or_id)
-                .map(|a| a.name.clone()),
-              ResourceTargetVariant::Build => all_resources
-                .builds
-                .get(&name_or_id)
-                .map(|b| b.name.clone()),
-              ResourceTargetVariant::Builder => all_resources
-                .builders
-                .get(&name_or_id)
-                .map(|b| b.name.clone()),
-              ResourceTargetVariant::Deployment => all_resources
-                .deployments
-                .get(&name_or_id)
-                .map(|d| d.name.clone()),
-              ResourceTargetVariant::Procedure => all_resources
-                .procedures
-                .get(&name_or_id)
-                .map(|p| p.name.clone()),
-              ResourceTargetVariant::Action => all_resources
-                .actions
-                .get(&name_or_id)
-                .map(|p| p.name.clone()),
-              ResourceTargetVariant::Repo => all_resources
-                .repos
-                .get(&name_or_id)
-                .map(|r| r.name.clone()),
+                .map(|s| s.name.clone()),
               ResourceTargetVariant::Server => all_resources
                 .servers
                 .get(&name_or_id)
@@ -172,10 +148,38 @@ impl Resolve<ExecuteArgs> for RunSync {
                 .stacks
                 .get(&name_or_id)
                 .map(|s| s.name.clone()),
+              ResourceTargetVariant::Deployment => all_resources
+                .deployments
+                .get(&name_or_id)
+                .map(|d| d.name.clone()),
+              ResourceTargetVariant::Build => all_resources
+                .builds
+                .get(&name_or_id)
+                .map(|b| b.name.clone()),
+              ResourceTargetVariant::Repo => all_resources
+                .repos
+                .get(&name_or_id)
+                .map(|r| r.name.clone()),
+              ResourceTargetVariant::Procedure => all_resources
+                .procedures
+                .get(&name_or_id)
+                .map(|p| p.name.clone()),
+              ResourceTargetVariant::Action => all_resources
+                .actions
+                .get(&name_or_id)
+                .map(|p| p.name.clone()),
               ResourceTargetVariant::ResourceSync => all_resources
                 .syncs
                 .get(&name_or_id)
                 .map(|s| s.name.clone()),
+              ResourceTargetVariant::Builder => all_resources
+                .builders
+                .get(&name_or_id)
+                .map(|b| b.name.clone()),
+              ResourceTargetVariant::Alerter => all_resources
+                .alerters
+                .get(&name_or_id)
+                .map(|a| a.name.clone()),
               ResourceTargetVariant::System => None,
             },
             Err(_) => Some(name_or_id),

bin/core/src/api/listener/integrations/github.rs

@@ -5,10 +5,9 @@ use hmac::{Hmac, Mac};
 use serde::Deserialize;
 use sha2::Sha256;
 
-use crate::{
-  config::core_config,
-  listener::{ExtractBranch, VerifySecret},
-};
+use crate::config::core_config;
+
+use super::{ExtractBranch, VerifySecret};
 
 type HmacSha256 = Hmac<Sha256>;
 
@@ -18,7 +17,7 @@ pub struct Github;
 impl VerifySecret for Github {
   #[instrument("VerifyGithubSecret", skip_all)]
   fn verify_secret(
-    headers: HeaderMap,
+    headers: &HeaderMap,
     body: &str,
     custom_secret: &str,
   ) -> anyhow::Result<()> {

bin/core/src/api/listener/integrations/gitlab.rs

@@ -1,10 +1,10 @@
 use anyhow::{Context, anyhow};
+use axum::http::HeaderMap;
 use serde::Deserialize;
 
-use crate::{
-  config::core_config,
-  listener::{ExtractBranch, VerifySecret},
-};
+use crate::config::core_config;
+
+use super::{ExtractBranch, VerifySecret};
 
 /// Listener implementation for Gitlab type API
 pub struct Gitlab;
@@ -12,7 +12,7 @@ pub struct Gitlab;
 impl VerifySecret for Gitlab {
   #[instrument("VerifyGitlabSecret", skip_all)]
   fn verify_secret(
-    headers: axum::http::HeaderMap,
+    headers: &HeaderMap,
     _body: &str,
     custom_secret: &str,
   ) -> anyhow::Result<()> {

bin/core/src/api/listener/integrations/mod.rs

@@ -0,0 +1,4 @@
+pub mod github;
+pub mod gitlab;
+
+use super::{ExtractBranch, VerifySecret};

bin/core/src/api/listener/mod.rs

@@ -32,7 +32,7 @@ trait CustomSecret: KomodoResource {
 /// Implemented on the integration struct, eg [integrations::github::Github]
 trait VerifySecret {
   fn verify_secret(
-    headers: HeaderMap,
+    headers: &HeaderMap,
     body: &str,
     custom_secret: &str,
   ) -> anyhow::Result<()>;

bin/core/src/api/listener/router.rs

@@ -1,14 +1,22 @@
-use axum::{Router, extract::Path, http::HeaderMap, routing::post};
+use std::net::{IpAddr, SocketAddr};
+
+use axum::{
+  Router,
+  extract::{ConnectInfo, Path},
+  http::HeaderMap,
+  routing::post,
+};
 use komodo_client::entities::{
   action::Action, build::Build, procedure::Procedure, repo::Repo,
   resource::Resource, stack::Stack, sync::ResourceSync,
 };
+use rate_limit::WithFailureRateLimit;
 use reqwest::StatusCode;
 use serde::Deserialize;
 use serror::AddStatusCode;
 use tracing::Instrument;
 
-use crate::resource::KomodoResource;
+use crate::{resource::KomodoResource, state::auth_rate_limiter};
 
 use super::{
   CustomSecret, ExtractBranch, VerifySecret,
@@ -47,9 +55,9 @@ pub fn router<P: VerifySecret + ExtractBranch>() -> Router {
   .route(
     "/build/{id}",
     post(
-      |Path(Id { id }), headers: HeaderMap, body: String| async move {
+      |Path(Id { id }), headers: HeaderMap, ConnectInfo(info): ConnectInfo<SocketAddr>, body: String| async move {
         let build =
-          auth_webhook::<P, Build>(&id, headers, &body).await?;
+          auth_webhook::<P, Build>(&id, &headers, info.ip(), &body).await?;
         tokio::spawn(async move {
           let span = info_span!("BuildWebhook", id);
           async {
@@ -73,9 +81,9 @@ pub fn router<P: VerifySecret + ExtractBranch>() -> Router {
   .route(
     "/repo/{id}/{option}",
     post(
-      |Path(IdAndOption::<RepoWebhookOption> { id, option }), headers: HeaderMap, body: String| async move {
+      |Path(IdAndOption::<RepoWebhookOption> { id, option }), headers: HeaderMap, ConnectInfo(info): ConnectInfo<SocketAddr>, body: String| async move {
         let repo =
-          auth_webhook::<P, Repo>(&id, headers, &body).await?;
+          auth_webhook::<P, Repo>(&id, &headers, info.ip(), &body).await?;
         tokio::spawn(async move {
           let span = info_span!("RepoWebhook", id);
           async {
@@ -99,9 +107,9 @@ pub fn router<P: VerifySecret + ExtractBranch>() -> Router {
   .route(
     "/stack/{id}/{option}",
     post(
-      |Path(IdAndOption::<StackWebhookOption> { id, option }), headers: HeaderMap, body: String| async move {
+      |Path(IdAndOption::<StackWebhookOption> { id, option }), headers: HeaderMap, ConnectInfo(info): ConnectInfo<SocketAddr>, body: String| async move {
         let stack =
-          auth_webhook::<P, Stack>(&id, headers, &body).await?;
+          auth_webhook::<P, Stack>(&id, &headers, info.ip(), &body).await?;
         tokio::spawn(async move {
           let span = info_span!("StackWebhook", id);
           async {
@@ -125,9 +133,9 @@ pub fn router<P: VerifySecret + ExtractBranch>() -> Router {
   .route(
     "/sync/{id}/{option}",
     post(
-      |Path(IdAndOption::<SyncWebhookOption> { id, option }), headers: HeaderMap, body: String| async move {
+      |Path(IdAndOption::<SyncWebhookOption> { id, option }), headers: HeaderMap, ConnectInfo(info): ConnectInfo<SocketAddr>, body: String| async move {
         let sync =
-          auth_webhook::<P, ResourceSync>(&id, headers, &body).await?;
+          auth_webhook::<P, ResourceSync>(&id, &headers, info.ip(), &body).await?;
         tokio::spawn(async move {
           let span = info_span!("ResourceSyncWebhook", id);
           async {
@@ -151,9 +159,9 @@ pub fn router<P: VerifySecret + ExtractBranch>() -> Router {
   .route(
     "/procedure/{id}/{branch}",
     post(
-      |Path(IdAndBranch { id, branch }), headers: HeaderMap, body: String| async move {
+      |Path(IdAndBranch { id, branch }), headers: HeaderMap, ConnectInfo(info): ConnectInfo<SocketAddr>, body: String| async move {
         let procedure =
-          auth_webhook::<P, Procedure>(&id, headers, &body).await?;
+          auth_webhook::<P, Procedure>(&id, &headers, info.ip(), &body).await?;
         tokio::spawn(async move {
           let span = info_span!("ProcedureWebhook", id);
           async {
@@ -177,9 +185,9 @@ pub fn router<P: VerifySecret + ExtractBranch>() -> Router {
   .route(
     "/action/{id}/{branch}",
     post(
-      |Path(IdAndBranch { id, branch }), headers: HeaderMap, body: String| async move {
+      |Path(IdAndBranch { id, branch }), headers: HeaderMap, ConnectInfo(info): ConnectInfo<SocketAddr>, body: String| async move {
         let action =
-          auth_webhook::<P, Action>(&id, headers, &body).await?;
+          auth_webhook::<P, Action>(&id, &headers, info.ip(), &body).await?;
         tokio::spawn(async move {
           let span = info_span!("ActionWebhook", id);
           async {
@@ -204,17 +212,26 @@ pub fn router<P: VerifySecret + ExtractBranch>() -> Router {
 
 async fn auth_webhook<P, R>(
   id: &str,
-  headers: HeaderMap,
+  headers: &HeaderMap,
+  ip: IpAddr,
   body: &str,
 ) -> serror::Result<Resource<R::Config, R::Info>>
 where
   P: VerifySecret,
   R: KomodoResource + CustomSecret,
 {
-  let resource = crate::resource::get::<R>(id)
-    .await
-    .status_code(StatusCode::BAD_REQUEST)?;
-  P::verify_secret(headers, body, R::custom_secret(&resource))
-    .status_code(StatusCode::UNAUTHORIZED)?;
-  Ok(resource)
+  async {
+    let resource = crate::resource::get::<R>(id)
+      .await
+      .status_code(StatusCode::BAD_REQUEST)?;
+    P::verify_secret(headers, body, R::custom_secret(&resource))
+      .status_code(StatusCode::UNAUTHORIZED)?;
+    serror::Result::Ok(resource)
+  }
+  .with_failure_rate_limit_using_headers(
+    auth_rate_limiter(),
+    headers,
+    Some(ip),
+  )
+  .await
 }

bin/core/src/api/mod.rs

@@ -1,11 +1,93 @@
+use axum::{
+  Router,
+  http::{HeaderName, HeaderValue},
+  routing::get,
+};
+use tower_http::{
+  services::{ServeDir, ServeFile},
+  set_header::SetResponseHeaderLayer,
+};
+use tower_sessions::{
+  Expiry, MemoryStore, SessionManagerLayer,
+  cookie::{SameSite, time::Duration},
+};
+
+use crate::{
+  config::{core_config, core_host, cors_layer},
+  ts_client,
+};
+
 pub mod auth;
 pub mod execute;
 pub mod read;
-pub mod terminal;
 pub mod user;
 pub mod write;
 
+mod listener;
+mod terminal;
+mod ws;
+
 #[derive(serde::Deserialize)]
 struct Variant {
   variant: String,
 }
+
+pub fn app() -> Router {
+  let config = core_config();
+
+  // Setup static frontend services
+  let frontend_path = &config.frontend_path;
+  let frontend_index =
+    ServeFile::new(format!("{frontend_path}/index.html"));
+  let serve_frontend = ServeDir::new(frontend_path)
+    .not_found_service(frontend_index.clone());
+
+  Router::new()
+    .route("/version", get(|| async { env!("CARGO_PKG_VERSION") }))
+    .nest("/auth", auth::router())
+    .nest("/user", user::router())
+    .nest("/read", read::router())
+    .nest("/write", write::router())
+    .nest("/execute", execute::router())
+    .nest("/terminal", terminal::router())
+    .nest("/listener", listener::router())
+    .nest("/ws", ws::router())
+    .nest("/client", ts_client::router())
+    .layer(memory_session_layer())
+    .fallback_service(serve_frontend)
+    .layer(cors_layer())
+    .layer(SetResponseHeaderLayer::overriding(
+      HeaderName::from_static("x-content-type-options"),
+      HeaderValue::from_static("nosniff"),
+    ))
+    .layer(SetResponseHeaderLayer::overriding(
+      HeaderName::from_static("x-frame-options"),
+      HeaderValue::from_static("DENY"),
+    ))
+    .layer(SetResponseHeaderLayer::overriding(
+      HeaderName::from_static("x-xss-protection"),
+      HeaderValue::from_static("1; mode=block"),
+    ))
+    .layer(SetResponseHeaderLayer::overriding(
+      HeaderName::from_static("referrer-policy"),
+      HeaderValue::from_static("strict-origin-when-cross-origin"),
+    ))
+}
+
+const MEMORY_SESSION_EXPIRY_SECONDS: i64 = 60;
+
+fn memory_session_layer() -> SessionManagerLayer<MemoryStore> {
+  let config = core_config();
+  let mut layer = SessionManagerLayer::new(MemoryStore::default())
+    .with_expiry(Expiry::OnInactivity(Duration::seconds(
+      MEMORY_SESSION_EXPIRY_SECONDS,
+    )))
+    .with_secure(config.host.starts_with("https://"))
+    // Needs Lax in order for sessions to work
+    // accross oauth redirects.
+    .with_same_site(SameSite::Lax);
+  if let Some(domain) = core_host().and_then(|url| url.domain()) {
+    layer = layer.with_domain(domain);
+  }
+  layer
+}

bin/core/src/api/read/action.rs

@@ -46,8 +46,13 @@ impl Resolve<ReadArgs> for ListActions {
       get_all_tags(None).await?
     };
     Ok(
-      resource::list_for_user::<Action>(self.query, user, &all_tags)
-        .await?,
+      resource::list_for_user::<Action>(
+        self.query,
+        user,
+        PermissionLevel::Read.into(),
+        &all_tags,
+      )
+      .await?,
     )
   }
 }
@@ -64,7 +69,10 @@ impl Resolve<ReadArgs> for ListFullActions {
     };
     Ok(
       resource::list_full_for_user::<Action>(
-        self.query, user, &all_tags,
+        self.query,
+        user,
+        PermissionLevel::Read.into(),
+        &all_tags,
       )
       .await?,
     )
@@ -100,6 +108,7 @@ impl Resolve<ReadArgs> for GetActionsSummary {
     let actions = resource::list_full_for_user::<Action>(
       Default::default(),
       user,
+      PermissionLevel::Read.into(),
       &[],
     )
     .await

bin/core/src/api/read/alert.rs

@@ -8,15 +8,15 @@ use komodo_client::{
   api::read::{
     GetAlert, GetAlertResponse, ListAlerts, ListAlertsResponse,
   },
-  entities::{
-    deployment::Deployment, server::Server, stack::Stack,
-    sync::ResourceSync,
-  },
+  entities::permission::PermissionLevel,
 };
 use resolver_api::Resolve;
 
 use crate::{
-  config::core_config, permission::get_resource_ids_for_user,
+  config::core_config,
+  permission::{
+    check_user_target_access, user_resource_target_query,
+  },
   state::db_client,
 };
 
@@ -29,25 +29,10 @@ impl Resolve<ReadArgs> for ListAlerts {
     self,
     ReadArgs { user }: &ReadArgs,
   ) -> serror::Result<ListAlertsResponse> {
-    let mut query = self.query.unwrap_or_default();
-    if !user.admin && !core_config().transparent_mode {
-      let server_ids =
-        get_resource_ids_for_user::<Server>(user).await?;
-      let stack_ids =
-        get_resource_ids_for_user::<Stack>(user).await?;
-      let deployment_ids =
-        get_resource_ids_for_user::<Deployment>(user).await?;
-      let sync_ids =
-        get_resource_ids_for_user::<ResourceSync>(user).await?;
-      query.extend(doc! {
-        "$or": [
-          { "target.type": "Server", "target.id": { "$in": &server_ids } },
-          { "target.type": "Stack", "target.id": { "$in": &stack_ids } },
-          { "target.type": "Deployment", "target.id": { "$in": &deployment_ids } },
-          { "target.type": "ResourceSync", "target.id": { "$in": &sync_ids } },
-        ]
-      });
-    }
+    // Alerts
+    let query = user_resource_target_query(user, self.query)
+      .await?
+      .unwrap_or_default();
 
     let alerts = find_collect(
       &db_client().alerts,
@@ -76,13 +61,21 @@ impl Resolve<ReadArgs> for ListAlerts {
 impl Resolve<ReadArgs> for GetAlert {
   async fn resolve(
     self,
-    _: &ReadArgs,
+    ReadArgs { user }: &ReadArgs,
   ) -> serror::Result<GetAlertResponse> {
-    Ok(
-      find_one_by_id(&db_client().alerts, &self.id)
-        .await
-        .context("failed to query db for alert")?
-        .context("no alert found with given id")?,
+    let alert = find_one_by_id(&db_client().alerts, &self.id)
+      .await
+      .context("failed to query db for alert")?
+      .context("no alert found with given id")?;
+    if user.admin || core_config().transparent_mode {
+      return Ok(alert);
+    }
+    check_user_target_access(
+      &alert.target,
+      user,
+      PermissionLevel::Read.into(),
     )
+    .await?;
+    Ok(alert)
   }
 }

bin/core/src/api/read/alerter.rs

@@ -11,8 +11,10 @@ use komodo_client::{
 use resolver_api::Resolve;
 
 use crate::{
-  helpers::query::get_all_tags, permission::get_check_permissions,
-  resource, state::db_client,
+  helpers::query::get_all_tags,
+  permission::{get_check_permissions, list_resource_ids_for_user},
+  resource,
+  state::db_client,
 };
 
 use super::ReadArgs;
@@ -44,8 +46,13 @@ impl Resolve<ReadArgs> for ListAlerters {
       get_all_tags(None).await?
     };
     Ok(
-      resource::list_for_user::<Alerter>(self.query, user, &all_tags)
-        .await?,
+      resource::list_for_user::<Alerter>(
+        self.query,
+        user,
+        PermissionLevel::Read.into(),
+        &all_tags,
+      )
+      .await?,
     )
   }
 }
@@ -62,7 +69,10 @@ impl Resolve<ReadArgs> for ListFullAlerters {
     };
     Ok(
       resource::list_full_for_user::<Alerter>(
-        self.query, user, &all_tags,
+        self.query,
+        user,
+        PermissionLevel::Read.into(),
+        &all_tags,
       )
       .await?,
     )
@@ -74,9 +84,11 @@ impl Resolve<ReadArgs> for GetAlertersSummary {
     self,
     ReadArgs { user }: &ReadArgs,
   ) -> serror::Result<GetAlertersSummaryResponse> {
-    let query = match resource::get_resource_object_ids_for_user::<
-      Alerter,
-    >(user)
+    let query = match list_resource_ids_for_user::<Alerter>(
+      None,
+      user,
+      PermissionLevel::Read.into(),
+    )
     .await?
     {
       Some(ids) => doc! {

bin/core/src/api/read/build.rs

@@ -6,7 +6,7 @@ use database::mungos::{
   find::find_collect,
   mongodb::{bson::doc, options::FindOptions},
 };
-use futures::TryStreamExt;
+use futures_util::TryStreamExt;
 use komodo_client::{
   api::read::*,
   entities::{
@@ -54,8 +54,13 @@ impl Resolve<ReadArgs> for ListBuilds {
       get_all_tags(None).await?
     };
     Ok(
-      resource::list_for_user::<Build>(self.query, user, &all_tags)
-        .await?,
+      resource::list_for_user::<Build>(
+        self.query,
+        user,
+        PermissionLevel::Read.into(),
+        &all_tags,
+      )
+      .await?,
     )
   }
 }
@@ -72,7 +77,10 @@ impl Resolve<ReadArgs> for ListFullBuilds {
     };
     Ok(
       resource::list_full_for_user::<Build>(
-        self.query, user, &all_tags,
+        self.query,
+        user,
+        PermissionLevel::Read.into(),
+        &all_tags,
       )
       .await?,
     )
@@ -108,6 +116,7 @@ impl Resolve<ReadArgs> for GetBuildsSummary {
     let builds = resource::list_full_for_user::<Build>(
       Default::default(),
       user,
+      PermissionLevel::Read.into(),
       &[],
     )
     .await
@@ -271,7 +280,10 @@ impl Resolve<ReadArgs> for ListCommonBuildExtraArgs {
       get_all_tags(None).await?
     };
     let builds = resource::list_full_for_user::<Build>(
-      self.query, user, &all_tags,
+      self.query,
+      user,
+      PermissionLevel::Read.into(),
+      &all_tags,
     )
     .await
     .context("failed to get resources matching query")?;

bin/core/src/api/read/builder.rs

@@ -11,8 +11,10 @@ use komodo_client::{
 use resolver_api::Resolve;
 
 use crate::{
-  helpers::query::get_all_tags, permission::get_check_permissions,
-  resource, state::db_client,
+  helpers::query::get_all_tags,
+  permission::{get_check_permissions, list_resource_ids_for_user},
+  resource,
+  state::db_client,
 };
 
 use super::ReadArgs;
@@ -44,8 +46,13 @@ impl Resolve<ReadArgs> for ListBuilders {
       get_all_tags(None).await?
     };
     Ok(
-      resource::list_for_user::<Builder>(self.query, user, &all_tags)
-        .await?,
+      resource::list_for_user::<Builder>(
+        self.query,
+        user,
+        PermissionLevel::Read.into(),
+        &all_tags,
+      )
+      .await?,
     )
   }
 }
@@ -62,7 +69,10 @@ impl Resolve<ReadArgs> for ListFullBuilders {
     };
     Ok(
       resource::list_full_for_user::<Builder>(
-        self.query, user, &all_tags,
+        self.query,
+        user,
+        PermissionLevel::Read.into(),
+        &all_tags,
       )
       .await?,
     )
@@ -74,9 +84,11 @@ impl Resolve<ReadArgs> for GetBuildersSummary {
     self,
     ReadArgs { user }: &ReadArgs,
   ) -> serror::Result<GetBuildersSummaryResponse> {
-    let query = match resource::get_resource_object_ids_for_user::<
-      Builder,
-    >(user)
+    let query = match list_resource_ids_for_user::<Builder>(
+      None,
+      user,
+      PermissionLevel::Read.into(),
+    )
     .await?
     {
       Some(ids) => doc! {

bin/core/src/api/read/deployment.rs

@@ -4,23 +4,31 @@ use anyhow::{Context, anyhow};
 use komodo_client::{
   api::read::*,
   entities::{
+    SwarmOrServer,
     deployment::{
       Deployment, DeploymentActionState, DeploymentConfig,
       DeploymentListItem, DeploymentState,
     },
-    docker::container::{Container, ContainerStats},
+    docker::{
+      container::{Container, ContainerStats},
+      service::SwarmService,
+    },
     permission::PermissionLevel,
     server::{Server, ServerState},
     update::Log,
   },
 };
 use periphery_client::api::{self, container::InspectContainer};
+use reqwest::StatusCode;
 use resolver_api::Resolve;
+use serror::AddStatusCodeError as _;
 
 use crate::{
-  helpers::{periphery_client, query::get_all_tags},
+  helpers::{
+    periphery_client, query::get_all_tags, swarm::swarm_request,
+  },
   permission::get_check_permissions,
-  resource,
+  resource::{self, setup_deployment_execution},
   state::{
     action_states, deployment_status_cache, server_status_cache,
   },
@@ -56,7 +64,10 @@ impl Resolve<ReadArgs> for ListDeployments {
     };
     let only_update_available = self.query.specific.update_available;
     let deployments = resource::list_for_user::<Deployment>(
-      self.query, user, &all_tags,
+      self.query,
+      user,
+      PermissionLevel::Read.into(),
+      &all_tags,
     )
     .await?;
     let deployments = if only_update_available {
@@ -83,7 +94,10 @@ impl Resolve<ReadArgs> for ListFullDeployments {
     };
     Ok(
       resource::list_full_for_user::<Deployment>(
-        self.query, user, &all_tags,
+        self.query,
+        user,
+        PermissionLevel::Read.into(),
+        &all_tags,
       )
       .await?,
     )
@@ -125,30 +139,40 @@ impl Resolve<ReadArgs> for GetDeploymentLog {
       tail,
       timestamps,
     } = self;
-    let Deployment {
-      name,
-      config: DeploymentConfig { server_id, .. },
-      ..
-    } = get_check_permissions::<Deployment>(
+
+    let (deployment, swarm_or_server) = setup_deployment_execution(
       &deployment,
       user,
       PermissionLevel::Read.logs(),
     )
     .await?;
-    if server_id.is_empty() {
-      return Ok(Log::default());
-    }
-    let server = resource::get::<Server>(&server_id).await?;
-    let res = periphery_client(&server)
-      .await?
-      .request(api::container::GetContainerLog {
-        name,
-        tail: cmp::min(tail, MAX_LOG_LENGTH),
-        timestamps,
-      })
+
+    let log = match swarm_or_server {
+      SwarmOrServer::Swarm(swarm) => swarm_request(
+        &swarm.config.server_ids,
+        periphery_client::api::swarm::GetSwarmServiceLog {
+          service: deployment.name,
+          tail,
+          timestamps,
+          no_task_ids: false,
+          no_resolve: false,
+          details: false,
+        },
+      )
       .await
-      .context("failed at call to periphery")?;
-    Ok(res)
+      .context("Failed to get service log from swarm")?,
+      SwarmOrServer::Server(server) => periphery_client(&server)
+        .await?
+        .request(api::container::GetContainerLog {
+          name: deployment.name,
+          tail: cmp::min(tail, MAX_LOG_LENGTH),
+          timestamps,
+        })
+        .await
+        .context("failed at call to periphery")?,
+    };
+
+    Ok(log)
   }
 }
 
@@ -164,32 +188,44 @@ impl Resolve<ReadArgs> for SearchDeploymentLog {
       invert,
       timestamps,
     } = self;
-    let Deployment {
-      name,
-      config: DeploymentConfig { server_id, .. },
-      ..
-    } = get_check_permissions::<Deployment>(
+
+    let (deployment, swarm_or_server) = setup_deployment_execution(
       &deployment,
       user,
       PermissionLevel::Read.logs(),
     )
     .await?;
-    if server_id.is_empty() {
-      return Ok(Log::default());
-    }
-    let server = resource::get::<Server>(&server_id).await?;
-    let res = periphery_client(&server)
-      .await?
-      .request(api::container::GetContainerLogSearch {
-        name,
-        terms,
-        combinator,
-        invert,
-        timestamps,
-      })
+
+    let log = match swarm_or_server {
+      SwarmOrServer::Swarm(swarm) => swarm_request(
+        &swarm.config.server_ids,
+        periphery_client::api::swarm::GetSwarmServiceLogSearch {
+          service: deployment.name,
+          terms,
+          combinator,
+          invert,
+          timestamps,
+          no_task_ids: false,
+          no_resolve: false,
+          details: false,
+        },
+      )
       .await
-      .context("failed at call to periphery")?;
-    Ok(res)
+      .context("Failed to search service log from swarm")?,
+      SwarmOrServer::Server(server) => periphery_client(&server)
+        .await?
+        .request(api::container::GetContainerLogSearch {
+          name: deployment.name,
+          terms,
+          combinator,
+          invert,
+          timestamps,
+        })
+        .await
+        .context("Failed to search container log from server")?,
+    };
+
+    Ok(log)
   }
 }
 
@@ -199,42 +235,78 @@ impl Resolve<ReadArgs> for InspectDeploymentContainer {
     ReadArgs { user }: &ReadArgs,
   ) -> serror::Result<Container> {
     let InspectDeploymentContainer { deployment } = self;
-    let Deployment {
-      name,
-      config: DeploymentConfig { server_id, .. },
-      ..
-    } = get_check_permissions::<Deployment>(
+    let (deployment, swarm_or_server) = setup_deployment_execution(
       &deployment,
       user,
       PermissionLevel::Read.inspect(),
     )
     .await?;
-    if server_id.is_empty() {
+
+    let SwarmOrServer::Server(server) = swarm_or_server else {
       return Err(
         anyhow!(
-          "Cannot inspect deployment, not attached to any server"
+          "InspectDeploymentContainer should not be called for Deployment in Swarm Mode"
         )
-        .into(),
+        .status_code(StatusCode::BAD_REQUEST),
       );
-    }
-    let server = resource::get::<Server>(&server_id).await?;
+    };
+
     let cache = server_status_cache()
       .get_or_insert_default(&server.id)
       .await;
+
     if cache.state != ServerState::Ok {
       return Err(
         anyhow!(
-          "Cannot inspect container: server is {:?}",
+          "Cannot inspect container: Server is {:?}",
           cache.state
         )
         .into(),
       );
     }
-    let res = periphery_client(&server)
+
+    periphery_client(&server)
       .await?
-      .request(InspectContainer { name })
-      .await?;
-    Ok(res)
+      .request(InspectContainer {
+        name: deployment.name,
+      })
+      .await
+      .context("Failed to inspect container on server")
+      .map_err(Into::into)
+  }
+}
+
+impl Resolve<ReadArgs> for InspectDeploymentSwarmService {
+  async fn resolve(
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<SwarmService> {
+    let InspectDeploymentSwarmService { deployment } = self;
+    let (deployment, swarm_or_server) = setup_deployment_execution(
+      &deployment,
+      user,
+      PermissionLevel::Read.logs(),
+    )
+    .await?;
+
+    let SwarmOrServer::Swarm(swarm) = swarm_or_server else {
+      return Err(
+        anyhow!(
+          "InspectDeploymentSwarmService should only be called for Deployment in Swarm Mode"
+        )
+        .status_code(StatusCode::BAD_REQUEST),
+      );
+    };
+
+    swarm_request(
+      &swarm.config.server_ids,
+      periphery_client::api::swarm::InspectSwarmService {
+        service: deployment.name,
+      },
+    )
+    .await
+    .context("Failed to inspect service on swarm")
+    .map_err(Into::into)
   }
 }
 
@@ -297,6 +369,7 @@ impl Resolve<ReadArgs> for GetDeploymentsSummary {
     let deployments = resource::list_full_for_user::<Deployment>(
       Default::default(),
       user,
+      PermissionLevel::Read.into(),
       &[],
     )
     .await
@@ -342,7 +415,10 @@ impl Resolve<ReadArgs> for ListCommonDeploymentExtraArgs {
       get_all_tags(None).await?
     };
     let deployments = resource::list_full_for_user::<Deployment>(
-      self.query, user, &all_tags,
+      self.query,
+      user,
+      PermissionLevel::Read.into(),
+      &all_tags,
     )
     .await
     .context("failed to get resources matching query")?;

bin/core/src/api/read/mod.rs

@@ -11,6 +11,7 @@ use komodo_client::{
     build::Build,
     builder::{Builder, BuilderConfig},
     config::{DockerRegistry, GitProvider},
+    permission::PermissionLevel,
     repo::Repo,
     server::Server,
     sync::ResourceSync,
@@ -48,8 +49,10 @@ mod repo;
 mod schedule;
 mod server;
 mod stack;
+mod swarm;
 mod sync;
 mod tag;
+mod terminal;
 mod toml;
 mod update;
 mod user;
@@ -73,36 +76,28 @@ enum ReadRequest {
   ListGitProvidersFromConfig(ListGitProvidersFromConfig),
   ListDockerRegistriesFromConfig(ListDockerRegistriesFromConfig),
 
-  // ==== USER ====
-  GetUsername(GetUsername),
-  GetPermission(GetPermission),
-  FindUser(FindUser),
-  ListUsers(ListUsers),
-  ListApiKeys(ListApiKeys),
-  ListApiKeysForServiceUser(ListApiKeysForServiceUser),
-  ListPermissions(ListPermissions),
-  ListUserTargetPermissions(ListUserTargetPermissions),
-
-  // ==== USER GROUP ====
-  GetUserGroup(GetUserGroup),
-  ListUserGroups(ListUserGroups),
-
-  // ==== PROCEDURE ====
-  GetProceduresSummary(GetProceduresSummary),
-  GetProcedure(GetProcedure),
-  GetProcedureActionState(GetProcedureActionState),
-  ListProcedures(ListProcedures),
-  ListFullProcedures(ListFullProcedures),
-
-  // ==== ACTION ====
-  GetActionsSummary(GetActionsSummary),
-  GetAction(GetAction),
-  GetActionActionState(GetActionActionState),
-  ListActions(ListActions),
-  ListFullActions(ListFullActions),
-
-  // ==== SCHEDULE ====
-  ListSchedules(ListSchedules),
+  // ==== SWARM ====
+  GetSwarmsSummary(GetSwarmsSummary),
+  GetSwarm(GetSwarm),
+  GetSwarmActionState(GetSwarmActionState),
+  ListSwarms(ListSwarms),
+  InspectSwarm(InspectSwarm),
+  ListFullSwarms(ListFullSwarms),
+  ListSwarmNodes(ListSwarmNodes),
+  InspectSwarmNode(InspectSwarmNode),
+  ListSwarmConfigs(ListSwarmConfigs),
+  InspectSwarmConfig(InspectSwarmConfig),
+  ListSwarmSecrets(ListSwarmSecrets),
+  InspectSwarmSecret(InspectSwarmSecret),
+  ListSwarmStacks(ListSwarmStacks),
+  InspectSwarmStack(InspectSwarmStack),
+  ListSwarmTasks(ListSwarmTasks),
+  InspectSwarmTask(InspectSwarmTask),
+  ListSwarmServices(ListSwarmServices),
+  InspectSwarmService(InspectSwarmService),
+  GetSwarmServiceLog(GetSwarmServiceLog),
+  SearchSwarmServiceLog(SearchSwarmServiceLog),
+  ListSwarmNetworks(ListSwarmNetworks),
 
   // ==== SERVER ====
   GetServersSummary(GetServersSummary),
@@ -110,9 +105,10 @@ enum ReadRequest {
   GetServerState(GetServerState),
   GetPeripheryInformation(GetPeripheryInformation),
   GetServerActionState(GetServerActionState),
-  GetHistoricalServerStats(GetHistoricalServerStats),
   ListServers(ListServers),
   ListFullServers(ListFullServers),
+
+  // ==== TERMINAL ====
   ListTerminals(ListTerminals),
 
   // ==== DOCKER ====
@@ -135,6 +131,7 @@ enum ReadRequest {
   // ==== SERVER STATS ====
   GetSystemInformation(GetSystemInformation),
   GetSystemStats(GetSystemStats),
+  GetHistoricalServerStats(GetHistoricalServerStats),
   ListSystemProcesses(ListSystemProcesses),
 
   // ==== STACK ====
@@ -144,6 +141,7 @@ enum ReadRequest {
   GetStackLog(GetStackLog),
   SearchStackLog(SearchStackLog),
   InspectStackContainer(InspectStackContainer),
+  InspectStackSwarmService(InspectStackSwarmService),
   ListStacks(ListStacks),
   ListFullStacks(ListFullStacks),
   ListStackServices(ListStackServices),
@@ -159,6 +157,7 @@ enum ReadRequest {
   GetDeploymentLog(GetDeploymentLog),
   SearchDeploymentLog(SearchDeploymentLog),
   InspectDeploymentContainer(InspectDeploymentContainer),
+  InspectDeploymentSwarmService(InspectDeploymentSwarmService),
   ListDeployments(ListDeployments),
   ListFullDeployments(ListFullDeployments),
   ListCommonDeploymentExtraArgs(ListCommonDeploymentExtraArgs),
@@ -180,6 +179,23 @@ enum ReadRequest {
   ListRepos(ListRepos),
   ListFullRepos(ListFullRepos),
 
+  // ==== PROCEDURE ====
+  GetProceduresSummary(GetProceduresSummary),
+  GetProcedure(GetProcedure),
+  GetProcedureActionState(GetProcedureActionState),
+  ListProcedures(ListProcedures),
+  ListFullProcedures(ListFullProcedures),
+
+  // ==== ACTION ====
+  GetActionsSummary(GetActionsSummary),
+  GetAction(GetAction),
+  GetActionActionState(GetActionActionState),
+  ListActions(ListActions),
+  ListFullActions(ListFullActions),
+
+  // ==== SCHEDULE ====
+  ListSchedules(ListSchedules),
+
   // ==== SYNC ====
   GetResourceSyncsSummary(GetResourceSyncsSummary),
   GetResourceSync(GetResourceSync),
@@ -207,6 +223,20 @@ enum ReadRequest {
   GetTag(GetTag),
   ListTags(ListTags),
 
+  // ==== USER ====
+  GetUsername(GetUsername),
+  GetPermission(GetPermission),
+  FindUser(FindUser),
+  ListUsers(ListUsers),
+  ListApiKeys(ListApiKeys),
+  ListApiKeysForServiceUser(ListApiKeysForServiceUser),
+  ListPermissions(ListPermissions),
+  ListUserTargetPermissions(ListUserTargetPermissions),
+
+  // ==== USER GROUP ====
+  GetUserGroup(GetUserGroup),
+  ListUserGroups(ListUserGroups),
+
   // ==== UPDATE ====
   GetUpdate(GetUpdate),
   ListUpdates(ListUpdates),
@@ -397,16 +427,19 @@ impl Resolve<ReadArgs> for ListGitProvidersFromConfig {
       resource::list_full_for_user::<Build>(
         Default::default(),
         user,
+        PermissionLevel::Read.into(),
         &[]
       ),
       resource::list_full_for_user::<Repo>(
         Default::default(),
         user,
+        PermissionLevel::Read.into(),
         &[]
       ),
       resource::list_full_for_user::<ResourceSync>(
         Default::default(),
         user,
+        PermissionLevel::Read.into(),
         &[]
       ),
     )?;

bin/core/src/api/read/procedure.rs

@@ -45,7 +45,10 @@ impl Resolve<ReadArgs> for ListProcedures {
     };
     Ok(
       resource::list_for_user::<Procedure>(
-        self.query, user, &all_tags,
+        self.query,
+        user,
+        PermissionLevel::Read.into(),
+        &all_tags,
       )
       .await?,
     )
@@ -64,7 +67,10 @@ impl Resolve<ReadArgs> for ListFullProcedures {
     };
     Ok(
       resource::list_full_for_user::<Procedure>(
-        self.query, user, &all_tags,
+        self.query,
+        user,
+        PermissionLevel::Read.into(),
+        &all_tags,
       )
       .await?,
     )
@@ -79,6 +85,7 @@ impl Resolve<ReadArgs> for GetProceduresSummary {
     let procedures = resource::list_full_for_user::<Procedure>(
       Default::default(),
       user,
+      PermissionLevel::Read.into(),
       &[],
     )
     .await

bin/core/src/api/read/repo.rs

@@ -44,8 +44,13 @@ impl Resolve<ReadArgs> for ListRepos {
       get_all_tags(None).await?
     };
     Ok(
-      resource::list_for_user::<Repo>(self.query, user, &all_tags)
-        .await?,
+      resource::list_for_user::<Repo>(
+        self.query,
+        user,
+        PermissionLevel::Read.into(),
+        &all_tags,
+      )
+      .await?,
     )
   }
 }
@@ -62,7 +67,10 @@ impl Resolve<ReadArgs> for ListFullRepos {
     };
     Ok(
       resource::list_full_for_user::<Repo>(
-        self.query, user, &all_tags,
+        self.query,
+        user,
+        PermissionLevel::Read.into(),
+        &all_tags,
       )
       .await?,
     )
@@ -98,6 +106,7 @@ impl Resolve<ReadArgs> for GetReposSummary {
     let repos = resource::list_full_for_user::<Repo>(
       Default::default(),
       user,
+      PermissionLevel::Read.into(),
       &[],
     )
     .await

bin/core/src/api/read/schedule.rs

@@ -1,9 +1,10 @@
-use futures::future::join_all;
+use futures_util::future::join_all;
 use komodo_client::{
   api::read::*,
   entities::{
     ResourceTarget,
     action::Action,
+    permission::PermissionLevel,
     procedure::Procedure,
     resource::{ResourceQuery, TemplatesQueryBehavior},
     schedule::Schedule,
@@ -35,6 +36,7 @@ impl Resolve<ReadArgs> for ListSchedules {
           specific: Default::default(),
         },
         &args.user,
+        PermissionLevel::Read.into(),
         &all_tags,
       ),
       list_full_for_user::<Procedure>(
@@ -46,6 +48,7 @@ impl Resolve<ReadArgs> for ListSchedules {
           specific: Default::default(),
         },
         &args.user,
+        PermissionLevel::Read.into(),
         &all_tags,
       )
     )?;

bin/core/src/api/read/server.rs

@@ -25,11 +25,10 @@ use komodo_client::{
       network::Network,
       volume::Volume,
     },
-    komodo_timestamp,
     permission::PermissionLevel,
     server::{
-      Server, ServerActionState, ServerListItem, ServerState,
-      TerminalInfo,
+      Server, ServerActionState, ServerListItem, ServerQuery,
+      ServerState,
     },
     stack::{Stack, StackServiceNames},
     stats::{SystemInformation, SystemProcess},
@@ -50,7 +49,7 @@ use tokio::sync::Mutex;
 
 use crate::{
   helpers::{periphery_client, query::get_all_tags},
-  permission::get_check_permissions,
+  permission::{get_check_permissions, list_resources_for_user},
   resource,
   stack::compose_container_match_regex,
   state::{action_states, db_client, server_status_cache},
@@ -66,6 +65,7 @@ impl Resolve<ReadArgs> for GetServersSummary {
     let servers = resource::list_for_user::<Server>(
       Default::default(),
       user,
+      PermissionLevel::Read.into(),
       &[],
     )
     .await?;
@@ -126,8 +126,13 @@ impl Resolve<ReadArgs> for ListServers {
       get_all_tags(None).await?
     };
     Ok(
-      resource::list_for_user::<Server>(self.query, user, &all_tags)
-        .await?,
+      resource::list_for_user::<Server>(
+        self.query,
+        user,
+        PermissionLevel::Read.into(),
+        &all_tags,
+      )
+      .await?,
     )
   }
 }
@@ -144,7 +149,10 @@ impl Resolve<ReadArgs> for ListFullServers {
     };
     Ok(
       resource::list_full_for_user::<Server>(
-        self.query, user, &all_tags,
+        self.query,
+        user,
+        PermissionLevel::Read.into(),
+        &all_tags,
       )
       .await?,
     )
@@ -375,8 +383,8 @@ impl Resolve<ReadArgs> for ListDockerContainers {
     let cache = server_status_cache()
       .get_or_insert_default(&server.id)
       .await;
-    if let Some(containers) = &cache.containers {
-      Ok(containers.clone())
+    if let Some(docker) = &cache.docker {
+      Ok(docker.containers.clone())
     } else {
       Ok(Vec::new())
     }
@@ -389,17 +397,12 @@ impl Resolve<ReadArgs> for ListAllDockerContainers {
     ReadArgs { user }: &ReadArgs,
   ) -> serror::Result<ListAllDockerContainersResponse> {
     let servers = resource::list_for_user::<Server>(
-      Default::default(),
+      ServerQuery::builder().names(self.servers.clone()).build(),
       user,
+      PermissionLevel::Read.into(),
       &[],
     )
-    .await?
-    .into_iter()
-    .filter(|server| {
-      self.servers.is_empty()
-        || self.servers.contains(&server.id)
-        || self.servers.contains(&server.name)
-    });
+    .await?;
 
     let mut containers = Vec::<ContainerListItem>::new();
 
@@ -407,9 +410,18 @@ impl Resolve<ReadArgs> for ListAllDockerContainers {
       let cache = server_status_cache()
         .get_or_insert_default(&server.id)
         .await;
-      if let Some(more_containers) = &cache.containers {
-        containers.extend(more_containers.clone());
-      }
+      let Some(docker) = &cache.docker else {
+        continue;
+      };
+      let more = docker
+        .containers
+        .iter()
+        .filter(|container| {
+          self.containers.is_empty()
+            || self.containers.contains(&container.name)
+        })
+        .cloned();
+      containers.extend(more);
     }
 
     Ok(containers)
@@ -424,6 +436,7 @@ impl Resolve<ReadArgs> for GetDockerContainersSummary {
     let servers = resource::list_full_for_user::<Server>(
       Default::default(),
       user,
+      PermissionLevel::Read.into(),
       &[],
     )
     .await
@@ -436,8 +449,8 @@ impl Resolve<ReadArgs> for GetDockerContainersSummary {
         .get_or_insert_default(&server.id)
         .await;
 
-      if let Some(containers) = &cache.containers {
-        for container in containers {
+      if let Some(docker) = &cache.docker {
+        for container in &docker.containers {
           res.total += 1;
           match container.state {
             ContainerStateStatusEnum::Created
@@ -575,12 +588,12 @@ impl Resolve<ReadArgs> for GetResourceMatchingContainer {
     }
 
     // then check stacks
-    let stacks =
-      resource::list_full_for_user_using_document::<Stack>(
-        doc! { "config.server_id": &server.id },
-        user,
-      )
-      .await?;
+    let stacks = list_resources_for_user::<Stack>(
+      doc! { "config.server_id": &server.id },
+      user,
+      PermissionLevel::Read.into(),
+    )
+    .await?;
 
     // check matching stack
     for stack in stacks {
@@ -629,8 +642,8 @@ impl Resolve<ReadArgs> for ListDockerNetworks {
     let cache = server_status_cache()
       .get_or_insert_default(&server.id)
       .await;
-    if let Some(networks) = &cache.networks {
-      Ok(networks.clone())
+    if let Some(docker) = &cache.docker {
+      Ok(docker.networks.clone())
     } else {
       Ok(Vec::new())
     }
@@ -682,8 +695,8 @@ impl Resolve<ReadArgs> for ListDockerImages {
     let cache = server_status_cache()
       .get_or_insert_default(&server.id)
       .await;
-    if let Some(images) = &cache.images {
-      Ok(images.clone())
+    if let Some(docker) = &cache.docker {
+      Ok(docker.images.clone())
     } else {
       Ok(Vec::new())
     }
@@ -763,8 +776,8 @@ impl Resolve<ReadArgs> for ListDockerVolumes {
     let cache = server_status_cache()
       .get_or_insert_default(&server.id)
       .await;
-    if let Some(volumes) = &cache.volumes {
-      Ok(volumes.clone())
+    if let Some(docker) = &cache.docker {
+      Ok(docker.volumes.clone())
     } else {
       Ok(Vec::new())
     }
@@ -813,76 +826,54 @@ impl Resolve<ReadArgs> for ListComposeProjects {
     let cache = server_status_cache()
       .get_or_insert_default(&server.id)
       .await;
-    if let Some(projects) = &cache.projects {
-      Ok(projects.clone())
+    if let Some(docker) = &cache.docker {
+      Ok(docker.projects.clone())
     } else {
       Ok(Vec::new())
     }
   }
 }
 
-#[derive(Default)]
-struct TerminalCacheItem {
-  list: Vec<TerminalInfo>,
-  ttl: i64,
-}
+// impl Resolve<ReadArgs> for ListAllTerminals {
+//   async fn resolve(
+//     self,
+//     args: &ReadArgs,
+//   ) -> Result<Self::Response, Self::Error> {
+//     // match self.tar
+//     let mut terminals = resource::list_full_for_user::<Server>(
+//       self.query, &args.user, &all_tags,
+//     )
+//     .await?
+//     .into_iter()
+//     .map(|server| async move {
+//       (
+//         list_terminals_inner(&server, self.fresh).await,
+//         (server.id, server.name),
+//       )
+//     })
+//     .collect::<FuturesUnordered<_>>()
+//     .collect::<Vec<_>>()
+//     .await
+//     .into_iter()
+//     .flat_map(|(terminals, server)| {
+//       let terminals = terminals.ok()?;
+//       Some((terminals, server))
+//     })
+//     .flat_map(|(terminals, (server_id, server_name))| {
+//       terminals.into_iter().map(move |info| {
+//         TerminalInfoWithServer::from_terminal_info(
+//           &server_id,
+//           &server_name,
+//           info,
+//         )
+//       })
+//     })
+//     .collect::<Vec<_>>();
 
-const TERMINAL_CACHE_TIMEOUT: i64 = 30_000;
+//     terminals.sort_by(|a, b| {
+//       a.server_name.cmp(&b.server_name).then(a.name.cmp(&b.name))
+//     });
 
-#[derive(Default)]
-struct TerminalCache(
-  std::sync::Mutex<
-    HashMap<String, Arc<tokio::sync::Mutex<TerminalCacheItem>>>,
-  >,
-);
-
-impl TerminalCache {
-  fn get_or_insert(
-    &self,
-    server_id: String,
-  ) -> Arc<tokio::sync::Mutex<TerminalCacheItem>> {
-    if let Some(cached) =
-      self.0.lock().unwrap().get(&server_id).cloned()
-    {
-      return cached;
-    }
-    let to_cache =
-      Arc::new(tokio::sync::Mutex::new(TerminalCacheItem::default()));
-    self.0.lock().unwrap().insert(server_id, to_cache.clone());
-    to_cache
-  }
-}
-
-fn terminals_cache() -> &'static TerminalCache {
-  static TERMINALS: OnceLock<TerminalCache> = OnceLock::new();
-  TERMINALS.get_or_init(Default::default)
-}
-
-impl Resolve<ReadArgs> for ListTerminals {
-  async fn resolve(
-    self,
-    ReadArgs { user }: &ReadArgs,
-  ) -> serror::Result<ListTerminalsResponse> {
-    let server = get_check_permissions::<Server>(
-      &self.server,
-      user,
-      PermissionLevel::Read.terminal(),
-    )
-    .await?;
-    let cache = terminals_cache().get_or_insert(server.id.clone());
-    let mut cache = cache.lock().await;
-    if self.fresh || komodo_timestamp() > cache.ttl {
-      cache.list = periphery_client(&server)
-        .await?
-        .request(periphery_client::api::terminal::ListTerminals {
-          container: None,
-        })
-        .await
-        .context("Failed to get fresh terminal list")?;
-      cache.ttl = komodo_timestamp() + TERMINAL_CACHE_TIMEOUT;
-      Ok(cache.list.clone())
-    } else {
-      Ok(cache.list.clone())
-    }
-  }
-}
+//     Ok(terminals)
+//   }
+// }

bin/core/src/api/read/stack.rs

@@ -4,9 +4,11 @@ use anyhow::{Context, anyhow};
 use komodo_client::{
   api::read::*,
   entities::{
-    docker::container::Container,
+    SwarmOrServer,
+    docker::{
+      container::Container, service::SwarmService, stack::SwarmStack,
+    },
     permission::PermissionLevel,
-    server::{Server, ServerState},
     stack::{Stack, StackActionState, StackListItem, StackState},
   },
 };
@@ -14,14 +16,18 @@ use periphery_client::api::{
   compose::{GetComposeLog, GetComposeLogSearch},
   container::InspectContainer,
 };
+use reqwest::StatusCode;
 use resolver_api::Resolve;
+use serror::AddStatusCodeError as _;
 
 use crate::{
-  helpers::{periphery_client, query::get_all_tags},
+  helpers::{
+    periphery_client, query::get_all_tags, swarm::swarm_request,
+  },
   permission::get_check_permissions,
   resource,
-  stack::get_stack_and_server,
-  state::{action_states, server_status_cache, stack_status_cache},
+  stack::setup_stack_execution,
+  state::{action_states, stack_status_cache},
 };
 
 use super::ReadArgs;
@@ -73,28 +79,53 @@ impl Resolve<ReadArgs> for GetStackLog {
   ) -> serror::Result<GetStackLogResponse> {
     let GetStackLog {
       stack,
-      services,
+      mut services,
       tail,
       timestamps,
     } = self;
-    let (stack, server) = get_stack_and_server(
+    let (stack, swarm_or_server) = setup_stack_execution(
       &stack,
       user,
       PermissionLevel::Read.logs(),
-      true,
     )
     .await?;
-    let res = periphery_client(&server)
-      .await?
-      .request(GetComposeLog {
-        project: stack.project_name(false),
-        services,
-        tail,
-        timestamps,
-      })
-      .await
-      .context("Failed to get stack log from periphery")?;
-    Ok(res)
+
+    let log = match swarm_or_server {
+      SwarmOrServer::Swarm(swarm) => {
+        let service = services.pop().context(
+          "Must pass single service for Swarm mode Stack logs",
+        )?;
+        swarm_request(
+          &swarm.config.server_ids,
+          periphery_client::api::swarm::GetSwarmServiceLog {
+            // The actual service name on swarm will be stackname_servicename
+            service: format!(
+              "{}_{service}",
+              stack.project_name(false)
+            ),
+            tail,
+            timestamps,
+            no_task_ids: false,
+            no_resolve: false,
+            details: false,
+          },
+        )
+        .await
+        .context("Failed to get stack service log from swarm")?
+      }
+      SwarmOrServer::Server(server) => periphery_client(&server)
+        .await?
+        .request(GetComposeLog {
+          project: stack.project_name(false),
+          services,
+          tail,
+          timestamps,
+        })
+        .await
+        .context("Failed to get stack log from periphery")?,
+    };
+
+    Ok(log)
   }
 }
 
@@ -105,32 +136,55 @@ impl Resolve<ReadArgs> for SearchStackLog {
   ) -> serror::Result<SearchStackLogResponse> {
     let SearchStackLog {
       stack,
-      services,
+      mut services,
       terms,
       combinator,
       invert,
       timestamps,
     } = self;
-    let (stack, server) = get_stack_and_server(
+    let (stack, swarm_or_server) = setup_stack_execution(
       &stack,
       user,
       PermissionLevel::Read.logs(),
-      true,
     )
     .await?;
-    let res = periphery_client(&server)
-      .await?
-      .request(GetComposeLogSearch {
-        project: stack.project_name(false),
-        services,
-        terms,
-        combinator,
-        invert,
-        timestamps,
-      })
-      .await
-      .context("Failed to search stack log from periphery")?;
-    Ok(res)
+
+    let log = match swarm_or_server {
+      SwarmOrServer::Swarm(swarm) => {
+        let service = services.pop().context(
+          "Must pass single service for Swarm mode Stack logs",
+        )?;
+        swarm_request(
+          &swarm.config.server_ids,
+          periphery_client::api::swarm::GetSwarmServiceLogSearch {
+            service,
+            terms,
+            combinator,
+            invert,
+            timestamps,
+            no_task_ids: false,
+            no_resolve: false,
+            details: false,
+          },
+        )
+        .await
+        .context("Failed to get stack service log from swarm")?
+      }
+      SwarmOrServer::Server(server) => periphery_client(&server)
+        .await?
+        .request(GetComposeLogSearch {
+          project: stack.project_name(false),
+          services,
+          terms,
+          combinator,
+          invert,
+          timestamps,
+        })
+        .await
+        .context("Failed to search stack log from periphery")?,
+    };
+
+    Ok(log)
   }
 }
 
@@ -140,38 +194,29 @@ impl Resolve<ReadArgs> for InspectStackContainer {
     ReadArgs { user }: &ReadArgs,
   ) -> serror::Result<Container> {
     let InspectStackContainer { stack, service } = self;
-    let stack = get_check_permissions::<Stack>(
+    let (stack, swarm_or_server) = setup_stack_execution(
       &stack,
       user,
       PermissionLevel::Read.inspect(),
     )
     .await?;
-    if stack.config.server_id.is_empty() {
-      return Err(
-        anyhow!("Cannot inspect stack, not attached to any server")
-          .into(),
-      );
-    }
-    let server =
-      resource::get::<Server>(&stack.config.server_id).await?;
-    let cache = server_status_cache()
-      .get_or_insert_default(&server.id)
-      .await;
-    if cache.state != ServerState::Ok {
+
+    let SwarmOrServer::Server(server) = swarm_or_server else {
       return Err(
         anyhow!(
-          "Cannot inspect container: server is {:?}",
-          cache.state
+          "InspectStackContainer should not be called for Stack in Swarm Mode"
         )
-        .into(),
+        .status_code(StatusCode::BAD_REQUEST),
       );
-    }
+    };
+
     let services = &stack_status_cache()
       .get(&stack.id)
       .await
       .unwrap_or_default()
       .curr
       .services;
+
     let Some(name) = services
       .iter()
       .find(|s| s.service == service)
@@ -181,14 +226,101 @@ impl Resolve<ReadArgs> for InspectStackContainer {
         "No service found matching '{service}'. Was the stack last deployed manually?"
       ).into());
     };
+
     let res = periphery_client(&server)
       .await?
       .request(InspectContainer { name })
-      .await?;
+      .await
+      .context("Failed to inspect container on server")?;
+
     Ok(res)
   }
 }
 
+impl Resolve<ReadArgs> for InspectStackSwarmService {
+  async fn resolve(
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<SwarmService> {
+    let InspectStackSwarmService { stack, service } = self;
+    let (stack, swarm_or_server) = setup_stack_execution(
+      &stack,
+      user,
+      PermissionLevel::Read.inspect(),
+    )
+    .await?;
+
+    let SwarmOrServer::Swarm(swarm) = swarm_or_server else {
+      return Err(
+        anyhow!(
+          "InspectStackSwarmService should only be called for Stack in Swarm Mode"
+        )
+        .status_code(StatusCode::BAD_REQUEST),
+      );
+    };
+
+    let services = &stack_status_cache()
+      .get(&stack.id)
+      .await
+      .unwrap_or_default()
+      .curr
+      .services;
+
+    let Some(service) = services
+      .iter()
+      .find(|s| s.service == service)
+      .and_then(|s| {
+        s.swarm_service.as_ref().and_then(|c| c.name.clone())
+      })
+    else {
+      return Err(anyhow!(
+        "No service found matching '{service}'. Was the stack last deployed manually?"
+      ).into());
+    };
+
+    swarm_request(
+      &swarm.config.server_ids,
+      periphery_client::api::swarm::InspectSwarmService { service },
+    )
+    .await
+    .context("Failed to inspect service on swarm")
+    .map_err(Into::into)
+  }
+}
+
+impl Resolve<ReadArgs> for InspectStackSwarmInfo {
+  async fn resolve(
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<SwarmStack> {
+    let (stack, swarm_or_server) = setup_stack_execution(
+      &self.stack,
+      user,
+      PermissionLevel::Read.inspect(),
+    )
+    .await?;
+
+    let SwarmOrServer::Swarm(swarm) = swarm_or_server else {
+      return Err(
+        anyhow!(
+          "InspectStackSwarmInfo should only be called for Stack in Swarm Mode"
+        )
+        .status_code(StatusCode::BAD_REQUEST),
+      );
+    };
+
+    swarm_request(
+      &swarm.config.server_ids,
+      periphery_client::api::swarm::InspectSwarmStack {
+        stack: stack.project_name(false),
+      },
+    )
+    .await
+    .context("Failed to inspect stack info on swarm")
+    .map_err(Into::into)
+  }
+}
+
 impl Resolve<ReadArgs> for ListCommonStackExtraArgs {
   async fn resolve(
     self,
@@ -200,10 +332,13 @@ impl Resolve<ReadArgs> for ListCommonStackExtraArgs {
       get_all_tags(None).await?
     };
     let stacks = resource::list_full_for_user::<Stack>(
-      self.query, user, &all_tags,
+      self.query,
+      user,
+      PermissionLevel::Read.into(),
+      &all_tags,
     )
     .await
-    .context("failed to get resources matching query")?;
+    .context("Failed to get resources matching query")?;
 
     // first collect with guaranteed uniqueness
     let mut res = HashSet::<String>::new();
@@ -231,10 +366,13 @@ impl Resolve<ReadArgs> for ListCommonStackBuildExtraArgs {
       get_all_tags(None).await?
     };
     let stacks = resource::list_full_for_user::<Stack>(
-      self.query, user, &all_tags,
+      self.query,
+      user,
+      PermissionLevel::Read.into(),
+      &all_tags,
     )
     .await
-    .context("failed to get resources matching query")?;
+    .context("Failed to get resources matching query")?;
 
     // first collect with guaranteed uniqueness
     let mut res = HashSet::<String>::new();
@@ -262,9 +400,13 @@ impl Resolve<ReadArgs> for ListStacks {
       get_all_tags(None).await?
     };
     let only_update_available = self.query.specific.update_available;
-    let stacks =
-      resource::list_for_user::<Stack>(self.query, user, &all_tags)
-        .await?;
+    let stacks = resource::list_for_user::<Stack>(
+      self.query,
+      user,
+      PermissionLevel::Read.into(),
+      &all_tags,
+    )
+    .await?;
     let stacks = if only_update_available {
       stacks
         .into_iter()
@@ -295,7 +437,10 @@ impl Resolve<ReadArgs> for ListFullStacks {
     };
     Ok(
       resource::list_full_for_user::<Stack>(
-        self.query, user, &all_tags,
+        self.query,
+        user,
+        PermissionLevel::Read.into(),
+        &all_tags,
       )
       .await?,
     )
@@ -331,10 +476,11 @@ impl Resolve<ReadArgs> for GetStacksSummary {
     let stacks = resource::list_full_for_user::<Stack>(
       Default::default(),
       user,
+      PermissionLevel::Read.into(),
       &[],
     )
     .await
-    .context("failed to get stacks from db")?;
+    .context("Failed to get stacks from database")?;
 
     let mut res = GetStacksSummaryResponse::default();
 

bin/core/src/api/read/swarm.rs

@@ -0,0 +1,519 @@
+use anyhow::{Context, anyhow};
+use komodo_client::{
+  api::read::*,
+  entities::{
+    permission::PermissionLevel,
+    swarm::{Swarm, SwarmActionState, SwarmListItem, SwarmState},
+  },
+};
+use resolver_api::Resolve;
+
+use crate::{
+  helpers::{query::get_all_tags, swarm::swarm_request},
+  permission::get_check_permissions,
+  resource,
+  state::{action_states, server_status_cache, swarm_status_cache},
+};
+
+use super::ReadArgs;
+
+impl Resolve<ReadArgs> for GetSwarm {
+  async fn resolve(
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<Swarm> {
+    Ok(
+      get_check_permissions::<Swarm>(
+        &self.swarm,
+        user,
+        PermissionLevel::Read.into(),
+      )
+      .await?,
+    )
+  }
+}
+
+impl Resolve<ReadArgs> for ListSwarms {
+  async fn resolve(
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<Vec<SwarmListItem>> {
+    let all_tags = if self.query.tags.is_empty() {
+      vec![]
+    } else {
+      get_all_tags(None).await?
+    };
+    Ok(
+      resource::list_for_user::<Swarm>(
+        self.query,
+        user,
+        PermissionLevel::Read.into(),
+        &all_tags,
+      )
+      .await?,
+    )
+  }
+}
+
+impl Resolve<ReadArgs> for ListFullSwarms {
+  async fn resolve(
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<ListFullSwarmsResponse> {
+    let all_tags = if self.query.tags.is_empty() {
+      vec![]
+    } else {
+      get_all_tags(None).await?
+    };
+    Ok(
+      resource::list_full_for_user::<Swarm>(
+        self.query,
+        user,
+        PermissionLevel::Read.into(),
+        &all_tags,
+      )
+      .await?,
+    )
+  }
+}
+
+impl Resolve<ReadArgs> for GetSwarmActionState {
+  async fn resolve(
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<SwarmActionState> {
+    let swarm = get_check_permissions::<Swarm>(
+      &self.swarm,
+      user,
+      PermissionLevel::Read.into(),
+    )
+    .await?;
+    let action_state = action_states()
+      .swarm
+      .get(&swarm.id)
+      .await
+      .unwrap_or_default()
+      .get()?;
+    Ok(action_state)
+  }
+}
+
+impl Resolve<ReadArgs> for GetSwarmsSummary {
+  async fn resolve(
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<GetSwarmsSummaryResponse> {
+    let swarms = resource::list_full_for_user::<Swarm>(
+      Default::default(),
+      user,
+      PermissionLevel::Read.into(),
+      &[],
+    )
+    .await
+    .context("failed to get swarms from db")?;
+
+    let mut res = GetSwarmsSummaryResponse::default();
+
+    let cache = swarm_status_cache();
+
+    for swarm in swarms {
+      res.total += 1;
+
+      match cache
+        .get(&swarm.id)
+        .await
+        .map(|status| status.state)
+        .unwrap_or_default()
+      {
+        SwarmState::Unknown => {
+          res.unknown += 1;
+        }
+        SwarmState::Healthy => {
+          res.healthy += 1;
+        }
+        SwarmState::Unhealthy => {
+          res.unhealthy += 1;
+        }
+      }
+    }
+
+    Ok(res)
+  }
+}
+
+impl Resolve<ReadArgs> for InspectSwarm {
+  async fn resolve(
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<InspectSwarmResponse> {
+    let swarm = get_check_permissions::<Swarm>(
+      &self.swarm,
+      user,
+      PermissionLevel::Read.inspect(),
+    )
+    .await?;
+    let cache =
+      swarm_status_cache().get_or_insert_default(&swarm.id).await;
+    let inspect = cache
+      .inspect
+      .as_ref()
+      .cloned()
+      .context("SwarmInspectInfo not available")?;
+    Ok(inspect)
+  }
+}
+
+impl Resolve<ReadArgs> for ListSwarmNodes {
+  async fn resolve(
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<ListSwarmNodesResponse> {
+    let swarm = get_check_permissions::<Swarm>(
+      &self.swarm,
+      user,
+      PermissionLevel::Read.into(),
+    )
+    .await?;
+    let cache =
+      swarm_status_cache().get_or_insert_default(&swarm.id).await;
+    if let Some(lists) = &cache.lists {
+      Ok(lists.nodes.clone())
+    } else {
+      Ok(Vec::new())
+    }
+  }
+}
+
+impl Resolve<ReadArgs> for InspectSwarmNode {
+  async fn resolve(
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<InspectSwarmNodeResponse> {
+    let swarm = get_check_permissions::<Swarm>(
+      &self.swarm,
+      user,
+      PermissionLevel::Read.inspect(),
+    )
+    .await?;
+    swarm_request(
+      &swarm.config.server_ids,
+      periphery_client::api::swarm::InspectSwarmNode {
+        node: self.node,
+      },
+    )
+    .await
+    .map_err(Into::into)
+  }
+}
+
+impl Resolve<ReadArgs> for ListSwarmServices {
+  async fn resolve(
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<ListSwarmServicesResponse> {
+    let swarm = get_check_permissions::<Swarm>(
+      &self.swarm,
+      user,
+      PermissionLevel::Read.into(),
+    )
+    .await?;
+    let cache =
+      swarm_status_cache().get_or_insert_default(&swarm.id).await;
+    if let Some(lists) = &cache.lists {
+      Ok(lists.services.clone())
+    } else {
+      Ok(Vec::new())
+    }
+  }
+}
+
+impl Resolve<ReadArgs> for InspectSwarmService {
+  async fn resolve(
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<InspectSwarmServiceResponse> {
+    let swarm = get_check_permissions::<Swarm>(
+      &self.swarm,
+      user,
+      PermissionLevel::Read.inspect(),
+    )
+    .await?;
+    swarm_request(
+      &swarm.config.server_ids,
+      periphery_client::api::swarm::InspectSwarmService {
+        service: self.service,
+      },
+    )
+    .await
+    .map_err(Into::into)
+  }
+}
+
+impl Resolve<ReadArgs> for GetSwarmServiceLog {
+  async fn resolve(
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<GetSwarmServiceLogResponse> {
+    let swarm = get_check_permissions::<Swarm>(
+      &self.swarm,
+      user,
+      PermissionLevel::Read.logs(),
+    )
+    .await?;
+    swarm_request(
+      &swarm.config.server_ids,
+      periphery_client::api::swarm::GetSwarmServiceLog {
+        service: self.service,
+        tail: self.tail,
+        timestamps: self.timestamps,
+        no_task_ids: self.no_task_ids,
+        no_resolve: self.no_resolve,
+        details: self.details,
+      },
+    )
+    .await
+    .map_err(Into::into)
+  }
+}
+
+impl Resolve<ReadArgs> for SearchSwarmServiceLog {
+  async fn resolve(
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<SearchSwarmServiceLogResponse> {
+    let swarm = get_check_permissions::<Swarm>(
+      &self.swarm,
+      user,
+      PermissionLevel::Read.logs(),
+    )
+    .await?;
+    swarm_request(
+      &swarm.config.server_ids,
+      periphery_client::api::swarm::GetSwarmServiceLogSearch {
+        service: self.service,
+        terms: self.terms,
+        combinator: self.combinator,
+        invert: self.invert,
+        timestamps: self.timestamps,
+        no_task_ids: self.no_task_ids,
+        no_resolve: self.no_resolve,
+        details: self.details,
+      },
+    )
+    .await
+    .map_err(Into::into)
+  }
+}
+
+impl Resolve<ReadArgs> for ListSwarmTasks {
+  async fn resolve(
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<ListSwarmTasksResponse> {
+    let swarm = get_check_permissions::<Swarm>(
+      &self.swarm,
+      user,
+      PermissionLevel::Read.into(),
+    )
+    .await?;
+    let cache =
+      swarm_status_cache().get_or_insert_default(&swarm.id).await;
+    if let Some(lists) = &cache.lists {
+      Ok(lists.tasks.clone())
+    } else {
+      Ok(Vec::new())
+    }
+  }
+}
+
+impl Resolve<ReadArgs> for InspectSwarmTask {
+  async fn resolve(
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<InspectSwarmTaskResponse> {
+    let swarm = get_check_permissions::<Swarm>(
+      &self.swarm,
+      user,
+      PermissionLevel::Read.inspect(),
+    )
+    .await?;
+    swarm_request(
+      &swarm.config.server_ids,
+      periphery_client::api::swarm::InspectSwarmTask {
+        task: self.task,
+      },
+    )
+    .await
+    .map_err(Into::into)
+  }
+}
+
+impl Resolve<ReadArgs> for ListSwarmSecrets {
+  async fn resolve(
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<ListSwarmSecretsResponse> {
+    let swarm = get_check_permissions::<Swarm>(
+      &self.swarm,
+      user,
+      PermissionLevel::Read.into(),
+    )
+    .await?;
+    let cache =
+      swarm_status_cache().get_or_insert_default(&swarm.id).await;
+    if let Some(lists) = &cache.lists {
+      Ok(lists.secrets.clone())
+    } else {
+      Ok(Vec::new())
+    }
+  }
+}
+
+impl Resolve<ReadArgs> for InspectSwarmSecret {
+  async fn resolve(
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<InspectSwarmSecretResponse> {
+    let swarm = get_check_permissions::<Swarm>(
+      &self.swarm,
+      user,
+      PermissionLevel::Read.inspect(),
+    )
+    .await?;
+    swarm_request(
+      &swarm.config.server_ids,
+      periphery_client::api::swarm::InspectSwarmSecret {
+        secret: self.secret,
+      },
+    )
+    .await
+    .map_err(Into::into)
+  }
+}
+
+impl Resolve<ReadArgs> for ListSwarmConfigs {
+  async fn resolve(
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<ListSwarmConfigsResponse> {
+    let swarm = get_check_permissions::<Swarm>(
+      &self.swarm,
+      user,
+      PermissionLevel::Read.into(),
+    )
+    .await?;
+    let cache =
+      swarm_status_cache().get_or_insert_default(&swarm.id).await;
+    if let Some(lists) = &cache.lists {
+      Ok(lists.configs.clone())
+    } else {
+      Ok(Vec::new())
+    }
+  }
+}
+
+impl Resolve<ReadArgs> for InspectSwarmConfig {
+  async fn resolve(
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<InspectSwarmConfigResponse> {
+    let swarm = get_check_permissions::<Swarm>(
+      &self.swarm,
+      user,
+      PermissionLevel::Read.inspect(),
+    )
+    .await?;
+    swarm_request(
+      &swarm.config.server_ids,
+      periphery_client::api::swarm::InspectSwarmConfig {
+        config: self.config,
+      },
+    )
+    .await
+    .map_err(Into::into)
+  }
+}
+
+impl Resolve<ReadArgs> for ListSwarmStacks {
+  async fn resolve(
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<ListSwarmStacksResponse> {
+    let swarm = get_check_permissions::<Swarm>(
+      &self.swarm,
+      user,
+      PermissionLevel::Read.into(),
+    )
+    .await?;
+    let cache =
+      swarm_status_cache().get_or_insert_default(&swarm.id).await;
+    if let Some(lists) = &cache.lists {
+      Ok(lists.stacks.clone())
+    } else {
+      Ok(Vec::new())
+    }
+  }
+}
+
+impl Resolve<ReadArgs> for InspectSwarmStack {
+  async fn resolve(
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<InspectSwarmStackResponse> {
+    let swarm = get_check_permissions::<Swarm>(
+      &self.swarm,
+      user,
+      PermissionLevel::Read.inspect(),
+    )
+    .await?;
+    swarm_request(
+      &swarm.config.server_ids,
+      periphery_client::api::swarm::InspectSwarmStack {
+        stack: self.stack,
+      },
+    )
+    .await
+    .map_err(Into::into)
+  }
+}
+
+impl Resolve<ReadArgs> for ListSwarmNetworks {
+  async fn resolve(
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> Result<Self::Response, Self::Error> {
+    let swarm = get_check_permissions::<Swarm>(
+      &self.swarm,
+      user,
+      PermissionLevel::Read.into(),
+    )
+    .await?;
+
+    let cache = server_status_cache();
+
+    for server_id in swarm.config.server_ids {
+      let Some(status) = cache.get(&server_id).await else {
+        continue;
+      };
+      let Some(docker) = &status.docker else {
+        continue;
+      };
+      let networks = docker
+        .networks
+        .iter()
+        .filter(|network| {
+          network.driver.as_deref() == Some("overlay")
+        })
+        .cloned()
+        .collect::<Vec<_>>();
+      return Ok(networks);
+    }
+
+    Err(
+      anyhow!(
+        "Failed to retrieve swarm networks from any manager node."
+      )
+      .into(),
+    )
+  }
+}

bin/core/src/api/read/sync.rs

@@ -45,7 +45,10 @@ impl Resolve<ReadArgs> for ListResourceSyncs {
     };
     Ok(
       resource::list_for_user::<ResourceSync>(
-        self.query, user, &all_tags,
+        self.query,
+        user,
+        PermissionLevel::Read.into(),
+        &all_tags,
       )
       .await?,
     )
@@ -64,7 +67,10 @@ impl Resolve<ReadArgs> for ListFullResourceSyncs {
     };
     Ok(
       resource::list_full_for_user::<ResourceSync>(
-        self.query, user, &all_tags,
+        self.query,
+        user,
+        PermissionLevel::Read.into(),
+        &all_tags,
       )
       .await?,
     )
@@ -101,6 +107,7 @@ impl Resolve<ReadArgs> for GetResourceSyncsSummary {
       resource::list_full_for_user::<ResourceSync>(
         Default::default(),
         user,
+        PermissionLevel::Read.into(),
         &[],
       )
       .await

bin/core/src/api/read/terminal.rs

@@ -0,0 +1,247 @@
+use anyhow::Context as _;
+use futures_util::{
+  FutureExt, StreamExt as _, stream::FuturesUnordered,
+};
+use komodo_client::{
+  api::read::{ListTerminals, ListTerminalsResponse},
+  entities::{
+    deployment::Deployment,
+    permission::PermissionLevel,
+    server::Server,
+    stack::Stack,
+    terminal::{Terminal, TerminalTarget},
+    user::User,
+  },
+};
+use reqwest::StatusCode;
+use resolver_api::Resolve;
+use serror::AddStatusCode;
+
+use crate::{
+  helpers::periphery_client, permission::get_check_permissions,
+  resource,
+};
+
+use super::ReadArgs;
+
+//
+
+impl Resolve<ReadArgs> for ListTerminals {
+  async fn resolve(
+    self,
+    ReadArgs { user }: &ReadArgs,
+  ) -> serror::Result<ListTerminalsResponse> {
+    let Some(target) = self.target else {
+      return list_all_terminals_for_user(user, self.use_names).await;
+    };
+    match &target {
+      TerminalTarget::Server { server } => {
+        let server = server
+          .as_ref()
+          .context("Must provide 'target.params.server'")
+          .status_code(StatusCode::BAD_REQUEST)?;
+        let server = get_check_permissions::<Server>(
+          server,
+          user,
+          PermissionLevel::Read.terminal(),
+        )
+        .await?;
+        list_terminals_on_server(&server, Some(target)).await
+      }
+      TerminalTarget::Container { server, .. } => {
+        let server = get_check_permissions::<Server>(
+          server,
+          user,
+          PermissionLevel::Read.terminal(),
+        )
+        .await?;
+        list_terminals_on_server(&server, Some(target)).await
+      }
+      TerminalTarget::Stack { stack, .. } => {
+        let server = get_check_permissions::<Stack>(
+          stack,
+          user,
+          PermissionLevel::Read.terminal(),
+        )
+        .await?
+        .config
+        .server_id;
+        let server = resource::get::<Server>(&server).await?;
+        list_terminals_on_server(&server, Some(target)).await
+      }
+      TerminalTarget::Deployment { deployment } => {
+        let server = get_check_permissions::<Deployment>(
+          deployment,
+          user,
+          PermissionLevel::Read.terminal(),
+        )
+        .await?
+        .config
+        .server_id;
+        let server = resource::get::<Server>(&server).await?;
+        list_terminals_on_server(&server, Some(target)).await
+      }
+    }
+  }
+}
+
+async fn list_all_terminals_for_user(
+  user: &User,
+  use_names: bool,
+) -> serror::Result<Vec<Terminal>> {
+  let (mut servers, stacks, deployments) = tokio::try_join!(
+    resource::list_full_for_user::<Server>(
+      Default::default(),
+      user,
+      PermissionLevel::Read.terminal(),
+      &[]
+    )
+    .map(|res| res.map(|servers| servers
+      .into_iter()
+      // true denotes user actually has permission on this Server.
+      .map(|server| (server, true))
+      .collect::<Vec<_>>())),
+    resource::list_full_for_user::<Stack>(
+      Default::default(),
+      user,
+      PermissionLevel::Read.terminal(),
+      &[]
+    ),
+    resource::list_full_for_user::<Deployment>(
+      Default::default(),
+      user,
+      PermissionLevel::Read.terminal(),
+      &[]
+    ),
+  )?;
+
+  // Ensure any missing servers are present to query
+  for stack in &stacks {
+    if !stack.config.server_id.is_empty()
+      && !servers
+        .iter()
+        .any(|(server, _)| server.id == stack.config.server_id)
+    {
+      let server =
+        resource::get::<Server>(&stack.config.server_id).await?;
+      servers.push((server, false));
+    }
+  }
+  for deployment in &deployments {
+    if !deployment.config.server_id.is_empty()
+      && !servers
+        .iter()
+        .any(|(server, _)| server.id == deployment.config.server_id)
+    {
+      let server =
+        resource::get::<Server>(&deployment.config.server_id).await?;
+      servers.push((server, false));
+    }
+  }
+
+  let mut terminals = servers
+    .into_iter()
+    .map(|(server, server_permission)| async move {
+      (
+        list_terminals_on_server(&server, None).await,
+        (server.id, server.name, server_permission),
+      )
+    })
+    .collect::<FuturesUnordered<_>>()
+    .collect::<Vec<_>>()
+    .await
+    .into_iter()
+    .flat_map(
+      |(terminals, (server_id, server_name, server_permission))| {
+        let terminals = terminals
+          .ok()?
+          .into_iter()
+          .filter_map(|mut terminal| {
+            // Only keep terminals with appropriate perms.
+            match terminal.target.clone() {
+              TerminalTarget::Server { .. } => server_permission
+                .then(|| {
+                  terminal.target = TerminalTarget::Server {
+                    server: Some(if use_names {
+                      server_name.clone()
+                    } else {
+                      server_id.clone()
+                    }),
+                  };
+                  terminal
+                }),
+              TerminalTarget::Container { container, .. } => {
+                server_permission.then(|| {
+                  terminal.target = TerminalTarget::Container {
+                    server: if use_names {
+                      server_name.clone()
+                    } else {
+                      server_id.clone()
+                    },
+                    container,
+                  };
+                  terminal
+                })
+              }
+              TerminalTarget::Stack { stack, service } => {
+                stacks.iter().find(|s| s.id == stack).map(|s| {
+                  terminal.target = TerminalTarget::Stack {
+                    stack: if use_names {
+                      s.name.clone()
+                    } else {
+                      s.id.clone()
+                    },
+                    service,
+                  };
+                  terminal
+                })
+              }
+              TerminalTarget::Deployment { deployment } => {
+                deployments.iter().find(|d| d.id == deployment).map(
+                  |d| {
+                    terminal.target = TerminalTarget::Deployment {
+                      deployment: if use_names {
+                        d.name.clone()
+                      } else {
+                        d.id.clone()
+                      },
+                    };
+                    terminal
+                  },
+                )
+              }
+            }
+          })
+          .collect::<Vec<_>>();
+
+        Some(terminals)
+      },
+    )
+    .flatten()
+    .collect::<Vec<_>>();
+
+  terminals.sort_by(|a, b| {
+    a.target.cmp(&b.target).then(a.name.cmp(&b.name))
+  });
+
+  Ok(terminals)
+}
+
+async fn list_terminals_on_server(
+  server: &Server,
+  target: Option<TerminalTarget>,
+) -> serror::Result<Vec<Terminal>> {
+  periphery_client(server)
+    .await?
+    .request(periphery_client::api::terminal::ListTerminals {
+      target,
+    })
+    .await
+    .with_context(|| {
+      format!(
+        "Failed to get Terminal list from Server {} ({})",
+        server.name, server.id
+      )
+    })
+    .map_err(Into::into)
+}

bin/core/src/api/read/toml.rs

@@ -11,7 +11,8 @@ use komodo_client::{
     builder::Builder, deployment::Deployment,
     permission::PermissionLevel, procedure::Procedure, repo::Repo,
     resource::ResourceQuery, server::Server, stack::Stack,
-    sync::ResourceSync, toml::ResourcesToml, user::User,
+    swarm::Swarm, sync::ResourceSync, toml::ResourcesToml,
+    user::User,
   },
 };
 use resolver_api::Resolve;
@@ -46,6 +47,7 @@ async fn get_all_targets(
     resource::list_full_for_user::<Alerter>(
       ResourceQuery::builder().tags(tags).build(),
       user,
+      PermissionLevel::Read.into(),
       &all_tags,
     )
     .await?
@@ -56,6 +58,7 @@ async fn get_all_targets(
     resource::list_full_for_user::<Builder>(
       ResourceQuery::builder().tags(tags).build(),
       user,
+      PermissionLevel::Read.into(),
       &all_tags,
     )
     .await?
@@ -66,6 +69,7 @@ async fn get_all_targets(
     resource::list_full_for_user::<Server>(
       ResourceQuery::builder().tags(tags).build(),
       user,
+      PermissionLevel::Read.into(),
       &all_tags,
     )
     .await?
@@ -76,6 +80,7 @@ async fn get_all_targets(
     resource::list_full_for_user::<Stack>(
       ResourceQuery::builder().tags(tags).build(),
       user,
+      PermissionLevel::Read.into(),
       &all_tags,
     )
     .await?
@@ -86,6 +91,7 @@ async fn get_all_targets(
     resource::list_full_for_user::<Deployment>(
       ResourceQuery::builder().tags(tags).build(),
       user,
+      PermissionLevel::Read.into(),
       &all_tags,
     )
     .await?
@@ -96,6 +102,7 @@ async fn get_all_targets(
     resource::list_full_for_user::<Build>(
       ResourceQuery::builder().tags(tags).build(),
       user,
+      PermissionLevel::Read.into(),
       &all_tags,
     )
     .await?
@@ -106,6 +113,7 @@ async fn get_all_targets(
     resource::list_full_for_user::<Repo>(
       ResourceQuery::builder().tags(tags).build(),
       user,
+      PermissionLevel::Read.into(),
       &all_tags,
     )
     .await?
@@ -116,6 +124,7 @@ async fn get_all_targets(
     resource::list_full_for_user::<Procedure>(
       ResourceQuery::builder().tags(tags).build(),
       user,
+      PermissionLevel::Read.into(),
       &all_tags,
     )
     .await?
@@ -126,6 +135,7 @@ async fn get_all_targets(
     resource::list_full_for_user::<Action>(
       ResourceQuery::builder().tags(tags).build(),
       user,
+      PermissionLevel::Read.into(),
       &all_tags,
     )
     .await?
@@ -136,6 +146,7 @@ async fn get_all_targets(
     resource::list_full_for_user::<ResourceSync>(
       ResourceQuery::builder().tags(tags).build(),
       user,
+      PermissionLevel::Read.into(),
       &all_tags,
     )
     .await?
@@ -197,42 +208,21 @@ impl Resolve<ReadArgs> for ExportResourcesToToml {
     let ReadArgs { user } = args;
     for target in targets {
       match target {
-        ResourceTarget::Alerter(id) => {
-          let mut alerter = get_check_permissions::<Alerter>(
+        ResourceTarget::Swarm(id) => {
+          let mut swarm = get_check_permissions::<Swarm>(
             &id,
             user,
             PermissionLevel::Read.into(),
           )
           .await?;
-          Alerter::replace_ids(&mut alerter);
-          res.alerters.push(convert_resource::<Alerter>(
-            alerter,
+          Swarm::replace_ids(&mut swarm);
+          res.swarms.push(convert_resource::<Swarm>(
+            swarm,
             false,
             vec![],
             &id_to_tags,
           ))
         }
-        ResourceTarget::ResourceSync(id) => {
-          let mut sync = get_check_permissions::<ResourceSync>(
-            &id,
-            user,
-            PermissionLevel::Read.into(),
-          )
-          .await?;
-          if sync.config.file_contents.is_empty()
-            && (sync.config.files_on_host
-              || !sync.config.repo.is_empty()
-              || !sync.config.linked_repo.is_empty())
-          {
-            ResourceSync::replace_ids(&mut sync);
-            res.resource_syncs.push(convert_resource::<ResourceSync>(
-              sync,
-              false,
-              vec![],
-              &id_to_tags,
-            ))
-          }
-        }
         ResourceTarget::Server(id) => {
           let mut server = get_check_permissions::<Server>(
             &id,
@@ -248,31 +238,16 @@ impl Resolve<ReadArgs> for ExportResourcesToToml {
             &id_to_tags,
           ))
         }
-        ResourceTarget::Builder(id) => {
-          let mut builder = get_check_permissions::<Builder>(
+        ResourceTarget::Stack(id) => {
+          let mut stack = get_check_permissions::<Stack>(
             &id,
             user,
             PermissionLevel::Read.into(),
           )
           .await?;
-          Builder::replace_ids(&mut builder);
-          res.builders.push(convert_resource::<Builder>(
-            builder,
-            false,
-            vec![],
-            &id_to_tags,
-          ))
-        }
-        ResourceTarget::Build(id) => {
-          let mut build = get_check_permissions::<Build>(
-            &id,
-            user,
-            PermissionLevel::Read.into(),
-          )
-          .await?;
-          Build::replace_ids(&mut build);
-          res.builds.push(convert_resource::<Build>(
-            build,
+          Stack::replace_ids(&mut stack);
+          res.stacks.push(convert_resource::<Stack>(
+            stack,
             false,
             vec![],
             &id_to_tags,
@@ -293,6 +268,21 @@ impl Resolve<ReadArgs> for ExportResourcesToToml {
             &id_to_tags,
           ))
         }
+        ResourceTarget::Build(id) => {
+          let mut build = get_check_permissions::<Build>(
+            &id,
+            user,
+            PermissionLevel::Read.into(),
+          )
+          .await?;
+          Build::replace_ids(&mut build);
+          res.builds.push(convert_resource::<Build>(
+            build,
+            false,
+            vec![],
+            &id_to_tags,
+          ))
+        }
         ResourceTarget::Repo(id) => {
           let mut repo = get_check_permissions::<Repo>(
             &id,
@@ -308,21 +298,6 @@ impl Resolve<ReadArgs> for ExportResourcesToToml {
             &id_to_tags,
           ))
         }
-        ResourceTarget::Stack(id) => {
-          let mut stack = get_check_permissions::<Stack>(
-            &id,
-            user,
-            PermissionLevel::Read.into(),
-          )
-          .await?;
-          Stack::replace_ids(&mut stack);
-          res.stacks.push(convert_resource::<Stack>(
-            stack,
-            false,
-            vec![],
-            &id_to_tags,
-          ))
-        }
         ResourceTarget::Procedure(id) => {
           let mut procedure = get_check_permissions::<Procedure>(
             &id,
@@ -353,6 +328,57 @@ impl Resolve<ReadArgs> for ExportResourcesToToml {
             &id_to_tags,
           ));
         }
+        ResourceTarget::ResourceSync(id) => {
+          let mut sync = get_check_permissions::<ResourceSync>(
+            &id,
+            user,
+            PermissionLevel::Read.into(),
+          )
+          .await?;
+          if sync.config.file_contents.is_empty()
+            && (sync.config.files_on_host
+              || !sync.config.repo.is_empty()
+              || !sync.config.linked_repo.is_empty())
+          {
+            ResourceSync::replace_ids(&mut sync);
+            res.resource_syncs.push(convert_resource::<ResourceSync>(
+              sync,
+              false,
+              vec![],
+              &id_to_tags,
+            ))
+          }
+        }
+        ResourceTarget::Builder(id) => {
+          let mut builder = get_check_permissions::<Builder>(
+            &id,
+            user,
+            PermissionLevel::Read.into(),
+          )
+          .await?;
+          Builder::replace_ids(&mut builder);
+          res.builders.push(convert_resource::<Builder>(
+            builder,
+            false,
+            vec![],
+            &id_to_tags,
+          ))
+        }
+        ResourceTarget::Alerter(id) => {
+          let mut alerter = get_check_permissions::<Alerter>(
+            &id,
+            user,
+            PermissionLevel::Read.into(),
+          )
+          .await?;
+          Alerter::replace_ids(&mut alerter);
+          res.alerters.push(convert_resource::<Alerter>(
+            alerter,
+            false,
+            vec![],
+            &id_to_tags,
+          ))
+        }
         ResourceTarget::System(_) => continue,
       };
     }

bin/core/src/api/read/update.rs

@@ -1,6 +1,6 @@
 use std::collections::HashMap;
 
-use anyhow::{Context, anyhow};
+use anyhow::Context;
 use database::mungos::{
   by_id::find_one_by_id,
   find::find_collect,
@@ -9,18 +9,7 @@ use database::mungos::{
 use komodo_client::{
   api::read::{GetUpdate, ListUpdates, ListUpdatesResponse},
   entities::{
-    ResourceTarget,
-    action::Action,
-    alerter::Alerter,
-    build::Build,
-    builder::Builder,
-    deployment::Deployment,
     permission::PermissionLevel,
-    procedure::Procedure,
-    repo::Repo,
-    server::Server,
-    stack::Stack,
-    sync::ResourceSync,
     update::{Update, UpdateListItem},
     user::User,
   },
@@ -29,7 +18,9 @@ use resolver_api::Resolve;
 
 use crate::{
   config::core_config,
-  permission::{get_check_permissions, get_resource_ids_for_user},
+  permission::{
+    check_user_target_access, user_resource_target_query,
+  },
   state::db_client,
 };
 
@@ -42,120 +33,7 @@ impl Resolve<ReadArgs> for ListUpdates {
     self,
     ReadArgs { user }: &ReadArgs,
   ) -> serror::Result<ListUpdatesResponse> {
-    let query = if user.admin || core_config().transparent_mode {
-      self.query
-    } else {
-      let server_query = get_resource_ids_for_user::<Server>(user)
-        .await?
-        .map(|ids| {
-          doc! {
-            "target.type": "Server", "target.id": { "$in": ids }
-          }
-        })
-        .unwrap_or_else(|| doc! { "target.type": "Server" });
-
-      let deployment_query =
-        get_resource_ids_for_user::<Deployment>(user)
-          .await?
-          .map(|ids| {
-            doc! {
-              "target.type": "Deployment", "target.id": { "$in": ids }
-            }
-          })
-          .unwrap_or_else(|| doc! { "target.type": "Deployment" });
-
-      let stack_query = get_resource_ids_for_user::<Stack>(user)
-        .await?
-        .map(|ids| {
-          doc! {
-            "target.type": "Stack", "target.id": { "$in": ids }
-          }
-        })
-        .unwrap_or_else(|| doc! { "target.type": "Stack" });
-
-      let build_query = get_resource_ids_for_user::<Build>(user)
-        .await?
-        .map(|ids| {
-          doc! {
-            "target.type": "Build", "target.id": { "$in": ids }
-          }
-        })
-        .unwrap_or_else(|| doc! { "target.type": "Build" });
-
-      let repo_query = get_resource_ids_for_user::<Repo>(user)
-        .await?
-        .map(|ids| {
-          doc! {
-            "target.type": "Repo", "target.id": { "$in": ids }
-          }
-        })
-        .unwrap_or_else(|| doc! { "target.type": "Repo" });
-
-      let procedure_query =
-        get_resource_ids_for_user::<Procedure>(user)
-          .await?
-          .map(|ids| {
-            doc! {
-              "target.type": "Procedure", "target.id": { "$in": ids }
-            }
-          })
-          .unwrap_or_else(|| doc! { "target.type": "Procedure" });
-
-      let action_query = get_resource_ids_for_user::<Action>(user)
-        .await?
-        .map(|ids| {
-          doc! {
-            "target.type": "Action", "target.id": { "$in": ids }
-          }
-        })
-        .unwrap_or_else(|| doc! { "target.type": "Action" });
-
-      let builder_query = get_resource_ids_for_user::<Builder>(user)
-        .await?
-        .map(|ids| {
-          doc! {
-            "target.type": "Builder", "target.id": { "$in": ids }
-          }
-        })
-        .unwrap_or_else(|| doc! { "target.type": "Builder" });
-
-      let alerter_query = get_resource_ids_for_user::<Alerter>(user)
-        .await?
-        .map(|ids| {
-          doc! {
-            "target.type": "Alerter", "target.id": { "$in": ids }
-          }
-        })
-        .unwrap_or_else(|| doc! { "target.type": "Alerter" });
-
-      let resource_sync_query = get_resource_ids_for_user::<
-        ResourceSync,
-      >(user)
-      .await?
-      .map(|ids| {
-        doc! {
-          "target.type": "ResourceSync", "target.id": { "$in": ids }
-        }
-      })
-      .unwrap_or_else(|| doc! { "target.type": "ResourceSync" });
-
-      let mut query = self.query.unwrap_or_default();
-      query.extend(doc! {
-        "$or": [
-          server_query,
-          deployment_query,
-          stack_query,
-          build_query,
-          repo_query,
-          procedure_query,
-          action_query,
-          alerter_query,
-          builder_query,
-          resource_sync_query,
-        ]
-      });
-      query.into()
-    };
+    let query = user_resource_target_query(user, self.query).await?;
 
     let usernames = find_collect(&db_client().users, None, None)
       .await
@@ -222,93 +100,12 @@ impl Resolve<ReadArgs> for GetUpdate {
     if user.admin || core_config().transparent_mode {
       return Ok(update);
     }
-    match &update.target {
-      ResourceTarget::System(_) => {
-        return Err(
-          anyhow!("user must be admin to view system updates").into(),
-        );
-      }
-      ResourceTarget::Server(id) => {
-        get_check_permissions::<Server>(
-          id,
-          user,
-          PermissionLevel::Read.into(),
-        )
-        .await?;
-      }
-      ResourceTarget::Deployment(id) => {
-        get_check_permissions::<Deployment>(
-          id,
-          user,
-          PermissionLevel::Read.into(),
-        )
-        .await?;
-      }
-      ResourceTarget::Build(id) => {
-        get_check_permissions::<Build>(
-          id,
-          user,
-          PermissionLevel::Read.into(),
-        )
-        .await?;
-      }
-      ResourceTarget::Repo(id) => {
-        get_check_permissions::<Repo>(
-          id,
-          user,
-          PermissionLevel::Read.into(),
-        )
-        .await?;
-      }
-      ResourceTarget::Builder(id) => {
-        get_check_permissions::<Builder>(
-          id,
-          user,
-          PermissionLevel::Read.into(),
-        )
-        .await?;
-      }
-      ResourceTarget::Alerter(id) => {
-        get_check_permissions::<Alerter>(
-          id,
-          user,
-          PermissionLevel::Read.into(),
-        )
-        .await?;
-      }
-      ResourceTarget::Procedure(id) => {
-        get_check_permissions::<Procedure>(
-          id,
-          user,
-          PermissionLevel::Read.into(),
-        )
-        .await?;
-      }
-      ResourceTarget::Action(id) => {
-        get_check_permissions::<Action>(
-          id,
-          user,
-          PermissionLevel::Read.into(),
-        )
-        .await?;
-      }
-      ResourceTarget::ResourceSync(id) => {
-        get_check_permissions::<ResourceSync>(
-          id,
-          user,
-          PermissionLevel::Read.into(),
-        )
-        .await?;
-      }
-      ResourceTarget::Stack(id) => {
-        get_check_permissions::<Stack>(
-          id,
-          user,
-          PermissionLevel::Read.into(),
-        )
-        .await?;
-      }
-    }
+    check_user_target_access(
+      &update.target,
+      user,
+      PermissionLevel::Read.into(),
+    )
+    .await?;
     Ok(update)
   }
 }

bin/core/src/api/terminal.rs

@@ -1,26 +1,15 @@
 use anyhow::Context;
 use axum::{Extension, Router, middleware, routing::post};
-use komodo_client::{
-  api::terminal::*,
-  entities::{
-    deployment::Deployment, permission::PermissionLevel,
-    server::Server, stack::Stack, user::User,
-  },
-};
+use komodo_client::{api::terminal::*, entities::user::User};
 use serror::Json;
 
 use crate::{
-  auth::auth_request, helpers::periphery_client,
-  permission::get_check_permissions, resource::get,
-  state::stack_status_cache,
+  auth::auth_request, helpers::terminal::setup_target_for_user,
 };
 
 pub fn router() -> Router {
   Router::new()
     .route("/execute", post(execute_terminal))
-    .route("/execute/container", post(execute_container_exec))
-    .route("/execute/deployment", post(execute_deployment_exec))
-    .route("/execute/stack", post(execute_stack_exec))
     .layer(middleware::from_fn(auth_request))
 }
 
@@ -33,187 +22,29 @@ pub fn router() -> Router {
   skip_all,
   fields(
     operator = user.id,
-    server,
+    target,
     terminal,
+    init = format!("{init:?}")
   )
 )]
 async fn execute_terminal(
   Extension(user): Extension<User>,
   Json(ExecuteTerminalBody {
-    server,
+    target,
     terminal,
     command,
+    init,
   }): Json<ExecuteTerminalBody>,
 ) -> serror::Result<axum::body::Body> {
   info!("/terminal/execute request | user: {}", user.username);
 
-  let server = get_check_permissions::<Server>(
-    &server,
-    &user,
-    PermissionLevel::Read.terminal(),
-  )
-  .await?;
-
-  let stream = periphery_client(&server)
-    .await?
-    .execute_terminal(terminal, command)
-    .await
-    .context("Failed to execute command on periphery")?;
-
-  Ok(axum::body::Body::from_stream(stream))
-}
-
-// ======================
-//  ExecuteContainerExec
-// ======================
-
-#[instrument(
-  name = "ExecuteContainerExec",
-  skip_all,
-  fields(
-    operator = user.id,
-    server,
-    container,
-    shell,
-    recreate = format!("{recreate:?}"),
-  )
-)]
-async fn execute_container_exec(
-  Extension(user): Extension<User>,
-  Json(ExecuteContainerExecBody {
-    server,
-    container,
-    shell,
-    command,
-    recreate,
-  }): Json<ExecuteContainerExecBody>,
-) -> serror::Result<axum::body::Body> {
-  info!("ExecuteContainerExec request | user: {}", user.username);
-
-  let server = get_check_permissions::<Server>(
-    &server,
-    &user,
-    PermissionLevel::Read.terminal(),
-  )
-  .await?;
-
-  let periphery = periphery_client(&server).await?;
+  let (target, terminal, periphery) =
+    setup_target_for_user(target, terminal, init, &user).await?;
 
   let stream = periphery
-    .execute_container_exec(container, shell, command, recreate)
+    .execute_terminal(target, terminal, command)
     .await
-    .context(
-      "Failed to execute container exec command on periphery",
-    )?;
-
-  Ok(axum::body::Body::from_stream(stream))
-}
-
-// =======================
-//  ExecuteDeploymentExec
-// =======================
-
-#[instrument(
-  name = "ExecuteDeploymentExec",
-  skip_all,
-  fields(
-    operator = user.id,
-    deployment,
-    shell,
-    recreate = format!("{recreate:?}"),
-  )
-)]
-async fn execute_deployment_exec(
-  Extension(user): Extension<User>,
-  Json(ExecuteDeploymentExecBody {
-    deployment,
-    shell,
-    command,
-    recreate,
-  }): Json<ExecuteDeploymentExecBody>,
-) -> serror::Result<axum::body::Body> {
-  info!("ExecuteDeploymentExec request | user: {}", user.username);
-
-  let deployment = get_check_permissions::<Deployment>(
-    &deployment,
-    &user,
-    PermissionLevel::Read.terminal(),
-  )
-  .await?;
-
-  let server = get::<Server>(&deployment.config.server_id).await?;
-
-  let periphery = periphery_client(&server).await?;
-
-  let stream = periphery
-    .execute_container_exec(deployment.name, shell, command, recreate)
-    .await
-    .context(
-      "Failed to execute container exec command on periphery",
-    )?;
-
-  Ok(axum::body::Body::from_stream(stream))
-}
-
-// ==================
-//  ExecuteStackExec
-// ==================
-
-#[instrument(
-  name = "ExecuteStackExec",
-  skip_all,
-  fields(
-    operator = user.id,
-    stack,
-    service,
-    shell,
-    recreate = format!("{recreate:?}"),
-  )
-)]
-async fn execute_stack_exec(
-  Extension(user): Extension<User>,
-  Json(ExecuteStackExecBody {
-    stack,
-    service,
-    shell,
-    command,
-    recreate,
-  }): Json<ExecuteStackExecBody>,
-) -> serror::Result<axum::body::Body> {
-  info!("ExecuteStackExec request | user: {}", user.username);
-
-  let stack = get_check_permissions::<Stack>(
-    &stack,
-    &user,
-    PermissionLevel::Read.terminal(),
-  )
-  .await?;
-
-  let server = get::<Server>(&stack.config.server_id).await?;
-
-  let container = stack_status_cache()
-    .get(&stack.id)
-    .await
-    .context("could not get stack status")?
-    .curr
-    .services
-    .iter()
-    .find(|s| s.service == service)
-    .context("could not find service")?
-    .container
-    .as_ref()
-    .context("could not find service container")?
-    .name
-    .clone();
-
-  let periphery = periphery_client(&server).await?;
-
-  let stream = periphery
-    .execute_container_exec(container, shell, command, recreate)
-    .await
-    .context(
-      "Failed to execute container exec command on periphery",
-    )?;
+    .context("Failed to execute command on Terminal")?;
 
   Ok(axum::body::Body::from_stream(stream))
 }

bin/core/src/api/user.rs

@@ -1,209 +0,0 @@
-use std::{collections::VecDeque, time::Instant};
-
-use anyhow::{Context, anyhow};
-use axum::{
-  Extension, Json, Router, extract::Path, middleware, routing::post,
-};
-use database::mongo_indexed::doc;
-use database::mungos::{
-  by_id::update_one_by_id, mongodb::bson::to_bson,
-};
-use derive_variants::EnumVariants;
-use komodo_client::entities::random_string;
-use komodo_client::{
-  api::user::*,
-  entities::{api_key::ApiKey, komodo_timestamp, user::User},
-};
-use resolver_api::Resolve;
-use response::Response;
-use serde::{Deserialize, Serialize};
-use serde_json::json;
-use typeshare::typeshare;
-use uuid::Uuid;
-
-use crate::{
-  auth::auth_request, helpers::query::get_user, state::db_client,
-};
-
-use super::Variant;
-
-pub struct UserArgs {
-  pub user: User,
-}
-
-#[typeshare]
-#[derive(
-  Serialize, Deserialize, Debug, Clone, Resolve, EnumVariants,
-)]
-#[args(UserArgs)]
-#[response(Response)]
-#[error(serror::Error)]
-#[serde(tag = "type", content = "params")]
-enum UserRequest {
-  PushRecentlyViewed(PushRecentlyViewed),
-  SetLastSeenUpdate(SetLastSeenUpdate),
-  CreateApiKey(CreateApiKey),
-  DeleteApiKey(DeleteApiKey),
-}
-
-pub fn router() -> Router {
-  Router::new()
-    .route("/", post(handler))
-    .route("/{variant}", post(variant_handler))
-    .layer(middleware::from_fn(auth_request))
-}
-
-async fn variant_handler(
-  user: Extension<User>,
-  Path(Variant { variant }): Path<Variant>,
-  Json(params): Json<serde_json::Value>,
-) -> serror::Result<axum::response::Response> {
-  let req: UserRequest = serde_json::from_value(json!({
-    "type": variant,
-    "params": params,
-  }))?;
-  handler(user, Json(req)).await
-}
-
-async fn handler(
-  Extension(user): Extension<User>,
-  Json(request): Json<UserRequest>,
-) -> serror::Result<axum::response::Response> {
-  let timer = Instant::now();
-  let req_id = Uuid::new_v4();
-  debug!(
-    "/user request {req_id} | user: {} ({})",
-    user.username, user.id
-  );
-  let res = request.resolve(&UserArgs { user }).await;
-  if let Err(e) = &res {
-    warn!("/user request {req_id} error: {:#}", e.error);
-  }
-  let elapsed = timer.elapsed();
-  debug!("/user request {req_id} | resolve time: {elapsed:?}");
-  res.map(|res| res.0)
-}
-
-const RECENTLY_VIEWED_MAX: usize = 10;
-
-impl Resolve<UserArgs> for PushRecentlyViewed {
-  async fn resolve(
-    self,
-    UserArgs { user }: &UserArgs,
-  ) -> serror::Result<PushRecentlyViewedResponse> {
-    let user = get_user(&user.id).await?;
-
-    let (resource_type, id) = self.resource.extract_variant_id();
-    let update = match user.recents.get(&resource_type) {
-      Some(recents) => {
-        let mut recents = recents
-          .iter()
-          .filter(|_id| !id.eq(*_id))
-          .take(RECENTLY_VIEWED_MAX - 1)
-          .collect::<VecDeque<_>>();
-        recents.push_front(id);
-        doc! { format!("recents.{resource_type}"): to_bson(&recents)? }
-      }
-      None => {
-        doc! { format!("recents.{resource_type}"): [id] }
-      }
-    };
-    update_one_by_id(
-      &db_client().users,
-      &user.id,
-      database::mungos::update::Update::Set(update),
-      None,
-    )
-    .await
-    .with_context(|| {
-      format!("failed to update recents.{resource_type}")
-    })?;
-
-    Ok(PushRecentlyViewedResponse {})
-  }
-}
-
-impl Resolve<UserArgs> for SetLastSeenUpdate {
-  async fn resolve(
-    self,
-    UserArgs { user }: &UserArgs,
-  ) -> serror::Result<SetLastSeenUpdateResponse> {
-    update_one_by_id(
-      &db_client().users,
-      &user.id,
-      database::mungos::update::Update::Set(doc! {
-        "last_update_view": komodo_timestamp()
-      }),
-      None,
-    )
-    .await
-    .context("failed to update user last_update_view")?;
-    Ok(SetLastSeenUpdateResponse {})
-  }
-}
-
-const SECRET_LENGTH: usize = 40;
-const BCRYPT_COST: u32 = 10;
-
-impl Resolve<UserArgs> for CreateApiKey {
-  #[instrument(
-    "CreateApiKey",
-    skip_all,
-    fields(operator = user.id)
-  )]
-  async fn resolve(
-    self,
-    UserArgs { user }: &UserArgs,
-  ) -> serror::Result<CreateApiKeyResponse> {
-    let user = get_user(&user.id).await?;
-
-    let key = format!("K-{}", random_string(SECRET_LENGTH));
-    let secret = format!("S-{}", random_string(SECRET_LENGTH));
-    let secret_hash = bcrypt::hash(&secret, BCRYPT_COST)
-      .context("failed at hashing secret string")?;
-
-    let api_key = ApiKey {
-      name: self.name,
-      key: key.clone(),
-      secret: secret_hash,
-      user_id: user.id.clone(),
-      created_at: komodo_timestamp(),
-      expires: self.expires,
-    };
-    db_client()
-      .api_keys
-      .insert_one(api_key)
-      .await
-      .context("failed to create api key on db")?;
-    Ok(CreateApiKeyResponse { key, secret })
-  }
-}
-
-impl Resolve<UserArgs> for DeleteApiKey {
-  #[instrument(
-    "DeleteApiKey",
-    skip_all,
-    fields(operator = user.id)
-  )]
-  async fn resolve(
-    self,
-    UserArgs { user }: &UserArgs,
-  ) -> serror::Result<DeleteApiKeyResponse> {
-    let client = db_client();
-    let key = client
-      .api_keys
-      .find_one(doc! { "key": &self.key })
-      .await
-      .context("failed at db query")?
-      .context("no api key with key found")?;
-    if user.id != key.user_id {
-      return Err(anyhow!("api key does not belong to user").into());
-    }
-    client
-      .api_keys
-      .delete_one(doc! { "key": key.key })
-      .await
-      .context("failed to delete api key from db")?;
-    Ok(DeleteApiKeyResponse {})
-  }
-}

bin/core/src/api/user/api_key.rs

@@ -0,0 +1,97 @@
+use anyhow::{Context as _, anyhow};
+use database::bson::doc;
+use komodo_client::{
+  api::user::{
+    CreateApiKey, CreateApiKeyResponse, DeleteApiKey,
+    DeleteApiKeyResponse,
+  },
+  entities::{api_key::ApiKey, komodo_timestamp, random_string},
+};
+use reqwest::StatusCode;
+use resolver_api::Resolve;
+use serror::{AddStatusCode as _, AddStatusCodeError as _};
+
+use crate::{
+  helpers::{query::get_user, validations::validate_api_key_name},
+  state::db_client,
+};
+
+use super::UserArgs;
+
+const SECRET_LENGTH: usize = 40;
+const BCRYPT_COST: u32 = 10;
+
+impl Resolve<UserArgs> for CreateApiKey {
+  #[instrument(
+    "CreateApiKey",
+    skip_all,
+    fields(operator = user.id)
+  )]
+  async fn resolve(
+    self,
+    UserArgs { user, .. }: &UserArgs,
+  ) -> serror::Result<CreateApiKeyResponse> {
+    let user = get_user(&user.id).await?;
+
+    validate_api_key_name(&self.name)
+      .status_code(StatusCode::BAD_REQUEST)?;
+
+    let key = format!("K_{}_K", random_string(SECRET_LENGTH));
+    let secret = format!("S_{}_S", random_string(SECRET_LENGTH));
+    let secret_hash = bcrypt::hash(&secret, BCRYPT_COST)
+      .context("Failed at hashing secret string")?;
+
+    let api_key = ApiKey {
+      name: self.name,
+      key: key.clone(),
+      secret: secret_hash,
+      user_id: user.id.clone(),
+      created_at: komodo_timestamp(),
+      expires: self.expires,
+    };
+
+    db_client()
+      .api_keys
+      .insert_one(api_key)
+      .await
+      .context("Failed to create api key on database")?;
+
+    Ok(CreateApiKeyResponse { key, secret })
+  }
+}
+
+impl Resolve<UserArgs> for DeleteApiKey {
+  #[instrument(
+    "DeleteApiKey",
+    skip_all,
+    fields(operator = user.id)
+  )]
+  async fn resolve(
+    self,
+    UserArgs { user, .. }: &UserArgs,
+  ) -> serror::Result<DeleteApiKeyResponse> {
+    let client = db_client();
+
+    let key = client
+      .api_keys
+      .find_one(doc! { "key": &self.key })
+      .await
+      .context("Failed at database query")?
+      .context("No api key with key found")?;
+
+    if user.id != key.user_id {
+      return Err(
+        anyhow!("Api key does not belong to user")
+          .status_code(StatusCode::FORBIDDEN),
+      );
+    }
+
+    client
+      .api_keys
+      .delete_one(doc! { "key": key.key })
+      .await
+      .context("Failed to delete api key from database")?;
+
+    Ok(DeleteApiKeyResponse {})
+  }
+}

bin/core/src/api/user/mod.rs

@@ -0,0 +1,367 @@
+use std::{collections::VecDeque, time::Instant};
+
+use anyhow::{Context, anyhow};
+use axum::{
+  Extension, Json, Router, extract::Path, middleware, routing::post,
+};
+use database::{
+  bson::{doc, to_bson},
+  mungos::by_id::update_one_by_id,
+};
+use derive_variants::EnumVariants;
+use komodo_client::{
+  api::user::*,
+  entities::{NoData, komodo_timestamp, user::User},
+};
+use reqwest::StatusCode;
+use resolver_api::Resolve;
+use response::Response;
+use serde::{Deserialize, Serialize};
+use serde_json::json;
+use serror::{AddStatusCode as _, AddStatusCodeError};
+use tower_sessions::Session;
+use typeshare::typeshare;
+use uuid::Uuid;
+
+use crate::{
+  auth::auth_request,
+  config::core_config,
+  helpers::{
+    query::get_user,
+    validations::{validate_password, validate_username},
+  },
+  state::db_client,
+};
+
+use super::Variant;
+
+mod api_key;
+mod passkey;
+mod totp;
+
+pub struct UserArgs {
+  pub user: User,
+  /// Per-client session state
+  pub session: Option<Session>,
+}
+
+#[typeshare]
+#[derive(
+  Debug, Clone, Serialize, Deserialize, Resolve, EnumVariants,
+)]
+#[args(UserArgs)]
+#[response(Response)]
+#[error(serror::Error)]
+#[serde(tag = "type", content = "params")]
+enum UserRequest {
+  PushRecentlyViewed(PushRecentlyViewed),
+  SetLastSeenUpdate(SetLastSeenUpdate),
+  UpdateUsername(UpdateUsername),
+  UpdatePassword(UpdatePassword),
+  CreateApiKey(CreateApiKey),
+  DeleteApiKey(DeleteApiKey),
+  BeginTotpEnrollment(BeginTotpEnrollment),
+  ConfirmTotpEnrollment(ConfirmTotpEnrollment),
+  UnenrollTotp(UnenrollTotp),
+  BeginPasskeyEnrollment(BeginPasskeyEnrollment),
+  ConfirmPasskeyEnrollment(ConfirmPasskeyEnrollment),
+  UnenrollPasskey(UnenrollPasskey),
+  BeginThirdPartyLoginLink(BeginThirdPartyLoginLink),
+  UnlinkLogin(UnlinkLogin),
+  UpdateThirdPartySkip2fa(UpdateThirdPartySkip2fa),
+}
+
+pub fn router() -> Router {
+  Router::new()
+    .route("/", post(handler))
+    .route("/{variant}", post(variant_handler))
+    .layer(middleware::from_fn(auth_request))
+}
+
+async fn variant_handler(
+  session: Session,
+  user: Extension<User>,
+  Path(Variant { variant }): Path<Variant>,
+  Json(params): Json<serde_json::Value>,
+) -> serror::Result<axum::response::Response> {
+  let req: UserRequest = serde_json::from_value(json!({
+    "type": variant,
+    "params": params,
+  }))?;
+  handler(session, user, Json(req)).await
+}
+
+async fn handler(
+  session: Session,
+  Extension(user): Extension<User>,
+  Json(request): Json<UserRequest>,
+) -> serror::Result<axum::response::Response> {
+  let timer = Instant::now();
+  let req_id = Uuid::new_v4();
+  debug!(
+    "/user request {req_id} | user: {} ({})",
+    user.username, user.id
+  );
+  let res = request
+    .resolve(&UserArgs {
+      user,
+      session: Some(session),
+    })
+    .await;
+  if let Err(e) = &res {
+    warn!("/user request {req_id} error: {:#}", e.error);
+  }
+  let elapsed = timer.elapsed();
+  debug!("/user request {req_id} | resolve time: {elapsed:?}");
+  res.map(|res| res.0)
+}
+
+const RECENTLY_VIEWED_MAX: usize = 10;
+
+impl Resolve<UserArgs> for PushRecentlyViewed {
+  async fn resolve(
+    self,
+    UserArgs { user, .. }: &UserArgs,
+  ) -> serror::Result<PushRecentlyViewedResponse> {
+    let user = get_user(&user.id).await?;
+
+    let (resource_type, id) = self.resource.extract_variant_id();
+
+    let field = format!("recents.{resource_type}");
+
+    let update = match user.recents.get(&resource_type) {
+      Some(recents) => {
+        let mut recents = recents
+          .iter()
+          .filter(|_id| !id.eq(*_id))
+          .take(RECENTLY_VIEWED_MAX - 1)
+          .collect::<VecDeque<_>>();
+
+        recents.push_front(id);
+
+        doc! { &field: to_bson(&recents)? }
+      }
+      None => {
+        doc! { &field: [id] }
+      }
+    };
+
+    update_one_by_id(
+      &db_client().users,
+      &user.id,
+      database::mungos::update::Update::Set(update),
+      None,
+    )
+    .await
+    .with_context(|| format!("Failed to update user '{field}'"))?;
+
+    Ok(PushRecentlyViewedResponse {})
+  }
+}
+
+impl Resolve<UserArgs> for SetLastSeenUpdate {
+  async fn resolve(
+    self,
+    UserArgs { user, .. }: &UserArgs,
+  ) -> serror::Result<SetLastSeenUpdateResponse> {
+    update_one_by_id(
+      &db_client().users,
+      &user.id,
+      database::mungos::update::Update::Set(doc! {
+        "last_update_view": komodo_timestamp()
+      }),
+      None,
+    )
+    .await
+    .context("Failed to update user 'last_update_view'")?;
+
+    Ok(SetLastSeenUpdateResponse {})
+  }
+}
+
+//
+
+impl Resolve<UserArgs> for UpdateUsername {
+  #[instrument(
+    "UpdateUsername",
+    skip_all,
+    fields(
+      operator = user.id,
+      new_username = self.username,
+    )
+  )]
+  async fn resolve(
+    self,
+    UserArgs { user, .. }: &UserArgs,
+  ) -> serror::Result<UpdateUsernameResponse> {
+    check_locked(&user.username)?;
+
+    validate_username(&self.username)?;
+
+    let db = db_client();
+
+    if db
+      .users
+      .find_one(doc! { "username": &self.username })
+      .await
+      .context("Failed to query for existing users")?
+      .is_some()
+    {
+      return Err(anyhow!("Username already taken.").into());
+    }
+
+    let update = doc! { "$set": { "username": self.username } };
+
+    update_one_by_id(&db.users, &user.id, update, None)
+      .await
+      .context("Failed to update user username on database.")?;
+
+    Ok(NoData {})
+  }
+}
+
+//
+
+impl Resolve<UserArgs> for UpdatePassword {
+  #[instrument(
+    "UpdatePassword",
+    skip_all,
+    fields(operator = user.id)
+  )]
+  async fn resolve(
+    self,
+    UserArgs { user, .. }: &UserArgs,
+  ) -> serror::Result<UpdatePasswordResponse> {
+    check_locked(&user.username)?;
+
+    validate_password(&self.password)
+      .status_code(StatusCode::BAD_REQUEST)?;
+
+    db_client().set_user_password(user, &self.password).await?;
+
+    Ok(NoData {})
+  }
+}
+
+//
+
+#[derive(Serialize, Deserialize)]
+pub struct SessionThirdPartyLinkInfo {
+  pub user_id: String,
+}
+
+impl SessionThirdPartyLinkInfo {
+  pub const KEY: &str = "third-party-link-info";
+}
+
+impl Resolve<UserArgs> for BeginThirdPartyLoginLink {
+  #[instrument(
+    "BeginThirdPartyLoginLink",
+    skip_all,
+    fields(operator = user.id)
+  )]
+  async fn resolve(
+    self,
+    UserArgs { user, session }: &UserArgs,
+  ) -> serror::Result<BeginThirdPartyLoginLinkResponse> {
+    check_locked(&user.username)?;
+
+    let session = session.as_ref().context(
+      "Method called in invalid context. This should not happen",
+    )?;
+
+    session
+      .insert(
+        SessionThirdPartyLinkInfo::KEY,
+        SessionThirdPartyLinkInfo {
+          user_id: user.id.clone(),
+        },
+      )
+      .await
+      .context(
+        "Failed to insert third party link info into client session",
+      )?;
+
+    Ok(NoData {})
+  }
+}
+
+impl Resolve<UserArgs> for UnlinkLogin {
+  #[instrument(
+    "UnlinkLogin",
+    skip_all,
+    fields(operator = user.id)
+  )]
+  async fn resolve(
+    self,
+    UserArgs { user, .. }: &UserArgs,
+  ) -> serror::Result<UnlinkLoginResponse> {
+    check_locked(&user.username)?;
+
+    let field = match self.provider.to_lowercase().as_str() {
+      "local" => "linked_logins.Local",
+      "github" => "linked_logins.Github",
+      "google" => "linked_logins.Google",
+      "oidc" => "linked_logins.Oidc",
+      _ => {
+        return Err(
+          anyhow!(
+            "Unrecognized third party login provider: {}",
+            self.provider
+          )
+          .status_code(StatusCode::BAD_REQUEST),
+        );
+      }
+    };
+
+    let update = doc! {
+      "$unset": {
+        field: ""
+      }
+    };
+
+    update_one_by_id(&db_client().users, &user.id, update, None)
+      .await
+      .context("Failed to unlink third partly login on database")?;
+
+    Ok(NoData {})
+  }
+}
+
+impl Resolve<UserArgs> for UpdateThirdPartySkip2fa {
+  #[instrument(
+    "UpdateThirdPartySkip2fa",
+    skip_all,
+    fields(operator = user.id)
+  )]
+  async fn resolve(
+    self,
+    UserArgs { user, .. }: &UserArgs,
+  ) -> serror::Result<UpdateThirdPartySkip2faResponse> {
+    check_locked(&user.username)?;
+
+    let update = doc! {
+      "$set": {
+        "third_party_skip_2fa": self.skip
+      }
+    };
+
+    update_one_by_id(&db_client().users, &user.id, update, None)
+      .await
+      .context("Failed to unlink third partly login on database")?;
+
+    Ok(NoData {})
+  }
+}
+
+fn check_locked(username: &str) -> serror::Result<()> {
+  for locked_username in &core_config().lock_login_credentials_for {
+    if locked_username == "__ALL__" || *locked_username == username {
+      return Err(
+        anyhow!("User login credentials are locked.")
+          .status_code(StatusCode::UNAUTHORIZED),
+      );
+    }
+  }
+  Ok(())
+}

bin/core/src/api/user/passkey.rs

@@ -0,0 +1,151 @@
+use anyhow::Context as _;
+use database::{
+  bson::{doc, to_bson},
+  mungos::by_id::update_one_by_id,
+};
+use komodo_client::{
+  api::user::{
+    BeginPasskeyEnrollment, BeginPasskeyEnrollmentResponse,
+    ConfirmPasskeyEnrollment, ConfirmPasskeyEnrollmentResponse,
+    UnenrollPasskey, UnenrollPasskeyResponse,
+  },
+  entities::komodo_timestamp,
+};
+use resolver_api::Resolve;
+use serde::{Deserialize, Serialize};
+use uuid::Uuid;
+use webauthn_rs::prelude::PasskeyRegistration;
+
+use crate::state::{db_client, webauthn};
+
+use super::UserArgs;
+
+#[derive(Serialize, Deserialize)]
+struct SessionPasskeyEnrollment {
+  user_id: String,
+  state: PasskeyRegistration,
+}
+impl SessionPasskeyEnrollment {
+  const KEY: &str = "passkey-enrollment";
+}
+
+impl Resolve<UserArgs> for BeginPasskeyEnrollment {
+  #[instrument(
+    "BeginPasskeyEnrollment",
+    skip_all,
+    fields(operator = user.id)
+  )]
+  async fn resolve(
+    self,
+    UserArgs { user, session }: &UserArgs,
+  ) -> serror::Result<BeginPasskeyEnrollmentResponse> {
+    super::check_locked(&user.username)?;
+
+    let session = session.as_ref().context(
+      "Method called in invalid context. This should not happen",
+    )?;
+
+    let webauthn = webauthn().context(
+      "No webauthn provider available, invalid KOMODO_HOST config",
+    )?;
+
+    // Get two parts from this, the first is returned to the client.
+    // The second must stay server side and is used in confirmation flow.
+    let (challenge, state) = webauthn.start_passkey_registration(
+      Uuid::new_v4(),
+      &user.username,
+      &user.username,
+      None,
+    )?;
+
+    session
+      .insert(
+        SessionPasskeyEnrollment::KEY,
+        SessionPasskeyEnrollment {
+          user_id: user.id.clone(),
+          state
+        },
+      )
+      .await
+      .context(
+        "Failed to store passkey enrollment state in server side client session",
+      )?;
+
+    Ok(challenge)
+  }
+}
+
+//
+
+impl Resolve<UserArgs> for ConfirmPasskeyEnrollment {
+  #[instrument(
+    "ConfirmPasskeyEnrollment",
+    skip_all,
+    fields(operator = user.id)
+  )]
+  async fn resolve(
+    self,
+    UserArgs { user, session }: &UserArgs,
+  ) -> serror::Result<ConfirmPasskeyEnrollmentResponse> {
+    let session = session.as_ref().context(
+      "Method called in invalid context. This should not happen",
+    )?;
+
+    let webauthn = webauthn().context(
+      "No webauthn provider available, invalid KOMODO_HOST config",
+    )?;
+
+    let SessionPasskeyEnrollment { user_id, state } = session
+      .remove(SessionPasskeyEnrollment::KEY)
+      .await
+      .context("Passkey enrollment was not initiated correctly")?
+      .context(
+        "Passkey enrollment was not initiated correctly or timed out",
+      )?;
+
+    let passkey = webauthn
+      .finish_passkey_registration(&self.credential, &state)
+      .context("Failed to finish passkey registration")?;
+
+    let passkey = to_bson(&passkey)
+      .context("Failed to serialize passkey to BSON")?;
+
+    let update = doc! {
+      "$set": {
+        "passkey.passkey": passkey,
+        "passkey.created_at": komodo_timestamp()
+      }
+    };
+
+    update_one_by_id(&db_client().users, &user_id, update, None)
+      .await
+      .context("Failed to update user passkey options on database")?;
+
+    Ok(ConfirmPasskeyEnrollmentResponse {})
+  }
+}
+
+//
+
+impl Resolve<UserArgs> for UnenrollPasskey {
+  #[instrument(
+    "UnenrollPasskey",
+    skip_all,
+    fields(operator = user.id)
+  )]
+  async fn resolve(
+    self,
+    UserArgs { user, .. }: &UserArgs,
+  ) -> serror::Result<UnenrollPasskeyResponse> {
+    let update = doc! {
+      "$set": {
+        "passkey.passkey": null,
+        "passkey.created_at": 0
+      }
+    };
+    update_one_by_id(&db_client().users, &user.id, update, None)
+      .await
+      .context("Failed to update user passkey options on database")?;
+    Ok(UnenrollPasskeyResponse {})
+  }
+}

bin/core/src/api/user/totp.rs

@@ -0,0 +1,158 @@
+use anyhow::{Context as _, anyhow};
+use data_encoding::BASE32_NOPAD;
+use database::{
+  bson::doc, hash_password, mungos::by_id::update_one_by_id,
+};
+use komodo_client::{
+  api::user::{
+    BeginTotpEnrollment, BeginTotpEnrollmentResponse,
+    ConfirmTotpEnrollment, ConfirmTotpEnrollmentResponse,
+    UnenrollTotp, UnenrollTotpResponse,
+  },
+  entities::{komodo_timestamp, random_bytes, random_string},
+};
+use reqwest::StatusCode;
+use resolver_api::Resolve;
+use serde::{Deserialize, Serialize};
+use serror::AddStatusCodeError as _;
+
+use crate::{auth::totp::make_totp, state::db_client};
+
+use super::UserArgs;
+
+const TOTP_ENROLLMENT_SECRET_LENGTH: usize = 20;
+
+#[derive(Serialize, Deserialize)]
+struct SessionTotpEnrollment {
+  secret: Vec<u8>,
+}
+impl SessionTotpEnrollment {
+  const KEY: &str = "totp-enrollment";
+}
+
+impl Resolve<UserArgs> for BeginTotpEnrollment {
+  #[instrument(
+    "BeginTotpEnrollment",
+    skip_all,
+    fields(operator = user.id)
+  )]
+  async fn resolve(
+    self,
+    UserArgs { user, session }: &UserArgs,
+  ) -> serror::Result<BeginTotpEnrollmentResponse> {
+    super::check_locked(&user.username)?;
+
+    let session = session.as_ref().context(
+      "Method called in invalid context. This should not happen",
+    )?;
+
+    let secret = random_bytes(TOTP_ENROLLMENT_SECRET_LENGTH);
+    let totp = make_totp(secret.clone(), user.id.clone())?;
+    let png = totp
+      .get_qr_base64()
+      .map_err(anyhow::Error::msg)
+      .context("Failed to generate QR code png")?;
+    session
+      .insert(
+        SessionTotpEnrollment::KEY,
+        SessionTotpEnrollment { secret },
+      )
+      .await?;
+
+    Ok(BeginTotpEnrollmentResponse {
+      uri: totp.get_url(),
+      png,
+    })
+  }
+}
+
+impl Resolve<UserArgs> for ConfirmTotpEnrollment {
+  #[instrument(
+    "ConfirmTotpEnrollment",
+    skip_all,
+    fields(operator = user.id)
+  )]
+  async fn resolve(
+    self,
+    UserArgs { user, session }: &UserArgs,
+  ) -> serror::Result<ConfirmTotpEnrollmentResponse> {
+    let session = session.as_ref().context(
+      "Method called in invalid context. This should not happen",
+    )?;
+
+    let SessionTotpEnrollment { secret } = session
+      .remove(SessionTotpEnrollment::KEY)
+      .await
+      .context("Totp enrollment was not initiated correctly")?
+      .context(
+        "Totp enrollment was not initiated correctly or timed out",
+      )?;
+
+    let encoded_secret = BASE32_NOPAD.encode(&secret);
+
+    let totp = make_totp(secret, None)?;
+
+    let valid = totp
+      .check_current(&self.code)
+      .context("Failed to check code validity")?;
+
+    if !valid {
+      return Err(anyhow!(
+        "The provided code was not valid. Please try BeginTotpEnrollment flow again."
+      ).status_code(StatusCode::BAD_REQUEST));
+    }
+
+    let recovery_codes =
+      (0..10).map(|_| random_string(20)).collect::<Vec<_>>();
+    let hashed_recovery_codes = recovery_codes
+      .iter()
+      .map(hash_password)
+      .collect::<anyhow::Result<Vec<_>>>()
+      .context("Failed to generate valid recovery codes")?;
+
+    update_one_by_id(
+      &db_client().users,
+      &user.id,
+      doc! {
+        "$set": {
+          "totp.secret": encoded_secret,
+          "totp.confirmed_at": komodo_timestamp(),
+          "totp.recovery_codes": hashed_recovery_codes,
+        }
+      },
+      None,
+    )
+    .await
+    .context("Failed to update user totp fields on database")?;
+
+    Ok(ConfirmTotpEnrollmentResponse { recovery_codes })
+  }
+}
+
+impl Resolve<UserArgs> for UnenrollTotp {
+  #[instrument(
+    "UnenrollTotp",
+    skip_all,
+    fields(operator = user.id)
+  )]
+  async fn resolve(
+    self,
+    UserArgs { user, .. }: &UserArgs,
+  ) -> serror::Result<UnenrollTotpResponse> {
+    update_one_by_id(
+      &db_client().users,
+      &user.id,
+      doc! {
+        "$set": {
+          "totp.secret": "",
+          "totp.confirmed_at": 0,
+          "totp.recovery_codes": [],
+        }
+      },
+      None,
+    )
+    .await
+    .context("Failed to clear user totp fields on database")?;
+    Ok(UnenrollTotpResponse {})
+  }
+}

bin/core/src/api/write/build.rs

@@ -390,23 +390,28 @@ impl Resolve<WriteArgs> for RefreshBuildCache {
       None
     };
 
-    let (
-      remote_path,
-      remote_contents,
-      remote_error,
-      latest_hash,
-      latest_message,
-    ) = if build.config.files_on_host {
+    let RemoteDockerfileContents {
+      path,
+      contents,
+      error,
+      hash,
+      message,
+    } = if build.config.files_on_host {
       // =============
       // FILES ON HOST
       // =============
       match get_on_host_dockerfile(&build).await {
         Ok(FileContents { path, contents }) => {
-          (Some(path), Some(contents), None, None, None)
-        }
-        Err(e) => {
-          (None, None, Some(format_serror(&e.into())), None, None)
+          RemoteDockerfileContents {
+            path: Some(path),
+            contents: Some(contents),
+            ..Default::default()
+          }
         }
+        Err(e) => RemoteDockerfileContents {
+          error: Some(format_serror(&e.into())),
+          ..Default::default()
+        },
       }
     } else if let Some(repo) = &repo {
       let Some(res) = get_git_remote(&build, repo.into()).await?
@@ -426,7 +431,7 @@ impl Resolve<WriteArgs> for RefreshBuildCache {
       // =============
       // UI BASED FILE
       // =============
-      (None, None, None, None, None)
+      RemoteDockerfileContents::default()
     };
 
     let info = BuildInfo {
@@ -434,11 +439,11 @@ impl Resolve<WriteArgs> for RefreshBuildCache {
       built_hash: build.info.built_hash,
       built_message: build.info.built_message,
       built_contents: build.info.built_contents,
-      remote_path,
-      remote_contents,
-      remote_error,
-      latest_hash,
-      latest_message,
+      remote_path: path,
+      remote_contents: contents,
+      remote_error: error,
+      latest_hash: hash,
+      latest_message: message,
     };
 
     let info = to_document(&info)
@@ -530,15 +535,7 @@ async fn get_on_host_dockerfile(
 async fn get_git_remote(
   build: &Build,
   mut clone_args: RepoExecutionArgs,
-) -> anyhow::Result<
-  Option<(
-    Option<String>,
-    Option<String>,
-    Option<String>,
-    Option<String>,
-    Option<String>,
-  )>,
-> {
+) -> anyhow::Result<Option<RemoteDockerfileContents>> {
   if clone_args.provider.is_empty() {
     // Nothing to do here
     return Ok(None);
@@ -565,7 +562,17 @@ async fn get_git_remote(
     access_token,
   )
   .await
-  .context("failed to clone build repo")?;
+  .context("Failed to clone Build repo")?;
+
+  // Ensure clone / pull successful,
+  // propogate error log -> 'errored' and return.
+  if let Some(failure) = res.logs.iter().find(|log| !log.success) {
+    return Ok(Some(RemoteDockerfileContents {
+      path: Some(format!("Failed at: {}", failure.stage)),
+      error: Some(failure.combined()),
+      ..Default::default()
+    }));
+  }
 
   let relative_path = PathBuf::from(&build.config.build_path)
     .join(&build.config.dockerfile_path);
@@ -578,11 +585,20 @@ async fn get_git_remote(
       Ok(contents) => (Some(contents), None),
       Err(e) => (None, Some(format_serror(&e.into()))),
     };
-  Ok(Some((
-    Some(relative_path.display().to_string()),
+  Ok(Some(RemoteDockerfileContents {
+    path: Some(relative_path.display().to_string()),
     contents,
     error,
-    res.commit_hash,
-    res.commit_message,
-  )))
+    hash: res.commit_hash,
+    message: res.commit_message,
+  }))
+}
+
+#[derive(Default)]
+pub struct RemoteDockerfileContents {
+  pub path: Option<String>,
+  pub contents: Option<String>,
+  pub error: Option<String>,
+  pub hash: Option<String>,
+  pub message: Option<String>,
 }

bin/core/src/api/write/mod.rs

@@ -23,7 +23,7 @@ mod alerter;
 mod build;
 mod builder;
 mod deployment;
-mod onboarding_key;
+mod onboarding;
 mod permissions;
 mod procedure;
 mod provider;
@@ -32,8 +32,10 @@ mod resource;
 mod server;
 mod service_user;
 mod stack;
+mod swarm;
 mod sync;
 mod tag;
+mod terminal;
 mod user;
 mod user_group;
 mod variable;
@@ -52,36 +54,16 @@ pub struct WriteArgs {
 #[error(serror::Error)]
 #[serde(tag = "type", content = "params")]
 pub enum WriteRequest {
-  // ==== USER ====
-  CreateLocalUser(CreateLocalUser),
-  UpdateUserUsername(UpdateUserUsername),
-  UpdateUserPassword(UpdateUserPassword),
-  DeleteUser(DeleteUser),
-
-  // ==== SERVICE USER ====
-  CreateServiceUser(CreateServiceUser),
-  UpdateServiceUserDescription(UpdateServiceUserDescription),
-  CreateApiKeyForServiceUser(CreateApiKeyForServiceUser),
-  DeleteApiKeyForServiceUser(DeleteApiKeyForServiceUser),
-
-  // ==== USER GROUP ====
-  CreateUserGroup(CreateUserGroup),
-  RenameUserGroup(RenameUserGroup),
-  DeleteUserGroup(DeleteUserGroup),
-  AddUserToUserGroup(AddUserToUserGroup),
-  RemoveUserFromUserGroup(RemoveUserFromUserGroup),
-  SetUsersInUserGroup(SetUsersInUserGroup),
-  SetEveryoneUserGroup(SetEveryoneUserGroup),
-
-  // ==== PERMISSIONS ====
-  UpdateUserAdmin(UpdateUserAdmin),
-  UpdateUserBasePermissions(UpdateUserBasePermissions),
-  UpdatePermissionOnResourceType(UpdatePermissionOnResourceType),
-  UpdatePermissionOnTarget(UpdatePermissionOnTarget),
-
   // ==== RESOURCE ====
   UpdateResourceMeta(UpdateResourceMeta),
 
+  // ==== SWARM ====
+  CreateSwarm(CreateSwarm),
+  CopySwarm(CopySwarm),
+  DeleteSwarm(DeleteSwarm),
+  UpdateSwarm(UpdateSwarm),
+  RenameSwarm(RenameSwarm),
+
   // ==== SERVER ====
   CreateServer(CreateServer),
   CopyServer(CopyServer),
@@ -89,11 +71,14 @@ pub enum WriteRequest {
   UpdateServer(UpdateServer),
   RenameServer(RenameServer),
   CreateNetwork(CreateNetwork),
+  UpdateServerPublicKey(UpdateServerPublicKey),
+  RotateServerKeys(RotateServerKeys),
+
+  // ==== TERMINAL ====
   CreateTerminal(CreateTerminal),
   DeleteTerminal(DeleteTerminal),
   DeleteAllTerminals(DeleteAllTerminals),
-  UpdateServerPublicKey(UpdateServerPublicKey),
-  RotateServerKeys(RotateServerKeys),
+  BatchDeleteAllTerminals(BatchDeleteAllTerminals),
 
   // ==== STACK ====
   CreateStack(CreateStack),
@@ -121,13 +106,6 @@ pub enum WriteRequest {
   WriteBuildFileContents(WriteBuildFileContents),
   RefreshBuildCache(RefreshBuildCache),
 
-  // ==== BUILDER ====
-  CreateBuilder(CreateBuilder),
-  CopyBuilder(CopyBuilder),
-  DeleteBuilder(DeleteBuilder),
-  UpdateBuilder(UpdateBuilder),
-  RenameBuilder(RenameBuilder),
-
   // ==== REPO ====
   CreateRepo(CreateRepo),
   CopyRepo(CopyRepo),
@@ -136,13 +114,6 @@ pub enum WriteRequest {
   RenameRepo(RenameRepo),
   RefreshRepoCache(RefreshRepoCache),
 
-  // ==== ALERTER ====
-  CreateAlerter(CreateAlerter),
-  CopyAlerter(CopyAlerter),
-  DeleteAlerter(DeleteAlerter),
-  UpdateAlerter(UpdateAlerter),
-  RenameAlerter(RenameAlerter),
-
   // ==== PROCEDURE ====
   CreateProcedure(CreateProcedure),
   CopyProcedure(CopyProcedure),
@@ -167,6 +138,50 @@ pub enum WriteRequest {
   CommitSync(CommitSync),
   RefreshResourceSyncPending(RefreshResourceSyncPending),
 
+  // ==== BUILDER ====
+  CreateBuilder(CreateBuilder),
+  CopyBuilder(CopyBuilder),
+  DeleteBuilder(DeleteBuilder),
+  UpdateBuilder(UpdateBuilder),
+  RenameBuilder(RenameBuilder),
+
+  // ==== ALERTER ====
+  CreateAlerter(CreateAlerter),
+  CopyAlerter(CopyAlerter),
+  DeleteAlerter(DeleteAlerter),
+  UpdateAlerter(UpdateAlerter),
+  RenameAlerter(RenameAlerter),
+
+  // ==== ONBOARDING KEY ====
+  CreateOnboardingKey(CreateOnboardingKey),
+  UpdateOnboardingKey(UpdateOnboardingKey),
+  DeleteOnboardingKey(DeleteOnboardingKey),
+
+  // ==== USER ====
+  CreateLocalUser(CreateLocalUser),
+  DeleteUser(DeleteUser),
+
+  // ==== SERVICE USER ====
+  CreateServiceUser(CreateServiceUser),
+  UpdateServiceUserDescription(UpdateServiceUserDescription),
+  CreateApiKeyForServiceUser(CreateApiKeyForServiceUser),
+  DeleteApiKeyForServiceUser(DeleteApiKeyForServiceUser),
+
+  // ==== USER GROUP ====
+  CreateUserGroup(CreateUserGroup),
+  RenameUserGroup(RenameUserGroup),
+  DeleteUserGroup(DeleteUserGroup),
+  AddUserToUserGroup(AddUserToUserGroup),
+  RemoveUserFromUserGroup(RemoveUserFromUserGroup),
+  SetUsersInUserGroup(SetUsersInUserGroup),
+  SetEveryoneUserGroup(SetEveryoneUserGroup),
+
+  // ==== PERMISSIONS ====
+  UpdateUserAdmin(UpdateUserAdmin),
+  UpdateUserBasePermissions(UpdateUserBasePermissions),
+  UpdatePermissionOnResourceType(UpdatePermissionOnResourceType),
+  UpdatePermissionOnTarget(UpdatePermissionOnTarget),
+
   // ==== TAG ====
   CreateTag(CreateTag),
   DeleteTag(DeleteTag),
@@ -188,11 +203,6 @@ pub enum WriteRequest {
   UpdateDockerRegistryAccount(UpdateDockerRegistryAccount),
   DeleteDockerRegistryAccount(DeleteDockerRegistryAccount),
 
-  // ==== ONBOARDING KEY ====
-  CreateOnboardingKey(CreateOnboardingKey),
-  UpdateOnboardingKey(UpdateOnboardingKey),
-  DeleteOnboardingKey(DeleteOnboardingKey),
-
   // ==== ALERT ====
   CloseAlert(CloseAlert),
 }

bin/core/src/api/write/onboarding.rs

@@ -45,7 +45,7 @@ impl Resolve<WriteArgs> for CreateOnboardingKey {
     let private_key = if let Some(private_key) = self.private_key {
       private_key
     } else {
-      format!("O-{}", random_string(30))
+      format!("O_{}_O", random_string(28))
     };
     let public_key = EncodedKeyPair::from_private_key(&private_key)?
       .public

bin/core/src/api/write/permissions.rs

@@ -307,8 +307,8 @@ async fn extract_user_target_with_validation(
         .users
         .find_one(filter)
         .await
-        .context("failed to query db for users")?
-        .context("no matching user found")?
+        .context("Failed to query db for users")?
+        .context("No matching user found")?
         .id;
       Ok((UserTargetVariant::User, id))
     }
@@ -321,8 +321,8 @@ async fn extract_user_target_with_validation(
         .user_groups
         .find_one(filter)
         .await
-        .context("failed to query db for user_groups")?
-        .context("no matching user_group found")?
+        .context("Failed to query db for user_groups")?
+        .context("No matching user_group found")?
         .id;
       Ok((UserTargetVariant::UserGroup, id))
     }
@@ -338,47 +338,19 @@ async fn extract_resource_target_with_validation(
       let res = resource_target.extract_variant_id();
       Ok((res.0, res.1.clone()))
     }
-    ResourceTarget::Build(ident) => {
+    ResourceTarget::Swarm(ident) => {
       let filter = match ObjectId::from_str(ident) {
         Ok(id) => doc! { "_id": id },
         Err(_) => doc! { "name": ident },
       };
       let id = db_client()
-        .builds
+        .swarms
         .find_one(filter)
         .await
-        .context("failed to query db for builds")?
-        .context("no matching build found")?
+        .context("Failed to query db for swarms")?
+        .context("No matching server found")?
         .id;
-      Ok((ResourceTargetVariant::Build, id))
-    }
-    ResourceTarget::Builder(ident) => {
-      let filter = match ObjectId::from_str(ident) {
-        Ok(id) => doc! { "_id": id },
-        Err(_) => doc! { "name": ident },
-      };
-      let id = db_client()
-        .builders
-        .find_one(filter)
-        .await
-        .context("failed to query db for builders")?
-        .context("no matching builder found")?
-        .id;
-      Ok((ResourceTargetVariant::Builder, id))
-    }
-    ResourceTarget::Deployment(ident) => {
-      let filter = match ObjectId::from_str(ident) {
-        Ok(id) => doc! { "_id": id },
-        Err(_) => doc! { "name": ident },
-      };
-      let id = db_client()
-        .deployments
-        .find_one(filter)
-        .await
-        .context("failed to query db for deployments")?
-        .context("no matching deployment found")?
-        .id;
-      Ok((ResourceTargetVariant::Deployment, id))
+      Ok((ResourceTargetVariant::Server, id))
     }
     ResourceTarget::Server(ident) => {
       let filter = match ObjectId::from_str(ident) {
@@ -389,11 +361,53 @@ async fn extract_resource_target_with_validation(
         .servers
         .find_one(filter)
         .await
-        .context("failed to query db for servers")?
-        .context("no matching server found")?
+        .context("Failed to query db for servers")?
+        .context("No matching server found")?
         .id;
       Ok((ResourceTargetVariant::Server, id))
     }
+    ResourceTarget::Stack(ident) => {
+      let filter = match ObjectId::from_str(ident) {
+        Ok(id) => doc! { "_id": id },
+        Err(_) => doc! { "name": ident },
+      };
+      let id = db_client()
+        .stacks
+        .find_one(filter)
+        .await
+        .context("Failed to query db for stacks")?
+        .context("No matching stack found")?
+        .id;
+      Ok((ResourceTargetVariant::Stack, id))
+    }
+    ResourceTarget::Deployment(ident) => {
+      let filter = match ObjectId::from_str(ident) {
+        Ok(id) => doc! { "_id": id },
+        Err(_) => doc! { "name": ident },
+      };
+      let id = db_client()
+        .deployments
+        .find_one(filter)
+        .await
+        .context("Failed to query db for deployments")?
+        .context("No matching deployment found")?
+        .id;
+      Ok((ResourceTargetVariant::Deployment, id))
+    }
+    ResourceTarget::Build(ident) => {
+      let filter = match ObjectId::from_str(ident) {
+        Ok(id) => doc! { "_id": id },
+        Err(_) => doc! { "name": ident },
+      };
+      let id = db_client()
+        .builds
+        .find_one(filter)
+        .await
+        .context("Failed to query db for builds")?
+        .context("No matching build found")?
+        .id;
+      Ok((ResourceTargetVariant::Build, id))
+    }
     ResourceTarget::Repo(ident) => {
       let filter = match ObjectId::from_str(ident) {
         Ok(id) => doc! { "_id": id },
@@ -403,25 +417,11 @@ async fn extract_resource_target_with_validation(
         .repos
         .find_one(filter)
         .await
-        .context("failed to query db for repos")?
-        .context("no matching repo found")?
+        .context("Failed to query db for repos")?
+        .context("No matching repo found")?
         .id;
       Ok((ResourceTargetVariant::Repo, id))
     }
-    ResourceTarget::Alerter(ident) => {
-      let filter = match ObjectId::from_str(ident) {
-        Ok(id) => doc! { "_id": id },
-        Err(_) => doc! { "name": ident },
-      };
-      let id = db_client()
-        .alerters
-        .find_one(filter)
-        .await
-        .context("failed to query db for alerters")?
-        .context("no matching alerter found")?
-        .id;
-      Ok((ResourceTargetVariant::Alerter, id))
-    }
     ResourceTarget::Procedure(ident) => {
       let filter = match ObjectId::from_str(ident) {
         Ok(id) => doc! { "_id": id },
@@ -431,8 +431,8 @@ async fn extract_resource_target_with_validation(
         .procedures
         .find_one(filter)
         .await
-        .context("failed to query db for procedures")?
-        .context("no matching procedure found")?
+        .context("Failed to query db for procedures")?
+        .context("No matching procedure found")?
         .id;
       Ok((ResourceTargetVariant::Procedure, id))
     }
@@ -445,8 +445,8 @@ async fn extract_resource_target_with_validation(
         .actions
         .find_one(filter)
         .await
-        .context("failed to query db for actions")?
-        .context("no matching action found")?
+        .context("Failed to query db for actions")?
+        .context("No matching action found")?
         .id;
       Ok((ResourceTargetVariant::Action, id))
     }
@@ -459,24 +459,38 @@ async fn extract_resource_target_with_validation(
         .resource_syncs
         .find_one(filter)
         .await
-        .context("failed to query db for resource syncs")?
-        .context("no matching resource sync found")?
+        .context("Failed to query db for resource syncs")?
+        .context("No matching resource sync found")?
         .id;
       Ok((ResourceTargetVariant::ResourceSync, id))
     }
-    ResourceTarget::Stack(ident) => {
+    ResourceTarget::Builder(ident) => {
       let filter = match ObjectId::from_str(ident) {
         Ok(id) => doc! { "_id": id },
         Err(_) => doc! { "name": ident },
       };
       let id = db_client()
-        .stacks
+        .builders
         .find_one(filter)
         .await
-        .context("failed to query db for stacks")?
-        .context("no matching stack found")?
+        .context("Failed to query db for builders")?
+        .context("No matching builder found")?
         .id;
-      Ok((ResourceTargetVariant::Stack, id))
+      Ok((ResourceTargetVariant::Builder, id))
+    }
+    ResourceTarget::Alerter(ident) => {
+      let filter = match ObjectId::from_str(ident) {
+        Ok(id) => doc! { "_id": id },
+        Err(_) => doc! { "name": ident },
+      };
+      let id = db_client()
+        .alerters
+        .find_one(filter)
+        .await
+        .context("Failed to query db for alerters")?
+        .context("No matching alerter found")?
+        .id;
+      Ok((ResourceTargetVariant::Alerter, id))
     }
   }
 }

bin/core/src/api/write/resource.rs

@@ -5,7 +5,8 @@ use komodo_client::{
   entities::{
     ResourceTarget, action::Action, alerter::Alerter, build::Build,
     builder::Builder, deployment::Deployment, procedure::Procedure,
-    repo::Repo, server::Server, stack::Stack, sync::ResourceSync,
+    repo::Repo, server::Server, stack::Stack, swarm::Swarm,
+    sync::ResourceSync,
   },
 };
 use reqwest::StatusCode;
@@ -45,9 +46,15 @@ impl Resolve<WriteArgs> for UpdateResourceMeta {
             .status_code(StatusCode::BAD_REQUEST),
         );
       }
+      ResourceTarget::Swarm(id) => {
+        resource::update_meta::<Swarm>(&id, meta, args).await?;
+      }
       ResourceTarget::Server(id) => {
         resource::update_meta::<Server>(&id, meta, args).await?;
       }
+      ResourceTarget::Stack(id) => {
+        resource::update_meta::<Stack>(&id, meta, args).await?;
+      }
       ResourceTarget::Deployment(id) => {
         resource::update_meta::<Deployment>(&id, meta, args).await?;
       }
@@ -57,12 +64,6 @@ impl Resolve<WriteArgs> for UpdateResourceMeta {
       ResourceTarget::Repo(id) => {
         resource::update_meta::<Repo>(&id, meta, args).await?;
       }
-      ResourceTarget::Builder(id) => {
-        resource::update_meta::<Builder>(&id, meta, args).await?;
-      }
-      ResourceTarget::Alerter(id) => {
-        resource::update_meta::<Alerter>(&id, meta, args).await?;
-      }
       ResourceTarget::Procedure(id) => {
         resource::update_meta::<Procedure>(&id, meta, args).await?;
       }
@@ -73,8 +74,11 @@ impl Resolve<WriteArgs> for UpdateResourceMeta {
         resource::update_meta::<ResourceSync>(&id, meta, args)
           .await?;
       }
-      ResourceTarget::Stack(id) => {
-        resource::update_meta::<Stack>(&id, meta, args).await?;
+      ResourceTarget::Builder(id) => {
+        resource::update_meta::<Builder>(&id, meta, args).await?;
+      }
+      ResourceTarget::Alerter(id) => {
+        resource::update_meta::<Alerter>(&id, meta, args).await?;
       }
     }
     Ok(UpdateResourceMetaResponse {})

bin/core/src/api/write/server.rs

@@ -3,7 +3,7 @@ use formatting::{bold, format_serror};
 use komodo_client::{
   api::write::*,
   entities::{
-    NoData, Operation,
+    Operation,
     permission::PermissionLevel,
     server::{Server, ServerInfo},
     to_docker_compatible_name,
@@ -187,109 +187,6 @@ impl Resolve<WriteArgs> for CreateNetwork {
   }
 }
 
-impl Resolve<WriteArgs> for CreateTerminal {
-  #[instrument(
-    "CreateTerminal",
-    skip_all,
-    fields(
-      operator = user.id,
-      server = self.server,
-      terminal = self.name,
-      command = self.command,
-      recreate = format!("{:?}", self.recreate),
-    )
-  )]
-  async fn resolve(
-    self,
-    WriteArgs { user }: &WriteArgs,
-  ) -> serror::Result<NoData> {
-    let server = get_check_permissions::<Server>(
-      &self.server,
-      user,
-      PermissionLevel::Write.terminal(),
-    )
-    .await?;
-
-    let periphery = periphery_client(&server).await?;
-
-    periphery
-      .request(api::terminal::CreateTerminal {
-        name: self.name,
-        command: self.command,
-        recreate: self.recreate,
-      })
-      .await
-      .context("Failed to create terminal on Periphery")?;
-
-    Ok(NoData {})
-  }
-}
-
-impl Resolve<WriteArgs> for DeleteTerminal {
-  #[instrument(
-    "DeleteTerminal",
-    skip_all,
-    fields(
-      operator = user.id,
-      server = self.server,
-      terminal = self.terminal,
-    )
-  )]
-  async fn resolve(
-    self,
-    WriteArgs { user }: &WriteArgs,
-  ) -> serror::Result<NoData> {
-    let server = get_check_permissions::<Server>(
-      &self.server,
-      user,
-      PermissionLevel::Write.terminal(),
-    )
-    .await?;
-
-    let periphery = periphery_client(&server).await?;
-
-    periphery
-      .request(api::terminal::DeleteTerminal {
-        terminal: self.terminal,
-      })
-      .await
-      .context("Failed to delete terminal on Periphery")?;
-
-    Ok(NoData {})
-  }
-}
-
-impl Resolve<WriteArgs> for DeleteAllTerminals {
-  #[instrument(
-    "DeleteAllTerminals",
-    skip_all,
-    fields(
-      operator = user.id,
-      server = self.server,
-    )
-  )]
-  async fn resolve(
-    self,
-    WriteArgs { user }: &WriteArgs,
-  ) -> serror::Result<NoData> {
-    let server = get_check_permissions::<Server>(
-      &self.server,
-      user,
-      PermissionLevel::Write.terminal(),
-    )
-    .await?;
-
-    let periphery = periphery_client(&server).await?;
-
-    periphery
-      .request(api::terminal::DeleteAllTerminals {})
-      .await
-      .context("Failed to delete all terminals on Periphery")?;
-
-    Ok(NoData {})
-  }
-}
-
 //
 
 impl Resolve<WriteArgs> for UpdateServerPublicKey {

bin/core/src/api/write/service_user.rs

@@ -1,10 +1,5 @@
-use std::str::FromStr;
-
 use anyhow::{Context, anyhow};
-use database::mungos::{
-  by_id::find_one_by_id,
-  mongodb::bson::{doc, oid::ObjectId},
-};
+use database::mungos::{by_id::find_one_by_id, mongodb::bson::doc};
 use komodo_client::{
   api::{user::CreateApiKey, write::*},
   entities::{
@@ -12,9 +7,15 @@ use komodo_client::{
     user::{User, UserConfig},
   },
 };
+use reqwest::StatusCode;
 use resolver_api::Resolve;
+use serror::{AddStatusCode as _, AddStatusCodeError as _};
 
-use crate::{api::user::UserArgs, state::db_client};
+use crate::{
+  api::user::UserArgs,
+  helpers::validations::{validate_api_key_name, validate_username},
+  state::db_client,
+};
 
 use super::WriteArgs;
 
@@ -33,30 +34,38 @@ impl Resolve<WriteArgs> for CreateServiceUser {
     WriteArgs { user }: &WriteArgs,
   ) -> serror::Result<CreateServiceUserResponse> {
     if !user.admin {
-      return Err(anyhow!("user not admin").into());
-    }
-    if ObjectId::from_str(&self.username).is_ok() {
       return Err(
-        anyhow!("username cannot be valid ObjectId").into(),
+        anyhow!("Only Admins can manage Service Users")
+          .status_code(StatusCode::FORBIDDEN),
       );
     }
+
+    validate_username(&self.username)
+      .status_code(StatusCode::BAD_REQUEST)?;
+
     let config = UserConfig::Service {
       description: self.description,
     };
+
     let mut user = User {
       id: Default::default(),
       username: self.username,
       config,
+      totp: Default::default(),
+      passkey: Default::default(),
+      third_party_skip_2fa: Default::default(),
       enabled: true,
       admin: false,
       super_admin: false,
       create_server_permissions: false,
       create_build_permissions: false,
       last_update_view: 0,
+      linked_logins: Default::default(),
       recents: Default::default(),
       all: Default::default(),
       updated_at: komodo_timestamp(),
     };
+
     user.id = db_client()
       .users
       .insert_one(&user)
@@ -66,6 +75,7 @@ impl Resolve<WriteArgs> for CreateServiceUser {
       .as_object_id()
       .context("inserted id is not object id")?
       .to_string();
+
     Ok(user)
   }
 }
@@ -85,18 +95,28 @@ impl Resolve<WriteArgs> for UpdateServiceUserDescription {
     WriteArgs { user }: &WriteArgs,
   ) -> serror::Result<UpdateServiceUserDescriptionResponse> {
     if !user.admin {
-      return Err(anyhow!("user not admin").into());
+      return Err(
+        anyhow!("Only Admins can manage Service Users")
+          .status_code(StatusCode::FORBIDDEN),
+      );
     }
+
     let db = db_client();
+
     let service_user = db
       .users
       .find_one(doc! { "username": &self.username })
       .await
-      .context("failed to query db for user")?
-      .context("no user with given username")?;
+      .context("Failed to query db for user")?
+      .context("No user with given username")?;
+
     let UserConfig::Service { .. } = &service_user.config else {
-      return Err(anyhow!("user is not service user").into());
+      return Err(
+        anyhow!("Target user is not Service User")
+          .status_code(StatusCode::FORBIDDEN),
+      );
     };
+
     db.users
       .update_one(
         doc! { "username": &self.username },
@@ -104,13 +124,15 @@ impl Resolve<WriteArgs> for UpdateServiceUserDescription {
       )
       .await
       .context("failed to update user on db")?;
-    let res = db
+
+    let service_user = db
       .users
       .find_one(doc! { "username": &self.username })
       .await
       .context("failed to query db for user")?
       .context("user with username not found")?;
-    Ok(res)
+
+    Ok(service_user)
   }
 }
 
@@ -130,21 +152,36 @@ impl Resolve<WriteArgs> for CreateApiKeyForServiceUser {
     WriteArgs { user }: &WriteArgs,
   ) -> serror::Result<CreateApiKeyForServiceUserResponse> {
     if !user.admin {
-      return Err(anyhow!("user not admin").into());
+      return Err(
+        anyhow!("Only Admins can manage Service Users")
+          .status_code(StatusCode::FORBIDDEN),
+      );
     }
+
+    validate_api_key_name(&self.name)
+      .status_code(StatusCode::BAD_REQUEST)?;
+
     let service_user =
       find_one_by_id(&db_client().users, &self.user_id)
         .await
-        .context("failed to query db for user")?
-        .context("no user found with id")?;
+        .context("Failed to query db for user")?
+        .context("No user found with id")?;
+
     let UserConfig::Service { .. } = &service_user.config else {
-      return Err(anyhow!("user is not service user").into());
+      return Err(
+        anyhow!("Target user is not Service User")
+          .status_code(StatusCode::FORBIDDEN),
+      );
     };
+
     CreateApiKey {
       name: self.name,
       expires: self.expires,
     }
-    .resolve(&UserArgs { user: service_user })
+    .resolve(&UserArgs {
+      user: service_user,
+      session: None,
+    })
     .await
   }
 }
@@ -163,23 +200,34 @@ impl Resolve<WriteArgs> for DeleteApiKeyForServiceUser {
     WriteArgs { user }: &WriteArgs,
   ) -> serror::Result<DeleteApiKeyForServiceUserResponse> {
     if !user.admin {
-      return Err(anyhow!("user not admin").into());
+      return Err(
+        anyhow!("Only Admins can manage Service Users")
+          .status_code(StatusCode::FORBIDDEN),
+      );
     }
+
     let db = db_client();
+
     let api_key = db
       .api_keys
       .find_one(doc! { "key": &self.key })
       .await
       .context("failed to query db for api key")?
       .context("did not find matching api key")?;
+
     let service_user =
       find_one_by_id(&db_client().users, &api_key.user_id)
         .await
         .context("failed to query db for user")?
         .context("no user found with id")?;
+
     let UserConfig::Service { .. } = &service_user.config else {
-      return Err(anyhow!("user is not service user").into());
+      return Err(
+        anyhow!("Target user is not Service User")
+          .status_code(StatusCode::FORBIDDEN),
+      );
     };
+
     db.api_keys
       .delete_one(doc! { "key": self.key })
       .await

bin/core/src/api/write/stack.rs

@@ -10,7 +10,6 @@ use komodo_client::{
     all_logs_success,
     permission::PermissionLevel,
     repo::Repo,
-    server::ServerState,
     stack::{Stack, StackInfo},
     update::Update,
     user::stack_user,
@@ -25,9 +24,8 @@ use resolver_api::Resolve;
 use crate::{
   config::core_config,
   helpers::{
-    periphery_client,
-    query::get_server_with_state,
-    stack_git_token,
+    query::get_swarm_or_server,
+    stack_git_token, swarm_or_server_request,
     update::{add_update, make_update},
   },
   permission::get_check_permissions,
@@ -204,32 +202,24 @@ async fn write_stack_file_contents_on_host(
   contents: String,
   mut update: Update,
 ) -> serror::Result<Update> {
-  if stack.config.server_id.is_empty() {
-    return Err(anyhow!(
-      "Cannot write file, Files on host Stack has not configured a Server"
-    ).into());
-  }
-  let (server, state) =
-    get_server_with_state(&stack.config.server_id).await?;
-  if state != ServerState::Ok {
-    return Err(
-      anyhow!(
-        "Cannot write file when server is unreachable or disabled"
-      )
-      .into(),
-    );
-  }
-  match periphery_client(&server)
-    .await?
-    .request(WriteComposeContentsToHost {
+  let swarm_or_server = get_swarm_or_server(
+    &stack.config.swarm_id,
+    &stack.config.server_id,
+  )
+  .await?;
+
+  let res = swarm_or_server_request(
+    &swarm_or_server,
+    WriteComposeContentsToHost {
       name: stack.name,
       run_directory: stack.config.run_directory,
       file_path,
       contents,
-    })
-    .await
-    .context("Failed to write contents to host")
-  {
+    },
+  )
+  .await;
+
+  match res {
     Ok(log) => {
       update.logs.push(log);
     }
@@ -239,7 +229,7 @@ async fn write_stack_file_contents_on_host(
         format_serror(&e.into()),
       );
     }
-  };
+  }
 
   if !all_logs_success(&update.logs) {
     update.finalize();
@@ -459,26 +449,22 @@ impl Resolve<WriteArgs> for RefreshStackCache {
       // =============
       // FILES ON HOST
       // =============
-      let (server, state) = if stack.config.server_id.is_empty() {
-        (None, ServerState::Disabled)
-      } else {
-        let (server, state) =
-          get_server_with_state(&stack.config.server_id).await?;
-        (Some(server), state)
-      };
-      if state != ServerState::Ok {
-        (vec![], None, None, None, None)
-      } else if let Some(server) = server {
+      if let Ok(swarm_or_server) = get_swarm_or_server(
+        &stack.config.swarm_id,
+        &stack.config.server_id,
+      )
+      .await
+      {
         let GetComposeContentsOnHostResponse { contents, errors } =
-          match periphery_client(&server)
-            .await?
-            .request(GetComposeContentsOnHost {
+          match swarm_or_server_request(
+            &swarm_or_server,
+            GetComposeContentsOnHost {
               file_paths: stack.all_file_dependencies(),
               name: stack.name.clone(),
               run_directory: stack.config.run_directory.clone(),
-            })
-            .await
-            .context("failed to get compose file contents from host")
+            },
+          )
+          .await
           {
             Ok(res) => res,
             Err(e) => GetComposeContentsOnHostResponse {
@@ -489,7 +475,6 @@ impl Resolve<WriteArgs> for RefreshStackCache {
               }],
             },
           };
-
         let project_name = stack.project_name(true);
 
         let mut services = Vec::new();

bin/core/src/api/write/swarm.rs

@@ -0,0 +1,108 @@
+use komodo_client::{
+  api::write::*,
+  entities::{
+    permission::PermissionLevel, swarm::Swarm, update::Update,
+  },
+};
+use resolver_api::Resolve;
+
+use crate::{permission::get_check_permissions, resource};
+
+use super::WriteArgs;
+
+impl Resolve<WriteArgs> for CreateSwarm {
+  #[instrument(
+    "CreateSwarm",
+    skip_all,
+    fields(
+      operator = user.id,
+      swarm = self.name,
+      config = serde_json::to_string(&self.config).unwrap(),
+    )
+  )]
+  async fn resolve(
+    self,
+    WriteArgs { user }: &WriteArgs,
+  ) -> serror::Result<Swarm> {
+    resource::create::<Swarm>(&self.name, self.config, None, user)
+      .await
+  }
+}
+
+impl Resolve<WriteArgs> for CopySwarm {
+  #[instrument(
+    "CopySwarm",
+    skip_all,
+    fields(
+      operator = user.id,
+      swarm = self.name,
+      copy_swarm = self.id,
+    )
+  )]
+  async fn resolve(
+    self,
+    WriteArgs { user }: &WriteArgs,
+  ) -> serror::Result<Swarm> {
+    let Swarm { config, .. } = get_check_permissions::<Swarm>(
+      &self.id,
+      user,
+      PermissionLevel::Read.into(),
+    )
+    .await?;
+    resource::create::<Swarm>(&self.name, config.into(), None, user)
+      .await
+  }
+}
+
+impl Resolve<WriteArgs> for DeleteSwarm {
+  #[instrument(
+    "DeleteSwarm",
+    skip_all,
+    fields(
+      operator = user.id,
+      swarm = self.id,
+    )
+  )]
+  async fn resolve(
+    self,
+    WriteArgs { user }: &WriteArgs,
+  ) -> serror::Result<Swarm> {
+    Ok(resource::delete::<Swarm>(&self.id, user).await?)
+  }
+}
+
+impl Resolve<WriteArgs> for UpdateSwarm {
+  #[instrument(
+    "UpdateSwarm",
+    skip_all,
+    fields(
+      operator = user.id,
+      swarm = self.id,
+      update = serde_json::to_string(&self.config).unwrap()
+    )
+  )]
+  async fn resolve(
+    self,
+    WriteArgs { user }: &WriteArgs,
+  ) -> serror::Result<Swarm> {
+    Ok(resource::update::<Swarm>(&self.id, self.config, user).await?)
+  }
+}
+
+impl Resolve<WriteArgs> for RenameSwarm {
+  #[instrument(
+    "RenameSwarm",
+    skip_all,
+    fields(
+      operator = user.id,
+      swarm = self.id,
+      new_name = self.name
+    )
+  )]
+  async fn resolve(
+    self,
+    WriteArgs { user }: &WriteArgs,
+  ) -> serror::Result<Update> {
+    Ok(resource::rename::<Swarm>(&self.id, &self.name, user).await?)
+  }
+}

bin/core/src/api/write/terminal.rs

@@ -0,0 +1,309 @@
+use anyhow::Context as _;
+use futures_util::{StreamExt as _, stream::FuturesUnordered};
+use komodo_client::{
+  api::write::*,
+  entities::{
+    NoData, deployment::Deployment, permission::PermissionLevel,
+    server::Server, stack::Stack, terminal::TerminalTarget,
+    user::User,
+  },
+};
+use periphery_client::api;
+use reqwest::StatusCode;
+use resolver_api::Resolve;
+use serror::AddStatusCode;
+
+use crate::{
+  helpers::{
+    periphery_client,
+    query::get_all_tags,
+    terminal::{
+      create_container_terminal_inner,
+      get_deployment_periphery_container,
+      get_stack_service_periphery_container,
+    },
+  },
+  permission::get_check_permissions,
+  resource,
+};
+
+use super::WriteArgs;
+
+//
+
+impl Resolve<WriteArgs> for CreateTerminal {
+  #[instrument(
+    "CreateTerminal",
+    skip_all,
+    fields(
+      operator = user.id,
+      terminal = self.name,
+      target = format!("{:?}", self.target),
+      command = self.command,
+      mode = format!("{:?}", self.mode),
+      recreate = format!("{:?}", self.recreate),
+    )
+  )]
+  async fn resolve(
+    self,
+    WriteArgs { user }: &WriteArgs,
+  ) -> serror::Result<NoData> {
+    match self.target.clone() {
+      TerminalTarget::Server { server } => {
+        let server = server
+          .context("Must provide 'target.params.server'")
+          .status_code(StatusCode::BAD_REQUEST)?;
+        create_server_terminal(self, server, user).await?;
+      }
+      TerminalTarget::Container { server, container } => {
+        create_container_terminal(self, server, container, user)
+          .await?;
+      }
+      TerminalTarget::Stack { stack, service } => {
+        let service = service
+          .context("Must provide 'target.params.service'")
+          .status_code(StatusCode::BAD_REQUEST)?;
+        create_stack_service_terminal(self, stack, service, user)
+          .await?;
+      }
+      TerminalTarget::Deployment { deployment } => {
+        create_deployment_terminal(self, deployment, user).await?;
+      }
+    };
+    Ok(NoData {})
+  }
+}
+
+async fn create_server_terminal(
+  CreateTerminal {
+    name,
+    command,
+    recreate,
+    target: _,
+    mode: _,
+  }: CreateTerminal,
+  server: String,
+  user: &User,
+) -> anyhow::Result<()> {
+  let server = get_check_permissions::<Server>(
+    &server,
+    user,
+    PermissionLevel::Read.terminal(),
+  )
+  .await?;
+
+  let periphery = periphery_client(&server).await?;
+
+  periphery
+    .request(api::terminal::CreateServerTerminal {
+      name,
+      command,
+      recreate,
+    })
+    .await
+    .context("Failed to create Server Terminal on Periphery")?;
+
+  Ok(())
+}
+
+async fn create_container_terminal(
+  req: CreateTerminal,
+  server: String,
+  container: String,
+  user: &User,
+) -> anyhow::Result<()> {
+  let server = get_check_permissions::<Server>(
+    &server,
+    user,
+    PermissionLevel::Read.terminal(),
+  )
+  .await?;
+  let periphery = periphery_client(&server).await?;
+  create_container_terminal_inner(req, &periphery, container).await
+}
+
+async fn create_stack_service_terminal(
+  req: CreateTerminal,
+  stack: String,
+  service: String,
+  user: &User,
+) -> anyhow::Result<()> {
+  let (_, periphery, container) =
+    get_stack_service_periphery_container(&stack, &service, user)
+      .await?;
+  create_container_terminal_inner(req, &periphery, container).await
+}
+
+async fn create_deployment_terminal(
+  req: CreateTerminal,
+  deployment: String,
+  user: &User,
+) -> anyhow::Result<()> {
+  let (_, periphery, container) =
+    get_deployment_periphery_container(&deployment, user).await?;
+  create_container_terminal_inner(req, &periphery, container).await
+}
+
+//
+
+impl Resolve<WriteArgs> for DeleteTerminal {
+  #[instrument(
+    "DeleteTerminal",
+    skip_all,
+    fields(
+      operator = user.id,
+      target = format!("{:?}", self.target),
+      terminal = self.terminal,
+    )
+  )]
+  async fn resolve(
+    self,
+    WriteArgs { user }: &WriteArgs,
+  ) -> serror::Result<NoData> {
+    let server = match &self.target {
+      TerminalTarget::Server { server } => {
+        let server = server
+          .as_ref()
+          .context("Must provide 'target.params.server'")
+          .status_code(StatusCode::BAD_REQUEST)?;
+        get_check_permissions::<Server>(
+          server,
+          user,
+          PermissionLevel::Read.terminal(),
+        )
+        .await?
+      }
+      TerminalTarget::Container { server, .. } => {
+        get_check_permissions::<Server>(
+          server,
+          user,
+          PermissionLevel::Read.terminal(),
+        )
+        .await?
+      }
+      TerminalTarget::Stack { stack, .. } => {
+        let server = get_check_permissions::<Stack>(
+          stack,
+          user,
+          PermissionLevel::Read.terminal(),
+        )
+        .await?
+        .config
+        .server_id;
+        resource::get::<Server>(&server).await?
+      }
+      TerminalTarget::Deployment { deployment } => {
+        let server = get_check_permissions::<Deployment>(
+          deployment,
+          user,
+          PermissionLevel::Read.terminal(),
+        )
+        .await?
+        .config
+        .server_id;
+        resource::get::<Server>(&server).await?
+      }
+    };
+
+    let periphery = periphery_client(&server).await?;
+
+    periphery
+      .request(api::terminal::DeleteTerminal {
+        target: self.target,
+        terminal: self.terminal,
+      })
+      .await
+      .context("Failed to delete terminal on Periphery")?;
+
+    Ok(NoData {})
+  }
+}
+
+//
+
+impl Resolve<WriteArgs> for DeleteAllTerminals {
+  #[instrument(
+    "DeleteAllTerminals",
+    skip_all,
+    fields(
+      operator = user.id,
+      server = self.server,
+    )
+  )]
+  async fn resolve(
+    self,
+    WriteArgs { user }: &WriteArgs,
+  ) -> serror::Result<NoData> {
+    let server = get_check_permissions::<Server>(
+      &self.server,
+      user,
+      PermissionLevel::Read.terminal(),
+    )
+    .await?;
+
+    let periphery = periphery_client(&server).await?;
+
+    periphery
+      .request(api::terminal::DeleteAllTerminals {})
+      .await
+      .context("Failed to delete all terminals on Periphery")?;
+
+    Ok(NoData {})
+  }
+}
+
+//
+
+impl Resolve<WriteArgs> for BatchDeleteAllTerminals {
+  #[instrument(
+    "BatchDeleteAllTerminals",
+    skip_all,
+    fields(
+      operator = user.id,
+      query = format!("{:?}", self.query),
+    )
+  )]
+  async fn resolve(
+    self,
+    WriteArgs { user }: &WriteArgs,
+  ) -> Result<Self::Response, Self::Error> {
+    let all_tags = if self.query.tags.is_empty() {
+      vec![]
+    } else {
+      get_all_tags(None).await?
+    };
+
+    resource::list_full_for_user::<Server>(
+      self.query,
+      user,
+      PermissionLevel::Read.terminal(),
+      &all_tags,
+    )
+    .await?
+    .into_iter()
+    .map(|server| async move {
+      let res = async {
+        let periphery = periphery_client(&server).await?;
+
+        periphery
+          .request(api::terminal::DeleteAllTerminals {})
+          .await
+          .context("Failed to delete all terminals on Periphery")?;
+
+        anyhow::Ok(())
+      }
+      .await;
+      if let Err(e) = res {
+        warn!(
+          "Failed to delete all terminals on {} ({}) | {e:#}",
+          server.name, server.id
+        )
+      }
+    })
+    .collect::<FuturesUnordered<_>>()
+    .collect::<Vec<_>>()
+    .await;
+
+    Ok(NoData {})
+  }
+}

bin/core/src/api/write/user.rs

@@ -8,16 +8,16 @@ use database::{
 };
 use komodo_client::{
   api::write::*,
-  entities::{
-    NoData,
-    user::{User, UserConfig},
-  },
+  entities::user::{User, UserConfig},
 };
 use reqwest::StatusCode;
 use resolver_api::Resolve;
-use serror::AddStatusCodeError;
+use serror::{AddStatusCode as _, AddStatusCodeError};
 
-use crate::{config::core_config, state::db_client};
+use crate::{
+  helpers::validations::{validate_password, validate_username},
+  state::db_client,
+};
 
 use super::WriteArgs;
 
@@ -38,24 +38,15 @@ impl Resolve<WriteArgs> for CreateLocalUser {
   ) -> serror::Result<CreateLocalUserResponse> {
     if !admin.admin {
       return Err(
-        anyhow!("This method is admin-only.")
+        anyhow!("This method is Admin Only.")
           .status_code(StatusCode::FORBIDDEN),
       );
     }
 
-    if self.username.is_empty() {
-      return Err(anyhow!("Username cannot be empty.").into());
-    }
-
-    if ObjectId::from_str(&self.username).is_ok() {
-      return Err(
-        anyhow!("Username cannot be valid ObjectId").into(),
-      );
-    }
-
-    if self.password.is_empty() {
-      return Err(anyhow!("Password cannot be empty.").into());
-    }
+    validate_username(&self.username)
+      .status_code(StatusCode::BAD_REQUEST)?;
+    validate_password(&self.password)
+      .status_code(StatusCode::BAD_REQUEST)?;
 
     let db = db_client();
 
@@ -87,6 +78,10 @@ impl Resolve<WriteArgs> for CreateLocalUser {
       config: UserConfig::Local {
         password: hashed_password,
       },
+      linked_logins: Default::default(),
+      totp: Default::default(),
+      passkey: Default::default(),
+      third_party_skip_2fa: Default::default(),
     };
 
     user.id = db_client()
@@ -107,91 +102,6 @@ impl Resolve<WriteArgs> for CreateLocalUser {
 
 //
 
-impl Resolve<WriteArgs> for UpdateUserUsername {
-  #[instrument(
-    "UpdateUserUsername",
-    skip_all,
-    fields(
-      operator = user.id,
-      new_username = self.username,
-    )
-  )]
-  async fn resolve(
-    self,
-    WriteArgs { user }: &WriteArgs,
-  ) -> serror::Result<UpdateUserUsernameResponse> {
-    for locked_username in &core_config().lock_login_credentials_for {
-      if locked_username == "__ALL__"
-        || *locked_username == user.username
-      {
-        return Err(
-          anyhow!("User not allowed to update their username.")
-            .into(),
-        );
-      }
-    }
-    if self.username.is_empty() {
-      return Err(anyhow!("Username cannot be empty.").into());
-    }
-
-    if ObjectId::from_str(&self.username).is_ok() {
-      return Err(
-        anyhow!("Username cannot be valid ObjectId").into(),
-      );
-    }
-
-    let db = db_client();
-    if db
-      .users
-      .find_one(doc! { "username": &self.username })
-      .await
-      .context("Failed to query for existing users")?
-      .is_some()
-    {
-      return Err(anyhow!("Username already taken.").into());
-    }
-    let id = ObjectId::from_str(&user.id)
-      .context("User id not valid ObjectId.")?;
-    db.users
-      .update_one(
-        doc! { "_id": id },
-        doc! { "$set": { "username": self.username } },
-      )
-      .await
-      .context("Failed to update user username on database.")?;
-    Ok(NoData {})
-  }
-}
-
-//
-
-impl Resolve<WriteArgs> for UpdateUserPassword {
-  #[instrument(
-    "UpdateUserPassword",
-    skip_all,
-    fields(operator = user.id)
-  )]
-  async fn resolve(
-    self,
-    WriteArgs { user }: &WriteArgs,
-  ) -> serror::Result<UpdateUserPasswordResponse> {
-    for locked_username in &core_config().lock_login_credentials_for {
-      if locked_username == "__ALL__"
-        || *locked_username == user.username
-      {
-        return Err(
-          anyhow!("User not allowed to update their password.")
-            .into(),
-        );
-      }
-    }
-    db_client().set_user_password(user, &self.password).await?;
-    Ok(NoData {})
-  }
-}
-
-//
-
 impl Resolve<WriteArgs> for DeleteUser {
   #[instrument(
     "DeleteUser",
@@ -211,15 +121,19 @@ impl Resolve<WriteArgs> for DeleteUser {
           .status_code(StatusCode::FORBIDDEN),
       );
     }
+
     if admin.username == self.user || admin.id == self.user {
       return Err(anyhow!("User cannot delete themselves.").into());
     }
+
     let query = if let Ok(id) = ObjectId::from_str(&self.user) {
       doc! { "_id": id }
     } else {
       doc! { "username": self.user }
     };
+
     let db = db_client();
+
     let Some(user) = db
       .users
       .find_one(query.clone())
@@ -230,21 +144,25 @@ impl Resolve<WriteArgs> for DeleteUser {
         anyhow!("No user found with given id / username").into(),
       );
     };
+
     if user.super_admin {
       return Err(
         anyhow!("Cannot delete a super admin user.").into(),
       );
     }
+
     if user.admin && !admin.super_admin {
       return Err(
         anyhow!("Only a Super Admin can delete an admin user.")
           .into(),
       );
     }
+
     db.users
       .delete_one(query)
       .await
       .context("Failed to delete user from database")?;
+
     // Also remove user id from all user groups
     if let Err(e) = db
       .user_groups
@@ -253,6 +171,7 @@ impl Resolve<WriteArgs> for DeleteUser {
     {
       warn!("Failed to remove deleted user from user groups | {e:?}");
     };
+
     Ok(user)
   }
 }

bin/core/src/api/write/variable.rs

@@ -6,12 +6,13 @@ use komodo_client::{
 };
 use reqwest::StatusCode;
 use resolver_api::Resolve;
-use serror::AddStatusCodeError;
+use serror::{AddStatusCode as _, AddStatusCodeError};
 
 use crate::{
   helpers::{
     query::get_variable,
     update::{add_update, make_update},
+    validations::{validate_variable_name, validate_variable_value},
   },
   state::db_client,
 };
@@ -35,7 +36,7 @@ impl Resolve<WriteArgs> for CreateVariable {
   ) -> serror::Result<CreateVariableResponse> {
     if !user.admin {
       return Err(
-        anyhow!("Only admins can create variables")
+        anyhow!("Only Admins can create Variables")
           .status_code(StatusCode::FORBIDDEN),
       );
     }
@@ -47,6 +48,11 @@ impl Resolve<WriteArgs> for CreateVariable {
       is_secret,
     } = self;
 
+    validate_variable_name(&name)
+      .status_code(StatusCode::BAD_REQUEST)?;
+    validate_variable_value(&value)
+      .status_code(StatusCode::BAD_REQUEST)?;
+
     let variable = Variable {
       name,
       value,
@@ -58,7 +64,7 @@ impl Resolve<WriteArgs> for CreateVariable {
       .variables
       .insert_one(&variable)
       .await
-      .context("Failed to create variable on db")?;
+      .context("Failed to create Variable on db")?;
 
     let mut update = make_update(
       ResourceTarget::system(),
@@ -67,7 +73,8 @@ impl Resolve<WriteArgs> for CreateVariable {
     );
 
     update
-      .push_simple_log("create variable", format!("{variable:#?}"));
+      .push_simple_log("Create Variable", format!("{variable:#?}"));
+
     update.finalize();
 
     add_update(update).await?;
@@ -91,13 +98,18 @@ impl Resolve<WriteArgs> for UpdateVariableValue {
   ) -> serror::Result<UpdateVariableValueResponse> {
     if !user.admin {
       return Err(
-        anyhow!("Only admins can update variables")
+        anyhow!("Only Admins can update Variables")
           .status_code(StatusCode::FORBIDDEN),
       );
     }
 
     let UpdateVariableValue { name, value } = self;
 
+    validate_variable_name(&name)
+      .status_code(StatusCode::BAD_REQUEST)?;
+    validate_variable_value(&value)
+      .status_code(StatusCode::BAD_REQUEST)?;
+
     let variable = get_variable(&name).await?;
 
     if value == variable.value {
@@ -156,10 +168,11 @@ impl Resolve<WriteArgs> for UpdateVariableDescription {
   ) -> serror::Result<UpdateVariableDescriptionResponse> {
     if !user.admin {
       return Err(
-        anyhow!("Only admins can update variables")
+        anyhow!("Only Admins can update Variables")
           .status_code(StatusCode::FORBIDDEN),
       );
     }
+
     db_client()
       .variables
       .update_one(
@@ -168,6 +181,7 @@ impl Resolve<WriteArgs> for UpdateVariableDescription {
       )
       .await
       .context("Failed to update variable description on db")?;
+
     Ok(get_variable(&self.name).await?)
   }
 }
@@ -188,10 +202,11 @@ impl Resolve<WriteArgs> for UpdateVariableIsSecret {
   ) -> serror::Result<UpdateVariableIsSecretResponse> {
     if !user.admin {
       return Err(
-        anyhow!("Only admins can update variables")
+        anyhow!("Only Admins can update Variables")
           .status_code(StatusCode::FORBIDDEN),
       );
     }
+
     db_client()
       .variables
       .update_one(
@@ -199,7 +214,8 @@ impl Resolve<WriteArgs> for UpdateVariableIsSecret {
         doc! { "$set": { "is_secret": self.is_secret } },
       )
       .await
-      .context("Failed to update variable is secret on db")?;
+      .context("Failed to update Variable 'is_secret' on db")?;
+
     Ok(get_variable(&self.name).await?)
   }
 }
@@ -219,16 +235,18 @@ impl Resolve<WriteArgs> for DeleteVariable {
   ) -> serror::Result<DeleteVariableResponse> {
     if !user.admin {
       return Err(
-        anyhow!("Only admins can delete variables")
+        anyhow!("Only Admins can delete Variables")
           .status_code(StatusCode::FORBIDDEN),
       );
     }
+
     let variable = get_variable(&self.name).await?;
+
     db_client()
       .variables
       .delete_one(doc! { "name": &self.name })
       .await
-      .context("Failed to delete variable on db")?;
+      .context("Failed to delete Variable on db")?;
 
     let mut update = make_update(
       ResourceTarget::system(),

bin/core/src/api/ws/mod.rs

@@ -0,0 +1,101 @@
+use std::net::IpAddr;
+
+use crate::{
+  auth::{auth_api_key_check_enabled, auth_jwt_check_enabled},
+  helpers::query::get_user,
+  state::auth_rate_limiter,
+};
+use anyhow::{Context, anyhow};
+use axum::{
+  Router,
+  extract::ws::{self, WebSocket},
+  http::HeaderMap,
+  routing::get,
+};
+use komodo_client::{entities::user::User, ws::WsLoginMessage};
+use rate_limit::WithFailureRateLimit;
+use reqwest::StatusCode;
+use serror::{AddStatusCode, AddStatusCodeError};
+
+mod terminal;
+mod update;
+
+pub fn router() -> Router {
+  Router::new()
+    // Periphery facing
+    .route("/periphery", get(crate::connection::server::handler))
+    // User facing
+    .route("/update", get(update::handler))
+    .route("/terminal", get(terminal::handler))
+}
+
+async fn user_ws_login(
+  mut socket: WebSocket,
+  headers: &HeaderMap,
+  fallback_ip: IpAddr,
+) -> Option<(WebSocket, User)> {
+  let res = async {
+    let message = match socket
+      .recv()
+      .await
+      .context("Failed to receive message over socket: Closed")
+      .status_code(StatusCode::BAD_REQUEST)?
+      .context("Failed to recieve message over socket: Error")
+      .status_code(StatusCode::BAD_REQUEST)?
+    {
+      ws::Message::Text(utf8_bytes) => utf8_bytes.to_string(),
+      ws::Message::Binary(bytes) => String::from_utf8(bytes.into())
+        .context("Received invalid message bytes: Not UTF-8")
+        .status_code(StatusCode::BAD_REQUEST)?,
+      message => {
+        return Err(
+          anyhow!("Received invalid message: {message:?}")
+            .status_code(StatusCode::BAD_REQUEST),
+        );
+      }
+    };
+
+    match WsLoginMessage::from_json_str(&message)
+      .context("Invalid login message")
+      .status_code(StatusCode::BAD_REQUEST)?
+    {
+      WsLoginMessage::Jwt { jwt } => auth_jwt_check_enabled(&jwt)
+        .await
+        .status_code(StatusCode::UNAUTHORIZED),
+      WsLoginMessage::ApiKeys { key, secret } => {
+        auth_api_key_check_enabled(&key, &secret)
+          .await
+          .status_code(StatusCode::UNAUTHORIZED)
+      }
+    }
+  }
+  .with_failure_rate_limit_using_headers(
+    auth_rate_limiter(),
+    headers,
+    Some(fallback_ip),
+  )
+  .await;
+  match res {
+    Ok(user) => {
+      let _ = socket.send(ws::Message::text("LOGGED_IN")).await;
+      Some((socket, user))
+    }
+    Err(e) => {
+      let _ = socket
+        .send(ws::Message::text(format!(
+          "[{}]: {:#}",
+          e.status, e.error
+        )))
+        .await;
+      None
+    }
+  }
+}
+
+async fn check_user_valid(user_id: &str) -> anyhow::Result<User> {
+  let user = get_user(user_id).await?;
+  if !user.enabled {
+    return Err(anyhow!("User not enabled"));
+  }
+  Ok(user)
+}

bin/core/src/api/ws/terminal.rs

@@ -0,0 +1,188 @@
+use std::net::SocketAddr;
+
+use anyhow::anyhow;
+use axum::{
+  extract::{ConnectInfo, FromRequestParts, WebSocketUpgrade, ws},
+  http::{HeaderMap, request},
+  response::IntoResponse,
+};
+use bytes::Bytes;
+use futures_util::{SinkExt, StreamExt as _};
+use komodo_client::{
+  api::terminal::ConnectTerminalQuery, entities::user::User,
+};
+use periphery_client::api::terminal::DisconnectTerminal;
+use serde::de::DeserializeOwned;
+use tokio_util::sync::CancellationToken;
+
+use crate::{
+  helpers::terminal::setup_target_for_user,
+  periphery::{PeripheryClient, terminal::ConnectTerminalResponse},
+  state::periphery_connections,
+};
+
+#[instrument("ConnectTerminal", skip(ws))]
+pub async fn handler(
+  Qs(query): Qs<ConnectTerminalQuery>,
+  ConnectInfo(info): ConnectInfo<SocketAddr>,
+  headers: HeaderMap,
+  ws: WebSocketUpgrade,
+) -> impl IntoResponse {
+  let ip = info.ip();
+  ws.on_upgrade(move |socket| async move {
+    let Some((mut client_socket, user)) =
+      super::user_ws_login(socket, &headers, ip).await
+    else {
+      return;
+    };
+
+    let (periphery, response) =
+      match setup_forwarding(query, &user).await {
+        Ok(response) => response,
+        Err(e) => {
+          let _ = client_socket
+            .send(ws::Message::text(format!("ERROR: {e:#}")))
+            .await;
+          let _ = client_socket.close().await;
+          return;
+        }
+      };
+
+    forward_ws_channel(periphery, client_socket, response).await
+  })
+}
+
+async fn setup_forwarding(
+  ConnectTerminalQuery {
+    target,
+    terminal,
+    init,
+  }: ConnectTerminalQuery,
+  user: &User,
+) -> anyhow::Result<(PeripheryClient, ConnectTerminalResponse)> {
+  let (target, terminal, periphery) =
+    setup_target_for_user(target, terminal, init, user).await?;
+  let response = periphery.connect_terminal(terminal, target).await?;
+  Ok((periphery, response))
+}
+
+async fn forward_ws_channel(
+  periphery: PeripheryClient,
+  client_socket: axum::extract::ws::WebSocket,
+  ConnectTerminalResponse {
+    channel,
+    sender: periphery_sender,
+    receiver: mut periphery_receiver,
+  }: ConnectTerminalResponse,
+) {
+  let (mut client_send, mut client_receive) = client_socket.split();
+  let cancel = CancellationToken::new();
+
+  periphery_receiver.set_cancel(cancel.clone());
+
+  trace!("starting ws exchange");
+
+  let core_to_periphery = async {
+    loop {
+      let client_recv_res = tokio::select! {
+        res = client_receive.next() => res,
+        _ = cancel.cancelled() => break,
+      };
+      let bytes = match client_recv_res {
+        Some(Ok(ws::Message::Binary(bytes))) => bytes.into(),
+        Some(Ok(ws::Message::Text(text))) => {
+          let bytes: Bytes = text.into();
+          bytes.into()
+        }
+        Some(Ok(ws::Message::Close(_frame))) => {
+          break;
+        }
+        Some(Err(_e)) => {
+          break;
+        }
+        None => {
+          break;
+        }
+        // Ignore
+        Some(Ok(_)) => continue,
+      };
+      if let Err(_e) =
+        periphery_sender.send_terminal(channel, Ok(bytes)).await
+      {
+        break;
+      };
+    }
+    cancel.cancel();
+    let _ = periphery_sender
+      .send_terminal(channel, Err(anyhow!("Client disconnected")))
+      .await;
+  };
+
+  let periphery_to_core = async {
+    loop {
+      // Already adheres to cancellation token
+      match periphery_receiver.recv().await {
+        Ok(Ok(bytes)) => {
+          if let Err(e) =
+            client_send.send(ws::Message::Binary(bytes.into())).await
+          {
+            debug!("{e:?}");
+            break;
+          };
+        }
+        Ok(Err(e)) => {
+          let _ = client_send
+            .send(ws::Message::text(format!("{e:#}")))
+            .await;
+          break;
+        }
+        Err(_) => {
+          let _ =
+            client_send.send(ws::Message::text("STREAM EOF")).await;
+          break;
+        }
+      }
+    }
+    let _ = client_send.close().await;
+    cancel.cancel();
+  };
+
+  tokio::join!(core_to_periphery, periphery_to_core);
+
+  // Cleanup
+  if let Err(e) =
+    periphery.request(DisconnectTerminal { channel }).await
+  {
+    warn!(
+      "Failed to disconnect Periphery terminal forwarding | {e:#}",
+    )
+  }
+  if let Some(connection) =
+    periphery_connections().get(&periphery.id).await
+  {
+    connection.terminals.remove(&channel).await;
+  }
+}
+
+pub struct Qs<T>(pub T);
+
+impl<S, T> FromRequestParts<S> for Qs<T>
+where
+  S: Send + Sync,
+  T: DeserializeOwned,
+{
+  type Rejection = axum::response::Response;
+
+  async fn from_request_parts(
+    parts: &mut request::Parts,
+    _state: &S,
+  ) -> Result<Self, Self::Rejection> {
+    let raw = parts.uri.query().unwrap_or_default();
+    serde_qs::from_str::<T>(raw).map(Qs).map_err(|e| {
+      axum::response::IntoResponse::into_response((
+        axum::http::StatusCode::BAD_REQUEST,
+        format!("Failed to parse request query: {e}"),
+      ))
+    })
+  }
+}

bin/core/src/api/ws/update.rs

@@ -1,9 +1,12 @@
+use std::net::SocketAddr;
+
 use anyhow::anyhow;
 use axum::{
-  extract::{WebSocketUpgrade, ws::Message},
+  extract::{ConnectInfo, WebSocketUpgrade, ws::Message},
+  http::HeaderMap,
   response::IntoResponse,
 };
-use futures::{SinkExt, StreamExt};
+use futures_util::{SinkExt, StreamExt};
 use komodo_client::entities::{
   ResourceTarget, permission::PermissionLevel, user::User,
 };
@@ -16,17 +19,24 @@ use crate::helpers::{
   channel::update_channel, query::get_user_permission_on_target,
 };
 
-pub async fn handler(ws: WebSocketUpgrade) -> impl IntoResponse {
+pub async fn handler(
+  headers: HeaderMap,
+  ConnectInfo(info): ConnectInfo<SocketAddr>,
+  ws: WebSocketUpgrade,
+) -> impl IntoResponse {
   // get a reveiver for internal update messages.
   let mut receiver = update_channel().receiver.resubscribe();
+  let ip = info.ip();
 
   // handle http -> ws updgrade
-  ws.on_upgrade(|socket| async move {
-    let Some((socket, user)) = super::user_ws_login(socket).await else {
-      return
+  ws.on_upgrade(move |socket| async move {
+    let Some((client_socket, user)) =
+      super::user_ws_login(socket, &headers, ip).await
+    else {
+      return;
     };
 
-    let (mut ws_sender, mut ws_reciever) = socket.split();
+    let (mut ws_sender, mut ws_reciever) = client_socket.split();
 
     let cancel = CancellationToken::new();
     let cancel_clone = cancel.clone();

bin/core/src/auth/github/client.rs

@@ -7,15 +7,15 @@ use komodo_client::entities::{
 };
 use reqwest::StatusCode;
 use serde::{Deserialize, Serialize, de::DeserializeOwned};
-use tokio::sync::Mutex;
 
 use crate::{auth::STATE_PREFIX_LENGTH, config::core_config};
 
-pub fn github_oauth_client() -> &'static Option<GithubOauthClient> {
+pub fn github_oauth_client() -> Option<&'static GithubOauthClient> {
   static GITHUB_OAUTH_CLIENT: OnceLock<Option<GithubOauthClient>> =
     OnceLock::new();
   GITHUB_OAUTH_CLIENT
     .get_or_init(|| GithubOauthClient::new(core_config()))
+    .as_ref()
 }
 
 pub struct GithubOauthClient {
@@ -24,7 +24,6 @@ pub struct GithubOauthClient {
   client_secret: String,
   redirect_uri: String,
   scopes: String,
-  states: Mutex<Vec<String>>,
   user_agent: String,
 }
 
@@ -69,15 +68,14 @@ impl GithubOauthClient {
       redirect_uri: format!("{host}/auth/github/callback"),
       user_agent: Default::default(),
       scopes: Default::default(),
-      states: Default::default(),
     }
     .into()
   }
 
-  pub async fn get_login_redirect_url(
+  pub async fn get_state_and_login_redirect_url(
     &self,
     redirect: Option<String>,
-  ) -> String {
+  ) -> (String, String) {
     let state_prefix = random_string(STATE_PREFIX_LENGTH);
     let state = match redirect {
       Some(redirect) => format!("{state_prefix}{redirect}"),
@@ -87,22 +85,7 @@ impl GithubOauthClient {
       "https://github.com/login/oauth/authorize?state={state}&client_id={}&redirect_uri={}&scope={}",
       self.client_id, self.redirect_uri, self.scopes
     );
-    let mut states = self.states.lock().await;
-    states.push(state);
-    redirect_url
-  }
-
-  pub async fn check_state(&self, state: &str) -> bool {
-    let mut contained = false;
-    self.states.lock().await.retain(|s| {
-      if s.as_str() == state {
-        contained = true;
-        false
-      } else {
-        true
-      }
-    });
-    contained
+    (state, redirect_url)
   }
 
   pub async fn get_access_token(

bin/core/src/auth/github/mod.rs

@@ -1,20 +1,40 @@
+use std::net::SocketAddr;
+
 use anyhow::{Context, anyhow};
 use axum::{
-  Router, extract::Query, response::Redirect, routing::get,
+  Router,
+  extract::{ConnectInfo, Query},
+  http::HeaderMap,
+  response::Redirect,
+  routing::get,
 };
-use database::mongo_indexed::Document;
 use database::mungos::mongodb::bson::doc;
-use komodo_client::entities::{
-  komodo_timestamp, random_string,
-  user::{User, UserConfig},
+use database::{
+  mongo_indexed::Document, mungos::by_id::update_one_by_id,
 };
+use futures_util::TryFutureExt;
+use komodo_client::{
+  api::auth::UserIdOrTwoFactor,
+  entities::{
+    komodo_timestamp, random_string,
+    user::{User, UserConfig},
+  },
+};
+use rate_limit::WithFailureRateLimit;
 use reqwest::StatusCode;
-use serde::Deserialize;
-use serror::AddStatusCode;
+use serde::{Deserialize, Serialize};
+use serror::{AddStatusCode, AddStatusCodeError as _};
+use tower_sessions::Session;
 
 use crate::{
+  api::user::SessionThirdPartyLinkInfo,
+  auth::{
+    SessionPasskeyLogin, SessionTotpLogin, SessionUserId,
+    format_redirect, github::client::GithubOauthClient,
+  },
   config::core_config,
-  state::{db_client, jwt_client},
+  helpers::query::{find_github_user, get_user},
+  state::{auth_rate_limiter, db_client, webauthn},
 };
 
 use self::client::github_oauth_client;
@@ -25,27 +45,89 @@ pub mod client;
 
 pub fn router() -> Router {
   Router::new()
-    .route(
-      "/login",
-      get(|Query(query): Query<RedirectQuery>| async {
-        Redirect::to(
-          &github_oauth_client()
-            .as_ref()
-            // OK: the router is only mounted in case that the client is populated
-            .unwrap()
-            .get_login_redirect_url(query.redirect)
-            .await,
-        )
-      }),
-    )
+    .route("/login", get(login))
+    .route("/link", get(link))
     .route(
       "/callback",
-      get(|query| async {
-        callback(query).await.status_code(StatusCode::UNAUTHORIZED)
-      }),
+      get(
+        |query,
+         session: Session,
+         headers: HeaderMap,
+         ConnectInfo(info): ConnectInfo<SocketAddr>| async move {
+          callback(query, session)
+            .map_err(|e| e.status_code(StatusCode::UNAUTHORIZED))
+            .with_failure_rate_limit_using_headers(
+              auth_rate_limiter(),
+              &headers,
+              Some(info.ip()),
+            )
+            .await
+        },
+      ),
     )
 }
 
+#[derive(Serialize, Deserialize)]
+struct SessionGithubVerificationInfo {
+  state: String,
+}
+
+impl SessionGithubVerificationInfo {
+  const KEY: &str = "github-verification-info";
+}
+
+async fn login(
+  session: Session,
+  Query(query): Query<RedirectQuery>,
+) -> serror::Result<Redirect> {
+  let (state, uri) = github_oauth_client()
+    .as_ref()
+    .context("Github Oauth not configured")
+    .status_code(StatusCode::UNAUTHORIZED)?
+    .get_state_and_login_redirect_url(query.redirect)
+    .await;
+  session
+    .insert(
+      SessionGithubVerificationInfo::KEY,
+      SessionGithubVerificationInfo { state },
+    )
+    .await
+    .context("Failed to insert github oauth session state")?;
+  Ok(Redirect::to(&uri))
+}
+
+#[derive(Serialize, Deserialize)]
+struct SessionGithubLinkInfo {
+  user_id: String,
+  state: String,
+}
+
+impl SessionGithubLinkInfo {
+  const KEY: &str = "github-link-info";
+}
+
+async fn link(session: Session) -> serror::Result<Redirect> {
+  let SessionThirdPartyLinkInfo { user_id } = session
+    .remove(SessionThirdPartyLinkInfo::KEY)
+    .await
+    .context("Invalid session third party link info.")?
+    .context("Missing session third party link info")?;
+  let (state, uri) = github_oauth_client()
+    .as_ref()
+    .context("Github Oauth not configured")
+    .status_code(StatusCode::UNAUTHORIZED)?
+    .get_state_and_login_redirect_url(None)
+    .await;
+  session
+    .insert(
+      SessionGithubLinkInfo::KEY,
+      SessionGithubLinkInfo { user_id, state },
+    )
+    .await
+    .context("Failed to insert session link info")?;
+  Ok(Redirect::to(&uri))
+}
+
 #[derive(Debug, Deserialize)]
 struct CallbackQuery {
   state: String,
@@ -54,25 +136,92 @@ struct CallbackQuery {
 
 async fn callback(
   Query(query): Query<CallbackQuery>,
+  session: Session,
 ) -> anyhow::Result<Redirect> {
-  let client = github_oauth_client().as_ref().unwrap();
-  if !client.check_state(&query.state).await {
-    return Err(anyhow!("state mismatch"));
+  let client =
+    github_oauth_client().context("Missing github oauth client")?;
+
+  // Check first if this is a link callback
+  // and use the linking handler if so.
+  if let Ok(Some(info)) =
+    session.remove(SessionGithubLinkInfo::KEY).await
+  {
+    return link_github_callback(client, info, query).await;
   }
+
+  let SessionGithubVerificationInfo { state } = session
+    .remove(SessionGithubVerificationInfo::KEY)
+    .await
+    .context("Invalid github oauth session state")?
+    .context("Missing github oauth session state")?;
+
+  if query.state != state {
+    return Err(anyhow!("State mismatch"));
+  }
+
   let token = client.get_access_token(&query.code).await?;
+
   let github_user =
     client.get_github_user(&token.access_token).await?;
+
   let github_id = github_user.id.to_string();
+
   let db_client = db_client();
-  let user = db_client
-    .users
-    .find_one(doc! { "config.data.github_id": &github_id })
-    .await
-    .context("failed at find user query from database")?;
-  let jwt = match user {
-    Some(user) => jwt_client()
-      .encode(user.id)
-      .context("failed to generate jwt")?,
+
+  let user = find_github_user(&github_id).await?;
+
+  let user_id_or_two_factor = match user {
+    Some(user) => {
+      match (
+        user.third_party_skip_2fa,
+        user.passkey.passkey,
+        user.totp.enrolled(),
+      ) {
+        // Skip / No 2FA
+        (true, _, _) | (false, None, false) => {
+          session
+            .insert(
+              SessionUserId::KEY,
+              SessionUserId(user.id.clone()),
+            )
+            .await
+            .context("Failed to store user id for client session")?;
+          UserIdOrTwoFactor::UserId(user.id)
+        }
+        // WebAuthn Passkey 2FA
+        (false, Some(passkey), _) => {
+          let webauthn = webauthn().context(
+            "No webauthn provider available, invalid KOMODO_HOST config",
+          )?;
+          let (response, server_state) = webauthn
+            .start_passkey_authentication(&[passkey])
+            .context("Failed to start passkey authentication flow")?;
+          session
+            .insert(
+              SessionPasskeyLogin::KEY,
+              SessionPasskeyLogin {
+                user_id: user.id,
+                state: server_state,
+              },
+            )
+            .await?;
+          UserIdOrTwoFactor::Passkey(response)
+        }
+        // TOTP 2FA
+        (false, None, true) => {
+          session
+            .insert(
+              SessionTotpLogin::KEY,
+              SessionTotpLogin { user_id: user.id },
+            )
+            .await
+            .context(
+              "Failed to store totp login state in for user session",
+            )?;
+          UserIdOrTwoFactor::Totp {}
+        }
+      }
+    }
     None => {
       let ts = komodo_timestamp();
       let no_users_exist =
@@ -111,28 +260,90 @@ async fn callback(
           github_id,
           avatar: github_user.avatar_url,
         },
+        linked_logins: Default::default(),
+        totp: Default::default(),
+        passkey: Default::default(),
+        third_party_skip_2fa: Default::default(),
       };
       let user_id = db_client
         .users
         .insert_one(user)
         .await
-        .context("failed to create user on mongo")?
+        .context("Failed to create user on mongo")?
         .inserted_id
         .as_object_id()
         .context("inserted_id is not ObjectId")?
         .to_string();
-      jwt_client()
-        .encode(user_id)
-        .context("failed to generate jwt")?
+      UserIdOrTwoFactor::UserId(user_id)
     }
   };
-  let exchange_token = jwt_client().create_exchange_token(jwt).await;
-  let redirect = &query.state[STATE_PREFIX_LENGTH..];
-  let redirect_url = if redirect.is_empty() {
-    format!("{}?token={exchange_token}", core_config().host)
-  } else {
-    let splitter = if redirect.contains('?') { '&' } else { '?' };
-    format!("{redirect}{splitter}token={exchange_token}")
+  let redirect = Some(&query.state[STATE_PREFIX_LENGTH..]);
+  match user_id_or_two_factor {
+    UserIdOrTwoFactor::UserId(_) => {
+      Ok(format_redirect(redirect, "redeem_ready=true"))
+    }
+    UserIdOrTwoFactor::Totp {} => {
+      Ok(format_redirect(redirect, "totp=true"))
+    }
+    UserIdOrTwoFactor::Passkey(passkey) => {
+      let passkey = serde_json::to_string(&passkey)
+        .context("Failed to serialize passkey response")?;
+      let passkey = urlencoding::encode(&passkey);
+      Ok(format_redirect(redirect, &format!("passkey={passkey}")))
+    }
+  }
+}
+
+/// This intercepts during the normal oauth callback if
+/// 'github-link-info' is found on session.
+async fn link_github_callback(
+  client: &GithubOauthClient,
+  SessionGithubLinkInfo { user_id, state }: SessionGithubLinkInfo,
+  query: CallbackQuery,
+) -> anyhow::Result<Redirect> {
+  if query.state != state {
+    return Err(anyhow!("State mismatch"));
+  }
+
+  let token = client.get_access_token(&query.code).await?;
+
+  let github_user =
+    client.get_github_user(&token.access_token).await?;
+  let github_id = github_user.id.to_string();
+
+  if let Some(existing_user) = find_github_user(&github_id).await? {
+    if existing_user.id == user_id {
+      // Link is already complete, this is a no-op
+      return Ok(Redirect::to(&format!(
+        "{}/settings",
+        core_config().host,
+      )));
+    } else {
+      return Err(anyhow!("Account already linked to another user."));
+    }
+  }
+
+  let user = get_user(&user_id).await?;
+
+  if let UserConfig::Github { .. } = &user.config {
+    return Err(anyhow!(
+      "User is primary Github user, cannot link another Github login."
+    ));
+  }
+
+  let update = doc! {
+    "$set": {
+      "linked_logins.Github.type": "Github",
+      "linked_logins.Github.data.github_id": &github_id,
+      "linked_logins.Github.data.avatar": &github_user.avatar_url,
+    }
   };
-  Ok(Redirect::to(&redirect_url))
+
+  update_one_by_id(&db_client().users, &user_id, update, None)
+    .await
+    .context(
+      "Failed to link Github login to existing user on database",
+    )?;
+
+  Ok(Redirect::to(&format!("{}/settings", core_config().host,)))
 }

bin/core/src/auth/google/client.rs

@@ -1,22 +1,22 @@
 use std::sync::OnceLock;
 
 use anyhow::{Context, anyhow};
-use jsonwebtoken::{DecodingKey, Validation, decode};
+use jsonwebtoken::dangerous::insecure_decode;
 use komodo_client::entities::{
   config::core::{CoreConfig, OauthCredentials},
   random_string,
 };
 use reqwest::StatusCode;
 use serde::{Deserialize, de::DeserializeOwned};
-use tokio::sync::Mutex;
 
 use crate::{auth::STATE_PREFIX_LENGTH, config::core_config};
 
-pub fn google_oauth_client() -> &'static Option<GoogleOauthClient> {
+pub fn google_oauth_client() -> Option<&'static GoogleOauthClient> {
   static GOOGLE_OAUTH_CLIENT: OnceLock<Option<GoogleOauthClient>> =
     OnceLock::new();
   GOOGLE_OAUTH_CLIENT
     .get_or_init(|| GoogleOauthClient::new(core_config()))
+    .as_ref()
 }
 
 pub struct GoogleOauthClient {
@@ -25,7 +25,6 @@ pub struct GoogleOauthClient {
   client_secret: String,
   redirect_uri: String,
   scopes: String,
-  states: Mutex<Vec<String>>,
   user_agent: String,
 }
 
@@ -77,16 +76,15 @@ impl GoogleOauthClient {
       client_secret: secret.clone(),
       redirect_uri: format!("{host}/auth/google/callback"),
       user_agent: String::from("komodo"),
-      states: Default::default(),
       scopes,
     }
     .into()
   }
 
-  pub async fn get_login_redirect_url(
+  pub async fn get_state_and_login_redirect_url(
     &self,
     redirect: Option<String>,
-  ) -> String {
+  ) -> (String, String) {
     let state_prefix = random_string(STATE_PREFIX_LENGTH);
     let state = match redirect {
       Some(redirect) => format!("{state_prefix}{redirect}"),
@@ -96,22 +94,7 @@ impl GoogleOauthClient {
       "https://accounts.google.com/o/oauth2/v2/auth?response_type=code&state={state}&client_id={}&redirect_uri={}&scope={}",
       self.client_id, self.redirect_uri, self.scopes
     );
-    let mut states = self.states.lock().await;
-    states.push(state);
-    redirect_url
-  }
-
-  pub async fn check_state(&self, state: &str) -> bool {
-    let mut contained = false;
-    self.states.lock().await.retain(|s| {
-      if s.as_str() == state {
-        contained = true;
-        false
-      } else {
-        true
-      }
-    });
-    contained
+    (state, redirect_url)
   }
 
   pub async fn get_access_token(
@@ -138,15 +121,8 @@ impl GoogleOauthClient {
     &self,
     id_token: &str,
   ) -> anyhow::Result<GoogleUser> {
-    let mut v = Validation::new(Default::default());
-    v.insecure_disable_signature_validation();
-    v.validate_aud = false;
-    let res = decode::<GoogleUser>(
-      id_token,
-      &DecodingKey::from_secret(b""),
-      &v,
-    )
-    .context("failed to decode google id token")?;
+    let res = insecure_decode::<GoogleUser>(id_token)
+      .context("failed to decode google id token")?;
     Ok(res.claims)
   }
 

bin/core/src/auth/google/mod.rs

@@ -1,21 +1,41 @@
+use std::net::SocketAddr;
+
 use anyhow::{Context, anyhow};
 use async_timing_util::unix_timestamp_ms;
 use axum::{
-  Router, extract::Query, response::Redirect, routing::get,
+  Router,
+  extract::{ConnectInfo, Query},
+  http::HeaderMap,
+  response::Redirect,
+  routing::get,
 };
-use database::mongo_indexed::Document;
 use database::mungos::mongodb::bson::doc;
-use komodo_client::entities::{
-  random_string,
-  user::{User, UserConfig},
+use database::{
+  mongo_indexed::Document, mungos::by_id::update_one_by_id,
 };
+use futures_util::TryFutureExt;
+use komodo_client::{
+  api::auth::UserIdOrTwoFactor,
+  entities::{
+    random_string,
+    user::{User, UserConfig},
+  },
+};
+use rate_limit::WithFailureRateLimit;
 use reqwest::StatusCode;
-use serde::Deserialize;
-use serror::AddStatusCode;
+use serde::{Deserialize, Serialize};
+use serror::{AddStatusCode, AddStatusCodeError as _};
+use tower_sessions::Session;
 
 use crate::{
+  api::user::SessionThirdPartyLinkInfo,
+  auth::{
+    SessionPasskeyLogin, SessionTotpLogin, SessionUserId,
+    format_redirect, google::client::GoogleOauthClient,
+  },
   config::core_config,
-  state::{db_client, jwt_client},
+  helpers::query::{find_google_user, get_user},
+  state::{auth_rate_limiter, db_client, webauthn},
 };
 
 use self::client::google_oauth_client;
@@ -26,27 +46,89 @@ pub mod client;
 
 pub fn router() -> Router {
   Router::new()
-    .route(
-      "/login",
-      get(|Query(query): Query<RedirectQuery>| async move {
-        Redirect::to(
-          &google_oauth_client()
-            .as_ref()
-            // OK: its not mounted unless the client is populated
-            .unwrap()
-            .get_login_redirect_url(query.redirect)
-            .await,
-        )
-      }),
-    )
+    .route("/login", get(login))
+    .route("/link", get(link))
     .route(
       "/callback",
-      get(|query| async {
-        callback(query).await.status_code(StatusCode::UNAUTHORIZED)
-      }),
+      get(
+        |query,
+         session,
+         headers: HeaderMap,
+         ConnectInfo(info): ConnectInfo<SocketAddr>| async move {
+          callback(session, query)
+            .map_err(|e| e.status_code(StatusCode::UNAUTHORIZED))
+            .with_failure_rate_limit_using_headers(
+              auth_rate_limiter(),
+              &headers,
+              Some(info.ip()),
+            )
+            .await
+        },
+      ),
     )
 }
 
+#[derive(Serialize, Deserialize)]
+struct SessionGoogleVerificationInfo {
+  state: String,
+}
+
+impl SessionGoogleVerificationInfo {
+  const KEY: &str = "google-verification-info";
+}
+
+async fn login(
+  session: Session,
+  Query(query): Query<RedirectQuery>,
+) -> serror::Result<Redirect> {
+  let (state, uri) = google_oauth_client()
+    .as_ref()
+    .context("Google Oauth not configured")
+    .status_code(StatusCode::UNAUTHORIZED)?
+    .get_state_and_login_redirect_url(query.redirect)
+    .await;
+  session
+    .insert(
+      SessionGoogleVerificationInfo::KEY,
+      SessionGoogleVerificationInfo { state },
+    )
+    .await
+    .context("Failed to insert google oauth session state")?;
+  Ok(Redirect::to(&uri))
+}
+
+#[derive(Serialize, Deserialize)]
+struct SessionGoogleLinkInfo {
+  user_id: String,
+  state: String,
+}
+
+impl SessionGoogleLinkInfo {
+  const KEY: &str = "google-link-info";
+}
+
+async fn link(session: Session) -> serror::Result<Redirect> {
+  let SessionThirdPartyLinkInfo { user_id } = session
+    .remove(SessionThirdPartyLinkInfo::KEY)
+    .await
+    .context("Invalid session third party link info.")?
+    .context("Missing session third party link info")?;
+  let (state, uri) = google_oauth_client()
+    .as_ref()
+    .context("Google Oauth not configured")
+    .status_code(StatusCode::UNAUTHORIZED)?
+    .get_state_and_login_redirect_url(None)
+    .await;
+  session
+    .insert(
+      SessionGoogleLinkInfo::KEY,
+      SessionGoogleLinkInfo { user_id, state },
+    )
+    .await
+    .context("Failed to insert session link info")?;
+  Ok(Redirect::to(&uri))
+}
+
 #[derive(Debug, Deserialize)]
 struct CallbackQuery {
   state: Option<String>,
@@ -55,37 +137,102 @@ struct CallbackQuery {
 }
 
 async fn callback(
+  session: Session,
   Query(query): Query<CallbackQuery>,
 ) -> anyhow::Result<Redirect> {
-  // Safe: the method is only called after the client is_some
-  let client = google_oauth_client().as_ref().unwrap();
+  let client =
+    google_oauth_client().context("Missing google oauth client")?;
+
   if let Some(error) = query.error {
-    return Err(anyhow!("auth error from google: {error}"));
+    return Err(anyhow!("Auth error from google: {error}"));
   }
-  let state = query
+
+  let client_state = query
     .state
-    .context("callback query does not contain state")?;
-  if !client.check_state(&state).await {
-    return Err(anyhow!("state mismatch"));
+    .context("Callback query does not contain state")?;
+  let code =
+    query.code.context("callback query does not contain code")?;
+
+  // Check first if this is a link callback
+  // and use the linking handler if so.
+  if let Ok(Some(info)) =
+    session.remove(SessionGoogleLinkInfo::KEY).await
+  {
+    return link_google_callback(client, info, client_state, code)
+      .await;
   }
-  let token = client
-    .get_access_token(
-      &query.code.context("callback query does not contain code")?,
-    )
-    .await?;
+
+  let SessionGoogleVerificationInfo { state } = session
+    .get(SessionGoogleVerificationInfo::KEY)
+    .await
+    .context("Invalid google oauth session state")?
+    .context("Missing google oauth session state")?;
+
+  if client_state != state {
+    return Err(anyhow!("State mismatch"));
+  }
+
+  let token = client.get_access_token(&code).await?;
+
   let google_user = client.get_google_user(&token.id_token)?;
   let google_id = google_user.id.to_string();
-  let db_client = db_client();
-  let user = db_client
-    .users
-    .find_one(doc! { "config.data.google_id": &google_id })
-    .await
-    .context("failed at find user query from mongo")?;
-  let jwt = match user {
-    Some(user) => jwt_client()
-      .encode(user.id)
-      .context("failed to generate jwt")?,
+
+  let user = find_google_user(&google_id).await?;
+
+  let user_id_or_two_factor = match user {
+    Some(user) => {
+      match (
+        user.third_party_skip_2fa,
+        user.passkey.passkey,
+        user.totp.enrolled(),
+      ) {
+        // Skip / No 2FA
+        (true, _, _) | (false, None, false) => {
+          session
+            .insert(
+              SessionUserId::KEY,
+              SessionUserId(user.id.clone()),
+            )
+            .await
+            .context("Failed to store user id for client session")?;
+          UserIdOrTwoFactor::UserId(user.id)
+        }
+        // WebAuthn Passkey 2FA
+        (false, Some(passkey), _) => {
+          let webauthn = webauthn().context(
+            "No webauthn provider available, invalid KOMODO_HOST config",
+          )?;
+          let (response, server_state) = webauthn
+            .start_passkey_authentication(&[passkey])
+            .context("Failed to start passkey authentication flow")?;
+          session
+            .insert(
+              SessionPasskeyLogin::KEY,
+              SessionPasskeyLogin {
+                user_id: user.id,
+                state: server_state,
+              },
+            )
+            .await?;
+          UserIdOrTwoFactor::Passkey(response)
+        }
+        // TOTP 2FA
+        (false, None, true) => {
+          session
+            .insert(
+              SessionTotpLogin::KEY,
+              SessionTotpLogin { user_id: user.id },
+            )
+            .await
+            .context(
+              "Failed to store totp login state in for user session",
+            )?;
+          UserIdOrTwoFactor::Totp {}
+        }
+      }
+    }
     None => {
+      let db_client = db_client();
       let ts = unix_timestamp_ms() as i64;
       let no_users_exist =
         db_client.users.find_one(Document::new()).await?.is_none();
@@ -128,28 +275,90 @@ async fn callback(
           google_id,
           avatar: google_user.picture,
         },
+        linked_logins: Default::default(),
+        totp: Default::default(),
+        passkey: Default::default(),
+        third_party_skip_2fa: Default::default(),
       };
       let user_id = db_client
         .users
         .insert_one(user)
         .await
-        .context("failed to create user on mongo")?
+        .context("Failed to create user on mongo")?
         .inserted_id
         .as_object_id()
         .context("inserted_id is not ObjectId")?
         .to_string();
-      jwt_client()
-        .encode(user_id)
-        .context("failed to generate jwt")?
+      UserIdOrTwoFactor::UserId(user_id)
     }
   };
-  let exchange_token = jwt_client().create_exchange_token(jwt).await;
-  let redirect = &state[STATE_PREFIX_LENGTH..];
-  let redirect_url = if redirect.is_empty() {
-    format!("{}?token={exchange_token}", core_config().host)
-  } else {
-    let splitter = if redirect.contains('?') { '&' } else { '?' };
-    format!("{redirect}{splitter}token={exchange_token}")
+  let redirect = Some(&state[STATE_PREFIX_LENGTH..]);
+  match user_id_or_two_factor {
+    UserIdOrTwoFactor::UserId(_) => {
+      Ok(format_redirect(redirect, "redeem_ready=true"))
+    }
+    UserIdOrTwoFactor::Totp {} => {
+      Ok(format_redirect(redirect, "totp=true"))
+    }
+    UserIdOrTwoFactor::Passkey(passkey) => {
+      let passkey = serde_json::to_string(&passkey)
+        .context("Failed to serialize passkey response")?;
+      let passkey = urlencoding::encode(&passkey);
+      Ok(format_redirect(redirect, &format!("passkey={passkey}")))
+    }
+  }
+}
+
+/// This intercepts during the normal oauth callback if
+/// 'google-link-info' is found on session.
+async fn link_google_callback(
+  client: &GoogleOauthClient,
+  SessionGoogleLinkInfo { user_id, state }: SessionGoogleLinkInfo,
+  client_state: String,
+  code: String,
+) -> anyhow::Result<Redirect> {
+  if client_state != state {
+    return Err(anyhow!("State mismatch"));
+  }
+
+  let token = client.get_access_token(&code).await?;
+
+  let google_user = client.get_google_user(&token.id_token)?;
+  let google_id = google_user.id.to_string();
+
+  if let Some(existing_user) = find_google_user(&google_id).await? {
+    if existing_user.id == user_id {
+      // Link is already complete, this is a no-op
+      return Ok(Redirect::to(&format!(
+        "{}/settings",
+        core_config().host,
+      )));
+    } else {
+      return Err(anyhow!("Account already linked to another user."));
+    }
+  }
+
+  let user = get_user(&user_id).await?;
+
+  if let UserConfig::Google { .. } = &user.config {
+    return Err(anyhow!(
+      "User is primary Google user, cannot link another Google login."
+    ));
+  }
+
+  let update = doc! {
+    "$set": {
+      "linked_logins.Google.type": "Google",
+      "linked_logins.Google.data.google_id": &google_id,
+      "linked_logins.Google.data.avatar": &google_user.picture,
+    }
   };
-  Ok(Redirect::to(&redirect_url))
+
+  update_one_by_id(&db_client().users, &user_id, update, None)
+    .await
+    .context(
+      "Failed to link Google login to existing user on database",
+    )?;
+
+  Ok(Redirect::to(&format!("{}/settings", core_config().host,)))
 }

bin/core/src/auth/jwt.rs

@@ -1,9 +1,5 @@
-use std::collections::HashMap;
-
-use anyhow::{Context, anyhow};
-use async_timing_util::{
-  Timelength, get_timelength_in_ms, unix_timestamp_ms,
-};
+use anyhow::Context;
+use async_timing_util::{get_timelength_in_ms, unix_timestamp_ms};
 use database::mungos::mongodb::bson::doc;
 use jsonwebtoken::{
   DecodingKey, EncodingKey, Header, Validation, decode, encode,
@@ -13,9 +9,6 @@ use komodo_client::{
   entities::{config::core::CoreConfig, random_string},
 };
 use serde::{Deserialize, Serialize};
-use tokio::sync::Mutex;
-
-type ExchangeTokenMap = Mutex<HashMap<String, (JwtResponse, u128)>>;
 
 #[derive(Serialize, Deserialize, Clone)]
 pub struct JwtClaims {
@@ -30,7 +23,6 @@ pub struct JwtClient {
   encoding_key: EncodingKey,
   decoding_key: DecodingKey,
   ttl_ms: u128,
-  exchange_tokens: ExchangeTokenMap,
 }
 
 impl JwtClient {
@@ -48,7 +40,6 @@ impl JwtClient {
       ttl_ms: get_timelength_in_ms(
         config.jwt_ttl.to_string().parse()?,
       ),
-      exchange_tokens: Default::default(),
     })
   }
 
@@ -64,46 +55,13 @@ impl JwtClient {
       exp,
     };
     let jwt = encode(&self.header, &claims, &self.encoding_key)
-      .context("failed at signing claim")?;
+      .context("Failed at signing claim")?;
     Ok(JwtResponse { user_id, jwt })
   }
 
   pub fn decode(&self, jwt: &str) -> anyhow::Result<JwtClaims> {
     decode::<JwtClaims>(jwt, &self.decoding_key, &self.validation)
       .map(|res| res.claims)
-      .context("failed to decode token claims")
-  }
-
-  pub async fn create_exchange_token(
-    &self,
-    jwt: JwtResponse,
-  ) -> String {
-    let exchange_token = random_string(40);
-    self.exchange_tokens.lock().await.insert(
-      exchange_token.clone(),
-      (
-        jwt,
-        unix_timestamp_ms()
-          + get_timelength_in_ms(Timelength::OneMinute),
-      ),
-    );
-    exchange_token
-  }
-
-  pub async fn redeem_exchange_token(
-    &self,
-    exchange_token: &str,
-  ) -> anyhow::Result<JwtResponse> {
-    let (jwt, valid_until) = self
-      .exchange_tokens
-      .lock()
-      .await
-      .remove(exchange_token)
-      .context("invalid exchange token: unrecognized")?;
-    if unix_timestamp_ms() < valid_until {
-      Ok(jwt)
-    } else {
-      Err(anyhow!("invalid exchange token: expired"))
-    }
+      .context("Failed to decode token claims")
   }
 }

bin/core/src/auth/local.rs

@@ -1,148 +1,261 @@
-use std::str::FromStr;
+use std::sync::{Arc, OnceLock};
 
 use anyhow::{Context, anyhow};
 use async_timing_util::unix_timestamp_ms;
 use database::{
   hash_password,
-  mungos::mongodb::bson::{Document, doc, oid::ObjectId},
+  mungos::mongodb::bson::{Document, doc},
 };
 use komodo_client::{
   api::auth::{
     LoginLocalUser, LoginLocalUserResponse, SignUpLocalUser,
     SignUpLocalUserResponse,
   },
-  entities::user::{User, UserConfig},
+  entities::user::{User, UserConfig, UserConfigVariant},
 };
+use rate_limit::{RateLimiter, WithFailureRateLimit};
+use reqwest::StatusCode;
 use resolver_api::Resolve;
+use serror::{AddStatusCode as _, AddStatusCodeError};
+use tower_sessions::Session;
 
 use crate::{
   api::auth::AuthArgs,
+  auth::{SessionPasskeyLogin, SessionTotpLogin},
   config::core_config,
-  state::{db_client, jwt_client},
+  helpers::validations::{validate_password, validate_username},
+  state::{auth_rate_limiter, db_client, jwt_client, webauthn},
 };
 
 impl Resolve<AuthArgs> for SignUpLocalUser {
   #[instrument("SignUpLocalUser", skip(self))]
   async fn resolve(
     self,
-    _: &AuthArgs,
+    AuthArgs {
+      headers,
+      ip,
+      session,
+    }: &AuthArgs,
   ) -> serror::Result<SignUpLocalUserResponse> {
-    let core_config = core_config();
-
-    if !core_config.local_auth {
-      return Err(anyhow!("Local auth is not enabled").into());
-    }
-
-    if self.username.is_empty() {
-      return Err(anyhow!("Username cannot be empty string").into());
-    }
-
-    if ObjectId::from_str(&self.username).is_ok() {
-      return Err(
-        anyhow!("Username cannot be valid ObjectId").into(),
-      );
-    }
-
-    if self.password.is_empty() {
-      return Err(anyhow!("Password cannot be empty string").into());
-    }
-
-    let db = db_client();
-
-    let no_users_exist =
-      db.users.find_one(Document::new()).await?.is_none();
-
-    if !no_users_exist && core_config.disable_user_registration {
-      return Err(anyhow!("User registration is disabled").into());
-    }
-
-    if db
-      .users
-      .find_one(doc! { "username": &self.username })
+    sign_up_local_user(self)
+      .with_failure_rate_limit_using_headers(
+        auth_rate_limiter(),
+        headers,
+        Some(*ip),
+      )
       .await
-      .context("Failed to query for existing users")?
-      .is_some()
-    {
-      return Err(anyhow!("Username already taken.").into());
-    }
-
-    let ts = unix_timestamp_ms() as i64;
-    let hashed_password = hash_password(self.password)?;
-
-    let user = User {
-      id: Default::default(),
-      username: self.username,
-      enabled: no_users_exist || core_config.enable_new_users,
-      admin: no_users_exist,
-      super_admin: no_users_exist,
-      create_server_permissions: no_users_exist,
-      create_build_permissions: no_users_exist,
-      updated_at: ts,
-      last_update_view: 0,
-      recents: Default::default(),
-      all: Default::default(),
-      config: UserConfig::Local {
-        password: hashed_password,
-      },
-    };
-
-    let user_id = db_client()
-      .users
-      .insert_one(user)
-      .await
-      .context("failed to create user")?
-      .inserted_id
-      .as_object_id()
-      .context("inserted_id is not ObjectId")?
-      .to_string();
-
-    jwt_client()
-      .encode(user_id.clone())
-      .context("failed to generate jwt for user")
-      .map_err(Into::into)
   }
 }
 
+async fn sign_up_local_user(
+  req: SignUpLocalUser,
+) -> serror::Result<SignUpLocalUserResponse> {
+  let config = core_config();
+
+  if !config.local_auth {
+    return Err(anyhow!("Local auth is not enabled").into());
+  }
+
+  validate_username(&req.username)
+    .status_code(StatusCode::BAD_REQUEST)?;
+  validate_password(&req.password)
+    .status_code(StatusCode::BAD_REQUEST)?;
+
+  let db = db_client();
+
+  let no_users_exist =
+    db.users.find_one(Document::new()).await?.is_none();
+
+  if !no_users_exist && config.disable_user_registration {
+    return Err(
+      anyhow!("User registration is disabled")
+        .status_code(StatusCode::UNAUTHORIZED),
+    );
+  }
+
+  if db
+    .users
+    .find_one(doc! { "username": &req.username })
+    .await
+    .context("Failed to query for existing users")?
+    .is_some()
+  {
+    // When user registration is enabled, there is no way around allowing
+    // potential attackers to gain some insight about which usernames exist
+    // if they are allowed to register accounts. Since this can be easily inferred,
+    // might as well be clear. The auth rate limiter is critical here.
+    return Err(
+      anyhow!("Username already taken.")
+        .status_code(StatusCode::BAD_REQUEST),
+    );
+  }
+
+  let ts = unix_timestamp_ms() as i64;
+  let hashed_password = hash_password(req.password)?;
+
+  let user = User {
+    id: Default::default(),
+    username: req.username,
+    enabled: no_users_exist || config.enable_new_users,
+    admin: no_users_exist,
+    super_admin: no_users_exist,
+    create_server_permissions: no_users_exist,
+    create_build_permissions: no_users_exist,
+    updated_at: ts,
+    last_update_view: 0,
+    recents: Default::default(),
+    all: Default::default(),
+    config: UserConfig::Local {
+      password: hashed_password,
+    },
+    linked_logins: Default::default(),
+    totp: Default::default(),
+    passkey: Default::default(),
+    third_party_skip_2fa: Default::default(),
+  };
+
+  let user_id = db_client()
+    .users
+    .insert_one(user)
+    .await
+    .context("Failed to create user on database")?
+    .inserted_id
+    .as_object_id()
+    .context("The 'inserted_id' is not ObjectId")?
+    .to_string();
+
+  jwt_client()
+    .encode(user_id)
+    .context("Failed to generate JWT for user")
+    .map_err(Into::into)
+}
+
+/// Local login method has a dedicated rate limiter
+/// so the UI background calls using existing JWT do
+/// not influence the number of attempts user has
+/// to log in.
+fn login_local_user_rate_limiter() -> &'static RateLimiter {
+  static LOGIN_LOCAL_USER_RATE_LIMITER: OnceLock<Arc<RateLimiter>> =
+    OnceLock::new();
+  LOGIN_LOCAL_USER_RATE_LIMITER.get_or_init(|| {
+    let config = core_config();
+    RateLimiter::new(
+      config.auth_rate_limit_disabled,
+      config.auth_rate_limit_max_attempts as usize,
+      config.auth_rate_limit_window_seconds,
+    )
+  })
+}
+
 impl Resolve<AuthArgs> for LoginLocalUser {
   async fn resolve(
     self,
-    _: &AuthArgs,
+    AuthArgs {
+      headers,
+      ip,
+      session,
+    }: &AuthArgs,
   ) -> serror::Result<LoginLocalUserResponse> {
-    if !core_config().local_auth {
-      return Err(anyhow!("local auth is not enabled").into());
-    }
+    login_local_user(
+      self,
+      session
+        .as_ref()
+        .context("Method called in context without session")?,
+    )
+    .with_failure_rate_limit_using_headers(
+      login_local_user_rate_limiter(),
+      headers,
+      Some(*ip),
+    )
+    .await
+  }
+}
 
-    let user = db_client()
-      .users
-      .find_one(doc! { "username": &self.username })
-      .await
-      .context("failed at db query for users")?
-      .with_context(|| {
-        format!("did not find user with username {}", self.username)
-      })?;
+async fn login_local_user(
+  req: LoginLocalUser,
+  session: &Session,
+) -> serror::Result<LoginLocalUserResponse> {
+  if !core_config().local_auth {
+    return Err(
+      anyhow!("Local auth is not enabled")
+        .status_code(StatusCode::UNAUTHORIZED),
+    );
+  }
 
-    let UserConfig::Local {
-      password: user_pw_hash,
-    } = user.config
-    else {
+  validate_username(&req.username)
+    .status_code(StatusCode::BAD_REQUEST)?;
+
+  let user = db_client()
+    .users
+    .find_one(doc! { "username": &req.username })
+    .await
+    .context("Failed at db query for users")?
+    .context("Invalid login credentials")
+    .status_code(StatusCode::UNAUTHORIZED)?;
+
+  let user_pw_hash =
+    if let UserConfig::Local { password } = &user.config {
+      password
+    } else if let Some(UserConfig::Local { password }) =
+      user.linked_logins.get(UserConfigVariant::Local)
+    {
+      password
+    } else {
       return Err(
-        anyhow!(
-          "non-local auth users can not log in with a password"
-        )
-        .into(),
+        anyhow!("Invalid login credentials")
+          .status_code(StatusCode::UNAUTHORIZED),
       );
     };
 
-    let verified = bcrypt::verify(self.password, &user_pw_hash)
-      .context("failed at verify password")?;
+  let verified = bcrypt::verify(req.password, user_pw_hash)
+    .context("Invalid login credentials")
+    .status_code(StatusCode::UNAUTHORIZED)?;
 
-    if !verified {
-      return Err(anyhow!("invalid credentials").into());
+  if !verified {
+    return Err(
+      anyhow!("Invalid login credentials")
+        .status_code(StatusCode::UNAUTHORIZED),
+    );
+  }
+
+  match (user.passkey.passkey, user.totp.enrolled()) {
+    // WebAuthn 2FA
+    (Some(passkey), _) => {
+      let webauthn = webauthn().context(
+        "No webauthn provider available, invalid KOMODO_HOST config",
+      )?;
+      let (response, server_state) = webauthn
+        .start_passkey_authentication(&[passkey])
+        .context("Failed to start passkey authentication flow")?;
+      session
+        .insert(
+          SessionPasskeyLogin::KEY,
+          SessionPasskeyLogin {
+            user_id: user.id,
+            state: server_state,
+          },
+        )
+        .await?;
+      Ok(LoginLocalUserResponse::Passkey(response))
+    }
+    // TOTP 2FA
+    (None, true) => {
+      session
+        .insert(
+          SessionTotpLogin::KEY,
+          SessionTotpLogin { user_id: user.id },
+        )
+        .await?;
+      Ok(LoginLocalUserResponse::Totp {})
+    }
+    // No 2FA, can return JWT immediately
+    (None, false) => {
+      jwt_client()
+        .encode(user.id)
+        // This is in internal error (500), not auth error
+        .context("Failed to generate JWT for user")
+        .map(LoginLocalUserResponse::Jwt)
+        .map_err(Into::into)
     }
-
-    jwt_client()
-      .encode(user.id.clone())
-      .context("failed at generating jwt for user")
-      .map_err(Into::into)
   }
 }

bin/core/src/auth/mod.rs

@@ -1,18 +1,26 @@
+use std::net::SocketAddr;
+
 use anyhow::{Context, anyhow};
 use async_timing_util::unix_timestamp_ms;
 use axum::{
-  extract::Request, http::HeaderMap, middleware::Next,
-  response::Response,
+  extract::{ConnectInfo, Request},
+  http::HeaderMap,
+  middleware::Next,
+  response::{Redirect, Response},
 };
 use database::mungos::mongodb::bson::doc;
+use futures_util::TryFutureExt;
 use komodo_client::entities::{komodo_timestamp, user::User};
+use rate_limit::WithFailureRateLimit;
 use reqwest::StatusCode;
-use serde::Deserialize;
-use serror::AddStatusCode;
+use serde::{Deserialize, Serialize};
+use serror::AddStatusCodeError as _;
+use webauthn_rs::prelude::PasskeyAuthentication;
 
 use crate::{
+  config::core_config,
   helpers::query::get_user,
-  state::{db_client, jwt_client},
+  state::{auth_rate_limiter, db_client, jwt_client},
 };
 
 use self::jwt::JwtClaims;
@@ -21,24 +29,66 @@ pub mod github;
 pub mod google;
 pub mod jwt;
 pub mod oidc;
+pub mod totp;
 
 mod local;
 
+/// Length of random token in Oauth / OIDC 'state'
 const STATE_PREFIX_LENGTH: usize = 20;
+/// JWT Clock skew tolerance in milliseconds (10 seconds for JWTs)
+const JWT_CLOCK_SKEW_TOLERANCE_MS: u128 = 10 * 1000;
+/// Api Key Clock skew tolerance in milliseconds (5 minutes for Api Keys)
+const API_KEY_CLOCK_SKEW_TOLERANCE_MS: i64 = 5 * 60 * 1000;
 
 #[derive(Debug, Deserialize)]
 struct RedirectQuery {
   redirect: Option<String>,
 }
 
+#[derive(Serialize, Deserialize)]
+pub struct SessionUserId(pub String);
+impl SessionUserId {
+  pub const KEY: &str = "user-id";
+}
+
+#[derive(Serialize, Deserialize)]
+pub struct SessionTotpLogin {
+  pub user_id: String,
+}
+impl SessionTotpLogin {
+  pub const KEY: &str = "totp-login";
+}
+
+#[derive(Serialize, Deserialize)]
+pub struct SessionPasskeyLogin {
+  pub user_id: String,
+  /// This must stay server side only
+  pub state: PasskeyAuthentication,
+}
+impl SessionPasskeyLogin {
+  pub const KEY: &str = "passkey-login";
+}
+
 pub async fn auth_request(
   headers: HeaderMap,
   mut req: Request,
   next: Next,
 ) -> serror::Result<Response> {
-  let user = authenticate_check_enabled(&headers)
-    .await
-    .status_code(StatusCode::UNAUTHORIZED)?;
+  let fallback = req
+    .extensions()
+    .get::<ConnectInfo<SocketAddr>>()
+    .map(|addr| addr.ip());
+  let mut user = authenticate_check_enabled(&headers)
+    .map_err(|e| e.status_code(StatusCode::UNAUTHORIZED))
+    .with_failure_rate_limit_using_headers(
+      auth_rate_limiter(),
+      &headers,
+      fallback,
+    )
+    .await?;
+  // Sanitize the user for safety before
+  // attaching to the request handlers.
+  user.sanitize();
   req.extensions_mut().insert(user);
   Ok(next.run(req).await)
 }
@@ -53,23 +103,21 @@ pub async fn get_user_id_from_headers(
   ) {
     (Some(jwt), _, _) => {
       // USE JWT
-      let jwt = jwt.to_str().context("jwt is not str")?;
-      auth_jwt_get_user_id(jwt)
-        .await
-        .context("failed to authenticate jwt")
+      let jwt = jwt.to_str().context("JWT is not valid UTF-8")?;
+      auth_jwt_get_user_id(jwt).await
     }
     (None, Some(key), Some(secret)) => {
       // USE API KEY / SECRET
-      let key = key.to_str().context("key is not str")?;
-      let secret = secret.to_str().context("secret is not str")?;
-      auth_api_key_get_user_id(key, secret)
-        .await
-        .context("failed to authenticate api key")
+      let key =
+        key.to_str().context("X-API-KEY is not valid UTF-8")?;
+      let secret =
+        secret.to_str().context("X-API-SECRET is not valid UTF-8")?;
+      auth_api_key_get_user_id(key, secret).await
     }
     _ => {
       // AUTH FAIL
       Err(anyhow!(
-        "must attach either AUTHORIZATION header with jwt OR pass X-API-KEY and X-API-SECRET"
+        "Must attach either AUTHORIZATION header with jwt OR pass X-API-KEY and X-API-SECRET"
       ))
     }
   }
@@ -79,22 +127,30 @@ pub async fn authenticate_check_enabled(
   headers: &HeaderMap,
 ) -> anyhow::Result<User> {
   let user_id = get_user_id_from_headers(headers).await?;
-  let user = get_user(&user_id).await?;
+  let user = get_user(&user_id)
+    .await
+    .map_err(|_| anyhow!("Invalid user credentials"))?;
   if user.enabled {
     Ok(user)
   } else {
-    Err(anyhow!("user not enabled"))
+    Err(anyhow!("Invalid user credentials"))
   }
 }
 
 pub async fn auth_jwt_get_user_id(
   jwt: &str,
 ) -> anyhow::Result<String> {
-  let claims: JwtClaims = jwt_client().decode(jwt)?;
-  if claims.exp > unix_timestamp_ms() {
+  let claims: JwtClaims = jwt_client()
+    .decode(jwt)
+    .map_err(|_| anyhow!("Invalid user credentials"))?;
+  // Apply clock skew tolerance.
+  // Token is valid if expiration is greater than (now - tolerance)
+  if claims.exp
+    > unix_timestamp_ms().saturating_sub(JWT_CLOCK_SKEW_TOLERANCE_MS)
+  {
     Ok(claims.id)
   } else {
-    Err(anyhow!("token has expired"))
+    Err(anyhow!("Invalid user credentials"))
   }
 }
 
@@ -113,19 +169,25 @@ pub async fn auth_api_key_get_user_id(
     .api_keys
     .find_one(doc! { "key": key })
     .await
-    .context("failed to query db")?
-    .context("no api key matching key")?;
-  if key.expires != 0 && key.expires < komodo_timestamp() {
-    return Err(anyhow!("api key expired"));
+    .context("Failed to query db")?
+    .context("Invalid user credentials")?;
+  // Apply clock skew tolerance.
+  // Token is invalid if expiration is less than (now - tolerance)
+  if key.expires != 0
+    && key.expires
+      < komodo_timestamp()
+        .saturating_sub(API_KEY_CLOCK_SKEW_TOLERANCE_MS)
+  {
+    return Err(anyhow!("Invalid user credentials"));
   }
   if bcrypt::verify(secret, &key.secret)
-    .context("failed to verify secret hash")?
+    .map_err(|_| anyhow!("Invalid user credentials"))?
   {
     // secret matches
     Ok(key.user_id)
   } else {
     // secret mismatch
-    Err(anyhow!("invalid api secret"))
+    Err(anyhow!("Invalid user credentials"))
   }
 }
 
@@ -142,6 +204,18 @@ async fn check_enabled(user_id: String) -> anyhow::Result<User> {
   if user.enabled {
     Ok(user)
   } else {
-    Err(anyhow!("user not enabled"))
+    Err(anyhow!("Invalid user credentials"))
   }
 }
+
+fn format_redirect(redirect: Option<&str>, extra: &str) -> Redirect {
+  let redirect_url = if let Some(redirect) = redirect
+    && !redirect.is_empty()
+  {
+    let splitter = if redirect.contains('?') { '&' } else { '?' };
+    format!("{redirect}{splitter}{extra}")
+  } else {
+    format!("{}?{extra}", core_config().host)
+  };
+  Redirect::to(&redirect_url)
+}

bin/core/src/auth/oidc/client.rs

@@ -1,16 +1,33 @@
 use std::{sync::OnceLock, time::Duration};
 
-use anyhow::Context;
+use anyhow::{Context, anyhow};
 use arc_swap::ArcSwapOption;
 use openidconnect::{
-  Client, ClientId, ClientSecret, EmptyAdditionalClaims,
-  EndpointMaybeSet, EndpointNotSet, EndpointSet, IssuerUrl,
-  RedirectUrl, StandardErrorResponse, core::*,
+  AccessToken, AccessTokenHash, AuthorizationCode, Client, ClientId,
+  ClientSecret, CsrfToken, EmptyAdditionalClaims, EndpointMaybeSet,
+  EndpointNotSet, EndpointSet, IssuerUrl, Nonce, OAuth2TokenResponse,
+  PkceCodeChallenge, PkceCodeVerifier, RedirectUrl, Scope,
+  StandardErrorResponse, SubjectIdentifier, TokenResponse as _,
+  UserInfoClaims, core::*,
 };
 
 use crate::config::core_config;
 
-type OidcClient = Client<
+static APP_USER_AGENT: &str =
+  concat!("Komodo/", env!("CARGO_PKG_VERSION"),);
+
+fn oidc_reqwest_client() -> &'static reqwest::Client {
+  static REQWEST: OnceLock<reqwest::Client> = OnceLock::new();
+  REQWEST.get_or_init(|| {
+    reqwest::Client::builder()
+      .redirect(reqwest::redirect::Policy::none())
+      .user_agent(APP_USER_AGENT)
+      .build()
+      .expect("Invalid OIDC reqwest client")
+  })
+}
+
+pub type InnerOidcClient = Client<
   EmptyAdditionalClaims,
   CoreAuthDisplay,
   CoreGenderClaim,
@@ -30,6 +47,8 @@ type OidcClient = Client<
   EndpointMaybeSet,
 >;
 
+pub struct OidcClient(InnerOidcClient);
+
 pub fn oidc_client() -> &'static ArcSwapOption<OidcClient> {
   static OIDC_CLIENT: OnceLock<ArcSwapOption<OidcClient>> =
     OnceLock::new();
@@ -66,7 +85,7 @@ async fn reset_oidc_client() -> anyhow::Result<()> {
   // Use OpenID Connect Discovery to fetch the provider metadata.
   let provider_metadata = CoreProviderMetadata::discover_async(
     IssuerUrl::new(config.oidc_provider.clone())?,
-    super::reqwest_client(),
+    oidc_reqwest_client(),
   )
   .await
   .context("Failed to get OIDC /.well-known/openid-configuration")?;
@@ -87,7 +106,112 @@ async fn reset_oidc_client() -> anyhow::Result<()> {
     core_config().host
   ))?);
 
-  oidc_client().store(Some(client.into()));
+  oidc_client().store(Some(OidcClient(client).into()));
 
   Ok(())
 }
+
+impl OidcClient {
+  pub fn authorize_url(
+    &self,
+    pkce_challenge: PkceCodeChallenge,
+  ) -> (url::Url, CsrfToken, Nonce) {
+    self
+      .0
+      .authorize_url(
+        CoreAuthenticationFlow::AuthorizationCode,
+        CsrfToken::new_random,
+        Nonce::new_random,
+      )
+      .set_pkce_challenge(pkce_challenge)
+      .add_scope(Scope::new("openid".to_string()))
+      .add_scope(Scope::new("profile".to_string()))
+      .add_scope(Scope::new("email".to_string()))
+      .url()
+  }
+
+  /// Applied security validations and extracts the
+  /// oidc user id.
+  pub async fn validate_extract_subject_and_token(
+    &self,
+    (client, server): (CsrfToken, String),
+    code: String,
+    pkce_verifier: PkceCodeVerifier,
+    nonce: Nonce,
+  ) -> anyhow::Result<(SubjectIdentifier, AccessToken)> {
+    // Validate CSRF tokens match
+    if client.secret() != &server {
+      return Err(anyhow!("CSRF token invalid"));
+    }
+
+    let reqwest_client = oidc_reqwest_client();
+    let token_response = self
+      .0
+      .exchange_code(AuthorizationCode::new(code))
+      .context("Failed to get Oauth token at exchange code")?
+      .set_pkce_verifier(pkce_verifier)
+      .request_async(reqwest_client)
+      .await
+      .context("Failed to get Oauth token")?;
+
+    // Extract the ID token claims after verifying its authenticity and nonce.
+    let id_token = token_response
+      .id_token()
+      .context("OIDC Server did not return an ID token")?;
+
+    // Some providers attach additional audiences, they must be added here
+    // so token verification succeeds.
+    let verifier = self.0.id_token_verifier();
+    let additional_audiences =
+      &core_config().oidc_additional_audiences;
+    let verifier = if additional_audiences.is_empty() {
+      verifier
+    } else {
+      verifier.set_other_audience_verifier_fn(|aud| {
+        additional_audiences.contains(aud)
+      })
+    };
+
+    let claims = id_token
+      .claims(&verifier, &nonce)
+      .context("Failed to verify token claims. This issue may be temporary (60 seconds max).")?;
+
+    // Verify the access token hash to ensure that the access token hasn't been substituted for
+    // another user's.
+    if let Some(expected_access_token_hash) =
+      claims.access_token_hash()
+    {
+      let actual_access_token_hash = AccessTokenHash::from_token(
+        &token_response.access_token().clone(),
+        id_token.signing_alg()?,
+        id_token.signing_key(&verifier)?,
+      )?;
+      if actual_access_token_hash != *expected_access_token_hash {
+        return Err(anyhow!("Invalid access token"));
+      }
+    }
+
+    Ok((
+      claims.subject().clone(),
+      token_response.access_token().clone(),
+    ))
+  }
+
+  pub async fn fetch_user_info(
+    &self,
+    access_token: AccessToken,
+    subject: SubjectIdentifier,
+  ) -> anyhow::Result<
+    UserInfoClaims<EmptyAdditionalClaims, CoreGenderClaim>,
+  > {
+    self
+      .0
+      .user_info(access_token, Some(subject))
+      .context("Invalid user info request")?
+      .request_async::<EmptyAdditionalClaims, _, CoreGenderClaim>(
+        oidc_reqwest_client(),
+      )
+      .await
+      .context("Failed to fetch OIDC user info")
+  }
+}

bin/core/src/auth/oidc/mod.rs

@@ -1,80 +1,102 @@
-use std::sync::OnceLock;
+use std::net::SocketAddr;
 
 use anyhow::{Context, anyhow};
 use axum::{
-  Router, extract::Query, response::Redirect, routing::get,
+  Router,
+  extract::{ConnectInfo, Query},
+  http::HeaderMap,
+  response::Redirect,
+  routing::get,
 };
 use client::oidc_client;
-use dashmap::DashMap;
-use database::mungos::mongodb::bson::{Document, doc};
-use komodo_client::entities::{
-  komodo_timestamp, random_string,
-  user::{User, UserConfig},
+use database::mungos::{
+  by_id::update_one_by_id,
+  mongodb::bson::{Document, doc},
+};
+use futures_util::TryFutureExt;
+use komodo_client::{
+  api::auth::UserIdOrTwoFactor,
+  entities::{
+    komodo_timestamp, random_string,
+    user::{User, UserConfig},
+  },
 };
 use openidconnect::{
-  AccessTokenHash, AuthorizationCode, CsrfToken,
-  EmptyAdditionalClaims, Nonce, OAuth2TokenResponse,
-  PkceCodeChallenge, PkceCodeVerifier, Scope, TokenResponse,
-  core::{CoreAuthenticationFlow, CoreGenderClaim},
+  CsrfToken, Nonce, PkceCodeChallenge, PkceCodeVerifier,
 };
+use rate_limit::WithFailureRateLimit;
 use reqwest::StatusCode;
-use serde::Deserialize;
-use serror::AddStatusCode;
+use serde::{Deserialize, Serialize};
+use serror::{AddStatusCode as _, AddStatusCodeError};
+use tower_sessions::Session;
 
 use crate::{
+  api::user::SessionThirdPartyLinkInfo,
+  auth::{
+    SessionPasskeyLogin, SessionTotpLogin, SessionUserId,
+    format_redirect, oidc::client::OidcClient,
+  },
   config::core_config,
-  state::{db_client, jwt_client},
+  helpers::query::{find_oidc_user, get_user},
+  state::{auth_rate_limiter, db_client, webauthn},
 };
 
 use super::RedirectQuery;
 
 pub mod client;
 
-static APP_USER_AGENT: &str =
-  concat!("Komodo/", env!("CARGO_PKG_VERSION"),);
-
-fn reqwest_client() -> &'static reqwest::Client {
-  static REQWEST: OnceLock<reqwest::Client> = OnceLock::new();
-  REQWEST.get_or_init(|| {
-    reqwest::Client::builder()
-      .redirect(reqwest::redirect::Policy::none())
-      .user_agent(APP_USER_AGENT)
-      .build()
-      .expect("Invalid OIDC reqwest client")
-  })
-}
-
-/// CSRF tokens can only be used once from the callback,
-/// and must be used within this timeframe
-const CSRF_VALID_FOR_MS: i64 = 120_000; // 2 minutes for user to log in.
-
-type RedirectUrl = Option<String>;
-/// Maps the csrf secrets to other information added in the "login" method (before auth provider redirect).
-/// This information is retrieved in the "callback" method (after auth provider redirect).
-type VerifierMap =
-  DashMap<String, (PkceCodeVerifier, Nonce, RedirectUrl, i64)>;
-fn verifier_tokens() -> &'static VerifierMap {
-  static VERIFIERS: OnceLock<VerifierMap> = OnceLock::new();
-  VERIFIERS.get_or_init(Default::default)
-}
-
 pub fn router() -> Router {
   Router::new()
     .route(
       "/login",
-      get(|query| async {
-        login(query).await.status_code(StatusCode::UNAUTHORIZED)
+      get(|session, query| async {
+        login(session, query)
+          .await
+          .status_code(StatusCode::UNAUTHORIZED)
+      }),
+    )
+    .route(
+      "/link",
+      get(|session| async {
+        link(session).await.status_code(StatusCode::UNAUTHORIZED)
       }),
     )
     .route(
       "/callback",
-      get(|query| async {
-        callback(query).await.status_code(StatusCode::UNAUTHORIZED)
-      }),
+      get(
+        |query,
+         session,
+         headers: HeaderMap,
+         ConnectInfo(info): ConnectInfo<SocketAddr>| async move {
+          callback(session, query)
+            .map_err(|e| e.status_code(StatusCode::UNAUTHORIZED))
+            .with_failure_rate_limit_using_headers(
+              auth_rate_limiter(),
+              &headers,
+              Some(info.ip()),
+            )
+            .await
+        },
+      ),
     )
 }
 
+type RedirectUrl = Option<String>;
+
+#[derive(Serialize, Deserialize)]
+struct SessionOidcVerificationInfo {
+  csrf_token: String,
+  pkce_verifier: PkceCodeVerifier,
+  nonce: Nonce,
+  redirect: RedirectUrl,
+}
+
+impl SessionOidcVerificationInfo {
+  const KEY: &str = "oidc-verification-info";
+}
+
 async fn login(
+  session: Session,
   Query(RedirectQuery { redirect }): Query<RedirectQuery>,
 ) -> anyhow::Result<Redirect> {
   let client = oidc_client().load();
@@ -85,32 +107,76 @@ async fn login(
     PkceCodeChallenge::new_random_sha256();
 
   // Generate the authorization URL.
-  let (auth_url, csrf_token, nonce) = client
-    .authorize_url(
-      CoreAuthenticationFlow::AuthorizationCode,
-      CsrfToken::new_random,
-      Nonce::new_random,
-    )
-    .set_pkce_challenge(pkce_challenge)
-    .add_scope(Scope::new("openid".to_string()))
-    .add_scope(Scope::new("profile".to_string()))
-    .add_scope(Scope::new("email".to_string()))
-    .url();
+  let (auth_url, csrf_token, nonce) =
+    client.authorize_url(pkce_challenge);
 
   // Data inserted here will be matched on callback side for csrf protection.
-  verifier_tokens().insert(
-    csrf_token.secret().clone(),
-    (
-      pkce_verifier,
-      nonce,
-      redirect,
-      komodo_timestamp() + CSRF_VALID_FOR_MS,
-    ),
-  );
+  session
+    .insert(
+      SessionOidcVerificationInfo::KEY,
+      SessionOidcVerificationInfo {
+        csrf_token: csrf_token.secret().clone(),
+        pkce_verifier,
+        nonce,
+        redirect,
+      },
+    )
+    .await
+    .context("Failed to insert session verification info")?;
 
+  auth_redirect(auth_url.as_str())
+}
+
+#[derive(Serialize, Deserialize)]
+struct SessionOidcLinkInfo {
+  user_id: String,
+  csrf_token: String,
+  pkce_verifier: PkceCodeVerifier,
+  nonce: Nonce,
+}
+
+impl SessionOidcLinkInfo {
+  const KEY: &str = "oidc-link-info";
+}
+
+async fn link(session: Session) -> anyhow::Result<Redirect> {
+  let SessionThirdPartyLinkInfo { user_id } = session
+    .remove(SessionThirdPartyLinkInfo::KEY)
+    .await
+    .context("Invalid session third party link info.")?
+    .context("Missing session third party link info")?;
+
+  let client = oidc_client().load();
+  let client =
+    client.as_ref().context("OIDC Client not configured")?;
+
+  let (pkce_challenge, pkce_verifier) =
+    PkceCodeChallenge::new_random_sha256();
+
+  // Generate the authorization URL.
+  let (auth_url, csrf_token, nonce) =
+    client.authorize_url(pkce_challenge);
+
+  session
+    .insert(
+      SessionOidcLinkInfo::KEY,
+      SessionOidcLinkInfo {
+        user_id,
+        csrf_token: csrf_token.secret().clone(),
+        pkce_verifier,
+        nonce,
+      },
+    )
+    .await
+    .context("Failed to insert session link info")?;
+
+  auth_redirect(auth_url.as_str())
+}
+
+/// Applies 'oidc_redirect_host'
+fn auth_redirect(auth_url: &str) -> anyhow::Result<Redirect> {
   let config = core_config();
   let redirect = if !config.oidc_redirect_host.is_empty() {
-    let auth_url = auth_url.as_str();
     let (protocol, rest) = auth_url
       .split_once("://")
       .context("Invalid URL: Missing protocol (eg 'https://')")?;
@@ -123,21 +189,21 @@ async fn login(
       &config.oidc_redirect_host,
     ))
   } else {
-    Redirect::to(auth_url.as_str())
+    Redirect::to(auth_url)
   };
-
   Ok(redirect)
 }
 
 #[derive(Debug, Deserialize)]
-struct CallbackQuery {
+pub struct OidcCallbackQuery {
   state: Option<String>,
   code: Option<String>,
   error: Option<String>,
 }
 
 async fn callback(
-  Query(query): Query<CallbackQuery>,
+  session: Session,
+  Query(query): Query<OidcCallbackQuery>,
 ) -> anyhow::Result<Redirect> {
   let client = oidc_client().load();
   let client =
@@ -152,79 +218,93 @@ async fn callback(
     query.state.context("Provider did not return state")?,
   );
 
-  let (_, (pkce_verifier, nonce, redirect, valid_until)) =
-    verifier_tokens()
-      .remove(state.secret())
-      .context("CSRF token invalid")?;
-
-  if komodo_timestamp() > valid_until {
-    return Err(anyhow!(
-      "CSRF token invalid (Timed out). The token must be used within 2 minutes."
-    ));
-  }
-
-  let reqwest_client = reqwest_client();
-  let token_response = client
-    .exchange_code(AuthorizationCode::new(code))
-    .context("Failed to get Oauth token at exchange code")?
-    .set_pkce_verifier(pkce_verifier)
-    .request_async(reqwest_client)
-    .await
-    .context("Failed to get Oauth token")?;
-
-  // Extract the ID token claims after verifying its authenticity and nonce.
-  let id_token = token_response
-    .id_token()
-    .context("OIDC Server did not return an ID token")?;
-
-  // Some providers attach additional audiences, they must be added here
-  // so token verification succeeds.
-  let verifier = client.id_token_verifier();
-  let additional_audiences = &core_config().oidc_additional_audiences;
-  let verifier = if additional_audiences.is_empty() {
-    verifier
-  } else {
-    verifier.set_other_audience_verifier_fn(|aud| {
-      additional_audiences.contains(aud)
-    })
-  };
-
-  let claims = id_token
-    .claims(&verifier, &nonce)
-    .context("Failed to verify token claims. This issue may be temporary (60 seconds max).")?;
-
-  // Verify the access token hash to ensure that the access token hasn't been substituted for
-  // another user's.
-  if let Some(expected_access_token_hash) = claims.access_token_hash()
+  // Check first if this is a link callback
+  // and use the linking handler if so.
+  if let Ok(Some(info)) =
+    session.remove(SessionOidcLinkInfo::KEY).await
   {
-    let actual_access_token_hash = AccessTokenHash::from_token(
-      token_response.access_token(),
-      id_token.signing_alg()?,
-      id_token.signing_key(&verifier)?,
-    )?;
-    if actual_access_token_hash != *expected_access_token_hash {
-      return Err(anyhow!("Invalid access token"));
-    }
+    return link_oidc_callback(client, info, state, code).await;
   }
 
-  let user_id = claims.subject().as_str();
-
-  let db_client = db_client();
-  let user = db_client
-    .users
-    .find_one(doc! {
-      "config.data.provider": &core_config().oidc_provider,
-      "config.data.user_id": user_id
-    })
+  let SessionOidcVerificationInfo {
+    csrf_token,
+    pkce_verifier,
+    nonce,
+    redirect,
+  } = session
+    .remove(SessionOidcVerificationInfo::KEY)
     .await
-    .context("failed at find user query from database")?;
+    .context("Invalid session verification info.")?
+    .context(
+      "Missing session verification info for CSRF protection.",
+    )?;
 
-  let jwt = match user {
-    Some(user) => jwt_client()
-      .encode(user.id)
-      .context("failed to generate jwt")?,
+  let (subject, token) = client
+    .validate_extract_subject_and_token(
+      (state, csrf_token),
+      code,
+      pkce_verifier,
+      nonce,
+    )
+    .await?;
+
+  let user = find_oidc_user(&subject).await?;
+
+  let user_id_or_two_factor = match user {
+    Some(user) => {
+      match (
+        user.third_party_skip_2fa,
+        user.passkey.passkey,
+        user.totp.enrolled(),
+      ) {
+        // Skip / No 2FA
+        (true, _, _) | (false, None, false) => {
+          session
+            .insert(
+              SessionUserId::KEY,
+              SessionUserId(user.id.clone()),
+            )
+            .await
+            .context("Failed to store user id for client session")?;
+          UserIdOrTwoFactor::UserId(user.id)
+        }
+        // WebAuthn Passkey 2FA
+        (false, Some(passkey), _) => {
+          let webauthn = webauthn().context(
+            "No webauthn provider available, invalid KOMODO_HOST config",
+          )?;
+          let (response, server_state) = webauthn
+            .start_passkey_authentication(&[passkey])
+            .context("Failed to start passkey authentication flow")?;
+          session
+            .insert(
+              SessionPasskeyLogin::KEY,
+              SessionPasskeyLogin {
+                user_id: user.id,
+                state: server_state,
+              },
+            )
+            .await?;
+          UserIdOrTwoFactor::Passkey(response)
+        }
+        // TOTP 2FA
+        (false, None, true) => {
+          session
+            .insert(
+              SessionTotpLogin::KEY,
+              SessionTotpLogin { user_id: user.id },
+            )
+            .await
+            .context(
+              "Failed to store totp login state in for user session",
+            )?;
+          UserIdOrTwoFactor::Totp {}
+        }
+      }
+    }
     None => {
       let ts = komodo_timestamp();
+      let db_client = db_client();
       let no_users_exist =
         db_client.users.find_one(Document::new()).await?.is_none();
       let core_config = core_config();
@@ -233,17 +313,8 @@ async fn callback(
       }
 
       // Fetch user info
-      let user_info = client
-        .user_info(
-          token_response.access_token().clone(),
-          claims.subject().clone().into(),
-        )
-        .context("Invalid user info request")?
-        .request_async::<EmptyAdditionalClaims, _, CoreGenderClaim>(
-          reqwest_client,
-        )
-        .await
-        .context("Failed to fetch user info for new user")?;
+      let user_info =
+        client.fetch_user_info(token, subject.clone()).await?;
 
       // Will use preferred_username, then email, then user_id if it isn't available.
       let mut username = user_info
@@ -253,7 +324,7 @@ async fn callback(
           let email = user_info
             .email()
             .map(|email| email.as_str())
-            .unwrap_or(user_id);
+            .unwrap_or(subject.as_str());
           if core_config.oidc_use_full_email {
             email
           } else {
@@ -291,8 +362,12 @@ async fn callback(
         all: Default::default(),
         config: UserConfig::Oidc {
           provider: core_config.oidc_provider.clone(),
-          user_id: user_id.to_string(),
+          user_id: subject.to_string(),
         },
+        linked_logins: Default::default(),
+        totp: Default::default(),
+        passkey: Default::default(),
+        third_party_skip_2fa: Default::default(),
       };
 
       let user_id = db_client
@@ -305,17 +380,88 @@ async fn callback(
         .context("inserted_id is not ObjectId")?
         .to_string();
 
-      jwt_client()
-        .encode(user_id)
-        .context("failed to generate jwt")?
+      UserIdOrTwoFactor::UserId(user_id)
     }
   };
-  let exchange_token = jwt_client().create_exchange_token(jwt).await;
-  let redirect_url = if let Some(redirect) = redirect {
-    let splitter = if redirect.contains('?') { '&' } else { '?' };
-    format!("{redirect}{splitter}token={exchange_token}")
-  } else {
-    format!("{}?token={exchange_token}", core_config().host)
+
+  match user_id_or_two_factor {
+    UserIdOrTwoFactor::UserId(_) => {
+      Ok(format_redirect(redirect.as_deref(), "redeem_ready=true"))
+    }
+    UserIdOrTwoFactor::Totp {} => {
+      Ok(format_redirect(redirect.as_deref(), "totp=true"))
+    }
+    UserIdOrTwoFactor::Passkey(passkey) => {
+      let passkey = serde_json::to_string(&passkey)
+        .context("Failed to serialize passkey response")?;
+      let passkey = urlencoding::encode(&passkey);
+      Ok(format_redirect(
+        redirect.as_deref(),
+        &format!("passkey={passkey}"),
+      ))
+    }
+  }
+}
+
+/// This intercepts during the normal oauth callback if
+/// 'oidc-link-info' is found on session.
+async fn link_oidc_callback(
+  client: &OidcClient,
+  SessionOidcLinkInfo {
+    user_id,
+    csrf_token,
+    pkce_verifier,
+    nonce,
+  }: SessionOidcLinkInfo,
+  state: CsrfToken,
+  code: String,
+) -> anyhow::Result<Redirect> {
+  let (subject, _) = client
+    .validate_extract_subject_and_token(
+      (state, csrf_token),
+      code,
+      pkce_verifier,
+      nonce,
+    )
+    .await?;
+
+  let oidc_provider = &core_config().oidc_provider;
+  let oidc_user_id = subject.as_str();
+
+  // Ensure there are no other existing users with this login linked.
+  if let Some(existing_user) = find_oidc_user(&subject).await? {
+    if existing_user.id == user_id {
+      // Link is already complete, this is a no-op
+      return Ok(Redirect::to(&format!(
+        "{}/settings",
+        core_config().host,
+      )));
+    } else {
+      return Err(anyhow!("Account already linked to another user."));
+    }
+  }
+
+  let user = get_user(&user_id).await?;
+
+  if let UserConfig::Oidc { .. } = &user.config {
+    return Err(anyhow!(
+      "User is primary Oidc user, cannot link another Oidc login."
+    ));
+  }
+
+  let update = doc! {
+    "$set": {
+      "linked_logins.Oidc.type": "Oidc",
+      "linked_logins.Oidc.data.provider": oidc_provider,
+      "linked_logins.Oidc.data.user_id": oidc_user_id,
+    }
   };
-  Ok(Redirect::to(&redirect_url))
+
+  update_one_by_id(&db_client().users, &user_id, update, None)
+    .await
+    .context(
+      "Failed to link OIDC login to existing user on database",
+    )?;
+
+  Ok(Redirect::to(&format!("{}/settings", core_config().host,)))
 }

bin/core/src/auth/totp.rs

@@ -0,0 +1,17 @@
+use anyhow::Context as _;
+
+pub fn make_totp(
+  secret_bytes: Vec<u8>,
+  account_name: impl Into<Option<String>>,
+) -> anyhow::Result<totp_rs::TOTP> {
+  totp_rs::TOTP::new(
+    totp_rs::Algorithm::SHA1,
+    6,
+    1,
+    30,
+    secret_bytes,
+    Some(String::from("Komodo")),
+    account_name.into().unwrap_or_default(),
+  )
+  .context("Failed to construct TOTP")
+}

bin/core/src/cloud/aws/ec2.rs

@@ -11,7 +11,7 @@ use aws_sdk_ec2::{
     Tag, TagSpecification,
   },
 };
-use base64::Engine;
+use data_encoding::BASE64_NOPAD;
 use komodo_client::entities::{
   ResourceTarget,
   alert::{Alert, AlertData, SeverityLevel},
@@ -127,10 +127,7 @@ pub async fn launch_ec2_instance(
     )
     .min_count(1)
     .max_count(1)
-    .user_data(
-      base64::engine::general_purpose::STANDARD_NO_PAD
-        .encode(user_data),
-    );
+    .user_data(BASE64_NOPAD.encode(user_data.as_bytes()));
 
   let res = req
     .send()

bin/core/src/config.rs

@@ -1,6 +1,7 @@
 use std::{path::PathBuf, sync::OnceLock};
 
 use anyhow::Context;
+use axum::http::HeaderValue;
 use colored::Colorize;
 use config::ConfigLoader;
 use environment_file::{
@@ -14,6 +15,7 @@ use komodo_client::entities::{
   logger::LogConfig,
 };
 use noise::key::{RotatableKeyPair, SpkiPublicKey};
+use tower_http::cors::CorsLayer;
 
 /// Should call in startup to ensure Core errors without valid private key.
 pub fn core_keys() -> &'static RotatableKeyPair {
@@ -89,6 +91,70 @@ pub fn periphery_public_keys() -> Option<&'static [SpkiPublicKey]> {
     .as_deref()
 }
 
+/// Creates a CORS layer based on the Core configuration.
+///
+/// - If `cors_allowed_origins` is empty: Allows all origins (backward compatibility)
+/// - If `cors_allowed_origins` is set: Only allows the specified origins
+/// - Methods and headers are always allowed (Any)
+/// - Credentials are only allowed if `cors_allow_credentials` is true
+pub fn cors_layer() -> CorsLayer {
+  let config = core_config();
+  let mut cors = CorsLayer::new()
+    .allow_methods(tower_http::cors::AllowMethods::mirror_request())
+    .allow_headers(tower_http::cors::AllowHeaders::mirror_request())
+    .allow_credentials(config.cors_allow_credentials);
+  if config.cors_allowed_origins.is_empty() {
+    warn!(
+      "CORS using allowed origin 'Any' (*). Use KOMODO_CORS_ALLOWED_ORIGINS to configure specific origins."
+    );
+    cors = cors.allow_origin(tower_http::cors::Any)
+  } else {
+    let allowed_origins = config
+      .cors_allowed_origins
+      .iter()
+      .filter_map(|origin| {
+        HeaderValue::from_str(origin)
+          .inspect_err(|e| {
+            warn!("Invalid CORS allowed origin: {origin} | {e:?}")
+          })
+          .ok()
+      })
+      .collect::<Vec<_>>();
+    info!("CORS using allowed origin/s: {allowed_origins:?}");
+    cors = cors.allow_origin(allowed_origins);
+  };
+  cors
+}
+
+pub fn monitoring_interval() -> async_timing_util::Timelength {
+  static MONITORING_INTERVAL: OnceLock<
+    async_timing_util::Timelength,
+  > = OnceLock::new();
+  *MONITORING_INTERVAL.get_or_init(|| {
+    core_config().monitoring_interval.try_into().unwrap_or_else(
+      |_| {
+        error!("Invalid 'monitoring_interval', using default 15-sec");
+        async_timing_util::Timelength::FifteenSeconds
+      },
+    )
+  })
+}
+
+pub fn core_host() -> Option<&'static url::Url> {
+  static CORE_URL: OnceLock<Option<url::Url>> = OnceLock::new();
+  CORE_URL
+    .get_or_init(|| {
+      url::Url::parse(&core_config().host)
+      .inspect_err(|e| {
+        warn!(
+          "Invalid KOMODO_HOST: not URL. Passkeys won't work. | {e:?}"
+        )
+      })
+      .ok()
+    })
+    .as_ref()
+}
+
 pub fn core_config() -> &'static CoreConfig {
   static CORE_CONFIG: OnceLock<CoreConfig> = OnceLock::new();
   CORE_CONFIG.get_or_init(|| {
@@ -281,6 +347,21 @@ pub fn core_config() -> &'static CoreConfig {
         .komodo_frontend_path
         .unwrap_or(config.frontend_path),
       jwt_ttl: env.komodo_jwt_ttl.unwrap_or(config.jwt_ttl),
+      auth_rate_limit_disabled: env
+        .komodo_auth_rate_limit_disabled
+        .unwrap_or(config.auth_rate_limit_disabled),
+      auth_rate_limit_max_attempts: env
+        .komodo_auth_rate_limit_max_attempts
+        .unwrap_or(config.auth_rate_limit_max_attempts),
+      auth_rate_limit_window_seconds: env
+        .komodo_auth_rate_limit_window_seconds
+        .unwrap_or(config.auth_rate_limit_window_seconds),
+      cors_allowed_origins: env
+        .komodo_cors_allowed_origins
+        .unwrap_or(config.cors_allowed_origins),
+      cors_allow_credentials: env
+        .komodo_cors_allow_credentials
+        .unwrap_or(config.cors_allow_credentials),
       sync_directory: env
         .komodo_sync_directory
         .unwrap_or(config.sync_directory),
@@ -336,6 +417,9 @@ pub fn core_config() -> &'static CoreConfig {
         .komodo_lock_login_credentials_for
         .unwrap_or(config.lock_login_credentials_for),
       local_auth: env.komodo_local_auth.unwrap_or(config.local_auth),
+      min_password_length: env
+        .komodo_min_password_length
+        .unwrap_or(config.min_password_length),
       logging: LogConfig {
         level: env
           .komodo_logging_level

bin/core/src/connection/client.rs

@@ -45,7 +45,6 @@ impl PeripheryConnectionArgs<'_> {
       periphery_connections().insert(id.clone(), self).await;
 
     let responses = connection.responses.clone();
-    let terminals = connection.terminals.clone();
 
     tokio::spawn(async move {
       loop {
@@ -91,11 +90,7 @@ impl PeripheryConnectionArgs<'_> {
       }
     });
 
-    Ok(PeripheryClient {
-      id,
-      responses,
-      terminals,
-    })
+    Ok(PeripheryClient { id, responses })
   }
 }
 

bin/core/src/connection/mod.rs

@@ -31,7 +31,7 @@ use transport::{
   },
   channel::{BufferedReceiver, Sender, buffered_channel},
   websocket::{
-    Websocket, WebsocketMessage, WebsocketReceiver as _,
+    Websocket, WebsocketReceiver as _, WebsocketReceiverExt,
     WebsocketSender as _,
   },
 };
@@ -257,7 +257,8 @@ impl<'a> From<&'a OwnedPeripheryConnectionArgs>
 pub type ResponseChannels =
   CloneCache<Uuid, Sender<EncodedResponse<EncodedJsonMessage>>>;
 
-pub type TerminalChannels = CloneCache<Uuid, Sender<Vec<u8>>>;
+pub type TerminalChannels =
+  CloneCache<Uuid, Sender<anyhow::Result<Vec<u8>>>>;
 
 #[derive(Debug)]
 pub struct PeripheryConnection {
@@ -366,8 +367,22 @@ impl PeripheryConnection {
 
     let forward_writes = async {
       loop {
-        let Ok(message) = receiver.recv().await else {
-          break;
+        let message = match tokio::time::timeout(
+          Duration::from_secs(5),
+          receiver.recv(),
+        )
+        .await
+        {
+          Ok(Ok(message)) => message,
+          Ok(Err(_)) => break,
+          // Handle sending Ping
+          Err(_) => {
+            if let Err(e) = ws_write.ping().await {
+              self.set_error(e).await;
+              break;
+            }
+            continue;
+          }
         };
         match ws_write.send(message.into_bytes()).await {
           Ok(_) => receiver.clear_buffer(),
@@ -384,19 +399,13 @@ impl PeripheryConnection {
 
     let handle_reads = async {
       loop {
-        match ws_read.recv().await {
-          Ok(WebsocketMessage::Message(message)) => {
-            self.handle_incoming_message(message).await
-          }
-          Ok(WebsocketMessage::Close(_))
-          | Ok(WebsocketMessage::Closed) => {
-            self.set_error(anyhow!("Connection closed")).await;
-            break;
-          }
+        match ws_read.recv_message().await {
+          Ok(message) => self.handle_incoming_message(message).await,
           Err(e) => {
             self.set_error(e).await;
+            break;
           }
-        };
+        }
       }
       // Cancel again if not already
       cancel.cancel();
@@ -409,15 +418,8 @@ impl PeripheryConnection {
 
   pub async fn handle_incoming_message(
     &self,
-    message: EncodedTransportMessage,
+    message: TransportMessage,
   ) {
-    let message: TransportMessage = match message.decode() {
-      Ok(res) => res,
-      Err(e) => {
-        warn!("Failed to parse Message bytes | {e:#}");
-        return;
-      }
-    };
     match message {
       TransportMessage::Response(data) => {
         match data.decode().map(ResponseMessage::into_inner) {

bin/core/src/helpers/action_state.rs

@@ -1,3 +1,16 @@
+//! # Action State Management
+//!
+//! This module provides thread-safe state management for resource exections.
+//! It prevents concurrent execution of exections on the same resource using
+//! a Mutex-based locking mechanism with RAII guards.
+//!
+//! ## Safety
+//!
+//! - Uses RAII pattern to ensure locks are always released
+//! - Handles lock poisoning gracefully
+//! - Prevents race conditions through per-resource locks
+//! - No deadlock risk: each resource has independent locks
+
 use std::sync::{Arc, Mutex};
 
 use anyhow::anyhow;
@@ -9,12 +22,13 @@ use komodo_client::{
     deployment::DeploymentActionState,
     procedure::ProcedureActionState, repo::RepoActionState,
     server::ServerActionState, stack::StackActionState,
-    sync::ResourceSyncActionState,
+    swarm::SwarmActionState, sync::ResourceSyncActionState,
   },
 };
 
 #[derive(Default)]
 pub struct ActionStates {
+  pub swarm: CloneCache<String, Arc<ActionState<SwarmActionState>>>,
   pub server: CloneCache<String, Arc<ActionState<ServerActionState>>>,
   pub stack: CloneCache<String, Arc<ActionState<StackActionState>>>,
   pub deployment:
@@ -28,7 +42,16 @@ pub struct ActionStates {
     CloneCache<String, Arc<ActionState<ResourceSyncActionState>>>,
 }
 
-/// Need to be able to check "busy" with write lock acquired.
+/// Thread-safe state container for resource executions.
+///
+/// Uses a Mutex to prevent concurrent executions and provides
+/// RAII-based locking through [UpdateGuard].
+///
+/// # Safety
+///
+/// - Each resource has its own ActionState instance
+/// - State is reset to default when [UpdateGuard] is dropped
+/// - Lock poisoning error handling is handled gracefully with anyhow::Error
 #[derive(Default)]
 pub struct ActionState<States: Default + Send + 'static>(
   Mutex<States>,
@@ -42,7 +65,7 @@ impl<States: Default + Busy + Copy + Send + 'static>
       *self
         .0
         .lock()
-        .map_err(|e| anyhow!("action state lock poisoned | {e:?}"))?,
+        .map_err(|e| anyhow!("Action state lock poisoned | {e:?}"))?,
     )
   }
 
@@ -51,14 +74,33 @@ impl<States: Default + Busy + Copy + Send + 'static>
       self
         .0
         .lock()
-        .map_err(|e| anyhow!("action state lock poisoned | {e:?}"))?
+        .map_err(|e| anyhow!("Action state lock poisoned | {e:?}"))?
         .busy(),
     )
   }
 
-  /// Will acquire lock, check busy, and if not will
-  /// run the provided update function on the states.
-  /// Returns a guard that returns the states to default (not busy) when dropped.
+  /// Acquires lock, checks if resource is busy, and if not,
+  /// runs the provided update function on the states.
+  ///
+  /// Returns an `UpdateGuard` that automatically resets the state
+  /// to default (not busy) when dropped.
+  ///
+  /// # Errors
+  ///
+  /// Returns an error if:
+  /// - The lock is poisoned
+  /// - The resource is currently busy
+  ///
+  /// # Example
+  ///
+  /// ```rust
+  /// let guard = action_state.update(|state| {
+  ///   *state = SomeNewState;
+  /// })?;
+  /// // State is locked and marked as busy
+  /// // ... perform work ...
+  /// drop(guard) // Guard is dropped, state returns to default
+  /// ```
   pub fn update(
     &self,
     update_fn: impl Fn(&mut States),
@@ -91,10 +133,22 @@ impl<States: Default + Busy + Copy + Send + 'static>
   }
 }
 
-/// When dropped will return the inner state to default.
-/// The inner mutex guard must already be dropped BEFORE this is dropped,
-/// which is guaranteed as the inner guard is dropped by all public methods before
-/// user could drop UpdateGuard.
+/// RAII guard that automatically resets the action state when dropped.
+///
+/// # Safety
+///
+/// The inner mutex guard is guaranteed to be dropped before this guard
+/// is dropped, preventing deadlocks. This is ensured by all public methods
+/// that create UpdateGuard instances.
+///
+/// # Behavior
+///
+/// When dropped, this guard will:
+/// 1. Re-acquire the lock
+/// 2. Call the provided return function (typically resetting to default)
+/// 3. Release the lock
+///
+/// If the lock is poisoned, an error is logged but the drop continues.
 pub struct UpdateGuard<'a, States: Default + Send + 'static>(
   &'a Mutex<States>,
   Box<dyn Fn(&mut States) + Send>,

bin/core/src/helpers/all_resources.rs

@@ -3,11 +3,12 @@ use std::collections::HashMap;
 use komodo_client::entities::{
   action::Action, alerter::Alerter, build::Build, builder::Builder,
   deployment::Deployment, procedure::Procedure, repo::Repo,
-  server::Server, stack::Stack, sync::ResourceSync,
+  server::Server, stack::Stack, swarm::Swarm, sync::ResourceSync,
 };
 
 #[derive(Debug, Default)]
 pub struct AllResourcesById {
+  pub swarms: HashMap<String, Swarm>,
   pub servers: HashMap<String, Server>,
   pub deployments: HashMap<String, Deployment>,
   pub stacks: HashMap<String, Stack>,
@@ -27,6 +28,10 @@ impl AllResourcesById {
     let id_to_tags = &map;
     let match_tags = &[];
     Ok(Self {
+      swarms: crate::resource::get_id_to_resource_map::<Swarm>(
+        id_to_tags, match_tags,
+      )
+      .await?,
       servers: crate::resource::get_id_to_resource_map::<Server>(
         id_to_tags, match_tags,
       )

bin/core/src/helpers/mod.rs

@@ -4,6 +4,7 @@ use anyhow::{Context, anyhow};
 use database::mongo_indexed::Document;
 use database::mungos::mongodb::bson::{Bson, doc};
 use indexmap::IndexSet;
+use komodo_client::entities::SwarmOrServer;
 use komodo_client::entities::{
   ResourceTarget,
   build::Build,
@@ -15,7 +16,11 @@ use komodo_client::entities::{
   stack::Stack,
   user::User,
 };
+use resolver_api::HasResponse;
+use serde::Serialize;
+use serde::de::DeserializeOwned;
 
+use crate::helpers::swarm::swarm_request;
 use crate::{
   config::core_config, connection::PeripheryConnectionArgs,
   periphery::PeripheryClient, state::db_client,
@@ -30,9 +35,10 @@ pub mod matcher;
 pub mod procedure;
 pub mod prune;
 pub mod query;
+pub mod swarm;
+pub mod terminal;
 pub mod update;
-
-// pub mod resource;
+pub mod validations;
 
 pub fn empty_or_only_spaces(word: &str) -> bool {
   if word.is_empty() {
@@ -263,3 +269,21 @@ pub fn repo_link(
   }
   res
 }
+
+pub async fn swarm_or_server_request<T>(
+  swarm_or_server: &SwarmOrServer,
+  request: T,
+) -> anyhow::Result<T::Response>
+where
+  T: std::fmt::Debug + Clone + Serialize + HasResponse,
+  T::Response: DeserializeOwned,
+{
+  match swarm_or_server {
+    SwarmOrServer::Swarm(swarm) => {
+      swarm_request(&swarm.config.server_ids, request).await
+    }
+    SwarmOrServer::Server(server) => {
+      periphery_client(server).await?.request(request).await
+    }
+  }
+}

bin/core/src/helpers/procedure.rs

@@ -3,13 +3,14 @@ use std::time::{Duration, Instant};
 use anyhow::{Context, anyhow};
 use database::mungos::by_id::find_one_by_id;
 use formatting::{Color, bold, colored, format_serror, muted};
-use futures::future::join_all;
+use futures_util::future::join_all;
 use komodo_client::{
   api::execute::*,
   entities::{
     action::Action,
     build::Build,
     deployment::Deployment,
+    permission::PermissionLevel,
     procedure::Procedure,
     repo::Repo,
     stack::Stack,
@@ -456,7 +457,7 @@ async fn execute_execution(
           .resolve(&ExecuteArgs { user, update, id })
           .await
           .map_err(|e| e.error)
-          .context("Failed at RemoveDeployment"),
+          .context("Failed at DestroyDeployment"),
         &update_id,
       )
       .await?
@@ -1161,6 +1162,159 @@ async fn execute_execution(
       )
       .await?
     }
+    Execution::RemoveSwarmNodes(req) => {
+      let req = ExecuteRequest::RemoveSwarmNodes(req);
+      let update = init_execution_update(&req, &user).await?;
+      let ExecuteRequest::RemoveSwarmNodes(req) = req else {
+        unreachable!()
+      };
+      let update_id = update.id.clone();
+      handle_resolve_result(
+        req
+          .resolve(&ExecuteArgs { user, update, id })
+          .await
+          .map_err(|e| e.error)
+          .context("Failed at RemoveSwarmNodes"),
+        &update_id,
+      )
+      .await?
+    }
+    Execution::RemoveSwarmStacks(req) => {
+      let req = ExecuteRequest::RemoveSwarmStacks(req);
+      let update = init_execution_update(&req, &user).await?;
+      let ExecuteRequest::RemoveSwarmStacks(req) = req else {
+        unreachable!()
+      };
+      let update_id = update.id.clone();
+      handle_resolve_result(
+        req
+          .resolve(&ExecuteArgs { user, update, id })
+          .await
+          .map_err(|e| e.error)
+          .context("Failed at RemoveSwarmStacks"),
+        &update_id,
+      )
+      .await?
+    }
+    Execution::RemoveSwarmServices(req) => {
+      let req = ExecuteRequest::RemoveSwarmServices(req);
+      let update = init_execution_update(&req, &user).await?;
+      let ExecuteRequest::RemoveSwarmServices(req) = req else {
+        unreachable!()
+      };
+      let update_id = update.id.clone();
+      handle_resolve_result(
+        req
+          .resolve(&ExecuteArgs { user, update, id })
+          .await
+          .map_err(|e| e.error)
+          .context("Failed at RemoveSwarmServices"),
+        &update_id,
+      )
+      .await?
+    }
+    Execution::CreateSwarmConfig(req) => {
+      let req = ExecuteRequest::CreateSwarmConfig(req);
+      let update = init_execution_update(&req, &user).await?;
+      let ExecuteRequest::CreateSwarmConfig(req) = req else {
+        unreachable!()
+      };
+      let update_id = update.id.clone();
+      handle_resolve_result(
+        req
+          .resolve(&ExecuteArgs { user, update, id })
+          .await
+          .map_err(|e| e.error)
+          .context("Failed at CreateSwarmConfig"),
+        &update_id,
+      )
+      .await?
+    }
+    Execution::RotateSwarmConfig(req) => {
+      let req = ExecuteRequest::RotateSwarmConfig(req);
+      let update = init_execution_update(&req, &user).await?;
+      let ExecuteRequest::RotateSwarmConfig(req) = req else {
+        unreachable!()
+      };
+      let update_id = update.id.clone();
+      handle_resolve_result(
+        req
+          .resolve(&ExecuteArgs { user, update, id })
+          .await
+          .map_err(|e| e.error)
+          .context("Failed at RotateSwarmConfig"),
+        &update_id,
+      )
+      .await?
+    }
+    Execution::RemoveSwarmConfigs(req) => {
+      let req = ExecuteRequest::RemoveSwarmConfigs(req);
+      let update = init_execution_update(&req, &user).await?;
+      let ExecuteRequest::RemoveSwarmConfigs(req) = req else {
+        unreachable!()
+      };
+      let update_id = update.id.clone();
+      handle_resolve_result(
+        req
+          .resolve(&ExecuteArgs { user, update, id })
+          .await
+          .map_err(|e| e.error)
+          .context("Failed at RemoveSwarmConfigs"),
+        &update_id,
+      )
+      .await?
+    }
+    Execution::CreateSwarmSecret(req) => {
+      let req = ExecuteRequest::CreateSwarmSecret(req);
+      let update = init_execution_update(&req, &user).await?;
+      let ExecuteRequest::CreateSwarmSecret(req) = req else {
+        unreachable!()
+      };
+      let update_id = update.id.clone();
+      handle_resolve_result(
+        req
+          .resolve(&ExecuteArgs { user, update, id })
+          .await
+          .map_err(|e| e.error)
+          .context("Failed at CreateSwarmSecret"),
+        &update_id,
+      )
+      .await?
+    }
+    Execution::RotateSwarmSecret(req) => {
+      let req = ExecuteRequest::RotateSwarmSecret(req);
+      let update = init_execution_update(&req, &user).await?;
+      let ExecuteRequest::RotateSwarmSecret(req) = req else {
+        unreachable!()
+      };
+      let update_id = update.id.clone();
+      handle_resolve_result(
+        req
+          .resolve(&ExecuteArgs { user, update, id })
+          .await
+          .map_err(|e| e.error)
+          .context("Failed at RotateSwarmSecret"),
+        &update_id,
+      )
+      .await?
+    }
+    Execution::RemoveSwarmSecrets(req) => {
+      let req = ExecuteRequest::RemoveSwarmSecrets(req);
+      let update = init_execution_update(&req, &user).await?;
+      let ExecuteRequest::RemoveSwarmSecrets(req) = req else {
+        unreachable!()
+      };
+      let update_id = update.id.clone();
+      handle_resolve_result(
+        req
+          .resolve(&ExecuteArgs { user, update, id })
+          .await
+          .map_err(|e| e.error)
+          .context("Failed at RemoveSwarmSecrets"),
+        &update_id,
+      )
+      .await?
+    }
     Execution::ClearRepoCache(req) => {
       let req = ExecuteRequest::ClearRepoCache(req);
       let update = init_execution_update(&req, &user).await?;
@@ -1311,6 +1465,7 @@ async fn extend_batch_exection<E: ExtendBatch>(
     pattern,
     Default::default(),
     procedure_user(),
+    PermissionLevel::Read.into(),
     &[],
   )
   .await?

bin/core/src/helpers/prune.rs

@@ -3,7 +3,7 @@ use async_timing_util::{
   ONE_DAY_MS, Timelength, unix_timestamp_ms, wait_until_timelength,
 };
 use database::mungos::{find::find_collect, mongodb::bson::doc};
-use futures::{StreamExt, stream::FuturesUnordered};
+use futures_util::{StreamExt, stream::FuturesUnordered};
 use periphery_client::api::docker::PruneImages;
 
 use crate::{config::core_config, state::db_client};

bin/core/src/helpers/query.rs

@@ -11,7 +11,7 @@ use database::mungos::{
 use komodo_client::{
   busy::Busy,
   entities::{
-    Operation, ResourceTarget, ResourceTargetVariant,
+    Operation, ResourceTarget, ResourceTargetVariant, SwarmOrServer,
     action::{Action, ActionState},
     alerter::Alerter,
     build::Build,
@@ -25,6 +25,7 @@ use komodo_client::{
     repo::Repo,
     server::{Server, ServerState},
     stack::{Stack, StackServiceNames, StackState},
+    swarm::Swarm,
     sync::ResourceSync,
     tag::Tag,
     update::Update,
@@ -33,9 +34,11 @@ use komodo_client::{
     variable::Variable,
   },
 };
+use openidconnect::SubjectIdentifier;
 
 use crate::{
   config::core_config,
+  helpers::swarm::swarm_request,
   permission::get_user_permission_on_resource,
   resource::{self, KomodoResource},
   stack::compose_container_match_regex,
@@ -55,8 +58,19 @@ pub async fn get_user(user: &str) -> anyhow::Result<User> {
     .users
     .find_one(id_or_username_filter(user))
     .await
-    .context("failed to query mongo for user")?
-    .with_context(|| format!("no user found with {user}"))
+    .context("Failed to query mongo for user")?
+    .with_context(|| format!("No user found matching '{user}'"))
+}
+
+pub async fn get_swarm_reachability(
+  swarm: &Swarm,
+) -> anyhow::Result<()> {
+  swarm_request(
+    &swarm.config.server_ids,
+    periphery_client::api::GetVersion {},
+  )
+  .await
+  .map(|_| ())
 }
 
 pub async fn get_server_with_state(
@@ -181,7 +195,9 @@ pub fn get_stack_state_from_containers(
 pub async fn get_stack_state(
   stack: &Stack,
 ) -> anyhow::Result<StackState> {
-  if stack.config.server_id.is_empty() {
+  if stack.config.swarm_id.is_empty()
+    && stack.config.server_id.is_empty()
+  {
     return Ok(StackState::Down);
   }
   let state = stack_status_cache()
@@ -287,24 +303,24 @@ pub async fn get_user_permission_on_target(
 ) -> anyhow::Result<PermissionLevelAndSpecifics> {
   match target {
     ResourceTarget::System(_) => Ok(PermissionLevel::None.into()),
-    ResourceTarget::Build(id) => {
-      get_user_permission_on_resource::<Build>(user, id).await
-    }
-    ResourceTarget::Builder(id) => {
-      get_user_permission_on_resource::<Builder>(user, id).await
-    }
-    ResourceTarget::Deployment(id) => {
-      get_user_permission_on_resource::<Deployment>(user, id).await
+    ResourceTarget::Swarm(id) => {
+      get_user_permission_on_resource::<Swarm>(user, id).await
     }
     ResourceTarget::Server(id) => {
       get_user_permission_on_resource::<Server>(user, id).await
     }
+    ResourceTarget::Stack(id) => {
+      get_user_permission_on_resource::<Stack>(user, id).await
+    }
+    ResourceTarget::Deployment(id) => {
+      get_user_permission_on_resource::<Deployment>(user, id).await
+    }
+    ResourceTarget::Build(id) => {
+      get_user_permission_on_resource::<Build>(user, id).await
+    }
     ResourceTarget::Repo(id) => {
       get_user_permission_on_resource::<Repo>(user, id).await
     }
-    ResourceTarget::Alerter(id) => {
-      get_user_permission_on_resource::<Alerter>(user, id).await
-    }
     ResourceTarget::Procedure(id) => {
       get_user_permission_on_resource::<Procedure>(user, id).await
     }
@@ -314,8 +330,11 @@ pub async fn get_user_permission_on_target(
     ResourceTarget::ResourceSync(id) => {
       get_user_permission_on_resource::<ResourceSync>(user, id).await
     }
-    ResourceTarget::Stack(id) => {
-      get_user_permission_on_resource::<Stack>(user, id).await
+    ResourceTarget::Builder(id) => {
+      get_user_permission_on_resource::<Builder>(user, id).await
+    }
+    ResourceTarget::Alerter(id) => {
+      get_user_permission_on_resource::<Alerter>(user, id).await
     }
   }
 }
@@ -447,3 +466,97 @@ pub async fn get_procedure_state(id: &String) -> ProcedureState {
   }
   procedure_state_cache().get(id).await.unwrap_or_default()
 }
+
+/// Get's a resource's assigned swarm or server, with swarm taking precedence.
+/// Makes sure the target is reachable before passing along for commands.
+pub async fn get_swarm_or_server(
+  swarm_id: &str,
+  server_id: &str,
+) -> anyhow::Result<SwarmOrServer> {
+  if !swarm_id.is_empty() {
+    let swarm = resource::get::<Swarm>(swarm_id).await?;
+
+    // Errors if not reachable, and returns the error
+    get_swarm_reachability(&swarm).await?;
+
+    return Ok(SwarmOrServer::Swarm(swarm));
+  }
+
+  if server_id.is_empty() {
+    return Err(anyhow!(
+      "Neither Swarm nor Server has been configured in this resource."
+    ));
+  }
+
+  let (server, state) = get_server_with_state(server_id).await?;
+
+  if state != ServerState::Ok {
+    return Err(anyhow!(
+      "Cannot send command when Server is unreachable or disabled"
+    ));
+  }
+
+  Ok(SwarmOrServer::Server(server))
+}
+
+pub async fn find_github_user(
+  github_id: &str,
+) -> anyhow::Result<Option<User>> {
+  db_client()
+    .users
+    .find_one(doc! {
+      // Find either primary or linked Github user
+      "$or": [
+        // User is primary Github user
+        { "config.data.github_id": &github_id },
+        // User has linked this Github login
+        { "linked_logins.Github.data.github_id": &github_id }
+      ]
+    })
+    .await
+    .context("Failed at find user query from database")
+}
+
+pub async fn find_google_user(
+  google_id: &str,
+) -> anyhow::Result<Option<User>> {
+  db_client()
+    .users
+    .find_one(doc! {
+      // Find either primary or linked Google user
+      "$or": [
+        // User is primary Google user
+        { "config.data.google_id": &google_id },
+        // User has linked this Google login
+        { "linked_logins.Google.data.google_id": &google_id }
+      ]
+    })
+    .await
+    .context("Failed at find user query from database")
+}
+
+pub async fn find_oidc_user(
+  subject: &SubjectIdentifier,
+) -> anyhow::Result<Option<User>> {
+  let oidc_provider = &core_config().oidc_provider;
+  let oidc_user_id = subject.as_str();
+  db_client()
+    .users
+    .find_one(doc! {
+      // Find either primary or linked Oidc user
+      "$or": [
+        // User is primary Oidc user
+        {
+          "config.data.provider": oidc_provider,
+          "config.data.user_id": oidc_user_id
+        },
+        // User has linked this Oidc login
+        {
+          "linked_logins.Oidc.data.provider": oidc_provider,
+          "linked_logins.Oidc.data.user_id": oidc_user_id
+        }
+      ]
+    })
+    .await
+    .context("Failed at find user query from database")
+}

bin/core/src/helpers/swarm.rs

@@ -0,0 +1,70 @@
+use std::time::Duration;
+
+use anyhow::anyhow;
+use komodo_client::entities::server::{Server, ServerState};
+use resolver_api::HasResponse;
+use serde::{Serialize, de::DeserializeOwned};
+
+use crate::{
+  helpers::periphery_client, resource, state::server_status_cache,
+};
+
+pub async fn swarm_request<T>(
+  server_ids: &[String],
+  request: T,
+) -> anyhow::Result<T::Response>
+where
+  T: std::fmt::Debug + Clone + Serialize + HasResponse,
+  T::Response: DeserializeOwned,
+{
+  swarm_request_custom_timeout(
+    server_ids,
+    request,
+    Duration::from_secs(10),
+  )
+  .await
+}
+
+pub async fn swarm_request_custom_timeout<T>(
+  server_ids: &[String],
+  request: T,
+  timeout: Duration,
+) -> anyhow::Result<T::Response>
+where
+  T: std::fmt::Debug + Clone + Serialize + HasResponse,
+  T::Response: DeserializeOwned,
+{
+  let status_cache = server_status_cache();
+  let mut err = Option::<anyhow::Error>::None;
+  for server_id in server_ids {
+    let Some(ServerState::Ok) =
+      status_cache.get(server_id).await.map(|status| status.state)
+    else {
+      err = Some(anyhow!("No managers connected"));
+      continue;
+    };
+    let Ok(periphery) =
+      (match resource::get::<Server>(server_id).await {
+        Ok(server) => periphery_client(&server).await,
+        Err(e) => {
+          err = Some(e);
+          continue;
+        }
+      })
+    else {
+      continue;
+    };
+    match periphery
+      .request_custom_timeout(request.clone(), timeout)
+      .await
+    {
+      Ok(res) => return Ok(res),
+      Err(e) => err = Some(e),
+    }
+  }
+  Err(
+    err
+      .unwrap_or_else(|| anyhow!("Unknown error"))
+      .context("Failed to request from Swarm"),
+  )
+}

bin/core/src/helpers/terminal.rs

@@ -0,0 +1,345 @@
+use anyhow::{Context as _, anyhow};
+use komodo_client::{
+  api::{terminal::InitTerminal, write::CreateTerminal},
+  entities::{
+    deployment::Deployment,
+    permission::PermissionLevel,
+    server::Server,
+    stack::Stack,
+    terminal::{ContainerTerminalMode, TerminalTarget},
+    user::User,
+  },
+};
+use periphery_client::api;
+
+use crate::{
+  helpers::periphery_client, periphery::PeripheryClient,
+  permission::get_check_permissions, resource,
+  state::stack_status_cache,
+};
+
+pub async fn setup_target_for_user(
+  target: TerminalTarget,
+  terminal: Option<String>,
+  init: Option<InitTerminal>,
+  user: &User,
+) -> anyhow::Result<(TerminalTarget, String, PeripheryClient)> {
+  match target {
+    TerminalTarget::Server { server } => {
+      setup_server_target_for_user(
+        server.context("Missing 'target.params.server'")?,
+        terminal,
+        init,
+        user,
+      )
+      .await
+    }
+    TerminalTarget::Container { server, container } => {
+      setup_container_target_for_user(
+        server, container, terminal, init, user,
+      )
+      .await
+    }
+    TerminalTarget::Stack { stack, service } => {
+      setup_stack_service_target_for_user(
+        stack,
+        service.context("Missing 'target.params.service'")?,
+        terminal,
+        init,
+        user,
+      )
+      .await
+    }
+    TerminalTarget::Deployment { deployment } => {
+      setup_deployment_target_for_user(
+        deployment, terminal, init, user,
+      )
+      .await
+    }
+  }
+}
+
+async fn setup_server_target_for_user(
+  server: String,
+  terminal: Option<String>,
+  init: Option<InitTerminal>,
+  user: &User,
+) -> anyhow::Result<(TerminalTarget, String, PeripheryClient)> {
+  let server = get_check_permissions::<Server>(
+    &server,
+    user,
+    PermissionLevel::Read.terminal(),
+  )
+  .await?;
+
+  let terminal = terminal.unwrap_or_else(|| {
+    init
+      .as_ref()
+      .and_then(|init| init.command.clone())
+      .unwrap_or_else(|| String::from("term"))
+  });
+
+  let periphery = periphery_client(&server).await?;
+
+  if let Some(init) = init {
+    periphery
+      .request(api::terminal::CreateServerTerminal {
+        name: terminal.clone(),
+        command: init.command,
+        recreate: init.recreate,
+      })
+      .await
+      .context("Failed to create Server Terminal on Periphery")?;
+  }
+
+  Ok((
+    TerminalTarget::Server {
+      server: Some(server.id),
+    },
+    terminal,
+    periphery,
+  ))
+}
+
+async fn setup_container_target_for_user(
+  server: String,
+  container: String,
+  terminal: Option<String>,
+  init: Option<InitTerminal>,
+  user: &User,
+) -> anyhow::Result<(TerminalTarget, String, PeripheryClient)> {
+  let server = get_check_permissions::<Server>(
+    &server,
+    user,
+    PermissionLevel::Read.terminal(),
+  )
+  .await?;
+
+  let terminal = default_container_terminal_name(
+    terminal,
+    &container,
+    init.as_ref(),
+  );
+
+  let periphery = periphery_client(&server).await?;
+
+  let target = TerminalTarget::Container {
+    server: server.id,
+    container: container.clone(),
+  };
+
+  if let Some(init) = init {
+    create_container_terminal_inner(
+      CreateTerminal {
+        name: terminal.clone(),
+        target: target.clone(),
+        command: init.command,
+        mode: init.mode,
+        recreate: init.recreate,
+      },
+      &periphery,
+      container,
+    )
+    .await?;
+  }
+
+  Ok((target, terminal, periphery))
+}
+
+async fn setup_stack_service_target_for_user(
+  stack: String,
+  service: String,
+  terminal: Option<String>,
+  init: Option<InitTerminal>,
+  user: &User,
+) -> anyhow::Result<(TerminalTarget, String, PeripheryClient)> {
+  let (target, periphery, container) =
+    get_stack_service_periphery_container(&stack, &service, user)
+      .await?;
+
+  let terminal = default_container_terminal_name(
+    terminal,
+    &container,
+    init.as_ref(),
+  );
+
+  if let Some(init) = init {
+    create_container_terminal_inner(
+      CreateTerminal {
+        name: terminal.clone(),
+        target: target.clone(),
+        command: init.command,
+        mode: init.mode,
+        recreate: init.recreate,
+      },
+      &periphery,
+      container,
+    )
+    .await?;
+  }
+
+  Ok((target, terminal, periphery))
+}
+
+async fn setup_deployment_target_for_user(
+  deployment: String,
+  terminal: Option<String>,
+  init: Option<InitTerminal>,
+  user: &User,
+) -> anyhow::Result<(TerminalTarget, String, PeripheryClient)> {
+  let (target, periphery, container) =
+    get_deployment_periphery_container(&deployment, user).await?;
+
+  let terminal = default_container_terminal_name(
+    terminal,
+    &container,
+    init.as_ref(),
+  );
+
+  if let Some(init) = init {
+    create_container_terminal_inner(
+      CreateTerminal {
+        name: terminal.clone(),
+        target: target.clone(),
+        command: init.command,
+        mode: init.mode,
+        recreate: init.recreate,
+      },
+      &periphery,
+      container,
+    )
+    .await?;
+  }
+
+  Ok((target, terminal, periphery))
+}
+
+fn default_container_terminal_name(
+  terminal: Option<String>,
+  container: &str,
+  init: Option<&InitTerminal>,
+) -> String {
+  terminal.unwrap_or_else(|| {
+    init
+      .as_ref()
+      .map(|init| {
+        init.command.clone().unwrap_or_else(|| {
+          init.mode.unwrap_or_default().as_ref().to_string()
+        })
+      })
+      .unwrap_or_else(|| container.to_string())
+  })
+}
+
+pub async fn create_container_terminal_inner(
+  CreateTerminal {
+    name,
+    target,
+    command,
+    mode,
+    recreate,
+  }: CreateTerminal,
+  periphery: &PeripheryClient,
+  container: String,
+) -> anyhow::Result<()> {
+  match mode.unwrap_or_default() {
+    ContainerTerminalMode::Exec => periphery
+      .request(periphery_client::api::terminal::CreateContainerExecTerminal {
+        name,
+        target,
+        container,
+        command,
+        recreate,
+      })
+      .await
+      .context(
+        "Failed to create Container Exec Terminal on Periphery",
+      )?,
+    ContainerTerminalMode::Attach => periphery
+      .request(periphery_client::api::terminal::CreateContainerAttachTerminal {
+        name,
+        target,
+        container,
+        recreate,
+      })
+      .await
+      .context(
+        "Failed to create Container Attach Terminal on Periphery",
+      )?,
+  };
+  Ok(())
+}
+
+pub async fn get_stack_service_periphery_container(
+  stack: &str,
+  service: &str,
+  user: &User,
+) -> anyhow::Result<(TerminalTarget, PeripheryClient, String)> {
+  let stack = get_check_permissions::<Stack>(
+    stack,
+    user,
+    PermissionLevel::Read.terminal(),
+  )
+  .await?;
+
+  let server =
+    resource::get::<Server>(&stack.config.server_id).await?;
+
+  let Some(status) = stack_status_cache().get(&stack.id).await else {
+    return Err(anyhow!("Could not get Stack status"));
+  };
+
+  let container = status
+    .curr
+    .services
+    .iter()
+    .find(|s| s.service.as_str() == service)
+    .with_context(|| {
+      format!("Did not find Stack service matching {service}")
+    })?
+    .container
+    .as_ref()
+    .with_context(|| {
+      format!("Did not find container for Stack service {service}")
+    })?
+    .name
+    .clone();
+
+  let periphery = periphery_client(&server).await?;
+
+  Ok((
+    TerminalTarget::Stack {
+      stack: stack.id,
+      service: Some(service.to_string()),
+    },
+    periphery,
+    container,
+  ))
+}
+
+pub async fn get_deployment_periphery_container(
+  deployment: &str,
+  user: &User,
+) -> anyhow::Result<(TerminalTarget, PeripheryClient, String)> {
+  let deployment = get_check_permissions::<Deployment>(
+    deployment,
+    user,
+    PermissionLevel::Read.terminal(),
+  )
+  .await?;
+
+  let server =
+    resource::get::<Server>(&deployment.config.server_id).await?;
+
+  let periphery = periphery_client(&server).await?;
+
+  let container = deployment.name.clone();
+
+  Ok((
+    TerminalTarget::Deployment {
+      deployment: deployment.id,
+    },
+    periphery,
+    container,
+  ))
+}

bin/core/src/helpers/update.rs

@@ -14,6 +14,7 @@ use komodo_client::entities::{
   repo::Repo,
   server::Server,
   stack::Stack,
+  swarm::Swarm,
   sync::ResourceSync,
   update::{Update, UpdateListItem},
   user::User,
@@ -121,6 +122,62 @@ pub async fn init_execution_update(
   user: &User,
 ) -> anyhow::Result<Update> {
   let (operation, target) = match &request {
+    // Swarm
+    ExecuteRequest::RemoveSwarmNodes(data) => (
+      Operation::RemoveSwarmNodes,
+      ResourceTarget::Swarm(
+        resource::get::<Swarm>(&data.swarm).await?.id,
+      ),
+    ),
+    ExecuteRequest::RemoveSwarmStacks(data) => (
+      Operation::RemoveSwarmStacks,
+      ResourceTarget::Swarm(
+        resource::get::<Swarm>(&data.swarm).await?.id,
+      ),
+    ),
+    ExecuteRequest::RemoveSwarmServices(data) => (
+      Operation::RemoveSwarmServices,
+      ResourceTarget::Swarm(
+        resource::get::<Swarm>(&data.swarm).await?.id,
+      ),
+    ),
+    ExecuteRequest::CreateSwarmConfig(data) => (
+      Operation::CreateSwarmConfig,
+      ResourceTarget::Swarm(
+        resource::get::<Swarm>(&data.swarm).await?.id,
+      ),
+    ),
+    ExecuteRequest::RotateSwarmConfig(data) => (
+      Operation::RotateSwarmConfig,
+      ResourceTarget::Swarm(
+        resource::get::<Swarm>(&data.swarm).await?.id,
+      ),
+    ),
+    ExecuteRequest::RemoveSwarmConfigs(data) => (
+      Operation::RemoveSwarmConfigs,
+      ResourceTarget::Swarm(
+        resource::get::<Swarm>(&data.swarm).await?.id,
+      ),
+    ),
+    ExecuteRequest::CreateSwarmSecret(data) => (
+      Operation::CreateSwarmSecret,
+      ResourceTarget::Swarm(
+        resource::get::<Swarm>(&data.swarm).await?.id,
+      ),
+    ),
+    ExecuteRequest::RotateSwarmSecret(data) => (
+      Operation::RotateSwarmSecret,
+      ResourceTarget::Swarm(
+        resource::get::<Swarm>(&data.swarm).await?.id,
+      ),
+    ),
+    ExecuteRequest::RemoveSwarmSecrets(data) => (
+      Operation::RemoveSwarmSecrets,
+      ResourceTarget::Swarm(
+        resource::get::<Swarm>(&data.swarm).await?.id,
+      ),
+    ),
+
     // Server
     ExecuteRequest::StartContainer(data) => (
       Operation::StartContainer,

bin/core/src/helpers/validations.rs

@@ -0,0 +1,85 @@
+//! # Input Validation Module
+//!
+//! This module provides validation functions for user inputs to prevent
+//! invalid data from entering the system and improve security.
+
+use anyhow::Context;
+use validations::{StringValidator, StringValidatorMatches};
+
+use crate::config::core_config;
+
+/// Minimum length for usernames
+pub const MIN_USERNAME_LENGTH: usize = 1;
+/// Maximum length for usernames
+pub const MAX_USERNAME_LENGTH: usize = 100;
+
+/// Validate usernames
+///
+/// - Between [MIN_USERNAME_LENGTH] and [MAX_USERNAME_LENGTH] characters
+/// - Matches `^[a-zA-Z0-9._@-]+$`
+pub fn validate_username(username: &str) -> anyhow::Result<()> {
+  StringValidator::default()
+    .min_length(MIN_USERNAME_LENGTH)
+    .max_length(MAX_USERNAME_LENGTH)
+    .matches(StringValidatorMatches::Username)
+    .validate(username)
+    .context("Failed to validate username")
+}
+
+/// Maximum length for passwords
+pub const MAX_PASSWORD_LENGTH: usize = 1000;
+
+/// Validate passwords
+///
+/// - Between [CoreConfig::min_password_length][komodo_client::entities::config::core::CoreConfig::min_password_length] and [MAX_PASSWORD_LENGTH] characters
+pub fn validate_password(password: &str) -> anyhow::Result<()> {
+  StringValidator::default()
+    .min_length(core_config().min_password_length as usize)
+    .max_length(MAX_PASSWORD_LENGTH)
+    .validate(password)
+    .context("Failed to validate password")
+}
+
+/// Maximum length for API key names
+pub const MAX_API_KEY_NAME_LENGTH: usize = 200;
+
+/// Validate api key names
+///
+/// - Greater than [MAX_API_KEY_NAME_LENGTH] characters
+pub fn validate_api_key_name(name: &str) -> anyhow::Result<()> {
+  StringValidator::default()
+    .max_length(MAX_API_KEY_NAME_LENGTH)
+    .validate(name)
+    .context("Failed to validate api key name")
+}
+
+/// Minimum length for variable names
+pub const MIN_VARIABLE_NAME_LENGTH: usize = 1;
+/// Maximum length for variable names
+pub const MAX_VARIABLE_NAME_LENGTH: usize = 500;
+
+/// Validate variable names
+///
+/// - Between [MIN_VARIABLE_NAME_LENGTH] and [MAX_VARIABLE_NAME_LENGTH] characters
+/// - Matches `^[a-zA-Z_][a-zA-Z0-9_]*$`
+pub fn validate_variable_name(name: &str) -> anyhow::Result<()> {
+  StringValidator::default()
+    .min_length(MIN_VARIABLE_NAME_LENGTH)
+    .max_length(MAX_VARIABLE_NAME_LENGTH)
+    .matches(StringValidatorMatches::VariableName)
+    .validate(name)
+    .context("Failed to validate variable name")
+}
+
+/// Maximum length for variable values
+pub const MAX_VARIABLE_VALUE_LENGTH: usize = 10000;
+
+/// Validate variable values
+///
+/// - Less than [MAX_VARIABLE_VALUE_LENGTH] characters
+pub fn validate_variable_value(value: &str) -> anyhow::Result<()> {
+  StringValidator::default()
+    .max_length(MAX_VARIABLE_VALUE_LENGTH)
+    .validate(value)
+    .context("Failed to validate variable value")
+}