deploy 2.0.0-dev-53

Cargo.toml

@@ -8,7 +8,7 @@ members = [
 ]
 
 [workspace.package]
-version = "2.0.0-dev-36"
+version = "2.0.0-dev-53"
 edition = "2024"
 authors = ["mbecker20 <becker.maxh@gmail.com>"]
 license = "GPL-3.0-or-later"
@@ -25,9 +25,11 @@ periphery_client = { path = "client/periphery/rs" }
 environment_file = { path = "lib/environment_file" }
 environment = { path = "lib/environment" }
 interpolate = { path = "lib/interpolate" }
+secret_file = { path = "lib/secret_file" }
 formatting = { path = "lib/formatting" }
 transport = { path = "lib/transport" }
 database = { path = "lib/database" }
+encoding = { path = "lib/encoding" }
 response = { path = "lib/response" }
 command = { path = "lib/command" }
 config = { path = "lib/config" }
@@ -77,7 +79,7 @@ bson = { version = "2.15.0" } # must keep in sync with mongodb version
 serde_yaml_ng = "0.10.0"
 serde_json = "1.0.145"
 serde_qs = "0.15.0"
-toml = "0.9.7"
+toml = "0.9.8"
 url = "2.5.7"
 
 # ERROR
@@ -100,12 +102,11 @@ envy = "0.4.2"
 
 # CRYPTO / AUTH
 uuid = { version = "1.18.1", features = ["v4", "fast-rng", "serde"] }
-jsonwebtoken = { version = "9.3.1", default-features = false } # locked back with octorust
+jsonwebtoken = { version = "10.0.0", features = ["aws_lc_rs"] } # locked back with octorust
 rustls = { version = "0.23.32", features = ["aws-lc-rs"] }
 pem-rfc7468 = { version = "0.7.0", features = ["alloc"] }
 openidconnect = "4.0.1"
 urlencoding = "2.1.3"
-nom_pem = "4.0.0"
 bcrypt = "0.17.1"
 base64 = "0.22.1"
 pkcs8 = "0.10.2"
@@ -124,9 +125,9 @@ bollard = "0.19.3"
 sysinfo = "0.37.1"
 
 # CLOUD
-aws-config = "1.8.6"
-aws-sdk-ec2 = "1.170.1"
-aws-credential-types = "1.2.6"
+aws-config = "1.8.8"
+aws-sdk-ec2 = "1.172.0"
+aws-credential-types = "1.2.8"
 
 ## CRON
 english-to-cron = "0.1.6"
@@ -140,9 +141,8 @@ derive_builder = "0.20.2"
 shell-escape = "0.1.5"
 comfy-table = "7.2.1"
 typeshare = "1.0.4"
-octorust = "0.10.0"
 dashmap = "6.1.0"
 wildcard = "0.3.0"
 colored = "3.0.0"
-regex = "1.11.3"
+regex = "1.12.1"
 bytes = "1.10.1"

bin/cli/src/command/database.rs

@@ -358,7 +358,9 @@ async fn v1_downgrade(yes: bool) -> anyhow::Result<()> {
     .await
     .context("Failed to downgrade Server schema")?;
 
-  info!("V1 Downgrade complete. Ready to downgrade to komodo-core:1 ✅");
+  info!(
+    "V1 Downgrade complete. Ready to downgrade to komodo-core:1 ✅"
+  );
 
   Ok(())
 }

bin/cli/src/command/execute.rs

@@ -233,6 +233,9 @@ pub async fn handle(
     Execution::RotateAllServerKeys(data) => {
       println!("{}: {data:?}", "Data".dimmed())
     }
+    Execution::RotateCoreKeys(data) => {
+      println!("{}: {data:?}", "Data".dimmed())
+    }
     Execution::Sleep(data) => {
       println!("{}: {data:?}", "Data".dimmed())
     }
@@ -501,6 +504,10 @@ pub async fn handle(
       .execute(request)
       .await
       .map(|u| ExecutionResult::Single(u.into())),
+    Execution::RotateCoreKeys(request) => client
+      .execute(request)
+      .await
+      .map(|u| ExecutionResult::Single(u.into())),
     Execution::Sleep(request) => {
       let duration =
         Duration::from_millis(request.duration_ms as u64);

bin/cli/src/config.rs

@@ -28,7 +28,7 @@ pub fn cli_env() -> &'static Env {
     {
       Ok(env) => env,
       Err(e) => {
-        panic!("{e:?}");
+        panic!("{e:?}")
       }
     }
   })

bin/core/Cargo.toml

@@ -19,9 +19,11 @@ komodo_client = { workspace = true, features = ["mongo"] }
 periphery_client.workspace = true
 environment_file.workspace = true
 interpolate.workspace = true
+secret_file.workspace = true
 formatting.workspace = true
 transport.workspace = true
 database.workspace = true
+encoding.workspace = true
 response.workspace = true
 command.workspace = true
 config.workspace = true
@@ -55,7 +57,6 @@ serde_yaml_ng.workspace = true
 typeshare.workspace = true
 chrono-tz.workspace = true
 indexmap.workspace = true
-octorust.workspace = true
 wildcard.workspace = true
 arc-swap.workspace = true
 colored.workspace = true
@@ -63,7 +64,6 @@ dashmap.workspace = true
 tracing.workspace = true
 reqwest.workspace = true
 futures.workspace = true
-nom_pem.workspace = true
 dotenvy.workspace = true
 anyhow.workspace = true
 croner.workspace = true

bin/core/aio.Dockerfile

@@ -57,6 +57,8 @@ ENV KOMODO_CLI_CONFIG_KEYWORDS="*config.*,*komodo.cli*.*"
 
 CMD [ "core" ]
 
+# Label to prevent Komodo from stopping with StopAllContainers
+LABEL komodo.skip="true"
 # Label for Ghcr
 LABEL org.opencontainers.image.source="https://github.com/moghtech/komodo"
 LABEL org.opencontainers.image.description="Komodo Core"

bin/core/multi-arch.Dockerfile

@@ -53,6 +53,8 @@ ENV KOMODO_CLI_CONFIG_KEYWORDS="*config.*,*komodo.cli*.*"
 
 CMD [ "core" ]
 
+# Label to prevent Komodo from stopping with StopAllContainers
+LABEL komodo.skip="true"
 # Label for Ghcr
 LABEL org.opencontainers.image.source="https://github.com/moghtech/komodo"
 LABEL org.opencontainers.image.description="Komodo Core"

bin/core/single-arch.Dockerfile

@@ -42,6 +42,8 @@ ENV KOMODO_CLI_CONFIG_KEYWORDS="*config.*,*komodo.cli*.*"
 
 CMD [ "core" ]
 
+# Label to prevent Komodo from stopping with StopAllContainers
+LABEL komodo.skip="true"
 # Label for Ghcr
 LABEL org.opencontainers.image.source="https://github.com/moghtech/komodo"
 LABEL org.opencontainers.image.description="Komodo Core"

bin/core/src/alert/discord.rs

@@ -50,7 +50,7 @@ pub async fn send_alert(
       match alert.level {
         SeverityLevel::Ok => {
           format!(
-            "{level} | **{name}**{region} is now **reachable**\n{link}"
+            "{level} | **{name}**{region} is now **connected**\n{link}"
           )
         }
         SeverityLevel::Critical => {
@@ -241,31 +241,33 @@ pub async fn send_alert(
     }
     AlertData::None {} => Default::default(),
   };
-  if !content.is_empty() {
-    let VariablesAndSecrets { variables, secrets } =
-      get_variables_and_secrets().await?;
-    let mut url_interpolated = url.to_string();
-
-    let mut interpolator =
-      Interpolator::new(Some(&variables), &secrets);
-
-    interpolator.interpolate_string(&mut url_interpolated)?;
-
-    send_message(&url_interpolated, &content)
-      .await
-      .map_err(|e| {
-        let replacers = interpolator
-          .secret_replacers
-          .into_iter()
-          .collect::<Vec<_>>();
-        let sanitized_error =
-          svi::replace_in_string(&format!("{e:?}"), &replacers);
-        anyhow::Error::msg(format!(
-          "Error with slack request: {sanitized_error}"
-        ))
-      })?;
+  
+  if content.is_empty() {
+    return Ok(());
   }
-  Ok(())
+
+  let VariablesAndSecrets { variables, secrets } =
+    get_variables_and_secrets().await?;
+  let mut url_interpolated = url.to_string();
+
+  let mut interpolator =
+    Interpolator::new(Some(&variables), &secrets);
+
+  interpolator.interpolate_string(&mut url_interpolated)?;
+
+  send_message(&url_interpolated, &content)
+    .await
+    .map_err(|e| {
+      let replacers = interpolator
+        .secret_replacers
+        .into_iter()
+        .collect::<Vec<_>>();
+      let sanitized_error =
+        svi::replace_in_string(&format!("{e:?}"), &replacers);
+      anyhow::Error::msg(format!(
+        "Error with slack request: {sanitized_error}"
+      ))
+    })
 }
 
 async fn send_message(

bin/core/src/alert/mod.rs

@@ -294,7 +294,7 @@ fn standard_alert_content(alert: &Alert) -> String {
       let link = resource_link(ResourceTargetVariant::Server, id);
       match alert.level {
         SeverityLevel::Ok => {
-          format!("{level} | {name}{region} is now reachable\n{link}")
+          format!("{level} | {name}{region} is now connected\n{link}")
         }
         SeverityLevel::Critical => {
           let err = err

bin/core/src/alert/ntfy.rs

@@ -9,10 +9,32 @@ pub async fn send_alert(
   alert: &Alert,
 ) -> anyhow::Result<()> {
   let content = standard_alert_content(alert);
-  if !content.is_empty() {
-    send_message(url, email, content).await?;
+  if content.is_empty() {
+    return Ok(());
   }
-  Ok(())
+
+  let VariablesAndSecrets { variables, secrets } =
+    get_variables_and_secrets().await?;
+  let mut url_interpolated = url.to_string();
+
+  let mut interpolator =
+    Interpolator::new(Some(&variables), &secrets);
+
+  interpolator.interpolate_string(&mut url_interpolated)?;
+
+  send_message(&url_interpolated, email, content)
+    .await
+    .map_err(|e| {
+      let replacers = interpolator
+        .secret_replacers
+        .into_iter()
+        .collect::<Vec<_>>();
+      let sanitized_error =
+        svi::replace_in_string(&format!("{e:?}"), &replacers);
+      anyhow::Error::msg(format!(
+        "Error with slack request: {sanitized_error}"
+      ))
+    })
 }
 
 async fn send_message(
@@ -22,7 +44,7 @@ async fn send_message(
 ) -> anyhow::Result<()> {
   let mut request = http_client()
     .post(url)
-    .header("Title", "ntfy Alert")
+    .header("Title", "Komodo Alert")
     .body(content);
 
   if let Some(email) = email {
@@ -43,9 +65,7 @@ async fn send_message(
       )
     })?;
     Err(anyhow!(
-      "Failed to send message to ntfy | {} | {}",
-      status,
-      text
+      "Failed to send message to ntfy | {status} | {text}",
     ))
   }
 }

bin/core/src/alert/pushover.rs

@@ -8,10 +8,30 @@ pub async fn send_alert(
   alert: &Alert,
 ) -> anyhow::Result<()> {
   let content = standard_alert_content(alert);
-  if !content.is_empty() {
-    send_message(url, content).await?;
+  if content.is_empty() {
+    return Ok(());
   }
-  Ok(())
+
+  let VariablesAndSecrets { variables, secrets } =
+    get_variables_and_secrets().await?;
+  let mut url_interpolated = url.to_string();
+
+  let mut interpolator =
+    Interpolator::new(Some(&variables), &secrets);
+
+  interpolator.interpolate_string(&mut url_interpolated)?;
+
+  send_message(&url_interpolated, content).await.map_err(|e| {
+    let replacers = interpolator
+      .secret_replacers
+      .into_iter()
+      .collect::<Vec<_>>();
+    let sanitized_error =
+      svi::replace_in_string(&format!("{e:?}"), &replacers);
+    anyhow::Error::msg(format!(
+      "Error with slack request: {sanitized_error}"
+    ))
+  })
 }
 
 async fn send_message(

bin/core/src/alert/slack.rs

@@ -64,11 +64,11 @@ pub async fn send_alert(
       match alert.level {
         SeverityLevel::Ok => {
           let text =
-            format!("{level} | *{name}*{region} is now *reachable*");
+            format!("{level} | *{name}*{region} is now *connected*");
           let blocks = vec![
             Block::header(level),
             Block::section(format!(
-              "*{name}*{region} is now *reachable*"
+              "*{name}*{region} is now *connnected*"
             )),
           ];
           (text, blocks.into())
@@ -468,31 +468,32 @@ pub async fn send_alert(
     }
     AlertData::None {} => Default::default(),
   };
-  if !text.is_empty() {
-    let VariablesAndSecrets { variables, secrets } =
-      get_variables_and_secrets().await?;
-    let mut url_interpolated = url.to_string();
-
-    let mut interpolator =
-      Interpolator::new(Some(&variables), &secrets);
-
-    interpolator.interpolate_string(&mut url_interpolated)?;
-
-    let slack = ::slack::Client::new(url_interpolated);
-    slack
-      .send_owned_message_single(&text, blocks.as_deref())
-      .await
-      .map_err(|e| {
-        let replacers = interpolator
-          .secret_replacers
-          .into_iter()
-          .collect::<Vec<_>>();
-        let sanitized_error =
-          svi::replace_in_string(&format!("{e:?}"), &replacers);
-        anyhow::Error::msg(format!(
-          "Error with slack request: {sanitized_error}"
-        ))
-      })?;
+  if text.is_empty() {
+    return Ok(());
   }
+  let VariablesAndSecrets { variables, secrets } =
+    get_variables_and_secrets().await?;
+  let mut url_interpolated = url.to_string();
+
+  let mut interpolator =
+    Interpolator::new(Some(&variables), &secrets);
+
+  interpolator.interpolate_string(&mut url_interpolated)?;
+
+  let slack = ::slack::Client::new(url_interpolated);
+  slack
+    .send_owned_message_single(&text, blocks.as_deref())
+    .await
+    .map_err(|e| {
+      let replacers = interpolator
+        .secret_replacers
+        .into_iter()
+        .collect::<Vec<_>>();
+      let sanitized_error =
+        svi::replace_in_string(&format!("{e:?}"), &replacers);
+      anyhow::Error::msg(format!(
+        "Error with slack request: {sanitized_error}"
+      ))
+    })?;
   Ok(())
 }

bin/core/src/api/execute/action.rs

@@ -142,15 +142,11 @@ impl Resolve<ExecuteArgs> for RunAction {
     let file = format!("{}.ts", random_string(10));
     let path = core_config().action_directory.join(&file);
 
-    if let Some(parent) = path.parent() {
-      fs::create_dir_all(parent)
-        .await
-        .with_context(|| format!("Failed to initialize Action file parent directory {parent:?}"))?;
-    }
-
-    fs::write(&path, contents).await.with_context(|| {
-      format!("Failed to write action file to {path:?}")
-    })?;
+    secret_file::write_async(&path, contents)
+      .await
+      .with_context(|| {
+        format!("Failed to write action file to {path:?}")
+      })?;
 
     let CoreConfig { ssl_enabled, .. } = core_config();
 

bin/core/src/api/execute/maintenance.rs

@@ -2,19 +2,23 @@ use std::{fmt::Write as _, sync::OnceLock};
 
 use anyhow::{Context, anyhow};
 use command::run_komodo_command;
-use database::mungos::{find::find_collect, mongodb::bson::doc};
+use database::{
+  bson::{Document, doc},
+  mungos::find::find_collect,
+};
 use formatting::{bold, format_serror};
-use futures::StreamExt;
+use futures::{StreamExt, stream::FuturesOrdered};
 use komodo_client::{
   api::execute::{
     BackupCoreDatabase, ClearRepoCache, GlobalAutoUpdate,
-    RotateAllServerKeys,
+    RotateAllServerKeys, RotateCoreKeys,
   },
   entities::{
     deployment::DeploymentState, server::ServerState,
     stack::StackState,
   },
 };
+use periphery_client::api;
 use reqwest::StatusCode;
 use resolver_api::Resolve;
 use serror::AddStatusCodeError;
@@ -24,8 +28,8 @@ use crate::{
   api::execute::{
     ExecuteArgs, pull_deployment_inner, pull_stack_inner,
   },
-  config::core_config,
-  helpers::update::update_update,
+  config::{core_config, core_keys},
+  helpers::{periphery_client, update::update_update},
   resource::rotate_server_keys,
   state::{
     db_client, deployment_status_cache, server_status_cache,
@@ -330,6 +334,11 @@ fn global_rotate_lock() -> &'static Mutex<()> {
 }
 
 impl Resolve<ExecuteArgs> for RotateAllServerKeys {
+  #[instrument(
+    name = "RotateAllServerKeys",
+    skip(user, update),
+    fields(user_id = user.id, update_id = update.id)
+  )]
   async fn resolve(
     self,
     ExecuteArgs { user, update }: &ExecuteArgs,
@@ -343,7 +352,7 @@ impl Resolve<ExecuteArgs> for RotateAllServerKeys {
 
     let _lock = global_rotate_lock()
       .try_lock()
-      .context("Rotate All Server Keys already in progress...")?;
+      .context("Key rotation already in progress...")?;
 
     let mut update = update.clone();
 
@@ -351,7 +360,7 @@ impl Resolve<ExecuteArgs> for RotateAllServerKeys {
 
     let mut servers = db_client()
       .servers
-      .find(doc! { "config.enabled": true })
+      .find(Document::new())
       .await
       .context("Failed to query servers from database")?;
 
@@ -367,6 +376,14 @@ impl Resolve<ExecuteArgs> for RotateAllServerKeys {
           continue;
         }
       };
+      if !server.config.auto_rotate_keys {
+        let _ = write!(
+          &mut log,
+          "\nSkipping {}: Key Rotation Disabled ⚙️",
+          bold(&server.name)
+        );
+        continue;
+      }
       let Some(status) = server_status_cache.get(&server.id).await
       else {
         let _ = write!(
@@ -376,14 +393,24 @@ impl Resolve<ExecuteArgs> for RotateAllServerKeys {
         );
         continue;
       };
-      if !matches!(status.state, ServerState::Ok) {
-        let _ = write!(
-          &mut log,
-          "\nSkipping {}: {} ⚠️",
-          bold(&server.name),
-          status.state
-        );
-        continue;
+      match status.state {
+        ServerState::Disabled => {
+          let _ = write!(
+            &mut log,
+            "\nSkipping {}: Server Disabled ⚙️",
+            bold(&server.name)
+          );
+          continue;
+        }
+        ServerState::NotOk => {
+          let _ = write!(
+            &mut log,
+            "\nSkipping {}: Server Not Ok ⚠️",
+            bold(&server.name)
+          );
+          continue;
+        }
+        _ => {}
       }
       match rotate_server_keys(&server).await {
         Ok(_) => {
@@ -415,3 +442,126 @@ impl Resolve<ExecuteArgs> for RotateAllServerKeys {
     Ok(update)
   }
 }
+
+impl Resolve<ExecuteArgs> for RotateCoreKeys {
+  #[instrument(
+    name = "RotateCoreKeys",
+    skip(user, update),
+    fields(user_id = user.id, update_id = update.id)
+  )]
+  async fn resolve(
+    self,
+    ExecuteArgs { user, update }: &ExecuteArgs,
+  ) -> Result<Self::Response, Self::Error> {
+    if !user.admin {
+      return Err(
+        anyhow!("This method is admin only.")
+          .status_code(StatusCode::FORBIDDEN),
+      );
+    }
+
+    let _lock = global_rotate_lock()
+      .try_lock()
+      .context("Key rotation already in progress...")?;
+
+    let mut update = update.clone();
+
+    update_update(update.clone()).await?;
+
+    let core_keys = core_keys();
+
+    if !core_keys.rotatable() {
+      return Err(anyhow!("Core `private_key` must be pointing to file, for example 'file:/config/keys/core.key'").into());
+    };
+
+    let server_status_cache = server_status_cache();
+    let servers =
+      find_collect(&db_client().servers, Document::new(), None)
+        .await
+        .context("Failed to query servers from database")?
+        .into_iter()
+        .map(|server| async move {
+          let state = server_status_cache
+            .get(&server.id)
+            .await
+            .map(|s| s.state)
+            .unwrap_or(ServerState::NotOk);
+          (server, state)
+        })
+        .collect::<FuturesOrdered<_>>()
+        .collect::<Vec<_>>()
+        .await;
+
+    if !self.force
+      && let Some((server, _)) = servers
+        .iter()
+        .find(|(_, state)| matches!(state, ServerState::NotOk))
+    {
+      return Err(
+        anyhow!("Server {} is NotOk, stopping key rotation. Pass `force: true` to continue anyways.", server.name).into(),
+      );
+    }
+
+    let public_key = core_keys.rotate().await?.into_inner();
+
+    info!("New Public Key: {public_key}");
+
+    let mut log = format!("New Public Key: {public_key}\n");
+
+    for (server, state) in servers {
+      match state {
+        ServerState::Disabled => {
+          let _ = write!(
+            &mut log,
+            "\nSkipping {}: Server Disabled ⚙️",
+            bold(&server.name)
+          );
+          continue;
+        }
+        ServerState::NotOk => {
+          // Shouldn't be reached unless 'force: true'
+          let _ = write!(
+            &mut log,
+            "\nSkipping {}: Server Not Ok ⚠️",
+            bold(&server.name)
+          );
+          continue;
+        }
+        _ => {}
+      }
+      let periphery = periphery_client(&server).await?;
+      let res = periphery
+        .request(api::keys::RotateCorePublicKey {
+          public_key: public_key.clone(),
+        })
+        .await;
+      match res {
+        Ok(_) => {
+          let _ = write!(
+            &mut log,
+            "\nRotated key for {} ✅",
+            bold(&server.name)
+          );
+        }
+        Err(e) => {
+          update.push_error_log(
+            "Key Rotation Failure",
+            format_serror(
+              &e.context(format!(
+                "Failed to rotate for {}. The new Core public key will have to be added manually.",
+                bold(&server.name)
+              ))
+              .into(),
+            ),
+          );
+        }
+      }
+    }
+
+    update.push_simple_log("Rotate Core Keys", log);
+    update.finalize();
+    update_update(update.clone()).await?;
+
+    Ok(update)
+  }
+}

bin/core/src/api/execute/mod.rs

@@ -150,6 +150,7 @@ pub enum ExecuteRequest {
   BackupCoreDatabase(BackupCoreDatabase),
   GlobalAutoUpdate(GlobalAutoUpdate),
   RotateAllServerKeys(RotateAllServerKeys),
+  RotateCoreKeys(RotateCoreKeys),
 }
 
 pub fn router() -> Router {

bin/core/src/api/read/build.rs

@@ -12,7 +12,6 @@ use komodo_client::{
   entities::{
     Operation,
     build::{Build, BuildActionState, BuildListItem, BuildState},
-    config::core::CoreConfig,
     permission::PermissionLevel,
     update::UpdateStatus,
   },
@@ -20,13 +19,10 @@ use komodo_client::{
 use resolver_api::Resolve;
 
 use crate::{
-  config::core_config,
   helpers::query::get_all_tags,
   permission::get_check_permissions,
   resource,
-  state::{
-    action_states, build_state_cache, db_client, github_client,
-  },
+  state::{action_states, build_state_cache, db_client},
 };
 
 use super::ReadArgs;
@@ -306,81 +302,3 @@ impl Resolve<ReadArgs> for ListCommonBuildExtraArgs {
     Ok(res)
   }
 }
-
-impl Resolve<ReadArgs> for GetBuildWebhookEnabled {
-  async fn resolve(
-    self,
-    ReadArgs { user }: &ReadArgs,
-  ) -> serror::Result<GetBuildWebhookEnabledResponse> {
-    let Some(github) = github_client() else {
-      return Ok(GetBuildWebhookEnabledResponse {
-        managed: false,
-        enabled: false,
-      });
-    };
-
-    let build = get_check_permissions::<Build>(
-      &self.build,
-      user,
-      PermissionLevel::Read.into(),
-    )
-    .await?;
-
-    if build.config.git_provider != "github.com"
-      || build.config.repo.is_empty()
-    {
-      return Ok(GetBuildWebhookEnabledResponse {
-        managed: false,
-        enabled: false,
-      });
-    }
-
-    let mut split = build.config.repo.split('/');
-    let owner = split.next().context("Build repo has no owner")?;
-
-    let Some(github) = github.get(owner) else {
-      return Ok(GetBuildWebhookEnabledResponse {
-        managed: false,
-        enabled: false,
-      });
-    };
-
-    let repo =
-      split.next().context("Build repo has no repo after the /")?;
-
-    let github_repos = github.repos();
-
-    let webhooks = github_repos
-      .list_all_webhooks(owner, repo)
-      .await
-      .context("failed to list all webhooks on repo")?
-      .body;
-
-    let CoreConfig {
-      host,
-      webhook_base_url,
-      ..
-    } = core_config();
-
-    let host = if webhook_base_url.is_empty() {
-      host
-    } else {
-      webhook_base_url
-    };
-    let url = format!("{host}/listener/github/build/{}", build.id);
-
-    for webhook in webhooks {
-      if webhook.active && webhook.config.url == url {
-        return Ok(GetBuildWebhookEnabledResponse {
-          managed: true,
-          enabled: true,
-        });
-      }
-    }
-
-    Ok(GetBuildWebhookEnabledResponse {
-      managed: true,
-      enabled: false,
-    })
-  }
-}

bin/core/src/api/read/mod.rs

@@ -1,4 +1,4 @@
-use std::{collections::HashSet, sync::OnceLock, time::Instant};
+use std::{collections::HashSet, time::Instant};
 
 use anyhow::{Context, anyhow};
 use axum::{
@@ -28,7 +28,7 @@ use uuid::Uuid;
 
 use crate::{
   auth::auth_request,
-  config::{core_config, core_public_key},
+  config::{core_config, core_keys},
   helpers::periphery_client,
   resource,
 };
@@ -142,7 +142,6 @@ enum ReadRequest {
   GetStacksSummary(GetStacksSummary),
   GetStack(GetStack),
   GetStackActionState(GetStackActionState),
-  GetStackWebhooksEnabled(GetStackWebhooksEnabled),
   GetStackLog(GetStackLog),
   SearchStackLog(SearchStackLog),
   InspectStackContainer(InspectStackContainer),
@@ -171,7 +170,6 @@ enum ReadRequest {
   GetBuildActionState(GetBuildActionState),
   GetBuildMonthlyStats(GetBuildMonthlyStats),
   ListBuildVersions(ListBuildVersions),
-  GetBuildWebhookEnabled(GetBuildWebhookEnabled),
   ListBuilds(ListBuilds),
   ListFullBuilds(ListFullBuilds),
   ListCommonBuildExtraArgs(ListCommonBuildExtraArgs),
@@ -180,7 +178,6 @@ enum ReadRequest {
   GetReposSummary(GetReposSummary),
   GetRepo(GetRepo),
   GetRepoActionState(GetRepoActionState),
-  GetRepoWebhooksEnabled(GetRepoWebhooksEnabled),
   ListRepos(ListRepos),
   ListFullRepos(ListFullRepos),
 
@@ -188,7 +185,6 @@ enum ReadRequest {
   GetResourceSyncsSummary(GetResourceSyncsSummary),
   GetResourceSync(GetResourceSync),
   GetResourceSyncActionState(GetResourceSyncActionState),
-  GetSyncWebhooksEnabled(GetSyncWebhooksEnabled),
   ListResourceSyncs(ListResourceSyncs),
   ListFullResourceSyncs(ListFullResourceSyncs),
 
@@ -281,11 +277,13 @@ impl Resolve<ReadArgs> for GetVersion {
   }
 }
 
-fn core_info() -> &'static GetCoreInfoResponse {
-  static CORE_INFO: OnceLock<GetCoreInfoResponse> = OnceLock::new();
-  CORE_INFO.get_or_init(|| {
+impl Resolve<ReadArgs> for GetCoreInfo {
+  async fn resolve(
+    self,
+    _: &ReadArgs,
+  ) -> serror::Result<GetCoreInfoResponse> {
     let config = core_config();
-    GetCoreInfoResponse {
+    let info = GetCoreInfoResponse {
       title: config.title.clone(),
       monitoring_interval: config.monitoring_interval,
       webhook_base_url: if config.webhook_base_url.is_empty() {
@@ -299,24 +297,10 @@ fn core_info() -> &'static GetCoreInfoResponse {
       disable_non_admin_create: config.disable_non_admin_create,
       disable_websocket_reconnect: config.disable_websocket_reconnect,
       enable_fancy_toml: config.enable_fancy_toml,
-      github_webhook_owners: config
-        .github_webhook_app
-        .installations
-        .iter()
-        .map(|i| i.namespace.to_string())
-        .collect(),
       timezone: config.timezone.clone(),
-      public_key: core_public_key().to_string(),
-    }
-  })
-}
-
-impl Resolve<ReadArgs> for GetCoreInfo {
-  async fn resolve(
-    self,
-    _: &ReadArgs,
-  ) -> serror::Result<GetCoreInfoResponse> {
-    Ok(core_info().clone())
+      public_key: core_keys().load().public.to_string(),
+    };
+    Ok(info)
   }
 }
 

bin/core/src/api/read/onboarding_key.rs

@@ -1,3 +1,5 @@
+use std::cmp::Ordering;
+
 use anyhow::{Context, anyhow};
 use database::mungos::find::find_collect;
 use komodo_client::api::read::{
@@ -22,9 +24,28 @@ impl Resolve<ReadArgs> for ListOnboardingKeys {
           .status_code(StatusCode::FORBIDDEN),
       );
     }
-    find_collect(&db_client().onboarding_keys, None, None)
-      .await
-      .context("Failed to query database for Server onboarding keys")
-      .map_err(Into::into)
+
+    let mut keys =
+      find_collect(&db_client().onboarding_keys, None, None)
+        .await
+        .context(
+          "Failed to query database for Server onboarding keys",
+        )?;
+
+    // No expiry keys first, followed
+    keys.sort_by(|a, b| {
+      if a.expires == b.expires {
+        Ordering::Equal
+      } else if a.expires == 0 {
+        Ordering::Less
+      } else if b.expires == 0 {
+        Ordering::Greater
+      } else {
+        // Descending
+        b.expires.cmp(&a.expires)
+      }
+    });
+
+    Ok(keys)
   }
 }

bin/core/src/api/read/repo.rs

@@ -2,7 +2,6 @@ use anyhow::Context;
 use komodo_client::{
   api::read::*,
   entities::{
-    config::core::CoreConfig,
     permission::PermissionLevel,
     repo::{Repo, RepoActionState, RepoListItem, RepoState},
   },
@@ -10,11 +9,10 @@ use komodo_client::{
 use resolver_api::Resolve;
 
 use crate::{
-  config::core_config,
   helpers::query::get_all_tags,
   permission::get_check_permissions,
   resource,
-  state::{action_states, github_client, repo_state_cache},
+  state::{action_states, repo_state_cache},
 };
 
 use super::ReadArgs;
@@ -159,104 +157,3 @@ impl Resolve<ReadArgs> for GetReposSummary {
     Ok(res)
   }
 }
-
-impl Resolve<ReadArgs> for GetRepoWebhooksEnabled {
-  async fn resolve(
-    self,
-    ReadArgs { user }: &ReadArgs,
-  ) -> serror::Result<GetRepoWebhooksEnabledResponse> {
-    let Some(github) = github_client() else {
-      return Ok(GetRepoWebhooksEnabledResponse {
-        managed: false,
-        clone_enabled: false,
-        pull_enabled: false,
-        build_enabled: false,
-      });
-    };
-
-    let repo = get_check_permissions::<Repo>(
-      &self.repo,
-      user,
-      PermissionLevel::Read.into(),
-    )
-    .await?;
-
-    if repo.config.git_provider != "github.com"
-      || repo.config.repo.is_empty()
-    {
-      return Ok(GetRepoWebhooksEnabledResponse {
-        managed: false,
-        clone_enabled: false,
-        pull_enabled: false,
-        build_enabled: false,
-      });
-    }
-
-    let mut split = repo.config.repo.split('/');
-    let owner = split.next().context("Repo repo has no owner")?;
-
-    let Some(github) = github.get(owner) else {
-      return Ok(GetRepoWebhooksEnabledResponse {
-        managed: false,
-        clone_enabled: false,
-        pull_enabled: false,
-        build_enabled: false,
-      });
-    };
-
-    let repo_name =
-      split.next().context("Repo repo has no repo after the /")?;
-
-    let github_repos = github.repos();
-
-    let webhooks = github_repos
-      .list_all_webhooks(owner, repo_name)
-      .await
-      .context("failed to list all webhooks on repo")?
-      .body;
-
-    let CoreConfig {
-      host,
-      webhook_base_url,
-      ..
-    } = core_config();
-
-    let host = if webhook_base_url.is_empty() {
-      host
-    } else {
-      webhook_base_url
-    };
-    let clone_url =
-      format!("{host}/listener/github/repo/{}/clone", repo.id);
-    let pull_url =
-      format!("{host}/listener/github/repo/{}/pull", repo.id);
-    let build_url =
-      format!("{host}/listener/github/repo/{}/build", repo.id);
-
-    let mut clone_enabled = false;
-    let mut pull_enabled = false;
-    let mut build_enabled = false;
-
-    for webhook in webhooks {
-      if !webhook.active {
-        continue;
-      }
-      if webhook.config.url == clone_url {
-        clone_enabled = true
-      }
-      if webhook.config.url == pull_url {
-        pull_enabled = true
-      }
-      if webhook.config.url == build_url {
-        build_enabled = true
-      }
-    }
-
-    Ok(GetRepoWebhooksEnabledResponse {
-      managed: true,
-      clone_enabled,
-      pull_enabled,
-      build_enabled,
-    })
-  }
-}

bin/core/src/api/read/server.rs

@@ -885,7 +885,9 @@ impl Resolve<ReadArgs> for ListTerminals {
     if self.fresh || komodo_timestamp() > cache.ttl {
       cache.list = periphery_client(&server)
         .await?
-        .request(periphery_client::api::terminal::ListTerminals {})
+        .request(periphery_client::api::terminal::ListTerminals {
+          container: None,
+        })
         .await
         .context("Failed to get fresh terminal list")?;
       cache.ttl = komodo_timestamp() + TERMINAL_CACHE_TIMEOUT;

bin/core/src/api/read/stack.rs

@@ -4,7 +4,6 @@ use anyhow::{Context, anyhow};
 use komodo_client::{
   api::read::*,
   entities::{
-    config::core::CoreConfig,
     docker::container::Container,
     permission::PermissionLevel,
     server::{Server, ServerState},
@@ -18,15 +17,11 @@ use periphery_client::api::{
 use resolver_api::Resolve;
 
 use crate::{
-  config::core_config,
   helpers::{periphery_client, query::get_all_tags},
   permission::get_check_permissions,
   resource,
   stack::get_stack_and_server,
-  state::{
-    action_states, github_client, server_status_cache,
-    stack_status_cache,
-  },
+  state::{action_states, server_status_cache, stack_status_cache},
 };
 
 use super::ReadArgs;
@@ -378,91 +373,3 @@ impl Resolve<ReadArgs> for GetStacksSummary {
     Ok(res)
   }
 }
-
-impl Resolve<ReadArgs> for GetStackWebhooksEnabled {
-  async fn resolve(
-    self,
-    ReadArgs { user }: &ReadArgs,
-  ) -> serror::Result<GetStackWebhooksEnabledResponse> {
-    let Some(github) = github_client() else {
-      return Ok(GetStackWebhooksEnabledResponse {
-        managed: false,
-        refresh_enabled: false,
-        deploy_enabled: false,
-      });
-    };
-
-    let stack = get_check_permissions::<Stack>(
-      &self.stack,
-      user,
-      PermissionLevel::Read.into(),
-    )
-    .await?;
-
-    if stack.config.git_provider != "github.com"
-      || stack.config.repo.is_empty()
-    {
-      return Ok(GetStackWebhooksEnabledResponse {
-        managed: false,
-        refresh_enabled: false,
-        deploy_enabled: false,
-      });
-    }
-
-    let mut split = stack.config.repo.split('/');
-    let owner = split.next().context("Sync repo has no owner")?;
-
-    let Some(github) = github.get(owner) else {
-      return Ok(GetStackWebhooksEnabledResponse {
-        managed: false,
-        refresh_enabled: false,
-        deploy_enabled: false,
-      });
-    };
-
-    let repo_name =
-      split.next().context("Repo repo has no repo after the /")?;
-
-    let github_repos = github.repos();
-
-    let webhooks = github_repos
-      .list_all_webhooks(owner, repo_name)
-      .await
-      .context("failed to list all webhooks on repo")?
-      .body;
-
-    let CoreConfig {
-      host,
-      webhook_base_url,
-      ..
-    } = core_config();
-
-    let host = if webhook_base_url.is_empty() {
-      host
-    } else {
-      webhook_base_url
-    };
-    let refresh_url =
-      format!("{host}/listener/github/stack/{}/refresh", stack.id);
-    let deploy_url =
-      format!("{host}/listener/github/stack/{}/deploy", stack.id);
-
-    let mut refresh_enabled = false;
-    let mut deploy_enabled = false;
-
-    for webhook in webhooks {
-      if webhook.active && webhook.config.url == refresh_url {
-        refresh_enabled = true
-      }
-      if webhook.active && webhook.config.url == deploy_url {
-        deploy_enabled = true
-      }
-    }
-
-    Ok(GetStackWebhooksEnabledResponse {
-      managed: true,
-      refresh_enabled,
-      deploy_enabled,
-    })
-  }
-}

bin/core/src/api/read/sync.rs

@@ -2,7 +2,6 @@ use anyhow::Context;
 use komodo_client::{
   api::read::*,
   entities::{
-    config::core::CoreConfig,
     permission::PermissionLevel,
     sync::{
       ResourceSync, ResourceSyncActionState, ResourceSyncListItem,
@@ -12,11 +11,8 @@ use komodo_client::{
 use resolver_api::Resolve;
 
 use crate::{
-  config::core_config,
-  helpers::query::get_all_tags,
-  permission::get_check_permissions,
-  resource,
-  state::{action_states, github_client},
+  helpers::query::get_all_tags, permission::get_check_permissions,
+  resource, state::action_states,
 };
 
 use super::ReadArgs;
@@ -154,91 +150,3 @@ impl Resolve<ReadArgs> for GetResourceSyncsSummary {
     Ok(res)
   }
 }
-
-impl Resolve<ReadArgs> for GetSyncWebhooksEnabled {
-  async fn resolve(
-    self,
-    ReadArgs { user }: &ReadArgs,
-  ) -> serror::Result<GetSyncWebhooksEnabledResponse> {
-    let Some(github) = github_client() else {
-      return Ok(GetSyncWebhooksEnabledResponse {
-        managed: false,
-        refresh_enabled: false,
-        sync_enabled: false,
-      });
-    };
-
-    let sync = get_check_permissions::<ResourceSync>(
-      &self.sync,
-      user,
-      PermissionLevel::Read.into(),
-    )
-    .await?;
-
-    if sync.config.git_provider != "github.com"
-      || sync.config.repo.is_empty()
-    {
-      return Ok(GetSyncWebhooksEnabledResponse {
-        managed: false,
-        refresh_enabled: false,
-        sync_enabled: false,
-      });
-    }
-
-    let mut split = sync.config.repo.split('/');
-    let owner = split.next().context("Sync repo has no owner")?;
-
-    let Some(github) = github.get(owner) else {
-      return Ok(GetSyncWebhooksEnabledResponse {
-        managed: false,
-        refresh_enabled: false,
-        sync_enabled: false,
-      });
-    };
-
-    let repo_name =
-      split.next().context("Repo repo has no repo after the /")?;
-
-    let github_repos = github.repos();
-
-    let webhooks = github_repos
-      .list_all_webhooks(owner, repo_name)
-      .await
-      .context("failed to list all webhooks on repo")?
-      .body;
-
-    let CoreConfig {
-      host,
-      webhook_base_url,
-      ..
-    } = core_config();
-
-    let host = if webhook_base_url.is_empty() {
-      host
-    } else {
-      webhook_base_url
-    };
-    let refresh_url =
-      format!("{host}/listener/github/sync/{}/refresh", sync.id);
-    let sync_url =
-      format!("{host}/listener/github/sync/{}/sync", sync.id);
-
-    let mut refresh_enabled = false;
-    let mut sync_enabled = false;
-
-    for webhook in webhooks {
-      if webhook.active && webhook.config.url == refresh_url {
-        refresh_enabled = true
-      }
-      if webhook.active && webhook.config.url == sync_url {
-        sync_enabled = true
-      }
-    }
-
-    Ok(GetSyncWebhooksEnabledResponse {
-      managed: true,
-      refresh_enabled,
-      sync_enabled,
-    })
-  }
-}

bin/core/src/api/terminal.rs

@@ -95,6 +95,7 @@ async fn execute_container_exec_inner(
     container,
     shell,
     command,
+    recreate,
   }: ExecuteContainerExecBody,
   user: User,
 ) -> serror::Result<axum::body::Body> {
@@ -110,7 +111,7 @@ async fn execute_container_exec_inner(
   let periphery = periphery_client(&server).await?;
 
   let stream = periphery
-    .execute_container_exec(container, shell, command)
+    .execute_container_exec(container, shell, command, recreate)
     .await
     .context(
       "Failed to execute container exec command on periphery",
@@ -143,6 +144,7 @@ async fn execute_deployment_exec_inner(
     deployment,
     shell,
     command,
+    recreate,
   }: ExecuteDeploymentExecBody,
   user: User,
 ) -> serror::Result<axum::body::Body> {
@@ -160,7 +162,7 @@ async fn execute_deployment_exec_inner(
   let periphery = periphery_client(&server).await?;
 
   let stream = periphery
-    .execute_container_exec(deployment.name, shell, command)
+    .execute_container_exec(deployment.name, shell, command, recreate)
     .await
     .context(
       "Failed to execute container exec command on periphery",
@@ -194,6 +196,7 @@ async fn execute_stack_exec_inner(
     service,
     shell,
     command,
+    recreate,
   }: ExecuteStackExecBody,
   user: User,
 ) -> serror::Result<axum::body::Body> {
@@ -226,7 +229,7 @@ async fn execute_stack_exec_inner(
   let periphery = periphery_client(&server).await?;
 
   let stream = periphery
-    .execute_container_exec(container, shell, command)
+    .execute_container_exec(container, shell, command, recreate)
     .await
     .context(
       "Failed to execute container exec command on periphery",

bin/core/src/api/write/alert.rs

@@ -0,0 +1,33 @@
+use std::str::FromStr;
+
+use anyhow::{Context, anyhow};
+use database::mungos::mongodb::bson::{doc, oid::ObjectId};
+use komodo_client::{api::write::CloseAlert, entities::NoData};
+use reqwest::StatusCode;
+use resolver_api::Resolve;
+use serror::AddStatusCodeError;
+
+use crate::{api::write::WriteArgs, state::db_client};
+
+impl Resolve<WriteArgs> for CloseAlert {
+  async fn resolve(
+    self,
+    WriteArgs { user: admin }: &WriteArgs,
+  ) -> Result<Self::Response, Self::Error> {
+    if !admin.admin {
+      return Err(
+        anyhow!("This call is admin only")
+          .status_code(StatusCode::FORBIDDEN),
+      );
+    }
+    db_client()
+      .alerts
+      .update_one(
+        doc! { "_id": ObjectId::from_str(&self.id)? },
+        doc! { "$set": { "resolved": true } },
+      )
+      .await
+      .context("Failed to close Alert on database")?;
+    Ok(NoData {})
+  }
+}

bin/core/src/api/write/build.rs

@@ -11,18 +11,14 @@ use komodo_client::{
   entities::{
     FileContents, NoData, Operation, RepoExecutionArgs,
     all_logs_success,
-    build::{Build, BuildInfo, PartialBuildConfig},
+    build::{Build, BuildInfo},
     builder::{Builder, BuilderConfig},
-    config::core::CoreConfig,
     permission::PermissionLevel,
     repo::Repo,
     server::ServerState,
     update::Update,
   },
 };
-use octorust::types::{
-  ReposCreateWebhookRequest, ReposCreateWebhookRequestConfig,
-};
 use periphery_client::api::build::{
   GetDockerfileContentsOnHost, WriteDockerfileContentsToHost,
 };
@@ -40,7 +36,7 @@ use crate::{
   periphery::PeripheryClient,
   permission::get_check_permissions,
   resource,
-  state::{db_client, github_client},
+  state::db_client,
 };
 
 use super::WriteArgs;
@@ -272,8 +268,9 @@ async fn write_dockerfile_contents_git(
     return Ok(update);
   }
 
-  if let Err(e) =
-    fs::write(&full_path, &contents).await.with_context(|| {
+  if let Err(e) = secret_file::write_async(&full_path, &contents)
+    .await
+    .with_context(|| {
       format!("Failed to write dockerfile contents to {full_path:?}")
     })
   {
@@ -439,7 +436,6 @@ async fn get_on_host_periphery(
           &config,
         ),
         config.insecure_tls,
-        &config.passkey,
       )
       .await?;
       // Poll for connection to be estalished
@@ -546,201 +542,3 @@ async fn get_git_remote(
     res.commit_message,
   )))
 }
-
-impl Resolve<WriteArgs> for CreateBuildWebhook {
-  #[instrument(name = "CreateBuildWebhook", skip(args))]
-  async fn resolve(
-    self,
-    args: &WriteArgs,
-  ) -> serror::Result<CreateBuildWebhookResponse> {
-    let Some(github) = github_client() else {
-      return Err(
-        anyhow!(
-          "github_webhook_app is not configured in core config toml"
-        )
-        .into(),
-      );
-    };
-
-    let WriteArgs { user } = args;
-
-    let build = get_check_permissions::<Build>(
-      &self.build,
-      user,
-      PermissionLevel::Write.into(),
-    )
-    .await?;
-
-    if build.config.repo.is_empty() {
-      return Err(
-        anyhow!("No repo configured, can't create webhook").into(),
-      );
-    }
-
-    let mut split = build.config.repo.split('/');
-    let owner = split.next().context("Build repo has no owner")?;
-
-    let Some(github) = github.get(owner) else {
-      return Err(
-        anyhow!("Cannot manage repo webhooks under owner {owner}")
-          .into(),
-      );
-    };
-
-    let repo =
-      split.next().context("Build repo has no repo after the /")?;
-
-    let github_repos = github.repos();
-
-    // First make sure the webhook isn't already created (inactive ones are ignored)
-    let webhooks = github_repos
-      .list_all_webhooks(owner, repo)
-      .await
-      .context("failed to list all webhooks on repo")?
-      .body;
-
-    let CoreConfig {
-      host,
-      webhook_base_url,
-      webhook_secret,
-      ..
-    } = core_config();
-
-    let webhook_secret = if build.config.webhook_secret.is_empty() {
-      webhook_secret
-    } else {
-      &build.config.webhook_secret
-    };
-
-    let host = if webhook_base_url.is_empty() {
-      host
-    } else {
-      webhook_base_url
-    };
-    let url = format!("{host}/listener/github/build/{}", build.id);
-
-    for webhook in webhooks {
-      if webhook.active && webhook.config.url == url {
-        return Ok(NoData {});
-      }
-    }
-
-    // Now good to create the webhook
-    let request = ReposCreateWebhookRequest {
-      active: Some(true),
-      config: Some(ReposCreateWebhookRequestConfig {
-        url,
-        secret: webhook_secret.to_string(),
-        content_type: String::from("json"),
-        insecure_ssl: None,
-        digest: Default::default(),
-        token: Default::default(),
-      }),
-      events: vec![String::from("push")],
-      name: String::from("web"),
-    };
-    github_repos
-      .create_webhook(owner, repo, &request)
-      .await
-      .context("failed to create webhook")?;
-
-    if !build.config.webhook_enabled {
-      UpdateBuild {
-        id: build.id,
-        config: PartialBuildConfig {
-          webhook_enabled: Some(true),
-          ..Default::default()
-        },
-      }
-      .resolve(args)
-      .await
-      .map_err(|e| e.error)
-      .context("failed to update build to enable webhook")?;
-    }
-
-    Ok(NoData {})
-  }
-}
-
-impl Resolve<WriteArgs> for DeleteBuildWebhook {
-  #[instrument(name = "DeleteBuildWebhook", skip(user))]
-  async fn resolve(
-    self,
-    WriteArgs { user }: &WriteArgs,
-  ) -> serror::Result<DeleteBuildWebhookResponse> {
-    let Some(github) = github_client() else {
-      return Err(
-        anyhow!(
-          "github_webhook_app is not configured in core config toml"
-        )
-        .into(),
-      );
-    };
-
-    let build = get_check_permissions::<Build>(
-      &self.build,
-      user,
-      PermissionLevel::Write.into(),
-    )
-    .await?;
-
-    if build.config.git_provider != "github.com" {
-      return Err(
-        anyhow!("Can only manage github.com repo webhooks").into(),
-      );
-    }
-
-    if build.config.repo.is_empty() {
-      return Err(
-        anyhow!("No repo configured, can't delete webhook").into(),
-      );
-    }
-
-    let mut split = build.config.repo.split('/');
-    let owner = split.next().context("Build repo has no owner")?;
-
-    let Some(github) = github.get(owner) else {
-      return Err(
-        anyhow!("Cannot manage repo webhooks under owner {owner}")
-          .into(),
-      );
-    };
-
-    let repo =
-      split.next().context("Build repo has no repo after the /")?;
-
-    let github_repos = github.repos();
-
-    let webhooks = github_repos
-      .list_all_webhooks(owner, repo)
-      .await
-      .context("failed to list all webhooks on repo")?
-      .body;
-
-    let CoreConfig {
-      host,
-      webhook_base_url,
-      ..
-    } = core_config();
-
-    let host = if webhook_base_url.is_empty() {
-      host
-    } else {
-      webhook_base_url
-    };
-    let url = format!("{host}/listener/github/build/{}", build.id);
-
-    for webhook in webhooks {
-      if webhook.active && webhook.config.url == url {
-        github_repos
-          .delete_webhook(owner, repo, webhook.id)
-          .await
-          .context("failed to delete webhook")?;
-        return Ok(NoData {});
-      }
-    }
-
-    // No webhook to delete, all good
-    Ok(NoData {})
-  }
-}

bin/core/src/api/write/mod.rs

@@ -19,6 +19,7 @@ use crate::auth::auth_request;
 use super::Variant;
 
 mod action;
+mod alert;
 mod alerter;
 mod build;
 mod builder;
@@ -103,8 +104,6 @@ pub enum WriteRequest {
   RenameStack(RenameStack),
   WriteStackFileContents(WriteStackFileContents),
   RefreshStackCache(RefreshStackCache),
-  CreateStackWebhook(CreateStackWebhook),
-  DeleteStackWebhook(DeleteStackWebhook),
 
   // ==== DEPLOYMENT ====
   CreateDeployment(CreateDeployment),
@@ -122,8 +121,6 @@ pub enum WriteRequest {
   RenameBuild(RenameBuild),
   WriteBuildFileContents(WriteBuildFileContents),
   RefreshBuildCache(RefreshBuildCache),
-  CreateBuildWebhook(CreateBuildWebhook),
-  DeleteBuildWebhook(DeleteBuildWebhook),
 
   // ==== BUILDER ====
   CreateBuilder(CreateBuilder),
@@ -139,8 +136,6 @@ pub enum WriteRequest {
   UpdateRepo(UpdateRepo),
   RenameRepo(RenameRepo),
   RefreshRepoCache(RefreshRepoCache),
-  CreateRepoWebhook(CreateRepoWebhook),
-  DeleteRepoWebhook(DeleteRepoWebhook),
 
   // ==== ALERTER ====
   CreateAlerter(CreateAlerter),
@@ -172,8 +167,6 @@ pub enum WriteRequest {
   WriteSyncFileContents(WriteSyncFileContents),
   CommitSync(CommitSync),
   RefreshResourceSyncPending(RefreshResourceSyncPending),
-  CreateSyncWebhook(CreateSyncWebhook),
-  DeleteSyncWebhook(DeleteSyncWebhook),
 
   // ==== TAG ====
   CreateTag(CreateTag),
@@ -200,6 +193,9 @@ pub enum WriteRequest {
   CreateOnboardingKey(CreateOnboardingKey),
   UpdateOnboardingKey(UpdateOnboardingKey),
   DeleteOnboardingKey(DeleteOnboardingKey),
+
+  // ==== ALERT ====
+  CloseAlert(CloseAlert),
 }
 
 pub fn router() -> Router {

bin/core/src/api/write/repo.rs

@@ -1,4 +1,4 @@
-use anyhow::{Context, anyhow};
+use anyhow::Context;
 use database::mongo_indexed::doc;
 use database::mungos::{
   by_id::update_one_by_id, mongodb::bson::to_document,
@@ -7,19 +7,14 @@ use formatting::format_serror;
 use komodo_client::{
   api::write::*,
   entities::{
-    NoData, Operation, RepoExecutionArgs,
-    config::core::CoreConfig,
-    komodo_timestamp,
+    NoData, Operation, RepoExecutionArgs, komodo_timestamp,
     permission::PermissionLevel,
-    repo::{PartialRepoConfig, Repo, RepoInfo},
+    repo::{Repo, RepoInfo},
     server::Server,
     to_path_compatible_name,
     update::{Log, Update},
   },
 };
-use octorust::types::{
-  ReposCreateWebhookRequest, ReposCreateWebhookRequestConfig,
-};
 use periphery_client::api;
 use resolver_api::Resolve;
 
@@ -31,7 +26,7 @@ use crate::{
   },
   permission::get_check_permissions,
   resource,
-  state::{action_states, db_client, github_client},
+  state::{action_states, db_client},
 };
 
 use super::WriteArgs;
@@ -235,220 +230,3 @@ impl Resolve<WriteArgs> for RefreshRepoCache {
     Ok(NoData {})
   }
 }
-
-impl Resolve<WriteArgs> for CreateRepoWebhook {
-  #[instrument(name = "CreateRepoWebhook", skip(args))]
-  async fn resolve(
-    self,
-    args: &WriteArgs,
-  ) -> serror::Result<CreateRepoWebhookResponse> {
-    let Some(github) = github_client() else {
-      return Err(
-        anyhow!(
-          "github_webhook_app is not configured in core config toml"
-        )
-        .into(),
-      );
-    };
-
-    let repo = get_check_permissions::<Repo>(
-      &self.repo,
-      &args.user,
-      PermissionLevel::Write.into(),
-    )
-    .await?;
-
-    if repo.config.repo.is_empty() {
-      return Err(
-        anyhow!("No repo configured, can't create webhook").into(),
-      );
-    }
-
-    let mut split = repo.config.repo.split('/');
-    let owner = split.next().context("Repo repo has no owner")?;
-
-    let Some(github) = github.get(owner) else {
-      return Err(
-        anyhow!("Cannot manage repo webhooks under owner {owner}")
-          .into(),
-      );
-    };
-
-    let repo_name =
-      split.next().context("Repo repo has no repo after the /")?;
-
-    let github_repos = github.repos();
-
-    // First make sure the webhook isn't already created (inactive ones are ignored)
-    let webhooks = github_repos
-      .list_all_webhooks(owner, repo_name)
-      .await
-      .context("failed to list all webhooks on repo")?
-      .body;
-
-    let CoreConfig {
-      host,
-      webhook_base_url,
-      webhook_secret,
-      ..
-    } = core_config();
-
-    let webhook_secret = if repo.config.webhook_secret.is_empty() {
-      webhook_secret
-    } else {
-      &repo.config.webhook_secret
-    };
-
-    let host = if webhook_base_url.is_empty() {
-      host
-    } else {
-      webhook_base_url
-    };
-    let url = match self.action {
-      RepoWebhookAction::Clone => {
-        format!("{host}/listener/github/repo/{}/clone", repo.id)
-      }
-      RepoWebhookAction::Pull => {
-        format!("{host}/listener/github/repo/{}/pull", repo.id)
-      }
-      RepoWebhookAction::Build => {
-        format!("{host}/listener/github/repo/{}/build", repo.id)
-      }
-    };
-
-    for webhook in webhooks {
-      if webhook.active && webhook.config.url == url {
-        return Ok(NoData {});
-      }
-    }
-
-    // Now good to create the webhook
-    let request = ReposCreateWebhookRequest {
-      active: Some(true),
-      config: Some(ReposCreateWebhookRequestConfig {
-        url,
-        secret: webhook_secret.to_string(),
-        content_type: String::from("json"),
-        insecure_ssl: None,
-        digest: Default::default(),
-        token: Default::default(),
-      }),
-      events: vec![String::from("push")],
-      name: String::from("web"),
-    };
-    github_repos
-      .create_webhook(owner, repo_name, &request)
-      .await
-      .context("failed to create webhook")?;
-
-    if !repo.config.webhook_enabled {
-      UpdateRepo {
-        id: repo.id,
-        config: PartialRepoConfig {
-          webhook_enabled: Some(true),
-          ..Default::default()
-        },
-      }
-      .resolve(args)
-      .await
-      .map_err(|e| e.error)
-      .context("failed to update repo to enable webhook")?;
-    }
-
-    Ok(NoData {})
-  }
-}
-
-impl Resolve<WriteArgs> for DeleteRepoWebhook {
-  #[instrument(name = "DeleteRepoWebhook", skip(user))]
-  async fn resolve(
-    self,
-    WriteArgs { user }: &WriteArgs,
-  ) -> serror::Result<DeleteRepoWebhookResponse> {
-    let Some(github) = github_client() else {
-      return Err(
-        anyhow!(
-          "github_webhook_app is not configured in core config toml"
-        )
-        .into(),
-      );
-    };
-
-    let repo = get_check_permissions::<Repo>(
-      &self.repo,
-      user,
-      PermissionLevel::Write.into(),
-    )
-    .await?;
-
-    if repo.config.git_provider != "github.com" {
-      return Err(
-        anyhow!("Can only manage github.com repo webhooks").into(),
-      );
-    }
-
-    if repo.config.repo.is_empty() {
-      return Err(
-        anyhow!("No repo configured, can't create webhook").into(),
-      );
-    }
-
-    let mut split = repo.config.repo.split('/');
-    let owner = split.next().context("Repo repo has no owner")?;
-
-    let Some(github) = github.get(owner) else {
-      return Err(
-        anyhow!("Cannot manage repo webhooks under owner {owner}")
-          .into(),
-      );
-    };
-
-    let repo_name =
-      split.next().context("Repo repo has no repo after the /")?;
-
-    let github_repos = github.repos();
-
-    // First make sure the webhook isn't already created (inactive ones are ignored)
-    let webhooks = github_repos
-      .list_all_webhooks(owner, repo_name)
-      .await
-      .context("failed to list all webhooks on repo")?
-      .body;
-
-    let CoreConfig {
-      host,
-      webhook_base_url,
-      ..
-    } = core_config();
-
-    let host = if webhook_base_url.is_empty() {
-      host
-    } else {
-      webhook_base_url
-    };
-    let url = match self.action {
-      RepoWebhookAction::Clone => {
-        format!("{host}/listener/github/repo/{}/clone", repo.id)
-      }
-      RepoWebhookAction::Pull => {
-        format!("{host}/listener/github/repo/{}/pull", repo.id)
-      }
-      RepoWebhookAction::Build => {
-        format!("{host}/listener/github/repo/{}/build", repo.id)
-      }
-    };
-
-    for webhook in webhooks {
-      if webhook.active && webhook.config.url == url {
-        github_repos
-          .delete_webhook(owner, repo_name, webhook.id)
-          .await
-          .context("failed to delete webhook")?;
-        return Ok(NoData {});
-      }
-    }
-
-    // No webhook to delete, all good
-    Ok(NoData {})
-  }
-}

bin/core/src/api/write/stack.rs

@@ -8,18 +8,14 @@ use komodo_client::{
   entities::{
     FileContents, NoData, Operation, RepoExecutionArgs,
     all_logs_success,
-    config::core::CoreConfig,
     permission::PermissionLevel,
     repo::Repo,
     server::ServerState,
-    stack::{PartialStackConfig, Stack, StackInfo},
+    stack::{Stack, StackInfo},
     update::Update,
     user::stack_user,
   },
 };
-use octorust::types::{
-  ReposCreateWebhookRequest, ReposCreateWebhookRequestConfig,
-};
 use periphery_client::api::compose::{
   GetComposeContentsOnHost, GetComposeContentsOnHostResponse,
   WriteComposeContentsToHost,
@@ -40,7 +36,7 @@ use crate::{
     remote::{RemoteComposeContents, get_repo_compose_contents},
     services::extract_services_into_res,
   },
-  state::{db_client, github_client},
+  state::db_client,
 };
 
 use super::WriteArgs;
@@ -567,216 +563,3 @@ impl Resolve<WriteArgs> for RefreshStackCache {
     Ok(NoData {})
   }
 }
-
-impl Resolve<WriteArgs> for CreateStackWebhook {
-  #[instrument(name = "CreateStackWebhook", skip(args))]
-  async fn resolve(
-    self,
-    args: &WriteArgs,
-  ) -> serror::Result<CreateStackWebhookResponse> {
-    let WriteArgs { user } = args;
-
-    let Some(github) = github_client() else {
-      return Err(
-        anyhow!(
-          "github_webhook_app is not configured in core config toml"
-        )
-        .into(),
-      );
-    };
-
-    let stack = get_check_permissions::<Stack>(
-      &self.stack,
-      user,
-      PermissionLevel::Write.into(),
-    )
-    .await?;
-
-    if stack.config.repo.is_empty() {
-      return Err(
-        anyhow!("No repo configured, can't create webhook").into(),
-      );
-    }
-
-    let mut split = stack.config.repo.split('/');
-    let owner = split.next().context("Stack repo has no owner")?;
-
-    let Some(github) = github.get(owner) else {
-      return Err(
-        anyhow!("Cannot manage repo webhooks under owner {owner}")
-          .into(),
-      );
-    };
-
-    let repo =
-      split.next().context("Stack repo has no repo after the /")?;
-
-    let github_repos = github.repos();
-
-    // First make sure the webhook isn't already created (inactive ones are ignored)
-    let webhooks = github_repos
-      .list_all_webhooks(owner, repo)
-      .await
-      .context("failed to list all webhooks on repo")?
-      .body;
-
-    let CoreConfig {
-      host,
-      webhook_base_url,
-      webhook_secret,
-      ..
-    } = core_config();
-
-    let webhook_secret = if stack.config.webhook_secret.is_empty() {
-      webhook_secret
-    } else {
-      &stack.config.webhook_secret
-    };
-
-    let host = if webhook_base_url.is_empty() {
-      host
-    } else {
-      webhook_base_url
-    };
-    let url = match self.action {
-      StackWebhookAction::Refresh => {
-        format!("{host}/listener/github/stack/{}/refresh", stack.id)
-      }
-      StackWebhookAction::Deploy => {
-        format!("{host}/listener/github/stack/{}/deploy", stack.id)
-      }
-    };
-
-    for webhook in webhooks {
-      if webhook.active && webhook.config.url == url {
-        return Ok(NoData {});
-      }
-    }
-
-    // Now good to create the webhook
-    let request = ReposCreateWebhookRequest {
-      active: Some(true),
-      config: Some(ReposCreateWebhookRequestConfig {
-        url,
-        secret: webhook_secret.to_string(),
-        content_type: String::from("json"),
-        insecure_ssl: None,
-        digest: Default::default(),
-        token: Default::default(),
-      }),
-      events: vec![String::from("push")],
-      name: String::from("web"),
-    };
-    github_repos
-      .create_webhook(owner, repo, &request)
-      .await
-      .context("failed to create webhook")?;
-
-    if !stack.config.webhook_enabled {
-      UpdateStack {
-        id: stack.id,
-        config: PartialStackConfig {
-          webhook_enabled: Some(true),
-          ..Default::default()
-        },
-      }
-      .resolve(args)
-      .await
-      .map_err(|e| e.error)
-      .context("failed to update stack to enable webhook")?;
-    }
-
-    Ok(NoData {})
-  }
-}
-
-impl Resolve<WriteArgs> for DeleteStackWebhook {
-  #[instrument(name = "DeleteStackWebhook", skip(user))]
-  async fn resolve(
-    self,
-    WriteArgs { user }: &WriteArgs,
-  ) -> serror::Result<DeleteStackWebhookResponse> {
-    let Some(github) = github_client() else {
-      return Err(
-        anyhow!(
-          "github_webhook_app is not configured in core config toml"
-        )
-        .into(),
-      );
-    };
-
-    let stack = get_check_permissions::<Stack>(
-      &self.stack,
-      user,
-      PermissionLevel::Write.into(),
-    )
-    .await?;
-
-    if stack.config.git_provider != "github.com" {
-      return Err(
-        anyhow!("Can only manage github.com repo webhooks").into(),
-      );
-    }
-
-    if stack.config.repo.is_empty() {
-      return Err(
-        anyhow!("No repo configured, can't create webhook").into(),
-      );
-    }
-
-    let mut split = stack.config.repo.split('/');
-    let owner = split.next().context("Stack repo has no owner")?;
-
-    let Some(github) = github.get(owner) else {
-      return Err(
-        anyhow!("Cannot manage repo webhooks under owner {owner}")
-          .into(),
-      );
-    };
-
-    let repo =
-      split.next().context("Sync repo has no repo after the /")?;
-
-    let github_repos = github.repos();
-
-    // First make sure the webhook isn't already created (inactive ones are ignored)
-    let webhooks = github_repos
-      .list_all_webhooks(owner, repo)
-      .await
-      .context("failed to list all webhooks on repo")?
-      .body;
-
-    let CoreConfig {
-      host,
-      webhook_base_url,
-      ..
-    } = core_config();
-
-    let host = if webhook_base_url.is_empty() {
-      host
-    } else {
-      webhook_base_url
-    };
-    let url = match self.action {
-      StackWebhookAction::Refresh => {
-        format!("{host}/listener/github/stack/{}/refresh", stack.id)
-      }
-      StackWebhookAction::Deploy => {
-        format!("{host}/listener/github/stack/{}/deploy", stack.id)
-      }
-    };
-
-    for webhook in webhooks {
-      if webhook.active && webhook.config.url == url {
-        github_repos
-          .delete_webhook(owner, repo, webhook.id)
-          .await
-          .context("failed to delete webhook")?;
-        return Ok(NoData {});
-      }
-    }
-
-    // No webhook to delete, all good
-    Ok(NoData {})
-  }
-}

bin/core/src/api/write/sync.rs

@@ -12,14 +12,13 @@ use formatting::format_serror;
 use komodo_client::{
   api::{read::ExportAllResourcesToToml, write::*},
   entities::{
-    self, NoData, Operation, RepoExecutionArgs, ResourceTarget,
+    self, Operation, RepoExecutionArgs, ResourceTarget,
     action::Action,
     alert::{Alert, AlertData, SeverityLevel},
     alerter::Alerter,
     all_logs_success,
     build::Build,
     builder::Builder,
-    config::core::CoreConfig,
     deployment::Deployment,
     komodo_timestamp,
     permission::PermissionLevel,
@@ -27,18 +26,12 @@ use komodo_client::{
     repo::Repo,
     server::Server,
     stack::Stack,
-    sync::{
-      PartialResourceSyncConfig, ResourceSync, ResourceSyncInfo,
-      SyncDeployUpdate,
-    },
+    sync::{ResourceSync, ResourceSyncInfo, SyncDeployUpdate},
     to_path_compatible_name,
     update::{Log, Update},
     user::sync_user,
   },
 };
-use octorust::types::{
-  ReposCreateWebhookRequest, ReposCreateWebhookRequestConfig,
-};
 use resolver_api::Resolve;
 
 use crate::{
@@ -53,7 +46,7 @@ use crate::{
   },
   permission::get_check_permissions,
   resource,
-  state::{db_client, github_client},
+  state::db_client,
   sync::{
     deploy::SyncDeployParams, remote::RemoteResources,
     view::push_updates_for_view,
@@ -206,15 +199,7 @@ async fn write_sync_file_contents_on_host(
     .context("Invalid resource path")?;
   let full_path = root.join(&resource_path).join(&file_path);
 
-  if let Some(parent) = full_path.parent() {
-    tokio::fs::create_dir_all(parent).await.with_context(|| {
-      format!(
-        "Failed to initialize resource file parent directory {parent:?}"
-      )
-    })?;
-  }
-
-  if let Err(e) = tokio::fs::write(&full_path, &contents)
+  if let Err(e) = secret_file::write_async(&full_path, &contents)
     .await
     .with_context(|| {
       format!(
@@ -491,12 +476,7 @@ impl Resolve<WriteArgs> for CommitSync {
         .sync_directory
         .join(to_path_compatible_name(&sync.name))
         .join(&resource_path);
-      if let Some(parent) = file_path.parent() {
-        tokio::fs::create_dir_all(parent)
-          .await
-          .with_context(|| format!("Failed to initialize resource file parent directory {parent:?}"))?;
-      };
-      if let Err(e) = tokio::fs::write(&file_path, &res.toml)
+      if let Err(e) = secret_file::write_async(&file_path, &res.toml)
         .await
         .with_context(|| {
           format!("Failed to write resource file to {file_path:?}",)
@@ -982,215 +962,3 @@ impl Resolve<WriteArgs> for RefreshResourceSyncPending {
     Ok(crate::resource::get::<ResourceSync>(&sync.id).await?)
   }
 }
-
-impl Resolve<WriteArgs> for CreateSyncWebhook {
-  #[instrument(name = "CreateSyncWebhook", skip(args))]
-  async fn resolve(
-    self,
-    args: &WriteArgs,
-  ) -> serror::Result<CreateSyncWebhookResponse> {
-    let WriteArgs { user } = args;
-    let Some(github) = github_client() else {
-      return Err(
-        anyhow!(
-          "github_webhook_app is not configured in core config toml"
-        )
-        .into(),
-      );
-    };
-
-    let sync = get_check_permissions::<ResourceSync>(
-      &self.sync,
-      user,
-      PermissionLevel::Write.into(),
-    )
-    .await?;
-
-    if sync.config.repo.is_empty() {
-      return Err(
-        anyhow!("No repo configured, can't create webhook").into(),
-      );
-    }
-
-    let mut split = sync.config.repo.split('/');
-    let owner = split.next().context("Sync repo has no owner")?;
-
-    let Some(github) = github.get(owner) else {
-      return Err(
-        anyhow!("Cannot manage repo webhooks under owner {owner}")
-          .into(),
-      );
-    };
-
-    let repo =
-      split.next().context("Repo repo has no repo after the /")?;
-
-    let github_repos = github.repos();
-
-    // First make sure the webhook isn't already created (inactive ones are ignored)
-    let webhooks = github_repos
-      .list_all_webhooks(owner, repo)
-      .await
-      .context("failed to list all webhooks on repo")?
-      .body;
-
-    let CoreConfig {
-      host,
-      webhook_base_url,
-      webhook_secret,
-      ..
-    } = core_config();
-
-    let webhook_secret = if sync.config.webhook_secret.is_empty() {
-      webhook_secret
-    } else {
-      &sync.config.webhook_secret
-    };
-
-    let host = if webhook_base_url.is_empty() {
-      host
-    } else {
-      webhook_base_url
-    };
-    let url = match self.action {
-      SyncWebhookAction::Refresh => {
-        format!("{host}/listener/github/sync/{}/refresh", sync.id)
-      }
-      SyncWebhookAction::Sync => {
-        format!("{host}/listener/github/sync/{}/sync", sync.id)
-      }
-    };
-
-    for webhook in webhooks {
-      if webhook.active && webhook.config.url == url {
-        return Ok(NoData {});
-      }
-    }
-
-    // Now good to create the webhook
-    let request = ReposCreateWebhookRequest {
-      active: Some(true),
-      config: Some(ReposCreateWebhookRequestConfig {
-        url,
-        secret: webhook_secret.to_string(),
-        content_type: String::from("json"),
-        insecure_ssl: None,
-        digest: Default::default(),
-        token: Default::default(),
-      }),
-      events: vec![String::from("push")],
-      name: String::from("web"),
-    };
-    github_repos
-      .create_webhook(owner, repo, &request)
-      .await
-      .context("failed to create webhook")?;
-
-    if !sync.config.webhook_enabled {
-      UpdateResourceSync {
-        id: sync.id,
-        config: PartialResourceSyncConfig {
-          webhook_enabled: Some(true),
-          ..Default::default()
-        },
-      }
-      .resolve(args)
-      .await
-      .map_err(|e| e.error)
-      .context("failed to update sync to enable webhook")?;
-    }
-
-    Ok(NoData {})
-  }
-}
-
-impl Resolve<WriteArgs> for DeleteSyncWebhook {
-  #[instrument(name = "DeleteSyncWebhook", skip(user))]
-  async fn resolve(
-    self,
-    WriteArgs { user }: &WriteArgs,
-  ) -> serror::Result<DeleteSyncWebhookResponse> {
-    let Some(github) = github_client() else {
-      return Err(
-        anyhow!(
-          "github_webhook_app is not configured in core config toml"
-        )
-        .into(),
-      );
-    };
-
-    let sync = get_check_permissions::<ResourceSync>(
-      &self.sync,
-      user,
-      PermissionLevel::Write.into(),
-    )
-    .await?;
-
-    if sync.config.git_provider != "github.com" {
-      return Err(
-        anyhow!("Can only manage github.com repo webhooks").into(),
-      );
-    }
-
-    if sync.config.repo.is_empty() {
-      return Err(
-        anyhow!("No repo configured, can't create webhook").into(),
-      );
-    }
-
-    let mut split = sync.config.repo.split('/');
-    let owner = split.next().context("Sync repo has no owner")?;
-
-    let Some(github) = github.get(owner) else {
-      return Err(
-        anyhow!("Cannot manage repo webhooks under owner {owner}")
-          .into(),
-      );
-    };
-
-    let repo =
-      split.next().context("Sync repo has no repo after the /")?;
-
-    let github_repos = github.repos();
-
-    // First make sure the webhook isn't already created (inactive ones are ignored)
-    let webhooks = github_repos
-      .list_all_webhooks(owner, repo)
-      .await
-      .context("failed to list all webhooks on repo")?
-      .body;
-
-    let CoreConfig {
-      host,
-      webhook_base_url,
-      ..
-    } = core_config();
-
-    let host = if webhook_base_url.is_empty() {
-      host
-    } else {
-      webhook_base_url
-    };
-    let url = match self.action {
-      SyncWebhookAction::Refresh => {
-        format!("{host}/listener/github/sync/{}/refresh", sync.id)
-      }
-      SyncWebhookAction::Sync => {
-        format!("{host}/listener/github/sync/{}/sync", sync.id)
-      }
-    };
-
-    for webhook in webhooks {
-      if webhook.active && webhook.config.url == url {
-        github_repos
-          .delete_webhook(owner, repo, webhook.id)
-          .await
-          .context("failed to delete webhook")?;
-        return Ok(NoData {});
-      }
-    }
-
-    // No webhook to delete, all good
-    Ok(NoData {})
-  }
-}

bin/core/src/auth/jwt.rs

@@ -18,7 +18,7 @@ use crate::helpers::random_string;
 
 type ExchangeTokenMap = Mutex<HashMap<String, (JwtResponse, u128)>>;
 
-#[derive(Serialize, Deserialize)]
+#[derive(Serialize, Deserialize, Clone)]
 pub struct JwtClaims {
   pub id: String,
   pub iat: u128,

bin/core/src/config.rs

@@ -9,36 +9,20 @@ use environment_file::{
 use komodo_client::entities::{
   config::{
     DatabaseConfig,
-    core::{
-      AwsCredentials, CoreConfig, Env, GithubWebhookAppConfig,
-      GithubWebhookAppInstallationConfig, OauthCredentials,
-    },
+    core::{AwsCredentials, CoreConfig, Env, OauthCredentials},
   },
   logger::LogConfig,
 };
-use noise::key::{SpkiPublicKey, load_maybe_generate_private_key};
+use noise::key::{RotatableKeyPair, SpkiPublicKey};
 
 /// Should call in startup to ensure Core errors without valid private key.
-pub fn core_private_key() -> &'static String {
-  static CORE_PRIVATE_KEY: OnceLock<String> = OnceLock::new();
-  CORE_PRIVATE_KEY.get_or_init(|| {
-    let config = core_config();
-    if let Some(path) = config.private_key.strip_prefix("file:") {
-      load_maybe_generate_private_key(path).unwrap()
-    } else {
-      config.private_key.clone()
-    }
-  })
-}
-
-/// Should call in startup to ensure Core errors without valid private key.
-pub fn core_public_key() -> &'static String {
-  static CORE_PUBLIC_KEY: OnceLock<String> = OnceLock::new();
-  CORE_PUBLIC_KEY.get_or_init(|| {
-    SpkiPublicKey::from_private_key(core_private_key())
-      .context("Got invalid private key")
-      .unwrap()
-      .into_inner()
+pub fn core_keys() -> &'static RotatableKeyPair {
+  static CORE_KEYS: OnceLock<RotatableKeyPair> = OnceLock::new();
+  CORE_KEYS.get_or_init(|| {
+    RotatableKeyPair::from_private_key_spec(
+      &core_config().private_key,
+    )
+    .unwrap()
   })
 }
 
@@ -109,12 +93,13 @@ pub fn core_config() -> &'static CoreConfig {
   static CORE_CONFIG: OnceLock<CoreConfig> = OnceLock::new();
   CORE_CONFIG.get_or_init(|| {
     let env: Env = match envy::from_env()
-      .context("Failed to parse Komodo Core environment") {
-        Ok(env) => env,
-        Err(e) => {
-          panic!("{e:?}");
-        }
-      };
+      .context("Failed to parse Komodo Core environment")
+    {
+      Ok(env) => env,
+      Err(e) => {
+        panic!("{e:?}");
+      }
+    };
     let config = if env.komodo_config_paths.is_empty() {
       println!(
         "{}: No config paths found, using default config",
@@ -122,7 +107,8 @@ pub fn core_config() -> &'static CoreConfig {
       );
       CoreConfig::default()
     } else {
-      let config_keywords = env.komodo_config_keywords
+      let config_keywords = env
+        .komodo_config_keywords
         .iter()
         .map(String::as_str)
         .collect::<Vec<_>>();
@@ -132,7 +118,8 @@ pub fn core_config() -> &'static CoreConfig {
         "Config File Keywords".dimmed(),
       );
       (ConfigLoader {
-        paths: &env.komodo_config_paths
+        paths: &env
+          .komodo_config_paths
           .iter()
           .map(PathBuf::as_path)
           .collect::<Vec<_>>(),
@@ -141,57 +128,53 @@ pub fn core_config() -> &'static CoreConfig {
         merge_nested: env.komodo_merge_nested_config,
         extend_array: env.komodo_extend_config_arrays,
         debug_print: env.komodo_config_debug,
-      }).load::<CoreConfig>()
+      })
+      .load::<CoreConfig>()
       .expect("Failed at parsing config from paths")
     };
 
-    let installations = match (
-      maybe_read_list_from_file(
-        env.komodo_github_webhook_app_installations_ids_file,
-        env.komodo_github_webhook_app_installations_ids
-      ),
-      env.komodo_github_webhook_app_installations_namespaces
-    ) {
-      (Some(ids), Some(namespaces)) => {
-        if ids.len() != namespaces.len() {
-          panic!("KOMODO_GITHUB_WEBHOOK_APP_INSTALLATIONS_IDS length and KOMODO_GITHUB_WEBHOOK_APP_INSTALLATIONS_NAMESPACES length mismatch. Got {ids:?} and {namespaces:?}")
-        }
-        ids
-          .into_iter()
-          .zip(namespaces)
-          .map(|(id, namespace)| GithubWebhookAppInstallationConfig {
-            id,
-            namespace
-          })
-          .collect()
-      },
-      (Some(_), None) | (None, Some(_)) => {
-        panic!("Got only one of KOMODO_GITHUB_WEBHOOK_APP_INSTALLATIONS_IDS or KOMODO_GITHUB_WEBHOOK_APP_INSTALLATIONS_NAMESPACES, both MUST be provided");
-      }
-      (None, None) => {
-        config.github_webhook_app.installations
-      }
-    };
-
     // recreating CoreConfig here makes sure apply all env overrides applied.
     CoreConfig {
       // Secret things overridden with file
-      private_key: maybe_read_item_from_file(env.komodo_private_key_file, env.komodo_private_key)
-        .unwrap_or(config.private_key),
-      passkey: maybe_read_item_from_file(env.komodo_passkey_file, env.komodo_passkey)
-        .or(config.passkey),
-      jwt_secret: maybe_read_item_from_file(env.komodo_jwt_secret_file, env.komodo_jwt_secret).unwrap_or(config.jwt_secret),
-      webhook_secret: maybe_read_item_from_file(env.komodo_webhook_secret_file, env.komodo_webhook_secret)
-        .unwrap_or(config.webhook_secret),
+      private_key: maybe_read_item_from_file(
+        env.komodo_private_key_file,
+        env.komodo_private_key,
+      )
+      .unwrap_or(config.private_key),
+      passkey: maybe_read_item_from_file(
+        env.komodo_passkey_file,
+        env.komodo_passkey,
+      )
+      .or(config.passkey),
+      jwt_secret: maybe_read_item_from_file(
+        env.komodo_jwt_secret_file,
+        env.komodo_jwt_secret,
+      )
+      .unwrap_or(config.jwt_secret),
+      webhook_secret: maybe_read_item_from_file(
+        env.komodo_webhook_secret_file,
+        env.komodo_webhook_secret,
+      )
+      .unwrap_or(config.webhook_secret),
       database: DatabaseConfig {
-        uri: maybe_read_item_from_file(env.komodo_database_uri_file,env.komodo_database_uri).unwrap_or(config.database.uri),
-        address: env.komodo_database_address.unwrap_or(config.database.address),
-        username: maybe_read_item_from_file(env.komodo_database_username_file,env
-          .komodo_database_username)
-          .unwrap_or(config.database.username),
-        password: maybe_read_item_from_file(env.komodo_database_password_file,env
-          .komodo_database_password)
-          .unwrap_or(config.database.password),
+        uri: maybe_read_item_from_file(
+          env.komodo_database_uri_file,
+          env.komodo_database_uri,
+        )
+        .unwrap_or(config.database.uri),
+        address: env
+          .komodo_database_address
+          .unwrap_or(config.database.address),
+        username: maybe_read_item_from_file(
+          env.komodo_database_username_file,
+          env.komodo_database_username,
+        )
+        .unwrap_or(config.database.username),
+        password: maybe_read_item_from_file(
+          env.komodo_database_password_file,
+          env.komodo_database_password,
+        )
+        .unwrap_or(config.database.password),
         app_name: env
           .komodo_database_app_name
           .unwrap_or(config.database.app_name),
@@ -201,64 +184,82 @@ pub fn core_config() -> &'static CoreConfig {
       },
       init_admin_username: maybe_read_item_from_file(
         env.komodo_init_admin_username_file,
-        env.komodo_init_admin_username
-      ).or(config.init_admin_username),
+        env.komodo_init_admin_username,
+      )
+      .or(config.init_admin_username),
       init_admin_password: maybe_read_item_from_file(
         env.komodo_init_admin_password_file,
-        env.komodo_init_admin_password
-      ).unwrap_or(config.init_admin_password),
-      oidc_enabled: env.komodo_oidc_enabled.unwrap_or(config.oidc_enabled),
-      oidc_provider: env.komodo_oidc_provider.unwrap_or(config.oidc_provider),
-      oidc_redirect_host: env.komodo_oidc_redirect_host.unwrap_or(config.oidc_redirect_host),
-      oidc_client_id: maybe_read_item_from_file(env.komodo_oidc_client_id_file,env
-        .komodo_oidc_client_id)
-        .unwrap_or(config.oidc_client_id),
-      oidc_client_secret: maybe_read_item_from_file(env.komodo_oidc_client_secret_file,env
-        .komodo_oidc_client_secret)
-        .unwrap_or(config.oidc_client_secret),
-      oidc_use_full_email: env.komodo_oidc_use_full_email
+        env.komodo_init_admin_password,
+      )
+      .unwrap_or(config.init_admin_password),
+      oidc_enabled: env
+        .komodo_oidc_enabled
+        .unwrap_or(config.oidc_enabled),
+      oidc_provider: env
+        .komodo_oidc_provider
+        .unwrap_or(config.oidc_provider),
+      oidc_redirect_host: env
+        .komodo_oidc_redirect_host
+        .unwrap_or(config.oidc_redirect_host),
+      oidc_client_id: maybe_read_item_from_file(
+        env.komodo_oidc_client_id_file,
+        env.komodo_oidc_client_id,
+      )
+      .unwrap_or(config.oidc_client_id),
+      oidc_client_secret: maybe_read_item_from_file(
+        env.komodo_oidc_client_secret_file,
+        env.komodo_oidc_client_secret,
+      )
+      .unwrap_or(config.oidc_client_secret),
+      oidc_use_full_email: env
+        .komodo_oidc_use_full_email
         .unwrap_or(config.oidc_use_full_email),
-      oidc_additional_audiences: maybe_read_list_from_file(env.komodo_oidc_additional_audiences_file,env
-        .komodo_oidc_additional_audiences)
-        .unwrap_or(config.oidc_additional_audiences),
+      oidc_additional_audiences: maybe_read_list_from_file(
+        env.komodo_oidc_additional_audiences_file,
+        env.komodo_oidc_additional_audiences,
+      )
+      .unwrap_or(config.oidc_additional_audiences),
       google_oauth: OauthCredentials {
         enabled: env
           .komodo_google_oauth_enabled
           .unwrap_or(config.google_oauth.enabled),
-        id: maybe_read_item_from_file(env.komodo_google_oauth_id_file,env
-          .komodo_google_oauth_id)
-          .unwrap_or(config.google_oauth.id),
-        secret: maybe_read_item_from_file(env.komodo_google_oauth_secret_file,env
-          .komodo_google_oauth_secret)
-          .unwrap_or(config.google_oauth.secret),
+        id: maybe_read_item_from_file(
+          env.komodo_google_oauth_id_file,
+          env.komodo_google_oauth_id,
+        )
+        .unwrap_or(config.google_oauth.id),
+        secret: maybe_read_item_from_file(
+          env.komodo_google_oauth_secret_file,
+          env.komodo_google_oauth_secret,
+        )
+        .unwrap_or(config.google_oauth.secret),
       },
       github_oauth: OauthCredentials {
         enabled: env
           .komodo_github_oauth_enabled
           .unwrap_or(config.github_oauth.enabled),
-        id: maybe_read_item_from_file(env.komodo_github_oauth_id_file,env
-          .komodo_github_oauth_id)
-          .unwrap_or(config.github_oauth.id),
-        secret: maybe_read_item_from_file(env.komodo_github_oauth_secret_file,env
-          .komodo_github_oauth_secret)
-          .unwrap_or(config.github_oauth.secret),
+        id: maybe_read_item_from_file(
+          env.komodo_github_oauth_id_file,
+          env.komodo_github_oauth_id,
+        )
+        .unwrap_or(config.github_oauth.id),
+        secret: maybe_read_item_from_file(
+          env.komodo_github_oauth_secret_file,
+          env.komodo_github_oauth_secret,
+        )
+        .unwrap_or(config.github_oauth.secret),
       },
       aws: AwsCredentials {
-        access_key_id: maybe_read_item_from_file(env.komodo_aws_access_key_id_file, env
-          .komodo_aws_access_key_id)
-          .unwrap_or(config.aws.access_key_id),
-        secret_access_key: maybe_read_item_from_file(env.komodo_aws_secret_access_key_file, env
-          .komodo_aws_secret_access_key)
-          .unwrap_or(config.aws.secret_access_key),
-      },
-      github_webhook_app: GithubWebhookAppConfig {
-        app_id: maybe_read_item_from_file(env.komodo_github_webhook_app_app_id_file, env
-          .komodo_github_webhook_app_app_id)
-          .unwrap_or(config.github_webhook_app.app_id),
-        pk_path: env
-          .komodo_github_webhook_app_pk_path
-          .unwrap_or(config.github_webhook_app.pk_path),
-        installations,
+        access_key_id: maybe_read_item_from_file(
+          env.komodo_aws_access_key_id_file,
+          env.komodo_aws_access_key_id,
+        )
+        .unwrap_or(config.aws.access_key_id),
+        secret_access_key: maybe_read_item_from_file(
+          env.komodo_aws_secret_access_key_file,
+          env.komodo_aws_secret_access_key,
+        )
+        .unwrap_or(config.aws.secret_access_key),
       },
 
       // Non secrets
@@ -267,13 +268,19 @@ pub fn core_config() -> &'static CoreConfig {
       port: env.komodo_port.unwrap_or(config.port),
       bind_ip: env.komodo_bind_ip.unwrap_or(config.bind_ip),
       timezone: env.komodo_timezone.unwrap_or(config.timezone),
-      periphery_public_keys: env.komodo_periphery_public_keys.or(config.periphery_public_keys),
-      first_server_address: env.komodo_first_server_address.or(config.first_server_address),
-      first_server_name: env.komodo_first_server_name.or(config.first_server_name),
-      frontend_path: env.komodo_frontend_path.unwrap_or(config.frontend_path),
-      jwt_ttl: env
-        .komodo_jwt_ttl
-        .unwrap_or(config.jwt_ttl),
+      periphery_public_keys: env
+        .komodo_periphery_public_keys
+        .or(config.periphery_public_keys),
+      first_server_address: env
+        .komodo_first_server_address
+        .or(config.first_server_address),
+      first_server_name: env
+        .komodo_first_server_name
+        .or(config.first_server_name),
+      frontend_path: env
+        .komodo_frontend_path
+        .unwrap_or(config.frontend_path),
+      jwt_ttl: env.komodo_jwt_ttl.unwrap_or(config.jwt_ttl),
       sync_directory: env
         .komodo_sync_directory
         .unwrap_or(config.sync_directory),
@@ -304,24 +311,31 @@ pub fn core_config() -> &'static CoreConfig {
       ui_write_disabled: env
         .komodo_ui_write_disabled
         .unwrap_or(config.ui_write_disabled),
-      disable_confirm_dialog: env.komodo_disable_confirm_dialog
+      disable_confirm_dialog: env
+        .komodo_disable_confirm_dialog
         .unwrap_or(config.disable_confirm_dialog),
-      disable_websocket_reconnect: env.komodo_disable_websocket_reconnect
+      disable_websocket_reconnect: env
+        .komodo_disable_websocket_reconnect
         .unwrap_or(config.disable_websocket_reconnect),
-      enable_new_users: env.komodo_enable_new_users
+      enable_new_users: env
+        .komodo_enable_new_users
         .unwrap_or(config.enable_new_users),
-      disable_user_registration: env.komodo_disable_user_registration
+      disable_user_registration: env
+        .komodo_disable_user_registration
         .unwrap_or(config.disable_user_registration),
-      disable_non_admin_create: env.komodo_disable_non_admin_create
+      disable_non_admin_create: env
+        .komodo_disable_non_admin_create
         .unwrap_or(config.disable_non_admin_create),
-      disable_init_resources: env.komodo_disable_init_resources
+      disable_init_resources: env
+        .komodo_disable_init_resources
         .unwrap_or(config.disable_init_resources),
-      enable_fancy_toml: env.komodo_enable_fancy_toml
+      enable_fancy_toml: env
+        .komodo_enable_fancy_toml
         .unwrap_or(config.enable_fancy_toml),
-      lock_login_credentials_for: env.komodo_lock_login_credentials_for
+      lock_login_credentials_for: env
+        .komodo_lock_login_credentials_for
         .unwrap_or(config.lock_login_credentials_for),
-      local_auth: env.komodo_local_auth
-        .unwrap_or(config.local_auth),
+      local_auth: env.komodo_local_auth.unwrap_or(config.local_auth),
       logging: LogConfig {
         level: env
           .komodo_logging_level
@@ -329,9 +343,11 @@ pub fn core_config() -> &'static CoreConfig {
         stdio: env
           .komodo_logging_stdio
           .unwrap_or(config.logging.stdio),
-        pretty: env.komodo_logging_pretty
+        pretty: env
+          .komodo_logging_pretty
           .unwrap_or(config.logging.pretty),
-        location: env.komodo_logging_location
+        location: env
+          .komodo_logging_location
           .unwrap_or(config.logging.location),
         otlp_endpoint: env
           .komodo_logging_otlp_endpoint
@@ -340,12 +356,24 @@ pub fn core_config() -> &'static CoreConfig {
           .komodo_logging_opentelemetry_service_name
           .unwrap_or(config.logging.opentelemetry_service_name),
       },
-      pretty_startup_config: env.komodo_pretty_startup_config.unwrap_or(config.pretty_startup_config),
-      unsafe_unsanitized_startup_config: env.komodo_unsafe_unsanitized_startup_config.unwrap_or(config.unsafe_unsanitized_startup_config),
-      internet_interface: env.komodo_internet_interface.unwrap_or(config.internet_interface),
-      ssl_enabled: env.komodo_ssl_enabled.unwrap_or(config.ssl_enabled),
-      ssl_key_file: env.komodo_ssl_key_file.unwrap_or(config.ssl_key_file),
-      ssl_cert_file: env.komodo_ssl_cert_file.unwrap_or(config.ssl_cert_file),
+      pretty_startup_config: env
+        .komodo_pretty_startup_config
+        .unwrap_or(config.pretty_startup_config),
+      unsafe_unsanitized_startup_config: env
+        .komodo_unsafe_unsanitized_startup_config
+        .unwrap_or(config.unsafe_unsanitized_startup_config),
+      internet_interface: env
+        .komodo_internet_interface
+        .unwrap_or(config.internet_interface),
+      ssl_enabled: env
+        .komodo_ssl_enabled
+        .unwrap_or(config.ssl_enabled),
+      ssl_key_file: env
+        .komodo_ssl_key_file
+        .unwrap_or(config.ssl_key_file),
+      ssl_cert_file: env
+        .komodo_ssl_cert_file
+        .unwrap_or(config.ssl_cert_file),
 
       // These can't be overridden on env
       secrets: config.secrets,

bin/core/src/connection/client.rs

@@ -1,21 +1,24 @@
-use std::{sync::Arc, time::Duration};
+use std::time::Duration;
 
 use anyhow::{Context, anyhow};
-use periphery_client::CONNECTION_RETRY_SECONDS;
-use serror::{deserialize_error_bytes, serialize_error_bytes};
+use periphery_client::{
+  CONNECTION_RETRY_SECONDS, transport::LoginMessage,
+};
 use transport::{
-  MessageState,
   auth::{
     AddressConnectionIdentifiers, ClientLoginFlow,
     ConnectionIdentifiers,
   },
   fix_ws_address,
-  websocket::{Websocket, tungstenite::TungsteniteWebsocket},
+  websocket::{
+    Websocket, WebsocketExt as _, login::LoginWebsocketExt,
+    tungstenite::TungsteniteWebsocket,
+  },
 };
 
 use crate::{
   config::{core_config, core_connection_query},
-  periphery::ConnectionChannels,
+  periphery::PeripheryClient,
   state::periphery_connections,
 };
 
@@ -26,8 +29,7 @@ impl PeripheryConnectionArgs<'_> {
     self,
     id: String,
     insecure: bool,
-    passkey: String,
-  ) -> anyhow::Result<Arc<ConnectionChannels>> {
+  ) -> anyhow::Result<PeripheryClient> {
     let Some(address) = self.address else {
       return Err(anyhow!(
         "Cannot spawn client connection with empty address"
@@ -42,7 +44,8 @@ impl PeripheryConnectionArgs<'_> {
     let (connection, mut receiver) =
       periphery_connections().insert(id.clone(), self).await;
 
-    let channels = connection.channels.clone();
+    let responses = connection.responses.clone();
+    let terminals = connection.terminals.clone();
 
     tokio::spawn(async move {
       loop {
@@ -73,9 +76,8 @@ impl PeripheryConnectionArgs<'_> {
           core_connection_query().as_bytes(),
         );
 
-        if let Err(e) = connection
-          .client_login(&mut socket, identifiers, &passkey)
-          .await
+        if let Err(e) =
+          connection.client_login(&mut socket, identifiers).await
         {
           connection.set_error(e).await;
           tokio::time::sleep(Duration::from_secs(
@@ -89,7 +91,11 @@ impl PeripheryConnectionArgs<'_> {
       }
     });
 
-    Ok(channels)
+    Ok(PeripheryClient {
+      id,
+      responses,
+      terminals,
+    })
   }
 }
 
@@ -100,28 +106,20 @@ impl PeripheryConnection {
     &self,
     socket: &mut TungsteniteWebsocket,
     identifiers: ConnectionIdentifiers<'_>,
-    // for legacy auth
-    passkey: &str,
   ) -> anyhow::Result<()> {
     // Get the required auth type
-    let bytes = socket
-      .recv_bytes()
-      .with_timeout(Duration::from_secs(2))
-      .await?
-      .context("Failed to receive login type indicator")?;
+    let v1_passkey_flow =
+      socket
+        .recv_login_v1_passkey_flow()
+        .await
+        .context("Failed to receive Login V1PasskeyFlow message")?;
 
-    match bytes.iter().as_slice() {
-      // Noise auth
-      &[0] => {
-        self
-          .handle_login::<_, ClientLoginFlow>(socket, identifiers)
-          .await
-      }
-      // Passkey auth
-      &[1] => handle_passkey_login(socket, passkey).await,
-      other => Err(anyhow!(
-        "Receieved invalid login type pattern: {other:?}"
-      )),
+    if v1_passkey_flow {
+      handle_passkey_login(socket, self.args.passkey.as_deref()).await
+    } else {
+      self
+        .handle_login::<_, ClientLoginFlow>(socket, identifiers)
+        .await
     }
   }
 }
@@ -129,47 +127,37 @@ impl PeripheryConnection {
 async fn handle_passkey_login(
   socket: &mut TungsteniteWebsocket,
   // for legacy auth
-  passkey: &str,
+  passkey: Option<&str>,
 ) -> anyhow::Result<()> {
   let res = async {
-    let mut passkey = if passkey.is_empty() {
+    let passkey = if let Some(passkey) = passkey {
+      passkey.as_bytes().to_vec()
+    } else {
       core_config()
         .passkey
         .as_deref()
         .context("Periphery requires passkey auth")?
         .as_bytes()
         .to_vec()
-    } else {
-      passkey.as_bytes().to_vec()
     };
-    passkey.push(MessageState::Successful.as_byte());
 
     socket
-      .send(passkey.into())
+      .send_message(LoginMessage::V1Passkey(passkey))
       .await
-      .context("Failed to send passkey")?;
+      .context("Failed to send Login V1Passkey message")?;
 
     // Receive login state message and return based on value
-    let state_msg = socket
-      .recv_bytes()
+    socket
+      .recv_login_success()
       .await
-      .context("Failed to receive authentication state message")?;
-    let state = state_msg.last().context(
-      "Authentication state message did not contain state byte",
-    )?;
-    match MessageState::from_byte(*state) {
-      MessageState::Successful => anyhow::Ok(()),
-      _ => Err(deserialize_error_bytes(
-        &state_msg[..(state_msg.len() - 1)],
-      )),
-    }
+      .context("Failed to receive Login Success message")?;
+
+    anyhow::Ok(())
   }
   .await;
   if let Err(e) = res {
-    let mut bytes = serialize_error_bytes(&e);
-    bytes.push(MessageState::Failed.as_byte());
     if let Err(e) = socket
-      .send(bytes.into())
+      .send_login_error(&e)
       .await
       .context("Failed to send login failed to client")
     {
@@ -177,7 +165,7 @@ async fn handle_passkey_login(
       warn!("{e:#}");
     }
     // Close socket
-    let _ = socket.close(None).await;
+    let _ = socket.close().await;
     // Return the original error
     Err(e)
   } else {

bin/core/src/connection/mod.rs

@@ -7,36 +7,38 @@ use std::{
 };
 
 use anyhow::anyhow;
-use bytes::Bytes;
 use cache::CloneCache;
 use database::mungos::{by_id::update_one_by_id, mongodb::bson::doc};
+use encoding::{
+  CastBytes as _, Decode as _, EncodedJsonMessage, EncodedResponse,
+  WithChannel,
+};
 use komodo_client::entities::{
   builder::{AwsBuilderConfig, UrlBuilderConfig},
   optional_str,
   server::Server,
 };
-use serror::serror_into_anyhow_error;
-use tokio::sync::{
-  RwLock,
-  mpsc::{Sender, error::SendError},
+use periphery_client::transport::{
+  EncodedTransportMessage, ResponseMessage, TransportMessage,
 };
+use serror::serror_into_anyhow_error;
+use tokio::sync::RwLock;
 use tokio_util::sync::CancellationToken;
 use transport::{
   auth::{
     ConnectionIdentifiers, LoginFlow, LoginFlowArgs,
     PublicKeyValidator,
   },
-  bytes::id_from_transport_bytes,
-  channel::{BufferedReceiver, buffered_channel},
+  channel::{BufferedReceiver, Sender, buffered_channel},
   websocket::{
     Websocket, WebsocketMessage, WebsocketReceiver as _,
     WebsocketSender as _,
   },
 };
+use uuid::Uuid;
 
 use crate::{
-  config::{core_private_key, periphery_public_keys},
-  periphery::ConnectionChannels,
+  config::{core_keys, periphery_public_keys},
   state::db_client,
 };
 
@@ -56,7 +58,10 @@ impl PeripheryConnections {
     &self,
     server_id: String,
     args: PeripheryConnectionArgs<'_>,
-  ) -> (Arc<PeripheryConnection>, BufferedReceiver<Bytes>) {
+  ) -> (
+    Arc<PeripheryConnection>,
+    BufferedReceiver<EncodedTransportMessage>,
+  ) {
     let (connection, receiver) = if let Some(existing_connection) =
       self.0.remove(&server_id).await
     {
@@ -97,6 +102,9 @@ pub struct PeripheryConnectionArgs<'a> {
   pub id: &'a str,
   pub address: Option<&'a str>,
   periphery_public_key: Option<&'a str>,
+  /// V1 legacy support.
+  /// Only possible for Core -> Periphery.
+  passkey: Option<&'a str>,
 }
 
 impl PublicKeyValidator for PeripheryConnectionArgs<'_> {
@@ -110,12 +118,11 @@ impl PublicKeyValidator for PeripheryConnectionArgs<'_> {
         self.id.to_string(),
         Some(public_key.clone()),
       );
-      let e = anyhow!("{public_key} is invalid")
+      anyhow!("{public_key} is invalid")
         .context(
           "Ensure public key matches configured Periphery Public Key",
         )
-        .context("Core failed to validate Periphery public key");
-      e
+        .context("Core failed to validate Periphery public key")
     };
     let core_to_periphery = self.address.is_some();
     match (self.periphery_public_key, core_to_periphery) {
@@ -150,6 +157,7 @@ impl<'a> PeripheryConnectionArgs<'a> {
       id: &server.id,
       address: optional_str(&server.config.address),
       periphery_public_key: optional_str(&server.info.public_key),
+      passkey: optional_str(&server.config.passkey),
     }
   }
 
@@ -163,6 +171,7 @@ impl<'a> PeripheryConnectionArgs<'a> {
       periphery_public_key: optional_str(
         &config.periphery_public_key,
       ),
+      passkey: optional_str(&config.passkey),
     }
   }
 
@@ -177,6 +186,7 @@ impl<'a> PeripheryConnectionArgs<'a> {
       periphery_public_key: optional_str(
         &config.periphery_public_key,
       ),
+      passkey: None,
     }
   }
 
@@ -187,6 +197,7 @@ impl<'a> PeripheryConnectionArgs<'a> {
       periphery_public_key: self
         .periphery_public_key
         .map(str::to_string),
+      passkey: self.passkey.map(str::to_string),
     }
   }
 
@@ -209,6 +220,9 @@ pub struct OwnedPeripheryConnectionArgs {
   /// If None, must have 'periphery_public_keys' set
   /// in Core config, or will error
   pub periphery_public_key: Option<String>,
+  /// V1 legacy support.
+  /// Only possible for Core -> Periphery connection.
+  pub passkey: Option<String>,
 }
 
 impl OwnedPeripheryConnectionArgs {
@@ -217,6 +231,7 @@ impl OwnedPeripheryConnectionArgs {
       id: &self.id,
       address: self.address.as_deref(),
       periphery_public_key: self.periphery_public_key.as_deref(),
+      passkey: self.passkey.as_deref(),
     }
   }
 }
@@ -237,12 +252,18 @@ impl<'a> From<&'a OwnedPeripheryConnectionArgs>
   }
 }
 
+/// Sends None as InProgress ping.
+pub type ResponseChannels =
+  CloneCache<Uuid, Sender<EncodedResponse<EncodedJsonMessage>>>;
+
+pub type TerminalChannels = CloneCache<Uuid, Sender<Vec<u8>>>;
+
 #[derive(Debug)]
 pub struct PeripheryConnection {
   /// The connection args
   pub args: OwnedPeripheryConnectionArgs,
   /// Send and receive bytes over the connection socket.
-  pub sender: Sender<Bytes>,
+  pub sender: Sender<EncodedTransportMessage>,
   /// Cancel the connection
   pub cancel: CancellationToken,
   /// Whether Periphery is currently connected.
@@ -251,14 +272,19 @@ pub struct PeripheryConnection {
   // at the same server id.
   /// Stores latest connection error
   pub error: Arc<RwLock<Option<serror::Serror>>>,
-  /// Forward bytes from Periphery to specific channel handlers.
-  pub channels: Arc<ConnectionChannels>,
+  /// Forward bytes from Periphery to response channel handlers.
+  pub responses: Arc<ResponseChannels>,
+  /// Forward bytes from Periphery to terminal channel handlers.
+  pub terminals: Arc<TerminalChannels>,
 }
 
 impl PeripheryConnection {
   pub fn new(
     args: impl Into<OwnedPeripheryConnectionArgs>,
-  ) -> (Arc<PeripheryConnection>, BufferedReceiver<Bytes>) {
+  ) -> (
+    Arc<PeripheryConnection>,
+    BufferedReceiver<EncodedTransportMessage>,
+  ) {
     let (sender, receiever) = buffered_channel();
     (
       PeripheryConnection {
@@ -267,7 +293,8 @@ impl PeripheryConnection {
         cancel: CancellationToken::new(),
         connected: AtomicBool::new(false),
         error: Default::default(),
-        channels: Default::default(),
+        responses: Default::default(),
+        terminals: Default::default(),
       }
       .into(),
       receiever,
@@ -277,7 +304,10 @@ impl PeripheryConnection {
   pub fn with_new_args(
     &self,
     args: impl Into<OwnedPeripheryConnectionArgs>,
-  ) -> (Arc<PeripheryConnection>, BufferedReceiver<Bytes>) {
+  ) -> (
+    Arc<PeripheryConnection>,
+    BufferedReceiver<EncodedTransportMessage>,
+  ) {
     // Ensure this connection is cancelled.
     self.cancel();
     let (sender, receiever) = buffered_channel();
@@ -288,7 +318,8 @@ impl PeripheryConnection {
         cancel: CancellationToken::new(),
         connected: AtomicBool::new(false),
         error: self.error.clone(),
-        channels: self.channels.clone(),
+        responses: self.responses.clone(),
+        terminals: self.terminals.clone(),
       }
       .into(),
       receiever,
@@ -303,7 +334,7 @@ impl PeripheryConnection {
     L::login(LoginFlowArgs {
       socket,
       identifiers,
-      private_key: core_private_key(),
+      private_key: core_keys().load().private.as_str(),
       public_key_validator: self.args.borrow(),
     })
     .await?;
@@ -315,7 +346,7 @@ impl PeripheryConnection {
   pub async fn handle_socket<W: Websocket>(
     &self,
     socket: W,
-    receiver: &mut BufferedReceiver<Bytes>,
+    receiver: &mut BufferedReceiver<EncodedTransportMessage>,
   ) {
     let cancel = self.cancel.child_token();
 
@@ -324,42 +355,32 @@ impl PeripheryConnection {
 
     let (mut ws_write, mut ws_read) = socket.split();
 
+    ws_read.set_cancel(cancel.clone());
+    receiver.set_cancel(cancel.clone());
+
     let forward_writes = async {
       loop {
-        let next = tokio::select! {
-          next = receiver.recv() => next,
-          _ = cancel.cancelled() => break,
+        let Ok(message) = receiver.recv().await else {
+          break;
         };
-
-        let message = match next {
-          Some(request) => Bytes::copy_from_slice(request),
-          // Sender Dropped (shouldn't happen, a reference is held on 'connection').
-          None => break,
-        };
-
-        match ws_write.send(message).await {
+        match ws_write.send(message.into_bytes()).await {
           Ok(_) => receiver.clear_buffer(),
           Err(e) => {
-            self.set_error(e.into()).await;
+            self.set_error(e).await;
             break;
           }
         }
       }
       // Cancel again if not already
-      let _ = ws_write.close(None).await;
+      let _ = ws_write.close().await;
       cancel.cancel();
     };
 
     let handle_reads = async {
       loop {
-        let next = tokio::select! {
-          next = ws_read.recv() => next,
-          _ = cancel.cancelled() => break,
-        };
-
-        match next {
-          Ok(WebsocketMessage::Binary(bytes)) => {
-            self.handle_incoming_bytes(bytes).await
+        match ws_read.recv().await {
+          Ok(WebsocketMessage::Message(message)) => {
+            self.handle_incoming_message(message).await
           }
           Ok(WebsocketMessage::Close(_))
           | Ok(WebsocketMessage::Closed) => {
@@ -367,7 +388,7 @@ impl PeripheryConnection {
             break;
           }
           Err(e) => {
-            self.set_error(e.into()).await;
+            self.set_error(e).await;
           }
         };
       }
@@ -380,33 +401,69 @@ impl PeripheryConnection {
     self.set_connected(false);
   }
 
-  pub async fn handle_incoming_bytes(&self, bytes: Bytes) {
-    let id = match id_from_transport_bytes(&bytes) {
+  pub async fn handle_incoming_message(
+    &self,
+    message: EncodedTransportMessage,
+  ) {
+    let message: TransportMessage = match message.decode() {
       Ok(res) => res,
       Err(e) => {
-        // TODO: handle better
-        warn!("Failed to read id | {e:#}");
+        warn!("Failed to parse Message bytes | {e:#}");
         return;
       }
     };
-    let Some(channel) = self.channels.get(&id).await else {
-      // TODO: handle better
-      debug!("Failed to send response | No response channel found");
-      return;
-    };
-    if let Err(e) = channel.send(bytes).await {
-      // TODO: handle better
-      warn!("Failed to send response | Channel failure | {e:#}");
+    match message {
+      TransportMessage::Response(data) => {
+        match data.decode().map(ResponseMessage::into_inner) {
+          Ok(WithChannel { channel, data }) => {
+            let Some(response_channel) =
+              self.responses.get(&channel).await
+            else {
+              warn!(
+                "Failed to forward Response message | No response channel found at {channel}"
+              );
+              return;
+            };
+            if let Err(e) = response_channel.send(data).await {
+              warn!(
+                "Failed to forward Response | Response channel failure at {channel} | {e:#}"
+              );
+            }
+          }
+          Err(e) => {
+            warn!("Failed to read Response message | {e:#}");
+          }
+        }
+      }
+      TransportMessage::Terminal(data) => match data.decode() {
+        Ok(WithChannel {
+          channel: channel_id,
+          data,
+        }) => {
+          let Some(channel) = self.terminals.get(&channel_id).await
+          else {
+            warn!(
+              "Failed to forward Terminal message | No terminal channel found at {channel_id}"
+            );
+            return;
+          };
+          if let Err(e) = channel.send(data).await {
+            warn!(
+              "Failed to forward Terminal message | Channel failure at {channel_id} | {e:#}"
+            );
+          }
+        }
+        Err(e) => {
+          warn!("Failed to read Terminal message | {e:#}");
+        }
+      },
+      //
+      other => {
+        warn!("Received unexpected transport message | {other:?}");
+      }
     }
   }
 
-  pub async fn send(
-    &self,
-    value: Bytes,
-  ) -> Result<(), SendError<Bytes>> {
-    self.sender.send(value).await
-  }
-
   pub fn set_connected(&self, connected: bool) {
     self.connected.store(connected, atomic::Ordering::Relaxed);
   }

bin/core/src/connection/server.rs

@@ -1,4 +1,4 @@
-use std::{str::FromStr, time::Duration};
+use std::str::FromStr;
 
 use anyhow::{Context, anyhow};
 use axum::{
@@ -6,31 +6,36 @@ use axum::{
   http::{HeaderMap, StatusCode},
   response::Response,
 };
-use bytes::Bytes;
 use database::mungos::mongodb::bson::{doc, oid::ObjectId};
 use komodo_client::{
   api::write::{CreateBuilder, CreateServer, UpdateResourceMeta},
   entities::{
     builder::{PartialBuilderConfig, PartialServerBuilderConfig},
+    komodo_timestamp,
     onboarding_key::OnboardingKey,
     server::{PartialServerConfig, Server},
     user::system_user,
   },
 };
+use periphery_client::{
+  api::PeripheryConnectionQuery, transport::LoginMessage,
+};
 use resolver_api::Resolve;
 use serror::{AddStatusCode, AddStatusCodeError};
 use transport::{
-  MessageState, PeripheryConnectionQuery,
   auth::{
     HeaderConnectionIdentifiers, LoginFlow, LoginFlowArgs,
     PublicKeyValidator, ServerLoginFlow,
   },
-  websocket::{Websocket, axum::AxumWebsocket},
+  websocket::{
+    Websocket, WebsocketExt as _, axum::AxumWebsocket,
+    login::LoginWebsocketExt,
+  },
 };
 
 use crate::{
   api::write::WriteArgs,
-  config::core_private_key,
+  config::core_keys,
   helpers::query::id_or_name_filter,
   resource::KomodoResource,
   state::{db_client, periphery_connections},
@@ -119,9 +124,11 @@ async fn existing_server_handler(
       format!("server={}", urlencoding::encode(&server_query));
     let mut socket = AxumWebsocket(socket);
 
-    if let Err(e) = socket.send(Bytes::from_owner([0])).await.context(
-      "Failed to send the login flow indicator over connnection",
-    ) {
+    if let Err(e) = socket
+      .send_message(LoginMessage::OnboardingFlow(false))
+      .await
+      .context("Failed to send Login OnboardingFlow false message")
+    {
       connection.set_error(e).await;
       return;
     };
@@ -151,8 +158,8 @@ async fn onboard_server_handler(
       format!("server={}", urlencoding::encode(&server_query));
     let mut socket = AxumWebsocket(socket);
 
-    if let Err(e) = socket.send(Bytes::from_owner([1])).await.context(
-      "Failed to send the login flow indicator over connnection",
+    if let Err(e) = socket.send_message(LoginMessage::OnboardingFlow(true)).await.context(
+      "Failed to send Login OnboardingFlow true message",
     ).context("Server onboarding error") {
       warn!("{e:#}");
       return;
@@ -161,7 +168,7 @@ async fn onboard_server_handler(
     let onboarding_key = match ServerLoginFlow::login(LoginFlowArgs {
       socket: &mut socket,
       identifiers: identifiers.build(query.as_bytes()),
-      private_key: core_private_key(),
+      private_key: core_keys().load().private.as_str(),
       public_key_validator: CreationKeyValidator,
     })
     .await
@@ -173,19 +180,10 @@ async fn onboard_server_handler(
       }
     };
 
-    let res = socket
-      .recv_bytes()
-      .with_timeout(Duration::from_secs(2))
-      .await
-      .and_then(|res| {
-        res.and_then(|public_key_bytes| {
-          String::from_utf8(public_key_bytes.into())
-            .context("Public key bytes are not valid utf8")
-        })
-      });
-
     // Post onboarding login 1: Receive public key
-    let public_key = match res
+    let public_key = match socket
+      .recv_login_public_key()
+      .await
     {
       Ok(public_key) => public_key,
       Err(e) => {
@@ -194,11 +192,9 @@ async fn onboard_server_handler(
       }
     };
 
-    
-
     let server_id = match create_server_maybe_builder(
       server_query,
-      public_key,
+      public_key.into_inner(),
       onboarding_key.copy_server,
       onboarding_key.tags,
       onboarding_key.create_builder
@@ -207,7 +203,7 @@ async fn onboard_server_handler(
       Err(e) => {
         warn!("{e:#}");
         if let Err(e) = socket
-          .send_error(&e)
+          .send_login_error(&e)
           .await
           .context("Failed to send Server creation failed to client")
         {
@@ -219,9 +215,9 @@ async fn onboard_server_handler(
     };
 
     if let Err(e) = socket
-      .send(MessageState::Successful.into())
+      .send_message(LoginMessage::Success)
       .await
-      .context("Failed to send Server creation successful to client")
+      .context("Failed to send Login Onboarding Successful message")
     {
       // Log additional error
       warn!("{e:#}");
@@ -229,7 +225,7 @@ async fn onboard_server_handler(
 
     // Server created, close and trigger reconnect
     // and handling using existing server handler.
-    let _ = socket.close(None).await;
+    let _ = socket.close().await;
 
     // Add the server to onboarding key "Onboarded"
     let res = db_client()
@@ -257,14 +253,23 @@ async fn create_server_maybe_builder(
       ..Default::default()
     }
   } else {
-    let config = match db_client().servers.find_one(id_or_name_filter(&copy_server)).await {
+    let config = match db_client()
+      .servers
+      .find_one(id_or_name_filter(&copy_server))
+      .await
+    {
       Ok(Some(server)) => server.config,
       Ok(None) => {
-        warn!("Server onboarding: Failed to find Server {}", copy_server);
+        warn!(
+          "Server onboarding: Failed to find Server {}",
+          copy_server
+        );
         Default::default()
       }
       Err(e) => {
-        warn!("Failed to query database for onboarding key 'copy_server' | {e:?}");
+        warn!(
+          "Failed to query database for onboarding key 'copy_server' | {e:?}"
+        );
         Default::default()
       }
     };
@@ -340,10 +345,14 @@ impl PublicKeyValidator for CreationKeyValidator {
       .await
       .context("Failed to query database for Server onboarding keys")?
       .context("Matching Server onboarding key not found")?;
-    if onboarding_key.enabled {
+    // Check enabled and not expired.
+    if onboarding_key.enabled
+      && (onboarding_key.expires == 0
+        || onboarding_key.expires > komodo_timestamp())
+    {
       Ok(onboarding_key)
     } else {
-      Err(anyhow!("Onboarding key is disabled"))
+      Err(anyhow!("Onboarding key is invalid"))
     }
   }
 }

bin/core/src/helpers/builder.rs

@@ -54,7 +54,6 @@ pub async fn get_builder_periphery(
           &config,
         ),
         config.insecure_tls,
-        &config.passkey,
       )
       .await?;
       periphery
@@ -116,7 +115,6 @@ async fn get_aws_builder(
       &config,
     ),
     config.insecure_tls,
-    "",
   )
   .await?;
 

bin/core/src/helpers/mod.rs

@@ -195,7 +195,6 @@ pub async fn periphery_client(
   PeripheryClient::new(
     PeripheryConnectionArgs::from_server(server),
     server.config.insecure_tls,
-    &server.config.passkey,
   )
   .await
 }

bin/core/src/helpers/procedure.rs

@@ -1226,6 +1226,23 @@ async fn execute_execution(
       )
       .await?
     }
+    Execution::RotateCoreKeys(req) => {
+      let req = ExecuteRequest::RotateCoreKeys(req);
+      let update = init_execution_update(&req, &user).await?;
+      let ExecuteRequest::RotateCoreKeys(req) = req else {
+        unreachable!()
+      };
+      let update_id = update.id.clone();
+      handle_resolve_result(
+        req
+          .resolve(&ExecuteArgs { user, update })
+          .await
+          .map_err(|e| e.error)
+          .context("Failed at RotateCoreKeys"),
+        &update_id,
+      )
+      .await?
+    }
     Execution::Sleep(req) => {
       let duration = Duration::from_millis(req.duration_ms as u64);
       tokio::time::sleep(duration).await;

bin/core/src/helpers/update.rs

@@ -523,6 +523,9 @@ pub async fn init_execution_update(
     ExecuteRequest::RotateAllServerKeys(_data) => {
       (Operation::RotateAllServerKeys, ResourceTarget::system())
     }
+    ExecuteRequest::RotateCoreKeys(_data) => {
+      (Operation::RotateCoreKeys, ResourceTarget::system())
+    }
   };
 
   let mut update = make_update(target, operation, user);

bin/core/src/main.rs

@@ -6,14 +6,14 @@ extern crate tracing;
 use std::{net::SocketAddr, str::FromStr};
 
 use anyhow::Context;
-use axum::Router;
+use axum::{Router, routing::get};
 use axum_server::{Handle, tls_rustls::RustlsConfig};
 use tower_http::{
   cors::{Any, CorsLayer},
   services::{ServeDir, ServeFile},
 };
 
-use crate::config::{core_config, core_public_key};
+use crate::config::{core_config, core_keys};
 
 mod alert;
 mod api;
@@ -54,7 +54,7 @@ async fn app() -> anyhow::Result<()> {
   }
 
   // Init + log public key. Will crash if invalid private key here.
-  info!("Public Key: {}", core_public_key());
+  info!("Public Key: {}", core_keys().load().public);
 
   rustls::crypto::aws_lc_rs::default_provider()
     .install_default()
@@ -90,6 +90,7 @@ async fn app() -> anyhow::Result<()> {
     .not_found_service(frontend_index.clone());
 
   let app = Router::new()
+    .route("/version", get(|| async { env!("CARGO_PKG_VERSION") }))
     .nest("/auth", api::auth::router())
     .nest("/user", api::user::router())
     .nest("/read", api::read::router())

bin/core/src/periphery/mod.rs

@@ -1,42 +1,35 @@
 use std::{sync::Arc, time::Duration};
 
 use anyhow::{Context, anyhow};
-use bytes::Bytes;
-use cache::CloneCache;
+use encoding::Decode as _;
 use periphery_client::api;
 use resolver_api::HasResponse;
 use serde::{Serialize, de::DeserializeOwned};
 use serde_json::json;
-use serror::deserialize_error_bytes;
-use tokio::sync::mpsc::{self, Sender};
-use tracing::warn;
-use transport::{
-  MessageState,
-  bytes::{from_transport_bytes, to_transport_bytes},
-};
+use transport::channel::channel;
 use uuid::Uuid;
 
 use crate::{
-  connection::{PeripheryConnection, PeripheryConnectionArgs},
+  connection::{
+    PeripheryConnection, PeripheryConnectionArgs, ResponseChannels,
+    TerminalChannels,
+  },
   state::periphery_connections,
 };
 
 pub mod terminal;
 
-pub type ConnectionChannels = CloneCache<Uuid, Sender<Bytes>>;
-
 #[derive(Debug)]
 pub struct PeripheryClient {
   pub id: String,
-  channels: Arc<ConnectionChannels>,
+  pub responses: Arc<ResponseChannels>,
+  pub terminals: Arc<TerminalChannels>,
 }
 
 impl PeripheryClient {
   pub async fn new(
     args: PeripheryConnectionArgs<'_>,
     insecure_tls: bool,
-    // deprecated.
-    passkey: &str,
   ) -> anyhow::Result<PeripheryClient> {
     let connections = periphery_connections();
 
@@ -47,21 +40,17 @@ impl PeripheryClient {
       if args.address.is_none() {
         return Err(anyhow!("Server {id} is not connected"));
       }
-      let channels = args
-        .spawn_client_connection(
-          id.clone(),
-          insecure_tls,
-          passkey.to_string(),
-        )
-        .await?;
-      return Ok(PeripheryClient { id, channels });
+      return args
+        .spawn_client_connection(id.clone(), insecure_tls)
+        .await;
     };
 
     // Ensure the connection args are unchanged.
     if args.matches(&connection.args) {
       return Ok(PeripheryClient {
         id,
-        channels: connection.channels.clone(),
+        responses: connection.responses.clone(),
+        terminals: connection.terminals.clone(),
       });
     }
 
@@ -77,18 +66,12 @@ impl PeripheryClient {
         .with_context(|| format!("Server {id} is not connected"))?;
       Ok(PeripheryClient {
         id,
-        channels: connection.channels.clone(),
+        responses: connection.responses.clone(),
+        terminals: connection.terminals.clone(),
       })
     } else {
       // Core -> Periphery connection
-      let channels = args
-        .spawn_client_connection(
-          id.clone(),
-          insecure_tls,
-          passkey.to_string(),
-        )
-        .await?;
-      Ok(PeripheryClient { id, channels })
+      args.spawn_client_connection(id.clone(), insecure_tls).await
     }
   }
 
@@ -123,83 +106,40 @@ impl PeripheryClient {
     // Polls connected 3 times before bailing
     connection.bail_if_not_connected().await?;
 
-    let id = Uuid::new_v4();
-    let (response_sender, mut response_receiever) =
-      mpsc::channel(1000);
-    self.channels.insert(id, response_sender).await;
-
-    let req_type = T::req_type();
-    let data = serde_json::to_vec(&json!({
-      "type": req_type,
-      "params": request
-    }))
-    .context("Failed to serialize request to bytes")?;
+    let channel_id = Uuid::new_v4();
+    let (response_sender, mut response_receiever) = channel();
+    self.responses.insert(channel_id, response_sender).await;
 
     if let Err(e) = connection
-      .send(to_transport_bytes(data, id, MessageState::Request))
+      .sender
+      .send_request(
+        channel_id,
+        &json!({
+          "type": T::req_type(),
+          "params": request
+        }),
+      )
       .await
       .context("Failed to send request over channel")
     {
       // cleanup
-      self.channels.remove(&id).await;
+      self.terminals.remove(&channel_id).await;
       return Err(e);
     }
 
     // Poll for the associated response
     loop {
-      let next = tokio::select! {
-        msg = response_receiever.recv() => msg,
-        // Periphery will send InProgress every 5s to avoid timeout
-        _ = tokio::time::sleep(Duration::from_secs(10)) => {
-          return Err(anyhow!("Response timed out"));
-        }
+      let message = response_receiever
+        .recv()
+        .with_timeout(Duration::from_secs(10))
+        .await?;
+
+      // Still in progress, sent to avoid timeout.
+      let Some(message) = message.decode()? else {
+        continue;
       };
 
-      let bytes = match next {
-        Some(bytes) => bytes,
-        None => {
-          return Err(anyhow!(
-            "Sender dropped before response was recieved"
-          ));
-        }
-      };
-
-      let (state, data) = match from_transport_bytes(bytes) {
-        Ok((data, _, state)) if !data.is_empty() => (state, data),
-        // Ignore no data cases
-        Ok(_) => continue,
-        Err(e) => {
-          warn!(
-            "Server {} | Received invalid message | {e:#}",
-            self.id
-          );
-          continue;
-        }
-      };
-      match state {
-        // TODO: improve the allocation in .to_vec
-        MessageState::Successful => {
-          // cleanup
-          self.channels.remove(&id).await;
-          return serde_json::from_slice(&data)
-            .context("Failed to parse successful response");
-        }
-        MessageState::Failed => {
-          // cleanup
-          self.channels.remove(&id).await;
-          return Err(deserialize_error_bytes(&data));
-        }
-        MessageState::InProgress => continue,
-        // Shouldn't be received by this receiver
-        other => {
-          // TODO: delete log
-          warn!(
-            "Server {} | Got other message over over response channel: {other:?}",
-            self.id
-          );
-          continue;
-        }
-      }
+      return message.decode();
     }
   }
 }

bin/core/src/periphery/terminal.rs

@@ -8,12 +8,15 @@ use anyhow::Context;
 use bytes::Bytes;
 use cache::CloneCache;
 use futures::Stream;
-use periphery_client::api::terminal::{
-  ConnectContainerExec, ConnectTerminal, END_OF_OUTPUT,
-  ExecuteContainerExec, ExecuteTerminal,
+use komodo_client::api::write::TerminalRecreateMode;
+use periphery_client::{
+  api::terminal::{
+    ConnectContainerAttach, ConnectContainerExec, ConnectTerminal,
+    END_OF_OUTPUT, ExecuteContainerExec, ExecuteTerminal,
+  },
+  transport::EncodedTransportMessage,
 };
-use tokio::sync::mpsc::{Receiver, Sender, channel};
-use transport::bytes::data_from_transport_bytes;
+use transport::channel::{Receiver, Sender, channel};
 use uuid::Uuid;
 
 use crate::{
@@ -24,7 +27,11 @@ impl PeripheryClient {
   pub async fn connect_terminal(
     &self,
     terminal: String,
-  ) -> anyhow::Result<(Uuid, Sender<Bytes>, Receiver<Bytes>)> {
+  ) -> anyhow::Result<(
+    Uuid,
+    Sender<EncodedTransportMessage>,
+    Receiver<Vec<u8>>,
+  )> {
     tracing::trace!(
       "request | type: ConnectTerminal | terminal name: {terminal}",
     );
@@ -34,22 +41,35 @@ impl PeripheryClient {
         || format!("No connection found for server {}", self.id),
       )?;
 
-    let id = self
+    let channel_id = self
       .request(ConnectTerminal { terminal })
       .await
       .context("Failed to create terminal connection")?;
 
-    let (sender, receiever) = channel(1024);
-    connection.channels.insert(id, sender).await;
+    let (sender, receiever) = channel();
+    connection.terminals.insert(channel_id, sender).await;
 
-    Ok((id, connection.sender.clone(), receiever))
+    connection
+      .sender
+      .send_terminal(channel_id, Bytes::new())
+      .await
+      .context(
+        "Failed to send TerminalTrigger to begin forwarding.",
+      )?;
+
+    Ok((channel_id, connection.sender.clone(), receiever))
   }
 
   pub async fn connect_container_exec(
     &self,
     container: String,
     shell: String,
-  ) -> anyhow::Result<(Uuid, Sender<Bytes>, Receiver<Bytes>)> {
+    recreate: TerminalRecreateMode,
+  ) -> anyhow::Result<(
+    Uuid,
+    Sender<EncodedTransportMessage>,
+    Receiver<Vec<u8>>,
+  )> {
     tracing::trace!(
       "request | type: ConnectContainerExec | container name: {container} | shell: {shell}",
     );
@@ -59,15 +79,67 @@ impl PeripheryClient {
         || format!("No connection found for server {}", self.id),
       )?;
 
-    let id = self
-      .request(ConnectContainerExec { container, shell })
+    let channel_id = self
+      .request(ConnectContainerExec {
+        container,
+        shell,
+        recreate,
+      })
       .await
       .context("Failed to create container exec connection")?;
 
-    let (sender, receiever) = channel(1000);
-    connection.channels.insert(id, sender).await;
+    let (sender, receiever) = channel();
+    connection.terminals.insert(channel_id, sender).await;
 
-    Ok((id, connection.sender.clone(), receiever))
+    connection
+      .sender
+      .send_terminal(channel_id, Bytes::new())
+      .await
+      .context(
+        "Failed to send TerminalTrigger to begin forwarding.",
+      )?;
+
+    Ok((channel_id, connection.sender.clone(), receiever))
+  }
+
+  pub async fn connect_container_attach(
+    &self,
+    container: String,
+    recreate: TerminalRecreateMode,
+  ) -> anyhow::Result<(
+    Uuid,
+    Sender<EncodedTransportMessage>,
+    Receiver<Vec<u8>>,
+  )> {
+    tracing::trace!(
+      "request | type: ConnectContainerAttach | container name: {container}",
+    );
+
+    let connection =
+      periphery_connections().get(&self.id).await.with_context(
+        || format!("No connection found for server {}", self.id),
+      )?;
+
+    let channel = self
+      .request(ConnectContainerAttach {
+        container,
+        recreate,
+      })
+      .await
+      .context("Failed to create container attach connection")?;
+
+    let (sender, receiever) = transport::channel::channel();
+    connection.terminals.insert(channel, sender).await;
+
+    connection
+      .sender
+      .send_terminal(channel, Bytes::new())
+      .await
+      .context(
+        "Failed to send TerminalTrigger to begin forwarding.",
+      )?;
+
+    Ok((channel, connection.sender.clone(), receiever))
   }
 
   /// Executes command on specified terminal,
@@ -89,7 +161,7 @@ impl PeripheryClient {
     terminal: String,
     command: String,
   ) -> anyhow::Result<
-    impl Stream<Item = anyhow::Result<Bytes>> + 'static,
+    impl Stream<Item = anyhow::Result<Vec<u8>>> + 'static,
   > {
     tracing::trace!(
       "sending request | type: ExecuteTerminal | terminal name: {terminal} | command: {command}",
@@ -100,19 +172,29 @@ impl PeripheryClient {
         || format!("No connection found for server {}", self.id),
       )?;
 
-    let id = self
+    let channel_id = self
       .request(ExecuteTerminal { terminal, command })
       .await
       .context("Failed to create execute terminal connection")?;
 
-    let (sender, receiver) = channel(1000);
+    let (terminal_sender, terminal_receiver) = channel();
+    connection
+      .terminals
+      .insert(channel_id, terminal_sender)
+      .await;
 
-    connection.channels.insert(id, sender).await;
+    connection
+      .sender
+      .send_terminal(channel_id, Bytes::new())
+      .await
+      .context(
+        "Failed to send TerminalTrigger to begin forwarding.",
+      )?;
 
     Ok(ReceiverStream {
-      id,
-      receiver,
-      channels: connection.channels.clone(),
+      channel_id,
+      receiver: terminal_receiver,
+      channels: connection.terminals.clone(),
     })
   }
 
@@ -135,6 +217,7 @@ impl PeripheryClient {
     container: String,
     shell: String,
     command: String,
+    recreate: TerminalRecreateMode,
   ) -> anyhow::Result<ReceiverStream> {
     tracing::trace!(
       "sending request | type: ExecuteContainerExec | container: {container} | shell: {shell} | command: {command}",
@@ -145,50 +228,58 @@ impl PeripheryClient {
         || format!("No connection found for server {}", self.id),
       )?;
 
-    let id = self
+    let channel_id = self
       .request(ExecuteContainerExec {
         container,
         shell,
         command,
+        recreate,
       })
       .await
       .context("Failed to create execute terminal connection")?;
 
-    let (sender, receiver) = channel(1000);
+    let (terminal_sender, terminal_receiver) = channel();
+    connection
+      .terminals
+      .insert(channel_id, terminal_sender)
+      .await;
 
-    connection.channels.insert(id, sender).await;
+    // Trigger forwarding to begin now that forwarding channel is ready.
+    // This is required to not miss messages.
+    connection
+      .sender
+      .send_terminal(channel_id, Bytes::new())
+      .await?;
 
     Ok(ReceiverStream {
-      id,
-      receiver,
-      channels: connection.channels.clone(),
+      channel_id,
+      receiver: terminal_receiver,
+      channels: connection.terminals.clone(),
     })
   }
 }
 
 pub struct ReceiverStream {
-  id: Uuid,
-  channels: Arc<CloneCache<Uuid, Sender<Bytes>>>,
-  receiver: Receiver<Bytes>,
+  channel_id: Uuid,
+  channels: Arc<CloneCache<Uuid, Sender<Vec<u8>>>>,
+  receiver: Receiver<Vec<u8>>,
 }
 
 impl Stream for ReceiverStream {
-  type Item = anyhow::Result<Bytes>;
+  type Item = anyhow::Result<Vec<u8>>;
   fn poll_next(
     mut self: Pin<&mut Self>,
     cx: &mut task::Context<'_>,
   ) -> Poll<Option<Self::Item>> {
-    match self
-      .receiver
-      .poll_recv(cx)
-      .map(|bytes| bytes.map(data_from_transport_bytes))
-    {
-      Poll::Ready(Some(Ok(bytes))) if bytes == END_OF_OUTPUT => {
+    match self.receiver.poll_recv(cx) {
+      Poll::Ready(Some(bytes))
+        if bytes == END_OF_OUTPUT.as_bytes() =>
+      {
         self.cleanup();
         Poll::Ready(None)
       }
-      Poll::Ready(Some(Ok(bytes))) => Poll::Ready(Some(Ok(bytes))),
-      Poll::Ready(Some(Err(e))) => Poll::Ready(Some(Err(e))),
+      Poll::Ready(Some(bytes)) => Poll::Ready(Some(Ok(bytes))),
+      // Poll::Ready(Some(Err(e))) => Poll::Ready(Some(Err(e))),
       Poll::Ready(None) => {
         self.cleanup();
         Poll::Ready(None)
@@ -202,7 +293,7 @@ impl ReceiverStream {
   fn cleanup(&self) {
     // Not the prettiest but it should be fine
     let channels = self.channels.clone();
-    let id = self.id;
+    let id = self.channel_id;
     tokio::spawn(async move {
       channels.remove(&id).await;
     });

bin/core/src/resource/procedure.rs

@@ -781,6 +781,13 @@ async fn validate_config(
             ));
           }
         }
+        Execution::RotateCoreKeys(_params) => {
+          if !user.admin {
+            return Err(anyhow!(
+              "Non admin user cannot trigger rotate core keys"
+            ));
+          }
+        }
         Execution::Sleep(_) => {}
       }
     }

bin/core/src/resource/server.rs

@@ -72,14 +72,14 @@ impl super::KomodoResource for Server {
       version,
       public_key,
       terminals_disabled,
-      container_exec_disabled,
+      container_terminals_disabled,
     ) = match status.as_ref().and_then(|s| s.periphery_info.as_ref())
     {
       Some(info) => (
         Some(info.version.clone()),
         Some(info.public_key.clone()),
         info.terminals_disabled,
-        info.container_exec_disabled,
+        info.container_terminals_disabled,
       ),
       None => (None, None, true, true),
     };
@@ -111,7 +111,7 @@ impl super::KomodoResource for Server {
           server.info.attempted_public_key,
         ),
         terminals_disabled,
-        container_exec_disabled,
+        container_terminals_disabled,
       },
     }
   }
@@ -245,6 +245,14 @@ impl super::KomodoResource for Server {
       )
       .await;
 
+    let _ = db_client()
+      .onboarding_keys
+      .update_many(
+        doc! { "copy_server": &id },
+        doc! { "$set": { "copy_server": "" } },
+      )
+      .await;
+
     Ok(())
   }
 
@@ -281,7 +289,7 @@ pub async fn update_server_public_key(
 pub async fn rotate_server_keys(
   server: &Server,
 ) -> anyhow::Result<()> {
-  let periphery = periphery_client(&server).await?;
+  let periphery = periphery_client(server).await?;
   let public_key = periphery
     .request(api::keys::RotatePrivateKey {})
     .await

bin/core/src/schedule.rs

@@ -46,10 +46,10 @@ pub fn spawn_schedule_executor() {
         match next_run {
           Ok(next_run_time) if current_time >= next_run_time => {
             tokio::spawn(async move {
-              match &target {
+              match target {
                 ResourceTarget::Action(id) => {
                   let action = match crate::resource::get::<Action>(
-                    id,
+                    &id,
                   )
                   .await
                   {
@@ -61,6 +61,24 @@ pub fn spawn_schedule_executor() {
                       return;
                     }
                   };
+
+                  if action.config.schedule_alert {
+                    let alert = Alert {
+                      id: Default::default(),
+                      target: ResourceTarget::Action(id.clone()),
+                      ts: komodo_timestamp(),
+                      resolved_ts: Some(komodo_timestamp()),
+                      resolved: true,
+                      level: SeverityLevel::Ok,
+                      data: AlertData::ScheduleRun {
+                        resource_type: ResourceTargetVariant::Action,
+                        id: id.clone(),
+                        name: action.name.clone(),
+                      },
+                    };
+                    send_alerts(&[alert]).await
+                  }
+
                   let request =
                     ExecuteRequest::RunAction(RunAction {
                       action: id.clone(),
@@ -80,10 +98,12 @@ pub fn spawn_schedule_executor() {
                       return;
                     }
                   };
+
                   let ExecuteRequest::RunAction(request) = request
                   else {
                     unreachable!()
                   };
+
                   if let Err(e) = request
                     .resolve(&ExecuteArgs {
                       user: action_user().to_owned(),
@@ -95,28 +115,13 @@ pub fn spawn_schedule_executor() {
                       "Scheduled action run on {id} failed | {e:?}"
                     );
                   }
+
                   update_schedule(&action);
-                  if action.config.schedule_alert {
-                    let alert = Alert {
-                      id: Default::default(),
-                      target,
-                      ts: komodo_timestamp(),
-                      resolved_ts: Some(komodo_timestamp()),
-                      resolved: true,
-                      level: SeverityLevel::Ok,
-                      data: AlertData::ScheduleRun {
-                        resource_type: ResourceTargetVariant::Action,
-                        id: action.id,
-                        name: action.name,
-                      },
-                    };
-                    send_alerts(&[alert]).await
-                  }
                 }
                 ResourceTarget::Procedure(id) => {
                   let procedure = match crate::resource::get::<
                     Procedure,
-                  >(id)
+                  >(&id)
                   .await
                   {
                     Ok(procedure) => procedure,
@@ -127,6 +132,25 @@ pub fn spawn_schedule_executor() {
                       return;
                     }
                   };
+
+                  if procedure.config.schedule_alert {
+                    let alert = Alert {
+                      id: Default::default(),
+                      target: ResourceTarget::Procedure(id.clone()),
+                      ts: komodo_timestamp(),
+                      resolved_ts: Some(komodo_timestamp()),
+                      resolved: true,
+                      level: SeverityLevel::Ok,
+                      data: AlertData::ScheduleRun {
+                        resource_type:
+                          ResourceTargetVariant::Procedure,
+                        id: id.clone(),
+                        name: procedure.name.clone(),
+                      },
+                    };
+                    send_alerts(&[alert]).await
+                  }
+
                   let request =
                     ExecuteRequest::RunProcedure(RunProcedure {
                       procedure: id.clone(),
@@ -145,6 +169,7 @@ pub fn spawn_schedule_executor() {
                       return;
                     }
                   };
+
                   let ExecuteRequest::RunProcedure(request) = request
                   else {
                     unreachable!()
@@ -160,24 +185,8 @@ pub fn spawn_schedule_executor() {
                       "Scheduled procedure run on {id} failed | {e:?}"
                     );
                   }
+
                   update_schedule(&procedure);
-                  if procedure.config.schedule_alert {
-                    let alert = Alert {
-                      id: Default::default(),
-                      target,
-                      ts: komodo_timestamp(),
-                      resolved_ts: Some(komodo_timestamp()),
-                      resolved: true,
-                      level: SeverityLevel::Ok,
-                      data: AlertData::ScheduleRun {
-                        resource_type:
-                          ResourceTargetVariant::Procedure,
-                        id: procedure.id,
-                        name: procedure.name,
-                      },
-                    };
-                    send_alerts(&[alert]).await
-                  }
                 }
                 _ => unreachable!(),
               }

bin/core/src/state.rs

@@ -1,22 +1,12 @@
-use std::{
-  collections::HashMap,
-  sync::{Arc, OnceLock},
-};
+use std::sync::{Arc, OnceLock};
 
 use anyhow::Context;
 use arc_swap::ArcSwap;
 use cache::CloneCache;
 use komodo_client::entities::{
-  action::ActionState,
-  build::BuildState,
-  config::core::{CoreConfig, GithubWebhookAppConfig},
-  deployment::DeploymentState,
-  procedure::ProcedureState,
-  repo::RepoState,
-  stack::StackState,
-};
-use octorust::auth::{
-  Credentials, InstallationTokenGenerator, JWTCredentials,
+  action::ActionState, build::BuildState,
+  deployment::DeploymentState, procedure::ProcedureState,
+  repo::RepoState, stack::StackState,
 };
 
 use crate::{
@@ -149,73 +139,3 @@ pub fn all_resources_cache() -> &'static ArcSwap<AllResourcesById> {
     OnceLock::new();
   ALL_RESOURCES.get_or_init(Default::default)
 }
-
-pub fn github_client()
--> Option<&'static HashMap<String, octorust::Client>> {
-  static GITHUB_CLIENT: OnceLock<
-    Option<HashMap<String, octorust::Client>>,
-  > = OnceLock::new();
-  GITHUB_CLIENT
-    .get_or_init(|| {
-      let CoreConfig {
-        github_webhook_app:
-          GithubWebhookAppConfig {
-            app_id,
-            installations,
-            pk_path,
-            ..
-          },
-        ..
-      } = core_config();
-      if *app_id == 0 || installations.is_empty() {
-        return None;
-      }
-      let private_key = match std::fs::read(pk_path).with_context(|| format!("github webhook app | failed to load private key at {pk_path}")) {
-        Ok(key) => key,
-        Err(e) => {
-          error!("{e:#}");
-          return None;
-        }
-      };
-
-      let private_key = match nom_pem::decode_block(&private_key) {
-        Ok(key) => key,
-        Err(e) => {
-          error!("github webhook app | failed to decode private key at {pk_path} | {e:?}");
-          return None;
-        }
-      };
-
-      let jwt = match JWTCredentials::new(*app_id, private_key.data).context("failed to initialize github JWTCredentials") {
-        Ok(jwt) => jwt,
-        Err(e) => {
-          error!("github webhook app | failed to make github JWTCredentials | pk path: {pk_path} | {e:#}");
-          return None
-        }
-      };
-
-      let mut clients =
-        HashMap::with_capacity(installations.capacity());
-
-      for installation in installations {
-        let token_generator = InstallationTokenGenerator::new(
-          installation.id,
-          jwt.clone(),
-        );
-        let client = match octorust::Client::new(
-          "github-app",
-          Credentials::InstallationToken(token_generator),
-        ).with_context(|| format!("failed to initialize github webhook client for installation {}", installation.id)) {
-          Ok(client) => client,
-          Err(e) => {
-            error!("{e:#}");
-            continue;
-          }
-        };
-        clients.insert(installation.namespace.to_string(), client);
-      }
-
-      Some(clients)
-    })
-    .as_ref()
-}

bin/core/src/sync/resources.rs

@@ -693,6 +693,7 @@ impl ResourceSyncTrait for Procedure {
           Execution::BackupCoreDatabase(_) => {}
           Execution::GlobalAutoUpdate(_) => {}
           Execution::RotateAllServerKeys(_) => {}
+          Execution::RotateCoreKeys(_) => {}
           Execution::Sleep(_) => {}
         }
       }

bin/core/src/sync/toml.rs

@@ -811,7 +811,8 @@ impl ToToml for Procedure {
           | Execution::ClearRepoCache(_)
           | Execution::BackupCoreDatabase(_)
           | Execution::GlobalAutoUpdate(_)
-          | Execution::RotateAllServerKeys(_) => {}
+          | Execution::RotateAllServerKeys(_)
+          | Execution::RotateCoreKeys(_) => {}
         }
       }
     }

bin/core/src/ws/container.rs

@@ -4,22 +4,25 @@ use axum::{
 };
 use futures::SinkExt;
 use komodo_client::{
-  api::terminal::ConnectContainerExecQuery,
+  api::terminal::{
+    ConnectContainerAttachQuery, ConnectContainerExecQuery,
+  },
   entities::{permission::PermissionLevel, server::Server},
 };
 
 use crate::permission::get_check_permissions;
 
 #[instrument(name = "ConnectContainerExec", skip(ws))]
-pub async fn terminal(
+pub async fn exec(
   Query(ConnectContainerExecQuery {
     server,
     container,
     shell,
+    recreate,
   }): Query<ConnectContainerExecQuery>,
   ws: WebSocketUpgrade,
 ) -> impl IntoResponse {
-  ws.on_upgrade(|socket| async move {
+  ws.on_upgrade(async move |socket| {
     let Some((mut client_socket, user)) =
       super::user_ws_login(socket).await
     else {
@@ -44,11 +47,56 @@ pub async fn terminal(
       }
     };
 
-    super::handle_container_terminal(
+    super::handle_container_exec_terminal(
       client_socket,
       &server,
       container,
       shell,
+      recreate,
+    )
+    .await
+  })
+}
+
+#[instrument(name = "ConnectContainerAttach", skip(ws))]
+pub async fn attach(
+  Query(ConnectContainerAttachQuery {
+    server,
+    container,
+    recreate,
+  }): Query<ConnectContainerAttachQuery>,
+  ws: WebSocketUpgrade,
+) -> impl IntoResponse {
+  ws.on_upgrade(async move |socket| {
+    let Some((mut client_socket, user)) =
+      super::user_ws_login(socket).await
+    else {
+      return;
+    };
+
+    let server = match get_check_permissions::<Server>(
+      &server,
+      &user,
+      PermissionLevel::Read.terminal(),
+    )
+    .await
+    {
+      Ok(server) => server,
+      Err(e) => {
+        debug!("could not get server | {e:#}");
+        let _ = client_socket
+          .send(Message::text(format!("ERROR: {e:#}")))
+          .await;
+        let _ = client_socket.close().await;
+        return;
+      }
+    };
+
+    super::handle_container_attach_terminal(
+      client_socket,
+      &server,
+      container,
+      recreate,
     )
     .await
   })

bin/core/src/ws/deployment.rs

@@ -4,7 +4,9 @@ use axum::{
 };
 use futures::SinkExt;
 use komodo_client::{
-  api::terminal::ConnectDeploymentExecQuery,
+  api::terminal::{
+    ConnectDeploymentAttachQuery, ConnectDeploymentExecQuery,
+  },
   entities::{
     deployment::Deployment, permission::PermissionLevel,
     server::Server,
@@ -14,13 +16,15 @@ use komodo_client::{
 use crate::{permission::get_check_permissions, resource::get};
 
 #[instrument(name = "ConnectDeploymentExec", skip(ws))]
-pub async fn terminal(
-  Query(ConnectDeploymentExecQuery { deployment, shell }): Query<
-    ConnectDeploymentExecQuery,
-  >,
+pub async fn exec(
+  Query(ConnectDeploymentExecQuery {
+    deployment,
+    shell,
+    recreate,
+  }): Query<ConnectDeploymentExecQuery>,
   ws: WebSocketUpgrade,
 ) -> impl IntoResponse {
-  ws.on_upgrade(|socket| async move {
+  ws.on_upgrade(async move |socket| {
     let Some((mut client_socket, user)) =
       super::user_ws_login(socket).await
     else {
@@ -58,11 +62,68 @@ pub async fn terminal(
         }
       };
 
-    super::handle_container_terminal(
+    super::handle_container_exec_terminal(
       client_socket,
       &server,
       deployment.name,
       shell,
+      recreate,
+    )
+    .await
+  })
+}
+
+#[instrument(name = "ConnectDeploymentAttach", skip(ws))]
+pub async fn attach(
+  Query(ConnectDeploymentAttachQuery {
+    deployment,
+    recreate,
+  }): Query<ConnectDeploymentAttachQuery>,
+  ws: WebSocketUpgrade,
+) -> impl IntoResponse {
+  ws.on_upgrade(async move |socket| {
+    let Some((mut client_socket, user)) =
+      super::user_ws_login(socket).await
+    else {
+      return;
+    };
+
+    let deployment = match get_check_permissions::<Deployment>(
+      &deployment,
+      &user,
+      PermissionLevel::Read.terminal(),
+    )
+    .await
+    {
+      Ok(deployment) => deployment,
+      Err(e) => {
+        debug!("could not get deployment | {e:#}");
+        let _ = client_socket
+          .send(Message::text(format!("ERROR: {e:#}")))
+          .await;
+        let _ = client_socket.close().await;
+        return;
+      }
+    };
+
+    let server =
+      match get::<Server>(&deployment.config.server_id).await {
+        Ok(server) => server,
+        Err(e) => {
+          debug!("could not get server | {e:#}");
+          let _ = client_socket
+            .send(Message::text(format!("ERROR: {e:#}")))
+            .await;
+          let _ = client_socket.close().await;
+          return;
+        }
+      };
+
+    super::handle_container_attach_terminal(
+      client_socket,
+      &server,
+      deployment.name,
+      recreate,
     )
     .await
   })

bin/core/src/ws/mod.rs

@@ -7,22 +7,22 @@ use crate::{
 use anyhow::anyhow;
 use axum::{
   Router,
-  extract::ws::{Message, WebSocket},
+  extract::ws::{self, WebSocket},
   routing::get,
 };
 use bytes::Bytes;
 use futures::{SinkExt, StreamExt};
 use komodo_client::{
+  api::write::TerminalRecreateMode,
   entities::{server::Server, user::User},
   ws::WsLoginMessage,
 };
-use periphery_client::api::terminal::DisconnectTerminal;
-use tokio::sync::mpsc::{Receiver, Sender};
-use tokio_util::sync::CancellationToken;
-use transport::{
-  MessageState,
-  bytes::{data_from_transport_bytes, to_transport_bytes},
+use periphery_client::{
+  api::terminal::DisconnectTerminal,
+  transport::EncodedTransportMessage,
 };
+use tokio_util::sync::CancellationToken;
+use transport::channel::{Receiver, Sender};
 use uuid::Uuid;
 
 mod container;
@@ -38,9 +38,12 @@ pub fn router() -> Router {
     // User facing
     .route("/update", get(update::handler))
     .route("/terminal", get(terminal::handler))
-    .route("/container/terminal", get(container::terminal))
-    .route("/deployment/terminal", get(deployment::terminal))
-    .route("/stack/terminal", get(stack::terminal))
+    .route("/container/terminal", get(container::exec))
+    .route("/container/terminal/attach", get(container::attach))
+    .route("/deployment/terminal", get(deployment::exec))
+    .route("/deployment/terminal/attach", get(deployment::attach))
+    .route("/stack/terminal", get(stack::exec))
+    .route("/stack/terminal/attach", get(stack::attach))
 }
 
 #[instrument(level = "debug")]
@@ -48,7 +51,7 @@ async fn user_ws_login(
   mut socket: WebSocket,
 ) -> Option<(WebSocket, User)> {
   let login_msg = match socket.recv().await {
-    Some(Ok(Message::Text(login_msg))) => {
+    Some(Ok(ws::Message::Text(login_msg))) => {
       LoginMessage::Ok(login_msg.to_string())
     }
     Some(Ok(msg)) => {
@@ -64,7 +67,7 @@ async fn user_ws_login(
   let login_msg = match login_msg {
     LoginMessage::Ok(login_msg) => login_msg,
     LoginMessage::Err(msg) => {
-      let _ = socket.send(Message::text(msg)).await;
+      let _ = socket.send(ws::Message::text(msg)).await;
       let _ = socket.close().await;
       return None;
     }
@@ -74,12 +77,12 @@ async fn user_ws_login(
     Ok(WsLoginMessage::Jwt { jwt }) => {
       match auth_jwt_check_enabled(&jwt).await {
         Ok(user) => {
-          let _ = socket.send(Message::text("LOGGED_IN")).await;
+          let _ = socket.send(ws::Message::text("LOGGED_IN")).await;
           Some((socket, user))
         }
         Err(e) => {
           let _ = socket
-            .send(Message::text(format!(
+            .send(ws::Message::text(format!(
               "failed to authenticate user using jwt | {e:#}"
             )))
             .await;
@@ -92,12 +95,12 @@ async fn user_ws_login(
     Ok(WsLoginMessage::ApiKeys { key, secret }) => {
       match auth_api_key_check_enabled(&key, &secret).await {
         Ok(user) => {
-          let _ = socket.send(Message::text("LOGGED_IN")).await;
+          let _ = socket.send(ws::Message::text("LOGGED_IN")).await;
           Some((socket, user))
         }
         Err(e) => {
           let _ = socket
-            .send(Message::text(format!(
+            .send(ws::Message::text(format!(
               "failed to authenticate user using api keys | {e:#}"
             )))
             .await;
@@ -108,7 +111,7 @@ async fn user_ws_login(
     }
     Err(e) => {
       let _ = socket
-        .send(Message::text(format!(
+        .send(ws::Message::text(format!(
           "failed to parse login message: {e:#}"
         )))
         .await;
@@ -134,11 +137,12 @@ async fn check_user_valid(user_id: &str) -> anyhow::Result<User> {
   Ok(user)
 }
 
-async fn handle_container_terminal(
+async fn handle_container_exec_terminal(
   mut client_socket: WebSocket,
   server: &Server,
   container: String,
   shell: String,
+  recreate: TerminalRecreateMode,
 ) {
   let periphery = match crate::helpers::periphery_client(server).await
   {
@@ -146,7 +150,7 @@ async fn handle_container_terminal(
     Err(e) => {
       debug!("couldn't get periphery | {e:#}");
       let _ = client_socket
-        .send(Message::text(format!("ERROR: {e:#}")))
+        .send(ws::Message::text(format!("ERROR: {e:#}")))
         .await;
       let _ = client_socket.close().await;
       return;
@@ -156,14 +160,17 @@ async fn handle_container_terminal(
   trace!("connecting to periphery container exec websocket");
 
   let (periphery_connection_id, periphery_sender, periphery_receiver) =
-    match periphery.connect_container_exec(container, shell).await {
+    match periphery
+      .connect_container_exec(container, shell, recreate)
+      .await
+    {
       Ok(ws) => ws,
       Err(e) => {
         debug!(
           "Failed connect to periphery container exec websocket | {e:#}"
         );
         let _ = client_socket
-          .send(Message::text(format!("ERROR: {e:#}")))
+          .send(ws::Message::text(format!("ERROR: {e:#}")))
           .await;
         let _ = client_socket.close().await;
         return;
@@ -182,35 +189,84 @@ async fn handle_container_terminal(
   .await
 }
 
+async fn handle_container_attach_terminal(
+  mut client_socket: WebSocket,
+  server: &Server,
+  container: String,
+  recreate: TerminalRecreateMode,
+) {
+  let periphery = match crate::helpers::periphery_client(server).await
+  {
+    Ok(periphery) => periphery,
+    Err(e) => {
+      debug!("couldn't get periphery | {e:#}");
+      let _ = client_socket
+        .send(ws::Message::text(format!("ERROR: {e:#}")))
+        .await;
+      let _ = client_socket.close().await;
+      return;
+    }
+  };
+
+  trace!("connecting to periphery container exec websocket");
+
+  let (periphery_connection_id, periphery_sender, periphery_receiver) =
+    match periphery
+      .connect_container_attach(container, recreate)
+      .await
+    {
+      Ok(ws) => ws,
+      Err(e) => {
+        debug!(
+          "Failed connect to periphery container attach websocket | {e:#}"
+        );
+        let _ = client_socket
+          .send(ws::Message::text(format!("ERROR: {e:#}")))
+          .await;
+        let _ = client_socket.close().await;
+        return;
+      }
+    };
+
+  trace!("connected to periphery container attach websocket");
+
+  forward_ws_channel(
+    periphery,
+    client_socket,
+    periphery_connection_id,
+    periphery_sender,
+    periphery_receiver,
+  )
+  .await
+}
+
 async fn forward_ws_channel(
   periphery: PeripheryClient,
   client_socket: axum::extract::ws::WebSocket,
   periphery_connection_id: Uuid,
-  periphery_sender: Sender<Bytes>,
-  mut periphery_receiver: Receiver<Bytes>,
+  periphery_sender: Sender<EncodedTransportMessage>,
+  mut periphery_receiver: Receiver<Vec<u8>>,
 ) {
-  let (mut core_send, mut core_receive) = client_socket.split();
+  let (mut client_send, mut client_receive) = client_socket.split();
   let cancel = CancellationToken::new();
 
+  periphery_receiver.set_cancel(cancel.clone());
+
   trace!("starting ws exchange");
 
   let core_to_periphery = async {
     loop {
-      let res = tokio::select! {
-        res = core_receive.next() => res,
+      let client_recv_res = tokio::select! {
+        res = client_receive.next() => res,
         _ = cancel.cancelled() => {
           trace!("core to periphery read: cancelled from inside");
           break;
         }
       };
-      match res {
-        Some(Ok(Message::Binary(data))) => {
+      match client_recv_res {
+        Some(Ok(ws::Message::Binary(bytes))) => {
           if let Err(e) = periphery_sender
-            .send(to_transport_bytes(
-              data.into(),
-              periphery_connection_id,
-              MessageState::Terminal,
-            ))
+            .send_terminal(periphery_connection_id, bytes)
             .await
           {
             debug!("Failed to send terminal message | {e:?}",);
@@ -218,14 +274,10 @@ async fn forward_ws_channel(
             break;
           };
         }
-        Some(Ok(Message::Text(data))) => {
-          let data: Bytes = data.into();
+        Some(Ok(ws::Message::Text(text))) => {
+          let bytes: Bytes = text.into();
           if let Err(e) = periphery_sender
-            .send(to_transport_bytes(
-              data.into(),
-              periphery_connection_id,
-              MessageState::Terminal,
-            ))
+            .send_terminal(periphery_connection_id, bytes)
             .await
           {
             debug!("Failed to send terminal message | {e:?}",);
@@ -234,7 +286,7 @@ async fn forward_ws_channel(
           };
         }
         // TODO: Disconnect from periphery when client disconnects
-        Some(Ok(Message::Close(_frame))) => {
+        Some(Ok(ws::Message::Close(_frame))) => {
           cancel.cancel();
           break;
         }
@@ -254,26 +306,20 @@ async fn forward_ws_channel(
 
   let periphery_to_core = async {
     loop {
-      let res = tokio::select! {
-        res = periphery_receiver.recv() => res.map(data_from_transport_bytes),
-        _ = cancel.cancelled() => {
-          trace!("periphery to core read: cancelled from inside");
-          break;
-        }
-      };
-      match res {
-        Some(Ok(bytes)) => {
-          if let Err(e) = core_send.send(Message::Binary(bytes)).await
+      // Already adheres to cancellation token
+      match periphery_receiver.recv().await {
+        Ok(bytes) => {
+          if let Err(e) =
+            client_send.send(ws::Message::Binary(bytes.into())).await
           {
             debug!("{e:?}");
             cancel.cancel();
             break;
           };
         }
-        // No data, ignore
-        Some(Err(_e)) => {}
-        None => {
-          let _ = core_send.send(Message::text("STREAM EOF")).await;
+        Err(_) => {
+          let _ =
+            client_send.send(ws::Message::text("STREAM EOF")).await;
           cancel.cancel();
           break;
         }
@@ -297,6 +343,6 @@ async fn forward_ws_channel(
   if let Some(connection) =
     periphery_connections().get(&periphery.id).await
   {
-    connection.channels.remove(&periphery_connection_id).await;
+    connection.terminals.remove(&periphery_connection_id).await;
   }
 }

bin/core/src/ws/stack.rs

@@ -4,7 +4,7 @@ use axum::{
 };
 use futures::SinkExt;
 use komodo_client::{
-  api::terminal::ConnectStackExecQuery,
+  api::terminal::{ConnectStackAttachQuery, ConnectStackExecQuery},
   entities::{
     permission::PermissionLevel, server::Server, stack::Stack,
   },
@@ -16,97 +16,135 @@ use crate::{
 };
 
 #[instrument(name = "ConnectStackExec", skip(ws))]
-pub async fn terminal(
+pub async fn exec(
   Query(ConnectStackExecQuery {
     stack,
     service,
     shell,
+    recreate,
   }): Query<ConnectStackExecQuery>,
   ws: WebSocketUpgrade,
 ) -> impl IntoResponse {
-  ws.on_upgrade(|socket| async move {
-    let Some((mut client_socket, user)) =
-      super::user_ws_login(socket).await
+  ws.on_upgrade(async move |socket| {
+    let Some((client_socket, server, container)) =
+      login_get_server_container(socket, &stack, &service).await
     else {
       return;
     };
 
-    let stack = match get_check_permissions::<Stack>(
-      &stack,
-      &user,
-      PermissionLevel::Read.terminal(),
-    )
-    .await
-    {
-      Ok(stack) => stack,
-      Err(e) => {
-        debug!("could not get stack | {e:#}");
-        let _ = client_socket
-          .send(Message::text(format!("ERROR: {e:#}")))
-          .await;
-        let _ = client_socket.close().await;
-        return;
-      }
-    };
-
-    let server = match get::<Server>(&stack.config.server_id).await {
-      Ok(server) => server,
-      Err(e) => {
-        debug!("could not get server | {e:#}");
-        let _ = client_socket
-          .send(Message::text(format!("ERROR: {e:#}")))
-          .await;
-        let _ = client_socket.close().await;
-        return;
-      }
-    };
-
-    let Some(status) = stack_status_cache().get(&stack.id).await
-    else {
-      debug!("could not get stack status");
-      let _ = client_socket
-        .send(Message::text(String::from(
-          "ERROR: could not get stack status",
-        )))
-        .await;
-      let _ = client_socket.close().await;
-      return;
-    };
-
-    let container = match status
-      .curr
-      .services
-      .iter()
-      .find(|s| s.service == service)
-      .map(|s| s.container.as_ref())
-    {
-      Some(Some(container)) => container.name.clone(),
-      Some(None) => {
-        let _ = client_socket
-          .send(Message::text(format!(
-            "ERROR: Service {service} container could not be found"
-          )))
-          .await;
-        let _ = client_socket.close().await;
-        return;
-      }
-      None => {
-        let _ = client_socket
-          .send(Message::text(format!(
-            "ERROR: Service {service} could not be found"
-          )))
-          .await;
-        let _ = client_socket.close().await;
-        return;
-      }
-    };
-
-    super::handle_container_terminal(
+    super::handle_container_exec_terminal(
       client_socket,
       &server,
       container,
       shell,
+      recreate,
     )
     .await
   })
 }
+
+#[instrument(name = "ConnectStackAttach", skip(ws))]
+pub async fn attach(
+  Query(ConnectStackAttachQuery {
+    stack,
+    service,
+    recreate,
+  }): Query<ConnectStackAttachQuery>,
+  ws: WebSocketUpgrade,
+) -> impl IntoResponse {
+  ws.on_upgrade(async move |socket| {
+    let Some((client_socket, server, container)) =
+      login_get_server_container(socket, &stack, &service).await
+    else {
+      return;
+    };
+
+    super::handle_container_attach_terminal(
+      client_socket,
+      &server,
+      container,
+      recreate,
+    )
+    .await
+  })
+}
+
+async fn login_get_server_container(
+  socket: axum::extract::ws::WebSocket,
+  stack: &str,
+  service: &str,
+) -> Option<(axum::extract::ws::WebSocket, Server, String)> {
+  let (mut client_socket, user) =
+    super::user_ws_login(socket).await?;
+
+  let stack = match get_check_permissions::<Stack>(
+    stack,
+    &user,
+    PermissionLevel::Read.terminal(),
+  )
+  .await
+  {
+    Ok(stack) => stack,
+    Err(e) => {
+      debug!("could not get stack | {e:#}");
+      let _ = client_socket
+        .send(Message::text(format!("ERROR: {e:#}")))
+        .await;
+      let _ = client_socket.close().await;
+      return None;
+    }
+  };
+
+  let server = match get::<Server>(&stack.config.server_id).await {
+    Ok(server) => server,
+    Err(e) => {
+      debug!("could not get server | {e:#}");
+      let _ = client_socket
+        .send(Message::text(format!("ERROR: {e:#}")))
+        .await;
+      let _ = client_socket.close().await;
+      return None;
+    }
+  };
+
+  let Some(status) = stack_status_cache().get(&stack.id).await else {
+    debug!("could not get stack status");
+    let _ = client_socket
+      .send(Message::text(String::from(
+        "ERROR: could not get stack status",
+      )))
+      .await;
+    let _ = client_socket.close().await;
+    return None;
+  };
+
+  let container = match status
+    .curr
+    .services
+    .iter()
+    .find(|s| s.service == service)
+    .map(|s| s.container.as_ref())
+  {
+    Some(Some(container)) => container.name.clone(),
+    Some(None) => {
+      let _ = client_socket
+        .send(Message::text(format!(
+          "ERROR: Service {service} container could not be found"
+        )))
+        .await;
+      let _ = client_socket.close().await;
+      return None;
+    }
+    None => {
+      let _ = client_socket
+        .send(Message::text(format!(
+          "ERROR: Service {service} could not be found"
+        )))
+        .await;
+      let _ = client_socket.close().await;
+      return None;
+    }
+  };
+
+  Some((client_socket, server, container))
+}

bin/periphery/Cargo.toml

@@ -20,9 +20,10 @@ periphery_client.workspace = true
 environment_file.workspace = true
 environment.workspace = true
 interpolate.workspace = true
+secret_file.workspace = true
 formatting.workspace = true
 transport.workspace = true
-response.workspace = true
+encoding.workspace = true
 command.workspace = true
 config.workspace = true
 logger.workspace = true

bin/periphery/aio.Dockerfile

@@ -31,6 +31,9 @@ ENV PERIPHERY_PRIVATE_KEY="file:/config/keys/periphery.key"
 
 CMD [ "periphery" ]
 
+# Label to prevent Komodo from stopping with StopAllContainers
+LABEL komodo.skip="true"
+# Label for ghcr
 LABEL org.opencontainers.image.source="https://github.com/moghtech/komodo"
 LABEL org.opencontainers.image.description="Komodo Periphery"
 LABEL org.opencontainers.image.licenses="GPL-3.0"

bin/periphery/multi-arch.Dockerfile

@@ -34,6 +34,9 @@ ENV PERIPHERY_PRIVATE_KEY="file:/config/keys/periphery.key"
 
 CMD [ "periphery" ]
 
+# Label to prevent Komodo from stopping with StopAllContainers
+LABEL komodo.skip="true"
+# Label for ghcr
 LABEL org.opencontainers.image.source="https://github.com/moghtech/komodo"
 LABEL org.opencontainers.image.description="Komodo Periphery"
 LABEL org.opencontainers.image.licenses="GPL-3.0"

bin/periphery/single-arch.Dockerfile

@@ -23,6 +23,9 @@ ENV PERIPHERY_PRIVATE_KEY="file:/config/keys/periphery.key"
 
 CMD [ "periphery" ]
 
+# Label to prevent Komodo from stopping with StopAllContainers
+LABEL komodo.skip="true"
+# Label for ghcr
 LABEL org.opencontainers.image.source="https://github.com/moghtech/komodo"
 LABEL org.opencontainers.image.description="Komodo Periphery"
 LABEL org.opencontainers.image.licenses="GPL-3.0"

bin/periphery/src/api/build/helpers.rs

@@ -26,14 +26,7 @@ pub async fn write_dockerfile(
       .components()
       .collect::<PathBuf>();
 
-    // Ensure parent directory exists
-    if let Some(parent) = full_dockerfile_path.parent() && !parent.exists() {
-      tokio::fs::create_dir_all(parent)
-        .await
-        .with_context(|| format!("Failed to initialize dockerfile parent directory {parent:?}"))?;
-    }
-
-    tokio::fs::write(&full_dockerfile_path, dockerfile).await.with_context(|| {
+    secret_file::write_async(&full_dockerfile_path, dockerfile).await.with_context(|| {
       format!(
         "Failed to write dockerfile contents to {full_dockerfile_path:?}"
       )
@@ -86,12 +79,14 @@ pub async fn parse_secret_args(
     }
     // Write the value to file to mount
     let path = build_dir.join(variable);
-    tokio::fs::write(&path, value).await.with_context(|| {
-      format!(
-        "Failed to write build secret {variable} to {}",
-        path.display()
-      )
-    })?;
+    secret_file::write_async(&path, value).await.with_context(
+      || {
+        format!(
+          "Failed to write build secret {variable} to {}",
+          path.display()
+        )
+      },
+    )?;
     // Extend the command
     write!(
       &mut res,

bin/periphery/src/api/build/mod.rs

@@ -38,7 +38,7 @@ impl Resolve<super::Args> for GetDockerfileContentsOnHost {
   async fn resolve(
     self,
     _: &super::Args,
-  ) -> serror::Result<GetDockerfileContentsOnHostResponse> {
+  ) -> anyhow::Result<GetDockerfileContentsOnHostResponse> {
     let GetDockerfileContentsOnHost {
       name,
       build_path,
@@ -84,7 +84,7 @@ impl Resolve<super::Args> for WriteDockerfileContentsToHost {
       dockerfile_path = &self.dockerfile_path,
     )
   )]
-  async fn resolve(self, _: &super::Args) -> serror::Result<Log> {
+  async fn resolve(self, _: &super::Args) -> anyhow::Result<Log> {
     let WriteDockerfileContentsToHost {
       name,
       build_path,
@@ -106,9 +106,13 @@ impl Resolve<super::Args> for WriteDockerfileContentsToHost {
         .await
         .with_context(|| format!("Failed to initialize dockerfile parent directory {parent:?}"))?;
     }
-    fs::write(&full_path, contents).await.with_context(|| {
-      format!("Failed to write dockerfile contents to {full_path:?}")
-    })?;
+    secret_file::write_async(&full_path, contents)
+      .await
+      .with_context(|| {
+        format!(
+          "Failed to write dockerfile contents to {full_path:?}"
+        )
+      })?;
     Ok(Log::simple(
       "Write dockerfile to host",
       format!("dockerfile contents written to {full_path:?}"),
@@ -121,7 +125,7 @@ impl Resolve<super::Args> for build::Build {
   async fn resolve(
     self,
     _: &super::Args,
-  ) -> serror::Result<Vec<Log>> {
+  ) -> anyhow::Result<Vec<Log>> {
     let build::Build {
       mut build,
       repo: linked_repo,
@@ -168,7 +172,9 @@ impl Resolve<super::Args> for build::Build {
       && linked_repo.is_none()
       && dockerfile.is_empty()
     {
-      return Err(anyhow!("Build must be files on host mode, have a repo attached, or have dockerfile contents set to build").into());
+      return Err(anyhow!(
+        "Build must be files on host mode, have a repo attached, or have dockerfile contents set to build"
+      ));
     }
 
     let registry_tokens = registry_tokens
@@ -319,7 +325,7 @@ impl Resolve<super::Args> for build::Build {
 
 impl Resolve<super::Args> for PruneBuilders {
   #[instrument(name = "PruneBuilders", skip_all)]
-  async fn resolve(self, _: &super::Args) -> serror::Result<Log> {
+  async fn resolve(self, _: &super::Args) -> anyhow::Result<Log> {
     let command = String::from("docker builder prune -a -f");
     Ok(run_komodo_command("Prune Builders", None, command).await)
   }
@@ -329,7 +335,7 @@ impl Resolve<super::Args> for PruneBuilders {
 
 impl Resolve<super::Args> for PruneBuildx {
   #[instrument(name = "PruneBuildx", skip_all)]
-  async fn resolve(self, _: &super::Args) -> serror::Result<Log> {
+  async fn resolve(self, _: &super::Args) -> anyhow::Result<Log> {
     let command = String::from("docker buildx prune -a -f");
     Ok(run_komodo_command("Prune Buildx", None, command).await)
   }

bin/periphery/src/api/compose/helpers.rs

@@ -166,8 +166,7 @@ pub async fn pull_or_clone_stack(
     replacers: Default::default(),
   }
   .resolve(req_args)
-  .await
-  .map_err(|e| e.error)?;
+  .await?;
 
   Ok(root)
 }

bin/periphery/src/api/compose/mod.rs

@@ -1,3 +1,5 @@
+use std::{borrow::Cow, path::PathBuf};
+
 use anyhow::{Context, anyhow};
 use command::{
   run_komodo_command, run_komodo_command_with_sanitization,
@@ -18,8 +20,6 @@ use periphery_client::api::compose::*;
 use resolver_api::Resolve;
 use serde::{Deserialize, Serialize};
 use shell_escape::unix::escape;
-use std::{borrow::Cow, path::PathBuf};
-use tokio::fs;
 
 use crate::{
   config::periphery_config,
@@ -94,7 +94,7 @@ pub struct DockerComposeLsItem {
 
 impl Resolve<super::Args> for GetComposeLog {
   #[instrument(name = "GetComposeLog", level = "debug")]
-  async fn resolve(self, _: &super::Args) -> serror::Result<Log> {
+  async fn resolve(self, _: &super::Args) -> anyhow::Result<Log> {
     let GetComposeLog {
       project,
       services,
@@ -117,7 +117,7 @@ impl Resolve<super::Args> for GetComposeLog {
 
 impl Resolve<super::Args> for GetComposeLogSearch {
   #[instrument(name = "GetComposeLogSearch", level = "debug")]
-  async fn resolve(self, _: &super::Args) -> serror::Result<Log> {
+  async fn resolve(self, _: &super::Args) -> anyhow::Result<Log> {
     let GetComposeLogSearch {
       project,
       services,
@@ -148,7 +148,7 @@ impl Resolve<super::Args> for GetComposeContentsOnHost {
   async fn resolve(
     self,
     _: &super::Args,
-  ) -> serror::Result<GetComposeContentsOnHostResponse> {
+  ) -> anyhow::Result<GetComposeContentsOnHostResponse> {
     let GetComposeContentsOnHost {
       name,
       run_directory,
@@ -160,12 +160,6 @@ impl Resolve<super::Args> for GetComposeContentsOnHost {
     let run_directory =
       root.join(&run_directory).components().collect::<PathBuf>();
 
-    if !run_directory.exists() {
-      fs::create_dir_all(&run_directory)
-        .await
-        .context("Failed to initialize run directory")?;
-    }
-
     let mut res = GetComposeContentsOnHostResponse::default();
 
     for file in file_paths {
@@ -173,11 +167,13 @@ impl Resolve<super::Args> for GetComposeContentsOnHost {
         .join(&file.path)
         .components()
         .collect::<PathBuf>();
-      match fs::read_to_string(&full_path).await.with_context(|| {
-        format!(
-          "Failed to read compose file contents at {full_path:?}"
-        )
-      }) {
+      match tokio::fs::read_to_string(&full_path).await.with_context(
+        || {
+          format!(
+            "Failed to read compose file contents at {full_path:?}"
+          )
+        },
+      ) {
         Ok(contents) => {
           // The path we store here has to be the same as incoming file path in the array,
           // in order for WriteComposeContentsToHost to write to the correct path.
@@ -213,7 +209,7 @@ impl Resolve<super::Args> for WriteComposeContentsToHost {
       file_path = &self.file_path,
     )
   )]
-  async fn resolve(self, _: &super::Args) -> serror::Result<Log> {
+  async fn resolve(self, _: &super::Args) -> anyhow::Result<Log> {
     let WriteComposeContentsToHost {
       name,
       run_directory,
@@ -227,17 +223,13 @@ impl Resolve<super::Args> for WriteComposeContentsToHost {
       .join(file_path)
       .components()
       .collect::<PathBuf>();
-    // Ensure parent directory exists
-    if let Some(parent) = file_path.parent() {
-      fs::create_dir_all(&parent)
-        .await
-        .with_context(|| format!("Failed to initialize compose file parent directory {parent:?}"))?;
-    }
-    fs::write(&file_path, contents).await.with_context(|| {
-      format!(
-        "Failed to write compose file contents to {file_path:?}"
-      )
-    })?;
+    secret_file::write_async(&file_path, contents)
+      .await
+      .with_context(|| {
+        format!(
+          "Failed to write compose file contents to {file_path:?}"
+        )
+      })?;
     Ok(Log::simple(
       "Write contents to host",
       format!("File contents written to {file_path:?}"),
@@ -260,7 +252,7 @@ impl Resolve<super::Args> for WriteCommitComposeContents {
   async fn resolve(
     self,
     args: &super::Args,
-  ) -> serror::Result<RepoExecutionResponse> {
+  ) -> anyhow::Result<RepoExecutionResponse> {
     let WriteCommitComposeContents {
       stack,
       repo,
@@ -295,7 +287,6 @@ impl Resolve<super::Args> for WriteCommitComposeContents {
       &stack.config.branch,
     )
     .await
-    .map_err(Into::into)
   }
 }
 
@@ -313,7 +304,7 @@ impl Resolve<super::Args> for ComposePull {
   async fn resolve(
     self,
     args: &super::Args,
-  ) -> serror::Result<ComposePullResponse> {
+  ) -> anyhow::Result<ComposePullResponse> {
     let ComposePull {
       mut stack,
       repo,
@@ -373,7 +364,7 @@ impl Resolve<super::Args> for ComposePull {
     // Validate files
     for (full_path, path) in &file_paths {
       if !full_path.exists() {
-        return Err(anyhow!("Missing compose file at {path}").into());
+        return Err(anyhow!("Missing compose file at {path}"));
       }
     }
 
@@ -428,7 +419,7 @@ impl Resolve<super::Args> for ComposeUp {
   async fn resolve(
     self,
     args: &super::Args,
-  ) -> serror::Result<ComposeUpResponse> {
+  ) -> anyhow::Result<ComposeUpResponse> {
     let ComposeUp {
       mut stack,
       repo,
@@ -686,7 +677,7 @@ impl Resolve<super::Args> for ComposeUp {
 
 impl Resolve<super::Args> for ComposeExecution {
   #[instrument(name = "ComposeExecution")]
-  async fn resolve(self, _: &super::Args) -> serror::Result<Log> {
+  async fn resolve(self, _: &super::Args) -> anyhow::Result<Log> {
     let ComposeExecution { project, command } = self;
     let docker_compose = docker_compose();
     let log = run_komodo_command(
@@ -703,7 +694,7 @@ impl Resolve<super::Args> for ComposeExecution {
 
 impl Resolve<super::Args> for ComposeRun {
   #[instrument(name = "ComposeRun", level = "debug", skip_all, fields(stack = &self.stack.name, service = &self.service))]
-  async fn resolve(self, args: &super::Args) -> serror::Result<Log> {
+  async fn resolve(self, args: &super::Args) -> anyhow::Result<Log> {
     let ComposeRun {
       mut stack,
       repo,

bin/periphery/src/api/compose/write.rs

@@ -167,8 +167,7 @@ async fn write_stack_linked_repo<'a>(
       replacers,
     }
     .resolve(req_args)
-    .await
-    .map_err(|e| e.error)?
+    .await?
   } else {
     PullOrCloneRepo {
       args,
@@ -181,8 +180,7 @@ async fn write_stack_linked_repo<'a>(
       replacers,
     }
     .resolve(req_args)
-    .await
-    .map_err(|e| e.error)?
+    .await?
   };
 
   res.logs().extend(clone_res.res.logs);
@@ -253,8 +251,7 @@ async fn write_stack_inline_repo<'a>(
       replacers: Default::default(),
     }
     .resolve(req_args)
-    .await
-    .map_err(|e| e.error)?
+    .await?
   } else {
     PullOrCloneRepo {
       args,
@@ -267,8 +264,7 @@ async fn write_stack_inline_repo<'a>(
       replacers: Default::default(),
     }
     .resolve(req_args)
-    .await
-    .map_err(|e| e.error)?
+    .await?
   };
 
   res.logs().extend(clone_res.res.logs);
@@ -354,7 +350,7 @@ async fn write_stack_ui_defined(
     .components()
     .collect::<PathBuf>();
 
-  fs::write(&file_path, &stack.config.file_contents)
+  secret_file::write_async(&file_path, &stack.config.file_contents)
     .await
     .with_context(|| {
       format!("Failed to write compose file to {file_path:?}")

bin/periphery/src/api/container.rs

@@ -12,10 +12,9 @@ use periphery_client::api::container::*;
 use resolver_api::Resolve;
 
 use crate::{
-  docker::{
-    docker_client, stats::get_container_stats, stop_container_command,
-  },
+  docker::{stats::get_container_stats, stop_container_command},
   helpers::format_log_grep,
+  state::docker_client,
 };
 
 // ======
@@ -29,8 +28,13 @@ impl Resolve<super::Args> for InspectContainer {
   async fn resolve(
     self,
     _: &super::Args,
-  ) -> serror::Result<Container> {
-    Ok(docker_client().inspect_container(&self.name).await?)
+  ) -> anyhow::Result<Container> {
+    let client = docker_client().load();
+    let client = client
+      .iter()
+      .next()
+      .context("Could not connect to docker client")?;
+    client.inspect_container(&self.name).await
   }
 }
 
@@ -38,7 +42,7 @@ impl Resolve<super::Args> for InspectContainer {
 
 impl Resolve<super::Args> for GetContainerLog {
   #[instrument(name = "GetContainerLog", level = "debug")]
-  async fn resolve(self, _: &super::Args) -> serror::Result<Log> {
+  async fn resolve(self, _: &super::Args) -> anyhow::Result<Log> {
     let GetContainerLog {
       name,
       tail,
@@ -59,7 +63,7 @@ impl Resolve<super::Args> for GetContainerLog {
 
 impl Resolve<super::Args> for GetContainerLogSearch {
   #[instrument(name = "GetContainerLogSearch", level = "debug")]
-  async fn resolve(self, _: &super::Args) -> serror::Result<Log> {
+  async fn resolve(self, _: &super::Args) -> anyhow::Result<Log> {
     let GetContainerLogSearch {
       name,
       terms,
@@ -90,7 +94,7 @@ impl Resolve<super::Args> for GetContainerStats {
   async fn resolve(
     self,
     _: &super::Args,
-  ) -> serror::Result<ContainerStats> {
+  ) -> anyhow::Result<ContainerStats> {
     let mut stats = get_container_stats(Some(self.name)).await?;
     let stats =
       stats.pop().context("No stats found for container")?;
@@ -105,11 +109,13 @@ impl Resolve<super::Args> for GetFullContainerStats {
   async fn resolve(
     self,
     _: &super::Args,
-  ) -> serror::Result<FullContainerStats> {
-    docker_client()
-      .full_container_stats(&self.name)
-      .await
-      .map_err(Into::into)
+  ) -> anyhow::Result<FullContainerStats> {
+    let client = docker_client().load();
+    let client = client
+      .iter()
+      .next()
+      .context("Could not connect to docker client")?;
+    client.full_container_stats(&self.name).await
   }
 }
 
@@ -120,8 +126,8 @@ impl Resolve<super::Args> for GetContainerStatsList {
   async fn resolve(
     self,
     _: &super::Args,
-  ) -> serror::Result<Vec<ContainerStats>> {
-    Ok(get_container_stats(None).await?)
+  ) -> anyhow::Result<Vec<ContainerStats>> {
+    get_container_stats(None).await
   }
 }
 
@@ -131,7 +137,7 @@ impl Resolve<super::Args> for GetContainerStatsList {
 
 impl Resolve<super::Args> for StartContainer {
   #[instrument(name = "StartContainer")]
-  async fn resolve(self, _: &super::Args) -> serror::Result<Log> {
+  async fn resolve(self, _: &super::Args) -> anyhow::Result<Log> {
     Ok(
       run_komodo_command(
         "Docker Start",
@@ -147,7 +153,7 @@ impl Resolve<super::Args> for StartContainer {
 
 impl Resolve<super::Args> for RestartContainer {
   #[instrument(name = "RestartContainer")]
-  async fn resolve(self, _: &super::Args) -> serror::Result<Log> {
+  async fn resolve(self, _: &super::Args) -> anyhow::Result<Log> {
     Ok(
       run_komodo_command(
         "Docker Restart",
@@ -163,7 +169,7 @@ impl Resolve<super::Args> for RestartContainer {
 
 impl Resolve<super::Args> for PauseContainer {
   #[instrument(name = "PauseContainer")]
-  async fn resolve(self, _: &super::Args) -> serror::Result<Log> {
+  async fn resolve(self, _: &super::Args) -> anyhow::Result<Log> {
     Ok(
       run_komodo_command(
         "Docker Pause",
@@ -177,7 +183,7 @@ impl Resolve<super::Args> for PauseContainer {
 
 impl Resolve<super::Args> for UnpauseContainer {
   #[instrument(name = "UnpauseContainer")]
-  async fn resolve(self, _: &super::Args) -> serror::Result<Log> {
+  async fn resolve(self, _: &super::Args) -> anyhow::Result<Log> {
     Ok(
       run_komodo_command(
         "Docker Unpause",
@@ -193,7 +199,7 @@ impl Resolve<super::Args> for UnpauseContainer {
 
 impl Resolve<super::Args> for StopContainer {
   #[instrument(name = "StopContainer")]
-  async fn resolve(self, _: &super::Args) -> serror::Result<Log> {
+  async fn resolve(self, _: &super::Args) -> anyhow::Result<Log> {
     let StopContainer { name, signal, time } = self;
     let command = stop_container_command(&name, signal, time);
     let log = run_komodo_command("Docker Stop", None, command).await;
@@ -220,7 +226,7 @@ impl Resolve<super::Args> for StopContainer {
 
 impl Resolve<super::Args> for RemoveContainer {
   #[instrument(name = "RemoveContainer")]
-  async fn resolve(self, _: &super::Args) -> serror::Result<Log> {
+  async fn resolve(self, _: &super::Args) -> anyhow::Result<Log> {
     let RemoveContainer { name, signal, time } = self;
     let stop_command = stop_container_command(&name, signal, time);
     let command =
@@ -254,7 +260,7 @@ impl Resolve<super::Args> for RemoveContainer {
 
 impl Resolve<super::Args> for RenameContainer {
   #[instrument(name = "RenameContainer")]
-  async fn resolve(self, _: &super::Args) -> serror::Result<Log> {
+  async fn resolve(self, _: &super::Args) -> anyhow::Result<Log> {
     let RenameContainer {
       curr_name,
       new_name,
@@ -268,7 +274,7 @@ impl Resolve<super::Args> for RenameContainer {
 
 impl Resolve<super::Args> for PruneContainers {
   #[instrument(name = "PruneContainers", skip_all)]
-  async fn resolve(self, _: &super::Args) -> serror::Result<Log> {
+  async fn resolve(self, _: &super::Args) -> anyhow::Result<Log> {
     let command = String::from("docker container prune -f");
     Ok(run_komodo_command("Prune Containers", None, command).await)
   }
@@ -281,14 +287,21 @@ impl Resolve<super::Args> for StartAllContainers {
   async fn resolve(
     self,
     _: &super::Args,
-  ) -> serror::Result<Vec<Log>> {
-    let containers = docker_client()
+  ) -> anyhow::Result<Vec<Log>> {
+    let client = docker_client().load();
+    let client = client
+      .iter()
+      .next()
+      .context("Could not connect to docker client")?;
+    let containers = client
       .list_containers()
       .await
       .context("failed to list all containers on host")?;
     let futures = containers.iter().filter_map(
       |ContainerListItem { name, labels, .. }| {
-        if labels.contains_key("komodo.skip") {
+        if let Some(skip) = labels.get("komodo.skip")
+          && skip != "false"
+        {
           return None;
         }
         let command = format!("docker start {name}");
@@ -308,14 +321,21 @@ impl Resolve<super::Args> for RestartAllContainers {
   async fn resolve(
     self,
     _: &super::Args,
-  ) -> serror::Result<Vec<Log>> {
-    let containers = docker_client()
+  ) -> anyhow::Result<Vec<Log>> {
+    let client = docker_client().load();
+    let client = client
+      .iter()
+      .next()
+      .context("Could not connect to docker client")?;
+    let containers = client
       .list_containers()
       .await
       .context("failed to list all containers on host")?;
     let futures = containers.iter().filter_map(
       |ContainerListItem { name, labels, .. }| {
-        if labels.contains_key("komodo.skip") {
+        if let Some(skip) = labels.get("komodo.skip")
+          && skip != "false"
+        {
           return None;
         }
         let command = format!("docker restart {name}");
@@ -335,14 +355,21 @@ impl Resolve<super::Args> for PauseAllContainers {
   async fn resolve(
     self,
     _: &super::Args,
-  ) -> serror::Result<Vec<Log>> {
-    let containers = docker_client()
+  ) -> anyhow::Result<Vec<Log>> {
+    let client = docker_client().load();
+    let client = client
+      .iter()
+      .next()
+      .context("Could not connect to docker client")?;
+    let containers = client
       .list_containers()
       .await
       .context("failed to list all containers on host")?;
     let futures = containers.iter().filter_map(
       |ContainerListItem { name, labels, .. }| {
-        if labels.contains_key("komodo.skip") {
+        if let Some(skip) = labels.get("komodo.skip")
+          && skip != "false"
+        {
           return None;
         }
         let command = format!("docker pause {name}");
@@ -362,14 +389,21 @@ impl Resolve<super::Args> for UnpauseAllContainers {
   async fn resolve(
     self,
     _: &super::Args,
-  ) -> serror::Result<Vec<Log>> {
-    let containers = docker_client()
+  ) -> anyhow::Result<Vec<Log>> {
+    let client = docker_client().load();
+    let client = client
+      .iter()
+      .next()
+      .context("Could not connect to docker client")?;
+    let containers = client
       .list_containers()
       .await
       .context("failed to list all containers on host")?;
     let futures = containers.iter().filter_map(
       |ContainerListItem { name, labels, .. }| {
-        if labels.contains_key("komodo.skip") {
+        if let Some(skip) = labels.get("komodo.skip")
+          && skip != "false"
+        {
           return None;
         }
         let command = format!("docker unpause {name}");
@@ -389,14 +423,21 @@ impl Resolve<super::Args> for StopAllContainers {
   async fn resolve(
     self,
     _: &super::Args,
-  ) -> serror::Result<Vec<Log>> {
-    let containers = docker_client()
+  ) -> anyhow::Result<Vec<Log>> {
+    let client = docker_client().load();
+    let client = client
+      .iter()
+      .next()
+      .context("Could not connect to docker client")?;
+    let containers = client
       .list_containers()
       .await
       .context("failed to list all containers on host")?;
     let futures = containers.iter().filter_map(
       |ContainerListItem { name, labels, .. }| {
-        if labels.contains_key("komodo.skip") {
+        if let Some(skip) = labels.get("komodo.skip")
+          && skip != "false"
+        {
           return None;
         }
         Some(async move {

bin/periphery/src/api/deploy.rs

@@ -33,7 +33,7 @@ impl Resolve<super::Args> for Deploy {
       stop_time = self.stop_time,
     )
   )]
-  async fn resolve(self, args: &super::Args) -> serror::Result<Log> {
+  async fn resolve(self, args: &super::Args) -> anyhow::Result<Log> {
     let Deploy {
       mut deployment,
       stop_signal,

bin/periphery/src/api/docker.rs

@@ -1,5 +1,6 @@
 use std::sync::OnceLock;
 
+use anyhow::Context;
 use cache::TimeoutCache;
 use command::run_komodo_command;
 use komodo_client::entities::{
@@ -15,7 +16,7 @@ use komodo_client::entities::{
 use periphery_client::api::docker::*;
 use resolver_api::Resolve;
 
-use crate::docker::{docker_client, docker_login};
+use crate::{docker::docker_login, state::docker_client};
 
 // =====
 // IMAGE
@@ -23,8 +24,13 @@ use crate::docker::{docker_client, docker_login};
 
 impl Resolve<super::Args> for InspectImage {
   #[instrument(name = "InspectImage", level = "debug")]
-  async fn resolve(self, _: &super::Args) -> serror::Result<Image> {
-    Ok(docker_client().inspect_image(&self.name).await?)
+  async fn resolve(self, _: &super::Args) -> anyhow::Result<Image> {
+    let client = docker_client().load();
+    let client = client
+      .iter()
+      .next()
+      .context("Could not connect to docker client")?;
+    client.inspect_image(&self.name).await
   }
 }
 
@@ -35,8 +41,13 @@ impl Resolve<super::Args> for ImageHistory {
   async fn resolve(
     self,
     _: &super::Args,
-  ) -> serror::Result<Vec<ImageHistoryResponseItem>> {
-    Ok(docker_client().image_history(&self.name).await?)
+  ) -> anyhow::Result<Vec<ImageHistoryResponseItem>> {
+    let client = docker_client().load();
+    let client = client
+      .iter()
+      .next()
+      .context("Could not connect to docker client")?;
+    client.image_history(&self.name).await
   }
 }
 
@@ -53,7 +64,7 @@ fn pull_cache() -> &'static TimeoutCache<String, Log> {
 
 impl Resolve<super::Args> for PullImage {
   #[instrument(name = "PullImage", skip_all, fields(name = &self.name))]
-  async fn resolve(self, _: &super::Args) -> serror::Result<Log> {
+  async fn resolve(self, _: &super::Args) -> anyhow::Result<Log> {
     let PullImage {
       name,
       account,
@@ -69,7 +80,7 @@ impl Resolve<super::Args> for PullImage {
 
     // Early return from cache if lasted pulled with PULL_TIMEOUT
     if locked.last_ts + PULL_TIMEOUT > komodo_timestamp() {
-      return locked.clone_res().map_err(Into::into);
+      return locked.clone_res();
     }
 
     let res = async {
@@ -94,7 +105,7 @@ impl Resolve<super::Args> for PullImage {
     // then immediately also use this same result.
     locked.set(&res, komodo_timestamp());
 
-    res.map_err(Into::into)
+    res
   }
 }
 
@@ -102,7 +113,7 @@ impl Resolve<super::Args> for PullImage {
 
 impl Resolve<super::Args> for DeleteImage {
   #[instrument(name = "DeleteImage")]
-  async fn resolve(self, _: &super::Args) -> serror::Result<Log> {
+  async fn resolve(self, _: &super::Args) -> anyhow::Result<Log> {
     let command = format!("docker image rm {}", self.name);
     Ok(run_komodo_command("Delete Image", None, command).await)
   }
@@ -112,7 +123,7 @@ impl Resolve<super::Args> for DeleteImage {
 
 impl Resolve<super::Args> for PruneImages {
   #[instrument(name = "PruneImages")]
-  async fn resolve(self, _: &super::Args) -> serror::Result<Log> {
+  async fn resolve(self, _: &super::Args) -> anyhow::Result<Log> {
     let command = String::from("docker image prune -a -f");
     Ok(run_komodo_command("Prune Images", None, command).await)
   }
@@ -124,8 +135,13 @@ impl Resolve<super::Args> for PruneImages {
 
 impl Resolve<super::Args> for InspectNetwork {
   #[instrument(name = "InspectNetwork", level = "debug")]
-  async fn resolve(self, _: &super::Args) -> serror::Result<Network> {
-    Ok(docker_client().inspect_network(&self.name).await?)
+  async fn resolve(self, _: &super::Args) -> anyhow::Result<Network> {
+    let client = docker_client().load();
+    let client = client
+      .iter()
+      .next()
+      .context("Could not connect to docker client")?;
+    client.inspect_network(&self.name).await
   }
 }
 
@@ -133,7 +149,7 @@ impl Resolve<super::Args> for InspectNetwork {
 
 impl Resolve<super::Args> for CreateNetwork {
   #[instrument(name = "CreateNetwork", skip(self))]
-  async fn resolve(self, _: &super::Args) -> serror::Result<Log> {
+  async fn resolve(self, _: &super::Args) -> anyhow::Result<Log> {
     let CreateNetwork { name, driver } = self;
     let driver = match driver {
       Some(driver) => format!(" -d {driver}"),
@@ -148,7 +164,7 @@ impl Resolve<super::Args> for CreateNetwork {
 
 impl Resolve<super::Args> for DeleteNetwork {
   #[instrument(name = "DeleteNetwork", skip(self))]
-  async fn resolve(self, _: &super::Args) -> serror::Result<Log> {
+  async fn resolve(self, _: &super::Args) -> anyhow::Result<Log> {
     let command = format!("docker network rm {}", self.name);
     Ok(run_komodo_command("Delete Network", None, command).await)
   }
@@ -158,7 +174,7 @@ impl Resolve<super::Args> for DeleteNetwork {
 
 impl Resolve<super::Args> for PruneNetworks {
   #[instrument(name = "PruneNetworks", skip(self))]
-  async fn resolve(self, _: &super::Args) -> serror::Result<Log> {
+  async fn resolve(self, _: &super::Args) -> anyhow::Result<Log> {
     let command = String::from("docker network prune -f");
     Ok(run_komodo_command("Prune Networks", None, command).await)
   }
@@ -170,8 +186,13 @@ impl Resolve<super::Args> for PruneNetworks {
 
 impl Resolve<super::Args> for InspectVolume {
   #[instrument(name = "InspectVolume", level = "debug")]
-  async fn resolve(self, _: &super::Args) -> serror::Result<Volume> {
-    Ok(docker_client().inspect_volume(&self.name).await?)
+  async fn resolve(self, _: &super::Args) -> anyhow::Result<Volume> {
+    let client = docker_client().load();
+    let client = client
+      .iter()
+      .next()
+      .context("Could not connect to docker client")?;
+    client.inspect_volume(&self.name).await
   }
 }
 
@@ -179,7 +200,7 @@ impl Resolve<super::Args> for InspectVolume {
 
 impl Resolve<super::Args> for DeleteVolume {
   #[instrument(name = "DeleteVolume")]
-  async fn resolve(self, _: &super::Args) -> serror::Result<Log> {
+  async fn resolve(self, _: &super::Args) -> anyhow::Result<Log> {
     let command = format!("docker volume rm {}", self.name);
     Ok(run_komodo_command("Delete Volume", None, command).await)
   }
@@ -189,7 +210,7 @@ impl Resolve<super::Args> for DeleteVolume {
 
 impl Resolve<super::Args> for PruneVolumes {
   #[instrument(name = "PruneVolumes")]
-  async fn resolve(self, _: &super::Args) -> serror::Result<Log> {
+  async fn resolve(self, _: &super::Args) -> anyhow::Result<Log> {
     let command = String::from("docker volume prune -a -f");
     Ok(run_komodo_command("Prune Volumes", None, command).await)
   }

bin/periphery/src/api/git.rs

@@ -1,5 +1,4 @@
 use anyhow::{Context, anyhow};
-use axum::http::StatusCode;
 use formatting::format_serror;
 use komodo_client::entities::{
   DefaultRepoFolder, LatestCommit, update::Log,
@@ -10,7 +9,6 @@ use periphery_client::api::git::{
   RenameRepo,
 };
 use resolver_api::Resolve;
-use serror::AddStatusCodeError;
 use std::path::PathBuf;
 use tokio::fs;
 
@@ -23,7 +21,7 @@ impl Resolve<super::Args> for GetLatestCommit {
   async fn resolve(
     self,
     _: &super::Args,
-  ) -> serror::Result<Option<LatestCommit>> {
+  ) -> anyhow::Result<Option<LatestCommit>> {
     let repo_path = match self.path {
       Some(p) => PathBuf::from(p),
       None => periphery_config().repo_dir().join(self.name),
@@ -48,7 +46,7 @@ impl Resolve<super::Args> for CloneRepo {
   async fn resolve(
     self,
     _: &super::Args,
-  ) -> serror::Result<PeripheryRepoExecutionResponse> {
+  ) -> anyhow::Result<PeripheryRepoExecutionResponse> {
     let CloneRepo {
       args,
       git_token,
@@ -75,7 +73,6 @@ impl Resolve<super::Args> for CloneRepo {
       replacers,
     )
     .await
-    .map_err(Into::into)
   }
 }
 
@@ -93,7 +90,7 @@ impl Resolve<super::Args> for PullRepo {
   async fn resolve(
     self,
     _: &super::Args,
-  ) -> serror::Result<PeripheryRepoExecutionResponse> {
+  ) -> anyhow::Result<PeripheryRepoExecutionResponse> {
     let PullRepo {
       args,
       git_token,
@@ -119,7 +116,6 @@ impl Resolve<super::Args> for PullRepo {
       replacers,
     )
     .await
-    .map_err(Into::into)
   }
 }
 
@@ -137,7 +133,7 @@ impl Resolve<super::Args> for PullOrCloneRepo {
   async fn resolve(
     self,
     _: &super::Args,
-  ) -> serror::Result<PeripheryRepoExecutionResponse> {
+  ) -> anyhow::Result<PeripheryRepoExecutionResponse> {
     let PullOrCloneRepo {
       args,
       git_token,
@@ -165,7 +161,6 @@ impl Resolve<super::Args> for PullOrCloneRepo {
       replacers,
     )
     .await
-    .map_err(Into::into)
   }
 }
 
@@ -173,7 +168,7 @@ impl Resolve<super::Args> for PullOrCloneRepo {
 
 impl Resolve<super::Args> for RenameRepo {
   #[instrument(name = "RenameRepo")]
-  async fn resolve(self, _: &super::Args) -> serror::Result<Log> {
+  async fn resolve(self, _: &super::Args) -> anyhow::Result<Log> {
     let RenameRepo {
       curr_name,
       new_name,
@@ -194,7 +189,7 @@ impl Resolve<super::Args> for RenameRepo {
 
 impl Resolve<super::Args> for DeleteRepo {
   #[instrument(name = "DeleteRepo")]
-  async fn resolve(self, _: &super::Args) -> serror::Result<Log> {
+  async fn resolve(self, _: &super::Args) -> anyhow::Result<Log> {
     let DeleteRepo { name, is_build } = self;
     // If using custom clone path, it will be passed by core instead of name.
     // So the join will resolve to just the absolute path.
@@ -222,16 +217,13 @@ impl Resolve<super::Args> for DeleteRepo {
 
 fn default_folder(
   default_folder: DefaultRepoFolder,
-) -> serror::Result<PathBuf> {
+) -> anyhow::Result<PathBuf> {
   match default_folder {
     DefaultRepoFolder::Stacks => Ok(periphery_config().stack_dir()),
     DefaultRepoFolder::Builds => Ok(periphery_config().build_dir()),
     DefaultRepoFolder::Repos => Ok(periphery_config().repo_dir()),
-    DefaultRepoFolder::NotApplicable => {
-      Err(
-        anyhow!("The clone args should not have a default_folder of NotApplicable using this method.")
-          .status_code(StatusCode::BAD_REQUEST)
-      )
-    }
+    DefaultRepoFolder::NotApplicable => Err(anyhow!(
+      "The clone args should not have a default_folder of NotApplicable using this method."
+    )),
   }
 }

bin/periphery/src/api/keys.rs

@@ -1,16 +1,13 @@
-use std::sync::Arc;
-
 use komodo_client::entities::NoData;
-use noise::key::{
-  EncodedKeyPair, SpkiPublicKey, generate_write_keys,
-};
+use noise::key::SpkiPublicKey;
 use periphery_client::api::keys::{
   RotateCorePublicKey, RotatePrivateKey, RotatePrivateKeyResponse,
 };
 use resolver_api::Resolve;
 
-use crate::config::{
-  core_public_keys, periphery_config, periphery_private_key,
+use crate::{
+  config::periphery_config,
+  state::{core_public_keys, periphery_keys},
 };
 
 //
@@ -19,50 +16,43 @@ impl Resolve<super::Args> for RotatePrivateKey {
   async fn resolve(
     self,
     _: &super::Args,
-  ) -> serror::Result<RotatePrivateKeyResponse> {
-    let config = periphery_config();
-    let keys = match config.private_key.as_ref() {
-      Some(private_key) => match private_key.strip_prefix("file:") {
-        None => EncodedKeyPair::from_private_key(private_key)?,
-        Some(path) => generate_write_keys(path)?,
-      },
-      None => generate_write_keys(
-        config.root_directory.join("keys/periphery.key"),
-      )?,
-    };
-    // Store new private key for next auth
-    periphery_private_key()
-      .store(Arc::new(keys.private.into_inner()));
-    Ok(RotatePrivateKeyResponse {
-      public_key: keys.public.into_inner(),
-    })
+  ) -> anyhow::Result<RotatePrivateKeyResponse> {
+    let public_key = periphery_keys().rotate().await?.into_inner();
+    info!("New Public Key: {public_key}");
+    Ok(RotatePrivateKeyResponse { public_key })
   }
 }
 
 //
 
 impl Resolve<super::Args> for RotateCorePublicKey {
-  async fn resolve(self, _: &super::Args) -> serror::Result<NoData> {
+  async fn resolve(self, _: &super::Args) -> anyhow::Result<NoData> {
     let config = periphery_config();
-    let (Some(core_public_keys_spec), Some(core_public_keys)) =
-      (config.core_public_keys.as_ref(), core_public_keys())
+
+    let Some(core_public_keys_spec) =
+      config.core_public_keys.as_ref()
     else {
       return Ok(NoData {});
     };
-    let Some(core_public_key_path) = core_public_keys_spec
-      .first()
-      .and_then(|key| key.strip_prefix("file:"))
+
+    let Some(path) = core_public_keys_spec
+      .iter()
+      // Finds the first Core Public Key in spec with `file` prefix.
+      .find_map(|public_keys| public_keys.strip_prefix("file:"))
     else {
       return Ok(NoData {});
     };
+
     let public_key = SpkiPublicKey::from(self.public_key);
-    public_key.write_pem(core_public_key_path)?;
-    let mut new_core_public_keys = vec![public_key];
-    // This only replaces the first, extend the rest.
-    new_core_public_keys
-      .extend_from_slice(&core_public_keys.load().as_slice()[1..]);
-    // Store the new keys for the next auth
-    core_public_keys.store(Arc::new(new_core_public_keys));
+
+    // Check equality at path before trying to rewrite.
+    match SpkiPublicKey::from_file(path) {
+      Ok(existing) if existing == public_key => {}
+      _ => public_key.write_pem_async(path).await?,
+    }
+
+    core_public_keys().refresh();
+
     Ok(NoData {})
   }
 }

bin/periphery/src/api/mod.rs

@@ -1,5 +1,6 @@
 use command::run_komodo_command;
 use derive_variants::EnumVariants;
+use encoding::{EncodedJsonMessage, EncodedResponse};
 use futures::FutureExt;
 use komodo_client::entities::{
   config::{DockerRegistry, GitProvider},
@@ -12,14 +13,12 @@ use periphery_client::api::{
   stats::*, terminal::*, *,
 };
 use resolver_api::Resolve;
-use response::JsonBytes;
 use serde::{Deserialize, Serialize};
 
 use crate::{
   api::compose::list_compose_projects,
-  config::{periphery_config, periphery_public_key},
-  docker::docker_client,
-  stats::stats_client,
+  config::periphery_config,
+  state::{docker_client, periphery_keys, stats_client},
 };
 
 pub mod terminal;
@@ -41,8 +40,8 @@ pub struct Args {
   Serialize, Deserialize, Debug, Clone, Resolve, EnumVariants,
 )]
 #[args(Args)]
-#[response(JsonBytes)]
-#[error(serror::Error)]
+#[response(EncodedResponse<EncodedJsonMessage>)]
+#[error(anyhow::Error)]
 #[variant_derive(Debug)]
 #[serde(tag = "type", content = "params")]
 #[allow(clippy::enum_variant_names, clippy::large_enum_variant)]
@@ -144,6 +143,7 @@ pub enum PeripheryRequest {
   DeleteAllTerminals(DeleteAllTerminals),
   ConnectTerminal(ConnectTerminal),
   ConnectContainerExec(ConnectContainerExec),
+  ConnectContainerAttach(ConnectContainerAttach),
   DisconnectTerminal(DisconnectTerminal),
   ExecuteTerminal(ExecuteTerminal),
   ExecuteContainerExec(ExecuteContainerExec),
@@ -160,7 +160,7 @@ impl Resolve<Args> for GetHealth {
   async fn resolve(
     self,
     _: &Args,
-  ) -> serror::Result<GetHealthResponse> {
+  ) -> anyhow::Result<GetHealthResponse> {
     Ok(GetHealthResponse {})
   }
 }
@@ -172,7 +172,7 @@ impl Resolve<Args> for GetVersion {
   async fn resolve(
     self,
     _: &Args,
-  ) -> serror::Result<GetVersionResponse> {
+  ) -> anyhow::Result<GetVersionResponse> {
     Ok(GetVersionResponse {
       version: env!("CARGO_PKG_VERSION").to_string(),
     })
@@ -185,22 +185,25 @@ impl Resolve<Args> for PollStatus {
   async fn resolve(
     self,
     _: &Args,
-  ) -> serror::Result<PollStatusResponse> {
+  ) -> anyhow::Result<PollStatusResponse> {
     // Docker lists
     let docker_lists = async {
-      let docker = docker_client();
+      let client = docker_client().load();
+      let Some(client) = client.iter().next() else {
+        return Default::default();
+      };
       let containers =
-        docker.list_containers().await.unwrap_or_default();
+        client.list_containers().await.unwrap_or_default();
       // Todo: handle errors better
       (
         tokio::join!(
-          docker
+          client
             .list_networks(&containers)
             .map(Result::unwrap_or_default),
-          docker
+          client
             .list_images(&containers)
             .map(Result::unwrap_or_default),
-          docker
+          client
             .list_volumes(&containers)
             .map(Result::unwrap_or_default)
         ),
@@ -229,10 +232,12 @@ impl Resolve<Args> for PollStatus {
     Ok(PollStatusResponse {
       periphery_info: PeripheryInformation {
         version: env!("CARGO_PKG_VERSION").to_string(),
-        public_key: periphery_public_key().load().to_string(),
+        public_key: periphery_keys().load().public.to_string(),
         terminals_disabled: config.disable_terminals,
-        container_exec_disabled: config.disable_container_exec,
+        container_terminals_disabled: config
+          .disable_container_terminals,
         stats_polling_rate: config.stats_polling_rate,
+        docker_connected: docker_client().load().is_some(),
       },
       system_info: stats_client.info.clone(),
       system_stats,
@@ -252,7 +257,7 @@ impl Resolve<Args> for GetSystemProcesses {
   async fn resolve(
     self,
     _: &Args,
-  ) -> serror::Result<Vec<SystemProcess>> {
+  ) -> anyhow::Result<Vec<SystemProcess>> {
     Ok(stats_client().read().await.get_processes())
   }
 }
@@ -264,7 +269,7 @@ impl Resolve<Args> for ListGitProviders {
   async fn resolve(
     self,
     _: &Args,
-  ) -> serror::Result<Vec<GitProvider>> {
+  ) -> anyhow::Result<Vec<GitProvider>> {
     Ok(periphery_config().git_providers.0.clone())
   }
 }
@@ -278,7 +283,7 @@ impl Resolve<Args> for ListDockerRegistries {
   async fn resolve(
     self,
     _: &Args,
-  ) -> serror::Result<Vec<DockerRegistry>> {
+  ) -> anyhow::Result<Vec<DockerRegistry>> {
     Ok(periphery_config().docker_registries.0.clone())
   }
 }
@@ -287,7 +292,7 @@ impl Resolve<Args> for ListDockerRegistries {
 
 impl Resolve<Args> for ListSecrets {
   #[instrument(name = "ListSecrets", level = "debug", skip_all)]
-  async fn resolve(self, _: &Args) -> serror::Result<Vec<String>> {
+  async fn resolve(self, _: &Args) -> anyhow::Result<Vec<String>> {
     Ok(
       periphery_config()
         .secrets
@@ -300,7 +305,7 @@ impl Resolve<Args> for ListSecrets {
 
 impl Resolve<Args> for PruneSystem {
   #[instrument(name = "PruneSystem", skip_all)]
-  async fn resolve(self, _: &Args) -> serror::Result<Log> {
+  async fn resolve(self, _: &Args) -> anyhow::Result<Log> {
     let command = String::from("docker system prune -a -f --volumes");
     Ok(run_komodo_command("Prune System", None, command).await)
   }

bin/periphery/src/api/terminal.rs

@@ -1,23 +1,26 @@
 use std::{sync::Arc, time::Duration};
 
 use anyhow::{Context, anyhow};
-use axum::http::StatusCode;
-use bytes::Bytes;
 use futures::{Stream, StreamExt, TryStreamExt};
-use komodo_client::{
-  api::write::TerminalRecreateMode,
-  entities::{KOMODO_EXIT_CODE, NoData, server::TerminalInfo},
+use komodo_client::entities::{
+  ContainerTerminalMode, KOMODO_EXIT_CODE, NoData,
+  server::TerminalInfo,
+};
+use periphery_client::{
+  api::terminal::*, transport::EncodedTransportMessage,
 };
-use periphery_client::api::terminal::*;
 use resolver_api::Resolve;
-use serror::AddStatusCodeError;
-use tokio::sync::mpsc::Sender;
 use tokio_util::{codec::LinesCodecError, sync::CancellationToken};
-use transport::{MessageState, bytes::to_transport_bytes};
+use transport::channel::{BufferedChannel, Sender};
 use uuid::Uuid;
 
 use crate::{
-  config::periphery_config, connection::core_channels, terminal::*,
+  config::periphery_config,
+  state::{
+    TerminalChannel, core_connections, terminal_channels,
+    terminal_triggers,
+  },
+  terminal::*,
 };
 
 //
@@ -27,9 +30,9 @@ impl Resolve<super::Args> for ListTerminals {
   async fn resolve(
     self,
     _: &super::Args,
-  ) -> serror::Result<Vec<TerminalInfo>> {
+  ) -> anyhow::Result<Vec<TerminalInfo>> {
     clean_up_terminals().await;
-    Ok(list_terminals().await)
+    Ok(list_terminals(self.container.as_deref()).await)
   }
 }
 
@@ -37,17 +40,15 @@ impl Resolve<super::Args> for ListTerminals {
 
 impl Resolve<super::Args> for CreateTerminal {
   #[instrument(name = "CreateTerminal", level = "debug")]
-  async fn resolve(self, _: &super::Args) -> serror::Result<NoData> {
+  async fn resolve(self, _: &super::Args) -> anyhow::Result<NoData> {
     if periphery_config().disable_terminals {
-      return Err(
-        anyhow!("Terminals are disabled in the periphery config")
-          .status_code(StatusCode::FORBIDDEN),
-      );
+      return Err(anyhow!(
+        "Terminals are disabled in the periphery config"
+      ));
     }
-    create_terminal(self.name, self.command, self.recreate)
+    create_terminal(self.name, self.command, self.recreate, None)
       .await
       .map(|_| NoData {})
-      .map_err(Into::into)
   }
 }
 
@@ -55,7 +56,7 @@ impl Resolve<super::Args> for CreateTerminal {
 
 impl Resolve<super::Args> for DeleteTerminal {
   #[instrument(name = "DeleteTerminal", level = "debug")]
-  async fn resolve(self, _: &super::Args) -> serror::Result<NoData> {
+  async fn resolve(self, _: &super::Args) -> anyhow::Result<NoData> {
     delete_terminal(&self.terminal).await;
     Ok(NoData {})
   }
@@ -65,7 +66,7 @@ impl Resolve<super::Args> for DeleteTerminal {
 
 impl Resolve<super::Args> for DeleteAllTerminals {
   #[instrument(name = "DeleteAllTerminals", level = "debug")]
-  async fn resolve(self, _: &super::Args) -> serror::Result<NoData> {
+  async fn resolve(self, _: &super::Args) -> anyhow::Result<NoData> {
     delete_all_terminals().await;
     Ok(NoData {})
   }
@@ -75,30 +76,26 @@ impl Resolve<super::Args> for DeleteAllTerminals {
 
 impl Resolve<super::Args> for ConnectTerminal {
   #[instrument(name = "ConnectTerminal", level = "debug")]
-  async fn resolve(self, args: &super::Args) -> serror::Result<Uuid> {
+  async fn resolve(self, args: &super::Args) -> anyhow::Result<Uuid> {
     if periphery_config().disable_terminals {
-      return Err(
-        anyhow!("Terminals are disabled in the periphery config")
-          .status_code(StatusCode::FORBIDDEN),
-      );
+      return Err(anyhow!(
+        "Terminals are disabled in the periphery config"
+      ));
     }
 
-    let channel =
-      core_channels().get(&args.core).await.with_context(|| {
-        format!("Failed to find channel for {}", args.core)
-      })?;
+    let connection =
+      core_connections().get(&args.core).await.with_context(
+        || format!("Failed to find channel for {}", args.core),
+      )?;
 
     clean_up_terminals().await;
 
     let terminal = get_terminal(&self.terminal).await?;
 
-    let id = Uuid::new_v4();
+    let channel =
+      spawn_terminal_forwarding(connection, terminal).await;
 
-    tokio::spawn(async move {
-      handle_terminal_forwarding(&channel.sender, id, terminal).await
-    });
-
-    Ok(id)
+    Ok(channel)
   }
 }
 
@@ -106,46 +103,88 @@ impl Resolve<super::Args> for ConnectTerminal {
 
 impl Resolve<super::Args> for ConnectContainerExec {
   #[instrument(name = "ConnectContainerExec", level = "debug")]
-  async fn resolve(self, args: &super::Args) -> serror::Result<Uuid> {
-    if periphery_config().disable_container_exec {
-      return Err(
-        anyhow!("Container exec is disabled in the periphery config")
-          .into(),
-      );
+  async fn resolve(self, args: &super::Args) -> anyhow::Result<Uuid> {
+    if periphery_config().disable_container_terminals {
+      return Err(anyhow!(
+        "Container Terminals are disabled in the periphery config"
+      ));
     }
 
-    let channel =
-      core_channels().get(&args.core).await.with_context(|| {
-        format!("Failed to find channel for {}", args.core)
-      })?;
+    let connection =
+      core_connections().get(&args.core).await.with_context(
+        || format!("Failed to find channel for {}", args.core),
+      )?;
 
-    let ConnectContainerExec { container, shell } = self;
+    let ConnectContainerExec {
+      container,
+      shell,
+      recreate,
+    } = self;
 
     if container.contains("&&") || shell.contains("&&") {
-      return Err(
-        anyhow!(
-          "The use of '&&' is forbidden in the container name or shell"
-        )
-        .into(),
-      );
+      return Err(anyhow!(
+        "The use of '&&' is forbidden in the container name or shell"
+      ));
     }
 
     // Create (recreate if shell changed)
     let terminal = create_terminal(
       container.clone(),
       format!("docker exec -it {container} {shell}"),
-      TerminalRecreateMode::DifferentCommand,
+      recreate,
+      Some((container, ContainerTerminalMode::Exec)),
     )
     .await
     .context("Failed to create terminal for container exec")?;
 
-    let id = Uuid::new_v4();
+    let channel =
+      spawn_terminal_forwarding(connection, terminal).await;
 
-    tokio::spawn(async move {
-      handle_terminal_forwarding(&channel.sender, id, terminal).await
-    });
+    Ok(channel)
+  }
+}
 
-    Ok(id)
+//
+
+impl Resolve<super::Args> for ConnectContainerAttach {
+  #[instrument(name = "ConnectContainerAttach", level = "debug")]
+  async fn resolve(self, args: &super::Args) -> anyhow::Result<Uuid> {
+    if periphery_config().disable_container_terminals {
+      return Err(anyhow!(
+        "Container Terminals are disabled in the periphery config"
+      ));
+    }
+
+    let connection =
+      core_connections().get(&args.core).await.with_context(
+        || format!("Failed to find channel for {}", args.core),
+      )?;
+
+    let ConnectContainerAttach {
+      container,
+      recreate,
+    } = self;
+
+    if container.contains("&&") {
+      return Err(anyhow!(
+        "The use of '&&' is forbidden in the container name"
+      ));
+    }
+
+    // Create (recreate if shell changed)
+    let terminal = create_terminal(
+      container.clone(),
+      format!("docker attach {container} --sig-proxy=false"),
+      recreate,
+      Some((container, ContainerTerminalMode::Attach)),
+    )
+    .await
+    .context("Failed to create terminal for container attach")?;
+
+    let channel =
+      spawn_terminal_forwarding(connection, terminal).await;
+
+    Ok(channel)
   }
 }
 
@@ -153,11 +192,10 @@ impl Resolve<super::Args> for ConnectContainerExec {
 
 impl Resolve<super::Args> for DisconnectTerminal {
   #[instrument(name = "DisconnectTerminal", level = "debug")]
-  async fn resolve(self, _: &super::Args) -> serror::Result<NoData> {
-    if let Some((_, cancel)) =
-      terminal_channels().remove(&self.id).await
+  async fn resolve(self, _: &super::Args) -> anyhow::Result<NoData> {
+    if let Some(channel) = terminal_channels().remove(&self.id).await
     {
-      cancel.cancel();
+      channel.cancel.cancel();
     }
     Ok(NoData {})
   }
@@ -167,37 +205,39 @@ impl Resolve<super::Args> for DisconnectTerminal {
 
 impl Resolve<super::Args> for ExecuteTerminal {
   #[instrument(name = "ExecuteTerminal", level = "debug")]
-  async fn resolve(self, args: &super::Args) -> serror::Result<Uuid> {
+  async fn resolve(self, args: &super::Args) -> anyhow::Result<Uuid> {
     if periphery_config().disable_terminals {
-      return Err(
-        anyhow!("Terminals are disabled in the periphery config")
-          .status_code(StatusCode::FORBIDDEN),
-      );
+      return Err(anyhow!(
+        "Terminals are disabled in the Periphery config"
+      ));
     }
 
     let channel =
-      core_channels().get(&args.core).await.with_context(|| {
-        format!("Failed to find channel for {}", args.core)
-      })?;
+      core_connections().get(&args.core).await.with_context(
+        || format!("Failed to find channel for {}", args.core),
+      )?;
 
     let terminal = get_terminal(&self.terminal).await?;
 
-    let stdout =
-      setup_execute_command_on_terminal(&terminal, &self.command)
-        .await?;
+    let channel_id = Uuid::new_v4();
 
-    let id = Uuid::new_v4();
+    let stdout = setup_execute_command_on_terminal(
+      channel_id,
+      &terminal,
+      &self.command,
+    )
+    .await?;
 
     tokio::spawn(async move {
       forward_execute_command_on_terminal_response(
         &channel.sender,
-        id,
+        channel_id,
         stdout,
       )
       .await
     });
 
-    Ok(id)
+    Ok(channel_id)
   }
 }
 
@@ -205,34 +245,36 @@ impl Resolve<super::Args> for ExecuteTerminal {
 
 impl Resolve<super::Args> for ExecuteContainerExec {
   #[instrument(name = "ExecuteContainerExec", level = "debug")]
-  async fn resolve(self, args: &super::Args) -> serror::Result<Uuid> {
-    if periphery_config().disable_container_exec {
-      return Err(
-        anyhow!("Container exec is disabled in the periphery config")
-          .into(),
-      );
+  async fn resolve(self, args: &super::Args) -> anyhow::Result<Uuid> {
+    if periphery_config().disable_container_terminals {
+      return Err(anyhow!(
+        "Container Terminals are disabled in the Periphery config"
+      ));
     }
 
     let Self {
       container,
       shell,
       command,
+      recreate,
     } = self;
 
     if container.contains("&&") || shell.contains("&&") {
-      return Err(
-        anyhow!(
-          "The use of '&&' is forbidden in the container name or shell"
-        )
-        .into(),
-      );
+      return Err(anyhow!(
+        "The use of '&&' is forbidden in the container name or shell"
+      ));
     }
 
-    // Create terminal (recreate if shell changed)
+    let channel =
+      core_connections().get(&args.core).await.with_context(
+        || format!("Failed to find channel for {}", args.core),
+      )?;
+
     let terminal = create_terminal(
       container.clone(),
       format!("docker exec -it {container} {shell}"),
-      TerminalRecreateMode::DifferentCommand,
+      recreate,
+      Some((container, ContainerTerminalMode::Exec)),
     )
     .await
     .context("Failed to create terminal for container exec")?;
@@ -240,59 +282,84 @@ impl Resolve<super::Args> for ExecuteContainerExec {
     // Wait a bit for terminal to initialize
     tokio::time::sleep(Duration::from_millis(500)).await;
 
-    let stdout =
-      setup_execute_command_on_terminal(&terminal, &command).await?;
+    let channel_id = Uuid::new_v4();
 
-    let id = Uuid::new_v4();
-
-    let channel =
-      core_channels().get(&args.core).await.with_context(|| {
-        format!("Failed to find channel for {}", args.core)
-      })?;
+    let stdout = setup_execute_command_on_terminal(
+      channel_id, &terminal, &command,
+    )
+    .await?;
 
     tokio::spawn(async move {
       forward_execute_command_on_terminal_response(
         &channel.sender,
-        id,
+        channel_id,
         stdout,
       )
       .await
     });
 
-    Ok(id)
+    Ok(channel_id)
   }
 }
 
-async fn handle_terminal_forwarding(
-  sender: &Sender<Bytes>,
-  id: Uuid,
+async fn spawn_terminal_forwarding(
+  connection: Arc<BufferedChannel<EncodedTransportMessage>>,
   terminal: Arc<Terminal>,
-) {
+) -> Uuid {
+  let channel = Uuid::new_v4();
   let cancel = CancellationToken::new();
 
-  terminal_channels()
-    .insert(id, (terminal.stdin.clone(), cancel.clone()))
-    .await;
+  tokio::join!(
+    terminal_channels().insert(
+      channel,
+      Arc::new(TerminalChannel {
+        sender: terminal.stdin.clone(),
+        cancel: cancel.clone(),
+      }),
+    ),
+    terminal_triggers().insert(channel),
+  );
+
+  tokio::spawn(async move {
+    handle_terminal_forwarding(
+      &connection.sender,
+      channel,
+      terminal,
+      cancel,
+    )
+    .await
+  });
+
+  channel
+}
+
+async fn handle_terminal_forwarding(
+  sender: &Sender<EncodedTransportMessage>,
+  channel: Uuid,
+  terminal: Arc<Terminal>,
+  cancel: CancellationToken,
+) {
+  // This waits to begin forwarding until Core sends the None byte start trigger.
+  // This ensures no messages are lost before channels on both sides are set up.
+  if let Err(e) = terminal_triggers().wait(&channel).await {
+    warn!(
+      "Failed to init terminal | Failed to receive begin trigger | {e:#}"
+    );
+    terminal_channels().remove(&channel).await;
+    return;
+  }
 
   let init_res = async {
     let (a, b) = terminal.history.bytes_parts();
     if !a.is_empty() {
       sender
-        .send(to_transport_bytes(
-          a.into(),
-          id,
-          MessageState::Terminal,
-        ))
+        .send_terminal(channel, a)
         .await
         .context("Failed to send history part a")?;
     }
     if !b.is_empty() {
       sender
-        .send(to_transport_bytes(
-          b.into(),
-          id,
-          MessageState::Terminal,
-        ))
+        .send_terminal(channel, b)
         .await
         .context("Failed to send history part b")?;
     }
@@ -303,7 +370,7 @@ async fn handle_terminal_forwarding(
   if let Err(e) = init_res {
     // TODO: Handle error
     warn!("Failed to init terminal | {e:#}");
-    terminal_channels().remove(&id).await;
+    terminal_channels().remove(&channel).await;
     return;
   }
 
@@ -311,7 +378,7 @@ async fn handle_terminal_forwarding(
   let mut stdout = terminal.stdout.resubscribe();
   loop {
     let res = tokio::select! {
-      res = stdout.recv() => res.context("Failed to get message over stdout receiver"),
+      res = stdout.recv() => res,
       _ = terminal.cancel.cancelled() => {
         trace!("ws write: cancelled from outside");
         // let _ = ws_sender.send("PTY KILLED")).await;
@@ -328,49 +395,39 @@ async fn handle_terminal_forwarding(
         break
       }
     };
-    match res {
-      Ok(bytes) => {
-        if let Err(e) = sender
-          .send(to_transport_bytes(
-            bytes.into(),
-            id,
-            MessageState::Terminal,
-          ))
-          .await
-        {
-          debug!("Failed to send to WS: {e:?}");
-          cancel.cancel();
-          break;
-        }
-      }
+
+    let bytes = match res {
+      Ok(bytes) => bytes,
       Err(e) => {
-        debug!("PTY -> WS channel read error: {e:?}");
-        let _ = sender
-          .send(to_transport_bytes(
-            format!("ERROR: {e:#}").into(),
-            id,
-            MessageState::Terminal,
-          ))
-          .await;
+        debug!("Terminal receiver failed | {e:?}");
         terminal.cancel();
         break;
       }
+    };
+
+    if let Err(e) = sender.send_terminal(channel, bytes).await {
+      debug!("Failed to send to WS: {e:?}");
+      cancel.cancel();
+      break;
     }
   }
 
   // Clean up
-  if let Some((_, cancel)) = terminal_channels().remove(&id).await {
-    trace!("Cancel called for {id}");
-    cancel.cancel();
+  if let Some(terminal_channel) =
+    terminal_channels().remove(&channel).await
+  {
+    trace!("Cancel called for {channel}");
+    terminal_channel.cancel.cancel();
   }
   clean_up_terminals().await;
 }
 
 /// This is run before spawning task handler
 async fn setup_execute_command_on_terminal(
+  channel_id: Uuid,
   terminal: &Terminal,
   command: &str,
-) -> serror::Result<
+) -> anyhow::Result<
   impl Stream<Item = Result<String, LinesCodecError>> + 'static,
 > {
   // Read the bytes into lines
@@ -391,7 +448,7 @@ async fn setup_execute_command_on_terminal(
 
   terminal
     .stdin
-    .send(StdinMsg::Bytes(Bytes::from(full_command)))
+    .send(StdinMsg::Bytes(full_command.into()))
     .await
     .context("Failed to send command to terminal stdin")?;
 
@@ -406,47 +463,41 @@ async fn setup_execute_command_on_terminal(
       // Keep looping until the start sentinel received.
       Some(_) => {}
       None => {
-        return Err(
-          anyhow!(
-            "Stdout stream terminated before start sentinel received"
-          )
-          .into(),
-        );
+        return Err(anyhow!(
+          "Stdout stream terminated before start sentinel received"
+        ));
       }
     }
   }
 
+  terminal_triggers().insert(channel_id).await;
+
   Ok(stdout)
 }
 
 async fn forward_execute_command_on_terminal_response(
-  sender: &Sender<Bytes>,
-  id: Uuid,
+  sender: &Sender<EncodedTransportMessage>,
+  channel: Uuid,
   mut stdout: impl Stream<Item = Result<String, LinesCodecError>> + Unpin,
 ) {
+  // This waits to begin forwarding until Core sends the None byte start trigger.
+  // This ensures no messages are lost before channels on both sides are set up.
+  if let Err(e) = terminal_triggers().wait(&channel).await {
+    warn!("{e:#}");
+    return;
+  }
+
   loop {
     match stdout.next().await {
       Some(Ok(line)) if line.as_str() == END_OF_OUTPUT => {
-        if let Err(e) = sender
-          .send(to_transport_bytes(
-            line.into(),
-            id,
-            MessageState::Terminal,
-          ))
-          .await
-        {
+        if let Err(e) = sender.send_terminal(channel, line).await {
           warn!("Got ws_sender send error on END_OF_OUTPUT | {e:?}");
         }
         break;
       }
       Some(Ok(line)) => {
-        if let Err(e) = sender
-          .send(to_transport_bytes(
-            (line + "\n").into(),
-            id,
-            MessageState::Terminal,
-          ))
-          .await
+        if let Err(e) =
+          sender.send_terminal(channel, line + "\n").await
         {
           warn!("Got ws_sender send error | {e:?}");
           break;

bin/periphery/src/config.rs

@@ -1,11 +1,5 @@
-use std::{
-  fs::read_to_string,
-  path::PathBuf,
-  sync::{Arc, OnceLock},
-};
+use std::{path::PathBuf, sync::OnceLock};
 
-use anyhow::Context;
-use arc_swap::ArcSwap;
 use clap::Parser;
 use colored::Colorize;
 use config::ConfigLoader;
@@ -16,74 +10,6 @@ use komodo_client::entities::{
   config::periphery::{CliArgs, Env, PeripheryConfig},
   logger::{LogConfig, LogLevel},
 };
-use noise::key::{SpkiPublicKey, load_maybe_generate_private_key};
-
-/// Should call in startup to ensure Periphery errors without valid private key.
-pub fn periphery_private_key() -> &'static ArcSwap<String> {
-  static PERIPHERY_PRIVATE_KEY: OnceLock<ArcSwap<String>> =
-    OnceLock::new();
-  PERIPHERY_PRIVATE_KEY.get_or_init(|| {
-    let config = periphery_config();
-    let private_key = config.private_key.clone().unwrap_or(format!(
-      "file:{}/keys/periphery.key",
-      config.root_directory.display()
-    ));
-    let private_key =
-      if let Some(path) = private_key.strip_prefix("file:") {
-        load_maybe_generate_private_key(path).unwrap()
-      } else {
-        private_key
-      };
-    ArcSwap::new(Arc::new(private_key))
-  })
-}
-
-/// Should call in startup to ensure Periphery errors without valid private key.
-pub fn periphery_public_key() -> &'static ArcSwap<SpkiPublicKey> {
-  static PERIPHERY_PUBLIC_KEY: OnceLock<ArcSwap<SpkiPublicKey>> =
-    OnceLock::new();
-  PERIPHERY_PUBLIC_KEY.get_or_init(|| {
-    let public_key = SpkiPublicKey::from_private_key(
-      periphery_private_key().load().as_str(),
-    )
-    .context("Got invalid private key")
-    .unwrap();
-    ArcSwap::new(Arc::new(public_key))
-  })
-}
-
-pub fn core_public_keys()
--> Option<&'static ArcSwap<Vec<SpkiPublicKey>>> {
-  static CORE_PUBLIC_KEYS: OnceLock<
-    Option<ArcSwap<Vec<SpkiPublicKey>>>,
-  > = OnceLock::new();
-  CORE_PUBLIC_KEYS
-    .get_or_init(|| {
-      periphery_config().core_public_keys.as_ref().map(
-        |public_keys| {
-          let public_keys = public_keys
-            .iter()
-            .map(|public_key| {
-              let maybe_pem = if let Some(path) =
-                public_key.strip_prefix("file:")
-              {
-                read_to_string(path)
-                  .with_context(|| {
-                    format!("Failed to read public key at {path:?}")
-                  })
-                  .unwrap()
-              } else {
-                public_key.clone()
-              };
-              SpkiPublicKey::from_maybe_pem(&maybe_pem).unwrap()
-            })
-            .collect::<Vec<_>>();
-          ArcSwap::new(Arc::new(public_keys))
-        },
-      )
-    })
-    .as_ref()
-}
 
 pub fn periphery_args() -> &'static CliArgs {
   static PERIPHERY_ARGS: OnceLock<CliArgs> = OnceLock::new();
@@ -164,11 +90,13 @@ pub fn periphery_config() -> &'static PeripheryConfig {
       .or(config.passkeys),
       core_addresses: env
         .periphery_core_addresses
-        .or(config.core_addresses),
+        .unwrap_or(config.core_addresses),
       core_tls_insecure_skip_verify: env
         .periphery_core_tls_insecure_skip_verify
         .unwrap_or(config.core_tls_insecure_skip_verify),
-      connect_as: env.periphery_connect_as.or(config.connect_as),
+      connect_as: env
+        .periphery_connect_as
+        .unwrap_or(config.connect_as),
       server_enabled: env
         .periphery_server_enabled
         .unwrap_or(config.server_enabled),
@@ -183,9 +111,9 @@ pub fn periphery_config() -> &'static PeripheryConfig {
       disable_terminals: env
         .periphery_disable_terminals
         .unwrap_or(config.disable_terminals),
-      disable_container_exec: env
-        .periphery_disable_container_exec
-        .unwrap_or(config.disable_container_exec),
+      disable_container_terminals: env
+        .periphery_disable_container_terminals
+        .unwrap_or(config.disable_container_terminals),
       stats_polling_rate: env
         .periphery_stats_polling_rate
         .unwrap_or(config.stats_polling_rate),

bin/periphery/src/connection/client.rs

@@ -2,32 +2,35 @@ use std::{sync::Arc, time::Duration};
 
 use anyhow::{Context, anyhow};
 use axum::http::{HeaderValue, StatusCode};
-use bytes::Bytes;
-use periphery_client::CONNECTION_RETRY_SECONDS;
-use serror::deserialize_error_bytes;
+use periphery_client::{
+  CONNECTION_RETRY_SECONDS, transport::LoginMessage,
+};
 use transport::{
-  MessageState,
   auth::{
     AddressConnectionIdentifiers, ClientLoginFlow,
     ConnectionIdentifiers, LoginFlow, LoginFlowArgs,
   },
   fix_ws_address,
-  websocket::{Websocket, tungstenite::TungsteniteWebsocket},
+  websocket::{
+    WebsocketExt, login::LoginWebsocketExt,
+    tungstenite::TungsteniteWebsocket,
+  },
 };
 
 use crate::{
   api::Args,
-  config::{periphery_config, periphery_public_key},
-  connection::{CorePublicKeyValidator, core_channels},
+  config::periphery_config,
+  connection::core_public_keys,
+  state::{core_connections, periphery_keys},
 };
 
-pub async fn handler(
-  address: &str,
-  connect_as: &str,
-) -> anyhow::Result<()> {
+pub async fn handler(address: &str) -> anyhow::Result<()> {
   let address = fix_ws_address(address);
   let identifiers = AddressConnectionIdentifiers::extract(&address)?;
-  let query = format!("server={}", urlencoding::encode(connect_as));
+  let query = format!(
+    "server={}",
+    urlencoding::encode(&periphery_config().connect_as)
+  );
   let endpoint = format!("{address}/ws/periphery?{query}");
 
   info!("Initiating outbound connection to {endpoint}");
@@ -41,13 +44,13 @@ pub async fn handler(
   });
 
   let channel =
-    core_channels().get_or_insert_default(&args.core).await;
+    core_connections().get_or_insert_default(&args.core).await;
 
   let mut receiver = channel.receiver()?;
 
   loop {
     let (mut socket, accept) =
-      match connect_websocket(&endpoint, connect_as).await {
+      match connect_websocket(&endpoint).await {
         Ok(res) => res,
         Err(e) => {
           if !already_logged_connection_error {
@@ -68,13 +71,12 @@ pub async fn handler(
 
     // Receive whether to use Server connection flow vs Server onboarding flow.
 
-    let flow_bytes = match socket
-      .recv_bytes()
-      .with_timeout(Duration::from_secs(2))
-      .await?
-      .context("Failed to receive login flow indicator")
+    let onboarding_flow = match socket
+      .recv_login_onboarding_flow()
+      .await
+      .context("Failed to receive Login OnboardingFlow message")
     {
-      Ok(flow_bytes) => flow_bytes,
+      Ok(onboarding_flow) => onboarding_flow,
       Err(e) => {
         if !already_logged_connection_error {
           warn!("{e:#}");
@@ -97,128 +99,98 @@ pub async fn handler(
     let identifiers =
       identifiers.build(accept.as_bytes(), query.as_bytes());
 
-    match flow_bytes.iter().as_slice() {
-      // Connection (standard) flow
-      &[0] => {
-        if let Err(e) = super::handle_login::<_, ClientLoginFlow>(
-          &mut socket,
-          identifiers,
-        )
-        .await
-        {
-          if !already_logged_login_error {
-            warn!("Failed to login | {e:#}");
-            already_logged_login_error = true;
-          }
-          tokio::time::sleep(Duration::from_secs(
-            CONNECTION_RETRY_SECONDS,
-          ))
-          .await;
-          continue;
-        }
-
-        already_logged_login_error = false;
-
-        super::handle_socket(
-          socket,
-          &args,
-          &channel.sender,
-          &mut receiver,
-        )
-        .await
-      }
-      // Creation
-      &[1] => {
-        if let Err(e) =
-          handle_onboarding(connect_as, socket, identifiers).await
-        {
-          if !already_logged_onboarding_error {
-            error!("{e:#}");
-            already_logged_onboarding_error = true;
-          }
-          tokio::time::sleep(Duration::from_secs(
-            CONNECTION_RETRY_SECONDS,
-          ))
-          .await;
-          continue;
-        };
-      }
-      // Other (error)
-      other => {
-        if !already_logged_connection_error {
-          warn!("Receieved invalid login flow pattern: {other:?}");
-          already_logged_connection_error = true;
-          // If error transitions from login to connection,
-          // set to false to see login error after reconnect.
-          already_logged_login_error = false;
+    if onboarding_flow {
+      if let Err(e) = handle_onboarding(socket, identifiers).await {
+        if !already_logged_onboarding_error {
+          error!("{e:#}");
+          already_logged_onboarding_error = true;
         }
         tokio::time::sleep(Duration::from_secs(
           CONNECTION_RETRY_SECONDS,
         ))
         .await;
+        continue;
+      };
+    } else {
+      if let Err(e) = super::handle_login::<_, ClientLoginFlow>(
+        &mut socket,
+        identifiers,
+      )
+      .await
+      {
+        if !already_logged_login_error {
+          warn!("Failed to login | {e:#}");
+          already_logged_login_error = true;
+        }
+        tokio::time::sleep(Duration::from_secs(
+          CONNECTION_RETRY_SECONDS,
+        ))
+        .await;
+        continue;
       }
-    };
+
+      already_logged_login_error = false;
+
+      super::handle_socket(
+        socket,
+        &args,
+        &channel.sender,
+        &mut receiver,
+      )
+      .await
+    }
   }
 }
 
 async fn handle_onboarding(
-  connect_as: &str,
   mut socket: TungsteniteWebsocket,
   identifiers: ConnectionIdentifiers<'_>,
 ) -> anyhow::Result<()> {
-  let onboarding_key = periphery_config()
+  let config = periphery_config();
+  let onboarding_key = config
     .onboarding_key
     .as_deref()
-    .with_context(|| format!("Server {connect_as} does not exist, and no PERIPHERY_ONBOARDING_KEY is provided."))?;
+    .with_context(|| format!("Server {} does not exist, and no PERIPHERY_ONBOARDING_KEY is provided.", config.connect_as))?;
 
   ClientLoginFlow::login(LoginFlowArgs {
     private_key: onboarding_key,
     identifiers,
-    public_key_validator: CorePublicKeyValidator,
+    public_key_validator: core_public_keys(),
     socket: &mut socket,
   })
   .await?;
 
   // Post onboarding login 1: Send public key
   socket
-    .send(Bytes::copy_from_slice(
-      periphery_public_key().load().as_bytes(),
+    .send_message(LoginMessage::PublicKey(
+      periphery_keys().load().public.clone(),
     ))
     .await
     .context("Failed to send public key bytes")?;
 
-  let res = socket
-    .recv_bytes()
-    .with_timeout(Duration::from_secs(2))
-    .await?
+  socket
+    .recv_login_success()
+    .await
     .context("Failed to receive Server creation result")?;
 
-  match res.last().map(|byte| MessageState::from_byte(*byte)) {
-    Some(MessageState::Successful) => {
-      info!(
-        "Server onboarding flow for '{connect_as}' successful ✅"
-      );
-      Ok(())
-    }
-    Some(MessageState::Failed) => {
-      Err(deserialize_error_bytes(&res[..(res.len() - 1)]))
-    }
-    other => Err(anyhow!(
-      "Got unrecognized onboarding flow response: {other:?}"
-    )),
-  }
+  info!(
+    "Server onboarding flow for '{}' successful ✅",
+    config.connect_as
+  );
+
+  Ok(())
 }
 
 async fn connect_websocket(
   url: &str,
-  connect_as: &str,
 ) -> anyhow::Result<(TungsteniteWebsocket, HeaderValue)> {
-  TungsteniteWebsocket::connect_maybe_tls_insecure(url, periphery_config().core_tls_insecure_skip_verify)
+  let config = periphery_config();
+  TungsteniteWebsocket::connect_maybe_tls_insecure(url, config.core_tls_insecure_skip_verify)
     .await
     .map_err(|e| match e.status {
-      StatusCode::NOT_FOUND => anyhow!("404 Not Found: Server '{connect_as}' does not exist."),
-      StatusCode::BAD_REQUEST => anyhow!("400 Bad Request: Server '{connect_as}' is disabled or configured to make Core → Periphery connection"),
-      StatusCode::UNAUTHORIZED => anyhow!("401 Unauthorized: Only one Server connected as '{connect_as}' is allowed. Or the Core reverse proxy needs to forward host and websocket headers."),
+      StatusCode::NOT_FOUND => anyhow!("404 Not Found: Server '{}' does not exist.", config.connect_as),
+      StatusCode::BAD_REQUEST => anyhow!("400 Bad Request: Server '{}' is disabled or configured to make Core → Periphery connection", config.connect_as),
+      StatusCode::UNAUTHORIZED => anyhow!("401 Unauthorized: Only one Server connected as '{}' is allowed. Or the Core reverse proxy needs to forward host and websocket headers.", config.connect_as),
       _ => e.error,
     })
 }

bin/periphery/src/connection/mod.rs

@@ -1,70 +1,45 @@
-use std::{
-  sync::{Arc, OnceLock},
-  time::Duration,
-};
+use std::{sync::Arc, time::Duration};
 
 use anyhow::anyhow;
-use bytes::Bytes;
-use cache::CloneCache;
+use encoding::{
+  CastBytes as _, Decode as _, Encode as _, WithChannel,
+};
+use periphery_client::transport::{
+  EncodedRequestMessage, EncodedTransportMessage, RequestMessage,
+  TransportMessage,
+};
 use resolver_api::Resolve;
-use response::JsonBytes;
-use serror::serialize_error_bytes;
-use tokio::sync::mpsc::Sender;
 use transport::{
-  MessageState,
   auth::{
     ConnectionIdentifiers, LoginFlow, LoginFlowArgs,
     PublicKeyValidator,
   },
-  bytes::{
-    data_from_transport_bytes, id_state_from_transport_bytes,
-    to_transport_bytes,
-  },
-  channel::{BufferedChannel, BufferedReceiver},
+  channel::{BufferedReceiver, Sender},
   websocket::{
-    Websocket, WebsocketMessage, WebsocketReceiver,
-    WebsocketSender as _,
+    Websocket, WebsocketReceiverExt as _, WebsocketSender as _,
   },
 };
-use uuid::Uuid;
 
 use crate::{
   api::{Args, PeripheryRequest},
-  config::{
-    core_public_keys, periphery_config, periphery_private_key,
-  },
+  config::periphery_config,
+  state::{CorePublicKeys, core_public_keys, periphery_keys},
 };
 
 pub mod client;
 pub mod server;
 
-// Core Address / Host -> Channel
-pub type CoreChannels =
-  CloneCache<String, Arc<BufferedChannel<Bytes>>>;
-
-pub fn core_channels() -> &'static CoreChannels {
-  static CORE_CHANNELS: OnceLock<CoreChannels> = OnceLock::new();
-  CORE_CHANNELS.get_or_init(Default::default)
-}
-
-pub struct CorePublicKeyValidator;
-
-impl PublicKeyValidator for CorePublicKeyValidator {
+impl PublicKeyValidator for &CorePublicKeys {
   type ValidationResult = ();
   async fn validate(&self, public_key: String) -> anyhow::Result<()> {
-    if let Some(public_keys) = core_public_keys()
-      && public_keys
-        .load()
-        .iter()
-        .all(|expected| public_key != expected.as_str())
-    {
+    if self.is_valid(&public_key).await {
+      Ok(())
+    } else {
       Err(
         anyhow!("{public_key} is invalid")
           .context("Ensure public key matches one of the 'core_public_keys' in periphery config (PERIPHERY_CORE_PUBLIC_KEYS)")
           .context("Periphery failed to validate Core public key"),
       )
-    } else {
-      Ok(())
     }
   }
 }
@@ -76,8 +51,8 @@ async fn handle_login<W: Websocket, L: LoginFlow>(
   L::login(LoginFlowArgs {
     socket,
     identifiers,
-    private_key: periphery_private_key().load().as_str(),
-    public_key_validator: CorePublicKeyValidator,
+    private_key: periphery_keys().load().private.as_str(),
+    public_key_validator: core_public_keys(),
   })
   .await
 }
@@ -85,17 +60,17 @@ async fn handle_login<W: Websocket, L: LoginFlow>(
 async fn handle_socket<W: Websocket>(
   socket: W,
   args: &Arc<Args>,
-  sender: &Sender<Bytes>,
-  receiver: &mut BufferedReceiver<Bytes>,
+  sender: &Sender<EncodedTransportMessage>,
+  receiver: &mut BufferedReceiver<EncodedTransportMessage>,
 ) {
   let config = periphery_config();
   info!(
     "Logged in to Komodo Core {} websocket{}",
     args.core,
-    if config.core_addresses.is_some()
-      && let Some(connect_as) = &config.connect_as
+    if !config.core_addresses.is_empty()
+      && !config.connect_as.is_empty()
     {
-      format!(" as Server {connect_as}")
+      format!(" as Server {}", config.connect_as)
     } else {
       String::new()
     }
@@ -105,19 +80,19 @@ async fn handle_socket<W: Websocket>(
 
   let forward_writes = async {
     loop {
-      let msg = match receiver.recv().await {
-        // Sender Dropped (shouldn't happen, it is static).
-        None => break,
-        // This has to copy the bytes to follow ownership rules.
-        Some(msg) => Bytes::copy_from_slice(msg),
+      let message = match receiver.recv().await {
+        Ok(message) => message,
+        Err(e) => {
+          warn!("{e:#}");
+          break;
+        }
       };
-      match ws_write.send(msg).await {
+      match ws_write.send(message.into_bytes()).await {
         // Clears the stored message from receiver buffer.
-        // TODO: Move after response ack.
         Ok(_) => receiver.clear_buffer(),
         Err(e) => {
           warn!("Failed to send response | {e:?}");
-          let _ = ws_write.close(None).await;
+          let _ = ws_write.close().await;
           break;
         }
       }
@@ -126,23 +101,23 @@ async fn handle_socket<W: Websocket>(
 
   let handle_reads = async {
     loop {
-      match ws_read.recv().await {
-        Ok(WebsocketMessage::Binary(bytes)) => {
-          handle_incoming_bytes(args, sender, bytes).await
-        }
-        Ok(WebsocketMessage::Close(frame)) => {
-          warn!("Connection closed with frame: {frame:?}");
-          break;
-        }
-        Ok(WebsocketMessage::Closed) => {
-          warn!("Connection already closed");
-          break;
-        }
+      let message = match ws_read.recv_message().await {
+        Ok(res) => res,
         Err(e) => {
-          warn!("Failed to read websocket message | {e:?}");
+          warn!("{e:#}");
           break;
         }
       };
+      match message {
+        TransportMessage::Request(message) => {
+          handle_request(args.clone(), sender.clone(), message)
+        }
+        TransportMessage::Terminal(message) => {
+          crate::terminal::handle_message(message).await
+        }
+        // Rest shouldn't be received by Periphery
+        _ => {}
+      }
     }
   };
 
@@ -152,73 +127,31 @@ async fn handle_socket<W: Websocket>(
   }
 }
 
-async fn handle_incoming_bytes(
-  args: &Arc<Args>,
-  sender: &Sender<Bytes>,
-  bytes: Bytes,
-) {
-  let (id, state) = match id_state_from_transport_bytes(&bytes) {
-    Ok(res) => res,
-    Err(e) => {
-      warn!("Failed to parse transport bytes | {e:#}");
-      return;
-    }
-  };
-  match state {
-    MessageState::Request => {
-      handle_request(args.clone(), sender.clone(), id, bytes)
-    }
-    MessageState::Terminal => {
-      crate::terminal::handle_incoming_message(id, bytes).await
-    }
-    // Shouldn't be received by Periphery
-    MessageState::InProgress => {}
-    MessageState::Successful => {}
-    MessageState::Failed => {}
-  }
-}
-
 fn handle_request(
   args: Arc<Args>,
-  sender: Sender<Bytes>,
-  req_id: Uuid,
-  bytes: Bytes,
+  sender: Sender<EncodedTransportMessage>,
+  message: EncodedRequestMessage,
 ) {
   tokio::spawn(async move {
-    let request = match data_from_transport_bytes(bytes) {
-      Ok(req) if !req.is_empty() => req,
-      _ => {
-        return;
-      }
-    };
-
-    let request =
-      match serde_json::from_slice::<PeripheryRequest>(&request) {
-        Ok(req) => req,
+    let WithChannel {
+      channel,
+      data: request,
+    }: WithChannel<PeripheryRequest> =
+      match message.decode().and_then(RequestMessage::map_decode) {
+        Ok(res) => res,
         Err(e) => {
           // TODO: handle:
-          warn!("Failed to parse transport bytes | {e:#}");
+          warn!("Failed to parse Request bytes | {e:#}");
           return;
         }
       };
 
     let resolve_response = async {
-      let (state, data) = match request.resolve(&args).await {
-        Ok(JsonBytes::Ok(res)) => (MessageState::Successful, res),
-        Ok(JsonBytes::Err(e)) => (
-          MessageState::Failed,
-          serialize_error_bytes(
-            &anyhow::Error::new(e)
-              .context("Failed to serialize response body"),
-          ),
-        ),
-        Err(e) => {
-          (MessageState::Failed, serialize_error_bytes(&e.error))
-        }
+      let response = match request.resolve(&args).await {
+        Ok(res) => res,
+        Err(e) => (&e).encode(),
       };
-      if let Err(e) =
-        sender.send(to_transport_bytes(data, req_id, state)).await
-      {
+      if let Err(e) = sender.send_response(channel, response).await {
         error!("Failed to send response over channel | {e:?}");
       }
     };
@@ -226,14 +159,7 @@ fn handle_request(
     let ping_in_progress = async {
       loop {
         tokio::time::sleep(Duration::from_secs(5)).await;
-        if let Err(e) = sender
-          .send(to_transport_bytes(
-            Vec::new(),
-            req_id,
-            MessageState::InProgress,
-          ))
-          .await
-        {
+        if let Err(e) = sender.send_in_progress(channel).await {
           error!("Failed to ping in progress over channel | {e:?}");
         }
       }

bin/periphery/src/connection/server.rs

@@ -18,24 +18,23 @@ use axum::{
   routing::get,
 };
 use axum_server::tls_rustls::RustlsConfig;
-use bytes::Bytes;
-use serror::{
-  AddStatusCode, AddStatusCodeError, deserialize_error_bytes,
-  serialize_error_bytes,
+use periphery_client::{
+  api::CoreConnectionQuery, transport::LoginMessage,
 };
+use serror::{AddStatusCode, AddStatusCodeError};
 use transport::{
-  CoreConnectionQuery, MessageState,
   auth::{
     ConnectionIdentifiers, HeaderConnectionIdentifiers,
     ServerLoginFlow,
   },
-  websocket::{Websocket, axum::AxumWebsocket},
+  websocket::{
+    Websocket, WebsocketExt, axum::AxumWebsocket,
+    login::LoginWebsocketExt,
+  },
 };
 
 use crate::{
-  api::Args,
-  config::{core_public_keys, periphery_config},
-  connection::core_channels,
+  api::Args, config::periphery_config, state::core_connections,
 };
 
 pub async fn run() -> anyhow::Result<()> {
@@ -47,6 +46,7 @@ pub async fn run() -> anyhow::Result<()> {
     .context("failed to parse listen address")?;
 
   let app = Router::new()
+    .route("/version", get(|| async { env!("CARGO_PKG_VERSION") }))
     .route("/", get(crate::connection::server::handler))
     .layer(middleware::from_fn(guard_request_by_ip))
     .into_make_service_with_connect_info::<SocketAddr>();
@@ -90,7 +90,7 @@ async fn handler(
   let args = Arc::new(Args { core });
 
   let channel =
-    core_channels().get_or_insert_default(&args.core).await;
+    core_connections().get_or_insert_default(&args.core).await;
 
   // Ensure the receiver is free before upgrading connection.
   // Due to ownership, it needs to be re-locked inside the ws handler,
@@ -107,6 +107,28 @@ async fn handler(
   Ok(ws.on_upgrade(|socket| async move {
     let mut socket = AxumWebsocket(socket);
 
+    // Make sure receiver locked over the login.
+    let mut receiver = match channel.receiver() {
+      Ok(receiver) => receiver,
+      Err(e) => {
+        warn!("Failed to forward connection | {e:#}");
+
+        if let Err(e) = socket
+          .send_login_error(&e)
+          .await
+          .context("Failed to send forward failed to client")
+        {
+          // Log additional error
+          warn!("{e:#}");
+        }
+
+        // Close socket
+        let _ = socket.close().await;
+
+        return;
+      }
+    };
+
     let query = format!("core={}", urlencoding::encode(&args.core));
 
     if let Err(e) =
@@ -125,30 +147,6 @@ async fn handler(
     already_logged_login_error()
       .store(false, atomic::Ordering::Relaxed);
 
-    let mut receiver = match channel.receiver() {
-      Ok(receiver) => receiver,
-      Err(e) => {
-        warn!("Failed to forward connection | {e:#}");
-
-        let mut bytes = serialize_error_bytes(&e);
-        bytes.push(MessageState::Failed.as_byte());
-
-        if let Err(e) = socket
-          .send(bytes.into())
-          .await
-          .context("Failed to send forward failed to client")
-        {
-          // Log additional error
-          warn!("{e:#}");
-        }
-
-        // Close socket
-        let _ = socket.close(None).await;
-
-        return;
-      }
-    };
-
     super::handle_socket(
       socket,
       &args,
@@ -165,13 +163,13 @@ async fn handle_login(
   socket: &mut AxumWebsocket,
   identifiers: ConnectionIdentifiers<'_>,
 ) -> anyhow::Result<()> {
-  match (core_public_keys(), &periphery_config().passkeys) {
+  let config = periphery_config();
+  match (&config.core_public_keys, &config.passkeys) {
     (Some(_), _) | (_, None) => {
-      // Send login type [0] (Noise auth)
       socket
-        .send(Bytes::from_owner([0]))
+        .send_message(LoginMessage::V1PasskeyFlow(false))
         .await
-        .context("Failed to send login type indicator")?;
+        .context("Failed to send Login V1PasskeyFlow message")?;
       super::handle_login::<_, ServerLoginFlow>(socket, identifiers)
         .await
     }
@@ -185,55 +183,43 @@ async fn handle_passkey_login(
   socket: &mut AxumWebsocket,
   passkeys: &[String],
 ) -> anyhow::Result<()> {
-  warn!(
-    "Authenticating using Passkeys. Set 'core_public_key' (PERIPHERY_CORE_PUBLIC_KEY) instead to enhance security."
-  );
+  if !already_logged_login_error().load(atomic::Ordering::Relaxed) {
+    warn!(
+      "Authenticating using Passkeys. Set 'core_public_key' (PERIPHERY_CORE_PUBLIC_KEY) instead to enhance security."
+    );
+  };
   let res = async {
-    // Send login type
     socket
-      // Passkey auth: [1]
-      .send(Bytes::from_owner([1]))
+      .send_message(LoginMessage::V1PasskeyFlow(true))
       .await
       .context("Failed to send login type indicator")?;
 
     // Receieve passkey
-    let bytes = socket
-      .recv_bytes()
+    let passkey = socket
+      .recv_login_v1_passkey()
       .await
-      .context("Failed to receive passkey from Core")?;
-    let passkey = match MessageState::from_byte(
-      *bytes.last().context("passkey message is empty")?,
-    ) {
-      MessageState::Successful => &bytes[..(bytes.len() - 1)],
-      _ => {
-        return Err(deserialize_error_bytes(
-          &bytes[..(bytes.len() - 1)],
-        ));
-      }
-    };
+      .context("Failed to receive Login V1Passkey from Core")?;
 
     if passkeys
       .iter()
       .any(|expected_passkey| expected_passkey.as_bytes() == passkey)
     {
       socket
-        .send(MessageState::Successful.into())
+        .send_message(LoginMessage::Success)
         .await
         .context("Failed to send login type indicator")?;
       Ok(())
     } else {
       let e = anyhow!("Invalid passkey");
-      let mut bytes = serialize_error_bytes(&e);
-      bytes.push(MessageState::Failed.as_byte());
       if let Err(e) = socket
-        .send(bytes.into())
+        .send_login_error(&e)
         .await
         .context("Failed to send login failed")
       {
         // Log additional error
         warn!("{e:#}");
         // Close socket
-        let _ = socket.close(None).await;
+        let _ = socket.close().await;
       }
       // Return the original error
       Err(e)
@@ -241,10 +227,8 @@ async fn handle_passkey_login(
   }
   .await;
   if let Err(e) = res {
-    let mut bytes = serialize_error_bytes(&e);
-    bytes.push(MessageState::Failed.as_byte());
     if let Err(e) = socket
-      .send(bytes.into())
+      .send_login_error(&e)
       .await
       .context("Failed to send login failed to client")
     {
@@ -252,7 +236,7 @@ async fn handle_passkey_login(
       warn!("{e:#}");
     }
     // Close socket
-    let _ = socket.close(None).await;
+    let _ = socket.close().await;
     // Return the original error
     Err(e)
   } else {

bin/periphery/src/docker/containers.rs

@@ -9,7 +9,9 @@ use komodo_client::entities::docker::{
   container::*,
 };
 
-use super::{DockerClient, stats::container_stats};
+use crate::state::container_stats;
+
+use super::DockerClient;
 
 impl DockerClient {
   pub async fn list_containers(

bin/periphery/src/docker/mod.rs

@@ -1,6 +1,4 @@
-use std::sync::OnceLock;
-
-use anyhow::anyhow;
+use anyhow::{Context, anyhow};
 use bollard::Docker;
 use command::run_komodo_command;
 use komodo_client::entities::{TerminationSignal, update::Log};
@@ -13,21 +11,15 @@ mod images;
 mod networks;
 mod volumes;
 
-pub fn docker_client() -> &'static DockerClient {
-  static DOCKER_CLIENT: OnceLock<DockerClient> = OnceLock::new();
-  DOCKER_CLIENT.get_or_init(Default::default)
-}
-
 pub struct DockerClient {
   docker: Docker,
 }
 
-impl Default for DockerClient {
-  fn default() -> DockerClient {
-    DockerClient {
-      docker: Docker::connect_with_defaults()
-        .expect("failed to connect to docker daemon"),
-    }
+impl DockerClient {
+  pub fn connect() -> anyhow::Result<DockerClient> {
+    let docker = Docker::connect_with_defaults()
+      .context("Failed to connect to docker api. Docker monitoring won't work and will return empty results.")?;
+    Ok(DockerClient { docker })
   }
 }
 

bin/periphery/src/docker/stats.rs

@@ -1,10 +1,6 @@
-use std::{
-  collections::HashMap,
-  sync::{Arc, OnceLock},
-};
+use std::{collections::HashMap, sync::Arc};
 
 use anyhow::{Context, anyhow};
-use arc_swap::ArcSwap;
 use async_timing_util::wait_until_timelength;
 use bollard::{models, query_parameters::StatsOptionsBuilder};
 use futures::StreamExt;
@@ -19,15 +15,10 @@ use komodo_client::entities::docker::{
 };
 use run_command::async_run_command;
 
-use crate::{config::periphery_config, docker::DockerClient};
-
-pub type ContainerStatsMap = HashMap<String, ContainerStats>;
-
-pub fn container_stats() -> &'static ArcSwap<ContainerStatsMap> {
-  static CONTAINER_STATS: OnceLock<ArcSwap<ContainerStatsMap>> =
-    OnceLock::new();
-  CONTAINER_STATS.get_or_init(Default::default)
-}
+use crate::{
+  config::periphery_config, docker::DockerClient,
+  state::container_stats,
+};
 
 pub fn spawn_polling_thread() {
   tokio::spawn(async move {

bin/periphery/src/main.rs

@@ -1,7 +1,10 @@
 use futures::{StreamExt, stream::FuturesUnordered};
 use komodo_client::entities::config::periphery::Command;
 
-use crate::config::{periphery_args, periphery_public_key};
+use crate::{
+  config::periphery_args,
+  state::{core_public_keys, periphery_keys},
+};
 
 #[macro_use]
 extern crate tracing;
@@ -11,6 +14,7 @@ mod config;
 mod connection;
 mod docker;
 mod helpers;
+mod state;
 mod stats;
 mod terminal;
 
@@ -27,7 +31,10 @@ async fn app() -> anyhow::Result<()> {
   }
 
   // Init + log public key. Will crash if invalid private key here.
-  info!("Public Key: {}", periphery_public_key().load());
+  info!("Public Key: {}", periphery_keys().load().public);
+
+  // Init core public keys. Will crash if invalid core public keys here.
+  core_public_keys();
 
   rustls::crypto::aws_lc_rs::default_provider()
     .install_default()
@@ -39,20 +46,16 @@ async fn app() -> anyhow::Result<()> {
   let mut handles = FuturesUnordered::new();
 
   // Spawn client side connections
-  match (&config.core_addresses, &config.connect_as) {
-    (Some(addresses), Some(connect_as)) => {
-      for address in addresses {
-        handles.push(tokio::spawn(connection::client::handler(
-          address, connect_as,
-        )));
-      }
+  if !config.core_addresses.is_empty() && config.connect_as.is_empty()
+  {
+    warn!(
+      "'core_addresses' are defined for outbound connection, but missing 'connect_as' (PERIPHERY_CONNECT_AS)."
+    );
+  } else {
+    for address in &config.core_addresses {
+      handles
+        .push(tokio::spawn(connection::client::handler(address)));
     }
-    (Some(_), None) => {
-      warn!(
-        "'core_addresses' are defined for outbound connection, but missing 'connect_as' (PERIPHERY_CONNECT_AS)."
-      );
-    }
-    _ => {}
   }
 
   // Spawn server connection handler

bin/periphery/src/state.rs

@@ -0,0 +1,292 @@
+use std::{
+  collections::HashMap,
+  path::PathBuf,
+  str::FromStr,
+  sync::{Arc, OnceLock},
+};
+
+use anyhow::{Context, anyhow};
+use arc_swap::ArcSwap;
+use cache::CloneCache;
+use komodo_client::entities::docker::container::ContainerStats;
+use noise::key::{RotatableKeyPair, SpkiPublicKey};
+use periphery_client::transport::EncodedTransportMessage;
+use tokio::sync::{Mutex, RwLock, mpsc, oneshot};
+use tokio_util::sync::CancellationToken;
+use transport::channel::BufferedChannel;
+use uuid::Uuid;
+
+use crate::{
+  config::periphery_config,
+  docker::DockerClient,
+  stats::StatsClient,
+  terminal::{StdinMsg, Terminal},
+};
+
+/// Should call in startup to ensure Periphery errors without valid private key.
+pub fn periphery_keys() -> &'static RotatableKeyPair {
+  static PERIPHERY_KEYS: OnceLock<RotatableKeyPair> = OnceLock::new();
+  PERIPHERY_KEYS.get_or_init(|| {
+    let config = periphery_config();
+    if let Some(private_key_spec) = config.private_key.as_deref() {
+      RotatableKeyPair::from_private_key_spec(private_key_spec)
+    } else {
+      RotatableKeyPair::from_private_key_spec(&format!(
+        "file:{}",
+        config
+          .root_directory
+          .join("keys/periphery.key")
+          .to_str()
+          .expect("Invalid root directory")
+      ))
+    }
+    .unwrap()
+  })
+}
+
+pub fn core_public_keys() -> &'static CorePublicKeys {
+  static CORE_PUBLIC_KEYS: OnceLock<CorePublicKeys> = OnceLock::new();
+  CORE_PUBLIC_KEYS.get_or_init(CorePublicKeys::default)
+}
+
+pub struct CorePublicKeys {
+  keys: ArcSwap<Vec<SpkiPublicKey>>,
+  /// If any keys fail to write, store them here.
+  /// For Periphery -> Core connection, Periphery will
+  /// write the Core pub keys to these files as they connect.
+  to_write: ArcSwap<Vec<PathBuf>>,
+}
+
+impl Default for CorePublicKeys {
+  fn default() -> Self {
+    let keys = CorePublicKeys {
+      keys: Default::default(),
+      to_write: Default::default(),
+    };
+    keys.refresh();
+    keys
+  }
+}
+
+impl CorePublicKeys {
+  pub fn load(&self) -> arc_swap::Guard<Arc<Vec<SpkiPublicKey>>> {
+    self.keys.load()
+  }
+
+  pub async fn is_valid(&self, public_key: &str) -> bool {
+    // For Periphery -> Core connection, maybe init
+    // Core public key file if it doesn't exist.
+    self.maybe_write(public_key).await;
+    let keys = self.keys.load();
+    keys.is_empty() || keys.iter().any(|pk| pk.as_str() == public_key)
+  }
+
+  async fn maybe_write(&self, public_key: &str) {
+    let to_write = self.to_write.load();
+    let path = match to_write.as_slice() {
+      // Do nothing if empty
+      [] => return,
+      [path, _rest @ ..] => path,
+    };
+    let public_key = match SpkiPublicKey::from_maybe_pem(public_key) {
+      Ok(public_key) => public_key,
+      Err(e) => {
+        error!("Invalid incoming public key | {e:#}");
+        return;
+      }
+    };
+    if let Err(e) = public_key.write_pem_async(path).await {
+      warn!("Failed to pin incoming public key | {e:#}");
+    }
+    self.refresh();
+  }
+
+  pub fn refresh(&self) {
+    let config = periphery_config();
+    let Some(core_public_keys) = config.core_public_keys.as_ref()
+    else {
+      return;
+    };
+    let mut to_write = Vec::new();
+    let core_public_keys = core_public_keys
+      .iter()
+      .flat_map(|public_key| {
+        if let Some(path) = public_key.strip_prefix("file:")
+        {
+          match (SpkiPublicKey::from_file(path), config.server_enabled) {
+            (Ok(public_key), _) => Some(public_key),
+            (Err(e), false) => {
+              // If only outbound connections, only warn.
+              // It will be written when Core public key received.
+              warn!("{e:#}");
+              let Ok(path) = PathBuf::from_str(path);
+              to_write.push(path);
+              None
+            }
+            (Err(e), true) => {
+              // This is too dangerous to allow if server_enabled.
+              error!("{e:#}");
+              std::process::exit(1)
+            }
+          }
+        } else {
+          SpkiPublicKey::from_maybe_pem(public_key)
+            .context("Invalid hardcoded public key. If this is supposed to point to file, add 'file:' prefix.")
+            .inspect_err(|e| {
+              error!("{e:#}");
+              std::process::exit(1)
+            })
+            .ok()
+        }
+      })
+      .collect::<Vec<_>>();
+    self.keys.store(Arc::new(core_public_keys));
+    self.to_write.store(Arc::new(to_write));
+  }
+}
+
+// Core Address / Host -> Channel
+pub type CoreConnection = BufferedChannel<EncodedTransportMessage>;
+pub type CoreConnections = CloneCache<String, Arc<CoreConnection>>;
+
+pub fn core_connections() -> &'static CoreConnections {
+  static CORE_CONNECTIONS: OnceLock<CoreConnections> =
+    OnceLock::new();
+  CORE_CONNECTIONS.get_or_init(Default::default)
+}
+
+pub fn stats_client() -> &'static RwLock<StatsClient> {
+  static STATS_CLIENT: OnceLock<RwLock<StatsClient>> =
+    OnceLock::new();
+  STATS_CLIENT.get_or_init(|| RwLock::new(StatsClient::default()))
+}
+
+pub type PtyName = String;
+pub type PtyMap =
+  tokio::sync::RwLock<HashMap<PtyName, Arc<Terminal>>>;
+
+pub fn terminals() -> &'static PtyMap {
+  static TERMINALS: OnceLock<PtyMap> = OnceLock::new();
+  TERMINALS.get_or_init(Default::default)
+}
+
+pub type TerminalChannels = CloneCache<Uuid, Arc<TerminalChannel>>;
+
+pub fn terminal_channels() -> &'static TerminalChannels {
+  static TERMINAL_CHANNELS: OnceLock<TerminalChannels> =
+    OnceLock::new();
+  TERMINAL_CHANNELS.get_or_init(Default::default)
+}
+
+#[derive(Debug)]
+pub struct TerminalChannel {
+  pub sender: mpsc::Sender<StdinMsg>,
+  pub cancel: CancellationToken,
+}
+
+pub fn terminal_triggers() -> &'static TerminalTriggers {
+  static TERMINAL_TRIGGERS: OnceLock<TerminalTriggers> =
+    OnceLock::new();
+  TERMINAL_TRIGGERS.get_or_init(Default::default)
+}
+
+/// Periphery must wait for Core to finish setting
+/// up channel forwarding before sending message,
+/// or the first sent messages may be missed.
+#[derive(Default)]
+pub struct TerminalTriggers(CloneCache<Uuid, Arc<TerminalTrigger>>);
+
+impl TerminalTriggers {
+  pub async fn insert(&self, channel: Uuid) {
+    let (sender, receiver) = oneshot::channel();
+    let trigger = Arc::new(TerminalTrigger {
+      sender: Some(sender).into(),
+      receiver: Some(receiver).into(),
+    });
+    self.0.insert(channel, trigger).await;
+  }
+
+  pub async fn send(&self, channel: &Uuid) -> anyhow::Result<()> {
+    let trigger = self.0.get(channel).await.with_context(|| {
+      format!("No trigger found for channel {channel}")
+    })?;
+    trigger.send().await
+  }
+
+  pub async fn wait(&self, channel: &Uuid) -> anyhow::Result<()> {
+    let trigger = self.0.get(channel).await.with_context(|| {
+      format!("No trigger found for channel {channel}")
+    })?;
+    trigger.wait().await?;
+    self.0.remove(channel).await;
+    Ok(())
+  }
+}
+
+#[derive(Debug)]
+pub struct TerminalTrigger {
+  sender: Mutex<Option<oneshot::Sender<()>>>,
+  receiver: Mutex<Option<oneshot::Receiver<()>>>,
+}
+
+impl TerminalTrigger {
+  /// This consumes the Trigger Sender.
+  pub async fn send(&self) -> anyhow::Result<()> {
+    let mut sender = self.sender.lock().await;
+    let sender = sender
+      .take()
+      .context("Called TerminalTrigger 'send' more than once.")?;
+    sender
+      .send(())
+      .map_err(|_| anyhow!("TerminalTrigger sender already used"))
+  }
+
+  /// This consumes the Trigger Receiver.
+  pub async fn wait(&self) -> anyhow::Result<()> {
+    let mut receiver = self.receiver.lock().await;
+    let receiver = receiver
+      .take()
+      .context("Called TerminalTrigger 'wait' more than once.")?;
+    receiver.await.context("Failed to receive TerminalTrigger")
+  }
+}
+
+pub fn docker_client() -> &'static SwappableDockerClient {
+  static DOCKER_CLIENT: OnceLock<SwappableDockerClient> =
+    OnceLock::new();
+  DOCKER_CLIENT.get_or_init(SwappableDockerClient::init)
+}
+
+#[derive(Default)]
+pub struct SwappableDockerClient(ArcSwap<Option<DockerClient>>);
+
+impl SwappableDockerClient {
+  pub fn init() -> Self {
+    let docker = DockerClient::connect()
+      // Only logs on first init, although keeps trying to connect
+      .inspect_err(|e| warn!("{e:#}"))
+      .ok();
+    Self(ArcSwap::new(Arc::new(docker)))
+  }
+
+  pub fn load(&self) -> arc_swap::Guard<Arc<Option<DockerClient>>> {
+    let res = self.0.load();
+    if res.is_some() {
+      return res;
+    }
+    self.reload();
+    self.0.load()
+  }
+
+  pub fn reload(&self) {
+    self.0.store(Arc::new(DockerClient::connect().ok()));
+  }
+}
+
+pub type ContainerStatsMap = HashMap<String, ContainerStats>;
+
+pub fn container_stats() -> &'static ArcSwap<ContainerStatsMap> {
+  static CONTAINER_STATS: OnceLock<ArcSwap<ContainerStatsMap>> =
+    OnceLock::new();
+  CONTAINER_STATS.get_or_init(Default::default)
+}

bin/periphery/src/stats.rs

@@ -1,4 +1,4 @@
-use std::{cmp::Ordering, sync::OnceLock};
+use std::cmp::Ordering;
 
 use async_timing_util::wait_until_timelength;
 use komodo_client::entities::stats::{
@@ -6,15 +6,8 @@ use komodo_client::entities::stats::{
   SystemProcess, SystemStats,
 };
 use sysinfo::{ProcessRefreshKind, ProcessesToUpdate, System};
-use tokio::sync::RwLock;
 
-use crate::config::periphery_config;
-
-pub fn stats_client() -> &'static RwLock<StatsClient> {
-  static STATS_CLIENT: OnceLock<RwLock<StatsClient>> =
-    OnceLock::new();
-  STATS_CLIENT.get_or_init(|| RwLock::new(StatsClient::default()))
-}
+use crate::{config::periphery_config, state::stats_client};
 
 /// This should be called before starting the server in main.rs.
 /// Keeps the cached stats up to date

bin/periphery/src/terminal.rs

@@ -1,43 +1,32 @@
-use std::{
-  collections::{HashMap, VecDeque},
-  sync::{Arc, OnceLock},
-  time::Duration,
-};
+use std::{collections::VecDeque, sync::Arc, time::Duration};
 
 use anyhow::{Context, anyhow};
 use bytes::Bytes;
-use cache::CloneCache;
+use encoding::{Decode as _, WithChannel};
 use komodo_client::{
-  api::write::TerminalRecreateMode, entities::server::TerminalInfo,
+  api::write::TerminalRecreateMode,
+  entities::{ContainerTerminalMode, server::TerminalInfo},
 };
+use periphery_client::transport::EncodedTerminalMessage;
 use portable_pty::{CommandBuilder, PtySize, native_pty_system};
 use tokio::sync::{broadcast, mpsc};
 use tokio_util::sync::CancellationToken;
-use transport::bytes::data_from_transport_bytes;
-use uuid::Uuid;
 
-pub type TerminalChannels =
-  CloneCache<Uuid, (mpsc::Sender<StdinMsg>, CancellationToken)>;
+use crate::state::{terminal_channels, terminal_triggers, terminals};
 
-pub fn terminal_channels() -> &'static TerminalChannels {
-  static TERMINAL_CHANNELS: OnceLock<TerminalChannels> =
-    OnceLock::new();
-  TERMINAL_CHANNELS.get_or_init(Default::default)
-}
-
-pub async fn handle_incoming_message(id: Uuid, bytes: Bytes) {
-  let Some((channel, _)) = terminal_channels().get(&id).await else {
-    warn!("No terminal channel for {id}");
-    return;
-  };
-  let Ok(data) = data_from_transport_bytes(bytes) else {
-    warn!("Got terminal message with no data for {id}");
-    return;
+pub async fn handle_message(message: EncodedTerminalMessage) {
+  let WithChannel {
+    channel: channel_id,
+    mut data,
+  } = match message.decode() {
+    Ok(res) => res,
+    Err(e) => {
+      warn!("Received invalid Terminal bytes | {e:#}");
+      return;
+    }
   };
   let msg = match data.first() {
-    Some(&0x00) => {
-      StdinMsg::Bytes(Bytes::copy_from_slice(&data[1..]))
-    }
+    Some(&0x00) => StdinMsg::Bytes(data.drain(1..).collect()),
     Some(&0xFF) => {
       if let Ok(dimensions) =
         serde_json::from_slice::<ResizeDimensions>(&data[1..])
@@ -48,23 +37,29 @@ pub async fn handle_incoming_message(id: Uuid, bytes: Bytes) {
       }
     }
     Some(_) => StdinMsg::Bytes(data),
-    // No data
-    None => return,
+    // Empty bytes are the "begin" trigger for Terminal Executions
+    None => {
+      if let Err(e) = terminal_triggers().send(&channel_id).await {
+        warn!("{e:#}")
+      }
+      return;
+    }
   };
-  if let Err(e) = channel.send(msg).await {
-    warn!("No receiver for {id} | {e:?}");
+  let Some(channel) = terminal_channels().get(&channel_id).await
+  else {
+    warn!("No terminal channel for {channel_id}");
+    return;
+  };
+  if let Err(e) = channel.sender.send(msg).await {
+    warn!("No receiver for {channel_id} | {e:?}");
   };
 }
 
-type PtyName = String;
-type PtyMap = tokio::sync::RwLock<HashMap<PtyName, Arc<Terminal>>>;
-type StdinSender = mpsc::Sender<StdinMsg>;
-type StdoutReceiver = broadcast::Receiver<Bytes>;
-
 pub async fn create_terminal(
   name: String,
   command: String,
   recreate: TerminalRecreateMode,
+  container: Option<(String, ContainerTerminalMode)>,
 ) -> anyhow::Result<Arc<Terminal>> {
   trace!(
     "CreateTerminal: {name} | command: {command} | recreate: {recreate:?}"
@@ -84,7 +79,7 @@ pub async fn create_terminal(
     }
   }
   let terminal = Arc::new(
-    Terminal::new(command)
+    Terminal::new(command, container)
       .await
       .context("Failed to init terminal")?,
   );
@@ -100,11 +95,25 @@ pub async fn delete_terminal(name: &str) {
   }
 }
 
-pub async fn list_terminals() -> Vec<TerminalInfo> {
+pub async fn list_terminals(
+  container: Option<&str>,
+) -> Vec<TerminalInfo> {
   let mut terminals = terminals()
     .read()
     .await
     .iter()
+    .filter(|(_, terminal)| {
+      // If no container passed, returns all
+      let Some(container) = container else {
+        return true;
+      };
+      let Some(term_container) =
+        terminal.container.as_ref().map(|c| c.0.as_str())
+      else {
+        return false;
+      };
+      term_container == container
+    })
     .map(|(name, terminal)| TerminalInfo {
       name: name.to_string(),
       command: terminal.command.clone(),
@@ -144,11 +153,6 @@ pub async fn delete_all_terminals() {
   tokio::time::sleep(Duration::from_millis(100)).await;
 }
 
-fn terminals() -> &'static PtyMap {
-  static TERMINALS: OnceLock<PtyMap> = OnceLock::new();
-  TERMINALS.get_or_init(Default::default)
-}
-
 #[derive(Clone, serde::Deserialize)]
 pub struct ResizeDimensions {
   rows: u16,
@@ -157,10 +161,13 @@ pub struct ResizeDimensions {
 
 #[derive(Clone)]
 pub enum StdinMsg {
-  Bytes(Bytes),
+  Bytes(Vec<u8>),
   Resize(ResizeDimensions),
 }
 
+pub type StdinSender = mpsc::Sender<StdinMsg>;
+pub type StdoutReceiver = broadcast::Receiver<Bytes>;
+
 pub struct Terminal {
   /// The command that was used as the root command, eg `shell`
   command: String,
@@ -171,10 +178,16 @@ pub struct Terminal {
   pub stdout: StdoutReceiver,
 
   pub history: Arc<History>,
+
+  /// If terminal is for a container.
+  pub container: Option<(String, ContainerTerminalMode)>,
 }
 
 impl Terminal {
-  async fn new(command: String) -> anyhow::Result<Terminal> {
+  async fn new(
+    command: String,
+    container: Option<(String, ContainerTerminalMode)>,
+  ) -> anyhow::Result<Terminal> {
     trace!("Creating terminal with command: {command}");
 
     let terminal = native_pty_system()
@@ -340,6 +353,7 @@ impl Terminal {
       stdin,
       stdout,
       history,
+      container,
     })
   }
 

client/core/rs/src/api/execute/maintenance.rs

@@ -95,3 +95,34 @@ pub struct GlobalAutoUpdate {}
 #[response(Update)]
 #[error(serror::Error)]
 pub struct RotateAllServerKeys {}
+
+//
+
+/// **Admin only.** Rotates the Core private key,
+/// and all Server public keys.
+/// Response: [Update].
+///
+/// If any Server is `NotOk`, this will fail.
+/// To proceed anyways, pass `force: true`.
+#[typeshare]
+#[derive(
+  Debug,
+  Clone,
+  PartialEq,
+  Serialize,
+  Deserialize,
+  Resolve,
+  EmptyTraits,
+  Parser,
+)]
+#[empty_traits(KomodoExecuteRequest)]
+#[response(Update)]
+#[error(serror::Error)]
+pub struct RotateCoreKeys {
+  /// Force the rotation to proceed even if a Server is `NotOk`.
+  /// The Core Public Key in Periphery config may have to be updated manually.
+  /// (alias: `f`)
+  #[serde(default)]
+  #[clap(long, short, alias = "f", default_value_t = false)]
+  pub force: bool,
+}

client/core/rs/src/api/execute/mod.rs

@@ -161,12 +161,17 @@ pub enum Execution {
 
   // MAINTENANCE
   ClearRepoCache(ClearRepoCache),
-  #[clap(alias = "backup-database", alias = "backup-db", alias = "backup")]
+  #[clap(
+    alias = "backup-database",
+    alias = "backup-db",
+    alias = "backup"
+  )]
   BackupCoreDatabase(BackupCoreDatabase),
   #[clap(alias = "auto-update")]
   GlobalAutoUpdate(GlobalAutoUpdate),
   #[clap(alias = "rotate-keys")]
   RotateAllServerKeys(RotateAllServerKeys),
+  RotateCoreKeys(RotateCoreKeys),
 
   // SLEEP
   Sleep(Sleep),

client/core/rs/src/api/read/build.rs

@@ -236,30 +236,3 @@ pub struct ListCommonBuildExtraArgs {
 
 #[typeshare]
 pub type ListCommonBuildExtraArgsResponse = Vec<String>;
-
-//
-
-/// Get whether a Build's target repo has a webhook for the build configured. Response: [GetBuildWebhookEnabledResponse].
-#[typeshare]
-#[derive(
-  Serialize, Deserialize, Debug, Clone, Resolve, EmptyTraits,
-)]
-#[empty_traits(KomodoReadRequest)]
-#[response(GetBuildWebhookEnabledResponse)]
-#[error(serror::Error)]
-pub struct GetBuildWebhookEnabled {
-  /// Id or name
-  #[serde(alias = "id", alias = "name")]
-  pub build: String,
-}
-
-/// Response for [GetBuildWebhookEnabled]
-#[typeshare]
-#[derive(Serialize, Deserialize, Debug, Clone)]
-pub struct GetBuildWebhookEnabledResponse {
-  /// Whether the repo webhooks can even be managed.
-  /// The repo owner must be in `github_webhook_app.owners` list to be managed.
-  pub managed: bool,
-  /// Whether pushes to branch trigger build. Will always be false if managed is false.
-  pub enabled: bool,
-}

client/core/rs/src/api/read/mod.rs

@@ -108,8 +108,6 @@ pub struct GetCoreInfoResponse {
   pub disable_non_admin_create: bool,
   /// Whether confirm dialog should be disabled
   pub disable_confirm_dialog: bool,
-  /// The repo owners for which github webhook management api is available
-  pub github_webhook_owners: Vec<String>,
   /// Whether to disable websocket automatic reconnect.
   pub disable_websocket_reconnect: bool,
   /// Whether to enable fancy toml highlighting.

client/core/rs/src/api/read/repo.rs

@@ -117,34 +117,3 @@ pub struct GetReposSummaryResponse {
   /// The number of repos with unknown state.
   pub unknown: u32,
 }
-
-//
-
-/// Get a target Repo's configured webhooks. Response: [GetRepoWebhooksEnabledResponse].
-#[typeshare]
-#[derive(
-  Serialize, Deserialize, Debug, Clone, Resolve, EmptyTraits,
-)]
-#[empty_traits(KomodoReadRequest)]
-#[response(GetRepoWebhooksEnabledResponse)]
-#[error(serror::Error)]
-pub struct GetRepoWebhooksEnabled {
-  /// Id or name
-  #[serde(alias = "id", alias = "name")]
-  pub repo: String,
-}
-
-/// Response for [GetRepoWebhooksEnabled]
-#[typeshare]
-#[derive(Serialize, Deserialize, Debug, Clone)]
-pub struct GetRepoWebhooksEnabledResponse {
-  /// Whether the repo webhooks can even be managed.
-  /// The repo owner must be in `github_webhook_app.owners` list to be managed.
-  pub managed: bool,
-  /// Whether pushes to branch trigger clone. Will always be false if managed is false.
-  pub clone_enabled: bool,
-  /// Whether pushes to branch trigger pull. Will always be false if managed is false.
-  pub pull_enabled: bool,
-  /// Whether pushes to branch trigger build. Will always be false if managed is false.
-  pub build_enabled: bool,
-}

client/core/rs/src/api/read/stack.rs

@@ -276,32 +276,3 @@ pub struct GetStacksSummaryResponse {
   /// The number of stacks with Unknown state.
   pub unknown: u32,
 }
-
-//
-
-/// Get a target stack's configured webhooks. Response: [GetStackWebhooksEnabledResponse].
-#[typeshare]
-#[derive(
-  Serialize, Deserialize, Debug, Clone, Resolve, EmptyTraits,
-)]
-#[empty_traits(KomodoReadRequest)]
-#[response(GetStackWebhooksEnabledResponse)]
-#[error(serror::Error)]
-pub struct GetStackWebhooksEnabled {
-  /// Id or name
-  #[serde(alias = "id", alias = "name")]
-  pub stack: String,
-}
-
-/// Response for [GetStackWebhooksEnabled]
-#[typeshare]
-#[derive(Serialize, Deserialize, Debug, Clone)]
-pub struct GetStackWebhooksEnabledResponse {
-  /// Whether the repo webhooks can even be managed.
-  /// The repo owner must be in `github_webhook_app.owners` list to be managed.
-  pub managed: bool,
-  /// Whether pushes to branch trigger refresh. Will always be false if managed is false.
-  pub refresh_enabled: bool,
-  /// Whether pushes to branch trigger stack execution. Will always be false if managed is false.
-  pub deploy_enabled: bool,
-}

client/core/rs/src/api/read/sync.rs

@@ -116,32 +116,3 @@ pub struct GetResourceSyncsSummaryResponse {
   /// The number of syncs with unknown state.
   pub unknown: u32,
 }
-
-//
-
-/// Get a target Sync's configured webhooks. Response: [GetSyncWebhooksEnabledResponse].
-#[typeshare]
-#[derive(
-  Serialize, Deserialize, Debug, Clone, Resolve, EmptyTraits,
-)]
-#[empty_traits(KomodoReadRequest)]
-#[response(GetSyncWebhooksEnabledResponse)]
-#[error(serror::Error)]
-pub struct GetSyncWebhooksEnabled {
-  /// Id or name
-  #[serde(alias = "id", alias = "name")]
-  pub sync: String,
-}
-
-/// Response for [GetSyncWebhooksEnabled]
-#[typeshare]
-#[derive(Serialize, Deserialize, Debug, Clone)]
-pub struct GetSyncWebhooksEnabledResponse {
-  /// Whether the repo webhooks can even be managed.
-  /// The repo owner must be in `github_webhook_app.owners` list to be managed.
-  pub managed: bool,
-  /// Whether pushes to branch trigger refresh. Will always be false if managed is false.
-  pub refresh_enabled: bool,
-  /// Whether pushes to branch trigger sync execution. Will always be false if managed is false.
-  pub sync_enabled: bool,
-}

client/core/rs/src/api/terminal.rs

@@ -1,8 +1,9 @@
 use serde::{Deserialize, Serialize};
 use typeshare::typeshare;
 
+use crate::api::write::TerminalRecreateMode;
+
 /// Query to connect to a terminal (interactive shell over websocket) on the given server.
-/// TODO: Document calling.
 #[typeshare]
 #[derive(Serialize, Deserialize, Debug, Clone)]
 pub struct ConnectTerminalQuery {
@@ -16,23 +17,18 @@ pub struct ConnectTerminalQuery {
 }
 
 /// Execute a terminal command on the given server.
-/// TODO: Document calling.
 #[typeshare]
 #[derive(Serialize, Deserialize, Debug, Clone)]
 pub struct ExecuteTerminalBody {
   /// Server Id or name
   pub server: String,
   /// The name of the terminal on the server to use to execute.
-  /// If the terminal at name exists, it will be used to execute the command.
-  /// Otherwise, a new terminal will be created for this command, which will
-  /// persist until it exits or is deleted.
   pub terminal: String,
   /// The command to execute.
   pub command: String,
 }
 
 /// Query to connect to a container exec session (interactive shell over websocket) on the given server.
-/// TODO: Document calling.
 #[typeshare]
 #[derive(Serialize, Deserialize, Debug, Clone)]
 pub struct ConnectContainerExecQuery {
@@ -42,10 +38,27 @@ pub struct ConnectContainerExecQuery {
   pub container: String,
   /// The shell to use (eg. `sh` or `bash`)
   pub shell: String,
+  /// Specify the recreate behavior.
+  /// Default is 'DifferentCommand'
+  #[serde(default = "default_container_recreate_mode")]
+  pub recreate: TerminalRecreateMode,
+}
+
+/// Query to connect to a container attach session (interactive shell over websocket) on the given server.
+#[typeshare]
+#[derive(Serialize, Deserialize, Debug, Clone)]
+pub struct ConnectContainerAttachQuery {
+  /// Server Id or name
+  pub server: String,
+  /// The container name
+  pub container: String,
+  /// Specify the recreate behavior.
+  /// Default is 'DifferentCommand'
+  #[serde(default = "default_container_recreate_mode")]
+  pub recreate: TerminalRecreateMode,
 }
 
 /// Execute a command in the given containers shell.
-/// TODO: Document calling.
 #[typeshare]
 #[derive(Serialize, Deserialize, Debug, Clone)]
 pub struct ExecuteContainerExecBody {
@@ -57,11 +70,14 @@ pub struct ExecuteContainerExecBody {
   pub shell: String,
   /// The command to execute.
   pub command: String,
+  /// Specify the recreate behavior.
+  /// Default is 'DifferentCommand'
+  #[serde(default = "default_container_recreate_mode")]
+  pub recreate: TerminalRecreateMode,
 }
 
 /// Query to connect to a container exec session (interactive shell over websocket) on the given Deployment.
 /// This call will use access to the Deployment Terminal to permission the call.
-/// TODO: Document calling.
 #[typeshare]
 #[derive(Serialize, Deserialize, Debug, Clone)]
 pub struct ConnectDeploymentExecQuery {
@@ -69,10 +85,26 @@ pub struct ConnectDeploymentExecQuery {
   pub deployment: String,
   /// The shell to use (eg. `sh` or `bash`)
   pub shell: String,
+  /// Specify the recreate behavior.
+  /// Default is 'DifferentCommand'
+  #[serde(default = "default_container_recreate_mode")]
+  pub recreate: TerminalRecreateMode,
+}
+
+/// Query to connect to a container attach session (interactive shell over websocket) on the given Deployment.
+/// This call will use access to the Deployment Terminal to permission the call.
+#[typeshare]
+#[derive(Serialize, Deserialize, Debug, Clone)]
+pub struct ConnectDeploymentAttachQuery {
+  /// Deployment Id or name
+  pub deployment: String,
+  /// Specify the recreate behavior.
+  /// Default is 'DifferentCommand'
+  #[serde(default = "default_container_recreate_mode")]
+  pub recreate: TerminalRecreateMode,
 }
 
 /// Execute a command in the given containers shell.
-/// TODO: Document calling.
 #[typeshare]
 #[derive(Serialize, Deserialize, Debug, Clone)]
 pub struct ExecuteDeploymentExecBody {
@@ -82,11 +114,14 @@ pub struct ExecuteDeploymentExecBody {
   pub shell: String,
   /// The command to execute.
   pub command: String,
+  /// Specify the recreate behavior.
+  /// Default is 'DifferentCommand'
+  #[serde(default = "default_container_recreate_mode")]
+  pub recreate: TerminalRecreateMode,
 }
 
 /// Query to connect to a container exec session (interactive shell over websocket) on the given Stack / service.
 /// This call will use access to the Stack Terminal to permission the call.
-/// TODO: Document calling.
 #[typeshare]
 #[derive(Serialize, Deserialize, Debug, Clone)]
 pub struct ConnectStackExecQuery {
@@ -96,10 +131,28 @@ pub struct ConnectStackExecQuery {
   pub service: String,
   /// The shell to use (eg. `sh` or `bash`)
   pub shell: String,
+  /// Specify the recreate behavior.
+  /// Default is 'DifferentCommand'
+  #[serde(default = "default_container_recreate_mode")]
+  pub recreate: TerminalRecreateMode,
+}
+
+/// Query to connect to a container attach session (interactive shell over websocket) on the given Stack / service.
+/// This call will use access to the Stack Terminal to permission the call.
+#[typeshare]
+#[derive(Serialize, Deserialize, Debug, Clone)]
+pub struct ConnectStackAttachQuery {
+  /// Stack Id or name
+  pub stack: String,
+  /// The service name to attach to
+  pub service: String,
+  /// Specify the recreate behavior.
+  /// Default is 'DifferentCommand'
+  #[serde(default = "default_container_recreate_mode")]
+  pub recreate: TerminalRecreateMode,
 }
 
 /// Execute a command in the given containers shell.
-/// TODO: Document calling.
 #[typeshare]
 #[derive(Serialize, Deserialize, Debug, Clone)]
 pub struct ExecuteStackExecBody {
@@ -111,4 +164,12 @@ pub struct ExecuteStackExecBody {
   pub shell: String,
   /// The command to execute.
   pub command: String,
+  /// Specify the recreate behavior.
+  /// Default is 'DifferentCommand'
+  #[serde(default = "default_container_recreate_mode")]
+  pub recreate: TerminalRecreateMode,
+}
+
+fn default_container_recreate_mode() -> TerminalRecreateMode {
+  TerminalRecreateMode::DifferentCommand
 }

client/core/rs/src/api/write/action.rs

@@ -4,7 +4,6 @@ use serde::{Deserialize, Serialize};
 use typeshare::typeshare;
 
 use crate::entities::{
-  NoData,
   action::{_PartialActionConfig, Action},
   update::Update,
 };
@@ -104,41 +103,3 @@ pub struct RenameAction {
   /// The new name.
   pub name: String,
 }
-
-/// Create a webhook on the github action attached to the Action resource.
-/// passed in request. Response: [CreateActionWebhookResponse]
-#[typeshare]
-#[derive(
-  Serialize, Deserialize, Debug, Clone, Resolve, EmptyTraits,
-)]
-#[empty_traits(KomodoWriteRequest)]
-#[response(CreateActionWebhookResponse)]
-#[error(serror::Error)]
-pub struct CreateActionWebhook {
-  /// Id or name
-  #[serde(alias = "id", alias = "name")]
-  pub action: String,
-}
-
-#[typeshare]
-pub type CreateActionWebhookResponse = NoData;
-
-//
-
-/// Delete the webhook on the github action attached to the Action resource.
-/// passed in request. Response: [DeleteActionWebhookResponse]
-#[typeshare]
-#[derive(
-  Serialize, Deserialize, Debug, Clone, Resolve, EmptyTraits,
-)]
-#[empty_traits(KomodoWriteRequest)]
-#[response(DeleteActionWebhookResponse)]
-#[error(serror::Error)]
-pub struct DeleteActionWebhook {
-  /// Id or name
-  #[serde(alias = "id", alias = "name")]
-  pub action: String,
-}
-
-#[typeshare]
-pub type DeleteActionWebhookResponse = NoData;

client/core/rs/src/api/write/alert.rs

@@ -0,0 +1,22 @@
+use derive_empty_traits::EmptyTraits;
+use resolver_api::Resolve;
+use serde::{Deserialize, Serialize};
+use typeshare::typeshare;
+
+use crate::{api::write::KomodoWriteRequest, entities::NoData};
+
+//
+
+/// **Admin only.** Close the Alert at the given id.
+/// Response: [NoData]
+#[typeshare]
+#[derive(
+  Debug, Clone, Serialize, Deserialize, Resolve, EmptyTraits,
+)]
+#[empty_traits(KomodoWriteRequest)]
+#[response(NoData)]
+#[error(serror::Error)]
+pub struct CloseAlert {
+  /// The id of the Alert to close.
+  pub id: String,
+}

client/core/rs/src/api/write/build.rs

@@ -138,43 +138,3 @@ pub struct RefreshBuildCache {
   #[serde(alias = "id", alias = "name")]
   pub build: String,
 }
-
-//
-
-/// Create a webhook on the github repo attached to the build
-/// passed in request. Response: [CreateBuildWebhookResponse]
-#[typeshare]
-#[derive(
-  Serialize, Deserialize, Debug, Clone, Resolve, EmptyTraits,
-)]
-#[empty_traits(KomodoWriteRequest)]
-#[response(CreateBuildWebhookResponse)]
-#[error(serror::Error)]
-pub struct CreateBuildWebhook {
-  /// Id or name
-  #[serde(alias = "id", alias = "name")]
-  pub build: String,
-}
-
-#[typeshare]
-pub type CreateBuildWebhookResponse = NoData;
-
-//
-
-/// Delete a webhook on the github repo attached to the build
-/// passed in request. Response: [CreateBuildWebhookResponse]
-#[typeshare]
-#[derive(
-  Serialize, Deserialize, Debug, Clone, Resolve, EmptyTraits,
-)]
-#[empty_traits(KomodoWriteRequest)]
-#[response(DeleteBuildWebhookResponse)]
-#[error(serror::Error)]
-pub struct DeleteBuildWebhook {
-  /// Id or name
-  #[serde(alias = "id", alias = "name")]
-  pub build: String,
-}
-
-#[typeshare]
-pub type DeleteBuildWebhookResponse = NoData;

client/core/rs/src/api/write/mod.rs

@@ -1,4 +1,5 @@
 mod action;
+mod alert;
 mod alerter;
 mod api_key;
 mod build;
@@ -19,6 +20,7 @@ mod user_group;
 mod variable;
 
 pub use action::*;
+pub use alert::*;
 pub use alerter::*;
 pub use api_key::*;
 pub use build::*;

client/core/rs/src/api/write/repo.rs

@@ -133,45 +133,3 @@ pub enum RepoWebhookAction {
   Pull,
   Build,
 }
-
-/// Create a webhook on the github repo attached to the (Komodo) Repo resource.
-/// passed in request. Response: [CreateRepoWebhookResponse]
-#[typeshare]
-#[derive(
-  Serialize, Deserialize, Debug, Clone, Resolve, EmptyTraits,
-)]
-#[empty_traits(KomodoWriteRequest)]
-#[response(CreateRepoWebhookResponse)]
-#[error(serror::Error)]
-pub struct CreateRepoWebhook {
-  /// Id or name
-  #[serde(alias = "id", alias = "name")]
-  pub repo: String,
-  /// "Clone" or "Pull" or "Build"
-  pub action: RepoWebhookAction,
-}
-
-#[typeshare]
-pub type CreateRepoWebhookResponse = NoData;
-
-//
-
-/// Delete the webhook on the github repo attached to the (Komodo) Repo resource.
-/// passed in request. Response: [DeleteRepoWebhookResponse]
-#[typeshare]
-#[derive(
-  Serialize, Deserialize, Debug, Clone, Resolve, EmptyTraits,
-)]
-#[empty_traits(KomodoWriteRequest)]
-#[response(DeleteRepoWebhookResponse)]
-#[error(serror::Error)]
-pub struct DeleteRepoWebhook {
-  /// Id or name
-  #[serde(alias = "id", alias = "name")]
-  pub repo: String,
-  /// "Clone" or "Pull" or "Build"
-  pub action: RepoWebhookAction,
-}
-
-#[typeshare]
-pub type DeleteRepoWebhookResponse = NoData;

client/core/rs/src/api/write/server.rs

@@ -134,7 +134,7 @@ pub struct CreateNetwork {
 /// Configures the behavior of [CreateTerminal] if the
 /// specified terminal name already exists.
 #[typeshare]
-#[derive(Serialize, Deserialize, Debug, Clone, Copy, Default)]
+#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize)]
 pub enum TerminalRecreateMode {
   /// Never kill the old terminal if it already exists.
   /// If the command is different, returns error.

client/core/rs/src/api/write/stack.rs

@@ -155,45 +155,3 @@ pub enum StackWebhookAction {
   Refresh,
   Deploy,
 }
-
-/// Create a webhook on the github repo attached to the stack
-/// passed in request. Response: [CreateStackWebhookResponse]
-#[typeshare]
-#[derive(
-  Serialize, Deserialize, Debug, Clone, Resolve, EmptyTraits,
-)]
-#[empty_traits(KomodoWriteRequest)]
-#[response(CreateStackWebhookResponse)]
-#[error(serror::Error)]
-pub struct CreateStackWebhook {
-  /// Id or name
-  #[serde(alias = "id", alias = "name")]
-  pub stack: String,
-  /// "Refresh" or "Deploy"
-  pub action: StackWebhookAction,
-}
-
-#[typeshare]
-pub type CreateStackWebhookResponse = NoData;
-
-//
-
-/// Delete the webhook on the github repo attached to the stack
-/// passed in request. Response: [DeleteStackWebhookResponse]
-#[typeshare]
-#[derive(
-  Serialize, Deserialize, Debug, Clone, Resolve, EmptyTraits,
-)]
-#[empty_traits(KomodoWriteRequest)]
-#[response(DeleteStackWebhookResponse)]
-#[error(serror::Error)]
-pub struct DeleteStackWebhook {
-  /// Id or name
-  #[serde(alias = "id", alias = "name")]
-  pub stack: String,
-  /// "Refresh" or "Deploy"
-  pub action: StackWebhookAction,
-}
-
-#[typeshare]
-pub type DeleteStackWebhookResponse = NoData;

client/core/rs/src/api/write/sync.rs

@@ -5,7 +5,6 @@ use serde::{Deserialize, Serialize};
 use typeshare::typeshare;
 
 use crate::entities::{
-  NoData,
   sync::{_PartialResourceSyncConfig, ResourceSync},
   update::Update,
 };
@@ -178,45 +177,3 @@ pub enum SyncWebhookAction {
   Refresh,
   Sync,
 }
-
-/// Create a webhook on the github repo attached to the sync
-/// passed in request. Response: [CreateSyncWebhookResponse]
-#[typeshare]
-#[derive(
-  Serialize, Deserialize, Debug, Clone, Resolve, EmptyTraits,
-)]
-#[empty_traits(KomodoWriteRequest)]
-#[response(CreateSyncWebhookResponse)]
-#[error(serror::Error)]
-pub struct CreateSyncWebhook {
-  /// Id or name
-  #[serde(alias = "id", alias = "name")]
-  pub sync: String,
-  /// "Refresh" or "Sync"
-  pub action: SyncWebhookAction,
-}
-
-#[typeshare]
-pub type CreateSyncWebhookResponse = NoData;
-
-//
-
-/// Delete the webhook on the github repo attached to the sync
-/// passed in request. Response: [DeleteSyncWebhookResponse]
-#[typeshare]
-#[derive(
-  Serialize, Deserialize, Debug, Clone, Resolve, EmptyTraits,
-)]
-#[empty_traits(KomodoWriteRequest)]
-#[response(DeleteSyncWebhookResponse)]
-#[error(serror::Error)]
-pub struct DeleteSyncWebhook {
-  /// Id or name
-  #[serde(alias = "id", alias = "name")]
-  pub sync: String,
-  /// "Refresh" or "Sync"
-  pub action: SyncWebhookAction,
-}
-
-#[typeshare]
-pub type DeleteSyncWebhookResponse = NoData;

client/core/rs/src/entities/config/core.rs

@@ -214,25 +214,6 @@ pub struct Env {
   /// Override `github_oauth.secret` from file
   pub komodo_github_oauth_secret_file: Option<PathBuf>,
 
-  /// Override `github_webhook_app.app_id`
-  pub komodo_github_webhook_app_app_id: Option<i64>,
-  /// Override `github_webhook_app.app_id` from file
-  pub komodo_github_webhook_app_app_id_file: Option<PathBuf>,
-  /// Override `github_webhook_app.installations[i].id`. Accepts comma seperated list.
-  ///
-  /// Note. Paired by index with values in `komodo_github_webhook_app_installations_namespaces`
-  pub komodo_github_webhook_app_installations_ids: Option<Vec<i64>>,
-  /// Override `github_webhook_app.installations[i].id` from file
-  pub komodo_github_webhook_app_installations_ids_file:
-    Option<PathBuf>,
-  /// Override `github_webhook_app.installations[i].namespace`. Accepts comma seperated list.
-  ///
-  /// Note. Paired by index with values in `komodo_github_webhook_app_installations_ids`
-  pub komodo_github_webhook_app_installations_namespaces:
-    Option<Vec<String>>,
-  /// Override `github_webhook_app.pk_path`
-  pub komodo_github_webhook_app_pk_path: Option<String>,
-
   /// Override `database.uri`
   #[serde(alias = "komodo_mongo_uri")]
   pub komodo_database_uri: Option<String>,
@@ -544,11 +525,6 @@ pub struct CoreConfig {
   #[serde(default)]
   pub webhook_base_url: String,
 
-  /// Configure a Github Webhook app.
-  /// Allows users to manage repo webhooks from within the Komodo UI.
-  #[serde(default)]
-  pub github_webhook_app: GithubWebhookAppConfig,
-
   // ===========
   // = Logging =
   // ===========
@@ -782,7 +758,6 @@ impl Default for CoreConfig {
       github_oauth: Default::default(),
       webhook_secret: Default::default(),
       webhook_base_url: Default::default(),
-      github_webhook_app: Default::default(),
       logging: Default::default(),
       pretty_startup_config: Default::default(),
       unsafe_unsanitized_startup_config: Default::default(),
@@ -879,7 +854,6 @@ impl CoreConfig {
       },
       webhook_secret: empty_or_redacted(&config.webhook_secret),
       webhook_base_url: config.webhook_base_url,
-      github_webhook_app: config.github_webhook_app,
       database: config.database.sanitized(),
       aws: AwsCredentials {
         access_key_id: empty_or_redacted(&config.aws.access_key_id),
@@ -942,38 +916,3 @@ pub struct AwsCredentials {
   /// The aws SECRET_ACCESS_KEY
   pub secret_access_key: String,
 }
-
-/// Provide configuration for a Github Webhook app.
-#[derive(Debug, Clone, Deserialize)]
-pub struct GithubWebhookAppConfig {
-  /// Github app id
-  pub app_id: i64,
-  /// Configure the app installations on multiple accounts / organizations.
-  pub installations: Vec<GithubWebhookAppInstallationConfig>,
-  /// Private key path. Default: /github-private-key.pem.
-  #[serde(default = "default_private_key_path")]
-  pub pk_path: String,
-}
-
-fn default_private_key_path() -> String {
-  String::from("/github/private-key.pem")
-}
-
-impl Default for GithubWebhookAppConfig {
-  fn default() -> Self {
-    GithubWebhookAppConfig {
-      app_id: 0,
-      installations: Default::default(),
-      pk_path: default_private_key_path(),
-    }
-  }
-}
-
-/// Provide configuration for a Github Webhook app installation.
-#[derive(Debug, Clone, Deserialize)]
-pub struct GithubWebhookAppInstallationConfig {
-  /// The installation ID
-  pub id: i64,
-  /// The user or organization name
-  pub namespace: String,
-}

client/core/rs/src/entities/config/periphery.rs

@@ -18,7 +18,10 @@ use serde::Deserialize;
 use std::{collections::HashMap, path::PathBuf};
 
 use crate::{
-  deserializers::{ForgivingVec, option_string_list_deserializer},
+  deserializers::{
+    ForgivingVec, option_string_list_deserializer,
+    string_list_deserializer,
+  },
   entities::{
     Timelength,
     logger::{LogConfig, LogLevel, StdioLogMode},
@@ -172,8 +175,9 @@ pub struct Env {
   pub periphery_build_dir: Option<PathBuf>,
   /// Override `disable_terminals`
   pub periphery_disable_terminals: Option<bool>,
-  /// Override `disable_container_exec`
-  pub periphery_disable_container_exec: Option<bool>,
+  /// Override `disable_container_terminals`
+  #[serde(alias = "periphery_disable_container_exec")]
+  pub periphery_disable_container_terminals: Option<bool>,
   /// Override `stats_polling_rate`
   pub periphery_stats_polling_rate: Option<Timelength>,
   /// Override `container_stats_polling_rate`
@@ -258,10 +262,9 @@ pub struct PeripheryConfig {
   #[serde(
     default,
     alias = "core_address",
-    deserialize_with = "option_string_list_deserializer",
-    skip_serializing_if = "Option::is_none"
+    deserialize_with = "string_list_deserializer"
   )]
-  pub core_addresses: Option<Vec<String>>,
+  pub core_addresses: Vec<String>,
 
   /// Allow Periphery to connect to Core
   /// without validating the Core certs
@@ -269,8 +272,8 @@ pub struct PeripheryConfig {
   pub core_tls_insecure_skip_verify: bool,
 
   /// Server name / id to connect as
-  #[serde(skip_serializing_if = "Option::is_none")]
-  pub connect_as: Option<String>,
+  #[serde(default)]
+  pub connect_as: String,
 
   // ======================
   // = INBOUND CONNECTION =
@@ -351,11 +354,11 @@ pub struct PeripheryConfig {
   #[serde(default)]
   pub disable_terminals: bool,
 
-  /// Whether to disable the container exec api
+  /// Whether to disable the container exec / attach api
   /// and disallow remote container shell access.
   /// Default: false
-  #[serde(default)]
-  pub disable_container_exec: bool,
+  #[serde(default, alias = "disable_container_exec")]
+  pub disable_container_terminals: bool,
 
   /// The rate at which the system stats will be polled to update the cache.
   /// Options: https://docs.rs/komodo_client/latest/komodo_client/entities/enum.Timelength.html
@@ -443,9 +446,9 @@ impl Default for PeripheryConfig {
       onboarding_key: None,
       core_public_keys: None,
       passkeys: None,
-      core_addresses: None,
+      core_addresses: Default::default(),
       core_tls_insecure_skip_verify: Default::default(),
-      connect_as: None,
+      connect_as: Default::default(),
       server_enabled: default_server_enabled(),
       port: default_periphery_port(),
       bind_ip: default_periphery_bind_ip(),
@@ -454,7 +457,7 @@ impl Default for PeripheryConfig {
       stack_dir: None,
       build_dir: None,
       disable_terminals: Default::default(),
-      disable_container_exec: Default::default(),
+      disable_container_terminals: Default::default(),
       stats_polling_rate: default_stats_polling_rate(),
       container_stats_polling_rate:
         default_container_stats_polling_rate(),
@@ -504,7 +507,7 @@ impl PeripheryConfig {
       stack_dir: self.stack_dir.clone(),
       build_dir: self.build_dir.clone(),
       disable_terminals: self.disable_terminals,
-      disable_container_exec: self.disable_container_exec,
+      disable_container_terminals: self.disable_container_terminals,
       stats_polling_rate: self.stats_polling_rate,
       container_stats_polling_rate: self.container_stats_polling_rate,
       legacy_compose_cli: self.legacy_compose_cli,

client/core/rs/src/entities/mod.rs

@@ -605,6 +605,28 @@ pub struct RepoExecutionResponse {
   pub commit_message: Option<String>,
 }
 
+#[typeshare]
+#[derive(
+  Debug,
+  Clone,
+  Copy,
+  PartialEq,
+  Eq,
+  Hash,
+  Default,
+  Serialize,
+  Deserialize,
+  Display,
+  EnumString,
+)]
+#[serde(rename_all = "lowercase")]
+#[strum(serialize_all = "lowercase")]
+pub enum ContainerTerminalMode {
+  #[default]
+  Exec,
+  Attach,
+}
+
 #[typeshare]
 #[derive(
   Debug,
@@ -1175,6 +1197,7 @@ pub enum Operation {
   BackupCoreDatabase,
   GlobalAutoUpdate,
   RotateAllServerKeys,
+  RotateCoreKeys,
 
   // variable
   CreateVariable,

client/core/rs/src/entities/onboarding_key.rs

@@ -25,6 +25,11 @@ pub struct OnboardingKey {
   #[serde(default)]
   pub enabled: bool,
 
+  /// Expiry of key, or 0 if never expires
+  #[serde(default)]
+  #[cfg_attr(feature = "mongo", index)]
+  pub expires: I64,
+
   /// Name associated with the api key for management
   #[serde(default)]
   pub name: String,
@@ -37,10 +42,6 @@ pub struct OnboardingKey {
   #[serde(default)]
   pub created_at: I64,
 
-  /// Expiry of key, or 0 if never expires
-  #[serde(default)]
-  pub expires: I64,
-
   /// Default tags to give to Servers created with this key.
   #[serde(default)]
   pub tags: Vec<String>,