implement core variables / secrets on the backend

Cargo.lock

@@ -2281,6 +2281,7 @@ dependencies = [
  "serror",
  "sha2",
  "slack_client_rs",
+ "svi",
  "termination_signal",
  "tokio",
  "tokio-util",
@@ -3514,9 +3515,9 @@ checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc"
 
 [[package]]
 name = "svi"
-version = "0.1.4"
+version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "78a65bb8a7e77c020a52094de0c5cab44506727339282368418b285fe9bc92ef"
+checksum = "6136b927b6a7bd321e3ebfd1380d396a6952d4f5c14dcd0893724110d79a5c3b"
 dependencies = [
  "thiserror",
 ]

Cargo.toml

@@ -31,7 +31,7 @@ mongo_indexed = "0.3.0"
 resolver_api = "1.1.0"
 parse_csl = "0.1.0"
 mungos = "0.5.6"
-svi = "0.1.4"
+svi = "1.0.0"
 
 # ASYNC
 tokio = { version = "1.37.0", features = ["full"] }

bin/core/Cargo.toml

@@ -29,6 +29,7 @@ resolver_api.workspace = true
 parse_csl.workspace = true
 mungos.workspace = true
 slack.workspace = true
+svi.workspace = true
 # external
 urlencoding.workspace = true
 aws-sdk-ec2.workspace = true

bin/core/src/api/execute/build.rs

@@ -1,4 +1,4 @@
-use std::time::Duration;
+use std::{collections::HashSet, time::Duration};
 
 use anyhow::{anyhow, Context};
 use futures::future::join_all;
@@ -45,7 +45,7 @@ use crate::{
   helpers::{
     channel::build_cancel_channel,
     periphery_client,
-    query::get_deployment_state,
+    query::{get_deployment_state, get_global_variables},
     update::{add_update, make_update, update_update},
   },
   resource::{self, refresh_build_state_cache},
@@ -139,6 +139,7 @@ impl Resolve<RunBuild, User> for State {
       };
 
     let core_config = core_config();
+    let variables = get_global_variables().await?;
 
     // CLONE REPO
 
@@ -182,11 +183,35 @@ impl Resolve<RunBuild, User> for State {
         .get(&build.config.docker_account)
         .cloned();
 
+      // Interpolate variables / secrets into build args
+      let mut replacers = HashSet::new();
+      for arg in &mut build.config.build_args {
+        // first pass - global variables - don't need replacers.
+        let (res, _) = svi::interpolate_variables(
+          &arg.value,
+          &variables,
+          svi::Interpolator::DoubleBrackets,
+          false,
+        )
+        .context("failed to interpolate global variables")?;
+        // second pass - core secrets - need replacers.
+        let (res, more_replacers) = svi::interpolate_variables(
+          &res,
+          &core_config.secrets,
+          svi::Interpolator::DoubleBrackets,
+          false,
+        )
+        .context("failed to interpolate core secrets")?;
+        replacers.extend(more_replacers);
+        arg.value = res;
+      }
+
       let res = tokio::select! {
         res = periphery
           .request(api::build::Build {
             build: build.clone(),
             docker_token,
+            replacers: replacers.into_iter().collect(),
           }) => res.context("failed at call to periphery to build"),
         _ = cancel.cancelled() => {
           info!("build cancelled during build, cleaning up builder");

bin/core/src/api/execute/deployment.rs

@@ -1,3 +1,5 @@
+use std::collections::HashSet;
+
 use anyhow::{anyhow, Context};
 use futures::future::join_all;
 use monitor_client::{
@@ -22,7 +24,7 @@ use crate::{
   config::core_config,
   helpers::{
     periphery_client,
-    query::get_server_with_status,
+    query::{get_global_variables, get_server_with_status},
     update::{add_update, make_update, update_update},
   },
   monitor::update_cache_for_server,
@@ -49,6 +51,10 @@ impl Resolve<Deploy, User> for State {
       )
       .await?;
 
+    if deployment.config.server_id.is_empty() {
+      return Err(anyhow!("deployment has no server configured"));
+    }
+
     // get the action state for the deployment (or insert default).
     let action_state = action_states()
       .deployment
@@ -60,10 +66,6 @@ impl Resolve<Deploy, User> for State {
     let _action_guard =
       action_state.update(|state| state.deploying = true)?;
 
-    if deployment.config.server_id.is_empty() {
-      return Err(anyhow!("deployment has no server configured"));
-    }
-
     let (server, status) =
       get_server_with_status(&deployment.config.server_id).await?;
     if status != ServerState::Ok {
@@ -83,9 +85,11 @@ impl Resolve<Deploy, User> for State {
         } else {
           version
         };
+        // replace image with corresponding build image.
         deployment.config.image = DeploymentImage::Image {
           image: format!("{image_name}:{version}"),
         };
+        // set docker account to match build docker account if it's not overridden by deployment
         if deployment.config.docker_account.is_empty() {
           deployment.config.docker_account =
             build.config.docker_account;
@@ -95,6 +99,32 @@ impl Resolve<Deploy, User> for State {
       DeploymentImage::Image { .. } => Version::default(),
     };
 
+    let variables = get_global_variables().await?;
+    let core_config = core_config();
+
+    // Interpolate variables into environment
+    let mut replacers = HashSet::new();
+    for env in &mut deployment.config.environment {
+      // first pass - global variables - don't need replacers.
+      let (res, _) = svi::interpolate_variables(
+        &env.value,
+        &variables,
+        svi::Interpolator::DoubleBrackets,
+        false,
+      )
+      .context("failed to interpolate global variables")?;
+      // second pass - core secrets - need replacers.
+      let (res, more_replacers) = svi::interpolate_variables(
+        &res,
+        &core_config.secrets,
+        svi::Interpolator::DoubleBrackets,
+        false,
+      )
+      .context("failed to interpolate core secrets")?;
+      replacers.extend(more_replacers);
+      env.value = res;
+    }
+
     let mut update =
       make_update(&deployment, Operation::DeployContainer, &user);
     update.in_progress();
@@ -102,7 +132,7 @@ impl Resolve<Deploy, User> for State {
 
     update.id = add_update(update.clone()).await?;
 
-    let docker_token = core_config()
+    let docker_token = core_config
       .docker_accounts
       .get(&deployment.config.docker_account)
       .cloned();
@@ -113,6 +143,7 @@ impl Resolve<Deploy, User> for State {
         stop_signal,
         stop_time,
         docker_token,
+        replacers: replacers.into_iter().collect(),
       })
       .await
     {

bin/core/src/api/read/mod.rs

@@ -28,6 +28,7 @@ mod toml;
 mod update;
 mod user;
 mod user_group;
+mod variable;
 
 #[typeshare]
 #[derive(Serialize, Deserialize, Debug, Clone, Resolver)]
@@ -141,6 +142,10 @@ enum ReadRequest {
   GetSystemStats(GetSystemStats),
   #[to_string_resolver]
   GetSystemProcesses(GetSystemProcesses),
+
+  // ==== VARIABLE ====
+  GetVariable(GetVariable),
+  ListVariables(ListVariables),
 }
 
 pub fn router() -> Router {

bin/core/src/api/read/variable.rs

@@ -0,0 +1,43 @@
+use anyhow::Context;
+use monitor_client::{
+  api::read::{
+    GetVariable, GetVariableResponse, ListVariables,
+    ListVariablesResponse,
+  },
+  entities::user::User,
+};
+use mungos::find::find_collect;
+use resolver_api::Resolve;
+
+use crate::{
+  config::core_config,
+  helpers::query::get_variable,
+  state::{db_client, State},
+};
+
+impl Resolve<GetVariable, User> for State {
+  async fn resolve(
+    &self,
+    GetVariable { name }: GetVariable,
+    _: User,
+  ) -> anyhow::Result<GetVariableResponse> {
+    get_variable(&name).await
+  }
+}
+
+impl Resolve<ListVariables, User> for State {
+  async fn resolve(
+    &self,
+    ListVariables {}: ListVariables,
+    _: User,
+  ) -> anyhow::Result<ListVariablesResponse> {
+    let variables =
+      find_collect(&db_client().await.variables, None, None)
+        .await
+        .context("failed to query db for variables")?;
+    Ok(ListVariablesResponse {
+      variables,
+      secrets: core_config().secrets.keys().cloned().collect(),
+    })
+  }
+}

bin/core/src/api/write/mod.rs

@@ -26,6 +26,7 @@ mod server_template;
 mod tag;
 mod user;
 mod user_group;
+mod variable;
 
 #[typeshare]
 #[derive(Serialize, Deserialize, Debug, Clone, Resolver)]
@@ -116,6 +117,12 @@ enum WriteRequest {
   DeleteTag(DeleteTag),
   RenameTag(RenameTag),
   UpdateTagsOnResource(UpdateTagsOnResource),
+
+  // ==== VARIABLE ====
+  CreateVariable(CreateVariable),
+  UpdateVariableValue(UpdateVariableValue),
+  UpdateVariableDescription(UpdateVariableDescription),
+  DeleteVariable(DeleteVariable),
 }
 
 pub fn router() -> Router {

bin/core/src/api/write/variable.rs

@@ -0,0 +1,165 @@
+use anyhow::{anyhow, Context};
+use monitor_client::{
+  api::write::{
+    CreateVariable, CreateVariableResponse, DeleteVariable,
+    DeleteVariableResponse, UpdateVariableDescription,
+    UpdateVariableDescriptionResponse, UpdateVariableValue,
+    UpdateVariableValueResponse,
+  },
+  entities::{
+    update::ResourceTarget, user::User, variable::Variable, Operation,
+  },
+};
+use mungos::mongodb::bson::doc;
+use resolver_api::Resolve;
+
+use crate::{
+  helpers::{
+    query::get_variable,
+    update::{add_update, make_update},
+  },
+  state::{db_client, State},
+};
+
+impl Resolve<CreateVariable, User> for State {
+  async fn resolve(
+    &self,
+    CreateVariable {
+      name,
+      value,
+      description,
+    }: CreateVariable,
+    user: User,
+  ) -> anyhow::Result<CreateVariableResponse> {
+    if !user.admin {
+      return Err(anyhow!("only admins can create variables"));
+    }
+
+    let variable = Variable {
+      name,
+      value,
+      description,
+    };
+
+    db_client()
+      .await
+      .variables
+      .insert_one(&variable, None)
+      .await
+      .context("failed to create variable on db")?;
+
+    let mut update = make_update(
+      ResourceTarget::system(),
+      Operation::CreateVariable,
+      &user,
+    );
+
+    update
+      .push_simple_log("create variable", format!("{variable:#?}"));
+    update.finalize();
+
+    add_update(update).await?;
+
+    get_variable(&variable.name).await
+  }
+}
+
+impl Resolve<UpdateVariableValue, User> for State {
+  async fn resolve(
+    &self,
+    UpdateVariableValue { name, value }: UpdateVariableValue,
+    user: User,
+  ) -> anyhow::Result<UpdateVariableValueResponse> {
+    if !user.admin {
+      return Err(anyhow!("only admins can create variables"));
+    }
+
+    let variable = get_variable(&name).await?;
+
+    db_client()
+      .await
+      .variables
+      .update_one(
+        doc! { "name": &name },
+        doc! { "$set": { "value": &value } },
+        None,
+      )
+      .await
+      .context("failed to update variable value on db")?;
+
+    let mut update = make_update(
+      ResourceTarget::system(),
+      Operation::UpdateVariableValue,
+      &user,
+    );
+
+    update.push_simple_log(
+      "update variable value",
+      format!(
+        "<span class=\"text-muted-foreground\">variable</span>: '{name}'\n<span class=\"text-muted-foreground\">from</span>: <span class=\"text-red-500\">{}</span>\n<span class=\"text-muted-foreground\">to</span>:   <span class=\"text-green-500\">{value}</span>",
+        variable.value
+      ),
+    );
+    update.finalize();
+
+    add_update(update).await?;
+
+    get_variable(&name).await
+  }
+}
+
+impl Resolve<UpdateVariableDescription, User> for State {
+  async fn resolve(
+    &self,
+    UpdateVariableDescription { name, description }: UpdateVariableDescription,
+    user: User,
+  ) -> anyhow::Result<UpdateVariableDescriptionResponse> {
+    if !user.admin {
+      return Err(anyhow!("only admins can create variables"));
+    }
+    db_client()
+      .await
+      .variables
+      .update_one(
+        doc! { "name": &name },
+        doc! { "$set": { "description": &description } },
+        None,
+      )
+      .await
+      .context("failed to update variable description on db")?;
+    get_variable(&name).await
+  }
+}
+
+impl Resolve<DeleteVariable, User> for State {
+  async fn resolve(
+    &self,
+    DeleteVariable { name }: DeleteVariable,
+    user: User,
+  ) -> anyhow::Result<DeleteVariableResponse> {
+    if !user.admin {
+      return Err(anyhow!("only admins can create variables"));
+    }
+    let variable = get_variable(&name).await?;
+    db_client()
+      .await
+      .variables
+      .delete_one(doc! { "name": &name }, None)
+      .await
+      .context("failed to delete variable on db")?;
+
+    let mut update = make_update(
+      ResourceTarget::system(),
+      Operation::DeleteVariable,
+      &user,
+    );
+
+    update
+      .push_simple_log("delete variable", format!("{variable:#?}"));
+    update.finalize();
+
+    add_update(update).await?;
+
+    Ok(variable)
+  }
+}

bin/core/src/db.rs

@@ -16,6 +16,7 @@ use monitor_client::entities::{
   update::Update,
   user::User,
   user_group::UserGroup,
+  variable::Variable,
 };
 use mungos::{
   init::MongoBuilder,
@@ -28,6 +29,7 @@ pub struct DbClient {
   pub permissions: Collection<Permission>,
   pub api_keys: Collection<ApiKey>,
   pub tags: Collection<Tag>,
+  pub variables: Collection<Variable>,
   pub updates: Collection<Update>,
   pub alerts: Collection<Alert>,
   pub stats: Collection<SystemStatsRecord>,
@@ -80,20 +82,16 @@ impl DbClient {
     let db = client.database(db_name);
 
     let client = DbClient {
-      users: mongo_indexed::collection::<User>(&db, true).await?,
-      user_groups: mongo_indexed::collection::<UserGroup>(&db, true)
-        .await?,
-      permissions: mongo_indexed::collection::<Permission>(&db, true)
-        .await?,
-      api_keys: mongo_indexed::collection::<ApiKey>(&db, true)
-        .await?,
-      tags: mongo_indexed::collection::<Tag>(&db, true).await?,
-      updates: mongo_indexed::collection::<Update>(&db, true).await?,
-      alerts: mongo_indexed::collection::<Alert>(&db, true).await?,
-      stats: mongo_indexed::collection::<SystemStatsRecord>(
-        &db, true,
-      )
-      .await?,
+      users: mongo_indexed::collection(&db, true).await?,
+      user_groups: mongo_indexed::collection(&db, true).await?,
+      permissions: mongo_indexed::collection(&db, true).await?,
+      api_keys: mongo_indexed::collection(&db, true).await?,
+      tags: mongo_indexed::collection(&db, true).await?,
+      variables: mongo_indexed::collection(&db, true).await?,
+      updates: mongo_indexed::collection(&db, true).await?,
+      alerts: mongo_indexed::collection(&db, true).await?,
+      stats: mongo_indexed::collection(&db, true).await?,
+      // RESOURCES
       servers: resource_collection(&db, "Server").await?,
       deployments: resource_collection(&db, "Deployment").await?,
       builds: resource_collection(&db, "Build").await?,
@@ -103,6 +101,7 @@ impl DbClient {
       procedures: resource_collection(&db, "Procedure").await?,
       server_templates: resource_collection(&db, "ServerTemplate")
         .await?,
+      //
       db,
     };
     Ok(client)

bin/core/src/helpers/query.rs

@@ -1,4 +1,7 @@
-use std::{collections::HashSet, str::FromStr};
+use std::{
+  collections::{HashMap, HashSet},
+  str::FromStr,
+};
 
 use anyhow::{anyhow, Context};
 use monitor_client::entities::{
@@ -8,6 +11,7 @@ use monitor_client::entities::{
   tag::Tag,
   update::ResourceTargetVariant,
   user::{admin_service_user, User},
+  variable::Variable,
 };
 use mungos::{
   by_id::find_one_by_id,
@@ -198,3 +202,27 @@ pub fn id_or_name_filter(id_or_name: &str) -> Document {
     Err(_) => doc! { "name": id_or_name },
   }
 }
+
+pub async fn get_global_variables(
+) -> anyhow::Result<HashMap<String, String>> {
+  Ok(
+    find_collect(&db_client().await.variables, None, None)
+      .await
+      .context("failed to get all variables from db")?
+      .into_iter()
+      .map(|variable| (variable.name, variable.value))
+      .collect(),
+  )
+}
+
+pub async fn get_variable(name: &str) -> anyhow::Result<Variable> {
+  db_client()
+    .await
+    .variables
+    .find_one(doc! { "name": &name }, None)
+    .await
+    .context("failed at call to db")?
+    .with_context(|| {
+      format!("no variable found with given name: {name}")
+    })
+}

bin/periphery/src/api/build.rs

@@ -12,16 +12,17 @@ use crate::{
 };
 
 impl Resolve<Build> for State {
-  #[instrument(name = "Build", skip(self))]
+  #[instrument(name = "Build", skip(self, replacers))]
   async fn resolve(
     &self,
     Build {
       build,
       docker_token,
+      replacers,
     }: Build,
     _: (),
   ) -> anyhow::Result<Vec<Log>> {
-    docker::build::build(&build, docker_token).await
+    docker::build::build(&build, docker_token, replacers).await
   }
 }
 

bin/periphery/src/api/container.rs

@@ -187,7 +187,7 @@ impl Resolve<PruneContainers> for State {
 //
 
 impl Resolve<Deploy> for State {
-  #[instrument(name = "Deploy", skip(self))]
+  #[instrument(name = "Deploy", skip(self, replacers))]
   async fn resolve(
     &self,
     Deploy {
@@ -195,6 +195,7 @@ impl Resolve<Deploy> for State {
       docker_token,
       stop_signal,
       stop_time,
+      replacers,
     }: Deploy,
     _: (),
   ) -> anyhow::Result<Log> {
@@ -207,6 +208,7 @@ impl Resolve<Deploy> for State {
         .unwrap_or(deployment.config.termination_timeout)
         .into(),
       docker_token,
+      replacers,
     )
     .await;
     Ok(res)

bin/periphery/src/helpers/docker/build.rs

@@ -20,7 +20,7 @@ pub async fn prune_images() -> Log {
   run_monitor_command("prune images", command).await
 }
 
-#[instrument(skip(docker_token))]
+#[instrument(skip(docker_token, core_replacers))]
 pub async fn build(
   Build {
     name,
@@ -41,6 +41,7 @@ pub async fn build(
     ..
   }: &Build,
   docker_token: Option<String>,
+  core_replacers: Vec<(String, String)>,
 ) -> anyhow::Result<Vec<Log>> {
   let mut logs = Vec::new();
   let docker_token = match (
@@ -90,14 +91,17 @@ pub async fn build(
     info!("finished building docker image");
     logs.push(build_log);
   } else {
-    let (command, replacers) = svi::interpolate_variables(
+    let (command, mut replacers) = svi::interpolate_variables(
       &command,
       &periphery_config().secrets,
       svi::Interpolator::DoubleBrackets,
+      true,
     )
     .context(
       "failed to interpolate secrets into docker build command",
     )?;
+    replacers.extend(core_replacers);
+
     let mut build_log =
       run_monitor_command("docker build", command).await;
     build_log.command =

bin/periphery/src/helpers/docker/container.rs

@@ -178,12 +178,13 @@ async fn pull_image(image: &str) -> Log {
   run_monitor_command("docker pull", command).await
 }
 
-#[instrument(skip(docker_token))]
+#[instrument(skip(docker_token, core_replacers))]
 pub async fn deploy(
   deployment: &Deployment,
   stop_signal: Option<TerminationSignal>,
   stop_time: Option<i32>,
   docker_token: Option<String>,
+  core_replacers: Vec<(String, String)>,
 ) -> Log {
   let docker_token = match (
     docker_token,
@@ -244,12 +245,14 @@ pub async fn deploy(
       &command,
       &periphery_config().secrets,
       svi::Interpolator::DoubleBrackets,
+      true,
     )
     .context("failed to interpolate secrets into docker run command");
     if let Err(e) = command {
       return Log::error("docker run", format!("{e:?}"));
     }
-    let (command, replacers) = command.unwrap();
+    let (command, mut replacers) = command.unwrap();
+    replacers.extend(core_replacers);
     let mut log = run_monitor_command("docker run", command).await;
     log.command = svi::replace_in_string(&log.command, &replacers);
     log.stdout = svi::replace_in_string(&log.stdout, &replacers);

client/core/rs/src/api/read/mod.rs

@@ -19,6 +19,7 @@ mod toml;
 mod update;
 mod user;
 mod user_group;
+mod variable;
 
 pub use alert::*;
 pub use alerter::*;
@@ -36,6 +37,7 @@ pub use toml::*;
 pub use update::*;
 pub use user::*;
 pub use user_group::*;
+pub use variable::*;
 
 use crate::entities::Timelength;
 

client/core/rs/src/api/read/variable.rs

@@ -0,0 +1,46 @@
+use derive_empty_traits::EmptyTraits;
+use resolver_api::derive::Request;
+use serde::{Deserialize, Serialize};
+use typeshare::typeshare;
+
+use crate::entities::variable::Variable;
+
+use super::MonitorReadRequest;
+
+/// List all available global variables.
+/// Response: [Variable]
+#[typeshare]
+#[derive(
+  Serialize, Deserialize, Debug, Clone, Request, EmptyTraits,
+)]
+#[empty_traits(MonitorReadRequest)]
+#[response(GetVariableResponse)]
+pub struct GetVariable {
+  /// The name of the variable to get.
+  pub name: String,
+}
+
+#[typeshare]
+pub type GetVariableResponse = Variable;
+
+//
+
+/// List all available global variables.
+/// Response: [ListVariablesResponse]
+#[typeshare]
+#[derive(
+  Serialize, Deserialize, Debug, Clone, Request, EmptyTraits,
+)]
+#[empty_traits(MonitorReadRequest)]
+#[response(ListVariablesResponse)]
+pub struct ListVariables {}
+
+/// The response of [ListVariables].
+#[typeshare]
+#[derive(Serialize, Deserialize, Debug, Clone)]
+pub struct ListVariablesResponse {
+  /// The available global variables.
+  pub variables: Vec<Variable>,
+  /// The available global secret keys
+  pub secrets: Vec<String>,
+}

client/core/rs/src/api/write/mod.rs

@@ -12,6 +12,7 @@ mod server_template;
 mod tags;
 mod user;
 mod user_group;
+mod variable;
 
 pub use alerter::*;
 pub use api_key::*;
@@ -27,5 +28,6 @@ pub use server_template::*;
 pub use tags::*;
 pub use user::*;
 pub use user_group::*;
+pub use variable::*;
 
 pub trait MonitorWriteRequest: resolver_api::HasResponse {}

client/core/rs/src/api/write/variable.rs

@@ -0,0 +1,83 @@
+use derive_empty_traits::EmptyTraits;
+use resolver_api::derive::Request;
+use serde::{Deserialize, Serialize};
+use typeshare::typeshare;
+
+use crate::entities::variable::Variable;
+
+use super::MonitorWriteRequest;
+
+/// **Admin only.** Create variable. Response: [Variable].
+#[typeshare]
+#[derive(
+  Debug, Clone, Serialize, Deserialize, Request, EmptyTraits,
+)]
+#[empty_traits(MonitorWriteRequest)]
+#[response(CreateVariableResponse)]
+pub struct CreateVariable {
+  /// The name of the variable to create.
+  pub name: String,
+  /// The initial value of the variable. defualt: "".
+  #[serde(default)]
+  pub value: String,
+  /// The initial value of the description. default: "".
+  #[serde(default)]
+  pub description: String,
+}
+
+#[typeshare]
+pub type CreateVariableResponse = Variable;
+
+//
+
+/// **Admin only.** Update variable. Response: [Variable].
+#[typeshare]
+#[derive(
+  Debug, Clone, Serialize, Deserialize, Request, EmptyTraits,
+)]
+#[empty_traits(MonitorWriteRequest)]
+#[response(UpdateVariableValueResponse)]
+pub struct UpdateVariableValue {
+  /// The name of the variable to update.
+  pub name: String,
+  /// The value to set.
+  pub value: String,
+}
+
+#[typeshare]
+pub type UpdateVariableValueResponse = Variable;
+
+//
+
+/// **Admin only.** Update variable. Response: [Variable].
+#[typeshare]
+#[derive(
+  Debug, Clone, Serialize, Deserialize, Request, EmptyTraits,
+)]
+#[empty_traits(MonitorWriteRequest)]
+#[response(UpdateVariableDescriptionResponse)]
+pub struct UpdateVariableDescription {
+  /// The name of the variable to update.
+  pub name: String,
+  /// The description to set.
+  pub description: String,
+}
+
+#[typeshare]
+pub type UpdateVariableDescriptionResponse = Variable;
+
+//
+
+/// **Admin only.** Delete a variable. Response: [Variable].
+#[typeshare]
+#[derive(
+  Debug, Clone, Serialize, Deserialize, Request, EmptyTraits,
+)]
+#[empty_traits(MonitorWriteRequest)]
+#[response(DeleteVariableResponse)]
+pub struct DeleteVariable {
+  pub name: String,
+}
+
+#[typeshare]
+pub type DeleteVariableResponse = Variable;

client/core/rs/src/entities/config/core.rs

@@ -1,13 +1,13 @@
 //! # Configuring the Core API
-//! 
+//!
 //! Monitor core is configured by parsing base configuration file ([CoreConfig]), and overriding
 //! any fields given in the file with ones provided on the environment ([Env]).
-//! 
+//!
 //! The recommended method for running monitor core is via the docker image. This image has a default
 //! configuration file provided in the image, meaning any custom configuration can be provided
 //! on the environment alone. However, if a custom configuration file is prefered, it can be mounted
 //! into the image at `/config/config.toml`.
-//! 
+//!
 
 use std::collections::HashMap;
 
@@ -108,25 +108,25 @@ fn default_config_path() -> String {
 }
 
 /// # Core Configuration File
-/// 
+///
 /// The Core API initializes it's configuration by reading the environment,
 /// parsing the [CoreConfig] schema from the file path specified by `env.monitor_config_path`,
 /// and then applying any config field overrides specified in the environment.
-/// 
+///
 /// *Note.* The monitor core docker image includes the default core configuration found below.
 /// To configure the core api, you can either mount your own custom configuration file
 /// to `/config/config.toml` inside the container, or simply override whichever fields
 /// you need using the environment.
-/// 
+///
 /// ## Example TOML
 /// ```toml
 /// ## this will be the document title on the web page (shows up as text in the browser tab).
 /// ## default: 'Monitor'
 /// title = "Monitor"
 ///
-/// ## required for oauth functionality. this should be the url used to access monitor in browser, 
+/// ## required for oauth functionality. this should be the url used to access monitor in browser,
 /// ## potentially behind DNS.
-/// ## eg https://monitor.dev or http://12.34.56.78:9000. 
+/// ## eg https://monitor.dev or http://12.34.56.78:9000.
 /// ## this should match the address configured in your oauth app.
 /// ## no default
 /// host = "https://monitor.dev"
@@ -157,7 +157,7 @@ fn default_config_path() -> String {
 ///
 /// ## specify how long an issued jwt stays valid.
 /// ## all jwts are invalidated on application restart.
-/// ## default: 1-day. 
+/// ## default: 1-day.
 /// ## options: 1-hr, 12-hr, 1-day, 3-day, 1-wk, 2-wk, 30-day
 /// jwt_valid_for = "1-day"
 ///
@@ -166,7 +166,7 @@ fn default_config_path() -> String {
 /// ## options: 5-sec, 15-sec, 30-sec, 1-min, 2-min, 5-min, 15-min
 /// monitoring_interval = "15-sec"
 ///
-/// ## number of days to keep stats around, or 0 to disable pruning. 
+/// ## number of days to keep stats around, or 0 to disable pruning.
 /// ## stats older than this number of days are deleted daily
 /// ## default: 0 (pruning disabled)
 /// keep_stats_for_days = 0
@@ -215,12 +215,17 @@ fn default_config_path() -> String {
 /// ## provide aws api keys for ephemeral builders
 /// # aws.access_key_id = "your_aws_key_id"
 /// # aws.secret_access_key = "your_aws_secret_key"
-/// 
+///
+/// ## provide core-base secrets
+/// [secrets]
+/// # SECRET_1 = "value_1"
+/// # SECRET_2 = "value_2"
+///
 /// ## provide core-based github accounts
 /// [github_accounts]
 /// # github_username_1 = "github_token_1"
 /// # github_username_2 = "github_token_2"
-/// 
+///
 /// ## provide core-based docker accounts
 /// [docker_accounts]
 /// # docker_username_1 = "docker_token_1"
@@ -312,7 +317,13 @@ pub struct CoreConfig {
   /// Configure AWS credentials to use with AWS builds / server launches.
   #[serde(default)]
   pub aws: AwsCredentials,
-  
+
+  /// Configure core-based secrets. These will be preferentially interpolated into
+  /// values if they contain a matching secret. Otherwise, the periphery will have to have the
+  /// secret configured.
+  #[serde(default)]
+  pub secrets: HashMap<String, String>,
+
   /// Configure core-based github accounts. These will be preferentially attached to build / repo clone
   /// requests if they contain a matching github account. Otherwise, the periphery will have to have the
   /// account configured.

client/core/rs/src/entities/mod.rs

@@ -45,6 +45,7 @@ pub mod update;
 pub mod user;
 /// Subtypes of [UserGroup][user_group::UserGroup].
 pub mod user_group;
+pub mod variable;
 
 #[typeshare(serialized_as = "number")]
 pub type I64 = i64;
@@ -433,6 +434,11 @@ pub enum Operation {
   UpdateServerTemplate,
   DeleteServerTemplate,
   LaunchServer,
+
+  // variable
+  CreateVariable,
+  UpdateVariableValue,
+  DeleteVariable,
 }
 
 #[typeshare]

client/core/rs/src/entities/variable.rs

@@ -0,0 +1,21 @@
+use serde::{Deserialize, Serialize};
+use typeshare::typeshare;
+
+/// A non-secret global variable which can be interpolated into deployment
+/// environment variable values and build argument values.
+#[typeshare]
+#[derive(Serialize, Deserialize, Debug, Clone, Default)]
+#[cfg_attr(
+  feature = "mongo",
+  derive(mongo_indexed::derive::MongoIndexed)
+)]
+pub struct Variable {
+  /// Unique name associated with the variable.
+  /// Instances of '[[variable.name]]' in value will be replaced with 'variable.value'.
+  #[cfg_attr(feature = "mongo", unique_index)]
+  pub name: String,
+  /// The value associated with the variable.
+  pub value: String,
+  /// A description for the variable.
+  pub description: String,
+}

client/core/ts/src/responses.ts

@@ -111,6 +111,10 @@ export type ReadResponses = {
   // ==== SERVER STATS ====
   GetSystemStats: Types.GetSystemStatsResponse;
   GetSystemProcesses: Types.GetSystemProcessesResponse;
+
+  // ==== VARIABLE ====
+  GetVariable: Types.GetVariableResponse;
+  ListVariables: Types.ListVariablesResponse;
 };
 
 export type WriteResponses = {
@@ -198,6 +202,12 @@ export type WriteResponses = {
   DeleteTag: Types.Tag;
   RenameTag: Types.Tag;
   UpdateTagsOnResource: Types.UpdateTagsOnResourceResponse;
+
+  // ==== VARIABLE ====
+  CreateVariable: Types.CreateVariableResponse;
+  UpdateVariableValue: Types.UpdateVariableValueResponse;
+  UpdateVariableDescription: Types.UpdateVariableDescriptionResponse;
+  DeleteVariable: Types.DeleteVariableResponse;
 };
 
 export type ExecuteResponses = {

client/core/ts/src/types.ts

@@ -1130,6 +1130,7 @@ export enum Operation {
 	PruneNetworksServer = "PruneNetworksServer",
 	CreateNetwork = "CreateNetwork",
 	DeleteNetwork = "DeleteNetwork",
+	StopAllContainers = "StopAllContainers",
 	CreateBuild = "CreateBuild",
 	UpdateBuild = "UpdateBuild",
 	DeleteBuild = "DeleteBuild",
@@ -1143,7 +1144,6 @@ export enum Operation {
 	DeleteDeployment = "DeleteDeployment",
 	DeployContainer = "DeployContainer",
 	StopContainer = "StopContainer",
-	StopAllContainers = "StopAllContainers",
 	StartContainer = "StartContainer",
 	RemoveContainer = "RemoveContainer",
 	RenameDeployment = "RenameDeployment",
@@ -1163,6 +1163,9 @@ export enum Operation {
 	UpdateServerTemplate = "UpdateServerTemplate",
 	DeleteServerTemplate = "DeleteServerTemplate",
 	LaunchServer = "LaunchServer",
+	CreateVariable = "CreateVariable",
+	UpdateVariableValue = "UpdateVariableValue",
+	DeleteVariable = "DeleteVariable",
 }
 
 export enum UpdateStatus {
@@ -1230,6 +1233,24 @@ export type GetUserGroupResponse = UserGroup;
 
 export type ListUserGroupsResponse = UserGroup[];
 
+/**
+ * A non-secret global variable which can be interpolated into deployment
+ * environment variable values and build argument values.
+ */
+export interface Variable {
+	/**
+	 * Unique name associated with the variable.
+	 * Instances of '[[variable.name]]' in value will be replaced with 'variable.value'.
+	 */
+	name: string;
+	/** The value associated with the variable. */
+	value: string;
+	/** A description for the variable. */
+	description: string;
+}
+
+export type GetVariableResponse = Variable;
+
 export type DeleteApiKeyResponse = NoData;
 
 /** Response for [CreateApiKey]. */
@@ -1273,6 +1294,14 @@ export type CreateServiceUserResponse = User;
 
 export type UpdateServiceUserDescriptionResponse = User;
 
+export type CreateVariableResponse = Variable;
+
+export type UpdateVariableValueResponse = Variable;
+
+export type UpdateVariableDescriptionResponse = Variable;
+
+export type DeleteVariableResponse = Variable;
+
 export type _PartialCustomAlerterConfig = Partial<CustomAlerterConfig>;
 
 export type _PartialSlackAlerterConfig = Partial<SlackAlerterConfig>;
@@ -1754,15 +1783,13 @@ export interface GetBuildMonthlyStatsResponse {
 }
 
 /**
- * Paginated endpoint for versions of the build that were built in the past and available for deployment,
+ * Retrieve versions of the build that were built in the past and available for deployment,
  * sorted by most recent first.
  * Response: [GetBuildVersionsResponse].
  */
 export interface GetBuildVersions {
 	/** Id or name */
 	build: string;
-	/** The page of data. Default is 0, which corrensponds to the most recent versions. */
-	page?: number;
 	/** Filter to only include versions matching this major version. */
 	major?: number;
 	/** Filter to only include versions matching this minor version. */
@@ -2462,6 +2489,30 @@ export interface GetUserGroup {
 export interface ListUserGroups {
 }
 
+/**
+ * List all available global variables.
+ * Response: [Variable]
+ */
+export interface GetVariable {
+	/** The name of the variable to get. */
+	name: string;
+}
+
+/**
+ * List all available global variables.
+ * Response: [ListVariablesResponse]
+ */
+export interface ListVariables {
+}
+
+/** The response of [ListVariables]. */
+export interface ListVariablesResponse {
+	/** The available global variables. */
+	variables: Variable[];
+	/** The available global secret keys */
+	secrets: string[];
+}
+
 export type PartialAlerterConfig = 
 	| { type: "Custom", params: _PartialCustomAlerterConfig }
 	| { type: "Slack", params: _PartialSlackAlerterConfig };
@@ -3076,6 +3127,37 @@ export interface SetUsersInUserGroup {
 	users: string[];
 }
 
+/** **Admin only.** Create variable. Response: [Variable]. */
+export interface CreateVariable {
+	/** The name of the variable to create. */
+	name: string;
+	/** The initial value of the variable. defualt: "". */
+	value?: string;
+	/** The initial value of the description. default: "". */
+	description?: string;
+}
+
+/** **Admin only.** Update variable. Response: [Variable]. */
+export interface UpdateVariableValue {
+	/** The name of the variable to update. */
+	name: string;
+	/** The value to set. */
+	value: string;
+}
+
+/** **Admin only.** Update variable. Response: [Variable]. */
+export interface UpdateVariableDescription {
+	/** The name of the variable to update. */
+	name: string;
+	/** The description to set. */
+	description: string;
+}
+
+/** **Admin only.** Delete a variable. Response: [Variable]. */
+export interface DeleteVariable {
+	name: string;
+}
+
 /** Configuration for a custom alerter. */
 export interface CustomAlerterConfig {
 	/** The http/s endpoint to send the POST to */
@@ -3319,7 +3401,9 @@ export type ReadRequest =
 	| { type: "GetAlert", params: GetAlert }
 	| { type: "GetSystemInformation", params: GetSystemInformation }
 	| { type: "GetSystemStats", params: GetSystemStats }
-	| { type: "GetSystemProcesses", params: GetSystemProcesses };
+	| { type: "GetSystemProcesses", params: GetSystemProcesses }
+	| { type: "GetVariable", params: GetVariable }
+	| { type: "ListVariables", params: ListVariables };
 
 export type WriteRequest = 
 	| { type: "CreateApiKey", params: CreateApiKey }
@@ -3377,7 +3461,11 @@ export type WriteRequest =
 	| { type: "CreateTag", params: CreateTag }
 	| { type: "DeleteTag", params: DeleteTag }
 	| { type: "RenameTag", params: RenameTag }
-	| { type: "UpdateTagsOnResource", params: UpdateTagsOnResource };
+	| { type: "UpdateTagsOnResource", params: UpdateTagsOnResource }
+	| { type: "CreateVariable", params: CreateVariable }
+	| { type: "UpdateVariableValue", params: UpdateVariableValue }
+	| { type: "UpdateVariableDescription", params: UpdateVariableDescription }
+	| { type: "DeleteVariable", params: DeleteVariable };
 
 export type WsLoginMessage = 
 	| { type: "Jwt", params: {

client/periphery/rs/src/api/build.rs

@@ -10,6 +10,8 @@ pub struct Build {
   pub build: monitor_client::entities::build::Build,
   /// Override docker token with one sent from core.
   pub docker_token: Option<String>,
+  /// Propogate any secret replacers from core interpolation.
+  pub replacers: Vec<(String, String)>,
 }
 
 pub type BuildResponse = Vec<Log>;

client/periphery/rs/src/api/container.rs

@@ -105,4 +105,6 @@ pub struct Deploy {
   pub docker_token: Option<String>,
   pub stop_signal: Option<TerminationSignal>,
   pub stop_time: Option<i32>,
+  /// Propogate any secret replacers from core interpolation.
+  pub replacers: Vec<(String, String)>,
 }

config_example/core.config.example.toml

@@ -90,6 +90,11 @@ mongo.address = "localhost:27017"
 # aws.access_key_id = "your_aws_key_id"
 # aws.secret_access_key = "your_aws_secret_key"
 
+## provide core-base secrets
+# [secrets]
+# SECRET_1 = "value_1"
+# SECRET_2 = "value_2"
+
 ## provide core-based github accounts
 # [github_accounts]
 # github_username_1 = "github_token_1"