From a288edcf61043b1fa9de6a6a08b5fca231499967 Mon Sep 17 00:00:00 2001 From: mbecker20 Date: Sat, 11 Mar 2023 23:34:17 +0000 Subject: [PATCH] 0.2.8 implement secret interpolation on builds and deployments --- Cargo.lock | 44 ++++++++++++++--------- cli/Cargo.toml | 2 +- core/Cargo.toml | 2 +- lib/db_client/Cargo.toml | 2 +- lib/helpers/Cargo.toml | 2 +- lib/monitor_client/Cargo.toml | 4 +-- lib/periphery_client/Cargo.toml | 2 +- lib/types/Cargo.toml | 2 +- lib/types/src/build.rs | 5 +++ lib/types/src/deployment.rs | 5 +++ periphery/Cargo.toml | 7 ++-- periphery/src/api/build.rs | 9 ++++- periphery/src/api/container.rs | 8 ++++- periphery/src/helpers/docker/build.rs | 19 ++++++++-- periphery/src/helpers/docker/container.rs | 20 +++++++++-- periphery/src/main.rs | 13 ++++--- 16 files changed, 105 insertions(+), 41 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0db053dea..bb3d6a46f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -734,7 +734,7 @@ dependencies = [ [[package]] name = "core" -version = "0.2.7" +version = "0.2.8" dependencies = [ "anyhow", "async_timing_util", @@ -753,7 +753,7 @@ dependencies = [ "hmac", "jwt", "monitor_helpers", - "monitor_types 0.2.7", + "monitor_types 0.2.8", "mungos", "periphery_client", "serde", @@ -987,10 +987,10 @@ checksum = "23d8666cb01533c39dde32bcbab8e227b4ed6679b2c925eba05feabea39508fb" [[package]] name = "db_client" -version = "0.2.7" +version = "0.2.8" dependencies = [ "anyhow", - "monitor_types 0.2.7", + "monitor_types 0.2.8", "mungos", ] @@ -1837,12 +1837,12 @@ dependencies = [ [[package]] name = "monitor_cli" -version = "0.2.7" +version = "0.2.8" dependencies = [ "async_timing_util", "clap", "colored", - "monitor_types 0.2.7", + "monitor_types 0.2.8", "rand", "run_command", "serde", @@ -1854,12 +1854,12 @@ dependencies = [ [[package]] name = "monitor_client" -version = "0.2.7" +version = "0.2.8" dependencies = [ "anyhow", "envy", "futures-util", - "monitor_types 0.2.7 (registry+https://github.com/rust-lang/crates.io-index)", + "monitor_types 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", "reqwest", "serde", "serde_derive", @@ -1871,11 +1871,11 @@ dependencies = [ [[package]] name = "monitor_helpers" -version = "0.2.7" +version = "0.2.8" dependencies = [ "anyhow", "axum", - "monitor_types 0.2.7", + "monitor_types 0.2.8", "rand", "serde", "serde_json", @@ -1884,7 +1884,7 @@ dependencies = [ [[package]] name = "monitor_periphery" -version = "0.2.7" +version = "0.2.8" dependencies = [ "anyhow", "async_timing_util", @@ -1896,11 +1896,12 @@ dependencies = [ "envy", "futures", "monitor_helpers", - "monitor_types 0.2.7", + "monitor_types 0.2.8", "run_command", "serde", "serde_derive", "serde_json", + "svi", "sysinfo", "tokio", "toml", @@ -1909,7 +1910,7 @@ dependencies = [ [[package]] name = "monitor_types" -version = "0.2.7" +version = "0.2.8" dependencies = [ "anyhow", "bollard", @@ -1926,9 +1927,9 @@ dependencies = [ [[package]] name = "monitor_types" -version = "0.2.7" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1c280239929526ffd057372240260b6a78e7f62bbbc061218a46f607f176f3e" +checksum = "66b027ddf58aa5db73f63e4d2fd4426c41dd1db9b48e88aef6ba24731aa75874" dependencies = [ "anyhow", "bollard", @@ -2184,11 +2185,11 @@ checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" [[package]] name = "periphery_client" -version = "0.2.7" +version = "0.2.8" dependencies = [ "anyhow", "futures-util", - "monitor_types 0.2.7", + "monitor_types 0.2.8", "reqwest", "serde", "serde_json", @@ -2862,6 +2863,15 @@ version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" +[[package]] +name = "svi" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec1ee5e6cf961310f3b4ba037f6a3680fc264f9077e0b9f16a0d7cc8d0ade140" +dependencies = [ + "thiserror", +] + [[package]] name = "syn" version = "1.0.109" diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 4e7f15a71..1b32641ea 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "monitor_cli" -version = "0.2.7" +version = "0.2.8" edition = "2021" authors = ["MoghTech"] description = "monitor cli | tools to setup monitor system" diff --git a/core/Cargo.toml b/core/Cargo.toml index affa8f883..9668a3d00 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "core" -version = "0.2.7" +version = "0.2.8" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/lib/db_client/Cargo.toml b/lib/db_client/Cargo.toml index b8d8323cc..06c4d13c3 100644 --- a/lib/db_client/Cargo.toml +++ b/lib/db_client/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "db_client" -version = "0.2.7" +version = "0.2.8" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/lib/helpers/Cargo.toml b/lib/helpers/Cargo.toml index 5334c0492..f17c8b609 100644 --- a/lib/helpers/Cargo.toml +++ b/lib/helpers/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "monitor_helpers" -version = "0.2.7" +version = "0.2.8" edition = "2021" authors = ["MoghTech"] description = "helpers used as dependency for mogh tech monitor" diff --git a/lib/monitor_client/Cargo.toml b/lib/monitor_client/Cargo.toml index a1ce4ccaa..a15f99f3f 100644 --- a/lib/monitor_client/Cargo.toml +++ b/lib/monitor_client/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "monitor_client" -version = "0.2.7" +version = "0.2.8" edition = "2021" authors = ["MoghTech"] description = "a client to interact with the monitor system" @@ -9,7 +9,7 @@ license = "GPL-3.0-or-later" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -monitor_types = "0.2.7" +monitor_types = "0.2.8" # monitor_types = { path = "../types" } reqwest = { version = "0.11", features = ["json"] } tokio-tungstenite = { version = "0.18", features=["native-tls"] } diff --git a/lib/periphery_client/Cargo.toml b/lib/periphery_client/Cargo.toml index 495193b65..9f7e64a38 100644 --- a/lib/periphery_client/Cargo.toml +++ b/lib/periphery_client/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "periphery_client" -version = "0.2.7" +version = "0.2.8" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/lib/types/Cargo.toml b/lib/types/Cargo.toml index 13273d312..6dfaab76c 100644 --- a/lib/types/Cargo.toml +++ b/lib/types/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "monitor_types" -version = "0.2.7" +version = "0.2.8" edition = "2021" authors = ["MoghTech"] description = "types for the mogh tech monitor" diff --git a/lib/types/src/build.rs b/lib/types/src/build.rs index a5d0d8c91..585560ae2 100644 --- a/lib/types/src/build.rs +++ b/lib/types/src/build.rs @@ -37,6 +37,11 @@ pub struct Build { #[builder(setter(skip))] pub permissions: PermissionsMap, + #[serde(default)] + #[builder(default)] + #[diff(attr(#[serde(skip_serializing_if = "Option::is_none")]))] + pub skip_secret_interp: bool, + #[builder(default)] #[diff(attr(#[serde(skip_serializing_if = "option_diff_no_change")]))] pub server_id: Option, // server which this image should be built on diff --git a/lib/types/src/deployment.rs b/lib/types/src/deployment.rs index ed130138e..455d51340 100644 --- a/lib/types/src/deployment.rs +++ b/lib/types/src/deployment.rs @@ -37,6 +37,11 @@ pub struct Deployment { #[builder(setter(skip))] pub permissions: PermissionsMap, + #[serde(default)] + #[builder(default)] + #[diff(attr(#[serde(skip_serializing_if = "Option::is_none")]))] + pub skip_secret_interp: bool, + #[builder(default)] #[diff(attr(#[serde(skip_serializing_if = "docker_run_args_diff_no_change")]))] pub docker_run_args: DockerRunArgs, diff --git a/periphery/Cargo.toml b/periphery/Cargo.toml index 18cad1898..d6e07c365 100644 --- a/periphery/Cargo.toml +++ b/periphery/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "monitor_periphery" -version = "0.2.7" +version = "0.2.8" edition = "2021" authors = ["MoghTech"] description = "monitor periphery binary | run monitor periphery as system daemon" @@ -17,12 +17,10 @@ helpers = { package = "monitor_helpers", path = "../lib/helpers" } types = { package = "monitor_types", path = "../lib/types" } run_command = { version = "0.0.5", features = ["async_tokio"] } async_timing_util = "0.1.14" -tokio = { version = "1.25", features = ["full"] } -# tokio-util = "0.7" +tokio = { version = "1.26", features = ["full"] } axum = { version = "0.6", features = ["ws"] } tower = { version = "0.4", features = ["full"] } futures = "0.3" -# futures-util = "0.3.25" dotenv = "0.15" serde = "1.0" serde_derive = "1.0" @@ -34,3 +32,4 @@ sysinfo = "0.28" toml = "0.7" daemonize = "0.5.0" clap = { version = "4.1", features = ["derive"] } +svi = "0.1.3" diff --git a/periphery/src/api/build.rs b/periphery/src/api/build.rs index a6c036e84..e7ad909ff 100644 --- a/periphery/src/api/build.rs +++ b/periphery/src/api/build.rs @@ -26,7 +26,14 @@ async fn build_image( tokio::spawn(async move { let logs = match get_docker_token(&build.docker_account, &config) { Ok(docker_token) => { - match docker::build(&build, config.repo_dir.clone(), docker_token).await { + match docker::build( + &build, + config.repo_dir.clone(), + docker_token, + &config.secrets, + ) + .await + { Ok(logs) => logs, Err(e) => vec![Log::error("build", format!("{e:#?}"))], } diff --git a/periphery/src/api/container.rs b/periphery/src/api/container.rs index 0244f0059..f94ac4158 100644 --- a/periphery/src/api/container.rs +++ b/periphery/src/api/container.rs @@ -109,7 +109,13 @@ async fn deploy( ) -> anyhow::Result> { let log = match get_docker_token(&deployment.docker_run_args.docker_account, &config) { Ok(docker_token) => tokio::spawn(async move { - docker::deploy(&deployment, &docker_token, config.repo_dir.clone()).await + docker::deploy( + &deployment, + &docker_token, + config.repo_dir.clone(), + &config.secrets, + ) + .await }) .await .context("failed at spawn thread for deploy")?, diff --git a/periphery/src/helpers/docker/build.rs b/periphery/src/helpers/docker/build.rs index 89442394d..5bd0145ee 100644 --- a/periphery/src/helpers/docker/build.rs +++ b/periphery/src/helpers/docker/build.rs @@ -1,4 +1,4 @@ -use std::path::PathBuf; +use std::{collections::HashMap, path::PathBuf}; use anyhow::{anyhow, Context}; use helpers::to_monitor_name; @@ -20,10 +20,12 @@ pub async fn build( docker_build_args, docker_account, docker_organization, + skip_secret_interp, .. }: &Build, mut repo_dir: PathBuf, docker_token: Option, + secrets: &HashMap, ) -> anyhow::Result> { let mut logs = Vec::new(); let DockerBuildArgs { @@ -55,8 +57,19 @@ pub async fn build( "cd {} && docker build {build_args}{image_tags} -f {dockerfile_path} .{docker_push}", build_dir.display() ); - let build_log = run_monitor_command("docker build", command).await; - logs.push(build_log); + if *skip_secret_interp { + let build_log = run_monitor_command("docker build", command).await; + logs.push(build_log); + } else { + let (command, replacers) = + svi::interpolate_variables(&command, secrets, svi::Interpolator::DoubleBrackets) + .context("failed to interpolate secrets into docker build command")?; + let mut build_log = run_monitor_command("docker build", command).await; + build_log.command = svi::replace_in_string(&build_log.command, &replacers); + build_log.stdout = svi::replace_in_string(&build_log.stdout, &replacers); + build_log.stderr = svi::replace_in_string(&build_log.stderr, &replacers); + logs.push(build_log); + } Ok(logs) } diff --git a/periphery/src/helpers/docker/container.rs b/periphery/src/helpers/docker/container.rs index 4d2a0dd15..29688ad79 100644 --- a/periphery/src/helpers/docker/container.rs +++ b/periphery/src/helpers/docker/container.rs @@ -1,4 +1,4 @@ -use std::path::PathBuf; +use std::{collections::HashMap, path::PathBuf}; use anyhow::{anyhow, Context}; use helpers::to_monitor_name; @@ -78,6 +78,7 @@ pub async fn deploy( deployment: &Deployment, docker_token: &Option, repo_dir: PathBuf, + secrets: &HashMap, ) -> Log { if let Err(e) = docker_login(&deployment.docker_run_args.docker_account, docker_token).await { return Log::error("docker login", format!("{e:#?}")); @@ -85,7 +86,22 @@ pub async fn deploy( let _ = pull_image(&deployment.docker_run_args.image).await; let _ = stop_and_remove_container(&to_monitor_name(&deployment.name)).await; let command = docker_run_command(deployment, repo_dir); - run_monitor_command("docker run", command).await + if deployment.skip_secret_interp { + run_monitor_command("docker run", command).await + } else { + let command = + svi::interpolate_variables(&command, secrets, svi::Interpolator::DoubleBrackets) + .context("failed to interpolate secrets into docker run command"); + if let Err(e) = command { + return Log::error("docker run", format!("{e:?}")); + } + let (command, replacers) = command.unwrap(); + let mut log = run_monitor_command("docker run", command).await; + log.command = svi::replace_in_string(&log.command, &replacers); + log.stdout = svi::replace_in_string(&log.stdout, &replacers); + log.stderr = svi::replace_in_string(&log.stderr, &replacers); + log + } } pub fn docker_run_command( diff --git a/periphery/src/main.rs b/periphery/src/main.rs index fd5b203e8..39b67fce5 100644 --- a/periphery/src/main.rs +++ b/periphery/src/main.rs @@ -14,7 +14,7 @@ mod helpers; type PeripheryConfigExtension = Extension>; type HomeDirExtension = Extension>; -fn main() { +fn main() -> anyhow::Result<()> { let (args, port, config, home_dir) = config::load(); if args.daemon { @@ -29,7 +29,9 @@ fn main() { } } - run_periphery_server(port, config, home_dir) + run_periphery_server(port, config, home_dir)?; + + Ok(()) } #[tokio::main] @@ -37,11 +39,12 @@ async fn run_periphery_server( port: u16, config: PeripheryConfigExtension, home_dir: HomeDirExtension, -) { +) -> anyhow::Result<()> { let app = api::router(config, home_dir); axum::Server::bind(&get_socket_addr(port)) .serve(app.into_make_service_with_connect_info::()) - .await - .expect("monitor periphery axum server crashed"); + .await?; + + Ok(()) }