add a bunch of secret / config examples. begin builder

.vscode/tasks.json

@@ -52,6 +52,14 @@
 				"cwd": "${workspaceFolder}/periphery"
 			},
 		},
+		{
+			"type": "cargo",
+			"command": "run",
+			"label": "run builder",
+			"options": {
+				"cwd": "${workspaceFolder}/builder"
+			},
+		},
 		{
 			"type": "cargo",
 			"command": "run",

builder/.env.example

@@ -0,0 +1,2 @@
+PORT=9001
+SECRETS_PATH=./secrets.example.toml

builder/secrets.example.json

@@ -0,0 +1,10 @@
+{
+	"github_accounts": {
+		"github_username1": "github_token1",
+		"github_username2": "github_token2"
+	},
+	"docker_accounts": {
+		"docker_username1": "docker_token1",
+		"docker_username2": "docker_token2"
+	}
+}

builder/secrets.example.toml

@@ -0,0 +1,7 @@
+[github_accounts]
+github_username1 = "github_token1"
+github_username2 = "github_token2"
+
+[docker_accounts]
+docker_username1 = "docker_token1"
+docker_username2 = "docker_token2"

builder/src/api/get_accounts.rs

@@ -0,0 +1,30 @@
+use axum::{extract::Path, Extension, Json};
+use mungos::Deserialize;
+use types::AccountType;
+
+use crate::BuilderSecretsExtension;
+
+#[derive(Deserialize, Debug)]
+pub struct GetAccountsPath {
+    account_type: AccountType,
+}
+
+pub async fn get_accounts(
+    Extension(secrets): BuilderSecretsExtension,
+    Path(path): Path<GetAccountsPath>,
+) -> Json<Vec<String>> {
+    match path.account_type {
+        AccountType::Github => {
+            let mut accounts: Vec<String> =
+                secrets.github_accounts.keys().map(|k| k.clone()).collect();
+            accounts.sort();
+            Json(accounts)
+        }
+        AccountType::Docker => {
+            let mut accounts: Vec<String> =
+                secrets.docker_accounts.keys().map(|k| k.clone()).collect();
+            accounts.sort();
+            Json(accounts)
+        }
+    }
+}

builder/src/api/mod.rs

@@ -0,0 +1,10 @@
+use axum::{routing::get, Router};
+
+mod get_accounts;
+
+pub fn router() -> Router {
+    Router::new().route(
+        "get_accounts/:account_type",
+        get(get_accounts::get_accounts),
+    )
+}

builder/src/config.rs

@@ -1,22 +1,37 @@
-use std::{net::SocketAddr, str::FromStr};
-
-use dotenv::dotenv;
-use mungos::Deserialize;
-
-#[derive(Deserialize)]
-struct Env {
-    port: u16,
-}
-
-pub fn load() {
-    pub fn load() -> (SocketAddr) {
-        dotenv().ok();
-
-        let env = envy::from_env::<Env>().unwrap();
-
-        let socket_addr = SocketAddr::from_str(&format!("0.0.0.0:{}", env.port))
-            .expect("failed to parse socket addr");
-
-        (socket_addr)
-    }
-}
+use std::{net::SocketAddr, str::FromStr, sync::Arc};
+
+use axum::Extension;
+use dotenv::dotenv;
+use ::helpers::parse_config_file;
+use mungos::Deserialize;
+
+use crate::BuilderSecretsExtension;
+
+#[derive(Deserialize)]
+struct Env {
+    #[serde(default = "default_port")]
+    port: u16,
+    #[serde(default = "default_config_path")]
+    secrets_path: String,
+}
+
+pub fn load() -> (SocketAddr, BuilderSecretsExtension) {
+    dotenv().ok();
+
+    let env = envy::from_env::<Env>().unwrap();
+
+    let socket_addr = SocketAddr::from_str(&format!("0.0.0.0:{}", env.port))
+        .expect("failed to parse socket addr");
+
+    let secrets = parse_config_file(&env.secrets_path).expect("failed to parse config");
+
+    (socket_addr, Extension(Arc::new(secrets)))
+}
+
+fn default_port() -> u16 {
+    9001
+}
+
+fn default_config_path() -> String {
+    "/secrets/secrets.json".to_string()
+}

builder/src/main.rs

@@ -1,6 +1,21 @@
-mod config;
-
-#[tokio::main]
-async fn main() {
-    let (socket_addr) = config::load();
-}
+use std::sync::Arc;
+
+use axum::Extension;
+use types::BuilderSecrets;
+
+mod api;
+mod config;
+
+type BuilderSecretsExtension = Extension<Arc<BuilderSecrets>>;
+
+#[tokio::main]
+async fn main() {
+    let (socket_addr, secrets) = config::load();
+
+    let app = api::router().layer(secrets);
+
+    axum::Server::bind(&socket_addr)
+        .serve(app.into_make_service())
+        .await
+        .expect("server crashed");
+}

core/config.example.json

@@ -1,15 +1,13 @@
 {
 	"port": 9000,
-	"docker_accounts": {},
-	"github_accounts": {},
+	"jwt_secret": "your_jwt_secret",
+	"jwt_valid_for": "1-wk",
+	"slack_url": "your_slack_app_webhook_url",
+	"github_webhook_secret": "your_random_webhook_secret",
 	"github_oauth": {
 		"id": "your_client_id",
 		"secret": "your_client_secret"
 	},
-	"github_webhook_secret": "your_random_webhook_secret",
-	"jwt_secret": "your_jwt_secret",
-	"jwt_valid_for": "1-wk",
-	"slack_url": "your_slack_app_webhook_url",
 	"mongo": {
 		"uri": "your_mongo_uri",
 		"app_name": "monitor_core",

core/config.example.toml

@@ -11,10 +11,4 @@ secret = "your_client_secret"
 [mongo]
 uri = "your_mongo_uri"
 app_name = "monitor_core"
-db_name = "monitor"
-
-[github_accounts]
-github_username = "github_token"
-
-[docker_accounts]
-docker_username = "docker_token"
+db_name = "monitor"

core/src/config.rs

@@ -21,7 +21,7 @@ struct Env {
 pub fn load() -> CoreConfig {
     dotenv().ok();
     let env: Env = envy::from_env().expect("failed to parse environment variables");
-    parse_config_file(&env.config_path)
+    parse_config_file(&env.config_path).expect("failed to parse config")
 }
 
 pub fn default_config_path() -> String {

core/src/main.rs

@@ -1,9 +1,9 @@
 #![allow(unused)]
 
+use ::helpers::docker::DockerClient;
 use auth::JwtClient;
 use axum::{http::StatusCode, Router};
 use db::DbClient;
-use ::helpers::docker::DockerClient;
 
 mod api;
 mod auth;

lib/db_client/src/lib.rs

@@ -1,57 +1,52 @@
-use std::{sync::Arc, time::Duration};
-
-use axum::Extension;
-use collections::{
-    builds_collection, deployments_collection, procedures_collection, servers_collection,
-    updates_collection, users_collection,
-};
-use mungos::{Collection, Mungos};
-use types::{Build, Deployment, Procedure, Server, Update, User, MongoConfig};
-
-mod collections;
-
-pub type DbExtension = Extension<Arc<DbClient>>;
-
-pub struct DbClient {
-    pub users: Collection<User>,
-    pub servers: Collection<Server>,
-    pub deployments: Collection<Deployment>,
-    pub builds: Collection<Build>,
-    pub procedures: Collection<Procedure>,
-    pub updates: Collection<Update>,
-}
-
-impl DbClient {
-    pub async fn extension(config: MongoConfig) -> DbExtension {
-        let db_name = &config.db_name;
-        let mungos = Mungos::new(
-            &config.uri,
-            &config.app_name,
-            Duration::from_secs(3),
-            None,
-        )
-        .await
-        .expect("failed to initialize mungos");
-        let client = DbClient {
-            users: users_collection(&mungos, db_name)
-                .await
-                .expect("failed to make users collection"),
-            servers: servers_collection(&mungos, db_name)
-                .await
-                .expect("failed to make servers collection"),
-            deployments: deployments_collection(&mungos, db_name)
-                .await
-                .expect("failed to make deployments collection"),
-            builds: builds_collection(&mungos, db_name)
-                .await
-                .expect("failed to make builds collection"),
-            updates: updates_collection(&mungos, db_name)
-                .await
-                .expect("failed to make updates collection"),
-            procedures: procedures_collection(&mungos, db_name)
-                .await
-                .expect("failed to make procedures collection"),
-        };
-        Extension(Arc::new(client))
-    }
-}
+use std::{sync::Arc, time::Duration};
+
+use axum::Extension;
+use collections::{
+    builds_collection, deployments_collection, procedures_collection, servers_collection,
+    updates_collection, users_collection,
+};
+use mungos::{Collection, Mungos};
+use types::{Build, Deployment, MongoConfig, Procedure, Server, Update, User};
+
+mod collections;
+
+pub type DbExtension = Extension<Arc<DbClient>>;
+
+pub struct DbClient {
+    pub users: Collection<User>,
+    pub servers: Collection<Server>,
+    pub deployments: Collection<Deployment>,
+    pub builds: Collection<Build>,
+    pub procedures: Collection<Procedure>,
+    pub updates: Collection<Update>,
+}
+
+impl DbClient {
+    pub async fn extension(config: MongoConfig) -> DbExtension {
+        let db_name = &config.db_name;
+        let mungos = Mungos::new(&config.uri, &config.app_name, Duration::from_secs(3), None)
+            .await
+            .expect("failed to initialize mungos");
+        let client = DbClient {
+            users: users_collection(&mungos, db_name)
+                .await
+                .expect("failed to make users collection"),
+            servers: servers_collection(&mungos, db_name)
+                .await
+                .expect("failed to make servers collection"),
+            deployments: deployments_collection(&mungos, db_name)
+                .await
+                .expect("failed to make deployments collection"),
+            builds: builds_collection(&mungos, db_name)
+                .await
+                .expect("failed to make builds collection"),
+            updates: updates_collection(&mungos, db_name)
+                .await
+                .expect("failed to make updates collection"),
+            procedures: procedures_collection(&mungos, db_name)
+                .await
+                .expect("failed to make procedures collection"),
+        };
+        Extension(Arc::new(client))
+    }
+}

lib/helpers/Cargo.toml

@@ -7,7 +7,6 @@ edition = "2021"
 
 [dependencies]
 types = { path = "../types" }
-run_command = "0.0.5"
 async_timing_util = "0.1.11"
 bollard = "0.13"
 anyhow = "1.0"
@@ -15,4 +14,5 @@ axum = "0.5"
 serde = "1.0"
 serde_derive = "1.0"
 serde_json = "1.0"
-toml = "0.5"
+toml = "0.5"
+run_command = { version = "0.0.5", features = ["async_tokio"] }

lib/helpers/src/docker.rs

@@ -4,12 +4,14 @@ use anyhow::anyhow;
 use async_timing_util::unix_timestamp_ms;
 use axum::Extension;
 use bollard::{container::ListContainersOptions, Docker};
-use run_command::{async_run_command, CommandOutput};
+use ::run_command::async_run_command;
 use types::{
     BasicContainerInfo, Build, Conversion, Deployment, DockerRunArgs, EnvironmentVar, Log,
     RestartMode,
 };
 
+use crate::output_into_log;
+
 pub type DockerExtension = Extension<Arc<DockerClient>>;
 
 pub struct DockerClient {
@@ -169,19 +171,6 @@ fn parse_post_image(post_image: &Option<String>) -> String {
 
 // BUILD COMMANDS
 
-pub async fn docker_build(_build: &Build) -> (bool, Log) {
+pub async fn docker_build(_build: &Build) -> Log {
     todo!()
 }
-
-fn output_into_log(stage: &str, command: String, start_ts: i64, output: CommandOutput) -> Log {
-    let success = output.success();
-    Log {
-        stage: stage.to_string(),
-        stdout: output.stdout,
-        stderr: output.stderr,
-        command,
-        success,
-        start_ts,
-        end_ts: unix_timestamp_ms() as i64,
-    }
-}

lib/helpers/src/git.rs

@@ -1,5 +1,5 @@
 use async_timing_util::unix_timestamp_ms;
-use run_command::async_run_command;
+use ::run_command::async_run_command;
 use types::{Build, Deployment, Log};
 
 pub async fn clone_build_repo(

lib/helpers/src/lib.rs

@@ -1,19 +1,38 @@
 use std::{fs::File, io::Read};
 
+use anyhow::Context;
+use async_timing_util::unix_timestamp_ms;
+use run_command::CommandOutput;
 use serde::de::DeserializeOwned;
+use types::Log;
 
 pub mod docker;
 pub mod git;
 
-pub fn parse_config_file<T: DeserializeOwned>(path: &str) -> T {
+pub fn parse_config_file<T: DeserializeOwned>(path: &str) -> anyhow::Result<T> {
     let mut file = File::open(&path).expect(&format!("failed to find config at {path}"));
-    if path.ends_with("toml") {
+    let config = if path.ends_with("toml") {
         let mut contents = String::new();
-        file.read_to_string(&mut contents).expect(&format!("failed to read toml at {path}"));
-        toml::from_str(&contents).expect(&format!("failed to parse toml at {path}"))
+        file.read_to_string(&mut contents)
+            .context(format!("failed to read toml at {path}"))?;
+        toml::from_str(&contents).context(format!("failed to parse toml at {path}"))?
     } else if path.ends_with("json") {
-        serde_json::from_reader(file).expect(&format!("failed to parse json at {path}"))
+        serde_json::from_reader(file).context(format!("failed to parse json at {path}"))?
     } else {
         panic!("unsupported config file type: {}", path)
+    };
+    Ok(config)
+}
+
+pub fn output_into_log(stage: &str, command: String, start_ts: i64, output: CommandOutput) -> Log {
+    let success = output.success();
+    Log {
+        stage: stage.to_string(),
+        stdout: output.stdout,
+        stderr: output.stderr,
+        command,
+        success,
+        start_ts,
+        end_ts: unix_timestamp_ms() as i64,
     }
-}
+}

lib/types/src/lib.rs

@@ -229,11 +229,7 @@ pub struct CoreConfig {
     // port the core web server runs on
     pub port: u16,
 
-    // docker integration
-    pub docker_accounts: DockerAccounts,
-
     // github integration
-    pub github_accounts: GithubAccounts,
     pub github_oauth: OauthCredentials,
     pub github_webhook_secret: String,
 
@@ -273,6 +269,14 @@ pub struct PeripherySecrets {
     pub github_accounts: GithubAccounts,
 }
 
+#[derive(Deserialize, Debug)]
+pub struct BuilderSecrets {
+    #[serde(default)]
+    pub docker_accounts: DockerAccounts,
+    #[serde(default)]
+    pub github_accounts: GithubAccounts,
+}
+
 #[derive(Deserialize, Debug)]
 pub struct UserCredentials {
     pub username: String,
@@ -303,6 +307,14 @@ pub struct SystemNetwork {
     pub transmitted: f64, // in kB
 }
 
+#[derive(Serialize, Deserialize, Debug, Display, EnumString, PartialEq, Hash, Eq, Clone, Copy)]
+#[serde(rename_all = "snake_case")]
+#[strum(serialize_all = "snake_case")]
+pub enum AccountType {
+    Github,
+    Docker,
+}
+
 #[derive(Serialize, Deserialize, Debug, Display, EnumString, PartialEq, Hash, Eq, Clone, Copy)]
 #[serde(rename_all = "snake_case")]
 #[strum(serialize_all = "snake_case")]

periphery/.env.example

@@ -0,0 +1 @@
+SECRETS_PATH=./secrets.example.toml

periphery/secrets.example.json

@@ -0,0 +1,10 @@
+{
+	"github_accounts": {
+		"github_username1": "github_token1",
+		"github_username2": "github_token2"
+	},
+	"docker_accounts": {
+		"docker_username1": "docker_token1",
+		"docker_username2": "docker_token2"
+	}
+}

periphery/secrets.example.toml

@@ -0,0 +1,7 @@
+[github_accounts]
+github_username1 = "github_token1"
+github_username2 = "github_token2"
+
+[docker_accounts]
+docker_username1 = "docker_token1"
+docker_username2 = "docker_token2"

periphery/src/config.rs

@@ -1,9 +1,9 @@
 use std::{fs::File, io::Read};
 
 use dotenv::dotenv;
+use helpers::parse_config_file;
 use serde::Deserialize;
 use types::PeripherySecrets;
-use helpers::parse_config_file;
 
 #[derive(Deserialize)]
 struct Env {
@@ -16,7 +16,7 @@ struct Env {
 pub fn load() -> (u16, PeripherySecrets) {
     dotenv().ok();
     let env: Env = envy::from_env().expect("failed to parse env");
-    let secrets = parse_config_file(&env.secrets_path);
+    let secrets = parse_config_file(&env.secrets_path).expect("failed to parse secrets file");
     (env.port, secrets)
 }
 
@@ -26,4 +26,4 @@ fn default_port() -> u16 {
 
 fn default_secrets_path() -> String {
     "/secrets/secrets.toml".to_string()
-}
+}