From 402259ef1142903b9ace184f20a41f318c9ad20f Mon Sep 17 00:00:00 2001 From: mbecker20 Date: Sat, 24 Jun 2023 00:42:14 +0000 Subject: [PATCH] implement build request --- core/src/auth/mod.rs | 2 +- core/src/cloud/mod.rs | 7 +- core/src/requests/api/build.rs | 250 +++++++++++++++++++--- core/src/requests/api/deployment.rs | 4 +- lib/helpers/src/lib.rs | 18 -- lib/types/src/entities/build.rs | 8 +- lib/types/src/entities/mod.rs | 15 ++ lib/types/src/lib.rs | 17 ++ periphery/src/helpers/docker/build.rs | 6 +- periphery/src/helpers/docker/container.rs | 6 +- periphery/src/requests/build.rs | 3 +- periphery/src/requests/container.rs | 5 +- 12 files changed, 275 insertions(+), 66 deletions(-) diff --git a/core/src/auth/mod.rs b/core/src/auth/mod.rs index 106d208cc..08b10ab05 100644 --- a/core/src/auth/mod.rs +++ b/core/src/auth/mod.rs @@ -20,7 +20,7 @@ use crate::{ state::{State, StateExtension}, }; -pub use self::jwt::{JwtClient, RequestUser, RequestUserExtension}; +pub use self::jwt::{InnerRequestUser, JwtClient, RequestUser, RequestUserExtension}; pub use github::client::GithubOauthClient; pub use google::client::GoogleOauthClient; diff --git a/core/src/cloud/mod.rs b/core/src/cloud/mod.rs index 850bead86..bd69d65bf 100644 --- a/core/src/cloud/mod.rs +++ b/core/src/cloud/mod.rs @@ -1,5 +1,6 @@ pub mod aws; -pub enum InstanceCleanupData { - Aws { instance_id: String, region: String, } -} \ No newline at end of file +pub enum BuildCleanupData { + Server { repo_name: String }, + Aws { instance_id: String, region: String }, +} diff --git a/core/src/requests/api/build.rs b/core/src/requests/api/build.rs index fc6f54337..5ad1aa2fc 100644 --- a/core/src/requests/api/build.rs +++ b/core/src/requests/api/build.rs @@ -1,13 +1,15 @@ -use std::pin::Pin; +use std::time::Duration; use anyhow::{anyhow, Context}; use async_trait::async_trait; -use futures::Future; -use monitor_helpers::{all_logs_success, monitor_timestamp}; +use futures::future::join_all; +use monitor_helpers::monitor_timestamp; use monitor_types::{ + all_logs_success, entities::{ build::{Build, BuildBuilderConfig}, builder::{AwsBuilder, BuilderConfig}, + deployment::DockerContainerState, update::{Log, Update, UpdateStatus, UpdateTarget}, Operation, PermissionLevel, }, @@ -15,12 +17,15 @@ use monitor_types::{ requests::api::{CreateBuild, DeleteBuild, GetBuild, ListBuilds, RunBuild, UpdateBuild}, }; use mungos::mongodb::bson::{doc, to_bson}; -use periphery_client::PeripheryClient; +use periphery_client::{ + requests::{self, GetVersionResponse}, + PeripheryClient, +}; use resolver_api::Resolve; use crate::{ - auth::RequestUser, - cloud::{aws::Ec2Instance, InstanceCleanupData}, + auth::{InnerRequestUser, RequestUser}, + cloud::{aws::Ec2Instance, BuildCleanupData}, helpers::empty_or_only_spaces, state::State, }; @@ -78,6 +83,7 @@ impl Resolve for State { name, created_at: start_ts, updated_at: start_ts, + last_built_at: 0, permissions: [(user.id.clone(), PermissionLevel::Update)] .into_iter() .collect(), @@ -273,6 +279,8 @@ impl Resolve for State { }; update.id = self.add_update(update.clone()).await?; + // GET BUILDER PERIPHERY + let builder = self.get_build_builder(&build, &mut update).await; if let Err(e) = &builder { @@ -286,11 +294,63 @@ impl Resolve for State { let (periphery, cleanup_data) = builder.unwrap(); - // ... + // CLONE REPO + + let clone_success = match periphery + .request(requests::CloneRepo { + args: (&build).into(), + }) + .await + { + Ok(clone_logs) => { + let success = all_logs_success(&clone_logs); + update.logs.extend(clone_logs); + success + } + Err(e) => { + update + .logs + .push(Log::error("clone repo", format!("{e:#?}"))); + false + } + }; + + if clone_success { + match periphery + .request(requests::Build { + build: build.clone(), + }) + .await + .context("failed at call to periphery to build") + { + Ok(logs) => update.logs.extend(logs), + Err(e) => update.logs.push(Log::error("build", format!("{e:#?}"))), + }; + } + + if all_logs_success(&update.logs) { + let _ = self + .db + .builds + .update_one::( + &build.id, + mungos::Update::Set(doc! { + "version": to_bson(&build.config.version) + .context("failed at converting version to bson")?, + "last_built_at": monitor_timestamp(), + }), + ) + .await; + } + + // CLEANUP AND FINALIZE UPDATE self.cleanup_builder_instance(cleanup_data, &mut update) .await; + self.handle_post_build_redeploy(&build.id, &mut update) + .await; + update.finalize(); self.update_update(update.clone()).await?; @@ -318,19 +378,33 @@ impl Resolve for State { } } +const BUILDER_POLL_RATE_SECS: u64 = 2; +const BUILDER_POLL_MAX_TRIES: usize = 30; + impl State { async fn get_build_builder( &self, build: &Build, update: &mut Update, - ) -> anyhow::Result<(PeripheryClient, Option)> { + ) -> anyhow::Result<(PeripheryClient, BuildCleanupData)> { match &build.config.builder { BuildBuilderConfig::Server { server_id } => { + if server_id.is_empty() { + return Err(anyhow!("build has not configured a builder")); + } let server = self.get_server(server_id).await?; let periphery = self.periphery_client(&server); - Ok((periphery, None)) + Ok(( + periphery, + BuildCleanupData::Server { + repo_name: build.name.clone(), + }, + )) } BuildBuilderConfig::Builder { builder_id } => { + if builder_id.is_empty() { + return Err(anyhow!("build has not configured a builder")); + } let builder = self.get_builder(builder_id).await?; match builder.config { BuilderConfig::AwsBuilder(config) => { @@ -346,7 +420,9 @@ impl State { build: &Build, builder: AwsBuilder, update: &mut Update, - ) -> anyhow::Result<(PeripheryClient, Option)> { + ) -> anyhow::Result<(PeripheryClient, BuildCleanupData)> { + let start_create_ts = monitor_timestamp(); + let instance_name = format!( "BUILDER-{}-v{}", build.name, @@ -355,34 +431,71 @@ impl State { let Ec2Instance { instance_id, ip } = self.create_ec2_instance(&instance_name, &builder).await?; - update - .logs - .push(Log::simple("started builder instance", format!(""))); + let readable_sec_group_ids = builder.security_group_ids.join(", "); + let AwsBuilder { + ami_id, + instance_type, + volume_gb, + subnet_id, + .. + } = builder; + + let log = Log { + stage: "start build instance".to_string(), + success: true, + stdout: format!("instance id: {instance_id}\nami id: {ami_id}\ninstance type: {instance_type}\nvolume size: {volume_gb} GB\nsubnet id: {subnet_id}\nsecurity groups: {readable_sec_group_ids}"), + start_ts: start_create_ts, + end_ts: monitor_timestamp(), + ..Default::default() + }; + + update.logs.push(log); self.update_update(update.clone()).await?; let periphery = PeripheryClient::new(format!("http://{ip}:8000"), &self.config.passkey); - Ok(( - periphery, - InstanceCleanupData::Aws { - instance_id, - region: builder.region, + let start_connect_ts = monitor_timestamp(); + let mut res = Ok(GetVersionResponse { + version: String::new(), + }); + for _ in 0..BUILDER_POLL_MAX_TRIES { + let version = periphery + .request(requests::GetVersion {}) + .await + .context("failed to reach periphery client on builder"); + if let Ok(GetVersionResponse { version }) = &version { + let connect_log = Log { + stage: "build instance connected".to_string(), + success: true, + stdout: format!( + "established contact with periphery on builder\nperiphery version: v{}", + version + ), + start_ts: start_connect_ts, + end_ts: monitor_timestamp(), + ..Default::default() + }; + update.logs.push(connect_log); + self.update_update(update.clone()).await?; + return Ok(( + periphery, + BuildCleanupData::Aws { + instance_id, + region: builder.region, + }, + )); } - .into(), - )) + res = version; + tokio::time::sleep(Duration::from_secs(BUILDER_POLL_RATE_SECS)).await; + } + Err(anyhow!("{:#?}", res.err().unwrap())) } - async fn cleanup_builder_instance( - &self, - cleanup_data: Option, - update: &mut Update, - ) { - if cleanup_data.is_none() { - return; - } - match cleanup_data.unwrap() { - InstanceCleanupData::Aws { + async fn cleanup_builder_instance(&self, cleanup_data: BuildCleanupData, update: &mut Update) { + match cleanup_data { + BuildCleanupData::Server { repo_name } => {} + BuildCleanupData::Aws { instance_id, region, } => { @@ -401,4 +514,81 @@ impl State { } } } + + async fn handle_post_build_redeploy(&self, build_id: &str, update: &mut Update) { + let redeploy_deployments = self + .db + .deployments + .get_some( + doc! { "build_id": build_id, "redeploy_on_build": true }, + None, + ) + .await; + + if let Ok(deployments) = redeploy_deployments { + let futures = deployments.into_iter().map(|d| async move { + let request_user: RequestUser = InnerRequestUser { + id: "auto redeploy".to_string(), + is_admin: true, + ..Default::default() + } + .into(); + let state = self.get_deployment_state(&d).await.unwrap_or_default(); + if state == DockerContainerState::Running { + // Some(( + // d.id.clone(), + // self.deploy_container( + // &d.id, + // &RequestUser { + // id: "auto redeploy".to_string(), + // is_admin: true, + // ..Default::default() + // }, + // None, + // None, + // ) + // .await, + // )) + Some(()) + } else { + None + } + }); + + let redeploy_results = join_all(futures).await; + + let mut redeploys = Vec::::new(); + let mut redeploy_failures = Vec::::new(); + + // for res in redeploy_results { + // if res.is_none() { + // continue; + // } + // let (id, res) = res.unwrap(); + // match res { + // Ok(_) => redeploys.push(id), + // Err(e) => redeploy_failures.push(format!("{id}: {e:#?}")), + // } + // } + + if !redeploys.is_empty() { + update.logs.push(Log::simple( + "redeploy", + format!("redeployed deployments: {}", redeploys.join(", ")), + )) + } + + if !redeploy_failures.is_empty() { + update.logs.push(Log::simple( + "redeploy failures", + redeploy_failures.join("\n"), + )) + } + } else if let Err(e) = redeploy_deployments { + update.logs.push(Log::simple( + "redeploys failed", + format!("failed to get deployments to redeploy: {e:#?}"), + )) + } + } } diff --git a/core/src/requests/api/deployment.rs b/core/src/requests/api/deployment.rs index 2fa0b8ebb..80230aaa9 100644 --- a/core/src/requests/api/deployment.rs +++ b/core/src/requests/api/deployment.rs @@ -1,6 +1,6 @@ use anyhow::{anyhow, Context}; use async_trait::async_trait; -use monitor_helpers::{all_logs_success, monitor_timestamp, to_monitor_name}; +use monitor_helpers::{monitor_timestamp, to_monitor_name}; use monitor_types::{ entities::{ deployment::{Deployment, DockerContainerState}, @@ -11,7 +11,7 @@ use monitor_types::{ requests::api::{ CreateDeployment, DeleteDeployment, GetDeployment, ListDeployments, RenameDeployment, UpdateDeployment, - }, + }, all_logs_success, }; use mungos::mongodb::bson::{doc, to_bson}; use periphery_client::requests; diff --git a/lib/helpers/src/lib.rs b/lib/helpers/src/lib.rs index 68908bcdd..8db650cc6 100644 --- a/lib/helpers/src/lib.rs +++ b/lib/helpers/src/lib.rs @@ -1,27 +1,9 @@ use async_timing_util::unix_timestamp_ms; -use monitor_types::entities::update::Log; pub fn to_monitor_name(name: &str) -> String { name.to_lowercase().replace(' ', "_") } -pub fn optional_string(string: &str) -> Option { - if string.is_empty() { - None - } else { - Some(string.to_string()) - } -} - pub fn monitor_timestamp() -> i64 { unix_timestamp_ms() as i64 } - -pub fn all_logs_success(logs: &Vec) -> bool { - for log in logs { - if !log.success { - return false; - } - } - true -} \ No newline at end of file diff --git a/lib/types/src/entities/build.rs b/lib/types/src/entities/build.rs index e1e01be7d..69fb21a2c 100644 --- a/lib/types/src/entities/build.rs +++ b/lib/types/src/entities/build.rs @@ -40,6 +40,10 @@ pub struct Build { #[builder(setter(skip))] pub updated_at: I64, + #[serde(default)] + #[builder(setter(skip))] + pub last_built_at: I64, + pub config: BuildConfig, } @@ -161,6 +165,8 @@ pub enum BuildBuilderConfig { impl Default for BuildBuilderConfig { fn default() -> Self { - Self::Server { server_id: Default::default() } + Self::Server { + server_id: Default::default(), + } } } diff --git a/lib/types/src/entities/mod.rs b/lib/types/src/entities/mod.rs index 5fa703c87..727b14ffd 100644 --- a/lib/types/src/entities/mod.rs +++ b/lib/types/src/entities/mod.rs @@ -6,6 +6,8 @@ use serde::{Deserialize, Serialize}; use strum_macros::{Display, EnumString}; use typeshare::typeshare; +use crate::optional_string; + pub mod build; pub mod builder; pub mod deployment; @@ -93,6 +95,19 @@ pub struct CloneArgs { pub github_account: Option, } +impl From<&self::build::Build> for CloneArgs { + fn from(build: &self::build::Build) -> CloneArgs { + CloneArgs { + name: build.name.clone(), + repo: optional_string(&build.config.repo), + branch: optional_string(&build.config.branch), + on_clone: build.config.pre_build.clone().into(), + on_pull: None, + github_account: optional_string(&build.config.github_account), + } + } +} + #[typeshare] #[derive( Serialize, Deserialize, Debug, Display, EnumString, PartialEq, Hash, Eq, Clone, Copy, Default, diff --git a/lib/types/src/lib.rs b/lib/types/src/lib.rs index 043682efa..4610c00c1 100644 --- a/lib/types/src/lib.rs +++ b/lib/types/src/lib.rs @@ -13,3 +13,20 @@ pub type MongoDocument = mungos::mongodb::bson::Document; fn i64_is_zero(n: &I64) -> bool { *n == 0 } + +pub fn all_logs_success(logs: &Vec) -> bool { + for log in logs { + if !log.success { + return false; + } + } + true +} + +pub fn optional_string(string: &str) -> Option { + if string.is_empty() { + None + } else { + Some(string.to_string()) + } +} \ No newline at end of file diff --git a/periphery/src/helpers/docker/build.rs b/periphery/src/helpers/docker/build.rs index 7971368b1..4dd2bcc92 100644 --- a/periphery/src/helpers/docker/build.rs +++ b/periphery/src/helpers/docker/build.rs @@ -1,12 +1,12 @@ use std::{collections::HashMap, path::PathBuf}; use anyhow::Context; -use monitor_helpers::{optional_string, to_monitor_name}; -use monitor_types::entities::{ +use monitor_helpers::to_monitor_name; +use monitor_types::{entities::{ build::{Build, BuildConfig}, update::Log, EnvironmentVar, Version, -}; +}, optional_string}; use crate::helpers::run_monitor_command; diff --git a/periphery/src/helpers/docker/container.rs b/periphery/src/helpers/docker/container.rs index cf1ea9adc..b21886777 100644 --- a/periphery/src/helpers/docker/container.rs +++ b/periphery/src/helpers/docker/container.rs @@ -1,15 +1,15 @@ use std::collections::HashMap; use anyhow::{anyhow, Context}; -use monitor_helpers::{optional_string, to_monitor_name}; -use monitor_types::entities::{ +use monitor_helpers::to_monitor_name; +use monitor_types::{entities::{ deployment::{ Conversion, Deployment, DeploymentConfig, DockerContainerStats, RestartMode, TerminationSignal, }, update::Log, EnvironmentVar, -}; +}, optional_string}; use run_command::async_run_command; use crate::helpers::{docker::parse_extra_args, run_monitor_command}; diff --git a/periphery/src/requests/build.rs b/periphery/src/requests/build.rs index 0b8b95c79..4f30df93b 100644 --- a/periphery/src/requests/build.rs +++ b/periphery/src/requests/build.rs @@ -1,6 +1,5 @@ use async_trait::async_trait; -use monitor_helpers::optional_string; -use monitor_types::entities::{server::docker_image::ImageSummary, update::Log}; +use monitor_types::{entities::{server::docker_image::ImageSummary, update::Log}, optional_string}; use resolver_api::{derive::Request, Resolve}; use serde::{Deserialize, Serialize}; diff --git a/periphery/src/requests/container.rs b/periphery/src/requests/container.rs index 54cf98a0a..6363747a9 100644 --- a/periphery/src/requests/container.rs +++ b/periphery/src/requests/container.rs @@ -1,9 +1,8 @@ use anyhow::{anyhow, Context}; -use monitor_helpers::optional_string; -use monitor_types::entities::{ +use monitor_types::{entities::{ deployment::{BasicContainerInfo, Deployment, DockerContainerStats, TerminationSignal}, update::Log, -}; +}, optional_string}; use resolver_api::{derive::Request, Resolve}; use serde::{Deserialize, Serialize};