work on Aws builds / RunBuild

This commit is contained in:
mbecker20
2023-06-23 08:07:35 +00:00
parent 2affc5f555
commit 2d4c682fac
15 changed files with 1033 additions and 36 deletions

481
Cargo.lock generated
View File

@@ -19,6 +19,15 @@ dependencies = [
"version_check",
]
[[package]]
name = "aho-corasick"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "43f6cb1bf222025340178f382c426f13757b2960e89779dfcb319c32542a5a41"
dependencies = [
"memchr",
]
[[package]]
name = "android-tzdata"
version = "0.1.1"
@@ -108,8 +117,8 @@ checksum = "9de37a9f9ebc2a377f452555a2278e89548c7746c9a47cf3047b30366af7168e"
dependencies = [
"serde",
"serde_derive",
"strum",
"strum_macros",
"strum 0.24.1",
"strum_macros 0.24.3",
"tokio",
]
@@ -130,6 +139,326 @@ version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
[[package]]
name = "aws-config"
version = "0.55.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bcdcf0d683fe9c23d32cf5b53c9918ea0a500375a9fb20109802552658e576c9"
dependencies = [
"aws-credential-types",
"aws-http",
"aws-sdk-sso",
"aws-sdk-sts",
"aws-smithy-async",
"aws-smithy-client",
"aws-smithy-http",
"aws-smithy-http-tower",
"aws-smithy-json",
"aws-smithy-types",
"aws-types",
"bytes",
"fastrand",
"hex",
"http",
"hyper",
"ring",
"time",
"tokio",
"tower",
"tracing",
"zeroize",
]
[[package]]
name = "aws-credential-types"
version = "0.55.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1fcdb2f7acbc076ff5ad05e7864bdb191ca70a6fd07668dc3a1a8bcd051de5ae"
dependencies = [
"aws-smithy-async",
"aws-smithy-types",
"fastrand",
"tokio",
"tracing",
"zeroize",
]
[[package]]
name = "aws-endpoint"
version = "0.55.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8cce1c41a6cfaa726adee9ebb9a56fcd2bbfd8be49fd8a04c5e20fd968330b04"
dependencies = [
"aws-smithy-http",
"aws-smithy-types",
"aws-types",
"http",
"regex",
"tracing",
]
[[package]]
name = "aws-http"
version = "0.55.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aadbc44e7a8f3e71c8b374e03ecd972869eb91dd2bc89ed018954a52ba84bc44"
dependencies = [
"aws-credential-types",
"aws-smithy-http",
"aws-smithy-types",
"aws-types",
"bytes",
"http",
"http-body",
"lazy_static",
"percent-encoding",
"pin-project-lite",
"tracing",
]
[[package]]
name = "aws-sdk-ec2"
version = "0.28.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eab2493c5857725eeafe12ec66ba4ce6feb3355e3af6828d9ef28d6152972a27"
dependencies = [
"aws-credential-types",
"aws-endpoint",
"aws-http",
"aws-sig-auth",
"aws-smithy-async",
"aws-smithy-client",
"aws-smithy-http",
"aws-smithy-http-tower",
"aws-smithy-json",
"aws-smithy-query",
"aws-smithy-types",
"aws-smithy-xml",
"aws-types",
"bytes",
"fastrand",
"http",
"regex",
"tokio-stream",
"tower",
"tracing",
]
[[package]]
name = "aws-sdk-sso"
version = "0.28.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c8b812340d86d4a766b2ca73f740dfd47a97c2dff0c06c8517a16d88241957e4"
dependencies = [
"aws-credential-types",
"aws-endpoint",
"aws-http",
"aws-sig-auth",
"aws-smithy-async",
"aws-smithy-client",
"aws-smithy-http",
"aws-smithy-http-tower",
"aws-smithy-json",
"aws-smithy-types",
"aws-types",
"bytes",
"http",
"regex",
"tokio-stream",
"tower",
"tracing",
]
[[package]]
name = "aws-sdk-sts"
version = "0.28.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "265fac131fbfc188e5c3d96652ea90ecc676a934e3174eaaee523c6cec040b3b"
dependencies = [
"aws-credential-types",
"aws-endpoint",
"aws-http",
"aws-sig-auth",
"aws-smithy-async",
"aws-smithy-client",
"aws-smithy-http",
"aws-smithy-http-tower",
"aws-smithy-json",
"aws-smithy-query",
"aws-smithy-types",
"aws-smithy-xml",
"aws-types",
"bytes",
"http",
"regex",
"tower",
"tracing",
]
[[package]]
name = "aws-sig-auth"
version = "0.55.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3b94acb10af0c879ecd5c7bdf51cda6679a0a4f4643ce630905a77673bfa3c61"
dependencies = [
"aws-credential-types",
"aws-sigv4",
"aws-smithy-http",
"aws-types",
"http",
"tracing",
]
[[package]]
name = "aws-sigv4"
version = "0.55.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9d2ce6f507be68e968a33485ced670111d1cbad161ddbbab1e313c03d37d8f4c"
dependencies = [
"aws-smithy-http",
"form_urlencoded",
"hex",
"hmac",
"http",
"once_cell",
"percent-encoding",
"regex",
"sha2",
"time",
"tracing",
]
[[package]]
name = "aws-smithy-async"
version = "0.55.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "13bda3996044c202d75b91afeb11a9afae9db9a721c6a7a427410018e286b880"
dependencies = [
"futures-util",
"pin-project-lite",
"tokio",
"tokio-stream",
]
[[package]]
name = "aws-smithy-client"
version = "0.55.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0a86aa6e21e86c4252ad6a0e3e74da9617295d8d6e374d552be7d3059c41cedd"
dependencies = [
"aws-smithy-async",
"aws-smithy-http",
"aws-smithy-http-tower",
"aws-smithy-types",
"bytes",
"fastrand",
"http",
"http-body",
"hyper",
"hyper-rustls",
"lazy_static",
"pin-project-lite",
"rustls",
"tokio",
"tower",
"tracing",
]
[[package]]
name = "aws-smithy-http"
version = "0.55.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2b3b693869133551f135e1f2c77cb0b8277d9e3e17feaf2213f735857c4f0d28"
dependencies = [
"aws-smithy-types",
"bytes",
"bytes-utils",
"futures-core",
"http",
"http-body",
"hyper",
"once_cell",
"percent-encoding",
"pin-project-lite",
"pin-utils",
"tokio",
"tokio-util",
"tracing",
]
[[package]]
name = "aws-smithy-http-tower"
version = "0.55.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3ae4f6c5798a247fac98a867698197d9ac22643596dc3777f0c76b91917616b9"
dependencies = [
"aws-smithy-http",
"aws-smithy-types",
"bytes",
"http",
"http-body",
"pin-project-lite",
"tower",
"tracing",
]
[[package]]
name = "aws-smithy-json"
version = "0.55.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "23f9f42fbfa96d095194a632fbac19f60077748eba536eb0b9fecc28659807f8"
dependencies = [
"aws-smithy-types",
]
[[package]]
name = "aws-smithy-query"
version = "0.55.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "98819eb0b04020a1c791903533b638534ae6c12e2aceda3e6e6fba015608d51d"
dependencies = [
"aws-smithy-types",
"urlencoding",
]
[[package]]
name = "aws-smithy-types"
version = "0.55.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "16a3d0bf4f324f4ef9793b86a1701d9700fbcdbd12a846da45eed104c634c6e8"
dependencies = [
"base64-simd",
"itoa",
"num-integer",
"ryu",
"time",
]
[[package]]
name = "aws-smithy-xml"
version = "0.55.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b1b9d12875731bd07e767be7baad95700c3137b56730ec9ddeedb52a5e5ca63b"
dependencies = [
"xmlparser",
]
[[package]]
name = "aws-types"
version = "0.55.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6dd209616cc8d7bfb82f87811a5c655dc97537f592689b18743bddf5dc5c4829"
dependencies = [
"aws-credential-types",
"aws-smithy-async",
"aws-smithy-client",
"aws-smithy-http",
"aws-smithy-types",
"http",
"rustc_version 0.4.0",
"tracing",
]
[[package]]
name = "axum"
version = "0.6.18"
@@ -195,6 +524,16 @@ version = "0.21.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "604178f6c5c21f02dc555784810edfb88d34ac2c73b2eae109655649ee73ce3d"
[[package]]
name = "base64-simd"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "339abbe78e73178762e23bea9dfd08e697eb3f3301cd4be981c0f78ba5859195"
dependencies = [
"outref",
"vsimd",
]
[[package]]
name = "bcrypt"
version = "0.14.0"
@@ -323,6 +662,16 @@ version = "1.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be"
[[package]]
name = "bytes-utils"
version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e47d3a8076e283f3acd27400535992edb3ba4b5bb72f8891ad8fbe7932a7d4b9"
dependencies = [
"bytes",
"either",
]
[[package]]
name = "cc"
version = "1.0.79"
@@ -1055,6 +1404,21 @@ dependencies = [
"want",
]
[[package]]
name = "hyper-rustls"
version = "0.23.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1788965e61b367cd03a62950836d5cd41560c3577d90e40e0819373194d1661c"
dependencies = [
"http",
"hyper",
"log",
"rustls",
"rustls-native-certs",
"tokio",
"tokio-rustls",
]
[[package]]
name = "hyper-tls"
version = "0.5.0"
@@ -1455,6 +1819,8 @@ dependencies = [
"anyhow",
"async-trait",
"async_timing_util",
"aws-config",
"aws-sdk-ec2",
"axum",
"bcrypt",
"dotenv",
@@ -1549,16 +1915,16 @@ dependencies = [
"resolver_api",
"serde",
"serde_json",
"strum",
"strum_macros",
"strum 0.25.0",
"strum_macros 0.25.0",
"typeshare",
]
[[package]]
name = "mungos"
version = "0.4.2"
version = "0.4.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "996c88d186685b01835dbfcfa061b94de43b370d64273fd80751c8fac0d406d4"
checksum = "78e0e07595834795a2884abf009363c3884f3e5ca0ed63748e586aa62efcd26b"
dependencies = [
"anyhow",
"async-trait",
@@ -1572,9 +1938,9 @@ dependencies = [
[[package]]
name = "mungos_derive"
version = "0.4.1"
version = "0.4.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0035357ee0b70d919d1767f06d180bb098bfb59cbf4b74cb8540544530de1862"
checksum = "11f93e42a586114214dd192d81518125f8d7fe6deda892c19f98f4d22c1cf905"
dependencies = [
"proc-macro2",
"quote",
@@ -1608,6 +1974,16 @@ dependencies = [
"winapi",
]
[[package]]
name = "num-integer"
version = "0.1.45"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9"
dependencies = [
"autocfg",
"num-traits",
]
[[package]]
name = "num-traits"
version = "0.2.15"
@@ -1686,6 +2062,12 @@ dependencies = [
"vcpkg",
]
[[package]]
name = "outref"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4030760ffd992bef45b0ae3f10ce1aba99e33464c90d14dd7c039884963ddc7a"
[[package]]
name = "parking_lot"
version = "0.12.1"
@@ -1889,6 +2271,23 @@ dependencies = [
"bitflags",
]
[[package]]
name = "regex"
version = "1.8.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d0ab3ca65655bb1e41f2a8c8cd662eb4fb035e67c3f78da1d61dffe89d07300f"
dependencies = [
"aho-corasick",
"memchr",
"regex-syntax",
]
[[package]]
name = "regex-syntax"
version = "0.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "436b050e76ed2903236f032a59761c1eb99e1b0aead2c257922771dab1fc8c78"
[[package]]
name = "reqwest"
version = "0.11.18"
@@ -2038,6 +2437,18 @@ dependencies = [
"webpki",
]
[[package]]
name = "rustls-native-certs"
version = "0.6.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00"
dependencies = [
"openssl-probe",
"rustls-pemfile",
"schannel",
"security-framework",
]
[[package]]
name = "rustls-pemfile"
version = "1.0.2"
@@ -2393,6 +2804,12 @@ version = "0.24.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "063e6045c0e62079840579a7e47a355ae92f60eb74daaf156fb1e84ba164e63f"
[[package]]
name = "strum"
version = "0.25.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "290d54ea6f91c969195bdbcd7442c8c2a2ba87da8bf60a7ee86a235d4bc1e125"
[[package]]
name = "strum_macros"
version = "0.24.3"
@@ -2406,6 +2823,19 @@ dependencies = [
"syn 1.0.109",
]
[[package]]
name = "strum_macros"
version = "0.25.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fe9f3bd7d2e45dcc5e265fbb88d6513e4747d8ef9444cf01a533119bce28a157"
dependencies = [
"heck",
"proc-macro2",
"quote",
"rustversion",
"syn 2.0.18",
]
[[package]]
name = "subtle"
version = "2.5.0"
@@ -2635,6 +3065,17 @@ dependencies = [
"webpki",
]
[[package]]
name = "tokio-stream"
version = "0.1.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "397c988d37662c7dda6d2208364a706264bf3d6138b11d436cbac0ad38832842"
dependencies = [
"futures-core",
"pin-project-lite",
"tokio",
]
[[package]]
name = "tokio-tungstenite"
version = "0.18.0"
@@ -2758,9 +3199,21 @@ dependencies = [
"cfg-if",
"log",
"pin-project-lite",
"tracing-attributes",
"tracing-core",
]
[[package]]
name = "tracing-attributes"
version = "0.1.26"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.18",
]
[[package]]
name = "tracing-core"
version = "0.1.31"
@@ -2967,6 +3420,12 @@ version = "0.9.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
[[package]]
name = "vsimd"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64"
[[package]]
name = "want"
version = "0.3.0"
@@ -3275,6 +3734,12 @@ dependencies = [
"tap",
]
[[package]]
name = "xmlparser"
version = "0.13.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4d25c75bf9ea12c4040a97f829154768bbbce366287e2dc044af160cd79a13fd"
[[package]]
name = "zeroize"
version = "1.6.0"

View File

@@ -38,8 +38,8 @@ bson = "2.6.1"
bollard = "0.14.0"
derive_builder = "0.12"
typeshare = "1.0.1"
strum = "0.24"
strum_macros = "0.24"
strum = "0.25"
strum_macros = "0.25"
sysinfo = "0.29"
async-trait = "0.1"
urlencoding = "2.1"
@@ -48,6 +48,8 @@ jwt = "0.16"
hmac = "0.12"
sha2 = "0.10"
bcrypt = "0.14"
aws-config = "0.55"
aws-sdk-ec2 = "0.28"
proc-macro2 = "1.0"
quote = "1.0"
syn = "2.0"
@@ -61,6 +63,6 @@ partial_derive2 = "0.1.4"
make_option = "0.1.3"
resolver_api = "0.1.5"
parse_csl = "0.1.0"
mungos = "0.4.2"
mungos = "0.4.4"
svi = "0.1.4"

View File

@@ -35,6 +35,8 @@ sha2.workspace = true
bcrypt.workspace = true
async-trait.workspace = true
futures.workspace = true
aws-config.workspace = true
aws-sdk-ec2.workspace = true
# mogh
async_timing_util.workspace = true
merge_config_files.workspace = true

192
core/src/cloud/aws.rs Normal file
View File

@@ -0,0 +1,192 @@
use std::time::Duration;
use anyhow::{anyhow, Context};
use aws_sdk_ec2::{
config::Region,
types::{
BlockDeviceMapping, EbsBlockDevice, InstanceNetworkInterfaceSpecification,
InstanceStateChange, InstanceStateName, InstanceStatus, InstanceType, ResourceType, Tag,
TagSpecification,
},
Client,
};
use monitor_types::entities::builder::AwsBuilder;
use crate::state::State;
const POLL_RATE_SECS: u64 = 2;
const MAX_POLL_TRIES: usize = 30;
pub struct Ec2Instance {
pub instance_id: String,
pub ip: String,
}
impl State {
async fn create_ec2_client(&self, region: String) -> Client {
// There may be a better way to pass these keys to client
std::env::set_var("AWS_ACCESS_KEY_ID", &self.config.aws.access_key_id);
std::env::set_var("AWS_SECRET_ACCESS_KEY", &self.config.aws.secret_access_key);
let region = Region::new(region);
let config = aws_config::from_env().region(region).load().await;
Client::new(&config)
}
pub async fn create_ec2_instance(
&self,
instance_name: &str,
AwsBuilder {
region,
instance_type,
volume_gb,
ami_id,
subnet_id,
security_group_ids,
key_pair_name,
assign_public_ip,
}: &AwsBuilder,
) -> anyhow::Result<Ec2Instance> {
let instance_type = InstanceType::from(instance_type.as_str());
if let InstanceType::Unknown(t) = instance_type {
return Err(anyhow!("unknown instance type {t:?}"));
}
let client = self.create_ec2_client(region.clone()).await;
let res = client
.run_instances()
.image_id(ami_id)
.instance_type(instance_type)
.block_device_mappings(
BlockDeviceMapping::builder()
.set_device_name(String::from("/dev/sda1").into())
.set_ebs(
EbsBlockDevice::builder()
.volume_size(*volume_gb)
.build()
.into(),
)
.build(),
)
.network_interfaces(
InstanceNetworkInterfaceSpecification::builder()
.subnet_id(subnet_id)
.associate_public_ip_address(*assign_public_ip)
.set_groups(security_group_ids.to_vec().into())
.device_index(0)
.build(),
)
.key_name(key_pair_name)
.tag_specifications(
TagSpecification::builder()
.tags(Tag::builder().key("Name").value(instance_name).build())
.resource_type(ResourceType::Instance)
.build(),
)
.min_count(1)
.max_count(1)
.send()
.await
.context("failed to start builder ec2 instance")?;
let instance = res
.instances()
.ok_or(anyhow!("got None for created instances"))?
.get(0)
.ok_or(anyhow!("instances array is empty"))?;
let instance_id = instance
.instance_id()
.ok_or(anyhow!("instance does not have instance_id"))?
.to_string();
for _ in 0..MAX_POLL_TRIES {
let state_name = get_ec2_instance_state_name(&client, &instance_id).await?;
if state_name == Some(InstanceStateName::Running) {
let ip = if *assign_public_ip {
get_ec2_instance_public_ip(&client, &instance_id).await?
} else {
instance
.private_ip_address()
.ok_or(anyhow!("instance does not have private ip"))?
.to_string()
};
return Ok(Ec2Instance { instance_id, ip });
}
tokio::time::sleep(Duration::from_secs(POLL_RATE_SECS)).await;
}
Err(anyhow!("instance not running after polling"))
}
pub async fn terminate_ec2_instance(
&self,
region: String,
instance_id: &str,
) -> anyhow::Result<InstanceStateChange> {
let client = self.create_ec2_client(region).await;
let res = client
.terminate_instances()
.instance_ids(instance_id)
.send()
.await
.context("failed to terminate instance from aws")?
.terminating_instances()
.ok_or(anyhow!("terminating instances is None"))?
.get(0)
.ok_or(anyhow!("terminating instances is empty"))?
.to_owned();
Ok(res)
}
}
async fn get_ec2_instance_status(
client: &Client,
instance_id: &str,
) -> anyhow::Result<Option<InstanceStatus>> {
let status = client
.describe_instance_status()
.instance_ids(instance_id)
.send()
.await
.context("failed to get instance status from aws")?
.instance_statuses()
.ok_or(anyhow!("instance statuses is None"))?
.get(0)
.map(|s| s.to_owned());
Ok(status)
}
async fn get_ec2_instance_state_name(
client: &Client,
instance_id: &str,
) -> anyhow::Result<Option<InstanceStateName>> {
let status = get_ec2_instance_status(client, instance_id).await?;
if status.is_none() {
return Ok(None);
}
let state = status
.unwrap()
.instance_state()
.ok_or(anyhow!("instance state is None"))?
.name()
.ok_or(anyhow!("instance state name is None"))?
.to_owned();
Ok(Some(state))
}
async fn get_ec2_instance_public_ip(client: &Client, instance_id: &str) -> anyhow::Result<String> {
let ip = client
.describe_instances()
.instance_ids(instance_id)
.send()
.await
.context("failed to get instance status from aws")?
.reservations()
.ok_or(anyhow!("instance reservations is None"))?
.get(0)
.ok_or(anyhow!("instance reservations is empty"))?
.instances()
.ok_or(anyhow!("instances is None"))?
.get(0)
.ok_or(anyhow!("instances is empty"))?
.public_ip_address()
.ok_or(anyhow!("instance has no public ip"))?
.to_string();
Ok(ip)
}

5
core/src/cloud/mod.rs Normal file
View File

@@ -0,0 +1,5 @@
pub mod aws;
pub enum InstanceCleanupData {
Aws { instance_id: String, region: String, }
}

View File

@@ -1,5 +1,6 @@
use monitor_types::entities::{
build::Build, deployment::Deployment, server::Server, update::Update, user::User,
build::Build, builder::Builder, deployment::Deployment, server::Server, update::Update,
user::User,
};
use mungos::{Collection, Indexed, Mungos};
@@ -10,6 +11,7 @@ pub struct DbClient {
pub servers: Collection<Server>,
pub deployments: Collection<Deployment>,
pub builds: Collection<Build>,
pub builders: Collection<Builder>,
pub updates: Collection<Update>,
}
@@ -25,8 +27,8 @@ impl DbClient {
servers: Server::collection(&mungos, &config.mongo.db_name, true).await?,
deployments: Deployment::collection(&mungos, &config.mongo.db_name, true).await?,
builds: Build::collection(&mungos, &config.mongo.db_name, true).await?,
builders: Builder::collection(&mungos, &config.mongo.db_name, true).await?,
updates: Update::collection(&mungos, &config.mongo.db_name, true).await?,
// mungos,
};
Ok(client)
}

View File

@@ -9,7 +9,7 @@ use monitor_types::{
server::{Server, ServerStatus},
update::Update,
user::User,
PermissionLevel,
PermissionLevel, builder::Builder,
},
permissioned::Permissioned,
};
@@ -201,6 +201,45 @@ impl State {
Ok(build.get_user_permissions(user_id))
}
pub async fn get_builder(&self, builder_id: &str) -> anyhow::Result<Builder> {
self.db
.builders
.find_one_by_id(builder_id)
.await?
.context(format!("did not find any builder with id {builder_id}"))
}
pub async fn get_builder_check_permissions(
&self,
builder_id: &str,
user: &RequestUser,
permission_level: PermissionLevel,
) -> anyhow::Result<Builder> {
let builder = self
.db
.builders
.find_one_by_id(builder_id)
.await?
.context(format!("did not find any builder with id {builder_id}"))?;
let permissions = builder.get_user_permissions(&user.id);
if user.is_admin || permissions >= permission_level {
Ok(builder)
} else {
Err(anyhow!(
"user does not have required permissions on this builder"
))
}
}
pub async fn get_user_permission_on_builder(
&self,
user_id: &str,
builder_id: &str,
) -> anyhow::Result<PermissionLevel> {
let builder = self.get_builder(builder_id).await?;
Ok(builder.get_user_permissions(user_id))
}
pub async fn send_update(&self, update: Update) -> anyhow::Result<()> {
self.update.sender.lock().await.send(update)?;
Ok(())

View File

@@ -5,6 +5,7 @@ use axum::{Extension, Router};
use termination_signal::tokio::immediate_term_handle;
mod auth;
mod cloud;
mod config;
mod db;
mod helpers;

View File

@@ -1,19 +1,29 @@
use std::pin::Pin;
use anyhow::{anyhow, Context};
use async_trait::async_trait;
use futures::Future;
use monitor_helpers::{all_logs_success, monitor_timestamp};
use monitor_types::{
entities::{
build::Build,
build::{Build, BuildBuilderConfig},
builder::{AwsBuilder, BuilderConfig},
update::{Log, Update, UpdateStatus, UpdateTarget},
Operation, PermissionLevel,
},
permissioned::Permissioned,
requests::api::{CreateBuild, DeleteBuild, GetBuild, ListBuilds, UpdateBuild},
requests::api::{CreateBuild, DeleteBuild, GetBuild, ListBuilds, RunBuild, UpdateBuild},
};
use mungos::mongodb::bson::{doc, to_bson};
use periphery_client::PeripheryClient;
use resolver_api::Resolve;
use crate::{auth::RequestUser, helpers::empty_or_only_spaces, state::State};
use crate::{
auth::RequestUser,
cloud::{aws::Ec2Instance, InstanceCleanupData},
helpers::empty_or_only_spaces,
state::State,
};
#[async_trait]
impl Resolve<GetBuild, RequestUser> for State {
@@ -57,7 +67,7 @@ impl Resolve<CreateBuild, RequestUser> for State {
CreateBuild { name, config }: CreateBuild,
user: RequestUser,
) -> anyhow::Result<Build> {
if let Some(server_id) = &config.server_id {
if let Some(BuildBuilderConfig::Server { server_id }) = &config.builder {
self.get_server_check_permissions(server_id, &user, PermissionLevel::Update)
.await
.context("cannot create build on this server")?;
@@ -233,3 +243,162 @@ impl Resolve<UpdateBuild, RequestUser> for State {
res
}
}
#[async_trait]
impl Resolve<RunBuild, RequestUser> for State {
async fn resolve(
&self,
RunBuild { build_id }: RunBuild,
user: RequestUser,
) -> anyhow::Result<Update> {
if self.action_states.build.busy(&build_id).await {
return Err(anyhow!("build busy"));
}
let mut build = self
.get_build_check_permissions(&build_id, &user, PermissionLevel::Execute)
.await?;
let inner = || async move {
build.config.version.increment();
let mut update = Update {
target: UpdateTarget::Build(build.id.clone()),
operation: Operation::RunBuild,
start_ts: monitor_timestamp(),
status: UpdateStatus::InProgress,
success: true,
operator: user.id.clone(),
version: build.config.version.clone(),
..Default::default()
};
update.id = self.add_update(update.clone()).await?;
let builder = self.get_build_builder(&build, &mut update).await;
if let Err(e) = &builder {
update
.logs
.push(Log::error("get builder", format!("{e:#?}")));
update.finalize();
self.update_update(update.clone()).await?;
return Ok(update);
}
let (periphery, cleanup_data) = builder.unwrap();
// ...
self.cleanup_builder_instance(cleanup_data, &mut update)
.await;
update.finalize();
self.update_update(update.clone()).await?;
Ok(update)
};
self.action_states
.build
.update_entry(build_id.clone(), |entry| {
entry.building = true;
})
.await;
let res = inner().await;
self.action_states
.build
.update_entry(build_id, |entry| {
entry.building = false;
})
.await;
res
}
}
impl State {
async fn get_build_builder(
&self,
build: &Build,
update: &mut Update,
) -> anyhow::Result<(PeripheryClient, Option<InstanceCleanupData>)> {
match &build.config.builder {
BuildBuilderConfig::Server { server_id } => {
let server = self.get_server(server_id).await?;
let periphery = self.periphery_client(&server);
Ok((periphery, None))
}
BuildBuilderConfig::Builder { builder_id } => {
let builder = self.get_builder(builder_id).await?;
match builder.config {
BuilderConfig::AwsBuilder(config) => {
self.get_aws_builder(build, config, update).await
}
}
}
}
}
async fn get_aws_builder(
&self,
build: &Build,
builder: AwsBuilder,
update: &mut Update,
) -> anyhow::Result<(PeripheryClient, Option<InstanceCleanupData>)> {
let instance_name = format!(
"BUILDER-{}-v{}",
build.name,
build.config.version.to_string()
);
let Ec2Instance { instance_id, ip } =
self.create_ec2_instance(&instance_name, &builder).await?;
update
.logs
.push(Log::simple("started builder instance", format!("")));
self.update_update(update.clone()).await?;
let periphery = PeripheryClient::new(format!("http://{ip}:8000"), &self.config.passkey);
Ok((
periphery,
InstanceCleanupData::Aws {
instance_id,
region: builder.region,
}
.into(),
))
}
async fn cleanup_builder_instance(
&self,
cleanup_data: Option<InstanceCleanupData>,
update: &mut Update,
) {
if cleanup_data.is_none() {
return;
}
match cleanup_data.unwrap() {
InstanceCleanupData::Aws {
instance_id,
region,
} => {
let res = self
.terminate_ec2_instance(region, &instance_id)
.await
.context("failed to terminate ec2 instance");
let log = match res {
Ok(_) => Log::simple(
"terminate instance",
format!("terminate instance id {}", instance_id),
),
Err(e) => Log::error("terminate instance", format!("{e:#?}")),
};
update.logs.push(log);
}
}
}
}

View File

@@ -182,6 +182,12 @@ impl State {
let permissions = self.get_user_permission_on_build(user_id, build_id).await?;
(permissions, "build")
}
UpdateTarget::Builder(builder_id) => {
let permissions = self
.get_user_permission_on_builder(user_id, builder_id)
.await?;
(permissions, "builder")
}
// UpdateTarget::Procedure(procedure_id) => {
// let permissions = db_client
// .get_user_permission_on_procedure(user_id, procedure_id)

View File

@@ -1,4 +1,4 @@
use bson::serde_helpers::hex_string_as_object_id;
use bson::{doc, serde_helpers::hex_string_as_object_id};
use derive_builder::Builder;
use mungos::MungosIndexed;
use partial_derive2::Partial;
@@ -47,11 +47,11 @@ pub struct Build {
#[derive(Serialize, Deserialize, Debug, Clone, Builder, Partial, MungosIndexed)]
#[partial_derive(Serialize, Deserialize, Debug, Clone)]
#[skip_serializing_none]
#[doc_index(doc! { "builder.type": 1 })]
#[sparse_doc_index(doc! { "builder.params.server_id": 1 })]
#[sparse_doc_index(doc! { "builder.params.builder_id": 1 })]
pub struct BuildConfig {
#[index]
#[serde(default)]
#[builder(default)]
pub server_id: String,
pub builder: BuildBuilderConfig,
#[serde(default)]
#[builder(default)]
@@ -125,7 +125,7 @@ fn default_dockerfile_path() -> String {
impl From<PartialBuildConfig> for BuildConfig {
fn from(value: PartialBuildConfig) -> BuildConfig {
BuildConfig {
server_id: value.server_id.unwrap_or_default(),
builder: value.builder.unwrap_or_default(),
skip_secret_interp: value.skip_secret_interp.unwrap_or_default(),
version: value.version.unwrap_or_default(),
repo: value.repo.unwrap_or_default(),
@@ -150,3 +150,17 @@ pub struct BuildActionState {
pub building: bool,
pub updating: bool,
}
#[typeshare]
#[derive(Serialize, Deserialize, Debug, Clone, MungosIndexed)]
#[serde(tag = "type", content = "params")]
pub enum BuildBuilderConfig {
Server { server_id: String },
Builder { builder_id: String },
}
impl Default for BuildBuilderConfig {
fn default() -> Self {
Self::Server { server_id: Default::default() }
}
}

View File

@@ -0,0 +1,84 @@
use bson::serde_helpers::hex_string_as_object_id;
use derive_builder::Builder;
use mungos::MungosIndexed;
use serde::{Deserialize, Serialize};
use typeshare::typeshare;
use crate::{i64_is_zero, I64};
use super::PermissionsMap;
#[typeshare]
#[derive(Serialize, Deserialize, Debug, Clone, Builder, MungosIndexed)]
pub struct Builder {
#[serde(
default,
rename = "_id",
skip_serializing_if = "String::is_empty",
with = "hex_string_as_object_id"
)]
#[builder(setter(skip))]
pub id: String,
#[unique_index]
pub name: String,
#[serde(default)]
#[builder(default)]
pub description: String,
#[serde(default)]
#[builder(setter(skip))]
pub permissions: PermissionsMap,
#[serde(default, skip_serializing_if = "i64_is_zero")]
#[builder(setter(skip))]
pub created_at: I64,
#[serde(default)]
#[builder(setter(skip))]
pub updated_at: I64,
pub config: BuilderConfig,
}
#[typeshare]
#[derive(Serialize, Deserialize, Debug, Clone, MungosIndexed)]
#[serde(tag = "type", content = "params")]
pub enum BuilderConfig {
AwsBuilder(AwsBuilder),
}
#[typeshare]
#[derive(Serialize, Deserialize, Debug, Clone, Builder, MungosIndexed)]
pub struct AwsBuilder {
#[serde(default = "default_region")]
#[builder(default = "default_region()")]
pub region: String,
#[serde(default = "default_instance_type")]
#[builder(default = "default_instance_type()")]
pub instance_type: String,
#[serde(default = "default_volume_gb")]
#[builder(default = "default_volume_gb()")]
pub volume_gb: i32,
pub ami_id: String,
pub subnet_id: String,
pub security_group_ids: Vec<String>,
pub key_pair_name: String,
pub assign_public_ip: bool,
}
fn default_region() -> String {
String::from("us-east-1")
}
fn default_instance_type() -> String {
String::from("c5.2xlarge")
}
fn default_volume_gb() -> i32 {
20
}

View File

@@ -7,6 +7,7 @@ use strum_macros::{Display, EnumString};
use typeshare::typeshare;
pub mod build;
pub mod builder;
pub mod deployment;
pub mod repo;
pub mod server;
@@ -199,16 +200,7 @@ impl Default for &PermissionLevel {
#[typeshare]
#[derive(
Serialize,
Deserialize,
Debug,
Default,
PartialEq,
Hash,
Eq,
Clone,
Copy,
MungosIndexed,
Serialize, Deserialize, Debug, Default, PartialEq, Hash, Eq, Clone, Copy, MungosIndexed,
)]
pub enum Operation {
// do nothing
@@ -228,7 +220,7 @@ pub enum Operation {
CreateBuild,
UpdateBuild,
DeleteBuild,
BuildBuild,
RunBuild,
// deployment
CreateDeployment,

View File

@@ -29,6 +29,23 @@ pub struct Update {
pub version: Version,
}
impl Update {
pub fn finalize(&mut self) {
self.success = all_logs_success(&self.logs);
self.end_ts = Some(unix_timestamp_ms() as i64);
self.status = UpdateStatus::Complete;
}
}
fn all_logs_success(logs: &Vec<Log>) -> bool {
for log in logs {
if !log.success {
return false;
}
}
true
}
#[typeshare]
#[derive(Serialize, Deserialize, Debug, Clone, Default)]
pub struct Log {
@@ -74,6 +91,7 @@ pub enum UpdateTarget {
#[default]
System,
Build(String),
Builder(String),
Deployment(String),
Server(String),
// Procedure(String),

View File

@@ -1,4 +1,4 @@
use crate::entities::{PermissionsMap, PermissionLevel, deployment::Deployment, build::Build, server::Server};
use crate::entities::{PermissionsMap, PermissionLevel, deployment::Deployment, build::Build, server::Server, builder::Builder};
pub trait Permissioned {
fn permissions_map(&self) -> &PermissionsMap;
@@ -24,4 +24,10 @@ impl Permissioned for Server {
fn permissions_map(&self) -> &PermissionsMap {
&self.permissions
}
}
impl Permissioned for Builder {
fn permissions_map(&self) -> &PermissionsMap {
&self.permissions
}
}