update aws sdk verison and implement merge_config_files

2023-04-01 07:06:17 +00:00
parent 84d45c5df8
commit 10b6a9482b
10 changed files with 82 additions and 167 deletions
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -86,9 +86,9 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"

 [[package]]
 name = "aws-config"
-version = "0.54.1"
+version = "0.55.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3c3d1e2a1f1ab3ac6c4b884e37413eaa03eb9d901e4fc68ee8f5c1d49721680e"
+checksum = "1854be4730cc87602316707045a5c0585287419d54f293bbb52a82c895d9086a"
 dependencies = [
 "aws-credential-types",
 "aws-http",
@@ -102,6 +102,7 @@ dependencies = [
 "aws-smithy-types",
 "aws-types",
 "bytes",
+ "fastrand",
 "hex",
 "http",
 "hyper",
@@ -115,12 +116,13 @@ dependencies = [

 [[package]]
 name = "aws-credential-types"
-version = "0.54.1"
+version = "0.55.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bb0696a0523a39a19087747e4dafda0362dc867531e3d72a3f195564c84e5e08"
+checksum = "77e37e62f59cf3284067337da7467d842df8cfe3f5e5c06487ac7521819cf16d"
 dependencies = [
 "aws-smithy-async",
 "aws-smithy-types",
+ "fastrand",
 "tokio",
 "tracing",
 "zeroize",
@@ -128,9 +130,9 @@ dependencies = [

 [[package]]
 name = "aws-endpoint"
-version = "0.54.1"
+version = "0.55.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "80a4f935ab6a1919fbfd6102a80c4fccd9ff5f47f94ba154074afe1051903261"
+checksum = "9f38276d5875b19a9bb2b4ae049fd776c932fcc62068f78b71ce475093ccb4c8"
 dependencies = [
 "aws-smithy-http",
 "aws-smithy-types",
@@ -142,9 +144,9 @@ dependencies = [

 [[package]]
 name = "aws-http"
-version = "0.54.1"
+version = "0.55.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "82976ca4e426ee9ca3ffcf919d9b2c8d14d0cd80d43cc02173737a8f07f28d4d"
+checksum = "67224bfece71e21a63ae82b1ebbfda05be28678a0fab06def03229c7a445d3fb"
 dependencies = [
 "aws-credential-types",
 "aws-smithy-http",
@@ -161,9 +163,9 @@ dependencies = [

 [[package]]
 name = "aws-sdk-ec2"
-version = "0.24.0"
+version = "0.25.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b40ee2d853d8300a49513778beb79b1574ff9e9c94b30b1531bc0171d730ad64"
+checksum = "fd180c0cb50c9b4306288a9c67f51a0e252ce09f05462b34a437b8b37107982e"
 dependencies = [
 "aws-credential-types",
 "aws-endpoint",
@@ -189,9 +191,9 @@ dependencies = [

 [[package]]
 name = "aws-sdk-sso"
-version = "0.24.0"
+version = "0.25.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ca0119bacf0c42f587506769390983223ba834e605f049babe514b2bd646dbb2"
+checksum = "f4c10657158e12163d6b3fb1e0c9154e43656843794a3071d9ee63ec82a4de2d"
 dependencies = [
 "aws-credential-types",
 "aws-endpoint",
@@ -209,13 +211,14 @@ dependencies = [
 "regex",
 "tokio-stream",
 "tower",
+ "tracing",
 ]

 [[package]]
 name = "aws-sdk-sts"
-version = "0.24.0"
+version = "0.25.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "270b6a33969ebfcb193512fbd5e8ee5306888ad6c6d5d775cdbfb2d50d94de26"
+checksum = "706a308b7277ac9aac78ab37933509659c6f978a8473e95d8e7a8103093133c3"
 dependencies = [
 "aws-credential-types",
 "aws-endpoint",
@@ -239,9 +242,9 @@ dependencies = [

 [[package]]
 name = "aws-sig-auth"
-version = "0.54.1"
+version = "0.55.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "660a02a98ab1af83bd8d714afbab2d502ba9b18c49e7e4cddd6bf8837ff778cb"
+checksum = "cebfa0f118afd7197185d5e097bfcdfca9f8410dca50435d67784405f1fd5a05"
 dependencies = [
 "aws-credential-types",
 "aws-sigv4",
@@ -253,9 +256,9 @@ dependencies = [

 [[package]]
 name = "aws-sigv4"
-version = "0.54.1"
+version = "0.55.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cdaf11005b7444e6cd66f600d09861a3aeb6eb89a0f003c7c9820dbab2d15297"
+checksum = "19a4f5c05c8646d12b7bb3f18c04edc5ac5e8928ab80e1649e568190f2bc7b79"
 dependencies = [
 "aws-smithy-http",
 "form_urlencoded",
@@ -272,9 +275,9 @@ dependencies = [

 [[package]]
 name = "aws-smithy-async"
-version = "0.54.4"
+version = "0.55.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "63c712a28a4f2f2139759235c08bf98aca99d4fdf1b13c78c5f95613df0a5db9"
+checksum = "8cd4b9b7d99263f75304fc1fcd752361cbc4cbf068b832acd8daeaaff44267eb"
 dependencies = [
 "futures-util",
 "pin-project-lite",
@@ -284,9 +287,9 @@ dependencies = [

 [[package]]
 name = "aws-smithy-client"
-version = "0.54.4"
+version = "0.55.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "104ca17f56cde00a10207169697dfe9c6810db339d52fb352707e64875b30a44"
+checksum = "748298b60bbd0594223ea136ceed2ed4b6d50970bcefa69a5ff6d710ce593854"
 dependencies = [
 "aws-smithy-async",
 "aws-smithy-http",
@@ -300,6 +303,7 @@ dependencies = [
 "hyper-rustls",
 "lazy_static",
 "pin-project-lite",
+ "rustls",
 "tokio",
 "tower",
 "tracing",
@@ -307,9 +311,9 @@ dependencies = [

 [[package]]
 name = "aws-smithy-http"
-version = "0.54.4"
+version = "0.55.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "873f316f1833add0d3aa54ed1b0cd252ddd88c792a0cf839886400099971e844"
+checksum = "d78510732b81040689dc146e3693bfbcf388ab88cbda667d3ef67f8869b0744a"
 dependencies = [
 "aws-smithy-types",
 "bytes",
@@ -329,9 +333,9 @@ dependencies = [

 [[package]]
 name = "aws-smithy-http-tower"
-version = "0.54.4"
+version = "0.55.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4f38231d3f5dac9ac7976f44e12803add1385119ffca9e5f050d8e980733d164"
+checksum = "dc33689c27bbd8184412b45c4d1ab795d9a35402562d9fde6c53695a90969740"
 dependencies = [
 "aws-smithy-http",
 "aws-smithy-types",
@@ -345,18 +349,18 @@ dependencies = [

 [[package]]
 name = "aws-smithy-json"
-version = "0.54.4"
+version = "0.55.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4bd83ff2b79e9f729746fcc8ad798676b68fe6ea72986571569a5306a277a182"
+checksum = "ada31cab1b1d1f0abc5c4d1183de5b278597704851aa703801b82feabf19aa74"
 dependencies = [
 "aws-smithy-types",
 ]

 [[package]]
 name = "aws-smithy-query"
-version = "0.54.4"
+version = "0.55.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a2f0445dafe9d2cd50b44339ae3c3ed46549aad8ac696c52ad660b3e7ae8682b"
+checksum = "b55358401b657d192f70f093927f01d73cc4859e2907956b20c4043c76624006"
 dependencies = [
 "aws-smithy-types",
 "urlencoding",
@@ -364,9 +368,9 @@ dependencies = [

 [[package]]
 name = "aws-smithy-types"
-version = "0.54.4"
+version = "0.55.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8161232eda10290f5136610a1eb9de56aceaccd70c963a26a260af20ac24794f"
+checksum = "474d145c2e0f82892841d2502bd546ca0dbc1e4e242c3563d96e7061054c268f"
 dependencies = [
 "base64-simd",
 "itoa",
@@ -377,18 +381,18 @@ dependencies = [

 [[package]]
 name = "aws-smithy-xml"
-version = "0.54.4"
+version = "0.55.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "343ffe9a9bb3f542675f4df0e0d5933513d6ad038ca3907ad1767ba690a99684"
+checksum = "bb159921734d090b01c586a4fef73964f42fcb7eb53a8184b2db374bd6a6fc99"
 dependencies = [
 "xmlparser",
 ]

 [[package]]
 name = "aws-types"
-version = "0.54.1"
+version = "0.55.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f8f15b34253b68cde08e39b0627cc6101bcca64351229484b4743392c035d057"
+checksum = "81fb02591b5075d318e0083dcb76df0e151db4ce48f987ecd00e5b53c7a6ba59"
 dependencies = [
 "aws-credential-types",
 "aws-smithy-async",
@@ -758,6 +762,7 @@ dependencies = [
 "hex",
 "hmac",
 "jwt",
+ "merge_config_files",
 "monitor_helpers",
 "monitor_types 0.2.12",
 "mungos",
@@ -1443,9 +1448,9 @@ checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421"

 [[package]]
 name = "hyper"
-version = "0.14.24"
+version = "0.14.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5e011372fa0b68db8350aa7a248930ecc7839bf46d8485577d69f117a75f164c"
+checksum = "cc5e554ff619822309ffd57d8734d77cd5ce6238bc956f037ea06c58238c9899"
 dependencies = [
 "bytes",
 "futures-channel",
@@ -1768,6 +1773,18 @@ dependencies = [
 "autocfg",
 ]

+[[package]]
+name = "merge_config_files"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5b3bdb9ecb523dd42ecdbd73c127b0f765142ab3399cf890d6fa07605e3e2f57"
+dependencies = [
+ "serde",
+ "serde_json",
+ "thiserror",
+ "toml",
+]
+
 [[package]]
 name = "mime"
 version = "0.3.16"
@@ -1915,6 +1932,7 @@ dependencies = [
 "dotenv",
 "envy",
 "futures",
+ "merge_config_files",
 "monitor_helpers",
 "monitor_types 0.2.12",
 "run_command",
--- a/core/Cargo.toml
+++ b/core/Cargo.toml
@@ -35,5 +35,6 @@ futures-util = "0.3"
 diff-struct = "0.5"
 typeshare = "1.0.0"
 hex = "0.4"
-aws-config = "0.54"
-aws-sdk-ec2 = "0.24"
+aws-config = "0.55"
+aws-sdk-ec2 = "0.25"
+merge_config_files = "0.1.1"
--- a/core/src/actions/build.rs
+++ b/core/src/actions/build.rs
@@ -1,6 +1,7 @@
 use std::time::Duration;

 use anyhow::{anyhow, Context};
+use aws_sdk_ec2::Client;
 use diff::Diff;
 use helpers::{all_logs_success, to_monitor_name};
 use mungos::{doc, to_bson};
@@ -14,7 +15,7 @@ use types::{
 use crate::{
    auth::RequestUser,
    cloud::aws::{
-        self, create_ec2_client, create_instance_with_ami, terminate_ec2_instance, Ec2Instance,
+        create_ec2_client, create_instance_with_ami, terminate_ec2_instance, Ec2Instance,
    },
    helpers::empty_or_only_spaces,
    state::State,
@@ -426,7 +427,7 @@ impl State {
    async fn create_ec2_instance_for_build(
        &self,
        build: &Build,
-    ) -> anyhow::Result<(Ec2Instance, Option<aws::Client>, Vec<Log>)> {
+    ) -> anyhow::Result<(Ec2Instance, Option<Client>, Vec<Log>)> {
        if build.aws_config.is_none() {
            return Err(anyhow!("build has no aws_config attached"));
        }
@@ -527,7 +528,7 @@ impl State {

    async fn terminate_ec2_instance(
        &self,
-        aws_client: aws::Client,
+        aws_client: Client,
        server: &Ec2Instance,
        update: &mut Update,
    ) {
--- a/core/src/cloud/aws.rs
+++ b/core/src/cloud/aws.rs
@@ -1,14 +1,14 @@
 use std::time::Duration;

 use anyhow::{anyhow, Context};
-use aws_sdk_ec2::model::{
-    BlockDeviceMapping, EbsBlockDevice, InstanceNetworkInterfaceSpecification, InstanceStateChange,
-    InstanceStateName, InstanceStatus, ResourceType, Tag, TagSpecification,
-};
-pub use aws_sdk_ec2::{
-    model::InstanceType,
-    output::{DescribeInstanceStatusOutput, TerminateInstancesOutput},
-    Client, Region,
+use aws_sdk_ec2::{
+    config::Region,
+    types::{
+        BlockDeviceMapping, EbsBlockDevice, InstanceNetworkInterfaceSpecification,
+        InstanceStateChange, InstanceStateName, InstanceStatus, InstanceType, ResourceType, Tag,
+        TagSpecification,
+    },
+    Client,
 };
 use types::Server;

--- a/core/src/config.rs
+++ b/core/src/config.rs
@@ -1,6 +1,6 @@
 use axum_extra::routing::SpaRouter;
 use dotenv::dotenv;
-use helpers::parse_config_file;
+use merge_config_files::parse_config_file;
 use mungos::Deserialize;
 use types::CoreConfig;

@@ -15,7 +15,7 @@ struct Env {
 pub fn load() -> (CoreConfig, SpaRouter) {
    dotenv().ok();
    let env: Env = envy::from_env().expect("failed to parse environment variables");
-    let config = parse_config_file(&env.config_path).expect("failed to parse config");
+    let config = parse_config_file(env.config_path).expect("failed to parse config");
    let spa_router = SpaRouter::new("/assets", env.frontend_path);
    (config, spa_router)
 }
--- a/lib/helpers/src/lib.rs
+++ b/lib/helpers/src/lib.rs
@@ -1,113 +1,10 @@
-use std::{borrow::Borrow, fs::File, io::Read, net::SocketAddr, str::FromStr};
+use std::{borrow::Borrow, net::SocketAddr, str::FromStr};

-use anyhow::{anyhow, Context};
+use anyhow::anyhow;
 use axum::http::StatusCode;
 use rand::{distributions::Alphanumeric, Rng};
-use serde::de::DeserializeOwned;
-use serde_json::{Map, Value};
 use types::Log;

-pub fn parse_config_files<'a, T: DeserializeOwned>(
-    paths: impl IntoIterator<Item = impl Borrow<String>>,
-    merge_nested: bool,
-    extend_array: bool,
-) -> anyhow::Result<T> {
-    let mut target = Map::new();
-    for path in paths {
-        target = merge_objects(
-            target,
-            parse_config_file(path.borrow())?,
-            merge_nested,
-            extend_array,
-        )?;
-    }
-    serde_json::from_str(&serde_json::to_string(&target)?)
-        .context("failed to parse final config into expected type")
-}
-
-pub fn parse_config_file<T: DeserializeOwned>(path: &str) -> anyhow::Result<T> {
-    let mut file = File::open(&path).expect(&format!("failed to find config at {path}"));
-    let config = if path.ends_with("toml") {
-        let mut contents = String::new();
-        file.read_to_string(&mut contents)
-            .context(format!("failed to read toml at {path}"))?;
-        toml::from_str(&contents).context(format!("failed to parse toml at {path}"))?
-    } else if path.ends_with("json") {
-        serde_json::from_reader(file).context(format!("failed to parse json at {path}"))?
-    } else {
-        panic!("unsupported config file type: {}", path)
-    };
-    Ok(config)
-}
-
-/// object is serde_json::Map<String, serde_json::Value>
-/// source will overide target
-/// will recurse when field is object if merge_object = true, otherwise object will be replaced
-/// will extend when field is array if extend_array = true, otherwise array will be replaced
-/// will return error when types on source and target fields do not match
-fn merge_objects(
-    mut target: Map<String, Value>,
-    source: Map<String, Value>,
-    merge_nested: bool,
-    extend_array: bool,
-) -> anyhow::Result<Map<String, Value>> {
-    for (key, value) in source {
-        let curr = target.remove(&key);
-        if curr.is_none() {
-            target.insert(key, value);
-            continue;
-        }
-        let curr = curr.unwrap();
-        match curr {
-            Value::Object(target_obj) => {
-                if !merge_nested {
-                    target.insert(key, value);
-                    continue;
-                }
-                match value {
-                    Value::Object(source_obj) => {
-                        target.insert(
-                            key,
-                            Value::Object(merge_objects(
-                                target_obj,
-                                source_obj,
-                                merge_nested,
-                                extend_array,
-                            )?),
-                        );
-                    }
-                    _ => {
-                        return Err(anyhow!(
-                            "types on field {key} do not match. got {value:?}, expected object"
-                        ))
-                    }
-                }
-            }
-            Value::Array(mut target_arr) => {
-                if !extend_array {
-                    target.insert(key, value);
-                    continue;
-                }
-                match value {
-                    Value::Array(source_arr) => {
-                        target_arr.extend(source_arr);
-                        target.insert(key, Value::Array(target_arr));
-                    }
-                    _ => {
-                        return Err(anyhow!(
-                            "types on field {key} do not match. got {value:?}, expected array"
-                        ))
-                    }
-                }
-            }
-            _ => {
-                target.insert(key, value);
-            }
-        }
-    }
-    Ok(target)
-}
-
 pub fn parse_comma_seperated_list<T: FromStr>(
    comma_sep_list: impl Borrow<str>,
 ) -> anyhow::Result<Vec<T>> {
--- a/periphery/Cargo.toml
+++ b/periphery/Cargo.toml
@@ -33,3 +33,4 @@ toml = "0.7"
 daemonize = "0.5.0"
 clap = { version = "4.1", features = ["derive"] }
 svi = "0.1.3"
+merge_config_files = "0.1.1"
--- a/periphery/src/config.rs
+++ b/periphery/src/config.rs
@@ -3,7 +3,8 @@ use std::sync::Arc;
 use axum::Extension;
 use clap::Parser;
 use dotenv::dotenv;
-use helpers::{parse_comma_seperated_list, parse_config_files};
+use helpers::parse_comma_seperated_list;
+use merge_config_files::parse_config_files;
 use serde::Deserialize;
 use types::PeripheryConfig;

@@ -63,9 +64,9 @@ pub fn load() -> (Args, u16, PeripheryConfigExtension, HomeDirExtension) {
        )
        .into_iter()
        .map(|p| p.replace("~", &home_dir))
-        .collect();
+        .collect::<Vec<_>>();
    let config = parse_config_files::<PeripheryConfig>(
-        &config_paths,
+        config_paths.clone(),
        args.merge_nested_config,
        args.merge_nested_config,
    )
--- a/periphery/src/helpers/docker/build.rs
+++ b/periphery/src/helpers/docker/build.rs
@@ -49,11 +49,7 @@ pub async fn build(
    };
    let build_args = parse_build_args(build_args);
    let extra_args = parse_extra_args(extra_args);
-    let buildx = if *use_buildx {
-        " buildx"
-    } else {
-        ""
-    };
+    let buildx = if *use_buildx { " buildx" } else { "" };
    let image_name = get_image_name(&name, docker_account, docker_organization);
    let image_tags = image_tags(&image_name, &version);
    let docker_push = if using_account {
--- a/periphery/src/helpers/docker/container.rs
+++ b/periphery/src/helpers/docker/container.rs
@@ -7,7 +7,7 @@ use types::{
    Conversion, Deployment, DockerContainerStats, DockerRunArgs, EnvironmentVar, Log, RestartMode,
 };

-use crate::helpers::{run_monitor_command, docker::parse_extra_args};
+use crate::helpers::{docker::parse_extra_args, run_monitor_command};

 use super::docker_login;