ComputerSurge/komodo
3
0
Fork 0
forked from github-starred/komodo
Code Pull Requests Packages Releases Activity

clean up ecr

Browse Source
This commit is contained in:
mbecker20
2024-06-23 02:22:14 -07:00
parent bfe762b71a
commit 7d396dd539
13 changed files with 103 additions and 135 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
Cargo.lock generated
View File

@@ -518,17 +518,6 @@ dependencies = [
"tracing",
]
[[package]]
name = "aws_ecr"
version = "1.9.0"
dependencies = [
"anyhow",
"aws-config",
"aws-sdk-ecr",
"run_command",
"tracing",
]
[[package]]
name = "axum"
version = "0.6.20"
@@ -2266,7 +2255,7 @@ dependencies = [
"async_timing_util",
"aws-config",
"aws-sdk-ec2",
"aws_ecr",
"aws-sdk-ecr",
"axum 0.7.5",
"axum-extra",
"base64 0.22.1",
@@ -2292,6 +2281,7 @@ dependencies = [
"rand",
"reqwest 0.12.4",
"resolver_api",
"run_command",
"serde",
"serde_json",
"serror",
@@ -2317,7 +2307,6 @@ version = "1.9.0"
dependencies = [
"anyhow",
"async_timing_util",
"aws_ecr",
"axum 0.7.5",
"axum-extra",
"bollard",

1
Cargo.toml
View File

@@ -19,7 +19,6 @@ monitor_client = "1.8.0"
periphery_client = { path = "client/periphery/rs" }
formatting = { path = "lib/formatting" }
command = { path = "lib/command" }
aws_ecr = { path = "lib/aws_ecr" }
logger = { path = "lib/logger" }
git = { path = "lib/git" }

3
bin/core/Cargo.toml
View File

@@ -18,7 +18,6 @@ path = "src/main.rs"
monitor_client = { workspace = true, features = ["mongo"] }
periphery_client.workspace = true
formatting.workspace = true
aws_ecr.workspace = true
logger.workspace = true
git.workspace = true
# mogh
@@ -30,6 +29,7 @@ derive_variants.workspace = true
mongo_indexed.workspace = true
resolver_api.workspace = true
toml_pretty.workspace = true
run_command.workspace = true
parse_csl.workspace = true
mungos.workspace = true
slack.workspace = true
@@ -38,6 +38,7 @@ svi.workspace = true
ordered_hash_map.workspace = true
urlencoding.workspace = true
aws-sdk-ec2.workspace = true
aws-sdk-ecr.workspace = true
aws-config.workspace = true
tokio-util.workspace = true
axum-extra.workspace = true

37
bin/core/src/api/execute/build.rs
View File

@@ -12,12 +12,13 @@ use monitor_client::{
all_logs_success,
build::{Build, CloudRegistryConfig, ImageRegistry},
builder::{AwsBuilderConfig, Builder, BuilderConfig},
config::core::AwsEcrConfig,
config::core::{AwsEcrConfig, AwsEcrConfigWithCredentials},
deployment::DeploymentState,
monitor_timestamp,
permission::PermissionLevel,
server::{stats::SeverityLevel, Server},
server_template::aws::AwsServerTemplateConfig,
to_monitor_name,
update::{Log, Update},
user::{auto_redeploy_user, User},
},
@@ -39,9 +40,12 @@ use tokio_util::sync::CancellationToken;
use crate::{
cloud::{
aws::ec2::{
launch_ec2_instance, terminate_ec2_instance_with_retry,
Ec2Instance,
aws::{
ec2::{
launch_ec2_instance, terminate_ec2_instance_with_retry,
Ec2Instance,
},
ecr,
},
BuildCleanupData,
},
@@ -762,23 +766,32 @@ async fn validate_account_extract_registry_token_aws_ecr(
ImageRegistry::AwsEcr(label) => {
let config = core_config().aws_ecr_registries.get(label);
let token = match config {
Some(AwsEcrConfig {
Some(AwsEcrConfigWithCredentials {
region,
access_key_id,
secret_access_key,
..
}) => Some(
aws_ecr::get_ecr_token(
region,
}) => {
ecr::maybe_create_repo(
&to_monitor_name(&build.name),
region.to_string(),
access_key_id,
secret_access_key,
)
.await
.context("failed to get aws ecr token")?,
),
.await?;
Some(
ecr::get_ecr_token(
region,
access_key_id,
secret_access_key,
)
.await
.context("failed to get aws ecr token")?,
)
}
None => None,
};
Ok((token, config.cloned()))
Ok((token, config.map(AwsEcrConfig::from)))
}
ImageRegistry::Custom(_) => {
Err(anyhow!("Custom image registry is not implemented"))

16
bin/core/src/api/execute/deployment.rs
View File

@@ -7,6 +7,7 @@ use monitor_client::{
api::execute::*,
entities::{
build::{Build, ImageRegistry},
config::core::AwsEcrConfig,
deployment::{Deployment, DeploymentImage},
get_image_name,
permission::PermissionLevel,
@@ -84,7 +85,10 @@ impl Resolve<Deploy, (User, Update)> for State {
DeploymentImage::Build { build_id, version } => {
let build = resource::get::<Build>(&build_id).await?;
let image_name = get_image_name(&build, |label| {
core_config().aws_ecr_registries.get(label)
core_config()
.aws_ecr_registries
.get(label)
.map(AwsEcrConfig::from)
})
.context("failed to create image name")?;
let version = if version.is_none() {
@@ -177,9 +181,13 @@ impl Resolve<Deploy, (User, Update)> for State {
core_config.github_accounts.get(&params.account).cloned(),
None,
),
ImageRegistry::AwsEcr(label) => {
(None, core_config.aws_ecr_registries.get(label).cloned())
}
ImageRegistry::AwsEcr(label) => (
None,
core_config
.aws_ecr_registries
.get(label)
.map(AwsEcrConfig::from),
),
ImageRegistry::Custom(_) => {
return Err(anyhow!("Custom ImageRegistry not yet supported"))
}

57
lib/aws_ecr/src/lib.rs → bin/core/src/cloud/aws/ecr.rs
View File

@@ -4,7 +4,7 @@ use aws_sdk_ecr::Client as EcrClient;
use run_command::async_run_command;
#[tracing::instrument(skip(access_key_id, secret_access_key))]
pub async fn make_ecr_client(
async fn make_ecr_client(
region: String,
access_key_id: &str,
secret_access_key: &str,
@@ -19,35 +19,16 @@ pub async fn make_ecr_client(
EcrClient::new(&config)
}
/// Gets a token docker login.
///
/// Requires the aws cli be installed on the host
#[tracing::instrument(skip(access_key_id, secret_access_key))]
pub async fn get_ecr_token(
region: &str,
pub async fn maybe_create_repo(
repo: &str,
region: String,
access_key_id: &str,
secret_access_key: &str,
) -> anyhow::Result<String> {
let log = async_run_command(&format!(
"AWS_ACCESS_KEY_ID={access_key_id} AWS_SECRET_ACCESS_KEY={secret_access_key} aws ecr get-login-password --region {region}"
))
.await;
if log.success() {
Ok(log.stdout)
} else {
Err(
anyhow!("stdout: {} | stderr: {}", log.stdout, log.stderr)
.context("failed to get aws ecr login token"),
)
}
}
#[tracing::instrument(skip(client))]
pub async fn maybe_create_repo(
client: &EcrClient,
repo: &str,
) -> anyhow::Result<()> {
let client =
make_ecr_client(region, access_key_id, secret_access_key).await;
let existing = client
.describe_repositories()
.send()
@@ -75,3 +56,27 @@ pub async fn maybe_create_repo(
Ok(())
}
/// Gets a token docker login.
///
/// Requires the aws cli be installed on the host
#[tracing::instrument(skip(access_key_id, secret_access_key))]
pub async fn get_ecr_token(
region: &str,
access_key_id: &str,
secret_access_key: &str,
) -> anyhow::Result<String> {
let log = async_run_command(&format!(
"AWS_ACCESS_KEY_ID={access_key_id} AWS_SECRET_ACCESS_KEY={secret_access_key} aws ecr get-login-password --region {region}"
))
.await;
if log.success() {
Ok(log.stdout)
} else {
Err(
anyhow!("stdout: {} | stderr: {}", log.stdout, log.stderr)
.context("failed to get aws ecr login token"),
)
}
}

1
bin/core/src/cloud/aws/mod.rs
View File

@@ -1 +1,2 @@
pub mod ec2;
pub mod ecr;

1
bin/periphery/Cargo.toml
View File

@@ -19,7 +19,6 @@ monitor_client = { workspace = true, features = ["docker"] }
periphery_client.workspace = true
formatting.workspace = true
command.workspace = true
aws_ecr.workspace = true
logger.workspace = true
git.workspace = true
# mogh

39
bin/periphery/src/docker/build.rs
View File

@@ -2,7 +2,7 @@ use anyhow::Context;
use command::run_monitor_command;
use formatting::format_serror;
use monitor_client::entities::{
build::{Build, BuildConfig, ImageRegistry},
build::{Build, BuildConfig},
config::core::AwsEcrConfig,
get_image_name, optional_string, to_monitor_name,
update::Log,
@@ -68,11 +68,6 @@ pub async fn build(
let name = to_monitor_name(name);
// Only needed for aws ecr
maybe_create_repo(&name, image_registry, aws_ecr)
.await
.context("failed to create new repo for the build")?;
// Get paths
let build_dir =
periphery_config().repo_dir.join(&name).join(build_path);
@@ -82,7 +77,7 @@ pub async fn build(
};
// Get command parts
let image_name = get_image_name(build, |_| aws_ecr)
let image_name = get_image_name(build, |_| aws_ecr.cloned())
.context("failed to make image name")?;
let build_args = parse_build_args(build_args);
let labels = parse_labels(labels);
@@ -145,33 +140,3 @@ fn parse_build_args(build_args: &[EnvironmentVar]) -> String {
.collect::<Vec<_>>()
.join("")
}
#[instrument(skip_all)]
async fn maybe_create_repo(
name: &str,
image_registry: &ImageRegistry,
aws_ecr: Option<&AwsEcrConfig>,
) -> anyhow::Result<()> {
match image_registry {
ImageRegistry::AwsEcr(label) => {
let AwsEcrConfig {
region,
access_key_id,
secret_access_key,
..
} = aws_ecr.with_context(|| {
format!("did not find any aws ecr config for {label}")
})?;
let client = aws_ecr::make_ecr_client(
region.to_string(),
access_key_id,
secret_access_key,
)
.await;
aws_ecr::maybe_create_repo(&client, name).await
}
_ => Ok(()),
}
}

24
bin/periphery/src/docker/mod.rs
View File

@@ -76,26 +76,12 @@ pub async fn docker_login(
}
}
ImageRegistry::AwsEcr(label) => {
let AwsEcrConfig {
region,
account_id,
access_key_id,
secret_access_key,
} = aws_ecr.with_context(|| {
format!("Could not find aws ecr config for label {label}")
})?;
let registry_token = match registry_token {
Some(token) => token.to_string(),
None => aws_ecr::get_ecr_token(
region,
access_key_id,
secret_access_key,
)
.await
let AwsEcrConfig { region, account_id } = aws_ecr
.with_context(|| {
format!("failed to get aws ecr token for {label}")
})?,
};
format!("Could not find aws ecr config for label {label}")
})?;
let registry_token = registry_token
.context("aws ecr build missing registry token from core")?;
let log = async_run_command(&format!("docker login {account_id}.dkr.ecr.{region}.amazonaws.com -u AWS -p {registry_token}")).await;
if log.success() {
Ok(true)

22
client/core/rs/src/entities/config/core.rs
View File

@@ -397,7 +397,7 @@ pub struct CoreConfig {
/// Configure aws ecr registries.
#[serde(default, alias = "aws_ecr_registry")]
pub aws_ecr_registries: HashMap<String, AwsEcrConfig>,
pub aws_ecr_registries: HashMap<String, AwsEcrConfigWithCredentials>,
}
fn default_title() -> String {
@@ -538,7 +538,7 @@ pub struct HetznerCredentials {
/// Provide configuration for an Aws Ecr registry.
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct AwsEcrConfig {
pub struct AwsEcrConfigWithCredentials {
/// The Aws region
pub region: String,
/// The Aws account id
@@ -548,3 +548,21 @@ pub struct AwsEcrConfig {
/// The Aws SECRET_ACCESS_KEY
pub secret_access_key: String,
}
/// Provide configuration for an Aws Ecr registry.
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct AwsEcrConfig {
/// The Aws region
pub region: String,
/// The Aws account id
pub account_id: String,
}
impl AwsEcrConfig {
pub fn from(config: &AwsEcrConfigWithCredentials) -> AwsEcrConfig {
AwsEcrConfig {
region: config.region.to_string(),
account_id: config.account_id.to_string(),
}
}
}

6
client/core/rs/src/entities/mod.rs
View File

@@ -107,13 +107,13 @@ pub fn optional_string(string: &str) -> Option<String> {
}
}
pub fn get_image_name<'a, 'b>(
pub fn get_image_name(
build::Build {
name,
config: build::BuildConfig { image_registry, .. },
..
}: &'a build::Build,
aws_ecr: impl FnOnce(&'a String) -> Option<&'b AwsEcrConfig>,
}: &build::Build,
aws_ecr: impl FnOnce(&String) -> Option<AwsEcrConfig>,
) -> anyhow::Result<String> {
let name = to_monitor_name(name);
let name = match image_registry {

16
lib/aws_ecr/Cargo.toml
View File

@@ -1,16 +0,0 @@
[package]
name = "aws_ecr"
version.workspace = true
edition.workspace = true
authors.workspace = true
license.workspace = true
repository.workspace = true
homepage.workspace = true
[dependencies]
run_command.workspace = true
#
aws-sdk-ecr.workspace = true
aws-config.workspace = true
tracing.workspace = true
anyhow.workspace = true