ComputerSurge/komodo
3
0
Fork 0
forked from github-starred/komodo
Code Pull Requests 1 Actions Packages Releases Activity

sync working clean

Browse Source
This commit is contained in:
mbecker20
2024-03-27 06:35:57 -07:00
parent 73c17c68c4
commit 1b1650eba4
9 changed files with 129 additions and 145 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
bin/monrun/resources/deployments.toml
View File

@@ -1,9 +1,20 @@
[[deployment]]
name = "monitor-core-v1-dev"
description = "v1 dev core api"
tags = ["monitor", "v1", "dev"]
name = "monitor-core-v1"
description = "v1 core api"
tags = ["monitor", "v1", "api"]
config.server = "mogh-server"
[[deployment.config.environment]]
variable = "TEST_1"
value = "VALUE"
[[deployment.config.environment]]
variable = "TEST_2"
value = "VALUE"
[[deployment]]
name = "monitor-frontend-v1"
description = "v1 frontend"
tags = ["monitor", "v1", "frontend"]
config.server = "mogh-server"
[[deployment.config.environment]]
variable = "TEST_1"
value = "VALUE"

28
bin/monrun/src/sync/mod.rs
View File

@@ -47,15 +47,22 @@ pub async fn run_sync(path: &Path) -> anyhow::Result<()> {
wait_for_enter("CONTINUE")?;
// Run these first, which require no name -> id replacement
Alerter::run_updates(alerter_updates, alerter_creates).await;
Builder::run_updates(builder_updates, builder_creates).await;
// No deps
Server::run_updates(server_updates, server_creates).await;
Alerter::run_updates(alerter_updates, alerter_creates).await;
// Dependant on server
Builder::run_updates(builder_updates, builder_creates).await;
Repo::run_updates(repo_updates, repo_creates).await;
// Dependant on builder
Build::run_updates(build_updates, build_creates).await;
// Dependant on server / builder
Deployment::run_updates(deployment_updates, deployment_creates)
.await;
Repo::run_updates(repo_updates, repo_creates).await;
// Dependant on everything
Procedure::run_updates(procedure_updates, procedure_creates).await;
Ok(())
@@ -68,7 +75,6 @@ type UpdatesResult<T> = (ToUpdate<T>, ToCreate<T>);
pub trait ResourceSync {
type PartialConfig: Clone + Send + 'static;
type ListItemInfo: 'static;
type ExtLookup: Send + Sync;
fn display() -> &'static str;
@@ -77,18 +83,14 @@ pub trait ResourceSync {
fn name_to_resource(
) -> &'static HashMap<String, ResourceListItem<Self::ListItemInfo>>;
async fn init_lookup_data() -> Self::ExtLookup;
/// Returns created id
async fn create(
resource: Resource<Self::PartialConfig>,
ext_lookup: &Self::ExtLookup,
) -> anyhow::Result<String>;
async fn update(
id: String,
resource: Resource<Self::PartialConfig>,
ext_lookup: &Self::ExtLookup,
) -> anyhow::Result<()>;
fn get_updates(
@@ -151,8 +153,6 @@ pub trait ResourceSync {
.map(|tag| (tag.name, tag.id))
.collect::<HashMap<_, _>>();
let ext_lookup = Self::init_lookup_data().await;
let log_after = !to_update.is_empty() || !to_create.is_empty();
for (id, resource) in to_update {
@@ -160,9 +160,7 @@ pub trait ResourceSync {
let name = resource.name.clone();
let tags = resource.tags.clone();
let description = resource.description.clone();
if let Err(e) =
Self::update(id.clone(), resource, &ext_lookup).await
{
if let Err(e) = Self::update(id.clone(), resource).await {
warn!("failed to update {} {name} | {e:#}", Self::display());
}
Self::update_tags(
@@ -180,7 +178,7 @@ pub trait ResourceSync {
let name = resource.name.clone();
let tags = resource.tags.clone();
let description = resource.description.clone();
let id = match Self::create(resource, &ext_lookup).await {
let id = match Self::create(resource).await {
Ok(id) => id,
Err(e) => {
warn!(

5
bin/monrun/src/sync/resources/alerter.rs
View File

@@ -16,7 +16,6 @@ use crate::{
impl ResourceSync for Alerter {
type PartialConfig = PartialAlerterConfig;
type ListItemInfo = AlerterListItemInfo;
type ExtLookup = ();
fn display() -> &'static str {
"alerter"
@@ -32,11 +31,8 @@ impl ResourceSync for Alerter {
name_to_alerter()
}
async fn init_lookup_data() -> Self::ExtLookup {}
async fn create(
resource: Resource<Self::PartialConfig>,
_: &(),
) -> anyhow::Result<String> {
monitor_client()
.write(CreateAlerter {
@@ -50,7 +46,6 @@ impl ResourceSync for Alerter {
async fn update(
id: String,
resource: Resource<Self::PartialConfig>,
_: &(),
) -> anyhow::Result<()> {
monitor_client()
.write(UpdateAlerter {

34
bin/monrun/src/sync/resources/build.rs
View File

@@ -1,10 +1,7 @@
use std::collections::HashMap;
use monitor_client::{
api::{
read::ListBuilders,
write::{CreateBuild, UpdateBuild},
},
api::write::{CreateBuild, UpdateBuild},
entities::{
build::{Build, BuildListItemInfo, PartialBuildConfig},
resource::{Resource, ResourceListItem},
@@ -19,8 +16,6 @@ use crate::{
impl ResourceSync for Build {
type PartialConfig = PartialBuildConfig;
type ListItemInfo = BuildListItemInfo;
/// Builder Name to Id
type ExtLookup = HashMap<String, String>;
fn display() -> &'static str {
"build"
@@ -36,26 +31,9 @@ impl ResourceSync for Build {
name_to_build()
}
async fn init_lookup_data() -> Self::ExtLookup {
monitor_client()
.read(ListBuilders::default())
.await
.expect("failed to get builders")
.into_iter()
.map(|b| (b.name, b.id))
.collect::<HashMap<_, _>>()
}
async fn create(
mut resource: Resource<Self::PartialConfig>,
builder_name_to_id: &Self::ExtLookup,
resource: Resource<Self::PartialConfig>,
) -> anyhow::Result<String> {
// at this point the 'builder_id' is the name
resource.config.builder_id = resource
.config
.builder_id
.and_then(|id| builder_name_to_id.get(&id).cloned());
monitor_client()
.write(CreateBuild {
name: resource.name,
@@ -67,14 +45,8 @@ impl ResourceSync for Build {
async fn update(
id: String,
mut resource: Resource<Self::PartialConfig>,
builder_name_to_id: &Self::ExtLookup,
resource: Resource<Self::PartialConfig>,
) -> anyhow::Result<()> {
// at this point the 'builder_id' is the name
resource.config.builder_id = resource
.config
.builder_id
.and_then(|id| builder_name_to_id.get(&id).cloned());
monitor_client()
.write(UpdateBuild {
id,

5
bin/monrun/src/sync/resources/builder.rs
View File

@@ -16,7 +16,6 @@ use crate::{
impl ResourceSync for Builder {
type PartialConfig = PartialBuilderConfig;
type ListItemInfo = BuilderListItemInfo;
type ExtLookup = ();
fn display() -> &'static str {
"builder"
@@ -32,11 +31,8 @@ impl ResourceSync for Builder {
name_to_builder()
}
async fn init_lookup_data() -> Self::ExtLookup {}
async fn create(
resource: Resource<Self::PartialConfig>,
_: &(),
) -> anyhow::Result<String> {
monitor_client()
.write(CreateBuilder {
@@ -50,7 +46,6 @@ impl ResourceSync for Builder {
async fn update(
id: String,
resource: Resource<Self::PartialConfig>,
_: &(),
) -> anyhow::Result<()> {
monitor_client()
.write(UpdateBuilder {

73
bin/monrun/src/sync/resources/deployment.rs
View File

@@ -1,14 +1,10 @@
use std::collections::HashMap;
use monitor_client::{
api::{
read::{ListBuilds, ListServers},
write,
},
api::write,
entities::{
deployment::{
Deployment, DeploymentImage, DeploymentListItemInfo,
PartialDeploymentConfig,
Deployment, DeploymentListItemInfo, PartialDeploymentConfig,
},
resource::{Resource, ResourceListItem},
update::ResourceTarget,
@@ -19,15 +15,9 @@ use crate::{
maps::name_to_deployment, monitor_client, sync::ResourceSync,
};
pub struct DeploymentExtLookup {
pub servers: HashMap<String, String>,
pub builds: HashMap<String, String>,
}
impl ResourceSync for Deployment {
type PartialConfig = PartialDeploymentConfig;
type ListItemInfo = DeploymentListItemInfo;
type ExtLookup = DeploymentExtLookup;
fn display() -> &'static str {
"deployment"
@@ -43,31 +33,9 @@ impl ResourceSync for Deployment {
name_to_deployment()
}
async fn init_lookup_data() -> Self::ExtLookup {
let servers = monitor_client()
.read(ListServers::default())
.await
.expect("failed to get servers")
.into_iter()
.map(|b| (b.name, b.id))
.collect::<HashMap<_, _>>();
let builds = monitor_client()
.read(ListBuilds::default())
.await
.expect("failed to get builds")
.into_iter()
.map(|b| (b.name, b.id))
.collect::<HashMap<_, _>>();
DeploymentExtLookup { servers, builds }
}
async fn create(
mut resource: Resource<Self::PartialConfig>,
lookup: &Self::ExtLookup,
resource: Resource<Self::PartialConfig>,
) -> anyhow::Result<String> {
handle_name_to_id_switch(&mut resource.config, lookup);
monitor_client()
.write(write::CreateDeployment {
name: resource.name,
@@ -79,10 +47,8 @@ impl ResourceSync for Deployment {
async fn update(
id: String,
mut resource: Resource<Self::PartialConfig>,
lookup: &Self::ExtLookup,
resource: Resource<Self::PartialConfig>,
) -> anyhow::Result<()> {
handle_name_to_id_switch(&mut resource.config, lookup);
monitor_client()
.write(write::UpdateDeployment {
id,
@@ -92,34 +58,3 @@ impl ResourceSync for Deployment {
Ok(())
}
}
fn handle_name_to_id_switch(
config: &mut PartialDeploymentConfig,
lookup: &DeploymentExtLookup,
) {
config.server_id = config
.server_id
.as_ref()
.and_then(|name| lookup.servers.get(name).cloned());
if let Some(DeploymentImage::Build {
build_id: name,
version,
}) = &config.image
{
match lookup.builds.get(name).cloned() {
Some(build_id) => {
config.image = DeploymentImage::Build {
build_id,
version: version.clone(),
}
.into();
}
None => {
config.image = DeploymentImage::Image {
image: String::new(),
}
.into();
}
}
}
}

102
bin/monrun/src/sync/resources/procedure.rs
View File

@@ -1,7 +1,10 @@
use std::collections::HashMap;
use monitor_client::{
api::write::{CreateProcedure, UpdateProcedure},
api::{
read::ListTags,
write::{CreateProcedure, UpdateProcedure},
},
entities::{
procedure::{Procedure, ProcedureConfig, ProcedureListItemInfo},
resource::{Resource, ResourceListItem},
@@ -10,13 +13,14 @@ use monitor_client::{
};
use crate::{
maps::name_to_procedure, monitor_client, sync::ResourceSync,
maps::name_to_procedure,
monitor_client,
sync::{ResourceSync, ToCreate, ToUpdate},
};
impl ResourceSync for Procedure {
type PartialConfig = ProcedureConfig;
type ListItemInfo = ProcedureListItemInfo;
type ExtLookup = ();
fn display() -> &'static str {
"procedure"
@@ -32,11 +36,8 @@ impl ResourceSync for Procedure {
name_to_procedure()
}
async fn init_lookup_data() -> Self::ExtLookup {}
async fn create(
resource: Resource<Self::PartialConfig>,
_: &Self::ExtLookup,
) -> anyhow::Result<String> {
monitor_client()
.write(CreateProcedure {
@@ -50,7 +51,6 @@ impl ResourceSync for Procedure {
async fn update(
id: String,
resource: Resource<Self::PartialConfig>,
_: &Self::ExtLookup,
) -> anyhow::Result<()> {
monitor_client()
.write(UpdateProcedure {
@@ -60,4 +60,92 @@ impl ResourceSync for Procedure {
.await?;
Ok(())
}
async fn run_updates(
mut to_update: ToUpdate<Self::PartialConfig>,
mut to_create: ToCreate<Self::PartialConfig>,
) {
let mut tag_name_to_id = monitor_client()
.read(ListTags::default())
.await
.expect("failed to ListTags mid run")
.into_iter()
.map(|tag| (tag.name, tag.id))
.collect::<HashMap<_, _>>();
if to_update.is_empty() && to_create.is_empty() {
return;
}
for i in 0..10 {
let mut to_pull = Vec::new();
for (id, resource) in &to_update {
// Update resource
let name = resource.name.clone();
let tags = resource.tags.clone();
let description = resource.description.clone();
if let Err(e) =
Self::update(id.clone(), resource.clone()).await
{
if i == 9 {
warn!(
"failed to update {} {name} | {e:#}",
Self::display()
);
}
}
Self::update_tags(
id.clone(),
&name,
&tags,
&mut tag_name_to_id,
)
.await;
Self::update_description(id.clone(), description).await;
info!("{} {name} updated", Self::display());
// have to clone out so to_update is mutable
to_pull.push(id.clone());
}
to_update.retain(|(id, _)| !to_pull.contains(id));
let mut to_pull = Vec::new();
for resource in &to_create {
let name = resource.name.clone();
let tags = resource.tags.clone();
let description = resource.description.clone();
let id = match Self::create(resource.clone()).await {
Ok(id) => id,
Err(e) => {
if i == 9 {
warn!(
"failed to create {} {name} | {e:#}",
Self::display(),
);
}
continue;
}
};
Self::update_tags(
id.clone(),
&name,
&tags,
&mut tag_name_to_id,
)
.await;
Self::update_description(id, description).await;
info!("{} {name} created", Self::display());
to_pull.push(name);
}
to_create.retain(|resource| !to_pull.contains(&resource.name));
if to_update.is_empty() && to_create.is_empty() {
info!(
"============ {}s synced ✅ ============",
Self::display()
);
return;
}
}
warn!("procedure sync loop exited after max iterations");
}
}

5
bin/monrun/src/sync/resources/repo.rs
View File

@@ -14,7 +14,6 @@ use crate::{maps::name_to_repo, monitor_client, sync::ResourceSync};
impl ResourceSync for Repo {
type PartialConfig = PartialRepoConfig;
type ListItemInfo = RepoInfo;
type ExtLookup = ();
fn display() -> &'static str {
"repo"
@@ -30,11 +29,8 @@ impl ResourceSync for Repo {
name_to_repo()
}
async fn init_lookup_data() -> Self::ExtLookup {}
async fn create(
resource: Resource<Self::PartialConfig>,
_: &(),
) -> anyhow::Result<String> {
monitor_client()
.write(CreateRepo {
@@ -48,7 +44,6 @@ impl ResourceSync for Repo {
async fn update(
id: String,
resource: Resource<Self::PartialConfig>,
_: &(),
) -> anyhow::Result<()> {
monitor_client()
.write(UpdateRepo {

5
bin/monrun/src/sync/resources/server.rs
View File

@@ -16,7 +16,6 @@ use crate::{
impl ResourceSync for Server {
type ListItemInfo = ServerListItemInfo;
type PartialConfig = PartialServerConfig;
type ExtLookup = ();
fn display() -> &'static str {
"server"
@@ -32,11 +31,8 @@ impl ResourceSync for Server {
name_to_server()
}
async fn init_lookup_data() -> Self::ExtLookup {}
async fn create(
resource: Resource<Self::PartialConfig>,
_: &(),
) -> anyhow::Result<String> {
monitor_client()
.write(CreateServer {
@@ -50,7 +46,6 @@ impl ResourceSync for Server {
async fn update(
id: String,
resource: Resource<Self::PartialConfig>,
_: &(),
) -> anyhow::Result<()> {
monitor_client()
.write(UpdateServer {