ComputerSurge/komodo
3
0
Fork 0
forked from github-starred/komodo
Code Pull Requests 1 Actions Packages Releases Activity

implement RunSync

Browse Source
This commit is contained in:
mbecker20
2024-06-07 02:43:45 -07:00
parent bf85e886bd
commit 49f1d40ce8
26 changed files with 1338 additions and 814 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
Cargo.lock generated
View File

@@ -2166,12 +2166,12 @@ dependencies = [
"clap",
"colored",
"futures",
"merge_config_files",
"monitor_client",
"partial_derive2",
"serde",
"serde_json",
"strum 0.26.2",
"sync",
"tokio",
"toml",
"tracing",
@@ -2247,7 +2247,6 @@ dependencies = [
"sha2",
"slack_client_rs",
"svi",
"sync",
"termination_signal",
"tokio",
"tokio-util",
@@ -3219,13 +3218,12 @@ dependencies = [
[[package]]
name = "serror"
version = "0.3.5"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d4c2d61f8b059a9ac51ab4f65ea9874cb1b8a6e57f2ebc6d284988f6b877253b"
checksum = "a58297fe0d139a2950d2f474c3f5d0614eecc2b90f3bfe3ee28dca36dfcfd78d"
dependencies = [
"anyhow",
"axum 0.7.5",
"axum-extra",
"serde",
"serde_json",
]
@@ -3479,19 +3477,6 @@ dependencies = [
"unicode-ident",
]
[[package]]
name = "sync"
version = "1.6.2"
dependencies = [
"anyhow",
"colored",
"monitor_client",
"partial_derive2",
"serde",
"toml",
"tracing",
]
[[package]]
name = "sync_wrapper"
version = "0.1.2"

3
Cargo.toml
View File

@@ -19,12 +19,11 @@ monitor_client = "1.6.2"
periphery_client = { path = "client/periphery/rs" }
command = { path = "lib/command" }
logger = { path = "lib/logger" }
sync = { path = "lib/sync" }
git = { path = "lib/git" }
# MOGH
run_command = { version = "0.0.6", features = ["async_tokio"] }
serror = { version = "0.3.5", default-features = false }
serror = { version = "0.4.3", default-features = false }
slack = { version = "0.1.0", package = "slack_client_rs" }
derive_default_builder = "0.1.8"
derive_empty_traits = "0.1.0"

2
bin/cli/Cargo.toml
View File

@@ -17,11 +17,11 @@ path = "src/main.rs"
[dependencies]
# local
monitor_client.workspace = true
sync.workspace = true
# mogh
partial_derive2.workspace = true
# external
tracing-subscriber.workspace = true
merge_config_files.workspace = true
serde_json.workspace = true
futures.workspace = true
tracing.workspace = true

1
bin/cli/src/helpers.rs
View File

@@ -2,7 +2,6 @@ use std::io::Read;
use anyhow::Context;
use colored::Colorize;
use serde::de::DeserializeOwned;
pub fn wait_for_enter(press_enter_to: &str) -> anyhow::Result<()> {
println!(

3
bin/cli/src/state.rs
View File

@@ -1,6 +1,7 @@
use std::sync::OnceLock;
use clap::Parser;
use merge_config_files::parse_config_file;
use monitor_client::MonitorClient;
pub fn cli_args() -> &'static crate::args::CliArgs {
@@ -23,7 +24,7 @@ pub fn monitor_client() -> &'static MonitorClient {
}
(url, key, secret) => {
let mut creds: crate::args::CredsFile =
sync::file::parse_toml_file(&cli_args().creds)
parse_config_file(cli_args().creds.as_str())
.expect("failed to parse monitor credentials");
if let Some(url) = url {

27
bin/cli/src/sync/file.rs
View File

@@ -7,6 +7,7 @@ use std::{
use anyhow::{anyhow, Context};
use colored::Colorize;
use monitor_client::entities::toml::ResourcesToml;
use serde::de::DeserializeOwned;
pub fn read_resources(path: &str) -> anyhow::Result<ResourcesToml> {
let mut res = ResourcesToml::default();
@@ -30,14 +31,13 @@ fn read_resources_recursive(
{
return Ok(());
}
let more =
match sync::file::parse_toml_file::<ResourcesToml>(path) {
Ok(res) => res,
Err(e) => {
warn!("failed to parse {:?}. skipping file | {e:#}", path);
return Ok(());
}
};
let more = match parse_toml_file::<ResourcesToml>(path) {
Ok(res) => res,
Err(e) => {
warn!("failed to parse {:?}. skipping file | {e:#}", path);
return Ok(());
}
};
info!(
"{} from {}",
"adding resources".green().bold(),
@@ -45,12 +45,13 @@ fn read_resources_recursive(
);
resources.servers.extend(more.servers);
resources.deployments.extend(more.deployments);
resources.repos.extend(more.repos);
resources.builds.extend(more.builds);
resources.repos.extend(more.repos);
resources.procedures.extend(more.procedures);
resources.builders.extend(more.builders);
resources.alerters.extend(more.alerters);
resources.server_templates.extend(more.server_templates);
resources.syncs.extend(more.syncs);
resources.user_groups.extend(more.user_groups);
resources.variables.extend(more.variables);
Ok(())
@@ -69,3 +70,11 @@ fn read_resources_recursive(
Err(anyhow!("resources path is neither file nor directory"))
}
}
fn parse_toml_file<T: DeserializeOwned>(
path: impl AsRef<std::path::Path>,
) -> anyhow::Result<T> {
let contents = std::fs::read_to_string(path)
.context("failed to read file contents")?;
toml::from_str(&contents).context("failed to parse toml contents")
}

26
bin/cli/src/sync/mod.rs
View File

@@ -24,14 +24,6 @@ pub async fn run(path: &str, delete: bool) -> anyhow::Result<()> {
info!("computing sync actions...");
let (
server_template_creates,
server_template_updates,
server_template_deletes,
) = resource::get_updates::<ServerTemplate>(
resources.server_templates,
delete,
)?;
let (server_creates, server_updates, server_deletes) =
resource::get_updates::<Server>(resources.servers, delete)?;
let (deployment_creates, deployment_updates, deployment_deletes) =
@@ -41,14 +33,22 @@ pub async fn run(path: &str, delete: bool) -> anyhow::Result<()> {
)?;
let (build_creates, build_updates, build_deletes) =
resource::get_updates::<Build>(resources.builds, delete)?;
let (builder_creates, builder_updates, builder_deletes) =
resource::get_updates::<Builder>(resources.builders, delete)?;
let (alerter_creates, alerter_updates, alerter_deletes) =
resource::get_updates::<Alerter>(resources.alerters, delete)?;
let (repo_creates, repo_updates, repo_deletes) =
resource::get_updates::<Repo>(resources.repos, delete)?;
let (procedure_creates, procedure_updates, procedure_deletes) =
resource::get_updates::<Procedure>(resources.procedures, delete)?;
let (builder_creates, builder_updates, builder_deletes) =
resource::get_updates::<Builder>(resources.builders, delete)?;
let (alerter_creates, alerter_updates, alerter_deletes) =
resource::get_updates::<Alerter>(resources.alerters, delete)?;
let (
server_template_creates,
server_template_updates,
server_template_deletes,
) = resource::get_updates::<ServerTemplate>(
resources.server_templates,
delete,
)?;
let (variable_creates, variable_updates, variable_deletes) =
variables::get_updates(resources.variables, delete)?;
@@ -116,7 +116,7 @@ pub async fn run(path: &str, delete: bool) -> anyhow::Result<()> {
Build::run_updates(build_creates, build_updates, build_deletes)
.await;
// Dependant on server / builder
// Dependant on server / build
Deployment::run_updates(
deployment_creates,
deployment_updates,

3
bin/cli/src/sync/resources/procedure.rs
View File

@@ -133,6 +133,7 @@ impl ResourceSync for Procedure {
Self::display()
);
}
continue;
}
}
@@ -169,7 +170,7 @@ impl ResourceSync for Procedure {
to_create.retain(|resource| !to_pull.contains(&resource.name));
if to_update.is_empty() && to_create.is_empty() {
info!("all procedures synced");
// info!("all procedures synced");
return;
}
}

1
bin/core/Cargo.toml
View File

@@ -18,7 +18,6 @@ path = "src/main.rs"
monitor_client = { workspace = true, features = ["mongo"] }
periphery_client.workspace = true
logger.workspace = true
sync.workspace = true
git.workspace = true
# mogh
serror = { workspace = true, features = ["axum"] }

227
bin/core/src/api/execute/sync.rs
View File

@@ -2,13 +2,33 @@ use anyhow::Context;
use monitor_client::{
api::execute::RunSync,
entities::{
permission::PermissionLevel, sync::ResourceSync, update::Update,
self,
alerter::Alerter,
build::Build,
builder::Builder,
deployment::Deployment,
permission::PermissionLevel,
procedure::Procedure,
repo::Repo,
server::Server,
server_template::ServerTemplate,
update::{Log, Update},
user::User,
},
};
use resolver_api::Resolve;
use crate::{helpers::update::update_update, resource, state::State};
use crate::{
helpers::{
query::get_id_to_tags,
sync::resource::{
get_updates_for_execution, AllResourcesById, ResourceSync,
},
update::update_update,
},
resource,
state::State,
};
impl Resolve<RunSync, (User, Update)> for State {
async fn resolve(
@@ -16,15 +36,13 @@ impl Resolve<RunSync, (User, Update)> for State {
RunSync { sync }: RunSync,
(user, mut update): (User, Update),
) -> anyhow::Result<Update> {
let sync = resource::get_check_permissions::<ResourceSync>(
&sync,
&user,
PermissionLevel::Execute,
)
let sync = resource::get_check_permissions::<
entities::sync::ResourceSync,
>(&sync, &user, PermissionLevel::Execute)
.await?;
let (res, logs) =
crate::helpers::sync::get_remote_resources(&sync)
crate::helpers::sync::remote::get_remote_resources(&sync)
.await
.context("failed to get remote resources")?;
@@ -33,6 +51,197 @@ impl Resolve<RunSync, (User, Update)> for State {
let resources = res?;
todo!()
let all_resources = AllResourcesById::load().await?;
let id_to_tags = get_id_to_tags(None).await?;
let (servers_to_create, servers_to_update, servers_to_delete) =
get_updates_for_execution::<Server>(
resources.servers,
sync.config.delete,
&all_resources,
&id_to_tags,
)
.await?;
let (
deployments_to_create,
deployments_to_update,
deployments_to_delete,
) = get_updates_for_execution::<Deployment>(
resources.deployments,
sync.config.delete,
&all_resources,
&id_to_tags,
)
.await?;
let (builds_to_create, builds_to_update, builds_to_delete) =
get_updates_for_execution::<Build>(
resources.builds,
sync.config.delete,
&all_resources,
&id_to_tags,
)
.await?;
let (repos_to_create, repos_to_update, repos_to_delete) =
get_updates_for_execution::<Repo>(
resources.repos,
sync.config.delete,
&all_resources,
&id_to_tags,
)
.await?;
let (
procedures_to_create,
procedures_to_update,
procedures_to_delete,
) = get_updates_for_execution::<Procedure>(
resources.procedures,
sync.config.delete,
&all_resources,
&id_to_tags,
)
.await?;
let (builders_to_create, builders_to_update, builders_to_delete) =
get_updates_for_execution::<Builder>(
resources.builders,
sync.config.delete,
&all_resources,
&id_to_tags,
)
.await?;
let (alerters_to_create, alerters_to_update, alerters_to_delete) =
get_updates_for_execution::<Alerter>(
resources.alerters,
sync.config.delete,
&all_resources,
&id_to_tags,
)
.await?;
let (
server_templates_to_create,
server_templates_to_update,
server_templates_to_delete,
) = get_updates_for_execution::<ServerTemplate>(
resources.server_templates,
sync.config.delete,
&all_resources,
&id_to_tags,
)
.await?;
let (
resource_syncs_to_create,
resource_syncs_to_update,
resource_syncs_to_delete,
) = get_updates_for_execution::<entities::sync::ResourceSync>(
resources.syncs,
sync.config.delete,
&all_resources,
&id_to_tags,
)
.await?;
// =================
// No deps
maybe_extend(
&mut update.logs,
entities::sync::ResourceSync::run_updates(
resource_syncs_to_create,
resource_syncs_to_update,
resource_syncs_to_delete,
)
.await,
);
maybe_extend(
&mut update.logs,
ServerTemplate::run_updates(
server_templates_to_create,
server_templates_to_update,
server_templates_to_delete,
)
.await,
);
maybe_extend(
&mut update.logs,
Server::run_updates(
servers_to_create,
servers_to_update,
servers_to_delete,
)
.await,
);
maybe_extend(
&mut update.logs,
Alerter::run_updates(
alerters_to_create,
alerters_to_update,
alerters_to_delete,
)
.await,
);
// Dependent on server
maybe_extend(
&mut update.logs,
Builder::run_updates(
builders_to_create,
builders_to_update,
builders_to_delete,
)
.await,
);
maybe_extend(
&mut update.logs,
Repo::run_updates(
repos_to_create,
repos_to_update,
repos_to_delete,
)
.await,
);
// Dependant on builder
maybe_extend(
&mut update.logs,
Build::run_updates(
builds_to_create,
builds_to_update,
builds_to_delete,
)
.await,
);
// Dependant on server / build
maybe_extend(
&mut update.logs,
Deployment::run_updates(
deployments_to_create,
deployments_to_update,
deployments_to_delete,
)
.await,
);
// Dependant on everything
maybe_extend(
&mut update.logs,
Procedure::run_updates(
procedures_to_create,
procedures_to_update,
procedures_to_delete,
)
.await,
);
update.finalize();
update_update(update.clone()).await?;
Ok(update)
}
}
fn maybe_extend(logs: &mut Vec<Log>, log: Option<Log>) {
if let Some(log) = log {
logs.push(log);
}
}

24
bin/core/src/api/read/toml.rs
View File

@@ -21,6 +21,7 @@ use monitor_client::{
resource::{Resource, ResourceQuery},
server::Server,
server_template::ServerTemplate,
sync::ResourceSync,
toml::{
PermissionToml, ResourceToml, ResourcesToml, UserGroupToml,
},
@@ -167,6 +168,15 @@ impl Resolve<ExportResourcesToToml, User> for State {
.await?;
res.alerters.push(convert_resource(alerter, &names.tags))
}
ResourceTarget::ResourceSync(id) => {
let sync = resource::get_check_permissions::<ResourceSync>(
&id,
&user,
PermissionLevel::Read,
)
.await?;
res.syncs.push(convert_resource(sync, &names.tags))
}
ResourceTarget::ServerTemplate(id) => {
let template = resource::get_check_permissions::<
ServerTemplate,
@@ -268,9 +278,6 @@ impl Resolve<ExportResourcesToToml, User> for State {
format!("failed to add procedure {id}")
})?;
}
ResourceTarget::ResourceSync(id) => {
todo!()
}
ResourceTarget::System(_) => continue,
};
}
@@ -364,7 +371,9 @@ async fn add_procedure(
Execution::PruneContainers(exec) => exec.server.clone_from(
names.servers.get(&exec.server).unwrap_or(&String::new()),
),
Execution::RunSync(exec) => todo!(),
Execution::RunSync(exec) => exec.sync.clone_from(
names.syncs.get(&exec.sync).unwrap_or(&String::new()),
),
Execution::None(_) => continue,
}
}
@@ -384,6 +393,7 @@ struct ResourceNames {
repos: HashMap<String, String>,
deployments: HashMap<String, String>,
procedures: HashMap<String, String>,
syncs: HashMap<String, String>,
}
impl ResourceNames {
@@ -432,6 +442,12 @@ impl ResourceNames {
.into_iter()
.map(|t| (t.id, t.name))
.collect::<HashMap<_, _>>(),
syncs: find_collect(&db.resource_syncs, None, None)
.await
.context("failed to get all resource syncs")?
.into_iter()
.map(|t| (t.id, t.name))
.collect::<HashMap<_, _>>(),
})
}
}

10
bin/core/src/helpers/procedure.rs
View File

@@ -238,7 +238,15 @@ async fn execute_execution(
.context("failed at PruneContainers")?
}
Execution::RunSync(req) => {
todo!()
let req = ExecuteRequest::RunSync(req);
let update = init_execution_update(&req, &user).await?;
let ExecuteRequest::RunSync(req) = req else {
unreachable!()
};
State
.resolve(req, (user, update))
.await
.context("failed at RunSync")?
}
};
if update.success {

12
bin/core/src/helpers/query.rs
View File

@@ -107,6 +107,18 @@ pub async fn get_tag_check_owner(
Err(anyhow!("user must be tag owner or admin"))
}
pub async fn get_id_to_tags(
filter: impl Into<Option<Document>>,
) -> anyhow::Result<HashMap<String, Tag>> {
let res = find_collect(&db_client().await.tags, filter, None)
.await
.context("failed to query db for tags")?
.into_iter()
.map(|tag| (tag.id.clone(), tag))
.collect();
Ok(res)
}
#[instrument(level = "debug")]
pub async fn get_user_user_group_ids(
user_id: &str,

52
lib/sync/src/file.rs → bin/core/src/helpers/sync/file.rs
View File

@@ -1,19 +1,24 @@
use std::{fs, path::Path};
use anyhow::{anyhow, Context};
use colored::Colorize;
use monitor_client::entities::toml::ResourcesToml;
use monitor_client::entities::{toml::ResourcesToml, update::Log};
use serde::de::DeserializeOwned;
pub fn read_resources(path: &Path) -> anyhow::Result<ResourcesToml> {
use super::{colored, muted};
pub fn read_resources(
path: &Path,
) -> anyhow::Result<(ResourcesToml, Log)> {
let mut res = ResourcesToml::default();
read_resources_recursive(path, &mut res)?;
Ok(res)
let mut log = format!("reading resources from {path:?}");
read_resources_recursive(path, &mut res, &mut log)?;
Ok((res, Log::simple("read remote resources", log)))
}
fn read_resources_recursive(
path: &Path,
resources: &mut ResourcesToml,
log: &mut String,
) -> anyhow::Result<()> {
let res =
fs::metadata(path).context("failed to get path metadata")?;
@@ -28,26 +33,28 @@ fn read_resources_recursive(
let more = match parse_toml_file::<ResourcesToml>(path) {
Ok(res) => res,
Err(e) => {
tracing::warn!(
"failed to parse {:?}. skipping file | {e:#}",
path
);
warn!("failed to parse {:?}. skipping file | {e:#}", path);
return Ok(());
}
};
tracing::info!(
"{} from {}",
"adding resources".green().bold(),
path.display().to_string().blue().bold()
);
log.push('\n');
log.push_str(&format!(
"{}: {} from {}",
muted("INFO"),
colored("adding resources", "green"),
colored(&path.display().to_string(), "blue")
));
resources.servers.extend(more.servers);
resources.deployments.extend(more.deployments);
resources.repos.extend(more.repos);
resources.builds.extend(more.builds);
resources.repos.extend(more.repos);
resources.procedures.extend(more.procedures);
resources.builders.extend(more.builders);
resources.alerters.extend(more.alerters);
resources.server_templates.extend(more.server_templates);
resources.syncs.extend(more.syncs);
resources.user_groups.extend(more.user_groups);
resources.variables.extend(more.variables);
Ok(())
@@ -55,12 +62,15 @@ fn read_resources_recursive(
let directory = fs::read_dir(path)
.context("failed to read directory contents")?;
for entry in directory.into_iter().flatten() {
if let Err(e) =
read_resources_recursive(&entry.path(), resources)
let path = entry.path();
if let Err(e) = read_resources_recursive(&path, resources, log)
{
tracing::warn!(
"failed to read additional resources at path | {e:#}"
);
log.push('\n');
log.push_str(&format!(
"{}: failed to read additional resources from {} | {e:#}",
colored("ERROR", "red"),
colored(&path.display().to_string(), "blue")
));
}
}
Ok(())
@@ -69,7 +79,7 @@ fn read_resources_recursive(
}
}
pub fn parse_toml_file<T: DeserializeOwned>(
fn parse_toml_file<T: DeserializeOwned>(
path: impl AsRef<std::path::Path>,
) -> anyhow::Result<T> {
let contents = std::fs::read_to_string(path)

17
bin/core/src/helpers/sync/mod.rs Normal file
View File

@@ -0,0 +1,17 @@
pub mod remote;
pub mod resource;
mod file;
mod resources;
fn muted(content: &str) -> String {
format!("<span class=\"text-muted-foreground\">{content}</span>")
}
fn bold(content: &str) -> String {
format!("<span class=\"font-bold\">{content}</span>")
}
fn colored(content: &str, color: &str) -> String {
format!("<span class=\"text-{color}-500\">{content}</span>")
}

12
bin/core/src/helpers/sync.rs → bin/core/src/helpers/sync/remote.rs
View File

@@ -2,7 +2,6 @@ use anyhow::{anyhow, Context};
use monitor_client::entities::{
sync::ResourceSync, toml::ResourcesToml, update::Log, CloneArgs,
};
use sync::file::read_resources;
use crate::config::core_config;
@@ -24,7 +23,7 @@ pub async fn get_remote_resources(
.transpose()?
.cloned();
let clone_logs =
let mut logs =
git::clone(clone_args, &config.sync_directory, github_token)
.await
.context("failed to clone resource repo")?;
@@ -32,11 +31,16 @@ pub async fn get_remote_resources(
let repo_path = config.sync_directory.join(&sync.name);
let resource_path = repo_path.join(&sync.config.resource_path);
let res = read_resources(&resource_path);
let res = super::file::read_resources(&resource_path).map(
|(resources, log)| {
logs.push(log);
resources
},
);
if let Err(e) = std::fs::remove_dir_all(&repo_path) {
warn!("failed to remove sync repo directory | {e:?}")
}
Ok((res, clone_logs))
Ok((res, logs))
}

443
bin/core/src/helpers/sync/resource.rs Normal file
View File

@@ -0,0 +1,443 @@
use std::collections::HashMap;
use anyhow::Context;
use monitor_client::{
api::write::{UpdateDescription, UpdateTagsOnResource},
entities::{
self,
alerter::Alerter,
build::Build,
builder::Builder,
deployment::Deployment,
procedure::Procedure,
repo::Repo,
server::Server,
server_template::ServerTemplate,
tag::Tag,
toml::ResourceToml,
update::{Log, ResourceTarget},
user::sync_user,
},
};
use mungos::find::find_collect;
use partial_derive2::MaybeNone;
use resolver_api::Resolve;
use crate::{resource::MonitorResource, state::State};
use super::{bold, colored, muted};
pub type ToUpdate<T> = Vec<ToUpdateItem<T>>;
pub type ToCreate<T> = Vec<ResourceToml<T>>;
/// Vec of resource names
pub type ToDelete = Vec<String>;
type UpdatesResult<T> = (ToCreate<T>, ToUpdate<T>, ToDelete);
pub struct ToUpdateItem<T> {
pub id: String,
pub resource: ResourceToml<T>,
pub update_description: bool,
pub update_tags: bool,
}
pub trait ResourceSync: MonitorResource + Sized {
fn resource_target(id: String) -> ResourceTarget;
/// Diffs the declared toml (partial) against the full existing config.
/// Removes all fields from toml (partial) that haven't changed.
fn get_diff(
original: Self::Config,
update: Self::PartialConfig,
resources: &AllResourcesById,
) -> anyhow::Result<Self::ConfigDiff>;
async fn run_updates(
to_create: ToCreate<Self::PartialConfig>,
to_update: ToUpdate<Self::PartialConfig>,
to_delete: ToDelete,
) -> Option<Log> {
if to_create.is_empty()
&& to_update.is_empty()
&& to_delete.is_empty()
{
return None;
}
let mut log =
format!("running updates on {}s", Self::resource_type());
for resource in to_create {
let name = resource.name.clone();
let tags = resource.tags.clone();
let description = resource.description.clone();
let id = match crate::resource::create::<Self>(
&resource.name,
resource.config,
sync_user(),
)
.await
{
Ok(resource) => resource.id,
Err(e) => {
log.push('\n');
log.push_str(&format!(
"{}: failed to create {} '{}' | {e:#}",
colored("ERROR", "red"),
Self::resource_type(),
bold(&name)
));
continue;
}
};
run_update_tags::<Self>(id.clone(), &name, tags, &mut log)
.await;
run_update_description::<Self>(
id,
&name,
description,
&mut log,
)
.await;
log.push_str(&format!(
"\n{}: {} {} '{}'",
muted("INFO"),
colored("created", "green"),
Self::resource_type(),
bold(&name)
));
}
for ToUpdateItem {
id,
resource,
update_description,
update_tags,
} in to_update
{
// Update resource
let name = resource.name.clone();
let tags = resource.tags.clone();
let description = resource.description.clone();
if update_description {
run_update_description::<Self>(
id.clone(),
&name,
description,
&mut log,
)
.await;
}
if update_tags {
run_update_tags::<Self>(id.clone(), &name, tags, &mut log)
.await;
}
if !resource.config.is_none() {
log.push('\n');
if let Err(e) = crate::resource::update::<Self>(
&id,
resource.config,
sync_user(),
)
.await
{
log.push_str(&format!(
"{}: failed to update config on {} '{}' | {e:#}",
colored("ERROR", "red"),
Self::resource_type(),
bold(&name),
))
} else {
log.push_str(&format!(
"{}: {} {} '{}' configuration",
muted("INFO"),
colored("updated", "blue"),
Self::resource_type(),
bold(&name)
));
}
}
}
for resource in to_delete {
log.push('\n');
if let Err(e) =
crate::resource::delete::<Self>(&resource, sync_user()).await
{
log.push_str(&format!(
"{}: failed to delete {} '{}' | {e:#}",
colored("ERROR", "red"),
Self::resource_type(),
bold(&resource),
))
} else {
log.push_str(&format!(
"{}: {} {} '{}'",
muted("INFO"),
colored("deleted", "red"),
Self::resource_type(),
bold(&resource)
));
}
}
Some(Log::simple(
&format!("Update {}s", Self::resource_type()),
log,
))
}
}
/// Gets all the resources to update. For use in sync execution.
pub async fn get_updates_for_execution<Resource: ResourceSync>(
resources: Vec<ResourceToml<Resource::PartialConfig>>,
delete: bool,
all_resources: &AllResourcesById,
id_to_tags: &HashMap<String, Tag>,
) -> anyhow::Result<UpdatesResult<Resource::PartialConfig>> {
let map = find_collect(Resource::coll().await, None, None)
.await
.context("failed to get resources from db")?
.into_iter()
.map(|r| (r.name.clone(), r))
.collect::<HashMap<_, _>>();
let mut to_create = ToCreate::<Resource::PartialConfig>::new();
let mut to_update = ToUpdate::<Resource::PartialConfig>::new();
let mut to_delete = ToDelete::new();
if delete {
for resource in map.values() {
if !resources.iter().any(|r| r.name == resource.name) {
to_delete.push(resource.name.clone());
}
}
}
for mut resource in resources {
match map.get(&resource.name) {
Some(original) => {
// First merge toml resource config (partial) onto default resource config.
// Makes sure things that aren't defined in toml (come through as None) actually get removed.
let config: Resource::Config = resource.config.into();
resource.config = config.into();
let diff = Resource::get_diff(
original.config.clone(),
resource.config,
all_resources,
)?;
let original_tags = original
.tags
.iter()
.filter_map(|id| id_to_tags.get(id).map(|t| t.name.clone()))
.collect::<Vec<_>>();
// Only proceed if there are any fields to update,
// or a change to tags / description
if diff.is_none()
&& resource.description == original.description
&& resource.tags == original_tags
{
continue;
}
// println!(
// "\n{}: {}: '{}'\n-------------------",
// "UPDATE".blue(),
// Resource::display(),
// resource.name.bold(),
// );
// let mut lines = Vec::<String>::new();
// if resource.description != original.description {
// lines.push(format!(
// "{}: 'description'\n{}: {}\n{}: {}",
// "field".dimmed(),
// "from".dimmed(),
// original.description.red(),
// "to".dimmed(),
// resource.description.green()
// ))
// }
// if resource.tags != original_tags {
// let from = format!("{:?}", original_tags).red();
// let to = format!("{:?}", resource.tags).green();
// lines.push(format!(
// "{}: 'tags'\n{}: {from}\n{}: {to}",
// "field".dimmed(),
// "from".dimmed(),
// "to".dimmed(),
// ));
// }
// lines.extend(diff.iter_field_diffs().map(
// |FieldDiff { field, from, to }| {
// format!(
// "{}: '{field}'\n{}: {}\n{}: {}",
// "field".dimmed(),
// "from".dimmed(),
// from.red(),
// "to".dimmed(),
// to.green()
// )
// },
// ));
// println!("{}", lines.join("\n-------------------\n"));
// Minimizes updates through diffing.
resource.config = diff.into();
let update = ToUpdateItem {
id: original.id.clone(),
update_description: resource.description
!= original.description,
update_tags: resource.tags != original_tags,
resource,
};
to_update.push(update);
}
None => {
// println!(
// "\n{}: {}: {}\n{}: {}\n{}: {:?}\n{}: {}",
// "CREATE".green(),
// Resource::display(),
// resource.name.bold().green(),
// "description".dimmed(),
// resource.description,
// "tags".dimmed(),
// resource.tags,
// "config".dimmed(),
// serde_json::to_string_pretty(&resource.config)?
// );
to_create.push(resource);
}
}
}
for name in &to_delete {
// println!(
// "\n{}: {}: '{}'\n-------------------",
// "DELETE".red(),
// Resource::display(),
// name.bold(),
// );
}
Ok((to_create, to_update, to_delete))
}
pub async fn run_update_tags<Resource: ResourceSync>(
id: String,
name: &str,
tags: Vec<String>,
log: &mut String,
) {
// Update tags
log.push('\n');
if let Err(e) = State
.resolve(
UpdateTagsOnResource {
target: Resource::resource_target(id),
tags,
},
sync_user().to_owned(),
)
.await
{
log.push_str(&format!(
"{}: failed to update tags on {} '{}' | {e:#}",
colored("ERROR", "red"),
Resource::resource_type(),
bold(name),
))
} else {
log.push_str(&format!(
"{}: {} {} '{}' tags",
muted("INFO"),
colored("updated", "blue"),
Resource::resource_type(),
bold(name)
));
}
}
pub async fn run_update_description<Resource: ResourceSync>(
id: String,
name: &str,
description: String,
log: &mut String,
) {
log.push('\n');
if let Err(e) = State
.resolve(
UpdateDescription {
target: Resource::resource_target(id.clone()),
description,
},
sync_user().to_owned(),
)
.await
{
log.push_str(&format!(
"{}: failed to update description on {} '{}' | {e:#}",
colored("ERROR", "red"),
Resource::resource_type(),
bold(name),
))
} else {
log.push_str(&format!(
"{}: {} {} '{}' description",
muted("INFO"),
colored("updated", "blue"),
Resource::resource_type(),
bold(name)
));
}
}
pub struct AllResourcesById {
pub servers: HashMap<String, Server>,
pub deployments: HashMap<String, Deployment>,
pub builds: HashMap<String, Build>,
pub repos: HashMap<String, Repo>,
pub procedures: HashMap<String, Procedure>,
pub builders: HashMap<String, Builder>,
pub alerters: HashMap<String, Alerter>,
pub templates: HashMap<String, ServerTemplate>,
pub syncs: HashMap<String, entities::sync::ResourceSync>,
}
impl AllResourcesById {
pub async fn load() -> anyhow::Result<Self> {
Ok(Self {
servers: crate::resource::get_id_to_resource_map::<Server>()
.await?,
deployments: crate::resource::get_id_to_resource_map::<
Deployment,
>()
.await?,
builds: crate::resource::get_id_to_resource_map::<Build>()
.await?,
repos: crate::resource::get_id_to_resource_map::<Repo>()
.await?,
procedures:
crate::resource::get_id_to_resource_map::<Procedure>().await?,
builders: crate::resource::get_id_to_resource_map::<Builder>()
.await?,
alerters: crate::resource::get_id_to_resource_map::<Alerter>()
.await?,
templates: crate::resource::get_id_to_resource_map::<
ServerTemplate,
>()
.await?,
syncs: crate::resource::get_id_to_resource_map::<
entities::sync::ResourceSync,
>()
.await?,
})
}
}

468
bin/core/src/helpers/sync/resources.rs Normal file
View File

@@ -0,0 +1,468 @@
use monitor_client::{
api::execute::Execution,
entities::{
self,
alerter::Alerter,
build::Build,
builder::{Builder, BuilderConfig},
deployment::{Deployment, DeploymentImage},
procedure::Procedure,
repo::Repo,
server::Server,
server_template::ServerTemplate,
update::{Log, ResourceTarget},
user::sync_user,
},
};
use partial_derive2::{MaybeNone, PartialDiff};
use crate::{
helpers::sync::{
bold, colored, muted,
resource::{
run_update_description, run_update_tags, ResourceSync,
ToUpdateItem,
},
},
resource::MonitorResource,
};
use super::resource::{
AllResourcesById, ToCreate, ToDelete, ToUpdate,
};
impl ResourceSync for Server {
fn resource_target(id: String) -> ResourceTarget {
ResourceTarget::Server(id)
}
fn get_diff(
original: Self::Config,
update: Self::PartialConfig,
_resources: &AllResourcesById,
) -> anyhow::Result<Self::ConfigDiff> {
Ok(original.partial_diff(update))
}
}
impl ResourceSync for Build {
fn resource_target(id: String) -> ResourceTarget {
ResourceTarget::Build(id)
}
fn get_diff(
mut original: Self::Config,
update: Self::PartialConfig,
resources: &AllResourcesById,
) -> anyhow::Result<Self::ConfigDiff> {
original.builder_id = resources
.builders
.get(&original.builder_id)
.map(|b| b.name.clone())
.unwrap_or_default();
Ok(original.partial_diff(update))
}
}
impl ResourceSync for Deployment {
fn resource_target(id: String) -> ResourceTarget {
ResourceTarget::Deployment(id)
}
fn get_diff(
mut original: Self::Config,
update: Self::PartialConfig,
resources: &AllResourcesById,
) -> anyhow::Result<Self::ConfigDiff> {
// need to replace the server id with name
original.server_id = resources
.servers
.get(&original.server_id)
.map(|s| s.name.clone())
.unwrap_or_default();
// need to replace the build id with name
if let DeploymentImage::Build { build_id, version } =
&original.image
{
original.image = DeploymentImage::Build {
build_id: resources
.builds
.get(build_id)
.map(|b| b.name.clone())
.unwrap_or_default(),
version: version.clone(),
};
}
Ok(original.partial_diff(update))
}
}
impl ResourceSync for Repo {
fn resource_target(id: String) -> ResourceTarget {
ResourceTarget::Repo(id)
}
fn get_diff(
mut original: Self::Config,
update: Self::PartialConfig,
resources: &AllResourcesById,
) -> anyhow::Result<Self::ConfigDiff> {
// Need to replace server id with name
original.server_id = resources
.servers
.get(&original.server_id)
.map(|s| s.name.clone())
.unwrap_or_default();
Ok(original.partial_diff(update))
}
}
impl ResourceSync for Alerter {
fn resource_target(id: String) -> ResourceTarget {
ResourceTarget::Alerter(id)
}
fn get_diff(
original: Self::Config,
update: Self::PartialConfig,
_resources: &AllResourcesById,
) -> anyhow::Result<Self::ConfigDiff> {
Ok(original.partial_diff(update))
}
}
impl ResourceSync for Builder {
fn resource_target(id: String) -> ResourceTarget {
ResourceTarget::Builder(id)
}
fn get_diff(
mut original: Self::Config,
update: Self::PartialConfig,
resources: &AllResourcesById,
) -> anyhow::Result<Self::ConfigDiff> {
// need to replace server builder id with name
if let BuilderConfig::Server(config) = &mut original {
config.server_id = resources
.servers
.get(&config.server_id)
.map(|s| s.name.clone())
.unwrap_or_default();
}
Ok(original.partial_diff(update))
}
}
impl ResourceSync for ServerTemplate {
fn resource_target(id: String) -> ResourceTarget {
ResourceTarget::ServerTemplate(id)
}
fn get_diff(
original: Self::Config,
update: Self::PartialConfig,
_resources: &AllResourcesById,
) -> anyhow::Result<Self::ConfigDiff> {
Ok(original.partial_diff(update))
}
}
impl ResourceSync for entities::sync::ResourceSync {
fn resource_target(id: String) -> ResourceTarget {
ResourceTarget::ResourceSync(id)
}
fn get_diff(
original: Self::Config,
update: Self::PartialConfig,
_resources: &AllResourcesById,
) -> anyhow::Result<Self::ConfigDiff> {
Ok(original.partial_diff(update))
}
}
impl ResourceSync for Procedure {
fn resource_target(id: String) -> ResourceTarget {
ResourceTarget::Procedure(id)
}
fn get_diff(
mut original: Self::Config,
update: Self::PartialConfig,
resources: &AllResourcesById,
) -> anyhow::Result<Self::ConfigDiff> {
for stage in &mut original.stages {
for execution in &mut stage.executions {
match &mut execution.execution {
Execution::None(_) => {}
Execution::RunProcedure(config) => {
config.procedure = resources
.procedures
.get(&config.procedure)
.map(|p| p.name.clone())
.unwrap_or_default();
}
Execution::RunBuild(config) => {
config.build = resources
.builds
.get(&config.build)
.map(|b| b.name.clone())
.unwrap_or_default();
}
Execution::Deploy(config) => {
config.deployment = resources
.deployments
.get(&config.deployment)
.map(|d| d.name.clone())
.unwrap_or_default();
}
Execution::StartContainer(config) => {
config.deployment = resources
.deployments
.get(&config.deployment)
.map(|d| d.name.clone())
.unwrap_or_default();
}
Execution::StopContainer(config) => {
config.deployment = resources
.deployments
.get(&config.deployment)
.map(|d| d.name.clone())
.unwrap_or_default();
}
Execution::RemoveContainer(config) => {
config.deployment = resources
.deployments
.get(&config.deployment)
.map(|d| d.name.clone())
.unwrap_or_default();
}
Execution::CloneRepo(config) => {
config.repo = resources
.repos
.get(&config.repo)
.map(|d| d.name.clone())
.unwrap_or_default();
}
Execution::PullRepo(config) => {
config.repo = resources
.repos
.get(&config.repo)
.map(|d| d.name.clone())
.unwrap_or_default();
}
Execution::StopAllContainers(config) => {
config.server = resources
.servers
.get(&config.server)
.map(|d| d.name.clone())
.unwrap_or_default();
}
Execution::PruneNetworks(config) => {
config.server = resources
.servers
.get(&config.server)
.map(|d| d.name.clone())
.unwrap_or_default();
}
Execution::PruneImages(config) => {
config.server = resources
.servers
.get(&config.server)
.map(|d| d.name.clone())
.unwrap_or_default();
}
Execution::PruneContainers(config) => {
config.server = resources
.servers
.get(&config.server)
.map(|d| d.name.clone())
.unwrap_or_default();
}
Execution::RunSync(config) => {
config.sync = resources
.syncs
.get(&config.sync)
.map(|s| s.name.clone())
.unwrap_or_default();
}
}
}
}
Ok(original.partial_diff(update))
}
async fn run_updates(
mut to_create: ToCreate<Self::PartialConfig>,
mut to_update: ToUpdate<Self::PartialConfig>,
to_delete: ToDelete,
) -> Option<Log> {
if to_create.is_empty()
&& to_update.is_empty()
&& to_delete.is_empty()
{
return None;
}
let mut log =
format!("running updates on {}s", Self::resource_type());
for name in to_delete {
if let Err(e) =
crate::resource::delete::<Procedure>(&name, sync_user()).await
{
log.push_str(&format!(
"{}: failed to delete {} '{}' | {e:#}",
colored("ERROR", "red"),
Self::resource_type(),
bold(&name),
))
} else {
log.push_str(&format!(
"{}: {} {} '{}'",
muted("INFO"),
colored("deleted", "red"),
Self::resource_type(),
bold(&name)
));
}
}
if to_update.is_empty() && to_create.is_empty() {
return Some(Log::simple(
&format!("Update {}s", Self::resource_type()),
log,
));
}
for i in 0..10 {
let mut to_pull = Vec::new();
for ToUpdateItem {
id,
resource,
update_description,
update_tags,
} in &to_update
{
// Update resource
let name = resource.name.clone();
let tags = resource.tags.clone();
let description = resource.description.clone();
if *update_description {
run_update_description::<Procedure>(
id.clone(),
&name,
description,
&mut log,
)
.await;
}
if *update_tags {
run_update_tags::<Procedure>(
id.clone(),
&name,
tags,
&mut log,
)
.await;
}
if !resource.config.is_none() {
if let Err(e) = crate::resource::update::<Procedure>(
id,
resource.config.clone(),
sync_user(),
)
.await
{
if i == 9 {
log.push('\n');
log.push_str(&format!(
"{}: failed to update {} '{}' | {e:#}",
colored("ERROR", "red"),
Self::resource_type(),
bold(&name)
));
}
continue;
}
}
log.push('\n');
log.push_str(&format!(
"{}: {} '{}' updated",
muted("INFO"),
Self::resource_type(),
bold(&name)
));
// have to clone out so to_update is mutable
to_pull.push(id.clone());
}
//
to_update.retain(|resource| !to_pull.contains(&resource.id));
let mut to_pull = Vec::new();
for resource in &to_create {
let name = resource.name.clone();
let tags = resource.tags.clone();
let description = resource.description.clone();
let id = match crate::resource::create::<Procedure>(
&name,
resource.config.clone(),
sync_user(),
)
.await
{
Ok(resource) => resource.id,
Err(e) => {
if i == 9 {
log.push('\n');
log.push_str(&format!(
"{}: failed to create {} '{}' | {e:#}",
colored("ERROR", "red"),
Self::resource_type(),
bold(&name)
));
}
continue;
}
};
run_update_tags::<Procedure>(
id.clone(),
&name,
tags,
&mut log,
)
.await;
run_update_description::<Procedure>(
id,
&name,
description,
&mut log,
)
.await;
log.push_str(&format!(
"\n{}: {} {} '{}'",
muted("INFO"),
colored("created", "green"),
Self::resource_type(),
bold(&name)
));
to_pull.push(name);
}
to_create.retain(|resource| !to_pull.contains(&resource.name));
if to_update.is_empty() && to_create.is_empty() {
// info!("all procedures synced");
return Some(Log::simple("Update Procedures", log));
}
}
warn!("procedure sync loop exited after max iterations");
todo!()
}
}

21
bin/core/src/resource/mod.rs
View File

@@ -1,4 +1,4 @@
use std::str::FromStr;
use std::{collections::HashMap, str::FromStr};
use anyhow::{anyhow, Context};
use futures::future::join_all;
@@ -63,14 +63,16 @@ pub use repo::{
/// Implement on each monitor resource for common methods
pub trait MonitorResource {
type ListItem: Serialize + Send;
type Config: Send
type Config: Clone
+ Send
+ Sync
+ Unpin
+ Serialize
+ DeserializeOwned
+ From<Self::PartialConfig>
+ PartialDiff<Self::PartialConfig, Self::ConfigDiff>
+ 'static;
type PartialConfig: Into<Self::Config> + Serialize;
type PartialConfig: From<Self::Config> + Serialize + MaybeNone;
type ConfigDiff: Into<Self::PartialConfig>
+ Serialize
+ Diff
@@ -270,6 +272,19 @@ async fn list_full_for_user_using_document<T: MonitorResource>(
})
}
pub async fn get_id_to_resource_map<T: MonitorResource>(
) -> anyhow::Result<HashMap<String, Resource<T::Config, T::Info>>> {
let res = find_collect(T::coll().await, None, None)
.await
.with_context(|| {
format!("failed to pull {}s from mongo", T::resource_type())
})?
.into_iter()
.map(|r| (r.id.clone(), r))
.collect();
Ok(res)
}
// =======
// CREATE
// =======

9
bin/core/src/resource/procedure.rs
View File

@@ -15,6 +15,7 @@ use monitor_client::{
repo::Repo,
resource::Resource,
server::Server,
sync::ResourceSync,
update::{ResourceTargetVariant, Update},
user::User,
Operation,
@@ -271,7 +272,13 @@ async fn validate_config(
params.server = server.id;
}
Execution::RunSync(params) => {
todo!()
let sync = super::get_check_permissions::<ResourceSync>(
&params.sync,
user,
PermissionLevel::Execute,
)
.await?;
params.sync = sync.id;
}
}
}

48
client/core/rs/src/entities/toml.rs
View File

@@ -6,20 +6,13 @@ use super::{
permission::PermissionLevel, procedure::PartialProcedureConfig,
repo::PartialRepoConfig, server::PartialServerConfig,
server_template::PartialServerTemplateConfig,
update::ResourceTarget, variable::Variable,
sync::PartialResourceSyncConfig, update::ResourceTarget,
variable::Variable,
};
/// Specifies resources to sync on monitor
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct ResourcesToml {
#[serde(
default,
rename = "server_template",
skip_serializing_if = "Vec::is_empty"
)]
pub server_templates:
Vec<ResourceToml<PartialServerTemplateConfig>>,
#[serde(
default,
rename = "server",
@@ -27,13 +20,6 @@ pub struct ResourcesToml {
)]
pub servers: Vec<ResourceToml<PartialServerConfig>>,
#[serde(
default,
rename = "build",
skip_serializing_if = "Vec::is_empty"
)]
pub builds: Vec<ResourceToml<PartialBuildConfig>>,
#[serde(
default,
rename = "deployment",
@@ -43,10 +29,10 @@ pub struct ResourcesToml {
#[serde(
default,
rename = "builder",
rename = "build",
skip_serializing_if = "Vec::is_empty"
)]
pub builders: Vec<ResourceToml<PartialBuilderConfig>>,
pub builds: Vec<ResourceToml<PartialBuildConfig>>,
#[serde(
default,
@@ -55,6 +41,20 @@ pub struct ResourcesToml {
)]
pub repos: Vec<ResourceToml<PartialRepoConfig>>,
#[serde(
default,
rename = "procedure",
skip_serializing_if = "Vec::is_empty"
)]
pub procedures: Vec<ResourceToml<PartialProcedureConfig>>,
#[serde(
default,
rename = "builder",
skip_serializing_if = "Vec::is_empty"
)]
pub builders: Vec<ResourceToml<PartialBuilderConfig>>,
#[serde(
default,
rename = "alerter",
@@ -64,10 +64,18 @@ pub struct ResourcesToml {
#[serde(
default,
rename = "procedure",
rename = "server_template",
skip_serializing_if = "Vec::is_empty"
)]
pub procedures: Vec<ResourceToml<PartialProcedureConfig>>,
pub server_templates:
Vec<ResourceToml<PartialServerTemplateConfig>>,
#[serde(
default,
rename = "sync",
skip_serializing_if = "Vec::is_empty"
)]
pub syncs: Vec<ResourceToml<PartialResourceSyncConfig>>,
#[serde(
default,

19
client/core/rs/src/entities/user.rs
View File

@@ -87,7 +87,10 @@ impl User {
}
pub fn is_service_user(user_id: &str) -> bool {
matches!(user_id, "Procedure" | "Github" | "Auto Redeploy")
matches!(
user_id,
"Procedure" | "Github" | "Auto Redeploy" | "Resource Sync"
)
}
}
@@ -96,6 +99,7 @@ pub fn admin_service_user(user_id: &str) -> Option<User> {
"Procedure" => procedure_user().to_owned().into(),
"Github" => github_user().to_owned().into(),
"Auto Redeploy" => auto_redeploy_user().to_owned().into(),
"Resource Sync" => sync_user().to_owned().into(),
_ => None,
}
}
@@ -139,6 +143,19 @@ pub fn auto_redeploy_user() -> &'static User {
})
}
pub fn sync_user() -> &'static User {
static SYNC_USER: OnceLock<User> = OnceLock::new();
SYNC_USER.get_or_init(|| {
let id_name = String::from("Resource Sync");
User {
id: id_name.clone(),
username: id_name,
admin: true,
..Default::default()
}
})
}
#[typeshare]
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(tag = "type", content = "data")]

19
lib/sync/Cargo.toml
View File

@@ -1,19 +0,0 @@
[package]
name = "sync"
version.workspace = true
edition.workspace = true
authors.workspace = true
license.workspace = true
repository.workspace = true
homepage.workspace = true
[dependencies]
monitor_client.workspace = true
#
partial_derive2.workspace = true
#
tracing.workspace = true
colored.workspace = true
anyhow.workspace = true
serde.workspace = true
toml.workspace = true

3
lib/sync/src/lib.rs
View File

@@ -1,3 +0,0 @@
pub mod file;
pub mod resource;
pub mod resources;

438
lib/sync/src/resource.rs
View File

@@ -1,438 +0,0 @@
use std::collections::HashMap;
use monitor_client::entities::{
resource::Resource, tag::Tag, toml::ResourceToml,
update::ResourceTarget,
};
use partial_derive2::{Diff, MaybeNone, PartialDiff};
use serde::Serialize;
pub type ToUpdate<T> = Vec<ToUpdateItem<T>>;
pub type ToCreate<T> = Vec<ResourceToml<T>>;
/// Vec of resource names
pub type ToDelete = Vec<String>;
type UpdatesResult<T> = (ToCreate<T>, ToUpdate<T>, ToDelete);
pub struct ToUpdateItem<T> {
pub id: String,
pub resource: ResourceToml<T>,
pub update_description: bool,
pub update_tags: bool,
}
/// Implement this one depending on environment
pub trait ResourceSync: Sized {
type Config: Clone
+ Default
+ Send
+ From<Self::PartialConfig>
+ PartialDiff<Self::PartialConfig, Self::ConfigDiff>
+ 'static;
type Info: Default + 'static;
type PartialConfig: std::fmt::Debug
+ Clone
+ Send
+ From<Self::Config>
+ Serialize
+ MaybeNone
+ From<Self::ConfigDiff>
+ 'static;
type ConfigDiff: Diff + MaybeNone;
fn name_to_resource(
) -> &'static HashMap<String, Resource<Self::Config, Self::Info>>;
/// Creates the resource and returns created id.
#[allow(async_fn_in_trait)]
async fn create(
resource: ResourceToml<Self::PartialConfig>,
) -> anyhow::Result<String>;
/// Updates the resource at id with the partial config.
#[allow(async_fn_in_trait)]
async fn update(
id: String,
resource: ResourceToml<Self::PartialConfig>,
) -> anyhow::Result<()>;
/// Deletes the target resource
#[allow(async_fn_in_trait)]
async fn delete(id_or_name: String) -> anyhow::Result<()>;
#[allow(async_fn_in_trait)]
async fn update_tags(
id: String,
tags: Vec<String>,
) -> anyhow::Result<()>;
#[allow(async_fn_in_trait)]
async fn update_description(
id: String,
description: String,
) -> anyhow::Result<()>;
/// Diffs the declared toml (partial) against the full existing config.
/// Removes all fields from toml (partial) that haven't changed.
fn get_diff(
original: Self::Config,
update: Self::PartialConfig,
) -> anyhow::Result<Self::ConfigDiff>;
}
pub trait ResourceSyncOuter<
Implementer: ResourceSync,
Logger: SyncLogger<Implementer>,
> where
Self: Sized,
{
fn display() -> &'static str;
fn resource_target(id: String) -> ResourceTarget;
#[allow(async_fn_in_trait)]
async fn run_updates(
to_create: ToCreate<Implementer::PartialConfig>,
to_update: ToUpdate<Implementer::PartialConfig>,
to_delete: ToDelete,
) {
for resource in to_create {
let name = resource.name.clone();
let tags = resource.tags.clone();
let description = resource.description.clone();
let id = match Implementer::create(resource).await {
Ok(id) => id,
Err(e) => {
Logger::log_failed_create(&name, e);
// warn!(
// "failed to create {} {name} | {e:#}",
// Self::display(),
// );
continue;
}
};
run_update_tags::<Implementer, Self, Logger>(
id.clone(),
&name,
tags,
)
.await;
run_update_description::<Implementer, Self, Logger>(
id,
&name,
description,
)
.await;
Logger::log_created(&name);
// info!(
// "{} {} '{}'",
// "created".green().bold(),
// Self::display(),
// name.bold(),
// );
}
for ToUpdateItem {
id,
resource,
update_description,
update_tags,
} in to_update
{
// Update resource
let name = resource.name.clone();
let tags = resource.tags.clone();
let description = resource.description.clone();
if update_description {
run_update_description::<Implementer, Self, Logger>(
id.clone(),
&name,
description,
)
.await;
}
if update_tags {
run_update_tags::<Implementer, Self, Logger>(
id.clone(),
&name,
tags,
)
.await;
}
if !resource.config.is_none() {
if let Err(e) = Implementer::update(id, resource).await {
Logger::log_failed_update(&name, e);
// warn!(
// "failed to update config on {} {name} | {e:#}",
// Self::display()
// );
} else {
Logger::log_updated(&name);
// info!(
// "{} {} '{}' configuration",
// "updated".blue().bold(),
// Self::display(),
// name.bold(),
// );
}
}
}
for resource in to_delete {
if let Err(e) = Implementer::delete(resource.clone()).await {
Logger::log_failed_delete(&resource, e);
// warn!(
// "failed to delete {} {resource} | {e:#}",
// Self::display()
// );
} else {
Logger::log_deleted(&resource);
// info!(
// "{} {} '{}'",
// "deleted".red().bold(),
// Self::display(),
// resource.bold(),
// );
}
}
}
}
pub trait SyncLogger<Implementer: ResourceSync>
where
Self: Sized,
{
fn log_to_create(
resource: &ResourceToml<Implementer::PartialConfig>,
);
fn log_to_update(
name: &str,
description: &str,
tags: &[String],
diff: &Implementer::ConfigDiff,
);
fn log_to_delete(name: &str);
fn log_created(name: &str);
fn log_failed_create(name: &str, e: anyhow::Error);
fn log_updated(name: &str);
fn log_failed_update(name: &str, e: anyhow::Error);
fn log_deleted(name: &str);
fn log_failed_delete(name: &str, e: anyhow::Error);
fn log_tags_updated(name: &str);
fn log_failed_tag_update(name: &str, e: anyhow::Error);
fn log_description_updated(name: &str);
fn log_failed_description_update(name: &str, e: anyhow::Error);
fn log_procedure_sync_failed_max_iter();
}
pub trait IdToTag {
fn id_to_tag() -> HashMap<String, Tag>;
}
/// Gets all the resources to update, logging along the way.
pub fn get_updates<Implementer, Resource, Tags, Logger>(
resources: Vec<ResourceToml<Implementer::PartialConfig>>,
delete: bool,
) -> anyhow::Result<UpdatesResult<Implementer::PartialConfig>>
where
Implementer: ResourceSync,
Resource: ResourceSyncOuter<Implementer, Logger>,
Tags: IdToTag,
Logger: SyncLogger<Implementer>,
{
let map = Implementer::name_to_resource();
let mut to_create = ToCreate::<Implementer::PartialConfig>::new();
let mut to_update = ToUpdate::<Implementer::PartialConfig>::new();
let mut to_delete = ToDelete::new();
if delete {
for resource in map.values() {
if !resources.iter().any(|r| r.name == resource.name) {
to_delete.push(resource.name.clone());
}
}
}
for mut resource in resources {
match map.get(&resource.name) {
Some(original) => {
// First merge toml resource config (partial) onto default resource config.
// Makes sure things that aren't defined in toml (come through as None) actually get removed.
let config: Implementer::Config = resource.config.into();
resource.config = config.into();
let diff = Implementer::get_diff(
original.config.clone(),
resource.config,
)?;
let tags = Tags::id_to_tag();
let original_tags = original
.tags
.iter()
.filter_map(|id| tags.get(id).map(|t| t.name.clone()))
.collect::<Vec<_>>();
// Only proceed if there are any fields to update,
// or a change to tags / description
if diff.is_none()
&& resource.description == original.description
&& resource.tags == original_tags
{
continue;
}
Logger::log_to_update(
&resource.name,
&resource.description,
&resource.tags,
&diff,
);
// println!(
// "\n{}: {}: '{}'\n-------------------",
// "UPDATE".blue(),
// Resource::display(),
// resource.name.bold(),
// );
// let mut lines = Vec::<String>::new();
// if resource.description != original.description {
// lines.push(format!(
// "{}: 'description'\n{}: {}\n{}: {}",
// "field".dimmed(),
// "from".dimmed(),
// original.description.red(),
// "to".dimmed(),
// resource.description.green()
// ))
// }
// if resource.tags != original_tags {
// let from = format!("{:?}", original_tags).red();
// let to = format!("{:?}", resource.tags).green();
// lines.push(format!(
// "{}: 'tags'\n{}: {from}\n{}: {to}",
// "field".dimmed(),
// "from".dimmed(),
// "to".dimmed(),
// ));
// }
// lines.extend(diff.iter_field_diffs().map(
// |FieldDiff { field, from, to }| {
// format!(
// "{}: '{field}'\n{}: {}\n{}: {}",
// "field".dimmed(),
// "from".dimmed(),
// from.red(),
// "to".dimmed(),
// to.green()
// )
// },
// ));
// println!("{}", lines.join("\n-------------------\n"));
// Minimizes updates through diffing.
resource.config = diff.into();
let update = ToUpdateItem {
id: original.id.clone(),
update_description: resource.description
!= original.description,
update_tags: resource.tags != original_tags,
resource,
};
to_update.push(update);
}
None => {
Logger::log_to_create(&resource);
// println!(
// "\n{}: {}: {}\n{}: {}\n{}: {:?}\n{}: {}",
// "CREATE".green(),
// Resource::display(),
// resource.name.bold().green(),
// "description".dimmed(),
// resource.description,
// "tags".dimmed(),
// resource.tags,
// "config".dimmed(),
// serde_json::to_string_pretty(&resource.config)?
// );
to_create.push(resource);
}
}
}
for name in &to_delete {
Logger::log_to_delete(name);
// println!(
// "\n{}: {}: '{}'\n-------------------",
// "DELETE".red(),
// Resource::display(),
// name.bold(),
// );
}
Ok((to_create, to_update, to_delete))
}
pub async fn run_update_tags<Implementer, Resource, Logger>(
id: String,
name: &str,
tags: Vec<String>,
) where
Implementer: ResourceSync,
Resource: ResourceSyncOuter<Implementer, Logger>,
Logger: SyncLogger<Implementer>,
{
// Update tags
if let Err(e) = Implementer::update_tags(id, tags).await {
Logger::log_failed_tag_update(name, e);
// tracing::warn!(
// "failed to update tags on {} {name} | {e:#}",
// Resource::display(),
// );
} else {
Logger::log_tags_updated(name);
// tracing::info!(
// "{} {} '{}' tags",
// "updated".blue().bold(),
// Resource::display(),
// name.bold(),
// );
}
}
pub async fn run_update_description<Implementer, Resource, Logger>(
id: String,
name: &str,
description: String,
) where
Implementer: ResourceSync,
Resource: ResourceSyncOuter<Implementer, Logger>,
Logger: SyncLogger<Implementer>,
{
if let Err(e) =
Implementer::update_description(id, description).await
{
Logger::log_failed_description_update(name, e);
// warn!("failed to update resource {id} description | {e:#}");
} else {
Logger::log_description_updated(name);
// info!(
// "{} {} '{}' description",
// "updated".blue().bold(),
// Resource::display(),
// name.bold(),
// );
}
}

243
lib/sync/src/resources.rs
View File

@@ -1,243 +0,0 @@
use monitor_client::entities::{
alerter::Alerter, build::Build, builder::Builder,
deployment::Deployment, procedure::Procedure, repo::Repo,
server::Server, server_template::ServerTemplate, sync,
update::ResourceTarget,
};
use partial_derive2::MaybeNone;
use crate::resource::{
ResourceSync, ResourceSyncOuter, SyncLogger, ToUpdateItem,
};
impl<Implementer: ResourceSync, Logger: SyncLogger<Implementer>>
ResourceSyncOuter<Implementer, Logger> for Server
{
fn display() -> &'static str {
"server"
}
fn resource_target(id: String) -> ResourceTarget {
ResourceTarget::Server(id)
}
}
impl<Implementer: ResourceSync, Logger: SyncLogger<Implementer>>
ResourceSyncOuter<Implementer, Logger> for Deployment
{
fn display() -> &'static str {
"deployment"
}
fn resource_target(id: String) -> ResourceTarget {
ResourceTarget::Deployment(id)
}
}
impl<Implementer: ResourceSync, Logger: SyncLogger<Implementer>>
ResourceSyncOuter<Implementer, Logger> for Build
{
fn display() -> &'static str {
"build"
}
fn resource_target(id: String) -> ResourceTarget {
ResourceTarget::Build(id)
}
}
impl<Implementer: ResourceSync, Logger: SyncLogger<Implementer>>
ResourceSyncOuter<Implementer, Logger> for Repo
{
fn display() -> &'static str {
"repo"
}
fn resource_target(id: String) -> ResourceTarget {
ResourceTarget::Repo(id)
}
}
impl<Implementer: ResourceSync, Logger: SyncLogger<Implementer>>
ResourceSyncOuter<Implementer, Logger> for Alerter
{
fn display() -> &'static str {
"alerter"
}
fn resource_target(id: String) -> ResourceTarget {
ResourceTarget::Alerter(id)
}
}
impl<Implementer: ResourceSync, Logger: SyncLogger<Implementer>>
ResourceSyncOuter<Implementer, Logger> for Builder
{
fn display() -> &'static str {
"builder"
}
fn resource_target(id: String) -> ResourceTarget {
ResourceTarget::Builder(id)
}
}
impl<Implementer: ResourceSync, Logger: SyncLogger<Implementer>>
ResourceSyncOuter<Implementer, Logger> for ServerTemplate
{
fn display() -> &'static str {
"server_template"
}
fn resource_target(id: String) -> ResourceTarget {
ResourceTarget::ServerTemplate(id)
}
}
impl<Implementer: ResourceSync, Logger: SyncLogger<Implementer>>
ResourceSyncOuter<Implementer, Logger> for sync::ResourceSync
{
fn display() -> &'static str {
"resource_sync"
}
fn resource_target(id: String) -> ResourceTarget {
ResourceTarget::ResourceSync(id)
}
}
impl<Implementer: ResourceSync, Logger: SyncLogger<Implementer>>
ResourceSyncOuter<Implementer, Logger> for Procedure
{
fn display() -> &'static str {
"procedure"
}
fn resource_target(id: String) -> ResourceTarget {
ResourceTarget::Procedure(id)
}
async fn run_updates(
mut to_create: crate::resource::ToCreate<
<Implementer as ResourceSync>::PartialConfig,
>,
mut to_update: crate::resource::ToUpdate<
<Implementer as ResourceSync>::PartialConfig,
>,
to_delete: crate::resource::ToDelete,
) {
for name in to_delete {
if let Err(e) = Implementer::delete(name.clone()).await {
Logger::log_failed_delete(&name, e);
// warn!("failed to delete procedure {name} | {e:#}",);
} else {
Logger::log_deleted(&name);
// info!(
// "{} procedure '{}'",
// "deleted".red().bold(),
// name.bold(),
// );
}
}
if to_update.is_empty() && to_create.is_empty() {
return;
}
for i in 0..10 {
let mut to_pull = Vec::new();
for ToUpdateItem {
id,
resource,
update_description,
update_tags,
} in &to_update
{
// Update resource
let name = resource.name.clone();
let tags = resource.tags.clone();
let description = resource.description.clone();
if *update_description {
crate::resource::run_update_description::<
Implementer,
Self,
Logger,
>(id.clone(), &name, description)
.await;
}
if *update_tags {
crate::resource::run_update_tags::<Implementer, Self, Logger>(
id.clone(),
&name,
tags,
)
.await;
}
if !resource.config.is_none() {
if let Err(e) =
Implementer::update(id.clone(), resource.clone()).await
{
if i == 9 {
Logger::log_failed_update(&name, e);
// warn!(
// "failed to update {} {name} | {e:#}",
// Self::display()
// );
}
}
}
// info!("{} {name} updated", Self::display());
Logger::log_updated(&name);
// have to clone out so to_update is mutable
to_pull.push(id.clone());
}
//
to_update.retain(|resource| !to_pull.contains(&resource.id));
let mut to_pull = Vec::new();
for resource in &to_create {
let name = resource.name.clone();
let tags = resource.tags.clone();
let description = resource.description.clone();
let id = match Implementer::create(resource.clone()).await {
Ok(id) => id,
Err(e) => {
if i == 9 {
// warn!(
// "failed to create {} {name} | {e:#}",
// Self::display(),
// );
Logger::log_failed_create(&name, e);
}
continue;
}
};
crate::resource::run_update_tags::<Implementer, Self, Logger>(
id.clone(),
&name,
tags,
)
.await;
crate::resource::run_update_description::<
Implementer,
Self,
Logger,
>(id, &name, description)
.await;
Logger::log_created(&name);
// info!("{} {name} created", Self::display());
to_pull.push(name);
}
to_create.retain(|resource| !to_pull.contains(&resource.name));
if to_update.is_empty() && to_create.is_empty() {
// info!("all procedures synced");
return;
}
}
Logger::log_procedure_sync_failed_max_iter();
// warn!("procedure sync loop exited after max iterations");
}
}