From c12eb4045910594adb631b5a1961f514caf095c6 Mon Sep 17 00:00:00 2001 From: mbecker20 Date: Fri, 7 Jul 2023 06:55:47 +0000 Subject: [PATCH] add api to search resources by tag --- Cargo.lock | 1 - Cargo.toml | 1 - core/src/db.rs | 4 +- core/src/requests/read/deployment.rs | 7 +- core/src/requests/read/mod.rs | 4 + core/src/requests/read/search.rs | 203 ++++++++++++++++++++++++++ core/src/requests/read/server.rs | 1 + lib/types/Cargo.toml | 1 - lib/types/src/entities/build.rs | 3 +- lib/types/src/entities/builder.rs | 3 +- lib/types/src/entities/deployment.rs | 3 +- lib/types/src/entities/mod.rs | 1 + lib/types/src/entities/repo.rs | 3 +- lib/types/src/entities/server/mod.rs | 3 +- lib/types/src/entities/tag.rs | 66 +++++++++ lib/types/src/entities/update.rs | 5 +- lib/types/src/entities/user.rs | 3 +- lib/types/src/requests/read/mod.rs | 2 + lib/types/src/requests/read/search.rs | 25 ++++ lib/types/src/requests/read/server.rs | 1 + lib/types/src/requests/write/tags.rs | 43 +++++- 21 files changed, 356 insertions(+), 27 deletions(-) create mode 100644 core/src/requests/read/search.rs create mode 100644 lib/types/src/entities/tag.rs create mode 100644 lib/types/src/requests/read/search.rs diff --git a/Cargo.lock b/Cargo.lock index 1221694ec..eb4bf3c06 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1908,7 +1908,6 @@ dependencies = [ "async-trait", "async_timing_util", "bollard", - "bson", "derive_builder", "derive_variants", "make_option", diff --git a/Cargo.toml b/Cargo.toml index 608366616..088332833 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,7 +32,6 @@ envy = "0.4" anyhow = "1.0" log = "0.4" simple_logger = "4.1" -bson = "2.6.1" bollard = "0.14.0" derive_builder = "0.12" typeshare = "1.0.1" diff --git a/core/src/db.rs b/core/src/db.rs index 5c788178b..859e5ad89 100644 --- a/core/src/db.rs +++ b/core/src/db.rs @@ -1,6 +1,6 @@ use monitor_types::entities::{ build::Build, builder::Builder, deployment::Deployment, repo::Repo, server::Server, - update::Update, user::User, + update::Update, user::User, tag::CustomTag, }; use mungos::{Collection, Indexed, Mungos}; @@ -13,6 +13,7 @@ pub struct DbClient { pub builds: Collection, pub builders: Collection, pub repos: Collection, + pub tags: Collection, pub updates: Collection, } @@ -30,6 +31,7 @@ impl DbClient { builds: Build::collection(&mungos, &config.mongo.db_name, true).await?, builders: Builder::collection(&mungos, &config.mongo.db_name, true).await?, repos: Repo::collection(&mungos, &config.mongo.db_name, true).await?, + tags: CustomTag::collection(&mungos, &config.mongo.db_name, true).await?, updates: Update::collection(&mungos, &config.mongo.db_name, true).await?, }; Ok(client) diff --git a/core/src/requests/read/deployment.rs b/core/src/requests/read/deployment.rs index af7944da0..5a7f35aca 100644 --- a/core/src/requests/read/deployment.rs +++ b/core/src/requests/read/deployment.rs @@ -7,7 +7,7 @@ use monitor_types::{ entities::{ deployment::{ Deployment, DeploymentActionState, DeploymentConfig, DeploymentImage, - DockerContainerState, DockerContainerStats, + DockerContainerStats, }, update::{Log, UpdateStatus}, Operation, PermissionLevel, @@ -64,10 +64,7 @@ impl Resolve for State { id: deployment.id, name: deployment.name, tags: deployment.tags, - state: status - .as_ref() - .map(|s| s.state) - .unwrap_or(DockerContainerState::Unknown), + state: status.as_ref().map(|s| s.state).unwrap_or_default(), status: status .as_ref() .and_then(|s| s.container.as_ref().and_then(|c| c.status.to_owned())), diff --git a/core/src/requests/read/mod.rs b/core/src/requests/read/mod.rs index aaf3e5988..193e495e7 100644 --- a/core/src/requests/read/mod.rs +++ b/core/src/requests/read/mod.rs @@ -20,6 +20,7 @@ mod builder; mod deployment; mod repo; mod server; +mod search; #[typeshare] #[derive(Serialize, Deserialize, Debug, Clone, Resolver)] @@ -28,6 +29,9 @@ mod server; #[serde(tag = "type", content = "params")] #[allow(clippy::enum_variant_names, clippy::large_enum_variant)] pub enum ReadRequest { + // ==== SEARCH ==== + FindResources(FindResources), + // ==== SERVER ==== GetPeripheryVersion(GetPeripheryVersion), GetSystemInformation(GetSystemInformation), diff --git a/core/src/requests/read/search.rs b/core/src/requests/read/search.rs new file mode 100644 index 000000000..e9630e4f8 --- /dev/null +++ b/core/src/requests/read/search.rs @@ -0,0 +1,203 @@ +use std::str::FromStr; + +use anyhow::Context; +use async_trait::async_trait; +use monitor_types::{ + entities::{ + tag::Tag, + update::ResourceTargetVariant::{self, *}, + PermissionLevel, + }, + permissioned::Permissioned, + requests::read::{ + BuildListItem, DeploymentListItem, FindResources, FindResourcesResponse, RepoListItem, + ServerListItem, + }, +}; +use mungos::mongodb::bson::{doc, oid::ObjectId, Document}; +use resolver_api::Resolve; + +use crate::{auth::RequestUser, state::State}; + +const ALL_RESOURCE_TYPES: [ResourceTargetVariant; 4] = [Server, Build, Deployment, Repo]; + +#[async_trait] +impl Resolve for State { + async fn resolve( + &self, + FindResources { tags }: FindResources, + user: RequestUser, + ) -> anyhow::Result { + let SeperateTags { + resource_types, + server_ids, + custom_tag_ids, + } = seperate_tags(tags); + + let mut query = Document::new(); + + if !custom_tag_ids.is_empty() { + query.insert("tags", doc! { "$all": custom_tag_ids }); + } + + let mut response = FindResourcesResponse::default(); + + for resource_type in resource_types { + match resource_type { + Server => { + let servers = if server_ids.is_empty() { + self.db.servers.get_some(query.clone(), None).await? + } else { + let server_ids = server_ids + .iter() + .map(|id| { + ObjectId::from_str(id) + .context("failed to parse server id as ObjectId") + }) + .collect::>>()?; + self.db + .servers + .get_some(doc! { "_id": { "$in": server_ids } }, None) + .await? + } + .into_iter() + .filter(|s| s.get_user_permissions(&user.id) > PermissionLevel::None); + for server in servers { + let status = self + .server_status_cache + .get(&server.id) + .await + .map(|s| s.status) + .unwrap_or_default(); + let item = ServerListItem { + status, + id: server.id, + name: server.name, + tags: server.tags, + }; + response.servers.push(item); + } + } + Deployment => { + let mut query = query.clone(); + + if !server_ids.is_empty() { + query.insert("config.server_id", doc! { "$in": &server_ids }); + } + + let deployments = self + .db + .deployments + .get_some(query, None) + .await? + .into_iter() + .filter(|d| d.get_user_permissions(&user.id) > PermissionLevel::Read); + + for deployment in deployments { + let status = self.deployment_status_cache.get(&deployment.id).await; + let item = DeploymentListItem { + id: deployment.id, + name: deployment.name, + tags: deployment.tags, + state: status.as_ref().map(|s| s.state).unwrap_or_default(), + status: status.as_ref().and_then(|s| { + s.container.as_ref().and_then(|c| c.status.to_owned()) + }), + image: String::new(), + version: String::new(), + }; + response.deployments.push(item); + } + } + Build => { + let mut query = query.clone(); + + if !server_ids.is_empty() { + query.insert( + "config.builder.params.server_id", + doc! { "$in": &server_ids }, + ); + } + + let builds = self + .db + .builds + .get_some(query, None) + .await? + .into_iter() + .filter(|d| d.get_user_permissions(&user.id) > PermissionLevel::Read); + + for build in builds { + let item = BuildListItem { + id: build.id, + name: build.name, + tags: build.tags, + last_built_at: build.last_built_at, + version: build.config.version, + }; + response.builds.push(item); + } + } + Repo => { + let mut query = query.clone(); + + if !server_ids.is_empty() { + query.insert("config.server_id", doc! { "$in": &server_ids }); + } + + let repos = self + .db + .repos + .get_some(query, None) + .await? + .into_iter() + .filter(|d| d.get_user_permissions(&user.id) > PermissionLevel::Read); + + for repo in repos { + let item = RepoListItem { + id: repo.id, + name: repo.name, + tags: repo.tags, + last_pulled_at: repo.last_pulled_at, + }; + response.repos.push(item); + } + } + _ => unreachable!(), + } + } + + Ok(response) + } +} + +#[derive(Default)] +struct SeperateTags { + resource_types: Vec, + server_ids: Vec, + custom_tag_ids: Vec, +} + +fn seperate_tags(tags: Vec) -> SeperateTags { + let mut seperated = SeperateTags::default(); + + for tag in tags { + match tag { + Tag::Custom { tag_id } => seperated.custom_tag_ids.push(tag_id), + Tag::Server { server_id } => seperated.server_ids.push(server_id), + Tag::ResourceType { resource } => { + if !matches!(resource, Builder | System,) + && !seperated.resource_types.contains(&resource) + { + seperated.resource_types.push(resource); + } + } + } + } + + if seperated.resource_types.is_empty() { + seperated.resource_types = ALL_RESOURCE_TYPES.to_vec(); + } + + seperated +} diff --git a/core/src/requests/read/server.rs b/core/src/requests/read/server.rs index 8aea149bd..d54c92694 100644 --- a/core/src/requests/read/server.rs +++ b/core/src/requests/read/server.rs @@ -73,6 +73,7 @@ impl Resolve for State { ServerListItem { id: server.id, name: server.name, + tags: server.tags, status: status.map(|s| s.status).unwrap_or_default(), } }); diff --git a/lib/types/Cargo.toml b/lib/types/Cargo.toml index d0493066d..46163a724 100644 --- a/lib/types/Cargo.toml +++ b/lib/types/Cargo.toml @@ -16,7 +16,6 @@ strum.workspace = true strum_macros.workspace = true async-trait.workspace = true anyhow.workspace = true -bson.workspace = true async_timing_util.workspace = true resolver_api.workspace = true derive_builder.workspace = true diff --git a/lib/types/src/entities/build.rs b/lib/types/src/entities/build.rs index 36ef9415b..0cced3a99 100644 --- a/lib/types/src/entities/build.rs +++ b/lib/types/src/entities/build.rs @@ -1,7 +1,6 @@ -use bson::{doc, serde_helpers::hex_string_as_object_id}; use derive_builder::Builder; use derive_variants::EnumVariants; -use mungos::MungosIndexed; +use mungos::{MungosIndexed, mongodb::bson::{serde_helpers::hex_string_as_object_id, doc}}; use partial_derive2::Partial; use serde::{Deserialize, Serialize}; use strum_macros::{Display, EnumString}; diff --git a/lib/types/src/entities/builder.rs b/lib/types/src/entities/builder.rs index ef68b8564..6169eed09 100644 --- a/lib/types/src/entities/builder.rs +++ b/lib/types/src/entities/builder.rs @@ -1,7 +1,6 @@ -use bson::serde_helpers::hex_string_as_object_id; use derive_builder::Builder; use derive_variants::EnumVariants; -use mungos::MungosIndexed; +use mungos::{MungosIndexed, mongodb::bson::serde_helpers::hex_string_as_object_id}; use partial_derive2::Partial; use serde::{Deserialize, Serialize}; use strum_macros::{Display, EnumString}; diff --git a/lib/types/src/entities/deployment.rs b/lib/types/src/entities/deployment.rs index 181a53cd4..7936611a2 100644 --- a/lib/types/src/entities/deployment.rs +++ b/lib/types/src/entities/deployment.rs @@ -1,7 +1,6 @@ -use bson::{doc, serde_helpers::hex_string_as_object_id}; use derive_builder::Builder; use derive_variants::EnumVariants; -use mungos::MungosIndexed; +use mungos::{mongodb::bson::{serde_helpers::hex_string_as_object_id, doc}, MungosIndexed}; use partial_derive2::Partial; use serde::{Deserialize, Serialize}; use strum_macros::{Display, EnumString}; diff --git a/lib/types/src/entities/mod.rs b/lib/types/src/entities/mod.rs index 8e3021fc4..88adc3326 100644 --- a/lib/types/src/entities/mod.rs +++ b/lib/types/src/entities/mod.rs @@ -13,6 +13,7 @@ pub mod builder; pub mod deployment; pub mod repo; pub mod server; +pub mod tag; pub mod update; pub mod user; diff --git a/lib/types/src/entities/repo.rs b/lib/types/src/entities/repo.rs index 8e12b6f17..eae2262ff 100644 --- a/lib/types/src/entities/repo.rs +++ b/lib/types/src/entities/repo.rs @@ -1,6 +1,5 @@ -use bson::serde_helpers::hex_string_as_object_id; use derive_builder::Builder; -use mungos::MungosIndexed; +use mungos::{MungosIndexed, mongodb::bson::serde_helpers::hex_string_as_object_id}; use partial_derive2::Partial; use serde::{Deserialize, Serialize}; use typeshare::typeshare; diff --git a/lib/types/src/entities/server/mod.rs b/lib/types/src/entities/server/mod.rs index c9c455ff4..c964a8ef5 100644 --- a/lib/types/src/entities/server/mod.rs +++ b/lib/types/src/entities/server/mod.rs @@ -1,6 +1,5 @@ -use bson::serde_helpers::hex_string_as_object_id; use derive_builder::Builder; -use mungos::MungosIndexed; +use mungos::{MungosIndexed, mongodb::bson::serde_helpers::hex_string_as_object_id}; use partial_derive2::Partial; use serde::{Deserialize, Serialize}; use typeshare::typeshare; diff --git a/lib/types/src/entities/tag.rs b/lib/types/src/entities/tag.rs new file mode 100644 index 000000000..0544cb174 --- /dev/null +++ b/lib/types/src/entities/tag.rs @@ -0,0 +1,66 @@ +use derive_builder::Builder; +use derive_variants::EnumVariants; +use mungos::{ + mongodb::bson::{doc, serde_helpers::hex_string_as_object_id}, + MungosIndexed, +}; +use partial_derive2::Partial; +use serde::{Deserialize, Serialize}; +use strum_macros::{Display, EnumString}; +use typeshare::typeshare; + +use crate::MongoId; + +use super::update::ResourceTargetVariant; + +#[typeshare] +#[derive(Serialize, Deserialize, Debug, Clone, EnumVariants)] +#[variant_derive(Serialize, Deserialize, Debug, Clone, Copy, Display, EnumString)] +#[serde(tag = "type", content = "params")] +pub enum Tag { + ResourceType { resource: ResourceTargetVariant }, // filter by resource type + Server { server_id: String }, // filter by server, eg deployments, builds, repos + Custom { tag_id: String }, // filter by presence of custom tag on resource +} + +#[typeshare] +#[derive(Serialize, Deserialize, Debug, Clone, Builder, MungosIndexed, Partial)] +#[partial_derive(Serialize, Deserialize, Debug, Clone)] +#[unique_doc_index(doc! { "name": 1, "category": 1 })] +pub struct CustomTag { + #[serde( + default, + rename = "_id", + skip_serializing_if = "String::is_empty", + with = "hex_string_as_object_id" + )] + #[builder(setter(skip))] + pub id: MongoId, + + #[index] + pub name: String, + + #[serde(default)] + #[builder(default)] + #[index] + pub category: String, + + #[serde(default)] + #[builder(default)] + pub color: TagColor, +} + +#[typeshare] +#[derive( + Serialize, Deserialize, Debug, Clone, Copy, Display, EnumString, MungosIndexed, Default, +)] +pub enum TagColor { + #[default] + Red, + Green, + Blue, + Yellow, + Purple, + Magenta, + Cyan, +} diff --git a/lib/types/src/entities/update.rs b/lib/types/src/entities/update.rs index f769a6542..9275b5447 100644 --- a/lib/types/src/entities/update.rs +++ b/lib/types/src/entities/update.rs @@ -1,7 +1,6 @@ use async_timing_util::unix_timestamp_ms; -use bson::serde_helpers::hex_string_as_object_id; use derive_variants::EnumVariants; -use mungos::MungosIndexed; +use mungos::{MungosIndexed, mongodb::bson::serde_helpers::hex_string_as_object_id}; use serde::{Deserialize, Serialize}; use strum_macros::{Display, EnumString}; use typeshare::typeshare; @@ -88,7 +87,7 @@ impl Log { #[typeshare] #[derive(Serialize, Deserialize, Debug, Clone, Default, MungosIndexed, EnumVariants)] -#[variant_derive(Serialize, Deserialize, Debug, Clone, Copy, Display, EnumString)] +#[variant_derive(Serialize, Deserialize, Debug, Clone, Copy, Display, EnumString, PartialEq, Eq)] #[serde(tag = "type", content = "id")] pub enum ResourceTarget { #[default] diff --git a/lib/types/src/entities/user.rs b/lib/types/src/entities/user.rs index 280dcdd91..5216694dd 100644 --- a/lib/types/src/entities/user.rs +++ b/lib/types/src/entities/user.rs @@ -1,5 +1,4 @@ -use bson::serde_helpers::hex_string_as_object_id; -use mungos::MungosIndexed; +use mungos::{MungosIndexed, mongodb::bson::serde_helpers::hex_string_as_object_id}; use serde::{Deserialize, Serialize}; use typeshare::typeshare; diff --git a/lib/types/src/requests/read/mod.rs b/lib/types/src/requests/read/mod.rs index 4db1232b5..89955ff42 100644 --- a/lib/types/src/requests/read/mod.rs +++ b/lib/types/src/requests/read/mod.rs @@ -6,6 +6,7 @@ mod build; mod builder; mod deployment; mod repo; +mod search; mod server; mod update; @@ -13,6 +14,7 @@ pub use build::*; pub use builder::*; pub use deployment::*; pub use repo::*; +pub use search::*; pub use server::*; pub use update::*; diff --git a/lib/types/src/requests/read/search.rs b/lib/types/src/requests/read/search.rs new file mode 100644 index 000000000..2adb8a7a3 --- /dev/null +++ b/lib/types/src/requests/read/search.rs @@ -0,0 +1,25 @@ +use resolver_api::derive::Request; +use serde::{Serialize, Deserialize}; +use typeshare::typeshare; + +use crate::entities::tag::Tag; + +use super::{ServerListItem, DeploymentListItem, BuildListItem, RepoListItem}; + +// + +#[typeshare] +#[derive(Serialize, Deserialize, Debug, Clone, Request)] +#[response(FindResourcesResponse)] +pub struct FindResources { + pub tags: Vec, +} + +#[typeshare] +#[derive(Serialize, Deserialize, Debug, Clone, Default)] +pub struct FindResourcesResponse { + pub servers: Vec, + pub deployments: Vec, + pub builds: Vec, + pub repos: Vec, +} \ No newline at end of file diff --git a/lib/types/src/requests/read/server.rs b/lib/types/src/requests/read/server.rs index 950d4cc89..4cc9e234c 100644 --- a/lib/types/src/requests/read/server.rs +++ b/lib/types/src/requests/read/server.rs @@ -42,6 +42,7 @@ pub struct ServerListItem { pub id: String, pub name: String, pub status: ServerStatus, + pub tags: Vec, } // diff --git a/lib/types/src/requests/write/tags.rs b/lib/types/src/requests/write/tags.rs index 2d77133d8..74a256172 100644 --- a/lib/types/src/requests/write/tags.rs +++ b/lib/types/src/requests/write/tags.rs @@ -2,7 +2,44 @@ use resolver_api::derive::Request; use serde::{Serialize, Deserialize}; use typeshare::typeshare; -use crate::entities::update::ResourceTarget; +use crate::entities::{update::ResourceTarget, tag::{CustomTag, TagColor, PartialCustomTag}}; + +#[typeshare(serialized_as = "Partial")] +type _PartialCustomTag = PartialCustomTag; + +// + +#[typeshare] +#[derive(Serialize, Deserialize, Debug, Clone, Request)] +#[response(CustomTag)] +pub struct CreateTag { + pub name: String, + + #[serde(default)] + pub category: String, + + #[serde(default)] + pub color: TagColor +} + +// + +#[typeshare] +#[derive(Serialize, Deserialize, Debug, Clone, Request)] +#[response(CustomTag)] +pub struct DeleteTag { + pub id: String, +} + +// + +#[typeshare] +#[derive(Serialize, Deserialize, Debug, Clone, Request)] +#[response(CustomTag)] +pub struct UpdateTag { + pub id: String, + pub config: _PartialCustomTag +} // @@ -11,7 +48,7 @@ use crate::entities::update::ResourceTarget; #[response(())] pub struct AddTags { pub target: ResourceTarget, - pub tags: Vec, + pub tags: Vec, // custom tag ids } // @@ -21,5 +58,5 @@ pub struct AddTags { #[response(())] pub struct RemoveTags { pub target: ResourceTarget, - pub tags: Vec, + pub tags: Vec, // custom tag ids }