Files
komodo/bin/core/src/api/read/mod.rs
Maxwell Becker 7d4376f426 1.15.0 (#90)
* attach env_file to compose build and compose pull stages

* fmt and bump rust version

* bump dependencies

* ignored for Sqlite message

* fix Build secret args info

* improve secret arguments info

* improve environment, ports, volumes deserializers

* rename `mongo` to `database` in config

* support _FILE in secret env vars

* improve setup - simpler compose

* remove aws ecr container registry support, alpine dockerfiles

* log periphery config

* ssl_enabled mode

* log http vs https

* periphery client accept untrust ssl certs

* fix nav issue from links

* configurable ssl

* KOMODO_ENSURE_SERVER -> KOMODO_FIRST_SERVER

* mount proc and ssl volume

* managed sync

* validate files on host resource path

* remove sync repo not configured guards

* disable confirm dialog

* fix sync hash / message Option

* try dev dockerfile

* refresh sync resources after commit

* socket invalidate handling

* delete dev dockerfile

* Commit Changes

* Add Info tab to syncs

* fix new Info parsing issue with serde default

* refresh stack cache on create / update

* managed syncs can't sync themselves

* managed syncs seems to work

* bump thiserror

* use alpine as main dockerfile

* apt add --no-cache

* disable user write perms, super admin perms to manage admins

* manage admin user UI

* implement disable non admin create frontend

* disable create non admin

* Copy button shown based on permission

* warning message on managed sync

* implement monaco editor

* impl simple match tags config

* resource sync support match tags

* more match tag filtering

* improve config with better saving diffs

* export button use monaco

* deser Conversions with wrapping strings

* envs editing

* don't delete variables / user groups if match tags defined

* env from_str improve

* improve dashboards

* remove core ca stuff for now

* move periphery ssl gen to dedicated file

* default server address periphery:8120

* clean up ssl configs

* server dashboard

* nice test compose

* add discord alerter

* discord alerter

* stack hideInfo logic

* compose setup

* alert table

* improve config hover card style

* update min editor height and stack config

* Feat: Styling Updates (#94)

* sidebar takes full screen height

* add bg accent to navbar

* add aschild prop to topbar alerts trigger

* stylize resource rows

* internally scrollable data tables

* better hover color for outlined button

* always show scrollbar to prevent layout shift

* better hover color for navbar

* rearrange buttons

* fix table and resource row styles

* cleanup scrollbar css

* use page for dashboard instead of section

* fix padding

* resource sync refactor and env keep comments

* frontend build

* improve configs

* config nice

* Feat/UI (#95)

* stylize resource rows

* internally scrollable data tables

* fix table and resource row styles

* use page for dashboard instead of section

* fix padding

* add `ResourcePageHeader` to required components

* add generic resource page header component

* add resource page headers for all components

* add resource notificaitons component

* add `TextUpdateMenu2` for use in resource page

* cleanup resource notificaitons

* update resource page layout

* ui edits

* sync kind of work

* clean up unused import

* syncs seem to work

* new sync pending

* monaco diff hide unchanged regions

* update styling all in config  resource select links

* confirm update default strings

* move procedure Add Stage to left

* update colors / styles

* frontend build

* backend for write file contents to host

* compose reference ports comment out

* server config

* ensure parent directory created

* fix frontend build

* remove default stack run_directory

* fix periphery compose deploy response set

* update compose files

* move server stats under tabs

* fix deployment list item getting correct image when not deployed

* stack updates cache after file write

* edit files on host

* clean up unused imports

* top level config update assignment must be spread

* update deps, move alert module

* move stack module

* move sync module

* move to sync db_client usage after init

* support generic OIDC provider

* init builders / server templates specifying https

* special cases for server / deployment state

* improve alert details

* add builder template `use_https` config

* try downgrade aws sdk ec2 for x86 build

* update debian dockerfiles to rm lists/*

* optionally configure seperate KOMODO_OIDC_REDIRECT

* add defaults to compose.env

* keep tags / search right aligned when view only

* clean up configs

* remove unused migrator deps

* update roadmap support generic OIDC

* initialize sync use confirm button

* key_value syntax highlighting

* smaller debian dockerfiles

* clean up deps.sh

* debian dockerifle

* New config layout (#96)

* new config layout

* fix image config layout and components config

* fix dom nesting and cleanup components

* fix label, make switches flex row

* ensure smooth scroll on hash navigations

* width 180 on config sidebar

* slight edits to config

* log whether https builder

* DISABLED <switch> ENABLED

* fix some more config

* smaller checked component

* server config looking good

* auto initialize compose files when files on host

* stack files on host good

* stack config nice

* remove old config

* deployments looking good

* build looking good

* Repo good

* nice config for builders

* alerter good

* server template config

* syncs good

* tweak stack config

* use status badge for update tables

* unified update page using router params

* replace /updates with unified updates page

* redirect all resource updates to unified update page

* fix reset handling

* unmount legacy page

* try periphery rustls

* rm unused import

* fix broken deps

* add unified alerts apge

* mount new alerts, remove old alerts page

* reroute resource alerts to unified alerts page

* back to periphery openssl

* ssl_enabled defaults to false for backward compat

* reqwest need json feature

* back to og yaml monaco

* Uncomment config fields for clearer config

* clean up compose env

* implement pull or clone, avoid deleting repo directory

* refactor mongo configuration params

* all configs respect empty string null

* add back status to header

* build toml don't have version if not auto incrementing

* fix comile

* fix repo pull cd to correct dir

* fix core pull_or_clone directory

* improve statuses

* remove ' ' from kv list parser

* longer CSRF valid for, to give time to login / accept

* don't compute diff / execute if there are any file_errors

* PartialBuilderConfig enum user inner option

* move errors to top

* fix toml init serializer

* server template and bulder manually add config.params line

* better way to check builder / template params empty

* improve build configs

* merge links into network area deployment

* default periphery config

* improve SystemCommand editor

* better Repo server / builder Info

* improve Alerts / Updates with ResourceSelector

* fix unused frontend

* update ResourceSync description

* toml use [resource.config] syntax

* update toml syntax

* update Build.image_registry schema

* fix repo / stack resource link alias

* reorder image registry

* align toml / yaml parser style

* some config updates

---------

Co-authored-by: Karamvir Singh <67458484+karamvirsingh98@users.noreply.github.com>
Co-authored-by: kv <karamvir.singh98@gmail.com>
2024-10-06 23:54:23 -07:00

575 lines
16 KiB
Rust

use std::{collections::HashSet, sync::OnceLock, time::Instant};
use anyhow::{anyhow, Context};
use axum::{middleware, routing::post, Extension, Router};
use axum_extra::{headers::ContentType, TypedHeader};
use komodo_client::{
api::read::*,
entities::{
build::Build,
builder::{Builder, BuilderConfig},
config::{DockerRegistry, GitProvider},
repo::Repo,
server::Server,
sync::ResourceSync,
user::User,
ResourceTarget,
},
};
use resolver_api::{
derive::Resolver, Resolve, ResolveToString, Resolver,
};
use serde::{Deserialize, Serialize};
use serror::Json;
use typeshare::typeshare;
use uuid::Uuid;
use crate::{
auth::auth_request, config::core_config, helpers::periphery_client,
resource, state::State,
};
mod alert;
mod alerter;
mod build;
mod builder;
mod deployment;
mod permission;
mod procedure;
mod provider;
mod repo;
mod search;
mod server;
mod server_template;
mod stack;
mod sync;
mod tag;
mod toml;
mod update;
mod user;
mod user_group;
mod variable;
#[typeshare]
#[derive(Serialize, Deserialize, Debug, Clone, Resolver)]
#[resolver_target(State)]
#[resolver_args(User)]
#[serde(tag = "type", content = "params")]
enum ReadRequest {
#[to_string_resolver]
GetVersion(GetVersion),
#[to_string_resolver]
GetCoreInfo(GetCoreInfo),
ListSecrets(ListSecrets),
ListGitProvidersFromConfig(ListGitProvidersFromConfig),
ListDockerRegistriesFromConfig(ListDockerRegistriesFromConfig),
// ==== USER ====
GetUsername(GetUsername),
GetPermissionLevel(GetPermissionLevel),
FindUser(FindUser),
ListUsers(ListUsers),
ListApiKeys(ListApiKeys),
ListApiKeysForServiceUser(ListApiKeysForServiceUser),
ListPermissions(ListPermissions),
ListUserTargetPermissions(ListUserTargetPermissions),
// ==== USER GROUP ====
GetUserGroup(GetUserGroup),
ListUserGroups(ListUserGroups),
// ==== SEARCH ====
FindResources(FindResources),
// ==== PROCEDURE ====
GetProceduresSummary(GetProceduresSummary),
GetProcedure(GetProcedure),
GetProcedureActionState(GetProcedureActionState),
ListProcedures(ListProcedures),
ListFullProcedures(ListFullProcedures),
// ==== SERVER TEMPLATE ====
GetServerTemplate(GetServerTemplate),
GetServerTemplatesSummary(GetServerTemplatesSummary),
ListServerTemplates(ListServerTemplates),
ListFullServerTemplates(ListFullServerTemplates),
// ==== SERVER ====
GetServersSummary(GetServersSummary),
GetServer(GetServer),
GetServerState(GetServerState),
GetPeripheryVersion(GetPeripheryVersion),
GetServerActionState(GetServerActionState),
GetHistoricalServerStats(GetHistoricalServerStats),
ListServers(ListServers),
ListFullServers(ListFullServers),
InspectDockerContainer(InspectDockerContainer),
GetResourceMatchingContainer(GetResourceMatchingContainer),
GetContainerLog(GetContainerLog),
SearchContainerLog(SearchContainerLog),
InspectDockerNetwork(InspectDockerNetwork),
InspectDockerImage(InspectDockerImage),
ListDockerImageHistory(ListDockerImageHistory),
InspectDockerVolume(InspectDockerVolume),
#[to_string_resolver]
ListDockerContainers(ListDockerContainers),
#[to_string_resolver]
ListDockerNetworks(ListDockerNetworks),
#[to_string_resolver]
ListDockerImages(ListDockerImages),
#[to_string_resolver]
ListDockerVolumes(ListDockerVolumes),
#[to_string_resolver]
ListComposeProjects(ListComposeProjects),
// ==== DEPLOYMENT ====
GetDeploymentsSummary(GetDeploymentsSummary),
GetDeployment(GetDeployment),
GetDeploymentContainer(GetDeploymentContainer),
GetDeploymentActionState(GetDeploymentActionState),
GetDeploymentStats(GetDeploymentStats),
GetDeploymentLog(GetDeploymentLog),
SearchDeploymentLog(SearchDeploymentLog),
ListDeployments(ListDeployments),
ListFullDeployments(ListFullDeployments),
ListCommonDeploymentExtraArgs(ListCommonDeploymentExtraArgs),
// ==== BUILD ====
GetBuildsSummary(GetBuildsSummary),
GetBuild(GetBuild),
GetBuildActionState(GetBuildActionState),
GetBuildMonthlyStats(GetBuildMonthlyStats),
ListBuildVersions(ListBuildVersions),
GetBuildWebhookEnabled(GetBuildWebhookEnabled),
ListBuilds(ListBuilds),
ListFullBuilds(ListFullBuilds),
ListCommonBuildExtraArgs(ListCommonBuildExtraArgs),
// ==== REPO ====
GetReposSummary(GetReposSummary),
GetRepo(GetRepo),
GetRepoActionState(GetRepoActionState),
GetRepoWebhooksEnabled(GetRepoWebhooksEnabled),
ListRepos(ListRepos),
ListFullRepos(ListFullRepos),
// ==== SYNC ====
GetResourceSyncsSummary(GetResourceSyncsSummary),
GetResourceSync(GetResourceSync),
GetResourceSyncActionState(GetResourceSyncActionState),
GetSyncWebhooksEnabled(GetSyncWebhooksEnabled),
ListResourceSyncs(ListResourceSyncs),
ListFullResourceSyncs(ListFullResourceSyncs),
// ==== STACK ====
GetStacksSummary(GetStacksSummary),
GetStack(GetStack),
GetStackActionState(GetStackActionState),
GetStackWebhooksEnabled(GetStackWebhooksEnabled),
GetStackServiceLog(GetStackServiceLog),
SearchStackServiceLog(SearchStackServiceLog),
ListStacks(ListStacks),
ListFullStacks(ListFullStacks),
ListStackServices(ListStackServices),
ListCommonStackExtraArgs(ListCommonStackExtraArgs),
ListCommonStackBuildExtraArgs(ListCommonStackBuildExtraArgs),
// ==== BUILDER ====
GetBuildersSummary(GetBuildersSummary),
GetBuilder(GetBuilder),
ListBuilders(ListBuilders),
ListFullBuilders(ListFullBuilders),
// ==== ALERTER ====
GetAlertersSummary(GetAlertersSummary),
GetAlerter(GetAlerter),
ListAlerters(ListAlerters),
ListFullAlerters(ListFullAlerters),
// ==== TOML ====
ExportAllResourcesToToml(ExportAllResourcesToToml),
ExportResourcesToToml(ExportResourcesToToml),
// ==== TAG ====
GetTag(GetTag),
ListTags(ListTags),
// ==== UPDATE ====
GetUpdate(GetUpdate),
ListUpdates(ListUpdates),
// ==== ALERT ====
ListAlerts(ListAlerts),
GetAlert(GetAlert),
// ==== SERVER STATS ====
#[to_string_resolver]
GetSystemInformation(GetSystemInformation),
#[to_string_resolver]
GetSystemStats(GetSystemStats),
#[to_string_resolver]
ListSystemProcesses(ListSystemProcesses),
// ==== VARIABLE ====
GetVariable(GetVariable),
ListVariables(ListVariables),
// ==== PROVIDER ====
GetGitProviderAccount(GetGitProviderAccount),
ListGitProviderAccounts(ListGitProviderAccounts),
GetDockerRegistryAccount(GetDockerRegistryAccount),
ListDockerRegistryAccounts(ListDockerRegistryAccounts),
}
pub fn router() -> Router {
Router::new()
.route("/", post(handler))
.layer(middleware::from_fn(auth_request))
}
#[instrument(name = "ReadHandler", level = "debug", skip(user), fields(user_id = user.id))]
async fn handler(
Extension(user): Extension<User>,
Json(request): Json<ReadRequest>,
) -> serror::Result<(TypedHeader<ContentType>, String)> {
let timer = Instant::now();
let req_id = Uuid::new_v4();
debug!("/read request | user: {}", user.username);
let res =
State
.resolve_request(request, user)
.await
.map_err(|e| match e {
resolver_api::Error::Serialization(e) => {
anyhow!("{e:?}").context("response serialization error")
}
resolver_api::Error::Inner(e) => e,
});
if let Err(e) = &res {
debug!("/read request {req_id} error: {e:#}");
}
let elapsed = timer.elapsed();
debug!("/read request {req_id} | resolve time: {elapsed:?}");
Ok((TypedHeader(ContentType::json()), res?))
}
fn version() -> &'static String {
static VERSION: OnceLock<String> = OnceLock::new();
VERSION.get_or_init(|| {
serde_json::to_string(&GetVersionResponse {
version: env!("CARGO_PKG_VERSION").to_string(),
})
.context("failed to serialize GetVersionResponse")
.unwrap()
})
}
impl ResolveToString<GetVersion, User> for State {
async fn resolve_to_string(
&self,
GetVersion {}: GetVersion,
_: User,
) -> anyhow::Result<String> {
Ok(version().to_string())
}
}
fn core_info() -> &'static String {
static CORE_INFO: OnceLock<String> = OnceLock::new();
CORE_INFO.get_or_init(|| {
let config = core_config();
let info = GetCoreInfoResponse {
title: config.title.clone(),
monitoring_interval: config.monitoring_interval,
webhook_base_url: if config.webhook_base_url.is_empty() {
config.host.clone()
} else {
config.webhook_base_url.clone()
},
transparent_mode: config.transparent_mode,
ui_write_disabled: config.ui_write_disabled,
disable_confirm_dialog: config.disable_confirm_dialog,
disable_non_admin_create: config.disable_non_admin_create,
github_webhook_owners: config
.github_webhook_app
.installations
.iter()
.map(|i| i.namespace.to_string())
.collect(),
};
serde_json::to_string(&info)
.context("failed to serialize GetCoreInfoResponse")
.unwrap()
})
}
impl ResolveToString<GetCoreInfo, User> for State {
async fn resolve_to_string(
&self,
GetCoreInfo {}: GetCoreInfo,
_: User,
) -> anyhow::Result<String> {
Ok(core_info().to_string())
}
}
impl Resolve<ListSecrets, User> for State {
async fn resolve(
&self,
ListSecrets { target }: ListSecrets,
_: User,
) -> anyhow::Result<ListSecretsResponse> {
let mut secrets = core_config()
.secrets
.keys()
.cloned()
.collect::<HashSet<_>>();
if let Some(target) = target {
let server_id = match target {
ResourceTarget::Server(id) => Some(id),
ResourceTarget::Builder(id) => {
match resource::get::<Builder>(&id).await?.config {
BuilderConfig::Server(config) => Some(config.server_id),
BuilderConfig::Aws(config) => {
secrets.extend(config.secrets);
None
}
}
}
_ => {
return Err(anyhow!("target must be `Server` or `Builder`"))
}
};
if let Some(id) = server_id {
let server = resource::get::<Server>(&id).await?;
let more = periphery_client(&server)?
.request(periphery_client::api::ListSecrets {})
.await
.with_context(|| {
format!(
"failed to get secrets from server {}",
server.name
)
})?;
secrets.extend(more);
}
}
let mut secrets = secrets.into_iter().collect::<Vec<_>>();
secrets.sort();
Ok(secrets)
}
}
impl Resolve<ListGitProvidersFromConfig, User> for State {
async fn resolve(
&self,
ListGitProvidersFromConfig { target }: ListGitProvidersFromConfig,
user: User,
) -> anyhow::Result<ListGitProvidersFromConfigResponse> {
let mut providers = core_config().git_providers.clone();
if let Some(target) = target {
match target {
ResourceTarget::Server(id) => {
merge_git_providers_for_server(&mut providers, &id).await?;
}
ResourceTarget::Builder(id) => {
match resource::get::<Builder>(&id).await?.config {
BuilderConfig::Server(config) => {
merge_git_providers_for_server(
&mut providers,
&config.server_id,
)
.await?;
}
BuilderConfig::Aws(config) => {
merge_git_providers(
&mut providers,
config.git_providers,
);
}
}
}
_ => {
return Err(anyhow!("target must be `Server` or `Builder`"))
}
}
}
let (builds, repos, syncs) = tokio::try_join!(
resource::list_full_for_user::<Build>(
Default::default(),
&user
),
resource::list_full_for_user::<Repo>(Default::default(), &user),
resource::list_full_for_user::<ResourceSync>(
Default::default(),
&user
),
)?;
for build in builds {
if !providers
.iter()
.any(|provider| provider.domain == build.config.git_provider)
{
providers.push(GitProvider {
domain: build.config.git_provider,
https: build.config.git_https,
accounts: Default::default(),
});
}
}
for repo in repos {
if !providers
.iter()
.any(|provider| provider.domain == repo.config.git_provider)
{
providers.push(GitProvider {
domain: repo.config.git_provider,
https: repo.config.git_https,
accounts: Default::default(),
});
}
}
for sync in syncs {
if !providers
.iter()
.any(|provider| provider.domain == sync.config.git_provider)
{
providers.push(GitProvider {
domain: sync.config.git_provider,
https: sync.config.git_https,
accounts: Default::default(),
});
}
}
providers.sort();
Ok(providers)
}
}
impl Resolve<ListDockerRegistriesFromConfig, User> for State {
async fn resolve(
&self,
ListDockerRegistriesFromConfig { target }: ListDockerRegistriesFromConfig,
_: User,
) -> anyhow::Result<ListDockerRegistriesFromConfigResponse> {
let mut registries = core_config().docker_registries.clone();
if let Some(target) = target {
match target {
ResourceTarget::Server(id) => {
merge_docker_registries_for_server(&mut registries, &id)
.await?;
}
ResourceTarget::Builder(id) => {
match resource::get::<Builder>(&id).await?.config {
BuilderConfig::Server(config) => {
merge_docker_registries_for_server(
&mut registries,
&config.server_id,
)
.await?;
}
BuilderConfig::Aws(config) => {
merge_docker_registries(
&mut registries,
config.docker_registries,
);
}
}
}
_ => {
return Err(anyhow!("target must be `Server` or `Builder`"))
}
}
}
registries.sort();
Ok(registries)
}
}
async fn merge_git_providers_for_server(
providers: &mut Vec<GitProvider>,
server_id: &str,
) -> anyhow::Result<()> {
let server = resource::get::<Server>(server_id).await?;
let more = periphery_client(&server)?
.request(periphery_client::api::ListGitProviders {})
.await
.with_context(|| {
format!(
"failed to get git providers from server {}",
server.name
)
})?;
merge_git_providers(providers, more);
Ok(())
}
fn merge_git_providers(
providers: &mut Vec<GitProvider>,
more: Vec<GitProvider>,
) {
for incoming_provider in more {
if let Some(provider) = providers
.iter_mut()
.find(|provider| provider.domain == incoming_provider.domain)
{
for account in incoming_provider.accounts {
if !provider.accounts.contains(&account) {
provider.accounts.push(account);
}
}
} else {
providers.push(incoming_provider);
}
}
}
async fn merge_docker_registries_for_server(
registries: &mut Vec<DockerRegistry>,
server_id: &str,
) -> anyhow::Result<()> {
let server = resource::get::<Server>(server_id).await?;
let more = periphery_client(&server)?
.request(periphery_client::api::ListDockerRegistries {})
.await
.with_context(|| {
format!(
"failed to get docker registries from server {}",
server.name
)
})?;
merge_docker_registries(registries, more);
Ok(())
}
fn merge_docker_registries(
registries: &mut Vec<DockerRegistry>,
more: Vec<DockerRegistry>,
) {
for incoming_registry in more {
if let Some(registry) = registries
.iter_mut()
.find(|registry| registry.domain == incoming_registry.domain)
{
for account in incoming_registry.accounts {
if !registry.accounts.contains(&account) {
registry.accounts.push(account);
}
}
} else {
registries.push(incoming_registry);
}
}
}