github-starred/komodo
2
0
Fork 1
mirror of https://github.com/moghtech/komodo.git synced 2026-04-28 03:38:55 -05:00
Code Issues 3.1k Packages Projects Releases 103 Wiki Activity

implement basic deployment api

Browse Source
This commit is contained in:
mbecker20
2023-06-24 07:21:15 +00:00
parent 402259ef11
commit b33501cce6
24 changed files with 1343 additions and 142 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.vscode/tasks.json vendored
View File

@@ -85,7 +85,7 @@
},
{
"type": "shell",
"command": "typeshare ./lib/types --lang=typescript --output-file=./lib/src/types.ts && typeshare ./lib/core_api --lang=typescript --output-file=./frontend/src/core_api.ts",
"command": "typeshare ./lib/types --lang=typescript --output-file=./lib/ts_client/src/types.ts",
"label": "generate typescript types",
"problemMatcher": []
}

11
Cargo.lock generated
View File

@@ -1830,7 +1830,6 @@ dependencies = [
"jwt",
"log",
"merge_config_files",
"monitor_helpers",
"monitor_types",
"mungos",
"parse_csl",
@@ -1851,15 +1850,6 @@ dependencies = [
"uuid",
]
[[package]]
name = "monitor_helpers"
version = "1.0.0"
dependencies = [
"async_timing_util",
"monitor_types",
"serde",
]
[[package]]
name = "monitor_macros"
version = "1.0.0"
@@ -1883,7 +1873,6 @@ dependencies = [
"envy",
"log",
"merge_config_files",
"monitor_helpers",
"monitor_types",
"parse_csl",
"resolver_api",

1
Cargo.toml
View File

@@ -11,7 +11,6 @@ license = "GPL-3.0-or-later"
[workspace.dependencies]
# local
monitor_macros = { path = "lib/macros" }
monitor_helpers = { path = "lib/helpers" }
monitor_types = { path = "lib/types" }
monitor_client = { path = "lib/rs_client" }
periphery_client = { path = "lib/periphery_client" }

1
core/Cargo.toml
View File

@@ -10,7 +10,6 @@ license.workspace = true
[dependencies]
# local
monitor_types.workspace = true
monitor_helpers.workspace = true
periphery_client.workspace = true
# external
tokio.workspace = true

3
core/src/auth/github/mod.rs
View File

@@ -1,7 +1,6 @@
use anyhow::{anyhow, Context};
use axum::{extract::Query, http::StatusCode, response::Redirect, routing::get, Router};
use monitor_helpers::monitor_timestamp;
use monitor_types::entities::user::User;
use monitor_types::{entities::user::User, monitor_timestamp};
use mungos::mongodb::bson::doc;
use serde::Deserialize;

55
core/src/helpers.rs
View File

@@ -54,6 +54,25 @@ impl State {
Ok((server, status))
}
pub async fn get_server_status(
&self,
server_id: &str,
) -> anyhow::Result<ServerStatus> {
let server = self.get_server(server_id).await?;
if !server.config.enabled {
return Ok(ServerStatus::Disabled);
}
let status = match self
.periphery_client(&server)
.request(requests::GetHealth {})
.await
{
Ok(_) => ServerStatus::Ok,
Err(_) => ServerStatus::NotOk,
};
Ok(status)
}
pub async fn get_server_check_permissions(
&self,
server_id: &str,
@@ -285,22 +304,22 @@ impl<T: Clone + Default> Cache<T> {
self.cache.read().await.get(key).cloned()
}
pub async fn get_or_default(&self, key: String) -> T {
let mut cache = self.cache.write().await;
cache.entry(key).or_default().clone()
}
// pub async fn get_or_default(&self, key: String) -> T {
// let mut cache = self.cache.write().await;
// cache.entry(key).or_default().clone()
// }
pub async fn get_list(&self, filter: Option<impl Fn(&String, &T) -> bool>) -> Vec<T> {
let cache = self.cache.read().await;
match filter {
Some(filter) => cache
.iter()
.filter(|(k, v)| filter(k, v))
.map(|(_, e)| e.clone())
.collect(),
None => cache.iter().map(|(_, e)| e.clone()).collect(),
}
}
// pub async fn get_list(&self, filter: Option<impl Fn(&String, &T) -> bool>) -> Vec<T> {
// let cache = self.cache.read().await;
// match filter {
// Some(filter) => cache
// .iter()
// .filter(|(k, v)| filter(k, v))
// .map(|(_, e)| e.clone())
// .collect(),
// None => cache.iter().map(|(_, e)| e.clone()).collect(),
// }
// }
pub async fn insert(&self, key: String, val: T) {
self.cache.write().await.insert(key, val);
@@ -311,9 +330,9 @@ impl<T: Clone + Default> Cache<T> {
handler(cache.entry(key).or_default());
}
pub async fn clear(&self) {
self.cache.write().await.clear();
}
// pub async fn clear(&self) {
// self.cache.write().await.clear();
// }
pub async fn remove(&self, key: &str) {
self.cache.write().await.remove(key);

122
core/src/requests/api/build.rs
View File

@@ -3,7 +3,6 @@ use std::time::Duration;
use anyhow::{anyhow, Context};
use async_trait::async_trait;
use futures::future::join_all;
use monitor_helpers::monitor_timestamp;
use monitor_types::{
all_logs_success,
entities::{
@@ -13,8 +12,11 @@ use monitor_types::{
update::{Log, Update, UpdateStatus, UpdateTarget},
Operation, PermissionLevel,
},
monitor_timestamp,
permissioned::Permissioned,
requests::api::{CreateBuild, DeleteBuild, GetBuild, ListBuilds, RunBuild, UpdateBuild},
requests::api::{
CreateBuild, DeleteBuild, Deploy, GetBuild, ListBuilds, RunBuild, UpdateBuild,
},
};
use mungos::mongodb::bson::{doc, to_bson};
use periphery_client::{
@@ -72,10 +74,27 @@ impl Resolve<CreateBuild, RequestUser> for State {
CreateBuild { name, config }: CreateBuild,
user: RequestUser,
) -> anyhow::Result<Build> {
if let Some(BuildBuilderConfig::Server { server_id }) = &config.builder {
self.get_server_check_permissions(server_id, &user, PermissionLevel::Update)
.await
.context("cannot create build on this server")?;
if let Some(builder) = &config.builder {
match builder {
BuildBuilderConfig::Server { server_id } => {
self.get_server_check_permissions(
server_id,
&user,
PermissionLevel::Update,
)
.await
.context("cannot create build on this server. user must have update permissions on the server.")?;
}
BuildBuilderConfig::Builder { builder_id } => {
self.get_builder_check_permissions(
builder_id,
&user,
PermissionLevel::Read,
)
.await
.context("cannot create build using this builder. user must have at least read permissions on the builder.")?;
}
}
}
let start_ts = monitor_timestamp();
let build = Build {
@@ -190,6 +209,29 @@ impl Resolve<UpdateBuild, RequestUser> for State {
let inner = || async move {
let start_ts = monitor_timestamp();
if let Some(builder) = &config.builder {
match builder {
BuildBuilderConfig::Server { server_id } => {
self.get_server_check_permissions(
server_id,
&user,
PermissionLevel::Update,
)
.await
.context("cannot create build on this server. user must have update permissions on the server.")?;
}
BuildBuilderConfig::Builder { builder_id } => {
self.get_builder_check_permissions(
builder_id,
&user,
PermissionLevel::Read,
)
.await
.context("cannot create build using this builder. user must have at least read permissions on the builder.")?;
}
}
}
if let Some(build_args) = &mut config.build_args {
build_args.retain(|v| {
!empty_or_only_spaces(&v.variable) && !empty_or_only_spaces(&v.value)
@@ -345,7 +387,7 @@ impl Resolve<RunBuild, RequestUser> for State {
// CLEANUP AND FINALIZE UPDATE
self.cleanup_builder_instance(cleanup_data, &mut update)
self.cleanup_builder_instance(periphery, cleanup_data, &mut update)
.await;
self.handle_post_build_redeploy(&build.id, &mut update)
@@ -492,9 +534,18 @@ impl State {
Err(anyhow!("{:#?}", res.err().unwrap()))
}
async fn cleanup_builder_instance(&self, cleanup_data: BuildCleanupData, update: &mut Update) {
async fn cleanup_builder_instance(
&self,
periphery: PeripheryClient,
cleanup_data: BuildCleanupData,
update: &mut Update,
) {
match cleanup_data {
BuildCleanupData::Server { repo_name } => {}
BuildCleanupData::Server { repo_name } => {
let _ = periphery
.request(requests::DeleteRepo { name: repo_name })
.await;
}
BuildCleanupData::Aws {
instance_id,
region,
@@ -526,30 +577,29 @@ impl State {
.await;
if let Ok(deployments) = redeploy_deployments {
let futures = deployments.into_iter().map(|d| async move {
let futures = deployments.into_iter().map(|deployment| async move {
let request_user: RequestUser = InnerRequestUser {
id: "auto redeploy".to_string(),
is_admin: true,
..Default::default()
}
.into();
let state = self.get_deployment_state(&d).await.unwrap_or_default();
let state = self
.get_deployment_state(&deployment)
.await
.unwrap_or_default();
if state == DockerContainerState::Running {
// Some((
// d.id.clone(),
// self.deploy_container(
// &d.id,
// &RequestUser {
// id: "auto redeploy".to_string(),
// is_admin: true,
// ..Default::default()
// },
// None,
// None,
// )
// .await,
// ))
Some(())
let res = self
.resolve(
Deploy {
deployment_id: deployment.id.clone(),
stop_signal: None,
stop_time: None,
},
request_user,
)
.await;
Some((deployment.id.clone(), res))
} else {
None
}
@@ -560,16 +610,16 @@ impl State {
let mut redeploys = Vec::<String>::new();
let mut redeploy_failures = Vec::<String>::new();
// for res in redeploy_results {
// if res.is_none() {
// continue;
// }
// let (id, res) = res.unwrap();
// match res {
// Ok(_) => redeploys.push(id),
// Err(e) => redeploy_failures.push(format!("{id}: {e:#?}")),
// }
// }
for res in redeploy_results {
if res.is_none() {
continue;
}
let (id, res) = res.unwrap();
match res {
Ok(_) => redeploys.push(id),
Err(e) => redeploy_failures.push(format!("{id}: {e:#?}")),
}
}
if !redeploys.is_empty() {
update.logs.push(Log::simple(

407
core/src/requests/api/deployment.rs
View File

@@ -1,17 +1,20 @@
use anyhow::{anyhow, Context};
use async_trait::async_trait;
use monitor_helpers::{monitor_timestamp, to_monitor_name};
use monitor_types::{
all_logs_success,
entities::{
deployment::{Deployment, DockerContainerState},
deployment::{Deployment, DeploymentImage, DockerContainerState},
server::ServerStatus,
update::{Log, Update, UpdateStatus, UpdateTarget},
Operation, PermissionLevel,
Operation, PermissionLevel, Version,
},
get_image_name, monitor_timestamp,
permissioned::Permissioned,
requests::api::{
CreateDeployment, DeleteDeployment, GetDeployment, ListDeployments, RenameDeployment,
UpdateDeployment,
}, all_logs_success,
CreateDeployment, DeleteDeployment, Deploy, GetDeployment, ListDeployments,
RemoveContainer, RenameDeployment, StartContainer, StopContainer, UpdateDeployment,
},
to_monitor_name,
};
use mungos::mongodb::bson::{doc, to_bson};
use periphery_client::requests;
@@ -70,12 +73,12 @@ impl Resolve<CreateDeployment, RequestUser> for State {
if let Some(server_id) = &config.server_id {
self.get_server_check_permissions(server_id, &user, PermissionLevel::Update)
.await
.context("cannot create deployment on this server")?;
.context("cannot create deployment on this server. user must have update permissions on the server to perform this action.")?;
}
if let Some(build_id) = &config.build_id {
if let Some(DeploymentImage::Build { build_id, .. }) = &config.image {
self.get_build_check_permissions(build_id, &user, PermissionLevel::Read)
.await
.context("cannot create deployment with this build attached")?;
.context("cannot create deployment with this build attached. user must have at least read permissions on the build to perform this action.")?;
}
let start_ts = monitor_timestamp();
let deployment = Deployment {
@@ -237,6 +240,17 @@ impl Resolve<UpdateDeployment, RequestUser> for State {
let inner = || async move {
let start_ts = monitor_timestamp();
if let Some(server_id) = &config.server_id {
self.get_server_check_permissions(server_id, &user, PermissionLevel::Update)
.await
.context("cannot create deployment on this server. user must have update permissions on the server to perform this action.")?;
}
if let Some(DeploymentImage::Build { build_id, .. }) = &config.image {
self.get_build_check_permissions(build_id, &user, PermissionLevel::Read)
.await
.context("cannot create deployment with this build attached. user must have at least read permissions on the build to perform this action.")?;
}
if let Some(volumes) = &mut config.volumes {
volumes.retain(|v| {
!empty_or_only_spaces(&v.local) && !empty_or_only_spaces(&v.container)
@@ -404,3 +418,378 @@ impl Resolve<RenameDeployment, RequestUser> for State {
res
}
}
#[async_trait]
impl Resolve<Deploy, RequestUser> for State {
async fn resolve(
&self,
Deploy {
deployment_id,
stop_signal,
stop_time,
}: Deploy,
user: RequestUser,
) -> anyhow::Result<Update> {
if self.action_states.deployment.busy(&deployment_id).await {
return Err(anyhow!("deployment busy"));
}
let mut deployment = self
.get_deployment_check_permissions(&deployment_id, &user, PermissionLevel::Execute)
.await?;
if deployment.config.server_id.is_empty() {
return Err(anyhow!("deployment has no server configured"));
}
let (server, status) = self
.get_server_with_status(&deployment.config.server_id)
.await?;
if status != ServerStatus::Ok {
return Err(anyhow!(
"cannot send action when server is unreachable or disabled"
));
}
let periphery = self.periphery_client(&server);
let inner = || async move {
let start_ts = monitor_timestamp();
let version = match deployment.config.image {
DeploymentImage::Build { build_id, version } => {
let build = self.get_build(&build_id).await?;
let image_name = get_image_name(&build);
let version = if version.is_none() {
build.config.version
} else {
version
};
deployment.config.image = DeploymentImage::Image {
image: format!("{image_name}:{}", version.to_string()),
};
if deployment.config.docker_account.is_empty() {
deployment.config.docker_account = build.config.docker_account;
}
version
}
DeploymentImage::Image { .. } => Version::default(),
};
let mut update = Update {
target: UpdateTarget::Deployment(deployment.id.clone()),
operation: Operation::DeployContainer,
start_ts,
status: UpdateStatus::InProgress,
success: true,
operator: user.id.clone(),
version,
..Default::default()
};
update.id = self.add_update(update.clone()).await?;
let log = match periphery
.request(requests::Deploy {
deployment,
stop_signal,
stop_time,
})
.await
{
Ok(log) => log,
Err(e) => Log::error("deploy container", format!("{e:#?}")),
};
update.logs.push(log);
update.finalize();
self.update_cache_for_server(&server).await;
self.update_update(update.clone()).await?;
Ok(update)
};
self.action_states
.deployment
.update_entry(deployment_id.to_string(), |entry| {
entry.deploying = true;
})
.await;
let res = inner().await;
self.action_states
.deployment
.update_entry(deployment_id, |entry| {
entry.deploying = false;
})
.await;
res
}
}
#[async_trait]
impl Resolve<StartContainer, RequestUser> for State {
async fn resolve(
&self,
StartContainer { deployment_id }: StartContainer,
user: RequestUser,
) -> anyhow::Result<Update> {
if self.action_states.deployment.busy(&deployment_id).await {
return Err(anyhow!("deployment busy"));
}
let deployment = self
.get_deployment_check_permissions(&deployment_id, &user, PermissionLevel::Execute)
.await?;
if deployment.config.server_id.is_empty() {
return Err(anyhow!("deployment has no server configured"));
}
let (server, status) = self
.get_server_with_status(&deployment.config.server_id)
.await?;
if status != ServerStatus::Ok {
return Err(anyhow!(
"cannot send action when server is unreachable or disabled"
));
}
let periphery = self.periphery_client(&server);
let inner = || async move {
let start_ts = monitor_timestamp();
let mut update = Update {
target: UpdateTarget::Deployment(deployment.id.clone()),
operation: Operation::StartContainer,
start_ts,
status: UpdateStatus::InProgress,
success: true,
operator: user.id.clone(),
..Default::default()
};
update.id = self.add_update(update.clone()).await?;
let log = match periphery
.request(requests::StartContainer {
name: deployment.name.clone(),
})
.await
{
Ok(log) => log,
Err(e) => Log::error("start container", format!("{e:#?}")),
};
update.logs.push(log);
update.finalize();
self.update_cache_for_server(&server).await;
self.update_update(update.clone()).await?;
Ok(update)
};
self.action_states
.deployment
.update_entry(deployment_id.to_string(), |entry| {
entry.starting = true;
})
.await;
let res = inner().await;
self.action_states
.deployment
.update_entry(deployment_id, |entry| {
entry.starting = false;
})
.await;
res
}
}
#[async_trait]
impl Resolve<StopContainer, RequestUser> for State {
async fn resolve(
&self,
StopContainer {
deployment_id,
signal,
time,
}: StopContainer,
user: RequestUser,
) -> anyhow::Result<Update> {
if self.action_states.deployment.busy(&deployment_id).await {
return Err(anyhow!("deployment busy"));
}
let deployment = self
.get_deployment_check_permissions(&deployment_id, &user, PermissionLevel::Execute)
.await?;
if deployment.config.server_id.is_empty() {
return Err(anyhow!("deployment has no server configured"));
}
let (server, status) = self
.get_server_with_status(&deployment.config.server_id)
.await?;
if status != ServerStatus::Ok {
return Err(anyhow!(
"cannot send action when server is unreachable or disabled"
));
}
let periphery = self.periphery_client(&server);
let inner = || async move {
let start_ts = monitor_timestamp();
let mut update = Update {
target: UpdateTarget::Deployment(deployment.id.clone()),
operation: Operation::StopContainer,
start_ts,
status: UpdateStatus::InProgress,
success: true,
operator: user.id.clone(),
..Default::default()
};
update.id = self.add_update(update.clone()).await?;
let log = match periphery
.request(requests::StopContainer {
name: deployment.name.clone(),
signal: signal
.unwrap_or(deployment.config.termination_signal)
.into(),
time: time.unwrap_or(deployment.config.termination_timeout).into(),
})
.await
{
Ok(log) => log,
Err(e) => Log::error("stop container", format!("{e:#?}")),
};
update.logs.push(log);
update.finalize();
self.update_cache_for_server(&server).await;
self.update_update(update.clone()).await?;
Ok(update)
};
self.action_states
.deployment
.update_entry(deployment_id.to_string(), |entry| {
entry.stopping = true;
})
.await;
let res = inner().await;
self.action_states
.deployment
.update_entry(deployment_id, |entry| {
entry.stopping = false;
})
.await;
res
}
}
#[async_trait]
impl Resolve<RemoveContainer, RequestUser> for State {
async fn resolve(
&self,
RemoveContainer {
deployment_id,
signal,
time,
}: RemoveContainer,
user: RequestUser,
) -> anyhow::Result<Update> {
if self.action_states.deployment.busy(&deployment_id).await {
return Err(anyhow!("deployment busy"));
}
let deployment = self
.get_deployment_check_permissions(&deployment_id, &user, PermissionLevel::Execute)
.await?;
if deployment.config.server_id.is_empty() {
return Err(anyhow!("deployment has no server configured"));
}
let (server, status) = self
.get_server_with_status(&deployment.config.server_id)
.await?;
if status != ServerStatus::Ok {
return Err(anyhow!(
"cannot send action when server is unreachable or disabled"
));
}
let periphery = self.periphery_client(&server);
let inner = || async move {
let start_ts = monitor_timestamp();
let mut update = Update {
target: UpdateTarget::Deployment(deployment.id.clone()),
operation: Operation::RemoveContainer,
start_ts,
status: UpdateStatus::InProgress,
success: true,
operator: user.id.clone(),
..Default::default()
};
update.id = self.add_update(update.clone()).await?;
let log = match periphery
.request(requests::RemoveContainer {
name: deployment.name.clone(),
signal: signal
.unwrap_or(deployment.config.termination_signal)
.into(),
time: time.unwrap_or(deployment.config.termination_timeout).into(),
})
.await
{
Ok(log) => log,
Err(e) => Log::error("stop container", format!("{e:#?}")),
};
update.logs.push(log);
update.finalize();
self.update_cache_for_server(&server).await;
self.update_update(update.clone()).await?;
Ok(update)
};
self.action_states
.deployment
.update_entry(deployment_id.to_string(), |entry| {
entry.removing = true;
})
.await;
let res = inner().await;
self.action_states
.deployment
.update_entry(deployment_id, |entry| {
entry.removing = false;
})
.await;
res
}
}

12
core/src/requests/api/mod.rs
View File

@@ -74,6 +74,18 @@ pub enum ApiRequest {
DeleteDeployment(DeleteDeployment),
UpdateDeployment(UpdateDeployment),
RenameDeployment(RenameDeployment),
//
// ==== BUILD ====
//
GetBuild(GetBuild),
ListBuilds(ListBuilds),
// CRUD
CreateBuild(CreateBuild),
DeleteBuild(DeleteBuild),
UpdateBuild(UpdateBuild),
// ACTIONS
RunBuild(RunBuild),
}
pub fn router() -> Router {

2
core/src/requests/api/secret.rs
View File

@@ -1,7 +1,7 @@
use anyhow::{anyhow, Context};
use async_trait::async_trait;
use monitor_helpers::monitor_timestamp;
use monitor_types::{
monitor_timestamp,
entities::user::ApiSecret,
requests::api::{CreateLoginSecret, CreateLoginSecretResponse, DeleteLoginSecret},
};

2
core/src/requests/api/server.rs
View File

@@ -1,6 +1,5 @@
use anyhow::{anyhow, Context};
use async_trait::async_trait;
use monitor_helpers::monitor_timestamp;
use monitor_types::{
entities::{
deployment::BasicContainerInfo,
@@ -11,6 +10,7 @@ use monitor_types::{
update::{Log, Update, UpdateStatus, UpdateTarget},
Operation, PermissionLevel,
},
monitor_timestamp,
permissioned::Permissioned,
requests::api::{
CreateServer, DeleteServer, GetAllSystemStats, GetBasicSystemStats, GetCpuUsage,

13
lib/helpers/Cargo.toml
View File

@@ -1,13 +0,0 @@
[package]
name = "monitor_helpers"
version.workspace = true
edition.workspace = true
authors.workspace = true
license.workspace = true
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
monitor_types.workspace = true
serde.workspace = true
async_timing_util.workspace = true

9
lib/helpers/src/lib.rs
View File

@@ -1,9 +0,0 @@
use async_timing_util::unix_timestamp_ms;
pub fn to_monitor_name(name: &str) -> String {
name.to_lowercase().replace(' ', "_")
}
pub fn monitor_timestamp() -> i64 {
unix_timestamp_ms() as i64
}

650
lib/ts_client/src/types.ts Normal file
View File

@@ -0,0 +1,650 @@
/*
Generated by typeshare 1.6.0
*/
export enum PermissionLevel {
None = "none",
Read = "read",
Execute = "execute",
Update = "update",
}
export type PermissionsMap = Record<string, PermissionLevel>;
export type I64 = number;
export type MongoDocument = any;
export type BuildBuilderConfig =
| { type: "Server", params: {
server_id: string;
}}
| { type: "Builder", params: {
builder_id: string;
}};
export interface Version {
major: number;
minor: number;
patch: number;
}
export interface SystemCommand {
path?: string;
command?: string;
}
export interface EnvironmentVar {
variable: string;
value: string;
}
export interface BuildConfig {
builder: BuildBuilderConfig;
skip_secret_interp?: boolean;
version?: Version;
repo?: string;
branch: string;
github_account?: string;
docker_account?: string;
docker_organization?: string;
pre_build?: SystemCommand;
build_path: string;
dockerfile_path: string;
build_args?: EnvironmentVar[];
extra_args?: string[];
use_buildx?: boolean;
tags?: string[];
}
export interface Build {
_id?: string;
name: string;
description?: string;
permissions?: PermissionsMap;
created_at?: I64;
updated_at?: I64;
last_built_at?: I64;
config: BuildConfig;
}
export interface BuildActionState {
building: boolean;
updating: boolean;
}
export type BuilderConfig =
| { type: "AwsBuilder", params: AwsBuilder };
export interface Builder {
_id?: string;
name: string;
description?: string;
permissions?: PermissionsMap;
created_at?: I64;
updated_at?: I64;
config: BuilderConfig;
}
export interface AwsBuilder {
region: string;
instance_type: string;
volume_gb: number;
ami_id: string;
subnet_id: string;
security_group_ids: string[];
key_pair_name: string;
assign_public_ip: boolean;
}
export enum TerminationSignal {
SigHup = "SIGHUP",
SigInt = "SIGINT",
SigQuit = "SIGQUIT",
SigTerm = "SIGTERM",
}
export interface TerminationSignalLabel {
signal: TerminationSignal;
label: string;
}
export interface Conversion {
local: string;
container: string;
}
export enum RestartMode {
NoRestart = "no",
OnFailure = "on-failure",
Always = "always",
UnlessStopped = "unless-stopped",
}
export interface DeploymentConfig {
server_id?: string;
build_id?: string;
image?: string;
skip_secret_interp?: boolean;
redeploy_on_build?: boolean;
term_signal_labels: TerminationSignalLabel[];
termination_signal?: TerminationSignal;
termination_timeout: number;
ports?: Conversion[];
volumes?: Conversion[];
environment?: EnvironmentVar[];
network: string;
restart?: RestartMode;
post_image?: string;
container_user?: string;
extra_args?: string[];
docker_account?: string;
tags?: string[];
}
export interface Deployment {
_id?: string;
name: string;
description?: string;
permissions?: PermissionsMap;
created_at?: I64;
updated_at?: I64;
config: DeploymentConfig;
}
export enum DockerContainerState {
Unknown = "Unknown",
NotDeployed = "NotDeployed",
Created = "Created",
Restarting = "Restarting",
Running = "Running",
Removing = "Removing",
Paused = "Paused",
Exited = "Exited",
Dead = "Dead",
}
export interface BasicContainerInfo {
name: string;
id: string;
image: string;
state: DockerContainerState;
status?: string;
}
export interface DockerContainerStats {
name: string;
cpu_perc: string;
mem_perc: string;
mem_usage: string;
net_io: string;
block_io: string;
pids: string;
}
export interface DeploymentActionState {
deploying: boolean;
stopping: boolean;
starting: boolean;
removing: boolean;
pulling: boolean;
recloning: boolean;
updating: boolean;
renaming: boolean;
}
export interface CloneArgs {
name: string;
repo?: string;
branch?: string;
on_clone?: SystemCommand;
on_pull?: SystemCommand;
github_account?: string;
}
export interface RepoConfig {
repo: string;
branch: string;
github_account: string;
on_clone: SystemCommand;
on_pull: SystemCommand;
}
export interface Repo {
_id?: string;
name: string;
description?: string;
permissions?: PermissionsMap;
created_at?: string;
updated_at?: string;
tags?: string[];
config: RepoConfig;
}
export interface ImageSummary {
/** ID is the content-addressable ID of an image. This identifier is a content-addressable digest calculated from the image's configuration (which includes the digests of layers used by the image). Note that this digest differs from the `RepoDigests` below, which holds digests of image manifests that reference the image. */
Id: string;
/** ID of the parent image. Depending on how the image was created, this field may be empty and is only set for images that were built/created locally. This field is empty if the image was pulled from an image registry. */
ParentId: string;
/** List of image names/tags in the local image cache that reference this image. Multiple image tags can refer to the same image, and this list may be empty if no tags reference the image, in which case the image is \"untagged\", in which case it can still be referenced by its ID. */
RepoTags: string[];
/** List of content-addressable digests of locally available image manifests that the image is referenced from. Multiple manifests can refer to the same image. These digests are usually only available if the image was either pulled from a registry, or if the image was pushed to a registry, which is when the manifest is generated and its digest calculated. */
RepoDigests: string[];
/** Date and time at which the image was created as a Unix timestamp (number of seconds sinds EPOCH). */
Created: I64;
/** Total size of the image including all layers it is composed of. */
Size: I64;
/** Total size of image layers that are shared between this image and other images. This size is not calculated by default. `-1` indicates that the value has not been set / calculated. */
SharedSize: I64;
/** Total size of the image including all layers it is composed of. In versions of Docker before v1.10, this field was calculated from the image itself and all of its parent images. Docker v1.10 and up store images self-contained, and no longer use a parent-chain, making this field an equivalent of the Size field. This field is kept for backward compatibility, but may be removed in a future version of the API. */
VirtualSize: I64;
/** User-defined key/value metadata. */
Labels: Record<string, string>;
/** Number of containers using this image. Includes both stopped and running containers. This size is not calculated by default, and depends on which API endpoint is used. `-1` indicates that the value has not been set / calculated. */
Containers: I64;
}
export interface IpamConfig {
Subnet?: string;
IPRange?: string;
Gateway?: string;
AuxiliaryAddresses?: Record<string, string>;
}
export interface Ipam {
/** Name of the IPAM driver to use. */
Driver?: string;
/** List of IPAM configuration options, specified as a map: ``` {\"Subnet\": <CIDR>, \"IPRange\": <CIDR>, \"Gateway\": <IP address>, \"AuxAddress\": <device_name:IP address>} ``` */
Config?: IpamConfig[];
/** Driver-specific options, specified as a map. */
Options?: Record<string, string>;
}
export interface NetworkContainer {
Name?: string;
EndpointID?: string;
MacAddress?: string;
IPv4Address?: string;
IPv6Address?: string;
}
export interface DockerNetwork {
Name?: string;
Id?: string;
Created?: string;
Scope?: string;
Driver?: string;
EnableIPv6?: boolean;
IPAM?: Ipam;
Internal?: boolean;
Attachable?: boolean;
Ingress?: boolean;
Containers?: Record<string, NetworkContainer>;
Options?: Record<string, string>;
Labels?: Record<string, string>;
}
export interface ServerConfig {
address: string;
enabled: boolean;
auto_prune: boolean;
region?: string;
cpu_warning: number;
cpu_critical: number;
mem_warning: number;
mem_critical: number;
disk_warning: number;
disk_critical: number;
to_notify?: string[];
tags?: string[];
}
export interface Server {
_id?: string;
name: string;
description?: string;
permissions?: PermissionsMap;
created_at?: I64;
updated_at?: I64;
config: ServerConfig;
}
export interface ServerActionState {
pruning_networks: boolean;
pruning_containers: boolean;
pruning_images: boolean;
}
export interface SystemInformation {
name?: string;
os?: string;
kernel?: string;
core_count?: number;
host_name?: string;
cpu_brand: string;
}
export interface BasicSystemStats {
system_load: number;
cpu_perc: number;
cpu_freq_mhz: number;
mem_used_gb: number;
mem_total_gb: number;
disk_used_gb: number;
disk_total_gb: number;
}
export interface SingleCpuUsage {
name: string;
usage: number;
}
export interface CpuUsage {
cpu_perc: number;
cpu_freq_mhz: number;
cpus?: SingleCpuUsage[];
}
export interface SingleDiskUsage {
mount: string;
used_gb: number;
total_gb: number;
}
export interface DiskUsage {
used_gb: number;
total_gb: number;
read_kb: number;
write_kb: number;
disks?: SingleDiskUsage[];
}
export interface SystemNetwork {
name: string;
recieved_kb: number;
transmitted_kb: number;
}
export interface NetworkUsage {
recieved_kb: number;
transmitted_kb: number;
networks: SystemNetwork[];
}
export interface SystemProcess {
pid: number;
name: string;
exe?: string;
cmd: string[];
start_time?: number;
cpu_perc: number;
mem_mb: number;
disk_read_kb: number;
disk_write_kb: number;
}
export interface SystemComponent {
label: string;
temp: number;
max: number;
critical?: number;
}
export enum Timelength {
OneSecond = "1-sec",
FiveSeconds = "5-sec",
TenSeconds = "10-sec",
FifteenSeconds = "15-sec",
ThirtySeconds = "30-sec",
OneMinute = "1-min",
TwoMinutes = "2-min",
FiveMinutes = "5-min",
TenMinutes = "10-min",
FifteenMinutes = "15-min",
ThirtyMinutes = "30-min",
OneHour = "1-hr",
TwoHours = "2-hr",
SixHours = "6-hr",
EightHours = "8-hr",
TwelveHours = "12-hr",
OneDay = "1-day",
ThreeDay = "3-day",
OneWeek = "1-wk",
TwoWeeks = "2-wk",
ThirtyDays = "30-day",
}
export interface AllSystemStats {
basic: BasicSystemStats;
cpu: CpuUsage;
disk: DiskUsage;
network: NetworkUsage;
processes?: SystemProcess[];
components?: SystemComponent[];
polling_rate: Timelength;
refresh_ts: I64;
refresh_list_ts: I64;
}
export enum StatsState {
Ok = "Ok",
Warning = "Warning",
Critical = "Critical",
}
export interface ServerHealth {
cpu: StatsState;
mem: StatsState;
disk: StatsState;
disks: Record<string, StatsState>;
}
export type UpdateTarget =
| { type: "System", id?: undefined }
| { type: "Build", id: string }
| { type: "Builder", id: string }
| { type: "Deployment", id: string }
| { type: "Server", id: string };
export enum Operation {
None = "None",
CreateServer = "CreateServer",
UpdateServer = "UpdateServer",
DeleteServer = "DeleteServer",
RenameServer = "RenameServer",
PruneImagesServer = "PruneImagesServer",
PruneContainersServer = "PruneContainersServer",
PruneNetworksServer = "PruneNetworksServer",
CreateBuild = "CreateBuild",
UpdateBuild = "UpdateBuild",
DeleteBuild = "DeleteBuild",
RunBuild = "RunBuild",
CreateDeployment = "CreateDeployment",
UpdateDeployment = "UpdateDeployment",
DeleteDeployment = "DeleteDeployment",
DeployContainer = "DeployContainer",
StopContainer = "StopContainer",
StartContainer = "StartContainer",
RemoveContainer = "RemoveContainer",
PullDeployment = "PullDeployment",
RecloneDeployment = "RecloneDeployment",
RenameDeployment = "RenameDeployment",
CreateProcedure = "CreateProcedure",
UpdateProcedure = "UpdateProcedure",
DeleteProcedure = "DeleteProcedure",
CreateCommand = "CreateCommand",
UpdateCommand = "UpdateCommand",
DeleteCommand = "DeleteCommand",
RunCommand = "RunCommand",
CreateGroup = "CreateGroup",
UpdateGroup = "UpdateGroup",
DeleteGroup = "DeleteGroup",
ModifyUserEnabled = "ModifyUserEnabled",
ModifyUserCreateServerPermissions = "ModifyUserCreateServerPermissions",
ModifyUserCreateBuildPermissions = "ModifyUserCreateBuildPermissions",
ModifyUserPermissions = "ModifyUserPermissions",
AutoBuild = "AutoBuild",
AutoPull = "AutoPull",
}
export interface Log {
stage: string;
command: string;
stdout: string;
stderr: string;
success: boolean;
start_ts: I64;
end_ts: I64;
}
export enum UpdateStatus {
Queued = "Queued",
InProgress = "InProgress",
Complete = "Complete",
}
export interface Update {
_id?: string;
target: UpdateTarget;
operation: Operation;
logs: Log[];
start_ts: I64;
end_ts?: I64;
status: UpdateStatus;
success: boolean;
operator: string;
version: Version;
}
export interface ApiSecret {
name: string;
hash?: string;
created_at: I64;
expires?: I64;
}
export interface User {
_id?: string;
username: string;
enabled?: boolean;
admin?: boolean;
create_server_permissions?: boolean;
create_build_permissions?: boolean;
avatar?: string;
secrets?: ApiSecret[];
password?: string;
github_id?: string;
google_id?: string;
created_at?: I64;
updated_at?: I64;
}
export interface RunBuild {
build_id: string;
}
export interface RenameDeployment {
id: string;
name: string;
}
export interface Deploy {
deployment_id: string;
}
export interface StartContainer {
deployment_id: string;
}
export interface StopContainer {
deployment_id: string;
signal?: TerminationSignal;
time?: number;
}
export interface RemoveContainer {
deployment_id: string;
signal?: TerminationSignal;
time?: number;
}
export interface CreateLoginSecret {
name: string;
expires?: I64;
}
export interface CreateLoginSecretResponse {
secret: string;
}
export interface DeleteLoginSecret {
name: string;
}
export interface RenameServer {
id: string;
name: string;
}
export interface GetPeripheryVersion {
server_id: string;
}
export interface GetPeripheryVersionResponse {
version: string;
}
export interface GetAllSystemStats {
server_id: string;
}
export interface GetLoginOptions {
}
export interface GetLoginOptionsResponse {
local: boolean;
github: boolean;
google: boolean;
}
export interface CreateLocalUser {
username: string;
password: string;
}
export interface CreateLocalUserResponse {
jwt: string;
}
export interface LoginLocalUser {
username: string;
password: string;
}
export interface LoginLocalUserResponse {
jwt: string;
}
export interface ExchangeForJwt {
token: string;
}
export interface ExchangeForJwtResponse {
jwt: string;
}
export interface LoginWithSecret {
username: string;
secret: string;
}
export interface LoginWithSecretResponse {
jwt: string;
}
export enum ServerStatus {
NotOk = "NotOk",
Ok = "Ok",
Disabled = "Disabled",
}

30
lib/types/src/entities/deployment.rs
View File

@@ -1,4 +1,4 @@
use bson::serde_helpers::hex_string_as_object_id;
use bson::{doc, serde_helpers::hex_string_as_object_id};
use derive_builder::Builder;
use mungos::MungosIndexed;
use partial_derive2::Partial;
@@ -8,7 +8,7 @@ use typeshare::typeshare;
use crate::{i64_is_zero, I64};
use super::{EnvironmentVar, PermissionsMap};
use super::{EnvironmentVar, PermissionsMap, Version};
#[typeshare]
#[derive(Serialize, Deserialize, Debug, Clone, Builder, MungosIndexed)]
@@ -48,6 +48,8 @@ pub struct Deployment {
#[derive(Serialize, Deserialize, Debug, Clone, Builder, Partial, MungosIndexed)]
#[partial_derive(Serialize, Deserialize, Debug, Clone)]
#[skip_serializing_none]
#[doc_index(doc! { "image.type": 1 })]
#[sparse_doc_index(doc! { "image.params.build_id": 1 })]
pub struct DeploymentConfig {
#[serde(default)]
#[builder(default)]
@@ -56,12 +58,7 @@ pub struct DeploymentConfig {
#[serde(default)]
#[builder(default)]
#[index]
pub build_id: String,
#[serde(default)]
#[builder(default)]
pub image: String,
pub image: DeploymentImage,
#[serde(default)]
#[builder(default)]
@@ -140,7 +137,6 @@ impl From<PartialDeploymentConfig> for DeploymentConfig {
fn from(value: PartialDeploymentConfig) -> DeploymentConfig {
DeploymentConfig {
server_id: value.server_id.unwrap_or_default(),
build_id: value.build_id.unwrap_or_default(),
image: value.image.unwrap_or_default(),
skip_secret_interp: value.skip_secret_interp.unwrap_or_default(),
redeploy_on_build: value.redeploy_on_build.unwrap_or_default(),
@@ -165,6 +161,22 @@ impl From<PartialDeploymentConfig> for DeploymentConfig {
}
}
#[typeshare]
#[derive(Serialize, Deserialize, Debug, Clone, MungosIndexed)]
#[serde(tag = "type", content = "params")]
pub enum DeploymentImage {
Image { image: String },
Build { build_id: String, version: Version },
}
impl Default for DeploymentImage {
fn default() -> Self {
Self::Image {
image: Default::default(),
}
}
}
#[typeshare]
#[derive(Serialize, Deserialize, Debug, Clone, Default, PartialEq)]
pub struct Conversion {

4
lib/types/src/entities/mod.rs
View File

@@ -75,6 +75,10 @@ impl Version {
pub fn increment(&mut self) {
self.patch += 1;
}
pub fn is_none(&self) -> bool {
self.major == 0 && self.minor == 0 && self.patch == 0
}
}
#[typeshare]

39
lib/types/src/lib.rs
View File

@@ -1,3 +1,8 @@
use async_timing_util::unix_timestamp_ms;
use entities::{
build::{Build, BuildConfig},
update::Log,
};
use typeshare::typeshare;
pub mod busy;
@@ -14,7 +19,7 @@ fn i64_is_zero(n: &I64) -> bool {
*n == 0
}
pub fn all_logs_success(logs: &Vec<entities::update::Log>) -> bool {
pub fn all_logs_success(logs: &Vec<Log>) -> bool {
for log in logs {
if !log.success {
return false;
@@ -29,4 +34,34 @@ pub fn optional_string(string: &str) -> Option<String> {
} else {
Some(string.to_string())
}
}
}
pub fn get_image_name(
Build {
name,
config:
BuildConfig {
docker_organization,
docker_account,
..
},
..
}: &Build,
) -> String {
let name = to_monitor_name(name);
if !docker_organization.is_empty() {
format!("{docker_organization}/{name}")
} else if !docker_account.is_empty() {
format!("{docker_account}/{name}")
} else {
name
}
}
pub fn to_monitor_name(name: &str) -> String {
name.to_lowercase().replace(' ', "_")
}
pub fn monitor_timestamp() -> i64 {
unix_timestamp_ms() as i64
}

36
lib/types/src/requests/api/deployment.rs
View File

@@ -5,7 +5,7 @@ use resolver_api::derive::Request;
use serde::{Serialize, Deserialize};
use typeshare::typeshare;
use crate::{MongoDocument, entities::{deployment::{Deployment, PartialDeploymentConfig}, update::Update}};
use crate::{MongoDocument, entities::{deployment::{Deployment, PartialDeploymentConfig, TerminationSignal}, update::Update}};
//
@@ -19,4 +19,38 @@ derive_crud_requests!(Deployment);
pub struct RenameDeployment {
pub id: String,
pub name: String,
}
#[typeshare]
#[derive(Serialize, Deserialize, Debug, Clone, Request)]
#[response(Update)]
pub struct Deploy {
pub deployment_id: String,
pub stop_signal: Option<TerminationSignal>,
pub stop_time: Option<i32>,
}
#[typeshare]
#[derive(Serialize, Deserialize, Debug, Clone, Request)]
#[response(Update)]
pub struct StartContainer {
pub deployment_id: String,
}
#[typeshare]
#[derive(Serialize, Deserialize, Debug, Clone, Request)]
#[response(Update)]
pub struct StopContainer {
pub deployment_id: String,
pub signal: Option<TerminationSignal>,
pub time: Option<i32>,
}
#[typeshare]
#[derive(Serialize, Deserialize, Debug, Clone, Request)]
#[response(Update)]
pub struct RemoveContainer {
pub deployment_id: String,
pub signal: Option<TerminationSignal>,
pub time: Option<i32>,
}

1
periphery/Cargo.toml
View File

@@ -18,7 +18,6 @@ path = "src/main.rs"
[dependencies]
# local
monitor_types.workspace = true
monitor_helpers.workspace = true
# external
tokio.workspace = true
axum.workspace = true

14
periphery/src/helpers/docker/build.rs
View File

@@ -1,12 +1,14 @@
use std::{collections::HashMap, path::PathBuf};
use anyhow::Context;
use monitor_helpers::to_monitor_name;
use monitor_types::{entities::{
build::{Build, BuildConfig},
update::Log,
EnvironmentVar, Version,
}, optional_string};
use monitor_types::{
entities::{
build::{Build, BuildConfig},
update::Log,
EnvironmentVar, Version,
},
optional_string, to_monitor_name,
};
use crate::helpers::run_monitor_command;

38
periphery/src/helpers/docker/container.rs
View File

@@ -1,15 +1,17 @@
use std::collections::HashMap;
use anyhow::{anyhow, Context};
use monitor_helpers::to_monitor_name;
use monitor_types::{entities::{
deployment::{
Conversion, Deployment, DeploymentConfig, DockerContainerStats, RestartMode,
TerminationSignal,
use monitor_types::{
entities::{
deployment::{
Conversion, Deployment, DeploymentConfig, DeploymentImage, DockerContainerStats,
RestartMode, TerminationSignal,
},
update::Log,
EnvironmentVar,
},
update::Log,
EnvironmentVar,
}, optional_string};
optional_string, to_monitor_name,
};
use run_command::async_run_command;
use crate::helpers::{docker::parse_extra_args, run_monitor_command};
@@ -151,9 +153,23 @@ pub async fn deploy(
{
return Log::error("docker login", format!("{e:#?}"));
}
let _ = pull_image(&deployment.config.image).await;
let image = if let DeploymentImage::Image { image } = &deployment.config.image {
if image.is_empty() {
return Log::error(
"get image",
String::from("deployment does not have image attached"),
);
}
image
} else {
return Log::error(
"get image",
String::from("deployment does not have image attached"),
);
};
let _ = pull_image(image).await;
let _ = stop_and_remove_container(&deployment.name, stop_signal, stop_time).await;
let command = docker_run_command(deployment);
let command = docker_run_command(deployment, image);
if deployment.config.skip_secret_interp {
run_monitor_command("docker run", command).await
} else {
@@ -177,7 +193,6 @@ pub fn docker_run_command(
name,
config:
DeploymentConfig {
image,
volumes,
ports,
network,
@@ -190,6 +205,7 @@ pub fn docker_run_command(
},
..
}: &Deployment,
image: &str,
) -> String {
let name = to_monitor_name(name);
let container_user = parse_container_user(container_user);

6
periphery/src/helpers/git.rs
View File

@@ -2,8 +2,10 @@ use std::path::PathBuf;
use anyhow::anyhow;
use async_timing_util::unix_timestamp_ms;
use monitor_helpers::to_monitor_name;
use monitor_types::entities::{update::Log, CloneArgs, SystemCommand};
use monitor_types::{
entities::{update::Log, CloneArgs, SystemCommand},
to_monitor_name,
};
use run_command::async_run_command;
use super::run_monitor_command;

24
periphery/src/requests/container.rs
View File

@@ -1,8 +1,11 @@
use anyhow::{anyhow, Context};
use monitor_types::{entities::{
deployment::{BasicContainerInfo, Deployment, DockerContainerStats, TerminationSignal},
update::Log,
}, optional_string};
use monitor_types::{
entities::{
deployment::{BasicContainerInfo, Deployment, DockerContainerStats, TerminationSignal},
update::Log,
},
optional_string,
};
use resolver_api::{derive::Request, Resolve};
use serde::{Deserialize, Serialize};
@@ -179,7 +182,18 @@ impl Resolve<Deploy> for State {
let secrets = self.secrets.clone();
let log = match self.get_docker_token(&optional_string(&deployment.config.docker_account)) {
Ok(docker_token) => tokio::spawn(async move {
docker::deploy(&deployment, &docker_token, &secrets, stop_signal, stop_time).await
docker::deploy(
&deployment,
&docker_token,
&secrets,
stop_signal
.unwrap_or(deployment.config.termination_signal)
.into(),
stop_time
.unwrap_or(deployment.config.termination_timeout)
.into(),
)
.await
})
.await
.context("failed at spawn thread for deploy")?,

3
periphery/src/requests/git.rs
View File

@@ -1,5 +1,4 @@
use monitor_helpers::to_monitor_name;
use monitor_types::entities::{update::Log, CloneArgs, SystemCommand};
use monitor_types::{entities::{update::Log, CloneArgs, SystemCommand}, to_monitor_name};
use resolver_api::{derive::Request, Resolve};
use serde::{Deserialize, Serialize};