ComputerSurge/komodo
3
0
Fork 0
forked from github-starred/komodo
Code Pull Requests Packages Releases Activity

1.18.4 (#604)

Browse Source 
* update easy deps

* update otel deps

* implement template in types + update resource meta

* ts types

* dev-2

* dev-3 default template query is include

* Toggle resource is template in resource header

* dev-4 support CopyServer

* gen ts

* style template selector in New Resource menu

* fix new menu show 0

* add template market in omni search bar

* fix some dynamic import behavior

* template badge on dashboard

* dev-5

* standardize interpolation methods with nice api

* core use new interpolation methods

* refactor git usage

* dev-6 refactor interpolation / git methods

* fix pull stack passed replacers

*  new types

* remove redundant interpolation for build secret args

* clean up periphery docker client

* dev-7 include ports in container summary, see if they actually come through

* show container ports in container table

* refresh processes without tasks (more efficient)

* dev-8 keep container stats cache, include with ContainerListItem

* gen types

* display more container ports

* dev-9 fix repo clone when repo doesn't exist initially

* Add ports display to more spots

* fix function name

* add Periphery full container stats api, may be used later

* server container stats list

* dev-10

* 1.18.4 release

* Use reset instead of invalidate to fix GetUser spam on token expiry (#618)

---------

Co-authored-by: Jacky Fong <hello@huzky.dev>
This commit is contained in:
Maxwell Becker
2025-06-24 16:32:39 -07:00
committed by GitHub
parent 2205a81e79
commit 118ae9b92c
141 changed files with 5896 additions and 4453 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
lib/command/Cargo.toml
View File

@@ -10,6 +10,4 @@ homepage.workspace = true
[dependencies]
komodo_client.workspace = true
run_command.workspace = true
formatting.workspace = true
anyhow.workspace = true
svi.workspace = true

36
lib/command/src/lib.rs
View File

@@ -1,13 +1,10 @@
use std::{collections::HashMap, path::Path};
use std::path::Path;
use anyhow::Context;
use formatting::format_serror;
use komodo_client::{
entities::{komodo_timestamp, update::Log},
parsers::parse_multiline_command,
};
use run_command::{CommandOutput, async_run_command};
use svi::Interpolator;
pub async fn run_komodo_command(
stage: &str,
@@ -43,8 +40,7 @@ pub async fn run_komodo_command_multiline(
Some(run_komodo_command(stage, path, command).await)
}
/// Interpolates provided secrets into (potentially multiline) command,
/// executes the command, and sanitizes the output to avoid exposing the secrets.
/// Executes the command, and sanitizes the output to avoid exposing secrets in the log.
///
/// Checks to make sure the command is non-empty after being multiline-parsed.
///
@@ -52,30 +48,13 @@ pub async fn run_komodo_command_multiline(
/// and chains them together with '&&'.
/// Supports full line and end of line comments.
/// See [parse_multiline_command].
pub async fn run_komodo_command_with_interpolation(
pub async fn run_komodo_command_with_sanitization(
stage: &str,
path: impl Into<Option<&Path>>,
command: impl AsRef<str>,
parse_multiline: bool,
secrets: &HashMap<String, String>,
additional_replacers: &[(String, String)],
replacers: &[(String, String)],
) -> Option<Log> {
let (command, mut replacers) = match svi::interpolate_variables(
command.as_ref(),
secrets,
Interpolator::DoubleBrackets,
true,
)
.context("Failed to interpolate secrets")
{
Ok(res) => res,
Err(e) => {
return Some(Log::error(
&format!("{stage} - Interpolate Secrets"),
format_serror(&e.into()),
));
}
};
let mut log = if parse_multiline {
run_komodo_command_multiline(stage, path, command).await
} else {
@@ -83,10 +62,9 @@ pub async fn run_komodo_command_with_interpolation(
}?;
// Sanitize the command and output
replacers.extend_from_slice(additional_replacers);
log.command = svi::replace_in_string(&log.command, &replacers);
log.stdout = svi::replace_in_string(&log.stdout, &replacers);
log.stderr = svi::replace_in_string(&log.stderr, &replacers);
log.command = svi::replace_in_string(&log.command, replacers);
log.stdout = svi::replace_in_string(&log.stdout, replacers);
log.stderr = svi::replace_in_string(&log.stderr, replacers);
Some(log)
}

15
lib/environment/Cargo.toml Normal file
View File

@@ -0,0 +1,15 @@
[package]
name = "environment"
version.workspace = true
edition.workspace = true
authors.workspace = true
license.workspace = true
repository.workspace = true
homepage.workspace = true
[dependencies]
komodo_client.workspace = true
formatting.workspace = true
#
anyhow.workspace = true
tokio.workspace = true

67
lib/environment/src/lib.rs Normal file
View File

@@ -0,0 +1,67 @@
use std::path::{Path, PathBuf};
use anyhow::Context;
use formatting::format_serror;
use komodo_client::entities::{EnvironmentVar, update::Log};
/// If the environment was written and needs to be passed to the compose command,
/// will return the env file PathBuf.
/// Should ensure all logs are successful after calling.
pub async fn write_env_file(
environment: &[EnvironmentVar],
folder: &Path,
env_file_path: &str,
logs: &mut Vec<Log>,
) -> Option<PathBuf> {
let env_file_path =
folder.join(env_file_path).components().collect::<PathBuf>();
if environment.is_empty() {
// Still want to return Some(env_file_path) if the path
// already exists on the host and is a file.
// This is for "Files on Server" mode when user writes the env file themself.
if env_file_path.is_file() {
return Some(env_file_path);
}
return None;
}
let contents = environment
.iter()
.map(|env| format!("{}={}", env.variable, env.value))
.collect::<Vec<_>>()
.join("\n");
if let Some(parent) = env_file_path.parent() {
if let Err(e) = tokio::fs::create_dir_all(parent)
.await
.with_context(|| format!("Failed to initialize environment file parent directory {parent:?}"))
{
logs.push(Log::error(
"Write Environment File",
format_serror(&e.into()),
));
return None;
}
}
if let Err(e) = tokio::fs::write(&env_file_path, contents)
.await
.with_context(|| {
format!("Failed to write environment file to {env_file_path:?}")
})
{
logs.push(Log::error(
"Write Environment File",
format_serror(&e.into()),
));
return None;
}
logs.push(Log::simple(
"Write Environment File",
format!("Environment file written to {env_file_path:?}"),
));
Some(env_file_path)
}

1
lib/git/Cargo.toml
View File

@@ -14,7 +14,6 @@ command.workspace = true
cache.workspace = true
#
run_command.workspace = true
svi.workspace = true
#
tracing.workspace = true
anyhow.workspace = true

261
lib/git/src/clone.rs
View File

@@ -1,216 +1,123 @@
use std::{collections::HashMap, path::Path};
use std::{io::ErrorKind, path::Path};
use command::{
run_komodo_command, run_komodo_command_multiline,
run_komodo_command_with_interpolation,
};
use anyhow::Context;
use command::run_komodo_command;
use formatting::format_serror;
use komodo_client::entities::{
CloneArgs, EnvironmentVar, all_logs_success, komodo_timestamp,
RepoExecutionArgs, RepoExecutionResponse, all_logs_success,
update::Log,
};
use run_command::async_run_command;
use crate::{GitRes, get_commit_hash_log};
use crate::get_commit_hash_log;
/// Will delete the existing repo folder,
/// clone the repo, get the latest hash / message,
/// and run on_clone / on_pull.
///
/// Assumes all interpolation is already done and takes the list of replacers
/// for the On Clone command.
#[tracing::instrument(
level = "debug",
skip(
clone_args,
access_token,
environment,
secrets,
core_replacers
)
skip(clone_args, access_token)
)]
pub async fn clone<T>(
clone_args: T,
root_repo_dir: &Path,
access_token: Option<String>,
environment: &[EnvironmentVar],
env_file_path: &str,
// if skip_secret_interp is none, make sure to pass None here
secrets: Option<&HashMap<String, String>>,
core_replacers: &[(String, String)],
) -> anyhow::Result<GitRes>
) -> anyhow::Result<RepoExecutionResponse>
where
T: Into<CloneArgs> + std::fmt::Debug,
T: Into<RepoExecutionArgs> + std::fmt::Debug,
{
let args: CloneArgs = clone_args.into();
let path = args.path(root_repo_dir);
let args: RepoExecutionArgs = clone_args.into();
let repo_url = args.remote_url(access_token.as_deref())?;
let mut logs = clone_inner(
&repo_url,
&args.branch,
&args.commit,
&path,
access_token,
)
.await;
if !all_logs_success(&logs) {
tracing::warn!(
"Failed to clone repo at {path:?} | name: {} | {logs:?}",
args.name
);
return Ok(GitRes {
logs,
path,
hash: None,
message: None,
env_file_path: None,
});
}
tracing::debug!("repo at {path:?} cloned");
let (hash, message) = match get_commit_hash_log(&path).await {
Ok((log, hash, message)) => {
logs.push(log);
(Some(hash), Some(message))
}
Err(e) => {
logs.push(Log::simple(
"Latest Commit",
format_serror(
&e.context("Failed to get latest commit").into(),
),
));
(None, None)
}
let mut res = RepoExecutionResponse {
path: args.path(root_repo_dir),
logs: Vec::new(),
commit_hash: None,
commit_message: None,
};
let Ok((env_file_path, _replacers)) =
crate::environment::write_file(
environment,
env_file_path,
secrets,
&path,
&mut logs,
)
.await
else {
return Ok(GitRes {
logs,
path,
hash,
message,
env_file_path: None,
});
};
if let Some(command) = args.on_clone {
let on_clone_path = path.join(&command.path);
if let Some(log) = if let Some(secrets) = secrets {
run_komodo_command_with_interpolation(
"On Clone",
Some(on_clone_path.as_path()),
&command.command,
true,
secrets,
core_replacers,
)
.await
} else {
run_komodo_command_multiline(
"On Clone",
Some(on_clone_path.as_path()),
&command.command,
)
.await
} {
logs.push(log)
};
}
if let Some(command) = args.on_pull {
let on_pull_path = path.join(&command.path);
if let Some(log) = if let Some(secrets) = secrets {
run_komodo_command_with_interpolation(
"On Pull",
Some(on_pull_path.as_path()),
&command.command,
true,
secrets,
core_replacers,
)
.await
} else {
run_komodo_command_multiline(
"On Pull",
Some(on_pull_path.as_path()),
&command.command,
)
.await
} {
logs.push(log)
};
}
Ok(GitRes {
logs,
path,
hash,
message,
env_file_path,
})
}
async fn clone_inner(
repo_url: &str,
branch: &str,
commit: &Option<String>,
destination: &Path,
access_token: Option<String>,
) -> Vec<Log> {
let _ = tokio::fs::remove_dir_all(destination).await;
// Ensure parent folder exists
if let Some(parent) = destination.parent() {
let _ = tokio::fs::create_dir_all(parent).await;
if let Some(parent) = res.path.parent() {
if let Err(e) = tokio::fs::create_dir_all(parent)
.await
.context("Failed to create clone parent directory.")
{
res.logs.push(Log::error(
"Prepare Repo Root",
format_serror(&e.into()),
));
return Ok(res);
}
}
match tokio::fs::remove_dir_all(&res.path).await {
Err(e) if e.kind() != ErrorKind::NotFound => {
let e: anyhow::Error = e.into();
res.logs.push(Log::error(
"Clean Repo Root",
format_serror(
&e.context(
"Failed to remove existing repo root before clone.",
)
.into(),
),
));
return Ok(res);
}
_ => {}
}
let command = format!(
"git clone {repo_url} {} -b {branch}",
destination.display()
"git clone {repo_url} {} -b {}",
res.path.display(),
args.branch
);
let start_ts = komodo_timestamp();
let output = async_run_command(&command).await;
let success = output.success();
let (command, stderr) = if let Some(token) = access_token {
(
command.replace(&token, "<TOKEN>"),
output.stderr.replace(&token, "<TOKEN>"),
)
} else {
(command, output.stderr)
};
let mut logs = vec![Log {
stage: "Clone Repo".to_string(),
command,
success,
stdout: output.stdout,
stderr,
start_ts,
end_ts: komodo_timestamp(),
}];
if !logs[0].success {
return logs;
let mut log = run_komodo_command("Clone Repo", None, command).await;
if let Some(token) = access_token {
log.command = log.command.replace(&token, "<TOKEN>");
log.stdout = log.stdout.replace(&token, "<TOKEN>");
log.stderr = log.stderr.replace(&token, "<TOKEN>");
}
if let Some(commit) = commit {
res.logs.push(log);
if !all_logs_success(&res.logs) {
return Ok(res);
}
if let Some(commit) = args.commit {
let reset_log = run_komodo_command(
"set commit",
destination,
res.path.as_path(),
format!("git reset --hard {commit}",),
)
.await;
logs.push(reset_log);
res.logs.push(reset_log);
}
logs
if !all_logs_success(&res.logs) {
return Ok(res);
}
match get_commit_hash_log(&res.path)
.await
.context("Failed to get latest commit")
{
Ok((log, hash, message)) => {
res.logs.push(log);
res.commit_hash = Some(hash);
res.commit_message = Some(message);
}
Err(e) => {
res
.logs
.push(Log::simple("Latest Commit", format_serror(&e.into())));
}
};
Ok(res)
}

78
lib/git/src/commit.rs
View File

@@ -3,11 +3,13 @@ use std::path::{Path, PathBuf};
use anyhow::Context;
use command::run_komodo_command;
use formatting::format_serror;
use komodo_client::entities::{all_logs_success, update::Log};
use komodo_client::entities::{
RepoExecutionResponse, all_logs_success, update::Log,
};
use run_command::async_run_command;
use tokio::fs;
use crate::{GitRes, get_commit_hash_log};
use crate::get_commit_hash_log;
/// Write file, add, commit, force push.
/// Repo must be cloned.
@@ -15,31 +17,48 @@ pub async fn write_commit_file(
commit_msg: &str,
repo_dir: &Path,
// relative to repo root
file: &Path,
relative_file_path: &Path,
contents: &str,
branch: &str,
) -> anyhow::Result<GitRes> {
// Clean up the path by stripping any redundant `/./`
let path = repo_dir.join(file).components().collect::<PathBuf>();
) -> anyhow::Result<RepoExecutionResponse> {
let mut res = RepoExecutionResponse {
path: repo_dir.to_path_buf(),
logs: Vec::new(),
commit_hash: None,
commit_message: None,
};
if let Some(parent) = path.parent() {
// Clean up the path by stripping any redundant `/./`
let full_file_path = repo_dir
.join(relative_file_path)
.components()
.collect::<PathBuf>();
if let Some(parent) = full_file_path.parent() {
fs::create_dir_all(parent).await.with_context(|| {
format!("Failed to initialize file parent directory {parent:?}")
})?;
}
fs::write(&path, contents).await.with_context(|| {
format!("Failed to write contents to {path:?}")
})?;
fs::write(&full_file_path, contents)
.await
.with_context(|| {
format!("Failed to write contents to {full_file_path:?}")
})?;
let mut res = GitRes::default();
res.logs.push(Log::simple(
"Write file",
format!("File contents written to {path:?}"),
format!("File contents written to {full_file_path:?}"),
));
commit_file_inner(commit_msg, &mut res, repo_dir, file, branch)
.await;
commit_file_inner(
commit_msg,
&mut res,
repo_dir,
relative_file_path,
branch,
)
.await;
Ok(res)
}
@@ -52,16 +71,23 @@ pub async fn commit_file(
// relative to repo root
file: &Path,
branch: &str,
) -> GitRes {
let mut res = GitRes::default();
) -> RepoExecutionResponse {
let mut res = RepoExecutionResponse {
path: repo_dir.to_path_buf(),
logs: Vec::new(),
commit_hash: None,
commit_message: None,
};
commit_file_inner(commit_msg, &mut res, repo_dir, file, branch)
.await;
res
}
pub async fn commit_file_inner(
commit_msg: &str,
res: &mut GitRes,
res: &mut RepoExecutionResponse,
repo_dir: &Path,
// relative to repo root
file: &Path,
@@ -102,8 +128,8 @@ pub async fn commit_file_inner(
match get_commit_hash_log(repo_dir).await {
Ok((log, hash, message)) => {
res.logs.push(log);
res.hash = Some(hash);
res.message = Some(message);
res.commit_hash = Some(hash);
res.commit_message = Some(message);
}
Err(e) => {
res.logs.push(Log::error(
@@ -120,6 +146,7 @@ pub async fn commit_file_inner(
format!("git push --set-upstream origin {branch}"),
)
.await;
res.logs.push(push_log);
}
@@ -129,10 +156,15 @@ pub async fn commit_all(
repo_dir: &Path,
message: &str,
branch: &str,
) -> GitRes {
) -> RepoExecutionResponse {
ensure_global_git_config_set().await;
let mut res = GitRes::default();
let mut res = RepoExecutionResponse {
path: repo_dir.to_path_buf(),
logs: Vec::new(),
commit_hash: None,
commit_message: None,
};
let add_log =
run_komodo_command("Add Files", repo_dir, "git add -A").await;
@@ -155,8 +187,8 @@ pub async fn commit_all(
match get_commit_hash_log(repo_dir).await {
Ok((log, hash, message)) => {
res.logs.push(log);
res.hash = Some(hash);
res.message = Some(message);
res.commit_hash = Some(hash);
res.commit_message = Some(message);
}
Err(e) => {
res.logs.push(Log::error(

166
lib/git/src/environment.rs
View File

@@ -1,166 +0,0 @@
use std::{
collections::HashMap,
path::{Path, PathBuf},
};
use anyhow::Context;
use formatting::format_serror;
use komodo_client::entities::{EnvironmentVar, update::Log};
/// If the environment was written and needs to be passed to the compose command,
/// will return the env file PathBuf.
/// If variables were interpolated, will also return the sanitizing replacers.
pub async fn write_file(
environment: &[EnvironmentVar],
env_file_path: &str,
secrets: Option<&HashMap<String, String>>,
folder: &Path,
logs: &mut Vec<Log>,
) -> Result<(Option<PathBuf>, Option<Vec<(String, String)>>), ()> {
let env_file_path = folder.join(env_file_path);
if environment.is_empty() {
// Still want to return Some(env_file_path) if the path
// already exists on the host and is a file.
// This is for "Files on Server" mode when user writes the env file themself.
if env_file_path.is_file() {
return Ok((Some(env_file_path), None));
}
return Ok((None, None));
}
let contents = environment
.iter()
.map(|env| format!("{}={}", env.variable, env.value))
.collect::<Vec<_>>()
.join("\n");
let (contents, replacers) = if let Some(secrets) = secrets {
let res = svi::interpolate_variables(
&contents,
secrets,
svi::Interpolator::DoubleBrackets,
true,
)
.context("failed to interpolate secrets into environment");
let (contents, replacers) = match res {
Ok(res) => res,
Err(e) => {
logs.push(Log::error(
"Interpolate - Environment",
format_serror(&e.into()),
));
return Err(());
}
};
if !replacers.is_empty() {
logs.push(Log::simple(
"Interpolate - Environment",
replacers
.iter()
.map(|(_, variable)| format!("<span class=\"text-muted-foreground\">replaced:</span> {variable}"))
.collect::<Vec<_>>()
.join("\n"),
))
}
(contents, Some(replacers))
} else {
(contents, None)
};
if let Some(parent) = env_file_path.parent() {
if let Err(e) = tokio::fs::create_dir_all(parent)
.await
.with_context(|| format!("Failed to initialize environment file parent directory {parent:?}"))
{
logs.push(Log::error(
"Write Environment File",
format_serror(&e.into()),
));
return Err(());
}
}
if let Err(e) = tokio::fs::write(&env_file_path, contents)
.await
.with_context(|| {
format!("Failed to write environment file to {env_file_path:?}")
})
{
logs.push(Log::error(
"Write Environment File",
format_serror(&e.into()),
));
return Err(());
}
logs.push(Log::simple(
"Write Environment File",
format!("Environment file written to {env_file_path:?}"),
));
Ok((Some(env_file_path), replacers))
}
///
/// Will return the env file PathBuf.
pub async fn write_file_simple(
environment: &[EnvironmentVar],
env_file_path: &str,
folder: &Path,
logs: &mut Vec<Log>,
) -> anyhow::Result<Option<PathBuf>> {
let env_file_path = folder.join(env_file_path);
if environment.is_empty() {
// Still want to return Some(env_file_path) if the path
// already exists on the host and is a file.
// This is for "Files on Server" mode when user writes the env file themself.
if env_file_path.is_file() {
return Ok(Some(env_file_path));
}
return Ok(None);
}
let contents = environment
.iter()
.map(|env| format!("{}={}", env.variable, env.value))
.collect::<Vec<_>>()
.join("\n");
if let Some(parent) = env_file_path.parent() {
if let Err(e) = tokio::fs::create_dir_all(parent)
.await
.with_context(|| format!("Failed to initialize environment file parent directory {parent:?}"))
{
logs.push(Log::error(
"Write Environment File",
format_serror(&(&e).into()),
));
return Err(e);
}
}
if let Err(e) = tokio::fs::write(&env_file_path, contents)
.await
.with_context(|| {
format!("Failed to write environment file to {env_file_path:?}")
})
{
logs.push(Log::error(
"Write Environment file",
format_serror(&(&e).into()),
));
return Err(e);
}
logs.push(Log::simple(
"Write Environment File",
format!("Environment written to {env_file_path:?}"),
));
Ok(Some(env_file_path))
}

4
lib/git/src/init.rs
View File

@@ -3,12 +3,12 @@ use std::path::Path;
use command::run_komodo_command;
use formatting::format_serror;
use komodo_client::entities::{
CloneArgs, all_logs_success, update::Log,
RepoExecutionArgs, all_logs_success, update::Log,
};
pub async fn init_folder_as_repo(
folder_path: &Path,
args: &CloneArgs,
args: &RepoExecutionArgs,
access_token: Option<&str>,
logs: &mut Vec<Log>,
) {

13
lib/git/src/lib.rs
View File

@@ -1,4 +1,4 @@
use std::path::{Path, PathBuf};
use std::path::Path;
use anyhow::{Context, anyhow};
use formatting::{bold, muted};
@@ -8,8 +8,6 @@ use komodo_client::entities::{
use run_command::async_run_command;
use tracing::instrument;
pub mod environment;
mod clone;
mod commit;
mod init;
@@ -24,15 +22,6 @@ pub use crate::{
pull_or_clone::pull_or_clone,
};
#[derive(Debug, Default, Clone)]
pub struct GitRes {
pub logs: Vec<Log>,
pub path: PathBuf,
pub hash: Option<String>,
pub message: Option<String>,
pub env_file_path: Option<PathBuf>,
}
#[instrument(level = "debug")]
pub async fn get_commit_hash_info(
repo_dir: &Path,

193
lib/git/src/pull.rs
View File

@@ -1,28 +1,26 @@
use std::{
collections::HashMap,
path::{Path, PathBuf},
sync::OnceLock,
};
use cache::TimeoutCache;
use command::{
run_komodo_command, run_komodo_command_multiline,
run_komodo_command_with_interpolation,
};
use command::run_komodo_command;
use formatting::format_serror;
use komodo_client::entities::{
CloneArgs, EnvironmentVar, all_logs_success, komodo_timestamp,
update::Log,
RepoExecutionArgs, RepoExecutionResponse, all_logs_success,
komodo_timestamp, update::Log,
};
use crate::{GitRes, get_commit_hash_log};
use crate::get_commit_hash_log;
/// Wait this long after a pull to allow another pull through
const PULL_TIMEOUT: i64 = 5_000;
fn pull_cache() -> &'static TimeoutCache<PathBuf, GitRes> {
static PULL_CACHE: OnceLock<TimeoutCache<PathBuf, GitRes>> =
OnceLock::new();
fn pull_cache()
-> &'static TimeoutCache<PathBuf, RepoExecutionResponse> {
static PULL_CACHE: OnceLock<
TimeoutCache<PathBuf, RepoExecutionResponse>,
> = OnceLock::new();
PULL_CACHE.get_or_init(Default::default)
}
@@ -31,34 +29,29 @@ fn pull_cache() -> &'static TimeoutCache<PathBuf, GitRes> {
/// can change branch after clone, or even the remote.
#[tracing::instrument(
level = "debug",
skip(
clone_args,
access_token,
environment,
secrets,
core_replacers
)
skip(clone_args, access_token)
)]
#[allow(clippy::too_many_arguments)]
pub async fn pull<T>(
clone_args: T,
root_repo_dir: &Path,
access_token: Option<String>,
environment: &[EnvironmentVar],
env_file_path: &str,
// if skip_secret_interp is none, make sure to pass None here
secrets: Option<&HashMap<String, String>>,
core_replacers: &[(String, String)],
) -> anyhow::Result<GitRes>
) -> anyhow::Result<RepoExecutionResponse>
where
T: Into<CloneArgs> + std::fmt::Debug,
T: Into<RepoExecutionArgs> + std::fmt::Debug,
{
let args: CloneArgs = clone_args.into();
let path = args.path(root_repo_dir);
let args: RepoExecutionArgs = clone_args.into();
let repo_url = args.remote_url(access_token.as_deref())?;
let mut res = RepoExecutionResponse {
path: args.path(root_repo_dir),
logs: Vec::new(),
commit_hash: None,
commit_message: None,
};
// Acquire the path lock
let lock = pull_cache().get_lock(path.clone()).await;
let lock = pull_cache().get_lock(res.path.clone()).await;
// Lock the path lock, prevents simultaneous pulls by
// ensuring simultaneous pulls will wait for first to finish
@@ -71,33 +64,25 @@ where
}
let res = async {
let mut logs = Vec::new();
// Check for '.git' path to see if the folder is initialized as a git repo
let dot_git_path = path.join(".git");
let dot_git_path = res.path.join(".git");
if !dot_git_path.exists() {
crate::init::init_folder_as_repo(
&path,
&res.path,
&args,
access_token.as_deref(),
&mut logs,
&mut res.logs,
)
.await;
if !all_logs_success(&logs) {
return Ok(GitRes {
logs,
path,
hash: None,
message: None,
env_file_path: None,
});
if !all_logs_success(&res.logs) {
return Ok(res);
}
}
// Set remote url
let mut set_remote = run_komodo_command(
"Set git remote",
path.as_ref(),
"Set Git Remote",
res.path.as_ref(),
format!("git remote set-url origin {repo_url}"),
)
.await;
@@ -110,145 +95,75 @@ where
set_remote.stderr =
set_remote.stderr.replace(&token, "<TOKEN>");
}
logs.push(set_remote);
if !all_logs_success(&logs) {
return Ok(GitRes {
logs,
path,
hash: None,
message: None,
env_file_path: None,
});
res.logs.push(set_remote);
if !all_logs_success(&res.logs) {
return Ok(res);
}
// First fetch remote branches before checkout
let fetch = run_komodo_command(
"Git fetch",
path.as_ref(),
"Git Fetch",
res.path.as_ref(),
"git fetch --all --prune",
)
.await;
if !fetch.success {
logs.push(fetch);
return Ok(GitRes {
logs,
path,
hash: None,
message: None,
env_file_path: None,
});
res.logs.push(fetch);
return Ok(res);
}
let checkout = run_komodo_command(
"Checkout branch",
path.as_ref(),
res.path.as_ref(),
format!("git checkout -f {}", args.branch),
)
.await;
logs.push(checkout);
if !all_logs_success(&logs) {
return Ok(GitRes {
logs,
path,
hash: None,
message: None,
env_file_path: None,
});
res.logs.push(checkout);
if !all_logs_success(&res.logs) {
return Ok(res);
}
let pull_log = run_komodo_command(
"Git pull",
path.as_ref(),
res.path.as_ref(),
format!("git pull --rebase --force origin {}", args.branch),
)
.await;
logs.push(pull_log);
if !all_logs_success(&logs) {
return Ok(GitRes {
logs,
path,
hash: None,
message: None,
env_file_path: None,
});
res.logs.push(pull_log);
if !all_logs_success(&res.logs) {
return Ok(res);
}
if let Some(commit) = args.commit {
let reset_log = run_komodo_command(
"Set commit",
path.as_ref(),
res.path.as_ref(),
format!("git reset --hard {commit}"),
)
.await;
logs.push(reset_log);
res.logs.push(reset_log);
if !all_logs_success(&res.logs) {
return Ok(res);
}
}
let (hash, message) = match get_commit_hash_log(&path).await {
match get_commit_hash_log(&res.path).await {
Ok((log, hash, message)) => {
logs.push(log);
(Some(hash), Some(message))
res.logs.push(log);
res.commit_hash = Some(hash);
res.commit_message = Some(message);
}
Err(e) => {
logs.push(Log::simple(
res.logs.push(Log::simple(
"Latest Commit",
format_serror(
&e.context("Failed to get latest commit").into(),
),
));
(None, None)
}
};
let Ok((env_file_path, _replacers)) =
crate::environment::write_file(
environment,
env_file_path,
secrets,
&path,
&mut logs,
)
.await
else {
return Ok(GitRes {
logs,
path,
hash,
message,
env_file_path: None,
});
};
if let Some(command) = args.on_pull {
let on_pull_path = path.join(&command.path);
if let Some(log) = if let Some(secrets) = secrets {
run_komodo_command_with_interpolation(
"On Pull",
Some(on_pull_path.as_path()),
&command.command,
true,
secrets,
core_replacers,
)
.await
} else {
run_komodo_command_multiline(
"On Pull",
Some(on_pull_path.as_path()),
&command.command,
)
.await
} {
logs.push(log)
};
}
anyhow::Ok(GitRes {
logs,
path,
hash,
message,
env_file_path,
})
anyhow::Ok(res)
}
.await;

54
lib/git/src/pull_or_clone.rs
View File

@@ -1,61 +1,37 @@
use std::{collections::HashMap, path::Path};
use std::path::Path;
use komodo_client::entities::{CloneArgs, EnvironmentVar};
use crate::GitRes;
use komodo_client::entities::{
RepoExecutionArgs, RepoExecutionResponse,
};
/// This is a mix of clone / pull.
/// - If the folder doesn't exist, it will clone the repo.
/// - Second variable in tuple will be `true`
/// - If it does, it will ensure the remote is correct,
/// ensure the correct branch is (force) checked out,
/// force pull the repo, and switch to specified hash if provided.
#[tracing::instrument(
level = "debug",
skip(
clone_args,
access_token,
environment,
secrets,
core_replacers
)
skip(clone_args, access_token)
)]
pub async fn pull_or_clone<T>(
clone_args: T,
root_repo_dir: &Path,
access_token: Option<String>,
environment: &[EnvironmentVar],
env_file_path: &str,
// if skip_secret_interp is none, make sure to pass None here
secrets: Option<&HashMap<String, String>>,
core_replacers: &[(String, String)],
) -> anyhow::Result<GitRes>
) -> anyhow::Result<(RepoExecutionResponse, bool)>
where
T: Into<CloneArgs> + std::fmt::Debug,
T: Into<RepoExecutionArgs> + std::fmt::Debug,
{
let args: CloneArgs = clone_args.into();
let args: RepoExecutionArgs = clone_args.into();
let folder_path = args.path(root_repo_dir);
if folder_path.exists() {
crate::pull(
args,
root_repo_dir,
access_token,
environment,
env_file_path,
secrets,
core_replacers,
)
.await
crate::pull(args, root_repo_dir, access_token)
.await
.map(|r| (r, false))
} else {
crate::clone(
args,
root_repo_dir,
access_token,
environment,
env_file_path,
secrets,
core_replacers,
)
.await
crate::clone(args, root_repo_dir, access_token)
.await
.map(|r| (r, true))
}
}

15
lib/interpolate/Cargo.toml Normal file
View File

@@ -0,0 +1,15 @@
[package]
name = "interpolate"
version.workspace = true
edition.workspace = true
authors.workspace = true
license.workspace = true
repository.workspace = true
homepage.workspace = true
[dependencies]
komodo_client.workspace = true
#
svi.workspace = true
#
anyhow.workspace = true

181
lib/interpolate/src/lib.rs Normal file
View File

@@ -0,0 +1,181 @@
use std::collections::{HashMap, HashSet};
use anyhow::Context;
use komodo_client::entities::{
EnvironmentVar, build::Build, deployment::Deployment, repo::Repo,
stack::Stack, update::Log,
};
pub struct Interpolator<'a> {
variables: Option<&'a HashMap<String, String>>,
secrets: &'a HashMap<String, String>,
variable_replacers: HashSet<(String, String)>,
pub secret_replacers: HashSet<(String, String)>,
}
impl<'a> Interpolator<'a> {
pub fn new(
variables: Option<&'a HashMap<String, String>>,
secrets: &'a HashMap<String, String>,
) -> Interpolator<'a> {
Interpolator {
variables,
secrets,
variable_replacers: Default::default(),
secret_replacers: Default::default(),
}
}
pub fn interpolate_stack(
&mut self,
stack: &mut Stack,
) -> anyhow::Result<&mut Self> {
if stack.config.skip_secret_interp {
return Ok(self);
}
self
.interpolate_string(&mut stack.config.file_contents)?
.interpolate_string(&mut stack.config.environment)?
.interpolate_string(&mut stack.config.pre_deploy.command)?
.interpolate_string(&mut stack.config.post_deploy.command)?
.interpolate_extra_args(&mut stack.config.extra_args)?
.interpolate_extra_args(&mut stack.config.build_extra_args)
}
pub fn interpolate_repo(
&mut self,
repo: &mut Repo,
) -> anyhow::Result<&mut Self> {
if repo.config.skip_secret_interp {
return Ok(self);
}
self
.interpolate_string(&mut repo.config.environment)?
.interpolate_string(&mut repo.config.on_clone.command)?
.interpolate_string(&mut repo.config.on_pull.command)
}
pub fn interpolate_build(
&mut self,
build: &mut Build,
) -> anyhow::Result<&mut Self> {
if build.config.skip_secret_interp {
return Ok(self);
}
self
.interpolate_string(&mut build.config.build_args)?
.interpolate_string(&mut build.config.secret_args)?
.interpolate_string(&mut build.config.labels)?
.interpolate_string(&mut build.config.pre_build.command)?
.interpolate_string(&mut build.config.dockerfile)?
.interpolate_extra_args(&mut build.config.extra_args)
}
pub fn interpolate_deployment(
&mut self,
deployment: &mut Deployment,
) -> anyhow::Result<&mut Self> {
if deployment.config.skip_secret_interp {
return Ok(self);
}
self
.interpolate_string(&mut deployment.config.environment)?
.interpolate_string(&mut deployment.config.ports)?
.interpolate_string(&mut deployment.config.volumes)?
.interpolate_string(&mut deployment.config.labels)?
.interpolate_string(&mut deployment.config.command)?
.interpolate_extra_args(&mut deployment.config.extra_args)
}
pub fn interpolate_string(
&mut self,
target: &mut String,
) -> anyhow::Result<&mut Self> {
if target.is_empty() {
return Ok(self);
}
// first pass - variables
let res = if let Some(variables) = self.variables {
let (res, more_replacers) = svi::interpolate_variables(
target,
variables,
svi::Interpolator::DoubleBrackets,
false,
)
.with_context(|| {
format!(
"failed to interpolate variables into target '{target}'",
)
})?;
self.variable_replacers.extend(more_replacers);
res
} else {
target.to_string()
};
// second pass - secrets
let (res, more_replacers) = svi::interpolate_variables(
&res,
self.secrets,
svi::Interpolator::DoubleBrackets,
false,
)
.with_context(|| {
format!("failed to interpolate secrets into target '{target}'",)
})?;
self.secret_replacers.extend(more_replacers);
// Set with result
*target = res;
Ok(self)
}
pub fn interpolate_extra_args(
&mut self,
extra_args: &mut Vec<String>,
) -> anyhow::Result<&mut Self> {
for arg in extra_args {
self
.interpolate_string(arg)
.context("failed interpolation into extra arg")?;
}
Ok(self)
}
pub fn interpolate_env_vars(
&mut self,
env_vars: &mut Vec<EnvironmentVar>,
) -> anyhow::Result<&mut Self> {
for var in env_vars {
self
.interpolate_string(&mut var.value)
.context("failed interpolation into variable value")?;
}
Ok(self)
}
pub fn push_logs(&self, logs: &mut Vec<Log>) {
// Show which variables / values were interpolated
if !self.variable_replacers.is_empty() {
logs.push(Log::simple("Interpolate Variables", self.variable_replacers
.iter()
.map(|(value, variable)| format!("<span class=\"text-muted-foreground\">{variable} =></span> {value}"))
.collect::<Vec<_>>()
.join("\n")));
}
// Only show names of interpolated secrets
if !self.secret_replacers.is_empty() {
logs.push(
Log::simple("Interpolate Secrets",
self.secret_replacers
.iter()
.map(|(_, variable)| format!("<span class=\"text-muted-foreground\">replaced:</span> {variable}"))
.collect::<Vec<_>>()
.join("\n"),)
);
}
}
}