github-starred/KohakuHub
2
0
Fork 0
mirror of https://github.com/KohakuBlueleaf/KohakuHub.git synced 2026-03-09 07:12:07 -05:00
Code Issues Packages Projects Releases Wiki Activity

fix scripts bugs

Browse Source
This commit is contained in:
Kohaku-Blueleaf
2025-10-22 21:20:54 +08:00
parent 80315c53e9
commit de7cc89e47
3 changed files with 59 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
scripts/db_migrations/014_user_external_tokens.py
View File

@@ -104,7 +104,7 @@ def run():
db.connect(reuse_if_open=True)
# Check if should skip
if should_skip_due_to_future_migrations(db, cfg, MIGRATION_NUMBER):
if should_skip_due_to_future_migrations(MIGRATION_NUMBER, db, cfg):
print(
f"\n⚠️ Skipping migration {MIGRATION_NUMBER} - already applied or superseded by future migrations\n"
)

48
scripts/generate_docker_compose.py
View File

@@ -544,9 +544,7 @@ builtin = true
print(f" python scripts/generate_docker_compose.py --config {output_path}")
def migrate_existing_config(
docker_compose_path: Path, config_toml_path: Path
) -> dict:
def migrate_existing_config(docker_compose_path: Path, config_toml_path: Path) -> dict:
"""Migrate existing configuration files interactively.
Reads existing values and only prompts for new fields.
@@ -569,7 +567,9 @@ def migrate_existing_config(
# Extract environment variables
for line in content.split("\n"):
match = re.match(r"\s*- (KOHAKU_HUB_\w+)=(.+?)(?:\s+#.*)?$", line.strip())
match = re.match(
r"\s*- (KOHAKU_HUB_\w+)=(.+?)(?:\s+#.*)?$", line.strip()
)
if match:
key, value = match.groups()
existing_env[key] = value.strip()
@@ -619,9 +619,7 @@ def migrate_existing_config(
db_url = get_existing("KOHAKU_HUB_DATABASE_URL", "app.database_url")
if db_url and db_url.startswith("postgresql://"):
# Parse: postgresql://user:pass@host:port/db
match = re.match(
r"postgresql://([^:]+):([^@]+)@([^:]+):(\d+)/(.+)", db_url
)
match = re.match(r"postgresql://([^:]+):([^@]+)@([^:]+):(\d+)/(.+)", db_url)
if match:
user, password, host, port, db = match.groups()
config["postgres_user"] = user
@@ -661,16 +659,28 @@ def migrate_existing_config(
config["s3_builtin"] = s3_endpoint and "minio" in s3_endpoint
print(f"Using: {'Built-in MinIO' if config['s3_builtin'] else 'External S3'}")
config["s3_access_key"] = get_existing("KOHAKU_HUB_S3_ACCESS_KEY", "s3.access_key") or "minioadmin"
config["s3_secret_key"] = get_existing("KOHAKU_HUB_S3_SECRET_KEY", "s3.secret_key") or "minioadmin"
config["s3_access_key"] = (
get_existing("KOHAKU_HUB_S3_ACCESS_KEY", "s3.access_key") or "minioadmin"
)
config["s3_secret_key"] = (
get_existing("KOHAKU_HUB_S3_SECRET_KEY", "s3.secret_key") or "minioadmin"
)
config["s3_endpoint"] = s3_endpoint or "http://minio:9000"
config["s3_region"] = get_existing("KOHAKU_HUB_S3_REGION", "s3.region") or "us-east-1"
config["s3_signature_version"] = get_existing("KOHAKU_HUB_S3_SIGNATURE_VERSION", "s3.signature_version") or ""
config["s3_region"] = (
get_existing("KOHAKU_HUB_S3_REGION", "s3.region") or "us-east-1"
)
config["s3_signature_version"] = (
get_existing("KOHAKU_HUB_S3_SIGNATURE_VERSION", "s3.signature_version") or ""
)
# Security Configuration
print("\n--- Security Configuration ---")
config["session_secret"] = get_existing("KOHAKU_HUB_SESSION_SECRET", "auth.session_secret")
config["admin_secret"] = get_existing("KOHAKU_HUB_ADMIN_SECRET_TOKEN", "admin.secret_token")
config["session_secret"] = get_existing(
"KOHAKU_HUB_SESSION_SECRET", "auth.session_secret"
)
config["admin_secret"] = get_existing(
"KOHAKU_HUB_ADMIN_SECRET_TOKEN", "admin.secret_token"
)
# NEW FIELD: database_key
config["database_key"] = get_existing("KOHAKU_HUB_DATABASE_KEY", "app.database_key")
@@ -679,7 +689,9 @@ def migrate_existing_config(
default_db_key = generate_secret(32)
print(f" Generated: {default_db_key}")
use_generated = ask_yes_no("Use generated database key?", default=True)
config["database_key"] = default_db_key if use_generated else ask_string("Database encryption key")
config["database_key"] = (
default_db_key if use_generated else ask_string("Database encryption key")
)
else:
print(f" Database key: (exists)")
@@ -735,7 +747,9 @@ def main():
existing_docker_compose = repo_root / "docker-compose.yml"
existing_config_toml = repo_root / "config.toml"
has_existing_config = existing_docker_compose.exists() or existing_config_toml.exists()
has_existing_config = (
existing_docker_compose.exists() or existing_config_toml.exists()
)
if has_existing_config and not args.config:
print("🔍 Found existing configuration files:")
@@ -1014,9 +1028,6 @@ token_expire_days = 365
enabled = true
secret_token = "{config['admin_secret']}"
[app]
database_key = "{config['database_key']}" # For encrypting external fallback tokens
[quota]
default_user_private_quota_bytes = 10_000_000 # 10MB
default_user_public_quota_bytes = 100_000_000 # 100MB
@@ -1035,6 +1046,7 @@ base_url = "http://localhost:48888" # Dev server URL
api_base = "/api"
db_backend = "postgres"
database_url = "{db_url}"
database_key = "{config['database_key']}" # For encrypting external fallback tokens
# LFS Configuration (sizes in decimal: 1MB = 1,000,000 bytes)
lfs_threshold_bytes = 5_000_000 # 5MB - files larger use LFS
lfs_multipart_threshold_bytes = 100_000_000 # 100MB - files larger use multipart upload

34
scripts/migrate_config.py
View File

@@ -75,7 +75,11 @@ def read_existing_docker_compose(filepath: Path) -> dict:
in_environment = True
elif in_hub_api and in_environment:
# Check if we've left the environment section
if stripped and not stripped.startswith("-") and not stripped.startswith("#"):
if (
stripped
and not stripped.startswith("-")
and not stripped.startswith("#")
):
# New section started
in_environment = False
in_hub_api = False
@@ -319,7 +323,9 @@ def write_docker_compose(filepath: Path, env_vars: dict, base_content: str = Non
# Keep comment from original line
comment_match = re.search(r"(#.+)$", line)
comment = comment_match.group(1) if comment_match else ""
output_lines.append(f"{indent}- {key}={env_vars[key]} {comment}".rstrip())
output_lines.append(
f"{indent}- {key}={env_vars[key]} {comment}".rstrip()
)
continue
output_lines.append(line)
@@ -346,7 +352,9 @@ def write_config_toml(filepath: Path, config: dict):
for key, value in data.items():
if isinstance(value, dict):
# Nested section
write_section(f"{section_name}.{key}" if section_name else key, value, indent)
write_section(
f"{section_name}.{key}" if section_name else key, value, indent
)
elif isinstance(value, bool):
lines.append(f"{prefix}{key} = {str(value).lower()}")
elif isinstance(value, (int, float)):
@@ -355,7 +363,10 @@ def write_config_toml(filepath: Path, config: dict):
lines.append(f'{prefix}{key} = "{value}"')
elif isinstance(value, list):
# Simple list formatting
items = ", ".join(f'"{item}"' if isinstance(item, str) else str(item) for item in value)
items = ", ".join(
f'"{item}"' if isinstance(item, str) else str(item)
for item in value
)
lines.append(f"{prefix}{key} = [{items}]")
else:
lines.append(f'{prefix}{key} = "{value}"')
@@ -364,7 +375,16 @@ def write_config_toml(filepath: Path, config: dict):
lines.append("") # Blank line after section
# Write sections in order
for section in ["s3", "lakefs", "smtp", "auth", "admin", "app", "quota", "fallback"]:
for section in [
"s3",
"lakefs",
"smtp",
"auth",
"admin",
"app",
"quota",
"fallback",
]:
if section in config:
write_section(section, config[section])
@@ -465,7 +485,9 @@ def main():
print("\n💡 Next steps:")
print(" 1. Review the updated configuration files")
print(" 2. Restart services: docker-compose down && docker-compose up -d")
print(" 3. Run migrations: docker-compose exec hub-api python scripts/run_migrations.py")
print(
" 3. Run migrations: docker-compose exec hub-api python scripts/run_migrations.py"
)
print()