From 7173f1c598f613e91565f1bdbfe276e154a2a84a Mon Sep 17 00:00:00 2001 From: Kohaku-Blueleaf <59680068+KohakuBlueleaf@users.noreply.github.com> Date: Sat, 11 Oct 2025 00:14:23 +0800 Subject: [PATCH] fix ui and docker compose script --- scripts/README.md | 41 +++++++++ scripts/generate_docker_compose.py | 90 +++++++++++++++++-- .../src/components/common/CodeViewer.vue | 11 ++- .../src/components/repo/RepoViewer.vue | 31 +++++-- .../[name]/blob/[branch]/[...file].vue | 21 +++-- src/kohaku-hub-ui/src/pages/settings.vue | 11 ++- src/kohaku-hub-ui/src/utils/clipboard.js | 48 ++++++++++ 7 files changed, 225 insertions(+), 28 deletions(-) create mode 100644 src/kohaku-hub-ui/src/utils/clipboard.js diff --git a/scripts/README.md b/scripts/README.md index aad2dbe..a01764d 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -51,6 +51,11 @@ python scripts/generate_docker_compose.py --config kohakuhub.conf - Auto-generated admin secret token - Option to use custom secrets +5. **Network:** + - External Docker bridge network support + - Allows cross-compose communication with external PostgreSQL/S3 services + - Automatically added to hub-api and lakefs when using external services + **Example: Interactive Mode** ``` @@ -197,6 +202,10 @@ secret_key = minioadmin [security] session_secret = your-secret-here admin_secret = your-admin-secret + +[network] +# Optional: for cross-compose communication +external_network = shared-network ``` For external PostgreSQL or S3: @@ -215,8 +224,40 @@ endpoint = https://your-s3-endpoint.com access_key = your-access-key secret_key = your-secret-key region = us-east-1 + +[network] +# Required if external services are in different Docker Compose +external_network = shared-network ``` +**Using External Docker Network:** + +If PostgreSQL or S3 are in separate Docker Compose setups, you need a shared network: + +```bash +# Create the shared network first +docker network create shared-network + +# Your PostgreSQL docker-compose.yml +services: + postgres: + # ... your config + networks: + - shared-network + +networks: + shared-network: + external: true + +# Generate KohakuHub with external network +python scripts/generate_docker_compose.py --config kohakuhub.conf +``` + +The generator will automatically: +- Add the external network to `hub-api` and `lakefs` services +- Configure them to use both `default` (hub-net) and the external network +- Allow container name resolution across compose files + **Important Notes:** - Shell scripts automatically use LF line endings (configured in `.gitattributes`) - Database initialization runs automatically on LakeFS startup diff --git a/scripts/generate_docker_compose.py b/scripts/generate_docker_compose.py index 80b1833..7176fd3 100644 --- a/scripts/generate_docker_compose.py +++ b/scripts/generate_docker_compose.py @@ -146,7 +146,9 @@ def generate_lakefs_service(config: dict) -> str: force_path_style = "true" else: s3_endpoint = config["s3_endpoint"] - force_path_style = "true" if "minio" in s3_endpoint.lower() else "false" + # Use path-style for all non-AWS endpoints (MinIO, CloudFlare R2, custom S3) + # Only AWS S3 (*.amazonaws.com) should use virtual-hosted style + force_path_style = "false" if "amazonaws.com" in s3_endpoint.lower() else "true" # Add entrypoint and volumes for database initialization entrypoint_config = "" @@ -160,6 +162,16 @@ def generate_lakefs_service(config: dict) -> str: - ./scripts/lakefs-entrypoint.sh:/scripts/lakefs-entrypoint.sh:ro - ./scripts/init-databases.sh:/scripts/init-databases.sh:ro""" + # Add external network if needed (for external postgres or s3) + lakefs_networks_str = "" + if config.get("external_network") and ( + not config["postgres_builtin"] or not config["s3_builtin"] + ): + lakefs_networks_str = f""" networks: + - default + - {config['external_network']} +""" + return f""" lakefs: image: treeverse/lakefs:latest container_name: lakefs @@ -180,7 +192,7 @@ def generate_lakefs_service(config: dict) -> str: user: "${{UID}}:${{GID}}" {depends_on_str} volumes: {volumes_config} -""" +{lakefs_networks_str}""" def generate_hub_api_service(config: dict) -> str: @@ -197,6 +209,16 @@ def generate_hub_api_service(config: dict) -> str: for dep in depends_on: depends_on_str += f" - {dep}\n" + # Add external network if needed (for external postgres or s3) + networks_str = "" + if config.get("external_network") and ( + not config["postgres_builtin"] or not config["s3_builtin"] + ): + networks_str = f""" networks: + - default + - {config['external_network']} +""" + # Database configuration if config["postgres_builtin"]: db_url = f"postgresql://{config['postgres_user']}:{config['postgres_password']}@postgres:5432/{config['postgres_db']}" @@ -261,7 +283,7 @@ def generate_hub_api_service(config: dict) -> str: - KOHAKU_HUB_DEFAULT_ORG_PUBLIC_QUOTA_BYTES=100_000_000 volumes: - ./hub-meta/hub-api:/hub-api-creds -""" +{networks_str}""" def generate_hub_ui_service() -> str: @@ -299,8 +321,16 @@ def generate_docker_compose(config: dict) -> str: content = "# docker-compose.yml\n# Generated by KohakuHub docker-compose generator\n\nservices:\n" content += "\n".join(services) + + # Network configuration content += "\nnetworks:\n default:\n name: hub-net\n" + # Add external network if specified + if config.get("external_network"): + content += f""" {config['external_network']}: + external: true +""" + return content @@ -368,6 +398,13 @@ def load_config_file(config_path: Path) -> dict: config["session_secret"] = generate_secret() config["admin_secret"] = generate_secret() + # Network section + if parser.has_section("network"): + net = parser["network"] + config["external_network"] = net.get("external_network", fallback="") + else: + config["external_network"] = "" + return config @@ -418,6 +455,12 @@ secret_key = minioadmin # Session and admin secrets (auto-generated if not specified) # session_secret = your-session-secret-here # admin_secret = your-admin-secret-here + +[network] +# External bridge network (optional) +# Use this if PostgreSQL or S3 are in different Docker Compose setups +# Create the network first: docker network create shared-network +# external_network = shared-network """ output_path.write_text(template, encoding="utf-8") @@ -577,6 +620,26 @@ def interactive_config() -> dict: # LakeFS encryption key config["lakefs_encrypt_key"] = generate_secret() + # Network configuration + print() + print("--- Network Configuration ---") + use_external_network = False + if not config["postgres_builtin"] or not config["s3_builtin"]: + use_external_network = ask_yes_no( + "Use external Docker network for cross-compose communication?", + default=False, + ) + + if use_external_network: + config["external_network"] = ask_string( + "External network name", default="shared-network" + ) + print() + print(f"Note: Make sure the network exists:") + print(f" docker network create {config['external_network']}") + else: + config["external_network"] = "" + return config @@ -622,21 +685,34 @@ def generate_and_write_files(config: dict): print(f"S3 Storage: {'Built-in MinIO' if config['s3_builtin'] else 'Custom S3'}") if not config["s3_builtin"]: print(f" Endpoint: {config['s3_endpoint']}") + if config.get("external_network"): + print(f"External Network: {config['external_network']}") print(f"Session Secret: {config['session_secret'][:20]}...") print(f"Admin Secret: {config['admin_secret'][:20]}...") print("-" * 60) print() print("Next steps:") - print("1. Review the generated docker-compose.yml") - print("2. Build frontend: npm run build --prefix ./src/kohaku-hub-ui") - print("3. Start services: docker-compose up -d") + step_num = 1 + + if config.get("external_network"): + print(f"{step_num}. Create external network if not exists:") + print(f" docker network create {config['external_network']}") + step_num += 1 + print() + + print(f"{step_num}. Review the generated docker-compose.yml") + step_num += 1 + print(f"{step_num}. Build frontend: npm run build --prefix ./src/kohaku-hub-ui") + step_num += 1 + print(f"{step_num}. Start services: docker-compose up -d") print() if config["lakefs_use_postgres"]: print(" Note: Databases will be created automatically on first startup:") print(f" - {config['postgres_db']} (hub-api)") print(f" - {config['lakefs_db']} (LakeFS)") print() - print("4. Access at: http://localhost:28080") + step_num += 1 + print(f"{step_num}. Access at: http://localhost:28080") print() diff --git a/src/kohaku-hub-ui/src/components/common/CodeViewer.vue b/src/kohaku-hub-ui/src/components/common/CodeViewer.vue index 6cf20f8..d3be520 100644 --- a/src/kohaku-hub-ui/src/components/common/CodeViewer.vue +++ b/src/kohaku-hub-ui/src/components/common/CodeViewer.vue @@ -39,6 +39,7 @@