mirror of
https://github.com/KohakuBlueleaf/KohakuHub.git
synced 2026-04-29 02:48:33 -05:00
use correct size unit in every script
This commit is contained in:
@@ -53,7 +53,7 @@ def generate_structured_content(file_number, size_bytes):
|
||||
header = f"""# Test File {file_number:04d}
|
||||
|
||||
This is a generated test file for upload testing.
|
||||
File size: {size_bytes / (1024*1024):.2f} MB
|
||||
File size: {size_bytes / (1000*1000):.2f} MB
|
||||
|
||||
## Content Section
|
||||
|
||||
@@ -86,13 +86,13 @@ def main():
|
||||
output_dir = Path("large_file_test")
|
||||
num_files = 1000
|
||||
file_size_mb = 1.5
|
||||
file_size_bytes = int(file_size_mb * 1024 * 1024)
|
||||
file_size_bytes = int(file_size_mb * 1000 * 1000)
|
||||
|
||||
# Create output directory
|
||||
output_dir.mkdir(exist_ok=True)
|
||||
print(f"Creating {num_files} files of {file_size_mb}MB each...")
|
||||
print(f"Output directory: {output_dir.absolute()}")
|
||||
print(f"Total size: {num_files * file_size_mb / 1024:.2f} GB")
|
||||
print(f"Total size: {num_files * file_size_mb / 1000:.2f} GB")
|
||||
print()
|
||||
|
||||
# Generate files
|
||||
@@ -119,7 +119,7 @@ def main():
|
||||
|
||||
# Calculate actual size
|
||||
total_size = sum(f.stat().st_size for f in output_dir.glob("*.txt"))
|
||||
print(f"✓ Total size: {total_size / (1024**3):.2f} GB")
|
||||
print(f"✓ Total size: {total_size / (1000**3):.2f} GB")
|
||||
print()
|
||||
print("You can now test with:")
|
||||
print(" - Upload via web UI")
|
||||
|
||||
@@ -255,10 +255,9 @@ def generate_test_repo(base_path: str = "test_folder"):
|
||||
lfs_size = sum(size for _, size in lfs_files)
|
||||
regular_size = sum(size for _, size in regular_files)
|
||||
|
||||
print(f"\n Total size: {total_size / 1024 / 1024:.2f} MB")
|
||||
print(f" LFS size: {lfs_size / 1024 / 1024:.2f} MB")
|
||||
print(f" Regular size: {regular_size / 1024:.2f} KB")
|
||||
|
||||
print(f"\n Total size: {total_size / 1000 / 1000:.2f} MB")
|
||||
print(f" LFS size: {lfs_size / 1000 / 1000:.2f} MB")
|
||||
print(f" Regular size: {regular_size / 1000:.2f} KB")
|
||||
print(f"\n Directory structure:")
|
||||
print(f" - Root: 2 files")
|
||||
print(f" - config/: 2 files")
|
||||
|
||||
@@ -200,7 +200,7 @@ def display_summary(bucket, stats, detailed=False):
|
||||
|
||||
def display_quota_warning(total_size, quota_gb=10):
|
||||
"""Display warning if approaching quota limit."""
|
||||
quota_bytes = quota_gb * 1024**3
|
||||
quota_bytes = quota_gb * 1000**3
|
||||
percentage = (total_size / quota_bytes * 100) if quota_bytes > 0 else 0
|
||||
|
||||
console.print()
|
||||
|
||||
@@ -126,8 +126,8 @@ def step2_populate_mock_data():
|
||||
is_org=True,
|
||||
email=None,
|
||||
password_hash=None,
|
||||
private_quota_bytes=10 * 1024 * 1024 * 1024, # 10GB
|
||||
public_quota_bytes=50 * 1024 * 1024 * 1024, # 50GB
|
||||
private_quota_bytes=10 * 1000 * 1000 * 1000, # 10GB
|
||||
public_quota_bytes=50 * 1000 * 1000 * 1000, # 50GB
|
||||
)
|
||||
print(f" Created organization: {org.username}")
|
||||
|
||||
@@ -140,8 +140,8 @@ def step2_populate_mock_data():
|
||||
password_hash="dummy_hash",
|
||||
email_verified=True,
|
||||
is_active=True,
|
||||
private_quota_bytes=5 * 1024 * 1024 * 1024, # 5GB
|
||||
public_quota_bytes=20 * 1024 * 1024 * 1024, # 20GB
|
||||
private_quota_bytes=5 * 1000 * 1000 * 1000, # 5GB
|
||||
public_quota_bytes=20 * 1000 * 1000 * 1000, # 20GB
|
||||
)
|
||||
print(f" Created user: {user.username}")
|
||||
|
||||
@@ -154,7 +154,7 @@ def step2_populate_mock_data():
|
||||
private=False,
|
||||
owner=org,
|
||||
quota_bytes=None, # Inherit from org
|
||||
used_bytes=1024 * 1024 * 100, # 100MB
|
||||
used_bytes=100 * 1000 * 1000, # 100MB
|
||||
)
|
||||
print(f" Created repository: {repo1.full_id}")
|
||||
|
||||
@@ -165,8 +165,8 @@ def step2_populate_mock_data():
|
||||
full_id="test-user/test-dataset",
|
||||
private=True,
|
||||
owner=user,
|
||||
quota_bytes=2 * 1024 * 1024 * 1024, # Custom 2GB quota
|
||||
used_bytes=500 * 1024 * 1024, # 500MB used
|
||||
quota_bytes=2 * 1000 * 1000 * 1000, # Custom 2GB quota
|
||||
used_bytes=500 * 1000 * 1000, # 500MB used
|
||||
)
|
||||
print(f" Created repository: {repo2.full_id}")
|
||||
|
||||
@@ -346,11 +346,11 @@ def step4_verify_migration():
|
||||
print(" ✗ quota_bytes should still be NULL!")
|
||||
return False
|
||||
|
||||
if repo1.used_bytes != 1024 * 1024 * 100:
|
||||
if repo1.used_bytes != 100 * 1000 * 1000:
|
||||
print(f" ✗ used_bytes changed! Expected 104857600, got {repo1.used_bytes}")
|
||||
return False
|
||||
|
||||
if repo2.quota_bytes != 2 * 1024 * 1024 * 1024:
|
||||
if repo2.quota_bytes != 2 * 1000 * 1000 * 1000:
|
||||
print(f" ✗ quota_bytes changed! Expected 2147483648, got {repo2.quota_bytes}")
|
||||
return False
|
||||
|
||||
@@ -390,7 +390,7 @@ def step5_test_new_functionality():
|
||||
keep_versions = get_effective_lfs_keep_versions(repo)
|
||||
suffix_rules = get_effective_lfs_suffix_rules(repo)
|
||||
|
||||
print(f" Effective threshold: {threshold / (1024*1024):.1f} MB")
|
||||
print(f" Effective threshold: {threshold / (1000*1000):.1f} MB")
|
||||
print(f" Effective keep_versions: {keep_versions}")
|
||||
print(f" Suffix rules: {suffix_rules}")
|
||||
|
||||
@@ -411,9 +411,8 @@ def step5_test_new_functionality():
|
||||
return False
|
||||
|
||||
# Test 2: should_use_lfs with defaults
|
||||
test_small = should_use_lfs(repo, "config.json", 1024) # 1KB
|
||||
test_large = should_use_lfs(repo, "model.bin", 10 * 1024 * 1024) # 10MB
|
||||
|
||||
test_small = should_use_lfs(repo, "config.json", 1000) # 1KB
|
||||
test_large = should_use_lfs(repo, "model.bin", 10 * 1000 * 1000) # 10MB
|
||||
print(f" config.json (1KB) uses LFS: {test_small}")
|
||||
print(f" model.bin (10MB) uses LFS: {test_large}")
|
||||
|
||||
@@ -425,14 +424,13 @@ def step5_test_new_functionality():
|
||||
|
||||
# Test 3: Custom threshold
|
||||
print("\n Test 2: Custom threshold (1MB)")
|
||||
repo.lfs_threshold_bytes = 1024 * 1024
|
||||
repo.lfs_threshold_bytes = 1000 * 1000
|
||||
repo.save()
|
||||
|
||||
threshold = get_effective_lfs_threshold(repo)
|
||||
test_500kb = should_use_lfs(repo, "file.bin", 500 * 1024)
|
||||
test_2mb = should_use_lfs(repo, "file.bin", 2 * 1024 * 1024)
|
||||
|
||||
print(f" Effective threshold: {threshold / (1024*1024):.1f} MB")
|
||||
test_500kb = should_use_lfs(repo, "file.bin", 500 * 1000)
|
||||
test_2mb = should_use_lfs(repo, "file.bin", 2 * 1000 * 1000)
|
||||
print(f" Effective threshold: {threshold / (1000*1000):.1f} MB")
|
||||
print(f" file.bin (500KB) uses LFS: {test_500kb}")
|
||||
print(f" file.bin (2MB) uses LFS: {test_2mb}")
|
||||
|
||||
|
||||
Reference in New Issue
Block a user