From bb326a0f930fbc5d0270916ca19558557e98b245 Mon Sep 17 00:00:00 2001 From: Paula Barszcz Date: Thu, 25 Aug 2022 20:17:05 +0200 Subject: [PATCH] [DockerSize] Docker image size multi arch (#8290) * Get the size of the docker image taking architecture into account Co-authored-by: chris48s --- services/docker/docker-fixtures.js | 99 ++++++----------- services/docker/docker-helpers.js | 22 ++++ services/docker/docker-size.service.js | 125 +++++++++++++++++++--- services/docker/docker-size.spec.js | 107 +++++++++++++----- services/docker/docker-size.tester.js | 23 +++- services/docker/docker-version.service.js | 19 +--- 6 files changed, 269 insertions(+), 126 deletions(-) diff --git a/services/docker/docker-fixtures.js b/services/docker/docker-fixtures.js index 26218ca350..a8f17aaa7a 100644 --- a/services/docker/docker-fixtures.js +++ b/services/docker/docker-fixtures.js @@ -1,71 +1,36 @@ const sizeDataNoTagSemVerSort = [ - { name: 'master', full_size: 13449470 }, - { name: 'feature-smtps-support', full_size: 13449638 }, - { name: 'latest', full_size: 13448411 }, - { name: '4', full_size: 13448411 }, - { name: '4.3', full_size: 13448411 }, - { name: '4.3.0', full_size: 13448411 }, - { name: '4.2', full_size: 13443674 }, - { name: '4.2.0', full_size: 13443674 }, - { name: '4.1', full_size: 19244435 }, - { name: '4.1.0', full_size: 19244435 }, - { name: 'v4.0.0-alpha2', full_size: 10933605 }, - { name: 'v4.0.0-alpha1', full_size: 10933644 }, - { name: '4.0.0', full_size: 11512227 }, - { name: '4.0', full_size: 11512227 }, - { name: 'v2.1.9', full_size: 29739490 }, - { name: 'v2.1.10', full_size: 29739842 }, - { name: 'v3.0.0', full_size: 32882980 }, - { name: 'v3.0.1', full_size: 32880923 }, - { name: 'v3.1.0', full_size: 32441549 }, - { name: 'v3.1.1', full_size: 32441767 }, - { name: 'v3.1.2', full_size: 32442741 }, - { name: 'v3.1.3', full_size: 32442629 }, - { name: 'v3.1.4', full_size: 32478607 }, - { name: 'v3.2.0', full_size: 33489914 }, - { name: 'v3.3.0', full_size: 33628545 }, - { name: 'v3.3.1', full_size: 33629018 }, - { name: 'v3.3.3', full_size: 33628988 }, - { name: 'v3.3.4', full_size: 33629019 }, - { name: 'v3.3.6', full_size: 33628753 }, - { name: 'v3.3.7', full_size: 33629556 }, - { name: 'v3.3.8', full_size: 33644261 }, - { name: 'v3.3.9', full_size: 33644175 }, - { name: 'v3.3.10', full_size: 33644406 }, - { name: 'v3.3.11', full_size: 33644430 }, - { name: 'v3.3.12', full_size: 33644703 }, - { name: 'v3.3.13', full_size: 33644377 }, - { name: 'v3.3.15', full_size: 33644581 }, - { name: 'v3.3.16', full_size: 33644663 }, - { name: 'v3.3.17', full_size: 33644228 }, - { name: 'v3.3.18', full_size: 33644466 }, - { name: 'v3.3.19', full_size: 33644724 }, - { name: 'v3.4.0', full_size: 34918552 }, - { name: 'v3.4.2', full_size: 33605129 }, - { name: 'v3.5.0', full_size: 33582915 }, - { name: 'v3.6.0', full_size: 34789944 }, - { name: 'develop', full_size: 38129308 }, - { name: 'v3.7.0', full_size: 38179583 }, - { name: 'v3.7.1', full_size: 38614944 }, - { name: 'v3.8.0', full_size: 42962384 }, - { name: 'v3.8.1', full_size: 40000713 }, - { name: 'v3.8.2', full_size: 40000567 }, - { name: 'v3.8.3', full_size: 40040963 }, - { name: 'v3.9.0', full_size: 40044357 }, - { name: 'v3.9.1', full_size: 40048123 }, - { name: 'v3.9.2', full_size: 40047663 }, - { name: 'v3.9.3', full_size: 40048204 }, - { name: 'v3.9.4', full_size: 40049571 }, - { name: 'v3.9.5', full_size: 40049695 }, - { name: 'v3.10.0', full_size: 39940736 }, - { name: 'v3.11.0', full_size: 39928170 }, - { name: 'v3.12.0', full_size: 39966770 }, - { name: 'v3.13.0', full_size: 38556045 }, - { name: 'v3.14.0', full_size: 38574008 }, - { name: 'v3.15.0', full_size: 38578507 }, - { name: 'v3.16.0', full_size: 38852598 }, - { name: 'v3.16.1', full_size: 38851702 }, - { name: 'v3.16.2', full_size: 38969822 }, + { + full_size: 300000000, + name: 'v4.0.0-alpha2', + images: [ + { architecture: 'amd64', size: 219939484 }, + { architecture: 'arm64', size: 200000000 }, + ], + }, + { + full_size: 400000000, + name: 'v4.2.4', + images: [ + { architecture: 'amd64', size: 220000000 }, + { architecture: 'arm64', size: 210000000 }, + ], + }, + { + full_size: 100000000, + name: 'v3.9.7', + images: [ + { architecture: 'amd64', size: 120000000 }, + { architecture: 'arm64', size: 110000000 }, + ], + }, + { + full_size: 500000000, + name: 'latest', + images: [ + { architecture: 'amd64', size: 560000000 }, + { architecture: 'arm64', size: 460000000 }, + ], + }, ] const versionDataNoTagDateSort = { count: 4, diff --git a/services/docker/docker-helpers.js b/services/docker/docker-helpers.js index 0b9b1d6879..c7da530fd5 100644 --- a/services/docker/docker-helpers.js +++ b/services/docker/docker-helpers.js @@ -1,7 +1,28 @@ +import Joi from 'joi' // see https://github.com/badges/shields/pull/1690 import { NotFound } from '../index.js' const dockerBlue = '066da5' +// Valid architecture values: https://golang.org/doc/install/source#environment (GOARCH) +const archSchema = Joi.alternatives( + Joi.string().valid( + 'amd64', + 'arm', + 'arm64', + 's390x', + '386', + 'ppc64', + 'ppc64le', + 'wasm', + 'mips', + 'mipsle', + 'mips64', + 'mips64le', + 'riscv64' + ), + Joi.number().valid(386).cast('string') +) + function buildDockerUrl(badgeName, includeTagRoute) { if (includeTagRoute) { return { @@ -55,6 +76,7 @@ function getDigestSemVerMatches({ data, digest }) { } export { + archSchema, dockerBlue, buildDockerUrl, getDockerHubUser, diff --git a/services/docker/docker-size.service.js b/services/docker/docker-size.service.js index 16c7ea4946..4d4c5f6a6a 100644 --- a/services/docker/docker-size.service.js +++ b/services/docker/docker-size.service.js @@ -4,6 +4,7 @@ import { nonNegativeInteger } from '../validators.js' import { latest } from '../version.js' import { BaseJsonService, NotFound } from '../index.js' import { + archSchema, buildDockerUrl, getDockerHubUser, getMultiPageData, @@ -12,6 +13,12 @@ import { const buildSchema = Joi.object({ name: Joi.string().required(), full_size: nonNegativeInteger.required(), + images: Joi.array().items( + Joi.object({ + size: nonNegativeInteger.required(), + architecture: Joi.string().required(), + }) + ), }).required() const pagedSchema = Joi.object({ @@ -20,14 +27,37 @@ const pagedSchema = Joi.object({ Joi.object({ name: Joi.string().required(), full_size: nonNegativeInteger.required(), + images: Joi.array().items( + Joi.object({ + size: nonNegativeInteger.required(), + architecture: Joi.string().required(), + }) + ), }) ), }).required() const queryParamSchema = Joi.object({ sort: Joi.string().valid('date', 'semver').default('date'), + arch: archSchema, }).required() +// If user provided the arch parameter, +// check if any of the returned images has an architecture matching the arch parameter provided. +// If yes, return the size of the image with this arch. +// If not, throw the `NotFound` error. +// For details see: https://github.com/badges/shields/issues/8238 +function getImageSizeForArch(images, arch) { + const imgWithArch = Object.values(images).find( + img => img.architecture === arch + ) + + if (!imgWithArch) { + throw new NotFound({ prettyMessage: 'architecture not found' }) + } + return imgWithArch.size +} + export default class DockerSize extends BaseJsonService { static category = 'size' static route = { ...buildDockerUrl('image-size', true), queryParamSchema } @@ -46,6 +76,14 @@ export default class DockerSize extends BaseJsonService { queryParams: { sort: 'semver' }, staticPreview: this.render({ size: 136000000 }), }, + { + title: + 'Docker Image Size with architecture (latest by date/latest semver)', + pattern: ':user/:repo', + namedParams: { user: 'library', repo: 'mysql' }, + queryParams: { sort: 'date', arch: 'amd64' }, + staticPreview: this.render({ size: 146000000 }), + }, { title: 'Docker Image Size (tag)', pattern: ':user/:repo/:tag', @@ -73,30 +111,83 @@ export default class DockerSize extends BaseJsonService { }) } - transform({ tag, sort, data }) { - if (!tag && sort === 'date') { - if (data.count === 0) { - throw new NotFound({ prettyMessage: 'repository not found' }) + getSizeFromImageByLatestDate(data, arch) { + if (data.count === 0) { + throw new NotFound({ prettyMessage: 'repository not found' }) + } else { + const latestEntry = data.results[0] + + if (arch) { + return { size: getImageSizeForArch(latestEntry.images, arch) } } else { - return { size: data.results[0].full_size } + return { size: latestEntry.full_size } } - } else if (!tag && sort === 'semver') { - const [matches, versions] = data.reduce( - ([m, v], d) => { - m[d.name] = d.full_size - v.push(d.name) - return [m, v] - }, - [{}, []] - ) - const version = latest(versions) + } + } + + getSizeFromImageByLatestSemver(data, arch) { + // If no tag is specified, and sorting is by semver, first filter out the entry containing the latest semver from the response with Docker images. + // If no architecture is supplied by the user, return `full_size` from this entry. + // If the architecture is supplied by the user, check if any of the returned images for this entry has an architecture matching the arch parameter supplied by the user. + // If yes, return the size of the image with this arch. + // If not, throw the `NotFound` error. + + const [matches, versions, images] = data.reduce( + ([m, v, i], d) => { + m[d.name] = d.full_size + v.push(d.name) + i[d.name] = d.images + return [m, v, i] + }, + [{}, [], {}] + ) + + const version = latest(versions) + + let sizeOfImgWithArch + + if (arch) { + Object.keys(images).forEach(ver => { + if (ver === version) { + sizeOfImgWithArch = getImageSizeForArch(images[ver], arch) + return { size: sizeOfImgWithArch } + } + }) + + if (sizeOfImgWithArch) { + return { size: sizeOfImgWithArch } + } else { + throw new NotFound({ prettyMessage: 'architecture not found' }) + } + } else { return { size: matches[version] } + } + } + + getSizeFromTag(data, arch) { + // If the tag is specified, and the architecture is supplied by the user, + // check if any of the returned images has an architecture matching the arch parameter supplied by the user. + // If yes, return the size of the image with this arch. + // If no, throw the `NotFound` error. + // If no architecture is supplied by the user, return the value of the `full_size` from the response (the image with the `latest` tag). + if (arch) { + return { size: getImageSizeForArch(data.images, arch) } } else { return { size: data.full_size } } } - async handle({ user, repo, tag }, { sort }) { + transform({ tag, sort, data, arch }) { + if (!tag && sort === 'date') { + return this.getSizeFromImageByLatestDate(data, arch) + } else if (!tag && sort === 'semver') { + return this.getSizeFromImageByLatestSemver(data, arch) + } else { + return this.getSizeFromTag(data, arch) + } + } + + async handle({ user, repo, tag }, { sort, arch }) { let data if (!tag && sort === 'date') { @@ -111,7 +202,7 @@ export default class DockerSize extends BaseJsonService { data = await this.fetch({ user, repo, tag }) } - const { size } = await this.transform({ tag, sort, data }) + const { size } = await this.transform({ tag, sort, data, arch }) return this.constructor.render({ size }) } } diff --git a/services/docker/docker-size.spec.js b/services/docker/docker-size.spec.js index 946e0d161b..66c9c4270a 100644 --- a/services/docker/docker-size.spec.js +++ b/services/docker/docker-size.spec.js @@ -3,40 +3,99 @@ import DockerSize from './docker-size.service.js' import { sizeDataNoTagSemVerSort } from './docker-fixtures.js' describe('DockerSize', function () { - test(DockerSize.prototype.transform, () => { - given({ - tag: '', - sort: 'date', - data: { results: [{ name: 'next', full_size: 219939484 }] }, - }).expect({ + test(DockerSize.prototype.getSizeFromImageByLatestDate, () => { + given( + { + count: 0, + results: [], + }, + 'amd64' + ).expectError('Not Found: repository not found') + given( + { + count: 1, + results: [ + { + full_size: 300000000, + name: 'next', + images: [{ architecture: 'amd64', size: 219939484 }], + }, + ], + }, + 'amd64' + ).expect({ size: 219939484, }) given({ - tag: '', - sort: 'date', - data: { + count: 1, + results: [ + { + full_size: 300000000, + name: 'next', + images: [ + { architecture: 'amd64', size: 219939484 }, + { architecture: 'arm64', size: 200000000 }, + ], + }, + ], + }).expect({ + size: 300000000, + }) + given( + { + count: 1, results: [ - { name: 'latest', full_size: 74661264 }, - { name: 'arm64v8-latest', full_size: 76310416 }, - { name: 'arm32v7-latest', full_size: 68001970 }, - { name: 'amd64-latest', full_size: 74661264 }, + { + full_size: 300000000, + name: 'next', + images: [ + { architecture: 'amd64', size: 219939484 }, + { architecture: 'arm64', size: 200000000 }, + ], + }, ], }, - }).expect({ - size: 74661264, + 'arm64777' + ).expectError('Not Found: architecture not found') + }) + + test(DockerSize.prototype.getSizeFromTag, () => { + given( + { + full_size: 300000000, + name: 'next', + images: [{ architecture: 'amd64', size: 219939484 }], + }, + 'amd64' + ).expect({ + size: 219939484, }) given({ - tag: '', - sort: 'semver', - data: sizeDataNoTagSemVerSort, + full_size: 300000000, + name: 'next', + images: [{ architecture: 'amd64', size: 219939484 }], }).expect({ - size: 13448411, + size: 300000000, }) - given({ - tag: 'latest', - data: { name: 'latest', full_size: 13448411 }, - }).expect({ - size: 13448411, + given( + { + full_size: 300000000, + name: 'next', + images: [{ architecture: 'amd64', size: 219939484 }], + }, + 'arm64777' + ).expectError('Not Found: architecture not found') + }) + + test(DockerSize.prototype.getSizeFromImageByLatestSemver, () => { + given(sizeDataNoTagSemVerSort, 'amd64').expect({ + size: 220000000, }) + given(sizeDataNoTagSemVerSort).expect({ + size: 400000000, + }) + given(sizeDataNoTagSemVerSort, 'nonexistentArch').expectError( + 'Not Found: architecture not found' + ) }) }) diff --git a/services/docker/docker-size.tester.js b/services/docker/docker-size.tester.js index c476a773f3..f6b9014bfa 100644 --- a/services/docker/docker-size.tester.js +++ b/services/docker/docker-size.tester.js @@ -9,6 +9,13 @@ t.create('docker image size (valid, library)') message: isFileSize, }) +t.create('docker image size (valid, library, arch parameter )') + .get('/_/mysql.json?arch=amd64') + .expectBadge({ + label: 'image size', + message: isFileSize, + }) + t.create('docker image size (valid, library with tag)') .get('/_/alpine/latest.json') .expectBadge({ @@ -41,5 +48,19 @@ t.create('docker image size (invalid, unknown repository)') .get('/_/not-a-real-repo.json') .expectBadge({ label: 'image size', - message: 'repository not found', + message: 'repository or tag not found', + }) + +t.create('docker image size (invalid, wrong sorting method)') + .get('/jrottenberg/ffmpeg/3.2-alpine.json?sort=daterrr') + .expectBadge({ + label: 'image size', + message: 'invalid query parameter: sort', + }) + +t.create('docker image size (invalid, nonexisting arch)') + .get('/jrottenberg/ffmpeg/3.2-alpine.json?arch=nonexistingArch') + .expectBadge({ + label: 'image size', + message: 'invalid query parameter: arch', }) diff --git a/services/docker/docker-version.service.js b/services/docker/docker-version.service.js index 00bb00e985..594c8e9647 100644 --- a/services/docker/docker-version.service.js +++ b/services/docker/docker-version.service.js @@ -3,6 +3,7 @@ import { nonNegativeInteger } from '../validators.js' import { latest, renderVersionBadge } from '../version.js' import { BaseJsonService, NotFound, InvalidResponse } from '../index.js' import { + archSchema, buildDockerUrl, getDockerHubUser, getMultiPageData, @@ -26,23 +27,7 @@ const buildSchema = Joi.object({ const queryParamSchema = Joi.object({ sort: Joi.string().valid('date', 'semver').default('date'), - arch: Joi.string() - // Valid architecture values: https://golang.org/doc/install/source#environment (GOARCH) - .valid( - 'amd64', - 'arm', - 'arm64', - 's390x', - '386', - 'ppc64', - 'ppc64le', - 'wasm', - 'mips', - 'mipsle', - 'mips64', - 'mips64le' - ) - .default('amd64'), + arch: archSchema.default('amd64'), }).required() export default class DockerVersion extends BaseJsonService {