From 944b4cb66331695925b72ac065b0dd4421a5a402 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Sun, 1 Nov 2026 00:52:42 +0100 Subject: [PATCH] ci(publish): build container images on native arch runners Each distro now builds amd64 on ubuntu-latest and arm64 on ubuntu-24.04-arm instead of emulating arm64 under QEMU. image.sh gains --platform, which pushes one - tag, and --merge, which joins those tags under the release, stable and alias tags with buildx imagetools. The workflow splits into prepare, build and merge jobs; scripts/build/publish.sh follows the same sequence locally. Co-Authored-By: Claude Opus 5.5 (1M context) --- .github/workflows/publish-containers.yml | 83 +++++++++++-- scripts/build/image.sh | 50 +++++++- scripts/build/publish.sh | 16 ++- .../merge-container-images.sh | 15 +++ .../publish-container-images.sh | 10 +- tests/integration/test_build_image_publish.py | 116 ++++++++++++++++++ 6 files changed, 274 insertions(+), 16 deletions(-) create mode 100644 scripts/github/publish-containers/merge-container-images.sh create mode 100644 tests/integration/test_build_image_publish.py diff --git a/.github/workflows/publish-containers.yml b/.github/workflows/publish-containers.yml index 906a7f6..5866c11 100644 --- a/.github/workflows/publish-containers.yml +++ b/.github/workflows/publish-containers.yml @@ -6,13 +6,17 @@ on: types: [completed] jobs: - publish: + prepare: if: ${{ github.event.workflow_run.conclusion == 'success' }} runs-on: ubuntu-latest permissions: contents: read - packages: write + + outputs: + should_publish: ${{ steps.info.outputs.should_publish }} + version: ${{ steps.info.outputs.version }} + is_stable: ${{ steps.info.outputs.is_stable }} steps: - name: Checkout repository (with tags) @@ -36,24 +40,87 @@ jobs: if: steps.branch-check.outputs.is_on_main == 'true' run: bash scripts/github/publish-containers/compute-publish-container-info.sh + build: + needs: prepare + if: ${{ needs.prepare.outputs.should_publish == 'true' }} + runs-on: ${{ matrix.target.runner }} + + permissions: + contents: read + packages: write + + strategy: + fail-fast: false + matrix: + distro: &distros [arch, debian, ubuntu, fedora, centos] + target: + - platform: linux/amd64 + runner: ubuntu-latest + - platform: linux/arm64 + runner: ubuntu-24.04-arm + + steps: + - name: Checkout workflow_run commit + uses: actions/checkout@v7 + with: + ref: ${{ github.event.workflow_run.head_sha }} + - name: Set up Docker Buildx - if: ${{ steps.info.outputs.should_publish == 'true' }} uses: docker/setup-buildx-action@594f3bf4285d9ea8dc53c9a0c9c4092420091003 with: use: true - name: Login to GHCR - if: ${{ steps.info.outputs.should_publish == 'true' }} uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Publish all images - if: ${{ steps.info.outputs.should_publish == 'true' }} + - name: Build and push ${{ matrix.distro }} for ${{ matrix.target.platform }} env: OWNER: ${{ github.repository_owner }} - VERSION: ${{ steps.info.outputs.version }} - IS_STABLE: ${{ steps.info.outputs.is_stable }} + VERSION: ${{ needs.prepare.outputs.version }} + PKGMGR_DISTRO: ${{ matrix.distro }} + PLATFORM: ${{ matrix.target.platform }} run: bash scripts/github/publish-containers/publish-container-images.sh + + merge: + needs: [prepare, build] + if: ${{ !cancelled() && needs.prepare.outputs.should_publish == 'true' }} + runs-on: ubuntu-latest + + permissions: + contents: read + packages: write + + strategy: + fail-fast: false + matrix: + distro: *distros + + steps: + - name: Checkout workflow_run commit + uses: actions/checkout@v7 + with: + ref: ${{ github.event.workflow_run.head_sha }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@594f3bf4285d9ea8dc53c9a0c9c4092420091003 + with: + use: true + + - name: Login to GHCR + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Join both platforms of ${{ matrix.distro }} under the release tags + env: + OWNER: ${{ github.repository_owner }} + VERSION: ${{ needs.prepare.outputs.version }} + IS_STABLE: ${{ needs.prepare.outputs.is_stable }} + PKGMGR_DISTRO: ${{ matrix.distro }} + run: bash scripts/github/publish-containers/merge-container-images.sh diff --git a/scripts/build/image.sh b/scripts/build/image.sh index d3ab267..3c9e832 100755 --- a/scripts/build/image.sh +++ b/scripts/build/image.sh @@ -18,6 +18,8 @@ OWNER="" # e.g. github org/user REPO_PREFIX="pkgmgr" # image base name (pkgmgr) VERSION="" # X.Y.Z (required for --publish) IS_STABLE="false" # "true" -> publish stable tags +PLATFORM="" # e.g. linux/arm64 -> build and push only this platform +MERGE=0 # if 1 -> join the per-platform tags of --version under the publish tags DEFAULT_DISTRO="arch" usage() { @@ -39,9 +41,12 @@ Publish options: --repo-prefix Image base name (default: pkgmgr) --version Version for --publish --stable Whether to publish :stable tags (default: false) + --platform Build and push only this platform; with --publish as - + --merge Join the - tags of all platforms under the publish tags Notes: - --publish implies --push and requires --registry, --owner, and --version. +- --platform requires --push; --merge requires --publish and builds nothing. - Local build (no --push) uses "docker build" and creates a local image like "pkgmgr-arch". EOF } @@ -82,6 +87,12 @@ while [[ $# -gt 0 ]]; do [[ -n "${IS_STABLE}" ]] || { echo "ERROR: --stable requires a value (true|false)"; exit 2; } shift 2 ;; + --platform) + PLATFORM="${2:-}" + [[ -n "${PLATFORM}" ]] || { echo "ERROR: --platform requires a value"; exit 2; } + shift 2 + ;; + --merge) MERGE=1; shift ;; -h|--help) usage; exit 0 ;; *) echo "ERROR: Unknown argument: $1" >&2 @@ -117,6 +128,19 @@ if [[ "${PUBLISH}" == "1" ]]; then [[ -n "${VERSION}" ]] || { echo "ERROR: --publish requires --version"; exit 2; } fi +if [[ -n "${PLATFORM}" ]]; then + [[ "${PUSH}" == "1" ]] || { echo "ERROR: --platform requires --push or --publish"; exit 2; } + [[ ",$(resolve_platforms "$PKGMGR_DISTRO")," == *",${PLATFORM},"* ]] \ + || { echo "ERROR: ${PKGMGR_DISTRO} offers no platform ${PLATFORM}"; exit 2; } +fi + +if [[ "${MERGE}" == "1" ]]; then + [[ "${PUBLISH}" == "1" ]] || { echo "ERROR: --merge requires --publish"; exit 2; } + [[ -z "${PLATFORM}" ]] || { echo "ERROR: --merge and --platform exclude each other"; exit 2; } +fi + +DISTRO_TAG_BASE="${REGISTRY}/${OWNER}/${REPO_PREFIX}-${PKGMGR_DISTRO}" + # Guard: --push without --publish requires fully-qualified --tag if [[ "${PUSH}" == "1" && "${PUBLISH}" != "1" ]]; then if [[ "${IMAGE_TAG}" != */* ]]; then @@ -130,7 +154,7 @@ echo "------------------------------------------------------------" echo "[build] Building image" echo "distro = ${PKGMGR_DISTRO}" echo "BASE_IMAGE = ${BASE_IMAGE}" -echo "platforms = $(resolve_platforms "$PKGMGR_DISTRO")" +echo "platforms = ${PLATFORM:-$(resolve_platforms "$PKGMGR_DISTRO")}" if [[ "${NO_CACHE}" == "1" ]]; then echo "cache = disabled"; fi if [[ "${PUSH}" == "1" ]]; then echo "push = enabled"; fi if [[ "${PUBLISH}" == "1" ]]; then @@ -150,7 +174,7 @@ if [[ "${NO_CACHE}" == "1" ]]; then fi compute_publish_tags() { - local distro_tag_base="${REGISTRY}/${OWNER}/${REPO_PREFIX}-${PKGMGR_DISTRO}" + local distro_tag_base="${DISTRO_TAG_BASE}" local alias_tag_base="" if [[ "${PKGMGR_DISTRO}" == "${DEFAULT_DISTRO}" ]]; then @@ -176,10 +200,26 @@ compute_publish_tags() { printf '%s\n' "${tags[@]}" } -if [[ "${PUSH}" == "1" ]]; then +if [[ "${MERGE}" == "1" ]]; then + merge_args=(docker buildx imagetools create) + while IFS= read -r t; do + merge_args+=(-t "$t") + done < <(compute_publish_tags) + + IFS=',' read -ra platforms <<< "$(resolve_platforms "$PKGMGR_DISTRO")" + for p in "${platforms[@]}"; do + merge_args+=("${DISTRO_TAG_BASE}:${VERSION}-${p#*/}") + done + + echo "[build] Running: ${merge_args[*]}" + "${merge_args[@]}" + docker buildx imagetools inspect "${DISTRO_TAG_BASE}:${VERSION}" +elif [[ "${PUSH}" == "1" ]]; then bx_args=(docker buildx build --push) - if [[ "${PUBLISH}" == "1" ]]; then + if [[ "${PUBLISH}" == "1" && -n "${PLATFORM}" ]]; then + bx_args+=(-t "${DISTRO_TAG_BASE}:${VERSION}-${PLATFORM#*/}") + elif [[ "${PUBLISH}" == "1" ]]; then while IFS= read -r t; do bx_args+=(-t "$t") done < <(compute_publish_tags) @@ -187,7 +227,7 @@ if [[ "${PUSH}" == "1" ]]; then bx_args+=(-t "${IMAGE_TAG}") fi - bx_args+=(--platform "$(resolve_platforms "$PKGMGR_DISTRO")") + bx_args+=(--platform "${PLATFORM:-$(resolve_platforms "$PKGMGR_DISTRO")}") bx_args+=("${build_args[@]}") bx_args+=(.) diff --git a/scripts/build/publish.sh b/scripts/build/publish.sh index df20618..8c23239 100755 --- a/scripts/build/publish.sh +++ b/scripts/build/publish.sh @@ -14,6 +14,9 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +# shellcheck source=./scripts/build/base.sh +source "${SCRIPT_DIR}/base.sh" + REGISTRY="${REGISTRY:-ghcr.io}" IS_STABLE="${IS_STABLE:-false}" DISTROS="${DISTROS:-arch debian ubuntu fedora centos}" @@ -33,12 +36,23 @@ for d in ${DISTROS}; do echo "[publish] PKGMGR_DISTRO=${d}" echo "============================================================" + IFS=',' read -ra platforms <<< "$(resolve_platforms "${d}")" + for p in "${platforms[@]}"; do + PKGMGR_DISTRO="${d}" bash "${SCRIPT_DIR}/image.sh" \ + --publish \ + --registry "${REGISTRY}" \ + --owner "${OWNER}" \ + --version "${VERSION}" \ + --platform "${p}" + done + PKGMGR_DISTRO="${d}" bash "${SCRIPT_DIR}/image.sh" \ --publish \ --registry "${REGISTRY}" \ --owner "${OWNER}" \ --version "${VERSION}" \ - --stable "${IS_STABLE}" + --stable "${IS_STABLE}" \ + --merge done echo diff --git a/scripts/github/publish-containers/merge-container-images.sh b/scripts/github/publish-containers/merge-container-images.sh new file mode 100644 index 0000000..2bdf534 --- /dev/null +++ b/scripts/github/publish-containers/merge-container-images.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +set -euo pipefail + +: "${OWNER:?OWNER must be set}" +: "${VERSION:?VERSION must be set}" +: "${IS_STABLE:?IS_STABLE must be set}" +: "${PKGMGR_DISTRO:?PKGMGR_DISTRO must be set}" + +bash scripts/build/image.sh \ + --publish \ + --registry ghcr.io \ + --owner "${OWNER}" \ + --version "${VERSION}" \ + --stable "${IS_STABLE}" \ + --merge diff --git a/scripts/github/publish-containers/publish-container-images.sh b/scripts/github/publish-containers/publish-container-images.sh index d78d56e..15c72d9 100644 --- a/scripts/github/publish-containers/publish-container-images.sh +++ b/scripts/github/publish-containers/publish-container-images.sh @@ -3,6 +3,12 @@ set -euo pipefail : "${OWNER:?OWNER must be set}" : "${VERSION:?VERSION must be set}" -: "${IS_STABLE:?IS_STABLE must be set}" +: "${PKGMGR_DISTRO:?PKGMGR_DISTRO must be set}" +: "${PLATFORM:?PLATFORM must be set}" -bash scripts/build/publish.sh +bash scripts/build/image.sh \ + --publish \ + --registry ghcr.io \ + --owner "${OWNER}" \ + --version "${VERSION}" \ + --platform "${PLATFORM}" diff --git a/tests/integration/test_build_image_publish.py b/tests/integration/test_build_image_publish.py new file mode 100644 index 0000000..e80d072 --- /dev/null +++ b/tests/integration/test_build_image_publish.py @@ -0,0 +1,116 @@ +from __future__ import annotations + +import os +import subprocess +import tempfile +import unittest +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[2] +BASE = "--build-arg BASE_IMAGE=ghcr.io/kevinveenbirkenbach/base-{distro}:latest ." + + +class TestBuildImagePublish(unittest.TestCase): + def setUp(self) -> None: + self._tmp = tempfile.TemporaryDirectory() + tmp = Path(self._tmp.name) + self.log = tmp / "docker.log" + fake = tmp / "docker" + fake.write_text('#!/usr/bin/env bash\necho "$*" >> "$FAKE_DOCKER_LOG"\n') + fake.chmod(0o755) + self.env = { + k: v for k, v in os.environ.items() if not k.startswith("BASE_IMAGES_") + } + self.env["PATH"] = f"{tmp}{os.pathsep}{os.environ['PATH']}" + self.env["FAKE_DOCKER_LOG"] = str(self.log) + + def tearDown(self) -> None: + self._tmp.cleanup() + + def _run(self, script: str, distro: str, *args: str, **env: str): + result = subprocess.run( + ["bash", f"scripts/build/{script}", *args], + check=False, + cwd=REPO_ROOT, + env={**self.env, "PKGMGR_DISTRO": distro, **env}, + capture_output=True, + text=True, + ) + calls = self.log.read_text().splitlines() if self.log.exists() else [] + return result, calls + + def _publish(self, distro: str, *args: str): + return self._run( + "image.sh", + distro, + "--publish", + "--registry", + "ghcr.io", + "--owner", + "o", + "--version", + "2.1.2", + *args, + ) + + def test_platform_pushes_only_the_arch_tag(self) -> None: + result, calls = self._publish("debian", "--platform", "linux/arm64") + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual( + calls, + [ + ( + "buildx build --push -t ghcr.io/o/pkgmgr-debian:2.1.2-arm64 " + "--platform linux/arm64 " + ) + + BASE.format(distro="debian") + ], + ) + + def test_merge_joins_arch_tags_under_release_and_alias_tags(self) -> None: + result, calls = self._publish("arch", "--stable", "true", "--merge") + + self.assertEqual(result.returncode, 0, result.stderr) + tags = " ".join( + f"-t ghcr.io/o/{image}:{tag}" + for image in ("pkgmgr-arch", "pkgmgr") + for tag in ("latest", "2.1.2", "stable") + ) + self.assertEqual( + calls, + [ + ( + f"buildx imagetools create {tags} " + "ghcr.io/o/pkgmgr-arch:2.1.2-amd64 ghcr.io/o/pkgmgr-arch:2.1.2-arm64" + ), + "buildx imagetools inspect ghcr.io/o/pkgmgr-arch:2.1.2", + ], + ) + + def test_unknown_platform_and_merge_without_publish_are_refused(self) -> None: + unknown, _ = self._publish("debian", "--platform", "linux/s390x") + no_publish, calls = self._run("image.sh", "debian", "--merge") + + self.assertEqual(unknown.returncode, 2) + self.assertEqual(no_publish.returncode, 2) + self.assertEqual(calls, []) + + def test_publish_builds_each_platform_then_merges(self) -> None: + result, calls = self._run( + "publish.sh", "fedora", OWNER="o", VERSION="2.1.2", DISTROS="fedora" + ) + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual( + [c.split(" -t ")[0] for c in calls], + ["buildx build --push", "buildx build --push", "buildx imagetools create"] + + ["buildx imagetools inspect ghcr.io/o/pkgmgr-fedora:2.1.2"], + ) + self.assertIn("--platform linux/amd64", calls[0]) + self.assertIn("--platform linux/arm64", calls[1]) + self.assertNotIn(":stable", calls[2]) + + +if __name__ == "__main__": + unittest.main()