ci(publish): build container images on native arch runners
Each distro now builds amd64 on ubuntu-latest and arm64 on ubuntu-24.04-arm instead of emulating arm64 under QEMU. image.sh gains --platform, which pushes one <version>-<arch> tag, and --merge, which joins those tags under the release, stable and alias tags with buildx imagetools. The workflow splits into prepare, build and merge jobs; scripts/build/publish.sh follows the same sequence locally. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
919e6a6434
commit
944b4cb663
6 files changed
+274
-16
No files matched your search
@@ -6,13 +6,17 @@ on:
|
||||
types: [completed]
|
||||
|
||||
jobs:
|
||||
publish:
|
||||
prepare:
|
||||
if: ${{ github.event.workflow_run.conclusion == 'success' }}
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
|
||||
outputs:
|
||||
should_publish: ${{ steps.info.outputs.should_publish }}
|
||||
version: ${{ steps.info.outputs.version }}
|
||||
is_stable: ${{ steps.info.outputs.is_stable }}
|
||||
|
||||
steps:
|
||||
- name: Checkout repository (with tags)
|
||||
@@ -36,24 +40,87 @@ jobs:
|
||||
if: steps.branch-check.outputs.is_on_main == 'true'
|
||||
run: bash scripts/github/publish-containers/compute-publish-container-info.sh
|
||||
|
||||
build:
|
||||
needs: prepare
|
||||
if: ${{ needs.prepare.outputs.should_publish == 'true' }}
|
||||
runs-on: ${{ matrix.target.runner }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
distro: &distros [arch, debian, ubuntu, fedora, centos]
|
||||
target:
|
||||
- platform: linux/amd64
|
||||
runner: ubuntu-latest
|
||||
- platform: linux/arm64
|
||||
runner: ubuntu-24.04-arm
|
||||
|
||||
steps:
|
||||
- name: Checkout workflow_run commit
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
ref: ${{ github.event.workflow_run.head_sha }}
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
if: ${{ steps.info.outputs.should_publish == 'true' }}
|
||||
uses: docker/setup-buildx-action@594f3bf4285d9ea8dc53c9a0c9c4092420091003
|
||||
with:
|
||||
use: true
|
||||
|
||||
- name: Login to GHCR
|
||||
if: ${{ steps.info.outputs.should_publish == 'true' }}
|
||||
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Publish all images
|
||||
if: ${{ steps.info.outputs.should_publish == 'true' }}
|
||||
- name: Build and push ${{ matrix.distro }} for ${{ matrix.target.platform }}
|
||||
env:
|
||||
OWNER: ${{ github.repository_owner }}
|
||||
VERSION: ${{ steps.info.outputs.version }}
|
||||
IS_STABLE: ${{ steps.info.outputs.is_stable }}
|
||||
VERSION: ${{ needs.prepare.outputs.version }}
|
||||
PKGMGR_DISTRO: ${{ matrix.distro }}
|
||||
PLATFORM: ${{ matrix.target.platform }}
|
||||
run: bash scripts/github/publish-containers/publish-container-images.sh
|
||||
|
||||
merge:
|
||||
needs: [prepare, build]
|
||||
if: ${{ !cancelled() && needs.prepare.outputs.should_publish == 'true' }}
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
distro: *distros
|
||||
|
||||
steps:
|
||||
- name: Checkout workflow_run commit
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
ref: ${{ github.event.workflow_run.head_sha }}
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@594f3bf4285d9ea8dc53c9a0c9c4092420091003
|
||||
with:
|
||||
use: true
|
||||
|
||||
- name: Login to GHCR
|
||||
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Join both platforms of ${{ matrix.distro }} under the release tags
|
||||
env:
|
||||
OWNER: ${{ github.repository_owner }}
|
||||
VERSION: ${{ needs.prepare.outputs.version }}
|
||||
IS_STABLE: ${{ needs.prepare.outputs.is_stable }}
|
||||
PKGMGR_DISTRO: ${{ matrix.distro }}
|
||||
run: bash scripts/github/publish-containers/merge-container-images.sh
|
||||
+45
-5
@@ -18,6 +18,8 @@ OWNER="" # e.g. github org/user
|
||||
REPO_PREFIX="pkgmgr" # image base name (pkgmgr)
|
||||
VERSION="" # X.Y.Z (required for --publish)
|
||||
IS_STABLE="false" # "true" -> publish stable tags
|
||||
PLATFORM="" # e.g. linux/arm64 -> build and push only this platform
|
||||
MERGE=0 # if 1 -> join the per-platform tags of --version under the publish tags
|
||||
DEFAULT_DISTRO="arch"
|
||||
|
||||
usage() {
|
||||
@@ -39,9 +41,12 @@ Publish options:
|
||||
--repo-prefix <name> Image base name (default: pkgmgr)
|
||||
--version <X.Y.Z> Version for --publish
|
||||
--stable <true|false> Whether to publish :stable tags (default: false)
|
||||
--platform <os/arch> Build and push only this platform; with --publish as <version>-<arch>
|
||||
--merge Join the <version>-<arch> tags of all platforms under the publish tags
|
||||
|
||||
Notes:
|
||||
- --publish implies --push and requires --registry, --owner, and --version.
|
||||
- --platform requires --push; --merge requires --publish and builds nothing.
|
||||
- Local build (no --push) uses "docker build" and creates a local image like "pkgmgr-arch".
|
||||
EOF
|
||||
}
|
||||
@@ -82,6 +87,12 @@ while [[ $# -gt 0 ]]; do
|
||||
[[ -n "${IS_STABLE}" ]] || { echo "ERROR: --stable requires a value (true|false)"; exit 2; }
|
||||
shift 2
|
||||
;;
|
||||
--platform)
|
||||
PLATFORM="${2:-}"
|
||||
[[ -n "${PLATFORM}" ]] || { echo "ERROR: --platform requires a value"; exit 2; }
|
||||
shift 2
|
||||
;;
|
||||
--merge) MERGE=1; shift ;;
|
||||
-h|--help) usage; exit 0 ;;
|
||||
*)
|
||||
echo "ERROR: Unknown argument: $1" >&2
|
||||
@@ -117,6 +128,19 @@ if [[ "${PUBLISH}" == "1" ]]; then
|
||||
[[ -n "${VERSION}" ]] || { echo "ERROR: --publish requires --version"; exit 2; }
|
||||
fi
|
||||
|
||||
if [[ -n "${PLATFORM}" ]]; then
|
||||
[[ "${PUSH}" == "1" ]] || { echo "ERROR: --platform requires --push or --publish"; exit 2; }
|
||||
[[ ",$(resolve_platforms "$PKGMGR_DISTRO")," == *",${PLATFORM},"* ]] \
|
||||
|| { echo "ERROR: ${PKGMGR_DISTRO} offers no platform ${PLATFORM}"; exit 2; }
|
||||
fi
|
||||
|
||||
if [[ "${MERGE}" == "1" ]]; then
|
||||
[[ "${PUBLISH}" == "1" ]] || { echo "ERROR: --merge requires --publish"; exit 2; }
|
||||
[[ -z "${PLATFORM}" ]] || { echo "ERROR: --merge and --platform exclude each other"; exit 2; }
|
||||
fi
|
||||
|
||||
DISTRO_TAG_BASE="${REGISTRY}/${OWNER}/${REPO_PREFIX}-${PKGMGR_DISTRO}"
|
||||
|
||||
# Guard: --push without --publish requires fully-qualified --tag
|
||||
if [[ "${PUSH}" == "1" && "${PUBLISH}" != "1" ]]; then
|
||||
if [[ "${IMAGE_TAG}" != */* ]]; then
|
||||
@@ -130,7 +154,7 @@ echo "------------------------------------------------------------"
|
||||
echo "[build] Building image"
|
||||
echo "distro = ${PKGMGR_DISTRO}"
|
||||
echo "BASE_IMAGE = ${BASE_IMAGE}"
|
||||
echo "platforms = $(resolve_platforms "$PKGMGR_DISTRO")"
|
||||
echo "platforms = ${PLATFORM:-$(resolve_platforms "$PKGMGR_DISTRO")}"
|
||||
if [[ "${NO_CACHE}" == "1" ]]; then echo "cache = disabled"; fi
|
||||
if [[ "${PUSH}" == "1" ]]; then echo "push = enabled"; fi
|
||||
if [[ "${PUBLISH}" == "1" ]]; then
|
||||
@@ -150,7 +174,7 @@ if [[ "${NO_CACHE}" == "1" ]]; then
|
||||
fi
|
||||
|
||||
compute_publish_tags() {
|
||||
local distro_tag_base="${REGISTRY}/${OWNER}/${REPO_PREFIX}-${PKGMGR_DISTRO}"
|
||||
local distro_tag_base="${DISTRO_TAG_BASE}"
|
||||
local alias_tag_base=""
|
||||
|
||||
if [[ "${PKGMGR_DISTRO}" == "${DEFAULT_DISTRO}" ]]; then
|
||||
@@ -176,10 +200,26 @@ compute_publish_tags() {
|
||||
printf '%s\n' "${tags[@]}"
|
||||
}
|
||||
|
||||
if [[ "${PUSH}" == "1" ]]; then
|
||||
if [[ "${MERGE}" == "1" ]]; then
|
||||
merge_args=(docker buildx imagetools create)
|
||||
while IFS= read -r t; do
|
||||
merge_args+=(-t "$t")
|
||||
done < <(compute_publish_tags)
|
||||
|
||||
IFS=',' read -ra platforms <<< "$(resolve_platforms "$PKGMGR_DISTRO")"
|
||||
for p in "${platforms[@]}"; do
|
||||
merge_args+=("${DISTRO_TAG_BASE}:${VERSION}-${p#*/}")
|
||||
done
|
||||
|
||||
echo "[build] Running: ${merge_args[*]}"
|
||||
"${merge_args[@]}"
|
||||
docker buildx imagetools inspect "${DISTRO_TAG_BASE}:${VERSION}"
|
||||
elif [[ "${PUSH}" == "1" ]]; then
|
||||
bx_args=(docker buildx build --push)
|
||||
|
||||
if [[ "${PUBLISH}" == "1" ]]; then
|
||||
if [[ "${PUBLISH}" == "1" && -n "${PLATFORM}" ]]; then
|
||||
bx_args+=(-t "${DISTRO_TAG_BASE}:${VERSION}-${PLATFORM#*/}")
|
||||
elif [[ "${PUBLISH}" == "1" ]]; then
|
||||
while IFS= read -r t; do
|
||||
bx_args+=(-t "$t")
|
||||
done < <(compute_publish_tags)
|
||||
@@ -187,7 +227,7 @@ if [[ "${PUSH}" == "1" ]]; then
|
||||
bx_args+=(-t "${IMAGE_TAG}")
|
||||
fi
|
||||
|
||||
bx_args+=(--platform "$(resolve_platforms "$PKGMGR_DISTRO")")
|
||||
bx_args+=(--platform "${PLATFORM:-$(resolve_platforms "$PKGMGR_DISTRO")}")
|
||||
bx_args+=("${build_args[@]}")
|
||||
bx_args+=(.)
|
||||
|
||||
|
||||
@@ -14,6 +14,9 @@ set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
|
||||
# shellcheck source=./scripts/build/base.sh
|
||||
source "${SCRIPT_DIR}/base.sh"
|
||||
|
||||
REGISTRY="${REGISTRY:-ghcr.io}"
|
||||
IS_STABLE="${IS_STABLE:-false}"
|
||||
DISTROS="${DISTROS:-arch debian ubuntu fedora centos}"
|
||||
@@ -33,12 +36,23 @@ for d in ${DISTROS}; do
|
||||
echo "[publish] PKGMGR_DISTRO=${d}"
|
||||
echo "============================================================"
|
||||
|
||||
IFS=',' read -ra platforms <<< "$(resolve_platforms "${d}")"
|
||||
for p in "${platforms[@]}"; do
|
||||
PKGMGR_DISTRO="${d}" bash "${SCRIPT_DIR}/image.sh" \
|
||||
--publish \
|
||||
--registry "${REGISTRY}" \
|
||||
--owner "${OWNER}" \
|
||||
--version "${VERSION}" \
|
||||
--stable "${IS_STABLE}"
|
||||
--platform "${p}"
|
||||
done
|
||||
|
||||
PKGMGR_DISTRO="${d}" bash "${SCRIPT_DIR}/image.sh" \
|
||||
--publish \
|
||||
--registry "${REGISTRY}" \
|
||||
--owner "${OWNER}" \
|
||||
--version "${VERSION}" \
|
||||
--stable "${IS_STABLE}" \
|
||||
--merge
|
||||
done
|
||||
|
||||
echo
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
: "${OWNER:?OWNER must be set}"
|
||||
: "${VERSION:?VERSION must be set}"
|
||||
: "${IS_STABLE:?IS_STABLE must be set}"
|
||||
: "${PKGMGR_DISTRO:?PKGMGR_DISTRO must be set}"
|
||||
|
||||
bash scripts/build/image.sh \
|
||||
--publish \
|
||||
--registry ghcr.io \
|
||||
--owner "${OWNER}" \
|
||||
--version "${VERSION}" \
|
||||
--stable "${IS_STABLE}" \
|
||||
--merge
|
||||
@@ -3,6 +3,12 @@ set -euo pipefail
|
||||
|
||||
: "${OWNER:?OWNER must be set}"
|
||||
: "${VERSION:?VERSION must be set}"
|
||||
: "${IS_STABLE:?IS_STABLE must be set}"
|
||||
: "${PKGMGR_DISTRO:?PKGMGR_DISTRO must be set}"
|
||||
: "${PLATFORM:?PLATFORM must be set}"
|
||||
|
||||
bash scripts/build/publish.sh
|
||||
bash scripts/build/image.sh \
|
||||
--publish \
|
||||
--registry ghcr.io \
|
||||
--owner "${OWNER}" \
|
||||
--version "${VERSION}" \
|
||||
--platform "${PLATFORM}"
|
||||
@@ -0,0 +1,116 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
BASE = "--build-arg BASE_IMAGE=ghcr.io/kevinveenbirkenbach/base-{distro}:latest ."
|
||||
|
||||
|
||||
class TestBuildImagePublish(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self._tmp = tempfile.TemporaryDirectory()
|
||||
tmp = Path(self._tmp.name)
|
||||
self.log = tmp / "docker.log"
|
||||
fake = tmp / "docker"
|
||||
fake.write_text('#!/usr/bin/env bash\necho "$*" >> "$FAKE_DOCKER_LOG"\n')
|
||||
fake.chmod(0o755)
|
||||
self.env = {
|
||||
k: v for k, v in os.environ.items() if not k.startswith("BASE_IMAGES_")
|
||||
}
|
||||
self.env["PATH"] = f"{tmp}{os.pathsep}{os.environ['PATH']}"
|
||||
self.env["FAKE_DOCKER_LOG"] = str(self.log)
|
||||
|
||||
def tearDown(self) -> None:
|
||||
self._tmp.cleanup()
|
||||
|
||||
def _run(self, script: str, distro: str, *args: str, **env: str):
|
||||
result = subprocess.run(
|
||||
["bash", f"scripts/build/{script}", *args],
|
||||
check=False,
|
||||
cwd=REPO_ROOT,
|
||||
env={**self.env, "PKGMGR_DISTRO": distro, **env},
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
calls = self.log.read_text().splitlines() if self.log.exists() else []
|
||||
return result, calls
|
||||
|
||||
def _publish(self, distro: str, *args: str):
|
||||
return self._run(
|
||||
"image.sh",
|
||||
distro,
|
||||
"--publish",
|
||||
"--registry",
|
||||
"ghcr.io",
|
||||
"--owner",
|
||||
"o",
|
||||
"--version",
|
||||
"2.1.2",
|
||||
*args,
|
||||
)
|
||||
|
||||
def test_platform_pushes_only_the_arch_tag(self) -> None:
|
||||
result, calls = self._publish("debian", "--platform", "linux/arm64")
|
||||
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertEqual(
|
||||
calls,
|
||||
[
|
||||
(
|
||||
"buildx build --push -t ghcr.io/o/pkgmgr-debian:2.1.2-arm64 "
|
||||
"--platform linux/arm64 "
|
||||
)
|
||||
+ BASE.format(distro="debian")
|
||||
],
|
||||
)
|
||||
|
||||
def test_merge_joins_arch_tags_under_release_and_alias_tags(self) -> None:
|
||||
result, calls = self._publish("arch", "--stable", "true", "--merge")
|
||||
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
tags = " ".join(
|
||||
f"-t ghcr.io/o/{image}:{tag}"
|
||||
for image in ("pkgmgr-arch", "pkgmgr")
|
||||
for tag in ("latest", "2.1.2", "stable")
|
||||
)
|
||||
self.assertEqual(
|
||||
calls,
|
||||
[
|
||||
(
|
||||
f"buildx imagetools create {tags} "
|
||||
"ghcr.io/o/pkgmgr-arch:2.1.2-amd64 ghcr.io/o/pkgmgr-arch:2.1.2-arm64"
|
||||
),
|
||||
"buildx imagetools inspect ghcr.io/o/pkgmgr-arch:2.1.2",
|
||||
],
|
||||
)
|
||||
|
||||
def test_unknown_platform_and_merge_without_publish_are_refused(self) -> None:
|
||||
unknown, _ = self._publish("debian", "--platform", "linux/s390x")
|
||||
no_publish, calls = self._run("image.sh", "debian", "--merge")
|
||||
|
||||
self.assertEqual(unknown.returncode, 2)
|
||||
self.assertEqual(no_publish.returncode, 2)
|
||||
self.assertEqual(calls, [])
|
||||
|
||||
def test_publish_builds_each_platform_then_merges(self) -> None:
|
||||
result, calls = self._run(
|
||||
"publish.sh", "fedora", OWNER="o", VERSION="2.1.2", DISTROS="fedora"
|
||||
)
|
||||
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertEqual(
|
||||
[c.split(" -t ")[0] for c in calls],
|
||||
["buildx build --push", "buildx build --push", "buildx imagetools create"]
|
||||
+ ["buildx imagetools inspect ghcr.io/o/pkgmgr-fedora:2.1.2"],
|
||||
)
|
||||
self.assertIn("--platform linux/amd64", calls[0])
|
||||
self.assertIn("--platform linux/arm64", calls[1])
|
||||
self.assertNotIn(":stable", calls[2])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in new issue
Block a user