diff --git a/.github/workflows/publish-containers.yml b/.github/workflows/publish-containers.yml index 5866c11..5f4ec21 100644 --- a/.github/workflows/publish-containers.yml +++ b/.github/workflows/publish-containers.yml @@ -4,19 +4,20 @@ on: workflow_run: workflows: ["Mark stable commit"] types: [completed] + schedule: + - cron: "0 3 * * *" + workflow_dispatch: jobs: prepare: - if: ${{ github.event.workflow_run.conclusion == 'success' }} + if: ${{ github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success' }} runs-on: ubuntu-latest permissions: contents: read outputs: - should_publish: ${{ steps.info.outputs.should_publish }} - version: ${{ steps.info.outputs.version }} - is_stable: ${{ steps.info.outputs.is_stable }} + releases: ${{ steps.info.outputs.releases }} steps: - name: Checkout repository (with tags) @@ -24,25 +25,16 @@ jobs: with: fetch-depth: 0 - - name: Checkout workflow_run commit and refresh tags + - name: Resolve the releases this run publishes + id: info env: WORKFLOW_RUN_SHA: ${{ github.event.workflow_run.head_sha }} - run: bash scripts/github/publish-containers/checkout-workflow-run-commit.sh - - - name: Check whether tagged commit is on main - id: branch-check - env: - TARGET_SHA: ${{ github.event.workflow_run.head_sha }} - run: bash scripts/github/common/check-tagged-commit-on-main.sh - - - name: Compute version and stable flag - id: info - if: steps.branch-check.outputs.is_on_main == 'true' run: bash scripts/github/publish-containers/compute-publish-container-info.sh build: needs: prepare - if: ${{ needs.prepare.outputs.should_publish == 'true' }} + if: ${{ needs.prepare.result == 'success' && needs.prepare.outputs.releases != '[]' }} + name: Build ${{ matrix.distro }} ${{ matrix.release.version }} for ${{ matrix.target.platform }} runs-on: ${{ matrix.target.runner }} permissions: @@ -58,12 +50,13 @@ jobs: runner: ubuntu-latest - platform: linux/arm64 runner: ubuntu-24.04-arm + release: ${{ fromJSON(needs.prepare.outputs.releases) }} steps: - - name: Checkout workflow_run commit + - name: Checkout ${{ matrix.release.version }} uses: actions/checkout@v7 with: - ref: ${{ github.event.workflow_run.head_sha }} + ref: ${{ matrix.release.sha }} - name: Set up Docker Buildx uses: docker/setup-buildx-action@594f3bf4285d9ea8dc53c9a0c9c4092420091003 @@ -77,17 +70,18 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Build and push ${{ matrix.distro }} for ${{ matrix.target.platform }} + - name: Build and push ${{ matrix.distro }} ${{ matrix.release.version }} for ${{ matrix.target.platform }} env: OWNER: ${{ github.repository_owner }} - VERSION: ${{ needs.prepare.outputs.version }} + VERSION: ${{ matrix.release.version }} PKGMGR_DISTRO: ${{ matrix.distro }} PLATFORM: ${{ matrix.target.platform }} run: bash scripts/github/publish-containers/publish-container-images.sh merge: needs: [prepare, build] - if: ${{ !cancelled() && needs.prepare.outputs.should_publish == 'true' }} + if: ${{ !cancelled() && needs.prepare.result == 'success' && needs.prepare.outputs.releases != '[]' }} + name: Merge ${{ matrix.distro }} ${{ matrix.release.version }} runs-on: ubuntu-latest permissions: @@ -98,12 +92,13 @@ jobs: fail-fast: false matrix: distro: *distros + release: ${{ fromJSON(needs.prepare.outputs.releases) }} steps: - - name: Checkout workflow_run commit + - name: Checkout ${{ matrix.release.version }} uses: actions/checkout@v7 with: - ref: ${{ github.event.workflow_run.head_sha }} + ref: ${{ matrix.release.sha }} - name: Set up Docker Buildx uses: docker/setup-buildx-action@594f3bf4285d9ea8dc53c9a0c9c4092420091003 @@ -117,10 +112,10 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Join both platforms of ${{ matrix.distro }} under the release tags + - name: Join both platforms of ${{ matrix.distro }} ${{ matrix.release.version }} under its tags env: OWNER: ${{ github.repository_owner }} - VERSION: ${{ needs.prepare.outputs.version }} - IS_STABLE: ${{ needs.prepare.outputs.is_stable }} + VERSION: ${{ matrix.release.version }} + ALIASES: ${{ matrix.release.aliases }} PKGMGR_DISTRO: ${{ matrix.distro }} run: bash scripts/github/publish-containers/merge-container-images.sh diff --git a/README.md b/README.md index e5d7e0c..6a02dab 100644 --- a/README.md +++ b/README.md @@ -181,16 +181,50 @@ is a manifest list covering `linux/amd64` and `linux/arm64`. All images live under `ghcr.io/kevinveenbirkenbach/`: -| Tag | Contains | -| --------------------------- | ---------------------------------- | -| `pkgmgr-:latest` | newest build for that distribution | -| `pkgmgr-:` | one release | -| `pkgmgr-:stable` | newest release marked stable | +| Tag | Contains | +| ------------------------------------- | --------------------------------------------------------- | +| `pkgmgr-:latest` | newest build of the newest release on `main` | +| `pkgmgr-:stable` | newest build of the release the git tag `stable` marks | +| `pkgmgr-:` | newest build of the newest release of that major | +| `pkgmgr-:.` | newest build of the newest release of that minor | +| `pkgmgr-:` | newest build of that release | +| `pkgmgr-:.` | one build, never moves | `` is one of `arch`, `debian`, `ubuntu`, `fedora`, `centos`. The Arch image is published without the suffix as well, so `pkgmgr:stable` and `pkgmgr-arch:stable` are the same image. +`` is the PKGMGR release and `` the build time, so +`2.3.0.1767225600` is a full version an updater can order and pin. Every tag of +one build points at the same image. + +`stable` moves only once "Mark stable commit" has moved the git tag `stable`, +which it does after every test of that release passed. Every day at 03:00 UTC, +after the daily 00:00 UTC build of base-images, and on demand through the +workflow's manual trigger, two releases are rebuilt on the newest base image: + +- the newest `vX.Y.Z` release on `main`, moving `latest`, its prefixes and + ``, and adding `.`; +- the release the git tag `stable` marks, when that is an older one, moving + `stable` and ``, adding its own `.`, and moving a + prefix only while it is still the newest release of that prefix. + +With newest `2.4.0` and stable `2.3.0`, `2.4.0` takes `latest`, `2` and `2.4`, +and `2.3.0` takes `stable` and `2.3`. A security update in base-images thereby +reaches whoever pinned a major, a minor, a release or follows `stable`. + +Each release is compared with its own published ``. A rebuild whose +installed packages equal it pushes nothing, since a new timestamp tag would make +every pin bump for no change; a tag of that release that does not show the +published build yet, such as `stable` right after "Mark stable commit", is moved +without a new timestamp. + +A rebuild runs the build scripts of the release it rebuilds, not those of +`main`, so the skip and the tag rules above only hold for releases tagged after +this scheme landed. The scripts of an earlier release know no aliases: its merge +step fails and leaves its tags where they are, until a newer release replaces +it as the newest and the stable one. + For a clean environment that carries the system dependencies but no PKGMGR, pull `base-` from [base-images](https://github.com/kevinveenbirkenbach/base-images) directly. diff --git a/scripts/build/base.sh b/scripts/build/base.sh index f5dbd02..f5736be 100755 --- a/scripts/build/base.sh +++ b/scripts/build/base.sh @@ -30,3 +30,24 @@ resolve_platforms() { *) echo "ERROR: Unknown distro '$PKGMGR_DISTRO'" >&2; exit 1 ;; esac } + +package_set_cmd() { + local PKGMGR_DISTRO="$1" + case "$PKGMGR_DISTRO" in + arch|manjaro) echo 'pacman -Q' ;; + debian|ubuntu) echo 'dpkg-query -W' ;; + fedora|centos) echo 'rpm -qa | sort' ;; + *) echo "ERROR: Unknown distro '$PKGMGR_DISTRO'" >&2; exit 1 ;; + esac +} + +version_prefixes() { + local -a parts + IFS=. read -ra parts <<< "$1" + local acc="${parts[0]}" i + echo "${acc}" + for ((i = 1; i < ${#parts[@]}; i++)); do + acc+=".${parts[i]}" + echo "${acc}" + done +} diff --git a/scripts/build/image.sh b/scripts/build/image.sh index 3c9e832..56452ef 100755 --- a/scripts/build/image.sh +++ b/scripts/build/image.sh @@ -12,12 +12,13 @@ NO_CACHE=0 MISSING_ONLY=0 IMAGE_TAG="" # local image name or base tag (without registry) PUSH=0 # if 1 -> use buildx and push (requires docker buildx) -PUBLISH=0 # if 1 -> push with semantic tags (latest/version/stable + arch aliases) +PUBLISH=0 # if 1 -> push , . and the --aliases, plus arch aliases REGISTRY="" # e.g. ghcr.io OWNER="" # e.g. github org/user REPO_PREFIX="pkgmgr" # image base name (pkgmgr) VERSION="" # X.Y.Z (required for --publish) -IS_STABLE="false" # "true" -> publish stable tags +ALIASES="" # moving tags besides , e.g. "latest 2 2.4 stable" +ALIASES_SET=0 # 1 once --aliases was given, also when empty PLATFORM="" # e.g. linux/arm64 -> build and push only this platform MERGE=0 # if 1 -> join the per-platform tags of --version under the publish tags DEFAULT_DISTRO="arch" @@ -35,17 +36,23 @@ Build options: Publish options: --push Push the built image (uses docker buildx build --push) - --publish Publish semantic tags (latest, , optional stable) + arch aliases + --publish Publish , . and every --aliases tag, plus arch + aliases --registry Registry (e.g. ghcr.io) --owner Registry namespace (e.g. \${GITHUB_REPOSITORY_OWNER}) --repo-prefix Image base name (default: pkgmgr) --version Version for --publish - --stable Whether to publish :stable tags (default: false) - --platform Build and push only this platform; with --publish as - - --merge Join the - tags of all platforms under the publish tags + --aliases "" Moving tags besides , space separated, may be empty + (e.g. "latest 2 2.4 stable") + --platform Build and push only this platform; with --publish as -, + skipped when its package set equals the published + --merge Join the - tags of all platforms under the publish tags; + when already joins exactly these images, only the aliases + that do not are moved and no . is added Notes: - --publish implies --push and requires --registry, --owner, and --version. +- --publish without --platform requires --aliases. - --platform requires --push; --merge requires --publish and builds nothing. - Local build (no --push) uses "docker build" and creates a local image like "pkgmgr-arch". EOF @@ -82,9 +89,10 @@ while [[ $# -gt 0 ]]; do [[ -n "${VERSION}" ]] || { echo "ERROR: --version requires a value"; exit 2; } shift 2 ;; - --stable) - IS_STABLE="${2:-}" - [[ -n "${IS_STABLE}" ]] || { echo "ERROR: --stable requires a value (true|false)"; exit 2; } + --aliases) + [[ $# -ge 2 ]] || { echo "ERROR: --aliases requires a value, empty for none"; exit 2; } + ALIASES="$2" + ALIASES_SET=1 shift 2 ;; --platform) @@ -126,6 +134,8 @@ if [[ "${PUBLISH}" == "1" ]]; then [[ -n "${REGISTRY}" ]] || { echo "ERROR: --publish requires --registry"; exit 2; } [[ -n "${OWNER}" ]] || { echo "ERROR: --publish requires --owner"; exit 2; } [[ -n "${VERSION}" ]] || { echo "ERROR: --publish requires --version"; exit 2; } + [[ -n "${PLATFORM}" || "${ALIASES_SET}" == "1" ]] \ + || { echo "ERROR: --publish without --platform requires --aliases"; exit 2; } fi if [[ -n "${PLATFORM}" ]]; then @@ -140,6 +150,7 @@ if [[ "${MERGE}" == "1" ]]; then fi DISTRO_TAG_BASE="${REGISTRY}/${OWNER}/${REPO_PREFIX}-${PKGMGR_DISTRO}" +STAMP="$(date -u +%s)" # Guard: --push without --publish requires fully-qualified --tag if [[ "${PUSH}" == "1" && "${PUBLISH}" != "1" ]]; then @@ -162,7 +173,8 @@ if [[ "${PUBLISH}" == "1" ]]; then echo "registry = ${REGISTRY}" echo "owner = ${OWNER}" echo "version = ${VERSION}" - echo "stable = ${IS_STABLE}" + echo "stamp = ${STAMP}" + echo "aliases = ${ALIASES}" fi echo "------------------------------------------------------------" @@ -173,48 +185,97 @@ if [[ "${NO_CACHE}" == "1" ]]; then build_args+=(--no-cache) fi -compute_publish_tags() { - local distro_tag_base="${DISTRO_TAG_BASE}" - local alias_tag_base="" - +publish_bases() { + echo "${DISTRO_TAG_BASE}" if [[ "${PKGMGR_DISTRO}" == "${DEFAULT_DISTRO}" ]]; then - alias_tag_base="${REGISTRY}/${OWNER}/${REPO_PREFIX}" + echo "${REGISTRY}/${OWNER}/${REPO_PREFIX}" fi +} - local tags=() - tags+=("${distro_tag_base}:latest") - tags+=("${distro_tag_base}:${VERSION}") +moving_tags() { + local base alias + local -a aliases + read -ra aliases <<< "${ALIASES}" + while read -r base; do + echo "${base}:${VERSION}" + for alias in "${aliases[@]}"; do + echo "${base}:${alias}" + done + done < <(publish_bases) +} - if [[ "${IS_STABLE}" == "true" ]]; then - tags+=("${distro_tag_base}:stable") - fi +stamp_tags() { + local base + while read -r base; do + echo "${base}:${VERSION}.${STAMP}" + done < <(publish_bases) +} - if [[ -n "${alias_tag_base}" ]]; then - tags+=("${alias_tag_base}:latest") - tags+=("${alias_tag_base}:${VERSION}") - if [[ "${IS_STABLE}" == "true" ]]; then - tags+=("${alias_tag_base}:stable") - fi - fi +raw_index() { + docker buildx imagetools inspect --raw "$1" 2>/dev/null || true +} - printf '%s\n' "${tags[@]}" +digest_of() { + docker buildx imagetools inspect --format '{{json .Manifest.Digest}}' "$1" } if [[ "${MERGE}" == "1" ]]; then - merge_args=(docker buildx imagetools create) - while IFS= read -r t; do - merge_args+=(-t "$t") - done < <(compute_publish_tags) - + sources=() IFS=',' read -ra platforms <<< "$(resolve_platforms "$PKGMGR_DISTRO")" for p in "${platforms[@]}"; do - merge_args+=("${DISTRO_TAG_BASE}:${VERSION}-${p#*/}") + sources+=("${DISTRO_TAG_BASE}:${VERSION}-${p#*/}") done - echo "[build] Running: ${merge_args[*]}" - "${merge_args[@]}" - docker buildx imagetools inspect "${DISTRO_TAG_BASE}:${VERSION}" + mapfile -t moving < <(moving_tags) + planned="$(docker buildx imagetools create --dry-run -t "${DISTRO_TAG_BASE}:${VERSION}" "${sources[@]}")" + tags=() + if [[ "$(raw_index "${DISTRO_TAG_BASE}:${VERSION}")" == "${planned}" ]]; then + for t in "${moving[@]}"; do + if [[ "$(raw_index "${t}")" != "${planned}" ]]; then + tags+=("${t}") + fi + done + else + mapfile -t stamps < <(stamp_tags) + tags=("${moving[@]}" "${stamps[@]}") + fi + + if ((${#tags[@]} == 0)); then + echo "[build] ${PKGMGR_DISTRO} ${VERSION}: every tag already joins these platform images, nothing to push." + exit 0 + fi + + create_args=() + for t in "${tags[@]}"; do + create_args+=(-t "$t") + done + + echo "[build] Running: docker buildx imagetools create ${create_args[*]} ${sources[*]}" + docker buildx imagetools create "${create_args[@]}" "${sources[@]}" + + version_digest="$(digest_of "${DISTRO_TAG_BASE}:${VERSION}")" + for t in "${moving[@]}"; do + if [[ "$(digest_of "${t}")" != "${version_digest}" ]]; then + echo "[build] ❌ ${PKGMGR_DISTRO}: :${VERSION} and ${t} point at different images." >&2 + echo "[build] Whoever pinned ${t##*:} stopped receiving what ${VERSION} gets." >&2 + exit 1 + fi + done + echo "[build] ✅ ${PKGMGR_DISTRO}: every tag of ${VERSION} == ${version_digest}" elif [[ "${PUSH}" == "1" ]]; then + if [[ "${PUBLISH}" == "1" && -n "${PLATFORM}" ]]; then + probe="${REPO_PREFIX}-${PKGMGR_DISTRO}:probe" + docker buildx build --load --platform "${PLATFORM}" -t "${probe}" "${build_args[@]}" . + + package_set="$(package_set_cmd "${PKGMGR_DISTRO}")" + built_set="$(docker run --rm --platform "${PLATFORM}" --entrypoint bash "${probe}" -lc "${package_set}" | sha256sum)" + published_set="$(docker run --rm --pull always --platform "${PLATFORM}" --entrypoint bash "${DISTRO_TAG_BASE}:${VERSION}" -lc "${package_set}" 2>/dev/null | sha256sum || true)" + if [[ "${built_set}" == "${published_set}" ]]; then + echo "[build] ${PKGMGR_DISTRO} ${PLATFORM} unchanged since the published ${VERSION}, nothing to push." + exit 0 + fi + fi + bx_args=(docker buildx build --push) if [[ "${PUBLISH}" == "1" && -n "${PLATFORM}" ]]; then @@ -222,7 +283,10 @@ elif [[ "${PUSH}" == "1" ]]; then elif [[ "${PUBLISH}" == "1" ]]; then while IFS= read -r t; do bx_args+=(-t "$t") - done < <(compute_publish_tags) + done < <(moving_tags) + while IFS= read -r t; do + bx_args+=(-t "$t") + done < <(stamp_tags) else bx_args+=(-t "${IMAGE_TAG}") fi diff --git a/scripts/build/publish.sh b/scripts/build/publish.sh index 8c23239..9474c8a 100755 --- a/scripts/build/publish.sh +++ b/scripts/build/publish.sh @@ -6,10 +6,10 @@ set -euo pipefail # Required env: # OWNER (e.g. GITHUB_REPOSITORY_OWNER) # VERSION (e.g. 1.2.3) +# ALIASES moving tags besides VERSION, space separated, may be empty (e.g. "latest 1 1.2 stable") # # Optional env: # REGISTRY (default: ghcr.io) -# IS_STABLE (default: false) # DISTROS (default: "arch debian ubuntu fedora centos") SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" @@ -18,16 +18,16 @@ SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" source "${SCRIPT_DIR}/base.sh" REGISTRY="${REGISTRY:-ghcr.io}" -IS_STABLE="${IS_STABLE:-false}" DISTROS="${DISTROS:-arch debian ubuntu fedora centos}" : "${OWNER:?Environment variable OWNER must be set (e.g. github.repository_owner)}" : "${VERSION:?Environment variable VERSION must be set (e.g. 1.2.3)}" +: "${ALIASES?Environment variable ALIASES must be set, empty for none (e.g. \"latest 1 1.2 stable\")}" echo "[publish] REGISTRY=${REGISTRY}" echo "[publish] OWNER=${OWNER}" echo "[publish] VERSION=${VERSION}" -echo "[publish] IS_STABLE=${IS_STABLE}" +echo "[publish] ALIASES=${ALIASES}" echo "[publish] DISTROS=${DISTROS}" for d in ${DISTROS}; do @@ -51,7 +51,7 @@ for d in ${DISTROS}; do --registry "${REGISTRY}" \ --owner "${OWNER}" \ --version "${VERSION}" \ - --stable "${IS_STABLE}" \ + --aliases "${ALIASES}" \ --merge done diff --git a/scripts/github/publish-containers/checkout-workflow-run-commit.sh b/scripts/github/publish-containers/checkout-workflow-run-commit.sh deleted file mode 100644 index 3ae914a..0000000 --- a/scripts/github/publish-containers/checkout-workflow-run-commit.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -WORKFLOW_RUN_SHA="${WORKFLOW_RUN_SHA:?WORKFLOW_RUN_SHA must be set}" - -git checkout -f "${WORKFLOW_RUN_SHA}" -git fetch --tags --force -git tag --list 'stable' 'v*' --sort=version:refname | tail -n 20 diff --git a/scripts/github/publish-containers/compute-publish-container-info.sh b/scripts/github/publish-containers/compute-publish-container-info.sh index eabb673..7879d64 100644 --- a/scripts/github/publish-containers/compute-publish-container-info.sh +++ b/scripts/github/publish-containers/compute-publish-container-info.sh @@ -1,23 +1,91 @@ #!/usr/bin/env bash +# Param: WORKFLOW_RUN_SHA commit the "Mark stable commit" run tagged; empty on a scheduled or +# manual rebuild, which publishes the newest vX.Y.Z release on origin/main +# and the release the git tag stable marks +# Output: releases= in GITHUB_OUTPUT; aliases are the +# moving tags a release takes besides and . set -euo pipefail -SHA="$(git rev-parse HEAD)" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -V_TAG="$(git tag --points-at "${SHA}" --list 'v*' | sort -V | tail -n1)" -if [[ -z "${V_TAG}" ]]; then - echo "No version tag found for ${SHA}. Skipping publish." - echo "should_publish=false" >> "$GITHUB_OUTPUT" - exit 0 +# shellcheck source=scripts/build/base.sh +source "${SCRIPT_DIR}/../../build/base.sh" + +: "${WORKFLOW_RUN_SHA?WORKFLOW_RUN_SHA must be set, empty for a rebuild}" +: "${GITHUB_OUTPUT:?GITHUB_OUTPUT must be set}" + +RELEASE_TAG='^v[0-9]+\.[0-9]+\.[0-9]+$' + +release_at() { + local listed tag + listed="$(git tag --points-at "$1" --sort=-v:refname)" + tag="$(grep -m1 -E "${RELEASE_TAG}" <<< "${listed}" || true)" + echo "${tag#v}" +} + +newest_of() { + local prefix="$1" version + for version in "${RELEASES[@]}"; do + if [[ "${version}." == "${prefix}."* ]]; then + echo "${version}" + return 0 + fi + done +} + +aliases_of() { + local version="$1" prefix + local -a aliases=() + if [[ "${version}" == "${RELEASES[0]}" ]]; then + aliases+=(latest) + fi + while read -r prefix; do + if [[ "${prefix}" != "${version}" && "$(newest_of "${prefix}")" == "${version}" ]]; then + aliases+=("${prefix}") + fi + done < <(version_prefixes "${version}") + if [[ "${version}" == "${STABLE}" ]]; then + aliases+=(stable) + fi + echo "${aliases[*]}" +} + +listed="$(git tag --merged origin/main --sort=-v:refname)" +mapfile -t tags < <(grep -E "${RELEASE_TAG}" <<< "${listed}") +RELEASES=("${tags[@]#v}") + +STABLE="" +stable_sha="$(git rev-parse -q --verify 'refs/tags/stable^{commit}' || true)" +if [[ -n "${stable_sha}" ]]; then + STABLE="$(release_at "${stable_sha}")" fi -VERSION="${V_TAG#v}" +targets=() +if [[ -n "${WORKFLOW_RUN_SHA}" ]]; then + version="$(release_at "${WORKFLOW_RUN_SHA}")" + if ! git merge-base --is-ancestor "${WORKFLOW_RUN_SHA}" origin/main; then + echo "${WORKFLOW_RUN_SHA} is not on origin/main. Skipping publish." + elif [[ -z "${version}" ]]; then + echo "No vX.Y.Z tag found for ${WORKFLOW_RUN_SHA}. Skipping publish." + else + targets+=("${version}") + fi +else + ((${#RELEASES[@]} > 0)) || { echo "No vX.Y.Z tag is reachable from origin/main." >&2; exit 1; } + targets+=("${RELEASES[0]}") + if [[ -z "${STABLE}" ]]; then + echo "The git tag stable marks no vX.Y.Z release; the stable image stays where it is." + elif [[ "${STABLE}" != "${RELEASES[0]}" ]]; then + targets+=("${STABLE}") + fi +fi -STABLE_SHA="$(git rev-parse -q --verify 'refs/tags/stable^{commit}' 2>/dev/null || true)" -IS_STABLE=false -[[ -n "${STABLE_SHA}" && "${STABLE_SHA}" == "${SHA}" ]] && IS_STABLE=true +json="" +for version in "${targets[@]}"; do + sha="$(git rev-parse "v${version}^{commit}")" + moving="$(aliases_of "${version}")" + echo "Publishing ${version} at ${sha}, moving: ${moving}" + json+="${json:+,}{\"sha\":\"${sha}\",\"version\":\"${version}\",\"aliases\":\"${moving}\"}" +done -{ - echo "should_publish=true" - echo "version=${VERSION}" - echo "is_stable=${IS_STABLE}" -} >> "$GITHUB_OUTPUT" +echo "releases=[${json}]" >> "${GITHUB_OUTPUT}" diff --git a/scripts/github/publish-containers/merge-container-images.sh b/scripts/github/publish-containers/merge-container-images.sh index 2bdf534..5d69c59 100644 --- a/scripts/github/publish-containers/merge-container-images.sh +++ b/scripts/github/publish-containers/merge-container-images.sh @@ -3,7 +3,7 @@ set -euo pipefail : "${OWNER:?OWNER must be set}" : "${VERSION:?VERSION must be set}" -: "${IS_STABLE:?IS_STABLE must be set}" +: "${ALIASES?ALIASES must be set, empty for none}" : "${PKGMGR_DISTRO:?PKGMGR_DISTRO must be set}" bash scripts/build/image.sh \ @@ -11,5 +11,5 @@ bash scripts/build/image.sh \ --registry ghcr.io \ --owner "${OWNER}" \ --version "${VERSION}" \ - --stable "${IS_STABLE}" \ + --aliases "${ALIASES}" \ --merge diff --git a/tests/integration/test_build_image_publish.py b/tests/integration/test_build_image_publish.py index e80d072..8358381 100644 --- a/tests/integration/test_build_image_publish.py +++ b/tests/integration/test_build_image_publish.py @@ -8,6 +8,41 @@ from pathlib import Path REPO_ROOT = Path(__file__).resolve().parents[2] BASE = "--build-arg BASE_IMAGE=ghcr.io/kevinveenbirkenbach/base-{distro}:latest ." +STAMP = "1767225600" +DIGEST = "{{json .Manifest.Digest}}" + +FAKE_DOCKER = """#!/usr/bin/env bash +echo "$*" >> "$FAKE_DOCKER_LOG" +case "$*" in + "run "*" ghcr.io/"*) + for tag in ${FAKE_PUBLISHED_TAGS}; do + if [[ "$*" == *" ${tag} -lc "* ]]; then echo "${FAKE_PUBLISHED_SET}"; exit 0; fi + done + exit 1 ;; + "run "*) echo "${FAKE_BUILT_SET}" ;; + "buildx imagetools create --dry-run "*) echo "${FAKE_PLANNED_INDEX}" ;; + "buildx imagetools inspect --raw "*) + if [[ " ${FAKE_CURRENT_TAGS} " == *" ${!#} "* ]]; then + echo "${FAKE_PLANNED_INDEX}" + else + echo stale + fi ;; + "buildx imagetools inspect --format "*) + if [[ "${!#}" == "${FAKE_DRIFTED_TAG}" ]]; then echo old; else echo new; fi ;; +esac +""" + + +def refs(image: str, *tags: str) -> list[str]: + return [f"ghcr.io/o/{image}:{tag}" for tag in tags] + + +def sources(distro: str, version: str) -> str: + return " ".join(refs(f"pkgmgr-{distro}", f"{version}-amd64", f"{version}-arm64")) + + +def tag_args(tags: list[str]) -> str: + return " ".join(f"-t {tag}" for tag in tags) class TestBuildImagePublish(unittest.TestCase): @@ -16,13 +51,22 @@ class TestBuildImagePublish(unittest.TestCase): tmp = Path(self._tmp.name) self.log = tmp / "docker.log" fake = tmp / "docker" - fake.write_text('#!/usr/bin/env bash\necho "$*" >> "$FAKE_DOCKER_LOG"\n') + fake.write_text(FAKE_DOCKER) fake.chmod(0o755) + date = tmp / "date" + date.write_text(f"#!/usr/bin/env bash\necho {STAMP}\n") + date.chmod(0o755) self.env = { k: v for k, v in os.environ.items() if not k.startswith("BASE_IMAGES_") } self.env["PATH"] = f"{tmp}{os.pathsep}{os.environ['PATH']}" self.env["FAKE_DOCKER_LOG"] = str(self.log) + self.env["FAKE_BUILT_SET"] = "built" + self.env["FAKE_PUBLISHED_SET"] = "published" + self.env["FAKE_PUBLISHED_TAGS"] = "" + self.env["FAKE_PLANNED_INDEX"] = "planned" + self.env["FAKE_CURRENT_TAGS"] = "" + self.env["FAKE_DRIFTED_TAG"] = "" def tearDown(self) -> None: self._tmp.cleanup() @@ -39,7 +83,7 @@ class TestBuildImagePublish(unittest.TestCase): calls = self.log.read_text().splitlines() if self.log.exists() else [] return result, calls - def _publish(self, distro: str, *args: str): + def _publish(self, distro: str, version: str, *args: str, **env: str): return self._run( "image.sh", distro, @@ -49,67 +93,228 @@ class TestBuildImagePublish(unittest.TestCase): "--owner", "o", "--version", - "2.1.2", + version, *args, + **env, ) def test_platform_pushes_only_the_arch_tag(self) -> None: - result, calls = self._publish("debian", "--platform", "linux/arm64") + result, calls = self._publish("debian", "2.1.2", "--platform", "linux/arm64") self.assertEqual(result.returncode, 0, result.stderr) self.assertEqual( calls, [ + ("buildx build --load --platform linux/arm64 -t pkgmgr-debian:probe ") + + BASE.format(distro="debian"), + ( + "run --rm --platform linux/arm64 --entrypoint bash " + "pkgmgr-debian:probe -lc dpkg-query -W" + ), + ( + "run --rm --pull always --platform linux/arm64 --entrypoint bash " + "ghcr.io/o/pkgmgr-debian:2.1.2 -lc dpkg-query -W" + ), ( "buildx build --push -t ghcr.io/o/pkgmgr-debian:2.1.2-arm64 " "--platform linux/arm64 " ) - + BASE.format(distro="debian") + + BASE.format(distro="debian"), ], ) - def test_merge_joins_arch_tags_under_release_and_alias_tags(self) -> None: - result, calls = self._publish("arch", "--stable", "true", "--merge") + def test_platform_unchanged_since_its_own_published_version_pushes_nothing( + self, + ) -> None: + result, calls = self._publish( + "debian", + "2.3.0", + "--platform", + "linux/arm64", + FAKE_BUILT_SET="same", + FAKE_PUBLISHED_SET="same", + FAKE_PUBLISHED_TAGS="ghcr.io/o/pkgmgr-debian:2.3.0", + ) self.assertEqual(result.returncode, 0, result.stderr) - tags = " ".join( - f"-t ghcr.io/o/{image}:{tag}" - for image in ("pkgmgr-arch", "pkgmgr") - for tag in ("latest", "2.1.2", "stable") + self.assertIn("nothing to push", result.stdout) + self.assertFalse([c for c in calls if c.startswith("buildx build --push")]) + + def test_platform_pushes_when_its_packages_changed(self) -> None: + result, calls = self._publish( + "debian", + "2.3.0", + "--platform", + "linux/arm64", + FAKE_PUBLISHED_TAGS="ghcr.io/o/pkgmgr-debian:2.3.0", + ) + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertTrue(calls[-1].startswith("buildx build --push"), calls) + + def test_platform_pushes_while_its_version_is_not_out(self) -> None: + result, calls = self._publish( + "debian", + "2.4.0", + "--platform", + "linux/arm64", + FAKE_BUILT_SET="same", + FAKE_PUBLISHED_SET="same", + FAKE_PUBLISHED_TAGS="ghcr.io/o/pkgmgr-debian:latest", + ) + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn( + ( + "run --rm --pull always --platform linux/arm64 --entrypoint bash " + "ghcr.io/o/pkgmgr-debian:2.4.0 -lc dpkg-query -W" + ), + calls, + ) + self.assertTrue(calls[-1].startswith("buildx build --push"), calls) + + def test_merge_of_the_newest_release_tags_latest_and_its_prefixes(self) -> None: + result, calls = self._publish( + "arch", "2.4.0", "--aliases", "latest 2 2.4", "--merge" + ) + + self.assertEqual(result.returncode, 0, result.stderr) + moving = refs("pkgmgr-arch", "2.4.0", "latest", "2", "2.4") + refs( + "pkgmgr", "2.4.0", "latest", "2", "2.4" + ) + stamps = refs("pkgmgr-arch", f"2.4.0.{STAMP}") + refs( + "pkgmgr", f"2.4.0.{STAMP}" ) self.assertEqual( calls, [ ( - f"buildx imagetools create {tags} " - "ghcr.io/o/pkgmgr-arch:2.1.2-amd64 ghcr.io/o/pkgmgr-arch:2.1.2-arm64" + "buildx imagetools create --dry-run " + f"-t ghcr.io/o/pkgmgr-arch:2.4.0 {sources('arch', '2.4.0')}" ), - "buildx imagetools inspect ghcr.io/o/pkgmgr-arch:2.1.2", - ], + "buildx imagetools inspect --raw ghcr.io/o/pkgmgr-arch:2.4.0", + ( + f"buildx imagetools create {tag_args(moving + stamps)} " + f"{sources('arch', '2.4.0')}" + ), + f"buildx imagetools inspect --format {DIGEST} ghcr.io/o/pkgmgr-arch:2.4.0", + ] + + [f"buildx imagetools inspect --format {DIGEST} {t}" for t in moving], ) - def test_unknown_platform_and_merge_without_publish_are_refused(self) -> None: - unknown, _ = self._publish("debian", "--platform", "linux/s390x") - no_publish, calls = self._run("image.sh", "debian", "--merge") + def test_merge_of_an_older_stable_release_tags_stable_and_only_its_own_prefixes( + self, + ) -> None: + result, calls = self._publish( + "debian", "2.3.0", "--aliases", "2.3 stable", "--merge" + ) - self.assertEqual(unknown.returncode, 2) - self.assertEqual(no_publish.returncode, 2) - self.assertEqual(calls, []) + self.assertEqual(result.returncode, 0, result.stderr) + tags = refs("pkgmgr-debian", "2.3.0", "2.3", "stable", f"2.3.0.{STAMP}") + self.assertIn( + f"buildx imagetools create {tag_args(tags)} {sources('debian', '2.3.0')}", + calls, + ) - def test_publish_builds_each_platform_then_merges(self) -> None: - result, calls = self._run( - "publish.sh", "fedora", OWNER="o", VERSION="2.1.2", DISTROS="fedora" + def test_merge_moves_stable_to_an_unchanged_build_without_a_new_stamp( + self, + ) -> None: + result, calls = self._publish( + "debian", + "2.4.0", + "--aliases", + "latest 2 2.4 stable", + "--merge", + FAKE_CURRENT_TAGS=" ".join( + refs("pkgmgr-debian", "2.4.0", "latest", "2", "2.4") + ), ) self.assertEqual(result.returncode, 0, result.stderr) self.assertEqual( - [c.split(" -t ")[0] for c in calls], - ["buildx build --push", "buildx build --push", "buildx imagetools create"] - + ["buildx imagetools inspect ghcr.io/o/pkgmgr-fedora:2.1.2"], + [c for c in calls if c.startswith("buildx imagetools create -t")], + [ + "buildx imagetools create -t ghcr.io/o/pkgmgr-debian:stable " + + sources("debian", "2.4.0") + ], ) - self.assertIn("--platform linux/amd64", calls[0]) - self.assertIn("--platform linux/arm64", calls[1]) - self.assertNotIn(":stable", calls[2]) + + def test_merge_pushes_nothing_when_every_tag_shows_the_build(self) -> None: + result, calls = self._publish( + "arch", + "2.4.0", + "--aliases", + "latest 2 2.4 stable", + "--merge", + FAKE_CURRENT_TAGS=" ".join( + refs("pkgmgr-arch", "2.4.0", "latest", "2", "2.4", "stable") + + refs("pkgmgr", "2.4.0", "latest", "2", "2.4", "stable") + ), + ) + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("nothing to push", result.stdout) + self.assertEqual( + [c for c in calls if c.startswith("buildx imagetools create -t")], [] + ) + + def test_merge_fails_when_a_tag_drifts_from_its_version(self) -> None: + result, _ = self._publish( + "debian", + "2.1.2", + "--aliases", + "2.1", + "--merge", + FAKE_DRIFTED_TAG="ghcr.io/o/pkgmgr-debian:2.1", + ) + + self.assertEqual(result.returncode, 1) + self.assertIn( + ":2.1.2 and ghcr.io/o/pkgmgr-debian:2.1 point at different images", + result.stderr, + ) + + def test_unknown_platform_merge_without_publish_or_aliases_are_refused( + self, + ) -> None: + unknown, _ = self._publish("debian", "2.1.2", "--platform", "linux/s390x") + no_publish, _ = self._run("image.sh", "debian", "--merge") + no_aliases, calls = self._publish("debian", "2.1.2", "--merge") + + self.assertEqual(unknown.returncode, 2) + self.assertEqual(no_publish.returncode, 2) + self.assertEqual(no_aliases.returncode, 2) + self.assertEqual(calls, []) + + def test_publish_builds_each_platform_then_merges(self) -> None: + result, calls = self._run( + "publish.sh", + "fedora", + OWNER="o", + VERSION="2.1.2", + ALIASES="2.1 stable", + DISTROS="fedora", + ) + + self.assertEqual(result.returncode, 0, result.stderr) + pushes = [ + c + for c in calls + if c.startswith(("buildx build --push", "buildx imagetools create")) + ] + self.assertEqual( + [c.split(" -t ")[0] for c in pushes], + [ + "buildx build --push", + "buildx build --push", + "buildx imagetools create --dry-run", + "buildx imagetools create", + ], + ) + self.assertIn("--platform linux/amd64", pushes[0]) + self.assertIn("--platform linux/arm64", pushes[1]) + self.assertIn("-t ghcr.io/o/pkgmgr-fedora:stable ", pushes[3]) + self.assertNotIn(":latest", pushes[3]) if __name__ == "__main__": diff --git a/tests/integration/test_publish_containers_release.py b/tests/integration/test_publish_containers_release.py new file mode 100644 index 0000000..993e52d --- /dev/null +++ b/tests/integration/test_publish_containers_release.py @@ -0,0 +1,239 @@ +from __future__ import annotations + +import json +import os +import subprocess +import tempfile +import unittest +from pathlib import Path + +import yaml + +REPO_ROOT = Path(__file__).resolve().parents[2] +RESOLVER = ( + REPO_ROOT + / "scripts" + / "github" + / "publish-containers" + / "compute-publish-container-info.sh" +) +WORKFLOW = REPO_ROOT / ".github" / "workflows" / "publish-containers.yml" + +GIT_ENV = { + **os.environ, + "GIT_CONFIG_GLOBAL": os.devnull, + "GIT_CONFIG_NOSYSTEM": "1", + "GIT_AUTHOR_NAME": "test", + "GIT_AUTHOR_EMAIL": "test@example.invalid", + "GIT_COMMITTER_NAME": "test", + "GIT_COMMITTER_EMAIL": "test@example.invalid", +} + + +class TestComputePublishContainerInfo(unittest.TestCase): + def setUp(self) -> None: + self._tmp = tempfile.TemporaryDirectory() + tmp = Path(self._tmp.name) + self.repo = tmp / "repo" + self.repo.mkdir() + self.output = tmp / "github_output" + self.git("init", "-q", "-b", "main") + + def tearDown(self) -> None: + self._tmp.cleanup() + + def git(self, *args: str) -> str: + return subprocess.run( + ["git", *args], + cwd=self.repo, + env=GIT_ENV, + capture_output=True, + text=True, + check=True, + ).stdout.strip() + + def commit(self, *tags: str) -> str: + self.git("commit", "-q", "--allow-empty", "-m", " ".join(tags) or "untagged") + for tag in tags: + self.git("tag", tag) + return self.git("rev-parse", "HEAD") + + def push_main(self) -> None: + self.git("update-ref", "refs/remotes/origin/main", "HEAD") + + def resolve(self, workflow_run_sha: str): + self.output.write_text("") + result = subprocess.run( + ["bash", str(RESOLVER)], + cwd=self.repo, + env={ + **GIT_ENV, + "GITHUB_OUTPUT": str(self.output), + "WORKFLOW_RUN_SHA": workflow_run_sha, + }, + capture_output=True, + text=True, + check=False, + ) + outputs = dict( + line.split("=", 1) for line in self.output.read_text().splitlines() + ) + return result, outputs + + def releases(self, workflow_run_sha: str) -> list[dict[str, str]]: + result, outputs = self.resolve(workflow_run_sha) + self.assertEqual(result.returncode, 0, result.stderr) + return json.loads(outputs["releases"]) + + def test_a_rebuild_with_stable_on_the_newest_release_publishes_it_once( + self, + ) -> None: + self.commit("v2.3.0") + newest = self.commit("v2.4.0", "stable") + self.push_main() + + self.assertEqual( + self.releases(""), + [{"sha": newest, "version": "2.4.0", "aliases": "latest 2 2.4 stable"}], + ) + + def test_a_rebuild_with_stable_behind_publishes_the_newest_and_the_stable_release( + self, + ) -> None: + self.commit("v2.2.0") + stable = self.commit("v2.3.0", "stable") + newest = self.commit("v2.4.0") + self.commit("v2.5.0-rc1") + self.push_main() + + self.assertEqual( + self.releases(""), + [ + {"sha": newest, "version": "2.4.0", "aliases": "latest 2 2.4"}, + {"sha": stable, "version": "2.3.0", "aliases": "2.3 stable"}, + ], + ) + + def test_the_stable_release_takes_no_prefix_a_newer_release_leads(self) -> None: + stable = self.commit("v2.3.0", "stable") + self.commit("v2.3.1") + newest = self.commit("v2.4.0") + self.push_main() + + self.assertEqual( + self.releases(""), + [ + {"sha": newest, "version": "2.4.0", "aliases": "latest 2 2.4"}, + {"sha": stable, "version": "2.3.0", "aliases": "stable"}, + ], + ) + + def test_a_rebuild_publishes_the_newest_release_on_main_not_its_tip( + self, + ) -> None: + self.commit("v1.9.0") + newest = self.commit("v1.10.0") + self.commit() + self.push_main() + self.git("checkout", "-q", "-b", "side") + self.commit("v3.0.0") + + result, outputs = self.resolve("") + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual( + json.loads(outputs["releases"]), + [{"sha": newest, "version": "1.10.0", "aliases": "latest 1 1.10"}], + ) + self.assertIn("the stable image stays where it is", result.stdout) + + def test_a_workflow_run_publishes_the_commit_it_tagged(self) -> None: + self.commit("v2.3.0") + tagged = self.commit("v2.4.0", "stable") + self.push_main() + + self.assertEqual( + self.releases(tagged), + [{"sha": tagged, "version": "2.4.0", "aliases": "latest 2 2.4 stable"}], + ) + + def test_a_workflow_run_of_a_backport_moves_only_the_prefixes_it_leads( + self, + ) -> None: + self.commit("v2.3.0") + self.commit("v2.4.0", "stable") + backport = self.commit("v2.3.1") + self.push_main() + + self.assertEqual( + self.releases(backport), + [{"sha": backport, "version": "2.3.1", "aliases": "2.3"}], + ) + + def test_a_workflow_run_outside_main_publishes_nothing(self) -> None: + self.commit("v2.3.0") + self.push_main() + self.git("checkout", "-q", "-b", "side") + outside = self.commit("v9.0.0") + + self.assertEqual(self.releases(outside), []) + + def test_a_workflow_run_without_an_x_y_z_tag_publishes_nothing(self) -> None: + self.commit("v2.3.0") + candidate = self.commit("v2.4.0-rc1") + self.push_main() + + self.assertEqual(self.releases(candidate), []) + + def test_a_rebuild_without_a_release_on_main_fails(self) -> None: + self.commit("v1.0.0-rc1", "stable") + self.push_main() + + result, outputs = self.resolve("") + + self.assertEqual(result.returncode, 1) + self.assertEqual(outputs, {}) + self.assertIn("No vX.Y.Z tag is reachable from origin/main", result.stderr) + + +class TestPublishContainersWorkflow(unittest.TestCase): + def setUp(self) -> None: + self.workflow = yaml.safe_load(WORKFLOW.read_text()) + + def test_the_releases_are_rebuilt_daily_at_03_utc_and_on_demand(self) -> None: + triggers = self.workflow[True] + steps = { + step.get("id"): step for step in self.workflow["jobs"]["prepare"]["steps"] + } + + self.assertEqual(triggers["schedule"], [{"cron": "0 3 * * *"}]) + self.assertIn("workflow_dispatch", triggers) + self.assertEqual( + steps["info"]["run"], + "bash scripts/github/publish-containers/compute-publish-container-info.sh", + ) + self.assertEqual( + steps["info"]["env"]["WORKFLOW_RUN_SHA"], + "${{ github.event.workflow_run.head_sha }}", + ) + + def test_every_resolved_release_is_built_and_merged_with_its_own_tags( + self, + ) -> None: + jobs = self.workflow["jobs"] + merge_env = jobs["merge"]["steps"][-1]["env"] + + for job in ("build", "merge"): + self.assertEqual( + jobs[job]["strategy"]["matrix"]["release"], + "${{ fromJSON(needs.prepare.outputs.releases) }}", + ) + self.assertEqual( + jobs[job]["steps"][0]["with"]["ref"], "${{ matrix.release.sha }}" + ) + self.assertEqual(merge_env["VERSION"], "${{ matrix.release.version }}") + self.assertEqual(merge_env["ALIASES"], "${{ matrix.release.aliases }}") + + +if __name__ == "__main__": + unittest.main()