feat(publish): rebuild releases nightly and let stable follow the tested one

Every image build now carries <version> and the immutable
<version>.<unix-ts>; latest goes to the newest release on main, each
prefix (2, 2.3) to the newest release starting with it, and stable to the
release the git tag stable marks, which "Mark stable commit" sets only
after all tests passed.

publish-containers.yml runs daily at 03:00 UTC and on workflow_dispatch.
A rebuild publishes the newest release and, when the git tag stable sits
on an older one, that release too, so both get base-image security
updates. compute-publish-container-info.sh emits the release list with
each release's moving tags; image.sh takes them as --aliases instead of
--stable.

A platform build whose installed package set equals the published
<version> pushes nothing, and a merge whose image index equals the
published one only moves the tags not yet on it, without a new stamp.
Only vX.Y.Z tags count as releases; v2.4.0-rc1 no longer moves 2 or 2.4.

The build scripts come from the release commit: releases tagged before
this change keep the old scripts until a newer release replaces them.

tests/integration: 22 tests for release choice, aliases, the stable
rebuild, the push skip and the triggers.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/integration/test_build_image_publish.py

@@ -8,6 +8,41 @@ from pathlib import Path
 
 REPO_ROOT = Path(__file__).resolve().parents[2]
 BASE = "--build-arg BASE_IMAGE=ghcr.io/kevinveenbirkenbach/base-{distro}:latest ."
+STAMP = "1767225600"
+DIGEST = "{{json .Manifest.Digest}}"
+
+FAKE_DOCKER = """#!/usr/bin/env bash
+echo "$*" >> "$FAKE_DOCKER_LOG"
+case "$*" in
+  "run "*" ghcr.io/"*)
+    for tag in ${FAKE_PUBLISHED_TAGS}; do
+      if [[ "$*" == *" ${tag} -lc "* ]]; then echo "${FAKE_PUBLISHED_SET}"; exit 0; fi
+    done
+    exit 1 ;;
+  "run "*) echo "${FAKE_BUILT_SET}" ;;
+  "buildx imagetools create --dry-run "*) echo "${FAKE_PLANNED_INDEX}" ;;
+  "buildx imagetools inspect --raw "*)
+    if [[ " ${FAKE_CURRENT_TAGS} " == *" ${!#} "* ]]; then
+      echo "${FAKE_PLANNED_INDEX}"
+    else
+      echo stale
+    fi ;;
+  "buildx imagetools inspect --format "*)
+    if [[ "${!#}" == "${FAKE_DRIFTED_TAG}" ]]; then echo old; else echo new; fi ;;
+esac
+"""
+
+
+def refs(image: str, *tags: str) -> list[str]:
+    return [f"ghcr.io/o/{image}:{tag}" for tag in tags]
+
+
+def sources(distro: str, version: str) -> str:
+    return " ".join(refs(f"pkgmgr-{distro}", f"{version}-amd64", f"{version}-arm64"))
+
+
+def tag_args(tags: list[str]) -> str:
+    return " ".join(f"-t {tag}" for tag in tags)
 
 
 class TestBuildImagePublish(unittest.TestCase):
@@ -16,13 +51,22 @@ class TestBuildImagePublish(unittest.TestCase):
         tmp = Path(self._tmp.name)
         self.log = tmp / "docker.log"
         fake = tmp / "docker"
-        fake.write_text('#!/usr/bin/env bash\necho "$*" >> "$FAKE_DOCKER_LOG"\n')
+        fake.write_text(FAKE_DOCKER)
         fake.chmod(0o755)
+        date = tmp / "date"
+        date.write_text(f"#!/usr/bin/env bash\necho {STAMP}\n")
+        date.chmod(0o755)
         self.env = {
             k: v for k, v in os.environ.items() if not k.startswith("BASE_IMAGES_")
         }
         self.env["PATH"] = f"{tmp}{os.pathsep}{os.environ['PATH']}"
         self.env["FAKE_DOCKER_LOG"] = str(self.log)
+        self.env["FAKE_BUILT_SET"] = "built"
+        self.env["FAKE_PUBLISHED_SET"] = "published"
+        self.env["FAKE_PUBLISHED_TAGS"] = ""
+        self.env["FAKE_PLANNED_INDEX"] = "planned"
+        self.env["FAKE_CURRENT_TAGS"] = ""
+        self.env["FAKE_DRIFTED_TAG"] = ""
 
     def tearDown(self) -> None:
         self._tmp.cleanup()
@@ -39,7 +83,7 @@ class TestBuildImagePublish(unittest.TestCase):
         calls = self.log.read_text().splitlines() if self.log.exists() else []
         return result, calls
 
-    def _publish(self, distro: str, *args: str):
+    def _publish(self, distro: str, version: str, *args: str, **env: str):
         return self._run(
             "image.sh",
             distro,
@@ -49,67 +93,228 @@ class TestBuildImagePublish(unittest.TestCase):
             "--owner",
             "o",
             "--version",
-            "2.1.2",
+            version,
             *args,
+            **env,
         )
 
     def test_platform_pushes_only_the_arch_tag(self) -> None:
-        result, calls = self._publish("debian", "--platform", "linux/arm64")
+        result, calls = self._publish("debian", "2.1.2", "--platform", "linux/arm64")
 
         self.assertEqual(result.returncode, 0, result.stderr)
         self.assertEqual(
             calls,
             [
+                ("buildx build --load --platform linux/arm64 -t pkgmgr-debian:probe ")
+                + BASE.format(distro="debian"),
+                (
+                    "run --rm --platform linux/arm64 --entrypoint bash "
+                    "pkgmgr-debian:probe -lc dpkg-query -W"
+                ),
+                (
+                    "run --rm --pull always --platform linux/arm64 --entrypoint bash "
+                    "ghcr.io/o/pkgmgr-debian:2.1.2 -lc dpkg-query -W"
+                ),
                 (
                     "buildx build --push -t ghcr.io/o/pkgmgr-debian:2.1.2-arm64 "
                     "--platform linux/arm64 "
                 )
-                + BASE.format(distro="debian")
+                + BASE.format(distro="debian"),
             ],
         )
 
-    def test_merge_joins_arch_tags_under_release_and_alias_tags(self) -> None:
-        result, calls = self._publish("arch", "--stable", "true", "--merge")
+    def test_platform_unchanged_since_its_own_published_version_pushes_nothing(
+        self,
+    ) -> None:
+        result, calls = self._publish(
+            "debian",
+            "2.3.0",
+            "--platform",
+            "linux/arm64",
+            FAKE_BUILT_SET="same",
+            FAKE_PUBLISHED_SET="same",
+            FAKE_PUBLISHED_TAGS="ghcr.io/o/pkgmgr-debian:2.3.0",
+        )
 
         self.assertEqual(result.returncode, 0, result.stderr)
-        tags = " ".join(
-            f"-t ghcr.io/o/{image}:{tag}"
-            for image in ("pkgmgr-arch", "pkgmgr")
-            for tag in ("latest", "2.1.2", "stable")
+        self.assertIn("nothing to push", result.stdout)
+        self.assertFalse([c for c in calls if c.startswith("buildx build --push")])
+
+    def test_platform_pushes_when_its_packages_changed(self) -> None:
+        result, calls = self._publish(
+            "debian",
+            "2.3.0",
+            "--platform",
+            "linux/arm64",
+            FAKE_PUBLISHED_TAGS="ghcr.io/o/pkgmgr-debian:2.3.0",
+        )
+
+        self.assertEqual(result.returncode, 0, result.stderr)
+        self.assertTrue(calls[-1].startswith("buildx build --push"), calls)
+
+    def test_platform_pushes_while_its_version_is_not_out(self) -> None:
+        result, calls = self._publish(
+            "debian",
+            "2.4.0",
+            "--platform",
+            "linux/arm64",
+            FAKE_BUILT_SET="same",
+            FAKE_PUBLISHED_SET="same",
+            FAKE_PUBLISHED_TAGS="ghcr.io/o/pkgmgr-debian:latest",
+        )
+
+        self.assertEqual(result.returncode, 0, result.stderr)
+        self.assertIn(
+            (
+                "run --rm --pull always --platform linux/arm64 --entrypoint bash "
+                "ghcr.io/o/pkgmgr-debian:2.4.0 -lc dpkg-query -W"
+            ),
+            calls,
+        )
+        self.assertTrue(calls[-1].startswith("buildx build --push"), calls)
+
+    def test_merge_of_the_newest_release_tags_latest_and_its_prefixes(self) -> None:
+        result, calls = self._publish(
+            "arch", "2.4.0", "--aliases", "latest 2 2.4", "--merge"
+        )
+
+        self.assertEqual(result.returncode, 0, result.stderr)
+        moving = refs("pkgmgr-arch", "2.4.0", "latest", "2", "2.4") + refs(
+            "pkgmgr", "2.4.0", "latest", "2", "2.4"
+        )
+        stamps = refs("pkgmgr-arch", f"2.4.0.{STAMP}") + refs(
+            "pkgmgr", f"2.4.0.{STAMP}"
         )
         self.assertEqual(
             calls,
             [
                 (
-                    f"buildx imagetools create {tags} "
-                    "ghcr.io/o/pkgmgr-arch:2.1.2-amd64 ghcr.io/o/pkgmgr-arch:2.1.2-arm64"
+                    "buildx imagetools create --dry-run "
+                    f"-t ghcr.io/o/pkgmgr-arch:2.4.0 {sources('arch', '2.4.0')}"
                 ),
-                "buildx imagetools inspect ghcr.io/o/pkgmgr-arch:2.1.2",
-            ],
+                "buildx imagetools inspect --raw ghcr.io/o/pkgmgr-arch:2.4.0",
+                (
+                    f"buildx imagetools create {tag_args(moving + stamps)} "
+                    f"{sources('arch', '2.4.0')}"
+                ),
+                f"buildx imagetools inspect --format {DIGEST} ghcr.io/o/pkgmgr-arch:2.4.0",
+            ]
+            + [f"buildx imagetools inspect --format {DIGEST} {t}" for t in moving],
         )
 
-    def test_unknown_platform_and_merge_without_publish_are_refused(self) -> None:
-        unknown, _ = self._publish("debian", "--platform", "linux/s390x")
-        no_publish, calls = self._run("image.sh", "debian", "--merge")
+    def test_merge_of_an_older_stable_release_tags_stable_and_only_its_own_prefixes(
+        self,
+    ) -> None:
+        result, calls = self._publish(
+            "debian", "2.3.0", "--aliases", "2.3 stable", "--merge"
+        )
 
-        self.assertEqual(unknown.returncode, 2)
-        self.assertEqual(no_publish.returncode, 2)
-        self.assertEqual(calls, [])
+        self.assertEqual(result.returncode, 0, result.stderr)
+        tags = refs("pkgmgr-debian", "2.3.0", "2.3", "stable", f"2.3.0.{STAMP}")
+        self.assertIn(
+            f"buildx imagetools create {tag_args(tags)} {sources('debian', '2.3.0')}",
+            calls,
+        )
 
-    def test_publish_builds_each_platform_then_merges(self) -> None:
-        result, calls = self._run(
-            "publish.sh", "fedora", OWNER="o", VERSION="2.1.2", DISTROS="fedora"
+    def test_merge_moves_stable_to_an_unchanged_build_without_a_new_stamp(
+        self,
+    ) -> None:
+        result, calls = self._publish(
+            "debian",
+            "2.4.0",
+            "--aliases",
+            "latest 2 2.4 stable",
+            "--merge",
+            FAKE_CURRENT_TAGS=" ".join(
+                refs("pkgmgr-debian", "2.4.0", "latest", "2", "2.4")
+            ),
         )
 
         self.assertEqual(result.returncode, 0, result.stderr)
         self.assertEqual(
-            [c.split(" -t ")[0] for c in calls],
-            ["buildx build --push", "buildx build --push", "buildx imagetools create"]
-            + ["buildx imagetools inspect ghcr.io/o/pkgmgr-fedora:2.1.2"],
+            [c for c in calls if c.startswith("buildx imagetools create -t")],
+            [
+                "buildx imagetools create -t ghcr.io/o/pkgmgr-debian:stable "
+                + sources("debian", "2.4.0")
+            ],
         )
-        self.assertIn("--platform linux/amd64", calls[0])
-        self.assertIn("--platform linux/arm64", calls[1])
-        self.assertNotIn(":stable", calls[2])
+
+    def test_merge_pushes_nothing_when_every_tag_shows_the_build(self) -> None:
+        result, calls = self._publish(
+            "arch",
+            "2.4.0",
+            "--aliases",
+            "latest 2 2.4 stable",
+            "--merge",
+            FAKE_CURRENT_TAGS=" ".join(
+                refs("pkgmgr-arch", "2.4.0", "latest", "2", "2.4", "stable")
+                + refs("pkgmgr", "2.4.0", "latest", "2", "2.4", "stable")
+            ),
+        )
+
+        self.assertEqual(result.returncode, 0, result.stderr)
+        self.assertIn("nothing to push", result.stdout)
+        self.assertEqual(
+            [c for c in calls if c.startswith("buildx imagetools create -t")], []
+        )
+
+    def test_merge_fails_when_a_tag_drifts_from_its_version(self) -> None:
+        result, _ = self._publish(
+            "debian",
+            "2.1.2",
+            "--aliases",
+            "2.1",
+            "--merge",
+            FAKE_DRIFTED_TAG="ghcr.io/o/pkgmgr-debian:2.1",
+        )
+
+        self.assertEqual(result.returncode, 1)
+        self.assertIn(
+            ":2.1.2 and ghcr.io/o/pkgmgr-debian:2.1 point at different images",
+            result.stderr,
+        )
+
+    def test_unknown_platform_merge_without_publish_or_aliases_are_refused(
+        self,
+    ) -> None:
+        unknown, _ = self._publish("debian", "2.1.2", "--platform", "linux/s390x")
+        no_publish, _ = self._run("image.sh", "debian", "--merge")
+        no_aliases, calls = self._publish("debian", "2.1.2", "--merge")
+
+        self.assertEqual(unknown.returncode, 2)
+        self.assertEqual(no_publish.returncode, 2)
+        self.assertEqual(no_aliases.returncode, 2)
+        self.assertEqual(calls, [])
+
+    def test_publish_builds_each_platform_then_merges(self) -> None:
+        result, calls = self._run(
+            "publish.sh",
+            "fedora",
+            OWNER="o",
+            VERSION="2.1.2",
+            ALIASES="2.1 stable",
+            DISTROS="fedora",
+        )
+
+        self.assertEqual(result.returncode, 0, result.stderr)
+        pushes = [
+            c
+            for c in calls
+            if c.startswith(("buildx build --push", "buildx imagetools create"))
+        ]
+        self.assertEqual(
+            [c.split(" -t ")[0] for c in pushes],
+            [
+                "buildx build --push",
+                "buildx build --push",
+                "buildx imagetools create --dry-run",
+                "buildx imagetools create",
+            ],
+        )
+        self.assertIn("--platform linux/amd64", pushes[0])
+        self.assertIn("--platform linux/arm64", pushes[1])
+        self.assertIn("-t ghcr.io/o/pkgmgr-fedora:stable ", pushes[3])
+        self.assertNotIn(":latest", pushes[3])
 
 
 if __name__ == "__main__":

tests/integration/test_publish_containers_release.py

@@ -0,0 +1,239 @@
+from __future__ import annotations
+
+import json
+import os
+import subprocess
+import tempfile
+import unittest
+from pathlib import Path
+
+import yaml
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+RESOLVER = (
+    REPO_ROOT
+    / "scripts"
+    / "github"
+    / "publish-containers"
+    / "compute-publish-container-info.sh"
+)
+WORKFLOW = REPO_ROOT / ".github" / "workflows" / "publish-containers.yml"
+
+GIT_ENV = {
+    **os.environ,
+    "GIT_CONFIG_GLOBAL": os.devnull,
+    "GIT_CONFIG_NOSYSTEM": "1",
+    "GIT_AUTHOR_NAME": "test",
+    "GIT_AUTHOR_EMAIL": "test@example.invalid",
+    "GIT_COMMITTER_NAME": "test",
+    "GIT_COMMITTER_EMAIL": "test@example.invalid",
+}
+
+
+class TestComputePublishContainerInfo(unittest.TestCase):
+    def setUp(self) -> None:
+        self._tmp = tempfile.TemporaryDirectory()
+        tmp = Path(self._tmp.name)
+        self.repo = tmp / "repo"
+        self.repo.mkdir()
+        self.output = tmp / "github_output"
+        self.git("init", "-q", "-b", "main")
+
+    def tearDown(self) -> None:
+        self._tmp.cleanup()
+
+    def git(self, *args: str) -> str:
+        return subprocess.run(
+            ["git", *args],
+            cwd=self.repo,
+            env=GIT_ENV,
+            capture_output=True,
+            text=True,
+            check=True,
+        ).stdout.strip()
+
+    def commit(self, *tags: str) -> str:
+        self.git("commit", "-q", "--allow-empty", "-m", " ".join(tags) or "untagged")
+        for tag in tags:
+            self.git("tag", tag)
+        return self.git("rev-parse", "HEAD")
+
+    def push_main(self) -> None:
+        self.git("update-ref", "refs/remotes/origin/main", "HEAD")
+
+    def resolve(self, workflow_run_sha: str):
+        self.output.write_text("")
+        result = subprocess.run(
+            ["bash", str(RESOLVER)],
+            cwd=self.repo,
+            env={
+                **GIT_ENV,
+                "GITHUB_OUTPUT": str(self.output),
+                "WORKFLOW_RUN_SHA": workflow_run_sha,
+            },
+            capture_output=True,
+            text=True,
+            check=False,
+        )
+        outputs = dict(
+            line.split("=", 1) for line in self.output.read_text().splitlines()
+        )
+        return result, outputs
+
+    def releases(self, workflow_run_sha: str) -> list[dict[str, str]]:
+        result, outputs = self.resolve(workflow_run_sha)
+        self.assertEqual(result.returncode, 0, result.stderr)
+        return json.loads(outputs["releases"])
+
+    def test_a_rebuild_with_stable_on_the_newest_release_publishes_it_once(
+        self,
+    ) -> None:
+        self.commit("v2.3.0")
+        newest = self.commit("v2.4.0", "stable")
+        self.push_main()
+
+        self.assertEqual(
+            self.releases(""),
+            [{"sha": newest, "version": "2.4.0", "aliases": "latest 2 2.4 stable"}],
+        )
+
+    def test_a_rebuild_with_stable_behind_publishes_the_newest_and_the_stable_release(
+        self,
+    ) -> None:
+        self.commit("v2.2.0")
+        stable = self.commit("v2.3.0", "stable")
+        newest = self.commit("v2.4.0")
+        self.commit("v2.5.0-rc1")
+        self.push_main()
+
+        self.assertEqual(
+            self.releases(""),
+            [
+                {"sha": newest, "version": "2.4.0", "aliases": "latest 2 2.4"},
+                {"sha": stable, "version": "2.3.0", "aliases": "2.3 stable"},
+            ],
+        )
+
+    def test_the_stable_release_takes_no_prefix_a_newer_release_leads(self) -> None:
+        stable = self.commit("v2.3.0", "stable")
+        self.commit("v2.3.1")
+        newest = self.commit("v2.4.0")
+        self.push_main()
+
+        self.assertEqual(
+            self.releases(""),
+            [
+                {"sha": newest, "version": "2.4.0", "aliases": "latest 2 2.4"},
+                {"sha": stable, "version": "2.3.0", "aliases": "stable"},
+            ],
+        )
+
+    def test_a_rebuild_publishes_the_newest_release_on_main_not_its_tip(
+        self,
+    ) -> None:
+        self.commit("v1.9.0")
+        newest = self.commit("v1.10.0")
+        self.commit()
+        self.push_main()
+        self.git("checkout", "-q", "-b", "side")
+        self.commit("v3.0.0")
+
+        result, outputs = self.resolve("")
+
+        self.assertEqual(result.returncode, 0, result.stderr)
+        self.assertEqual(
+            json.loads(outputs["releases"]),
+            [{"sha": newest, "version": "1.10.0", "aliases": "latest 1 1.10"}],
+        )
+        self.assertIn("the stable image stays where it is", result.stdout)
+
+    def test_a_workflow_run_publishes_the_commit_it_tagged(self) -> None:
+        self.commit("v2.3.0")
+        tagged = self.commit("v2.4.0", "stable")
+        self.push_main()
+
+        self.assertEqual(
+            self.releases(tagged),
+            [{"sha": tagged, "version": "2.4.0", "aliases": "latest 2 2.4 stable"}],
+        )
+
+    def test_a_workflow_run_of_a_backport_moves_only_the_prefixes_it_leads(
+        self,
+    ) -> None:
+        self.commit("v2.3.0")
+        self.commit("v2.4.0", "stable")
+        backport = self.commit("v2.3.1")
+        self.push_main()
+
+        self.assertEqual(
+            self.releases(backport),
+            [{"sha": backport, "version": "2.3.1", "aliases": "2.3"}],
+        )
+
+    def test_a_workflow_run_outside_main_publishes_nothing(self) -> None:
+        self.commit("v2.3.0")
+        self.push_main()
+        self.git("checkout", "-q", "-b", "side")
+        outside = self.commit("v9.0.0")
+
+        self.assertEqual(self.releases(outside), [])
+
+    def test_a_workflow_run_without_an_x_y_z_tag_publishes_nothing(self) -> None:
+        self.commit("v2.3.0")
+        candidate = self.commit("v2.4.0-rc1")
+        self.push_main()
+
+        self.assertEqual(self.releases(candidate), [])
+
+    def test_a_rebuild_without_a_release_on_main_fails(self) -> None:
+        self.commit("v1.0.0-rc1", "stable")
+        self.push_main()
+
+        result, outputs = self.resolve("")
+
+        self.assertEqual(result.returncode, 1)
+        self.assertEqual(outputs, {})
+        self.assertIn("No vX.Y.Z tag is reachable from origin/main", result.stderr)
+
+
+class TestPublishContainersWorkflow(unittest.TestCase):
+    def setUp(self) -> None:
+        self.workflow = yaml.safe_load(WORKFLOW.read_text())
+
+    def test_the_releases_are_rebuilt_daily_at_03_utc_and_on_demand(self) -> None:
+        triggers = self.workflow[True]
+        steps = {
+            step.get("id"): step for step in self.workflow["jobs"]["prepare"]["steps"]
+        }
+
+        self.assertEqual(triggers["schedule"], [{"cron": "0 3 * * *"}])
+        self.assertIn("workflow_dispatch", triggers)
+        self.assertEqual(
+            steps["info"]["run"],
+            "bash scripts/github/publish-containers/compute-publish-container-info.sh",
+        )
+        self.assertEqual(
+            steps["info"]["env"]["WORKFLOW_RUN_SHA"],
+            "${{ github.event.workflow_run.head_sha }}",
+        )
+
+    def test_every_resolved_release_is_built_and_merged_with_its_own_tags(
+        self,
+    ) -> None:
+        jobs = self.workflow["jobs"]
+        merge_env = jobs["merge"]["steps"][-1]["env"]
+
+        for job in ("build", "merge"):
+            self.assertEqual(
+                jobs[job]["strategy"]["matrix"]["release"],
+                "${{ fromJSON(needs.prepare.outputs.releases) }}",
+            )
+            self.assertEqual(
+                jobs[job]["steps"][0]["with"]["ref"], "${{ matrix.release.sha }}"
+            )
+        self.assertEqual(merge_env["VERSION"], "${{ matrix.release.version }}")
+        self.assertEqual(merge_env["ALIASES"], "${{ matrix.release.aliases }}")
+
+
+if __name__ == "__main__":
+    unittest.main()