From e45b6ed4a56cb752c92351924721aba4123b587e Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Fri, 18 Sep 2026 15:51:25 +0200 Subject: [PATCH] refactor(docker): consume the base images instead of building them The `virgin` stage installed the distribution build dependencies, and that layer now lives in its own repository: https://github.com/kevinveenbirkenbach/base-images. What remains here is one image per distribution that installs pkgmgr on top of it. `slim` goes with it: it was published for every distribution and pulled by nothing. scripts/build/base.sh stops pinning five upstream images and resolves ghcr.io//base-: instead. Its env names follow the images repository rather than this one, so the code shows whose namespace that is. image.sh loses the --target axis, the -virgin/-slim tag suffixes with it, and passes --platform on push. Manjaro joins the set; os_resolver.sh already maps it onto arch, so it needs no dependency script of its own. The two virgin workflows pull the base image rather than building it. What they prove is unchanged: pkgmgr installs into an untouched container, as root and as an unprivileged user. test_distro_dependency_scripts_install_gpg_tools goes: it grepped the dependency scripts for gnupg, and those scripts are base-images' responsibility now, where the contract test runs `gpg --version` instead of reading a package list. Verified against locally built base images: the arch image builds FROM base-arch, and the full suite passes inside it - 405 unit, 90 integration and 49 e2e tests. Requires base-images to be published first; until then every build here fails at the pull. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/test-virgin-root.yml | 8 +-- .github/workflows/test-virgin-user.yml | 8 +-- Dockerfile | 56 +++---------------- Makefile | 24 ++------ scripts/build/base.sh | 32 +++++++---- scripts/build/image.sh | 29 +--------- scripts/build/publish.sh | 35 +----------- ...t_git_verification_runtime_dependencies.py | 28 ---------- 8 files changed, 48 insertions(+), 172 deletions(-) diff --git a/.github/workflows/test-virgin-root.yml b/.github/workflows/test-virgin-root.yml index ae66d6a..4eb55f4 100644 --- a/.github/workflows/test-virgin-root.yml +++ b/.github/workflows/test-virgin-root.yml @@ -24,10 +24,8 @@ jobs: - name: Show Docker version run: docker version - - name: Build virgin container (${{ matrix.distro }}) - run: | - set -euo pipefail - PKGMGR_DISTRO="${{ matrix.distro }}" make build-missing-virgin + - name: Pull base image (${{ matrix.distro }}) + run: docker pull "ghcr.io/kevinveenbirkenbach/base-${{ matrix.distro }}:latest" - name: Virgin ${{ matrix.distro }} pkgmgr test (root) run: | @@ -39,7 +37,7 @@ jobs: -v pkgmgr_pip_cache:/root/.cache/pip \ -e NIX_CONFIG="${NIX_CONFIG}" \ -w /opt/src/pkgmgr \ - "pkgmgr-${{ matrix.distro }}-virgin" \ + "ghcr.io/kevinveenbirkenbach/base-${{ matrix.distro }}:latest" \ bash -lc ' set -euo pipefail diff --git a/.github/workflows/test-virgin-user.yml b/.github/workflows/test-virgin-user.yml index e1aa92a..d872de0 100644 --- a/.github/workflows/test-virgin-user.yml +++ b/.github/workflows/test-virgin-user.yml @@ -24,10 +24,8 @@ jobs: - name: Show Docker version run: docker version - - name: Build virgin container (${{ matrix.distro }}) - run: | - set -euo pipefail - PKGMGR_DISTRO="${{ matrix.distro }}" make build-missing-virgin + - name: Pull base image (${{ matrix.distro }}) + run: docker pull "ghcr.io/kevinveenbirkenbach/base-${{ matrix.distro }}:latest" - name: Virgin ${{ matrix.distro }} pkgmgr test (user) run: | @@ -37,7 +35,7 @@ jobs: -v "$PWD":/opt/src/pkgmgr \ -e NIX_CONFIG="${NIX_CONFIG}" \ -w /opt/src/pkgmgr \ - "pkgmgr-${{ matrix.distro }}-virgin" \ + "ghcr.io/kevinveenbirkenbach/base-${{ matrix.distro }}:latest" \ bash -lc ' set -euo pipefail diff --git a/Dockerfile b/Dockerfile index 9ad35e2..418329f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,67 +1,29 @@ # syntax=docker/dockerfile:1 -# ------------------------------------------------------------ -# Base image selector — overridden by build args / Makefile -# ------------------------------------------------------------ +# One image, built on the distribution base published by +# https://github.com/kevinveenbirkenbach/base-images, which owns the build +# dependencies this used to install in a `virgin` stage of its own. +# +# hadolint ignore=DL3006,InvalidDefaultArgInFrom +# BASE_IMAGE carries no default on purpose: a default would build one +# distribution's image under another distribution's tag whenever the build arg +# is forgotten. scripts/build/base.sh is the only place that resolves it. ARG BASE_IMAGE +FROM ${BASE_IMAGE} -# ============================================================ -# Target: virgin -# - installs distro deps (incl. make) -# - no pkgmgr build -# - no entrypoint -# ============================================================ -FROM ${BASE_IMAGE} AS virgin SHELL ["/bin/bash", "-lc"] -RUN echo "BASE_IMAGE=${BASE_IMAGE}" && cat /etc/os-release || true - WORKDIR /build -# Copy scripts first so dependency installation can be cached -COPY scripts/installation/ scripts/installation/ - -# Install distro-specific build dependencies (including make) -RUN bash scripts/installation/dependencies.sh - -# Virgin default -CMD ["bash"] - - -# ============================================================ -# Target: full -# - inherits from virgin -# - builds + installs pkgmgr -# - sets entrypoint + default cmd -# - NOTE: does NOT run slim.sh (that is done in slim stage) -# ============================================================ -FROM virgin AS full - -WORKDIR /build - -# Copy full repository for build COPY . . -# Build and install distro-native package-manager package RUN set -eu; \ echo "Building and installing package-manager via make install..."; \ make install; \ rm -rf /build -# Entry point COPY scripts/docker/entry.sh /usr/local/bin/docker-entry.sh WORKDIR /opt/src/pkgmgr ENTRYPOINT ["/usr/local/bin/docker-entry.sh"] CMD ["pkgmgr", "--help"] - - -# ============================================================ -# Target: slim -# - based on full -# - runs slim.sh -# ============================================================ -FROM full AS slim - -COPY scripts/docker/slim.sh /usr/local/bin/slim.sh -RUN chmod +x /usr/local/bin/slim.sh && /usr/local/bin/slim.sh diff --git a/Makefile b/Makefile index f33e17d..14bab8f 100644 --- a/Makefile +++ b/Makefile @@ -17,20 +17,11 @@ export NIX_CONFIG # ------------------------------------------------------------ # Base images -# (kept for documentation/reference; actual build logic is in scripts/build) +# Published by https://github.com/kevinveenbirkenbach/base-images and resolved +# by scripts/build/base.sh; override the registry, owner or tag there. # ------------------------------------------------------------ -BASE_IMAGE_ARCH := archlinux:latest -BASE_IMAGE_DEBIAN := debian:stable-slim -BASE_IMAGE_UBUNTU := ubuntu:latest -BASE_IMAGE_FEDORA := fedora:latest -BASE_IMAGE_CENTOS := quay.io/centos/centos:stream9 - -# Make them available in scripts -export BASE_IMAGE_ARCH -export BASE_IMAGE_DEBIAN -export BASE_IMAGE_UBUNTU -export BASE_IMAGE_FEDORA -export BASE_IMAGE_CENTOS +BASE_IMAGES_TAG ?= latest +export BASE_IMAGES_TAG # PYthon Unittest Pattern TEST_PATTERN := test_*.py @@ -63,17 +54,12 @@ setup-nix: # Docker build targets (delegated to scripts/build) # ------------------------------------------------------------ build: - @bash scripts/build/image.sh --target virgin @bash scripts/build/image.sh -build-missing-virgin: - @bash scripts/build/image.sh --target virgin --missing - -build-missing: build-missing-virgin +build-missing: @bash scripts/build/image.sh --missing build-no-cache: - @bash scripts/build/image.sh --target virgin --no-cache @bash scripts/build/image.sh --no-cache build-no-cache-all: diff --git a/scripts/build/base.sh b/scripts/build/base.sh index ba90592..f5dbd02 100755 --- a/scripts/build/base.sh +++ b/scripts/build/base.sh @@ -1,20 +1,32 @@ #!/usr/bin/env bash set -euo pipefail -: "${BASE_IMAGE_ARCH:=archlinux:latest}" -: "${BASE_IMAGE_DEBIAN:=debian:stable-slim}" -: "${BASE_IMAGE_UBUNTU:=ubuntu:latest}" -: "${BASE_IMAGE_FEDORA:=fedora:latest}" -: "${BASE_IMAGE_CENTOS:=quay.io/centos/centos:latest}" +# The distribution bases this repository builds on. They are published by +# https://github.com/kevinveenbirkenbach/base-images, one multi-platform image +# per distribution, and carry the build dependencies the package build needs. +# +# Env overrides exist for testing an unpublished base; the registry namespace +# follows the images repository, not this one. + +: "${BASE_IMAGES_REGISTRY:=ghcr.io}" +: "${BASE_IMAGES_OWNER:=kevinveenbirkenbach}" +: "${BASE_IMAGES_TAG:=latest}" resolve_base_image() { local PKGMGR_DISTRO="$1" case "$PKGMGR_DISTRO" in - arch) echo "$BASE_IMAGE_ARCH" ;; - debian) echo "$BASE_IMAGE_DEBIAN" ;; - ubuntu) echo "$BASE_IMAGE_UBUNTU" ;; - fedora) echo "$BASE_IMAGE_FEDORA" ;; - centos) echo "$BASE_IMAGE_CENTOS" ;; + arch|manjaro|debian|ubuntu|fedora|centos) + echo "${BASE_IMAGES_REGISTRY}/${BASE_IMAGES_OWNER}/base-${PKGMGR_DISTRO}:${BASE_IMAGES_TAG}" + ;; + *) echo "ERROR: Unknown distro '$PKGMGR_DISTRO'" >&2; exit 1 ;; + esac +} + +# Platforms each distribution's base offers; see base-images/scripts/build/distros.sh. +resolve_platforms() { + local PKGMGR_DISTRO="$1" + case "$PKGMGR_DISTRO" in + arch|manjaro|debian|ubuntu|fedora|centos) echo "linux/amd64,linux/arm64" ;; *) echo "ERROR: Unknown distro '$PKGMGR_DISTRO'" >&2; exit 1 ;; esac } diff --git a/scripts/build/image.sh b/scripts/build/image.sh index e868ad9..d3ab267 100755 --- a/scripts/build/image.sh +++ b/scripts/build/image.sh @@ -10,7 +10,6 @@ source "${SCRIPT_DIR}/base.sh" NO_CACHE=0 MISSING_ONLY=0 -TARGET="" IMAGE_TAG="" # local image name or base tag (without registry) PUSH=0 # if 1 -> use buildx and push (requires docker buildx) PUBLISH=0 # if 1 -> push with semantic tags (latest/version/stable + arch aliases) @@ -23,9 +22,6 @@ DEFAULT_DISTRO="arch" usage() { local default_tag="pkgmgr-${PKGMGR_DISTRO}" - if [[ -n "${TARGET:-}" ]]; then - default_tag="${default_tag}-${TARGET}" - fi cat < $0 [options] @@ -33,7 +29,6 @@ Usage: PKGMGR_DISTRO= $0 [options] Build options: --missing Build only if the image does not already exist (local build only) --no-cache Build with --no-cache - --target Build a specific Dockerfile target (e.g. virgin, slim) --tag Override the output image tag (default: ${default_tag}) Publish options: @@ -47,7 +42,7 @@ Publish options: Notes: - --publish implies --push and requires --registry, --owner, and --version. -- Local build (no --push) uses "docker build" and creates local images like "pkgmgr-arch" / "pkgmgr-arch-virgin" / "pkgmgr-arch-slim". +- Local build (no --push) uses "docker build" and creates a local image like "pkgmgr-arch". EOF } @@ -55,11 +50,6 @@ while [[ $# -gt 0 ]]; do case "$1" in --no-cache) NO_CACHE=1; shift ;; --missing) MISSING_ONLY=1; shift ;; - --target) - TARGET="${2:-}" - [[ -n "${TARGET}" ]] || { echo "ERROR: --target requires a value (e.g. virgin|slim)"; exit 2; } - shift 2 - ;; --tag) IMAGE_TAG="${2:-}" [[ -n "${IMAGE_TAG}" ]] || { echo "ERROR: --tag requires a value"; exit 2; } @@ -104,9 +94,6 @@ done # Derive default local tag if not provided if [[ -z "${IMAGE_TAG}" ]]; then IMAGE_TAG="${REPO_PREFIX}-${PKGMGR_DISTRO}" - if [[ -n "${TARGET}" ]]; then - IMAGE_TAG="${IMAGE_TAG}-${TARGET}" - fi fi BASE_IMAGE="$(resolve_base_image "$PKGMGR_DISTRO")" @@ -143,7 +130,7 @@ echo "------------------------------------------------------------" echo "[build] Building image" echo "distro = ${PKGMGR_DISTRO}" echo "BASE_IMAGE = ${BASE_IMAGE}" -if [[ -n "${TARGET}" ]]; then echo "target = ${TARGET}"; fi +echo "platforms = $(resolve_platforms "$PKGMGR_DISTRO")" if [[ "${NO_CACHE}" == "1" ]]; then echo "cache = disabled"; fi if [[ "${PUSH}" == "1" ]]; then echo "push = enabled"; fi if [[ "${PUBLISH}" == "1" ]]; then @@ -162,23 +149,12 @@ if [[ "${NO_CACHE}" == "1" ]]; then build_args+=(--no-cache) fi -if [[ -n "${TARGET}" ]]; then - build_args+=(--target "${TARGET}") -fi - compute_publish_tags() { local distro_tag_base="${REGISTRY}/${OWNER}/${REPO_PREFIX}-${PKGMGR_DISTRO}" local alias_tag_base="" - if [[ -n "${TARGET}" ]]; then - distro_tag_base="${distro_tag_base}-${TARGET}" - fi - if [[ "${PKGMGR_DISTRO}" == "${DEFAULT_DISTRO}" ]]; then alias_tag_base="${REGISTRY}/${OWNER}/${REPO_PREFIX}" - if [[ -n "${TARGET}" ]]; then - alias_tag_base="${alias_tag_base}-${TARGET}" - fi fi local tags=() @@ -211,6 +187,7 @@ if [[ "${PUSH}" == "1" ]]; then bx_args+=(-t "${IMAGE_TAG}") fi + bx_args+=(--platform "$(resolve_platforms "$PKGMGR_DISTRO")") bx_args+=("${build_args[@]}") bx_args+=(.) diff --git a/scripts/build/publish.sh b/scripts/build/publish.sh index 5f20410..b1be3c9 100755 --- a/scripts/build/publish.sh +++ b/scripts/build/publish.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -euo pipefail -# Publish all distro images (full + virgin + slim) to a registry via image.sh --publish +# Publish one image per distro, each covering every platform its base offers. # # Required env: # OWNER (e.g. GITHUB_REPOSITORY_OWNER) @@ -12,8 +12,8 @@ set -euo pipefail # IS_STABLE (default: false) # DISTROS (default: "arch debian ubuntu fedora centos") # -# Notes: -# - This expects Dockerfile targets: virgin, full (default), slim +# The `virgin` variant moved to https://github.com/kevinveenbirkenbach/base-images +# and `slim` is gone: one image per distribution, no target to pick. SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" @@ -36,41 +36,12 @@ for d in ${DISTROS}; do echo "[publish] PKGMGR_DISTRO=${d}" echo "============================================================" - # ---------------------------------------------------------- - # virgin - # -> ghcr.io//pkgmgr--virgin:{latest,,stable?} - # ---------------------------------------------------------- - PKGMGR_DISTRO="${d}" bash "${SCRIPT_DIR}/image.sh" \ - --publish \ - --registry "${REGISTRY}" \ - --owner "${OWNER}" \ - --version "${VERSION}" \ - --stable "${IS_STABLE}" \ - --target virgin - - # ---------------------------------------------------------- - # full (default target) - # -> ghcr.io//pkgmgr-:{latest,,stable?} - # ---------------------------------------------------------- PKGMGR_DISTRO="${d}" bash "${SCRIPT_DIR}/image.sh" \ --publish \ --registry "${REGISTRY}" \ --owner "${OWNER}" \ --version "${VERSION}" \ --stable "${IS_STABLE}" - - # ---------------------------------------------------------- - # slim - # -> ghcr.io//pkgmgr--slim:{latest,,stable?} - # + alias for default distro: ghcr.io//pkgmgr-slim:{...} - # ---------------------------------------------------------- - PKGMGR_DISTRO="${d}" bash "${SCRIPT_DIR}/image.sh" \ - --publish \ - --registry "${REGISTRY}" \ - --owner "${OWNER}" \ - --version "${VERSION}" \ - --stable "${IS_STABLE}" \ - --target slim done echo diff --git a/tests/integration/test_git_verification_runtime_dependencies.py b/tests/integration/test_git_verification_runtime_dependencies.py index a9bf360..6c3a017 100644 --- a/tests/integration/test_git_verification_runtime_dependencies.py +++ b/tests/integration/test_git_verification_runtime_dependencies.py @@ -1,6 +1,5 @@ from __future__ import annotations -import re import unittest from pathlib import Path @@ -25,33 +24,6 @@ class TestGitVerificationRuntimeDependencies(unittest.TestCase): self.assertIn("pkgs.git", flake_text) self.assertIn("pkgs.gnupg", flake_text) - def test_distro_dependency_scripts_install_gpg_tools(self) -> None: - repo_root = _find_repo_root() - expected_packages = { - "arch": "gnupg", - "debian": "gnupg", - "ubuntu": "gnupg", - "fedora": "gnupg2", - "centos": "gnupg2", - } - - missing: list[str] = [] - for distro, package_name in expected_packages.items(): - script_path = ( - repo_root / "scripts" / "installation" / distro / "dependencies.sh" - ) - content = script_path.read_text(encoding="utf-8") - if not re.search(rf"\b{re.escape(package_name)}\b", content): - missing.append( - f"{distro}: expected package {package_name} in {script_path}" - ) - - if missing: - self.fail( - "Git signature verification runtime dependencies are incomplete:\n" - + "\n".join(f" - {item}" for item in missing) - ) - if __name__ == "__main__": unittest.main()