From 0dfbaa0f6b275fbc1ce9df8f1f782c0d721de4a4 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Fri, 12 Dec 2025 16:40:21 +0100 Subject: [PATCH] ci/docker: unify image build logic and run virgin tests across all distros Refactor Dockerfile into multi-stage virgin/full targets and introduce a single flag-based image build script. Standardize image naming, remove redundant build scripts, and update Makefile targets accordingly. CI workflows now build missing virgin images and run root and user tests consistently across all supported distributions. https://chatgpt.com/share/693c29d9-9b28-800f-a549-5661c783d968 --- .github/workflows/test-virgin-root.yml | 19 ++- .github/workflows/test-virgin-user.yml | 26 ++-- Dockerfile | 79 ++++++------ Makefile | 17 ++- .../build/{resolve-base-image.sh => base.sh} | 0 scripts/build/build-image-missing.sh | 24 ---- scripts/build/build-image-no-cache.sh | 15 --- scripts/build/build-image.sh | 14 -- scripts/build/image.sh | 120 ++++++++++++++++++ scripts/setup/nix.sh | 0 scripts/setup/venv.sh | 0 scripts/test/test-e2e.sh | 2 +- scripts/test/test-env-nix.sh | 2 +- scripts/test/test-env-virtual.sh | 2 +- scripts/test/test-integration.sh | 2 +- scripts/test/test-unit.sh | 2 +- 16 files changed, 200 insertions(+), 124 deletions(-) rename scripts/build/{resolve-base-image.sh => base.sh} (100%) delete mode 100755 scripts/build/build-image-missing.sh delete mode 100755 scripts/build/build-image-no-cache.sh delete mode 100755 scripts/build/build-image.sh create mode 100755 scripts/build/image.sh mode change 100644 => 100755 scripts/setup/nix.sh mode change 100644 => 100755 scripts/setup/venv.sh mode change 100644 => 100755 scripts/test/test-env-nix.sh diff --git a/.github/workflows/test-virgin-root.yml b/.github/workflows/test-virgin-root.yml index ba1e976..dbc90b4 100644 --- a/.github/workflows/test-virgin-root.yml +++ b/.github/workflows/test-virgin-root.yml @@ -7,6 +7,10 @@ jobs: test-virgin-root: runs-on: ubuntu-latest timeout-minutes: 45 + strategy: + fail-fast: false + matrix: + distro: [arch, debian, ubuntu, fedora, centos] steps: - name: Checkout repository @@ -15,7 +19,14 @@ jobs: - name: Show Docker version run: docker version - - name: Virgin Arch pkgmgr flake test (root) + # 🔹 BUILD virgin image if missing + - name: Build virgin container (${{ matrix.distro }}) + run: | + set -euo pipefail + distro="${{ matrix.distro }}" make build-missing + + # 🔹 RUN test inside virgin image + - name: Virgin ${{ matrix.distro }} pkgmgr test (root) run: | set -euo pipefail @@ -24,13 +35,10 @@ jobs: -v pkgmgr_repos:/root/Repositories \ -v pkgmgr_pip_cache:/root/.cache/pip \ -w /src \ - archlinux:latest \ + "pkgmgr-${{ matrix.distro }}-virgin" \ bash -lc ' set -euo pipefail - pacman -Syu --noconfirm git python python-pip nix make - - # Fix: allow git operations on mounted repo path git config --global --add safe.directory /src make install @@ -46,4 +54,3 @@ jobs: echo ">>> Running Nix-based: nix run .#pkgmgr -- version pkgmgr" nix run /src#pkgmgr -- version pkgmgr ' - diff --git a/.github/workflows/test-virgin-user.yml b/.github/workflows/test-virgin-user.yml index a981f6f..38f965d 100644 --- a/.github/workflows/test-virgin-user.yml +++ b/.github/workflows/test-virgin-user.yml @@ -7,6 +7,10 @@ jobs: test-virgin-user: runs-on: ubuntu-latest timeout-minutes: 45 + strategy: + fail-fast: false + matrix: + distro: [arch, debian, ubuntu, fedora, centos] steps: - name: Checkout repository @@ -15,19 +19,24 @@ jobs: - name: Show Docker version run: docker version - - name: Virgin Arch pkgmgr user test (non-root with sudo) + # 🔹 BUILD virgin image if missing + - name: Build virgin container (${{ matrix.distro }}) + run: | + set -euo pipefail + distro="${{ matrix.distro }}" make build-missing + + # 🔹 RUN test inside virgin image as non-root + - name: Virgin ${{ matrix.distro }} pkgmgr test (user) run: | set -euo pipefail docker run --rm \ -v "$PWD":/src \ -w /src \ - archlinux:latest \ + "pkgmgr-${{ matrix.distro }}-virgin" \ bash -lc ' set -euo pipefail - pacman -Syu --noconfirm git python python-pip sudo base-devel debugedit nix make - make install useradd -m dev @@ -35,7 +44,6 @@ jobs: chmod 0440 /etc/sudoers.d/dev chown -R dev:dev /src - # --- make Nix usable for non-root inside this container --- mkdir -p /nix/store /nix/var/nix /nix/var/log/nix /nix/var/nix/profiles chown -R dev:dev /nix chmod 0755 /nix @@ -45,21 +53,13 @@ jobs: set -euo pipefail cd /src - echo \">>> [dev] Using user: \$(whoami)\" - - echo \">>> [dev] Running make setup-venv...\" make setup-venv - - echo \">>> [dev] Activating venv...\" . \"\$HOME/.venvs/pkgmgr/bin/activate\" - echo \">>> [dev] Running: pkgmgr version pkgmgr\" pkgmgr version pkgmgr - echo \">>> [dev] Running Nix-based pkgmgr version...\" export NIX_REMOTE=local export NIX_CONFIG=\"experimental-features = nix-command flakes\" nix run /src#pkgmgr -- version pkgmgr " ' - diff --git a/Dockerfile b/Dockerfile index 6443b15..908ce53 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,54 +1,57 @@ +# syntax=docker/dockerfile:1 + # ------------------------------------------------------------ -# Base image selector — overridden by Makefile +# Base image selector — overridden by build args / Makefile # ------------------------------------------------------------ ARG BASE_IMAGE -FROM ${BASE_IMAGE} -RUN echo "BASE_IMAGE=${BASE_IMAGE}" && \ - cat /etc/os-release || true +# ============================================================ +# Target: virgin +# - installs distro deps (incl. make) +# - no pkgmgr build +# - no entrypoint +# ============================================================ +FROM ${BASE_IMAGE} AS virgin -# ------------------------------------------------------------ -# Nix environment defaults -# -# Nix itself is installed by your system packages (via init-nix.sh). -# Here we only define default configuration options. -# ------------------------------------------------------------ -ENV NIX_CONFIG="experimental-features = nix-command flakes" +RUN echo "BASE_IMAGE=${BASE_IMAGE}" && cat /etc/os-release || true - -# ------------------------------------------------------------ -# Copy scripts and install distro dependencies -# ------------------------------------------------------------ WORKDIR /build -# Copy only scripts first so dependency installation can run early -COPY scripts/ scripts/ -RUN find scripts -type f -name '*.sh' -exec chmod +x {} \; +# Copy scripts first so dependency installation can be cached +COPY scripts/installation/ scripts/installation/ -# ------------------------------------------------------------ -# Select distro-specific Docker entrypoint -# ------------------------------------------------------------ -# Docker entrypoint (distro-agnostic, nutzt package.sh) -# ------------------------------------------------------------ -COPY scripts/docker/entry.sh /usr/local/bin/docker-entry.sh -RUN chmod +x /usr/local/bin/docker-entry.sh +# Install distro-specific build dependencies (including make) +RUN bash scripts/installation/dependencies.sh -# ------------------------------------------------------------ -# Build and install distro-native package-manager package -# via Makefile `install` target -# ------------------------------------------------------------ +# Virgin default +CMD ["bash"] + + +# ============================================================ +# Target: full +# - inherits from virgin +# - builds + installs pkgmgr +# - sets entrypoint + default cmd +# ============================================================ +FROM virgin AS full + +# Nix environment defaults (only config; nix itself comes from deps/install flow) +ENV NIX_CONFIG="experimental-features = nix-command flakes" + +WORKDIR /build + +# Copy full repository for build COPY . . -RUN find scripts -type f -name '*.sh' -exec chmod +x {} \; -RUN set -e; \ - echo "Building and installing package-manager via make install..."; \ - make install; \ - rm -rf /build +# Build and install distro-native package-manager package +RUN set -euo pipefail; \ + echo "Building and installing package-manager via make install..."; \ + make install; \ + cd /; rm -rf /build + +# Entry point +COPY scripts/docker/entry.sh /usr/local/bin/docker-entry.sh -# ------------------------------------------------------------ -# Runtime working directory and dev entrypoint -# ------------------------------------------------------------ WORKDIR /src - ENTRYPOINT ["/usr/local/bin/docker-entry.sh"] CMD ["pkgmgr", "--help"] diff --git a/Makefile b/Makefile index 333ef53..bfb578a 100644 --- a/Makefile +++ b/Makefile @@ -57,10 +57,16 @@ setup-nix: # Docker build targets (delegated to scripts/build) # ------------------------------------------------------------ build: - @bash scripts/build/build-image.sh + @bash scripts/build/image.sh --target virgin + @bash scripts/build/image.sh + +build-missing: + @bash scripts/build/image.sh --target virgin --missing + @bash scripts/build/image.sh --missing build-no-cache: - @bash scripts/build/build-image-no-cache.sh + @bash scripts/build/image.sh --target virgin --no-cache + @bash scripts/build/image.sh --no-cache build-no-cache-all: @set -e; \ @@ -88,13 +94,6 @@ test-env-virtual: build-missing test-env-nix: build-missing @bash scripts/test/test-env-nix.sh - -# ------------------------------------------------------------ -# Build only missing container images -# ------------------------------------------------------------ -build-missing: - @bash scripts/build/build-image-missing.sh - # Combined test target for local + CI (unit + integration + e2e) test: test-env-virtual test-unit test-integration test-e2e diff --git a/scripts/build/resolve-base-image.sh b/scripts/build/base.sh similarity index 100% rename from scripts/build/resolve-base-image.sh rename to scripts/build/base.sh diff --git a/scripts/build/build-image-missing.sh b/scripts/build/build-image-missing.sh deleted file mode 100755 index 0e9f997..0000000 --- a/scripts/build/build-image-missing.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" -source "${SCRIPT_DIR}/resolve-base-image.sh" - -IMAGE="package-manager-test-$distro" -BASE_IMAGE="$(resolve_base_image "$distro")" - -if docker image inspect "$IMAGE" >/dev/null 2>&1; then - echo "[build-missing] Image already exists: $IMAGE (skipping)" - exit 0 -fi - -echo -echo "------------------------------------------------------------" -echo "[build-missing] Building missing image: $IMAGE" -echo "BASE_IMAGE = $BASE_IMAGE" -echo "------------------------------------------------------------" - -docker build \ - --build-arg BASE_IMAGE="$BASE_IMAGE" \ - -t "$IMAGE" \ - . diff --git a/scripts/build/build-image-no-cache.sh b/scripts/build/build-image-no-cache.sh deleted file mode 100755 index 839039a..0000000 --- a/scripts/build/build-image-no-cache.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" -source "${SCRIPT_DIR}/resolve-base-image.sh" - -base_image="$(resolve_base_image "$distro")" - -echo ">>> Building test image for distro '$distro' with NO CACHE (BASE_IMAGE=$base_image)..." - -docker build \ - --no-cache \ - --build-arg BASE_IMAGE="$base_image" \ - -t "package-manager-test-$distro" \ - . diff --git a/scripts/build/build-image.sh b/scripts/build/build-image.sh deleted file mode 100755 index 44598fa..0000000 --- a/scripts/build/build-image.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" -source "${SCRIPT_DIR}/resolve-base-image.sh" - -base_image="$(resolve_base_image "$distro")" - -echo ">>> Building test image for distro '$distro' (BASE_IMAGE=$base_image)..." - -docker build \ - --build-arg BASE_IMAGE="$base_image" \ - -t "package-manager-test-$distro" \ - . diff --git a/scripts/build/image.sh b/scripts/build/image.sh new file mode 100755 index 0000000..e3f6b5d --- /dev/null +++ b/scripts/build/image.sh @@ -0,0 +1,120 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Unified docker image builder for all distros. +# +# Supports: +# --missing Build only if image does not exist +# --no-cache Disable docker layer cache +# --target Dockerfile target (e.g. virgin|full) +# --tag Override image tag (default: pkgmgr-$distro[-$target]) +# +# Requires: +# - env var: distro (arch|debian|ubuntu|fedora|centos) +# - base.sh in same dir +# +# Examples: +# distro=arch bash scripts/build/image.sh +# distro=arch bash scripts/build/image.sh --no-cache +# distro=arch bash scripts/build/image.sh --missing +# distro=arch bash scripts/build/image.sh --target virgin +# distro=arch bash scripts/build/image.sh --target virgin --missing +# distro=arch bash scripts/build/image.sh --tag myimg:arch + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/base.sh" + +: "${distro:?Environment variable 'distro' must be set (arch|debian|ubuntu|fedora|centos)}" + +NO_CACHE=0 +MISSING_ONLY=0 +TARGET="" +IMAGE_TAG="" # derive later unless --tag is provided + +usage() { + local default_tag="pkgmgr-${distro}" + if [[ -n "${TARGET:-}" ]]; then + default_tag="${default_tag}-${TARGET}" + fi + + cat < $0 [--missing] [--no-cache] [--target ] [--tag ] + +Options: + --missing Build only if the image does not already exist + --no-cache Build with --no-cache + --target Build a specific Dockerfile target (e.g. virgin|full) + --tag Override the output image tag (default: ${default_tag}) + -h, --help Show help +EOF +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --no-cache) NO_CACHE=1; shift ;; + --missing) MISSING_ONLY=1; shift ;; + --target) + TARGET="${2:-}" + if [[ -z "${TARGET}" ]]; then + echo "ERROR: --target requires a value (e.g. virgin|full)" >&2 + exit 2 + fi + shift 2 + ;; + --tag) + IMAGE_TAG="${2:-}" + if [[ -z "${IMAGE_TAG}" ]]; then + echo "ERROR: --tag requires a value" >&2 + exit 2 + fi + shift 2 + ;; + -h|--help) usage; exit 0 ;; + *) + echo "ERROR: Unknown argument: $1" >&2 + usage + exit 2 + ;; + esac +done + +# Auto-tag: if --tag not provided, derive from distro (+ target suffix) +if [[ -z "${IMAGE_TAG}" ]]; then + IMAGE_TAG="pkgmgr-${distro}" + if [[ -n "${TARGET}" ]]; then + IMAGE_TAG="${IMAGE_TAG}-${TARGET}" + fi +fi + +BASE_IMAGE="$(resolve_base_image "$distro")" + +if [[ "${MISSING_ONLY}" == "1" ]]; then + if docker image inspect "${IMAGE_TAG}" >/dev/null 2>&1; then + echo "[build] Image already exists: ${IMAGE_TAG} (skipping due to --missing)" + exit 0 + fi +fi + +echo +echo "------------------------------------------------------------" +echo "[build] Building image: ${IMAGE_TAG}" +echo "distro = ${distro}" +echo "BASE_IMAGE = ${BASE_IMAGE}" +if [[ -n "${TARGET}" ]]; then echo "target = ${TARGET}"; fi +if [[ "${NO_CACHE}" == "1" ]]; then echo "cache = disabled"; fi +echo "------------------------------------------------------------" + +build_args=(--build-arg "BASE_IMAGE=${BASE_IMAGE}") + +if [[ "${NO_CACHE}" == "1" ]]; then + build_args+=(--no-cache) +fi + +if [[ -n "${TARGET}" ]]; then + build_args+=(--target "${TARGET}") +fi + +build_args+=(-t "${IMAGE_TAG}" .) + +docker build "${build_args[@]}" diff --git a/scripts/setup/nix.sh b/scripts/setup/nix.sh old mode 100644 new mode 100755 diff --git a/scripts/setup/venv.sh b/scripts/setup/venv.sh old mode 100644 new mode 100755 diff --git a/scripts/test/test-e2e.sh b/scripts/test/test-e2e.sh index 6e5bd6a..0ffaf97 100755 --- a/scripts/test/test-e2e.sh +++ b/scripts/test/test-e2e.sh @@ -12,7 +12,7 @@ docker run --rm \ -e REINSTALL_PKGMGR=1 \ -e TEST_PATTERN="${TEST_PATTERN}" \ --workdir /src \ - "package-manager-test-${distro}" \ + "pkgmgr-${distro}" \ bash -lc ' set -euo pipefail diff --git a/scripts/test/test-env-nix.sh b/scripts/test/test-env-nix.sh old mode 100644 new mode 100755 index 14dd59e..268fdc2 --- a/scripts/test/test-env-nix.sh +++ b/scripts/test/test-env-nix.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -euo pipefail -IMAGE="package-manager-test-${distro}" +IMAGE="pkgmgr-${distro}" echo "============================================================" echo ">>> Running Nix flake-only test in ${distro} container" diff --git a/scripts/test/test-env-virtual.sh b/scripts/test/test-env-virtual.sh index a50662e..3f00cfd 100755 --- a/scripts/test/test-env-virtual.sh +++ b/scripts/test/test-env-virtual.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -euo pipefail -IMAGE="package-manager-test-$distro" +IMAGE="pkgmgr-$distro" echo echo "------------------------------------------------------------" diff --git a/scripts/test/test-integration.sh b/scripts/test/test-integration.sh index a4fa60e..a0330ce 100755 --- a/scripts/test/test-integration.sh +++ b/scripts/test/test-integration.sh @@ -12,7 +12,7 @@ docker run --rm \ --workdir /src \ -e REINSTALL_PKGMGR=1 \ -e TEST_PATTERN="${TEST_PATTERN}" \ - "package-manager-test-${distro}" \ + "pkgmgr-${distro}" \ bash -lc ' set -e; git config --global --add safe.directory /src || true; diff --git a/scripts/test/test-unit.sh b/scripts/test/test-unit.sh index 5b8ec32..4af6222 100755 --- a/scripts/test/test-unit.sh +++ b/scripts/test/test-unit.sh @@ -12,7 +12,7 @@ docker run --rm \ --workdir /src \ -e REINSTALL_PKGMGR=1 \ -e TEST_PATTERN="${TEST_PATTERN}" \ - "package-manager-test-${distro}" \ + "pkgmgr-${distro}" \ bash -lc ' set -e; git config --global --add safe.directory /src || true;