From 5ca1adda7bdb9b9914c49501322844cabf133ca7 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Thu, 11 Dec 2025 14:48:36 +0100 Subject: [PATCH] Refactor CI distro handling and container build scripts - Introduce a GitHub Actions matrix for `test-container` and `test-e2e` to run against arch, debian, ubuntu, fedora, and centos - Run unit and integration tests only in the Arch container by passing `distro="arch"` via make in the corresponding workflows - Replace the global DISTROS loop with a single `distro` variable in the Makefile, defaulting to `arch`, and export it for all scripts - Update build scripts (build-image, build-image-no-cache, build-image-missing) to build images for the selected distro only - Simplify test-container script to validate a single distro image using the `distro` environment variable - Simplify E2E, unit, and integration test scripts to run against a single distro container instead of iterating over all distros https://chatgpt.com/share/693acbba-9e30-800f-94fb-fea4489e9078 --- .github/workflows/test-container.yml | 10 ++- .github/workflows/test-e2e.yml | 12 ++- .github/workflows/test-integration.yml | 2 +- .github/workflows/test-unit.yml | 2 +- Makefile | 9 ++- scripts/build/build-image-missing.sh | 41 ++++------ scripts/build/build-image-no-cache.sh | 16 ++-- scripts/build/build-image.sh | 14 ++-- scripts/test/test-container.sh | 61 +++++++-------- scripts/test/test-e2e.sh | 100 ++++++++++++------------- scripts/test/test-integration.sh | 2 - scripts/test/test-unit.sh | 2 - 12 files changed, 126 insertions(+), 145 deletions(-) diff --git a/.github/workflows/test-container.yml b/.github/workflows/test-container.yml index 911674f..a60c327 100644 --- a/.github/workflows/test-container.yml +++ b/.github/workflows/test-container.yml @@ -7,6 +7,10 @@ jobs: test-container: runs-on: ubuntu-latest timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + distro: [arch, debian, ubuntu, fedora, centos] steps: - name: Checkout repository @@ -15,5 +19,7 @@ jobs: - name: Show Docker version run: docker version - - name: Run container tests - run: make test-container + - name: Run container tests (${{ matrix.distro }}) + run: | + set -euo pipefail + distro="${{ matrix.distro }}" make test-container diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml index d5adc63..384b6fe 100644 --- a/.github/workflows/test-e2e.yml +++ b/.github/workflows/test-e2e.yml @@ -6,7 +6,11 @@ on: jobs: test-e2e: runs-on: ubuntu-latest - timeout-minutes: 60 # E2E + all distros can be heavier + timeout-minutes: 60 # E2E can be heavier + strategy: + fail-fast: false + matrix: + distro: [arch, debian, ubuntu, fedora, centos] steps: - name: Checkout repository @@ -15,5 +19,7 @@ jobs: - name: Show Docker version run: docker version - - name: Run E2E tests via make (all distros) - run: make test-e2e + - name: Run E2E tests via make (${{ matrix.distro }}) + run: | + set -euo pipefail + distro="${{ matrix.distro }}" make test-e2e diff --git a/.github/workflows/test-integration.yml b/.github/workflows/test-integration.yml index 438fa7d..78f090f 100644 --- a/.github/workflows/test-integration.yml +++ b/.github/workflows/test-integration.yml @@ -16,4 +16,4 @@ jobs: run: docker version - name: Run integration tests via make (Arch container) - run: make test-integration DISTROS="arch" + run: make test-integration distro="arch" diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index e776bd0..408173a 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -16,4 +16,4 @@ jobs: run: docker version - name: Run unit tests via make (Arch container) - run: make test-unit DISTROS="arch" + run: make test-unit distro="arch" diff --git a/Makefile b/Makefile index 06edc1e..abb545c 100644 --- a/Makefile +++ b/Makefile @@ -2,11 +2,15 @@ test build build-no-cache test-unit test-e2e test-integration \ test-container +# Distro +# Options: arch debian ubuntu fedora centos +distro ?= arch +export distro + # ------------------------------------------------------------ -# Distro list and base images +# Base images # (kept for documentation/reference; actual build logic is in scripts/build) # ------------------------------------------------------------ -DISTROS := arch debian ubuntu fedora centos BASE_IMAGE_ARCH := archlinux:latest BASE_IMAGE_DEBIAN := debian:stable-slim BASE_IMAGE_UBUNTU := ubuntu:latest @@ -14,7 +18,6 @@ BASE_IMAGE_FEDORA := fedora:latest BASE_IMAGE_CENTOS := quay.io/centos/centos:stream9 # Make them available in scripts -export DISTROS export BASE_IMAGE_ARCH export BASE_IMAGE_DEBIAN export BASE_IMAGE_UBUNTU diff --git a/scripts/build/build-image-missing.sh b/scripts/build/build-image-missing.sh index 3194961..0e9f997 100755 --- a/scripts/build/build-image-missing.sh +++ b/scripts/build/build-image-missing.sh @@ -4,32 +4,21 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" source "${SCRIPT_DIR}/resolve-base-image.sh" -echo "============================================================" -echo ">>> Building ONLY missing container images" -echo "============================================================" +IMAGE="package-manager-test-$distro" +BASE_IMAGE="$(resolve_base_image "$distro")" -for distro in $DISTROS; do - IMAGE="package-manager-test-$distro" - BASE_IMAGE="$(resolve_base_image "$distro")" - - if docker image inspect "$IMAGE" >/dev/null 2>&1; then - echo "[build-missing] Image already exists: $IMAGE (skipping)" - continue - fi - - echo - echo "------------------------------------------------------------" - echo "[build-missing] Building missing image: $IMAGE" - echo "BASE_IMAGE = $BASE_IMAGE" - echo "------------------------------------------------------------" - - docker build \ - --build-arg BASE_IMAGE="$BASE_IMAGE" \ - -t "$IMAGE" \ - . -done +if docker image inspect "$IMAGE" >/dev/null 2>&1; then + echo "[build-missing] Image already exists: $IMAGE (skipping)" + exit 0 +fi echo -echo "============================================================" -echo ">>> build-missing: Done" -echo "============================================================" +echo "------------------------------------------------------------" +echo "[build-missing] Building missing image: $IMAGE" +echo "BASE_IMAGE = $BASE_IMAGE" +echo "------------------------------------------------------------" + +docker build \ + --build-arg BASE_IMAGE="$BASE_IMAGE" \ + -t "$IMAGE" \ + . diff --git a/scripts/build/build-image-no-cache.sh b/scripts/build/build-image-no-cache.sh index 66ed058..839039a 100755 --- a/scripts/build/build-image-no-cache.sh +++ b/scripts/build/build-image-no-cache.sh @@ -4,14 +4,12 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" source "${SCRIPT_DIR}/resolve-base-image.sh" -for distro in $DISTROS; do - base_image="$(resolve_base_image "$distro")" +base_image="$(resolve_base_image "$distro")" - echo ">>> Building test image for distro '$distro' with NO CACHE (BASE_IMAGE=$base_image)..." +echo ">>> Building test image for distro '$distro' with NO CACHE (BASE_IMAGE=$base_image)..." - docker build \ - --no-cache \ - --build-arg BASE_IMAGE="$base_image" \ - -t "package-manager-test-$distro" \ - . -done +docker build \ + --no-cache \ + --build-arg BASE_IMAGE="$base_image" \ + -t "package-manager-test-$distro" \ + . diff --git a/scripts/build/build-image.sh b/scripts/build/build-image.sh index f3b373c..44598fa 100755 --- a/scripts/build/build-image.sh +++ b/scripts/build/build-image.sh @@ -4,13 +4,11 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" source "${SCRIPT_DIR}/resolve-base-image.sh" -for distro in $DISTROS; do - base_image="$(resolve_base_image "$distro")" +base_image="$(resolve_base_image "$distro")" - echo ">>> Building test image for distro '$distro' (BASE_IMAGE=$base_image)..." +echo ">>> Building test image for distro '$distro' (BASE_IMAGE=$base_image)..." - docker build \ - --build-arg BASE_IMAGE="$base_image" \ - -t "package-manager-test-$distro" \ - . -done +docker build \ + --build-arg BASE_IMAGE="$base_image" \ + -t "package-manager-test-$distro" \ + . diff --git a/scripts/test/test-container.sh b/scripts/test/test-container.sh index b5b7b03..5f60ec5 100755 --- a/scripts/test/test-container.sh +++ b/scripts/test/test-container.sh @@ -1,41 +1,30 @@ #!/usr/bin/env bash set -euo pipefail -echo "============================================================" -echo ">>> Running sanity test: verifying test containers start" -echo "============================================================" - -for distro in $DISTROS; do - IMAGE="package-manager-test-$distro" - - echo - echo "------------------------------------------------------------" - echo ">>> Testing container: $IMAGE" - echo "------------------------------------------------------------" - - echo "[test-container] Running: docker run --rm --entrypoint pkgmgr $IMAGE --help" - echo - - # Run the command and capture the output - if OUTPUT=$(docker run --rm \ - -e PKGMGR_DEV=1 \ - -v pkgmgr_nix_store_${distro}:/nix \ - -v "$(pwd):/src" \ - -v "pkgmgr_nix_cache_${distro}:/root/.cache/nix" \ - "$IMAGE" 2>&1); then - echo "$OUTPUT" - echo - echo "[test-container] SUCCESS: $IMAGE responded to 'pkgmgr --help'" - - else - echo "$OUTPUT" - echo - echo "[test-container] ERROR: $IMAGE failed to run 'pkgmgr --help'" - exit 1 - fi -done +IMAGE="package-manager-test-$distro" echo -echo "============================================================" -echo ">>> All containers passed the sanity check" -echo "============================================================" +echo "------------------------------------------------------------" +echo ">>> Testing container: $IMAGE" +echo "------------------------------------------------------------" + +echo "[test-container] Running: docker run --rm --entrypoint pkgmgr $IMAGE --help" +echo + +# Run the command and capture the output +if OUTPUT=$(docker run --rm \ + -e PKGMGR_DEV=1 \ + -v pkgmgr_nix_store_${distro}:/nix \ + -v "$(pwd):/src" \ + -v "pkgmgr_nix_cache_${distro}:/root/.cache/nix" \ + "$IMAGE" 2>&1); then + echo "$OUTPUT" + echo + echo "[test-container] SUCCESS: $IMAGE responded to 'pkgmgr --help'" + +else + echo "$OUTPUT" + echo + echo "[test-container] ERROR: $IMAGE failed to run 'pkgmgr --help'" + exit 1 +fi \ No newline at end of file diff --git a/scripts/test/test-e2e.sh b/scripts/test/test-e2e.sh index ddf6faa..28f66d2 100755 --- a/scripts/test/test-e2e.sh +++ b/scripts/test/test-e2e.sh @@ -1,64 +1,60 @@ #!/usr/bin/env bash set -euo pipefail -echo ">>> Running E2E tests in all distros: $DISTROS" +echo "============================================================" +echo ">>> Running E2E tests: $distro" +echo "============================================================" -for distro in $DISTROS; do - echo "============================================================" - echo ">>> Running E2E tests: $distro" - echo "============================================================" +docker run --rm \ + -v "$(pwd):/src" \ + -v "pkgmgr_nix_store_${distro}:/nix" \ + -v "pkgmgr_nix_cache_${distro}:/root/.cache/nix" \ + -e PKGMGR_DEV=1 \ + -e TEST_PATTERN="${TEST_PATTERN}" \ + --workdir /src \ + "package-manager-test-${distro}" \ + bash -lc ' + set -euo pipefail - docker run --rm \ - -v "$(pwd):/src" \ - -v "pkgmgr_nix_store_${distro}:/nix" \ - -v "pkgmgr_nix_cache_${distro}:/root/.cache/nix" \ - -e PKGMGR_DEV=1 \ - -e TEST_PATTERN="${TEST_PATTERN}" \ - --workdir /src \ - "package-manager-test-${distro}" \ - bash -lc ' - set -euo pipefail + # Load distro info + if [ -f /etc/os-release ]; then + . /etc/os-release + fi - # Load distro info - if [ -f /etc/os-release ]; then - . /etc/os-release - fi + echo "Running tests inside distro: ${ID:-unknown}" - echo "Running tests inside distro: ${ID:-unknown}" + # Load Nix environment if available + if [ -f "/nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh" ]; then + . "/nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh" + fi - # Load Nix environment if available - if [ -f "/nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh" ]; then - . "/nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh" - fi + if [ -f "$HOME/.nix-profile/etc/profile.d/nix.sh" ]; then + . "$HOME/.nix-profile/etc/profile.d/nix.sh" + fi - if [ -f "$HOME/.nix-profile/etc/profile.d/nix.sh" ]; then - . "$HOME/.nix-profile/etc/profile.d/nix.sh" - fi + PATH="/nix/var/nix/profiles/default/bin:$HOME/.nix-profile/bin:$PATH" - PATH="/nix/var/nix/profiles/default/bin:$HOME/.nix-profile/bin:$PATH" + command -v nix >/dev/null || { + echo "ERROR: nix not found." + exit 1 + } - command -v nix >/dev/null || { - echo "ERROR: nix not found." - exit 1 - } + # Mark the mounted repository as safe to avoid Git ownership errors. + # Newer Git (e.g. on Ubuntu) complains about the gitdir (/src/.git), + # older versions about the worktree (/src). Nix turns "." into the + # flake input "git+file:///src", which then uses Git under the hood. + if command -v git >/dev/null 2>&1; then + # Worktree path + git config --global --add safe.directory /src || true + # Gitdir path shown in the "dubious ownership" error + git config --global --add safe.directory /src/.git || true + # Ephemeral CI containers: allow all paths as a last resort + git config --global --add safe.directory '*' || true + fi - # Mark the mounted repository as safe to avoid Git ownership errors. - # Newer Git (e.g. on Ubuntu) complains about the gitdir (/src/.git), - # older versions about the worktree (/src). Nix turns "." into the - # flake input "git+file:///src", which then uses Git under the hood. - if command -v git >/dev/null 2>&1; then - # Worktree path - git config --global --add safe.directory /src || true - # Gitdir path shown in the "dubious ownership" error - git config --global --add safe.directory /src/.git || true - # Ephemeral CI containers: allow all paths as a last resort - git config --global --add safe.directory '*' || true - fi - - # Run the E2E tests inside the Nix development shell - nix develop .#default --no-write-lock-file -c \ - python3 -m unittest discover \ - -s /src/tests/e2e \ - -p "$TEST_PATTERN" - ' -done + # Run the E2E tests inside the Nix development shell + nix develop .#default --no-write-lock-file -c \ + python3 -m unittest discover \ + -s /src/tests/e2e \ + -p "$TEST_PATTERN" + ' diff --git a/scripts/test/test-integration.sh b/scripts/test/test-integration.sh index 42bd607..75141de 100755 --- a/scripts/test/test-integration.sh +++ b/scripts/test/test-integration.sh @@ -1,8 +1,6 @@ #!/usr/bin/env bash set -euo pipefail -: "${distro:=arch}" - echo "============================================================" echo ">>> Running INTEGRATION tests in ${distro} container" echo "============================================================" diff --git a/scripts/test/test-unit.sh b/scripts/test/test-unit.sh index 7c0f979..68a406b 100755 --- a/scripts/test/test-unit.sh +++ b/scripts/test/test-unit.sh @@ -1,8 +1,6 @@ #!/usr/bin/env bash set -euo pipefail -: "${distro:=arch}" - echo "============================================================" echo ">>> Running UNIT tests in ${distro} container" echo "============================================================"