From 324f6db1f34ac3f98620bc297a6f3513a2a1be50 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Fri, 12 Dec 2025 12:15:40 +0100 Subject: [PATCH] ci: split container tests into virtualenv and Nix flake environments Refactor CI to clearly separate virtualenv-based container tests from pure Nix flake tests across all distros (arch, debian, ubuntu, fedora, centos). Introduce dedicated test-env-nix workflow and Makefile targets, rename former container tests to test-env-virtual, and update stable pipeline dependencies. Improve Nix reliability in containers by fixing installer permissions and explicitly validating nix availability and version during image build and tests. --- .github/workflows/ci.yml | 7 ++- .github/workflows/mark-stable.yml | 10 ++- .github/workflows/test-env-nix.yml | 26 ++++++++ ...est-container.yml => test-env-virtual.yml} | 4 +- Dockerfile | 5 ++ Makefile | 26 +++++--- README.md | 2 +- packaging/arch/package-manager.install | 4 +- packaging/debian/postinst | 6 +- packaging/fedora/package-manager.spec | 7 +-- scripts/init-nix.sh | 10 +++ scripts/installation/venv-create.sh | 62 ++++++++++++++++--- scripts/test/test-env-nix.sh | 48 ++++++++++++++ ...{test-container.sh => test-env-virtual.sh} | 10 +-- 14 files changed, 184 insertions(+), 43 deletions(-) create mode 100644 .github/workflows/test-env-nix.yml rename .github/workflows/{test-container.yml => test-env-virtual.yml} (86%) create mode 100644 scripts/test/test-env-nix.sh rename scripts/test/{test-container.sh => test-env-virtual.sh} (65%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b35fb5e..4ad2598 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,8 +13,11 @@ jobs: test-integration: uses: ./.github/workflows/test-integration.yml - test-container: - uses: ./.github/workflows/test-container.yml + test-env-virtual: + uses: ./.github/workflows/test-env-virtual.yml + + test-env-nix: + uses: ./.github/workflows/test-env-nix.yml test-e2e: uses: ./.github/workflows/test-e2e.yml diff --git a/.github/workflows/mark-stable.yml b/.github/workflows/mark-stable.yml index 66d9d8e..72a13c1 100644 --- a/.github/workflows/mark-stable.yml +++ b/.github/workflows/mark-stable.yml @@ -14,8 +14,11 @@ jobs: test-integration: uses: ./.github/workflows/test-integration.yml - test-container: - uses: ./.github/workflows/test-container.yml + test-env-virtual: + uses: ./.github/workflows/test-env-virtual.yml + + test-env-nix: + uses: ./.github/workflows/test-env-nix.yml test-e2e: uses: ./.github/workflows/test-e2e.yml @@ -30,7 +33,8 @@ jobs: needs: - test-unit - test-integration - - test-container + - test-env-nix + - test-env-virtual - test-e2e - test-virgin-user - test-virgin-root diff --git a/.github/workflows/test-env-nix.yml b/.github/workflows/test-env-nix.yml new file mode 100644 index 0000000..b41fdf5 --- /dev/null +++ b/.github/workflows/test-env-nix.yml @@ -0,0 +1,26 @@ +name: Test Virgin Nix (flake only) + +on: + workflow_call: + +jobs: + test-env-nix: + runs-on: ubuntu-latest + timeout-minutes: 45 + + strategy: + fail-fast: false + matrix: + distro: [arch, debian, ubuntu, fedora, centos] + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Show Docker version + run: docker version + + - name: Nix flake-only test (${{ matrix.distro }}) + run: | + set -euo pipefail + distro="${{ matrix.distro }}" make test-env-nix diff --git a/.github/workflows/test-container.yml b/.github/workflows/test-env-virtual.yml similarity index 86% rename from .github/workflows/test-container.yml rename to .github/workflows/test-env-virtual.yml index 7aef50a..e3a4680 100644 --- a/.github/workflows/test-container.yml +++ b/.github/workflows/test-env-virtual.yml @@ -4,7 +4,7 @@ on: workflow_call: jobs: - test-container: + test-env-virtual: runs-on: ubuntu-latest timeout-minutes: 30 strategy: @@ -25,4 +25,4 @@ jobs: - name: Run container tests (${{ matrix.distro }}) run: | set -euo pipefail - distro="${{ matrix.distro }}" make test-container + distro="${{ matrix.distro }}" make test-env-virtual diff --git a/Dockerfile b/Dockerfile index 76f90d9..cd14075 100644 --- a/Dockerfile +++ b/Dockerfile @@ -52,6 +52,11 @@ RUN set -e; \ make install; \ rm -rf /build +# ------------------------------------------------------------ +# Show Nix Version +# ------------------------------------------------------------ +RUN command -v nix && nix --version + # ------------------------------------------------------------ # Runtime working directory and dev entrypoint # ------------------------------------------------------------ diff --git a/Makefile b/Makefile index 8763d40..1423e4c 100644 --- a/Makefile +++ b/Makefile @@ -1,9 +1,10 @@ .PHONY: install setup uninstall \ - test build build-no-cache test-unit test-e2e test-integration \ - test-container + test build build-no-cache build-no-cache-all build-missing delete-volumes purge \ + test-unit test-e2e test-integration test-env-virtual test-env-nix # Distro # Options: arch debian ubuntu fedora centos +DISTROS ?= arch debian ubuntu fedora centos distro ?= arch export distro @@ -37,11 +38,18 @@ setup: # ------------------------------------------------------------ # Docker build targets (delegated to scripts/build) # ------------------------------------------------------------ +build: + @bash scripts/build/build-image.sh + build-no-cache: @bash scripts/build/build-image-no-cache.sh -build: - @bash scripts/build/build-image.sh +build-no-cache-all: + @set -e; \ + for d in $(DISTROS); do \ + echo "=== build-no-cache: $$d ==="; \ + distro="$$d" $(MAKE) build-no-cache; \ + done # ------------------------------------------------------------ # Test targets (delegated to scripts/test) @@ -56,8 +64,12 @@ test-integration: build-missing test-e2e: build-missing @bash scripts/test/test-e2e.sh -test-container: build-missing - @bash scripts/test/test-container.sh +test-env-virtual: build-missing + @bash scripts/test/test-env-virtual.sh + +test-env-nix: build-missing + @bash scripts/test/test-env-nix.sh + # ------------------------------------------------------------ # Build only missing container images @@ -66,7 +78,7 @@ build-missing: @bash scripts/build/build-image-missing.sh # Combined test target for local + CI (unit + integration + e2e) -test: test-container test-unit test-integration test-e2e +test: test-env-virtual test-unit test-integration test-e2e delete-volumes: @docker volume rm pkgmgr_nix_store_${distro} pkgmgr_nix_cache_${distro} || true diff --git a/README.md b/README.md index c63353a..341a9a3 100644 --- a/README.md +++ b/README.md @@ -98,7 +98,7 @@ The following diagram gives a full overview of: ![PKGMGR Architecture](assets/map.png) -**Diagram status:** 11 December 2025 +**Diagram status:** 12 December 2025 **Always-up-to-date version:** [https://s.veen.world/pkgmgrmp](https://s.veen.world/pkgmgrmp) --- diff --git a/packaging/arch/package-manager.install b/packaging/arch/package-manager.install index 0bdff27..3ed7da1 100644 --- a/packaging/arch/package-manager.install +++ b/packaging/arch/package-manager.install @@ -1,9 +1,9 @@ post_install() { - /usr/lib/package-manager/init-nix.sh || true + /usr/lib/package-manager/init-nix.sh || echo ">>> ERROR: /usr/lib/package-manager/init-nix.sh not found or not executable." } post_upgrade() { - /usr/lib/package-manager/init-nix.sh || true + /usr/lib/package-manager/init-nix.sh || echo ">>> ERROR: /usr/lib/package-manager/init-nix.sh not found or not executable." } post_remove() { diff --git a/packaging/debian/postinst b/packaging/debian/postinst index 899b26c..17cc3c6 100755 --- a/packaging/debian/postinst +++ b/packaging/debian/postinst @@ -3,11 +3,7 @@ set -e case "$1" in configure) - if [ -x /usr/lib/package-manager/init-nix.sh ]; then - /usr/lib/package-manager/init-nix.sh || true - else - echo ">>> Warning: /usr/lib/package-manager/init-nix.sh not found or not executable." - fi + /usr/lib/package-manager/init-nix.sh || echo ">>> ERROR: /usr/lib/package-manager/init-nix.sh not found or not executable." ;; esac diff --git a/packaging/fedora/package-manager.spec b/packaging/fedora/package-manager.spec index 15f087e..1afbe8f 100644 --- a/packaging/fedora/package-manager.spec +++ b/packaging/fedora/package-manager.spec @@ -60,12 +60,7 @@ rm -rf \ %{buildroot}/usr/lib/package-manager/.gitkeep || true %post -# Initialize Nix (if needed) after installing the package-manager files. -if [ -x /usr/lib/package-manager/init-nix.sh ]; then - /usr/lib/package-manager/init-nix.sh || true -else - echo ">>> Warning: /usr/lib/package-manager/init-nix.sh not found or not executable." -fi +/usr/lib/package-manager/init-nix.sh || echo ">>> ERROR: /usr/lib/package-manager/init-nix.sh not found or not executable." %postun echo ">>> package-manager removed. Nix itself was not removed." diff --git a/scripts/init-nix.sh b/scripts/init-nix.sh index 6a7dc4c..7044482 100755 --- a/scripts/init-nix.sh +++ b/scripts/init-nix.sh @@ -80,6 +80,13 @@ install_nix_with_retry() { installer="$(mktemp -t nix-installer.XXXXXX)" + # ------------------------------------------------------------------------- + # FIX: mktemp creates files with 0600 by default, which breaks when we later + # run the installer as a different user (e.g., 'nix' in container+root). + # Make it readable and (best-effort) owned by the target user. + # ------------------------------------------------------------------------- + chmod 0644 "${installer}" + echo "[init-nix] Downloading Nix installer from ${NIX_INSTALL_URL} with retry (max ${NIX_DOWNLOAD_MAX_TIME}s)..." while true; do @@ -103,6 +110,9 @@ install_nix_with_retry() { done if [[ -n "${run_as}" ]]; then + # Best-effort: ensure the target user can read the downloaded installer + chown "${run_as}:${run_as}" "${installer}" 2>/dev/null || true + echo "[init-nix] Running installer as user '${run_as}' with mode '${mode}'..." if command -v sudo >/dev/null 2>&1; then sudo -u "${run_as}" bash -lc "sh '${installer}' ${mode_flag}" diff --git a/scripts/installation/venv-create.sh b/scripts/installation/venv-create.sh index 8d341b8..4c712a5 100644 --- a/scripts/installation/venv-create.sh +++ b/scripts/installation/venv-create.sh @@ -3,42 +3,84 @@ set -euo pipefail # venv-create.sh # -# Small helper to create/update a Python virtual environment for pkgmgr. +# Create/update a Python virtual environment for pkgmgr and install dependencies. +# +# Priority order: +# 1) pyproject.toml -> pip install (editable by default) +# 2) requirements.txt +# 3) _requirements.txt (legacy) # # Usage: # PKGMGR_VENV_DIR=/home/dev/.venvs/pkgmgr bash scripts/installation/venv-create.sh # or # bash scripts/installation/venv-create.sh /home/dev/.venvs/pkgmgr +# +# Optional: +# PKGMGR_PIP_EDITABLE=0 # install non-editable (default: 1) +# PKGMGR_PIP_EXTRAS="dev,test" # install extras: .[dev,test] +# PKGMGR_PREFER_NIX=1 # print Nix hint and exit non-zero PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" cd "${PROJECT_ROOT}" VENV_DIR="${PKGMGR_VENV_DIR:-${1:-${HOME}/.venvs/pkgmgr}}" +PIP_EDITABLE="${PKGMGR_PIP_EDITABLE:-1}" +PIP_EXTRAS="${PKGMGR_PIP_EXTRAS:-}" +PREFER_NIX="${PKGMGR_PREFER_NIX:-0}" echo "[venv-create] Using VENV_DIR=${VENV_DIR}" +if [[ "${PREFER_NIX}" == "1" ]]; then + echo "[venv-create] PKGMGR_PREFER_NIX=1 set." + echo "[venv-create] Hint: Use Nix instead of a venv for reproducible installs:" + echo "[venv-create] nix develop" + echo "[venv-create] nix run .#pkgmgr -- --help" + exit 2 +fi + echo "[venv-create] Ensuring virtualenv parent directory exists..." mkdir -p "$(dirname "${VENV_DIR}")" if [[ ! -d "${VENV_DIR}" ]]; then - echo "[venv-create] Creating virtual environment at: ${VENV_DIR}" - python3 -m venv "${VENV_DIR}" + echo "[venv-create] Creating virtual environment at: ${VENV_DIR}" + python3 -m venv "${VENV_DIR}" else - echo "[venv-create] Virtual environment already exists at: ${VENV_DIR}" + echo "[venv-create] Virtual environment already exists at: ${VENV_DIR}" fi echo "[venv-create] Installing Python tooling into venv..." "${VENV_DIR}/bin/python" -m ensurepip --upgrade "${VENV_DIR}/bin/pip" install --upgrade pip setuptools wheel -if [[ -f "requirements.txt" ]]; then - echo "[venv-create] Installing dependencies from requirements.txt..." - "${VENV_DIR}/bin/pip" install -r requirements.txt +# --------------------------------------------------------------------------- +# Install dependencies +# --------------------------------------------------------------------------- +if [[ -f "pyproject.toml" ]]; then + echo "[venv-create] Detected pyproject.toml. Installing project via pip..." + + target="." + if [[ -n "${PIP_EXTRAS}" ]]; then + target=".[${PIP_EXTRAS}]" + fi + + if [[ "${PIP_EDITABLE}" == "1" ]]; then + echo "[venv-create] pip install -e ${target}" + "${VENV_DIR}/bin/pip" install -e "${target}" + else + echo "[venv-create] pip install ${target}" + "${VENV_DIR}/bin/pip" install "${target}" + fi + +elif [[ -f "requirements.txt" ]]; then + echo "[venv-create] Installing dependencies from requirements.txt..." + "${VENV_DIR}/bin/pip" install -r requirements.txt + elif [[ -f "_requirements.txt" ]]; then - echo "[venv-create] Installing dependencies from _requirements.txt..." - "${VENV_DIR}/bin/pip" install -r _requirements.txt + echo "[venv-create] Installing dependencies from _requirements.txt (legacy)..." + "${VENV_DIR}/bin/pip" install -r _requirements.txt + else - echo "[venv-create] No requirements.txt or _requirements.txt found. Skipping dependency installation." + echo "[venv-create] No pyproject.toml, requirements.txt, or _requirements.txt found. Skipping dependency installation." fi echo "[venv-create] Done." diff --git a/scripts/test/test-env-nix.sh b/scripts/test/test-env-nix.sh new file mode 100644 index 0000000..c71ea37 --- /dev/null +++ b/scripts/test/test-env-nix.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +set -euo pipefail + +IMAGE="package-manager-test-${distro}" + +echo "============================================================" +echo ">>> Running Nix flake-only test in ${distro} container" +echo ">>> Image: ${IMAGE}" +echo "============================================================" + +docker run --rm \ + -v "$(pwd):/src" \ + -v "pkgmgr_nix_store_${distro}:/nix" \ + -v "pkgmgr_nix_cache_${distro}:/root/.cache/nix" \ + --workdir /src \ + -e PKGMGR_DEV=0 \ + "${IMAGE}" \ + bash -lc ' + set -euo pipefail + + if command -v git >/dev/null 2>&1; then + git config --global --add safe.directory /src || true + git config --global --add safe.directory /src/.git || true + git config --global --add safe.directory "*" || true + fi + + echo ">>> preflight: nix must exist in image" + if ! command -v nix >/dev/null 2>&1; then + echo "NO_NIX" + echo "ERROR: nix not found in image '\'''"${IMAGE}"''\'' (distro='"${distro}"')" + echo "HINT: Ensure Nix is installed during image build for this distro." + exit 1 + fi + + echo ">>> nix version" + nix --version + + echo ">>> nix flake show" + nix flake show . --no-write-lock-file >/dev/null + + echo ">>> nix build .#default" + nix build .#default --no-link --no-write-lock-file + + echo ">>> nix run .#pkgmgr -- --help" + nix run .#pkgmgr -- --help --no-write-lock-file + + echo ">>> OK: Nix flake-only test succeeded." + ' diff --git a/scripts/test/test-container.sh b/scripts/test/test-env-virtual.sh similarity index 65% rename from scripts/test/test-container.sh rename to scripts/test/test-env-virtual.sh index 1d018fd..25e6ad3 100755 --- a/scripts/test/test-container.sh +++ b/scripts/test/test-env-virtual.sh @@ -5,12 +5,12 @@ IMAGE="package-manager-test-$distro" echo echo "------------------------------------------------------------" -echo ">>> Testing container: $IMAGE" +echo ">>> Testing VENV: $IMAGE" echo "------------------------------------------------------------" -echo "[test-container] Inspect image metadata:" +echo "[test-env-virtual] Inspect image metadata:" docker image inspect "$IMAGE" | sed -n '1,40p' -echo "[test-container] Running: docker run --rm --entrypoint pkgmgr $IMAGE --help" +echo "[test-env-virtual] Running: docker run --rm --entrypoint pkgmgr $IMAGE --help" echo # Run the command and capture the output @@ -22,11 +22,11 @@ if OUTPUT=$(docker run --rm \ "$IMAGE" 2>&1); then echo "$OUTPUT" echo - echo "[test-container] SUCCESS: $IMAGE responded to 'pkgmgr --help'" + echo "[test-env-virtual] SUCCESS: $IMAGE responded to 'pkgmgr --help'" else echo "$OUTPUT" echo - echo "[test-container] ERROR: $IMAGE failed to run 'pkgmgr --help'" + echo "[test-env-virtual] ERROR: $IMAGE failed to run 'pkgmgr --help'" exit 1 fi \ No newline at end of file