**Aur builder: add retry logic for yay clone to recover from GitHub 504 errors**

Implemented a robust retry mechanism for cloning the yay AUR helper during Arch dependency installation.
The new logic retries the git clone operation for up to 5 minutes with a 20-second pause between attempts, allowing the build to proceed even when GitHub intermittently returns HTTP 504 errors.

This improves the stability of Arch container builds, especially under network pressure or transient upstream outages.
The yay build process now only starts once the clone step completes successfully.

https://chatgpt.com/share/693b102b-fdb0-800f-9f2e-d4840f14d329

CHANGELOG.md

@@ -1,3 +1,50 @@
+## [0.10.1] - 2025-12-11
+
+* Fixed Debian\Ubuntu to pass container e2e tests
+
+
+## [0.10.0] - 2025-12-11
+
+* **Changes since v0.9.1**
+
+**Mirror System**
+
+* Added SSH mirror support including multi-push and remote probing
+* Introduced mirror management commands and refactored the CLI parser into modules
+
+**CI/CD**
+
+* Migrated to reusable workflows with improved debugging instrumentation
+* Made stable-tag automation reliable for workflow_run events and permissions
+* Ensured deterministic test results by rebuilding all test containers with no-cache
+
+**E2E and Container Tests**
+
+* Fixed Git safe.directory handling across all containers
+* Restored Dockerfile ENTRYPOINT to resolve Nix TLS issues
+* Fixed missing volume errors and hardened the E2E runner
+* Added full Nix flake E2E test matrix across all distro containers
+* Disabled Nix sandboxing for cross-distro builds where required
+
+**Nix and Python Environment**
+
+* Unified Nix Python environment and introduced lazy CLI imports
+* Ensured PyYAML availability and improved Python 3.13 compatibility
+* Refactored flake.nix to remove side effects and rely on generic python3
+
+**Packaging**
+
+* Removed Debian’s hard dependency on Nix
+* Restructured packaging layout and refined build paths
+* Excluded assets from Arch PKGBUILD rsync
+* Cleaned up obsolete ignore files
+
+**Repository Layout**
+
+* Restructured repository to align local, Nix-based, and distro-based build workflows
+* Added Arch support and refined build/purge scripts
+
+
 ## [0.9.1] - 2025-12-10
 
 * * Refactored installer: new `venv-create.sh`, cleaner root/user setup flow, updated README with architecture map.

flake.nix

@@ -36,7 +36,7 @@
         rec {
           pkgmgr = pyPkgs.buildPythonApplication {
             pname   = "package-manager";
-            version = "0.9.1";
+            version = "0.10.1";
 
             # Use the git repo as source
             src = ./.;

packaging/debian/control

@@ -9,7 +9,7 @@ Homepage: https://github.com/kevinveenbirkenbach/package-manager
 
 Package: package-manager
 Architecture: any
-Depends: ${misc:Depends}
+Depends: sudo, ${misc:Depends}
 Description: Wrapper that runs Kevin's package-manager via Nix flake
  This package provides the `pkgmgr` command, which runs Kevin's package
  manager via a local Nix flake

pyproject.toml

@@ -7,7 +7,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "package-manager"
-version = "0.9.1"
+version = "0.10.1"
 description = "Kevin's package-manager tool (pkgmgr)"
 readme = "README.md"
 requires-python = ">=3.11"

scripts/init-nix.sh

@@ -45,6 +45,26 @@ ensure_nix_on_path() {
   fi
 }
 
+# ---------------------------------------------------------------------------
+# Helper: ensure Nix build group and users exist (build-users-group = nixbld)
+# ---------------------------------------------------------------------------
+ensure_nix_build_group() {
+  # Ensure nixbld group (build-users-group for Nix)
+  if ! getent group nixbld >/dev/null 2>&1; then
+    echo "[init-nix] Creating group 'nixbld'..."
+    groupadd -r nixbld
+  fi
+
+  # Ensure Nix build users (nixbld1..nixbld10) as members of nixbld
+  for i in $(seq 1 10); do
+    if ! id "nixbld$i" >/dev/null 2>&1; then
+      echo "[init-nix] Creating build user nixbld$i..."
+      # -r: system account, -g: primary group, -G: supplementary (ensures membership is listed)
+      useradd -r -g nixbld -G nixbld -s /usr/sbin/nologin "nixbld$i"
+    fi
+  done
+}
+
 # ---------------------------------------------------------------------------
 # Fast path: Nix already available
 # ---------------------------------------------------------------------------
@@ -76,20 +96,8 @@ fi
 if [[ "${IN_CONTAINER}" -eq 1 && "${EUID:-0}" -eq 0 ]]; then
   echo "[init-nix] Running as root inside a container – using dedicated 'nix' user."
 
-  # Ensure nixbld group (required by Nix)
-  if ! getent group nixbld >/dev/null 2>&1; then
-    echo "[init-nix] Creating group 'nixbld'..."
-    groupadd -r nixbld
-  fi
-
-  # Ensure Nix build users (nixbld1..nixbld10) as members of nixbld
-  for i in $(seq 1 10); do
-    if ! id "nixbld$i" >/dev/null 2>&1; then
-      echo "[init-nix] Creating build user nixbld$i..."
-      # -r: system account, -g: primary group, -G: supplementary (ensures membership is listed)
-      useradd -r -g nixbld -G nixbld -s /usr/sbin/nologin "nixbld$i"
-    fi
-  done
+  # Ensure build group/users for Nix
+  ensure_nix_build_group
 
   # Ensure "nix" user (home at /home/nix)
   if ! id nix >/dev/null 2>&1; then
@@ -187,14 +195,25 @@ if [[ "${IN_CONTAINER}" -eq 0 ]]; then
   # Real host
   if command -v systemctl >/dev/null 2>&1; then
     echo "[init-nix] Host with systemd – using multi-user install (--daemon)."
+    if [[ "${EUID:-0}" -eq 0 ]]; then
+      # Prepare build-users-group for Nix daemon installs
+      ensure_nix_build_group
+    fi
     sh <(curl -L https://nixos.org/nix/install) --daemon
   else
     if [[ "${EUID:-0}" -eq 0 ]]; then
       echo "[init-nix] WARNING: Running as root without systemd on host."
       echo "[init-nix] Falling back to single-user install (--no-daemon), but this is not recommended."
+
+      # IMPORTANT: This is where Debian/Ubuntu inside your CI end up.
+      # We must ensure 'nixbld' exists before running the installer,
+      # otherwise modern Nix fails with: "the group 'nixbld' ... does not exist".
+      ensure_nix_build_group
+
       sh <(curl -L https://nixos.org/nix/install) --no-daemon
     else
       echo "[init-nix] Non-root host without systemd – using single-user install (--no-daemon)."
+      # Non-root cannot create nixbld group; rely on upstream defaults
       sh <(curl -L https://nixos.org/nix/install) --no-daemon
     fi
   fi

scripts/installation/arch/aur-builder-setup.sh

@@ -45,8 +45,42 @@ else
 fi
 
 echo "[aur-builder-setup] Ensuring yay is installed for aur_builder..."
+
 if ! "${RUN_AS_AUR[@]}" 'command -v yay >/dev/null 2>&1'; then
-  "${RUN_AS_AUR[@]}" 'cd ~ && rm -rf yay && git clone https://aur.archlinux.org/yay.git && cd yay && makepkg -si --noconfirm'
+  echo "[aur-builder-setup] yay not found – starting retry sequence for download..."
+
+  MAX_TIME=300         # 5 minutes
+  SLEEP_INTERVAL=20    # 20 seconds
+  ELAPSED=0
+
+  while true; do
+    if "${RUN_AS_AUR[@]}" '
+        set -euo pipefail
+        cd ~
+        rm -rf yay || true
+        git clone https://aur.archlinux.org/yay.git yay
+    '; then
+        echo "[aur-builder-setup] yay repository cloned successfully."
+        break
+    fi
+
+    echo "[aur-builder-setup] git clone failed (likely 504). Retrying in ${SLEEP_INTERVAL}s..."
+    sleep "${SLEEP_INTERVAL}"
+    ELAPSED=$((ELAPSED + SLEEP_INTERVAL))
+
+    if (( ELAPSED >= MAX_TIME )); then
+      echo "[aur-builder-setup] ERROR: Aborted after 5 minutes of retry attempts."
+      exit 1
+    fi
+  done
+
+  # Now build yay after successful clone
+  "${RUN_AS_AUR[@]}" '
+      set -euo pipefail
+      cd ~/yay
+      makepkg -si --noconfirm
+  '
+
 else
   echo "[aur-builder-setup] yay already installed."
 fi

scripts/pkgmgr-wrapper.sh

@@ -8,19 +8,18 @@ fi
 
 FLAKE_DIR="/usr/lib/package-manager"
 
-# ------------------------------------------------------------
-# Try to ensure that "nix" is on PATH
-# ------------------------------------------------------------
+# ---------------------------------------------------------------------------
+# Try to ensure that "nix" is on PATH (common locations + container user)
+# ---------------------------------------------------------------------------
 if ! command -v nix >/dev/null 2>&1; then
-  # Common locations for Nix installations
   CANDIDATES=(
     "/nix/var/nix/profiles/default/bin/nix"
     "${HOME:-/root}/.nix-profile/bin/nix"
+    "/home/nix/.nix-profile/bin/nix"
   )
 
   for candidate in "${CANDIDATES[@]}"; do
     if [[ -x "$candidate" ]]; then
-      # Prepend the directory of the candidate to PATH
       PATH="$(dirname "$candidate"):${PATH}"
       export PATH
       break
@@ -28,13 +27,22 @@ if ! command -v nix >/dev/null 2>&1; then
   done
 fi
 
-# ------------------------------------------------------------
-# Primary (and only) path: use Nix flake if available
-# ------------------------------------------------------------
+# ---------------------------------------------------------------------------
+# If nix is still missing, try to run init-nix.sh once
+# ---------------------------------------------------------------------------
+if ! command -v nix >/dev/null 2>&1; then
+  if [[ -x "${FLAKE_DIR}/init-nix.sh" ]]; then
+    "${FLAKE_DIR}/init-nix.sh" || true
+  fi
+fi
+
+# ---------------------------------------------------------------------------
+# Primary path: use Nix flake if available
+# ---------------------------------------------------------------------------
 if command -v nix >/dev/null 2>&1; then
   exec nix run "${FLAKE_DIR}#pkgmgr" -- "$@"
 fi
 
-echo "[pkgmgr-wrapper] ERROR: 'nix' binary not found on PATH."
+echo "[pkgmgr-wrapper] ERROR: 'nix' binary not found on PATH after init."
 echo "[pkgmgr-wrapper] Nix is required to run pkgmgr (no Python fallback)."
 exit 1

src/pkgmgr/actions/install/installers/nix_flake.py

@@ -139,22 +139,27 @@ class NixFlakeInstaller(BaseInstaller):
 
         for output, allow_failure in outputs:
             cmd = f"nix profile install {ctx.repo_dir}#{output}"
+            print(f"[INFO] Running: {cmd}")
+            ret = os.system(cmd)
 
-            try:
-                run_command(
-                    cmd,
-                    cwd=ctx.repo_dir,
-                    preview=ctx.preview,
-                    allow_failure=allow_failure,
-                )
+            # Extract real exit code from os.system() result
+            if os.WIFEXITED(ret):
+                exit_code = os.WEXITSTATUS(ret)
+            else:
+                # abnormal termination (signal etc.) – keep raw value
+                exit_code = ret
+
+            if exit_code == 0:
                 print(f"Nix flake output '{output}' successfully installed.")
-            except SystemExit as e:
-                print(f"[Error] Failed to install Nix flake output '{output}': {e}")
-                if not allow_failure:
-                    # Mandatory output failed → fatal for the pipeline.
-                    raise
-                # Optional output failed → log and continue.
-                print(
-                    "[Warning] Continuing despite failure to install "
-                    f"optional output '{output}'."
-                )
+                continue
+
+            print(f"[Error] Failed to install Nix flake output '{output}'")
+            print(f"[Error] Command exited with code {exit_code}")
+
+            if not allow_failure:
+                raise SystemExit(exit_code)
+
+            print(
+                "[Warning] Continuing despite failure to install "
+                f"optional output '{output}'."
+            )

tests/unit/pkgmgr/actions/install/installers/test_nix_flake.py

@@ -1,140 +1,206 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
-import os
-import unittest
-from unittest import mock
-from unittest.mock import MagicMock, patch
+"""
+Unit tests for NixFlakeInstaller using unittest (no pytest).
+
+Covers:
+- Successful installation (exit_code == 0)
+- Mandatory failure → SystemExit with correct code
+- Optional failure (pkgmgr default) → no raise, but warning
+- supports() behavior incl. PKGMGR_DISABLE_NIX_FLAKE_INSTALLER
+"""
+
+import io
+import os
+import shutil
+import tempfile
+import unittest
+from contextlib import redirect_stdout
+from unittest.mock import patch
 
-from pkgmgr.actions.install.context import RepoContext
 from pkgmgr.actions.install.installers.nix_flake import NixFlakeInstaller
 
 
+class DummyCtx:
+    """Minimal context object to satisfy NixFlakeInstaller.run() / supports()."""
+
+    def __init__(self, identifier: str, repo_dir: str, preview: bool = False):
+        self.identifier = identifier
+        self.repo_dir = repo_dir
+        self.preview = preview
+
+
 class TestNixFlakeInstaller(unittest.TestCase):
     def setUp(self) -> None:
-        self.repo = {"repository": "package-manager"}
-        # Important: identifier "pkgmgr" triggers both "pkgmgr" and "default"
-        self.ctx = RepoContext(
-            repo=self.repo,
-            identifier="pkgmgr",
-            repo_dir="/tmp/repo",
-            repositories_base_dir="/tmp",
-            bin_dir="/bin",
-            all_repos=[self.repo],
-            no_verification=False,
-            preview=False,
-            quiet=False,
-            clone_mode="ssh",
-            update_dependencies=False,
-        )
-        self.installer = NixFlakeInstaller()
+        # Create a temporary repository directory with a flake.nix file
+        self._tmpdir = tempfile.mkdtemp(prefix="nix_flake_test_")
+        self.repo_dir = self._tmpdir
+        flake_path = os.path.join(self.repo_dir, "flake.nix")
+        with open(flake_path, "w", encoding="utf-8") as f:
+            f.write("{}\n")
 
-    @patch("pkgmgr.actions.install.installers.nix_flake.os.path.exists")
-    @patch("pkgmgr.actions.install.installers.nix_flake.shutil.which")
-    def test_supports_true_when_nix_and_flake_exist(
-        self,
-        mock_which: MagicMock,
-        mock_exists: MagicMock,
-    ) -> None:
-        mock_which.return_value = "/usr/bin/nix"
-        mock_exists.return_value = True
+        # Ensure the disable env var is not set by default
+        os.environ.pop("PKGMGR_DISABLE_NIX_FLAKE_INSTALLER", None)
 
-        with patch.dict(os.environ, {"PKGMGR_DISABLE_NIX_FLAKE_INSTALLER": ""}, clear=False):
-            self.assertTrue(self.installer.supports(self.ctx))
+    def tearDown(self) -> None:
+        # Cleanup temporary directory
+        if os.path.isdir(self._tmpdir):
+            shutil.rmtree(self._tmpdir, ignore_errors=True)
 
-        mock_which.assert_called_once_with("nix")
-        mock_exists.assert_called_once_with(
-            os.path.join(self.ctx.repo_dir, self.installer.FLAKE_FILE)
-        )
+    def _enable_nix_in_module(self, which_patch):
+        """Ensure shutil.which('nix') in nix_flake module returns a path."""
+        which_patch.return_value = "/usr/bin/nix"
 
-    @patch("pkgmgr.actions.install.installers.nix_flake.os.path.exists")
-    @patch("pkgmgr.actions.install.installers.nix_flake.shutil.which")
-    def test_supports_false_when_nix_missing(
-        self,
-        mock_which: MagicMock,
-        mock_exists: MagicMock,
-    ) -> None:
-        mock_which.return_value = None
-        mock_exists.return_value = True  # flake exists but nix is missing
-
-        with patch.dict(os.environ, {"PKGMGR_DISABLE_NIX_FLAKE_INSTALLER": ""}, clear=False):
-            self.assertFalse(self.installer.supports(self.ctx))
-
-    @patch("pkgmgr.actions.install.installers.nix_flake.os.path.exists")
-    @patch("pkgmgr.actions.install.installers.nix_flake.shutil.which")
-    def test_supports_false_when_disabled_via_env(
-        self,
-        mock_which: MagicMock,
-        mock_exists: MagicMock,
-    ) -> None:
-        mock_which.return_value = "/usr/bin/nix"
-        mock_exists.return_value = True
-
-        with patch.dict(
-            os.environ,
-            {"PKGMGR_DISABLE_NIX_FLAKE_INSTALLER": "1"},
-            clear=False,
-        ):
-            self.assertFalse(self.installer.supports(self.ctx))
-
-    @patch("pkgmgr.actions.install.installers.nix_flake.NixFlakeInstaller.supports")
-    @patch("pkgmgr.actions.install.installers.nix_flake.run_command")
-    def test_run_removes_old_profile_and_installs_outputs(
-        self,
-        mock_run_command: MagicMock,
-        mock_supports: MagicMock,
-    ) -> None:
+    def test_nix_flake_run_success(self):
         """
-        run() should:
-        - remove the old profile
-        - install both 'pkgmgr' and 'default' outputs for identifier 'pkgmgr'
-        - call commands in the correct order
+        When os.system returns a successful exit code, the installer
+        should report success and not raise.
         """
-        mock_supports.return_value = True
+        ctx = DummyCtx(identifier="some-lib", repo_dir=self.repo_dir)
 
-        commands: list[str] = []
+        installer = NixFlakeInstaller()
 
-        def side_effect(cmd: str, cwd: str | None = None, preview: bool = False, **_: object) -> None:
-            commands.append(cmd)
+        buf = io.StringIO()
+        with patch(
+            "pkgmgr.actions.install.installers.nix_flake.shutil.which"
+        ) as which_mock, patch(
+            "pkgmgr.actions.install.installers.nix_flake.os.system"
+        ) as system_mock, redirect_stdout(buf):
+            self._enable_nix_in_module(which_mock)
 
-        mock_run_command.side_effect = side_effect
+            # Simulate os.system returning success (exit code 0)
+            system_mock.return_value = 0
 
-        with patch.dict(os.environ, {"PKGMGR_DISABLE_NIX_FLAKE_INSTALLER": ""}, clear=False):
-            self.installer.run(self.ctx)
+            # Sanity: supports() must be True
+            self.assertTrue(installer.supports(ctx))
 
-        remove_cmd = f"nix profile remove {self.installer.PROFILE_NAME} || true"
-        install_pkgmgr_cmd = f"nix profile install {self.ctx.repo_dir}#pkgmgr"
-        install_default_cmd = f"nix profile install {self.ctx.repo_dir}#default"
+            installer.run(ctx)
 
-        self.assertIn(remove_cmd, commands)
-        self.assertIn(install_pkgmgr_cmd, commands)
-        self.assertIn(install_default_cmd, commands)
+        out = buf.getvalue()
+        self.assertIn("[INFO] Running: nix profile install", out)
+        self.assertIn("Nix flake output 'default' successfully installed.", out)
 
-        self.assertEqual(commands[0], remove_cmd)
-
-    @patch("pkgmgr.actions.install.installers.nix_flake.shutil.which")
-    @patch("pkgmgr.actions.install.installers.nix_flake.run_command")
-    def test_ensure_old_profile_removed_ignores_systemexit(
-        self,
-        mock_run_command: MagicMock,
-        mock_which: MagicMock,
-    ) -> None:
-        mock_which.return_value = "/usr/bin/nix"
-
-        def side_effect(cmd: str, cwd: str | None = None, preview: bool = False, **_: object) -> None:
-            raise SystemExit(1)
-
-        mock_run_command.side_effect = side_effect
-
-        self.installer._ensure_old_profile_removed(self.ctx)
-
-        remove_cmd = f"nix profile remove {self.installer.PROFILE_NAME} || true"
-        mock_run_command.assert_called_with(
-            remove_cmd,
-            cwd=self.ctx.repo_dir,
-            preview=self.ctx.preview,
+        # Ensure the nix command was actually invoked
+        system_mock.assert_called_with(
+            f"nix profile install {self.repo_dir}#default"
         )
 
+    def test_nix_flake_run_mandatory_failure_raises(self):
+        """
+        For a generic repository (identifier not pkgmgr/package-manager),
+        `default` is mandatory and a non-zero exit code should raise SystemExit
+        with the real exit code (e.g. 1, not 256).
+        """
+        ctx = DummyCtx(identifier="some-lib", repo_dir=self.repo_dir)
+        installer = NixFlakeInstaller()
+
+        buf = io.StringIO()
+        with patch(
+            "pkgmgr.actions.install.installers.nix_flake.shutil.which"
+        ) as which_mock, patch(
+            "pkgmgr.actions.install.installers.nix_flake.os.system"
+        ) as system_mock, redirect_stdout(buf):
+            self._enable_nix_in_module(which_mock)
+
+            # Simulate os.system returning encoded status for exit code 1
+            # os.system encodes exit code as (exit_code << 8)
+            system_mock.return_value = 1 << 8
+
+            self.assertTrue(installer.supports(ctx))
+
+            with self.assertRaises(SystemExit) as cm:
+                installer.run(ctx)
+
+        # The real exit code should be 1 (not 256)
+        self.assertEqual(cm.exception.code, 1)
+
+        out = buf.getvalue()
+        self.assertIn("[INFO] Running: nix profile install", out)
+        self.assertIn("[Error] Failed to install Nix flake output 'default'", out)
+        self.assertIn("[Error] Command exited with code 1", out)
+
+    def test_nix_flake_run_optional_failure_does_not_raise(self):
+        """
+        For the package-manager repository, the 'default' output is optional.
+        Failure to install it must not raise, but should log a warning instead.
+        """
+        ctx = DummyCtx(identifier="pkgmgr", repo_dir=self.repo_dir)
+        installer = NixFlakeInstaller()
+
+        calls = []
+
+        def fake_system(cmd: str) -> int:
+            calls.append(cmd)
+            # First call (pkgmgr) → success
+            if len(calls) == 1:
+                return 0
+            # Second call (default) → failure (exit code 1 encoded)
+            return 1 << 8
+
+        buf = io.StringIO()
+        with patch(
+            "pkgmgr.actions.install.installers.nix_flake.shutil.which"
+        ) as which_mock, patch(
+            "pkgmgr.actions.install.installers.nix_flake.os.system",
+            side_effect=fake_system,
+        ), redirect_stdout(buf):
+            self._enable_nix_in_module(which_mock)
+
+            self.assertTrue(installer.supports(ctx))
+
+            # Optional failure must NOT raise
+            installer.run(ctx)
+
+        out = buf.getvalue()
+
+        # Both outputs should have been mentioned
+        self.assertIn(
+            "attempting to install profile outputs: pkgmgr, default", out
+        )
+
+        # First output ("pkgmgr") succeeded
+        self.assertIn(
+            "Nix flake output 'pkgmgr' successfully installed.", out
+        )
+
+        # Second output ("default") failed but did not raise
+        self.assertIn(
+            "[Error] Failed to install Nix flake output 'default'", out
+        )
+        self.assertIn("[Error] Command exited with code 1", out)
+        self.assertIn(
+            "Continuing despite failure to install optional output 'default'.",
+            out,
+        )
+
+        # Ensure we actually called os.system twice (pkgmgr and default)
+        self.assertEqual(len(calls), 2)
+        self.assertIn(
+            f"nix profile install {self.repo_dir}#pkgmgr",
+            calls[0],
+        )
+        self.assertIn(
+            f"nix profile install {self.repo_dir}#default",
+            calls[1],
+        )
+
+    def test_nix_flake_supports_respects_disable_env(self):
+        """
+        PKGMGR_DISABLE_NIX_FLAKE_INSTALLER=1 must disable the installer,
+        even if flake.nix exists and nix is available.
+        """
+        ctx = DummyCtx(identifier="pkgmgr", repo_dir=self.repo_dir)
+        installer = NixFlakeInstaller()
+
+        with patch(
+            "pkgmgr.actions.install.installers.nix_flake.shutil.which"
+        ) as which_mock:
+            self._enable_nix_in_module(which_mock)
+            os.environ["PKGMGR_DISABLE_NIX_FLAKE_INSTALLER"] = "1"
+
+            self.assertFalse(installer.supports(ctx))
+
 
 if __name__ == "__main__":
     unittest.main()