diff --git a/skills/kvb-check-gate/scripts/gate.py b/skills/kvb-check-gate/scripts/gate.py index 8d364ef..a95504f 100755 --- a/skills/kvb-check-gate/scripts/gate.py +++ b/skills/kvb-check-gate/scripts/gate.py @@ -28,15 +28,42 @@ Args: command's process group while hold keeps waiting for it. For deploys and every other heavy command that is no `make test*` target. - The worktree lock is `kvb-gate-run.lock` in the worktree's git dir, - taken without waiting and passed to make or the command, so it stays - held while they or a process they left running on this host live; a - process inside a container (`docker compose exec`) runs on there after - they die, and the lock no longer covers it. run and hold write their PID - and command into it, and a busy message names them. `hotfix.py sync` - takes `kvb-gate-sync.lock` exclusively, recording its PID and command - there, then the worktree lock, and releases them in reverse order; run - and hold take the worktree lock while holding `kvb-gate-sync.lock` + The worktree lock is `kvb-gate-run.lock` in `/tmp/agents-locks/`, + key the first 16 hex digits of the SHA-256 of the worktree's real + top-level path: a fixed path no environment variable moves, so every + caller sharing this /tmp meets the same lock whether the git dir is + writable or not; a reboot ends every holder, so it needs no durability. + Both directories are created with mode 700 where missing; each level and + lock file is opened relative to its parent without following a link and + refused unless it is the current user's and not writable by group or + others, a lock file a regular file with one link. /tmp belongs to one + mount namespace, so run, hold and sync also hold the git dir lock, + `kvb-gate-run.lock` in the worktree's git dir: the one inode that a + caller with another /tmp (an agentbox box with the project bind-mounted, + the host) and a gate.py from before /tmp/agents-locks share. It is + created with mode 600 where missing, opened read-only where the git dir + or the file refuses writing, opened without following a link and refused + unless it is a regular file of the current user with one link; while it + is missing and cannot be created, only the /tmp lock guards the worktree + and a caller with another /tmp is not kept out. `kvb-gate.json` and + `kvb-gate.json.lock` in the git common dir pass the same checks, and + `kvb-gate.json` is replaced by a new file, never written in place; run + opens the lock, creates and removes a temporary file beside + `kvb-gate.json` and reads it before it takes the locks, so a read-only + git common dir or a refused `kvb-gate.json` ends it before any target + runs. Both + locks are taken without waiting and passed to make or the command, so + they stay held while they or a process they left running on this host + live; a process inside a container (`docker compose exec`) runs on there + after they die, and the locks no longer cover it. run and hold write + their PID and command into both, the git dir lock only when it opened + for writing, and a busy message names them; one on the git dir lock + shows its last recorded line as possibly stale, since a holder that + opened it read-only records nothing there. `hotfix.py sync` takes + `kvb-gate-sync.lock` beside the worktree lock exclusively, recording its + PID and command there, then the worktree lock and the git dir lock, + recording them in both too, and releases them in reverse + order; run and hold take the locks while holding `kvb-gate-sync.lock` shared, so they wait for a running sync, naming it, for at most SYNC_WAIT_SECONDS, and a busy worktree lock then means another run or hold. @@ -58,28 +85,38 @@ Output: Exit codes: list: 0 done, 2 usage error. - status: 0 gate green, 1 gate open, 2 usage error. + status: 0 gate green, 1 gate open, 2 usage error or a `kvb-gate.json` + that cannot be opened or trusted. run: 0 every target green, 1 a target red, 2 usage error, an excluded - target, dirty tracked files or HEAD moved during a run, 3 void: the remaining targets are - neither run nor recorded, 75 (EX_TEMPFAIL) busy: another run or hold - holds the worktree, or a sync still holds it after SYNC_WAIT_SECONDS. + target, a worktree lock, git dir lock, `kvb-gate.json` or its lock + that cannot be created, opened or trusted, a git common dir it cannot + write (before any target runs), dirty tracked files or + HEAD moved during a run, 3 void: the remaining targets are neither + run nor recorded, 75 (EX_TEMPFAIL) busy: another run, hold or sync + holds the worktree or its git dir lock, or a sync still holds it + after SYNC_WAIT_SECONDS. hold: the command's exit code unchanged, 128+N when signal N ended it; - before the command started 2 on a usage error, 75 busy as for run, - 128+N when signal N reached hold. + before the command started 2 on a usage error, a worktree lock or + git dir lock that cannot be created, opened or trusted or a command + that cannot start, 75 busy as for run, 128+N when signal N reached + hold. """ from __future__ import annotations import argparse import contextlib +import errno import fcntl import glob +import hashlib import json import os import pwd import re import shlex import signal +import stat import subprocess import sys import tempfile @@ -109,6 +146,12 @@ POLL_SECONDS = 0.1 BUSY = os.EX_TEMPFAIL FORWARDED = (signal.SIGINT, signal.SIGTERM, signal.SIGHUP) HOTFIX = Path(__file__).resolve().parents[2] / "kvb-git-hotfix" / "scripts" / "hotfix.py" +LOCK_BASE = Path("/tmp/agents-locks") +KEY_HEX = 16 +DIRECTORY_MODE = 0o700 +LOCK_MODE = 0o600 +READ_ONLY = (errno.EROFS, errno.EACCES, errno.EPERM) +HOLDER_BYTES = 4096 CONFIG = Path(pwd.getpwuid(os.getuid()).pw_dir) / ".claude" / "agents-council" / "config.json" EXCLUDED = "gate_excluded_targets" @@ -241,21 +284,61 @@ def state_path(repo: Path) -> Path: def load(path: Path) -> dict: - if not path.exists(): - return {} try: - data = json.loads(path.read_text(encoding="utf-8")) - except ValueError: - fail(f"{path} is not valid JSON; move it aside to start over") + handle = os.fdopen(git_file(path, os.O_RDONLY), encoding="utf-8") + except FileNotFoundError: + return {} + except OSError as error: + fail(f"cannot read {path}: {error.strerror or error}") + with handle: + try: + data = json.load(handle) + except ValueError: + fail(f"{path} is not valid JSON; move it aside to start over") if not isinstance(data, dict): fail(f"{path} holds no object; move it aside to start over") return data -def record(path: Path, sha: str, target: str, entry: dict) -> None: +def unwritable(path: Path, error: OSError) -> NoReturn: + fail(f"cannot write {path}: {error.strerror or error}; nothing ran. Run `gate.py run` as an operator row on the " + "host, where the git common dir is writable; `make ` alone runs but records nothing") + + +def state_lock(path: Path) -> TextIO: + """Args: + path: STATE in the git common dir. + + Returns: + Its lock file, opened by git_file() for reading and writing, once a + temporary file could be created and removed beside path and load() + read path; exit 2 naming the host-side route to a writable git common + dir, or as load() does, otherwise. + """ + lock_path = path.with_name(f"{STATE}.lock") + try: + lock = os.fdopen(git_file(lock_path, os.O_RDWR | os.O_CREAT), "r+", encoding="utf-8") + except OSError as error: + unwritable(lock_path, error) + try: + handle, temp = tempfile.mkstemp(dir=path.parent, prefix=f".{STATE}.") + except OSError as error: + lock.close() + unwritable(path.parent, error) + os.close(handle) + os.unlink(temp) + try: + load(path) + except SystemExit: + lock.close() + raise + return lock + + +def record(lock: TextIO, path: Path, sha: str, target: str, entry: dict) -> None: # Linked worktrees share the git common dir, so parallel runs write this file. - with open(path.with_name(f"{STATE}.lock"), "w", encoding="utf-8") as lock: - fcntl.flock(lock, fcntl.LOCK_EX) + fcntl.flock(lock, fcntl.LOCK_EX) + try: data = load(path) data.setdefault(sha, {})[target] = entry handle, temp = tempfile.mkstemp(dir=path.parent, prefix=f".{STATE}.") @@ -263,6 +346,8 @@ def record(path: Path, sha: str, target: str, entry: dict) -> None: json.dump(data, out, indent=1, sort_keys=True) out.write("\n") os.replace(temp, path) + finally: + fcntl.flock(lock, fcntl.LOCK_UN) def first_failure(log: Path, checks: list[re.Pattern]) -> str | None: @@ -314,19 +399,20 @@ def claim(lock: TextIO, command: list[str]) -> None: lock: the open, exclusively held lock file. command: the holder's command line. """ + lock.seek(0) lock.truncate(0) lock.write(f"{os.getpid()}\t{shlex.join(command)}\n") lock.flush() -def holder(path: Path) -> str: +def holder(fd: int) -> str: """Name the holder that `claim` recorded in a lock file. Args: - path: the lock file. + fd: an open descriptor of the lock file. """ try: - pid, _, command = path.read_text(encoding="utf-8").strip().partition("\t") + pid, _, command = os.pread(fd, HOLDER_BYTES, 0).decode("utf-8", "replace").strip().partition("\t") except OSError: pid, command = "", "" return f"pid {pid}: {command}" if pid and command else "holder not recorded" @@ -351,34 +437,169 @@ def shared(sync: TextIO, repo: Path, caught: list[int]) -> None: if caught: sys.exit(128 + caught[0]) if time.monotonic() >= deadline: - busy(f"a hotfix.py sync still holds {repo} after {SYNC_WAIT_SECONDS} s ({holder(Path(sync.name))}); " + busy(f"a hotfix.py sync still holds {repo} after {SYNC_WAIT_SECONDS} s ({holder(sync.fileno())}); " "retry once it ends") if not waiting: - print(f"gate: a hotfix.py sync merges in {repo} ({holder(Path(sync.name))}); waiting for it at most " + print(f"gate: a hotfix.py sync merges in {repo} ({holder(sync.fileno())}); waiting for it at most " f"{SYNC_WAIT_SECONDS} s", file=sys.stderr, flush=True) waiting = True time.sleep(POLL_SECONDS) +class Busy(Exception): + pass + + +def owned(fd: int, path: Path, regular: bool, private: bool) -> int: + """Args: + fd: an open descriptor of path, closed on a refusal. + path: named in the refusal. + regular: whether path is a lock or state file rather than a directory. + private: whether group and others must not be able to write path. + + Returns: + fd when it is a directory, or with regular a regular file with one + link, of the current user that, with private, group and others cannot + write; exit 2 naming path otherwise. + """ + info = os.fstat(fd) + if regular: + kind, fits = f"regular file of uid {os.getuid()} with one link", stat.S_ISREG(info.st_mode) and info.st_nlink == 1 + else: + kind, fits = f"directory of uid {os.getuid()}", stat.S_ISDIR(info.st_mode) + if fits and info.st_uid == os.getuid() and not (private and info.st_mode & 0o022): + return fd + os.close(fd) + fail(f"{path} must be a {kind}" + (" that group and others cannot write" if private else "")) + + +def git_file(path: Path, flags: int) -> int: + """Args: + path: a lock or state file in a git dir, where a sandboxed caller may + have put a link, FIFO or second name. + flags: os.open flags; O_NOFOLLOW, O_NONBLOCK and O_CLOEXEC are added, + and a created file gets LOCK_MODE. + + Returns: + Its descriptor once owned() passes it as a regular file, without the + private check; OSError of the open propagates. + """ + return owned(os.open(path, flags | os.O_NOFOLLOW | os.O_NONBLOCK | os.O_CLOEXEC, LOCK_MODE), path, True, False) + + +def lock_files(top: Path, base: Path) -> tuple[TextIO, TextIO]: + """Args: + top: the worktree's top-level directory, as `git rev-parse + --show-toplevel` prints it. + base: the directory holding one lock directory per worktree, LOCK_BASE + outside tests. + + Returns: + RUN_LOCK and SYNC_LOCK of base/, open for reading and writing, + key the first KEY_HEX hex digits of the SHA-256 of the real path of + top. base and base/ are created with DIRECTORY_MODE where + missing; each level and lock file is opened relative to its parent's + descriptor without following a link and passes owned() before + anything is written to it. Exit 2 naming the level otherwise. + """ + key = hashlib.sha256(os.fsencode(os.path.realpath(top))).hexdigest()[:KEY_HEX] + where = base.parent + try: + fd = os.open(where, os.O_RDONLY | os.O_DIRECTORY | os.O_CLOEXEC) + for name in (base.name, key): + where = where / name + with contextlib.suppress(FileExistsError): + os.mkdir(name, DIRECTORY_MODE, dir_fd=fd) + parent, fd = fd, os.open(name, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | os.O_CLOEXEC, dir_fd=fd) + os.close(parent) + owned(fd, where, False, True) + files = [] + for name in (RUN_LOCK, SYNC_LOCK): + where = base / key / name + flags = os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW | os.O_NONBLOCK | os.O_CLOEXEC + opened = owned(os.open(name, flags, LOCK_MODE, dir_fd=fd), where, True, True) + files.append(os.fdopen(opened, "r+", encoding="utf-8")) + os.close(fd) + except OSError as error: + fail(f"cannot open the worktree lock {where}: {error.strerror or error}") + return files[0], files[1] + + +def git_dir_file(path: Path) -> TextIO | None: + """Args: + path: RUN_LOCK in a worktree's git dir. + + Returns: + path opened by git_file() for reading and writing, created where + missing, or read-only where the git dir or the file refuses writing; + None when it is missing and cannot be created; exit 2 when it cannot + be opened otherwise. + """ + try: + return os.fdopen(git_file(path, os.O_RDWR | os.O_CREAT), "r+", encoding="utf-8") + except OSError as error: + if error.errno not in READ_ONLY: + fail(f"cannot open the git dir lock {path}: {error.strerror or error}") + try: + return os.fdopen(git_file(path, os.O_RDONLY), "r", encoding="utf-8") + except FileNotFoundError: + return None + except OSError as error: + fail(f"cannot open the git dir lock {path}: {error.strerror or error}") + + @contextlib.contextmanager -def worktree_lock(repo: Path, command: list[str], caught: list[int]) -> Iterator[int]: - """Hold the worktree lock; yields its file descriptor for pass_fds. +def git_dir_lock(git_dir: Path, command: list[str]) -> Iterator[int | None]: + """Args: + git_dir: the worktree's git dir, one inode for every mount namespace + that sees the worktree and for gate.py from before LOCK_BASE. + command: the holder's command line, recorded in it when git_dir_file() + opened it for writing. + + Returns: + A context holding RUN_LOCK of git_dir exclusively and yielding its + descriptor, or holding nothing and yielding None when git_dir_file() + finds it missing and not creatable; Busy with the line last recorded + there, labelled possibly stale, while another run, hold or sync holds it. + """ + lock = git_dir_file(git_dir / RUN_LOCK) + if lock is None: + yield None + return + with lock: + try: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + raise Busy(f"last recorded {holder(lock.fileno())}, possibly stale") from None + if lock.writable(): + claim(lock, command) + yield lock.fileno() + + +@contextlib.contextmanager +def worktree_lock(repo: Path, command: list[str], caught: list[int]) -> Iterator[tuple[int, ...]]: + """Hold the worktree lock and the git dir lock; yields their file descriptors for pass_fds. Args: - repo: the worktree whose git dir holds RUN_LOCK and SYNC_LOCK. - command: the holder's command line, recorded in RUN_LOCK. + repo: the worktree whose lock_files and git_dir_lock are taken. + command: the holder's command line, recorded in both RUN_LOCKs. caught: signals the caller caught; the first ends a wait for a sync with 128+N. """ git_dir = Path(git(repo, "rev-parse", "--absolute-git-dir").strip()) - with open(git_dir / RUN_LOCK, "a", encoding="utf-8") as lock: - with open(git_dir / SYNC_LOCK, "a", encoding="utf-8") as sync: + run_lock, sync_lock = lock_files(Path(git(repo, "rev-parse", "--show-toplevel").strip()), LOCK_BASE) + with run_lock as lock, contextlib.ExitStack() as held: + with sync_lock as sync: shared(sync, repo, caught) try: fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) except BlockingIOError: - busy(f"another gate.py run or hold holds {repo} ({holder(git_dir / RUN_LOCK)})") + busy(f"another gate.py run or hold holds {repo} ({holder(lock.fileno())})") + try: + git_fd = held.enter_context(git_dir_lock(git_dir, command)) + except Busy as other: + busy(f"another gate.py run or hold or a hotfix.py sync holds the git dir lock of {repo} ({other})") claim(lock, command) - yield lock.fileno() + yield (lock.fileno(),) if git_fd is None else (lock.fileno(), git_fd) def repository(value: str) -> Path: @@ -483,7 +704,11 @@ def cmd_run(args: argparse.Namespace) -> int: barred = sorted(set(args.target) & excluded()) if barred: fail(f"excluded from every gate by {EXCLUDED} in {CONFIG}: {', '.join(barred)}") - with worktree_lock(repo, ["gate.py", "run", "--repo", str(repo), *target_flags(args.target)], []) as lock: + state = state_path(repo) + with ( + state_lock(state) as state_file, + worktree_lock(repo, ["gate.py", "run", "--repo", str(repo), *target_flags(args.target)], []) as locks, + ): sha = head(repo) dirty, untracked = tree(repo) if dirty: @@ -492,7 +717,6 @@ def cmd_run(args: argparse.Namespace) -> int: print(f"untracked\t{len(untracked)} file(s), not part of the gate") log_dir = Path(args.log_dir).resolve() log_dir.mkdir(parents=True, exist_ok=True) - state = state_path(repo) red = False order = list(dict.fromkeys(args.target)) checking = hotfix_checks() @@ -505,7 +729,7 @@ def cmd_run(args: argparse.Namespace) -> int: with open(log, "w", encoding="utf-8") as out: code = subprocess.run( ["make", "-C", str(repo), target], - stdin=subprocess.DEVNULL, stdout=out, stderr=subprocess.STDOUT, pass_fds=(lock,), check=False, + stdin=subprocess.DEVNULL, stdout=out, stderr=subprocess.STDOUT, pass_fds=locks, check=False, ).returncode if head(repo) != sha: fail(f"HEAD moved during {target}; nothing recorded for it") @@ -525,7 +749,7 @@ def cmd_run(args: argparse.Namespace) -> int: "fail_regex": [c.pattern for c in checks]} if reasons: entry["reason"] = "; ".join(reasons) - record(state, sha, target, entry) + record(state_file, state, sha, target, entry) print("\t".join([status, target, str(log), cell(entry.get("reason"))])) if changed: fail(f"{target} left tracked files dirty; remaining targets not run") @@ -551,11 +775,11 @@ def cmd_hold(args: argparse.Namespace) -> int: for signum in FORWARDED: signal.signal(signum, forward) - with worktree_lock(repo, ["gate.py", "hold", "--repo", str(repo), "--", *args.command], caught) as lock: + with worktree_lock(repo, ["gate.py", "hold", "--repo", str(repo), "--", *args.command], caught) as locks: if caught: return 128 + caught[0] try: - child = subprocess.Popen(args.command, cwd=repo, pass_fds=(lock,), start_new_session=True) + child = subprocess.Popen(args.command, cwd=repo, pass_fds=locks, start_new_session=True) except OSError as error: fail(f"cannot start {args.command[0]}: {error.strerror}") for signum in caught: diff --git a/skills/kvb-git-hotfix/scripts/hotfix.py b/skills/kvb-git-hotfix/scripts/hotfix.py index 0273232..f8fab5f 100755 --- a/skills/kvb-git-hotfix/scripts/hotfix.py +++ b/skills/kvb-git-hotfix/scripts/hotfix.py @@ -46,9 +46,10 @@ Args: tracked tree, merge origin/ and each base up the chain, each with `git merge --no-verify --no-edit`. From the tree check to the last merge or abort it holds `kvb-gate-sync.lock`, with its PID and - command recorded there, and the worktree lock of the kvb-check-gate - skill's gate.py, and refuses while a `gate.py run` or `gate.py hold` - holds it, naming the holder. A watch.py busy process it sees in + command recorded there, and the worktree lock and git dir lock of the + kvb-check-gate skill's gate.py, and refuses while a `gate.py run` or + `gate.py hold` or, by the git dir lock, a sync with another /tmp + holds one, naming the holder. A watch.py busy process it sees in also refuses, but processes in another sandbox's PID namespace stay unseen, so the lock is the guard. The default branch is origin/HEAD; --default is read only when that ref is missing and must agree with it otherwise. @@ -742,27 +743,30 @@ def cmd_void(rasci: ModuleType, args: argparse.Namespace) -> int: @contextlib.contextmanager def locked(repo: str, command: list[str]) -> Iterator[None]: - """Hold gate.py's sync lock and worktree lock of the worktree. + """Hold gate.py's sync lock, worktree lock and git dir lock of the worktree. Args: repo: the worktree. - command: sync's command line, recorded in the sync lock. + command: sync's command line, recorded in every lock it holds that it can write. """ if not GATE.is_file(): yield return gate = module("gate", GATE) git_dir = Path(git_out(repo, "rev-parse", "--absolute-git-dir").strip()) - with ( - open(git_dir / gate.SYNC_LOCK, "a", encoding="utf-8") as sync, - open(git_dir / gate.RUN_LOCK, "a", encoding="utf-8") as run, - ): + run_lock, sync_lock = gate.lock_files(Path(git_out(repo, "rev-parse", "--show-toplevel").strip()), gate.LOCK_BASE) + with sync_lock as sync, run_lock as run, contextlib.ExitStack() as held: fcntl.flock(sync, fcntl.LOCK_EX) gate.claim(sync, command) try: fcntl.flock(run, fcntl.LOCK_EX | fcntl.LOCK_NB) except BlockingIOError: - fail(f"a gate.py run or hold holds {repo} ({gate.holder(git_dir / gate.RUN_LOCK)}); sync once it ends") + fail(f"a gate.py run or hold holds {repo} ({gate.holder(run.fileno())}); sync once it ends") + try: + held.enter_context(gate.git_dir_lock(git_dir, command)) + except gate.Busy as other: + fail(f"a gate.py run or hold or another sync holds the git dir lock of {repo} ({other}); sync once it ends") + gate.claim(run, command) yield diff --git a/tests/test_gate.py b/tests/test_gate.py index 417d53d..4995386 100644 --- a/tests/test_gate.py +++ b/tests/test_gate.py @@ -5,6 +5,7 @@ from __future__ import annotations import argparse import contextlib import fcntl +import hashlib import importlib.util import io import json @@ -12,6 +13,7 @@ import os import pwd import shutil import signal +import stat import subprocess import sys import tempfile @@ -26,6 +28,18 @@ SCRIPT = REPO_ROOT / "skills" / "kvb-check-gate" / "scripts" / "gate.py" HOTFIX = REPO_ROOT / "skills" / "kvb-git-hotfix" / "scripts" / "hotfix.py" RUN_LOCK = "kvb-gate-run.lock" SYNC_LOCK = "kvb-gate-sync.lock" +STATE = "kvb-gate.json" +LOCK_BASE = Path("/tmp/agents-locks") +EARLIER_GATE = ( + "import fcntl, sys\n" + "with open(sys.argv[1], 'a', encoding='utf-8') as lock:\n" + " try:\n" + " fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)\n" + " except BlockingIOError:\n" + " print('held')\n" + " else:\n" + " print('free')\n" +) GREEN = { "Makefile": ( "include mk/*.mk\n" @@ -70,6 +84,7 @@ RED = { "tracked.txt": "one\n", } USAGE = {"Makefile": "test:\n\t@true\nbuild:\n\t@true\n"} +MARKING = {"Makefile": ".PHONY: test\ntest:\n\t@touch \"$$MARKER\"\n"} VOID = {"Makefile": ".PHONY: test test-a test-b\ntest:\n\t@echo all\ntest-a:\n\t@echo a\ntest-b:\n\t@echo b\n"} ORACLE = {"Makefile": ".PHONY: test test-oracle\ntest:\n\t@echo all\ntest-oracle:\n\t@echo oracle\n"} FAILED = ("--fail-regex", r"FAILED TARGETS: \S") @@ -95,6 +110,17 @@ def load_gate() -> object: return gate +def lock_dir(worktree: Path) -> Path: + return LOCK_BASE / hashlib.sha256(os.fsencode(str(worktree))).hexdigest()[:16] + + +def drop_lock_dir(path: Path) -> None: + if path.is_symlink(): + path.unlink() + else: + shutil.rmtree(path, ignore_errors=True) + + class GateCase(unittest.TestCase): files: ClassVar[dict[str, str]] @@ -105,6 +131,7 @@ class GateCase(unittest.TestCase): self.repo = self.root / "repo" self.logs = self.root / "logs" self.repo.mkdir() + self.addCleanup(drop_lock_dir, lock_dir(self.repo)) self.env = { "PATH": "/usr/bin:/bin", "HOME": tmp.name, @@ -378,16 +405,268 @@ class TestWorktreeLock(GateCase): files = VOID def lock(self, name: str) -> Path: - return Path(self._git("rev-parse", "--absolute-git-dir").strip()) / name + LOCK_BASE.mkdir(mode=0o700, exist_ok=True) + lock_dir(self.repo).mkdir(mode=0o700, exist_ok=True) + return lock_dir(self.repo) / name - def lock_free(self) -> bool: - with open(self.lock(RUN_LOCK), "a", encoding="utf-8") as handle: + def test_a_read_only_git_dir_holds_no_lock_and_the_hold_still_runs(self): + git_dir = self.repo / ".git" + git_dir.chmod(0o555) + self.addCleanup(git_dir.chmod, 0o755) + result = self.gate("hold", "--repo", str(self.repo), "--", "sh", "-c", "exit 7") + self.assertEqual(result.returncode, 7, result.stderr) + self.assertEqual(list(git_dir.glob("kvb-gate-*.lock")), []) + locks = lock_dir(self.repo) + self.assertEqual(sorted(path.name for path in locks.iterdir()), [RUN_LOCK, SYNC_LOCK]) + self.assertEqual(stat.S_IMODE(locks.stat().st_mode), 0o700) + self.assertEqual({stat.S_IMODE((locks / name).stat().st_mode) for name in (RUN_LOCK, SYNC_LOCK)}, {0o600}) + + def test_the_lock_path_ignores_tmpdir_council_root_state_and_home(self): + with open(self.lock(RUN_LOCK), "a", encoding="utf-8") as lock: + fcntl.flock(lock, fcntl.LOCK_EX) + for name in ("TMPDIR", "AGENTS_COUNCIL_ROOT", "XDG_STATE_HOME", "HOME"): + for value in (str(self.root / f"other-{name}"), "relative"): + with self.subTest(name=name, value=value): + result = subprocess.run( + [sys.executable, str(SCRIPT), "hold", "--repo", str(self.repo), "--", "true"], + cwd=self.root, env={**self.env, name: value}, capture_output=True, text=True, check=False, + ) + self.assertEqual(result.returncode, 75, result.stderr) + self.assertEqual([path.name for path in self.root.iterdir() if path.name.startswith(("other-", "relative"))], []) + + def test_a_link_shared_directory_or_second_name_at_any_level_is_refused(self): + gate = load_gate() + elsewhere, victim = self.root / "elsewhere", self.root / "victim" + elsewhere.mkdir() + victim.write_text("keep\n", encoding="utf-8") + key = lock_dir(self.repo).name + for case, refusal in ( + ("base link", "cannot open the worktree lock {base}:"), + ("key link", "cannot open the worktree lock {base}/{key}:"), + ("run link", f"cannot open the worktree lock {{base}}/{{key}}/{RUN_LOCK}:"), + ("sync link", f"cannot open the worktree lock {{base}}/{{key}}/{SYNC_LOCK}:"), + ("base shared", f"{{base}} must be a directory of uid {os.getuid()} that group and others cannot write"), + ("key shared", f"{{base}}/{{key}} must be a directory of uid {os.getuid()} that group and others cannot write"), + ("run hardlink", f"{{base}}/{{key}}/{RUN_LOCK} must be a regular file of uid {os.getuid()} with one link"), + ("run fifo", f"{{base}}/{{key}}/{RUN_LOCK} must be a regular file of uid {os.getuid()} with one link"), + ): + with self.subTest(case): + base = self.root / case.replace(" ", "-") + level, kind = case.split() + path = {"base": base, "key": base / key, "run": base / key / RUN_LOCK, "sync": base / key / SYNC_LOCK}[level] + for directory in {"base": (), "key": (base,)}.get(level, (base, base / key)): + directory.mkdir(mode=0o700) + if kind == "link": + path.symlink_to(elsewhere if level in ("base", "key") else elsewhere / path.name) + elif kind == "shared": + path.mkdir() + path.chmod(0o777) + elif kind == "hardlink": + os.link(victim, path) + else: + os.mkfifo(path, 0o600) + with ( + contextlib.redirect_stderr(io.StringIO()) as err, + self.assertRaises(SystemExit) as raised, + ): + gate.lock_files(self.repo, base) + self.assertEqual(raised.exception.code, 2) + self.assertIn(refusal.format(base=base, key=key), err.getvalue()) + self.assertEqual(list(elsewhere.iterdir()), []) + self.assertEqual(victim.read_text(encoding="utf-8"), "keep\n") + marker = self.root / "ran" + lock_dir(self.repo).parent.mkdir(mode=0o700, exist_ok=True) + lock_dir(self.repo).symlink_to(elsewhere, target_is_directory=True) + result = self.gate("hold", "--repo", str(self.repo), "--", "touch", str(marker)) + self.assertEqual(result.returncode, 2, result.stderr) + self.assertIn(f"gate: cannot open the worktree lock {lock_dir(self.repo)}:", result.stderr) + self.assertNotIn("Traceback", result.stderr) + self.assertEqual((list(elsewhere.iterdir()), marker.exists()), ([], False)) + + def test_a_held_git_dir_lock_keeps_run_and_hold_out_and_a_read_only_one_is_still_held(self): + git_lock = self.repo / ".git" / RUN_LOCK + with open(git_lock, "a", encoding="utf-8") as lock: + fcntl.flock(lock, fcntl.LOCK_EX) + lock.write("4242\tgate.py hold -- make deploy\n") + lock.flush() + run = self.run_targets("test-a", extra=()) + hold = self.gate("hold", "--repo", str(self.repo), "--", "true") + for result in (run, hold): + with self.subTest(command=result.args[2]): + self.assertEqual(result.returncode, 75, result.stderr) + self.assertIn(f"another gate.py run or hold or a hotfix.py sync holds the git dir lock of {self.repo} " + "(last recorded pid 4242: gate.py hold -- make deploy, possibly stale)", result.stderr) + git_lock.chmod(0o400) + git_dir = self.repo / ".git" + git_dir.chmod(0o555) + self.addCleanup(git_dir.chmod, 0o755) + reader = EARLIER_GATE.replace("'a'", "'r'") + result = self.gate("hold", "--repo", str(self.repo), "--", sys.executable, "-c", reader, str(git_lock)) + self.assertEqual((result.returncode, result.stdout), (0, "held\n"), result.stderr) + self.assertEqual(git_lock.read_text(encoding="utf-8"), "4242\tgate.py hold -- make deploy\n") + + def test_a_missing_git_dir_lock_is_created_and_held_so_an_earlier_gate_py_stays_out(self): + git_lock = self.repo / ".git" / RUN_LOCK + self.assertFalse(git_lock.exists()) + result = self.gate("hold", "--repo", str(self.repo), "--", sys.executable, "-c", EARLIER_GATE, str(git_lock)) + self.assertEqual((result.returncode, result.stdout), (0, "held\n"), result.stderr) + self.assertEqual(stat.S_IMODE(git_lock.stat().st_mode), 0o600) + self.assertIn(f"\tgate.py hold --repo {self.repo} -- {sys.executable} -c ", git_lock.read_text(encoding="utf-8")) + self.assertTrue(self.free(git_lock)) + + def test_a_gate_py_with_a_tmp_of_its_own_is_kept_out_by_the_git_dir_lock(self): + host, box = load_gate(), load_gate() + for gate, name in ((host, "host-tmp"), (box, "box-tmp")): + (self.root / name).mkdir(mode=0o700) + gate.LOCK_BASE = self.root / name / "agents-locks" + with ( + host.worktree_lock(self.repo, ["gate.py", "hold", "--", "make", "deploy"], []), + contextlib.redirect_stderr(io.StringIO()) as err, + self.assertRaises(SystemExit) as raised, + box.worktree_lock(self.repo, ["gate.py", "hold"], []), + ): + self.fail("a gate.py with a /tmp of its own took the worktree lock") + self.assertEqual(raised.exception.code, 75) + self.assertIn(f"holds the git dir lock of {self.repo} (last recorded pid {os.getpid()}: gate.py hold -- make " + "deploy, possibly stale)", err.getvalue()) + with box.worktree_lock(self.repo, ["gate.py", "hold"], []) as fds: + self.assertEqual(len(fds), 2) + + def test_a_read_only_git_dir_lock_holder_is_named_live_by_its_tmp_lock_and_the_git_dir_line_as_stale(self): + git_lock = self.repo / ".git" / RUN_LOCK + git_lock.write_text("4242\tgate.py run --repo OLD-FINISHED-RUN\n", encoding="utf-8") + git_lock.chmod(0o400) + self.addCleanup(git_lock.chmod, 0o600) + live, box = load_gate(), load_gate() + (self.root / "box-tmp").mkdir(mode=0o700) + box.LOCK_BASE = self.root / "box-tmp" / "agents-locks" + with live.worktree_lock(self.repo, ["gate.py", "hold", "--", "live"], []): + same_tmp = self.gate("hold", "--repo", str(self.repo), "--", "true") + with ( + contextlib.redirect_stderr(io.StringIO()) as err, + self.assertRaises(SystemExit) as raised, + box.worktree_lock(self.repo, ["gate.py", "hold"], []), + ): + self.fail("a gate.py with a /tmp of its own took the worktree lock") + self.assertEqual(same_tmp.returncode, 75, same_tmp.stderr) + self.assertIn(f"another gate.py run or hold holds {self.repo} (pid {os.getpid()}: gate.py hold -- live)", + same_tmp.stderr) + self.assertEqual(raised.exception.code, 75) + self.assertIn("(last recorded pid 4242: gate.py run --repo OLD-FINISHED-RUN, possibly stale)", err.getvalue()) + self.assertEqual(git_lock.read_text(encoding="utf-8"), "4242\tgate.py run --repo OLD-FINISHED-RUN\n") + + def test_a_fifo_link_or_second_name_as_git_dir_lock_ends_the_hold_with_2_at_once(self): + git_lock = self.repo / ".git" / RUN_LOCK + victim, marker = self.root / "victim", self.root / "ran" + victim.write_text("keep\n", encoding="utf-8") + second_name = f"{git_lock} must be a regular file of uid {os.getuid()} with one link" + for case, plant, refusal in ( + ("fifo", lambda: os.mkfifo(git_lock, 0o600), second_name), + ("hardlink", lambda: os.link(victim, git_lock), second_name), + ("symlink", lambda: git_lock.symlink_to(victim), f"cannot open the git dir lock {git_lock}:"), + ): + with self.subTest(case): + plant() + try: + result = subprocess.run( + [sys.executable, str(SCRIPT), "hold", "--repo", str(self.repo), "--", "touch", str(marker)], + env=self.env, capture_output=True, text=True, timeout=30, check=False, + ) + finally: + git_lock.unlink() + self.assertEqual(result.returncode, 2, result.stderr) + self.assertIn(refusal, result.stderr) + self.assertNotIn("Traceback", result.stderr) + self.assertFalse(marker.exists()) + self.assertEqual(victim.read_text(encoding="utf-8"), "keep\n") + + def test_record_and_status_never_follow_a_link_or_second_name_in_the_git_common_dir(self): + gate = load_gate() + victim = self.root / "victim.json" + secret = '{"auths": {"registry": "secret"}}\n' + for case in ("lock link", "lock hardlink", "state link", "state hardlink"): + with self.subTest(case): + common = self.root / case.replace(" ", "-") + common.mkdir() + victim.write_text(secret, encoding="utf-8") + planted = common / (f"{STATE}.lock" if case.startswith("lock") else STATE) + if case.endswith("hardlink"): + os.link(victim, planted) + else: + planted.symlink_to(victim) + with ( + contextlib.redirect_stderr(io.StringIO()) as err, + self.assertRaises(SystemExit) as raised, + gate.state_lock(common / STATE) as lock, + ): + gate.record(lock, common / STATE, "0" * 40, "test-a", {"status": "green"}) + self.assertEqual(raised.exception.code, 2) + self.assertIn(str(planted), err.getvalue()) + self.assertEqual(victim.read_text(encoding="utf-8"), secret) + self.assertEqual(planted.is_symlink(), case.endswith(" link")) + self.assertEqual([path.name for path in common.iterdir() if path.name.startswith(f".{STATE}.")], []) + self.assertEqual((common / STATE).exists() and not (common / STATE).is_symlink(), + case == "state hardlink") + state = self.repo / ".git" / STATE + for case, plant, refusal in ( + ("link", lambda: state.symlink_to(victim), f"gate: cannot read {state}:"), + ("fifo", lambda: os.mkfifo(state, 0o600), f"gate: {state} must be a regular file of uid {os.getuid()}"), + ): + with self.subTest(f"status through a {case}"): + plant() + try: + result = subprocess.run( + [sys.executable, str(SCRIPT), "status", "--repo", str(self.repo), "--json"], + env=self.env, capture_output=True, text=True, timeout=30, check=False, + ) + finally: + state.unlink() + self.assertEqual((result.returncode, result.stdout), (2, ""), result.stderr) + self.assertIn(refusal, result.stderr) + self.assertNotIn("secret", result.stdout + result.stderr) + + def test_each_worktree_has_its_own_lock_and_a_subdirectory_shares_its_worktrees(self): + other = self.root / "other" + self._git("worktree", "add", "-q", "-b", "other", str(other)) + self.addCleanup(drop_lock_dir, lock_dir(other)) + (self.repo / "sub").mkdir() + self.assertNotEqual(lock_dir(self.repo), lock_dir(other)) + with open(self.lock(RUN_LOCK), "a", encoding="utf-8") as lock: + fcntl.flock(lock, fcntl.LOCK_EX) + busy = self.gate("hold", "--repo", str(self.repo / "sub"), "--", "true") + free = self.gate("hold", "--repo", str(other), "--", "true") + self.assertEqual((busy.returncode, free.returncode), (75, 0), busy.stderr + free.stderr) + self.assertTrue((lock_dir(other) / RUN_LOCK).is_file()) + + def test_a_lock_or_command_that_cannot_start_ends_the_hold_non_zero(self): + marker = self.root / "ran" + locks = self.lock(RUN_LOCK).parent + locks.chmod(0o500) + self.addCleanup(locks.chmod, 0o700) + unwritable = self.gate("hold", "--repo", str(self.repo), "--", "touch", str(marker)) + locks.chmod(0o700) + missing = self.gate("hold", "--repo", str(self.repo), "--", str(self.root / "no-such-command")) + for result, message in ( + (unwritable, f"gate: cannot open the worktree lock {locks / RUN_LOCK}: Permission denied"), + (missing, "gate: cannot start"), + ): + with self.subTest(message): + self.assertEqual(result.returncode, 2, result.stderr) + self.assertIn(message, result.stderr) + self.assertNotIn("Traceback", result.stderr) + self.assertFalse(marker.exists()) + + def free(self, path: Path) -> bool: + with open(path, "a", encoding="utf-8") as handle: try: fcntl.flock(handle, fcntl.LOCK_EX | fcntl.LOCK_NB) except BlockingIOError: return False return True + def lock_free(self) -> bool: + return self.free(self.lock(RUN_LOCK)) + def test_a_busy_worktree_makes_run_and_hold_exit_75_naming_the_holder(self): with open(self.lock(RUN_LOCK), "a", encoding="utf-8") as lock: fcntl.flock(lock, fcntl.LOCK_EX) @@ -459,8 +738,10 @@ class TestWorktreeLock(GateCase): self.assertEqual(result.returncode, 128 + signal.SIGTERM) self.assertEqual(self.gate("hold", "--repo", str(self.repo)).returncode, 2) - def test_an_orphan_of_a_hold_keeps_the_lock(self): + def test_an_orphan_of_a_hold_keeps_both_locks(self): pid_file = self.root / "deploy.pid" + git_lock = self.repo / ".git" / RUN_LOCK + git_lock.touch(mode=0o600) holder = subprocess.Popen( [sys.executable, str(SCRIPT), "hold", "--repo", str(self.repo), "--", "sh", "-c", f"echo $$ > {pid_file}.tmp; mv {pid_file}.tmp {pid_file}; exec sleep 60"], @@ -477,14 +758,15 @@ class TestWorktreeLock(GateCase): self.addCleanup(self.kill, deploy) holder.kill() holder.wait() + self.assertFalse(self.free(git_lock), "the git dir lock died with the hold") result = self.run_targets("test-a", extra=()) self.assertEqual(result.returncode, 75) self.assertIn(f"another gate.py run or hold holds {self.repo} (pid {holder.pid}: " f"gate.py hold --repo {self.repo} -- sh -c", result.stderr) self.kill(deploy) deadline = time.monotonic() + 30 - while not self.lock_free(): - self.assertLess(time.monotonic(), deadline, "the killed deploy kept the lock") + while not (self.lock_free() and self.free(git_lock)): + self.assertLess(time.monotonic(), deadline, "the killed deploy kept a lock") time.sleep(0.05) self.assertEqual(self.run_targets("test-a", extra=()).returncode, 0) @@ -536,6 +818,54 @@ class TestWorktreeLock(GateCase): self.assertTrue(self.lock_free()) +class TestReadOnlyCommonDir(GateCase): + files = MARKING + + def test_a_read_only_common_dir_ends_run_with_2_before_any_target_runs(self): + marker = self.root / "ran" + self.env["MARKER"] = str(marker) + git_dir = self.repo / ".git" + self.addCleanup(git_dir.chmod, 0o755) + for case in ("no state yet", "state of an earlier run"): + with self.subTest(case): + git_dir.chmod(0o755) + if case == "state of an earlier run": + self.assertEqual(self.run_targets("test", extra=()).returncode, 0) + marker.unlink() + before = {path.name: path.read_bytes() for path in git_dir.glob(f"{STATE}*")} + git_dir.chmod(0o555) + result = self.run_targets("test", extra=()) + self.assertEqual((result.returncode, marker.exists()), (2, False), result.stdout + result.stderr) + self.assertIn("nothing ran", result.stderr) + self.assertIn("as an operator row on the host", result.stderr) + self.assertIn("`make ` alone runs but records nothing", result.stderr) + self.assertNotIn("queue.py", result.stderr) + self.assertNotIn("Traceback", result.stderr) + self.assertEqual({path.name: path.read_bytes() for path in git_dir.glob(f"{STATE}*")}, before) + self.assertEqual(list(git_dir.glob(f".{STATE}.*")), []) + + def test_a_linked_or_invalid_state_file_ends_run_with_2_before_any_target_runs(self): + marker, victim = self.root / "ran", self.root / "victim.json" + self.env["MARKER"] = str(marker) + victim.write_text('{"auths": {"registry": "secret"}}\n', encoding="utf-8") + state = self.repo / ".git" / STATE + for case, plant, refusal in ( + ("link", lambda: state.symlink_to(victim), f"gate: cannot read {state}:"), + ("invalid JSON", lambda: state.write_text("{broken\n", encoding="utf-8"), f"gate: {state} is not valid JSON"), + ): + with self.subTest(case): + plant() + try: + result = self.run_targets("test", extra=()) + finally: + state.unlink() + self.assertEqual((result.returncode, marker.exists()), (2, False), result.stdout + result.stderr) + self.assertIn(refusal, result.stderr) + self.assertNotIn("secret", result.stdout + result.stderr) + self.assertEqual(victim.read_text(encoding="utf-8"), '{"auths": {"registry": "secret"}}\n') + self.assertEqual(list((self.repo / ".git").glob(f".{STATE}.*")), []) + + class TestUsage(GateCase): files = USAGE diff --git a/tests/test_hotfix.py b/tests/test_hotfix.py index 5d95ac8..1cbbe5f 100644 --- a/tests/test_hotfix.py +++ b/tests/test_hotfix.py @@ -5,6 +5,7 @@ from __future__ import annotations import argparse import contextlib import fcntl +import hashlib import importlib.util import io import json @@ -37,6 +38,7 @@ PROBE = ( SIGNATURE = r"connection refused .* port 5432" RUN_LOCK = "kvb-gate-run.lock" SYNC_LOCK = "kvb-gate-sync.lock" +LOCK_BASE = Path("/tmp/agents-locks") IDENTITY = { "GIT_AUTHOR_NAME": "Test", "GIT_AUTHOR_EMAIL": "test@example.org", @@ -53,6 +55,16 @@ def load_hotfix() -> object: return hotfix +def gate_locks(worktree: Path) -> Path: + return LOCK_BASE / hashlib.sha256(os.fsencode(str(worktree))).hexdigest()[:16] + + +def gate_lock(worktree: Path, name: str) -> Path: + LOCK_BASE.mkdir(mode=0o700, exist_ok=True) + gate_locks(worktree).mkdir(mode=0o700, exist_ok=True) + return gate_locks(worktree) / name + + class TestOriginUrls(unittest.TestCase): def test_spellings_of_one_repository_normalize_alike(self): hotfix = load_hotfix() @@ -172,6 +184,8 @@ class TestHotfix(unittest.TestCase): self.seed = self.published(self.origin, self.base / "seed", "app.txt") self.work = self.base / "work" + for worktree in (self.work, self.base / "feature", self.base / "line", self.base / "lone"): + self.addCleanup(shutil.rmtree, gate_locks(worktree), True) self.git(self.base, "clone", "-q", str(self.origin), str(self.work)) self.feature = self.base / "feature" self.git(self.work, "worktree", "add", "-q", "-b", "feature", str(self.feature), "origin/main") @@ -709,12 +723,20 @@ class TestHotfix(unittest.TestCase): def test_sync_refuses_while_a_gate_or_a_busy_process_holds_the_worktree(self): before = self.head(self.work) - git_dir = Path(self.git(self.work, "rev-parse", "--absolute-git-dir").strip()) - with open(git_dir / RUN_LOCK, "a", encoding="utf-8") as lock: + with open(gate_lock(self.work, RUN_LOCK), "a", encoding="utf-8") as lock: fcntl.flock(lock, fcntl.LOCK_EX) result = self.sync(self.work, "work/side") self.assertEqual(result.returncode, 2) self.assertIn(f"a gate.py run or hold holds {self.work}", result.stderr) + git_lock = Path(self.git(self.work, "rev-parse", "--absolute-git-dir").strip()) / RUN_LOCK + with open(git_lock, "a", encoding="utf-8") as lock: + fcntl.flock(lock, fcntl.LOCK_EX) + lock.write("4242\tgate.py hold -- make deploy\n") + lock.flush() + result = self.sync(self.work, "work/side") + self.assertEqual(result.returncode, 2) + self.assertIn(f"a gate.py run or hold or another sync holds the git dir lock of {self.work} " + "(last recorded pid 4242: gate.py hold -- make deploy, possibly stale)", result.stderr) config = self.base / "config.json" config.write_text(json.dumps({"busy_processes": ["sleep"]}), encoding="utf-8") self.env["AGENTS_COUNCIL_CONFIG"] = str(config) @@ -771,7 +793,7 @@ class TestHotfix(unittest.TestCase): self.assertIn(f"a gate.py run or hold holds {self.work} (pid {holder.pid}: " f"gate.py hold --repo {self.work} -- sh", result.stderr) self.assertEqual(self.head(self.work), before) - lock = Path(self.git(self.work, "rev-parse", "--absolute-git-dir").strip()) / RUN_LOCK + lock = gate_lock(self.work, RUN_LOCK) os.kill(deploy, signal.SIGKILL) deadline = time.monotonic() + 30 while not self.lock_free(lock): @@ -783,15 +805,17 @@ class TestHotfix(unittest.TestCase): line = self.base / "line" self.git(self.work, "worktree", "add", "-q", str(line), "line") self.commit(line, "side.txt", "other\n", "clashing line work") - lock = Path(self.git(self.work, "rev-parse", "--absolute-git-dir").strip()) / RUN_LOCK - probe, calls = self.base / "probe.py", self.base / "lock-states.log" + lock = gate_lock(self.work, RUN_LOCK) + git_lock = self.work / ".git" / RUN_LOCK + probe, calls, git_calls = self.base / "probe.py", self.base / "lock-states.log", self.base / "git-dir-states.log" probe.write_text(PROBE, encoding="utf-8") wrapper = self.base / "bin" / "git" wrapper.parent.mkdir() real = shutil.which("git", path=self.env["PATH"]) wrapper.write_text( "#!/bin/sh\n" - f"case \" $* \" in *\" fetch \"*|*\" merge \"*) {sys.executable} {probe} {lock} \"$*\" >> {calls} ;; esac\n" + f"case \" $* \" in *\" fetch \"*|*\" merge \"*) {sys.executable} {probe} {lock} \"$*\" >> {calls}\n" + f" {sys.executable} {probe} {git_lock} \"$*\" >> {git_calls} ;; esac\n" f"exec {real} \"$@\"\n", encoding="utf-8", ) @@ -799,13 +823,16 @@ class TestHotfix(unittest.TestCase): traced = {**self.env, "PATH": f"{wrapper.parent}:{self.env['PATH']}"} result = self.hotfix("sync", "--repo", str(self.work), "--task", "work/side", env=traced) self.assertEqual(result.returncode, 4, result.stderr) - states = [entry.split(" ", 1) for entry in calls.read_text(encoding="utf-8").splitlines()] - commands = [command.split(" ")[2] + (" --abort" if command.endswith("--abort") else "") for _, command in states] - self.assertEqual(commands, ["fetch", "merge", "merge", "merge --abort"]) - self.assertEqual([state for state, _ in states], ["free", "held", "held", "held"], states) - self.assertTrue(self.lock_free(lock)) - self.assertIn(f"\thotfix.py sync --repo {self.work} --task work/side\n", - lock.with_name(SYNC_LOCK).read_text(encoding="utf-8")) + for log, path in ((calls, lock), (git_calls, git_lock)): + with self.subTest(lock=str(path)): + states = [entry.split(" ", 1) for entry in log.read_text(encoding="utf-8").splitlines()] + commands = [command.split(" ")[2] + (" --abort" if command.endswith("--abort") else "") + for _, command in states] + self.assertEqual(commands, ["fetch", "merge", "merge", "merge --abort"]) + self.assertEqual([state for state, _ in states], ["free", "held", "held", "held"], states) + self.assertTrue(self.lock_free(path)) + for path in (lock.with_name(SYNC_LOCK), lock, git_lock): + self.assertIn(f"\thotfix.py sync --repo {self.work} --task work/side\n", path.read_text(encoding="utf-8")) def test_sync_and_update_keep_git_maintenance_off(self): calls = self.base / "git-calls.log"