feat(kvb-git-hotfix): share hotfixes in the register and sync before every action

The Maintainer messaged every agent about every hotfix, waking each for a full turn. Hotfixes are now register rows hotfix/<slug> (repository identity by git common dir and normalized origin URL, branch, sha, PR, failing-line signature, targets, stage), written with hotfix.py add/update and queried only when an agent acts, never by cron: pending lists merged hotfixes missing from HEAD (filtered by targets) and open ones; match finds hotfixes whose signature fits a failing line, evaluated in one worker under a shared deadline; void tells gate.py run, before every target, when a merged hotfix missing from HEAD makes the rest of the run worthless (exit 3), the operator's one exception to never cutting a job short; sync fetches first, then under the worktree locks merges origin/<default> and every base up the funnel chain recorded with rasci.py set --base, refusing a detached or foreign branch, a diverged base or a busy worktree, aborting cleanly on a conflict.

gate.py gains hold, which runs any heavy command, deploys included, under the worktree lock, forwards signals, returns the command's code and exits 75 when busy; run holds the same lock and takes the sync lock shared. kvb-check-gate, kvb-ci-triage, kvb-org-organize (Peer protocol, capacity), kvb-org-coordinate and kvb-org-split follow; rasci gaps, wsjf and artifacts skip hotfix and operator rows. Three independent skeptic rounds found 34 defects (three of them high), all fixed with tests; negative controls fail against the earlier code.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/test_gate.py

@@ -1,17 +1,28 @@
-"""Exercise gate.py: static `make test*` listing, runs judged per target, and the per-HEAD gate."""
+"""Exercise gate.py: static `make test*` listing, runs judged per target, hotfix voids, and the per-HEAD gate."""
 
 from __future__ import annotations
 
+import contextlib
+import fcntl
+import importlib.util
+import io
 import json
+import os
+import shutil
+import signal
 import subprocess
 import sys
 import tempfile
+import time
 import unittest
 from pathlib import Path
 from typing import ClassVar
 
 REPO_ROOT = Path(__file__).resolve().parents[1]
 SCRIPT = REPO_ROOT / "skills" / "kvb-check-gate" / "scripts" / "gate.py"
+HOTFIX = REPO_ROOT / "skills" / "kvb-git-hotfix" / "scripts" / "hotfix.py"
+RUN_LOCK = "kvb-gate-run.lock"
+SYNC_LOCK = "kvb-gate-sync.lock"
 GREEN = {
     "Makefile": (
         "include mk/*.mk\n"
@@ -56,7 +67,28 @@ RED = {
     "tracked.txt": "one\n",
 }
 USAGE = {"Makefile": "test:\n\t@true\nbuild:\n\t@true\n"}
+VOID = {"Makefile": ".PHONY: test test-a test-b\ntest:\n\t@echo all\ntest-a:\n\t@echo a\ntest-b:\n\t@echo b\n"}
 FAILED = ("--fail-regex", r"FAILED TARGETS: \S")
+SYNC_HOLDER = "4242\thotfix.py sync --task t\n"
+COUNTING_CHILD = (
+    "import os, signal, sys, time\n"
+    "got = []\n"
+    "signal.signal(signal.SIGINT, lambda signum, frame: got.append(signum))\n"
+    "with open(sys.argv[1] + '.tmp', 'w', encoding='utf-8') as out:\n"
+    "    out.write(str(os.getpid()))\n"
+    "os.replace(sys.argv[1] + '.tmp', sys.argv[1])\n"
+    "while not got:\n"
+    "    time.sleep(0.05)\n"
+    "time.sleep(0.5)\n"
+    "sys.exit(10 + len(got))\n"
+)
+
+
+def load_gate() -> object:
+    spec = importlib.util.spec_from_file_location("gate_under_test", SCRIPT)
+    gate = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(gate)
+    return gate
 
 
 class GateCase(unittest.TestCase):
@@ -72,6 +104,7 @@ class GateCase(unittest.TestCase):
         self.env = {
             "PATH": "/usr/bin:/bin",
             "HOME": tmp.name,
+            "AGENTS_COUNCIL_ROOT": str(self.root / "council"),
             "GIT_CONFIG_NOSYSTEM": "1",
             "GIT_AUTHOR_NAME": "t",
             "GIT_AUTHOR_EMAIL": "t@example.org",
@@ -199,6 +232,263 @@ class TestRedPath(GateCase):
         self.assertEqual(rows["test"]["status"], "missing")
 
 
+class TestHotfixes(GateCase):
+    files = VOID
+
+    def setUp(self):
+        super().setUp()
+        origin = self.root / "origin.git"
+        self._git("init", "-q", "--bare", "-b", "main", str(origin))
+        self._git("remote", "add", "origin", str(origin))
+        self._git("push", "-q", "-u", "origin", "main")
+        self._git("remote", "set-head", "origin", "main")
+
+    def hotfix(self, slug: str, status: str, *targets: str) -> str:
+        parent = "refs/remotes/origin/main" if status == "merged" else "HEAD"
+        sha = self._git("commit-tree", "HEAD^{tree}", "-p", parent, "-m", slug).strip()
+        if status == "merged":
+            self._git("push", "-q", "origin", f"{sha}:refs/heads/main")
+        flags = [part for target in targets for part in ("--target", target)]
+        result = subprocess.run(
+            [sys.executable, str(HOTFIX), "add", slug, "--repo", str(self.repo), "--branch", f"fix/{slug}",
+             "--sha", sha, "--signature", "boom", "--status", status, "--session", "t", *flags],
+            env=self.env, capture_output=True, text=True, check=False,
+        )
+        self.assertEqual(result.returncode, 0, result.stderr)
+        return sha
+
+    def results(self) -> dict[str, str]:
+        return {r["target"]: r["status"] for r in self.status(self.repo)[1]["targets"]}
+
+    def test_a_hotfix_for_any_remaining_target_voids_the_run_before_it_starts(self):
+        self.hotfix("lint", "pr")
+        self.hotfix("b", "merged", "test-b")
+        void = "void: merge main for hotfix b, then rerun\n"
+        result = self.run_targets("test-a", "test-b", extra=())
+        self.assertEqual((result.returncode, result.stdout), (3, void), result.stderr)
+        self.assertEqual(self.results(), {"test": "missing", "test-a": "missing", "test-b": "missing"})
+        self.assertEqual(self.run_targets("test-a", extra=()).returncode, 0)
+        self.assertEqual((self.run_targets("test", extra=()).returncode, self.results()["test"]), (3, "missing"))
+
+        self._git("merge", "-q", "--ff-only", "origin/main")
+        self.assertEqual(self.run_targets("test", "test-a", "test-b", extra=()).returncode, 0)
+        self.hotfix("any", "merged")
+        result = self.run_targets("test-a", extra=())
+        self.assertEqual((result.returncode, result.stdout), (3, "void: merge main for hotfix any, then rerun\n"))
+
+    def test_status_keeps_the_gate_open_while_a_merged_hotfix_is_missing(self):
+        self.assertEqual(self.run_targets("test", "test-a", "test-b", extra=()).returncode, 0)
+        self.hotfix("lint", "pr", "test-a")
+        self.assertEqual(self.last_status_line(), "gate: green")
+        self.hotfix("rollout", "merged", "deploy")
+        self.assertEqual(self.last_status_line(), "gate: green")
+        self.hotfix("db", "merged", "test-a")
+        code, status = self.status(self.repo)
+        self.assertEqual((code, status["gate"], status["open"]), (1, "open", ["missing hotfix: db"]))
+        self.assertEqual(self.last_status_line(), "gate: open (missing hotfix: db)")
+
+    def test_a_failing_or_missing_hotfix_check_warns_and_the_run_goes_on(self):
+        register = self.root / "council" / "rasci.json"
+        register.parent.mkdir()
+        legacy = {"stage": "merged", "repo": str(self.root / "gone"), "branch": "fix/legacy", "sha": "0" * 40,
+                  "signature": "boom", "targets": [], "status": "done", "roles": {}}
+        register.write_text(json.dumps({"hotfix/legacy": legacy}), encoding="utf-8")
+        result = self.run_targets("test-a", extra=())
+        self.assertEqual(result.returncode, 0, result.stderr)
+        self.assertIn("hotfix/legacy: repository unknown", result.stderr)
+        self.assertNotIn("going on without its check", result.stderr)
+        register.write_text("{", encoding="utf-8")
+        result = self.run_targets("test-a", extra=())
+        self.assertEqual(result.returncode, 0, result.stderr)
+        self.assertIn("gate: hotfix.py void exited 2; going on without its check", result.stderr)
+        status = self.gate("status", "--repo", str(self.repo))
+        self.assertIn("gate: hotfix.py pending exited 2; going on without its check", status.stderr)
+
+        skills = self.root / "lone" / "skills"
+        shutil.copytree(SCRIPT.parents[1], skills / "kvb-check-gate", ignore=shutil.ignore_patterns("__pycache__"))
+        lone = subprocess.run(
+            [sys.executable, str(skills / "kvb-check-gate" / "scripts" / "gate.py"), "run", "--repo", str(self.repo),
+             "--target", "test-b", "--log-dir", str(self.logs)],
+            env=self.env, capture_output=True, text=True, check=False,
+        )
+        self.assertEqual(lone.returncode, 0, lone.stderr)
+        self.assertIn("kvb-git-hotfix/scripts/hotfix.py is missing; merged hotfixes are not checked", lone.stderr)
+        self.assertEqual(self.results()["test-b"], "green")
+
+    def test_verdict_lines_count_even_when_hotfix_exits_non_zero(self):
+        gate = load_gate()
+        stub = self.root / "stub.py"
+        gate.HOTFIX = stub
+        for printed, slugs, warned in (("void\thotfix/x\tabc\tall targets\n", ["x"], False), ("", [], True)):
+            with self.subTest(printed=printed):
+                stub.write_text(f"import sys\nsys.stdout.write({printed!r})\nsys.exit(2)\n", encoding="utf-8")
+                with contextlib.redirect_stderr(io.StringIO()) as err:
+                    self.assertEqual(gate.voiding_hotfixes(self.repo, ["test-a"]), slugs)
+                self.assertEqual("hotfix.py void exited 2; going on without its check" in err.getvalue(), warned)
+
+
+class TestWorktreeLock(GateCase):
+    files = VOID
+
+    def lock(self, name: str) -> Path:
+        return Path(self._git("rev-parse", "--absolute-git-dir").strip()) / name
+
+    def lock_free(self) -> bool:
+        with open(self.lock(RUN_LOCK), "a", encoding="utf-8") as handle:
+            try:
+                fcntl.flock(handle, fcntl.LOCK_EX | fcntl.LOCK_NB)
+            except BlockingIOError:
+                return False
+            return True
+
+    def test_a_busy_worktree_makes_run_and_hold_exit_75_naming_the_holder(self):
+        with open(self.lock(RUN_LOCK), "a", encoding="utf-8") as lock:
+            fcntl.flock(lock, fcntl.LOCK_EX)
+            lock.write("4242\tgate.py hold -- make deploy\n")
+            lock.flush()
+            run = self.run_targets("test-a", extra=())
+            hold = self.gate("hold", "--repo", str(self.repo), "--", "true")
+        for result in (run, hold):
+            with self.subTest(command=result.args[2]):
+                self.assertEqual(result.returncode, 75, result.stderr)
+                self.assertIn(f"another gate.py run or hold holds {self.repo} (pid 4242: gate.py hold -- make deploy)",
+                              result.stderr)
+        self.assertEqual(self.run_targets("test-a", extra=()).returncode, 0)
+        self.assertIn(f"\tgate.py run --repo {self.repo} --target test-a\n",
+                      self.lock(RUN_LOCK).read_text(encoding="utf-8"))
+
+    def test_hold_passes_the_commands_own_exit_codes_through(self):
+        for code in (1, 2, 3, 75):
+            with self.subTest(code=code):
+                result = self.gate("hold", "--repo", str(self.repo), "--", "sh", "-c", f"exit {code}")
+                self.assertEqual(result.returncode, code)
+
+    def test_hold_forwards_signals_to_the_command_and_waits_for_it(self):
+        for name, signum, group, code in (
+            ("SIGINT to the process group, as Ctrl-C sends it", signal.SIGINT, True, 11),
+            ("SIGTERM to hold only", signal.SIGTERM, False, 128 + signal.SIGTERM),
+            ("SIGHUP to hold only", signal.SIGHUP, False, 128 + signal.SIGHUP),
+        ):
+            with self.subTest(name):
+                pid_file = self.root / f"child-{signum}.pid"
+                holder = subprocess.Popen(
+                    [sys.executable, str(SCRIPT), "hold", "--repo", str(self.repo), "--",
+                     sys.executable, "-c", COUNTING_CHILD, str(pid_file)],
+                    env=self.env, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE,
+                    text=True, start_new_session=True,
+                )
+                self.addCleanup(holder.stderr.close)
+                self.addCleanup(holder.wait)
+                self.addCleanup(holder.kill)
+                deadline = time.monotonic() + 30
+                while not pid_file.exists():
+                    self.assertIsNone(holder.poll())
+                    self.assertLess(time.monotonic(), deadline)
+                    time.sleep(0.05)
+                child = int(pid_file.read_text(encoding="utf-8"))
+                self.addCleanup(self.kill, child)
+                (os.killpg if group else os.kill)(holder.pid, signum)
+                returned = holder.wait(timeout=30)
+                self.kill(child)
+                err = holder.stderr.read()
+                self.assertEqual(returned, code, err)
+                self.assertNotIn("Traceback", err)
+                self.assertTrue(self.lock_free())
+
+    def test_hold_runs_the_command_in_the_worktree_under_the_lock(self):
+        probe = (
+            "import fcntl, os, sys\n"
+            "with open(sys.argv[1], 'a', encoding='utf-8') as lock:\n"
+            "    try:\n"
+            "        fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)\n"
+            "    except BlockingIOError:\n"
+            "        print('held', os.getcwd())\n"
+            "sys.exit(5)\n"
+        )
+        result = self.gate("hold", "--repo", str(self.repo), "--", sys.executable, "-c", probe, str(self.lock(RUN_LOCK)))
+        self.assertEqual((result.returncode, result.stdout), (5, f"held {self.repo}\n"), result.stderr)
+        self.assertTrue(self.lock_free())
+        result = self.gate("hold", "--repo", str(self.repo), "--", "sh", "-c", "kill -TERM $$")
+        self.assertEqual(result.returncode, 128 + signal.SIGTERM)
+        self.assertEqual(self.gate("hold", "--repo", str(self.repo)).returncode, 2)
+
+    def test_an_orphan_of_a_hold_keeps_the_lock(self):
+        pid_file = self.root / "deploy.pid"
+        holder = subprocess.Popen(
+            [sys.executable, str(SCRIPT), "hold", "--repo", str(self.repo), "--",
+             "sh", "-c", f"echo $$ > {pid_file}.tmp; mv {pid_file}.tmp {pid_file}; exec sleep 60"],
+            env=self.env, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
+        )
+        self.addCleanup(holder.wait)
+        self.addCleanup(holder.kill)
+        deadline = time.monotonic() + 30
+        while not pid_file.exists():
+            self.assertIsNone(holder.poll())
+            self.assertLess(time.monotonic(), deadline)
+            time.sleep(0.05)
+        deploy = int(pid_file.read_text(encoding="utf-8"))
+        self.addCleanup(self.kill, deploy)
+        holder.kill()
+        holder.wait()
+        result = self.run_targets("test-a", extra=())
+        self.assertEqual(result.returncode, 75)
+        self.assertIn(f"another gate.py run or hold holds {self.repo} (pid {holder.pid}: "
+                      f"gate.py hold --repo {self.repo} -- sh -c", result.stderr)
+        self.kill(deploy)
+        deadline = time.monotonic() + 30
+        while not self.lock_free():
+            self.assertLess(time.monotonic(), deadline, "the killed deploy kept the lock")
+            time.sleep(0.05)
+        self.assertEqual(self.run_targets("test-a", extra=()).returncode, 0)
+
+    def kill(self, pid: int) -> None:
+        with contextlib.suppress(ProcessLookupError):
+            os.kill(pid, signal.SIGKILL)
+
+    def test_a_run_waits_for_a_running_sync_instead_of_failing(self):
+        with (
+            open(self.lock(SYNC_LOCK), "a", encoding="utf-8") as sync,
+            open(self.lock(RUN_LOCK), "a", encoding="utf-8") as lock,
+        ):
+            fcntl.flock(sync, fcntl.LOCK_EX)
+            sync.write(SYNC_HOLDER)
+            sync.flush()
+            fcntl.flock(lock, fcntl.LOCK_EX)
+            run = subprocess.Popen(
+                [sys.executable, str(SCRIPT), "run", "--repo", str(self.repo), "--target", "test-a",
+                 "--log-dir", str(self.logs)],
+                env=self.env, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True,
+            )
+            self.addCleanup(run.wait)
+            self.addCleanup(run.kill)
+            self.assertEqual(
+                run.stderr.readline(),
+                f"gate: a hotfix.py sync merges in {self.repo} (pid 4242: hotfix.py sync --task t); waiting for it at "
+                f"most {load_gate().SYNC_WAIT_SECONDS} s\n",
+            )
+        out, err = run.communicate(timeout=60)
+        self.assertEqual(run.returncode, 0, err)
+        self.assertTrue(out.startswith("green\ttest-a\t"), out)
+
+    def test_a_sync_outlasting_the_wait_ends_it_with_75_naming_the_sync(self):
+        gate = load_gate()
+        gate.SYNC_WAIT_SECONDS = 1
+        with open(self.lock(SYNC_LOCK), "a", encoding="utf-8") as sync:
+            fcntl.flock(sync, fcntl.LOCK_EX)
+            sync.write(SYNC_HOLDER)
+            sync.flush()
+            with (
+                contextlib.redirect_stderr(io.StringIO()) as err,
+                self.assertRaises(SystemExit) as raised,
+                gate.worktree_lock(self.repo, ["gate.py", "hold"], []),
+            ):
+                self.fail("the worktree lock was taken during a sync")
+        self.assertEqual(raised.exception.code, 75)
+        self.assertIn(f"a hotfix.py sync still holds {self.repo} after 1 s (pid 4242: hotfix.py sync --task t)",
+                      err.getvalue())
+        self.assertTrue(self.lock_free())
+
+
 class TestUsage(GateCase):
     files = USAGE