fix(democracy): skip worktrees with a running make in the stop-stage cleanup

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Kevin Veen-BirkenbachandClaude Opus 5.5 committed 2026-10-31 17:14:53 +01:00
1 parent b69a7f85d9
commit ed0ea62b52
3 files changed
+36 -2

No files matched your search

+2 -1
View File
@@ -95,7 +95,8 @@ stays silent below the thresholds.
- **Warn**: the chair and every session get the biggest consumers; the chair
plans a cleanup and grants no large slot.
- **Stop**: new heavy jobs are blocked in every session (teardown commands
never), and the hook runs the configured low-risk cleanup by itself.
never), and the hook runs the configured low-risk cleanup by itself; it
skips every worktree where a gate or deploy process still runs.
- **Critical**: additionally a desktop alarm to the operator. A running stack
is torn down only with the operator's OK.
- A measured fall that would reach the critical threshold within the
+18 -1
View File
@@ -25,7 +25,8 @@ projection_minutes, interval_seconds, cache_interval_seconds, heavy_commands,
never_block (regexes), cleanup (shell commands), worktree_cleanup (shell
commands with {worktree}), worktrees_command (shell command printing the
running stacks' worktree paths, one per line), worktree_marker (path relative
to a worktree that must exist), cache_dirs, cache_budget_percent,
to a worktree that must exist), busy_processes (process names; a worktree
where one of them runs is skipped), cache_dirs, cache_budget_percent,
slot_need_gb {kind: GB}, notify (argv the alert text is appended to),
load_limits {cpu_percent, mem_percent, sample_seconds, kinds}: admit for a
kind listed in kinds also needs CPU busy and RAM used below the limits.
@@ -313,9 +314,25 @@ def running_worktrees(cfg: dict) -> list[str]:
return [t for t in trees if os.path.exists(os.path.join(t, cfg["worktree_marker"]))]
def busy(tree: str, names: list[str]) -> bool:
root = os.path.realpath(tree)
for pid in filter(str.isdigit, os.listdir("/proc")):
try:
if Path(f"/proc/{pid}/comm").read_text(encoding="utf-8").strip() not in names:
continue
cwd = os.readlink(f"/proc/{pid}/cwd")
except OSError:
continue
if cwd == root or cwd.startswith(root + os.sep):
return True
return False
def run_cleanup(cfg: dict) -> list[str]:
commands = list(cfg["cleanup"])
for tree in running_worktrees(cfg):
if busy(tree, cfg["busy_processes"]):
continue
commands += [t.format(worktree=shlex.quote(tree)) for t in cfg["worktree_cleanup"]]
results = []
for command in commands:
+16
View File
@@ -66,6 +66,7 @@ class TestWatchCommands(unittest.TestCase):
"worktree_cleanup": [],
"worktrees_command": "true",
"worktree_marker": "Makefile",
"busy_processes": ["sleep"],
"cache_dirs": [str(self.cache)],
"cache_budget_percent": 3,
"slot_need_gb": {"swarm": 30, "test": 0.001},
@@ -158,6 +159,21 @@ class TestWatchCommands(unittest.TestCase):
self.assertEqual((marked / "cleaned").read_text(encoding="utf-8"), "x\n")
self.assertFalse((foreign / "cleaned").exists())
def test_full_cleanup_skips_a_worktree_where_a_busy_process_runs(self):
idle, gated = self.base / "idle", self.base / "gated"
for tree in (idle, gated):
(tree / "sub").mkdir(parents=True)
(tree / "Makefile").write_text("", encoding="utf-8")
gate = subprocess.Popen(["sleep", "60"], cwd=gated / "sub")
self.addCleanup(gate.wait)
self.addCleanup(gate.kill)
self.cfg["worktrees_command"] = f"echo {idle}; echo {gated}"
self.cfg["worktree_cleanup"] = ["echo x >> {worktree}/cleaned"]
self._write_config()
self.assertEqual(self._run("cleanup", "--full").returncode, 0)
self.assertTrue((idle / "cleaned").exists())
self.assertFalse((gated / "cleaned").exists())
def test_without_config_every_command_is_a_no_op(self):
self.config.unlink()
result = self._run("status")