fix(democracy): skip worktrees with a running make in the stop-stage cleanup
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
b69a7f85d9
commit
ed0ea62b52
3 files changed
+36
-2
No files matched your search
@@ -95,7 +95,8 @@ stays silent below the thresholds.
|
||||
- **Warn**: the chair and every session get the biggest consumers; the chair
|
||||
plans a cleanup and grants no large slot.
|
||||
- **Stop**: new heavy jobs are blocked in every session (teardown commands
|
||||
never), and the hook runs the configured low-risk cleanup by itself.
|
||||
never), and the hook runs the configured low-risk cleanup by itself; it
|
||||
skips every worktree where a gate or deploy process still runs.
|
||||
- **Critical**: additionally a desktop alarm to the operator. A running stack
|
||||
is torn down only with the operator's OK.
|
||||
- A measured fall that would reach the critical threshold within the
|
||||
|
||||
@@ -25,7 +25,8 @@ projection_minutes, interval_seconds, cache_interval_seconds, heavy_commands,
|
||||
never_block (regexes), cleanup (shell commands), worktree_cleanup (shell
|
||||
commands with {worktree}), worktrees_command (shell command printing the
|
||||
running stacks' worktree paths, one per line), worktree_marker (path relative
|
||||
to a worktree that must exist), cache_dirs, cache_budget_percent,
|
||||
to a worktree that must exist), busy_processes (process names; a worktree
|
||||
where one of them runs is skipped), cache_dirs, cache_budget_percent,
|
||||
slot_need_gb {kind: GB}, notify (argv the alert text is appended to),
|
||||
load_limits {cpu_percent, mem_percent, sample_seconds, kinds}: admit for a
|
||||
kind listed in kinds also needs CPU busy and RAM used below the limits.
|
||||
@@ -313,9 +314,25 @@ def running_worktrees(cfg: dict) -> list[str]:
|
||||
return [t for t in trees if os.path.exists(os.path.join(t, cfg["worktree_marker"]))]
|
||||
|
||||
|
||||
def busy(tree: str, names: list[str]) -> bool:
|
||||
root = os.path.realpath(tree)
|
||||
for pid in filter(str.isdigit, os.listdir("/proc")):
|
||||
try:
|
||||
if Path(f"/proc/{pid}/comm").read_text(encoding="utf-8").strip() not in names:
|
||||
continue
|
||||
cwd = os.readlink(f"/proc/{pid}/cwd")
|
||||
except OSError:
|
||||
continue
|
||||
if cwd == root or cwd.startswith(root + os.sep):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def run_cleanup(cfg: dict) -> list[str]:
|
||||
commands = list(cfg["cleanup"])
|
||||
for tree in running_worktrees(cfg):
|
||||
if busy(tree, cfg["busy_processes"]):
|
||||
continue
|
||||
commands += [t.format(worktree=shlex.quote(tree)) for t in cfg["worktree_cleanup"]]
|
||||
results = []
|
||||
for command in commands:
|
||||
|
||||
@@ -66,6 +66,7 @@ class TestWatchCommands(unittest.TestCase):
|
||||
"worktree_cleanup": [],
|
||||
"worktrees_command": "true",
|
||||
"worktree_marker": "Makefile",
|
||||
"busy_processes": ["sleep"],
|
||||
"cache_dirs": [str(self.cache)],
|
||||
"cache_budget_percent": 3,
|
||||
"slot_need_gb": {"swarm": 30, "test": 0.001},
|
||||
@@ -158,6 +159,21 @@ class TestWatchCommands(unittest.TestCase):
|
||||
self.assertEqual((marked / "cleaned").read_text(encoding="utf-8"), "x\n")
|
||||
self.assertFalse((foreign / "cleaned").exists())
|
||||
|
||||
def test_full_cleanup_skips_a_worktree_where_a_busy_process_runs(self):
|
||||
idle, gated = self.base / "idle", self.base / "gated"
|
||||
for tree in (idle, gated):
|
||||
(tree / "sub").mkdir(parents=True)
|
||||
(tree / "Makefile").write_text("", encoding="utf-8")
|
||||
gate = subprocess.Popen(["sleep", "60"], cwd=gated / "sub")
|
||||
self.addCleanup(gate.wait)
|
||||
self.addCleanup(gate.kill)
|
||||
self.cfg["worktrees_command"] = f"echo {idle}; echo {gated}"
|
||||
self.cfg["worktree_cleanup"] = ["echo x >> {worktree}/cleaned"]
|
||||
self._write_config()
|
||||
self.assertEqual(self._run("cleanup", "--full").returncode, 0)
|
||||
self.assertTrue((idle / "cleaned").exists())
|
||||
self.assertFalse((gated / "cleaned").exists())
|
||||
|
||||
def test_without_config_every_command_is_a_no_op(self):
|
||||
self.config.unlink()
|
||||
result = self._run("status")
|
||||
|
||||
Reference in new issue
Block a user