fix(democracy): skip worktrees with a running make in the stop-stage cleanup
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
b69a7f85d9
commit
ed0ea62b52
3 files changed
+36
-2
No files matched your search
@@ -95,7 +95,8 @@ stays silent below the thresholds.
|
|||||||
- **Warn**: the chair and every session get the biggest consumers; the chair
|
- **Warn**: the chair and every session get the biggest consumers; the chair
|
||||||
plans a cleanup and grants no large slot.
|
plans a cleanup and grants no large slot.
|
||||||
- **Stop**: new heavy jobs are blocked in every session (teardown commands
|
- **Stop**: new heavy jobs are blocked in every session (teardown commands
|
||||||
never), and the hook runs the configured low-risk cleanup by itself.
|
never), and the hook runs the configured low-risk cleanup by itself; it
|
||||||
|
skips every worktree where a gate or deploy process still runs.
|
||||||
- **Critical**: additionally a desktop alarm to the operator. A running stack
|
- **Critical**: additionally a desktop alarm to the operator. A running stack
|
||||||
is torn down only with the operator's OK.
|
is torn down only with the operator's OK.
|
||||||
- A measured fall that would reach the critical threshold within the
|
- A measured fall that would reach the critical threshold within the
|
||||||
|
|||||||
@@ -25,7 +25,8 @@ projection_minutes, interval_seconds, cache_interval_seconds, heavy_commands,
|
|||||||
never_block (regexes), cleanup (shell commands), worktree_cleanup (shell
|
never_block (regexes), cleanup (shell commands), worktree_cleanup (shell
|
||||||
commands with {worktree}), worktrees_command (shell command printing the
|
commands with {worktree}), worktrees_command (shell command printing the
|
||||||
running stacks' worktree paths, one per line), worktree_marker (path relative
|
running stacks' worktree paths, one per line), worktree_marker (path relative
|
||||||
to a worktree that must exist), cache_dirs, cache_budget_percent,
|
to a worktree that must exist), busy_processes (process names; a worktree
|
||||||
|
where one of them runs is skipped), cache_dirs, cache_budget_percent,
|
||||||
slot_need_gb {kind: GB}, notify (argv the alert text is appended to),
|
slot_need_gb {kind: GB}, notify (argv the alert text is appended to),
|
||||||
load_limits {cpu_percent, mem_percent, sample_seconds, kinds}: admit for a
|
load_limits {cpu_percent, mem_percent, sample_seconds, kinds}: admit for a
|
||||||
kind listed in kinds also needs CPU busy and RAM used below the limits.
|
kind listed in kinds also needs CPU busy and RAM used below the limits.
|
||||||
@@ -313,9 +314,25 @@ def running_worktrees(cfg: dict) -> list[str]:
|
|||||||
return [t for t in trees if os.path.exists(os.path.join(t, cfg["worktree_marker"]))]
|
return [t for t in trees if os.path.exists(os.path.join(t, cfg["worktree_marker"]))]
|
||||||
|
|
||||||
|
|
||||||
|
def busy(tree: str, names: list[str]) -> bool:
|
||||||
|
root = os.path.realpath(tree)
|
||||||
|
for pid in filter(str.isdigit, os.listdir("/proc")):
|
||||||
|
try:
|
||||||
|
if Path(f"/proc/{pid}/comm").read_text(encoding="utf-8").strip() not in names:
|
||||||
|
continue
|
||||||
|
cwd = os.readlink(f"/proc/{pid}/cwd")
|
||||||
|
except OSError:
|
||||||
|
continue
|
||||||
|
if cwd == root or cwd.startswith(root + os.sep):
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
def run_cleanup(cfg: dict) -> list[str]:
|
def run_cleanup(cfg: dict) -> list[str]:
|
||||||
commands = list(cfg["cleanup"])
|
commands = list(cfg["cleanup"])
|
||||||
for tree in running_worktrees(cfg):
|
for tree in running_worktrees(cfg):
|
||||||
|
if busy(tree, cfg["busy_processes"]):
|
||||||
|
continue
|
||||||
commands += [t.format(worktree=shlex.quote(tree)) for t in cfg["worktree_cleanup"]]
|
commands += [t.format(worktree=shlex.quote(tree)) for t in cfg["worktree_cleanup"]]
|
||||||
results = []
|
results = []
|
||||||
for command in commands:
|
for command in commands:
|
||||||
|
|||||||
@@ -66,6 +66,7 @@ class TestWatchCommands(unittest.TestCase):
|
|||||||
"worktree_cleanup": [],
|
"worktree_cleanup": [],
|
||||||
"worktrees_command": "true",
|
"worktrees_command": "true",
|
||||||
"worktree_marker": "Makefile",
|
"worktree_marker": "Makefile",
|
||||||
|
"busy_processes": ["sleep"],
|
||||||
"cache_dirs": [str(self.cache)],
|
"cache_dirs": [str(self.cache)],
|
||||||
"cache_budget_percent": 3,
|
"cache_budget_percent": 3,
|
||||||
"slot_need_gb": {"swarm": 30, "test": 0.001},
|
"slot_need_gb": {"swarm": 30, "test": 0.001},
|
||||||
@@ -158,6 +159,21 @@ class TestWatchCommands(unittest.TestCase):
|
|||||||
self.assertEqual((marked / "cleaned").read_text(encoding="utf-8"), "x\n")
|
self.assertEqual((marked / "cleaned").read_text(encoding="utf-8"), "x\n")
|
||||||
self.assertFalse((foreign / "cleaned").exists())
|
self.assertFalse((foreign / "cleaned").exists())
|
||||||
|
|
||||||
|
def test_full_cleanup_skips_a_worktree_where_a_busy_process_runs(self):
|
||||||
|
idle, gated = self.base / "idle", self.base / "gated"
|
||||||
|
for tree in (idle, gated):
|
||||||
|
(tree / "sub").mkdir(parents=True)
|
||||||
|
(tree / "Makefile").write_text("", encoding="utf-8")
|
||||||
|
gate = subprocess.Popen(["sleep", "60"], cwd=gated / "sub")
|
||||||
|
self.addCleanup(gate.wait)
|
||||||
|
self.addCleanup(gate.kill)
|
||||||
|
self.cfg["worktrees_command"] = f"echo {idle}; echo {gated}"
|
||||||
|
self.cfg["worktree_cleanup"] = ["echo x >> {worktree}/cleaned"]
|
||||||
|
self._write_config()
|
||||||
|
self.assertEqual(self._run("cleanup", "--full").returncode, 0)
|
||||||
|
self.assertTrue((idle / "cleaned").exists())
|
||||||
|
self.assertFalse((gated / "cleaned").exists())
|
||||||
|
|
||||||
def test_without_config_every_command_is_a_no_op(self):
|
def test_without_config_every_command_is_a_no_op(self):
|
||||||
self.config.unlink()
|
self.config.unlink()
|
||||||
result = self._run("status")
|
result = self._run("status")
|
||||||
|
|||||||
Reference in new issue
Block a user