fix(orchestrator): settle queue jobs whose holder is gone instead of showing them running

The hub read queue.json alone, so an active entry whose hold had exited kept showing as running. client/queue_live.py probes each hold lock as queue.py's held() does (shared non-blocking flock, hard links probed too) and reads the whole queue-history.jsonl, cached by inode, size and mtime; an active entry with a history row started after its enqueue ends with that row's exit, one whose holder is gone becomes gone. The outcome lives in the hub-owned field hub_outcome; queue.py status is never rewritten. resources.py uses the same settled(), plan medians skip the queue kind, and a guard test keeps every client module in the Dockerfile COPY line. Also closes fd leaks in read_history and records.read_regular.

MEDUSA: 11 findings in resources.py (lines 73, 81, 262, 445 x2, 576, 227, 479, 524, 578) and crons.py:125 accepted as false positives after an independent review, operator pick 2026-11-12.

🎯 Fix: 92 %
💥 Break: 5 %
💸 Cost: 100 %
⏱️ Time: 100 %
🛡️ Security: 100 %
🏆 Score: 87
🚦 Gate: ✅ 90/5, cost and time <= 100, security 100 %

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Kevin Veen-BirkenbachandClaude Opus 5.5 committed 2026-11-12 23:09:32 +00:00
1 parent 3418738993
commit 85b79de0d8
15 files changed
+534 -39

No files matched your search

+22 -2
View File
@@ -1,5 +1,6 @@
from __future__ import annotations
import fcntl
import json
import os
import subprocess
@@ -108,10 +109,29 @@ class TestParsing(unittest.TestCase):
with tempfile.TemporaryDirectory() as tmp:
council = Path(tmp)
self.assertEqual(sampler.queue(council), {"waiting": 0, "active": 0})
entries = [{"status": "waiting"}, {"status": "waiting"}, {"status": "active"}, {"status": "done"}, "junk"]
entries = [{"status": status, "enqueued": NOW} for status in ("waiting", "waiting", "active", "done")] + [{"status": "active"}, "junk"]
(council / "queue.json").write_text(json.dumps({"entries": entries}), encoding="utf-8")
self.assertEqual(sampler.queue(council), {"waiting": 2, "active": 1})
def test_the_queue_counts_only_live_entries_as_the_hub_settles_them(self):
with tempfile.TemporaryDirectory() as tmp:
council = Path(tmp)
(council / "holds").mkdir()
handles = [(council / "holds" / f"7-{n}.lock").open("w", encoding="utf-8") for n in (1, 2, 3)]
for handle in handles:
self.addCleanup(handle.close)
fcntl.flock(handle, fcntl.LOCK_EX)
entries = [
{"id": "test-1", "status": "waiting", "lock": "7-1", "enqueued": NOW},
{"id": "test-2", "status": "active", "lock": "7-2", "enqueued": NOW},
{"id": "quality-high-3", "status": "active", "lock": "7-3", "enqueued": NOW},
{"id": "test-4", "status": "waiting", "lock": "7-4", "enqueued": NOW},
{"id": "test-5", "status": "active", "lock": "7-5", "enqueued": NOW},
]
(council / "queue.json").write_text(json.dumps({"entries": entries}), encoding="utf-8")
(council / "queue-history.jsonl").write_text(json.dumps({"id": "quality-high-3-1794514233361361126", "start": NOW + 10, "end": NOW + 20, "exit": 2}) + "\n", encoding="utf-8")
self.assertEqual(sampler.queue(council), {"waiting": 1, "active": 1})
class TestCpu(unittest.TestCase):
def test_cpu_is_the_busy_share_since_the_ticks_the_run_before_kept_and_the_sampler_never_sleeps(self):
@@ -233,7 +253,7 @@ class TestSamplerInScratchHome(unittest.TestCase):
(stubs / tool).chmod(0o755)
self.sys = host_sys(base / "sys")
self.log = base / "calls.jsonl"
(self.council / "queue.json").write_text(json.dumps({"entries": [{"status": "waiting"}, {"status": "active"}, {"status": "active"}]}), encoding="utf-8")
(self.council / "queue.json").write_text(json.dumps({"entries": [{"status": status, "enqueued": NOW} for status in ("waiting", "active", "active")]}), encoding="utf-8")
self.env = {
**os.environ, "HOME": str(self.home), "PATH": f"{stubs}:/usr/bin:/bin",
"STUB_LOG": str(self.log), "STUB_DF": DF, "STUB_NVIDIA_SMI": NVIDIA_SMI.splitlines()[0] + "\n",