fix(orchestrator): settle queue jobs whose holder is gone instead of showing them running
The hub read queue.json alone, so an active entry whose hold had exited kept showing as running. client/queue_live.py probes each hold lock as queue.py's held() does (shared non-blocking flock, hard links probed too) and reads the whole queue-history.jsonl, cached by inode, size and mtime; an active entry with a history row started after its enqueue ends with that row's exit, one whose holder is gone becomes gone. The outcome lives in the hub-owned field hub_outcome; queue.py status is never rewritten. resources.py uses the same settled(), plan medians skip the queue kind, and a guard test keeps every client module in the Dockerfile COPY line. Also closes fd leaks in read_history and records.read_regular. MEDUSA: 11 findings in resources.py (lines 73, 81, 262, 445 x2, 576, 227, 479, 524, 578) and crons.py:125 accepted as false positives after an independent review, operator pick 2026-11-12. 🎯 Fix: 92 % 💥 Break: 5 % 💸 Cost: 100 % ⏱️ Time: 100 % 🛡️ Security: 100 % 🏆 Score: 87 🚦 Gate: ✅ 90/5, cost and time <= 100, security 100 % Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
3418738993
commit
85b79de0d8
15 files changed
+534
-39
No files matched your search
@@ -1,5 +1,6 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import fcntl
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
@@ -108,10 +109,29 @@ class TestParsing(unittest.TestCase):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
council = Path(tmp)
|
||||
self.assertEqual(sampler.queue(council), {"waiting": 0, "active": 0})
|
||||
entries = [{"status": "waiting"}, {"status": "waiting"}, {"status": "active"}, {"status": "done"}, "junk"]
|
||||
entries = [{"status": status, "enqueued": NOW} for status in ("waiting", "waiting", "active", "done")] + [{"status": "active"}, "junk"]
|
||||
(council / "queue.json").write_text(json.dumps({"entries": entries}), encoding="utf-8")
|
||||
self.assertEqual(sampler.queue(council), {"waiting": 2, "active": 1})
|
||||
|
||||
def test_the_queue_counts_only_live_entries_as_the_hub_settles_them(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
council = Path(tmp)
|
||||
(council / "holds").mkdir()
|
||||
handles = [(council / "holds" / f"7-{n}.lock").open("w", encoding="utf-8") for n in (1, 2, 3)]
|
||||
for handle in handles:
|
||||
self.addCleanup(handle.close)
|
||||
fcntl.flock(handle, fcntl.LOCK_EX)
|
||||
entries = [
|
||||
{"id": "test-1", "status": "waiting", "lock": "7-1", "enqueued": NOW},
|
||||
{"id": "test-2", "status": "active", "lock": "7-2", "enqueued": NOW},
|
||||
{"id": "quality-high-3", "status": "active", "lock": "7-3", "enqueued": NOW},
|
||||
{"id": "test-4", "status": "waiting", "lock": "7-4", "enqueued": NOW},
|
||||
{"id": "test-5", "status": "active", "lock": "7-5", "enqueued": NOW},
|
||||
]
|
||||
(council / "queue.json").write_text(json.dumps({"entries": entries}), encoding="utf-8")
|
||||
(council / "queue-history.jsonl").write_text(json.dumps({"id": "quality-high-3-1794514233361361126", "start": NOW + 10, "end": NOW + 20, "exit": 2}) + "\n", encoding="utf-8")
|
||||
self.assertEqual(sampler.queue(council), {"waiting": 1, "active": 1})
|
||||
|
||||
|
||||
class TestCpu(unittest.TestCase):
|
||||
def test_cpu_is_the_busy_share_since_the_ticks_the_run_before_kept_and_the_sampler_never_sleeps(self):
|
||||
@@ -233,7 +253,7 @@ class TestSamplerInScratchHome(unittest.TestCase):
|
||||
(stubs / tool).chmod(0o755)
|
||||
self.sys = host_sys(base / "sys")
|
||||
self.log = base / "calls.jsonl"
|
||||
(self.council / "queue.json").write_text(json.dumps({"entries": [{"status": "waiting"}, {"status": "active"}, {"status": "active"}]}), encoding="utf-8")
|
||||
(self.council / "queue.json").write_text(json.dumps({"entries": [{"status": status, "enqueued": NOW} for status in ("waiting", "active", "active")]}), encoding="utf-8")
|
||||
self.env = {
|
||||
**os.environ, "HOME": str(self.home), "PATH": f"{stubs}:/usr/bin:/bin",
|
||||
"STUB_LOG": str(self.log), "STUB_DF": DF, "STUB_NVIDIA_SMI": NVIDIA_SMI.splitlines()[0] + "\n",
|
||||
|
||||
Reference in new issue
Block a user