fix(orchestrator): settle queue jobs whose holder is gone instead of showing them running
The hub read queue.json alone, so an active entry whose hold had exited kept showing as running. client/queue_live.py probes each hold lock as queue.py's held() does (shared non-blocking flock, hard links probed too) and reads the whole queue-history.jsonl, cached by inode, size and mtime; an active entry with a history row started after its enqueue ends with that row's exit, one whose holder is gone becomes gone. The outcome lives in the hub-owned field hub_outcome; queue.py status is never rewritten. resources.py uses the same settled(), plan medians skip the queue kind, and a guard test keeps every client module in the Dockerfile COPY line. Also closes fd leaks in read_history and records.read_regular. MEDUSA: 11 findings in resources.py (lines 73, 81, 262, 445 x2, 576, 227, 479, 524, 578) and crons.py:125 accepted as false positives after an independent review, operator pick 2026-11-12. 🎯 Fix: 92 % 💥 Break: 5 % 💸 Cost: 100 % ⏱️ Time: 100 % 🛡️ Security: 100 % 🏆 Score: 87 🚦 Gate: ✅ 90/5, cost and time <= 100, security 100 % Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
3418738993
commit
85b79de0d8
15 files changed
+534
-39
No files matched your search
@@ -241,6 +241,10 @@ def rules_dir(root: Path) -> Path:
|
||||
return root / "rules"
|
||||
|
||||
|
||||
def client_calls(tree: ast.AST) -> list[ast.Call]:
|
||||
return [node for node in ast.walk(tree) if isinstance(node, ast.Call) and "client_module" in (getattr(node.func, "id", ""), getattr(node.func, "attr", ""))]
|
||||
|
||||
|
||||
class TestDiffsModule(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
@@ -825,12 +829,18 @@ class TestSharedLimits(unittest.TestCase):
|
||||
self.assertLessEqual(len(json.dumps(patch, ensure_ascii=False).encode("utf-8")), hub_limits.DIFF_LIMIT)
|
||||
self.assertGreater(len(permission_hook.encoded({"diff": [{"patch": patch}]})), hub_limits.DIFF_BODY_LIMIT)
|
||||
|
||||
def test_the_hub_image_copies_every_client_module_signed_loads(self):
|
||||
tree = ast.parse((HUB / "kvb_orchestrator" / "signed.py").read_text(encoding="utf-8"))
|
||||
loaded = {f"{node.args[0].value}.py" for node in ast.walk(tree) if isinstance(node, ast.Call) and getattr(node.func, "id", "") == "client_module"}
|
||||
def test_client_module_calls_are_found_bare_and_as_attributes(self):
|
||||
found = client_calls(ast.parse('signed.client_module("a", {})\nclient_module("b", {})\nother("c")\nx.other("d")'))
|
||||
self.assertEqual([node.args[0].value for node in found], ["a", "b"])
|
||||
|
||||
def test_the_hub_image_copies_every_client_module_the_hub_loads(self):
|
||||
trees = [ast.parse(path.read_text(encoding="utf-8")) for path in sorted((HUB / "kvb_orchestrator").glob("*.py"))]
|
||||
calls = [node for tree in trees for node in client_calls(tree)]
|
||||
self.assertTrue(all(node.args and isinstance(node.args[0], ast.Constant) for node in calls))
|
||||
loaded = {f"{node.args[0].value}.py" for node in calls}
|
||||
copies = [line.split()[1:] for line in (HUB / "Dockerfile").read_text(encoding="utf-8").splitlines() if line.startswith("COPY ")]
|
||||
copied = {Path(source).name for words in copies if words[-1] == "/app/client/" for source in words[:-1]}
|
||||
self.assertIn("hub_limits.py", loaded)
|
||||
self.assertLessEqual({"hub_limits.py", "queue_live.py"}, loaded)
|
||||
self.assertLessEqual(loaded, copied)
|
||||
|
||||
|
||||
|
||||
@@ -74,6 +74,7 @@ console.log(JSON.stringify({
|
||||
steps: {whole: stepsOf(70 * H, now), window: stepsOf(90 * H, 99 * H), burnt: P.steps(burnt, burnPoints, 0)},
|
||||
etas: {none: eta("none"), points: eta("points"), loop: [...P.etas(loop, looped, {now, estimation: "none"}).keys()].sort()},
|
||||
medians: P.medians(rows),
|
||||
queueMedians: P.medians([...rows, row("queue/done", {kind: "queue", status: "done", ended: true, start: 0, end: 50 * H, points: 1})]),
|
||||
median: [P.median([]), P.median([3, 1, 2]), P.median([4, 1, 3, 2])],
|
||||
bars: {noneAlways: lay("none", "always"), pointsAlways: lay("points", "always"), pointsOnly: lay("points", "points"), noneOnly: lay("none", "points"), real: lay("points", "real")},
|
||||
loop: P.bars(loop, looped, {now, estimation: "none", forecast: "always"}).map((bar) => bar.task),
|
||||
@@ -157,6 +158,9 @@ class TestPlanModule(unittest.TestCase):
|
||||
self.assertEqual(self.data["medians"], {"duration": 2 * HOUR, "perPoint": 1.5 * HOUR})
|
||||
self.assertEqual(self.data["median"], [None, 2, 2.5])
|
||||
|
||||
def test_a_done_queue_job_never_moves_the_medians(self):
|
||||
self.assertEqual(self.data["queueMedians"], self.data["medians"])
|
||||
|
||||
def test_without_estimate_a_forecast_is_as_long_as_the_median_duration_behind_its_predecessors(self):
|
||||
self.assertEqual(self.data["bars"]["noneAlways"], [
|
||||
["work/a", 80, 82, False, False],
|
||||
|
||||
+192
-2
@@ -1,5 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import fcntl
|
||||
import functools
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
@@ -12,13 +14,14 @@ from unittest import mock
|
||||
|
||||
from hub_case import HubCase
|
||||
from hub_support import REPO_ROOT, iso, key, runner, signing, verifier
|
||||
from kvb_orchestrator import settings, skill_scripts, tasks, tree
|
||||
from kvb_orchestrator import crons, records, settings, skill_scripts, tasks, tree
|
||||
|
||||
BROKEN = {"pull": [], "push": [], "source": "invalid"}
|
||||
FLOW = {"pull": ["origin/main", "store-i18n"], "push": ["main", "production"], "source": "file"}
|
||||
PRIO = ["set", "work/feature", "--prio", "3"]
|
||||
ESTIMATE = ["estimate", "work/feature", "Operator", "--value", "8", "--complexity", "3"]
|
||||
QUEUE_SCRIPT = REPO_ROOT / "skills" / "kvb-org-queue" / "scripts" / "queue.py"
|
||||
PROOF = "/home/me/worktrees/github.com/infinito-nexus/core/proof/karakalpak-archive-role"
|
||||
|
||||
|
||||
def change(argv: list, task: str = "work/feature") -> dict:
|
||||
@@ -136,11 +139,198 @@ class TestHubTasks(HubCase):
|
||||
self.assertEqual((command["kind"], command["title"], command["editable"], command["open_work"]), ("command", "git push", False, ["task"]))
|
||||
gate, deploy = rows["queue/gate-7"], rows["queue/deploy-8"]
|
||||
self.assertEqual((gate["kind"], gate["status"], gate["title"], gate["worker"], gate["editable"], gate["flow"]), ("queue", "awaiting", "make test", "Stage", False, FLOW))
|
||||
self.assertEqual(gate["queue"], {"kind": "gate", "parent": "work/feature", "enqueued": now - 60})
|
||||
self.assertEqual(gate["queue"], {"kind": "gate", "parent": "work/feature", "enqueued": now - 60, "exit": None})
|
||||
self.assertEqual((deploy["status"], deploy["flow"], deploy["queue"]["parent"]), ("running", None, "work/pinned"))
|
||||
done = rows["work/done"]
|
||||
self.assertEqual((done["ended"], done["editable"], done["open_work"], done["worktree"]), (True, False, [], ""))
|
||||
|
||||
def held_lock(self, tag: str, mode: int = fcntl.LOCK_EX) -> None:
|
||||
holds = self.council / "holds"
|
||||
holds.mkdir(parents=True, exist_ok=True)
|
||||
handle = (holds / f"{tag}.lock").open("w", encoding="utf-8")
|
||||
self.addCleanup(handle.close)
|
||||
fcntl.flock(handle, mode)
|
||||
|
||||
def queue_job(self, now: float, ident: str, status: str, task: str, lock: str, age: float) -> dict:
|
||||
granted = now - age + 10 if status == "active" else None
|
||||
return {"id": ident, "kind": ident.rpartition("-")[0], "task": task, "workdir": PROOF, "session": "s1", "name": "Production", "lock": lock, "pid": 7,
|
||||
"status": status, "enqueued": now - age, "granted": granted, "cmd": "make quality-high"}
|
||||
|
||||
def test_a_queue_job_with_a_history_row_or_a_gone_holder_is_no_longer_running(self):
|
||||
now = time.time()
|
||||
for tag in ("7-1410", "7-1451", "7-5", "7-1387", "7-1500", "7-1501", "7-1502"):
|
||||
self.held_lock(tag)
|
||||
self.held_lock("7-1481", fcntl.LOCK_SH)
|
||||
(self.council / "holds" / "7-1465.lock").write_text("", encoding="utf-8")
|
||||
(self.council / "holds" / "7-1480.lock").symlink_to(self.council / "holds" / "7-1451.lock")
|
||||
job = functools.partial(self.queue_job, now)
|
||||
queue = {"seq": 1502, "entries": [
|
||||
job("quality-high-1410", "active", "karakalpak-archive", "7-1410", 1200),
|
||||
job("test-nostack-1464", "active", "karakalpak-archive", "7-1464", 300),
|
||||
job("test-nostack-1465", "active", "iam/final-066-backchannel", "7-1465", 200),
|
||||
job("test-nostack-1466", "active", "iam/final-066-backchannel", "7-1466", 100),
|
||||
job("quality-high-1451", "active", "work/gitea-oidc-auto-link", "7-1451", 900),
|
||||
job("test-nostack-1480", "active", "work/symlinked", "7-1480", 80),
|
||||
job("test-nostack-1481", "active", "iam/final-066-backchannel", "7-1481", 70),
|
||||
job("gate-5", "active", "work/reused", "7-5", 60),
|
||||
job("test-1387", "waiting", "work/security-page-optimize", "7-1387", 50),
|
||||
job("test-1388", "waiting", "work/security-page-optimize", "7-1388", 40),
|
||||
job("test-nostack-1471", "waiting", "work/orchestrator-firewall", "../holds/7-1471", 30),
|
||||
job("test-1500", "ended", "work/foreign", "7-1500", 25),
|
||||
job("test-1501", "gone", "work/foreign", "7-1501", 24),
|
||||
{**job("test-1502", "active", "work/foreign", "7-1502", 20), "hub_outcome": {"state": "ended", "exit": 9}, "exit": 9, "end": now},
|
||||
]}
|
||||
history = [
|
||||
{"end": now - 600, "exit": 1, "id": "quality-high-1410-1794513000000000000", "kind": "quality-high", "start": now - 1180, "workdir": PROOF},
|
||||
{"concurrent": 2, "cpu_before": 53.9, "cpu_max": 100.0, "disk_gb": 1.91, "end": now - 5, "exit": 2, "free_before_gb": 113.12,
|
||||
"id": "quality-high-1410-1794514233361361126", "kind": "quality-high", "mem_before": 43.2, "mem_max": 84.3, "min_free_gb": 111.21,
|
||||
"minutes": 18.13, "signature": "infinito-nexus-core: make quality-high", "start": now - 1185, "workdir": PROOF},
|
||||
{"end": now - 20, "exit": 0, "id": "test-nostack-1464-1794514708907109250", "kind": "test-nostack", "start": now - 290, "workdir": PROOF},
|
||||
{"end": now - 3, "exit": 0, "id": "test-nostack-1466-retry", "kind": "test-nostack", "start": now - 90, "workdir": PROOF},
|
||||
{"end": now - 7200, "exit": 1, "id": "gate-5-1794500000000000000", "kind": "gate", "start": now - 7300, "workdir": PROOF},
|
||||
{"end": now - 1, "exit": 0, "id": "gate-5x-1794500000000000000", "kind": "gate", "start": now - 30, "workdir": PROOF},
|
||||
]
|
||||
self.write_council({"queue.json": queue, "queue-history.jsonl": self.jsonl(history)})
|
||||
state = self.snapshot()
|
||||
rows = {row["task"]: row for row in state["tasks"] if row["kind"] == "queue"}
|
||||
|
||||
def seen(ident: str) -> tuple:
|
||||
row = rows[f"queue/{ident}"]
|
||||
return row["status"], row["ended"], row["end"], row["queue"].get("exit")
|
||||
|
||||
self.assertEqual(seen("quality-high-1410"), ("stopped", True, now - 5, 2))
|
||||
self.assertEqual(rows["queue/quality-high-1410"]["start"], now - 1190)
|
||||
self.assertEqual(seen("test-nostack-1464"), ("done", True, now - 20, 0))
|
||||
self.assertEqual(seen("test-nostack-1465"), ("stopped", True, None, None))
|
||||
self.assertEqual(seen("test-nostack-1466"), ("stopped", True, None, None))
|
||||
self.assertEqual(seen("test-nostack-1481"), ("stopped", True, None, None))
|
||||
self.assertEqual(seen("test-nostack-1480"), ("stopped", True, None, None))
|
||||
live = ("quality-high-1451", "gate-5", "test-1502")
|
||||
shown = ("quality-high-1410", "test-nostack-1464", "test-nostack-1465", "test-nostack-1466", "test-nostack-1480", "test-nostack-1481", "test-1387", "test-1500", "test-1501", *live)
|
||||
self.assertEqual(sorted(rows), sorted(f"queue/{ident}" for ident in shown))
|
||||
self.assertEqual([seen(ident) for ident in live], [("running", False, None, None)] * 3)
|
||||
self.assertEqual([seen(ident) for ident in ("test-1387", "test-1500", "test-1501")], [("awaiting", False, None, None)] * 3)
|
||||
self.assertEqual(state["queue_line"], ["work/gitea-oidc-auto-link", "work/reused", "work/foreign", "work/security-page-optimize"])
|
||||
schedule = {row["job"]: (row["expression"], row["active"]) for row in crons.queue_rows(self.council, self.httpd.hub.history.rows)}
|
||||
self.assertEqual(schedule["quality-high-1410"], ("quality-high · exit 2", False))
|
||||
self.assertEqual(schedule["test-nostack-1465"], ("test-nostack · holder gone without an exit code", False))
|
||||
self.assertEqual(schedule["quality-high-1451"], ("quality-high · running", True))
|
||||
self.assertEqual(schedule["test-1500"], ("test · waiting", True))
|
||||
|
||||
def test_the_history_is_read_after_the_hold_locks_and_parsed_once_per_change(self):
|
||||
now = time.time()
|
||||
self.held_lock("7-1")
|
||||
queue = {"seq": 2, "entries": [self.queue_job(now, "test-1", "active", "work/a", "7-1", 100), self.queue_job(now, "test-2", "active", "work/b", "7-2", 50)]}
|
||||
self.write_council({"queue.json": queue, "queue-history.jsonl": self.jsonl([{"id": "other-9-1", "start": now - 500, "end": now - 400, "exit": 0}])})
|
||||
probe, read = crons.queue_live.holder_gone, crons.queue_live.read_history
|
||||
reads = []
|
||||
|
||||
def leaving(council: Path, tag: object) -> bool:
|
||||
if tag == "7-2":
|
||||
with (self.council / "queue-history.jsonl").open("a", encoding="utf-8") as handle:
|
||||
handle.write(json.dumps({"id": "test-2-1794514233361361126", "start": now - 40, "end": now - 1, "exit": 3}) + "\n")
|
||||
return probe(council, tag)
|
||||
|
||||
def counted(path: Path) -> list[dict]:
|
||||
reads.append(path.name)
|
||||
return read(path)
|
||||
|
||||
with mock.patch.object(crons.queue_live, "holder_gone", leaving), mock.patch.object(crons.queue_live, "read_history", counted):
|
||||
rows = {row["task"]: row for row in self.snapshot()["tasks"] if row["kind"] == "queue"}
|
||||
self.assertEqual((rows["queue/test-2"]["status"], rows["queue/test-2"]["queue"]["exit"]), ("stopped", 3))
|
||||
self.assertEqual(rows["queue/test-1"]["status"], "running")
|
||||
self.assertEqual(len(reads), 2)
|
||||
self.httpd.hub.history.rows()
|
||||
self.assertEqual(len(reads), 2)
|
||||
|
||||
def test_the_whole_history_is_read_and_a_bad_byte_costs_its_line_only(self):
|
||||
path = self.council / "queue-history.jsonl"
|
||||
self.council.mkdir(parents=True, exist_ok=True)
|
||||
rows = b"".join(json.dumps({"id": f"gate-{n}-1", "start": n, "pad": "x" * 1000}).encode() + b"\n" for n in range(6000))
|
||||
path.write_bytes(b'{"id": "bad-\xff-1"}\n' + rows + b"[[[\n" + b'{"id": "last-1"}')
|
||||
found = crons.queue_live.read_history(path)
|
||||
self.assertGreater(path.stat().st_size, 1 << 22)
|
||||
self.assertEqual((len(found), found[1]["id"], found[-2]["id"], found[-1]["id"]), (6002, "gate-0-1", "gate-5999-1", "last-1"))
|
||||
|
||||
def test_reading_a_directory_as_history_or_json_leaks_no_descriptor(self):
|
||||
folder = self.council / "queue-history.jsonl"
|
||||
folder.mkdir(parents=True)
|
||||
before = len(os.listdir("/proc/self/fd"))
|
||||
for _ in range(50):
|
||||
self.assertEqual(crons.queue_live.read_history(folder), [])
|
||||
self.assertIsNone(records.read_json(folder))
|
||||
self.assertEqual(records.read_jsonl(folder), [])
|
||||
self.assertLessEqual(len(os.listdir("/proc/self/fd")), before + 2)
|
||||
|
||||
def test_the_history_cache_reparses_on_a_new_inode_mtime_or_size_alone(self):
|
||||
path = self.council / "queue-history.jsonl"
|
||||
self.council.mkdir(parents=True, exist_ok=True)
|
||||
history = crons.History(path)
|
||||
|
||||
def write(text: str, ns: int) -> None:
|
||||
path.write_text(text, encoding="utf-8")
|
||||
os.utime(path, ns=(ns, ns))
|
||||
|
||||
write('{"id": "a-1"}\n', 10**18)
|
||||
self.assertEqual([row["id"] for row in history.rows()], ["a-1"])
|
||||
replacement = self.council / "replacement"
|
||||
replacement.write_text('{"id": "b-1"}\n', encoding="utf-8")
|
||||
os.utime(replacement, ns=(10**18, 10**18))
|
||||
os.replace(replacement, path)
|
||||
self.assertEqual([row["id"] for row in history.rows()], ["b-1"])
|
||||
write('{"id": "c-1"}\n', 10**18 + 1)
|
||||
self.assertEqual([row["id"] for row in history.rows()], ["c-1"])
|
||||
write('{"id": "c-1"}\n{"id": "d-1"}\n', 10**18 + 1)
|
||||
self.assertEqual([row["id"] for row in history.rows()], ["c-1", "d-1"])
|
||||
|
||||
def test_a_hard_link_to_a_live_hold_lock_keeps_the_hold_alive(self):
|
||||
self.held_lock("7-20")
|
||||
holds = self.council / "holds"
|
||||
os.link(holds / "7-20.lock", holds / "7-21.lock")
|
||||
self.assertEqual((holds / "7-20.lock").stat().st_nlink, 2)
|
||||
self.assertEqual([crons.queue_live.holder_gone(self.council, tag) for tag in ("7-20", "7-21")], [False, False])
|
||||
|
||||
@unittest.skipIf(os.geteuid() == 0, "root reads a file of mode 0")
|
||||
def test_a_failed_history_read_keeps_the_rows_before_and_retries_next_time(self):
|
||||
path = self.council / "queue-history.jsonl"
|
||||
self.write_council({"queue-history.jsonl": '{"id": "a-1"}\n'})
|
||||
history = crons.History(path)
|
||||
self.assertEqual([row["id"] for row in history.rows()], ["a-1"])
|
||||
with path.open("a", encoding="utf-8") as handle:
|
||||
handle.write('{"id": "b-1"}\n')
|
||||
path.chmod(0)
|
||||
self.addCleanup(path.chmod, 0o600)
|
||||
self.assertEqual([row["id"] for row in history.rows()], ["a-1"])
|
||||
self.assertIsNone(crons.queue_live.read_history(path))
|
||||
path.chmod(0o600)
|
||||
self.assertEqual([row["id"] for row in history.rows()], ["a-1", "b-1"])
|
||||
|
||||
def test_the_hub_judges_every_hold_lock_as_queue_py_held_does(self):
|
||||
queue_py = load_queue()
|
||||
holds = self.council / "holds"
|
||||
self.held_lock("7-1")
|
||||
(holds / "7-2.lock").write_text("", encoding="utf-8")
|
||||
(holds / "7-3.lock").symlink_to(holds / "7-1.lock")
|
||||
(holds / "7-7.lock").write_text("", encoding="utf-8")
|
||||
os.link(holds / "7-7.lock", holds / "7-6.lock")
|
||||
decided = {"held": "7-1", "free": "7-2", "missing": "7-404", "malformed": "x-1", "traversal": "../holds/7-1", "none": None}
|
||||
refused = {"symlink": "7-3", "hard link": "7-6"}
|
||||
hub = {name: crons.queue_live.holder_gone(self.council, tag) for name, tag in {**decided, **refused}.items()}
|
||||
with mock.patch.object(queue_py, "HOLDS", holds), mock.patch.object(queue_py.watch, "ROOT", self.council):
|
||||
for name, tag in refused.items():
|
||||
with self.subTest(refused=name), self.assertRaises(OSError):
|
||||
queue_py.held(tag)
|
||||
with self.subTest(refused=name), self.assertRaises(OSError), queue_py.nofollow.locked(self.council, holds / f"{tag}.lock", 0o600, None):
|
||||
pass
|
||||
verdicts = {name: queue_py.held(tag) for name, tag in decided.items()}
|
||||
self.assertEqual({name: not hub[name] for name in decided}, verdicts)
|
||||
self.assertEqual(verdicts, {"held": True, "free": False, "missing": False, "malformed": False, "traversal": False, "none": False})
|
||||
self.assertEqual({name: hub[name] for name in refused}, {"symlink": True, "hard link": True})
|
||||
if os.geteuid() != 0:
|
||||
(holds / "7-9.lock").write_text("", encoding="utf-8")
|
||||
(holds / "7-9.lock").chmod(0)
|
||||
self.assertFalse(crons.queue_live.holder_gone(self.council, "7-9"))
|
||||
|
||||
def test_only_the_newest_ended_tasks_are_listed(self):
|
||||
register = {f"work/old-{n:03d}": {"status": "done", "end": f"2026-11-{1 + n // 24:02d}T{n % 24:02d}:00:00+00:00"} for n in range(30)}
|
||||
self.write_council({"rasci.json": {**register, "work/open": {"status": "queued"}}})
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import fcntl
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
@@ -108,10 +109,29 @@ class TestParsing(unittest.TestCase):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
council = Path(tmp)
|
||||
self.assertEqual(sampler.queue(council), {"waiting": 0, "active": 0})
|
||||
entries = [{"status": "waiting"}, {"status": "waiting"}, {"status": "active"}, {"status": "done"}, "junk"]
|
||||
entries = [{"status": status, "enqueued": NOW} for status in ("waiting", "waiting", "active", "done")] + [{"status": "active"}, "junk"]
|
||||
(council / "queue.json").write_text(json.dumps({"entries": entries}), encoding="utf-8")
|
||||
self.assertEqual(sampler.queue(council), {"waiting": 2, "active": 1})
|
||||
|
||||
def test_the_queue_counts_only_live_entries_as_the_hub_settles_them(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
council = Path(tmp)
|
||||
(council / "holds").mkdir()
|
||||
handles = [(council / "holds" / f"7-{n}.lock").open("w", encoding="utf-8") for n in (1, 2, 3)]
|
||||
for handle in handles:
|
||||
self.addCleanup(handle.close)
|
||||
fcntl.flock(handle, fcntl.LOCK_EX)
|
||||
entries = [
|
||||
{"id": "test-1", "status": "waiting", "lock": "7-1", "enqueued": NOW},
|
||||
{"id": "test-2", "status": "active", "lock": "7-2", "enqueued": NOW},
|
||||
{"id": "quality-high-3", "status": "active", "lock": "7-3", "enqueued": NOW},
|
||||
{"id": "test-4", "status": "waiting", "lock": "7-4", "enqueued": NOW},
|
||||
{"id": "test-5", "status": "active", "lock": "7-5", "enqueued": NOW},
|
||||
]
|
||||
(council / "queue.json").write_text(json.dumps({"entries": entries}), encoding="utf-8")
|
||||
(council / "queue-history.jsonl").write_text(json.dumps({"id": "quality-high-3-1794514233361361126", "start": NOW + 10, "end": NOW + 20, "exit": 2}) + "\n", encoding="utf-8")
|
||||
self.assertEqual(sampler.queue(council), {"waiting": 1, "active": 1})
|
||||
|
||||
|
||||
class TestCpu(unittest.TestCase):
|
||||
def test_cpu_is_the_busy_share_since_the_ticks_the_run_before_kept_and_the_sampler_never_sleeps(self):
|
||||
@@ -233,7 +253,7 @@ class TestSamplerInScratchHome(unittest.TestCase):
|
||||
(stubs / tool).chmod(0o755)
|
||||
self.sys = host_sys(base / "sys")
|
||||
self.log = base / "calls.jsonl"
|
||||
(self.council / "queue.json").write_text(json.dumps({"entries": [{"status": "waiting"}, {"status": "active"}, {"status": "active"}]}), encoding="utf-8")
|
||||
(self.council / "queue.json").write_text(json.dumps({"entries": [{"status": status, "enqueued": NOW} for status in ("waiting", "active", "active")]}), encoding="utf-8")
|
||||
self.env = {
|
||||
**os.environ, "HOME": str(self.home), "PATH": f"{stubs}:/usr/bin:/bin",
|
||||
"STUB_LOG": str(self.log), "STUB_DF": DF, "STUB_NVIDIA_SMI": NVIDIA_SMI.splitlines()[0] + "\n",
|
||||
|
||||
Reference in new issue
Block a user