diff --git a/orchestrator/Dockerfile b/orchestrator/Dockerfile index 3597835..1e4cc45 100644 --- a/orchestrator/Dockerfile +++ b/orchestrator/Dockerfile @@ -36,7 +36,7 @@ RUN chmod 0755 /opt/voices /opt/whisper COPY server.py /app/server.py COPY kvb_orchestrator /app/kvb_orchestrator -COPY client/hub_limits.py client/signing.py client/passkey.py client/verifier.py /app/client/ +COPY client/hub_limits.py client/signing.py client/passkey.py client/verifier.py client/queue_live.py /app/client/ COPY templates /app/templates COPY vendor /app/vendor diff --git a/orchestrator/client/queue_live.py b/orchestrator/client/queue_live.py new file mode 100644 index 0000000..5f943a3 --- /dev/null +++ b/orchestrator/client/queue_live.py @@ -0,0 +1,181 @@ +from __future__ import annotations + +import errno +import fcntl +import json +import os +import re +import stat +from collections.abc import Callable +from pathlib import Path + +HISTORY = "queue-history.jsonl" +HOLDS = "holds" +HOLD_TAG = re.compile(r"\d+-\d+") +JOB_SUFFIX = re.compile(r"\d+") +READ_FLAGS = os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK | os.O_CLOEXEC +DIRECTORY = os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | os.O_CLOEXEC +REFUSED = frozenset({errno.ELOOP, errno.ENOTDIR}) +NO_HISTORY = frozenset({errno.ENOENT, errno.ELOOP, errno.ENOTDIR}) + + +def moment(value: object) -> float | None: + if isinstance(value, bool) or not isinstance(value, int | float) or not 0 <= value < 1e11: + return None + return float(value) + + +def opened(council: Path, tag: str) -> tuple[int | None, bool]: + """Returns: + (descriptor, gone): the descriptor of holds/.lock, opened as + queue.py's open_below walks to it, and False; else None and True + where queue.py's held() finds no lock (a missing directory or file) + or open_below refuses one (a link or a non-directory on the way), + None and False where the open fails otherwise. + """ + try: + folder = os.open(council / HOLDS, DIRECTORY) + except FileNotFoundError: + return None, True + except OSError as error: + return None, error.errno in REFUSED + try: + return os.open(f"{tag}.lock", READ_FLAGS, dir_fd=folder), False + except FileNotFoundError: + return None, True + except OSError as error: + return None, error.errno in REFUSED + finally: + os.close(folder) + + +def holder_gone(council: Path, tag: object) -> bool: + """Args: + council: the council directory. + tag: the lock tag of an entry that has one; queue.py's living() + decides by held(tag) alone then. + + Returns: + not held(tag) for every lock queue.py's held() decides: True for a + tag queue.py would not write, a missing lock and a lock a + non-blocking shared flock gets, False for a held one. A hard-linked + lock gets the same probe. A link or a non-regular file, which no + hold can create, is True; a lock that cannot be opened or flocked + otherwise (EACCES) is False. + """ + if not isinstance(tag, str) or not HOLD_TAG.fullmatch(tag): + return True + fd, gone = opened(council, tag) + if fd is None: + return gone + try: + if not stat.S_ISREG(os.fstat(fd).st_mode): + return True + fcntl.flock(fd, fcntl.LOCK_SH | fcntl.LOCK_NB) + except OSError: + return False + finally: + os.close(fd) + return True + + +def read_history(path: Path) -> list[dict] | None: + """Args: + path: a queue-history.jsonl; a missing file, a link or another + non-regular file reads as no rows. + + Returns: + The JSON object rows of the whole file, in file order; an undecodable + byte or a bad line costs that line only. None when the read fails + otherwise (EACCES, EIO, EMFILE). + """ + try: + fd = os.open(path, READ_FLAGS) + except OSError as error: + return [] if error.errno in NO_HISTORY else None + try: + if not stat.S_ISREG(os.fstat(fd).st_mode): + return [] + with open(fd, "rb", closefd=False) as handle: + data = handle.read() + except OSError: + return None + finally: + os.close(fd) + rows = [] + for line in data.decode("utf-8", errors="replace").splitlines(): + try: + row = json.loads(line) + except (ValueError, RecursionError): + continue + if isinstance(row, dict): + rows.append(row) + return rows + + +def history_jobs(rows: list[dict]) -> dict[str, list[dict]]: + """Returns: + The history rows with a start by entry id, the row id without its + last dash and digits. + """ + jobs: dict[str, list[dict]] = {} + for row in rows: + entry_id, _, suffix = str(row.get("id", "")).rpartition("-") + if entry_id and JOB_SUFFIX.fullmatch(suffix) and moment(row.get("start")) is not None: + jobs.setdefault(entry_id, []).append(row) + return jobs + + +def exit_of(row: dict) -> int | None: + code = row.get("exit") + return code if isinstance(code, int) and not isinstance(code, bool) else None + + +def outcome(entry: dict, created: float, gone: bool, jobs: dict[str, list[dict]]) -> tuple[bool, dict | None]: + """Args: + entry: a queue.json entry. + created: its enqueue time. + gone: holder_gone() of its lock, False without a lock key. + jobs: history_jobs(). + + Returns: + (kept, outcome): an active entry whose job has a history row started + after the enqueue is kept with {"state": "ended", "exit", "end"} of + the newest such row by end; else an active one whose holder is gone + is kept with {"state": "gone", "exit": None, "end": None}; else any + other entry whose holder is gone is not kept; else the entry is kept + live with outcome None. + """ + active = entry.get("status") == "active" + ident = entry.get("id") + rows = [row for row in jobs.get(ident, []) if (moment(row.get("start")) or 0.0) >= created] if active and isinstance(ident, str) else [] + if rows: + row = max(rows, key=lambda found: moment(found.get("end")) or 0.0) + return True, {"state": "ended", "exit": exit_of(row), "end": moment(row.get("end"))} + if gone: + return active, {"state": "gone", "exit": None, "end": None} if active else None + return True, None + + +def settled(council: Path, found: list[tuple[dict, float]], history: Callable[[], list[dict]]) -> list[tuple[dict, float, dict | None]]: + """Args: + council: the council directory. + found: the queue.json entries with their enqueue time. + history: returns the queue-history.jsonl rows, None for an + unreadable file; called once, after every hold lock is probed, + and only while an entry is active. + + Returns: + Each kept entry with its enqueue time and outcome, as outcome() + reads it, in the order of found; an entry without a lock key, which + queue.py's living() judges by pids of other PID namespaces, is never + gone and ends only by its history row. + """ + gone = ["lock" in entry and holder_gone(council, entry["lock"]) for entry, _ in found] + jobs = history_jobs(history() or []) if any(entry.get("status") == "active" for entry, _ in found) else {} + kept = [] + for (entry, created), dead in zip(found, gone, strict=True): + keep, result = outcome(entry, created, dead, jobs) + if keep: + kept.append((entry, created, result)) + return kept diff --git a/orchestrator/client/resources.py b/orchestrator/client/resources.py index 25106a5..a8abc74 100755 --- a/orchestrator/client/resources.py +++ b/orchestrator/client/resources.py @@ -44,6 +44,7 @@ from __future__ import annotations import argparse import base64 import datetime +import functools import hashlib import json import os @@ -55,6 +56,8 @@ import time from collections.abc import Callable from pathlib import Path +import queue_live + INTERVAL = 60 BUCKET = 900 HOUR = 3600 @@ -226,9 +229,19 @@ def gpus(sys_root: Path, run: Callable[[list[str]], str | None]) -> list[dict]: def queue(council: Path) -> dict: + """Returns: + {waiting, active}: the queue.json entries with an enqueue time that + queue_live.settled() keeps without an outcome, as the hub counts them. + """ data = read_json(council / "queue.json") entries = data.get("entries") if isinstance(data, dict) else None - statuses = [entry.get("status") for entry in entries if isinstance(entry, dict)] if isinstance(entries, list) else [] + found = [] + for entry in entries if isinstance(entries, list) else []: + created = queue_live.moment(entry.get("enqueued")) if isinstance(entry, dict) else None + if created is not None: + found.append((entry, created)) + history = functools.partial(queue_live.read_history, council / queue_live.HISTORY) + statuses = [entry.get("status") for entry, _, result in queue_live.settled(council, found, history) if result is None] return {"waiting": statuses.count("waiting"), "active": statuses.count("active")} diff --git a/orchestrator/kvb_orchestrator/crons.py b/orchestrator/kvb_orchestrator/crons.py index 04240e5..b4f3ab0 100644 --- a/orchestrator/kvb_orchestrator/crons.py +++ b/orchestrator/kvb_orchestrator/crons.py @@ -1,15 +1,23 @@ from __future__ import annotations +import os +import threading +from collections.abc import Callable +from dataclasses import dataclass, field from pathlib import Path from .council import agent from .events import event from .names import Directory from .records import epoch, read_json, shown +from .signed import client_module + +queue_live = client_module("queue_live", {}) CRON_ACTIVE = 900 TIMER_KEYS = ("OnCalendar", "OnActiveSec", "OnBootSec", "OnStartupSec", "OnUnitActiveSec", "OnUnitInactiveSec") CRON_LABELS = {"cron": "Cron", "wakeup": "Wakeup", "background": "Background", "timer": "Timer", "queue": "Queue"} +OUTCOME = "hub_outcome" def cron_state(cron: dict, beats: dict[str, float], now: float) -> dict: @@ -41,10 +49,52 @@ def timer_rows(folder: Path) -> list[dict]: return rows -def queue_entries(council: Path) -> list[tuple[dict, float]]: - """Returns: - Each kvb-org-queue entry of queue.json with its enqueue time; an - entry without one is no entry. +@dataclass +class History: + path: Path + lock: threading.Lock = field(default_factory=threading.Lock) + cached: list[dict] = field(default_factory=list) + stamp: tuple[int, int, int] | None = None + + def rows(self) -> list[dict]: + """Returns: + queue_live.read_history() of the whole file, parsed again only + when its inode, size or mtime changed; after a failed read the + rows read before, and the next call reads again. The list is + shared, read only. + """ + with self.lock: + try: + info = os.lstat(self.path) + except FileNotFoundError: + self.cached, self.stamp = [], None + return self.cached + except OSError: + self.stamp = None + return self.cached + stamp = (info.st_ino, info.st_size, info.st_mtime_ns) + if stamp != self.stamp: + found = queue_live.read_history(self.path) + if found is None: + self.stamp = None + else: + self.cached, self.stamp = found, stamp + return self.cached + + +def outcome_of(entry: dict) -> dict | None: + found = entry.get(OUTCOME) + return found if isinstance(found, dict) else None + + +def queue_entries(council: Path, history: Callable[[], list[dict]]) -> list[tuple[dict, float]]: + """Args: + council: the council directory. + history: returns the queue-history.jsonl rows, History.rows. + + Returns: + Each entry of queue.json with an enqueue time that queue_live.settled() + keeps, with that time, as a copy whose OUTCOME is its outcome. """ data = read_json(council / "queue.json") entries = data.get("entries") if isinstance(data, dict) else None @@ -53,17 +103,26 @@ def queue_entries(council: Path) -> list[tuple[dict, float]]: created = epoch(entry.get("enqueued")) if isinstance(entry, dict) else None if created is not None: found.append((entry, created)) - return found + return [({**entry, OUTCOME: result}, created) for entry, created, result in queue_live.settled(council, found, history)] -def queue_rows(council: Path) -> list[dict]: +def queue_state(entry: dict) -> str: + found = outcome_of(entry) + if found is None: + return "running" if entry.get("status") == "active" else "waiting" + if found.get("state") == "gone": + return "holder gone without an exit code" + code = found.get("exit") + return "ended" if code is None else f"exit {code}" + + +def queue_rows(council: Path, history: Callable[[], list[dict]]) -> list[dict]: rows = [] - for entry, created in queue_entries(council): - running = "running" if entry.get("status") == "active" else "waiting" + for entry, created in queue_entries(council, history): rows.append({ "key": f"queue:{shown(entry.get('id'))}", "type": "queue", "session": shown(entry.get("session")), "sub": "", "created": created, - "deleted": None, "until": None, "job": shown(entry.get("id")), "expression": f"{shown(entry.get('kind'))} · {running}", - "prompt": shown(entry.get("cmd")), "reason": shown(entry.get("task")), "active": True, "agent": agent(entry.get("name")), + "deleted": None, "until": None, "job": shown(entry.get("id")), "expression": f"{shown(entry.get('kind'))} · {queue_state(entry)}", + "prompt": shown(entry.get("cmd")), "reason": shown(entry.get("task")), "active": outcome_of(entry) is None, "agent": agent(entry.get("name")), }) return rows diff --git a/orchestrator/kvb_orchestrator/hub.py b/orchestrator/kvb_orchestrator/hub.py index cfb1126..a9a2faf 100644 --- a/orchestrator/kvb_orchestrator/hub.py +++ b/orchestrator/kvb_orchestrator/hub.py @@ -13,6 +13,7 @@ from .asking import Asking from .batches import Pending from .choosing import Choosing from .cleanup import Cleaning, Spool +from .crons import History, queue_live from .denying import Denying from .engines import Engines, Speech from .events import Log, Watch @@ -103,6 +104,7 @@ class Hub(Mcp, Reviewing, Asking, Messaging, Choosing, Approving, Denying, Reque touched: Touched voice_notes: VoiceNotes replay: Path + history: History def hosts(self) -> tuple[str, str]: return f"127.0.0.1:{self.public_port}", f"localhost:{self.public_port}" @@ -142,6 +144,7 @@ def open_hub(council: Path, pages: Path, public_port: int, state: Path, host_sta timers=places.timers, kept=places.kept, council=council, + history=History(council / queue_live.HISTORY), pages=pages, public_port=public_port, reviews=reviews, diff --git a/orchestrator/kvb_orchestrator/records.py b/orchestrator/kvb_orchestrator/records.py index de0acdc..0fa6dd6 100644 --- a/orchestrator/kvb_orchestrator/records.py +++ b/orchestrator/kvb_orchestrator/records.py @@ -34,10 +34,14 @@ def read_regular(path: Path, limit: int | None) -> str: Returns: The file's text, UTF-8 decoded. """ - with open(os.open(path, READ_FLAGS), "rb") as handle: - if not stat.S_ISREG(os.fstat(handle.fileno()).st_mode): + fd = os.open(path, READ_FLAGS) + try: + if not stat.S_ISREG(os.fstat(fd).st_mode): raise OSError(errno.EINVAL, "not a regular file", str(path)) - data = handle.read() if limit is None else handle.read(limit + 1) + with open(fd, "rb", closefd=False) as handle: + data = handle.read() if limit is None else handle.read(limit + 1) + finally: + os.close(fd) if limit is not None and len(data) > limit: raise OSError(errno.EFBIG, f"larger than {limit} bytes", str(path)) return data.decode("utf-8") diff --git a/orchestrator/kvb_orchestrator/sources.py b/orchestrator/kvb_orchestrator/sources.py index 27980e2..348fa6a 100644 --- a/orchestrator/kvb_orchestrator/sources.py +++ b/orchestrator/kvb_orchestrator/sources.py @@ -9,7 +9,7 @@ from .errors import ToolError from .fields import single from .lifetimes import edge_times, lifetime, session_gone, subagent_rows from .names import PSEUDO, SESSION_ID, Directory, title_sightings -from .records import epoch, iso_epoch, read_jsonl +from .records import epoch, iso_epoch from .register import REGISTER, read_register from .tree import assign @@ -48,10 +48,10 @@ class Sources: tasks = read_register(self.council / REGISTER) reviews = self.reviews.listed() scan = assign(self.tree.get(), tasks, read_claims(self.council), reviews) - history = read_jsonl(self.council / "queue-history.jsonl") + history = self.history.rows() return {**enrich(scan, tasks, reviews, self.touched, history, self.jobs.listed()), "cleanups": self.cleanup_results(), "gates": self.gate_view(scan)} def schedules(self, claims: list[dict], now: float) -> list[dict]: beats = {claim["session"]: claim["beat"] for claim in claims if claim["beat"] is not None} rows = [cron_state(cron, beats, now) for cron in self.transcripts.schedules()] - return rows + timer_rows(self.timers) + queue_rows(self.council) + return rows + timer_rows(self.timers) + queue_rows(self.council, self.history.rows) diff --git a/orchestrator/kvb_orchestrator/tasks.py b/orchestrator/kvb_orchestrator/tasks.py index 35fe0bd..e6b47d0 100644 --- a/orchestrator/kvb_orchestrator/tasks.py +++ b/orchestrator/kvb_orchestrator/tasks.py @@ -4,6 +4,7 @@ import itertools import statistics from .council import HUMAN, agent +from .crons import outcome_of from .records import epoch, iso_epoch, shown from .register import ENDED, OPERATOR, parties, register_row, status_of from .signed import signing @@ -152,12 +153,21 @@ def task_row(task: str, row: dict, trees: list[dict]) -> dict: } +def queue_status(entry: dict, found: dict | None) -> str: + if found is not None: + return "done" if found.get("state") == "ended" and found.get("exit") == 0 else "stopped" + return "running" if entry.get("status") == "active" else "awaiting" + + def queue_row(entry: dict, created: float, trees: list[dict]) -> dict: text = {name: shown(entry.get(name))[:TEXT_SHOWN] for name in ("id", "cmd", "kind", "task", "workdir")} + found = outcome_of(entry) + ended = found is not None + granted = entry.get("status") == "active" or ended return { "task": f"{QUEUE}{text['id']}", "title": text["cmd"], - "status": "running" if entry.get("status") == "active" else "awaiting", + "status": queue_status(entry, found), "worker": agent(entry.get("name"))[:TEXT_SHOWN] or "-", "responsible": "-", "accountable": "-", @@ -167,7 +177,7 @@ def queue_row(entry: dict, created: float, trees: list[dict]) -> dict: "met": 0, "total": 0, "kind": "queue", - "ended": False, + "ended": ended, "editable": False, "movable": False, "prio": None, @@ -175,13 +185,13 @@ def queue_row(entry: dict, created: float, trees: list[dict]) -> dict: "wsjf": None, "value": None, "points": None, - "start": (epoch(entry.get("granted")) or created) if entry.get("status") == "active" else None, - "end": None, + "start": (epoch(entry.get("granted")) or created) if granted else None, + "end": epoch(found.get("end")) if found is not None else None, "updated": None, "after": [], "base": "-", "roles": {}, - "queue": {"kind": text["kind"], "parent": text["task"], "enqueued": created}, + "queue": {"kind": text["kind"], "parent": text["task"], "enqueued": created, "exit": found.get("exit") if found is not None else None}, **located(entry, trees, False), } @@ -253,7 +263,7 @@ def queue_chain(queue: list[tuple[dict, float]], tasks: dict[str, dict], agents: holder = order[name] if isinstance(name, str) and name in order else len(order) return (not entry.get("hotfix"), place - hours, holder, seq if isinstance(seq, int) and not isinstance(seq, bool) else 0) - active = sorted((pair for pair in queue if pair[0].get("status") == "active"), key=lambda pair: epoch(pair[0].get("granted")) or pair[1]) + active = sorted((pair for pair in queue if pair[0].get("status") == "active" and outcome_of(pair[0]) is None), key=lambda pair: epoch(pair[0].get("granted")) or pair[1]) waiting = sorted((pair for pair in queue if pair[0].get("status") == "waiting"), key=waiting_key) return [(shown(entry.get("id"))[:TEXT_SHOWN], entry.get("status") == "waiting") for entry, _ in [*active, *waiting]] diff --git a/orchestrator/kvb_orchestrator/view.py b/orchestrator/kvb_orchestrator/view.py index 65886ce..02ec9b8 100644 --- a/orchestrator/kvb_orchestrator/view.py +++ b/orchestrator/kvb_orchestrator/view.py @@ -85,7 +85,7 @@ class View: def state(self, begin: float, end: float, now: float, steps: int) -> dict: tasks = read_register(self.council / REGISTER) news = read_jsonl(self.council / NEWS) - history = read_jsonl(self.council / "queue-history.jsonl") + history = self.history.rows() claims = read_claims(self.council) names = sessions(claims) directory = self.directory(claims, news, now) @@ -105,7 +105,7 @@ class View: people = {node["id"]: {"name": node["label"], "now": directory.now(node["id"]) or node["label"], "names": node["names"]} for node in drawn["nodes"] if node["kind"] != "subagent"} settings = self.settings.get() inbox = self.inbox(now, tasks, directory) - queue = queue_entries(self.council) + queue = queue_entries(self.council, self.history.rows) order = stored_order(self.council / AGENT_ORDER) rows = task_rows(tasks, queue, self.tree.get()) chain = queue_chain(queue, tasks, order, now) diff --git a/orchestrator/templates/page-tasks.js b/orchestrator/templates/page-tasks.js index 9f85940..d86994a 100644 --- a/orchestrator/templates/page-tasks.js +++ b/orchestrator/templates/page-tasks.js @@ -47,7 +47,8 @@ function taskPlace(row) { if (row.kind === "queue") { - return h("span", {}, `${row.queue.kind} for ${row.queue.parent} · queued `, when(row.queue.enqueued)); + const outcome = !row.ended ? null : row.queue.exit !== null ? ` · exit ${row.queue.exit}` : " · ended without an exit code"; + return h("span", {}, `${row.queue.kind} for ${row.queue.parent} · queued `, when(row.queue.enqueued), outcome); } const place = row.worktree || (row.workdir !== "-" ? row.workdir : ""); return place ? h("code", {title: row.repo || place}, place) : null; diff --git a/orchestrator/templates/plan.js b/orchestrator/templates/plan.js index 52b3873..ccb0f7f 100644 --- a/orchestrator/templates/plan.js +++ b/orchestrator/templates/plan.js @@ -23,12 +23,12 @@ const Plan = (() => { } /** - * @param rows - task rows with status, start, end (epoch seconds or null) and points (median complexity or null). - * @returns {duration, perPoint}: the median seconds a done task took from start to end, and the median seconds - * per point of the done tasks with points; null where no done task gives one. + * @param rows - task rows with kind, status, start, end (epoch seconds or null) and points (median complexity or null). + * @returns {duration, perPoint}: the median seconds a done task, never a queue job, took from start to end, and the + * median seconds per point of the done tasks with points; null where no done task gives one. */ function medians(rows) { - const done = rows.filter((row) => row.status === "done" && row.start !== null && row.end !== null && row.end >= row.start); + const done = rows.filter((row) => row.kind !== "queue" && row.status === "done" && row.start !== null && row.end !== null && row.end >= row.start); return { duration: median(done.map((row) => row.end - row.start)), perPoint: median(done.filter((row) => row.points).map((row) => (row.end - row.start) / row.points)), diff --git a/tests/test_hub_operator_tasks.py b/tests/test_hub_operator_tasks.py index 2c1da2b..0ecfb78 100644 --- a/tests/test_hub_operator_tasks.py +++ b/tests/test_hub_operator_tasks.py @@ -241,6 +241,10 @@ def rules_dir(root: Path) -> Path: return root / "rules" +def client_calls(tree: ast.AST) -> list[ast.Call]: + return [node for node in ast.walk(tree) if isinstance(node, ast.Call) and "client_module" in (getattr(node.func, "id", ""), getattr(node.func, "attr", ""))] + + class TestDiffsModule(unittest.TestCase): @classmethod def setUpClass(cls): @@ -825,12 +829,18 @@ class TestSharedLimits(unittest.TestCase): self.assertLessEqual(len(json.dumps(patch, ensure_ascii=False).encode("utf-8")), hub_limits.DIFF_LIMIT) self.assertGreater(len(permission_hook.encoded({"diff": [{"patch": patch}]})), hub_limits.DIFF_BODY_LIMIT) - def test_the_hub_image_copies_every_client_module_signed_loads(self): - tree = ast.parse((HUB / "kvb_orchestrator" / "signed.py").read_text(encoding="utf-8")) - loaded = {f"{node.args[0].value}.py" for node in ast.walk(tree) if isinstance(node, ast.Call) and getattr(node.func, "id", "") == "client_module"} + def test_client_module_calls_are_found_bare_and_as_attributes(self): + found = client_calls(ast.parse('signed.client_module("a", {})\nclient_module("b", {})\nother("c")\nx.other("d")')) + self.assertEqual([node.args[0].value for node in found], ["a", "b"]) + + def test_the_hub_image_copies_every_client_module_the_hub_loads(self): + trees = [ast.parse(path.read_text(encoding="utf-8")) for path in sorted((HUB / "kvb_orchestrator").glob("*.py"))] + calls = [node for tree in trees for node in client_calls(tree)] + self.assertTrue(all(node.args and isinstance(node.args[0], ast.Constant) for node in calls)) + loaded = {f"{node.args[0].value}.py" for node in calls} copies = [line.split()[1:] for line in (HUB / "Dockerfile").read_text(encoding="utf-8").splitlines() if line.startswith("COPY ")] copied = {Path(source).name for words in copies if words[-1] == "/app/client/" for source in words[:-1]} - self.assertIn("hub_limits.py", loaded) + self.assertLessEqual({"hub_limits.py", "queue_live.py"}, loaded) self.assertLessEqual(loaded, copied) diff --git a/tests/test_hub_plan.py b/tests/test_hub_plan.py index 9e96219..cb232ae 100644 --- a/tests/test_hub_plan.py +++ b/tests/test_hub_plan.py @@ -74,6 +74,7 @@ console.log(JSON.stringify({ steps: {whole: stepsOf(70 * H, now), window: stepsOf(90 * H, 99 * H), burnt: P.steps(burnt, burnPoints, 0)}, etas: {none: eta("none"), points: eta("points"), loop: [...P.etas(loop, looped, {now, estimation: "none"}).keys()].sort()}, medians: P.medians(rows), + queueMedians: P.medians([...rows, row("queue/done", {kind: "queue", status: "done", ended: true, start: 0, end: 50 * H, points: 1})]), median: [P.median([]), P.median([3, 1, 2]), P.median([4, 1, 3, 2])], bars: {noneAlways: lay("none", "always"), pointsAlways: lay("points", "always"), pointsOnly: lay("points", "points"), noneOnly: lay("none", "points"), real: lay("points", "real")}, loop: P.bars(loop, looped, {now, estimation: "none", forecast: "always"}).map((bar) => bar.task), @@ -157,6 +158,9 @@ class TestPlanModule(unittest.TestCase): self.assertEqual(self.data["medians"], {"duration": 2 * HOUR, "perPoint": 1.5 * HOUR}) self.assertEqual(self.data["median"], [None, 2, 2.5]) + def test_a_done_queue_job_never_moves_the_medians(self): + self.assertEqual(self.data["queueMedians"], self.data["medians"]) + def test_without_estimate_a_forecast_is_as_long_as_the_median_duration_behind_its_predecessors(self): self.assertEqual(self.data["bars"]["noneAlways"], [ ["work/a", 80, 82, False, False], diff --git a/tests/test_hub_tasks.py b/tests/test_hub_tasks.py index 1807a4d..a6ed439 100644 --- a/tests/test_hub_tasks.py +++ b/tests/test_hub_tasks.py @@ -1,5 +1,7 @@ from __future__ import annotations +import fcntl +import functools import importlib.util import json import os @@ -12,13 +14,14 @@ from unittest import mock from hub_case import HubCase from hub_support import REPO_ROOT, iso, key, runner, signing, verifier -from kvb_orchestrator import settings, skill_scripts, tasks, tree +from kvb_orchestrator import crons, records, settings, skill_scripts, tasks, tree BROKEN = {"pull": [], "push": [], "source": "invalid"} FLOW = {"pull": ["origin/main", "store-i18n"], "push": ["main", "production"], "source": "file"} PRIO = ["set", "work/feature", "--prio", "3"] ESTIMATE = ["estimate", "work/feature", "Operator", "--value", "8", "--complexity", "3"] QUEUE_SCRIPT = REPO_ROOT / "skills" / "kvb-org-queue" / "scripts" / "queue.py" +PROOF = "/home/me/worktrees/github.com/infinito-nexus/core/proof/karakalpak-archive-role" def change(argv: list, task: str = "work/feature") -> dict: @@ -136,11 +139,198 @@ class TestHubTasks(HubCase): self.assertEqual((command["kind"], command["title"], command["editable"], command["open_work"]), ("command", "git push", False, ["task"])) gate, deploy = rows["queue/gate-7"], rows["queue/deploy-8"] self.assertEqual((gate["kind"], gate["status"], gate["title"], gate["worker"], gate["editable"], gate["flow"]), ("queue", "awaiting", "make test", "Stage", False, FLOW)) - self.assertEqual(gate["queue"], {"kind": "gate", "parent": "work/feature", "enqueued": now - 60}) + self.assertEqual(gate["queue"], {"kind": "gate", "parent": "work/feature", "enqueued": now - 60, "exit": None}) self.assertEqual((deploy["status"], deploy["flow"], deploy["queue"]["parent"]), ("running", None, "work/pinned")) done = rows["work/done"] self.assertEqual((done["ended"], done["editable"], done["open_work"], done["worktree"]), (True, False, [], "")) + def held_lock(self, tag: str, mode: int = fcntl.LOCK_EX) -> None: + holds = self.council / "holds" + holds.mkdir(parents=True, exist_ok=True) + handle = (holds / f"{tag}.lock").open("w", encoding="utf-8") + self.addCleanup(handle.close) + fcntl.flock(handle, mode) + + def queue_job(self, now: float, ident: str, status: str, task: str, lock: str, age: float) -> dict: + granted = now - age + 10 if status == "active" else None + return {"id": ident, "kind": ident.rpartition("-")[0], "task": task, "workdir": PROOF, "session": "s1", "name": "Production", "lock": lock, "pid": 7, + "status": status, "enqueued": now - age, "granted": granted, "cmd": "make quality-high"} + + def test_a_queue_job_with_a_history_row_or_a_gone_holder_is_no_longer_running(self): + now = time.time() + for tag in ("7-1410", "7-1451", "7-5", "7-1387", "7-1500", "7-1501", "7-1502"): + self.held_lock(tag) + self.held_lock("7-1481", fcntl.LOCK_SH) + (self.council / "holds" / "7-1465.lock").write_text("", encoding="utf-8") + (self.council / "holds" / "7-1480.lock").symlink_to(self.council / "holds" / "7-1451.lock") + job = functools.partial(self.queue_job, now) + queue = {"seq": 1502, "entries": [ + job("quality-high-1410", "active", "karakalpak-archive", "7-1410", 1200), + job("test-nostack-1464", "active", "karakalpak-archive", "7-1464", 300), + job("test-nostack-1465", "active", "iam/final-066-backchannel", "7-1465", 200), + job("test-nostack-1466", "active", "iam/final-066-backchannel", "7-1466", 100), + job("quality-high-1451", "active", "work/gitea-oidc-auto-link", "7-1451", 900), + job("test-nostack-1480", "active", "work/symlinked", "7-1480", 80), + job("test-nostack-1481", "active", "iam/final-066-backchannel", "7-1481", 70), + job("gate-5", "active", "work/reused", "7-5", 60), + job("test-1387", "waiting", "work/security-page-optimize", "7-1387", 50), + job("test-1388", "waiting", "work/security-page-optimize", "7-1388", 40), + job("test-nostack-1471", "waiting", "work/orchestrator-firewall", "../holds/7-1471", 30), + job("test-1500", "ended", "work/foreign", "7-1500", 25), + job("test-1501", "gone", "work/foreign", "7-1501", 24), + {**job("test-1502", "active", "work/foreign", "7-1502", 20), "hub_outcome": {"state": "ended", "exit": 9}, "exit": 9, "end": now}, + ]} + history = [ + {"end": now - 600, "exit": 1, "id": "quality-high-1410-1794513000000000000", "kind": "quality-high", "start": now - 1180, "workdir": PROOF}, + {"concurrent": 2, "cpu_before": 53.9, "cpu_max": 100.0, "disk_gb": 1.91, "end": now - 5, "exit": 2, "free_before_gb": 113.12, + "id": "quality-high-1410-1794514233361361126", "kind": "quality-high", "mem_before": 43.2, "mem_max": 84.3, "min_free_gb": 111.21, + "minutes": 18.13, "signature": "infinito-nexus-core: make quality-high", "start": now - 1185, "workdir": PROOF}, + {"end": now - 20, "exit": 0, "id": "test-nostack-1464-1794514708907109250", "kind": "test-nostack", "start": now - 290, "workdir": PROOF}, + {"end": now - 3, "exit": 0, "id": "test-nostack-1466-retry", "kind": "test-nostack", "start": now - 90, "workdir": PROOF}, + {"end": now - 7200, "exit": 1, "id": "gate-5-1794500000000000000", "kind": "gate", "start": now - 7300, "workdir": PROOF}, + {"end": now - 1, "exit": 0, "id": "gate-5x-1794500000000000000", "kind": "gate", "start": now - 30, "workdir": PROOF}, + ] + self.write_council({"queue.json": queue, "queue-history.jsonl": self.jsonl(history)}) + state = self.snapshot() + rows = {row["task"]: row for row in state["tasks"] if row["kind"] == "queue"} + + def seen(ident: str) -> tuple: + row = rows[f"queue/{ident}"] + return row["status"], row["ended"], row["end"], row["queue"].get("exit") + + self.assertEqual(seen("quality-high-1410"), ("stopped", True, now - 5, 2)) + self.assertEqual(rows["queue/quality-high-1410"]["start"], now - 1190) + self.assertEqual(seen("test-nostack-1464"), ("done", True, now - 20, 0)) + self.assertEqual(seen("test-nostack-1465"), ("stopped", True, None, None)) + self.assertEqual(seen("test-nostack-1466"), ("stopped", True, None, None)) + self.assertEqual(seen("test-nostack-1481"), ("stopped", True, None, None)) + self.assertEqual(seen("test-nostack-1480"), ("stopped", True, None, None)) + live = ("quality-high-1451", "gate-5", "test-1502") + shown = ("quality-high-1410", "test-nostack-1464", "test-nostack-1465", "test-nostack-1466", "test-nostack-1480", "test-nostack-1481", "test-1387", "test-1500", "test-1501", *live) + self.assertEqual(sorted(rows), sorted(f"queue/{ident}" for ident in shown)) + self.assertEqual([seen(ident) for ident in live], [("running", False, None, None)] * 3) + self.assertEqual([seen(ident) for ident in ("test-1387", "test-1500", "test-1501")], [("awaiting", False, None, None)] * 3) + self.assertEqual(state["queue_line"], ["work/gitea-oidc-auto-link", "work/reused", "work/foreign", "work/security-page-optimize"]) + schedule = {row["job"]: (row["expression"], row["active"]) for row in crons.queue_rows(self.council, self.httpd.hub.history.rows)} + self.assertEqual(schedule["quality-high-1410"], ("quality-high · exit 2", False)) + self.assertEqual(schedule["test-nostack-1465"], ("test-nostack · holder gone without an exit code", False)) + self.assertEqual(schedule["quality-high-1451"], ("quality-high · running", True)) + self.assertEqual(schedule["test-1500"], ("test · waiting", True)) + + def test_the_history_is_read_after_the_hold_locks_and_parsed_once_per_change(self): + now = time.time() + self.held_lock("7-1") + queue = {"seq": 2, "entries": [self.queue_job(now, "test-1", "active", "work/a", "7-1", 100), self.queue_job(now, "test-2", "active", "work/b", "7-2", 50)]} + self.write_council({"queue.json": queue, "queue-history.jsonl": self.jsonl([{"id": "other-9-1", "start": now - 500, "end": now - 400, "exit": 0}])}) + probe, read = crons.queue_live.holder_gone, crons.queue_live.read_history + reads = [] + + def leaving(council: Path, tag: object) -> bool: + if tag == "7-2": + with (self.council / "queue-history.jsonl").open("a", encoding="utf-8") as handle: + handle.write(json.dumps({"id": "test-2-1794514233361361126", "start": now - 40, "end": now - 1, "exit": 3}) + "\n") + return probe(council, tag) + + def counted(path: Path) -> list[dict]: + reads.append(path.name) + return read(path) + + with mock.patch.object(crons.queue_live, "holder_gone", leaving), mock.patch.object(crons.queue_live, "read_history", counted): + rows = {row["task"]: row for row in self.snapshot()["tasks"] if row["kind"] == "queue"} + self.assertEqual((rows["queue/test-2"]["status"], rows["queue/test-2"]["queue"]["exit"]), ("stopped", 3)) + self.assertEqual(rows["queue/test-1"]["status"], "running") + self.assertEqual(len(reads), 2) + self.httpd.hub.history.rows() + self.assertEqual(len(reads), 2) + + def test_the_whole_history_is_read_and_a_bad_byte_costs_its_line_only(self): + path = self.council / "queue-history.jsonl" + self.council.mkdir(parents=True, exist_ok=True) + rows = b"".join(json.dumps({"id": f"gate-{n}-1", "start": n, "pad": "x" * 1000}).encode() + b"\n" for n in range(6000)) + path.write_bytes(b'{"id": "bad-\xff-1"}\n' + rows + b"[[[\n" + b'{"id": "last-1"}') + found = crons.queue_live.read_history(path) + self.assertGreater(path.stat().st_size, 1 << 22) + self.assertEqual((len(found), found[1]["id"], found[-2]["id"], found[-1]["id"]), (6002, "gate-0-1", "gate-5999-1", "last-1")) + + def test_reading_a_directory_as_history_or_json_leaks_no_descriptor(self): + folder = self.council / "queue-history.jsonl" + folder.mkdir(parents=True) + before = len(os.listdir("/proc/self/fd")) + for _ in range(50): + self.assertEqual(crons.queue_live.read_history(folder), []) + self.assertIsNone(records.read_json(folder)) + self.assertEqual(records.read_jsonl(folder), []) + self.assertLessEqual(len(os.listdir("/proc/self/fd")), before + 2) + + def test_the_history_cache_reparses_on_a_new_inode_mtime_or_size_alone(self): + path = self.council / "queue-history.jsonl" + self.council.mkdir(parents=True, exist_ok=True) + history = crons.History(path) + + def write(text: str, ns: int) -> None: + path.write_text(text, encoding="utf-8") + os.utime(path, ns=(ns, ns)) + + write('{"id": "a-1"}\n', 10**18) + self.assertEqual([row["id"] for row in history.rows()], ["a-1"]) + replacement = self.council / "replacement" + replacement.write_text('{"id": "b-1"}\n', encoding="utf-8") + os.utime(replacement, ns=(10**18, 10**18)) + os.replace(replacement, path) + self.assertEqual([row["id"] for row in history.rows()], ["b-1"]) + write('{"id": "c-1"}\n', 10**18 + 1) + self.assertEqual([row["id"] for row in history.rows()], ["c-1"]) + write('{"id": "c-1"}\n{"id": "d-1"}\n', 10**18 + 1) + self.assertEqual([row["id"] for row in history.rows()], ["c-1", "d-1"]) + + def test_a_hard_link_to_a_live_hold_lock_keeps_the_hold_alive(self): + self.held_lock("7-20") + holds = self.council / "holds" + os.link(holds / "7-20.lock", holds / "7-21.lock") + self.assertEqual((holds / "7-20.lock").stat().st_nlink, 2) + self.assertEqual([crons.queue_live.holder_gone(self.council, tag) for tag in ("7-20", "7-21")], [False, False]) + + @unittest.skipIf(os.geteuid() == 0, "root reads a file of mode 0") + def test_a_failed_history_read_keeps_the_rows_before_and_retries_next_time(self): + path = self.council / "queue-history.jsonl" + self.write_council({"queue-history.jsonl": '{"id": "a-1"}\n'}) + history = crons.History(path) + self.assertEqual([row["id"] for row in history.rows()], ["a-1"]) + with path.open("a", encoding="utf-8") as handle: + handle.write('{"id": "b-1"}\n') + path.chmod(0) + self.addCleanup(path.chmod, 0o600) + self.assertEqual([row["id"] for row in history.rows()], ["a-1"]) + self.assertIsNone(crons.queue_live.read_history(path)) + path.chmod(0o600) + self.assertEqual([row["id"] for row in history.rows()], ["a-1", "b-1"]) + + def test_the_hub_judges_every_hold_lock_as_queue_py_held_does(self): + queue_py = load_queue() + holds = self.council / "holds" + self.held_lock("7-1") + (holds / "7-2.lock").write_text("", encoding="utf-8") + (holds / "7-3.lock").symlink_to(holds / "7-1.lock") + (holds / "7-7.lock").write_text("", encoding="utf-8") + os.link(holds / "7-7.lock", holds / "7-6.lock") + decided = {"held": "7-1", "free": "7-2", "missing": "7-404", "malformed": "x-1", "traversal": "../holds/7-1", "none": None} + refused = {"symlink": "7-3", "hard link": "7-6"} + hub = {name: crons.queue_live.holder_gone(self.council, tag) for name, tag in {**decided, **refused}.items()} + with mock.patch.object(queue_py, "HOLDS", holds), mock.patch.object(queue_py.watch, "ROOT", self.council): + for name, tag in refused.items(): + with self.subTest(refused=name), self.assertRaises(OSError): + queue_py.held(tag) + with self.subTest(refused=name), self.assertRaises(OSError), queue_py.nofollow.locked(self.council, holds / f"{tag}.lock", 0o600, None): + pass + verdicts = {name: queue_py.held(tag) for name, tag in decided.items()} + self.assertEqual({name: not hub[name] for name in decided}, verdicts) + self.assertEqual(verdicts, {"held": True, "free": False, "missing": False, "malformed": False, "traversal": False, "none": False}) + self.assertEqual({name: hub[name] for name in refused}, {"symlink": True, "hard link": True}) + if os.geteuid() != 0: + (holds / "7-9.lock").write_text("", encoding="utf-8") + (holds / "7-9.lock").chmod(0) + self.assertFalse(crons.queue_live.holder_gone(self.council, "7-9")) + def test_only_the_newest_ended_tasks_are_listed(self): register = {f"work/old-{n:03d}": {"status": "done", "end": f"2026-11-{1 + n // 24:02d}T{n % 24:02d}:00:00+00:00"} for n in range(30)} self.write_council({"rasci.json": {**register, "work/open": {"status": "queued"}}}) diff --git a/tests/test_resources_sample.py b/tests/test_resources_sample.py index cd6dbfc..01f08af 100644 --- a/tests/test_resources_sample.py +++ b/tests/test_resources_sample.py @@ -1,5 +1,6 @@ from __future__ import annotations +import fcntl import json import os import subprocess @@ -108,10 +109,29 @@ class TestParsing(unittest.TestCase): with tempfile.TemporaryDirectory() as tmp: council = Path(tmp) self.assertEqual(sampler.queue(council), {"waiting": 0, "active": 0}) - entries = [{"status": "waiting"}, {"status": "waiting"}, {"status": "active"}, {"status": "done"}, "junk"] + entries = [{"status": status, "enqueued": NOW} for status in ("waiting", "waiting", "active", "done")] + [{"status": "active"}, "junk"] (council / "queue.json").write_text(json.dumps({"entries": entries}), encoding="utf-8") self.assertEqual(sampler.queue(council), {"waiting": 2, "active": 1}) + def test_the_queue_counts_only_live_entries_as_the_hub_settles_them(self): + with tempfile.TemporaryDirectory() as tmp: + council = Path(tmp) + (council / "holds").mkdir() + handles = [(council / "holds" / f"7-{n}.lock").open("w", encoding="utf-8") for n in (1, 2, 3)] + for handle in handles: + self.addCleanup(handle.close) + fcntl.flock(handle, fcntl.LOCK_EX) + entries = [ + {"id": "test-1", "status": "waiting", "lock": "7-1", "enqueued": NOW}, + {"id": "test-2", "status": "active", "lock": "7-2", "enqueued": NOW}, + {"id": "quality-high-3", "status": "active", "lock": "7-3", "enqueued": NOW}, + {"id": "test-4", "status": "waiting", "lock": "7-4", "enqueued": NOW}, + {"id": "test-5", "status": "active", "lock": "7-5", "enqueued": NOW}, + ] + (council / "queue.json").write_text(json.dumps({"entries": entries}), encoding="utf-8") + (council / "queue-history.jsonl").write_text(json.dumps({"id": "quality-high-3-1794514233361361126", "start": NOW + 10, "end": NOW + 20, "exit": 2}) + "\n", encoding="utf-8") + self.assertEqual(sampler.queue(council), {"waiting": 1, "active": 1}) + class TestCpu(unittest.TestCase): def test_cpu_is_the_busy_share_since_the_ticks_the_run_before_kept_and_the_sampler_never_sleeps(self): @@ -233,7 +253,7 @@ class TestSamplerInScratchHome(unittest.TestCase): (stubs / tool).chmod(0o755) self.sys = host_sys(base / "sys") self.log = base / "calls.jsonl" - (self.council / "queue.json").write_text(json.dumps({"entries": [{"status": "waiting"}, {"status": "active"}, {"status": "active"}]}), encoding="utf-8") + (self.council / "queue.json").write_text(json.dumps({"entries": [{"status": status, "enqueued": NOW} for status in ("waiting", "active", "active")]}), encoding="utf-8") self.env = { **os.environ, "HOME": str(self.home), "PATH": f"{stubs}:/usr/bin:/bin", "STUB_LOG": str(self.log), "STUB_DF": DF, "STUB_NVIDIA_SMI": NVIDIA_SMI.splitlines()[0] + "\n",