diff --git a/orchestrator/kvb_orchestrator/hub.py b/orchestrator/kvb_orchestrator/hub.py index 2eb787c..85e2813 100644 --- a/orchestrator/kvb_orchestrator/hub.py +++ b/orchestrator/kvb_orchestrator/hub.py @@ -31,6 +31,7 @@ from .messaging import Messaging from .names import Names from .offices import Offices from .presence import SNAPSHOT +from .queue_order import QUEUE_SCRIPT, QueueOrder from .receipts import ( NAMES, READS, @@ -127,6 +128,7 @@ class Hub(Mcp, Reviewing, Asking, Messaging, Choosing, Approving, Denying, Reque voice_notes: VoiceNotes replay: Path history: History + queue_order: QueueOrder view_store: ViewStore def hosts(self) -> tuple[str, str]: @@ -193,6 +195,7 @@ def open_hub(council: Path, pages: Path, public_port: int, state: Path, host_sta kept=places.kept, council=council, history=History(council / queue_live.HISTORY), + queue_order=QueueOrder(script=scripts.skills / QUEUE_SCRIPT, council=council, home=state), pages=pages, public_port=public_port, reviews=reviews, diff --git a/orchestrator/kvb_orchestrator/queue_order.py b/orchestrator/kvb_orchestrator/queue_order.py new file mode 100644 index 0000000..c861c89 --- /dev/null +++ b/orchestrator/kvb_orchestrator/queue_order.py @@ -0,0 +1,193 @@ +from __future__ import annotations + +import json +import math +import subprocess +import sys +import threading +from dataclasses import dataclass, field +from pathlib import Path + +from .crons import outcome_of +from .records import shown +from .skill_scripts import script_env +from .tasks import QUEUE_SHOWN, TEXT_SHOWN, prio_of + +QUEUE_SCRIPT = "kvb-org-queue/scripts/queue.py" +ORDER_ARGS = ("list", "--json") +ORDER_SECONDS = 20 +ORDER_TIMEOUT = 10 +ORDER_LIMIT = 1 << 20 +TRACE_SHOWN = 20 +POSITION_SHOWN = 32 +CLASSES = ("hotfix", "fix", "feature", "main") + + +def amount(value: object) -> float | None: + if isinstance(value, bool) or not isinstance(value, int | float) or not math.isfinite(value) or value < 0: + return None + return float(value) + + +def text(value: object) -> str: + return value[:TEXT_SHOWN] if isinstance(value, str) else "" + + +def position_of(value: object) -> int | str | None: + if isinstance(value, int) and not isinstance(value, bool) and value >= 0: + return value + return value[:POSITION_SHOWN] if isinstance(value, str) and value else None + + +def steps_of(value: object) -> list[str]: + return [step[:TEXT_SHOWN] for step in value[:TRACE_SHOWN] if isinstance(step, str)] if isinstance(value, list) else [] + + +def bounded(value: object) -> str | None: + return value if isinstance(value, str) and len(value) <= TEXT_SHOWN else None + + +def score_of(value: object) -> float | None: + return amount(value.get("score") if isinstance(value, dict) else value) + + +def queue_job(raw: object) -> dict | None: + """Args: + raw: one row of `queue.py list --json`. + + Returns: + The row as the board shows it, each field of one type and bounded: + position an int or a word such as run, class one of CLASSES, prio an + int, wsjf (a number or {"score"}), waited_s and eta_s non-negative + numbers, each else None; trace at most TRACE_SHOWN strings; agent + falls back to name. None for a row without a string id. + """ + if not isinstance(raw, dict) or not isinstance(raw.get("id"), str) or not raw["id"]: + return None + klass = bounded(raw.get("class")) + return { + "position": position_of(raw.get("position")), + "id": raw["id"][:TEXT_SHOWN], + "kind": text(raw.get("kind")), + "task": bounded(raw.get("task")), + "session": text(raw.get("session")), + "agent": text(raw.get("agent", raw.get("name"))), + "class": klass if klass in CLASSES else None, + "prio": prio_of(raw), + "wsjf": score_of(raw.get("wsjf")), + "waited_s": amount(raw.get("waited_s")), + "eta_s": amount(raw.get("eta_s")), + "reason": text(raw.get("reason")), + "trace": steps_of(raw.get("trace")), + } + + +def lines_of(stdout: str) -> list | None: + try: + return [json.loads(line) for line in stdout.splitlines() if line.strip()] or None + except (ValueError, RecursionError): + return None + + +def parsed(stdout: str) -> object: + try: + return json.loads(stdout) + except (ValueError, RecursionError): + return lines_of(stdout) + + +def rows_of(stdout: str) -> list | None: + data = parsed(stdout) + if isinstance(data, dict): + data = data.get("entries") if "entries" in data else [data] if "id" in data else None + return data if isinstance(data, list) else None + + +def queue_jobs(stdout: str) -> list[dict] | None: + """Args: + stdout: what `queue.py list --json` printed: a JSON list of rows, + an object holding them under entries, or one row object per + line. + + Returns: + queue_job() of each row in grant order, each id once, at most + QUEUE_SHOWN; None when the output holds no such rows. + """ + rows = rows_of(stdout) + if rows is None: + return None + found: dict[str, dict] = {} + for job in map(queue_job, rows): + if job is not None and job["id"] not in found: + found[job["id"]] = job + return list(found.values())[:QUEUE_SHOWN] + + +def live_ids(queue: list[tuple[dict, float]]) -> frozenset[str]: + return frozenset(shown(entry.get("id")) for entry, _ in queue if outcome_of(entry) is None) + + +def grant_order(chain: list[tuple[str, bool]], jobs: list[dict] | None) -> list[tuple[str, bool]]: + """Args: + chain: queue_chain(), the hub's own grant order. + jobs: queue_jobs() of the installed queue.py, None without them. + + Returns: + The chain in queue.py's order: the ids jobs lists in its order, the + others after them in the chain's; the chain itself without jobs. + """ + if jobs is None: + return chain + place = {job["id"]: index for index, job in enumerate(jobs)} + return sorted(chain, key=lambda pair: place.get(pair[0], len(place))) + + +@dataclass +class QueueOrder: + script: Path + council: Path + home: Path + lock: threading.Lock = field(default_factory=threading.Lock) + cached: list[dict] | None = None + live: frozenset[str] | None = None + at: float = 0.0 + + def run(self) -> list[dict] | None: + try: + done = subprocess.run( + [sys.executable, str(self.script), *ORDER_ARGS], + capture_output=True, + timeout=ORDER_TIMEOUT, + env=script_env(self.home, self.council), + stdin=subprocess.DEVNULL, + check=False, + ) + except (OSError, subprocess.TimeoutExpired): + return None + if done.returncode != 0 or len(done.stdout) > ORDER_LIMIT: + return None + return queue_jobs(done.stdout.decode("utf-8", errors="replace")) + + def get(self, live: frozenset[str], now: float) -> list[dict] | None: + """Args: + live: live_ids() of the queue entries the hub reads. + now: epoch seconds. + + Returns: + [] while no entry is live, without a run. Else queue_jobs() of a + `queue.py list --json` run, run again once ORDER_SECONDS passed + or the live ids changed; while another request runs it, the rows + read before. None while the installed queue.py lists no JSON, + fails or overruns ORDER_TIMEOUT, so the board keeps the hub's own + order and the page says why. + """ + if not live: + return [] + if not self.lock.acquire(blocking=False): + return self.cached + try: + if live != self.live or now - self.at >= ORDER_SECONDS: + self.cached, self.live, self.at = self.run(), live, now + return self.cached + finally: + self.lock.release() diff --git a/orchestrator/kvb_orchestrator/skill_scripts.py b/orchestrator/kvb_orchestrator/skill_scripts.py index e9fd76f..b24d092 100644 --- a/orchestrator/kvb_orchestrator/skill_scripts.py +++ b/orchestrator/kvb_orchestrator/skill_scripts.py @@ -34,6 +34,10 @@ def bounded(stdout: str) -> str: return "\n".join([first if len(first) <= FIRST_LINE_LIMIT else "", ELIDED, rest[-OUTPUT_LIMIT:]]) +def script_env(home: Path, council: Path) -> dict[str, str]: + return {"PATH": SAFE_PATH, "HOME": str(home), "AGENTS_COUNCIL_ROOT": str(council), "PYTHONDONTWRITEBYTECODE": "1"} + + def script_module(script: Path) -> ModuleType | None: """Returns: The installed script as a module, loaded again after an install @@ -73,7 +77,7 @@ class Scripts: def run(self, script: str, args: list[str]) -> dict: command = " ".join([Path(script).name, *args]) - env = {"PATH": SAFE_PATH, "HOME": str(self.home), "AGENTS_COUNCIL_ROOT": str(self.council), "PYTHONDONTWRITEBYTECODE": "1"} + env = script_env(self.home, self.council) try: done = subprocess.run( [sys.executable, str(self.skills / script), *args], diff --git a/orchestrator/kvb_orchestrator/view.py b/orchestrator/kvb_orchestrator/view.py index 8724ea9..06069a8 100644 --- a/orchestrator/kvb_orchestrator/view.py +++ b/orchestrator/kvb_orchestrator/view.py @@ -24,6 +24,7 @@ from .messaging import message_row from .orbits import TRACK_STEPS, track from .phrasing import joined, phrase_parts from .presence import presence, read_snapshot +from .queue_order import grant_order, live_ids from .receipts import inline, read_of from .records import epoch, iso_epoch, read_jsonl from .register import ( @@ -120,9 +121,10 @@ class View: settings = self.settings.get() inbox = self.inbox(now, tasks, directory) queue = queue_entries(self.council, self.history.rows) + jobs = self.queue_order.get(live_ids(queue), now) order = stored_order(self.council / AGENT_ORDER) rows = task_rows(tasks, queue, self.tree.get()) - chain = queue_chain(queue, tasks, order, now) + chain = grant_order(queue_chain(queue, tasks, order, now), jobs) return { "settings": settings, "media": media_origins(settings), @@ -151,6 +153,7 @@ class View: "tasks": rows, "task_edges": task_edges(rows, chain), "queue_line": queue_line(chain, queue), + "queue_jobs": jobs, "agent_order": agent_order(tasks, claims, order), "radio": radio, "claims": claims, diff --git a/orchestrator/kvb_orchestrator/viewstore.py b/orchestrator/kvb_orchestrator/viewstore.py index e4d4ef5..e4c7650 100644 --- a/orchestrator/kvb_orchestrator/viewstore.py +++ b/orchestrator/kvb_orchestrator/viewstore.py @@ -17,6 +17,7 @@ VIEW = { "resources": {"hidden": [], "tokens": "interval", "combined": False, "combinedHidden": []}, "burn": {"overlays": []}, "home": {"pins": []}, + "kanban": {"hidden": []}, } VIEW_PATH = re.compile(r"[A-Za-z]{1,32}(?:\.[A-Za-z]{1,32})?") SERIES_ID = re.compile(r"[A-Za-z0-9:_./-]{1,200}") @@ -24,6 +25,7 @@ PIN_KIND = re.compile(r"[A-Za-z0-9:_./-]{1,32}") IDS_LIMIT = 500 TOKEN_MODES = ("interval", "accumulated") OVERLAYS = ("fix", "break", "tokens", "velocity", "security", "score") +KANBAN_LAYERS = ("line", "jobs") PINS_LIMIT = 64 SPAN = (1, 12) METRIC = "metric" @@ -41,6 +43,12 @@ def series_ids(value: object, name: str) -> list[str]: return list(value) +def members(value: object, name: str, what: str, allowed: tuple[str, ...]) -> list[str]: + if not isinstance(value, list) or not all(isinstance(item, str) and item in allowed for item in value) or len(set(value)) != len(value): + raise ToolError(f"{name} must list distinct {what} of {', '.join(allowed)}") + return list(value) + + def span(value: object, name: str) -> int: low, high = SPAN if isinstance(value, bool) or not isinstance(value, int) or not low <= value <= high: @@ -140,11 +148,9 @@ def valid_view(raw: object) -> dict: resources = section(data["resources"], "resources", tuple(VIEW["resources"])) burn = section(data["burn"], "burn", tuple(VIEW["burn"])) home = section(data["home"], "home", tuple(VIEW["home"])) + kanban = section(data["kanban"], "kanban", tuple(VIEW["kanban"])) if not isinstance(resources["tokens"], str) or resources["tokens"] not in TOKEN_MODES: raise ToolError(f"resources.tokens must be one of {', '.join(TOKEN_MODES)}") - overlays = burn["overlays"] - if not isinstance(overlays, list) or not all(isinstance(item, str) and item in OVERLAYS for item in overlays) or len(set(overlays)) != len(overlays): - raise ToolError(f"burn.overlays must list distinct metrics of {', '.join(OVERLAYS)}") return { "resources": { "hidden": series_ids(resources["hidden"], "resources.hidden"), @@ -152,11 +158,20 @@ def valid_view(raw: object) -> dict: "combined": flag(resources["combined"], "resources.combined"), "combinedHidden": series_ids(resources["combinedHidden"], "resources.combinedHidden"), }, - "burn": {"overlays": list(overlays)}, + "burn": {"overlays": members(burn["overlays"], "burn.overlays", "metrics", OVERLAYS)}, "home": {"pins": pins(home["pins"])}, + "kanban": {"hidden": members(kanban["hidden"], "kanban.hidden", "layers", KANBAN_LAYERS)}, } +def upgraded(raw: object) -> object: + """Returns: + A stored document of an older hub with each VIEW section it lacks + at its default; anything else as it is. + """ + return {**copy.deepcopy(VIEW), **raw} if isinstance(raw, dict) else raw + + def slot(view: dict, path: object) -> tuple[dict, str]: """Args: view: a document of the shape of VIEW. @@ -188,7 +203,7 @@ class ViewStore: self.file = file self.lock = threading.Lock() try: - self.current = valid_view(read_json(file)) + self.current = valid_view(upgraded(read_json(file))) except ToolError: self.current = copy.deepcopy(VIEW) diff --git a/orchestrator/server.py b/orchestrator/server.py index 637e0e2..f3e88a7 100755 --- a/orchestrator/server.py +++ b/orchestrator/server.py @@ -18,7 +18,8 @@ --host-state: the state directory's path on the host, named in the radio rows that carry an upload or a voice recording. --skills: installed skills directory holding the kvb-org-radio, - kvb-org-rasci and kvb-org-suspend scripts. + kvb-org-rasci and kvb-org-suspend scripts, and the kvb-org-queue + queue.py. --transcripts: directory the Claude Code transcripts (*.jsonl) lie under. --templates: directory holding index.html.j2 and the *.js modules it includes. diff --git a/orchestrator/templates/kanban.js b/orchestrator/templates/kanban.js index 52c9cb8..0243804 100644 --- a/orchestrator/templates/kanban.js +++ b/orchestrator/templates/kanban.js @@ -6,7 +6,13 @@ const Kanban = (() => { const FREE_FROM = 4; const OFF = ["command"]; const DAY = 86400; + const HOUR = 3600; + const MINUTE = 60; const REFUSED = ["failed", "refused"]; + const QUEUE = "queue/"; + const RUNNING = "run"; + const QUEUE_LAYERS = [["line", "Queue line", "k-queue-line"], ["jobs", "Queue jobs", "k-queue-jobs"]]; + const CLASS_TONES = {hotfix: "bad", fix: "warn", feature: "info", main: "violet"}; const onBoard = (row) => !OFF.includes(row.kind); const laneOf = (prio) => (prio === null ? "unpinned" : prio >= FREE_FROM ? "p4" : `p${Math.max(prio, 1)}`); @@ -164,7 +170,102 @@ const Kanban = (() => { return failed.length ? {failed, applied: batch.filter((job) => job.status === "done").length, total: batch.length} : null; } - return {COLUMNS, LANES, FREE_FROM, onBoard, columnOf, laneOf, labelOf, board, freePin, move, stage, pending, lines, why}; + /** + * @param seconds - a span in seconds. + * @returns the span in words, cut to its two largest units: "45 s", "12 min", "2 h 5 min", "3 d 4 h". + */ + function span(seconds) { + const whole = Math.max(0, Math.floor(seconds)); + if (whole < MINUTE) { + return `${whole} s`; + } + if (whole < HOUR) { + return `${Math.floor(whole / MINUTE)} min`; + } + const [big, unit, small, rest] = whole < DAY ? [HOUR, "h", MINUTE, "min"] : [DAY, "d", HOUR, "h"]; + const part = Math.floor((whole % big) / small); + return part ? `${Math.floor(whole / big)} ${unit} ${part} ${rest}` : `${Math.floor(whole / big)} ${unit}`; + } + + /** + * @param rows - the shown cards' rows. + * @param line - the hub's queue_line: tasks holding a live queue entry, in grant order. + * @param jobs - the hub's queue_jobs: the rows of `queue.py list --json` in grant order, null where the installed + * queue.py lists no JSON. + * @returns {task: {place, of, jobs}} for each shown card the queue concerns: place its 1-based stop on the queue + * line (null for a card off the line), of the line's length, and jobs its queue jobs in grant order, a queue card's + * own job and a task card's jobs by task; jobs stays empty without queue_jobs. + */ + function queued(rows, line, jobs) { + const found = {}; + for (const row of rows) { + const place = line.indexOf(row.task); + const own = (jobs || []).filter((job) => (row.kind === "queue" ? `${QUEUE}${job.id}` === row.task : job.task === row.task)); + if (place >= 0 || own.length) { + found[row.task] = {place: place >= 0 ? place + 1 : null, of: line.length, jobs: own}; + } + } + return found; + } + + const placeOf = (job) => (typeof job.position === "number" ? `#${job.position}` : job.position === RUNNING ? "running" : job.position || "?"); + + /** + * @param job - one queue_jobs row. + * @returns the chips of its card overlay in display order, each {key, text, hint, tone}: its grant position, class, + * pin, WSJF, the gate it runs (its kind), how long it waits and its ETA; a class, pin, WSJF or wait the row lacks + * leaves its chip out, an unknown ETA reads "ETA ?". + */ + function jobChips(job) { + const running = job.position === RUNNING; + const chips = [{key: "position", text: placeOf(job), hint: `${job.id}: ${running ? "granted, running now" : `grant position ${placeOf(job)}`}`, tone: "neutral"}]; + if (job.class) { + chips.push({key: "class", text: job.class, hint: `${job.id}: class ${job.class}`, tone: CLASS_TONES[job.class]}); + } + if (job.prio !== null) { + chips.push({key: "prio", text: `P${job.prio}`, hint: `${job.id}: its task is pinned P${job.prio}`, tone: "neutral"}); + } + if (job.wsjf !== null) { + chips.push({key: "wsjf", text: `WSJF ${Number(job.wsjf.toFixed(2))}`, hint: `${job.id}: WSJF ${job.wsjf} of its task`, tone: "neutral"}); + } + chips.push({key: "gate", text: job.kind || "?", hint: `${job.id}: runs the gate ${job.kind || "of an unknown kind"}`, tone: "teal"}); + if (job.waited_s !== null) { + chips.push({key: "waited", text: `${running ? "waited" : "waits"} ${span(job.waited_s)}`, hint: `${job.id}: ${running ? "waited" : "waits"} ${span(job.waited_s)} in the queue`, tone: "neutral"}); + } + const eta = job.eta_s === null ? "no ETA learned yet" : `${running ? "ends" : "starts"} in about ${span(job.eta_s)}`; + chips.push({key: "eta", text: job.eta_s === null ? "ETA ?" : `ETA ${span(job.eta_s)}`, hint: `${job.id}: ${eta}`, tone: "neutral"}); + return chips; + } + + /** + * @param job - one queue_jobs row. + * @returns {label, place, steps}: the popover's name, where the job stands (#n, running or the word queue.py gave) + * and the rule trace that put it there, in the order queue.py applied it; a row without a trace shows its reason + * as the only step, one without either says so. + */ + function whyHere(job) { + const steps = job.trace.length ? job.trace : [job.reason || "queue.py gave no trace and no reason for this job"]; + return {label: `Why here: ${job.id} at ${placeOf(job)}`, place: placeOf(job), steps}; + } + + /** + * @param hidden - the kanban.hidden view-store list: the queue layers switched off. + * @returns {line, jobs}: whether the numbered queue line and the job overlay of the cards show. + */ + function layers(hidden) { + return Object.fromEntries(QUEUE_LAYERS.map(([id]) => [id, !hidden.includes(id)])); + } + + /** + * @param jobs - the hub's queue_jobs. + * @returns them with waited_s and eta_s in whole minutes. + */ + function queueSig(jobs) { + const minutes = (value) => (value === null ? null : Math.floor(value / MINUTE)); + return jobs === null ? null : jobs.map((job) => ({...job, waited_s: minutes(job.waited_s), eta_s: minutes(job.eta_s)})); + } + + return {COLUMNS, LANES, FREE_FROM, QUEUE_LAYERS, onBoard, columnOf, laneOf, labelOf, board, freePin, move, stage, pending, lines, why, span, queued, jobChips, whyHere, layers, queueSig}; })(); if (typeof module === "object" && module.exports) { module.exports = Kanban; diff --git a/orchestrator/templates/page-kanban.js b/orchestrator/templates/page-kanban.js index 8f90a8e..23189eb 100644 --- a/orchestrator/templates/page-kanban.js +++ b/orchestrator/templates/page-kanban.js @@ -1,7 +1,9 @@ const KANBAN_LABELS = {blocked: ["Blocked", "bad"], stopped: ["Stopped", "neutral"]}; const KANBAN_COLUMN_ICONS = {queued: "bi-hourglass", running: "bi-play-circle", done: "bi-check2-circle"}; const KANBAN_DRAG = "application/x-kvb-task"; - const kanban = {staged: [], collapsed: new Set(), old: false, dragging: null}; + const KANBAN_LAYERS = "kanban.hidden"; + const KANBAN_JOBS_SHOWN = 3; + const kanban = {staged: [], collapsed: new Set(), old: false, dragging: null, queue: {marks: {}, line: true, jobs: true}, bound: false}; function restage(next) { kanban.staged = next; @@ -81,6 +83,65 @@ const cardOf = (task) => () => [...document.querySelectorAll("#main .kanban-card")].find((node) => node.dataset.task === task) || null; + function whyButton(task, id) { + const card = cardOf(task)(); + return card ? [...card.querySelectorAll(".queue-why")].find((node) => node.dataset.job === id) || null : null; + } + + function openWhy(task, job) { + const found = Kanban.whyHere(job); + const anchor = () => whyButton(task, job.id); + const body = h("div", {class: "popup-body"}, h("ol", {class: "queue-trace", "aria-label": `Rule trace of ${job.id}`}, found.steps.map((step) => h("li", {}, step))), + job.reason && job.trace.length ? h("p", {class: "queue-reason"}, `Now: ${job.reason}`) : null); + openPopup(found.label, [popupHead(icon("bi-signpost-split"), found.label, h("code", {class: "task-id"}, job.task || task)), body], anchor, () => { + const button = anchor(); + if (button) { + button.focus(); + } + }); + const close = popupBox().querySelector(".popup-head button"); + if (close) { + close.focus(); + } + } + + function queuePlace(row) { + const mark = kanban.queue.marks[row.task]; + if (!kanban.queue.line || !mark || mark.place === null) { + return null; + } + return h("span", {class: "queue-place", "data-place": mark.place, role: "img", ...named(`Queue stop ${mark.place} of ${mark.of}`, `${row.task}: stop ${mark.place} of ${mark.of} on the queue line, in grant order`)}, String(mark.place)); + } + + function chipHint(job, chip) { + if (chip.key === "waited") { + return `${chip.hint}, since ${absolute.format(stampOf(state.now - job.waited_s))}`; + } + return chip.key === "eta" && job.eta_s !== null ? `${chip.hint}, about ${absolute.format(stampOf(state.now + job.eta_s))}` : chip.hint; + } + + function queueJob(row, job) { + const chips = Kanban.jobChips(job).map((chip) => h("span", {class: "pill queue-chip", "data-chip": chip.key, "data-tone": chip.tone, title: chipHint(job, chip)}, chip.text)); + const open = (event) => { + event.stopPropagation(); + openWhy(row.task, job); + }; + const hint = `Why ${job.id} stands at ${Kanban.whyHere(job).place}: the rules queue.py applied, in order`; + const why = h("button", {type: "button", class: "btn small ghost queue-why", "data-job": job.id, ...named(`Why here: ${job.id}`, hint), on: {click: open}}, icon("bi-question-circle"), h("span", {class: "queue-why-text"}, "Why here?")); + return h("li", {class: "queue-job", "data-job": job.id, "data-class": job.class || "", "data-kind": job.kind}, + h("div", {class: "queue-chips"}, chips, why), job.reason ? h("p", {class: "queue-reason", title: job.reason}, job.reason) : null); + } + + function queueJobs(row) { + const mark = kanban.queue.marks[row.task]; + if (!kanban.queue.jobs || !mark || !mark.jobs.length) { + return null; + } + const more = mark.jobs.length - KANBAN_JOBS_SHOWN; + return h("ul", {class: "queue-jobs", "aria-label": `Queue jobs of ${row.task}`}, mark.jobs.slice(0, KANBAN_JOBS_SHOWN).map((job) => queueJob(row, job)), + more > 0 ? h("li", {class: "queue-more muted"}, `+${more} more queue jobs`) : null); + } + function kanbanCard(row, lane, column, cards) { const open = () => taskSelect(row.task, cardOf(row.task)); const own = (event) => !event.target.closest("button, summary, details, select, .kanban-handle"); @@ -112,7 +173,7 @@ }, }, }, - h("div", {class: "head-row", "data-star-slot": ""}, h("code", {class: "task-id"}, row.task), cardLabels(row), h("span", {class: "spacer"}), + h("div", {class: "head-row", "data-star-slot": ""}, queuePlace(row), h("code", {class: "task-id"}, row.task), cardLabels(row), h("span", {class: "spacer"}), row.pending ? h("span", {class: "kanban-pending", role: "img", ...named("Staged", row.pending.map(shownArgv).join("\n"))}, icon("bi-pencil")) : null, openJobs(row.task).length ? h("span", {class: "kanban-wait", role: "img", ...named("At the runner", `${row.task}: a signed change waits for the runner`)}, icon("bi-hourglass-split")) : null, row.editable ? h("span", {class: "kanban-handle", role: "img", draggable: "true", ...named("Follows", `Drag onto the card that waits for ${row.task}`), on: {dragstart: dragFrom(row.task, "after"), dragend: () => { @@ -121,6 +182,7 @@ row.movable ? kanbanMenu(row, cards) : null), h("span", {class: "kanban-title"}, titleOf(row)), row.worker !== "-" ? h("span", {class: "muted"}, row.worker) : null, + queueJobs(row), cardWhy(row.task)); } @@ -168,17 +230,39 @@ const stalled = h("span", {class: "pill", id: "kanban-stale", "data-tone": "warn", role: "img", ...named(`Stale: ${stale}`, `Stale: ${stale} tasks without a register change for ${Plan.STALE_DAYS} days, counted apart`)}, icon("bi-hourglass-bottom"), String(stale)); const listed = count ? h("ol", {class: "kanban-staged", id: "kanban-staged", "aria-label": `${words}: the passkey signs exactly these`}, kanban.staged.map(({task, argv}) => h("li", {"data-task": task}, h("code", {}, shownArgv(argv))))) : null; - return [h("div", {class: "kanban-bar", role: "toolbar", "aria-label": "Board"}, apply, discard, older, stalled), listed]; + return [h("div", {class: "kanban-bar", role: "toolbar", "aria-label": "Board"}, apply, discard, older, stalled, queueLegend()), listed]; + } + + function queueLegend() { + const layers = LegendChips.render(Kanban.QUEUE_LAYERS.map(([id, label, tone]) => ({id, label, tone})), KANBAN_LAYERS, "hidden"); + const words = "The installed queue.py lists no JSON (queue.py list --json): the board numbers the grant order but shows no class, gate, ETA or rule trace"; + const only = state.queue_jobs === null && state.queue_line.length ? h("span", {class: "pill", id: "kanban-queue-order", "data-tone": "warn", role: "img", ...named("Queue order only", words)}, icon("bi-info-circle"), "Order only") : null; + return [layers, only]; + } + + function bindLayers() { + if (!kanban.bound) { + kanban.bound = true; + ViewStore.on(KANBAN_LAYERS, () => { + taskView.sig = ""; + drawTasks(); + }); + } } function kanbanLines(box, rows) { const svg = box.querySelector("svg.kanban-lines"); const base = box.getBoundingClientRect(); - const rectOf = (task) => { - const node = [...box.querySelectorAll(".kanban-card")].find((card) => card.dataset.task === task); + const rectIn = (node) => { const found = node && node.getClientRects().length ? node.getBoundingClientRect() : null; return found && {x: found.left - base.left, y: found.top - base.top, w: found.width, h: found.height}; }; + const cardNode = (task) => [...box.querySelectorAll(".kanban-card")].find((card) => card.dataset.task === task) || null; + const rectOf = (task) => rectIn(cardNode(task)); + const stopOf = (task) => { + const card = cardNode(task); + return rectIn(card && card.querySelector(".queue-place")) || rectIn(card); + }; const centre = (r) => ({x: r.x + r.w / 2, y: r.y + r.h / 2}); const rim = (r, toward) => { const c = centre(r); @@ -187,7 +271,7 @@ return {x: c.x + dx * scale, y: c.y + dy * scale}; }; const hub = new Map(state.tasks.map((row) => [row.task, row])); - const {arrows, queue} = Kanban.lines(rows, state.queue_line); + const {arrows, queue} = Kanban.lines(rows, kanban.queue.line ? state.queue_line : []); const marks = []; for (const {from, to} of arrows) { const [a, b] = [rectOf(from), rectOf(to)]; @@ -199,7 +283,7 @@ s("title", {}, `${to} follows ${from}: click to remove`), s("path", {class: "kanban-hit", d}), s("path", {class: "kanban-link", d, "marker-end": "url(#kanban-tip)"}))); } } - const stops = queue.map(rectOf).filter(Boolean).map(centre); + const stops = queue.map(stopOf).filter(Boolean).map(centre); if (stops.length > 1) { marks.push(s("polyline", {class: "kanban-queue", "data-tasks": queue.filter((task) => rectOf(task)).join(" "), points: stops.map((at) => `${fixed(at.x)},${fixed(at.y)}`).join(" ")}, s("title", {}, "Queue order"))); } @@ -215,6 +299,8 @@ } const ids = new Set(shown.map((row) => row.task)); const rows = Kanban.pending(state.tasks, kanban.staged).filter((row) => ids.has(row.task) && Kanban.onBoard(row)); + bindLayers(); + kanban.queue = {marks: Kanban.queued(rows, state.queue_line, state.queue_jobs), ...Kanban.layers(ViewStore.get(KANBAN_LAYERS))}; const {lanes} = Kanban.board(rows, {now: state.now, old: kanban.old}); const cards = lanes.flatMap((lane) => Object.values(lane.cells).flat()); const head = h("div", {class: "kanban-head"}, TASK_COLUMNS.map(([id, label]) => h("h3", {class: "task-column group-title", "data-column": id}, icon(KANBAN_COLUMN_ICONS[id]), label, diff --git a/orchestrator/templates/page-tasks.js b/orchestrator/templates/page-tasks.js index efcc131..b8fad5e 100644 --- a/orchestrator/templates/page-tasks.js +++ b/orchestrator/templates/page-tasks.js @@ -191,7 +191,7 @@ stage3d("tasks", dom.stage); } const q = taskView.q.trim().toLowerCase(); - const sig = JSON.stringify([state.tasks, state.jobs, state.task_edges, settings.estimation, settings.trend, taskView.filter, taskView.sort, taskView.descending, taskView.forecast, taskView.scope, q, waitingCount(), Math.floor(nowSeconds() / 60)]); + const sig = JSON.stringify([state.tasks, state.jobs, state.task_edges, state.queue_line, Kanban.queueSig(state.queue_jobs), settings.estimation, settings.trend, taskView.filter, taskView.sort, taskView.descending, taskView.forecast, taskView.scope, q, waitingCount(), Math.floor(nowSeconds() / 60)]); if (sig === taskView.sig) { return; } diff --git a/orchestrator/templates/style-kanban.css b/orchestrator/templates/style-kanban.css index b7f1fa3..66f8ce5 100644 --- a/orchestrator/templates/style-kanban.css +++ b/orchestrator/templates/style-kanban.css @@ -32,6 +32,20 @@ svg .kanban-hit { fill: none; stroke: transparent; stroke-width: 12; pointer-eve svg .kanban-arrow.pending .kanban-link { opacity: .55; } svg .kanban-tip { fill: var(--violet); } svg .kanban-queue { fill: none; stroke: var(--warn); stroke-width: 1.6; stroke-dasharray: 6 4; } +.kanban-card .queue-place + .task-id { flex-basis: calc(100% - 2.4em); } +.queue-place { display: inline-grid; flex: none; place-items: center; min-width: 1.7em; height: 1.7em; padding: 0 4px; border: 2px solid var(--warn); border-radius: 999px; background: var(--warn-soft); color: var(--warn); font-size: var(--fs-2xs); font-weight: 750; font-variant-numeric: tabular-nums; line-height: 1; } +.queue-jobs { display: grid; gap: 4px; min-width: 0; margin: 0; padding: 0; list-style: none; } +.queue-job { display: grid; gap: 2px; min-width: 0; padding-top: 4px; border-top: 1px dashed var(--line); } +.queue-chips { display: flex; flex-wrap: wrap; align-items: center; gap: 3px; min-width: 0; } +.queue-chips .pill { max-width: 100%; padding: 0 6px; line-height: 18px; white-space: normal; overflow-wrap: anywhere; } +.queue-chips .queue-why { min-width: 0; padding: 0 6px; font-size: var(--fs-xs); } +.queue-reason, .queue-more { margin: 0; color: var(--muted); font-size: var(--fs-2xs); overflow-wrap: anywhere; } +.queue-trace { display: grid; gap: 4px; margin: 0; padding-left: 1.4em; overflow-wrap: anywhere; } +.legend-chip.k-queue-line { --c: var(--warn); } +.legend-chip.k-queue-jobs { --c: var(--teal); } +@media (max-width: 640px) { + .queue-why-text { display: none; } +} svg .burn-finish-line { fill: none; stroke: var(--c); stroke-width: 2; stroke-dasharray: 8 3; } svg .burn-finish-end { fill: var(--c); } svg.burn .k-fixed, .burn-key.k-fixed { --c: var(--violet); } diff --git a/tests/e2e/test_queue_overlay_e2e.py b/tests/e2e/test_queue_overlay_e2e.py new file mode 100644 index 0000000..c6afa63 --- /dev/null +++ b/tests/e2e/test_queue_overlay_e2e.py @@ -0,0 +1,221 @@ +from __future__ import annotations + +import json +import os +import shutil +import sys +import time +import unittest +from pathlib import Path + +HERE = Path(__file__).resolve().parent +sys.path[:0] = [str(HERE.parent), str(HERE)] + +import test_orchestrator_e2e as base +from hub_case import HubCase +from test_hub_queue_overlay import ( + HUB_ORDER, + LINE_ORDER, + NEW_QUEUE, + OLD_QUEUE, + ROWS, + board_council, + fake_queue, +) + +OUT = Path(os.environ.get("E2E_OUT", "/tmp/queue-overlay-shots")) +CHIPS = { + "q-1": [["position", "running"], ["class", "fix"], ["prio", "P2"], ["gate", "test"], ["waited", "waited 1 min"], ["eta", "ETA 4 min"]], + "q-2": [["position", "#1"], ["class", "hotfix"], ["prio", "P1"], ["wsjf", "WSJF 2.67"], ["gate", "quality-high"], ["waited", "waits 21 min"], ["eta", "ETA 8 min"]], + "q-4": [["position", "#2"], ["class", "main"], ["gate", "deploy"], ["waited", "waits 1 h 6 min"], ["eta", "ETA ?"]], + "q-3": [["position", "#3"], ["class", "feature"], ["wsjf", "WSJF 1.5"], ["gate", "test"], ["waited", "waits 30 s"], ["eta", "ETA 1 h 5 min"]], +} +READ = """ + const card = (task) => all("#main .kanban-card").find((node) => node.dataset.task === task) || null; + const chip = (id) => all('#main .legend-chip[data-path="kanban.hidden"]').find((node) => node.dataset.chip === id) || null; + const centre = (node, at) => { + const box = node.getBoundingClientRect(); + return [box.left + box.width / 2 - at.left, box.top + box.height / 2 - at.top]; + }; + const read = () => { + const at = document.querySelector("#main .kanban").getBoundingClientRect(); + const line = document.querySelector("#main svg.kanban-lines .kanban-queue"); + const points = line ? line.getAttribute("points").trim().split(/\\s+/).map((pair) => pair.split(",").map(Number)) : []; + const badges = (line ? line.dataset.tasks.split(" ") : []).map((task) => card(task).querySelector(".queue-place")).filter(Boolean).map((node) => centre(node, at)); + const spills = (node) => { + const own = node.getBoundingClientRect(); + const host = node.closest(".kanban-card").getBoundingClientRect(); + return own.right > host.right + 1 || own.left < host.left - 1 || node.scrollWidth > node.clientWidth + 1; + }; + return { + places: Object.fromEntries(all("#main .kanban-card .queue-place").map((node) => [node.closest(".kanban-card").dataset.task, [node.dataset.place, name(node)]])), + line: line ? line.dataset.tasks : null, + stops: points.length > 1 && points.length === badges.length && points.every(([x, y], index) => Math.abs(x - badges[index][0]) < 1.5 && Math.abs(y - badges[index][1]) < 1.5), + jobs: Object.fromEntries(all("#main .kanban-card").filter((node) => node.querySelector(".queue-job")).map((node) => [node.dataset.task, + all(".queue-job", node).map((job) => [job.dataset.job, all(".queue-chip", job).map((item) => [item.dataset.chip, text(item)]), text(job.querySelector(".queue-reason"))])])), + legend: all('#main .legend-chip[data-path="kanban.hidden"]').map((node) => [node.dataset.chip, node.getAttribute("aria-pressed")]), + order: name(document.getElementById("kanban-queue-order")), + buttons: all("#main .queue-why").map((node) => [node.dataset.job, name(node), node.tabIndex, node.getClientRects().length > 0]), + spill: all("#main .queue-job, #main .queue-place").filter(spills).length, + unlabelled: all("#main .kanban-box svg.bi").map((svg) => svg.parentElement).filter((owner) => !text(owner) && !(owner.getAttribute("title") && owner.getAttribute("aria-label"))).length, + wide: document.documentElement.scrollWidth > window.innerWidth + 1, + }; + }; +""" +BOARD = """(async () => {@HELPERS@@READ@ + if (!(await until(() => card("work/alpha") && document.querySelector("#main .queue-job") && document.querySelector("#main svg.kanban-lines .kanban-queue"), 10000))) { + return {error: text(document.getElementById("main"))}; + } + await sleep(400); + const into = Date.now() % 60000; + if (into > 45000) { + await sleep(61000 - into); + } + const before = read(); + const why = card("work/alpha").querySelector(".queue-why"); + why.scrollIntoView({block: "center"}); + why.focus(); + return {before, focused: document.activeElement === why ? name(why) : null}; +})()""" +POPUP = """(async () => {@HELPERS@ + const dialog = await until(() => document.getElementById("popup").open && document.getElementById("popup"), 4000); + if (!dialog) { + return {error: "no popup after Enter", focused: document.activeElement ? document.activeElement.outerHTML.slice(0, 200) : null}; + } + await sleep(300); + const box = dialog.querySelector(".popup-box").getBoundingClientRect(); + return { + label: dialog.getAttribute("aria-label"), steps: all("#popup .queue-trace li").map(text), now: text(dialog.querySelector(".queue-reason")), + focus: Boolean(document.activeElement && document.activeElement.closest("#popup")), inside: box.left >= 0 && box.right <= window.innerWidth + 1, + wide: document.documentElement.scrollWidth > window.innerWidth + 1, + }; +})()""" +LAYERS = """(async () => {@HELPERS@@READ@ + document.activeElement.dispatchEvent(new KeyboardEvent("keydown", {key: "Escape", bubbles: true})); + await sleep(300); + const back = document.activeElement; + const closed = [document.getElementById("popup").open, back && back.closest(".kanban-card") ? [back.closest(".kanban-card").dataset.task, back.dataset.job] : null]; + chip("jobs").click(); + await until(() => !document.querySelector("#main .queue-job"), 4000); + chip("line").click(); + await until(() => !document.querySelector("#main .queue-place"), 4000); + await sleep(1200); + return {closed, off: read()}; +})()""" +KEPT = """(async () => {@HELPERS@@READ@ + if (!(await until(() => card("work/alpha") && chip("line"), 10000))) { + return {error: text(document.getElementById("main"))}; + } + await sleep(400); + const kept = read(); + chip("jobs").click(); + await until(() => document.querySelector("#main .queue-job"), 4000); + chip("line").click(); + await until(() => document.querySelector("#main .queue-place"), 4000); + await sleep(1200); + return {kept, back: read()}; +})()""" +ORDER_ONLY = """(async () => {@HELPERS@@READ@ + if (!(await until(() => card("work/alpha") && document.querySelector("#main .queue-place") && document.getElementById("kanban-queue-order"), 10000))) { + return {error: text(document.getElementById("main"))}; + } + await sleep(400); + return read(); +})()""" + + +def probe(template: str) -> str: + return template.replace("@HELPERS@", base.HELPERS).replace("@READ@", READ) + + +def jobs_shown() -> dict: + """Returns: + The overlay each card must carry: every task card its job, each + queue card its own, chips as CHIPS and the reason of the fixture. + """ + shown = {} + for row in ROWS: + entry = [[row["id"], CHIPS[row["id"]], row["reason"]]] + shown[row["task"]] = entry + shown[f"queue/{row['id']}"] = entry + return shown + + +@unittest.skipUnless(shutil.which(os.environ.get("CHROMIUM", "chromium")), "no Chromium") +class TestQueueOverlay(HubCase): + drive = base.TestOrchestratorE2E.drive + + def setUp(self): + super().setUp() + OUT.mkdir(parents=True, exist_ok=True) + self.write_council(board_council(time.time())) + self.fakes = self.state.parent / "fake-skills" + self.httpd.hub.queue_order.script = fake_queue(self.fakes / "new", NEW_QUEUE, ROWS) + + def steps(self) -> list[dict]: + local = f"http://localhost:{self.port}/" + steps = [] + for width, height in base.WIDTHS: + steps += [ + {"width": width, "height": height, "url": f"{local}?{width}#tasks/board", "wait": 1500, "probe": probe(BOARD), "shot": str(OUT / f"chromium-{width}-queue-board.png")}, + {"width": width, "height": height, "url": None, "wait": 0, "keys": ["Enter"], "probe": probe(POPUP), "shot": str(OUT / f"chromium-{width}-queue-why.png")}, + {"width": width, "height": height, "url": None, "wait": 0, "probe": probe(LAYERS), "shot": str(OUT / f"chromium-{width}-queue-layers-off.png")}, + {"width": width, "height": height, "url": f"{local}?{width}-reloaded#tasks/board", "wait": 1500, "probe": probe(KEPT), "shot": str(OUT / f"chromium-{width}-queue-layers-back.png")}, + ] + return steps + + def check_fit(self, read: dict) -> None: + self.assertEqual((read["unlabelled"], read["spill"], read["wide"]), (0, 0, False), read) + + def check_full(self, read: dict) -> None: + self.assertEqual(read["places"], {task: [str(place), f"Queue stop {place} of 4"] for place, task in enumerate(LINE_ORDER, 1)}) + self.assertEqual((read["line"], read["stops"]), (" ".join(LINE_ORDER), True), read) + self.assertEqual(read["jobs"], jobs_shown()) + self.assertEqual((read["legend"], read["order"]), ([["line", "true"], ["jobs", "true"]], None)) + self.assertEqual(sorted(read["buttons"]), sorted([job, f"Why here: {job}", 0, True] for job in [row["id"] for row in ROWS] * 2)) + self.check_fit(read) + + def check_off(self, read: dict) -> None: + self.assertEqual((read["places"], read["line"], read["jobs"], read["buttons"]), ({}, None, {}, []), read) + self.assertEqual(read["legend"], [["line", "false"], ["jobs", "false"]]) + self.check_fit(read) + + def test_chromium_numbers_the_line_shows_each_job_explains_it_by_keyboard_and_keeps_the_switched_layers_at_three_widths(self): + data = self.drive(self.steps(), None) + (OUT / "chromium-queue-overlay.json").write_text(json.dumps(data, indent=1), encoding="utf-8") + self.assertEqual([error for error in data["errors"] if "/favicon.ico" not in error], []) + results = iter(data["results"]) + for width, _ in base.WIDTHS: + with self.subTest(width=width): + board, popup, layers, kept = next(results), next(results), next(results), next(results) + self.assertNotIn("error", board, board) + self.check_full(board["before"]) + self.assertEqual(board["focused"], "Why here: q-2") + self.assertEqual((popup.get("label"), popup.get("steps"), popup.get("now")), ("Why here: q-2 at #1", ROWS[1]["trace"], f"Now: {ROWS[1]['reason']}"), popup) + self.assertEqual((popup["focus"], popup["inside"], popup["wide"]), (True, True, False), popup) + self.assertEqual(layers["closed"], [False, ["work/alpha", "q-2"]]) + self.check_off(layers["off"]) + self.assertNotIn("error", kept, kept) + self.check_off(kept["kept"]) + self.check_full(kept["back"]) + self.assertEqual(self.httpd.hub.view_store.get()["kanban"], {"hidden": []}) + + def test_chromium_keeps_the_numbered_hub_order_and_says_why_without_list_json_at_three_widths(self): + order = self.httpd.hub.queue_order + order.script, order.live = fake_queue(self.fakes / "old", OLD_QUEUE, ROWS), None + local = f"http://localhost:{self.port}/" + steps = [{"width": width, "height": height, "url": f"{local}?{width}-old#tasks/board", "wait": 1500, "probe": probe(ORDER_ONLY), "shot": str(OUT / f"chromium-{width}-queue-order-only.png")} for width, height in base.WIDTHS] + data = self.drive(steps, None) + (OUT / "chromium-queue-order-only.json").write_text(json.dumps(data, indent=1), encoding="utf-8") + self.assertEqual([error for error in data["errors"] if "/favicon.ico" not in error], []) + for (width, _), read in zip(base.WIDTHS, data["results"], strict=True): + with self.subTest(width=width): + self.assertNotIn("error", read, read) + self.assertEqual(read["places"], {task: [str(place), f"Queue stop {place} of 4"] for place, task in enumerate(HUB_ORDER, 1)}) + self.assertEqual((read["line"], read["stops"], read["jobs"], read["buttons"]), (" ".join(HUB_ORDER), True, {}, []), read) + self.assertEqual((read["legend"], read["order"]), ([["line", "true"], ["jobs", "true"]], "Queue order only")) + self.check_fit(read) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/queue-list.json b/tests/queue-list.json new file mode 100644 index 0000000..a2c015a --- /dev/null +++ b/tests/queue-list.json @@ -0,0 +1,6 @@ +[ + {"position": "run", "id": "q-1", "kind": "test", "task": "work/bravo", "session": "s-docs", "agent": "Docs", "class": "fix", "prio": 2, "wsjf": null, "waited_s": 95, "eta_s": 240, "reason": "granted on its last poll", "trace": ["granted at its last poll", "slot test:1 of 2"]}, + {"position": 1, "id": "q-2", "kind": "quality-high", "task": "work/alpha", "session": "s-stage", "agent": "Stage", "class": "hotfix", "prio": 1, "wsjf": 2.6667, "waited_s": 1260, "eta_s": 480, "reason": "RAM 61 % used + 15 % still to come reach the limit 75 %", "trace": ["hotfix first", "rank 0", "line slot:2 behind q-1", "RAM 61 % + 15 % reaches 75 %"]}, + {"position": 2, "id": "q-4", "kind": "deploy", "task": "work/delta", "session": "s-skills", "agent": "Skills", "class": "main", "prio": null, "wsjf": null, "waited_s": 4000, "eta_s": null, "reason": "waits behind q-2", "trace": []}, + {"position": 3, "id": "q-3", "kind": "test", "task": "work/charlie", "session": "s-stage", "agent": "Stage", "class": "feature", "prio": null, "wsjf": 1.5, "waited_s": 30, "eta_s": 3900, "reason": "all 1 test slots taken", "trace": ["rank 7", "waited 0 h", "line slot:2 behind q-2", "RAM 52 % + 15 GB fits"]} +] diff --git a/tests/test_hub_queue_overlay.py b/tests/test_hub_queue_overlay.py new file mode 100644 index 0000000..7e98e73 --- /dev/null +++ b/tests/test_hub_queue_overlay.py @@ -0,0 +1,307 @@ +from __future__ import annotations + +import copy +import json +import os +import subprocess +import tempfile +import time +import unittest +from pathlib import Path +from unittest import mock + +from hub_case import HubCase +from hub_support import HUB +from kvb_orchestrator import queue_order +from kvb_orchestrator.errors import ToolError +from kvb_orchestrator.viewstore import VIEW, ViewStore, valid_view + +KANBAN = HUB / "templates" / "kanban.js" +FIXTURE = Path(__file__).resolve().parent / "queue-list.json" +ROWS = json.loads(FIXTURE.read_text(encoding="utf-8")) +NEW_QUEUE = """import json +import os +import pathlib +import sys + +here = pathlib.Path(__file__) +with here.with_suffix(".calls").open("a", encoding="utf-8") as calls: + calls.write(" ".join(sys.argv[1:]) + "\\n") +here.with_suffix(".env").write_text(json.dumps(dict(os.environ)), encoding="utf-8") +if sys.argv[1:] != ["list", "--json"]: + sys.exit(2) +sys.stdout.write(here.with_name("rows.json").read_text(encoding="utf-8")) +""" +OLD_QUEUE = """import argparse + +parser = argparse.ArgumentParser(prog="queue.py") +commands = parser.add_subparsers(dest="command", required=True) +commands.add_parser("list") +parser.parse_args() +print("position\\tid\\tstatus") +""" +SLOW_QUEUE = "import time\ntime.sleep(5)\n" +FAILING_QUEUE = "import sys\nsys.exit(1)\n" +LINE_ORDER = ["work/bravo", "work/alpha", "work/delta", "work/charlie"] +HUB_ORDER = ["work/bravo", "work/alpha", "work/charlie", "work/delta"] +MODEL_PROBE = """ +const K = require(process.argv[1]); +const rows = JSON.parse(process.argv[2]); +const card = (task, kind) => ({task, kind: kind || "task"}); +const cards = [card("work/alpha"), card("work/bravo"), card("work/charlie"), card("work/delta"), card("work/echo"), card("queue/q-2", "queue"), card("queue/q-9", "queue")]; +const line = ["work/bravo", "work/alpha", "work/delta", "work/charlie", "work/hidden"]; +const marks = (jobs) => Object.fromEntries(Object.entries(K.queued(cards, line, jobs)).map(([task, mark]) => [task, [mark.place, mark.of, mark.jobs.map((job) => job.id)]])); +console.log(JSON.stringify({ + marks: marks(rows), + degraded: marks(null), + chips: Object.fromEntries(rows.map((job) => [job.id, K.jobChips(job).map((chip) => [chip.key, chip.text, chip.tone])])), + hints: K.jobChips(rows[1]).map((chip) => chip.hint), + why: rows.map(K.whyHere), + bare: K.whyHere({...rows[2], reason: ""}), + spans: [0, 59, 60, 3599, 3600, 3660, 86399, 86400, 90000, 172800].map(K.span), + layers: [[], ["jobs"], ["line", "jobs"]].map(K.layers), + legend: K.QUEUE_LAYERS, + sig: [K.queueSig(null), K.queueSig([{...rows[0], waited_s: 119, eta_s: null}])], +})); +""" + + +def fake_queue(folder: Path, source: str, rows: object) -> Path: + """Args: + folder: an empty directory the fake owns. + source: the fake queue.py. + rows: what its `list --json` prints, JSON-encoded unless a string. + + Returns: + The fake queue.py; each run appends its arguments to queue.calls + and writes its environment to queue.env. + """ + folder.mkdir(parents=True, exist_ok=True) + script = folder / "queue.py" + script.write_text(source, encoding="utf-8") + (folder / "rows.json").write_text(rows if isinstance(rows, str) else json.dumps(rows), encoding="utf-8") + return script + + +def calls(script: Path) -> list[str]: + log = script.with_suffix(".calls") + return log.read_text(encoding="utf-8").splitlines() if log.exists() else [] + + +def board_council(now: float) -> dict: + """Returns: + A register and a queue whose grant order the hub ranks as HUB_ORDER + and the fixture of queue.py ranks as LINE_ORDER. + """ + entry = {"cmd": "make test", "granted": None, "hotfix": None, "status": "waiting"} + return { + "rasci.json": { + "work/alpha": {"title": "Alpha", "status": "queued", "worker": "Stage", "prio": 1}, + "work/bravo": {"title": "Bravo", "status": "running", "worker": "Docs", "prio": 2}, + "work/charlie": {"title": "Charlie", "status": "queued", "worker": "Stage", "estimates": {"Stage": {"value": 3, "complexity": 2}}}, + "work/delta": {"title": "Delta", "status": "queued", "worker": "Skills"}, + }, + "queue.json": {"seq": 4, "entries": [ + {**entry, "id": "q-1", "seq": 1, "kind": "test", "task": "work/bravo", "name": "Docs", "status": "active", "enqueued": now - 200, "granted": now - 100}, + {**entry, "id": "q-2", "seq": 2, "kind": "quality-high", "task": "work/alpha", "name": "Stage", "hotfix": "alpha", "enqueued": now - 1260}, + {**entry, "id": "q-3", "seq": 3, "kind": "test", "task": "work/charlie", "name": "Stage", "enqueued": now - 30}, + {**entry, "id": "q-4", "seq": 4, "kind": "deploy", "task": "work/delta", "name": "Skills", "enqueued": now - 100}, + ]}, + } + + +class TestQueueRows(unittest.TestCase): + def test_the_fixture_reads_back_field_for_field_in_grant_order(self): + self.assertEqual(queue_order.queue_jobs(FIXTURE.read_text(encoding="utf-8")), ROWS) + self.assertEqual([job["id"] for job in queue_order.queue_jobs(json.dumps(ROWS))], ["q-1", "q-2", "q-4", "q-3"]) + + def test_a_malformed_row_is_bounded_and_one_without_an_id_or_seen_before_is_dropped(self): + odd = { + "id": "a", "position": True, "kind": 7, "task": "t" * 501, "name": "Docs", "class": "urgent", "prio": True, + "wsjf": {"score": 3.5}, "waited_s": -1, "eta_s": "soon", "reason": "r" * 600, "trace": ["one", 5, None, *(f"s{n}" for n in range(30))], + } + found = queue_order.queue_jobs(json.dumps([{"id": ""}, "x", {"id": 5}, odd, {**odd, "kind": "again"}, {"id": "b", "position": "w" * 40, "wsjf": 2, "eta_s": 0}])) + self.assertEqual(found, [ + {"position": None, "id": "a", "kind": "", "task": None, "session": "", "agent": "Docs", "class": None, "prio": None, "wsjf": 3.5, + "waited_s": None, "eta_s": None, "reason": "r" * 500, "trace": ["one", *(f"s{n}" for n in range(17))]}, + {"position": "w" * 32, "id": "b", "kind": "", "task": None, "session": "", "agent": "", "class": None, "prio": None, "wsjf": 2.0, + "waited_s": None, "eta_s": 0.0, "reason": "", "trace": []}, + ]) + self.assertIsNone(queue_order.queue_jobs('[{"id": "n", "waited_s": NaN}]')[0]["waited_s"]) + + def test_rows_under_entries_and_one_row_per_line_read_and_other_output_reads_as_none(self): + self.assertEqual(queue_order.queue_jobs(json.dumps({"entries": ROWS})), ROWS) + self.assertEqual(queue_order.queue_jobs("\n".join(json.dumps(row) for row in ROWS) + "\n"), ROWS) + self.assertEqual(queue_order.queue_jobs("[]"), []) + self.assertEqual([job["id"] for job in queue_order.queue_jobs('{"id": "q-9"}')], ["q-9"]) + for output in ("", "position\tid\n1\tq-1\n", "5", '{"entries": 5}', '{"error": "no config"}', '{"id": "q-1"}\nnot json\n'): + with self.subTest(output=output): + self.assertIsNone(queue_order.queue_jobs(output)) + + def test_the_grant_order_follows_queue_py_and_keeps_the_hub_order_without_it(self): + chain = [("q-1", False), ("q-2", True), ("q-3", True), ("q-9", True), ("q-4", True)] + self.assertEqual(queue_order.grant_order(chain, ROWS), [("q-1", False), ("q-2", True), ("q-4", True), ("q-3", True), ("q-9", True)]) + self.assertIs(queue_order.grant_order(chain, None), chain) + + +class TestQueueOrderRuns(unittest.TestCase): + def setUp(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.base = Path(tmp.name) + + def order(self, source: str, rows: object) -> queue_order.QueueOrder: + return queue_order.QueueOrder(script=fake_queue(self.base / "skills", source, rows), council=self.base / "council", home=self.base / "home") + + def test_it_runs_once_per_window_again_when_the_live_jobs_change_and_never_while_none_is_live(self): + order = self.order(NEW_QUEUE, ROWS) + one, two = frozenset({"q-1"}), frozenset({"q-1", "q-2"}) + self.assertEqual(order.get(one, 1000.0), ROWS) + self.assertEqual(order.get(one, 1000.0 + queue_order.ORDER_SECONDS - 1), ROWS) + self.assertEqual(calls(order.script), ["list --json"]) + order.get(two, 1010.0) + order.get(two, 1010.0 + queue_order.ORDER_SECONDS) + self.assertEqual(order.get(frozenset(), 5000.0), []) + self.assertEqual(calls(order.script), ["list --json"] * 3) + + def test_a_request_meeting_a_running_read_gets_the_rows_read_before(self): + order = self.order(NEW_QUEUE, ROWS) + self.assertEqual(order.get(frozenset({"q-1"}), 1000.0), ROWS) + with order.lock: + self.assertEqual(order.get(frozenset({"q-1", "q-2"}), 2000.0), ROWS) + self.assertEqual(calls(order.script), ["list --json"]) + + def test_an_old_queue_py_a_failure_text_output_a_missing_script_and_an_overrun_read_as_none(self): + cases = {"old": (OLD_QUEUE, ROWS), "failing": (FAILING_QUEUE, ROWS), "text": (NEW_QUEUE, "position\tid\n1\tq-1\n"), "slow": (SLOW_QUEUE, ROWS)} + for name, (source, rows) in cases.items(): + with self.subTest(case=name), mock.patch.object(queue_order, "ORDER_TIMEOUT", 0.5): + order = queue_order.QueueOrder(script=fake_queue(self.base / name, source, rows), council=self.base, home=self.base) + started = time.monotonic() + self.assertIsNone(order.get(frozenset({"q-1"}), 1000.0)) + self.assertLess(time.monotonic() - started, 4) + missing = queue_order.QueueOrder(script=self.base / "gone" / "queue.py", council=self.base, home=self.base) + self.assertIsNone(missing.get(frozenset({"q-1"}), 1000.0)) + + def test_the_run_sees_the_council_and_the_state_home_and_nothing_of_the_hub_environment(self): + order = self.order(NEW_QUEUE, ROWS) + with mock.patch.dict(os.environ, {"KVB_HUB_SECRET": "x"}): + order.get(frozenset({"q-1"}), 1000.0) + seen = json.loads(order.script.with_suffix(".env").read_text(encoding="utf-8")) + self.assertEqual((seen["HOME"], seen["AGENTS_COUNCIL_ROOT"], seen["PATH"]), (str(self.base / "home"), str(self.base / "council"), "/usr/local/bin:/usr/bin:/bin")) + self.assertNotIn("KVB_HUB_SECRET", seen) + + +class TestHubQueueOverlay(HubCase): + def install(self, source: str) -> Path: + script = fake_queue(self.state.parent / "fake-skills", source, ROWS) + self.httpd.hub.queue_order.script = script + return script + + def queue_edges(self, state: dict) -> list[tuple[str, str]]: + return [(edge["from"], edge["to"]) for edge in state["task_edges"] if edge["kind"] == "queue"] + + def test_the_state_carries_the_queue_py_rows_and_orders_the_line_and_the_queue_edges_by_them(self): + self.write_council(board_council(time.time())) + script = self.install(NEW_QUEUE) + state = self.snapshot() + self.assertEqual(state["queue_jobs"], ROWS) + self.assertEqual(state["queue_line"], LINE_ORDER) + self.assertEqual(self.queue_edges(state), [("queue/q-1", "queue/q-2"), ("queue/q-2", "queue/q-4"), ("queue/q-4", "queue/q-3")]) + self.snapshot() + self.assertEqual(calls(script), ["list --json"]) + + def test_an_install_without_list_json_keeps_today_s_order_and_sends_no_rows(self): + self.write_council(board_council(time.time())) + self.install(OLD_QUEUE) + state = self.snapshot() + self.assertIsNone(state["queue_jobs"]) + self.assertEqual(state["queue_line"], HUB_ORDER) + self.assertEqual(self.queue_edges(state), [("queue/q-1", "queue/q-2"), ("queue/q-2", "queue/q-3"), ("queue/q-3", "queue/q-4")]) + + def test_a_queue_without_a_live_entry_runs_nothing(self): + council = board_council(time.time()) + self.write_council({**council, "queue.json": {"seq": 0, "entries": []}}) + script = self.install(NEW_QUEUE) + state = self.snapshot() + self.assertEqual((state["queue_jobs"], state["queue_line"], calls(script)), ([], [], [])) + + def test_the_page_embeds_the_layer_list_and_ships_the_overlay(self): + html = self.index() + self.assertIn('"kanban": {"hidden": []}', html) + self.assertIn(KANBAN.read_text(encoding="utf-8").strip(), html) + page = (HUB / "templates" / "page-kanban.js").read_text(encoding="utf-8") + for needle in ('LegendChips.render(Kanban.QUEUE_LAYERS', 'KANBAN_LAYERS, "hidden")', "Kanban.queued(rows, state.queue_line, state.queue_jobs)", "openPopup(found.label"): + with self.subTest(needle=needle): + self.assertIn(needle, page) + self.assertIn("Kanban.queueSig(state.queue_jobs)", (HUB / "templates" / "page-tasks.js").read_text(encoding="utf-8")) + + +class TestOverlayModel(unittest.TestCase): + @classmethod + def setUpClass(cls): + done = subprocess.run(["node", "-e", MODEL_PROBE, str(KANBAN), json.dumps(ROWS)], capture_output=True, text=True, timeout=60, check=True) + cls.data = json.loads(done.stdout) + + def test_each_card_on_the_line_gets_its_stop_and_its_jobs_and_a_queue_card_its_own_job(self): + self.assertEqual(self.data["marks"], { + "work/alpha": [2, 5, ["q-2"]], "work/bravo": [1, 5, ["q-1"]], "work/charlie": [4, 5, ["q-3"]], "work/delta": [3, 5, ["q-4"]], "queue/q-2": [None, 5, ["q-2"]], + }) + + def test_without_queue_py_rows_the_line_stays_numbered_and_no_card_shows_jobs(self): + self.assertEqual(self.data["degraded"], {"work/alpha": [2, 5, []], "work/bravo": [1, 5, []], "work/charlie": [4, 5, []], "work/delta": [3, 5, []]}) + + def test_the_chips_show_position_class_pin_wsjf_gate_wait_and_eta(self): + self.assertEqual(self.data["chips"], { + "q-1": [["position", "running", "neutral"], ["class", "fix", "warn"], ["prio", "P2", "neutral"], ["gate", "test", "teal"], ["waited", "waited 1 min", "neutral"], ["eta", "ETA 4 min", "neutral"]], + "q-2": [["position", "#1", "neutral"], ["class", "hotfix", "bad"], ["prio", "P1", "neutral"], ["wsjf", "WSJF 2.67", "neutral"], ["gate", "quality-high", "teal"], + ["waited", "waits 21 min", "neutral"], ["eta", "ETA 8 min", "neutral"]], + "q-4": [["position", "#2", "neutral"], ["class", "main", "violet"], ["gate", "deploy", "teal"], ["waited", "waits 1 h 6 min", "neutral"], ["eta", "ETA ?", "neutral"]], + "q-3": [["position", "#3", "neutral"], ["class", "feature", "info"], ["wsjf", "WSJF 1.5", "neutral"], ["gate", "test", "teal"], ["waited", "waits 30 s", "neutral"], ["eta", "ETA 1 h 5 min", "neutral"]], + }) + self.assertEqual(self.data["hints"], [ + "q-2: grant position #1", "q-2: class hotfix", "q-2: its task is pinned P1", "q-2: WSJF 2.6667 of its task", "q-2: runs the gate quality-high", + "q-2: waits 21 min in the queue", "q-2: starts in about 8 min", + ]) + + def test_why_here_lists_the_trace_in_order_and_falls_back_to_the_reason(self): + self.assertEqual(self.data["why"], [ + {"label": "Why here: q-1 at running", "place": "running", "steps": ["granted at its last poll", "slot test:1 of 2"]}, + {"label": "Why here: q-2 at #1", "place": "#1", "steps": ["hotfix first", "rank 0", "line slot:2 behind q-1", "RAM 61 % + 15 % reaches 75 %"]}, + {"label": "Why here: q-4 at #2", "place": "#2", "steps": ["waits behind q-2"]}, + {"label": "Why here: q-3 at #3", "place": "#3", "steps": ["rank 7", "waited 0 h", "line slot:2 behind q-2", "RAM 52 % + 15 GB fits"]}, + ]) + self.assertEqual(self.data["bare"]["steps"], ["queue.py gave no trace and no reason for this job"]) + + def test_spans_read_in_their_two_largest_units(self): + self.assertEqual(self.data["spans"], ["0 s", "59 s", "1 min", "59 min", "1 h", "1 h 1 min", "23 h 59 min", "1 d", "1 d 1 h", "2 d"]) + + def test_the_legend_switches_each_layer_and_the_redraw_signature_ignores_seconds(self): + self.assertEqual(self.data["legend"], [["line", "Queue line", "k-queue-line"], ["jobs", "Queue jobs", "k-queue-jobs"]]) + self.assertEqual(self.data["layers"], [{"line": True, "jobs": True}, {"line": True, "jobs": False}, {"line": False, "jobs": False}]) + self.assertEqual(self.data["sig"], [None, [{**ROWS[0], "waited_s": 1, "eta_s": None}]]) + + +class TestKanbanView(unittest.TestCase): + def test_the_layer_list_starts_empty_takes_each_known_layer_once_and_refuses_others(self): + self.assertEqual(VIEW["kanban"], {"hidden": []}) + doc = copy.deepcopy(VIEW) + doc["kanban"]["hidden"] = ["jobs", "line"] + self.assertEqual(valid_view(doc), doc) + for hidden in (["grid"], ["line", "line"], "line", [None]): + with self.subTest(hidden=hidden), self.assertRaises(ToolError): + valid_view({**doc, "kanban": {"hidden": hidden}}) + with self.assertRaises(ToolError): + valid_view({key: value for key, value in doc.items() if key != "kanban"}) + + def test_a_view_stored_by_an_older_hub_keeps_its_sections_and_gains_the_layers(self): + with tempfile.TemporaryDirectory() as folder: + stored = Path(folder) / "view.json" + old = {key: value for key, value in copy.deepcopy(VIEW).items() if key != "kanban"} + old["home"]["pins"] = [{"kind": "panel", "id": "resources-hardware", "w": 6, "h": 3}] + old["burn"]["overlays"] = ["fix"] + stored.write_text(json.dumps(old), encoding="utf-8") + self.assertEqual(ViewStore(stored).get(), {**old, "kanban": {"hidden": []}}) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_hub_state.py b/tests/test_hub_state.py index 743743a..3263333 100644 --- a/tests/test_hub_state.py +++ b/tests/test_hub_state.py @@ -151,7 +151,7 @@ class TestHubState(HubCase): set(state), { "now", "since", "until", "first", "alarm_since", "reviews", "asks", "decisions", "messages", "operator", - "register", "tasks", "task_edges", "queue_line", "agent_order", "radio", "claims", "grants", "night", "night_parts", "presence", "silence", "graph", + "register", "tasks", "task_edges", "queue_line", "queue_jobs", "agent_order", "radio", "claims", "grants", "night", "night_parts", "presence", "silence", "graph", "inbox", "pending", "rules", "approvers", "jobs", "views", "llm", "people", "settings", "media", "speech", "badges", "kinds", }, diff --git a/tests/test_hub_view.py b/tests/test_hub_view.py index 261b6bd..391f849 100644 --- a/tests/test_hub_view.py +++ b/tests/test_hub_view.py @@ -32,8 +32,9 @@ FULL = { "resources": {"hidden": ["host:CPU", "agent:Stage", "disk:/mnt_20_data"], "tokens": "accumulated", "combined": True, "combinedHidden": ["burn:scope"]}, "burn": {"overlays": ["fix", "score"]}, "home": {"pins": [{"kind": "panel", "id": "resources-hardware", "w": 12, "h": 4}, {"kind": "task", "id": "work/feature", "w": 1, "h": 12}]}, + "kanban": {"hidden": ["line"]}, } -PIN = {"kind": "panel", "id": "resources-hardware", "w": 6, "h": 3} +PIN ={"kind": "panel", "id": "resources-hardware", "w": 6, "h": 3} EMBEDDED = re.compile(r'') MAKE = """ const make = (tag) => { @@ -288,7 +289,7 @@ class TestViewDocument(unittest.TestCase): def test_the_defaults_and_a_full_document_pass_unchanged(self): self.assertEqual(valid_view(copy.deepcopy(VIEW)), VIEW) self.assertEqual(valid_view(copy.deepcopy(FULL)), FULL) - self.assertEqual(VIEW, {"resources": {"hidden": [], "tokens": "interval", "combined": False, "combinedHidden": []}, "burn": {"overlays": []}, "home": {"pins": []}}) + self.assertEqual(VIEW, {"resources": {"hidden": [], "tokens": "interval", "combined": False, "combinedHidden": []}, "burn": {"overlays": []}, "home": {"pins": []}, "kanban": {"hidden": []}}) def test_unknown_or_missing_keys_and_malformed_values_are_refused(self): def changed(section: str | None, key: str | None, value: object) -> dict: