diff --git a/skills/kvb-org-queue-cleanup/scripts/cleanup.py b/skills/kvb-org-queue-cleanup/scripts/cleanup.py index f8a9465..0e35250 100755 --- a/skills/kvb-org-queue-cleanup/scripts/cleanup.py +++ b/skills/kvb-org-queue-cleanup/scripts/cleanup.py @@ -348,7 +348,7 @@ def outcome(groups: dict[str, list[dict]], gone: list[dict]) -> dict[str, list[s def send(cls: str, to: list[str], text: str) -> None: - radio.transmit(cls, to, text, STATION, []) + radio.transmit(cls, to, text, STATION, [], True) def capped(ids: list[str]) -> str: diff --git a/skills/kvb-org-queue/SKILL.md b/skills/kvb-org-queue/SKILL.md index 4798809..457f15e 100644 --- a/skills/kvb-org-queue/SKILL.md +++ b/skills/kvb-org-queue/SKILL.md @@ -1,12 +1,12 @@ --- name: kvb-org-queue description: > - Queue every heavy job (gate, deploy, build, swarm), never ask the chair: - hotfix first, then rank. + Queue heavy jobs (gate, deploy, build, swarm) without the chair: hotfix, + then rank; FIFO per worktree. --- -`${CLAUDE_SKILL_DIR}/scripts/queue.py` replaces asking the chair for heavy -slots with one queue that every waiting agent evaluates by the same rules. +`${CLAUDE_SKILL_DIR}/scripts/queue.py` is one queue all waiting agents +evaluate alike, never the chair. ## Run a heavy job @@ -14,71 +14,76 @@ slots with one queue that every waiting agent evaluates by the same rules. queue.py hold --kind --task -- ``` -Start it in the background (`run_in_background` or equivalent) and end the -turn: hold runs the command on its grant and exits with its code. +Background it and end the turn; hold runs the command on grant, exiting with +its code. - ``: a key of `slot_need_gb` in `~/.claude/agents-council/config.json`. - ``: your open `kvb-org-rasci` task; its rank places you. - `--hotfix `: the job proves the register row `hotfix/` (`kvb-git-hotfix`) and goes first. -- `--need-gb ` overrides the learned disk need. +- `--need-gb ` overrides learned disk need. - Start hold in the job's worktree: its git toplevel is the workdir. Other runtimes add `--session `. - In a worktree the command stays `gate.py run|hold` (`kvb-check-gate`): `queue.py hold ... -- python3 run ...`. -One test file or module, or ruff, without Docker self-admits with -`watch.py admit --kind test-nostack` (`kvb-org-organize`); any other test -run goes through hold with kind `test`. +One test file or module, or ruff, without Docker self-admits +(`watch.py admit --kind test-nostack`, `kvb-org-organize`); other test runs +hold kind `test`. ## Order and grant -Every 20 s each waiting hold evaluates the queue under one lock: +Every 20 s each waiting hold evaluates the queue, locked: -1. Hotfix jobs first, then the register rank (`rasci.py show`), one place +1. Hotfixes first, then the register rank (`rasci.py show`), one place earlier per whole hour waited, then arrival; an open `after` task holds a - job. -2. In that order, a job is blocked at disk stage stop or critical, at warn - when it needs disk, at memory stage stop, during a radio-silence mayday, - while its kind's slots are taken, or while its workdir runs another job; - `exclusive_kinds` run only beside `light_kinds`. -3. A job whose disk need would cut below the stop threshold, or whose - footprint would push the running ones past `budget_percent`, reserves it - against the jobs behind it but `light_kinds`, which meet only running and - granted jobs, so a stream of light jobs can delay a heavy head; a job - alone always fits. + job. Main first needs Main in `rasci.py agents` (else no tiebreak); a job + aged an hour still passes Main's. +2. A job waits at disk stage stop or critical, at warn when it needs disk, + at memory stage stop, during a radio-silence mayday, while its kind's + slots are taken, or behind an earlier job of its worktree, compose stack + or `INFINITO_INSTANCE` slot; a job passes one that can never start, a + light job one awaiting only the machine, a hotfix all but hotfixes and + running jobs; `exclusive_kinds` run + only beside `light_kinds`. +3. A job fits if its disk need keeps the stop threshold free and its + footprint the running ones within `budget_percent` (alone always), and + passes blocked jobs, but the head (a blocked hotfix, an exclusive job + running jobs keep out, one stuck a minute or an hour old, or for other + lines a Main job whose line runs or awaits the machine) only taking + nothing it lacks, its last slot included, and only light while it is a + hotfix, exclusive or held by queue load. 4. An unblocked job counts as granted for the jobs behind it. 5. Every job, alone too, needs CPU, and RAM used plus what it and the - running jobs may still take, below `load_limits`. + running jobs may still take, below `load_limits`; outside load makes no + head and holds no line. +6. A stall escalates itself (`ESCALATION` in queue.py). ## History and learning -Every finished job appends a row to `queue-history.jsonl`, from which disk -need, slot limits and footprints are learned; read `learning.md` next to this -file before changing a seed or a limit. +Finished jobs teach disk need, slot limits and footprints via +`queue-history.jsonl`; read `learning.md` before changing a seed or limit. ## Inspect and cancel ```bash -queue.py list +queue.py list [--json] queue.py history queue.py cancel ``` -`list` shows each job's position, ETA and blocking reason, `history` the -learned values, `cancel` removes your own waiting job. +`list` shows position, ETA and blocking reason (`--json`: rule trace), `history` the learned values, `cancel` removes your waiting job. ## Rules - Never ask the chair for a heavy slot or announce a grant; the chair handles - only exceptions such as a stuck queue or a hung job (`capacity.md` of - `kvb-org-organize`). + only a stuck queue or hung job (`capacity.md` of `kvb-org-organize`). - A running job is never cut short for the queue; only the operator orders - one down, and `kvb-git-hotfix` may void a gate. A `kvb-org-queue-cleanup` - round deletes only entries whose lock no living process holds. + one down, and `kvb-git-hotfix` may void a gate. `kvb-org-queue-cleanup` + deletes only entries whose lock no living process holds. - With `queue_enforce: true`, the `block` hook (PreToolUse on Bash) refuses `heavy_commands`, image builds, `make install|verify-*`, tests beyond one file and `gate.py run|hold` outside `queue.py hold`. -- Without the config, hold exits 2: tell the operator and ask the chair - (`kvb-org-organize`) until it exists. A waiting hold rereads it every poll - (a broken file keeps the last good one), so a limit change needs no re-hold. +- Without the config, hold exits 2: tell the operator, ask the chair + (`kvb-org-organize`). A waiting hold rereads it every poll + (a broken file keeps the last good one). diff --git a/skills/kvb-org-queue/scripts/queue.py b/skills/kvb-org-queue/scripts/queue.py index 693c195..995ed23 100755 --- a/skills/kvb-org-queue/scripts/queue.py +++ b/skills/kvb-org-queue/scripts/queue.py @@ -4,7 +4,7 @@ Usage: queue.py hold --kind KIND --task TASK [--need-gb N] [--hotfix SLUG] [--session ID] [--receipt LEVEL --log PATH] -- CMD... - queue.py list + queue.py list [--json] queue.py history queue.py cancel ID [--session ID] queue.py block hook (PreToolUse on Bash) @@ -30,14 +30,22 @@ Environment: Args: hold: enqueue CMD, then every POLL_SECONDS evaluate the whole queue under - the lock until this entry is granted; measure disk, RAM and CPU, run + the lock until this entry is granted; while the head waits blocked, + that poll escalates over kvb-org-radio by ESCALATION, starts the + watch.py low-risk cleanup once per head when disk alone (its stage + or the floor) blocks it and nothing waiting fits, logs its report + once and posts to Main once when disk still blocks the head + (stalled()); its mayday sets no radio silence, and any silence + stops grants; a drain starts only after DEBOUNCE_SECONDS of polls + proposing it (latched()); measure disk, RAM and CPU, run CMD in the current directory with inherited stdio in a session of its own, forward SIGINT and SIGTERM to its process group, measure every POLL_SECONDS and at its end, record its history row and leave the queue. The entry carries its code (a hash of this file and the SIBLINGS); a poll that finds other code installed in ~/.agents/skills leaves the queue ungranted and restarts the same hold - with the installed queue.py. + with the installed queue.py, which keeps its entry id, place and + enqueue time (hand_over()). --kind: a key of slot_need_gb in the config. --task: an open task of the kvb-org-rasci register; its rank orders the queue, and the operator's agent order (`rasci.py agents @@ -59,10 +67,14 @@ Args: list: a header, then one line per entry: position (run for active jobs), id, status, kind, task, need GB, minutes waited, ETA minutes (until a running job ends or a waiting one starts, from learned medians; - while - one is unknown), workdir, blocking reason (the one the holder saw at - its last poll, host load included; prefixed when the entry runs other - code than the installed one). Entries of dead holders are dropped - first. + one is unknown), workdir, reason (the one the holder saw at its last + poll, host load included, such as `waits in line behind ` or + `drained for `, the head's after `escalated : ` once the + stall reached it; `backfilled` for a job started past a blocked one; + prefixed when the entry runs other code than the installed one). + Entries of dead holders are dropped first. + --json: print the same rows as a JSON list (listed()), each with + the trace of the evaluate() pass that set its reason. history: a markdown table per command signature and one per kind of what the last 30 days of history teach, with each slot limit's source. cancel ID: remove the caller's waiting entry ID; its hold exits 3. @@ -82,10 +94,8 @@ yet) may sum to at most these percentage points, a job alone always fits; slot_limits {kind: N}, optional, caps the learned slot limit of a kind from above (history.slot_limits); exclusive_kinds [kind] and light_kinds [kind], optional: a job of an exclusive kind runs only beside jobs of light kinds, and -no job of another kind starts beside it; a light job meets only the budget and -disk of running and granted jobs, never a reservation; queue_enforce (bool), optional, read by block -only. The watch.py keys disk, thresholds_gb (stop is the floor every grant -leaves free), slot_need_gb (the disk seed per kind), load_limits (every grant +no job of another kind starts beside it; queue_enforce (bool), optional, read by block only. The watch.py keys +disk, thresholds_gb (stop is the floor every grant leaves free), slot_need_gb (the disk seed per kind), load_limits (every grant also needs CPU busy below cpu_percent and RAM used plus the job's learned RAM share below mem_percent; both bound the learned slot limits), memory_limits (no grant at memory stage stop), heavy_commands and never_block are used as @@ -122,11 +132,12 @@ import os import re import shlex import signal +import stat import subprocess import sys import time from collections import Counter -from collections.abc import Iterator +from collections.abc import Callable, Iterator from pathlib import Path from types import ModuleType from typing import NoReturn @@ -165,6 +176,36 @@ XDIST_LONG = "--numprocesses=" ADDOPTS = "PYTEST_ADDOPTS=" FROZEN = "footprint" ALIASES = "aliases" +CLAIMS = "claims" +DRAIN = "drain" +GRANTED = "granted_at" +STALL = "stall" +CLEANUP = "cleanup" +HANDOVER = "handover" +STACK = "stack:" +COMPOSE_FILES = ("compose.yaml", "compose.yml", "docker-compose.yaml", "docker-compose.yml") +PROJECT_ENV = re.compile(r"^COMPOSE_PROJECT_NAME=(.*)$", re.MULTILINE) +COMPOSE_NAME = re.compile(r"^name:[ \t]*([^#\n]*)", re.MULTILINE) +PROJECT_CHARS = re.compile(r"[a-z0-9_-]") +SLOT = "slot:" +SLOT_ENV = re.compile(r"^INFINITO_INSTANCE=(.*)$", re.MULTILINE) +SLOT_FILES = ("custom.env", ".env", "default.env") +BLOCKED = "blocked_since" +DEBOUNCE_SECONDS = 3 * POLL_SECONDS +DRAIN_AFTER_SECONDS = HOUR +IDLE_SECONDS = 30 * 60 +RADIO = SKILLS / "kvb-org-radio" / "scripts" / "radio.py" +RADIO_SECONDS = 30 +PREFIX = "kvb-org-queue: " +TEXT = "" +ESCALATION = ( + (30, "post", ("post", TEXT, "--to", "Main")), + (60, "call", ("send", "call", TEXT, "--to", "Main")), + (120, "security", ("send", "security", TEXT)), + (240, "pan-pan", ("send", "pan-pan", TEXT)), + (24 * 60, "mayday", ("send", "mayday", TEXT, "--no-silence")), +) +STEPS = {step: (minutes, argv) for minutes, step, argv in ESCALATION} @functools.cache @@ -200,6 +241,14 @@ class Stale(Exception): pass +class Disk(str): + pass + + +class Outside(str): + pass + + def fail(message: str) -> NoReturn: print(f"queue: {message}", file=sys.stderr) sys.exit(2) @@ -377,6 +426,57 @@ def key_of(entry: dict) -> tuple[str, tuple[str, ...]]: return entry["kind"], aliases_of(entry) +def claims_of(path: str) -> list[str]: + """Returns: + What a job whose workdir is path holds alone: path, then + `stack:` when path holds a compose file, the project being + COMPOSE_PROJECT_NAME of path/.env, else the top-level `name:` of the + first of COMPOSE_FILES there, else the name of path normalized as + docker compose normalizes it, then `slot:` for the + INFINITO_INSTANCE of the first of SLOT_FILES below path that pins + one, so custom.env wins and an unpinned core checkout holds the slot + of its default.env. + """ + slots = slot_of(path) + files = tuple(name for name in COMPOSE_FILES if os.path.isfile(os.path.join(path, name))) + if not files: + return [path, *slots] + named = [value for value in pinned(path, PROJECT_ENV, (".env",)) if value][-1:] + named = named or [value for value in pinned(path, COMPOSE_NAME, files[:1]) if value][:1] + project = named[-1] if named else "".join(PROJECT_CHARS.findall(os.path.basename(path).lower())) + return [path, f"{STACK}{project.lstrip('_-')}", *slots] + + +def slot_of(path: str) -> list[str]: + for name in SLOT_FILES: + found = [value for value in pinned(path, SLOT_ENV, (name,)) if value] + if found: + return [f"{SLOT}{found[-1]}"] + return [] + + +def pinned(path: str, key: re.Pattern, names: tuple[str, ...]) -> list[str]: + """Returns: + The values the env files names below path give the key, unquoted, + in file order; an unreadable file gives none. + """ + found: list[str] = [] + for name in names: + try: + found += [value.strip().strip("\"'") for value in key.findall(Path(path, name).read_text(encoding="utf-8"))] + except (OSError, UnicodeDecodeError): + continue + return found + + +def claims(entry: dict) -> list[str]: + """Returns: + The claims stored at enqueue (claims_of(); the workdir alone for an + entry an older queue stored without them). + """ + return entry[CLAIMS] if CLAIMS in entry else [entry["workdir"]] + + def identity(pid: int) -> str | None: """Name a live process by its start time, so a reused pid does not match. @@ -464,14 +564,50 @@ def store() -> Iterator[dict]: nofollow.replace(watch.ROOT, STORE, json.dumps(data, indent=1, sort_keys=True), 0o600) -def ordered(waiting: list[dict], ranks: dict[str, int], agents: dict[str, int], now: float) -> list[dict]: - def key(entry: dict) -> tuple: - hours = int((now - entry["enqueued"]) // HOUR) - rank = ranks[entry["task"]] if entry["task"] in ranks else len(ranks) - agent = agents[entry["name"]] if entry["name"] in agents else len(agents) - return (not entry["hotfix"], rank - hours, agent, entry["seq"]) +def priority(entry: dict, ranks: dict[str, int], agents: dict[str, int], now: float) -> tuple[tuple, list[str]]: + """Returns: + The sort key of entry in ordered() (hotfix first, register rank less + the whole hours waited, agent order, arrival) and the trace steps + that name it. + """ + hours = int((now - entry["enqueued"]) // HOUR) + rank = ranks[entry["task"]] if entry["task"] in ranks else len(ranks) + agent = agents[entry["name"]] if entry["name"] in agents else len(agents) + steps = [ + *(["hotfix first"] if entry["hotfix"] else []), + (f"rank {rank}" if entry["task"] in ranks else "unranked") + (f", {hours} h waited: {rank - hours}" if hours else ""), + *([f"agent order {agent}"] if entry["name"] in agents else []), + f"arrival {entry['seq']}", + ] + return (not entry["hotfix"], rank - hours, agent, entry["seq"]), steps - return sorted(waiting, key=key) + +def ordered(waiting: list[dict], ranks: dict[str, int], agents: dict[str, int], now: float) -> list[dict]: + return sorted(waiting, key=lambda entry: priority(entry, ranks, agents, now)[0]) + + +def in_lines(order: list[dict]) -> list[dict]: + """Returns: + order with every entry moved behind each entry of order enqueued + before it that shares one of its claims (claims()), so a line runs + by enqueue time alone, whatever the rank; a hotfix moves behind + older hotfixes only. + """ + held = {e["id"]: set(claims(e)) for e in order} + placed: dict[str, dict] = {} + + def place(entry: dict) -> None: + if entry["id"] in placed: + return + for older in order: + earlier = (older["enqueued"], older["seq"]) < (entry["enqueued"], entry["seq"]) + if earlier and held[older["id"]] & held[entry["id"]] and (older["hotfix"] or not entry["hotfix"]): + place(older) + placed[entry["id"]] = entry + + for entry in order: + place(entry) + return list(placed.values()) def over_budget(used: dict[str, float], share: dict[str, float], budget: dict[str, float]) -> str | None: @@ -481,13 +617,14 @@ def over_budget(used: dict[str, float], share: dict[str, float], budget: dict[st return ", ".join( f"{resource.upper()} {used[resource]:.0f} % in use or reserved + {share[resource]:.0f} % > budget {budget[resource]:g} %" for resource in over - ) + "; reserved" + ) def evaluate(entries: list[dict], free_gb: float, floor: float, stage: str, memory: str, silenced: bool, slot_limits: dict[str, int], footprints: dict[str, dict[str, float]], budget: dict[str, float], ranks: dict[str, int], agents: dict[str, int], loaded: dict[str, str], - exclusive: set[str], light: set[str], now: float) -> dict[str, str | None]: + exclusive: set[str], light: set[str], drain: str | None, + now: float, *, trace: dict[str, list[str]] | None = None) -> tuple[dict[str, str | None], str | None, list[dict]]: """Decide every waiting entry in grant order, without disk or processes. Args: @@ -509,71 +646,202 @@ def evaluate(entries: list[dict], free_gb: float, floor: float, stage: str, memo all of them. loaded: {entry id: why live CPU or RAM holds it back (load_reason())} for the waiting entries the host load keeps out. - exclusive: kinds that run only beside light kinds (exclusive_kinds). + exclusive: kinds that run only beside light kinds (exclusive_kinds, + alone_reason()); they wait in grant order, in no line of their own. light: kinds allowed beside an exclusive one (light_kinds). + drain: the entry the previous pass drained for (DRAIN), else None. now: epoch seconds; each whole hour waited moves an entry one rank up. + trace: when given, filled with {waiting entry id: the steps that + decided it, in order}: its priority() steps, `drain head`, + `passes : it waits for the machine` (or ` can never start, + `) per entry it passes, then the + rule that blocked it, or `takes nothing lacks`, the fit and + `backfilled past ` for a granted one. Returns: - {entry id: None when granted, else the blocking reason}, in grant order. - An unblocked entry counts as granted for the entries after it, whoever - holds it, so a lower entry never takes a higher entry's slot; one - blocked by disk or budget reserves its need against the entries after it. - A light entry meets only the disk, RAM and CPU of running and granted - entries: reservations, a head the host load keeps out included, hold - back heavy kinds alone, so light jobs that keep coming can delay a heavy - head without bound. A hotfix entry that passes every check but the host - load holds back every entry after it, so lighter jobs cannot keep the - load up. While no queue job is active, any other entry the host load - alone keeps out reserves nothing and holds nothing back: that load lies - outside the queue, so holding the queue frees none of it. An exclusive - entry that only running jobs of other kinds keep out holds back every - entry after it but light ones, so none overtakes it. A waiting entry - whose holder has not polled for CLAIM_SECONDS reserves nothing and comes - last. + ({entry id: None when granted, else the blocking reason, a Disk when + the disk stage or the floor alone blocks it} in grant order, the entry + the next pass drains for (drained()), the polling waiting entries in + grant order). A granted entry counts against the disk, budget and + slots of the entries after it, whoever holds it. A blocked entry + reserves nothing, so a later entry that fits starts past it + (backfill), except the head: drain, or before it a blocked hotfix + that is the first blocked entry not unstartable() or an exclusive entry that only + running jobs of other kinds keep out (alone_reason()). Behind the + head every later entry, light ones too, starts only when it takes + nothing the head still lacks, the last slot of its kind included + (takes(); beside a head that is + exclusive or waits for an exclusive job only light jobs count); every + later one but light ones waits behind a + hotfix head, and behind any head whose kind is exclusive or that live + load holds while queue jobs run; light ones wait too while load the + queue causes holds a hotfix head. An entry never passes an active, + granted or waiting entry enqueued before it that shares one of its + claims (claims(): worktree, compose stack, slot): those form its + line, which runs by enqueue time alone (in_lines()), never by rank, + except that a hotfix waits behind no older entry but a hotfix or an + active one, and a light entry passes a waiting one that is no hotfix + and waits only for the machine (its slot, alone_reason() or a hold + behind an entry of other claims), which then waits for that light run; + a head keeps its slot, so that delays it by one light run at most. A + granted entry holds its claims for the rest of the pass. A waiting + entry of the session Main that could start, whose line only runs, + waits for the machine or is the head, reserves like a head against the + later entries of other claims: its slot, its exclusive room and its + budget (held_for()). + An entry that could not start with no queue job running (its disk + need cuts into floor, its kind has no slot, or load holds it that no + queue job causes: an Outside reason, or no queue job active) is no + head; one with an Outside reason holds no line either. A waiting + entry whose holder has not polled for CLAIM_SECONDS reserves nothing + and comes last. An entry passes a waiting entry of its line that never + could start (unstartable()) and takes its place in that line. """ active = [e for e in entries if e["status"] == "active"] - available = free_gb - sum(e["need_gb"] for e in active) used = {resource: sum(footprints[e["id"]][resource] for e in active) for resource in budget} - taken = len(active) - granted = used, available, taken + available = free_gb - sum(e["need_gb"] for e in active) + lit = [e for e in active if e["kind"] in light] + beside = {resource: sum(footprints[e["id"]][resource] for e in lit) for resource in budget} + room = free_gb - sum(e["need_gb"] for e in lit) counts = Counter(e["kind"] for e in active) - busy = {e["workdir"] for e in active} + lines = {claim: e["id"] for e in active for claim in claims(e)} + polling = [e for e in entries if e["status"] == "waiting" and not silent(e, now)] + order = in_lines(ordered(polling, ranks, agents, now)) + never = {e["id"]: unstartable(e, free_gb, floor, slot_limits) for e in order} + could = { + e["id"] for e in order if not never[e["id"]] + and not (e["id"] in loaded and (not active or isinstance(loaded[e["id"]], Outside))) + } + drain = drain if drain in could else None + steps = {} if trace is None else trace verdicts: dict[str, str | None] = {} - waiting = [e for e in entries if e["status"] == "waiting"] - held = drain = None - for entry in ordered([e for e in waiting if not silent(e, now)], ranks, agents, now): - if held is not None: - verdicts[entry["id"]] = f"waits behind hotfix {held}, which the host load holds back" - continue - kind, need, workdir, share = entry["kind"], entry["need_gb"], entry["workdir"], footprints[entry["id"]] - if drain is not None and kind not in light: - verdicts[entry["id"]] = f"waits behind {drain}, which needs the host beside light kinds only" - continue - reason = host_reason(stage, memory, silenced, need) or slot_reason(kind, workdir, slot_limits, counts, busy) - if reason is None: - reason = alone_reason(kind, counts, exclusive, light) - if reason is not None and kind in exclusive: - drain = entry["id"] - if reason is None: - pool_used, pool_available, pool_taken = granted if kind in light else (used, available, taken) - reason, took, disk = resources(need, share, pool_used, pool_available, floor, budget, pool_taken) - if reason is None and entry["id"] in loaded and (entry["hotfix"] or not active): - reason = loaded[entry["id"]] - if entry["hotfix"]: - held = entry["id"] + blocked: list[dict] = [] + able: list[dict] = [] + reserve: tuple[dict[str, float], float, bool, str, str, str] | None = None + machine: set[str] = set() + holds: list[tuple[dict[str, float], float, str, str, set[str]]] = [] + running = {e["id"] for e in active} + for entry in order: + kind, need, share, mine = entry["kind"], entry["need_gb"], footprints[entry["id"]], claims(entry) + taken = sum(counts.values()) + lifted = [c for c in mine if never.get(lines.get(c))] + passed = list(dict.fromkeys(lines[c] for c in mine if c in lifted or kind in light and lines.get(c) in machine)) + kept = [c for c in mine if lines.get(c) not in passed] + owners = {lines[c] for c in kept if c in lines} + lined = bool(owners) + lead = behind(entry, kept, lines, able, reserve[4] if reserve else drain, exclusive, light, loaded) + alone, slot = alone_reason(kind, counts, exclusive, light), slot_reason(kind, slot_limits, counts) + reason = blocking( + lead and lead[0], disk_reason(stage, need), + host_reason(memory, silenced) or slot or alone or (over_budget(used, share, budget) if taken else None), + None if available - need >= floor else floor_reason(need, floor), loaded.get(entry["id"]), + ) + step = lead[1] if lead else reason + if reason is None and reserve is not None: + slotted = reserve[5] if kind == reserve[5] and counts[kind] + 1 >= slot_limits.get(kind, 0) else None + short = takes(need, share, *((beside, room) if reserve[2] else (used, available)), floor, budget, reserve[:2], slotted) + reason = reserve[3] if short else None + step = f"{reserve[3]}: takes {short}" if short else f"takes nothing {reserve[4]} lacks" + pools = used, available, beside, room + why = None if reason is not None else next(((h[2], s) for h in holds if not h[4] & set(mine) and ( + s := held_for(kind, need, share, h, counts, slot_limits, exclusive, light, pools, floor, budget))), None) + if why: + reason, step = f"reserved for {why[0]}", f"reserved for {why[0]}: {why[1]}" + for claim in () if isinstance(loaded.get(entry["id"]), Outside) else mine: + if reason is None or claim in lifted: + lines[claim] = entry["id"] else: - used, available = plus(used, took), available - disk - if reason is None: - taken += 1 - counts[kind] += 1 - busy.add(workdir) - if entry["id"] not in loaded: - granted = plus(granted[0], took), granted[1] - disk, granted[2] + 1 + lines.setdefault(claim, entry["id"]) + mark = [ + *priority(entry, ranks, agents, now)[1], *(["drain head"] if entry["id"] == drain else []), + *(f"passes {o}: {o} can never start, {never[o]}" if never.get(o) else f"passes {o}: it waits for the machine" for o in passed), + ] + if reason is None: + steps[entry["id"]] = [*mark, *fitted(step, used, share, need, available, blocked)] + used, available = plus(used, share), available - need + if kind in light: + beside, room = plus(beside, share), room - need + counts[kind] += 1 + else: + steps[entry["id"]] = [*mark, str(step)] + blocked.append(entry) + able += [] if never[entry["id"]] else [entry] + first = entry["hotfix"] and able == [entry] or kind in exclusive and reason == alone + if reserve is None and (entry["id"] == drain or entry["id"] in could and first): + held = f"waits behind hotfix {entry['id']}" if entry["hotfix"] else f"drained for {entry['id']}" + reserve = share, need, kind in exclusive or reason == alone, held, entry["id"], kind + if not entry["hotfix"] and not lined and (lead or reason in (alone, slot)): + machine.add(entry["id"]) + settled = running | machine | {i for i, v in verdicts.items() if v is None} | {reserve[4] if reserve else None} + if lined and entry["name"] == "Main" and entry["id"] in could and entry["id"] != (reserve and reserve[4]) and owners <= settled: + holds.append((share, need, entry["id"], kind, set(mine))) verdicts[entry["id"]] = reason - for entry in waiting: - if silent(entry, now): + for entry in entries: + if entry["status"] == "waiting" and silent(entry, now): verdicts[entry["id"]] = f"no poll for {now - entry['polled']:.0f} s; reserves nothing until it polls again" - return verdicts + steps[entry["id"]] = [verdicts[entry["id"]]] + return verdicts, drained(order, verdicts, drain, could, now), order + + +def fitted(step: str | None, used: dict[str, float], share: dict[str, float], need: float, available: float, + blocked: list[dict]) -> list[str]: + """Returns: + The trace steps of a granted entry: step when a head reserves, its + fit beside used and available, and the blocked entry it passes. + """ + fit = ", ".join(f"{r.upper()} {used[r]:.0f} % + {share[r]:.0f} %" for r in used) + f", {need:g} GB of {available:.0f} GB fits" + return [*([step] if step else []), fit, *([f"backfilled past {blocked[0]['id']}"] if blocked else [])] + + +def blocking(lead: str | None, staged: str | None, rest: str | None, floored: str | None, + late: str | None) -> str | None: + """Returns: + The first reason of lead, staged, rest, floored and late, a Disk + when only staged or floored, the disk checks, hold the entry back. + """ + reason = lead or staged or rest or floored or late + return Disk(reason) if reason is not None and lead is None and rest is None and late is None else reason + + +def behind(entry: dict, mine: list[str], lines: dict[str, str], blocked: list[dict], drain: str | None, + exclusive: set[str], light: set[str], loaded: dict[str, str]) -> tuple[str, str] | None: + """Returns: + (why an entry before it holds entry back, its trace step), else None: + the first of blocked (the blocked entries not unstartable()) is a + hotfix and entry is not light, or load + the queue causes holds that hotfix (loaded, no Outside reason); an + entry enqueued before it shares one of its claims mine (lines maps a + claim to the first entry holding it); or drain, the entry that + reserves (evaluate()), is blocked before it, entry is not light, and + live load holds drain or its kind is exclusive. + """ + first = blocked[0] if blocked else None + if first and first["hotfix"]: + pressed = first["id"] in loaded and not isinstance(loaded[first["id"]], Outside) + if entry["kind"] not in light or pressed: + return f"waits behind hotfix {first['id']}", f"hotfix {first['id']} blocked ahead" + (": load holds it" if pressed else "") + ahead = [(claim, lines[claim]) for claim in mine if claim in lines] + if ahead: + return f"waits in line behind {ahead[0][1]}", f"line {ahead[0][0]} behind {ahead[0][1]}" + for e in blocked: + if entry["kind"] not in light and e["id"] == drain and (drain in loaded or e["kind"] in exclusive): + return f"drained for {drain}", f"drained for {drain}: " + ("load holds it" if drain in loaded else f"{e['kind']} is exclusive") + return None + + +def drained(order: list[dict], verdicts: dict[str, str | None], drain: str | None, could: set[str], + now: float) -> str | None: + """Returns: + The entry the next pass drains for, among could, the entries of order + that could start with no queue job running: drain while it still + waits, else the first blocked one that waited DRAIN_AFTER_SECONDS, + else the first of order once none of order fits; None when there is + none. + """ + blocked = [e["id"] for e in order if verdicts[e["id"]] is not None] + aged = [e["id"] for e in order if e["id"] in blocked and now - e["enqueued"] >= DRAIN_AFTER_SECONDS] + stuck = [order[0]["id"]] if order and len(blocked) == len(order) else [] + return next((e for e in [drain, *aged, *stuck] if e in could), None) def silent(entry: dict, now: float) -> bool: @@ -591,25 +859,72 @@ def plus(used: dict[str, float], share: dict[str, float]) -> dict[str, float]: return {resource: used[resource] + share[resource] for resource in used} -def resources(need: float, share: dict[str, float], used: dict[str, float], available: float, floor: float, - budget: dict[str, float], taken: int) -> tuple[str | None, dict[str, float], float]: +def takes(need: float, share: dict[str, float], used: dict[str, float], available: float, floor: float, + budget: dict[str, float], reserve: tuple[dict[str, float], float], slotted: str | None) -> str | None: """Returns: - (reason or None, RAM and CPU share, disk GB) the job takes or - reserves: one passing the budget beside other jobs reserves its share, - one cutting into the floor its disk, a granted one takes both. + What a job of disk need and footprint share, beside jobs that use + used and leave available GB, takes that the head's reserve + (footprint, disk need) leaves too short: `its slot` when + slotted names the head's kind whose last free slot it takes, the + budget resources it pushes past, or disk; None when it takes none of + a short resource. """ - if taken and (reason := over_budget(used, share, budget)) is not None: - return reason, share, 0 - if available - need < floor: - return f"{need:g} GB would cut into the {floor:g} GB floor; reserved", dict.fromkeys(share, 0), need - return None, share, need + held, held_need = reserve + over = plus(plus(used, held), share) + short = [r.upper() for r in budget if share[r] > 0 and over[r] > budget[r]] + if slotted: + return f"its {slotted} slot" + if short: + return " and ".join(short) + return "disk" if need > 0 and available - held_need - need < floor else None -def host_reason(stage: str, memory: str, silenced: bool, need: float) -> str | None: +def held_for(kind: str, need: float, share: dict[str, float], hold: tuple[dict[str, float], float, str, str, set[str]], + counts: Counter, slot_limits: dict[str, int], exclusive: set[str], light: set[str], + pools: tuple[dict[str, float], float, dict[str, float], float], floor: float, budget: dict[str, float]) -> str | None: + """Returns: + Why a job of kind, disk need and footprint share takes what hold + (footprint, disk need, id, kind, claims), a job whose line only runs + or waits for the machine, lacks: ` is exclusive` when one of + the two is exclusive and the other not light, else `takes` what + takes() finds beside pools (used, available, then the light jobs' + beside and room, which count when hold's kind is exclusive), its last + free slot of hold's kind included; None when it takes nothing. + """ + held = hold[3] + if held in exclusive and kind not in light or kind in exclusive and held not in light: + return f"{held if held in exclusive else kind} is exclusive" + used, available, beside, room = pools + slotted = held if kind == held and counts[kind] + 1 >= slot_limits.get(kind, 0) else None + short = takes(need, share, *((beside, room) if held in exclusive else (used, available)), floor, budget, hold[:2], slotted) + return f"takes {short}" if short else None + + +def unstartable(entry: dict, free_gb: float, floor: float, slot_limits: dict[str, int]) -> str | None: + """Returns: + Why entry could not start even with no queue job running: ` + slots` when its kind's cap is 0 or missing, floor_reason() + when its disk need cuts into floor of free_gb; else None. + """ + cap = slot_limits.get(entry["kind"], 0) + if cap <= 0: + return f"{cap} {entry['kind']} slots" + return floor_reason(entry["need_gb"], floor) if free_gb - entry["need_gb"] < floor else None + + +def floor_reason(need: float, floor: float) -> str: + return f"{need:g} GB would cut into the {floor:g} GB floor" + + +def disk_reason(stage: str, need: float) -> str | None: if stage in ("stop", "critical"): return f"disk stage {stage}" if stage == "warn" and need > 0: return "disk stage warn admits only jobs that need no disk" + return None + + +def host_reason(memory: str, silenced: bool) -> str | None: if memory == "stop": return "memory stage stop" if silenced: @@ -617,13 +932,11 @@ def host_reason(stage: str, memory: str, silenced: bool, need: float) -> str | N return None -def slot_reason(kind: str, workdir: str, slot_limits: dict[str, int], counts: Counter, busy: set[str]) -> str | None: +def slot_reason(kind: str, slot_limits: dict[str, int], counts: Counter) -> str | None: if kind not in slot_limits: return f"slot_limits has no {kind}" if counts[kind] >= slot_limits[kind]: return f"all {slot_limits[kind]} {kind} slots taken" - if workdir in busy: - return f"{workdir} runs another job" return None @@ -691,17 +1004,28 @@ def footprint(entry: dict, signatures: dict[tuple[str, str], dict], kinds: dict[ def held_back(cfg: dict, entries: list[dict], now: float, signatures: dict[str, dict], kinds: dict[str, dict], cpu: float, mem: float) -> dict[str, str]: + """Returns: + {waiting entry id: load_reason()} for the waiting entries the host + load keeps out, an Outside reason when the load would keep the entry + out alone with every queue job gone: CPU less the active jobs' CPU + shares, RAM less the ramped ones' RAM shares. + """ + active = [e for e in entries if e["status"] == "active"] + ramped = sum(ram_share(e, signatures, kinds) for e in active if now - e.get("granted", now) >= RAMP_SECONDS) + busy = sum((footprint(e, signatures, kinds) or {"cpu": 0})["cpu"] for e in active) loaded = {} for entry in entries: if entry["status"] == "waiting": pending = pending_ram(entries, entry["id"], now, signatures, kinds) if (reason := load_reason(cfg["load_limits"], cpu, mem, pending)) is not None: - loaded[entry["id"]] = reason + alone = load_reason(cfg["load_limits"], cpu - busy, mem - ramped, ram_share(entry, signatures, kinds)) + loaded[entry["id"]] = reason if alone is None else Outside(reason) return loaded def decide(cfg: dict, entries: list[dict], free_gb: float, now: float, signatures: dict[str, dict], - kinds: dict[str, dict], loaded: dict[str, str]) -> dict[str, str | None]: + kinds: dict[str, dict], loaded: dict[str, str], drain: str | None, *, + trace: dict[str, list[str]] | None = None) -> tuple[dict[str, str | None], str | None, list[dict]]: slots = {kind: limit for kind, (limit, _) in limits(cfg, kinds, entries).items()} budget = cfg["budget_percent"] footprints = {} @@ -709,12 +1033,207 @@ def decide(cfg: dict, entries: list[dict], free_gb: float, now: float, signature share = footprint(entry, signatures, kinds) footprints[entry["id"]] = dict(budget) if share is None else share waits = after_waits(entries) - return evaluate( + steps = {} if trace is None else trace + verdicts, drain, order = evaluate( [e for e in entries if e["id"] not in waits], free_gb, cfg["thresholds_gb"]["stop"], stage(cfg, free_gb, now), watch.memory_stage(watch.memory(), cfg), silenced(now), slots, footprints, budget, register_ranks(), agent_ranks(), loaded, - set(cfg.get("exclusive_kinds", [])), set(cfg.get("light_kinds", [])), now, - ) | waits + set(cfg.get("exclusive_kinds", [])), set(cfg.get("light_kinds", [])), drain, now, trace=steps, + ) + steps.update({entry_id: [reason] for entry_id, reason in waits.items()}) + return verdicts | waits, drain, order + + +def latched(data: dict, drain: str | None, now: float) -> str | None: + """Returns: + drain when the store data drains for it already, or once a new drain + was proposed at every poll for DEBOUNCE_SECONDS (BLOCKED keeps since + when), else None; a pass proposing none clears BLOCKED. + """ + if drain is None or drain == data.get(DRAIN): + data.pop(BLOCKED, None) + return drain + data.setdefault(BLOCKED, now) + return drain if now - data[BLOCKED] >= DEBOUNCE_SECONDS else None + + +def labelled(verdicts: dict[str, str | None], fired: list[str]) -> dict[str, str | None]: + """Returns: + verdicts with `backfilled` for a granted entry after a blocked one, + and `escalated : ` before the first entry's reason for the last + step fired in the stall. + """ + shown: dict[str, str | None] = {} + for entry_id, reason in verdicts.items(): + if reason is None: + shown[entry_id] = "backfilled" if any(shown.values()) else None + else: + shown[entry_id] = f"escalated {fired[-1]}: {reason}" if fired and not shown else reason + return shown + + +def escalate(stall: dict, grant: float | None, since: float, hotfix: bool, + now: float) -> tuple[dict, str | None]: + """Args: + stall: the stored STALL, {} for none; one stored for another grant, + or last advanced more than IDLE_SECONDS before since, counts as + none, so a grant or an idle queue resets it, and nothing else: a + restart, rejoin, cancel or new hold keeps its clock. + grant: epoch seconds of the last grant on the host, None for none. + since: epoch seconds the oldest polling waiting entry was enqueued; + a new stall's clock starts at the later of grant and since. + hotfix: whether a hotfix job runs ahead of the head; the clock + stands still while one does. + now: epoch seconds. + + Returns: + (the stall to store, the ESCALATION step to fire now or None). Each + step fires once per stall; steps the clock passed unseen are marked + fired with the last of them, which alone fires. + """ + fresh = {"grant": grant, "since": since if grant is None else max(grant, since), "seen": now, "fired": []} + if stall.get("grant") != grant or not fresh.keys() <= stall.keys() or since - stall["seen"] > IDLE_SECONDS: + stall = fresh + start = now if hotfix else stall["since"] + due = [step for minutes, step, _ in ESCALATION if now - start >= minutes * 60 and step not in stall["fired"]] + return {**stall, "since": start, "seen": now, "fired": [*stall["fired"], *due]}, due[-1] if due else None + + +def cleanup_due(state: dict, head: str, disk: bool, now: float) -> tuple[dict, bool]: + """Returns: + (the CLEANUP to store: the stored one while head stays the head, + whatever grants pass beside it, else a new one for head; whether to + start the low-risk cleanup now, once per head, when disk alone + blocks it). + """ + if state.get("head") != head: + state = {"head": head, "cleaned": False, "cleaned_at": None, "settled": False, "noticed": False} + if disk and not state["cleaned"]: + return {**state, "cleaned": True, "cleaned_at": now}, True + return state, False + + +def settle(stall: dict, outcome: Callable[[], tuple[bool, list[tuple[float, str]]]], + short: bool) -> tuple[dict, str | None, bool]: + """Args: + stall: the CLEANUP cleanup_due() returned. + outcome: () -> (whether a cleanup holds the watch.py cleanup lock, + the cleanup alerts as (epoch seconds, text)); called only while + the head's cleanup has started and not yet settled. + short: whether disk alone still blocks the head. + + Returns: + (the CLEANUP to store, the line to log once the cleanup has ended + (its newest report since it started, or that it left none), whether + to send the notice now); both come once per head. + """ + log = None + if stall["cleaned"] and not stall["settled"]: + cleaning, reports = outcome() + if cleaning: + return stall, None, False + texts = [text for at, text in reports if at >= stall["cleaned_at"]] + log = f"low-risk cleanup: {texts[-1]}" if texts else "low-risk cleanup ended without a report" + stall = {**stall, "settled": True} + notice = stall["settled"] and short and not stall["noticed"] + return {**stall, "noticed": stall["noticed"] or notice}, log, notice + + +def cleanup_outcome(now: float) -> tuple[bool, list[tuple[float, str]]]: + """Returns: + (whether a watch.py cleanup holds its lock, younger than LOCK_TTL, + at now; the cleanup alerts of watch.py as (epoch seconds, text), + none while it does or when the file is unreadable). + """ + try: + held = os.lstat(watch.ROOT / watch.CLEANUP_LOCK) + except OSError: + held = None + if held is not None and stat.S_ISDIR(held.st_mode) and now - held.st_mtime <= watch.LOCK_TTL: + return True, [] + try: + rows = [json.loads(line) for _, line in watch.history_rows(nofollow.read_text(watch.ROOT, watch.ALERTS))] + except OSError: + return False, [] + return False, [ + (row["id"] / 1e9, row["text"]) for row in rows + if row.get("stage") == "cleanup" and isinstance(row["id"], int) and isinstance(row.get("text"), str) + ] + + +def stalled(data: dict, order: list[dict], verdicts: dict[str, str | None], floor: float, free_gb: float, + outcome: Callable[[], tuple[bool, list[tuple[float, str]]]], + now: float) -> tuple[list[tuple[str, list[str]]], bool, str | None]: + """Returns: + ([(label, radio.py argv)] to send: the ESCALATION step due, then + the notice that the head cannot fit after the cleanup; whether to + start the cleanup; the line to log about the cleanup, else None). + """ + fits = None in verdicts.values() + head = next((e for e in order if isinstance(verdicts[e["id"]], Disk)), None) if fits else next(iter(order), None) + if head is None: + return [], False, None + reason = verdicts[head["id"]] + active = [e for e in data["entries"] if e["status"] == "active"] + grants = [*([data[GRANTED]] if GRANTED in data else []), *(e["granted"] for e in active)] + disk = isinstance(reason, Disk) + sends = [] + if not fits: + data[STALL], step = escalate( + data.get(STALL) or {}, max(grants) if grants else None, min(e["enqueued"] for e in order), + any(e["hotfix"] for e in active), now, + ) + sends = [] if step is None else [(f"escalated {step}", radio_words(step, ( + f"{PREFIX}no grant for {STEPS[step][0]} min; head {head['id']} ({head['kind']}, {head['task']}) waits: {reason}" + )))] + state, clean = cleanup_due(data.get(CLEANUP) or {}, head["id"], disk, now) + log = None + if not clean: + state, log, notice = settle(state, outcome, disk) + reach = max(free_gb - floor, 0) if reason == floor_reason(head["need_gb"], floor) else 0 + if notice: + sends.append(("noticed that the head cannot fit", radio_words("post", ( + f"{PREFIX}head {head['id']} needs {head['need_gb']:g} GB, max reachable {reach:.0f} GB " + f"after cleanup ({reason}); operator cleanup needed" + )))) + data[CLEANUP] = state + return sends, clean, log + + +def radio_words(step: str, text: str) -> list[str]: + return [text if word == TEXT else word for word in STEPS[step][1]] + + +def cleaned(state: dict) -> tuple[dict, str]: + """Returns: + (the CLEANUP state, settled when the cleanup did not start, so no + report is awaited; the line to log). + """ + try: + watch.spawn_cleanup("--full") + except OSError as error: + return {**state, "settled": True}, f"low-risk cleanup did not start: {error}" + return state, "disk alone blocks the head; low-risk cleanup started" + + +def act(sends: list[tuple[str, list[str]]], logs: list[str], session: str) -> None: + """Args: + sends: [(label, radio.py argv)] to send over kvb-org-radio, each + reported once by its label. + logs: lines to log. + session: the session that sends the radio rows. + """ + for log in logs: + print(f"queue: {log}", file=sys.stderr, flush=True) + for label, words in sends: + try: + sent = subprocess.run( + [EXE, str(RADIO), *words, "--session", session], stdin=subprocess.DEVNULL, capture_output=True, + timeout=RADIO_SECONDS, check=False, + ).returncode + except (OSError, subprocess.TimeoutExpired) as error: + sent = error + print(f"queue: {label}" if sent == 0 else f"queue: {label} failed: {sent}", file=sys.stderr, flush=True) def load_reason(limits: dict, cpu: float, mem: float, ram_share: float) -> str | None: @@ -768,13 +1287,33 @@ def workdir() -> str: def enqueue(fields: dict) -> dict: + """Returns: + The entry of fields; a hold that hand_over() left for the installed + code at most CLAIM_SECONDS before, of the same pid, session and + command, keeps its id, seq and enqueue time. + """ with store() as data: - data["seq"] += 1 - entry = {"id": f"{fields['kind']}-{data['seq']}", "seq": data["seq"], **fields} + kept = {pid: h for pid, h in data.get(HANDOVER, {}).items() if fields["enqueued"] - h["at"] <= CLAIM_SECONDS} + mine = kept.pop(str(fields["pid"]), None) + if mine is not None and (mine["session"], mine["cmd"]) == (fields["session"], fields["cmd"]): + entry = {**fields, "id": mine["id"], "seq": mine["seq"], "enqueued": mine["enqueued"]} + else: + data["seq"] += 1 + entry = {"id": f"{fields['kind']}-{data['seq']}", "seq": data["seq"], **fields} + data[HANDOVER] = kept data["entries"].append(entry) return entry +def hand_over(entry_id: str) -> None: + with store() as data: + for entry in [e for e in data["entries"] if e["id"] == entry_id]: + data["entries"].remove(entry) + data.setdefault(HANDOVER, {})[str(entry["pid"])] = { + **{key: entry[key] for key in ("id", "seq", "enqueued", "session", "cmd")}, "at": time.time(), + } + + def update(entry_id: str, **fields: object) -> None: with store() as data: for entry in data["entries"]: @@ -823,13 +1362,26 @@ def wait(cfg: dict, entry_id: str, caught: list[int]) -> tuple[int | None, int]: return CANCELLED, 0 rejoin(data, mine[0], now) loaded = held_back(cfg, data["entries"], now, signatures, kinds, cpu, mem) - verdicts = decide(cfg, data["entries"], free, now, signatures, kinds, loaded) - reason = verdicts[entry_id] or loaded.get(entry_id) - mine[0]["reason"] = reason - if reason is None: + steps: dict[str, list[str]] = {} + verdicts, drain, order = decide(cfg, data["entries"], free, now, signatures, kinds, loaded, data.get(DRAIN), trace=steps) + data[DRAIN] = latched(data, drain, now) + mine[0]["trace"] = steps[entry_id] + if verdicts[entry_id] is None: concurrent = sum(e["status"] == "active" for e in data["entries"]) - mine[0].update(status="active", granted=now, **{FROZEN: footprint(mine[0], signatures, kinds)}) + mine[0].update(status="active", granted=now, reason=labelled(verdicts, [])[entry_id], + **{FROZEN: footprint(mine[0], signatures, kinds)}) + data[GRANTED] = now + data.pop(STALL, None) return None, concurrent + sends, clean, log = stalled(data, order, verdicts, cfg["thresholds_gb"]["stop"], free, + functools.partial(cleanup_outcome, now), now) + logs = [] if log is None else [log] + if clean: + data[CLEANUP], started = cleaned(data[CLEANUP]) + logs.append(started) + reason = labelled(verdicts, data[STALL]["fired"] if STALL in data else [])[entry_id] + mine[0]["reason"] = reason + act(sends, logs, mine[0]["session"]) position = list(verdicts).index(entry_id) + 1 if (position, reason) != shown: print(f"queue: {entry_id} waits at position {position}: {reason}", file=sys.stderr, flush=True) @@ -951,10 +1503,14 @@ def cmd_hold(args: argparse.Namespace) -> int: "workdir": here, "session": session, "name": council.live_sessions().get(session) or session, "lock": tag, "code": CODE, "pid": os.getpid(), "pid_start": identity(os.getpid()), "child": None, "child_start": None, "status": "waiting", "enqueued": time.time(), "granted": None, - "cmd": shlex.join(args.command), ALIASES: list(names), + "cmd": shlex.join(args.command), ALIASES: list(names), CLAIMS: claims_of(here), }) try: - code, concurrent = wait(cfg, entry["id"], caught) + try: + code, concurrent = wait(cfg, entry["id"], caught) + except Stale: + hand_over(entry["id"]) + raise if code is not None: return code if caught: @@ -983,40 +1539,93 @@ def cmd_hold(args: argparse.Namespace) -> int: leave(entry["id"]) -def cmd_list(_: argparse.Namespace) -> int: +def cmd_list(args: argparse.Namespace) -> int: cfg = config() free, _ = watch.disk(cfg) now = time.time() signatures, kinds = learned(cfg, now) + steps: dict[str, list[str]] = {} with store() as data: data["entries"] = prune(data["entries"]) entries = list(data["entries"]) - verdicts = decide(cfg, entries, free, now, signatures, kinds, {}) + verdicts, _, _ = decide(cfg, entries, free, now, signatures, kinds, {}, data.get(DRAIN), trace=steps) + verdicts = labelled(verdicts, (data.get(STALL) or {}).get("fired", [])) by_id = {e["id"]: e for e in entries} running = [(key_of(e), e["granted"]) for e in entries if e["status"] == "active"] rows = [ - ("run", e, history.eta(signatures, [(key_of(e), e["granted"])], [], now), "-") + (None, e, history.eta(signatures, [(key_of(e), e["granted"])], [], now), e.get("reason") or "-", e.get("trace", [])) for e in entries if e["status"] == "active" ] ahead: list[tuple[str, str]] = [] for n, (entry_id, reason) in enumerate(verdicts.items(), 1): e = by_id[entry_id] - polled = reason if silent(e, now) else e.get("reason") or reason - rows.append((str(n), e, history.eta(signatures, running, ahead, now), polled or "granted on its next poll")) + own = not silent(e, now) and bool(e.get("reason")) + rows.append((n, e, history.eta(signatures, running, ahead, now), (e["reason"] if own else reason) or "granted on its next poll", + e.get("trace", steps[entry_id]) if own else steps[entry_id])) ahead.append(key_of(e)) host = code_of(HOST_SKILLS) or CODE + rows = [(position, e, eta, stamped(e, reason, host), trace) for position, e, eta, reason, trace in rows] + if args.json: + rasci = sibling("rasci") + print(json.dumps(listed(rows, rasci.merged(False), rasci.wsjf, now), indent=1)) + return 0 print("position\tid\tstatus\tkind\ttask\tneed_gb\twaited_min\teta_min\tworkdir\treason") - for position, e, eta, reason in rows: - if e.get("code") != host: - reason = f"queue code {e.get('code') or 'without a stamp'}, the host runs {host}: {reason}" + for position, e, eta, reason, _ in rows: waited = ((e["granted"] or now) - e["enqueued"]) / 60 print(tab_line( - [position, e["id"], e["status"], e["kind"], e["task"], f"{e['need_gb']:g}", f"{waited:.0f}", + ["run" if position is None else position, e["id"], e["status"], e["kind"], e["task"], f"{e['need_gb']:g}", f"{waited:.0f}", "-" if eta is None else f"{eta:.0f}", e["workdir"], reason] )) return 0 +def stamped(entry: dict, reason: str, host: str) -> str: + if entry.get("code") == host: + return reason + return f"queue code {entry.get('code') or 'without a stamp'}, the host runs {host}: {reason}" + + +def job_class(entry: dict, pin: int | None) -> str: + """Returns: + hotfix for a --hotfix job or a task pinned in the hotfix band (prio + 1), main for a job of the session named Main, fix for the fix band + (prio 2), else feature. + """ + if entry["hotfix"] or pin == 1: + return "hotfix" + if entry["name"] == "Main": + return "main" + return "fix" if pin == 2 else "feature" + + +def listed(rows: list[tuple], register: dict[str, dict], score: Callable[[dict], tuple | None], + now: float) -> list[dict]: + """Args: + rows: (position, None while active, entry, ETA minutes or None, the + reason list shows, the trace steps) in list order. + register: the merged kvb-org-rasci rows by task. + score: rasci.wsjf, (value, complexity, WSJF) of a row or None. + now: epoch seconds. + + Returns: + One object per row: position, id, status, kind, task, session, + agent, class (job_class()), prio (the task's pin or None), wsjf (or + None), waited_s, eta_s (or None), reason and trace, the steps of + evaluate() that decided the entry at the poll that set its reason. + """ + shown = [] + for position, e, eta, reason, trace in rows: + row = register.get(e["task"], {}) + wsjf = score(row) + shown.append({ + "position": position, "id": e["id"], "status": e["status"], "kind": e["kind"], "task": e["task"], + "session": e.get("session"), "agent": e["name"], "class": job_class(e, row.get("prio")), "prio": row.get("prio"), + "wsjf": None if wsjf is None else wsjf[2], "waited_s": round((e["granted"] or now) - e["enqueued"]), + "eta_s": None if eta is None else round(eta * 60), "reason": reason, "trace": trace, + }) + return shown + + def tab_line(fields: list[object]) -> str: return "\t".join("".join(c if c.isprintable() else c.encode("unicode_escape").decode() for c in str(f)) for f in fields) @@ -1147,7 +1756,7 @@ def cli() -> argparse.ArgumentParser: hold.add_argument("--receipt") hold.add_argument("--log") hold.add_argument("command", nargs="+") - sub.add_parser("list") + sub.add_parser("list").add_argument("--json", action="store_true") sub.add_parser("history") cancel = sub.add_parser("cancel") cancel.add_argument("id") diff --git a/skills/kvb-org-radio/SKILL.md b/skills/kvb-org-radio/SKILL.md index d9acbfb..039ee00 100644 --- a/skills/kvb-org-radio/SKILL.md +++ b/skills/kvb-org-radio/SKILL.md @@ -27,7 +27,7 @@ status, result or question that can wait. ## Radio: immediate `send` reaches the addressee at its next tool step and wakes it when idle: -a full turn of every receiver, very expensive. +a full turn per receiver, very expensive. | Class | Addressee | Use | | --- | --- | --- | @@ -37,7 +37,8 @@ a full turn of every receiver, very expensive. | pan-pan | all | urgent trouble that blocks others and needs help soon | | mayday | all | shared state in danger (disk full, a broken shared branch, a leaked credential) | -A mayday also holds radio silence for two hours: a hook refuses every +A mayday holds radio silence two hours, except `--no-silence` or under +another's silence, which notify the operator instead: a hook refuses every Edit, Write and Bash command except those the config's `never_block` and `mayday_allow` start with and bare `radio.py` calls (text in single quotes, no shell operators), until its sender runs `radio.py clear` once resolved. The sender keeps working; an agent that diff --git a/skills/kvb-org-radio/scripts/radio.py b/skills/kvb-org-radio/scripts/radio.py index c288ddc..a1b6377 100755 --- a/skills/kvb-org-radio/scripts/radio.py +++ b/skills/kvb-org-radio/scripts/radio.py @@ -16,12 +16,15 @@ Environment: Args: send "" [--to ...] [--wake] [--sid ] - [--session ]: the radio, delivered at the addressee's next tool - step and waking it. + [--no-silence] [--session ]: the radio, delivered at the + addressee's next tool step and waking it. call needs --to; broadcast, security, pan-pan and mayday go to all and take no --to. direct is the old alias: with --wake a call, without it a post. mayday also sets radio silence - (/radio-silence.json) for two hours. Prints `ts: `, the + (/radio-silence.json) for two hours, unless --no-silence or + while the silence of another sender holds, which it never replaces; + --no-silence goes with mayday only; a mayday that sets no silence + tells the operator through the notify argv at once. Prints `ts: `, the stored row's ts, flushed right after it is stored, then `sent ...` and `wake: SendMessage to ` for every addressee without a live listener. --sid pins a call with a single --to to that session as in @@ -365,10 +368,14 @@ def store(entries: list[dict]) -> float: return entries[-1]["ts"] -def transmit(cls: str, to: list[str], text: str, caller: str | None, sessions: list[str]) -> int: +def transmit(cls: str, to: list[str], text: str, caller: str | None, sessions: list[str], silencing: bool) -> int: """Args: sessions: the session ids of the addressees, kept under `to_sessions` beside their names; none keeps the row as before. + silencing: whether a mayday sets radio silence; it never replaces + the active silence of another sender, and renewing its own keeps + its helpers. A mayday that sets no silence runs the notify argv + itself, since no silence brings it to the hooks. """ if cls in ADDRESSED and not to: fail(f"{cls} needs --to") @@ -383,11 +390,15 @@ def transmit(cls: str, to: list[str], text: str, caller: str | None, sessions: l with organize("nofollow").locked(ROOT, LOCK, 0o600, None): ts = store([{"from": sender, "to": to, "text": text, "class": cls, "v": VERSION, **({"to_sessions": sessions} if sessions else {})}]) print(f"{STORED}{ts!r}", flush=True) - if cls == "mayday": + active = (silence() or {}) if cls == "mayday" else {} + silenced = silencing and active.get("by", sender) == sender + if cls == "mayday" and silenced: write_atomic( SILENCE, - json.dumps({"since": ts, "until": ts + SILENCE_SECONDS, "by": sender, "text": text}), + json.dumps({**active, "since": ts, "until": ts + SILENCE_SECONDS, "by": sender, "text": text}), ) + if cls == "mayday" and not silenced: + notify_once({"since": ts, "by": sender, "text": text}) print(f"sent {cls} to {', '.join(clip(name) for name in to) or 'all'}") if cls in RADIO: for name in unreached(cls, to, sid, sessions): @@ -403,11 +414,13 @@ def cmd_send(args: argparse.Namespace) -> int: fail("--wake only with direct; a call always wakes") if args.sid and (cls != "call" or len(args.to or []) != 1): fail("--sid takes a call with a single --to") - return transmit(cls, args.to or [], args.text, args.session, [args.sid] if args.sid else []) + if args.no_silence and cls != "mayday": + fail("--no-silence goes with mayday only") + return transmit(cls, args.to or [], args.text, args.session, [args.sid] if args.sid else [], not args.no_silence) def cmd_post(args: argparse.Namespace) -> int: - return transmit("post" if args.to else "newsletter", args.to or [], args.text, args.session, []) + return transmit("post" if args.to else "newsletter", args.to or [], args.text, args.session, [], False) def cmd_clear(args: argparse.Namespace) -> int: @@ -664,7 +677,7 @@ def cmd_feedback_send(args: argparse.Namespace) -> int: fail("--sid takes a single --to") chosen = read_json(FEEDBACK) for name in args.to: - transmit(feedback_of(chosen, args.sid or name), [name], args.text, args.session, [args.sid] if args.sid else []) + transmit(feedback_of(chosen, args.sid or name), [name], args.text, args.session, [args.sid] if args.sid else [], False) return 0 @@ -677,6 +690,7 @@ def main() -> int: p.add_argument("--to", action="append") p.add_argument("--wake", action="store_true") p.add_argument("--sid") + p.add_argument("--no-silence", action="store_true") p.add_argument("--session") p.set_defaults(func=cmd_send) p = sub.add_parser("post") diff --git a/tests/queue-replay-20261114.json b/tests/queue-replay-20261114.json new file mode 100644 index 0000000..5ab2838 --- /dev/null +++ b/tests/queue-replay-20261114.json @@ -0,0 +1,3055 @@ +{ + "jobs": [ + { + "id": "test-nostack-1838", + "seq": 101838, + "kind": "test-nostack", + "task": "t:/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/avatar", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 0, + "workdir": "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/avatar", + "claims": [ + "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/avatar" + ], + "dur": 2535.0, + "enqueued": 1794611059.6730757 + }, + { + "id": "quality-high-1855", + "seq": 101855, + "kind": "quality-high", + "task": "t:/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/fix/web-app-docs/cli-replay-ci-severity", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 15, + "workdir": "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/fix/web-app-docs/cli-replay-ci-severity", + "claims": [ + "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/fix/web-app-docs/cli-replay-ci-severity", + "stack:cli-replay-ci-severity", + "slot:8" + ], + "dur": 1035.6000000000001, + "enqueued": 1794611059.6730757 + }, + { + "id": "test-nostack-1873", + "seq": 101873, + "kind": "test-nostack", + "task": "t:/home/kevinveenbirkenbach/Repositories/github.com/kevinveenbirkenbach/skills/.worktrees/feature/quality-adoption", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 0, + "workdir": "/home/kevinveenbirkenbach/Repositories/github.com/kevinveenbirkenbach/skills/.worktrees/feature/quality-adoption", + "claims": [ + "/home/kevinveenbirkenbach/Repositories/github.com/kevinveenbirkenbach/skills/.worktrees/feature/quality-adoption" + ], + "dur": 87.6, + "enqueued": 1794611059.6730757 + }, + { + "id": "test-nostack-1883", + "seq": 101883, + "kind": "test-nostack", + "task": "t:/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/quality/.worktrees/feature/benchmarks", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 0, + "workdir": "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/quality/.worktrees/feature/benchmarks", + "claims": [ + "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/quality/.worktrees/feature/benchmarks" + ], + "dur": 454.8, + "enqueued": 1794611067.8671985 + }, + { + "id": "quality-high-1884", + "seq": 101884, + "kind": "quality-high", + "task": "t:/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/fix/ci/vulnerability-db-artifact-handoff", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 15, + "workdir": "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/fix/ci/vulnerability-db-artifact-handoff", + "claims": [ + "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/fix/ci/vulnerability-db-artifact-handoff", + "stack:vulnerability-db-artifact-handoff", + "slot:10" + ], + "dur": 661.1999999999999, + "enqueued": 1794612406.7663507 + }, + { + "id": "test-nostack-1885", + "seq": 101885, + "kind": "test-nostack", + "task": "t:/home/kevinveenbirkenbach/Repositories/github.com/kevinveenbirkenbach/skills/.worktrees/unify/skills-20261112-night", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 0, + "workdir": "/home/kevinveenbirkenbach/Repositories/github.com/kevinveenbirkenbach/skills/.worktrees/unify/skills-20261112-night", + "claims": [ + "/home/kevinveenbirkenbach/Repositories/github.com/kevinveenbirkenbach/skills/.worktrees/unify/skills-20261112-night" + ], + "dur": 717.6, + "enqueued": 1794612406.7663507 + }, + { + "id": "test-nostack-1886", + "seq": 101886, + "kind": "test-nostack", + "task": "t:/home/kevinveenbirkenbach/Repositories/github.com/kevinveenbirkenbach/skills/.worktrees/unify/skills-20261112-night", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 0, + "workdir": "/home/kevinveenbirkenbach/Repositories/github.com/kevinveenbirkenbach/skills/.worktrees/unify/skills-20261112-night", + "claims": [ + "/home/kevinveenbirkenbach/Repositories/github.com/kevinveenbirkenbach/skills/.worktrees/unify/skills-20261112-night" + ], + "dur": 2323.7999999999997, + "enqueued": 1794612552.1318529 + }, + { + "id": "test-nostack-1887", + "seq": 101887, + "kind": "test-nostack", + "task": "t:/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/gui/.worktrees/feature/web-app-cst-store/slim-store-platform-roles", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 0, + "workdir": "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/gui/.worktrees/feature/web-app-cst-store/slim-store-platform-roles", + "claims": [ + "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/gui/.worktrees/feature/web-app-cst-store/slim-store-platform-roles", + "stack:slim-store-platform-roles" + ], + "dur": 379.8, + "enqueued": 1794612552.1318529 + }, + { + "id": "test-nostack-1888", + "seq": 101888, + "kind": "test-nostack", + "task": "t:/home/kevinveenbirkenbach/Repositories/github.com/kevinveenbirkenbach/skills/.worktrees/feature/quality-adoption", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 0, + "workdir": "/home/kevinveenbirkenbach/Repositories/github.com/kevinveenbirkenbach/skills/.worktrees/feature/quality-adoption", + "claims": [ + "/home/kevinveenbirkenbach/Repositories/github.com/kevinveenbirkenbach/skills/.worktrees/feature/quality-adoption" + ], + "dur": 466.8, + "enqueued": 1794613105.1446853 + }, + { + "id": "test-nostack-1895", + "seq": 101895, + "kind": "test-nostack", + "task": "t:/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/rehearsal/production", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 0, + "workdir": "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/rehearsal/production", + "claims": [ + "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/rehearsal/production" + ], + "dur": 1305.0, + "enqueued": 1794613271.6440656 + }, + { + "id": "quality-high-1900", + "seq": 101900, + "kind": "quality-high", + "task": "t:/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/fix/inventory/tools-runtime-severity", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 15, + "workdir": "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/fix/inventory/tools-runtime-severity", + "claims": [ + "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/fix/inventory/tools-runtime-severity", + "stack:tools-runtime-severity", + "slot:11" + ], + "dur": 1087.8, + "enqueued": 1794615775.5323043 + }, + { + "id": "quality-high-1903", + "seq": 101903, + "kind": "quality-high", + "task": "t:/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/fix/web-app-gitea/oidc-auto-link", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 15, + "workdir": "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/fix/web-app-gitea/oidc-auto-link", + "claims": [ + "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/fix/web-app-gitea/oidc-auto-link", + "stack:oidc-auto-link", + "slot:194" + ], + "dur": 619.2, + "enqueued": 1794616886.7265294 + }, + { + "id": "quality-high-1905", + "seq": 101905, + "kind": "quality-high", + "task": "t:/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/fix/make/cross-worktree-env", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 15, + "workdir": "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/fix/make/cross-worktree-env", + "claims": [ + "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/fix/make/cross-worktree-env", + "stack:cross-worktree-env", + "slot:12" + ], + "dur": 1065.6000000000001, + "enqueued": 1794616886.7265294 + }, + { + "id": "test-1908", + "seq": 101908, + "kind": "test", + "task": "t:/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/chore/cosmos-role-verify", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 15, + "workdir": "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/chore/cosmos-role-verify", + "claims": [ + "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/chore/cosmos-role-verify" + ], + "dur": 521.4, + "enqueued": 1794617966.3740077 + }, + { + "id": "quality-high-1912", + "seq": 101912, + "kind": "quality-high", + "task": "t:/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/chore/image/bake-lint-tools", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 15, + "workdir": "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/chore/image/bake-lint-tools", + "claims": [ + "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/chore/image/bake-lint-tools", + "stack:bake-lint-tools", + "slot:253" + ], + "dur": 782.4, + "enqueued": 1794617966.3740077 + }, + { + "id": "quality-high-1919", + "seq": 101919, + "kind": "quality-high", + "task": "t:/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/feature/tokens-css", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 15, + "workdir": "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/feature/tokens-css", + "claims": [ + "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/feature/tokens-css", + "stack:tokens-css", + "slot:14" + ], + "dur": 5.3999999999999995, + "enqueued": 1794617966.3740077 + }, + { + "id": "iteration-1920", + "seq": 101920, + "kind": "iteration", + "task": "t:/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/quality/.worktrees/feature/egress-tests", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 20, + "workdir": "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/quality/.worktrees/feature/egress-tests", + "claims": [ + "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/quality/.worktrees/feature/egress-tests" + ], + "dur": 10.799999999999999, + "enqueued": 1794617966.3740077 + }, + { + "id": "test-1924", + "seq": 101924, + "kind": "test", + "task": "t:/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/gui/.worktrees/feature/web-app-cst-store/money-integrity", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 15, + "workdir": "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/gui/.worktrees/feature/web-app-cst-store/money-integrity", + "claims": [ + "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/gui/.worktrees/feature/web-app-cst-store/money-integrity", + "stack:money-integrity" + ], + "dur": 439.20000000000005, + "enqueued": 1794617966.3740077 + }, + { + "id": "iteration-1928", + "seq": 101928, + "kind": "iteration", + "task": "t:/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/security/.worktrees/feature/cosmos-sky", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 20, + "workdir": "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/security/.worktrees/feature/cosmos-sky", + "claims": [ + "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/security/.worktrees/feature/cosmos-sky", + "stack:cosmos-sky" + ], + "dur": 7.800000000000001, + "enqueued": 1794617966.3740077 + }, + { + "id": "test-1929", + "seq": 101929, + "kind": "test", + "task": "t:/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/avatar", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 15, + "workdir": "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/avatar", + "claims": [ + "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/avatar" + ], + "dur": 229.8, + "enqueued": 1794617966.3740077 + }, + { + "id": "quality-high-1930", + "seq": 101930, + "kind": "quality-high", + "task": "t:/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/verify/main-hotfixes-20261114", + "name": "Main", + "main": true, + "hotfix": null, + "need_gb": 15, + "workdir": "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/verify/main-hotfixes-20261114", + "claims": [ + "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/verify/main-hotfixes-20261114" + ], + "dur": 1030.2, + "enqueued": 1794617966.3740077 + }, + { + "id": "test-nostack-1931", + "seq": 101931, + "kind": "test-nostack", + "task": "t:/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/fix/make/cross-worktree-env", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 0, + "workdir": "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/fix/make/cross-worktree-env", + "claims": [ + "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/fix/make/cross-worktree-env", + "stack:cross-worktree-env", + "slot:12" + ], + "dur": 5.3999999999999995, + "enqueued": 1794618186.661939 + }, + { + "id": "iteration-1932", + "seq": 101932, + "kind": "iteration", + "task": "t:/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/chore/cosmos-role-verify", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 20, + "workdir": "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/chore/cosmos-role-verify", + "claims": [ + "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/chore/cosmos-role-verify" + ], + "dur": 636.0, + "enqueued": 1794620051.2977545 + }, + { + "id": "iteration-1933", + "seq": 101933, + "kind": "iteration", + "task": "t:/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/security/.worktrees/feature/cosmos-sky", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 20, + "workdir": "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/security/.worktrees/feature/cosmos-sky", + "claims": [ + "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/security/.worktrees/feature/cosmos-sky", + "stack:cosmos-sky" + ], + "dur": 7.800000000000001, + "enqueued": 1794620114.506888 + }, + { + "id": "iteration-1934", + "seq": 101934, + "kind": "iteration", + "task": "t:/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/security/.worktrees/feature/cosmos-sky", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 20, + "workdir": "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/security/.worktrees/feature/cosmos-sky", + "claims": [ + "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/security/.worktrees/feature/cosmos-sky", + "stack:cosmos-sky" + ], + "dur": 160.2, + "enqueued": 1794620226.5240426 + }, + { + "id": "test-nostack-1935", + "seq": 101935, + "kind": "test-nostack", + "task": "t:/home/kevinveenbirkenbach/Repositories/github.com/kevinveenbirkenbach/infinito-nexus-core", + "name": "Main", + "main": true, + "hotfix": null, + "need_gb": 0, + "workdir": "/home/kevinveenbirkenbach/Repositories/github.com/kevinveenbirkenbach/infinito-nexus-core", + "claims": [ + "/home/kevinveenbirkenbach/Repositories/github.com/kevinveenbirkenbach/infinito-nexus-core", + "stack:infinito-nexus-core", + "slot:0" + ], + "dur": 13.8, + "enqueued": 1794620372.3420076 + }, + { + "id": "iteration-1936", + "seq": 101936, + "kind": "iteration", + "task": "t:/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/security/.worktrees/feature/cosmos-sky", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 20, + "workdir": "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/security/.worktrees/feature/cosmos-sky", + "claims": [ + "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/security/.worktrees/feature/cosmos-sky", + "stack:cosmos-sky" + ], + "dur": 121.79999999999998, + "enqueued": 1794620420.814787 + }, + { + "id": "test-nostack-1937", + "seq": 101937, + "kind": "test-nostack", + "task": "t:/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/chore/cosmos-role-verify", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 0, + "workdir": "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/chore/cosmos-role-verify", + "claims": [ + "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/chore/cosmos-role-verify" + ], + "dur": 6.6, + "enqueued": 1794620624.7329404 + }, + { + "id": "iteration-1938", + "seq": 101938, + "kind": "iteration", + "task": "t:/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/security/.worktrees/feature/cosmos-sky", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 20, + "workdir": "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/security/.worktrees/feature/cosmos-sky", + "claims": [ + "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/security/.worktrees/feature/cosmos-sky", + "stack:cosmos-sky" + ], + "dur": 6.6, + "enqueued": 1794620624.7329404 + }, + { + "id": "iteration-1939", + "seq": 101939, + "kind": "iteration", + "task": "t:/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/security/.worktrees/feature/cosmos-sky", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 20, + "workdir": "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/security/.worktrees/feature/cosmos-sky", + "claims": [ + "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/security/.worktrees/feature/cosmos-sky", + "stack:cosmos-sky" + ], + "dur": 48.0, + "enqueued": 1794620668.5262458 + }, + { + "id": "iteration-1940", + "seq": 101940, + "kind": "iteration", + "task": "t:/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/security/.worktrees/feature/cosmos-sky", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 20, + "workdir": "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/security/.worktrees/feature/cosmos-sky", + "claims": [ + "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/security/.worktrees/feature/cosmos-sky", + "stack:cosmos-sky" + ], + "dur": 10.799999999999999, + "enqueued": 1794620746.1917279 + }, + { + "id": "iteration-1941", + "seq": 101941, + "kind": "iteration", + "task": "t:/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/chore/cosmos-role-verify", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 20, + "workdir": "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/chore/cosmos-role-verify", + "claims": [ + "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/chore/cosmos-role-verify" + ], + "dur": 580.2, + "enqueued": 1794620783.543189 + }, + { + "id": "test-nostack-1942", + "seq": 101942, + "kind": "test-nostack", + "task": "t:/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/chore/cosmos-role-verify", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 0, + "workdir": "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/chore/cosmos-role-verify", + "claims": [ + "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/chore/cosmos-role-verify" + ], + "dur": 16.200000000000003, + "enqueued": 1794620795.3532555 + }, + { + "id": "iteration-1943", + "seq": 101943, + "kind": "iteration", + "task": "t:/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/security/.worktrees/feature/cosmos-sky", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 20, + "workdir": "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/security/.worktrees/feature/cosmos-sky", + "claims": [ + "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/security/.worktrees/feature/cosmos-sky", + "stack:cosmos-sky" + ], + "dur": 7.800000000000001, + "enqueued": 1794620795.3532555 + }, + { + "id": "iteration-1944", + "seq": 101944, + "kind": "iteration", + "task": "t:/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/security/.worktrees/feature/cosmos-sky", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 20, + "workdir": "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/security/.worktrees/feature/cosmos-sky", + "claims": [ + "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/security/.worktrees/feature/cosmos-sky", + "stack:cosmos-sky" + ], + "dur": 105.0, + "enqueued": 1794620831.5061061 + }, + { + "id": "iteration-1945", + "seq": 101945, + "kind": "iteration", + "task": "t:/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/security/.worktrees/feature/cosmos-sky", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 20, + "workdir": "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/security/.worktrees/feature/cosmos-sky", + "claims": [ + "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/security/.worktrees/feature/cosmos-sky", + "stack:cosmos-sky" + ], + "dur": 30.0, + "enqueued": 1794620981.3912404 + }, + { + "id": "iteration-1946", + "seq": 101946, + "kind": "iteration", + "task": "t:/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/security/.worktrees/feature/cosmos-sky", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 20, + "workdir": "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/security/.worktrees/feature/cosmos-sky", + "claims": [ + "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/security/.worktrees/feature/cosmos-sky", + "stack:cosmos-sky" + ], + "dur": 9.6, + "enqueued": 1794621040.3421578 + }, + { + "id": "iteration-1947", + "seq": 101947, + "kind": "iteration", + "task": "t:/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/security/.worktrees/feature/cosmos-sky", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 20, + "workdir": "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/security/.worktrees/feature/cosmos-sky", + "claims": [ + "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/security/.worktrees/feature/cosmos-sky", + "stack:cosmos-sky" + ], + "dur": 735.6, + "enqueued": 1794621090.8770251 + }, + { + "id": "test-1948", + "seq": 101948, + "kind": "test", + "task": "t:/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/quality/.worktrees/feature/e2e-on-main", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 15, + "workdir": "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/quality/.worktrees/feature/e2e-on-main", + "claims": [ + "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/quality/.worktrees/feature/e2e-on-main" + ], + "dur": 139.8, + "enqueued": 1794621834.8067539 + }, + { + "id": "test-1949", + "seq": 101949, + "kind": "test", + "task": "t:/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/chore/cosmos-svc-stack-proof", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 15, + "workdir": "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/chore/cosmos-svc-stack-proof", + "claims": [ + "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/chore/cosmos-svc-stack-proof" + ], + "dur": 1231.2, + "enqueued": 1794621834.8067539 + }, + { + "id": "iteration-1950", + "seq": 101950, + "kind": "iteration", + "task": "t:/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/chore/cosmos-role-verify", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 20, + "workdir": "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/chore/cosmos-role-verify", + "claims": [ + "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/chore/cosmos-role-verify" + ], + "dur": 6.6, + "enqueued": 1794623087.8126159 + }, + { + "id": "iteration-1951", + "seq": 101951, + "kind": "iteration", + "task": "t:/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/security/.worktrees/feature/cosmos-sky", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 20, + "workdir": "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/security/.worktrees/feature/cosmos-sky", + "claims": [ + "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/security/.worktrees/feature/cosmos-sky", + "stack:cosmos-sky" + ], + "dur": 696.0, + "enqueued": 1794623253.1670656 + }, + { + "id": "iteration-1952", + "seq": 101952, + "kind": "iteration", + "task": "t:/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/chore/cosmos-role-verify", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 20, + "workdir": "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/chore/cosmos-role-verify", + "claims": [ + "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/chore/cosmos-role-verify" + ], + "dur": 6.6, + "enqueued": 1794623307.2139034 + }, + { + "id": "iteration-1953", + "seq": 101953, + "kind": "iteration", + "task": "t:/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/chore/cosmos-role-verify", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 20, + "workdir": "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/chore/cosmos-role-verify", + "claims": [ + "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/chore/cosmos-role-verify" + ], + "dur": 35.4, + "enqueued": 1794623397.3255267 + }, + { + "id": "test-1954", + "seq": 101954, + "kind": "test", + "task": "t:/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/quality/.worktrees/feature/e2e-on-main", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 15, + "workdir": "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/quality/.worktrees/feature/e2e-on-main", + "claims": [ + "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/quality/.worktrees/feature/e2e-on-main" + ], + "dur": 174.0, + "enqueued": 1794623935.733968 + }, + { + "id": "test-nostack-1955", + "seq": 101955, + "kind": "test-nostack", + "task": "t:/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/gui/.worktrees/feature/web-app-cst-store/slim-store-platform-roles", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 0, + "workdir": "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/gui/.worktrees/feature/web-app-cst-store/slim-store-platform-roles", + "claims": [ + "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/gui/.worktrees/feature/web-app-cst-store/slim-store-platform-roles", + "stack:slim-store-platform-roles" + ], + "dur": 110.4, + "enqueued": 1794623935.733968 + }, + { + "id": "test-1956", + "seq": 101956, + "kind": "test", + "task": "t:/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/quality/.worktrees/feature/e2e-on-main", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 15, + "workdir": "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/quality/.worktrees/feature/e2e-on-main", + "claims": [ + "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/quality/.worktrees/feature/e2e-on-main" + ], + "dur": 19.8, + "enqueued": 1794624222.2138777 + }, + { + "id": "test-1957", + "seq": 101957, + "kind": "test", + "task": "t:/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/quality/.worktrees/feature/e2e-on-main", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 15, + "workdir": "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/quality/.worktrees/feature/e2e-on-main", + "claims": [ + "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/quality/.worktrees/feature/e2e-on-main" + ], + "dur": 18.6, + "enqueued": 1794624327.1261904 + }, + { + "id": "test-nostack-1958", + "seq": 101958, + "kind": "test-nostack", + "task": "t:/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/gui/.worktrees/feature/web-app-cst-store/slim-store-platform-roles", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 0, + "workdir": "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/gui/.worktrees/feature/web-app-cst-store/slim-store-platform-roles", + "claims": [ + "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/gui/.worktrees/feature/web-app-cst-store/slim-store-platform-roles", + "stack:slim-store-platform-roles" + ], + "dur": 105.0, + "enqueued": 1794624408.5033197 + }, + { + "id": "test-1959", + "seq": 101959, + "kind": "test", + "task": "t:/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/quality/.worktrees/feature/e2e-on-main", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 15, + "workdir": "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/quality/.worktrees/feature/e2e-on-main", + "claims": [ + "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/quality/.worktrees/feature/e2e-on-main" + ], + "dur": 155.39999999999998, + "enqueued": 1794624524.5966284 + }, + { + "id": "test-1960", + "seq": 101960, + "kind": "test", + "task": "t:/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/quality/.worktrees/feature/e2e-on-main", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 15, + "workdir": "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/quality/.worktrees/feature/e2e-on-main", + "claims": [ + "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/quality/.worktrees/feature/e2e-on-main" + ], + "dur": 18.6, + "enqueued": 1794624715.1702557 + }, + { + "id": "test-1961", + "seq": 101961, + "kind": "test", + "task": "t:/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/quality/.worktrees/feature/e2e-on-main", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 15, + "workdir": "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/quality/.worktrees/feature/e2e-on-main", + "claims": [ + "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/quality/.worktrees/feature/e2e-on-main" + ], + "dur": 242.4, + "enqueued": 1794624826.5787725 + }, + { + "id": "test-1962", + "seq": 101962, + "kind": "test", + "task": "t:/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/quality/.worktrees/feature/e2e-on-main", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 15, + "workdir": "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/quality/.worktrees/feature/e2e-on-main", + "claims": [ + "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/quality/.worktrees/feature/e2e-on-main" + ], + "dur": 291.0, + "enqueued": 1794625119.271377 + }, + { + "id": "iteration-1963", + "seq": 101963, + "kind": "iteration", + "task": "t:/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/quality/.worktrees/feature/playwright-csp", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 20, + "workdir": "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/quality/.worktrees/feature/playwright-csp", + "claims": [ + "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/quality/.worktrees/feature/playwright-csp" + ], + "dur": 1108.8, + "enqueued": 1794625743.1497972 + }, + { + "id": "iteration-1964", + "seq": 101964, + "kind": "iteration", + "task": "t:/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/quality/.worktrees/feature/playwright-csp", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 20, + "workdir": "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/quality/.worktrees/feature/playwright-csp", + "claims": [ + "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/quality/.worktrees/feature/playwright-csp" + ], + "dur": 141.0, + "enqueued": 1794627242.7408538 + }, + { + "id": "iteration-1965", + "seq": 101965, + "kind": "iteration", + "task": "t:/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/quality/.worktrees/feature/playwright-csp", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 20, + "workdir": "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/quality/.worktrees/feature/playwright-csp", + "claims": [ + "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/quality/.worktrees/feature/playwright-csp" + ], + "dur": 622.1999999999999, + "enqueued": 1794627584.405832 + }, + { + "id": "iteration-1966", + "seq": 101966, + "kind": "iteration", + "task": "t:/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/quality/.worktrees/feature/playwright-csp", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 20, + "workdir": "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/quality/.worktrees/feature/playwright-csp", + "claims": [ + "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/quality/.worktrees/feature/playwright-csp" + ], + "dur": 1624.8, + "enqueued": 1794628603.3751512 + }, + { + "id": "test-nostack-1968", + "seq": 101968, + "kind": "test-nostack", + "task": "t:/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/quality/.worktrees/fix/role-svc-stack", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 0, + "workdir": "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/quality/.worktrees/fix/role-svc-stack", + "claims": [ + "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/quality/.worktrees/fix/role-svc-stack" + ], + "dur": 11.4, + "enqueued": 1794630632.9562223 + }, + { + "id": "iteration-1969", + "seq": 101969, + "kind": "iteration", + "task": "t:/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/quality/.worktrees/feature/playwright-csp", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 20, + "workdir": "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/quality/.worktrees/feature/playwright-csp", + "claims": [ + "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/quality/.worktrees/feature/playwright-csp" + ], + "dur": 1158.6, + "enqueued": 1794630707.2312706 + }, + { + "id": "iteration-1970", + "seq": 101970, + "kind": "iteration", + "task": "t:/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/quality/.worktrees/feature/playwright-csp", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 20, + "workdir": "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/quality/.worktrees/feature/playwright-csp", + "claims": [ + "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/quality/.worktrees/feature/playwright-csp" + ], + "dur": 1173.0, + "enqueued": 1794631949.5232105 + }, + { + "id": "test-nostack-1971", + "seq": 101971, + "kind": "test-nostack", + "task": "t:/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/gui/.worktrees/feature/web-app-cst-store/slim-store-platform-roles", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 0, + "workdir": "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/gui/.worktrees/feature/web-app-cst-store/slim-store-platform-roles", + "claims": [ + "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/gui/.worktrees/feature/web-app-cst-store/slim-store-platform-roles", + "stack:slim-store-platform-roles" + ], + "dur": 105.0, + "enqueued": 1794632146.9577649 + }, + { + "id": "test-nostack-1972", + "seq": 101972, + "kind": "test-nostack", + "task": "t:/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/gui/.worktrees/feature/web-app-cst-store/slim-store-platform-roles", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 0, + "workdir": "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/gui/.worktrees/feature/web-app-cst-store/slim-store-platform-roles", + "claims": [ + "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/gui/.worktrees/feature/web-app-cst-store/slim-store-platform-roles", + "stack:slim-store-platform-roles" + ], + "dur": 105.0, + "enqueued": 1794632590.7863858 + }, + { + "id": "iteration-1973", + "seq": 101973, + "kind": "iteration", + "task": "t:/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/quality/.worktrees/feature/playwright-csp", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 20, + "workdir": "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/quality/.worktrees/feature/playwright-csp", + "claims": [ + "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/quality/.worktrees/feature/playwright-csp" + ], + "dur": 797.4, + "enqueued": 1794633408.2415342 + }, + { + "id": "test-nostack-1974", + "seq": 101974, + "kind": "test-nostack", + "task": "t:/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/gui/.worktrees/fix/web-app-cst-store/tenant-data-leaks", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 0, + "workdir": "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/gui/.worktrees/fix/web-app-cst-store/tenant-data-leaks", + "claims": [ + "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/gui/.worktrees/fix/web-app-cst-store/tenant-data-leaks" + ], + "dur": 122.99999999999999, + "enqueued": 1794634394.2724197 + }, + { + "id": "iteration-1975", + "seq": 101975, + "kind": "iteration", + "task": "t:/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/quality/.worktrees/feature/playwright-csp", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 20, + "workdir": "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/quality/.worktrees/feature/playwright-csp", + "claims": [ + "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/quality/.worktrees/feature/playwright-csp" + ], + "dur": 1947.0000000000002, + "enqueued": 1794634496.6960294 + }, + { + "id": "test-nostack-1976", + "seq": 101976, + "kind": "test-nostack", + "task": "t:/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/gui/.worktrees/fix/web-app-cst-store/tenant-data-leaks", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 0, + "workdir": "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/gui/.worktrees/fix/web-app-cst-store/tenant-data-leaks", + "claims": [ + "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/gui/.worktrees/fix/web-app-cst-store/tenant-data-leaks" + ], + "dur": 96.0, + "enqueued": 1794636596.8188114 + }, + { + "id": "iteration-1977", + "seq": 101977, + "kind": "iteration", + "task": "t:/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/quality/.worktrees/feature/playwright-csp", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 20, + "workdir": "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/quality/.worktrees/feature/playwright-csp", + "claims": [ + "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/quality/.worktrees/feature/playwright-csp" + ], + "dur": 30.0, + "enqueued": 1794636616.5507662 + }, + { + "id": "iteration-1978", + "seq": 101978, + "kind": "iteration", + "task": "t:/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/quality/.worktrees/feature/playwright-csp", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 20, + "workdir": "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/quality/.worktrees/feature/playwright-csp", + "claims": [ + "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/quality/.worktrees/feature/playwright-csp" + ], + "dur": 1615.2, + "enqueued": 1794636731.0313532 + }, + { + "id": "iteration-1979", + "seq": 101979, + "kind": "iteration", + "task": "t:/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/quality/.worktrees/feature/playwright-csp", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 20, + "workdir": "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/quality/.worktrees/feature/playwright-csp", + "claims": [ + "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/quality/.worktrees/feature/playwright-csp" + ], + "dur": 2544.6, + "enqueued": 1794638576.6743174 + }, + { + "id": "test-nostack-1980", + "seq": 101980, + "kind": "test-nostack", + "task": "t:/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/feature/iam-profile-rbac", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 0, + "workdir": "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/feature/iam-profile-rbac", + "claims": [ + "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/feature/iam-profile-rbac", + "stack:iam-profile-rbac", + "slot:108" + ], + "dur": 756.5999999999999, + "enqueued": 1794641307.2764425 + }, + { + "id": "iteration-1981", + "seq": 101981, + "kind": "iteration", + "task": "t:/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/quality/.worktrees/feature/playwright-csp", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 20, + "workdir": "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/quality/.worktrees/feature/playwright-csp", + "claims": [ + "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/quality/.worktrees/feature/playwright-csp" + ], + "dur": 2612.4, + "enqueued": 1794641307.2764425 + }, + { + "id": "iteration-1982", + "seq": 101982, + "kind": "iteration", + "task": "t:/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/chore/cosmos-role-verify", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 20, + "workdir": "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/chore/cosmos-role-verify", + "claims": [ + "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/chore/cosmos-role-verify" + ], + "dur": 617.4, + "enqueued": 1794641740.4039161 + }, + { + "id": "test-nostack-1983", + "seq": 101983, + "kind": "test-nostack", + "task": "t:/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/gui/.worktrees/feature/web-app-cst-store/slim-store-platform-roles", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 0, + "workdir": "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/gui/.worktrees/feature/web-app-cst-store/slim-store-platform-roles", + "claims": [ + "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/gui/.worktrees/feature/web-app-cst-store/slim-store-platform-roles", + "stack:slim-store-platform-roles" + ], + "dur": 117.6, + "enqueued": 1794642383.1472366 + }, + { + "id": "test-nostack-1984", + "seq": 101984, + "kind": "test-nostack", + "task": "t:/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/gui/.worktrees/feature/web-app-cst-store/slim-store-platform-roles", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 0, + "workdir": "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/gui/.worktrees/feature/web-app-cst-store/slim-store-platform-roles", + "claims": [ + "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/gui/.worktrees/feature/web-app-cst-store/slim-store-platform-roles", + "stack:slim-store-platform-roles" + ], + "dur": 121.2, + "enqueued": 1794643513.3667147 + }, + { + "id": "test-nostack-1985", + "seq": 101985, + "kind": "test-nostack", + "task": "t:/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/fix/update/repo-ref-must-resolve", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 0, + "workdir": "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/fix/update/repo-ref-must-resolve", + "claims": [ + "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/fix/update/repo-ref-must-resolve" + ], + "dur": 45.6, + "enqueued": 1794643621.852624 + }, + { + "id": "test-nostack-1986", + "seq": 101986, + "kind": "test-nostack", + "task": "t:/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/fix/web-app-pihole/onion-handover", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 0, + "workdir": "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/fix/web-app-pihole/onion-handover", + "claims": [ + "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/fix/web-app-pihole/onion-handover" + ], + "dur": 6.0, + "enqueued": 1794643668.4548438 + }, + { + "id": "test-nostack-1987", + "seq": 101987, + "kind": "test-nostack", + "task": "t:/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/fix/web-app-pihole/onion-handover", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 0, + "workdir": "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/fix/web-app-pihole/onion-handover", + "claims": [ + "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/fix/web-app-pihole/onion-handover" + ], + "dur": 93.0, + "enqueued": 1794643711.1901598 + }, + { + "id": "test-nostack-1988", + "seq": 101988, + "kind": "test-nostack", + "task": "t:/tmp", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 0, + "workdir": "/tmp", + "claims": [ + "/tmp" + ], + "dur": 5.3999999999999995, + "enqueued": 1794643807.7290268 + }, + { + "id": "test-nostack-1989", + "seq": 101989, + "kind": "test-nostack", + "task": "t:/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/fix/update/hold-blocked-pins", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 0, + "workdir": "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/fix/update/hold-blocked-pins", + "claims": [ + "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/fix/update/hold-blocked-pins", + "stack:hold-blocked-pins", + "slot:0" + ], + "dur": 89.4, + "enqueued": 1794643807.7290268 + }, + { + "id": "test-nostack-1990", + "seq": 101990, + "kind": "test-nostack", + "task": "t:/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/fix/tests/swarm-watchdog-progress", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 0, + "workdir": "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/fix/tests/swarm-watchdog-progress", + "claims": [ + "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/fix/tests/swarm-watchdog-progress" + ], + "dur": 11.4, + "enqueued": 1794643832.7381885 + }, + { + "id": "test-nostack-1991", + "seq": 101991, + "kind": "test-nostack", + "task": "t:/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/fix/tests/swarm-watchdog-progress", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 0, + "workdir": "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/fix/tests/swarm-watchdog-progress", + "claims": [ + "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/fix/tests/swarm-watchdog-progress" + ], + "dur": 8.4, + "enqueued": 1794643918.881824 + }, + { + "id": "test-nostack-1992", + "seq": 101992, + "kind": "test-nostack", + "task": "t:/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/fix/dsk-virtualbox/ci-runner-410", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 0, + "workdir": "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/fix/dsk-virtualbox/ci-runner-410", + "claims": [ + "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/fix/dsk-virtualbox/ci-runner-410" + ], + "dur": 18.6, + "enqueued": 1794643959.7522821 + }, + { + "id": "iteration-1993", + "seq": 101993, + "kind": "iteration", + "task": "t:/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/quality/.worktrees/feature/playwright-csp", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 20, + "workdir": "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/quality/.worktrees/feature/playwright-csp", + "claims": [ + "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/quality/.worktrees/feature/playwright-csp" + ], + "dur": 2896.8, + "enqueued": 1794644700.7398243 + }, + { + "id": "test-nostack-1995", + "seq": 101995, + "kind": "test-nostack", + "task": "t:/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/fix/update/hold-blocked-pins", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 0, + "workdir": "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/fix/update/hold-blocked-pins", + "claims": [ + "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/fix/update/hold-blocked-pins", + "stack:hold-blocked-pins", + "slot:0" + ], + "dur": 96.60000000000001, + "enqueued": 1794645201.2362478 + }, + { + "id": "test-nostack-1996", + "seq": 101996, + "kind": "test-nostack", + "task": "t:/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/fix/web-app-pihole/onion-handover", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 0, + "workdir": "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/fix/web-app-pihole/onion-handover", + "claims": [ + "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/fix/web-app-pihole/onion-handover" + ], + "dur": 62.400000000000006, + "enqueued": 1794645315.4696426 + }, + { + "id": "test-nostack-1997", + "seq": 101997, + "kind": "test-nostack", + "task": "t:/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/fix/dsk-virtualbox/ci-runner-410", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 0, + "workdir": "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/fix/dsk-virtualbox/ci-runner-410", + "claims": [ + "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/fix/dsk-virtualbox/ci-runner-410" + ], + "dur": 20.400000000000002, + "enqueued": 1794645315.4696426 + }, + { + "id": "test-nostack-1998", + "seq": 101998, + "kind": "test-nostack", + "task": "t:/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/fix/tests/swarm-watchdog-progress", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 0, + "workdir": "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/fix/tests/swarm-watchdog-progress", + "claims": [ + "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/fix/tests/swarm-watchdog-progress" + ], + "dur": 9.0, + "enqueued": 1794645378.8097925 + }, + { + "id": "test-nostack-1999", + "seq": 101999, + "kind": "test-nostack", + "task": "t:/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/fix/tests/swarm-watchdog-progress", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 0, + "workdir": "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/fix/tests/swarm-watchdog-progress", + "claims": [ + "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/fix/tests/swarm-watchdog-progress" + ], + "dur": 13.8, + "enqueued": 1794645457.425492 + }, + { + "id": "test-nostack-2000", + "seq": 102000, + "kind": "test-nostack", + "task": "t:/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/fix/dsk-gnt-cursor/vendor-repo-off", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 0, + "workdir": "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/fix/dsk-gnt-cursor/vendor-repo-off", + "claims": [ + "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/fix/dsk-gnt-cursor/vendor-repo-off" + ], + "dur": 68.39999999999999, + "enqueued": 1794645673.3301532 + }, + { + "id": "test-nostack-2001", + "seq": 102001, + "kind": "test-nostack", + "task": "t:/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/fix/update/dnf-makecache-assume-yes", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 0, + "workdir": "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/fix/update/dnf-makecache-assume-yes", + "claims": [ + "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/fix/update/dnf-makecache-assume-yes" + ], + "dur": 21.0, + "enqueued": 1794647275.2544863 + }, + { + "id": "test-nostack-2002", + "seq": 102002, + "kind": "test-nostack", + "task": "t:/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/fix/dsk-gnt-cursor/vendor-repo-off", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 0, + "workdir": "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/fix/dsk-gnt-cursor/vendor-repo-off", + "claims": [ + "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/fix/dsk-gnt-cursor/vendor-repo-off" + ], + "dur": 30.0, + "enqueued": 1794647617.7398634 + }, + { + "id": "iteration-2003", + "seq": 102003, + "kind": "iteration", + "task": "t:/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/quality/.worktrees/feature/playwright-csp", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 20, + "workdir": "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/quality/.worktrees/feature/playwright-csp", + "claims": [ + "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/quality/.worktrees/feature/playwright-csp" + ], + "dur": 3868.2, + "enqueued": 1794647766.5256798 + }, + { + "id": "test-nostack-2004", + "seq": 102004, + "kind": "test-nostack", + "task": "t:/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/fix/svc-bkp-volume-2-local/baudolo-8", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 0, + "workdir": "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/fix/svc-bkp-volume-2-local/baudolo-8", + "claims": [ + "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/fix/svc-bkp-volume-2-local/baudolo-8", + "stack:baudolo-8", + "slot:13" + ], + "dur": 7.199999999999999, + "enqueued": 1794650786.7636828 + }, + { + "id": "test-nostack-2005", + "seq": 102005, + "kind": "test-nostack", + "task": "t:/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/fix/svc-bkp-volume-2-local/baudolo-8", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 0, + "workdir": "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/fix/svc-bkp-volume-2-local/baudolo-8", + "claims": [ + "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/fix/svc-bkp-volume-2-local/baudolo-8", + "stack:baudolo-8", + "slot:13" + ], + "dur": 16.8, + "enqueued": 1794650832.55184 + }, + { + "id": "test-nostack-2006", + "seq": 102006, + "kind": "test-nostack", + "task": "t:/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/fix/svc-bkp-volume-2-local/baudolo-8", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 0, + "workdir": "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/fix/svc-bkp-volume-2-local/baudolo-8", + "claims": [ + "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/fix/svc-bkp-volume-2-local/baudolo-8", + "stack:baudolo-8", + "slot:13" + ], + "dur": 30.0, + "enqueued": 1794651022.4366412 + }, + { + "id": "test-nostack-2007", + "seq": 102007, + "kind": "test-nostack", + "task": "t:/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/fix/make/cross-worktree-env", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 0, + "workdir": "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/fix/make/cross-worktree-env", + "claims": [ + "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/fix/make/cross-worktree-env", + "stack:cross-worktree-env", + "slot:12" + ], + "dur": 330.0, + "enqueued": 1794651675.1879473 + }, + { + "id": "iteration-2008", + "seq": 102008, + "kind": "iteration", + "task": "t:/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/quality/.worktrees/feature/playwright-csp", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 20, + "workdir": "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/quality/.worktrees/feature/playwright-csp", + "claims": [ + "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/quality/.worktrees/feature/playwright-csp" + ], + "dur": 304.8, + "enqueued": 1794651960.9293716 + }, + { + "id": "test-nostack-2010", + "seq": 102010, + "kind": "test-nostack", + "task": "t:/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/gui", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 0, + "workdir": "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/gui", + "claims": [ + "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/gui", + "stack:gui" + ], + "dur": 6.0, + "enqueued": 1794653280.0467408 + }, + { + "id": "test-nostack-2011", + "seq": 102011, + "kind": "test-nostack", + "task": "t:/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/gui/.worktrees/feature/web-app-cst-store/slim-store-platform-roles", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 0, + "workdir": "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/gui/.worktrees/feature/web-app-cst-store/slim-store-platform-roles", + "claims": [ + "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/gui/.worktrees/feature/web-app-cst-store/slim-store-platform-roles", + "stack:slim-store-platform-roles" + ], + "dur": 114.0, + "enqueued": 1794653332.5699947 + }, + { + "id": "iteration-2012", + "seq": 102012, + "kind": "iteration", + "task": "t:/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/quality/.worktrees/main", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 20, + "workdir": "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/quality/.worktrees/main", + "claims": [ + "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/quality/.worktrees/main" + ], + "dur": 297.0, + "enqueued": 1794653948.7949274 + }, + { + "id": "iteration-2013", + "seq": 102013, + "kind": "iteration", + "task": "t:/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/quality/.worktrees/fix/verify-database", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 20, + "workdir": "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/quality/.worktrees/fix/verify-database", + "claims": [ + "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/quality/.worktrees/fix/verify-database" + ], + "dur": 60.6, + "enqueued": 1794657947.1725764 + }, + { + "id": "test-nostack-2014", + "seq": 102014, + "kind": "test-nostack", + "task": "t:/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/gui/.worktrees/feature/web-app-cst-store/money-integrity", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 0, + "workdir": "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/gui/.worktrees/feature/web-app-cst-store/money-integrity", + "claims": [ + "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/gui/.worktrees/feature/web-app-cst-store/money-integrity", + "stack:money-integrity" + ], + "dur": 130.2, + "enqueued": 1794657947.1725764 + }, + { + "id": "test-nostack-2015", + "seq": 102015, + "kind": "test-nostack", + "task": "t:/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/fix/sys-ctl-hlth-runtime/trivy-rhel10-library-only", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 0, + "workdir": "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/fix/sys-ctl-hlth-runtime/trivy-rhel10-library-only", + "claims": [ + "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/fix/sys-ctl-hlth-runtime/trivy-rhel10-library-only", + "stack:trivy-rhel10-library-only", + "slot:7" + ], + "dur": 9.0, + "enqueued": 1794657947.1725764 + }, + { + "id": "test-nostack-2016", + "seq": 102016, + "kind": "test-nostack", + "task": "t:/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/fix/sys-ctl-hlth-runtime/trivy-rhel10-library-only", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 0, + "workdir": "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/fix/sys-ctl-hlth-runtime/trivy-rhel10-library-only", + "claims": [ + "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/fix/sys-ctl-hlth-runtime/trivy-rhel10-library-only", + "stack:trivy-rhel10-library-only", + "slot:7" + ], + "dur": 9.0, + "enqueued": 1794658011.6031766 + }, + { + "id": "test-2017", + "seq": 102017, + "kind": "test", + "task": "t:/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/fix/sys-ctl-hlth-runtime/trivy-rhel10-library-only", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 15, + "workdir": "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/fix/sys-ctl-hlth-runtime/trivy-rhel10-library-only", + "claims": [ + "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/fix/sys-ctl-hlth-runtime/trivy-rhel10-library-only", + "stack:trivy-rhel10-library-only", + "slot:7" + ], + "dur": 132.6, + "enqueued": 1794658202.0725489 + }, + { + "id": "test-2018", + "seq": 102018, + "kind": "test", + "task": "t:/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/avatar", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 15, + "workdir": "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/avatar", + "claims": [ + "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/avatar" + ], + "dur": 7.199999999999999, + "enqueued": 1794659162.7829883 + }, + { + "id": "test-2019", + "seq": 102019, + "kind": "test", + "task": "t:/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/fix/sys-ctl-hlth-runtime/trivy-rhel10-library-only", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 15, + "workdir": "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/fix/sys-ctl-hlth-runtime/trivy-rhel10-library-only", + "claims": [ + "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/fix/sys-ctl-hlth-runtime/trivy-rhel10-library-only", + "stack:trivy-rhel10-library-only", + "slot:7" + ], + "dur": 115.8, + "enqueued": 1794659162.7829883 + }, + { + "id": "quality-high-2020", + "seq": 102020, + "kind": "quality-high", + "task": "t:/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/feature/tokens-css", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 15, + "workdir": "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/feature/tokens-css", + "claims": [ + "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/feature/tokens-css", + "stack:tokens-css", + "slot:14" + ], + "dur": 593.4000000000001, + "enqueued": 1794659926.5323298 + }, + { + "id": "gui-dind-2021", + "seq": 102021, + "kind": "gui-dind", + "task": "t:/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/gui/.worktrees/feature/web-app-cst-store/money-integrity", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 65, + "workdir": "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/gui/.worktrees/feature/web-app-cst-store/money-integrity", + "claims": [ + "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/gui/.worktrees/feature/web-app-cst-store/money-integrity", + "stack:money-integrity" + ], + "dur": 105.0, + "enqueued": 1794659926.5323298 + }, + { + "id": "test-nostack-2022", + "seq": 102022, + "kind": "test-nostack", + "task": "t:/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/gui/.worktrees/feature/web-app-cst-store/slim-store-platform-roles", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 0, + "workdir": "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/gui/.worktrees/feature/web-app-cst-store/slim-store-platform-roles", + "claims": [ + "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/gui/.worktrees/feature/web-app-cst-store/slim-store-platform-roles", + "stack:slim-store-platform-roles" + ], + "dur": 118.2, + "enqueued": 1794660003.2000523 + }, + { + "id": "test-2023", + "seq": 102023, + "kind": "test", + "task": "t:/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/avatar", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 15, + "workdir": "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/avatar", + "claims": [ + "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/avatar" + ], + "dur": 6.6, + "enqueued": 1794660160.0300958 + }, + { + "id": "test-nostack-2024", + "seq": 102024, + "kind": "test-nostack", + "task": "t:/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/fix/main/test-lint-size-and-spot", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 0, + "workdir": "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/fix/main/test-lint-size-and-spot", + "claims": [ + "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/fix/main/test-lint-size-and-spot", + "stack:test-lint-size-and-spot", + "slot:0" + ], + "dur": 13.2, + "enqueued": 1794660160.0300958 + }, + { + "id": "test-nostack-2025", + "seq": 102025, + "kind": "test-nostack", + "task": "t:/home/kevinveenbirkenbach/Repositories/github.com/kevinveenbirkenbach/skills", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 0, + "workdir": "/home/kevinveenbirkenbach/Repositories/github.com/kevinveenbirkenbach/skills", + "claims": [ + "/home/kevinveenbirkenbach/Repositories/github.com/kevinveenbirkenbach/skills" + ], + "dur": 94.2, + "enqueued": 1794660668.6558936 + }, + { + "id": "test-2026", + "seq": 102026, + "kind": "test", + "task": "t:/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/avatar", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 15, + "workdir": "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/avatar", + "claims": [ + "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/avatar" + ], + "dur": 6.6, + "enqueued": 1794660778.0215502 + }, + { + "id": "test-nostack-2030", + "seq": 102030, + "kind": "test-nostack", + "task": "t:/home/kevinveenbirkenbach/Repositories/github.com/kevinveenbirkenbach/skills", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 0, + "workdir": "/home/kevinveenbirkenbach/Repositories/github.com/kevinveenbirkenbach/skills", + "claims": [ + "/home/kevinveenbirkenbach/Repositories/github.com/kevinveenbirkenbach/skills" + ], + "dur": 529.8, + "enqueued": 1794661230.5860317 + }, + { + "id": "test-nostack-2031", + "seq": 102031, + "kind": "test-nostack", + "task": "t:/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/fix/dsk-blender/arch-arm64-package", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 0, + "workdir": "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/fix/dsk-blender/arch-arm64-package", + "claims": [ + "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/fix/dsk-blender/arch-arm64-package", + "stack:arch-arm64-package", + "slot:0" + ], + "dur": 24.599999999999998, + "enqueued": 1794661379.0173752 + }, + { + "id": "iteration-1", + "seq": 200001, + "kind": "iteration", + "task": "t:/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/quality/.worktrees/fix/verify-database", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 20, + "workdir": "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/quality/.worktrees/fix/verify-database", + "claims": [ + "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/quality/.worktrees/fix/verify-database" + ], + "dur": 96.60000000000001, + "enqueued": 1794664961.5303812 + }, + { + "id": "iteration-2", + "seq": 200002, + "kind": "iteration", + "task": "t:/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/security/feature/page-optimize", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 20, + "workdir": "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/security/feature/page-optimize", + "claims": [ + "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/security/feature/page-optimize", + "stack:page-optimize" + ], + "dur": 714.0, + "enqueued": 1794664965.807971 + }, + { + "id": "quality-high-5", + "seq": 200005, + "kind": "quality-high", + "task": "t:/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/feature/tokens-css", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 15, + "workdir": "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/feature/tokens-css", + "claims": [ + "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/feature/tokens-css", + "stack:tokens-css", + "slot:14" + ], + "dur": 5.3999999999999995, + "enqueued": 1794665706.3554428 + }, + { + "id": "iteration-7", + "seq": 200007, + "kind": "iteration", + "task": "t:/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/quality/.worktrees/fix/verify-database", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 20, + "workdir": "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/quality/.worktrees/fix/verify-database", + "claims": [ + "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/quality/.worktrees/fix/verify-database" + ], + "dur": 79.80000000000001, + "enqueued": 1794665706.3554428 + }, + { + "id": "iteration-9", + "seq": 200009, + "kind": "iteration", + "task": "t:/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/quality/.worktrees/fix/verify-database", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 20, + "workdir": "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/quality/.worktrees/fix/verify-database", + "claims": [ + "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/quality/.worktrees/fix/verify-database" + ], + "dur": 63.0, + "enqueued": 1794665846.469447 + }, + { + "id": "test-10", + "seq": 200010, + "kind": "test", + "task": "t:/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/avatar", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 15, + "workdir": "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/avatar", + "claims": [ + "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/avatar" + ], + "dur": 6.0, + "enqueued": 1794666087.3280525 + }, + { + "id": "iteration-11", + "seq": 200011, + "kind": "iteration", + "task": "t:/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/quality/.worktrees/fix/verify-database", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 20, + "workdir": "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/quality/.worktrees/fix/verify-database", + "claims": [ + "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/quality/.worktrees/fix/verify-database" + ], + "dur": 79.80000000000001, + "enqueued": 1794666106.885998 + }, + { + "id": "test-12", + "seq": 200012, + "kind": "test", + "task": "t:/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/avatar", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 15, + "workdir": "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/avatar", + "claims": [ + "/home/kevinveenbirkenbach/Repositories/github.com/infinito-nexus/avatar" + ], + "dur": 84.0, + "enqueued": 1794666216.6864936 + }, + { + "id": "test-nostack-13", + "seq": 200013, + "kind": "test-nostack", + "task": "t:/tmp/claude-1000/-home-kevinveenbirkenbach-Repositories-github-com-kevinveenbirkenbach-infinito-nexus-core/af1817ec-1709-4426-98d3-34c3a98c48cf/scratchpad/codescan-merge", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 0, + "workdir": "/tmp/claude-1000/-home-kevinveenbirkenbach-Repositories-github-com-kevinveenbirkenbach-infinito-nexus-core/af1817ec-1709-4426-98d3-34c3a98c48cf/scratchpad/codescan-merge", + "claims": [ + "/tmp/claude-1000/-home-kevinveenbirkenbach-Repositories-github-com-kevinveenbirkenbach-infinito-nexus-core/af1817ec-1709-4426-98d3-34c3a98c48cf/scratchpad/codescan-merge", + "stack:codescan-merge", + "slot:0" + ], + "dur": 39.6, + "enqueued": 1794666851.3462508 + }, + { + "id": "test-nostack-14", + "seq": 200014, + "kind": "test-nostack", + "task": "t:/tmp/claude-1000/-home-kevinveenbirkenbach-Repositories-github-com-kevinveenbirkenbach-infinito-nexus-core/af1817ec-1709-4426-98d3-34c3a98c48cf/scratchpad/codescan-merge", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 0, + "workdir": "/tmp/claude-1000/-home-kevinveenbirkenbach-Repositories-github-com-kevinveenbirkenbach-infinito-nexus-core/af1817ec-1709-4426-98d3-34c3a98c48cf/scratchpad/codescan-merge", + "claims": [ + "/tmp/claude-1000/-home-kevinveenbirkenbach-Repositories-github-com-kevinveenbirkenbach-infinito-nexus-core/af1817ec-1709-4426-98d3-34c3a98c48cf/scratchpad/codescan-merge", + "stack:codescan-merge", + "slot:0" + ], + "dur": 41.4, + "enqueued": 1794666971.5129383 + }, + { + "id": "quality-high-15", + "seq": 200015, + "kind": "quality-high", + "task": "t:/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/production", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 15, + "workdir": "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/production", + "claims": [ + "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/production", + "stack:production", + "slot:174" + ], + "dur": 5.3999999999999995, + "enqueued": 1794667211.9268482 + }, + { + "id": "quality-high-16", + "seq": 200016, + "kind": "quality-high", + "task": "t:/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/chore/verify/main-hotfixes-20261114", + "name": "Main", + "main": true, + "hotfix": null, + "need_gb": 15, + "workdir": "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/chore/verify/main-hotfixes-20261114", + "claims": [ + "/home/kevinveenbirkenbach/.local/share/worktrees/github.com/infinito-nexus/core/chore/verify/main-hotfixes-20261114", + "stack:main-hotfixes-20261114", + "slot:254" + ], + "dur": 5.3999999999999995, + "enqueued": 1794667211.9268482 + }, + { + "id": "test-19", + "seq": 200019, + "kind": "test", + "task": "t:/home/kevinveenbirkenbach/Repositories/github.com/kevinveenbirkenbach/skills/.worktrees/fix/hub-cleanup-pkgmgr-local", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 15, + "workdir": "/home/kevinveenbirkenbach/Repositories/github.com/kevinveenbirkenbach/skills/.worktrees/fix/hub-cleanup-pkgmgr-local", + "claims": [ + "/home/kevinveenbirkenbach/Repositories/github.com/kevinveenbirkenbach/skills/.worktrees/fix/hub-cleanup-pkgmgr-local" + ], + "dur": 1461.0, + "enqueued": 1794667211.9268482 + }, + { + "id": "test-22", + "seq": 200022, + "kind": "test", + "task": "t:/home/kevinveenbirkenbach/Repositories/github.com/kevinveenbirkenbach/skills/.worktrees/fix/hub-speech-bubbles-hidden", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 15, + "workdir": "/home/kevinveenbirkenbach/Repositories/github.com/kevinveenbirkenbach/skills/.worktrees/fix/hub-speech-bubbles-hidden", + "claims": [ + "/home/kevinveenbirkenbach/Repositories/github.com/kevinveenbirkenbach/skills/.worktrees/fix/hub-speech-bubbles-hidden" + ], + "dur": 75.0, + "enqueued": 1794668187.4213681 + }, + { + "id": "test-26", + "seq": 200026, + "kind": "test", + "task": "t:/home/kevinveenbirkenbach/Repositories/github.com/kevinveenbirkenbach/skills", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 15, + "workdir": "/home/kevinveenbirkenbach/Repositories/github.com/kevinveenbirkenbach/skills", + "claims": [ + "/home/kevinveenbirkenbach/Repositories/github.com/kevinveenbirkenbach/skills" + ], + "dur": 142.20000000000002, + "enqueued": 1794668187.4213681 + }, + { + "id": "test-27", + "seq": 200027, + "kind": "test", + "task": "t:/home/kevinveenbirkenbach/Repositories/github.com/kevinveenbirkenbach/skills", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 15, + "workdir": "/home/kevinveenbirkenbach/Repositories/github.com/kevinveenbirkenbach/skills", + "claims": [ + "/home/kevinveenbirkenbach/Repositories/github.com/kevinveenbirkenbach/skills" + ], + "dur": 48.0, + "enqueued": 1794668187.4213681 + }, + { + "id": "test-nostack-34", + "seq": 200034, + "kind": "test-nostack", + "task": "t:/home/kevinveenbirkenbach/Repositories/github.com/kevinveenbirkenbach/infinito-nexus-core/.claude/worktrees/agent-a8c3ad1a9d2b6944e", + "name": "Main", + "main": true, + "hotfix": null, + "need_gb": 0, + "workdir": "/home/kevinveenbirkenbach/Repositories/github.com/kevinveenbirkenbach/infinito-nexus-core/.claude/worktrees/agent-a8c3ad1a9d2b6944e", + "claims": [ + "/home/kevinveenbirkenbach/Repositories/github.com/kevinveenbirkenbach/infinito-nexus-core/.claude/worktrees/agent-a8c3ad1a9d2b6944e", + "stack:agent-a8c3ad1a9d2b6944e", + "slot:0" + ], + "dur": 105.0, + "enqueued": 1794668187.4213681 + }, + { + "id": "test-nostack-37", + "seq": 200037, + "kind": "test-nostack", + "task": "t:/home/kevinveenbirkenbach/Repositories/github.com/kevinveenbirkenbach/infinito-nexus-core/.claude/worktrees/agent-a8c3ad1a9d2b6944e", + "name": "Main", + "main": true, + "hotfix": null, + "need_gb": 0, + "workdir": "/home/kevinveenbirkenbach/Repositories/github.com/kevinveenbirkenbach/infinito-nexus-core/.claude/worktrees/agent-a8c3ad1a9d2b6944e", + "claims": [ + "/home/kevinveenbirkenbach/Repositories/github.com/kevinveenbirkenbach/infinito-nexus-core/.claude/worktrees/agent-a8c3ad1a9d2b6944e", + "stack:agent-a8c3ad1a9d2b6944e", + "slot:0" + ], + "dur": 455.4, + "enqueued": 1794668370.0796046 + }, + { + "id": "test-39", + "seq": 200039, + "kind": "test", + "task": "t:/home/kevinveenbirkenbach/Repositories/github.com/kevinveenbirkenbach/skills/.worktrees/fix/kvb-org-queue-backfill", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 15, + "workdir": "/home/kevinveenbirkenbach/Repositories/github.com/kevinveenbirkenbach/skills/.worktrees/fix/kvb-org-queue-backfill", + "claims": [ + "/home/kevinveenbirkenbach/Repositories/github.com/kevinveenbirkenbach/skills/.worktrees/fix/kvb-org-queue-backfill" + ], + "dur": 186.6, + "enqueued": 1794668732.2392776 + }, + { + "id": "test-nostack-40", + "seq": 200040, + "kind": "test-nostack", + "task": "t:/home/kevinveenbirkenbach/Repositories/github.com/kevinveenbirkenbach/skills/.worktrees/fix/kvb-org-queue-backfill", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 0, + "workdir": "/home/kevinveenbirkenbach/Repositories/github.com/kevinveenbirkenbach/skills/.worktrees/fix/kvb-org-queue-backfill", + "claims": [ + "/home/kevinveenbirkenbach/Repositories/github.com/kevinveenbirkenbach/skills/.worktrees/fix/kvb-org-queue-backfill" + ], + "dur": 4.8, + "enqueued": 1794668732.2392776 + }, + { + "id": "test-nostack-41", + "seq": 200041, + "kind": "test-nostack", + "task": "t:/home/kevinveenbirkenbach/Repositories/github.com/kevinveenbirkenbach/skills/.worktrees/fix/kvb-org-queue-backfill", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 0, + "workdir": "/home/kevinveenbirkenbach/Repositories/github.com/kevinveenbirkenbach/skills/.worktrees/fix/kvb-org-queue-backfill", + "claims": [ + "/home/kevinveenbirkenbach/Repositories/github.com/kevinveenbirkenbach/skills/.worktrees/fix/kvb-org-queue-backfill" + ], + "dur": 5.3999999999999995, + "enqueued": 1794668758.7126744 + }, + { + "id": "test-nostack-42", + "seq": 200042, + "kind": "test-nostack", + "task": "t:/home/kevinveenbirkenbach/Repositories/github.com/kevinveenbirkenbach/skills/.worktrees/fix/kvb-org-queue-backfill", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 0, + "workdir": "/home/kevinveenbirkenbach/Repositories/github.com/kevinveenbirkenbach/skills/.worktrees/fix/kvb-org-queue-backfill", + "claims": [ + "/home/kevinveenbirkenbach/Repositories/github.com/kevinveenbirkenbach/skills/.worktrees/fix/kvb-org-queue-backfill" + ], + "dur": 5.3999999999999995, + "enqueued": 1794668788.64045 + }, + { + "id": "test-43", + "seq": 200043, + "kind": "test", + "task": "t:/home/kevinveenbirkenbach/Repositories/github.com/kevinveenbirkenbach/skills/.worktrees/fix/hub-cleanup-pkgmgr-local", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 15, + "workdir": "/home/kevinveenbirkenbach/Repositories/github.com/kevinveenbirkenbach/skills/.worktrees/fix/hub-cleanup-pkgmgr-local", + "claims": [ + "/home/kevinveenbirkenbach/Repositories/github.com/kevinveenbirkenbach/skills/.worktrees/fix/hub-cleanup-pkgmgr-local" + ], + "dur": 30.0, + "enqueued": 1794669030.6856914 + }, + { + "id": "test-44", + "seq": 200044, + "kind": "test", + "task": "t:/home/kevinveenbirkenbach/Repositories/github.com/kevinveenbirkenbach/skills/.worktrees/fix/hub-cleanup-pkgmgr-local", + "name": "other", + "main": false, + "hotfix": null, + "need_gb": 15, + "workdir": "/home/kevinveenbirkenbach/Repositories/github.com/kevinveenbirkenbach/skills/.worktrees/fix/hub-cleanup-pkgmgr-local", + "claims": [ + "/home/kevinveenbirkenbach/Repositories/github.com/kevinveenbirkenbach/skills/.worktrees/fix/hub-cleanup-pkgmgr-local" + ], + "dur": 31.200000000000003, + "enqueued": 1794669087.7331543 + } + ], + "master": { + "plain": { + "test-nostack-1838": 0.0, + "quality-high-1855": 0.0, + "test-nostack-1873": 0.0, + "test-nostack-1883": 0.19676462014516194, + "quality-high-1884": 0.0, + "test-nostack-1885": 0.0, + "test-nostack-1886": 9.666666666666666, + "test-nostack-1887": 0.0, + "test-nostack-1888": 0.11645279328028361, + "test-nostack-1895": 0.0, + "quality-high-1900": 0.0, + "quality-high-1903": 0.0, + "quality-high-1905": 10.333333333333334, + "test-1908": 10.333333333333334, + "quality-high-1912": 19.333333333333332, + "quality-high-1919": 32.666666666666664, + "iteration-1920": 33.0, + "test-1924": 33.333333333333336, + "iteration-1928": 40.666666666666664, + "test-1929": 41.0, + "quality-high-1930": 45.0, + "test-nostack-1931": 6.661867813269297, + "iteration-1932": 27.58460421959559, + "iteration-1933": 26.53111866315206, + "iteration-1934": 24.997499418258666, + "test-nostack-1935": 0.23386666774749756, + "iteration-1936": 24.759320346514382, + "test-nostack-1937": 0.027351121107737224, + "iteration-1938": 23.694017787774403, + "iteration-1939": 23.297462697823843, + "iteration-1940": 23.003037997086842, + "iteration-1941": 26.04718031088511, + "test-nostack-1942": 0.18367920319239298, + "iteration-1943": 22.517012536525726, + "iteration-1944": 22.247798359394075, + "iteration-1945": 21.749712789058684, + "iteration-1946": 21.433864164352418, + "iteration-1947": 20.92494970957438, + "test-1948": 20.859454230467477, + "test-1949": 23.192787563800813, + "iteration-1950": 22.976023197174072, + "iteration-1951": 20.220115701357525, + "iteration-1952": 19.652668404579163, + "iteration-1953": 18.48414134979248, + "test-1954": 20.510667328039805, + "test-nostack-1955": 0.17733399470647176, + "test-1956": 18.736002167065937, + "test-1957": 17.320796954631806, + "test-nostack-1958": 0.2978447993596395, + "test-1959": 14.362956321239471, + "test-1960": 13.853395867347718, + "test-1961": 12.329920585950216, + "test-1962": 11.785043843587239, + "iteration-1963": 6.387070175011953, + "iteration-1964": 0.0, + "iteration-1965": 0.0, + "iteration-1966": 0.0, + "test-nostack-1968": 0.0, + "iteration-1969": 0.0, + "iteration-1970": 0.0, + "test-nostack-1971": 0.0, + "test-nostack-1972": 0.0, + "iteration-1973": 0.0, + "test-nostack-1974": 0.0, + "iteration-1975": 0.0, + "test-nostack-1976": 0.0, + "iteration-1977": 0.004467419783274333, + "iteration-1978": 0.0, + "iteration-1979": 0.0, + "test-nostack-1980": 0.0, + "iteration-1981": 0.0, + "iteration-1982": 0.0, + "test-nostack-1983": 0.0, + "test-nostack-1984": 0.0, + "test-nostack-1985": 0.0, + "test-nostack-1986": 0.0, + "test-nostack-1987": 0.0, + "test-nostack-1988": 0.0, + "test-nostack-1989": 0.0, + "test-nostack-1990": 0.0, + "test-nostack-1991": 0.0, + "test-nostack-1992": 0.0, + "iteration-1993": 0.0, + "test-nostack-1995": 0.0, + "test-nostack-1996": 0.0, + "test-nostack-1997": 0.0, + "test-nostack-1998": 0.0, + "test-nostack-1999": 0.0, + "test-nostack-2000": 0.0, + "test-nostack-2001": 0.0, + "test-nostack-2002": 0.0, + "iteration-2003": 0.0, + "test-nostack-2004": 0.0, + "test-nostack-2005": 0.0, + "test-nostack-2006": 0.0, + "test-nostack-2007": 0.0, + "iteration-2008": 0.0, + "test-nostack-2010": 0.0, + "test-nostack-2011": 0.0, + "iteration-2012": 0.0, + "iteration-2013": 0.0, + "test-nostack-2014": 0.0, + "test-nostack-2015": 0.0, + "test-nostack-2016": 0.0, + "test-2017": 0.0, + "test-2018": 0.0, + "test-2019": 0.3333333333333333, + "quality-high-2020": 0.0, + "gui-dind-2021": 10.0, + "test-nostack-2022": 0.0555379589398702, + "test-2023": 8.108370566368103, + "test-nostack-2024": 0.10837056636810302, + "test-nostack-2025": 0.0, + "test-2026": 0.0, + "test-nostack-2030": 0.0, + "test-nostack-2031": 0.0, + "iteration-1": 0.0, + "iteration-2": 0.2620401700337728, + "quality-high-5": 0.0, + "iteration-7": 0.3333333333333333, + "iteration-9": 0.0, + "test-10": 0.0, + "iteration-11": 0.007367575168609619, + "test-12": 0.0, + "test-nostack-13": 0.0, + "test-nostack-14": 0.0, + "quality-high-15": 0.0, + "quality-high-16": 0.3333333333333333, + "test-19": 0.6666666666666666, + "test-22": 9.0, + "test-26": 10.333333333333334, + "test-27": 13.0, + "test-nostack-34": 0.0, + "test-nostack-37": 0.28902939160664876, + "test-39": 4.919701508680979, + "test-nostack-40": 0.25303484201431276, + "test-nostack-41": 0.14514489571253458, + "test-nostack-42": 0.31301530202229816, + "test-43": 3.278927946090698, + "test-44": 2.9948035637537638 + }, + "ram45": { + "test-nostack-1838": 0.0, + "quality-high-1855": null, + "test-nostack-1873": 0.0, + "test-nostack-1883": 0.19676462014516194, + "quality-high-1884": null, + "test-nostack-1885": 0.21511208216349284, + "test-nostack-1886": 9.792353713512421, + "test-nostack-1887": 0.125687046845754, + "test-nostack-1888": 0.2421398401260376, + "test-nostack-1895": 0.13381683429082233, + "quality-high-1900": null, + "quality-high-1903": null, + "quality-high-1905": null, + "test-1908": 0.2216511329015096, + "quality-high-1912": null, + "quality-high-1919": null, + "iteration-1920": 9.22165113290151, + "test-1924": 9.554984466234844, + "iteration-1928": 16.888317799568178, + "test-1929": 17.22165113290151, + "quality-high-1930": null, + "test-nostack-1931": 5.550185612837473, + "iteration-1932": 0.13958868583043416, + "iteration-1933": 9.752769796053569, + "iteration-1934": 8.219150551160176, + "test-nostack-1935": 0.12218446731567383, + "iteration-1936": 7.980971479415894, + "test-nostack-1937": 1.249002254009247, + "iteration-1938": 14.58233558734258, + "iteration-1939": 14.185780497392019, + "iteration-1940": 13.891355796655018, + "iteration-1941": 1.9354981104532878, + "test-nostack-1942": 0.07199700276056925, + "iteration-1943": 13.405330336093902, + "iteration-1944": 13.136116158962249, + "iteration-1945": 12.638030588626862, + "iteration-1946": 12.322181963920594, + "iteration-1947": 11.813267509142557, + "test-1948": 11.747772030035655, + "test-1949": 14.081105363368987, + "iteration-1950": 13.864340996742248, + "iteration-1951": 11.1084335009257, + "iteration-1952": 21.874319537480673, + "iteration-1953": 20.70579248269399, + "test-1954": 12.39898512760798, + "test-nostack-1955": 0.06565179427464803, + "test-1956": 10.624319966634115, + "test-1957": 9.209114754199982, + "test-nostack-1958": 0.18616259892781575, + "test-1959": 6.251274120807648, + "test-1960": 5.741713666915894, + "test-1961": 4.218238385518392, + "test-1962": 3.673361643155416, + "iteration-1963": 0.275387974580129, + "iteration-1964": 0.2822036981582642, + "iteration-1965": 0.2544540603955587, + "iteration-1966": 0.2716320753097534, + "test-nostack-1968": 0.11194755633672078, + "iteration-1969": 0.20736341873804728, + "iteration-1970": 0.16916441917419434, + "test-nostack-1971": 0.21192184686660767, + "test-nostack-1972": 0.14811149835586548, + "iteration-1973": 0.19052569071451822, + "test-nostack-1974": 0.090010933081309, + "iteration-1975": 0.7162841041882833, + "test-nostack-1976": 0.04757107098897298, + "iteration-1977": 1.3853718241055806, + "iteration-1978": 0.14402870734532675, + "iteration-1979": 0.04997930526733398, + "test-nostack-1980": 0.20661055246988932, + "iteration-1981": 0.20661055246988932, + "iteration-1982": 36.65448599259059, + "test-nostack-1983": 0.27543065150578816, + "test-nostack-1984": 0.1051060160001119, + "test-nostack-1985": 0.2970075289408366, + "test-nostack-1986": 0.1869705319404602, + "test-nostack-1987": 0.14138193130493165, + "test-nostack-1988": 0.19906748135884603, + "test-nostack-1989": 0.19906748135884603, + "test-nostack-1990": 0.1155814528465271, + "test-nostack-1991": 0.013187527656555176, + "test-nostack-1992": 0.3320132255554199, + "iteration-1993": 0.3155541896820068, + "test-nostack-1995": 0.3072804649670919, + "test-nostack-1996": 0.07005721728006999, + "test-nostack-1997": 0.07005721728006999, + "test-nostack-1998": 0.014388052622477214, + "test-nostack-1999": 0.03745972712834676, + "test-nostack-2000": 0.10571537415186565, + "test-nostack-2001": 0.07364315589269002, + "test-nostack-2002": 0.03222020467122396, + "iteration-2003": 0.21912326415379843, + "test-nostack-2004": 0.21515654722849528, + "test-nostack-2005": 0.11868726015090943, + "test-nostack-2006": 0.287273907661438, + "test-nostack-2007": 0.07475214004516602, + "iteration-2008": 0.979061734676361, + "test-nostack-2010": 0.3271055817604065, + "test-nostack-2011": 0.11838468313217163, + "iteration-2012": 0.18130247195561727, + "iteration-2013": 0.20834165414174396, + "test-nostack-2014": 0.20834165414174396, + "test-nostack-2015": 0.20834165414174396, + "test-nostack-2016": 0.1344983180363973, + "test-2017": 0.29334211349487305, + "test-2018": 0.28150145610173544, + "test-2019": 0.6148347894350688, + "quality-high-2020": null, + "gui-dind-2021": 0.21901243130366008, + "test-nostack-2022": 0.2745503902435303, + "test-2023": 0.3273829976717631, + "test-nostack-2024": 0.3273829976717631, + "test-nostack-2025": 0.1836197018623352, + "test-2026": 0.02752542495727539, + "test-nostack-2030": 0.15145073334376016, + "test-nostack-2031": 0.010928340752919515, + "iteration-1": 0.30237824122111, + "iteration-2": 0.23108507792154948, + "quality-high-5": null, + "iteration-7": 0.22196054855982464, + "iteration-9": 0.2200604796409607, + "test-10": 0.2057503859202067, + "iteration-11": 0.21311796108881634, + "test-12": 0.04977636734644572, + "test-nostack-13": 0.138780415058136, + "test-nostack-14": 0.13600229024887084, + "quality-high-15": null, + "quality-high-16": null, + "test-19": 0.1291037917137146, + "test-22": 8.537528459231059, + "test-26": 10.537528459231059, + "test-27": 18.204195125897726, + "test-nostack-34": 8.537528459231059, + "test-nostack-37": 7.493224517504374, + "test-39": 10.123896634578704, + "test-nostack-40": 0.7905633012453716, + "test-nostack-41": 0.6826733549435934, + "test-nostack-42": 0.5172104279200236, + "test-43": 8.483123071988423, + "test-44": 8.19899868965149 + }, + "hotfix": { + "test-nostack-1838": 0.0, + "quality-high-1855": 0.0, + "test-nostack-1873": 0.0, + "test-nostack-1883": 0.19676462014516194, + "quality-high-1884": 0.0, + "test-nostack-1885": 0.0, + "test-nostack-1886": 9.666666666666666, + "test-nostack-1887": 0.0, + "test-nostack-1888": 0.11645279328028361, + "test-nostack-1895": 0.0, + "quality-high-1900": 0.0, + "quality-high-1903": 0.0, + "quality-high-1905": 10.333333333333334, + "test-1908": 27.666666666666668, + "quality-high-1912": 36.666666666666664, + "quality-high-1919": 50.0, + "iteration-1920": 50.333333333333336, + "test-1924": 50.666666666666664, + "iteration-1928": 58.0, + "test-1929": 58.333333333333336, + "quality-high-1930": 10.333333333333334, + "test-nostack-1931": 6.661867813269297, + "iteration-1932": 27.58460421959559, + "iteration-1933": 26.53111866315206, + "iteration-1934": 24.997499418258666, + "test-nostack-1935": 0.23386666774749756, + "iteration-1936": 24.759320346514382, + "test-nostack-1937": 0.027351121107737224, + "iteration-1938": 23.694017787774403, + "iteration-1939": 23.297462697823843, + "iteration-1940": 23.003037997086842, + "iteration-1941": 26.04718031088511, + "test-nostack-1942": 0.18367920319239298, + "iteration-1943": 22.517012536525726, + "iteration-1944": 22.247798359394075, + "iteration-1945": 21.749712789058684, + "iteration-1946": 21.433864164352418, + "iteration-1947": 20.92494970957438, + "test-1948": 20.859454230467477, + "test-1949": 23.192787563800813, + "iteration-1950": 22.976023197174072, + "iteration-1951": 20.220115701357525, + "iteration-1952": 19.652668404579163, + "iteration-1953": 18.48414134979248, + "test-1954": 20.510667328039805, + "test-nostack-1955": 0.17733399470647176, + "test-1956": 18.736002167065937, + "test-1957": 17.320796954631806, + "test-nostack-1958": 0.2978447993596395, + "test-1959": 14.362956321239471, + "test-1960": 13.853395867347718, + "test-1961": 12.329920585950216, + "test-1962": 11.785043843587239, + "iteration-1963": 6.387070175011953, + "iteration-1964": 0.0, + "iteration-1965": 0.0, + "iteration-1966": 0.0, + "test-nostack-1968": 0.0, + "iteration-1969": 0.0, + "iteration-1970": 0.0, + "test-nostack-1971": 0.0, + "test-nostack-1972": 0.0, + "iteration-1973": 0.0, + "test-nostack-1974": 0.0, + "iteration-1975": 0.0, + "test-nostack-1976": 0.0, + "iteration-1977": 0.004467419783274333, + "iteration-1978": 0.0, + "iteration-1979": 0.0, + "test-nostack-1980": 0.0, + "iteration-1981": 0.0, + "iteration-1982": 0.0, + "test-nostack-1983": 0.0, + "test-nostack-1984": 0.0, + "test-nostack-1985": 0.0, + "test-nostack-1986": 0.0, + "test-nostack-1987": 0.0, + "test-nostack-1988": 0.0, + "test-nostack-1989": 0.0, + "test-nostack-1990": 0.0, + "test-nostack-1991": 0.0, + "test-nostack-1992": 0.0, + "iteration-1993": 0.0, + "test-nostack-1995": 0.0, + "test-nostack-1996": 0.0, + "test-nostack-1997": 0.0, + "test-nostack-1998": 0.0, + "test-nostack-1999": 0.0, + "test-nostack-2000": 0.0, + "test-nostack-2001": 0.0, + "test-nostack-2002": 0.0, + "iteration-2003": 0.0, + "test-nostack-2004": 0.0, + "test-nostack-2005": 0.0, + "test-nostack-2006": 0.0, + "test-nostack-2007": 0.0, + "iteration-2008": 0.0, + "test-nostack-2010": 0.0, + "test-nostack-2011": 0.0, + "iteration-2012": 0.0, + "iteration-2013": 0.0, + "test-nostack-2014": 0.0, + "test-nostack-2015": 0.0, + "test-nostack-2016": 0.0, + "test-2017": 0.0, + "test-2018": 0.0, + "test-2019": 0.3333333333333333, + "quality-high-2020": 0.0, + "gui-dind-2021": 10.0, + "test-nostack-2022": 0.0555379589398702, + "test-2023": 8.108370566368103, + "test-nostack-2024": 0.10837056636810302, + "test-nostack-2025": 0.0, + "test-2026": 0.0, + "test-nostack-2030": 0.0, + "test-nostack-2031": 0.0, + "iteration-1": 0.0, + "iteration-2": 0.2620401700337728, + "quality-high-5": 0.0, + "iteration-7": 0.3333333333333333, + "iteration-9": 0.0, + "test-10": 0.0, + "iteration-11": 0.007367575168609619, + "test-12": 0.0, + "test-nostack-13": 0.0, + "test-nostack-14": 0.0, + "quality-high-15": 0.3333333333333333, + "quality-high-16": 0.0, + "test-19": 0.6666666666666666, + "test-22": 9.0, + "test-26": 10.333333333333334, + "test-27": 13.0, + "test-nostack-34": 0.0, + "test-nostack-37": 0.28902939160664876, + "test-39": 4.919701508680979, + "test-nostack-40": 0.25303484201431276, + "test-nostack-41": 0.14514489571253458, + "test-nostack-42": 0.31301530202229816, + "test-43": 3.278927946090698, + "test-44": 2.9948035637537638 + }, + "lines": { + "test-nostack-1838": 0.0, + "quality-high-1855": 0.0, + "test-nostack-1873": 0.0, + "test-nostack-1883": 0.19676462014516194, + "quality-high-1884": 0.0, + "test-nostack-1885": 0.0, + "test-nostack-1886": 9.666666666666666, + "test-nostack-1887": 0.0, + "test-nostack-1888": 0.11645279328028361, + "test-nostack-1895": 0.0, + "quality-high-1900": 0.0, + "quality-high-1903": 0.0, + "quality-high-1905": 10.333333333333334, + "quality-high-1930-fill0": 0.0, + "quality-high-1930-fill1": 0.0, + "quality-high-1930-fill2": 0.0, + "quality-high-1930-fill3": 0.0, + "quality-high-1930-older-light": 28.666666666666668, + "quality-high-1930-older-gate": 37.166666666666664, + "test-1908": 27.666666666666668, + "quality-high-1912": 54.0, + "quality-high-1919": 67.33333333333333, + "iteration-1920": 67.66666666666667, + "test-1924": 68.0, + "iteration-1928": 75.33333333333333, + "test-1929": 75.66666666666667, + "quality-high-1930": 10.333333333333334, + "test-nostack-1931": 9.99520114660263, + "iteration-1932": 44.917937552928926, + "iteration-1933": 43.86445199648539, + "iteration-1934": 42.330832751592, + "test-nostack-1935": 0.23386666774749756, + "iteration-1936": 42.092653679847714, + "test-nostack-1937": 0.027351121107737224, + "iteration-1938": 41.027351121107735, + "iteration-1939": 40.630796031157175, + "iteration-1940": 40.33637133042018, + "iteration-1941": 43.380513644218446, + "test-nostack-1942": 0.18367920319239298, + "iteration-1943": 39.85034586985906, + "iteration-1944": 39.5811316927274, + "iteration-1945": 39.08304612239202, + "iteration-1946": 38.76719749768575, + "iteration-1947": 38.25828304290771, + "test-1948": 38.19278756380081, + "test-1949": 40.52612089713414, + "iteration-1950": 40.30935653050741, + "iteration-1951": 37.55344903469086, + "iteration-1952": 36.986001737912495, + "iteration-1953": 35.817474683125816, + "test-1954": 37.84400066137314, + "test-nostack-1955": 0.17733399470647176, + "test-1956": 36.06933550039927, + "test-1957": 34.65413028796514, + "test-nostack-1958": 0.2978447993596395, + "test-1959": 31.696289654572805, + "test-1960": 31.18672920068105, + "test-1961": 29.663253919283548, + "test-1962": 29.11837717692057, + "iteration-1963": 23.720403508345285, + "iteration-1964": 17.333333333333332, + "iteration-1965": 14.305583695570627, + "iteration-1966": 8.0, + "test-nostack-1968": 0.0, + "iteration-1969": 0.3333333333333333, + "iteration-1970": 0.0, + "test-nostack-1971": 0.0, + "test-nostack-1972": 0.0, + "iteration-1973": 0.0, + "test-nostack-1974": 0.0, + "iteration-1975": 0.0, + "test-nostack-1976": 0.0, + "iteration-1977": 0.004467419783274333, + "iteration-1978": 0.0, + "iteration-1979": 0.0, + "test-nostack-1980": 0.0, + "iteration-1981": 0.0, + "iteration-1982": 0.0, + "test-nostack-1983": 0.0, + "test-nostack-1984": 0.0, + "test-nostack-1985": 0.0, + "test-nostack-1986": 0.0, + "test-nostack-1987": 0.0, + "test-nostack-1988": 0.0, + "test-nostack-1989": 0.0, + "test-nostack-1990": 0.0, + "test-nostack-1991": 0.0, + "test-nostack-1992": 0.0, + "iteration-1993": 0.0, + "test-nostack-1995": 0.0, + "test-nostack-1996": 0.0, + "test-nostack-1997": 0.0, + "test-nostack-1998": 0.0, + "test-nostack-1999": 0.0, + "test-nostack-2000": 0.0, + "test-nostack-2001": 0.0, + "test-nostack-2002": 0.0, + "iteration-2003": 0.0, + "test-nostack-2004": 0.0, + "test-nostack-2005": 0.0, + "test-nostack-2006": 0.0, + "test-nostack-2007": 0.0, + "iteration-2008": 0.0, + "test-nostack-2010": 0.0, + "test-nostack-2011": 0.0, + "iteration-2012": 0.0, + "iteration-2013": 0.0, + "test-nostack-2014": 0.0, + "test-nostack-2015": 0.0, + "test-nostack-2016": 0.0, + "test-2017": 0.0, + "test-2018": 0.0, + "test-2019": 0.3333333333333333, + "quality-high-2020": 0.0, + "gui-dind-2021": 10.0, + "test-nostack-2022": 0.0555379589398702, + "test-2023": 8.108370566368103, + "test-nostack-2024": 0.10837056636810302, + "test-nostack-2025": 0.0, + "test-2026": 0.0, + "test-nostack-2030": 0.0, + "test-nostack-2031": 0.0, + "iteration-1": 0.0, + "iteration-2": 0.2620401700337728, + "quality-high-5": 0.0, + "iteration-7": 0.3333333333333333, + "iteration-9": 0.0, + "test-10": 0.0, + "iteration-11": 0.007367575168609619, + "test-12": 0.0, + "test-nostack-13": 0.0, + "test-nostack-14": 0.0, + "quality-high-16-fill0": 0.0, + "quality-high-16-fill1": 0.0, + "quality-high-16-fill2": 0.0, + "quality-high-16-fill3": 0.0, + "quality-high-16-older-light": 14.666666666666666, + "quality-high-16-older-gate": 0.16666666666666666, + "quality-high-15": 0.3333333333333333, + "quality-high-16": 0.0, + "test-19": 0.6666666666666666, + "test-22": 9.0, + "test-26": 10.333333333333334, + "test-27": 13.0, + "test-nostack-34": 0.0, + "test-nostack-37": 0.28902939160664876, + "test-39": 4.919701508680979, + "test-nostack-40": 0.25303484201431276, + "test-nostack-41": 0.14514489571253458, + "test-nostack-42": 0.31301530202229816, + "test-43": 3.278927946090698, + "test-44": 2.9948035637537638 + }, + "real": { + "test-nostack-1838": 0.0, + "quality-high-1855": 0.0, + "test-nostack-1873": 0.0, + "test-nostack-1883": 0.19676462014516194, + "quality-high-1884": 0.0, + "test-nostack-1885": 0.0, + "test-nostack-1886": 9.666666666666666, + "test-nostack-1887": 0.0, + "test-nostack-1888": 0.11645279328028361, + "test-nostack-1895": 0.0, + "quality-high-1900": 0.0, + "quality-high-1903": 40.666666666666664, + "quality-high-1905": 0.0, + "test-1908": 0.00587536096572876, + "quality-high-1912": 9.005875360965728, + "quality-high-1919": 22.339208694299064, + "iteration-1920": 104.33920869429906, + "test-1924": 97.00587536096573, + "iteration-1928": 113.67254202763239, + "test-1929": 93.00587536096573, + "quality-high-1930": 33.00587536096573, + "test-nostack-1931": 0.001076507568359375, + "iteration-1932": 15.59047958056132, + "iteration-1933": 78.20366069078446, + "iteration-1934": 81.0033747792244, + "test-nostack-1935": 0.2397420287132263, + "iteration-1936": 85.76519570748012, + "test-nostack-1937": 0.033226482073465985, + "iteration-1938": 84.69989314874013, + "iteration-1939": 84.30333805878958, + "iteration-1940": 84.00891335805257, + "iteration-1941": 14.05305567185084, + "test-nostack-1942": 0.18955456415812175, + "iteration-1943": 83.52288789749146, + "iteration-1944": 83.2536737203598, + "iteration-1945": 82.75558815002441, + "iteration-1946": 82.43973952531815, + "iteration-1947": 81.93082507054011, + "test-1948": 40.19866292476654, + "test-1949": 6.531996258099874, + "iteration-1950": 6.315231891473134, + "iteration-1951": 58.22599106232325, + "iteration-1952": 2.991877098878225, + "iteration-1953": 1.8233500440915427, + "test-1954": 7.516542689005534, + "test-nostack-1955": 0.1832093556722005, + "test-1956": 5.741877528031667, + "test-1957": 4.326672315597534, + "test-nostack-1958": 0.30372016032536825, + "test-1959": 1.3688316822052002, + "test-1960": 0.859271228313446, + "test-1961": 0.002462613582611084, + "test-1962": 2.457585871219635, + "iteration-1963": 0.05961220264434815, + "iteration-1964": 0.0, + "iteration-1965": 0.0, + "iteration-1966": 0.0, + "test-nostack-1968": 0.0, + "iteration-1969": 0.0, + "iteration-1970": 0.0, + "test-nostack-1971": 0.0, + "test-nostack-1972": 0.0, + "iteration-1973": 0.0, + "test-nostack-1974": 0.0, + "iteration-1975": 0.0, + "test-nostack-1976": 0.0, + "iteration-1977": 0.004467419783274333, + "iteration-1978": 0.0, + "iteration-1979": 0.0, + "test-nostack-1980": 60.0, + "iteration-1981": 60.0, + "iteration-1982": 52.78120877345403, + "test-nostack-1983": 43.40215343236923, + "test-nostack-1984": 26.56516213019689, + "test-nostack-1985": 21.757063643137613, + "test-nostack-1986": 21.313693312803903, + "test-nostack-1987": 20.93477137883504, + "test-nostack-1988": 19.992456928888956, + "test-nostack-1989": 18.32579026222229, + "test-nostack-1990": 17.908970900376637, + "test-nostack-1991": 16.806576975186665, + "test-nostack-1992": 15.792069339752198, + "iteration-1993": 47.108943637212114, + "test-nostack-1995": 0.10066991249720256, + "test-nostack-1996": 0.196779998143514, + "test-nostack-1997": 0.196779998143514, + "test-nostack-1998": 0.1411108334859212, + "test-nostack-1999": 0.16418250799179077, + "test-nostack-2000": 0.23243815501530965, + "test-nostack-2001": 0.20036593675613404, + "test-nostack-2002": 0.0, + "iteration-2003": 44.333333333333336, + "test-nostack-2004": 0.0, + "test-nostack-2005": 0.0, + "test-nostack-2006": 0.0, + "test-nostack-2007": 0.0, + "iteration-2008": 39.0, + "test-nostack-2010": 0.014710513750712077, + "test-nostack-2011": 0.13932294845581056, + "iteration-2012": 0.20224073727925618, + "iteration-2013": 0.0, + "test-nostack-2014": 0.0, + "test-nostack-2015": 0.0, + "test-nostack-2016": 0.0, + "test-2017": 0.0, + "test-2018": 2.0, + "test-2019": 0.0, + "quality-high-2020": 0.0, + "gui-dind-2021": 10.333333333333334, + "test-nostack-2022": 0.0555379589398702, + "test-2023": 6.108370566368103, + "test-nostack-2024": 0.10837056636810302, + "test-nostack-2025": 0.0, + "test-2026": 0.0, + "test-nostack-2030": 0.0, + "test-nostack-2031": 0.0, + "iteration-1": 0.0, + "iteration-2": 0.2620401700337728, + "quality-high-5": 0.0, + "iteration-7": 0.3333333333333333, + "iteration-9": 0.0, + "test-10": 0.0, + "iteration-11": 0.007367575168609619, + "test-12": 0.0, + "test-nostack-13": 0.0, + "test-nostack-14": 0.0, + "quality-high-15": 0.3333333333333333, + "quality-high-16": 0.0, + "test-19": 0.6666666666666666, + "test-22": 12.666666666666666, + "test-26": 9.0, + "test-27": 11.666666666666666, + "test-nostack-34": 0.0, + "test-nostack-37": 0.28902939160664876, + "test-39": 4.919701508680979, + "test-nostack-40": 0.25303484201431276, + "test-nostack-41": 0.14514489571253458, + "test-nostack-42": 0.31301530202229816, + "test-43": 3.278927946090698, + "test-44": 2.9948035637537638 + } + } +} diff --git a/tests/test_queue.py b/tests/test_queue.py index 9d9cec3..b812fd5 100644 --- a/tests/test_queue.py +++ b/tests/test_queue.py @@ -2,8 +2,10 @@ from __future__ import annotations +import contextlib import fcntl import importlib.util +import io import json import os import shutil @@ -37,6 +39,7 @@ BASE = { "loaded": {}, "exclusive": set(), "light": set(), + "drain": None, "now": NOW, } CHILD = ( @@ -76,6 +79,10 @@ def job(seq: int, **fields: object) -> dict: } +def idle() -> tuple[bool, list]: + return False, [] + + def stored(seq: int, status: str, pid: int, pid_start: str | None, child: int | None, child_start: str | None) -> dict: return { @@ -86,23 +93,31 @@ def stored(seq: int, status: str, pid: int, pid_start: str | None, child: int | } -class TestEvaluate(unittest.TestCase): +class Passes: @classmethod def setUpClass(cls): cls.queue = _load(QUEUE) - def decide(self, entries: list[dict], **overrides: object) -> dict: + def run_pass(self, entries: list[dict], **overrides: object) -> tuple[dict, str | None, list[dict]]: footprints = {e["id"]: {"ram": 0, "cpu": 0} for e in entries} | overrides.pop("footprints", {}) return self.queue.evaluate(entries, **{**BASE, "footprints": footprints, **overrides}) - def test_the_budget_refuses_a_job_whose_share_would_pass_it_and_reserves_it(self): + def decide(self, entries: list[dict], **overrides: object) -> dict: + return self.run_pass(entries, **overrides)[0] + + +class TestEvaluate(Passes, unittest.TestCase): + def test_the_budget_refuses_a_job_whose_share_would_pass_it_and_a_smaller_one_backfills(self): running = job(1, kind="test", status="active") candidate = job(2, kind="gui-dind", workdir="/x") small = job(3, task="work/b", workdir="/y") shares = {"j1": {"ram": 60, "cpu": 40}, "j2": {"ram": 30, "cpu": 20}, "j3": {"ram": 10, "cpu": 5}} verdicts = self.decide([running, candidate, small], slot_limits={"test": 5, "gui-dind": 1}, footprints=shares) - self.assertEqual(verdicts["j2"], "RAM 60 % in use or reserved + 30 % > budget 75 %; reserved") - self.assertEqual(verdicts["j3"], "RAM 90 % in use or reserved + 10 % > budget 75 %; reserved") + self.assertEqual(verdicts["j2"], "RAM 60 % in use or reserved + 30 % > budget 75 %") + self.assertIsNone(verdicts["j3"]) + fat = {**shares, "j3": {"ram": 16, "cpu": 5}} + verdicts = self.decide([running, candidate, small], slot_limits={"test": 5, "gui-dind": 1}, footprints=fat) + self.assertEqual(verdicts["j3"], "RAM 60 % in use or reserved + 16 % > budget 75 %") unbounded = {"ram": 1000, "cpu": 1000} self.assertEqual( self.decide([running, candidate, small], slot_limits={"test": 5, "gui-dind": 1}, footprints=shares, budget=unbounded), @@ -113,12 +128,12 @@ class TestEvaluate(unittest.TestCase): alone = self.decide([job(1)], footprints={"j1": {"ram": 90, "cpu": 150}}) self.assertEqual(alone, {"j1": None}) busy = self.decide([job(1, status="active"), job(2)], footprints={"j1": {"ram": 1, "cpu": 70}, "j2": {"ram": 1, "cpu": 40}}) - self.assertEqual(busy, {"j2": "CPU 70 % in use or reserved + 40 % > budget 100 %; reserved"}) + self.assertEqual(busy, {"j2": "CPU 70 % in use or reserved + 40 % > budget 100 %"}) def test_a_share_beyond_the_budget_runs_only_alone_even_beside_a_job_of_no_share(self): whole = {"ram": 80, "cpu": 1} beside = self.decide([job(1, status="active"), job(2)], footprints={"j1": {"ram": 0, "cpu": 0}, "j2": whole}) - self.assertEqual(beside, {"j2": "RAM 0 % in use or reserved + 80 % > budget 75 %; reserved"}) + self.assertEqual(beside, {"j2": "RAM 0 % in use or reserved + 80 % > budget 75 %"}) self.assertEqual(self.decide([job(2)], footprints={"j2": whole}), {"j2": None}) def test_load_counts_the_ram_in_use_plus_what_is_still_to_come(self): @@ -309,18 +324,21 @@ class TestEvaluate(unittest.TestCase): {"j1": "quality-high runs only beside test-nostack, but iteration runs"}, ) - def test_an_exclusive_entry_kept_out_by_a_running_job_holds_back_lower_non_light_entries(self): + def test_r5_an_exclusive_entry_kept_out_by_a_running_job_holds_back_lower_non_light_entries_at_once(self): gates = { "exclusive": {"quality-high"}, "light": {"test-nostack"}, "slot_limits": {"quality-high": 1, "iteration": 2, "test-nostack": 2}, } running = job(1, kind="iteration", status="active") gate, deploy, light = job(2, kind="quality-high"), job(3, kind="iteration", task="work/b"), job(4, kind="test-nostack", task="work/c") - self.assertEqual(self.decide([running, gate, deploy, light], **gates), { - "j2": "quality-high runs only beside test-nostack, but iteration runs", - "j3": "waits behind j2, which needs the host beside light kinds only", - "j4": None, - }) + held = {"j2": "quality-high runs only beside test-nostack, but iteration runs", "j3": "drained for j2", "j4": None} + verdicts, drain, _ = self.run_pass([running, gate, deploy, light], **gates) + self.assertEqual((verdicts, drain), (held, None)) + self.assertEqual(self.decide([running, gate, deploy, light], drain="j2", **gates), held) + greedy = {"j1": {"ram": 30, "cpu": 30}, "j2": {"ram": 40, "cpu": 40}, "j3": {"ram": 5, "cpu": 5}, "j4": {"ram": 30, "cpu": 5}} + self.assertEqual(self.decide([running, gate, deploy, light], footprints=greedy, **gates)["j4"], None) + self.assertEqual(self.decide([running, gate, deploy, light], footprints=greedy | {"j4": {"ram": 40, "cpu": 70}}, **gates)["j4"], + "drained for j2") self.assertEqual( self.decide([gate, deploy, light], **gates), {"j2": None, "j3": "quality-high runs only beside test-nostack", "j4": None}, @@ -350,11 +368,11 @@ class TestEvaluate(unittest.TestCase): self.queue.rejoin(data, polling, NOW) self.assertEqual((polling["seq"], polling["enqueued"], polling["polled"], data["seq"]), (3, NOW - HOUR, NOW, 8)) - def test_disk_reservation_keeps_smaller_jobs_from_starving_a_big_one(self): + def test_a_smaller_job_backfills_past_a_big_head_the_disk_floor_blocks(self): big, small = job(1, need_gb=50), job(2, task="work/b", need_gb=5) - verdicts = self.decide([big, small], free_gb=40.0) - self.assertEqual(verdicts["j1"], "50 GB would cut into the 10 GB floor; reserved") - self.assertEqual(verdicts["j2"], "5 GB would cut into the 10 GB floor; reserved") + verdicts, drain, _ = self.run_pass([big, small], free_gb=40.0) + self.assertEqual(verdicts, {"j1": "50 GB would cut into the 10 GB floor", "j2": None}) + self.assertIsNone(drain) self.assertIsNone(self.decide([small], free_gb=40.0)["j2"]) running = job(3, need_gb=25, status="active") self.assertIsNone(self.decide([running, job(4, need_gb=5)], free_gb=40.0)["j4"]) @@ -377,38 +395,33 @@ class TestEvaluate(unittest.TestCase): job(1, workdir="/w", status="active"), job(2, workdir="/w"), job(3, workdir="/x"), job(4, task="work/b", workdir="/x"), ]) - self.assertEqual(verdicts, {"j2": "/w runs another job", "j3": None, "j4": "/x runs another job"}) + self.assertEqual(verdicts, {"j2": "waits in line behind j1", "j3": None, "j4": "waits in line behind j3"}) def test_a_hotfix_the_host_load_holds_back_holds_back_every_entry_after_it_and_a_normal_one_does_not(self): load = "RAM 55 % used + 42 % still to come reach the limit 90 %" hotfix, light = job(1, hotfix="main-red"), job(2, task="work/b") - self.assertEqual( - self.decide([hotfix, light], loaded={"j1": load}), - {"j1": load, "j2": "waits behind hotfix j1, which the host load holds back"}, - ) + self.assertEqual(self.decide([hotfix, light], loaded={"j1": load}), {"j1": load, "j2": "waits behind hotfix j1"}) self.assertEqual(self.decide([job(1), light], loaded={"j1": load}), {"j1": load, "j2": None}) self.assertEqual(self.decide([hotfix, light]), {"j1": None, "j2": None}) - def test_with_no_job_active_a_head_the_host_load_alone_keeps_out_reserves_nothing_and_with_one_active_it_does(self): + def test_r5_a_head_the_host_load_keeps_out_reserves_nothing_and_exclusive_kinds_share_no_line(self): load = "RAM 55 % used + 41 % still to come reach the limit 90 %" head, twin = job(1, kind="quality-high"), job(2, kind="quality-high", task="work/b") gate, light = job(3, task="work/b"), job(4, kind="test-nostack", task="work/c") shares = {"j1": {"ram": 41, "cpu": 62}, "j2": {"ram": 41, "cpu": 62}, "j3": {"ram": 20, "cpu": 40}, "j4": {"ram": 6, "cpu": 40}} rules = {"slot_limits": {"quality-high": 1, "test": 5, "test-nostack": 6}, "exclusive": {"quality-high", "test"}, "light": {"test-nostack"}} loaded = {"j1": load, "j2": load} - self.assertEqual( - self.decide([head, twin, gate, light], loaded=loaded, footprints=shares, **rules), - {"j1": load, "j2": load, "j3": None, "j4": None}, - ) + line = {"j2": load, "j3": None, "j4": None} + self.assertEqual(self.decide([head, twin, gate, light], loaded=loaded, footprints=shares, **rules), {"j1": load, **line}) running = job(5, kind="test-nostack", task="work/c", status="active") self.assertEqual( self.decide([running, head, twin, gate, light], loaded=loaded, footprints=shares | {"j5": {"ram": 6, "cpu": 11}}, **rules), - {"j1": None, "j2": "all 1 quality-high slots taken", "j3": "test runs only beside test-nostack, but quality-high runs", "j4": None}, + {"j1": load, **line}, ) - def test_a_light_entry_meets_only_running_and_granted_jobs_while_heavy_ones_keep_every_reservation(self): + def test_a_blocked_entry_reserves_nothing_for_light_and_heavy_kinds_alike(self): rules = {"slot_limits": {"test": 5, "test-nostack": 5, "deploy": 1}, "light": {"test-nostack"}} - with self.subTest("budget reservations hold back heavy kinds only"): + with self.subTest("budget"): running, head, twin = job(1, status="active"), job(2), job(3, task="work/b") light, heavy = job(4, kind="test-nostack", task="work/c", need_gb=0), job(5, task="work/c") shares = { @@ -416,35 +429,723 @@ class TestEvaluate(unittest.TestCase): "j4": {"ram": 6, "cpu": 11}, "j5": {"ram": 5, "cpu": 5}, } self.assertEqual(self.decide([running, head, twin, light, heavy], footprints=shares, **rules), { - "j2": "RAM 30 % in use or reserved + 50 % > budget 75 %, CPU 50 % in use or reserved + 60 % > budget 100 %; reserved", - "j3": "RAM 80 % in use or reserved + 50 % > budget 75 %, CPU 110 % in use or reserved + 60 % > budget 100 %; reserved", + "j2": "RAM 30 % in use or reserved + 50 % > budget 75 %, CPU 50 % in use or reserved + 60 % > budget 100 %", + "j3": "RAM 30 % in use or reserved + 50 % > budget 75 %, CPU 50 % in use or reserved + 60 % > budget 100 %", "j4": None, - "j5": "RAM 136 % in use or reserved + 5 % > budget 75 %, CPU 181 % in use or reserved + 5 % > budget 100 %; reserved", + "j5": None, }) - with self.subTest("disk reservations hold back heavy kinds only"): + with self.subTest("disk"): big, small, late = job(1, need_gb=50), job(2, kind="test-nostack", task="work/b", need_gb=0), job(3, task="work/c", need_gb=5) self.assertEqual(self.decide([big, small, late], free_gb=40.0, **rules), { - "j1": "50 GB would cut into the 10 GB floor; reserved", "j2": None, "j3": "5 GB would cut into the 10 GB floor; reserved", + "j1": "50 GB would cut into the 10 GB floor", "j2": None, "j3": None, }) - with self.subTest("a granted light counts for later lights, a blocked one reserves against heavy kinds"): + with self.subTest("a granted light counts for every later entry"): running = job(1, status="active") first, second = job(2, kind="test-nostack", need_gb=0), job(3, kind="test-nostack", task="work/b", need_gb=0) heavy = job(4, task="work/c") shares = {"j1": {"ram": 10, "cpu": 60}, "j2": {"ram": 0, "cpu": 30}, "j3": {"ram": 0, "cpu": 30}, "j4": {"ram": 5, "cpu": 5}} self.assertEqual(self.decide([running, first, second, heavy], footprints=shares, **rules), { - "j2": None, "j3": "CPU 90 % in use or reserved + 30 % > budget 100 %; reserved", - "j4": "CPU 120 % in use or reserved + 5 % > budget 100 %; reserved", + "j2": None, "j3": "CPU 90 % in use or reserved + 30 % > budget 100 %", "j4": None, }) - with self.subTest("a head the host load keeps out while a job runs reserves against heavy kinds only"): + with self.subTest("a head the host load keeps out while a job runs"): load = "RAM 55 % used + 41 % still to come reach the limit 90 %" running, head = job(1, kind="test-nostack", status="active", need_gb=0), job(2, kind="deploy") heavy, light = job(3, task="work/b"), job(4, kind="test-nostack", task="work/c", need_gb=0) shares = {"j1": {"ram": 0, "cpu": 10}, "j2": {"ram": 0, "cpu": 60}, "j3": {"ram": 0, "cpu": 40}, "j4": {"ram": 0, "cpu": 40}} self.assertEqual(self.decide([running, head, heavy, light], loaded={"j2": load}, footprints=shares, **rules), { - "j2": None, "j3": "CPU 70 % in use or reserved + 40 % > budget 100 %; reserved", "j4": None, + "j2": load, "j3": None, "j4": None, }) +class TestBackfill(Passes, unittest.TestCase): + def test_the_live_case_small_jobs_backfill_past_a_65_gb_head_and_the_head_takes_the_next_capacity(self): + rules = {"floor": 15.0, "slot_limits": {"gui-dind": 1, "iteration": 8}} + head = job(1926, kind="gui-dind", need_gb=65, workdir="/store") + small = [job(n, kind="iteration", task="work/b", need_gb=gb) for n, gb in enumerate((9, 12, 20, 15, 10, 18, 14, 11), 1)] + + def queue(running: set[int], waiting: list[int]) -> list[dict]: + return [head] + [{**small[n - 1], "status": "active", "granted": NOW - 60} for n in running] + [ + small[n - 1] for n in waiting + ] + + verdicts, drain, _ = self.run_pass(queue(set(), list(range(1, 9))), free_gb=47.0, **rules) + self.assertEqual(self.queue.labelled(verdicts, []), { + "j1926": "65 GB would cut into the 15 GB floor", "j1": "backfilled", "j2": "backfilled", + "j3": "20 GB would cut into the 15 GB floor", "j4": "15 GB would cut into the 15 GB floor", "j5": "backfilled", + "j6": "18 GB would cut into the 15 GB floor", "j7": "14 GB would cut into the 15 GB floor", + "j8": "11 GB would cut into the 15 GB floor", + }) + self.assertIsNone(drain) + verdicts, drain, order = self.run_pass(queue({1, 2, 5}, [3, 4, 6, 7, 8]), free_gb=47.0, **rules) + self.assertNotIn(None, verdicts.values()) + self.assertIsNone(drain, "47 GB free never holds 65 GB above the floor, so draining would only idle the host") + self.assertEqual(self.queue.stalled({"entries": queue({1, 2, 5}, [])}, order, verdicts, 15.0, 47.0, lambda: (False, []), NOW)[:2], ([], True)) + verdicts, drain, _ = self.run_pass(queue({1, 2, 5}, [3, 4, 6, 7, 8]), free_gb=85.0, **rules) + self.assertEqual((verdicts["j3"], verdicts["j4"], drain), (None, None, None)) + verdicts, drain, _ = self.run_pass(queue({1, 2, 3, 4, 5}, [6, 7, 8]), free_gb=85.0, **rules) + self.assertNotIn(None, verdicts.values()) + self.assertEqual(drain, "j1926") + backfill = self.decide(queue({2, 3, 4, 5}, [6, 7, 8]), free_gb=85.0, **rules) + drained = self.decide(queue({2, 3, 4, 5}, [6, 7, 8]), free_gb=85.0, drain="j1926", **rules) + self.assertEqual((backfill["j8"], drained["j8"]), (None, "drained for j1926")) + self.assertEqual(drained["j6"], "18 GB would cut into the 15 GB floor") + verdicts, _, order = self.run_pass(queue(set(), [6, 7, 8]), free_gb=85.0, drain="j1926", **rules) + self.assertEqual((order[0]["id"], verdicts["j1926"]), ("j1926", None)) + + def test_a_line_runs_strictly_in_order_by_worktree_and_stack_and_exclusive_kinds_form_none(self): + load = "CPU 95 % busy reaches the limit 90 %" + disk, older = job(13, status="active", workdir="/z", need_gb=10), job(1, workdir="/core", need_gb=25) + younger, other, last = job(2, task="work/b", workdir="/core"), job(3, task="work/b", workdir="/x"), job(4, task="work/c", workdir="/core") + self.assertEqual(self.decide([disk, older, younger, other, last], free_gb=40.0), { + "j1": "25 GB would cut into the 10 GB floor", "j2": "waits in line behind j1", "j3": None, + "j4": "waits in line behind j1", + }) + self.assertEqual(self.decide([{**older, "need_gb": 50}, younger, other, last], free_gb=40.0), { + "j1": "50 GB would cut into the 10 GB floor", "j2": None, "j3": None, "j4": "waits in line behind j2", + }) + stacked = [job(5, claims=["/a", "stack:core"]), job(6, task="work/b", claims=["/b", "stack:core"]), job(7, task="work/b", claims=["/c"])] + self.assertEqual(self.decide(stacked, loaded={"j5": load}), {"j5": load, "j6": "waits in line behind j5", "j7": None}) + rules = {"exclusive": {"test"}, "slot_limits": {"test": 5, "deploy": 1}} + machine = [job(8, workdir="/g"), job(9, task="work/b", workdir="/h"), job(10, kind="deploy", task="work/c", workdir="/i")] + self.assertEqual(self.decide(machine, loaded={"j8": load}, **rules), {"j8": load, "j9": None, "j10": "test runs only beside nothing"}) + running = [job(11, workdir="/core", status="active"), job(12, task="work/b", workdir="/core", hotfix="fix-1")] + self.assertEqual(self.decide(running), {"j12": "waits in line behind j11"}) + + def test_r5_a_blocked_hotfix_head_that_could_start_is_passed_only_by_light_jobs_taking_nothing_it_lacks(self): + rules = {"free_gb": 40.0, "light": {"test-nostack"}, "slot_limits": {"test": 5, "test-nostack": 5}} + light = job(3, kind="test-nostack", task="work/c", need_gb=0, workdir="/l") + hotfix, small, disk = job(1, need_gb=15, hotfix="main-red"), job(2, task="work/b", workdir="/s"), job(8, status="active", need_gb=20, workdir="/d") + self.assertEqual(self.decide([disk, hotfix, small, light], **rules), { + "j1": "15 GB would cut into the 10 GB floor", "j2": "waits behind hotfix j1", "j3": None, + }) + self.assertEqual(self.decide([{**hotfix, "need_gb": 50}, small, light], **rules), + {"j1": "50 GB would cut into the 10 GB floor", "j2": None, "j3": None}) + running, fixing = job(9, status="active", workdir="/r"), job(1, hotfix="main-red") + shares = {"j9": {"ram": 50, "cpu": 10}, "j1": {"ram": 30, "cpu": 10}, "j2": {"ram": 5, "cpu": 5}, "j3": {"ram": 10, "cpu": 5}} + self.assertEqual(self.decide([running, fixing, small, light], footprints=shares, **rules), { + "j1": "RAM 50 % in use or reserved + 30 % > budget 75 %", "j2": "waits behind hotfix j1", "j3": "waits behind hotfix j1", + }) + self.assertEqual(self.decide([running, fixing, light], footprints=shares | {"j3": {"ram": 0, "cpu": 5}}, **rules)["j3"], None) + pressed = self.decide([running, fixing, light], loaded={"j1": "CPU 95 % busy reaches the limit 90 %"}, **rules) + self.assertEqual(pressed["j3"], "waits behind hotfix j1") + second = job(4, task="work/c", hotfix="fix-2", workdir="/f") + self.assertEqual(self.decide([disk, hotfix, second], **rules)["j4"], "waits behind hotfix j1") + normal = {**hotfix, "hotfix": None, "need_gb": 50} + self.assertEqual(self.decide([normal, small, light], **rules), {"j1": "50 GB would cut into the 10 GB floor", "j2": None, "j3": None}) + + def test_a_drain_starts_only_once_nothing_fits_and_ends_when_its_head_leaves_or_cannot_fit_the_disk(self): + running = job(9, need_gb=40, status="active", workdir="/r") + big, small = job(1, need_gb=50), job(2, task="work/b", need_gb=25, workdir="/s") + late, light = job(3, task="work/c", need_gb=5, workdir="/t"), job(4, kind="test-nostack", task="work/c", need_gb=0, workdir="/l") + rules = {"free_gb": 70.0, "light": {"test-nostack"}, "slot_limits": {"test": 5, "test-nostack": 5}} + self.assertIsNone(self.run_pass([running, big, small, late], **rules)[1]) + self.assertEqual(self.run_pass([running, big, small], **rules)[1], "j1") + self.assertEqual(self.decide([running, big, small, late, light], drain="j1", **rules), { + "j1": "50 GB would cut into the 10 GB floor", "j2": "25 GB would cut into the 10 GB floor", + "j3": "drained for j1", "j4": None, + }) + self.assertEqual(self.run_pass([running, big, small, late], drain="j1", **rules)[1], "j1") + self.assertEqual(self.decide([big, small, late], drain="j1", **rules), { + "j1": None, "j2": "25 GB would cut into the 10 GB floor", "j3": None, + }) + self.assertIsNone(self.run_pass([running, small, late], drain="j1", **rules)[1]) + self.assertIsNone(self.run_pass([running, big, small], drain="j1", **{**rules, "free_gb": 55.0})[1]) + deploying, head = job(9, kind="deploy", need_gb=0, status="active", workdir="/r"), job(1, kind="deploy", need_gb=20) + fits, wide = job(3, task="work/c", need_gb=5, workdir="/t"), job(5, task="work/c", need_gb=45, workdir="/u") + slots = {**rules, "slot_limits": {"test": 5, "deploy": 1}} + self.assertEqual(self.decide([deploying, head, fits, wide], drain="j1", **slots), { + "j1": "all 1 deploy slots taken", "j3": None, "j5": "drained for j1", + }) + + def test_claims_name_the_worktree_and_the_compose_stack_its_directory_derives(self): + with tempfile.TemporaryDirectory() as tmp: + plain, named, stack = Path(tmp, "plain"), Path(tmp, "Core.Stack_1"), Path(tmp, "pinned") + for path in (plain, named, stack): + path.mkdir() + (named / "compose.yml").write_text("services: {}\n", encoding="utf-8") + (stack / "docker-compose.yaml").write_text("services: {}\n", encoding="utf-8") + (stack / ".env").write_text("A=1\nCOMPOSE_PROJECT_NAME='shared'\n", encoding="utf-8") + self.assertEqual(self.queue.claims_of(str(plain)), [str(plain)]) + self.assertEqual(self.queue.claims_of(str(named)), [str(named), "stack:corestack_1"]) + self.assertEqual(self.queue.claims_of(str(stack)), [str(stack), "stack:shared"]) + self.assertEqual(self.queue.claims(job(1, kind="test")), ["/w1"]) + self.assertEqual(self.queue.claims(job(1, claims=["/a", "stack:x"])), ["/a", "stack:x"]) + + def test_r2_worktrees_pinned_to_one_slot_share_a_line(self): + with tempfile.TemporaryDirectory() as tmp: + base = Path(tmp) + pins = {"a": ("custom.env", "INFINITO_INSTANCE=174\n"), "b": (".env", 'X=1\nINFINITO_INSTANCE="174"\n'), + "c": ("custom.env", "INFINITO_INSTANCE=175\n"), "core": ("default.env", "INFINITO_INSTANCE=0\n")} + for name, (pin, text) in pins.items(): + (base / name).mkdir() + (base / name / "compose.yml").write_text("services: {}\n", encoding="utf-8") + (base / name / pin).write_text(text, encoding="utf-8") + (base / "a" / "default.env").write_text("INFINITO_INSTANCE=0\n", encoding="utf-8") + claims = {name: self.queue.claims_of(str(base / name)) for name in pins} + (base / "d").mkdir() + (base / "d" / "custom.env").write_text("INFINITO_INSTANCE=174\n", encoding="utf-8") + (base / "d" / ".env").write_text("INFINITO_INSTANCE=180\n", encoding="utf-8") + drifted = self.queue.claims_of(str(base / "d")) + self.assertEqual(claims["a"], [str(base / "a"), "stack:a", "slot:174"]) + self.assertEqual(claims["b"], [str(base / "b"), "stack:b", "slot:174"]) + self.assertEqual(claims["core"], [str(base / "core"), "stack:core", "slot:0"]) + self.assertEqual(drifted, [str(base / "d"), "slot:174"]) + load = "CPU 95 % busy reaches the limit 90 %" + entries = [job(1, claims=claims["a"]), job(2, task="work/b", claims=claims["b"]), job(3, task="work/b", claims=claims["c"])] + self.assertEqual(self.decide(entries, loaded={"j1": load}), {"j1": load, "j2": "waits in line behind j1", "j3": None}) + + def test_disk_reservation_keeps_smaller_jobs_from_starving_a_big_one(self): + big, small, later = job(1, workdir="/w", need_gb=50, enqueued=NOW - 60), job(2, workdir="/w", need_gb=5, enqueued=NOW - 30), job(3, workdir="/w", need_gb=5) + verdicts, _, order = self.run_pass([big, small], free_gb=64.0, floor=15.0) + self.assertEqual(verdicts, {"j1": "50 GB would cut into the 15 GB floor", "j2": None}) + data = {"entries": [big, small], "granted_at": NOW} + self.assertEqual(self.queue.stalled(data, order, verdicts, 15.0, 64.0, idle, NOW)[:2], ([], True)) + sends, clean, _ = self.queue.stalled(data, order, verdicts, 15.0, 64.0, idle, NOW + 60) + self.assertEqual(([label for label, _ in sends], clean), (["noticed that the head cannot fit"], False)) + self.assertIn("head j1 needs 50 GB, max reachable 49 GB", " ".join(word for _, words in sends for word in words)) + self.assertEqual(self.decide([big, later], free_gb=66.0, floor=15.0), {"j1": None, "j3": "waits in line behind j1"}) + + def test_labels_mark_backfilled_entries_and_the_escalated_head(self): + verdicts = {"h": "blocked", "a": None, "b": "waits in line behind h"} + self.assertEqual(self.queue.labelled(verdicts, ["post", "call"]), { + "h": "escalated call: blocked", "a": "backfilled", "b": "waits in line behind h", + }) + self.assertEqual(self.queue.labelled({"a": None, "h": "blocked"}, []), {"a": None, "h": "blocked"}) + + def test_r3_p1_disk_alone_in_the_warn_and_stop_bands_starts_the_cleanup_and_the_notice(self): + head = job(1, kind="deploy", need_gb=20, enqueued=NOW - 60) + bands = ((30.0, "ok", "20 GB would cut into the 15 GB floor", 15), (35.0, "warn", "disk stage warn admits only jobs that need no disk", 0), + (14.0, "stop", "disk stage stop", 0)) + for free, stage, reason, reach in bands: + with self.subTest(stage=stage): + verdicts, _, order = self.run_pass([head], free_gb=free, stage=stage, floor=15.0) + data = {"entries": [head], "granted_at": NOW - 60} + polls = [self.queue.stalled(data, order, verdicts, 15.0, free, idle, NOW + at) for at in (0, 20, 40)] + text = f"kvb-org-queue: head j1 needs 20 GB, max reachable {reach} GB after cleanup ({reason}); operator cleanup needed" + self.assertEqual(verdicts, {"j1": reason}) + self.assertEqual(polls, [ + ([], True, None), ([("noticed that the head cannot fit", ["post", text, "--to", "Main"])], False, + "low-risk cleanup ended without a report"), ([], False, None), + ]) + + def test_r3_s2_a_head_live_load_holds_behind_the_floor_is_no_disk_alone_stall(self): + head, load = job(1, need_gb=70), "CPU 95 % busy reaches the limit 90 %" + verdicts, _, order = self.run_pass([head], free_gb=80.0, floor=15.0, loaded={"j1": load}) + self.assertEqual(verdicts, {"j1": "70 GB would cut into the 15 GB floor"}) + self.assertEqual(self.queue.stalled({"entries": [head], "granted_at": NOW}, order, verdicts, 15.0, 80.0, idle, NOW), ([], False, None)) + + def test_r3_p4_a_line_runs_by_enqueue_time_whatever_the_rank(self): + load = "CPU 95 % busy reaches the limit 90 %" + older, younger, other = job(1, task="work/c", workdir="/core", enqueued=NOW - 1800), job(2, workdir="/core"), job(3, task="work/b", workdir="/x") + verdicts = self.decide([younger, other, older], loaded={"j1": load}) + self.assertEqual(list(verdicts.items()), [("j1", load), ("j2", "waits in line behind j1"), ("j3", None)]) + normal, hotfix = job(4, workdir="/core", need_gb=50, enqueued=NOW - 60), job(5, workdir="/core", hotfix="main-red") + self.assertEqual(self.decide([hotfix, normal, job(6, task="work/b", workdir="/s")], free_gb=40.0), { + "j5": None, "j4": "waits in line behind j5", "j6": None, + }) + + def test_r3_p5_a_drain_head_that_live_load_holds_holds_back_every_entry_after_it(self): + running, head, small = job(9, status="active", workdir="/r", granted=NOW - HOUR), job(1), job(2, task="work/b", workdir="/s") + shares = {"j9": {"ram": 30, "cpu": 10}, "j1": {"ram": 40, "cpu": 10}, "j2": {"ram": 5, "cpu": 5}} + ram = "RAM 60 % used + 40 % still to come reach the limit 90 %" + self.assertEqual(self.run_pass([running, head, small], footprints=shares, loaded={"j1": ram, "j2": ram})[1], "j1") + self.assertEqual(self.decide([running, head, small], footprints=shares, loaded={"j1": ram}, drain="j1"), { + "j1": ram, "j2": "drained for j1", + }) + self.assertEqual(self.decide([running, head, small], footprints=shares, loaded={"j1": ram}), {"j1": ram, "j2": None}) + + def test_r3_p6_a_light_job_takes_nothing_the_drain_head_still_lacks(self): + rules = {"light": {"test-nostack"}, "slot_limits": {"test": 5, "test-nostack": 5}, "drain": "j1"} + running, head = job(9, status="active", workdir="/r"), job(1) + light = job(2, kind="test-nostack", task="work/b", workdir="/l") + shares = {"j9": {"ram": 40, "cpu": 0}, "j1": {"ram": 45, "cpu": 0}, "j2": {"ram": 35, "cpu": 0}} + self.assertEqual(self.decide([running, head, light], footprints=shares, **rules), { + "j1": "RAM 40 % in use or reserved + 45 % > budget 75 %", "j2": "drained for j1", + }) + self.assertIsNone(self.decide([running, head, light], footprints={**shares, "j2": {"ram": 0, "cpu": 0}}, **rules)["j2"]) + + def test_r3_p7_one_spike_poll_never_starts_a_drain_and_a_minute_of_blocked_polls_does(self): + running, head = job(9, need_gb=40, status="active", workdir="/r"), job(1, need_gb=50) + small, spike = job(2, task="work/b", need_gb=5, workdir="/s"), "CPU 99 % busy reaches the limit 90 %" + data: dict = {"drain": None} + + def poll(at: float, loaded: dict[str, str]) -> str | None: + _, drain, _ = self.run_pass([running, head, small], free_gb=70.0, loaded=loaded, drain=data["drain"], now=at) + data["drain"] = self.queue.latched(data, drain, at) + return data["drain"] + + self.assertEqual([poll(NOW, {"j1": spike, "j2": spike}), poll(NOW + 20, {})], [None, None]) + self.assertNotIn(self.queue.BLOCKED, data) + self.assertEqual([poll(NOW + 40 + 20 * n, {"j2": spike}) for n in range(4)], [None, None, None, "j1"]) + + def test_r3_s1_a_steady_stream_starts_the_head_within_the_age_limit_plus_one_job(self): + rules = {"free_gb": 100.0, "floor": 15.0, "slot_limits": {"test": 1000}} + head = job(1, need_gb=70, workdir="/h") + active, waiting, data = [job(2, need_gb=20, status="active", granted=NOW - 60, workdir="/w2")], [head], {"drain": None} + for poll in range(4 * 180): + at = NOW + poll * 20 + active = [e for e in active if at - e["granted"] < 1800] + waiting.append(job(poll + 3, task="work/b", need_gb=0 if poll % 45 else 20, enqueued=at)) + verdicts, drain, _ = self.run_pass([*active, *waiting], drain=data["drain"], now=at, **rules) + data["drain"] = self.queue.latched(data, drain, at) + if verdicts["j1"] is None: + self.assertLessEqual(at - NOW, self.queue.DRAIN_AFTER_SECONDS + 1800 + self.queue.DEBOUNCE_SECONDS) + return + active += [{**e, "status": "active", "granted": at} for e in waiting if verdicts[e["id"]] is None] + waiting = [e for e in waiting if verdicts[e["id"]] is not None] + self.fail("a steady stream starved the head") + + def test_r4_load_from_outside_the_queue_drains_for_no_head_and_holds_back_no_job(self): + rules = {"exclusive": {"quality-high", "test"}, "light": {"test-nostack"}, + "slot_limits": {"quality-high": 1, "test": 1, "test-nostack": 4, "iteration": 3}} + head, light = job(1, kind="quality-high", enqueued=NOW - 3700), job(2, kind="test-nostack", task="work/b", workdir="/l") + heavy, running = job(3, kind="iteration", task="work/b", workdir="/i"), job(9, kind="test-nostack", status="active", workdir="/r") + ram = "RAM 45 % used + 41 % still to come reach the limit 85 %" + outside = self.queue.Outside(ram) + self.assertEqual(self.run_pass([head, light, heavy], loaded={"j1": ram}, drain="j1", **rules)[:2], + ({"j1": ram, "j2": None, "j3": None}, None)) + self.assertEqual(self.run_pass([running, head, light, heavy], loaded={"j1": outside}, drain="j1", **rules)[:2], + ({"j1": ram, "j2": None, "j3": None}, None)) + self.assertEqual(self.run_pass([running, head, light, heavy], loaded={"j1": ram}, drain="j1", **rules)[:2], + ({"j1": ram, "j2": None, "j3": "drained for j1"}, "j1")) + twin = job(4, kind="test", task="work/c", workdir="/w1") + self.assertEqual((self.decide([head, twin], loaded={"j1": outside}, **rules), self.decide([head, twin], loaded={"j1": ram}, **rules)), + ({"j1": ram, "j4": None}, {"j1": ram, "j4": "waits in line behind j1"})) + + def test_r4_a_head_aged_out_waiting_only_for_an_exclusive_job_holds_back_no_light_job(self): + rules = {"exclusive": {"quality-high", "test"}, "light": {"test-nostack"}, "budget": {"ram": 75, "cpu": 100}, + "slot_limits": {"quality-high": 1, "test": 1, "test-nostack": 4}} + running = job(9, kind="test", status="active", workdir="/r", granted=NOW - 300) + head, light = job(1, kind="quality-high", enqueued=NOW - 3700), job(2, kind="test-nostack", task="work/b", workdir="/l") + shares = {"j9": {"ram": 34.5, "cpu": 24.5}, "j1": {"ram": 40.6, "cpu": 59.8}, "j2": {"ram": 7.2, "cpu": 9.8}} + verdicts, drain, _ = self.run_pass([running, head, light], footprints=shares, drain="j1", **rules) + self.assertEqual((verdicts, drain), ({"j1": "quality-high runs only beside test-nostack, but test runs", "j2": None}, "j1")) + greedy = {**shares, "j2": {"ram": 40.0, "cpu": 9.8}} + self.assertEqual(self.decide([running, head, light], footprints=greedy, drain="j1", **rules)["j2"], "drained for j1") + + +class TestTrace(Passes, unittest.TestCase): + def traced(self, entries: list[dict], **overrides: object) -> tuple[dict, dict]: + trace: dict[str, list[str]] = {} + verdicts = self.run_pass(entries, trace=trace, **overrides)[0] + again: dict[str, list[str]] = {} + self.run_pass(entries, trace=again, **overrides) + self.assertEqual(again, trace) + return verdicts, trace + + def test_r5_the_trace_names_the_order_and_the_fit_of_a_granted_entry(self): + fit = "RAM 0 % + 0 %, CPU 0 % + 0 %, 1 GB of 100 GB fits" + cases = { + "hotfix first": ([job(1, hotfix="main-red")], {}, ["hotfix first", "rank 0", "arrival 1", fit]), + "rank and age": ([job(1, task="work/c", enqueued=NOW - 2 * HOUR)], {}, ["rank 2, 2 h waited: 0", "arrival 1", fit]), + "unranked with agent order": ([job(1, task="work/z")], {"agents": {"s1": 0}}, ["unranked", "agent order 0", "arrival 1", fit]), + "backfill": ([job(1, need_gb=95), job(2, task="work/b")], {}, + ["rank 1", "arrival 2", "RAM 0 % + 0 %, CPU 0 % + 0 %, 1 GB of 100 GB fits", "backfilled past j1"]), + } + for name, (entries, overrides, steps) in cases.items(): + with self.subTest(name): + verdicts, trace = self.traced(entries, **overrides) + self.assertEqual((verdicts[entries[-1]["id"]], trace[entries[-1]["id"]]), (None, steps)) + + def test_r5_the_trace_ends_in_the_rule_that_blocked_the_entry(self): + load = "CPU 95 % busy reaches the limit 90 %" + running = job(9, status="active", workdir="/r") + gates = {"exclusive": {"quality-high"}, "light": {"test-nostack"}, "slot_limits": {"quality-high": 1, "iteration": 2, "test-nostack": 2, "test": 5}} + greedy = {"j9": {"ram": 30, "cpu": 30}, "j1": {"ram": 40, "cpu": 40}, "j2": {"ram": 40, "cpu": 70}, "j3": {"ram": 0, "cpu": 5}} + machine = [job(9, kind="iteration", status="active", workdir="/r"), job(1, kind="quality-high"), job(2, kind="iteration", task="work/b")] + cases = { + "line": ([job(9, kind="deploy", status="active", workdir="/r"), job(1, kind="deploy", workdir="/core"), job(2, task="work/b", workdir="/core")], + {}, "j2", "line /core behind j1"), + "hotfix ahead that could start": ([job(9, kind="deploy", status="active", workdir="/r"), job(1, kind="deploy", hotfix="main-red"), + job(2, task="work/b")], {}, "j2", "hotfix j1 blocked ahead"), + "hotfix pressed": ([running, job(1, hotfix="main-red"), job(2, kind="test-nostack", task="work/b")], + {"loaded": {"j1": load}, **gates}, "j2", "hotfix j1 blocked ahead: load holds it"), + "exclusive head": (machine, gates, "j2", "drained for j1: quality-high is exclusive"), + "load head": ([running, job(1), job(2, task="work/b")], {"loaded": {"j1": load}, "drain": "j1"}, "j2", + "drained for j1: load holds it"), + "takes what the head lacks": ([*machine[:2], job(2, kind="test-nostack", task="work/b")], {"footprints": greedy, **gates}, "j2", + "drained for j1: takes RAM and CPU"), + "disk stage": ([job(1)], {"stage": "stop"}, "j1", "disk stage stop"), + "memory stage": ([job(1)], {"memory": "stop"}, "j1", "memory stage stop"), + "slot": ([job(1, kind="deploy", status="active"), job(2, kind="deploy", task="work/b")], {}, "j2", "all 1 deploy slots taken"), + "alone": (machine, gates, "j1", "quality-high runs only beside test-nostack, but iteration runs"), + "budget": ([running, job(1)], {"footprints": {"j9": {"ram": 50, "cpu": 0}, "j1": {"ram": 30, "cpu": 0}}}, "j1", + "RAM 50 % in use or reserved + 30 % > budget 75 %"), + "floor": ([job(1, need_gb=95)], {}, "j1", "95 GB would cut into the 10 GB floor"), + "load": ([job(1)], {"loaded": {"j1": load}}, "j1", load), + } + for name, (entries, overrides, target, step) in cases.items(): + with self.subTest(name): + verdicts, trace = self.traced(entries, **overrides) + self.assertIsNotNone(verdicts[target]) + self.assertEqual(trace[target][-1], step) + + def test_r5_the_trace_marks_the_drain_head_a_head_a_light_job_takes_nothing_from_and_a_silent_entry(self): + gates = {"exclusive": {"quality-high"}, "light": {"test-nostack"}, "slot_limits": {"quality-high": 1, "iteration": 2, "test-nostack": 2}} + machine = [job(9, kind="iteration", status="active", workdir="/r"), job(1, kind="quality-high"), job(2, kind="test-nostack", task="work/b")] + _, trace = self.traced(machine, drain="j1", **gates) + self.assertEqual(trace["j1"][-2:], ["drain head", "quality-high runs only beside test-nostack, but iteration runs"]) + self.assertEqual(trace["j2"][-3:], ["takes nothing j1 lacks", "RAM 0 % + 0 %, CPU 0 % + 0 %, 1 GB of 99 GB fits", "backfilled past j1"]) + verdicts, trace = self.traced([job(1, polled=NOW - 600)]) + self.assertEqual(trace["j1"], [verdicts["j1"]]) + + def test_r6_a_light_job_passes_an_older_job_of_its_worktree_only_while_that_one_waits_for_the_machine(self): + gates = {"exclusive": {"quality-high"}, "light": {"test-nostack"}, "slot_limits": {"quality-high": 1, "iteration": 2, "test-nostack": 2}} + gate = job(9, kind="quality-high", status="active", workdir="/r") + heavy, light = job(1, kind="iteration", workdir="/core"), job(2, kind="test-nostack", task="work/b", workdir="/core") + verdicts, trace = self.traced([gate, heavy, light], **gates) + self.assertEqual(verdicts, {"j1": "quality-high runs only beside test-nostack", "j2": None}) + self.assertIn("passes j1: it waits for the machine", trace["j2"]) + self.assertEqual(self.decide([{**light, "status": "active"}, heavy], **gates), {"j1": "waits in line behind j2"}) + stacked = [gate, job(8, kind="iteration", status="active", claims=["/x", "stack:core"]), + {**heavy, "claims": ["/core", "stack:core"]}, {**light, "claims": ["/core"]}] + self.assertEqual(self.decide(stacked, **gates)["j2"], "waits in line behind j1") + self.assertEqual(self.decide([gate, {**heavy, "hotfix": "main-red"}, light], **gates)["j2"], "waits in line behind j1") + + def test_r6_beside_a_head_waiting_for_an_exclusive_job_only_light_jobs_count(self): + gates = {"exclusive": {"quality-high"}, "light": {"test-nostack"}, "slot_limits": {"quality-high": 1, "gui-dind": 1, "test-nostack": 2}} + gate, head = job(9, kind="quality-high", status="active", workdir="/r"), job(1, kind="gui-dind") + light = job(2, kind="test-nostack", task="work/b") + shares = {"j9": {"ram": 40.6, "cpu": 59.8}, "j1": {"ram": 20.7, "cpu": 63.6}, "j2": {"ram": 7.2, "cpu": 9.8}} + verdicts, trace = self.traced([gate, head, light], footprints=shares, drain="j1", **gates) + self.assertEqual(verdicts["j2"], None) + self.assertIn("takes nothing j1 lacks", trace["j2"]) + greedy = shares | {"j2": {"ram": 7.2, "cpu": 40.0}} + self.assertEqual(self.decide([gate, head, light], footprints=greedy, drain="j1", **gates)["j2"], "drained for j1") + + def test_r7_a_hotfix_never_waits_in_line_behind_an_older_waiting_entry_that_is_no_hotfix(self): + gates = {"exclusive": {"quality-high"}, "light": {"test-nostack"}, + "slot_limits": {"quality-high": 1, "iteration": 3, "test-nostack": 1}} + filled = job(9, id="A", kind="test-nostack", status="active", workdir="/a") + older = job(1, id="L", kind="test-nostack", workdir="/main", enqueued=NOW - 20, name="Main") + hotfix = job(2, id="H", kind="quality-high", workdir="/main", enqueued=NOW - 10, name="Main", hotfix="main-red") + self.assertEqual(self.decide([filled, older, hotfix], **gates), {"H": None, "L": "waits in line behind H"}) + gate = job(8, id="Q", kind="quality-high", status="active", workdir="/other") + main = {**older, "id": "X", "kind": "quality-high"} + other = job(3, id="I", kind="iteration", task="work/b", workdir="/b", enqueued=NOW) + verdicts, trace = self.traced([gate, main, hotfix, other], **gates) + self.assertEqual(verdicts, {"H": "all 1 quality-high slots taken", "X": "waits behind hotfix H", "I": "waits behind hotfix H"}) + self.assertEqual(trace["H"][0], "hotfix first") + self.assertEqual(self.decide([main, hotfix, other], **gates), { + "H": None, "X": "waits in line behind H", "I": "quality-high runs only beside test-nostack", + }) + + def test_r7_a_drain_head_keeps_its_slot_and_its_line_delays_it_by_one_light_run_at_most(self): + gates = {"exclusive": {"quality-high"}, "light": {"test-nostack"}, "slot_limits": {"iteration": 3, "test-nostack": 4}} + busy = [job(7, id="Ib", kind="iteration", status="active", workdir="/b"), job(8, id="Ic", kind="iteration", status="active", workdir="/c")] + head = job(1, id="H", kind="iteration", workdir="/h", enqueued=NOW - 5000) + rival = job(2, id="I2", kind="iteration", workdir="/v", enqueued=NOW - 4000) + running = job(9, id="L1", kind="test-nostack", status="active", workdir="/h") + verdicts, trace = self.traced([*busy, running, head, rival], drain="H", **gates) + self.assertEqual(verdicts, {"H": "waits in line behind L1", "I2": "drained for H"}) + self.assertEqual(trace["I2"][-1], "drained for H: takes its iteration slot") + later = job(3, id="L2", kind="test-nostack", workdir="/h", enqueued=NOW - 100) + elsewhere = job(4, id="L3", kind="test-nostack", workdir="/z", enqueued=NOW - 50) + verdicts, trace = self.traced([*busy, {**rival, "status": "active"}, head, later, elsewhere], drain="H", **gates) + self.assertEqual(verdicts, {"H": "all 3 iteration slots taken", "L2": None, "L3": None}) + self.assertEqual(trace["L2"][-4:-2], ["passes H: it waits for the machine", "takes nothing H lacks"]) + self.assertIn("takes nothing H lacks", trace["L3"]) + next_one = job(5, id="L4", kind="test-nostack", workdir="/h", enqueued=NOW - 20) + third = job(6, id="I3", kind="iteration", workdir="/w", enqueued=NOW - 10) + freed = [busy[0], {**rival, "status": "active"}, {**later, "status": "active"}, head, next_one, third] + self.assertEqual(self.decide(freed, drain="H", **gates), {"H": "waits in line behind L2", "L4": "waits in line behind L2", "I3": "drained for H"}) + self.assertEqual(self.decide([busy[0], {**rival, "status": "active"}, head, next_one, third], drain="H", **gates), + {"H": None, "L4": "waits in line behind H", "I3": "all 3 iteration slots taken"}) + + def test_r7_a_granted_entry_owns_its_line_for_the_rest_of_the_pass(self): + gates = {"exclusive": {"quality-high"}, "light": {"test-nostack"}, "slot_limits": {"quality-high": 1, "test-nostack": 4}} + gate = job(9, id="Q", kind="quality-high", status="active", workdir="/other") + waiting = job(1, id="M", kind="quality-high", workdir="/x", enqueued=NOW - 30) + first, second = (job(n, id=f"L{n - 1}", kind="test-nostack", workdir="/x", enqueued=NOW - 30 + n) for n in (2, 3)) + verdicts, trace = self.traced([gate, waiting, first, second], **gates) + self.assertEqual(verdicts, {"M": "all 1 quality-high slots taken", "L1": None, "L2": "waits in line behind L1"}) + self.assertEqual(trace["L2"][-1], "line /x behind L1") + + def test_r8_a_main_job_whose_line_runs_or_waits_for_the_machine_reserves_against_younger_lines(self): + gates = {"exclusive": {"quality-high", "test"}, "light": {"test-nostack"}, + "slot_limits": {"quality-high": 1, "test": 1, "iteration": 2, "test-nostack": 4}} + filled = [job(10 + n, id=f"F{n}", kind="test-nostack", status="active", workdir=f"/f{n}") for n in range(4)] + light = job(1, id="L", kind="test-nostack", workdir="/main", enqueued=NOW - 30, name="Main") + gate = job(2, id="X", kind="quality-high", workdir="/main", enqueued=NOW - 20, name="Main") + rival = job(3, id="T", kind="test", task="work/b", workdir="/t", enqueued=NOW - 10) + verdicts, trace = self.traced([*filled, light, gate, rival], **gates) + self.assertEqual(verdicts, {"L": "all 4 test-nostack slots taken", "X": "waits in line behind L", "T": "reserved for X"}) + self.assertEqual(trace["T"][-1], "reserved for X: quality-high is exclusive") + self.assertEqual(self.decide([*filled[:3], {**light, "status": "active"}, gate, rival], **gates)["T"], "reserved for X") + self.assertEqual(self.decide([*filled, light, {**gate, "name": "s1"}, rival], **gates)["T"], None) + running = job(4, id="R", kind="iteration", status="active", workdir="/w") + deploy, other = job(5, id="D", kind="iteration", workdir="/w", name="Main"), job(6, id="E", kind="iteration", task="work/b", workdir="/e") + small = job(7, id="S", kind="test-nostack", task="work/c", workdir="/s") + verdicts, trace = self.traced([running, deploy, other, small], **gates) + self.assertEqual(verdicts, {"D": "waits in line behind R", "E": "reserved for D", "S": None}) + self.assertEqual(trace["E"][-1], "reserved for D: takes its iteration slot") + chained = [job(4, id="Z", kind="iteration", status="active", claims=["/z"]), job(5, id="Y", kind="iteration", claims=["/z", "/main"], enqueued=NOW - 600), + {**gate, "claims": ["/main"]}, {**other, "id": "T"}] + self.assertEqual(self.decide(chained, **gates), {"Y": "waits in line behind Z", "X": "waits in line behind Y", "T": None}) + + def test_r10_a_job_passes_a_holder_of_its_line_that_can_never_start_and_takes_its_place(self): + gates = {"exclusive": {"quality-high"}, "light": {"test-nostack"}, + "slot_limits": {"quality-high": 1, "iteration": 2, "gui-dind": 1, "swarm": 0}} + holder = job(1, id="O", kind="swarm", workdir="/main", enqueued=NOW - 30, name="Main") + gate = job(2, id="X", kind="quality-high", workdir="/main", enqueued=NOW - 20, name="Main") + others = [job(3, id="I", kind="iteration", task="work/b", workdir="/b"), job(4, id="G", kind="gui-dind", task="work/c", workdir="/c")] + verdicts, trace = self.traced([holder, gate, *others], **gates) + beside = "quality-high runs only beside test-nostack" + self.assertEqual(verdicts, {"O": "all 0 swarm slots taken", "X": None, "I": beside, "G": beside}) + self.assertIn("passes O: O can never start, 0 swarm slots", trace["X"]) + running, later = job(9, id="Q", kind="quality-high", status="active", workdir="/q"), job(5, id="Y", kind="iteration", workdir="/main") + self.assertEqual(self.decide([running, holder, gate, later], **gates), + {"O": "all 0 swarm slots taken", "X": "all 1 quality-high slots taken", "Y": "waits in line behind X"}) + huge = {**holder, "kind": "iteration", "need_gb": 200} + verdicts, trace = self.traced([huge, gate], **gates) + self.assertEqual(verdicts["X"], None) + self.assertIn("passes O: O can never start, 200 GB would cut into the 10 GB floor", trace["X"]) + + def test_r11_a_hotfix_that_can_never_start_holds_no_one_back_and_heads_nothing(self): + gates = {"exclusive": {"quality-high"}, "light": {"test-nostack"}, + "slot_limits": {"swarm": 0, "iteration": 2, "gui-dind": 1, "test-nostack": 2, "quality-high": 1}} + doomed = job(1, id="H", kind="swarm", workdir="/main", hotfix="main-red", name="Main", enqueued=NOW - 30) + others = [job(2, id="I", kind="iteration", task="work/b", workdir="/b"), job(3, id="G", kind="gui-dind", task="work/c", workdir="/c"), + job(4, id="L", kind="test-nostack", task="work/c", workdir="/d")] + self.assertEqual(self.decide([doomed, *others], **gates), {"H": "all 0 swarm slots taken", "I": None, "G": None, "L": None}) + gate, able = job(9, id="Q", kind="quality-high", status="active", workdir="/q"), job(5, id="H2", kind="quality-high", workdir="/h2", + hotfix="main-red", enqueued=NOW - 20) + self.assertEqual(self.decide([gate, doomed, able, others[0]], **gates), + {"H": "all 0 swarm slots taken", "H2": "all 1 quality-high slots taken", "I": "waits behind hotfix H2"}) + + def test_r11_a_holder_that_could_not_start_regains_its_place_once_it_can(self): + gates = {"exclusive": {"quality-high"}, "light": {"test-nostack"}, "slot_limits": {"quality-high": 1, "iteration": 2, "swarm": 0}} + line = [job(1, id="O", kind="swarm", workdir="/main", enqueued=NOW - 30), job(2, id="X", kind="quality-high", workdir="/main", enqueued=NOW - 20), + job(3, id="Y", kind="iteration", workdir="/main", enqueued=NOW - 10)] + self.assertEqual(self.decide(line, **gates), {"O": "all 0 swarm slots taken", "X": None, "Y": "waits in line behind X"}) + back = {**gates, "slot_limits": {**gates["slot_limits"], "swarm": 1}} + self.assertEqual(self.decide(line, **back), {"O": None, "X": "waits in line behind O", "Y": "waits in line behind O"}) + + def test_r7_the_stack_comes_from_the_env_then_the_compose_name_then_the_folder(self): + with tempfile.TemporaryDirectory() as tmp: + files = { + "a/compose.yml": "name: security-check-inspect\nservices: {}\n", + "b/compose.yaml": 'name: "security-check-inspect" # shared\nservices:\n web:\n name: other\n', + "c/compose.yml": "name: security-check-inspect\n", "c/.env": "COMPOSE_PROJECT_NAME=own\n", + "d/docker-compose.yml": "services:\n name: nested\n", + } + for name, text in files.items(): + Path(tmp, name).parent.mkdir(exist_ok=True) + Path(tmp, name).write_text(text, encoding="utf-8") + stacks = {name: self.queue.claims_of(str(Path(tmp, name)))[1:] for name in "abcd"} + self.assertEqual(stacks, {"a": ["stack:security-check-inspect"], "b": ["stack:security-check-inspect"], "c": ["stack:own"], "d": ["stack:d"]}) + + +class TestEscalation(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.queue = _load(QUEUE) + + def climb(self, stall: dict, grant: float, minutes: list[float], **flags: bool) -> tuple[dict, list]: + steps = [] + for minute in minutes: + stall, step = self.queue.escalate(json.loads(json.dumps(stall)), grant, grant, flags.get("hotfix", False), + grant + minute * 60) + steps.append(step) + return stall, steps + + def test_each_step_fires_once_at_its_threshold_and_the_state_survives_a_restart(self): + minutes = [29, 30, 31, 59, 60, 119, 120, 239, 240, 24 * 60 - 1, 24 * 60, 48 * 60] + stall, steps = self.climb({}, NOW, minutes) + self.assertEqual(steps, [None, "post", None, None, "call", None, "security", None, "pan-pan", None, "mayday", None]) + self.assertEqual(stall["fired"], ["post", "call", "security", "pan-pan", "mayday"]) + self.assertEqual([threshold for threshold, _, _ in self.queue.ESCALATION], [30, 60, 120, 240, 24 * 60]) + + def test_a_new_grant_resets_the_stall_and_a_late_first_look_fires_only_the_highest_due_step(self): + stall, _ = self.climb({}, NOW, [30]) + stall, steps = self.climb(stall, NOW + 3600, [29, 30]) + self.assertEqual((steps, stall["fired"]), ([None, "post"], ["post"])) + stall, step = self.queue.escalate({}, NOW, NOW, False, NOW + 5 * HOUR) + self.assertEqual((step, stall["fired"]), ("pan-pan", ["post", "call", "security", "pan-pan"])) + self.assertIsNone(self.queue.escalate({}, NOW, NOW + HOUR, False, NOW + HOUR + 29 * 60)[1]) + + def test_the_clock_stands_still_while_a_hotfix_runs_ahead_of_the_head(self): + stall, steps = self.climb({}, NOW, [20, 50], hotfix=True) + self.assertEqual(steps, [None, None]) + stall, steps = self.climb(stall, NOW, [79, 80]) + self.assertEqual(steps, [None, "post"]) + + def test_r3_p3_a_restart_at_50_min_keeps_the_clock_and_an_idle_queue_restarts_it(self): + data, fired = {"entries": [], "granted_at": NOW}, {} + for minute in range(130): + head = job(1 if minute < 50 else 2, kind="deploy", enqueued=NOW if minute < 50 else NOW + 50 * 60) + data["entries"] = [head] + sends, _, _ = self.queue.stalled(data, [head], {head["id"]: "all 1 deploy slots taken"}, 15.0, 100.0, idle, NOW + minute * 60) + fired.update((label, minute) for label, _ in sends) + self.assertEqual(fired, {"escalated post": 30, "escalated call": 60, "escalated security": 120}) + late = job(3, kind="deploy", enqueued=NOW + 10 * HOUR) + data["entries"] = [late] + self.assertEqual(self.queue.stalled(data, [late], {"j3": "all 1 deploy slots taken"}, 15.0, 100.0, idle, NOW + 10 * HOUR + 60)[0], []) + + def test_r4_a_head_that_never_fits_gets_one_cleanup_and_one_notice_whatever_starts_beside_it(self): + head = job(1, kind="test", need_gb=60, enqueued=NOW - 600) + data, now, cleanups, notices = {"entries": [head], "granted_at": NOW - 600}, NOW, 0, [] + for cycle in range(3): + for _ in range(3): + sends, clean, _ = self.queue.stalled(data, [head], {"j1": self.queue.Disk("60 GB would cut into the 15 GB floor")}, + 15.0, 40.0, idle, now) + cleanups += clean + notices += [label for label, _ in sends if label.startswith("noticed")] + now += 20 + data["granted_at"] = now + data.pop("stall", None) + now += 20 + self.assertEqual((cleanups, notices), (1, ["noticed that the head cannot fit"])) + other = job(2, kind="test", need_gb=60) + self.assertTrue(self.queue.stalled(data, [other], {"j2": self.queue.Disk("60 GB would cut into the 15 GB floor")}, + 15.0, 40.0, idle, now)[1]) + + def test_the_cleanup_runs_once_per_head_when_disk_alone_blocks_it(self): + state, clean = self.queue.cleanup_due({}, "j1", True, NOW) + again = self.queue.cleanup_due(state, "j1", True, NOW + 60) + fresh = self.queue.cleanup_due(again[0], "j2", True, NOW + 180) + self.assertEqual((clean, again[1], fresh[1], state["cleaned_at"]), (True, False, True, NOW)) + self.assertFalse(self.queue.cleanup_due({}, "j1", False, NOW)[1]) + + def test_stalled_reads_the_store_names_the_head_and_skips_a_hotfix_ahead(self): + head = job(1926, kind="gui-dind", need_gb=65, enqueued=NOW - 40 * 60) + verdicts = {"j1926": self.queue.Disk("65 GB would cut into the 15 GB floor"), "j2": "all 1 test slots taken"} + order = [head, job(2)] + data = {"entries": [head, job(2)], "granted_at": NOW - 31 * 60} + text = "kvb-org-queue: no grant for 30 min; head j1926 (gui-dind, work/a) waits: 65 GB would cut into the 15 GB floor" + notice = ("noticed that the head cannot fit", ["post", ( + f"kvb-org-queue: head j1926 needs 65 GB, max reachable 32 GB after cleanup ({verdicts['j1926']}); operator cleanup needed" + ), "--to", "Main"]) + self.assertEqual(self.queue.stalled(data, order, verdicts, 15.0, 47.0, idle, NOW), + ([("escalated post", ["post", text, "--to", "Main"])], True, None)) + self.assertEqual(self.queue.stalled(data, order, verdicts, 15.0, 47.0, idle, NOW + 60), + ([notice], False, "low-risk cleanup ended without a report")) + self.assertEqual(self.queue.stalled(data, order, verdicts, 15.0, 47.0, idle, NOW + 120), ([], False, None)) + hotfix = {"entries": [head, job(3, status="active", hotfix="main-red", granted=NOW - 9 * HOUR)], "granted_at": NOW - 9 * HOUR} + self.assertEqual(self.queue.stalled(hotfix, order, verdicts, 15.0, 100.0, idle, NOW)[0], []) + self.assertEqual(self.queue.stalled({"entries": []}, order, {**verdicts, "j2": None}, 15.0, 100.0, idle, NOW), ([], True, None)) + self.assertEqual(self.queue.stalled({"entries": []}, order, {**verdicts, "j1926": None}, 15.0, 100.0, idle, NOW), ([], False, None)) + + def test_r2_a_head_that_cannot_fit_after_the_cleanup_gets_one_notice_per_head(self): + head = job(1926, kind="gui-dind", need_gb=65, enqueued=NOW - 5 * 60) + floored = self.queue.Disk("65 GB would cut into the 15 GB floor") + order, data = [head], {"entries": [head], "granted_at": NOW - 60} + report = "RESOURCE CLEANUP: freed 1.0 GB (0.0 GB caches), / now 47.0 GB free; 3 commands ok." + notice = ("noticed that the head cannot fit", [ + "post", f"kvb-org-queue: head j1926 needs 65 GB, max reachable 32 GB after cleanup ({floored}); operator cleanup needed", + "--to", "Main", + ]) + + def poll(at: float, cleaning: bool, reason: str = floored) -> tuple: + return self.queue.stalled( + data, order, {"j1926": reason}, 15.0, 47.0, lambda: (cleaning, [(NOW - 600, "older"), (at - 10, report)]), at + ) + + self.assertEqual(poll(NOW, False), ([], True, None)) + self.assertEqual(poll(NOW + 20, True), ([], False, None)) + self.assertEqual(poll(NOW + 40, False), ([notice], False, f"low-risk cleanup: {report}")) + self.assertEqual(poll(NOW + 60, False), ([], False, None)) + data["granted_at"] = NOW + 100 + self.assertEqual([poll(NOW + 200, False), poll(NOW + 220, False)], [([], False, None)] * 2) + data.pop("cleanup") + self.assertEqual(poll(NOW + 400, False), ([], True, None)) + self.assertEqual(poll(NOW + 420, False, "all 1 gui-dind slots taken"), ([], False, f"low-risk cleanup: {report}")) + + def test_r2_a_cleanup_that_fails_in_the_sandbox_is_logged_once_and_the_queue_keeps_polling(self): + failed = "RESOURCE CLEANUP: freed 0.0 GB (0.0 GB caches), / now 47.0 GB free; 0 commands ok, failed: exit 1: docker image prune" + stall, clean = self.queue.cleanup_due({}, "j1", True, NOW) + lines = [] + for _ in range(3): + stall, log, _ = self.queue.settle(stall, lambda: (False, [(NOW + 5, failed)]), False) + lines.append(log) + self.assertEqual((clean, lines), (True, [f"low-risk cleanup: {failed}", None, None])) + refused = mock.Mock(side_effect=PermissionError(13, "Permission denied")) + stall, _ = self.queue.cleanup_due({}, "j1", True, NOW) + with mock.patch.object(self.queue.watch, "spawn_cleanup", refused): + stall, started = self.queue.cleaned(stall) + self.assertEqual(started, "low-risk cleanup did not start: [Errno 13] Permission denied") + stall, log, notice = self.queue.settle(stall, idle, True) + self.assertEqual((log, notice, self.queue.settle(stall, idle, True)[1:]), (None, True, (None, False))) + stamp = 1_800_000_005 * 10**9 + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp).resolve() + with mock.patch.object(self.queue.watch, "ROOT", root), mock.patch.object(self.queue.watch, "ALERTS", root / "alerts.jsonl"): + self.assertEqual(self.queue.cleanup_outcome(NOW), (False, [])) + (root / "alerts.jsonl").write_text("\n".join([ + json.dumps({"id": stamp, "stage": "cleanup", "text": failed}), "{broken", + json.dumps({"id": 7, "stage": "stop", "text": "RESOURCE STOP"}), + ]) + "\n", encoding="utf-8") + (root / "cleanup.lock").mkdir() + now = time.time() + self.assertEqual(self.queue.cleanup_outcome(now), (True, [])) + self.assertEqual(self.queue.cleanup_outcome(now + 3600), (False, [(stamp / 1e9, failed)])) + + def test_act_starts_the_cleanup_and_sends_each_step_with_the_radio_flags(self): + with tempfile.TemporaryDirectory() as tmp: + seen = Path(tmp, "argv.json") + fake = Path(tmp, "radio.py") + fake.write_text(f"import json, sys\njson.dump(sys.argv[1:], open({str(seen)!r}, 'w'))\n", encoding="utf-8") + spawn = mock.Mock() + with mock.patch.object(self.queue, "RADIO", fake), mock.patch.object(self.queue.watch, "spawn_cleanup", spawn), \ + contextlib.redirect_stderr(io.StringIO()) as err: + stall, started = self.queue.cleaned({"settled": False}) + self.queue.act([("escalated call", self.queue.radio_words("call", "head waits"))], ["low-risk cleanup: x", started], "s1") + self.queue.act([], [], "s1") + self.assertEqual(json.loads(seen.read_text(encoding="utf-8")), ["send", "call", "head waits", "--to", "Main", "--session", "s1"]) + spawn.assert_called_once_with("--full") + self.assertEqual(stall, {"settled": False}) + self.assertEqual(err.getvalue(), ( + "queue: low-risk cleanup: x\nqueue: disk alone blocks the head; low-risk cleanup started\nqueue: escalated call\n" + )) + + def council_env(self, tmp: str) -> tuple[Path, dict[str, str]]: + root = Path(tmp).resolve() / "council" + return root, {"PATH": os.environ["PATH"], "HOME": tmp, "AGENTS_COUNCIL_ROOT": str(root), + "AGENTS_COUNCIL_CONFIG": str(Path(tmp, "none.json"))} + + def test_every_step_is_a_command_the_installed_radio_accepts_and_its_mayday_sets_no_silence(self): + with tempfile.TemporaryDirectory() as tmp: + root, env = self.council_env(tmp) + with mock.patch.dict(os.environ, env, clear=True), contextlib.redirect_stderr(io.StringIO()) as err: + for _, step, _ in self.queue.ESCALATION: + self.queue.act([(f"escalated {step}", self.queue.radio_words(step, f"kvb-org-queue: stall {step}"))], [], "s1") + rows = [json.loads(line) for line in (root / "news.jsonl").read_text(encoding="utf-8").splitlines()] + silent = (root / "radio-silence.json").exists() + self.assertEqual(err.getvalue().splitlines(), [f"queue: escalated {step}" for _, step, _ in self.queue.ESCALATION]) + self.assertEqual([(row["class"], row["to"], row["text"]) for row in rows], [ + ("post", ["Main"], "kvb-org-queue: stall post"), ("call", ["Main"], "kvb-org-queue: stall call"), + ("security", [], "kvb-org-queue: stall security"), ("pan-pan", [], "kvb-org-queue: stall pan-pan"), + ("mayday", [], "kvb-org-queue: stall mayday"), + ]) + self.assertFalse(silent) + + def test_r3_p2_a_mayday_never_replaces_the_silence_of_another_sender_and_any_silence_stops_grants(self): + with tempfile.TemporaryDirectory() as tmp: + root, env = self.council_env(tmp) + root.mkdir() + silence, now = root / "radio-silence.json", time.time() + spoof = {"since": now, "until": now + HOUR, "by": "Operator", "text": "kvb-org-queue: no grant for 1440 min"} + silence.write_text(json.dumps(spoof), encoding="utf-8") + + def send(*words: str) -> int: + return subprocess.run( + [sys.executable, str(self.queue.RADIO), "send", *words, "--session", "s2"], env=env, + stdin=subprocess.DEVNULL, capture_output=True, timeout=60, check=False, + ).returncode + + codes = [send("mayday", "disk full"), send("mayday", "disk full", "--no-silence"), + send("call", "x", "--to", "Main", "--no-silence")] + kept = json.loads(silence.read_text(encoding="utf-8")) + with mock.patch.object(self.queue.watch, "ROOT", root), mock.patch.object(self.queue, "SILENCE", silence): + stops = self.queue.silenced(now) + silence.unlink() + quiet = [send("mayday", "disk full", "--no-silence"), silence.exists()] + send("mayday", "disk full") + own = json.loads(silence.read_text(encoding="utf-8")) + self.assertEqual((codes, kept, stops, quiet), ([0, 0, 2], spoof, True, [0, False])) + self.assertEqual((own["text"], own["by"] != "Operator", own["until"] - own["since"]), ("disk full", True, 7200)) + + class TestHistory(unittest.TestCase): @classmethod def setUpClass(cls): @@ -755,6 +1456,7 @@ class TestQueueCommands(unittest.TestCase): signature = f"{self.base.name}: {Path(sys.executable).name} -c" self.assertEqual((entry["aliases"], entry["footprint"]), ([signature], None)) self.assertEqual(self.entries(), []) + self.assertGreaterEqual(json.loads((self.root / "queue.json").read_text(encoding="utf-8"))["granted_at"], entry["enqueued"]) [row] = [json.loads(line) for line in self.history_file.read_text(encoding="utf-8").splitlines()] self.assertEqual( [row[k] for k in ("kind", "signature", "workdir", "exit", "concurrent")], @@ -826,6 +1528,18 @@ class TestQueueCommands(unittest.TestCase): self.assertFalse(ran.exists()) self.assertEqual(self.entries(), []) + def test_r4_a_restart_onto_the_installed_code_keeps_the_entry_its_place_and_its_enqueue_time(self): + host = self.install_host_skills() + host.write_text(host.read_text(encoding="utf-8") + "\n", encoding="utf-8") + self.cfg["slot_limits"]["deploy"] = 0 + self.write_config() + hold, err = self.blocked_hold("true") + data = json.loads((self.root / "queue.json").read_text(encoding="utf-8")) + hold.send_signal(signal.SIGTERM) + self.assertEqual(hold.wait(timeout=30), 128 + signal.SIGTERM) + self.assertIn("Restarting this hold with", err.read_text(encoding="utf-8")) + self.assertEqual(([e["id"] for e in data["entries"]], data["seq"], data.get("handover")), (["deploy-1"], 1, {})) + def test_list_marks_an_entry_on_other_code_than_the_host_install(self): queue = _load(QUEUE) me, mine = os.getpid(), queue.identity(os.getpid()) @@ -838,6 +1552,116 @@ class TestQueueCommands(unittest.TestCase): self.assertEqual(reasons[0], "granted on its next poll") self.assertTrue(reasons[1].startswith(f"queue code without a stamp, the host runs {queue.CODE}: "), reasons[1]) + def test_list_names_the_line_the_backfill_and_the_escalated_head(self): + queue = _load(QUEUE) + me, mine = os.getpid(), queue.identity(os.getpid()) + self.cfg["slot_limits"] = {"test": 5, "deploy": 1} + self.write_config() + + def entry(seq: int, status: str, **fields: object) -> dict: + return {**stored(seq, status, me, mine, me if status == "active" else None, mine if status == "active" else None), + "code": queue.CODE, **fields} + + (self.root / "queue.json").write_text(json.dumps({"seq": 4, "entries": [ + entry(1, "waiting", id="deploy-1", kind="deploy", workdir="/core"), + entry(2, "waiting", workdir="/core"), + entry(3, "waiting", workdir="/x"), + entry(4, "active", kind="deploy", workdir="/y", reason="backfilled", footprint={"ram": 0, "cpu": 0}), + ], "stall": {"grant": 0, "since": 0, "fired": ["post"], "cleaned": False}}), encoding="utf-8") + listing = self.run_queue("list") + self.assertEqual(listing.returncode, 0, listing.stderr) + self.assertEqual({line.split("\t")[1]: line.split("\t")[-1] for line in listing.stdout.splitlines()[1:]}, { + "test-4": "backfilled", "deploy-1": "escalated post: all 1 deploy slots taken", + "test-2": "waits in line behind deploy-1", "test-3": "backfilled", + }) + + def test_r5_list_json_gives_each_row_its_class_prio_wsjf_waits_and_the_trace_of_the_pass_that_set_its_reason(self): + queue = _load(QUEUE) + me, mine = os.getpid(), queue.identity(os.getpid()) + self.cfg["slot_limits"] = {"test": 5, "deploy": 0} + self.write_config() + register = json.loads((self.root / "rasci.json").read_text(encoding="utf-8")) + register["work/a"].update(prio=2, estimates={"Operator": {"value": 8, "complexity": 2}}) + (self.root / "rasci.json").write_text(json.dumps(register), encoding="utf-8") + now = time.time() + + def entry(seq: int, status: str, **fields: object) -> dict: + return {**stored(seq, status, me, mine, me if status == "active" else None, mine if status == "active" else None), + "code": queue.CODE, "enqueued": now - 120, **fields} + + polled = ["unranked", "arrival 2", "line /core behind deploy-1"] + (self.root / "queue.json").write_text(json.dumps({"seq": 3, "entries": [ + entry(1, "waiting", id="deploy-1", kind="deploy", workdir="/core", hotfix="fix-1"), + entry(2, "waiting", workdir="/core", name="Main", polled=now, reason="waits in line behind deploy-1", trace=polled), + entry(3, "active", workdir="/y", granted=now - 60, reason="backfilled", trace=["arrival 3"], footprint={"ram": 0, "cpu": 0}), + ]}), encoding="utf-8") + listing = self.run_queue("list", "--json") + self.assertEqual(listing.returncode, 0, listing.stderr) + rows = json.loads(listing.stdout) + self.assertEqual([list(row) for row in rows], [[ + "position", "id", "status", "kind", "task", "session", "agent", "class", "prio", "wsjf", "waited_s", "eta_s", "reason", "trace", + ]] * 3) + shown = {row["id"]: row for row in rows} + self.assertEqual([(row["position"], row["id"]) for row in rows], [(None, "test-3"), (1, "deploy-1"), (2, "test-2")]) + self.assertEqual({i: (r["class"], r["prio"], r["wsjf"], r["agent"]) for i, r in shown.items()}, { + "test-3": ("fix", 2, 4.0, "s1"), "deploy-1": ("hotfix", 2, 4.0, "s1"), "test-2": ("main", 2, 4.0, "Main"), + }) + self.assertEqual({i: (r["reason"], r["trace"]) for i, r in shown.items()}, { + "test-3": ("backfilled", ["arrival 3"]), + "deploy-1": ("all 0 deploy slots taken", ["hotfix first", "rank 0", "arrival 1", "all 0 deploy slots taken"]), + "test-2": ("waits in line behind deploy-1", polled), + }) + self.assertTrue(all(118 <= r["waited_s"] <= 180 for i, r in shown.items() if i != "test-3")) + self.assertEqual((shown["test-3"]["waited_s"], shown["test-2"]["eta_s"]), (60, None)) + + def test_a_poll_of_a_stalled_queue_escalates_over_the_radio_once(self): + self.cfg["slot_limits"]["deploy"] = 0 + self.write_config() + holds = self.root / "holds" + holds.mkdir() + script = "import fcntl, sys, time; h = open(sys.argv[1], 'w'); fcntl.flock(h, fcntl.LOCK_EX); print('locked', flush=True); time.sleep(60)" + holder = subprocess.Popen([sys.executable, "-c", script, str(holds / "5-6.lock")], stdout=subprocess.PIPE, text=True) + self.addCleanup(holder.wait) + self.addCleanup(holder.kill) + self.addCleanup(holder.stdout.close) + self.assertEqual(holder.stdout.readline().strip(), "locked") + now = time.time() + head = {**stored(1, "waiting", 1, "1", None, None), "id": "deploy-1", "kind": "deploy", "lock": "5-6", + "workdir": "/core", "enqueued": now - 40 * 60, "polled": now} + (self.root / "queue.json").write_text(json.dumps({"seq": 1, "entries": [head], "granted_at": now - 31 * 60}), + encoding="utf-8") + err = self.base / "hold.err" + with open(err, "w", encoding="utf-8") as handle: + hold = subprocess.Popen( + [sys.executable, str(QUEUE), "hold", "--kind", "deploy", "--task", "work/a", "--session", "s1", "--", "true"], + cwd=self.base, env=self.env, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=handle, + ) + self.addCleanup(hold.wait) + self.addCleanup(hold.kill) + self.wait_for(lambda: "waits at position 2" in err.read_text(encoding="utf-8")) + hold.send_signal(signal.SIGTERM) + self.assertEqual(hold.wait(timeout=30), 128 + signal.SIGTERM) + self.assertEqual(err.read_text(encoding="utf-8").splitlines(), [ + "queue: escalated post", "queue: deploy-2 waits at position 2: all 0 deploy slots taken", + ]) + [row] = [json.loads(line) for line in (self.root / "news.jsonl").read_text(encoding="utf-8").splitlines()] + self.assertEqual((row["class"], row["to"], row["from"], row["text"]), ( + "post", ["Main"], "s1", + "kvb-org-queue: no grant for 30 min; head deploy-1 (deploy, work/a) waits: all 0 deploy slots taken", + )) + data = json.loads((self.root / "queue.json").read_text(encoding="utf-8")) + self.assertEqual((data["stall"]["fired"], data["cleanup"]["head"], data["cleanup"]["cleaned"]), (["post"], "deploy-1", False)) + + def test_r3_any_radio_silence_stops_a_grant_even_one_in_the_queues_own_words(self): + now = time.time() + silence = {"since": now, "until": now + 7200, "by": "s1", + "text": "kvb-org-queue: no grant for 1440 min; head deploy-9 (deploy, work/a) waits: x"} + (self.root / "radio-silence.json").write_text(json.dumps(silence), encoding="utf-8") + hold, err = self.blocked_hold("true") + hold.send_signal(signal.SIGTERM) + self.assertEqual(hold.wait(timeout=30), 128 + signal.SIGTERM) + self.assertEqual(err.read_text(encoding="utf-8"), "queue: deploy-1 waits at position 1: radio silence\n") + def test_hold_refuses_bad_input_with_exit_2_and_enqueues_nothing(self): def hold(*args: str) -> list[str]: return ["hold", "--session", "s1", *args] diff --git a/tests/test_queue_replay.py b/tests/test_queue_replay.py new file mode 100644 index 0000000..c9fba60 --- /dev/null +++ b/tests/test_queue_replay.py @@ -0,0 +1,296 @@ +from __future__ import annotations + +import importlib.util +import json +import os +import statistics +import tempfile +import unittest +from collections import Counter +from pathlib import Path +from unittest import mock + +REPO_ROOT = Path(__file__).resolve().parents[1] +QUEUE = REPO_ROOT / "skills" / "kvb-org-queue" / "scripts" / "queue.py" +FIXTURE = Path(__file__).resolve().parent / "queue-replay-20261114.json" +FP = { + "quality-high": {"ram": 40.6, "cpu": 59.8}, "test": {"ram": 34.5, "cpu": 24.5}, + "test-nostack": {"ram": 7.2, "cpu": 9.8}, "iteration": {"ram": 10.4, "cpu": 7.9}, + "gui-dind": {"ram": 20.7, "cpu": 63.6}, "swarm": {"ram": 40.6, "cpu": 59.8}, "operator": {"ram": 10.0, "cpu": 10.0}, +} +SLOTS = {"gui-dind": 1, "iteration": 3, "operator": 1, "quality-high": 1, "swarm": 1, "test": 1, "test-nostack": 4} +BUDGET = {"ram": 75, "cpu": 100} +LIMITS = {"cpu_percent": 101, "mem_percent": 85} +EXCLUSIVE = {"quality-high", "test"} +LIGHT = {"test-nostack"} +FLOOR = 15.0 +STEP = 20 +RAMP = 900 +HOUR = 3600 +HORIZON = 24 * HOUR +LINES_MEAN = 13.53 +SCENARIOS = { + "plain": {}, "ram45": {"baseline": 45.0}, "hotfix": {"hotfix": True}, "lines": {"hotfix": True, "lines": True, "agents": True}, + "real": {"free": 110.0, "ranked": True, "agents": True}, +} + + +def load(path: Path, name: str): + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def scenario(jobs: list[dict], name: str) -> list[dict]: + """Returns: + jobs as the scenario name replays them: `hotfix` marks every Main + quality-high job a hotfix; `lines` adds before each such hotfix four + light jobs that fill the test-nostack slots, then in the hotfix's + claims an older light job and an older quality-high of Main, the + shapes of the skeptic's probes P3 and P3b, each with a seq below + every job enqueued after it (placed()); the older quality-high is + held to bound() and rivals() instead of master's wait. + """ + spec = SCENARIOS[name] + if not spec.get("hotfix"): + return jobs + marked = [{**j, "hotfix": "main" if j["main"] and j["kind"] == "quality-high" else None} for j in jobs] + if not spec.get("lines"): + return marked + added = [] + for h in (j for j in marked if j["hotfix"]): + base = {"task": h["task"], "need_gb": 0, "hotfix": None, "enqueued": h["enqueued"] - 60} + added += [{**base, "id": f"{h['id']}-fill{k}", "seq": placed(marked, h["enqueued"] - 90, 0.1 + 0.1 * k), "kind": "test-nostack", + "name": "other", "main": False, "workdir": f"/replay/{h['id']}/fill{k}", "claims": [f"/replay/{h['id']}/fill{k}"], + "enqueued": h["enqueued"] - 90, "dur": 900.0} for k in range(4)] + added += [{**base, "id": f"{h['id']}-older-light", "seq": placed(marked, base["enqueued"], 0.5), "kind": "test-nostack", + "name": "Main", "main": True, "workdir": h["workdir"], "claims": h["claims"], "dur": 300.0}, + {**base, "id": f"{h['id']}-older-gate", "seq": placed(marked, h["enqueued"] - 30, 0.6), "kind": "quality-high", + "name": "Main", "main": True, "workdir": h["workdir"], "claims": h["claims"], "need_gb": 15, + "enqueued": h["enqueued"] - 30, "dur": h["dur"]}] + return sorted(marked + added, key=lambda j: j["seq"]) + + +def placed(jobs: list[dict], enqueued: float, fraction: float) -> float: + """Returns: + A seq fraction (0 to 1) below the lowest seq of the jobs enqueued + after enqueued, so arrival order agrees with the enqueue stamps. + """ + return min(j["seq"] for j in jobs if j["enqueued"] > enqueued) - 1 + fraction + + +def world(name: str, jobs: list[dict]) -> dict: + """Returns: + The host of scenario name: baseline (RAM % used outside the queue, + None for none), free (t -> free disk in GB; `real` drops it below + FLOOR for the hour after the middle job's enqueue), ranks (`real`: + every task by name) and agents (`real`, `lines`: Main first, then by + name). + """ + spec = SCENARIOS[name] + dip = sorted(j["enqueued"] for j in jobs)[len(jobs) // 2] + return { + "baseline": spec.get("baseline"), + "free": (lambda t: 12.0 if dip <= t < dip + HOUR else spec["free"]) if "free" in spec else (lambda t: 1000.0), + "ranks": {task: n for n, task in enumerate(sorted({j["task"] for j in jobs}))} if spec.get("ranked") else {}, + "agents": {agent: n for n, agent in enumerate(sorted({j["name"] for j in jobs}, key=lambda a: (a != "Main", a)))} + if spec.get("agents") else {}, + } + + +def loaded(queue, active: list[dict], waiting: list[dict], baseline: float | None, t: float) -> dict[str, str]: + """Returns: + {waiting job id: load_reason()} with RAM used = baseline + the ramped + active jobs' RAM, the young ones' RAM still to come; queue.Outside + when the baseline alone keeps the job out (held_back()). + """ + if baseline is None: + return {} + ramped = sum(FP[a["kind"]]["ram"] for a in active if t - a["granted"] >= RAMP) + young = sum(FP[a["kind"]]["ram"] for a in active if t - a["granted"] < RAMP) + out = {} + for j in waiting: + reason = queue.load_reason(LIMITS, 0.0, baseline + ramped, young + FP[j["kind"]]["ram"]) + if reason is not None: + alone = queue.load_reason(LIMITS, 0.0, baseline, FP[j["kind"]]["ram"]) + out[j["id"]] = reason if alone is None else queue.Outside(reason) + return out + + +def simulate(queue, jobs: list[dict], name: str, branch: bool, events: list | None = None) -> dict[str, float | None]: + """Args: + queue: the queue.py module to replay. + jobs: the scenario's jobs, each with its claims, enqueued and dur. + name: the scenario, whose world() the replay runs on. + branch: whether queue has this branch's evaluate (drain, latched(), + stalled()) or master's. + events: when given, the branch's stall events are appended as (t, + label, head id): each ESCALATION send, `cleanup` for each + cleanup started, `notice` for each notice. + + Returns: + {job id: minutes from enqueue to grant, None when it never started + within HORIZON after the last enqueue}; every STEP seconds each + waiting job polls and every job granted starts at once. + """ + host = world(name, jobs) + pending = sorted(({**j, "status": "waiting", "granted": None} for j in jobs), key=lambda j: (j["enqueued"], j["seq"])) + t, end = pending[0]["enqueued"], max(j["enqueued"] for j in pending) + HORIZON + waiting: list[dict] = [] + active: list[dict] = [] + data: dict = {"drain": None} + grants: dict[str, float] = {} + while (pending or waiting) and t <= end: + active = [a for a in active if a["granted"] + a["dur"] > t] + while pending and pending[0]["enqueued"] <= t: + waiting.append(pending.pop(0)) + if not waiting: + t = pending[0]["enqueued"] + continue + for j in waiting: + j["polled"] = t + entries = active + waiting + load, free = loaded(queue, active, waiting, host["baseline"], t), host["free"](t) + args = (entries, free, FLOOR, "stop" if free < FLOOR else "ok", "ok", False, SLOTS, {j["id"]: FP[j["kind"]] for j in entries}, + BUDGET, host["ranks"], host["agents"], load, EXCLUSIVE, LIGHT) + if branch: + verdicts, drain, order = queue.evaluate(*args, data["drain"], t) + data["drain"] = queue.latched(data, drain, t) + go = [j for j in waiting if verdicts[j["id"]] is None] + if events is not None: + data["entries"] = entries + sends, clean, _ = queue.stalled(data, order, verdicts, FLOOR, free, lambda: (False, []), t) + events += [(t, "notice" if label.startswith("noticed") else label, order[0]["id"]) for label, _ in sends] + events += [(t, "cleanup", order[0]["id"])] if clean else [] + else: + verdicts = queue.evaluate(*args, t) + go = [j for j in waiting if verdicts[j["id"]] is None and j["id"] not in load] + for j in go: + j.update(status="active", granted=t) + grants[j["id"]] = t + waiting.remove(j) + active.append(j) + if go and branch: + data[queue.GRANTED] = t + data.pop(queue.STALL, None) + t += STEP + return {j["id"]: (grants[j["id"]] - j["enqueued"]) / 60 if j["id"] in grants else None for j in jobs} + + +def mean(waits: dict[str, float | None], jobs: list[dict]) -> float: + cutoff = max(j["enqueued"] for j in jobs) + HORIZON + return statistics.mean(w if w is not None else (cutoff - j["enqueued"]) / 60 for j, w in ((j, waits[j["id"]]) for j in jobs)) + + +def overlaps(waits: dict[str, float | None], jobs: list[dict]) -> list[tuple[str, str]]: + runs = [(j, j["enqueued"] + waits[j["id"]] * 60) for j in jobs if waits[j["id"]] is not None] + return [(a["id"], b["id"]) for n, (a, ga) in enumerate(runs) for b, gb in runs[n + 1:] + if set(a["claims"]) & set(b["claims"]) and ga < gb + b["dur"] and gb < ga + a["dur"]] + + +def started(waits: dict[str, float | None], job: dict) -> float: + return job["enqueued"] + waits[job["id"]] * 60 + + +def took(e: dict, g: float, x: dict, runs: list[tuple[dict, float]]) -> bool: + """Returns: + Whether e, starting at g beside runs ((job, start)), took what x + needs: its slot (e's was the last free one), its exclusive room (one + of the two exclusive, the other not light) or its budget (only the + light jobs count beside an exclusive x). + """ + running = [j for j, s in runs if j is not e and s <= g < s + j["dur"]] + beside = [j for j in running if j["kind"] in LIGHT or x["kind"] not in EXCLUSIVE] + slot = e["kind"] == x["kind"] and sum(j["kind"] == e["kind"] for j in running) + 1 >= SLOTS[e["kind"]] + room = x["kind"] in EXCLUSIVE and e["kind"] not in LIGHT or e["kind"] in EXCLUSIVE and x["kind"] not in LIGHT + return slot or room or any(sum(FP[j["kind"]][r] for j in beside) + FP[e["kind"]][r] + FP[x["kind"]][r] > BUDGET[r] for r in BUDGET) + + +def bound(waits: dict[str, float | None], jobs: list[dict], x: dict) -> float: + """Returns: + The latest grant the rule allows x: one STEP after the later of its + enqueue and the end of every job started before it that is older and + shares one of its claims or is a hotfix. + """ + ends = [started(waits, j) + j["dur"] for j in jobs if j is not x and waits[j["id"]] is not None and started(waits, j) < started(waits, x) + and (j["enqueued"] < x["enqueued"] and set(j["claims"]) & set(x["claims"]) or j["hotfix"])] + return max([x["enqueued"], *ends]) + STEP + + +def rivals(waits: dict[str, float | None], jobs: list[dict], x: dict) -> list[str]: + """Returns: + The jobs after x in arrival (seq), of other claims and no hotfix, + that started while x waited and took what it needs (took()). + """ + runs = [(j, started(waits, j)) for j in jobs if waits[j["id"]] is not None] + return [e["id"] for e, g in runs if not e["hotfix"] and e["seq"] > x["seq"] and not set(e["claims"]) & set(x["claims"]) + and x["enqueued"] <= g < started(waits, x) and took(e, g, x, runs)] + + +class TestReplay(unittest.TestCase): + @classmethod + def setUpClass(cls): + tmp = tempfile.TemporaryDirectory() + cls.addClassCleanup(tmp.cleanup) + with mock.patch.dict(os.environ, {"AGENTS_COUNCIL_ROOT": str(Path(tmp.name).resolve() / "council")}): + cls.queue = load(QUEUE, "queue_replay_under_test") + cls.fixture = json.loads(FIXTURE.read_text(encoding="utf-8")) + cls.jobs = cls.fixture["jobs"] + cls.scenes = {name: scenario(cls.jobs, name) for name in SCENARIOS} + cls.events: list = [] + cls.waits = {name: simulate(cls.queue, cls.scenes[name], name, True, cls.events if name == "real" else None) for name in SCENARIOS} + + def test_the_fixture_is_todays_140_finished_jobs_with_frozen_claims_and_masters_waits(self): + self.assertEqual(len(self.jobs), 140) + self.assertTrue(all("claims" in j and j["dur"] > 0 for j in self.jobs)) + self.assertEqual(set(self.fixture["master"]), set(SCENARIOS)) + self.assertEqual(Counter(j["id"].rsplit("-", 1)[-1] for j in self.scenes["lines"] if j["id"].startswith("quality-high-1930-")), + Counter({"fill0": 1, "fill1": 1, "fill2": 1, "fill3": 1, "light": 1, "gate": 1})) + + def test_the_mean_wait_stays_within_a_minute_of_master_and_lines_within_a_minute_of_its_main_first_mean(self): + for name in ("plain", "ram45", "hotfix", "real"): + jobs = self.scenes[name] + with self.subTest(scenario=name): + self.assertLessEqual(mean(self.waits[name], jobs), mean(self.fixture["master"][name], jobs) + 1) + with self.subTest(scenario="lines"): + self.assertLessEqual(mean(self.waits["lines"], self.scenes["lines"]), LINES_MEAN + 1, + "lines trades mean wait for Main's bound, capped a minute above the accepted cost") + + def test_no_scenario_starves_a_job_master_runs(self): + for name in SCENARIOS: + with self.subTest(scenario=name): + starved = {i for i, w in self.waits[name].items() if w is None} + self.assertEqual(starved - {i for i, w in self.fixture["master"][name].items() if w is None}, set()) + + def test_no_hotfix_and_no_main_job_waits_longer_than_on_master(self): + for name in ("plain", "hotfix", "lines", "real"): + master, branch = self.fixture["master"][name], self.waits[name] + jobs = [j for j in self.scenes[name] if (j["main"] or j["hotfix"]) and not j["id"].endswith("-older-gate")] + with self.subTest(scenario=name): + self.assertEqual([(j["id"], master[j["id"]], branch[j["id"]]) for j in jobs if branch[j["id"]] is None or ( + master[j["id"]] is not None and branch[j["id"]] > master[j["id"]] + STEP / 60)], []) + + def test_an_older_main_gate_starts_by_its_bound_and_no_younger_rival_takes_its_room(self): + waits, jobs = self.waits["lines"], self.scenes["lines"] + gates = [j for j in jobs if j["id"].endswith("-older-gate")] + self.assertEqual(len(gates), 2) + self.assertEqual({x["id"]: round((bound(waits, jobs, x) - x["enqueued"]) / 60, 1) for x in gates}, + {"quality-high-16-older-gate": 19.5, "quality-high-1930-older-gate": 33.5}) + self.assertEqual([(x["id"], waits[x["id"]]) for x in gates if waits[x["id"]] is None or started(waits, x) > bound(waits, jobs, x)], []) + self.assertEqual({x["id"]: rivals(waits, jobs, x) for x in gates if rivals(waits, jobs, x)}, {}) + + def test_no_two_jobs_of_one_claim_run_at_once(self): + for name, jobs in self.scenes.items(): + with self.subTest(scenario=name): + self.assertEqual(overlaps(self.waits[name], jobs), []) + + def test_the_disk_dip_starts_one_cleanup_and_one_notice_per_head_and_escalates(self): + per_head = Counter((label, head) for _, label, head in self.events if label in ("cleanup", "notice")) + self.assertTrue(per_head) + self.assertEqual({count for count in per_head.values()}, {1}) + self.assertIn("escalated post", {label for _, label, _ in self.events}) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_radio.py b/tests/test_radio.py index be0c02e..a9320bf 100644 --- a/tests/test_radio.py +++ b/tests/test_radio.py @@ -394,6 +394,22 @@ class TestRadio(unittest.TestCase): self._prompt("s3") self.assertEqual(log.read_text(encoding="utf-8"), "Agents radio MAYDAY by One: smoke\n") + def test_a_mayday_that_sets_no_silence_notifies_at_once_and_none_replaces_another_senders_silence(self): + log = self.root.parent / "notified.log" + script = "import sys; open(sys.argv[1], 'a').write(sys.argv[2] + '\\n')" + self.config.write_text(json.dumps({"notify": [sys.executable, "-c", script, str(log)]}), encoding="utf-8") + self._send("s1", "send", "mayday", "smoke") + self._send("s2", "send", "mayday", "queue stall", "--no-silence") + self._send("s3", "send", "mayday", "leaked credential") + silence = json.loads((self.root / "radio-silence.json").read_text(encoding="utf-8")) + self._prompt("s2") + self._prompt("s3") + self.assertEqual((silence["by"], silence["text"]), ("One", "smoke")) + self.assertEqual(log.read_text(encoding="utf-8").splitlines(), [ + "Agents radio MAYDAY by Two: queue stall", "Agents radio MAYDAY by Three: leaked credential", + "Agents radio MAYDAY by One: smoke", + ]) + def test_listener_wakes_on_mayday(self): proc = self._listen("s2") self.assertEqual(self._send("s1", "send", "mayday", "host down"), ["sent mayday to all", "wake: SendMessage to Three"])