Merge branch 'fix/hub-queue-stale-running' into unify/skills-20261112-night
This commit is contained in:
commit
1ae790028f
15 files changed
+534
-39
No files matched your search
@@ -36,7 +36,7 @@ RUN chmod 0755 /opt/voices /opt/whisper
|
|||||||
|
|
||||||
COPY server.py /app/server.py
|
COPY server.py /app/server.py
|
||||||
COPY kvb_orchestrator /app/kvb_orchestrator
|
COPY kvb_orchestrator /app/kvb_orchestrator
|
||||||
COPY client/hub_limits.py client/signing.py client/passkey.py client/verifier.py /app/client/
|
COPY client/hub_limits.py client/signing.py client/passkey.py client/verifier.py client/queue_live.py /app/client/
|
||||||
COPY templates /app/templates
|
COPY templates /app/templates
|
||||||
COPY vendor /app/vendor
|
COPY vendor /app/vendor
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,181 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import errno
|
||||||
|
import fcntl
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import stat
|
||||||
|
from collections.abc import Callable
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
HISTORY = "queue-history.jsonl"
|
||||||
|
HOLDS = "holds"
|
||||||
|
HOLD_TAG = re.compile(r"\d+-\d+")
|
||||||
|
JOB_SUFFIX = re.compile(r"\d+")
|
||||||
|
READ_FLAGS = os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK | os.O_CLOEXEC
|
||||||
|
DIRECTORY = os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | os.O_CLOEXEC
|
||||||
|
REFUSED = frozenset({errno.ELOOP, errno.ENOTDIR})
|
||||||
|
NO_HISTORY = frozenset({errno.ENOENT, errno.ELOOP, errno.ENOTDIR})
|
||||||
|
|
||||||
|
|
||||||
|
def moment(value: object) -> float | None:
|
||||||
|
if isinstance(value, bool) or not isinstance(value, int | float) or not 0 <= value < 1e11:
|
||||||
|
return None
|
||||||
|
return float(value)
|
||||||
|
|
||||||
|
|
||||||
|
def opened(council: Path, tag: str) -> tuple[int | None, bool]:
|
||||||
|
"""Returns:
|
||||||
|
(descriptor, gone): the descriptor of holds/<tag>.lock, opened as
|
||||||
|
queue.py's open_below walks to it, and False; else None and True
|
||||||
|
where queue.py's held() finds no lock (a missing directory or file)
|
||||||
|
or open_below refuses one (a link or a non-directory on the way),
|
||||||
|
None and False where the open fails otherwise.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
folder = os.open(council / HOLDS, DIRECTORY)
|
||||||
|
except FileNotFoundError:
|
||||||
|
return None, True
|
||||||
|
except OSError as error:
|
||||||
|
return None, error.errno in REFUSED
|
||||||
|
try:
|
||||||
|
return os.open(f"{tag}.lock", READ_FLAGS, dir_fd=folder), False
|
||||||
|
except FileNotFoundError:
|
||||||
|
return None, True
|
||||||
|
except OSError as error:
|
||||||
|
return None, error.errno in REFUSED
|
||||||
|
finally:
|
||||||
|
os.close(folder)
|
||||||
|
|
||||||
|
|
||||||
|
def holder_gone(council: Path, tag: object) -> bool:
|
||||||
|
"""Args:
|
||||||
|
council: the council directory.
|
||||||
|
tag: the lock tag of an entry that has one; queue.py's living()
|
||||||
|
decides by held(tag) alone then.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
not held(tag) for every lock queue.py's held() decides: True for a
|
||||||
|
tag queue.py would not write, a missing lock and a lock a
|
||||||
|
non-blocking shared flock gets, False for a held one. A hard-linked
|
||||||
|
lock gets the same probe. A link or a non-regular file, which no
|
||||||
|
hold can create, is True; a lock that cannot be opened or flocked
|
||||||
|
otherwise (EACCES) is False.
|
||||||
|
"""
|
||||||
|
if not isinstance(tag, str) or not HOLD_TAG.fullmatch(tag):
|
||||||
|
return True
|
||||||
|
fd, gone = opened(council, tag)
|
||||||
|
if fd is None:
|
||||||
|
return gone
|
||||||
|
try:
|
||||||
|
if not stat.S_ISREG(os.fstat(fd).st_mode):
|
||||||
|
return True
|
||||||
|
fcntl.flock(fd, fcntl.LOCK_SH | fcntl.LOCK_NB)
|
||||||
|
except OSError:
|
||||||
|
return False
|
||||||
|
finally:
|
||||||
|
os.close(fd)
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def read_history(path: Path) -> list[dict] | None:
|
||||||
|
"""Args:
|
||||||
|
path: a queue-history.jsonl; a missing file, a link or another
|
||||||
|
non-regular file reads as no rows.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
The JSON object rows of the whole file, in file order; an undecodable
|
||||||
|
byte or a bad line costs that line only. None when the read fails
|
||||||
|
otherwise (EACCES, EIO, EMFILE).
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
fd = os.open(path, READ_FLAGS)
|
||||||
|
except OSError as error:
|
||||||
|
return [] if error.errno in NO_HISTORY else None
|
||||||
|
try:
|
||||||
|
if not stat.S_ISREG(os.fstat(fd).st_mode):
|
||||||
|
return []
|
||||||
|
with open(fd, "rb", closefd=False) as handle:
|
||||||
|
data = handle.read()
|
||||||
|
except OSError:
|
||||||
|
return None
|
||||||
|
finally:
|
||||||
|
os.close(fd)
|
||||||
|
rows = []
|
||||||
|
for line in data.decode("utf-8", errors="replace").splitlines():
|
||||||
|
try:
|
||||||
|
row = json.loads(line)
|
||||||
|
except (ValueError, RecursionError):
|
||||||
|
continue
|
||||||
|
if isinstance(row, dict):
|
||||||
|
rows.append(row)
|
||||||
|
return rows
|
||||||
|
|
||||||
|
|
||||||
|
def history_jobs(rows: list[dict]) -> dict[str, list[dict]]:
|
||||||
|
"""Returns:
|
||||||
|
The history rows with a start by entry id, the row id without its
|
||||||
|
last dash and digits.
|
||||||
|
"""
|
||||||
|
jobs: dict[str, list[dict]] = {}
|
||||||
|
for row in rows:
|
||||||
|
entry_id, _, suffix = str(row.get("id", "")).rpartition("-")
|
||||||
|
if entry_id and JOB_SUFFIX.fullmatch(suffix) and moment(row.get("start")) is not None:
|
||||||
|
jobs.setdefault(entry_id, []).append(row)
|
||||||
|
return jobs
|
||||||
|
|
||||||
|
|
||||||
|
def exit_of(row: dict) -> int | None:
|
||||||
|
code = row.get("exit")
|
||||||
|
return code if isinstance(code, int) and not isinstance(code, bool) else None
|
||||||
|
|
||||||
|
|
||||||
|
def outcome(entry: dict, created: float, gone: bool, jobs: dict[str, list[dict]]) -> tuple[bool, dict | None]:
|
||||||
|
"""Args:
|
||||||
|
entry: a queue.json entry.
|
||||||
|
created: its enqueue time.
|
||||||
|
gone: holder_gone() of its lock, False without a lock key.
|
||||||
|
jobs: history_jobs().
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
(kept, outcome): an active entry whose job has a history row started
|
||||||
|
after the enqueue is kept with {"state": "ended", "exit", "end"} of
|
||||||
|
the newest such row by end; else an active one whose holder is gone
|
||||||
|
is kept with {"state": "gone", "exit": None, "end": None}; else any
|
||||||
|
other entry whose holder is gone is not kept; else the entry is kept
|
||||||
|
live with outcome None.
|
||||||
|
"""
|
||||||
|
active = entry.get("status") == "active"
|
||||||
|
ident = entry.get("id")
|
||||||
|
rows = [row for row in jobs.get(ident, []) if (moment(row.get("start")) or 0.0) >= created] if active and isinstance(ident, str) else []
|
||||||
|
if rows:
|
||||||
|
row = max(rows, key=lambda found: moment(found.get("end")) or 0.0)
|
||||||
|
return True, {"state": "ended", "exit": exit_of(row), "end": moment(row.get("end"))}
|
||||||
|
if gone:
|
||||||
|
return active, {"state": "gone", "exit": None, "end": None} if active else None
|
||||||
|
return True, None
|
||||||
|
|
||||||
|
|
||||||
|
def settled(council: Path, found: list[tuple[dict, float]], history: Callable[[], list[dict]]) -> list[tuple[dict, float, dict | None]]:
|
||||||
|
"""Args:
|
||||||
|
council: the council directory.
|
||||||
|
found: the queue.json entries with their enqueue time.
|
||||||
|
history: returns the queue-history.jsonl rows, None for an
|
||||||
|
unreadable file; called once, after every hold lock is probed,
|
||||||
|
and only while an entry is active.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Each kept entry with its enqueue time and outcome, as outcome()
|
||||||
|
reads it, in the order of found; an entry without a lock key, which
|
||||||
|
queue.py's living() judges by pids of other PID namespaces, is never
|
||||||
|
gone and ends only by its history row.
|
||||||
|
"""
|
||||||
|
gone = ["lock" in entry and holder_gone(council, entry["lock"]) for entry, _ in found]
|
||||||
|
jobs = history_jobs(history() or []) if any(entry.get("status") == "active" for entry, _ in found) else {}
|
||||||
|
kept = []
|
||||||
|
for (entry, created), dead in zip(found, gone, strict=True):
|
||||||
|
keep, result = outcome(entry, created, dead, jobs)
|
||||||
|
if keep:
|
||||||
|
kept.append((entry, created, result))
|
||||||
|
return kept
|
||||||
@@ -44,6 +44,7 @@ from __future__ import annotations
|
|||||||
import argparse
|
import argparse
|
||||||
import base64
|
import base64
|
||||||
import datetime
|
import datetime
|
||||||
|
import functools
|
||||||
import hashlib
|
import hashlib
|
||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
@@ -55,6 +56,8 @@ import time
|
|||||||
from collections.abc import Callable
|
from collections.abc import Callable
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
|
import queue_live
|
||||||
|
|
||||||
INTERVAL = 60
|
INTERVAL = 60
|
||||||
BUCKET = 900
|
BUCKET = 900
|
||||||
HOUR = 3600
|
HOUR = 3600
|
||||||
@@ -226,9 +229,19 @@ def gpus(sys_root: Path, run: Callable[[list[str]], str | None]) -> list[dict]:
|
|||||||
|
|
||||||
|
|
||||||
def queue(council: Path) -> dict:
|
def queue(council: Path) -> dict:
|
||||||
|
"""Returns:
|
||||||
|
{waiting, active}: the queue.json entries with an enqueue time that
|
||||||
|
queue_live.settled() keeps without an outcome, as the hub counts them.
|
||||||
|
"""
|
||||||
data = read_json(council / "queue.json")
|
data = read_json(council / "queue.json")
|
||||||
entries = data.get("entries") if isinstance(data, dict) else None
|
entries = data.get("entries") if isinstance(data, dict) else None
|
||||||
statuses = [entry.get("status") for entry in entries if isinstance(entry, dict)] if isinstance(entries, list) else []
|
found = []
|
||||||
|
for entry in entries if isinstance(entries, list) else []:
|
||||||
|
created = queue_live.moment(entry.get("enqueued")) if isinstance(entry, dict) else None
|
||||||
|
if created is not None:
|
||||||
|
found.append((entry, created))
|
||||||
|
history = functools.partial(queue_live.read_history, council / queue_live.HISTORY)
|
||||||
|
statuses = [entry.get("status") for entry, _, result in queue_live.settled(council, found, history) if result is None]
|
||||||
return {"waiting": statuses.count("waiting"), "active": statuses.count("active")}
|
return {"waiting": statuses.count("waiting"), "active": statuses.count("active")}
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -1,15 +1,23 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
|
import threading
|
||||||
|
from collections.abc import Callable
|
||||||
|
from dataclasses import dataclass, field
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
from .council import agent
|
from .council import agent
|
||||||
from .events import event
|
from .events import event
|
||||||
from .names import Directory
|
from .names import Directory
|
||||||
from .records import epoch, read_json, shown
|
from .records import epoch, read_json, shown
|
||||||
|
from .signed import client_module
|
||||||
|
|
||||||
|
queue_live = client_module("queue_live", {})
|
||||||
|
|
||||||
CRON_ACTIVE = 900
|
CRON_ACTIVE = 900
|
||||||
TIMER_KEYS = ("OnCalendar", "OnActiveSec", "OnBootSec", "OnStartupSec", "OnUnitActiveSec", "OnUnitInactiveSec")
|
TIMER_KEYS = ("OnCalendar", "OnActiveSec", "OnBootSec", "OnStartupSec", "OnUnitActiveSec", "OnUnitInactiveSec")
|
||||||
CRON_LABELS = {"cron": "Cron", "wakeup": "Wakeup", "background": "Background", "timer": "Timer", "queue": "Queue"}
|
CRON_LABELS = {"cron": "Cron", "wakeup": "Wakeup", "background": "Background", "timer": "Timer", "queue": "Queue"}
|
||||||
|
OUTCOME = "hub_outcome"
|
||||||
|
|
||||||
|
|
||||||
def cron_state(cron: dict, beats: dict[str, float], now: float) -> dict:
|
def cron_state(cron: dict, beats: dict[str, float], now: float) -> dict:
|
||||||
@@ -41,10 +49,52 @@ def timer_rows(folder: Path) -> list[dict]:
|
|||||||
return rows
|
return rows
|
||||||
|
|
||||||
|
|
||||||
def queue_entries(council: Path) -> list[tuple[dict, float]]:
|
@dataclass
|
||||||
"""Returns:
|
class History:
|
||||||
Each kvb-org-queue entry of queue.json with its enqueue time; an
|
path: Path
|
||||||
entry without one is no entry.
|
lock: threading.Lock = field(default_factory=threading.Lock)
|
||||||
|
cached: list[dict] = field(default_factory=list)
|
||||||
|
stamp: tuple[int, int, int] | None = None
|
||||||
|
|
||||||
|
def rows(self) -> list[dict]:
|
||||||
|
"""Returns:
|
||||||
|
queue_live.read_history() of the whole file, parsed again only
|
||||||
|
when its inode, size or mtime changed; after a failed read the
|
||||||
|
rows read before, and the next call reads again. The list is
|
||||||
|
shared, read only.
|
||||||
|
"""
|
||||||
|
with self.lock:
|
||||||
|
try:
|
||||||
|
info = os.lstat(self.path)
|
||||||
|
except FileNotFoundError:
|
||||||
|
self.cached, self.stamp = [], None
|
||||||
|
return self.cached
|
||||||
|
except OSError:
|
||||||
|
self.stamp = None
|
||||||
|
return self.cached
|
||||||
|
stamp = (info.st_ino, info.st_size, info.st_mtime_ns)
|
||||||
|
if stamp != self.stamp:
|
||||||
|
found = queue_live.read_history(self.path)
|
||||||
|
if found is None:
|
||||||
|
self.stamp = None
|
||||||
|
else:
|
||||||
|
self.cached, self.stamp = found, stamp
|
||||||
|
return self.cached
|
||||||
|
|
||||||
|
|
||||||
|
def outcome_of(entry: dict) -> dict | None:
|
||||||
|
found = entry.get(OUTCOME)
|
||||||
|
return found if isinstance(found, dict) else None
|
||||||
|
|
||||||
|
|
||||||
|
def queue_entries(council: Path, history: Callable[[], list[dict]]) -> list[tuple[dict, float]]:
|
||||||
|
"""Args:
|
||||||
|
council: the council directory.
|
||||||
|
history: returns the queue-history.jsonl rows, History.rows.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Each entry of queue.json with an enqueue time that queue_live.settled()
|
||||||
|
keeps, with that time, as a copy whose OUTCOME is its outcome.
|
||||||
"""
|
"""
|
||||||
data = read_json(council / "queue.json")
|
data = read_json(council / "queue.json")
|
||||||
entries = data.get("entries") if isinstance(data, dict) else None
|
entries = data.get("entries") if isinstance(data, dict) else None
|
||||||
@@ -53,17 +103,26 @@ def queue_entries(council: Path) -> list[tuple[dict, float]]:
|
|||||||
created = epoch(entry.get("enqueued")) if isinstance(entry, dict) else None
|
created = epoch(entry.get("enqueued")) if isinstance(entry, dict) else None
|
||||||
if created is not None:
|
if created is not None:
|
||||||
found.append((entry, created))
|
found.append((entry, created))
|
||||||
return found
|
return [({**entry, OUTCOME: result}, created) for entry, created, result in queue_live.settled(council, found, history)]
|
||||||
|
|
||||||
|
|
||||||
def queue_rows(council: Path) -> list[dict]:
|
def queue_state(entry: dict) -> str:
|
||||||
|
found = outcome_of(entry)
|
||||||
|
if found is None:
|
||||||
|
return "running" if entry.get("status") == "active" else "waiting"
|
||||||
|
if found.get("state") == "gone":
|
||||||
|
return "holder gone without an exit code"
|
||||||
|
code = found.get("exit")
|
||||||
|
return "ended" if code is None else f"exit {code}"
|
||||||
|
|
||||||
|
|
||||||
|
def queue_rows(council: Path, history: Callable[[], list[dict]]) -> list[dict]:
|
||||||
rows = []
|
rows = []
|
||||||
for entry, created in queue_entries(council):
|
for entry, created in queue_entries(council, history):
|
||||||
running = "running" if entry.get("status") == "active" else "waiting"
|
|
||||||
rows.append({
|
rows.append({
|
||||||
"key": f"queue:{shown(entry.get('id'))}", "type": "queue", "session": shown(entry.get("session")), "sub": "", "created": created,
|
"key": f"queue:{shown(entry.get('id'))}", "type": "queue", "session": shown(entry.get("session")), "sub": "", "created": created,
|
||||||
"deleted": None, "until": None, "job": shown(entry.get("id")), "expression": f"{shown(entry.get('kind'))} · {running}",
|
"deleted": None, "until": None, "job": shown(entry.get("id")), "expression": f"{shown(entry.get('kind'))} · {queue_state(entry)}",
|
||||||
"prompt": shown(entry.get("cmd")), "reason": shown(entry.get("task")), "active": True, "agent": agent(entry.get("name")),
|
"prompt": shown(entry.get("cmd")), "reason": shown(entry.get("task")), "active": outcome_of(entry) is None, "agent": agent(entry.get("name")),
|
||||||
})
|
})
|
||||||
return rows
|
return rows
|
||||||
|
|
||||||
|
|||||||
@@ -13,6 +13,7 @@ from .asking import Asking
|
|||||||
from .batches import Pending
|
from .batches import Pending
|
||||||
from .choosing import Choosing
|
from .choosing import Choosing
|
||||||
from .cleanup import Cleaning, Spool
|
from .cleanup import Cleaning, Spool
|
||||||
|
from .crons import History, queue_live
|
||||||
from .denying import Denying
|
from .denying import Denying
|
||||||
from .engines import Engines, Speech
|
from .engines import Engines, Speech
|
||||||
from .events import Log, Watch
|
from .events import Log, Watch
|
||||||
@@ -103,6 +104,7 @@ class Hub(Mcp, Reviewing, Asking, Messaging, Choosing, Approving, Denying, Reque
|
|||||||
touched: Touched
|
touched: Touched
|
||||||
voice_notes: VoiceNotes
|
voice_notes: VoiceNotes
|
||||||
replay: Path
|
replay: Path
|
||||||
|
history: History
|
||||||
|
|
||||||
def hosts(self) -> tuple[str, str]:
|
def hosts(self) -> tuple[str, str]:
|
||||||
return f"127.0.0.1:{self.public_port}", f"localhost:{self.public_port}"
|
return f"127.0.0.1:{self.public_port}", f"localhost:{self.public_port}"
|
||||||
@@ -142,6 +144,7 @@ def open_hub(council: Path, pages: Path, public_port: int, state: Path, host_sta
|
|||||||
timers=places.timers,
|
timers=places.timers,
|
||||||
kept=places.kept,
|
kept=places.kept,
|
||||||
council=council,
|
council=council,
|
||||||
|
history=History(council / queue_live.HISTORY),
|
||||||
pages=pages,
|
pages=pages,
|
||||||
public_port=public_port,
|
public_port=public_port,
|
||||||
reviews=reviews,
|
reviews=reviews,
|
||||||
|
|||||||
@@ -34,10 +34,14 @@ def read_regular(path: Path, limit: int | None) -> str:
|
|||||||
Returns:
|
Returns:
|
||||||
The file's text, UTF-8 decoded.
|
The file's text, UTF-8 decoded.
|
||||||
"""
|
"""
|
||||||
with open(os.open(path, READ_FLAGS), "rb") as handle:
|
fd = os.open(path, READ_FLAGS)
|
||||||
if not stat.S_ISREG(os.fstat(handle.fileno()).st_mode):
|
try:
|
||||||
|
if not stat.S_ISREG(os.fstat(fd).st_mode):
|
||||||
raise OSError(errno.EINVAL, "not a regular file", str(path))
|
raise OSError(errno.EINVAL, "not a regular file", str(path))
|
||||||
data = handle.read() if limit is None else handle.read(limit + 1)
|
with open(fd, "rb", closefd=False) as handle:
|
||||||
|
data = handle.read() if limit is None else handle.read(limit + 1)
|
||||||
|
finally:
|
||||||
|
os.close(fd)
|
||||||
if limit is not None and len(data) > limit:
|
if limit is not None and len(data) > limit:
|
||||||
raise OSError(errno.EFBIG, f"larger than {limit} bytes", str(path))
|
raise OSError(errno.EFBIG, f"larger than {limit} bytes", str(path))
|
||||||
return data.decode("utf-8")
|
return data.decode("utf-8")
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ from .errors import ToolError
|
|||||||
from .fields import single
|
from .fields import single
|
||||||
from .lifetimes import edge_times, lifetime, session_gone, subagent_rows
|
from .lifetimes import edge_times, lifetime, session_gone, subagent_rows
|
||||||
from .names import PSEUDO, SESSION_ID, Directory, title_sightings
|
from .names import PSEUDO, SESSION_ID, Directory, title_sightings
|
||||||
from .records import epoch, iso_epoch, read_jsonl
|
from .records import epoch, iso_epoch
|
||||||
from .register import REGISTER, read_register
|
from .register import REGISTER, read_register
|
||||||
from .tree import assign
|
from .tree import assign
|
||||||
|
|
||||||
@@ -48,10 +48,10 @@ class Sources:
|
|||||||
tasks = read_register(self.council / REGISTER)
|
tasks = read_register(self.council / REGISTER)
|
||||||
reviews = self.reviews.listed()
|
reviews = self.reviews.listed()
|
||||||
scan = assign(self.tree.get(), tasks, read_claims(self.council), reviews)
|
scan = assign(self.tree.get(), tasks, read_claims(self.council), reviews)
|
||||||
history = read_jsonl(self.council / "queue-history.jsonl")
|
history = self.history.rows()
|
||||||
return {**enrich(scan, tasks, reviews, self.touched, history, self.jobs.listed()), "cleanups": self.cleanup_results(), "gates": self.gate_view(scan)}
|
return {**enrich(scan, tasks, reviews, self.touched, history, self.jobs.listed()), "cleanups": self.cleanup_results(), "gates": self.gate_view(scan)}
|
||||||
|
|
||||||
def schedules(self, claims: list[dict], now: float) -> list[dict]:
|
def schedules(self, claims: list[dict], now: float) -> list[dict]:
|
||||||
beats = {claim["session"]: claim["beat"] for claim in claims if claim["beat"] is not None}
|
beats = {claim["session"]: claim["beat"] for claim in claims if claim["beat"] is not None}
|
||||||
rows = [cron_state(cron, beats, now) for cron in self.transcripts.schedules()]
|
rows = [cron_state(cron, beats, now) for cron in self.transcripts.schedules()]
|
||||||
return rows + timer_rows(self.timers) + queue_rows(self.council)
|
return rows + timer_rows(self.timers) + queue_rows(self.council, self.history.rows)
|
||||||
@@ -4,6 +4,7 @@ import itertools
|
|||||||
import statistics
|
import statistics
|
||||||
|
|
||||||
from .council import HUMAN, agent
|
from .council import HUMAN, agent
|
||||||
|
from .crons import outcome_of
|
||||||
from .records import epoch, iso_epoch, shown
|
from .records import epoch, iso_epoch, shown
|
||||||
from .register import ENDED, OPERATOR, parties, register_row, status_of
|
from .register import ENDED, OPERATOR, parties, register_row, status_of
|
||||||
from .signed import signing
|
from .signed import signing
|
||||||
@@ -152,12 +153,21 @@ def task_row(task: str, row: dict, trees: list[dict]) -> dict:
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def queue_status(entry: dict, found: dict | None) -> str:
|
||||||
|
if found is not None:
|
||||||
|
return "done" if found.get("state") == "ended" and found.get("exit") == 0 else "stopped"
|
||||||
|
return "running" if entry.get("status") == "active" else "awaiting"
|
||||||
|
|
||||||
|
|
||||||
def queue_row(entry: dict, created: float, trees: list[dict]) -> dict:
|
def queue_row(entry: dict, created: float, trees: list[dict]) -> dict:
|
||||||
text = {name: shown(entry.get(name))[:TEXT_SHOWN] for name in ("id", "cmd", "kind", "task", "workdir")}
|
text = {name: shown(entry.get(name))[:TEXT_SHOWN] for name in ("id", "cmd", "kind", "task", "workdir")}
|
||||||
|
found = outcome_of(entry)
|
||||||
|
ended = found is not None
|
||||||
|
granted = entry.get("status") == "active" or ended
|
||||||
return {
|
return {
|
||||||
"task": f"{QUEUE}{text['id']}",
|
"task": f"{QUEUE}{text['id']}",
|
||||||
"title": text["cmd"],
|
"title": text["cmd"],
|
||||||
"status": "running" if entry.get("status") == "active" else "awaiting",
|
"status": queue_status(entry, found),
|
||||||
"worker": agent(entry.get("name"))[:TEXT_SHOWN] or "-",
|
"worker": agent(entry.get("name"))[:TEXT_SHOWN] or "-",
|
||||||
"responsible": "-",
|
"responsible": "-",
|
||||||
"accountable": "-",
|
"accountable": "-",
|
||||||
@@ -167,7 +177,7 @@ def queue_row(entry: dict, created: float, trees: list[dict]) -> dict:
|
|||||||
"met": 0,
|
"met": 0,
|
||||||
"total": 0,
|
"total": 0,
|
||||||
"kind": "queue",
|
"kind": "queue",
|
||||||
"ended": False,
|
"ended": ended,
|
||||||
"editable": False,
|
"editable": False,
|
||||||
"movable": False,
|
"movable": False,
|
||||||
"prio": None,
|
"prio": None,
|
||||||
@@ -175,13 +185,13 @@ def queue_row(entry: dict, created: float, trees: list[dict]) -> dict:
|
|||||||
"wsjf": None,
|
"wsjf": None,
|
||||||
"value": None,
|
"value": None,
|
||||||
"points": None,
|
"points": None,
|
||||||
"start": (epoch(entry.get("granted")) or created) if entry.get("status") == "active" else None,
|
"start": (epoch(entry.get("granted")) or created) if granted else None,
|
||||||
"end": None,
|
"end": epoch(found.get("end")) if found is not None else None,
|
||||||
"updated": None,
|
"updated": None,
|
||||||
"after": [],
|
"after": [],
|
||||||
"base": "-",
|
"base": "-",
|
||||||
"roles": {},
|
"roles": {},
|
||||||
"queue": {"kind": text["kind"], "parent": text["task"], "enqueued": created},
|
"queue": {"kind": text["kind"], "parent": text["task"], "enqueued": created, "exit": found.get("exit") if found is not None else None},
|
||||||
**located(entry, trees, False),
|
**located(entry, trees, False),
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -253,7 +263,7 @@ def queue_chain(queue: list[tuple[dict, float]], tasks: dict[str, dict], agents:
|
|||||||
holder = order[name] if isinstance(name, str) and name in order else len(order)
|
holder = order[name] if isinstance(name, str) and name in order else len(order)
|
||||||
return (not entry.get("hotfix"), place - hours, holder, seq if isinstance(seq, int) and not isinstance(seq, bool) else 0)
|
return (not entry.get("hotfix"), place - hours, holder, seq if isinstance(seq, int) and not isinstance(seq, bool) else 0)
|
||||||
|
|
||||||
active = sorted((pair for pair in queue if pair[0].get("status") == "active"), key=lambda pair: epoch(pair[0].get("granted")) or pair[1])
|
active = sorted((pair for pair in queue if pair[0].get("status") == "active" and outcome_of(pair[0]) is None), key=lambda pair: epoch(pair[0].get("granted")) or pair[1])
|
||||||
waiting = sorted((pair for pair in queue if pair[0].get("status") == "waiting"), key=waiting_key)
|
waiting = sorted((pair for pair in queue if pair[0].get("status") == "waiting"), key=waiting_key)
|
||||||
return [(shown(entry.get("id"))[:TEXT_SHOWN], entry.get("status") == "waiting") for entry, _ in [*active, *waiting]]
|
return [(shown(entry.get("id"))[:TEXT_SHOWN], entry.get("status") == "waiting") for entry, _ in [*active, *waiting]]
|
||||||
|
|
||||||
|
|||||||
@@ -85,7 +85,7 @@ class View:
|
|||||||
def state(self, begin: float, end: float, now: float, steps: int) -> dict:
|
def state(self, begin: float, end: float, now: float, steps: int) -> dict:
|
||||||
tasks = read_register(self.council / REGISTER)
|
tasks = read_register(self.council / REGISTER)
|
||||||
news = read_jsonl(self.council / NEWS)
|
news = read_jsonl(self.council / NEWS)
|
||||||
history = read_jsonl(self.council / "queue-history.jsonl")
|
history = self.history.rows()
|
||||||
claims = read_claims(self.council)
|
claims = read_claims(self.council)
|
||||||
names = sessions(claims)
|
names = sessions(claims)
|
||||||
directory = self.directory(claims, news, now)
|
directory = self.directory(claims, news, now)
|
||||||
@@ -105,7 +105,7 @@ class View:
|
|||||||
people = {node["id"]: {"name": node["label"], "now": directory.now(node["id"]) or node["label"], "names": node["names"]} for node in drawn["nodes"] if node["kind"] != "subagent"}
|
people = {node["id"]: {"name": node["label"], "now": directory.now(node["id"]) or node["label"], "names": node["names"]} for node in drawn["nodes"] if node["kind"] != "subagent"}
|
||||||
settings = self.settings.get()
|
settings = self.settings.get()
|
||||||
inbox = self.inbox(now, tasks, directory)
|
inbox = self.inbox(now, tasks, directory)
|
||||||
queue = queue_entries(self.council)
|
queue = queue_entries(self.council, self.history.rows)
|
||||||
order = stored_order(self.council / AGENT_ORDER)
|
order = stored_order(self.council / AGENT_ORDER)
|
||||||
rows = task_rows(tasks, queue, self.tree.get())
|
rows = task_rows(tasks, queue, self.tree.get())
|
||||||
chain = queue_chain(queue, tasks, order, now)
|
chain = queue_chain(queue, tasks, order, now)
|
||||||
|
|||||||
@@ -47,7 +47,8 @@
|
|||||||
|
|
||||||
function taskPlace(row) {
|
function taskPlace(row) {
|
||||||
if (row.kind === "queue") {
|
if (row.kind === "queue") {
|
||||||
return h("span", {}, `${row.queue.kind} for ${row.queue.parent} · queued `, when(row.queue.enqueued));
|
const outcome = !row.ended ? null : row.queue.exit !== null ? ` · exit ${row.queue.exit}` : " · ended without an exit code";
|
||||||
|
return h("span", {}, `${row.queue.kind} for ${row.queue.parent} · queued `, when(row.queue.enqueued), outcome);
|
||||||
}
|
}
|
||||||
const place = row.worktree || (row.workdir !== "-" ? row.workdir : "");
|
const place = row.worktree || (row.workdir !== "-" ? row.workdir : "");
|
||||||
return place ? h("code", {title: row.repo || place}, place) : null;
|
return place ? h("code", {title: row.repo || place}, place) : null;
|
||||||
|
|||||||
@@ -23,12 +23,12 @@ const Plan = (() => {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @param rows - task rows with status, start, end (epoch seconds or null) and points (median complexity or null).
|
* @param rows - task rows with kind, status, start, end (epoch seconds or null) and points (median complexity or null).
|
||||||
* @returns {duration, perPoint}: the median seconds a done task took from start to end, and the median seconds
|
* @returns {duration, perPoint}: the median seconds a done task, never a queue job, took from start to end, and the
|
||||||
* per point of the done tasks with points; null where no done task gives one.
|
* median seconds per point of the done tasks with points; null where no done task gives one.
|
||||||
*/
|
*/
|
||||||
function medians(rows) {
|
function medians(rows) {
|
||||||
const done = rows.filter((row) => row.status === "done" && row.start !== null && row.end !== null && row.end >= row.start);
|
const done = rows.filter((row) => row.kind !== "queue" && row.status === "done" && row.start !== null && row.end !== null && row.end >= row.start);
|
||||||
return {
|
return {
|
||||||
duration: median(done.map((row) => row.end - row.start)),
|
duration: median(done.map((row) => row.end - row.start)),
|
||||||
perPoint: median(done.filter((row) => row.points).map((row) => (row.end - row.start) / row.points)),
|
perPoint: median(done.filter((row) => row.points).map((row) => (row.end - row.start) / row.points)),
|
||||||
|
|||||||
@@ -241,6 +241,10 @@ def rules_dir(root: Path) -> Path:
|
|||||||
return root / "rules"
|
return root / "rules"
|
||||||
|
|
||||||
|
|
||||||
|
def client_calls(tree: ast.AST) -> list[ast.Call]:
|
||||||
|
return [node for node in ast.walk(tree) if isinstance(node, ast.Call) and "client_module" in (getattr(node.func, "id", ""), getattr(node.func, "attr", ""))]
|
||||||
|
|
||||||
|
|
||||||
class TestDiffsModule(unittest.TestCase):
|
class TestDiffsModule(unittest.TestCase):
|
||||||
@classmethod
|
@classmethod
|
||||||
def setUpClass(cls):
|
def setUpClass(cls):
|
||||||
@@ -825,12 +829,18 @@ class TestSharedLimits(unittest.TestCase):
|
|||||||
self.assertLessEqual(len(json.dumps(patch, ensure_ascii=False).encode("utf-8")), hub_limits.DIFF_LIMIT)
|
self.assertLessEqual(len(json.dumps(patch, ensure_ascii=False).encode("utf-8")), hub_limits.DIFF_LIMIT)
|
||||||
self.assertGreater(len(permission_hook.encoded({"diff": [{"patch": patch}]})), hub_limits.DIFF_BODY_LIMIT)
|
self.assertGreater(len(permission_hook.encoded({"diff": [{"patch": patch}]})), hub_limits.DIFF_BODY_LIMIT)
|
||||||
|
|
||||||
def test_the_hub_image_copies_every_client_module_signed_loads(self):
|
def test_client_module_calls_are_found_bare_and_as_attributes(self):
|
||||||
tree = ast.parse((HUB / "kvb_orchestrator" / "signed.py").read_text(encoding="utf-8"))
|
found = client_calls(ast.parse('signed.client_module("a", {})\nclient_module("b", {})\nother("c")\nx.other("d")'))
|
||||||
loaded = {f"{node.args[0].value}.py" for node in ast.walk(tree) if isinstance(node, ast.Call) and getattr(node.func, "id", "") == "client_module"}
|
self.assertEqual([node.args[0].value for node in found], ["a", "b"])
|
||||||
|
|
||||||
|
def test_the_hub_image_copies_every_client_module_the_hub_loads(self):
|
||||||
|
trees = [ast.parse(path.read_text(encoding="utf-8")) for path in sorted((HUB / "kvb_orchestrator").glob("*.py"))]
|
||||||
|
calls = [node for tree in trees for node in client_calls(tree)]
|
||||||
|
self.assertTrue(all(node.args and isinstance(node.args[0], ast.Constant) for node in calls))
|
||||||
|
loaded = {f"{node.args[0].value}.py" for node in calls}
|
||||||
copies = [line.split()[1:] for line in (HUB / "Dockerfile").read_text(encoding="utf-8").splitlines() if line.startswith("COPY ")]
|
copies = [line.split()[1:] for line in (HUB / "Dockerfile").read_text(encoding="utf-8").splitlines() if line.startswith("COPY ")]
|
||||||
copied = {Path(source).name for words in copies if words[-1] == "/app/client/" for source in words[:-1]}
|
copied = {Path(source).name for words in copies if words[-1] == "/app/client/" for source in words[:-1]}
|
||||||
self.assertIn("hub_limits.py", loaded)
|
self.assertLessEqual({"hub_limits.py", "queue_live.py"}, loaded)
|
||||||
self.assertLessEqual(loaded, copied)
|
self.assertLessEqual(loaded, copied)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -74,6 +74,7 @@ console.log(JSON.stringify({
|
|||||||
steps: {whole: stepsOf(70 * H, now), window: stepsOf(90 * H, 99 * H), burnt: P.steps(burnt, burnPoints, 0)},
|
steps: {whole: stepsOf(70 * H, now), window: stepsOf(90 * H, 99 * H), burnt: P.steps(burnt, burnPoints, 0)},
|
||||||
etas: {none: eta("none"), points: eta("points"), loop: [...P.etas(loop, looped, {now, estimation: "none"}).keys()].sort()},
|
etas: {none: eta("none"), points: eta("points"), loop: [...P.etas(loop, looped, {now, estimation: "none"}).keys()].sort()},
|
||||||
medians: P.medians(rows),
|
medians: P.medians(rows),
|
||||||
|
queueMedians: P.medians([...rows, row("queue/done", {kind: "queue", status: "done", ended: true, start: 0, end: 50 * H, points: 1})]),
|
||||||
median: [P.median([]), P.median([3, 1, 2]), P.median([4, 1, 3, 2])],
|
median: [P.median([]), P.median([3, 1, 2]), P.median([4, 1, 3, 2])],
|
||||||
bars: {noneAlways: lay("none", "always"), pointsAlways: lay("points", "always"), pointsOnly: lay("points", "points"), noneOnly: lay("none", "points"), real: lay("points", "real")},
|
bars: {noneAlways: lay("none", "always"), pointsAlways: lay("points", "always"), pointsOnly: lay("points", "points"), noneOnly: lay("none", "points"), real: lay("points", "real")},
|
||||||
loop: P.bars(loop, looped, {now, estimation: "none", forecast: "always"}).map((bar) => bar.task),
|
loop: P.bars(loop, looped, {now, estimation: "none", forecast: "always"}).map((bar) => bar.task),
|
||||||
@@ -157,6 +158,9 @@ class TestPlanModule(unittest.TestCase):
|
|||||||
self.assertEqual(self.data["medians"], {"duration": 2 * HOUR, "perPoint": 1.5 * HOUR})
|
self.assertEqual(self.data["medians"], {"duration": 2 * HOUR, "perPoint": 1.5 * HOUR})
|
||||||
self.assertEqual(self.data["median"], [None, 2, 2.5])
|
self.assertEqual(self.data["median"], [None, 2, 2.5])
|
||||||
|
|
||||||
|
def test_a_done_queue_job_never_moves_the_medians(self):
|
||||||
|
self.assertEqual(self.data["queueMedians"], self.data["medians"])
|
||||||
|
|
||||||
def test_without_estimate_a_forecast_is_as_long_as_the_median_duration_behind_its_predecessors(self):
|
def test_without_estimate_a_forecast_is_as_long_as_the_median_duration_behind_its_predecessors(self):
|
||||||
self.assertEqual(self.data["bars"]["noneAlways"], [
|
self.assertEqual(self.data["bars"]["noneAlways"], [
|
||||||
["work/a", 80, 82, False, False],
|
["work/a", 80, 82, False, False],
|
||||||
|
|||||||
+192
-2
@@ -1,5 +1,7 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import fcntl
|
||||||
|
import functools
|
||||||
import importlib.util
|
import importlib.util
|
||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
@@ -12,13 +14,14 @@ from unittest import mock
|
|||||||
|
|
||||||
from hub_case import HubCase
|
from hub_case import HubCase
|
||||||
from hub_support import REPO_ROOT, iso, key, runner, signing, verifier
|
from hub_support import REPO_ROOT, iso, key, runner, signing, verifier
|
||||||
from kvb_orchestrator import settings, skill_scripts, tasks, tree
|
from kvb_orchestrator import crons, records, settings, skill_scripts, tasks, tree
|
||||||
|
|
||||||
BROKEN = {"pull": [], "push": [], "source": "invalid"}
|
BROKEN = {"pull": [], "push": [], "source": "invalid"}
|
||||||
FLOW = {"pull": ["origin/main", "store-i18n"], "push": ["main", "production"], "source": "file"}
|
FLOW = {"pull": ["origin/main", "store-i18n"], "push": ["main", "production"], "source": "file"}
|
||||||
PRIO = ["set", "work/feature", "--prio", "3"]
|
PRIO = ["set", "work/feature", "--prio", "3"]
|
||||||
ESTIMATE = ["estimate", "work/feature", "Operator", "--value", "8", "--complexity", "3"]
|
ESTIMATE = ["estimate", "work/feature", "Operator", "--value", "8", "--complexity", "3"]
|
||||||
QUEUE_SCRIPT = REPO_ROOT / "skills" / "kvb-org-queue" / "scripts" / "queue.py"
|
QUEUE_SCRIPT = REPO_ROOT / "skills" / "kvb-org-queue" / "scripts" / "queue.py"
|
||||||
|
PROOF = "/home/me/worktrees/github.com/infinito-nexus/core/proof/karakalpak-archive-role"
|
||||||
|
|
||||||
|
|
||||||
def change(argv: list, task: str = "work/feature") -> dict:
|
def change(argv: list, task: str = "work/feature") -> dict:
|
||||||
@@ -136,11 +139,198 @@ class TestHubTasks(HubCase):
|
|||||||
self.assertEqual((command["kind"], command["title"], command["editable"], command["open_work"]), ("command", "git push", False, ["task"]))
|
self.assertEqual((command["kind"], command["title"], command["editable"], command["open_work"]), ("command", "git push", False, ["task"]))
|
||||||
gate, deploy = rows["queue/gate-7"], rows["queue/deploy-8"]
|
gate, deploy = rows["queue/gate-7"], rows["queue/deploy-8"]
|
||||||
self.assertEqual((gate["kind"], gate["status"], gate["title"], gate["worker"], gate["editable"], gate["flow"]), ("queue", "awaiting", "make test", "Stage", False, FLOW))
|
self.assertEqual((gate["kind"], gate["status"], gate["title"], gate["worker"], gate["editable"], gate["flow"]), ("queue", "awaiting", "make test", "Stage", False, FLOW))
|
||||||
self.assertEqual(gate["queue"], {"kind": "gate", "parent": "work/feature", "enqueued": now - 60})
|
self.assertEqual(gate["queue"], {"kind": "gate", "parent": "work/feature", "enqueued": now - 60, "exit": None})
|
||||||
self.assertEqual((deploy["status"], deploy["flow"], deploy["queue"]["parent"]), ("running", None, "work/pinned"))
|
self.assertEqual((deploy["status"], deploy["flow"], deploy["queue"]["parent"]), ("running", None, "work/pinned"))
|
||||||
done = rows["work/done"]
|
done = rows["work/done"]
|
||||||
self.assertEqual((done["ended"], done["editable"], done["open_work"], done["worktree"]), (True, False, [], ""))
|
self.assertEqual((done["ended"], done["editable"], done["open_work"], done["worktree"]), (True, False, [], ""))
|
||||||
|
|
||||||
|
def held_lock(self, tag: str, mode: int = fcntl.LOCK_EX) -> None:
|
||||||
|
holds = self.council / "holds"
|
||||||
|
holds.mkdir(parents=True, exist_ok=True)
|
||||||
|
handle = (holds / f"{tag}.lock").open("w", encoding="utf-8")
|
||||||
|
self.addCleanup(handle.close)
|
||||||
|
fcntl.flock(handle, mode)
|
||||||
|
|
||||||
|
def queue_job(self, now: float, ident: str, status: str, task: str, lock: str, age: float) -> dict:
|
||||||
|
granted = now - age + 10 if status == "active" else None
|
||||||
|
return {"id": ident, "kind": ident.rpartition("-")[0], "task": task, "workdir": PROOF, "session": "s1", "name": "Production", "lock": lock, "pid": 7,
|
||||||
|
"status": status, "enqueued": now - age, "granted": granted, "cmd": "make quality-high"}
|
||||||
|
|
||||||
|
def test_a_queue_job_with_a_history_row_or_a_gone_holder_is_no_longer_running(self):
|
||||||
|
now = time.time()
|
||||||
|
for tag in ("7-1410", "7-1451", "7-5", "7-1387", "7-1500", "7-1501", "7-1502"):
|
||||||
|
self.held_lock(tag)
|
||||||
|
self.held_lock("7-1481", fcntl.LOCK_SH)
|
||||||
|
(self.council / "holds" / "7-1465.lock").write_text("", encoding="utf-8")
|
||||||
|
(self.council / "holds" / "7-1480.lock").symlink_to(self.council / "holds" / "7-1451.lock")
|
||||||
|
job = functools.partial(self.queue_job, now)
|
||||||
|
queue = {"seq": 1502, "entries": [
|
||||||
|
job("quality-high-1410", "active", "karakalpak-archive", "7-1410", 1200),
|
||||||
|
job("test-nostack-1464", "active", "karakalpak-archive", "7-1464", 300),
|
||||||
|
job("test-nostack-1465", "active", "iam/final-066-backchannel", "7-1465", 200),
|
||||||
|
job("test-nostack-1466", "active", "iam/final-066-backchannel", "7-1466", 100),
|
||||||
|
job("quality-high-1451", "active", "work/gitea-oidc-auto-link", "7-1451", 900),
|
||||||
|
job("test-nostack-1480", "active", "work/symlinked", "7-1480", 80),
|
||||||
|
job("test-nostack-1481", "active", "iam/final-066-backchannel", "7-1481", 70),
|
||||||
|
job("gate-5", "active", "work/reused", "7-5", 60),
|
||||||
|
job("test-1387", "waiting", "work/security-page-optimize", "7-1387", 50),
|
||||||
|
job("test-1388", "waiting", "work/security-page-optimize", "7-1388", 40),
|
||||||
|
job("test-nostack-1471", "waiting", "work/orchestrator-firewall", "../holds/7-1471", 30),
|
||||||
|
job("test-1500", "ended", "work/foreign", "7-1500", 25),
|
||||||
|
job("test-1501", "gone", "work/foreign", "7-1501", 24),
|
||||||
|
{**job("test-1502", "active", "work/foreign", "7-1502", 20), "hub_outcome": {"state": "ended", "exit": 9}, "exit": 9, "end": now},
|
||||||
|
]}
|
||||||
|
history = [
|
||||||
|
{"end": now - 600, "exit": 1, "id": "quality-high-1410-1794513000000000000", "kind": "quality-high", "start": now - 1180, "workdir": PROOF},
|
||||||
|
{"concurrent": 2, "cpu_before": 53.9, "cpu_max": 100.0, "disk_gb": 1.91, "end": now - 5, "exit": 2, "free_before_gb": 113.12,
|
||||||
|
"id": "quality-high-1410-1794514233361361126", "kind": "quality-high", "mem_before": 43.2, "mem_max": 84.3, "min_free_gb": 111.21,
|
||||||
|
"minutes": 18.13, "signature": "infinito-nexus-core: make quality-high", "start": now - 1185, "workdir": PROOF},
|
||||||
|
{"end": now - 20, "exit": 0, "id": "test-nostack-1464-1794514708907109250", "kind": "test-nostack", "start": now - 290, "workdir": PROOF},
|
||||||
|
{"end": now - 3, "exit": 0, "id": "test-nostack-1466-retry", "kind": "test-nostack", "start": now - 90, "workdir": PROOF},
|
||||||
|
{"end": now - 7200, "exit": 1, "id": "gate-5-1794500000000000000", "kind": "gate", "start": now - 7300, "workdir": PROOF},
|
||||||
|
{"end": now - 1, "exit": 0, "id": "gate-5x-1794500000000000000", "kind": "gate", "start": now - 30, "workdir": PROOF},
|
||||||
|
]
|
||||||
|
self.write_council({"queue.json": queue, "queue-history.jsonl": self.jsonl(history)})
|
||||||
|
state = self.snapshot()
|
||||||
|
rows = {row["task"]: row for row in state["tasks"] if row["kind"] == "queue"}
|
||||||
|
|
||||||
|
def seen(ident: str) -> tuple:
|
||||||
|
row = rows[f"queue/{ident}"]
|
||||||
|
return row["status"], row["ended"], row["end"], row["queue"].get("exit")
|
||||||
|
|
||||||
|
self.assertEqual(seen("quality-high-1410"), ("stopped", True, now - 5, 2))
|
||||||
|
self.assertEqual(rows["queue/quality-high-1410"]["start"], now - 1190)
|
||||||
|
self.assertEqual(seen("test-nostack-1464"), ("done", True, now - 20, 0))
|
||||||
|
self.assertEqual(seen("test-nostack-1465"), ("stopped", True, None, None))
|
||||||
|
self.assertEqual(seen("test-nostack-1466"), ("stopped", True, None, None))
|
||||||
|
self.assertEqual(seen("test-nostack-1481"), ("stopped", True, None, None))
|
||||||
|
self.assertEqual(seen("test-nostack-1480"), ("stopped", True, None, None))
|
||||||
|
live = ("quality-high-1451", "gate-5", "test-1502")
|
||||||
|
shown = ("quality-high-1410", "test-nostack-1464", "test-nostack-1465", "test-nostack-1466", "test-nostack-1480", "test-nostack-1481", "test-1387", "test-1500", "test-1501", *live)
|
||||||
|
self.assertEqual(sorted(rows), sorted(f"queue/{ident}" for ident in shown))
|
||||||
|
self.assertEqual([seen(ident) for ident in live], [("running", False, None, None)] * 3)
|
||||||
|
self.assertEqual([seen(ident) for ident in ("test-1387", "test-1500", "test-1501")], [("awaiting", False, None, None)] * 3)
|
||||||
|
self.assertEqual(state["queue_line"], ["work/gitea-oidc-auto-link", "work/reused", "work/foreign", "work/security-page-optimize"])
|
||||||
|
schedule = {row["job"]: (row["expression"], row["active"]) for row in crons.queue_rows(self.council, self.httpd.hub.history.rows)}
|
||||||
|
self.assertEqual(schedule["quality-high-1410"], ("quality-high · exit 2", False))
|
||||||
|
self.assertEqual(schedule["test-nostack-1465"], ("test-nostack · holder gone without an exit code", False))
|
||||||
|
self.assertEqual(schedule["quality-high-1451"], ("quality-high · running", True))
|
||||||
|
self.assertEqual(schedule["test-1500"], ("test · waiting", True))
|
||||||
|
|
||||||
|
def test_the_history_is_read_after_the_hold_locks_and_parsed_once_per_change(self):
|
||||||
|
now = time.time()
|
||||||
|
self.held_lock("7-1")
|
||||||
|
queue = {"seq": 2, "entries": [self.queue_job(now, "test-1", "active", "work/a", "7-1", 100), self.queue_job(now, "test-2", "active", "work/b", "7-2", 50)]}
|
||||||
|
self.write_council({"queue.json": queue, "queue-history.jsonl": self.jsonl([{"id": "other-9-1", "start": now - 500, "end": now - 400, "exit": 0}])})
|
||||||
|
probe, read = crons.queue_live.holder_gone, crons.queue_live.read_history
|
||||||
|
reads = []
|
||||||
|
|
||||||
|
def leaving(council: Path, tag: object) -> bool:
|
||||||
|
if tag == "7-2":
|
||||||
|
with (self.council / "queue-history.jsonl").open("a", encoding="utf-8") as handle:
|
||||||
|
handle.write(json.dumps({"id": "test-2-1794514233361361126", "start": now - 40, "end": now - 1, "exit": 3}) + "\n")
|
||||||
|
return probe(council, tag)
|
||||||
|
|
||||||
|
def counted(path: Path) -> list[dict]:
|
||||||
|
reads.append(path.name)
|
||||||
|
return read(path)
|
||||||
|
|
||||||
|
with mock.patch.object(crons.queue_live, "holder_gone", leaving), mock.patch.object(crons.queue_live, "read_history", counted):
|
||||||
|
rows = {row["task"]: row for row in self.snapshot()["tasks"] if row["kind"] == "queue"}
|
||||||
|
self.assertEqual((rows["queue/test-2"]["status"], rows["queue/test-2"]["queue"]["exit"]), ("stopped", 3))
|
||||||
|
self.assertEqual(rows["queue/test-1"]["status"], "running")
|
||||||
|
self.assertEqual(len(reads), 2)
|
||||||
|
self.httpd.hub.history.rows()
|
||||||
|
self.assertEqual(len(reads), 2)
|
||||||
|
|
||||||
|
def test_the_whole_history_is_read_and_a_bad_byte_costs_its_line_only(self):
|
||||||
|
path = self.council / "queue-history.jsonl"
|
||||||
|
self.council.mkdir(parents=True, exist_ok=True)
|
||||||
|
rows = b"".join(json.dumps({"id": f"gate-{n}-1", "start": n, "pad": "x" * 1000}).encode() + b"\n" for n in range(6000))
|
||||||
|
path.write_bytes(b'{"id": "bad-\xff-1"}\n' + rows + b"[[[\n" + b'{"id": "last-1"}')
|
||||||
|
found = crons.queue_live.read_history(path)
|
||||||
|
self.assertGreater(path.stat().st_size, 1 << 22)
|
||||||
|
self.assertEqual((len(found), found[1]["id"], found[-2]["id"], found[-1]["id"]), (6002, "gate-0-1", "gate-5999-1", "last-1"))
|
||||||
|
|
||||||
|
def test_reading_a_directory_as_history_or_json_leaks_no_descriptor(self):
|
||||||
|
folder = self.council / "queue-history.jsonl"
|
||||||
|
folder.mkdir(parents=True)
|
||||||
|
before = len(os.listdir("/proc/self/fd"))
|
||||||
|
for _ in range(50):
|
||||||
|
self.assertEqual(crons.queue_live.read_history(folder), [])
|
||||||
|
self.assertIsNone(records.read_json(folder))
|
||||||
|
self.assertEqual(records.read_jsonl(folder), [])
|
||||||
|
self.assertLessEqual(len(os.listdir("/proc/self/fd")), before + 2)
|
||||||
|
|
||||||
|
def test_the_history_cache_reparses_on_a_new_inode_mtime_or_size_alone(self):
|
||||||
|
path = self.council / "queue-history.jsonl"
|
||||||
|
self.council.mkdir(parents=True, exist_ok=True)
|
||||||
|
history = crons.History(path)
|
||||||
|
|
||||||
|
def write(text: str, ns: int) -> None:
|
||||||
|
path.write_text(text, encoding="utf-8")
|
||||||
|
os.utime(path, ns=(ns, ns))
|
||||||
|
|
||||||
|
write('{"id": "a-1"}\n', 10**18)
|
||||||
|
self.assertEqual([row["id"] for row in history.rows()], ["a-1"])
|
||||||
|
replacement = self.council / "replacement"
|
||||||
|
replacement.write_text('{"id": "b-1"}\n', encoding="utf-8")
|
||||||
|
os.utime(replacement, ns=(10**18, 10**18))
|
||||||
|
os.replace(replacement, path)
|
||||||
|
self.assertEqual([row["id"] for row in history.rows()], ["b-1"])
|
||||||
|
write('{"id": "c-1"}\n', 10**18 + 1)
|
||||||
|
self.assertEqual([row["id"] for row in history.rows()], ["c-1"])
|
||||||
|
write('{"id": "c-1"}\n{"id": "d-1"}\n', 10**18 + 1)
|
||||||
|
self.assertEqual([row["id"] for row in history.rows()], ["c-1", "d-1"])
|
||||||
|
|
||||||
|
def test_a_hard_link_to_a_live_hold_lock_keeps_the_hold_alive(self):
|
||||||
|
self.held_lock("7-20")
|
||||||
|
holds = self.council / "holds"
|
||||||
|
os.link(holds / "7-20.lock", holds / "7-21.lock")
|
||||||
|
self.assertEqual((holds / "7-20.lock").stat().st_nlink, 2)
|
||||||
|
self.assertEqual([crons.queue_live.holder_gone(self.council, tag) for tag in ("7-20", "7-21")], [False, False])
|
||||||
|
|
||||||
|
@unittest.skipIf(os.geteuid() == 0, "root reads a file of mode 0")
|
||||||
|
def test_a_failed_history_read_keeps_the_rows_before_and_retries_next_time(self):
|
||||||
|
path = self.council / "queue-history.jsonl"
|
||||||
|
self.write_council({"queue-history.jsonl": '{"id": "a-1"}\n'})
|
||||||
|
history = crons.History(path)
|
||||||
|
self.assertEqual([row["id"] for row in history.rows()], ["a-1"])
|
||||||
|
with path.open("a", encoding="utf-8") as handle:
|
||||||
|
handle.write('{"id": "b-1"}\n')
|
||||||
|
path.chmod(0)
|
||||||
|
self.addCleanup(path.chmod, 0o600)
|
||||||
|
self.assertEqual([row["id"] for row in history.rows()], ["a-1"])
|
||||||
|
self.assertIsNone(crons.queue_live.read_history(path))
|
||||||
|
path.chmod(0o600)
|
||||||
|
self.assertEqual([row["id"] for row in history.rows()], ["a-1", "b-1"])
|
||||||
|
|
||||||
|
def test_the_hub_judges_every_hold_lock_as_queue_py_held_does(self):
|
||||||
|
queue_py = load_queue()
|
||||||
|
holds = self.council / "holds"
|
||||||
|
self.held_lock("7-1")
|
||||||
|
(holds / "7-2.lock").write_text("", encoding="utf-8")
|
||||||
|
(holds / "7-3.lock").symlink_to(holds / "7-1.lock")
|
||||||
|
(holds / "7-7.lock").write_text("", encoding="utf-8")
|
||||||
|
os.link(holds / "7-7.lock", holds / "7-6.lock")
|
||||||
|
decided = {"held": "7-1", "free": "7-2", "missing": "7-404", "malformed": "x-1", "traversal": "../holds/7-1", "none": None}
|
||||||
|
refused = {"symlink": "7-3", "hard link": "7-6"}
|
||||||
|
hub = {name: crons.queue_live.holder_gone(self.council, tag) for name, tag in {**decided, **refused}.items()}
|
||||||
|
with mock.patch.object(queue_py, "HOLDS", holds), mock.patch.object(queue_py.watch, "ROOT", self.council):
|
||||||
|
for name, tag in refused.items():
|
||||||
|
with self.subTest(refused=name), self.assertRaises(OSError):
|
||||||
|
queue_py.held(tag)
|
||||||
|
with self.subTest(refused=name), self.assertRaises(OSError), queue_py.nofollow.locked(self.council, holds / f"{tag}.lock", 0o600, None):
|
||||||
|
pass
|
||||||
|
verdicts = {name: queue_py.held(tag) for name, tag in decided.items()}
|
||||||
|
self.assertEqual({name: not hub[name] for name in decided}, verdicts)
|
||||||
|
self.assertEqual(verdicts, {"held": True, "free": False, "missing": False, "malformed": False, "traversal": False, "none": False})
|
||||||
|
self.assertEqual({name: hub[name] for name in refused}, {"symlink": True, "hard link": True})
|
||||||
|
if os.geteuid() != 0:
|
||||||
|
(holds / "7-9.lock").write_text("", encoding="utf-8")
|
||||||
|
(holds / "7-9.lock").chmod(0)
|
||||||
|
self.assertFalse(crons.queue_live.holder_gone(self.council, "7-9"))
|
||||||
|
|
||||||
def test_only_the_newest_ended_tasks_are_listed(self):
|
def test_only_the_newest_ended_tasks_are_listed(self):
|
||||||
register = {f"work/old-{n:03d}": {"status": "done", "end": f"2026-11-{1 + n // 24:02d}T{n % 24:02d}:00:00+00:00"} for n in range(30)}
|
register = {f"work/old-{n:03d}": {"status": "done", "end": f"2026-11-{1 + n // 24:02d}T{n % 24:02d}:00:00+00:00"} for n in range(30)}
|
||||||
self.write_council({"rasci.json": {**register, "work/open": {"status": "queued"}}})
|
self.write_council({"rasci.json": {**register, "work/open": {"status": "queued"}}})
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import fcntl
|
||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
import subprocess
|
import subprocess
|
||||||
@@ -108,10 +109,29 @@ class TestParsing(unittest.TestCase):
|
|||||||
with tempfile.TemporaryDirectory() as tmp:
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
council = Path(tmp)
|
council = Path(tmp)
|
||||||
self.assertEqual(sampler.queue(council), {"waiting": 0, "active": 0})
|
self.assertEqual(sampler.queue(council), {"waiting": 0, "active": 0})
|
||||||
entries = [{"status": "waiting"}, {"status": "waiting"}, {"status": "active"}, {"status": "done"}, "junk"]
|
entries = [{"status": status, "enqueued": NOW} for status in ("waiting", "waiting", "active", "done")] + [{"status": "active"}, "junk"]
|
||||||
(council / "queue.json").write_text(json.dumps({"entries": entries}), encoding="utf-8")
|
(council / "queue.json").write_text(json.dumps({"entries": entries}), encoding="utf-8")
|
||||||
self.assertEqual(sampler.queue(council), {"waiting": 2, "active": 1})
|
self.assertEqual(sampler.queue(council), {"waiting": 2, "active": 1})
|
||||||
|
|
||||||
|
def test_the_queue_counts_only_live_entries_as_the_hub_settles_them(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
council = Path(tmp)
|
||||||
|
(council / "holds").mkdir()
|
||||||
|
handles = [(council / "holds" / f"7-{n}.lock").open("w", encoding="utf-8") for n in (1, 2, 3)]
|
||||||
|
for handle in handles:
|
||||||
|
self.addCleanup(handle.close)
|
||||||
|
fcntl.flock(handle, fcntl.LOCK_EX)
|
||||||
|
entries = [
|
||||||
|
{"id": "test-1", "status": "waiting", "lock": "7-1", "enqueued": NOW},
|
||||||
|
{"id": "test-2", "status": "active", "lock": "7-2", "enqueued": NOW},
|
||||||
|
{"id": "quality-high-3", "status": "active", "lock": "7-3", "enqueued": NOW},
|
||||||
|
{"id": "test-4", "status": "waiting", "lock": "7-4", "enqueued": NOW},
|
||||||
|
{"id": "test-5", "status": "active", "lock": "7-5", "enqueued": NOW},
|
||||||
|
]
|
||||||
|
(council / "queue.json").write_text(json.dumps({"entries": entries}), encoding="utf-8")
|
||||||
|
(council / "queue-history.jsonl").write_text(json.dumps({"id": "quality-high-3-1794514233361361126", "start": NOW + 10, "end": NOW + 20, "exit": 2}) + "\n", encoding="utf-8")
|
||||||
|
self.assertEqual(sampler.queue(council), {"waiting": 1, "active": 1})
|
||||||
|
|
||||||
|
|
||||||
class TestCpu(unittest.TestCase):
|
class TestCpu(unittest.TestCase):
|
||||||
def test_cpu_is_the_busy_share_since_the_ticks_the_run_before_kept_and_the_sampler_never_sleeps(self):
|
def test_cpu_is_the_busy_share_since_the_ticks_the_run_before_kept_and_the_sampler_never_sleeps(self):
|
||||||
@@ -233,7 +253,7 @@ class TestSamplerInScratchHome(unittest.TestCase):
|
|||||||
(stubs / tool).chmod(0o755)
|
(stubs / tool).chmod(0o755)
|
||||||
self.sys = host_sys(base / "sys")
|
self.sys = host_sys(base / "sys")
|
||||||
self.log = base / "calls.jsonl"
|
self.log = base / "calls.jsonl"
|
||||||
(self.council / "queue.json").write_text(json.dumps({"entries": [{"status": "waiting"}, {"status": "active"}, {"status": "active"}]}), encoding="utf-8")
|
(self.council / "queue.json").write_text(json.dumps({"entries": [{"status": status, "enqueued": NOW} for status in ("waiting", "active", "active")]}), encoding="utf-8")
|
||||||
self.env = {
|
self.env = {
|
||||||
**os.environ, "HOME": str(self.home), "PATH": f"{stubs}:/usr/bin:/bin",
|
**os.environ, "HOME": str(self.home), "PATH": f"{stubs}:/usr/bin:/bin",
|
||||||
"STUB_LOG": str(self.log), "STUB_DF": DF, "STUB_NVIDIA_SMI": NVIDIA_SMI.splitlines()[0] + "\n",
|
"STUB_LOG": str(self.log), "STUB_DF": DF, "STUB_NVIDIA_SMI": NVIDIA_SMI.splitlines()[0] + "\n",
|
||||||
|
|||||||
Reference in new issue
Block a user