feat(ci-artifacts,merge-fix): cluster failed CI jobs and search refs through scripts

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

skills/ci-artifacts/SKILL.md

@@ -12,23 +12,18 @@ failure without downloading its artifacts is guessing with extra steps.
 
 ## Getting the log
 
-`gh run view --log` refuses to serve anything while a run is unfinished:
+For a GitHub Actions run, one call does it:
 
-    run <id> is still in progress; logs will be available when it is complete
+    ${CLAUDE_SKILL_DIR}/scripts/ci_failures.py <RUN_ID> [--repo <owner>/<repo>] [--artifacts]
 
-That is not a dead end — individual *jobs* that have already finished are
-served by the API:
+It prints how far HEAD moved since the run's commit, saves every finished
+failed job's log to `/tmp/ci-logs-<run>/` (colour stripped, even while the run
+is still in progress), and clusters the jobs by the normalised error line that
+ended them. Jobs still running are listed, not fetched: their API log can stop hours
+early. The clustering is a heuristic; check each boundary against the logs.
 
-    gh api /repos/<owner>/<repo>/actions/jobs/<JOB_ID>/logs
-
-Enumerate the failing jobs and their ids first:
-
-    gh run view <RUN_ID> --repo <owner>/<repo> --json jobs \
-      --jq '.jobs[] | select(.conclusion=="failure") | "\(.databaseId)\t\(.name)"'
-
-Save every log to a file (`/tmp/ci-logs-<run>/<job>.log`) and grep the file
-repeatedly. Re-fetching a log to find the line you just lost costs minutes;
-grepping the saved copy costs milliseconds.
+Grep the saved files repeatedly. Re-fetching a log to find the line you just
+lost costs minutes; grepping the saved copy costs milliseconds.
 
 ## Getting the artifacts
 

skills/ci-artifacts/scripts/ci_failures.py

@@ -0,0 +1,149 @@
+#!/usr/bin/env python3
+"""Fetch the failed jobs of a GitHub Actions run and cluster them by error line.
+
+Usage:
+    ci_failures.py RUN_ID [--repo OWNER/REPO] [--out DIR] [--artifacts]
+
+Prints how far HEAD moved since the run's commit, saves each finished failed
+job's log (colour stripped) under DIR (default /tmp/ci-logs-RUN_ID), and
+groups the jobs by the normalised error line that ended them (the last one). Jobs still running are
+listed but not fetched: their API log can stop hours early. The clustering is
+a heuristic; confirm a cluster's boundary against the logs.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import re
+import subprocess
+import sys
+from pathlib import Path
+
+FAILED = {"failure", "cancelled", "timed_out"}
+ANSI = re.compile(r"\x1b\[[0-9;]*[A-Za-z]")
+STAMP = re.compile(r"^\d{4}-\d\d-\d\dT[\d:.]+Z\s?")
+GENERIC = re.compile(r"Process completed with exit code|failed=0\b|FAILED - RETRYING")
+ANNOTATION = re.compile(r"##\[error\]")
+FATAL = re.compile(r"fatal: \[|FAILED!|Traceback|AssertionError|panic:")
+SIGNAL = re.compile(r"\b(ERROR|Error|FAILED|FAIL|fatal|Exception|failed)\b")
+TASK = re.compile(r"^TASK \[")
+REFUSAL = "terminal escape sequences"
+
+
+def run(*cmd: str) -> subprocess.CompletedProcess:
+    return subprocess.run(list(cmd), capture_output=True, text=True, check=False)
+
+
+def clean(line: str) -> str:
+    return STAMP.sub("", ANSI.sub("", line)).strip()
+
+
+def normalise(line: str) -> str:
+    line = re.sub(r"\b[0-9a-f]{7,}\b", "<hex>", line)
+    line = re.sub(r"\d+", "<n>", line)
+    return line[:200]
+
+
+def output_lines(text: str) -> list[tuple[str, str]]:
+    kept, echo, task = [], False, ""
+    for raw in text.splitlines():
+        line = clean(raw)
+        if line.startswith("##[group]Run "):
+            echo = True
+        elif line.startswith("##[endgroup]"):
+            echo = False
+        elif TASK.match(line):
+            task = line
+        elif line and not echo and not GENERIC.search(line):
+            kept.append((task, line))
+    return kept
+
+
+def signature(text: str) -> str:
+    lines = output_lines(text)
+    for pattern in (ANNOTATION, FATAL, SIGNAL):
+        for task, line in reversed(lines):
+            if pattern.search(line):
+                found = ANNOTATION.sub("", line).strip()
+                return normalise(f"{task} {found}" if pattern is FATAL and task else found)
+    return normalise(lines[-1][1]) if lines else "<empty log>"
+
+
+def cluster(jobs: list[tuple[str, str]]) -> dict[str, list[str]]:
+    groups: dict[str, list[str]] = {}
+    for job, text in jobs:
+        groups.setdefault(signature(text), []).append(job)
+    return dict(sorted(groups.items(), key=lambda item: -len(item[1])))
+
+
+def behind(sha: str) -> str:
+    if run("git", "cat-file", "-e", f"{sha}^{{commit}}").returncode:
+        return "behind=unknown (run commit not in this clone)"
+    count = run("git", "rev-list", "--count", f"{sha}..HEAD").stdout.strip()
+    moved = {p.split("/")[0] for p in run("git", "diff", "--name-only", f"{sha}..HEAD").stdout.split()}
+    return f"behind={count} moved={','.join(sorted(moved)) or '-'}"
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(prog="ci_failures")
+    parser.add_argument("run_id")
+    parser.add_argument("--repo")
+    parser.add_argument("--out")
+    parser.add_argument("--artifacts", action="store_true")
+    args = parser.parse_args()
+    repo = args.repo or run("gh", "repo", "view", "--json", "nameWithOwner", "-q", ".nameWithOwner").stdout.strip()
+    if not repo:
+        parser.error("no --repo and gh cannot resolve the current repository")
+    meta = run("gh", "api", f"/repos/{repo}/actions/runs/{args.run_id}")
+    # per_page=100 on a large matrix answers HTTP 502; 30 per page goes through.
+    listing = run("gh", "api", "--paginate", f"/repos/{repo}/actions/runs/{args.run_id}/jobs?per_page=30",
+                  "--jq", ".jobs[] | {id, name, status, conclusion}")
+    if meta.returncode or listing.returncode:
+        print((meta.stderr or listing.stderr).strip(), file=sys.stderr)
+        return 1
+    data = json.loads(meta.stdout)
+    jobs = [json.loads(line) for line in listing.stdout.splitlines() if line]
+    out = Path(args.out or f"/tmp/ci-logs-{args.run_id}")
+    out.mkdir(parents=True, exist_ok=True)
+    print(f"run={args.run_id} repo={repo} head={data['head_sha'][:10]} status={data['status']} "
+          f"jobs={len(jobs)} {behind(data['head_sha'])}")
+    running = [j["name"] for j in jobs if j["status"] != "completed"]
+    if running:
+        print(f"running ({len(running)}, not fetched): {', '.join(running)}")
+    fetched, missing = [], []
+    for job in jobs:
+        if job["status"] != "completed" or job["conclusion"] not in FAILED:
+            continue
+        log = run("gh", "api", "--allow-escape-sequences", f"/repos/{repo}/actions/jobs/{job['id']}/logs")
+        if log.returncode or REFUSAL in log.stdout[:200]:
+            missing.append(f"{job['conclusion']}\t{job['name']}\t{log.stderr.strip()[:80]}")
+            continue
+        slug = re.sub(r"[^A-Za-z0-9._-]+", "_", job["name"])[:80]
+        path = out / f"{job['id']}-{slug}.log"
+        path.write_text("\n".join(ANSI.sub("", line) for line in log.stdout.splitlines()), encoding="utf-8")
+        text = "##[error]cancelled" if job["conclusion"] == "cancelled" else log.stdout
+        fetched.append((f"{job['conclusion']}\t{job['name']}\t{path}", text))
+    if missing:
+        (out / "no-log.txt").write_text("\n".join(missing) + "\n", encoding="utf-8")
+        counts = {c: sum(m.startswith(c) for m in missing) for c in FAILED}
+        summary = ", ".join(f"{n} {c}" for c, n in sorted(counts.items()) if n)
+        print(f"no log for {len(missing)} jobs ({summary}; never started or expired): {out / 'no-log.txt'}")
+    for number, (sig, jobs) in enumerate(cluster(fetched).items(), 1):
+        print(f"cluster {number} ({len(jobs)} jobs): {sig}")
+        for job in jobs[:3]:
+            print(f"  {job}")
+        if len(jobs) > 3:
+            members = out / f"cluster-{number}.txt"
+            members.write_text("\n".join(jobs) + "\n", encoding="utf-8")
+            print(f"  ... +{len(jobs) - 3} more: {members}")
+    if args.artifacts:
+        listing = run("gh", "api", f"/repos/{repo}/actions/runs/{args.run_id}/artifacts", "--paginate",
+                      "--jq", '.artifacts[] | "\\(.name)\\t\\(.size_in_bytes)"')
+        print("artifacts:")
+        print(listing.stdout.rstrip() or "  none")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

skills/commit-next/SKILL.md

@@ -9,8 +9,8 @@ description: >
 ---
 
 1. `git fetch --all --prune --tags`, then hunt the staged fix across every
-   ref with the merge-fix search steps (`git log --all` by `--grep`, by
-   `-S<symbol>`, by path). Describe the staged bundle in one line first so a
+   ref with the merge-fix skill's `find_fix.py` (by `--grep`, by `-S`, by
+   `--path`). Describe the staged bundle in one line first so a
    wrong search intent is visible. Only the search runs here — never
    cherry-pick or merge, and never touch the staging index or the tree.
 2. A similar fix on another branch is a stop: run the dialectic skill on the

skills/democracy/SKILL.md

@@ -111,8 +111,8 @@ records the result.
 The one protocol for every exchange between agents; `coordinate`, `triage`,
 `tune`, `funnel` and `cleanup` use it.
 
-1. **Look**: `council.py list` and the refs (`git log --all` by message, by
-   `-S`, by path). A claim or a committed fix answers the question without a
+1. **Look**: `council.py list` and the refs (the merge-fix skill's
+   `find_fix.py`). A claim or a committed fix answers the question without a
    message.
 2. **Claim** the office (Election).
 3. **One message per peer** (in Claude Code: `ListAgents`, then

skills/merge-fix/SKILL.md

@@ -35,21 +35,18 @@ one line so a wrong reading is visible before the search.
 
 ## 4. Search every ref
 
-Run all three; they find different things:
+Search by keyword, by literal and by path in one call; they find different
+things:
 
 ```bash
-git log --all --oneline --regexp-ignore-case --grep='<keyword>'
-git log --all --oneline -S'<symbol or literal>'
-git log --all --oneline -- '<path or glob>'
+${CLAUDE_SKILL_DIR}/scripts/find_fix.py --grep '<keyword>' -S '<symbol or literal>' --path '<path or glob>'
 ```
 
-Then reduce the hits:
-
-- Drop anything already merged: `git merge-base --is-ancestor <sha> HEAD`
-  exits 0 → the fix is already in, say so and stop.
-- Locate the rest: `git branch -a --contains <sha>`.
-- Read the actual change: `git show --stat <sha>` and the diff. A matching
-  commit subject is a lead, not evidence; the diff decides.
+It drops every hit already in HEAD (`in_head` counts them; all in HEAD means
+the fix is already in, say so and stop) and prints the rest with their
+branches and files. Then read the actual change: `git show --stat <sha>` and
+the diff. A matching commit subject is a lead, not evidence, and `-S` also
+matches the commit that removed the literal; the diff decides.
 
 ## 5. Present the candidates
 

skills/merge-fix/scripts/find_fix.py

@@ -0,0 +1,67 @@
+#!/usr/bin/env python3
+"""Find commits on any ref that match a fix, skipping those already in HEAD.
+
+Usage:
+    find_fix.py [--grep KEYWORD]... [-S LITERAL]... [--path PATH]... [--fetch]
+
+Prints one line per candidate: sha, up to three branches, subject, files
+touched. -S also matches commits that removed the literal; read the diff.
+Run it inside the repository; it never checks anything out.
+"""
+
+from __future__ import annotations
+
+import argparse
+import subprocess
+import sys
+
+LIMIT = 30
+
+
+def git(*args: str) -> subprocess.CompletedProcess:
+    return subprocess.run(["git", *args], capture_output=True, text=True, check=False)
+
+
+def search(args: argparse.Namespace) -> list[str]:
+    queries = [["--regexp-ignore-case", f"--grep={k}"] for k in args.grep]
+    queries += [[f"-S{s}"] for s in args.S]
+    queries += [["--", p] for p in args.path]
+    found: list[str] = []
+    for query in queries:
+        for sha in git("log", "--all", "--format=%H", *query).stdout.split():
+            if sha not in found:
+                found.append(sha)
+    return found
+
+
+def describe(sha: str) -> str:
+    subject = git("log", "-1", "--format=%h %s", sha).stdout.strip()
+    branches = git("branch", "-a", "--contains", sha, "--format=%(refname:short)").stdout.split()
+    files = [f for f in git("show", "--name-only", "--format=", sha).stdout.split("\n") if f]
+    shown = ",".join(branches[:3]) + (f",+{len(branches) - 3}" if len(branches) > 3 else "")
+    return f"{subject}\t{shown or '-'}\t{len(files)} files: {','.join(files[:5])}"
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(prog="find_fix")
+    parser.add_argument("--grep", action="append", default=[])
+    parser.add_argument("-S", action="append", default=[])
+    parser.add_argument("--path", action="append", default=[])
+    parser.add_argument("--fetch", action="store_true")
+    args = parser.parse_args()
+    if not (args.grep or args.S or args.path):
+        parser.error("give at least one --grep, -S or --path")
+    if args.fetch:
+        git("fetch", "--all", "--prune", "--tags")
+    hits = search(args)
+    outside = [sha for sha in hits if git("merge-base", "--is-ancestor", sha, "HEAD").returncode]
+    print(f"hits={len(hits)} in_head={len(hits) - len(outside)} candidates={len(outside)}")
+    for sha in outside[:LIMIT]:
+        print(describe(sha))
+    if len(outside) > LIMIT:
+        print(f"... {len(outside) - LIMIT} more, narrow the search")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

skills/observe/SKILL.md

@@ -27,7 +27,8 @@ gave one): after each observation, schedule the next wake-up yourself and pass
 the same `/observe` prompt back so the next firing repeats the watch.
 
 1. **Poll the target.** Pull its current state (job list, check conclusions, the
-   log's tail). Enumerate the failures exactly as the `triage` skill's step 1
+   log's tail); for a GitHub Actions run, the ci-artifacts skill's
+   `ci_failures.py <run>`. Enumerate the failures exactly as the `triage` skill's step 1
    does: failure, cancelled, timed-out; a downstream aggregate that fails only
    because an upstream did is not its own failure.
 2. **Diff against the baseline.** A failure is *new* only if it is not in the

skills/triage/SKILL.md

@@ -15,6 +15,10 @@ root cause is proven.
 
 ## Procedure
 
+For a GitHub Actions run, the ci-artifacts skill's `ci_failures.py <run>`
+does the mechanics of steps 0 to 2 in one call: distance to HEAD, the failed
+jobs' logs, and a first clustering. The judgement in those steps stays yours.
+
 0. **Check the run against HEAD before anything else.** A CI run is a snapshot
    of one commit, and the branch has moved since. Run
    `git log --oneline <run-sha>..HEAD | wc -l` as the first command of the

skills/tune/SKILL.md

@@ -93,7 +93,7 @@ Discard a before/after pair, not just its noise, when:
 
 ## Before you optimize
 
-- **Search every branch** (`git log --all -S<symbol>`, by path) for a change
+- **Search every branch** (the merge-fix skill's `find_fix.py -S <symbol> --path <path>`) for a change
   that already removes or reshapes the step. A fix on a sibling branch can make
   the whole candidate worth zero.
 - A step that is long by design (a backup and restore drill, an end-to-end

tests/test_ci_tools.py

@@ -0,0 +1,96 @@
+"""Exercise ci_failures clustering and find_fix's ref search."""
+
+from __future__ import annotations
+
+import importlib.util
+import shutil
+import subprocess
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+CI_FAILURES = REPO_ROOT / "skills" / "ci-artifacts" / "scripts" / "ci_failures.py"
+FIND_FIX = REPO_ROOT / "skills" / "merge-fix" / "scripts" / "find_fix.py"
+
+
+def _load(path: Path):
+    spec = importlib.util.spec_from_file_location(path.stem, path)
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+class TestCiFailures(unittest.TestCase):
+    def setUp(self):
+        self.ci = _load(CI_FAILURES)
+
+    def test_jobs_differing_only_in_identifiers_share_a_cluster(self):
+        one = "2026-10-30T10:00:00.1Z setup ok\n##[error]Timeout on host 10.0.0.12 after 300s\n"
+        two = "\x1b[31m2026-10-30T11:00:00.1Z ##[error]Timeout on host 10.0.0.47 after 299s\x1b[0m\n"
+        other = "PLAY RECAP failed=0\nfatal: unable to reach registry\n"
+        groups = self.ci.cluster([("a", one), ("b", two), ("c", other)])
+        self.assertEqual(list(groups.values()), [["a", "b"], ["c"]])
+
+    def test_command_echo_and_retries_are_skipped_and_task_named(self):
+        text = (
+            '##[group]Run make deploy || echo "failed"\n'
+            "make deploy\n"
+            "##[endgroup]\n"
+            "FAILED - RETRYING: wait (3 retries left)\n"
+            "TASK [web-app-y : Probe] ****\n"
+            "Traceback (most recent call last): recovered on retry\n"
+            "TASK [web-app-x : Wait for health] ****\n"
+            'fatal: [localhost]: FAILED! => {"msg": "timeout 120"}\n'
+            "##[error]Process completed with exit code 2.\n"
+        )
+        self.assertEqual(
+            self.ci.signature(text),
+            'TASK [web-app-x : Wait for health] **** fatal: [localhost]: FAILED! => {"msg": "timeout <n>"}',
+        )
+
+    def test_generic_exit_code_line_is_not_the_signature(self):
+        text = "Error: disk full\n##[error]Process completed with exit code 1.\n"
+        self.assertEqual(self.ci.signature(text), "Error: disk full")
+
+
+@unittest.skipUnless(shutil.which("git"), "git not installed")
+class TestFindFix(unittest.TestCase):
+    def setUp(self):
+        self.tmp = tempfile.TemporaryDirectory()
+        self.addCleanup(self.tmp.cleanup)
+        self.repo = Path(self.tmp.name)
+        self.env = {
+            "PATH": "/usr/bin:/bin",
+            "HOME": str(self.repo),
+            "GIT_AUTHOR_NAME": "t",
+            "GIT_AUTHOR_EMAIL": "t@t",
+            "GIT_COMMITTER_NAME": "t",
+            "GIT_COMMITTER_EMAIL": "t@t",
+        }
+        self._git("init", "-q", "-b", "main")
+        self._git("commit", "-q", "--allow-empty", "-m", "fix: probe in head")
+        self._git("switch", "-q", "-c", "side")
+        self._git("commit", "-q", "--allow-empty", "-m", "fix: probe on side")
+        self._git("switch", "-q", "main")
+
+    def _git(self, *args: str) -> None:
+        subprocess.run(["git", *args], check=True, capture_output=True, env=self.env, cwd=self.repo)
+
+    def test_only_commits_outside_head_are_candidates(self):
+        result = subprocess.run(
+            [sys.executable, str(FIND_FIX), "--grep", "probe"],
+            capture_output=True,
+            text=True,
+            check=True,
+            env=self.env,
+            cwd=self.repo,
+        )
+        self.assertIn("hits=2 in_head=1 candidates=1", result.stdout)
+        self.assertIn("fix: probe on side", result.stdout)
+        self.assertNotIn("probe in head", result.stdout)
+
+
+if __name__ == "__main__":
+    unittest.main()