From 6544f01eef03394403bb35fdaf86a72a33019f4c Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Fri, 30 Oct 2026 15:53:55 +0100 Subject: [PATCH] feat(ci-artifacts,merge-fix): cluster failed CI jobs and search refs through scripts Co-Authored-By: Claude Opus 5.5 (1M context) --- skills/ci-artifacts/SKILL.md | 23 ++-- skills/ci-artifacts/scripts/ci_failures.py | 149 +++++++++++++++++++++ skills/commit-next/SKILL.md | 4 +- skills/democracy/SKILL.md | 4 +- skills/merge-fix/SKILL.md | 19 ++- skills/merge-fix/scripts/find_fix.py | 67 +++++++++ skills/observe/SKILL.md | 3 +- skills/triage/SKILL.md | 4 + skills/tune/SKILL.md | 2 +- tests/test_ci_tools.py | 96 +++++++++++++ 10 files changed, 340 insertions(+), 31 deletions(-) create mode 100755 skills/ci-artifacts/scripts/ci_failures.py create mode 100755 skills/merge-fix/scripts/find_fix.py create mode 100644 tests/test_ci_tools.py diff --git a/skills/ci-artifacts/SKILL.md b/skills/ci-artifacts/SKILL.md index b06bd32..6d5a3f3 100644 --- a/skills/ci-artifacts/SKILL.md +++ b/skills/ci-artifacts/SKILL.md @@ -12,23 +12,18 @@ failure without downloading its artifacts is guessing with extra steps. ## Getting the log -`gh run view --log` refuses to serve anything while a run is unfinished: +For a GitHub Actions run, one call does it: - run is still in progress; logs will be available when it is complete + ${CLAUDE_SKILL_DIR}/scripts/ci_failures.py [--repo /] [--artifacts] -That is not a dead end — individual *jobs* that have already finished are -served by the API: +It prints how far HEAD moved since the run's commit, saves every finished +failed job's log to `/tmp/ci-logs-/` (colour stripped, even while the run +is still in progress), and clusters the jobs by the normalised error line that +ended them. Jobs still running are listed, not fetched: their API log can stop hours +early. The clustering is a heuristic; check each boundary against the logs. - gh api /repos///actions/jobs//logs - -Enumerate the failing jobs and their ids first: - - gh run view --repo / --json jobs \ - --jq '.jobs[] | select(.conclusion=="failure") | "\(.databaseId)\t\(.name)"' - -Save every log to a file (`/tmp/ci-logs-/.log`) and grep the file -repeatedly. Re-fetching a log to find the line you just lost costs minutes; -grepping the saved copy costs milliseconds. +Grep the saved files repeatedly. Re-fetching a log to find the line you just +lost costs minutes; grepping the saved copy costs milliseconds. ## Getting the artifacts diff --git a/skills/ci-artifacts/scripts/ci_failures.py b/skills/ci-artifacts/scripts/ci_failures.py new file mode 100755 index 0000000..1d89948 --- /dev/null +++ b/skills/ci-artifacts/scripts/ci_failures.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +"""Fetch the failed jobs of a GitHub Actions run and cluster them by error line. + +Usage: + ci_failures.py RUN_ID [--repo OWNER/REPO] [--out DIR] [--artifacts] + +Prints how far HEAD moved since the run's commit, saves each finished failed +job's log (colour stripped) under DIR (default /tmp/ci-logs-RUN_ID), and +groups the jobs by the normalised error line that ended them (the last one). Jobs still running are +listed but not fetched: their API log can stop hours early. The clustering is +a heuristic; confirm a cluster's boundary against the logs. +""" + +from __future__ import annotations + +import argparse +import json +import re +import subprocess +import sys +from pathlib import Path + +FAILED = {"failure", "cancelled", "timed_out"} +ANSI = re.compile(r"\x1b\[[0-9;]*[A-Za-z]") +STAMP = re.compile(r"^\d{4}-\d\d-\d\dT[\d:.]+Z\s?") +GENERIC = re.compile(r"Process completed with exit code|failed=0\b|FAILED - RETRYING") +ANNOTATION = re.compile(r"##\[error\]") +FATAL = re.compile(r"fatal: \[|FAILED!|Traceback|AssertionError|panic:") +SIGNAL = re.compile(r"\b(ERROR|Error|FAILED|FAIL|fatal|Exception|failed)\b") +TASK = re.compile(r"^TASK \[") +REFUSAL = "terminal escape sequences" + + +def run(*cmd: str) -> subprocess.CompletedProcess: + return subprocess.run(list(cmd), capture_output=True, text=True, check=False) + + +def clean(line: str) -> str: + return STAMP.sub("", ANSI.sub("", line)).strip() + + +def normalise(line: str) -> str: + line = re.sub(r"\b[0-9a-f]{7,}\b", "", line) + line = re.sub(r"\d+", "", line) + return line[:200] + + +def output_lines(text: str) -> list[tuple[str, str]]: + kept, echo, task = [], False, "" + for raw in text.splitlines(): + line = clean(raw) + if line.startswith("##[group]Run "): + echo = True + elif line.startswith("##[endgroup]"): + echo = False + elif TASK.match(line): + task = line + elif line and not echo and not GENERIC.search(line): + kept.append((task, line)) + return kept + + +def signature(text: str) -> str: + lines = output_lines(text) + for pattern in (ANNOTATION, FATAL, SIGNAL): + for task, line in reversed(lines): + if pattern.search(line): + found = ANNOTATION.sub("", line).strip() + return normalise(f"{task} {found}" if pattern is FATAL and task else found) + return normalise(lines[-1][1]) if lines else "" + + +def cluster(jobs: list[tuple[str, str]]) -> dict[str, list[str]]: + groups: dict[str, list[str]] = {} + for job, text in jobs: + groups.setdefault(signature(text), []).append(job) + return dict(sorted(groups.items(), key=lambda item: -len(item[1]))) + + +def behind(sha: str) -> str: + if run("git", "cat-file", "-e", f"{sha}^{{commit}}").returncode: + return "behind=unknown (run commit not in this clone)" + count = run("git", "rev-list", "--count", f"{sha}..HEAD").stdout.strip() + moved = {p.split("/")[0] for p in run("git", "diff", "--name-only", f"{sha}..HEAD").stdout.split()} + return f"behind={count} moved={','.join(sorted(moved)) or '-'}" + + +def main() -> int: + parser = argparse.ArgumentParser(prog="ci_failures") + parser.add_argument("run_id") + parser.add_argument("--repo") + parser.add_argument("--out") + parser.add_argument("--artifacts", action="store_true") + args = parser.parse_args() + repo = args.repo or run("gh", "repo", "view", "--json", "nameWithOwner", "-q", ".nameWithOwner").stdout.strip() + if not repo: + parser.error("no --repo and gh cannot resolve the current repository") + meta = run("gh", "api", f"/repos/{repo}/actions/runs/{args.run_id}") + # per_page=100 on a large matrix answers HTTP 502; 30 per page goes through. + listing = run("gh", "api", "--paginate", f"/repos/{repo}/actions/runs/{args.run_id}/jobs?per_page=30", + "--jq", ".jobs[] | {id, name, status, conclusion}") + if meta.returncode or listing.returncode: + print((meta.stderr or listing.stderr).strip(), file=sys.stderr) + return 1 + data = json.loads(meta.stdout) + jobs = [json.loads(line) for line in listing.stdout.splitlines() if line] + out = Path(args.out or f"/tmp/ci-logs-{args.run_id}") + out.mkdir(parents=True, exist_ok=True) + print(f"run={args.run_id} repo={repo} head={data['head_sha'][:10]} status={data['status']} " + f"jobs={len(jobs)} {behind(data['head_sha'])}") + running = [j["name"] for j in jobs if j["status"] != "completed"] + if running: + print(f"running ({len(running)}, not fetched): {', '.join(running)}") + fetched, missing = [], [] + for job in jobs: + if job["status"] != "completed" or job["conclusion"] not in FAILED: + continue + log = run("gh", "api", "--allow-escape-sequences", f"/repos/{repo}/actions/jobs/{job['id']}/logs") + if log.returncode or REFUSAL in log.stdout[:200]: + missing.append(f"{job['conclusion']}\t{job['name']}\t{log.stderr.strip()[:80]}") + continue + slug = re.sub(r"[^A-Za-z0-9._-]+", "_", job["name"])[:80] + path = out / f"{job['id']}-{slug}.log" + path.write_text("\n".join(ANSI.sub("", line) for line in log.stdout.splitlines()), encoding="utf-8") + text = "##[error]cancelled" if job["conclusion"] == "cancelled" else log.stdout + fetched.append((f"{job['conclusion']}\t{job['name']}\t{path}", text)) + if missing: + (out / "no-log.txt").write_text("\n".join(missing) + "\n", encoding="utf-8") + counts = {c: sum(m.startswith(c) for m in missing) for c in FAILED} + summary = ", ".join(f"{n} {c}" for c, n in sorted(counts.items()) if n) + print(f"no log for {len(missing)} jobs ({summary}; never started or expired): {out / 'no-log.txt'}") + for number, (sig, jobs) in enumerate(cluster(fetched).items(), 1): + print(f"cluster {number} ({len(jobs)} jobs): {sig}") + for job in jobs[:3]: + print(f" {job}") + if len(jobs) > 3: + members = out / f"cluster-{number}.txt" + members.write_text("\n".join(jobs) + "\n", encoding="utf-8") + print(f" ... +{len(jobs) - 3} more: {members}") + if args.artifacts: + listing = run("gh", "api", f"/repos/{repo}/actions/runs/{args.run_id}/artifacts", "--paginate", + "--jq", '.artifacts[] | "\\(.name)\\t\\(.size_in_bytes)"') + print("artifacts:") + print(listing.stdout.rstrip() or " none") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/skills/commit-next/SKILL.md b/skills/commit-next/SKILL.md index 79a0f78..fc22b26 100644 --- a/skills/commit-next/SKILL.md +++ b/skills/commit-next/SKILL.md @@ -9,8 +9,8 @@ description: > --- 1. `git fetch --all --prune --tags`, then hunt the staged fix across every - ref with the merge-fix search steps (`git log --all` by `--grep`, by - `-S`, by path). Describe the staged bundle in one line first so a + ref with the merge-fix skill's `find_fix.py` (by `--grep`, by `-S`, by + `--path`). Describe the staged bundle in one line first so a wrong search intent is visible. Only the search runs here — never cherry-pick or merge, and never touch the staging index or the tree. 2. A similar fix on another branch is a stop: run the dialectic skill on the diff --git a/skills/democracy/SKILL.md b/skills/democracy/SKILL.md index b41f87d..e83087b 100644 --- a/skills/democracy/SKILL.md +++ b/skills/democracy/SKILL.md @@ -111,8 +111,8 @@ records the result. The one protocol for every exchange between agents; `coordinate`, `triage`, `tune`, `funnel` and `cleanup` use it. -1. **Look**: `council.py list` and the refs (`git log --all` by message, by - `-S`, by path). A claim or a committed fix answers the question without a +1. **Look**: `council.py list` and the refs (the merge-fix skill's + `find_fix.py`). A claim or a committed fix answers the question without a message. 2. **Claim** the office (Election). 3. **One message per peer** (in Claude Code: `ListAgents`, then diff --git a/skills/merge-fix/SKILL.md b/skills/merge-fix/SKILL.md index c217f23..e92ca6f 100644 --- a/skills/merge-fix/SKILL.md +++ b/skills/merge-fix/SKILL.md @@ -35,21 +35,18 @@ one line so a wrong reading is visible before the search. ## 4. Search every ref -Run all three; they find different things: +Search by keyword, by literal and by path in one call; they find different +things: ```bash -git log --all --oneline --regexp-ignore-case --grep='' -git log --all --oneline -S'' -git log --all --oneline -- '' +${CLAUDE_SKILL_DIR}/scripts/find_fix.py --grep '' -S '' --path '' ``` -Then reduce the hits: - -- Drop anything already merged: `git merge-base --is-ancestor HEAD` - exits 0 → the fix is already in, say so and stop. -- Locate the rest: `git branch -a --contains `. -- Read the actual change: `git show --stat ` and the diff. A matching - commit subject is a lead, not evidence; the diff decides. +It drops every hit already in HEAD (`in_head` counts them; all in HEAD means +the fix is already in, say so and stop) and prints the rest with their +branches and files. Then read the actual change: `git show --stat ` and +the diff. A matching commit subject is a lead, not evidence, and `-S` also +matches the commit that removed the literal; the diff decides. ## 5. Present the candidates diff --git a/skills/merge-fix/scripts/find_fix.py b/skills/merge-fix/scripts/find_fix.py new file mode 100755 index 0000000..29b4981 --- /dev/null +++ b/skills/merge-fix/scripts/find_fix.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +"""Find commits on any ref that match a fix, skipping those already in HEAD. + +Usage: + find_fix.py [--grep KEYWORD]... [-S LITERAL]... [--path PATH]... [--fetch] + +Prints one line per candidate: sha, up to three branches, subject, files +touched. -S also matches commits that removed the literal; read the diff. +Run it inside the repository; it never checks anything out. +""" + +from __future__ import annotations + +import argparse +import subprocess +import sys + +LIMIT = 30 + + +def git(*args: str) -> subprocess.CompletedProcess: + return subprocess.run(["git", *args], capture_output=True, text=True, check=False) + + +def search(args: argparse.Namespace) -> list[str]: + queries = [["--regexp-ignore-case", f"--grep={k}"] for k in args.grep] + queries += [[f"-S{s}"] for s in args.S] + queries += [["--", p] for p in args.path] + found: list[str] = [] + for query in queries: + for sha in git("log", "--all", "--format=%H", *query).stdout.split(): + if sha not in found: + found.append(sha) + return found + + +def describe(sha: str) -> str: + subject = git("log", "-1", "--format=%h %s", sha).stdout.strip() + branches = git("branch", "-a", "--contains", sha, "--format=%(refname:short)").stdout.split() + files = [f for f in git("show", "--name-only", "--format=", sha).stdout.split("\n") if f] + shown = ",".join(branches[:3]) + (f",+{len(branches) - 3}" if len(branches) > 3 else "") + return f"{subject}\t{shown or '-'}\t{len(files)} files: {','.join(files[:5])}" + + +def main() -> int: + parser = argparse.ArgumentParser(prog="find_fix") + parser.add_argument("--grep", action="append", default=[]) + parser.add_argument("-S", action="append", default=[]) + parser.add_argument("--path", action="append", default=[]) + parser.add_argument("--fetch", action="store_true") + args = parser.parse_args() + if not (args.grep or args.S or args.path): + parser.error("give at least one --grep, -S or --path") + if args.fetch: + git("fetch", "--all", "--prune", "--tags") + hits = search(args) + outside = [sha for sha in hits if git("merge-base", "--is-ancestor", sha, "HEAD").returncode] + print(f"hits={len(hits)} in_head={len(hits) - len(outside)} candidates={len(outside)}") + for sha in outside[:LIMIT]: + print(describe(sha)) + if len(outside) > LIMIT: + print(f"... {len(outside) - LIMIT} more, narrow the search") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/skills/observe/SKILL.md b/skills/observe/SKILL.md index f16ed32..91605a6 100644 --- a/skills/observe/SKILL.md +++ b/skills/observe/SKILL.md @@ -27,7 +27,8 @@ gave one): after each observation, schedule the next wake-up yourself and pass the same `/observe` prompt back so the next firing repeats the watch. 1. **Poll the target.** Pull its current state (job list, check conclusions, the - log's tail). Enumerate the failures exactly as the `triage` skill's step 1 + log's tail); for a GitHub Actions run, the ci-artifacts skill's + `ci_failures.py `. Enumerate the failures exactly as the `triage` skill's step 1 does: failure, cancelled, timed-out; a downstream aggregate that fails only because an upstream did is not its own failure. 2. **Diff against the baseline.** A failure is *new* only if it is not in the diff --git a/skills/triage/SKILL.md b/skills/triage/SKILL.md index 349b6ae..ca66cf5 100644 --- a/skills/triage/SKILL.md +++ b/skills/triage/SKILL.md @@ -15,6 +15,10 @@ root cause is proven. ## Procedure +For a GitHub Actions run, the ci-artifacts skill's `ci_failures.py ` +does the mechanics of steps 0 to 2 in one call: distance to HEAD, the failed +jobs' logs, and a first clustering. The judgement in those steps stays yours. + 0. **Check the run against HEAD before anything else.** A CI run is a snapshot of one commit, and the branch has moved since. Run `git log --oneline ..HEAD | wc -l` as the first command of the diff --git a/skills/tune/SKILL.md b/skills/tune/SKILL.md index 900300d..cf96f51 100644 --- a/skills/tune/SKILL.md +++ b/skills/tune/SKILL.md @@ -93,7 +93,7 @@ Discard a before/after pair, not just its noise, when: ## Before you optimize -- **Search every branch** (`git log --all -S`, by path) for a change +- **Search every branch** (the merge-fix skill's `find_fix.py -S --path `) for a change that already removes or reshapes the step. A fix on a sibling branch can make the whole candidate worth zero. - A step that is long by design (a backup and restore drill, an end-to-end diff --git a/tests/test_ci_tools.py b/tests/test_ci_tools.py new file mode 100644 index 0000000..a29a74b --- /dev/null +++ b/tests/test_ci_tools.py @@ -0,0 +1,96 @@ +"""Exercise ci_failures clustering and find_fix's ref search.""" + +from __future__ import annotations + +import importlib.util +import shutil +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] +CI_FAILURES = REPO_ROOT / "skills" / "ci-artifacts" / "scripts" / "ci_failures.py" +FIND_FIX = REPO_ROOT / "skills" / "merge-fix" / "scripts" / "find_fix.py" + + +def _load(path: Path): + spec = importlib.util.spec_from_file_location(path.stem, path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class TestCiFailures(unittest.TestCase): + def setUp(self): + self.ci = _load(CI_FAILURES) + + def test_jobs_differing_only_in_identifiers_share_a_cluster(self): + one = "2026-10-30T10:00:00.1Z setup ok\n##[error]Timeout on host 10.0.0.12 after 300s\n" + two = "\x1b[31m2026-10-30T11:00:00.1Z ##[error]Timeout on host 10.0.0.47 after 299s\x1b[0m\n" + other = "PLAY RECAP failed=0\nfatal: unable to reach registry\n" + groups = self.ci.cluster([("a", one), ("b", two), ("c", other)]) + self.assertEqual(list(groups.values()), [["a", "b"], ["c"]]) + + def test_command_echo_and_retries_are_skipped_and_task_named(self): + text = ( + '##[group]Run make deploy || echo "failed"\n' + "make deploy\n" + "##[endgroup]\n" + "FAILED - RETRYING: wait (3 retries left)\n" + "TASK [web-app-y : Probe] ****\n" + "Traceback (most recent call last): recovered on retry\n" + "TASK [web-app-x : Wait for health] ****\n" + 'fatal: [localhost]: FAILED! => {"msg": "timeout 120"}\n' + "##[error]Process completed with exit code 2.\n" + ) + self.assertEqual( + self.ci.signature(text), + 'TASK [web-app-x : Wait for health] **** fatal: [localhost]: FAILED! => {"msg": "timeout "}', + ) + + def test_generic_exit_code_line_is_not_the_signature(self): + text = "Error: disk full\n##[error]Process completed with exit code 1.\n" + self.assertEqual(self.ci.signature(text), "Error: disk full") + + +@unittest.skipUnless(shutil.which("git"), "git not installed") +class TestFindFix(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.repo = Path(self.tmp.name) + self.env = { + "PATH": "/usr/bin:/bin", + "HOME": str(self.repo), + "GIT_AUTHOR_NAME": "t", + "GIT_AUTHOR_EMAIL": "t@t", + "GIT_COMMITTER_NAME": "t", + "GIT_COMMITTER_EMAIL": "t@t", + } + self._git("init", "-q", "-b", "main") + self._git("commit", "-q", "--allow-empty", "-m", "fix: probe in head") + self._git("switch", "-q", "-c", "side") + self._git("commit", "-q", "--allow-empty", "-m", "fix: probe on side") + self._git("switch", "-q", "main") + + def _git(self, *args: str) -> None: + subprocess.run(["git", *args], check=True, capture_output=True, env=self.env, cwd=self.repo) + + def test_only_commits_outside_head_are_candidates(self): + result = subprocess.run( + [sys.executable, str(FIND_FIX), "--grep", "probe"], + capture_output=True, + text=True, + check=True, + env=self.env, + cwd=self.repo, + ) + self.assertIn("hits=2 in_head=1 candidates=1", result.stdout) + self.assertIn("fix: probe on side", result.stdout) + self.assertNotIn("probe in head", result.stdout) + + +if __name__ == "__main__": + unittest.main()