From 58f4acdad90293073f48b5669df9bdf7482ffe7b Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Tue, 17 Nov 2026 18:40:53 +0000 Subject: [PATCH] fix(kvb-ci-artifacts): Ansible results tolerated by failed_when false or ignore_errors never become the failure signature MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ci_failures.py tracks each Ansible result block and drops it when failed_when_result is false or when the task ends with ...ignoring, so a tolerated probe's stderr no longer masks the task that really ended the job. Three tests pin the tolerated, ignored and still-failing cases; SKILL.md says so in fewer tokens than before. 🎯 Fix: 92 % 💥 Break: 3 % 💸 Tokens: 98 % ⏱️ Velocity: – 🛡️ Security: 100 % 💰 Value: ? 🏆 Score: 91 🚦 Gate: ✅ 90/5, tokens and velocity <= 100, security 100 % Co-Authored-By: Claude Opus 5.5 --- skills/kvb-ci-artifacts/SKILL.md | 12 +++--- .../kvb-ci-artifacts/scripts/ci_failures.py | 32 ++++++++++---- tests/test_ci_tools.py | 42 +++++++++++++++++++ 3 files changed, 73 insertions(+), 13 deletions(-) diff --git a/skills/kvb-ci-artifacts/SKILL.md b/skills/kvb-ci-artifacts/SKILL.md index 8ea3000..bb3db5d 100644 --- a/skills/kvb-ci-artifacts/SKILL.md +++ b/skills/kvb-ci-artifacts/SKILL.md @@ -5,8 +5,7 @@ description: > cause-holding artifacts. Trigger to explain a CI failure, and from triage. --- -A job log tells you what broke. An artifact tells you why. Investigating a CI -failure without downloading its artifacts is guessing with extra steps. +A job log tells you what broke. An artifact tells you why. ## Getting the log @@ -17,11 +16,12 @@ For a GitHub Actions run, one call does it: It prints how far HEAD moved since the run's commit, saves every finished failed job's log to `/tmp/ci-logs-/` (colour stripped, even while the run is still in progress), and clusters the jobs by the normalised error line that -ended them. Jobs still running are listed, not fetched: their API log can stop hours -early. The clustering is a heuristic; check each boundary against the logs. +ended them. Results that `failed_when: false` or `ignore_errors` tolerate never +end a job. Jobs still running are listed, not fetched: their API log can stop +hours early. The clustering is a heuristic; check each boundary against the +logs. -Grep the saved files repeatedly. Re-fetching a log to find the line you just -lost costs minutes; grepping the saved copy costs milliseconds. +Grep the saved files repeatedly instead of re-fetching a log. ## Getting the artifacts diff --git a/skills/kvb-ci-artifacts/scripts/ci_failures.py b/skills/kvb-ci-artifacts/scripts/ci_failures.py index 03f654f..e8174fc 100755 --- a/skills/kvb-ci-artifacts/scripts/ci_failures.py +++ b/skills/kvb-ci-artifacts/scripts/ci_failures.py @@ -32,6 +32,9 @@ LINT_CODE = re.compile(r"^[A-Z]{1,4}\d{3,4}\b") LINT_ARROW = re.compile(r"^-->\s+\S+:\d+:\d+") SIGNAL = re.compile(r"\b(ERROR|Error|FAILED|FAIL|fatal|Exception|failed)\b") TASK = re.compile(r"^TASK \[") +RESULT = re.compile(r"^(ok|changed|fatal|failed|skipping): \[") +TOLERATED = re.compile(r"""\bfailed_when_result["']?\s*:\s*false\b""", re.IGNORECASE) +IGNORING = "...ignoring" REFUSAL = "terminal escape sequences" @@ -39,10 +42,6 @@ def run(*cmd: str) -> subprocess.CompletedProcess: return subprocess.run(list(cmd), capture_output=True, text=True, check=False) -def clean(line: str) -> str: - return STAMP.sub("", ANSI.sub("", line)).strip() - - def normalise(line: str) -> str: line = re.sub(r"\b[0-9a-f]{7,}\b", "", line) line = re.sub(r"\d+", "", line) @@ -50,19 +49,38 @@ def normalise(line: str) -> str: def output_lines(text: str) -> list[tuple[str, str]]: + """Returns: + The (task, line) pairs that can carry a job's error: without command + echo, generic make and retry lines, Ansible results a + `failed_when: false` tolerated and every result of a task that + `ignore_errors` let pass (`...ignoring`). + """ kept, echo, task = [], False, "" + started, block, tolerated = 0, None, False for raw in text.splitlines(): - line = clean(raw) + body = STAMP.sub("", ANSI.sub("", raw)) + line = body.strip() if PROCESS_END.search(line): break - if line.startswith("##[group]Run "): + if block is not None and line and not body[:1].isspace(): + if tolerated: + del kept[block:] + block = None + if line == IGNORING: + del kept[started:] + elif line.startswith("##[group]Run "): echo = True elif line.startswith("##[endgroup]"): echo = False elif TASK.match(line): - task = line + task, started = line, len(kept) elif line and not echo and not GENERIC.search(line): + if RESULT.match(line): + block, tolerated = len(kept), False + tolerated = tolerated or (block is not None and bool(TOLERATED.search(line))) kept.append((task, line)) + if block is not None and tolerated: + del kept[block:] return kept diff --git a/tests/test_ci_tools.py b/tests/test_ci_tools.py index 352dbf1..f432fa2 100644 --- a/tests/test_ci_tools.py +++ b/tests/test_ci_tools.py @@ -68,6 +68,48 @@ class TestCiFailures(unittest.TestCase): text = "Error: disk full\n##[error]Process completed with exit code 1.\n" self.assertEqual(self.ci.signature(text), "Error: disk full") + def test_results_a_failed_when_false_tolerated_never_become_the_signature(self): + stamp = "2026-11-17T16:00:00.1Z " + text = ( + f"{stamp}TASK [svc-db-redis : Pull image] ****\n" + f"{stamp}Error: image pull timed out\n" + f"{stamp}TASK [svc-db-redis : Probe the container] ****\n" + f'{stamp}ok: [localhost] => {{"changed": false, "failed_when_result": false, "rc": 1, ' + '"stderr": "Error response from daemon: No such container: redis-central"}\n' + f"{stamp}TASK [web-app-discourse : Probe the container] ****\n" + f"{stamp}ok: [localhost] => changed=false\n" + f"{stamp} failed_when_result: false\n" + f"{stamp} stderr: 'Error response from daemon: No such container: discourse'\n" + f"{stamp}PLAY RECAP\n" + "##[error]Process completed with exit code 2.\n" + ) + self.assertEqual(self.ci.signature(text), "Error: image pull timed out") + + def test_results_ignore_errors_let_pass_never_become_the_signature(self): + text = ( + "TASK [web-app-x : Wait for health] ****\n" + 'fatal: [localhost]: FAILED! => {"msg": "timeout 120"}\n' + "TASK [svc-ai-hermes : Probe the container] ****\n" + 'failed: [localhost] (item=hermes) => {"stderr": "Error: No such container: hermes"}\n' + 'fatal: [localhost]: FAILED! => {"msg": "One or more items failed"}\n' + "...ignoring\n" + "##[error]Process completed with exit code 2.\n" + ) + self.assertEqual( + self.ci.signature(text), + 'TASK [web-app-x : Wait for health] **** fatal: [localhost]: FAILED! => {"msg": "timeout "}', + ) + + def test_a_failed_result_without_tolerance_stays_the_signature(self): + text = ( + "TASK [svc-db-redis : Probe the container] ****\n" + "fatal: [localhost]: FAILED! => changed=false\n" + " failed_when_result: true\n" + " stderr: 'No such container: redis-central'\n" + "##[error]Process completed with exit code 2.\n" + ) + self.assertEqual(self.ci.signature(text), "TASK [svc-db-redis : Probe the container] **** fatal: [localhost]: FAILED! => changed=false") + @unittest.skipUnless(shutil.which("git"), "git not installed") class TestFindFix(unittest.TestCase):