fix(kvb-ci-artifacts): Ansible results tolerated by failed_when false or ignore_errors never become the failure signature

ci_failures.py tracks each Ansible result block and drops it when failed_when_result is false or when the task ends with ...ignoring, so a tolerated probe's stderr no longer masks the task that really ended the job. Three tests pin the tolerated, ignored and still-failing cases; SKILL.md says so in fewer tokens than before.

🎯 Fix: 92 %
💥 Break: 3 %
💸 Tokens: 98 %
⏱️ Velocity: –
🛡️ Security: 100 %
💰 Value: ?
🏆 Score: 91
🚦 Gate: ✅ 90/5, tokens and velocity <= 100, security 100 %

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Kevin Veen-BirkenbachandClaude Opus 5.5 committed 2026-11-17 18:40:53 +00:00
1 parent e8a652a5a5
commit 58f4acdad9
3 files changed
+73 -13

No files matched your search

+6 -6
View File
@@ -5,8 +5,7 @@ description: >
cause-holding artifacts. Trigger to explain a CI failure, and from triage. cause-holding artifacts. Trigger to explain a CI failure, and from triage.
--- ---
A job log tells you what broke. An artifact tells you why. Investigating a CI A job log tells you what broke. An artifact tells you why.
failure without downloading its artifacts is guessing with extra steps.
## Getting the log ## Getting the log
@@ -17,11 +16,12 @@ For a GitHub Actions run, one call does it:
It prints how far HEAD moved since the run's commit, saves every finished It prints how far HEAD moved since the run's commit, saves every finished
failed job's log to `/tmp/ci-logs-<run>/` (colour stripped, even while the run failed job's log to `/tmp/ci-logs-<run>/` (colour stripped, even while the run
is still in progress), and clusters the jobs by the normalised error line that is still in progress), and clusters the jobs by the normalised error line that
ended them. Jobs still running are listed, not fetched: their API log can stop hours ended them. Results that `failed_when: false` or `ignore_errors` tolerate never
early. The clustering is a heuristic; check each boundary against the logs. end a job. Jobs still running are listed, not fetched: their API log can stop
hours early. The clustering is a heuristic; check each boundary against the
logs.
Grep the saved files repeatedly. Re-fetching a log to find the line you just Grep the saved files repeatedly instead of re-fetching a log.
lost costs minutes; grepping the saved copy costs milliseconds.
## Getting the artifacts ## Getting the artifacts
+25 -7
View File
@@ -32,6 +32,9 @@ LINT_CODE = re.compile(r"^[A-Z]{1,4}\d{3,4}\b")
LINT_ARROW = re.compile(r"^-->\s+\S+:\d+:\d+") LINT_ARROW = re.compile(r"^-->\s+\S+:\d+:\d+")
SIGNAL = re.compile(r"\b(ERROR|Error|FAILED|FAIL|fatal|Exception|failed)\b") SIGNAL = re.compile(r"\b(ERROR|Error|FAILED|FAIL|fatal|Exception|failed)\b")
TASK = re.compile(r"^TASK \[") TASK = re.compile(r"^TASK \[")
RESULT = re.compile(r"^(ok|changed|fatal|failed|skipping): \[")
TOLERATED = re.compile(r"""\bfailed_when_result["']?\s*:\s*false\b""", re.IGNORECASE)
IGNORING = "...ignoring"
REFUSAL = "terminal escape sequences" REFUSAL = "terminal escape sequences"
@@ -39,10 +42,6 @@ def run(*cmd: str) -> subprocess.CompletedProcess:
return subprocess.run(list(cmd), capture_output=True, text=True, check=False) return subprocess.run(list(cmd), capture_output=True, text=True, check=False)
def clean(line: str) -> str:
return STAMP.sub("", ANSI.sub("", line)).strip()
def normalise(line: str) -> str: def normalise(line: str) -> str:
line = re.sub(r"\b[0-9a-f]{7,}\b", "<hex>", line) line = re.sub(r"\b[0-9a-f]{7,}\b", "<hex>", line)
line = re.sub(r"\d+", "<n>", line) line = re.sub(r"\d+", "<n>", line)
@@ -50,19 +49,38 @@ def normalise(line: str) -> str:
def output_lines(text: str) -> list[tuple[str, str]]: def output_lines(text: str) -> list[tuple[str, str]]:
"""Returns:
The (task, line) pairs that can carry a job's error: without command
echo, generic make and retry lines, Ansible results a
`failed_when: false` tolerated and every result of a task that
`ignore_errors` let pass (`...ignoring`).
"""
kept, echo, task = [], False, "" kept, echo, task = [], False, ""
started, block, tolerated = 0, None, False
for raw in text.splitlines(): for raw in text.splitlines():
line = clean(raw) body = STAMP.sub("", ANSI.sub("", raw))
line = body.strip()
if PROCESS_END.search(line): if PROCESS_END.search(line):
break break
if line.startswith("##[group]Run "): if block is not None and line and not body[:1].isspace():
if tolerated:
del kept[block:]
block = None
if line == IGNORING:
del kept[started:]
elif line.startswith("##[group]Run "):
echo = True echo = True
elif line.startswith("##[endgroup]"): elif line.startswith("##[endgroup]"):
echo = False echo = False
elif TASK.match(line): elif TASK.match(line):
task = line task, started = line, len(kept)
elif line and not echo and not GENERIC.search(line): elif line and not echo and not GENERIC.search(line):
if RESULT.match(line):
block, tolerated = len(kept), False
tolerated = tolerated or (block is not None and bool(TOLERATED.search(line)))
kept.append((task, line)) kept.append((task, line))
if block is not None and tolerated:
del kept[block:]
return kept return kept
+42
View File
@@ -68,6 +68,48 @@ class TestCiFailures(unittest.TestCase):
text = "Error: disk full\n##[error]Process completed with exit code 1.\n" text = "Error: disk full\n##[error]Process completed with exit code 1.\n"
self.assertEqual(self.ci.signature(text), "Error: disk full") self.assertEqual(self.ci.signature(text), "Error: disk full")
def test_results_a_failed_when_false_tolerated_never_become_the_signature(self):
stamp = "2026-11-17T16:00:00.1Z "
text = (
f"{stamp}TASK [svc-db-redis : Pull image] ****\n"
f"{stamp}Error: image pull timed out\n"
f"{stamp}TASK [svc-db-redis : Probe the container] ****\n"
f'{stamp}ok: [localhost] => {{"changed": false, "failed_when_result": false, "rc": 1, '
'"stderr": "Error response from daemon: No such container: redis-central"}\n'
f"{stamp}TASK [web-app-discourse : Probe the container] ****\n"
f"{stamp}ok: [localhost] => changed=false\n"
f"{stamp} failed_when_result: false\n"
f"{stamp} stderr: 'Error response from daemon: No such container: discourse'\n"
f"{stamp}PLAY RECAP\n"
"##[error]Process completed with exit code 2.\n"
)
self.assertEqual(self.ci.signature(text), "Error: image pull timed out")
def test_results_ignore_errors_let_pass_never_become_the_signature(self):
text = (
"TASK [web-app-x : Wait for health] ****\n"
'fatal: [localhost]: FAILED! => {"msg": "timeout 120"}\n'
"TASK [svc-ai-hermes : Probe the container] ****\n"
'failed: [localhost] (item=hermes) => {"stderr": "Error: No such container: hermes"}\n'
'fatal: [localhost]: FAILED! => {"msg": "One or more items failed"}\n'
"...ignoring\n"
"##[error]Process completed with exit code 2.\n"
)
self.assertEqual(
self.ci.signature(text),
'TASK [web-app-x : Wait for health] **** fatal: [localhost]: FAILED! => {"msg": "timeout <n>"}',
)
def test_a_failed_result_without_tolerance_stays_the_signature(self):
text = (
"TASK [svc-db-redis : Probe the container] ****\n"
"fatal: [localhost]: FAILED! => changed=false\n"
" failed_when_result: true\n"
" stderr: 'No such container: redis-central'\n"
"##[error]Process completed with exit code 2.\n"
)
self.assertEqual(self.ci.signature(text), "TASK [svc-db-redis : Probe the container] **** fatal: [localhost]: FAILED! => changed=false")
@unittest.skipUnless(shutil.which("git"), "git not installed") @unittest.skipUnless(shutil.which("git"), "git not installed")
class TestFindFix(unittest.TestCase): class TestFindFix(unittest.TestCase):