fix(kvb-ci-artifacts): Ansible results tolerated by failed_when false or ignore_errors never become the failure signature
ci_failures.py tracks each Ansible result block and drops it when failed_when_result is false or when the task ends with ...ignoring, so a tolerated probe's stderr no longer masks the task that really ended the job. Three tests pin the tolerated, ignored and still-failing cases; SKILL.md says so in fewer tokens than before. 🎯 Fix: 92 % 💥 Break: 3 % 💸 Tokens: 98 % ⏱️ Velocity: – 🛡️ Security: 100 % 💰 Value: ? 🏆 Score: 91 🚦 Gate: ✅ 90/5, tokens and velocity <= 100, security 100 % Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
e8a652a5a5
commit
58f4acdad9
3 files changed
+73
-13
No files matched your search
@@ -5,8 +5,7 @@ description: >
|
|||||||
cause-holding artifacts. Trigger to explain a CI failure, and from triage.
|
cause-holding artifacts. Trigger to explain a CI failure, and from triage.
|
||||||
---
|
---
|
||||||
|
|
||||||
A job log tells you what broke. An artifact tells you why. Investigating a CI
|
A job log tells you what broke. An artifact tells you why.
|
||||||
failure without downloading its artifacts is guessing with extra steps.
|
|
||||||
|
|
||||||
## Getting the log
|
## Getting the log
|
||||||
|
|
||||||
@@ -17,11 +16,12 @@ For a GitHub Actions run, one call does it:
|
|||||||
It prints how far HEAD moved since the run's commit, saves every finished
|
It prints how far HEAD moved since the run's commit, saves every finished
|
||||||
failed job's log to `/tmp/ci-logs-<run>/` (colour stripped, even while the run
|
failed job's log to `/tmp/ci-logs-<run>/` (colour stripped, even while the run
|
||||||
is still in progress), and clusters the jobs by the normalised error line that
|
is still in progress), and clusters the jobs by the normalised error line that
|
||||||
ended them. Jobs still running are listed, not fetched: their API log can stop hours
|
ended them. Results that `failed_when: false` or `ignore_errors` tolerate never
|
||||||
early. The clustering is a heuristic; check each boundary against the logs.
|
end a job. Jobs still running are listed, not fetched: their API log can stop
|
||||||
|
hours early. The clustering is a heuristic; check each boundary against the
|
||||||
|
logs.
|
||||||
|
|
||||||
Grep the saved files repeatedly. Re-fetching a log to find the line you just
|
Grep the saved files repeatedly instead of re-fetching a log.
|
||||||
lost costs minutes; grepping the saved copy costs milliseconds.
|
|
||||||
|
|
||||||
## Getting the artifacts
|
## Getting the artifacts
|
||||||
|
|
||||||
|
|||||||
@@ -32,6 +32,9 @@ LINT_CODE = re.compile(r"^[A-Z]{1,4}\d{3,4}\b")
|
|||||||
LINT_ARROW = re.compile(r"^-->\s+\S+:\d+:\d+")
|
LINT_ARROW = re.compile(r"^-->\s+\S+:\d+:\d+")
|
||||||
SIGNAL = re.compile(r"\b(ERROR|Error|FAILED|FAIL|fatal|Exception|failed)\b")
|
SIGNAL = re.compile(r"\b(ERROR|Error|FAILED|FAIL|fatal|Exception|failed)\b")
|
||||||
TASK = re.compile(r"^TASK \[")
|
TASK = re.compile(r"^TASK \[")
|
||||||
|
RESULT = re.compile(r"^(ok|changed|fatal|failed|skipping): \[")
|
||||||
|
TOLERATED = re.compile(r"""\bfailed_when_result["']?\s*:\s*false\b""", re.IGNORECASE)
|
||||||
|
IGNORING = "...ignoring"
|
||||||
REFUSAL = "terminal escape sequences"
|
REFUSAL = "terminal escape sequences"
|
||||||
|
|
||||||
|
|
||||||
@@ -39,10 +42,6 @@ def run(*cmd: str) -> subprocess.CompletedProcess:
|
|||||||
return subprocess.run(list(cmd), capture_output=True, text=True, check=False)
|
return subprocess.run(list(cmd), capture_output=True, text=True, check=False)
|
||||||
|
|
||||||
|
|
||||||
def clean(line: str) -> str:
|
|
||||||
return STAMP.sub("", ANSI.sub("", line)).strip()
|
|
||||||
|
|
||||||
|
|
||||||
def normalise(line: str) -> str:
|
def normalise(line: str) -> str:
|
||||||
line = re.sub(r"\b[0-9a-f]{7,}\b", "<hex>", line)
|
line = re.sub(r"\b[0-9a-f]{7,}\b", "<hex>", line)
|
||||||
line = re.sub(r"\d+", "<n>", line)
|
line = re.sub(r"\d+", "<n>", line)
|
||||||
@@ -50,19 +49,38 @@ def normalise(line: str) -> str:
|
|||||||
|
|
||||||
|
|
||||||
def output_lines(text: str) -> list[tuple[str, str]]:
|
def output_lines(text: str) -> list[tuple[str, str]]:
|
||||||
|
"""Returns:
|
||||||
|
The (task, line) pairs that can carry a job's error: without command
|
||||||
|
echo, generic make and retry lines, Ansible results a
|
||||||
|
`failed_when: false` tolerated and every result of a task that
|
||||||
|
`ignore_errors` let pass (`...ignoring`).
|
||||||
|
"""
|
||||||
kept, echo, task = [], False, ""
|
kept, echo, task = [], False, ""
|
||||||
|
started, block, tolerated = 0, None, False
|
||||||
for raw in text.splitlines():
|
for raw in text.splitlines():
|
||||||
line = clean(raw)
|
body = STAMP.sub("", ANSI.sub("", raw))
|
||||||
|
line = body.strip()
|
||||||
if PROCESS_END.search(line):
|
if PROCESS_END.search(line):
|
||||||
break
|
break
|
||||||
if line.startswith("##[group]Run "):
|
if block is not None and line and not body[:1].isspace():
|
||||||
|
if tolerated:
|
||||||
|
del kept[block:]
|
||||||
|
block = None
|
||||||
|
if line == IGNORING:
|
||||||
|
del kept[started:]
|
||||||
|
elif line.startswith("##[group]Run "):
|
||||||
echo = True
|
echo = True
|
||||||
elif line.startswith("##[endgroup]"):
|
elif line.startswith("##[endgroup]"):
|
||||||
echo = False
|
echo = False
|
||||||
elif TASK.match(line):
|
elif TASK.match(line):
|
||||||
task = line
|
task, started = line, len(kept)
|
||||||
elif line and not echo and not GENERIC.search(line):
|
elif line and not echo and not GENERIC.search(line):
|
||||||
|
if RESULT.match(line):
|
||||||
|
block, tolerated = len(kept), False
|
||||||
|
tolerated = tolerated or (block is not None and bool(TOLERATED.search(line)))
|
||||||
kept.append((task, line))
|
kept.append((task, line))
|
||||||
|
if block is not None and tolerated:
|
||||||
|
del kept[block:]
|
||||||
return kept
|
return kept
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -68,6 +68,48 @@ class TestCiFailures(unittest.TestCase):
|
|||||||
text = "Error: disk full\n##[error]Process completed with exit code 1.\n"
|
text = "Error: disk full\n##[error]Process completed with exit code 1.\n"
|
||||||
self.assertEqual(self.ci.signature(text), "Error: disk full")
|
self.assertEqual(self.ci.signature(text), "Error: disk full")
|
||||||
|
|
||||||
|
def test_results_a_failed_when_false_tolerated_never_become_the_signature(self):
|
||||||
|
stamp = "2026-11-17T16:00:00.1Z "
|
||||||
|
text = (
|
||||||
|
f"{stamp}TASK [svc-db-redis : Pull image] ****\n"
|
||||||
|
f"{stamp}Error: image pull timed out\n"
|
||||||
|
f"{stamp}TASK [svc-db-redis : Probe the container] ****\n"
|
||||||
|
f'{stamp}ok: [localhost] => {{"changed": false, "failed_when_result": false, "rc": 1, '
|
||||||
|
'"stderr": "Error response from daemon: No such container: redis-central"}\n'
|
||||||
|
f"{stamp}TASK [web-app-discourse : Probe the container] ****\n"
|
||||||
|
f"{stamp}ok: [localhost] => changed=false\n"
|
||||||
|
f"{stamp} failed_when_result: false\n"
|
||||||
|
f"{stamp} stderr: 'Error response from daemon: No such container: discourse'\n"
|
||||||
|
f"{stamp}PLAY RECAP\n"
|
||||||
|
"##[error]Process completed with exit code 2.\n"
|
||||||
|
)
|
||||||
|
self.assertEqual(self.ci.signature(text), "Error: image pull timed out")
|
||||||
|
|
||||||
|
def test_results_ignore_errors_let_pass_never_become_the_signature(self):
|
||||||
|
text = (
|
||||||
|
"TASK [web-app-x : Wait for health] ****\n"
|
||||||
|
'fatal: [localhost]: FAILED! => {"msg": "timeout 120"}\n'
|
||||||
|
"TASK [svc-ai-hermes : Probe the container] ****\n"
|
||||||
|
'failed: [localhost] (item=hermes) => {"stderr": "Error: No such container: hermes"}\n'
|
||||||
|
'fatal: [localhost]: FAILED! => {"msg": "One or more items failed"}\n'
|
||||||
|
"...ignoring\n"
|
||||||
|
"##[error]Process completed with exit code 2.\n"
|
||||||
|
)
|
||||||
|
self.assertEqual(
|
||||||
|
self.ci.signature(text),
|
||||||
|
'TASK [web-app-x : Wait for health] **** fatal: [localhost]: FAILED! => {"msg": "timeout <n>"}',
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_a_failed_result_without_tolerance_stays_the_signature(self):
|
||||||
|
text = (
|
||||||
|
"TASK [svc-db-redis : Probe the container] ****\n"
|
||||||
|
"fatal: [localhost]: FAILED! => changed=false\n"
|
||||||
|
" failed_when_result: true\n"
|
||||||
|
" stderr: 'No such container: redis-central'\n"
|
||||||
|
"##[error]Process completed with exit code 2.\n"
|
||||||
|
)
|
||||||
|
self.assertEqual(self.ci.signature(text), "TASK [svc-db-redis : Probe the container] **** fatal: [localhost]: FAILED! => changed=false")
|
||||||
|
|
||||||
|
|
||||||
@unittest.skipUnless(shutil.which("git"), "git not installed")
|
@unittest.skipUnless(shutil.which("git"), "git not installed")
|
||||||
class TestFindFix(unittest.TestCase):
|
class TestFindFix(unittest.TestCase):
|
||||||
|
|||||||
Reference in new issue
Block a user