diff --git a/skills/kvb-org-organize/capacity.md b/skills/kvb-org-organize/capacity.md index 67e673b..be5c2db 100644 --- a/skills/kvb-org-organize/capacity.md +++ b/skills/kvb-org-organize/capacity.md @@ -51,7 +51,10 @@ and stays silent below the thresholds. plans a cleanup and grants no large slot. - **Stop**: new heavy jobs are blocked in every session (teardown commands never), and the hook runs the configured low-risk cleanup by itself; it - skips every worktree where a gate or deploy process still runs. + skips every worktree where a gate or deploy process still runs. Its + `docker image prune` removes only images whose repository and digest are + both ``, so pulls by digest stay; `system prune`, `rmi`, `image rm` + and `compose down --rmi` are refused. An accident guard, not a sandbox. - **Critical**: additionally a desktop alarm to the operator. Running jobs and their stacks still keep running; only the operator can make an exception. - A measured fall that would reach the critical threshold within the diff --git a/skills/kvb-org-organize/scripts/watch.py b/skills/kvb-org-organize/scripts/watch.py index dd66b69..8fe5367 100755 --- a/skills/kvb-org-organize/scripts/watch.py +++ b/skills/kvb-org-organize/scripts/watch.py @@ -24,6 +24,13 @@ Usage: configured low-risk cleanup commands watch.py status print the stage computed from the live free space and the stored samples, and the stored measurement's age in seconds + watch.py guard REAL REPORT ARGS... + the docker and docker-compose shim of the cleanup shell + lines: an image prune becomes the leftover step on the + same daemon, a system prune, rmi, image rm, image + remove or compose --rmi is refused with exit 3, + anything else execs REAL; result lines are appended to + REPORT Environment: AGENTS_COUNCIL_ROOT: shared state directory; /tmp/agents-council unless set, @@ -42,8 +49,22 @@ Environment: Config keys, all required: disk, thresholds_gb {warn, stop, critical}, projection_minutes, interval_seconds, cache_interval_seconds, heavy_commands, -never_block (regexes), cleanup (shell commands), worktree_cleanup (argv lists -with {project}, run without a shell in /, so no worktree file is read), +never_block (regexes), cleanup (shell commands; a plain `docker image prune` +with nothing but -a, -f, --all, --force or --filter X never runs: the first +stands for the built-in step that lists the dangling images and removes, +without force, each whose repository and digest are both , so tagged +images, pulls by digest and images a container uses stay; every other line +runs with the guard shims of docker and docker-compose first on PATH, which +turn each image prune into that step on the same daemon and refuse system +prune, rmi, image rm, image remove and compose --rmi; a line that would reach +docker past the shims, by a path in command position, a PATH reset or a +privilege switch, is refused and none of it runs; this guards operator-written +lines against accidents, and a computed docker path, a script resetting PATH +or the docker API socket are out of its scope), +worktree_cleanup (argv lists with {project}, run without a shell in /, so no +worktree file is read; `docker compose -p {project} exec -T docker +image prune` with prune flags runs the leftover step inside that daemon, any +other entry naming an image removal is refused), worktrees_command (shell command printing per running stack its worktree path and compose project, tab-separated, one per line; a project that is not a compose project name is skipped), worktree_marker (path relative @@ -74,9 +95,11 @@ import math import os import re import shlex +import shutil import stat import subprocess import sys +import tempfile import time from pathlib import Path from types import ModuleType @@ -134,6 +157,46 @@ COMPOSE_VALUES = frozenset({ "--progress", }) BUILDING = frozenset({"up", "run", "create"}) +LEFTOVERS = ["images", "--filter", "dangling=true", "--format", "{{.ID}} {{.Repository}} {{.Digest}}"] +IMAGE_ID = re.compile(r"(?:sha256:)?[0-9a-f]{12,64}") +NONE = "" +PRUNE_COMMANDS = (["image", "prune"], ["images", "prune"]) +PRUNE_FLAG = re.compile(r"-[af]+|--force|--all|--filter=.+", re.DOTALL) +DOCKER_GLOBALS = frozenset({ + "-H", "--host", "-c", "--context", "--config", "-l", "--log-level", "--tlscacert", "--tlscert", "--tlskey", +}) +SPARING = "a plain docker image prune removes the build leftovers and keeps them" +REMOVALS = { + ("system", "prune"): f"docker system prune also removes digest-pinned images; {SPARING}", + ("rmi",): f"docker rmi can remove digest-pinned images; {SPARING}", + ("image", "rm"): f"docker image rm can remove digest-pinned images; {SPARING}", + ("image", "remove"): f"docker image remove can remove digest-pinned images; {SPARING}", +} +COMPOSE_RMI = f"compose down --rmi removes the services' images, digest-pinned ones included; {SPARING}" +GUARDED_BINARIES = ("docker", "docker-compose") +INNER_REMOVAL = re.compile(r"\bdocker\b.*\b(?:images? (?:prune|rm|remove)|system prune|rmi)\b", re.DOTALL) +WORKTREE_REMOVAL = re.compile(r"\b(?:images? (?:prune|rm|remove)|system prune|rmi)\b") +QUOTING = re.compile(r"[\"'`\\]") +UNGUARDED = re.compile(r"\bPATH=|\bcommand\s+-p\b|\benv\b[^;&|\n]*\s(?:-i|--ignore-environment)\b") +PRIVILEGED = re.compile(r"\b(?:sudo|doas|su|pkexec|run0|systemd-run)\b") +IN_USE = re.compile( + r"^Error response from daemon: conflict: unable to delete \S+ \((?:must|cannot) be forced\) - " + r"image is being used by (?:running|stopped) container \S+$", + re.MULTILINE, +) +OK, KEPT = "ok: ", "kept (in use): " +GUARD_REFUSED = 3 +SHIM = '#!/bin/sh\nexec {python} -I {watch} guard {real} {report} "$@"\n' +REFUSALS = { + "unguarded": ( + "docker run by a path, PATH=, env -i, command -p, sudo, doas, su, pkexec, run0 or systemd-run would " + "bypass the docker guard; call docker by name" + ), + "nested": ( + "it would remove images in the worktree's daemon; only docker compose -p {project} exec -T " + "docker image prune [-a|--all] [-f|--force] [--filter X] is supported" + ), +} SEPARATOR = re.compile(r"[();<>|&\n]+") PUNCTUATION = "();<>|&\n" WORD_BREAK = " \t\n;&|()<>" @@ -1200,20 +1263,251 @@ def busy(tree: str, names: list[str]) -> bool: return False +def prune_flags(words: list[str]) -> bool: + flags = iter(words) + return all(PRUNE_FLAG.fullmatch(word) or (word == "--filter" and next(flags, None) is not None) for word in flags) + + +def plain_prune(command: str) -> bool: + """Returns: + Whether the line parses to exactly `docker image prune` or `docker + images prune` followed by nothing but -a, -f, --all, --force and + --filter X. + """ + try: + words = shlex.split(command) + except ValueError: + return False + return words[:1] == ["docker"] and words[1:3] in PRUNE_COMMANDS and prune_flags(words[3:]) + + +def prune_verdict(command: str) -> str: + """Returns: + "prune" for a plain_prune() line, run in process; "unguarded" for a + line whose text, with whitespace collapsed and with or without its + quotes, backslashes and backticks, would reach docker past the guard + shim (an absolute docker path, a PATH reset, or docker with a + privilege switch); else "run", through the shell with the shim first + on PATH. + """ + if plain_prune(command): + return "prune" + texts = {" ".join(text.split()) for text in (command, QUOTING.sub("", command))} + if absolute_docker(command) or any( + UNGUARDED.search(text) or ("docker" in text and PRIVILEGED.search(text)) for text in texts + ): + return "unguarded" + return "run" + + +def absolute_docker(command: str) -> bool: + """Returns: + Whether a simple command of the line, or of a shell's -c code in it, + runs a program named by a path ending in /docker or /docker-compose; + a path in an argument, such as /var/lib/docker, does not count. + """ + try: + parts = segments(command) + except ValueError: + return False + for part in parts: + run = program(part) + code = inline_code(run) + if code is not None and absolute_docker(code): + return True + if run and run[0].endswith(tuple(f"/{name}" for name in GUARDED_BINARIES)): + return True + return False + + +def docker_split(args: list[str]) -> tuple[list[str], list[str]]: + """Returns: + docker's global options with their values, and the subcommand with + its arguments. + """ + at = 0 + while at < len(args) and args[at].startswith("-"): + at += 2 if args[at] in DOCKER_GLOBALS else 1 + return args[:at], args[at:] + + +def guard_verdict(rest: list[str]) -> str | None: + """Returns: + "prune" for an image prune, the reason a removal is refused, or None + for a command docker runs as asked. + """ + if rest[:2] in PRUNE_COMMANDS: + return "prune" + for words, reason in REMOVALS.items(): + if tuple(rest[: len(words)]) == words: + return reason + if rest[:1] == ["compose"] and any(arg == "--rmi" or arg.startswith("--rmi=") for arg in rest): + return COMPOSE_RMI + if INNER_REMOVAL.search(" ".join(rest[1:])): + return REFUSALS["nested"] + return None + + +def cmd_guard(real: str, report: str, args: list[str]) -> int: + """Args: + real: the docker or docker-compose binary the shim stands in for. + report: the file the result lines are appended to. + args: its arguments as the shell expanded them. + + Returns: + 0 after a clean leftover step, 1 after one with failures, + GUARD_REFUSED for a refused removal; any other command replaces this + process with the real binary. + """ + name = os.path.basename(real) + options, rest = docker_split(args) if name == "docker" else ([], ["compose", *args]) + verdict = guard_verdict(rest) + if verdict is None: + os.execv(real, [real, *args]) + if verdict == "prune": + lines = remove_leftovers([real, *options]) + else: + lines = [f"refused ({verdict}): {shlex.join([name, *args])}".replace("\n", " ")] + print(f"docker guard: {lines[0]}", file=sys.stderr) + with open(report, "a", encoding="utf-8") as handle: + handle.writelines(f"{line}\n" for line in lines) + if verdict != "prune": + return GUARD_REFUSED + return 1 if cleanup_failures(lines) else 0 + + +def run_step(command: str | list[str], shell: bool, env: dict[str, str] | None = None) -> tuple[int | str, bytes, bytes]: + """Returns: + The exit code, or "timeout" or the OS error that kept the command from + running, and its standard output and standard error. + """ + try: + done = subprocess.run(command, shell=shell, cwd="/", env=env, capture_output=True, timeout=900, check=False) + except subprocess.TimeoutExpired: + return "timeout", b"", b"" + except OSError as error: + return error.strerror or type(error).__name__, b"", b"" + return done.returncode, done.stdout, done.stderr + + +def result_line(code: int | str, shown: str) -> str: + return f"{OK if code == 0 else f'failed ({code}): '}{shown}" + + +def shown(prefix: list[str], args: list[str]) -> str: + return shlex.join([os.path.basename(prefix[0]), *prefix[1:], *args]) + + +def remove_leftovers(prefix: list[str]) -> list[str]: + """Args: + prefix: the argv that reaches the daemon, up to and including its + docker binary and global options. + + Returns: + The result lines of listing the dangling images and of removing each + one, without force, whose repository and digest are both ; a + pull by digest keeps both and stays, an image a container uses is + `kept (in use)`. + """ + code, out, _ = run_step([*prefix, *LEFTOVERS], False) + lines = [result_line(code, shown(prefix, LEFTOVERS))] + rows = (row.split() for row in out.decode(errors="replace").splitlines()) if code == 0 else () + for image in dict.fromkeys(fields[0] for fields in rows if fields[1:] == [NONE, NONE] and IMAGE_ID.fullmatch(fields[0])): + code, _, err = run_step([*prefix, "rmi", image], False) + in_use = code != 0 and IN_USE.search(err.decode(errors="replace")) is not None + lines.append(KEPT + shown(prefix, ["rmi", image]) if in_use else result_line(code, shown(prefix, ["rmi", image]))) + return lines + + +def cleanup_failures(results: list[str]) -> list[str]: + return [line for line in results if not line.startswith((OK, KEPT))] + + +def guard_env(directory: Path, report: Path, binaries: dict[str, str | None]) -> dict[str, str]: + """Args: + binaries: the real path of each of GUARDED_BINARIES, None when absent. + + Returns: + The environment of the cleanup shell lines: per binary found, a shim + of its name in the private directory, first on PATH, that hands every + call to `watch.py guard`. + """ + env = dict(os.environ) + found = {name: real for name, real in binaries.items() if real is not None} + for name, real in found.items(): + shim = directory / name + quoted = {key: shlex.quote(value) for key, value in ( + ("python", sys.executable), ("watch", str(Path(__file__).resolve())), ("real", real), + ("report", str(report)), + )} + shim.write_text(SHIM.format(**quoted), encoding="utf-8") + shim.chmod(0o700) + if found: + env["PATH"] = os.pathsep.join(filter(None, (str(directory), env.get("PATH", "")))) + return env + + +def guarded(command: str, env: dict[str, str], report: Path) -> list[str]: + report.unlink(missing_ok=True) + code = run_step(command, True, env)[0] + return [result_line(code, command), *(report.read_text(encoding="utf-8").splitlines() if report.exists() else ())] + + +def worktree_prune(argv: list[str]) -> bool: + return ( + argv[:3] == ["docker", "compose", "-p"] and argv[4:6] == ["exec", "-T"] and len(argv) > 9 + and not argv[6].startswith("-") and argv[7] == "docker" and argv[8:10] in PRUNE_COMMANDS + and prune_flags(argv[10:]) + ) + + +def worktree_step(argv: list[str]) -> list[str]: + """Returns: + The result lines of one worktree command: the leftover step inside the + worktree's daemon for the exact `docker compose -p exec -T + docker image prune` form, a refusal for any other command + naming an image removal, else its own result. + """ + if worktree_prune(argv): + return remove_leftovers(argv[:8]) + text = " ".join(argv) + if WORKTREE_REMOVAL.search(text) or WORKTREE_REMOVAL.search(QUOTING.sub("", text)): + return [f"refused ({REFUSALS['nested']}): {shlex.join(argv)}"] + return [result_line(run_step(argv, False)[0], shlex.join(argv))] + + def run_cleanup(cfg: dict) -> list[str]: - commands = [(command, True) for command in cfg["cleanup"]] - for tree, project in running_worktrees(cfg): - if busy(tree, cfg["busy_processes"]): - continue - commands += [([arg.format(project=project) for arg in argv], False) for argv in cfg["worktree_cleanup"]] - results = [] - for command, shell in commands: - try: - code = subprocess.run(command, shell=shell, cwd="/", capture_output=True, timeout=900, check=False).returncode - except subprocess.TimeoutExpired: - code = "timeout" - results.append(f"{'ok' if code == 0 else f'failed ({code})'}: {command if shell else shlex.join(command)}") - return results + """Returns: + The result lines of the configured cleanup commands, every plain + `docker image prune` among them replaced by one remove_leftovers() in + the first one's place, every line prune_verdict() refuses reported as + `refused (): ` with none of it run and every other + line followed by the lines its guarded docker calls reported, then + those of the worktree commands. + """ + worktree_commands = [ + [arg.format(project=project) for arg in argv] + for tree, project in running_worktrees(cfg) + if not busy(tree, cfg["busy_processes"]) + for argv in cfg["worktree_cleanup"] + ] + binaries = {name: shutil.which(name) for name in GUARDED_BINARIES} + real = binaries["docker"] + results: list[str] = [] + pruned = False + with tempfile.TemporaryDirectory(prefix="kvb-docker-guard-") as directory: + report = Path(directory) / "report" + env = guard_env(Path(directory), report, binaries) + for command in cfg["cleanup"]: + verdict = prune_verdict(command) + if verdict == "run": + results += guarded(command, env, report) + elif verdict in REFUSALS: + results.append(f"refused ({REFUSALS[verdict]}): {command}") + elif not pruned: + pruned = True + results += remove_leftovers([real or "docker"]) + return results + [line for argv in worktree_commands for line in worktree_step(argv)] def cmd_cleanup(cfg: dict, args: argparse.Namespace) -> int: @@ -1231,7 +1525,7 @@ def cmd_cleanup(cfg: dict, args: argparse.Namespace) -> int: results = run_cleanup(cfg) if args.full else [] cache_gb = trim_caches(cfg) after, _ = disk(cfg) - failed = [r for r in results if not r.startswith("ok")] + failed = cleanup_failures(results) if args.full or cache_gb: alert( "cleanup", @@ -1261,6 +1555,8 @@ def cmd_status(cfg: dict) -> int: def main() -> int: + if sys.argv[1:2] == ["guard"] and len(sys.argv) > 3: + return cmd_guard(sys.argv[2], sys.argv[3], sys.argv[4:]) parser = argparse.ArgumentParser(prog="watch") sub = parser.add_subparsers(dest="cmd", required=True) sub.add_parser("watch") diff --git a/tests/test_watch.py b/tests/test_watch.py index b290f8f..bffe47e 100644 --- a/tests/test_watch.py +++ b/tests/test_watch.py @@ -7,12 +7,15 @@ import importlib.util import json import os import re +import shlex +import shutil import subprocess import sys import tempfile import time import unittest from pathlib import Path +from unittest import mock REPO_ROOT = Path(__file__).resolve().parents[1] WATCH = REPO_ROOT / "skills" / "kvb-org-organize" / "scripts" / "watch.py" @@ -21,6 +24,76 @@ OPEN_MEMORY = {"available": -1, "swap_free": -1, "psi_full": 1000, "container_sw SHUT_MEMORY = {"available": 101, "swap_free": 101, "psi_full": 0, "container_swap_gb": 0} MEMINFO = "MemTotal: 1000 kB\nMemAvailable: 120 kB\nSwapTotal: 2000 kB\nSwapFree: 300 kB\n" PSI = "some avg10=30.00 avg60=1.00 avg300=0.50 total=1\nfull avg10=12.50 avg60=0.80 avg300=0.10 total=1\n" +LISTED = ["images", "--filter", "dangling=true", "--format", "{{.ID}} {{.Repository}} {{.Digest}}"] +LISTING = ( + "aaaaaaaaaaaa \n" + f"bbbbbbbbbbbb ghcr.io/infinito-nexus/core/debian sha256:110c8487{'0' * 56}\n" + f"cccccccccccc quay.io/oauth2-proxy/oauth2-proxy sha256:{'c' * 64}\n" + "eeeeeeeeeeee quay.io/oauth2-proxy/oauth2-proxy \n" + "dddddddddddd \n" +) +REMOVED = [LISTED, ["rmi", "aaaaaaaaaaaa"], ["rmi", "dddddddddddd"]] +LEFTOVER_IDS = {"aaaaaaaaaaaa", "dddddddddddd"} +FORBIDDEN = (["image", "prune"], ["images", "prune"], ["system", "prune"], ["image", "rm"], ["image", "remove"]) +GUARDED = ( + "docker image pr\\\nune -f", "docker image \\\nprune -f", "docker image $'prune' -f", "docker image $'\\x70rune' -f", + "P=prune; docker image $P -f", "docker image ${X:-prune} -f", "docker image {prune,} -f", + "docker image remove $(docker image ls -qf dangling=true)", + "docker rmi $(docker images | grep '' | awk '{print $3}')", + "docker images | awk '// {print $3}' | xargs -r docker rmi", + "docker images -qf dangling=true | xargs -r docker image remove", "clean-images", + "echo ZG9ja2VyIGltYWdlIHBydW5lIC1m | base64 -d | sh", "docker IMAGE PRUNE -f", "docker images prune -f", + "docker image prune -f", "docker image prune -f # nightly", "docker image prune -f > /dev/null 2>&1", + "docker image prune -af --filter until=24h", 'docker image prune --filter "$(docker system prune -af)" -f', + "docker image prune -f --filter", "docker image prune --force=true", 'echo "$(docker image prune -f)"', + "echo `docker image prune -f`", "eval 'docker image prune -f'", "sh -c 'docker image prune -f'", + "docker image prune -f; docker builder prune -f", "docker system prune -af", + "docker rmi $(docker images -qf dangling=true)", "docker images -q --filter dangling=true | xargs docker rmi", + "docker image prune -f '", "docker compose -p stack-a exec -T infinito docker image prune -f", + "docker container prune -f --filter label=image-prune", "docker container prune -f --filter 'label=image prune'", +) +UNGUARDED = ( + "/usr/bin/docker image prune -f", "D=/usr/bin; $D/docker image prune -f", "sudo docker image prune -f", + "doas docker image prune -f", "su -c 'docker image prune -f'", "pkexec docker image prune -f", + "run0 docker image prune -f", "systemd-run docker image prune -f", "PATH=/usr/bin docker image prune -f", + "env -i docker image prune -f", "command -p docker image prune -f", +) +GLOBALS = {"-H", "--host", "-c", "--context"} +TOOLS = ("sh", "grep", "awk", "xargs", "base64", "env", "rm") +FAKE_DOCKER = """#!{python} +import json +import os +import sys +from pathlib import Path + +base = Path({base!r}) +with (base / "calls.jsonl").open("a", encoding="utf-8") as calls: + calls.write(json.dumps(sys.argv[1:]) + "\\n") +with (base / "hosts.jsonl").open("a", encoding="utf-8") as hosts: + hosts.write(json.dumps(os.environ.get("DOCKER_HOST", "")) + "\\n") +args = sys.argv[1:] +if "docker" in args: + args = args[args.index("docker") + 1 :] +while args and args[0].startswith("-"): + args = args[2:] if args[0] in {globals!r} else args[1:] + + +def listed(name): + return (base / name).exists() and args[1] in (base / name).read_text(encoding="utf-8").split() + + +if args[:1] == ["images"]: + if not (base / "images.txt").exists(): + sys.exit("Cannot connect to the Docker daemon") + sys.stdout.write((base / "images.txt").read_text(encoding="utf-8")) +elif args[:1] == ["rmi"] and listed("in-use.txt"): + sys.exit( + f"Error response from daemon: conflict: unable to delete {{args[1]}} (must be forced) - " + "image is being used by stopped container 0123456789ab" + ) +elif args[:1] == ["rmi"] and listed("gone.txt"): + sys.exit(f"Error response from daemon: No such image: {{args[1]}}") +""" def _load(): @@ -698,5 +771,249 @@ class TestWatchCommands(unittest.TestCase): self.assertEqual((result.returncode, result.stdout), (0, "")) +class TestImagePrune(unittest.TestCase): + def setUp(self): + self.watch = _load() + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.base = Path(tmp.name) + self.bin = self.base / "bin" + self.bin.mkdir() + fake = self.bin / "docker" + for name in ("docker", "docker-compose"): + fake = self.bin / name + fake.write_text(FAKE_DOCKER.format(python=sys.executable, base=str(self.base), globals=GLOBALS), encoding="utf-8") + fake.chmod(0o755) + for tool in TOOLS: + (self.bin / tool).symlink_to(shutil.which(tool, path="/usr/bin:/bin")) + script = self.bin / "clean-images" + script.write_text("#!/bin/sh\ndocker image prune -f\n", encoding="utf-8") + script.chmod(0o755) + (self.base / "images.txt").write_text(LISTING, encoding="utf-8") + path = mock.patch.dict(os.environ, {"PATH": str(self.bin)}) + path.start() + self.addCleanup(path.stop) + + def _cleanup(self, *commands: str, worktree: list[list[str]] | None = None, stacks: str = "true") -> list[str]: + for name in ("calls.jsonl", "hosts.jsonl"): + (self.base / name).unlink(missing_ok=True) + cfg = { + "cleanup": list(commands), "worktree_cleanup": worktree or [], "worktrees_command": stacks, + "worktree_marker": "Makefile", "busy_processes": [], + } + return self.watch.run_cleanup(cfg) + + def _read(self, name: str) -> list: + path = self.base / name + return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines()] if path.exists() else [] + + def _calls(self) -> list[list[str]]: + return self._read("calls.jsonl") + + def _removals(self) -> list[list[str]]: + found = [] + for call in self._calls(): + args = call[call.index("docker") + 1 :] if "docker" in call else call + while args and args[0].startswith("-"): + args = args[2:] if args[0] in GLOBALS else args[1:] + leftover = args[:1] == ["rmi"] and len(args) == 2 and args[1] in LEFTOVER_IDS + if args[:2] in FORBIDDEN or (args[:1] == ["rmi"] and not leftover): + found.append(call) + return found + + def test_full_cleanup_removes_build_leftovers_and_keeps_digest_pinned_images(self): + results = self._cleanup("docker builder prune -f", "docker image prune -f", "docker image prune -a -f") + self.assertEqual(self._calls(), [["builder", "prune", "-f"], *REMOVED]) + self.assertEqual(results, [ + "ok: docker builder prune -f", f"ok: {shlex.join(['docker', *LISTED])}", + "ok: docker rmi aaaaaaaaaaaa", "ok: docker rmi dddddddddddd", + ]) + + def test_only_a_plain_image_prune_maps_to_the_leftover_step(self): + for command in ( + "docker image prune -f", "docker image prune -a", "docker image prune --all --force --filter until=24h", + "docker images prune -af", " docker image prune -f --filter=label=x", "docker 'image' prune -f", + ): + with self.subTest(command=command): + self.assertEqual(self.watch.prune_verdict(command), "prune") + self._cleanup(command) + self.assertEqual(self._calls(), REMOVED) + for command in ( + "docker builder prune -f", "docker container prune -f --filter until=1h", "docker images", + "docker rmi abcdef123456", + ): + with self.subTest(command=command): + self.assertEqual(self.watch.prune_verdict(command), "run") + + def test_no_shell_line_gets_a_prune_or_removal_past_the_guard(self): + for command in GUARDED: + with self.subTest(command=command): + self._cleanup(command) + self.assertEqual(self._removals(), []) + results = self._cleanup("P=prune; docker image $P -f") + self.assertEqual(self._calls(), REMOVED) + self.assertEqual(results, [ + "ok: P=prune; docker image $P -f", f"ok: {shlex.join(['docker', *LISTED])}", + "ok: docker rmi aaaaaaaaaaaa", "ok: docker rmi dddddddddddd", + ]) + results = self._cleanup("docker system prune -af") + self.assertEqual(results, [ + f"failed ({self.watch.GUARD_REFUSED}): docker system prune -af", + f"refused ({self.watch.REMOVALS[('system', 'prune')]}): docker system prune -af", + ]) + self.assertEqual(self._calls(), []) + + def test_a_line_the_guard_cannot_see_is_refused_and_none_of_it_runs(self): + for command in (*UNGUARDED, f"{self.bin}/docker image prune -f"): + with self.subTest(command=command): + self.assertEqual(self.watch.prune_verdict(command), "unguarded") + command = f"{self.bin}/docker image prune -f" + self.assertEqual(self._cleanup(command), [f"refused ({self.watch.REFUSALS['unguarded']}): {command}"]) + self.assertEqual(self._calls(), []) + + def test_a_compose_down_with_rmi_is_refused_and_a_plain_down_runs(self): + for command in ( + "docker compose -p x down --rmi all", "docker compose -p x down --rmi local", + "docker compose -p x down --rmi=all", "docker-compose -p x down --rmi all", + "R=all; docker compose -p x down --rmi $R", + ): + with self.subTest(command=command): + results = self._cleanup(command) + self.assertEqual(self._calls(), []) + self.assertTrue(results[1].startswith(f"refused ({self.watch.COMPOSE_RMI}): "), results) + for command, call in ( + ("docker compose -p x down", ["compose", "-p", "x", "down"]), ("docker-compose -p x down", ["-p", "x", "down"]), + ): + with self.subTest(command=command): + self.assertEqual(self._cleanup(command), [f"ok: {command}"]) + self.assertEqual(self._calls(), [call]) + tree = self.base / "tree" + tree.mkdir() + (tree / "Makefile").write_text("", encoding="utf-8") + down = ["docker", "compose", "-p", "{project}", "down", "--rmi", "all"] + results = self._cleanup(worktree=[down], stacks=f"printf '%s\\t%s\\n' {tree} stack-a") + self.assertEqual(results, [f"refused ({self.watch.REFUSALS['nested']}): docker compose -p stack-a down --rmi all"]) + self.assertEqual(self._calls(), []) + + def test_only_a_docker_run_by_path_counts_as_unguarded(self): + leftover = self.base / "var" / "lib" / "docker" / "tmp" / "x" + leftover.mkdir(parents=True) + command = f"rm -rf {leftover}" + self.assertEqual(self._cleanup(command), [f"ok: {command}"]) + self.assertFalse(leftover.exists()) + for command, verdict in ( + ("du -sh /var/lib/docker", "run"), ("ls /usr/bin/docker", "run"), + ("curl --unix-socket /var/run/docker.sock http://localhost/images/json", "run"), + ("/usr/bin/docker image prune -f", "unguarded"), ("D=/usr/bin; $D/docker image prune -f", "unguarded"), + ("xargs /usr/bin/docker rmi", "unguarded"), ("sh -c '/usr/bin/docker image prune -f'", "unguarded"), + ("echo $(/usr/bin/docker image prune -f)", "unguarded"), ("env FOO=1 /usr/bin/docker image prune -f", "unguarded"), + ("/usr/local/bin/docker-compose -p x down --rmi all", "unguarded"), + ): + with self.subTest(command=command): + self.assertEqual(self.watch.prune_verdict(command), verdict) + + def test_a_host_or_context_prune_runs_the_leftover_step_on_that_daemon(self): + for command, options in ( + ("docker -H tcp://other:2375 image prune -f", ["-H", "tcp://other:2375"]), + ("docker --context remote image prune -af", ["--context", "remote"]), + ("env DOCKER_HOST=tcp://other:2375 docker image prune -f", []), + ): + with self.subTest(command=command): + self._cleanup(command) + self.assertEqual(self._calls(), [[*options, *call] for call in REMOVED]) + self.assertEqual(self._read("hosts.jsonl"), ["tcp://other:2375"] * 3) + + def test_the_live_worktree_entries_prune_leftovers_and_run_the_builder_prune_unchanged(self): + tree = self.base / "tree" + tree.mkdir() + (tree / "Makefile").write_text("", encoding="utf-8") + exec_prefix = ["docker", "compose", "-p", "{project}", "exec", "-T", "infinito", "docker"] + live = [ + [*exec_prefix, "image", "prune", "-f"], + [*exec_prefix, "builder", "prune", "-f", "--filter", "until=24h"], + ] + results = self._cleanup(worktree=live, stacks=f"printf '%s\\t%s\\n' {tree} stack-a") + prefix = ["compose", "-p", "stack-a", "exec", "-T", "infinito", "docker"] + builder = [*prefix, "builder", "prune", "-f", "--filter", "until=24h"] + self.assertEqual(self._calls(), [*([*prefix, *call] for call in REMOVED), builder]) + self.assertEqual(results, [ + *(f"ok: {shlex.join(['docker', *prefix, *call])}" for call in REMOVED), f"ok: {shlex.join(['docker', *builder])}", + ]) + + def test_the_live_worktree_prune_runs_the_leftover_step_inside_that_daemon(self): + tree = self.base / "tree" + tree.mkdir() + (tree / "Makefile").write_text("", encoding="utf-8") + live = ["docker", "compose", "-p", "{project}", "exec", "-T", "infinito", "docker", "image", "prune", "-f"] + inner = ["docker", "compose", "-p", "{project}", "exec", "-T", "infinito", "sh", "-c", "docker rmi abc"] + results = self._cleanup(worktree=[live, inner], stacks=f"printf '%s\\t%s\\n' {tree} stack-a") + prefix = ["compose", "-p", "stack-a", "exec", "-T", "infinito", "docker"] + self.assertEqual(self._calls(), [[*prefix, *call] for call in REMOVED]) + self.assertEqual(results, [ + f"ok: {shlex.join(['docker', *prefix, *LISTED])}", f"ok: {shlex.join(['docker', *prefix, *REMOVED[1]])}", + f"ok: {shlex.join(['docker', *prefix, *REMOVED[2]])}", + f"refused ({self.watch.REFUSALS['nested']}): {shlex.join([arg.format(project='stack-a') for arg in inner])}", + ]) + + def test_the_live_cleanup_entries_prune_leftovers_and_run_the_others_unchanged(self): + repo = self.base / "repo" + images = repo / "state" / "e2e" / "store-images" + images.mkdir(parents=True) + for name in ("store-api.tar", "store-web.tar"): + (images / name).write_bytes(b"x") + live = [ + "docker image prune -f", + "docker container prune -f --filter until=1h", + ( + 'if [ -z "$(docker ps -q --filter label=com.docker.compose.project=infinito-nexus-gui)" ]; then rm -f ' + f"{repo}/state/e2e/store-images/store-api.tar {repo}/state/e2e/store-images/store-web.tar; fi" + ), + ] + self.assertEqual([self.watch.prune_verdict(command) for command in live], ["prune", "run", "run"]) + results = self._cleanup(*live) + self.assertEqual(self._calls(), [ + *REMOVED, ["container", "prune", "-f", "--filter", "until=1h"], + ["ps", "-q", "--filter", "label=com.docker.compose.project=infinito-nexus-gui"], + ]) + self.assertEqual(results, [ + f"ok: {shlex.join(['docker', *LISTED])}", "ok: docker rmi aaaaaaaaaaaa", "ok: docker rmi dddddddddddd", + f"ok: {live[1]}", f"ok: {live[2]}", + ]) + self.assertEqual(os.listdir(images), []) + + def test_an_image_a_container_uses_is_kept_and_stays_out_of_the_alert(self): + (self.base / "in-use.txt").write_text("aaaaaaaaaaaa\n", encoding="utf-8") + (self.base / "gone.txt").write_text("dddddddddddd\n", encoding="utf-8") + results = self._cleanup("docker image prune -f", "docker builder prune -f") + self.assertEqual(results[1:], [ + "kept (in use): docker rmi aaaaaaaaaaaa", "failed (1): docker rmi dddddddddddd", "ok: docker builder prune -f", + ]) + self.assertEqual(self.watch.cleanup_failures(results), ["failed (1): docker rmi dddddddddddd"]) + held = "Error response from daemon: conflict: unable to delete f18bae17841e" + for err, in_use in ( + (f"{held} (must be forced) - image is being used by stopped container 3c9a\n", True), + (f"{held} (cannot be forced) - image is being used by running container 3c9a\n", True), + (f"{held} (cannot be forced) - image has dependent child images\n", False), + (f"{held} (must be forced) - image is referenced in multiple repositories\n", False), + (f"WARNING: runtime\n{held} (cannot be forced) - image has dependent child images\n", False), + (f"{held}: container store is locked\n", False), + ("Error response from daemon: No such image: f18bae17841e\n", False), + ): + with self.subTest(err=err): + self.assertEqual(self.watch.IN_USE.search(err) is not None, in_use) + + def test_an_unreachable_daemon_is_reported_and_removes_nothing(self): + (self.base / "images.txt").unlink() + results = self._cleanup("docker image prune -f", "docker builder prune -f") + self.assertEqual(results, [f"failed (1): {shlex.join(['docker', *LISTED])}", "ok: docker builder prune -f"]) + self.assertEqual(self._calls(), [LISTED, ["builder", "prune", "-f"]]) + + def test_a_missing_docker_binary_is_reported_not_raised(self): + (self.bin / "docker").unlink() + results = self._cleanup("docker image prune -f") + self.assertEqual(len(results), 1, results) + self.assertRegex(results[0], rf"^failed \(.+\): {re.escape(shlex.join(['docker', *LISTED]))}$") + + if __name__ == "__main__": unittest.main()