diff --git a/orchestrator/client/cleanup.py b/orchestrator/client/cleanup.py index 14cbbeb..8cb1e84 100755 --- a/orchestrator/client/cleanup.py +++ b/orchestrator/client/cleanup.py @@ -196,9 +196,9 @@ class Cleanup: reason = outcome if isinstance(outcome, str) and outcome else stderr.strip()[-500:] or f"pkgmgr reported no result for {item['branch']}" return {**item, "status": "failed", "reason": reason[:500], "code": code, "applied": applied} - def job(self, path: Path, now: float) -> dict: + def job(self, folder: int, path: Path, now: float) -> dict: try: - job = json.loads(path.read_text(encoding="utf-8")) + job = json.loads(verifier.read_regular(path, folder)) except (OSError, ValueError) as error: return {"id": path.stem, "finished": now, "items": [], "error": f"unreadable job: {error}"} items = job.get("items") if isinstance(job, dict) else None @@ -214,15 +214,10 @@ class Cleanup: def cycle(jobs: Path, results: Path, cleanup: Cleanup) -> int: - handled = 0 - for path in sorted(jobs.iterdir()): - if not JOB.fullmatch(path.name) or not path.is_file(): - path.unlink(missing_ok=True) - continue - verifier.write_private(results / path.name, cleanup.job(path, time.time())) - path.unlink(missing_ok=True) - handled += 1 - return handled + def run(folder: int, name: str) -> dict | None: + return cleanup.job(folder, Path(name), time.time()) if JOB.fullmatch(name) else None + + return verifier.drain(jobs, results, run) def main() -> int: @@ -232,7 +227,6 @@ def main() -> int: parser.add_argument(name, type=Path, required=True) args = parser.parse_args() trust = verifier.Trust(approvers=args.approvers, replay=args.replay, origin=signing.origin_of(args.port)) - args.results.mkdir(mode=0o700, parents=True, exist_ok=True) cycle(args.jobs, args.results, Cleanup(trust, args.tree, args.pkgmgr)) return 0 diff --git a/orchestrator/client/offices.py b/orchestrator/client/offices.py index bd06b17..b71b642 100755 --- a/orchestrator/client/offices.py +++ b/orchestrator/client/offices.py @@ -29,7 +29,6 @@ Returns: from __future__ import annotations import argparse -import errno import json import os import re @@ -47,39 +46,18 @@ JOB = re.compile(r"[0-9a-f]{16}\.json") SAFE_PATH = "/usr/local/bin:/usr/bin:/bin" CLAIM = "claim.json" CLAIM_SHAPE = (("session", str), ("runtime", str), ("beat", (int, float))) -DIRECTORY = os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | os.O_CLOEXEC -FILE = os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK | os.O_CLOEXEC -READ_LIMIT = 1 << 20 +DIRECTORY = verifier.FOLDER OUTPUT_LIMIT = 4000 TIMEOUT = 120 -def read_regular(name: str | Path, dir_fd: int | None) -> bytes: - """Args: - name: the file, relative to dir_fd or a path; a link, FIFO or other - non-regular file raises OSError, so does one over READ_LIMIT bytes. - dir_fd: the open directory name is relative to, or None. - - Returns: - The file's bytes. - """ - fd = os.open(name, FILE, dir_fd=dir_fd) - with open(fd, "rb") as handle: - if not stat.S_ISREG(os.fstat(fd).st_mode): - raise OSError(errno.EINVAL, "not a regular file", str(name)) - data = handle.read(READ_LIMIT + 1) - if len(data) > READ_LIMIT: - raise OSError(errno.EFBIG, f"larger than {READ_LIMIT} bytes", str(name)) - return data - - def claim_in(office_fd: int) -> dict | None: """Returns: The claim of the open office directory in the shape council.py reads, or None. """ try: - claim = json.loads(read_regular(CLAIM, office_fd)) + claim = json.loads(verifier.read_regular(CLAIM, office_fd)) except (OSError, ValueError, RecursionError): return None shaped = isinstance(claim, dict) and all(isinstance(claim.get(key), kind) for key, kind in CLAIM_SHAPE) @@ -268,9 +246,9 @@ class Offices: vacant = "; the office is vacant now" if item["action"] == "takeover" else "" return ended("failed", f"{output.strip()[-500:]}{vacant}", done.returncode, output) - def job(self, path: Path, now: float) -> dict: + def job(self, folder: int, path: Path, now: float) -> dict: try: - job = json.loads(read_regular(path, None)) + job = json.loads(verifier.read_regular(path, folder)) except (OSError, ValueError, RecursionError) as error: return {"id": path.stem, "finished": now, "items": [], "error": f"unreadable job: {error}"} items = job.get("items") if isinstance(job, dict) else None @@ -281,16 +259,10 @@ class Offices: def cycle(jobs: Path, results: Path, offices: Offices) -> int: - handled = 0 - for path in sorted(jobs.iterdir()): - if JOB.fullmatch(path.name): - verifier.write_private(results / path.name, offices.job(path, time.time())) - handled += 1 - if path.is_dir() and not path.is_symlink(): - shutil.rmtree(path) - else: - path.unlink(missing_ok=True) - return handled + def run(folder: int, name: str) -> dict | None: + return offices.job(folder, Path(name), time.time()) if JOB.fullmatch(name) else None + + return verifier.drain(jobs, results, run) def main() -> int: @@ -300,7 +272,6 @@ def main() -> int: parser.add_argument(name, type=Path, required=True) args = parser.parse_args() trust = verifier.Trust(approvers=args.approvers, replay=args.replay, origin=signing.origin_of(args.port)) - args.results.mkdir(mode=0o700, parents=True, exist_ok=True) cycle(args.jobs, args.results, Offices(trust, args.council, args.script, Path.home())) return 0 diff --git a/orchestrator/client/verifier.py b/orchestrator/client/verifier.py index 722c32b..62b2ab9 100644 --- a/orchestrator/client/verifier.py +++ b/orchestrator/client/verifier.py @@ -1,10 +1,16 @@ from __future__ import annotations +import contextlib import datetime +import errno import fcntl import json import os -import tempfile +import secrets +import shutil +import stat +import sys +from collections.abc import Callable, Iterator from pathlib import Path from typing import NamedTuple @@ -18,6 +24,10 @@ from signing import ( rule_allows, ) +FOLDER = os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | os.O_CLOEXEC +FILE = os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK | os.O_CLOEXEC +READ_LIMIT = 1 << 20 + class Trust(NamedTuple): approvers: Path @@ -32,20 +42,195 @@ def pinned(path: Path) -> list[dict]: return entries -def write_private(path: Path, data: dict) -> None: - path.parent.mkdir(mode=0o700, parents=True, exist_ok=True) - descriptor, name = tempfile.mkstemp(dir=path.parent, prefix=f".{path.name}-") +def write_at(folder: int, name: str, data: dict) -> None: + """Args: + folder: the open directory the file goes into. + name: the file's name there; a link of that name is replaced, never + followed. + data: written as JSON with mode 0600, by rename. + """ + temp = f".{name}-{secrets.token_hex(8)}" + descriptor = os.open(temp, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW | os.O_CLOEXEC, 0o600, dir_fd=folder) try: with os.fdopen(descriptor, "w", encoding="utf-8") as handle: json.dump(data, handle, indent=1, sort_keys=True) handle.flush() os.fsync(handle.fileno()) - os.replace(name, path) + os.replace(temp, name, src_dir_fd=folder, dst_dir_fd=folder) except OSError: - Path(name).unlink(missing_ok=True) + with contextlib.suppress(FileNotFoundError): + os.unlink(temp, dir_fd=folder) raise +def write_private(path: Path, data: dict) -> None: + path.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + folder = os.open(path.parent, os.O_RDONLY | os.O_DIRECTORY | os.O_CLOEXEC) + try: + write_at(folder, path.name, data) + finally: + os.close(folder) + + +def read_regular(name: str | Path, dir_fd: int | None) -> bytes: + """Args: + name: the file, relative to dir_fd or a path; a link, FIFO or other + non-regular file raises OSError, so does one over READ_LIMIT bytes. + dir_fd: the open directory name is relative to, or None. + + Returns: + The file's bytes. + """ + fd = os.open(name, FILE, dir_fd=dir_fd) + with open(fd, "rb") as handle: + if not stat.S_ISREG(os.fstat(fd).st_mode): + raise OSError(errno.EINVAL, "not a regular file", str(name)) + data = handle.read(READ_LIMIT + 1) + if len(data) > READ_LIMIT: + raise OSError(errno.EFBIG, f"larger than {READ_LIMIT} bytes", str(name)) + return data + + +def descend(folder: int, part: str, create: bool, follow: bool) -> int: + """Args: + folder: an open directory, closed once part is open. + part: one name in it. + create: make a missing part with mode 0700. + follow: follow part when it is a link. + + Returns: + The open directory part. + """ + try: + child = os.open(part, FOLDER, dir_fd=folder) + except FileNotFoundError: + if not create: + raise + with contextlib.suppress(FileExistsError): + os.mkdir(part, 0o700, dir_fd=folder) + child = os.open(part, FOLDER, dir_fd=folder) + except OSError as error: + if error.errno not in (errno.ELOOP, errno.ENOTDIR) or not follow: + raise + child = os.open(part, FOLDER & ~os.O_NOFOLLOW, dir_fd=folder) + os.close(folder) + return child + + +@contextlib.contextmanager +def private_dir(path: Path, create: bool) -> Iterator[int]: + """Args: + path: the directory, opened from / one name at a time by descend(); + a link is followed only by a process other than root, and only + while every directory entered so far is root's and no one else + can write it. + create: make missing directories with mode 0700. + + Returns: + The open directory, closed on exit. + + Raises: + Refused unless it is a directory of this effective uid that no one + else can write, reached without a link descend() did not follow. + """ + folder = os.open("/", FOLDER) + rooted = os.geteuid() != 0 + try: + for part in path.absolute().parts[1:]: + held = os.fstat(folder) + rooted = rooted and held.st_uid == 0 and not held.st_mode & 0o022 + folder = descend(folder, part, create, rooted) + found = os.fstat(folder) + except OSError as error: + os.close(folder) + raise Refused(f"{path}: {error.filename}: {error.strerror or error}") from error + if found.st_uid != os.geteuid() or found.st_mode & 0o022: + os.close(folder) + raise Refused(f"{path} is no directory of uid {os.geteuid()}, or others can write it") + try: + yield folder + finally: + os.close(folder) + + +def remove_at(folder: int, name: str) -> None: + """Args: + folder: the open directory. + name: the entry to remove: a real directory with all below it through + shutil.rmtree, which refuses a link swapped in; anything else as + the name itself, never what a link names. + """ + try: + if stat.S_ISDIR(os.lstat(name, dir_fd=folder).st_mode): + shutil.rmtree(name, dir_fd=folder) + else: + os.unlink(name, dir_fd=folder) + except FileNotFoundError: + return + + +def unreported(folder: int, out: int, name: str, result: dict, error: OSError) -> None: + """Args: + folder: the open jobs directory. + out: the open results directory. + name: the job whose result could not be written; it moves to + ..unreported in out, or is removed where that move fails, so + it never runs again. + result: its result, printed on stderr as one line. + error: why the result could not be written. + """ + kept = f".{Path(name).stem}.unreported" + print(f"no result for {name}: {error}; the job is kept as {kept} beside the results: {json.dumps(result, sort_keys=True)}", file=sys.stderr) + try: + os.rename(name, kept, src_dir_fd=folder, dst_dir_fd=out) + except OSError: + try: + remove_at(folder, name) + except OSError as stuck: + print(f"kept {name}: {stuck}", file=sys.stderr) + + +def drain(jobs: Path, results: Path, run: Callable[[int, str], dict | None]) -> int: + """Args: + jobs: the directory the hub writes jobs into. + results: the directory each result goes to under its job's name; + made when missing. + run: called with the open jobs directory and each entry's name, in + sorted order; returns the entry's result, or None for no job. + + Returns: + The number of results written. Every entry is removed after its + result, each write and removal relative to the open directories; a + result that cannot be written goes to unreported(), a removal that + fails prints one line on stderr, and the cycle goes on; a refused + directory changes nothing and prints one line. + """ + with contextlib.ExitStack() as stack: + try: + folder = stack.enter_context(private_dir(jobs, False)) + out = stack.enter_context(private_dir(results, True)) + except Refused as error: + print(f"refused, nothing done: {error}", file=sys.stderr) + return 0 + with os.scandir(folder) as entries: + names = sorted(entry.name for entry in entries) + handled = 0 + for name in names: + result = run(folder, name) + if result is not None: + try: + write_at(out, name, result) + handled += 1 + except OSError as error: + unreported(folder, out, name, result, error) + continue + try: + remove_at(folder, name) + except OSError as error: + print(f"kept {name}: {error}", file=sys.stderr) + return handled + + def pin(path: Path, entry: dict, now: float) -> list[dict]: stamp = datetime.datetime.fromtimestamp(now, datetime.UTC).isoformat(timespec="seconds") entries = [old for old in load_pinned(path) if old["id"] != entry["id"]] + [{**entry, "pinned": stamp}] diff --git a/tests/test_hub_cleanup.py b/tests/test_hub_cleanup.py index 634e2ca..7f6712a 100644 --- a/tests/test_hub_cleanup.py +++ b/tests/test_hub_cleanup.py @@ -175,8 +175,8 @@ class TestHostCleanup(unittest.TestCase): self.repo = base / "repos" / "demo" self.repo.mkdir(parents=True) self.jobs, self.results = base / "jobs", base / "results" - self.jobs.mkdir() - self.results.mkdir() + self.jobs.mkdir(mode=0o700) + self.results.mkdir(mode=0o700) self.fake = base / "bin" / "pkgmgr" self.fake.parent.mkdir() self.fake.write_text(FAKE_PKGMGR.format(python=sys.executable), encoding="utf-8") diff --git a/tests/test_hub_cleanup_token.py b/tests/test_hub_cleanup_token.py index 879e5b5..25ecb66 100644 --- a/tests/test_hub_cleanup_token.py +++ b/tests/test_hub_cleanup_token.py @@ -180,8 +180,8 @@ class TestHostCleanupWithoutToken(unittest.TestCase): self.git("symbolic-ref", "refs/remotes/origin/HEAD", "refs/remotes/origin/main") self.git("update-ref", "refs/remotes/origin/remote-merged", self.tip("remote-merged")) self.jobs, self.results = base / "jobs", base / "results" - self.jobs.mkdir() - self.results.mkdir() + self.jobs.mkdir(mode=0o700) + self.results.mkdir(mode=0o700) self.fake = base / "bin" / "pkgmgr" self.fake.parent.mkdir() self.fake.write_text(SCRATCH_PKGMGR.format(python=sys.executable), encoding="utf-8") diff --git a/tests/test_hub_offices.py b/tests/test_hub_offices.py index e6a140f..9d094b3 100644 --- a/tests/test_hub_offices.py +++ b/tests/test_hub_offices.py @@ -160,8 +160,8 @@ class TestHostOffices(unittest.TestCase): for sid, name in (("s1", "One"), ("s2", "Two")): (sessions / f"{sid}.json").write_text(json.dumps({"sessionId": sid, "name": name}), encoding="utf-8") self.jobs, self.results = base / "jobs", base / "results" - self.jobs.mkdir() - self.results.mkdir() + self.jobs.mkdir(mode=0o700) + self.results.mkdir(mode=0o700) self.approvers = base / "config" / "approvers.json" self.trust = verifier.Trust(approvers=self.approvers, replay=base / "replay" / "offices.json", origin=ORIGIN) self.signer = key(-7, "main") @@ -271,7 +271,7 @@ class TestHostOffices(unittest.TestCase): planted.write_text(json.dumps({"v": 1, "id": linked, "created": "x", "items": [appoint], "proof": self.proof([appoint], self.signer)}), encoding="utf-8") (self.jobs / f"{linked}.json").symlink_to(planted) os.mkfifo(self.jobs / f"{fifo}.json", 0o600) - (self.jobs / f"{large}.json").write_text(" " * (host.READ_LIMIT + 1), encoding="utf-8") + (self.jobs / f"{large}.json").write_text(" " * (verifier.READ_LIMIT + 1), encoding="utf-8") self.assertEqual(host.cycle(self.jobs, self.results, self.runner), 3) for job_id in (linked, fifo, large): with self.subTest(job=job_id): diff --git a/tests/test_hub_spool_links.py b/tests/test_hub_spool_links.py new file mode 100644 index 0000000..5aa6fb8 --- /dev/null +++ b/tests/test_hub_spool_links.py @@ -0,0 +1,270 @@ +from __future__ import annotations + +import contextlib +import errno +import hashlib +import importlib.util +import io +import json +import os +import shutil +import stat +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +from hub_support import HUB, ORIGIN, verifier + +JOB_ID = "0123456789abcdef" +JOB = f"{JOB_ID}.json" +OTHER_ID = "fedcba9876543210" +ME = os.getuid() + + +def load(name: str): + spec = importlib.util.spec_from_file_location(f"kvb_orchestrator_{name}_spool_links", HUB / "client" / f"{name}.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +offices = load("offices") +cleanup = load("cleanup") + + +def digest(root: Path) -> list[tuple[str, str, str]]: + found = [] + for folder, dirs, files in os.walk(root): + for name in dirs + files: + path = Path(folder) / name + mode = path.lstat().st_mode + if stat.S_ISLNK(mode): + content = f"link {os.readlink(path)}" + elif stat.S_ISREG(mode): + content = hashlib.sha256(path.read_bytes()).hexdigest() + else: + content = "dir" + found.append((str(path.relative_to(root)), oct(mode), content)) + return sorted(found) + + +def root_owned(*folders: Path, mine: bool = False): + """Args: + folders: the directories os.fstat reports as root's that no one else + can write. + mine: report every inode of this uid the same way. + """ + inodes = {(folder.stat().st_dev, folder.stat().st_ino) for folder in folders} + real = os.fstat + + def fstat(fd: int) -> os.stat_result: + found = real(fd) + if (found.st_dev, found.st_ino) not in inodes and not (mine and found.st_uid == ME): + return found + fields = list(found[:10]) + fields[0] &= ~0o022 + fields[4] = 0 + return os.stat_result(fields) + + return mock.patch("os.fstat", side_effect=fstat) + + +class SpoolLinks: + module = None + + def setUp(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + base = Path(tmp.name) + self.spool = base / "state" / "spool" + self.jobs, self.results = self.spool / "jobs", self.spool / "results" + self.victim = base / "victim" + for folder in (self.jobs, self.results, self.victim / "jobs" / "sub", self.victim / "results"): + folder.mkdir(mode=0o700, parents=True) + for path in (self.victim / "jobs" / JOB, self.victim / "jobs" / "keep.txt", self.victim / "jobs" / "sub" / "inner.txt", self.victim / "results" / JOB): + self.plant_job(path) + self.before = digest(self.victim) + trust = verifier.Trust(approvers=base / "approvers.json", replay=base / "replay.json", origin=ORIGIN) + self.runner = self.unit(trust, base) + + def plant_job(self, path: Path | None = None) -> None: + (path or self.jobs / JOB).write_text(json.dumps({"v": 1, "id": JOB_ID, "items": []}), encoding="utf-8") + + def cycle(self) -> tuple[int, list[str]]: + stderr = io.StringIO() + with contextlib.redirect_stderr(stderr): + handled = self.module.cycle(self.jobs, self.results, self.runner) + return handled, stderr.getvalue().splitlines() + + def assert_refused(self) -> None: + handled, lines = self.cycle() + self.assertEqual((handled, len(lines)), (0, 1), lines) + self.assertIn("refused, nothing done", lines[0]) + self.assertEqual(digest(self.victim), self.before) + + def test_a_linked_jobs_dir_is_refused(self): + self.jobs.rmdir() + self.jobs.symlink_to(self.victim / "jobs") + self.assert_refused() + self.assertEqual(list(self.results.iterdir()), []) + + def test_a_linked_results_dir_is_refused(self): + self.results.rmdir() + self.results.symlink_to(self.victim / "results") + self.plant_job() + self.assert_refused() + self.assertEqual([path.name for path in self.jobs.iterdir()], [JOB]) + + def test_a_linked_spool_dir_above_both_is_refused(self): + shutil.rmtree(self.spool) + self.spool.symlink_to(self.victim) + self.assert_refused() + + def test_a_jobs_dir_others_can_write_is_refused(self): + self.plant_job() + self.jobs.chmod(0o770) + self.assert_refused() + self.assertEqual([path.name for path in self.jobs.iterdir()], [JOB]) + + def test_a_linked_job_file_is_removed_as_the_link(self): + (self.jobs / JOB).symlink_to(self.victim / "jobs" / JOB) + (self.jobs / "keep.txt").symlink_to(self.victim / "jobs" / "keep.txt") + self.cycle() + self.assertEqual(list(self.jobs.iterdir()), []) + self.assertEqual(digest(self.victim), self.before) + self.assertIn("unreadable job", json.loads((self.results / JOB).read_text(encoding="utf-8"))["error"]) + + def test_a_subdir_that_is_or_holds_a_link_is_removed_without_following(self): + (self.jobs / "linked").symlink_to(self.victim / "jobs") + held = self.jobs / "sub" / "deeper" + held.mkdir(parents=True) + (self.jobs / "sub" / "dir").symlink_to(self.victim / "jobs") + (self.jobs / "sub" / "file").symlink_to(self.victim / "jobs" / "keep.txt") + (held / "victim").symlink_to(self.victim) + self.cycle() + self.assertEqual(list(self.jobs.iterdir()), []) + self.assertEqual(digest(self.victim), self.before) + + def test_spool_dirs_swapped_for_links_mid_cycle_change_nothing_outside(self): + self.plant_job() + (self.jobs / "keep.txt").write_text("spool", encoding="utf-8") + (self.jobs / "sub").mkdir() + moved = self.spool / "moved" + real = self.runner.job + + def swap(*args): + if not moved.exists(): + moved.mkdir() + for name in ("jobs", "results"): + (self.spool / name).rename(moved / name) + (self.spool / name).symlink_to(self.victim / name) + return real(*args) + + with mock.patch.object(self.runner, "job", side_effect=swap): + self.assertEqual(self.cycle()[0], 1) + self.assertEqual(digest(self.victim), self.before) + self.assertEqual((list((moved / "jobs").iterdir()), [path.name for path in (moved / "results").iterdir()]), ([], [JOB])) + + def test_a_subdir_swapped_for_a_link_before_its_removal_is_kept(self): + (self.jobs / "sub").mkdir() + (self.jobs / "sub" / "inner.txt").write_text("spool", encoding="utf-8") + real = shutil.rmtree + + def swap(path, *args, **kwargs): + if not (self.spool / "moved").exists(): + (self.jobs / "sub").rename(self.spool / "moved") + (self.jobs / "sub").symlink_to(self.victim / "jobs") + return real(path, *args, **kwargs) + + with mock.patch("shutil.rmtree", side_effect=swap): + handled, lines = self.cycle() + self.assertEqual(digest(self.victim), self.before) + self.assertEqual((handled, len(lines), (self.jobs / "sub").is_symlink()), (0, 1, True), lines) + + @unittest.skipIf(ME == 0, "a unit running as root follows no link") + def test_a_link_in_a_dir_only_root_can_write_is_followed_by_a_unit_of_another_uid(self): + real = self.spool.parent / "real" + self.spool.rename(real) + self.spool.symlink_to(real) + self.plant_job() + self.assert_refused() + with root_owned(self.spool.parent, *self.spool.parent.parents): + self.assertEqual(self.cycle(), (1, [])) + self.assertEqual(([path.name for path in (real / "jobs").iterdir()], [path.name for path in (real / "results").iterdir()]), ([], [JOB])) + + def test_a_unit_running_as_root_follows_no_link(self): + self.jobs.rmdir() + self.jobs.symlink_to(self.victim / "jobs") + with root_owned(self.spool, *self.spool.parents, mine=True), mock.patch("os.getuid", return_value=0), mock.patch("os.geteuid", return_value=0): + self.assert_refused() + + def test_a_link_in_a_root_dir_below_a_dir_of_another_uid_is_not_followed(self): + self.jobs.rmdir() + self.jobs.symlink_to(self.victim / "jobs") + with root_owned(self.spool, *self.spool.parent.parent.parents): + self.assert_refused() + + def plant_other(self) -> None: + (self.jobs / f"{OTHER_ID}.json").write_text(json.dumps({"v": 1, "id": OTHER_ID, "items": []}), encoding="utf-8") + + def test_a_result_that_cannot_be_written_keeps_its_job_beside_the_results(self): + self.plant_job() + self.plant_other() + (self.results / JOB).mkdir() + (self.results / JOB / "planted").write_text("hub", encoding="utf-8") + handled, lines = self.cycle() + self.assertEqual((handled, len(lines)), (1, 1), lines) + self.assertIn(f"no result for {JOB}", lines[0]) + self.assertIn("malformed job", lines[0]) + self.assertEqual(list(self.jobs.iterdir()), []) + self.assertEqual(sorted(path.name for path in self.results.iterdir()), [f".{JOB_ID}.unreported", JOB, f"{OTHER_ID}.json"]) + self.assertEqual([path.name for path in (self.results / JOB).iterdir()], ["planted"]) + self.assertEqual(json.loads((self.results / f".{JOB_ID}.unreported").read_text(encoding="utf-8"))["id"], JOB_ID) + + def test_a_full_disk_keeps_every_job_unreported_and_runs_none_again(self): + self.plant_job() + self.plant_other() + with mock.patch("os.fsync", side_effect=OSError(errno.ENOSPC, "No space left on device")): + handled, lines = self.cycle() + self.assertEqual((handled, len(lines)), (0, 2), lines) + self.assertEqual(sorted(path.name for path in self.results.iterdir()), [f".{JOB_ID}.unreported", f".{OTHER_ID}.unreported"]) + self.assertEqual(self.cycle(), (0, [])) + self.assertEqual(len(list(self.results.iterdir())), 2) + + def test_a_job_that_can_be_neither_kept_nor_removed_leaves_the_cycle_going(self): + self.plant_job() + self.plant_other() + for name in (JOB, f".{JOB_ID}.unreported"): + (self.results / name).mkdir() + (self.results / name / "planted").write_text("hub", encoding="utf-8") + real = verifier.remove_at + + def stuck(folder, name): + if name == JOB: + raise PermissionError(errno.EACCES, "Permission denied") + return real(folder, name) + + with mock.patch.object(verifier, "remove_at", side_effect=stuck): + handled, lines = self.cycle() + self.assertEqual((handled, len(lines)), (1, 2), lines) + self.assertIn(f"kept {JOB}", lines[1]) + self.assertTrue((self.results / f"{OTHER_ID}.json").is_file()) + + +class TestOfficesSpoolLinks(SpoolLinks, unittest.TestCase): + module = offices + + def unit(self, trust: verifier.Trust, base: Path): + return offices.Offices(trust, base / "council", base / "council.py", base / "home") + + +class TestCleanupSpoolLinks(SpoolLinks, unittest.TestCase): + module = cleanup + + def unit(self, trust: verifier.Trust, base: Path): + return cleanup.Cleanup(trust, base / "tree.json", base / "pkgmgr") + + +if __name__ == "__main__": + unittest.main()