Merge branch 'fix/client-jobs-dir-symlink' into unify/skills-20261112-night
This commit is contained in:
commit
93f002e681
7 files changed
+482
-62
No files matched your search
@@ -196,9 +196,9 @@ class Cleanup:
|
||||
reason = outcome if isinstance(outcome, str) and outcome else stderr.strip()[-500:] or f"pkgmgr reported no result for {item['branch']}"
|
||||
return {**item, "status": "failed", "reason": reason[:500], "code": code, "applied": applied}
|
||||
|
||||
def job(self, path: Path, now: float) -> dict:
|
||||
def job(self, folder: int, path: Path, now: float) -> dict:
|
||||
try:
|
||||
job = json.loads(path.read_text(encoding="utf-8"))
|
||||
job = json.loads(verifier.read_regular(path, folder))
|
||||
except (OSError, ValueError) as error:
|
||||
return {"id": path.stem, "finished": now, "items": [], "error": f"unreadable job: {error}"}
|
||||
items = job.get("items") if isinstance(job, dict) else None
|
||||
@@ -214,15 +214,10 @@ class Cleanup:
|
||||
|
||||
|
||||
def cycle(jobs: Path, results: Path, cleanup: Cleanup) -> int:
|
||||
handled = 0
|
||||
for path in sorted(jobs.iterdir()):
|
||||
if not JOB.fullmatch(path.name) or not path.is_file():
|
||||
path.unlink(missing_ok=True)
|
||||
continue
|
||||
verifier.write_private(results / path.name, cleanup.job(path, time.time()))
|
||||
path.unlink(missing_ok=True)
|
||||
handled += 1
|
||||
return handled
|
||||
def run(folder: int, name: str) -> dict | None:
|
||||
return cleanup.job(folder, Path(name), time.time()) if JOB.fullmatch(name) else None
|
||||
|
||||
return verifier.drain(jobs, results, run)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
@@ -232,7 +227,6 @@ def main() -> int:
|
||||
parser.add_argument(name, type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
trust = verifier.Trust(approvers=args.approvers, replay=args.replay, origin=signing.origin_of(args.port))
|
||||
args.results.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||
cycle(args.jobs, args.results, Cleanup(trust, args.tree, args.pkgmgr))
|
||||
return 0
|
||||
|
||||
|
||||
@@ -29,7 +29,6 @@ Returns:
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import errno
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
@@ -47,39 +46,18 @@ JOB = re.compile(r"[0-9a-f]{16}\.json")
|
||||
SAFE_PATH = "/usr/local/bin:/usr/bin:/bin"
|
||||
CLAIM = "claim.json"
|
||||
CLAIM_SHAPE = (("session", str), ("runtime", str), ("beat", (int, float)))
|
||||
DIRECTORY = os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | os.O_CLOEXEC
|
||||
FILE = os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK | os.O_CLOEXEC
|
||||
READ_LIMIT = 1 << 20
|
||||
DIRECTORY = verifier.FOLDER
|
||||
OUTPUT_LIMIT = 4000
|
||||
TIMEOUT = 120
|
||||
|
||||
|
||||
def read_regular(name: str | Path, dir_fd: int | None) -> bytes:
|
||||
"""Args:
|
||||
name: the file, relative to dir_fd or a path; a link, FIFO or other
|
||||
non-regular file raises OSError, so does one over READ_LIMIT bytes.
|
||||
dir_fd: the open directory name is relative to, or None.
|
||||
|
||||
Returns:
|
||||
The file's bytes.
|
||||
"""
|
||||
fd = os.open(name, FILE, dir_fd=dir_fd)
|
||||
with open(fd, "rb") as handle:
|
||||
if not stat.S_ISREG(os.fstat(fd).st_mode):
|
||||
raise OSError(errno.EINVAL, "not a regular file", str(name))
|
||||
data = handle.read(READ_LIMIT + 1)
|
||||
if len(data) > READ_LIMIT:
|
||||
raise OSError(errno.EFBIG, f"larger than {READ_LIMIT} bytes", str(name))
|
||||
return data
|
||||
|
||||
|
||||
def claim_in(office_fd: int) -> dict | None:
|
||||
"""Returns:
|
||||
The claim of the open office directory in the shape council.py
|
||||
reads, or None.
|
||||
"""
|
||||
try:
|
||||
claim = json.loads(read_regular(CLAIM, office_fd))
|
||||
claim = json.loads(verifier.read_regular(CLAIM, office_fd))
|
||||
except (OSError, ValueError, RecursionError):
|
||||
return None
|
||||
shaped = isinstance(claim, dict) and all(isinstance(claim.get(key), kind) for key, kind in CLAIM_SHAPE)
|
||||
@@ -268,9 +246,9 @@ class Offices:
|
||||
vacant = "; the office is vacant now" if item["action"] == "takeover" else ""
|
||||
return ended("failed", f"{output.strip()[-500:]}{vacant}", done.returncode, output)
|
||||
|
||||
def job(self, path: Path, now: float) -> dict:
|
||||
def job(self, folder: int, path: Path, now: float) -> dict:
|
||||
try:
|
||||
job = json.loads(read_regular(path, None))
|
||||
job = json.loads(verifier.read_regular(path, folder))
|
||||
except (OSError, ValueError, RecursionError) as error:
|
||||
return {"id": path.stem, "finished": now, "items": [], "error": f"unreadable job: {error}"}
|
||||
items = job.get("items") if isinstance(job, dict) else None
|
||||
@@ -281,16 +259,10 @@ class Offices:
|
||||
|
||||
|
||||
def cycle(jobs: Path, results: Path, offices: Offices) -> int:
|
||||
handled = 0
|
||||
for path in sorted(jobs.iterdir()):
|
||||
if JOB.fullmatch(path.name):
|
||||
verifier.write_private(results / path.name, offices.job(path, time.time()))
|
||||
handled += 1
|
||||
if path.is_dir() and not path.is_symlink():
|
||||
shutil.rmtree(path)
|
||||
else:
|
||||
path.unlink(missing_ok=True)
|
||||
return handled
|
||||
def run(folder: int, name: str) -> dict | None:
|
||||
return offices.job(folder, Path(name), time.time()) if JOB.fullmatch(name) else None
|
||||
|
||||
return verifier.drain(jobs, results, run)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
@@ -300,7 +272,6 @@ def main() -> int:
|
||||
parser.add_argument(name, type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
trust = verifier.Trust(approvers=args.approvers, replay=args.replay, origin=signing.origin_of(args.port))
|
||||
args.results.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||
cycle(args.jobs, args.results, Offices(trust, args.council, args.script, Path.home()))
|
||||
return 0
|
||||
|
||||
|
||||
@@ -1,10 +1,16 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import datetime
|
||||
import errno
|
||||
import fcntl
|
||||
import json
|
||||
import os
|
||||
import tempfile
|
||||
import secrets
|
||||
import shutil
|
||||
import stat
|
||||
import sys
|
||||
from collections.abc import Callable, Iterator
|
||||
from pathlib import Path
|
||||
from typing import NamedTuple
|
||||
|
||||
@@ -18,6 +24,10 @@ from signing import (
|
||||
rule_allows,
|
||||
)
|
||||
|
||||
FOLDER = os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | os.O_CLOEXEC
|
||||
FILE = os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK | os.O_CLOEXEC
|
||||
READ_LIMIT = 1 << 20
|
||||
|
||||
|
||||
class Trust(NamedTuple):
|
||||
approvers: Path
|
||||
@@ -32,20 +42,195 @@ def pinned(path: Path) -> list[dict]:
|
||||
return entries
|
||||
|
||||
|
||||
def write_private(path: Path, data: dict) -> None:
|
||||
path.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||
descriptor, name = tempfile.mkstemp(dir=path.parent, prefix=f".{path.name}-")
|
||||
def write_at(folder: int, name: str, data: dict) -> None:
|
||||
"""Args:
|
||||
folder: the open directory the file goes into.
|
||||
name: the file's name there; a link of that name is replaced, never
|
||||
followed.
|
||||
data: written as JSON with mode 0600, by rename.
|
||||
"""
|
||||
temp = f".{name}-{secrets.token_hex(8)}"
|
||||
descriptor = os.open(temp, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW | os.O_CLOEXEC, 0o600, dir_fd=folder)
|
||||
try:
|
||||
with os.fdopen(descriptor, "w", encoding="utf-8") as handle:
|
||||
json.dump(data, handle, indent=1, sort_keys=True)
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
os.replace(name, path)
|
||||
os.replace(temp, name, src_dir_fd=folder, dst_dir_fd=folder)
|
||||
except OSError:
|
||||
Path(name).unlink(missing_ok=True)
|
||||
with contextlib.suppress(FileNotFoundError):
|
||||
os.unlink(temp, dir_fd=folder)
|
||||
raise
|
||||
|
||||
|
||||
def write_private(path: Path, data: dict) -> None:
|
||||
path.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||
folder = os.open(path.parent, os.O_RDONLY | os.O_DIRECTORY | os.O_CLOEXEC)
|
||||
try:
|
||||
write_at(folder, path.name, data)
|
||||
finally:
|
||||
os.close(folder)
|
||||
|
||||
|
||||
def read_regular(name: str | Path, dir_fd: int | None) -> bytes:
|
||||
"""Args:
|
||||
name: the file, relative to dir_fd or a path; a link, FIFO or other
|
||||
non-regular file raises OSError, so does one over READ_LIMIT bytes.
|
||||
dir_fd: the open directory name is relative to, or None.
|
||||
|
||||
Returns:
|
||||
The file's bytes.
|
||||
"""
|
||||
fd = os.open(name, FILE, dir_fd=dir_fd)
|
||||
with open(fd, "rb") as handle:
|
||||
if not stat.S_ISREG(os.fstat(fd).st_mode):
|
||||
raise OSError(errno.EINVAL, "not a regular file", str(name))
|
||||
data = handle.read(READ_LIMIT + 1)
|
||||
if len(data) > READ_LIMIT:
|
||||
raise OSError(errno.EFBIG, f"larger than {READ_LIMIT} bytes", str(name))
|
||||
return data
|
||||
|
||||
|
||||
def descend(folder: int, part: str, create: bool, follow: bool) -> int:
|
||||
"""Args:
|
||||
folder: an open directory, closed once part is open.
|
||||
part: one name in it.
|
||||
create: make a missing part with mode 0700.
|
||||
follow: follow part when it is a link.
|
||||
|
||||
Returns:
|
||||
The open directory part.
|
||||
"""
|
||||
try:
|
||||
child = os.open(part, FOLDER, dir_fd=folder)
|
||||
except FileNotFoundError:
|
||||
if not create:
|
||||
raise
|
||||
with contextlib.suppress(FileExistsError):
|
||||
os.mkdir(part, 0o700, dir_fd=folder)
|
||||
child = os.open(part, FOLDER, dir_fd=folder)
|
||||
except OSError as error:
|
||||
if error.errno not in (errno.ELOOP, errno.ENOTDIR) or not follow:
|
||||
raise
|
||||
child = os.open(part, FOLDER & ~os.O_NOFOLLOW, dir_fd=folder)
|
||||
os.close(folder)
|
||||
return child
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def private_dir(path: Path, create: bool) -> Iterator[int]:
|
||||
"""Args:
|
||||
path: the directory, opened from / one name at a time by descend();
|
||||
a link is followed only by a process other than root, and only
|
||||
while every directory entered so far is root's and no one else
|
||||
can write it.
|
||||
create: make missing directories with mode 0700.
|
||||
|
||||
Returns:
|
||||
The open directory, closed on exit.
|
||||
|
||||
Raises:
|
||||
Refused unless it is a directory of this effective uid that no one
|
||||
else can write, reached without a link descend() did not follow.
|
||||
"""
|
||||
folder = os.open("/", FOLDER)
|
||||
rooted = os.geteuid() != 0
|
||||
try:
|
||||
for part in path.absolute().parts[1:]:
|
||||
held = os.fstat(folder)
|
||||
rooted = rooted and held.st_uid == 0 and not held.st_mode & 0o022
|
||||
folder = descend(folder, part, create, rooted)
|
||||
found = os.fstat(folder)
|
||||
except OSError as error:
|
||||
os.close(folder)
|
||||
raise Refused(f"{path}: {error.filename}: {error.strerror or error}") from error
|
||||
if found.st_uid != os.geteuid() or found.st_mode & 0o022:
|
||||
os.close(folder)
|
||||
raise Refused(f"{path} is no directory of uid {os.geteuid()}, or others can write it")
|
||||
try:
|
||||
yield folder
|
||||
finally:
|
||||
os.close(folder)
|
||||
|
||||
|
||||
def remove_at(folder: int, name: str) -> None:
|
||||
"""Args:
|
||||
folder: the open directory.
|
||||
name: the entry to remove: a real directory with all below it through
|
||||
shutil.rmtree, which refuses a link swapped in; anything else as
|
||||
the name itself, never what a link names.
|
||||
"""
|
||||
try:
|
||||
if stat.S_ISDIR(os.lstat(name, dir_fd=folder).st_mode):
|
||||
shutil.rmtree(name, dir_fd=folder)
|
||||
else:
|
||||
os.unlink(name, dir_fd=folder)
|
||||
except FileNotFoundError:
|
||||
return
|
||||
|
||||
|
||||
def unreported(folder: int, out: int, name: str, result: dict, error: OSError) -> None:
|
||||
"""Args:
|
||||
folder: the open jobs directory.
|
||||
out: the open results directory.
|
||||
name: the job whose result could not be written; it moves to
|
||||
.<id>.unreported in out, or is removed where that move fails, so
|
||||
it never runs again.
|
||||
result: its result, printed on stderr as one line.
|
||||
error: why the result could not be written.
|
||||
"""
|
||||
kept = f".{Path(name).stem}.unreported"
|
||||
print(f"no result for {name}: {error}; the job is kept as {kept} beside the results: {json.dumps(result, sort_keys=True)}", file=sys.stderr)
|
||||
try:
|
||||
os.rename(name, kept, src_dir_fd=folder, dst_dir_fd=out)
|
||||
except OSError:
|
||||
try:
|
||||
remove_at(folder, name)
|
||||
except OSError as stuck:
|
||||
print(f"kept {name}: {stuck}", file=sys.stderr)
|
||||
|
||||
|
||||
def drain(jobs: Path, results: Path, run: Callable[[int, str], dict | None]) -> int:
|
||||
"""Args:
|
||||
jobs: the directory the hub writes jobs into.
|
||||
results: the directory each result goes to under its job's name;
|
||||
made when missing.
|
||||
run: called with the open jobs directory and each entry's name, in
|
||||
sorted order; returns the entry's result, or None for no job.
|
||||
|
||||
Returns:
|
||||
The number of results written. Every entry is removed after its
|
||||
result, each write and removal relative to the open directories; a
|
||||
result that cannot be written goes to unreported(), a removal that
|
||||
fails prints one line on stderr, and the cycle goes on; a refused
|
||||
directory changes nothing and prints one line.
|
||||
"""
|
||||
with contextlib.ExitStack() as stack:
|
||||
try:
|
||||
folder = stack.enter_context(private_dir(jobs, False))
|
||||
out = stack.enter_context(private_dir(results, True))
|
||||
except Refused as error:
|
||||
print(f"refused, nothing done: {error}", file=sys.stderr)
|
||||
return 0
|
||||
with os.scandir(folder) as entries:
|
||||
names = sorted(entry.name for entry in entries)
|
||||
handled = 0
|
||||
for name in names:
|
||||
result = run(folder, name)
|
||||
if result is not None:
|
||||
try:
|
||||
write_at(out, name, result)
|
||||
handled += 1
|
||||
except OSError as error:
|
||||
unreported(folder, out, name, result, error)
|
||||
continue
|
||||
try:
|
||||
remove_at(folder, name)
|
||||
except OSError as error:
|
||||
print(f"kept {name}: {error}", file=sys.stderr)
|
||||
return handled
|
||||
|
||||
|
||||
def pin(path: Path, entry: dict, now: float) -> list[dict]:
|
||||
stamp = datetime.datetime.fromtimestamp(now, datetime.UTC).isoformat(timespec="seconds")
|
||||
entries = [old for old in load_pinned(path) if old["id"] != entry["id"]] + [{**entry, "pinned": stamp}]
|
||||
|
||||
@@ -175,8 +175,8 @@ class TestHostCleanup(unittest.TestCase):
|
||||
self.repo = base / "repos" / "demo"
|
||||
self.repo.mkdir(parents=True)
|
||||
self.jobs, self.results = base / "jobs", base / "results"
|
||||
self.jobs.mkdir()
|
||||
self.results.mkdir()
|
||||
self.jobs.mkdir(mode=0o700)
|
||||
self.results.mkdir(mode=0o700)
|
||||
self.fake = base / "bin" / "pkgmgr"
|
||||
self.fake.parent.mkdir()
|
||||
self.fake.write_text(FAKE_PKGMGR.format(python=sys.executable), encoding="utf-8")
|
||||
|
||||
@@ -180,8 +180,8 @@ class TestHostCleanupWithoutToken(unittest.TestCase):
|
||||
self.git("symbolic-ref", "refs/remotes/origin/HEAD", "refs/remotes/origin/main")
|
||||
self.git("update-ref", "refs/remotes/origin/remote-merged", self.tip("remote-merged"))
|
||||
self.jobs, self.results = base / "jobs", base / "results"
|
||||
self.jobs.mkdir()
|
||||
self.results.mkdir()
|
||||
self.jobs.mkdir(mode=0o700)
|
||||
self.results.mkdir(mode=0o700)
|
||||
self.fake = base / "bin" / "pkgmgr"
|
||||
self.fake.parent.mkdir()
|
||||
self.fake.write_text(SCRATCH_PKGMGR.format(python=sys.executable), encoding="utf-8")
|
||||
|
||||
@@ -160,8 +160,8 @@ class TestHostOffices(unittest.TestCase):
|
||||
for sid, name in (("s1", "One"), ("s2", "Two")):
|
||||
(sessions / f"{sid}.json").write_text(json.dumps({"sessionId": sid, "name": name}), encoding="utf-8")
|
||||
self.jobs, self.results = base / "jobs", base / "results"
|
||||
self.jobs.mkdir()
|
||||
self.results.mkdir()
|
||||
self.jobs.mkdir(mode=0o700)
|
||||
self.results.mkdir(mode=0o700)
|
||||
self.approvers = base / "config" / "approvers.json"
|
||||
self.trust = verifier.Trust(approvers=self.approvers, replay=base / "replay" / "offices.json", origin=ORIGIN)
|
||||
self.signer = key(-7, "main")
|
||||
@@ -271,7 +271,7 @@ class TestHostOffices(unittest.TestCase):
|
||||
planted.write_text(json.dumps({"v": 1, "id": linked, "created": "x", "items": [appoint], "proof": self.proof([appoint], self.signer)}), encoding="utf-8")
|
||||
(self.jobs / f"{linked}.json").symlink_to(planted)
|
||||
os.mkfifo(self.jobs / f"{fifo}.json", 0o600)
|
||||
(self.jobs / f"{large}.json").write_text(" " * (host.READ_LIMIT + 1), encoding="utf-8")
|
||||
(self.jobs / f"{large}.json").write_text(" " * (verifier.READ_LIMIT + 1), encoding="utf-8")
|
||||
self.assertEqual(host.cycle(self.jobs, self.results, self.runner), 3)
|
||||
for job_id in (linked, fifo, large):
|
||||
with self.subTest(job=job_id):
|
||||
|
||||
@@ -0,0 +1,270 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import errno
|
||||
import hashlib
|
||||
import importlib.util
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import stat
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest import mock
|
||||
|
||||
from hub_support import HUB, ORIGIN, verifier
|
||||
|
||||
JOB_ID = "0123456789abcdef"
|
||||
JOB = f"{JOB_ID}.json"
|
||||
OTHER_ID = "fedcba9876543210"
|
||||
ME = os.getuid()
|
||||
|
||||
|
||||
def load(name: str):
|
||||
spec = importlib.util.spec_from_file_location(f"kvb_orchestrator_{name}_spool_links", HUB / "client" / f"{name}.py")
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
offices = load("offices")
|
||||
cleanup = load("cleanup")
|
||||
|
||||
|
||||
def digest(root: Path) -> list[tuple[str, str, str]]:
|
||||
found = []
|
||||
for folder, dirs, files in os.walk(root):
|
||||
for name in dirs + files:
|
||||
path = Path(folder) / name
|
||||
mode = path.lstat().st_mode
|
||||
if stat.S_ISLNK(mode):
|
||||
content = f"link {os.readlink(path)}"
|
||||
elif stat.S_ISREG(mode):
|
||||
content = hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
else:
|
||||
content = "dir"
|
||||
found.append((str(path.relative_to(root)), oct(mode), content))
|
||||
return sorted(found)
|
||||
|
||||
|
||||
def root_owned(*folders: Path, mine: bool = False):
|
||||
"""Args:
|
||||
folders: the directories os.fstat reports as root's that no one else
|
||||
can write.
|
||||
mine: report every inode of this uid the same way.
|
||||
"""
|
||||
inodes = {(folder.stat().st_dev, folder.stat().st_ino) for folder in folders}
|
||||
real = os.fstat
|
||||
|
||||
def fstat(fd: int) -> os.stat_result:
|
||||
found = real(fd)
|
||||
if (found.st_dev, found.st_ino) not in inodes and not (mine and found.st_uid == ME):
|
||||
return found
|
||||
fields = list(found[:10])
|
||||
fields[0] &= ~0o022
|
||||
fields[4] = 0
|
||||
return os.stat_result(fields)
|
||||
|
||||
return mock.patch("os.fstat", side_effect=fstat)
|
||||
|
||||
|
||||
class SpoolLinks:
|
||||
module = None
|
||||
|
||||
def setUp(self):
|
||||
tmp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(tmp.cleanup)
|
||||
base = Path(tmp.name)
|
||||
self.spool = base / "state" / "spool"
|
||||
self.jobs, self.results = self.spool / "jobs", self.spool / "results"
|
||||
self.victim = base / "victim"
|
||||
for folder in (self.jobs, self.results, self.victim / "jobs" / "sub", self.victim / "results"):
|
||||
folder.mkdir(mode=0o700, parents=True)
|
||||
for path in (self.victim / "jobs" / JOB, self.victim / "jobs" / "keep.txt", self.victim / "jobs" / "sub" / "inner.txt", self.victim / "results" / JOB):
|
||||
self.plant_job(path)
|
||||
self.before = digest(self.victim)
|
||||
trust = verifier.Trust(approvers=base / "approvers.json", replay=base / "replay.json", origin=ORIGIN)
|
||||
self.runner = self.unit(trust, base)
|
||||
|
||||
def plant_job(self, path: Path | None = None) -> None:
|
||||
(path or self.jobs / JOB).write_text(json.dumps({"v": 1, "id": JOB_ID, "items": []}), encoding="utf-8")
|
||||
|
||||
def cycle(self) -> tuple[int, list[str]]:
|
||||
stderr = io.StringIO()
|
||||
with contextlib.redirect_stderr(stderr):
|
||||
handled = self.module.cycle(self.jobs, self.results, self.runner)
|
||||
return handled, stderr.getvalue().splitlines()
|
||||
|
||||
def assert_refused(self) -> None:
|
||||
handled, lines = self.cycle()
|
||||
self.assertEqual((handled, len(lines)), (0, 1), lines)
|
||||
self.assertIn("refused, nothing done", lines[0])
|
||||
self.assertEqual(digest(self.victim), self.before)
|
||||
|
||||
def test_a_linked_jobs_dir_is_refused(self):
|
||||
self.jobs.rmdir()
|
||||
self.jobs.symlink_to(self.victim / "jobs")
|
||||
self.assert_refused()
|
||||
self.assertEqual(list(self.results.iterdir()), [])
|
||||
|
||||
def test_a_linked_results_dir_is_refused(self):
|
||||
self.results.rmdir()
|
||||
self.results.symlink_to(self.victim / "results")
|
||||
self.plant_job()
|
||||
self.assert_refused()
|
||||
self.assertEqual([path.name for path in self.jobs.iterdir()], [JOB])
|
||||
|
||||
def test_a_linked_spool_dir_above_both_is_refused(self):
|
||||
shutil.rmtree(self.spool)
|
||||
self.spool.symlink_to(self.victim)
|
||||
self.assert_refused()
|
||||
|
||||
def test_a_jobs_dir_others_can_write_is_refused(self):
|
||||
self.plant_job()
|
||||
self.jobs.chmod(0o770)
|
||||
self.assert_refused()
|
||||
self.assertEqual([path.name for path in self.jobs.iterdir()], [JOB])
|
||||
|
||||
def test_a_linked_job_file_is_removed_as_the_link(self):
|
||||
(self.jobs / JOB).symlink_to(self.victim / "jobs" / JOB)
|
||||
(self.jobs / "keep.txt").symlink_to(self.victim / "jobs" / "keep.txt")
|
||||
self.cycle()
|
||||
self.assertEqual(list(self.jobs.iterdir()), [])
|
||||
self.assertEqual(digest(self.victim), self.before)
|
||||
self.assertIn("unreadable job", json.loads((self.results / JOB).read_text(encoding="utf-8"))["error"])
|
||||
|
||||
def test_a_subdir_that_is_or_holds_a_link_is_removed_without_following(self):
|
||||
(self.jobs / "linked").symlink_to(self.victim / "jobs")
|
||||
held = self.jobs / "sub" / "deeper"
|
||||
held.mkdir(parents=True)
|
||||
(self.jobs / "sub" / "dir").symlink_to(self.victim / "jobs")
|
||||
(self.jobs / "sub" / "file").symlink_to(self.victim / "jobs" / "keep.txt")
|
||||
(held / "victim").symlink_to(self.victim)
|
||||
self.cycle()
|
||||
self.assertEqual(list(self.jobs.iterdir()), [])
|
||||
self.assertEqual(digest(self.victim), self.before)
|
||||
|
||||
def test_spool_dirs_swapped_for_links_mid_cycle_change_nothing_outside(self):
|
||||
self.plant_job()
|
||||
(self.jobs / "keep.txt").write_text("spool", encoding="utf-8")
|
||||
(self.jobs / "sub").mkdir()
|
||||
moved = self.spool / "moved"
|
||||
real = self.runner.job
|
||||
|
||||
def swap(*args):
|
||||
if not moved.exists():
|
||||
moved.mkdir()
|
||||
for name in ("jobs", "results"):
|
||||
(self.spool / name).rename(moved / name)
|
||||
(self.spool / name).symlink_to(self.victim / name)
|
||||
return real(*args)
|
||||
|
||||
with mock.patch.object(self.runner, "job", side_effect=swap):
|
||||
self.assertEqual(self.cycle()[0], 1)
|
||||
self.assertEqual(digest(self.victim), self.before)
|
||||
self.assertEqual((list((moved / "jobs").iterdir()), [path.name for path in (moved / "results").iterdir()]), ([], [JOB]))
|
||||
|
||||
def test_a_subdir_swapped_for_a_link_before_its_removal_is_kept(self):
|
||||
(self.jobs / "sub").mkdir()
|
||||
(self.jobs / "sub" / "inner.txt").write_text("spool", encoding="utf-8")
|
||||
real = shutil.rmtree
|
||||
|
||||
def swap(path, *args, **kwargs):
|
||||
if not (self.spool / "moved").exists():
|
||||
(self.jobs / "sub").rename(self.spool / "moved")
|
||||
(self.jobs / "sub").symlink_to(self.victim / "jobs")
|
||||
return real(path, *args, **kwargs)
|
||||
|
||||
with mock.patch("shutil.rmtree", side_effect=swap):
|
||||
handled, lines = self.cycle()
|
||||
self.assertEqual(digest(self.victim), self.before)
|
||||
self.assertEqual((handled, len(lines), (self.jobs / "sub").is_symlink()), (0, 1, True), lines)
|
||||
|
||||
@unittest.skipIf(ME == 0, "a unit running as root follows no link")
|
||||
def test_a_link_in_a_dir_only_root_can_write_is_followed_by_a_unit_of_another_uid(self):
|
||||
real = self.spool.parent / "real"
|
||||
self.spool.rename(real)
|
||||
self.spool.symlink_to(real)
|
||||
self.plant_job()
|
||||
self.assert_refused()
|
||||
with root_owned(self.spool.parent, *self.spool.parent.parents):
|
||||
self.assertEqual(self.cycle(), (1, []))
|
||||
self.assertEqual(([path.name for path in (real / "jobs").iterdir()], [path.name for path in (real / "results").iterdir()]), ([], [JOB]))
|
||||
|
||||
def test_a_unit_running_as_root_follows_no_link(self):
|
||||
self.jobs.rmdir()
|
||||
self.jobs.symlink_to(self.victim / "jobs")
|
||||
with root_owned(self.spool, *self.spool.parents, mine=True), mock.patch("os.getuid", return_value=0), mock.patch("os.geteuid", return_value=0):
|
||||
self.assert_refused()
|
||||
|
||||
def test_a_link_in_a_root_dir_below_a_dir_of_another_uid_is_not_followed(self):
|
||||
self.jobs.rmdir()
|
||||
self.jobs.symlink_to(self.victim / "jobs")
|
||||
with root_owned(self.spool, *self.spool.parent.parent.parents):
|
||||
self.assert_refused()
|
||||
|
||||
def plant_other(self) -> None:
|
||||
(self.jobs / f"{OTHER_ID}.json").write_text(json.dumps({"v": 1, "id": OTHER_ID, "items": []}), encoding="utf-8")
|
||||
|
||||
def test_a_result_that_cannot_be_written_keeps_its_job_beside_the_results(self):
|
||||
self.plant_job()
|
||||
self.plant_other()
|
||||
(self.results / JOB).mkdir()
|
||||
(self.results / JOB / "planted").write_text("hub", encoding="utf-8")
|
||||
handled, lines = self.cycle()
|
||||
self.assertEqual((handled, len(lines)), (1, 1), lines)
|
||||
self.assertIn(f"no result for {JOB}", lines[0])
|
||||
self.assertIn("malformed job", lines[0])
|
||||
self.assertEqual(list(self.jobs.iterdir()), [])
|
||||
self.assertEqual(sorted(path.name for path in self.results.iterdir()), [f".{JOB_ID}.unreported", JOB, f"{OTHER_ID}.json"])
|
||||
self.assertEqual([path.name for path in (self.results / JOB).iterdir()], ["planted"])
|
||||
self.assertEqual(json.loads((self.results / f".{JOB_ID}.unreported").read_text(encoding="utf-8"))["id"], JOB_ID)
|
||||
|
||||
def test_a_full_disk_keeps_every_job_unreported_and_runs_none_again(self):
|
||||
self.plant_job()
|
||||
self.plant_other()
|
||||
with mock.patch("os.fsync", side_effect=OSError(errno.ENOSPC, "No space left on device")):
|
||||
handled, lines = self.cycle()
|
||||
self.assertEqual((handled, len(lines)), (0, 2), lines)
|
||||
self.assertEqual(sorted(path.name for path in self.results.iterdir()), [f".{JOB_ID}.unreported", f".{OTHER_ID}.unreported"])
|
||||
self.assertEqual(self.cycle(), (0, []))
|
||||
self.assertEqual(len(list(self.results.iterdir())), 2)
|
||||
|
||||
def test_a_job_that_can_be_neither_kept_nor_removed_leaves_the_cycle_going(self):
|
||||
self.plant_job()
|
||||
self.plant_other()
|
||||
for name in (JOB, f".{JOB_ID}.unreported"):
|
||||
(self.results / name).mkdir()
|
||||
(self.results / name / "planted").write_text("hub", encoding="utf-8")
|
||||
real = verifier.remove_at
|
||||
|
||||
def stuck(folder, name):
|
||||
if name == JOB:
|
||||
raise PermissionError(errno.EACCES, "Permission denied")
|
||||
return real(folder, name)
|
||||
|
||||
with mock.patch.object(verifier, "remove_at", side_effect=stuck):
|
||||
handled, lines = self.cycle()
|
||||
self.assertEqual((handled, len(lines)), (1, 2), lines)
|
||||
self.assertIn(f"kept {JOB}", lines[1])
|
||||
self.assertTrue((self.results / f"{OTHER_ID}.json").is_file())
|
||||
|
||||
|
||||
class TestOfficesSpoolLinks(SpoolLinks, unittest.TestCase):
|
||||
module = offices
|
||||
|
||||
def unit(self, trust: verifier.Trust, base: Path):
|
||||
return offices.Offices(trust, base / "council", base / "council.py", base / "home")
|
||||
|
||||
|
||||
class TestCleanupSpoolLinks(SpoolLinks, unittest.TestCase):
|
||||
module = cleanup
|
||||
|
||||
def unit(self, trust: verifier.Trust, base: Path):
|
||||
return cleanup.Cleanup(trust, base / "tree.json", base / "pkgmgr")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in new issue
Block a user