Merge branch 'fix/client-jobs-dir-symlink' into unify/skills-20261112-night
This commit is contained in:
commit
93f002e681
7 files changed
+482
-62
No files matched your search
@@ -175,8 +175,8 @@ class TestHostCleanup(unittest.TestCase):
|
||||
self.repo = base / "repos" / "demo"
|
||||
self.repo.mkdir(parents=True)
|
||||
self.jobs, self.results = base / "jobs", base / "results"
|
||||
self.jobs.mkdir()
|
||||
self.results.mkdir()
|
||||
self.jobs.mkdir(mode=0o700)
|
||||
self.results.mkdir(mode=0o700)
|
||||
self.fake = base / "bin" / "pkgmgr"
|
||||
self.fake.parent.mkdir()
|
||||
self.fake.write_text(FAKE_PKGMGR.format(python=sys.executable), encoding="utf-8")
|
||||
|
||||
@@ -180,8 +180,8 @@ class TestHostCleanupWithoutToken(unittest.TestCase):
|
||||
self.git("symbolic-ref", "refs/remotes/origin/HEAD", "refs/remotes/origin/main")
|
||||
self.git("update-ref", "refs/remotes/origin/remote-merged", self.tip("remote-merged"))
|
||||
self.jobs, self.results = base / "jobs", base / "results"
|
||||
self.jobs.mkdir()
|
||||
self.results.mkdir()
|
||||
self.jobs.mkdir(mode=0o700)
|
||||
self.results.mkdir(mode=0o700)
|
||||
self.fake = base / "bin" / "pkgmgr"
|
||||
self.fake.parent.mkdir()
|
||||
self.fake.write_text(SCRATCH_PKGMGR.format(python=sys.executable), encoding="utf-8")
|
||||
|
||||
@@ -160,8 +160,8 @@ class TestHostOffices(unittest.TestCase):
|
||||
for sid, name in (("s1", "One"), ("s2", "Two")):
|
||||
(sessions / f"{sid}.json").write_text(json.dumps({"sessionId": sid, "name": name}), encoding="utf-8")
|
||||
self.jobs, self.results = base / "jobs", base / "results"
|
||||
self.jobs.mkdir()
|
||||
self.results.mkdir()
|
||||
self.jobs.mkdir(mode=0o700)
|
||||
self.results.mkdir(mode=0o700)
|
||||
self.approvers = base / "config" / "approvers.json"
|
||||
self.trust = verifier.Trust(approvers=self.approvers, replay=base / "replay" / "offices.json", origin=ORIGIN)
|
||||
self.signer = key(-7, "main")
|
||||
@@ -271,7 +271,7 @@ class TestHostOffices(unittest.TestCase):
|
||||
planted.write_text(json.dumps({"v": 1, "id": linked, "created": "x", "items": [appoint], "proof": self.proof([appoint], self.signer)}), encoding="utf-8")
|
||||
(self.jobs / f"{linked}.json").symlink_to(planted)
|
||||
os.mkfifo(self.jobs / f"{fifo}.json", 0o600)
|
||||
(self.jobs / f"{large}.json").write_text(" " * (host.READ_LIMIT + 1), encoding="utf-8")
|
||||
(self.jobs / f"{large}.json").write_text(" " * (verifier.READ_LIMIT + 1), encoding="utf-8")
|
||||
self.assertEqual(host.cycle(self.jobs, self.results, self.runner), 3)
|
||||
for job_id in (linked, fifo, large):
|
||||
with self.subTest(job=job_id):
|
||||
|
||||
@@ -0,0 +1,270 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import errno
|
||||
import hashlib
|
||||
import importlib.util
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import stat
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest import mock
|
||||
|
||||
from hub_support import HUB, ORIGIN, verifier
|
||||
|
||||
JOB_ID = "0123456789abcdef"
|
||||
JOB = f"{JOB_ID}.json"
|
||||
OTHER_ID = "fedcba9876543210"
|
||||
ME = os.getuid()
|
||||
|
||||
|
||||
def load(name: str):
|
||||
spec = importlib.util.spec_from_file_location(f"kvb_orchestrator_{name}_spool_links", HUB / "client" / f"{name}.py")
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
offices = load("offices")
|
||||
cleanup = load("cleanup")
|
||||
|
||||
|
||||
def digest(root: Path) -> list[tuple[str, str, str]]:
|
||||
found = []
|
||||
for folder, dirs, files in os.walk(root):
|
||||
for name in dirs + files:
|
||||
path = Path(folder) / name
|
||||
mode = path.lstat().st_mode
|
||||
if stat.S_ISLNK(mode):
|
||||
content = f"link {os.readlink(path)}"
|
||||
elif stat.S_ISREG(mode):
|
||||
content = hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
else:
|
||||
content = "dir"
|
||||
found.append((str(path.relative_to(root)), oct(mode), content))
|
||||
return sorted(found)
|
||||
|
||||
|
||||
def root_owned(*folders: Path, mine: bool = False):
|
||||
"""Args:
|
||||
folders: the directories os.fstat reports as root's that no one else
|
||||
can write.
|
||||
mine: report every inode of this uid the same way.
|
||||
"""
|
||||
inodes = {(folder.stat().st_dev, folder.stat().st_ino) for folder in folders}
|
||||
real = os.fstat
|
||||
|
||||
def fstat(fd: int) -> os.stat_result:
|
||||
found = real(fd)
|
||||
if (found.st_dev, found.st_ino) not in inodes and not (mine and found.st_uid == ME):
|
||||
return found
|
||||
fields = list(found[:10])
|
||||
fields[0] &= ~0o022
|
||||
fields[4] = 0
|
||||
return os.stat_result(fields)
|
||||
|
||||
return mock.patch("os.fstat", side_effect=fstat)
|
||||
|
||||
|
||||
class SpoolLinks:
|
||||
module = None
|
||||
|
||||
def setUp(self):
|
||||
tmp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(tmp.cleanup)
|
||||
base = Path(tmp.name)
|
||||
self.spool = base / "state" / "spool"
|
||||
self.jobs, self.results = self.spool / "jobs", self.spool / "results"
|
||||
self.victim = base / "victim"
|
||||
for folder in (self.jobs, self.results, self.victim / "jobs" / "sub", self.victim / "results"):
|
||||
folder.mkdir(mode=0o700, parents=True)
|
||||
for path in (self.victim / "jobs" / JOB, self.victim / "jobs" / "keep.txt", self.victim / "jobs" / "sub" / "inner.txt", self.victim / "results" / JOB):
|
||||
self.plant_job(path)
|
||||
self.before = digest(self.victim)
|
||||
trust = verifier.Trust(approvers=base / "approvers.json", replay=base / "replay.json", origin=ORIGIN)
|
||||
self.runner = self.unit(trust, base)
|
||||
|
||||
def plant_job(self, path: Path | None = None) -> None:
|
||||
(path or self.jobs / JOB).write_text(json.dumps({"v": 1, "id": JOB_ID, "items": []}), encoding="utf-8")
|
||||
|
||||
def cycle(self) -> tuple[int, list[str]]:
|
||||
stderr = io.StringIO()
|
||||
with contextlib.redirect_stderr(stderr):
|
||||
handled = self.module.cycle(self.jobs, self.results, self.runner)
|
||||
return handled, stderr.getvalue().splitlines()
|
||||
|
||||
def assert_refused(self) -> None:
|
||||
handled, lines = self.cycle()
|
||||
self.assertEqual((handled, len(lines)), (0, 1), lines)
|
||||
self.assertIn("refused, nothing done", lines[0])
|
||||
self.assertEqual(digest(self.victim), self.before)
|
||||
|
||||
def test_a_linked_jobs_dir_is_refused(self):
|
||||
self.jobs.rmdir()
|
||||
self.jobs.symlink_to(self.victim / "jobs")
|
||||
self.assert_refused()
|
||||
self.assertEqual(list(self.results.iterdir()), [])
|
||||
|
||||
def test_a_linked_results_dir_is_refused(self):
|
||||
self.results.rmdir()
|
||||
self.results.symlink_to(self.victim / "results")
|
||||
self.plant_job()
|
||||
self.assert_refused()
|
||||
self.assertEqual([path.name for path in self.jobs.iterdir()], [JOB])
|
||||
|
||||
def test_a_linked_spool_dir_above_both_is_refused(self):
|
||||
shutil.rmtree(self.spool)
|
||||
self.spool.symlink_to(self.victim)
|
||||
self.assert_refused()
|
||||
|
||||
def test_a_jobs_dir_others_can_write_is_refused(self):
|
||||
self.plant_job()
|
||||
self.jobs.chmod(0o770)
|
||||
self.assert_refused()
|
||||
self.assertEqual([path.name for path in self.jobs.iterdir()], [JOB])
|
||||
|
||||
def test_a_linked_job_file_is_removed_as_the_link(self):
|
||||
(self.jobs / JOB).symlink_to(self.victim / "jobs" / JOB)
|
||||
(self.jobs / "keep.txt").symlink_to(self.victim / "jobs" / "keep.txt")
|
||||
self.cycle()
|
||||
self.assertEqual(list(self.jobs.iterdir()), [])
|
||||
self.assertEqual(digest(self.victim), self.before)
|
||||
self.assertIn("unreadable job", json.loads((self.results / JOB).read_text(encoding="utf-8"))["error"])
|
||||
|
||||
def test_a_subdir_that_is_or_holds_a_link_is_removed_without_following(self):
|
||||
(self.jobs / "linked").symlink_to(self.victim / "jobs")
|
||||
held = self.jobs / "sub" / "deeper"
|
||||
held.mkdir(parents=True)
|
||||
(self.jobs / "sub" / "dir").symlink_to(self.victim / "jobs")
|
||||
(self.jobs / "sub" / "file").symlink_to(self.victim / "jobs" / "keep.txt")
|
||||
(held / "victim").symlink_to(self.victim)
|
||||
self.cycle()
|
||||
self.assertEqual(list(self.jobs.iterdir()), [])
|
||||
self.assertEqual(digest(self.victim), self.before)
|
||||
|
||||
def test_spool_dirs_swapped_for_links_mid_cycle_change_nothing_outside(self):
|
||||
self.plant_job()
|
||||
(self.jobs / "keep.txt").write_text("spool", encoding="utf-8")
|
||||
(self.jobs / "sub").mkdir()
|
||||
moved = self.spool / "moved"
|
||||
real = self.runner.job
|
||||
|
||||
def swap(*args):
|
||||
if not moved.exists():
|
||||
moved.mkdir()
|
||||
for name in ("jobs", "results"):
|
||||
(self.spool / name).rename(moved / name)
|
||||
(self.spool / name).symlink_to(self.victim / name)
|
||||
return real(*args)
|
||||
|
||||
with mock.patch.object(self.runner, "job", side_effect=swap):
|
||||
self.assertEqual(self.cycle()[0], 1)
|
||||
self.assertEqual(digest(self.victim), self.before)
|
||||
self.assertEqual((list((moved / "jobs").iterdir()), [path.name for path in (moved / "results").iterdir()]), ([], [JOB]))
|
||||
|
||||
def test_a_subdir_swapped_for_a_link_before_its_removal_is_kept(self):
|
||||
(self.jobs / "sub").mkdir()
|
||||
(self.jobs / "sub" / "inner.txt").write_text("spool", encoding="utf-8")
|
||||
real = shutil.rmtree
|
||||
|
||||
def swap(path, *args, **kwargs):
|
||||
if not (self.spool / "moved").exists():
|
||||
(self.jobs / "sub").rename(self.spool / "moved")
|
||||
(self.jobs / "sub").symlink_to(self.victim / "jobs")
|
||||
return real(path, *args, **kwargs)
|
||||
|
||||
with mock.patch("shutil.rmtree", side_effect=swap):
|
||||
handled, lines = self.cycle()
|
||||
self.assertEqual(digest(self.victim), self.before)
|
||||
self.assertEqual((handled, len(lines), (self.jobs / "sub").is_symlink()), (0, 1, True), lines)
|
||||
|
||||
@unittest.skipIf(ME == 0, "a unit running as root follows no link")
|
||||
def test_a_link_in_a_dir_only_root_can_write_is_followed_by_a_unit_of_another_uid(self):
|
||||
real = self.spool.parent / "real"
|
||||
self.spool.rename(real)
|
||||
self.spool.symlink_to(real)
|
||||
self.plant_job()
|
||||
self.assert_refused()
|
||||
with root_owned(self.spool.parent, *self.spool.parent.parents):
|
||||
self.assertEqual(self.cycle(), (1, []))
|
||||
self.assertEqual(([path.name for path in (real / "jobs").iterdir()], [path.name for path in (real / "results").iterdir()]), ([], [JOB]))
|
||||
|
||||
def test_a_unit_running_as_root_follows_no_link(self):
|
||||
self.jobs.rmdir()
|
||||
self.jobs.symlink_to(self.victim / "jobs")
|
||||
with root_owned(self.spool, *self.spool.parents, mine=True), mock.patch("os.getuid", return_value=0), mock.patch("os.geteuid", return_value=0):
|
||||
self.assert_refused()
|
||||
|
||||
def test_a_link_in_a_root_dir_below_a_dir_of_another_uid_is_not_followed(self):
|
||||
self.jobs.rmdir()
|
||||
self.jobs.symlink_to(self.victim / "jobs")
|
||||
with root_owned(self.spool, *self.spool.parent.parent.parents):
|
||||
self.assert_refused()
|
||||
|
||||
def plant_other(self) -> None:
|
||||
(self.jobs / f"{OTHER_ID}.json").write_text(json.dumps({"v": 1, "id": OTHER_ID, "items": []}), encoding="utf-8")
|
||||
|
||||
def test_a_result_that_cannot_be_written_keeps_its_job_beside_the_results(self):
|
||||
self.plant_job()
|
||||
self.plant_other()
|
||||
(self.results / JOB).mkdir()
|
||||
(self.results / JOB / "planted").write_text("hub", encoding="utf-8")
|
||||
handled, lines = self.cycle()
|
||||
self.assertEqual((handled, len(lines)), (1, 1), lines)
|
||||
self.assertIn(f"no result for {JOB}", lines[0])
|
||||
self.assertIn("malformed job", lines[0])
|
||||
self.assertEqual(list(self.jobs.iterdir()), [])
|
||||
self.assertEqual(sorted(path.name for path in self.results.iterdir()), [f".{JOB_ID}.unreported", JOB, f"{OTHER_ID}.json"])
|
||||
self.assertEqual([path.name for path in (self.results / JOB).iterdir()], ["planted"])
|
||||
self.assertEqual(json.loads((self.results / f".{JOB_ID}.unreported").read_text(encoding="utf-8"))["id"], JOB_ID)
|
||||
|
||||
def test_a_full_disk_keeps_every_job_unreported_and_runs_none_again(self):
|
||||
self.plant_job()
|
||||
self.plant_other()
|
||||
with mock.patch("os.fsync", side_effect=OSError(errno.ENOSPC, "No space left on device")):
|
||||
handled, lines = self.cycle()
|
||||
self.assertEqual((handled, len(lines)), (0, 2), lines)
|
||||
self.assertEqual(sorted(path.name for path in self.results.iterdir()), [f".{JOB_ID}.unreported", f".{OTHER_ID}.unreported"])
|
||||
self.assertEqual(self.cycle(), (0, []))
|
||||
self.assertEqual(len(list(self.results.iterdir())), 2)
|
||||
|
||||
def test_a_job_that_can_be_neither_kept_nor_removed_leaves_the_cycle_going(self):
|
||||
self.plant_job()
|
||||
self.plant_other()
|
||||
for name in (JOB, f".{JOB_ID}.unreported"):
|
||||
(self.results / name).mkdir()
|
||||
(self.results / name / "planted").write_text("hub", encoding="utf-8")
|
||||
real = verifier.remove_at
|
||||
|
||||
def stuck(folder, name):
|
||||
if name == JOB:
|
||||
raise PermissionError(errno.EACCES, "Permission denied")
|
||||
return real(folder, name)
|
||||
|
||||
with mock.patch.object(verifier, "remove_at", side_effect=stuck):
|
||||
handled, lines = self.cycle()
|
||||
self.assertEqual((handled, len(lines)), (1, 2), lines)
|
||||
self.assertIn(f"kept {JOB}", lines[1])
|
||||
self.assertTrue((self.results / f"{OTHER_ID}.json").is_file())
|
||||
|
||||
|
||||
class TestOfficesSpoolLinks(SpoolLinks, unittest.TestCase):
|
||||
module = offices
|
||||
|
||||
def unit(self, trust: verifier.Trust, base: Path):
|
||||
return offices.Offices(trust, base / "council", base / "council.py", base / "home")
|
||||
|
||||
|
||||
class TestCleanupSpoolLinks(SpoolLinks, unittest.TestCase):
|
||||
module = cleanup
|
||||
|
||||
def unit(self, trust: verifier.Trust, base: Path):
|
||||
return cleanup.Cleanup(trust, base / "tree.json", base / "pkgmgr")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in new issue
Block a user