diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..4baa6b7 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,22 @@ +# Agent Instructions + +## Scope against the skills repository + +package-manager holds deterministic code that an operator can run on its own, +without an agent: repository, branch, worktree, release, packaging and +development tooling. Agent coordination (claims, messages, queues, scores, +prompts, session data) belongs to the +[skills repository](https://github.com/kevinveenbirkenbach/skills), whose skills +stay thin wrappers that call `pkgmgr`. + +- A new helper that a human could sensibly run alone goes here as a `pkgmgr` + command, with its tests, never as a script inside a skill. +- Output that an agent filters further (claims, busy agents, scores) is + emitted as `--json`; the filtering stays in the skill. +- When the split of a helper is unclear, the operator decides. + +## Working copy + +The installed `pkgmgr` may run straight from the main checkout, so a +half-edited tree breaks it for everyone. Edit only in a worktree +(`pkgmgr worktree up `), never in the main checkout. diff --git a/README.md b/README.md index 6a02dab..fd08045 100644 --- a/README.md +++ b/README.md @@ -70,6 +70,43 @@ versioning features it can drive **end-to-end workflows**: --- +### Worktrees and garbage collection + +| Command | Does | +| --- | --- | +| `pkgmgr worktree up [--base REF] [--path DIR]` | Checks the branch out under `////` or at `--path`, assigns a free slot and runs the pinned `worktree-init` hook | +| `pkgmgr worktree cleanup [--apply]` | Removes clean worktrees whose commits are merged or pushed: pinned `worktree-fini` hook, compose stack, checkout; the branch stays | +| `pkgmgr branch cleanup [--apply]` | Removes merged branches that no worktree holds, locally and on the remotes | +| `pkgmgr branch tree [--format json\|yaml] [--output FILE]` | Writes every managed repository's branches and worktrees with their cleanup state, without fetching | +| `pkgmgr gc [--all] [--apply]` | Both cleanups in one pass, for the current repository or every managed one | +| `pkgmgr worktree orphans --stale-days N --top N` | Lists merged and abandoned branches and worktrees with a proposal each | + +A hook runs only when the `worktree_hooks` config pins its command under the +`//` of a repository managed in the pkgmgr +config, and only for that repository's directory; origin's URL, which the +repository's own config sets, never selects a pin. `{path}` and `{slot}` are +replaced; a +`worktree-init` or `worktree-fini` target in the repository's Makefile is +reported, never run, because an agent may have written it. The pinned program +must be an absolute path or a root-owned command on a `PATH` entry that root +owns and no group or other user can write, up to `/`, so a binary planted in a +user-writable directory such as a virtualenv never runs. A bare name runs as +the resolved file that was checked. Inside a user namespace other than the +initial one, root means the overflow uid, under which the real root's files +show there, as do those of every other unmapped owner. Outside such a +namespace a directory on a nosuid mount, where udisks2 and FUSE put user +mounts, never counts as root's. The hook runs with +stdin closed, only those `PATH` entries and a 30 minute timeout. Removal walks to +the worktree without following a symbolic link and skips a worktree path that +is not its own realpath. A pinned command that reads repository files, such as `make` or a +script in the repository, trusts the worktree's content, tracked or ignored. + +Every cleanup is a dry run until `--apply`, and `--json` prints the plan for +other tools. The [agent skills](https://github.com/kevinveenbirkenbach/skills) +wrap these commands; see [AGENTS.md](AGENTS.md) for the split. + +--- + ## Architecture & Setup Map 🗺️ The following diagram gives a full overview of: diff --git a/src/pkgmgr/actions/branch/__init__.py b/src/pkgmgr/actions/branch/__init__.py index 73cf94d..9016441 100644 --- a/src/pkgmgr/actions/branch/__init__.py +++ b/src/pkgmgr/actions/branch/__init__.py @@ -4,14 +4,14 @@ Public API for branch actions. from .close_branch import close_branch from .drop_branch import drop_branch -from .gc import branch_gc, branch_purge +from .gc import branch_purge, cleanup from .open_branch import open_branch from .tree import branch_tree, refresh_tree_entry __all__ = [ - "branch_gc", "branch_purge", "branch_tree", + "cleanup", "close_branch", "drop_branch", "open_branch", diff --git a/src/pkgmgr/actions/branch/gc.py b/src/pkgmgr/actions/branch/gc.py index 2a4ab95..05a23ba 100644 --- a/src/pkgmgr/actions/branch/gc.py +++ b/src/pkgmgr/actions/branch/gc.py @@ -3,25 +3,33 @@ from __future__ import annotations import json import os import re -import shutil import subprocess import sys import time from collections.abc import Callable from dataclasses import asdict, dataclass, field +from pkgmgr.actions.worktree.hooks import FINI_HOOK, hook_command, run_hook from pkgmgr.core.git.commands import delete_local_branch from pkgmgr.core.git.errors import GitRunError -from pkgmgr.core.git.queries import list_remotes, resolve_base_branch -from pkgmgr.core.git.run import run +from pkgmgr.core.git.queries import ( + FILTER_CONFIG, + list_remotes, + local_config, + resolve_base_branch, +) +from pkgmgr.core.git.run import END_OF_OPTIONS, run from .gc_forge import ForgeFacts, fetch_forge_facts +from .nofollow import canonical, remove_tree +Registrations = tuple[str, dict[str, list[str]]] Snapshot = tuple[ dict[str, str], dict[str, dict[str, str]], dict[int, str], dict[str, tuple[str | None, int | None, int | None, bool]], + Registrations, ] # PROC_PID_INIT_INO: only the initial PID namespace sees every process of the host. @@ -31,11 +39,8 @@ COMPOSE_PROJECT = "com.docker.compose.project" COMPOSE_WORKING_DIR = "com.docker.compose.project.working_dir" LOCK_PID = re.compile(r"\bpid (\d+)") TRACK = re.compile(r"(ahead|behind) (\d+)") -UNSAFE_LOCAL_CONFIG = re.compile(r"^(local|worktree)\t(filter\.\S+)") GITLINK = "160000" -# ref and remote names come from repositories an agent may write: one named -# like --output= would otherwise make rev-list create that file -END_OF_OPTIONS = "--end-of-options" +STATUS_PATH_FIELD = {"1": 8, "2": 9, "u": 10, "?": 1} ALWAYS_PROTECTED = ("main", "master") LABELS = { "merged-clean": "DELETE", @@ -48,6 +53,8 @@ REFUSAL = ( "Refusing: not in the host PID namespace (e.g. a sandbox), so running " "processes and worktree locks cannot be verified. Run it on the host." ) +SCOPES = ("worktree", "branch", "all", "purge") +NOT_LINKED = "not a linked worktree of this repository" @dataclass @@ -87,8 +94,12 @@ class Item: compose_projects: list[str] = field(default_factory=list) images: list[str] = field(default_factory=list) state: str = "" + merged: bool = False + dirty: int = 0 merged_by: str | None = None merged_into: str | None = None + unpushed: int | None = None + last_activity: int | None = None size_bytes: int | None = None skip: str | None = None actions: list[str] = field(default_factory=list) @@ -97,7 +108,7 @@ class Item: @property def eligible(self) -> bool: - return self.state == "merged-clean" and self.skip is None and bool(self.actions) + return self.skip is None and bool(self.actions) def matches(self, name: str) -> bool: return name == self.branch or ( @@ -119,6 +130,9 @@ class Repo: lines: dict[str, tuple[str, set[str]]] sizes: bool min_age: float | None + scope: str + activity: bool + hooks: dict[str, list[str]] warnings: list[str] = field(default_factory=list) items: list[Item] = field(default_factory=list) @@ -178,6 +192,54 @@ def _inside(path: str, root: str) -> bool: return path == root or path.startswith(root.rstrip("/") + "/") +def _registrations(repo_root: str) -> Registrations: + base = os.path.join(_common_dir(repo_root), "worktrees") + claims: dict[str, list[str]] = {} + for entry in sorted(os.listdir(base)) if os.path.isdir(base) else []: + entry_dir = os.path.join(base, entry) + target = _pointer(os.path.join(entry_dir, "gitdir"), entry_dir) + if target: + claims.setdefault(target, []).append(entry) + return base, claims + + +def _linked_meta(registrations: Registrations, path: str) -> str | None: + """ + Resolve a worktree's metadata directory through its `.git` file. + + Args: + registrations: `/worktrees` and, per gitdir target, the + registrations naming it, from _registrations. + path: worktree path as `git worktree list` derives it from the + writable `/worktrees//gitdir`, so possibly forged. + + Returns: + `/worktrees/` when path, the registration and + `/worktrees` are canonical, path's `.git` resolves to that + registration and it is the only registration whose gitdir names + `/.git`, else None. + """ + base, claims = registrations + if not (canonical(path) and canonical(base)): + return None + git_file = os.path.join(path, ".git") + meta = _pointer(git_file, path) + if meta is None or os.path.dirname(meta) != base or not canonical(meta): + return None + return meta if claims.get(git_file) == [os.path.basename(meta)] else None + + +def _pointer(file: str, relative_to: str) -> str | None: + if not os.path.isfile(file): + return None + try: + with open(file, encoding="utf-8") as fh: + target = fh.read().strip().removeprefix("gitdir: ") + except (OSError, ValueError): + return None + return os.path.normpath(os.path.join(relative_to, target)) + + def _worktrees(repo_root: str) -> list[Worktree]: trees: list[Worktree] = [] fields: dict[str, str] = {} @@ -304,7 +366,19 @@ def _merged_by(repo: Repo, sha: str) -> str | None: if number: return f"pr#{number}" line = _in_line(repo, sha) - return f"line:{line}" if line else None + if line: + return f"line:{line}" + return "patch" if _patch_merged(repo, sha) else None + + +def _patch_merged(repo: Repo, sha: str) -> bool: + cherry = _git(["cherry", END_OF_OPTIONS, repo.default, sha], repo.repo).splitlines() + if not cherry or any(not line.startswith("- ") for line in cherry): + return False + outside = set( + _git(["rev-list", END_OF_OPTIONS, f"{repo.default}..{sha}"], repo.repo).split() + ) + return outside == {line[2:] for line in cherry} def _du_bytes(path: str) -> int | None: @@ -314,17 +388,24 @@ def _du_bytes(path: str) -> int | None: return int(out[0]) * 1024 if out and out[0].isdigit() else None -def _age_hours(repo: Repo, shas: list[str], tree: Worktree | None) -> float: +def _last_activity( + repo: Repo, shas: list[str], tree: Worktree | None, changed: list[str] +) -> int: stamps = [int(_git(["log", "-1", "--format=%ct", sha], repo.repo)) for sha in shas] if tree and os.path.isfile(os.path.join(tree.path, ".git")): with open(os.path.join(tree.path, ".git"), encoding="utf-8") as fh: index = os.path.join(fh.read().strip().removeprefix("gitdir: "), "index") if os.path.isfile(index): stamps.append(int(os.path.getmtime(index))) - return (time.time() - max(stamps)) / 3600 + for path in changed: + try: + stamps.append(int(os.lstat(os.path.join(tree.path, path)).st_mtime)) + except OSError: + continue + return max(stamps) -def _human(size: int) -> str: +def human_size(size: int) -> str: value = float(size) for unit in ("K", "M", "G"): value /= 1024 @@ -356,20 +437,12 @@ def _skip( lock = _lock_skip(tree.locked, repo.host_ns) if lock: return lock - busy = sorted(pid for pid, cwd in cwds.items() if _inside(cwd, tree.path)) - if busy: - return f"process: pid {busy[0]} ({_comm(busy[0])})" - return None + return _busy(cwds, tree.path) -def _unsafe_config(path: str) -> list[str]: - try: - out = _git(["config", "--show-scope", "--get-regexp", r"^filter\."], path) - except GitRunError as exc: - if exc.returncode == 1: - return [] - raise - return [m.group(2) for m in map(UNSAFE_LOCAL_CONFIG.match, out.splitlines()) if m] +def _busy(cwds: dict[int, str], path: str) -> str | None: + busy = sorted(pid for pid, cwd in cwds.items() if _inside(cwd, path)) + return f"process: pid {busy[0]} ({_comm(busy[0])})" if busy else None def _has_submodules(path: str) -> bool: @@ -379,39 +452,59 @@ def _has_submodules(path: str) -> bool: ) -def _status(path: str) -> tuple[int, int, int, int]: - lines = _git( - ["status", "--porcelain=v2", "--untracked-files=all"], path - ).splitlines() - codes = [line.split(" ", 2)[1] for line in lines if line[:1] in "12u"] +def _status(path: str) -> tuple[list[str], int, int, int]: + records: list[str] = [] + paths: list[str] = [] + fields = iter( + _git(["status", "--porcelain=v2", "-z", "--untracked-files=all"], path).split( + "\0" + ) + ) + for record in fields: + kind = record[:1] + if kind not in STATUS_PATH_FIELD: + continue + records.append(record) + paths.append(record.split(" ", STATUS_PATH_FIELD[kind])[-1]) + if kind == "2": + next(fields, None) + codes = [record.split(" ", 2)[1] for record in records if record[:1] in "12u"] return ( - len(lines), + paths, sum(code[0] != "." for code in codes), sum(code[1] != "." for code in codes), - sum(line.startswith("? ") for line in lines), + sum(record.startswith("? ") for record in records), ) -def _read_worktree(item: Item, checkout: bool) -> tuple[int, str | None]: +def _read_worktree(item: Item, checkout: bool) -> tuple[int, str | None, list[str]]: if item.worktree is None: - return 0, None + return 0, None, [] if not checkout: item.worktree_state = "missing" - return 0, None + return 0, None, [] try: if _has_submodules(item.worktree): item.worktree_state = "changed" - return -1, "status skipped: submodules, whose config the scan cannot vet" - unsafe = _unsafe_config(item.worktree) + return ( + -1, + "status skipped: submodules, whose config the scan cannot vet", + [], + ) + unsafe = local_config(item.worktree, FILTER_CONFIG) if unsafe: item.worktree_state = "changed" - return -1, f"status skipped: {', '.join(unsafe)} in the repository config" - dirty, item.staged, item.unstaged, item.untracked = _status(item.worktree) + return ( + -1, + f"status skipped: {', '.join(unsafe)} in the repository config", + [], + ) + paths, item.staged, item.unstaged, item.untracked = _status(item.worktree) except GitRunError as exc: item.worktree_state = "changed" - return -1, f"status failed: {_describe(exc)}" - item.worktree_state = "changed" if dirty else "clean" - return dirty, None + return -1, f"status failed: {_describe(exc)}", [] + item.worktree_state = "changed" if paths else "clean" + return len(paths), None, paths def _build( @@ -420,7 +513,7 @@ def _build( tree: Worktree | None, snapshot: Snapshot, ) -> Item: - local, remote, cwds, tracking = snapshot + local, remote, cwds, tracking, registrations = snapshot upstream, ahead, behind, gone = tracking.get( branch or "", (None, None, None, False) ) @@ -438,7 +531,8 @@ def _build( remotes=dict(remote.get(branch, {})) if branch else {}, ) checkout = tree is not None and not tree.prunable and os.path.isdir(tree.path) - dirty, unread = _read_worktree(item, checkout) + dirty, unread, changed = _read_worktree(item, checkout) + item.dirty = max(dirty, 0) if branch in repo.protected: item.state, item.reason = "protected", "default or protected branch" return item @@ -451,6 +545,18 @@ def _build( shas = [item.local_sha, *item.remotes.values(), tree.head if tree else None] verdicts = {sha: _merged_by(repo, sha) for sha in dict.fromkeys(shas) if sha} unmerged = [sha for sha, verdict in verdicts.items() if verdict is None] + item.merged = bool(verdicts) and not unmerged + + head = tree.head if checkout else item.local_sha + if unmerged and head: + tips = {sha for shas in remote.values() for sha in shas.values()} + item.unpushed = ( + 0 + if head in tips + else int( + _git(["rev-list", "--count", head, "--not", "--remotes"], repo.repo) + ) + ) if unread: item.state, item.reason = "dirty", unread @@ -489,32 +595,89 @@ def _build( if repo.sizes and checkout and not tree.main: item.size_bytes = _du_bytes(tree.path) + if verdicts and (repo.activity or repo.min_age is not None): + item.last_activity = _last_activity( + repo, list(verdicts), tree if checkout else None, changed + ) item.skip = _skip(repo, item, tree, checkout, cwds) - if item.skip is None and repo.min_age is not None and verdicts: - age = _age_hours(repo, list(verdicts), tree if checkout else None) - if age < repo.min_age: - item.skip = f"younger than {repo.min_age:g} h" - for project in item.compose_projects: - item.actions += [f"compose-down:{project}", f"images:{project}"] - if tree: - item.actions.append("worktree") - if item.local_sha: - item.actions.append("local-branch") - item.actions += [f"remote-branch:{name}" for name in item.remotes] + if ( + item.skip is None + and repo.min_age is not None + and item.last_activity + and (time.time() - item.last_activity) / 3600 < repo.min_age + ): + item.skip = f"younger than {repo.min_age:g} h" + item.actions = _actions(repo, item, tree, checkout, dirty, registrations) return item +def _actions( + repo: Repo, + item: Item, + tree: Worktree | None, + checkout: bool, + dirty: int, + registrations: Registrations, +) -> list[str]: + tree_part: list[str] = [] + if tree: + if ( + checkout + and not tree.main + and _linked_meta(registrations, tree.path) is None + ): + item.skip = NOT_LINKED + return [] + if checkout and FINI_HOOK in repo.hooks: + tree_part.append(f"hook:{FINI_HOOK}") + for project in item.compose_projects: + tree_part += [f"compose-down:{project}", f"images:{project}"] + tree_part.append("worktree") + branch_part = ["local-branch"] if item.local_sha else [] + branch_part += [f"remote-branch:{name}" for name in item.remotes] + + if repo.scope == "purge": + return tree_part + branch_part + tree_ok = ( + tree is not None + and not tree.main + and dirty == 0 + and (item.merged or item.unpushed == 0) + ) + branch_ok = item.state == "merged-clean" + if repo.scope == "worktree": + return tree_part if tree_ok else [] + if repo.scope == "branch": + if branch_ok and tree is not None and item.skip is None: + item.skip = f"checked out in worktree {tree.path}" + return branch_part if branch_ok and tree is None else [] + actions = tree_part if tree_ok else [] + if branch_ok and (tree is None or tree_ok): + actions += branch_part + return actions + + def _snapshot(repo: Repo) -> tuple[list[Worktree], Snapshot]: local, remote, tracking = _refs(repo.repo, repo.remotes) cwds = _process_cwds() if repo.host_ns else {} - return _worktrees(repo.repo), (local, remote, cwds, tracking) + return _worktrees(repo.repo), ( + local, + remote, + cwds, + tracking, + _registrations(repo.repo), + ) def _items(repo: Repo) -> list[Item]: trees, snapshot = _snapshot(repo) - local, remote, _, _ = snapshot + local, remote, *_ = snapshot by_branch = {t.branch: t for t in trees if t.branch} - names = sorted(set(local) | set(remote) | set(by_branch)) + names = ( + sorted(by_branch) + if repo.scope == "worktree" + else sorted(set(local) | set(remote) | set(by_branch)) + ) items = [_build(repo, name, by_branch.get(name), snapshot) for name in names] items += [ _build(repo, None, tree, snapshot) @@ -581,6 +744,9 @@ def plan_repo( merged_into: list[str], sizes: bool, min_age: float | None, + scope: str, + activity: bool, + hooks: dict[str, dict[str, list[str]]], fetch: bool = True, forge_source: Callable[[dict[str, str]], ForgeFacts] | None = None, ) -> Repo: @@ -593,13 +759,20 @@ def plan_repo( host_ns: whether this process runs in the host PID namespace. merged_into: integration lines (refs) that also count as merged; each is protected. sizes: measure each worktree's disk usage into size_bytes. - min_age: skip items whose newest commit or worktree index is younger (hours). + min_age: skip items whose newest commit, worktree index or changed file is younger (hours). + scope: one of SCOPES; decides which actions an item gets. + activity: record last_activity on every item, not only under min_age. + hooks: realpath of a managed repository's root -> hook name -> argv, + as the CLI binds `worktree_hooks`; only repo_root's entry is used, + and only while repo_root's git directory is its own. fetch: fetch every remote first; the tree scan reads the known refs only. forge_source: remote name -> URL to forge facts; fetch_forge_facts by default. Returns: The repository with its classified items. """ + if scope not in SCOPES: + raise ValueError(f"unknown scope {scope!r}") warnings: list[str] = [] if fetch: try: @@ -659,12 +832,29 @@ def plan_repo( lines=lines, sizes=sizes, min_age=min_age, + scope=scope, + activity=activity, + hooks=_pins(hooks, repo_root, warnings), warnings=warnings, ) repo.items = _items(repo) return repo +def _pins( + hooks: dict[str, dict[str, list[str]]], repo_root: str, warnings: list[str] +) -> dict[str, list[str]]: + root = os.path.realpath(repo_root) + pinned = hooks.get(root, {}) + if not pinned: + return {} + owner = os.path.realpath(os.path.dirname(_common_dir(repo_root))) + if owner != root: + warnings.append(f"pinned hooks skipped: its git directory belongs to {owner}") + return {} + return pinned + + def _foreign_owned(path: str) -> bool: uid = os.getuid() if os.lstat(path).st_uid != uid: @@ -677,12 +867,9 @@ def _foreign_owned(path: str) -> bool: def _remove_worktree(repo_root: str, path: str) -> None: - with open(os.path.join(path, ".git"), encoding="utf-8") as fh: - meta = os.path.normpath( - os.path.join(path, fh.read().strip().removeprefix("gitdir: ")) - ) - if os.path.dirname(meta) != os.path.join(_common_dir(repo_root), "worktrees"): - raise RuntimeError(f"unexpected worktree metadata path {meta}") + meta = _linked_meta(_registrations(repo_root), path) + if meta is None: + raise RuntimeError(f"{path}: {NOT_LINKED}") if _foreign_owned(path): parent, name = os.path.split(path.rstrip("/")) @@ -699,10 +886,10 @@ def _remove_worktree(repo_root: str, path: str) -> None: ] ) else: - shutil.rmtree(path) + remove_tree(path) if os.path.lexists(path): raise RuntimeError(f"worktree {path} still exists; metadata {meta} kept") - shutil.rmtree(meta) + remove_tree(meta) def _rebuild(repo: Repo, item: Item) -> Item: @@ -726,9 +913,41 @@ def _fingerprint(item: Item) -> tuple: item.remotes, item.worktree, item.compose_projects, + item.actions, ) +def _run_fini_hook(repo: Repo, path: str) -> None: + run_hook( + FINI_HOOK, + hook_command(FINI_HOOK, repo.hooks[FINI_HOOK], path, None), + path, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + + +def _sha(cwd: str, ref: str) -> str | None: + try: + return _git(["rev-parse", "--verify", "--quiet", END_OF_OPTIONS, ref], cwd) + except GitRunError: + return None + + +def _changed_by_hook(repo: Repo, item: Item) -> str | None: + path = item.worktree + dirty, unread, _ = _read_worktree(Item(branch=None, worktree=path), True) + if unread or dirty: + return unread or f"dirty: {dirty} paths" + if _sha(path, "HEAD") != item.head: + return "HEAD moved" + if item.branch and _sha(repo.repo, f"refs/heads/{item.branch}") != item.local_sha: + return f"branch {item.branch} moved" + return _busy(_process_cwds(), path) if repo.host_ns else None + + def execute(repo: Repo, item: Item) -> None: fresh = _rebuild(repo, item) if _fingerprint(fresh) != _fingerprint(item): @@ -736,19 +955,31 @@ def execute(repo: Repo, item: Item) -> None: return notes: list[str] = [] + removes_tree = "worktree" in item.actions + if removes_tree and _linked_meta(_registrations(repo.repo), item.worktree) is None: + item.result = f"skipped: {NOT_LINKED}" + return try: - for project in item.compose_projects: + if f"hook:{FINI_HOOK}" in item.actions: + _run_fini_hook(repo, item.worktree) + changed = _changed_by_hook(repo, item) + if changed: + item.result = f"skipped: changed during {FINI_HOOK} ({changed})" + return + for project in item.compose_projects if removes_tree else []: _docker(["compose", "-p", project, "down", "-v", "--remove-orphans"]) - for image in item.images: + for image in item.images if removes_tree else []: try: _docker(["image", "rm", image]) except subprocess.CalledProcessError: notes.append(f"image kept: {image}") - if item.worktree: + if removes_tree: _remove_worktree(repo.repo, item.worktree) - if item.local_sha: + if "local-branch" in item.actions: delete_local_branch(item.branch, cwd=repo.repo, force=True) for remote, sha in item.remotes.items(): + if f"remote-branch:{remote}" not in item.actions: + continue run( [ "push", @@ -765,15 +996,23 @@ def execute(repo: Repo, item: Item) -> None: item.result = "; ".join(["done", *notes]) +def _label(item: Item) -> str: + if item.skip and (item.actions or item.state == "merged-clean"): + return "SKIP" + if item.actions: + return "REMOVE" if item.state in ("unmerged", "dirty") else "DELETE" + return "KEEP" if item.state == "merged-clean" else LABELS[item.state] + + def _line(item: Item) -> str: - label = "SKIP" if item.state == "merged-clean" and item.skip else LABELS[item.state] + label = _label(item) parts = [item.reason] if item.skip and item.state != "protected": parts.append(item.skip) if item.worktree and item.state != "stale": parts.append(f"worktree {item.worktree}") if item.size_bytes is not None: - parts.append(f"size {_human(item.size_bytes)}") + parts.append(f"size {human_size(item.size_bytes)}") if item.compose_projects: parts.append("stack " + ", ".join(item.compose_projects)) if item.remotes: @@ -819,7 +1058,7 @@ def _render( ) -def _prepare( +def prepare( repo_roots: list[str], include: list[str], exclude: list[str], @@ -858,9 +1097,10 @@ def _finish(repos: list[Repo]) -> int: return 1 if any(result.startswith("failed") for result in results) else 0 -def branch_gc( +def cleanup( repo_roots: list[str], *, + scope: str, apply: bool, as_json: bool, include: list[str], @@ -868,12 +1108,17 @@ def branch_gc( merged_into: list[str], sizes: bool, min_age: float | None, + hooks: dict[str, dict[str, list[str]]], ) -> int: """ - Plan, and with apply=True execute, the cleanup of merged and clean branches. + Plan, and with apply=True execute, a cleanup of worktrees, branches or both. Args: repo_roots: main worktree roots of the repositories to scan. + scope: `worktree` removes clean worktrees whose commits are merged or + pushed (hook, stack, checkout; the branch stays); `branch` removes + merged branches without a worktree, local and remote; `all` does + both in one pass. apply: execute the plan instead of only printing it. as_json: print the plan as JSON instead of text lines. include: only items whose branch or worktree path matches one of these. @@ -881,15 +1126,26 @@ def branch_gc( merged_into: integration lines (refs) that also count as merged. sizes: measure each worktree's disk usage into the plan. min_age: skip items younger than this many hours. + hooks: pinned hooks by managed repository root realpath, from the + CLI; a worktree-fini hook runs only for a root pinned there. Returns: 0 on success, 1 when an applied item failed, 2 when refused. """ + if scope not in ("worktree", "branch", "all"): + raise ValueError(f"unknown cleanup scope {scope!r}") if apply and not host_pid_namespace(): print(REFUSAL, file=sys.stderr) return 2 - options = {"merged_into": merged_into, "sizes": sizes, "min_age": min_age} - prepared = _prepare(repo_roots, include, exclude, options) + options = { + "merged_into": merged_into, + "sizes": sizes, + "min_age": min_age, + "scope": scope, + "activity": False, + "hooks": hooks, + } + prepared = prepare(repo_roots, include, exclude, options) if prepared is None: return 2 repos, host_ns, chosen = prepared @@ -900,7 +1156,7 @@ def branch_gc( if item.eligible and chosen(item): execute(repo, item) print( - f"[gc] {item.branch or item.worktree}: {item.result}", + f"[{scope}] {item.branch or item.worktree}: {item.result}", file=sys.stderr, ) _render(repos, host_ns, chosen, as_json) @@ -915,6 +1171,7 @@ def branch_purge( merged_into: list[str], sizes: bool, min_age: float | None, + hooks: dict[str, dict[str, list[str]]], ) -> int: """ Delete unmerged or dirty items after a y/N confirmation per item. @@ -926,6 +1183,8 @@ def branch_purge( merged_into: integration lines (refs) that also count as merged. sizes: measure each worktree's disk usage into the prompt line. min_age: skip items younger than this many hours. + hooks: pinned hooks by managed repository root realpath, from the + CLI; a worktree-fini hook runs only for a root pinned there. Returns: 0 on success, 1 when a confirmed item failed, 2 when refused. @@ -933,8 +1192,15 @@ def branch_purge( if not host_pid_namespace(): print(REFUSAL, file=sys.stderr) return 2 - options = {"merged_into": merged_into, "sizes": sizes, "min_age": min_age} - prepared = _prepare(repo_roots, include, exclude, options) + options = { + "merged_into": merged_into, + "sizes": sizes, + "min_age": min_age, + "scope": "purge", + "activity": False, + "hooks": hooks, + } + prepared = prepare(repo_roots, include, exclude, options) if prepared is None: return 2 repos, _, chosen = prepared diff --git a/src/pkgmgr/actions/branch/nofollow.py b/src/pkgmgr/actions/branch/nofollow.py new file mode 100644 index 0000000..3aa5390 --- /dev/null +++ b/src/pkgmgr/actions/branch/nofollow.py @@ -0,0 +1,44 @@ +from __future__ import annotations + +import os +import shutil + +DIR_FLAGS = os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW + + +def canonical(path: str) -> bool: + """ + Args: + path: a path as git or a plan names it. + + Returns: + True when path is absolute and already its own realpath: no symbolic + link and no `..` component that the kernel could resolve elsewhere. + """ + return os.path.isabs(path) and os.path.realpath(path) == path + + +def remove_tree(path: str) -> None: + """ + Delete a directory tree without following a symbolic link anywhere on + the way to it, even one swapped in after the caller checked the path. + + Args: + path: absolute, canonical directory to delete. + + Raises: + OSError: a component of path is a symbolic link or missing, or the + deletion failed. + """ + parent, name = os.path.split(path) + fd = os.open(os.sep, os.O_RDONLY | os.O_DIRECTORY) + try: + for component in [c for c in parent.split(os.sep) if c]: + child = os.open(component, DIR_FLAGS, dir_fd=fd) + os.close(fd) + fd = child + # Trip-wire: rmtree(dir_fd=) needs Python 3.11; the fd link pins the + # opened parent on every version, and rmtree refuses a symlinked name. + shutil.rmtree(f"/proc/self/fd/{fd}/{name}") + finally: + os.close(fd) diff --git a/src/pkgmgr/actions/branch/tree.py b/src/pkgmgr/actions/branch/tree.py index 2240027..cabc752 100644 --- a/src/pkgmgr/actions/branch/tree.py +++ b/src/pkgmgr/actions/branch/tree.py @@ -221,6 +221,9 @@ def _scan( merged_into=[], sizes=False, min_age=None, + scope="all", + activity=True, + hooks={}, fetch=False, forge_source=source, ) diff --git a/src/pkgmgr/actions/worktree/__init__.py b/src/pkgmgr/actions/worktree/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/pkgmgr/actions/worktree/hooks.py b/src/pkgmgr/actions/worktree/hooks.py new file mode 100644 index 0000000..f220591 --- /dev/null +++ b/src/pkgmgr/actions/worktree/hooks.py @@ -0,0 +1,144 @@ +from __future__ import annotations + +import os +import re +import shutil +import signal +import subprocess +import time + +from .trust import root_resolved, trusted_path + +INIT_HOOK = "worktree-init" +FINI_HOOK = "worktree-fini" +MAKEFILES = ("GNUmakefile", "makefile", "Makefile") +MAKEFILE_LIMIT = 1 << 20 +HOOK_TIMEOUT = 1800 +HOOK_GRACE = 5 +PLACEHOLDER = re.compile(r"\{(path|slot)\}") + + +def defined_in(path: str, target: str) -> str | None: + """ + Name the makefile in path that defines target, read as text and never run: + make expands $(shell) and include lines even under -n. + + Args: + path: directory that may hold a makefile. + target: make target name. + + Returns: + The first makefile, in make's lookup order, with a `:` rule; + None when none has one. + """ + rule = re.compile(rf"^{re.escape(target)}\s*:", re.MULTILINE) + for name in MAKEFILES: + file = os.path.join(path, name) + if os.path.islink(file) or not os.path.isfile(file): + continue + with open(file, encoding="utf-8", errors="replace") as fh: + if rule.search(fh.read(MAKEFILE_LIMIT)): + return name + return None + + +def hook_command(name: str, argv: list[str], path: str, slot: int | None) -> list[str]: + """ + Fill a pinned hook command in one pass and resolve its program. + + Args: + name: the hook, for error messages. + argv: the command pinned in the `worktree_hooks` config. + path: the worktree, replacing `{path}`. + slot: the worktree's slot, replacing `{slot}`; None leaves it as is. + + Returns: + The argv to run; its program is an absolute path, for a bare name the + resolved file that was checked, so a multi-call binary sees that + file's name; pin the link's absolute path to keep the link's name. + + Raises: + RuntimeError: the program is neither absolute nor a bare name found, + as a root-controlled file, on a root-owned PATH entry no group or + other user can write; anything relative resolves in the worktree. + """ + + def fill(match: re.Match[str]) -> str: + if match.group(1) == "path": + return path + return match.group(0) if slot is None else str(slot) + + filled = [PLACEHOLDER.sub(fill, arg) for arg in argv] + program = filled[0] if filled else "" + if not os.path.isabs(program): + found = ( + shutil.which(program, path=trusted_path()) + if program and "/" not in program + else None + ) + resolved = root_resolved(found) if found else None + if resolved is None: + raise RuntimeError( + f"{name} pin {program!r} is neither an absolute path nor a " + "root-owned command on a root-owned PATH entry" + ) + program = resolved + return [program, *filled[1:]] + + +def run_hook( + name: str, command: list[str], path: str, *, check: bool, **kwargs +) -> subprocess.CompletedProcess: + """ + Run a resolved hook command in the worktree. + + Args: + name: the hook, for error messages. + command: the argv from hook_command. + path: the worktree to run in. + check: raise CalledProcessError on a non-zero exit. + kwargs: further subprocess.Popen arguments, such as output handling. + + Returns: + The completed process; stdin was closed, PATH held only its + root-controlled entries and HOOK_TIMEOUT applied to its whole process + group. + + Raises: + RuntimeError: the hook ran longer than HOOK_TIMEOUT, or no PATH entry + is root-controlled. + """ + with subprocess.Popen( + command, + cwd=path, + stdin=subprocess.DEVNULL, + env={**os.environ, "PATH": trusted_path()}, + start_new_session=True, + **kwargs, + ) as process: + try: + stdout, stderr = process.communicate(timeout=HOOK_TIMEOUT) + except subprocess.TimeoutExpired as exc: + _stop_group(process) + raise RuntimeError(f"{name} timed out after {HOOK_TIMEOUT} s") from exc + if check and process.returncode: + raise subprocess.CalledProcessError( + process.returncode, command, output=stdout, stderr=stderr + ) + return subprocess.CompletedProcess(command, process.returncode, stdout, stderr) + + +def _stop_group(process: subprocess.Popen) -> None: + for sig in (signal.SIGTERM, signal.SIGKILL): + try: + os.killpg(process.pid, sig) + except OSError: + return + deadline = time.monotonic() + HOOK_GRACE + while time.monotonic() < deadline: + process.poll() + try: + os.killpg(process.pid, 0) + except OSError: + return + time.sleep(0.05) diff --git a/src/pkgmgr/actions/worktree/orphans.py b/src/pkgmgr/actions/worktree/orphans.py new file mode 100644 index 0000000..e5440f7 --- /dev/null +++ b/src/pkgmgr/actions/worktree/orphans.py @@ -0,0 +1,193 @@ +from __future__ import annotations + +import json +import time +from dataclasses import asdict + +from pkgmgr.actions.branch.gc import Item, human_size, prepare + +DAY = 86400 +ORDER = ("merged-clean", "merged-dirty", "stale-open", "stale", "active") +PROPOSALS = { + "merged-clean": "pkgmgr gc", + "merged-dirty": "ask", + "stale-open": "ask", + "stale": "git worktree prune", + "active": "keep", +} +GUARDS = ("process:", "lock:") +HEADER = ( + "Class", + "Branch", + "Worktree", + "Dirty", + "Unpushed", + "Last activity", + "Proposal", +) + + +def classify(item: Item, now: float, limit: float) -> str: + """ + Sort one scanned item into an orphan class. + + Args: + item: item from a scan with activity recorded. + now: current epoch seconds. + limit: seconds of inactivity after which unfinished work is stale. + + Returns: + One of ORDER. + """ + if item.state == "stale": + return "stale" + if ( + item.state == "protected" + or (item.branch or "").startswith("archive/") + or item.skip == "checked out in main worktree" + ): + return "active" + if item.merged: + return "merged-dirty" if item.state == "dirty" else "merged-clean" + pending = item.state == "dirty" or bool(item.unpushed) or not item.remotes + idle = item.last_activity is not None and now - item.last_activity > limit + guarded = bool(item.skip) and item.skip.startswith(GUARDS) + return "stale-open" if pending and idle and not guarded else "active" + + +def proposal(item: Item, cls: str) -> str: + if cls == "active": + return "keep" + if item.skip and item.skip.startswith(GUARDS): + return f"keep ({item.skip})" + return PROPOSALS[cls] + + +def _cells(row: dict, now: float, sizes: bool) -> list[str]: + activity = row["last_activity"] + when = ( + f"{time.strftime('%Y-%m-%d', time.localtime(activity))} " + f"({int((now - activity) // DAY)} d)" + if activity + else "-" + ) + values = [ + row["class"], + row["branch"] or "(detached)", + row["worktree"] or "-", + *( + [human_size(row["size_bytes"]) if row["size_bytes"] else "-"] + if sizes + else [] + ), + str(row["dirty"]), + "-" if row["unpushed"] is None else str(row["unpushed"]), + when, + row["proposal"], + ] + return [value.replace("|", "\\|") for value in values] + + +def _table(rows: list[dict], now: float, sizes: bool) -> list[str]: + header = [*HEADER[:3], *(["Size"] if sizes else []), *HEADER[3:]] + lines = [ + "| " + " | ".join(header) + " |", + "| " + " | ".join("---" for _ in header) + " |", + ] + return lines + ["| " + " | ".join(_cells(r, now, sizes)) + " |" for r in rows] + + +def worktree_orphans( + repo_roots: list[str], + *, + stale_days: int, + top: int, + as_json: bool, + include: list[str], + exclude: list[str], + merged_into: list[str], + sizes: bool, +) -> int: + """ + List merged and abandoned branches and worktrees with a proposal each. + + Read-only apart from the fetch the scan does. Merged covers ancestry, a + merged pull request, an integration line and patch equivalence. + + Args: + repo_roots: main worktree roots of the repositories to scan. + stale_days: unfinished work idle for longer is stale-open. + top: rows per class in the text table; 0 shows the counts only. + as_json: print every row, active ones included, as JSON. + include: only items whose branch or worktree path matches one of these. + exclude: drop items whose branch or worktree path matches one of these. + merged_into: integration lines (refs) that also count as merged. + sizes: measure each worktree's disk usage and order by it. + + Returns: + 0 on success, 2 when refused. + """ + options = { + "merged_into": merged_into, + "sizes": sizes, + "min_age": None, + "scope": "all", + "activity": True, + "hooks": {}, + } + prepared = prepare(repo_roots, include, exclude, options) + if prepared is None: + return 2 + repos, host_ns, chosen = prepared + now = time.time() + limit = stale_days * DAY + + report = [] + for repo in repos: + rows = [] + for item in repo.items: + if not chosen(item): + continue + cls = classify(item, now, limit) + rows.append({**asdict(item), "class": cls, "proposal": proposal(item, cls)}) + rows.sort( + key=lambda r: ( + ORDER.index(r["class"]), + -(r["size_bytes"] or 0) if sizes else 0, + r["last_activity"] or 0, + ) + ) + report.append({"repo": repo.repo, "warnings": repo.warnings, "rows": rows}) + + if as_json: + print( + json.dumps( + { + "host_pid_namespace": host_ns, + "stale_days": stale_days, + "repos": report, + }, + indent=2, + ) + ) + return 0 + for entry in report: + print(f"== {entry['repo']} (stale after {stale_days} days)") + for warning in entry["warnings"]: + print(f"WARN {warning}") + for cls in ORDER: + count = sum(r["class"] == cls for r in entry["rows"]) + print(f"- {cls}: {count} ({PROPOSALS[cls]})") + shown = [ + row + for cls in ORDER[:-1] + for row in [r for r in entry["rows"] if r["class"] == cls][:top] + ] + if shown: + print() + print("\n".join(_table(shown, now, sizes))) + if not host_ns: + print( + "WARN busy and lock checks unavailable: not in the host PID namespace" + ) + return 0 diff --git a/src/pkgmgr/actions/worktree/slots.py b/src/pkgmgr/actions/worktree/slots.py new file mode 100644 index 0000000..bb06619 --- /dev/null +++ b/src/pkgmgr/actions/worktree/slots.py @@ -0,0 +1,40 @@ +from __future__ import annotations + +import os + +SLOT_FILE = "pkgmgr-slot" + + +def used_slots(common_dir: str) -> set[int]: + base = os.path.join(common_dir, "worktrees") + slots: set[int] = set() + for entry in os.listdir(base) if os.path.isdir(base) else []: + try: + with open(os.path.join(base, entry, SLOT_FILE), encoding="utf-8") as fh: + slots.add(int(fh.read().strip())) + except (OSError, ValueError): + continue + return slots + + +def next_slot(used: set[int]) -> int: + slot = 1 + while slot in used: + slot += 1 + return slot + + +def write_slot(path: str, slot: int) -> None: + """ + Record a slot in the metadata of a worktree this process just added. + + Args: + path: the new worktree, whose `.git` file names its metadata. + slot: the slot to record. + """ + with open(os.path.join(path, ".git"), encoding="utf-8") as fh: + meta = os.path.normpath( + os.path.join(path, fh.read().strip().removeprefix("gitdir: ")) + ) + with open(os.path.join(meta, SLOT_FILE), "w", encoding="utf-8") as fh: + fh.write(f"{slot}\n") diff --git a/src/pkgmgr/actions/worktree/trust.py b/src/pkgmgr/actions/worktree/trust.py new file mode 100644 index 0000000..92f2397 --- /dev/null +++ b/src/pkgmgr/actions/worktree/trust.py @@ -0,0 +1,93 @@ +from __future__ import annotations + +import os +import stat + +ROOT_UID = 0 +UID_MAP = "/proc/self/uid_map" +INITIAL_MAP = [[0, 0, 4294967295]] +OVERFLOW_UID = "/proc/sys/kernel/overflowuid" +SHARED_WRITE = stat.S_IWGRP | stat.S_IWOTH + + +def root_uids() -> frozenset[int]: + """ + Returns: + The owner uids that stand for root. In the initial user namespace, + whose uid_map is the identity over every uid, that is 0. In any other + namespace a mapped uid 0 may be any user, so only the overflow uid + counts: the real root's files show up under it there, as do those of + every other unmapped owner. Empty when a file it needs is unreadable + or this process's own uid is unmapped, since its own files then show + up as the overflow uid too. + """ + try: + with open(UID_MAP, encoding="ascii") as fh: + ranges = [[int(n) for n in line.split()] for line in fh if line.strip()] + if ranges == INITIAL_MAP: + return frozenset({ROOT_UID}) + with open(OVERFLOW_UID, encoding="ascii") as fh: + overflow = int(fh.read()) + except (OSError, ValueError): + return frozenset() + if os.geteuid() == overflow: + return frozenset() + return frozenset({overflow}) + + +def root_resolved(path: str) -> str | None: + """ + Args: + path: an absolute file or directory. + + Returns: + The path, resolved once, when it and every directory above it are + owned by root and writable by neither group nor others, so no other + user can replace what it names; None otherwise. Callers keep and use + the returned path: resolving path again can follow a link swapped in + after this check. In the initial user namespace a component on a + nosuid mount also yields None: a desktop user can mount an image of + root-owned files through udisks2 or FUSE, always nosuid, while the + system's own directories never are; every bubblewrap mount is nosuid, + so the rule cannot apply inside one. + """ + owners = root_uids() + initial = owners == {ROOT_UID} + resolved = os.path.realpath(path) + current = resolved + while True: + try: + info = os.stat(current) + nosuid = initial and os.statvfs(current).f_flag & os.ST_NOSUID + except OSError: + return None + if info.st_uid not in owners or info.st_mode & SHARED_WRITE or nosuid: + return None + parent = os.path.dirname(current) + if parent == current: + return resolved + current = parent + + +def trusted_path() -> str: + """ + Returns: + The PATH entries that resolve to root-controlled directories, else + those of os.defpath, each in its resolved form and joined by + os.pathsep, so a symbolic link another user can retarget never stays + on it. + + Raises: + RuntimeError: no entry of either qualifies; an empty PATH would make + the hook search its working directory, the worktree. + """ + for source in (os.get_exec_path(), os.defpath.split(os.pathsep)): + resolved = (root_resolved(entry) for entry in source if os.path.isabs(entry)) + entries = dict.fromkeys( + entry for entry in resolved if entry and os.path.isdir(entry) + ) + if entries: + return os.pathsep.join(entries) + raise RuntimeError( + "no PATH entry is a root-owned directory closed to other writers" + ) diff --git a/src/pkgmgr/actions/worktree/up.py b/src/pkgmgr/actions/worktree/up.py new file mode 100644 index 0000000..e513ddf --- /dev/null +++ b/src/pkgmgr/actions/worktree/up.py @@ -0,0 +1,226 @@ +from __future__ import annotations + +import json +import os +import re +import sys + +from pkgmgr.core.git.errors import GitRunError +from pkgmgr.core.git.queries import FILTER_CONFIG, local_config +from pkgmgr.core.git.run import END_OF_OPTIONS, run + +from .hooks import INIT_HOOK, defined_in, hook_command, run_hook +from .slots import next_slot, used_slots, write_slot + +PORT_PREFIX = re.compile(r"^\d+/") +HOST_AND_PATH = re.compile(r"([^:/]+)[:/](.*)") +NETWORK_CONFIG = ( + r"^(core\.(sshcommand|gitproxy)|remote\..*\.(uploadpack|receivepack)" + r"|credential\..*helper|url\..*\.insteadof|protocol\..*allow)$" +) + + +def worktree_location(url: str) -> str: + """ + Map a remote URL to `//`. + + Args: + url: remote URL in https, ssh or scp form. + + Returns: + The host followed by the repository path, without `.git` or a port; + a local path loses its leading slash, so it always stays relative. + """ + rest = url.removesuffix(".git").split("://", 1)[-1].split("@", 1)[-1] + match = HOST_AND_PATH.match(rest) + if not match: + return rest.strip("/") + host, path = match.groups() + return f"{host}/{PORT_PREFIX.sub('', path).strip('/')}" + + +def _common_dir(cwd: str) -> str: + return run( + ["rev-parse", "--path-format=absolute", "--git-common-dir"], cwd=cwd, scan=True + ) + + +def _inside(path: str, root: str) -> bool: + real, base = os.path.realpath(path), os.path.realpath(root) + return os.path.commonpath([real, base]) == base + + +def _ref_exists(repo: str, ref: str) -> bool: + try: + run( + ["rev-parse", "--verify", "--quiet", f"{ref}^{{commit}}"], + cwd=repo, + scan=True, + ) + except GitRunError: + return False + return True + + +def _commit_of(repo: str, ref: str) -> str: + # worktree add reads a start point that begins with "-" as HEAD even after + # "--" or --end-of-options, so it only ever gets the resolved commit id + return run( + ["rev-parse", "--verify", "--quiet", END_OF_OPTIONS, f"{ref}^{{commit}}"], + cwd=repo, + scan=True, + ) + + +def _default_base(repo: str) -> str: + try: + return run( + ["symbolic-ref", "--short", "refs/remotes/origin/HEAD"], cwd=repo, scan=True + ) + except GitRunError: + return "origin/main" if _ref_exists(repo, "origin/main") else "origin/master" + + +def worktree_up( + repo_dir: str, + branch: str, + *, + base: str | None, + worktrees_dir: str, + path: str | None, + hooks: dict[str, dict[str, list[str]]], + as_json: bool, +) -> int: + """ + Check a branch out into its own worktree under the standard path. + + Args: + repo_dir: any directory inside the repository. + branch: branch to check out; created from base when it exists nowhere. + base: start point of a new branch; None takes origin's default branch. + worktrees_dir: root of the standard path + `////`. + path: exact checkout path instead of the standard one, e.g. a + directory an agent sandbox releases; None keeps the standard path. + hooks: pinned hooks by managed repository root realpath, from the + CLI; the worktree-init hook runs only when the repository root is + pinned there, never from the Makefile or by origin's URL. + as_json: print the result as JSON instead of text. + + Returns: + 0 when the worktree is ready, 1 when the worktree-init hook failed + (the worktree stays), 2 when refused. + """ + try: + common = _common_dir(repo_dir) + root = os.path.dirname(common) + url = run(["remote", "get-url", "--", "origin"], cwd=root, scan=True) + run(["check-ref-format", "--branch", branch], cwd=root, scan=True) + filters = local_config(root, FILTER_CONFIG) + except GitRunError as exc: + print(f"Refusing: {str(exc).splitlines()[0]}", file=sys.stderr) + return 2 + if filters: + print( + f"Refusing: {', '.join(filters)} in the repository config would run " + "on the checkout", + file=sys.stderr, + ) + return 2 + + location = worktree_location(url) + if path: + path = os.path.abspath(os.path.expanduser(path)) + else: + standard = os.path.expanduser(worktrees_dir) + path = os.path.join(standard, location, branch) + if not _inside(path, standard): + print(f"Refusing: {path} lies outside {standard}", file=sys.stderr) + return 2 + if os.path.lexists(path): + print(f"Refusing: {path} already exists", file=sys.stderr) + return 2 + + slot = next_slot(used_slots(common)) + pinned = hooks.get(os.path.realpath(root), {}).get(INIT_HOOK) + try: + command = hook_command(INIT_HOOK, pinned, path, slot) if pinned else None + except RuntimeError as exc: + print(f"Refusing: {exc}", file=sys.stderr) + return 2 + + warnings: list[str] = [] + network = local_config(root, NETWORK_CONFIG) + if network: + warnings.append(f"fetch skipped: {', '.join(network)} in the repository config") + else: + try: + run( + ["fetch", "--quiet", "--no-auto-maintenance", "origin"], + cwd=root, + scan=True, + ) + except GitRunError as exc: + warnings.append(f"fetch failed: {str(exc).splitlines()[0]}") + + os.makedirs(os.path.dirname(path), exist_ok=True) + start: str | None = None + if _ref_exists(root, f"refs/heads/{branch}"): + origin, add = "existing", ["worktree", "add", "--", path, branch] + elif _ref_exists(root, f"refs/remotes/origin/{branch}"): + origin = f"tracking origin/{branch}" + add = [ + "worktree", + "add", + "--track", + "-b", + branch, + "--", + path, + f"origin/{branch}", + ] + else: + start = base or _default_base(root) + origin = f"new from {start}" + add = ["worktree", "add", "--no-track", "-b", branch, "--", path] + try: + if start is not None: + add.append(_commit_of(root, start)) + run(add, cwd=root, scan=True) + except GitRunError as exc: + print(f"Refusing: {str(exc).splitlines()[0]}", file=sys.stderr) + return 2 + + write_slot(path, slot) + + hook = "none" + code = 0 + if command: + try: + ran = run_hook(INIT_HOOK, command, path, check=False, stdout=sys.stderr) + failure = None if ran.returncode == 0 else str(ran.returncode) + except (RuntimeError, OSError) as exc: + failure = str(exc) + hook = INIT_HOOK if failure is None else f"{INIT_HOOK} failed ({failure})" + code = 0 if failure is None else 1 + else: + makefile = defined_in(path, INIT_HOOK) + if makefile: + hook = f"not pinned: {INIT_HOOK} in {makefile}" + + result = { + "path": path, + "branch": branch, + "origin": origin, + "slot": slot, + "hook": hook, + "warnings": warnings, + } + if as_json: + print(json.dumps(result, indent=2)) + else: + for warning in warnings: + print(f"WARN {warning}") + for key in ("path", "branch", "origin", "slot", "hook"): + print(f"{key:<8} {result[key]}") + return code diff --git a/src/pkgmgr/cli/commands/__init__.py b/src/pkgmgr/cli/commands/__init__.py index ce6fb90..690f26b 100644 --- a/src/pkgmgr/cli/commands/__init__.py +++ b/src/pkgmgr/cli/commands/__init__.py @@ -10,6 +10,7 @@ from .release import handle_release from .repos import handle_repos_command from .tools import handle_tools_command from .version import handle_version +from .worktree import handle_gc, handle_worktree __all__ = [ "handle_archive", @@ -17,6 +18,7 @@ __all__ = [ "handle_changelog", "handle_code_scanning", "handle_config", + "handle_gc", "handle_make", "handle_mirror_command", "handle_publish", @@ -24,4 +26,5 @@ __all__ = [ "handle_repos_command", "handle_tools_command", "handle_version", + "handle_worktree", ] diff --git a/src/pkgmgr/cli/commands/branch.py b/src/pkgmgr/cli/commands/branch.py index cfee18b..a178ea7 100644 --- a/src/pkgmgr/cli/commands/branch.py +++ b/src/pkgmgr/cli/commands/branch.py @@ -4,8 +4,8 @@ import os import sys from pkgmgr.actions.branch import ( - branch_gc, branch_purge, + cleanup, close_branch, drop_branch, open_branch, @@ -17,22 +17,42 @@ from pkgmgr.core.git.errors import GitBaseError from pkgmgr.core.repository.dir import get_repo_dir +def _repo_dir(ctx: CLIContext, repo: dict) -> str: + return ( + os.path.expanduser(str(repo["directory"])) + if repo.get("directory") + else get_repo_dir(ctx.repositories_base_dir, repo) + ) + + def _known_dirs(ctx: CLIContext) -> list[str]: - return [ - os.path.expanduser(str(r["directory"])) - if r.get("directory") - else get_repo_dir(ctx.repositories_base_dir, r) - for r in ctx.all_repositories - ] + return [_repo_dir(ctx, r) for r in ctx.all_repositories] -def _gc_repo_roots(args, ctx: CLIContext) -> list[str]: - if not args.all: - return [main_repo_root(".")] - return [d for d in _known_dirs(ctx) if os.path.exists(os.path.join(d, ".git"))] +def pinned_hooks(ctx: CLIContext) -> dict[str, dict[str, list[str]]]: + """ + Bind the `worktree_hooks` config to managed repositories, never to a + remote URL, which the repository's own config sets. + + Args: + ctx: CLI context; each `worktree_hooks` key names the + `//` of a `repositories` entry. + + Returns: + Realpath of each pinned managed repository's directory -> hook name + -> argv; keys that match no managed repository are dropped. + """ + pins = ctx.config_merged["worktree_hooks"] + return { + os.path.realpath(_repo_dir(ctx, repo)): pins[key] + for repo in ctx.config_merged["repositories"] + if all(repo.get(part) for part in ("provider", "account", "repository")) + and (key := f"{repo['provider']}/{repo['account']}/{repo['repository']}") + in pins + } -def _refresh(roots: list[str]) -> None: +def refresh_tree(roots: list[str]) -> None: for root in roots: try: tree.refresh_tree_entry(root) @@ -40,19 +60,58 @@ def _refresh(roots: list[str]) -> None: print(f"WARN tree file not rewritten for {root}: {exc}", file=sys.stderr) +def scan_repo_roots(args, ctx: CLIContext) -> list[str]: + """ + Resolve the repositories a scan covers: the current one, or every + pkgmgr-managed one with --all. Exits with 2 outside a repository. + """ + if not args.all: + try: + return [main_repo_root(".")] + except GitBaseError as exc: + print(f"Not inside a git repository: {exc}", file=sys.stderr) + sys.exit(2) + return [d for d in _known_dirs(ctx) if os.path.exists(os.path.join(d, ".git"))] + + +def scan_options(args) -> dict: + return { + "include": args.branch, + "exclude": args.exclude, + "merged_into": args.merged_into, + "sizes": args.sizes, + } + + +def run_cleanup(args, ctx: CLIContext, scope: str) -> None: + roots = scan_repo_roots(args, ctx) + code = cleanup( + roots, + scope=scope, + apply=args.apply, + as_json=args.json, + min_age=args.min_age, + hooks=pinned_hooks(ctx), + **scan_options(args), + ) + if args.apply: + refresh_tree(roots) + sys.exit(code) + + def handle_branch(args, ctx: CLIContext) -> None: """ Handle `pkgmgr branch` subcommands. Currently supported: - - pkgmgr branch open [] [--base ] - - pkgmgr branch close [] [--base ] [--force|-f] - - pkgmgr branch drop [] [--base ] [--force|-f] - - pkgmgr branch gc [--apply] [--json] [--all] [--branch N]... [--exclude N]... - [--merged-into REF]... [--sizes] [--min-age HOURS] - - pkgmgr branch purge [--all] [--branch N]... [--exclude N]... - [--merged-into REF]... [--sizes] [--min-age HOURS] - - pkgmgr branch tree [--format json|yaml] [--output FILE] + - pkgmgr branch open [] [--base ] + - pkgmgr branch close [] [--base ] [--force|-f] + - pkgmgr branch drop [] [--base ] [--force|-f] + - pkgmgr branch cleanup [--apply] [--json] [--all] [--branch N]... [--exclude N]... + [--merged-into REF]... [--sizes] [--min-age HOURS] + - pkgmgr branch purge [--all] [--branch N]... [--exclude N]... + [--merged-into REF]... [--sizes] [--min-age HOURS] + - pkgmgr branch tree [--format json|yaml] [--output FILE] """ if args.subcommand == "tree": sys.exit( @@ -64,25 +123,18 @@ def handle_branch(args, ctx: CLIContext) -> None: ) ) - if args.subcommand in ("gc", "purge"): - try: - roots = _gc_repo_roots(args, ctx) - except GitBaseError as exc: - print(f"Not inside a git repository: {exc}", file=sys.stderr) - sys.exit(2) - options = { - "include": args.branch, - "exclude": args.exclude, - "merged_into": args.merged_into, - "sizes": args.sizes, - "min_age": args.min_age, - } - if args.subcommand == "gc": - code = branch_gc(roots, apply=args.apply, as_json=args.json, **options) - else: - code = branch_purge(roots, **options) - if args.subcommand == "purge" or args.apply: - _refresh(roots) + if args.subcommand == "cleanup": + run_cleanup(args, ctx, "branch") + + if args.subcommand == "purge": + roots = scan_repo_roots(args, ctx) + code = branch_purge( + roots, + min_age=args.min_age, + hooks=pinned_hooks(ctx), + **scan_options(args), + ) + refresh_tree(roots) sys.exit(code) if args.subcommand == "open": @@ -91,7 +143,7 @@ def handle_branch(args, ctx: CLIContext) -> None: base_branch=getattr(args, "base", "main"), cwd=".", ) - _refresh([main_repo_root(".")]) + refresh_tree([main_repo_root(".")]) return if args.subcommand == "close": @@ -101,7 +153,7 @@ def handle_branch(args, ctx: CLIContext) -> None: cwd=".", force=getattr(args, "force", False), ) - _refresh([main_repo_root(".")]) + refresh_tree([main_repo_root(".")]) return if args.subcommand == "drop": @@ -111,7 +163,7 @@ def handle_branch(args, ctx: CLIContext) -> None: cwd=".", force=getattr(args, "force", False), ) - _refresh([main_repo_root(".")]) + refresh_tree([main_repo_root(".")]) return print(f"Unknown branch subcommand: {args.subcommand}") diff --git a/src/pkgmgr/cli/commands/worktree.py b/src/pkgmgr/cli/commands/worktree.py new file mode 100644 index 0000000..64c9ff0 --- /dev/null +++ b/src/pkgmgr/cli/commands/worktree.py @@ -0,0 +1,59 @@ +from __future__ import annotations + +import sys + +from pkgmgr.actions.branch.gc import main_repo_root +from pkgmgr.actions.worktree.orphans import worktree_orphans +from pkgmgr.actions.worktree.up import worktree_up +from pkgmgr.cli.context import CLIContext + +from .branch import ( + pinned_hooks, + refresh_tree, + run_cleanup, + scan_options, + scan_repo_roots, +) + + +def handle_worktree(args, ctx: CLIContext) -> None: + """ + Handle `pkgmgr worktree` subcommands. + + - pkgmgr worktree up [--base REF] [--path DIR] [--json] + - pkgmgr worktree cleanup [--apply] [--json] [scan options] [--min-age HOURS] + - pkgmgr worktree orphans --stale-days N --top N [--json] [scan options] + """ + if args.subcommand == "up": + code = worktree_up( + ".", + args.name, + base=args.base, + worktrees_dir=ctx.config_merged["directories"]["worktrees"], + path=args.path, + hooks=pinned_hooks(ctx), + as_json=args.json, + ) + refresh_tree([main_repo_root(".")]) + sys.exit(code) + + if args.subcommand == "cleanup": + run_cleanup(args, ctx, "worktree") + + if args.subcommand == "orphans": + sys.exit( + worktree_orphans( + scan_repo_roots(args, ctx), + stale_days=args.stale_days, + top=args.top, + as_json=args.json, + **scan_options(args), + ) + ) + + print(f"Unknown worktree subcommand: {args.subcommand}") + sys.exit(2) + + +def handle_gc(args, ctx: CLIContext) -> None: + run_cleanup(args, ctx, "all") diff --git a/src/pkgmgr/cli/dispatch.py b/src/pkgmgr/cli/dispatch.py index 654e896..ace965b 100644 --- a/src/pkgmgr/cli/dispatch.py +++ b/src/pkgmgr/cli/dispatch.py @@ -10,6 +10,7 @@ from pkgmgr.cli.commands import ( handle_changelog, handle_code_scanning, handle_config, + handle_gc, handle_make, handle_mirror_command, handle_publish, @@ -17,6 +18,7 @@ from pkgmgr.cli.commands import ( handle_repos_command, handle_tools_command, handle_version, + handle_worktree, ) from pkgmgr.cli.context import CLIContext from pkgmgr.cli.proxy import maybe_handle_proxy @@ -163,6 +165,14 @@ def dispatch_command(args, ctx: CLIContext) -> None: handle_branch(args, ctx) return + if args.command == "worktree": + handle_worktree(args, ctx) + return + + if args.command == "gc": + handle_gc(args, ctx) + return + if args.command == "mirror": handle_mirror_command(ctx, args, selected) return diff --git a/src/pkgmgr/cli/parser/__init__.py b/src/pkgmgr/cli/parser/__init__.py index 4f366ee..5ed5dc2 100644 --- a/src/pkgmgr/cli/parser/__init__.py +++ b/src/pkgmgr/cli/parser/__init__.py @@ -10,6 +10,7 @@ from .changelog_cmd import add_changelog_subparser from .code_scanning_cmd import add_code_scanning_subparser from .common import SortedSubParsersAction from .config_cmd import add_config_subparsers +from .gc_cmd import add_gc_subparser from .install_update import add_install_update_subparsers from .list_cmd import add_list_subparser from .make_cmd import add_make_subparsers @@ -18,6 +19,7 @@ from .navigation_cmd import add_navigation_subparsers from .publish_cmd import add_publish_subparser from .release_cmd import add_release_subparser from .version_cmd import add_version_subparser +from .worktree_cmd import add_worktree_subparsers def create_parser(description_text: str) -> argparse.ArgumentParser: @@ -58,6 +60,8 @@ def create_parser(description_text: str) -> argparse.ArgumentParser: add_navigation_subparsers(subparsers) add_branch_subparsers(subparsers) + add_worktree_subparsers(subparsers) + add_gc_subparser(subparsers) add_release_subparser(subparsers) add_publish_subparser(subparsers) diff --git a/src/pkgmgr/cli/parser/branch_cmd.py b/src/pkgmgr/cli/parser/branch_cmd.py index dda094b..2dc8104 100644 --- a/src/pkgmgr/cli/parser/branch_cmd.py +++ b/src/pkgmgr/cli/parser/branch_cmd.py @@ -97,29 +97,29 @@ def add_branch_subparsers( help="Skip confirmation prompt and drop the branch directly", ) - branch_gc = branch_subparsers.add_parser( - "gc", + branch_cleanup = branch_subparsers.add_parser( + "cleanup", help=( - "Remove merged and clean branches with their worktrees and compose " - "stacks (dry-run unless --apply)" + "Remove merged branches that no worktree holds, locally and on the " + "remotes (dry-run unless --apply)" ), ) - branch_gc.add_argument( - "--apply", - action="store_true", - help="Execute the plan instead of only printing it", - ) - branch_gc.add_argument( - "--json", - action="store_true", - help="Print the plan as JSON", + add_apply_options(branch_cleanup) + + branch_purge = branch_subparsers.add_parser( + "purge", + help="Delete unmerged or dirty branches after a confirmation per item", ) + for sub in (branch_cleanup, branch_purge): + add_scan_options(sub) + add_min_age_option(sub) + branch_tree = branch_subparsers.add_parser( "tree", help=( "Print the branches and worktrees of every pkgmgr-managed repository " - "with their gc state, without fetching" + "with their cleanup state, without fetching" ), ) branch_tree.add_argument( @@ -134,50 +134,62 @@ def add_branch_subparsers( help="Write atomically to FILE instead of stdout", ) - branch_purge = branch_subparsers.add_parser( - "purge", - help="Delete unmerged or dirty branches after a confirmation per item", + +def add_apply_options(sub: argparse.ArgumentParser) -> None: + sub.add_argument( + "--apply", + action="store_true", + help="Execute the plan instead of only printing it", + ) + sub.add_argument( + "--json", + action="store_true", + help="Print the plan as JSON", ) - for sub in (branch_gc, branch_purge): - sub.add_argument( - "--all", - action="store_true", - help="Scan every pkgmgr-managed repository instead of the current one", - ) - sub.add_argument( - "--branch", - action="append", - default=[], - metavar="NAME", - help="Only this branch (or worktree path); repeatable", - ) - sub.add_argument( - "--exclude", - action="append", - default=[], - metavar="NAME", - help="Skip this branch (or worktree path); repeatable", - ) - sub.add_argument( - "--merged-into", - action="append", - default=[], - metavar="REF", - help=( - "Also count an item as merged when its commits are in REF (an " - "integration line); REF is protected and a checkout at its tip " - "counts as fresh; repeatable" - ), - ) - sub.add_argument( - "--sizes", - action="store_true", - help="Measure each worktree's disk usage (du) into the plan", - ) - sub.add_argument( - "--min-age", - type=float, - metavar="HOURS", - help="Skip items whose newest commit or worktree index is younger", - ) + +def add_scan_options(sub: argparse.ArgumentParser) -> None: + sub.add_argument( + "--all", + action="store_true", + help="Scan every pkgmgr-managed repository instead of the current one", + ) + sub.add_argument( + "--branch", + action="append", + default=[], + metavar="NAME", + help="Only this branch (or worktree path); repeatable", + ) + sub.add_argument( + "--exclude", + action="append", + default=[], + metavar="NAME", + help="Skip this branch (or worktree path); repeatable", + ) + sub.add_argument( + "--merged-into", + action="append", + default=[], + metavar="REF", + help=( + "Also count an item as merged when its commits are in REF (an " + "integration line); REF is protected and a checkout at its tip " + "counts as fresh; repeatable" + ), + ) + sub.add_argument( + "--sizes", + action="store_true", + help="Measure each worktree's disk usage (du) into the plan", + ) + + +def add_min_age_option(sub: argparse.ArgumentParser) -> None: + sub.add_argument( + "--min-age", + type=float, + metavar="HOURS", + help="Skip items whose newest commit, worktree index or changed file is younger", + ) diff --git a/src/pkgmgr/cli/parser/gc_cmd.py b/src/pkgmgr/cli/parser/gc_cmd.py new file mode 100644 index 0000000..a8e30c7 --- /dev/null +++ b/src/pkgmgr/cli/parser/gc_cmd.py @@ -0,0 +1,18 @@ +from __future__ import annotations + +import argparse + +from .branch_cmd import add_apply_options, add_min_age_option, add_scan_options + + +def add_gc_subparser(subparsers: argparse._SubParsersAction) -> None: + gc = subparsers.add_parser( + "gc", + help=( + "Garbage-collect the current repository, or every one with --all: " + "worktree cleanup and branch cleanup in one pass (dry-run unless --apply)" + ), + ) + add_apply_options(gc) + add_scan_options(gc) + add_min_age_option(gc) diff --git a/src/pkgmgr/cli/parser/worktree_cmd.py b/src/pkgmgr/cli/parser/worktree_cmd.py new file mode 100644 index 0000000..77e3eb7 --- /dev/null +++ b/src/pkgmgr/cli/parser/worktree_cmd.py @@ -0,0 +1,74 @@ +from __future__ import annotations + +import argparse + +from .branch_cmd import add_apply_options, add_min_age_option, add_scan_options + + +def add_worktree_subparsers(subparsers: argparse._SubParsersAction) -> None: + worktree_parser = subparsers.add_parser( + "worktree", + help="Worktree utilities: create, clean up and find abandoned worktrees", + ) + worktree_subparsers = worktree_parser.add_subparsers( + dest="subcommand", + help="Worktree subcommands", + required=True, + ) + + up = worktree_subparsers.add_parser( + "up", + help=( + "Check a branch out under ///" + "/, assign a slot and run `make worktree-init SLOT=` " + "when the project defines it" + ), + ) + up.add_argument("name", help="Branch to check out; created when it exists nowhere") + up.add_argument( + "--base", + metavar="REF", + help="Start point of a new branch (origin's default branch when omitted)", + ) + up.add_argument( + "--path", + metavar="DIR", + help=( + "Exact checkout path instead of the standard one, e.g. a directory " + "an agent sandbox releases" + ), + ) + up.add_argument("--json", action="store_true", help="Print the result as JSON") + + cleanup = worktree_subparsers.add_parser( + "cleanup", + help=( + "Remove clean worktrees whose commits are merged or pushed: " + "`make worktree-fini`, compose stack, checkout; the branch stays " + "(dry-run unless --apply)" + ), + ) + add_apply_options(cleanup) + add_scan_options(cleanup) + add_min_age_option(cleanup) + + orphans = worktree_subparsers.add_parser( + "orphans", + help="List merged and abandoned branches and worktrees with a proposal each", + ) + add_scan_options(orphans) + orphans.add_argument( + "--stale-days", + type=int, + required=True, + metavar="DAYS", + help="Unfinished work idle for longer counts as stale-open", + ) + orphans.add_argument( + "--top", + type=int, + required=True, + metavar="N", + help="Rows per class in the text table; 0 prints the counts only", + ) + orphans.add_argument("--json", action="store_true", help="Print every row as JSON") diff --git a/src/pkgmgr/config/defaults.yaml b/src/pkgmgr/config/defaults.yaml index 1f6b95b..00a8b57 100644 --- a/src/pkgmgr/config/defaults.yaml +++ b/src/pkgmgr/config/defaults.yaml @@ -4,6 +4,13 @@ directories: repositories: ~/Repositories/ workspaces: ~/Workspaces/ binaries: ~/.local/bin/ + worktrees: ~/.local/share/worktrees/ +# worktree_hooks: // of an entry under +# repositories -> worktree-init / worktree-fini -> argv, with {path} and {slot} +# replaced; it runs only for that entry's directory, never selected by origin's +# URL, and without an entry no hook runs, whatever the repository's Makefile +# defines +worktree_hooks: {} repositories: - account: kevinveenbirkenbach description: A configurable Python package manager that automates repository tasks—including cloning, installation, updates, and status reporting—based on a YAML configuration file for streamlined software management which gives you access to the Kevin Veen-Birkenbach Code Universe. diff --git a/src/pkgmgr/core/config/load.py b/src/pkgmgr/core/config/load.py index 16f9d13..2563f08 100644 --- a/src/pkgmgr/core/config/load.py +++ b/src/pkgmgr/core/config/load.py @@ -144,6 +144,7 @@ def _load_layer_dir( { "directories": {...}, "repositories": [...], + : dicts deep-merged, other values from the last file, } """ defaults: dict[str, Any] = {"directories": {}, "repositories": []} @@ -165,21 +166,16 @@ def _load_layer_dir( for path in yaml_files: data = _load_yaml_file(path) - category_name = path.stem - - dirs = data.get("directories") - if isinstance(dirs, dict): - defaults.setdefault("directories", {}) - _deep_merge(defaults["directories"], dirs) - - repos = data.get("repositories") + repos = data.pop("repositories", None) if isinstance(repos, list): - defaults.setdefault("repositories", []) _merge_repo_lists( defaults["repositories"], repos, - category_name=category_name, + category_name=path.stem, ) + if not isinstance(data.get("directories", {}), dict): + del data["directories"] + _deep_merge(defaults, data) return defaults @@ -229,7 +225,8 @@ def load_config(user_config_path: str) -> dict[str, Any]: Steps: 1. Determine ~/.config/pkgmgr/ (or dir of user_config_path). 2. Load all *.yml/*.yaml in that dir (except the user config file) as defaults. - 3. If nothing found, fall back to package defaults. + 3. If nothing found, fall back to package defaults. Package keys other + than directories and repositories always lie under the layers. 4. Load the user config file (if present). 5. Merge: - directories: deep-merge (defaults <- user) @@ -247,13 +244,14 @@ def load_config(user_config_path: str) -> dict[str, Any]: # 1+2) Defaults from user directory defaults = _load_layer_dir(config_dir, skip_filename=user_cfg_name) + package = _load_defaults_from_package_or_project() # 3) Fallback to package defaults if not defaults["directories"] and not defaults["repositories"]: - defaults = _load_defaults_from_package_or_project() - - defaults.setdefault("directories", {}) - defaults.setdefault("repositories", []) + defaults["directories"] = package["directories"] + defaults["repositories"] = package["repositories"] + del package["directories"], package["repositories"] + defaults = _deep_merge(package, defaults) # 4) User config user_cfg: dict[str, Any] = {} diff --git a/src/pkgmgr/core/git/queries/__init__.py b/src/pkgmgr/core/git/queries/__init__.py index 8eeced6..4fccb42 100644 --- a/src/pkgmgr/core/git/queries/__init__.py +++ b/src/pkgmgr/core/git/queries/__init__.py @@ -20,6 +20,7 @@ from .get_tags_at_ref import GitTagsAtRefQueryError, get_tags_at_ref from .get_upstream_ref import get_upstream_ref from .list_remotes import list_remotes from .list_tags import list_tags +from .local_config import FILTER_CONFIG, local_config from .probe_remote_reachable import ( probe_remote_reachable, probe_remote_reachable_detail, @@ -27,6 +28,7 @@ from .probe_remote_reachable import ( from .resolve_base_branch import GitBaseBranchNotFoundError, resolve_base_branch __all__ = [ + "FILTER_CONFIG", "GitBaseBranchNotFoundError", "GitChangelogQueryError", "GitLatestSigningKeyQueryError", @@ -46,6 +48,7 @@ __all__ = [ "get_upstream_ref", "list_remotes", "list_tags", + "local_config", "probe_remote_reachable", "probe_remote_reachable_detail", "resolve_base_branch", diff --git a/src/pkgmgr/core/git/queries/local_config.py b/src/pkgmgr/core/git/queries/local_config.py new file mode 100644 index 0000000..120bc03 --- /dev/null +++ b/src/pkgmgr/core/git/queries/local_config.py @@ -0,0 +1,35 @@ +from __future__ import annotations + +from ..errors import GitRunError +from ..run import run + +FILTER_CONFIG = r"^filter\." +LOCAL_SCOPES = ("local", "worktree") + + +def local_config(path: str, pattern: str) -> list[str]: + """ + List the config keys matching pattern that the repository itself sets. + + Args: + path: any directory inside the repository. + pattern: `git config --get-regexp` pattern over the canonical key names. + + Returns: + Each matching key set in the local or worktree scope, in config order; + global values never count, system config is not read. + """ + try: + out = run( + ["config", "--show-scope", "--get-regexp", pattern], cwd=path, scan=True + ) + except GitRunError as exc: + if exc.returncode == 1: + return [] + raise + keys: list[str] = [] + for line in out.splitlines(): + scope, _, rest = line.partition("\t") + if scope in LOCAL_SCOPES: + keys.append(rest.split(" ", 1)[0]) + return keys diff --git a/src/pkgmgr/core/git/run.py b/src/pkgmgr/core/git/run.py index 9fef59d..31bd175 100644 --- a/src/pkgmgr/core/git/run.py +++ b/src/pkgmgr/core/git/run.py @@ -36,6 +36,9 @@ SCAN_UNSET = ( "PAGER", ) SCAN_TIMEOUT = 60 +# ref and remote names come from repositories an agent may write: one named +# like --output= would otherwise make rev-list create that file +END_OF_OPTIONS = "--end-of-options" def _is_not_repo_error(stderr: str) -> bool: diff --git a/tests/integration/test_branch_gc.py b/tests/integration/test_branch_gc.py index 13e1d5a..9d1e5dc 100644 --- a/tests/integration/test_branch_gc.py +++ b/tests/integration/test_branch_gc.py @@ -7,6 +7,7 @@ import subprocess import tempfile import unittest from contextlib import redirect_stdout +from types import SimpleNamespace from unittest.mock import patch from pkgmgr.actions.branch import gc @@ -42,7 +43,7 @@ def docker_available() -> bool: return True -class TestBranchGc(unittest.TestCase): +class GcFixture: def setUp(self) -> None: self._tmp = tempfile.TemporaryDirectory() self.root = os.path.realpath(self._tmp.name) @@ -105,6 +106,9 @@ class TestBranchGc(unittest.TestCase): merged_into: list[str] | None = None, sizes: bool = False, min_age: float | None = None, + scope: str = "all", + activity: bool = False, + hooks: dict | None = None, ) -> gc.Repo: with ( patch.object(gc, "fetch_forge_facts", return_value=self.forge), @@ -117,6 +121,9 @@ class TestBranchGc(unittest.TestCase): merged_into=merged_into or [], sizes=sizes, min_age=min_age, + scope=scope, + activity=activity, + hooks=hooks or {}, ) def _integration_line(self) -> str: @@ -136,6 +143,8 @@ class TestBranchGc(unittest.TestCase): def _items(self, plan: gc.Repo) -> dict[str, gc.Item]: return {item.branch: item for item in plan.items if item.branch} + +class TestBranchGc(GcFixture, unittest.TestCase): def test_classifies_every_branch(self) -> None: items = self._items(self._plan(host_ns=True, cwds={})) @@ -379,7 +388,7 @@ class TestBranchGc(unittest.TestCase): self.assertEqual(git(self.repo, "for-each-ref", "refs/heads/-evil"), "") self.assertEqual(git(other, "for-each-ref", "refs/heads/-evil"), "") - def test_branch_gc_json_and_refusal(self) -> None: + def test_cleanup_json_and_refusal(self) -> None: out = io.StringIO() with ( patch.object(gc, "host_pid_namespace", return_value=True), @@ -388,8 +397,9 @@ class TestBranchGc(unittest.TestCase): patch.object(gc, "_process_cwds", return_value={}), redirect_stdout(out), ): - code = gc.branch_gc( + code = gc.cleanup( [self.repo], + scope="all", apply=False, as_json=True, include=["merged"], @@ -397,6 +407,7 @@ class TestBranchGc(unittest.TestCase): merged_into=["origin/main"], sizes=True, min_age=None, + hooks={}, ) data = json.loads(out.getvalue()) @@ -408,8 +419,9 @@ class TestBranchGc(unittest.TestCase): self.assertGreater(items[0]["size_bytes"], 0) with patch.object(gc, "host_pid_namespace", return_value=False): self.assertEqual( - gc.branch_gc( + gc.cleanup( [self.repo], + scope="all", apply=True, as_json=False, include=[], @@ -417,15 +429,16 @@ class TestBranchGc(unittest.TestCase): merged_into=[], sizes=False, min_age=None, + hooks={}, ), 2, ) - def test_cli_forwards_gc_flags(self) -> None: + def test_cli_forwards_cleanup_flags(self) -> None: args = create_parser("test").parse_args( [ "branch", - "gc", + "cleanup", "--apply", "--json", "--branch", @@ -443,16 +456,31 @@ class TestBranchGc(unittest.TestCase): "24", ] ) + pins = {"worktree-fini": ["true"]} + managed = { + "provider": "host", + "account": "o", + "repository": "r", + "directory": self.repo, + } + ctx = SimpleNamespace( + config_merged={ + "worktree_hooks": {"host/o/r": pins}, + "repositories": [managed], + } + ) with ( patch("pkgmgr.cli.commands.branch.main_repo_root", return_value=self.repo), - patch("pkgmgr.cli.commands.branch.branch_gc", return_value=1) as run_gc, + patch("pkgmgr.cli.commands.branch.cleanup", return_value=1) as run_gc, + patch("pkgmgr.cli.commands.branch.refresh_tree") as refresh, self.assertRaises(SystemExit) as exit_ctx, ): - handle_branch(args, ctx=None) + handle_branch(args, ctx=ctx) self.assertEqual(exit_ctx.exception.code, 1) run_gc.assert_called_once_with( [self.repo], + scope="branch", apply=True, as_json=True, include=["a", "b"], @@ -460,7 +488,9 @@ class TestBranchGc(unittest.TestCase): merged_into=["origin/line", "fork/line"], sizes=True, min_age=24.0, + hooks={os.path.realpath(self.repo): pins}, ) + refresh.assert_called_once_with([self.repo]) if __name__ == "__main__": diff --git a/tests/integration/test_branch_tree.py b/tests/integration/test_branch_tree.py index 9704235..90a400b 100644 --- a/tests/integration/test_branch_tree.py +++ b/tests/integration/test_branch_tree.py @@ -208,8 +208,9 @@ class TestBranchTree(unittest.TestCase): patch.object(gc, "fetch_forge_facts", return_value=ForgeFacts()), redirect_stdout(out), ): - gc.branch_gc( + gc.cleanup( [self.repo], + scope="all", apply=False, as_json=True, include=[], @@ -217,6 +218,7 @@ class TestBranchTree(unittest.TestCase): merged_into=[], sizes=False, min_age=None, + hooks={}, ) planned = { (item["branch"], item["worktree"]): ( diff --git a/tests/integration/test_worktree.py b/tests/integration/test_worktree.py new file mode 100644 index 0000000..67ed563 --- /dev/null +++ b/tests/integration/test_worktree.py @@ -0,0 +1,761 @@ +from __future__ import annotations + +import io +import json +import os +import shlex +import shutil +import sys +import time +import unittest +from contextlib import redirect_stderr, redirect_stdout +from types import SimpleNamespace +from unittest.mock import patch + +import yaml + +from pkgmgr.actions.branch import gc +from pkgmgr.actions.worktree import hooks, orphans, up +from pkgmgr.cli.commands.worktree import handle_gc, handle_worktree +from pkgmgr.cli.context import CLIContext +from pkgmgr.cli.dispatch import dispatch_command +from pkgmgr.cli.parser import create_parser +from pkgmgr.core.config.load import load_config +from tests.integration.test_branch_gc import GcFixture, git + + +class WorktreeFixture(GcFixture): + def _pushed_worktree(self, name: str, makefile: str | None) -> str: + git(self.repo, "checkout", "-b", name, "main") + if makefile is None: + self._commit(name) + else: + with open(os.path.join(self.repo, "Makefile"), "w") as fh: + fh.write(makefile) + git(self.repo, "add", "Makefile") + git(self.repo, "commit", "-m", f"{name} makefile") + git(self.repo, "push", "origin", name) + git(self.repo, "checkout", "main") + path = os.path.join(self.root, f"wt-{name}") + git(self.repo, "worktree", "add", path, name) + return path + + def _execute(self, plan: gc.Repo, item: gc.Item) -> None: + with patch.object(gc, "_process_cwds", return_value={}): + gc.execute(plan, item) + + def _pinned(self, hook: str, argv: list[str]) -> dict: + return {os.path.realpath(self.repo): {hook: argv}} + + +class TestCleanupScopes(WorktreeFixture, unittest.TestCase): + def test_worktree_scope_removes_pushed_worktree_and_keeps_branch(self) -> None: + path = self._pushed_worktree("pushed", None) + plan = self._plan(host_ns=True, cwds={}, scope="worktree") + items = self._items(plan) + + self.assertNotIn("squashed", items) + item = items["pushed"] + self.assertEqual(item.state, "unmerged") + self.assertEqual(item.unpushed, 0) + self.assertEqual(item.actions, ["worktree"]) + self.assertEqual(gc._label(item), "REMOVE") + self._execute(plan, item) + + self.assertEqual(item.result, "done") + self.assertFalse(os.path.exists(path)) + self.assertNotEqual(git(self.repo, "branch", "--list", "pushed"), "") + self.assertNotEqual(git(self.origin, "branch", "--list", "pushed"), "") + + def test_worktree_scope_keeps_unpushed_worktree(self) -> None: + path = self._pushed_worktree("ahead", None) + with open(os.path.join(path, "more.txt"), "w") as fh: + fh.write("more") + git(path, "add", "more.txt") + git(path, "commit", "-m", "more") + item = self._items(self._plan(host_ns=True, cwds={}, scope="worktree"))["ahead"] + + self.assertEqual(item.unpushed, 1) + self.assertEqual(item.actions, []) + self.assertFalse(item.eligible) + + def test_branch_scope_skips_checked_out_and_takes_free_branches(self) -> None: + items = self._items(self._plan(host_ns=True, cwds={}, scope="branch")) + + self.assertTrue(items["merged"].skip.startswith("checked out in worktree")) + self.assertEqual(items["merged"].actions, []) + self.assertEqual(gc._label(items["merged"]), "SKIP") + self.assertEqual( + items["squashed"].actions, ["local-branch", "remote-branch:origin"] + ) + self.assertTrue(items["squashed"].eligible) + + def test_patch_equivalent_branch_counts_as_merged(self) -> None: + git(self.repo, "checkout", "-b", "picked", "main") + self._commit("picked") + git(self.repo, "push", "origin", "picked") + sha = git(self.repo, "rev-parse", "HEAD") + git(self.repo, "checkout", "main") + self._commit("main-moved") + git(self.repo, "cherry-pick", sha) + git(self.repo, "push", "origin", "main") + item = self._items(self._plan(host_ns=True, cwds={}))["picked"] + + self.assertEqual(item.merged_by, "patch") + self.assertEqual(item.state, "merged-clean") + + def test_dotdot_behind_a_symlink_never_reaches_another_checkout(self) -> None: + victim = os.path.join(self.root, "victim") + os.makedirs(victim) + git(victim, "init", "--quiet", "-b", "main") + git(victim, "commit", "--quiet", "--allow-empty", "-m", "unpushed work") + git(self.repo, "branch", "evil", "main") + hop = os.path.join(self.repo, "A") + os.makedirs(os.path.join(hop, "victim")) + os.symlink(victim, os.path.join(hop, "L")) + meta = os.path.join(self.repo, ".git", "worktrees", "evil") + os.makedirs(meta) + files = { + os.path.join(hop, "victim", ".git"): f"gitdir: {meta}\n", + os.path.join(meta, "gitdir"): os.path.join( + hop, "L", "..", "victim", ".git" + ), + os.path.join(meta, "HEAD"): "ref: refs/heads/evil\n", + os.path.join(meta, "commondir"): "../..\n", + } + for name, text in files.items(): + with open(name, "w") as fh: + fh.write(text + ("" if text.endswith("\n") else "\n")) + plan = self._plan(host_ns=True, cwds={}, scope="worktree") + item = self._items(plan)["evil"] + + self._execute(plan, item) + + self.assertNotIn("worktree", item.actions) + self.assertTrue(os.path.isdir(os.path.join(victim, ".git"))) + + def test_pinned_fini_hook_runs_before_removal(self) -> None: + marker = os.path.join(self.root, "fini-ran") + path = self._pushed_worktree("hooked", None) + pins = self._fini_pins(["touch", marker]) + plan = self._plan(host_ns=True, cwds={}, scope="worktree", hooks=pins) + item = self._items(plan)["hooked"] + + self.assertEqual(item.actions, ["hook:worktree-fini", "worktree"]) + self._execute(plan, item) + + self.assertEqual(item.result, "done") + self.assertTrue(os.path.exists(marker)) + self.assertFalse(os.path.exists(path)) + + def _fini_pins(self, argv: list[str]) -> dict: + return self._pinned("worktree-fini", argv) + + def _cleanup_merged_forged_to(self, target: str) -> dict: + meta = os.path.join(self.repo, ".git", "worktrees", "wt-merged") + with open(os.path.join(meta, "gitdir"), "w") as fh: + fh.write(os.path.join(target, ".git") + "\n") + pins = self._fini_pins(["touch", "{path}/fini-ran"]) + out = io.StringIO() + with ( + patch.object(gc, "host_pid_namespace", return_value=True), + patch.object(gc, "list_compose_containers", return_value=[]), + patch.object(gc, "fetch_forge_facts", return_value=self.forge), + patch.object(gc, "_process_cwds", return_value={}), + redirect_stdout(out), + redirect_stderr(io.StringIO()), + ): + gc.cleanup( + [self.repo], + scope="worktree", + apply=True, + as_json=True, + include=["merged"], + exclude=[], + merged_into=[], + sizes=False, + min_age=None, + hooks=pins, + ) + return json.loads(out.getvalue())["repos"][0]["items"][0] + + def test_forged_gitdir_runs_nothing_in_another_checkout(self) -> None: + other = os.path.join(self.root, "other") + git(self.root, "clone", "--quiet", self.repo, other) + + item = self._cleanup_merged_forged_to(other) + + self.assertFalse(os.path.exists(os.path.join(other, "fini-ran"))) + self.assertEqual(item["worktree"], other) + self.assertEqual(item["skip"], gc.NOT_LINKED) + self.assertEqual(item["actions"], []) + self.assertTrue(os.path.isdir(os.path.join(other, ".git"))) + + def test_gitdir_forged_to_a_sibling_worktree_runs_nothing_there(self) -> None: + sibling = self._pushed_worktree("sibling", None) + + item = self._cleanup_merged_forged_to(sibling) + + self.assertFalse(os.path.exists(os.path.join(sibling, "fini-ran"))) + self.assertEqual(item["worktree"], sibling) + self.assertEqual(item["skip"], gc.NOT_LINKED) + self.assertEqual(item["actions"], []) + self.assertTrue(os.path.isdir(sibling)) + + def test_fini_hook_commit_keeps_worktree_and_branch(self) -> None: + commit = [ + "git", + "-c", + "user.name=t", + "-c", + "user.email=t@example.org", + "-c", + "commit.gpgsign=false", + "commit", + "--quiet", + "--allow-empty", + "-m", + "hook", + ] + plan = self._plan(host_ns=True, cwds={}, hooks=self._fini_pins(commit)) + item = self._items(plan)["merged"] + self.assertIn("local-branch", item.actions) + + self._execute(plan, item) + + self.assertEqual( + item.result, "skipped: changed during worktree-fini (HEAD moved)" + ) + self.assertTrue(os.path.isdir(self.merged_wt)) + self.assertNotEqual(git(self.repo, "rev-parse", "merged"), item.local_sha) + + def test_fini_hook_timeout_kills_its_process_group(self) -> None: + self._pushed_worktree("spawner", None) + marker = os.path.join(self.root, "grandchild-ran") + script = f"(sleep 1; touch {shlex.quote(marker)}) & sleep 30" + pins = self._fini_pins(["sh", "-c", script]) + plan = self._plan(host_ns=True, cwds={}, scope="worktree", hooks=pins) + item = self._items(plan)["spawner"] + + with patch.object(hooks, "HOOK_TIMEOUT", 0.3): + self._execute(plan, item) + time.sleep(1.5) + + self.assertEqual(item.result, "failed: worktree-fini timed out after 0.3 s") + self.assertFalse(os.path.exists(marker)) + + def test_fini_hook_timeout_fails_without_removal(self) -> None: + path = self._pushed_worktree("slow", None) + sleep = [sys.executable, "-c", "import time; time.sleep(30)"] + pins = self._fini_pins(sleep) + plan = self._plan(host_ns=True, cwds={}, scope="worktree", hooks=pins) + item = self._items(plan)["slow"] + + with patch.object(hooks, "HOOK_TIMEOUT", 0.5): + self._execute(plan, item) + + self.assertEqual(item.result, "failed: worktree-fini timed out after 0.5 s") + self.assertTrue(os.path.isdir(path)) + + def test_fini_hook_output_keeps_the_worktree(self) -> None: + path = self._pushed_worktree("dumps", None) + pins = self._fini_pins(["touch", "{path}/dump.sql"]) + plan = self._plan(host_ns=True, cwds={}, scope="worktree", hooks=pins) + item = self._items(plan)["dumps"] + + self._execute(plan, item) + + self.assertEqual( + item.result, "skipped: changed during worktree-fini (dirty: 1 paths)" + ) + self.assertTrue(os.path.isfile(os.path.join(path, "dump.sql"))) + + @unittest.skipUnless(shutil.which("make"), "make is not available") + def test_makefile_hooks_never_run_unpinned(self) -> None: + marker = os.path.join(self.root, "fini-ran") + planted = f"worktree-fini:\n\ttouch {marker}\n" + path = self._pushed_worktree("planted", makefile=planted) + with open(os.path.join(self.repo, ".git", "info", "exclude"), "a") as fh: + fh.write("GNUmakefile\n") + with open(os.path.join(path, "GNUmakefile"), "w") as fh: + fh.write(planted) + plan = self._plan(host_ns=True, cwds={}, scope="worktree") + item = self._items(plan)["planted"] + + self.assertEqual(item.actions, ["worktree"]) + self._execute(plan, item) + + self.assertEqual(item.result, "done") + self.assertFalse(os.path.exists(marker)) + + def test_activity_is_recorded_on_request(self) -> None: + quiet = self._items(self._plan(host_ns=True, cwds={})) + loud = self._items(self._plan(host_ns=True, cwds={}, activity=True)) + + self.assertIsNone(quiet["merged"].last_activity) + self.assertLessEqual(loud["merged"].last_activity, time.time()) + + +class TestWorktreeUp(WorktreeFixture, unittest.TestCase): + def setUp(self) -> None: + super().setUp() + self.worktrees = os.path.join(self.root, "wts") + + def _up( + self, branch: str, path: str | None, hooks: dict | None = None + ) -> tuple[int, dict]: + out = io.StringIO() + with redirect_stdout(out): + code = up.worktree_up( + self.repo, + branch, + base=None, + worktrees_dir=self.worktrees, + path=path, + hooks=hooks or {}, + as_json=True, + ) + return code, json.loads(out.getvalue()) if code != 2 else {} + + def _commit_file(self, name: str, text: str) -> None: + with open(os.path.join(self.repo, name), "w") as fh: + fh.write(text) + git(self.repo, "add", name) + git(self.repo, "commit", "-m", name) + git(self.repo, "push", "origin", "main") + + def _commit_script(self, name: str) -> str: + marker = os.path.join(self.root, f"{name}-ran") + script = os.path.join(self.repo, name) + with open(script, "w") as fh: + fh.write(f"#!/bin/sh\ntouch {marker}\n") + os.chmod(script, 0o755) + git(self.repo, "add", name) + git(self.repo, "commit", "-m", name) + git(self.repo, "push", "origin", "main") + return marker + + def _pins(self, argv: list[str]) -> dict: + return self._pinned("worktree-init", argv) + + def test_slot_in_branch_name_stays_literal(self) -> None: + write_path = "import sys; open('path.txt', 'w').write(sys.argv[1])" + pins = self._pins([sys.executable, "-c", write_path, "{path}"]) + + code, result = self._up("feature/x{slot}", None, pins) + + self.assertEqual(code, 0) + self.assertTrue(result["path"].endswith("/feature/x{slot}")) + with open(os.path.join(result["path"], "path.txt")) as fh: + self.assertEqual(fh.read(), result["path"]) + + @unittest.skipUnless(shutil.which("make"), "make is not available") + def test_empty_path_entry_never_runs_a_planted_command(self) -> None: + marker = self._commit_script("make") + search = os.pathsep + os.environ["PATH"] + + with patch.dict(os.environ, {"PATH": search}): + code, result = self._up("feature/p", None, self._pins(["make", "-v"])) + + self.assertEqual(code, 0) + self.assertEqual(result["hook"], "worktree-init") + self.assertFalse(os.path.exists(marker)) + + def test_bare_pin_skips_a_path_entry_another_user_can_write(self) -> None: + planted = os.path.join(self.root, "planted-bin") + os.makedirs(planted) + marker = os.path.join(self.root, "planted-ran") + fake = os.path.join(planted, "true") + with open(fake, "w") as fh: + fh.write(f"#!/bin/sh\ntouch {marker}\n") + os.chmod(fake, 0o755) + if os.getuid() == 0: + os.chown(planted, 65534, 65534) + os.chown(fake, 65534, 65534) + search = planted + os.pathsep + os.environ["PATH"] + + with patch.dict(os.environ, {"PATH": search}): + code, result = self._up("feature/t", None, self._pins(["true"])) + + self.assertEqual((code, result["hook"]), (0, "worktree-init")) + self.assertFalse(os.path.exists(marker)) + + def test_relative_pin_is_refused_before_checkout(self) -> None: + marker = self._commit_script("init.sh") + + code, _ = self._up("feature/r", None, self._pins(["./init.sh"])) + + self.assertEqual(code, 2) + self.assertFalse(os.path.exists(marker)) + self.assertFalse(os.path.exists(self.worktrees)) + + def test_init_hook_timeout_fails_and_keeps_worktree(self) -> None: + pins = self._pins([sys.executable, "-c", "import time; time.sleep(30)"]) + + with patch.object(hooks, "HOOK_TIMEOUT", 0.5): + code, result = self._up("feature/t", None, pins) + + self.assertEqual(code, 1) + self.assertEqual( + result["hook"], + "worktree-init failed (worktree-init timed out after 0.5 s)", + ) + self.assertTrue(os.path.isdir(result["path"])) + + def test_location_of_remote_urls(self) -> None: + cases = { + "git@github.com:owner/repo.git": "github.com/owner/repo", + "https://github.com/owner/repo": "github.com/owner/repo", + "ssh://git@host:2201/owner/repo.git": "host/owner/repo", + "/tmp/x/origin.git": "tmp/x/origin", + } + for url, expected in cases.items(): + with self.subTest(url=url): + self.assertEqual(up.worktree_location(url), expected) + + def test_new_branches_get_path_slots_and_pinned_init_hook(self) -> None: + write_slot = "import sys; open('slot.txt', 'w').write(sys.argv[1])" + hooks = self._pins([sys.executable, "-c", write_slot, "{slot}"]) + + code, first = self._up("feature/x", None, hooks) + expected = os.path.join( + self.worktrees, up.worktree_location(self.origin), "feature/x" + ) + self.assertEqual(code, 0) + self.assertEqual(first["path"], expected) + self.assertEqual(first["slot"], 1) + self.assertEqual(first["hook"], "worktree-init") + self.assertTrue(first["origin"].startswith("new from origin/main")) + with open(os.path.join(expected, "slot.txt")) as fh: + self.assertEqual(fh.read().strip(), "1") + + _, second = self._up("feature/y", None, hooks) + self.assertEqual(second["slot"], 2) + self.assertEqual(self._up("feature/x", None, hooks)[0], 2) + + @unittest.skipUnless(shutil.which("make"), "make is not available") + def test_unpinned_makefile_is_never_run_or_probed(self) -> None: + parsed = os.path.join(self.root, "parsed") + ran = os.path.join(self.root, "ran") + self._commit_file( + "Makefile", f"X := $(shell touch {parsed})\nworktree-init:\n\ttouch {ran}\n" + ) + + code, result = self._up("feature/m", None) + + self.assertEqual(code, 0) + self.assertEqual(result["hook"], "not pinned: worktree-init in Makefile") + self.assertFalse(os.path.exists(parsed)) + self.assertFalse(os.path.exists(ran)) + + def test_origin_url_cannot_escape_the_worktrees_dir(self) -> None: + git(self.repo, "remote", "set-url", "origin", "https://host/../../escape") + + self.assertEqual(self._up("feature/e", None)[0], 2) + self.assertFalse(os.path.exists(os.path.join(self.root, "escape"))) + + def test_default_base_named_like_an_option_is_a_ref(self) -> None: + git(self.repo, "checkout", "-b", "side", "main") + self._commit("side") + sha = git(self.repo, "rev-parse", "HEAD") + git(self.repo, "checkout", "main") + git(self.repo, "update-ref", "refs/heads/-f", sha) + git(self.repo, "symbolic-ref", "refs/remotes/origin/HEAD", "refs/heads/-f") + + code, result = self._up("feature/o", None) + + self.assertEqual(code, 0) + self.assertEqual(git(result["path"], "rev-parse", "HEAD"), sha) + + def test_local_filter_config_blocks_the_checkout(self) -> None: + marker = os.path.join(self.root, "smudged") + self._commit_file(".gitattributes", "* filter=evil\n") + git(self.repo, "config", "filter.evil.smudge", f"touch {marker}; cat") + + self.assertEqual(self._up("feature/s", None)[0], 2) + self.assertFalse(os.path.exists(marker)) + + def test_local_ssh_command_skips_the_fetch(self) -> None: + marker = os.path.join(self.root, "ssh-ran") + git(self.repo, "config", "core.sshCommand", f"touch {marker}; false") + git(self.repo, "config", "remote.origin.pushurl", self.origin) + git(self.repo, "remote", "set-url", "origin", "ssh://example.invalid/r.git") + + code, result = self._up("feature/f", None) + + self.assertEqual(code, 0) + self.assertFalse(os.path.exists(marker)) + self.assertIn("core.sshcommand", " ".join(result["warnings"])) + + def test_remote_branch_is_tracked(self) -> None: + git(self.repo, "checkout", "-b", "remote-only", "main") + self._commit("remote-only") + git(self.repo, "push", "origin", "remote-only") + git(self.repo, "checkout", "main") + git(self.repo, "branch", "-D", "remote-only") + + code, result = self._up("remote-only", None) + + self.assertEqual(code, 0) + self.assertEqual(result["origin"], "tracking origin/remote-only") + self.assertEqual(result["hook"], "none") + self.assertEqual( + git(result["path"], "rev-parse", "--abbrev-ref", "@{u}"), + "origin/remote-only", + ) + + def test_explicit_path_replaces_standard_path(self) -> None: + released = os.path.join(self.root, "released", "feature-z") + + code, result = self._up("feature/z", released) + + self.assertEqual(code, 0) + self.assertEqual(result["path"], released) + self.assertEqual( + git(released, "rev-parse", "--abbrev-ref", "HEAD"), "feature/z" + ) + self.assertFalse(os.path.exists(self.worktrees)) + + def test_invalid_branch_is_refused(self) -> None: + self.assertEqual(self._up("../evil", None)[0], 2) + self.assertFalse(os.path.exists(self.worktrees)) + + +class TestWorktreeOrphans(WorktreeFixture, unittest.TestCase): + def _report(self, cwds: dict[int, str]) -> dict[str, dict]: + out = io.StringIO() + with ( + patch.object(gc, "host_pid_namespace", return_value=True), + patch.object(gc, "list_compose_containers", return_value=[]), + patch.object(gc, "fetch_forge_facts", return_value=self.forge), + patch.object(gc, "_process_cwds", return_value=cwds), + redirect_stdout(out), + ): + code = orphans.worktree_orphans( + [self.repo], + stale_days=0, + top=5, + as_json=True, + include=[], + exclude=[], + merged_into=[], + sizes=False, + ) + self.assertEqual(code, 0) + rows = json.loads(out.getvalue())["repos"][0]["rows"] + return {row["branch"]: row for row in rows if row["branch"]} + + def test_classes_and_proposals(self) -> None: + git(self.repo, "checkout", "-b", "lonely", "main") + self._commit("lonely") + git(self.repo, "checkout", "main") + time.sleep(1.1) + rows = self._report({}) + + self.assertEqual(rows["main"]["class"], "active") + self.assertEqual(rows["merged"]["class"], "merged-clean") + self.assertEqual(rows["merged"]["proposal"], "pkgmgr gc") + self.assertEqual(rows["dirty"]["class"], "merged-dirty") + self.assertEqual(rows["lonely"]["class"], "stale-open") + self.assertEqual(rows["lonely"]["unpushed"], 1) + self.assertEqual(rows["wip"]["class"], "active") + + def test_busy_worktree_is_kept(self) -> None: + rows = self._report({4242: self.merged_wt}) + + self.assertEqual(rows["merged"]["class"], "merged-clean") + self.assertTrue( + rows["merged"]["proposal"].startswith("keep (process: pid 4242") + ) + + +class TestLoadedConfigCli(WorktreeFixture, unittest.TestCase): + HOOKS = ("worktree-init", "worktree-fini") + + def setUp(self) -> None: + super().setUp() + self.addCleanup(os.chdir, os.getcwd()) + self.markers = {h: os.path.join(self.root, f"{h}-ran") for h in self.HOOKS} + + def _ctx(self, directory: str, pinned: bool) -> CLIContext: + location = up.worktree_location(self.origin) + provider, _, rest = location.partition("/") + account, _, repository = rest.rpartition("/") + config: dict = { + "directories": {"worktrees": os.path.join(self.root, "wts")}, + "repositories": [ + { + "provider": provider, + "account": account, + "repository": repository, + "directory": directory, + } + ], + } + if pinned: + config["worktree_hooks"] = { + location: {h: ["touch", m] for h, m in self.markers.items()} + } + config_dir = os.path.join(self.root, "config") + os.makedirs(config_dir, exist_ok=True) + user_config = os.path.join(config_dir, "config.yaml") + with open(user_config, "w") as fh: + yaml.safe_dump(config, fh) + merged = load_config(user_config) + return CLIContext( + config_merged=merged, + repositories_base_dir=os.path.join(self.root, "repos"), + all_repositories=merged["repositories"], + binaries_dir=os.path.join(self.root, "bin"), + user_config_path=user_config, + ) + + def _run(self, argv: list[str], ctx: CLIContext) -> tuple[int, str]: + os.chdir(self.repo) + out = io.StringIO() + with ( + patch.object(gc, "host_pid_namespace", return_value=True), + patch.object(gc, "list_compose_containers", return_value=[]), + patch.object(gc, "fetch_forge_facts", return_value=self.forge), + patch.object(gc, "_process_cwds", return_value={}), + patch.object(gc, "_docker", return_value=""), + patch("pkgmgr.cli.commands.branch.refresh_tree"), + patch("pkgmgr.cli.commands.worktree.refresh_tree"), + redirect_stdout(out), + self.assertRaises(SystemExit) as exit_ctx, + ): + dispatch_command(create_parser("test").parse_args(argv), ctx) + return exit_ctx.exception.code, out.getvalue() + + def test_gc_dry_run_with_config_lacking_worktree_hooks(self) -> None: + code, out = self._run(["gc", "--json"], self._ctx(self.repo, pinned=False)) + + self.assertEqual(code, 0) + self.assertEqual(json.loads(out)["repos"][0]["repo"], self.repo) + + def test_origin_url_of_a_managed_repository_selects_no_pin(self) -> None: + managed = os.path.join(self.root, "managed") + git(self.root, "clone", "--quiet", self.repo, managed) + ctx = self._ctx(managed, pinned=True) + + code, out = self._run(["worktree", "up", "feature/a", "--json"], ctx) + self.assertEqual((code, json.loads(out)["hook"]), (0, "none")) + self.assertEqual(self._run(["gc", "--apply", "--branch", "merged"], ctx)[0], 0) + + self.assertFalse(os.path.exists(self.markers["worktree-init"])) + self.assertFalse(os.path.exists(self.markers["worktree-fini"])) + self.assertFalse(os.path.exists(self.merged_wt)) + + def test_pinned_directory_borrowing_another_git_dir_runs_no_pin(self) -> None: + borrower = os.path.join(self.root, "borrower") + git(self.root, "init", "--quiet", "-b", "main", borrower) + with open(os.path.join(borrower, ".git", "commondir"), "w") as fh: + fh.write(os.path.join(self.repo, ".git") + "\n") + ctx = self._ctx(borrower, pinned=True) + + code, out = self._run( + ["gc", "--all", "--apply", "--json", "--branch", "merged"], ctx + ) + + warnings = json.loads(out)["repos"][0]["warnings"] + self.assertEqual(code, 0) + self.assertTrue(any("pinned hooks skipped" in w for w in warnings)) + self.assertFalse(os.path.exists(self.markers["worktree-fini"])) + + def test_managed_repository_runs_its_pins(self) -> None: + ctx = self._ctx(self.repo, pinned=True) + + code, out = self._run(["worktree", "up", "feature/b", "--json"], ctx) + self.assertEqual((code, json.loads(out)["hook"]), (0, "worktree-init")) + self.assertEqual(self._run(["gc", "--apply", "--branch", "merged"], ctx)[0], 0) + + self.assertTrue(os.path.exists(self.markers["worktree-init"])) + self.assertTrue(os.path.exists(self.markers["worktree-fini"])) + self.assertFalse(os.path.exists(self.merged_wt)) + + +class TestWorktreeCli(unittest.TestCase): + def test_branch_gc_is_gone(self) -> None: + with self.assertRaises(SystemExit): + create_parser("test").parse_args(["branch", "gc"]) + + def test_up_forwards_to_worktree_up(self) -> None: + args = create_parser("test").parse_args( + [ + "worktree", + "up", + "feature/x", + "--base", + "origin/main", + "--path", + "/srv/wt/x", + "--json", + ] + ) + pins = {"worktree-init": ["true"]} + ctx = SimpleNamespace( + repositories_base_dir="/srv", + config_merged={ + "directories": {"worktrees": "~/wt"}, + "repositories": [ + {"provider": "host", "account": "o", "repository": "r"} + ], + "worktree_hooks": {"host/o/r": pins, "host/o/unmanaged": pins}, + }, + ) + with ( + patch("pkgmgr.cli.commands.worktree.worktree_up", return_value=0) as run_up, + patch("pkgmgr.cli.commands.worktree.refresh_tree") as refresh, + self.assertRaises(SystemExit) as exit_ctx, + ): + handle_worktree(args, ctx) + + self.assertEqual(exit_ctx.exception.code, 0) + run_up.assert_called_once_with( + ".", + "feature/x", + base="origin/main", + worktrees_dir="~/wt", + path="/srv/wt/x", + hooks={os.path.realpath("/srv/host/o/r"): pins}, + as_json=True, + ) + refresh.assert_called_once() + + def test_cleanup_and_gc_pick_their_scope(self) -> None: + for argv, handler, scope in ( + (["worktree", "cleanup", "--apply"], handle_worktree, "worktree"), + (["gc", "--apply"], handle_gc, "all"), + ): + with self.subTest(scope=scope): + args = create_parser("test").parse_args(argv) + ctx = SimpleNamespace( + config_merged={"worktree_hooks": {}, "repositories": []} + ) + with ( + patch( + "pkgmgr.cli.commands.branch.main_repo_root", return_value="/r" + ), + patch( + "pkgmgr.cli.commands.branch.cleanup", return_value=0 + ) as run_cleanup, + patch("pkgmgr.cli.commands.branch.refresh_tree") as refresh, + self.assertRaises(SystemExit), + ): + handler(args, ctx) + self.assertEqual(run_cleanup.call_args.kwargs["scope"], scope) + self.assertTrue(run_cleanup.call_args.kwargs["apply"]) + self.assertEqual(run_cleanup.call_args.kwargs["hooks"], {}) + refresh.assert_called_once_with(["/r"]) + + def test_orphans_requires_explicit_limits(self) -> None: + with self.assertRaises(SystemExit): + create_parser("test").parse_args(["worktree", "orphans"]) + args = create_parser("test").parse_args( + ["worktree", "orphans", "--stale-days", "7", "--top", "5"] + ) + self.assertEqual((args.stale_days, args.top), (7, 5)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/pkgmgr/actions/worktree/__init__.py b/tests/unit/pkgmgr/actions/worktree/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/unit/pkgmgr/actions/worktree/test_hooks.py b/tests/unit/pkgmgr/actions/worktree/test_hooks.py new file mode 100644 index 0000000..958570e --- /dev/null +++ b/tests/unit/pkgmgr/actions/worktree/test_hooks.py @@ -0,0 +1,46 @@ +from __future__ import annotations + +import unittest +from unittest.mock import patch + +from pkgmgr.actions.worktree import hooks + + +class TestHookCommand(unittest.TestCase): + def _command(self, argv: list[str]) -> list[str]: + return hooks.hook_command(hooks.INIT_HOOK, argv, "/worktree", 3) + + def test_bare_pin_runs_the_path_root_resolved_returned(self) -> None: + with ( + patch.object(hooks, "trusted_path", return_value="/usr/bin"), + patch.object(hooks.shutil, "which", return_value="/usr/bin/sh"), + patch.object(hooks, "root_resolved", return_value="/usr/bin/dash"), + ): + self.assertEqual( + self._command(["sh", "{path}", "{slot}"]), + ["/usr/bin/dash", "/worktree", "3"], + ) + + def test_bare_pin_to_a_file_not_root_controlled_is_refused(self) -> None: + with ( + patch.object(hooks, "trusted_path", return_value="/usr/bin"), + patch.object(hooks.shutil, "which", return_value="/usr/bin/sh"), + patch.object(hooks, "root_resolved", return_value=None), + self.assertRaises(RuntimeError), + ): + self._command(["sh"]) + + def test_relative_pin_with_a_slash_is_refused(self) -> None: + with self.assertRaises(RuntimeError): + self._command(["./init.sh"]) + + def test_empty_pin_is_refused(self) -> None: + with self.assertRaises(RuntimeError): + self._command([]) + + def test_absolute_pin_is_kept_verbatim(self) -> None: + self.assertEqual(self._command(["/usr/bin/sh"]), ["/usr/bin/sh"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/pkgmgr/actions/worktree/test_trust.py b/tests/unit/pkgmgr/actions/worktree/test_trust.py new file mode 100644 index 0000000..6799a12 --- /dev/null +++ b/tests/unit/pkgmgr/actions/worktree/test_trust.py @@ -0,0 +1,162 @@ +from __future__ import annotations + +import os +import shutil +import tempfile +import unittest +from types import SimpleNamespace +from unittest.mock import patch + +from pkgmgr.actions.worktree import trust + +ROOT_DIR = 0o040755 + + +class TestRootUids(unittest.TestCase): + def _uids( + self, uid_map: str, overflow: str | None = "65534\n", euid: int = 1000 + ) -> frozenset[int]: + with tempfile.TemporaryDirectory() as tmp: + mapping = os.path.join(tmp, "uid_map") + overflow_file = os.path.join(tmp, "overflowuid") + with open(mapping, "w") as fh: + fh.write(uid_map) + if overflow is not None: + with open(overflow_file, "w") as fh: + fh.write(overflow) + with ( + patch.object(trust, "UID_MAP", mapping), + patch.object(trust, "OVERFLOW_UID", overflow_file), + patch.object(trust.os, "geteuid", return_value=euid), + ): + return trust.root_uids() + + def test_initial_namespace_trusts_only_uid_0(self) -> None: + self.assertEqual(self._uids(" 0 0 4294967295\n"), {0}) + + def test_namespace_without_uid_0_trusts_only_the_overflow_uid(self) -> None: + self.assertEqual(self._uids("1000 1000 1\n"), {65534}) + + def test_namespace_mapping_uid_0_to_a_user_trusts_only_the_overflow_uid( + self, + ) -> None: + self.assertEqual(self._uids("0 1000 1\n"), {65534}) + + def test_nested_namespace_mapping_a_user_to_parent_0_trusts_only_overflow( + self, + ) -> None: + self.assertEqual(self._uids("1000 0 1\n"), {65534}) + + def test_own_uid_unmapped_trusts_no_uid(self) -> None: + self.assertEqual(self._uids("", euid=65534), frozenset()) + + def test_unreadable_map_trusts_no_uid(self) -> None: + with patch.object(trust, "UID_MAP", "/nonexistent/uid_map"): + self.assertEqual(trust.root_uids(), frozenset()) + + def test_unreadable_overflow_uid_trusts_no_uid(self) -> None: + self.assertEqual(self._uids("1000 1000 1\n", overflow=None), frozenset()) + + def test_malformed_map_trusts_no_uid(self) -> None: + self.assertEqual(self._uids("0 zero 1\n"), frozenset()) + + +class TestRootResolved(unittest.TestCase): + PATH = "/usr/bin/tool" + + def _resolve( + self, + stats: dict[str, tuple[int, int]], + owners: frozenset[int] = frozenset({0}), + nosuid: frozenset[str] = frozenset(), + ) -> str | None: + def fake_stat(path: str) -> SimpleNamespace: + if path not in stats: + raise FileNotFoundError(path) + uid, mode = stats[path] + return SimpleNamespace(st_uid=uid, st_mode=mode) + + def fake_statvfs(path: str) -> SimpleNamespace: + return SimpleNamespace(f_flag=os.ST_NOSUID if path in nosuid else 0) + + with ( + patch.object(trust, "root_uids", return_value=owners), + patch.object(trust.os.path, "realpath", side_effect=lambda p: p), + patch.object(trust.os, "stat", side_effect=fake_stat), + patch.object(trust.os, "statvfs", side_effect=fake_statvfs), + ): + return trust.root_resolved(self.PATH) + + def _tree(self) -> dict[str, tuple[int, int]]: + return { + "/": (0, ROOT_DIR), + "/usr": (0, ROOT_DIR), + "/usr/bin": (0, ROOT_DIR), + self.PATH: (0, 0o100755), + } + + def test_root_owned_chain_returns_the_resolved_path(self) -> None: + self.assertEqual(self._resolve(self._tree()), self.PATH) + + def test_non_root_ancestor_is_refused(self) -> None: + stats = self._tree() + stats["/usr"] = (1000, ROOT_DIR) + self.assertIsNone(self._resolve(stats)) + + def test_group_writable_leaf_is_refused(self) -> None: + stats = self._tree() + stats[self.PATH] = (0, 0o100775) + self.assertIsNone(self._resolve(stats)) + + def test_other_writable_ancestor_is_refused(self) -> None: + stats = self._tree() + stats["/usr/bin"] = (0, 0o040757) + self.assertIsNone(self._resolve(stats)) + + def test_stat_error_is_refused(self) -> None: + stats = self._tree() + del stats["/usr"] + self.assertIsNone(self._resolve(stats)) + + def test_nosuid_component_is_refused_in_the_initial_namespace(self) -> None: + self.assertIsNone(self._resolve(self._tree(), nosuid=frozenset({"/usr"}))) + + def test_nosuid_is_ignored_inside_another_namespace(self) -> None: + stats = {path: (65534, mode) for path, (_, mode) in self._tree().items()} + everything = frozenset(stats) + self.assertEqual( + self._resolve(stats, owners=frozenset({65534}), nosuid=everything), + self.PATH, + ) + + +class TestTrustedPath(unittest.TestCase): + def test_symlinked_entry_is_kept_only_resolved(self) -> None: + system = os.path.dirname(os.path.realpath(shutil.which("true"))) + with tempfile.TemporaryDirectory() as tmp: + link = os.path.join(tmp, "bin") + os.symlink(system, link) + with patch.dict(os.environ, {"PATH": link}): + self.assertEqual(trust.trusted_path(), system) + + def test_entry_keeps_the_resolution_it_was_checked_under(self) -> None: + system = os.path.dirname(os.path.realpath(shutil.which("true"))) + with tempfile.TemporaryDirectory() as tmp: + link = os.path.join(tmp, "bin") + os.symlink(system, link) + with ( + patch.dict(os.environ, {"PATH": link}), + patch.object(trust.os.path, "realpath", side_effect=[system, tmp]), + ): + self.assertEqual(trust.trusted_path(), system) + + def test_no_root_controlled_entry_is_refused(self) -> None: + with ( + patch.object(trust, "root_resolved", return_value=None), + self.assertRaises(RuntimeError), + ): + trust.trusted_path() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/pkgmgr/core/config/test_load.py b/tests/unit/pkgmgr/core/config/test_load.py index 4ff0198..3244067 100644 --- a/tests/unit/pkgmgr/core/config/test_load.py +++ b/tests/unit/pkgmgr/core/config/test_load.py @@ -18,6 +18,8 @@ from pkgmgr.core.config.load import ( load_config, ) +LAYER_REPO = {"provider": "p", "account": "a", "repository": "r"} + class DeepMergeTests(unittest.TestCase): def test_deep_merge_overrides_scalars_and_merges_dicts(self): @@ -267,5 +269,42 @@ class LoadConfigIntegrationUnitTests(unittest.TestCase): self.assertIsInstance(merged["repositories"], list) +class PackageDefaultKeysTests(unittest.TestCase): + def _load(self, files: dict[str, dict]) -> dict: + with tempfile.TemporaryDirectory() as td: + for name, data in files.items(): + (Path(td) / name).write_text(yaml.safe_dump(data), encoding="utf-8") + return load_config(str(Path(td) / "config.yaml")) + + def test_empty_user_dir_gets_worktree_hooks(self): + self.assertEqual(self._load({})["worktree_hooks"], {}) + + def test_layer_with_only_repositories_keeps_worktree_hooks(self): + merged = self._load({"defaults.yaml": {"repositories": [LAYER_REPO]}}) + + self.assertEqual(merged["worktree_hooks"], {}) + self.assertEqual(merged["repositories"][0]["repository"], "r") + + def test_layers_deep_merge_other_keys(self): + merged = self._load( + { + "a.yaml": { + "repositories": [LAYER_REPO], + "worktree_hooks": {"h/o/a": {"worktree-init": ["a"]}}, + }, + "b.yaml": {"worktree_hooks": {"h/o/b": {"worktree-init": ["b"]}}}, + } + ) + + self.assertEqual(set(merged["worktree_hooks"]), {"h/o/a", "h/o/b"}) + + def test_user_config_sets_worktree_hooks(self): + hooks = {"h/o/r": {"worktree-fini": ["/bin/true"]}} + + merged = self._load({"config.yaml": {"worktree_hooks": hooks}}) + + self.assertEqual(merged["worktree_hooks"], hooks) + + if __name__ == "__main__": unittest.main()