Merge branch 'fix/hub-cleanup-pkgmgr-local' into unify/skills-20261114

This commit is contained in:
Kevin Veen-Birkenbach committed 2026-11-14 21:52:08 +00:00
commit 568747651b
19 files changed
+1534 -189

No files matched your search

+217 -66
View File
@@ -15,18 +15,28 @@
Returns:
Runs once over every job in --jobs, as the operator on the host: per item
it verifies the operator's passkey signature over repository, branch,
worktree and tip, then in the repository runs `pkgmgr branch cleanup
--json --branch <name>` as a dry run and, only when that plan names
exactly this branch as merged-clean without skip reason at the signed
tip and worktree and deletes no remote branch, `pkgmgr branch cleanup
--apply --json --branch <name>`. While the tree file says pkgmgr has no GitHub token
(delete_ready false), only a branch the dry run shows merged by ancestry
into the default branch and without a remote branch is deleted; the
others are refused with "token needed". Every other item is refused and
nothing changes for it. An applied item is done only when pkgmgr's
result for it is done; a skipped one or one without result is failed
with pkgmgr's reason. Writes each item's result, rewrites the tree file
after a deletion and removes the job. The exit code is 0.
worktree and tip; then per repository it runs one `pkgmgr branch cleanup
--local --json --branch <name>...` dry run over its signed branches and
one `pkgmgr branch cleanup --apply --local --json --branch <name>...`
over those the plan names exactly at the signed tip and worktree and
deletes exactly the local branch of (signing.local_keeps, the predicate
the hub offers by); pkgmgr deletes only a local branch whose planned tip
is an ancestor of the default branch, by update-ref with that tip. While
the tree file says pkgmgr has no GitHub token (delete_ready false), only
a branch the dry run shows merged by ancestry into the default branch
and without a remote branch is deleted; the others are refused with
"token needed". Every other item is refused and nothing changes for it.
An applied item is done only when pkgmgr's result for it is done, with a
warning when pkgmgr deleted it at another merged tip than the signed
one; one the apply keeps is failed with pkgmgr's skip reason. A signed
branch pkgmgr did not report deleted whose ref is gone is gone, never
done: "deleted outside this run" when pkgmgr answered without deleting
it (a failed result, no result, or "Unknown --branch"), "gone after an
interrupted apply" when the apply ended without JSON; a present one is
failed. When a dry run or apply answers "Unknown --branch" for gone
branches, it runs once more without them.
Writes each item's result, rewrites the tree file after a deletion or a
gone branch and removes the job. The exit code is 0.
"""
from __future__ import annotations
@@ -49,7 +59,10 @@ UNSAFE = re.compile(r"[*?\[\]\\\s\x00-\x1f\x7f]|^-|\.\.|@\{")
OUTPUT_LIMIT = 16 << 10
TIMEOUT = 600
DONE = "done"
REMOTE_ACTION = "remote-branch:"
GONE = "gone"
DELETED_OUTSIDE = "deleted outside this run"
GONE_INTERRUPTED = "gone after an interrupted apply"
UNKNOWN = re.compile(r"^Unknown --branch: (.*)$", re.MULTILINE)
TOKEN_NEEDED = "token needed: without the GitHub token only a local branch merged by ancestry into the default branch and without a remote branch is deleted"
# the cwd is a repository agents can write: its config may name hooks, and
# pkgmgr releases before 4c8b137 run git branch -D without scan mode
@@ -84,23 +97,57 @@ def tree_facts(tree: Path) -> tuple[dict[str, str], bool]:
return {}, False
def run(pkgmgr: Path, argv: list[str], cwd: str) -> tuple[int, str, str]:
def run(pkgmgr: Path, argv: list[str], cwd: str, timeout: int) -> tuple[int, str, str]:
"""Args:
pkgmgr: the pkgmgr executable.
argv: its arguments.
cwd: the repository it runs in, without hooks or fsmonitor.
timeout: seconds before pkgmgr is killed.
Returns:
Exit code, stdout and the tail of stderr; 127 with the error when
pkgmgr could not start or was killed at the timeout.
"""
try:
done = subprocess.run([str(pkgmgr), *argv], cwd=cwd, env={**os.environ, **git_overrides()}, stdin=subprocess.DEVNULL, capture_output=True, text=True, errors="replace", timeout=TIMEOUT, check=False)
done = subprocess.run([str(pkgmgr), *argv], cwd=cwd, env={**os.environ, **git_overrides()}, stdin=subprocess.DEVNULL, capture_output=True, text=True, errors="replace", timeout=timeout, check=False)
except (OSError, subprocess.TimeoutExpired) as error:
return 127, "", str(error)
return done.returncode, done.stdout, done.stderr[-OUTPUT_LIMIT:]
def plan_argv(branch: str, apply: bool) -> list[str]:
def branch_tip(repository: str, branch: str) -> str | None:
"""Returns:
The local branch's tip as git reads it in the repository and no
parent of it, "" when the repository has no such branch, None when
git cannot tell.
"""
env = {**os.environ, **git_overrides(), "GIT_CEILING_DIRECTORIES": str(Path(repository).parent)}
try:
done = subprocess.run(["git", "rev-parse", "--verify", "--quiet", f"refs/heads/{branch}"], cwd=repository, env=env, stdin=subprocess.DEVNULL, capture_output=True, text=True, timeout=60, check=False)
except (OSError, subprocess.TimeoutExpired):
return None
if done.returncode == 0 and TIP.fullmatch(done.stdout.strip()):
return done.stdout.strip()
return "" if done.returncode == 1 and not done.stdout.strip() else None
def unknown(stderr: str) -> set[str]:
"""Returns:
The branch names pkgmgr's "Unknown --branch: a, b" line names.
"""
return {name for found in UNKNOWN.findall(stderr) for name in found.split(", ")}
def plan_argv(branches: list[str], apply: bool) -> list[str]:
"""Args:
branch: the exact branch name.
branches: the exact branch names of one repository.
apply: execute the plan instead of the dry run.
Returns:
The `pkgmgr branch cleanup` arguments for this branch's plan in JSON.
The `pkgmgr branch cleanup --local` arguments for these branches'
plan in JSON: local branches only, no forge token, no remote delete.
"""
return ["branch", "cleanup", *(["--apply"] if apply else []), "--json", "--branch", branch]
return ["branch", "cleanup", *(["--apply"] if apply else []), "--local", "--json", *[part for branch in branches for part in ("--branch", branch)]]
def plan_items(stdout: str) -> list[dict] | None:
@@ -126,20 +173,85 @@ def shaped(item: object) -> bool:
return item["repository"].startswith("/") and bool(TIP.fullmatch(item["tip"])) and (item["worktree"] == "" or item["worktree"].startswith("/"))
def mismatch(entry: dict, item: dict) -> str | None:
if entry.get("state") != "merged-clean":
return f"state is {entry.get('state')}, not merged-clean"
if entry.get("skip"):
return f"gc skip reason: {entry['skip']}"
def tip_changed(entry: dict, item: dict) -> str | None:
tip = entry.get("local_sha") or entry.get("head")
if tip != item["tip"]:
return f"tip changed: signed {item['tip'][:12]}, now {str(tip)[:12]}"
return None if tip == item["tip"] else f"tip changed: signed {item['tip'][:12]}, now {str(tip)[:12]}"
def mismatch(entry: dict, item: dict) -> str | None:
changed = tip_changed(entry, item)
if changed:
return changed
if (entry.get("worktree") or "") != item["worktree"]:
return f"worktree changed: signed {item['worktree'] or 'none'}, now {entry.get('worktree') or 'none'}"
remote = [str(action) for action in entry.get("actions") or [] if str(action).startswith(REMOTE_ACTION)]
if remote:
return f"the plan also deletes {', '.join(remote)[:200]}; a signed delete is local only"
return None
kept = signing.local_keeps(entry)
return f"pkgmgr branch cleanup --local keeps it: {kept}" if kept else None
def refused(item: dict, reason: str) -> dict:
return {**item, "status": "refused", "reason": reason}
def applied_result(item: dict, applied: list[dict] | None, code: int, stderr: str) -> dict:
"""Args:
item: one signed item the apply chose.
applied: every item `--apply` printed for its repository, None when
it printed no JSON.
code: the apply's exit code, 1 when any of its items failed.
stderr: the apply's error output.
Returns:
The item done only when pkgmgr's result for it is done, with a
warning as reason when pkgmgr deleted it at another tip than the
signed one, which update-ref only does for the tip pkgmgr planned as
merged; failed with pkgmgr's skip reason when the apply kept it.
Without a done result its ref decides: gone is gone, "gone after an
interrupted apply" when the apply printed no JSON and named no
unknown branch, else "deleted outside this run"; present is failed
with pkgmgr's reason.
"""
mine = next((entry for entry in applied or [] if entry.get("branch") == item["branch"]), {})
outcome = mine.get("result")
moved = tip_changed(mine, item) if mine else None
if isinstance(outcome, str) and outcome.split(";")[0] == DONE:
warning = f"deleted at another merged tip than the signed dry run: {moved}" if moved else ""
return {**item, "status": DONE, "reason": warning, "code": code, "applied": [mine]}
if mine.get("skip") and not outcome:
reason = f"pkgmgr branch cleanup --local keeps it: {mine['skip']}"
elif moved:
reason = f"the apply planned another tip than the signed dry run: {moved}"
elif not (isinstance(outcome, str) and outcome.startswith("skipped")) and branch_tip(item["repository"], item["branch"]) == "":
why = GONE_INTERRUPTED if applied is None and not unknown(stderr) else DELETED_OUTSIDE
return {**item, "status": GONE, "reason": why, "code": code, "applied": [mine] if mine else []}
else:
reason = outcome if isinstance(outcome, str) and outcome else stderr.strip()[-500:] or f"pkgmgr reported no result for {item['branch']}"
return {**item, "status": "failed", "reason": reason[:500], "code": code, "applied": [mine] if mine else []}
def screened(items: list[dict], plan: list[dict], default: str, ready: bool) -> dict[str, dict]:
"""Args:
items: admitted items of one repository, each branch once.
plan: the items of their dry run.
default: the repository's default branch.
ready: whether pkgmgr holds the GitHub token.
Returns:
The refusal by branch of each item the plan does not delete exactly
at its signed tip and worktree, or without the token not token-free;
of every item when the plan names other branches too.
"""
others = sorted({str(entry.get("branch")) for entry in plan} - {item["branch"] for item in items})
if others:
return {item["branch"]: refused(item, f"the dry run names other branches too, a pattern match: {', '.join(others)[:500]}") for item in items}
refusals = {}
for item in items:
mine = [entry for entry in plan if entry.get("branch") == item["branch"]]
reason = mismatch(mine[0], item) if len(mine) == 1 else f"the dry run names {len(mine)} items for {item['branch']}"
if reason is None and not ready and not signing.token_free(mine[0], default):
reason = TOKEN_NEEDED
if reason is not None:
refusals[item["branch"]] = refused(item, reason)
return refusals
class Cleanup:
@@ -148,53 +260,78 @@ class Cleanup:
self.tree = tree
self.pkgmgr = pkgmgr
def item(self, item: dict, proof: object, known: dict[str, str], ready: bool, now: float) -> dict:
def admitted(self, item: dict, proof: object, known: dict[str, str], now: float) -> str | None:
"""Args:
item: one signed item with repository, branch, worktree and tip.
proof: the job's {"payload", "assertion"}.
known: the repositories the tree file says pkgmgr knows, each
with its default branch.
ready: whether pkgmgr holds the GitHub token; without it only a
branch signing.token_free() accepts in the dry run is deleted.
known: the repositories the tree file says pkgmgr knows.
now: the current time in epoch seconds.
Returns:
The item with status done, refused or failed and the reason;
refused items changed nothing, failed ones are those pkgmgr did
not report done.
Why the item is refused before any dry run; None when the
operator's signature lists it, pkgmgr knows its repository and
its branch is an exact name.
"""
def refused(reason: str) -> dict:
return {**item, "status": "refused", "reason": reason}
try:
verifier.check(proof, signing.cleanup_item(item["repository"], item["branch"], item["worktree"], item["tip"]), "", self.trust, now)
except (signing.Refused, OSError) as error:
return refused(f"no valid operator signature: {error}")
return f"no valid operator signature: {error}"
if item["repository"] not in known or not Path(item["repository"]).is_dir():
return refused("unknown repository: pkgmgr does not list it")
return "unknown repository: pkgmgr does not list it"
if UNSAFE.search(item["branch"]) or not item["branch"]:
return refused("the branch name holds pattern or option characters; only an exact branch name is deleted")
code, stdout, stderr = run(self.pkgmgr, plan_argv(item["branch"], False), item["repository"])
return "the branch name holds pattern or option characters; only an exact branch name is deleted"
return None
def attempt(self, items: list[dict], apply: bool, results: dict[str, dict]) -> tuple[list[dict], int, str, str]:
"""Args:
items: items of one repository to plan or apply.
apply: run the apply instead of the dry run.
results: gets, by branch, each item that run names unknown whose
ref is gone, as gone "deleted outside this run".
Returns:
The items of the last run with its exit code, stdout and stderr;
after an "Unknown --branch" for gone branches the run repeats
once without them.
"""
location = items[0]["repository"]
code, stdout, stderr = run(self.pkgmgr, plan_argv([item["branch"] for item in items], apply), location, TIMEOUT)
named = unknown(stderr) if code != 0 else set()
gone = [item for item in items if item["branch"] in named and branch_tip(location, item["branch"]) == ""]
if gone:
results.update({item["branch"]: {**item, "status": GONE, "reason": DELETED_OUTSIDE, "code": code} for item in gone})
items = [item for item in items if item not in gone]
if items:
code, stdout, stderr = run(self.pkgmgr, plan_argv([item["branch"] for item in items], apply), location, TIMEOUT)
return items, code, stdout, stderr
def repository(self, items: list[dict], default: str, ready: bool) -> list[dict]:
"""Args:
items: admitted items of one repository, each branch once.
default: the repository's default branch.
ready: whether pkgmgr holds the GitHub token; without it only a
branch signing.token_free() accepts in the dry run is deleted.
Returns:
Each item with status done, gone, refused or failed and the
reason, from one dry run over all of them and one apply over
those the dry run plans to delete at the signed tip and worktree,
each repeated once without the branches it names unknown and
gone; refused items changed nothing.
"""
results: dict[str, dict] = {}
planned, code, stdout, stderr = self.attempt(items, False, results)
plan = plan_items(stdout) if code == 0 else None
if plan is None:
return refused(f"the dry run failed ({code}): {stderr.strip()[-500:]}")
others = sorted({str(entry.get("branch")) for entry in plan} - {item["branch"]})
if others:
return refused(f"the dry run names other branches too, a pattern match: {', '.join(others)[:500]}")
if len(plan) != 1:
return refused(f"the dry run names {len(plan)} items for {item['branch']}")
reason = mismatch(plan[0], item)
if reason:
return refused(reason)
if not ready and not signing.token_free(plan[0], known[item["repository"]]):
return refused(TOKEN_NEEDED)
code, stdout, stderr = run(self.pkgmgr, plan_argv(item["branch"], True), item["repository"])
applied = plan_items(stdout) or []
outcome = next((entry.get("result") for entry in applied if entry.get("branch") == item["branch"]), None)
if code == 0 and isinstance(outcome, str) and outcome.split(";")[0] == DONE:
return {**item, "status": "done", "reason": "", "code": code, "applied": applied}
reason = outcome if isinstance(outcome, str) and outcome else stderr.strip()[-500:] or f"pkgmgr reported no result for {item['branch']}"
return {**item, "status": "failed", "reason": reason[:500], "code": code, "applied": applied}
results.update({item["branch"]: refused(item, f"the dry run failed ({code}): {stderr.strip()[-500:]}") for item in planned})
else:
results.update(screened(planned, plan, default, ready))
chosen = [item for item in planned if item["branch"] not in results]
if chosen:
applying, code, stdout, stderr = self.attempt(chosen, True, results)
applied = plan_items(stdout)
results.update({item["branch"]: applied_result(item, applied, code, stderr) for item in applying})
return [results[item["branch"]] for item in items]
def job(self, folder: int, path: Path, now: float) -> dict:
try:
@@ -205,10 +342,24 @@ class Cleanup:
if not isinstance(items, list) or not items or not all(map(shaped, items)) or job.get("id") != path.stem:
return {"id": path.stem, "finished": now, "items": [], "error": "malformed job: refused, nothing deleted"}
known, ready = tree_facts(self.tree)
results = [self.item(item, job.get("proof"), known, ready, now) for item in items]
results: list[dict | None] = [None] * len(items)
groups: dict[str, list[int]] = {}
seen: set[tuple[str, str]] = set()
for index, item in enumerate(items):
reason = self.admitted(item, job.get("proof"), known, now)
if reason is None and (item["repository"], item["branch"]) in seen:
reason = "the job lists this branch twice"
if reason is not None:
results[index] = refused(item, reason)
continue
seen.add((item["repository"], item["branch"]))
groups.setdefault(item["repository"], []).append(index)
for location, indexes in groups.items():
for index, outcome in zip(indexes, self.repository([items[index] for index in indexes], known[location], ready)):
results[index] = outcome
result = {"id": path.stem, "finished": time.time(), "items": results}
if any(entry["status"] == "done" for entry in results):
code, _, stderr = run(self.pkgmgr, ["branch", "tree", "--format", "json", "--output", str(self.tree)], str(self.tree.parent))
if any(entry["status"] in (DONE, GONE) for entry in results):
code, _, stderr = run(self.pkgmgr, ["branch", "tree", "--format", "json", "--output", str(self.tree)], str(self.tree.parent), TIMEOUT)
result["tree"] = {"code": code, "error": stderr.strip()[-500:]}
return result
+159
View File
@@ -0,0 +1,159 @@
#!/usr/bin/env python3
"""Args:
--tree: the tree file `pkgmgr branch tree` writes; the plan goes beside
it as local-plan.json, in the directory the orchestrator mounts
read-only.
--pkgmgr: the pkgmgr executable built from a signed tag.
--budget: seconds one repository's dry run may take, well below the
tree unit's TimeoutStartSec.
Returns:
Keeps local-plan.json beside the tree file: the dry run of `pkgmgr
branch cleanup --local --json` of every repository the tree file says
pkgmgr knows, run in it as the cleanup runner runs its dry run
(cleanup.run: no repository hook or fsmonitor), named by its real path.
A repository whose refs, HEAD, config, reftable and worktree heads
(fingerprint) are unchanged since its last attempt is not planned
again. Before each dry run the file is rewritten atomically with that
repository marked stale ("replan started <time> has not finished") at
the attempted fingerprint, and again after it, so a kill keeps every
plan written before; a dry run that fails or outlasts --budget keeps
the repository's old items, marked stale with the reason and time, and
is retried only once its fingerprint changes. The hub offers nothing
from a stale repository. Deleting local-plan.json plans every
repository again. The exit code is 0 when every dry run of this run
printed a plan, else 1.
"""
from __future__ import annotations
import argparse
import datetime
import hashlib
import json
import os
import sys
import tempfile
from pathlib import Path
import cleanup
LOCAL_PLAN = "local-plan.json"
PLAN_ARGV = ["branch", "cleanup", "--local", "--json"]
GIT_FILES = ("HEAD", "config", "packed-refs", "reftable/tables.list")
WORKTREE_FILES = ("HEAD", "gitdir", "locked", "config.worktree")
def git_dir(repo: Path) -> Path:
dot = repo / ".git"
if dot.is_file():
return (repo / dot.read_text(encoding="utf-8").strip().removeprefix("gitdir: ")).resolve()
return dot
def fingerprint(repo: str) -> str | None:
"""Returns:
sha256 over name, mtime and size of every file the `--local` plan
reads from the repository's git dir: HEAD, config, packed-refs, the
reftable table list, every ref, and each worktree's HEAD, gitdir,
lock and config; None when the git dir cannot be read.
"""
try:
root = git_dir(Path(repo))
paths = [root / name for name in GIT_FILES]
for folder, _, files in os.walk(root / "refs"):
paths += [Path(folder, name) for name in files]
worktrees = root / "worktrees"
if worktrees.is_dir():
paths += [entry / name for entry in worktrees.iterdir() for name in WORKTREE_FILES]
digest = hashlib.sha256()
for path in sorted(paths):
try:
info = path.stat()
except FileNotFoundError:
continue
digest.update(f"{path.relative_to(root)}\0{info.st_mtime_ns}\0{info.st_size}\n".encode())
return digest.hexdigest()
except OSError:
return None
def known_repos(tree: Path) -> list[str]:
data = json.loads(tree.read_text(encoding="utf-8"))
return [entry["path"] for entry in data["repos"] if entry.get("known") is True and entry.get("missing") is not True and isinstance(entry.get("path"), str)]
def old_plans(target: Path) -> dict[str, dict]:
try:
return {entry["repo"]: entry for entry in json.loads(target.read_text(encoding="utf-8"))["repos"] if isinstance(entry, dict) and isinstance(entry.get("repo"), str)}
except (OSError, ValueError, KeyError, TypeError, AttributeError):
return {}
def plan(pkgmgr: Path, repo: str, budget: int) -> tuple[dict | None, str]:
"""Returns:
The repository's {"repo", "default", "merged_into", "warnings",
"items"} as pkgmgr prints it, named by its real path, and ""; or
None and "timed out" or "failed" when pkgmgr printed no plan of
exactly one repository within the budget.
"""
code, stdout, stderr = cleanup.run(pkgmgr, PLAN_ARGV, repo, budget)
if cleanup.plan_items(stdout) is None:
return None, "timed out" if code == 127 and "timed out" in stderr else "failed"
repos = json.loads(stdout)["repos"]
if len(repos) != 1 or not isinstance(repos[0].get("repo"), str):
return None, "failed"
return {**repos[0], "repo": os.path.realpath(repos[0]["repo"])}, ""
def write(target: Path, data: dict) -> None:
with tempfile.NamedTemporaryFile("w", dir=target.parent, prefix=f".{target.name}-", delete=False, encoding="utf-8") as handle:
json.dump(data, handle)
os.replace(handle.name, target)
def now() -> str:
return datetime.datetime.now(datetime.UTC).isoformat(timespec="seconds")
def main() -> int:
parser = argparse.ArgumentParser(prog="local_plan")
parser.add_argument("--tree", type=Path, required=True)
parser.add_argument("--pkgmgr", type=Path, required=True)
parser.add_argument("--budget", type=int, required=True)
args = parser.parse_args()
target = args.tree.with_name(LOCAL_PLAN)
try:
known = [os.path.realpath(repo) for repo in known_repos(args.tree)]
except (OSError, ValueError, KeyError, TypeError, AttributeError) as error:
print(f"local_plan: tree file {args.tree} unreadable ({error}), the old plan stays", file=sys.stderr)
return 1
plans = {repo: entry for repo, entry in old_plans(target).items() if repo in known}
def save() -> None:
write(target, {"repos": [plans[repo] for repo in known if repo in plans]})
failed = []
for repo in known:
current = fingerprint(repo)
if current is not None and plans.get(repo, {}).get("fingerprint") == current:
continue
previous = plans.get(repo, {"repo": repo, "default": None, "merged_into": [], "warnings": [], "items": []})
plans[repo] = {**previous, "fingerprint": current, "stale": f"replan started {now()} has not finished"}
save()
planned, reason = plan(args.pkgmgr, repo, args.budget)
if planned is None:
plans[repo] = {**previous, "fingerprint": current, "stale": f"replan {reason} at {now()}"}
failed.append(f"{repo}: {reason}")
else:
plans[repo] = {**planned, "fingerprint": current}
save()
if not known:
save()
for line in failed:
print(f"local_plan: no --local plan for {line}, the hub offers no delete there until its refs change", file=sys.stderr)
return 1 if failed else 0
if __name__ == "__main__":
raise SystemExit(main())
+31 -2
View File
@@ -35,6 +35,7 @@ GATE_ACTIONS = ("commit", "merge", "push", "pr")
GATE_REPO = re.compile(r"/[^\x00-\x1f\x7f]{0,4095}")
GATE_BRANCH = re.compile(r"(?![-/])(?!.*\.\.)(?!.*@\{)(?!.*//)[^\x00-\x20\x7f~^:?*\[\\]{1,255}(?<![/.])")
TOKEN_FREE_MERGE = "ancestor"
LOCAL_DELETE = ["local-branch"]
APPROVAL_KEYS = {"v", "kind", "items", "nonce", "expires"}
RULE_KEYS = {"v", "kind", "id", "kinds", "tool", "pattern", "scope", "expires"}
TARGET = "agents"
@@ -273,7 +274,8 @@ def names_default(ref: object, default: object) -> bool:
"""Args:
ref: a ref as pkgmgr names it, bare such as main or behind one
remote such as origin/main, the remote-tracking ref it compares
against when the remote has a HEAD.
against when the remote has a HEAD; `branch cleanup --local`
names it in full, refs/remotes/origin/main.
default: the repository's default branch as the tree file names it,
without the remote (pkgmgr tree.py strips it).
@@ -282,7 +284,7 @@ def names_default(ref: object, default: object) -> bool:
"""
if not isinstance(ref, str) or not isinstance(default, str) or not default:
return False
remote, _, name = ref.partition("/")
remote, _, name = ref.removeprefix("refs/remotes/").partition("/")
return ref == default or (bool(remote) and name == default)
@@ -300,6 +302,33 @@ def token_free(entry: dict, default: object) -> bool:
return entry.get("merged_by") == TOKEN_FREE_MERGE and names_default(entry.get("merged_into"), default) and not entry.get("remotes")
def local_keeps(entry: dict) -> str:
"""Args:
entry: one item of `pkgmgr branch cleanup --local --json`, the dry
run of what `--apply --local` does to this branch's local tip.
Returns:
Why that dry run keeps the branch, in pkgmgr's words where it gives
them: a state other than merged-clean, its skip reason (for a tip
that is no ancestor of the default branch "merged by patch, tip is
no ancestor of refs/remotes/origin/main"), a remote branch delete,
or a plan without exactly the local-branch delete; "" when it
deletes exactly the local branch, the only case the hub offers and
the host runs.
"""
if entry.get("state") != "merged-clean":
return f"state is {entry.get('state')}, not merged-clean"
if entry.get("skip"):
return str(entry["skip"])
actions = [str(action) for action in entry.get("actions") or []]
remote = [action for action in actions if action.startswith("remote-branch")]
if remote:
return f"the plan also deletes {', '.join(remote)[:200]}; a signed delete is local only"
if actions != LOCAL_DELETE:
return f"the dry run plans {', '.join(actions)[:200] or 'no delete'}, not exactly the local branch"
return ""
def approval_payload(hashes: list[str], nonce: str, expires: int) -> dict:
return {"v": VERSION, "kind": "approve", "items": sorted(set(hashes)), "nonce": nonce, "expires": expires}
+6 -4
View File
@@ -22,7 +22,7 @@ ITEM_LIMIT = 100
RESULTS_SHOWN = 20
TOKEN_ROW = "pkgmgr-github-token"
TOKEN_COMMAND = "env secret-tool store --label=pkgmgr-github service pkgmgr:github username 'github.com|-' </dev/tty"
TOKEN_REASON = "Store the GitHub token in the Secret Service under the key pkgmgr reads; Clean all merged then also deletes remote and squash-merged branches."
TOKEN_REASON = "Store the GitHub token in the Secret Service under the key pkgmgr reads; Delete all merged then also deletes the local branch of a merged branch that has a remote branch; never the remote branch."
TOKEN_ADDER = "Operator"
@@ -71,8 +71,10 @@ class Cleaning:
Returns:
The signed items, each still a merged-clean branch without a gc
skip reason at the same tip in the current tree file; without
the GitHub token (delete_ready false) each also token-free.
skip reason at the same tip in the current tree file whose
`--local` dry run deletes exactly its local branch at that tip
(tree.kept_by); without the GitHub token (delete_ready false)
each also token-free.
"""
if not isinstance(raw, list) or not 1 <= len(raw) <= ITEM_LIMIT:
raise ToolError(f"items must be a list of 1 to {ITEM_LIMIT} objects")
@@ -90,7 +92,7 @@ class Cleaning:
if entry in locked and entry not in known:
raise ToolError(f"delete blocked: {scan.get('delete_reason') or 'delete_ready is false'}")
if entry not in known:
raise ToolError(f"{entry.get('branch')!s:.200} in {entry.get('repository')!s:.200} is no merged-clean branch without skip reason at that tip in the tree file")
raise ToolError(f"{entry.get('branch')!s:.200} in {entry.get('repository')!s:.200} is no merged-clean branch without skip reason that the --local dry run deletes at that tip")
if entry in chosen:
raise ToolError("items must be distinct")
chosen.append(entry)
+108 -14
View File
@@ -20,6 +20,7 @@ INVALID = "invalid"
TREE_HEARTBEAT = "tree-heartbeat"
GC_SKILL = "kvb-git-gc"
GC_COMPOSE_DOWN = ("docker", "compose", "-p", "{project}", "down", "-v", "--remove-orphans")
LOCAL_PLAN = "local-plan.json"
def flow(item: dict) -> dict | None:
@@ -72,15 +73,64 @@ def held_by(worktree: str) -> str:
return f"held by worktree {worktree}: pkgmgr branch cleanup skips it, use branch gc ({GC_SKILL}, pkgmgr gc)"
def dry_run(item: dict) -> list[str]:
"""Returns:
def kept_by(planned: dict | None, tip: str, upstream_of: str, stale: str) -> str:
"""Args:
planned: this branch's item of the `pkgmgr branch cleanup --local`
dry run beside the tree file, None when it has none.
tip: the branch's local tip in the tree file.
upstream_of: a branch of the same dry run whose local upstream this
branch is, "" for none; pkgmgr's _config_keeps keeps it when the
host's `--branch` dry run chooses it alone.
stale: why the repository's dry run is stale, "" when it is fresh.
Returns:
Why the hub offers no delete: a stale dry run, or from that dry run
what keeps the branch (signing.local_keeps, the predicate the host
runner refuses by, then the local upstream), or that it has not
planned this tip yet; "" when it deletes exactly the local branch at
this tip.
"""
if stale:
return f"pkgmgr branch cleanup --local plan stale: {stale}; use branch gc ({GC_SKILL}, pkgmgr gc)"
if planned is None or planned.get("local_sha") != tip:
return f"no pkgmgr branch cleanup --local dry run of {tip[:12]} yet: the tree timer writes {LOCAL_PLAN} within a minute"
keeps = signing.local_keeps(planned) or (f"kept: local upstream of {upstream_of}" if upstream_of else "")
return f"pkgmgr branch cleanup --local keeps it: {keeps}; use branch gc ({GC_SKILL}, pkgmgr gc)" if keeps else ""
def local_upstreams(items: list[dict]) -> dict[str, str]:
"""Args:
items: one repository's items of the `--local` dry run, each with
its `%(upstream:short)` as upstream.
Returns:
Each branch that is the local upstream of another branch there, with
the first such branch by name, as pkgmgr's _config_keeps names it.
"""
names = {item["branch"] for item in items}
found: dict[str, str] = {}
for item in sorted(items, key=lambda entry: entry["branch"]):
upstream = str(item.get("upstream") or "")
parent = upstream if upstream in names else upstream.removeprefix("heads/")
if parent in names and parent != item["branch"]:
found.setdefault(parent, item["branch"])
return found
def dry_run(item: dict, kept: str) -> list[str]:
"""Args:
item: one tree item.
kept: kept_by() for it, "" when the hub offers its delete.
Returns:
What the runner's `pkgmgr branch cleanup` does for one tree item,
read from the gc plan the tree file carries: it deletes the local
branch and never touches a worktree, Compose stack or volume; a
remote branch stays, the runner refuses a plan that deletes one. A
branch held by a worktree it skips; its lines name what branch gc
would do instead, its `docker compose down -v` deleting the stack's
volumes.
branch held by a worktree it skips, and one the `--local` dry run
keeps or has not planned it does not delete; their lines name what
branch gc would do instead, its `docker compose down -v` deleting
the stack's volumes.
"""
tip = (item["local_sha"] or item["head"] or "")[:12]
merged = f", merged by {item['merged_by']}" if item["merged_by"] else ""
@@ -91,6 +141,10 @@ def dry_run(item: dict) -> list[str]:
lines += [f"branch gc would run {' '.join(GC_COMPOSE_DOWN).format(project=name)}, which deletes the volumes of {name}" for name in item["compose_projects"]]
lines += [f"branch gc would remove the image {name}" for name in item["images"]]
return lines
if kept:
lines.append(kept)
lines += [f"branch gc would delete the local branch {item['branch']}" if action == "local-branch" else f"branch gc would delete {action}" for action in item["actions"]]
return lines
for action in item["actions"]:
if action == "local-branch":
lines.append(f"deletes the local branch {item['branch']}")
@@ -101,14 +155,25 @@ def dry_run(item: dict) -> list[str]:
return lines
def branch(item: dict, path: str, default: object) -> dict:
def branch(item: dict, path: str, default: object, planned: dict | None, upstream_of: str, stale: str) -> dict:
"""Args:
item: one tree item with a branch.
path: its repository's path.
default: the repository's default branch.
planned: the branch's item of the `--local` dry run, None without.
upstream_of: as kept_by() takes it.
stale: as kept_by() takes it.
"""
remote = next(iter(item["remotes"].values()), "")
head = item["local_sha"] or item["head"] or remote
held = held_by(item["worktree"]) if item["worktree"] and item["state"] == "merged-clean" else ""
open_to_delete = item["state"] == "merged-clean" and not item["skip"] and bool(item["local_sha"]) and not item["worktree"]
candidate = item["state"] == "merged-clean" and bool(item["local_sha"]) and not item["worktree"]
kept = kept_by(planned, item["local_sha"], upstream_of, stale) if candidate else ""
open_to_delete = candidate and not item["skip"] and not kept
return {
"held": held,
"token_free": signing.token_free(item, default),
"kept": kept,
"token_free": signing.token_free(planned, default) if candidate and planned is not None else False,
"name": item["branch"],
"head": head,
"upstream": item["upstream"] or "",
@@ -120,22 +185,45 @@ def branch(item: dict, path: str, default: object) -> dict:
"merged_into": item["merged_into"],
"skip": item["skip"],
"reason": item["reason"],
"plan": dry_run(item),
"plan": dry_run(item, kept),
"cleanup": signing.cleanup_item(path, item["branch"], item["worktree"] or "", item["local_sha"]) if open_to_delete else None,
"worktrees": [worktree(item)] if item["worktree"] else [],
}
def repo(entry: dict) -> dict:
def repo(entry: dict, local: dict[tuple[str, str], tuple[dict, str]], stale: dict[str, str]) -> dict:
items = entry["items"]
return {
**{key: value for key, value in entry.items() if key != "items"},
"name": "/".join(Path(entry["path"]).parts[-LABEL_PARTS:]),
"branches": [branch(item, entry["path"], entry.get("default")) for item in items if item["branch"]],
"branches": [branch(item, entry["path"], entry.get("default"), *local.get((entry["path"], item["branch"]), (None, "")), stale.get(entry["path"], "")) for item in items if item["branch"]],
"detached": [worktree(item) for item in items if item["detached"]],
}
def local_plan(path: Path) -> tuple[dict[tuple[str, str], tuple[dict, str]], dict[str, str]]:
"""Returns:
Every item of the `pkgmgr branch cleanup --local` dry run that
client/local_plan.py writes at path, keyed by repository path and
branch, with the branch whose local upstream it is (local_upstreams,
"" for none); and each repository whose dry run is stale, with why.
Both are empty when the file is missing or unreadable, so the hub
offers no delete.
"""
try:
data = json.loads(path.read_text(encoding="utf-8"))
found, stale = {}, {}
for repository in data["repos"]:
items = [item for item in repository["items"] if isinstance(item, dict) and isinstance(item.get("branch"), str)]
upstreams = local_upstreams(items)
found.update({(repository["repo"], item["branch"]): (item, upstreams.get(item["branch"], "")) for item in items})
if repository.get("stale"):
stale[repository["repo"]] = str(repository["stale"])[:200]
return found, stale
except (OSError, ValueError, KeyError, TypeError, AttributeError):
return {}, {}
def offered(scan: dict, ready: bool) -> list[dict]:
"""Args:
scan: the tree.
@@ -151,8 +239,9 @@ def offered(scan: dict, ready: bool) -> list[dict]:
def read_tree(path: Path) -> dict:
try:
data = json.loads(path.read_text(encoding="utf-8"))
local, stale = local_plan(path.with_name(LOCAL_PLAN))
return {
"repos": sorted((repo(entry) for entry in data["repos"]), key=lambda entry: entry["name"]),
"repos": sorted((repo(entry, local, stale) for entry in data["repos"]), key=lambda entry: entry["name"]),
"scanned": iso_epoch(data["generated_at"]),
"delete_ready": data["delete_ready"],
"delete_reason": data["delete_reason"],
@@ -204,7 +293,7 @@ class Tree:
heartbeat: Path
lock: threading.Lock
cached: dict | None
stamp: tuple[int, int] | None
stamp: tuple[tuple[int, int], tuple[int, int] | None] | None
def beat(self) -> None:
self.heartbeat.touch()
@@ -218,7 +307,12 @@ class Tree:
f"no pkgmgr tree file at {self.file}: {error.strerror}. The tree timer runs `pkgmgr branch tree`, which the installed pkgmgr "
"lacks until a release ships it: release pkgmgr and install it, then the timer writes the file within a minute."
)}
stamp = (info.st_mtime_ns, info.st_size)
try:
planned = self.file.with_name(LOCAL_PLAN).stat()
plan_stamp = (planned.st_mtime_ns, planned.st_size)
except OSError:
plan_stamp = None
stamp = ((info.st_mtime_ns, info.st_size), plan_stamp)
if self.cached is None or stamp != self.stamp:
self.cached, self.stamp = read_tree(self.file), stamp
return self.cached
@@ -1,11 +1,12 @@
[Unit]
Description=Agent Orchestrator tree: pkgmgr branch tree into the file the orchestrator reads, while its page polls
Description=Agent Orchestrator tree: pkgmgr branch tree and the pkgmgr branch cleanup --local dry run into the files the orchestrator reads, while its page polls
[Service]
Type=oneshot
Environment=PATH=/usr/bin:/bin
ExecCondition=/bin/sh -c 'test -n "$(find %h/.local/state/kvb-orchestrator/tree-heartbeat -mmin -5 2>/dev/null)"'
ExecStart=%h/.local/share/kvb-orchestrator/pkgmgr/bin/pkgmgr branch tree --format json --output %h/.local/state/pkgmgr/tree/branch-tree.json
ExecStart=/usr/bin/env python3 %h/.local/share/kvb-orchestrator/client/local_plan.py --tree %h/.local/state/pkgmgr/tree/branch-tree.json --pkgmgr %h/.local/share/kvb-orchestrator/pkgmgr/bin/pkgmgr --budget 240
Nice=10
IOSchedulingClass=idle
CPUQuota=50%
+9 -7
View File
@@ -1,6 +1,6 @@
const CLEANUP_POLL = 2000;
const CLEANUP_PATIENCE = 900000;
const RESULT_LABELS = {done: ["deleted", "ok"], refused: ["refused", "bad"], failed: ["failed", "bad"]};
const RESULT_LABELS = {done: ["deleted", "ok"], gone: ["gone", "warn"], refused: ["refused", "bad"], failed: ["failed", "bad"]};
function planOf(item) {
const repo = (tree.data.repos || []).find((entry) => entry.path === item.repository);
@@ -8,16 +8,18 @@
return {repo, entry};
}
function deleteDialog(items, locked = [], held = []) {
function deleteDialog(items, locked = [], held = [], kept = []) {
if (!items.length) {
return;
}
const rows = [...items, ...locked, ...held].map((item) => {
const rows = [...items, ...locked, ...held, ...kept].map((item) => {
const {repo, entry} = planOf(item);
const waits = locked.includes(item);
const kept = held.includes(item);
return h("li", {class: waits ? "dry-run locked" : kept ? "dry-run held" : "dry-run", "data-token": waits ? "needed" : null, "data-held": kept && entry ? entry.held : null},
h("div", {class: "row"}, h("strong", {}, revealed(repo ? repo.name : item.repository)), h("code", {}, revealed(item.branch)), waits ? chip("bi-key", "token needed", "warn") : kept ? chip("bi-folder-symlink", "held by a worktree: not deleted here", "neutral") : null),
const holds = held.includes(item);
const keeps = kept.includes(item);
return h("li", {class: waits ? "dry-run locked" : holds ? "dry-run held" : keeps ? "dry-run kept" : "dry-run", "data-token": waits ? "needed" : null, "data-held": holds && entry ? entry.held : null, "data-kept": keeps && entry ? entry.kept : null},
h("div", {class: "row"}, h("strong", {}, revealed(repo ? repo.name : item.repository)), h("code", {}, revealed(item.branch)),
waits ? chip("bi-key", "token needed", "warn") : holds ? chip("bi-folder-symlink", "held by a worktree: not deleted here", "neutral") : keeps ? chip("bi-sign-merge-left", "kept by the --local dry run: not deleted here", "neutral") : null),
h("pre", {}, (entry ? entry.plan : [`${item.branch} @ ${item.tip.slice(0, 12)}`]).map(revealed).join("\n")));
});
const outcome = h("div", {class: "mini", "aria-live": "polite"});
@@ -29,7 +31,7 @@
}
});
openSheet(items.length > 1 ? `Delete ${items.length} merged branches` : `Delete ${items[0].branch}`, [
h("p", {class: "muted"}, "Dry run of pkgmgr branch cleanup for exactly these entries: it deletes the local branch and never touches a worktree, Compose stack or volume. No remote branch: the host refuses an entry whose plan deletes one. A branch held by a worktree is not deleted here: branch gc (kvb-git-gc, pkgmgr gc) removes it, and its docker compose down -v deletes the stack's volumes. The host checks your signature and refuses every entry whose state has changed since."),
h("p", {class: "muted"}, "Dry run of pkgmgr branch cleanup for exactly these entries: it deletes the local branch and never touches a worktree, Compose stack or volume. No remote branch: the host refuses an entry whose plan deletes one. A branch held by a worktree is not deleted here: branch gc (kvb-git-gc, pkgmgr gc) removes it, and its docker compose down -v deletes the stack's volumes. Neither is a branch the pkgmgr branch cleanup --local dry run keeps, such as one merged by pull request, squash or patch whose local tip is no ancestor of the default branch: branch gc removes it. The host checks your signature and refuses every entry whose state has changed since."),
h("ul", {class: "mini", "aria-label": "Dry run"}, rows),
outcome,
h("div", {class: "actions"}, h("span", {class: "spacer"}), btn("Cancel", "btn ghost", closeSheet), confirm),
+12 -4
View File
@@ -27,12 +27,15 @@
const FORGE_UNAVAILABLE = "forge unavailable";
const deletableNow = (entry) => Boolean(entry.cleanup) && (tree.data.delete_ready === true || entry.token_free === true);
const tokenLocked = (entry) => tree.data.delete_ready !== true && entry.state === "merged-clean" && !entry.held && (entry.cleanup ? entry.token_free !== true : entry.skip === FORGE_UNAVAILABLE);
const tokenLocked = (entry) => tree.data.delete_ready !== true && entry.state === "merged-clean" && !entry.held && !entry.kept && (entry.cleanup ? entry.token_free !== true : entry.skip === FORGE_UNAVAILABLE);
function deleteButton(entry) {
if (entry.held) {
return h("span", {class: "hint held", title: entry.held}, chip("bi-folder-symlink", entry.held, "neutral"));
}
if (entry.kept) {
return h("span", {class: "hint kept", title: entry.kept}, chip("bi-sign-merge-left", entry.kept, "neutral"));
}
if (!entry.cleanup) {
return tokenLocked(entry) ? chip("bi-key", "token needed", "warn") : null;
}
@@ -53,14 +56,17 @@
const items = branches.filter(deletableNow).map((entry) => entry.cleanup);
const waiting = branches.filter(tokenLocked);
const locked = waiting.filter((entry) => entry.cleanup).map((entry) => entry.cleanup);
const held = (tree.data.repos || []).flatMap((repo) => repo.branches.filter((entry) => entry.held).map((entry) => ({repository: repo.path, branch: entry.name, tip: entry.head})));
const preview = (key) => (tree.data.repos || []).flatMap((repo) => repo.branches.filter((entry) => entry[key]).map((entry) => ({repository: repo.path, branch: entry.name, tip: entry.head})));
const held = preview("held");
const kept = preview("kept");
const ready = items.length > 0;
const hint = !ready && tree.data.delete_ready !== true ? `Delete all merged blocked: ${tree.data.delete_reason || "delete_ready is false"}` : items.length ? `Delete all merged: ${items.length} branches after dry run and signature` : "Delete all merged: no merged branch without a skip reason";
const button = iconBtn("bi-trash3-fill", "Delete all merged", "btn small danger", () => deleteDialog(items, locked, held), hint);
const button = iconBtn("bi-trash3-fill", "Delete all merged", "btn small danger", () => deleteDialog(items, locked, held, kept), hint);
button.disabled = !ready;
return h("span", {class: "hint", title: hint}, button, items.length ? counter(items.length) : null,
waiting.length ? chip("bi-key", `${waiting.length} token needed`, "warn", waiting.length) : null,
held.length ? chip("bi-folder-symlink", `${held.length} held by a worktree: use branch gc`, "neutral", held.length) : null,
kept.length ? chip("bi-sign-merge-left", `${kept.length} kept by the --local dry run: see each dry run`, "neutral", kept.length) : null,
tree.data.delete_ready !== true ? iconBtn("bi-key-fill", "Store token", "btn small soft", storeToken, "Store token: add the operator row that stores the GitHub token in the Secret Service") : null);
}
@@ -92,8 +98,10 @@
return found.length ? found[found.length - 1] : null;
}
const RESULT_CHIPS = {done: ["bi-check2-all", "deleted", "ok"], gone: ["bi-question-octagon", "gone", "warn"], refused: ["bi-x-octagon", "refused", "bad"], failed: ["bi-exclamation-triangle", "failed", "bad"]};
function resultChip(result) {
const [name, label, tone] = result.status === "done" ? ["bi-check2-all", "deleted", "ok"] : result.status === "refused" ? ["bi-x-octagon", "refused", "bad"] : ["bi-exclamation-triangle", "failed", "bad"];
const [name, label, tone] = RESULT_CHIPS[result.status] || RESULT_CHIPS.failed;
return chip(name, `${label}${result.reason ? `: ${result.reason}` : ""}`, tone);
}