diff --git a/skills/kvb-git-merge-fix/scripts/find_fix.py b/skills/kvb-git-merge-fix/scripts/find_fix.py index 29b4981..3329614 100755 --- a/skills/kvb-git-merge-fix/scripts/find_fix.py +++ b/skills/kvb-git-merge-fix/scripts/find_fix.py @@ -5,33 +5,69 @@ Usage: find_fix.py [--grep KEYWORD]... [-S LITERAL]... [--path PATH]... [--fetch] Prints one line per candidate: sha, up to three branches, subject, files -touched. -S also matches commits that removed the literal; read the diff. +touched. The filters are alternatives: a commit is a hit when it matches any +one of them. -S also matches commits that removed the literal; read the diff. Run it inside the repository; it never checks anything out. """ from __future__ import annotations import argparse +import os import subprocess import sys +from collections.abc import Callable +from concurrent.futures import ThreadPoolExecutor LIMIT = 30 +WORKERS = os.cpu_count() or 1 +SLICES_PER_WORKER = 4 -def git(*args: str) -> subprocess.CompletedProcess: - return subprocess.run(["git", *args], capture_output=True, text=True, check=False) +def git(*args: str, stdin: str | None = None) -> subprocess.CompletedProcess: + return subprocess.run(["git", *args], input=stdin, capture_output=True, text=True, check=False) -def search(args: argparse.Namespace) -> list[str]: - queries = [["--regexp-ignore-case", f"--grep={k}"] for k in args.grep] - queries += [[f"-S{s}"] for s in args.S] - queries += [["--", p] for p in args.path] - found: list[str] = [] +def log_all(pool: ThreadPoolExecutor, *query: str) -> Callable[[], list[str]]: + job = pool.submit(git, "log", "--all", "--format=%H", *query) + return lambda: job.result().stdout.split() + + +def pickaxe(pool: ThreadPoolExecutor, literal: str, rank: dict[str, int]) -> Callable[[], list[str]]: + """Run `git log --all -S` as parallel slices of the commit list. + + The slices are strided because the costly commits cluster in time. + + Args: + pool: executor the slice runs are submitted to. + literal: the -S literal. + rank: every commit of `git rev-list --all`, mapped to its position. + + Returns: + A callable yielding the matching commits in `git log --all` order. + """ + commits = list(rank) + parts = min(len(commits), WORKERS * SLICES_PER_WORKER) + jobs = [ + pool.submit( + git, "log", "--no-walk=unsorted", "--stdin", "--format=%H", f"-S{literal}", stdin="\n".join(commits[i::parts]) + "\n" + ) + for i in range(parts) + ] + return lambda: sorted((sha for job in jobs for sha in job.result().stdout.split()), key=rank.__getitem__) + + +def search(args: argparse.Namespace, pool: ThreadPoolExecutor) -> list[str]: + queries = [log_all(pool, "--regexp-ignore-case", f"--grep={k}") for k in args.grep] + if args.S: + rank = {sha: i for i, sha in enumerate(git("rev-list", "--all").stdout.split())} + queries += [pickaxe(pool, s, rank) for s in args.S] + queries += [log_all(pool, "--", p) for p in args.path] + found: dict[str, None] = {} for query in queries: - for sha in git("log", "--all", "--format=%H", *query).stdout.split(): - if sha not in found: - found.append(sha) - return found + for sha in query(): + found.setdefault(sha) + return list(found) def describe(sha: str) -> str: @@ -53,11 +89,18 @@ def main() -> int: parser.error("give at least one --grep, -S or --path") if args.fetch: git("fetch", "--all", "--prune", "--tags") - hits = search(args) - outside = [sha for sha in hits if git("merge-base", "--is-ancestor", sha, "HEAD").returncode] - print(f"hits={len(hits)} in_head={len(hits) - len(outside)} candidates={len(outside)}") - for sha in outside[:LIMIT]: - print(describe(sha)) + pool = ThreadPoolExecutor(max_workers=WORKERS) + try: + head = pool.submit(git, "rev-list", "HEAD") + hits = search(args, pool) + in_head = set(head.result().stdout.split()) + outside = [sha for sha in hits if sha not in in_head] + print(f"hits={len(hits)} in_head={len(hits) - len(outside)} candidates={len(outside)}") + for line in pool.map(describe, outside[:LIMIT]): + print(line) + finally: + # A plain `with` would start every queued git slice after Ctrl-C. + pool.shutdown(cancel_futures=True) if len(outside) > LIMIT: print(f"... {len(outside) - LIMIT} more, narrow the search") return 0 diff --git a/tests/test_ci_tools.py b/tests/test_ci_tools.py index 3a48da7..352dbf1 100644 --- a/tests/test_ci_tools.py +++ b/tests/test_ci_tools.py @@ -89,21 +89,47 @@ class TestFindFix(unittest.TestCase): self._git("commit", "-q", "--allow-empty", "-m", "fix: probe on side") self._git("switch", "-q", "main") - def _git(self, *args: str) -> None: - subprocess.run(["git", *args], check=True, capture_output=True, env=self.env, cwd=self.repo) + def _git(self, *args: str) -> str: + return subprocess.run( + ["git", *args], check=True, capture_output=True, text=True, env=self.env, cwd=self.repo + ).stdout - def test_only_commits_outside_head_are_candidates(self): + def _find_fix(self, *args: str) -> list[str]: result = subprocess.run( - [sys.executable, str(FIND_FIX), "--grep", "probe"], + [sys.executable, str(FIND_FIX), *args], capture_output=True, text=True, check=True, env=self.env, cwd=self.repo, ) - self.assertIn("hits=2 in_head=1 candidates=1", result.stdout) - self.assertIn("fix: probe on side", result.stdout) - self.assertNotIn("probe in head", result.stdout) + return result.stdout.splitlines() + + def _commit_on_side(self, name: str, text: str, message: str) -> None: + self._git("switch", "-q", "side") + (self.repo / name).write_text(text) + self._git("add", name) + self._git("commit", "-q", "-m", message) + self._git("switch", "-q", "main") + + def test_only_commits_outside_head_are_candidates(self): + lines = self._find_fix("--grep", "probe") + self.assertEqual(lines[0], "hits=2 in_head=1 candidates=1") + self.assertIn("fix: probe on side", "\n".join(lines)) + self.assertNotIn("probe in head", "\n".join(lines)) + + def test_pickaxe_lists_what_git_log_all_finds_in_its_order(self): + for step in range(12): + self._commit_on_side("probe.txt", "needle\n" * (step % 3), f"step {step}") + expected = self._git("log", "--all", "--format=%h %s", "-Sneedle").splitlines() + lines = self._find_fix("-S", "needle") + self.assertEqual(lines[0], f"hits={len(expected)} in_head=0 candidates={len(expected)}") + self.assertEqual([line.split("\t")[0] for line in lines[1:]], expected) + + def test_filters_are_alternatives_not_narrowing(self): + self._commit_on_side("other.txt", "x\n", "chore: unrelated") + lines = self._find_fix("--grep", "probe", "--path", "other.txt") + self.assertEqual(lines[0], "hits=3 in_head=1 candidates=2") if __name__ == "__main__":