Merge branch 'perf/merge-fix/find-fix-speed'

find_fix.py decides in_head with one rev-list, runs -S as parallel slices and builds the candidate lines in parallel. Verified before the merge on the precomputed merge tree: test_ci_tools 7/7, byte-identical output on 3 calls in the skills repo and 2 in infinito-nexus-core, -S on core 62.4 s to 9.6 s.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

skills/kvb-git-merge-fix/scripts/find_fix.py

@@ -5,33 +5,69 @@ Usage:
     find_fix.py [--grep KEYWORD]... [-S LITERAL]... [--path PATH]... [--fetch]
 
 Prints one line per candidate: sha, up to three branches, subject, files
-touched. -S also matches commits that removed the literal; read the diff.
+touched. The filters are alternatives: a commit is a hit when it matches any
+one of them. -S also matches commits that removed the literal; read the diff.
 Run it inside the repository; it never checks anything out.
 """
 
 from __future__ import annotations
 
 import argparse
+import os
 import subprocess
 import sys
+from collections.abc import Callable
+from concurrent.futures import ThreadPoolExecutor
 
 LIMIT = 30
+WORKERS = os.cpu_count() or 1
+SLICES_PER_WORKER = 4
 
 
-def git(*args: str) -> subprocess.CompletedProcess:
-    return subprocess.run(["git", *args], capture_output=True, text=True, check=False)
+def git(*args: str, stdin: str | None = None) -> subprocess.CompletedProcess:
+    return subprocess.run(["git", *args], input=stdin, capture_output=True, text=True, check=False)
 
 
-def search(args: argparse.Namespace) -> list[str]:
-    queries = [["--regexp-ignore-case", f"--grep={k}"] for k in args.grep]
-    queries += [[f"-S{s}"] for s in args.S]
-    queries += [["--", p] for p in args.path]
-    found: list[str] = []
+def log_all(pool: ThreadPoolExecutor, *query: str) -> Callable[[], list[str]]:
+    job = pool.submit(git, "log", "--all", "--format=%H", *query)
+    return lambda: job.result().stdout.split()
+
+
+def pickaxe(pool: ThreadPoolExecutor, literal: str, rank: dict[str, int]) -> Callable[[], list[str]]:
+    """Run `git log --all -S<literal>` as parallel slices of the commit list.
+
+    The slices are strided because the costly commits cluster in time.
+
+    Args:
+        pool: executor the slice runs are submitted to.
+        literal: the -S literal.
+        rank: every commit of `git rev-list --all`, mapped to its position.
+
+    Returns:
+        A callable yielding the matching commits in `git log --all` order.
+    """
+    commits = list(rank)
+    parts = min(len(commits), WORKERS * SLICES_PER_WORKER)
+    jobs = [
+        pool.submit(
+            git, "log", "--no-walk=unsorted", "--stdin", "--format=%H", f"-S{literal}", stdin="\n".join(commits[i::parts]) + "\n"
+        )
+        for i in range(parts)
+    ]
+    return lambda: sorted((sha for job in jobs for sha in job.result().stdout.split()), key=rank.__getitem__)
+
+
+def search(args: argparse.Namespace, pool: ThreadPoolExecutor) -> list[str]:
+    queries = [log_all(pool, "--regexp-ignore-case", f"--grep={k}") for k in args.grep]
+    if args.S:
+        rank = {sha: i for i, sha in enumerate(git("rev-list", "--all").stdout.split())}
+        queries += [pickaxe(pool, s, rank) for s in args.S]
+    queries += [log_all(pool, "--", p) for p in args.path]
+    found: dict[str, None] = {}
     for query in queries:
-        for sha in git("log", "--all", "--format=%H", *query).stdout.split():
-            if sha not in found:
-                found.append(sha)
-    return found
+        for sha in query():
+            found.setdefault(sha)
+    return list(found)
 
 
 def describe(sha: str) -> str:
@@ -53,11 +89,18 @@ def main() -> int:
         parser.error("give at least one --grep, -S or --path")
     if args.fetch:
         git("fetch", "--all", "--prune", "--tags")
-    hits = search(args)
-    outside = [sha for sha in hits if git("merge-base", "--is-ancestor", sha, "HEAD").returncode]
-    print(f"hits={len(hits)} in_head={len(hits) - len(outside)} candidates={len(outside)}")
-    for sha in outside[:LIMIT]:
-        print(describe(sha))
+    pool = ThreadPoolExecutor(max_workers=WORKERS)
+    try:
+        head = pool.submit(git, "rev-list", "HEAD")
+        hits = search(args, pool)
+        in_head = set(head.result().stdout.split())
+        outside = [sha for sha in hits if sha not in in_head]
+        print(f"hits={len(hits)} in_head={len(hits) - len(outside)} candidates={len(outside)}")
+        for line in pool.map(describe, outside[:LIMIT]):
+            print(line)
+    finally:
+        # A plain `with` would start every queued git slice after Ctrl-C.
+        pool.shutdown(cancel_futures=True)
     if len(outside) > LIMIT:
         print(f"... {len(outside) - LIMIT} more, narrow the search")
     return 0

tests/test_ci_tools.py

@@ -89,21 +89,47 @@ class TestFindFix(unittest.TestCase):
         self._git("commit", "-q", "--allow-empty", "-m", "fix: probe on side")
         self._git("switch", "-q", "main")
 
-    def _git(self, *args: str) -> None:
-        subprocess.run(["git", *args], check=True, capture_output=True, env=self.env, cwd=self.repo)
+    def _git(self, *args: str) -> str:
+        return subprocess.run(
+            ["git", *args], check=True, capture_output=True, text=True, env=self.env, cwd=self.repo
+        ).stdout
 
-    def test_only_commits_outside_head_are_candidates(self):
+    def _find_fix(self, *args: str) -> list[str]:
         result = subprocess.run(
-            [sys.executable, str(FIND_FIX), "--grep", "probe"],
+            [sys.executable, str(FIND_FIX), *args],
             capture_output=True,
             text=True,
             check=True,
             env=self.env,
             cwd=self.repo,
         )
-        self.assertIn("hits=2 in_head=1 candidates=1", result.stdout)
-        self.assertIn("fix: probe on side", result.stdout)
-        self.assertNotIn("probe in head", result.stdout)
+        return result.stdout.splitlines()
+
+    def _commit_on_side(self, name: str, text: str, message: str) -> None:
+        self._git("switch", "-q", "side")
+        (self.repo / name).write_text(text)
+        self._git("add", name)
+        self._git("commit", "-q", "-m", message)
+        self._git("switch", "-q", "main")
+
+    def test_only_commits_outside_head_are_candidates(self):
+        lines = self._find_fix("--grep", "probe")
+        self.assertEqual(lines[0], "hits=2 in_head=1 candidates=1")
+        self.assertIn("fix: probe on side", "\n".join(lines))
+        self.assertNotIn("probe in head", "\n".join(lines))
+
+    def test_pickaxe_lists_what_git_log_all_finds_in_its_order(self):
+        for step in range(12):
+            self._commit_on_side("probe.txt", "needle\n" * (step % 3), f"step {step}")
+        expected = self._git("log", "--all", "--format=%h %s", "-Sneedle").splitlines()
+        lines = self._find_fix("-S", "needle")
+        self.assertEqual(lines[0], f"hits={len(expected)} in_head=0 candidates={len(expected)}")
+        self.assertEqual([line.split("\t")[0] for line in lines[1:]], expected)
+
+    def test_filters_are_alternatives_not_narrowing(self):
+        self._commit_on_side("other.txt", "x\n", "chore: unrelated")
+        lines = self._find_fix("--grep", "probe", "--path", "other.txt")
+        self.assertEqual(lines[0], "hits=3 in_head=1 candidates=2")
 
 
 if __name__ == "__main__":