fix(kvb-check-security): never cache a partial MEDUSA run as complete
security.py cached any run that exited cleanly, so a run MEDUSA cut short by its file size, line or path limits, or one that skipped an unscannable type, was stored and reported as 100 %. It now predicts which changed files MEDUSA takes, from MEDUSA's own limits and exclusion defaults read from its source, and caches a result only when MEDUSA reports exactly that count. Files it does not scan are listed as not covered; a diff with nothing scannable, an unreadable limit or a count mismatch exits 4 instead of scoring 100. Coverage details move to coverage.md, keeping SKILL.md under the word cap. 🎯 Fix: 95 % 💥 Break: 2 % 💸 Cost: 93 % ⏱️ Time: 100 % 🛡️ Security: 100 % 🏆 Score: 100 🚦 Gate: ✅ 90/5, cost and time <= 100, security 100 % Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
3418738993
commit
e7133af073
4 files changed
+513
-59
No files matched your search
+187
-8
@@ -16,7 +16,7 @@ REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||
SECURITY = REPO_ROOT / "skills" / "kvb-check-security" / "scripts" / "security.py"
|
||||
GIT_DIR = os.path.dirname(shutil.which("git"))
|
||||
PLACEHOLDER = "placeholder-value-never-shown"
|
||||
FAKE = """#!{python}
|
||||
FAKE = """#!{shebang}
|
||||
import json, os, pathlib, sys
|
||||
args = sys.argv[1:]
|
||||
pathlib.Path(__file__).with_name("argv.json").write_text(json.dumps(args))
|
||||
@@ -24,9 +24,19 @@ pathlib.Path(__file__).with_name("home.txt").write_text(os.environ["HOME"])
|
||||
pathlib.Path(os.environ["HOME"], ".medusa").mkdir()
|
||||
target, out = pathlib.Path(args[1]), pathlib.Path(args[args.index("-o") + 1])
|
||||
out.mkdir(parents=True)
|
||||
findings = []
|
||||
limit = int(os.environ.get("FAKE_MEDUSA_FILE_LIMIT", str(2 * 1024 * 1024)))
|
||||
findings, taken, counted = [], 0, 0
|
||||
for path in sorted(p for p in target.rglob("*") if p.is_file()):
|
||||
for number, text in enumerate(path.read_text().splitlines(), 1):
|
||||
if path.suffix not in (".py", ".json") and not path.name.startswith("Dockerfile"):
|
||||
continue
|
||||
if path.name == os.environ.get("FAKE_MEDUSA_LOSE") or any(p in str(path.relative_to(target)) for p in ("build", "env")):
|
||||
continue
|
||||
taken += 1
|
||||
if path.stat().st_size > limit:
|
||||
continue
|
||||
counted += path.read_bytes().count(b"\\n") + 1
|
||||
cut = int(os.environ.get("FAKE_MEDUSA_LINE_LIMIT", "50000"))
|
||||
for number, text in enumerate(path.read_text().split("\\n")[:cut], 1):
|
||||
for severity in ("CRITICAL", "HIGH", "MEDIUM", "LOW", "INFO"):
|
||||
if "MARK-" + severity in text:
|
||||
findings.append({{
|
||||
@@ -35,8 +45,35 @@ for path in sorted(p for p in target.rglob("*") if p.is_file()):
|
||||
}})
|
||||
if "MARK-FILE" in text:
|
||||
findings.append({{"severity": "MEDIUM", "rule_id": "FAKE-FILE", "file": str(path), "line": None}})
|
||||
report = {{"issues": findings}} if os.environ.get("FAKE_MEDUSA_SHAPE") == "other" else {{"findings": findings}}
|
||||
if not taken:
|
||||
print("No files found to scan")
|
||||
sys.exit(0)
|
||||
report = {{"findings": findings, "scan_summary": {{"files_scanned": taken, "lines_scanned": counted}}}}
|
||||
if os.environ.get("FAKE_MEDUSA_SHAPE") == "other":
|
||||
report = {{"issues": findings}}
|
||||
if os.environ.get("FAKE_MEDUSA_SHAPE") == "nocounts":
|
||||
report = {{"findings": findings, "scan_summary": {{"files_scanned": taken}}}}
|
||||
(out / "medusa-scan-20261104-120000.json").write_text(json.dumps(report))
|
||||
sys.stdout.write(os.environ.get("FAKE_MEDUSA_STDOUT", "").replace("TREE", str(target)))
|
||||
sys.exit(int(os.environ.get("FAKE_MEDUSA_EXIT", "0")))
|
||||
"""
|
||||
BASE = "class RuleBasedScanner:\n MAX_RULE_SCAN_LINES = {lines}\n"
|
||||
CONFIG = """@dataclass
|
||||
class MedusaConfig:
|
||||
exclude_paths: List[str] = field(default_factory=lambda: ["build/", "env/"])
|
||||
exclude_files: List[str] = field(default_factory=lambda: ["*.min.js"])
|
||||
"""
|
||||
PARALLEL = """MAX_SCAN_FILE_SIZE = {size}
|
||||
|
||||
|
||||
class MedusaParallelScanner:
|
||||
FILE_SCANNERS = {{".py": "python", ".json": "json"}}
|
||||
|
||||
def find_scannable_files(self):
|
||||
special_exact_names: Set[str] = {{"mcp.json"}}
|
||||
special_prefixes = ("Dockerfile", ".env")
|
||||
env_suffix = ".env"
|
||||
ai_context_dir_names: Set[str] = {{".claude"}}
|
||||
"""
|
||||
HAND = "review trust boundaries, secrets and injection in the diff by hand, then pass --security '~<n>' to kvb-check-score"
|
||||
|
||||
@@ -46,10 +83,20 @@ class TestSecurity(unittest.TestCase):
|
||||
tmp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(tmp.cleanup)
|
||||
self.base = Path(tmp.name)
|
||||
self.bin = self.base / "bin"
|
||||
self.bin.mkdir()
|
||||
medusa = self.bin / "medusa"
|
||||
medusa.write_text(FAKE.format(python=sys.executable), encoding="utf-8")
|
||||
venv = self.base / "venv"
|
||||
self.bin = venv / "bin"
|
||||
self.bin.mkdir(parents=True)
|
||||
real = Path(sys.executable).resolve()
|
||||
(self.bin / "python").symlink_to(real)
|
||||
(venv / "pyvenv.cfg").write_text(f"home = {real.parent}\ninclude-system-site-packages = false\n", encoding="utf-8")
|
||||
version = f"python{sys.version_info.major}.{sys.version_info.minor}"
|
||||
self.package = venv / "lib" / version / "site-packages" / "medusa"
|
||||
self.limits()
|
||||
(self.package / "__init__.py").write_text("", encoding="utf-8")
|
||||
(self.package / "config.py").write_text(CONFIG, encoding="utf-8")
|
||||
self.medusa = self.bin / "medusa"
|
||||
self.medusa.write_text(FAKE.format(shebang=f"{self.bin / 'python'} -E"), encoding="utf-8")
|
||||
medusa = self.medusa
|
||||
medusa.chmod(0o755)
|
||||
self.repo = self.base / "repo"
|
||||
self.repo.mkdir()
|
||||
@@ -70,6 +117,11 @@ class TestSecurity(unittest.TestCase):
|
||||
self.git("add", "-A")
|
||||
self.git("commit", "-qm", "base")
|
||||
|
||||
def limits(self, size: str = "2 * 1024 * 1024", lines: str = "50_000") -> None:
|
||||
for name, text in (("core/parallel.py", PARALLEL.format(size=size)), ("scanners/base.py", BASE.format(lines=lines))):
|
||||
(self.package / name).parent.mkdir(parents=True, exist_ok=True)
|
||||
(self.package / name).write_text(text, encoding="utf-8")
|
||||
|
||||
def git(self, *args: str) -> None:
|
||||
subprocess.run(["git", *args], cwd=self.repo, env=self.env, capture_output=True, check=True)
|
||||
|
||||
@@ -311,6 +363,133 @@ class TestSecurity(unittest.TestCase):
|
||||
self.assertTrue(argv.exists())
|
||||
self.assertEqual(list(folder.glob("*.json")), [])
|
||||
|
||||
def test_an_incomplete_run_is_never_cached_and_never_scored(self):
|
||||
self.write("app.py", "print(2)\n")
|
||||
argv = self.bin / "argv.json"
|
||||
folder = self.base / "council" / "security-cache"
|
||||
warn = "⚠️ "
|
||||
rerun = "medusa run incomplete, nothing cached, rerun once the host is idle"
|
||||
cases = (
|
||||
({"FAKE_MEDUSA_STDOUT": f"Found 1 scannable files\n{warn}1 scanner error(s)/timeout(s) during scan: "
|
||||
"OWASPLLMScanner (1)\n"}, f"{rerun}: 1 scanner error(s)/timeout(s) during scan: OWASPLLMScanner (1)"),
|
||||
({"FAKE_MEDUSA_STDOUT": f"{warn}1 director(ies) could not be read and were skipped (e.g. TREE/locked)\n"},
|
||||
f"{rerun}: 1 director(ies) could not be read and were skipped (e.g. locked)"),
|
||||
({"FAKE_MEDUSA_EXIT": "3"}, f"{rerun}: medusa exit 3"),
|
||||
({"FAKE_MEDUSA_FILE_LIMIT": "4"}, f"{rerun}: medusa counted 0 of 2 lines"),
|
||||
({"FAKE_MEDUSA_SHAPE": "nocounts"},
|
||||
("medusa report unreadable: medusa-scan-20261104-120000.json holds no files_scanned and lines_scanned "
|
||||
"counts (medusa exit 0)")),
|
||||
)
|
||||
for extra, problem in cases:
|
||||
with self.subTest(problem=problem):
|
||||
result = self.scan(env={**self.env, **extra})
|
||||
self.assertEqual((result.stdout, result.returncode), (f"🛡️ security – ({problem}: {HAND})\n", 4))
|
||||
self.assertEqual(list(folder.glob("*.json")), [])
|
||||
result = self.scan("--json", env={**self.env, "FAKE_MEDUSA_EXIT": "3"})
|
||||
self.assertEqual(json.loads(result.stdout), {"security": None, "problem": f"{rerun}: medusa exit 3"})
|
||||
argv.unlink()
|
||||
complete = self.scan(env=self.env)
|
||||
self.assertEqual((complete.stdout, complete.returncode), ("🛡️ security 100 % (medusa: 0 findings in 1 file)\n", 0))
|
||||
self.assertTrue(argv.exists())
|
||||
self.assertEqual(len(list(folder.glob("*.json"))), 1)
|
||||
argv.unlink()
|
||||
again = self.scan(env=self.env)
|
||||
self.assertEqual((again.stdout, again.returncode), (complete.stdout, complete.returncode))
|
||||
self.assertFalse(argv.exists())
|
||||
|
||||
def test_a_file_medusa_never_covers_is_named_with_its_limit_and_never_cached(self):
|
||||
argv = self.bin / "argv.json"
|
||||
folder = self.base / "council" / "security-cache"
|
||||
never = "medusa never covers this diff, nothing cached"
|
||||
big = "x = 1 # MARK-CRITICAL\n" + "#" * (2 * 1024 * 1024) + "\n"
|
||||
cases = (
|
||||
({"big.json": big}, {}, f"{never}: big.json has {len(big)} bytes, over medusa's 2097152 byte file limit", False),
|
||||
({"long.py": "x\n" * 50_000 + "y = 1 # MARK-CRITICAL\n"}, {},
|
||||
f"{never}: long.py has 50002 lines, over medusa's 50000 line regex limit", False),
|
||||
({"cr.py": "x\r" * 50_000 + "y = 1 # MARK-CRITICAL\r"}, {},
|
||||
f"{never}: cr.py has 50002 lines, over medusa's 50000 line regex limit", False),
|
||||
({"new.py": "x = 1 # MARK-CRITICAL\n"}, {"FAKE_MEDUSA_LOSE": "new.py"},
|
||||
f"{never}: medusa took up 1 of 2 files its rules take up", True),
|
||||
({}, {"FAKE_MEDUSA_STDOUT": "⚠️ Skipped TREE/app.py (3 MB exceeds 2 MB limit)\n"},
|
||||
f"{never}: Skipped app.py (3 MB exceeds 2 MB limit)", True),
|
||||
)
|
||||
self.write("app.py", "print(2)\n")
|
||||
for files, extra, problem, ran in cases:
|
||||
with self.subTest(problem=problem):
|
||||
argv.unlink(missing_ok=True)
|
||||
for name, text in files.items():
|
||||
self.write(name, text)
|
||||
result = self.scan(env={**self.env, **extra})
|
||||
self.assertEqual((result.stdout, result.returncode), (f"🛡️ security – ({problem}: {HAND})\n", 4))
|
||||
self.assertEqual(argv.exists(), ran)
|
||||
self.assertEqual(list(folder.glob("*.json")), [])
|
||||
for name in files:
|
||||
(self.repo / name).unlink()
|
||||
|
||||
def test_a_type_medusa_never_scans_is_listed_and_the_rest_scored(self):
|
||||
argv = self.bin / "argv.json"
|
||||
self.write("tpl.conf.j2", "token # MARK-CRITICAL\n")
|
||||
never = "medusa never covers this diff, nothing cached"
|
||||
result = self.scan(env=self.env)
|
||||
self.assertEqual((result.stdout, result.returncode), (
|
||||
f"🛡️ security – ({never}: medusa scans none of its files, not covered: tpl.conf.j2: {HAND})\n", 4,
|
||||
))
|
||||
self.assertFalse(argv.exists())
|
||||
self.write("app.py", "print(1)\nx = 1 # MARK-HIGH\n")
|
||||
scored = ("🛡️ security 75 % (medusa: 1 finding in 1 file: high app.py:2; not covered: tpl.conf.j2)\n", 3)
|
||||
result = self.scan(env=self.env)
|
||||
self.assertEqual((result.stdout, result.returncode), scored)
|
||||
argv.unlink()
|
||||
result = self.scan(env=self.env)
|
||||
self.assertEqual((result.stdout, result.returncode), scored)
|
||||
self.assertFalse(argv.exists())
|
||||
found = json.loads(self.scan("--json", env=self.env).stdout)
|
||||
self.assertEqual((found["files"], found["not_covered"]), (1, ["tpl.conf.j2"]))
|
||||
self.medusa.write_text(FAKE.format(shebang="/usr/bin/env -S python -E"), encoding="utf-8")
|
||||
result = self.scan(env=self.env)
|
||||
self.assertEqual((result.stdout, result.returncode), scored)
|
||||
self.assertTrue(argv.exists())
|
||||
(self.package / "scanners" / "base.py").unlink()
|
||||
result = self.scan(env=self.env)
|
||||
unread = f"{never}: medusa's own types, exclusions and limits are unreadable from its source: {HAND}"
|
||||
self.assertEqual((result.stdout, result.returncode), (f"🛡️ security – ({unread})\n", 4))
|
||||
|
||||
def test_a_path_medusa_excludes_is_listed_and_the_rest_scored(self):
|
||||
self.write("app.py", "print(1)\nx = 1 # MARK-HIGH\n")
|
||||
(self.repo / "cli" / "build").mkdir(parents=True)
|
||||
self.write("cli/build/run.py", "y = 1 # MARK-CRITICAL\n")
|
||||
self.write("environ.py", "z = 1 # MARK-CRITICAL\n")
|
||||
result = self.scan(env=self.env)
|
||||
self.assertEqual((result.stdout, result.returncode), (
|
||||
"🛡️ security 75 % (medusa: 1 finding in 1 file: high app.py:2; not covered: cli/build/run.py, environ.py)\n",
|
||||
3,
|
||||
))
|
||||
self.write("app.py", "print(1)\n")
|
||||
result = self.scan(env=self.env)
|
||||
never = "medusa never covers this diff, nothing cached"
|
||||
none = f"{never}: medusa scans none of its files, not covered: cli/build/run.py, environ.py: {HAND}"
|
||||
self.assertEqual((result.stdout, result.returncode), (f"🛡️ security – ({none})\n", 4))
|
||||
|
||||
def test_medusa_limits_come_from_its_source(self):
|
||||
mark = "y = 1 # MARK-CRITICAL\n"
|
||||
cases = (
|
||||
(("64", "50_000"), ("64", "50000"), "wide.py", "#" * 70 + "\n" + mark,
|
||||
"wide.py has 94 bytes, over medusa's 64 byte file limit"),
|
||||
(("2 * 1024", "100"), ("2048", "100"), "tall.py", "\n" * 120 + mark,
|
||||
"tall.py has 122 lines, over medusa's 100 line regex limit"),
|
||||
)
|
||||
for (size, lines), (fake_size, fake_lines), name, text, problem in cases:
|
||||
with self.subTest(problem=problem):
|
||||
self.limits(size=size, lines=lines)
|
||||
self.write(name, text)
|
||||
fake = {"FAKE_MEDUSA_FILE_LIMIT": fake_size, "FAKE_MEDUSA_LINE_LIMIT": fake_lines}
|
||||
result = self.scan(env={**self.env, **fake})
|
||||
self.assertEqual(
|
||||
(result.stdout, result.returncode),
|
||||
(f"🛡️ security – (medusa never covers this diff, nothing cached: {problem}: {HAND})\n", 4),
|
||||
)
|
||||
(self.repo / name).unlink()
|
||||
|
||||
def test_usage_errors_exit_2(self):
|
||||
result = subprocess.run(
|
||||
[sys.executable, str(SECURITY), "--repo", str(self.repo), "--base", "nope"],
|
||||
|
||||
Reference in new issue
Block a user