Merge branch 'fix/kvb-check-security-mjs' into unify/skills-20261112-night

This commit is contained in:
Kevin Veen-Birkenbach committed 2026-11-12 23:15:57 +00:00
commit 5e97fd5b47
2 files changed
+108 -17

No files matched your search

+50 -16
View File
@@ -20,11 +20,16 @@ Scan:
directory because medusa creates ~/.medusa on start, read from directory because medusa creates ~/.medusa on start, read from
`<dir>/medusa-scan-*.json`: a `findings` list whose entries carry `<dir>/medusa-scan-*.json`: a `findings` list whose entries carry
`severity` (critical, high, medium, low or info), `rule_id`, `file` and `line`. `severity` (critical, high, medium, low or info), `rule_id`, `file` and `line`.
medusa skips the SKIPPED suffixes (.mjs, .cjs, any case), so such a file is
copied under its name with ALIAS (.js) appended, again while a path or
folder of the change holds that name, or under the sha1 of its path with
ALIAS when that file name would pass NAME_MAX bytes; its findings are
fingerprinted from that copy and reported and accepted at its repository path.
Cache: Cache:
The findings of a scan, keyed by the sha256 of this script, the resolved The findings of a scan, keyed by the sha256 of this script, the resolved
medusa binary with its mtime and size, PATH, every MEDUSA_* variable and medusa binary with its mtime and size, PATH, every MEDUSA_* variable and
the path, mode and content sha256 of every scanned file, in the tree path, repository path, mode and content sha256 of every scanned file, in
<council root>/security-cache/<key>.json for CACHE_SECONDS. A later scan <council root>/security-cache/<key>.json for CACHE_SECONDS. A later scan
with the same key reads them instead of running medusa; any edit of a with the same key reads them instead of running medusa; any edit of a
scanned file, of medusa or of this script misses. The folder is used only scanned file, of medusa or of this script misses. The folder is used only
@@ -82,6 +87,9 @@ SAFE = 100
SHOWN = 5 SHOWN = 5
REACH = 2 REACH = 2
WORKERS = 4 WORKERS = 4
SKIPPED = (".mjs", ".cjs")
ALIAS = ".js"
NAME_MAX = 255
CACHE = "security-cache" CACHE = "security-cache"
CACHE_SECONDS = 24 * 3600 CACHE_SECONDS = 24 * 3600
NOTHING = type(None) NOTHING = type(None)
@@ -120,18 +128,42 @@ def changed(root: Path, base: str) -> list[str]:
return sorted({p for p in listed.split("\0") if p}) return sorted({p for p in listed.split("\0") if p})
def copy(root: Path, paths: list[str], tree: Path) -> int: def alias(name: str, taken: set[str]) -> str:
copied = 0 """Returns:
the tree path of name, whose suffix medusa skips: its file name with
ALIAS appended, again while taken holds that path, or the sha1 of name
treated the same way when that file name would pass NAME_MAX bytes.
"""
folder = Path(name).parent
for stem in (Path(name).name, digest(name)):
scratch = stem + ALIAS
while (folder / scratch).as_posix() in taken:
scratch += ALIAS
if len(os.fsencode(scratch)) <= NAME_MAX:
break
return (folder / scratch).as_posix()
def copy(root: Path, paths: list[str], tree: Path) -> dict[str, str]:
"""Returns:
the tree path of every copied file mapped to its repository path.
"""
names = {}
taken = set(paths) | {parent.as_posix() for name in paths for parent in Path(name).parents}
for name in paths: for name in paths:
source = root / name source = root / name
# Trip-wire: a symlink's target may lie outside the repository; even a copied link lets the scan read it. # Trip-wire: a symlink's target may lie outside the repository; even a copied link lets the scan read it.
if source.is_symlink() or not source.is_file(): if source.is_symlink() or not source.is_file():
continue continue
target = tree / name scratch = name
if Path(name).suffix.lower() in SKIPPED:
scratch = alias(name, taken)
taken.add(scratch)
target = tree / scratch
target.parent.mkdir(parents=True, exist_ok=True) target.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(source, target) shutil.copy2(source, target)
copied += 1 names[scratch] = name
return copied return names
def located(name: str, tree: Path) -> str: def located(name: str, tree: Path) -> str:
@@ -240,7 +272,7 @@ def accept(argv: list[str]) -> int:
return 0 return 0
def findings(reports: Path, tree: Path) -> list[dict]: def findings(reports: Path, tree: Path, names: dict[str, str]) -> list[dict]:
files = sorted(reports.glob("medusa-scan-*.json")) if reports.is_dir() else [] files = sorted(reports.glob("medusa-scan-*.json")) if reports.is_dir() else []
if not files: if not files:
raise Unreadable(f"no medusa-scan-*.json in {reports}") raise Unreadable(f"no medusa-scan-*.json in {reports}")
@@ -257,12 +289,12 @@ def findings(reports: Path, tree: Path) -> list[dict]:
if severity not in WEIGHTS: if severity not in WEIGHTS:
raise Unreadable(f"{files[-1].name}: a finding without a known severity") raise Unreadable(f"{files[-1].name}: a finding without a known severity")
line = entry.get("line") if isinstance(entry.get("line"), int) else None line = entry.get("line") if isinstance(entry.get("line"), int) else None
path = located(str(entry.get("file") or "?"), tree) scratch = located(str(entry.get("file") or "?"), tree)
sha1, context = fingerprint(tree, path, line) sha1, context = fingerprint(tree, scratch, line)
found.append({ found.append({
"severity": severity, "severity": severity,
"rule": entry.get("rule_id") if isinstance(entry.get("rule_id"), str) else None, "rule": entry.get("rule_id") if isinstance(entry.get("rule_id"), str) else None,
"path": path, "path": names.get(scratch, scratch),
"line": line, "line": line,
"sha1": sha1, "sha1": sha1,
"context": context, "context": context,
@@ -271,14 +303,15 @@ def findings(reports: Path, tree: Path) -> list[dict]:
return sorted(found, key=lambda f: (rank.index(f["severity"]), f["path"], f["line"] or 0)) return sorted(found, key=lambda f: (rank.index(f["severity"]), f["path"], f["line"] or 0))
def keyed(medusa: str, tree: Path) -> str: def keyed(medusa: str, tree: Path, names: dict[str, str]) -> str:
binary = Path(medusa).resolve() binary = Path(medusa).resolve()
info = binary.stat() info = binary.stat()
key = hashlib.sha256(Path(__file__).read_bytes()) key = hashlib.sha256(Path(__file__).read_bytes())
key.update(f"\0{binary}\0{info.st_mtime_ns}\0{info.st_size}\0{os.environ.get('PATH', '')}\0".encode()) key.update(f"\0{binary}\0{info.st_mtime_ns}\0{info.st_size}\0{os.environ.get('PATH', '')}\0".encode())
key.update("".join(f"{k}={v}\0" for k, v in sorted(os.environ.items()) if k.startswith("MEDUSA_")).encode()) key.update("".join(f"{k}={v}\0" for k, v in sorted(os.environ.items()) if k.startswith("MEDUSA_")).encode())
for path in sorted(p for p in tree.rglob("*") if p.is_file()): for scratch, name in sorted(names.items()):
key.update(f"{path.relative_to(tree).as_posix()}\0{path.stat().st_mode & 0o777:o}\0".encode()) path = tree / scratch
key.update(f"{scratch}\0{name}\0{path.stat().st_mode & 0o777:o}\0".encode())
key.update(hashlib.sha256(path.read_bytes()).digest()) key.update(hashlib.sha256(path.read_bytes()).digest())
return key.hexdigest() return key.hexdigest()
@@ -339,10 +372,11 @@ def scan(medusa: str, root: Path, base: str, cache: Path | None) -> tuple[int, l
tree, reports, home = work / "tree", work / "reports", work / "home" tree, reports, home = work / "tree", work / "reports", work / "home"
tree.mkdir() tree.mkdir()
home.mkdir() home.mkdir()
scanned = copy(root, changed(root, base), tree) names = copy(root, changed(root, base), tree)
scanned = len(names)
if not scanned: if not scanned:
return 0, [] return 0, []
key = keyed(medusa, tree) key = keyed(medusa, tree, names)
hit = recalled(cache, key) if cache else None hit = recalled(cache, key) if cache else None
if hit is not None: if hit is not None:
return scanned, hit return scanned, hit
@@ -353,7 +387,7 @@ def scan(medusa: str, root: Path, base: str, cache: Path | None) -> tuple[int, l
env={**os.environ, "HOME": str(home)}, env={**os.environ, "HOME": str(home)},
) )
try: try:
found = findings(reports, tree) found = findings(reports, tree, names)
except Unreadable as error: except Unreadable as error:
raise Unreadable(f"{error} (medusa exit {result.returncode})") from None raise Unreadable(f"{error} (medusa exit {result.returncode})") from None
if cache: if cache:
+58 -1
View File
@@ -24,8 +24,11 @@ pathlib.Path(__file__).with_name("home.txt").write_text(os.environ["HOME"])
pathlib.Path(os.environ["HOME"], ".medusa").mkdir() pathlib.Path(os.environ["HOME"], ".medusa").mkdir()
target, out = pathlib.Path(args[1]), pathlib.Path(args[args.index("-o") + 1]) target, out = pathlib.Path(args[1]), pathlib.Path(args[args.index("-o") + 1])
out.mkdir(parents=True) out.mkdir(parents=True)
scannable = sorted(p for p in target.rglob("*") if p.is_file() and p.suffix.lower() not in (".mjs", ".cjs"))
if not scannable:
sys.exit(0)
findings = [] findings = []
for path in sorted(p for p in target.rglob("*") if p.is_file()): for path in scannable:
for number, text in enumerate(path.read_text().splitlines(), 1): for number, text in enumerate(path.read_text().splitlines(), 1):
for severity in ("CRITICAL", "HIGH", "MEDIUM", "LOW", "INFO"): for severity in ("CRITICAL", "HIGH", "MEDIUM", "LOW", "INFO"):
if "MARK-" + severity in text: if "MARK-" + severity in text:
@@ -74,6 +77,7 @@ class TestSecurity(unittest.TestCase):
subprocess.run(["git", *args], cwd=self.repo, env=self.env, capture_output=True, check=True) subprocess.run(["git", *args], cwd=self.repo, env=self.env, capture_output=True, check=True)
def write(self, path: str, text: str) -> None: def write(self, path: str, text: str) -> None:
(self.repo / path).parent.mkdir(parents=True, exist_ok=True)
(self.repo / path).write_text(text, encoding="utf-8") (self.repo / path).write_text(text, encoding="utf-8")
def scan(self, *extra: str, env: dict[str, str]) -> subprocess.CompletedProcess: def scan(self, *extra: str, env: dict[str, str]) -> subprocess.CompletedProcess:
@@ -221,6 +225,59 @@ class TestSecurity(unittest.TestCase):
result = self.scan(env=self.env) result = self.scan(env=self.env)
self.assertEqual(result.stdout, "🛡️ security 100 % (medusa: 0 findings, 2 accepted in 1 file)\n") self.assertEqual(result.stdout, "🛡️ security 100 % (medusa: 0 findings, 2 accepted in 1 file)\n")
def test_mjs_and_cjs_files_medusa_skips_are_scanned_and_reported_at_their_repository_path(self):
line = "run(request.query) // MARK-HIGH"
self.write("tool.mjs", f"import run from 'run';\n{line}\n")
result = self.scan(env=self.env)
self.assertEqual(
(result.stdout, result.returncode),
("🛡️ security 75 % (medusa: 1 finding in 1 file: high tool.mjs:2)\n", 3),
)
sha1 = hashlib.sha1(line.encode()).hexdigest()
result = self.accept(
"--rule", "FAKE-HIGH", "--path", "tool.mjs", "--sha1", sha1, "--line", "2", "--reason", "fixture"
)
self.assertEqual(result.returncode, 0, result.stderr)
self.write("tool.mjs.js", "a // MARK-MEDIUM\n")
self.write("lib.CJS", "b // MARK-LOW\n")
result = self.scan(env=self.env)
self.assertEqual(
(result.stdout, result.returncode),
("🛡️ security 88 % (medusa: 2 findings, 1 accepted in 3 files: medium tool.mjs.js:1, low lib.CJS:1)\n", 3),
)
def test_a_folder_named_like_the_alias_keeps_both_paths_apart(self):
self.write("tool.mjs", "a // MARK-HIGH\n")
self.write("tool.mjs.js/inner.py", "b # MARK-LOW\n")
result = self.scan(env=self.env)
self.assertEqual(
(result.stdout, result.returncode),
("🛡️ security 73 % (medusa: 2 findings in 2 files: high tool.mjs:1, low tool.mjs.js/inner.py:1)\n", 3),
)
def test_the_cache_keeps_the_repository_path_of_an_aliased_file(self):
text = "a // MARK-HIGH\n"
self.write("tool.mjs", text)
first = json.loads(self.scan("--json", env=self.env).stdout)
(self.repo / "tool.mjs").unlink()
self.write("tool.mjs.js", text)
second = json.loads(self.scan("--json", env=self.env).stdout)
self.assertEqual([f["path"] for f in first["findings"] + second["findings"]], ["tool.mjs", "tool.mjs.js"])
def test_a_nested_mjs_file_is_reported_at_its_nested_path(self):
self.write("a/b/tool.mjs", "a // MARK-MEDIUM\n")
result = self.scan(env=self.env)
self.assertEqual(
(result.stdout, result.returncode),
("🛡️ security 90 % (medusa: 1 finding in 1 file: medium a/b/tool.mjs:1)\n", 3),
)
def test_an_mjs_file_name_without_room_for_the_alias_is_scanned_under_a_short_one(self):
name = "t" * 250 + ".mjs"
self.write(name, "a // MARK-LOW\n")
found = json.loads(self.scan("--json", env=self.env).stdout)
self.assertEqual((found["files"], [(f["path"], f["line"]) for f in found["findings"]]), (1, [(name, 1)]))
def test_critical_exits_1_and_floors_at_0(self): def test_critical_exits_1_and_floors_at_0(self):
self.write("app.py", "a # MARK-CRITICAL\nb # MARK-CRITICAL\nc # MARK-CRITICAL\n") self.write("app.py", "a # MARK-CRITICAL\nb # MARK-CRITICAL\nc # MARK-CRITICAL\n")
result = self.scan(env=self.env) result = self.scan(env=self.env)