diff --git a/skills/kvb-check-security/scripts/security.py b/skills/kvb-check-security/scripts/security.py
index aeecb88..ececa9d 100755
--- a/skills/kvb-check-security/scripts/security.py
+++ b/skills/kvb-check-security/scripts/security.py
@@ -20,6 +20,13 @@ Scan:
directory because medusa creates ~/.medusa on start, read from
`
/medusa-scan-*.json`: a `findings` list whose entries carry
`severity` (critical, high, medium, low or info), `rule_id`, `file` and `line`.
+ medusa skips the SKIPPED suffixes (.mjs, .cjs, any case), so such a file is
+ copied under its name with ALIAS (.js) appended, again while a path or
+ folder of the change holds that name, or under the sha1 of its path with
+ ALIAS when that file name would pass NAME_MAX bytes; its findings are
+ fingerprinted from that copy and reported and accepted at its repository
+ path. Rules, coverage and not covered judge the tree path medusa scans and
+ name the repository path.
Rules: medusa's own RULES, read without running medusa from the medusa
package that the interpreter of medusa's shebang (an `env` program looked
up on PATH) imports with medusa's environment: from core/parallel.py the
@@ -49,8 +56,8 @@ Scan:
Cache:
The findings of a complete scan, keyed by the sha256 of this script, the
resolved medusa binary with its mtime and size, PATH, the files not covered,
- every MEDUSA_* variable and
- the path, mode and content sha256 of every scanned file, in
+ every MEDUSA_* variable and the tree path, repository path, mode and
+ content sha256 of every copied file, in
/security-cache/.json for CACHE_SECONDS. A later scan
with the same key reads them instead of running medusa; any edit of a
scanned file, of medusa or of this script misses. The folder is used only
@@ -114,6 +121,9 @@ SAFE = 100
SHOWN = 5
REACH = 2
WORKERS = 4
+SKIPPED = (".mjs", ".cjs")
+ALIAS = ".js"
+NAME_MAX = 255
CACHE = "security-cache"
CACHE_SECONDS = 24 * 3600
NOTHING = type(None)
@@ -188,18 +198,42 @@ def changed(root: Path, base: str) -> list[str]:
return sorted({p for p in listed.split("\0") if p})
-def copy(root: Path, paths: list[str], tree: Path) -> int:
- copied = 0
+def alias(name: str, taken: set[str]) -> str:
+ """Returns:
+ the tree path of name, whose suffix medusa skips: its file name with
+ ALIAS appended, again while taken holds that path, or the sha1 of name
+ treated the same way when that file name would pass NAME_MAX bytes.
+ """
+ folder = Path(name).parent
+ for stem in (Path(name).name, digest(name)):
+ scratch = stem + ALIAS
+ while (folder / scratch).as_posix() in taken:
+ scratch += ALIAS
+ if len(os.fsencode(scratch)) <= NAME_MAX:
+ break
+ return (folder / scratch).as_posix()
+
+
+def copy(root: Path, paths: list[str], tree: Path) -> dict[str, str]:
+ """Returns:
+ the tree path of every copied file mapped to its repository path.
+ """
+ names = {}
+ taken = set(paths) | {parent.as_posix() for name in paths for parent in Path(name).parents}
for name in paths:
source = root / name
# Trip-wire: a symlink's target may lie outside the repository; even a copied link lets the scan read it.
if source.is_symlink() or not source.is_file():
continue
- target = tree / name
+ scratch = name
+ if Path(name).suffix.lower() in SKIPPED:
+ scratch = alias(name, taken)
+ taken.add(scratch)
+ target = tree / scratch
target.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(source, target)
- copied += 1
- return copied
+ names[scratch] = name
+ return names
def located(name: str, tree: Path) -> str:
@@ -308,9 +342,9 @@ def accept(argv: list[str]) -> int:
return 0
-def findings(reports: Path, tree: Path) -> tuple[list[dict], dict]:
+def findings(reports: Path, tree: Path, names: dict[str, str]) -> tuple[list[dict], dict]:
"""Returns:
- the findings of medusa's report and its `scan_summary`.
+ the findings of medusa's report at their repository paths and its `scan_summary`.
"""
files = sorted(reports.glob("medusa-scan-*.json")) if reports.is_dir() else []
if not files:
@@ -331,12 +365,12 @@ def findings(reports: Path, tree: Path) -> tuple[list[dict], dict]:
if severity not in WEIGHTS:
raise Unreadable(f"{files[-1].name}: a finding without a known severity")
line = entry.get("line") if isinstance(entry.get("line"), int) else None
- path = located(str(entry.get("file") or "?"), tree)
- sha1, context = fingerprint(tree, path, line)
+ scratch = located(str(entry.get("file") or "?"), tree)
+ sha1, context = fingerprint(tree, scratch, line)
found.append({
"severity": severity,
"rule": entry.get("rule_id") if isinstance(entry.get("rule_id"), str) else None,
- "path": path,
+ "path": names.get(scratch, scratch),
"line": line,
"sha1": sha1,
"context": context,
@@ -512,15 +546,16 @@ def cut(code: int, said: list[str], summary: dict, sizes: dict[str, tuple[int, i
return None
-def keyed(medusa: str, tree: Path, missed: list[str]) -> str:
+def keyed(medusa: str, tree: Path, names: dict[str, str], missed: list[str]) -> str:
binary = Path(medusa).resolve()
info = binary.stat()
key = hashlib.sha256(Path(__file__).read_bytes())
key.update(f"\0{binary}\0{info.st_mtime_ns}\0{info.st_size}\0{os.environ.get('PATH', '')}\0".encode())
key.update("".join(f"{name}\0" for name in missed).encode() + b"\0")
key.update("".join(f"{k}={v}\0" for k, v in sorted(os.environ.items()) if k.startswith("MEDUSA_")).encode())
- for path in sorted(p for p in tree.rglob("*") if p.is_file()):
- key.update(f"{path.relative_to(tree).as_posix()}\0{path.stat().st_mode & 0o777:o}\0".encode())
+ for scratch, name in sorted(names.items()):
+ path = tree / scratch
+ key.update(f"{scratch}\0{name}\0{path.stat().st_mode & 0o777:o}\0".encode())
key.update(hashlib.sha256(path.read_bytes()).digest())
return key.hexdigest()
@@ -577,31 +612,30 @@ def remember(folder: Path, key: str, found: list[dict]) -> None:
def scan(medusa: str, root: Path, base: str, cache: Path | None) -> tuple[int, list[dict], list[str]]:
"""Returns:
- the count of scanned files, their findings and the copied files of a
- type medusa never scans.
+ the count of scanned files, their findings and the repository paths
+ of the copied files medusa never takes up.
"""
with tempfile.TemporaryDirectory(prefix="kvb-security-") as scratch:
work = Path(scratch)
tree, reports, home = work / "tree", work / "reports", work / "home"
tree.mkdir()
home.mkdir()
- copy(root, changed(root, base), tree)
- env = {**os.environ, "HOME": str(home)}
- sizes = sized(tree)
- if not sizes:
+ names = copy(root, changed(root, base), tree)
+ if not names:
return 0, [], []
+ env = {**os.environ, "HOME": str(home)}
known = rules(package(medusa, work, env))
if known is None:
raise Uncovered("medusa's own types, exclusions and limits are unreadable from its source")
- missed = [name for name in sizes if not taken(name, known)]
- sizes = {name: size for name, size in sizes.items() if name not in missed}
+ sizes = {names[scratch]: size for scratch, size in sized(tree).items() if taken(scratch, known)}
+ missed = sorted(set(names.values()) - sizes.keys())
scanned = len(sizes)
if not scanned:
raise Uncovered(f"medusa scans none of its files, not covered: {shown(missed)}")
over = oversized(sizes, known)
if over:
raise Uncovered(over)
- key = keyed(medusa, tree, missed)
+ key = keyed(medusa, tree, names, missed)
hit = recalled(cache, key) if cache else None
if hit is not None:
return scanned, hit, missed
@@ -611,7 +645,7 @@ def scan(medusa: str, root: Path, base: str, cache: Path | None) -> tuple[int, l
cwd=work, stdin=subprocess.DEVNULL, capture_output=True, check=False, env=env,
)
try:
- found, summary = findings(reports, tree)
+ found, summary = findings(reports, tree, names)
except Unreadable as error:
raise Unreadable(f"{error} (medusa exit {result.returncode})") from None
said = [line.replace(f"{tree}/", "").strip(" \t!⚠️") for line in result.stdout.decode(errors="replace").splitlines()]
diff --git a/tests/test_security.py b/tests/test_security.py
index 2bf2d07..e202b67 100644
--- a/tests/test_security.py
+++ b/tests/test_security.py
@@ -27,7 +27,7 @@ out.mkdir(parents=True)
limit = int(os.environ.get("FAKE_MEDUSA_FILE_LIMIT", str(2 * 1024 * 1024)))
findings, taken, counted = [], 0, 0
for path in sorted(p for p in target.rglob("*") if p.is_file()):
- if path.suffix not in (".py", ".json") and not path.name.startswith("Dockerfile"):
+ if path.suffix.lower() not in (".py", ".json", ".js") and not path.name.startswith("Dockerfile"):
continue
if path.name == os.environ.get("FAKE_MEDUSA_LOSE") or any(p in str(path.relative_to(target)) for p in ("build", "env")):
continue
@@ -67,7 +67,7 @@ PARALLEL = """MAX_SCAN_FILE_SIZE = {size}
class MedusaParallelScanner:
- FILE_SCANNERS = {{".py": "python", ".json": "json"}}
+ FILE_SCANNERS = {{".py": "python", ".json": "json", ".js": "javascript"}}
def find_scannable_files(self):
special_exact_names: Set[str] = {{"mcp.json"}}
@@ -126,6 +126,7 @@ class TestSecurity(unittest.TestCase):
subprocess.run(["git", *args], cwd=self.repo, env=self.env, capture_output=True, check=True)
def write(self, path: str, text: str) -> None:
+ (self.repo / path).parent.mkdir(parents=True, exist_ok=True)
(self.repo / path).write_text(text, encoding="utf-8")
def scan(self, *extra: str, env: dict[str, str]) -> subprocess.CompletedProcess:
@@ -273,6 +274,87 @@ class TestSecurity(unittest.TestCase):
result = self.scan(env=self.env)
self.assertEqual(result.stdout, "🛡️ security 100 % (medusa: 0 findings, 2 accepted in 1 file)\n")
+ def test_mjs_and_cjs_files_medusa_skips_are_scanned_and_reported_at_their_repository_path(self):
+ line = "run(request.query) // MARK-HIGH"
+ self.write("tool.mjs", f"import run from 'run';\n{line}\n")
+ result = self.scan(env=self.env)
+ self.assertEqual(
+ (result.stdout, result.returncode),
+ ("🛡️ security 75 % (medusa: 1 finding in 1 file: high tool.mjs:2)\n", 3),
+ )
+ sha1 = hashlib.sha1(line.encode()).hexdigest()
+ result = self.accept(
+ "--rule", "FAKE-HIGH", "--path", "tool.mjs", "--sha1", sha1, "--line", "2", "--reason", "fixture"
+ )
+ self.assertEqual(result.returncode, 0, result.stderr)
+ self.write("tool.mjs.js", "a // MARK-MEDIUM\n")
+ self.write("lib.CJS", "b // MARK-LOW\n")
+ result = self.scan(env=self.env)
+ self.assertEqual(
+ (result.stdout, result.returncode),
+ ("🛡️ security 88 % (medusa: 2 findings, 1 accepted in 3 files: medium tool.mjs.js:1, low lib.CJS:1)\n", 3),
+ )
+
+ def test_a_folder_named_like_the_alias_keeps_both_paths_apart(self):
+ self.write("tool.mjs", "a // MARK-HIGH\n")
+ self.write("tool.mjs.js/inner.py", "b # MARK-LOW\n")
+ result = self.scan(env=self.env)
+ self.assertEqual(
+ (result.stdout, result.returncode),
+ ("🛡️ security 73 % (medusa: 2 findings in 2 files: high tool.mjs:1, low tool.mjs.js/inner.py:1)\n", 3),
+ )
+
+ def test_the_cache_keeps_the_repository_path_of_an_aliased_file(self):
+ text = "a // MARK-HIGH\n"
+ self.write("tool.mjs", text)
+ first = json.loads(self.scan("--json", env=self.env).stdout)
+ (self.repo / "tool.mjs").unlink()
+ self.write("tool.mjs.js", text)
+ second = json.loads(self.scan("--json", env=self.env).stdout)
+ self.assertEqual([f["path"] for f in first["findings"] + second["findings"]], ["tool.mjs", "tool.mjs.js"])
+
+ def test_a_nested_mjs_file_is_reported_at_its_nested_path(self):
+ self.write("a/b/tool.mjs", "a // MARK-MEDIUM\n")
+ result = self.scan(env=self.env)
+ self.assertEqual(
+ (result.stdout, result.returncode),
+ ("🛡️ security 90 % (medusa: 1 finding in 1 file: medium a/b/tool.mjs:1)\n", 3),
+ )
+
+ def test_an_mjs_file_name_without_room_for_the_alias_is_scanned_under_a_short_one(self):
+ name = "t" * 250 + ".mjs"
+ self.write(name, "a // MARK-LOW\n")
+ found = json.loads(self.scan("--json", env=self.env).stdout)
+ self.assertEqual((found["files"], [(f["path"], f["line"]) for f in found["findings"]]), (1, [(name, 1)]))
+
+ def test_an_mjs_only_diff_scores_from_a_real_scan_and_only_a_complete_run_is_cached(self):
+ argv = self.bin / "argv.json"
+ folder = self.base / "council" / "security-cache"
+ self.write("tool.mjs", "a // MARK-HIGH\n")
+ rerun = "medusa run incomplete, nothing cached, rerun once the host is idle"
+ timeout = "1 scanner error(s)/timeout(s) during scan: JavaScriptScanner (1)"
+ cases = (
+ ({"FAKE_MEDUSA_STDOUT": f"⚠️ {timeout}\n"}, f"{rerun}: {timeout}"),
+ ({"FAKE_MEDUSA_FILE_LIMIT": "4"}, f"{rerun}: medusa counted 0 of 2 lines"),
+ )
+ for extra, problem in cases:
+ with self.subTest(problem=problem):
+ argv.unlink(missing_ok=True)
+ result = self.scan(env={**self.env, **extra})
+ self.assertEqual((result.stdout, result.returncode), (f"🛡️ security – ({problem}: {HAND})\n", 4))
+ self.assertTrue(argv.exists())
+ self.assertEqual(list(folder.glob("*.json")), [])
+ scored = ("🛡️ security 75 % (medusa: 1 finding in 1 file: high tool.mjs:1)\n", 3)
+ argv.unlink()
+ result = self.scan(env=self.env)
+ self.assertEqual((result.stdout, result.returncode), scored)
+ self.assertTrue(argv.exists())
+ self.assertEqual(len(list(folder.glob("*.json"))), 1)
+ argv.unlink()
+ result = self.scan(env=self.env)
+ self.assertEqual((result.stdout, result.returncode), scored)
+ self.assertFalse(argv.exists())
+
def test_critical_exits_1_and_floors_at_0(self):
self.write("app.py", "a # MARK-CRITICAL\nb # MARK-CRITICAL\nc # MARK-CRITICAL\n")
result = self.scan(env=self.env)