fix(kvb-check-security): scan files MEDUSA's substring path excludes drop
MEDUSA matches its exclude paths as substrings of the whole path (core/parallel.py), so plain source under utils/env/, distro.py, builder.py or .github/ went unscanned and a diff of only such files got no score. security.py now moves each file that only a path exclusion drops to kvb-scan/<sha1 of its path>/<name> in the scan tree, or to kvb-scan/<sha1><suffix> when the name itself holds a pattern, and reports and accepts its findings at the repository path. File patterns such as *.min.js still skip. Reported by Stage 2.0. 🎯 Fix: 95 % 💥 Break: 3 % 💸 Tokens: 94 % ⏱️ Velocity: – 🛡️ Security: 100 % 💰 Value: ? 🏆 Score: 98 🚦 Gate: ✅ 90/5, tokens and velocity <= 100, security 100 % Created-with: https://s.veen.world/skills Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
6b36e80dad
commit
731d0df43b
3 files changed
+81
-35
No files matched your search
@@ -3,9 +3,8 @@
|
||||
The score counts only files medusa scans; its types, exclusions and limits
|
||||
come from its source.
|
||||
|
||||
- `not covered`: a type medusa never scans (`*.j2`, `Makefile`) or a path its
|
||||
default exclusions drop (`env`, `build`, `dist`, `obj`, `vendor` and the
|
||||
like). Review them by hand.
|
||||
- `not covered`: a type medusa never scans (`*.j2`, `Makefile`) or a file
|
||||
pattern it skips (`*.min.js`, `*.map`). Review them by hand.
|
||||
- `run incomplete`: a scanner passed `--timeout` or the run `--deadline`,
|
||||
mostly on a loaded CPU or a long file. A plain `PER_SCANNER_TIMEOUT` number
|
||||
counts as ignoring `MEDUSA_SCANNER_TIMEOUT`, even if read elsewhere.
|
||||
|
||||
@@ -43,11 +43,16 @@ Scan:
|
||||
MAX_SCAN_FILE_SIZE; from scanners/base.py MAX_RULE_SCAN_LINES; from
|
||||
config.py the default exclude_paths and exclude_files. Without all of
|
||||
them no file can be proven covered and a diff with files gets no score.
|
||||
Not covered: a copied file of a type medusa never scans, or one its
|
||||
exclusions drop (a path part equal to or matching a path pattern, a path
|
||||
pattern anywhere inside the path, a file pattern), is named as such and
|
||||
left out of the score and its proof; a diff of such files only gets no
|
||||
score.
|
||||
Path exclusions: medusa drops a path whose part equals or matches a path
|
||||
pattern, or that holds a path pattern anywhere inside (env drops
|
||||
utils/env/ and environ.py alike), so a copied file they drop moves to
|
||||
NEUTRAL/<sha1 of its repository path>/<its name>, or to
|
||||
NEUTRAL/<that sha1><its suffix> when its name alone holds a pattern, and
|
||||
is reported and accepted at its repository path.
|
||||
Not covered: a copied file of a type medusa never scans, one a file
|
||||
pattern drops, or one no NEUTRAL path makes medusa take up, is named as
|
||||
such and left out of the score and its proof; a diff of such files only
|
||||
gets no score.
|
||||
Coverage, checked before medusa runs: no scanned file over medusa's
|
||||
MAX_SCAN_FILE_SIZE bytes (skipped) or MAX_RULE_SCAN_LINES lines, with CR,
|
||||
CRLF and LF as breaks as its scanners read text (regex scan cut). Such a
|
||||
@@ -146,6 +151,7 @@ CEILING = 86400
|
||||
SKIPPED = (".mjs", ".cjs")
|
||||
ALIAS = ".js"
|
||||
NAME_MAX = 255
|
||||
NEUTRAL = "kvb-scan"
|
||||
CACHE = "security-cache"
|
||||
CACHE_SECONDS = 24 * 3600
|
||||
NOTHING = type(None)
|
||||
@@ -616,25 +622,64 @@ def medusa_run(command: list[str], work: Path, env: dict[str, str], deadline: in
|
||||
return process.returncode, out.decode(errors="replace"), time.monotonic() - start
|
||||
|
||||
|
||||
def typed(path: PurePosixPath, found: Rules) -> bool:
|
||||
extension = os.path.splitext(path.name)[1].lower()
|
||||
return (
|
||||
extension in found.extensions or path.name in found.names or path.name.startswith(found.prefixes)
|
||||
or path.name.endswith(found.suffix)
|
||||
or (len(path.parts) == 2 and path.parts[0] in found.folders and extension == ".md")
|
||||
)
|
||||
|
||||
|
||||
def path_dropped(name: str, found: Rules) -> bool:
|
||||
"""Returns:
|
||||
whether a path exclusion of medusa drops this tree path: a part equal to
|
||||
or matching a path pattern, or a path pattern anywhere inside it, as
|
||||
medusa's find_scannable_files matches them.
|
||||
"""
|
||||
return any(
|
||||
part == pattern or WILD.search(pattern) and fnmatchcase(part, pattern)
|
||||
for part in PurePosixPath(name).parts for pattern in found.paths
|
||||
) or any(pattern in name or WILD.search(pattern) and fnmatchcase(name, f"*{pattern}*") for pattern in found.paths)
|
||||
|
||||
|
||||
def file_dropped(path: PurePosixPath, found: Rules) -> bool:
|
||||
return any(fnmatchcase(path.name, pattern) for pattern in found.files)
|
||||
|
||||
|
||||
def taken(name: str, found: Rules) -> bool:
|
||||
"""Returns:
|
||||
whether medusa's find_scannable_files takes up a file of this tree path:
|
||||
a type it scans, and no exclusion that drops it.
|
||||
"""
|
||||
path = PurePosixPath(name)
|
||||
extension = os.path.splitext(path.name)[1].lower()
|
||||
typed = (
|
||||
extension in found.extensions or path.name in found.names or path.name.startswith(found.prefixes)
|
||||
or path.name.endswith(found.suffix)
|
||||
or (len(path.parts) == 2 and path.parts[0] in found.folders and extension == ".md")
|
||||
)
|
||||
dropped = (
|
||||
any(part == pattern or WILD.search(pattern) and fnmatchcase(part, pattern)
|
||||
for part in path.parts for pattern in found.paths)
|
||||
or any(pattern in name or WILD.search(pattern) and fnmatchcase(name, f"*{pattern}*") for pattern in found.paths)
|
||||
or any(fnmatchcase(path.name, pattern) for pattern in found.files)
|
||||
)
|
||||
return typed and not dropped
|
||||
return typed(path, found) and not path_dropped(name, found) and not file_dropped(path, found)
|
||||
|
||||
|
||||
def relocated(names: dict[str, str], tree: Path, found: Rules) -> dict[str, str]:
|
||||
"""Args:
|
||||
names: copy() of the change, tree path to repository path.
|
||||
tree: the temporary tree medusa scans.
|
||||
found: medusa's rules.
|
||||
|
||||
Returns:
|
||||
names after each file a path exclusion drops moved in tree to
|
||||
NEUTRAL/<sha1 of its repository path>/<its name>, or to
|
||||
NEUTRAL/<that sha1><its suffix> where medusa takes up only that; a file
|
||||
neither path makes taken, as one a file pattern drops, stays put.
|
||||
"""
|
||||
moved = {}
|
||||
for scratch, name in names.items():
|
||||
target, path = scratch, PurePosixPath(scratch)
|
||||
if path_dropped(scratch, found) and not file_dropped(path, found):
|
||||
key = digest(name)
|
||||
candidates = (f"{NEUTRAL}/{key}/{path.name}", f"{NEUTRAL}/{key}{path.suffix}")
|
||||
target = next((candidate for candidate in candidates if taken(candidate, found)), scratch)
|
||||
if target != scratch:
|
||||
(tree / target).parent.mkdir(parents=True, exist_ok=True)
|
||||
(tree / scratch).rename(tree / target)
|
||||
moved[target] = name
|
||||
return moved
|
||||
|
||||
|
||||
def sized(tree: Path) -> dict[str, tuple[int, int, int]]:
|
||||
@@ -776,6 +821,7 @@ def scan(
|
||||
known = rules(folder)
|
||||
if known is None:
|
||||
raise Uncovered("medusa's own types, exclusions and limits are unreadable from its source")
|
||||
names = relocated(names, tree, known)
|
||||
sizes = {names[scratch]: size for scratch, size in sized(tree).items() if taken(scratch, known)}
|
||||
missed = sorted(set(names.values()) - sizes.keys())
|
||||
scanned = len(sizes)
|
||||
|
||||
+15
-14
@@ -609,21 +609,22 @@ class TestSecurity(unittest.TestCase):
|
||||
unread = f"{never}: medusa's own types, exclusions and limits are unreadable from its source: {HAND}"
|
||||
self.assertEqual((result.stdout, result.returncode), (f"🛡️ security – ({unread})\n", 4))
|
||||
|
||||
def test_a_path_medusa_excludes_is_listed_and_the_rest_scored(self):
|
||||
def test_a_file_a_path_exclusion_drops_is_scanned_and_named_at_its_repository_path(self):
|
||||
self.write("app.py", "print(1)\nx = 1 # MARK-HIGH\n")
|
||||
(self.repo / "cli" / "build").mkdir(parents=True)
|
||||
self.write("cli/build/run.py", "y = 1 # MARK-CRITICAL\n")
|
||||
self.write("environ.py", "z = 1 # MARK-CRITICAL\n")
|
||||
result = self.scan(env=self.env)
|
||||
self.assertEqual((result.stdout, result.returncode), (
|
||||
"🛡️ security 75 % (medusa: 1 finding in 1 file: high app.py:2; not covered: cli/build/run.py, environ.py)\n",
|
||||
3,
|
||||
))
|
||||
self.write("app.py", "print(1)\n")
|
||||
result = self.scan(env=self.env)
|
||||
never = "medusa never covers this diff, nothing cached"
|
||||
none = f"{never}: medusa scans none of its files, not covered: cli/build/run.py, environ.py: {HAND}"
|
||||
self.assertEqual((result.stdout, result.returncode), (f"🛡️ security – ({none})\n", 4))
|
||||
self.write("utils/env/run.py", "y = 1 # MARK-LOW\n")
|
||||
self.write("environ.py", "z = 1 # MARK-LOW\n")
|
||||
self.write("build/lib.min.js", "a // MARK-CRITICAL\n")
|
||||
found = json.loads(self.scan("--json", env=self.env).stdout)
|
||||
self.assertEqual(
|
||||
sorted((f["path"], f["line"], f["severity"]) for f in found["findings"]),
|
||||
[("app.py", 2, "high"), ("environ.py", 1, "low"), ("utils/env/run.py", 1, "low")],
|
||||
)
|
||||
self.assertEqual((found["files"], found["not_covered"]), (3, ["build/lib.min.js"]))
|
||||
sha1 = next(f["sha1"] for f in found["findings"] if f["path"] == "utils/env/run.py")
|
||||
accepted = self.accept("--rule", "FAKE-LOW", "--path", "utils/env/run.py", "--sha1", sha1, "--reason", "fixture")
|
||||
self.assertEqual(accepted.returncode, 0, accepted.stderr)
|
||||
again = json.loads(self.scan("--json", env=self.env).stdout)
|
||||
self.assertEqual(sorted(f["path"] for f in again["findings"]), ["app.py", "environ.py"])
|
||||
|
||||
def test_medusa_limits_come_from_its_source(self):
|
||||
mark = "y = 1 # MARK-CRITICAL\n"
|
||||
|
||||
Reference in new issue
Block a user