Merge branch 'fix/kvb-check-security-mjs' into unify/skills-20261112-night
This commit is contained in:
commit
5e97fd5b47
2 files changed
+108
-17
No files matched your search
@@ -20,11 +20,16 @@ Scan:
|
|||||||
directory because medusa creates ~/.medusa on start, read from
|
directory because medusa creates ~/.medusa on start, read from
|
||||||
`<dir>/medusa-scan-*.json`: a `findings` list whose entries carry
|
`<dir>/medusa-scan-*.json`: a `findings` list whose entries carry
|
||||||
`severity` (critical, high, medium, low or info), `rule_id`, `file` and `line`.
|
`severity` (critical, high, medium, low or info), `rule_id`, `file` and `line`.
|
||||||
|
medusa skips the SKIPPED suffixes (.mjs, .cjs, any case), so such a file is
|
||||||
|
copied under its name with ALIAS (.js) appended, again while a path or
|
||||||
|
folder of the change holds that name, or under the sha1 of its path with
|
||||||
|
ALIAS when that file name would pass NAME_MAX bytes; its findings are
|
||||||
|
fingerprinted from that copy and reported and accepted at its repository path.
|
||||||
|
|
||||||
Cache:
|
Cache:
|
||||||
The findings of a scan, keyed by the sha256 of this script, the resolved
|
The findings of a scan, keyed by the sha256 of this script, the resolved
|
||||||
medusa binary with its mtime and size, PATH, every MEDUSA_* variable and
|
medusa binary with its mtime and size, PATH, every MEDUSA_* variable and
|
||||||
the path, mode and content sha256 of every scanned file, in
|
the tree path, repository path, mode and content sha256 of every scanned file, in
|
||||||
<council root>/security-cache/<key>.json for CACHE_SECONDS. A later scan
|
<council root>/security-cache/<key>.json for CACHE_SECONDS. A later scan
|
||||||
with the same key reads them instead of running medusa; any edit of a
|
with the same key reads them instead of running medusa; any edit of a
|
||||||
scanned file, of medusa or of this script misses. The folder is used only
|
scanned file, of medusa or of this script misses. The folder is used only
|
||||||
@@ -82,6 +87,9 @@ SAFE = 100
|
|||||||
SHOWN = 5
|
SHOWN = 5
|
||||||
REACH = 2
|
REACH = 2
|
||||||
WORKERS = 4
|
WORKERS = 4
|
||||||
|
SKIPPED = (".mjs", ".cjs")
|
||||||
|
ALIAS = ".js"
|
||||||
|
NAME_MAX = 255
|
||||||
CACHE = "security-cache"
|
CACHE = "security-cache"
|
||||||
CACHE_SECONDS = 24 * 3600
|
CACHE_SECONDS = 24 * 3600
|
||||||
NOTHING = type(None)
|
NOTHING = type(None)
|
||||||
@@ -120,18 +128,42 @@ def changed(root: Path, base: str) -> list[str]:
|
|||||||
return sorted({p for p in listed.split("\0") if p})
|
return sorted({p for p in listed.split("\0") if p})
|
||||||
|
|
||||||
|
|
||||||
def copy(root: Path, paths: list[str], tree: Path) -> int:
|
def alias(name: str, taken: set[str]) -> str:
|
||||||
copied = 0
|
"""Returns:
|
||||||
|
the tree path of name, whose suffix medusa skips: its file name with
|
||||||
|
ALIAS appended, again while taken holds that path, or the sha1 of name
|
||||||
|
treated the same way when that file name would pass NAME_MAX bytes.
|
||||||
|
"""
|
||||||
|
folder = Path(name).parent
|
||||||
|
for stem in (Path(name).name, digest(name)):
|
||||||
|
scratch = stem + ALIAS
|
||||||
|
while (folder / scratch).as_posix() in taken:
|
||||||
|
scratch += ALIAS
|
||||||
|
if len(os.fsencode(scratch)) <= NAME_MAX:
|
||||||
|
break
|
||||||
|
return (folder / scratch).as_posix()
|
||||||
|
|
||||||
|
|
||||||
|
def copy(root: Path, paths: list[str], tree: Path) -> dict[str, str]:
|
||||||
|
"""Returns:
|
||||||
|
the tree path of every copied file mapped to its repository path.
|
||||||
|
"""
|
||||||
|
names = {}
|
||||||
|
taken = set(paths) | {parent.as_posix() for name in paths for parent in Path(name).parents}
|
||||||
for name in paths:
|
for name in paths:
|
||||||
source = root / name
|
source = root / name
|
||||||
# Trip-wire: a symlink's target may lie outside the repository; even a copied link lets the scan read it.
|
# Trip-wire: a symlink's target may lie outside the repository; even a copied link lets the scan read it.
|
||||||
if source.is_symlink() or not source.is_file():
|
if source.is_symlink() or not source.is_file():
|
||||||
continue
|
continue
|
||||||
target = tree / name
|
scratch = name
|
||||||
|
if Path(name).suffix.lower() in SKIPPED:
|
||||||
|
scratch = alias(name, taken)
|
||||||
|
taken.add(scratch)
|
||||||
|
target = tree / scratch
|
||||||
target.parent.mkdir(parents=True, exist_ok=True)
|
target.parent.mkdir(parents=True, exist_ok=True)
|
||||||
shutil.copy2(source, target)
|
shutil.copy2(source, target)
|
||||||
copied += 1
|
names[scratch] = name
|
||||||
return copied
|
return names
|
||||||
|
|
||||||
|
|
||||||
def located(name: str, tree: Path) -> str:
|
def located(name: str, tree: Path) -> str:
|
||||||
@@ -240,7 +272,7 @@ def accept(argv: list[str]) -> int:
|
|||||||
return 0
|
return 0
|
||||||
|
|
||||||
|
|
||||||
def findings(reports: Path, tree: Path) -> list[dict]:
|
def findings(reports: Path, tree: Path, names: dict[str, str]) -> list[dict]:
|
||||||
files = sorted(reports.glob("medusa-scan-*.json")) if reports.is_dir() else []
|
files = sorted(reports.glob("medusa-scan-*.json")) if reports.is_dir() else []
|
||||||
if not files:
|
if not files:
|
||||||
raise Unreadable(f"no medusa-scan-*.json in {reports}")
|
raise Unreadable(f"no medusa-scan-*.json in {reports}")
|
||||||
@@ -257,12 +289,12 @@ def findings(reports: Path, tree: Path) -> list[dict]:
|
|||||||
if severity not in WEIGHTS:
|
if severity not in WEIGHTS:
|
||||||
raise Unreadable(f"{files[-1].name}: a finding without a known severity")
|
raise Unreadable(f"{files[-1].name}: a finding without a known severity")
|
||||||
line = entry.get("line") if isinstance(entry.get("line"), int) else None
|
line = entry.get("line") if isinstance(entry.get("line"), int) else None
|
||||||
path = located(str(entry.get("file") or "?"), tree)
|
scratch = located(str(entry.get("file") or "?"), tree)
|
||||||
sha1, context = fingerprint(tree, path, line)
|
sha1, context = fingerprint(tree, scratch, line)
|
||||||
found.append({
|
found.append({
|
||||||
"severity": severity,
|
"severity": severity,
|
||||||
"rule": entry.get("rule_id") if isinstance(entry.get("rule_id"), str) else None,
|
"rule": entry.get("rule_id") if isinstance(entry.get("rule_id"), str) else None,
|
||||||
"path": path,
|
"path": names.get(scratch, scratch),
|
||||||
"line": line,
|
"line": line,
|
||||||
"sha1": sha1,
|
"sha1": sha1,
|
||||||
"context": context,
|
"context": context,
|
||||||
@@ -271,14 +303,15 @@ def findings(reports: Path, tree: Path) -> list[dict]:
|
|||||||
return sorted(found, key=lambda f: (rank.index(f["severity"]), f["path"], f["line"] or 0))
|
return sorted(found, key=lambda f: (rank.index(f["severity"]), f["path"], f["line"] or 0))
|
||||||
|
|
||||||
|
|
||||||
def keyed(medusa: str, tree: Path) -> str:
|
def keyed(medusa: str, tree: Path, names: dict[str, str]) -> str:
|
||||||
binary = Path(medusa).resolve()
|
binary = Path(medusa).resolve()
|
||||||
info = binary.stat()
|
info = binary.stat()
|
||||||
key = hashlib.sha256(Path(__file__).read_bytes())
|
key = hashlib.sha256(Path(__file__).read_bytes())
|
||||||
key.update(f"\0{binary}\0{info.st_mtime_ns}\0{info.st_size}\0{os.environ.get('PATH', '')}\0".encode())
|
key.update(f"\0{binary}\0{info.st_mtime_ns}\0{info.st_size}\0{os.environ.get('PATH', '')}\0".encode())
|
||||||
key.update("".join(f"{k}={v}\0" for k, v in sorted(os.environ.items()) if k.startswith("MEDUSA_")).encode())
|
key.update("".join(f"{k}={v}\0" for k, v in sorted(os.environ.items()) if k.startswith("MEDUSA_")).encode())
|
||||||
for path in sorted(p for p in tree.rglob("*") if p.is_file()):
|
for scratch, name in sorted(names.items()):
|
||||||
key.update(f"{path.relative_to(tree).as_posix()}\0{path.stat().st_mode & 0o777:o}\0".encode())
|
path = tree / scratch
|
||||||
|
key.update(f"{scratch}\0{name}\0{path.stat().st_mode & 0o777:o}\0".encode())
|
||||||
key.update(hashlib.sha256(path.read_bytes()).digest())
|
key.update(hashlib.sha256(path.read_bytes()).digest())
|
||||||
return key.hexdigest()
|
return key.hexdigest()
|
||||||
|
|
||||||
@@ -339,10 +372,11 @@ def scan(medusa: str, root: Path, base: str, cache: Path | None) -> tuple[int, l
|
|||||||
tree, reports, home = work / "tree", work / "reports", work / "home"
|
tree, reports, home = work / "tree", work / "reports", work / "home"
|
||||||
tree.mkdir()
|
tree.mkdir()
|
||||||
home.mkdir()
|
home.mkdir()
|
||||||
scanned = copy(root, changed(root, base), tree)
|
names = copy(root, changed(root, base), tree)
|
||||||
|
scanned = len(names)
|
||||||
if not scanned:
|
if not scanned:
|
||||||
return 0, []
|
return 0, []
|
||||||
key = keyed(medusa, tree)
|
key = keyed(medusa, tree, names)
|
||||||
hit = recalled(cache, key) if cache else None
|
hit = recalled(cache, key) if cache else None
|
||||||
if hit is not None:
|
if hit is not None:
|
||||||
return scanned, hit
|
return scanned, hit
|
||||||
@@ -353,7 +387,7 @@ def scan(medusa: str, root: Path, base: str, cache: Path | None) -> tuple[int, l
|
|||||||
env={**os.environ, "HOME": str(home)},
|
env={**os.environ, "HOME": str(home)},
|
||||||
)
|
)
|
||||||
try:
|
try:
|
||||||
found = findings(reports, tree)
|
found = findings(reports, tree, names)
|
||||||
except Unreadable as error:
|
except Unreadable as error:
|
||||||
raise Unreadable(f"{error} (medusa exit {result.returncode})") from None
|
raise Unreadable(f"{error} (medusa exit {result.returncode})") from None
|
||||||
if cache:
|
if cache:
|
||||||
|
|||||||
+58
-1
@@ -24,8 +24,11 @@ pathlib.Path(__file__).with_name("home.txt").write_text(os.environ["HOME"])
|
|||||||
pathlib.Path(os.environ["HOME"], ".medusa").mkdir()
|
pathlib.Path(os.environ["HOME"], ".medusa").mkdir()
|
||||||
target, out = pathlib.Path(args[1]), pathlib.Path(args[args.index("-o") + 1])
|
target, out = pathlib.Path(args[1]), pathlib.Path(args[args.index("-o") + 1])
|
||||||
out.mkdir(parents=True)
|
out.mkdir(parents=True)
|
||||||
|
scannable = sorted(p for p in target.rglob("*") if p.is_file() and p.suffix.lower() not in (".mjs", ".cjs"))
|
||||||
|
if not scannable:
|
||||||
|
sys.exit(0)
|
||||||
findings = []
|
findings = []
|
||||||
for path in sorted(p for p in target.rglob("*") if p.is_file()):
|
for path in scannable:
|
||||||
for number, text in enumerate(path.read_text().splitlines(), 1):
|
for number, text in enumerate(path.read_text().splitlines(), 1):
|
||||||
for severity in ("CRITICAL", "HIGH", "MEDIUM", "LOW", "INFO"):
|
for severity in ("CRITICAL", "HIGH", "MEDIUM", "LOW", "INFO"):
|
||||||
if "MARK-" + severity in text:
|
if "MARK-" + severity in text:
|
||||||
@@ -74,6 +77,7 @@ class TestSecurity(unittest.TestCase):
|
|||||||
subprocess.run(["git", *args], cwd=self.repo, env=self.env, capture_output=True, check=True)
|
subprocess.run(["git", *args], cwd=self.repo, env=self.env, capture_output=True, check=True)
|
||||||
|
|
||||||
def write(self, path: str, text: str) -> None:
|
def write(self, path: str, text: str) -> None:
|
||||||
|
(self.repo / path).parent.mkdir(parents=True, exist_ok=True)
|
||||||
(self.repo / path).write_text(text, encoding="utf-8")
|
(self.repo / path).write_text(text, encoding="utf-8")
|
||||||
|
|
||||||
def scan(self, *extra: str, env: dict[str, str]) -> subprocess.CompletedProcess:
|
def scan(self, *extra: str, env: dict[str, str]) -> subprocess.CompletedProcess:
|
||||||
@@ -221,6 +225,59 @@ class TestSecurity(unittest.TestCase):
|
|||||||
result = self.scan(env=self.env)
|
result = self.scan(env=self.env)
|
||||||
self.assertEqual(result.stdout, "🛡️ security 100 % (medusa: 0 findings, 2 accepted in 1 file)\n")
|
self.assertEqual(result.stdout, "🛡️ security 100 % (medusa: 0 findings, 2 accepted in 1 file)\n")
|
||||||
|
|
||||||
|
def test_mjs_and_cjs_files_medusa_skips_are_scanned_and_reported_at_their_repository_path(self):
|
||||||
|
line = "run(request.query) // MARK-HIGH"
|
||||||
|
self.write("tool.mjs", f"import run from 'run';\n{line}\n")
|
||||||
|
result = self.scan(env=self.env)
|
||||||
|
self.assertEqual(
|
||||||
|
(result.stdout, result.returncode),
|
||||||
|
("🛡️ security 75 % (medusa: 1 finding in 1 file: high tool.mjs:2)\n", 3),
|
||||||
|
)
|
||||||
|
sha1 = hashlib.sha1(line.encode()).hexdigest()
|
||||||
|
result = self.accept(
|
||||||
|
"--rule", "FAKE-HIGH", "--path", "tool.mjs", "--sha1", sha1, "--line", "2", "--reason", "fixture"
|
||||||
|
)
|
||||||
|
self.assertEqual(result.returncode, 0, result.stderr)
|
||||||
|
self.write("tool.mjs.js", "a // MARK-MEDIUM\n")
|
||||||
|
self.write("lib.CJS", "b // MARK-LOW\n")
|
||||||
|
result = self.scan(env=self.env)
|
||||||
|
self.assertEqual(
|
||||||
|
(result.stdout, result.returncode),
|
||||||
|
("🛡️ security 88 % (medusa: 2 findings, 1 accepted in 3 files: medium tool.mjs.js:1, low lib.CJS:1)\n", 3),
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_a_folder_named_like_the_alias_keeps_both_paths_apart(self):
|
||||||
|
self.write("tool.mjs", "a // MARK-HIGH\n")
|
||||||
|
self.write("tool.mjs.js/inner.py", "b # MARK-LOW\n")
|
||||||
|
result = self.scan(env=self.env)
|
||||||
|
self.assertEqual(
|
||||||
|
(result.stdout, result.returncode),
|
||||||
|
("🛡️ security 73 % (medusa: 2 findings in 2 files: high tool.mjs:1, low tool.mjs.js/inner.py:1)\n", 3),
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_the_cache_keeps_the_repository_path_of_an_aliased_file(self):
|
||||||
|
text = "a // MARK-HIGH\n"
|
||||||
|
self.write("tool.mjs", text)
|
||||||
|
first = json.loads(self.scan("--json", env=self.env).stdout)
|
||||||
|
(self.repo / "tool.mjs").unlink()
|
||||||
|
self.write("tool.mjs.js", text)
|
||||||
|
second = json.loads(self.scan("--json", env=self.env).stdout)
|
||||||
|
self.assertEqual([f["path"] for f in first["findings"] + second["findings"]], ["tool.mjs", "tool.mjs.js"])
|
||||||
|
|
||||||
|
def test_a_nested_mjs_file_is_reported_at_its_nested_path(self):
|
||||||
|
self.write("a/b/tool.mjs", "a // MARK-MEDIUM\n")
|
||||||
|
result = self.scan(env=self.env)
|
||||||
|
self.assertEqual(
|
||||||
|
(result.stdout, result.returncode),
|
||||||
|
("🛡️ security 90 % (medusa: 1 finding in 1 file: medium a/b/tool.mjs:1)\n", 3),
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_an_mjs_file_name_without_room_for_the_alias_is_scanned_under_a_short_one(self):
|
||||||
|
name = "t" * 250 + ".mjs"
|
||||||
|
self.write(name, "a // MARK-LOW\n")
|
||||||
|
found = json.loads(self.scan("--json", env=self.env).stdout)
|
||||||
|
self.assertEqual((found["files"], [(f["path"], f["line"]) for f in found["findings"]]), (1, [(name, 1)]))
|
||||||
|
|
||||||
def test_critical_exits_1_and_floors_at_0(self):
|
def test_critical_exits_1_and_floors_at_0(self):
|
||||||
self.write("app.py", "a # MARK-CRITICAL\nb # MARK-CRITICAL\nc # MARK-CRITICAL\n")
|
self.write("app.py", "a # MARK-CRITICAL\nb # MARK-CRITICAL\nc # MARK-CRITICAL\n")
|
||||||
result = self.scan(env=self.env)
|
result = self.scan(env=self.env)
|
||||||
|
|||||||
Reference in new issue
Block a user