diff --git a/skills/kvb-check-security/scripts/security.py b/skills/kvb-check-security/scripts/security.py index 113e189..894b4ef 100755 --- a/skills/kvb-check-security/scripts/security.py +++ b/skills/kvb-check-security/scripts/security.py @@ -23,7 +23,7 @@ Scan: The regular files `git diff ` names plus untracked ones, symlinks left out, copied to a temporary tree and scanned once with `medusa scan --format json -o --no-cache --workers `, w the - files scanned but at most WORKERS, HOME pointing into the temporary + files scanned but at most host_workers(), HOME pointing into the temporary directory because medusa creates ~/.medusa on start and KNOB set to --timeout (a medusa that ignores KNOB keeps its own cap), read from `/medusa-scan-*.json`: a `findings` list whose entries carry @@ -137,7 +137,8 @@ WEIGHTS = {"critical": 50, "high": 25, "medium": 10, "low": 2, "info": 0} SAFE = 100 SHOWN = 5 REACH = 2 -WORKERS = 4 +CORE_SHARE = 0.95 +CPUINFO = Path("/proc/cpuinfo") KNOB = "MEDUSA_SCANNER_TIMEOUT" TIMEOUT = 900 DEADLINE = 3600 @@ -213,6 +214,37 @@ class Rules(NamedTuple): files: tuple +def physical_cores(cpuinfo: str) -> int: + """Returns: + The distinct physical id and core id pairs cpuinfo lists, else the + logical CPU count. + """ + pairs, socket = set(), "" + for line in cpuinfo.splitlines(): + key, _, value = line.partition(":") + if key.strip() == "physical id": + socket = value.strip() + elif key.strip() == "core id": + pairs.add((socket, value.strip())) + return len(pairs) or os.cpu_count() or 1 + + +def scan_workers(cores: int) -> int: + """Returns: + CORE_SHARE of the cores rounded down, at least one core left free and + at least one worker. + """ + return max(1, min(int(cores * CORE_SHARE), cores - 1)) + + +def host_workers() -> int: + try: + cpuinfo = CPUINFO.read_text(encoding="utf-8", errors="replace") + except OSError: + cpuinfo = "" + return scan_workers(physical_cores(cpuinfo)) + + def fail(message: str) -> None: print(f"security: {message}", file=sys.stderr) sys.exit(2) @@ -758,7 +790,7 @@ def scan( return scanned, hit, missed code, out, lasted = medusa_run( [medusa, "scan", str(tree), "--format", "json", "-o", str(reports), "--no-cache", - "--workers", str(min(scanned, WORKERS))], + "--workers", str(min(scanned, host_workers()))], work, env, deadline, ) try: diff --git a/tests/test_security.py b/tests/test_security.py index 6f0c448..7e75363 100644 --- a/tests/test_security.py +++ b/tests/test_security.py @@ -111,6 +111,28 @@ def ended(pids: list[int]) -> None: pass +def security_module(): + spec = importlib.util.spec_from_file_location("kvb_check_security_workers", SECURITY) + security = importlib.util.module_from_spec(spec) + spec.loader.exec_module(security) + return security + + +class TestScanWorkers(unittest.TestCase): + def test_medusa_takes_95_percent_of_the_physical_cores_and_leaves_one_free(self): + security = security_module() + for cores, workers in ((1, 1), (2, 1), (4, 3), (14, 13), (20, 19), (40, 38), (100, 95)): + with self.subTest(cores=cores): + self.assertEqual(security.scan_workers(cores), workers) + + def test_physical_cores_count_core_ids_per_socket_and_skip_hyperthreads(self): + security = security_module() + block = "processor\t: {n}\nphysical id\t: {socket}\ncore id\t\t: {core}\n\n" + cpuinfo = "".join(block.format(n=n, socket=n // 4, core=n % 2) for n in range(8)) + self.assertEqual(security.physical_cores(cpuinfo), 4) + self.assertEqual(security.physical_cores(""), os.cpu_count()) + + class TestSecurity(unittest.TestCase): def setUp(self): tmp = tempfile.TemporaryDirectory() @@ -199,7 +221,8 @@ class TestSecurity(unittest.TestCase): self.assertEqual(result.returncode, 3, result.stderr) argv = json.loads((self.bin / "argv.json").read_text(encoding="utf-8")) self.assertEqual( - [argv[0], *argv[2:4], argv[4], *argv[6:]], ["scan", "--format", "json", "-o", "--no-cache", "--workers", "2"] + [argv[0], *argv[2:4], argv[4], *argv[6:]], + ["scan", "--format", "json", "-o", "--no-cache", "--workers", str(min(2, security_module().host_workers()))], ) result = self.scan("--json", env=self.env) self.assertNotIn(PLACEHOLDER, result.stdout + result.stderr)