From 1e66e7a2c983aea97c5d65560b8360278c9907f2 Mon Sep 17 00:00:00 2001 From: root Date: Wed, 23 Sep 2026 16:31:27 +0300 Subject: [PATCH 01/22] Secret Diagnostics module and small changes --- .gitignore | 5 + .../core/helpers/variable_evaluator.py | 2 +- deepsecrets/diagnostics/__init__.py | 5 + deepsecrets/diagnostics/__main__.py | 62 +++ deepsecrets/diagnostics/trace.py | 405 ++++++++++++++++++ deepsecrets/diagnostics/worker.py | 39 ++ pyproject.toml | 4 + tests/diagnostics/test_trace.py | 140 ++++++ 8 files changed, 661 insertions(+), 1 deletion(-) create mode 100644 deepsecrets/diagnostics/__init__.py create mode 100644 deepsecrets/diagnostics/__main__.py create mode 100644 deepsecrets/diagnostics/trace.py create mode 100644 deepsecrets/diagnostics/worker.py create mode 100644 tests/diagnostics/test_trace.py diff --git a/.gitignore b/.gitignore index 088ce28..c8d3301 100644 --- a/.gitignore +++ b/.gitignore @@ -60,3 +60,8 @@ pytest* /benchmarker/ /tests/fixtures/problem_files/ /*.json + +/docs/private/ +AGENTS.md +CLAUDE.md +research/ \ No newline at end of file diff --git a/deepsecrets/core/helpers/variable_evaluator.py b/deepsecrets/core/helpers/variable_evaluator.py index c982e99..95d79b2 100644 --- a/deepsecrets/core/helpers/variable_evaluator.py +++ b/deepsecrets/core/helpers/variable_evaluator.py @@ -106,7 +106,7 @@ def confidence_from_evaluation_result(self, result: EvaluationResult): # Monotonic in every input: more naming evidence, more entropy or a less natural value never lowers it. # Naming: 0.2 per point up to 20, then 0.6 per point up to 25, so a strong name alone reaches 7 (HIGH) # and needs a random-looking value to reach VERY-HIGH. Value: entropy score 0..40 -> 0..5, halved for - # natural-looking values. See docs/research/variable-scoring-balance.md. + # natural-looking values. See docs/private/research/variable-scoring-balance.md. naming = min(max(result.naming_and_content_score, 0), 25) var_part = 0.2 * min(naming, 20) + 0.6 * max(naming - 20, 0) entropy_part = min(max(result.entropy_score, 0), 40) / 40 * 5 * min(result.nonsence_value_score + 0.5, 1) diff --git a/deepsecrets/diagnostics/__init__.py b/deepsecrets/diagnostics/__init__.py new file mode 100644 index 0000000..b4631ac --- /dev/null +++ b/deepsecrets/diagnostics/__init__.py @@ -0,0 +1,5 @@ +"""Diagnostics: explain why a value was or was not reported, in terms of the scanner's own stages. + +Nothing in the scanning path imports this package. `trace.trace_file` replays one file through the same components +a scan uses and records every decision, so it changes nothing about what a scan reports. +""" diff --git a/deepsecrets/diagnostics/__main__.py b/deepsecrets/diagnostics/__main__.py new file mode 100644 index 0000000..6d7508a --- /dev/null +++ b/deepsecrets/diagnostics/__main__.py @@ -0,0 +1,62 @@ +"""python -m deepsecrets.diagnostics trace --cases cases.jsonl --out traces.jsonl [--processes N] [--max-file-mb M] + +Each input line: {"case_id", "path", "relative_path", "line", "end_line", "start_col", "end_col", "value"}. +Each output line is one trace (see trace.py). Files are traced once however many cases they hold. +""" + +import argparse +import json +import sys +from collections import defaultdict +from multiprocessing import get_context + +from deepsecrets.diagnostics.worker import init, trace_job + + +def main(argv=None) -> int: + parser = argparse.ArgumentParser(prog='python -m deepsecrets.diagnostics') + sub = parser.add_subparsers(dest='command', required=True) + t = sub.add_parser('trace', help='trace labelled spans through the scan pipeline') + t.add_argument('--cases', required=True) + t.add_argument('--out', required=True) + t.add_argument('--processes', type=int, default=1) + t.add_argument('--max-file-mb', type=float, default=20.0) + t.add_argument( + '--no-path-exclusions', + action='store_true', + help='replay files the built-in excluded_paths rules would skip, as `--excluded-paths disable` does', + ) + args = parser.parse_args(argv) + + by_file = defaultdict(list) + with open(args.cases, encoding='utf-8') as f: + for line in f: + if line.strip(): + case = json.loads(line) + by_file[(case['path'], case['relative_path'])].append(case) + max_bytes = int(args.max_file_mb * 1_000_000) if args.max_file_mb else 0 + jobs = [(key, cases, max_bytes) for key, cases in by_file.items()] + + written = 0 + with open(args.out, 'w', encoding='utf-8') as out: + path_exclusions = not args.no_path_exclusions + if args.processes > 1: + with get_context('spawn').Pool( + args.processes, initializer=init, initargs=(path_exclusions,), maxtasksperchild=20 + ) as pool: + for traces in pool.imap_unordered(trace_job, jobs): + for trace in traces: + out.write(json.dumps(trace, default=str) + '\n') + written += 1 + else: + init(path_exclusions) + for job in jobs: + for trace in trace_job(job): + out.write(json.dumps(trace, default=str) + '\n') + written += 1 + print(f'traced {written} cases in {len(jobs)} files', file=sys.stderr) + return 0 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/deepsecrets/diagnostics/trace.py b/deepsecrets/diagnostics/trace.py new file mode 100644 index 0000000..db9469b --- /dev/null +++ b/deepsecrets/diagnostics/trace.py @@ -0,0 +1,405 @@ +"""Instrumented replay of one file: selection, lexer, tokens, variable detection and suppression, the post-filter, +the semantic engine's gates, the variable evaluator rule by rule, the per-file value cache, regex hits and merging. + +Input is a list of spans (a line and column range, plus the expected value) inside one file; output is one trace per +span. The verdict names the first stage where an expected secret was lost, or, for a reported value, which engine and +rule produced it and what drove its score. The schema is versioned: consumers (the benchmark harness) depend on it. +""" + +from dataclasses import dataclass, field +from typing import Dict, List, Optional + +from deepsecrets.core.engines.regex import RegexEngine +from deepsecrets.core.engines.semantic import SemanticEngine, filenames_ignorelist +from deepsecrets.core.helpers.variable_evaluator import HOPELESS_THRESHOLD, VariableEvaluator +from deepsecrets.core.model.file import File +from deepsecrets.core.model.token import SemanticType, Token +from deepsecrets.core.rulesets.excluded_paths import ExcludedPathsBuilder +from deepsecrets.core.rulesets.regex import RegexRulesetBuilder +from deepsecrets.core.rulesets.variable_scoring import VariableScoringRulesetBuilder +from deepsecrets.core.tokenizers.cheap_var_search import CheapVarSearchTokenizer +from deepsecrets.core.tokenizers.full_content import FullContentTokenizer +from deepsecrets.core.tokenizers.helpers.semantic.deep_analyzer import DeepAnalyzer +from deepsecrets.core.tokenizers.helpers.semantic.var_detection.rules import ( + VariableDetectionRules, + VariableSuppressionRules, +) +from deepsecrets.core.tokenizers.lexer import LexerTokenizer +from deepsecrets.core.utils.file_analyzer import EngineWithTokenizer, FileAnalyzer +from deepsecrets.core.utils.finding_merger import FindingMerger +from deepsecrets.core.utils.fs import get_path_inside_package + +SCHEMA_VERSION = 1 +WINDOW_BEFORE, WINDOW_AFTER = 6, 3 + + +@dataclass +class Rulesets: + regex: list + scoring: list + excluded: list + + @classmethod + def builtin(cls, path_exclusions: bool = True) -> 'Rulesets': + """The shipped rulesets. `path_exclusions=False` mirrors a scan run with `--excluded-paths disable`, so a + file under node_modules/ is replayed instead of stopping at the selection stage.""" + regex = RegexRulesetBuilder() + regex.with_rules_from_file(get_path_inside_package('rules/regexes.json')) + scoring = VariableScoringRulesetBuilder() + scoring.with_rules_from_file(get_path_inside_package('rules/variable_scoring_rules.json')) + excluded = ExcludedPathsBuilder() + if path_exclusions: + excluded.with_rules_from_file(get_path_inside_package('rules/excluded_paths.json')) + return cls(regex=regex.rules, scoring=scoring.rules, excluded=excluded.rules) + + +@dataclass +class Span: + case_id: str + line: int + end_line: int + start_col: int + end_col: int + value: str + start: Optional[int] = None + end: Optional[int] = None + located: bool = False + + +class TracingFileAnalyzer(FileAnalyzer): + """FileAnalyzer with the per-token decisions of `_run_engine` recorded: skipped by the value cache, findings.""" + + def __init__(self, file: File) -> None: + super().__init__(file) + self.decisions: List[dict] = [] + + def _run_engine(self, et: EngineWithTokenizer): + results = [] + processed_values: Dict[int, bool] = {} + if et.tokenizer not in self.tokens: + self.tokens[et.tokenizer] = et.tokenizer.tokenize(self.file) + name = et.tokenizer.__class__.__name__ + for token in self.tokens[et.tokenizer]: + known = processed_values.get(token.val_hash()) + if known is not None and known is False: + self.decisions.append( + {'tokenizer': name, 'engine': et.engine.name, 'span': token.span, 'cache_skip': True} + ) + continue + processed_values[token.val_hash()] = False + findings = et.engine.search(token) + for finding in findings: + finding.map_on_file(file=self.file, relative_start=token.span[0]) + results.append(finding) + processed_values[token.val_hash()] = True + self.decisions.append( + { + 'tokenizer': name, + 'engine': et.engine.name, + 'span': token.span, + 'cache_skip': False, + 'findings': len(findings), + } + ) + return results + + +def _overlaps(span, start: int, end: int) -> bool: + return span is not None and span[0] < end and start < span[1] + + +def _rule_ref(rule, rules) -> str: + index = next((i for i, r in enumerate(rules) if r is rule), -1) + language = getattr(rule, 'language', None) + return f'{getattr(language, "name", language)}#{index}' + + +def _type_name(token: Token) -> str: + return '.'.join(str(token.type[0]).split('.')[1:]) if token.type else '?' + + +def _locate(file: File, span: Span) -> None: + offsets = file.line_offsets + if span.line not in offsets: + return + line_start = offsets[span.line][0] + candidate = line_start + max(span.start_col, 1) - 1 + if file.content[candidate : candidate + len(span.value)] == span.value: + span.start, span.end, span.located = candidate, candidate + len(span.value), True + return + last = offsets.get(max(span.end_line, span.line), offsets[span.line])[1] + found = file.content.find(span.value, line_start, last + 1) if span.value else -1 + if found != -1: + span.start, span.end, span.located = found, found + len(span.value), True + return + # the value is not at the labelled place: keep the column range so tokens there can still be described + span.start = candidate + span.end = line_start + max(span.end_col, span.start_col + 1) - 1 + + +def _evaluate(evaluator: VariableEvaluator, scoring_rules: list, variable) -> dict: + context = variable.context + running, fired, hopeless = 0, [], False + for rule in scoring_rules: + if rule.match_by_context(context): + running += rule.score + fired.append({'id': rule.id, 'score': rule.score, 'running': running}) + if running <= HOPELESS_THRESHOLD: + hopeless = True + break + result = evaluator.evaluate(variable) + counterfactual = {} + for entry in fired: + without = VariableEvaluator([r for r in scoring_rules if r.id != entry['id']]) + counterfactual[entry['id']] = without.evaluate(variable).is_dangerous + return { + 'name': context.name, + 'rules': fired, + 'hopeless_exit': hopeless, + 'naming_score': result.naming_and_content_score if not hopeless else running, + 'entropy': round(result.entropy, 3), + 'entropy_score': round(result.entropy_score, 2), + 'naturalness': round(1 - result.nonsence_value_score, 3) if not hopeless else None, + 'dangerous': result.is_dangerous, + 'confidence': result.export_confidence, + 'rule_emitted': ('S105' if result.entropy_score > 0 else 'S106') if result.is_dangerous else None, + 'dangerous_without': counterfactual, + } + + +def _gate(token: Token, file: File) -> Optional[str]: + """SemanticEngine.search's early returns, in its order.""" + if token.length == file.length: + return 'whole_file_token' + if any(name in (file.path or '') for name in filenames_ignorelist): + return 'ignored_file_name' + if len(token.content) == 1: + return 'single_character' + if len(token.content.split(' ')) > 1: + return 'contains_space' + return None + + +@dataclass +class FileReplay: + file: File + relative_path: str + rulesets: Rulesets + lexer_name: Optional[str] = None + language: Optional[str] = None + raw_tokens: List[Token] = field(default_factory=list) + stream: str = '' + detections: List[dict] = field(default_factory=list) + prefilter_vars: List[Token] = field(default_factory=list) + production_vars: List[Token] = field(default_factory=list) + cheap_vars: List[Token] = field(default_factory=list) + findings: list = field(default_factory=list) + decisions: List[dict] = field(default_factory=list) + excluded_by: Optional[str] = None + + def run(self) -> 'FileReplay': + for rule in self.rulesets.excluded: + if rule.match(self.relative_path): + self.excluded_by = rule.pattern.pattern + break + + # tokens and regions before any variable analysis, with the type stream the detection rules read + raw = LexerTokenizer(deep_token_inspection=False) + raw.tokenize(self.file, post_filter=False) + self.lexer_name = raw.lexer.name if raw.lexer else None + self.language = getattr(getattr(raw, 'language', None), 'name', None) + self.raw_tokens, self.stream = list(raw.tokens), raw.token_stream + + detection_rules = VariableDetectionRules.rules + for region in getattr(raw, 'regions', None) or []: + if region.language is None: + continue + spans = [] + for rule in VariableSuppressionRules.for_language(region.language): + spans.extend(rule.match(region.tokens, region.stream)) + # the same collapsing DeepAnalyzer applies before it checks containment + spans = DeepAnalyzer([], post_filter=False)._collapse_suppression_regions(spans) + for rule in VariableDetectionRules.for_language(region.language): + for var in rule.match(region.tokens, region.stream): + suppressor = next((s for s in spans if var.span[0] >= s[0] and var.span[1] <= s[1]), None) + name_line = self.file.get_line_number(var.name_token.span[0]) if var.name_token else None + value_line = self.file.get_line_number(var.value_token.span[0]) if var.value_token else None + self.detections.append( + { + 'rule': _rule_ref(rule, detection_rules), + 'region_language': getattr(region.language, 'name', None), + 'name': var.name_token.content if var.name_token else getattr(var, 'name_override', None), + 'value_span': var.value_token.span if var.value_token else None, + 'name_value_lines_apart': (value_line - name_line) if name_line and value_line else None, + 'suppressed_by': list(suppressor) if suppressor else None, + 'creds_probability': rule.creds_probability, + } + ) + + pre = LexerTokenizer(deep_token_inspection=True) + pre.tokenize(self.file, post_filter=False) + self.prefilter_vars = pre.get_variables() + prod = LexerTokenizer(deep_token_inspection=True) + prod.tokenize(self.file) + self.production_vars = prod.get_variables() + cheap = CheapVarSearchTokenizer() + cheap.tokenize(self.file) + self.cheap_vars = cheap.get_variables() + + # the production pipeline for this file, as scan_modes/cli.py builds it, with the value cache recorded + regex_engine = RegexEngine(ruleset=self.rulesets.regex) + semantic_engine = SemanticEngine(regex_engine, ruleset=self.rulesets.scoring) + analyzer = TracingFileAnalyzer(self.file) + analyzer.add_engine(regex_engine, [FullContentTokenizer()]) + analyzer.add_engine(semantic_engine, [LexerTokenizer(deep_token_inspection=True), CheapVarSearchTokenizer()]) + self.findings = FindingMerger(analyzer.process()).merge() + self.decisions = analyzer.decisions + return self + + def window(self, index: int) -> Optional[str]: + if len(self.stream) != len(self.raw_tokens): + return None + before = self.stream[max(0, index - WINDOW_BEFORE) : index] + after = self.stream[index + 1 : index + 1 + WINDOW_AFTER] + return (before + '[' + self.stream[index] + ']' + after).replace('\n', '⏎') + + +def _trace_span(replay: FileReplay, span: Span, evaluator: VariableEvaluator) -> dict: + file = replay.file + # a label whose line is not in the file overlaps nothing + s, e = (span.start, span.end) if span.start is not None else (-1, -1) + trace = { + 'schema': SCHEMA_VERSION, + 'case_id': span.case_id, + 'located': span.located, + 'selection': {'excluded_by': replay.excluded_by}, + 'lexer': {'name': replay.lexer_name, 'language': replay.language, 'extension': file.extension}, + } + + covering = [i for i, t in enumerate(replay.raw_tokens) if _overlaps(t.span, s, e)] + if covering: + first = replay.raw_tokens[covering[0]] + contain = ( + 'split' if len(covering) > 1 else ('exact' if first.span[0] == s and first.span[1] == e else 'embedded') + ) + trace['token'] = { + 'type': _type_name(first), + 'contain': contain, + 'in_comment': any('Comment' in str(t) for t in first.type), + 'window': replay.window(covering[0]), + } + else: + trace['token'] = None + + detections = [d for d in replay.detections if _overlaps(d['value_span'], s, e)] + trace['detections'] = detections + in_pre = [t for t in replay.prefilter_vars if _overlaps(t.span, s, e)] + in_prod = [t for t in replay.production_vars if _overlaps(t.span, s, e)] + in_cheap = [t for t in replay.cheap_vars if _overlaps(t.span, s, e)] + trace['variables'] = {'lex_prefilter': bool(in_pre), 'lex': bool(in_prod), 'cheap': bool(in_cheap)} + + evaluations = [] + for source, tokens in (('lex', in_prod), ('cheap', in_cheap)): + for token in tokens: + if token.semantic is None or token.semantic.type != SemanticType.VARIABLE: + continue + entry = {'source': source, 'value_exact': token.span[0] == s and token.span[1] == e} + entry['gate'] = _gate(token, file) + if token.semantic.creds_probability == 9: + entry['s107'] = True + if entry['gate'] is None: + entry.update(_evaluate(evaluator, replay.rulesets.scoring, token.semantic.payload)) + evaluations.append(entry) + trace['evaluations'] = evaluations + + cache_skips = [ + d['tokenizer'] + for d in replay.decisions + if d['cache_skip'] and d['engine'] == 'semantic' and _overlaps(d['span'], s, e) + ] + reported = [f for f in replay.findings if f.start_offset < e and s < f.end_offset] + trace['production'] = { + 'findings': [ + { + 'rule': f.final_rule.id if f.final_rule else f.rules[0].id, + 'confidence': max(r.confidence for r in f.rules), + 'exact': f.start_offset == s and f.end_offset == e, + } + for f in (_with_final_rule(f) for f in reported) + ], + 'cache_skipped': sorted(set(cache_skips)), + } + trace['verdict'] = _verdict(trace) + return trace + + +def _with_final_rule(finding): + finding.choose_final_rule() + return finding + + +def _verdict(t: dict) -> dict: + """The first stage that explains a missing value; for a reported value, what reported it.""" + if t['production']['findings']: + rules = sorted({f['rule'] for f in t['production']['findings']}) + semantic = any(r.split('-')[0] in ('S105', 'S106', 'S107') for r in rules) + return { + 'stage': 'reported', + 'component': 'semantic_engine' if semantic else 'regex_engine', + 'detail': ','.join(rules), + } + if t['selection']['excluded_by']: + return {'stage': 'selection', 'component': 'excluded_paths', 'detail': t['selection']['excluded_by']} + if not t['located']: + return {'stage': 'label', 'component': 'dataset', 'detail': 'value not found where the label points'} + has_cheap = t['variables']['cheap'] + if t['lexer']['name'] is None and not has_cheap: + return {'stage': 'lexer', 'component': 'lexer_finder', 'detail': 'no lexer'} + if t['token'] is None and not has_cheap: + return {'stage': 'token', 'component': 'lexer_tokenizer', 'detail': 'no token covers the value'} + if not t['detections'] and not has_cheap: + detail = 'in a comment' if (t['token'] or {}).get('in_comment') else 'no detection rule matched' + return {'stage': 'detection', 'component': 'variable_detection', 'detail': detail} + if t['detections'] and all(d['suppressed_by'] for d in t['detections']) and not has_cheap: + return {'stage': 'detection', 'component': 'variable_suppression', 'detail': 'suppressed'} + if t['variables']['lex_prefilter'] and not t['variables']['lex'] and not has_cheap: + return {'stage': 'post_filter', 'component': 'deep_analyzer', 'detail': 'removed by final_cleanup'} + evaluations = t['evaluations'] + if not evaluations: + return { + 'stage': 'detection', + 'component': 'variable_detection', + 'detail': 'detected but no variable reached the engine', + } + if all(ev.get('gate') for ev in evaluations): + return {'stage': 'gate', 'component': 'semantic_engine', 'detail': evaluations[0]['gate']} + if any(ev.get('dangerous') for ev in evaluations): + if t['production']['cache_skipped']: + return { + 'stage': 'emission', + 'component': 'file_analyzer', + 'detail': 'value cache: seen earlier with no finding', + } + return {'stage': 'emission', 'component': 'finding_merger', 'detail': 'dangerous but not reported'} + return {'stage': 'evaluation', 'component': 'variable_evaluator', 'detail': 'scored as not dangerous'} + + +def trace_file(path: str, relative_path: str, spans: List[dict], rulesets: Optional[Rulesets] = None) -> List[dict]: + """One trace per span. `spans` items: case_id, line, end_line, start_col, end_col, value.""" + rulesets = rulesets or Rulesets.builtin() + file = File(path=path, relative_path=relative_path) + replay = FileReplay(file=file, relative_path=relative_path, rulesets=rulesets).run() + evaluator = VariableEvaluator(rulesets.scoring) + traces = [] + for raw in spans: + span = Span( + case_id=str(raw['case_id']), + line=int(raw['line']), + end_line=int(raw.get('end_line') or raw['line']), + start_col=int(raw.get('start_col') or 1), + end_col=int(raw.get('end_col') or 1), + value=raw.get('value') or '', + ) + _locate(file, span) + traces.append(_trace_span(replay, span, evaluator)) + return traces diff --git a/deepsecrets/diagnostics/worker.py b/deepsecrets/diagnostics/worker.py new file mode 100644 index 0000000..7ba9d4c --- /dev/null +++ b/deepsecrets/diagnostics/worker.py @@ -0,0 +1,39 @@ +"""Pool workers for `python -m deepsecrets.diagnostics`. + +They live in an ordinary module because `spawn` children never re-import a package's `__main__.py`, so functions +defined there cannot be found by pool workers. +""" + +import os +import traceback + +RULESETS = None + + +def init(path_exclusions: bool = True): + global RULESETS + from deepsecrets.diagnostics.trace import Rulesets + + RULESETS = Rulesets.builtin(path_exclusions) + + +def trace_job(job): + (path, relative_path), cases, max_bytes = job + from deepsecrets.diagnostics.trace import SCHEMA_VERSION, trace_file + + def failed(stage, detail): + return [ + { + 'schema': SCHEMA_VERSION, + 'case_id': c['case_id'], + 'verdict': {'stage': stage, 'component': 'tracer', 'detail': detail}, + } + for c in cases + ] + + try: + if max_bytes and os.path.getsize(path) > max_bytes: + return failed('skipped_large', f'{os.path.getsize(path)} bytes') + return trace_file(path, relative_path, cases, RULESETS) + except Exception as e: # one broken file must not lose the others + return failed('trace_error', f'{type(e).__name__}: {e}\n{traceback.format_exc(limit=3)}') diff --git a/pyproject.toml b/pyproject.toml index 7b1ad2e..6605eb8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,6 +49,10 @@ repository = "https://github.com/ntoskernel/deepsecrets" [project.scripts] deepsecrets = "deepsecrets:__main__.runnable_entrypoint" +[tool.poetry] +# The diagnostics tracer (python -m deepsecrets.diagnostics) is developer tooling: in the repository, not in builds. +exclude = ["deepsecrets/diagnostics"] + [tool.poetry.group.test.dependencies] pytest = "9.1.1" diff --git a/tests/diagnostics/test_trace.py b/tests/diagnostics/test_trace.py new file mode 100644 index 0000000..071ac1f --- /dev/null +++ b/tests/diagnostics/test_trace.py @@ -0,0 +1,140 @@ +"""The diagnostic replay names the stage where a value was lost, or what reported it. Values are synthetic.""" + +import json + +import pytest + +from deepsecrets.diagnostics.__main__ import main +from deepsecrets.diagnostics.trace import Rulesets, trace_file + +TOKEN = 'Hk3vQ9ZpL2mXw7RtYb4NcF8aZ' +SOURCE = 'password = "T7r$kq92LmZx"\n' f'# key: {TOKEN}\n' f'client = Client("{TOKEN}")\n' f'session_id = "{TOKEN}"\n' + + +def case(case_id, line, value): + text = SOURCE.split('\n')[line - 1] + col = text.index(value) + 1 + return dict(case_id=case_id, line=line, end_line=line, start_col=col, end_col=col + len(value), value=value) + + +@pytest.fixture() +def traces(tmp_path): + path = tmp_path / 'settings.py' + path.write_text(SOURCE) + cases = [case('pw', 1, 'T7r$kq92LmZx'), case('comment', 2, TOKEN), case('ctor', 3, TOKEN), case('sess', 4, TOKEN)] + return {t['case_id']: t for t in trace_file(str(path), 'src/settings.py', cases)} + + +def test_reported_value_names_its_engine_and_rule(traces): + t = traces['pw'] + assert t['verdict'] == {'stage': 'reported', 'component': 'semantic_engine', 'detail': 'S105'} + evaluation = t['evaluations'][0] + assert evaluation['dangerous'] is True + # counterfactual: the finding stands on one naming rule + assert evaluation['dangerous_without'] == {'SEM_VAR_HIGH_CONFIDENCE_FULLNAME_2': False} + + +def test_secret_in_a_comment_is_lost_at_detection(traces): + t = traces['comment'] + assert t['verdict']['stage'] == 'detection' and t['verdict']['detail'] == 'in a comment' + assert t['token']['in_comment'] is True + + +def test_constructor_argument_is_not_a_variable(traces): + t = traces['ctor'] + assert t['verdict'] == { + 'stage': 'detection', + 'component': 'variable_detection', + 'detail': 'no detection rule matched', + } + assert t['token']['window'].endswith('p[L]p⏎n') + + +def test_red_flag_name_is_rejected_by_the_evaluator(traces): + t = traces['sess'] + assert t['verdict']['stage'] == 'evaluation' + assert {'SEM_VAR_NAME_SLICE_REDFLAGS', 'SEM_VAR_NONSECRET_LAST_SLICE'} <= { + r['id'] for r in t['evaluations'][0]['rules'] + } + + +def test_value_not_where_the_label_points(tmp_path): + path = tmp_path / 'a.py' + path.write_text(SOURCE) + moved = dict(case('x', 1, 'T7r$kq92LmZx'), line=3, end_line=3) + [t] = trace_file(str(path), 'a.py', [moved]) + assert t['located'] is False + + +def test_excluded_path_stops_the_replay_unless_exclusions_are_off(tmp_path): + path = tmp_path / 'settings.py' + path.write_text(SOURCE) + # a value nothing reports: only then does the verdict fall through to the selection stage + cases = [case('ctor', 3, TOKEN)] + vendored = 'node_modules/pkg/settings.py' + [stopped] = trace_file(str(path), vendored, cases) + assert stopped['verdict'] == { + 'stage': 'selection', + 'component': 'excluded_paths', + 'detail': '.*node_modules\\/.*', + } + # a scan run with --excluded-paths disable reads the file, so the replay has to as well, and then the verdict + # names the stage that really lost the value + [replayed] = trace_file(str(path), vendored, cases, Rulesets.builtin(path_exclusions=False)) + assert replayed['verdict'] == { + 'stage': 'detection', + 'component': 'variable_detection', + 'detail': 'no detection rule matched', + } + + +def test_cli_replays_excluded_paths_on_request(tmp_path): + path = tmp_path / 'settings.py' + path.write_text(SOURCE) + cases = tmp_path / 'cases.jsonl' + cases.write_text( + json.dumps(dict(case('ctor', 3, TOKEN), path=str(path), relative_path='node_modules/p/settings.py')) + ) + out = tmp_path / 'traces.jsonl' + assert main(['trace', '--cases', str(cases), '--out', str(out)]) == 0 + assert json.loads(out.read_text())['verdict']['stage'] == 'selection' + assert main(['trace', '--cases', str(cases), '--out', str(out), '--no-path-exclusions']) == 0 + assert json.loads(out.read_text())['verdict']['stage'] == 'detection' + + +def test_cli_writes_one_trace_per_case(tmp_path): + path = tmp_path / 'settings.py' + path.write_text(SOURCE) + cases = tmp_path / 'cases.jsonl' + cases.write_text( + '\n'.join( + json.dumps(dict(case(i, line, v), path=str(path), relative_path='settings.py')) + for i, line, v in (('a', 1, 'T7r$kq92LmZx'), ('b', 4, TOKEN)) + ) + ) + out = tmp_path / 'traces.jsonl' + assert main(['trace', '--cases', str(cases), '--out', str(out)]) == 0 + lines = [json.loads(line) for line in out.read_text().splitlines()] + assert sorted(t['case_id'] for t in lines) == ['a', 'b'] + + +def test_cli_with_a_process_pool(tmp_path): + # spawn children never re-import a package's __main__.py: the pool workers must live in an importable module + paths = [] + for i in range(3): + path = tmp_path / f'settings_{i}.py' + path.write_text(SOURCE) + paths.append(path) + cases = tmp_path / 'cases.jsonl' + cases.write_text( + '\n'.join( + json.dumps(dict(case(f'c{i}', 1, 'T7r$kq92LmZx'), path=str(p), relative_path=p.name)) + for i, p in enumerate(paths) + ) + ) + out = tmp_path / 'traces.jsonl' + assert main(['trace', '--cases', str(cases), '--out', str(out), '--processes', '2']) == 0 + verdicts = { + json.loads(line)['case_id']: json.loads(line)['verdict']['stage'] for line in out.read_text().splitlines() + } + assert verdicts == {'c0': 'reported', 'c1': 'reported', 'c2': 'reported'} From 1ab35bf1ef5c28cb93887916d5420bb776d81a79 Mon Sep 17 00:00:00 2001 From: Nikolai Khechumov Date: Mon, 28 Sep 2026 09:29:04 +0000 Subject: [PATCH 02/22] Build lexer tokens on demand from a token table The lexer created a Token object for every piece of every file, although most are never read. It now records rows in a compact TokenTable and hands out LazyTokens that build a token only when variable detection or an engine asks for it. Type chains and stream items are computed once per Pygments type, and SingleTokenImprover decides from the current token and the two before it instead of rebuilding tokens. Output is identical to 2.1.1: test_lexer_equivalence compares every fixture with a frozen copy of the 2.1.1 tokenizer. --- .../helpers/semantic/deep_analyzer.py | 40 +++- .../helpers/single_token_improver.py | 138 ++++++------- .../core/tokenizers/helpers/token_table.py | 99 ++++++++++ .../core/tokenizers/helpers/type_stream.py | 50 ++++- deepsecrets/core/tokenizers/lexer.py | 60 +++--- .../core/tokenizers/test_lexer_equivalence.py | 184 ++++++++++++++++++ .../tokenizers/test_single_token_improver.py | 124 +++++++----- tests/core/tokenizers/test_token_table.py | 59 ++++++ 8 files changed, 594 insertions(+), 160 deletions(-) create mode 100644 deepsecrets/core/tokenizers/helpers/token_table.py create mode 100644 tests/core/tokenizers/test_lexer_equivalence.py create mode 100644 tests/core/tokenizers/test_token_table.py diff --git a/deepsecrets/core/tokenizers/helpers/semantic/deep_analyzer.py b/deepsecrets/core/tokenizers/helpers/semantic/deep_analyzer.py index 494b862..194480c 100644 --- a/deepsecrets/core/tokenizers/helpers/semantic/deep_analyzer.py +++ b/deepsecrets/core/tokenizers/helpers/semantic/deep_analyzer.py @@ -11,7 +11,8 @@ VariableDetectionRules, VariableSuppressionRules, ) -from deepsecrets.core.tokenizers.helpers.type_stream import types_to_filter_before, types_to_filter_after +from deepsecrets.core.tokenizers.helpers.token_table import LazyTokens +from deepsecrets.core.tokenizers.helpers.type_stream import is_filtered_ttype, is_filtered_type empty_tokens = ['\n', '\t', "'", "''", '"', '""'] @@ -48,9 +49,8 @@ def run(self): region.tokens = updated_tokens def analyze_token_sequence(self, language: Language, tokens: List[Token], stream: str) -> Set[Token]: - tokens_all = OrderedSet(tokens) if language is None: - return tokens_all + return OrderedSet(tokens) exclude_after = set() @@ -128,16 +128,46 @@ def _collapse_suppression_regions(self, suppression_regions): return regions def final_cleanup(self, tokens_all: Sequence[Token], tokens_to_be_excluded: Sequence[Token]) -> List[Token]: + if isinstance(tokens_all, LazyTokens): + return self._final_cleanup_rows(tokens_all, tokens_to_be_excluded) + if not isinstance(tokens_all, OrderedSet): tokens_all = OrderedSet(tokens_all) tokens_all = tokens_all - tokens_to_be_excluded final = [] for token in tokens_all: - if any(type in token.type for type in types_to_filter_before): # type: ignore + if is_filtered_type(token.type): + continue + + if token.content.replace(' ', '') in empty_tokens: + continue + + final.append(token) + + return final + + def _final_cleanup_rows(self, tokens: LazyTokens, tokens_to_be_excluded: Sequence[Token]) -> List[Token]: + """`final_cleanup` over a lazy view, building only the tokens it keeps. A row nothing has read yet was never + handed out, so it cannot be excluded, and its type and content decide it.""" + table = tokens.table + final = [] + for index in range(tokens.start, tokens.stop): + token = table.built[index] + if token is None: + if is_filtered_ttype(table.ttypes[index]): # type: ignore + continue + + if table.contents[index].replace(' ', '') in empty_tokens: + continue + + final.append(table.token(index)) + continue + + if token in tokens_to_be_excluded: continue - if any(type in token.type for type in types_to_filter_after): # type: ignore + if is_filtered_type(token.type): continue if token.content.replace(' ', '') in empty_tokens: diff --git a/deepsecrets/core/tokenizers/helpers/single_token_improver.py b/deepsecrets/core/tokenizers/helpers/single_token_improver.py index 0395f4c..fe0f572 100644 --- a/deepsecrets/core/tokenizers/helpers/single_token_improver.py +++ b/deepsecrets/core/tokenizers/helpers/single_token_improver.py @@ -1,22 +1,25 @@ import regex as re -from typing import Callable, List +from typing import Callable, List, Optional, Sequence, Tuple -from pygments.token import Token as PygmentsToken +from pygments.token import Token as PygmentsToken, _TokenType -from deepsecrets.core.model.token import Token from deepsecrets.core.tokenizers.helpers.semantic.language import Language -from deepsecrets.core.tokenizers.helpers.semantic.var_detection.detector import Match, RegionDetector -from deepsecrets.core.tokenizers.helpers.type_stream import token_to_typestream_item +from deepsecrets.core.tokenizers.helpers.type_stream import type_chain -# `curl -u `: the token after a literal `-u` -CURL_CREDENTIALS_DETECTOR = RegionDetector( - stream_pattern=re.compile('(L)(L)$'), - match_rules={1: Match(values=[re.compile('^-u$')])}, - match_semantics={}, -) +# A piece the current token is split into: its content and the Pygments type it gets. +Part = Tuple[str, _TokenType] + +CURL_CREDENTIALS_FLAG = re.compile('^-u$') class SingleTokenImprover: + """Splits one lexer token into several in known cases. + + An improvement decides from the current token (Pygments type, sanitised content, type-stream item) and the two + tokens before it as the tokenizer has emitted them (their stream items and contents), so the tokenizer never has + to build a `Token` to ask. It returns None to keep the token, or the parts that replace it, in order. + """ + language: Language acc: dict[Language, List[Callable]] @@ -27,78 +30,59 @@ def __init__(self, lang: Language) -> None: # Language.PHP: [self._php_variable_dollar_sign_breakdown], # TODO: Uncomment in v2.1 } - def improve(self, so_far_tokens: List[Token], so_far_type_stream: str, current_token: Token) -> List[Token]: - checkers: List[Callable] = self.acc.get(Language.ANY, []) - checkers.extend(self.acc.get(self.language, [])) + def applies(self) -> bool: + """Whether `improve` can change anything for this language; if not, it keeps every token.""" + return bool(self.acc.get(Language.ANY) or self.acc.get(self.language)) - tokens = [] - for improvement in checkers: - tokens.extend(improvement(so_far_tokens, so_far_type_stream, current_token)) + def improve( + self, ttype: _TokenType, content: str, item: str, previous_items: str, previous_contents: Sequence[str] + ) -> Optional[List[Part]]: + """`previous_items` and `previous_contents` hold at most the two tokens emitted before this one.""" + for improvement in self.acc.get(Language.ANY, []) + self.acc.get(self.language, []): + parts = improvement(ttype, content, item, previous_items, previous_contents) + if parts is not None: + return parts + return None - if len(tokens) == 0: - return [current_token] + def _php_variable_dollar_sign_breakdown( + self, ttype: _TokenType, content: str, item: str, previous_items: str, previous_contents: Sequence[str] + ) -> Optional[List[Part]]: + if PygmentsToken.Name.Variable not in type_chain(ttype): + return None - return tokens + if not content.startswith('$'): + return None - def _php_variable_dollar_sign_breakdown( - self, so_far_tokens: List[Token], so_far_type_stream: str, current_token: Token - ) -> List[Token]: - target_token_type = PygmentsToken.Name.Variable - if target_token_type not in current_token.type: - return [current_token] - - if not current_token.content.startswith('$'): - return [current_token] - - first_part = current_token.content[0] - second_part = current_token.content[1:] - - final = [] - fp_token = Token( - file=current_token.file, - content=first_part, - span=current_token.file.get_span_for_string(first_part, between=current_token.span), - ) - fp_token.set_type([PygmentsToken.Operator]) - final.append(fp_token) + return [(content[0], PygmentsToken.Operator), (content[1:], PygmentsToken.Name.Variable)] - sp_token = Token( - file=current_token.file, - content=second_part, - span=current_token.file.get_span_for_string(first_part, between=current_token.span), + def _curl_argstring_breakdown( + self, ttype: _TokenType, content: str, item: str, previous_items: str, previous_contents: Sequence[str] + ) -> Optional[List[Part]]: + # `curl -u `: a plain word (`L`) right after a literal `-u`, split on ':' into user and password. + after_flag = ( + item == 'L' + and previous_items[-1:] == 'L' + and CURL_CREDENTIALS_FLAG.match(previous_contents[-1]) is not None + ) + # Kept from the stream regex '(L)(L)$' this replaced: its `$` also matched before a final newline, so a + # newline two tokens after `-u` counts as the credentials too (and is dropped below: it has no ':'). + before_newline = ( + item == '\n' + and previous_items[-2:] == 'LL' + and CURL_CREDENTIALS_FLAG.match(previous_contents[-2]) is not None ) - sp_token.set_type([PygmentsToken.Name.Variable]) - final.append(sp_token) + if not (after_flag or before_newline): + return None - return [fp_token, sp_token] + new_parts = content.split(':') + if len(new_parts) == 1: + # Dropped, as before this was rewritten: indexing the missing second part raised, and the tokenizer skips + # a token that raises. The dropped token never becomes "the token before", so the next word is judged + # against `-u` again: one without ':' is dropped too, one with ':' is split as the credentials + # (`curl -u admin https://x` reports `https` = `//x`). KI-TOK-01. + return [] - def _curl_argstring_breakdown( - self, so_far_tokens: List[Token], so_far_type_stream: str, current_token: Token - ) -> List[Token]: - # '(L)(L)$' can only match the last two stream items, or the two before a trailing newline, - # so matching the tail (aligned with the last two tokens) is equivalent to matching the whole - # stream, and keeps this O(1) per token instead of O(tokens so far). - tail_length = min(2, len(so_far_tokens)) - tail_tokens = so_far_tokens[len(so_far_tokens) - tail_length :] - projected_tail = so_far_type_stream[len(so_far_type_stream) - tail_length :] - projected_tail += token_to_typestream_item(current_token) - - match = CURL_CREDENTIALS_DETECTOR.match(tail_tokens, projected_tail) - if not match: - return [current_token] - - new_parts = current_token.content.split(':') if new_parts[0] == '' or new_parts[1] == '': - return [current_token] - - final = [] - for part in new_parts: - t = Token( - file=current_token.file, - content=part, - span=current_token.file.get_span_for_string(part, between=current_token.span), - ) - t.set_type([PygmentsToken.Text]) - final.append(t) - - return final + return None + + return [(part, PygmentsToken.Text) for part in new_parts] diff --git a/deepsecrets/core/tokenizers/helpers/token_table.py b/deepsecrets/core/tokenizers/helpers/token_table.py new file mode 100644 index 0000000..e265c7d --- /dev/null +++ b/deepsecrets/core/tokenizers/helpers/token_table.py @@ -0,0 +1,99 @@ +"""The lexer's tokens as rows, built into `Token` objects only when something reads them. + +Most tokens a lexer produces are punctuation, operators, names and masked types that the type stream needs but +`DeepAnalyzer.final_cleanup` drops by type alone. `LexerTokenizer` therefore records each token as a row (Pygments +type, sanitised content, raw offsets) and hands out `LazyTokens` views over the table. Reading an index builds that +row's `Token`, with the span and type chain an eager build would give it, and caches it: a token keeps one identity +for the life of the table, which `DeepAnalyzer`'s identity-based exclusions rely on. +""" + +from collections.abc import Sequence +from typing import Iterator, List, Optional + +from pygments.token import _TokenType + +from deepsecrets.core.model.file import File +from deepsecrets.core.model.token import Token +from deepsecrets.core.tokenizers.helpers.type_stream import type_chain + + +class TokenTable: + file: File + ttypes: List[Optional[_TokenType]] + contents: List[str] + starts: List[int] + ends: List[int] + built: List[Optional[Token]] + + def __init__(self, file: File) -> None: + self.file = file + self.ttypes = [] + self.contents = [] + self.starts = [] + self.ends = [] + self.built = [] + + def __len__(self) -> int: + return len(self.contents) + + def add(self, ttype: _TokenType, content: str, start: int, end: int) -> None: + """A lexer token: `content` is sanitised, `start` and `end` are the raw token's offsets in the file.""" + self.ttypes.append(ttype) + self.contents.append(content) + self.starts.append(start) + self.ends.append(end) + self.built.append(None) + + def add_token(self, token: Token) -> None: + """A token that already exists, such as a part `SingleTokenImprover` split off.""" + self.ttypes.append(None) + self.contents.append(token.content) + self.starts.append(-1) + self.ends.append(-1) + self.built.append(token) + + def token(self, index: int) -> Token: + token = self.built[index] + if token is None: + content = self.contents[index] + # the span is searched for within one character of the raw token, as the tokenizer always did + span = self.file.get_span_for_string(content, between=[self.starts[index] - 1, self.ends[index] + 1]) + token = Token(file=self.file, content=content, span=span) + token.set_type(list(type_chain(self.ttypes[index]))) # type: ignore + self.built[index] = token + return token + + +class LazyTokens(Sequence): + """A read-only list of the table's rows `start` to `stop`: an index builds that row's token, a slice with step 1 + is a view (any other step gives a list).""" + + __slots__ = ('table', 'start', 'stop') + + def __init__(self, table: TokenTable, start: int = 0, stop: Optional[int] = None) -> None: + self.table = table + self.start = start + self.stop = len(table) if stop is None else stop + + def __len__(self) -> int: + return self.stop - self.start + + def __getitem__(self, index): + if isinstance(index, slice): + start, stop, step = index.indices(len(self)) + if step == 1: + return LazyTokens(self.table, self.start + start, self.start + max(start, stop)) + return [self.table.token(self.start + i) for i in range(start, stop, step)] + + if index < 0: + index += len(self) + if not 0 <= index < len(self): + raise IndexError('token index out of range') + return self.table.token(self.start + index) + + def __iter__(self) -> Iterator[Token]: + for index in range(self.start, self.stop): + yield self.table.token(index) + + def __repr__(self) -> str: # pragma: no cover + return f'LazyTokens(rows {self.start} to {self.stop} of {len(self.table)})' diff --git a/deepsecrets/core/tokenizers/helpers/type_stream.py b/deepsecrets/core/tokenizers/helpers/type_stream.py index 601be58..9b603a7 100644 --- a/deepsecrets/core/tokenizers/helpers/type_stream.py +++ b/deepsecrets/core/tokenizers/helpers/type_stream.py @@ -1,4 +1,6 @@ -from pygments.token import Token as PygmentsToken +from typing import Dict, Sequence, Tuple + +from pygments.token import Token as PygmentsToken, _TokenType from deepsecrets.core.model.token import Token @@ -66,3 +68,49 @@ def token_to_typestream_item(token: Token) -> str: return 'u' return acc.get(token.type[0], '?') # type: ignore + + +# The tables below depend only on a token's Pygments type (and, for the stream item, on whether its content is a +# newline), so they are filled once per type rather than once per token. +_chains: Dict[_TokenType, Tuple[_TokenType, ...]] = {} +_stream_items: Dict[_TokenType, str] = {} +_dropped: Dict[_TokenType, bool] = {} + + +def type_chain(ttype: _TokenType) -> Tuple[_TokenType, ...]: + """The type and its parents, most specific first, without the root `Token`: what `Token.type` holds.""" + chain = _chains.get(ttype) + if chain is None: + types = [ttype] + while types[-1].parent is not None and types[-1].parent != PygmentsToken: + types.append(types[-1].parent) + chain = _chains[ttype] = tuple(types) + return chain + + +def stream_item(ttype: _TokenType, content: str) -> str: + """`token_to_typestream_item` for a token of this type and content, without building the token.""" + if content == '\n': + return '\n' + + item = _stream_items.get(ttype) + if item is None: + chain = type_chain(ttype) + masked = any(type in chain for type in types_to_filter_before) and not any( + type in chain for type in types_not_to_filter_before + ) + item = _stream_items[ttype] = 'u' if masked else acc.get(ttype, '?') + return item + + +def is_filtered_type(types: Sequence[_TokenType]) -> bool: + """The type half of `DeepAnalyzer.final_cleanup`: masked types and the stream's structure are both dropped.""" + return any(type in types for type in types_to_filter_before) or any(type in types for type in types_to_filter_after) + + +def is_filtered_ttype(ttype: _TokenType) -> bool: + """`is_filtered_type` for a token that still has the type chain of its Pygments type.""" + dropped = _dropped.get(ttype) + if dropped is None: + dropped = _dropped[ttype] = is_filtered_type(type_chain(ttype)) + return dropped diff --git a/deepsecrets/core/tokenizers/lexer.py b/deepsecrets/core/tokenizers/lexer.py index ddee11f..eab97a8 100644 --- a/deepsecrets/core/tokenizers/lexer.py +++ b/deepsecrets/core/tokenizers/lexer.py @@ -1,4 +1,4 @@ -from typing import List, Set, Type, Union +from typing import List, Set, Union from deepsecrets.core.model.tokenized_region import TokenizedRegion @@ -12,9 +12,8 @@ from deepsecrets.core.model.token import Token from deepsecrets.core.tokenizers.helpers.semantic.language import Language from deepsecrets.core.tokenizers.helpers.single_token_improver import SingleTokenImprover -from deepsecrets.core.tokenizers.helpers.type_stream import ( - token_to_typestream_item, -) +from deepsecrets.core.tokenizers.helpers.token_table import LazyTokens, TokenTable +from deepsecrets.core.tokenizers.helpers.type_stream import stream_item, token_to_typestream_item, type_chain from deepsecrets.core.tokenizers.itokenizer import Tokenizer from deepsecrets.core.utils.lexer_finder import LexerFinder @@ -25,16 +24,6 @@ class LexerTokenizer(Tokenizer): language: Language = None regions: Set[TokenizedRegion] = None - def _get_types_for_token(self, token: PygmentsToken) -> List[Type]: # type: ignore - types = [] - types.append(token) - if token.parent is not None: - if token.parent == PygmentsToken: - return types - deep = self._get_types_for_token(token.parent) - types.extend(deep) - return types - def sanitize(self, content: str) -> Union[str, bool]: quotes = ["'", "''", '"', '""'] @@ -79,11 +68,15 @@ def tokenize(self, file: File, post_filter=True) -> List[Token]: raw_tokens = list(self.lexer.get_tokens_unprocessed(file.content)) single_token_improver = SingleTokenImprover(lang=self.language) + improving = single_token_improver.applies() + # Tokens are recorded as rows and built only when something reads them (see helpers/token_table.py). + table = TokenTable(file) + stream: List[str] = [] current_position = 0 # TODO: Token.Error creates millions of bullshit - for offset, types, content in raw_tokens: - types: List[Type] = self._get_types_for_token(types) + for offset, ttype, content in raw_tokens: + types = type_chain(ttype) start = current_position end = start + len(content) current_position = end @@ -98,19 +91,25 @@ def tokenize(self, file: File, post_filter=True) -> List[Token]: if not content: continue - span = file.get_span_for_string(content, between=[start - 1, end + 1]) - token = Token(file=file, content=content, span=span) - token.set_type(types) - - improved_tokens = single_token_improver.improve(self.tokens, self.token_stream, token) - - self.tokens.extend(improved_tokens) - self.add_to_token_stream(improved_tokens) + item = stream_item(ttype, content) + parts = None + if improving: + previous = ''.join(stream[-2:]) + parts = single_token_improver.improve(ttype, content, item, previous, table.contents[-2:]) + + if parts is None: + table.add(ttype, content, start, end) + stream.append(item) + else: + self._add_parts(table, stream, content, start, end, parts) except Exception as e: str(e) self.on_new_offset_processed(new_offset=current_position / file.length) + self.tokens = LazyTokens(table) + self.token_stream = ''.join(stream) + self.regions: Set[TokenizedRegion] = SubFileRegionsHelper( file=file, language=self.language, @@ -127,9 +126,16 @@ def tokenize(self, file: File, post_filter=True) -> List[Token]: self.silent_regions = deep_analyzer.silent_regions return self.tokens - def add_to_token_stream(self, tokens: List[Token]) -> None: - for token in tokens: - self.token_stream += token_to_typestream_item(token=token) + def _add_parts(self, table: TokenTable, stream: List[str], content: str, start: int, end: int, parts) -> None: + """The tokens an improvement split a token into, searched for inside the span the whole token would have had. + No parts drops the token.""" + file = table.file + span = file.get_span_for_string(content, between=[start - 1, end + 1]) + for part, part_type in parts: + token = Token(file=file, content=part, span=file.get_span_for_string(part, between=span)) + token.set_type([part_type]) + table.add_token(token) + stream.append(token_to_typestream_item(token=token)) def print_token_type_stream(self) -> None: print(self.token_stream) diff --git a/tests/core/tokenizers/test_lexer_equivalence.py b/tests/core/tokenizers/test_lexer_equivalence.py new file mode 100644 index 0000000..e06c847 --- /dev/null +++ b/tests/core/tokenizers/test_lexer_equivalence.py @@ -0,0 +1,184 @@ +"""`LexerTokenizer` builds `Token` objects lazily; its output must equal the eager loop it replaced, token by token.""" + +import glob +import os +from typing import List, Type + +import pytest +import regex as re +from pygments.token import Token as PygmentsToken + +from deepsecrets.core.model.file import File +from deepsecrets.core.model.token import Token +from deepsecrets.core.tokenizers.helpers import subfile_regions_helper +from deepsecrets.core.tokenizers.helpers.semantic.deep_analyzer import DeepAnalyzer +from deepsecrets.core.tokenizers.helpers.semantic.language import Language +from deepsecrets.core.tokenizers.helpers.semantic.var_detection.detector import Match, RegionDetector +from deepsecrets.core.tokenizers.helpers.subfile_regions_helper import SubFileRegionsHelper +from deepsecrets.core.tokenizers.helpers.token_table import TokenTable +from deepsecrets.core.tokenizers.helpers.type_stream import token_to_typestream_item +from deepsecrets.core.tokenizers.lexer import LexerTokenizer + +SHELL = "curl -u admin\ncurl -u 'login01:password01' -s https://x\nsort -u f.txt\ncurl -u 'u:p' -s x\n" +CURL_CREDENTIALS_DETECTOR = RegionDetector( + stream_pattern=re.compile('(L)(L)$'), + match_rules={1: Match(values=[re.compile('^-u$')])}, + match_semantics={}, +) + + +def curl_argstring_breakdown(so_far_tokens, so_far_type_stream, current_token): + """`SingleTokenImprover._curl_argstring_breakdown` as of 2.1.1, the only improvement (shell only).""" + tail_length = min(2, len(so_far_tokens)) + tail_tokens = so_far_tokens[len(so_far_tokens) - tail_length :] + projected_tail = so_far_type_stream[len(so_far_type_stream) - tail_length :] + projected_tail += token_to_typestream_item(current_token) + if not CURL_CREDENTIALS_DETECTOR.match(tail_tokens, projected_tail): + return [current_token] + new_parts = current_token.content.split(':') + if new_parts[0] == '' or new_parts[1] == '': + return [current_token] + final = [] + for part in new_parts: + t = Token( + file=current_token.file, + content=part, + span=current_token.file.get_span_for_string(part, between=current_token.span), + ) + t.set_type([PygmentsToken.Text]) + final.append(t) + return final + + +class EagerLexerTokenizer(LexerTokenizer): + """`LexerTokenizer.tokenize` as it was before tokens were built lazily: one `Token` per lexer token.""" + + def _get_types_for_token(self, token) -> List[Type]: + types = [token] + if token.parent is not None: + if token.parent == PygmentsToken: + return types + types.extend(self._get_types_for_token(token.parent)) + return types + + def tokenize(self, file: File, post_filter=True) -> List[Token]: + self.regions = set() + self.token_stream = '' + self.lexer = self._find_lexer_for_file(file) + if not self.lexer: + return self.tokens + try: + self.language = Language.from_text(self.lexer.filenames[0]) + except (ValueError, IndexError): + self.language = Language.from_text(file.extension) + except Exception: + pass + + raw_tokens = list(self.lexer.get_tokens_unprocessed(file.content)) + current_position = 0 + for _, types, content in raw_tokens: + types = self._get_types_for_token(types) + start = current_position + end = start + len(content) + current_position = end + if current_position > file.length: + continue + try: + if PygmentsToken.Error in types and len(types) == 1: + continue + content = self.sanitize(content) + if not content: + continue + span = file.get_span_for_string(content, between=[start - 1, end + 1]) + token = Token(file=file, content=content, span=span) + token.set_type(types) + improved_tokens = [token] + if self.language == Language.SHELL: + improved_tokens = curl_argstring_breakdown(self.tokens, self.token_stream, token) + self.tokens.extend(improved_tokens) + for improved in improved_tokens: + self.token_stream += token_to_typestream_item(token=improved) + except Exception as e: + str(e) + + self.regions = SubFileRegionsHelper( + file=file, language=self.language, tokens=self.tokens, stream=self.token_stream + ).find() + deep_analyzer = DeepAnalyzer( + regions=self.regions, deep_inspection=self.settings.deep_token_inspection, post_filter=post_filter + ) + self.tokens = deep_analyzer.get_final_tokens() + self.silent_regions = deep_analyzer.silent_regions + return self.tokens + + +def _signature(tokenizer: LexerTokenizer, tokens: List[Token]): + out = [] + for t in tokens: + semantic = None + if t.semantic is not None: + var = t.semantic.payload + semantic = ( + t.semantic.type.name, + t.semantic.creds_probability, + None if var.name_token is None else (var.name_token.content, tuple(var.name_token.span)), + var.value_token is t, + tuple(var.span), + id(var.found_by), + ) + out.append((t.content, tuple(t.span) if t.span else None, tuple(str(x) for x in t.type), semantic)) + regions = sorted((r.substitute_start_index, r.substitute_end_index, str(r.language)) for r in tokenizer.regions) + return out, tokenizer.token_stream, regions, tokenizer.silent_regions + + +def _files(): + # problem_files/ exists only on developer machines, so it is left out to keep the test the same everywhere + paths = sorted(p for p in glob.glob('tests/fixtures/**/*', recursive=True) if '/problem_files/' not in p) + return [File(path=p) for p in paths if os.path.isfile(p)] + [File(path='/tmp/curl.sh', content=SHELL)] + + +def _both(file, monkeypatch, **settings): + post_filter = settings.pop('post_filter') + lazy = LexerTokenizer(**settings) + lazy_result = _signature(lazy, lazy.tokenize(file, post_filter=post_filter)) + with monkeypatch.context() as m: + # sub-file regions (code blocks in Markdown, YAML block scalars) are re-lexed by a nested tokenizer + m.setattr(subfile_regions_helper, 'LexerTokenizer', EagerLexerTokenizer) + eager = EagerLexerTokenizer(**settings) + eager_result = _signature(eager, eager.tokenize(file, post_filter=post_filter)) + return lazy_result, eager_result + + +def test_every_fixture_tokenizes_as_before(monkeypatch): + files = _files() + assert len(files) > 80 + for file in files: + lazy, eager = _both(file, monkeypatch, deep_token_inspection=True, post_filter=True) + assert lazy == eager, file.path + + +@pytest.mark.parametrize('deep, post_filter', [(True, False), (False, False)]) +def test_unfiltered_settings_tokenize_as_before(monkeypatch, deep, post_filter): + # what the diagnostics tracer and the sub-file re-lex use; the files with sub-file regions or an improver + for file in _files(): + if not file.path.endswith(('.md', '.yaml', '.yml', '.sh')): + continue + lazy, eager = _both(file, monkeypatch, deep_token_inspection=deep, post_filter=post_filter) + assert lazy == eager, file.path + + +def test_most_tokens_are_never_built(monkeypatch): + tables = [] + record = TokenTable.__init__ + + def capture(self, file): + record(self, file) + tables.append(self) + + monkeypatch.setattr(TokenTable, '__init__', capture) + final = LexerTokenizer(deep_token_inspection=True).tokenize(File(path='tests/fixtures/2.py')) + + [table] = tables + built = sum(token is not None for token in table.built) + # every kept token is built; the rows cleanup drops by type alone are not, except where detection read them + assert len(final) <= built < len(table) / 2 diff --git a/tests/core/tokenizers/test_single_token_improver.py b/tests/core/tokenizers/test_single_token_improver.py index 3c3e06c..df3c673 100644 --- a/tests/core/tokenizers/test_single_token_improver.py +++ b/tests/core/tokenizers/test_single_token_improver.py @@ -1,20 +1,22 @@ -import glob +import random import pytest import regex as re +from pygments.token import Token as PygmentsToken from deepsecrets.core.model.file import File from deepsecrets.core.model.token import Token from deepsecrets.core.tokenizers.helpers import single_token_improver as sti from deepsecrets.core.tokenizers.helpers.semantic.var_detection.detector import Match, RegionDetector -from deepsecrets.core.tokenizers.helpers.type_stream import token_to_typestream_item +from deepsecrets.core.tokenizers.helpers.type_stream import stream_item, token_to_typestream_item from deepsecrets.core.tokenizers.lexer import LexerTokenizer SHELL = "curl -u admin\ncurl -u 'login01:password01' -s https://x\nsort -u f.txt\ncurl -u 'u:p' -s x\n" +TEXT, NEWLINE, OPERATOR = PygmentsToken.Text, PygmentsToken.Text.Whitespace, PygmentsToken.Operator -def _full_stream_reference(self, so_far_tokens, so_far_type_stream, current_token): - # the implementation before the tail check: the detector ran over the whole stream for every token +def _full_stream_reference(so_far_tokens, so_far_type_stream, current_token): + # the rule as first written: the stream regex over everything emitted so far, for every token projected = so_far_type_stream + token_to_typestream_item(current_token) rule = RegionDetector( stream_pattern=re.compile('(L)(L)$'), @@ -28,36 +30,69 @@ def _full_stream_reference(self, so_far_tokens, so_far_type_stream, current_toke return [current_token] final = [] for part in parts: - token = Token( - file=current_token.file, - content=part, - span=current_token.file.get_span_for_string(part, between=current_token.span), - ) - token.set_type([sti.PygmentsToken.Text]) + token = Token(file=current_token.file, content=part, span=[0, 0]) + token.set_type([PygmentsToken.Text]) final.append(token) return final -def _snapshot(file: File): - tokens = LexerTokenizer(deep_token_inspection=True).tokenize(file) - return [ - (t.content, tuple(t.span) if t.span else None, str(t.type[:1]), getattr(t.semantic, 'creds_probability', None)) - for t in tokens - ] +def _emit_with_reference(sequence): + file = File(path='/tmp/r.sh', content='x') + tokens, stream = [], '' + for ttype, content in sequence: + token = Token(file=file, content=content, span=[0, 0]) + token.set_type([ttype]) + try: + parts = _full_stream_reference(tokens, stream, token) + except IndexError: + continue # the tokenizer skipped a token whose improvement raised + tokens.extend(parts) + stream += ''.join(token_to_typestream_item(t) for t in parts) + return [(t.content, item) for t, item in zip(tokens, stream)] -def _shell_files(): - fixtures = [p for p in glob.glob('tests/fixtures/**/*', recursive=True) if p.endswith(('.sh', '.md'))] - return [File(path=p) for p in fixtures] + [File(path='/tmp/curl.sh', content=SHELL)] - - -def test_tail_check_matches_full_stream_reference(monkeypatch): - for file in _shell_files(): - current = _snapshot(file) - with monkeypatch.context() as m: - m.setattr(sti.SingleTokenImprover, '_curl_argstring_breakdown', _full_stream_reference) - reference = _snapshot(file) - assert current == reference, file.path +def _emit_with_improver(sequence): + improver = sti.SingleTokenImprover(sti.Language.SHELL) + contents, stream = [], [] + for ttype, content in sequence: + item = stream_item(ttype, content) + parts = improver.improve(ttype, content, item, ''.join(stream[-2:]), contents[-2:]) + for part, part_type in [(content, ttype)] if parts is None else parts: + contents.append(part) + stream.append(stream_item(part_type, part)) + return list(zip(contents, stream)) + + +def test_same_decisions_as_the_whole_stream_rule(): + words = [(TEXT, '-u'), (TEXT, 'admin'), (TEXT, 'a:b'), (TEXT, ':b'), (TEXT, 'a:'), (TEXT, 'a:b:c')] + other = [(NEWLINE, '\n'), (OPERATOR, '='), (PygmentsToken.Keyword, 'if')] + rnd = random.Random(7) + for _ in range(3000): + sequence = [rnd.choice(words * 2 + other) for _ in range(rnd.randrange(1, 9))] + assert _emit_with_improver(sequence) == _emit_with_reference(sequence), sequence + + +@pytest.mark.parametrize( + 'previous_items, previous_contents, content, item, expected', + [ + ('L', ['-u'], 'login:password', 'L', [('login', TEXT), ('password', TEXT)]), + ('uL', ['if', '-u'], 'a:b:c', 'L', [('a', TEXT), ('b', TEXT), ('c', TEXT)]), + ('L', ['-u'], ':password', 'L', None), + ('L', ['-u'], 'login:', 'L', None), + ('', [], 'a:b', 'L', None), + ('o', ['-u'], 'a:b', 'L', None), + ('L', ['-x'], 'a:b', 'L', None), + ('L', ['-u'], '=', 'o', None), + # today's behaviour, probably unintended: no ':' after `-u` drops the token + ('L', ['-u'], 'admin', 'L', []), + # and the stream regex's `$` matched before a final newline, so a newline two tokens after `-u` is dropped + ('LL', ['-u', 'login:'], '\n', '\n', []), + ('LL', ['x', '-u'], '\n', '\n', None), + ], +) +def test_curl_credentials_cases(previous_items, previous_contents, content, item, expected): + improver = sti.SingleTokenImprover(sti.Language.SHELL) + assert improver.improve(TEXT, content, item, previous_items, previous_contents) == expected def test_curl_credentials_still_split(): @@ -70,35 +105,24 @@ def test_curl_credentials_still_split(): assert SHELL[slice(*by_content['password01'].span)] == 'password01' -def test_detector_sees_constant_sized_input(monkeypatch): +def test_improver_sees_only_the_two_previous_tokens(monkeypatch): # matching the whole stream per token made shell tokenization quadratic in file length seen = [] - original = sti.CURL_CREDENTIALS_DETECTOR.__class__.match + original = sti.SingleTokenImprover.improve - def spy(self, tokens, stream): - seen.append((len(tokens), len(stream))) - return original(self, tokens, stream) + def spy(self, ttype, content, item, previous_items, previous_contents): + seen.append((len(previous_items), len(previous_contents))) + return original(self, ttype, content, item, previous_items, previous_contents) - monkeypatch.setattr(sti.CURL_CREDENTIALS_DETECTOR.__class__, 'match', spy) + monkeypatch.setattr(sti.SingleTokenImprover, 'improve', spy) content = ''.join(f'echo "step {i}" && export VAR_{i}=value{i}\n' for i in range(300)) LexerTokenizer(deep_token_inspection=True).tokenize(File(path='/tmp/long.sh', content=content)) assert len(seen) > 1000 - assert max(tokens for tokens, _ in seen) <= 2 - assert max(stream for _, stream in seen) <= 3 - - -@pytest.mark.parametrize('prefix_tokens', [0, 1, 2, 5]) -def test_short_prefixes(prefix_tokens): - file = File(path='/tmp/x.sh', content='curl -u a:b\n') - improver = sti.SingleTokenImprover(sti.Language.SHELL) - so_far = [Token(file=file, content='-u', span=[5, 7]) for _ in range(prefix_tokens)] - for token in so_far: - token.set_type([sti.PygmentsToken.Text]) - current = Token(file=file, content='a:b', span=[8, 11]) - current.set_type([sti.PygmentsToken.Text]) + assert max(max(pair) for pair in seen) <= 2 - result = improver._curl_argstring_breakdown(so_far, 'L' * prefix_tokens, current) - reference = _full_stream_reference(improver, so_far, 'L' * prefix_tokens, current) - assert [t.content for t in result] == [t.content for t in reference] +def test_only_shell_is_improved(): + assert sti.SingleTokenImprover(sti.Language.SHELL).applies() + assert not sti.SingleTokenImprover(sti.Language.PYTHON).applies() + assert not sti.SingleTokenImprover(None).applies() diff --git a/tests/core/tokenizers/test_token_table.py b/tests/core/tokenizers/test_token_table.py new file mode 100644 index 0000000..80e8ae9 --- /dev/null +++ b/tests/core/tokenizers/test_token_table.py @@ -0,0 +1,59 @@ +import pytest +from pygments.token import Token as PygmentsToken + +from deepsecrets.core.model.file import File +from deepsecrets.core.tokenizers.helpers.token_table import LazyTokens, TokenTable + +CONTENT = 'key = "value"\n' + + +@pytest.fixture() +def table(): + table = TokenTable(File(path='/tmp/t.py', content=CONTENT)) + table.add(PygmentsToken.Name, 'key', 0, 3) + table.add(PygmentsToken.Operator, '=', 4, 5) + table.add(PygmentsToken.Literal.String.Double, 'value', 6, 13) # raw token '"value"', sanitised + table.add(PygmentsToken.Text.Whitespace, '\n', 13, 14) + return table + + +def test_reading_builds_one_token_and_keeps_it(table): + tokens = LazyTokens(table) + assert table.built == [None] * 4 + + value = tokens[2] + assert (value.content, value.span, value.type) == ( + 'value', + (7, 12), + [PygmentsToken.Literal.String.Double, PygmentsToken.Literal.String, PygmentsToken.Literal], + ) + assert tokens[2] is value and tokens[1:3][1] is value + assert [t is not None for t in table.built] == [False, False, True, False] + + +def test_slices_are_views_with_list_bounds(table): + tokens = LazyTokens(table) + middle = tokens[1:3] + assert isinstance(middle, LazyTokens) and len(middle) == 2 + assert [t.content for t in middle] == ['=', 'value'] + assert len(tokens[3:1]) == 0 and len(tokens[-2:]) == 2 and len(tokens[:100]) == 4 + assert [t.content for t in middle[-1:]] == ['value'] + assert [t.content for t in tokens[::2]] == ['key', 'value'] + + +def test_indexes_behave_like_a_list(table): + tokens = LazyTokens(table) + assert tokens[-1].content == '\n' and tokens[-4].content == 'key' + with pytest.raises(IndexError): + tokens[4] + with pytest.raises(IndexError): + tokens[1:3][2] + assert [t.content for t in tokens] == ['key', '=', 'value', '\n'] + + +def test_existing_tokens_are_kept_as_they_are(table): + tokens = LazyTokens(table) + part = tokens[0] + other = TokenTable(table.file) + other.add_token(part) + assert LazyTokens(other)[0] is part From 9efab0bf30abb4583ab65bd4537e96e1ecaad3ff Mon Sep 17 00:00:00 2001 From: Nikolai Khechumov Date: Mon, 28 Sep 2026 09:29:04 +0000 Subject: [PATCH 03/22] Keep the naturalness model's bloom filter packed The model expanded its bloom filter into a list of ints at import: 585 ms and 31 MB in every worker. The bits now stay packed in bytes and are tested in place, with the same answers. --- .../core/helpers/naturalness_scorer.py | 18 +++++++------- tests/core/helpers/test_naturalness_scorer.py | 24 +++++++++++++++++++ 2 files changed, 32 insertions(+), 10 deletions(-) create mode 100644 tests/core/helpers/test_naturalness_scorer.py diff --git a/deepsecrets/core/helpers/naturalness_scorer.py b/deepsecrets/core/helpers/naturalness_scorer.py index d8016a9..4ab5611 100644 --- a/deepsecrets/core/helpers/naturalness_scorer.py +++ b/deepsecrets/core/helpers/naturalness_scorer.py @@ -14,7 +14,9 @@ def __init__(self, expected_elements=370000, false_positive_rate=0.01): self.fp_rate = false_positive_rate self.bit_size = int(-(self.num_words * math.log(self.fp_rate)) / (math.log(2) ** 2)) self.num_hashes = int((self.bit_size / self.num_words) * math.log(2)) - self.bit_array = [0] * self.bit_size + # The filter's bits, packed eight to a byte, most significant bit first. Kept packed: expanding them into a + # list of ints took 585 ms and 31 MB in every worker, for the same answers. + self.bits = bytes((self.bit_size + 7) // 8) # Trigrams frequency dictionary self.trigram_counts = collections.Counter() @@ -33,8 +35,9 @@ def _get_hashes(self, item): def _in_dictionary(self, word): if len(word) < 2: return False + bits = self.bits for position in self._get_hashes(word): - if self.bit_array[position] == 0: + if not (bits[position >> 3] >> (7 - (position & 7))) & 1: return False return True @@ -54,14 +57,9 @@ def load_from_json(cls, filepath): instance.total_trigrams = model_data["total_trigrams"] instance.trigram_counts = collections.Counter(model_data["trigram_counts"]) - byte_data = bytes.fromhex(model_data["bit_array_hex"]) - bit_array = [] - for byte in byte_data: - for i in range(7, -1, -1): - bit = (byte >> i) & 1 - bit_array.append(bit) - - instance.bit_array = bit_array + instance.bits = bytes.fromhex(model_data["bit_array_hex"]) + if len(instance.bits) * 8 < instance.bit_size: + raise ValueError('the model holds fewer bits than its bit_size') return instance def _get_trigram_score(self, substring): diff --git a/tests/core/helpers/test_naturalness_scorer.py b/tests/core/helpers/test_naturalness_scorer.py new file mode 100644 index 0000000..6d2e22a --- /dev/null +++ b/tests/core/helpers/test_naturalness_scorer.py @@ -0,0 +1,24 @@ +import hashlib +import random +import string + +from deepsecrets.core.helpers.naturalness_scorer import naturalness_scorer + + +def test_bloom_filter_stays_packed(): + # expanding the bits into a list of ints cost 585 ms and 31 MB in every worker + assert isinstance(naturalness_scorer.bits, bytes) + assert len(naturalness_scorer.bits) * 8 >= naturalness_scorer.bit_size + assert not hasattr(naturalness_scorer, 'bit_array') + + +def test_membership_answers_match_the_unpacked_filter(): + # the answers the list-of-ints filter gave for these words, recorded before it was packed + rnd = random.Random(11) + words = [''.join(rnd.choice(string.ascii_lowercase) for _ in range(rnd.randrange(2, 9))) for _ in range(5000)] + answers = ''.join('1' if naturalness_scorer._in_dictionary(word) else '0' for word in words) + + assert answers.count('1') == 915 + assert hashlib.sha256(answers.encode()).hexdigest() == ( + '1f8bbaf5cb466b5f25b0ad12a596e8def0dfddf3edd52923ecfbf34b1e7b9e03' + ) From b1af09f5bbc5a3eeb816a9f5a3b44b90e8160cf8 Mon Sep 17 00:00:00 2001 From: Nikolai Khechumov Date: Mon, 28 Sep 2026 09:29:04 +0000 Subject: [PATCH 04/22] Stop scoring random letters as natural language Random letters-only strings (an AWS key id in capitals, a keyboard-mash password) could score as English because the dictionary's short entries and the bloom filter's false positives cover them. A string whose whole-string trigram score is below PLAUSIBLE_TRIGRAMS is now not language (KI-ENG-02). Two known_imbalance cases pass and lose their marker. --- .../core/helpers/naturalness_scorer.py | 9 ++++++++ tests/core/helpers/test_naturalness_scorer.py | 22 +++++++++++++++++++ .../helpers/test_variable_scoring_cases.py | 2 -- 3 files changed, 31 insertions(+), 2 deletions(-) diff --git a/deepsecrets/core/helpers/naturalness_scorer.py b/deepsecrets/core/helpers/naturalness_scorer.py index 4ab5611..f237480 100644 --- a/deepsecrets/core/helpers/naturalness_scorer.py +++ b/deepsecrets/core/helpers/naturalness_scorer.py @@ -5,6 +5,11 @@ from deepsecrets.core.utils.fs import get_path_inside_package +# Below this whole-string trigram score a string is not language, however well the dictionary covers it: the +# dictionary's many short entries and the bloom filter's false positives cover random letters too (KI-ENG-02). On +# 44 words and placeholders against 440 random letter strings, words score 0.40 and above, random strings 0.38 at most. +PLAUSIBLE_TRIGRAMS = 0.35 + class NaturalnessScorer: @@ -95,6 +100,10 @@ def calculate_score(self, string): if not string.isalpha() or len(string) == 0: return 0.0 + plausibility = self._get_trigram_score(string) + if plausibility < PLAUSIBLE_TRIGRAMS: + return round(plausibility, 4) + n = len(string) dp = [(0, [])] * (n + 1) diff --git a/tests/core/helpers/test_naturalness_scorer.py b/tests/core/helpers/test_naturalness_scorer.py index 6d2e22a..6880deb 100644 --- a/tests/core/helpers/test_naturalness_scorer.py +++ b/tests/core/helpers/test_naturalness_scorer.py @@ -22,3 +22,25 @@ def test_membership_answers_match_the_unpacked_filter(): assert hashlib.sha256(answers.encode()).hexdigest() == ( '1f8bbaf5cb466b5f25b0ad12a596e8def0dfddf3edd52923ecfbf34b1e7b9e03' ) + + +def test_scores_of_known_values(): + assert naturalness_scorer.calculate_score('correcthorsebatterystaple') == 1.0 + # a keyboard mash: the dictionary covered it (0.64) until implausible trigrams stopped counting as language + assert naturalness_scorer.calculate_score('xqzvbnmkjh') == 0.0 + assert naturalness_scorer.calculate_score('kPq9Zr2vXt') == 0.0 + + +def test_random_letters_are_not_language_and_words_still_are(): + import random + import string + + rng = random.Random(7) + randoms = [ + ''.join(rng.choice(string.ascii_lowercase) for _ in range(n)) for n in (8, 12, 16, 20, 32) for _ in range(20) + ] + words = ['password', 'adminadmin', 'sunshine', 'changeme', 'mysecretpassword', 'letmein', 'iloveyou', 'placeholder'] + # a few short random strings read as pronounceable (4 of these 100, all 8 to 12 letters): a threshold high enough + # to reject them would start rejecting real words, whose lowest trigram score was 0.40 + assert sum(naturalness_scorer.calculate_score(r) >= 0.6 for r in randoms) <= len(randoms) // 20 + assert all(naturalness_scorer.calculate_score(w) >= 0.6 for w in words) diff --git a/tests/core/helpers/test_variable_scoring_cases.py b/tests/core/helpers/test_variable_scoring_cases.py index 9c962f8..74ac72c 100644 --- a/tests/core/helpers/test_variable_scoring_cases.py +++ b/tests/core/helpers/test_variable_scoring_cases.py @@ -54,14 +54,12 @@ def known_imbalance(reason: str): LOWER_20, CODE, id='client_secret-random-lowercase', - marks=known_imbalance('naturalness scores random letters-only strings as natural language (-50 > +30)'), ), pytest.param( 'secret', MIXED_LETTERS_16, CODE, id='secret-random-mixedcase-letters', - marks=known_imbalance('naturalness scores random letters-only strings as natural language (-50 > +30)'), ), pytest.param('access_token', B64URL_DASH, CODE, id='access_token-b64url-leading-dash'), pytest.param('access_token', B64URL_UNDERSCORE, CODE, id='access_token-b64url-leading-underscore'), From 70a9fb0a299f08eb332d17160624654d3a095511 Mon Sep 17 00:00:00 2001 From: Nikolai Khechumov Date: Mon, 28 Sep 2026 09:29:04 +0000 Subject: [PATCH 05/22] Rename a local variable spelled with a Cyrillic letter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `patterns` in Rule.fill_confidence_and_file_patterns was written with a Cyrillic 'е' (KI-DM-02). --- deepsecrets/core/model/rules/rule.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deepsecrets/core/model/rules/rule.py b/deepsecrets/core/model/rules/rule.py index 00efcc6..bb5655f 100644 --- a/deepsecrets/core/model/rules/rule.py +++ b/deepsecrets/core/model/rules/rule.py @@ -20,8 +20,8 @@ class Rule(BaseModel): def fill_confidence_and_file_patterns(cls, values: Dict) -> Dict: file_patterns = values.get('applicable_file_patterns', []) if len(file_patterns) > 0: - pattеrns = [re.compile(p) for p in file_patterns] - values['applicable_file_patterns'] = pattеrns + patterns = [re.compile(p) for p in file_patterns] + values['applicable_file_patterns'] = patterns if values.get('confidence', None) is None and values.get('id') is not None: values['confidence'] = 10 From 54e35b786cf7702567b4a534bddc5f23f2fed3a0 Mon Sep 17 00:00:00 2001 From: Nikolai Khechumov Date: Mon, 28 Sep 2026 09:29:04 +0000 Subject: [PATCH 06/22] Rework the worker process model - Workers start from a forkserver where the platform has one (Linux, macOS), spawn elsewhere; the server preloads the scanner and freezes the garbage collector, so workers share it. --multiprocessing-context keeps choosing spawn or fork. The progress manager uses the pool's context. - CI mode: no live display and no manager process, a plain progress line every 30 s. On inside CI services and when output is not a terminal; --ci/--no-ci force it. - A worker killed outright (OOM killer, SIGKILL) no longer hangs the scan: its file is failed with WorkerLostError (KI-CLI-37). - A worker that cannot load the analyzer bundle stops the scan with WorkerStartupError instead of hanging it; a bundle deleted mid-scan is staged again (KI-DM-30). - Multiprocessing sockets go to a short directory when TMPDIR is too long (KI-CLI-40), and the CLI waits for the forkserver after the scan so the workers' CPU time reaches the caller (KI-CLI-42). --- README.md | 1 + deepsecrets/cli.py | 49 +++- deepsecrets/config.py | 8 +- deepsecrets/core/modes/iscan_mode.py | 213 ++++++++++++++++-- deepsecrets/core/utils/environment.py | 37 +++ deepsecrets/core/utils/gc_freeze.py | 7 + .../core/utils/multiprocessing_setup.py | 142 ++++++++++++ deepsecrets/scan_modes/cli.py | 10 +- tests/cli/test_cli.py | 27 +++ tests/core/utils/test_environment.py | 36 +++ .../core/utils/test_multiprocessing_setup.py | 47 ++++ .../scan_modes/test_scan_mode_regressions.py | 199 +++++++++++++++- 12 files changed, 738 insertions(+), 38 deletions(-) create mode 100644 deepsecrets/core/utils/environment.py create mode 100644 deepsecrets/core/utils/gc_freeze.py create mode 100644 deepsecrets/core/utils/multiprocessing_setup.py create mode 100644 tests/core/utils/test_environment.py create mode 100644 tests/core/utils/test_multiprocessing_setup.py diff --git a/README.md b/README.md index 1c04379..05d42f1 100644 --- a/README.md +++ b/README.md @@ -65,6 +65,7 @@ The `--help` command is always ready to guide you, but here are the key flags yo * `--hashed-values /path/to/hashes.json`: Provide a list of pre-hashed known production secrets to search for them securely. * `--excluded-paths /path/to/exclusions.json`: Override or extend the default paths ignored during scanning. * `--disable-masking`: Keep potential secrets unmasked in the output report *(see caution below)*. +* `--ci` / `--no-ci`: Turn CI mode on or off. CI mode prints a plain progress line every 30 seconds instead of the live progress display. It turns on by itself inside CI services (GitHub Actions, GitLab CI, Jenkins and others) and whenever the output is not a terminal. ### Github Actions Integration diff --git a/deepsecrets/cli.py b/deepsecrets/cli.py index 8507991..898b1a0 100644 --- a/deepsecrets/cli.py +++ b/deepsecrets/cli.py @@ -22,6 +22,9 @@ from deepsecrets.core.utils.fs import get_abspath, get_path_inside_package from deepsecrets.core.utils.log import logger from deepsecrets.scan_modes.cli import CliScanMode +from deepsecrets.core.modes.iscan_mode import WorkerStartupError +from deepsecrets.core.utils.environment import is_ci_environment +from deepsecrets.core.utils.multiprocessing_setup import TempDirTooLongError, default_start_method, stop_forkserver from rich.progress import SpinnerColumn, TextColumn, BarColumn, TaskProgressColumn from rich.panel import Panel @@ -195,9 +198,19 @@ def _build_argparser(self) -> None: parser.add_argument( '--multiprocessing-context', type=str, - default='spawn', + default=None, choices=['fork', 'spawn', 'forkserver'], - help='Control the multiprocessing context\n', + help='How worker processes start. Default: forkserver where available (Linux, macOS), spawn elsewhere.\n' + 'forkserver workers share the scanner already loaded in the server instead of loading it again.\n', + ) + + parser.add_argument( + '--ci', + action=argparse.BooleanOptionalAction, + default=None, + help='CI mode: no live progress bars, a plain progress line every 30 s, and no progress manager process.\n' + 'Default: on inside a CI service (CI, GITHUB_ACTIONS, GITLAB_CI and similar variables) or when the\n' + 'output is not a terminal; --no-ci forces the live display.\n', ) parser.add_argument('--outfile', required=True, type=str) @@ -252,7 +265,8 @@ def parse_arguments(self) -> None: config.set_oneshot_path(user_args.oneshot) config.set_max_file_size(user_args.max_file_size) config.set_process_count(user_args.process_count) - config.set_mp_context(user_args.multiprocessing_context) + config.set_mp_context(user_args.multiprocessing_context or default_start_method()) + config.set_ci_mode(is_ci_environment() if user_args.ci is None else user_args.ci) config.set_verbose(user_args.verbose) config.output = Output(type=user_args.outformat, path=user_args.outfile) @@ -371,19 +385,32 @@ def start(self) -> int: # pragma: nocover f'[bold yellow]:warning:[/bold yellow] The tool will return code of {ReturnCodes.FINDINGS_DETECTED} if any findings are detected\n' ) - mode = CliScanMode(config=config) + try: + mode = CliScanMode(config=config) + except TempDirTooLongError as e: + console.print(f'[bold red][!] Cannot start worker processes: {e}.[/bold red]') + return ReturnCodes.ERROR console.line() - mode.set_progress_bar(progress_bar) - mode.progress_bar.set_start_time(startup_time) - mode.progress_bar.start() + if not config.ci_mode: + mode.set_progress_bar(progress_bar) + mode.progress_bar.set_start_time(startup_time) + mode.progress_bar.start() findings: List[Finding] errors: Dict[str, List[str]] timings: Dict[str, int] - findings, errors, timings = mode.run() + try: + findings, errors, timings = mode.run() + except WorkerStartupError as e: + if mode.progress_bar is not None: + mode.progress_bar.stop() + console.print(f'[bold red][!] Scan stopped: {e}. Every file it took would fail the same way.[/bold red]') + mode.dispose() + stop_forkserver() + return ReturnCodes.ERROR ''' for finding in findings: @@ -392,7 +419,8 @@ def start(self) -> int: # pragma: nocover finding.file = None ''' - mode.progress_bar.stop() + if mode.progress_bar is not None: + mode.progress_bar.stop() finish_time = datetime.now() report_path = get_abspath(config.output.path) @@ -474,6 +502,9 @@ def start(self) -> int: # pragma: nocover console.print(Align('[bold green]FINISHED', align='center')) console.line(2) mode.dispose() + # the pool and the manager are gone: wait for the forkserver, so whoever waits for this process sees the + # workers' CPU time (KI-CLI-42) + stop_forkserver() if len(findings) > 0 and config.return_code_if_findings: return ReturnCodes.FINDINGS_DETECTED diff --git a/deepsecrets/config.py b/deepsecrets/config.py index 86b712e..3efab66 100644 --- a/deepsecrets/config.py +++ b/deepsecrets/config.py @@ -7,6 +7,7 @@ from deepsecrets.core.utils.exceptions import FileNotFoundException from deepsecrets.core.utils.fs import get_abspath, path_exists +from deepsecrets.core.utils.multiprocessing_setup import default_start_method FALLBACK_PROCESS_COUNT = 4 @@ -28,7 +29,7 @@ class Config: workdir_path: str oneshot_path: str max_file_size: int = 0 # 0 means no limit - mp_context: str = 'spawn' + mp_context: str = default_start_method() engines: List[Type] = [] rulesets: Dict[Type, List[str]] = {} global_exclusion_paths: List[str] = [] @@ -38,6 +39,8 @@ class Config: disable_masking: bool report_diagnostics: bool = False verbose: bool = False + # no live terminal UI and no progress manager; the CLI turns it on in CI and when output is not a terminal + ci_mode: bool = False _benchmarking_mode: bool @@ -90,6 +93,9 @@ def set_max_file_size(self, size: int) -> None: def set_mp_context(self, context: str) -> None: self.mp_context = context + def set_ci_mode(self, enabled: bool) -> None: + self.ci_mode = enabled + def set_process_count(self, count: int) -> None: if count > 0: self.process_count = count diff --git a/deepsecrets/core/modes/iscan_mode.py b/deepsecrets/core/modes/iscan_mode.py index f9eb31e..2ee7bb6 100644 --- a/deepsecrets/core/modes/iscan_mode.py +++ b/deepsecrets/core/modes/iscan_mode.py @@ -10,7 +10,7 @@ from queue import SimpleQueue import regex as re -from multiprocessing import Manager, get_context +import contextlib from multiprocessing.managers import DictProxy import errno import os @@ -31,12 +31,29 @@ from deepsecrets.core.utils.finding_merger import FindingMerger from deepsecrets.core.utils.fs import get_abspath, get_relative_path from deepsecrets.core.utils.log import logger +from deepsecrets.core.utils.multiprocessing_setup import pool_context, start_manager +from deepsecrets.utils import setup_interrupts_for_subprocess from rich.progress import Progress as ProgressBar from rich.live import Live from rich.text import Text +class WorkerStartupError(RuntimeError): + """A pool worker could not load the analyzer bundle, so every file it took would fail the same way.""" + + +class WorkerLostError(RuntimeError): + """The worker process analysing a file exited without reporting a result (killed, or out of memory).""" + + +# seconds between noticing that a worker exited and failing the file it held, so a result it sent just before +# exiting can still arrive +LOST_WORKER_GRACE_SECONDS = 0.5 +# in CI mode, a plain progress line every this many seconds +CI_PROGRESS_INTERVAL_SECONDS = 30.0 + + @dataclass class FileJob: internal_id: int @@ -96,6 +113,8 @@ class ScanMode: files: Dict[str, FileOutcome] _mp_manager = None + # modules a forkserver imports before forking workers, so they share them (see pool_context) + worker_modules: List[str] = [] # ONLY IN BENCHMARKING MODE _oneshot_file: Optional[File] = None @@ -103,19 +122,29 @@ class ScanMode: def __init__(self, config: Config, pool_engine: Optional[Any] = None) -> None: console.print('[*] Looking for applicable files...', end='') console.line() + self.mp_context = pool_context(config.mp_context, self.worker_modules) if pool_engine is None: - self.pool_engine = get_context(config.mp_context).Pool + self.pool_engine = self.mp_context.Pool else: self.pool_engine = pool_engine - self._mp_manager = Manager() - self.active_task_reporter = self._mp_manager.dict({}) + # CI mode reports no per-file progress, so it needs no manager process (and no manager socket) + self._mp_manager = None if config.ci_mode else start_manager(self.mp_context) + self.active_task_reporter = self._mp_manager.dict({}) if self._mp_manager is not None else None self.progress_bar = None self.config = config self.file_results = [] self.file_jobs = {} self.failed_jobs = SimpleQueue() + self.done_jobs = SimpleQueue() + self.startup_error: Optional[str] = None + # task id -> pid of the worker that took it, written by the worker (pool_wrapper), for reap_lost_jobs + self.task_pids = None + self.lost_jobs: Dict[int, int] = {} + self._worker_pids_seen: set = set() + self._worker_gone_at: Dict[int, float] = {} + self._worker_pids_handled: set = set() self.stats = Stats() self.files = {} @@ -216,8 +245,72 @@ def refresh_jobs_progress_bars(self): def _on_job_error(self, task_id: int, error: BaseException) -> None: # Called by the pool's result handler thread when a job raised + if isinstance(error, WorkerStartupError): + self.startup_error = str(error) self.failed_jobs.put(task_id) + def _on_job_done(self, task_id: int, result: PerFileAnalysisResult) -> None: + # Called by the pool's result handler thread when a job returned; CI mode counts completion from this + self.done_jobs.put((task_id, len(result.findings or []))) + + def _new_task_pids(self) -> Any: + # one slot per task id (they start at 1); shared memory, so a worker's write costs no message to the parent + self.task_pids = self.mp_context.RawArray('i', len(self.filepaths) + 1) + return self.task_pids + + def reap_done_jobs(self) -> None: + while not self.done_jobs.empty(): + task_id, findings = self.done_jobs.get() + if task_id in self.stats.finished_ids: + continue + self.stats.total_findings += findings + self.stats.new_finished(task_id) + + def reap_lost_jobs(self, pool: Any) -> None: + """Fail the file a killed worker was analysing (KI-CLI-37). + + A worker killed outright (OOM killer, SIGKILL) takes its task with it: the pool starts a replacement but never + reports the task, so the scan used to wait forever. Each worker writes its pid into `task_pids` when it takes + a file; a pid that has left the pool's worker list for longer than LOST_WORKER_GRACE_SECONDS marks every + unfinished file it took as lost.""" + if self.task_pids is None: + return + live = {pid for pid in (getattr(worker, 'pid', None) for worker in list(getattr(pool, '_pool', []))) if pid} + if not live and not self._worker_pids_seen: + return # a thread pool: its workers cannot exit on their own + self._worker_pids_seen |= live + now = time.monotonic() + for pid in self._worker_pids_seen - live - self._worker_pids_handled: + self._worker_gone_at.setdefault(pid, now) + gone = {pid for pid, at in self._worker_gone_at.items() if now - at >= LOST_WORKER_GRACE_SECONDS} + gone -= self._worker_pids_handled + if not gone: + return + self._worker_pids_handled |= gone + for task_id, job in self.file_jobs.items(): + pid = self.task_pids[task_id] + if pid not in gone or task_id in self.lost_jobs or job.result_holder.ready(): + continue + self.lost_jobs[task_id] = pid + fail_lost_task( + job.result_holder, + WorkerLostError( + f'the worker process {pid} exited while analysing this file (killed, or out of memory)' + ), + ) + + def report_plain_progress(self, force: bool = False) -> None: + """CI mode's progress: a plain line every CI_PROGRESS_INTERVAL_SECONDS instead of live bars.""" + now = time.monotonic() + if not force and now - self._last_plain_progress < CI_PROGRESS_INTERVAL_SECONDS: + return + self._last_plain_progress = now + console.print( + f'[*] {self.stats.finished}/{self.stats.total_files} files scanned, ' + f'{self.stats.failed_files} failed, {self.stats.total_findings} findings before merging', + highlight=False, + ) + def reap_failed_jobs(self) -> None: # A job that raised never reports 'finished', so without this run() would poll forever while not self.failed_jobs.empty(): @@ -227,7 +320,8 @@ def reap_failed_jobs(self) -> None: self.stats.failed_files += 1 self.stats.new_finished(task_id) - self.active_task_reporter.pop(task_id, None) + if self.active_task_reporter is not None: + self.active_task_reporter.pop(task_id, None) job = self.file_jobs.get(task_id) if job is not None and job.pb_task_id is not None: @@ -257,13 +351,17 @@ def run(self) -> Tuple[List[Finding], Dict[str, List[str]], Dict[str, int]]: if proc_count == 0: return final, self.per_file_errors(), self.per_file_timings() - overall_progress_task = self.progress_bar.add_task( - "[green bold]OVERALL\nPROGRESS\n", - visible=True, - findings='F: 0', - errors='ERR: 0', - size=f'0/{self.stats.total_files}', - ) + overall_progress_task = None + if self.progress_bar is not None: + overall_progress_task = self.progress_bar.add_task( + "[green bold]OVERALL\nPROGRESS\n", + visible=True, + findings='F: 0', + errors='ERR: 0', + size=f'0/{self.stats.total_files}', + ) + ci_mode = self.config.ci_mode + self._last_plain_progress = time.monotonic() if PROFILER_ON: for file in self.filepaths: @@ -281,7 +379,7 @@ def run(self) -> Tuple[List[Finding], Dict[str, List[str]], Dict[str, int]]: self.pool_engine( processes=proc_count, initializer=init_worker, - initargs=(stage_bundle(bundle, staging), self.active_task_reporter), + initargs=(stage_bundle(bundle, staging), self.active_task_reporter, self._new_task_pids()), ) as pool, ): tid = 0 @@ -290,6 +388,7 @@ def run(self) -> Tuple[List[Finding], Dict[str, List[str]], Dict[str, int]]: result = pool.apply_async( pool_wrapper, (self._per_file_analyzer, tid, file), + callback=partial(self._on_job_done, tid) if ci_mode else None, error_callback=partial(self._on_job_error, tid), ) self.file_results.append(result) @@ -298,12 +397,23 @@ def run(self) -> Tuple[List[Finding], Dict[str, List[str]], Dict[str, int]]: self.stats.total_files = len(self.file_jobs.keys()) while self.stats.finished < self.stats.total_files: + # a worker that cannot load the bundle would fail every file it takes: stop, don't poll + # for results that cannot come (KI-DM-30) + if self.startup_error is not None: + raise WorkerStartupError(self.startup_error) + keep_bundle_staged(bundle, staging) self.refresh_jobs_progress_bars() + self.reap_done_jobs() + self.reap_lost_jobs(pool) self.reap_failed_jobs() self.refresh_overall_progress_bar(overall_progress_task) + if ci_mode: + self.report_plain_progress() time.sleep(0.1) # self.refresh_overall_debug_progress_bar(overall_debug) self.stop_progress_bar(overall_progress_task) + if ci_mode: + self.report_plain_progress(force=True) console.print('[*] Collecting results..') pool.join() @@ -357,7 +467,8 @@ def run(self) -> Tuple[List[Finding], Dict[str, List[str]], Dict[str, int]]: def dispose(self): self.task_reporter = None - self._mp_manager.shutdown() + if self._mp_manager is not None: + self._mp_manager.shutdown() print() def _get_files_list(self) -> List[str]: @@ -375,12 +486,14 @@ def _get_files_list(self) -> List[str]: flist.append(path) return flist - with Live(console=console, refresh_per_second=5) as live: + # CI mode: no live counter, which only redraws in a build log + with contextlib.nullcontext() if self.config.ci_mode else Live(console=console, refresh_per_second=5) as live: total_files = 0 for fpath, _, files in os.walk(get_abspath(self.config.workdir_path)): for filename in files: total_files += 1 - live.update(Text(text=f'Found {total_files} files, {self.stats.skipped_files} will be skipped')) + if live is not None: + live.update(Text(text=f'Found {total_files} files, {self.stats.skipped_files} will be skipped')) full_path = os.path.join(fpath, filename) rel_path = get_relative_path(full_path, self.config.workdir_path) exclusion = self._matching_exclusion(rel_path) @@ -499,22 +612,78 @@ def stage_bundle(bundle: AnalyzerBundle, directory: str) -> str: 33 KB) the parent blocks until the child has re-imported the package, so workers start one at a time: 64 of them took 55 s. A path is a few bytes, however many rules there are (KI-CLI-32). """ + os.makedirs(directory, mode=0o700, exist_ok=True) path = os.path.join(directory, 'bundle.pickle') with open(path, 'wb') as f: pickle.dump(bundle, f, protocol=pickle.HIGHEST_PROTOCOL) return path -def init_worker(bundle_path: str, task_reporter: DictProxy) -> None: # pragma: nocover - # Pool initializer: runs once per worker. Pickling the bundle (compiled rules) and the proxy - # with every task cost more than analysing a typical small file. - global _worker_bundle, _worker_task_reporter +def keep_bundle_staged(bundle: AnalyzerBundle, directory: str) -> None: + """Stage the bundle again if something deleted it mid-scan, such as a temporary-file cleaner. Workers that already + started do not need it, but the pool replaces any worker that exits, and the replacement loads it afresh.""" + if not os.path.exists(os.path.join(directory, 'bundle.pickle')): + stage_bundle(bundle, directory) + + +# seconds a worker waits before its one retry of a failed bundle load, which covers the parent re-staging it +BUNDLE_RETRY_DELAY = 0.5 +_worker_bundle_path: Optional[str] = None +_worker_startup_error: Optional[str] = None +_worker_task_pids: Any = None + + +def _load_bundle(bundle_path: str) -> AnalyzerBundle: # written by this scan into a private temporary directory (mkdtemp creates it 0700) with open(bundle_path, 'rb') as f: - _worker_bundle = pickle.load(f) + return pickle.load(f) + + +def init_worker(bundle_path: str, task_reporter: DictProxy, task_pids: Any = None) -> None: # pragma: nocover + # Pool initializer: runs once per worker. Pickling the bundle (compiled rules) and the proxy + # with every task cost more than analysing a typical small file. + # It must not raise: the pool would replace the worker, the replacement would fail the same way, and the scan + # would wait forever for results (KI-DM-30). The failure is kept for the worker's first task to report. + global _worker_bundle, _worker_task_reporter, _worker_bundle_path, _worker_startup_error, _worker_task_pids + # a forkserver worker inherits the server's handlers, where this module was imported as the main process, so the + # import-time call at the top of this module did nothing there: Ctrl+C belongs to the parent + setup_interrupts_for_subprocess() _worker_task_reporter = task_reporter + _worker_task_pids = task_pids + _worker_bundle_path = bundle_path + try: + _worker_bundle = _load_bundle(bundle_path) + _worker_startup_error = None + except Exception as e: + _worker_bundle = None + _worker_startup_error = f'{type(e).__name__}: {e}' + + +def _bundle_or_raise() -> AnalyzerBundle: + global _worker_bundle, _worker_startup_error + if _worker_bundle is None: + time.sleep(BUNDLE_RETRY_DELAY) + try: + _worker_bundle = _load_bundle(_worker_bundle_path) + _worker_startup_error = None + except Exception as e: + raise WorkerStartupError( + f'a worker could not load the analyzer bundle ({_worker_startup_error}; retry: {type(e).__name__}: {e})' + ) from e + return _worker_bundle def pool_wrapper(runner: Callable, task_id: Optional[int], file: str) -> PerFileAnalysisResult: # pragma: nocover - result = runner(_worker_bundle, file, task_id, _worker_task_reporter) + if _worker_task_pids is not None and isinstance(task_id, int) and 0 <= task_id < len(_worker_task_pids): + _worker_task_pids[task_id] = os.getpid() # lets the parent fail this file if this process dies (KI-CLI-37) + result = runner(_bundle_or_raise(), file, task_id, _worker_task_reporter) return result + + +def fail_lost_task(result: AsyncResult, error: BaseException) -> None: + """Complete a task whose worker died with `error`, as if the task had raised it: the error callback runs, `get()` + raises it, and the pool stops waiting for it, so `join()` returns. Uses `ApplyResult._set`, the method the pool's + own result handler calls; a result that arrives later is ignored by the pool.""" + setter = getattr(result, '_set', None) + if setter is not None and not result.ready(): + setter(0, (False, error)) diff --git a/deepsecrets/core/utils/environment.py b/deepsecrets/core/utils/environment.py new file mode 100644 index 0000000..2cdae9e --- /dev/null +++ b/deepsecrets/core/utils/environment.py @@ -0,0 +1,37 @@ +import os +import sys +from typing import Mapping, Optional, TextIO + +# Set by the CI services that do not also set CI=true. +CI_VARIABLES = ( + 'GITHUB_ACTIONS', + 'GITLAB_CI', + 'CIRCLECI', + 'TRAVIS', + 'BUILDKITE', + 'TF_BUILD', # Azure Pipelines + 'JENKINS_URL', + 'TEAMCITY_VERSION', + 'BITBUCKET_BUILD_NUMBER', + 'CODEBUILD_BUILD_ID', # AWS CodeBuild + 'DRONE', + 'APPVEYOR', +) +FALSE_VALUES = ('', '0', 'false', 'no', 'off') + + +def is_ci_environment(environ: Optional[Mapping[str, str]] = None, stream: Optional[TextIO] = None) -> bool: + """Whether to run without the live terminal UI: inside a CI service, or when the output is not a terminal + (piped, redirected, or captured by a build log), where live progress bars only add redraw work and noise.""" + environ = os.environ if environ is None else environ + stream = sys.stdout if stream is None else stream + + if environ.get('CI', '').strip().lower() not in FALSE_VALUES: + return True + if any(environ.get(name, '').strip().lower() not in FALSE_VALUES for name in CI_VARIABLES): + return True + + try: + return not stream.isatty() + except (AttributeError, ValueError): # a replaced or closed stream + return True diff --git a/deepsecrets/core/utils/gc_freeze.py b/deepsecrets/core/utils/gc_freeze.py new file mode 100644 index 0000000..39d7739 --- /dev/null +++ b/deepsecrets/core/utils/gc_freeze.py @@ -0,0 +1,7 @@ +"""Imported last by the forkserver, after the scanner's modules: moves everything loaded so far out of the garbage +collector's reach, so a worker forked from the server does not write to those pages while collecting, and they stay +shared with the server instead of being copied into every worker.""" + +import gc + +gc.freeze() diff --git a/deepsecrets/core/utils/multiprocessing_setup.py b/deepsecrets/core/utils/multiprocessing_setup.py new file mode 100644 index 0000000..7ffa200 --- /dev/null +++ b/deepsecrets/core/utils/multiprocessing_setup.py @@ -0,0 +1,142 @@ +"""How the scan starts its processes: the start method, where multiprocessing puts its sockets, and the manager. + +multiprocessing creates Unix sockets for the progress manager and for the forkserver at +`/pymp-XXXXXXXX/listener-XXXXXXXX`, 32 characters below the temporary directory. A socket path is at most +107 bytes on Linux and 103 on macOS, so a long `TMPDIR` made every scan die with a bare `EOFError` (KI-CLI-40). The +functions here put multiprocessing's private directory somewhere short instead, before any socket is created. +""" + +import contextlib +import os +import sys +import tempfile +import time +from multiprocessing import connection, forkserver, get_all_start_methods, get_context, process, util +from multiprocessing.context import BaseContext +from multiprocessing.managers import SyncManager +from typing import List, Optional + +SOCKET_PATH_MAX = 103 # macOS: 104 bytes with the terminating NUL; Linux allows 108 +SOCKET_BELOW_TEMP_DIR = len('/pymp-XXXXXXXX/listener-XXXXXXXX') +SOCKET_BELOW_MP_DIR = len('/listener-XXXXXXXX') +SHORT_TEMP_DIRS = ('/tmp', '/var/tmp', '/dev/shm') + +# imported by the forkserver before it forks any worker, so every worker shares them instead of importing them again +FREEZE_MODULE = 'deepsecrets.core.utils.gc_freeze' +# how long stop_forkserver waits for the server to exit +FORKSERVER_STOP_SECONDS = 10.0 + + +class TempDirTooLongError(RuntimeError): + """No directory was found where multiprocessing's socket paths fit.""" + + +def default_start_method() -> str: + """`forkserver` where the platform has it (Linux, macOS): workers are forked from a server that has already + imported the scanner, so they share its memory and start faster. `spawn` elsewhere (Windows).""" + return 'forkserver' if 'forkserver' in get_all_start_methods() else 'spawn' + + +def ensure_socket_friendly_tempdir() -> Optional[str]: + """Create multiprocessing's private directory where a socket path inside it fits, and return it. + + Uses `TMPDIR` when it is short enough, else the first of `SHORT_TEMP_DIRS` that is writable. Must run before + multiprocessing creates its directory, that is before the first manager or forkserver starts. `None` on Windows, + where multiprocessing uses named pipes.""" + if sys.platform == 'win32': + return None + + existing = process.current_process()._config.get('tempdir') + if existing: + if len(existing) + SOCKET_BELOW_MP_DIR > SOCKET_PATH_MAX: + raise TempDirTooLongError(_too_long_message(existing)) + return existing + + base = tempfile.gettempdir() + if len(base) + SOCKET_BELOW_TEMP_DIR <= SOCKET_PATH_MAX: + return util.get_temp_dir() + + for candidate in SHORT_TEMP_DIRS: + if len(candidate) + SOCKET_BELOW_TEMP_DIR > SOCKET_PATH_MAX: + continue + if not os.path.isdir(candidate) or not os.access(candidate, os.W_OK | os.X_OK): + continue + saved = tempfile.tempdir + tempfile.tempdir = candidate + try: + # created once per process and removed at exit by multiprocessing itself + return util.get_temp_dir() + finally: + tempfile.tempdir = saved + + raise TempDirTooLongError(_too_long_message(base)) + + +def _too_long_message(directory: str) -> str: + limit = SOCKET_PATH_MAX - SOCKET_BELOW_TEMP_DIR + return ( + f'the temporary directory {directory} is {len(directory)} characters long, and no shorter one is writable; ' + f'multiprocessing needs one of at most {limit} characters for its sockets. Set TMPDIR to a shorter directory' + ) + + +def pool_context(start_method: str, preload: List[str]) -> BaseContext: + """The multiprocessing context for the worker pool. For `forkserver`, the server imports `preload` and then + freezes the garbage collector, so the workers forked from it share those modules' memory.""" + ctx = get_context(start_method) + if start_method == 'forkserver': + ensure_socket_friendly_tempdir() + # takes effect when the server starts, at the first manager or pool of the process + ctx.set_forkserver_preload(list(preload) + [FREEZE_MODULE]) + return ctx + + +def stop_forkserver(timeout: float = FORKSERVER_STOP_SECONDS) -> bool: + """Stop this process's forkserver, if one runs, and wait for it; True when it has exited (or none ran). + + The server reaps the workers it forked, so their CPU time reaches this process only through this wait. Without it + the server outlives the scan as an orphan, and `time`, or a benchmark harness that waits for the scanner, sees the + main process alone (KI-CLI-42). Call it when the pool and the manager are gone: every process the server started + holds the server's alive pipe, so it exits only after the last of them. The wait is bounded for that reason; on + timeout the server exits later, with its last child. The next pool starts a new server. + + CPython's `ForkServer._stop` would wait without a bound, so this does the same steps with one.""" + server = getattr(forkserver, '_forkserver', None) + if server is None: + return True + if not all(hasattr(server, a) for a in ('_lock', '_forkserver_pid', '_forkserver_alive_fd', '_forkserver_address')): + return False # a CPython whose forkserver is built differently: leave it to exit on its own + with server._lock: + pid = server._forkserver_pid + if pid is None: + return True + # closing our end of the alive pipe asks the server to exit once no child holds it either + os.close(server._forkserver_alive_fd) + server._forkserver_alive_fd = None + server._forkserver_pid = None + address, server._forkserver_address = server._forkserver_address, None + if address and not util.is_abstract_socket_namespace(address): + with contextlib.suppress(OSError): + os.unlink(address) + deadline = time.monotonic() + timeout + while True: + try: + if os.waitpid(pid, os.WNOHANG)[0]: + return True + except ChildProcessError: + return True + if time.monotonic() >= deadline: + return False + time.sleep(0.01) + + +def start_manager(ctx: BaseContext) -> SyncManager: + """What `ctx.Manager()` does, with the socket address chosen here: a spawned or forkserver-started manager + would otherwise pick its own, under the original `TMPDIR`.""" + address = None + if sys.platform != 'win32': + ensure_socket_friendly_tempdir() + address = connection.arbitrary_address('AF_UNIX') + manager = SyncManager(address=address, ctx=ctx) + manager.start() + return manager diff --git a/deepsecrets/scan_modes/cli.py b/deepsecrets/scan_modes/cli.py index 4bd2b76..065ba01 100644 --- a/deepsecrets/scan_modes/cli.py +++ b/deepsecrets/scan_modes/cli.py @@ -24,6 +24,8 @@ class CliScanMode(ScanMode): + # the per-file analyzer below and everything it imports: engines, tokenizers, the naturalness model + worker_modules = ['deepsecrets.scan_modes.cli'] def prepare_for_scan(self) -> None: self.engines_enabled: Dict[str, bool] = {} @@ -81,12 +83,12 @@ def __finalize(result: PerFileAnalysisResult): file = File(path=file, relative_path=get_relative_path(file, bundle.workdir)) except Exception as e: logger.error(f'Unable to open the file: {e}') - lifecycle.on_failure(task_reporter[task_id]) + lifecycle.on_failure(task_reporter[task_id] if task_reporter is not None else None) result.status = 'unreadable' return __finalize(result) if file.length == 0: - lifecycle.on_finish(task_reporter[task_id]) + lifecycle.on_finish(task_reporter[task_id] if task_reporter is not None else None) result.status = 'empty' return __finalize(result) @@ -128,7 +130,7 @@ def __finalize(result: PerFileAnalysisResult): if PROFILER_ON: pass - if task_reporter is not None: - lifecycle.on_finish(task_reporter.get('task_id')) + # always: it records the end time; with no reporter (CI mode) it reports nothing + lifecycle.on_finish(task_reporter.get('task_id') if task_reporter is not None else None) return __finalize(result) diff --git a/tests/cli/test_cli.py b/tests/cli/test_cli.py index d4f9f61..2f8c814 100644 --- a/tests/cli/test_cli.py +++ b/tests/cli/test_cli.py @@ -4,6 +4,7 @@ from deepsecrets.core.engines.hashed_secret import HashedSecretEngine from deepsecrets.core.engines.regex import RegexEngine from deepsecrets.core.rulesets.hashed_secrets import HashedSecretsRulesetBuilder +from deepsecrets.core.utils.multiprocessing_setup import default_start_method @pytest.fixture(scope='module') @@ -95,3 +96,29 @@ def test_hashed_values_registers_hashed_engine(): finally: # the config singleton outlives this test (KI-CLI-12) config.rulesets.pop(HashedSecretsRulesetBuilder, None) + + +@pytest.mark.parametrize( + 'extra, detected, ci_mode', + [([], True, True), ([], False, False), (['--ci'], False, True), (['--no-ci'], True, False)], +) +def test_ci_mode_follows_the_flags_then_the_environment(monkeypatch, extra, detected, ci_mode): + monkeypatch.setattr('deepsecrets.cli.is_ci_environment', lambda: detected) + tool = DeepSecretsCliTool(args=['', '--target-dir', '/app/tests/fixtures/', '--outfile', '/tmp/x.sarif'] + extra) + config = tool.get_current_config() + monkeypatch.setattr(config, 'ci_mode', not ci_mode) # the singleton keeps what earlier tests set + tool.parse_arguments() + + assert config.ci_mode is ci_mode + + +@pytest.mark.parametrize( + 'extra, expected', [([], default_start_method()), (['--multiprocessing-context', 'spawn'], 'spawn')] +) +def test_multiprocessing_context_defaults_to_the_platform_start_method(monkeypatch, extra, expected): + tool = DeepSecretsCliTool(args=['', '--target-dir', '/app/tests/fixtures/', '--outfile', '/tmp/x.sarif'] + extra) + config = tool.get_current_config() + monkeypatch.setattr(config, 'mp_context', 'fork') + tool.parse_arguments() + + assert config.mp_context == expected diff --git a/tests/core/utils/test_environment.py b/tests/core/utils/test_environment.py new file mode 100644 index 0000000..2ebc486 --- /dev/null +++ b/tests/core/utils/test_environment.py @@ -0,0 +1,36 @@ +import io + +import pytest + +from deepsecrets.core.utils.environment import is_ci_environment + + +class Terminal(io.StringIO): + def isatty(self): + return True + + +@pytest.mark.parametrize( + 'environ, expected', + [ + ({}, False), + ({'CI': 'true'}, True), + ({'CI': '1'}, True), + ({'CI': 'false'}, False), + ({'CI': ''}, False), + ({'GITHUB_ACTIONS': 'true'}, True), + ({'TF_BUILD': 'True'}, True), + ({'JENKINS_URL': 'https://ci.example/'}, True), + ({'GITLAB_CI': 'false'}, False), + ], +) +def test_ci_variables_on_a_terminal(environ, expected): + assert is_ci_environment(environ, Terminal()) is expected + + +def test_output_that_is_not_a_terminal_counts_as_ci(): + assert is_ci_environment({}, io.StringIO()) is True + + +def test_a_stream_without_isatty_counts_as_ci(): + assert is_ci_environment({}, object()) is True diff --git a/tests/core/utils/test_multiprocessing_setup.py b/tests/core/utils/test_multiprocessing_setup.py new file mode 100644 index 0000000..2c9f81c --- /dev/null +++ b/tests/core/utils/test_multiprocessing_setup.py @@ -0,0 +1,47 @@ +import os +import subprocess +import sys +from pathlib import Path + +from deepsecrets.core.utils.multiprocessing_setup import SOCKET_BELOW_TEMP_DIR, SOCKET_PATH_MAX, default_start_method + +# a fresh interpreter: multiprocessing picks its private directory once per process +SCRIPT = ''' +from multiprocessing import connection, util +from deepsecrets.core.utils.multiprocessing_setup import pool_context, start_manager +ctx = pool_context('forkserver', []) +manager = start_manager(ctx) +shared = manager.dict({'a': 1}) +with ctx.Pool(2) as pool: + squares = pool.map(abs, [-1, -2, -3]) +print(len(util.get_temp_dir()), shared['a'], squares) +manager.shutdown() +''' + + +def test_default_start_method_is_forkserver_where_available(): + assert default_start_method() == ('forkserver' if sys.platform != 'win32' else 'spawn') + + +def test_a_long_tmpdir_no_longer_breaks_the_manager_or_the_forkserver(tmp_path: Path): + # KI-CLI-40: /pymp-XXXXXXXX/listener-XXXXXXXX must stay under the socket path limit + long_dir = tmp_path / ('d' * 60) / ('e' * 60) + long_dir.mkdir(parents=True) + assert len(str(long_dir)) + SOCKET_BELOW_TEMP_DIR > SOCKET_PATH_MAX + env = dict(os.environ, TMPDIR=str(long_dir), PYTHONPATH=os.getcwd()) + + broken = subprocess.run( + [sys.executable, '-c', 'from multiprocessing import Manager; Manager()'], + env=env, + capture_output=True, + text=True, + timeout=120, + ) + if sys.version_info < (3, 14): # 3.14 picks a short directory itself (multiprocessing.util._get_base_temp_dir) + assert broken.returncode != 0 and 'EOFError' in broken.stderr # what used to happen to every scan + + fixed = subprocess.run([sys.executable, '-c', SCRIPT], env=env, capture_output=True, text=True, timeout=120) + assert fixed.returncode == 0, fixed.stderr + tempdir_length, value, squares = fixed.stdout.split(' ', 2) + assert int(tempdir_length) + len('/listener-XXXXXXXX') <= SOCKET_PATH_MAX + assert value == '1' and squares.strip() == '[1, 2, 3]' diff --git a/tests/scan_modes/test_scan_mode_regressions.py b/tests/scan_modes/test_scan_mode_regressions.py index 0ca5703..e0a90be 100644 --- a/tests/scan_modes/test_scan_mode_regressions.py +++ b/tests/scan_modes/test_scan_mode_regressions.py @@ -1,9 +1,14 @@ import errno +import gc import os import pickle import shutil +import signal +import subprocess +import sys import threading from dataclasses import FrozenInstanceError, fields +from multiprocessing import forkserver, get_all_start_methods from multiprocessing.pool import ThreadPool from pathlib import Path from unittest.mock import Mock @@ -20,6 +25,7 @@ from deepsecrets.core.rulesets.regex import RegexRulesetBuilder from deepsecrets.core.utils.fs import get_path_inside_package from deepsecrets.core.modes import iscan_mode +from deepsecrets.core.utils.multiprocessing_setup import pool_context from deepsecrets.scan_modes.cli import CliScanMode HASHED_SECRET = '$ecRetT0F1nD' # sha1 listed in tests/fixtures/hashed_secrets.json, present in tests/fixtures/1.py @@ -177,7 +183,9 @@ def test_bundle_reaches_workers_once_not_per_task(): findings, errors, _ = _run_with_timeout(mode) pool = RecordingPool.last - bundle_path, reporter = pool.init_kwargs['initargs'] + bundle_path, reporter, task_pids = pool.init_kwargs['initargs'] + # one shared slot per task id, where each worker records its pid (KI-CLI-37) + assert len(task_pids) == len(mode.filepaths) + 1 and task_pids is mode.task_pids # the bundle travels as a path, so the payload written to each spawned child stays small (KI-CLI-32) assert isinstance(bundle_path, str) and len(pickle.dumps(bundle_path)) < 1024 assert not os.path.exists(bundle_path) # its staging directory goes with the pool @@ -195,7 +203,7 @@ def test_bundle_reaches_workers_once_not_per_task(): def test_hashed_scan_through_process_pool(tmp_path: Path): - # the combination that used to hang: HashedSecretEngine registered, run in a real spawn pool + # the combination that used to hang: HashedSecretEngine registered, run in a real process pool shutil.copy('tests/fixtures/1.py', tmp_path / '1.py') config = Config() config.set_workdir(str(tmp_path)) @@ -279,3 +287,190 @@ def fake_getsize(path): assert mode.files[str(victim)].skip_reason == 'unreadable:EACCES' finally: mode.dispose() + + +def _run_capturing(mode: CliScanMode, timeout: int = 60): + outcome = {} + + def target(): + try: + outcome['result'] = mode.run() + except BaseException as e: # the thread must report what run() raised + outcome['error'] = e + + thread = threading.Thread(target=target, daemon=True) + thread.start() + thread.join(timeout=timeout) + assert not thread.is_alive(), 'scan did not terminate' + return outcome + + +@pytest.mark.parametrize('start_method', ['forkserver', 'spawn', 'fork']) +# the scan runs in a thread here (_run_capturing), so fork warns about forking a multi-threaded process +@pytest.mark.filterwarnings('ignore:This process .* is multi-threaded:DeprecationWarning') +def test_worker_that_cannot_load_the_bundle_stops_the_scan(start_method): + # KI-DM-30: an initializer that raised killed the worker, the pool replaced it, the replacement failed the same + # way, and run() polled forever. Now the first task on such a worker fails with WorkerStartupError and run() stops. + def corrupted_pool(**kwargs): + with open(kwargs['initargs'][0], 'wb') as f: + f.write(b'not a pickle') + return pool_context(start_method, CliScanMode.worker_modules).Pool(**kwargs) + + config = _config('tests/fixtures/extless') + config.set_process_count(2) + mode = CliScanMode(config=config, pool_engine=corrupted_pool) + _mock_progress_bar(mode) + try: + outcome = _run_capturing(mode, timeout=120) + assert isinstance(outcome.get('error'), iscan_mode.WorkerStartupError) + assert 'UnpicklingError' in str(outcome['error']) + finally: + mode.dispose() + + +def test_bundle_deleted_before_workers_start_is_staged_again(monkeypatch): + # a temporary-file cleaner can delete the staged bundle; the scan re-stages it and the workers' retry succeeds + monkeypatch.setattr(iscan_mode, 'BUNDLE_RETRY_DELAY', 0.5) + + def pool_after_cleaner(**kwargs): + os.remove(kwargs['initargs'][0]) + return ThreadPool(**kwargs) + + config = _config('tests/fixtures/extless') + mode = CliScanMode(config=config, pool_engine=pool_after_cleaner) + _mock_progress_bar(mode) + try: + outcome = _run_capturing(mode) + assert 'error' not in outcome, outcome.get('error') + assert mode.stats.finished == 4 and mode.stats.failed_files == 0 + finally: + mode.dispose() + + +def test_initializer_never_raises(tmp_path: Path, monkeypatch): + monkeypatch.setattr(iscan_mode, 'BUNDLE_RETRY_DELAY', 0) + iscan_mode.init_worker(str(tmp_path / 'missing.pickle'), {}) + assert iscan_mode._worker_bundle is None + assert iscan_mode._worker_startup_error.startswith('FileNotFoundError') + with pytest.raises(iscan_mode.WorkerStartupError, match='could not load the analyzer bundle'): + iscan_mode.pool_wrapper(lambda *args: None, 1, 'any.py') + + +class WorkerKillingScanMode(CliScanMode): + @staticmethod + def _per_file_analyzer(bundle, file, task_id=None, task_reporter=None): # type: ignore + if file.endswith('/json'): + os.kill(os.getpid(), signal.SIGKILL) # what the OOM killer does + return CliScanMode._per_file_analyzer(bundle, file, task_id, task_reporter) + + +@pytest.mark.parametrize('ci_mode', [True, False]) +def test_killed_worker_fails_its_file_instead_of_hanging_the_scan(ci_mode): + # KI-CLI-37: the pool replaced the dead worker but never reported its task, and run() waited forever + config = _config('tests/fixtures/extless') + config.set_process_count(2) + config.set_ci_mode(ci_mode) + mode = WorkerKillingScanMode(config=config) + _mock_progress_bar(mode) + try: + outcome = _run_capturing(mode, timeout=120) + assert 'error' not in outcome, outcome.get('error') + _, errors, _ = outcome['result'] + killed = '/app/tests/fixtures/extless/json' + assert len(errors[killed]) == 1 and errors[killed][0].startswith('WorkerLostError: the worker process') + assert mode.stats.finished == 4 and mode.stats.failed_files == 1 + assert list(mode.lost_jobs) == [next(t for t, j in mode.file_jobs.items() if j.name == killed)] + finally: + mode.dispose() + + +def _sigint_handler_is_ignored(bundle, file, task_id=None, task_reporter=None): + return signal.getsignal(signal.SIGINT) is signal.SIG_IGN + + +@pytest.mark.parametrize('start_method', ['forkserver', 'spawn', 'fork']) +def test_pool_workers_leave_ctrl_c_to_the_parent(tmp_path: Path, start_method): + # init_worker is the only place a worker ignores SIGINT (KI-CLI-17): forkserver children inherit the server's + # default handler, fork children the parent's, and importing iscan_mode no longer changes a handler + # finalise earlier tests' pools now: collected while this pool starts the resource tracker, their semaphores + # would call into it reentrantly, which it refuses with a "might leak" warning + gc.collect() + ctx = pool_context(start_method, CliScanMode.worker_modules) + path = iscan_mode.stage_bundle(AnalyzerBundle(workdir=str(tmp_path)), str(tmp_path)) + with ctx.Pool(2, initializer=iscan_mode.init_worker, initargs=(path, None)) as pool: + answers = pool.starmap(iscan_mode.pool_wrapper, [(_sigint_handler_is_ignored, i, 'x') for i in range(4)]) + assert answers == [True] * 4 + + +def test_ci_mode_needs_no_manager_and_finds_the_same(): + results = {} + for ci_mode in (False, True): + config = _config('tests/fixtures/extless') + config.set_process_count(2) + config.set_ci_mode(ci_mode) + mode = CliScanMode(config=config) + _mock_progress_bar(mode) + try: + assert (mode._mp_manager is None) is ci_mode + findings, errors, _ = _run_with_timeout(mode, timeout=120) + results[ci_mode] = (sorted((f.file.path, f.start_offset, f.detection) for f in findings), errors) + assert mode.stats.finished == 4 + if ci_mode: + assert mode.stats.total_findings == len(findings) + finally: + mode.dispose() + assert results[True] == results[False] + + +FORKSERVER_STOP_SCRIPT = """ +import os, sys, time +from multiprocessing import forkserver +from deepsecrets.config import Config, Output +from deepsecrets.core.engines.regex import RegexEngine +from deepsecrets.core.rulesets.regex import RegexRulesetBuilder +from deepsecrets.core.utils.multiprocessing_setup import pool_context, start_manager, stop_forkserver +from deepsecrets.scan_modes.cli import CliScanMode + +config = Config() +config.set_workdir('tests/fixtures/extless') +config.engines.append(RegexEngine) +config.add_ruleset(RegexRulesetBuilder, ['tests/fixtures/regexes.json']) +config.output = Output(type='sarif', path='/tmp/unused.sarif') +config.set_process_count(2) +config.set_mp_context('forkserver') +config.set_ci_mode(True) +mode = CliScanMode(config=config) +mode.run() +mode.dispose() +server = forkserver._forkserver._forkserver_pid +print('stopped', stop_forkserver()) +try: + os.waitpid(server, os.WNOHANG) + print('reaped', False) +except ChildProcessError: + print('reaped', True) + +# another process the server started holds its alive pipe: the stop must not wait for it +extra = start_manager(pool_context('forkserver', [])) +server = forkserver._forkserver._forkserver_pid +began = time.monotonic() +print('stopped', stop_forkserver(timeout=0.5), 'fast' if time.monotonic() - began < 5 else 'slow') +extra.shutdown() +os.waitpid(server, 0) # exits with its last child +print('exited', True) +""" + + +@pytest.mark.skipif('forkserver' not in get_all_start_methods(), reason='no forkserver on this platform') +def test_stop_forkserver_accounts_the_workers_and_never_hangs(): + # KI-CLI-42: the server reaps the workers; left running, it outlived the scan as an orphan, so their CPU time + # never reached `time` or a harness that waits for the scanner (7x too little on the fixtures). In a subprocess, + # because other tests leave forkserver-started managers alive in this one. + env = dict(os.environ, PYTHONPATH=os.getcwd()) + out = subprocess.run( + [sys.executable, '-c', FORKSERVER_STOP_SCRIPT], env=env, capture_output=True, text=True, timeout=180 + ) + assert out.returncode == 0, out.stderr + assert out.stdout.split('\n')[-5:] == ['stopped True', 'reaped True', 'stopped False fast', 'exited True', ''] + + From 1124c76d0456adc7738b4936921dd898a10da3b9 Mon Sep 17 00:00:00 2001 From: Nikolai Khechumov Date: Mon, 28 Sep 2026 09:29:04 +0000 Subject: [PATCH 07/22] Answer file-path-only scoring rules once per file SEM_VAR_FILE_PATHS reads nothing but the file path, so its answer is the same for every variable of a file; the evaluator now remembers it per rule. --- .../core/helpers/variable_evaluator.py | 22 +++++++++++-- .../core/model/rules/variable_scoring.py | 6 ++++ tests/core/helpers/test_variable_evaluator.py | 31 +++++++++++++++++++ 3 files changed, 56 insertions(+), 3 deletions(-) diff --git a/deepsecrets/core/helpers/variable_evaluator.py b/deepsecrets/core/helpers/variable_evaluator.py index 95d79b2..8649ea4 100644 --- a/deepsecrets/core/helpers/variable_evaluator.py +++ b/deepsecrets/core/helpers/variable_evaluator.py @@ -1,5 +1,5 @@ from dataclasses import dataclass, field -from typing import List, Union +from typing import Dict, List, Optional, Tuple, Union from deepsecrets.core.helpers.entropy import EntropyHelper from deepsecrets.core.model.rules.variable_scoring import VariableScoringRule from deepsecrets.core.model.semantic import Context, Variable @@ -41,6 +41,22 @@ class VariableEvaluator: def __init__(self, rules: List[VariableScoringRule]) -> None: self.rules = rules + # A rule that reads nothing but the file path (SEM_VAR_FILE_PATHS) gives one answer per file, and a scan + # evaluates a file's variables together, so each such rule remembers its last path and answer. + self._path_only = [rule.reads_only_filepath() for rule in rules] + self._last_path_answer: Dict[int, Tuple[Optional[str], bool]] = {} + + def _fires(self, index: int, rule: VariableScoringRule, context: Context) -> bool: + if not self._path_only[index]: + return rule.match_by_context(context) + + last = self._last_path_answer.get(index) + if last is not None and last[0] == context.filepath: + return last[1] + + fired = rule.match_by_context(context) + self._last_path_answer[index] = (context.filepath, fired) + return fired def calculate_entropy_score(self, entropy: float) -> float: if entropy == 0: @@ -60,8 +76,8 @@ def evaluate(self, variable: Union[Variable | Context]) -> EvaluationResult: naming_and_content_score = 0 matched_rules = [] - for rule in self.rules: - fired = rule.match_by_context(context) + for index, rule in enumerate(self.rules): + fired = self._fires(index, rule, context) if fired: naming_and_content_score += rule.score matched_rules.append(rule.id) diff --git a/deepsecrets/core/model/rules/variable_scoring.py b/deepsecrets/core/model/rules/variable_scoring.py index 15d259e..ec3d9ca 100644 --- a/deepsecrets/core/model/rules/variable_scoring.py +++ b/deepsecrets/core/model/rules/variable_scoring.py @@ -79,6 +79,12 @@ def build_numeric(cls, values: Dict) -> Dict: return values + def reads_only_filepath(self) -> bool: + """Whether the answer depends on the file path alone, and so is the same for every variable in a file.""" + return self.target == Target.FILEPATH and all( + condition.target == Target.FILEPATH for condition in (*self.when, *self.unless) + ) + def _get_content_for_matching(self, context: Context): field = target_to_fields.get(self.target) return getattr(context, field) diff --git a/tests/core/helpers/test_variable_evaluator.py b/tests/core/helpers/test_variable_evaluator.py index 7042b8d..53d89a0 100644 --- a/tests/core/helpers/test_variable_evaluator.py +++ b/tests/core/helpers/test_variable_evaluator.py @@ -229,3 +229,34 @@ def test_18(variable_scoring_rules): result: EvaluationResult = ve.evaluate(ctx) assert result.is_dangerous is True assert result.export_confidence >= 6 + + +def test_path_only_rule_is_matched_once_per_file(variable_scoring_rules, monkeypatch): + # SEM_VAR_FILE_PATHS reads only the path, the same for every variable in a file; its `.*`-led pattern cost up to + # 70 us per variable on long paths + path_rules = [rule for rule in variable_scoring_rules if rule.reads_only_filepath()] + assert [rule.id for rule in path_rules] == ['SEM_VAR_FILE_PATHS'] + calls = [] + original = type(path_rules[0]).match_by_context + + def counting(self, context): + if self.reads_only_filepath(): + calls.append(context.filepath) + return original(self, context) + + monkeypatch.setattr(type(path_rules[0]), 'match_by_context', counting) + ve = VariableEvaluator(variable_scoring_rules) + for name in ('db_password', 'api_token', 'client_secret', 'auth_key'): + ve.evaluate(Context(name=name, value='vhpn6mbsvhpn6mbsvhpn6mbs', filepath='src/app/settings.py')) + assert calls == ['src/app/settings.py'] + + +def test_path_answer_follows_the_path(variable_scoring_rules): + # a long-lived evaluator (the research scripts keep one) sees paths change back and forth + ve = VariableEvaluator(variable_scoring_rules) + fresh = lambda ctx: VariableEvaluator(variable_scoring_rules).evaluate(ctx) # noqa: E731 + for filepath in ('src/app.py', 'src/tests/app.py', 'src/app.py', 'web/bundle.min.js', 'web/bundle.min.js'): + ctx = Context(name='api_token', value='vhpn6mbsvhpn6mbsvhpn6mbs', filepath=filepath) + cached, expected = ve.evaluate(ctx), fresh(ctx) + assert cached.matched_rules == expected.matched_rules, filepath + assert ('SEM_VAR_FILE_PATHS' in cached.matched_rules) == (filepath != 'src/app.py'), filepath From 8470188f0c7c841f5adb49a6ad289f7cb0ad6d72 Mon Sep 17 00:00:00 2001 From: Nikolai Khechumov Date: Mon, 28 Sep 2026 09:29:04 +0000 Subject: [PATCH 08/22] Give files over --deep-max-size the fast analysis Files larger than --deep-max-size (default 250,000 bytes) get the regex rules and the cheap variable search without the language lexer, which cost most of the scan time on them and rarely found more. 0 analyses every file fully. The hashed engine keeps the lexer on every file. --report-diagnostics records each file's depth. --- README.md | 1 + deepsecrets/cli.py | 19 +++++- deepsecrets/config.py | 16 +++++ deepsecrets/core/model/internal/processing.py | 6 ++ deepsecrets/core/model/response/dojo_sarif.py | 2 + deepsecrets/core/modes/iscan_mode.py | 3 + deepsecrets/scan_modes/cli.py | 18 +++++- tests/cli/test_cli.py | 11 ++++ tests/output/test_sarif_diagnostics.py | 16 +++++ .../scan_modes/test_scan_mode_regressions.py | 62 ++++++++++++++++++- 10 files changed, 150 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 05d42f1..80fab4a 100644 --- a/README.md +++ b/README.md @@ -64,6 +64,7 @@ The `--help` command is always ready to guide you, but here are the key flags yo * `--regex-rules /path/to/rules.json`: Supply your own custom regex ruleset. * `--hashed-values /path/to/hashes.json`: Provide a list of pre-hashed known production secrets to search for them securely. * `--excluded-paths /path/to/exclusions.json`: Override or extend the default paths ignored during scanning. +* `--deep-max-size BYTES`: Files larger than this (default 250,000 bytes) get the fast analysis: the regex rules and the variable search, without the language lexer. Big files are mostly generated or minified, the lexer costs most of the scan time on them and rarely finds more there. `0` gives every file the full analysis. * `--disable-masking`: Keep potential secrets unmasked in the output report *(see caution below)*. * `--ci` / `--no-ci`: Turn CI mode on or off. CI mode prints a plain progress line every 30 seconds instead of the live progress display. It turns on by itself inside CI services (GitHub Actions, GitLab CI, Jenkins and others) and whenever the output is not a terminal. diff --git a/deepsecrets/cli.py b/deepsecrets/cli.py index 898b1a0..14ee4b5 100644 --- a/deepsecrets/cli.py +++ b/deepsecrets/cli.py @@ -6,7 +6,14 @@ from jschema_to_python.to_json import to_json from deepsecrets import MODULE_NAME, console -from deepsecrets.config import SCANNER_VERSION, SCANNER_VERSION_NUMERIC, Config, config, Output +from deepsecrets.config import ( + DEFAULT_DEEP_MAX_SIZE, + SCANNER_VERSION, + SCANNER_VERSION_NUMERIC, + Config, + config, + Output, +) from deepsecrets.core.engines.hashed_secret import HashedSecretEngine from deepsecrets.core.engines.regex import RegexEngine from deepsecrets.core.engines.semantic import SemanticEngine @@ -153,6 +160,15 @@ def _build_argparser(self) -> None: default=['built-in'], ) + parser.add_argument( + '--deep-max-size', + type=int, + default=DEFAULT_DEEP_MAX_SIZE, + help='Files larger than this many bytes get the fast analysis: the regex rules and the variable search,\n' + 'without the language lexer, which costs most of the scan time on big files and rarely finds more there.\n' + f'Default: {DEFAULT_DEEP_MAX_SIZE}. 0 gives every file the full analysis.\n', + ) + parser.add_argument( '--false-findings', nargs='*', @@ -264,6 +280,7 @@ def parse_arguments(self) -> None: config.set_workdir(user_args.target_dir) config.set_oneshot_path(user_args.oneshot) config.set_max_file_size(user_args.max_file_size) + config.set_deep_max_size(user_args.deep_max_size) config.set_process_count(user_args.process_count) config.set_mp_context(user_args.multiprocessing_context or default_start_method()) config.set_ci_mode(is_ci_environment() if user_args.ci is None else user_args.ci) diff --git a/deepsecrets/config.py b/deepsecrets/config.py index 3efab66..9e352a4 100644 --- a/deepsecrets/config.py +++ b/deepsecrets/config.py @@ -11,6 +11,17 @@ FALLBACK_PROCESS_COUNT = 4 +# files larger than this many bytes get the shallow analysis: the regex rules and the cheap variable search, no lexer. +# On SecretBench the lexer found 12 of its 21 unique secrets in smaller files and the other 9 only in files of 1 MB +# or more (minified JavaScript, one YAML file), while files over 250 KB took two thirds of its time. +DEFAULT_DEEP_MAX_SIZE = 250_000 + + +def deep_analysis(size: int, deep_max_size: int) -> bool: + """Whether a file of `size` bytes gets the full analysis, lexer included. `deep_max_size` 0 or less: every file.""" + return deep_max_size <= 0 or size <= deep_max_size + + SCANNER_NAME = "DeepSecrets" SCANNER_VERSION = "2.1.1" SCANNER_VERSION_NUMERIC = [int(subver) for subver in SCANNER_VERSION.split('.')] @@ -29,6 +40,8 @@ class Config: workdir_path: str oneshot_path: str max_file_size: int = 0 # 0 means no limit + # files above this many bytes get no lexer (see deep_analysis above); 0 means every file gets it + deep_max_size: int = DEFAULT_DEEP_MAX_SIZE mp_context: str = default_start_method() engines: List[Type] = [] rulesets: Dict[Type, List[str]] = {} @@ -90,6 +103,9 @@ def set_workdir(self, path: str) -> None: def set_max_file_size(self, size: int) -> None: self.max_file_size = size + def set_deep_max_size(self, size: int) -> None: + self.deep_max_size = size + def set_mp_context(self, context: str) -> None: self.mp_context = context diff --git a/deepsecrets/core/model/internal/processing.py b/deepsecrets/core/model/internal/processing.py index 06f2399..73cc765 100644 --- a/deepsecrets/core/model/internal/processing.py +++ b/deepsecrets/core/model/internal/processing.py @@ -2,6 +2,7 @@ from typing import Dict, List, Optional, Sequence from deepsecrets.core.model.file import File from deepsecrets.core.model.finding import Finding +from deepsecrets.config import DEFAULT_DEEP_MAX_SIZE from deepsecrets.core.model.rules.rule import Rule @@ -19,6 +20,8 @@ class AnalyzerBundle: # ruleset name -> rules rulesets: Dict[str, Sequence[Rule]] = field(default_factory=dict) benchmarking_mode: bool = False + # see deep_analysis + deep_max_size: int = DEFAULT_DEEP_MAX_SIZE @dataclass @@ -31,6 +34,9 @@ class PerFileAnalysisResult: processing_time_ms: float = field(default=0.0) # 'ok', 'empty' (0 bytes) or 'unreadable' (the file could not be opened) status: str = field(default='ok') + # 'deep' (lexer included) or 'shallow' (larger than deep_max_size: regex rules and the cheap variable search); + # None for a file that was never analysed (it could not be opened, or it was empty) + depth: Optional[str] = field(default=None) # ONLY FOR BENCHMARKING MODE _file: Optional[File] = None diff --git a/deepsecrets/core/model/response/dojo_sarif.py b/deepsecrets/core/model/response/dojo_sarif.py index b8ce1f0..dfc9847 100644 --- a/deepsecrets/core/model/response/dojo_sarif.py +++ b/deepsecrets/core/model/response/dojo_sarif.py @@ -197,6 +197,8 @@ def _add_diagnostics(self, run: Run) -> None: properties = {'status': 'error' if outcome.status == 'scheduled' else outcome.status} if outcome.collected: properties['scanTimeMs'] = outcome.time_ms + if outcome.depth: + properties['depth'] = outcome.depth if outcome.skip_reason: properties['skipReason'] = outcome.skip_reason try: diff --git a/deepsecrets/core/modes/iscan_mode.py b/deepsecrets/core/modes/iscan_mode.py index 2ee7bb6..ce8aede 100644 --- a/deepsecrets/core/modes/iscan_mode.py +++ b/deepsecrets/core/modes/iscan_mode.py @@ -74,6 +74,8 @@ class FileOutcome: time_ms: float = 0.0 collected: bool = False errors: List[str] = field(default_factory=list) + # 'deep' or 'shallow' once collected (the size tier, see config.deep_analysis) + depth: Optional[str] = None @property def time_seconds(self) -> int: @@ -448,6 +450,7 @@ def run(self) -> Tuple[List[Finding], Dict[str, List[str]], Dict[str, int]]: self._oneshot_file = analysis_result._file outcome.collected = True outcome.time_ms = analysis_result.processing_time_ms + outcome.depth = analysis_result.depth outcome.errors = analysis_result.errors # a file that logged an error is reported as failed even when the analysis returned status = analysis_result.status diff --git a/deepsecrets/scan_modes/cli.py b/deepsecrets/scan_modes/cli.py index 065ba01..e4d682e 100644 --- a/deepsecrets/scan_modes/cli.py +++ b/deepsecrets/scan_modes/cli.py @@ -9,6 +9,7 @@ from deepsecrets.core.engines.semantic import SemanticEngine from deepsecrets.core.model.file import File from deepsecrets.core.modes.iscan_mode import ScanMode +from deepsecrets.config import deep_analysis from deepsecrets.core.model.internal.processing import AnalyzerBundle, PerFileAnalysisResult from deepsecrets.core.rulesets.hashed_secrets import HashedSecretsRulesetBuilder from deepsecrets.core.rulesets.regex import RegexRulesetBuilder @@ -46,7 +47,12 @@ def prepare_for_scan(self) -> None: self.rulesets[builder.ruleset_name] = builder.rules def analyzer_bundle(self) -> AnalyzerBundle: - return replace(super().analyzer_bundle(), engines=self.engines_enabled, rulesets=self.rulesets) + return replace( + super().analyzer_bundle(), + engines=self.engines_enabled, + rulesets=self.rulesets, + deep_max_size=self.config.deep_max_size, + ) @staticmethod def _per_file_analyzer(bundle: AnalyzerBundle, file: Any, task_id: Optional[int] = None, task_reporter: Optional[Any] = None) -> PerFileAnalysisResult: # type: ignore @@ -95,6 +101,13 @@ def __finalize(result: PerFileAnalysisResult): file_analyzer = FileAnalyzer(file) file_analyzer.attach_global_task_reporter(task_reporter=task_reporter, task_id=task_id) + # the size tier: the lexer only for files up to deep_max_size bytes + try: + deep = deep_analysis(os.path.getsize(file.path), bundle.deep_max_size) + except OSError: + deep = True + result.depth = 'deep' if deep else 'shallow' + fct = FullContentTokenizer() cheap_var_search = CheapVarSearchTokenizer() lex = LexerTokenizer(deep_token_inspection=True) @@ -114,13 +127,14 @@ def __finalize(result: PerFileAnalysisResult): hashed_secret_engine = HashedSecretEngine( ruleset=bundle.rulesets.get(HashedSecretsRulesetBuilder.ruleset_name, []) ) + # hashed values are compared against lexer tokens, so this engine keeps the lexer on every file file_analyzer.add_engine(hashed_secret_engine, [lex]) if eng == SemanticEngine.name: semantic_engine = SemanticEngine( regex_engine, ruleset=bundle.rulesets.get(VariableScoringRulesetBuilder.ruleset_name, []) ) - file_analyzer.add_engine(semantic_engine, [lex, cheap_var_search]) + file_analyzer.add_engine(semantic_engine, [lex, cheap_var_search] if deep else [cheap_var_search]) try: result.findings = file_analyzer.process() diff --git a/tests/cli/test_cli.py b/tests/cli/test_cli.py index 2f8c814..2d597a4 100644 --- a/tests/cli/test_cli.py +++ b/tests/cli/test_cli.py @@ -1,6 +1,7 @@ import pytest from deepsecrets.cli import DeepSecretsCliTool +from deepsecrets.config import DEFAULT_DEEP_MAX_SIZE from deepsecrets.core.engines.hashed_secret import HashedSecretEngine from deepsecrets.core.engines.regex import RegexEngine from deepsecrets.core.rulesets.hashed_secrets import HashedSecretsRulesetBuilder @@ -122,3 +123,13 @@ def test_multiprocessing_context_defaults_to_the_platform_start_method(monkeypat tool.parse_arguments() assert config.mp_context == expected + + +@pytest.mark.parametrize('extra, expected', [([], DEFAULT_DEEP_MAX_SIZE), (['--deep-max-size', '0'], 0)]) +def test_deep_max_size_flag(monkeypatch, extra, expected): + tool = DeepSecretsCliTool(args=['', '--target-dir', '/app/tests/fixtures/', '--outfile', '/tmp/x.sarif'] + extra) + config = tool.get_current_config() + monkeypatch.setattr(config, 'deep_max_size', -1) + tool.parse_arguments() + + assert config.deep_max_size == expected diff --git a/tests/output/test_sarif_diagnostics.py b/tests/output/test_sarif_diagnostics.py index 1b72d61..3455ec5 100644 --- a/tests/output/test_sarif_diagnostics.py +++ b/tests/output/test_sarif_diagnostics.py @@ -64,3 +64,19 @@ def test_error_list_is_per_file(): assert get_error_list() == [] # the returned list is a copy, not the live buffer assert errors == ['first file failed'] + + +def test_diagnostics_say_which_files_got_no_lexer(target, tmp_path): + run = scan(target, tmp_path / 'report.sarif', '--report-diagnostics', '--deep-max-size', '10') + artifacts = {a['location']['uri']: a for a in run['artifacts']} + assert artifacts['src/settings.py']['properties']['depth'] == 'shallow' + assert 'depth' not in artifacts['node_modules/pkg/index.js']['properties'] # skipped, never analysed + # the variable search still reads a small assignment without the lexer + assert [r['locations'][0]['physicalLocation']['artifactLocation']['uri'] for r in run['results']] == [ + 'src/settings.py' + ] + + run = scan(target, tmp_path / 'report.sarif', '--report-diagnostics') + assert {a['location']['uri']: a for a in run['artifacts']}['src/settings.py']['properties']['depth'] == 'deep' + + diff --git a/tests/scan_modes/test_scan_mode_regressions.py b/tests/scan_modes/test_scan_mode_regressions.py index e0a90be..d2114f7 100644 --- a/tests/scan_modes/test_scan_mode_regressions.py +++ b/tests/scan_modes/test_scan_mode_regressions.py @@ -23,6 +23,8 @@ from deepsecrets.core.model.rules.regex import RegexRule from deepsecrets.core.rulesets.hashed_secrets import HashedSecretsRulesetBuilder from deepsecrets.core.rulesets.regex import RegexRulesetBuilder +from deepsecrets.core.tokenizers.lexer import LexerTokenizer +from deepsecrets.core.rulesets.variable_scoring import VariableScoringRulesetBuilder from deepsecrets.core.utils.fs import get_path_inside_package from deepsecrets.core.modes import iscan_mode from deepsecrets.core.utils.multiprocessing_setup import pool_context @@ -142,11 +144,18 @@ def test_analyzer_bundle_carries_what_workers_read(): try: bundle = mode.analyzer_bundle() - assert {f.name for f in fields(bundle)} == {'workdir', 'engines', 'rulesets', 'benchmarking_mode'} + assert {f.name for f in fields(bundle)} == { + 'workdir', + 'engines', + 'rulesets', + 'benchmarking_mode', + 'deep_max_size', + } assert bundle.workdir == '/app/tests/fixtures/extless' assert bundle.engines == {'regex': True} assert list(bundle.rulesets) == ['regex'] assert bundle.benchmarking_mode is False + assert bundle.deep_max_size == config.deep_max_size with pytest.raises(FrozenInstanceError): setattr(bundle, 'workdir', '/elsewhere') assert pickle.loads(pickle.dumps(bundle)) == bundle @@ -474,3 +483,54 @@ def test_stop_forkserver_accounts_the_workers_and_never_hangs(): assert out.stdout.split('\n')[-5:] == ['stopped True', 'reaped True', 'stopped False fast', 'exited True', ''] +@pytest.mark.parametrize('deep_max_size, lexed', [(0, True), (20_233, True), (20_232, False)]) +def test_size_tier_gives_files_over_deep_max_size_no_lexer(monkeypatch, deep_max_size, lexed): + # tests/fixtures/1.py is 20,233 bytes; the tier compares its size on disk with deep_max_size, inclusive + assert os.path.getsize('tests/fixtures/1.py') == 20_233 + tokenized = [] + + class RecordingLexer(LexerTokenizer): + def tokenize(self, file, *args, **kwargs): + tokenized.append(file.path) + return super().tokenize(file, *args, **kwargs) + + monkeypatch.setattr('deepsecrets.scan_modes.cli.LexerTokenizer', RecordingLexer) + scoring = VariableScoringRulesetBuilder().with_rules_from_file( + get_path_inside_package('rules/variable_scoring_rules.json') + ) + bundle = AnalyzerBundle( + workdir='/app/tests/fixtures', + engines={'semantic': True}, + rulesets={'variable_scoring': scoring.rules}, + deep_max_size=deep_max_size, + ) + result = CliScanMode._per_file_analyzer(bundle, '/app/tests/fixtures/1.py', 1, {}) + + assert result.status == 'ok' + assert result.depth == ('deep' if lexed else 'shallow') + assert bool(tokenized) is lexed + + +def test_hashed_values_keep_the_lexer_on_a_shallow_file(): + # hashed values are compared against lexer tokens: the tier must not cost --hashed-values its findings + builder = HashedSecretsRulesetBuilder().with_rules_from_file('tests/fixtures/hashed_secrets.json') + bundle = AnalyzerBundle( + workdir='/app/tests/fixtures', engines={'hashed': True}, rulesets={'hashed': builder.rules}, deep_max_size=1 + ) + result = CliScanMode._per_file_analyzer(bundle, '/app/tests/fixtures/1.py', 1, {}) + + assert result.depth == 'shallow' + assert [finding.detection for finding in result.findings] == [HASHED_SECRET] + + +def test_run_records_each_files_depth(): + config = _config('tests/fixtures/extless') + config.set_deep_max_size(100) # json (87 bytes) and yaml (53) stay deep; ini (301) and radius (17,883) do not + config.set_ci_mode(True) + mode = CliScanMode(config=config, pool_engine=ThreadPool) + try: + _run_with_timeout(mode) + depths = {os.path.basename(path): outcome.depth for path, outcome in mode.files.items()} + assert depths == {'json': 'deep', 'yaml': 'deep', 'ini': 'shallow', 'radius': 'shallow'} + finally: + mode.dispose() From 03032248da61ae0c88ea8dd06100bdbca83f2184 Mon Sep 17 00:00:00 2001 From: Nikolai Khechumov Date: Mon, 28 Sep 2026 09:29:04 +0000 Subject: [PATCH 09/22] Report S106 only under password- and secret-like names S106 reported a low-entropy value under any secret-like name. A value of 3 bits per character or less is now reported only when a rule marked allows_low_entropy fires on the name (SEM_VAR_HUMAN_CHOSEN_SECRET_NAMES: passwords and secrets people choose), not for issued keys and tokens. The diagnostics tracer predicts the rule the same way. --- deepsecrets/core/engines/semantic.py | 3 +- .../core/helpers/variable_evaluator.py | 6 ++++ .../core/model/rules/variable_scoring.py | 2 ++ deepsecrets/diagnostics/trace.py | 22 +++++++++++++-- deepsecrets/rules/variable_scoring_rules.json | 8 ++++++ tests/core/engines/semantic/test_semantic.py | 24 +++++++++++++++- .../helpers/test_variable_scoring_cases.py | 28 +++++++++++++++++++ 7 files changed, 89 insertions(+), 4 deletions(-) diff --git a/deepsecrets/core/engines/semantic.py b/deepsecrets/core/engines/semantic.py index 56b7f78..d94abce 100644 --- a/deepsecrets/core/engines/semantic.py +++ b/deepsecrets/core/engines/semantic.py @@ -91,7 +91,8 @@ def search(self, token: Token) -> List[Finding]: internal_score={'var': token.semantic.name} | evaluation_result.summary(), ) ) - else: + elif evaluation_result.allows_low_entropy: + # 3 bits or less: a password someone chose, or else a placeholder, an id or a map key findings.append( Finding( detection=token.content, diff --git a/deepsecrets/core/helpers/variable_evaluator.py b/deepsecrets/core/helpers/variable_evaluator.py index 8649ea4..85bdc5a 100644 --- a/deepsecrets/core/helpers/variable_evaluator.py +++ b/deepsecrets/core/helpers/variable_evaluator.py @@ -19,6 +19,9 @@ class EvaluationResult: export_confidence: int = 0 + # a fired rule allows a value with no entropy score to be reported: people choose passwords, not API keys + allows_low_entropy: bool = False + # < 3: 0 # 3-4: 0 -> 35 @@ -75,12 +78,14 @@ def evaluate(self, variable: Union[Variable | Context]) -> EvaluationResult: naming_and_content_score = 0 matched_rules = [] + allows_low_entropy = False for index, rule in enumerate(self.rules): fired = self._fires(index, rule, context) if fired: naming_and_content_score += rule.score matched_rules.append(rule.id) + allows_low_entropy = allows_low_entropy or rule.allows_low_entropy if naming_and_content_score <= HOPELESS_THRESHOLD: return EvaluationResult( @@ -113,6 +118,7 @@ def evaluate(self, variable: Union[Variable | Context]) -> EvaluationResult: matched_rules=matched_rules, nonsence_value_score=nonsense_value_score, is_dangerous=naming_and_content_score > DANGER_THRESHOLD, + allows_low_entropy=allows_low_entropy, ) result.export_confidence = self.confidence_from_evaluation_result(result) diff --git a/deepsecrets/core/model/rules/variable_scoring.py b/deepsecrets/core/model/rules/variable_scoring.py index ec3d9ca..90459f3 100644 --- a/deepsecrets/core/model/rules/variable_scoring.py +++ b/deepsecrets/core/model/rules/variable_scoring.py @@ -67,6 +67,8 @@ class VariableScoringRule(RegexRule): # the rule can only fire when every `when` condition matches and no `unless` condition does when: List[ScoringCondition] = Field(default=[]) unless: List[ScoringCondition] = Field(default=[]) + # when the rule fires, a dangerous variable is reported even if its value earns no entropy score (S106) + allows_low_entropy: bool = False def _is_threshold_type(self): return self.threshold is not None and self.method is not None diff --git a/deepsecrets/diagnostics/trace.py b/deepsecrets/diagnostics/trace.py index db9469b..e67be3c 100644 --- a/deepsecrets/diagnostics/trace.py +++ b/deepsecrets/diagnostics/trace.py @@ -150,6 +150,8 @@ def _evaluate(evaluator: VariableEvaluator, scoring_rules: list, variable) -> di result = evaluator.evaluate(variable) counterfactual = {} for entry in fired: + if not entry['score']: + continue without = VariableEvaluator([r for r in scoring_rules if r.id != entry['id']]) counterfactual[entry['id']] = without.evaluate(variable).is_dangerous return { @@ -162,11 +164,21 @@ def _evaluate(evaluator: VariableEvaluator, scoring_rules: list, variable) -> di 'naturalness': round(1 - result.nonsence_value_score, 3) if not hopeless else None, 'dangerous': result.is_dangerous, 'confidence': result.export_confidence, - 'rule_emitted': ('S105' if result.entropy_score > 0 else 'S106') if result.is_dangerous else None, + 'allows_low_entropy': result.allows_low_entropy, + 'rule_emitted': _rule_emitted(result), 'dangerous_without': counterfactual, } +def _rule_emitted(result) -> Optional[str]: + """The rule SemanticEngine.search reports for an evaluated variable, or None.""" + if not result.is_dangerous: + return None + if result.entropy_score > 0: + return 'S105' + return 'S106' if result.allows_low_entropy else None + + def _gate(token: Token, file: File) -> Optional[str]: """SemanticEngine.search's early returns, in its order.""" if token.length == file.length: @@ -373,7 +385,7 @@ def _verdict(t: dict) -> dict: } if all(ev.get('gate') for ev in evaluations): return {'stage': 'gate', 'component': 'semantic_engine', 'detail': evaluations[0]['gate']} - if any(ev.get('dangerous') for ev in evaluations): + if any(ev.get('rule_emitted') for ev in evaluations): if t['production']['cache_skipped']: return { 'stage': 'emission', @@ -381,6 +393,12 @@ def _verdict(t: dict) -> dict: 'detail': 'value cache: seen earlier with no finding', } return {'stage': 'emission', 'component': 'finding_merger', 'detail': 'dangerous but not reported'} + if any(ev.get('dangerous') for ev in evaluations): + return { + 'stage': 'evaluation', + 'component': 'semantic_engine', + 'detail': 'no entropy score, and no allows_low_entropy rule fired on the name (S106)', + } return {'stage': 'evaluation', 'component': 'variable_evaluator', 'detail': 'scored as not dangerous'} diff --git a/deepsecrets/rules/variable_scoring_rules.json b/deepsecrets/rules/variable_scoring_rules.json index bfb4951..0e4a7b3 100644 --- a/deepsecrets/rules/variable_scoring_rules.json +++ b/deepsecrets/rules/variable_scoring_rules.json @@ -154,5 +154,13 @@ "target": "NAME_NORMALIZED", "pattern": "^key$", "score": 2 + }, + { + "id": "SEM_VAR_HUMAN_CHOSEN_SECRET_NAMES", + "description": "Scores nothing: marks the names of secrets people choose (passwords, secret keys), the only ones legitimately low in entropy. A dangerous variable whose value earns no entropy score (3 bits or less) is reported as S106 only when an allows_low_entropy rule fired; issued keys and tokens are random, so under their names such a value is a placeholder, an id or a map key. The password half of SEM_VAR_HIGH_CONFIDENCE_FULLNAME_2.", + "target": "NAME_NORMALIZED", + "pattern": "(passw|pwd|pass$|secret)", + "score": 0, + "allows_low_entropy": true } ] \ No newline at end of file diff --git a/tests/core/engines/semantic/test_semantic.py b/tests/core/engines/semantic/test_semantic.py index 1601aed..21b1085 100644 --- a/tests/core/engines/semantic/test_semantic.py +++ b/tests/core/engines/semantic/test_semantic.py @@ -49,7 +49,9 @@ def test_toml_1(file: File): def test_toml_2(file: File): findings, tokens, _ = semantic_case(file) assert len(tokens) == 13 - assert len(findings) == 4 + # SERVICE_OAUTH, STT_KEY and TTS_KEY repeat one 8-character block (3.0 bits): too low in entropy for a key or + # token name, so no S106 + assert [f.internal_score['var'] for f in findings] == ['S3_SECRET'] @pytest.mark.fixture_file_path('2.sh') @@ -124,3 +126,23 @@ def test_6_php(file: File): def test_ini_in_txt(file: File): findings, tokens, vars = semantic_case(file) assert len(findings) == 1 + + +def test_low_entropy_values_are_reported_only_under_names_people_choose(tmp_path): + path = tmp_path / 'settings.py' + path.write_text( + 'password = "nacc6opq"\n' + 'db_pass = "nacc6opq"\n' + 'client_secret = "nacc6opq"\n' + 'api_key = "nacc6opq"\n' + 'auth_token = "nacc6opq"\n' + 'STT_KEY = "vhpn6mbsvhpn6mbs"\n' + 'refresh_token = "Hk3vQ9ZpL2mXw7Rt"\n' + ) + findings, _, _ = semantic_case(File(path=str(path), relative_path='settings.py')) + assert sorted((f.internal_score['var'], f.final_rule.id) for f in findings) == [ + ('client_secret', 'S106'), + ('db_pass', 'S106'), + ('password', 'S106'), + ('refresh_token', 'S105'), + ] diff --git a/tests/core/helpers/test_variable_scoring_cases.py b/tests/core/helpers/test_variable_scoring_cases.py index 74ac72c..c8ff6b6 100644 --- a/tests/core/helpers/test_variable_scoring_cases.py +++ b/tests/core/helpers/test_variable_scoring_cases.py @@ -197,3 +197,31 @@ def test_word_valued_password_is_reported_at_low_confidence(variable_scoring_rul result = VariableEvaluator(variable_scoring_rules).evaluate(Context(name=name, value=value, filepath=filepath)) assert result.is_dangerous is True, result assert result.export_confidence < 3, result + + +LOW_ENTROPY = 'nacc6opq' # 8 characters, 2.75 bits: earns no entropy score + + +@pytest.mark.parametrize( + 'name, allowed', + [ + pytest.param('password', True, id='password'), + pytest.param('DB_PWD', True, id='DB_PWD'), + pytest.param('smtp_passwd', True, id='smtp_passwd'), + pytest.param('db_pass', True, id='db_pass'), + pytest.param('client_secret', True, id='client_secret'), + pytest.param('SECRET_KEY', True, id='SECRET_KEY'), + pytest.param('api_key', False, id='api_key'), + pytest.param('auth_token', False, id='auth_token'), + pytest.param('SERVICE_OAUTH', False, id='SERVICE_OAUTH'), + pytest.param('private_key', False, id='private_key'), + pytest.param('kube_kubeconfig_token', False, id='kube_kubeconfig_token'), + ], +) +def test_only_names_people_choose_a_value_for_allow_low_entropy(variable_scoring_rules, name, allowed): + # issued keys and tokens are random: a value of 3 bits or less under their names is a placeholder, an id or a + # map key, so SemanticEngine reports S106 only when an allows_low_entropy rule fired + result = VariableEvaluator(variable_scoring_rules).evaluate(Context(name=name, value=LOW_ENTROPY, filepath=CODE)) + assert result.is_dangerous is True, result + assert result.entropy_score == 0, result + assert result.allows_low_entropy is allowed, result From 458cfbb493178286938efbe85e5273395f2575fa Mon Sep 17 00:00:00 2001 From: Nikolai Khechumov Date: Mon, 28 Sep 2026 09:29:04 +0000 Subject: [PATCH 10/22] Name merged findings deterministically; one severity for every tier - choose_final_rule breaks confidence ties by the rule's position in its ruleset, then its id, instead of the set order that varied with the hash seed (KI-DM-15). - Every confidence tier carries security-severity 10.00: severity is how bad a leak is and does not depend on confidence. --- deepsecrets/core/model/finding.py | 3 +- deepsecrets/core/model/response/dojo_sarif.py | 42 ++++++------------- deepsecrets/core/model/rules/rule.py | 3 ++ deepsecrets/core/rulesets/ibuilder.py | 1 + tests/core/model/test_finding.py | 20 +++++++++ 5 files changed, 39 insertions(+), 30 deletions(-) diff --git a/deepsecrets/core/model/finding.py b/deepsecrets/core/model/finding.py index 1f2f951..dca3d0e 100644 --- a/deepsecrets/core/model/finding.py +++ b/deepsecrets/core/model/finding.py @@ -73,7 +73,8 @@ def get_partial_fingerprint(self) -> str: return sha256(base.encode('utf-8')).hexdigest() def choose_final_rule(self) -> None: - self.final_rule = sorted(self.rules, key=lambda r: r.confidence, reverse=True)[0] + # highest confidence first; a tie goes to the earlier rule in its ruleset, then the id, never to set order + self.final_rule = min(self.rules, key=lambda r: (-r.confidence, r.rank, r.id)) def __hash__(self) -> int: # pragma: nocover if not self.file: diff --git a/deepsecrets/core/model/response/dojo_sarif.py b/deepsecrets/core/model/response/dojo_sarif.py index dfc9847..15f52fa 100644 --- a/deepsecrets/core/model/response/dojo_sarif.py +++ b/deepsecrets/core/model/response/dojo_sarif.py @@ -52,35 +52,19 @@ class DojoSarifResponseBuilder(BaseResponseBuilder): report: SarifLog - def _get_tier(self, confidence: int): - confidence_tiers = { - (9, float('inf')): { - 'suffix': '-VERY-HIGH', - 'precision': 'very-high', - 'severity': '10.00', - 'label': 'Very High', - }, - (6, 9): { - 'suffix': '-HIGH', - 'precision': 'high', - 'severity': '9.70', - 'label': 'High', - }, - (3, 6): { - 'suffix': '-MEDIUM', - 'precision': 'medium', - 'severity': '9.40', - 'label': 'Medium', - }, - (float('-inf'), 3): { - 'suffix': '-LOW', - 'precision': 'low', - 'severity': '9.10', - 'label': 'Low', - }, - } + # Severity is how bad a leak of the secret is, and it does not depend on how sure we are: every tier carries the + # same value, critical for GitHub (over 9.0), DefectDojo (9 or more) and GitLab alike. Confidence goes in the tier: + # the rule id suffix, GitHub's rule-level `precision`, and the result's `confidence` property. + SECURITY_SEVERITY = '10.00' + CONFIDENCE_TIERS = { + (9, float('inf')): {'suffix': '-VERY-HIGH', 'precision': 'very-high', 'label': 'Very High'}, + (6, 9): {'suffix': '-HIGH', 'precision': 'high', 'label': 'High'}, + (3, 6): {'suffix': '-MEDIUM', 'precision': 'medium', 'label': 'Medium'}, + (float('-inf'), 3): {'suffix': '-LOW', 'precision': 'low', 'label': 'Low'}, + } - for (start, end), value in confidence_tiers.items(): + def _get_tier(self, confidence: int): + for (start, end), value in self.CONFIDENCE_TIERS.items(): if start <= confidence < end: return value @@ -96,7 +80,7 @@ def _sarif_rule_meta_from_rule(self, rule: Rule) -> TierAwareSarifRuleMeta: id=f'{base_rule_id}{suffix}', payload={ 'shortDescription': {'text': f'{base_description} ({tier.get("label")} Confidence)'}, - 'properties': {'precision': tier.get("precision"), 'security-severity': tier.get("severity")}, + 'properties': {'precision': tier.get("precision"), 'security-severity': self.SECURITY_SEVERITY}, }, ) diff --git a/deepsecrets/core/model/rules/rule.py b/deepsecrets/core/model/rules/rule.py index bb5655f..eeb0dfd 100644 --- a/deepsecrets/core/model/rules/rule.py +++ b/deepsecrets/core/model/rules/rule.py @@ -11,6 +11,9 @@ class Rule(BaseModel): enabled: bool = Field(default=True) confidence: int = Field(default=10) is_dynamic_confidence: bool = Field(default=False) + # breaks confidence ties when merged findings pick a final rule: a lower rank wins. Ruleset builders number rules + # in file order; rules made in code (S105, S106, S107) keep the default and lose ties to a typed rule + rank: int = Field(default=1_000_000) applicable_file_patterns: List[re.Pattern] = Field(default=[]) model_config = ConfigDict(arbitrary_types_allowed=True) diff --git a/deepsecrets/core/rulesets/ibuilder.py b/deepsecrets/core/rulesets/ibuilder.py index 255ec8e..5266286 100644 --- a/deepsecrets/core/rulesets/ibuilder.py +++ b/deepsecrets/core/rulesets/ibuilder.py @@ -21,6 +21,7 @@ def with_rules_from_file(self, file: str) -> object: parsed_model: Rule = self.rule_model(**rule) if parsed_model.enabled is False: continue + parsed_model.rank = len(self.rules) self.rules.append(parsed_model) return self diff --git a/tests/core/model/test_finding.py b/tests/core/model/test_finding.py index edbfdc8..15e4b58 100644 --- a/tests/core/model/test_finding.py +++ b/tests/core/model/test_finding.py @@ -44,3 +44,23 @@ def test_1_finding(file: File, rule: Rule): assert new_finding.start_offset == TOKEN_SPAN[0] + FINDING_SPAN_INSIDE_TOKEN[0] assert new_finding.end_offset == TOKEN_SPAN[0] + FINDING_SPAN_INSIDE_TOKEN[1] + + +def test_final_rule_tie_goes_to_the_earlier_rule_whatever_the_order(): + # merged rules come out of a set, so their order depends on the hash seed (KI-DM-15) + typed = Rule(id='S18', confidence=10, rank=18) + generic = Rule(id='S26', confidence=10, rank=24) + semantic = Rule(id='S105', confidence=10) # made in code: default rank, loses ties to ruleset rules + weaker = Rule(id='S0', confidence=9, rank=0) + for rules in ([typed, generic, semantic, weaker], [semantic, weaker, generic, typed], [generic, typed]): + finding = Finding(detection='x', start_offset=0, end_offset=1, rules=list(rules)) + finding.choose_final_rule() + assert finding.final_rule.id == 'S18' + + +def test_ruleset_builder_ranks_rules_in_file_order(): + from deepsecrets.core.rulesets.regex import RegexRulesetBuilder + from deepsecrets.core.utils.fs import get_path_inside_package + + rules = RegexRulesetBuilder().with_rules_from_file(get_path_inside_package('rules/regexes.json')).rules + assert [r.rank for r in rules] == list(range(len(rules))) From 781b9a37e6df2221d318fa4d96dc492bde3739ca Mon Sep 17 00:00:00 2001 From: Nikolai Khechumov Date: Mon, 28 Sep 2026 09:29:04 +0000 Subject: [PATCH 11/22] Judge regex matches as candidates before reporting them A regex match is a candidate, judged in RegexEngine by its own scoring rules (rules/regex_candidate_scoring_rules.json, --regex-candidate-scoring-rules), the way a variable is judged by the variable-scoring rules. - Rules declare an evidence class (format or shape) and type their capture groups (value, name, user, host, kind); a value group can declare its encoding (base64, base32, hex, base58check). - Key rules judge the body between the markers: the payload without line breaks, quotes, concatenations, headers and decoration, then its foreign share, randomness and checksum. S5/S26 lose their negative patterns; S18 reports the service account's private key. - A rejected candidate is dropped, or kept flagged at confidence 0 when asked; a match covering a variable the semantic engine reported is corroborated and keeps its rule's confidence. - Payload patterns run in linear time; a comparison on a text field is refused when the rules load; an empty match is never reported. --- deepsecrets/cli.py | 22 ++ deepsecrets/core/engines/regex.py | 81 ++++- deepsecrets/core/engines/semantic.py | 13 +- deepsecrets/core/helpers/confidence.py | 18 + deepsecrets/core/helpers/content_analyzer.py | 4 +- deepsecrets/core/helpers/encoding.py | 151 ++++++++ .../core/helpers/regex_candidate_evaluator.py | 72 ++++ .../core/helpers/variable_evaluator.py | 14 +- deepsecrets/core/model/finding.py | 15 +- deepsecrets/core/model/regex_candidate.py | 95 +++++ deepsecrets/core/model/rules/regex.py | 89 ++++- .../model/rules/regex_candidate_scoring.py | 142 ++++++++ .../core/rulesets/regex_candidate_scoring.py | 7 + .../rules/regex_candidate_scoring_rules.json | 180 ++++++++++ deepsecrets/rules/regexes.json | 139 ++++++-- deepsecrets/scan_modes/cli.py | 8 +- tests/cli/test_cli.py | 3 +- tests/core/engines/regex/test_regex.py | 60 +++- tests/core/helpers/test_encoding.py | 139 ++++++++ .../helpers/test_regex_candidate_evaluator.py | 324 ++++++++++++++++++ 20 files changed, 1514 insertions(+), 62 deletions(-) create mode 100644 deepsecrets/core/helpers/confidence.py create mode 100644 deepsecrets/core/helpers/encoding.py create mode 100644 deepsecrets/core/helpers/regex_candidate_evaluator.py create mode 100644 deepsecrets/core/model/regex_candidate.py create mode 100644 deepsecrets/core/model/rules/regex_candidate_scoring.py create mode 100644 deepsecrets/core/rulesets/regex_candidate_scoring.py create mode 100644 deepsecrets/rules/regex_candidate_scoring_rules.json create mode 100644 tests/core/helpers/test_encoding.py create mode 100644 tests/core/helpers/test_regex_candidate_evaluator.py diff --git a/deepsecrets/cli.py b/deepsecrets/cli.py index 14ee4b5..acd1899 100644 --- a/deepsecrets/cli.py +++ b/deepsecrets/cli.py @@ -23,6 +23,7 @@ from deepsecrets.core.rulesets.false_findings import FalseFindingsBuilder from deepsecrets.core.rulesets.hashed_secrets import HashedSecretsRulesetBuilder from deepsecrets.core.rulesets.regex import RegexRulesetBuilder +from deepsecrets.core.rulesets.regex_candidate_scoring import RegexCandidateScoringRulesetBuilder from deepsecrets.core.rulesets.variable_scoring import VariableScoringRulesetBuilder from deepsecrets.core.ui.progress_bar import DSApplicationProgess from deepsecrets.core.ui.time_remaining_column import SyncedTimeRemainingColumn @@ -119,6 +120,19 @@ def _build_argparser(self) -> None: default=['built-in'], ) + parser.add_argument( + '--regex-candidate-scoring-rules', + nargs='*', + type=str, + help='Controls rules for judging regex matches (candidates) before they are reported: placeholders,\n' + 'language-like values, hosts, test paths. Used by regex rules that declare an "evidence" class.\n' + '- Ignore this argument to use the built-in ruleset\n' + "- Using your own rulesets disables the default one. Add 'built-in' to the args list to merge rulesets\n" + 'eq. --regex-candidate-scoring-rules built-in /root/my_candidate_rules.json\n' + '- Give it no value to report regex matches unjudged, as before 2.2\n', + default=['built-in'], + ) + parser.add_argument( '--hashed-values', nargs='*', @@ -303,6 +317,14 @@ def parse_arguments(self) -> None: config.engines.append(RegexEngine) config.add_ruleset(RegexRulesetBuilder, rules) + CANDIDATE_BUILTIN_RULESET = get_path_inside_package('rules/regex_candidate_scoring_rules.json') + if user_args.regex_candidate_scoring_rules is not None: + rules = [ + rule.replace('built-in', CANDIDATE_BUILTIN_RULESET) + for rule in user_args.regex_candidate_scoring_rules + ] + config.add_ruleset(RegexCandidateScoringRulesetBuilder, rules) + conf_semantic_analysis = user_args.semantic_analysis if conf_semantic_analysis is not None and conf_semantic_analysis != DISABLED: config.engines.append(SemanticEngine) diff --git a/deepsecrets/core/engines/regex.py b/deepsecrets/core/engines/regex.py index 7cfcd0e..a497cbb 100644 --- a/deepsecrets/core/engines/regex.py +++ b/deepsecrets/core/engines/regex.py @@ -1,8 +1,8 @@ -from typing import List, Tuple +from typing import List, Optional from deepsecrets.core.engines.iengine import IEngine from deepsecrets.core.model.finding import Finding -from deepsecrets.core.model.rules.regex import RegexRule +from deepsecrets.core.model.rules.regex import RegexRule, RuleMatch from deepsecrets.core.model.token import Token @@ -10,30 +10,79 @@ class RegexEngine(IEngine): name = 'regex' description = 'Scans by regex patterns provided by RegexRules' - def search(self, token: Token) -> List[Finding]: + def __init__( + self, ruleset: Optional[List] = None, candidate_rules: Optional[List] = None, report_rejected: bool = False + ) -> None: + super().__init__(ruleset=ruleset if ruleset is not None else []) + # a regex match is a candidate, judged here as it is found, as the semantic engine judges a variable + self.evaluator = None + if candidate_rules: + # imported here: the evaluator reads the naturalness model, which a regex-only engine does not need + from deepsecrets.core.helpers.regex_candidate_evaluator import RegexCandidateEvaluator + from deepsecrets.core.model.regex_candidate import RegexCandidateContext + + self.evaluator = RegexCandidateEvaluator(candidate_rules) + self.candidate_context = RegexCandidateContext + # rejected candidates are reported (confidence 0, flagged) only at --confidence-level all + self.report_rejected = report_rejected + # the diagnostics tracer sets a list here to record every rejection, in file offsets, without emitting them + self.rejected_log: Optional[list] = None + + def search(self, token: Token, corroborated: bool = False) -> List[Finding]: + """`corroborated`: the semantic engine reported this token's variable, so a match covering the whole token is + already judged and keeps its rule's confidence.""" results = [] for rule in self.ruleset: if not self.is_rule_applicable(token=token, rule=rule): continue - results.extend(self._check_rule(token, rule)) # type: ignore + results.extend(self._check_rule(token, rule, corroborated)) # type: ignore return results - def _check_rule(self, token: Token, rule: RegexRule) -> List[Finding]: + def _check_rule(self, token: Token, rule: RegexRule, corroborated: bool = False) -> List[Finding]: findings: List[Finding] = [] - # rule.match returns an array of (start, end) spans - detects: List[Tuple[int, int]] = rule.match(token) - - for start, end in detects: - findings.append( - Finding( - rules=[rule], - detection=token.content[start:end], - start_offset=start, - end_offset=end, - ) + for match in rule.matches(token): + if match.start == match.end: + continue # an empty group (password= with nothing after it): nothing to report + finding = Finding( + rules=[rule], + detection=token.content[match.start : match.end], + start_offset=match.start, + end_offset=match.end, ) + if self._judge(finding, token, rule, match, corroborated): + findings.append(finding) return findings + + def _judge(self, finding: Finding, token: Token, rule: RegexRule, match: RuleMatch, corroborated: bool) -> bool: + """Whether to emit the finding; sets its confidence, or marks it rejected.""" + if self.evaluator is None or rule.evidence is None: + return True + if corroborated and (match.start, match.end) == (0, len(token.content)): + return True + + context = self.candidate_context.from_match(rule, match.match, finding.detection, token.file.path) + verdict = self.evaluator.evaluate(context) + if verdict.rejected: + if self.rejected_log is not None: + offset = token.span[0] if token.span else 0 + self.rejected_log.append( + { + 'span': (offset + match.start, offset + match.end), + 'rule': rule.id, + 'evidence': rule.evidence.value, + } + | verdict.summary() + ) + if not self.report_rejected: + return False + finding.rejected = True + finding.rules = [rule.model_copy(update={'confidence': 0, 'is_dynamic_confidence': True})] + finding.internal_score = {'candidate': verdict.summary()} + return True + if verdict.confidence is not None: + finding.rules = [rule.model_copy(update={'confidence': verdict.confidence, 'is_dynamic_confidence': True})] + return True diff --git a/deepsecrets/core/engines/semantic.py b/deepsecrets/core/engines/semantic.py index d94abce..46f305e 100644 --- a/deepsecrets/core/engines/semantic.py +++ b/deepsecrets/core/engines/semantic.py @@ -41,11 +41,22 @@ def search(self, token: Token) -> List[Finding]: if fname in token.file.path: return findings + # the variable first: a regex match that covers the whole token of a variable reported here is corroborated, + # so the regex sub-engine does not judge it again + semantic_findings = self._evaluate_semantics(token) + if self.subengine is not None: # pragma: nocover - content_findings = ContentAnalyzer(self.subengine).analyze(token) + content_findings = ContentAnalyzer(self.subengine).analyze(token, corroborated=bool(semantic_findings)) if content_findings is not None: findings.extend(content_findings) + findings.extend(semantic_findings) + return findings + + def _evaluate_semantics(self, token: Token) -> List[Finding]: + """S107 for a dangerous condition, S105 or S106 for a variable the evaluator scores as dangerous.""" + findings: List[Finding] = [] + if token.semantic is None: return findings diff --git a/deepsecrets/core/helpers/confidence.py b/deepsecrets/core/helpers/confidence.py new file mode 100644 index 0000000..6180983 --- /dev/null +++ b/deepsecrets/core/helpers/confidence.py @@ -0,0 +1,18 @@ +"""The parts of a confidence both evaluators share: the entropy scale and what a random-looking value adds.""" + + +def entropy_score(bits: float) -> float: + """-1 for a single repeated character, 0 under 3 bits per character, 0 to 35 from 3 to 4 bits, 40 at 4 or more.""" + if bits == 0: + return -1 + if bits < 3: + return 0 + if bits < 4: + return (bits - 3) * 35 + return 40 + + +def randomness_points(score: float, nonsense: float) -> float: + """Up to 5 confidence points for a random-looking value: its entropy score (entropy_score, 0..40) scaled to 0..5, + halved for a value that reads as language (nonsense = 1 - naturalness).""" + return min(max(score, 0), 40) / 40 * 5 * min(nonsense + 0.5, 1) diff --git a/deepsecrets/core/helpers/content_analyzer.py b/deepsecrets/core/helpers/content_analyzer.py index 8a97cbc..fb393dc 100644 --- a/deepsecrets/core/helpers/content_analyzer.py +++ b/deepsecrets/core/helpers/content_analyzer.py @@ -16,10 +16,10 @@ def __init__(self, engine: IEngine) -> None: self.engine = engine self.uncover_tactics = [self._check_by_base64] - def analyze(self, token: Token) -> List[Finding]: + def analyze(self, token: Token, **search_options) -> List[Finding]: self.token = token self.uncover() - return self.engine.search(self.token) if self.engine is not None else [] + return self.engine.search(self.token, **search_options) if self.engine is not None else [] def uncover(self) -> None: for tactic in self.uncover_tactics: diff --git a/deepsecrets/core/helpers/encoding.py b/deepsecrets/core/helpers/encoding.py new file mode 100644 index 0000000..e10116b --- /dev/null +++ b/deepsecrets/core/helpers/encoding.py @@ -0,0 +1,151 @@ +"""What an encoded value is made of: its payload, the share of it outside its alphabet, how random it is for its +length, and whether its checksum holds. A regex rule declares the encoding on its value group (`"encoding": "base64"` +in a `match_rules` entry); see docs/private/research/regex-precision-and-gap.md.""" + +import hashlib +from functools import lru_cache +from math import exp, lgamma, log, log1p, log2 +from typing import Optional + +import regex as re + +BASE58 = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz' +ALPHABETS = { + 'base64': frozenset('ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/='), + 'base32': frozenset('ABCDEFGHIJKLMNOPQRSTUVWXYZ234567='), + 'hex': frozenset('0123456789abcdefABCDEF'), + 'base58check': frozenset(BASE58), +} +# the symbols a random value of the encoding draws from (padding and the two cases of hex are not extra symbols) +SYMBOLS = {'base64': 64, 'base32': 32, 'hex': 16, 'base58check': 58} + +# The payload is read line by line: what surrounds an encoded value in source code sits at the ends of its lines +# (indentation, quotes, a comment marker, a diff sign, a concatenation, a line continuation), while code between two +# key markers has foreign characters inside its lines. Decoration is stripped, the inside is kept as it is. + +# the armour of another block inside a body: the match ran across blocks. Another private key's armour (a key removed +# and one added in a diff, two keys in a row) goes like a header line; any other block's (a certificate, a public key) +# stays foreign, whatever its edges +# (a match starts at the first dash of a run: trying every dash of a long run would take quadratic time) +ARMOUR = re.compile(r'(?, a language's +# newline constant +LINE_BREAK = re.compile( + r'\\{1,2}[rn]|[03];|�?[da];||' + r'\b(?:vbCrLf|vbNewLine|vbLf|Environment\.NewLine|PHP_EOL|os\.linesep)\b|\bSystem\.lineSeparator\(\)', + re.IGNORECASE, +) +# what escaping leaves (\/ in JSON, \" and \\ in string literals): a backslash is never part of an encoded value +BACKSLASH = re.compile(r'\\+') +# an interpolated name standing for a line break: {lineEnding}, ${nl}, #{eol} +PLACEHOLDER = re.compile(r'[$#]?\{[A-Za-z_][\w.]*(?:\(\))?\}') +# an armour header (Proc-Type, DEK-Info, Version, Comment…): no line of an encoded payload has a colon +HEADER_LINE = re.compile(r'^[^\w\n]*[A-Za-z][A-Za-z0-9-]*:[^\n]*$', re.MULTILINE) +# two string literals joined: "…" + "…" (Java, JS), "…" . "…" (PHP), "…" & "…" (VB), '…', '…' (a list), "…" $"…" (C#) +# (the whitespace after the operator belongs to it: two adjacent \s* would take quadratic time on a long run) +CONCATENATION = re.compile(r'["\'`]\s*(?:[+.,&]\s*)?[$@]?["\'`]') +# decoration at the start and the end of a line. A + (Java, JS) or a . (PHP) is decoration only next to a quote +# (+ "…" starting a line, "…" + ending one): elsewhere a + is base64, and a dot before the closing quote is an +# ellipsis (MIIE...), a truncated key rather than decoration +LEAD = re.compile(r'^(?:[ \t"\'`#*;>!|$@(\[,&-]|[+.](?=[ \t]*["\'`$@]))+', re.MULTILINE) +TRAIL = re.compile(r'(?:[ \t"\'`,;&_)\]|>-]|(?<=["\'`][ \t]*)[+.])+$', re.MULTILINE) +# a line left with nothing but the concatenation sign +LONE_PLUS = re.compile(r'^[ \t]*[+.][ \t]*$', re.MULTILINE) +WHITESPACE = re.compile(r'\s+') + +HEX = re.compile(r'[0-9a-f]+|[0-9A-F]+') +BASE32 = re.compile(r'[A-Z2-7]+') +CLASSES = ((re.compile(r'\d'), 10), (re.compile('[a-z]'), 26), (re.compile('[A-Z]'), 26)) +ALPHANUMERIC = re.compile(r'[A-Za-z0-9]') +# up to this length the expected entropy of a random string is summed exactly, beyond it approximated +EXACT_UP_TO = 2048 + + +def _other_armour_foreign(armour: re.Match) -> str: + return '' if 'PRIVATE' in armour.group(0).upper() else '~' * len(armour.group(0)) + + +def payload(text: str) -> str: + """The encoded value alone. Line breaks written as escapes, entities or constants become line breaks, header lines + go, and each line loses the decoration at its ends. A key body in a PEM file, a JSON string, a Java, Go, PHP, VB or + C# concatenation, a comment, a diff or an XML document gives the same payload.""" + text = ARMOUR.sub(_other_armour_foreign, text) + text = LINE_BREAK.sub('\n', text) + text = BACKSLASH.sub('', text) + text = PLACEHOLDER.sub('', text) + text = HEADER_LINE.sub('', text) + text = CONCATENATION.sub('\n', text) + text = TRAIL.sub('', LEAD.sub('', text)) + text = LONE_PLUS.sub('', text) + text = TRAIL.sub('', LEAD.sub('', text)) + return WHITESPACE.sub('', text) + + +def foreign_share(value: str, encoding: Optional[str]) -> float: + """The share of the value's characters outside its encoding's alphabet; 0 when no encoding is declared.""" + if not encoding or not value: + return 0.0 + alphabet = ALPHABETS[encoding] + return sum(1 for char in value if char not in alphabet) / len(value) + + +def symbols(value: str, encoding: Optional[str]) -> int: + """The number of symbols the value is drawn from: its encoding's, else read from the characters present.""" + if encoding: + return SYMBOLS[encoding] + if HEX.fullmatch(value): + return 16 + if BASE32.fullmatch(value): + return 32 + count = sum(size for pattern, size in CLASSES if pattern.search(value)) + count += len(set(ALPHANUMERIC.sub('', value))) + return max(count, 2) + + +@lru_cache(maxsize=4096) +def expected_entropy(length: int, alphabet_size: int) -> float: + """The Shannon entropy, in bits per character, a uniformly random string of this length over this many symbols is + expected to show. Exact (each symbol's count is binomial) up to EXACT_UP_TO characters; beyond it the + Miller–Madow approximation, which is within a thousandth of a bit there.""" + if length < 2 or alphabet_size < 2: + return 0.0 + if length > EXACT_UP_TO: + return log2(alphabet_size) - (alphabet_size - 1) / (2 * length * log(2)) + p = 1 / alphabet_size + log_p, log_q, log_n = log(p), log1p(-p), lgamma(length + 1) + mean = length * p + per_symbol = 0.0 + for count in range(1, length + 1): + pmf = exp(log_n - lgamma(count + 1) - lgamma(length - count + 1) + count * log_p + (length - count) * log_q) + if count > mean and pmf < 1e-12: + break + share = count / length + per_symbol -= pmf * share * log2(share) + return alphabet_size * per_symbol + + +def randomness(value: str, entropy: float, encoding: Optional[str]) -> float: + """The value's entropy over the entropy a random string of its length and alphabet is expected to have: about 1 + for a random value, lower for repetition and structure. The spread shrinks with length: a random 16-character value + can fall to 0.8 by chance, a random 1,600-character key body stays above 0.98.""" + expected = expected_entropy(len(value), symbols(value, encoding)) + return entropy / expected if expected > 0 else 0.0 + + +def checksum(value: str, encoding: Optional[str]) -> str: + """'valid' or 'invalid' for an encoding that carries a checksum, else 'none'. base58check (Bitcoin WIF keys and + addresses): the last 4 bytes are the first 4 of a double SHA-256 of the rest.""" + if encoding != 'base58check': + return 'none' + number = 0 + for char in value: + digit = BASE58.find(char) + if digit < 0: + return 'invalid' + number = number * 58 + digit + raw = number.to_bytes((number.bit_length() + 7) // 8, 'big') + raw = b'\0' * (len(value) - len(value.lstrip('1'))) + raw + if len(raw) < 5: + return 'invalid' + body, check = raw[:-4], raw[-4:] + return 'valid' if hashlib.sha256(hashlib.sha256(body).digest()).digest()[:4] == check else 'invalid' diff --git a/deepsecrets/core/helpers/regex_candidate_evaluator.py b/deepsecrets/core/helpers/regex_candidate_evaluator.py new file mode 100644 index 0000000..6f19b1d --- /dev/null +++ b/deepsecrets/core/helpers/regex_candidate_evaluator.py @@ -0,0 +1,72 @@ +from dataclasses import dataclass, field +from typing import Dict, List, Optional + +from deepsecrets.core.helpers.confidence import entropy_score, randomness_points +from deepsecrets.core.model.regex_candidate import RegexCandidateContext +from deepsecrets.core.model.rules.regex import Evidence +from deepsecrets.core.model.rules.regex_candidate_scoring import RegexCandidateScoringRule + +# The confidence a `shape` match earns from itself, before its value is looked at: what a strong name gives a variable +# (S105's 0.2 points per naming point, up to 20). Rule scores are confidence points taken from it or given back. +CORROBORATION = 4 +# A candidate is rejected when its total reaches this: one warning sign (-8) its cases did not buy back. A rule meant +# only to lower a kept candidate's confidence must stay small: a warning bought back leaves a total of -4, 4 points +# from here. +REJECT_AT = -8 + + +@dataclass +class CandidateVerdict: + rejected: bool + # for a kept `shape` candidate; None keeps the regex rule's own confidence (a `format` match proves its type) + confidence: Optional[int] + total: int = 0 + # each fired rule, with the name of the case that applied (None when no case did) + fired: Dict[str, Optional[str]] = field(default_factory=dict) + # for a rejected candidate: the fired rule that took the most points + reason: Optional[str] = None + + def summary(self) -> dict: + return { + 'rejected': self.rejected, + 'confidence': self.confidence, + 'total': self.total, + 'fired': self.fired, + 'reason': self.reason, + } + + +class RegexCandidateEvaluator: + """Judges regex candidates with the regex-candidate scoring rules, as VariableEvaluator judges variables with the + variable-scoring rules: every fired rule adds its score and its first matching case's. See + docs/private/research/regex-candidates.md.""" + + rules: List[RegexCandidateScoringRule] + + def __init__(self, rules: List[RegexCandidateScoringRule]) -> None: + self.rules = rules + + def evaluate(self, context: RegexCandidateContext) -> CandidateVerdict: + total = 0 + fired: Dict[str, Optional[str]] = {} + contributions: List[tuple] = [] + for rule in self.rules: + if not rule.fires(context): + continue + case = rule.case_for(context) + points = rule.score + (case.score if case is not None else 0) + total += points + fired[rule.id] = case.name if case is not None else None + contributions.append((points, rule.id)) + + if total <= REJECT_AT: + reason = min(contributions)[1] if contributions else None + return CandidateVerdict(rejected=True, confidence=0, total=total, fired=fired, reason=reason) + if context.evidence != Evidence.SHAPE.value: + return CandidateVerdict(rejected=False, confidence=None, total=total, fired=fired) + corroboration = min(max(CORROBORATION + total, 0), CORROBORATION) + randomness = randomness_points( + entropy_score(context.value_entropy), 1 - context.value_normalized_naturalness_score + ) + confidence = round(min(corroboration + randomness, 10)) + return CandidateVerdict(rejected=False, confidence=confidence, total=total, fired=fired) diff --git a/deepsecrets/core/helpers/variable_evaluator.py b/deepsecrets/core/helpers/variable_evaluator.py index 85bdc5a..b2084ae 100644 --- a/deepsecrets/core/helpers/variable_evaluator.py +++ b/deepsecrets/core/helpers/variable_evaluator.py @@ -1,5 +1,6 @@ from dataclasses import dataclass, field from typing import Dict, List, Optional, Tuple, Union +from deepsecrets.core.helpers.confidence import entropy_score, randomness_points from deepsecrets.core.helpers.entropy import EntropyHelper from deepsecrets.core.model.rules.variable_scoring import VariableScoringRule from deepsecrets.core.model.semantic import Context, Variable @@ -62,16 +63,7 @@ def _fires(self, index: int, rule: VariableScoringRule, context: Context) -> boo return fired def calculate_entropy_score(self, entropy: float) -> float: - if entropy == 0: - return -1 - - if entropy < 3: - return 0 - - if 3 <= entropy < 4: - return (entropy - 3) * 35 - - return 40 + return entropy_score(entropy) def evaluate(self, variable: Union[Variable | Context]) -> EvaluationResult: context = variable.context if isinstance(variable, Variable) else variable @@ -131,6 +123,6 @@ def confidence_from_evaluation_result(self, result: EvaluationResult): # natural-looking values. See docs/private/research/variable-scoring-balance.md. naming = min(max(result.naming_and_content_score, 0), 25) var_part = 0.2 * min(naming, 20) + 0.6 * max(naming - 20, 0) - entropy_part = min(max(result.entropy_score, 0), 40) / 40 * 5 * min(result.nonsence_value_score + 0.5, 1) + entropy_part = randomness_points(result.entropy_score, result.nonsence_value_score) return round(min(var_part + entropy_part, 10)) diff --git a/deepsecrets/core/model/finding.py b/deepsecrets/core/model/finding.py index dca3d0e..5fb04ef 100644 --- a/deepsecrets/core/model/finding.py +++ b/deepsecrets/core/model/finding.py @@ -24,6 +24,8 @@ class Finding(BaseModel): reason: str = Field(default='') final_rule: Optional[Rule] = Field(default=None) internal_score: Optional[dict] = Field(default_factory=dict) + # a regex candidate its evaluation rejected: reported only at --confidence-level all, with confidence 0 + rejected: bool = Field(default=False) _mapped_on_file: bool = PrivateAttr(default=False) model_config = ConfigDict(arbitrary_types_allowed=True) @@ -114,7 +116,16 @@ def merge(self, other: Any) -> bool: if other != self: return False - self.rules.extend(other.rules) - self.rules = list(set(self.rules)) + # One rule per id, in insertion order. The same rule reached through two passes keeps the first copy seen, as + # before, except that a kept finding's copy beats a rejected candidate's, and a static copy (a regex match the + # semantic engine corroborated) beats a dynamic one. + first, second = (other, self) if self.rejected and not other.rejected else (self, other) + best = {rule.id: rule for rule in first.rules} + for rule in second.rules: + kept = best.get(rule.id) + if kept is None or (kept.is_dynamic_confidence and not rule.is_dynamic_confidence): + best[rule.id] = rule + self.rules = list(best.values()) + self.rejected = self.rejected and other.rejected return True diff --git a/deepsecrets/core/model/regex_candidate.py b/deepsecrets/core/model/regex_candidate.py new file mode 100644 index 0000000..d4a0d90 --- /dev/null +++ b/deepsecrets/core/model/regex_candidate.py @@ -0,0 +1,95 @@ +from dataclasses import dataclass, field +from typing import Optional + +import regex as re + +from deepsecrets.core.helpers import encoding as encoded +from deepsecrets.core.helpers.entropy import EntropyHelper +from deepsecrets.core.model.rules.regex import GroupType, RegexRule +from deepsecrets.core.model.semantic import Context + +HOST_END = re.compile(r'[/:?#\s"\'),;\]\\]') +LOCAL_HOST = re.compile( + r'^(localhost|127\.|0\.0\.0\.0|10\.|192\.168\.|172\.(1[6-9]|2\d|3[01])\.|host\.docker\.internal|[^.]+$)', re.I +) +PLACEHOLDER_HOST = re.compile( + r'(^|\.)(example|foo|bar|baz|foobar|test|domain|yourdomain|mydomain|host|hostname|server|myserver|yourserver|' + r'invalid|local|localdomain|tld|xxx|company|acme)(\.[a-z]+)?$|[<>{}$%*]|^(example|foo|bar)\.', + re.I, +) + + +def host_kind(address: Optional[str]) -> str: + """'public', 'local', 'placeholder' or 'none' for the text a host group holds (after a URL's '@', after a + connection string's scheme): a dotted public name, localhost or a single-label service name, or a documentation + domain such as example.com or foobar.com.""" + host = HOST_END.split(address or '', maxsplit=1)[0] + if not host: + return 'none' + if PLACEHOLDER_HOST.search(host): + return 'placeholder' + if LOCAL_HOST.search(host): + return 'local' + return 'public' + + +@dataclass +class RegexCandidateContext(Context): + """A regex match seen as a candidate, like a variable: a value, sometimes a name, and what the rule's typed groups + hold around them. Name and value are normalised exactly as a variable's (Context); the regex-candidate scoring + rules read these fields by target (rules/regex_candidate_scoring_rules.json).""" + + rule: str = '' + evidence: str = '' + user: str = '' + host: str = '' + kind: str = '' + # the encoding the value group declares; with one, the value is its payload (core/helpers/encoding.py) + encoding: str = '' + host_kind: str = field(default='none', repr=False) + value_entropy: float = field(default=0.0, repr=False) + # the share of the value outside its encoding's alphabet (0 without an encoding) + value_foreign_share: float = field(default=0.0, repr=False) + # the value's entropy over a random string's of the same length and alphabet: about 1 when random + value_randomness: float = field(default=0.0, repr=False) + # 'valid' or 'invalid' for an encoding with a checksum, else 'none' + value_checksum: str = field(default='none', repr=False) + + def __post_init__(self): + if self.encoding: + self.value = encoded.payload(self.value) + super().__post_init__() + self.host_kind = host_kind(self.host) + self.value_entropy = EntropyHelper.get_for_string(self.value) + self.value_foreign_share = encoded.foreign_share(self.value, self.encoding) + self.value_randomness = encoded.randomness(self.value, self.value_entropy, self.encoding) + self.value_checksum = encoded.checksum(self.value, self.encoding) + + @classmethod + def from_match( + cls, rule: RegexRule, match: Optional[re.Match], reported: str, filepath: str + ) -> 'RegexCandidateContext': + """The candidate a rule's match makes. The value is the group typed `value`, else the rule's target group, else + the reported text; for a hit in decoded content the groups are the decoded text's. A value group that declares + an encoding gives its payload as the value: a key body without its line breaks, quotes and headers.""" + + def group(type: GroupType) -> str: + index = rule.group_of(type) + if index is None or match is None: + return '' + return match.group(index) or '' + + value_group = group(GroupType.VALUE) + value = value_group or (match.group(rule.target_group) if match is not None else '') or reported + encoding = rule.encoding_of_value() if value_group else None + return cls( + name=group(GroupType.NAME), + value=value or '', + filepath=filepath, + rule=rule.id, + evidence=rule.evidence.value if rule.evidence is not None else '', + user=group(GroupType.USER), + host=group(GroupType.HOST), + kind=group(GroupType.KIND), + encoding=encoding.value if encoding is not None else '', + ) diff --git a/deepsecrets/core/model/rules/regex.py b/deepsecrets/core/model/rules/regex.py index 488f3a2..feb2218 100644 --- a/deepsecrets/core/model/rules/regex.py +++ b/deepsecrets/core/model/rules/regex.py @@ -1,5 +1,7 @@ import regex as re -from typing import Dict, ForwardRef, List, Optional, Union +from dataclasses import dataclass +from enum import Enum +from typing import Dict, ForwardRef, List, Optional, Tuple, Union from pydantic import ConfigDict, Field, field_serializer, model_validator @@ -10,6 +12,45 @@ RegexRule = ForwardRef('RegexRule') +class GroupType(str, Enum): + """What a capture group of a regex rule holds, declared in its `match_rules` entry. A regex match typed this way is + a candidate: like a variable, it has a value and sometimes a name, plus what the rule knows around them.""" + + VALUE = 'value' # the text the candidate rules judge; default: the reported text (target_group) + NAME = 'name' # what the value is assigned to, like a variable's name + USER = 'user' # the account a credential belongs to + HOST = 'host' # the service a credential opens + KIND = 'kind' # a sub-type the rule distinguishes (an AWS key id's prefix) + + +class Encoding(str, Enum): + """How a value group's text is encoded, declared next to its type (`{"type": "value", "encoding": "base64"}`). The + candidate then judges the payload (core/helpers/encoding.py): the share outside the alphabet, the randomness + against it, the checksum.""" + + BASE64 = 'base64' + BASE32 = 'base32' + HEX = 'hex' + BASE58CHECK = 'base58check' # carries a checksum + + +class Evidence(str, Enum): + """What a regex rule's match proves, which decides the regex-candidate rules that apply to it.""" + + FORMAT = 'format' # the match proves the secret type: a key block, a prefixed or checksummed token + SHAPE = 'shape' # the match is only a shape (a URL password, an AWS key id) and needs corroboration + + +@dataclass +class RuleMatch: + """One match of a rule: the reported span, and the match itself for its groups. A hit in decoded content reports + the whole encoded token as its span, while `match` is the match in the decoded text.""" + + start: int + end: int + match: re.Match + + class RegexRule(Rule): # type: ignore pattern: re.Pattern negative_pattern: Optional[re.Pattern] = Field(default=None) @@ -18,6 +59,13 @@ class RegexRule(Rule): # type: ignore entropy_settings: Optional[float] = Field(default=None) escaping_needed: bool = False case_sensitive: bool = False + # which regex-candidate rules apply to this rule's matches (rules/regex_candidate_scoring_rules.json). None, the + # default for a user's own rules, leaves its matches unjudged, as before 2.2 + evidence: Optional[Evidence] = Field(default=None) + # on a match_rules entry: what its capture group holds (the text of the group must also match the entry) + type: Optional[GroupType] = Field(default=None) + # on a match_rules entry typed `value`: how the value is encoded + encoding: Optional[Encoding] = Field(default=None) model_config = ConfigDict(arbitrary_types_allowed=True) @@ -61,13 +109,40 @@ def build_pattern(cls, values: Dict) -> Dict: for _, match_rule in match_rules.items(): match_rule['id'] = '' match_rule['confidence'] = 9 + # a group can be typed without being constrained + match_rule.setdefault('pattern', '.*') return values + @model_validator(mode='after') + def one_group_per_type(self) -> 'RegexRule': + types = [rule.type for rule in (self.match_rules or {}).values() if rule.type is not None] + if len(types) != len(set(types)): + raise ValueError(f'rule {self.id}: a group type is declared twice in match_rules') + for rule in (self.match_rules or {}).values(): + if rule.encoding is not None and rule.type != GroupType.VALUE: + raise ValueError(f'rule {self.id}: only the group typed value declares an encoding') + return self + + def encoding_of_value(self) -> Optional[Encoding]: + """The encoding the value group declares, if any.""" + index = self.group_of(GroupType.VALUE) + return self.match_rules[index].encoding if index is not None else None # type: ignore + + def group_of(self, type: GroupType) -> Optional[int]: + """The capture group declared with this type, if any.""" + for group, rule in (self.match_rules or {}).items(): + if rule.type == type: + return int(group) + return None + def __hash__(self) -> int: # pragma: nocover return hash(self.id) - def match(self, token: Union[Token, str]) -> List[re.Match]: + def match(self, token: Union[Token, str]) -> List[Tuple[int, int]]: + return [(m.start, m.end) for m in self.matches(token)] + + def matches(self, token: Union[Token, str]) -> List[RuleMatch]: good_matches = [] contents = [] contents.append(token.content if isinstance(token, Token) else token) @@ -79,13 +154,15 @@ def match(self, token: Union[Token, str]) -> List[re.Match]: if self.negative_pattern is not None and self.negative_pattern.search(content) is not None: continue - matches = self.pattern.finditer(content) - - for match in matches: + for match in self.pattern.finditer(content): if not self._verify(match): continue - good_matches.append(match.span(self.target_group) if i == 0 else (0, len(contents[0]))) + if i == 0: + start, end = match.span(self.target_group) + good_matches.append(RuleMatch(start, end, match)) + else: + good_matches.append(RuleMatch(0, len(contents[0]), match)) return good_matches diff --git a/deepsecrets/core/model/rules/regex_candidate_scoring.py b/deepsecrets/core/model/rules/regex_candidate_scoring.py new file mode 100644 index 0000000..5bacad4 --- /dev/null +++ b/deepsecrets/core/model/rules/regex_candidate_scoring.py @@ -0,0 +1,142 @@ +import operator +from enum import Enum +from typing import Any, Dict, List, Optional + +import regex as re +from pydantic import BaseModel, ConfigDict, Field, field_serializer, model_validator + +from deepsecrets.core.model.rules.rule import Rule + + +class CandidateTarget(str, Enum): + """A field of RegexCandidateContext. The name is the field's, upper-cased.""" + + NAME_SPACED = 'NAME_SPACED' + NAME_NORMALIZED = 'NAME_NORMALIZED' + VALUE = 'VALUE' + VALUE_NORMALIZED = 'VALUE_NORMALIZED' + VALUE_LENGTH = 'VALUE_LENGTH' + VALUE_NORMALIZED_NATURALNESS_SCORE = 'VALUE_NORMALIZED_NATURALNESS_SCORE' + VALUE_ENTROPY = 'VALUE_ENTROPY' # Shannon entropy, bits per character + VALUE_RANDOMNESS = 'VALUE_RANDOMNESS' # entropy over a random string's of the same length and alphabet: ~1 random + VALUE_FOREIGN_SHARE = 'VALUE_FOREIGN_SHARE' # share of the value outside its declared encoding's alphabet + VALUE_CHECKSUM = 'VALUE_CHECKSUM' # valid, invalid, or none (no checksummed encoding declared) + FILEPATH = 'FILEPATH' + USER = 'USER' + HOST = 'HOST' + HOST_KIND = 'HOST_KIND' # public, local, placeholder or none + KIND = 'KIND' + RULE = 'RULE' # the regex rule's id + EVIDENCE = 'EVIDENCE' # the regex rule's evidence label: format or shape + + +OPS = { + '<=': operator.le, + '>=': operator.ge, + '==': operator.eq, + '!=': operator.ne, + '<': operator.lt, + '>': operator.gt, +} + + +# the fields a comparison (`method` with `threshold`) can test; the others are text +NUMERIC_TARGETS = frozenset( + { + CandidateTarget.VALUE_LENGTH, + CandidateTarget.VALUE_NORMALIZED_NATURALNESS_SCORE, + CandidateTarget.VALUE_ENTROPY, + CandidateTarget.VALUE_RANDOMNESS, + CandidateTarget.VALUE_FOREIGN_SHARE, + } +) + + +class CandidateTest(BaseModel): + """A test of one field, in exactly one form: `pattern` (a case-insensitive regex search on its text), `method` + with `threshold` (a numeric comparison), `equals` (its text is this value) or `in` (its text is one of these).""" + + target: CandidateTarget + pattern: Optional[re.Pattern] = None + method: Optional[str] = None + threshold: Optional[float] = None + equals: Optional[str] = None + in_: Optional[List[str]] = Field(default=None, alias='in') + + model_config = ConfigDict(arbitrary_types_allowed=True, populate_by_name=True) + + @model_validator(mode='before') + @classmethod + def compile_pattern(cls, values: Dict) -> Dict: + pattern = values.get('pattern') + if isinstance(pattern, str): + values['pattern'] = re.compile(pattern, re.IGNORECASE) + return values + + @model_validator(mode='after') + def one_form(self) -> 'CandidateTest': + forms = [ + self.pattern is not None, + self.method is not None or self.threshold is not None, + self.equals is not None, + self.in_ is not None, + ] + if sum(forms) != 1: + raise ValueError(f'{self.target.value}: give one of pattern, method and threshold, equals, or in') + if forms[1] and (self.method not in OPS or self.threshold is None): + raise ValueError(f'{self.target.value}: a comparison needs a method ({", ".join(OPS)}) and a threshold') + if forms[1] and self.target not in NUMERIC_TARGETS: + # caught here, when the rules load: at scan time the comparison would raise and abandon the file + raise ValueError(f'{self.target.value}: a comparison needs a numeric field, and this one is text') + return self + + @field_serializer('pattern') + def serialize_pattern(self, pattern: Optional[re.Pattern], _info): + return pattern.pattern if pattern is not None else None + + def holds(self, context: Any) -> bool: + content = getattr(context, self.target.value.lower()) + if self.pattern is not None: + return self.pattern.search(str(content)) is not None + if self.equals is not None: + return str(content) == self.equals + if self.in_ is not None: + return str(content) in self.in_ + return OPS[self.method](content, self.threshold) + + +class CandidateCondition(CandidateTest): + """A `when` condition, or one of a case's `if` conditions.""" + + +class CandidateCase(BaseModel): + """A sub-condition of a rule with its own score: applies when every `if` condition holds.""" + + name: Optional[str] = None + conditions: List[CandidateCondition] = Field(alias='if') + score: int + + model_config = ConfigDict(populate_by_name=True) + + def holds(self, context: Any) -> bool: + return all(condition.holds(context) for condition in self.conditions) + + +class RegexCandidateScoringRule(Rule, CandidateTest): + """A rule judging regex candidates (a separate ruleset from the variable-scoring rules). + + It fires when every `when` condition holds and its own test does. A fired rule adds its `score`, then the score of + the first of its `cases` that holds, if any. Rules add up; RegexCandidateEvaluator rejects a candidate whose total + reaches REJECT_AT and turns a kept `shape` candidate's total into its confidence. Scores are confidence points. + """ + + when: List[CandidateCondition] = Field(default=[]) + score: int = 0 + cases: List[CandidateCase] = Field(default=[]) + + def fires(self, context: Any) -> bool: + return all(condition.holds(context) for condition in self.when) and self.holds(context) + + def case_for(self, context: Any) -> Optional[CandidateCase]: + """For a rule that fired: the first case that holds.""" + return next((case for case in self.cases if case.holds(context)), None) diff --git a/deepsecrets/core/rulesets/regex_candidate_scoring.py b/deepsecrets/core/rulesets/regex_candidate_scoring.py new file mode 100644 index 0000000..3c86407 --- /dev/null +++ b/deepsecrets/core/rulesets/regex_candidate_scoring.py @@ -0,0 +1,7 @@ +from deepsecrets.core.model.rules.regex_candidate_scoring import RegexCandidateScoringRule +from deepsecrets.core.rulesets.ibuilder import IRulesetBuilder + + +class RegexCandidateScoringRulesetBuilder(IRulesetBuilder): + rule_model = RegexCandidateScoringRule + ruleset_name = 'regex_candidate_scoring' diff --git a/deepsecrets/rules/regex_candidate_scoring_rules.json b/deepsecrets/rules/regex_candidate_scoring_rules.json new file mode 100644 index 0000000..8f47705 --- /dev/null +++ b/deepsecrets/rules/regex_candidate_scoring_rules.json @@ -0,0 +1,180 @@ +[ + { + "id": "RC_VALUE_LENGTH", + "name": "Too short to be a secret", + "description": "A value of 4 characters or less", + "target": "VALUE_LENGTH", + "method": "<=", + "threshold": 4, + "score": -100 + }, + { + "id": "RC_FALSE_STARTING_SEQ", + "name": "A template, a boolean or markup, not a value", + "target": "VALUE", + "pattern": "^(\\$\\{|%env|true|false|#|\\(|<)", + "score": -100 + }, + { + "id": "RC_NOT_ENCODED", + "name": "Not in the encoding its group declares", + "description": "Over 2% of the payload outside the declared alphabet: code or an ellipsis between key markers, not a key body; an AWS key id with a 0, 1, 8 or 9 (they are base32). The allowance covers a stray comment prefix", + "target": "VALUE_FOREIGN_SHARE", + "method": ">", + "threshold": 0.02, + "score": -100 + }, + { + "id": "RC_BAD_CHECKSUM", + "name": "A checksummed value whose checksum fails", + "description": "A base58check value (a Bitcoin WIF key) whose last 4 bytes are not its double SHA-256: a run of base58 characters, not a key", + "target": "VALUE_CHECKSUM", + "equals": "invalid", + "score": -100 + }, + { + "id": "RC_NOT_RANDOM", + "name": "Far from random", + "description": "The value has under 75% of the entropy a random string of its length and alphabet is expected to have: a placeholder, a repeated character, a run of digits (xoxb-XXXXXXXX-..., AKIAAAAAAAAAAAAAAAAA)", + "target": "VALUE_RANDOMNESS", + "method": "<", + "threshold": 0.75, + "score": -8 + }, + { + "id": "RC_LONG_NOT_RANDOM", + "name": "A long value that is not random", + "description": "From 100 characters the estimate is tight (a random key body stays above 0.98), so under 85% is structure, not key material. The bar sits below real OpenSSH keys, whose fixed length fields put them at 0.91 to 0.93", + "when": [ + { + "target": "VALUE_LENGTH", + "method": ">=", + "threshold": 100 + } + ], + "target": "VALUE_RANDOMNESS", + "method": "<", + "threshold": 0.85, + "score": -8 + }, + { + "id": "RC_FORMAT_PLACEHOLDER", + "name": "A placeholder word in a match whose format proves its type", + "description": "A warning sign, bought back when the value is random: 4 bits per character or more", + "when": [ + { + "target": "EVIDENCE", + "equals": "format" + } + ], + "target": "VALUE_NORMALIZED", + "pattern": "^(.*${.*|.*custom.*|.*deadbeef.*|.*langua.*|.*right.*|.*debug.*|.*left.*|.*egmen.*|.*auto.*|.*exampl.*|add.*|use.*|.*launc.*|.*mode.*|.*conne.*|.*sume.*|.*prod.*|.*ward.*|.*serv.*|.*elem.*|.*contain.*|sav.*|.*attac.*|.*fail.*|.*displ.*|.*fake.*|meta.*|.*text.*|.*scop.*|.*tach.*|.*vigat.*|.*ploy.*|.*reat.*|.*your.*|access.*|.*project.*|.*process.*|get.*|set.*|is.*|has.*|.*obje.*|.*beaut.*|.*file.*|.*runn.*|.*confir.*|.*click.*|.*wast.*|.*liste.*|.*hand.*|.*json.*|.*cance.*|.*reduc.*|.*delet.*|.*move.*|.*compa.*|.*forma.*|.*item.*|.*focus.*|.*west.*|.*east.*|.*build.*|.*toggl.*|.*dispos.*|.*componen.*|.*prop.*|.*destr.*|.*string.*|.*receiv.*|.*rende.*|.*featu.*|.*close.*|.*open.*|.*subscr.*|.*coord.*|.*space.*|.*dimen.*|.*menu.*|.*purcha.*|.*order.*|.*report.*|.*identifi.*|.*time.*|.*simpl.*|.*offse.*|.*state.*|.*quantit.*|.*categ.*|.*identit.*|.*devic.*|.*memor.*|.*settin.*|.*locat.*|.*option.*|.*stor.*|record.*|.*versi.*|.*origi.*|.*year.*|.*week.*|.*detai.*|.*gatew.*|.*packa.*|.*prefer.*|.*chain.*|.*provid.*|.*phon.*|.*email.*|.*convers.*|.*perso.*|.*recomm.*|.*referen.*|.*master.*|.*impor.*|.*encrypt.*|.*sched.*|.*contac.*|.*verla.*|.*mediu.*|.*extern.*|.*intern.*|.*progr.*|.*contr.*|.*positi.*|.*facebook.*|.*scale.*|.*button.*|.*untime.*|.*modif.*|.*service.*|.*volum.*|.*length.*|.*guest.*|.*point.*|.*speci.*|.*networ.*|.*mount.*|.*instruc.*|.*kuberne.*|.*layer.*|.*notifi.*|.*submi.*|.*success.*|.*operat.*|.*percen.*|.*persona.*|.*direct.*|.*prefix.*|.*print.*|.*previ.*|.*price.*|.*protoco.*|.*reaso.*|.*count.*|.*reques.*|.*return.*|.*event.*|.*desig.*|.*place.*|.*select.*|.*search.*|.*share.*|.*shape.*|.*show.*|.*stora.*|.*width.*|.*plan.*|.*suggest.*|.*style.*|.*syste.*|.*targe.*|.*export.*|.*amou.*|.*actio.*|.*encod.*|.*decod.*|.*trans.*|.*page.*|.*local.*|.*symb.*|.*userid.*|.*useragent.*|.*mock.*|.*color.*|.*password.*|.*ctrl.*|.*help.*|.*numb.*|.*long.*|.*short.*|.*tip.*|.*down.*|.*head.*|.*foote.*|.*info.*|.*last.*|.*next.*|.*title.*|.*label.*|.*error.*|.*view.*|.*center.*|.*date.*|.*paren.*|.*able.*|.*examp.*|.*servi.*|.*class.*|.*prope.*|.*reset.*|.*eract.*|.*rule.*|.*track.*|.*cloud.*|.*call.*|.*index.*|.*limit.*|.*acce.*|.*param.*|.*note.*|.*name.*|.*descr.*|.*value.*|.*polic.*|.*part.*|.*correc.*|.*conte.*|.*media.*|.*exten.*|.*defau.*|.*fill.*|.*unive.*|.*star.*|.*addit.*|.*stac.*|.*excep.*|.*messa.*|.*zone.*|.*total.*|.*type.*|.*valid.*|.*instan.*|.*seria.*|.*docum.*|.*base.*|.*addre.*|.*size.*|.*first.*|.*sourc.*|.*cover.*|.*gener.*|.*config.*|.*with.*|.*window.*|.*support.*|.*comment.*|.*batch.*|.*vault.*|.*secret.*|.*copy.*|.*filte.*|.*clear.*|.*status.*|.*private.*|.*password.*|.*invalid.*|.*link.*|.*test.*|.*expir.*|.*empty.*|.*token.*|null|bearer|foo|foobar|xyz|undefined|none|.*todo.*|.*change.*|.*restore.*|[^A-Za-z0-9]*|%.*%|\\[.*\\]|{.*})$", + "score": -8, + "cases": [ + { + "name": "random enough to be a secret anyway", + "if": [ + { + "target": "VALUE_ENTROPY", + "method": ">=", + "threshold": 4.0 + } + ], + "score": 4 + } + ] + }, + { + "id": "RC_PLACEHOLDER", + "name": "A placeholder word in a shape match", + "description": "A warning sign that costs the match its corroboration. A public host cancels it: a real service's password with a placeholder word in it is weak, not an example. A random value buys it back", + "when": [ + { + "target": "EVIDENCE", + "equals": "shape" + } + ], + "target": "VALUE_NORMALIZED", + "pattern": "^(.*${.*|.*custom.*|.*deadbeef.*|.*langua.*|.*right.*|.*debug.*|.*left.*|.*egmen.*|.*auto.*|.*exampl.*|add.*|use.*|.*launc.*|.*mode.*|.*conne.*|.*sume.*|.*prod.*|.*ward.*|.*serv.*|.*elem.*|.*contain.*|sav.*|.*attac.*|.*fail.*|.*displ.*|.*fake.*|meta.*|.*text.*|.*scop.*|.*tach.*|.*vigat.*|.*ploy.*|.*reat.*|.*your.*|access.*|.*project.*|.*process.*|get.*|set.*|is.*|has.*|.*obje.*|.*beaut.*|.*file.*|.*runn.*|.*confir.*|.*click.*|.*wast.*|.*liste.*|.*hand.*|.*json.*|.*cance.*|.*reduc.*|.*delet.*|.*move.*|.*compa.*|.*forma.*|.*item.*|.*focus.*|.*west.*|.*east.*|.*build.*|.*toggl.*|.*dispos.*|.*componen.*|.*prop.*|.*destr.*|.*string.*|.*receiv.*|.*rende.*|.*featu.*|.*close.*|.*open.*|.*subscr.*|.*coord.*|.*space.*|.*dimen.*|.*menu.*|.*purcha.*|.*order.*|.*report.*|.*identifi.*|.*time.*|.*simpl.*|.*offse.*|.*state.*|.*quantit.*|.*categ.*|.*identit.*|.*devic.*|.*memor.*|.*settin.*|.*locat.*|.*option.*|.*stor.*|record.*|.*versi.*|.*origi.*|.*year.*|.*week.*|.*detai.*|.*gatew.*|.*packa.*|.*prefer.*|.*chain.*|.*provid.*|.*phon.*|.*email.*|.*convers.*|.*perso.*|.*recomm.*|.*referen.*|.*master.*|.*impor.*|.*encrypt.*|.*sched.*|.*contac.*|.*verla.*|.*mediu.*|.*extern.*|.*intern.*|.*progr.*|.*contr.*|.*positi.*|.*facebook.*|.*scale.*|.*button.*|.*untime.*|.*modif.*|.*service.*|.*volum.*|.*length.*|.*guest.*|.*point.*|.*speci.*|.*networ.*|.*mount.*|.*instruc.*|.*kuberne.*|.*layer.*|.*notifi.*|.*submi.*|.*success.*|.*operat.*|.*percen.*|.*persona.*|.*direct.*|.*prefix.*|.*print.*|.*previ.*|.*price.*|.*protoco.*|.*reaso.*|.*count.*|.*reques.*|.*return.*|.*event.*|.*desig.*|.*place.*|.*select.*|.*search.*|.*share.*|.*shape.*|.*show.*|.*stora.*|.*width.*|.*plan.*|.*suggest.*|.*style.*|.*syste.*|.*targe.*|.*export.*|.*amou.*|.*actio.*|.*encod.*|.*decod.*|.*trans.*|.*page.*|.*local.*|.*symb.*|.*userid.*|.*useragent.*|.*mock.*|.*color.*|.*password.*|.*ctrl.*|.*help.*|.*numb.*|.*long.*|.*short.*|.*tip.*|.*down.*|.*head.*|.*foote.*|.*info.*|.*last.*|.*next.*|.*title.*|.*label.*|.*error.*|.*view.*|.*center.*|.*date.*|.*paren.*|.*able.*|.*examp.*|.*servi.*|.*class.*|.*prope.*|.*reset.*|.*eract.*|.*rule.*|.*track.*|.*cloud.*|.*call.*|.*index.*|.*limit.*|.*acce.*|.*param.*|.*note.*|.*name.*|.*descr.*|.*value.*|.*polic.*|.*part.*|.*correc.*|.*conte.*|.*media.*|.*exten.*|.*defau.*|.*fill.*|.*unive.*|.*star.*|.*addit.*|.*stac.*|.*excep.*|.*messa.*|.*zone.*|.*total.*|.*type.*|.*valid.*|.*instan.*|.*seria.*|.*docum.*|.*base.*|.*addre.*|.*size.*|.*first.*|.*sourc.*|.*cover.*|.*gener.*|.*config.*|.*with.*|.*window.*|.*support.*|.*comment.*|.*batch.*|.*vault.*|.*secret.*|.*copy.*|.*filte.*|.*clear.*|.*status.*|.*private.*|.*password.*|.*invalid.*|.*link.*|.*test.*|.*expir.*|.*empty.*|.*token.*|null|bearer|foo|foobar|xyz|undefined|none|.*todo.*|.*change.*|.*restore.*|[^A-Za-z0-9]*|%.*%|\\[.*\\]|{.*})$", + "score": -8, + "cases": [ + { + "name": "a public host: a weak password, not an example", + "if": [ + { + "target": "HOST_KIND", + "equals": "public" + } + ], + "score": 8 + }, + { + "name": "random enough to be a secret anyway", + "if": [ + { + "target": "VALUE_ENTROPY", + "method": ">", + "threshold": 3.857142857142857 + } + ], + "score": 4 + } + ] + }, + { + "id": "RC_NATURAL_LANGUAGE", + "name": "A shape match that reads as language", + "description": "A warning sign that costs the match its corroboration, bought back by a random value; a public host lowers the bar to 3 bits per character", + "when": [ + { + "target": "EVIDENCE", + "equals": "shape" + } + ], + "target": "VALUE_NORMALIZED_NATURALNESS_SCORE", + "method": ">=", + "threshold": 0.6, + "score": -8, + "cases": [ + { + "name": "random enough to be a secret anyway", + "if": [ + { + "target": "VALUE_ENTROPY", + "method": ">", + "threshold": 3.857142857142857 + } + ], + "score": 4 + }, + { + "name": "a public host lowers the bar to 3 bits", + "if": [ + { + "target": "HOST_KIND", + "equals": "public" + }, + { + "target": "VALUE_ENTROPY", + "method": ">", + "threshold": 3.0 + } + ], + "score": 4 + } + ] + }, + { + "id": "RC_TEST_PATH", + "name": "A shape match in a test, mock or minified file", + "when": [ + { + "target": "EVIDENCE", + "equals": "shape" + } + ], + "target": "FILEPATH", + "pattern": "(.*\\.min\\.js$|/test/|/tests/|/spec/|/mock/|_test\\.)", + "score": -1 + } +] diff --git a/deepsecrets/rules/regexes.json b/deepsecrets/rules/regexes.json index d6f90b6..fe4f901 100644 --- a/deepsecrets/rules/regexes.json +++ b/deepsecrets/rules/regexes.json @@ -1,48 +1,79 @@ [ { "id": "S0", + "evidence": "format", "name": "Slack Token", "confidence": 10, "pattern": "(?=.{20,256})\\bxox[abpors]-(?:[A-Za-z0-9]+-){1,3}[A-Za-z0-9]+\\b" }, { "id": "S1", + "evidence": "format", "name": "RSA private key", "confidence": 10, - "pattern": "-BEGIN[\\s\\S]{0,10}RSA[\\s\\S]{0,10}PRIVATE[\\s\\S]{0,10}KEY-----[\\s\\S]{50,15000}?-----END[\\s\\S]{0,10}RSA[\\s\\S]{0,10}PRIVATE[\\s\\S]{0,10}KEY-" + "pattern": "-BEGIN[\\s\\S]{0,10}RSA[\\s\\S]{0,10}PRIVATE[\\s\\S]{0,10}KEY-----([\\s\\S]{50,15000}?)-----END[\\s\\S]{0,10}RSA[\\s\\S]{0,10}PRIVATE[\\s\\S]{0,10}KEY-", + "match_rules": { + "1": { + "type": "value", + "encoding": "base64" + } + } }, { "id": "S2", + "evidence": "format", "name": "SSH (OPENSSH) private key", "confidence": 10, - "pattern": "-BEGIN[\\S\\s]{0,10}OPENSSH[\\S\\s]{0,10}PRIVATE[\\S\\s]{0,10}KEY-----[\\s\\S]{50,15000}?-----END[\\S\\s]{0,10}OPENSSH[\\S\\s]{0,10}PRIVATE[\\S\\s]{0,10}KEY-" + "pattern": "-BEGIN[\\S\\s]{0,10}OPENSSH[\\S\\s]{0,10}PRIVATE[\\S\\s]{0,10}KEY-----([\\s\\S]{50,15000}?)-----END[\\S\\s]{0,10}OPENSSH[\\S\\s]{0,10}PRIVATE[\\S\\s]{0,10}KEY-", + "match_rules": { + "1": { + "type": "value", + "encoding": "base64" + } + } }, { "id": "S3", + "evidence": "format", "name": "SSH (DSA) private key", "confidence": 10, - "pattern": "-BEGIN[\\S\\s]{0,10}DSA[\\S\\s]{0,10}PRIVATE[\\S\\s]{0,10}KEY-----[\\s\\S]{50,15000}?-----END[\\S\\s]{0,10}DSA[\\S\\s]{0,10}PRIVATE[\\S\\s]{0,10}KEY-" + "pattern": "-BEGIN[\\S\\s]{0,10}DSA[\\S\\s]{0,10}PRIVATE[\\S\\s]{0,10}KEY-----([\\s\\S]{50,15000}?)-----END[\\S\\s]{0,10}DSA[\\S\\s]{0,10}PRIVATE[\\S\\s]{0,10}KEY-", + "match_rules": { + "1": { + "type": "value", + "encoding": "base64" + } + } }, { "id": "S4", + "evidence": "format", "name": "SSH (EC) private key", "confidence": 10, - "pattern": "-BEGIN[\\S\\s]{0,10}EC[\\S\\s]{0,10}PRIVATE[\\S\\s]{0,10}KEY-----[\\s\\S]{50,15000}?-----END[\\S\\s]{0,10}EC[\\S\\s]{0,10}PRIVATE[\\S\\s]{0,10}KEY-" + "pattern": "-BEGIN[\\S\\s]{0,10}EC[\\S\\s]{0,10}PRIVATE[\\S\\s]{0,10}KEY-----([\\s\\S]{50,15000}?)-----END[\\S\\s]{0,10}EC[\\S\\s]{0,10}PRIVATE[\\S\\s]{0,10}KEY-", + "match_rules": { + "1": { + "type": "value", + "encoding": "base64" + } + } }, { "id": "S5", + "evidence": "format", "name": "PGP private key block", "confidence": 10, "pattern": "-BEGIN[\\S\\s]{0,10}PGP[\\S\\s]{0,10}PRIVATE[\\S\\s]{0,10}KEY[\\S\\s]{0,10}BLOCK-----[^-]([\\s\\S]{50,15000}?)[^-]-----END[\\S\\s]{0,10}PGP[\\S\\s]{0,10}PRIVATE[\\S\\s]{0,10}KEY[\\S\\s]{0,10}BLOCK-", "match_rules": { "1": { - "pattern": ".*", - "negative_pattern": "{|}|<|>|(\\)\\))" + "type": "value", + "encoding": "base64" } } }, { "id": "S7", + "evidence": "format", "name": "Facebook Oauth", "confidence": 10, "pattern": "facebook.{0,100}?['\"][0-9a-f]{32}['\"]", @@ -50,91 +81,124 @@ }, { "id": "S8", + "evidence": "format", "name": "Twitter Oauth", "confidence": 10, "pattern": "twitter.{0,100}?['|\"][0-9a-zA-Z]{35,44}['|\"]" }, { "id": "S10", + "evidence": "format", "name": "Google Oauth", "confidence": 10, "pattern": "(\"client_secret\":\"[a-zA-Z0-9-_]{24}\")" }, { "id": "S12", + "evidence": "format", "name": "Heroku API Key", "confidence": 10, "pattern": "(?i)heroku.*?[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}" }, { "id": "S17", + "evidence": "format", "name": "Slack Webhook", "confidence": 10, "pattern": "https://hooks.slack.com/services/T[a-zA-Z0-9_]+/B[a-zA-Z0-9_]+/[a-zA-Z0-9_]+" }, { "id": "S18", - "name": "Google (GCP) Service-account", + "evidence": "format", + "name": "Google (GCP) service-account private key", "confidence": 10, - "pattern": "\"type\": \"service_account\"" + "pattern": "[\"']?type[\"']?\\s*:\\s*[\"']service_account[\"'][\\s\\S]{0,3000}?[\"']?private_key[\"']?\\s*:\\s*[\"']----(-BEGIN PRIVATE KEY-----((?:[A-Za-z0-9+/=\\s]|\\\\n|\\\\r){50,15000}?)-----END PRIVATE KEY-)----(?:\\\\n)?[\"']", + "target_group": 1, + "match_rules": { + "2": { + "type": "value", + "encoding": "base64" + } + }, + "description": "The private key of an object that declares itself a service account (the key, not the \"type\" marker; the body is base64 and escaped newlines only, so a short placeholder key cannot run into the next object)" }, { "id": "S19", + "evidence": "shape", "name": "Password in URL", "confidence": 10, - "pattern": "://([^.\"/@]+):([^.\"/@]+)@\\S+", + "pattern": "://([^.\"/@]+):([^.\"/@]+)@(\\S+)", "target_group": 2, "match_rules": { + "1": { + "type": "user" + }, "2": { + "type": "value", "pattern": "^[^\\${%<*][^ ]+[^}%>*]$", "negative_pattern": "^(.*symb.*|.*mock.*|.*color.*|.*password.*|.*ctrl.*|.*help.*|.*numb.*|.*long.*|.*short.*|.*tip.*|.*down.*|.*head.*|.*info.*|.*last.*|.*next.*|.*title.*|.*label.*|.*error.*|.*view.*|.*center.*|.*date.*|.*paren.*|.*able.*|.*examp.*|.*servi.*|.*class.*|.*prope.*|.*reset.*|.*eract.*|.*rule.*|.*track.*|.*cloud.*|.*call.*|.*index.*|.*limit.*|.*acces.*|.*param.*|.*note.*|.*name.*|.*descr.*|.*value.*|.*polic.*|.*part.*|.*correc.*|.*conte.*|.*media.*|.*exten.*|.*defau.*|.*fill.*|.*unive.*|.*star.*|.*addit.*|.*stac.*|.*excep.*|.*messa.*|.*zone.*|.*total.*|.*type.*|.*valid.*|.*instan.*|.*seria.*|.*docum.*|.*base.*|.*addre.*|.*size.*|.*first.*|.*sourc.*|.*cover.*|.*gener.*|.*config.*|.*with.*|.*window.*|.*support.*|.*comment.*|.*batch.*|.*vault.*|.*secret.*|.*copy.*|.*filte.*|.*clear.*|.*status.*|.*private.*|.*password.*|.*invalid.*|.*link.*|.*test.*|.*expir.*|.*empty.*|.*token.*|null|bearer|foo|foobar|xyz|undefined|none|.*todo.*|.*change.*|.*restore.*|[^A-Za-z0-9]*|%.*%|\\[.*\\]|{.*})$" + }, + "3": { + "type": "host" } } }, { "id": "S20", + "evidence": "shape", "name": "BAuth", "confidence": 10, "pattern": "Basic @[a-zA-Z0-9+/]+={0,2}" }, { "id": "S22", + "evidence": "format", "name": "Tableau PATS", "confidence": 0, "pattern": "\\bpersonalAccessTokenSecret\\b" }, { "id": "S23", + "evidence": "format", "name": "Tableau PAT", "confidence": 0, "pattern": "\\bpersonalAccessToken\\b" }, { "id": "S25", + "evidence": "format", "name": "Slack App Token", "confidence": 10, "pattern": "xapp-[0-9]+-[A-Za-z0-9_]+-[0-9]+-[a-f0-9]+" }, { "id": "S26", + "evidence": "format", "name": "Custom private key", "confidence": 10, "pattern": "-BEGIN[\\S\\s]{0,10}?PRIVATE[\\S\\s]{0,10}?KEY-----[^-]([\\s\\S]{50,15000}?)[^-]-----END[\\S\\s]{0,10}?PRIVATE[\\S\\s]{0,10}?KEY-", "match_rules": { "1": { - "pattern": ".*", - "negative_pattern": "{|}|<|>|(\\)\\))" + "type": "value", + "encoding": "base64" } } }, { "id": "S28", + "evidence": "shape", "name": "Suspicious password declaration", "pattern": "\\b(pass|password|pwd|passwd)\\b(\\W+)([A-Za-z0-9()$]*)\\b", "confidence": 0, "match_rules": { + "1": { + "type": "name" + }, "2": { "pattern": "^\\s*(?:'|:|=)+\\s*$" + }, + "3": { + "type": "value" } }, "target_group": 3, @@ -142,12 +206,14 @@ }, { "id": "S29", + "evidence": "format", "name": "Ansible vault", "confidence": 10, "pattern": "\\$ANSIBLE_VAULT;[0-9]\\.[0-9];AES256" }, { "id": "S30", + "evidence": "format", "name": "AWS MWS", "confidence": 10, "applicable_file_patterns": [ @@ -157,18 +223,21 @@ }, { "id": "S31", + "evidence": "format", "name": "Github App Token", "confidence": 10, "pattern": "\\b((?:ghp|gho|ghu|ghs|ghr)_[a-zA-Z0-9]{36,255})\\b" }, { "id": "S32", + "evidence": "format", "name": "Github OAuth", "confidence": 10, "pattern": "gho_[0-9a-zA-Z]{36}" }, { "id": "S33", + "evidence": "format", "name": "Github Personal Access Token", "confidence": 10, "pattern": "ghp_[0-9a-zA-Z]{36}" @@ -176,25 +245,38 @@ { "id": "S34", + "evidence": "format", "name": "Gitlab Personal Access Token", "confidence": 10, "pattern": "glpat-[0-9a-zA-Z\\-\\_]{20}" }, { "id": "S35", + "evidence": "shape", "name": "AWS Access Key ID", "confidence": 10, - "pattern": "\\b(A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)[A-Z0-9]{16,17}\\b", - "case_sensitive": true + "pattern": "\\b((A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)[A-Z0-9]{16,17})\\b", + "case_sensitive": true, + "match_rules": { + "1": { + "type": "value", + "encoding": "base32" + }, + "2": { + "type": "kind" + } + } }, { "id": "S36", + "evidence": "format", "description": "https://docs.stripe.com/keys", "name": "Stripe Secret", "confidence": 10, "pattern": "\\b(?:pk|sk|rk)_(?:test|live)_([0-9a-zA-Z]{24,45})\\b", "match_rules": { "1": { + "type": "value", "pattern": ".*", "negative_pattern": "^(.)\\1{10,}" } @@ -202,36 +284,53 @@ }, { "id": "S37", + "evidence": "format", "name": "BitCoin WIF Private Key", "confidence": 7, - "pattern": "\\b[5KL][1-9A-HJ-NP-Za-km-z]{50,51}\\b", - "case_sensitive": true + "pattern": "\\b([5KL][1-9A-HJ-NP-Za-km-z]{50,51})\\b", + "case_sensitive": true, + "match_rules": { + "1": { + "type": "value", + "encoding": "base58check" + } + } }, { "id": "S38", + "evidence": "format", "name": "Google API Key", "confidence": 10, "pattern": "\\bAIzaSy[A-Za-z0-9\\-_]{33}\\b" }, { "id": "S39", + "evidence": "format", "name": "Mailchimp API Key", "confidence": 6, "pattern": "\\b[0-9a-f]{32}-[a-z]{2}\\d{1,2}\\b" }, { "id": "S40", + "evidence": "format", "name": "LocationIQ Key", "confidence": 10, "pattern": "\\bpk\\.[a-zA-Z0-9]{30,32}\\b" }, { "id": "S41", + "evidence": "shape", "name": "SQL Server Connection String", "confidence": 10, - "pattern": "sqlserver://(?:.*?);.*?(?:password)=(.*?)(?:;|\\s|$)", - "target_group": 1 + "pattern": "sqlserver://(.*?);.*?(?:password)=(.*?)(?:;|\\s|$)", + "target_group": 2, + "match_rules": { + "1": { + "type": "host" + }, + "2": { + "type": "value" + } + } } - - -] +] diff --git a/deepsecrets/scan_modes/cli.py b/deepsecrets/scan_modes/cli.py index e4d682e..3eb11a8 100644 --- a/deepsecrets/scan_modes/cli.py +++ b/deepsecrets/scan_modes/cli.py @@ -13,6 +13,7 @@ from deepsecrets.core.model.internal.processing import AnalyzerBundle, PerFileAnalysisResult from deepsecrets.core.rulesets.hashed_secrets import HashedSecretsRulesetBuilder from deepsecrets.core.rulesets.regex import RegexRulesetBuilder +from deepsecrets.core.rulesets.regex_candidate_scoring import RegexCandidateScoringRulesetBuilder from deepsecrets.core.rulesets.variable_scoring import VariableScoringRulesetBuilder from deepsecrets.core.tokenizers.cheap_var_search import CheapVarSearchTokenizer from deepsecrets.core.tokenizers.full_content import FullContentTokenizer @@ -25,8 +26,9 @@ class CliScanMode(ScanMode): - # the per-file analyzer below and everything it imports: engines, tokenizers, the naturalness model - worker_modules = ['deepsecrets.scan_modes.cli'] + # the per-file analyzer below and everything it imports (engines, tokenizers, the naturalness model), and the + # regex-candidate evaluator, which RegexEngine imports only when it judges + worker_modules = ['deepsecrets.scan_modes.cli', 'deepsecrets.core.helpers.regex_candidate_evaluator'] def prepare_for_scan(self) -> None: self.engines_enabled: Dict[str, bool] = {} @@ -112,8 +114,10 @@ def __finalize(result: PerFileAnalysisResult): cheap_var_search = CheapVarSearchTokenizer() lex = LexerTokenizer(deep_token_inspection=True) + # regex matches are candidates, judged as they are found (rules/regex_candidate_scoring_rules.json) regex_engine = RegexEngine( ruleset=bundle.rulesets.get(RegexRulesetBuilder.ruleset_name, []), + candidate_rules=bundle.rulesets.get(RegexCandidateScoringRulesetBuilder.ruleset_name), ) for eng, enabled in bundle.engines.items(): diff --git a/tests/cli/test_cli.py b/tests/cli/test_cli.py index 2d597a4..ee8ddbe 100644 --- a/tests/cli/test_cli.py +++ b/tests/cli/test_cli.py @@ -50,7 +50,8 @@ def test_1_cli(args_1): config = tool.get_current_config() assert config is not None - assert len(config.rulesets) == 3 + # regex, regex-candidate scoring, variable scoring and false findings + assert len(config.rulesets) == 4 assert len(config.engines) == 2 assert len(config.global_exclusion_paths) == 1 diff --git a/tests/core/engines/regex/test_regex.py b/tests/core/engines/regex/test_regex.py index 4d1b9a4..aa0c855 100644 --- a/tests/core/engines/regex/test_regex.py +++ b/tests/core/engines/regex/test_regex.py @@ -4,7 +4,9 @@ from deepsecrets.core.model.file import File from deepsecrets.core.model.rules.regex import RegexRule from deepsecrets.core.model.token import Token +from deepsecrets.core.rulesets.regex_candidate_scoring import RegexCandidateScoringRulesetBuilder from deepsecrets.core.tokenizers.full_content import FullContentTokenizer +from deepsecrets.core.utils.fs import get_path_inside_package from tests.case_helpers import regex_case @@ -57,14 +59,29 @@ def test_go_7(file: File, regex_engine: RegexEngine): @pytest.mark.fixture_file_path('cases/private_keys.txt') def test_private_keys(file: File, regex_engine: RegexEngine): + # the two templates in the file (a key inside HTML entities, PGP armour built in minified code) are matched and + # rejected by the candidate rules as not base64, where a negative pattern on S5 and S26 used to stop the match + candidate_rules = ( + RegexCandidateScoringRulesetBuilder() + .with_rules_from_file(get_path_inside_package('rules/regex_candidate_scoring_rules.json')) + .rules + ) + engine = RegexEngine(ruleset=regex_engine.ruleset, candidate_rules=candidate_rules) findings, tokens, variables = regex_case( tokenizer=FullContentTokenizer(), - engine=regex_engine, + engine=engine, file=file, ) assert len(findings) == 2 + engine.rejected_log = [] + regex_case(tokenizer=FullContentTokenizer(), engine=engine, file=file) + assert sorted((entry['rule'], entry['reason']) for entry in engine.rejected_log) == [ + ('S26', 'RC_NOT_ENCODED'), + ('S5', 'RC_NOT_ENCODED'), + ] + def _token(content: str) -> Token: return Token(file=File(path=None, content=content + '\n'), content=content, span=[0, len(content)]) @@ -84,3 +101,44 @@ def test_rule_match_reports_decoded_content_as_whole_token(): assert rule.match(token) == [(0, len(token.content))] assert rule.match('SECRET_abc') == [] + + +def _service_account(body: str) -> str: + return ( + '{\n "type": "service_account",\n "project_id": "demo-project",\n "private_key_id": "0123abcd",\n' + f' "private_key": "-----BEGIN PRIVATE KEY-----\\n{body}\\n-----END PRIVATE KEY-----\\n",\n' + ' "client_email": "demo@demo-project.iam.gserviceaccount.com"\n}\n' + ) + + +def _scan(content: str, regex_engine: RegexEngine): + file = File(path='service_account.json', relative_path='service_account.json', content=content) + findings, _, _ = regex_case(tokenizer=FullContentTokenizer(), engine=regex_engine, file=file) + return findings + + +def test_s18_names_the_service_account_key_not_the_marker(regex_engine: RegexEngine): + import base64 + import hashlib + + # a synthetic key body: base64 of a hash chain, split into 64-character lines joined by escaped newlines + raw = b''.join(hashlib.sha256(bytes([i])).digest() for i in range(40)) + b64 = base64.b64encode(raw).decode() + body = '\\n'.join(b64[i : i + 64] for i in range(0, len(b64), 64)) + findings = _scan(_service_account(body), regex_engine) + + keys = [f for f in findings if 'S18' in {r.id for r in f.rules}] + assert len(keys) == 1 + # the same span as S26's generic private-key finding, so the two merge into one finding named by S18 + assert {r.id for r in keys[0].rules} == {'S18', 'S26'} + assert keys[0].final_rule.id == 'S18' + assert keys[0].detection.startswith('-BEGIN PRIVATE KEY-----') and keys[0].detection.endswith('END PRIVATE KEY-') + + +def test_s18_ignores_the_marker_and_placeholder_keys(regex_engine: RegexEngine): + marker_only = '{"type": "service_account", "client_email": "demo@demo-project.iam.gserviceaccount.com"}\n' + placeholder = _service_account('MIIEvQIBADAN') # a library fixture's stub: too short to be a key + for content in (marker_only, placeholder): + assert not [f for f in _scan(content, regex_engine) if 'S18' in {r.id for r in f.rules}] + + diff --git a/tests/core/helpers/test_encoding.py b/tests/core/helpers/test_encoding.py new file mode 100644 index 0000000..70823d8 --- /dev/null +++ b/tests/core/helpers/test_encoding.py @@ -0,0 +1,139 @@ +"""Encoded values: payload, foreign share, randomness against a random string, checksum. Values are synthetic.""" + +import base64 +import hashlib +import random +import textwrap +import time + +import pytest + +from deepsecrets.core.helpers import encoding +from deepsecrets.core.helpers.entropy import EntropyHelper + +RANDOM = random.Random(7) +BODY = base64.b64encode(bytes(RANDOM.randrange(256) for _ in range(1190))).decode() +LINES = textwrap.wrap(BODY, 64) + + +def base58check(payload: bytes) -> str: + raw = payload + hashlib.sha256(hashlib.sha256(payload).digest()).digest()[:4] + number, text = int.from_bytes(raw, 'big'), '' + while number: + number, digit = divmod(number, 58) + text = encoding.BASE58[digit] + text + return '1' * (len(raw) - len(raw.lstrip(b'\0'))) + text + + +def randomness(value, declared=None): + return encoding.randomness(value, EntropyHelper.get_for_string(value), declared) + + +FORMS = { + 'pem': '\n'.join(LINES), + 'encrypted pem': 'Proc-Type: 4,ENCRYPTED\nDEK-Info: AES-128-CBC,0123456789ABCDEF\n\n' + '\n'.join(LINES), + 'json string': '\\nProc-Type: 4,ENCRYPTED\\nDEK-Info: AES-128-CBC,0123456789ABCDEF\\n\\n' + + '\\n'.join(LINES) + + '\\n', + 'json, escaped slashes': '\\n'.join(line.replace('/', '\\/') for line in LINES), + 'java concatenation': '\\n" +\n' + '\n'.join(f' "{line}\\n" +' for line in LINES) + '\n "', + 'java, + first': '\n'.join(f' + "{line}\\n"' for line in LINES), + 'one-line concatenation': ' + '.join(f'"{line}"' for line in LINES), + 'go concatenation': '\n'.join(f'\t"{line}\\n" +' for line in LINES) + '\n\t""', + 'python list': '\n'.join(f" '{line}'," for line in LINES), + 'c# interpolation': '\n'.join(f'$"{line}{{lineEnding}}" +' for line in LINES), + 'vb concatenation': '\n'.join(f'"{line}" & vbCrLf & _' for line in LINES), + 'java line separator': '\n'.join(f' "{line}" + System.lineSeparator() +' for line in LINES), + 'php line ending': '\n'.join(f' "{line}" . PHP_EOL .' for line in LINES), + 'php, dot first': '\n'.join(f' . "{line}\\n"' for line in LINES), + 'xml entities': ' \n'.join(LINES), + 'xml hex entities': ' '.join(LINES), + 'html line breaks': '
\n'.join(LINES), + 'shell comment': '\n'.join(f'# {line}' for line in LINES), + 'block comment': '\n'.join(f' * {line}' for line in LINES), + 'markdown quote': '\n'.join(f'> {line}' for line in LINES), + 'removed in a diff': '\n'.join(f'-{line}' for line in textwrap.wrap(BODY, 40)), + 'yaml block': '\n'.join(f' {line}' for line in LINES), +} + + +def test_another_private_keys_armour_inside_a_body_is_dropped(): + # a diff that removes one key and adds another: the removed key's END line (------END…) is not an end marker to + # the key rules, so the match runs into the added key. Both halves are key material + removed = '\n'.join(f'-{line}' for line in LINES[:12]) + added = '\n'.join(f'+{line}' for line in LINES[12:]) + written = removed + '\n------END RSA PRIVATE KEY-----\n+-----BEGIN RSA PRIVATE KEY-----\n' + added + assert encoding.foreign_share(encoding.payload(written), 'base64') == 0 + + +@pytest.mark.parametrize('written', FORMS.values(), ids=FORMS.keys()) +def test_the_payload_is_the_same_however_the_body_is_written(written): + assert encoding.payload(written) == BODY + + +@pytest.mark.parametrize( + 'written', + [ + '\\n" + "\\n" + key.replace(/(.{64})/g, "$1\\n") + "\\n', + '\\n";\n result += base64.encode(chunk);\n result += "\\n', + 'MIIEpAIBAAKCAQEA...\n...your private key goes here, one line per 64 characters...', + '\nBad key, the certificate is fine\n-----END RSA PRIVATE KEY-----\n-----BEGIN CERTIFICATE-----\n' + LINES[0], + ], + ids=['code building a key', 'code appending chunks', 'ellipsis and prose', 'a match across two blocks'], +) +def test_code_and_placeholders_between_the_markers_stay_foreign(written): + assert encoding.foreign_share(encoding.payload(written), 'base64') > 0.02 + + +def test_an_ellipsis_before_the_closing_quote_stays_foreign(): + # a dot is decoration next to a quote only after it (PHP's "…" . PHP_EOL), not before it: "MIIE..." is truncated + assert encoding.foreign_share(encoding.payload('"' + LINES[0][:20] + '..."'), 'base64') > 0.02 + + +@pytest.mark.parametrize('text', ['-' * 20000, '"' + ' ' * 20000, '-----BEGIN RSA PRIVATE KEY-----\n' + '-' * 20000]) +def test_long_runs_take_linear_time(text): + # a long run of dashes or of blanks after a quote used to take seconds (every start of the run was retried) + started = time.perf_counter() + encoding.payload(text) + assert time.perf_counter() - started < 0.5 + + +def test_the_foreign_share_counts_characters_outside_the_alphabet(): + assert encoding.foreign_share(BODY, 'base64') == 0 + assert encoding.foreign_share('MIIEpAIBAAKCAQEA...' + 'A' * 81, 'base64') == 0.03 + assert encoding.foreign_share('AKIA0000000000000000', 'base32') == 0.8 + assert encoding.foreign_share('anything at all', None) == 0 + + +def test_the_expected_entropy_of_a_random_string(): + assert encoding.expected_entropy(1, 64) == 0 + lengths = [encoding.expected_entropy(n, 64) for n in (16, 64, 256, 1024, 2048)] + assert lengths == sorted(lengths) and lengths[-1] < 6 + # the approximation takes over where the exact sum stops, without a step + edge = encoding.EXACT_UP_TO + assert abs(encoding.expected_entropy(edge, 64) - encoding.expected_entropy(edge + 1, 64)) < 1e-3 + + +def test_randomness_is_about_one_for_a_random_value_and_low_for_structure(): + assert randomness(BODY, 'base64') > 0.98 + assert randomness('A' * 1024, 'base64') == 0 + assert randomness('AKIAAAAAAAAAAAAAAAAA', 'base32') < 0.75 + assert randomness('xoxb-XXXXXXXXXX-XXXXXXXXXX-XXXXXXXXXXXX') < 0.75 + + +@pytest.mark.parametrize( + 'value, symbols', + [('0123456789abcdef', 16), ('ABCDEFGHIJKLMNOPQRSTUVWXYZ234567', 32), ('xyz123', 36), ('aB3', 62), ('aB3-_', 64)], +) +def test_without_an_encoding_the_alphabet_is_read_from_the_value(value, symbols): + assert encoding.symbols(value, None) == symbols + + +def test_a_base58check_value_carries_its_checksum(): + key = base58check(b'\x80' + bytes(RANDOM.randrange(256) for _ in range(32))) + assert len(key) == 51 and key[0] == '5' + assert encoding.checksum(key, 'base58check') == 'valid' + altered = key[:-1] + ('2' if key[-1] != '2' else '3') + assert encoding.checksum(altered, 'base58check') == 'invalid' + assert encoding.checksum(key[:-1] + '0', 'base58check') == 'invalid' # 0 is not base58 + assert encoding.checksum(key, 'base64') == 'none' diff --git a/tests/core/helpers/test_regex_candidate_evaluator.py b/tests/core/helpers/test_regex_candidate_evaluator.py new file mode 100644 index 0000000..598cd6c --- /dev/null +++ b/tests/core/helpers/test_regex_candidate_evaluator.py @@ -0,0 +1,324 @@ +"""Regex candidates: typed groups, their own scoring rules, judgement in RegexEngine. Values are synthetic.""" + +import pytest + +from deepsecrets.core.engines.regex import RegexEngine +from deepsecrets.core.helpers.regex_candidate_evaluator import REJECT_AT, RegexCandidateEvaluator +from deepsecrets.core.model.file import File +from deepsecrets.core.model.finding import Finding +from deepsecrets.core.model.regex_candidate import RegexCandidateContext, host_kind +from deepsecrets.core.model.rules.regex import Evidence, GroupType, RegexRule +from deepsecrets.core.model.rules.regex_candidate_scoring import CandidateCondition, RegexCandidateScoringRule +from deepsecrets.core.model.rules.rule import Rule +from deepsecrets.core.rulesets.regex import RegexRulesetBuilder +from deepsecrets.core.rulesets.regex_candidate_scoring import RegexCandidateScoringRulesetBuilder +from deepsecrets.core.utils.fs import get_path_inside_package + +PATH = '/repo/src/settings.py' + + +@pytest.fixture(scope='module') +def candidate_rules(): + return ( + RegexCandidateScoringRulesetBuilder() + .with_rules_from_file(get_path_inside_package('rules/regex_candidate_scoring_rules.json')) + .rules + ) + + +@pytest.fixture(scope='module') +def rules(): + built = RegexRulesetBuilder().with_rules_from_file(get_path_inside_package('rules/regexes.json')).rules + return {rule.id: rule for rule in built} + + +def judge(candidate_rules, rule, text, path=PATH): + match = rule.matches(text)[0] + context = RegexCandidateContext.from_match(rule, match.match, text[match.start : match.end], path) + return context, RegexCandidateEvaluator(candidate_rules).evaluate(context) + + +def test_groups_are_typed_in_match_rules(rules): + s19, s41, s36 = rules['S19'], rules['S41'], rules['S36'] + assert (s19.group_of(GroupType.USER), s19.group_of(GroupType.VALUE), s19.group_of(GroupType.HOST)) == (1, 2, 3) + assert (s41.group_of(GroupType.HOST), s41.group_of(GroupType.VALUE)) == (1, 2) + assert s36.group_of(GroupType.VALUE) == 1 and s36.group_of(GroupType.HOST) is None + + +def test_a_group_type_is_declared_once(): + with pytest.raises(ValueError): + RegexRule(id='T', pattern='(a)(b)', match_rules={'1': {'type': 'host'}, '2': {'type': 'host'}}) + + +def test_a_typed_group_needs_no_pattern(): + rule = RegexRule(id='T', pattern='(a)(b)', match_rules={'1': {'type': 'name'}}) + assert rule.match('ab') == [(0, 2)] + + +def test_the_candidate_reads_its_groups(candidate_rules, rules): + context, _ = judge(candidate_rules, rules['S19'], 'db = "postgres://app:Tq7XbR2mKp9LvZ4w@db.billing-prod.net/x"') + assert (context.user, context.value, context.host_kind) == ('app', 'Tq7XbR2mKp9LvZ4w', 'public') + + +@pytest.mark.parametrize( + 'address, kind', + [ + ('db.billing-prod.net/x', 'public'), + ('localhost:5432', 'local'), + ('db:27017', 'local'), + ('foobar.com:1234/x', 'placeholder'), + ('api.example.com', 'placeholder'), + ('', 'none'), + ], +) +def test_host_kind(address, kind): + assert host_kind(address) == kind + + +def test_a_format_match_keeps_its_confidence_and_is_rejected_only_for_an_unrandom_placeholder(candidate_rules, rules): + # S36 judges the key body: the 'test' of sk_test_ is the key's mode, not a placeholder word + _, kept = judge(candidate_rules, rules['S36'], "k = 'sk_test_a1b2c3d4a1b2c3d4e5f6e5f6a1b2c3d4'") + assert not kept.rejected and kept.confidence is None + _, rejected = judge(candidate_rules, rules['S36'], "k = 'sk_test_exampleexampleexampleexa'") + assert rejected.rejected and rejected.reason == 'RC_FORMAT_PLACEHOLDER' + + +def test_a_shape_match_gets_a_confidence_from_its_value(candidate_rules, rules): + _, random = judge(candidate_rules, rules['S19'], 'postgres://app:Tq7XbR2mKp9LvZ4w@localhost/x') + assert not random.rejected and random.confidence == 9 + _, in_tests = judge( + candidate_rules, rules['S19'], 'postgres://app:Tq7XbR2mKp9LvZ4w@localhost/x', '/repo/tests/a.py' + ) + assert in_tests.confidence == 8 and in_tests.fired == {'RC_TEST_PATH': None} and in_tests.total == -1 + + +def test_a_word_as_a_password_is_rejected(candidate_rules, rules): + _, verdict = judge(candidate_rules, rules['S19'], 'postgres://app:sunshine@localhost/x') + assert verdict.rejected and verdict.reason == 'RC_NATURAL_LANGUAGE' and verdict.confidence == 0 + assert verdict.total == REJECT_AT and verdict.fired == {'RC_NATURAL_LANGUAGE': None} + + +def test_a_public_host_lowers_the_bar(candidate_rules, rules): + text = 'postgres://app:{}@{}/x' + _, public = judge(candidate_rules, rules['S19'], text.format('orangeelephantpillow', 'db.billing-prod.net')) + _, local = judge(candidate_rules, rules['S19'], text.format('orangeelephantpillow', 'localhost')) + assert not public.rejected and local.rejected + + +@pytest.mark.parametrize('host, kept', [('sqlprod.database.windows.net', True), ('localhost', False)]) +def test_a_placeholder_word_does_not_count_for_a_public_host(candidate_rules, rules, host, kept): + text = f'conn = "jdbc:sqlserver://{host}:1433;database=app;user=svc;password=MyPassword#2024;"' + _, verdict = judge(candidate_rules, rules['S41'], text) + assert verdict.rejected is not kept + + +def scan(engine, content, corroborated=False): + file = File(path=PATH, relative_path='src/settings.py', content=content) + from deepsecrets.core.model.token import Token + + token = Token(file=file, content=content, span=[0, len(content)]) + return engine.search(token, corroborated=corroborated) + + +def test_the_engine_drops_a_rejected_candidate_unless_asked(candidate_rules, rules): + content = 'url = "postgres://app:sunshine@localhost/x"' + assert scan(RegexEngine(ruleset=[rules['S19']], candidate_rules=candidate_rules), content) == [] + engine = RegexEngine(ruleset=[rules['S19']], candidate_rules=candidate_rules, report_rejected=True) + [finding] = scan(engine, content) + assert finding.rejected and finding.rules[0].confidence == 0 and finding.rules[0].is_dynamic_confidence + + +def test_the_engine_logs_rejections_for_the_tracer(candidate_rules, rules): + engine = RegexEngine(ruleset=[rules['S19']], candidate_rules=candidate_rules) + engine.rejected_log = [] + scan(engine, 'url = "postgres://app:sunshine@localhost/x"') + assert engine.rejected_log[0]['rule'] == 'S19' and engine.rejected_log[0]['reason'] == 'RC_NATURAL_LANGUAGE' + + +def test_a_match_the_semantic_engine_corroborated_is_not_judged_again(candidate_rules, rules): + # the whole token is the value of a variable the semantic engine reported: the rule keeps its own confidence + engine = RegexEngine(ruleset=[rules['S35']], candidate_rules=candidate_rules) + assert scan(engine, 'AKIAIOSFODNN7EXAMPLE') == [] + [finding] = scan(engine, 'AKIAIOSFODNN7EXAMPLE', corroborated=True) + assert finding.rules[0].confidence == 10 and not finding.rules[0].is_dynamic_confidence + + +def test_a_rule_without_an_evidence_class_is_not_judged(candidate_rules): + custom = RegexRule(id='CUSTOM1', pattern='sunshine') + [finding] = scan(RegexEngine(ruleset=[custom], candidate_rules=candidate_rules), 'x = "sunshine"') + assert not finding.rejected and finding.rules[0].confidence == 10 + + +def test_merging_keeps_one_rule_per_id(): + dynamic = Rule(id='S35', confidence=7, is_dynamic_confidence=True) + static = Rule(id='S35', confidence=10) + # a corroborated (static) copy beats a judged (dynamic) one + a = Finding(detection='x', start_offset=0, end_offset=1, rules=[dynamic]) + a.merge(Finding(detection='x', start_offset=0, end_offset=1, rules=[static])) + assert a.rules == [static] and a.rules[0].confidence == 10 + # between two dynamic copies the first seen stays, as before + low, high = Rule(id='S105', confidence=4, is_dynamic_confidence=True), Rule( + id='S105', confidence=9, is_dynamic_confidence=True + ) + b = Finding(detection='x', start_offset=0, end_offset=1, rules=[low]) + b.merge(Finding(detection='x', start_offset=0, end_offset=1, rules=[high])) + assert b.rules[0].confidence == 4 + # a kept finding's copy beats a rejected candidate's, and the merge is rejected only when both are + rejected = Finding( + detection='x', + start_offset=0, + end_offset=1, + rules=[Rule(id='S19', confidence=0, is_dynamic_confidence=True)], + rejected=True, + ) + rejected.merge( + Finding( + detection='x', + start_offset=0, + end_offset=1, + rules=[Rule(id='S19', confidence=6, is_dynamic_confidence=True)], + ) + ) + assert rejected.rules[0].confidence == 6 and not rejected.rejected + + +def test_a_random_value_buys_a_warning_back_through_a_case(candidate_rules, rules): + # a random value keeps the match but not its corroboration: -8 + 4 = -4, above the rejection line + context, verdict = judge(candidate_rules, rules['S36'], "k = 'sk_live_Xk9pQ2mZ7rT4vW8yB3nL6hJexample'") + assert not verdict.rejected and verdict.total == -4 + assert verdict.fired == {'RC_FORMAT_PLACEHOLDER': 'random enough to be a secret anyway'} + + +def test_a_public_host_cancels_a_placeholder_word(candidate_rules, rules): + text = 'conn = "jdbc:sqlserver://sqlprod.database.windows.net:1433;database=app;password=MyPassword#2024;"' + _, verdict = judge(candidate_rules, rules['S41'], text) + assert verdict.fired['RC_PLACEHOLDER'] == 'a public host: a weak password, not an example' + + +def rule(**fields): + fields.setdefault('id', 'T') + return RegexCandidateScoringRule(target='VALUE', pattern='.', **fields) + + +def test_only_the_first_case_that_holds_counts(): + context = RegexCandidateContext(name='', value='Tq7XbR2mKp9LvZ4w', filepath=PATH, evidence='shape') + both = rule( + score=-8, + cases=[ + {'name': 'first', 'if': [{'target': 'VALUE_ENTROPY', 'method': '>', 'threshold': 1}], 'score': 4}, + {'name': 'second', 'if': [{'target': 'VALUE_LENGTH', 'method': '>', 'threshold': 1}], 'score': 8}, + ], + ) + verdict = RegexCandidateEvaluator([both]).evaluate(context) + assert verdict.total == -4 and verdict.fired == {'T': 'first'} + + +def test_rules_add_up_and_reject_at_the_line(): + context = RegexCandidateContext(name='', value='Tq7XbR2mKp9LvZ4w', filepath=PATH, evidence='shape') + four = [rule(id=f'T{i}', score=-4) for i in range(2)] + assert RegexCandidateEvaluator(four[:1]).evaluate(context).rejected is False + assert RegexCandidateEvaluator(four).evaluate(context).rejected is True + + +def test_categorical_fields_are_compared_by_equality(): + context = RegexCandidateContext(name='', value='x', filepath=PATH, evidence='shape', host='db:5432') + assert CandidateCondition(target='EVIDENCE', equals='shape').holds(context) + assert not CandidateCondition(target='EVIDENCE', equals='shap').holds(context) + assert CandidateCondition(**{'target': 'HOST_KIND', 'in': ['local', 'none']}).holds(context) + + +@pytest.mark.parametrize( + 'fields', + [{}, {'equals': 'shape', 'pattern': 'x'}, {'method': '~', 'threshold': 1}, {'method': '>'}], +) +def test_a_condition_takes_exactly_one_form(fields): + with pytest.raises(ValueError): + CandidateCondition(target='EVIDENCE', **fields) + + +def test_the_evidence_label_is_format_or_shape(): + assert RegexRule(id='T', pattern='x', evidence='shape').evidence == Evidence.SHAPE + with pytest.raises(ValueError): + RegexRule(id='T', pattern='x', evidence='context') + + +def test_only_the_value_group_declares_an_encoding(): + with pytest.raises(ValueError): + RegexRule(id='T', pattern='(a)(b)', match_rules={'1': {'type': 'host', 'encoding': 'base64'}}) + assert RegexRule(id='T', pattern='(a)', match_rules={'1': {'type': 'value', 'encoding': 'hex'}}).encoding_of_value() + + +def pem(body_lines): + return '-----BEGIN RSA PRIVATE KEY-----\n' + '\n'.join(body_lines) + '\n-----END RSA PRIVATE KEY-----' + + +def test_a_key_rule_judges_the_body_not_the_armour(candidate_rules, rules): + import base64 + import random + import textwrap + + generator = random.Random(3) + body = base64.b64encode(bytes(generator.randrange(256) for _ in range(900))).decode() + context, kept = judge(candidate_rules, rules['S1'], pem(textwrap.wrap(body, 64))) + # the armour's PRIVATE no longer reaches the placeholder rule: the value is the body + assert context.value == body and context.encoding == 'base64' and not kept.rejected and kept.fired == {} + + +@pytest.mark.parametrize( + 'lines, reason', + [ + (['MIIEpAIBAAKCAQEA...', '...your private key goes here, one line per 64 characters...'], 'RC_NOT_ENCODED'), + (['A' * 64] * 4, 'RC_LONG_NOT_RANDOM'), + (['MIIEpAIBAAKCAQEA' + 'A' * 48, 'A' * 16], 'RC_NOT_RANDOM'), + ], + ids=['ellipsis', 'one repeated character, long', 'mostly one character, short'], +) +def test_a_key_body_that_is_not_key_material_is_rejected(candidate_rules, rules, lines, reason): + _, verdict = judge(candidate_rules, rules['S1'], pem(lines)) + assert verdict.rejected and verdict.reason == reason + + +def test_an_aws_key_id_is_base32(candidate_rules, rules): + context, verdict = judge(candidate_rules, rules['S35'], 'id = "AKIA0000000000000000"') + assert (context.kind, context.encoding) == ('AKIA', 'base32') + assert verdict.rejected and verdict.reason == 'RC_NOT_ENCODED' + + +def test_a_wif_key_needs_its_checksum(candidate_rules, rules): + import hashlib + + from deepsecrets.core.helpers.encoding import BASE58 + + raw = b'\x80' + hashlib.sha256(b'a synthetic key').digest() + raw += hashlib.sha256(hashlib.sha256(raw).digest()).digest()[:4] + number, key = int.from_bytes(raw, 'big'), '' + while number: + number, digit = divmod(number, 58) + key = BASE58[digit] + key + context, kept = judge(candidate_rules, rules['S37'], f'wif = "{key}"') + assert context.value_checksum == 'valid' and not kept.rejected + altered = key[:-1] + ('2' if key[-1] != '2' else '3') + _, rejected = judge(candidate_rules, rules['S37'], f'wif = "{altered}"') + assert rejected.rejected and rejected.reason == 'RC_BAD_CHECKSUM' + + +def test_a_placeholder_token_is_far_from_random(candidate_rules, rules): + _, verdict = judge(candidate_rules, rules['S0'], 'token = "xoxb-XXXXXXXXXX-XXXXXXXXXX-XXXXXXXXXXXX"') + assert verdict.rejected and verdict.reason == 'RC_NOT_RANDOM' + _, kept = judge(candidate_rules, rules['S0'], 'token = "xoxb-123456789012-123456789012-Zk3pQ9rT2mX7vB1nW4yL8hJd"') + assert not kept.rejected + + +def test_a_comparison_on_a_text_field_is_refused_when_the_rules_load(): + # at scan time it would raise TypeError ('<' between str and float) and abandon the rest of the file + with pytest.raises(ValueError, match='numeric field'): + RegexCandidateScoringRule(id='X', target='VALUE', method='<', threshold=3, score=-8) + RegexCandidateScoringRule(id='X', target='VALUE_LENGTH', method='<', threshold=3, score=-8) + + +@pytest.mark.parametrize('report_rejected', [False, True]) +def test_an_empty_match_is_never_reported(candidate_rules, rules, report_rejected): + # S41's password group can match nothing (password=;): 2.1 reported an empty detection, 2.2 rejected it + engine = RegexEngine(ruleset=[rules['S41']], candidate_rules=candidate_rules, report_rejected=report_rejected) + assert scan(engine, 'conn = "sqlserver://db.example.org;user=sa;password=;"') == [] + assert scan(RegexEngine(ruleset=[rules['S41']]), 'conn = "sqlserver://db.example.org;user=sa;password=;"') == [] From 816f23b2584b9b51903d1a9798f7516a665cd2b7 Mon Sep 17 00:00:00 2001 From: Nikolai Khechumov Date: Mon, 28 Sep 2026 09:29:04 +0000 Subject: [PATCH 12/22] Add --confidence-level all, low (default), medium, high or very-high: the lowest confidence to report. all also reports the regex candidates the evaluation rejected, at confidence 0 with an -INFO rule id. --- deepsecrets/cli.py | 14 ++++ deepsecrets/config.py | 12 +++ deepsecrets/core/model/internal/processing.py | 2 + deepsecrets/core/model/response/dojo_sarif.py | 35 +++++---- deepsecrets/core/modes/iscan_mode.py | 19 ++++- deepsecrets/scan_modes/cli.py | 2 + tests/output/test_confidence_levels.py | 78 +++++++++++++++++++ .../scan_modes/test_scan_mode_regressions.py | 1 + 8 files changed, 147 insertions(+), 16 deletions(-) create mode 100644 tests/output/test_confidence_levels.py diff --git a/deepsecrets/cli.py b/deepsecrets/cli.py index acd1899..db44f3a 100644 --- a/deepsecrets/cli.py +++ b/deepsecrets/cli.py @@ -7,6 +7,8 @@ from deepsecrets import MODULE_NAME, console from deepsecrets.config import ( + CONFIDENCE_LEVELS, + DEFAULT_CONFIDENCE_LEVEL, DEFAULT_DEEP_MAX_SIZE, SCANNER_VERSION, SCANNER_VERSION_NUMERIC, @@ -183,6 +185,17 @@ def _build_argparser(self) -> None: f'Default: {DEFAULT_DEEP_MAX_SIZE}. 0 gives every file the full analysis.\n', ) + parser.add_argument( + '--confidence-level', + type=str, + default=DEFAULT_CONFIDENCE_LEVEL, + choices=list(CONFIDENCE_LEVELS), + help='The lowest confidence to report, from all to very-high (default: low).\n' + '"low" reports every finding; "medium", "high" and "very-high" keep confidence 3, 6 and 9 or more.\n' + '"all" also reports the regex matches the scanner judged not to be secrets (placeholders, examples),\n' + 'at confidence 0 with an -INFO rule id.\n', + ) + parser.add_argument( '--false-findings', nargs='*', @@ -285,6 +298,7 @@ def parse_arguments(self) -> None: # set on every parse: the config singleton outlives one run config.set_report_diagnostics(user_args.report_diagnostics) + config.set_confidence_level(user_args.confidence_level) if user_args.benchmarking_mode: config._set_benchmarking_mode(True) diff --git a/deepsecrets/config.py b/deepsecrets/config.py index 9e352a4..ab1db7d 100644 --- a/deepsecrets/config.py +++ b/deepsecrets/config.py @@ -11,6 +11,11 @@ FALLBACK_PROCESS_COUNT = 4 +# --confidence-level: the lowest confidence a reported finding may have. 'all' also reports the regex candidates their +# evaluation rejected (confidence 0); 'low', the default, reports every finding the scanner keeps +CONFIDENCE_LEVELS = {'all': None, 'low': 0, 'medium': 3, 'high': 6, 'very-high': 9} +DEFAULT_CONFIDENCE_LEVEL = 'low' + # files larger than this many bytes get the shallow analysis: the regex rules and the cheap variable search, no lexer. # On SecretBench the lexer found 12 of its 21 unique secrets in smaller files and the other 9 only in files of 1 MB # or more (minified JavaScript, one YAML file), while files over 250 KB took two thirds of its time. @@ -51,6 +56,7 @@ class Config: return_code_if_findings: bool disable_masking: bool report_diagnostics: bool = False + confidence_level: str = DEFAULT_CONFIDENCE_LEVEL verbose: bool = False # no live terminal UI and no progress manager; the CLI turns it on in CI and when output is not a terminal ci_mode: bool = False @@ -64,6 +70,7 @@ def __init__(self) -> None: self.return_code_if_findings = False self.disable_masking = False self.report_diagnostics = False + self.confidence_level = DEFAULT_CONFIDENCE_LEVEL self._benchmarking_mode = False self.oneshot_path = None @@ -87,6 +94,11 @@ def set_disable_masking(self, state: bool): def set_report_diagnostics(self, state: bool): self.report_diagnostics = state + def set_confidence_level(self, level: str) -> None: + if level not in CONFIDENCE_LEVELS: + raise ValueError(f'unknown confidence level {level}') + self.confidence_level = level + def _set_path(self, path: str, field: str) -> None: if not path_exists(path): raise FileNotFoundException(f'{field} path does not exist ({path})') diff --git a/deepsecrets/core/model/internal/processing.py b/deepsecrets/core/model/internal/processing.py index 73cc765..473c127 100644 --- a/deepsecrets/core/model/internal/processing.py +++ b/deepsecrets/core/model/internal/processing.py @@ -22,6 +22,8 @@ class AnalyzerBundle: benchmarking_mode: bool = False # see deep_analysis deep_max_size: int = DEFAULT_DEEP_MAX_SIZE + # --confidence-level all: regex candidates their evaluation rejected are reported, flagged, at confidence 0 + report_rejected: bool = False @dataclass diff --git a/deepsecrets/core/model/response/dojo_sarif.py b/deepsecrets/core/model/response/dojo_sarif.py index 15f52fa..3fa125f 100644 --- a/deepsecrets/core/model/response/dojo_sarif.py +++ b/deepsecrets/core/model/response/dojo_sarif.py @@ -1,6 +1,6 @@ import os from dataclasses import dataclass -from typing import List, Set +from typing import List, Set, Tuple from sarif_om import ( SarifLog, @@ -62,19 +62,23 @@ class DojoSarifResponseBuilder(BaseResponseBuilder): (3, 6): {'suffix': '-MEDIUM', 'precision': 'medium', 'label': 'Medium'}, (float('-inf'), 3): {'suffix': '-LOW', 'precision': 'low', 'label': 'Low'}, } + # a regex candidate its evaluation rejected, reported only at --confidence-level all + INFO_TIER = {'suffix': '-INFO', 'precision': 'low', 'label': 'Info'} - def _get_tier(self, confidence: int): + def _get_tier(self, confidence: int, rejected: bool = False): + if rejected: + return self.INFO_TIER for (start, end), value in self.CONFIDENCE_TIERS.items(): if start <= confidence < end: return value - def _sarif_rule_meta_from_rule(self, rule: Rule) -> TierAwareSarifRuleMeta: + def _sarif_rule_meta_from_rule(self, rule: Rule, rejected: bool = False) -> TierAwareSarifRuleMeta: base_rule_id = rule.id base_description = rule.name - tier = self._get_tier(rule.confidence) - suffix = tier.get('suffix') if rule.is_dynamic_confidence is True else '' + tier = self._get_tier(rule.confidence, rejected) + suffix = tier.get('suffix') if rule.is_dynamic_confidence is True or rejected else '' return TierAwareSarifRuleMeta( id=f'{base_rule_id}{suffix}', @@ -125,25 +129,26 @@ def _convert_rules(self, rules: Set[TierAwareSarifRuleMeta]) -> List[ReportingDe def build(self) -> SarifLog: # type: ignore - rules: List[Rule] = list() + rules: List[Tuple[Rule, bool]] = list() for finding in self.findings: finding.choose_final_rule() region = self.get_region(finding=finding, masking=self.masking_enabled) context_region = self.get_context_region(finding=finding, masking=self.masking_enabled) - rules.append(finding.final_rule) - rule_meta = self._sarif_rule_meta_from_rule(finding.final_rule) + rules.append((finding.final_rule, finding.rejected)) + rule_meta = self._sarif_rule_meta_from_rule(finding.final_rule, finding.rejected) + properties = {'confidence': finding.final_rule.confidence} + text = f'[Confidence {finding.final_rule.confidence}/10] Secret in code: {finding.final_rule.name}' + if finding.rejected: + properties['rejected'] = True + text = f'[Rejected candidate] Possibly a secret in code: {finding.final_rule.name}' result = Result( rule_id=rule_meta.id, level='error', - properties={ - 'confidence': finding.final_rule.confidence, - }, - message=Message( - text=f'[Confidence {finding.final_rule.confidence}/10] Secret in code: {finding.final_rule.name}' - ), + properties=properties, + message=Message(text=text), locations=[ Location( physical_location=PhysicalLocation( @@ -160,7 +165,7 @@ def build(self) -> SarifLog: # type: ignore self.report.runs[0].results.append(result) - sarif_rules = self._convert_rules(set([self._sarif_rule_meta_from_rule(rule) for rule in rules])) + sarif_rules = self._convert_rules(set([self._sarif_rule_meta_from_rule(rule, rej) for rule, rej in rules])) self.report.runs[0].tool.driver.rules = sarif_rules mode = getattr(self, 'mode', None) diff --git a/deepsecrets/core/modes/iscan_mode.py b/deepsecrets/core/modes/iscan_mode.py index ce8aede..b14e2cc 100644 --- a/deepsecrets/core/modes/iscan_mode.py +++ b/deepsecrets/core/modes/iscan_mode.py @@ -20,7 +20,7 @@ from typing import Any, Callable, Dict, List, Optional, Tuple, Type from deepsecrets import PROFILER_ON, console -from deepsecrets.config import Config +from deepsecrets.config import CONFIDENCE_LEVELS, DEFAULT_CONFIDENCE_LEVEL, Config from deepsecrets.core.model.file import File from deepsecrets.core.model.finding import Finding from deepsecrets.core.model.internal.processing import AnalyzerBundle, PerFileAnalysisResult @@ -466,6 +466,7 @@ def run(self) -> Tuple[List[Finding], Dict[str, List[str]], Dict[str, int]]: console.print('[*] Filtering predefined false Findings..') fin = self.filter_false_positives(fin) + fin = self.filter_by_confidence_level(fin) return fin, self.per_file_errors(), self.per_file_timings() def dispose(self): @@ -583,6 +584,22 @@ def analyzer_bundle(self) -> AnalyzerBundle: def _per_file_analyzer(bundle: AnalyzerBundle, file: Any, task_id: Optional[int] = None, task_reporter: Optional[Any] = None) -> PerFileAnalysisResult: # type: ignore pass + def filter_by_confidence_level(self, results: List[Finding]) -> List[Finding]: + """--confidence-level: 'all' reports everything, rejected regex candidates included; the other levels drop those + and keep findings whose final rule has at least the level's confidence.""" + minimum = CONFIDENCE_LEVELS.get(getattr(self.config, 'confidence_level', DEFAULT_CONFIDENCE_LEVEL)) + if minimum is None: + return results + + kept: List[Finding] = [] + for finding in results: + if finding.rejected: + continue + finding.choose_final_rule() + if finding.final_rule.confidence >= minimum: + kept.append(finding) + return kept + def filter_false_positives(self, results: List[Finding]) -> List[Finding]: false_finding_rules = self.rulesets.get(FalseFindingsBuilder.ruleset_name) if false_finding_rules is None: diff --git a/deepsecrets/scan_modes/cli.py b/deepsecrets/scan_modes/cli.py index 3eb11a8..84cf189 100644 --- a/deepsecrets/scan_modes/cli.py +++ b/deepsecrets/scan_modes/cli.py @@ -54,6 +54,7 @@ def analyzer_bundle(self) -> AnalyzerBundle: engines=self.engines_enabled, rulesets=self.rulesets, deep_max_size=self.config.deep_max_size, + report_rejected=self.config.confidence_level == 'all', ) @staticmethod @@ -118,6 +119,7 @@ def __finalize(result: PerFileAnalysisResult): regex_engine = RegexEngine( ruleset=bundle.rulesets.get(RegexRulesetBuilder.ruleset_name, []), candidate_rules=bundle.rulesets.get(RegexCandidateScoringRulesetBuilder.ruleset_name), + report_rejected=bundle.report_rejected, ) for eng, enabled in bundle.engines.items(): diff --git a/tests/output/test_confidence_levels.py b/tests/output/test_confidence_levels.py new file mode 100644 index 0000000..1e104c6 --- /dev/null +++ b/tests/output/test_confidence_levels.py @@ -0,0 +1,78 @@ +"""Confidence tiers in SARIF, the INFO tier of rejected candidates, and --confidence-level.""" + +from types import SimpleNamespace + +import pytest + +from deepsecrets.cli import DeepSecretsCliTool +from deepsecrets.config import Config +from deepsecrets.core.model.finding import Finding +from deepsecrets.core.model.response.dojo_sarif import DojoSarifResponseBuilder +from deepsecrets.core.model.rules.rule import Rule +from deepsecrets.core.modes.iscan_mode import ScanMode + + +def meta(rule, rejected=False): + return DojoSarifResponseBuilder()._sarif_rule_meta_from_rule(rule, rejected) + + +def test_the_tier_names_the_confidence_and_the_severity_stays_the_same(): + high = meta(Rule(id='S35', name='AWS Access Key ID', confidence=7, is_dynamic_confidence=True)) + low = meta(Rule(id='S35', name='AWS Access Key ID', confidence=1, is_dynamic_confidence=True)) + static = meta(Rule(id='S0', name='Slack Token', confidence=10)) + assert (high.id, low.id, static.id) == ('S35-HIGH', 'S35-LOW', 'S0') + assert high.payload['properties']['precision'] == 'high' and low.payload['properties']['precision'] == 'low' + severities = {m.payload['properties']['security-severity'] for m in (high, low, static)} + assert severities == {DojoSarifResponseBuilder.SECURITY_SEVERITY} + + +def test_a_rejected_candidate_is_in_the_info_tier(): + info = meta(Rule(id='S19', name='Password in URL', confidence=0, is_dynamic_confidence=True), rejected=True) + assert info.id == 'S19-INFO' and info.payload['properties']['precision'] == 'low' + # a kept finding with confidence 0 stays LOW: INFO is for rejected candidates only + assert meta(Rule(id='S106', confidence=0, is_dynamic_confidence=True)).id == 'S106-LOW' + + +def finding(rule_id, confidence, rejected=False, start=0): + rule = Rule(id=rule_id, confidence=confidence, is_dynamic_confidence=True) + return Finding(detection='x', start_offset=start, end_offset=start + 1, rules=[rule], rejected=rejected) + + +@pytest.mark.parametrize( + 'level, expected', + [ + ('all', ['rejected', 'zero', 'low', 'medium', 'high', 'very-high']), + ('low', ['zero', 'low', 'medium', 'high', 'very-high']), + ('medium', ['medium', 'high', 'very-high']), + ('high', ['high', 'very-high']), + ('very-high', ['very-high']), + ], +) +def test_the_confidence_level_filters_the_report(level, expected): + findings = { + 'rejected': finding('S19', 0, rejected=True, start=0), + 'zero': finding('S106', 0, start=2), + 'low': finding('S105', 2, start=4), + 'medium': finding('S105', 3, start=6), + 'high': finding('S105', 6, start=8), + 'very-high': finding('S105', 9, start=10), + } + config = Config() + config.set_confidence_level(level) + kept = ScanMode.filter_by_confidence_level(SimpleNamespace(config=config), list(findings.values())) + assert [name for name, f in findings.items() if f in kept] == expected + + +def test_an_unknown_level_is_refused(): + with pytest.raises(ValueError): + Config().set_confidence_level('info') + + +def test_the_flag_sets_the_level(): + args = ['', '--target-dir', 'tests/fixtures', '--outfile', '/tmp/x.sarif', '--confidence-level', 'high'] + tool = DeepSecretsCliTool(args=args) + tool.parse_arguments() + assert tool.get_current_config().confidence_level == 'high' + tool = DeepSecretsCliTool(args=args[:-2]) + tool.parse_arguments() + assert tool.get_current_config().confidence_level == 'low' diff --git a/tests/scan_modes/test_scan_mode_regressions.py b/tests/scan_modes/test_scan_mode_regressions.py index d2114f7..1de74b2 100644 --- a/tests/scan_modes/test_scan_mode_regressions.py +++ b/tests/scan_modes/test_scan_mode_regressions.py @@ -150,6 +150,7 @@ def test_analyzer_bundle_carries_what_workers_read(): 'rulesets', 'benchmarking_mode', 'deep_max_size', + 'report_rejected', } assert bundle.workdir == '/app/tests/fixtures/extless' assert bundle.engines == {'regex': True} From f60a4f78c7f35f6991084f3da39c3b1a1cee807a Mon Sep 17 00:00:00 2001 From: Nikolai Khechumov Date: Mon, 28 Sep 2026 09:29:04 +0000 Subject: [PATCH 13/22] Add provider rules S42-S56 and cheap-search detectors - Regex rules S42-S56 for SonarQube, GitHub classic tokens, Algolia, Flickr, Bitly, Okta, Snyk, Telegram, Slack webhooks, Stripe, Facebook, Dropbox and Amplitude, in a keyword-first form adapted from gitleaks and Betterleaks. - The cheap variable search skips HTML tags within bounds where the lexer also reads the file (the wide skip stays above the size tier on purpose, KI-TOK-36), and reads command-line options and settings. --- .../core/tokenizers/cheap_var_search.py | 9 +- .../semantic/var_detection/detector.py | 3 + .../helpers/semantic/var_detection/rules.py | 44 +++- deepsecrets/rules/regexes.json | 235 ++++++++++++++++++ deepsecrets/scan_modes/cli.py | 2 +- tests/core/engines/regex/test_regex.py | 63 ++++- .../core/tokenizers/test_cheap_var_search.py | 29 ++- 7 files changed, 380 insertions(+), 5 deletions(-) diff --git a/deepsecrets/core/tokenizers/cheap_var_search.py b/deepsecrets/core/tokenizers/cheap_var_search.py index 3fc2857..6658a75 100644 --- a/deepsecrets/core/tokenizers/cheap_var_search.py +++ b/deepsecrets/core/tokenizers/cheap_var_search.py @@ -12,6 +12,13 @@ class CheapVarSearchTokenizer(Tokenizer): + def __init__(self, lexed: bool = False, **kwargs) -> None: + """`lexed`: the semantic engine also lexes this file, because it is within --deep-max-size; that selects the + detectors meant for such files (rules.py, `when_lexed`). The hashed engine's lexer does not count: it finds no + variables.""" + super().__init__(**kwargs) + self.lexed = lexed + def tokenize(self, file: File) -> List[Token]: language: Language = Language.ANY if file.extension is not None: @@ -20,7 +27,7 @@ def tokenize(self, file: File) -> List[Token]: except Exception as e: logger.exception(e) - rules: List[CheapVariableDetector] = CheapVariableDetectionRules.for_language(language) + rules: List[CheapVariableDetector] = CheapVariableDetectionRules.for_language(language, lexed=self.lexed) vars: List[Variable] = [] for rule in rules: vars.extend(rule.match(file.content)) diff --git a/deepsecrets/core/tokenizers/helpers/semantic/var_detection/detector.py b/deepsecrets/core/tokenizers/helpers/semantic/var_detection/detector.py index 5af30ec..826543d 100644 --- a/deepsecrets/core/tokenizers/helpers/semantic/var_detection/detector.py +++ b/deepsecrets/core/tokenizers/helpers/semantic/var_detection/detector.py @@ -170,6 +170,9 @@ def match(self, tokens: List[Token], token_stream: str) -> List['Variable']: class CheapVariableDetector(RegionDetector): + # None: every file. True: only a file the semantic engine also lexes (up to --deep-max-size); False: only a file + # it does not lex + when_lexed: Optional[bool] = None def match(self, content: str) -> List['Variable']: true_detections = [] diff --git a/deepsecrets/core/tokenizers/helpers/semantic/var_detection/rules.py b/deepsecrets/core/tokenizers/helpers/semantic/var_detection/rules.py index ce2be9e..e80e22a 100644 --- a/deepsecrets/core/tokenizers/helpers/semantic/var_detection/rules.py +++ b/deepsecrets/core/tokenizers/helpers/semantic/var_detection/rules.py @@ -414,15 +414,30 @@ class CheapVariableDetectionRules(VariableDetectionRules): match_rules={}, match_semantics={2: 'name', 4: 'value'}, ), - # looking for key-values when key is not in quoted, ignoring html tags (important really) + # looking for key-values when key is not in quoted, ignoring html tags (important really). In a file only the + # cheap search reads (over --deep-max-size), a "tag" runs to the next '>': in minified code `a', which the maintainer keeps on purpose as the precision guard of those files + # (KI-TOK-36; bounding it there costs 607 look-alike groups on SecretBench for 7 real) CheapVariableDetector( language=Language.ANY, + when_lexed=False, stream_pattern=re.compile( r'<[a-zA-Z]+[^>]*(*SKIP)(*F)|\b([a-zA-Z0-9_]+?)\b[\s]*?[:=][\s]*?([\'"])\b([^\'"()]+?)\b\2' ), match_rules={}, match_semantics={1: 'name', 3: 'value'}, ), + # the same where the lexer also reads the file: a tag is skipped only when it is one, `` + CheapVariableDetector( + language=Language.ANY, + when_lexed=True, + stream_pattern=re.compile( + r'<[a-zA-Z][\w:-]*(?:\s[^<>]{0,400})?/?>(*SKIP)(*F)|' + r'\b([a-zA-Z0-9_]+?)\b[\s]*?[:=][\s]*?([\'"])\b([^\'"()]+?)\b\2' + ), + match_rules={}, + match_semantics={1: 'name', 3: 'value'}, + ), # looking for random urls CheapVariableDetector( language=Language.ANY, @@ -441,6 +456,28 @@ class CheapVariableDetectionRules(VariableDetectionRules): match_rules={}, match_semantics={2: 'name', 4: 'value'}, ), + # a command-line option: -Dsonar.login=…, --api-key "…", --token=… + CheapVariableDetector( + language=Language.ANY, + stream_pattern=re.compile( + r'(?:^|\s)(?:-D|--?)([A-Za-z][\w.-]{1,60}[A-Za-z0-9])(?:=|[ \t]+)(["\']?)([^\s"\'`;|&]{6,200})\2' + r'(?=\s|$|[;|&])', + re.MULTILINE, + ), + overlapped=False, + match_rules={}, + match_semantics={1: 'name', 3: 'value'}, + ), + # .NET configuration: , where the name is an attribute's value + CheapVariableDetector( + language=Language.ANY, + stream_pattern=re.compile( + r' List[VariableDetector]: # type: ignore[override] + """The cheap detectors for a language; `lexed` says whether the semantic engine also lexes the file.""" + return [rule for rule in super().for_language(language) if rule.when_lexed in (None, lexed)] diff --git a/deepsecrets/rules/regexes.json b/deepsecrets/rules/regexes.json index fe4f901..8573ac6 100644 --- a/deepsecrets/rules/regexes.json +++ b/deepsecrets/rules/regexes.json @@ -332,5 +332,240 @@ "type": "value" } } + }, + { + "id": "S42", + "evidence": "shape", + "name": "SonarQube or SonarCloud token", + "confidence": 10, + "pattern": "((?:sonar[_.-]?(?:login|token))(?:[ \\t\\w.-]{0,20}))[\\s\\'\"]{0,3}(?:=|>|:{1,3}=|\\|\\||:|=>|\\?=|,)[\\x60\\'\"\\s=]{0,5}((?:squ_|sqp_|sqa_)?[a-z0-9=_\\-]{40})(?:[\\x60\\'\"\\s;]|\\\\[nr]|$)", + "target_group": 2, + "match_rules": { + "1": { + "type": "name" + }, + "2": { + "type": "value" + } + }, + "description": "A value in the provider's format assigned to a name that mentions the provider; the keyword-first form of `sonar-api-token` from gitleaks 8.30.1 (MIT, Copyright (c) 2019 Zachary Rice)" + }, + { + "id": "S43", + "evidence": "shape", + "name": "GitHub classic token under a GitHub name", + "confidence": 10, + "pattern": "((?:github[\\w.-]{0,20}?(?:token|pat|secret|key))(?:[ \\t\\w.-]{0,20}))[\\s\\'\"]{0,3}(?:=|>|:{1,3}=|\\|\\||:|=>|\\?=|,)[\\x60\\'\"\\s=]{0,5}([a-f0-9]{40})(?:[\\x60\\'\"\\s;]|\\\\[nr]|$)", + "target_group": 2, + "match_rules": { + "1": { + "type": "name" + }, + "2": { + "type": "value" + } + }, + "description": "Forty hex characters assigned to a name such as GITHUB_TOKEN or GITHUB_OAUTH_KEY: the token format before the ghp_ prefixes. Written in the frame of gitleaks 8.30.1 (MIT, Copyright (c) 2019 Zachary Rice)" + }, + { + "id": "S44", + "evidence": "shape", + "name": "Algolia API key", + "confidence": 10, + "pattern": "((?:algolia)(?:[ \\t\\w.-]{0,20}))[\\s\\'\"]{0,3}(?:=|>|:{1,3}=|\\|\\||:|=>|\\?=|,)[\\x60\\'\"\\s=]{0,5}([a-z0-9]{32})(?:[\\x60\\'\"\\s;]|\\\\[nr]|$)", + "target_group": 2, + "match_rules": { + "1": { + "type": "name" + }, + "2": { + "type": "value" + } + }, + "description": "A value in the provider's format assigned to a name that mentions the provider; the keyword-first form of `algolia-api-key` from gitleaks 8.30.1 (MIT, Copyright (c) 2019 Zachary Rice)" + }, + { + "id": "S45", + "evidence": "shape", + "name": "Algolia API key near its name", + "confidence": 10, + "pattern": "\\b(algolia\\w*(?:.|[\\n\\r]){0,32}?(?:secret|private|access|key|token|auth\\w*))(?:.|[\\n\\r]){0,32}?\\b([a-f0-9]{32})\\b", + "target_group": 2, + "match_rules": { + "1": { + "type": "name" + }, + "2": { + "type": "value" + } + }, + "description": "A value in the provider's format within a few characters of the provider's name and a secret word, as in a call argument; the frame of `bitly-access-token` from Betterleaks 1.8.1 (MIT)" + }, + { + "id": "S46", + "evidence": "shape", + "name": "Flickr API key", + "confidence": 10, + "pattern": "((?:flickr)(?:[ \\t\\w.-]{0,20}))[\\s\\'\"]{0,3}(?:=|>|:{1,3}=|\\|\\||:|=>|\\?=|,)[\\x60\\'\"\\s=]{0,5}([a-z0-9]{32})(?:[\\x60\\'\"\\s;]|\\\\[nr]|$)", + "target_group": 2, + "match_rules": { + "1": { + "type": "name" + }, + "2": { + "type": "value" + } + }, + "description": "A value in the provider's format assigned to a name that mentions the provider; the keyword-first form of `flickr-access-token` from gitleaks 8.30.1 (MIT, Copyright (c) 2019 Zachary Rice)" + }, + { + "id": "S47", + "evidence": "shape", + "name": "Bitly access token", + "confidence": 10, + "pattern": "\\b(bitly\\w*(?:.|[\\n\\r]){0,32}?(?:secret|private|access|key|token|auth\\w*))(?:.|[\\n\\r]){0,32}?\\b([a-f0-9]{40})\\b", + "target_group": 2, + "match_rules": { + "1": { + "type": "name" + }, + "2": { + "type": "value" + } + }, + "description": "A value in the provider's format within a few characters of the provider's name and a secret word, as in a call argument; adapted from `bitly-access-token` of Betterleaks 1.8.1 (MIT)" + }, + { + "id": "S48", + "evidence": "shape", + "name": "Okta API token", + "confidence": 10, + "pattern": "((?:okta)(?:[ \\t\\w.-]{0,20}))[\\s\\'\"]{0,3}(?:=|>|:{1,3}=|\\|\\||:|=>|\\?=|,)[\\x60\\'\"\\s=]{0,5}(00[\\w=\\-]{40})(?:[\\x60\\'\"\\s;]|\\\\[nr]|$)", + "target_group": 2, + "match_rules": { + "1": { + "type": "name" + }, + "2": { + "type": "value" + } + }, + "description": "A value in the provider's format assigned to a name that mentions the provider; the keyword-first form of `okta-access-token` from gitleaks 8.30.1 (MIT, Copyright (c) 2019 Zachary Rice)" + }, + { + "id": "S49", + "evidence": "shape", + "name": "Snyk API token", + "confidence": 10, + "pattern": "((?:snyk[_.-]?(?:(?:api|oauth)[_.-]?)?(?:key|token))(?:[ \\t\\w.-]{0,20}))[\\s\\'\"]{0,3}(?:=|>|:{1,3}=|\\|\\||:|=>|\\?=|,)[\\x60\\'\"\\s=]{0,5}([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})(?:[\\x60\\'\"\\s;]|\\\\[nr]|$)", + "target_group": 2, + "match_rules": { + "1": { + "type": "name" + }, + "2": { + "type": "value" + } + }, + "description": "A value in the provider's format assigned to a name that mentions the provider; the keyword-first form of `snyk-api-token` from gitleaks 8.30.1 (MIT, Copyright (c) 2019 Zachary Rice)" + }, + { + "id": "S50", + "evidence": "shape", + "name": "Telegram bot token under a Telegram name", + "confidence": 10, + "pattern": "((?:telegr)(?:[ \\t\\w.-]{0,20}))[\\s\\'\"]{0,3}(?:=|>|:{1,3}=|\\|\\||:|=>|\\?=|,)[\\x60\\'\"\\s=]{0,5}([0-9]{5,16}:A[a-z0-9_\\-]{34})(?:[\\x60\\'\"\\s;]|\\\\[nr]|$)", + "target_group": 2, + "match_rules": { + "1": { + "type": "name" + }, + "2": { + "type": "value" + } + }, + "description": "A value in the provider's format assigned to a name that mentions the provider; the keyword-first form of `telegram-bot-api-token` from gitleaks 8.30.1 (MIT, Copyright (c) 2019 Zachary Rice)" + }, + { + "id": "S51", + "evidence": "format", + "name": "Telegram bot token", + "confidence": 10, + "pattern": "\\b[0-9]{8,10}:AA[A-Za-z0-9_-]{33}\\b", + "case_sensitive": true, + "description": "A bot id, a colon, AA and 33 characters: the format proves the type" + }, + { + "id": "S52", + "evidence": "format", + "name": "Slack webhook (workflows, triggers, without a scheme)", + "confidence": 10, + "pattern": "(?:https?://)?hooks\\.slack\\.com/(?:services|workflows|triggers)/[A-Za-z0-9+/]{43,56}", + "description": "Widens S17 to the /workflows/ and /triggers/ URLs and to a URL without https://; `slack-webhook-url` from gitleaks 8.30.1 (MIT, Copyright (c) 2019 Zachary Rice)" + }, + { + "id": "S53", + "evidence": "format", + "name": "Stripe secret or restricted key, any length", + "confidence": 10, + "pattern": "\\b(?:sk|rk)_(?:test|live|prod)_([a-zA-Z0-9]{10,99})\\b", + "case_sensitive": true, + "match_rules": { + "1": { + "type": "value" + } + }, + "description": "sk_ and rk_ keys of 10 to 99 characters and prod keys, where S36 needs 24 to 45; `stripe-access-token` from gitleaks 8.30.1 (MIT, Copyright (c) 2019 Zachary Rice). Its placeholders fall to the randomness rule" + }, + { + "id": "S54", + "evidence": "shape", + "name": "Facebook app secret", + "confidence": 10, + "pattern": "((?:facebook)(?:[ \\t\\w.-]{0,20}))[\\s\\'\"]{0,3}(?:=|>|:{1,3}=|\\|\\||:|=>|\\?=|,)[\\x60\\'\"\\s=]{0,5}([a-f0-9]{32})(?:[\\x60\\'\"\\s;]|\\\\[nr]|$)", + "target_group": 2, + "match_rules": { + "1": { + "type": "name" + }, + "2": { + "type": "value" + } + }, + "description": "A value in the provider's format assigned to a name that mentions the provider; the keyword-first form of `facebook-secret` from gitleaks 8.30.1 (MIT, Copyright (c) 2019 Zachary Rice)" + }, + { + "id": "S55", + "evidence": "shape", + "name": "Dropbox app key", + "confidence": 10, + "pattern": "((?:dropbox)(?:[ \\t\\w.-]{0,20}))[\\s\\'\"]{0,3}(?:=|>|:{1,3}=|\\|\\||:|=>|\\?=|,)[\\x60\\'\"\\s=]{0,5}([a-z0-9]{15})(?:[\\x60\\'\"\\s;]|\\\\[nr]|$)", + "target_group": 2, + "match_rules": { + "1": { + "type": "name" + }, + "2": { + "type": "value" + } + }, + "description": "A value in the provider's format assigned to a name that mentions the provider; the keyword-first form of `dropbox-api-token` from gitleaks 8.30.1 (MIT, Copyright (c) 2019 Zachary Rice). The app key is the client half of an app's credentials" + }, + { + "id": "S56", + "evidence": "shape", + "name": "Amplitude API key", + "confidence": 10, + "pattern": "\\b(amplitude\\w*(?:.|[\\n\\r]){0,16}?(?:secret|private|access|key|token|auth\\w*))(?:.|[\\n\\r]){0,16}?\\b([a-f0-9]{32})\\b", + "target_group": 2, + "match_rules": { + "1": { + "type": "name" + }, + "2": { + "type": "value" + } + }, + "description": "A value in the provider's format within a few characters of the provider's name and a secret word, as in a call argument; adapted from `amplitude-secret-key` of Betterleaks 1.8.1 (MIT)" } ] diff --git a/deepsecrets/scan_modes/cli.py b/deepsecrets/scan_modes/cli.py index 84cf189..d6ad5b0 100644 --- a/deepsecrets/scan_modes/cli.py +++ b/deepsecrets/scan_modes/cli.py @@ -112,7 +112,7 @@ def __finalize(result: PerFileAnalysisResult): result.depth = 'deep' if deep else 'shallow' fct = FullContentTokenizer() - cheap_var_search = CheapVarSearchTokenizer() + cheap_var_search = CheapVarSearchTokenizer(lexed=deep) lex = LexerTokenizer(deep_token_inspection=True) # regex matches are candidates, judged as they are found (rules/regex_candidate_scoring_rules.json) diff --git a/tests/core/engines/regex/test_regex.py b/tests/core/engines/regex/test_regex.py index aa0c855..59ab88f 100644 --- a/tests/core/engines/regex/test_regex.py +++ b/tests/core/engines/regex/test_regex.py @@ -18,7 +18,7 @@ def test_1(file: File, regex_engine: RegexEngine): file=file, ) - assert len(findings) == 13 + assert len(findings) == 15 assert findings[0].final_rule.id == 'S0' assert findings[1].final_rule.id == 'S0' assert findings[2].final_rule.id == 'S1' @@ -33,6 +33,19 @@ def test_1(file: File, regex_engine: RegexEngine): assert findings[8].final_rule.id == 'S19' assert findings[8].detection == 'ridCNWnbTpavfVuJvWmS' + # S53 (any-length Stripe keys) matches the two placeholders S36's repeated-character pattern skips; unjudged here, + # they are rejected as far from random when the candidate rules judge them + assert [finding.final_rule.id for finding in findings[13:]] == ['S53', 'S53'] + candidate_rules = ( + RegexCandidateScoringRulesetBuilder() + .with_rules_from_file(get_path_inside_package('rules/regex_candidate_scoring_rules.json')) + .rules + ) + judged = RegexEngine(ruleset=regex_engine.ruleset, candidate_rules=candidate_rules) + judged.rejected_log = [] + regex_case(tokenizer=FullContentTokenizer(), engine=judged, file=file) + assert [entry['reason'] for entry in judged.rejected_log if entry['rule'] == 'S53'] == ['RC_NOT_RANDOM'] * 2 + @pytest.mark.fixture_file_path('extless/radius') def test_extless(file: File, regex_engine: RegexEngine): @@ -142,3 +155,51 @@ def test_s18_ignores_the_marker_and_placeholder_keys(regex_engine: RegexEngine): assert not [f for f in _scan(content, regex_engine) if 'S18' in {r.id for r in f.rules}] +def _random(alphabet: str, length: int, seed: int) -> str: + import random + + generator = random.Random(seed) + return ''.join(generator.choice(alphabet) for _ in range(length)) + + +HEX = '0123456789abcdef' +ALNUM = 'abcdefghijklmnopqrstuvwxyz0123456789' +MIXED = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789' +PROVIDER_CASES = { + 'S42': ('sonar.login={}', _random(ALNUM, 40, 1)), + 'S43': ('GITHUB_OAUTH_KEY="{}"', _random(HEX, 40, 2)), + 'S44': ("ALGOLIA_ADMIN_KEY: '{}'", _random(ALNUM, 32, 3)), + 'S45': ("const algoliaSearchKey = client('APPID', '{}')", _random(HEX, 32, 4)), + 'S46': ('flickr_api_key = "{}"', _random(ALNUM, 32, 5)), + 'S47': ('bitly access token: {}', _random(HEX, 40, 6)), + 'S48': ('okta.api-token={}', '00' + _random(MIXED, 40, 7)), + 'S49': ('SNYK_TOKEN={}', '1f0c2a3b-4d5e-4f60-8a7b-9c0d1e2f3a4b'), + 'S50': ('TELEGRAM_TOKEN="{}"', '123456789:AA' + _random(MIXED, 33, 8)), + 'S51': ('bot = Bot("{}")', '987654321:AA' + _random(MIXED, 33, 9)), + 'S52': ('url: {}', 'hooks.slack.com/workflows/' + _random(MIXED, 46, 10)), + 'S53': ('key = "{}"', 'sk_prod_' + _random(MIXED, 16, 11)), + 'S54': ('facebook_app_secret = "{}"', _random(HEX, 32, 12)), + 'S55': ('DROPBOX_APPKEY="{}"', _random(ALNUM, 15, 13)), + 'S56': ('amplitude.init(key, "{}")', _random(HEX, 32, 14)), +} + + +@pytest.mark.parametrize('rule_id', PROVIDER_CASES) +def test_provider_rules_find_their_tokens(regex_engine: RegexEngine, rule_id: str): + from deepsecrets.core.helpers.regex_candidate_evaluator import RegexCandidateEvaluator + from deepsecrets.core.model.regex_candidate import RegexCandidateContext + + template, value = PROVIDER_CASES[rule_id] + line = template.format(value) + rule = next(rule for rule in regex_engine.ruleset if rule.id == rule_id) + [match] = rule.matches(line) + assert value.endswith(line[match.start : match.end]) or line[match.start : match.end] in value + candidate_rules = ( + RegexCandidateScoringRulesetBuilder() + .with_rules_from_file(get_path_inside_package('rules/regex_candidate_scoring_rules.json')) + .rules + ) + context = RegexCandidateContext.from_match( + rule, match.match, line[match.start : match.end], '/repo/src/settings.py' + ) + assert not RegexCandidateEvaluator(candidate_rules).evaluate(context).rejected diff --git a/tests/core/tokenizers/test_cheap_var_search.py b/tests/core/tokenizers/test_cheap_var_search.py index 0a8e462..bf7a49a 100644 --- a/tests/core/tokenizers/test_cheap_var_search.py +++ b/tests/core/tokenizers/test_cheap_var_search.py @@ -15,4 +15,31 @@ def test_1(file: File, cheap_var_search_tokenizer: CheapVarSearchTokenizer): def test_2(file: File, cheap_var_search_tokenizer: CheapVarSearchTokenizer): _ = cheap_var_search_tokenizer.tokenize(file=file) variables = cheap_var_search_tokenizer.get_variables() - assert len(variables) == 24 + # 22 key-value pairs, and the --connect and --warehouse-dir options of a sqoop command (the command-option detector) + assert len(variables) == 24 + 2 + + +MINIFIED = 'if(a"' + + +@pytest.mark.parametrize('lexed, found', [(True, True), (False, False)]) +def test_the_tag_skip_depends_on_the_tier(lexed, found): + # where the lexer also reads the file a tag is skipped only when it is one; above the tier the wide skip stays, on + # purpose (KI-TOK-36), so `a' + tokenizer = CheapVarSearchTokenizer(lexed=lexed) + tokens = tokenizer.tokenize(File(path='/tmp/app.min.js', relative_path='app.min.js', content=MINIFIED)) + assert ('DROPBOX_APPKEY' in [token.semantic.payload.name for token in tokens]) is found + + +def test_command_options_and_dotnet_settings_are_variables(): + content = ( + 'mvn sonar:sonar -Dsonar.login=5f2b9c1e8a7d4036b1e9f0c2a4d6e8b0c1d3e5f7 -U\n' + 'tool --api-key "Zk3pQ9rT2mX7vB1n" --verbose\n' + '\n' + ) + tokens = CheapVarSearchTokenizer().tokenize(File(path='/tmp/build.sh', relative_path='build.sh', content=content)) + names = {token.semantic.payload.name: token.content for token in tokens} + assert names['sonar.login'].startswith('5f2b9c1e') and names['api-key'] == 'Zk3pQ9rT2mX7vB1n' + assert names['loginradius:apisecret'].startswith('3f9a1c2e') + + From 858aadb2ae8e0626dd148d5d85b60c49417b75c5 Mon Sep 17 00:00:00 2001 From: Nikolai Khechumov Date: Mon, 28 Sep 2026 09:29:04 +0000 Subject: [PATCH 14/22] Add --skip-bundles Also skip minified JavaScript, source maps and bundles (*.min.js, *.map, *.bundle.js). Off by default: the size tier already makes most of them cheap, and skipping them outright loses more secrets. --- README.md | 1 + deepsecrets/cli.py | 9 ++++++++ deepsecrets/rules/excluded_bundles.json | 5 +++++ tests/cli/test_cli.py | 15 +++++++++++++ .../scan_modes/test_scan_mode_regressions.py | 22 +++++++++++++++++++ 5 files changed, 52 insertions(+) create mode 100644 deepsecrets/rules/excluded_bundles.json diff --git a/README.md b/README.md index 80fab4a..045c5b5 100644 --- a/README.md +++ b/README.md @@ -65,6 +65,7 @@ The `--help` command is always ready to guide you, but here are the key flags yo * `--hashed-values /path/to/hashes.json`: Provide a list of pre-hashed known production secrets to search for them securely. * `--excluded-paths /path/to/exclusions.json`: Override or extend the default paths ignored during scanning. * `--deep-max-size BYTES`: Files larger than this (default 250,000 bytes) get the fast analysis: the regex rules and the variable search, without the language lexer. Big files are mostly generated or minified, the lexer costs most of the scan time on them and rarely finds more there. `0` gives every file the full analysis. +* `--skip-bundles`: Skip minified JavaScript, source maps and bundles (`*.min.js`, `*.map`, `*.bundle.js`) entirely, on top of the default exclusions. * `--disable-masking`: Keep potential secrets unmasked in the output report *(see caution below)*. * `--ci` / `--no-ci`: Turn CI mode on or off. CI mode prints a plain progress line every 30 seconds instead of the live progress display. It turns on by itself inside CI services (GitHub Actions, GitLab CI, Jenkins and others) and whenever the output is not a terminal. diff --git a/deepsecrets/cli.py b/deepsecrets/cli.py index db44f3a..6d24fc6 100644 --- a/deepsecrets/cli.py +++ b/deepsecrets/cli.py @@ -176,6 +176,13 @@ def _build_argparser(self) -> None: default=['built-in'], ) + parser.add_argument( + '--skip-bundles', + action='store_true', + help='Add minified JavaScript, source maps and bundles (*.min.js, *.map, *.bundle.js) to the built-in\n' + 'path exclusions. Most of them are larger than --deep-max-size and get the fast analysis anyway.\n', + ) + parser.add_argument( '--deep-max-size', type=int, @@ -321,6 +328,8 @@ def parse_arguments(self) -> None: EXCLUDE_PATHS_BUILTIN = get_path_inside_package('rules/excluded_paths.json') if user_args.excluded_paths is not None: rules = [rule.replace('built-in', EXCLUDE_PATHS_BUILTIN) for rule in user_args.excluded_paths] + if 'built-in' in user_args.excluded_paths and user_args.skip_bundles: + rules.append(get_path_inside_package('rules/excluded_bundles.json')) config.set_global_exclusion_paths(rules) config.engines = [] diff --git a/deepsecrets/rules/excluded_bundles.json b/deepsecrets/rules/excluded_bundles.json new file mode 100644 index 0000000..2a94a0a --- /dev/null +++ b/deepsecrets/rules/excluded_bundles.json @@ -0,0 +1,5 @@ +[ + {"name": "Minified JavaScript", "pattern": ".*\\.min\\.js$"}, + {"name": "Source maps", "pattern": ".*\\.map$"}, + {"name": "JavaScript bundles", "pattern": ".*\\.bundle\\.js$"} +] diff --git a/tests/cli/test_cli.py b/tests/cli/test_cli.py index ee8ddbe..15582a3 100644 --- a/tests/cli/test_cli.py +++ b/tests/cli/test_cli.py @@ -100,6 +100,21 @@ def test_hashed_values_registers_hashed_engine(): config.rulesets.pop(HashedSecretsRulesetBuilder, None) +@pytest.mark.parametrize( + 'extra, bundles_excluded', + [([], False), (['--skip-bundles'], True), (['--excluded-paths', 'disable', '--skip-bundles'], False)], +) +def test_bundle_exclusions_follow_the_flags(monkeypatch, extra, bundles_excluded): + tool = DeepSecretsCliTool(args=['', '--target-dir', '/app/tests/fixtures/', '--outfile', '/tmp/x.sarif'] + extra) + config = tool.get_current_config() + monkeypatch.setattr(config, 'global_exclusion_paths', []) # the singleton keeps what earlier tests added + tool.parse_arguments() + + names = [path.rsplit('/', 1)[-1] for path in config.global_exclusion_paths] + assert ('excluded_bundles.json' in names) is bundles_excluded + assert ('excluded_paths.json' in names) is ('disable' not in extra) + + @pytest.mark.parametrize( 'extra, detected, ci_mode', [([], True, True), ([], False, False), (['--ci'], False, True), (['--no-ci'], True, False)], diff --git a/tests/scan_modes/test_scan_mode_regressions.py b/tests/scan_modes/test_scan_mode_regressions.py index 1de74b2..e5341d3 100644 --- a/tests/scan_modes/test_scan_mode_regressions.py +++ b/tests/scan_modes/test_scan_mode_regressions.py @@ -366,6 +366,28 @@ def test_initializer_never_raises(tmp_path: Path, monkeypatch): iscan_mode.pool_wrapper(lambda *args: None, 1, 'any.py') +def test_skip_bundles_rules_exclude_minified_files_maps_and_bundles(tmp_path: Path): + for name in ('app.js', 'app.min.js', 'app.js.map', 'styles.css.map', 'main.bundle.js', 'bundle.js'): + (tmp_path / name).write_text('const a = 1;\n') + built_in = [get_path_inside_package(f'rules/{name}') for name in ('excluded_paths.json', 'excluded_bundles.json')] + + config = _config(str(tmp_path)) + config.set_global_exclusion_paths(built_in) + mode = CliScanMode(config=config) + try: + assert sorted(os.path.basename(p) for p in mode.filepaths) == ['app.js', 'bundle.js'] + finally: + mode.dispose() + + config = _config(str(tmp_path)) + config.set_global_exclusion_paths(built_in[:1]) # the default since the size tier; --skip-bundles adds the second + mode = CliScanMode(config=config) + try: + assert len(mode.filepaths) == 6 + finally: + mode.dispose() + + class WorkerKillingScanMode(CliScanMode): @staticmethod def _per_file_analyzer(bundle, file, task_id=None, task_reporter=None): # type: ignore From 28cbe83f9d35f55fbcea57ed889a5abcffca55d9 Mon Sep 17 00:00:00 2001 From: Nikolai Khechumov Date: Mon, 28 Sep 2026 09:29:04 +0000 Subject: [PATCH 15/22] Remove the jsx-lexer dependency LexerFinder re-executed the jsx-lexer source for every scanned file and the lexer was never chosen: Pygments' own JsxLexer answers the jsx and react aliases and file names (KI-TOK-18). --- deepsecrets/core/utils/lexer_finder.py | 10 ++++------ poetry.lock | 17 +---------------- pyproject.toml | 1 - 3 files changed, 5 insertions(+), 23 deletions(-) diff --git a/deepsecrets/core/utils/lexer_finder.py b/deepsecrets/core/utils/lexer_finder.py index 214f7c5..25f1f32 100644 --- a/deepsecrets/core/utils/lexer_finder.py +++ b/deepsecrets/core/utils/lexer_finder.py @@ -4,11 +4,10 @@ from deepsecrets.core.model.file import File from deepsecrets.core.utils.guess_filetype import FileTypeGuesser -from pygments.lexers import load_lexer_from_file, get_lexer_for_filename, get_lexer_by_name +from pygments.lexers import get_lexer_for_filename, get_lexer_by_name from pygments.util import ClassNotFound from pygments.formatters import RawTokenFormatter from pygments.lexers.special import RawTokenLexer -from jsx import lexer as lexer_mod class LexerFinder: @@ -21,13 +20,12 @@ class LexerFinder: probes: Dict def __init__(self) -> None: - self._init_custom_lexers() + # No JSX lexer to load: Pygments' own `pygments.lexers.jsx.JsxLexer` answers the `jsx` and `react` aliases and + # the `*.jsx` and `*.react` file names. The jsx-lexer package once loaded here re-executed its source for every + # file and was never the lexer chosen (KI-TOK-18); it is no longer a dependency self._init_alias_exceptions() self._init_probes() - def _init_custom_lexers(self): - load_lexer_from_file(lexer_mod.__file__, "JsxLexer") - def _init_alias_exceptions(self): self.alias_exceptions = {'js+react': 'react'} diff --git a/poetry.lock b/poetry.lock index 54f0104..1555012 100644 --- a/poetry.lock +++ b/poetry.lock @@ -322,21 +322,6 @@ docs = ["furo", "rst.linker (>=1.9)", "sphinx (>=3.5)"] packaging = ["build", "setuptools (>=61.2)", "setuptools_scm[toml] (>=6.0)", "twine"] testing = ["PyYAML", "atheris (>=2.3.0,<2.4.0) ; python_version < \"3.12\"", "bson", "ecdsa", "feedparser", "gmpy2", "numpy", "pandas", "pymongo", "pytest (>=6.0,!=8.1.*)", "pytest-benchmark", "pytest-benchmark[histogram]", "pytest-checkdocs (>=1.2.3)", "pytest-enabler (>=1.0.1)", "pytest-ruff (>=0.2.1)", "scikit-learn", "scipy (>=1.9.3) ; python_version > \"3.10\"", "scipy ; python_version <= \"3.10\"", "simplejson", "sqlalchemy", "ujson"] -[[package]] -name = "jsx-lexer" -version = "2.0.1" -description = "A JSX lexer for Pygments" -optional = false -python-versions = "*" -groups = ["main"] -files = [ - {file = "jsx-lexer-2.0.1.tar.gz", hash = "sha256:0d9aa653e74d7973d74021dde8349896c0df094d8e40349b92b35e0930ed7f71"}, - {file = "jsx_lexer-2.0.1-py2.py3-none-any.whl", hash = "sha256:508a08757764356aa36fd703596fdd59f789104f44b6568c7a14e27e62e57ad4"}, -] - -[package.dependencies] -Pygments = ">=2.12.0" - [[package]] name = "markdown-it-py" version = "4.2.0" @@ -1298,4 +1283,4 @@ typing-extensions = ">=4.12.0" [metadata] lock-version = "2.1" python-versions = ">=3.10,<4.0.0" -content-hash = "a97f41043d02746969b9d24e5eba3d7f67a6131c447627f9a73159252530755d" +content-hash = "f7e3ec827644dd6d664ac0d7b55ce329e378123005f3828a7ab12527b1d2cba2" diff --git a/pyproject.toml b/pyproject.toml index 6605eb8..4880219 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,7 +33,6 @@ dependencies = [ "ordered-set (==4.1.0)", "mmh3 (==5.3.0)", "regex (==2026.9.3)", - "jsx-lexer == 2.0.1", "aenum (==3.1.17)", "puppetparser == 0.2.14", "sarif-om == 1.0.4", From b978e66ff1f72d12f5f869e64c78cc59cc0481a7 Mon Sep 17 00:00:00 2001 From: Nikolai Khechumov Date: Mon, 28 Sep 2026 09:29:04 +0000 Subject: [PATCH 16/22] Keep the diagnostics tracer in step with the scanner The tracer replays the size tier, the cheap search's tier-dependent detectors, the regex-candidate judging and the allows_low_entropy gate, and names the stage where a value was lost (schema version 5). --- deepsecrets/diagnostics/__main__.py | 19 +++++++- deepsecrets/diagnostics/trace.py | 71 +++++++++++++++++++++++++---- deepsecrets/diagnostics/worker.py | 7 ++- tests/diagnostics/test_trace.py | 68 +++++++++++++++++++++++++++ 4 files changed, 151 insertions(+), 14 deletions(-) diff --git a/deepsecrets/diagnostics/__main__.py b/deepsecrets/diagnostics/__main__.py index 6d7508a..91351dd 100644 --- a/deepsecrets/diagnostics/__main__.py +++ b/deepsecrets/diagnostics/__main__.py @@ -10,6 +10,7 @@ from collections import defaultdict from multiprocessing import get_context +from deepsecrets.config import DEFAULT_DEEP_MAX_SIZE from deepsecrets.diagnostics.worker import init, trace_job @@ -26,6 +27,17 @@ def main(argv=None) -> int: action='store_true', help='replay files the built-in excluded_paths rules would skip, as `--excluded-paths disable` does', ) + t.add_argument( + '--skip-bundles', + action='store_true', + help='stop minified files, source maps and bundles at selection, as a scan with `--skip-bundles` does', + ) + t.add_argument( + '--deep-max-size', + type=int, + default=DEFAULT_DEEP_MAX_SIZE, + help='replay larger files without the lexer, as a scan with this `--deep-max-size` does (0: every file)', + ) args = parser.parse_args(argv) by_file = defaultdict(list) @@ -42,14 +54,17 @@ def main(argv=None) -> int: path_exclusions = not args.no_path_exclusions if args.processes > 1: with get_context('spawn').Pool( - args.processes, initializer=init, initargs=(path_exclusions,), maxtasksperchild=20 + args.processes, + initializer=init, + initargs=(path_exclusions, args.skip_bundles, args.deep_max_size), + maxtasksperchild=20, ) as pool: for traces in pool.imap_unordered(trace_job, jobs): for trace in traces: out.write(json.dumps(trace, default=str) + '\n') written += 1 else: - init(path_exclusions) + init(path_exclusions, args.skip_bundles, args.deep_max_size) for job in jobs: for trace in trace_job(job): out.write(json.dumps(trace, default=str) + '\n') diff --git a/deepsecrets/diagnostics/trace.py b/deepsecrets/diagnostics/trace.py index e67be3c..7b0aac7 100644 --- a/deepsecrets/diagnostics/trace.py +++ b/deepsecrets/diagnostics/trace.py @@ -6,9 +6,11 @@ rule produced it and what drove its score. The schema is versioned: consumers (the benchmark harness) depend on it. """ +import os from dataclasses import dataclass, field from typing import Dict, List, Optional +from deepsecrets.config import DEFAULT_DEEP_MAX_SIZE, deep_analysis from deepsecrets.core.engines.regex import RegexEngine from deepsecrets.core.engines.semantic import SemanticEngine, filenames_ignorelist from deepsecrets.core.helpers.variable_evaluator import HOPELESS_THRESHOLD, VariableEvaluator @@ -16,6 +18,7 @@ from deepsecrets.core.model.token import SemanticType, Token from deepsecrets.core.rulesets.excluded_paths import ExcludedPathsBuilder from deepsecrets.core.rulesets.regex import RegexRulesetBuilder +from deepsecrets.core.rulesets.regex_candidate_scoring import RegexCandidateScoringRulesetBuilder from deepsecrets.core.rulesets.variable_scoring import VariableScoringRulesetBuilder from deepsecrets.core.tokenizers.cheap_var_search import CheapVarSearchTokenizer from deepsecrets.core.tokenizers.full_content import FullContentTokenizer @@ -29,7 +32,7 @@ from deepsecrets.core.utils.finding_merger import FindingMerger from deepsecrets.core.utils.fs import get_path_inside_package -SCHEMA_VERSION = 1 +SCHEMA_VERSION = 5 # 2: the size tier ('tier' field, 'tier' verdict stage); 3: 'allows_low_entropy', S106 gating; 4: 'finding_evaluation'; 5: 'candidate_evaluation' (regex candidates judged in RegexEngine) replaces it WINDOW_BEFORE, WINDOW_AFTER = 6, 3 @@ -38,19 +41,37 @@ class Rulesets: regex: list scoring: list excluded: list + # the regex-candidate scoring rules (RegexEngine judges each regex match with them) + candidates: list = field(default_factory=list) + # the size tier of the scan being replayed (--deep-max-size) + deep_max_size: int = DEFAULT_DEEP_MAX_SIZE @classmethod - def builtin(cls, path_exclusions: bool = True) -> 'Rulesets': - """The shipped rulesets. `path_exclusions=False` mirrors a scan run with `--excluded-paths disable`, so a - file under node_modules/ is replayed instead of stopping at the selection stage.""" + def builtin( + cls, path_exclusions: bool = True, skip_bundles: bool = False, deep_max_size: int = DEFAULT_DEEP_MAX_SIZE + ) -> 'Rulesets': + """The shipped rulesets, as a scan with default flags uses them. `path_exclusions=False` mirrors + `--excluded-paths disable`, so a file under node_modules/ is replayed instead of stopping at the selection + stage; `skip_bundles=True` mirrors `--skip-bundles`, which adds minified files, source maps and bundles to the + built-in exclusions; `deep_max_size` mirrors `--deep-max-size`.""" regex = RegexRulesetBuilder() regex.with_rules_from_file(get_path_inside_package('rules/regexes.json')) scoring = VariableScoringRulesetBuilder() scoring.with_rules_from_file(get_path_inside_package('rules/variable_scoring_rules.json')) + candidates = RegexCandidateScoringRulesetBuilder() + candidates.with_rules_from_file(get_path_inside_package('rules/regex_candidate_scoring_rules.json')) excluded = ExcludedPathsBuilder() if path_exclusions: excluded.with_rules_from_file(get_path_inside_package('rules/excluded_paths.json')) - return cls(regex=regex.rules, scoring=scoring.rules, excluded=excluded.rules) + if skip_bundles: + excluded.with_rules_from_file(get_path_inside_package('rules/excluded_bundles.json')) + return cls( + regex=regex.rules, + scoring=scoring.rules, + excluded=excluded.rules, + candidates=candidates.rules, + deep_max_size=deep_max_size, + ) @dataclass @@ -197,6 +218,9 @@ class FileReplay: file: File relative_path: str rulesets: Rulesets + # False when the file is larger than rulesets.deep_max_size: the scan gives it no lexer + deep: bool = True + size: int = 0 lexer_name: Optional[str] = None language: Optional[str] = None raw_tokens: List[Token] = field(default_factory=list) @@ -207,6 +231,8 @@ class FileReplay: cheap_vars: List[Token] = field(default_factory=list) findings: list = field(default_factory=list) decisions: List[dict] = field(default_factory=list) + # regex candidates RegexEngine rejected, as it rejects them in a scan (not reported at the default level) + rejected: List[dict] = field(default_factory=list) excluded_by: Optional[str] = None def run(self) -> 'FileReplay': @@ -254,16 +280,22 @@ def run(self) -> 'FileReplay': prod = LexerTokenizer(deep_token_inspection=True) prod.tokenize(self.file) self.production_vars = prod.get_variables() - cheap = CheapVarSearchTokenizer() + # the tier decides the cheap detectors too (a bounded tag skip where the lexer also reads the file) + self.size = os.path.getsize(self.file.path) + self.deep = deep_analysis(self.size, self.rulesets.deep_max_size) + cheap = CheapVarSearchTokenizer(lexed=self.deep) cheap.tokenize(self.file) self.cheap_vars = cheap.get_variables() - # the production pipeline for this file, as scan_modes/cli.py builds it, with the value cache recorded - regex_engine = RegexEngine(ruleset=self.rulesets.regex) + # the production pipeline for this file, as scan_modes/cli.py builds it, with the value cache recorded; the + # lexer stages above still run on a shallow file, so a trace can say what the lexer would have found + regex_engine = RegexEngine(ruleset=self.rulesets.regex, candidate_rules=self.rulesets.candidates) + regex_engine.rejected_log = self.rejected semantic_engine = SemanticEngine(regex_engine, ruleset=self.rulesets.scoring) analyzer = TracingFileAnalyzer(self.file) analyzer.add_engine(regex_engine, [FullContentTokenizer()]) - analyzer.add_engine(semantic_engine, [LexerTokenizer(deep_token_inspection=True), CheapVarSearchTokenizer()]) + tokenizers = [LexerTokenizer(deep_token_inspection=True)] if self.deep else [] + analyzer.add_engine(semantic_engine, tokenizers + [CheapVarSearchTokenizer(lexed=self.deep)]) self.findings = FindingMerger(analyzer.process()).merge() self.decisions = analyzer.decisions return self @@ -286,6 +318,7 @@ def _trace_span(replay: FileReplay, span: Span, evaluator: VariableEvaluator) -> 'located': span.located, 'selection': {'excluded_by': replay.excluded_by}, 'lexer': {'name': replay.lexer_name, 'language': replay.language, 'extension': file.extension}, + 'tier': {'deep': replay.deep, 'size': replay.size, 'deep_max_size': replay.rulesets.deep_max_size}, } covering = [i for i, t in enumerate(replay.raw_tokens) if _overlaps(t.span, s, e)] @@ -311,7 +344,8 @@ def _trace_span(replay: FileReplay, span: Span, evaluator: VariableEvaluator) -> trace['variables'] = {'lex_prefilter': bool(in_pre), 'lex': bool(in_prod), 'cheap': bool(in_cheap)} evaluations = [] - for source, tokens in (('lex', in_prod), ('cheap', in_cheap)): + # a shallow file's lexer variables never reach the engine + for source, tokens in (('lex', in_prod if replay.deep else []), ('cheap', in_cheap)): for token in tokens: if token.semantic is None or token.semantic.type != SemanticType.VARIABLE: continue @@ -341,6 +375,11 @@ def _trace_span(replay: FileReplay, span: Span, evaluator: VariableEvaluator) -> ], 'cache_skipped': sorted(set(cache_skips)), } + trace['candidate_evaluation'] = [ + {k: (list(v) if isinstance(v, tuple) else v) for k, v in d.items()} + for d in replay.rejected + if _overlaps(d['span'], s, e) + ] trace['verdict'] = _verdict(trace) return trace @@ -364,7 +403,19 @@ def _verdict(t: dict) -> dict: return {'stage': 'selection', 'component': 'excluded_paths', 'detail': t['selection']['excluded_by']} if not t['located']: return {'stage': 'label', 'component': 'dataset', 'detail': 'value not found where the label points'} + if t.get('candidate_evaluation'): + first = t['candidate_evaluation'][0] + return { + 'stage': 'evaluation', + 'component': 'regex_candidate_evaluator', + 'detail': f"{first['rule']}: rejected at {first['total']} points, mostly {first['reason']} ({first['evidence']})", + } has_cheap = t['variables']['cheap'] + if not t['tier']['deep'] and not has_cheap: + detail = f"{t['tier']['size']} bytes, over --deep-max-size {t['tier']['deep_max_size']}: no lexer" + if t['variables']['lex']: + detail += ', which would have detected the variable' + return {'stage': 'tier', 'component': 'deep_max_size', 'detail': detail} if t['lexer']['name'] is None and not has_cheap: return {'stage': 'lexer', 'component': 'lexer_finder', 'detail': 'no lexer'} if t['token'] is None and not has_cheap: diff --git a/deepsecrets/diagnostics/worker.py b/deepsecrets/diagnostics/worker.py index 7ba9d4c..6850c39 100644 --- a/deepsecrets/diagnostics/worker.py +++ b/deepsecrets/diagnostics/worker.py @@ -6,15 +6,18 @@ import os import traceback +from typing import Optional RULESETS = None -def init(path_exclusions: bool = True): +def init(path_exclusions: bool = True, skip_bundles: bool = False, deep_max_size: Optional[int] = None): global RULESETS + from deepsecrets.config import DEFAULT_DEEP_MAX_SIZE from deepsecrets.diagnostics.trace import Rulesets - RULESETS = Rulesets.builtin(path_exclusions) + size = DEFAULT_DEEP_MAX_SIZE if deep_max_size is None else deep_max_size + RULESETS = Rulesets.builtin(path_exclusions, skip_bundles, size) def trace_job(job): diff --git a/tests/diagnostics/test_trace.py b/tests/diagnostics/test_trace.py index 071ac1f..7f0a0ae 100644 --- a/tests/diagnostics/test_trace.py +++ b/tests/diagnostics/test_trace.py @@ -138,3 +138,71 @@ def test_cli_with_a_process_pool(tmp_path): json.loads(line)['case_id']: json.loads(line)['verdict']['stage'] for line in out.read_text().splitlines() } assert verdicts == {'c0': 'reported', 'c1': 'reported', 'c2': 'reported'} + + +def test_builtin_rulesets_mirror_the_bundle_exclusions(): + def excludes(rulesets, path): + return any(rule.match(path) for rule in rulesets.excluded) + + assert not excludes(Rulesets.builtin(), 'web/app.min.js') + assert excludes(Rulesets.builtin(), 'node_modules/pkg/index.js') + assert excludes(Rulesets.builtin(skip_bundles=True), 'web/app.min.js') + assert not excludes(Rulesets.builtin(path_exclusions=False, skip_bundles=True), 'web/app.min.js') + + +def test_value_only_the_lexer_finds_is_lost_at_the_size_tier(tmp_path): + # a name two levels down in JSON: the lexer detects it, the cheap variable search does not + path = tmp_path / 'config.json' + path.write_text('{\n "service": {\n "auth": {\n "password": "T7r$kq92LmZx"\n }\n }\n}\n') + col = path.read_text().split('\n')[3].index('T7r') + 1 + nested = dict(case_id='nested', line=4, end_line=4, start_col=col, end_col=col + 12, value='T7r$kq92LmZx') + + deep = trace_file(str(path), 'config.json', [nested], Rulesets.builtin(deep_max_size=0))[0] + assert deep['verdict']['stage'] == 'reported' and deep['tier']['deep'] is True + + shallow = trace_file(str(path), 'config.json', [nested], Rulesets.builtin(deep_max_size=10))[0] + assert shallow['tier'] == {'deep': False, 'size': path.stat().st_size, 'deep_max_size': 10} + assert shallow['verdict'] == { + 'stage': 'tier', + 'component': 'deep_max_size', + 'detail': f'{path.stat().st_size} bytes, over --deep-max-size 10: no lexer, which would have detected the variable', + } + + +def test_low_entropy_value_under_a_key_name_is_dropped_by_the_engine(tmp_path): + path = tmp_path / 'settings.py' + # two values: the value cache would skip a repeat of one that produced no finding (KI-ENG-08) + path.write_text('api_key = "nacc6opq"\npassword = "qpo6ccan"\n') + spans = [ + dict(case_id=name, line=line, end_line=line, start_col=len(name) + 5, end_col=len(name) + 13, value=value) + for line, name, value in ((1, 'api_key', 'nacc6opq'), (2, 'password', 'qpo6ccan')) + ] + traces = {t['case_id']: t for t in trace_file(str(path), 'src/settings.py', spans)} + + key = traces['api_key'] + assert key['verdict']['stage'] == 'evaluation' and key['verdict']['component'] == 'semantic_engine' + assert key['evaluations'][0]['dangerous'] is True + assert key['evaluations'][0]['allows_low_entropy'] is False + assert key['evaluations'][0]['rule_emitted'] is None + + password = traces['password'] + assert password['verdict'] == {'stage': 'reported', 'component': 'semantic_engine', 'detail': 'S106'} + assert password['evaluations'][0]['rule_emitted'] == 'S106' + # a rule that scores nothing cannot change the verdict, so it gets no counterfactual + assert 'SEM_VAR_HUMAN_CHOSEN_SECRET_NAMES' not in password['evaluations'][0]['dangerous_without'] + + +def test_regex_candidate_rejected_by_its_evaluation(tmp_path): + path = tmp_path / 'settings.py' + # a dictionary word as a URL password: S19 matches it, the regex-candidate rules reject it + path.write_text('SERVER = "https://admin:sunshine@db.internal.example"\n') + value = 'sunshine' + col = path.read_text().index(value) + 1 + [t] = trace_file( + str(path), + 'src/settings.py', + [dict(case_id='w', line=1, end_line=1, start_col=col, end_col=col + len(value), value=value)], + ) + assert t['verdict']['stage'] == 'evaluation' and t['verdict']['component'] == 'regex_candidate_evaluator' + assert t['verdict']['detail'] == 'S19: rejected at -8 points, mostly RC_NATURAL_LANGUAGE (shape)' + assert t['candidate_evaluation'][0]['rejected'] is True From 82aa44fab30ee1b4b240462b0265aa1efb5b80ed Mon Sep 17 00:00:00 2001 From: Nikolai Khechumov Date: Mon, 28 Sep 2026 09:29:04 +0000 Subject: [PATCH 17/22] Build a fresh Config for every parse The config was a process-wide singleton that parse_arguments mutated, so a second run in the same process inherited the first one's exclusion files, rulesets and flags; after --disable-masking, a masked run still wrote plaintext secrets (KI-CLI-12). Each parse now builds its own Config and the module-level instance is gone. --- deepsecrets/cli.py | 11 +++--- deepsecrets/config.py | 3 -- tests/cli/test_cli.py | 50 ++++++++++++++++---------- tests/output/test_sarif_diagnostics.py | 4 --- 4 files changed, 39 insertions(+), 29 deletions(-) diff --git a/deepsecrets/cli.py b/deepsecrets/cli.py index 6d24fc6..6cbeecd 100644 --- a/deepsecrets/cli.py +++ b/deepsecrets/cli.py @@ -13,7 +13,6 @@ SCANNER_VERSION, SCANNER_VERSION_NUMERIC, Config, - config, Output, ) from deepsecrets.core.engines.hashed_secret import HashedSecretEngine @@ -72,6 +71,8 @@ class ReturnCodes: class DeepSecretsCliTool: argparser: argparse.ArgumentParser + # built by parse_arguments, one per parse + config: Config def __init__(self, args: List[str]): self.args = args @@ -294,6 +295,8 @@ def _build_argparser(self) -> None: self.argparser = parser def parse_arguments(self) -> None: + # a fresh config for every parse: nothing an earlier run set carries over (KI-CLI-12) + config = self.config = Config() user_args = self.argparser.parse_args(args=self.args[1:]) if user_args.verbose: @@ -303,7 +306,6 @@ def parse_arguments(self) -> None: if user_args.disable_masking: config.set_disable_masking(True) - # set on every parse: the config singleton outlives one run config.set_report_diagnostics(user_args.report_diagnostics) config.set_confidence_level(user_args.confidence_level) @@ -369,10 +371,10 @@ def parse_arguments(self) -> None: config.add_ruleset(FalseFindingsBuilder, conf_false_findings_ruleset) def get_current_config(self) -> Config: - return config + return self.config def _add_ignorefiles(self, files: List[str]): - config.set_global_exclusion_paths(files) + self.config.set_global_exclusion_paths(files) def start(self) -> int: # pragma: nocover startup_time = datetime.now() @@ -381,6 +383,7 @@ def start(self) -> int: # pragma: nocover except Exception as e: logger.exception(e) return ReturnCodes.ERROR + config = self.config if config.output.type == 'json': console.print('\n') diff --git a/deepsecrets/config.py b/deepsecrets/config.py index ab1db7d..7a9e00e 100644 --- a/deepsecrets/config.py +++ b/deepsecrets/config.py @@ -167,6 +167,3 @@ def _validate_paths(self, paths: List[str]) -> None: raise FileNotFoundException(f'File {path} does not exist') return - - -config = Config() diff --git a/tests/cli/test_cli.py b/tests/cli/test_cli.py index 15582a3..e4b393c 100644 --- a/tests/cli/test_cli.py +++ b/tests/cli/test_cli.py @@ -91,24 +91,41 @@ def test_hashed_values_registers_hashed_engine(): tool.parse_arguments() config = tool.get_current_config() - try: - assert HashedSecretEngine in config.engines - assert config.engines.count(RegexEngine) == 1 - assert config.rulesets[HashedSecretsRulesetBuilder] == ['/app/tests/fixtures/hashed_secrets.json'] - finally: - # the config singleton outlives this test (KI-CLI-12) - config.rulesets.pop(HashedSecretsRulesetBuilder, None) + assert HashedSecretEngine in config.engines + assert config.engines.count(RegexEngine) == 1 + assert config.rulesets[HashedSecretsRulesetBuilder] == ['/app/tests/fixtures/hashed_secrets.json'] + + +def test_a_parse_starts_from_a_fresh_config(): + # KI-CLI-12: the config was a process-wide singleton, so what one run set stayed set for the next, masking included + base = ['', '--target-dir', '/app/tests/fixtures/', '--outfile', '/tmp/x.sarif'] + first = DeepSecretsCliTool( + args=base + + ['--disable-masking', '--reflect-findings-in-return-code', '--benchmarking-mode'] + + ['--hashed-values', '/app/tests/fixtures/hashed_secrets.json'] + + ['--excluded-paths', '/app/tests/fixtures/false_findings.json'] + ) + first.parse_arguments() + second = DeepSecretsCliTool(args=base) + second.parse_arguments() + + config = second.get_current_config() + assert config is not first.get_current_config() + assert config.disable_masking is False + assert config.return_code_if_findings is False + assert config._benchmarking_mode is False + assert HashedSecretsRulesetBuilder not in config.rulesets + assert not any(path.endswith('false_findings.json') for path in config.global_exclusion_paths) @pytest.mark.parametrize( 'extra, bundles_excluded', [([], False), (['--skip-bundles'], True), (['--excluded-paths', 'disable', '--skip-bundles'], False)], ) -def test_bundle_exclusions_follow_the_flags(monkeypatch, extra, bundles_excluded): +def test_bundle_exclusions_follow_the_flags(extra, bundles_excluded): tool = DeepSecretsCliTool(args=['', '--target-dir', '/app/tests/fixtures/', '--outfile', '/tmp/x.sarif'] + extra) - config = tool.get_current_config() - monkeypatch.setattr(config, 'global_exclusion_paths', []) # the singleton keeps what earlier tests added tool.parse_arguments() + config = tool.get_current_config() names = [path.rsplit('/', 1)[-1] for path in config.global_exclusion_paths] assert ('excluded_bundles.json' in names) is bundles_excluded @@ -122,9 +139,8 @@ def test_bundle_exclusions_follow_the_flags(monkeypatch, extra, bundles_excluded def test_ci_mode_follows_the_flags_then_the_environment(monkeypatch, extra, detected, ci_mode): monkeypatch.setattr('deepsecrets.cli.is_ci_environment', lambda: detected) tool = DeepSecretsCliTool(args=['', '--target-dir', '/app/tests/fixtures/', '--outfile', '/tmp/x.sarif'] + extra) - config = tool.get_current_config() - monkeypatch.setattr(config, 'ci_mode', not ci_mode) # the singleton keeps what earlier tests set tool.parse_arguments() + config = tool.get_current_config() assert config.ci_mode is ci_mode @@ -132,20 +148,18 @@ def test_ci_mode_follows_the_flags_then_the_environment(monkeypatch, extra, dete @pytest.mark.parametrize( 'extra, expected', [([], default_start_method()), (['--multiprocessing-context', 'spawn'], 'spawn')] ) -def test_multiprocessing_context_defaults_to_the_platform_start_method(monkeypatch, extra, expected): +def test_multiprocessing_context_defaults_to_the_platform_start_method(extra, expected): tool = DeepSecretsCliTool(args=['', '--target-dir', '/app/tests/fixtures/', '--outfile', '/tmp/x.sarif'] + extra) - config = tool.get_current_config() - monkeypatch.setattr(config, 'mp_context', 'fork') tool.parse_arguments() + config = tool.get_current_config() assert config.mp_context == expected @pytest.mark.parametrize('extra, expected', [([], DEFAULT_DEEP_MAX_SIZE), (['--deep-max-size', '0'], 0)]) -def test_deep_max_size_flag(monkeypatch, extra, expected): +def test_deep_max_size_flag(extra, expected): tool = DeepSecretsCliTool(args=['', '--target-dir', '/app/tests/fixtures/', '--outfile', '/tmp/x.sarif'] + extra) - config = tool.get_current_config() - monkeypatch.setattr(config, 'deep_max_size', -1) tool.parse_arguments() + config = tool.get_current_config() assert config.deep_max_size == expected diff --git a/tests/output/test_sarif_diagnostics.py b/tests/output/test_sarif_diagnostics.py index 3455ec5..bdbd753 100644 --- a/tests/output/test_sarif_diagnostics.py +++ b/tests/output/test_sarif_diagnostics.py @@ -3,7 +3,6 @@ import pytest from deepsecrets.cli import DeepSecretsCliTool -from deepsecrets.config import config from deepsecrets.core.utils.log import clear_error_list, get_error_list, logger @@ -19,9 +18,6 @@ def target(tmp_path): def scan(target, outfile, *extra): - # the config singleton outlives other tests' runs; benchmarking mode would skip writing the report - config._set_benchmarking_mode(False) - config.set_oneshot_path(None) tool = DeepSecretsCliTool( args=['', '--target-dir', str(target), '--outfile', str(outfile), '--process-count', '1', *extra] ) From 82cf698c5fcfa2b58a030ad7a62758d1c090cbbc Mon Sep 17 00:00:00 2001 From: Nikolai Khechumov Date: Mon, 28 Sep 2026 09:29:04 +0000 Subject: [PATCH 18/22] Set the workers' SIGINT handler only in init_worker iscan_mode called setup_interrupts_for_subprocess() at import, which made any non-main process of an application that merely imported the scanner ignore Ctrl+C. init_worker already sets it for every start method (KI-CLI-17). --- deepsecrets/core/modes/iscan_mode.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/deepsecrets/core/modes/iscan_mode.py b/deepsecrets/core/modes/iscan_mode.py index b14e2cc..3c78605 100644 --- a/deepsecrets/core/modes/iscan_mode.py +++ b/deepsecrets/core/modes/iscan_mode.py @@ -1,7 +1,4 @@ from deepsecrets.core.ui.progress_bar import DSApplicationProgess -from deepsecrets.utils import setup_interrupts_for_subprocess - -setup_interrupts_for_subprocess() import time from dataclasses import dataclass, field @@ -665,8 +662,8 @@ def init_worker(bundle_path: str, task_reporter: DictProxy, task_pids: Any = Non # It must not raise: the pool would replace the worker, the replacement would fail the same way, and the scan # would wait forever for results (KI-DM-30). The failure is kept for the worker's first task to report. global _worker_bundle, _worker_task_reporter, _worker_bundle_path, _worker_startup_error, _worker_task_pids - # a forkserver worker inherits the server's handlers, where this module was imported as the main process, so the - # import-time call at the top of this module did nothing there: Ctrl+C belongs to the parent + # Ctrl+C belongs to the parent, so a worker ignores it. Set here, not at import: a forkserver worker inherits the + # server's default handler, and a call at import would reach any process that imports this module setup_interrupts_for_subprocess() _worker_task_reporter = task_reporter _worker_task_pids = task_pids From 846ca74f7bd3b5558bdb30924cc69b5bc15fe827 Mon Sep 17 00:00:00 2001 From: Nikolai Khechumov Date: Mon, 28 Sep 2026 09:29:04 +0000 Subject: [PATCH 19/22] Key the per-file value cache on the variable name too A value that produced no finding hid every later copy of it in the file, even under a secret-like name (SMTP_USER = 'superset' then SMTP_PASSWORD = 'superset'), and in 2.2 a rejected regex candidate did the same. The cache now keys on content and name (KI-ENG-08). --- deepsecrets/core/utils/file_analyzer.py | 20 ++++++++++--- deepsecrets/diagnostics/trace.py | 11 ++++---- tests/core/utils/test_file_analyzer.py | 37 +++++++++++++++++++++++++ 3 files changed, 59 insertions(+), 9 deletions(-) diff --git a/deepsecrets/core/utils/file_analyzer.py b/deepsecrets/core/utils/file_analyzer.py index c15c588..81ed33e 100644 --- a/deepsecrets/core/utils/file_analyzer.py +++ b/deepsecrets/core/utils/file_analyzer.py @@ -12,6 +12,17 @@ from deepsecrets.core.utils.progress import FileProgress +def answer_key(token: Token) -> tuple: + """What an engine's answer for a token depends on, as far as the per-file value cache is concerned: the content + and, for a variable's value, the variable's name. With the content alone (KI-ENG-08), `x = 'v'` without a finding + hid a later `password = 'v'`, and in 2.2 a value whose regex candidate was rejected hid the same value under a + secret-like name.""" + variable = token.semantic.payload if token.semantic is not None else None + name_token = getattr(variable, 'name_token', None) + name = name_token.content if name_token is not None else getattr(variable, 'name_override', None) + return token.content, name + + class EngineWithTokenizer(BaseModel): engine: IEngine tokenizer: Tokenizer @@ -69,7 +80,7 @@ def _add_silent_regions(self, regions: List): def _run_engine(self, et: EngineWithTokenizer) -> List[Finding]: results: List[Finding] = [] - processed_values: Dict[int, bool] = {} + processed_values: Dict[tuple, bool] = {} if et.tokenizer not in self.tokens: et.tokenizer.add_lifecycle_hooks(self.lifecycle) @@ -90,18 +101,19 @@ def _run_engine(self, et: EngineWithTokenizer) -> List[Finding]: name=et.tokenizer.__class__.__name__, ) - is_known_content = processed_values.get(token.val_hash()) + key = answer_key(token) + is_known_content = processed_values.get(key) if is_known_content is not None and is_known_content is False: continue - processed_values[token.val_hash()] = False + processed_values[key] = False findings: List[Finding] = et.engine.search(token) try: for finding in findings: finding.map_on_file(file=self.file, relative_start=token.span[0]) results.append(finding) - processed_values[token.val_hash()] = True + processed_values[key] = True except Exception as e: logger.exception(f'Unable to process token: {e}') diff --git a/deepsecrets/diagnostics/trace.py b/deepsecrets/diagnostics/trace.py index 7b0aac7..8a75891 100644 --- a/deepsecrets/diagnostics/trace.py +++ b/deepsecrets/diagnostics/trace.py @@ -28,7 +28,7 @@ VariableSuppressionRules, ) from deepsecrets.core.tokenizers.lexer import LexerTokenizer -from deepsecrets.core.utils.file_analyzer import EngineWithTokenizer, FileAnalyzer +from deepsecrets.core.utils.file_analyzer import EngineWithTokenizer, FileAnalyzer, answer_key from deepsecrets.core.utils.finding_merger import FindingMerger from deepsecrets.core.utils.fs import get_path_inside_package @@ -96,23 +96,24 @@ def __init__(self, file: File) -> None: def _run_engine(self, et: EngineWithTokenizer): results = [] - processed_values: Dict[int, bool] = {} + processed_values: Dict[tuple, bool] = {} if et.tokenizer not in self.tokens: self.tokens[et.tokenizer] = et.tokenizer.tokenize(self.file) name = et.tokenizer.__class__.__name__ for token in self.tokens[et.tokenizer]: - known = processed_values.get(token.val_hash()) + key = answer_key(token) # the scanner's cache key (FileAnalyzer._run_engine) + known = processed_values.get(key) if known is not None and known is False: self.decisions.append( {'tokenizer': name, 'engine': et.engine.name, 'span': token.span, 'cache_skip': True} ) continue - processed_values[token.val_hash()] = False + processed_values[key] = False findings = et.engine.search(token) for finding in findings: finding.map_on_file(file=self.file, relative_start=token.span[0]) results.append(finding) - processed_values[token.val_hash()] = True + processed_values[key] = True self.decisions.append( { 'tokenizer': name, diff --git a/tests/core/utils/test_file_analyzer.py b/tests/core/utils/test_file_analyzer.py index 5a13f41..92eeab7 100644 --- a/tests/core/utils/test_file_analyzer.py +++ b/tests/core/utils/test_file_analyzer.py @@ -15,3 +15,40 @@ def test_file_analyzer(file): findings = file_analyzer.process() assert findings is not None + + +def _builtin_bundle(workdir): + from deepsecrets.core.model.internal.processing import AnalyzerBundle + from deepsecrets.core.rulesets.regex import RegexRulesetBuilder + from deepsecrets.core.rulesets.regex_candidate_scoring import RegexCandidateScoringRulesetBuilder + from deepsecrets.core.rulesets.variable_scoring import VariableScoringRulesetBuilder + from deepsecrets.core.utils.fs import get_path_inside_package + + rulesets = {} + for builder, path in ( + (RegexRulesetBuilder, 'rules/regexes.json'), + (RegexCandidateScoringRulesetBuilder, 'rules/regex_candidate_scoring_rules.json'), + (VariableScoringRulesetBuilder, 'rules/variable_scoring_rules.json'), + ): + rulesets[builder.ruleset_name] = builder().with_rules_from_file(get_path_inside_package(path)).rules + return AnalyzerBundle(workdir=str(workdir), engines={'regex': True, 'semantic': True}, rulesets=rulesets) + + +@pytest.mark.parametrize( + 'first', + [ + 'x = "AKIAAAAABBBBBCCCCCDD"', # a regex candidate, rejected (not random) under a name that proves nothing + 'label = "Zx8vQ2mK7pLr4NtY"', # no finding at all + ], +) +def test_a_value_seen_under_another_name_is_still_evaluated(tmp_path, first): + # KI-ENG-08: the per-file value cache keyed on the content alone, so the second line was skipped + from deepsecrets.scan_modes.cli import CliScanMode + + value = first.split('"')[1] + target = tmp_path / 'settings.py' + target.write_text(f'{first}\naws_secret_access_key = "{value}"\n') + result = CliScanMode._per_file_analyzer(_builtin_bundle(tmp_path), str(target), 1, None) + + reported = [f for f in result.findings if f.detection == value] + assert reported and {f.start_line_number for f in reported} == {2} From 90cd89e219143d46c86dd1cd6930a90fb84ca447 Mon Sep 17 00:00:00 2001 From: Nikolai Khechumov Date: Mon, 28 Sep 2026 09:29:04 +0000 Subject: [PATCH 20/22] Tokenizer fixes: one file per call, streamed lexing, valid escapes - A reused CheapVarSearchTokenizer or LexerTokenizer returns only the current file's tokens. - The lexer iterates Pygments' token generator instead of building a list. - Invalid escape sequences in variable-detection patterns are doubled, so Python 3.12+ compiles the module without a SyntaxWarning. --- deepsecrets/core/tokenizers/cheap_var_search.py | 2 ++ .../core/tokenizers/helpers/semantic/deep_analyzer.py | 1 + .../tokenizers/helpers/semantic/var_detection/rules.py | 6 +++--- .../core/tokenizers/helpers/single_token_improver.py | 2 +- deepsecrets/core/tokenizers/lexer.py | 5 ++++- tests/core/tokenizers/test_cheap_var_search.py | 7 +++++++ 6 files changed, 18 insertions(+), 5 deletions(-) diff --git a/deepsecrets/core/tokenizers/cheap_var_search.py b/deepsecrets/core/tokenizers/cheap_var_search.py index 6658a75..b07838d 100644 --- a/deepsecrets/core/tokenizers/cheap_var_search.py +++ b/deepsecrets/core/tokenizers/cheap_var_search.py @@ -27,6 +27,8 @@ def tokenize(self, file: File) -> List[Token]: except Exception as e: logger.exception(e) + # one file per call: a tokenizer reused for another file must not hand back this one's tokens + self.tokens = [] rules: List[CheapVariableDetector] = CheapVariableDetectionRules.for_language(language, lexed=self.lexed) vars: List[Variable] = [] for rule in rules: diff --git a/deepsecrets/core/tokenizers/helpers/semantic/deep_analyzer.py b/deepsecrets/core/tokenizers/helpers/semantic/deep_analyzer.py index 194480c..688f437 100644 --- a/deepsecrets/core/tokenizers/helpers/semantic/deep_analyzer.py +++ b/deepsecrets/core/tokenizers/helpers/semantic/deep_analyzer.py @@ -131,6 +131,7 @@ def final_cleanup(self, tokens_all: Sequence[Token], tokens_to_be_excluded: Sequ if isinstance(tokens_all, LazyTokens): return self._final_cleanup_rows(tokens_all, tokens_to_be_excluded) + # a list or ordered set: only the eager reference tokenizer in tests passes one, the lexer passes LazyTokens if not isinstance(tokens_all, OrderedSet): tokens_all = OrderedSet(tokens_all) diff --git a/deepsecrets/core/tokenizers/helpers/semantic/var_detection/rules.py b/deepsecrets/core/tokenizers/helpers/semantic/var_detection/rules.py index e80e22a..3e7fa5c 100644 --- a/deepsecrets/core/tokenizers/helpers/semantic/var_detection/rules.py +++ b/deepsecrets/core/tokenizers/helpers/semantic/var_detection/rules.py @@ -27,7 +27,7 @@ class VariableDetectionRules: rules = [ VariableDetector( language=Language.PYTHON, - stream_pattern=re.compile('(n)(o|p)(?:\n?)(L)(?:\n|p|\?)'), # noqa + stream_pattern=re.compile('(n)(o|p)(?:\n?)(L)(?:\n|p|\\?)'), # noqa match_rules={2: Match(values=[re.compile('^=$'), re.compile('^:$')])}, match_semantics={1: 'name_token', 3: 'value_token'}, ), @@ -91,7 +91,7 @@ class VariableDetectionRules: ), VariableDetector( language=Language.GOLANG, - stream_pattern=re.compile('(n)(?:o|p){1,3}(\?|u)p(L)p'), # noqa + stream_pattern=re.compile('(n)(?:o|p){1,3}(\\?|u)p(L)p'), # noqa match_rules={2: Match(values=['byte', 'string'])}, match_semantics={1: 'name_token', 3: 'value_token'}, ), @@ -348,7 +348,7 @@ class VariableSuppressionRules(VariableDetectionRules): stream_pattern=re.compile('(n)(p)(n)(p)L'), match_rules={ 1: Match(values=[re.compile('^decode$'), re.compile('^decodeIfPresent$'), re.compile('^unbox$')]), - 2: Match(values=[re.compile('^\($')]), + 2: Match(values=[re.compile('^\\($')]), 3: Match(values=[re.compile('^(key|keyPath)$')]), 4: Match(values=[re.compile('^:$')]), }, diff --git a/deepsecrets/core/tokenizers/helpers/single_token_improver.py b/deepsecrets/core/tokenizers/helpers/single_token_improver.py index fe0f572..e8e8f8d 100644 --- a/deepsecrets/core/tokenizers/helpers/single_token_improver.py +++ b/deepsecrets/core/tokenizers/helpers/single_token_improver.py @@ -27,7 +27,7 @@ def __init__(self, lang: Language) -> None: self.language = lang self.acc = { Language.SHELL: [self._curl_argstring_breakdown], - # Language.PHP: [self._php_variable_dollar_sign_breakdown], # TODO: Uncomment in v2.1 + # Language.PHP: [self._php_variable_dollar_sign_breakdown], # off: it would change PHP tokens (KI-TOK-31) } def applies(self) -> bool: diff --git a/deepsecrets/core/tokenizers/lexer.py b/deepsecrets/core/tokenizers/lexer.py index eab97a8..49ad269 100644 --- a/deepsecrets/core/tokenizers/lexer.py +++ b/deepsecrets/core/tokenizers/lexer.py @@ -58,6 +58,8 @@ def tokenize(self, file: File, post_filter=True) -> List[Token]: self.lexer = self._find_lexer_for_file(file) if not self.lexer: + # not the previous file's tokens, should this tokenizer be reused + self.tokens = [] return self.tokens try: self.language: Language = Language.from_text(self.lexer.filenames[0]) @@ -66,7 +68,8 @@ def tokenize(self, file: File, post_filter=True) -> List[Token]: except Exception as e: logger.exception(e) - raw_tokens = list(self.lexer.get_tokens_unprocessed(file.content)) + # iterated once, as Pygments produces it: a list would hold every raw token of the file at once + raw_tokens = self.lexer.get_tokens_unprocessed(file.content) single_token_improver = SingleTokenImprover(lang=self.language) improving = single_token_improver.applies() # Tokens are recorded as rows and built only when something reads them (see helpers/token_table.py). diff --git a/tests/core/tokenizers/test_cheap_var_search.py b/tests/core/tokenizers/test_cheap_var_search.py index bf7a49a..4cc447d 100644 --- a/tests/core/tokenizers/test_cheap_var_search.py +++ b/tests/core/tokenizers/test_cheap_var_search.py @@ -43,3 +43,10 @@ def test_command_options_and_dotnet_settings_are_variables(): assert names['loginradius:apisecret'].startswith('3f9a1c2e') +def test_a_reused_tokenizer_returns_only_the_current_files_tokens(tmp_path): + first, second = tmp_path / 'a.txt', tmp_path / 'b.txt' + first.write_text('password = "hunter22hunter22"\n') + second.write_text('nothing to see here\n') + tokenizer = CheapVarSearchTokenizer(lexed=True) + assert tokenizer.tokenize(File(path=str(first))) + assert tokenizer.tokenize(File(path=str(second))) == [] From 48bf4d34d77310d6a42f416c51ca2fc6b9fcca2a Mon Sep 17 00:00:00 2001 From: Nikolai Khechumov Date: Mon, 28 Sep 2026 09:29:05 +0000 Subject: [PATCH 21/22] Exact summary counts in both display modes - The worker's last progress write used the literal key 'task_id' and lost the file analyzer's counts (KI-CLI-04). Tokens now travel in the result, and File Errors are recounted from the outcomes, so CI mode and the live display print the same summary. - The live display starts after the pool has created its workers, so fork workers are not forked from a process with the display's thread. - A file that was never analysed reports no depth. --- deepsecrets/cli.py | 2 +- deepsecrets/core/model/internal/processing.py | 2 ++ deepsecrets/core/modes/iscan_mode.py | 11 +++++++++-- deepsecrets/scan_modes/cli.py | 11 +++++++++-- tests/output/test_sarif_diagnostics.py | 5 +++++ tests/scan_modes/test_scan_mode_regressions.py | 8 +++++++- 6 files changed, 33 insertions(+), 6 deletions(-) diff --git a/deepsecrets/cli.py b/deepsecrets/cli.py index 6cbeecd..b18f721 100644 --- a/deepsecrets/cli.py +++ b/deepsecrets/cli.py @@ -459,9 +459,9 @@ def start(self) -> int: # pragma: nocover console.line() if not config.ci_mode: + # run() starts it once the worker pool exists mode.set_progress_bar(progress_bar) mode.progress_bar.set_start_time(startup_time) - mode.progress_bar.start() findings: List[Finding] errors: Dict[str, List[str]] diff --git a/deepsecrets/core/model/internal/processing.py b/deepsecrets/core/model/internal/processing.py index 473c127..497f144 100644 --- a/deepsecrets/core/model/internal/processing.py +++ b/deepsecrets/core/model/internal/processing.py @@ -39,6 +39,8 @@ class PerFileAnalysisResult: # 'deep' (lexer included) or 'shallow' (larger than deep_max_size: regex rules and the cheap variable search); # None for a file that was never analysed (it could not be opened, or it was empty) depth: Optional[str] = field(default=None) + # tokens the engines searched, for the report summary + tokens_processed: int = field(default=0) # ONLY FOR BENCHMARKING MODE _file: Optional[File] = None diff --git a/deepsecrets/core/modes/iscan_mode.py b/deepsecrets/core/modes/iscan_mode.py index 3c78605..bac00f2 100644 --- a/deepsecrets/core/modes/iscan_mode.py +++ b/deepsecrets/core/modes/iscan_mode.py @@ -206,7 +206,6 @@ def refresh_jobs_progress_bars(self): size='| ? Kb', ) - processed = current_state.get('processed', 0) findings = current_state.get('findings', 0) if finished is True: @@ -215,7 +214,6 @@ def refresh_jobs_progress_bars(self): if failure is True: self.stats.failed_files += 1 else: - self.stats.tokens_processed += processed self.stats.total_findings += findings if job.pb_task_id is not None: @@ -381,6 +379,10 @@ def run(self) -> Tuple[List[Finding], Dict[str, List[str]], Dict[str, int]]: initargs=(stage_bundle(bundle, staging), self.active_task_reporter, self._new_task_pids()), ) as pool, ): + # the live display starts after the pool has created its workers: under fork they are then forked + # from a process without the display's refresh thread + if self.progress_bar is not None: + self.progress_bar.start() tid = 0 for file in self.filepaths: tid += 1 @@ -448,6 +450,7 @@ def run(self) -> Tuple[List[Finding], Dict[str, List[str]], Dict[str, int]]: outcome.collected = True outcome.time_ms = analysis_result.processing_time_ms outcome.depth = analysis_result.depth + self.stats.tokens_processed += analysis_result.tokens_processed outcome.errors = analysis_result.errors # a file that logged an error is reported as failed even when the analysis returned status = analysis_result.status @@ -457,6 +460,10 @@ def run(self) -> Tuple[List[Finding], Dict[str, List[str]], Dict[str, int]]: continue final.extend(analysis_result.findings) + # the summary's count, from the outcomes: the same in CI mode (which counts only jobs that raised while it + # runs) and with the live display (which counts what the workers report) + self.stats.failed_files = sum(1 for outcome in self.files.values() if outcome.status in ('error', 'unreadable')) + console.line() console.print('[*] Merging similar findings..') fin = FindingMerger(final).merge() diff --git a/deepsecrets/scan_modes/cli.py b/deepsecrets/scan_modes/cli.py index d6ad5b0..581bb2a 100644 --- a/deepsecrets/scan_modes/cli.py +++ b/deepsecrets/scan_modes/cli.py @@ -146,11 +146,18 @@ def __finalize(result: PerFileAnalysisResult): result.findings = file_analyzer.process() except Exception as e: logger.exception(e) + result.tokens_processed = file_analyzer.progress.processed_count if PROFILER_ON: pass - # always: it records the end time; with no reporter (CI mode) it reports nothing - lifecycle.on_finish(task_reporter.get('task_id') if task_reporter is not None else None) + # always: it records the end time; with no reporter (CI mode) it reports nothing. The file analyzer's last + # report is merged in, so the final one keeps its token and finding counts, and a failure it caught stays one + # (KI-CLI-04) + child_report = task_reporter.get(task_id) if task_reporter is not None else None + if file_analyzer.progress.failure: + lifecycle.on_failure(child_report) + else: + lifecycle.on_finish(child_report) return __finalize(result) diff --git a/tests/output/test_sarif_diagnostics.py b/tests/output/test_sarif_diagnostics.py index bdbd753..18903ef 100644 --- a/tests/output/test_sarif_diagnostics.py +++ b/tests/output/test_sarif_diagnostics.py @@ -76,3 +76,8 @@ def test_diagnostics_say_which_files_got_no_lexer(target, tmp_path): assert {a['location']['uri']: a for a in run['artifacts']}['src/settings.py']['properties']['depth'] == 'deep' +def test_a_file_never_analysed_has_no_depth(tmp_path): + (tmp_path / 'empty.py').write_text('') + run = scan(tmp_path, tmp_path / 'report.sarif', '--report-diagnostics') + properties = {a['location']['uri']: a for a in run['artifacts']}['empty.py']['properties'] + assert properties['status'] != 'ok' and 'depth' not in properties diff --git a/tests/scan_modes/test_scan_mode_regressions.py b/tests/scan_modes/test_scan_mode_regressions.py index e5341d3..0b711f8 100644 --- a/tests/scan_modes/test_scan_mode_regressions.py +++ b/tests/scan_modes/test_scan_mode_regressions.py @@ -445,7 +445,13 @@ def test_ci_mode_needs_no_manager_and_finds_the_same(): try: assert (mode._mp_manager is None) is ci_mode findings, errors, _ = _run_with_timeout(mode, timeout=120) - results[ci_mode] = (sorted((f.file.path, f.start_offset, f.detection) for f in findings), errors) + # the summary's token count comes from the results, the same in both modes (it was 0 in CI mode) + assert mode.stats.tokens_processed > 0 + results[ci_mode] = ( + sorted((f.file.path, f.start_offset, f.detection) for f in findings), + errors, + mode.stats.tokens_processed, + ) assert mode.stats.finished == 4 if ci_mode: assert mode.stats.total_findings == len(findings) From 7aa2f31028e7040aafd47738df71f06503822c28 Mon Sep 17 00:00:00 2001 From: Nikolai Khechumov Date: Mon, 28 Sep 2026 09:29:05 +0000 Subject: [PATCH 22/22] Release 2.2.0 - Version 2.2.0; requires Python 3.11 (tomllib). - --outformat json is refused before the scan, as announced in 2.1.0. - start() split into plan, summary, report and release steps; the report is built before its file is opened. - The --outfile directory is checked while parsing; a bare --hashed-values disables the check; --skip-bundles works with custom --excluded-paths; exclusion files keep their order; --multiprocessing-context offers only the platform's start methods. - README: new flags, confidence tiers, Python version. --- .vscode/launch.json | 3 +- README.md | 10 +- deepsecrets/cli.py | 234 +++++++++++++++++++----------------------- deepsecrets/config.py | 7 +- poetry.lock | 91 +--------------- pyproject.toml | 4 +- tests/cli/test_cli.py | 46 ++++++++- 7 files changed, 168 insertions(+), 227 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 38d3b2f..22eb652 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -110,7 +110,8 @@ "--outfile", "./fdsafad.json", "--outformat", - "json", + "sarif", + "--multiprocessing-context", "spawn", "--process-count", "1", "--max-file-size", "0", "--disable-masking" diff --git a/README.md b/README.md index 045c5b5..5bece79 100644 --- a/README.md +++ b/README.md @@ -45,6 +45,8 @@ From PyPi $ pip install deepsecrets ``` +DeepSecrets needs Python 3.11 or later. + ## Scanning The easiest way to run a scan: @@ -65,7 +67,11 @@ The `--help` command is always ready to guide you, but here are the key flags yo * `--hashed-values /path/to/hashes.json`: Provide a list of pre-hashed known production secrets to search for them securely. * `--excluded-paths /path/to/exclusions.json`: Override or extend the default paths ignored during scanning. * `--deep-max-size BYTES`: Files larger than this (default 250,000 bytes) get the fast analysis: the regex rules and the variable search, without the language lexer. Big files are mostly generated or minified, the lexer costs most of the scan time on them and rarely finds more there. `0` gives every file the full analysis. -* `--skip-bundles`: Skip minified JavaScript, source maps and bundles (`*.min.js`, `*.map`, `*.bundle.js`) entirely, on top of the default exclusions. +* `--skip-bundles`: Skip minified JavaScript, source maps and bundles (`*.min.js`, `*.map`, `*.bundle.js`) entirely, on top of the other path exclusions. +* `--confidence-level {all,low,medium,high,very-high}`: The lowest confidence to report (default `low`, every finding). `medium`, `high` and `very-high` keep confidence 3, 6 and 9 or more; `all` also reports the regex matches the scanner judged not to be secrets (placeholders, examples), with an `-INFO` rule id. +* `--report-diagnostics`: Add every file found under the target to the SARIF report, with its status, scan time, skip reason and analysis depth. +* `--regex-candidate-scoring-rules /path/to/rules.json`: Supply your own rules for judging regex matches before they are reported. Give the flag no value to report regex matches unjudged. +* `--multiprocessing-context {forkserver,spawn,fork}`: How worker processes start. The default is `forkserver` where the platform has it (Linux, macOS): workers share the scanner already loaded in a server process. Windows uses `spawn`. All three give the same findings. * `--disable-masking`: Keep potential secrets unmasked in the output report *(see caution below)*. * `--ci` / `--no-ci`: Turn CI mode on or off. CI mode prints a plain progress line every 30 seconds instead of the live progress display. It turns on by itself inside CI services (GitHub Actions, GitLab CI, Jenkins and others) and whenever the output is not a terminal. @@ -116,7 +122,7 @@ Masking doesn't break the deduplication logic of downstream platforms (like Gith Every finding gets a confidence score. However, different security platforms parse SARIF metrics differently. To ensure compatibility across modern ASPM dashboards, DeepSecrets does the following: -* **Virtual Subrules (`rules[]`)**: Dynamically generates rules like `S105-LOW` or `S105-CRITICAL`. This forces GitHub Security and DefectDojo to map semantic precision variance properly without breaking native parsers. +* **Virtual Subrules (`rules[]`)**: Dynamically generates rules by confidence tier, from `S105-LOW` through `-MEDIUM` and `-HIGH` to `S105-VERY-HIGH` (and `-INFO` for the rejected matches `--confidence-level all` reports). This forces GitHub Security and DefectDojo to map semantic precision variance properly without breaking native parsers. * **Deterministic Result Level**: The tool always explicitly sets `level: error` in the `results[]` model. This acts as a universal fallback for CI/CD pipelines and older SAST parsers, ensuring that exposed secrets reliably break builds or block Pull Requests regardless of individual rule interpretations. diff --git a/deepsecrets/cli.py b/deepsecrets/cli.py index b18f721..e14eaba 100644 --- a/deepsecrets/cli.py +++ b/deepsecrets/cli.py @@ -1,9 +1,10 @@ import argparse from datetime import datetime -import json +import os from argparse import RawTextHelpFormatter from typing import Dict, List from jschema_to_python.to_json import to_json +from multiprocessing import get_all_start_methods from deepsecrets import MODULE_NAME, console from deepsecrets.config import ( @@ -11,7 +12,6 @@ DEFAULT_CONFIDENCE_LEVEL, DEFAULT_DEEP_MAX_SIZE, SCANNER_VERSION, - SCANNER_VERSION_NUMERIC, Config, Output, ) @@ -19,7 +19,6 @@ from deepsecrets.core.engines.regex import RegexEngine from deepsecrets.core.engines.semantic import SemanticEngine from deepsecrets.core.model.finding import Finding -from deepsecrets.core.model.response.builtin import BuiltinFormatResponseBuilder from deepsecrets.core.model.response.dojo_sarif import DojoSarifResponseBuilder from deepsecrets.core.rulesets.false_findings import FalseFindingsBuilder from deepsecrets.core.rulesets.hashed_secrets import HashedSecretsRulesetBuilder @@ -28,6 +27,7 @@ from deepsecrets.core.rulesets.variable_scoring import VariableScoringRulesetBuilder from deepsecrets.core.ui.progress_bar import DSApplicationProgess from deepsecrets.core.ui.time_remaining_column import SyncedTimeRemainingColumn +from deepsecrets.core.utils.exceptions import FileNotFoundException from deepsecrets.core.utils.fs import get_abspath, get_path_inside_package from deepsecrets.core.utils.log import logger from deepsecrets.scan_modes.cli import CliScanMode @@ -81,9 +81,6 @@ def __init__(self, args: List[str]): def say_hello(self) -> None: console.line(2) console.rule('DeepSecrets', characters='=') - - from rich.panel import Panel - console.print( Align( Panel.fit( @@ -140,7 +137,8 @@ def _build_argparser(self) -> None: '--hashed-values', nargs='*', type=str, - help='Path to your Hashed Values set.\n' "Don't set any value to disable this checks\n", + help='Path to your Hashed Values set.\n' + 'Leave the flag out, or give it no value, to disable these checks\n', ) parser.add_argument( @@ -180,8 +178,9 @@ def _build_argparser(self) -> None: parser.add_argument( '--skip-bundles', action='store_true', - help='Add minified JavaScript, source maps and bundles (*.min.js, *.map, *.bundle.js) to the built-in\n' - 'path exclusions. Most of them are larger than --deep-max-size and get the fast analysis anyway.\n', + help='Also skip minified JavaScript, source maps and bundles (*.min.js, *.map, *.bundle.js), whatever\n' + "--excluded-paths lists (except 'disable'). Most of them are larger than --deep-max-size and get the\n" + 'fast analysis anyway.\n', ) parser.add_argument( @@ -250,7 +249,8 @@ def _build_argparser(self) -> None: '--multiprocessing-context', type=str, default=None, - choices=['fork', 'spawn', 'forkserver'], + # the platform's own: fork and forkserver do not exist on Windows + choices=get_all_start_methods(), help='How worker processes start. Default: forkserver where available (Linux, macOS), spawn elsewhere.\n' 'forkserver workers share the scanner already loaded in the server instead of loading it again.\n', ) @@ -272,7 +272,7 @@ def _build_argparser(self) -> None: choices=['json', 'sarif', 'dojo-sarif'], help='"sarif": SARIF format (specification accurate, default)\n' '"dojo-sarif": SARIF format (compatible with DefectDojo\'s parser)\n' - '"json": old internal format (deprecated and will be removed soon)', + '"json": the old internal format, removed in 2.2.0 (exits with an error)', ) parser.add_argument( @@ -286,7 +286,8 @@ def _build_argparser(self) -> None: '--report-diagnostics', action='store_true', help='Add scan diagnostics to a SARIF report: every file found under the target directory in\n' - 'run.artifacts (scan time in ms, status, skip reason) and per-file errors in run.invocations.\n', + 'run.artifacts (scan time in ms, status, skip reason, analysis depth) and per-file errors in\n' + 'run.invocations.\n', ) parser.add_argument('--benchmarking-mode', help=argparse.SUPPRESS, action='store_true') @@ -299,9 +300,6 @@ def parse_arguments(self) -> None: config = self.config = Config() user_args = self.argparser.parse_args(args=self.args[1:]) - if user_args.verbose: - pass - # config.set_logging_level(logging.DEBUG) if user_args.disable_masking: config.set_disable_masking(True) @@ -323,6 +321,10 @@ def parse_arguments(self) -> None: config.set_ci_mode(is_ci_environment() if user_args.ci is None else user_args.ci) config.set_verbose(user_args.verbose) config.output = Output(type=user_args.outformat, path=user_args.outfile) + # checked now, not when the report is written after the whole scan + report_dir = os.path.dirname(get_abspath(user_args.outfile)) + if not os.path.isdir(report_dir): + raise FileNotFoundException(f'--outfile: the directory {report_dir} does not exist') if user_args.reflect_findings_in_return_code: config.return_code_if_findings = True @@ -330,12 +332,10 @@ def parse_arguments(self) -> None: EXCLUDE_PATHS_BUILTIN = get_path_inside_package('rules/excluded_paths.json') if user_args.excluded_paths is not None: rules = [rule.replace('built-in', EXCLUDE_PATHS_BUILTIN) for rule in user_args.excluded_paths] - if 'built-in' in user_args.excluded_paths and user_args.skip_bundles: + if user_args.skip_bundles and 'disable' not in user_args.excluded_paths: rules.append(get_path_inside_package('rules/excluded_bundles.json')) config.set_global_exclusion_paths(rules) - config.engines = [] - REGEX_BUILTIN_RULESET = get_path_inside_package('rules/regexes.json') if user_args.regex_rules is not None: rules = [rule.replace('built-in', REGEX_BUILTIN_RULESET) for rule in user_args.regex_rules] @@ -362,7 +362,9 @@ def parse_arguments(self) -> None: config.add_ruleset(VariableScoringRulesetBuilder, rules) conf_hashed_ruleset = user_args.hashed_values - if conf_hashed_ruleset is not None and conf_hashed_ruleset != DISABLED: + # the flag without a value disables the check, as its help says; an empty ruleset would only keep the lexer on + # every file for nothing (the hashed engine lexes files over --deep-max-size too) + if conf_hashed_ruleset and conf_hashed_ruleset != DISABLED: config.engines.append(HashedSecretEngine) config.add_ruleset(HashedSecretsRulesetBuilder, conf_hashed_ruleset) @@ -373,9 +375,6 @@ def parse_arguments(self) -> None: def get_current_config(self) -> Config: return self.config - def _add_ignorefiles(self, files: List[str]): - self.config.set_global_exclusion_paths(files) - def start(self) -> int: # pragma: nocover startup_time = datetime.now() try: @@ -386,69 +385,10 @@ def start(self) -> int: # pragma: nocover config = self.config if config.output.type == 'json': - console.print('\n') - if SCANNER_VERSION_NUMERIC[0] == 2 and SCANNER_VERSION_NUMERIC[1] < 2: - console.print( - Align( - Panel( - "The internal JSON report format is now DEPRECATED and will be removed in release 2.2.0\n\nConsider switching now.", - padding=(1, 2), - title=Text('SARIF IS NOW DEFAULT OUTPUT FORMAT', style='reverse'), - highlight=True, - subtitle=Text(' --outformat sarif ', style='reverse'), - title_align='center', - width=90, - subtitle_align='center', - box=box.HEAVY, - style='black on orange_red1', - expand=False, - ), - align='center', - ) - ) - - else: - console.print( - Align( - Panel( - f"The internal JSON report format was DEPRECATED since the release 2.2.0.\nNow ({SCANNER_VERSION}) it is REMOVED. Switch to SARIF\n.", - padding=(1, 2), - title=Text('SARIF IS NOW DEFAULT OUTPUT FORMAT', style='reverse'), - highlight=True, - subtitle=Text(' --outformat sarif ', style='reverse'), - title_align='center', - width=90, - subtitle_align='center', - box=box.HEAVY, - style='black on orange_red1', - expand=False, - ), - align='center', - ) - ) - return ReturnCodes.ERROR - - console.print('\n\n') - - console.rule( - f'Planning a scan against {config.workdir_path} using {config.process_count} process(es)', characters='=' - ) - console.line() - if config.disable_masking is True: - console.print( - '[bold red]:warning: SECRETS MASKING IS DISABLED. REPORT WILL CONTAIN SECRETS IN PLAINTEXT. BE CAREFUL!\n', - justify='center', - ) - else: - console.print( - '[bold green]:warning: SECRETS MASKING IS ENABLED. FINGERPRINTS ARE UNAFFECTED\n(downstream ASPM deduplication will work normally)\n', - justify='center', - ) + self._refuse_json_output() + return ReturnCodes.ERROR - if config.return_code_if_findings is True: - console.print( - f'[bold yellow]:warning:[/bold yellow] The tool will return code of {ReturnCodes.FINDINGS_DETECTED} if any findings are detected\n' - ) + self._print_plan() try: mode = CliScanMode(config=config) @@ -463,32 +403,80 @@ def start(self) -> int: # pragma: nocover mode.set_progress_bar(progress_bar) mode.progress_bar.set_start_time(startup_time) - findings: List[Finding] - errors: Dict[str, List[str]] - timings: Dict[str, int] - try: findings, errors, timings = mode.run() except WorkerStartupError as e: if mode.progress_bar is not None: mode.progress_bar.stop() console.print(f'[bold red][!] Scan stopped: {e}. Every file it took would fail the same way.[/bold red]') - mode.dispose() - stop_forkserver() + self._release(mode) return ReturnCodes.ERROR - ''' - for finding in findings: - if finding._mapped_on_file is False: - continue - finding.file = None - ''' - if mode.progress_bar is not None: mode.progress_bar.stop() - finish_time = datetime.now() report_path = get_abspath(config.output.path) + self._print_summary(mode, findings, errors, datetime.now() - startup_time, report_path) + + if config._benchmarking_mode is True: + return findings, errors, timings, mode._oneshot_file + + self._write_report(mode, findings, report_path) + self._say_goodbye(findings) + self._release(mode) + + if len(findings) > 0 and config.return_code_if_findings: + return ReturnCodes.FINDINGS_DETECTED + return ReturnCodes.OK + + @staticmethod + def _refuse_json_output() -> None: # pragma: nocover + console.print('\n') + console.print( + Align( + Panel( + 'The internal JSON report format was deprecated in 2.1 and is removed in 2.2.0.\n' + 'Use --outformat sarif (the default) or dojo-sarif.', + padding=(1, 2), + title=Text('SARIF IS NOW DEFAULT OUTPUT FORMAT', style='reverse'), + highlight=True, + subtitle=Text(' --outformat sarif ', style='reverse'), + title_align='center', + width=90, + subtitle_align='center', + box=box.HEAVY, + style='black on orange_red1', + expand=False, + ), + align='center', + ) + ) + + def _print_plan(self) -> None: # pragma: nocover + config = self.config + console.rule( + f'Planning a scan against {config.workdir_path} using {config.process_count} process(es)', characters='=' + ) + console.line() + if config.disable_masking is True: + console.print( + '[bold red]:warning: SECRETS MASKING IS DISABLED. REPORT WILL CONTAIN SECRETS IN PLAINTEXT. BE CAREFUL!\n', + justify='center', + ) + else: + console.print( + '[bold green]:warning: SECRETS MASKING IS ENABLED. FINGERPRINTS ARE UNAFFECTED\n(downstream ASPM deduplication will work normally)\n', + justify='center', + ) + + if config.return_code_if_findings is True: + console.print( + f'[bold yellow]:warning:[/bold yellow] The tool will return code of {ReturnCodes.FINDINGS_DETECTED} if any findings are detected\n' + ) + + def _print_summary( + self, mode: CliScanMode, findings: List[Finding], errors: Dict[str, List[str]], elapsed, report_path: str + ) -> None: # pragma: nocover console.line() console.print('[bold green]Scanning finished successfully', justify='center') console.line() @@ -509,7 +497,7 @@ def start(self) -> int: # pragma: nocover Align('Processed Files (Tokens)', vertical='middle'), f'{str(len(mode.filepaths))} ({mode.stats.tokens_processed})', ) - table.add_row(Align('Elapsed', vertical='middle'), f'{(finish_time-startup_time).total_seconds():.1f}s') + table.add_row(Align('Elapsed', vertical='middle'), f'{elapsed.total_seconds():.1f}s') errors_line_color = '[bold red]' if len(errors.keys()) > 0 else '[bold green]' table.add_row( Align(f'{errors_line_color}File Errors', vertical='middle'), @@ -522,36 +510,24 @@ def start(self) -> int: # pragma: nocover Align(f'{findings_line_color}Potential Findings', vertical='middle'), f'{findings_line_color}{str(len(findings))}', ) - table.add_row(Align(f'Report Location ({config.output.type})', vertical='middle'), report_path) + table.add_row(Align(f'Report Location ({self.config.output.type})', vertical='middle'), report_path) console.print(Align(table, align='center')) - if config._benchmarking_mode is True: - return findings, errors, timings, mode._oneshot_file - + def _write_report(self, mode: CliScanMode, findings: List[Finding], report_path: str) -> None: + # sarif and dojo-sarif are the same document; built before the file is opened, so a failure leaves no + # truncated report behind + report = ( + DojoSarifResponseBuilder() + .with_current_mode(mode) + .with_findings_list(findings) + .with_masking_enabled(not self.config.disable_masking) + .build() + ) with open(report_path, 'w+') as f: + f.write(to_json(report)) - if config.output.type == 'json': - json.dump( - BuiltinFormatResponseBuilder() - .with_current_mode(mode) - .with_findings_list(findings) - .with_masking_enabled(not config.disable_masking) - .build(), - f, - ) - - if config.output.type in ['sarif', 'dojo-sarif']: - f.write( - to_json( - DojoSarifResponseBuilder() - .with_current_mode(mode) - .with_findings_list(findings) - .with_masking_enabled(not config.disable_masking) - .build() - ) - ) - - if len(findings) > 0 and config.disable_masking: + def _say_goodbye(self, findings: List[Finding]) -> None: # pragma: nocover + if len(findings) > 0 and self.config.disable_masking: console.print( '[bold red]:warning: SECRETS MASKING WAS DISABLED, THE REPORT CONTAINS POTENTIAL SECRETS IN PLAINTEXT.\nBE CAREFUL!', justify='center', @@ -566,12 +542,10 @@ def start(self) -> int: # pragma: nocover console.line() console.print(Align('[bold green]FINISHED', align='center')) console.line(2) + + @staticmethod + def _release(mode: CliScanMode) -> None: mode.dispose() # the pool and the manager are gone: wait for the forkserver, so whoever waits for this process sees the # workers' CPU time (KI-CLI-42) stop_forkserver() - - if len(findings) > 0 and config.return_code_if_findings: - return ReturnCodes.FINDINGS_DETECTED - - return ReturnCodes.OK diff --git a/deepsecrets/config.py b/deepsecrets/config.py index 7a9e00e..5ca2ce8 100644 --- a/deepsecrets/config.py +++ b/deepsecrets/config.py @@ -28,8 +28,7 @@ def deep_analysis(size: int, deep_max_size: int) -> bool: SCANNER_NAME = "DeepSecrets" -SCANNER_VERSION = "2.1.1" -SCANNER_VERSION_NUMERIC = [int(subver) for subver in SCANNER_VERSION.split('.')] +SCANNER_VERSION = "2.2.0" SCANNER_URL = "https://github.com/ntoskernel/deepsecrets" MAX_LINE_LENGTH_FOR_CONTEXT = 300 @@ -151,7 +150,9 @@ def set_global_exclusion_paths(self, paths: List[str]) -> None: raise FileNotFoundException(f'global_exclusion_path does not exist ({path})') self.global_exclusion_paths.append(path) - self.global_exclusion_paths = list(set(self.global_exclusion_paths)) + # de-duplicated in order: the first file's patterns are matched first, so a skip reason names the same + # pattern on every run (a set's order changed with the hash seed) + self.global_exclusion_paths = list(dict.fromkeys(self.global_exclusion_paths)) def add_ruleset(self, type: Type, paths: List[str] = []) -> None: self._validate_paths(paths) diff --git a/poetry.lock b/poetry.lock index 1555012..671e59f 100644 --- a/poetry.lock +++ b/poetry.lock @@ -81,8 +81,6 @@ packaging = ">=22.0" pathspec = ">=1.0.0" platformdirs = ">=2" pytokens = ">=0.4.0,<0.5.0" -tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} -typing-extensions = {version = ">=4.0.1", markers = "python_version < \"3.11\""} [package.extras] colorama = ["colorama (>=0.4.3)"] @@ -249,31 +247,9 @@ files = [ {file = "coverage-7.16.0.tar.gz", hash = "sha256:077f0964087883176ff6ab9b074694cae29f8c708273b13ca62c183c6ed716cd"}, ] -[package.dependencies] -tomli = {version = "*", optional = true, markers = "python_full_version <= \"3.11.0a6\" and extra == \"toml\""} - [package.extras] toml = ["tomli ; python_full_version <= \"3.11.0a6\""] -[[package]] -name = "exceptiongroup" -version = "1.3.1" -description = "Backport of PEP 654 (exception groups)" -optional = false -python-versions = ">=3.7" -groups = ["test"] -markers = "python_version == \"3.10\"" -files = [ - {file = "exceptiongroup-1.3.1-py3-none-any.whl", hash = "sha256:a7a39a3bd276781e98394987d3a5701d0c4edffb633bb7a5144577f82c773598"}, - {file = "exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219"}, -] - -[package.dependencies] -typing-extensions = {version = ">=4.6.0", markers = "python_version < \"3.13\""} - -[package.extras] -test = ["pytest (>=6)"] - [[package]] name = "iniconfig" version = "2.3.0" @@ -830,12 +806,10 @@ files = [ [package.dependencies] colorama = {version = ">=0.4", markers = "sys_platform == \"win32\""} -exceptiongroup = {version = ">=1", markers = "python_version < \"3.11\""} iniconfig = ">=1.0.1" packaging = ">=22" pluggy = ">=1.5,<2" pygments = ">=2.7.2" -tomli = {version = ">=1", markers = "python_version < \"3.11\""} [package.extras] dev = ["argcomplete", "attrs (>=19.2)", "hypothesis (>=3.56)", "mock", "requests", "setuptools", "xmlschema"] @@ -1194,76 +1168,17 @@ enabler = ["pytest-enabler (>=2.2)"] test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21) ; python_version >= \"3.9\" and sys_platform != \"cygwin\"", "jaraco.envs (>=2.2)", "jaraco.path (>=3.7.2)", "jaraco.test (>=5.5)", "packaging (>=24.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-home (>=0.5)", "pytest-perf ; sys_platform != \"cygwin\"", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel (>=0.44.0)"] type = ["importlib_metadata (>=7.0.2) ; python_version < \"3.10\"", "jaraco.develop (>=7.21) ; sys_platform != \"cygwin\"", "mypy (==1.18.*)", "pytest-mypy"] -[[package]] -name = "tomli" -version = "2.4.1" -description = "A lil' TOML parser" -optional = false -python-versions = ">=3.8" -groups = ["dev", "test"] -files = [ - {file = "tomli-2.4.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f8f0fc26ec2cc2b965b7a3b87cd19c5c6b8c5e5f436b984e85f486d652285c30"}, - {file = "tomli-2.4.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4ab97e64ccda8756376892c53a72bd1f964e519c77236368527f758fbc36a53a"}, - {file = "tomli-2.4.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:96481a5786729fd470164b47cdb3e0e58062a496f455ee41b4403be77cb5a076"}, - {file = "tomli-2.4.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5a881ab208c0baf688221f8cecc5401bd291d67e38a1ac884d6736cbcd8247e9"}, - {file = "tomli-2.4.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:47149d5bd38761ac8be13a84864bf0b7b70bc051806bc3669ab1cbc56216b23c"}, - {file = "tomli-2.4.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ec9bfaf3ad2df51ace80688143a6a4ebc09a248f6ff781a9945e51937008fcbc"}, - {file = "tomli-2.4.1-cp311-cp311-win32.whl", hash = "sha256:ff2983983d34813c1aeb0fa89091e76c3a22889ee83ab27c5eeb45100560c049"}, - {file = "tomli-2.4.1-cp311-cp311-win_amd64.whl", hash = "sha256:5ee18d9ebdb417e384b58fe414e8d6af9f4e7a0ae761519fb50f721de398dd4e"}, - {file = "tomli-2.4.1-cp311-cp311-win_arm64.whl", hash = "sha256:c2541745709bad0264b7d4705ad453b76ccd191e64aa6f0fc66b69a293a45ece"}, - {file = "tomli-2.4.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:c742f741d58a28940ce01d58f0ab2ea3ced8b12402f162f4d534dfe18ba1cd6a"}, - {file = "tomli-2.4.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7f86fd587c4ed9dd76f318225e7d9b29cfc5a9d43de44e5754db8d1128487085"}, - {file = "tomli-2.4.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ff18e6a727ee0ab0388507b89d1bc6a22b138d1e2fa56d1ad494586d61d2eae9"}, - {file = "tomli-2.4.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:136443dbd7e1dee43c68ac2694fde36b2849865fa258d39bf822c10e8068eac5"}, - {file = "tomli-2.4.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:5e262d41726bc187e69af7825504c933b6794dc3fbd5945e41a79bb14c31f585"}, - {file = "tomli-2.4.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5cb41aa38891e073ee49d55fbc7839cfdb2bc0e600add13874d048c94aadddd1"}, - {file = "tomli-2.4.1-cp312-cp312-win32.whl", hash = "sha256:da25dc3563bff5965356133435b757a795a17b17d01dbc0f42fb32447ddfd917"}, - {file = "tomli-2.4.1-cp312-cp312-win_amd64.whl", hash = "sha256:52c8ef851d9a240f11a88c003eacb03c31fc1c9c4ec64a99a0f922b93874fda9"}, - {file = "tomli-2.4.1-cp312-cp312-win_arm64.whl", hash = "sha256:f758f1b9299d059cc3f6546ae2af89670cb1c4d48ea29c3cacc4fe7de3058257"}, - {file = "tomli-2.4.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:36d2bd2ad5fb9eaddba5226aa02c8ec3fa4f192631e347b3ed28186d43be6b54"}, - {file = "tomli-2.4.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:eb0dc4e38e6a1fd579e5d50369aa2e10acfc9cace504579b2faabb478e76941a"}, - {file = "tomli-2.4.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c7f2c7f2b9ca6bdeef8f0fa897f8e05085923eb091721675170254cbc5b02897"}, - {file = "tomli-2.4.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f3c6818a1a86dd6dca7ddcaaf76947d5ba31aecc28cb1b67009a5877c9a64f3f"}, - {file = "tomli-2.4.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d312ef37c91508b0ab2cee7da26ec0b3ed2f03ce12bd87a588d771ae15dcf82d"}, - {file = "tomli-2.4.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:51529d40e3ca50046d7606fa99ce3956a617f9b36380da3b7f0dd3dd28e68cb5"}, - {file = "tomli-2.4.1-cp313-cp313-win32.whl", hash = "sha256:2190f2e9dd7508d2a90ded5ed369255980a1bcdd58e52f7fe24b8162bf9fedbd"}, - {file = "tomli-2.4.1-cp313-cp313-win_amd64.whl", hash = "sha256:8d65a2fbf9d2f8352685bc1364177ee3923d6baf5e7f43ea4959d7d8bc326a36"}, - {file = "tomli-2.4.1-cp313-cp313-win_arm64.whl", hash = "sha256:4b605484e43cdc43f0954ddae319fb75f04cc10dd80d830540060ee7cd0243cd"}, - {file = "tomli-2.4.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:fd0409a3653af6c147209d267a0e4243f0ae46b011aa978b1080359fddc9b6cf"}, - {file = "tomli-2.4.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:a120733b01c45e9a0c34aeef92bf0cf1d56cfe81ed9d47d562f9ed591a9828ac"}, - {file = "tomli-2.4.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:559db847dc486944896521f68d8190be1c9e719fced785720d2216fe7022b662"}, - {file = "tomli-2.4.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:01f520d4f53ef97964a240a035ec2a869fe1a37dde002b57ebc4417a27ccd853"}, - {file = "tomli-2.4.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7f94b27a62cfad8496c8d2513e1a222dd446f095fca8987fceef261225538a15"}, - {file = "tomli-2.4.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ede3e6487c5ef5d28634ba3f31f989030ad6af71edfb0055cbbd14189ff240ba"}, - {file = "tomli-2.4.1-cp314-cp314-win32.whl", hash = "sha256:3d48a93ee1c9b79c04bb38772ee1b64dcf18ff43085896ea460ca8dec96f35f6"}, - {file = "tomli-2.4.1-cp314-cp314-win_amd64.whl", hash = "sha256:88dceee75c2c63af144e456745e10101eb67361050196b0b6af5d717254dddf7"}, - {file = "tomli-2.4.1-cp314-cp314-win_arm64.whl", hash = "sha256:b8c198f8c1805dc42708689ed6864951fd2494f924149d3e4bce7710f8eb5232"}, - {file = "tomli-2.4.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:d4d8fe59808a54658fcc0160ecfb1b30f9089906c50b23bcb4c69eddc19ec2b4"}, - {file = "tomli-2.4.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7008df2e7655c495dd12d2a4ad038ff878d4ca4b81fccaf82b714e07eae4402c"}, - {file = "tomli-2.4.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1d8591993e228b0c930c4bb0db464bdad97b3289fb981255d6c9a41aedc84b2d"}, - {file = "tomli-2.4.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:734e20b57ba95624ecf1841e72b53f6e186355e216e5412de414e3c51e5e3c41"}, - {file = "tomli-2.4.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8a650c2dbafa08d42e51ba0b62740dae4ecb9338eefa093aa5c78ceb546fcd5c"}, - {file = "tomli-2.4.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:504aa796fe0569bb43171066009ead363de03675276d2d121ac1a4572397870f"}, - {file = "tomli-2.4.1-cp314-cp314t-win32.whl", hash = "sha256:b1d22e6e9387bf4739fbe23bfa80e93f6b0373a7f1b96c6227c32bef95a4d7a8"}, - {file = "tomli-2.4.1-cp314-cp314t-win_amd64.whl", hash = "sha256:2c1c351919aca02858f740c6d33adea0c5deea37f9ecca1cc1ef9e884a619d26"}, - {file = "tomli-2.4.1-cp314-cp314t-win_arm64.whl", hash = "sha256:eab21f45c7f66c13f2a9e0e1535309cee140182a9cdae1e041d02e47291e8396"}, - {file = "tomli-2.4.1-py3-none-any.whl", hash = "sha256:0d85819802132122da43cb86656f8d1f8c6587d54ae7dcaf30e90533028b49fe"}, - {file = "tomli-2.4.1.tar.gz", hash = "sha256:7c7e1a961a0b2f2472c1ac5b69affa0ae1132c39adcb67aba98568702b9cc23f"}, -] -markers = {dev = "python_version == \"3.10\"", test = "python_full_version <= \"3.11.0a6\""} - [[package]] name = "typing-extensions" version = "4.15.0" description = "Backported and Experimental Type Hints for Python 3.9+" optional = false python-versions = ">=3.9" -groups = ["main", "dev", "test"] +groups = ["main"] files = [ {file = "typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548"}, {file = "typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466"}, ] -markers = {dev = "python_version == \"3.10\"", test = "python_version == \"3.10\""} [[package]] name = "typing-inspection" @@ -1282,5 +1197,5 @@ typing-extensions = ">=4.12.0" [metadata] lock-version = "2.1" -python-versions = ">=3.10,<4.0.0" -content-hash = "f7e3ec827644dd6d664ac0d7b55ce329e378123005f3828a7ab12527b1d2cba2" +python-versions = ">=3.11,<4.0.0" +content-hash = "1ad0862c42d61687c9a89081a313166b541afe055cb15a8c83fd2f9b8ccd84e5" diff --git a/pyproject.toml b/pyproject.toml index 4880219..85185bd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "deepsecrets" -version = "2.1.1" +version = "2.2.0" description = "A better tool for secrets scanning" license = { text = "MIT" } readme = "README.md" @@ -15,7 +15,7 @@ maintainers = [ keywords = ["security", "secrets", "credentials", "scanning", "appsec", "code", "search"] packages = [{include = "deepsecrets"}] -requires-python = ">=3.10,<4.0.0" +requires-python = ">=3.11,<4.0.0" classifiers = [ "Programming Language :: Python :: 3", diff --git a/tests/cli/test_cli.py b/tests/cli/test_cli.py index e4b393c..d4c0df6 100644 --- a/tests/cli/test_cli.py +++ b/tests/cli/test_cli.py @@ -1,10 +1,11 @@ import pytest -from deepsecrets.cli import DeepSecretsCliTool +from deepsecrets.cli import DeepSecretsCliTool, ReturnCodes from deepsecrets.config import DEFAULT_DEEP_MAX_SIZE from deepsecrets.core.engines.hashed_secret import HashedSecretEngine from deepsecrets.core.engines.regex import RegexEngine from deepsecrets.core.rulesets.hashed_secrets import HashedSecretsRulesetBuilder +from deepsecrets.core.utils.exceptions import FileNotFoundException from deepsecrets.core.utils.multiprocessing_setup import default_start_method @@ -163,3 +164,46 @@ def test_deep_max_size_flag(extra, expected): config = tool.get_current_config() assert config.deep_max_size == expected + + +def test_the_package_and_the_scanner_carry_the_same_version(): + import tomllib + + from deepsecrets.config import SCANNER_VERSION + + with open('/app/pyproject.toml', 'rb') as f: + assert tomllib.load(f)['project']['version'] == SCANNER_VERSION + + +def test_json_output_is_refused_before_the_scan(tmp_path): + report = tmp_path / 'report.json' + args = ['', '--target-dir', '/app/tests/fixtures/', '--outfile', str(report), '--outformat', 'json'] + assert DeepSecretsCliTool(args=args).start() == ReturnCodes.ERROR + assert not report.exists() + + +def test_hashed_values_without_a_value_disable_the_check(): + # an empty ruleset would still keep the lexer on files over --deep-max-size, for nothing + tool = DeepSecretsCliTool( + args=['', '--target-dir', '/app/tests/fixtures/', '--outfile', '/tmp/x.sarif', '--hashed-values'] + ) + tool.parse_arguments() + assert HashedSecretEngine not in tool.get_current_config().engines + + +def test_skip_bundles_applies_to_own_exclusion_files_too(): + own = '/app/tests/fixtures/false_findings.json' + args = ['', '--target-dir', '/app/tests/fixtures/', '--outfile', '/tmp/x.sarif', '--excluded-paths', own, own] + tool = DeepSecretsCliTool(args=args + ['--skip-bundles']) + tool.parse_arguments() + # de-duplicated in order: the first file's patterns are matched first on every run + names = [path.rsplit('/', 1)[-1] for path in tool.get_current_config().global_exclusion_paths] + assert names == ['false_findings.json', 'excluded_bundles.json'] + + +def test_a_missing_report_directory_is_refused_before_the_scan(tmp_path): + tool = DeepSecretsCliTool( + args=['', '--target-dir', '/app/tests/fixtures/', '--outfile', str(tmp_path / 'no' / 'r.sarif')] + ) + with pytest.raises(FileNotFoundException, match='does not exist'): + tool.parse_arguments()