Release 2.2

.gitignore

@@ -60,3 +60,8 @@ pytest*
 /benchmarker/
 /tests/fixtures/problem_files/
 /*.json
+
+/docs/private/
+AGENTS.md
+CLAUDE.md
+research/

.vscode/launch.json

@@ -110,7 +110,8 @@
                 "--outfile",
                 "./fdsafad.json",
                 "--outformat",
-                "json",
+                "sarif",
+                "--multiprocessing-context", "spawn",
                 "--process-count", "1",
                 "--max-file-size", "0",
                 "--disable-masking"

README.md

@@ -45,6 +45,8 @@ From PyPi
 $ pip install deepsecrets
 ```
 
+DeepSecrets needs Python 3.11 or later.
+
 
 ## Scanning
 The easiest way to run a scan:
@@ -64,7 +66,14 @@ The `--help` command is always ready to guide you, but here are the key flags yo
 * `--regex-rules /path/to/rules.json`: Supply your own custom regex ruleset.
 * `--hashed-values /path/to/hashes.json`: Provide a list of pre-hashed known production secrets to search for them securely.
 * `--excluded-paths /path/to/exclusions.json`: Override or extend the default paths ignored during scanning.
+* `--deep-max-size BYTES`: Files larger than this (default 250,000 bytes) get the fast analysis: the regex rules and the variable search, without the language lexer. Big files are mostly generated or minified, the lexer costs most of the scan time on them and rarely finds more there. `0` gives every file the full analysis.
+* `--skip-bundles`: Skip minified JavaScript, source maps and bundles (`*.min.js`, `*.map`, `*.bundle.js`) entirely, on top of the other path exclusions.
+* `--confidence-level {all,low,medium,high,very-high}`: The lowest confidence to report (default `low`, every finding). `medium`, `high` and `very-high` keep confidence 3, 6 and 9 or more; `all` also reports the regex matches the scanner judged not to be secrets (placeholders, examples), with an `-INFO` rule id.
+* `--report-diagnostics`: Add every file found under the target to the SARIF report, with its status, scan time, skip reason and analysis depth.
+* `--regex-candidate-scoring-rules /path/to/rules.json`: Supply your own rules for judging regex matches before they are reported. Give the flag no value to report regex matches unjudged.
+* `--multiprocessing-context {forkserver,spawn,fork}`: How worker processes start. The default is `forkserver` where the platform has it (Linux, macOS): workers share the scanner already loaded in a server process. Windows uses `spawn`. All three give the same findings.
 * `--disable-masking`: Keep potential secrets unmasked in the output report *(see caution below)*.
+* `--ci` / `--no-ci`: Turn CI mode on or off. CI mode prints a plain progress line every 30 seconds instead of the live progress display. It turns on by itself inside CI services (GitHub Actions, GitLab CI, Jenkins and others) and whenever the output is not a terminal.
 
 
 ### Github Actions Integration
@@ -113,7 +122,7 @@ Masking doesn't break the deduplication logic of downstream platforms (like Gith
 
 Every finding gets a confidence score. However, different security platforms parse SARIF metrics differently. To ensure compatibility across modern ASPM dashboards, DeepSecrets does the following:
 
-* **Virtual Subrules (`rules[]`)**: Dynamically generates rules like `S105-LOW` or `S105-CRITICAL`. This forces GitHub Security and DefectDojo to map semantic precision variance properly without breaking native parsers.
+* **Virtual Subrules (`rules[]`)**: Dynamically generates rules by confidence tier, from `S105-LOW` through `-MEDIUM` and `-HIGH` to `S105-VERY-HIGH` (and `-INFO` for the rejected matches `--confidence-level all` reports). This forces GitHub Security and DefectDojo to map semantic precision variance properly without breaking native parsers.
 
 * **Deterministic Result Level**: The tool always explicitly sets `level: error` in the `results[]` model. This acts as a universal fallback for CI/CD pipelines and older SAST parsers, ensuring that exposed secrets reliably break builds or block Pull Requests regardless of individual rule interpretations.
 

deepsecrets/config.py

@@ -7,12 +7,28 @@
 
 from deepsecrets.core.utils.exceptions import FileNotFoundException
 from deepsecrets.core.utils.fs import get_abspath, path_exists
+from deepsecrets.core.utils.multiprocessing_setup import default_start_method
 
 FALLBACK_PROCESS_COUNT = 4
 
+# --confidence-level: the lowest confidence a reported finding may have. 'all' also reports the regex candidates their
+# evaluation rejected (confidence 0); 'low', the default, reports every finding the scanner keeps
+CONFIDENCE_LEVELS = {'all': None, 'low': 0, 'medium': 3, 'high': 6, 'very-high': 9}
+DEFAULT_CONFIDENCE_LEVEL = 'low'
+
+# files larger than this many bytes get the shallow analysis: the regex rules and the cheap variable search, no lexer.
+# On SecretBench the lexer found 12 of its 21 unique secrets in smaller files and the other 9 only in files of 1 MB
+# or more (minified JavaScript, one YAML file), while files over 250 KB took two thirds of its time.
+DEFAULT_DEEP_MAX_SIZE = 250_000
+
+
+def deep_analysis(size: int, deep_max_size: int) -> bool:
+    """Whether a file of `size` bytes gets the full analysis, lexer included. `deep_max_size` 0 or less: every file."""
+    return deep_max_size <= 0 or size <= deep_max_size
+
+
 SCANNER_NAME = "DeepSecrets"
-SCANNER_VERSION = "2.1.1"
-SCANNER_VERSION_NUMERIC = [int(subver) for subver in SCANNER_VERSION.split('.')]
+SCANNER_VERSION = "2.2.0"
 SCANNER_URL = "https://github.com/ntoskernel/deepsecrets"
 
 MAX_LINE_LENGTH_FOR_CONTEXT = 300
@@ -28,7 +44,9 @@ class Config:
     workdir_path: str
     oneshot_path: str
     max_file_size: int = 0  # 0 means no limit
-    mp_context: str = 'spawn'
+    # files above this many bytes get no lexer (see deep_analysis above); 0 means every file gets it
+    deep_max_size: int = DEFAULT_DEEP_MAX_SIZE
+    mp_context: str = default_start_method()
     engines: List[Type] = []
     rulesets: Dict[Type, List[str]] = {}
     global_exclusion_paths: List[str] = []
@@ -37,7 +55,10 @@ class Config:
     return_code_if_findings: bool
     disable_masking: bool
     report_diagnostics: bool = False
+    confidence_level: str = DEFAULT_CONFIDENCE_LEVEL
     verbose: bool = False
+    # no live terminal UI and no progress manager; the CLI turns it on in CI and when output is not a terminal
+    ci_mode: bool = False
 
     _benchmarking_mode: bool
 
@@ -48,6 +69,7 @@ def __init__(self) -> None:
         self.return_code_if_findings = False
         self.disable_masking = False
         self.report_diagnostics = False
+        self.confidence_level = DEFAULT_CONFIDENCE_LEVEL
 
         self._benchmarking_mode = False
         self.oneshot_path = None
@@ -71,6 +93,11 @@ def set_disable_masking(self, state: bool):
     def set_report_diagnostics(self, state: bool):
         self.report_diagnostics = state
 
+    def set_confidence_level(self, level: str) -> None:
+        if level not in CONFIDENCE_LEVELS:
+            raise ValueError(f'unknown confidence level {level}')
+        self.confidence_level = level
+
     def _set_path(self, path: str, field: str) -> None:
         if not path_exists(path):
             raise FileNotFoundException(f'{field} path does not exist ({path})')
@@ -87,9 +114,15 @@ def set_workdir(self, path: str) -> None:
     def set_max_file_size(self, size: int) -> None:
         self.max_file_size = size
 
+    def set_deep_max_size(self, size: int) -> None:
+        self.deep_max_size = size
+
     def set_mp_context(self, context: str) -> None:
         self.mp_context = context
 
+    def set_ci_mode(self, enabled: bool) -> None:
+        self.ci_mode = enabled
+
     def set_process_count(self, count: int) -> None:
         if count > 0:
             self.process_count = count
@@ -117,7 +150,9 @@ def set_global_exclusion_paths(self, paths: List[str]) -> None:
                 raise FileNotFoundException(f'global_exclusion_path does not exist ({path})')
             self.global_exclusion_paths.append(path)
 
-        self.global_exclusion_paths = list(set(self.global_exclusion_paths))
+        # de-duplicated in order: the first file's patterns are matched first, so a skip reason names the same
+        # pattern on every run (a set's order changed with the hash seed)
+        self.global_exclusion_paths = list(dict.fromkeys(self.global_exclusion_paths))
 
     def add_ruleset(self, type: Type, paths: List[str] = []) -> None:
         self._validate_paths(paths)
@@ -133,6 +168,3 @@ def _validate_paths(self, paths: List[str]) -> None:
             raise FileNotFoundException(f'File {path} does not exist')
 
         return
-
-
-config = Config()

deepsecrets/core/engines/regex.py

@@ -1,39 +1,88 @@
-from typing import List, Tuple
+from typing import List, Optional
 
 from deepsecrets.core.engines.iengine import IEngine
 from deepsecrets.core.model.finding import Finding
-from deepsecrets.core.model.rules.regex import RegexRule
+from deepsecrets.core.model.rules.regex import RegexRule, RuleMatch
 from deepsecrets.core.model.token import Token
 
 
 class RegexEngine(IEngine):
     name = 'regex'
     description = 'Scans by regex patterns provided by RegexRules'
 
-    def search(self, token: Token) -> List[Finding]:
+    def __init__(
+        self, ruleset: Optional[List] = None, candidate_rules: Optional[List] = None, report_rejected: bool = False
+    ) -> None:
+        super().__init__(ruleset=ruleset if ruleset is not None else [])
+        # a regex match is a candidate, judged here as it is found, as the semantic engine judges a variable
+        self.evaluator = None
+        if candidate_rules:
+            # imported here: the evaluator reads the naturalness model, which a regex-only engine does not need
+            from deepsecrets.core.helpers.regex_candidate_evaluator import RegexCandidateEvaluator
+            from deepsecrets.core.model.regex_candidate import RegexCandidateContext
+
+            self.evaluator = RegexCandidateEvaluator(candidate_rules)
+            self.candidate_context = RegexCandidateContext
+        # rejected candidates are reported (confidence 0, flagged) only at --confidence-level all
+        self.report_rejected = report_rejected
+        # the diagnostics tracer sets a list here to record every rejection, in file offsets, without emitting them
+        self.rejected_log: Optional[list] = None
+
+    def search(self, token: Token, corroborated: bool = False) -> List[Finding]:
+        """`corroborated`: the semantic engine reported this token's variable, so a match covering the whole token is
+        already judged and keeps its rule's confidence."""
         results = []
 
         for rule in self.ruleset:
             if not self.is_rule_applicable(token=token, rule=rule):
                 continue
 
-            results.extend(self._check_rule(token, rule))  # type: ignore
+            results.extend(self._check_rule(token, rule, corroborated))  # type: ignore
         return results
 
-    def _check_rule(self, token: Token, rule: RegexRule) -> List[Finding]:
+    def _check_rule(self, token: Token, rule: RegexRule, corroborated: bool = False) -> List[Finding]:
         findings: List[Finding] = []
 
-        # rule.match returns an array of (start, end) spans
-        detects: List[Tuple[int, int]] = rule.match(token)
-
-        for start, end in detects:
-            findings.append(
-                Finding(
-                    rules=[rule],
-                    detection=token.content[start:end],
-                    start_offset=start,
-                    end_offset=end,
-                )
+        for match in rule.matches(token):
+            if match.start == match.end:
+                continue  # an empty group (password= with nothing after it): nothing to report
+            finding = Finding(
+                rules=[rule],
+                detection=token.content[match.start : match.end],
+                start_offset=match.start,
+                end_offset=match.end,
             )
+            if self._judge(finding, token, rule, match, corroborated):
+                findings.append(finding)
 
         return findings
+
+    def _judge(self, finding: Finding, token: Token, rule: RegexRule, match: RuleMatch, corroborated: bool) -> bool:
+        """Whether to emit the finding; sets its confidence, or marks it rejected."""
+        if self.evaluator is None or rule.evidence is None:
+            return True
+        if corroborated and (match.start, match.end) == (0, len(token.content)):
+            return True
+
+        context = self.candidate_context.from_match(rule, match.match, finding.detection, token.file.path)
+        verdict = self.evaluator.evaluate(context)
+        if verdict.rejected:
+            if self.rejected_log is not None:
+                offset = token.span[0] if token.span else 0
+                self.rejected_log.append(
+                    {
+                        'span': (offset + match.start, offset + match.end),
+                        'rule': rule.id,
+                        'evidence': rule.evidence.value,
+                    }
+                    | verdict.summary()
+                )
+            if not self.report_rejected:
+                return False
+            finding.rejected = True
+            finding.rules = [rule.model_copy(update={'confidence': 0, 'is_dynamic_confidence': True})]
+            finding.internal_score = {'candidate': verdict.summary()}
+            return True
+        if verdict.confidence is not None:
+            finding.rules = [rule.model_copy(update={'confidence': verdict.confidence, 'is_dynamic_confidence': True})]
+        return True

deepsecrets/core/engines/semantic.py

@@ -41,11 +41,22 @@ def search(self, token: Token) -> List[Finding]:
             if fname in token.file.path:
                 return findings
 
+        # the variable first: a regex match that covers the whole token of a variable reported here is corroborated,
+        # so the regex sub-engine does not judge it again
+        semantic_findings = self._evaluate_semantics(token)
+
         if self.subengine is not None:  # pragma: nocover
-            content_findings = ContentAnalyzer(self.subengine).analyze(token)
+            content_findings = ContentAnalyzer(self.subengine).analyze(token, corroborated=bool(semantic_findings))
             if content_findings is not None:
                 findings.extend(content_findings)
 
+        findings.extend(semantic_findings)
+        return findings
+
+    def _evaluate_semantics(self, token: Token) -> List[Finding]:
+        """S107 for a dangerous condition, S105 or S106 for a variable the evaluator scores as dangerous."""
+        findings: List[Finding] = []
+
         if token.semantic is None:
             return findings
 
@@ -91,7 +102,8 @@ def search(self, token: Token) -> List[Finding]:
                             internal_score={'var': token.semantic.name} | evaluation_result.summary(),
                         )
                     )
-                else:
+                elif evaluation_result.allows_low_entropy:
+                    # 3 bits or less: a password someone chose, or else a placeholder, an id or a map key
                     findings.append(
                         Finding(
                             detection=token.content,

deepsecrets/core/helpers/confidence.py

@@ -0,0 +1,18 @@
+"""The parts of a confidence both evaluators share: the entropy scale and what a random-looking value adds."""
+
+
+def entropy_score(bits: float) -> float:
+    """-1 for a single repeated character, 0 under 3 bits per character, 0 to 35 from 3 to 4 bits, 40 at 4 or more."""
+    if bits == 0:
+        return -1
+    if bits < 3:
+        return 0
+    if bits < 4:
+        return (bits - 3) * 35
+    return 40
+
+
+def randomness_points(score: float, nonsense: float) -> float:
+    """Up to 5 confidence points for a random-looking value: its entropy score (entropy_score, 0..40) scaled to 0..5,
+    halved for a value that reads as language (nonsense = 1 - naturalness)."""
+    return min(max(score, 0), 40) / 40 * 5 * min(nonsense + 0.5, 1)

deepsecrets/core/helpers/content_analyzer.py

@@ -16,10 +16,10 @@ def __init__(self, engine: IEngine) -> None:
         self.engine = engine
         self.uncover_tactics = [self._check_by_base64]
 
-    def analyze(self, token: Token) -> List[Finding]:
+    def analyze(self, token: Token, **search_options) -> List[Finding]:
         self.token = token
         self.uncover()
-        return self.engine.search(self.token) if self.engine is not None else []
+        return self.engine.search(self.token, **search_options) if self.engine is not None else []
 
     def uncover(self) -> None:
         for tactic in self.uncover_tactics: