v2.1.1

README.md

@@ -1,4 +1,5 @@
-# DeepSecrets 2.0 - a better tool for secrets scanning
+# [OWASP](https://owasp.org/projects/deepsecrets) DeepSecrets — a better tool for secrets scanning
+
 
 ![Tests Status](https://github.com/ntoskernel/deepsecrets/actions/workflows/run-tests.yml/badge.svg)
 

deepsecrets/config.py

@@ -11,7 +11,7 @@
 FALLBACK_PROCESS_COUNT = 4
 
 SCANNER_NAME = "DeepSecrets"
-SCANNER_VERSION = "2.1.0"
+SCANNER_VERSION = "2.1.1"
 SCANNER_VERSION_NUMERIC = [int(subver) for subver in SCANNER_VERSION.split('.')]
 SCANNER_URL = "https://github.com/ntoskernel/deepsecrets"
 

deepsecrets/core/modes/iscan_mode.py

@@ -12,6 +12,7 @@
 
 from multiprocessing import Manager, get_context
 from multiprocessing.managers import DictProxy
+import errno
 import os
 import pickle
 import tempfile
@@ -387,7 +388,21 @@ def _get_files_list(self) -> List[str]:
                         self._skip(full_path, f'excluded_path:{exclusion}')
                         continue
 
-                    if not self._size_check(full_path):
+                    if os.path.islink(full_path) and not os.path.exists(full_path):
+                        # a dangling symlink or a symlink loop -- os.path.exists is False for both. Skipping here
+                        # keeps it away from a worker that could not open it, and away from _size_check below,
+                        # where os.path.getsize raises once --max-file-size is set.
+                        self._skip(full_path, 'broken_symlink')
+                        continue
+
+                    try:
+                        size_ok = self._size_check(full_path)
+                    except OSError as e:
+                        # vanished or became unreadable between os.walk and here
+                        self._skip(full_path, f'unreadable:{errno.errorcode.get(e.errno, e.errno)}')
+                        continue
+
+                    if not size_ok:
                         self._skip(full_path, 'max_file_size')
                         '''
                         console.print(
@@ -427,6 +442,8 @@ def _matching_exclusion(self, path: str) -> Optional[str]:
         return None
 
     def _size_check(self, path: str):
+        """True when `path` is within `--max-file-size`. Raises `OSError` when the path cannot be stat'ed:
+        the caller records that under its own skip reason, distinct from a genuinely oversized file."""
         if self.config.max_file_size == 0:
             return True
 

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "deepsecrets"
-version = "2.1.0"
+version = "2.1.1"
 description = "A better tool for secrets scanning"
 license = { text = "MIT" }
 readme = "README.md"
@@ -42,7 +42,7 @@ dependencies = [
 ]
 
 [project.urls]
-homepage = "https://owasp.org/www-project-deepsecrets/"
+homepage = "https://owasp.org/projects/deepsecrets"
 repository = "https://github.com/ntoskernel/deepsecrets"
 "Bug Tracker" = "https://github.com/ntoskernel/deepsecrets/issues"
 
@@ -83,7 +83,6 @@ plugins = [
 pythonpath = [
   "."
 ]
-# the benchmark harness in deepsecrets_bench/ has its own tests and dependencies
 testpaths = ["tests"]
 
 [tool.black]

tests/scan_modes/test_scan_mode_regressions.py

@@ -1,3 +1,4 @@
+import errno
 import os
 import pickle
 import shutil
@@ -212,3 +213,69 @@ def test_hashed_scan_through_process_pool(tmp_path: Path):
         assert mode.stats.failed_files == 0
     finally:
         mode.dispose()
+
+
+def test_broken_symlink_does_not_abort_discovery(tmp_path: Path):
+    # a dangling symlink, and a symlink loop, used to raise out of _size_check and kill the whole
+    # scan before it started. --max-file-size must be set: at the default 0 the check short-circuits
+    # before os.path.getsize and this passes even against the unfixed code.
+    (tmp_path / 'real.py').write_text('password = "Xk9mQ2vL7pR4tZw8Jq"\n')
+    (tmp_path / 'dangling.py').symlink_to(tmp_path / 'gone.py')
+    (tmp_path / 'loop_a').symlink_to(tmp_path / 'loop_b')
+    (tmp_path / 'loop_b').symlink_to(tmp_path / 'loop_a')
+
+    config = _config(str(tmp_path))
+    config.set_max_file_size(10_000_000)
+    mode = CliScanMode(config=config)
+    try:
+        assert mode.filepaths == [str(tmp_path / 'real.py')]
+        assert mode.stats.skipped_files == 3
+        for name in ('dangling.py', 'loop_a', 'loop_b'):
+            assert mode.files[str(tmp_path / name)].status == 'skipped'
+            assert mode.files[str(tmp_path / name)].skip_reason == 'broken_symlink'
+    finally:
+        mode.dispose()
+
+
+def test_broken_symlink_skipped_without_a_size_limit(tmp_path: Path):
+    # at the default --max-file-size of 0 a dangling symlink never crashed, it reached a worker that
+    # could not open it and was reported as a per-file error. It is a skip with a reason now.
+    (tmp_path / 'real.py').write_text('password = "Xk9mQ2vL7pR4tZw8Jq"\n')
+    (tmp_path / 'dangling.py').symlink_to(tmp_path / 'gone.py')
+
+    config = _config(str(tmp_path))
+    assert config.max_file_size == 0
+    mode = CliScanMode(config=config)
+    try:
+        assert mode.filepaths == [str(tmp_path / 'real.py')]
+        assert mode.files[str(tmp_path / 'dangling.py')].skip_reason == 'broken_symlink'
+        # skipped files carry no error, so they stay out of the per-file error report
+        assert mode.per_file_errors() == {}
+    finally:
+        mode.dispose()
+
+
+def test_unreadable_file_is_not_filed_as_oversized(tmp_path: Path, monkeypatch):
+    # _size_check used to swallow OSError and return False, which the caller could only read as
+    # 'too big' -- so an unreadable file was reported as max_file_size in --report-diagnostics SARIF
+    (tmp_path / 'fine.txt').write_text('x' * 10)
+    victim = tmp_path / 'victim.txt'
+    victim.write_text('y' * 10)
+
+    real_getsize = os.path.getsize
+
+    def fake_getsize(path):
+        if str(path) == str(victim):
+            raise PermissionError(errno.EACCES, 'Permission denied', str(path))
+        return real_getsize(path)
+
+    monkeypatch.setattr(os.path, 'getsize', fake_getsize)
+
+    config = _config(str(tmp_path))
+    config.set_max_file_size(10_000_000)
+    mode = CliScanMode(config=config)
+    try:
+        assert mode.filepaths == [str(tmp_path / 'fine.txt')]
+        assert mode.files[str(victim)].skip_reason == 'unreadable:EACCES'
+    finally:
+        mode.dispose()