[detectors] Add Koala-36M Based Detector

benchmark/_common.py

@@ -29,6 +29,7 @@
     ContentDetector,
     HashDetector,
     HistogramDetector,
+    KoalaDetector,
     ThresholdDetector,
 )
 
@@ -39,6 +40,8 @@
     "detect-content": ContentDetector,
     "detect-hash": HashDetector,
     "detect-hist": HistogramDetector,
+    # TODO: KoalaDetector should be benchmarked without auto-downscaling (see its process_frame).
+    "detect-koala": KoalaDetector,
     "detect-threshold": ThresholdDetector,
 }
 

packaging/windows/requirements.txt

@@ -8,6 +8,7 @@ moviepy==2.2.1
 opencv-python-headless==5.0.0.93
 numpy==2.5.1
 platformdirs==4.11.0
+scikit-image==0.26.0
 tqdm==4.69.0
 
 # Build-only and test-only requirements.

pyproject.toml

@@ -66,6 +66,7 @@ dev = [
     "platformdirs",
     "pytest>=7.0",
     "pytest-rerunfailures",
+    "scikit-image",
     "tqdm",
 ]
 docs = ["Sphinx==7.0.1", "sphinx-copybutton==0.5.2"]

scenedetect/__init__.py

@@ -64,6 +64,7 @@
     ThresholdDetector as ThresholdDetector,
     HistogramDetector as HistogramDetector,
     HashDetector as HashDetector,
+    KoalaDetector as KoalaDetector,
 )
 from scenedetect.backends import (
     AVAILABLE_BACKENDS as AVAILABLE_BACKENDS,

scenedetect/_cli/__init__.py

@@ -43,6 +43,7 @@
     ContentDetector,
     HashDetector,
     HistogramDetector,
+    KoalaDetector,
     ThresholdDetector,
 )
 from scenedetect.platform import get_cv2_imwrite_params, get_system_version_info
@@ -943,6 +944,26 @@ def detect_hash_command(
     ctx.add_detector(HashDetector, detector_args)
 
 
+DETECT_KOALA_HELP = """Perform cut detection using the method described by Koala-36M (https://koala36m.github.io/). Requires the optional `scikit-image` package.
+
+Adjacent frames are scored using per-channel histogram correlation combined with the structural similarity of their edge maps. Cuts are found using an adaptive threshold calculated over a window of preceding scores, so all cuts are emitted once the whole video has been processed.
+
+This detector is experimental: its parameters are not yet configurable, and min-scene-len is not applied.
+
+Examples:
+
+    {scenedetect_with_video} detect-koala
+"""
+
+
+@click.command("detect-koala", cls=Command, help=DETECT_KOALA_HELP)
+@click.pass_context
+def detect_koala_command(ctx: click.Context):
+    ctx = ctx.obj
+    assert isinstance(ctx, CliContext)
+    ctx.add_detector(KoalaDetector, {})
+
+
 LOAD_SCENES_HELP = """Load scenes from CSV instead of detecting. Can be used with CSV generated by `list-scenes`. Scenes are loaded using the specified column as cut locations (frame number or timecode).
 
 Examples:
@@ -1875,6 +1896,7 @@ def save_otio_command(
 scenedetect.add_command(detect_content_command)
 scenedetect.add_command(detect_hash_command)
 scenedetect.add_command(detect_hist_command)
+scenedetect.add_command(detect_koala_command)
 scenedetect.add_command(detect_threshold_command)
 
 # Output

scenedetect/detectors/__init__.py

@@ -40,6 +40,7 @@
 from scenedetect.detectors.adaptive_detector import AdaptiveDetector as AdaptiveDetector
 from scenedetect.detectors.hash_detector import HashDetector as HashDetector
 from scenedetect.detectors.histogram_detector import HistogramDetector as HistogramDetector
+from scenedetect.detectors.koala_detector import KoalaDetector as KoalaDetector
 
 # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
 #                                                                             #

scenedetect/detectors/koala_detector.py

@@ -0,0 +1,112 @@
+#
+#            PySceneDetect: Python-Based Video Scene Detector
+#   -------------------------------------------------------------------
+#     [  Site:    https://scenedetect.com                           ]
+#     [  Docs:    https://scenedetect.com/docs/                     ]
+#     [  Github:  https://github.com/Breakthrough/PySceneDetect/    ]
+#
+# Copyright (C) 2024 Brandon Castellano <http://www.bcastell.com>.
+# PySceneDetect is licensed under the BSD 3-Clause License; see the
+# included LICENSE file, or visit one of the above pages for details.
+#
+""":class:`KoalaDetector` uses the detection method described by Koala-36M.
+See https://koala36m.github.io/ for details.
+
+TODO: Cite correctly.
+
+This detector requires the optional `scikit-image` package to be installed.
+
+This detector is available from the command-line as the `detect-koala` command.
+"""
+
+import cv2
+import numpy as np
+
+from scenedetect.common import FrameTimecode, TimecodeLike
+from scenedetect.detector import SceneDetector
+
+try:
+    from skimage.metrics import structural_similarity
+except ImportError:  # pragma: no cover
+    structural_similarity = None
+
+
+class KoalaDetector(SceneDetector):
+    """Detects cuts by scoring adjacent frames using per-channel histogram correlation combined
+    with the structural similarity of their edge maps, similar to the method described by
+    Koala-36M. Cuts are found using an adaptive threshold calculated over a window of preceding
+    scores, so all cuts are emitted from :meth:`post_process` once the whole video has been
+    processed."""
+
+    def __init__(self, min_scene_len: TimecodeLike = 15):
+        """
+        Arguments:
+            min_scene_len: Accepted for consistency with other detectors, but not currently used
+                by this detector (TODO).
+        """
+        if structural_similarity is None:
+            raise ImportError(
+                "KoalaDetector requires the `scikit-image` package (pip install scikit-image)."
+            )
+        super().__init__()
+        self._start_timecode: FrameTimecode | None = None
+        self._min_scene_len: TimecodeLike = min_scene_len
+        self._last_histogram: np.ndarray | None = None
+        self._last_edges: np.ndarray | None = None
+        self._scores: list[float] = []
+
+        # Tunables (TODO: Make these config params):
+
+        # Boxcar filter size (should be <= window size)
+        self._filter_size: int = 3
+        # Window to use for calculating threshold (should be >= filter size).
+        self._window_size: int = 8
+        # Multiplier for standard deviations when calculating threshold.
+        self._deviation: float = 3.0
+
+    def process_frame(self, timecode: FrameTimecode, frame_img: np.ndarray) -> list[FrameTimecode]:
+        assert structural_similarity is not None
+        # TODO: frame_img is already downscaled here. The same problem exists in HashDetector.
+        # For now we can just set downscale factor to 1 in SceneManager to work around the issue.
+        frame_img = cv2.resize(frame_img, (256, 256))
+        histogram = np.asarray(
+            [cv2.calcHist([c], [0], None, [254], [1, 255]) for c in cv2.split(frame_img)]
+        )
+        # TODO: Make the parameters below tunable.
+        frame_gray = cv2.resize(cv2.cvtColor(frame_img, cv2.COLOR_BGR2GRAY), (128, 128))
+        edges = np.maximum(frame_gray, cv2.Canny(frame_gray, 100, 200))
+        if self._start_timecode is None:
+            self._start_timecode = timecode
+        else:
+            delta_histogram = cv2.compareHist(self._last_histogram, histogram, cv2.HISTCMP_CORREL)
+            delta_edges = structural_similarity(self._last_edges, edges, data_range=255)
+            score = 4.61480465 * delta_histogram + 3.75211168 * delta_edges - 5.485968377115124
+            self._scores.append(score)
+        self._last_histogram = histogram
+        self._last_edges = edges
+        return []
+
+    def post_process(self, timecode: FrameTimecode) -> list[FrameTimecode]:
+        if self._start_timecode is None or not self._scores:
+            return []
+        cut_found = [score < 0.0 for score in self._scores]
+        cut_found.append(True)
+        boxcar = [1] * self._filter_size
+        cutoff = float(self._filter_size) / float(self._filter_size + 1)
+        filtered = np.convolve(self._scores, boxcar, mode="same")
+        for i in range(len(self._scores)):
+            if i >= self._window_size and filtered[i] < cutoff:
+                # TODO: Should we discard the N most extreme values before calculating threshold?
+                window = filtered[i - self._window_size : i]
+                threshold = window.mean() - (self._deviation * window.std())
+                if filtered[i] < threshold:
+                    cut_found[i] = True
+
+        cuts = []
+        last_cut = 0
+        for i in range(len(cut_found)):
+            if cut_found[i]:
+                if (i - last_cut) > self._window_size:
+                    cuts.append(last_cut)
+                last_cut = i + 1
+        return [self._start_timecode + cut for cut in cuts][1:]

tests/test_detectors.py

@@ -16,6 +16,7 @@
 test case material.
 """
 
+import importlib.util
 import os
 from dataclasses import dataclass
 
@@ -29,16 +30,21 @@
     ContentDetector,
     HashDetector,
     HistogramDetector,
+    KoalaDetector,
     ThresholdDetector,
 )
 
 # Untyped so each entry retains its concrete `type[...]` for parameterized construction
 # (calls below pass detector-specific kwargs like `min_scene_len`).
+# KoalaDetector requires the optional `scikit-image` package; it is only tested when installed.
+_HAVE_SKIMAGE = importlib.util.find_spec("skimage") is not None
+
 FAST_CUT_DETECTORS = (
     AdaptiveDetector,
     ContentDetector,
     HashDetector,
     HistogramDetector,
+    *((KoalaDetector,) if _HAVE_SKIMAGE else ()),
 )
 
 ALL_DETECTORS = (*FAST_CUT_DETECTORS, ThresholdDetector)
@@ -133,7 +139,9 @@ def get_fast_cut_test_cases():
             ),
             id=f"{detector_type.__name__}/m=30",
         )
+        # TODO: Make this work, right now min_scene_len isn't used by the detector.
         for detector_type in FAST_CUT_DETECTORS
+        if detector_type != KoalaDetector
     ]
     return test_cases
 
@@ -246,6 +254,9 @@ def test_detectors_with_stats(test_video_file):
 )
 def test_min_scene_len_accepts_time_values(detector_type, min_scene_len):
     """Detectors accept min_scene_len as int (frames), float (seconds), or str (timecode)."""
+    # TODO: Make this work, right now min_scene_len isn't used by the detector.
+    if detector_type is KoalaDetector:
+        pytest.skip("KoalaDetector does not apply min_scene_len yet.")
     test_case = TestCase(
         path=get_absolute_path("resources/goldeneye.mp4"),
         detector=detector_type(min_scene_len=min_scene_len),