From 9e343b9fd5fff84f847cfb1116cd7c8fbafc796a Mon Sep 17 00:00:00 2001 From: Breakthrough Date: Mon, 18 Nov 2024 22:04:17 -0500 Subject: [PATCH] [detectors] Implement Koala-36M Implement algorithm similar to that described in Koala-36M. Add `KoalaDetector` and `detect-koala` command. #441 --- benchmark/_common.py | 3 + packaging/windows/requirements.txt | 1 + pyproject.toml | 1 + scenedetect/__init__.py | 1 + scenedetect/_cli/__init__.py | 22 +++++ scenedetect/detectors/__init__.py | 1 + scenedetect/detectors/koala_detector.py | 112 ++++++++++++++++++++++++ tests/test_detectors.py | 11 +++ 8 files changed, 152 insertions(+) create mode 100644 scenedetect/detectors/koala_detector.py diff --git a/benchmark/_common.py b/benchmark/_common.py index c175dcc0..386a93e8 100644 --- a/benchmark/_common.py +++ b/benchmark/_common.py @@ -29,6 +29,7 @@ ContentDetector, HashDetector, HistogramDetector, + KoalaDetector, ThresholdDetector, ) @@ -39,6 +40,8 @@ "detect-content": ContentDetector, "detect-hash": HashDetector, "detect-hist": HistogramDetector, + # TODO: KoalaDetector should be benchmarked without auto-downscaling (see its process_frame). + "detect-koala": KoalaDetector, "detect-threshold": ThresholdDetector, } diff --git a/packaging/windows/requirements.txt b/packaging/windows/requirements.txt index dc31fa21..161ca886 100644 --- a/packaging/windows/requirements.txt +++ b/packaging/windows/requirements.txt @@ -8,6 +8,7 @@ moviepy==2.2.1 opencv-python-headless==5.0.0.93 numpy==2.5.1 platformdirs==4.11.0 +scikit-image==0.26.0 tqdm==4.69.0 # Build-only and test-only requirements. diff --git a/pyproject.toml b/pyproject.toml index 88f2f034..4acbb6cb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -66,6 +66,7 @@ dev = [ "platformdirs", "pytest>=7.0", "pytest-rerunfailures", + "scikit-image", "tqdm", ] docs = ["Sphinx==7.0.1", "sphinx-copybutton==0.5.2"] diff --git a/scenedetect/__init__.py b/scenedetect/__init__.py index f8b9ee73..0c7e0998 100644 --- a/scenedetect/__init__.py +++ b/scenedetect/__init__.py @@ -64,6 +64,7 @@ ThresholdDetector as ThresholdDetector, HistogramDetector as HistogramDetector, HashDetector as HashDetector, + KoalaDetector as KoalaDetector, ) from scenedetect.backends import ( AVAILABLE_BACKENDS as AVAILABLE_BACKENDS, diff --git a/scenedetect/_cli/__init__.py b/scenedetect/_cli/__init__.py index f75929be..646ee3cf 100644 --- a/scenedetect/_cli/__init__.py +++ b/scenedetect/_cli/__init__.py @@ -43,6 +43,7 @@ ContentDetector, HashDetector, HistogramDetector, + KoalaDetector, ThresholdDetector, ) from scenedetect.platform import get_cv2_imwrite_params, get_system_version_info @@ -943,6 +944,26 @@ def detect_hash_command( ctx.add_detector(HashDetector, detector_args) +DETECT_KOALA_HELP = """Perform cut detection using the method described by Koala-36M (https://koala36m.github.io/). Requires the optional `scikit-image` package. + +Adjacent frames are scored using per-channel histogram correlation combined with the structural similarity of their edge maps. Cuts are found using an adaptive threshold calculated over a window of preceding scores, so all cuts are emitted once the whole video has been processed. + +This detector is experimental: its parameters are not yet configurable, and min-scene-len is not applied. + +Examples: + + {scenedetect_with_video} detect-koala +""" + + +@click.command("detect-koala", cls=Command, help=DETECT_KOALA_HELP) +@click.pass_context +def detect_koala_command(ctx: click.Context): + ctx = ctx.obj + assert isinstance(ctx, CliContext) + ctx.add_detector(KoalaDetector, {}) + + LOAD_SCENES_HELP = """Load scenes from CSV instead of detecting. Can be used with CSV generated by `list-scenes`. Scenes are loaded using the specified column as cut locations (frame number or timecode). Examples: @@ -1875,6 +1896,7 @@ def save_otio_command( scenedetect.add_command(detect_content_command) scenedetect.add_command(detect_hash_command) scenedetect.add_command(detect_hist_command) +scenedetect.add_command(detect_koala_command) scenedetect.add_command(detect_threshold_command) # Output diff --git a/scenedetect/detectors/__init__.py b/scenedetect/detectors/__init__.py index 16238025..43cd5f05 100644 --- a/scenedetect/detectors/__init__.py +++ b/scenedetect/detectors/__init__.py @@ -40,6 +40,7 @@ from scenedetect.detectors.adaptive_detector import AdaptiveDetector as AdaptiveDetector from scenedetect.detectors.hash_detector import HashDetector as HashDetector from scenedetect.detectors.histogram_detector import HistogramDetector as HistogramDetector +from scenedetect.detectors.koala_detector import KoalaDetector as KoalaDetector # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # diff --git a/scenedetect/detectors/koala_detector.py b/scenedetect/detectors/koala_detector.py new file mode 100644 index 00000000..fc51a64b --- /dev/null +++ b/scenedetect/detectors/koala_detector.py @@ -0,0 +1,112 @@ +# +# PySceneDetect: Python-Based Video Scene Detector +# ------------------------------------------------------------------- +# [ Site: https://scenedetect.com ] +# [ Docs: https://scenedetect.com/docs/ ] +# [ Github: https://github.com/Breakthrough/PySceneDetect/ ] +# +# Copyright (C) 2024 Brandon Castellano . +# PySceneDetect is licensed under the BSD 3-Clause License; see the +# included LICENSE file, or visit one of the above pages for details. +# +""":class:`KoalaDetector` uses the detection method described by Koala-36M. +See https://koala36m.github.io/ for details. + +TODO: Cite correctly. + +This detector requires the optional `scikit-image` package to be installed. + +This detector is available from the command-line as the `detect-koala` command. +""" + +import cv2 +import numpy as np + +from scenedetect.common import FrameTimecode, TimecodeLike +from scenedetect.detector import SceneDetector + +try: + from skimage.metrics import structural_similarity +except ImportError: # pragma: no cover + structural_similarity = None + + +class KoalaDetector(SceneDetector): + """Detects cuts by scoring adjacent frames using per-channel histogram correlation combined + with the structural similarity of their edge maps, similar to the method described by + Koala-36M. Cuts are found using an adaptive threshold calculated over a window of preceding + scores, so all cuts are emitted from :meth:`post_process` once the whole video has been + processed.""" + + def __init__(self, min_scene_len: TimecodeLike = 15): + """ + Arguments: + min_scene_len: Accepted for consistency with other detectors, but not currently used + by this detector (TODO). + """ + if structural_similarity is None: + raise ImportError( + "KoalaDetector requires the `scikit-image` package (pip install scikit-image)." + ) + super().__init__() + self._start_timecode: FrameTimecode | None = None + self._min_scene_len: TimecodeLike = min_scene_len + self._last_histogram: np.ndarray | None = None + self._last_edges: np.ndarray | None = None + self._scores: list[float] = [] + + # Tunables (TODO: Make these config params): + + # Boxcar filter size (should be <= window size) + self._filter_size: int = 3 + # Window to use for calculating threshold (should be >= filter size). + self._window_size: int = 8 + # Multiplier for standard deviations when calculating threshold. + self._deviation: float = 3.0 + + def process_frame(self, timecode: FrameTimecode, frame_img: np.ndarray) -> list[FrameTimecode]: + assert structural_similarity is not None + # TODO: frame_img is already downscaled here. The same problem exists in HashDetector. + # For now we can just set downscale factor to 1 in SceneManager to work around the issue. + frame_img = cv2.resize(frame_img, (256, 256)) + histogram = np.asarray( + [cv2.calcHist([c], [0], None, [254], [1, 255]) for c in cv2.split(frame_img)] + ) + # TODO: Make the parameters below tunable. + frame_gray = cv2.resize(cv2.cvtColor(frame_img, cv2.COLOR_BGR2GRAY), (128, 128)) + edges = np.maximum(frame_gray, cv2.Canny(frame_gray, 100, 200)) + if self._start_timecode is None: + self._start_timecode = timecode + else: + delta_histogram = cv2.compareHist(self._last_histogram, histogram, cv2.HISTCMP_CORREL) + delta_edges = structural_similarity(self._last_edges, edges, data_range=255) + score = 4.61480465 * delta_histogram + 3.75211168 * delta_edges - 5.485968377115124 + self._scores.append(score) + self._last_histogram = histogram + self._last_edges = edges + return [] + + def post_process(self, timecode: FrameTimecode) -> list[FrameTimecode]: + if self._start_timecode is None or not self._scores: + return [] + cut_found = [score < 0.0 for score in self._scores] + cut_found.append(True) + boxcar = [1] * self._filter_size + cutoff = float(self._filter_size) / float(self._filter_size + 1) + filtered = np.convolve(self._scores, boxcar, mode="same") + for i in range(len(self._scores)): + if i >= self._window_size and filtered[i] < cutoff: + # TODO: Should we discard the N most extreme values before calculating threshold? + window = filtered[i - self._window_size : i] + threshold = window.mean() - (self._deviation * window.std()) + if filtered[i] < threshold: + cut_found[i] = True + + cuts = [] + last_cut = 0 + for i in range(len(cut_found)): + if cut_found[i]: + if (i - last_cut) > self._window_size: + cuts.append(last_cut) + last_cut = i + 1 + return [self._start_timecode + cut for cut in cuts][1:] diff --git a/tests/test_detectors.py b/tests/test_detectors.py index 6754d910..0756a961 100644 --- a/tests/test_detectors.py +++ b/tests/test_detectors.py @@ -16,6 +16,7 @@ test case material. """ +import importlib.util import os from dataclasses import dataclass @@ -29,16 +30,21 @@ ContentDetector, HashDetector, HistogramDetector, + KoalaDetector, ThresholdDetector, ) # Untyped so each entry retains its concrete `type[...]` for parameterized construction # (calls below pass detector-specific kwargs like `min_scene_len`). +# KoalaDetector requires the optional `scikit-image` package; it is only tested when installed. +_HAVE_SKIMAGE = importlib.util.find_spec("skimage") is not None + FAST_CUT_DETECTORS = ( AdaptiveDetector, ContentDetector, HashDetector, HistogramDetector, + *((KoalaDetector,) if _HAVE_SKIMAGE else ()), ) ALL_DETECTORS = (*FAST_CUT_DETECTORS, ThresholdDetector) @@ -133,7 +139,9 @@ def get_fast_cut_test_cases(): ), id=f"{detector_type.__name__}/m=30", ) + # TODO: Make this work, right now min_scene_len isn't used by the detector. for detector_type in FAST_CUT_DETECTORS + if detector_type != KoalaDetector ] return test_cases @@ -246,6 +254,9 @@ def test_detectors_with_stats(test_video_file): ) def test_min_scene_len_accepts_time_values(detector_type, min_scene_len): """Detectors accept min_scene_len as int (frames), float (seconds), or str (timecode).""" + # TODO: Make this work, right now min_scene_len isn't used by the detector. + if detector_type is KoalaDetector: + pytest.skip("KoalaDetector does not apply min_scene_len yet.") test_case = TestCase( path=get_absolute_path("resources/goldeneye.mp4"), detector=detector_type(min_scene_len=min_scene_len),