Adding GUI System Tests

CONTRIBUTING.md

@@ -41,6 +41,15 @@ The coverage report can be saved to the directory htmlcov by running the tests w
 
 For information on other coverage report formats, see https://pytest-cov.readthedocs.io/en/latest/reporting.html
 
+System tests which load the GUI and perform common workflows are run by default but can be excluded via
+
+    pytest tests --skip_system_tests
+
+Some tests can be time-consuming and are only run when necessary, i.e. they are skipped by default and marked as "slow".
+To include slow tests use the flag:
+
+    pytest tests --run_slow_tests
+
 Documentation
 -------------
 * The documentation will be hosted on GitHub pages.

rascal2/core/runner.py

@@ -81,7 +81,7 @@ def create_process(self):
         """Create process and multiprocessing event."""
         self.go_event = Event()
         self.exit_event = Event()
-        matlab_helper = MatlabHelper()
+        self.matlab_helper = MatlabHelper()
         self.process = Process(
             target=run,
             args=(
@@ -91,8 +91,8 @@ def create_process(self):
                 self.plot_queue,
                 self.go_event,
                 self.exit_event,
-                matlab_helper.ready_event,
-                matlab_helper.engine_output,
+                self.matlab_helper.ready_event,
+                self.matlab_helper.engine_output,
             ),
         )
 
@@ -199,6 +199,7 @@ def stop(self):
             self.process.kill()
         self.process = None
         self.clear_queues_and_events()
+        self.matlab_helper.close_event.set()
 
 
 def init_matlab_engine(engine_ready, engine_output, msg_queue):

requirements-dev.txt

@@ -3,3 +3,4 @@ pytest-cov
 ruff
 Sphinx
 pydata-sphinx-theme
+parameterized

tests/conftest.py

@@ -47,3 +47,40 @@ def teardown_mock_setting():
             target.stop()
 
     request.addfinalizer(teardown_mock_setting)
+
+
+def pytest_addoption(parser):
+    parser.addoption("--skip_system_tests", action="store_true", default=False, help="Skip GUI system tests")
+    parser.addoption("--run_slow_tests", action="store_true", default=False, help="Run slow tests")
+
+
+def pytest_configure(config):
+    config.addinivalue_line("markers", "system: GUI system tests")
+    config.addinivalue_line("markers", "unit: unit tests")
+    config.addinivalue_line("markers", "slow: marks slow tests which only run on review")
+
+
+allowed_markers = []
+skipped_tests = []
+
+
+def pytest_collection_modifyitems(config, items):
+    allowed_markers.append(pytest.mark.unit.mark)
+    if not config.getoption("--skip_system_tests"):
+        allowed_markers.append(pytest.mark.system.mark)
+    if config.getoption("--run_slow_tests"):
+        allowed_markers.append(pytest.mark.slow.mark)
+    for item in items:
+        if "gui_system" in item.nodeid:
+            item.add_marker(pytest.mark.system)
+        if "slow" in item.nodeid:
+            item.add_marker(pytest.mark.slow)
+        else:
+            item.add_marker(pytest.mark.unit)
+        for mark in item.own_markers:
+            if "parametrize" in mark.name:
+                continue
+            if mark not in allowed_markers:
+                item.add_marker(pytest.mark.skip(reason="Test not selected"))
+                skipped_tests.append(item.nodeid)
+                break

tests/system/gui_system_base.py

@@ -0,0 +1,81 @@
+import logging
+import os
+import sys
+import unittest
+from collections.abc import Callable
+
+from PyQt6.QtCore import Qt, QTimer
+from PyQt6.QtTest import QTest
+from PyQt6.QtWidgets import QApplication, QMessageBox
+
+from rascal2.config import setup_logging
+from rascal2.ui.view import MainWindowView
+
+SHOW_DELAY = 10  # Can be increased to watch tests
+SHORT_DELAY = 100
+
+
+def wait_until(
+    test_func: Callable[[], bool], delay=0.1, max_retry=100, message: str = "wait_until reached max retries"
+):
+    """Repeat test_func every delay seconds until it becomes true. Raises RuntimeError if max_retry is reached."""
+    for _ in range(max_retry):
+        if test_func():
+            return True
+        QTest.qWait(int(delay * 1000))
+    raise RuntimeError(message)
+
+
+class GuiSystemBase(unittest.TestCase):
+    app: QApplication = QApplication.instance()
+
+    def setUp(self) -> None:
+        setup_logging()
+        self.start_processes_old = os.getenv("START_PROCESSES")
+        os.environ["START_PROCESSES"] = "False"
+        self.no_exceptions = True
+
+        sys.excepthook = self.exception_hook
+        self.main_window = MainWindowView()
+        self.main_window.show()
+        QTest.qWait(SHORT_DELAY)
+
+    def tearDown(self) -> None:
+        if not self.no_exceptions:
+            raise Exception("An exception occurred in a PyQt slot")
+        sys.excepthook = sys.__excepthook__
+        logger = logging.getLogger("rascal2")
+        logger.handlers.clear()
+        QTimer.singleShot(SHORT_DELAY, lambda: self._click_messagebox(["Discard", "Don't Save"]))
+        self.main_window.close()
+        wait_until(
+            lambda: not self.main_window.isVisible(),
+            delay=0.05,
+            max_retry=200,
+            message="Main window did not close within 10 seconds",
+        )
+        del self.main_window
+        os.environ["START_PROCESSES"] = self.start_processes_old
+
+    def exception_hook(self, exc_type, exc_value, exc_traceback):
+        self.no_exceptions = False
+        sys.__excepthook__(exc_type, exc_value, exc_traceback)
+
+    @classmethod
+    def _click_messagebox(cls, button_text_list: list[str]):
+        """Needs to be queued with QTimer.singleShot before triggering the message box."""
+        no_buttons_found = True
+        button_texts = []
+        for widget in cls.app.topLevelWidgets():
+            if isinstance(widget, QMessageBox) and widget.isVisible():
+                for button_text in button_text_list:
+                    for button in widget.buttons():
+                        if button.text().replace("&", "") == button_text:
+                            QTest.mouseClick(button, Qt.MouseButton.LeftButton)
+                            no_buttons_found = False
+                    button_texts = [button.text() for button in widget.buttons()]
+                if no_buttons_found:
+                    raise ValueError(
+                        f"Could not find buttons '{button_text_list} in {button_texts}'.\n"
+                        f"Message box: {widget.windowTitle()} {widget.text()}"
+                    )

tests/system/gui_system_loading_test.py

@@ -0,0 +1,78 @@
+import random
+from pathlib import Path
+
+from parameterized import parameterized
+from PyQt6.QtTest import QTest
+from PyQt6.QtWidgets import QApplication
+
+from rascal2.dialogs.startup_dialog import LoadDialog
+from tests.system.gui_system_base import SHORT_DELAY, GuiSystemBase, wait_until
+
+EXAMPLES_PATH = Path(__file__, "../../../examples").resolve().as_posix()
+
+
+class TestGuiSystemLoading(GuiSystemBase):
+    def setUp(self) -> None:
+        super().setUp()
+
+    def tearDown(self) -> None:
+        super().tearDown()
+
+    def test_load(self):
+        QTest.qWait(SHORT_DELAY)
+        self.main_window.startup_dlg.import_project_button.click()
+        load_dialog = self.main_window.findChild(LoadDialog)
+        load_dialog.tabs.setCurrentIndex(2)
+        load_dialog.example_list_widget.itemClicked.emit(load_dialog.example_list_widget.item(0))
+        QTest.qWait(SHORT_DELAY)
+        assert self.main_window.presenter.model.project.name == "DSPC Standard Layers"
+
+    @parameterized.expand(
+        [
+            ("calculate", 142.346, 1000),
+            ("simplex", 13.1816, 1000),
+            ("de", 10.4746, 1000),
+        ]
+    )
+    def test_run(self, procedure_name, expected_chi, test_duration):
+        QTest.qWait(SHORT_DELAY)
+        self.main_window.startup_dlg.import_project_button.click()
+        load_dialog = self.main_window.findChild(LoadDialog)
+        load_dialog.tabs.setCurrentIndex(2)
+        load_dialog.example_list_widget.itemClicked.emit(load_dialog.example_list_widget.item(0))
+        wait_until(lambda: self.main_window.controls_widget.chi_squared.text() != "")
+        self.main_window.controls_widget.update_chi_squared("")
+        self.main_window.controls_widget.procedure_dropdown.setCurrentText(procedure_name)
+        QApplication.processEvents()
+        self.main_window.controls_widget.run_button.click()
+        wait_until(
+            lambda: "Finished RAT" in self.main_window.terminal_widget.text_area.toPlainText(), max_retry=test_duration
+        )
+        QTest.qWait(SHORT_DELAY)
+        assert self.main_window.controls_widget.chi_squared.text() == str(expected_chi)
+        assert self.main_window.presenter.runner.error is None
+
+    @parameterized.expand(
+        [
+            ("ns", 9.47214, 6000),
+            ("dream", 9.67234, 2000),
+        ]
+    )
+    def test_run_slow(self, procedure_name, expected_chi, test_duration):
+        random.seed(10)
+        QTest.qWait(SHORT_DELAY)
+        self.main_window.startup_dlg.import_project_button.click()
+        load_dialog = self.main_window.findChild(LoadDialog)
+        load_dialog.tabs.setCurrentIndex(2)
+        load_dialog.example_list_widget.itemClicked.emit(load_dialog.example_list_widget.item(0))
+        wait_until(lambda: self.main_window.controls_widget.chi_squared.text() != "")
+        self.main_window.controls_widget.update_chi_squared("")
+        self.main_window.controls_widget.procedure_dropdown.setCurrentText(procedure_name)
+        QApplication.processEvents()
+        self.main_window.controls_widget.run_button.click()
+        wait_until(
+            lambda: "Finished RAT" in self.main_window.terminal_widget.text_area.toPlainText(), max_retry=test_duration
+        )
+        QTest.qWait(SHORT_DELAY)
+        assert self.main_window.controls_widget.chi_squared.text() == str(expected_chi)
+        assert self.main_window.presenter.runner.error is None

tests/system/gui_system_main_window_test.py

@@ -0,0 +1,28 @@
+from rascal2 import RASCAL2_VERSION
+from tests.system.gui_system_base import GuiSystemBase
+
+
+class TestGuiSystemMainWindow(GuiSystemBase):
+    def setUp(self) -> None:
+        super().setUp()
+
+    def tearDown(self) -> None:
+        super().tearDown()
+
+    def test_main_window(self):
+        self.main_window.presenter.create_project("project", ".")
+        names = [win.windowTitle() for win in self.main_window.mdi.subWindowList()]
+        assert names == ["Fitting Controls", "Terminal", "Project", "Plots"]
+        assert (
+            self.main_window.terminal_widget.text_area.toPlainText()
+            == f"""
+ ███████████                       █████████    █████████   █████
+░░███░░░░░███                     ███░░░░░███  ███░░░░░███ ░░███
+ ░███    ░███   ██████    █████  ███     ░░░  ░███    ░███  ░███
+ ░██████████   ░░░░░███  ███░░  ░███          ░███████████  ░███
+ ░███░░░░░███   ███████ ░░█████ ░███          ░███░░░░░███  ░███
+ ░███    ░███  ███░░███  ░░░░███░░███     ███ ░███    ░███  ░███      █
+ █████   █████░░████████ ██████  ░░█████████  █████   █████ ███████████
+░░░░░   ░░░░░  ░░░░░░░░ ░░░░░░    ░░░░░░░░░  ░░░░░   ░░░░░ ░░░░░░░░░░░
+RasCAL-2: software for neutron reflectivity calculations v{RASCAL2_VERSION}"""
+        )