diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c824f819..b2942d73 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -41,6 +41,15 @@ The coverage report can be saved to the directory htmlcov by running the tests w For information on other coverage report formats, see https://pytest-cov.readthedocs.io/en/latest/reporting.html +System tests which load the GUI and perform common workflows are run by default but can be excluded via + + pytest tests --skip_system_tests + +Some tests can be time-consuming and are only run when necessary, i.e. they are skipped by default and marked as "slow". +To include slow tests use the flag: + + pytest tests --run_slow_tests + Documentation ------------- * The documentation will be hosted on GitHub pages. diff --git a/rascal2/core/runner.py b/rascal2/core/runner.py index 502c4801..e80a5d8a 100644 --- a/rascal2/core/runner.py +++ b/rascal2/core/runner.py @@ -81,7 +81,7 @@ def create_process(self): """Create process and multiprocessing event.""" self.go_event = Event() self.exit_event = Event() - matlab_helper = MatlabHelper() + self.matlab_helper = MatlabHelper() self.process = Process( target=run, args=( @@ -91,8 +91,8 @@ def create_process(self): self.plot_queue, self.go_event, self.exit_event, - matlab_helper.ready_event, - matlab_helper.engine_output, + self.matlab_helper.ready_event, + self.matlab_helper.engine_output, ), ) @@ -199,6 +199,7 @@ def stop(self): self.process.kill() self.process = None self.clear_queues_and_events() + self.matlab_helper.close_event.set() def init_matlab_engine(engine_ready, engine_output, msg_queue): diff --git a/requirements-dev.txt b/requirements-dev.txt index 8ffbdcac..ca69e882 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -3,3 +3,4 @@ pytest-cov ruff Sphinx pydata-sphinx-theme +parameterized diff --git a/tests/conftest.py b/tests/conftest.py index 6d7760a8..33e9d3c5 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -47,3 +47,40 @@ def teardown_mock_setting(): target.stop() request.addfinalizer(teardown_mock_setting) + + +def pytest_addoption(parser): + parser.addoption("--skip_system_tests", action="store_true", default=False, help="Skip GUI system tests") + parser.addoption("--run_slow_tests", action="store_true", default=False, help="Run slow tests") + + +def pytest_configure(config): + config.addinivalue_line("markers", "system: GUI system tests") + config.addinivalue_line("markers", "unit: unit tests") + config.addinivalue_line("markers", "slow: marks slow tests which only run on review") + + +allowed_markers = [] +skipped_tests = [] + + +def pytest_collection_modifyitems(config, items): + allowed_markers.append(pytest.mark.unit.mark) + if not config.getoption("--skip_system_tests"): + allowed_markers.append(pytest.mark.system.mark) + if config.getoption("--run_slow_tests"): + allowed_markers.append(pytest.mark.slow.mark) + for item in items: + if "gui_system" in item.nodeid: + item.add_marker(pytest.mark.system) + if "slow" in item.nodeid: + item.add_marker(pytest.mark.slow) + else: + item.add_marker(pytest.mark.unit) + for mark in item.own_markers: + if "parametrize" in mark.name: + continue + if mark not in allowed_markers: + item.add_marker(pytest.mark.skip(reason="Test not selected")) + skipped_tests.append(item.nodeid) + break diff --git a/tests/system/gui_system_base.py b/tests/system/gui_system_base.py new file mode 100644 index 00000000..90aabe0b --- /dev/null +++ b/tests/system/gui_system_base.py @@ -0,0 +1,81 @@ +import logging +import os +import sys +import unittest +from collections.abc import Callable + +from PyQt6.QtCore import Qt, QTimer +from PyQt6.QtTest import QTest +from PyQt6.QtWidgets import QApplication, QMessageBox + +from rascal2.config import setup_logging +from rascal2.ui.view import MainWindowView + +SHOW_DELAY = 10 # Can be increased to watch tests +SHORT_DELAY = 100 + + +def wait_until( + test_func: Callable[[], bool], delay=0.1, max_retry=100, message: str = "wait_until reached max retries" +): + """Repeat test_func every delay seconds until it becomes true. Raises RuntimeError if max_retry is reached.""" + for _ in range(max_retry): + if test_func(): + return True + QTest.qWait(int(delay * 1000)) + raise RuntimeError(message) + + +class GuiSystemBase(unittest.TestCase): + app: QApplication = QApplication.instance() + + def setUp(self) -> None: + setup_logging() + self.start_processes_old = os.getenv("START_PROCESSES") + os.environ["START_PROCESSES"] = "False" + self.no_exceptions = True + + sys.excepthook = self.exception_hook + self.main_window = MainWindowView() + self.main_window.show() + QTest.qWait(SHORT_DELAY) + + def tearDown(self) -> None: + if not self.no_exceptions: + raise Exception("An exception occurred in a PyQt slot") + sys.excepthook = sys.__excepthook__ + logger = logging.getLogger("rascal2") + logger.handlers.clear() + QTimer.singleShot(SHORT_DELAY, lambda: self._click_messagebox(["Discard", "Don't Save"])) + self.main_window.close() + wait_until( + lambda: not self.main_window.isVisible(), + delay=0.05, + max_retry=200, + message="Main window did not close within 10 seconds", + ) + del self.main_window + os.environ["START_PROCESSES"] = self.start_processes_old + + def exception_hook(self, exc_type, exc_value, exc_traceback): + self.no_exceptions = False + sys.__excepthook__(exc_type, exc_value, exc_traceback) + + @classmethod + def _click_messagebox(cls, button_text_list: list[str]): + """Needs to be queued with QTimer.singleShot before triggering the message box.""" + no_buttons_found = True + button_texts = [] + for widget in cls.app.topLevelWidgets(): + if isinstance(widget, QMessageBox) and widget.isVisible(): + for button_text in button_text_list: + for button in widget.buttons(): + if button.text().replace("&", "") == button_text: + QTest.mouseClick(button, Qt.MouseButton.LeftButton) + no_buttons_found = False + button_texts = [button.text() for button in widget.buttons()] + if no_buttons_found: + raise ValueError( + f"Could not find buttons '{button_text_list} in {button_texts}'.\n" + f"Message box: {widget.windowTitle()} {widget.text()}" + ) diff --git a/tests/system/gui_system_loading_test.py b/tests/system/gui_system_loading_test.py new file mode 100644 index 00000000..07797159 --- /dev/null +++ b/tests/system/gui_system_loading_test.py @@ -0,0 +1,78 @@ +import random +from pathlib import Path + +from parameterized import parameterized +from PyQt6.QtTest import QTest +from PyQt6.QtWidgets import QApplication + +from rascal2.dialogs.startup_dialog import LoadDialog +from tests.system.gui_system_base import SHORT_DELAY, GuiSystemBase, wait_until + +EXAMPLES_PATH = Path(__file__, "../../../examples").resolve().as_posix() + + +class TestGuiSystemLoading(GuiSystemBase): + def setUp(self) -> None: + super().setUp() + + def tearDown(self) -> None: + super().tearDown() + + def test_load(self): + QTest.qWait(SHORT_DELAY) + self.main_window.startup_dlg.import_project_button.click() + load_dialog = self.main_window.findChild(LoadDialog) + load_dialog.tabs.setCurrentIndex(2) + load_dialog.example_list_widget.itemClicked.emit(load_dialog.example_list_widget.item(0)) + QTest.qWait(SHORT_DELAY) + assert self.main_window.presenter.model.project.name == "DSPC Standard Layers" + + @parameterized.expand( + [ + ("calculate", 142.346, 1000), + ("simplex", 13.1816, 1000), + ("de", 10.4746, 1000), + ] + ) + def test_run(self, procedure_name, expected_chi, test_duration): + QTest.qWait(SHORT_DELAY) + self.main_window.startup_dlg.import_project_button.click() + load_dialog = self.main_window.findChild(LoadDialog) + load_dialog.tabs.setCurrentIndex(2) + load_dialog.example_list_widget.itemClicked.emit(load_dialog.example_list_widget.item(0)) + wait_until(lambda: self.main_window.controls_widget.chi_squared.text() != "") + self.main_window.controls_widget.update_chi_squared("") + self.main_window.controls_widget.procedure_dropdown.setCurrentText(procedure_name) + QApplication.processEvents() + self.main_window.controls_widget.run_button.click() + wait_until( + lambda: "Finished RAT" in self.main_window.terminal_widget.text_area.toPlainText(), max_retry=test_duration + ) + QTest.qWait(SHORT_DELAY) + assert self.main_window.controls_widget.chi_squared.text() == str(expected_chi) + assert self.main_window.presenter.runner.error is None + + @parameterized.expand( + [ + ("ns", 9.47214, 6000), + ("dream", 9.67234, 2000), + ] + ) + def test_run_slow(self, procedure_name, expected_chi, test_duration): + random.seed(10) + QTest.qWait(SHORT_DELAY) + self.main_window.startup_dlg.import_project_button.click() + load_dialog = self.main_window.findChild(LoadDialog) + load_dialog.tabs.setCurrentIndex(2) + load_dialog.example_list_widget.itemClicked.emit(load_dialog.example_list_widget.item(0)) + wait_until(lambda: self.main_window.controls_widget.chi_squared.text() != "") + self.main_window.controls_widget.update_chi_squared("") + self.main_window.controls_widget.procedure_dropdown.setCurrentText(procedure_name) + QApplication.processEvents() + self.main_window.controls_widget.run_button.click() + wait_until( + lambda: "Finished RAT" in self.main_window.terminal_widget.text_area.toPlainText(), max_retry=test_duration + ) + QTest.qWait(SHORT_DELAY) + assert self.main_window.controls_widget.chi_squared.text() == str(expected_chi) + assert self.main_window.presenter.runner.error is None diff --git a/tests/system/gui_system_main_window_test.py b/tests/system/gui_system_main_window_test.py new file mode 100644 index 00000000..cf11bcdc --- /dev/null +++ b/tests/system/gui_system_main_window_test.py @@ -0,0 +1,28 @@ +from rascal2 import RASCAL2_VERSION +from tests.system.gui_system_base import GuiSystemBase + + +class TestGuiSystemMainWindow(GuiSystemBase): + def setUp(self) -> None: + super().setUp() + + def tearDown(self) -> None: + super().tearDown() + + def test_main_window(self): + self.main_window.presenter.create_project("project", ".") + names = [win.windowTitle() for win in self.main_window.mdi.subWindowList()] + assert names == ["Fitting Controls", "Terminal", "Project", "Plots"] + assert ( + self.main_window.terminal_widget.text_area.toPlainText() + == f""" + ███████████ █████████ █████████ █████ +░░███░░░░░███ ███░░░░░███ ███░░░░░███ ░░███ + ░███ ░███ ██████ █████ ███ ░░░ ░███ ░███ ░███ + ░██████████ ░░░░░███ ███░░ ░███ ░███████████ ░███ + ░███░░░░░███ ███████ ░░█████ ░███ ░███░░░░░███ ░███ + ░███ ░███ ███░░███ ░░░░███░░███ ███ ░███ ░███ ░███ █ + █████ █████░░████████ ██████ ░░█████████ █████ █████ ███████████ +░░░░░ ░░░░░ ░░░░░░░░ ░░░░░░ ░░░░░░░░░ ░░░░░ ░░░░░ ░░░░░░░░░░░ +RasCAL-2: software for neutron reflectivity calculations v{RASCAL2_VERSION}""" + ) diff --git a/tests/test_ui.py b/tests/test_ui.py deleted file mode 100644 index 88b57760..00000000 --- a/tests/test_ui.py +++ /dev/null @@ -1,51 +0,0 @@ -import sys - -import pytest - -from rascal2.ui.view import MainWindowView - - -@pytest.fixture -def make_main_window(request): - def make(): - no_exceptions = True - - def exception_hook(exc_type, exc_value, exc_traceback): - nonlocal no_exceptions - no_exceptions = False - sys.__excepthook__(exc_type, exc_value, exc_traceback) - - sys.excepthook = exception_hook - - window = MainWindowView() - - def cleanup(): - nonlocal no_exceptions - if not no_exceptions: - raise Exception("An exception occurred in a PyQt slot") - sys.excepthook = sys.__excepthook__ - - request.addfinalizer(cleanup) - return window - - return make - - -def test_integration(qt_application, make_main_window): - _ = qt_application - window = make_main_window() - window.show() - print("test_integration 1") - - window.presenter.create_project("project", ".") - print("test_integration 2") - names = [win.windowTitle() for win in window.mdi.subWindowList()] - print("test_integration 3") - # QMDIArea is first in last out hence the reversed list - assert names == ["Fitting Controls", "Terminal", "Project", "Plots"] - print("test_integration 4") - - # Work through the different sections of the UI - - window.close() - print("test_integration 5")