From 4c3d20679d3936a04ac3aeb2c7334c1485dbc101 Mon Sep 17 00:00:00 2001 From: "Peter B. Johnson" Date: Tue, 15 Sep 2026 19:35:03 +0100 Subject: [PATCH 1/2] implement: Test against real Lambda Feedback exports (t19) --- docs/source/filters.py | 9 +- in2lambda/filters/__init__.py | 18 ++++ in2lambda/json_convert/json_convert.py | 5 +- .../minimal_template_question.json | 1 - in2lambda/main.py | 10 +- tests/conftest.py | 71 ++++++++++++++ tests/test_exports.py | 96 +++++++++++++++++++ tests/test_runner.py | 25 ++--- 8 files changed, 203 insertions(+), 32 deletions(-) create mode 100644 tests/test_exports.py diff --git a/docs/source/filters.py b/docs/source/filters.py index 2002768..483896f 100644 --- a/docs/source/filters.py +++ b/docs/source/filters.py @@ -1,6 +1,5 @@ import importlib import os -import pkgutil import shutil import subprocess from pathlib import Path @@ -19,13 +18,7 @@ def generate_filters_docs(): autosummary_directory.mkdir(exist_ok=True, parents=True) static_pdf_directory.mkdir(exist_ok=True) - filters = ( - i.name - for i in pkgutil.iter_modules(in2lambda.filters.__path__) - if i.name != "markdown" - ) - - for filter_name in filters: + for filter_name in in2lambda.filters.builtin_filters(): filter_module = importlib.import_module( f"in2lambda.filters.{filter_name}.filter" ) diff --git a/in2lambda/filters/__init__.py b/in2lambda/filters/__init__.py index 830e77d..9519688 100644 --- a/in2lambda/filters/__init__.py +++ b/in2lambda/filters/__init__.py @@ -1 +1,19 @@ """Subject specific panflute filters for parsing LaTeX documents.""" + +import pkgutil + + +def builtin_filters() -> list[str]: + """Lists the filters shipped with in2lambda. + + Each filter is a subpackage of this one; ``markdown`` is a helper module they share. + + Returns: + The filter names, as accepted by :func:`in2lambda.main.runner`. + + Examples: + >>> from in2lambda.filters import builtin_filters + >>> "PartsSepSol" in builtin_filters() + True + """ + return [i.name for i in pkgutil.iter_modules(__path__) if i.name != "markdown"] diff --git a/in2lambda/json_convert/json_convert.py b/in2lambda/json_convert/json_convert.py index dd85f23..0733454 100644 --- a/in2lambda/json_convert/json_convert.py +++ b/in2lambda/json_convert/json_convert.py @@ -96,12 +96,13 @@ def converter( ListQuestions[i].parts[j].worked_solution ) - # Output file + # Lambda Feedback names the file after the title with only spaces made + # underscores; path separators go too, so a title cannot leave the set folder. filename = ( "question_" + str(i).zfill(3) + "_" - + re.sub(r"[^\w\-_.]", "_", output["title"].strip()) + + re.sub(r"[\s/\\]", "_", output["title"].strip()) ) # write questions into directory diff --git a/in2lambda/json_convert/minimal_template_question.json b/in2lambda/json_convert/minimal_template_question.json index 1f96fc8..db97cf9 100644 --- a/in2lambda/json_convert/minimal_template_question.json +++ b/in2lambda/json_convert/minimal_template_question.json @@ -14,7 +14,6 @@ "answerContent": "", "responseAreas": [], "workedSolution": { - "title": "", "content": "Part worked solution here", "children": [] } diff --git a/in2lambda/main.py b/in2lambda/main.py index d5df1e2..261a18c 100644 --- a/in2lambda/main.py +++ b/in2lambda/main.py @@ -7,7 +7,6 @@ # sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) import importlib -import pkgutil import subprocess from typing import Optional @@ -168,14 +167,7 @@ def runner( # Python files in the subjects directory @click.argument( "chosen_filter", - type=click.Choice( - [ - i.name - for i in pkgutil.iter_modules(in2lambda.filters.__path__) - if i.name != "markdown" - ], - case_sensitive=False, - ), + type=click.Choice(in2lambda.filters.builtin_filters(), case_sensitive=False), ) @click.option( "--out", diff --git a/tests/conftest.py b/tests/conftest.py index 095b418..9a0eef1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,10 +1,22 @@ """Shared pytest fixtures for the in2lambda test suite.""" +import json import os +from pathlib import Path import pytest import in2lambda +from in2lambda.api.part import Part +from in2lambda.api.question import Question +from in2lambda.api.set import Set +from in2lambda.api.visibility_status import VisibilityController, VisibilityStatus + +EXPORTS_DIR = Path(__file__).parent / "fixtures" / "exports" +"""Real Lambda Feedback exports, one set per folder, exactly as the platform wrote them.""" + +EXPORTS = sorted(path for path in EXPORTS_DIR.iterdir() if path.is_dir()) +"""Every export folder, found rather than listed so that adding one needs no code.""" @pytest.fixture(scope="session") @@ -14,3 +26,62 @@ def filters_dir() -> str: Each filter ships a self-contained ``example.tex`` used by the end-to-end tests. """ return os.path.join(os.path.dirname(in2lambda.__file__), "filters") + + +def load_export(export_dir: Path) -> Set: + """Reads an exported set into the in2lambda model, keeping only what the model holds. + + The layout followed is the one described in ``fixtures/exports/README.md``. + + Args: + export_dir: A folder holding one exported set. + + Returns: + The set, with each question's images as absolute paths into ``media/``. + """ + (set_file,) = export_dir.glob("set_*.json") + set_json = json.loads(set_file.read_text()) + question_set = Set( + _name=set_json["name"], + _description=set_json["description"], + _finalAnswerVisibility=VisibilityController( + VisibilityStatus(set_json["finalAnswerVisibility"]) + ), + _workedSolutionVisibility=VisibilityController( + VisibilityStatus(set_json["workedSolutionVisibility"]) + ), + _structuredTutorialVisibility=VisibilityController( + VisibilityStatus(set_json["structuredTutorialVisibility"]) + ), + ) + + question_files = sorted( + export_dir.glob("question_*.json"), + key=lambda path: json.loads(path.read_text())["orderNumber"], + ) + media = sorted((export_dir / "media").glob("*")) + for question_file in question_files: + question_json = json.loads(question_file.read_text()) + question_set.questions.append( + Question( + title=question_json["title"], + main_text=question_json["masterContent"], + parts=[ + Part( + text=part["content"], + worked_solution=( + part["workedSolution"]["content"] + if "workedSolution" in part + else "" + ), + ) + for part in question_json["parts"] + ], + images=[ + str(image) + for image in media + if image.name.startswith(f"{question_file.stem}_") + ], + ) + ) + return question_set diff --git a/tests/test_exports.py b/tests/test_exports.py new file mode 100644 index 0000000..c460236 --- /dev/null +++ b/tests/test_exports.py @@ -0,0 +1,96 @@ +"""Round-trips every real Lambda Feedback export through the in2lambda model. + +Each folder in ``fixtures/exports`` is loaded into a :class:`~in2lambda.api.set.Set`, +written back with :meth:`~in2lambda.api.set.Set.to_json` and compared with the +original. The model holds far less than an export, so the comparison covers what it +does hold, the file names written, and that the writer emits no key Lambda Feedback +does not. +""" + +import json +from pathlib import Path + +import pytest +from conftest import EXPORTS, load_export + +from in2lambda.api.set import Set + +pytestmark = pytest.mark.parametrize("export_dir", EXPORTS, ids=lambda path: path.name) + + +def _write_back(question_set: Set, tmp_path: Path) -> Path: + question_set.to_json(str(tmp_path / "out")) + return tmp_path / "out" / question_set._name + + +def _relative_files(directory: Path) -> list[str]: + # Finder leaves .DS_Store beside files it has shown; it is not part of an export. + return sorted( + str(path.relative_to(directory)) + for path in directory.rglob("*") + if path.is_file() and not path.name.startswith(".") + ) + + +def _modelled(question_set: Set) -> dict: + # Visibility controllers have no equality, and image paths differ by where the + # set was read from, so compare their values and file names. + return { + "name": question_set._name, + "description": question_set._description, + "visibility": [ + str(question_set._finalAnswerVisibility), + str(question_set._workedSolutionVisibility), + str(question_set._structuredTutorialVisibility), + ], + "questions": [ + (q.title, q.main_text, q.parts, [Path(image).name for image in q.images]) + for q in question_set.questions + ], + } + + +def _key_paths(value, path: str = "") -> set[str]: + if isinstance(value, dict): + paths = set() + for key, item in value.items(): + paths |= {f"{path}.{key}"} | _key_paths(item, f"{path}.{key}") + return paths + if isinstance(value, list): + return set().union(*(_key_paths(item, f"{path}[]") for item in value)) + return set() + + +def _keys_by_kind(directory: Path) -> dict[str, set[str]]: + # Pooled over every set_ or question_ file rather than matched file to file: + # exports leave out components with no content, such as a part's worked solution. + keys: dict[str, set[str]] = {} + for file in directory.glob("*.json"): + kind = file.name.split("_")[0] + keys.setdefault(kind, set()).update(_key_paths(json.loads(file.read_text()))) + return keys + + +def test_export_round_trips(export_dir: Path, tmp_path: Path) -> None: + """Writing a loaded export reproduces its file names and reloads to the same set.""" + loaded = load_export(export_dir) + assert loaded.questions + assert all(question.main_text or question.parts for question in loaded.questions) + + written = _write_back(loaded, tmp_path) + + assert _relative_files(written) == _relative_files(export_dir) + assert _modelled(load_export(written)) == _modelled(loaded) + + +def test_written_keys_exist_in_export(export_dir: Path, tmp_path: Path) -> None: + """The writer emits no key, at any depth, that Lambda Feedback never exports there.""" + written = _keys_by_kind(_write_back(load_export(export_dir), tmp_path)) + exported = _keys_by_kind(export_dir) + + missing = { + kind: sorted(keys - exported[kind]) + for kind, keys in written.items() + if keys - exported[kind] + } + assert not missing, missing diff --git a/tests/test_runner.py b/tests/test_runner.py index df31429..7c39d51 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -1,9 +1,9 @@ """End-to-end tests for :func:`in2lambda.main.runner` across the built-in filters. Each built-in filter ships a self-contained ``example.tex`` that exercises the -document structure it targets. These tests run every filter over its own example -and check both the in-memory :class:`~in2lambda.api.set.Set` and the JSON/ZIP -files written to disk. +document structure it targets. These tests find every filter in the package, run +it over its own example and check both the in-memory :class:`~in2lambda.api.set.Set` +and the JSON/ZIP files written to disk, so a filter shipped without an example fails. """ import json @@ -12,15 +12,20 @@ import pytest from in2lambda.api.set import Set +from in2lambda.filters import builtin_filters from in2lambda.main import runner -BUILTIN_FILTERS = ["PartsSepSol", "PartsOneSol", "PartPartSolSol", "PartSolPartSol"] +def _example(filters_dir: str, filter_name: str) -> str: + path = os.path.join(filters_dir, filter_name, "example.tex") + assert os.path.isfile(path), f"{filter_name} ships no example.tex to test it with" + return path -@pytest.mark.parametrize("filter_name", BUILTIN_FILTERS) + +@pytest.mark.parametrize("filter_name", builtin_filters()) def test_runner_returns_populated_set(filter_name: str, filters_dir: str) -> None: """Every filter turns its example into a Set with at least one usable question.""" - result = runner(os.path.join(filters_dir, filter_name, "example.tex"), filter_name) + result = runner(_example(filters_dir, filter_name), filter_name) assert isinstance(result, Set) assert result.questions, f"{filter_name} produced no questions" @@ -29,17 +34,13 @@ def test_runner_returns_populated_set(filter_name: str, filters_dir: str) -> Non assert question.main_text or question.parts -@pytest.mark.parametrize("filter_name", BUILTIN_FILTERS) +@pytest.mark.parametrize("filter_name", builtin_filters()) def test_runner_writes_importable_json( filter_name: str, filters_dir: str, tmp_path ) -> None: """Passing an output directory produces the Lambda Feedback set/ dir and zip.""" out_dir = tmp_path / "out" - result = runner( - os.path.join(filters_dir, filter_name, "example.tex"), - filter_name, - str(out_dir), - ) + result = runner(_example(filters_dir, filter_name), filter_name, str(out_dir)) set_dir = out_dir / "set" assert set_dir.is_dir() From 8d9d6ca4e6bde26ef37ebf2d097cafbc25609ae1 Mon Sep 17 00:00:00 2001 From: "Peter B. Johnson" Date: Tue, 15 Sep 2026 21:31:52 +0100 Subject: [PATCH 2/2] implement: Test against real Lambda Feedback exports (t19) --- in2lambda/json_convert/json_convert.py | 5 ++-- tests/test_exports.py | 37 ++++++++++++++------------ 2 files changed, 23 insertions(+), 19 deletions(-) diff --git a/in2lambda/json_convert/json_convert.py b/in2lambda/json_convert/json_convert.py index 0733454..9c3dd29 100644 --- a/in2lambda/json_convert/json_convert.py +++ b/in2lambda/json_convert/json_convert.py @@ -97,12 +97,13 @@ def converter( ) # Lambda Feedback names the file after the title with only spaces made - # underscores; path separators go too, so a title cannot leave the set folder. + # underscores. Path separators go too, so a title cannot leave the set folder, + # and so do the characters Windows forbids in file names. filename = ( "question_" + str(i).zfill(3) + "_" - + re.sub(r"[\s/\\]", "_", output["title"].strip()) + + re.sub(r'[\s/\\<>:"|?*]', "_", output["title"].strip()) ) # write questions into directory diff --git a/tests/test_exports.py b/tests/test_exports.py index c460236..9d8f229 100644 --- a/tests/test_exports.py +++ b/tests/test_exports.py @@ -57,18 +57,21 @@ def _key_paths(value, path: str = "") -> set[str]: paths |= {f"{path}.{key}"} | _key_paths(item, f"{path}.{key}") return paths if isinstance(value, list): - return set().union(*(_key_paths(item, f"{path}[]") for item in value)) + return set().union( + *(_key_paths(item, f"{path}[{i}]") for i, item in enumerate(value)) + ) return set() -def _keys_by_kind(directory: Path) -> dict[str, set[str]]: - # Pooled over every set_ or question_ file rather than matched file to file: - # exports leave out components with no content, such as a part's worked solution. - keys: dict[str, set[str]] = {} - for file in directory.glob("*.json"): - kind = file.name.split("_")[0] - keys.setdefault(kind, set()).update(_key_paths(json.loads(file.read_text()))) - return keys +def _unexported_keys(written: dict, exported: dict) -> list[str]: + missing = _key_paths(written) - _key_paths(exported) + # Lambda Feedback leaves a part's workedSolution out of its export when the part + # has none, but the writer always emits one, so only then may it be absent. + for i, part in enumerate(written.get("parts", [])): + if not part["workedSolution"]["content"]: + prefix = f".parts[{i}].workedSolution" + missing = {key for key in missing if not key.startswith(prefix)} + return sorted(missing) def test_export_round_trips(export_dir: Path, tmp_path: Path) -> None: @@ -85,12 +88,12 @@ def test_export_round_trips(export_dir: Path, tmp_path: Path) -> None: def test_written_keys_exist_in_export(export_dir: Path, tmp_path: Path) -> None: """The writer emits no key, at any depth, that Lambda Feedback never exports there.""" - written = _keys_by_kind(_write_back(load_export(export_dir), tmp_path)) - exported = _keys_by_kind(export_dir) - - missing = { - kind: sorted(keys - exported[kind]) - for kind, keys in written.items() - if keys - exported[kind] - } + written = _write_back(load_export(export_dir), tmp_path) + + missing = {} + for file in written.glob("*.json"): + exported = json.loads((export_dir / file.name).read_text()) + keys = _unexported_keys(json.loads(file.read_text()), exported) + if keys: + missing[file.name] = keys assert not missing, missing