Test against real Lambda Feedback exports

docs/source/filters.py

@@ -1,6 +1,5 @@
 import importlib
 import os
-import pkgutil
 import shutil
 import subprocess
 from pathlib import Path
@@ -19,13 +18,7 @@ def generate_filters_docs():
     autosummary_directory.mkdir(exist_ok=True, parents=True)
     static_pdf_directory.mkdir(exist_ok=True)
 
-    filters = (
-        i.name
-        for i in pkgutil.iter_modules(in2lambda.filters.__path__)
-        if i.name != "markdown"
-    )
-
-    for filter_name in filters:
+    for filter_name in in2lambda.filters.builtin_filters():
         filter_module = importlib.import_module(
             f"in2lambda.filters.{filter_name}.filter"
         )

in2lambda/filters/__init__.py

@@ -1 +1,19 @@
 """Subject specific panflute filters for parsing LaTeX documents."""
+
+import pkgutil
+
+
+def builtin_filters() -> list[str]:
+    """Lists the filters shipped with in2lambda.
+
+    Each filter is a subpackage of this one; ``markdown`` is a helper module they share.
+
+    Returns:
+        The filter names, as accepted by :func:`in2lambda.main.runner`.
+
+    Examples:
+        >>> from in2lambda.filters import builtin_filters
+        >>> "PartsSepSol" in builtin_filters()
+        True
+    """
+    return [i.name for i in pkgutil.iter_modules(__path__) if i.name != "markdown"]

in2lambda/json_convert/json_convert.py

@@ -96,12 +96,14 @@ def converter(
                     ListQuestions[i].parts[j].worked_solution
                 )
 
-        # Output file
+        # Lambda Feedback names the file after the title with only spaces made
+        # underscores. Path separators go too, so a title cannot leave the set folder,
+        # and so do the characters Windows forbids in file names.
         filename = (
             "question_"
             + str(i).zfill(3)
             + "_"
-            + re.sub(r"[^\w\-_.]", "_", output["title"].strip())
+            + re.sub(r'[\s/\\<>:"|?*]', "_", output["title"].strip())
         )
 
         # write questions into directory

in2lambda/json_convert/minimal_template_question.json

@@ -14,7 +14,6 @@
       "answerContent": "",
       "responseAreas": [],
       "workedSolution": {
-        "title": "",
         "content": "Part worked solution here",
         "children": []
       }

in2lambda/main.py

@@ -7,7 +7,6 @@
 # sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
 
 import importlib
-import pkgutil
 import subprocess
 from typing import Optional
 
@@ -168,14 +167,7 @@ def runner(
 # Python files in the subjects directory
 @click.argument(
     "chosen_filter",
-    type=click.Choice(
-        [
-            i.name
-            for i in pkgutil.iter_modules(in2lambda.filters.__path__)
-            if i.name != "markdown"
-        ],
-        case_sensitive=False,
-    ),
+    type=click.Choice(in2lambda.filters.builtin_filters(), case_sensitive=False),
 )
 @click.option(
     "--out",

tests/conftest.py

@@ -1,10 +1,22 @@
 """Shared pytest fixtures for the in2lambda test suite."""
 
+import json
 import os
+from pathlib import Path
 
 import pytest
 
 import in2lambda
+from in2lambda.api.part import Part
+from in2lambda.api.question import Question
+from in2lambda.api.set import Set
+from in2lambda.api.visibility_status import VisibilityController, VisibilityStatus
+
+EXPORTS_DIR = Path(__file__).parent / "fixtures" / "exports"
+"""Real Lambda Feedback exports, one set per folder, exactly as the platform wrote them."""
+
+EXPORTS = sorted(path for path in EXPORTS_DIR.iterdir() if path.is_dir())
+"""Every export folder, found rather than listed so that adding one needs no code."""
 
 
 @pytest.fixture(scope="session")
@@ -14,3 +26,62 @@ def filters_dir() -> str:
     Each filter ships a self-contained ``example.tex`` used by the end-to-end tests.
     """
     return os.path.join(os.path.dirname(in2lambda.__file__), "filters")
+
+
+def load_export(export_dir: Path) -> Set:
+    """Reads an exported set into the in2lambda model, keeping only what the model holds.
+
+    The layout followed is the one described in ``fixtures/exports/README.md``.
+
+    Args:
+        export_dir: A folder holding one exported set.
+
+    Returns:
+        The set, with each question's images as absolute paths into ``media/``.
+    """
+    (set_file,) = export_dir.glob("set_*.json")
+    set_json = json.loads(set_file.read_text())
+    question_set = Set(
+        _name=set_json["name"],
+        _description=set_json["description"],
+        _finalAnswerVisibility=VisibilityController(
+            VisibilityStatus(set_json["finalAnswerVisibility"])
+        ),
+        _workedSolutionVisibility=VisibilityController(
+            VisibilityStatus(set_json["workedSolutionVisibility"])
+        ),
+        _structuredTutorialVisibility=VisibilityController(
+            VisibilityStatus(set_json["structuredTutorialVisibility"])
+        ),
+    )
+
+    question_files = sorted(
+        export_dir.glob("question_*.json"),
+        key=lambda path: json.loads(path.read_text())["orderNumber"],
+    )
+    media = sorted((export_dir / "media").glob("*"))
+    for question_file in question_files:
+        question_json = json.loads(question_file.read_text())
+        question_set.questions.append(
+            Question(
+                title=question_json["title"],
+                main_text=question_json["masterContent"],
+                parts=[
+                    Part(
+                        text=part["content"],
+                        worked_solution=(
+                            part["workedSolution"]["content"]
+                            if "workedSolution" in part
+                            else ""
+                        ),
+                    )
+                    for part in question_json["parts"]
+                ],
+                images=[
+                    str(image)
+                    for image in media
+                    if image.name.startswith(f"{question_file.stem}_")
+                ],
+            )
+        )
+    return question_set

tests/test_exports.py

@@ -0,0 +1,99 @@
+"""Round-trips every real Lambda Feedback export through the in2lambda model.
+
+Each folder in ``fixtures/exports`` is loaded into a :class:`~in2lambda.api.set.Set`,
+written back with :meth:`~in2lambda.api.set.Set.to_json` and compared with the
+original. The model holds far less than an export, so the comparison covers what it
+does hold, the file names written, and that the writer emits no key Lambda Feedback
+does not.
+"""
+
+import json
+from pathlib import Path
+
+import pytest
+from conftest import EXPORTS, load_export
+
+from in2lambda.api.set import Set
+
+pytestmark = pytest.mark.parametrize("export_dir", EXPORTS, ids=lambda path: path.name)
+
+
+def _write_back(question_set: Set, tmp_path: Path) -> Path:
+    question_set.to_json(str(tmp_path / "out"))
+    return tmp_path / "out" / question_set._name
+
+
+def _relative_files(directory: Path) -> list[str]:
+    # Finder leaves .DS_Store beside files it has shown; it is not part of an export.
+    return sorted(
+        str(path.relative_to(directory))
+        for path in directory.rglob("*")
+        if path.is_file() and not path.name.startswith(".")
+    )
+
+
+def _modelled(question_set: Set) -> dict:
+    # Visibility controllers have no equality, and image paths differ by where the
+    # set was read from, so compare their values and file names.
+    return {
+        "name": question_set._name,
+        "description": question_set._description,
+        "visibility": [
+            str(question_set._finalAnswerVisibility),
+            str(question_set._workedSolutionVisibility),
+            str(question_set._structuredTutorialVisibility),
+        ],
+        "questions": [
+            (q.title, q.main_text, q.parts, [Path(image).name for image in q.images])
+            for q in question_set.questions
+        ],
+    }
+
+
+def _key_paths(value, path: str = "") -> set[str]:
+    if isinstance(value, dict):
+        paths = set()
+        for key, item in value.items():
+            paths |= {f"{path}.{key}"} | _key_paths(item, f"{path}.{key}")
+        return paths
+    if isinstance(value, list):
+        return set().union(
+            *(_key_paths(item, f"{path}[{i}]") for i, item in enumerate(value))
+        )
+    return set()
+
+
+def _unexported_keys(written: dict, exported: dict) -> list[str]:
+    missing = _key_paths(written) - _key_paths(exported)
+    # Lambda Feedback leaves a part's workedSolution out of its export when the part
+    # has none, but the writer always emits one, so only then may it be absent.
+    for i, part in enumerate(written.get("parts", [])):
+        if not part["workedSolution"]["content"]:
+            prefix = f".parts[{i}].workedSolution"
+            missing = {key for key in missing if not key.startswith(prefix)}
+    return sorted(missing)
+
+
+def test_export_round_trips(export_dir: Path, tmp_path: Path) -> None:
+    """Writing a loaded export reproduces its file names and reloads to the same set."""
+    loaded = load_export(export_dir)
+    assert loaded.questions
+    assert all(question.main_text or question.parts for question in loaded.questions)
+
+    written = _write_back(loaded, tmp_path)
+
+    assert _relative_files(written) == _relative_files(export_dir)
+    assert _modelled(load_export(written)) == _modelled(loaded)
+
+
+def test_written_keys_exist_in_export(export_dir: Path, tmp_path: Path) -> None:
+    """The writer emits no key, at any depth, that Lambda Feedback never exports there."""
+    written = _write_back(load_export(export_dir), tmp_path)
+
+    missing = {}
+    for file in written.glob("*.json"):
+        exported = json.loads((export_dir / file.name).read_text())
+        keys = _unexported_keys(json.loads(file.read_text()), exported)
+        if keys:
+            missing[file.name] = keys
+    assert not missing, missing

tests/test_runner.py

@@ -1,9 +1,9 @@
 """End-to-end tests for :func:`in2lambda.main.runner` across the built-in filters.
 
 Each built-in filter ships a self-contained ``example.tex`` that exercises the
-document structure it targets. These tests run every filter over its own example
-and check both the in-memory :class:`~in2lambda.api.set.Set` and the JSON/ZIP
-files written to disk.
+document structure it targets. These tests find every filter in the package, run
+it over its own example and check both the in-memory :class:`~in2lambda.api.set.Set`
+and the JSON/ZIP files written to disk, so a filter shipped without an example fails.
 """
 
 import json
@@ -12,15 +12,20 @@
 import pytest
 
 from in2lambda.api.set import Set
+from in2lambda.filters import builtin_filters
 from in2lambda.main import runner
 
-BUILTIN_FILTERS = ["PartsSepSol", "PartsOneSol", "PartPartSolSol", "PartSolPartSol"]
 
+def _example(filters_dir: str, filter_name: str) -> str:
+    path = os.path.join(filters_dir, filter_name, "example.tex")
+    assert os.path.isfile(path), f"{filter_name} ships no example.tex to test it with"
+    return path
 
-@pytest.mark.parametrize("filter_name", BUILTIN_FILTERS)
+
+@pytest.mark.parametrize("filter_name", builtin_filters())
 def test_runner_returns_populated_set(filter_name: str, filters_dir: str) -> None:
     """Every filter turns its example into a Set with at least one usable question."""
-    result = runner(os.path.join(filters_dir, filter_name, "example.tex"), filter_name)
+    result = runner(_example(filters_dir, filter_name), filter_name)
 
     assert isinstance(result, Set)
     assert result.questions, f"{filter_name} produced no questions"
@@ -29,17 +34,13 @@ def test_runner_returns_populated_set(filter_name: str, filters_dir: str) -> Non
         assert question.main_text or question.parts
 
 
-@pytest.mark.parametrize("filter_name", BUILTIN_FILTERS)
+@pytest.mark.parametrize("filter_name", builtin_filters())
 def test_runner_writes_importable_json(
     filter_name: str, filters_dir: str, tmp_path
 ) -> None:
     """Passing an output directory produces the Lambda Feedback set/ dir and zip."""
     out_dir = tmp_path / "out"
-    result = runner(
-        os.path.join(filters_dir, filter_name, "example.tex"),
-        filter_name,
-        str(out_dir),
-    )
+    result = runner(_example(filters_dir, filter_name), filter_name, str(out_dir))
 
     set_dir = out_dir / "set"
     assert set_dir.is_dir()