feat: read and write detached expression subtrees

README.md

@@ -251,6 +251,30 @@ relations {
 }
 ```
 
+## Deeply nested expressions
+Protobuf parsers, including the one in the Python `protobuf` library, reject messages nested more
+than 100 levels deep, and each expression level takes several message levels. Substrait works
+around this with [detached expressions](https://substrait.io/serialization/binary_serialization/#bounded-expression-nesting):
+deep subtrees are moved to a table on the `PlanRel` or `ExtendedExpression` and referenced by
+ordinal.
+
+```python
+from substrait.proto import Plan
+from substrait.utils.detached import detach, resolve_detached
+
+data = detach(plan).SerializeToString()  # parses under the default limits
+plan = resolve_detached(Plan.FromString(data))  # ordinary expression trees again
+```
+
+`detach` returns a plan with nothing to detach as is. `infer_plan_schema`,
+`infer_extended_expression_schema`, the plan printer, and the builders resolve detached expressions
+on their own. Functions that take a bare `Rel` or `Expression` cannot, since the table lives on the
+containing message, so resolve the `Plan` or `ExtendedExpression` first.
+
+The builders work on the inlined tree, and the `protobuf` library copies messages passed to a
+constructor through its parser, so building on a detached plan works only while the plan still fits
+the limit once inlined. Detach the finished plan instead, just before serializing it.
+
 ## Produce a Substrait Plan with Ibis
 Let's use an existing Substrait producer, [Ibis](https://ibis-project.org), 
 to provide an example using Python Substrait as the consumer.

src/substrait/builders/extended_expression.py

@@ -26,6 +26,7 @@
     top_level_names,
     type_num_names,
 )
+from substrait.utils.detached import resolve_for_building
 
 # Monotonic source of unique RelCommon.rel_anchor values within a single build.
 # Not reset between builds by default; reset at each top-level materialization
@@ -162,10 +163,13 @@ def resolve_expression(
     whichever build produced it, so the collector re-derives them from the durable
     ``(urn, name)`` identities and the expression is rewritten to match -- the
     expression-level counterpart of ``builders.plan._bind``. Unchanged when the
-    numbering already agrees, as it does for anything this build resolved.
+    numbering already agrees, as it does for anything this build resolved. Its
+    detached subtrees are inlined, since their tables do not travel with the
+    expression into whatever is built from it.
     """
     if not isinstance(expression, stee.ExtendedExpression):
-        return expression(base_schema, registry)
+        return resolve_for_building(expression(base_schema, registry))
+    expression = resolve_for_building(expression)
     collector = current_collector()
     if collector is None:
         return expression
@@ -1077,6 +1081,9 @@ def _inner_rel(query, registry: ExtensionRegistry, base_schema) -> stalg.Rel:
         plan = query(registry) if callable(query) else query
     finally:
         outer_schemas.reset(token)
+    # Only the Rel travels into the Expression.Subquery; the PlanRel's detached
+    # tables stay behind with the plan, so inline its ordinals first.
+    plan = resolve_for_building(plan)
     # Fold the inner plan's extensions into this build before lifting its Rel: only
     # the Rel travels into the Expression.Subquery, so the declarations its function
     # references name stay behind with the plan. A plan built elsewhere therefore

src/substrait/builders/plan.py

@@ -40,6 +40,7 @@
     remap_function_references,
     type_num_names,
 )
+from substrait.utils.detached import resolve_for_building
 from substrait.version import substrait_version
 
 UnboundPlan = Callable[[ExtensionRegistry], stp.Plan]
@@ -73,9 +74,11 @@ def _bind(plan: PlanOrUnbound, registry: ExtensionRegistry) -> stp.Plan:
     resolver writes them).
 
     Every builder binds its inputs through here, so this is the single point at
-    which a foreign plan's anchor space is reconciled with ours.
+    which a foreign plan's anchor space is reconciled with ours. It is also where a
+    foreign plan's detached subtrees are inlined: their ordinals index tables on
+    the plan's own ``PlanRel``, which does not survive being built upon.
     """
-    bound = plan if isinstance(plan, stp.Plan) else plan(registry)
+    bound = resolve_for_building(plan if isinstance(plan, stp.Plan) else plan(registry))
     collector = current_collector()
     if collector is None:
         return bound

src/substrait/type_inference.py

@@ -7,6 +7,7 @@
 import substrait.type_pb2 as stt
 
 from substrait.utils import child_rels, iter_plan_rels, plan_subtrees, rel_anchor_of
+from substrait.utils.detached import resolve_present_tables
 
 
 class _SubtreeScope:
@@ -796,13 +797,21 @@ def infer_expression_type(
             return stt.Type(string=ecv.current_timezone)
         else:
             raise Exception(f"Unknown execution context variable {variable}")
+    elif rex_type == "detached_expression_ordinal":
+        raise ValueError(
+            f"detached expression ordinal {expression.detached_expression_ordinal} "
+            "can only be resolved against its containing PlanRel or "
+            "ExtendedExpression; call substrait.utils.detached.resolve_detached "
+            "on that message first"
+        )
     else:
         raise Exception(f"Unknown rex_type {rex_type}")
 
 
 def infer_extended_expression_schema(
     ee: stee.ExtendedExpression, *, registry=None
 ) -> stt.Type.Struct:
+    ee = resolve_present_tables(ee)
     exprs = [e for e in ee.referred_expr]
 
     types = [
@@ -1394,6 +1403,7 @@ def infer_rel_schema(rel: stalg.Rel, *, registry=None, subtrees=()) -> stt.Type.
 
 
 def infer_plan_schema(plan: stp.Plan, *, registry=None) -> stt.NamedStruct:
+    plan = resolve_present_tables(plan)
     # A Plan carries its shared subtrees in-band as the leading ``rel`` entries of
     # ``relations`` (the query root is the trailing ``root`` entry), so a
     # ReferenceRel anywhere in the tree resolves against them by ordinal. Wrap them