From 53aaf605fc68eef5c019af75f20eaac32a6f8d71 Mon Sep 17 00:00:00 2001 From: John Lambert Date: Thu, 24 Sep 2026 11:02:34 -0400 Subject: [PATCH 1/7] Prune reduplication matches whose copies cannot agree When unapplying an affix process rule that copies a part more than once, Morpher.PruneDisagreeingCopies (off by default) skips matches whose copies cannot unify segment by segment. Synthesis writes every copy from the same input, so such a match never survives synthesis. Copies containing an optional node or modified by the rule are always kept. Co-Authored-By: Claude Opus 5.5 --- .../Morpher.cs | 8 ++ .../AnalysisAffixProcessRule.cs | 3 +- .../AnalysisMorphologicalTransform.cs | 76 ++++++++++++++++++ .../AnalysisRealizationalAffixProcessRule.cs | 3 +- .../CopyAgreementPatternRule.cs | 38 +++++++++ .../MorphologicalOutputAction.cs | 5 ++ .../AffixProcessRuleTests.cs | 28 ++++--- .../CopyAgreementPruneTests.cs | 79 +++++++++++++++++++ 8 files changed, 225 insertions(+), 15 deletions(-) create mode 100644 src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/CopyAgreementPatternRule.cs create mode 100644 tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/CopyAgreementPruneTests.cs diff --git a/src/SIL.Machine.Morphology.HermitCrab/Morpher.cs b/src/SIL.Machine.Morphology.HermitCrab/Morpher.cs index 1cddacd0a..6ebfb6f9b 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/Morpher.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/Morpher.cs @@ -110,6 +110,14 @@ public ITraceManager TraceManager /// public bool AlwaysEnforceFinalTemplates { get; set; } + /// + /// When unapplying a rule that copies a part more than once (reduplication), skip matches whose + /// copies cannot unify segment by segment. Such a match can never survive synthesis, so this removes + /// only doomed analyses; on reduplication-heavy grammars it is the difference between a parse that + /// exhausts memory and one that finishes in seconds. + /// + public bool PruneDisagreeingCopies { get; set; } + /// /// Caps the concurrency used within a single parse or generation -- analysis cascade, /// affix-template unapplication and synthesis alike. A value of 1 runs the work fully diff --git a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisAffixProcessRule.cs b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisAffixProcessRule.cs index 11980053c..2a9f20053 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisAffixProcessRule.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisAffixProcessRule.cs @@ -29,7 +29,8 @@ public AnalysisAffixProcessRule(Morpher morpher, AffixProcessRule rule) foreach (AffixProcessAllomorph allo in rule.Allomorphs) { _rules.Add( - new MultiplePatternRule( + new CopyAgreementPatternRule( + morpher, new AnalysisAffixProcessAllomorphRuleSpec(allo), new MatcherSettings { diff --git a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisMorphologicalTransform.cs b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisMorphologicalTransform.cs index 93e7a0ef6..90948926d 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisMorphologicalTransform.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisMorphologicalTransform.cs @@ -46,6 +46,82 @@ protected IDictionary CapturedParts get { return _capturedParts; } } + internal bool HasRepeatedParts + { + get { return _capturedParts.Values.Any(count => count >= 2); } + } + + /// + /// Synthesis writes every copy of a part from the same input, and anything that later changes one + /// copy is unapplied before this rule, so copies proven to disagree segment by segment cannot lead + /// to a valid analysis. A copy containing an optional node (an unapplied deletion), or a part the + /// rule modifies, cannot be judged and never counts as disagreeing. + /// + internal bool HasDisagreeingCopies(Match match) + { + foreach (KeyValuePair capturedPart in _capturedParts) + { + string partName = capturedPart.Key; + int copyCount = capturedPart.Value; + if (copyCount < 2) + continue; + + if (_modifyFromInfos.ContainsKey(partName)) + continue; + + var copies = new List>(copyCount); + bool partUndecidable = false; + for (int i = 0; i < copyCount; i++) + { + GroupCapture capture = match.GroupCaptures[GetGroupName(partName, i)]; + if (!capture.Success) + { + partUndecidable = true; + break; + } + + List nodes = GetCapturedNodes(match.Input.Shape, capture.Range); + if (nodes.Any(node => node.Annotation.Optional)) + { + partUndecidable = true; + break; + } + + copies.Add(nodes.Where(node => node.Annotation.Type() == HCFeatureSystem.Segment).ToList()); + } + + if (partUndecidable) + continue; + + List firstCopy = copies[0]; + for (int i = 1; i < copies.Count; i++) + { + List otherCopy = copies[i]; + if (firstCopy.Count != otherCopy.Count) + return true; + + for (int j = 0; j < firstCopy.Count; j++) + { + if (!firstCopy[j].Annotation.FeatureStruct.IsUnifiable(otherCopy[j].Annotation.FeatureStruct)) + return true; + } + } + } + + return false; + } + + /// + /// Includes optional nodes the matcher skipped before the capture when they reach the start of the + /// shape, so an unapplied word-initial deletion is still seen. + /// + private static List GetCapturedNodes(Shape shape, Range range) + { + var nodes = new List(MorphologicalOutputAction.SkippedOptionalNodes(shape, range)); + nodes.AddRange(shape.GetNodes(range)); + return nodes; + } + public Pattern Pattern { get { return _pattern; } diff --git a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisRealizationalAffixProcessRule.cs b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisRealizationalAffixProcessRule.cs index 89e30fbe2..a3f3fc8cb 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisRealizationalAffixProcessRule.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisRealizationalAffixProcessRule.cs @@ -22,7 +22,8 @@ public AnalysisRealizationalAffixProcessRule(Morpher morpher, RealizationalAffix foreach (AffixProcessAllomorph allo in rule.Allomorphs) { _rules.Add( - new MultiplePatternRule( + new CopyAgreementPatternRule( + morpher, new AnalysisAffixProcessAllomorphRuleSpec(allo), new MatcherSettings { diff --git a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/CopyAgreementPatternRule.cs b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/CopyAgreementPatternRule.cs new file mode 100644 index 000000000..58d5cf58e --- /dev/null +++ b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/CopyAgreementPatternRule.cs @@ -0,0 +1,38 @@ +using System.Collections.Generic; +using SIL.Machine.Annotations; +using SIL.Machine.Matching; +using SIL.Machine.Rules; + +namespace SIL.Machine.Morphology.HermitCrab.MorphologicalRules +{ + internal sealed class CopyAgreementPatternRule : MultiplePatternRule + { + private readonly Morpher _morpher; + private readonly AnalysisAffixProcessAllomorphRuleSpec _spec; + + public CopyAgreementPatternRule( + Morpher morpher, + AnalysisAffixProcessAllomorphRuleSpec ruleSpec, + MatcherSettings matcherSettings + ) + : base(ruleSpec, matcherSettings) + { + _morpher = morpher; + _spec = ruleSpec; + } + + protected override IEnumerable ApplyImpl(Word input, ShapeNode start) + { + bool prune = _morpher.PruneDisagreeingCopies && _spec.HasRepeatedParts; + var results = new List(); + foreach (Match match in Matcher.AllMatches(input, start)) + { + if (prune && _spec.HasDisagreeingCopies(match)) + continue; + + results.Add(RuleSpec.ApplyRhs(this, match)); + } + return results; + } + } +} diff --git a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/MorphologicalOutputAction.cs b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/MorphologicalOutputAction.cs index 5595a232f..508a458c0 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/MorphologicalOutputAction.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/MorphologicalOutputAction.cs @@ -38,6 +38,11 @@ IDictionary capturedParts public abstract IEnumerable> Apply(Match match, Word output); protected IEnumerable GetSkippedOptionalNodes(Shape shape, Range range) + { + return SkippedOptionalNodes(shape, range); + } + + internal static IEnumerable SkippedOptionalNodes(Shape shape, Range range) { ShapeNode node = range.Start.Prev; var skippedNodes = new List(); diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/AffixProcessRuleTests.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/AffixProcessRuleTests.cs index 329074617..579d997db 100644 --- a/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/AffixProcessRuleTests.cs +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/AffixProcessRuleTests.cs @@ -965,8 +965,9 @@ public void SimulfixRules() AssertMorphsEqual(morpher.ParseWord("sɯɯpu"), "50 SIMUL"); } - [Test] - public void ReduplicationRules() + [TestCase(false)] + [TestCase(true)] + public void ReduplicationRules(bool pruneCopies) { var any = FeatureStruct.New().Symbol(HCFeatureSystem.Segment).Value; var cons = FeatureStruct @@ -1015,7 +1016,7 @@ public void ReduplicationRules() ); Morphophonemic.MorphologicalRules.Add(redup); - var morpher = new Morpher(TraceManager, Language); + var morpher = new Morpher(TraceManager, Language) { PruneDisagreeingCopies = pruneCopies }; AssertMorphsEqual(morpher.ParseWord("sasag"), "RED 32"); var voicing = new RewriteRule @@ -1033,7 +1034,7 @@ public void ReduplicationRules() ); Allophonic.PhonologicalRules.Add(voicing); - morpher = new Morpher(TraceManager, Language); + morpher = new Morpher(TraceManager, Language) { PruneDisagreeingCopies = pruneCopies }; AssertMorphsEqual(morpher.ParseWord("sazag"), "RED 32"); var affrication = new RewriteRule @@ -1050,7 +1051,7 @@ public void ReduplicationRules() ); Allophonic.PhonologicalRules.Add(affrication); - morpher = new Morpher(TraceManager, Language); + morpher = new Morpher(TraceManager, Language) { PruneDisagreeingCopies = pruneCopies }; AssertMorphsEqual(morpher.ParseWord("tsazag"), "RED 32"); redup.Allomorphs.Clear(); @@ -1066,7 +1067,7 @@ public void ReduplicationRules() } ); - morpher = new Morpher(TraceManager, Language); + morpher = new Morpher(TraceManager, Language) { PruneDisagreeingCopies = pruneCopies }; AssertMorphsEqual(morpher.ParseWord("tsagag"), "32 RED"); Allophonic.PhonologicalRules.Clear(); @@ -1084,7 +1085,7 @@ public void ReduplicationRules() } ); - morpher = new Morpher(TraceManager, Language); + morpher = new Morpher(TraceManager, Language) { PruneDisagreeingCopies = pruneCopies }; AssertMorphsEqual(morpher.ParseWord("sagsag"), "32 RED"); AssertMorphsEqual(morpher.ParseWord("sasibudbud"), "38 RED"); @@ -1115,7 +1116,7 @@ public void ReduplicationRules() ); Allophonic.PhonologicalRules.Add(gDelete); - morpher = new Morpher(TraceManager, Language); + morpher = new Morpher(TraceManager, Language) { PruneDisagreeingCopies = pruneCopies }; AssertMorphsEqual(morpher.ParseWord("saag"), "32 RED"); gDelete.Subrules.Clear(); @@ -1126,7 +1127,7 @@ public void ReduplicationRules() } ); - morpher = new Morpher(TraceManager, Language); + morpher = new Morpher(TraceManager, Language) { PruneDisagreeingCopies = pruneCopies }; AssertMorphsEqual(morpher.ParseWord("saga"), "32 RED"); Allophonic.PhonologicalRules.Clear(); @@ -1152,7 +1153,7 @@ public void ReduplicationRules() } ); - morpher = new Morpher(TraceManager, Language); + morpher = new Morpher(TraceManager, Language) { PruneDisagreeingCopies = pruneCopies }; AssertMorphsEqual(morpher.ParseWord("buiuib"), "30 RED", "31 RED"); } @@ -1900,8 +1901,9 @@ public void SubsumedAffix() AssertMorphsEqual(morpher.ParseWord("bubib"), "42 PRES", "43 PRES"); } - [Test] - public void ModifyFromInputRules() + [TestCase(false)] + [TestCase(true)] + public void ModifyFromInputRules(bool pruneCopies) { var any = FeatureStruct.New().Symbol(HCFeatureSystem.Segment).Value; var vowel = FeatureStruct @@ -1941,7 +1943,7 @@ public void ModifyFromInputRules() ); Morphophonemic.MorphologicalRules.Add(sSuffix); - var morpher = new Morpher(TraceManager, Language); + var morpher = new Morpher(TraceManager, Language) { PruneDisagreeingCopies = pruneCopies }; AssertMorphsEqual(morpher.ParseWord("puso"), "52 PL"); } diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/CopyAgreementPruneTests.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/CopyAgreementPruneTests.cs new file mode 100644 index 000000000..62206d6ca --- /dev/null +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/CopyAgreementPruneTests.cs @@ -0,0 +1,79 @@ +using NUnit.Framework; +using SIL.Machine.Annotations; +using SIL.Machine.FeatureModel; +using SIL.Machine.Matching; + +namespace SIL.Machine.Morphology.HermitCrab.MorphologicalRules; + +[TestFixture] +public class CopyAgreementPruneTests : HermitCrabTestBase +{ + [TestCase(false, 3)] + [TestCase(true, 1)] + public void FullCopyKeepsOnlyTheSplitWhoseCopiesAgree(bool pruneCopies, int expectedOutputs) + { + (string first, string second) = FindIncompatibleSegments(); + var anySegment = FeatureStruct.New().Symbol(HCFeatureSystem.Segment).Value; + AffixProcessRule rule = CreateCopyRule( + Pattern.New("copy").Annotation(anySegment).OneOrMore.Value + ); + var morpher = new Morpher(TraceManager, Language) { PruneDisagreeingCopies = pruneCopies }; + + List outputs = new AnalysisAffixProcessRule(morpher, rule) + .Apply(CreateInput(first + second + first + second)) + .ToList(); + + Assert.That(outputs.Count, Is.EqualTo(expectedOutputs)); + if (pruneCopies) + Assert.That(outputs.Single().Shape.Count, Is.EqualTo(2)); + } + + [TestCase(false, 1)] + [TestCase(true, 0)] + public void DisagreeingCopiesAreRemovedOnlyWhenPruning(bool pruneCopies, int expectedOutputs) + { + (string first, string second) = FindIncompatibleSegments(); + var anySegment = FeatureStruct.New().Symbol(HCFeatureSystem.Segment).Value; + AffixProcessRule rule = CreateCopyRule(Pattern.New("copy").Annotation(anySegment).Value); + var morpher = new Morpher(TraceManager, Language) { PruneDisagreeingCopies = pruneCopies }; + + List outputs = new AnalysisAffixProcessRule(morpher, rule).Apply(CreateInput(first + second)).ToList(); + + Assert.That(outputs.Count, Is.EqualTo(expectedOutputs)); + } + + private static AffixProcessRule CreateCopyRule(Pattern part) + { + var rule = new AffixProcessRule + { + Name = "full_copy", + Gloss = "RED", + RequiredSyntacticFeatureStruct = FeatureStruct.New().Value, + OutSyntacticFeatureStruct = FeatureStruct.New().Value, + }; + rule.Allomorphs.Add( + new AffixProcessAllomorph { Lhs = { part }, Rhs = { new CopyFromInput("copy"), new CopyFromInput("copy") } } + ); + return rule; + } + + private (string First, string Second) FindIncompatibleSegments() + { + CharacterDefinition[] segments = Table3 + .Where(definition => definition.Type == HCFeatureSystem.Segment) + .ToArray(); + var pair = segments + .SelectMany( + (first, index) => segments.Skip(index + 1).Select(second => new { First = first, Second = second }) + ) + .First(candidate => !candidate.First.FeatureStruct.IsUnifiable(candidate.Second.FeatureStruct)); + return (pair.First.Representations.First(), pair.Second.Representations.First()); + } + + private Word CreateInput(string representation) + { + var input = new Word(Surface, Table3.Segment(representation)) { AnalysisScope = new AnalysisScope() }; + input.Freeze(); + return input; + } +} From 1294977cdb3bd62ada3e731bb42d01d1b9a71f79 Mon Sep 17 00:00:00 2001 From: John Lambert Date: Thu, 24 Sep 2026 13:25:40 -0400 Subject: [PATCH 2/7] Pin copy pruning against a cap-limited deletion in one copy A self-feeding deletion strips two segments from the second copy only; with DeletionReapplications 0 analysis restores one. Forward generation defines the expected parse, and pruning must return the unpruned result. Co-Authored-By: Claude Opus 5.5 --- .../CopyAgreementPruneTests.cs | 68 +++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/CopyAgreementPruneTests.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/CopyAgreementPruneTests.cs index 62206d6ca..bfc421488 100644 --- a/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/CopyAgreementPruneTests.cs +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/CopyAgreementPruneTests.cs @@ -2,6 +2,7 @@ using SIL.Machine.Annotations; using SIL.Machine.FeatureModel; using SIL.Machine.Matching; +using SIL.Machine.Morphology.HermitCrab.PhonologicalRules; namespace SIL.Machine.Morphology.HermitCrab.MorphologicalRules; @@ -42,6 +43,73 @@ public void DisagreeingCopiesAreRemovedOnlyWhenPruning(bool pruneCopies, int exp Assert.That(outputs.Count, Is.EqualTo(expectedOutputs)); } + /// + /// The prune's weakest point: a self-feeding deletion strips two segments from the second copy only, + /// and = 0 lets analysis restore just one of them. + /// + [TestCase(0)] + [TestCase(1)] + public void CapLimitedDeletionInOneCopyParsesTheSameWithAndWithoutPruning(int deletionReapplications) + { + var any = FeatureStruct.New().Symbol(HCFeatureSystem.Segment).Value; + var vowel = FeatureStruct + .New(Language.PhonologicalFeatureSystem) + .Symbol(HCFeatureSystem.Segment) + .Symbol("voc+") + .Value; + AffixProcessRule redup = CreateCopyRule(Pattern.New("copy").Annotation(any).OneOrMore.Value); + redup.RequiredSyntacticFeatureStruct = FeatureStruct.New(Language.SyntacticFeatureSystem).Symbol("V").Value; + Morphophonemic.MorphologicalRules.Add(redup); + + var gDelete = new RewriteRule + { + Name = "g_delete", + Lhs = Pattern.New().Annotation(Character(Table1, "g")).Value, + }; + gDelete.Subrules.Add( + new RewriteSubrule { LeftEnvironment = Pattern.New().Annotation(vowel).Value } + ); + Allophonic.PhonologicalRules.Add(gDelete); + + LexEntry root = AddEntry( + "GG", + FeatureStruct.New(Language.SyntacticFeatureSystem).Symbol("V").Value, + Morphophonemic, + "ggasa" + ); + try + { + var generator = new Morpher(TraceManager, Language); + Assert.That( + generator.GenerateWords(root, new Morpheme[] { redup }, FeatureStruct.New().Value), + Is.EquivalentTo(new[] { "ggasaasa" }), + "forward derivation defines the expected parse" + ); + + string[] Parse(bool pruneCopies) + { + var morpher = new Morpher(TraceManager, Language) + { + DeletionReapplications = deletionReapplications, + PruneDisagreeingCopies = pruneCopies, + }; + return morpher + .ParseWord("ggasaasa") + .Select(w => string.Join(" ", w.AllomorphsInMorphOrder.Select(a => a.Morpheme.Gloss))) + .OrderBy(s => s, StringComparer.Ordinal) + .ToArray(); + } + + string[] unpruned = Parse(false); + Assert.That(unpruned, Does.Contain("GG RED"), "the unpruned engine finds the generated analysis"); + Assert.That(Parse(true), Is.EqualTo(unpruned)); + } + finally + { + Morphophonemic.Entries.Remove(root); + } + } + private static AffixProcessRule CreateCopyRule(Pattern part) { var rule = new AffixProcessRule From c91c40c74c4ea38d9ad045b13117384ea71c5e20 Mon Sep 17 00:00:00 2001 From: John Lambert Date: Thu, 24 Sep 2026 21:46:20 -0400 Subject: [PATCH 3/7] Turn copy-agreement pruning on by default Co-Authored-By: Claude Opus 5.5 --- src/SIL.Machine.Morphology.HermitCrab/Morpher.cs | 4 ++-- .../MorphologicalRules/CopyAgreementPruneTests.cs | 6 ++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/SIL.Machine.Morphology.HermitCrab/Morpher.cs b/src/SIL.Machine.Morphology.HermitCrab/Morpher.cs index 6ebfb6f9b..e44e60687 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/Morpher.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/Morpher.cs @@ -61,6 +61,7 @@ public Morpher(ITraceManager traceManager, Language lang, int maxDegreeOfParalle MaxStemCount = 2; MaxAlternatives = 0; MergeEquivalentAnalyses = true; + PruneDisagreeingCopies = true; LexEntrySelector = entry => true; RuleSelector = rule => true; @@ -113,8 +114,7 @@ public ITraceManager TraceManager /// /// When unapplying a rule that copies a part more than once (reduplication), skip matches whose /// copies cannot unify segment by segment. Such a match can never survive synthesis, so this removes - /// only doomed analyses; on reduplication-heavy grammars it is the difference between a parse that - /// exhausts memory and one that finishes in seconds. + /// only doomed analyses. On by default. /// public bool PruneDisagreeingCopies { get; set; } diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/CopyAgreementPruneTests.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/CopyAgreementPruneTests.cs index bfc421488..5f9a5d74c 100644 --- a/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/CopyAgreementPruneTests.cs +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/CopyAgreementPruneTests.cs @@ -9,6 +9,12 @@ namespace SIL.Machine.Morphology.HermitCrab.MorphologicalRules; [TestFixture] public class CopyAgreementPruneTests : HermitCrabTestBase { + [Test] + public void PruningIsOnByDefault() + { + Assert.That(new Morpher(TraceManager, Language).PruneDisagreeingCopies, Is.True); + } + [TestCase(false, 3)] [TestCase(true, 1)] public void FullCopyKeepsOnlyTheSplitWhoseCopiesAgree(bool pruneCopies, int expectedOutputs) From 21c662be24d5571a20f1e76790ef292e2639f6ac Mon Sep 17 00:00:00 2001 From: John Lambert Date: Fri, 25 Sep 2026 18:41:03 -0400 Subject: [PATCH 4/7] Compute HasRepeatedParts once per copy-agreement rule Rules that copy no part twice no longer rescan their captured parts on every unapplication. Mirrors PanGloss 77aca87f, which measured 0.62-0.64x of the prior time on Aweti with identical analyses. Co-Authored-By: Claude Opus 5.5 --- .../MorphologicalRules/CopyAgreementPatternRule.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/CopyAgreementPatternRule.cs b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/CopyAgreementPatternRule.cs index 58d5cf58e..78abce1ff 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/CopyAgreementPatternRule.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/CopyAgreementPatternRule.cs @@ -9,6 +9,7 @@ internal sealed class CopyAgreementPatternRule : MultiplePatternRule matcherSettings { _morpher = morpher; _spec = ruleSpec; + _hasRepeatedParts = ruleSpec.HasRepeatedParts; } protected override IEnumerable ApplyImpl(Word input, ShapeNode start) { - bool prune = _morpher.PruneDisagreeingCopies && _spec.HasRepeatedParts; + bool prune = _morpher.PruneDisagreeingCopies && _hasRepeatedParts; var results = new List(); foreach (Match match in Matcher.AllMatches(input, start)) { From 64337cd3519cb055ff0bfc49cb1bb367fc4a35fb Mon Sep 17 00:00:00 2001 From: John Lambert Date: Wed, 30 Sep 2026 15:20:24 -0400 Subject: [PATCH 5/7] Address review of copy pruning - Prune compounding rules as well as affix-process rules. - Compare every pair of copies, since unifiability is not transitive. - Skip pruning while tracing, as MergeEquivalentAnalyses does, so traces still show each doomed split failing in synthesis. - Collapse the SkippedOptionalNodes forwarder into a protected internal static GetSkippedOptionalNodes; split HasDisagreeingCopies into helpers. - Rename to DisagreeingCopiesPruningRule / PruneDisagreeingCopiesTests. - Add tests: realizational and compounding paths, three copies, parallelism and memo settings, partial morphemes with and without AlwaysEnforceFinalTemplates, DeletionReapplications = 2, tracing, and a counterbleeding opacity case pinned against forward generation. Co-Authored-By: Claude Opus 5.5 --- .../Morpher.cs | 7 +- .../AnalysisAffixProcessRule.cs | 2 +- .../AnalysisCompoundingRule.cs | 3 +- .../AnalysisMorphologicalTransform.cs | 93 ++--- .../AnalysisRealizationalAffixProcessRule.cs | 2 +- ...ule.cs => DisagreeingCopiesPruningRule.cs} | 10 +- .../MorphologicalOutputAction.cs | 7 +- .../CopyAgreementPruneTests.cs | 153 -------- .../PruneDisagreeingCopiesTests.cs | 367 ++++++++++++++++++ 9 files changed, 428 insertions(+), 216 deletions(-) rename src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/{CopyAgreementPatternRule.cs => DisagreeingCopiesPruningRule.cs} (77%) delete mode 100644 tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/CopyAgreementPruneTests.cs create mode 100644 tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/PruneDisagreeingCopiesTests.cs diff --git a/src/SIL.Machine.Morphology.HermitCrab/Morpher.cs b/src/SIL.Machine.Morphology.HermitCrab/Morpher.cs index e44e60687..1c7fea8fd 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/Morpher.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/Morpher.cs @@ -112,9 +112,10 @@ public ITraceManager TraceManager public bool AlwaysEnforceFinalTemplates { get; set; } /// - /// When unapplying a rule that copies a part more than once (reduplication), skip matches whose - /// copies cannot unify segment by segment. Such a match can never survive synthesis, so this removes - /// only doomed analyses. On by default. + /// When unapplying an affix-process or compounding rule that copies a part more than once + /// (reduplication), skip matches whose copies cannot unify segment by segment. Such a match can never + /// survive synthesis, so this removes only doomed analyses. On by default; ignored while tracing, so + /// a trace still shows each such match failing in synthesis. /// public bool PruneDisagreeingCopies { get; set; } diff --git a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisAffixProcessRule.cs b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisAffixProcessRule.cs index 2a9f20053..59014596d 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisAffixProcessRule.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisAffixProcessRule.cs @@ -29,7 +29,7 @@ public AnalysisAffixProcessRule(Morpher morpher, AffixProcessRule rule) foreach (AffixProcessAllomorph allo in rule.Allomorphs) { _rules.Add( - new CopyAgreementPatternRule( + new DisagreeingCopiesPruningRule( morpher, new AnalysisAffixProcessAllomorphRuleSpec(allo), new MatcherSettings diff --git a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisCompoundingRule.cs b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisCompoundingRule.cs index f77bf96e4..6dbf11b9f 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisCompoundingRule.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisCompoundingRule.cs @@ -28,7 +28,8 @@ public AnalysisCompoundingRule(Morpher morpher, CompoundingRule rule) foreach (CompoundingSubrule sr in rule.Subrules) { _rules.Add( - new MultiplePatternRule( + new DisagreeingCopiesPruningRule( + morpher, new AnalysisCompoundingSubruleRuleSpec(sr), new MatcherSettings { diff --git a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisMorphologicalTransform.cs b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisMorphologicalTransform.cs index 90948926d..c3244d021 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisMorphologicalTransform.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisMorphologicalTransform.cs @@ -54,74 +54,75 @@ internal bool HasRepeatedParts /// /// Synthesis writes every copy of a part from the same input, and anything that later changes one /// copy is unapplied before this rule, so copies proven to disagree segment by segment cannot lead - /// to a valid analysis. A copy containing an optional node (an unapplied deletion), or a part the - /// rule modifies, cannot be judged and never counts as disagreeing. + /// to a valid analysis. A part the rule modifies, or a copy containing an unapplied deletion, is + /// never reported as disagreeing. /// internal bool HasDisagreeingCopies(Match match) { foreach (KeyValuePair capturedPart in _capturedParts) { - string partName = capturedPart.Key; - int copyCount = capturedPart.Value; - if (copyCount < 2) + if (capturedPart.Value < 2 || _modifyFromInfos.ContainsKey(capturedPart.Key)) continue; - if (_modifyFromInfos.ContainsKey(partName)) - continue; - - var copies = new List>(copyCount); - bool partUndecidable = false; - for (int i = 0; i < copyCount; i++) + if ( + TryGetSegmentCopies(match, capturedPart.Key, capturedPart.Value, out List> copies) + && AnyCopiesDisagree(copies) + ) { - GroupCapture capture = match.GroupCaptures[GetGroupName(partName, i)]; - if (!capture.Success) - { - partUndecidable = true; - break; - } - - List nodes = GetCapturedNodes(match.Input.Shape, capture.Range); - if (nodes.Any(node => node.Annotation.Optional)) - { - partUndecidable = true; - break; - } - - copies.Add(nodes.Where(node => node.Annotation.Type() == HCFeatureSystem.Segment).ToList()); + return true; } + } - if (partUndecidable) - continue; + return false; + } + + private static bool TryGetSegmentCopies( + Match match, + string partName, + int copyCount, + out List> copies + ) + { + copies = new List>(copyCount); + for (int i = 0; i < copyCount; i++) + { + GroupCapture capture = match.GroupCaptures[GetGroupName(partName, i)]; + if (!capture.Success) + return false; + + // an unapplied word-initial deletion sits before the capture, not inside it + List nodes = MorphologicalOutputAction + .GetSkippedOptionalNodes(match.Input.Shape, capture.Range) + .Concat(match.Input.Shape.GetNodes(capture.Range)) + .ToList(); + if (nodes.Any(node => node.Annotation.Optional)) + return false; + + copies.Add(nodes.Where(node => node.Annotation.Type() == HCFeatureSystem.Segment).ToList()); + } + return true; + } - List firstCopy = copies[0]; - for (int i = 1; i < copies.Count; i++) + // unifiability is not transitive, so every pair is compared, not just each copy against the first + private static bool AnyCopiesDisagree(List> copies) + { + for (int i = 0; i < copies.Count; i++) + { + for (int j = i + 1; j < copies.Count; j++) { - List otherCopy = copies[i]; - if (firstCopy.Count != otherCopy.Count) + if (copies[i].Count != copies[j].Count) return true; - for (int j = 0; j < firstCopy.Count; j++) + for (int k = 0; k < copies[i].Count; k++) { - if (!firstCopy[j].Annotation.FeatureStruct.IsUnifiable(otherCopy[j].Annotation.FeatureStruct)) + if (!copies[i][k].Annotation.FeatureStruct.IsUnifiable(copies[j][k].Annotation.FeatureStruct)) return true; } } } - return false; } - /// - /// Includes optional nodes the matcher skipped before the capture when they reach the start of the - /// shape, so an unapplied word-initial deletion is still seen. - /// - private static List GetCapturedNodes(Shape shape, Range range) - { - var nodes = new List(MorphologicalOutputAction.SkippedOptionalNodes(shape, range)); - nodes.AddRange(shape.GetNodes(range)); - return nodes; - } - public Pattern Pattern { get { return _pattern; } diff --git a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisRealizationalAffixProcessRule.cs b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisRealizationalAffixProcessRule.cs index a3f3fc8cb..9862db6bf 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisRealizationalAffixProcessRule.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisRealizationalAffixProcessRule.cs @@ -22,7 +22,7 @@ public AnalysisRealizationalAffixProcessRule(Morpher morpher, RealizationalAffix foreach (AffixProcessAllomorph allo in rule.Allomorphs) { _rules.Add( - new CopyAgreementPatternRule( + new DisagreeingCopiesPruningRule( morpher, new AnalysisAffixProcessAllomorphRuleSpec(allo), new MatcherSettings diff --git a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/CopyAgreementPatternRule.cs b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/DisagreeingCopiesPruningRule.cs similarity index 77% rename from src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/CopyAgreementPatternRule.cs rename to src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/DisagreeingCopiesPruningRule.cs index 78abce1ff..e4431ab44 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/CopyAgreementPatternRule.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/DisagreeingCopiesPruningRule.cs @@ -5,15 +5,15 @@ namespace SIL.Machine.Morphology.HermitCrab.MorphologicalRules { - internal sealed class CopyAgreementPatternRule : MultiplePatternRule + internal sealed class DisagreeingCopiesPruningRule : MultiplePatternRule { private readonly Morpher _morpher; - private readonly AnalysisAffixProcessAllomorphRuleSpec _spec; + private readonly AnalysisMorphologicalTransformRuleSpec _spec; private readonly bool _hasRepeatedParts; - public CopyAgreementPatternRule( + public DisagreeingCopiesPruningRule( Morpher morpher, - AnalysisAffixProcessAllomorphRuleSpec ruleSpec, + AnalysisMorphologicalTransformRuleSpec ruleSpec, MatcherSettings matcherSettings ) : base(ruleSpec, matcherSettings) @@ -25,7 +25,7 @@ MatcherSettings matcherSettings protected override IEnumerable ApplyImpl(Word input, ShapeNode start) { - bool prune = _morpher.PruneDisagreeingCopies && _hasRepeatedParts; + bool prune = _morpher.PruneDisagreeingCopies && _hasRepeatedParts && !_morpher.TraceManager.IsTracing; var results = new List(); foreach (Match match in Matcher.AllMatches(input, start)) { diff --git a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/MorphologicalOutputAction.cs b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/MorphologicalOutputAction.cs index 508a458c0..53ed1ddd0 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/MorphologicalOutputAction.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/MorphologicalOutputAction.cs @@ -37,12 +37,7 @@ IDictionary capturedParts /// The output word synthesis. public abstract IEnumerable> Apply(Match match, Word output); - protected IEnumerable GetSkippedOptionalNodes(Shape shape, Range range) - { - return SkippedOptionalNodes(shape, range); - } - - internal static IEnumerable SkippedOptionalNodes(Shape shape, Range range) + protected internal static IEnumerable GetSkippedOptionalNodes(Shape shape, Range range) { ShapeNode node = range.Start.Prev; var skippedNodes = new List(); diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/CopyAgreementPruneTests.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/CopyAgreementPruneTests.cs deleted file mode 100644 index 5f9a5d74c..000000000 --- a/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/CopyAgreementPruneTests.cs +++ /dev/null @@ -1,153 +0,0 @@ -using NUnit.Framework; -using SIL.Machine.Annotations; -using SIL.Machine.FeatureModel; -using SIL.Machine.Matching; -using SIL.Machine.Morphology.HermitCrab.PhonologicalRules; - -namespace SIL.Machine.Morphology.HermitCrab.MorphologicalRules; - -[TestFixture] -public class CopyAgreementPruneTests : HermitCrabTestBase -{ - [Test] - public void PruningIsOnByDefault() - { - Assert.That(new Morpher(TraceManager, Language).PruneDisagreeingCopies, Is.True); - } - - [TestCase(false, 3)] - [TestCase(true, 1)] - public void FullCopyKeepsOnlyTheSplitWhoseCopiesAgree(bool pruneCopies, int expectedOutputs) - { - (string first, string second) = FindIncompatibleSegments(); - var anySegment = FeatureStruct.New().Symbol(HCFeatureSystem.Segment).Value; - AffixProcessRule rule = CreateCopyRule( - Pattern.New("copy").Annotation(anySegment).OneOrMore.Value - ); - var morpher = new Morpher(TraceManager, Language) { PruneDisagreeingCopies = pruneCopies }; - - List outputs = new AnalysisAffixProcessRule(morpher, rule) - .Apply(CreateInput(first + second + first + second)) - .ToList(); - - Assert.That(outputs.Count, Is.EqualTo(expectedOutputs)); - if (pruneCopies) - Assert.That(outputs.Single().Shape.Count, Is.EqualTo(2)); - } - - [TestCase(false, 1)] - [TestCase(true, 0)] - public void DisagreeingCopiesAreRemovedOnlyWhenPruning(bool pruneCopies, int expectedOutputs) - { - (string first, string second) = FindIncompatibleSegments(); - var anySegment = FeatureStruct.New().Symbol(HCFeatureSystem.Segment).Value; - AffixProcessRule rule = CreateCopyRule(Pattern.New("copy").Annotation(anySegment).Value); - var morpher = new Morpher(TraceManager, Language) { PruneDisagreeingCopies = pruneCopies }; - - List outputs = new AnalysisAffixProcessRule(morpher, rule).Apply(CreateInput(first + second)).ToList(); - - Assert.That(outputs.Count, Is.EqualTo(expectedOutputs)); - } - - /// - /// The prune's weakest point: a self-feeding deletion strips two segments from the second copy only, - /// and = 0 lets analysis restore just one of them. - /// - [TestCase(0)] - [TestCase(1)] - public void CapLimitedDeletionInOneCopyParsesTheSameWithAndWithoutPruning(int deletionReapplications) - { - var any = FeatureStruct.New().Symbol(HCFeatureSystem.Segment).Value; - var vowel = FeatureStruct - .New(Language.PhonologicalFeatureSystem) - .Symbol(HCFeatureSystem.Segment) - .Symbol("voc+") - .Value; - AffixProcessRule redup = CreateCopyRule(Pattern.New("copy").Annotation(any).OneOrMore.Value); - redup.RequiredSyntacticFeatureStruct = FeatureStruct.New(Language.SyntacticFeatureSystem).Symbol("V").Value; - Morphophonemic.MorphologicalRules.Add(redup); - - var gDelete = new RewriteRule - { - Name = "g_delete", - Lhs = Pattern.New().Annotation(Character(Table1, "g")).Value, - }; - gDelete.Subrules.Add( - new RewriteSubrule { LeftEnvironment = Pattern.New().Annotation(vowel).Value } - ); - Allophonic.PhonologicalRules.Add(gDelete); - - LexEntry root = AddEntry( - "GG", - FeatureStruct.New(Language.SyntacticFeatureSystem).Symbol("V").Value, - Morphophonemic, - "ggasa" - ); - try - { - var generator = new Morpher(TraceManager, Language); - Assert.That( - generator.GenerateWords(root, new Morpheme[] { redup }, FeatureStruct.New().Value), - Is.EquivalentTo(new[] { "ggasaasa" }), - "forward derivation defines the expected parse" - ); - - string[] Parse(bool pruneCopies) - { - var morpher = new Morpher(TraceManager, Language) - { - DeletionReapplications = deletionReapplications, - PruneDisagreeingCopies = pruneCopies, - }; - return morpher - .ParseWord("ggasaasa") - .Select(w => string.Join(" ", w.AllomorphsInMorphOrder.Select(a => a.Morpheme.Gloss))) - .OrderBy(s => s, StringComparer.Ordinal) - .ToArray(); - } - - string[] unpruned = Parse(false); - Assert.That(unpruned, Does.Contain("GG RED"), "the unpruned engine finds the generated analysis"); - Assert.That(Parse(true), Is.EqualTo(unpruned)); - } - finally - { - Morphophonemic.Entries.Remove(root); - } - } - - private static AffixProcessRule CreateCopyRule(Pattern part) - { - var rule = new AffixProcessRule - { - Name = "full_copy", - Gloss = "RED", - RequiredSyntacticFeatureStruct = FeatureStruct.New().Value, - OutSyntacticFeatureStruct = FeatureStruct.New().Value, - }; - rule.Allomorphs.Add( - new AffixProcessAllomorph { Lhs = { part }, Rhs = { new CopyFromInput("copy"), new CopyFromInput("copy") } } - ); - return rule; - } - - private (string First, string Second) FindIncompatibleSegments() - { - CharacterDefinition[] segments = Table3 - .Where(definition => definition.Type == HCFeatureSystem.Segment) - .ToArray(); - var pair = segments - .SelectMany( - (first, index) => segments.Skip(index + 1).Select(second => new { First = first, Second = second }) - ) - .First(candidate => !candidate.First.FeatureStruct.IsUnifiable(candidate.Second.FeatureStruct)); - return (pair.First.Representations.First(), pair.Second.Representations.First()); - } - - private Word CreateInput(string representation) - { - var input = new Word(Surface, Table3.Segment(representation)) { AnalysisScope = new AnalysisScope() }; - input.Freeze(); - return input; - } -} diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/PruneDisagreeingCopiesTests.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/PruneDisagreeingCopiesTests.cs new file mode 100644 index 000000000..2d2b3aaa0 --- /dev/null +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/PruneDisagreeingCopiesTests.cs @@ -0,0 +1,367 @@ +using NUnit.Framework; +using SIL.Machine.Annotations; +using SIL.Machine.FeatureModel; +using SIL.Machine.Matching; +using SIL.Machine.Morphology.HermitCrab.PhonologicalRules; + +namespace SIL.Machine.Morphology.HermitCrab.MorphologicalRules; + +public class PruneDisagreeingCopiesTests : HermitCrabTestBase +{ + private static readonly FeatureStruct AnySegment = FeatureStruct.New().Symbol(HCFeatureSystem.Segment).Value; + + [Test] + public void PruningIsOnByDefault() + { + Assert.That(new Morpher(TraceManager, Language).PruneDisagreeingCopies, Is.True); + } + + [TestCase(false, 3)] + [TestCase(true, 1)] + public void FullCopyKeepsOnlyTheSplitWhoseCopiesAgree(bool pruneCopies, int expectedOutputs) + { + (string first, string second) = FindIncompatibleSegments(); + AffixProcessRule rule = CreateCopyRule(FullPart()); + + List outputs = new AnalysisAffixProcessRule(CreateMorpher(pruneCopies), rule) + .Apply(CreateInput(first + second + first + second)) + .ToList(); + + Assert.That(outputs.Count, Is.EqualTo(expectedOutputs)); + if (pruneCopies) + Assert.That(outputs.Single().Shape.Count, Is.EqualTo(2)); + } + + [TestCase(false, 1)] + [TestCase(true, 0)] + public void DisagreeingCopiesAreRemovedOnlyWhenPruning(bool pruneCopies, int expectedOutputs) + { + (string first, string second) = FindIncompatibleSegments(); + AffixProcessRule rule = CreateCopyRule(SingleSegmentPart()); + + List outputs = new AnalysisAffixProcessRule(CreateMorpher(pruneCopies), rule) + .Apply(CreateInput(first + second)) + .ToList(); + + Assert.That(outputs.Count, Is.EqualTo(expectedOutputs)); + } + + [Test] + public void TracingKeepsDisagreeingCopies() + { + (string first, string second) = FindIncompatibleSegments(); + AffixProcessRule rule = CreateCopyRule(SingleSegmentPart()); + var input = new Word(Surface, Table3.Segment(first + second)) { AnalysisScope = new AnalysisScope() }; + TraceManager.AnalyzeWord(Language, input); + input.Freeze(); + TraceManager.IsTracing = true; + try + { + List outputs = new AnalysisAffixProcessRule(CreateMorpher(true), rule).Apply(input).ToList(); + + Assert.That(outputs.Count, Is.EqualTo(1)); + } + finally + { + TraceManager.IsTracing = false; + } + } + + [TestCase(false, 1)] + [TestCase(true, 0)] + public void EveryPairOfCopiesIsCompared(bool pruneCopies, int expectedOutputs) + { + // the first copy is underspecified, as unapplied phonology can leave it, so it unifies with both others + (string first, string second) = FindIncompatibleSegments(); + AffixProcessRule rule = CreateCopyRule(SingleSegmentPart(), copyCount: 3); + var input = new Word(Surface, Table3.Segment(first + first + second)) { AnalysisScope = new AnalysisScope() }; + input.Shape.First.Annotation.FeatureStruct = AnySegment.Clone(); + input.Freeze(); + + List outputs = new AnalysisAffixProcessRule(CreateMorpher(pruneCopies), rule).Apply(input).ToList(); + + Assert.That(outputs.Count, Is.EqualTo(expectedOutputs)); + } + + [TestCase(false, 3)] + [TestCase(true, 1)] + public void RealizationalFullCopyKeepsOnlyTheSplitWhoseCopiesAgree(bool pruneCopies, int expectedOutputs) + { + (string first, string second) = FindIncompatibleSegments(); + var rule = new RealizationalAffixProcessRule { Name = "real_copy", Gloss = "RED" }; + rule.Allomorphs.Add(CreateCopyAllomorph(FullPart(), copyCount: 2)); + + List outputs = new AnalysisRealizationalAffixProcessRule(CreateMorpher(pruneCopies), rule) + .Apply(CreateInput(first + second + first + second)) + .ToList(); + + Assert.That(outputs.Count, Is.EqualTo(expectedOutputs)); + } + + [TestCase(false, 3)] + [TestCase(true, 1)] + public void CompoundHeadCopiedTwiceKeepsOnlyTheSplitWhoseCopiesAgree(bool pruneCopies, int expectedOutputs) + { + var crule = new CompoundingRule { Name = "head_copy_compound" }; + crule.Subrules.Add( + new CompoundingSubrule + { + HeadLhs = { Pattern.New("head").Annotation(AnySegment).OneOrMore.Value }, + NonHeadLhs = { Pattern.New("nonHead").Annotation(AnySegment).OneOrMore.Value }, + Rhs = { new CopyFromInput("head"), new CopyFromInput("head"), new CopyFromInput("nonHead") }, + } + ); + Morphophonemic.MorphologicalRules.Add(crule); + + // "pu" is the only root the input can end in, so every analysis differs only in how "tata" splits + List outputs = new AnalysisCompoundingRule(CreateMorpher(pruneCopies), crule) + .Apply(CreateInput("tatapu")) + .ToList(); + + Assert.That(outputs.Count, Is.EqualTo(expectedOutputs)); + Assert.That(outputs.Select(w => w.CurrentNonHead.RootAllomorph.Morpheme.Gloss), Is.All.EqualTo("52")); + } + + [TestCase(0)] + [TestCase(1)] + [TestCase(2)] + public void ParsesTheSameUnderEveryParallelismSetting(int maxDegreeOfParallelism) + { + AffixProcessRule redup = CreateCopyRule(FullPart()); + redup.RequiredSyntacticFeatureStruct = FeatureStruct.New(Language.SyntacticFeatureSystem).Symbol("V").Value; + Morphophonemic.MorphologicalRules.Add(redup); + + string[] unpruned = ParseGlosses(CreateMorpher(false, maxDegreeOfParallelism: 1), "sagsag"); + Assert.That(unpruned, Is.EqualTo(new[] { "32 RED" })); + Assert.That(ParseGlosses(CreateMorpher(true, maxDegreeOfParallelism), "sagsag"), Is.EqualTo(unpruned)); + } + + [TestCase(false)] + [TestCase(true)] + public void PartialMorphemesParseTheSameWithAndWithoutPruning(bool alwaysEnforceFinalTemplates) + { + Morphophonemic.MorphologicalRules.Add(CreateCopyRule(FullPart())); + + string[] Parse(bool pruneCopies, string word) + { + Morpher morpher = CreateMorpher(pruneCopies); + Assert.That(morpher.IsPartial, Is.True, "entry 54 has an empty syntactic feature structure"); + morpher.AlwaysEnforceFinalTemplates = alwaysEnforceFinalTemplates; + return ParseGlosses(morpher, word); + } + + Assert.That(Parse(false, "pipi"), Does.Contain("54 RED")); + foreach (string word in new[] { "pipi", "sagsag", "sagsa" }) + Assert.That(Parse(true, word), Is.EqualTo(Parse(false, word)), word); + } + + /// + /// The prune's weakest point: a self-feeding deletion strips two segments from the second copy only, + /// and = 0 lets analysis restore just one of them. + /// + [TestCase(0)] + [TestCase(1)] + [TestCase(2)] + public void CapLimitedDeletionInOneCopyParsesTheSameWithAndWithoutPruning(int deletionReapplications) + { + var vowel = FeatureStruct + .New(Language.PhonologicalFeatureSystem) + .Symbol(HCFeatureSystem.Segment) + .Symbol("voc+") + .Value; + AffixProcessRule redup = CreateCopyRule(FullPart()); + redup.RequiredSyntacticFeatureStruct = FeatureStruct.New(Language.SyntacticFeatureSystem).Symbol("V").Value; + Morphophonemic.MorphologicalRules.Add(redup); + + var gDelete = new RewriteRule + { + Name = "g_delete", + Lhs = Pattern.New().Annotation(Character(Table1, "g")).Value, + }; + gDelete.Subrules.Add( + new RewriteSubrule { LeftEnvironment = Pattern.New().Annotation(vowel).Value } + ); + Allophonic.PhonologicalRules.Add(gDelete); + + LexEntry root = AddEntry( + "GG", + FeatureStruct.New(Language.SyntacticFeatureSystem).Symbol("V").Value, + Morphophonemic, + "ggasa" + ); + try + { + var generator = new Morpher(TraceManager, Language); + Assert.That( + generator.GenerateWords(root, new Morpheme[] { redup }, FeatureStruct.New().Value), + Is.EquivalentTo(new[] { "ggasaasa" }), + "forward derivation defines the expected parse" + ); + + string[] Parse(bool pruneCopies) + { + Morpher morpher = CreateMorpher(pruneCopies); + morpher.DeletionReapplications = deletionReapplications; + return ParseGlosses(morpher, "ggasaasa"); + } + + string[] unpruned = Parse(false); + Assert.That(unpruned, Does.Contain("GG RED"), "the unpruned engine finds the generated analysis"); + Assert.That(Parse(true), Is.EqualTo(unpruned)); + } + finally + { + Morphophonemic.Entries.Remove(root); + } + } + + /// + /// Counterbleeding opacity: voicing changes only the second copy, then raising destroys the vowel that + /// conditioned it, so analysis must restore the copy through an environment that is no longer on the surface. + /// + [Test] + public void OpaqueChangeToOneCopyParsesTheSameWithAndWithoutPruning() + { + AffixProcessRule redup = CreateCopyRule(FullPart()); + redup.RequiredSyntacticFeatureStruct = FeatureStruct.New(Language.SyntacticFeatureSystem).Symbol("V").Value; + Morphophonemic.MorphologicalRules.Add(redup); + + var voicing = new RewriteRule + { + Name = "voicing", + Lhs = Pattern.New().Annotation(Character(Table1, "s")).Value, + }; + voicing.Subrules.Add( + new RewriteSubrule + { + Rhs = Pattern + .New() + .Annotation( + FeatureStruct + .New(Language.PhonologicalFeatureSystem) + .Symbol(HCFeatureSystem.Segment) + .Symbol("vd+") + .Value + ) + .Value, + LeftEnvironment = Pattern.New().Annotation(Character(Table1, "a")).Value, + RightEnvironment = Pattern.New().Annotation(Character(Table1, "a")).Value, + } + ); + Allophonic.PhonologicalRules.Add(voicing); + + var raising = new RewriteRule + { + Name = "raising", + Lhs = Pattern.New().Annotation(Character(Table1, "a")).Value, + }; + raising.Subrules.Add( + new RewriteSubrule + { + Rhs = Pattern + .New() + .Annotation( + FeatureStruct + .New(Language.PhonologicalFeatureSystem) + .Symbol(HCFeatureSystem.Segment) + .Symbol("high+") + .Symbol("low-") + .Symbol("back-") + .Value + ) + .Value, + LeftEnvironment = Pattern.New().Annotation(Character(Table1, "z")).Value, + } + ); + Allophonic.PhonologicalRules.Add(raising); + + LexEntry root = AddEntry( + "SA", + FeatureStruct.New(Language.SyntacticFeatureSystem).Symbol("V").Value, + Morphophonemic, + "sa" + ); + try + { + var generator = new Morpher(TraceManager, Language); + Assert.That( + generator.GenerateWords(root, new Morpheme[] { redup }, FeatureStruct.New().Value), + Is.EquivalentTo(new[] { "sazi" }), + "forward derivation defines the expected parse" + ); + + string[] unpruned = ParseGlosses(CreateMorpher(false), "sazi"); + Assert.That(unpruned, Does.Contain("SA RED"), "the unpruned engine finds the generated analysis"); + Assert.That(ParseGlosses(CreateMorpher(true), "sazi"), Is.EqualTo(unpruned)); + } + finally + { + Morphophonemic.Entries.Remove(root); + } + } + + private Morpher CreateMorpher(bool pruneCopies, int maxDegreeOfParallelism = 0) + { + return new Morpher(TraceManager, Language, maxDegreeOfParallelism) { PruneDisagreeingCopies = pruneCopies }; + } + + private static string[] ParseGlosses(Morpher morpher, string word) + { + return morpher + .ParseWord(word) + .Select(w => string.Join(" ", w.AllomorphsInMorphOrder.Select(a => a.Morpheme.Gloss))) + .OrderBy(s => s, StringComparer.Ordinal) + .ToArray(); + } + + private static Pattern FullPart() + { + return Pattern.New("copy").Annotation(AnySegment).OneOrMore.Value; + } + + private static Pattern SingleSegmentPart() + { + return Pattern.New("copy").Annotation(AnySegment).Value; + } + + private static AffixProcessRule CreateCopyRule(Pattern part, int copyCount = 2) + { + var rule = new AffixProcessRule + { + Name = "full_copy", + Gloss = "RED", + RequiredSyntacticFeatureStruct = FeatureStruct.New().Value, + OutSyntacticFeatureStruct = FeatureStruct.New().Value, + }; + rule.Allomorphs.Add(CreateCopyAllomorph(part, copyCount)); + return rule; + } + + private static AffixProcessAllomorph CreateCopyAllomorph(Pattern part, int copyCount) + { + var allomorph = new AffixProcessAllomorph { Lhs = { part } }; + for (int i = 0; i < copyCount; i++) + allomorph.Rhs.Add(new CopyFromInput("copy")); + return allomorph; + } + + private (string First, string Second) FindIncompatibleSegments() + { + CharacterDefinition[] segments = Table3 + .Where(definition => definition.Type == HCFeatureSystem.Segment) + .ToArray(); + var pair = segments + .SelectMany( + (first, index) => segments.Skip(index + 1).Select(second => new { First = first, Second = second }) + ) + .First(candidate => !candidate.First.FeatureStruct.IsUnifiable(candidate.Second.FeatureStruct)); + return (pair.First.Representations.First(), pair.Second.Representations.First()); + } + + private Word CreateInput(string representation) + { + var input = new Word(Surface, Table3.Segment(representation)) { AnalysisScope = new AnalysisScope() }; + input.Freeze(); + return input; + } +} From 425f1f36c83c22eafde6554a54c7767ae4d3ab6c Mon Sep 17 00:00:00 2001 From: John Lambert Date: Fri, 9 Oct 2026 21:53:22 -0400 Subject: [PATCH 6/7] Prune disagreeing copies through Pattern.Acceptable, also while tracing The copy check is now the analysis rule spec's Pattern.Acceptable, so the three analysis rules use MultiplePatternRule unchanged and DisagreeingCopiesPruningRule is deleted. Affix-process specs set the predicate only when a part is copied, so other rules pay nothing; the compounding spec combines it with its existing head-capture check. Tracing no longer turns pruning off: a pruned match is a pattern that did not match, which a trace already leaves out. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../Morpher.cs | 3 +- .../AnalysisAffixProcessAllomorphRuleSpec.cs | 7 +++- .../AnalysisAffixProcessRule.cs | 5 +-- .../AnalysisCompoundingRule.cs | 5 +-- .../AnalysisCompoundingSubruleRuleSpec.cs | 7 ++-- .../AnalysisMorphologicalTransformRuleSpec.cs | 20 +++++++++- .../AnalysisRealizationalAffixProcessRule.cs | 5 +-- .../DisagreeingCopiesPruningRule.cs | 40 ------------------- .../PruneDisagreeingCopiesTests.cs | 4 +- 9 files changed, 36 insertions(+), 60 deletions(-) delete mode 100644 src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/DisagreeingCopiesPruningRule.cs diff --git a/src/SIL.Machine.Morphology.HermitCrab/Morpher.cs b/src/SIL.Machine.Morphology.HermitCrab/Morpher.cs index 1c7fea8fd..73c6007d2 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/Morpher.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/Morpher.cs @@ -114,8 +114,7 @@ public ITraceManager TraceManager /// /// When unapplying an affix-process or compounding rule that copies a part more than once /// (reduplication), skip matches whose copies cannot unify segment by segment. Such a match can never - /// survive synthesis, so this removes only doomed analyses. On by default; ignored while tracing, so - /// a trace still shows each such match failing in synthesis. + /// survive synthesis, so this removes only doomed analyses. On by default, including while tracing. /// public bool PruneDisagreeingCopies { get; set; } diff --git a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisAffixProcessAllomorphRuleSpec.cs b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisAffixProcessAllomorphRuleSpec.cs index 9ce94c148..895f9eb18 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisAffixProcessAllomorphRuleSpec.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisAffixProcessAllomorphRuleSpec.cs @@ -8,10 +8,13 @@ public class AnalysisAffixProcessAllomorphRuleSpec : AnalysisMorphologicalTransf { private readonly AffixProcessAllomorph _allomorph; - public AnalysisAffixProcessAllomorphRuleSpec(AffixProcessAllomorph allomorph) - : base(allomorph.Lhs, allomorph.Rhs) + public AnalysisAffixProcessAllomorphRuleSpec(AffixProcessAllomorph allomorph, Morpher morpher = null) + : base(allomorph.Lhs, allomorph.Rhs, morpher) { _allomorph = allomorph; + // the matcher builds a Match for every candidate it tests, so rules that copy nothing get no check + if (HasRepeatedParts) + Pattern.Acceptable = CopiesMayAgree; Pattern.Freeze(); } diff --git a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisAffixProcessRule.cs b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisAffixProcessRule.cs index 59014596d..d22149d32 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisAffixProcessRule.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisAffixProcessRule.cs @@ -29,9 +29,8 @@ public AnalysisAffixProcessRule(Morpher morpher, AffixProcessRule rule) foreach (AffixProcessAllomorph allo in rule.Allomorphs) { _rules.Add( - new DisagreeingCopiesPruningRule( - morpher, - new AnalysisAffixProcessAllomorphRuleSpec(allo), + new MultiplePatternRule( + new AnalysisAffixProcessAllomorphRuleSpec(allo, morpher), new MatcherSettings { Filter = ann => ann.Type() == HCFeatureSystem.Segment, diff --git a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisCompoundingRule.cs b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisCompoundingRule.cs index 6dbf11b9f..8df7e3900 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisCompoundingRule.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisCompoundingRule.cs @@ -28,9 +28,8 @@ public AnalysisCompoundingRule(Morpher morpher, CompoundingRule rule) foreach (CompoundingSubrule sr in rule.Subrules) { _rules.Add( - new DisagreeingCopiesPruningRule( - morpher, - new AnalysisCompoundingSubruleRuleSpec(sr), + new MultiplePatternRule( + new AnalysisCompoundingSubruleRuleSpec(sr, morpher), new MatcherSettings { Filter = ann => ann.Type() == HCFeatureSystem.Segment, diff --git a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisCompoundingSubruleRuleSpec.cs b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisCompoundingSubruleRuleSpec.cs index 50f6177bb..db11fa690 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisCompoundingSubruleRuleSpec.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisCompoundingSubruleRuleSpec.cs @@ -9,11 +9,12 @@ public class AnalysisCompoundingSubruleRuleSpec : AnalysisMorphologicalTransform { private readonly CompoundingSubrule _subrule; - public AnalysisCompoundingSubruleRuleSpec(CompoundingSubrule subrule) - : base(subrule.HeadLhs.Concat(subrule.NonHeadLhs), subrule.Rhs) + public AnalysisCompoundingSubruleRuleSpec(CompoundingSubrule subrule, Morpher morpher = null) + : base(subrule.HeadLhs.Concat(subrule.NonHeadLhs), subrule.Rhs, morpher) { _subrule = subrule; - Pattern.Acceptable = match => _subrule.HeadLhs.Any(part => IsPartCaptured(match, part.Name)); + Pattern.Acceptable = match => + _subrule.HeadLhs.Any(part => IsPartCaptured(match, part.Name)) && CopiesMayAgree(match); Pattern.Freeze(); } diff --git a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisMorphologicalTransformRuleSpec.cs b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisMorphologicalTransformRuleSpec.cs index a23ae84b1..826259988 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisMorphologicalTransformRuleSpec.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisMorphologicalTransformRuleSpec.cs @@ -9,11 +9,27 @@ public abstract class AnalysisMorphologicalTransformRuleSpec : AnalysisMorphologicalTransform, IPatternRuleSpec { + private readonly Morpher _morpher; + private readonly bool _hasRepeatedParts; + protected AnalysisMorphologicalTransformRuleSpec( IEnumerable> lhs, - IList rhs + IList rhs, + Morpher morpher ) - : base(lhs, rhs) { } + : base(lhs, rhs) + { + _morpher = morpher; + _hasRepeatedParts = HasRepeatedParts; + } + + protected bool CopiesMayAgree(Match match) + { + return _morpher == null + || !_morpher.PruneDisagreeingCopies + || !_hasRepeatedParts + || !HasDisagreeingCopies(match); + } public bool IsApplicable(Word input) { diff --git a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisRealizationalAffixProcessRule.cs b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisRealizationalAffixProcessRule.cs index 9862db6bf..544e531c6 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisRealizationalAffixProcessRule.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisRealizationalAffixProcessRule.cs @@ -22,9 +22,8 @@ public AnalysisRealizationalAffixProcessRule(Morpher morpher, RealizationalAffix foreach (AffixProcessAllomorph allo in rule.Allomorphs) { _rules.Add( - new DisagreeingCopiesPruningRule( - morpher, - new AnalysisAffixProcessAllomorphRuleSpec(allo), + new MultiplePatternRule( + new AnalysisAffixProcessAllomorphRuleSpec(allo, morpher), new MatcherSettings { Filter = ann => ann.Type() == HCFeatureSystem.Segment, diff --git a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/DisagreeingCopiesPruningRule.cs b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/DisagreeingCopiesPruningRule.cs deleted file mode 100644 index e4431ab44..000000000 --- a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/DisagreeingCopiesPruningRule.cs +++ /dev/null @@ -1,40 +0,0 @@ -using System.Collections.Generic; -using SIL.Machine.Annotations; -using SIL.Machine.Matching; -using SIL.Machine.Rules; - -namespace SIL.Machine.Morphology.HermitCrab.MorphologicalRules -{ - internal sealed class DisagreeingCopiesPruningRule : MultiplePatternRule - { - private readonly Morpher _morpher; - private readonly AnalysisMorphologicalTransformRuleSpec _spec; - private readonly bool _hasRepeatedParts; - - public DisagreeingCopiesPruningRule( - Morpher morpher, - AnalysisMorphologicalTransformRuleSpec ruleSpec, - MatcherSettings matcherSettings - ) - : base(ruleSpec, matcherSettings) - { - _morpher = morpher; - _spec = ruleSpec; - _hasRepeatedParts = ruleSpec.HasRepeatedParts; - } - - protected override IEnumerable ApplyImpl(Word input, ShapeNode start) - { - bool prune = _morpher.PruneDisagreeingCopies && _hasRepeatedParts && !_morpher.TraceManager.IsTracing; - var results = new List(); - foreach (Match match in Matcher.AllMatches(input, start)) - { - if (prune && _spec.HasDisagreeingCopies(match)) - continue; - - results.Add(RuleSpec.ApplyRhs(this, match)); - } - return results; - } - } -} diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/PruneDisagreeingCopiesTests.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/PruneDisagreeingCopiesTests.cs index 2d2b3aaa0..b3af6bc06 100644 --- a/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/PruneDisagreeingCopiesTests.cs +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/PruneDisagreeingCopiesTests.cs @@ -47,7 +47,7 @@ public void DisagreeingCopiesAreRemovedOnlyWhenPruning(bool pruneCopies, int exp } [Test] - public void TracingKeepsDisagreeingCopies() + public void TracingStillPrunesDisagreeingCopies() { (string first, string second) = FindIncompatibleSegments(); AffixProcessRule rule = CreateCopyRule(SingleSegmentPart()); @@ -59,7 +59,7 @@ public void TracingKeepsDisagreeingCopies() { List outputs = new AnalysisAffixProcessRule(CreateMorpher(true), rule).Apply(input).ToList(); - Assert.That(outputs.Count, Is.EqualTo(1)); + Assert.That(outputs, Is.Empty); } finally { From 81ff2feadebf59a174ea08af400fabb6c2d0b5e1 Mon Sep 17 00:00:00 2001 From: John Lambert Date: Sat, 10 Oct 2026 07:44:47 -0400 Subject: [PATCH 7/7] Line up copies across unapplied deletions instead of keeping them A copy containing an optional segment was always kept. Now copies are compared by alignment: an optional segment may stand for any number of segments of the other copy it unifies with, including none, and the copies disagree only if no alignment exists. A single optional node can stand for several segments because DeletionReapplications (0 by default) lets analysis restore fewer segments than a deletion removed; matching it to at most one segment pruned the valid parse in CapLimitedDeletionInOneCopyParsesTheSameWithAndWithoutPruning. Copies without optional nodes keep the linear comparison. Optional nodes that are not segments still make a copy undecidable. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../AnalysisMorphologicalTransform.cs | 59 ++++++++++++++++--- .../PruneDisagreeingCopiesTests.cs | 33 +++++++++++ 2 files changed, 84 insertions(+), 8 deletions(-) diff --git a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisMorphologicalTransform.cs b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisMorphologicalTransform.cs index c3244d021..5871dbb7c 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisMorphologicalTransform.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisMorphologicalTransform.cs @@ -54,8 +54,10 @@ internal bool HasRepeatedParts /// /// Synthesis writes every copy of a part from the same input, and anything that later changes one /// copy is unapplied before this rule, so copies proven to disagree segment by segment cannot lead - /// to a valid analysis. A part the rule modifies, or a copy containing an unapplied deletion, is - /// never reported as disagreeing. + /// to a valid analysis. An optional segment (an unapplied deletion) stands for any number of segments + /// it unifies with, because can restore fewer segments + /// than were deleted, so copies disagree only if no such choice lines them up. A part the rule + /// modifies, or a copy with an optional node that is not a segment, is never reported as disagreeing. /// internal bool HasDisagreeingCopies(Match match) { @@ -95,7 +97,7 @@ out List> copies .GetSkippedOptionalNodes(match.Input.Shape, capture.Range) .Concat(match.Input.Shape.GetNodes(capture.Range)) .ToList(); - if (nodes.Any(node => node.Annotation.Optional)) + if (nodes.Any(node => node.Annotation.Optional && node.Annotation.Type() != HCFeatureSystem.Segment)) return false; copies.Add(nodes.Where(node => node.Annotation.Type() == HCFeatureSystem.Segment).ToList()); @@ -110,17 +112,58 @@ private static bool AnyCopiesDisagree(List> copies) { for (int j = i + 1; j < copies.Count; j++) { - if (copies[i].Count != copies[j].Count) + if (!CanAlign(copies[i], copies[j])) return true; + } + } + return false; + } - for (int k = 0; k < copies[i].Count; k++) + private static bool CanAlign(List first, List second) + { + if (!first.Any(node => node.Annotation.Optional) && !second.Any(node => node.Annotation.Optional)) + { + if (first.Count != second.Count) + return false; + for (int k = 0; k < first.Count; k++) + { + if (!first[k].Annotation.FeatureStruct.IsUnifiable(second[k].Annotation.FeatureStruct)) + return false; + } + return true; + } + + // reachable[i, j]: the first i nodes of one copy can be lined up with the first j of the other; + // an optional node is passed over, or absorbs a node of the other copy and stays available + var reachable = new bool[first.Count + 1, second.Count + 1]; + reachable[0, 0] = true; + for (int i = 0; i <= first.Count; i++) + { + for (int j = 0; j <= second.Count; j++) + { + if (!reachable[i, j]) + continue; + bool firstOptional = i < first.Count && first[i].Annotation.Optional; + bool secondOptional = j < second.Count && second[j].Annotation.Optional; + if (firstOptional) + reachable[i + 1, j] = true; + if (secondOptional) + reachable[i, j + 1] = true; + if ( + i < first.Count + && j < second.Count + && first[i].Annotation.FeatureStruct.IsUnifiable(second[j].Annotation.FeatureStruct) + ) { - if (!copies[i][k].Annotation.FeatureStruct.IsUnifiable(copies[j][k].Annotation.FeatureStruct)) - return true; + reachable[i + 1, j + 1] = true; + if (firstOptional) + reachable[i, j + 1] = true; + if (secondOptional) + reachable[i + 1, j] = true; } } } - return false; + return reachable[first.Count, second.Count]; } public Pattern Pattern diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/PruneDisagreeingCopiesTests.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/PruneDisagreeingCopiesTests.cs index b3af6bc06..fa573bf21 100644 --- a/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/PruneDisagreeingCopiesTests.cs +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/PruneDisagreeingCopiesTests.cs @@ -83,6 +83,31 @@ public void EveryPairOfCopiesIsCompared(bool pruneCopies, int expectedOutputs) Assert.That(outputs.Count, Is.EqualTo(expectedOutputs)); } + [Test] + public void OptionalSegmentThatCannotLineUpIsPruned() + { + // skipping or keeping the restored segment, every split still pairs the two incompatible segments + (string first, string second) = FindIncompatibleSegments(); + AffixProcessRule rule = CreateCopyRule(FullPart()); + Word input = CreateInputWithOptionalSecondSegment(first + second + second); + + Assert.That(new AnalysisAffixProcessRule(CreateMorpher(false), rule).Apply(input), Is.Not.Empty); + Assert.That(new AnalysisAffixProcessRule(CreateMorpher(true), rule).Apply(input), Is.Empty); + } + + [Test] + public void OptionalSegmentThatLinesUpWhenSkippedIsKept() + { + (string first, string second) = FindIncompatibleSegments(); + AffixProcessRule rule = CreateCopyRule(FullPart()); + Word input = CreateInputWithOptionalSecondSegment(first + second + first); + + Assert.That( + new AnalysisAffixProcessRule(CreateMorpher(true), rule).Apply(input).Count(), + Is.EqualTo(new AnalysisAffixProcessRule(CreateMorpher(false), rule).Apply(input).Count()) + ); + } + [TestCase(false, 3)] [TestCase(true, 1)] public void RealizationalFullCopyKeepsOnlyTheSplitWhoseCopiesAgree(bool pruneCopies, int expectedOutputs) @@ -358,6 +383,14 @@ private static AffixProcessAllomorph CreateCopyAllomorph(Pattern