Prune reduplication matches whose copies cannot agree

src/SIL.Machine.Morphology.HermitCrab/Morpher.cs

@@ -61,6 +61,7 @@ public Morpher(ITraceManager traceManager, Language lang, int maxDegreeOfParalle
             MaxStemCount = 2;
             MaxAlternatives = 0;
             MergeEquivalentAnalyses = true;
+            PruneDisagreeingCopies = true;
             LexEntrySelector = entry => true;
             RuleSelector = rule => true;
 
@@ -110,6 +111,14 @@ public ITraceManager TraceManager
         /// </summary>
         public bool AlwaysEnforceFinalTemplates { get; set; }
 
+        /// <summary>
+        /// When unapplying an affix-process or compounding rule that copies a part more than once
+        /// (reduplication), skip matches whose copies cannot unify segment by segment. Such a match can never
+        /// survive synthesis, so this removes only doomed analyses. On by default; ignored while tracing, so
+        /// a trace still shows each such match failing in synthesis.
+        /// </summary>
+        public bool PruneDisagreeingCopies { get; set; }
+
         /// <summary>
         /// Caps the concurrency used within a single parse or generation -- analysis cascade,
         /// affix-template unapplication and synthesis alike. A value of 1 runs the work fully

src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisAffixProcessRule.cs

@@ -29,7 +29,8 @@ public AnalysisAffixProcessRule(Morpher morpher, AffixProcessRule rule)
             foreach (AffixProcessAllomorph allo in rule.Allomorphs)
             {
                 _rules.Add(
-                    new MultiplePatternRule<Word, ShapeNode>(
+                    new DisagreeingCopiesPruningRule(
+                        morpher,
                         new AnalysisAffixProcessAllomorphRuleSpec(allo),
                         new MatcherSettings<ShapeNode>
                         {

src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisCompoundingRule.cs

@@ -28,7 +28,8 @@ public AnalysisCompoundingRule(Morpher morpher, CompoundingRule rule)
             foreach (CompoundingSubrule sr in rule.Subrules)
             {
                 _rules.Add(
-                    new MultiplePatternRule<Word, ShapeNode>(
+                    new DisagreeingCopiesPruningRule(
+                        morpher,
                         new AnalysisCompoundingSubruleRuleSpec(sr),
                         new MatcherSettings<ShapeNode>
                         {

src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisMorphologicalTransform.cs

@@ -46,6 +46,83 @@ protected IDictionary<string, int> CapturedParts
             get { return _capturedParts; }
         }
 
+        internal bool HasRepeatedParts
+        {
+            get { return _capturedParts.Values.Any(count => count >= 2); }
+        }
+
+        /// <summary>
+        /// Synthesis writes every copy of a part from the same input, and anything that later changes one
+        /// copy is unapplied before this rule, so copies proven to disagree segment by segment cannot lead
+        /// to a valid analysis. A part the rule modifies, or a copy containing an unapplied deletion, is
+        /// never reported as disagreeing.
+        /// </summary>
+        internal bool HasDisagreeingCopies(Match<Word, ShapeNode> match)
+        {
+            foreach (KeyValuePair<string, int> capturedPart in _capturedParts)
+            {
+                if (capturedPart.Value < 2 || _modifyFromInfos.ContainsKey(capturedPart.Key))
+                    continue;
+
+                if (
+                    TryGetSegmentCopies(match, capturedPart.Key, capturedPart.Value, out List<List<ShapeNode>> copies)
+                    && AnyCopiesDisagree(copies)
+                )
+                {
+                    return true;
+                }
+            }
+
+            return false;
+        }
+
+        private static bool TryGetSegmentCopies(
+            Match<Word, ShapeNode> match,
+            string partName,
+            int copyCount,
+            out List<List<ShapeNode>> copies
+        )
+        {
+            copies = new List<List<ShapeNode>>(copyCount);
+            for (int i = 0; i < copyCount; i++)
+            {
+                GroupCapture<ShapeNode> capture = match.GroupCaptures[GetGroupName(partName, i)];
+                if (!capture.Success)
+                    return false;
+
+                // an unapplied word-initial deletion sits before the capture, not inside it
+                List<ShapeNode> nodes = MorphologicalOutputAction
+                    .GetSkippedOptionalNodes(match.Input.Shape, capture.Range)
+                    .Concat(match.Input.Shape.GetNodes(capture.Range))
+                    .ToList();
+                if (nodes.Any(node => node.Annotation.Optional))
+                    return false;
+
+                copies.Add(nodes.Where(node => node.Annotation.Type() == HCFeatureSystem.Segment).ToList());
+            }
+            return true;
+        }
+
+        // unifiability is not transitive, so every pair is compared, not just each copy against the first
+        private static bool AnyCopiesDisagree(List<List<ShapeNode>> copies)
+        {
+            for (int i = 0; i < copies.Count; i++)
+            {
+                for (int j = i + 1; j < copies.Count; j++)
+                {
+                    if (copies[i].Count != copies[j].Count)
+                        return true;
+
+                    for (int k = 0; k < copies[i].Count; k++)
+                    {
+                        if (!copies[i][k].Annotation.FeatureStruct.IsUnifiable(copies[j][k].Annotation.FeatureStruct))
+                            return true;
+                    }
+                }
+            }
+            return false;
+        }
+
         public Pattern<Word, ShapeNode> Pattern
         {
             get { return _pattern; }

src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisRealizationalAffixProcessRule.cs

@@ -22,7 +22,8 @@ public AnalysisRealizationalAffixProcessRule(Morpher morpher, RealizationalAffix
             foreach (AffixProcessAllomorph allo in rule.Allomorphs)
             {
                 _rules.Add(
-                    new MultiplePatternRule<Word, ShapeNode>(
+                    new DisagreeingCopiesPruningRule(
+                        morpher,
                         new AnalysisAffixProcessAllomorphRuleSpec(allo),
                         new MatcherSettings<ShapeNode>
                         {

src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/DisagreeingCopiesPruningRule.cs

@@ -0,0 +1,40 @@
+using System.Collections.Generic;
+using SIL.Machine.Annotations;
+using SIL.Machine.Matching;
+using SIL.Machine.Rules;
+
+namespace SIL.Machine.Morphology.HermitCrab.MorphologicalRules
+{
+    internal sealed class DisagreeingCopiesPruningRule : MultiplePatternRule<Word, ShapeNode>
+    {
+        private readonly Morpher _morpher;
+        private readonly AnalysisMorphologicalTransformRuleSpec _spec;
+        private readonly bool _hasRepeatedParts;
+
+        public DisagreeingCopiesPruningRule(
+            Morpher morpher,
+            AnalysisMorphologicalTransformRuleSpec ruleSpec,
+            MatcherSettings<ShapeNode> matcherSettings
+        )
+            : base(ruleSpec, matcherSettings)
+        {
+            _morpher = morpher;
+            _spec = ruleSpec;
+            _hasRepeatedParts = ruleSpec.HasRepeatedParts;
+        }
+
+        protected override IEnumerable<Word> ApplyImpl(Word input, ShapeNode start)
+        {
+            bool prune = _morpher.PruneDisagreeingCopies && _hasRepeatedParts && !_morpher.TraceManager.IsTracing;
+            var results = new List<Word>();
+            foreach (Match<Word, ShapeNode> match in Matcher.AllMatches(input, start))
+            {
+                if (prune && _spec.HasDisagreeingCopies(match))
+                    continue;
+
+                results.Add(RuleSpec.ApplyRhs(this, match));
+            }
+            return results;
+        }
+    }
+}

src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/MorphologicalOutputAction.cs

@@ -37,7 +37,7 @@ IDictionary<string, int> capturedParts
         /// <param name="output">The output word synthesis.</param>
         public abstract IEnumerable<Tuple<ShapeNode, ShapeNode>> Apply(Match<Word, ShapeNode> match, Word output);
 
-        protected IEnumerable<ShapeNode> GetSkippedOptionalNodes(Shape shape, Range<ShapeNode> range)
+        protected internal static IEnumerable<ShapeNode> GetSkippedOptionalNodes(Shape shape, Range<ShapeNode> range)
         {
             ShapeNode node = range.Start.Prev;
             var skippedNodes = new List<ShapeNode>();

tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/AffixProcessRuleTests.cs

@@ -965,8 +965,9 @@ public void SimulfixRules()
         AssertMorphsEqual(morpher.ParseWord("sɯɯpu"), "50 SIMUL");
     }
 
-    [Test]
-    public void ReduplicationRules()
+    [TestCase(false)]
+    [TestCase(true)]
+    public void ReduplicationRules(bool pruneCopies)
     {
         var any = FeatureStruct.New().Symbol(HCFeatureSystem.Segment).Value;
         var cons = FeatureStruct
@@ -1015,7 +1016,7 @@ public void ReduplicationRules()
         );
         Morphophonemic.MorphologicalRules.Add(redup);
 
-        var morpher = new Morpher(TraceManager, Language);
+        var morpher = new Morpher(TraceManager, Language) { PruneDisagreeingCopies = pruneCopies };
         AssertMorphsEqual(morpher.ParseWord("sasag"), "RED 32");
 
         var voicing = new RewriteRule
@@ -1033,7 +1034,7 @@ public void ReduplicationRules()
         );
         Allophonic.PhonologicalRules.Add(voicing);
 
-        morpher = new Morpher(TraceManager, Language);
+        morpher = new Morpher(TraceManager, Language) { PruneDisagreeingCopies = pruneCopies };
         AssertMorphsEqual(morpher.ParseWord("sazag"), "RED 32");
 
         var affrication = new RewriteRule
@@ -1050,7 +1051,7 @@ public void ReduplicationRules()
         );
         Allophonic.PhonologicalRules.Add(affrication);
 
-        morpher = new Morpher(TraceManager, Language);
+        morpher = new Morpher(TraceManager, Language) { PruneDisagreeingCopies = pruneCopies };
         AssertMorphsEqual(morpher.ParseWord("tsazag"), "RED 32");
 
         redup.Allomorphs.Clear();
@@ -1066,7 +1067,7 @@ public void ReduplicationRules()
             }
         );
 
-        morpher = new Morpher(TraceManager, Language);
+        morpher = new Morpher(TraceManager, Language) { PruneDisagreeingCopies = pruneCopies };
         AssertMorphsEqual(morpher.ParseWord("tsagag"), "32 RED");
 
         Allophonic.PhonologicalRules.Clear();
@@ -1084,7 +1085,7 @@ public void ReduplicationRules()
             }
         );
 
-        morpher = new Morpher(TraceManager, Language);
+        morpher = new Morpher(TraceManager, Language) { PruneDisagreeingCopies = pruneCopies };
         AssertMorphsEqual(morpher.ParseWord("sagsag"), "32 RED");
         AssertMorphsEqual(morpher.ParseWord("sasibudbud"), "38 RED");
 
@@ -1115,7 +1116,7 @@ public void ReduplicationRules()
         );
         Allophonic.PhonologicalRules.Add(gDelete);
 
-        morpher = new Morpher(TraceManager, Language);
+        morpher = new Morpher(TraceManager, Language) { PruneDisagreeingCopies = pruneCopies };
         AssertMorphsEqual(morpher.ParseWord("saag"), "32 RED");
 
         gDelete.Subrules.Clear();
@@ -1126,7 +1127,7 @@ public void ReduplicationRules()
             }
         );
 
-        morpher = new Morpher(TraceManager, Language);
+        morpher = new Morpher(TraceManager, Language) { PruneDisagreeingCopies = pruneCopies };
         AssertMorphsEqual(morpher.ParseWord("saga"), "32 RED");
 
         Allophonic.PhonologicalRules.Clear();
@@ -1152,7 +1153,7 @@ public void ReduplicationRules()
             }
         );
 
-        morpher = new Morpher(TraceManager, Language);
+        morpher = new Morpher(TraceManager, Language) { PruneDisagreeingCopies = pruneCopies };
         AssertMorphsEqual(morpher.ParseWord("buiuib"), "30 RED", "31 RED");
     }
 
@@ -1900,8 +1901,9 @@ public void SubsumedAffix()
         AssertMorphsEqual(morpher.ParseWord("bubib"), "42 PRES", "43 PRES");
     }
 
-    [Test]
-    public void ModifyFromInputRules()
+    [TestCase(false)]
+    [TestCase(true)]
+    public void ModifyFromInputRules(bool pruneCopies)
     {
         var any = FeatureStruct.New().Symbol(HCFeatureSystem.Segment).Value;
         var vowel = FeatureStruct
@@ -1941,7 +1943,7 @@ public void ModifyFromInputRules()
         );
         Morphophonemic.MorphologicalRules.Add(sSuffix);
 
-        var morpher = new Morpher(TraceManager, Language);
+        var morpher = new Morpher(TraceManager, Language) { PruneDisagreeingCopies = pruneCopies };
         AssertMorphsEqual(morpher.ParseWord("puso"), "52 PL");
     }