Fixes #2616. Support combining sequences that don't normalize.

Terminal.Gui/Application/Application.cs

@@ -67,6 +67,25 @@ public static string ToString (IConsoleDriver? driver)
                 {
                     sb.Append (sp);
                 }
+                else if (contents [r, c].CombiningMarks is { Count: > 0 })
+                {
+                    // AtlasEngine does not support NON-NORMALIZED combining marks in a way
+                    // compatible with the driver architecture. Any CMs (except in the first col)
+                    // are correctly combined with the base char, but are ALSO treated as 1 column
+                    // width codepoints E.g. `echo "[e`u{0301}`u{0301}]"` will output `[é  ]`.
+                    // 
+                    // For now, we just ignore the list of CMs.
+                    string combine = rune.ToString ();
+                    string? normalized = null;
+
+                    foreach (Rune combMark in contents [r, c].CombiningMarks)
+                    {
+                        combine += combMark;
+                        normalized = combine.Normalize (NormalizationForm.FormC);
+                    }
+
+                    sb.Append (normalized);
+                }
                 else
                 {
                     sb.Append ((char)rune.Value);
@@ -76,11 +95,6 @@ public static string ToString (IConsoleDriver? driver)
                 {
                     c++;
                 }
-
-                // See Issue #2616
-                //foreach (var combMark in contents [r, c].CombiningMarks) {
-                //	sb.Append ((char)combMark.Value);
-                //}
             }
 
             sb.AppendLine ();

Terminal.Gui/ConsoleDrivers/ConsoleDriver.cs

@@ -208,35 +208,19 @@ public void AddRune (Rune rune)
                     // b) Ignoring any CMs that don't normalize
                     if (Col > 0)
                     {
-                        if (Contents [Row, Col - 1].CombiningMarks.Count > 0)
+                        for (int i = Col; i > 0; i--)
                         {
-                            // Just add this mark to the list
-                            Contents [Row, Col - 1].CombiningMarks.Add (rune);
-
-                            // Ignore. Don't move to next column (let the driver figure out what to do).
-                        }
-                        else
-                        {
-                            // Attempt to normalize the cell to our left combined with this mark
-                            string combined = Contents [Row, Col - 1].Rune + rune.ToString ();
-
-                            // Normalize to Form C (Canonical Composition)
-                            string normalized = combined.Normalize (NormalizationForm.FormC);
-
-                            if (normalized.Length == 1)
+                            if (!Contents [Row, i - 1].Rune.IsCombiningMark () && Contents [Row, i - 1].Rune != Rune.ReplacementChar)
                             {
-                                // It normalized! We can just set the Cell to the left with the
-                                // normalized codepoint
-                                Contents [Row, Col - 1].Rune = (Rune)normalized [0];
-
-                                // Ignore. Don't move to next column because we're already there
-                            }
-                            else
-                            {
-                                // It didn't normalize. Add it to the Cell to left's CM list
-                                Contents [Row, Col - 1].CombiningMarks.Add (rune);
-
-                                // Ignore. Don't move to next column (let the driver figure out what to do).
+                                if (Contents [Row, i - 1].CombiningMarks is null)
+                                {
+                                    Contents [Row, i - 1].CombiningMarks = [];
+                                }
+                                // Just add this mark to the list
+                                Contents [Row, i - 1].CombiningMarks.Add (rune);
+                                Debug.Assert (Contents [Row, i - 1].CombiningMarks.Count > 0);
+
+                                break;
                             }
                         }
 

Terminal.Gui/ConsoleDrivers/CursesDriver/CursesDriver.cs

@@ -319,18 +319,19 @@
                         Rune rune = Contents [row, col].Rune;
                         output.Append (rune);
 
-                        if (Contents [row, col].CombiningMarks.Count > 0)
+                        if (Contents [row, col].CombiningMarks is { Count: > 0 })
                         {
                             // AtlasEngine does not support NON-NORMALIZED combining marks in a way
                             // compatible with the driver architecture. Any CMs (except in the first col)
                             // are correctly combined with the base char, but are ALSO treated as 1 column
                             // width codepoints E.g. `echo "[e`u{0301}`u{0301}]"` will output `[é  ]`.
                             // 
                             // For now, we just ignore the list of CMs.
-                            //foreach (var combMark in Contents [row, col].CombiningMarks) {
-                            //	output.Append (combMark);
-                            //}
-                            // WriteToConsole (output, ref lastCol, row, ref outputWidth);
+                            foreach (var combMark in Contents [row, col].CombiningMarks)
+                            {
+                                output.Append (combMark);
+                            }
+                            WriteToConsole (output, ref lastCol, row, ref outputWidth);
                         }
                         else if (rune.IsSurrogatePair () && rune.GetColumns () < 2)
                         {
@@ -580,7 +581,7 @@
 
     private Curses.Window? _window;
     private UnixMainLoop? _mainLoopDriver;
     private object _processInputToken;
 
     public override MainLoop Init ()
     {
@@ -725,7 +726,7 @@
 
                 while (wch2 == Curses.KeyMouse)
                 {
                     Key kea = null;
 
                     ConsoleKeyInfo [] cki =
                     {
@@ -734,7 +735,7 @@
                         new ('<', 0, false, false, false)
                     };
                     code = 0;
                     HandleEscSeqResponse (ref code, ref k, ref wch2, ref kea, ref cki);
                 }
 
                 return;
@@ -791,7 +792,7 @@
                 k = KeyCode.AltMask | MapCursesKey (wch);
             }
 
             Key key = null;
 
             if (code == 0)
             {
@@ -821,7 +822,7 @@
                     [
                         new ((char)KeyCode.Esc, 0, false, false, false), new ((char)wch2, 0, false, false, false)
                     ];
                     HandleEscSeqResponse (ref code, ref k, ref wch2, ref key, ref cki);
 
                     return;
                 }
@@ -952,7 +953,7 @@
                 EscSeqUtils.DecodeEscSeq (
                                           ref consoleKeyInfo,
                                           ref ck,
                                           cki,
                                           ref mod,
                                           out _,
                                           out _,
@@ -972,7 +973,7 @@
                         OnMouseEvent (new () { Flags = mf, Position = pos });
                     }
 
                     cki = null;
 
                     if (wch2 == 27)
                     {

Terminal.Gui/ConsoleDrivers/NetDriver/NetDriver.cs

@@ -167,18 +167,19 @@ public override bool UpdateScreen ()
                     Rune rune = Contents [row, col].Rune;
                     output.Append (rune);
 
-                    if (Contents [row, col].CombiningMarks.Count > 0)
+                    if (Contents [row, col].CombiningMarks is { Count: > 0 })
                     {
                         // AtlasEngine does not support NON-NORMALIZED combining marks in a way
                         // compatible with the driver architecture. Any CMs (except in the first col)
                         // are correctly combined with the base char, but are ALSO treated as 1 column
                         // width codepoints E.g. `echo "[e`u{0301}`u{0301}]"` will output `[é  ]`.
                         // 
                         // For now, we just ignore the list of CMs.
-                        //foreach (var combMark in Contents [row, col].CombiningMarks) {
-                        //	output.Append (combMark);
-                        //}
-                        // WriteToConsole (output, ref lastCol, row, ref outputWidth);
+                        foreach (var combMark in Contents [row, col].CombiningMarks)
+                        {
+                            output.Append (combMark);
+                        }
+                        WriteToConsole (output, ref lastCol, row, ref outputWidth);
                     }
                     else if (rune.IsSurrogatePair () && rune.GetColumns () < 2)
                     {

Terminal.Gui/ConsoleDrivers/NetDriver/NetMainLoop.cs

@@ -142,6 +142,11 @@ private void NetInputHandler ()
                     _waitForProbe.Reset ();
                 }
 
+                if (_inputHandlerTokenSource.IsCancellationRequested)
+                {
+                    return;
+                }
+
                 ProcessInputQueue ();
             }
             catch (OperationCanceledException)

Terminal.Gui/ConsoleDrivers/WindowsDriver/WindowsConsole.cs

@@ -197,6 +197,14 @@ public bool WriteToConsole (Size size, ExtendedCharInfo [] charInfoBuffer, Coord
                     {
                         _stringBuilder.Append (info.Char);
                     }
+
+                    if (info.CombiningMarks is { })
+                    {
+                        foreach (var combMark in info.CombiningMarks)
+                        {
+                            _stringBuilder.Append (combMark);
+                        }
+                    }
                 }
                 else
                 {
@@ -785,6 +793,7 @@ public struct ExtendedCharInfo
         public char Char { get; set; }
         public Attribute Attribute { get; set; }
         public bool Empty { get; set; } // TODO: Temp hack until virtual terminal sequences
+        internal List<char>? CombiningMarks;
 
         public ExtendedCharInfo (char character, Attribute attribute)
         {

Terminal.Gui/ConsoleDrivers/WindowsDriver/WindowsDriver.cs

@@ -40,7 +40,7 @@
     private Point _pointMove;
     private bool _processButtonClick;
 
     public WindowsDriver ()
     {
         if (Environment.OSVersion.Platform == PlatformID.Win32NT)
         {
@@ -356,22 +356,50 @@
 
                 _outputBuffer [position].Empty = false;
 
+                if (Contents [row, col].CombiningMarks is { Count: > 0 })
+                {
+                    _outputBuffer [position].CombiningMarks = [];
+
+                    foreach (var combMark in Contents [row, col].CombiningMarks)
+                    {
+                        _outputBuffer [position].CombiningMarks!.Add ((char)combMark.Value);
+                    }
+                }
+                else
+                {
+                    _outputBuffer [position].CombiningMarks = null;
+                }
+
                 if (Contents [row, col].Rune.IsBmp)
                 {
                     _outputBuffer [position].Char = (char)Contents [row, col].Rune.Value;
                 }
                 else
                 {
                     //_outputBuffer [position].Empty = true;
-                    _outputBuffer [position].Char = (char)Rune.ReplacementChar.Value;
+                    //_outputBuffer [position].Char = (char)Rune.ReplacementChar.Value;
+                    var rune = Contents [row, col].Rune;
+                    char [] surrogatePair = rune.ToString ().ToCharArray ();
+                    Debug.Assert (surrogatePair.Length == 2);
+                    _outputBuffer [position].Char = surrogatePair [0];
+
+                    if (_outputBuffer [position].CombiningMarks == null)
+                    {
+                        _outputBuffer [position].CombiningMarks = [];
+                        _outputBuffer [position].CombiningMarks!.Add (surrogatePair [1]);
+                    }
+                    else
+                    {
+                        _outputBuffer [position].CombiningMarks!.Insert (0, surrogatePair [1]);
+                    }
 
                     if (Contents [row, col].Rune.GetColumns () > 1 && col + 1 < Cols)
                     {
                         // TODO: This is a hack to deal with non-BMP and wide characters.
                         col++;
                         position = row * Cols + col;
                         _outputBuffer [position].Empty = false;
-                        _outputBuffer [position].Char = ' ';
+                        _outputBuffer [position].Char = '\0';
                     }
                 }
             }

Terminal.Gui/Drawing/Cell.cs

@@ -4,7 +4,7 @@
 ///     Represents a single row/column in a Terminal.Gui rendering surface (e.g. <see cref="LineCanvas"/> and
 ///     <see cref="IConsoleDriver"/>).
 /// </summary>
-public record struct Cell (Attribute? Attribute = null, bool IsDirty = false, Rune Rune = default)
+public record struct Cell (Attribute? Attribute = null, bool IsDirty = false, Rune Rune = default, List<Rune> CombiningMarks = null)
 {
     /// <summary>The attributes to use when drawing the Glyph.</summary>
     public Attribute? Attribute { get; set; } = Attribute;
@@ -23,13 +23,11 @@ public Rune Rune
         get => _rune;
         set
         {
-            CombiningMarks.Clear ();
+            CombiningMarks = null;
             _rune = value;
         }
     }
 
-    private List<Rune> _combiningMarks;
-
     /// <summary>
     ///     The combining marks for <see cref="Rune"/> that when combined makes this Cell a combining sequence. If
     ///     <see cref="CombiningMarks"/> empty, then <see cref="CombiningMarks"/> is ignored.
@@ -38,11 +36,7 @@ public Rune Rune
     ///     Only valid in the rare case where <see cref="Rune"/> is a combining sequence that could not be normalized to a
     ///     single Rune.
     /// </remarks>
-    internal List<Rune> CombiningMarks
-    {
-        get => _combiningMarks ?? [];
-        private set => _combiningMarks = value ?? [];
-    }
+    internal List<Rune> CombiningMarks { get; set; } = CombiningMarks;
 
     /// <inheritdoc/>
     public override string ToString () { return $"[{Rune}, {Attribute}]"; }

Terminal.Gui/Text/RuneExtensions.cs

@@ -129,7 +129,8 @@ public static bool IsCombiningMark (this Rune rune)
 
         return Rune.GetUnicodeCategory (rune) == UnicodeCategory.NonSpacingMark
                || category == UnicodeCategory.SpacingCombiningMark
-               || category == UnicodeCategory.EnclosingMark;
+               || category == UnicodeCategory.EnclosingMark
+               || category == UnicodeCategory.Format;
     }
 
     /// <summary>Reports whether a rune is a surrogate code point.</summary>

UICatalog/Scenarios/CombiningMarks.cs

@@ -12,22 +12,43 @@ public override void Main ()
         var top = new Toplevel ();
 
         top.DrawComplete += (s, e) =>
-                                   {
-                                       top.Move (0, 0);
-                                       top.AddStr ("Terminal.Gui only supports combining marks that normalize. See Issue #2616.");
-                                       top.Move (0, 2);
-                                       top.AddStr ("\u0301\u0301\u0328<- \"\\u301\\u301\\u328]\" using AddStr.");
-                                       top.Move (0, 3);
-                                       top.AddStr ("[a\u0301\u0301\u0328]<- \"[a\\u301\\u301\\u328]\" using AddStr.");
-                                       top.Move (0, 4);
-                                       top.AddRune ('[');
-                                       top.AddRune ('a');
-                                       top.AddRune ('\u0301');
-                                       top.AddRune ('\u0301');
-                                       top.AddRune ('\u0328');
-                                       top.AddRune (']');
-                                       top.AddStr ("<- \"[a\\u301\\u301\\u328]\" using AddRune for each.");
-                                   };
+                            {
+                                var i = -1;
+                                top.Move (0, ++i);
+                                top.AddStr ("Terminal.Gui only supports combining marks that normalize. See Issue #2616.");
+                                top.Move (0, ++i);
+                                top.AddStr ("\u0301\u0301\u0328<- \"\\u0301\\u0301\\u0328]\" using AddStr.");
+                                top.Move (0, ++i);
+                                top.AddStr ("[a\u0301\u0301\u0328]<- \"[a\\u0301\\u0301\\u0328]\" using AddStr.");
+                                top.Move (0, ++i);
+                                top.AddRune ('[');
+                                top.AddRune ('a');
+                                top.AddRune ('\u0301');
+                                top.AddRune ('\u0301');
+                                top.AddRune ('\u0328');
+                                top.AddRune (']');
+                                top.AddStr ("<- \"[a\\u0301\\u0301\\u0328]\" using AddRune for each.");
+                                top.Move (0, ++i + 1);
+                                top.AddStr ("[e\u0301\u0301\u0328]<- \"[e\\u0301\\u0301\\u0328]\" using AddStr.");
+                                top.Move (0, ++i);
+                                top.AddStr ("[e\u0301\u0328\u0301]<- \"[e\\u0301\\u0328\\u0301]\" using AddStr.");
+                                top.Move (0, ++i);
+                                top.AddStr ("[e\u0328\u0301]<- \"[e\\u0328\\u0301]\" using AddStr.");
+                                i++;
+                                top.Move (0, ++i);
+                                top.AddStr ("From now on we are using TextFormatter");
+                                TextFormatter tf = new () { Text = "[e\u0301\u0301\u0328]<- \"[e\\u0301\\u0301\\u0328]\" using TextFormatter." };
+                                tf.Draw (new (0, ++i, tf.Text.Length, 1), top.ColorScheme.Normal, top.ColorScheme.Normal);
+                                tf.Text = "[e\u0328\u0301]<- \"[e\\u0328\\u0301]\" using TextFormatter.";
+                                tf.Draw (new (0, ++i, tf.Text.Length, 1), top.ColorScheme.Normal, top.ColorScheme.Normal);
+                                i++;
+                                top.Move (0, ++i);
+                                top.AddStr ("From now on we are using Surrogate pairs with combining diacritics");
+                                top.Move (0, ++i);
+                                top.AddStr ("[\ud835\udc4b\u0302]<- \"[\\ud835\\udc4b\\u0302]\" using AddStr.");
+                                top.Move (0, ++i);
+                                top.AddStr ("[\ud83e\uddd1\u200d\ud83e\uddd2]<- \"[\\ud83e\\uddd1\\u200d\\ud83e\\uddd2]\" using AddStr.");
+                            };
 
         Application.Run (top);
         top.Dispose ();

UnitTests/ConsoleDrivers/AddRuneTests.cs

@@ -55,15 +55,19 @@ public void AddRune_Accented_Letter_With_Three_Combining_Unicode_Chars ()
 
         text = "\u0103\u0301";
         driver.AddStr (text);
-        Assert.Equal (expected, driver.Contents [0, 0].Rune);
+        Assert.Single (driver.Contents [0, 0].CombiningMarks);
+        string combined = driver.Contents [0, 0].Rune + driver.Contents [0, 0].CombiningMarks [0].ToString ();
+        Assert.Equal (expected, (Rune)combined.Normalize (NormalizationForm.FormC) [0]);
         Assert.Equal ((Rune)' ', driver.Contents [0, 1].Rune);
 
         driver.ClearContents ();
         driver.Move (0, 0);
 
         text = "\u0061\u0306\u0301";
         driver.AddStr (text);
-        Assert.Equal (expected, driver.Contents [0, 0].Rune);
+        Assert.Equal (2, driver.Contents [0, 0].CombiningMarks.Count);
+        combined = driver.Contents [0, 0].Rune + driver.Contents [0, 0].CombiningMarks [0].ToString () + driver.Contents [0, 0].CombiningMarks [1];
+        Assert.Equal (expected, (Rune)combined.Normalize (NormalizationForm.FormC) [0]);
         Assert.Equal ((Rune)' ', driver.Contents [0, 1].Rune);
 
         //		var s = "a\u0301\u0300\u0306";

UnitTests/ConsoleDrivers/ContentsTests.cs

@@ -56,31 +56,31 @@ public void AddStr_With_Combining_Characters (Type driverType)
         // a + ogonek + acute = <U+0061, U+0328, U+0301> ( ą́ )
         var ogonek = new Rune (0x0328); // Combining ogonek (a small hook or comma shape)
         combined = "a" + ogonek + acuteaccent;
-        expected = ("a" + ogonek).Normalize (NormalizationForm.FormC); // See Issue #2616
+        expected = ("á" + ogonek).Normalize (NormalizationForm.FormC); // See Issue #2616
 
         driver.Move (0, 0);
         driver.AddStr (combined);
         TestHelpers.AssertDriverContentsAre (expected, output, driver);
 
         // e + ogonek + acute = <U+0061, U+0328, U+0301> ( ę́́ )
         combined = "e" + ogonek + acuteaccent;
-        expected = ("e" + ogonek).Normalize (NormalizationForm.FormC); // See Issue #2616
+        expected = ("é" + ogonek).Normalize (NormalizationForm.FormC); // See Issue #2616
 
         driver.Move (0, 0);
         driver.AddStr (combined);
         TestHelpers.AssertDriverContentsAre (expected, output, driver);
 
         // i + ogonek + acute = <U+0061, U+0328, U+0301> ( į́́́ )
         combined = "i" + ogonek + acuteaccent;
-        expected = ("i" + ogonek).Normalize (NormalizationForm.FormC); // See Issue #2616
+        expected = ("í" + ogonek).Normalize (NormalizationForm.FormC); // See Issue #2616
 
         driver.Move (0, 0);
         driver.AddStr (combined);
         TestHelpers.AssertDriverContentsAre (expected, output, driver);
 
         // u + ogonek + acute = <U+0061, U+0328, U+0301> ( ų́́́́ )
         combined = "u" + ogonek + acuteaccent;
-        expected = ("u" + ogonek).Normalize (NormalizationForm.FormC); // See Issue #2616
+        expected = ("ú" + ogonek).Normalize (NormalizationForm.FormC); // See Issue #2616
 
         driver.Move (0, 0);
         driver.AddStr (combined);