Fixes #2616. Support combining sequences that don't normalize.

Terminal.Gui/Application/Application.cs

@@ -67,6 +67,28 @@ public static string ToString (IConsoleDriver? driver)
                 {
                     sb.Append (sp);
                 }
+                else if (contents [r, c].CombiningMarks is { Count: > 0 })
+                {
+                    // AtlasEngine does not support NON-NORMALIZED combining marks in a way
+                    // compatible with the driver architecture. Any CMs (except in the first col)
+                    // are correctly combined with the base char, but are ALSO treated as 1 column
+                    // width codepoints E.g. `echo "[e`u{0301}`u{0301}]"` will output `[é  ]`.
+                    // 
+                    // For now, we just ignore the list of CMs.
+                    string combine = rune.ToString ();
+                    string? normalized = null;
+
+                    foreach (Rune combMark in contents [r, c].CombiningMarks)
+                    {
+                        combine += combMark;
+                        normalized = combine.Normalize (NormalizationForm.FormC);
+                    }
+
+                    foreach (Rune enumerateRune in normalized!.EnumerateRunes ())
+                    {
+                        sb.Append (enumerateRune);
+                    }
+                }
                 else
                 {
                     sb.Append ((char)rune.Value);
@@ -76,11 +98,6 @@ public static string ToString (IConsoleDriver? driver)
                 {
                     c++;
                 }
-
-                // See Issue #2616
-                //foreach (var combMark in contents [r, c].CombiningMarks) {
-                //	sb.Append ((char)combMark.Value);
-                //}
             }
 
             sb.AppendLine ();

Terminal.Gui/ConsoleDrivers/ConsoleDriver.cs

@@ -1,6 +1,7 @@
 #nullable enable
 
 using System.Diagnostics;
+using System.Globalization;
 
 namespace Terminal.Gui;
 
@@ -54,6 +55,10 @@ public abstract class ConsoleDriver : IConsoleDriver
     // This is in addition to the dirty flag on each cell.
     internal bool []? _dirtyLines;
 
+    // Represent the necessary space character that must be added at
+    // the end of the line due the use of Format (Cf) unicode category
+    internal int []? _lineColsOffset;
+
     // QUESTION: When non-full screen apps are supported, will this represent the app size, or will that be in Application?
     /// <summary>Gets the location and size of the terminal screen.</summary>
     public Rectangle Screen => new (0, 0, Cols, Rows);
@@ -169,6 +174,13 @@ public virtual int Rows
     /// <summary>The topmost row in the terminal.</summary>
     public virtual int Top { get; set; } = 0;
 
+    /// <summary>
+    ///     Gets or sets whenever <see cref="Cell.CombiningMarks"/> is ignored or not.
+    /// </summary>
+    public static bool IgnoreIsCombiningMark { get; set; }
+
+    private Point? _lastValidAddRuneCell = null;
+
     /// <summary>Adds the specified rune to the display at the current cursor position.</summary>
     /// <remarks>
     ///     <para>
@@ -185,15 +197,15 @@ public virtual int Rows
     /// <param name="rune">Rune to add.</param>
     public void AddRune (Rune rune)
     {
-        int runeWidth = -1;
-        bool validLocation = IsValidLocation (rune, Col, Row);
-
         if (Contents is null)
         {
             return;
         }
 
+        int runeWidth = -1;
+        bool validLocation = IsValidLocation (rune, Col, Row);
         Rectangle clipRect = Clip!.GetBounds ();
+        bool wasAddedToCombiningMarks = false;
 
         if (validLocation)
         {
@@ -202,7 +214,7 @@ public void AddRune (Rune rune)
 
             lock (Contents)
             {
-                if (runeWidth == 0 && rune.IsCombiningMark ())
+                if (Rune.GetUnicodeCategory (rune) == UnicodeCategory.Format || (runeWidth == 0 && rune.IsCombiningMark ()))
                 {
                     // AtlasEngine does not support NON-NORMALIZED combining marks in a way
                     // compatible with the driver architecture. Any CMs (except in the first col)
@@ -212,50 +224,39 @@ public void AddRune (Rune rune)
                     // Until this is addressed (see Issue #), we do our best by
                     // a) Attempting to normalize any CM with the base char to it's left
                     // b) Ignoring any CMs that don't normalize
-                    if (Col > 0)
+                    if (Col > 0
+                        && _lastValidAddRuneCell is { }
+                        && _lastValidAddRuneCell.Value.Y == Row
+                        && Contents [Row, _lastValidAddRuneCell.Value.X].IsDirty)
                     {
-                        if (Contents [Row, Col - 1].CombiningMarks.Count > 0)
+                        if (Contents [Row, _lastValidAddRuneCell.Value.X].CombiningMarks is null)
                         {
-                            // Just add this mark to the list
-                            Contents [Row, Col - 1].CombiningMarks.Add (rune);
-
-                            // Ignore. Don't move to next column (let the driver figure out what to do).
+                            Contents [Row, _lastValidAddRuneCell.Value.X].CombiningMarks = [];
                         }
-                        else
-                        {
-                            // Attempt to normalize the cell to our left combined with this mark
-                            string combined = Contents [Row, Col - 1].Rune + rune.ToString ();
 
-                            // Normalize to Form C (Canonical Composition)
-                            string normalized = combined.Normalize (NormalizationForm.FormC);
-
-                            if (normalized.Length == 1)
-                            {
-                                // It normalized! We can just set the Cell to the left with the
-                                // normalized codepoint
-                                Contents [Row, Col - 1].Rune = (Rune)normalized [0];
+                        // Just add this mark to the list
+                        Contents [Row, _lastValidAddRuneCell.Value.X].CombiningMarks.Add (rune);
+                        wasAddedToCombiningMarks = true;
 
-                                // Ignore. Don't move to next column because we're already there
-                            }
-                            else
-                            {
-                                // It didn't normalize. Add it to the Cell to left's CM list
-                                Contents [Row, Col - 1].CombiningMarks.Add (rune);
-
-                                // Ignore. Don't move to next column (let the driver figure out what to do).
-                            }
+                        if (runeWidth == 0 && Rune.GetUnicodeCategory (rune) == UnicodeCategory.Format)
+                        {
+                            _lineColsOffset! [Row] += Contents [Row, _lastValidAddRuneCell.Value.X].Rune.GetColumns ();
                         }
 
-                        Contents [Row, Col - 1].Attribute = CurrentAttribute;
-                        Contents [Row, Col - 1].IsDirty = true;
+                        Contents [Row, _lastValidAddRuneCell.Value.X].Attribute = CurrentAttribute;
+                        Contents [Row, _lastValidAddRuneCell.Value.X].IsDirty = true;
                     }
                     else
                     {
                         // Most drivers will render a combining mark at col 0 as the mark
                         Contents [Row, Col].Rune = rune;
                         Contents [Row, Col].Attribute = CurrentAttribute;
                         Contents [Row, Col].IsDirty = true;
-                        Col++;
+
+                        if (runeWidth == 0)
+                        {
+                            Col++;
+                        }
                     }
                 }
                 else
@@ -325,7 +326,12 @@ public void AddRune (Rune rune)
             }
         }
 
-        if (runeWidth is < 0 or > 0)
+        if (!IgnoreIsCombiningMark && !wasAddedToCombiningMarks)
+        {
+            _lastValidAddRuneCell = new (Col, Row);
+        }
+
+        if (runeWidth is < 0 or > 0 && !wasAddedToCombiningMarks)
         {
             Col++;
         }
@@ -339,13 +345,10 @@ public void AddRune (Rune rune)
                 lock (Contents!)
                 {
                     // This is a double-width character, and we are not at the end of the line.
-                    // Col now points to the second column of the character. Ensure it doesn't
-                    // Get rendered.
-                    Contents [Row, Col].IsDirty = false;
+                    // Col now points to the second column of the character. Ensure it does get
+                    // rendered to allow the driver to handle it in its own way.
+                    Contents [Row, Col].IsDirty = true;
                     Contents [Row, Col].Attribute = CurrentAttribute;
-
-                    // TODO: Determine if we should wipe this out (for now now)
-                    //Contents [Row, Col].Rune = (Rune)' ';
                 }
             }
 
@@ -415,12 +418,14 @@ public void FillRect (Rectangle rect, Rune rune = default)
     public void ClearContents ()
     {
         Contents = new Cell [Rows, Cols];
+        _lastValidAddRuneCell = null;
 
         //CONCURRENCY: Unsynchronized access to Clip isn't safe.
         // TODO: ClearContents should not clear the clip; it should only clear the contents. Move clearing it elsewhere.
         Clip = new (Screen);
 
         _dirtyLines = new bool [Rows];
+        _lineColsOffset = new int [Rows];
 
         lock (Contents)
         {
@@ -499,11 +504,8 @@ public bool IsValidLocation (Rune rune, int col, int row)
         {
             return col >= 0 && row >= 0 && col < Cols && row < Rows && Clip!.Contains (col, row);
         }
-        else
-        {
 
-            return Clip!.Contains (col, row) || Clip!.Contains (col + 1, row);
-        }
+        return Clip!.Contains (col, row) || Clip!.Contains (col + 1, row);
     }
 
     /// <summary>Called when the terminal size changes. Fires the <see cref="SizeChanged"/> event.</summary>

Terminal.Gui/ConsoleDrivers/CursesDriver/CursesDriver.cs

@@ -318,31 +318,56 @@
 
                         outputWidth++;
                         Rune rune = Contents [row, col].Rune;
+
+                        if (rune == Rune.ReplacementChar)
+                        {
+                            continue;
+                        }
+
                         output.Append (rune);
 
-                        if (Contents [row, col].CombiningMarks.Count > 0)
+                        if (rune.IsCombiningMark ())
+                        {
+                            if (rune.GetColumns () == 0)
+                            {
+                                output.Append (' ');
+                            }
+                        }
+
+                        if (Contents [row, col].CombiningMarks is { Count: > 0 })
                         {
                             // AtlasEngine does not support NON-NORMALIZED combining marks in a way
                             // compatible with the driver architecture. Any CMs (except in the first col)
                             // are correctly combined with the base char, but are ALSO treated as 1 column
                             // width codepoints E.g. `echo "[e`u{0301}`u{0301}]"` will output `[é  ]`.
                             // 
                             // For now, we just ignore the list of CMs.
-                            //foreach (var combMark in Contents [row, col].CombiningMarks) {
-                            //	output.Append (combMark);
-                            //}
-                            // WriteToConsole (output, ref lastCol, row, ref outputWidth);
-                        }
-                        else if (rune.IsSurrogatePair () && rune.GetColumns () < 2)
-                        {
-                            WriteToConsole (output, ref lastCol, row, ref outputWidth);
-                            SetCursorPosition (col - 1, row);
+                            foreach (var combMark in Contents [row, col].CombiningMarks)
+                            {
+                                output.Append (combMark);
+                            }
+                            //WriteToConsole (output, ref lastCol, row, ref outputWidth);
                         }
+                        //else if (rune.IsSurrogatePair () && rune.GetColumns () < 2)
+                        //{
+                        //    WriteToConsole (output, ref lastCol, row, ref outputWidth);
+                        //    SetCursorPosition (col - 1, row);
+                        //}
 
                         Contents [row, col].IsDirty = false;
                     }
                 }
 
+                if (_lineColsOffset! [row] > 0)
+                {
+                    for (var i = 0; i < _lineColsOffset [row]; i++)
+                    {
+                        output.Append (' ');
+                    }
+
+                    _lineColsOffset! [row] = 0;
+                }
+
                 if (output.Length > 0)
                 {
                     SetCursorPosition (lastCol, row);
@@ -729,7 +754,7 @@
                 while (wch2 == Curses.KeyMouse)
                 {
                     // BUGBUG: Fix this nullable issue.
                     Key kea = null;
 
                     ConsoleKeyInfo [] cki =
                     {
@@ -739,7 +764,7 @@
                     };
                     code = 0;
                     // BUGBUG: Fix this nullable issue.
                     HandleEscSeqResponse (ref code, ref k, ref wch2, ref kea, ref cki);
                 }
 
                 return;
@@ -797,7 +822,7 @@
             }
 
             // BUGBUG: Fix this nullable issue.
             Key key = null;
 
             if (code == 0)
             {
@@ -828,7 +853,7 @@
                         new ((char)KeyCode.Esc, 0, false, false, false), new ((char)wch2, 0, false, false, false)
                     ];
                     // BUGBUG: Fix this nullable issue.
                     HandleEscSeqResponse (ref code, ref k, ref wch2, ref key, ref cki);
 
                     return;
                 }
@@ -980,7 +1005,7 @@
                 EscSeqUtils.DecodeEscSeq (
                                           ref consoleKeyInfo,
                                           ref ck,
                                           cki,
                                           ref mod,
                                           out _,
                                           out _,
@@ -1001,7 +1026,7 @@
                     }
 
                     // BUGBUG: Fix this nullable issue.
                     cki = null;
 
                     if (wch2 == 27)
                     {

Terminal.Gui/ConsoleDrivers/NetDriver/NetDriver.cs

@@ -166,31 +166,56 @@
 
                     outputWidth++;
                     Rune rune = Contents [row, col].Rune;
+
+                    if (rune == Rune.ReplacementChar)
+                    {
+                        continue;
+                    }
+
                     output.Append (rune);
 
-                    if (Contents [row, col].CombiningMarks.Count > 0)
+                    if (rune.IsCombiningMark ())
+                    {
+                        if (rune.GetColumns () == 0)
+                        {
+                            output.Append (' ');
+                        }
+                    }
+
+                    if (Contents [row, col].CombiningMarks is { Count: > 0 })
                     {
                         // AtlasEngine does not support NON-NORMALIZED combining marks in a way
                         // compatible with the driver architecture. Any CMs (except in the first col)
                         // are correctly combined with the base char, but are ALSO treated as 1 column
                         // width codepoints E.g. `echo "[e`u{0301}`u{0301}]"` will output `[é  ]`.
                         // 
                         // For now, we just ignore the list of CMs.
-                        //foreach (var combMark in Contents [row, col].CombiningMarks) {
-                        //	output.Append (combMark);
-                        //}
-                        // WriteToConsole (output, ref lastCol, row, ref outputWidth);
-                    }
-                    else if (rune.IsSurrogatePair () && rune.GetColumns () < 2)
-                    {
-                        WriteToConsole (output, ref lastCol, row, ref outputWidth);
-                        SetCursorPosition (col - 1, row);
+                        foreach (var combMark in Contents [row, col].CombiningMarks)
+                        {
+                            output.Append (combMark);
+                        }
+                        //WriteToConsole (output, ref lastCol, row, ref outputWidth);
                     }
+                    //else if (rune.IsSurrogatePair () && rune.GetColumns () < 2)
+                    //{
+                    //    WriteToConsole (output, ref lastCol, row, ref outputWidth);
+                    //    SetCursorPosition (col - 1, row);
+                    //}
 
                     Contents [row, col].IsDirty = false;
                 }
             }
 
+            if (_lineColsOffset! [row] > 0)
+            {
+                for (var i = 0; i < _lineColsOffset [row]; i++)
+                {
+                    output.Append (' ');
+                }
+
+                _lineColsOffset! [row] = 0;
+            }
+
             if (output.Length > 0)
             {
                 SetCursorPosition (lastCol, row);
@@ -226,7 +251,7 @@
 
     // BUGBUG: Fix this nullable issue.
     /// <inheritdoc />
     internal override IAnsiResponseParser GetParser () => _mainLoopDriver._netEvents.Parser;
     internal NetMainLoop? _mainLoopDriver;
 
     /// <inheritdoc />

Terminal.Gui/ConsoleDrivers/NetDriver/NetMainLoop.cs

@@ -139,6 +139,11 @@ private void NetInputHandler ()
                     _waitForProbe.Reset ();
                 }
 
+                if (_inputHandlerTokenSource.IsCancellationRequested)
+                {
+                    return;
+                }
+
                 ProcessInputQueue ();
             }
             catch (OperationCanceledException)