Fix inline predictive text and keyboard suggestion toolbar use cases.

.github/workflows/ios.yml

@@ -23,6 +23,10 @@ jobs:
 
     steps:
     - uses: actions/checkout@v4
+
+    - name: Setup keyboards
+      working-directory: platforms/ios/example
+      run: exec ./ios-keyboards-setup.sh
 
     - name: Install xcresultparser
       run: brew install a7ex/homebrew-formulae/xcresultparser
@@ -69,7 +73,7 @@ jobs:
         flags: unittests-ios, unittests
         # https://github.com/codecov/codecov-action/issues/557#issuecomment-1216749652
         token: ${{ secrets.CODECOV_TOKEN }}
-
+      
     - name: UI test coverage
       working-directory: platforms/ios/example
       run: exec ./ios-ui-test-coverage.sh

platforms/ios/example/Wysiwyg.xcodeproj/project.pbxproj

@@ -45,6 +45,7 @@
 		A6F4D0CF29AE0C1500087A3E /* Users.swift in Sources */ = {isa = PBXBuildFile; fileRef = A6F4D0CE29AE0C1500087A3E /* Users.swift */; };
 		A6F4D0D129AE0C3200087A3E /* Rooms.swift in Sources */ = {isa = PBXBuildFile; fileRef = A6F4D0D029AE0C3200087A3E /* Rooms.swift */; };
 		A6F4D0D329AE0C5100087A3E /* Commands.swift in Sources */ = {isa = PBXBuildFile; fileRef = A6F4D0D229AE0C5100087A3E /* Commands.swift */; };
+		A75C6AD22C3E989D0096D3A4 /* WysiwygUITests+Keyboard.swift in Sources */ = {isa = PBXBuildFile; fileRef = A75C6AD12C3E989D0096D3A4 /* WysiwygUITests+Keyboard.swift */; };
 /* End PBXBuildFile section */
 
 /* Begin PBXContainerItemProxy section */
@@ -97,6 +98,7 @@
 		A6F4D0CE29AE0C1500087A3E /* Users.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Users.swift; sourceTree = "<group>"; };
 		A6F4D0D029AE0C3200087A3E /* Rooms.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Rooms.swift; sourceTree = "<group>"; };
 		A6F4D0D229AE0C5100087A3E /* Commands.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Commands.swift; sourceTree = "<group>"; };
+		A75C6AD12C3E989D0096D3A4 /* WysiwygUITests+Keyboard.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "WysiwygUITests+Keyboard.swift"; sourceTree = "<group>"; };
 /* End PBXFileReference section */
 
 /* Begin PBXFrameworksBuildPhase section */
@@ -178,6 +180,7 @@
 				A64AB145296C759A00F08494 /* WysiwygUITests+Quotes.swift */,
 				A661FDA629B0ACB400E799A6 /* WysiwygUITests+Suggestions.swift */,
 				A64AB13D296C732500F08494 /* WysiwygUITests+Typing.swift */,
+				A75C6AD12C3E989D0096D3A4 /* WysiwygUITests+Keyboard.swift */,
 			);
 			path = WysiwygUITests;
 			sourceTree = "<group>";
@@ -280,6 +283,7 @@
 			isa = PBXNativeTarget;
 			buildConfigurationList = A6472CD12886CF840021A0E8 /* Build configuration list for PBXNativeTarget "WysiwygUITests" */;
 			buildPhases = (
+				A75C6AD32C3ECA450096D3A4 /* Force software keyboard on simulator */,
 				A6472CBD2886CF840021A0E8 /* Sources */,
 				A6472CBE2886CF840021A0E8 /* Frameworks */,
 				A6472CBF2886CF840021A0E8 /* Resources */,
@@ -392,6 +396,24 @@
 			shellPath = /bin/sh;
 			shellScript = "export PATH=\"$PATH:/opt/homebrew/bin\"\nif which swiftformat >/dev/null; then\n    swiftformat $PROJECT_DIR\n    swiftformat ../lib/WysiwygComposer\nelse\n    echo \"warning: SwiftFormat not installed, download from https://github.com/nicklockwood/SwiftFormat\"\nfi\n";
 		};
+		A75C6AD32C3ECA450096D3A4 /* Force software keyboard on simulator */ = {
+			isa = PBXShellScriptBuildPhase;
+			buildActionMask = 2147483647;
+			files = (
+			);
+			inputFileListPaths = (
+			);
+			inputPaths = (
+			);
+			name = "Force software keyboard on simulator";
+			outputFileListPaths = (
+			);
+			outputPaths = (
+			);
+			runOnlyForDeploymentPostprocessing = 0;
+			shellPath = /bin/sh;
+			shellScript = "external_kb_connected=false\n\nosascript -e 'quit app \"Simulator\"'\n\nSIMUS_KEYBOARD=$(/usr/libexec/PlistBuddy -c \"Print :DevicePreferences\" ~/Library/Preferences/com.apple.iphonesimulator.plist | perl -lne 'print $1 if /^    (\\S*) =/')\n\necho \"$SIMUS_KEYBOARD\" | while read -r a; do /usr/libexec/PlistBuddy -c \"Set :DevicePreferences:$a:ConnectHardwareKeyboard $external_kb_connected\" ~/Library/Preferences/com.apple.iphonesimulator.plist || /usr/libexec/PlistBuddy -c  \"Add :DevicePreferences:$a:ConnectHardwareKeyboard bool $external_kb_connected\" ~/Library/Preferences/com.apple.iphonesimulator.plist; done\n";
+		};
 /* End PBXShellScriptBuildPhase section */
 
 /* Begin PBXSourcesBuildPhase section */
@@ -432,6 +454,7 @@
 				A6BB18D729F9191C00EB6366 /* WysiwygUITests+Autocorrection.swift in Sources */,
 				A6472CC62886CF840021A0E8 /* WysiwygUITests.swift in Sources */,
 				A64AB140296C73CE00F08494 /* WysiwygUITests+Links.swift in Sources */,
+				A75C6AD22C3E989D0096D3A4 /* WysiwygUITests+Keyboard.swift in Sources */,
 				A661FDA729B0ACB400E799A6 /* WysiwygUITests+Suggestions.swift in Sources */,
 				A68E7141291D40710023CC04 /* WysiwygSharedConstants.swift in Sources */,
 				A64AB144296C747C00F08494 /* WysiwygUITests+Format.swift in Sources */,

platforms/ios/example/WysiwygUITests/WysiwygUITests+Keyboard.swift

@@ -0,0 +1,172 @@
+//
+// Copyright 2024 The Matrix.org Foundation C.I.C
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+import XCTest
+
+// These tests work on the assunmption that we always have the software keyboard enabled
+// which is handled through the run script
+// and that the simulator is configured with the following 2 keyboards: English (US) and Japanese Kana
+// Such configuration is done through a script called ios-keyboard-setup.sh but can also be done manually
+extension WysiwygUITests {
+    func testInlinePredictiveText() {
+        sleep(3)
+//        app.setupKeyboardLanguage(named: "English (US)")
+
+        // Sometimes autocorrection can break capitalisation, so we need to make sure the first letter is lowercase
+        app.keyboards.buttons["shift"].tap()
+        app.typeTextCharByCharUsingKeyboard("hello how a")
+        // We assert both the tree and textview content because the text view is containing the predictive text at that moment
+        // Which in the ui test is seen as part of the static text
+        assertTextViewContent("hello how are you")
+        assertTreeEquals(
+            """
+            └>"hello how a"
+            """
+        )
+        app.keys["space"].tap()
+        assertTextViewContent("hello how are you ")
+        assertTreeEquals(
+            """
+            └>"hello how are you "
+            """
+        )
+    }
+
+    func testInlinePredictiveTextIsIgnoredWhenSending() {
+        sleep(3)
+//        app.setupKeyboardLanguage(named: "English (US)")
+
+        // Sometimes autocorrection can break capitalisation, so we need to make sure the first letter is lowercase
+        app.keyboards.buttons["shift"].tap()
+        app.typeTextCharByCharUsingKeyboard("hello how")
+        // We assert both the tree and textview content because the text view is containing the predictive text at that moment
+        // Which in the ui test is seen as part of the static text
+        assertTextViewContent("hello how are you")
+        assertTreeEquals(
+            """
+            └>"hello how"
+            """
+        )
+        button(.sendButton).tap()
+        sleep(1)
+        assertContentText(plainText: "hello how", htmlText: "hello how")
+    }
+
+    func testInlinePredictiveTextIsIgnoredWhenDeleting() {
+        sleep(3)
+//        app.setupKeyboardLanguage(named: "English (US)")
+
+        // Sometimes autocorrection can break capitalisation, so we need to make sure the first letter is lowercase
+        app.keyboards.buttons["shift"].tap()
+        app.typeTextCharByCharUsingKeyboard("hello how")
+        app.keys["delete"].tap()
+        // We assert both the tree and textview content because the text view is containing the predictive text at that moment
+        // Which in the ui test is seen as part of the static text
+        assertTextViewContent("hello how are you")
+        assertTreeEquals(
+            """
+            └>"hello ho"
+            """
+        )
+        button(.sendButton).tap()
+        sleep(1)
+        assertContentText(plainText: "hello ho", htmlText: "hello ho")
+    }
+
+    func testDoubleSpaceIntoDot() {
+        sleep(3)
+//        app.setupKeyboardLanguage(named: "English (US)")
+
+        // Sometimes autocorrection can break capitalisation, so we need to make sure the first letter is lowercase
+        app.keyboards.buttons["shift"].tap()
+        app.typeTextCharByCharUsingKeyboard("hello")
+        app.keys["space"].tap()
+        app.keys["space"].tap()
+        assertTextViewContent("hello. ")
+        assertTreeEquals(
+            """
+            └>"hello. "
+            """
+        )
+    }
+
+    func testDotAfterInlinePredictiveText() {
+        sleep(3)
+//        app.setupKeyboardLanguage(named: "English (US)")
+
+        // Sometimes autocorrection can break capitalisation, so we need to make sure the first letter is lowercase
+        app.keyboards.buttons["shift"].tap()
+        app.typeTextCharByCharUsingKeyboard("hello how a")
+        // We assert both the tree and textview content because the text view is containing the predictive text at that moment
+        // Which in the ui test is seen as part of the static text
+        assertTextViewContent("hello how are you")
+        app.keys["space"].tap()
+        app.keys["more"].tap()
+        app.keys["."].tap()
+
+        // This optimisation to predictive inline text was introduced in 17.5
+        let correctText: String
+        if #available(iOS 17.5, *) {
+            correctText = "hello how are you."
+        } else {
+            correctText = "hello how are you ."
+        }
+        assertTextViewContent(correctText)
+        // In the failure case a second dot is added in the tree.
+        assertTreeEquals(
+            """
+            └>"\(correctText)"
+            """
+        )
+    }
+
+//    func testJapaneseKanaDeletion() {
+//        sleep(3)
+//        app.setupKeyboardLanguage(named: "日本語かな")
+//
+//        app.typeTextCharByCharUsingKeyboard("は")
+//        assertTextViewContent("は")
+//        assertTreeEquals(
+//            """
+//            └>"は"
+//            """
+//        )
+//        app.keys["delete"].tap()
+//        assertTextViewContent("")
+//        XCTAssertEqual(staticText(.treeText).label, "\n")
+//    }
+}
+
+private extension XCUIApplication {
+    func typeTextCharByCharUsingKeyboard(_ text: String) {
+        for char in text {
+            if char == " " {
+                keys["space"].tap()
+                continue
+            }
+            keys[String(char)].tap()
+        }
+    }
+
+    func setupKeyboardLanguage(named language: String) {
+        let nextKeyboard = buttons["Next keyboard"]
+        while let value = nextKeyboard.value as? String,
+              value != language {
+            nextKeyboard.tap()
+        }
+        nextKeyboard.tap()
+    }
+}

platforms/ios/example/WysiwygUITests/WysiwygUITests.swift

@@ -179,6 +179,11 @@ extension WysiwygUITests {
         XCTAssertTrue(pill.exists)
         XCTAssertEqual(pill.label, displayName)
     }
+
+    func assertContentText(plainText: String, htmlText: String) {
+        XCTAssert(staticText(.contentText).label == plainText)
+        XCTAssert(staticText(.htmlContentText).label == htmlText)
+    }
 }
 
 extension XCUIElement {

platforms/ios/example/ios-keyboards-setup.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+# Get the list of available simulators
+simulators=$(xcrun simctl list devices 17.2)
+
+# Find the current booted simulator
+test_simulator=$(echo "$simulators" | grep "iPhone 15 (" | awk -F '[()]' '{print $2}')
+
+echo "Found simulator: $test_simulator"
+
+# Define the path to the preferences plist file for the booted simulator
+plist_path=~/Library/Developer/CoreSimulator/Devices/$test_simulator/data/Library/Preferences/.GlobalPreferences.plist
+echo "Path: $plist_path"
+
+# Check if the plist file exists
+if [ ! -f "$plist_path" ]; then
+  echo "Preferences plist file not found: $plist_path"
+  exit 1
+fi
+
+# Clear the current keyboard layouts
+/usr/libexec/PlistBuddy -c "Delete :AppleKeyboards" "$plist_path" 2>/dev/null
+/usr/libexec/PlistBuddy -c "Add :AppleKeyboards array" "$plist_path"
+/usr/libexec/PlistBuddy -c "Add :AppleKeyboards:0 string 'en_US@sw=QWERTY;hw=Automatic'" "$plist_path"
+/usr/libexec/PlistBuddy -c "Add :AppleKeyboards:1 string 'ja_JP-Kana@sw=Kana;hw=Automatic'" "$plist_path"
+
+/usr/libexec/PlistBuddy -c "Print :AppleKeyboards array" "$plist_path" 2>/dev/null

platforms/ios/lib/WysiwygComposer/Sources/HTMLParser/Extensions/NSAttributedString+Range.swift

@@ -194,8 +194,8 @@ extension NSMutableAttributedString {
     /// - Returns: self (discardable)
     @discardableResult
     func removeDiscardableContent() -> Self {
-        discardableTextRanges().reversed().forEach {
-            replaceCharacters(in: $0, with: "")
+        for discardableTextRange in discardableTextRanges().reversed() {
+            replaceCharacters(in: discardableTextRange, with: "")
         }
 
         return self

platforms/ios/lib/WysiwygComposer/Sources/WysiwygComposer/Components/WysiwygComposerView/WysiwygComposerView.swift

@@ -85,7 +85,9 @@ public struct WysiwygComposerView: View {
 
     @ViewBuilder
     private var placeholderView: some View {
-        if viewModel.isContentEmpty, !viewModel.textView.isDictationRunning {
+        // The content can be empty but the textview not, e.g. if you start dictation
+        // but have not committed the text yet.
+        if viewModel.textView.attributedText.length == 0 {
             Text(placeholder)
                 .font(Font(UIFont.preferredFont(forTextStyle: .body)))
                 .foregroundColor(placeholderColor)
@@ -191,7 +193,10 @@ struct UITextViewWrapper: UIViewRepresentable {
                                       textView.logText,
                                       "Replacement: \"\(text)\""],
                                      functionName: #function)
-            return replaceText(range, text)
+            let change = replaceText(range, text)
+            Logger.textView.logDebug(["change: \(change)"],
+                                     functionName: #function)
+            return change
         }
 
         func textViewDidChange(_ textView: UITextView) {