Swift 6 support (#52)

* WIP

* working on the hybrid stuff

* Removing now-unneeded code, uncommenting map/catching

* Clean up package

* Remove those map/catching things, cannot quite get them right yet

* Simpler to just use an explicit argument

* Fix completionHandler type

* Local function workaround needed to fix race

* WIP

* It works

* Update readme

.editorconfig

@@ -0,0 +1,8 @@
+root = true
+
+[*]
+indent_style = tab
+end_of_line = lf
+charset = utf-8
+trim_trailing_whitespace = true
+insert_final_newline = true

Package.resolved

@@ -1,21 +1,21 @@
 {
+  "originHash" : "a818c4732d60e59901e640cf8c5de3900e8ec0ffaa1daa0a631b8676b0b17c4b",
   "pins" : [
     {
       "identity" : "rearrange",
       "kind" : "remoteSourceControl",
       "location" : "https://github.com/ChimeHQ/Rearrange",
       "state" : {
-        "revision" : "5ff7f3363f7a08f77e0d761e38e6add31c2136e1",
-        "version" : "1.8.1"
+        "revision" : "f1d74e1642956f0300756ad8d1d64e9034857bc3",
+        "version" : "2.0.0"
       }
     },
     {
       "identity" : "swifttreesitter",
       "kind" : "remoteSourceControl",
       "location" : "https://github.com/ChimeHQ/SwiftTreeSitter",
       "state" : {
-        "revision" : "36aa61d1b531f744f35229f010efba9c6d6cbbdd",
-        "version" : "0.9.0"
+        "revision" : "55f2c2fdaf859f86e4ea8513b9934badc7894019"
       }
     },
     {
@@ -28,5 +28,5 @@
       }
     }
   ],
-  "version" : 2
+  "version" : 3
 }

Package.swift

@@ -1,4 +1,4 @@
-// swift-tools-version: 5.8
+// swift-tools-version: 6.0
 
 import PackageDescription
 
@@ -15,28 +15,28 @@ let package = Package(
 		.library(name: "Neon", targets: ["Neon"]),
 	],
 	dependencies: [
-		.package(url: "https://github.com/ChimeHQ/SwiftTreeSitter", from: "0.9.0"),
-		.package(url: "https://github.com/ChimeHQ/Rearrange", from: "1.8.1"),
+		.package(url: "https://github.com/ChimeHQ/SwiftTreeSitter", revision: "55f2c2fdaf859f86e4ea8513b9934badc7894019"),
+		.package(url: "https://github.com/ChimeHQ/Rearrange", from: "2.0.0"),
 	],
 	targets: [
 		.target(name: "RangeState", dependencies: ["Rearrange"]),
 		.testTarget(name: "RangeStateTests", dependencies: ["RangeState"]),
 		.target(
 			name: "Neon",
 			dependencies: [
-                "RangeState",
-                "Rearrange",
-                "TreeSitterClient",
-                .product(name: "SwiftTreeSitterLayer", package: "SwiftTreeSitter"),
-            ]
+				"RangeState",
+				"Rearrange",
+				"TreeSitterClient",
+				.product(name: "SwiftTreeSitterLayer", package: "SwiftTreeSitter"),
+			]
 		),
 		.target(
 			name: "TreeSitterClient",
 			dependencies: [
 				"RangeState",
 				"Rearrange",
 				"SwiftTreeSitter",
-                .product(name: "SwiftTreeSitterLayer", package: "SwiftTreeSitter"),
+				.product(name: "SwiftTreeSitterLayer", package: "SwiftTreeSitter"),
 			]
 		),
 		.target(
@@ -50,13 +50,3 @@ let package = Package(
 		.testTarget(name: "TreeSitterClientTests", dependencies: ["TreeSitterClient", "NeonTestsTreeSitterSwift"])
 	]
 )
-
-let swiftSettings: [SwiftSetting] = [
-    .enableExperimentalFeature("StrictConcurrency")
-]
-
-for target in package.targets {
-    var settings = target.swiftSettings ?? []
-    settings.append(contentsOf: swiftSettings)
-    target.swiftSettings = settings
-}

Projects/NeonExample.xcodeproj/project.pbxproj

@@ -259,6 +259,10 @@
 				"INFOPLIST_KEY_UILaunchScreen_Generation[sdk=iphonesimulator*]" = YES;
 				"INFOPLIST_KEY_UIStatusBarStyle[sdk=iphoneos*]" = UIStatusBarStyleDefault;
 				"INFOPLIST_KEY_UIStatusBarStyle[sdk=iphonesimulator*]" = UIStatusBarStyleDefault;
+				IPHONEOS_DEPLOYMENT_TARGET = 16.0;
+				MACOSX_DEPLOYMENT_TARGET = 13.0;
+				TVOS_DEPLOYMENT_TARGET = 16.0;
+				WATCHOS_DEPLOYMENT_TARGET = 9.0;
 			};
 			name = Debug;
 		};
@@ -275,6 +279,10 @@
 				"INFOPLIST_KEY_UILaunchScreen_Generation[sdk=iphonesimulator*]" = YES;
 				"INFOPLIST_KEY_UIStatusBarStyle[sdk=iphoneos*]" = UIStatusBarStyleDefault;
 				"INFOPLIST_KEY_UIStatusBarStyle[sdk=iphonesimulator*]" = UIStatusBarStyleDefault;
+				IPHONEOS_DEPLOYMENT_TARGET = 16.0;
+				MACOSX_DEPLOYMENT_TARGET = 13.0;
+				TVOS_DEPLOYMENT_TARGET = 16.0;
+				WATCHOS_DEPLOYMENT_TARGET = 9.0;
 			};
 			name = Release;
 		};

Projects/NeonExample.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved

@@ -1,4 +1,5 @@
 {
+  "originHash" : "88a29eea1c1b10215ec2071d3cc04cae5d3b2418319c7e017e284ba383823031",
   "pins" : [
     {
       "identity" : "nsui",
@@ -13,16 +14,25 @@
       "kind" : "remoteSourceControl",
       "location" : "https://github.com/ChimeHQ/Rearrange",
       "state" : {
-        "revision" : "5ff7f3363f7a08f77e0d761e38e6add31c2136e1",
-        "version" : "1.8.1"
+        "revision" : "f1d74e1642956f0300756ad8d1d64e9034857bc3",
+        "version" : "2.0.0"
       }
     },
     {
       "identity" : "swifttreesitter",
       "kind" : "remoteSourceControl",
       "location" : "https://github.com/ChimeHQ/SwiftTreeSitter",
       "state" : {
-        "revision" : "b01904a3737649c1d8520106bbb285724fe5b0bb"
+        "revision" : "55f2c2fdaf859f86e4ea8513b9934badc7894019"
+      }
+    },
+    {
+      "identity" : "tree-sitter",
+      "kind" : "remoteSourceControl",
+      "location" : "https://github.com/tree-sitter/tree-sitter",
+      "state" : {
+        "revision" : "d97db6d63507eb62c536bcb2c4ac7d70c8ec665e",
+        "version" : "0.23.2"
       }
     },
     {
@@ -44,5 +54,5 @@
       }
     }
   ],
-  "version" : 2
+  "version" : 3
 }

Projects/NeonExample/TextViewController.swift

@@ -11,18 +11,14 @@ final class TextViewController: NSUIViewController {
 	private let highlighter: TextViewHighlighter
 
 	init() {
-		if #available(iOS 16.0, *) {
-			self.textView = NSUITextView(usingTextLayoutManager: false)
-		} else {
-			self.textView = NSUITextView()
-		}
+		self.textView = NSUITextView(usingTextLayoutManager: false)
 
 		self.highlighter = try! Self.makeHighlighter(for: textView)
 
 		super.init(nibName: nil, bundle: nil)
 
 		// enable non-continguous layout for TextKit 1
-        if #available(macOS 12.0, iOS 16.0, tvOS 15.0, *), textView.textLayoutManager == nil {
+		if textView.textLayoutManager == nil {
 			textView.nsuiLayoutManager?.allowsNonContiguousLayout = true
 		}
 	}
@@ -119,17 +115,17 @@ final class TextViewController: NSUIViewController {
 		// it wasn't that way on creation
 		highlighter.observeEnclosingScrollView()
 
-		regularTest()
+		regularTestWithSwiftCode()
 	}
 
-	func regularTest() {
+	func regularTestWithSwiftCode() {
 		let url = Bundle.main.url(forResource: "test", withExtension: "code")!
 		let content = try! String(contentsOf: url)
 
 		textView.text = content
 	}
 
-	func doBigTest() {
+	func doBigMarkdownTest() {
 		let url = Bundle.main.url(forResource: "big_test", withExtension: "md")!
 		let content = try! String(contentsOf: url)
 

README.md

@@ -51,7 +51,7 @@ Neon is made up of three parts: the core library, `RangeState` and `TreeSitterCl
 
 Neon's lowest-level component is called RangeState. This module contains the core building blocks used for the rest of the system. RangeState is built around the idea of hybrid synchronous/asynchronous execution. Making everything async is a lot easier, but that makes it impossible to provide a low-latency path for small documents. It is content-independent.
 
-- `Hybrid(Throwing)ValueProvider`: a fundamental type that defines work in terms of both synchronous and asynchronous functions
+- `HybridSyncAsyncValueProvider`: a fundamental type that defines work in terms of both synchronous and asynchronous functions
 - `RangeProcessor`: performs on-demand processing of range-based content (think parsing)
 - `RangeValidator`: building block for managing the validation of range-based content
 - `RangeInvalidationBuffer`: buffer and consolidate invalidations so they can be applied at the optimal time
@@ -60,8 +60,6 @@ Neon's lowest-level component is called RangeState. This module contains the cor
 
 Many of these support versionable content. If you are working with a backing store structure that supports efficient versioning, like a [piece table](https://en.wikipedia.org/wiki/Piece_table), expressing this to RangeState can improve its efficiency.
 
-It might be surprising to see that many of the types in RangeState are marked `@MainActor`. Right now, I have found no way to both support the hybrid sync/async functionality while also not being tied to a global actor. I think this is the most resonable trade-off, but I would very much like to lift this restriction. However, I believe it will require [language changes](https://forums.swift.org/t/isolation-assumptions/69514/47).
-
 ### Neon
 
 The top-level module includes systems for managing text styling. It is also text-system independent. It makes very few assumptions about how text is stored, displayed, or styled. It also includes some components for use with stock AppKit and UIKit systems. These are provided for easy integration, not maximum performance. 
@@ -84,7 +82,9 @@ I have not yet figured out a way to do this with TextKit 2, and it may not be po
 
 #### Performance
 
-Neon's performance is highly dependant on the text system integration. Every aspect is important as there are performance cliffs all around. But, priority range calcations (the visible set for most text views) are of particular importance. This is surprisingly challenging to do correctly with TextKit 1, and extremely hard with TextKit 2.
+Neon's performance is highly dependant on the text system integration. Every aspect is important as there are performance cliffs all around. But, priority range calculations (the visible set for most text views) are of particular importance. This is surprisingly challenging to do correctly with TextKit 1, and extremely hard with TextKit 2.
+
+Bottom line: Neon is extremely efficient. The bottlenecks will probably be your parsing system and text view. It can do a decent job of hiding parsing performance issues too. However, that doesn't mean it's perfect. There's always room for improvement and if you suspect a problem please do file an issue.
 
 ### TreeSitterClient
 
@@ -106,12 +106,16 @@ Neon was designed to accept and overlay token data from multiple sources simulta
 - Second pass: [tree-sitter](https://tree-sitter.github.io/tree-sitter/), which has good quality and **could** be low-latency
 - Third pass: [Language Server Protocol](https://microsoft.github.io/language-server-protocol/)'s [semantic tokens](https://microsoft.github.io/language-server-protocol/specifications/specification-current/#textDocument_semanticTokens), which can augment existing highlighting, but is high-latency
 
+An example of a first-pass system you might be interested in is [Lowlight](https://github.com/ChimeHQ/Lowlight).
+
 ### Theming
 
 A highlighting theme is really just a mapping from semantic labels to styles. Token data sources apply the semantic labels and the `TextSystemInterface` uses those labels to look up styling.
 
 This separation makes it very easy for you to do this look-up in a way that makes the most sense for whatever theming formats you'd like to support. This is also a convenient spot to adapt/modify the semantic labels coming from your data sources into a normalized form.
 
+If you are looking for some help here, check out [ThemePark](https://github.com/ChimeHQ/ThemePark).
+
 ## Usage
 
 ### TreeSitterClient
@@ -139,16 +143,15 @@ let clientConfig = TreeSitterClient.Configuration(
         // `languageConfigurationChanged(for:)`
         return nil
     },
-    contentProvider: { [textView] length in
-        // given a maximum needed length, produce a `Content` structure
-        // that will be used to access the text data
+    contentSnapshotProvider: { [textView] length in
+        // given a maximum needed length, produce a `ContentSnapshot` structure
+        // that will be used to access immutable text data
 
         // this can work for any system that efficiently produce a `String`
         return .init(string: textView.string)
     },
     lengthProvider: { [textView] in
         textView.string.utf16.count
-
     },
     invalidationHandler: { set in
         // take action on invalidated regions of the text

Sources/Neon/TextSystemInterface+Validation.swift

@@ -44,7 +44,10 @@ extension TextSystemInterface {
 				return validation(for: application, in: contentRange)
 			},
 			mainActorAsyncValue: { contentRange in
-				return await asyncValidate(contentRange, provider: provider.mainActorAsync)
+				await asyncValidate(
+					contentRange,
+					provider: { range in await provider.async(range) }
+				)
 			}
 		)
 	}
@@ -69,6 +72,10 @@ extension TextSystemInterface {
 	func validatorSecondaryHandler(
 		with provider: @escaping Styler.SecondaryValidationProvider
 	) -> SecondaryValidationProvider {
-		{ await asyncValidate($0, provider: provider) }
+		{ range in
+			await asyncValidate(range, provider: {
+				await provider($0)
+			})
+		}
 	}
 }

Sources/Neon/TextSystemStyler.swift

@@ -29,7 +29,8 @@ public final class TextSystemStyler<Interface: TextSystemInterface> {
 				versionedContent: textSystem.content,
 				provider: tokenValidator.validationProvider,
 				prioritySetProvider: { textSystem.visibleSet }
-			)
+			),
+            isolation: MainActor.shared
 		)
 	}
 
@@ -57,7 +58,7 @@ public final class TextSystemStyler<Interface: TextSystemInterface> {
 	public func visibleContentDidChange() {
 		let prioritySet = textSystem.visibleSet
 
-		validator.validate(.set(prioritySet), prioritizing: prioritySet)
+        validator.validate(.set(prioritySet), prioritizing: prioritySet, isolation: MainActor.shared)
 	}
 
 
@@ -68,12 +69,12 @@ public final class TextSystemStyler<Interface: TextSystemInterface> {
 	public func validate(_ target: RangeTarget) {
 		let prioritySet = textSystem.visibleSet
 
-		validator.validate(target, prioritizing: prioritySet)
+		validator.validate(target, prioritizing: prioritySet, isolation: MainActor.shared)
 	}
 
 	public func validate() {
 		let prioritySet = textSystem.visibleSet
 
-		validator.validate(.set(prioritySet), prioritizing: prioritySet)
+		validator.validate(.set(prioritySet), prioritizing: prioritySet, isolation: MainActor.shared)
 	}
 }

Sources/Neon/TextViewHighlighter.swift

@@ -35,6 +35,7 @@ extension TextView {
 /// A class that can connect `NSTextView`/`UITextView` to `TreeSitterClient`
 ///
 /// This class is a minimal implementation that can help perform highlighting for a TextView. It is compatible with both TextKit 1 and 2 views, and uses single-phase pass with tree-sitter. The created instance will become the delegate of the view's `NSTextStorage`.
+@available(macOS 13.0, macCatalyst 16.0, iOS 16.0, tvOS 16.0, watchOS 9.0, *)
 @MainActor
 public final class TextViewHighlighter {
 	private typealias Styler = TextSystemStyler<TextViewSystemInterface>
@@ -87,6 +88,7 @@ public final class TextViewHighlighter {
 			configuration: .init(
 				languageProvider: configuration.languageProvider,
 				contentProvider: { [interface] in interface.languageLayerContent(with: $0) },
+				contentSnapshopProvider: { [interface] in interface.languageLayerContentSnapshot(with: $0) },
 				lengthProvider: { [interface] in interface.content.currentLength },
 				invalidationHandler: { [buffer] in buffer.invalidate(.set($0)) },
 				locationTransformer: configuration.locationTransformer
@@ -131,7 +133,7 @@ public final class TextViewHighlighter {
 
 		try textView.getTextStorage().delegate = storageDelegate
 
-        observeEnclosingScrollView()
+		observeEnclosingScrollView()
 
 		invalidate(.all)
 	}
@@ -147,6 +149,7 @@ public final class TextViewHighlighter {
 	}
 }
 
+@available(macOS 13.0, iOS 16.0, tvOS 16.0, watchOS 9.0, *)
 extension TextViewHighlighter {
 	/// Begin monitoring for containing scroll view changes.
 	///
@@ -157,7 +160,7 @@ extension TextViewHighlighter {
 			print("warning: there is no enclosing scroll view")
 			return
 		}
-
+		
 		NotificationCenter.default.addObserver(
 			self,
 			selector: #selector(visibleContentChanged(_:)),
@@ -173,7 +176,7 @@ extension TextViewHighlighter {
 		)
 #elseif os(iOS) || os(visionOS)
 		self.frameObservation = textView.observe(\.contentOffset) { [weak self] view, _ in
-            MainActor.assumeIsolated {
+			MainActor.assumeIsolated {
 				guard let self = self else { return }
 
 				self.lastVisibleRange = self.textView.visibleTextRange