From 45b7fca6dbe1e68b44386166cbd5e9197ded34f1 Mon Sep 17 00:00:00 2001 From: Guillaume Lessard Date: Thu, 17 Apr 2025 15:24:34 -0700 Subject: [PATCH 1/3] [stdlib] internal UnsafeRawBufferPointer tweaks --- .../core/UnsafeRawBufferPointer.swift.gyb | 22 +++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/stdlib/public/core/UnsafeRawBufferPointer.swift.gyb b/stdlib/public/core/UnsafeRawBufferPointer.swift.gyb index 2ee50c1b77af2..cfa5af7adf7e1 100644 --- a/stdlib/public/core/UnsafeRawBufferPointer.swift.gyb +++ b/stdlib/public/core/UnsafeRawBufferPointer.swift.gyb @@ -100,6 +100,19 @@ public struct Unsafe${Mutable}RawBufferPointer { @usableFromInline internal let _position, _end: Unsafe${Mutable}RawPointer? + // This works around _debugPrecondition() impacting the performance of + // optimized code. (rdar://72246338) + @_alwaysEmitIntoClient + internal init( + @_nonEphemeral _uncheckedStart start: Unsafe${Mutable}RawPointer?, + count: Int + ) { + _internalInvariant(count >= 0) + _internalInvariant(unsafe count == 0 || start != nil) + unsafe _position = start + unsafe _end = start.map { unsafe $0 + _assumeNonNegative(count) } + } + /// Creates a buffer over the specified number of contiguous bytes starting /// at the given pointer. /// @@ -116,9 +129,14 @@ public struct Unsafe${Mutable}RawBufferPointer { _debugPrecondition(count >= 0, "${Self} with negative count") _debugPrecondition(unsafe count == 0 || start != nil, "${Self} has a nil start and nonzero count") + unsafe self.init(_uncheckedStart: start, count: count) + } - unsafe _position = start - unsafe _end = start.map { unsafe $0 + _assumeNonNegative(count) } + @safe + @_alwaysEmitIntoClient + public init(_empty: ()) { + unsafe _position = nil + unsafe _end = nil } } From aca36f5d40d04239076ddf31134f65734c71d667 Mon Sep 17 00:00:00 2001 From: Guillaume Lessard Date: Fri, 25 Apr 2025 10:50:51 -0700 Subject: [PATCH 2/3] [stdlib] refactor alignment checks --- stdlib/public/core/Hasher.swift | 4 +-- stdlib/public/core/Span/MutableSpan.swift | 10 +++--- stdlib/public/core/Span/Span.swift | 18 ++++------ .../public/core/UnsafeBufferPointer.swift.gyb | 18 +++++++++- stdlib/public/core/UnsafePointer.swift | 35 ++++++++++++++++--- .../core/UnsafeRawBufferPointer.swift.gyb | 17 ++++++--- stdlib/public/core/UnsafeRawPointer.swift | 28 ++++++++++----- 7 files changed, 91 insertions(+), 39 deletions(-) diff --git a/stdlib/public/core/Hasher.swift b/stdlib/public/core/Hasher.swift index f7650baf192f9..5de2806a1d569 100644 --- a/stdlib/public/core/Hasher.swift +++ b/stdlib/public/core/Hasher.swift @@ -227,9 +227,7 @@ extension Hasher { remaining -= c } } - _internalInvariant( - remaining == 0 || - Int(bitPattern: data) & (MemoryLayout.alignment - 1) == 0) + _internalInvariant(remaining == 0 || data._isWellAligned(for: UInt64.self)) // Load as many aligned words as there are in the input buffer while remaining >= MemoryLayout.size { diff --git a/stdlib/public/core/Span/MutableSpan.swift b/stdlib/public/core/Span/MutableSpan.swift index a9e40ca8167aa..f421faaf2fc42 100644 --- a/stdlib/public/core/Span/MutableSpan.swift +++ b/stdlib/public/core/Span/MutableSpan.swift @@ -72,9 +72,8 @@ extension MutableSpan where Element: ~Copyable { _unsafeElements buffer: UnsafeMutableBufferPointer ) { _precondition( - ((Int(bitPattern: buffer.baseAddress) & - (MemoryLayout.alignment &- 1)) == 0), - "baseAddress must be properly aligned to access Element" + buffer._isWellAligned(for: Element.self), + "buffer must be properly aligned to access Element" ) let ms = unsafe MutableSpan(_unchecked: buffer) self = unsafe _overrideLifetime(ms, borrowing: buffer) @@ -121,9 +120,8 @@ extension MutableSpan where Element: BitwiseCopyable { _unsafeBytes buffer: UnsafeMutableRawBufferPointer ) { _precondition( - ((Int(bitPattern: buffer.baseAddress) & - (MemoryLayout.alignment &- 1)) == 0), - "baseAddress must be properly aligned to access Element" + buffer._isWellAligned(for: Element.self), + "buffer must be properly aligned to access Element" ) let (byteCount, stride) = (buffer.count, MemoryLayout.stride) let (count, remainder) = byteCount.quotientAndRemainder(dividingBy: stride) diff --git a/stdlib/public/core/Span/Span.swift b/stdlib/public/core/Span/Span.swift index e27b7ac4f079a..fd582ee2b01d6 100644 --- a/stdlib/public/core/Span/Span.swift +++ b/stdlib/public/core/Span/Span.swift @@ -108,14 +108,11 @@ extension Span where Element: ~Copyable { public init( _unsafeElements buffer: UnsafeBufferPointer ) { - //FIXME: Workaround for https://github.com/swiftlang/swift/issues/77235 - let baseAddress = unsafe UnsafeRawPointer(buffer.baseAddress) _precondition( - ((Int(bitPattern: baseAddress) & - (MemoryLayout.alignment &- 1)) == 0), - "baseAddress must be properly aligned to access Element" + buffer._isWellAligned(for: Element.self), + "buffer must be properly aligned to access Element" ) - let span = unsafe Span(_unchecked: baseAddress, count: buffer.count) + let span = unsafe Span(_unchecked: buffer.baseAddress, count: buffer.count) // As a trivial value, 'baseAddress' does not formally depend on the // lifetime of 'buffer'. Make the dependence explicit. self = unsafe _overrideLifetime(span, borrowing: buffer) @@ -237,19 +234,16 @@ extension Span where Element: BitwiseCopyable { public init( _unsafeBytes buffer: UnsafeRawBufferPointer ) { - //FIXME: Workaround for https://github.com/swiftlang/swift/issues/77235 - let baseAddress = buffer.baseAddress _precondition( - ((Int(bitPattern: baseAddress) & - (MemoryLayout.alignment &- 1)) == 0), - "baseAddress must be properly aligned to access Element" + buffer._isWellAligned(for: Element.self), + "buffer must be properly aligned to access Element" ) let (byteCount, stride) = (buffer.count, MemoryLayout.stride) let (count, remainder) = byteCount.quotientAndRemainder(dividingBy: stride) _precondition( remainder == 0, "Span must contain a whole number of elements" ) - let span = unsafe Span(_unchecked: baseAddress, count: count) + let span = unsafe Span(_unchecked: buffer.baseAddress, count: count) // As a trivial value, 'baseAddress' does not formally depend on the // lifetime of 'buffer'. Make the dependence explicit. self = unsafe _overrideLifetime(span, borrowing: buffer) diff --git a/stdlib/public/core/UnsafeBufferPointer.swift.gyb b/stdlib/public/core/UnsafeBufferPointer.swift.gyb index ca385ef259525..173a77cc97592 100644 --- a/stdlib/public/core/UnsafeBufferPointer.swift.gyb +++ b/stdlib/public/core/UnsafeBufferPointer.swift.gyb @@ -780,6 +780,22 @@ extension Unsafe${Mutable}BufferPointer: % end } +extension Unsafe${Mutable}BufferPointer where Element: ~Copyable { + @safe + @_alwaysEmitIntoClient + public func _isWellAligned() -> Bool { + guard let p = baseAddress else { return true } + return p._isWellAligned() + } + + @safe + @_alwaysEmitIntoClient + public func _isWellAligned(for: T.Type) -> Bool { + guard let p = baseAddress else { return true } + return p._isWellAligned(for: T.self) + } +} + extension Unsafe${Mutable}BufferPointer { % if not Mutable: /// Creates a buffer over the same memory as the given buffer slice. @@ -1404,7 +1420,7 @@ extension Unsafe${Mutable}BufferPointer where Element: ~Copyable { } _debugPrecondition( - unsafe Int(bitPattern: .init(base)) & (MemoryLayout.alignment-1) == 0, + _isWellAligned(for: T.self), "baseAddress must be a properly aligned pointer for types Element and T" ) diff --git a/stdlib/public/core/UnsafePointer.swift b/stdlib/public/core/UnsafePointer.swift index db53ea3a0e57b..8d48534f096c8 100644 --- a/stdlib/public/core/UnsafePointer.swift +++ b/stdlib/public/core/UnsafePointer.swift @@ -400,8 +400,8 @@ extension UnsafePointer where Pointee: ~Copyable { capacity count: Int, _ body: (_ pointer: UnsafePointer) throws(E) -> Result ) throws(E) -> Result { - unsafe _debugPrecondition( - Int(bitPattern: .init(_rawValue)) & (MemoryLayout.alignment-1) == 0 && + _debugPrecondition( + _isWellAligned(for: T.self) && ( count == 1 || ( MemoryLayout.stride > MemoryLayout.stride ? MemoryLayout.stride % MemoryLayout.stride == 0 @@ -469,6 +469,19 @@ extension UnsafePointer where Pointee: ~Copyable { } } +extension UnsafePointer where Pointee: ~Copyable { + @safe + @_alwaysEmitIntoClient + public func _isWellAligned() -> Bool { + (Int(bitPattern: self) & (MemoryLayout.alignment &- 1)) == 0 + } + + @safe + @_alwaysEmitIntoClient + internal func _isWellAligned(for: T.Type) -> Bool { + UnsafeRawPointer(_rawValue)._isWellAligned(for: T.self) + } +} /// A pointer for accessing and manipulating data of a /// specific type. @@ -1248,8 +1261,8 @@ extension UnsafeMutablePointer where Pointee: ~Copyable { capacity count: Int, _ body: (_ pointer: UnsafeMutablePointer) throws(E) -> Result ) throws(E) -> Result { - unsafe _debugPrecondition( - Int(bitPattern: .init(_rawValue)) & (MemoryLayout.alignment-1) == 0 && + _debugPrecondition( + _isWellAligned(for: T.self) && ( count == 1 || ( MemoryLayout.stride > MemoryLayout.stride ? MemoryLayout.stride % MemoryLayout.stride == 0 @@ -1382,3 +1395,17 @@ extension UnsafeMutablePointer where Pointee: ~Copyable { )._unsafelyUnwrappedUnchecked } } + +extension UnsafeMutablePointer where Pointee: ~Copyable { + @safe + @_alwaysEmitIntoClient + public func _isWellAligned() -> Bool { + unsafe UnsafePointer(self)._isWellAligned() + } + + @safe + @_alwaysEmitIntoClient + internal func _isWellAligned(for: T.Type) -> Bool { + UnsafeRawPointer(_rawValue)._isWellAligned(for: T.self) + } +} diff --git a/stdlib/public/core/UnsafeRawBufferPointer.swift.gyb b/stdlib/public/core/UnsafeRawBufferPointer.swift.gyb index cfa5af7adf7e1..b94fea47a05c6 100644 --- a/stdlib/public/core/UnsafeRawBufferPointer.swift.gyb +++ b/stdlib/public/core/UnsafeRawBufferPointer.swift.gyb @@ -834,7 +834,7 @@ extension Unsafe${Mutable}RawBufferPointer { } _debugPrecondition( - Int(bitPattern: base) & (MemoryLayout.alignment-1) == 0, + _isWellAligned(for: S.Element.self), "buffer base address must be properly aligned to access S.Element" ) @@ -893,7 +893,7 @@ extension Unsafe${Mutable}RawBufferPointer { return unsafe .init(start: nil, count: 0) } _debugPrecondition( - Int(bitPattern: baseAddress) & (MemoryLayout.alignment-1) == 0, + _isWellAligned(for: C.Element.self), "buffer base address must be properly aligned to access C.Element" ) _precondition( @@ -918,7 +918,7 @@ extension Unsafe${Mutable}RawBufferPointer { } _internalInvariant(unsafe _end != nil) _debugPrecondition( - Int(bitPattern: baseAddress) & (MemoryLayout.alignment-1) == 0, + _isWellAligned(for: C.Element.self), "buffer base address must be properly aligned to access C.Element" ) var iterator = source.makeIterator() @@ -979,7 +979,7 @@ extension Unsafe${Mutable}RawBufferPointer { return unsafe .init(start: nil, count: 0) } _debugPrecondition( - Int(bitPattern: baseAddress) & (MemoryLayout.alignment-1) == 0, + _isWellAligned(for: T.self), "buffer base address must be properly aligned to access T" ) _precondition( @@ -1121,7 +1121,7 @@ extension Unsafe${Mutable}RawBufferPointer { return try unsafe body(.init(start: nil, count: 0)) } _debugPrecondition( - Int(bitPattern: s) & (MemoryLayout.alignment-1) == 0, + _isWellAligned(for: T.self), "baseAddress must be a properly aligned pointer for type T" ) // initializer ensures _end is nil only when _position is nil. @@ -1133,6 +1133,13 @@ extension Unsafe${Mutable}RawBufferPointer { return try unsafe body(.init(start: .init(s._rawValue), count: n)) } + @safe + @_alwaysEmitIntoClient + public func _isWellAligned(for type: T.Type) -> Bool { + guard let s = baseAddress else { return true } + return s._isWellAligned(for: T.self) + } + /// Returns a typed buffer to the memory referenced by this buffer, /// assuming that the memory is already bound to the specified type. /// diff --git a/stdlib/public/core/UnsafeRawPointer.swift b/stdlib/public/core/UnsafeRawPointer.swift index f3c4d16e04380..1d464a869e7e0 100644 --- a/stdlib/public/core/UnsafeRawPointer.swift +++ b/stdlib/public/core/UnsafeRawPointer.swift @@ -400,7 +400,7 @@ extension UnsafeRawPointer { _ body: (_ pointer: UnsafePointer) throws(E) -> Result ) throws(E) -> Result { _debugPrecondition( - Int(bitPattern: self) & (MemoryLayout.alignment-1) == 0, + _isWellAligned(for: T.self), "self must be a properly aligned pointer for type T" ) let binding = Builtin.bindMemory(_rawValue, count._builtinWordValue, T.self) @@ -446,8 +446,8 @@ extension UnsafeRawPointer { fromByteOffset offset: Int = 0, as type: T.Type ) -> T { - unsafe _debugPrecondition(0 == (UInt(bitPattern: self + offset) - & (UInt(MemoryLayout.alignment) - 1)), + unsafe _debugPrecondition( + (self + offset)._isWellAligned(for: T.self), "load from misaligned raw pointer") let rawPointer = unsafe (self + offset)._rawValue @@ -584,6 +584,12 @@ extension UnsafeRawPointer { return .init(Builtin.inttoptr_Word(bits._builtinWordValue)) } + @safe + @_alwaysEmitIntoClient + public func _isWellAligned(for type: T.Type) -> Bool { + unsafe self == self.alignedDown(for: type) + } + /// Obtain the next pointer whose bit pattern is a multiple of `alignment`. /// /// If the bit pattern of `self` is a multiple of `alignment`, @@ -1017,7 +1023,7 @@ extension UnsafeMutableRawPointer { _ body: (_ pointer: UnsafeMutablePointer) throws(E) -> Result ) throws(E) -> Result { _debugPrecondition( - Int(bitPattern: self) & (MemoryLayout.alignment-1) == 0, + _isWellAligned(for: T.self), "self must be a properly aligned pointer for type T" ) let binding = Builtin.bindMemory(_rawValue, count._builtinWordValue, T.self) @@ -1285,8 +1291,8 @@ extension UnsafeMutableRawPointer { fromByteOffset offset: Int = 0, as type: T.Type ) -> T { - unsafe _debugPrecondition(0 == (UInt(bitPattern: self + offset) - & (UInt(MemoryLayout.alignment) - 1)), + unsafe _debugPrecondition( + (self + offset)._isWellAligned(for: T.self), "load from misaligned raw pointer") let rawPointer = unsafe (self + offset)._rawValue @@ -1499,8 +1505,8 @@ extension UnsafeMutableRawPointer { internal func _legacy_se0349_storeBytes_internal( of value: T, toByteOffset offset: Int = 0, as type: T.Type ) { - unsafe _debugPrecondition(0 == (UInt(bitPattern: self + offset) - & (UInt(MemoryLayout.alignment) - 1)), + unsafe _debugPrecondition( + (self + offset)._isWellAligned(for: T.self), "storeBytes to misaligned raw pointer") var temp = value @@ -1588,6 +1594,12 @@ extension UnsafeMutableRawPointer { return .init(Builtin.inttoptr_Word(bits._builtinWordValue)) } + @safe + @_alwaysEmitIntoClient + public func _isWellAligned(for type: T.Type) -> Bool { + UnsafeRawPointer(self)._isWellAligned(for: type) + } + /// Obtain the next pointer whose bit pattern is a multiple of `alignment`. /// /// If the bit pattern of `self` is a multiple of `alignment`, From c5b3316728503927c26ee2f4c696bcdca3ebab0b Mon Sep 17 00:00:00 2001 From: Guillaume Lessard Date: Fri, 25 Apr 2025 10:51:26 -0700 Subject: [PATCH 3/3] [stdlib] the core of memory rebinding, unchecked --- stdlib/public/core/UnsafePointer.swift | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/stdlib/public/core/UnsafePointer.swift b/stdlib/public/core/UnsafePointer.swift index 8d48534f096c8..904369897780f 100644 --- a/stdlib/public/core/UnsafePointer.swift +++ b/stdlib/public/core/UnsafePointer.swift @@ -410,6 +410,19 @@ extension UnsafePointer where Pointee: ~Copyable { ), "self must be a properly aligned pointer for types Pointee and T" ) + return try unsafe _uncheckedWithMemoryRebound( + to: type, capacity: count, body) + } + + @_alwaysEmitIntoClient + @_transparent + internal func _uncheckedWithMemoryRebound< + T: ~Copyable, E: Error, Result: ~Copyable + >( + to type: T.Type, + capacity count: Int, + _ body: (_ pointer: UnsafePointer) throws(E) -> Result + ) throws(E) -> Result { let binding = Builtin.bindMemory(_rawValue, count._builtinWordValue, T.self) defer { Builtin.rebindMemory(_rawValue, binding) } return try unsafe body(.init(_rawValue))