From e5de7b6914e899fd492d2069c26eda9c9ee321cd Mon Sep 17 00:00:00 2001 From: Matt Bauman Date: Tue, 27 Feb 2018 13:44:38 -0600 Subject: [PATCH 01/12] Optimize dataids(::AbstractArray) I was previously calling objectid -- I should have been using pointer_from_objref. The former is a hash, the latter is much quicker and easier to compute. --- base/abstractarray.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/abstractarray.jl b/base/abstractarray.jl index 8571cc8ee90a5..de4277b7672f1 100644 --- a/base/abstractarray.jl +++ b/base/abstractarray.jl @@ -1529,7 +1529,7 @@ parts can specialize this method to return the concatenation of the `dataids` of their component parts. A typical definition for an array that wraps a parent is `Base.dataids(C::CustomArray) = dataids(C.parent)`. """ -dataids(A::AbstractArray) = (UInt(objectid(A)),) +dataids(A::AbstractArray) = (UInt(pointer_from_objref(A)),) dataids(A::Array) = (UInt(pointer(A)),) dataids(::AbstractRange) = () dataids(x) = () From b57a2a08eccaa389da2c74b1f470e91c67a5c31f Mon Sep 17 00:00:00 2001 From: Matt Bauman Date: Fri, 20 Jul 2018 14:22:08 -0400 Subject: [PATCH 02/12] slightly better abstract definition but not reliably inferable for nested structures --- base/abstractarray.jl | 9 +++++++- test/abstractarray.jl | 53 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 1 deletion(-) diff --git a/base/abstractarray.jl b/base/abstractarray.jl index de4277b7672f1..324f27369b9e2 100644 --- a/base/abstractarray.jl +++ b/base/abstractarray.jl @@ -1529,7 +1529,14 @@ parts can specialize this method to return the concatenation of the `dataids` of their component parts. A typical definition for an array that wraps a parent is `Base.dataids(C::CustomArray) = dataids(C.parent)`. """ -dataids(A::AbstractArray) = (UInt(pointer_from_objref(A)),) +function dataids(A::AbstractArray) + @inline + ids = _splatmap(dataids, ntuple(i -> getfield(A, i), Val(nfields(A)))) + if !isimmutable(A) + ids = (UInt(pointer_from_objref(A)), ids...) + end + return ids +end dataids(A::Array) = (UInt(pointer(A)),) dataids(::AbstractRange) = () dataids(x) = () diff --git a/test/abstractarray.jl b/test/abstractarray.jl index 47a81bbbf5b66..3457804fddc77 100644 --- a/test/abstractarray.jl +++ b/test/abstractarray.jl @@ -1309,6 +1309,59 @@ end end end +# Ensure dataids are inferrable for custom arrays +struct M0 <: AbstractArray{Int,2} end +struct M1{T} <: AbstractArray{Int,2} + x::T +end +struct M2{T,S} <: AbstractArray{Int,2} + x::T + y::S +end +struct M10{A,B,C,D,E,F,G,H,I,J} <: AbstractArray{Int,2} + a::A + b::B + c::C + d::D + e::E + f::F + g::G + h::H + i::I + j::J +end + +@testset "dataids" begin + @test @inferred(Base.dataids(M0())) === () + @test @inferred(Base.dataids(M1(1))) === () + @test @inferred(Base.dataids(M1(1:10))) === () + @test @inferred(Base.dataids(M10(1,2,3,4,5,6,7,8,9,0))) === () + + @test @inferred(Base.dataids(M1(M1([1])))) != Base.dataids(M1(M1([1]))) + @test @inferred(Base.dataids(M1(M2([1],2)))) != Base.dataids(M1(M2([1],2))) + @test @inferred(Base.dataids(M1(M2([1],[2])))) != Base.dataids(M1(M2([1],[2]))) + @test @inferred(Base.dataids(M10([1],[2],[3],[4],[5],[6],[7],[8],[9],[0]))) != Base.dataids(M10([1],[2],[3],[4],[5],[6],[7],[8],[9],[0])) + + x = [1] + y = [1] + mx = M1(x) + mxx = M2(x,x) + mxy = M2(x,y) + @test @inferred(Base.mightalias(mx,mx)) + @test @inferred(Base.mightalias(mx,mxx)) + @test @inferred(Base.mightalias(mx,mxy)) + @test @inferred(Base.mightalias(mxx,x)) + @test @inferred(Base.mightalias(x,mxy)) + @test !@inferred(Base.mightalias(mx, y)) + @test !@inferred(Base.mightalias(mxx, y)) + @test !@inferred(Base.mightalias(mxx, [1])) + @test !@inferred(Base.mightalias(mxy, 1:10)) + @test !@inferred(Base.mightalias(mxy, M0())) + @test !@inferred(Base.mightalias(mxy, [1])) + @test !@inferred(Base.mightalias(mxy, M1(1:10))) + @test !@inferred(Base.mightalias(mxy, M1([1]))) +end + @testset "Base.rest" begin a = reshape(1:4, 2, 2)' @test Base.rest(a) == a[:] From 77f7a9287fd76048be8ccd4c4d8bc44ec48d96d9 Mon Sep 17 00:00:00 2001 From: Martin Holters Date: Wed, 22 Aug 2018 11:14:44 +0200 Subject: [PATCH 03/12] Provide custom `dataids` for `Tridiagonal` --- stdlib/LinearAlgebra/src/tridiag.jl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/stdlib/LinearAlgebra/src/tridiag.jl b/stdlib/LinearAlgebra/src/tridiag.jl index 70b39ef37e203..c6dd67d535677 100644 --- a/stdlib/LinearAlgebra/src/tridiag.jl +++ b/stdlib/LinearAlgebra/src/tridiag.jl @@ -591,6 +591,8 @@ similar(M::Tridiagonal, ::Type{T}, dims::Union{Dims{1},Dims{2}}) where {T} = sim # Operations on Tridiagonal matrices copyto!(dest::Tridiagonal, src::Tridiagonal) = (copyto!(dest.dl, src.dl); copyto!(dest.d, src.d); copyto!(dest.du, src.du); dest) +Base.dataids(A::Tridiagonal) = (Base.dataids(A.dl), Base.dataids(A.d), Base.dataids(A.du)) + #Elementary operations for func in (:conj, :copy, :real, :imag) @eval function ($func)(M::Tridiagonal) From ae2d5bc068d57794039f674afd86a8d60e359255 Mon Sep 17 00:00:00 2001 From: Martin Holters Date: Wed, 22 Aug 2018 14:01:25 +0200 Subject: [PATCH 04/12] Further refine fallback dataids --- base/abstractarray.jl | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/base/abstractarray.jl b/base/abstractarray.jl index 324f27369b9e2..3f2902a36dbcb 100644 --- a/base/abstractarray.jl +++ b/base/abstractarray.jl @@ -1531,11 +1531,16 @@ their component parts. A typical definition for an array that wraps a parent is """ function dataids(A::AbstractArray) @inline - ids = _splatmap(dataids, ntuple(i -> getfield(A, i), Val(nfields(A)))) - if !isimmutable(A) - ids = (UInt(pointer_from_objref(A)), ids...) + if @generated + :(ids = tuple($([:(dataids(getfield(A, $i))...) for i in 1:fieldcount(A)]...))) + else + ids = _splatmap(dataids, ntuple(i -> getfield(A, i), Val(nfields(A)))) + end + if isimmutable(A) || !isempty(ids) + return ids + else + return (UInt(pointer_from_objref(A)),) end - return ids end dataids(A::Array) = (UInt(pointer(A)),) dataids(::AbstractRange) = () From ef1d35105b157a950cbfff0dffbb048ad37ef406 Mon Sep 17 00:00:00 2001 From: Martin Holters Date: Wed, 19 Aug 2020 10:54:46 +0200 Subject: [PATCH 05/12] Add custom implementation of `unaliascopy(::LogicalIndex)` --- base/multidimensional.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/base/multidimensional.jl b/base/multidimensional.jl index f793df068ec5a..240cef1c05b2c 100644 --- a/base/multidimensional.jl +++ b/base/multidimensional.jl @@ -790,6 +790,7 @@ LogicalIndex{Int}(mask::AbstractArray) = LogicalIndex{Int, typeof(mask)}(mask) size(L::LogicalIndex) = (L.sum,) length(L::LogicalIndex) = L.sum collect(L::LogicalIndex) = [i for i in L] +unaliascopy(L::TL) where {TL <: LogicalIndex} = TL(unaliascopy(L.mask)) show(io::IO, r::LogicalIndex) = print(io,collect(r)) print_array(io::IO, X::LogicalIndex) = print_array(io, collect(X)) # Iteration over LogicalIndex is very performance-critical, but it also must From 83d9a1cf48a3bf5045fe6f137eb52f147a84ecd7 Mon Sep 17 00:00:00 2001 From: Matt Bauman Date: Mon, 7 Aug 2023 12:27:19 -0400 Subject: [PATCH 06/12] add dataids(::AbstractDictionary) --- base/abstractarray.jl | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/base/abstractarray.jl b/base/abstractarray.jl index 3f2902a36dbcb..ec7f22242517e 100644 --- a/base/abstractarray.jl +++ b/base/abstractarray.jl @@ -1546,6 +1546,11 @@ dataids(A::Array) = (UInt(pointer(A)),) dataids(::AbstractRange) = () dataids(x) = () +# While dictionaries aren't typically involved in aliasing detection, some arrays +# (like DefaultArray) do use dictionaries as a backing element for their data. +# Defining this ensures those fields are appropriately involved in aliasing decisions. +dataids(dict::AbstractDict) = (UInt(pointer_from_objref(dict)),) + ## get (getindex with a default value) ## RangeVecIntList{A<:AbstractVector{Int}} = Union{Tuple{Vararg{Union{AbstractRange, AbstractVector{Int}}}}, From 67348e03eee8f78fdac80d757140a39389c063ca Mon Sep 17 00:00:00 2001 From: Matt Bauman Date: Mon, 7 Aug 2023 13:06:37 -0400 Subject: [PATCH 07/12] update docstring --- base/abstractarray.jl | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/base/abstractarray.jl b/base/abstractarray.jl index ec7f22242517e..3c3a1ea59ba9d 100644 --- a/base/abstractarray.jl +++ b/base/abstractarray.jl @@ -1524,10 +1524,9 @@ _isdisjoint(as::Tuple, bs::Tuple) = !(as[1] in bs) && _isdisjoint(tail(as), bs) Return a tuple of `UInt`s that represent the mutable data segments of an array. -Custom arrays that would like to opt-in to aliasing detection of their component -parts can specialize this method to return the concatenation of the `dataids` of -their component parts. A typical definition for an array that wraps a parent is -`Base.dataids(C::CustomArray) = dataids(C.parent)`. +The default implementation recursively combines the `dataids` of all fields of the struct. +Custom arrays only need to implement a custom `dataids` method if they depend upon non-array +fields to define their contents and are immutable. """ function dataids(A::AbstractArray) @inline From 5040206766283a15c5a5d22f6372982e1073b36d Mon Sep 17 00:00:00 2001 From: Matt Bauman Date: Tue, 8 Aug 2023 09:24:50 -0400 Subject: [PATCH 08/12] more careful dict handling only use pointer_from_objref for Dict itself; otherwise use objectid as some AbstractDicts are immutable --- base/abstractarray.jl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/base/abstractarray.jl b/base/abstractarray.jl index 3c3a1ea59ba9d..65ca38a9bfd67 100644 --- a/base/abstractarray.jl +++ b/base/abstractarray.jl @@ -1547,8 +1547,9 @@ dataids(x) = () # While dictionaries aren't typically involved in aliasing detection, some arrays # (like DefaultArray) do use dictionaries as a backing element for their data. -# Defining this ensures those fields are appropriately involved in aliasing decisions. -dataids(dict::AbstractDict) = (UInt(pointer_from_objref(dict)),) +# Defining this ensures those fields are involved in aliasing decisions. +dataids(dict::Dict) = (UInt(pointer_from_objref(dict)),) +dataids(dict::AbstractDict) = (UInt(objectid(dict)),) ## get (getindex with a default value) ## From 4832890cc30f71d077256a3c0e8bf5c98e7db766 Mon Sep 17 00:00:00 2001 From: Matt Bauman Date: Tue, 8 Aug 2023 14:48:57 -0400 Subject: [PATCH 09/12] remove dataids(::Dict) micro-optimization --- base/abstractarray.jl | 1 - 1 file changed, 1 deletion(-) diff --git a/base/abstractarray.jl b/base/abstractarray.jl index 65ca38a9bfd67..ddc1e6aad0827 100644 --- a/base/abstractarray.jl +++ b/base/abstractarray.jl @@ -1548,7 +1548,6 @@ dataids(x) = () # While dictionaries aren't typically involved in aliasing detection, some arrays # (like DefaultArray) do use dictionaries as a backing element for their data. # Defining this ensures those fields are involved in aliasing decisions. -dataids(dict::Dict) = (UInt(pointer_from_objref(dict)),) dataids(dict::AbstractDict) = (UInt(objectid(dict)),) ## get (getindex with a default value) ## From 1ba7f18ebfd583da97a4dd66686208bd36b97608 Mon Sep 17 00:00:00 2001 From: Matt Bauman Date: Wed, 9 Aug 2023 13:13:30 -0400 Subject: [PATCH 10/12] implement `unaliascopy(A::PermutedDimsArray)` --- base/permuteddimsarray.jl | 4 ++++ test/abstractarray.jl | 13 +++++++++++++ 2 files changed, 17 insertions(+) diff --git a/base/permuteddimsarray.jl b/base/permuteddimsarray.jl index 41c3636b40216..62445773d29f2 100644 --- a/base/permuteddimsarray.jl +++ b/base/permuteddimsarray.jl @@ -252,6 +252,10 @@ function _copy!(P::PermutedDimsArray{T,N,perm}, src) where {T,N,perm} return P end +function Base.unaliascopy(P::PermutedDimsArray)::typeof(P) + return (typeof(P))(Base.unaliascopy(P.parent)) +end + @noinline function _permutedims!(P::PermutedDimsArray, src, R1::CartesianIndices{0}, R2, R3, ds, dp) ip, is = axes(src, dp), axes(src, ds) for jo in first(ip):8:last(ip), io in first(is):8:last(is) diff --git a/test/abstractarray.jl b/test/abstractarray.jl index 3457804fddc77..efb07e074dbda 100644 --- a/test/abstractarray.jl +++ b/test/abstractarray.jl @@ -1158,6 +1158,7 @@ end Base.strides(S::Strider) = S.strides Base.elsize(::Type{<:Strider{T}}) where {T} = Base.elsize(Vector{T}) Base.unsafe_convert(::Type{Ptr{T}}, S::Strider{T}) where {T} = pointer(S.data, S.offset) +Base.unaliascopy(S::Strider)::typeof(S) = (typeof(S))(Base.unaliascopy(S.data), S.offset, S.strides, S.size) @testset "Simple 3d strided views and permutes" for sz in ((5, 3, 2), (7, 11, 13)) A = collect(reshape(1:prod(sz), sz)) @@ -1287,6 +1288,18 @@ end end end +@testset "PermutedDimsArray unaliasing" begin + A = [1 2; 3 4] + P = permutedims(A, (2,1)) + A .= PermutedDimsArray(A, (2,1)) + @test A == P + + A = [1 2; 3 4] + S = Strider(vec(A), strides(A), size(A)) + A .= PermutedDimsArray(S, (2, 1)) + @test A == P +end + @testset "first/last n elements of $(typeof(itr))" for itr in (collect(1:9), [1 4 7; 2 5 8; 3 6 9], ntuple(identity, 9)) From ef31a5f13ec44de860300b59c1f268b0bc5d1cca Mon Sep 17 00:00:00 2001 From: Matt Bauman Date: Wed, 9 Aug 2023 14:56:16 -0400 Subject: [PATCH 11/12] remove `dataids(::AbstractDict)` --- base/abstractarray.jl | 5 ----- 1 file changed, 5 deletions(-) diff --git a/base/abstractarray.jl b/base/abstractarray.jl index ddc1e6aad0827..90accddaf6ea2 100644 --- a/base/abstractarray.jl +++ b/base/abstractarray.jl @@ -1545,11 +1545,6 @@ dataids(A::Array) = (UInt(pointer(A)),) dataids(::AbstractRange) = () dataids(x) = () -# While dictionaries aren't typically involved in aliasing detection, some arrays -# (like DefaultArray) do use dictionaries as a backing element for their data. -# Defining this ensures those fields are involved in aliasing decisions. -dataids(dict::AbstractDict) = (UInt(objectid(dict)),) - ## get (getindex with a default value) ## RangeVecIntList{A<:AbstractVector{Int}} = Union{Tuple{Vararg{Union{AbstractRange, AbstractVector{Int}}}}, From c5d6f31a2a9b88f114897f3d78a07d7096e2413a Mon Sep 17 00:00:00 2001 From: Matt Bauman Date: Wed, 9 Aug 2023 15:02:12 -0400 Subject: [PATCH 12/12] slightly improved `dataids` docstring --- base/abstractarray.jl | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/base/abstractarray.jl b/base/abstractarray.jl index 90accddaf6ea2..490512f5381d0 100644 --- a/base/abstractarray.jl +++ b/base/abstractarray.jl @@ -1522,11 +1522,19 @@ _isdisjoint(as::Tuple, bs::Tuple) = !(as[1] in bs) && _isdisjoint(tail(as), bs) """ Base.dataids(A::AbstractArray) -Return a tuple of `UInt`s that represent the mutable data segments of an array. +Return a tuple of `UInt`s that identify the mutable data segments of an array. +These values are used to determine if two arrays might share memory with [`Base.mightalias`](@ref). The default implementation recursively combines the `dataids` of all fields of the struct. -Custom arrays only need to implement a custom `dataids` method if they depend upon non-array -fields to define their contents and are immutable. + +Custom arrays only need to implement a custom `dataids` method if: + +* they wish to ignore some fields (with non-empty `dataids`) in aliasing considerations; + for example this can be the case if an array is used to store intentionally-shared + metadata or other data that is not mutated by `setindex!` + +* or they depend upon non-array fields (with empty `dataids`) to define their indexable + contents that they wish to include in aliasing considerations. """ function dataids(A::AbstractArray) @inline