From 9d7d60d5e19fe22269135097fe131963f5920e8e Mon Sep 17 00:00:00 2001 From: SimonDanisch Date: Wed, 27 Jan 2016 23:08:48 +0100 Subject: [PATCH 1/4] make this work for FixedSizeArrays and scalars [WIP] --- src/StructsOfArrays.jl | 82 +++++++++++++++++++++++++++++++++++++----- 1 file changed, 74 insertions(+), 8 deletions(-) diff --git a/src/StructsOfArrays.jl b/src/StructsOfArrays.jl index 12c2751..611a7ae 100644 --- a/src/StructsOfArrays.jl +++ b/src/StructsOfArrays.jl @@ -1,19 +1,71 @@ module StructsOfArrays -export StructOfArrays +export StructOfArrays, ScalarRepeat immutable StructOfArrays{T,N,U<:Tuple} <: AbstractArray{T,N} arrays::U end -@generated function StructOfArrays{T}(::Type{T}, dims::Integer...) +type ScalarRepeat{T} + scalar::T +end +Base.ndims(::ScalarRepeat) = 1 +Base.getindex(s::ScalarRepeat, i...) = s.scalar +#should setindex! really be allowed? It will set the index for the whole row... +Base.setindex!{T}(s::ScalarRepeat{T}, value, i...) = (s.scalar = T(value)) +Base.eltype{T}(::ScalarRepeat{T}) = T + +Base.start(::ScalarRepeat) = 1 +Base.next(sr::ScalarRepeat, i) = sr.scalar, i+1 +Base.done(sr::ScalarRepeat, i) = false + + +# since this is used in hot loops, and T.types[.] doesn't play well with compiler +# this needs to be a generated function +@generated function is_tuple_struct{T}(::Type{T}) + is_ts = length(T.types) == 1 && T.types[1] <: Tuple + :($is_ts) +end +struct_eltypes{T}(struct::T) = struct_eltypes(T) +function struct_eltypes{T}(::Type{T}) + if is_tuple_struct(T) #special case tuple types (E.g. FixedSizeVectors) + return eltypes = T.types[1].parameters + else + return eltypes = T.types + end +end + +make_iterable(x::AbstractArray) = x +make_iterable(x) = ScalarRepeat(x) + +@generated function StructOfArrays{T}(::Type{T}, dim1::Integer, rest::Integer...) (!isleaftype(T) || T.mutable) && return :(throw(ArgumentError("can only create an StructOfArrays of leaf type immutables"))) isempty(T.types) && return :(throw(ArgumentError("cannot create an StructOfArrays of an empty or bitstype"))) + dims = (dim1, rest...) N = length(dims) - arrtuple = Tuple{[Array{T.types[i],N} for i = 1:length(T.types)]...} - :(StructOfArrays{T,$N,$arrtuple}(($([:(Array($(T.types[i]), dims)) for i = 1:length(T.types)]...),))) + eltypes = struct_eltypes(T) + arrtuple = Tuple{[Array{eltypes[i],N} for i = 1:length(eltypes)]...} + + :(StructOfArrays{T,$N,$arrtuple}( + ($([:(Array($(eltypes[i]), dims)) for i = 1:length(eltypes)]...),) + )) end StructOfArrays(T::Type, dims::Tuple{Vararg{Integer}}) = StructOfArrays(T, dims...) +function StructOfArrays(T::Type, a, rest...) + arrays = map(make_iterable, (a, rest...)) + N = ndims(arrays[1]) + eltypes = map(eltype, arrays) + s_eltypes = struct_eltypes(T) + any(ix->ix[1]!=ix[2], zip(eltypes,s_eltypes)) && throw(ArgumentError( + "fieldtypes of $T must be equal to eltypes of arrays: $eltypes" + )) + any(x->ndims(x)!=N, arrays) && throw(ArgumentError( + "cannot create an StructOfArrays from arrays with different ndims" + )) + arrtuple = Tuple{map(typeof, arrays)...} + StructOfArrays{T, N, arrtuple}(arrays) +end + Base.linearindexing{T<:StructOfArrays}(::Type{T}) = Base.LinearFast() @generated function Base.similar{T}(A::StructOfArrays, ::Type{T}, dims::Dims) @@ -31,18 +83,32 @@ Base.convert{T,S,N}(::Type{StructOfArrays{T}}, A::AbstractArray{S,N}) = Base.convert{T,N}(::Type{StructOfArrays}, A::AbstractArray{T,N}) = convert(StructOfArrays{T,N}, A) -Base.size(A::StructOfArrays) = size(A.arrays[1]) -Base.size(A::StructOfArrays, d) = size(A.arrays[1], d) +function Base.size(A::StructOfArrays) + for elem in A.arrays + if isa(elem, AbstractArray) + return size(elem) + end + end + () +end +Base.size(A::StructOfArrays, d) = size(A)[d] @generated function Base.getindex{T}(A::StructOfArrays{T}, i::Integer...) + n = length(struct_eltypes(T)) Expr(:block, Expr(:meta, :inline), - Expr(:new, T, [:(A.arrays[$j][i...]) for j = 1:length(T.types)]...)) + :($T($([:(A.arrays[$j][i...]) for j = 1:n]...))) + ) +end + +function _getindex{T}(x::T, i) + is_tuple_struct(T) ? x[i] : getfield(x, i) end @generated function Base.setindex!{T}(A::StructOfArrays{T}, x, i::Integer...) + n = length(struct_eltypes(T)) quote $(Expr(:meta, :inline)) v = convert(T, x) - $([:(A.arrays[$j][i...] = getfield(v, $j)) for j = 1:length(T.types)]...) + $([:(A.arrays[$j][i...] = _getindex(v, $j)) for j = 1:n]...) x end end From 333e5f58e08e8cf0b57ce51ac9e0b23d6f40b4d7 Mon Sep 17 00:00:00 2001 From: SimonDanisch Date: Sun, 31 Jan 2016 14:09:14 +0100 Subject: [PATCH 2/4] fix oversight --- src/StructsOfArrays.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/StructsOfArrays.jl b/src/StructsOfArrays.jl index 611a7ae..94f4e46 100644 --- a/src/StructsOfArrays.jl +++ b/src/StructsOfArrays.jl @@ -46,7 +46,7 @@ make_iterable(x) = ScalarRepeat(x) arrtuple = Tuple{[Array{eltypes[i],N} for i = 1:length(eltypes)]...} :(StructOfArrays{T,$N,$arrtuple}( - ($([:(Array($(eltypes[i]), dims)) for i = 1:length(eltypes)]...),) + ($([:(Array($(eltypes[i]), (dim1, rest...))) for i = 1:length(eltypes)]...),) )) end StructOfArrays(T::Type, dims::Tuple{Vararg{Integer}}) = StructOfArrays(T, dims...) From b24d328c159855ebd3668ab579cbb96146e4354e Mon Sep 17 00:00:00 2001 From: SimonDanisch Date: Wed, 3 Feb 2016 13:39:15 +0100 Subject: [PATCH 3/4] add topology independant constructor and getindex --- src/StructsOfArrays.jl | 198 +++++++++++++++++++++++++++-------------- 1 file changed, 132 insertions(+), 66 deletions(-) diff --git a/src/StructsOfArrays.jl b/src/StructsOfArrays.jl index 94f4e46..c9a84be 100644 --- a/src/StructsOfArrays.jl +++ b/src/StructsOfArrays.jl @@ -5,65 +5,35 @@ immutable StructOfArrays{T,N,U<:Tuple} <: AbstractArray{T,N} arrays::U end -type ScalarRepeat{T} - scalar::T -end -Base.ndims(::ScalarRepeat) = 1 -Base.getindex(s::ScalarRepeat, i...) = s.scalar -#should setindex! really be allowed? It will set the index for the whole row... -Base.setindex!{T}(s::ScalarRepeat{T}, value, i...) = (s.scalar = T(value)) -Base.eltype{T}(::ScalarRepeat{T}) = T - -Base.start(::ScalarRepeat) = 1 -Base.next(sr::ScalarRepeat, i) = sr.scalar, i+1 -Base.done(sr::ScalarRepeat, i) = false - - -# since this is used in hot loops, and T.types[.] doesn't play well with compiler -# this needs to be a generated function -@generated function is_tuple_struct{T}(::Type{T}) - is_ts = length(T.types) == 1 && T.types[1] <: Tuple - :($is_ts) -end -struct_eltypes{T}(struct::T) = struct_eltypes(T) -function struct_eltypes{T}(::Type{T}) - if is_tuple_struct(T) #special case tuple types (E.g. FixedSizeVectors) - return eltypes = T.types[1].parameters - else - return eltypes = T.types - end -end -make_iterable(x::AbstractArray) = x -make_iterable(x) = ScalarRepeat(x) - -@generated function StructOfArrays{T}(::Type{T}, dim1::Integer, rest::Integer...) +@generated function StructOfArrays{T}(::Type{T}, dims::Integer...) (!isleaftype(T) || T.mutable) && return :(throw(ArgumentError("can only create an StructOfArrays of leaf type immutables"))) isempty(T.types) && return :(throw(ArgumentError("cannot create an StructOfArrays of an empty or bitstype"))) - dims = (dim1, rest...) N = length(dims) - eltypes = struct_eltypes(T) - arrtuple = Tuple{[Array{eltypes[i],N} for i = 1:length(eltypes)]...} - - :(StructOfArrays{T,$N,$arrtuple}( - ($([:(Array($(eltypes[i]), (dim1, rest...))) for i = 1:length(eltypes)]...),) - )) + arrtuple = Tuple{[Array{T.types[i],N} for i = 1:length(T.types)]...} + :(StructOfArrays{T,$N,$arrtuple}(($([:(Array($(T.types[i]), dims)) for i = 1:length(T.types)]...),))) end StructOfArrays(T::Type, dims::Tuple{Vararg{Integer}}) = StructOfArrays(T, dims...) -function StructOfArrays(T::Type, a, rest...) - arrays = map(make_iterable, (a, rest...)) - N = ndims(arrays[1]) - eltypes = map(eltype, arrays) - s_eltypes = struct_eltypes(T) - any(ix->ix[1]!=ix[2], zip(eltypes,s_eltypes)) && throw(ArgumentError( - "fieldtypes of $T must be equal to eltypes of arrays: $eltypes" +function StructOfArrays(T::Type, first_array::AbstractArray, rest::AbstractArray...) + (!isleaftype(T) || T.mutable) && throw(ArgumentError( + "can only create an StructOfArrays of leaf type immutables" )) - any(x->ndims(x)!=N, arrays) && throw(ArgumentError( - "cannot create an StructOfArrays from arrays with different ndims" - )) - arrtuple = Tuple{map(typeof, arrays)...} - StructOfArrays{T, N, arrtuple}(arrays) + arrays = (first_array, rest...) + target_eltypes = flattened_bitstypes(T) + source_eltypes = DataType[] + #flatten array eltypes + for elem in arrays + append!(source_eltypes, flattened_bitstypes(eltype(elem))) + end + # flattened eltypes don't match with flattened struct type + if target_eltypes != source_eltypes + throw(ArgumentError( + "$target_eltypes\n\n$source_eltypes" + )) + end + typetuple = Tuple{map(typeof, arrays)...} + StructOfArrays{T, ndims(first_array), typetuple}(arrays) end Base.linearindexing{T<:StructOfArrays}(::Type{T}) = Base.LinearFast() @@ -83,32 +53,128 @@ Base.convert{T,S,N}(::Type{StructOfArrays{T}}, A::AbstractArray{S,N}) = Base.convert{T,N}(::Type{StructOfArrays}, A::AbstractArray{T,N}) = convert(StructOfArrays{T,N}, A) -function Base.size(A::StructOfArrays) - for elem in A.arrays - if isa(elem, AbstractArray) - return size(elem) +Base.size(A::StructOfArrays) = size(first(A.arrays)) +Base.size(A::StructOfArrays, d) = size(first(A.arrays), d) + + +fieldtypes{T<:Tuple}(::Type{T}) = (T.parameters...) +function fieldtypes{T}(::Type{T}) + if nfields(T) > 0 + return ntuple(i->fieldtype(T, i), nfields(T)) + else + return T + end +end + +""" +Returns a flattened and unflattened view of the elemenents of a type +E.g: +immutable X +x::Float32 +y::Float32 +end +immutable Y +a::X # tuples would get expanded as well +b::Float32 +c::Float32 +end +Would return +[Float32, Float32, Float32, Float32] +and +[(Y, [(X, [Float32, Float32]), Float32, Float32]] +""" +function flattened_bitstypes{T}(::Type{T}, flattened=DataType[]) + fields = fieldtypes(T) + if isa(fields, DataType) + if (!isleaftype(T) || T.mutable) + throw(ArgumentError("can only create an StructOfArrays of leaf type immutables")) + end + push!(flattened, fields) + return flattened + else + for T in fields + flattened_bitstypes(T, flattened) + end + end + flattened +end + +""" +Takes a tuple of array types with arbitrary structs as elements. +return `flat_indices` and `temporaries`. `flat_indices` is a vector with indices to every elemen in the array. +`temporaries` is a vector of temporaries, which effectively store the elemens from the arrays +E.g. +flatindexes((Vector{Vec3f0}) will return: +with `array_expr=(A.arrays)` and `index_expr=:([i...])`: +`temporaries`: + [:(value1 = A.arrays[i...])] +`flat_indices`: + [:(value1.(1).(1)), :(value1.(1).(2)), :(value1.(1).(3))] # .(1) to acces tuple of Vec3 +""" +function flatindexes(arrays) + temporaries = [] + flat_indices = [] + for (i, array) in enumerate(arrays) + tmpsym = symbol("value$i") + push!(temporaries, :($(tmpsym) = A.arrays[$i][i...])) + index_expr = :($tmpsym) + append!(flat_indices, flatindexes(eltype(array), index_expr, flat_indices)) + end + flat_indices, temporaries +end + +function flatindexes(T, index_expr, flat_indices) + fields = fieldtypes(T) + if isa(fields, DataType) + return [index_expr] + else + for (i,T) in enumerate(fields) + new_index_expr = :($(index_expr).($i)) + append!(flat_indices, flatindexes(T, new_index_expr, flat_indices)) end end - () + flat_indices end -Base.size(A::StructOfArrays, d) = size(A)[d] -@generated function Base.getindex{T}(A::StructOfArrays{T}, i::Integer...) - n = length(struct_eltypes(T)) - Expr(:block, Expr(:meta, :inline), - :($T($([:(A.arrays[$j][i...]) for j = 1:n]...))) - ) +""" +Creates a nested type T from elements in `flat_indices`. +`flat_indices` can be any array with expressions inside, as long as there is an +element for every field in `T`. +""" +function typecreator(T, lower_constr, flat_indices, i=1) + i>length(flat_indices) && return i + if T<:Tuple + constructor = Expr(:tuple) + else + constructor = Expr(:call, T) + end + push!(lower_constr.args, constructor) + fields = fieldtypes(T) + if isa(fields, DataType) + push!(constructor.args, flat_indices[i]) + return i+1 + else + for T in fields + i = typecreator(T, constructor, flat_indices, i) + end + end + return i end -function _getindex{T}(x::T, i) - is_tuple_struct(T) ? x[i] : getfield(x, i) + +@generated function Base.getindex{T, N, ArrayTypes}(A::StructOfArrays{T, N, ArrayTypes}, i::Integer...) + flat_indices, temporaries = flatindexes((ArrayTypes.parameters...)) + type_constructor = Expr(:block) + typecreator(T, type_constructor , flat_indices) + Expr(:block, Expr(:meta, :inline), temporaries..., type_constructor) end + + @generated function Base.setindex!{T}(A::StructOfArrays{T}, x, i::Integer...) - n = length(struct_eltypes(T)) quote $(Expr(:meta, :inline)) v = convert(T, x) - $([:(A.arrays[$j][i...] = _getindex(v, $j)) for j = 1:n]...) + $([:(A.arrays[$j][i...] = getfield(v, $j)) for j = 1:length(T.types)]...) x end end From a820ad2be10def88bfcc3543237f3b483c678270 Mon Sep 17 00:00:00 2001 From: SimonDanisch Date: Wed, 3 Feb 2016 14:38:16 +0100 Subject: [PATCH 4/4] comments and tests --- src/StructsOfArrays.jl | 31 ++++++++++++++++-------- test/runtests.jl | 53 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+), 10 deletions(-) diff --git a/src/StructsOfArrays.jl b/src/StructsOfArrays.jl index c9a84be..af64b26 100644 --- a/src/StructsOfArrays.jl +++ b/src/StructsOfArrays.jl @@ -28,10 +28,13 @@ function StructOfArrays(T::Type, first_array::AbstractArray, rest::AbstractArray end # flattened eltypes don't match with flattened struct type if target_eltypes != source_eltypes - throw(ArgumentError( - "$target_eltypes\n\n$source_eltypes" - )) + throw(ArgumentError("""$T does not match the given parameters. + Argument types: $(map(typeof, arrays)) + Flattened struct types: $target_eltypes + Flattened argument types: $source_eltypes + """)) end + # flattened they match! ♥💕 typetuple = Tuple{map(typeof, arrays)...} StructOfArrays{T, ndims(first_array), typetuple}(arrays) end @@ -56,7 +59,10 @@ Base.convert{T,N}(::Type{StructOfArrays}, A::AbstractArray{T,N}) = Base.size(A::StructOfArrays) = size(first(A.arrays)) Base.size(A::StructOfArrays, d) = size(first(A.arrays), d) - +""" +returns all field types of a composite type or tuple. +If it's neither composite, nor tuple, it will just return the DataType. +""" fieldtypes{T<:Tuple}(::Type{T}) = (T.parameters...) function fieldtypes{T}(::Type{T}) if nfields(T) > 0 @@ -118,7 +124,7 @@ function flatindexes(arrays) tmpsym = symbol("value$i") push!(temporaries, :($(tmpsym) = A.arrays[$i][i...])) index_expr = :($tmpsym) - append!(flat_indices, flatindexes(eltype(array), index_expr, flat_indices)) + flatindexes(eltype(array), index_expr, flat_indices) end flat_indices, temporaries end @@ -126,14 +132,15 @@ end function flatindexes(T, index_expr, flat_indices) fields = fieldtypes(T) if isa(fields, DataType) - return [index_expr] + push!(flat_indices, index_expr) + return nothing else for (i,T) in enumerate(fields) new_index_expr = :($(index_expr).($i)) - append!(flat_indices, flatindexes(T, new_index_expr, flat_indices)) + flatindexes(T, new_index_expr, flat_indices) end end - flat_indices + nothing end """ @@ -143,6 +150,8 @@ element for every field in `T`. """ function typecreator(T, lower_constr, flat_indices, i=1) i>length(flat_indices) && return i + # we need to special case tuples, since e.g. Tuple{Float32, Float32}(1f0, 1f0) + # is not defined. if T<:Tuple constructor = Expr(:tuple) else @@ -161,11 +170,13 @@ function typecreator(T, lower_constr, flat_indices, i=1) return i end - @generated function Base.getindex{T, N, ArrayTypes}(A::StructOfArrays{T, N, ArrayTypes}, i::Integer...) + #flatten the indices, flat_indices, temporaries = flatindexes((ArrayTypes.parameters...)) type_constructor = Expr(:block) - typecreator(T, type_constructor , flat_indices) + # create a constructor expression, which uses the flattened indexes to create the type + typecreator(T, type_constructor, flat_indices) + # put everything in a block! Expr(:block, Expr(:meta, :inline), temporaries..., type_constructor) end diff --git a/test/runtests.jl b/test/runtests.jl index a03e03a..cc94086 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -31,3 +31,56 @@ small = StructOfArrays(Complex64, 2) @test typeof(similar(small, SubString)) === Vector{SubString} @test typeof(similar(small, OneField)) === Vector{OneField} @test typeof(similar(small, Complex128)) <: StructOfArrays + +immutable Vec{N,T} + _::NTuple{N,T} +end +immutable HyperCube{N,T} + origin::Vec{N,T} + width::Vec{N,T} +end +immutable Instance{P, S, T, R} + primitive::P + scale::S + translation::T + rotation::R +end +immutable ScalarRepeat{T,N} <: AbstractArray{T,N} + value::T + size::NTuple{N,Int} +end +Base.size(sr::ScalarRepeat) = sr.size +Base.size(sr::ScalarRepeat, d) = sr.size[d] +Base.getindex(sr::ScalarRepeat, i...) = sr.value +Base.linearindexing{T<:ScalarRepeat}(::Type{T}) = Base.LinearFast() + + +function test_topologic_structs() + hco_x,hco_yz = rand(Float32, 10), [Vec{2,Float32}((rand(Float32), rand(Float32))) for i=1:10] + hcw_z,hcw_xy = rand(Float32, 10), [Vec{2,Float32}((rand(Float32), rand(Float32))) for i=1:10] + scale = ScalarRepeat(1f0, (10,)) + translation = ScalarRepeat(Vec{3, Float32}((2,1,3)), (10,)) + rotation = [Vec{4, Float32}((rand(Float32),rand(Float32),rand(Float32),rand(Float32))) for i=1:10] + soa = StructOfArrays( + Instance{HyperCube{3, Float32}, Float32, Vec{3, Float32}, Vec{4,Float32}}, + hco_x,hco_yz, hcw_xy, hcw_z, scale, translation, rotation + ) + zipped = zip(hco_x,hco_yz, hcw_xy, hcw_z, scale, translation, rotation) + for (i,(ox,oyz, wxy, wz, s, t, r)) in enumerate(zipped) + instance = soa[i] + @test instance.primitive.origin.(1).(1) === ox + @test instance.primitive.origin.(1).(2) === oyz.(1).(1) + @test instance.primitive.origin.(1).(3) === oyz.(1).(2) + + @test instance.primitive.width.(1).(1) === wxy.(1).(1) + @test instance.primitive.width.(1).(2) === wxy.(1).(2) + @test instance.primitive.width.(1).(3) === wz + + @test instance.scale === s + @test instance.translation === t + @test instance.rotation === r + + end +end + +test_topologic_structs()