feat: Julia helper functions using PyCall (#42)

* create julia_helpers.jl

* a helper to convert Julia Awkward array to Python

* type lock to a PrimitiveArray

* fix: use np.asarray

* convert only vectors of uint8

* use np.asarray instead

* check if PyCall is installed

* check PyCall installation

* check for PyCall in Main?

* move the import in

* Update and rename julia_helpers.jl to AwkwardPyCall.jl

* try reexport AwkwardArray

* add end of module

* feat: add python to julia functions

* fix: add JSON

* fix: check if PyCall is installed

* Update AwkwardPyCall.jl

* test: add tests

* test: add more tests

* fix: add unreleased AwkwardArray package for testing

* fix: add JSON for tests

* fix: reorder

* test: do not install python

* cleanup: remove python installation

* fix: format using JuliaFormatter and add coverage for tests

Author: ianna

.github/workflows/CI.yml

@@ -50,25 +50,20 @@ jobs:
       - name: Build Julia package
         uses: julia-actions/julia-buildpkg@v1
 
-      - name: Install PyCall dependencies
-        run: julia -e 'import Pkg; Pkg.add("PyCall"); Pkg.build("PyCall")'
-
       - name: Run Julia tests
         uses: julia-actions/julia-runtest@v1
         with:
           project: .
 
-      - name: Set up Python
-        uses: actions/setup-python@v2
-        with:
-          python-version: 3.12
-
       - name: Install Python dependencies
         run: |
           python -m pip install -r requirements/pycall.txt
 
-      # - name: Run Python tests
-      #   run: python test/test.py
+      - name: Install PyCall dependencies
+        run: julia -e 'import Pkg; Pkg.add("PyCall"); Pkg.build("PyCall")'
+  
+      - name: Run PyCall tests
+        run: julia --code-coverage --project=tests test/runpytests.jl
 
       - name: Generate and upload code coverage
         uses: julia-actions/julia-processcoverage@v1

src/pycall/AwkwardPyCall.jl

@@ -0,0 +1,43 @@
+module AwkwardPyCall
+
+using PyCall
+using JSON
+using AwkwardArray
+
+const ak = pyimport("awkward")
+const np = pyimport("numpy")
+
+function _as_numpy(array::AbstractVector{UInt8})
+    py_array = PyObject(array)
+    np.asarray(py_array, dtype = np.uint8)
+end
+
+function julia_array_to_python(array)
+    form, len, containers = AwkwardArray.to_buffers(array)
+
+    py_buffers = Dict{String,Any}()
+
+    for (key, buffer) in containers
+        py_buffers[key] = _as_numpy(buffer)
+    end
+
+    return ak.from_buffers(form, len, py_buffers)
+end
+
+function _as_julia(py_buffer)
+    uint8_buffer = reinterpret(UInt8, py_buffer)
+    return uint8_buffer
+end
+
+function python_array_to_julia(py_array)
+    form, len, containers = ak.to_buffers(py_array)
+
+    julia_buffers = Dict{String,AbstractVector{UInt8}}()
+    for (key, buffer) in containers
+        julia_buffers[key] = _as_julia(buffer)
+    end
+
+    return AwkwardArray.from_buffers(form.to_json(), len, julia_buffers)
+end
+
+end # module

src/tables.jl

@@ -4,11 +4,12 @@ Tables.columnaccess(::Type{RecordArray}) = true
 Tables.columns(x::RecordArray) = x.contents
 Tables.columnnames(x::RecordArray) = keys(x.contents)
 
-Tables.schema(x::RecordArray) = Tables.Schema(Tables.columnnames(x), eltype.(values(Tables.columns(x))))
+Tables.schema(x::RecordArray) =
+    Tables.Schema(Tables.columnnames(x), eltype.(values(Tables.columns(x))))
 
 function from_table(input)
     sch = Tables.schema(input)
-    NT = NamedTuple{sch.names, Base.Tuple{sch.types...}}
+    NT = NamedTuple{sch.names,Base.Tuple{sch.types...}}
     AwkwardType = layout_for(NT)
     out = AwkwardType()
 
@@ -18,4 +19,3 @@ function from_table(input)
 
     return out
 end
-

test/runpytests.jl

@@ -0,0 +1,76 @@
+using PyCall
+using Test
+
+using Pkg
+
+# FIXME: remove after AwkwardArray is released
+function add_unreleased_AwkwardArray_package()
+    # Specify the Git URL of the AwkwardArray package
+    git_url = "https://github.com/JuliaHEP/AwkwardArray.jl"
+
+    # Use Pkg to add the package
+    Pkg.add(PackageSpec(url = git_url))
+end
+
+# Call the function to add the unreleased AwkwardArray package
+add_unreleased_AwkwardArray_package()
+
+using AwkwardArray
+
+Pkg.add("JSON")
+using JSON
+
+include("../src/pycall/AwkwardPyCall.jl")
+
+import Main.AwkwardPyCall: ak
+
+import Main.AwkwardPyCall: julia_array_to_python
+
+# Test julia_array_to_python function
+@testset "julia_array_to_python tests" begin
+    array = AwkwardArray.ListOffsetArray(
+        [0, 3, 3, 5],
+        AwkwardArray.PrimitiveArray([1.1, 2.2, 3.3, 4.4, 5.5]),
+    )
+    # Test case 1: Check if the function returns an awkward array
+    @test isa(julia_array_to_python(array), PyObject)
+
+    # Test case 2: Check if the awkward array has the correct layout
+    py_array = julia_array_to_python(array)
+    @test typeof(py_array) == PyObject
+    @test ak.to_list(py_array) == [[1.1, 2.2, 3.3], [], [4.4, 5.5]]
+end
+
+import Main.AwkwardPyCall: python_array_to_julia
+
+# Test python_array_to_julia function
+@testset "python_array_to_julia tests" begin
+    py_array = ak.Array([[1.1, 2.2, 3.3], [], [4.4, 5.5]])
+
+    # Test case 1: Check if the function returns an awkward array
+    @test isa(
+        python_array_to_julia(py_array),
+        AwkwardArray.ListOffsetArray{
+            SubArray{Int64,1,Vector{Int64},Tuple{UnitRange{Int64}},true},
+            AwkwardArray.PrimitiveArray{
+                Float64,
+                SubArray{Float64,1,Vector{Float64},Tuple{UnitRange{Int64}},true},
+                :default,
+            },
+            :default,
+        },
+    )
+
+    # Test case 2: Check if the awkward array has the correct layout
+    array = python_array_to_julia(py_array)
+    @test typeof(array) == AwkwardArray.ListOffsetArray{
+        SubArray{Int64,1,Vector{Int64},Tuple{UnitRange{Int64}},true},
+        AwkwardArray.PrimitiveArray{
+            Float64,
+            SubArray{Float64,1,Vector{Float64},Tuple{UnitRange{Int64}},true},
+            :default,
+        },
+        :default,
+    }
+    @test array == [[1.1, 2.2, 3.3], [], [4.4, 5.5]]
+end

test/runtests.jl

@@ -3,7 +3,7 @@ using JSON
 using Test
 using Tables
 
-    ### PrimitiveArray #######################################################
+### PrimitiveArray #######################################################
 @testset "PrimitiveArray" begin
 
     begin
@@ -116,7 +116,7 @@ end
     end
 end
 
-    ### ListOffsetArray ######################################################
+### ListOffsetArray ######################################################
 @testset "ListOffsetArray" begin
 
     begin
@@ -233,7 +233,7 @@ end
     end
 end
 
-    ### ListArray ######################################################
+### ListArray ######################################################
 @testset "ListArray" begin
 
     begin
@@ -355,7 +355,7 @@ end
     end
 end
 
-    ### RegularArray #########################################################
+### RegularArray #########################################################
 
 @testset "RegularArray" begin
     begin
@@ -543,7 +543,7 @@ end
     end
 end
 
-    ### ListType with behavior = :string #####################################
+### ListType with behavior = :string #####################################
 
 @testset "ListType with behavior = :string" begin
     begin
@@ -900,7 +900,7 @@ end
     end
 
 end
-    ### ListType with other parameters #######################################
+### ListType with other parameters #######################################
 
 @testset "ListType with other parameters" begin
     begin
@@ -984,7 +984,7 @@ end
     end
 end
 
-    ### RecordArray ##########################################################
+### RecordArray ##########################################################
 
 @testset "RecordArray" begin
     begin
@@ -1282,7 +1282,7 @@ end
         )
     end
 end
-    ### TupleArray ##########################################################
+### TupleArray ##########################################################
 
 @testset "TupleArray" begin
     begin
@@ -1540,7 +1540,7 @@ end
     end
 end
 
-    ### IndexedArray #########################################################
+### IndexedArray #########################################################
 
 @testset "IndexedArray" begin
     begin
@@ -1709,7 +1709,7 @@ end
     end
 end
 
-    ### IndexedOptionArray ###################################################
+### IndexedOptionArray ###################################################
 
 @testset "IndexedOptionArray" begin
     begin
@@ -1826,7 +1826,7 @@ end
     end
 end
 
-    ### ByteMaskedArray ######################################################
+### ByteMaskedArray ######################################################
 
 @testset "ByteMaskedArray" begin
     begin
@@ -1984,7 +1984,7 @@ end
     end
 end
 
-    ### BitMaskedArray #######################################################
+### BitMaskedArray #######################################################
 
 @testset "BitMaskedArray" begin
     begin
@@ -2141,7 +2141,7 @@ end
     end
 end
 
-    ### UnmaskedArray ########################################################
+### UnmaskedArray ########################################################
 
 @testset "UnmaskedArray" begin
     begin
@@ -2270,7 +2270,7 @@ end
         )
     end
 end
-    ### UnionArray ###########################################################
+### UnionArray ###########################################################
 
 @testset "UnionArray" begin
     begin
@@ -2437,7 +2437,7 @@ end
         )
     end
 end
-    ### from_iter ############################################################
+### from_iter ############################################################
 
 @testset "from_iter" begin
     begin
@@ -2517,7 +2517,7 @@ end
     end
 end
 
-    ### from_buffers #########################################################
+### from_buffers #########################################################
 
 @testset "from_buffers" begin
     begin
@@ -3159,7 +3159,7 @@ end
     end
 
     @testset "Tables.jl intergration" begin
-        df = (; x = [[1], [2], [1,2,3]], y = [4.0, 5, 6])
+        df = (; x = [[1], [2], [1, 2, 3]], y = [4.0, 5, 6])
         awt = AwkwardArray.from_table(df)
         @test Tables.schema(df) == Tables.schema(awt)