Add support for Spark 4.0

.github/workflows/ci-check-jacoco.yml

@@ -10,9 +10,11 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
+        with:
+          persist-credentials: false
       - name: Setup Scala
-        uses: olafurpg/setup-scala@v10
+        uses: olafurpg/setup-scala@v14
         with:
           java-version: "[email protected]"
       - name: Build and run tests

.github/workflows/compatibility-check.yml

@@ -17,12 +17,15 @@ jobs:
         scala: [ 2.12, 2.13 ]
     name: Spark ${{ matrix.spark }}, Scala ${{ matrix.scala }}
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
       - name: Set up JDK 1.8
-        uses: actions/setup-java@v1
+        uses: actions/setup-java@v2
         with:
-          java-version: 1.8
-      - uses: actions/cache@v2
+          distribution: 'adopt'
+          java-version: 8
+      - uses: actions/cache@v4
         with:
           path: ~/.m2/repository
           key: ${{ runner.os }}-${{ matrix.scala }}-${{ hashFiles('**/pom.xml') }}

.github/workflows/test-and-verify.yml

@@ -12,16 +12,19 @@ jobs:
 
     strategy:
       matrix:
-        spark: [ 3.2, 3.3, 3.4, 3.5 ]
+        spark: [ 3.2, 3.3, 3.4, 3.5, 4.0 ]
         scala: [ 2.12, 2.13 ]
 
     name: Spark ${{ matrix.spark }}, Scala ${{ matrix.scala }}
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
       - name: Set up JDK 1.8
-        uses: actions/setup-java@v1
+        uses: actions/setup-java@v2
         with:
-          java-version: 1.8
+          distribution: 'adopt'
+          java-version: 8
       - name: Run tests
         run: mvn clean verify -Plicense-check,spark-${{ matrix.spark }},scala-${{ matrix.scala }}

pom.xml

@@ -389,19 +389,27 @@
         <profile>
             <id>spark-3.4</id>
             <properties>
-                <spark.version>3.4.2</spark.version>
+                <spark.version>3.4.4</spark.version>
                 <avro.version>1.11.1</avro.version>
                 <jackson.core.version>2.14.2</jackson.core.version>
             </properties>
         </profile>
         <profile>
             <id>spark-3.5</id>
             <properties>
-                <spark.version>3.5.0</spark.version>
+                <spark.version>3.5.6</spark.version>
                 <avro.version>1.11.2</avro.version>
                 <jackson.core.version>2.15.2</jackson.core.version>
             </properties>
         </profile>
+        <profile>
+            <id>spark-4.0</id>
+            <properties>
+                <spark.version>4.0.0</spark.version>
+                <avro.version>1.12.0</avro.version>
+                <jackson.core.version>2.18.2</jackson.core.version>
+            </properties>
+        </profile>
         <profile>
             <id>uber</id>
             <build>

src/main/scala/org/apache/spark/sql/avro/AbrisAvroDeserializer.scala

@@ -48,12 +48,12 @@ class AbrisAvroDeserializer(rootAvroType: Schema, rootCatalystType: DataType) {
         currCtor.newInstance(rootAvroType, rootCatalystType, "LEGACY")
       case currCtor if currCtor.getParameterTypes sameElements
         Array(schemaClz, dataTypeClz, stringClz, booleanClz) =>
-        // Spark 3.5.1 - 3.5.2
+        // Spark 3.5.1 - 3.5.6
         currCtor.newInstance(rootAvroType, rootCatalystType, "LEGACY", false: java.lang.Boolean)
       case currCtor if currCtor.getParameterTypes.toSeq sameElements
         Array(schemaClz, dataTypeClz, stringClz, booleanClz, stringClz) =>
-        // Spark 4.0.0-SNAPSHOT+
-        currCtor.newInstance(rootAvroType, rootCatalystType, "LEGACY", false: java.lang.Boolean, "")
+        // Spark 4.0.0
+        currCtor.newInstance(rootAvroType, rootCatalystType, "LEGACY", false: java.lang.Boolean, "") // ,-1
     } match {
       case Some(value: AvroDeserializer) =>
         value