fix(kafka): Handle message indices in proto data also for Glue Schema Registry (#1907)

* Bug fix to handle message indices in proto data

* Removing links and changing  log level from info to debug

* Upgrade lambda.events.version tp 3.16.0

* Update Glue schema id to 36. Add some tests to increase coverage. Use Kafka ByteUtils also in unit tests.

* Update lambda java events to 3.16.0 everywhere.

* Add sample generation code for all Protobuf cases.

---------

Co-authored-by: Karthik Puttaswamy <[email protected]>

Author: phipag

examples/powertools-examples-core-utilities/cdk/app/pom.xml

@@ -41,7 +41,7 @@
         <dependency>
             <groupId>com.amazonaws</groupId>
             <artifactId>aws-lambda-java-events</artifactId>
-            <version>3.15.0</version>
+            <version>3.16.0</version>
         </dependency>
         <dependency>
             <groupId>org.apache.logging.log4j</groupId>

examples/powertools-examples-core-utilities/gradle/build.gradle

@@ -26,7 +26,7 @@ dependencies {
     implementation 'com.amazonaws:aws-lambda-java-core:1.2.2'
     implementation 'com.fasterxml.jackson.core:jackson-annotations:2.13.2'
     implementation 'com.fasterxml.jackson.core:jackson-databind:2.13.2.2'
-    implementation 'com.amazonaws:aws-lambda-java-events:3.11.0'
+    implementation 'com.amazonaws:aws-lambda-java-events:3.16.0'
     implementation 'com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.13.2'
     implementation 'org.aspectj:aspectjrt:1.9.20.1'
     aspect 'software.amazon.lambda:powertools-tracing:2.1.0'

examples/powertools-examples-core-utilities/kotlin/build.gradle.kts

@@ -12,7 +12,7 @@ dependencies {
     implementation("com.amazonaws:aws-lambda-java-core:1.2.3")
     implementation("com.fasterxml.jackson.core:jackson-annotations:2.15.1")
     implementation("com.fasterxml.jackson.core:jackson-databind:2.15.3")
-    implementation("com.amazonaws:aws-lambda-java-events:3.11.3")
+    implementation("com.amazonaws:aws-lambda-java-events:3.16.0")
     implementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.15.2")
     implementation("org.aspectj:aspectjrt:1.9.20.1")
     aspect("software.amazon.lambda:powertools-tracing:2.1.0")
@@ -23,4 +23,4 @@ dependencies {
 
 kotlin {
     jvmToolchain(11)
-}
+}

examples/powertools-examples-core-utilities/sam-graalvm/pom.xml

@@ -39,7 +39,7 @@
         <dependency>
             <groupId>com.amazonaws</groupId>
             <artifactId>aws-lambda-java-events</artifactId>
-            <version>3.11.3</version>
+            <version>3.16.0</version>
         </dependency>
         <dependency>
             <groupId>org.aspectj</groupId>

examples/powertools-examples-core-utilities/sam/pom.xml

@@ -38,7 +38,7 @@
         <dependency>
             <groupId>com.amazonaws</groupId>
             <artifactId>aws-lambda-java-events</artifactId>
-            <version>3.15.0</version>
+            <version>3.16.0</version>
         </dependency>
         <dependency>
             <groupId>org.aspectj</groupId>

examples/powertools-examples-core-utilities/serverless/pom.xml

@@ -38,7 +38,7 @@
         <dependency>
             <groupId>com.amazonaws</groupId>
             <artifactId>aws-lambda-java-events</artifactId>
-            <version>3.15.0</version>
+            <version>3.16.0</version>
         </dependency>
         <dependency>
             <groupId>org.aspectj</groupId>

examples/powertools-examples-core-utilities/terraform/pom.xml

@@ -38,7 +38,7 @@
         <dependency>
             <groupId>com.amazonaws</groupId>
             <artifactId>aws-lambda-java-events</artifactId>
-            <version>3.15.0</version>
+            <version>3.16.0</version>
         </dependency>
         <dependency>
             <groupId>org.aspectj</groupId>

examples/powertools-examples-idempotency/pom.xml

@@ -52,7 +52,7 @@
         <dependency>
             <groupId>com.amazonaws</groupId>
             <artifactId>aws-lambda-java-events</artifactId>
-            <version>3.15.0</version>
+            <version>3.16.0</version>
         </dependency>
         <dependency>
             <groupId>org.aspectj</groupId>

examples/powertools-examples-kafka/events/kafka-protobuf-event.json

@@ -30,7 +30,11 @@
           {
             "headerKey": [104, 101, 97, 100, 101, 114, 86, 97, 108, 117, 101]
           }
-        ]
+        ],
+        "valueSchemaMetadata": {
+          "schemaId": "123",
+          "dataFormat": "PROTOBUF"
+        }
       },
       {
         "topic": "mytopic",
@@ -39,12 +43,34 @@
         "timestamp": 1545084650989,
         "timestampType": "CREATE_TIME",
         "key": null,
-        "value": "AgEACOkHEgZMYXB0b3AZUrgehes/j0A=",
+        "value": "BAIACOkHEgZMYXB0b3AZUrgehes/j0A=",
         "headers": [
           {
             "headerKey": [104, 101, 97, 100, 101, 114, 86, 97, 108, 117, 101]
           }
-        ]
+        ],
+        "valueSchemaMetadata": {
+          "schemaId": "456",
+          "dataFormat": "PROTOBUF"
+        }
+      },
+      {
+        "topic": "mytopic",
+        "partition": 0,
+        "offset": 18,
+        "timestamp": 1545084650990,
+        "timestampType": "CREATE_TIME",
+        "key": "NDI=",
+        "value": "AQjpBxIGTGFwdG9wGVK4HoXrP49A",
+        "headers": [
+          {
+            "headerKey": [104, 101, 97, 100, 101, 114, 86, 97, 108, 117, 101]
+          }
+        ],
+        "valueSchemaMetadata": {
+          "schemaId": "12345678-1234-1234-1234-123456789012",
+          "dataFormat": "PROTOBUF"
+        }
       }
     ]
   }

examples/powertools-examples-kafka/tools/pom.xml

@@ -13,6 +13,7 @@
         <maven.compiler.target>11</maven.compiler.target>
         <avro.version>1.12.0</avro.version>
         <protobuf.version>4.31.0</protobuf.version>
+        <kafka-clients.version>4.0.0</kafka-clients.version>
     </properties>
 
     <dependencies>
@@ -26,6 +27,11 @@
             <artifactId>protobuf-java</artifactId>
             <version>${protobuf.version}</version>
         </dependency>
+        <dependency>
+            <groupId>org.apache.kafka</groupId>
+            <artifactId>kafka-clients</artifactId>
+            <version>${kafka-clients.version}</version>
+        </dependency>
         <dependency>
             <groupId>com.fasterxml.jackson.core</groupId>
             <artifactId>jackson-databind</artifactId>