[!461] - Remove avro4s

# New features and improvements

- Remove avro4s from dependencies
- Update test where avro4s was used

# Breaking changes

none

# Migration

none

# Bug fixes

none

# How this feature was tested

Unit Test - Pipeline

# Related issue

Closes #577

Author: Matteo Seclì

consumers-spark/src/main/scala/it/agilelab/bigdata/wasp/consumers/spark/eventengine/MailStrategy.scala

@@ -223,7 +223,7 @@ class InnerMailStrategy(config: Config) {
       sb.toString
     }
 
-    println("EXPLODE QUERY: " + sqlQuery)
+//    println("EXPLODE QUERY: " + sqlQuery)
     val explodedMails = ss.sql(sqlQuery)
 
     ss.catalog.dropTempView(mailTableName)

consumers-spark/src/test/scala/it/agilelab/bigdata/wasp/consumers/spark/eventengine/InnerEventStrategySpec.scala

@@ -157,7 +157,7 @@ class InnerEventStrategySpec extends WordSpec with Matchers with BeforeAndAfter
     s"Process a big random sequence" in {
       import spark.implicits._
       val events: Array[Event] = target.transform(spark.sparkContext.parallelize(randomSeq).toDF).as[Event].collect()
-      println(events.length)
+      val _ = events.length
     }
 
     s"Find $totalEventsQuantity test events in mixedSeq which match the control event seq" in {

consumers-spark/src/test/scala/it/agilelab/bigdata/wasp/consumers/spark/http/data/FromKafka.scala

@@ -1,3 +1,68 @@
 package it.agilelab.bigdata.wasp.consumers.spark.http.data
 
-case class FromKafka(id: String, exampleAuthor: String, timestamp: Long)
+import org.apache.avro.generic.{GenericData, GenericDatumReader, GenericDatumWriter, GenericRecord}
+import org.apache.avro.io.{BinaryDecoder, BinaryEncoder, DatumReader, DatumWriter, DecoderFactory, EncoderFactory}
+import org.apache.avro.{Schema, SchemaBuilder}
+
+import java.io.ByteArrayOutputStream
+
+case class FromKafka(id: String, exampleAuthor: String, timestamp: Long)
+
+object FromKafka {
+  val schema: Schema = SchemaBuilder
+    .record("FromKafka")
+    .fields()
+    .name("id")
+    .`type`()
+    .stringType()
+    .noDefault()
+    .name("exampleAuthor")
+    .`type`()
+    .stringType()
+    .noDefault()
+    .name("timestamp")
+    .`type`()
+    .longType()
+    .noDefault()
+    .endRecord();
+
+  def toRecord(obj: FromKafka): GenericRecord = {
+    val record = new GenericData.Record(schema)
+    record.put("id", obj.id)
+    record.put("exampleAuthor", obj.exampleAuthor)
+    record.put("timestamp", obj.timestamp)
+    record
+  }
+
+  def fromRecord(record: GenericRecord): FromKafka = {
+    FromKafka(
+      record.get("id").toString,
+      record.get("exampleAuthor").toString,
+      record.get("timestamp").asInstanceOf[Long]
+    )
+  }
+
+  def serializeToBytes(obj: FromKafka, schema: Schema): Array[Byte] = {
+    val byteArrayOutputStream = new ByteArrayOutputStream()
+    val datumWriter: DatumWriter[GenericRecord] =
+      new GenericDatumWriter[GenericRecord](schema)
+    val encoder: BinaryEncoder =
+      EncoderFactory.get().binaryEncoder(byteArrayOutputStream, null)
+    datumWriter.write(toRecord(obj), encoder)
+    encoder.flush()
+    byteArrayOutputStream.toByteArray // Return the byte array
+  }
+
+  def deserializeFromBytes(
+      bytes: Array[Byte],
+      schema: Schema
+  ): FromKafka = {
+    val byteArrayInputStream = new java.io.ByteArrayInputStream(bytes)
+    val datumReader: DatumReader[GenericRecord] =
+      new GenericDatumReader[GenericRecord](schema)
+    val decoder: BinaryDecoder =
+      DecoderFactory.get().binaryDecoder(byteArrayInputStream, null)
+    val record = datumReader.read(null, decoder)
+    fromRecord(record)
+  }
+}

consumers-spark/src/test/scala/it/agilelab/bigdata/wasp/consumers/spark/http/data/SampleData.scala

@@ -1,3 +1,62 @@
 package it.agilelab.bigdata.wasp.consumers.spark.http.data
 
+import org.apache.avro.{Schema, SchemaBuilder}
+import org.apache.avro.generic.{GenericData, GenericDatumReader, GenericDatumWriter, GenericRecord}
+import org.apache.avro.io.{BinaryDecoder, BinaryEncoder, DatumReader, DatumWriter, DecoderFactory, EncoderFactory}
+
+import java.io.ByteArrayOutputStream
+
 case class SampleData(id: String, text: String)
+
+object SampleData {
+  val schema: Schema = SchemaBuilder
+    .record("SampleData")
+    .fields()
+    .name("id")
+    .`type`()
+    .stringType()
+    .noDefault()
+    .name("text")
+    .`type`()
+    .stringType()
+    .noDefault()
+    .endRecord();
+
+  def toRecord(obj: SampleData): GenericRecord = {
+    val record = new GenericData.Record(schema)
+    record.put("id", obj.id)
+    record.put("text", obj.text)
+    record
+  }
+
+  def fromRecord(record: GenericRecord): SampleData = {
+    SampleData(
+      record.get("id").toString,
+      record.get("text").toString,
+    )
+  }
+
+  def serializeToBytes(obj: SampleData, schema: Schema): Array[Byte] = {
+    val byteArrayOutputStream = new ByteArrayOutputStream()
+    val datumWriter: DatumWriter[GenericRecord] =
+      new GenericDatumWriter[GenericRecord](schema)
+    val encoder: BinaryEncoder =
+      EncoderFactory.get().binaryEncoder(byteArrayOutputStream, null)
+    datumWriter.write(toRecord(obj), encoder)
+    encoder.flush()
+    byteArrayOutputStream.toByteArray // Return the byte array
+  }
+
+  def deserializeFromBytes(
+                            bytes: Array[Byte],
+                            schema: Schema
+                          ): SampleData = {
+    val byteArrayInputStream = new java.io.ByteArrayInputStream(bytes)
+    val datumReader: DatumReader[GenericRecord] =
+      new GenericDatumReader[GenericRecord](schema)
+    val decoder: BinaryDecoder =
+      DecoderFactory.get().binaryDecoder(byteArrayInputStream, null)
+    val record = datumReader.read(null, decoder)
+    fromRecord(record)
+  }
+}

consumers-spark/src/test/scala/it/agilelab/bigdata/wasp/consumers/spark/http/topic/package.scala

@@ -1,6 +1,5 @@
 package it.agilelab.bigdata.wasp.consumers.spark.http
 
-import com.sksamuel.avro4s.AvroSchema
 import it.agilelab.bigdata.wasp.consumers.spark.http.data.{FromKafka, SampleData}
 import it.agilelab.bigdata.wasp.core.utils.JsonConverter
 import it.agilelab.bigdata.wasp.models.TopicModel
@@ -17,7 +16,7 @@ package object topic {
     valueFieldsNames = None,
     useAvroSchemaManager = false,
     schema = JsonConverter
-      .fromString(AvroSchema[FromKafka].toString(false))
+      .fromString(FromKafka.schema.toString(false))
       .getOrElse(org.mongodb.scala.bson.BsonDocument())
   )
 
@@ -32,7 +31,7 @@ package object topic {
     valueFieldsNames = None,
     useAvroSchemaManager = false,
     schema = JsonConverter
-      .fromString(AvroSchema[SampleData].toString(false))
+      .fromString(SampleData.schema.toString(false))
       .getOrElse(org.mongodb.scala.bson.BsonDocument())
   )
 }

consumers-spark/src/test/scala/it/agilelab/bigdata/wasp/consumers/spark/strategies/gdpr/ConfigUtilsSpec.scala

@@ -16,7 +16,7 @@ class ConfigUtilsSpec extends FlatSpec with Matchers with TryValues with BeforeA
         |{ "$KV_CONF_KEY" { "$KEYS_TO_DELETE_KEY" = [${keys.mkString(",")}], "$CORRELATION_ID_KEY" = "$correlationId" } }
         |""".stripMargin
 
-    println(stringConfig)
+//    println(stringConfig)
 
     val rootConfig = ConfigFactory.parseString(stringConfig)
     val config = ConfigUtils.getOptionalConfig(rootConfig, KV_CONF_KEY)

consumers-spark/src/test/scala/it/agilelab/bigdata/wasp/consumers/spark/strategies/gdpr/HdfsDataDeletionSpec.scala

@@ -635,7 +635,7 @@ class HdfsDataDeletionSpec extends FlatSpec with Matchers with TryValues with Be
 
     import spark.implicits._
     val fileNames = readFileNameAndId
-    println(fileNames)
+//    println(fileNames)
 
     val rawDataStoreConf = RawDataStoreConf(
       keyColumn,
@@ -713,7 +713,7 @@ class HdfsDataDeletionSpec extends FlatSpec with Matchers with TryValues with Be
 
     import spark.implicits._
     val fileNames = readFileNameAndId
-    println(fileNames)
+//    println(fileNames)
 
     val rawDataStoreConf = RawDataStoreConf(
       keyColumn,