Add E2E tests for Log4j2 (#4600)

* Add E2E tests for Log4j2

* Format code

* codereview fixes

* add to CI matrix

---------

Co-authored-by: Sentry Github Bot <[email protected]>

Author: adinauer

.github/workflows/system-tests-backend.yml

@@ -57,6 +57,9 @@ jobs:
           - sample: "sentry-samples-logback"
             agent: "false"
             agent-auto-init: "true"
+          - sample: "sentry-samples-log4j2"
+            agent: "false"
+            agent-auto-init: "true"
     steps:
       - uses: actions/checkout@v4
         with:

sentry-samples/sentry-samples-log4j2/build.gradle.kts

@@ -1,18 +1,85 @@
+import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
+
 plugins {
   java
   application
+  kotlin("jvm")
   alias(libs.plugins.gradle.versions)
+  id("com.github.johnrengelman.shadow") version "8.1.1"
 }
 
 application { mainClass.set("io.sentry.samples.log4j2.Main") }
 
+java.sourceCompatibility = JavaVersion.VERSION_17
+
+java.targetCompatibility = JavaVersion.VERSION_17
+
+repositories { mavenCentral() }
+
 configure<JavaPluginExtension> {
-  sourceCompatibility = JavaVersion.VERSION_1_8
-  targetCompatibility = JavaVersion.VERSION_1_8
+  sourceCompatibility = JavaVersion.VERSION_17
+  targetCompatibility = JavaVersion.VERSION_17
+}
+
+tasks.withType<KotlinCompile>().configureEach {
+  kotlinOptions {
+    freeCompilerArgs = listOf("-Xjsr305=strict")
+    jvmTarget = JavaVersion.VERSION_17.toString()
+  }
 }
 
 dependencies {
   implementation(projects.sentryLog4j2)
   implementation(libs.log4j.api)
   implementation(libs.log4j.core)
+
+  testImplementation(kotlin(Config.kotlinStdLib))
+  testImplementation(projects.sentry)
+  testImplementation(projects.sentrySystemTestSupport)
+  testImplementation(libs.kotlin.test.junit)
+  testImplementation(libs.slf4j.api)
+  testImplementation(libs.slf4j.jdk14)
+}
+
+// Configure the Shadow JAR (executable JAR with all dependencies)
+tasks.shadowJar {
+  manifest { attributes["Main-Class"] = "io.sentry.samples.log4j2.Main" }
+  archiveClassifier.set("") // Remove the classifier so it replaces the regular JAR
+  mergeServiceFiles()
+  // Use Log4j2 cache transformer to properly handle plugin files
+  transform(
+    com.github.jengelman.gradle.plugins.shadow.transformers.Log4j2PluginsCacheFileTransformer()
+  )
+}
+
+// Make the regular jar task depend on shadowJar
+tasks.jar {
+  enabled = false
+  dependsOn(tasks.shadowJar)
+}
+
+// Fix the startScripts task dependency
+tasks.startScripts { dependsOn(tasks.shadowJar) }
+
+configure<SourceSetContainer> { test { java.srcDir("src/test/java") } }
+
+tasks.register<Test>("systemTest").configure {
+  group = "verification"
+  description = "Runs the System tests"
+
+  outputs.upToDateWhen { false }
+
+  maxParallelForks = 1
+
+  // Cap JVM args per test
+  minHeapSize = "128m"
+  maxHeapSize = "1g"
+
+  filter { includeTestsMatching("io.sentry.systemtest*") }
+}
+
+tasks.named("test").configure {
+  require(this is Test)
+
+  filter { excludeTestsMatching("io.sentry.systemtest.*") }
 }

sentry-samples/sentry-samples-log4j2/src/test/kotlin/io/sentry/DummyTest.kt

@@ -0,0 +1,12 @@
+package io.sentry
+
+import kotlin.test.Test
+import kotlin.test.assertTrue
+
+class DummyTest {
+  @Test
+  fun `the only test`() {
+    // only needed to have more than 0 tests and not fail the build
+    assertTrue(true)
+  }
+}

sentry-samples/sentry-samples-log4j2/src/test/kotlin/io/sentry/systemtest/ConsoleApplicationSystemTest.kt

@@ -0,0 +1,67 @@
+package io.sentry.systemtest
+
+import io.sentry.SentryLevel
+import io.sentry.systemtest.util.TestHelper
+import java.util.concurrent.TimeUnit
+import org.junit.Assert
+import org.junit.Before
+import org.junit.Test
+
+class ConsoleApplicationSystemTest {
+  lateinit var testHelper: TestHelper
+
+  @Before
+  fun setup() {
+    testHelper = TestHelper("http://localhost:8000")
+    testHelper.reset()
+  }
+
+  @Test
+  fun `log4j2 application sends expected events when run as JAR`() {
+    val jarFile = testHelper.findJar("sentry-samples-log4j2")
+    val process =
+      testHelper.launch(
+        jarFile,
+        mapOf(
+          "SENTRY_DSN" to testHelper.dsn,
+          "SENTRY_TRACES_SAMPLE_RATE" to "1.0",
+          "SENTRY_ENABLE_PRETTY_SERIALIZATION_OUTPUT" to "false",
+          "SENTRY_DEBUG" to "true",
+        ),
+      )
+
+    process.waitFor(30, TimeUnit.SECONDS)
+    Assert.assertEquals(0, process.exitValue())
+
+    // Verify that we received the expected events
+    verifyExpectedEvents()
+  }
+
+  private fun verifyExpectedEvents() {
+    // Verify we received the RuntimeException
+    testHelper.ensureErrorReceived { event ->
+      event.exceptions?.any { ex ->
+        ex.type == "RuntimeException" && ex.value == "Invalid productId=445"
+      } == true &&
+        event.message?.formatted == "Something went wrong" &&
+        event.level?.name == "ERROR"
+    }
+
+    testHelper.ensureErrorReceived { event ->
+      event.breadcrumbs?.firstOrNull {
+        it.message == "Hello Sentry!" && it.level == SentryLevel.DEBUG
+      } != null
+    }
+
+    testHelper.ensureErrorReceived { event ->
+      event.breadcrumbs?.firstOrNull {
+        it.message == "User has made a purchase of product: 445" && it.level == SentryLevel.INFO
+      } != null
+    }
+
+    testHelper.ensureLogsReceived { logs, _ ->
+      testHelper.doesContainLogWithBody(logs, "User has made a purchase of product: 445") &&
+        testHelper.doesContainLogWithBody(logs, "Something went wrong")
+    }
+  }
+}

test/system-test-runner.py

@@ -615,6 +615,7 @@ def get_available_modules(self) -> List[ModuleConfig]:
             ModuleConfig("sentry-samples-console", "false", "true", "false"),
             ModuleConfig("sentry-samples-console-opentelemetry-noagent", "false", "true", "false"),
             ModuleConfig("sentry-samples-logback", "false", "true", "false"),
+            ModuleConfig("sentry-samples-log4j2", "false", "true", "false"),
         ]
 
     def _find_module_number(self, module_name: str, agent: str, auto_init: str) -> int: