Upgrade to log4j 2.25.1

build-tools-internal/version.properties

@@ -10,7 +10,7 @@ jackson           = 2.15.0
 snakeyaml         = 2.0
 icu4j             = 77.1
 supercsv          = 2.4.0
-log4j             = 2.19.0
+log4j             = 2.25.1
 slf4j             = 2.0.6
 ecsLogging        = 1.2.0
 jna               = 5.12.1

docs/changelog/132166.yaml

@@ -0,0 +1,6 @@
+pr: 132166
+summary: Upgrade to log4j 2.25.1
+area: Infra/Logging
+type: upgrade
+issues:
+  - 132035

gradle/verification-metadata.xml

@@ -502,6 +502,11 @@
             <sha256 value="3ef6c9f822b601aa151e10e123b49e5604243a4a99bcc47e4e1f9eea9781dc63" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="com.github.spotbugs" name="spotbugs-annotations" version="4.8.6">
+         <artifact name="spotbugs-annotations-4.8.6.jar">
+            <sha256 value="4548b74a815ed44f5480ca4f06204a8b00809dc7e5f6a825a9edf18f40377b65" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="com.github.spullara.mustache.java" name="compiler" version="0.9.10">
          <artifact name="compiler-0.9.10.jar">
             <sha256 value="2b5a9217811cb99846a473fa8e0d233eb33629347b7f44941f6c0fbd4cdf1038" origin="Generated by Gradle"/>
@@ -687,6 +692,11 @@
             <sha256 value="baf7d6ea97ce606c53e11b6854ba5f2ce7ef5c24dddf0afa18d1260bd25b002c" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="com.google.errorprone" name="error_prone_annotations" version="2.38.0">
+         <artifact name="error_prone_annotations-2.38.0.jar">
+            <sha256 value="6661d5335090a5fc61dd869d2095bc6c1e2156e3aa47a6e4ababdf64c99a7889" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="com.google.errorprone" name="error_prone_annotations" version="2.36.0">
          <artifact name="error_prone_annotations-2.36.0.jar">
             <sha256 value="77440e270b0bc9a249903c5a076c36a722c4886ca4f42675f2903a1c53ed61a5" origin="Generated by Gradle"/>
@@ -3110,6 +3120,11 @@
             <sha256 value="60480a6c81978a9871099e002997df84a05fc11789fb248e3e803a0dcb19e0cd" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.apache.logging.log4j" name="log4j-1.2-api" version="2.25.1">
+         <artifact name="log4j-1.2-api-2.25.1.jar">
+            <sha256 value="6b3babf23e3dd6f3d9db992beb7cf12ea96ba2a19164d61d1ab3e5ce7ba42c2b" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.apache.logging.log4j" name="log4j-api" version="2.17.1">
          <artifact name="log4j-api-2.17.1.jar">
             <sha256 value="b0d8a4c8ab4fb8b1888d0095822703b0e6d4793c419550203da9e69196161de4" origin="Generated by Gradle"/>
@@ -3175,11 +3190,21 @@
             <sha256 value="47f768ffd66107a66f0c2a19445ab1e42ce6719a7f30f9aa9ef96157c83949fd" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.apache.logging.log4j" name="log4j-jcl" version="2.25.1">
+         <artifact name="log4j-jcl-2.25.1.jar">
+            <sha256 value="f9dd2fdf1b924f2f037ea496f886c79940b24aab2e0f0eb1b008680d00265be6" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.apache.logging.log4j" name="log4j-slf4j-impl" version="2.19.0">
          <artifact name="log4j-slf4j-impl-2.19.0.jar">
             <sha256 value="015d5c229f3cd5c0ebf175c1da08d596d94043362ae9d92637d88848c90537c8" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.apache.logging.log4j" name="log4j-slf4j-impl" version="2.25.1">
+         <artifact name="log4j-slf4j-impl-2.25.1.jar">
+            <sha256 value="af6528455568ece7828b9eb621d1baad6341cd2608b12e47055bbba10f718248" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.apache.lucene" name="lucene-analysis-common" version="10.3.0">
          <artifact name="lucene-analysis-common-10.3.0.jar">
             <sha256 value="124bddea3fa0683d3e174552fba5fcf0f24a2f57f1ccf6957010d010d246db8b" origin="Generated by Gradle"/>

qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/config/log4j2.properties

@@ -30,7 +30,7 @@ appender.header_warning.type = HeaderWarningAppender
 appender.header_warning.name = header_warning
 
 logger.deprecation.name = deprecation
-logger.deprecation.level = deprecation
+logger.deprecation.level = warn
 logger.deprecation.appenderRef.deprecation_file.ref = deprecation_file
 logger.deprecation.appenderRef.header_warning.ref = header_warning
 logger.deprecation.additivity = false

qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/deprecation/log4j2.properties

@@ -24,7 +24,7 @@ appender.header_warning.type = HeaderWarningAppender
 appender.header_warning.name = header_warning
 
 logger.deprecation.name = deprecation
-logger.deprecation.level = deprecation
+logger.deprecation.level = warn
 logger.deprecation.appenderRef.deprecation_file.ref = deprecation_file
 logger.deprecation.appenderRef.header_warning.ref = header_warning
 logger.deprecation.additivity = false

qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/no_node_name/log4j2.properties

@@ -24,7 +24,7 @@ appender.header_warning.type = HeaderWarningAppender
 appender.header_warning.name = header_warning
 
 logger.deprecation.name = deprecation
-logger.deprecation.level = deprecation
+logger.deprecation.level = warn
 logger.deprecation.appenderRef.deprecation_file.ref = deprecation_file
 logger.deprecation.appenderRef.header_warning.ref = header_warning
 logger.deprecation.additivity = false

qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/settings/log4j2.properties

@@ -24,7 +24,7 @@ appender.header_warning.type = HeaderWarningAppender
 appender.header_warning.name = header_warning
 
 logger.deprecation.name = org.elasticsearch.deprecation.common.settings
-logger.deprecation.level = deprecation
+logger.deprecation.level = warn
 logger.deprecation.appenderRef.deprecation_console.ref = console
 logger.deprecation.appenderRef.deprecation_file.ref = deprecation_file
 logger.deprecation.appenderRef.header_warning.ref = header_warning

qa/logging-config/src/javaRestTest/resources/es-v7-log4j2.properties

@@ -102,7 +102,7 @@ appender.header_warning.type = HeaderWarningAppender
 appender.header_warning.name = header_warning
 #################################################
 logger.deprecation.name = org.elasticsearch.deprecation
-logger.deprecation.level = deprecation
+logger.deprecation.level = warn
 logger.deprecation.appenderRef.deprecation_rolling.ref = deprecation_rolling
 logger.deprecation.appenderRef.deprecation_rolling_old.ref = deprecation_rolling_old
 logger.deprecation.appenderRef.header_warning.ref = header_warning

server/build.gradle

@@ -202,12 +202,12 @@ tasks.named("thirdPartyAudit").configure {
             'com.fasterxml.jackson.databind.ser.impl.SimpleFilterProvider',
             'com.fasterxml.jackson.databind.ser.std.StdScalarSerializer',
             'com.fasterxml.jackson.databind.ser.std.StdSerializer',
+            'com.fasterxml.jackson.databind.util.ClassUtil',
             'com.fasterxml.jackson.dataformat.xml.JacksonXmlModule',
             'com.fasterxml.jackson.dataformat.xml.XmlMapper',
             'com.fasterxml.jackson.dataformat.xml.util.DefaultXmlPrettyPrinter',
-            'org.fusesource.jansi.Ansi',
-            'org.fusesource.jansi.AnsiRenderer$Code',
             'com.lmax.disruptor.EventFactory',
+            'com.lmax.disruptor.EventHandler',
             'com.lmax.disruptor.EventTranslator',
             'com.lmax.disruptor.EventTranslatorTwoArg',
             'com.lmax.disruptor.EventTranslatorVararg',
@@ -246,6 +246,7 @@ tasks.named("thirdPartyAudit").configure {
             'org.apache.commons.csv.QuoteMode',
             'org.apache.kafka.clients.producer.Producer',
             'org.apache.kafka.clients.producer.RecordMetadata',
+            'org.apache.kafka.common.serialization.ByteArraySerializer',
             'org.codehaus.stax2.XMLStreamWriter2',
             'org.jctools.queues.MpscArrayQueue',
             'org.osgi.framework.Bundle',
@@ -257,13 +258,24 @@ tasks.named("thirdPartyAudit").configure {
             'org.osgi.framework.ServiceReference',
             'org.osgi.framework.ServiceRegistration',
             'org.osgi.framework.SynchronousBundleListener',
+            'org.osgi.framework.wiring.BundleRevision',
             'org.osgi.framework.wiring.BundleWire',
             'org.osgi.framework.wiring.BundleWiring',
+            'org.zeromq.SocketType',
+            'org.zeromq.ZContext',
+            'org.zeromq.ZMonitor',
+            'org.zeromq.ZMonitor$Event',
+            'org.zeromq.ZMonitor$ZEvent',
             'org.zeromq.ZMQ$Context',
             'org.zeromq.ZMQ$Socket',
             'org.zeromq.ZMQ',
     )
     ignoreMissingClasses 'javax.xml.bind.DatatypeConverter'
+
+    ignoreViolations(
+        'org.apache.logging.log4j.core.util.internal.UnsafeUtil',
+        'org.apache.logging.log4j.core.util.internal.UnsafeUtil$1',
+    )
 }
 
 tasks.named("dependencyLicenses").configure {

server/src/main/java/org/elasticsearch/action/search/SearchPhaseExecutionException.java

@@ -26,21 +26,26 @@
 public class SearchPhaseExecutionException extends ElasticsearchException {
     private final String phaseName;
     private final ShardSearchFailure[] shardFailures;
+    private final ElasticsearchException guessedCause; // log4j requires a stable cause!
 
     public SearchPhaseExecutionException(String phaseName, String msg, ShardSearchFailure[] shardFailures) {
         this(phaseName, msg, null, shardFailures);
     }
 
+    @SuppressWarnings("this-escape")
     public SearchPhaseExecutionException(String phaseName, String msg, Throwable cause, ShardSearchFailure[] shardFailures) {
         super(msg, deduplicateCause(cause, shardFailures));
         this.phaseName = phaseName;
         this.shardFailures = shardFailures;
+        this.guessedCause = cause == null || super.getCause() == null ? guessFirstRootCause(shardFailures) : null;
     }
 
+    @SuppressWarnings("this-escape")
     public SearchPhaseExecutionException(StreamInput in) throws IOException {
         super(in);
         phaseName = in.readOptionalString();
         shardFailures = in.readArray(ShardSearchFailure::readShardSearchFailure, ShardSearchFailure[]::new);
+        guessedCause = super.getCause() == null ? guessFirstRootCause(shardFailures) : null;
     }
 
     @Override
@@ -96,14 +101,9 @@ public ShardSearchFailure[] shardFailures() {
 
     @Override
     public Throwable getCause() {
+        // note: log4j requires this to return a stable, consistent cause when called multiple times
         Throwable cause = super.getCause();
-        if (cause == null) {
-            // fall back to guessed root cause
-            for (ElasticsearchException rootCause : guessRootCauses()) {
-                return rootCause;
-            }
-        }
-        return cause;
+        return cause != null ? cause : guessedCause;
     }
 
     @Override
@@ -144,6 +144,16 @@ public ElasticsearchException[] guessRootCauses() {
         return rootCauses.toArray(new ElasticsearchException[0]);
     }
 
+    private static ElasticsearchException guessFirstRootCause(ShardSearchFailure[] shardFailures) {
+        for (ShardOperationFailedException failure : shardFailures) {
+            ElasticsearchException[] rootCauses = ElasticsearchException.guessRootCauses(failure.getCause());
+            if (rootCauses.length > 0) {
+                return rootCauses[0];
+            }
+        }
+        return null;
+    }
+
     @Override
     public String toString() {
         return "Failed to execute phase ["

test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java

@@ -152,9 +152,11 @@
 import org.elasticsearch.xcontent.XContentParser.Token;
 import org.elasticsearch.xcontent.XContentParserConfiguration;
 import org.elasticsearch.xcontent.XContentType;
+import org.hamcrest.Description;
 import org.hamcrest.Matcher;
 import org.hamcrest.MatcherAssert;
 import org.hamcrest.Matchers;
+import org.hamcrest.TypeSafeMatcher;
 import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.Before;
@@ -223,10 +225,9 @@
 import static java.util.Collections.emptyMap;
 import static org.elasticsearch.common.util.CollectionUtils.arrayAsArrayList;
 import static org.hamcrest.Matchers.anyOf;
-import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.empty;
-import static org.hamcrest.Matchers.emptyCollectionOf;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.everyItem;
 import static org.hamcrest.Matchers.hasItem;
 import static org.hamcrest.Matchers.startsWith;
 
@@ -823,11 +824,20 @@ public void log(StatusData data) {
 
     // Tolerate the absence or otherwise denial of these specific lookup classes.
     // At some future time, we should require the JDNI warning.
-    private static final List<String> LOG_4J_MSG_PREFIXES = List.of(
-        "JNDI lookup class is not available because this JRE does not support JNDI. "
-            + "JNDI string lookups will not be available, continuing configuration.",
-        "JMX runtime input lookup class is not available because this JRE does not support JMX. "
-            + "JMX lookups will not be available, continuing configuration. "
+    private static final Matcher<String> LOG_4J_MSG_PREFIXES = anyOf(
+        startsWith(
+            "JNDI lookup class is not available because this JRE does not support JNDI. "
+                + "JNDI string lookups will not be available, continuing configuration."
+        ),
+        startsWith(
+            "JMX runtime input lookup class is not available because this JRE does not support JMX. "
+                + "JMX lookups will not be available, continuing configuration. "
+        ),
+        // TODO migrate to annotation processor https://github.com/elastic/elasticsearch/issues/135022
+        startsWith("The use of package scanning to locate Log4j plugins is deprecated."),
+        startsWith("Some custom `Core` Log4j plugins are not properly registered"),
+        startsWith("Some custom `Converter` Log4j plugins are not properly registered"),
+        startsWith("No Root logger was configured, creating default ERROR-level Root logger with Console appender")
     );
 
     // separate method so that this can be checked again after suite scoped cluster is shut down
@@ -836,16 +846,25 @@ protected static void checkStaticState() throws Exception {
         assertThat(StatusLogger.getLogger().getLevel(), equalTo(Level.WARN));
         synchronized (statusData) {
             try {
-                // ensure that there are no status logger messages which would indicate a problem with our Log4j usage; we map the
-                // StatusData instances to Strings as otherwise their toString output is useless
-                assertThat(
-                    statusData.stream().map(status -> status.getMessage().getFormattedMessage()).collect(Collectors.toList()),
-                    anyOf(
-                        emptyCollectionOf(String.class),
-                        contains(startsWith(LOG_4J_MSG_PREFIXES.get(0)), startsWith(LOG_4J_MSG_PREFIXES.get(1))),
-                        contains(startsWith(LOG_4J_MSG_PREFIXES.get(1)))
-                    )
-                );
+                // ensure that there are no status logger messages which would indicate a problem with our Log4j usage;
+                assertThat(statusData, everyItem(new TypeSafeMatcher<StatusData>() {
+                    @Override
+                    protected boolean matchesSafely(StatusData item) {
+                        return LOG_4J_MSG_PREFIXES.matches(item.getFormattedStatus());
+                    }
+
+                    @Override
+                    public void describeTo(Description description) {
+                        LOG_4J_MSG_PREFIXES.describeTo(description);
+                    }
+
+                    @Override
+                    protected void describeMismatchSafely(StatusData item, Description mismatchDescription) {
+                        // make sure we see log4j exceptions in case of issues
+                        mismatchDescription.appendText("was ").appendValue(item.getFormattedStatus());
+                    }
+                }));
+
             } finally {
                 // we clear the list so that status data from other tests do not interfere with tests within the same JVM
                 statusData.clear();

test/framework/src/main/resources/log4j2-test.properties

@@ -10,5 +10,5 @@ appender.header_warning.type = HeaderWarningAppender
 appender.header_warning.name = header_warning
 
 logger.deprecation.name = org.elasticsearch.deprecation
-logger.deprecation.level = deprecation
+logger.deprecation.level = warn
 logger.deprecation.appenderRef.header_warning.ref = header_warning

x-pack/plugin/core/build.gradle

@@ -32,7 +32,7 @@ esplugin {
 tasks.named("dependencyLicenses").configure {
   mapping from: /http.*/, to: 'httpclient' // pulled in by rest client
   mapping from: /commons-.*/, to: 'commons' // pulled in by rest client
-  mapping from: /slf4j-.*/, to: 'slf4j' 
+  mapping from: /slf4j-.*/, to: 'slf4j'
 }
 
 configurations {
@@ -157,8 +157,6 @@ tasks.named("thirdPartyAudit").configure {
     'javax.jms.Message',
     // HttpClient5 can use Conscrypt (TLS using BoringSSL), but we don't want that
     'org.conscrypt.Conscrypt',
-    // SLF4j via HttpClient5
-    'org.slf4j.ext.EventData'
   )
 }
 

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/support/NoOpLogger.java

@@ -10,6 +10,7 @@
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.Marker;
 import org.apache.logging.log4j.message.EntryMessage;
+import org.apache.logging.log4j.message.FlowMessageFactory;
 import org.apache.logging.log4j.message.Message;
 import org.apache.logging.log4j.message.MessageFactory;
 import org.apache.logging.log4j.util.MessageSupplier;
@@ -930,6 +931,11 @@ public <MF extends MessageFactory> MF getMessageFactory() {
         return null;
     }
 
+    @Override
+    public FlowMessageFactory getFlowMessageFactory() {
+        return null;
+    }
+
     @Override
     public String getName() {
         return null;

x-pack/plugin/ent-search/build.gradle

@@ -63,13 +63,6 @@ tasks.named("dependencyLicenses") {
   mapping from: /jackson.*/, to: 'jackson'
 }
 
-tasks.named("thirdPartyAudit") {
-  ignoreMissingClasses(
-    // [missing classes] SLF4j includes an optional class that depends on an extension class (!)
-    'org.slf4j.ext.EventData'
-  )
-}
-
 tasks.named("yamlRestTest") {
   usesDefaultDistribution("uses the xpack/usage api")
 }

x-pack/plugin/identity-provider/build.gradle

@@ -249,8 +249,6 @@ tasks.named("thirdPartyAudit").configure {
         'org.bouncycastle.operator.jcajce.JcaContentSignerBuilder',
         'org.bouncycastle.util.Arrays',
         'org.bouncycastle.util.io.Streams',
-        // SLF4j
-        'org.slf4j.ext.EventData'
     )
 
   ignoreViolations(

x-pack/plugin/security/build.gradle

@@ -426,8 +426,6 @@ tasks.named("thirdPartyAudit").configure {
     'javax.activation.DataSource',
     'javax.activation.FileDataSource',
     'javax.activation.FileTypeMap',
-    // SLF4j
-    'org.slf4j.ext.EventData'
   )
 }