8365909: [REDO] Add a compilation timeout flag to catch long running compilations

src/hotspot/os/linux/compilerThreadTimeout_linux.cpp

@@ -0,0 +1,128 @@
+/*
+ * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "compiler/compilerThread.hpp"
+#include "compiler/compileTask.hpp"
+#include "compilerThreadTimeout_linux.hpp"
+#include "oops/method.hpp"
+#include "runtime/osThread.hpp"
+#include "signals_posix.hpp"
+#include "utilities/globalDefinitions.hpp"
+
+#include <pthread.h>
+
+#ifdef ASSERT
+void compiler_signal_handler(int signo, siginfo_t* info, void* context) {
+  CompilerThread::current()->timeout()->compiler_signal_handler(signo, info, context);
+}
+
+void CompilerThreadTimeoutLinux::compiler_signal_handler(int signo, siginfo_t* info, void* context) {
+  switch (signo) {
+    case TIMEOUT_SIGNAL: {
+      CompileTask* task = CompilerThread::current()->task();
+      const int SIZE = 512;
+      char method_name_buf[SIZE];
+      task->method()->name_and_sig_as_C_string(method_name_buf, SIZE);
+      assert(false, "compile task %d (%s) timed out after %zd ms",
+             task->compile_id(), method_name_buf, CompileTaskTimeout);
+    }
+    default: {
+      assert(false, "unexpected signal %d", signo);
+    }
+  }
+}
+#endif // ASSERT
+
+void CompilerThreadTimeoutLinux::arm() {
+#ifdef ASSERT
+  if (CompileTaskTimeout == 0) {
+    return;
+  }
+
+  const intx sec = (CompileTaskTimeout * NANOSECS_PER_MILLISEC) / NANOSECS_PER_SEC;
+  const intx nsec = (CompileTaskTimeout * NANOSECS_PER_MILLISEC) % NANOSECS_PER_SEC;
+  const struct timespec ts = {.tv_sec = sec, .tv_nsec = nsec};
+  const struct itimerspec its {.it_interval = ts, .it_value = ts};
+
+  // Start the timer.
+  timer_settime(_timer, 0, &its, nullptr);
+#endif // ASSERT
+}
+
+void CompilerThreadTimeoutLinux::disarm() {
+#ifdef ASSERT
+  if (CompileTaskTimeout == 0) {
+    return;
+  }
+
+  // Reset the timer by setting it to zero.
+  const struct itimerspec its {
+    .it_interval = {.tv_sec = 0, .tv_nsec=0},
+    .it_value = {.tv_sec = 0, .tv_nsec=0}
+  };
+  timer_settime(_timer, 0, &its, nullptr);
+#endif // ASSERT
+}
+
+bool CompilerThreadTimeoutLinux::init_timeout() {
+#ifdef ASSERT
+  if (CompileTaskTimeout == 0) {
+    return true;
+  }
+
+  JavaThread* thread = JavaThread::current();
+
+  // Create a POSIX timer sending SIGALRM to this thread only.
+  sigevent_t sev;
+  sev.sigev_value.sival_ptr = nullptr;
+  sev.sigev_signo = TIMEOUT_SIGNAL;
+  sev.sigev_notify = SIGEV_THREAD_ID;
+#ifdef MUSL_LIBC
+  sev.sigev_notify_thread_id = thread->osthread()->thread_id();
+#else
+  sev._sigev_un._tid = thread->osthread()->thread_id();
+#endif // MUSL_LIBC
+  clockid_t clock;
+  int err = pthread_getcpuclockid(thread->osthread()->pthread_id(), &clock);
+  if (err != 0) {
+    return false;
+  }
+  err = timer_create(clock, &sev, &_timer);
+  if (err != 0) {
+    return false;
+  }
+
+  // Install the signal handler and check that we do not have a conflicting handler.
+  struct sigaction sigact, sigact_old;
+  err = PosixSignals::install_sigaction_signal_handler(&sigact,
+                                                       &sigact_old,
+                                                       TIMEOUT_SIGNAL,
+                                                       (sa_sigaction_t)::compiler_signal_handler);
+  if (err != 0 || (sigact_old.sa_sigaction != sigact.sa_sigaction &&
+      sigact_old.sa_handler != SIG_DFL && sigact_old.sa_handler != SIG_IGN)) {
+    return false;
+  }
+#endif // ASSERT
+  return true;
+}

src/hotspot/os/linux/compilerThreadTimeout_linux.hpp

@@ -0,0 +1,51 @@
+/*
+ * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef LINUX_COMPILER_THREAD_TIMEOUT_LINUX_HPP
+#define LINUX_COMPILER_THREAD_TIMEOUT_LINUX_HPP
+
+#include "memory/allocation.hpp"
+#include "nmt/memTag.hpp"
+#include "utilities/macros.hpp"
+
+#include <csignal>
+#include <ctime>
+
+class CompilerThreadTimeoutLinux : public CHeapObj<mtCompiler> {
+#ifdef ASSERT
+ public:
+  static const int TIMEOUT_SIGNAL = SIGALRM;
+  void compiler_signal_handler(int signo, siginfo_t* info, void* context);
+ private:
+  timer_t          _timer;
+#endif // ASSERT
+ public:
+  CompilerThreadTimeoutLinux() DEBUG_ONLY(: _timer(nullptr)) {};
+
+  bool init_timeout();
+  void arm();
+  void disarm();
+};
+
+#endif //LINUX_COMPILER_THREAD_TIMEOUT_LINUX_HPP

src/hotspot/os/linux/globals_linux.hpp

@@ -89,6 +89,11 @@
   product(bool, PrintMemoryMapAtExit, false, DIAGNOSTIC,                \
           "Print an annotated memory map at exit")                      \
                                                                         \
+  develop(intx, CompileTaskTimeout, 0,                                  \
+          "Set the timeout for compile tasks' CPU time in milliseconds."\
+          " 0 = no timeout (default)")                                  \
+          range(0,1000000)                                              \
+                                                                        \
 // end of RUNTIME_OS_FLAGS
 
 //

src/hotspot/share/compiler/compileBroker.cpp

@@ -218,6 +218,7 @@ CompileTaskWrapper::CompileTaskWrapper(CompileTask* task) {
   CompilerThread* thread = CompilerThread::current();
   thread->set_task(task);
   CompileLog*     log  = thread->log();
+  thread->timeout()->arm();
   if (log != nullptr && !task->is_unloaded())  task->log_task_start(log);
 }
 
@@ -228,6 +229,7 @@ CompileTaskWrapper::~CompileTaskWrapper() {
   if (log != nullptr && !task->is_unloaded())  task->log_task_done(log);
   thread->set_task(nullptr);
   thread->set_env(nullptr);
+  thread->timeout()->disarm();
   if (task->is_blocking()) {
     bool free_task = false;
     {
@@ -1925,6 +1927,10 @@ void CompileBroker::compiler_thread_loop() {
     log->end_elem();
   }
 
+  if (!thread->init_compilation_timeout()) {
+    return;
+  }
+
   // If compiler thread/runtime initialization fails, exit the compiler thread
   if (!init_compiler_runtime()) {
     return;

src/hotspot/share/compiler/compilerThread.cpp

@@ -40,6 +40,7 @@ CompilerThread::CompilerThread(CompileQueue* queue,
   _can_call_java = false;
   _compiler = nullptr;
   _arena_stat = nullptr;
+  _timeout = nullptr;
 
 #ifndef PRODUCT
   _ideal_graph_printer = nullptr;
@@ -49,6 +50,7 @@ CompilerThread::CompilerThread(CompileQueue* queue,
 CompilerThread::~CompilerThread() {
   // Delete objects which were allocated on heap.
   delete _counters;
+  delete _timeout;
   // arenastat should have been deleted at the end of the compilation
   assert(_arena_stat == nullptr, "Should be null");
 }

src/hotspot/share/compiler/compilerThread.hpp

@@ -25,7 +25,14 @@
 #ifndef SHARE_COMPILER_COMPILERTHREAD_HPP
 #define SHARE_COMPILER_COMPILERTHREAD_HPP
 
+#include "memory/allocation.hpp"
+#include "nmt/memTag.hpp"
 #include "runtime/javaThread.hpp"
+#include "utilities/macros.hpp"
+
+#ifdef LINUX
+#include "compilerThreadTimeout_linux.hpp"
+#endif //LINUX
 
 class AbstractCompiler;
 class ArenaStatCounter;
@@ -38,10 +45,27 @@ class CompileQueue;
 class CompilerCounters;
 class IdealGraphPrinter;
 
+#ifndef LINUX
+class CompilerThreadTimeoutGeneric : public CHeapObj<mtCompiler> {
+ public:
+  CompilerThreadTimeoutGeneric() {};
+  void arm() {};
+  void disarm() {};
+  bool init_timeout() { return true; };
+};
+#endif // !LINUX
+
 // A thread used for Compilation.
 class CompilerThread : public JavaThread {
   friend class VMStructs;
   JVMCI_ONLY(friend class CompilerThreadCanCallJava;)
+
+#ifdef LINUX
+  typedef CompilerThreadTimeoutLinux Timeout;
+#else // LINUX
+  typedef CompilerThreadTimeoutGeneric Timeout;
+#endif // LINUX
+
  private:
   CompilerCounters* _counters;
 
@@ -57,6 +81,7 @@ class CompilerThread : public JavaThread {
 
   ArenaStatCounter*     _arena_stat;
 
+  Timeout*              _timeout;
  public:
 
   static CompilerThread* current() {
@@ -113,7 +138,13 @@ class CompilerThread : public JavaThread {
  public:
   IdealGraphPrinter *ideal_graph_printer()           { return _ideal_graph_printer; }
   void set_ideal_graph_printer(IdealGraphPrinter *n) { _ideal_graph_printer = n; }
-#endif
+#endif // !PRODUCT
+
+  Timeout* timeout() const { return _timeout; };
+  bool init_compilation_timeout() {
+    _timeout = new Timeout();
+    return _timeout->init_timeout();
+  };
 
   // Get/set the thread's current task
   CompileTask* task()                      { return _task; }

test/hotspot/jtreg/compiler/arguments/TestCompileTaskTimeout.java

@@ -0,0 +1,55 @@
+/*
+ * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+package compiler.arguments;
+
+/*
+ * @test TestCompileTaskTimeout
+ * @bug 8308094 8365909
+ * @requires vm.debug & vm.flagless & os.name == "Linux"
+ * @summary Check functionality of CompileTaskTimeout
+ * @library /test/lib
+ * @run driver compiler.arguments.TestCompileTaskTimeout
+ */
+
+import jdk.test.lib.process.ProcessTools;
+
+public class TestCompileTaskTimeout {
+
+    public static void main(String[] args) throws Throwable {
+        ProcessTools.executeTestJava("-Xcomp", "-XX:CompileTaskTimeout=1", "--version")
+                    .shouldHaveExitValue(134)
+                    .shouldContain("timed out after");
+
+        ProcessTools.executeTestJava("-Xcomp", "-XX:CompileTaskTimeout=1", "-XX:TieredStopAtLevel=3", "--version")
+                    .shouldHaveExitValue(134)
+                    .shouldContain("timed out after");
+
+        ProcessTools.executeTestJava("-Xcomp", "-XX:CompileTaskTimeout=1", "-XX:-TieredCompilation", "--version")
+                    .shouldHaveExitValue(134)
+                    .shouldContain("timed out after");
+
+        ProcessTools.executeTestJava("-Xcomp", "-XX:CompileTaskTimeout=200", "--version")
+                    .shouldHaveExitValue(0);
+    }
+}

test/hotspot/jtreg/runtime/signal/TestSigalrm.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2025, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -24,7 +24,7 @@
 
 /*
  * @test
- * @requires os.family != "windows" & os.family != "aix"
+ * @requires os.family != "windows" & os.family != "aix" & vm.flagless
  *
  * @summary converted from VM testbase runtime/signal/sigalrm01.
  * VM testbase keywords: [signal, runtime, linux, macosx]