Using Channel to load Java classes

.github/workflows/enso4igv.yml

@@ -79,7 +79,7 @@ jobs:
             target/lib/**
 
   build_windows_parser:
-    runs-on: windows-2019
+    runs-on: windows-2022
 
     steps:
       - uses: actions/checkout@v4

build.sbt

@@ -346,6 +346,8 @@ lazy val enso = (project in file("."))
     `interpreter-dsl-test`,
     `jna-wrapper`,
     `json-rpc-server`,
+    `jvm-channel`,
+    `jvm-interop`,
     `language-server`,
     `language-server-deps-wrapper`,
     launcher,
@@ -788,6 +790,8 @@ lazy val componentModulesPaths =
     (`logging-utils-akka` / Compile / exportedModuleBin).value,
     (`logging-service` / Compile / exportedModuleBin).value,
     (`logging-service-logback` / Compile / exportedModuleBin).value,
+    (`jvm-channel` / Compile / exportedModuleBin).value,
+    (`jvm-interop` / Compile / exportedModuleBin).value,
     (`os-environment` / Compile / exportedModuleBin).value,
     (`pkg` / Compile / exportedModuleBin).value,
     (`refactoring-utils` / Compile / exportedModuleBin).value,
@@ -3806,6 +3810,7 @@ lazy val `engine-runner` = project
       (`profiling-utils` / Compile / exportedModule).value,
       (`semver` / Compile / exportedModule).value,
       (`cli` / Compile / exportedModule).value,
+      (`jvm-channel` / Compile / exportedModule).value,
       (`os-environment` / Compile / exportedModule).value,
       (`distribution-manager` / Compile / exportedModule).value,
       (`editions` / Compile / exportedModule).value,
@@ -3828,6 +3833,10 @@ lazy val `engine-runner` = project
     NativeImage.additionalCp := {
       val runnerDeps =
         (Compile / fullClasspath).value.map(_.data.getAbsolutePath)
+      val jvmInteropDeps =
+        (`jvm-interop` / Compile / fullClasspath).value.map(
+          _.data.getAbsolutePath
+        )
       val runtimeDeps =
         (`runtime` / Compile / fullClasspath).value.map(_.data.getAbsolutePath)
       val loggingDeps =
@@ -3861,6 +3870,7 @@ lazy val `engine-runner` = project
       }
       val core = (
         runnerDeps ++
+          jvmInteropDeps ++
           runtimeDeps ++
           loggingDeps ++
           replDebugInstr ++
@@ -4253,13 +4263,75 @@ lazy val `benchmarks-common` =
     )
     .dependsOn(`polyglot-api`)
 
+lazy val `jvm-channel` =
+  project
+    .in(file("lib/java/jvm-channel"))
+    .enablePlugins(JPMSPlugin)
+    .settings(
+      customFrgaalJavaCompilerSettings("24"),
+      autoScalaLibrary := false,
+      (Test / fork) := true,
+      commands += WithDebugCommand.withDebug,
+      libraryDependencies ++= slf4jApi ++ Seq(
+        "org.graalvm.sdk" % "nativeimage"     % graalMavenPackagesVersion % "provided",
+        "org.graalvm.sdk" % "graal-sdk"       % graalMavenPackagesVersion % "provided",
+        "junit"           % "junit"           % junitVersion              % Test,
+        "com.github.sbt"  % "junit-interface" % junitIfVersion            % Test
+      ),
+      Compile / moduleDependencies ++= slf4jApi ++ Seq(
+        "org.graalvm.sdk"      % "nativeimage" % graalMavenPackagesVersion,
+        "org.graalvm.polyglot" % "polyglot"    % graalMavenPackagesVersion,
+        "org.graalvm.sdk"      % "word"        % graalMavenPackagesVersion
+      ),
+      Compile / internalModuleDependencies ++= Seq(
+        (`engine-common` / Compile / exportedModule).value,
+        (`persistance` / Compile / exportedModule).value
+      )
+    )
+    .dependsOn(`engine-common`)
+    .dependsOn(`persistance`)
+    .dependsOn(`persistance-dsl` % "provided")
+
+lazy val `jvm-interop` =
+  project
+    .in(file("lib/java/jvm-interop"))
+    .enablePlugins(JPMSPlugin)
+    .settings(
+      frgaalJavaCompilerSetting,
+      autoScalaLibrary := false,
+      (Test / fork) := true,
+      commands += WithDebugCommand.withDebug,
+      libraryDependencies ++= slf4jApi ++ Seq(
+        "org.graalvm.truffle" % "truffle-api"             % graalMavenPackagesVersion % "provided",
+        "org.graalvm.truffle" % "truffle-dsl-processor"   % graalMavenPackagesVersion % "provided",
+        "org.netbeans.api"    % "org-openide-util-lookup" % netbeansApiVersion        % "provided",
+        "org.graalvm.sdk"     % "graal-sdk"               % graalMavenPackagesVersion % Test,
+        "junit"               % "junit"                   % junitVersion              % Test,
+        "com.github.sbt"      % "junit-interface"         % junitIfVersion            % Test
+      ),
+      Compile / moduleDependencies ++= slf4jApi ++ Seq(
+        "org.netbeans.api"     % "org-openide-util-lookup" % netbeansApiVersion,
+        "org.graalvm.truffle"  % "truffle-api"             % graalMavenPackagesVersion,
+        "org.graalvm.sdk"      % "nativeimage"             % graalMavenPackagesVersion,
+        "org.graalvm.polyglot" % "polyglot"                % graalMavenPackagesVersion,
+        "org.graalvm.sdk"      % "word"                    % graalMavenPackagesVersion
+      ),
+      Compile / internalModuleDependencies ++= Seq(
+        (`jvm-channel` / Compile / exportedModule).value,
+        (`engine-common` / Compile / exportedModule).value,
+        (`persistance` / Compile / exportedModule).value
+      )
+    )
+    .dependsOn(`jvm-channel`)
+    .dependsOn(`persistance-dsl` % "provided")
+    .dependsOn(`test-utils` % Test)
+
 lazy val `os-environment` =
   project
     .in(file("lib/java/os-environment"))
     .enablePlugins(JPMSPlugin)
     .settings(
-      customFrgaalJavaCompilerSettings("24"),
-      scalaModuleDependencySetting,
+      frgaalJavaCompilerSetting,
       libraryDependencies ++= slf4jApi ++ Seq(
         "org.graalvm.sdk" % "nativeimage"     % graalMavenPackagesVersion % "provided",
         "org.graalvm.sdk" % "graal-sdk"       % graalMavenPackagesVersion % "provided",
@@ -4277,6 +4349,7 @@ lazy val `os-environment` =
       Compile / internalModuleDependencies ++= Seq(
         (`engine-common` / Compile / exportedModule).value,
         (`persistance` / Compile / exportedModule).value,
+        (`jvm-channel` / Compile / exportedModule).value,
         (`logging-utils` / Compile / exportedModule).value,
         (`logging-config` / Compile / exportedModule).value
       ),
@@ -4332,6 +4405,7 @@ lazy val `os-environment` =
         .value,
       Test / fork := true
     )
+    .dependsOn(`jvm-channel`)
     .dependsOn(`persistance`)
     .dependsOn(`persistance-dsl` % "provided")
     .dependsOn(`engine-common`)

engine/common/src/main/java/module-info.java

@@ -1,3 +1,5 @@
+import org.enso.common.PolyglotSymbolResolver;
+
 module org.enso.engine.common {
   requires org.graalvm.nativeimage;
   requires org.graalvm.polyglot;
@@ -6,4 +8,6 @@
   requires org.slf4j;
 
   exports org.enso.common;
+
+  uses PolyglotSymbolResolver;
 }

engine/common/src/main/java/org/enso/common/PolyglotSymbolResolver.java

@@ -0,0 +1,74 @@
+package org.enso.common;
+
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.ServiceLoader;
+
+/**
+ * Generic support for loading Java polyglot symbols. The resolver provides two kinds of interfaces:
+ *
+ * <ul>
+ *   <li>the client API - represented by all the <b>public static</b> methods
+ *   <li>the SPI - e.g. service provider interface - those are the <b>protected abstract</b> methods
+ * </ul>
+ *
+ * Those who tend to extend the capabilities of loading Java classes into Enso runtime shall
+ * register their own implementation visible via {@link ServiceLoader}.
+ *
+ * @see RuntimeOptions#HOST_CLASS_LOADING
+ */
+public abstract class PolyglotSymbolResolver {
+  private static final Collection<PolyglotSymbolResolver> ALL;
+
+  static {
+    var arr = new ArrayList<PolyglotSymbolResolver>();
+    for (var l : ServiceLoader.load(PolyglotSymbolResolver.class)) {
+      arr.add(l);
+    }
+    ALL = Collections.unmodifiableList(arr);
+  }
+
+  /**
+   * Search all providers for given name.
+   *
+   * @param name dot separated name to search for
+   * @return non-null object representing the name
+   * @throws java.lang.ClassNotFoundException if no name was found
+   */
+  public static Object loadClass(String name) throws ClassNotFoundException {
+    ClassNotFoundException ex = null;
+    for (var p : ALL) {
+      try {
+        var found = p.handleLoadClass(name);
+        assert found != null;
+        return found;
+      } catch (ClassNotFoundException cnfe) {
+        ex = cnfe;
+      }
+    }
+    if (ex == null) {
+      throw new ClassNotFoundException(name);
+    } else {
+      throw ex;
+    }
+  }
+
+  public static void addToClassPath(URL url) {
+    for (var p : ALL) {
+      p.handleAddToClassPath(url);
+    }
+  }
+
+  /**
+   * Subclasses implement this method to search for class with the provided name.
+   *
+   * @param name dot separated name to search for
+   * @return non-{@code null} object representing the name
+   * @throws java.lang.ClassNotFoundException if no name was found
+   */
+  protected abstract Object handleLoadClass(String name) throws ClassNotFoundException;
+
+  protected abstract void handleAddToClassPath(URL url);
+}

engine/common/src/main/java/org/enso/common/RuntimeOptions.java

@@ -35,6 +35,15 @@ private RuntimeOptions() {}
   private static final OptionDescriptor ENABLE_STATIC_ANALYSIS_DESCRIPTOR =
       OptionDescriptor.newBuilder(ENABLE_STATIC_ANALYSIS_KEY, ENABLE_STATIC_ANALYSIS).build();
 
+  public static final String HOST_CLASS_LOADING = optionName("classLoading");
+  public static final OptionKey<String> HOST_CLASS_LOADING_KEY = new OptionKey<>("hosted");
+  private static final OptionDescriptor HOST_CLASS_LOADING_DESCRIPTOR =
+      OptionDescriptor.newBuilder(HOST_CLASS_LOADING_KEY, HOST_CLASS_LOADING)
+          .help("Controls the way Enso runtime resolves polyglot java import statements")
+          .usageSyntax("Possible values are <hosted|service|all>")
+          .category(OptionCategory.INTERNAL)
+          .build();
+
   public static final String TREAT_WARNINGS_AS_ERRORS = optionName("treatWarningsAsErrors");
   public static final OptionKey<Boolean> TREAT_WARNINGS_AS_ERRORS_KEY = new OptionKey<>(false);
   private static final OptionDescriptor TREAT_WARNINGS_AS_ERRORS_DESCRIPTOR =
@@ -173,6 +182,7 @@ private RuntimeOptions() {}
               DISABLE_INLINE_CACHES_DESCRIPTOR,
               DISABLE_PRIVATE_CHECK_DESCRIPTOR,
               ENABLE_STATIC_ANALYSIS_DESCRIPTOR,
+              HOST_CLASS_LOADING_DESCRIPTOR,
               TREAT_WARNINGS_AS_ERRORS_DESCRIPTOR,
               ENABLE_AUTO_PARALLELISM_DESCRIPTOR,
               ENABLE_PROJECT_SUGGESTIONS_DESCRIPTOR,

engine/launcher/src/main/scala/org/enso/launcher/distribution/DefaultManagers.scala

@@ -51,7 +51,7 @@ object DefaultManagers {
         alwaysInstallMissing
       ),
       distributionManager,
-      new GraalVersionManager(distributionManager, LauncherEnvironment),
+      new GraalVersionManager(distributionManager),
       temporaryDirectoryManager,
       defaultResourceManager,
       EngineRepository.defaultEngineReleaseProvider,

engine/runner/src/main/java/module-info.java

@@ -8,6 +8,7 @@
   requires org.enso.librarymanager;
   requires org.enso.logging.config;
   requires org.enso.logging.utils;
+  requires org.enso.jvm.channel;
   requires org.enso.os.environment;
   requires org.enso.runtime.parser;
   requires org.enso.runtime.version.manager;

engine/runner/src/main/java/org/enso/runner/JavaFinder.java

@@ -79,7 +79,7 @@ private static Path findJavaExecutableInDistributionRuntimes() {
     if (distributionManager.isRunningPortable()) {
       logger.trace("Running in portable distribution");
     }
-    var graalVersionManager = new GraalVersionManager(distributionManager, env);
+    var graalVersionManager = new GraalVersionManager(distributionManager);
     var versionUsedForBuild =
         new GraalVMVersion(BuildVersion.graalVersion(), BuildVersion.javaVersion());
     var runtimeWithExactVersionMatch = graalVersionManager.findGraalRuntime(versionUsedForBuild);

engine/runner/src/main/java/org/enso/runner/Main.java

@@ -35,8 +35,8 @@
 import org.enso.distribution.DistributionManager;
 import org.enso.distribution.Environment;
 import org.enso.editions.DefaultEdition;
+import org.enso.jvm.channel.JVM;
 import org.enso.libraryupload.LibraryUploader.UploadFailedError;
-import org.enso.os.environment.jni.JVM;
 import org.enso.pkg.Contact;
 import org.enso.pkg.PackageManager;
 import org.enso.pkg.PackageManager$;
@@ -1574,6 +1574,15 @@ private void launch(String[] args) throws IOException, InterruptedException, URI
       }
     }
 
+    if (System.getProperty("java.home") == null) {
+      assert HostEnsoUtils.isAot() : "Otherwise java.home would be defined";
+      var exe = JavaFinder.findJavaExecutable();
+      if (exe != null) {
+        var path = exe.getParentFile().getParentFile().getAbsolutePath();
+        System.setProperty("java.home", path);
+        LOGGER.debug("Setting java.home property for AOT mode to {}", path);
+      }
+    }
     handleLaunch(originalCwdOrNull, line, logLevel, logMasking[0]);
   }
 

engine/runtime-compiler/src/test/java/org/enso/compiler/test/pass/PassPersistanceTest.java

@@ -79,11 +79,11 @@ private static <T> T serde(Class<T> clazz, T l, int expectedSize) throws IOExcep
 
   private static <T> T serde(Class<T> clazz, T l, int expectedSize, Function<Object, Object> fn)
       throws IOException {
-    var arr = POOL.write(l, fn);
+    var arr = POOL.withWriteReplace(fn).write(l);
     if (expectedSize >= 0) {
       assertEquals(expectedSize, arr.length - 12);
     }
-    var ref = POOL.read(arr, null);
+    var ref = POOL.read(arr);
     return ref.get(clazz);
   }
 }

engine/runtime-integration-tests/src/test/scala/org/enso/compiler/test/semantic/ImportExportTest.scala

@@ -1038,20 +1038,21 @@ class ImportExportTest
         .toList
         .collect({ case w: Warning.DuplicatedImport => w })
       warn.size shouldEqual 1
-      val arr = org.enso.interpreter.caches.PersistUtils.POOL.write(
-        mainIr,
-        {
-          case metadata: ProcessingPass.Metadata =>
-            metadata.prepareForSerialization(
-              ctx
-                .ensoContext()
-                .getCompiler
-                .context
-                .asInstanceOf[metadata.Compiler]
-            );
-          case obj => obj
-        }
-      );
+      val arr = org.enso.interpreter.caches.PersistUtils.POOL
+        .withWriteReplace(
+          {
+            case metadata: ProcessingPass.Metadata =>
+              metadata.prepareForSerialization(
+                ctx
+                  .ensoContext()
+                  .getCompiler
+                  .context
+                  .asInstanceOf[metadata.Compiler]
+              );
+            case obj => obj
+          }
+        )
+        .write(mainIr);
       arr should not be empty
     }
 
@@ -1079,20 +1080,21 @@ class ImportExportTest
         .asInstanceOf[errors.ImportExport.AmbiguousImport]
       ambiguousImport.symbolName shouldEqual "A_Type"
       try {
-        val arr = org.enso.interpreter.caches.PersistUtils.POOL.write(
-          mainIr,
-          {
-            case metadata: ProcessingPass.Metadata =>
-              metadata.prepareForSerialization(
-                ctx
-                  .ensoContext()
-                  .getCompiler
-                  .context
-                  .asInstanceOf[metadata.Compiler]
-              );
-            case obj => obj
-          }
-        );
+        val arr = org.enso.interpreter.caches.PersistUtils.POOL
+          .withWriteReplace(
+            {
+              case metadata: ProcessingPass.Metadata =>
+                metadata.prepareForSerialization(
+                  ctx
+                    .ensoContext()
+                    .getCompiler
+                    .context
+                    .asInstanceOf[metadata.Compiler]
+                );
+              case obj => obj
+            }
+          )
+          .write(mainIr);
         fail("Shouldn't return anything when there is an error" + arr)
       } catch {
         case ex: IOException =>