Use java.net.http.HttpClient instead of java.net.Http(s)URLConnection

Moving to java.net.http.HttpClient brings many benefits, including
HTTP/2 support and the ability to make asynchronous requests.

Major additions and changes:
- Introduce a lightweight org.pkl.core.http.HttpClient API.
  This keeps some flexibility and allows to enforce behavior
  such as setting the User-Agent header.
- Provide an implementation that delegates to java.net.http.HttpClient.
- Use HttpClient for all HTTP(s) requests across the codebase.
  This required adding an HttpClient parameter to constructors and
  factory methods of multiple classes, some of which are public APIs.
- Manage CA certificates per HTTP client instead of per JVM.
  This makes it unnecessary to set JVM-wide system/security properties
  and default SSLSocketFactory's.

Each HTTP client maintains its own connection pool and SSLContext.
For efficiency reasons, I've tried to reuse clients whenever feasible.
To avoid memory leaks, clients are not stored in static fields.

HTTP clients are expensive to create. For this reason,
EvaluatorBuilder defaults to a "lazy" client that creates the underlying
java.net.http.HttpClient on the first send (which may never happen).

Author: odenix

bench/src/jmh/java/org/pkl/core/ListSort.java

@@ -23,6 +23,7 @@
 import org.openjdk.jmh.annotations.*;
 import org.openjdk.jmh.util.TempFile;
 import org.openjdk.jmh.util.TempFileManager;
+import org.pkl.core.http.HttpClient;
 import org.pkl.core.module.ModuleKeyFactories;
 import org.pkl.core.repl.ReplRequest;
 import org.pkl.core.repl.ReplResponse;
@@ -39,6 +40,7 @@ public class ListSort {
   private static final ReplServer repl =
       new ReplServer(
           SecurityManagers.defaultManager,
+          HttpClient.dummyClient(),
           Loggers.stdErr(),
           List.of(ModuleKeyFactories.standardLibrary),
           List.of(ResourceReaders.file()),

docs/src/test/kotlin/DocSnippetTests.kt

@@ -25,6 +25,7 @@ import org.pkl.core.repl.ReplServer
 import org.pkl.core.resource.ResourceReaders
 import org.pkl.core.util.IoUtils
 import org.antlr.v4.runtime.ParserRuleContext
+import org.pkl.core.http.HttpClient
 import java.nio.file.Files
 import kotlin.io.path.isDirectory
 import kotlin.io.path.isRegularFile
@@ -78,6 +79,7 @@ class DocSnippetTestsEngine : HierarchicalTestEngine<DocSnippetTestsEngine.Execu
   override fun createExecutionContext(request: ExecutionRequest): ExecutionContext {
     val replServer = ReplServer(
       SecurityManagers.defaultManager,
+      HttpClient.dummyClient(),
       Loggers.stdErr(),
       listOf(
         ModuleKeyFactories.standardLibrary,

pkl-cli/pkl-cli.gradle.kts

@@ -156,7 +156,7 @@ fun Exec.configureExecutable(isEnabled: Boolean, outputFile: File, extraArgs: Li
         ,"--no-fallback"
         ,"-H:IncludeResources=org/pkl/core/stdlib/.*\\.pkl"
         ,"-H:IncludeResources=org/jline/utils/.*"
-        ,"-H:IncludeResources=org/pkl/commons/cli/commands/IncludedCARoots.pem"
+        ,"-H:IncludeResources=org/pkl/core/http/IncludedCARoots.pem"
         //,"-H:IncludeResources=org/pkl/core/Release.properties"
         ,"-H:IncludeResourceBundles=org.pkl.core.errorMessages"
         ,"--macro:truffle"

pkl-cli/src/main/kotlin/org/pkl/cli/CliDownloadPackageCommand.kt

@@ -31,7 +31,7 @@ class CliDownloadPackageCommand(
     if (moduleCacheDir == null) {
       throw CliException("Cannot download packages because no cache directory is specified.")
     }
-    val packageResolver = PackageResolver.getInstance(securityManager, moduleCacheDir)
+    val packageResolver = PackageResolver.getInstance(securityManager, httpClient, moduleCacheDir)
     val errors = mutableMapOf<PackageUri, Throwable>()
     for (pkg in packageUris) {
       try {

pkl-cli/src/main/kotlin/org/pkl/cli/CliProjectPackager.kt

@@ -82,6 +82,7 @@ class CliProjectPackager(
         outputPath,
         stackFrameTransformer,
         securityManager,
+        httpClient,
         skipPublishCheck,
         consoleWriter
       )

pkl-cli/src/main/kotlin/org/pkl/cli/CliProjectResolver.kt

@@ -40,6 +40,7 @@ class CliProjectResolver(
             SecurityManagers.defaultTrustLevels,
             rootDir
           ),
+          httpClient,
           moduleCacheDir
         )
       val dependencies = ProjectDependenciesResolver(project, packageResolver, errWriter).resolve()

pkl-cli/src/main/kotlin/org/pkl/cli/CliRepl.kt

@@ -36,6 +36,7 @@ internal class CliRepl(private val options: CliEvaluatorOptions) : CliCommand(op
             SecurityManagers.defaultTrustLevels,
             rootDir
           ),
+          httpClient,
           Loggers.stdErr(),
           listOf(
             ModuleKeyFactories.standardLibrary,

pkl-cli/src/main/kotlin/org/pkl/cli/CliServer.kt

@@ -25,7 +25,7 @@ import org.pkl.server.Server
 class CliServer(options: CliBaseOptions) : CliCommand(options) {
   override fun doRun() =
     try {
-      val server = Server(MessageTransports.stream(System.`in`, System.out))
+      val server = Server(MessageTransports.stream(System.`in`, System.out), httpClient)
       server.use { it.start() }
     } catch (e: ProtocolException) {
       throw CliException(e.message!!)

pkl-cli/src/test/kotlin/org/pkl/cli/CliDownloadPackageCommandTest.kt

@@ -202,7 +202,7 @@ class CliDownloadPackageCommandTest {
 
         Failed to download package://bogus.domain/[email protected] because:
         Exception when making request `GET https://bogus.domain/[email protected]`:
-        bogus.domain
+        Error connecting to host `bogus.domain`.
 
       """
           .trimIndent()

pkl-cli/src/test/kotlin/org/pkl/cli/CliEvaluatorTest.kt

@@ -33,7 +33,6 @@ import org.junit.jupiter.params.provider.EnumSource
 import org.pkl.commons.*
 import org.pkl.commons.cli.CliBaseOptions
 import org.pkl.commons.cli.CliException
-import org.pkl.commons.cli.commands.BaseOptions
 import org.pkl.commons.test.FileTestUtils
 import org.pkl.commons.test.PackageServer
 import org.pkl.core.OutputFormat
@@ -1177,9 +1176,7 @@ result = someLib.x
           sourceModules = listOf(moduleUri),
           workingDir = tempDir,
           moduleCacheDir = tempDir,
-          noCache = true,
-          // ensure we override any previously set root cert to the default buundle.
-          caCertificates = listOf(BaseOptions.Companion.includedCARootCerts())
+          noCache = true
         ),
       )
     val err = assertThrows<CliException> { CliEvaluator(options, consoleWriter = buffer).run() }