Skip to content

Commit d81a123

Browse files
bioballbioball
authored
Improve configuration and tests for native-image (#509)
* Don't expose JDK internal classes; instead solve msgpack issue with `--initialize-at-run-time`. * Use quick build mode for non-release builds: 40% faster compilation, 20% smaller executable. * Remove options that were commented out. * Also run ServerTest against native executable
1 parent a48748c commit d81a123

File tree

11 files changed

+231
-169
lines changed

11 files changed

+231
-169
lines changed

‎.circleci/config.yml‎

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ jobs:
1212
- run:
1313
command: |-
1414
export PATH=~/staticdeps/bin:$PATH
15-
./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results -DreleaseBuild=true pkl-cli:macExecutableAmd64 pkl-core:testMacExecutableAmd64
15+
./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results -DreleaseBuild=true pkl-cli:macExecutableAmd64 pkl-core:testMacExecutableAmd64 pkl-server:testNative
1616
name: gradle buildNative
1717
- persist_to_workspace:
1818
root: '.'
@@ -88,7 +88,7 @@ jobs:
8888
- run:
8989
command: |-
9090
export PATH=~/staticdeps/bin:$PATH
91-
./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results -DreleaseBuild=true pkl-cli:linuxExecutableAmd64 pkl-core:testLinuxExecutableAmd64
91+
./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results -DreleaseBuild=true pkl-cli:linuxExecutableAmd64 pkl-core:testLinuxExecutableAmd64 pkl-server:testNative
9292
name: gradle buildNative
9393
- persist_to_workspace:
9494
root: '.'
@@ -108,7 +108,7 @@ jobs:
108108
- run:
109109
command: |-
110110
export PATH=~/staticdeps/bin:$PATH
111-
./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results -DreleaseBuild=true pkl-cli:macExecutableAarch64 pkl-core:testMacExecutableAarch64
111+
./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results -DreleaseBuild=true pkl-cli:macExecutableAarch64 pkl-core:testMacExecutableAarch64 pkl-server:testNative
112112
name: gradle buildNative
113113
- persist_to_workspace:
114114
root: '.'
@@ -168,7 +168,7 @@ jobs:
168168
- run:
169169
command: |-
170170
export PATH=~/staticdeps/bin:$PATH
171-
./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results -DreleaseBuild=true pkl-cli:linuxExecutableAarch64 pkl-core:testLinuxExecutableAarch64
171+
./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results -DreleaseBuild=true pkl-cli:linuxExecutableAarch64 pkl-core:testLinuxExecutableAarch64 pkl-server:testNative
172172
name: gradle buildNative
173173
- persist_to_workspace:
174174
root: '.'
@@ -245,7 +245,7 @@ jobs:
245245
- run:
246246
command: |-
247247
export PATH=~/staticdeps/bin:$PATH
248-
./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results -DreleaseBuild=true pkl-cli:alpineExecutableAmd64 pkl-core:testAlpineExecutableAmd64
248+
./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results -DreleaseBuild=true pkl-cli:alpineExecutableAmd64 pkl-core:testAlpineExecutableAmd64 pkl-server:testNative
249249
name: gradle buildNative
250250
- persist_to_workspace:
251251
root: '.'
@@ -265,7 +265,7 @@ jobs:
265265
- run:
266266
command: |-
267267
export PATH=~/staticdeps/bin:$PATH
268-
./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results -DreleaseBuild=true pkl-cli:windowsExecutableAmd64 pkl-core:testWindowsExecutableAmd64
268+
./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results -DreleaseBuild=true pkl-cli:windowsExecutableAmd64 pkl-core:testWindowsExecutableAmd64 pkl-server:testNative
269269
name: gradle buildNative
270270
shell: bash.exe
271271
- persist_to_workspace:
@@ -288,7 +288,7 @@ jobs:
288288
- run:
289289
command: |-
290290
export PATH=~/staticdeps/bin:$PATH
291-
./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results pkl-cli:macExecutableAmd64 pkl-core:testMacExecutableAmd64
291+
./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results pkl-cli:macExecutableAmd64 pkl-core:testMacExecutableAmd64 pkl-server:testNative
292292
name: gradle buildNative
293293
- persist_to_workspace:
294294
root: '.'
@@ -364,7 +364,7 @@ jobs:
364364
- run:
365365
command: |-
366366
export PATH=~/staticdeps/bin:$PATH
367-
./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results pkl-cli:linuxExecutableAmd64 pkl-core:testLinuxExecutableAmd64
367+
./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results pkl-cli:linuxExecutableAmd64 pkl-core:testLinuxExecutableAmd64 pkl-server:testNative
368368
name: gradle buildNative
369369
- persist_to_workspace:
370370
root: '.'
@@ -384,7 +384,7 @@ jobs:
384384
- run:
385385
command: |-
386386
export PATH=~/staticdeps/bin:$PATH
387-
./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results pkl-cli:macExecutableAarch64 pkl-core:testMacExecutableAarch64
387+
./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results pkl-cli:macExecutableAarch64 pkl-core:testMacExecutableAarch64 pkl-server:testNative
388388
name: gradle buildNative
389389
- persist_to_workspace:
390390
root: '.'
@@ -444,7 +444,7 @@ jobs:
444444
- run:
445445
command: |-
446446
export PATH=~/staticdeps/bin:$PATH
447-
./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results pkl-cli:linuxExecutableAarch64 pkl-core:testLinuxExecutableAarch64
447+
./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results pkl-cli:linuxExecutableAarch64 pkl-core:testLinuxExecutableAarch64 pkl-server:testNative
448448
name: gradle buildNative
449449
- persist_to_workspace:
450450
root: '.'
@@ -521,7 +521,7 @@ jobs:
521521
- run:
522522
command: |-
523523
export PATH=~/staticdeps/bin:$PATH
524-
./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results pkl-cli:alpineExecutableAmd64 pkl-core:testAlpineExecutableAmd64
524+
./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results pkl-cli:alpineExecutableAmd64 pkl-core:testAlpineExecutableAmd64 pkl-server:testNative
525525
name: gradle buildNative
526526
- persist_to_workspace:
527527
root: '.'
@@ -541,7 +541,7 @@ jobs:
541541
- run:
542542
command: |-
543543
export PATH=~/staticdeps/bin:$PATH
544-
./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results pkl-cli:windowsExecutableAmd64 pkl-core:testWindowsExecutableAmd64
544+
./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results pkl-cli:windowsExecutableAmd64 pkl-core:testWindowsExecutableAmd64 pkl-server:testNative
545545
name: gradle buildNative
546546
shell: bash.exe
547547
- persist_to_workspace:

‎.circleci/jobs/BuildNativeJob.pkl‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -126,7 +126,7 @@ steps {
126126
}
127127
command = #"""
128128
export PATH=~/staticdeps/bin:$PATH
129-
./gradlew \#(module.gradleArgs) pkl-cli:\#(jobName) pkl-core:test\#(jobName.capitalize())
129+
./gradlew \#(module.gradleArgs) pkl-cli:\#(jobName) pkl-core:test\#(jobName.capitalize()) pkl-server:testNative
130130
"""#
131131
}
132132
new Config.PersistToWorkspaceStep {

‎pkl-cli/pkl-cli.gradle.kts‎

Lines changed: 15 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -146,10 +146,17 @@ fun Exec.configureExecutable(
146146
outputFile: Provider<RegularFile>,
147147
extraArgs: List<String> = listOf()
148148
) {
149-
inputs.files(sourceSets.main.map { it.output }).withPropertyName("mainSourceSets").withPathSensitivity(PathSensitivity.RELATIVE)
150-
inputs.files(configurations.runtimeClasspath).withPropertyName("runtimeClasspath").withNormalizer(ClasspathNormalizer::class)
149+
inputs.files(sourceSets.main.map { it.output })
150+
.withPropertyName("mainSourceSets")
151+
.withPathSensitivity(PathSensitivity.RELATIVE)
152+
inputs.files(configurations.runtimeClasspath)
153+
.withPropertyName("runtimeClasspath")
154+
.withNormalizer(ClasspathNormalizer::class)
151155
val nativeImageCommandName = if (buildInfo.os.isWindows) "native-image.cmd" else "native-image"
152-
inputs.files(file(graalVm.baseDir).resolve("bin/$nativeImageCommandName")).withPropertyName("graalVmNativeImage").withPathSensitivity(PathSensitivity.ABSOLUTE)
156+
inputs.files(file(graalVm.baseDir)
157+
.resolve("bin/$nativeImageCommandName"))
158+
.withPropertyName("graalVmNativeImage")
159+
.withPathSensitivity(PathSensitivity.ABSOLUTE)
153160
outputs.file(outputFile)
154161
outputs.cacheIf { true }
155162

@@ -165,47 +172,24 @@ fun Exec.configureExecutable(
165172
// that the "initialize everything at build time" *CLI* option is likely here to stay
166173
"--initialize-at-build-time="
167174
// needed for messagepack-java (see https://github.com/msgpack/msgpack-java/issues/600)
168-
,"--add-opens=java.base/java.nio=ALL-UNNAMED"
169-
,"--add-opens=java.base/sun.nio.ch=ALL-UNNAMED"
175+
,"--initialize-at-run-time=org.msgpack.core.buffer.DirectBufferAccess"
170176
,"--no-fallback"
171177
,"-H:IncludeResources=org/pkl/core/stdlib/.*\\.pkl"
172178
,"-H:IncludeResources=org/jline/utils/.*"
173179
,"-H:IncludeResources=org/pkl/certs/PklCARoots.pem"
174-
//,"-H:IncludeResources=org/pkl/core/Release.properties"
175180
,"-H:IncludeResourceBundles=org.pkl.core.errorMessages"
176181
,"--macro:truffle"
177182
,"-H:Class=org.pkl.cli.Main"
178183
,"-H:Name=${outputFile.get().asFile.name}"
179-
//,"--native-image-info"
180-
//,"-Dpolyglot.image-build-time.PreinitializeContexts=pkl"
181184
// the actual limit (currently) used by native-image is this number + 1400 (idea is to compensate for Truffle's own nodes)
182185
,"-H:MaxRuntimeCompileMethods=1800"
183186
,"-H:+EnforceMaxRuntimeCompileMethods"
184187
,"--enable-url-protocols=http,https"
185-
//,"--install-exit-handlers"
186188
,"-H:+ReportExceptionStackTraces"
187-
,"-H:-ParseRuntimeOptions" // disable automatic support for JVM CLI options (puts our main class in full control of argument parsing)
188-
//,"-H:+PrintAnalysisCallTree"
189-
//,"-H:PrintAnalysisCallTreeType=CSV"
190-
//,"-H:+PrintImageObjectTree"
191-
//,"--features=org.pkl.cli.svm.InitFeature"
192-
//,"-H:Dump=:2"
193-
//,"-H:MethodFilter=ModuleCache.getOrLoad*,VmLanguage.loadModule"
194-
//,"-g"
195-
//,"-verbose"
196-
//,"--debug-attach"
197-
//,"-H:+AllowVMInspection"
198-
//,"-H:+PrintHeapHistogram"
199-
//,"-H:+ReportDeletedElementsAtRuntime"
200-
//,"-H:+PrintMethodHistogram"
201-
//,"-H:+PrintRuntimeCompileMethods"
202-
//,"-H:NumberOfThreads=1"
203-
//,"-J-Dtruffle.TruffleRuntime=com.oracle.truffle.api.impl.DefaultTruffleRuntime"
204-
//,"-J-Dcom.oracle.truffle.aot=true"
205-
//,"-J:-ea"
206-
//,"-J:-esa"
207-
// for use with https://www.graalvm.org/docs/tools/dashboard/
208-
//,"-H:DashboardDump=dashboard.dump", "-H:+DashboardAll"
189+
// disable automatic support for JVM CLI options (puts our main class in full control of argument parsing)
190+
,"-H:-ParseRuntimeOptions"
191+
// quick build mode: 40% faster compilation, 20% smaller (but presumably also slower) executable
192+
,if (!buildInfo.isReleaseBuild) "-Ob" else ""
209193
// native-image rejects non-existing class path entries -> filter
210194
,"--class-path"
211195
,((sourceSets.main.get().output + configurations.runtimeClasspath.get())

‎pkl-commons-test/src/main/kotlin/org/pkl/commons/test/PklExecutablePaths.kt‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
/**
2+
* Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* https://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
package org.pkl.commons.test
17+
18+
import java.nio.file.Files
19+
import java.nio.file.Path
20+
import org.pkl.commons.test.FileTestUtils.rootProjectDir
21+
22+
object PklExecutablePaths {
23+
val macAarch64: Path = executablePath("pkl-macos-aarch64")
24+
val macAmd64: Path = executablePath("pkl-macos-amd64")
25+
val linuxAarch64: Path = executablePath("pkl-linux-aarch64")
26+
val linuxAmd64: Path = executablePath("pkl-linux-amd64")
27+
val alpineAmd64: Path = executablePath("pkl-alpine-linux-amd64")
28+
val windowsAmd64: Path = executablePath("pkl-windows-amd64.exe")
29+
30+
// order (aarch64 before amd64, linux before alpine) affects [firstExisting]
31+
val all: List<Path> =
32+
listOf(macAarch64, macAmd64, linuxAarch64, linuxAmd64, alpineAmd64, windowsAmd64)
33+
34+
val existing: List<Path>
35+
get() = all.filter(Files::exists)
36+
37+
val firstExisting: Path
38+
get() =
39+
existing.firstOrNull()
40+
?: throw AssertionError(
41+
"Native executable not found on system. " +
42+
"To fix this problem, run `./gradlew assembleNative`."
43+
)
44+
45+
private fun executablePath(name: String): Path =
46+
rootProjectDir.resolve("pkl-cli/build/executable").resolve(name)
47+
}

‎pkl-core/pkl-core.gradle.kts‎

Lines changed: 29 additions & 79 deletions
Original file line numberDiff line numberDiff line change
@@ -154,10 +154,7 @@ tasks.compileKotlin {
154154
}
155155

156156
tasks.test {
157-
inputs.dir("src/test/files/LanguageSnippetTests/input").withPropertyName("languageSnippetTestsInput").withPathSensitivity(PathSensitivity.RELATIVE)
158-
inputs.dir("src/test/files/LanguageSnippetTests/input-helper").withPropertyName("languageSnippetTestsInputHelper").withPathSensitivity(PathSensitivity.RELATIVE)
159-
inputs.dir("src/test/files/LanguageSnippetTests/output").withPropertyName("languageSnippetTestsOutput").withPathSensitivity(PathSensitivity.RELATIVE)
160-
157+
configureTest()
161158
useJUnitPlatform {
162159
excludeEngines("MacAmd64LanguageSnippetTestsEngine")
163160
excludeEngines("MacAarch64LanguageSnippetTestsEngine")
@@ -168,11 +165,7 @@ tasks.test {
168165
}
169166

170167
val testJavaExecutable by tasks.registering(Test::class) {
171-
inputs.dir("src/test/files/LanguageSnippetTests/input").withPropertyName("languageSnippetTestsInput").withPathSensitivity(PathSensitivity.RELATIVE)
172-
inputs.dir("src/test/files/LanguageSnippetTests/input-helper").withPropertyName("languageSnippetTestsInputHelper").withPathSensitivity(PathSensitivity.RELATIVE)
173-
inputs.dir("src/test/files/LanguageSnippetTests/output").withPropertyName("languageSnippetTestsOutput").withPathSensitivity(PathSensitivity.RELATIVE)
174-
175-
testClassesDirs = files(tasks.test.get().testClassesDirs)
168+
configureExecutableTest("LanguageSnippetTestsEngine")
176169
classpath =
177170
// compiled test classes
178171
sourceSets.test.get().output +
@@ -182,10 +175,6 @@ val testJavaExecutable by tasks.registering(Test::class) {
182175
// (test dependencies that are also main dependencies must already be contained in java executable;
183176
// to verify that we don't want to include them here)
184177
(configurations.testRuntimeClasspath.get() - configurations.runtimeClasspath.get())
185-
186-
useJUnitPlatform {
187-
includeEngines("LanguageSnippetTestsEngine")
188-
}
189178
}
190179

191180
tasks.check {
@@ -194,92 +183,32 @@ tasks.check {
194183

195184
val testMacExecutableAmd64 by tasks.registering(Test::class) {
196185
dependsOn(":pkl-cli:macExecutableAmd64")
197-
198-
inputs.dir("src/test/files/LanguageSnippetTests/input").withPropertyName("languageSnippetTestsInput").withPathSensitivity(PathSensitivity.RELATIVE)
199-
inputs.dir("src/test/files/LanguageSnippetTests/input-helper").withPropertyName("languageSnippetTestsInputHelper").withPathSensitivity(PathSensitivity.RELATIVE)
200-
inputs.dir("src/test/files/LanguageSnippetTests/output").withPropertyName("languageSnippetTestsOutput").withPathSensitivity(PathSensitivity.RELATIVE)
201-
202-
testClassesDirs = files(tasks.test.get().testClassesDirs)
203-
classpath = tasks.test.get().classpath
204-
205-
useJUnitPlatform {
206-
includeEngines("MacAmd64LanguageSnippetTestsEngine")
207-
}
186+
configureExecutableTest("MacAmd64LanguageSnippetTestsEngine")
208187
}
209188

210189
val testMacExecutableAarch64 by tasks.registering(Test::class) {
211190
dependsOn(":pkl-cli:macExecutableAarch64")
212-
213-
inputs.dir("src/test/files/LanguageSnippetTests/input")
214-
inputs.dir("src/test/files/LanguageSnippetTests/input-helper")
215-
inputs.dir("src/test/files/LanguageSnippetTests/output")
216-
217-
testClassesDirs = files(tasks.test.get().testClassesDirs)
218-
classpath = tasks.test.get().classpath
219-
220-
useJUnitPlatform {
221-
includeEngines("MacAarch64LanguageSnippetTestsEngine")
222-
}
191+
configureExecutableTest("MacAarch64LanguageSnippetTestsEngine")
223192
}
224193

225194
val testLinuxExecutableAmd64 by tasks.registering(Test::class) {
226195
dependsOn(":pkl-cli:linuxExecutableAmd64")
227-
228-
inputs.dir("src/test/files/LanguageSnippetTests/input")
229-
inputs.dir("src/test/files/LanguageSnippetTests/input-helper")
230-
inputs.dir("src/test/files/LanguageSnippetTests/output")
231-
232-
testClassesDirs = files(tasks.test.get().testClassesDirs)
233-
classpath = tasks.test.get().classpath
234-
235-
useJUnitPlatform {
236-
includeEngines("LinuxAmd64LanguageSnippetTestsEngine")
237-
}
196+
configureExecutableTest("LinuxAmd64LanguageSnippetTestsEngine")
238197
}
239198

240199
val testLinuxExecutableAarch64 by tasks.registering(Test::class) {
241200
dependsOn(":pkl-cli:linuxExecutableAarch64")
242-
243-
inputs.dir("src/test/files/LanguageSnippetTests/input")
244-
inputs.dir("src/test/files/LanguageSnippetTests/input-helper")
245-
inputs.dir("src/test/files/LanguageSnippetTests/output")
246-
247-
testClassesDirs = files(tasks.test.get().testClassesDirs)
248-
classpath = tasks.test.get().classpath
249-
250-
useJUnitPlatform {
251-
includeEngines("LinuxAarch64LanguageSnippetTestsEngine")
252-
}
201+
configureExecutableTest("LinuxAarch64LanguageSnippetTestsEngine")
253202
}
254203

255204
val testAlpineExecutableAmd64 by tasks.registering(Test::class) {
256205
dependsOn(":pkl-cli:alpineExecutableAmd64")
257-
258-
inputs.dir("src/test/files/LanguageSnippetTests/input")
259-
inputs.dir("src/test/files/LanguageSnippetTests/input-helper")
260-
inputs.dir("src/test/files/LanguageSnippetTests/output")
261-
262-
testClassesDirs = files(tasks.test.get().testClassesDirs)
263-
classpath = tasks.test.get().classpath
264-
265-
useJUnitPlatform {
266-
includeEngines("AlpineLanguageSnippetTestsEngine")
267-
}
206+
configureExecutableTest("AlpineLanguageSnippetTestsEngine")
268207
}
269208

270209
val testWindowsExecutableAmd64 by tasks.registering(Test::class) {
271210
dependsOn(":pkl-cli:windowsExecutableAmd64")
272-
273-
inputs.dir("src/test/files/LanguageSnippetTests/input")
274-
inputs.dir("src/test/files/LanguageSnippetTests/input-helper")
275-
inputs.dir("src/test/files/LanguageSnippetTests/output")
276-
277-
testClassesDirs = files(tasks.test.get().testClassesDirs)
278-
classpath = tasks.test.get().classpath
279-
280-
useJUnitPlatform {
281-
includeEngines("WindowsLanguageSnippetTestsEngine")
282-
}
211+
configureExecutableTest("WindowsLanguageSnippetTestsEngine")
283212
}
284213

285214
tasks.testNative {
@@ -316,3 +245,24 @@ spotless {
316245
target(files("src/main/antlr/PklParser.g4", "src/main/antlr/PklLexer.g4"))
317246
}
318247
}
248+
249+
private fun Test.configureTest() {
250+
inputs.dir("src/test/files/LanguageSnippetTests/input")
251+
.withPropertyName("languageSnippetTestsInput")
252+
.withPathSensitivity(PathSensitivity.RELATIVE)
253+
inputs.dir("src/test/files/LanguageSnippetTests/input-helper")
254+
.withPropertyName("languageSnippetTestsInputHelper")
255+
.withPathSensitivity(PathSensitivity.RELATIVE)
256+
inputs.dir("src/test/files/LanguageSnippetTests/output")
257+
.withPropertyName("languageSnippetTestsOutput")
258+
.withPathSensitivity(PathSensitivity.RELATIVE)
259+
}
260+
261+
private fun Test.configureExecutableTest(engineName: String) {
262+
configureTest()
263+
testClassesDirs = files(tasks.test.get().testClassesDirs)
264+
classpath = tasks.test.get().classpath
265+
useJUnitPlatform {
266+
includeEngines(engineName)
267+
}
268+
}

0 commit comments

Comments
 (0)