Skip to content

Commit 3646395

Browse files
nickvanickva
committed
Run test262 JS conformance tests for QuickJS
Test262 are 70k+ Javascript conformance tests [1]. There is an upstream PR [2] to run these in CI there but until then we can patch our own QuickJS and run them in our CI. One of the patches we have `04-test262-errors.patch` patches one of the exception we expect to get since modified the parser to accept the `function` expression syntax. For now run the test in the full CI only for Linux and skip s390x. Some architectures/os combinations show various discrepancies around TypedArrays or rounding errors. [1] https://github.com/tc39/test262 [2] bellard/quickjs#408
1 parent 53c3c13 commit 3646395

File tree

10 files changed

+102
-3
lines changed

10 files changed

+102
-3
lines changed

‎Makefile‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -354,6 +354,12 @@ weatherreport-test: devclean escriptize
354354
@dev/run "$(TEST_OPTS)" -n 1 -a adm:pass --no-eval \
355355
'bin/weatherreport --etc dev/lib/node1/etc --level error'
356356

357+
.PHONY: quickjs-test262
358+
# target: quickjs-javascript-tests - Run QuickJS JS conformance tests
359+
quickjs-test262: couch
360+
make -C src/couch_quickjs/quickjs test2-bootstrap
361+
make -C src/couch_quickjs/quickjs test2
362+
357363
################################################################################
358364
# Developing
359365
################################################################################

‎build-aux/Jenkinsfile.full‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ meta = [
2828
spidermonkey_vsn: '60',
2929
with_nouveau: true,
3030
with_clouseau: true,
31+
quickjs_test262: true,
3132
image: "apache/couchdbci-centos:8-erlang-${ERLANG_VERSION}"
3233
],
3334

@@ -36,6 +37,7 @@ meta = [
3637
spidermonkey_vsn: '78',
3738
with_nouveau: true,
3839
with_clouseau: true,
40+
quickjs_test262: true,
3941
image: "apache/couchdbci-centos:9-erlang-${ERLANG_VERSION}"
4042
],
4143

@@ -44,6 +46,7 @@ meta = [
4446
spidermonkey_vsn: '91',
4547
with_nouveau: true,
4648
with_clouseau: true,
49+
quickjs_test262: true,
4750
image: "apache/couchdbci-ubuntu:jammy-erlang-${ERLANG_VERSION}"
4851
],
4952

@@ -52,6 +55,7 @@ meta = [
5255
spidermonkey_vsn: '115',
5356
with_nouveau: true,
5457
with_clouseau: true,
58+
quickjs_test262: true,
5559
image: "apache/couchdbci-ubuntu:noble-erlang-${ERLANG_VERSION}"
5660
],
5761

@@ -60,6 +64,7 @@ meta = [
6064
spidermonkey_vsn: '78',
6165
with_nouveau: true,
6266
with_clouseau: true,
67+
quickjs_test262: true,
6368
image: "apache/couchdbci-debian:bookworm-erlang-${ERLANG_VERSION}",
6469
node_label: 'ppc64le'
6570
],
@@ -68,6 +73,9 @@ meta = [
6873
name: 'Debian s390x',
6974
spidermonkey_vsn: '78',
7075
with_nouveau: true,
76+
// QuickJS test262 shows a discrepancy typedarray-arg-set-values-same-buffer-other-type.js
77+
// Test262Error: 51539607552,42,0,4,5,6,7,8
78+
quickjs_test262: false,
7179
image: "apache/couchdbci-debian:bookworm-erlang-${ERLANG_VERSION}",
7280
node_label: 's390x'
7381
],
@@ -77,6 +85,7 @@ meta = [
7785
spidermonkey_vsn: '78',
7886
with_nouveau: true,
7987
with_clouseau: true,
88+
quickjs_test262: true,
8089
image: "apache/couchdbci-debian:bullseye-erlang-${ERLANG_VERSION}"
8190
],
8291

@@ -85,6 +94,8 @@ meta = [
8594
spidermonkey_vsn: '78',
8695
with_nouveau: true,
8796
with_clouseau: true,
97+
// Test this in in the bookworm-quickjs variant
98+
quickjs_test262: false,
8899
image: "apache/couchdbci-debian:bookworm-erlang-${ERLANG_VERSION}"
89100
],
90101

@@ -93,6 +104,7 @@ meta = [
93104
disable_spidermonkey: true,
94105
with_nouveau: true,
95106
with_clouseau: true,
107+
quickjs_test262: true,
96108
image: "apache/couchdbci-debian:bookworm-erlang-${ERLANG_VERSION}"
97109
],
98110

@@ -101,6 +113,7 @@ meta = [
101113
spidermonkey_vsn: '91',
102114
with_clouseau: true,
103115
clouseau_java_home: '/usr/local/openjdk8-jre',
116+
quickjs_test262: false,
104117
gnu_make: 'gmake'
105118
],
106119

@@ -111,6 +124,7 @@ meta = [
111124
disable_spidermonkey: true,
112125
with_clouseau: true,
113126
clouseau_java_home: '/usr/local/openjdk8-jre',
127+
quickjs_test262: false,
114128
gnu_make: 'gmake'
115129
],
116130

@@ -130,6 +144,7 @@ meta = [
130144
name: 'Windows 2022',
131145
spidermonkey_vsn: '128',
132146
with_clouseau: true,
147+
quickjs_test262: false,
133148
node_label: 'win'
134149
]
135150
]
@@ -187,6 +202,7 @@ def generateNativeStage(platform) {
187202
sh "${configure(meta[platform])}"
188203
sh '$MAKE'
189204
retry (3) {sh '$MAKE eunit'}
205+
if (meta[platform].quickjs_test262) {retry(3) {sh 'make quickjs-test262'}}
190206
retry (3) {sh '$MAKE elixir'}
191207
retry (3) {sh '$MAKE elixir-search'}
192208
retry (3) {sh '$MAKE mango-test'}
@@ -304,10 +320,12 @@ def generateContainerStage(platform) {
304320
unstash 'tarball'
305321
sh( script: "mkdir -p ${platform}/build", label: 'Create build directories' )
306322
sh( script: "tar -xf apache-couchdb-*.tar.gz -C ${platform}/build --strip-components=1", label: 'Unpack release' )
323+
quickjs_tests262 = meta[platform].quickjs_test262
307324
dir( "${platform}/build" ) {
308325
sh "${configure(meta[platform])}"
309326
sh 'make'
310327
retry(3) {sh 'make eunit'}
328+
if (meta[platform].quickjs_test262) {retry(3) {sh 'make quickjs-test262'}}
311329
retry(3) {sh 'make elixir'}
312330
retry(3) {sh 'make elixir-search'}
313331
retry(3) {sh 'make mango-test'}

‎src/couch_quickjs/.gitignore‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@
1616
/quickjs/qjscalc.c
1717
/quickjs/repl.c
1818
/quickjs/run-test262
19+
/quickjs/test262_report.txt
20+
/quickjs/test262/
1921
/quickjs/test_fib.c
2022
/quickjs/.github
2123
compile_commands.json

‎src/couch_quickjs/patches/02-test262-makefile.patch‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
--- quickjs-master/Makefile 2025-06-14 05:51:48
2+
+++ quickjs/Makefile 2025-06-20 18:03:41
3+
@@ -53,6 +53,10 @@
4+
#CONFIG_MSAN=y
5+
# use UB sanitizer
6+
#CONFIG_UBSAN=y
7+
+
8+
+# TEST262 bootstrap config: commit id and shallow "since" parameter
9+
+TEST262_COMMIT?=3316c0aaf676d657f5a6b33364fa7e579c78ac7f
10+
+TEST262_SINCE?=2025-05-21
11+
12+
OBJDIR=.obj
13+
14+
@@ -464,6 +468,15 @@
15+
microbench: qjs$(EXE)
16+
$(WINE) ./qjs$(EXE) --std tests/microbench.js
17+
18+
+ifeq ($(wildcard test262/features.txt),)
19+
+test2-bootstrap:
20+
+ git clone --single-branch --shallow-since=$(TEST262_SINCE) https://github.com/tc39/test262.git
21+
+ (cd test262 && git checkout -q $(TEST262_COMMIT) && patch -p1 < ../tests/test262.patch && cd ..)
22+
+else
23+
+test2-bootstrap:
24+
+ (cd test262 && git fetch && git reset --hard $(TEST262_COMMIT) && patch -p1 < ../tests/test262.patch && cd ..)
25+
+endif
26+
+
27+
ifeq ($(wildcard test262o/tests.txt),)
28+
test2o test2o-update:
29+
@echo test262o tests not installed

‎src/couch_quickjs/patches/03-test262-yield.patch‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
--- quickjs-master/tests/test262.patch 2025-06-14 05:51:48
2+
+++ quickjs/tests/test262.patch 2025-06-20 18:03:41
3+
@@ -14,9 +14,9 @@
4+
+// small: 200,
5+
+// long: 1000,
6+
+// huge: 10000,
7+
-+ yield: 20,
8+
-+ small: 20,
9+
-+ long: 100,
10+
++ yield: 40,
11+
++ small: 40,
12+
++ long: 200,
13+
+ huge: 1000,
14+
};
15+

‎src/couch_quickjs/patches/04-test262-errors.patch‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
--- quickjs-master/test262_errors.txt 2025-06-14 05:51:48
2+
+++ quickjs/test262_errors.txt 2025-06-20 18:03:41
3+
@@ -1,6 +1,8 @@
4+
test262/test/built-ins/Atomics/notify/retrieve-length-before-index-coercion-non-shared-detached.js:34: TypeError: ArrayBuffer is detached
5+
test262/test/built-ins/Atomics/notify/retrieve-length-before-index-coercion-non-shared-detached.js:34: strict mode: TypeError: ArrayBuffer is detached
6+
test262/test/language/module-code/top-level-await/module-graphs-does-not-hang.js:10: TypeError: $DONE() not called
7+
+test262/test/language/statements/expression/S12.4_A1.js:15: unexpected error type: Test262: This statement should not be evaluated.
8+
+test262/test/language/statements/expression/S12.4_A1.js:15: strict mode: unexpected error type: Test262: This statement should not be evaluated.
9+
test262/test/staging/sm/Date/UTC-convert-all-arguments.js:75: Test262Error: index 1: expected 42, got Error: didn't throw Expected SameValue(«Error: didn't throw», «42») to be true
10+
test262/test/staging/sm/Date/constructor-convert-all-arguments.js:75: Test262Error: index undefined: expected 42, got Error: didn't throw Expected SameValue(«Error: didn't throw», «42») to be true
11+
test262/test/staging/sm/Date/non-iso.js:76: Test262Error: Expected SameValue(«NaN», «-40071559730000») to be true

‎src/couch_quickjs/quickjs/Makefile‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,10 @@ PREFIX?=/usr/local
5454
# use UB sanitizer
5555
#CONFIG_UBSAN=y
5656

57+
# TEST262 bootstrap config: commit id and shallow "since" parameter
58+
TEST262_COMMIT?=3316c0aaf676d657f5a6b33364fa7e579c78ac7f
59+
TEST262_SINCE?=2025-05-21
60+
5761
OBJDIR=.obj
5862

5963
ifdef CONFIG_ASAN
@@ -464,6 +468,15 @@ stats: qjs$(EXE)
464468
microbench: qjs$(EXE)
465469
$(WINE) ./qjs$(EXE) --std tests/microbench.js
466470

471+
ifeq ($(wildcard test262/features.txt),)
472+
test2-bootstrap:
473+
git clone --single-branch --shallow-since=$(TEST262_SINCE) https://github.com/tc39/test262.git
474+
(cd test262 && git checkout -q $(TEST262_COMMIT) && patch -p1 < ../tests/test262.patch && cd ..)
475+
else
476+
test2-bootstrap:
477+
(cd test262 && git fetch && git reset --hard $(TEST262_COMMIT) && patch -p1 < ../tests/test262.patch && cd ..)
478+
endif
479+
467480
ifeq ($(wildcard test262o/tests.txt),)
468481
test2o test2o-update:
469482
@echo test262o tests not installed

‎src/couch_quickjs/quickjs/test262_errors.txt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
test262/test/built-ins/Atomics/notify/retrieve-length-before-index-coercion-non-shared-detached.js:34: TypeError: ArrayBuffer is detached
22
test262/test/built-ins/Atomics/notify/retrieve-length-before-index-coercion-non-shared-detached.js:34: strict mode: TypeError: ArrayBuffer is detached
33
test262/test/language/module-code/top-level-await/module-graphs-does-not-hang.js:10: TypeError: $DONE() not called
4+
test262/test/language/statements/expression/S12.4_A1.js:15: unexpected error type: Test262: This statement should not be evaluated.
5+
test262/test/language/statements/expression/S12.4_A1.js:15: strict mode: unexpected error type: Test262: This statement should not be evaluated.
46
test262/test/staging/sm/Date/UTC-convert-all-arguments.js:75: Test262Error: index 1: expected 42, got Error: didn't throw Expected SameValue(«Error: didn't throw», «42») to be true
57
test262/test/staging/sm/Date/constructor-convert-all-arguments.js:75: Test262Error: index undefined: expected 42, got Error: didn't throw Expected SameValue(«Error: didn't throw», «42») to be true
68
test262/test/staging/sm/Date/non-iso.js:76: Test262Error: Expected SameValue(«NaN», «-40071559730000») to be true

‎src/couch_quickjs/quickjs/tests/test262.patch‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,9 +14,9 @@ index 9828b15..4a5919d 100644
1414
+// small: 200,
1515
+// long: 1000,
1616
+// huge: 10000,
17-
+ yield: 20,
18-
+ small: 20,
19-
+ long: 100,
17+
+ yield: 40,
18+
+ small: 40,
19+
+ long: 200,
2020
+ huge: 1000,
2121
};
2222

‎src/couch_quickjs/update_patches.sh‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,9 @@ unzip -q -o master.zip
1717
echo " * updating 01-spidermonkey-185-mode.patch"
1818
set +e
1919
diff -u quickjs-master/quickjs.c quickjs/quickjs.c > patches/01-spidermonkey-185-mode.patch
20+
diff -u quickjs-master/Makefile quickjs/Makefile > patches/02-test262-makefile.patch
21+
diff -u quickjs-master/tests/test262.patch quickjs/tests/test262.patch > patches/03-test262-yield.patch
22+
diff -u quickjs-master/test262_errors.txt quickjs/test262_errors.txt > patches/04-test262-errors.patch
2023
set -e
2124
echo " * cleaning up"
2225
rm -rf master.zip quickjs-master

0 commit comments

Comments
 (0)