Run test262 tests in CI

Run test262 tests for Linux, macOS and Cosmopolitan

Add a Makefile `test2-bootstrap` helper to clone and patch test262 tests at
a particular commit. Running with the latest commit noticed a few tests were
failing and added them to the errors list.

The CI environment, especially for macOS, was a bit more unpredictable and was
failing in atomics tests due to lower timeouts in the `atomicsHelper` so double
the timeouts. The time didn't worsen too much. It still takes about 2-3 min.

Author: nickva

.github/workflows/ci.yml

@@ -21,7 +21,7 @@ jobs:
     strategy:
       fail-fast: false
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
         with:
           submodules: true
       - name: Build
@@ -36,14 +36,18 @@ jobs:
       - name: Run microbench
         run: |
           make microbench
+      - name: Run test262
+        run: |
+          make test2-bootstrap
+          make test2
 
   linux-lto:
     name: Linux LTO
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
         with:
           submodules: true
       - name: Build
@@ -62,11 +66,12 @@ jobs:
     strategy:
       fail-fast: false
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
         with:
           submodules: true
       - name: Install gcc-multilib
         run: |
+          sudo apt update
           sudo apt install -y gcc-multilib
       - name: Build
         run: |
@@ -78,7 +83,7 @@ jobs:
   linux-asan:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
         with:
           submodules: true
       - name: Build
@@ -93,7 +98,7 @@ jobs:
   linux-msan:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
         with:
           submodules: true
       - name: Build
@@ -110,7 +115,7 @@ jobs:
   linux-ubsan:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
         with:
           submodules: true
       - name: Build
@@ -128,7 +133,7 @@ jobs:
     strategy:
       fail-fast: false
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
       - name: Build
         run: |
           make -j$(getconf _NPROCESSORS_ONLN) CONFIG_WERROR=y
@@ -138,11 +143,15 @@ jobs:
       - name: Run built-in tests
         run: |
           make test
+      - name: Run test262
+        run: |
+          make test2-bootstrap
+          make test2
 
   macos-asan:
     runs-on: macos-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
       - name: Build
         run: |
           make -j$(getconf _NPROCESSORS_ONLN) CONFIG_WERROR=y CONFIG_ASAN=y
@@ -155,7 +164,7 @@ jobs:
   macos-ubsan:
     runs-on: macos-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
       - name: Build
         run: |
           make -j$(getconf _NPROCESSORS_ONLN) CONFIG_WERROR=y CONFIG_UBSAN=y
@@ -168,11 +177,16 @@ jobs:
   freebsd:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
       - name: Build + test
         uses: vmactions/freebsd-vm@v1
         with:
+          # This VM has a tendency to get stuck in the "VM is booting" cycle
+          # for quite a while, so set a shorter timeout
+          timeout-minutes: 15
           usesh: true
+          copyback: false
+          mem: 16384
           prepare: |
             pkg install -y gmake
           run: |
@@ -186,7 +200,7 @@ jobs:
     strategy:
       fail-fast: false
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
         with:
           submodules: true
       - name: Install Cosmopolitan
@@ -202,14 +216,18 @@ jobs:
       - name: Run built-in tests
         run: |
           make CONFIG_COSMO=y test
+      - name: Run test262
+        run: |
+          make test2-bootstrap
+          make CONFIG_COSMO=y test2
 
   mingw-windows:
     name: MinGW Windows target
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
         with:
           submodules: true
       - name: Install MinGW and Wine
@@ -240,7 +258,7 @@ jobs:
       run:
         shell: msys2 {0}
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
       - uses: msys2/setup-msys2@v2
         with:
           msystem: UCRT64
@@ -276,7 +294,7 @@ jobs:
           - linux/ppc64le
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
         with:
             submodules: recursive
       - name: Get qemu

Makefile

@@ -54,6 +54,10 @@ PREFIX?=/usr/local
 # use UB sanitizer
 #CONFIG_UBSAN=y
 
+# TEST262 bootstrap config: commit id and shallow "since" parameter
+TEST262_COMMIT?=af3d908437b0912513a594e7167f17658e72d88b
+TEST262_SINCE?=2025-09-14
+
 OBJDIR=.obj
 
 ifdef CONFIG_ASAN
@@ -464,6 +468,15 @@ stats: qjs$(EXE)
 microbench: qjs$(EXE)
 	$(WINE) ./qjs$(EXE) --std tests/microbench.js
 
+ifeq ($(wildcard test262/features.txt),)
+test2-bootstrap:
+	git clone --single-branch --shallow-since=$(TEST262_SINCE) https://github.com/tc39/test262.git
+	(cd test262 && git checkout -q $(TEST262_COMMIT) && patch -p1 < ../tests/test262.patch && cd ..)
+else
+test2-bootstrap:
+	(cd test262 && git fetch && git reset --hard $(TEST262_COMMIT) && patch -p1 < ../tests/test262.patch && cd ..)
+endif
+
 ifeq ($(wildcard test262o/tests.txt),)
 test2o test2o-update:
 	@echo test262o tests not installed

test262.conf

@@ -116,6 +116,7 @@ for-of
 generators
 globalThis
 hashbang
+immutable-arraybuffer=skip
 import-attributes
 import-defer=skip
 import.meta

tests/test262.patch

@@ -14,9 +14,9 @@ index 9828b15..4a5919d 100644
 +//  small: 200,
 +//  long: 1000,
 +//  huge: 10000,
-+  yield: 20,
-+  small: 20,
-+  long: 100,
++  yield: 40,
++  small: 40,
++  long: 200,
 +  huge: 1000,
  };