feat!: add Wasm support

Author: ObserverOfTime

build.zig

@@ -1,28 +1,47 @@
 const std = @import("std");
+const ts = @import("tree_sitter");
+
+const wasm_url = "https://github.com/tree-sitter/tree-sitter-c/releases/download/v0.24.1/tree-sitter-c.wasm";
 
 pub fn build(b: *std.Build) !void {
     const target = b.standardTargetOptions(.{});
     const optimize = b.standardOptimizeOption(.{});
 
-    const core = b.dependency("tree_sitter", .{
+    const options = b.addOptions();
+    const enable_wasm = b.option(bool, "enable-wasm", "Enable Wasm support") orelse false;
+    options.addOption(bool, "enable_wasm", enable_wasm);
+
+    const core = b.dependencyFromBuildZig(ts, .{
         .target = target,
         .optimize = optimize,
+        .amalgamated = true,
+        .@"build-shared" = false,
+        .@"enable-wasm" = enable_wasm,
     });
     const core_lib = core.artifact("tree-sitter");
+    const wasmtime = if (enable_wasm) core.builder.lazyDependency(ts.wasmtimeDep(target.result), .{}) else null;
 
     const module = b.addModule("tree_sitter", .{
         .root_source_file = b.path("src/root.zig"),
         .target = target,
         .optimize = optimize,
     });
     module.linkLibrary(core_lib);
+    module.addOptions("build", options);
+    if (wasmtime) |dep| {
+        module.addLibraryPath(dep.path("lib"));
+        module.linkSystemLibrary("wasmtime", .{
+            .preferred_link_mode = .static,
+        });
+    }
 
     const docs = b.addObject(.{
         .name = "tree_sitter",
         .root_source_file = b.path("src/root.zig"),
         .target = target,
         .optimize = .Debug,
     });
+    docs.root_module.addOptions("build", options);
 
     const install_docs = b.addInstallDirectory(.{
         .source_dir = docs.getEmittedDocs(),
@@ -39,9 +58,31 @@ pub fn build(b: *std.Build) !void {
         .optimize = optimize,
     });
     tests.linkLibrary(core_lib);
+    tests.root_module.addOptions("build", options);
+    if (wasmtime) |dep| {
+        tests.root_module.addLibraryPath(dep.path("lib"));
+        tests.root_module.linkSystemLibrary("wasmtime", .{
+            .preferred_link_mode = .static,
+        });
+        tests.root_module.linkSystemLibrary("unwind", .{});
+        if (target.result.os.tag == .windows) {
+            if (target.result.abi != .msvc) {
+                tests.root_module.linkSystemLibrary("unwind", .{});
+                tests.root_module.linkSystemLibrary("advapi32", .{});
+                tests.root_module.linkSystemLibrary("bcrypt", .{});
+                tests.root_module.linkSystemLibrary("ntdll", .{});
+                tests.root_module.linkSystemLibrary("ole32", .{});
+                tests.root_module.linkSystemLibrary("shell32", .{});
+                tests.root_module.linkSystemLibrary("userenv", .{});
+                tests.root_module.linkSystemLibrary("ws2_32", .{});
+            } else {
+                const fail = b.addFail("FIXME: cannot build with enable-wasm for MSVC");
+                tests.step.dependOn(&fail.step);
+            }
+        }
+    }
 
     const run_tests = b.addRunArtifact(tests);
-
     const test_step = b.step("test", "Run unit tests");
     test_step.dependOn(&run_tests.step);
 
@@ -55,6 +96,19 @@ pub fn build(b: *std.Build) !void {
                 .optimize = optimize,
             }) orelse continue;
             tests.linkLibrary(dep.artifact("tree-sitter-c"));
+
+            if (enable_wasm) {
+                // FIXME: prevent the file from being downloaded multiple times
+                std.log.info("Downloading {s}\n", .{ wasm_url });
+                const run_curl = b.addSystemCommand(&.{"curl", "-LSsf", wasm_url, "-o"});
+                const wasm_file = run_curl.addOutputFileArg("tree-sitter-c.wasm");
+                run_curl.expectStdErrEqual("");
+                tests.step.dependOn(&run_curl.step);
+                tests.root_module.addAnonymousImport("tree-sitter-c.wasm", .{
+                    .root_source_file = wasm_file,
+                });
+            }
+
             break;
         }
     }

build.zig.zon

@@ -2,15 +2,17 @@
     .name = .tree_sitter,
     .version = "0.25.1",
     .fingerprint = 0x841224b4264a3d65,
-    .minimum_zig_version = "0.14.0",
+    .minimum_zig_version = "0.14.1",
     .dependencies = .{
         .tree_sitter = .{
-            .url = "https://github.com/tree-sitter/tree-sitter/archive/refs/heads/release-0.25.tar.gz",
-            .hash = "tree_sitter-0.25.8-Tw2sRxW7CwCiJosGHB0dIWv0tROYNsI6hSEzNmsfOrhR",
+            // tree-sitter/tree-sitter#4743
+            .url = "https://github.com/tree-sitter/tree-sitter/archive/ef93906.tar.gz",
+            .hash = "tree_sitter-0.25.8-Tw2sR8q-CwBpDiIhhNPHvHyqWjnWrbaDAf9PRcvEyGkh",
         },
         .tree_sitter_c = .{
-            .url = "https://github.com/tree-sitter/tree-sitter-c/archive/976d785.tar.gz",
-            .hash = "tree_sitter_c-0.24.0-VJuGjG5iQADj4pKFZy8LQF8Pcisq3KN9rbKo8amF6uoJ",
+            // tree-sitter/tree-sitter-c#275
+            .url = "https://github.com/tree-sitter/tree-sitter-c/archive/3450141.tar.gz",
+            .hash = "tree_sitter_c-0.24.0-VJuGjP9iQAAhFIo6ihR4_n7OB3dn5BSU6_smy9xLwoL8",
             .lazy = true,
         },
     },

src/language.zig

@@ -71,6 +71,14 @@ pub const Language = opaque {
         return ts_language_abi_version(self);
     }
 
+    /// Check if the language came from a Wasm module.
+    ///
+    /// If so, then in order to use this language with a `Parser`,
+    /// that parser must have a `WasmStore` assigned.
+    pub inline fn isWasm(self: *const Language) bool {
+        return ts_language_is_wasm(self);
+    }
+
     /// Get the number of distinct node types in this language.
     pub inline fn symbolCount(self: *const Language) u32 {
         return ts_language_symbol_count(self);
@@ -156,6 +164,7 @@ extern fn ts_language_delete(self: *const Language) void;
 extern fn ts_language_field_count(self: *const Language) u32;
 extern fn ts_language_field_id_for_name(self: *const Language, name: [*]const u8, name_length: u32) u16;
 extern fn ts_language_field_name_for_id(self: *const Language, id: u16) ?[*:0]const u8;
+extern fn ts_language_is_wasm(language: *const Language) bool;
 extern fn ts_language_metadata(self: *const Language) ?*const LanguageMetadata;
 extern fn ts_language_name(self: *const Language) ?[*:0]const u8;
 extern fn ts_language_next_state(self: *const Language, state: u16, symbol: u16) u16;

src/parser.zig

@@ -1,3 +1,4 @@
+const build = @import("build");
 const std = @import("std");
 
 const Input = @import("types.zig").Input;
@@ -10,6 +11,7 @@ const Node = @import("node.zig").Node;
 const Point = @import("types.zig").Point;
 const Range = @import("types.zig").Range;
 const Tree = @import("tree.zig").Tree;
+const WasmStore = @import("wasm.zig").WasmStore;
 
 /// A stateful object that is used to produce
 /// a syntax tree based on some source code.
@@ -31,11 +33,10 @@ pub const Parser = opaque {
 
     /// Set the language that the parser should use for parsing.
     ///
-    /// Returns an error if the language has an incompatible version.
-    pub fn setLanguage(self: *Parser, language: ?*const Language) error{IncompatibleVersion}!void {
-        if (!ts_parser_set_language(self, language)) {
-            return error.IncompatibleVersion;
-        }
+    /// Returns an error if the language has an incompatible version,
+    /// or if it was loaded from Wasm and the parser lacks a Wasm store.
+    pub fn setLanguage(self: *Parser, language: ?*const Language) error{IncompatibleLanguage}!void {
+        if (!ts_parser_set_language(self, language)) return error.IncompatibleLanguage;
     }
 
     /// Get the parser's current logger.
@@ -185,6 +186,20 @@ pub const Parser = opaque {
         ts_parser_reset(self);
     }
 
+    /// Assign the given Wasm store to the parser.
+    ///
+    /// A parser must have a Wasm store in order to use Wasm languages.
+    pub fn setWasmStore(self: *Parser, store: *WasmStore) void {
+        if (comptime !build.enable_wasm) @compileError("Wasm is not supported");
+        ts_parser_set_wasm_store(self, store);
+    }
+
+    /// Remove the parser's current Wasm store, if any, and return it.
+    pub fn takeWasmStore(self: *Parser) ?*WasmStore {
+        if (comptime !build.enable_wasm) @compileError("Wasm is not supported");
+        return ts_parser_take_wasm_store(self);
+    }
+
     /// Set the file to which the parser should write debugging graphs
     /// during parsing. The graphs are formatted in the DOT language.
     ///
@@ -244,3 +259,5 @@ extern fn ts_parser_cancellation_flag(self: *const Parser) ?*const usize;
 extern fn ts_parser_set_logger(self: *Parser, logger: Logger) void;
 extern fn ts_parser_logger(self: *const Parser) Logger;
 extern fn ts_parser_print_dot_graphs(self: *Parser, fd: c_int) void;
+extern fn ts_parser_set_wasm_store(parser: *Parser, store: *WasmStore) void;
+extern fn ts_parser_take_wasm_store(parser: *Parser) ?*WasmStore;

src/root.zig

@@ -27,4 +27,8 @@ pub const QueryCursor = @import("query_cursor.zig").QueryCursor;
 pub const Tree = @import("tree.zig").Tree;
 pub const TreeCursor = @import("tree_cursor.zig").TreeCursor;
 
+const wasm = @import("wasm.zig");
+pub const WasmEngine = wasm.WasmEngine;
+pub const WasmStore = wasm.WasmStore;
+
 pub const setAllocator = @import("alloc.zig").setAllocator;

src/test.zig

@@ -1,3 +1,4 @@
+const build = @import("build");
 const std = @import("std");
 const testing = std.testing;
 const ts = @import("root.zig");
@@ -19,6 +20,7 @@ test "Language" {
     try testing.expect(language.isVisible(1));
     try testing.expect(!language.isSupertype(1));
     try testing.expect(language.nextState(1, 161) > 1);
+    try testing.expect(!language.isWasm());
 
     const copy = language.dupe();
     try testing.expectEqual(language, copy);
@@ -363,3 +365,40 @@ test "QueryCursor" {
     try testing.expectEqual(1, match.captures[0].index);
     try testing.expectEqualStrings("(", match.captures[0].node.type());
 }
+
+test "Wasm" {
+    if (comptime !build.enable_wasm) return error.SkipZigTest;
+
+    const engine = try ts.WasmEngine.init(null);
+    defer engine.deinit();
+
+    var error_message: []u8 = undefined;
+    const store = try ts.WasmStore.create(testing.allocator, engine, &error_message);
+    errdefer testing.allocator.free(error_message);
+    defer store.destroy();
+
+    const wasm = @embedFile("tree-sitter-c.wasm");
+    const language = try store.loadLanguage(testing.allocator, "c", wasm, &error_message);
+    errdefer testing.allocator.free(error_message);
+    defer language.destroy();
+
+    try testing.expect(language.isWasm());
+    try testing.expectEqual(1, store.languageCount());
+
+    const parser = ts.Parser.create();
+    defer parser.destroy();
+
+    try testing.expectError(error.IncompatibleLanguage, parser.setLanguage(language));
+    parser.setWasmStore(store);
+    defer _ = parser.takeWasmStore();
+    try parser.setLanguage(language);
+
+    const tree = try parser.parseBuffer("int main() {}", null, .UTF_8);
+    defer tree.destroy();
+
+    try testing.expectEqualStrings("translation_unit", tree.rootNode().type());
+
+    try testing.expectError(error.ParseError, store.loadLanguage(testing.allocator, "c", "", &error_message));
+    try testing.expectEqualStrings("failed to parse dylink section of wasm module", error_message);
+    testing.allocator.free(error_message);
+}