diff --git a/packages/cli/package.json b/packages/cli/package.json index 713acfaa46..b200963646 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -38,7 +38,6 @@ "@eslint/js": "~8.57.0", "@octokit/types": "^11.1.0", "@swc/core": "^1.3.76", - "@types/better-sqlite3": "^7.6.11", "@types/cors": "^2.8.17", "@types/fs-extra": "^9.0.13", "@types/gitconfiglocal": "^2.0.1", @@ -103,7 +102,6 @@ "applicationinsights": "^2.1.4", "async": "^3.2.4", "axios": "^0.27.2", - "better-sqlite3": "^11.5.0", "body-parser": "^1.20.2", "boxen": "^5.0.1", "chalk": "^4.1.2", @@ -131,6 +129,7 @@ "lunr": "^2.3.9", "minimatch": "^5.1.2", "moo": "^0.5.1", + "node-sqlite3-wasm": "^0.8.34", "open": "^8.2.1", "openapi-diff": "^0.23.6", "openapi-types": "^12.1.3", @@ -167,9 +166,7 @@ "resources", "built/appmap.html", "built/sequenceDiagram.html", - "built/docs", - "../../node_modules/better-sqlite3/build/Release/better_sqlite3.node", - "node_modules/better-sqlite3/build/Release/better_sqlite3.node" + "built/docs" ], "outputPath": "dist" }, diff --git a/packages/cli/src/cmds/search/search.ts b/packages/cli/src/cmds/search/search.ts index c7370a40f0..5f7f640582 100644 --- a/packages/cli/src/cmds/search/search.ts +++ b/packages/cli/src/cmds/search/search.ts @@ -1,5 +1,5 @@ +import sqlite3 from 'node-sqlite3-wasm'; import yargs from 'yargs'; -import sqlite3 from 'better-sqlite3'; import assert from 'assert'; import { readFileSync } from 'fs'; import { writeFile } from 'fs/promises'; @@ -186,7 +186,7 @@ export const handler = async (argv: ArgumentTypes) => { }; const index = await buildIndexInTempDir('appmaps', async (indexFile) => { - const db = new sqlite3(indexFile); + const db = new sqlite3.Database(indexFile); const fileIndex = new FileIndex(db); await buildAppMapIndex(fileIndex, [process.cwd()]); return fileIndex; diff --git a/packages/cli/src/rpc/explain/index-files.ts b/packages/cli/src/rpc/explain/index-files.ts index 2543e5eac2..1838dc41cd 100644 --- a/packages/cli/src/rpc/explain/index-files.ts +++ b/packages/cli/src/rpc/explain/index-files.ts @@ -1,4 +1,4 @@ -import sqlite3 from 'better-sqlite3'; +import type sqlite3 from 'node-sqlite3-wasm'; import { buildFileIndex, diff --git a/packages/cli/src/rpc/explain/index-snippets.ts b/packages/cli/src/rpc/explain/index-snippets.ts index ceb2d58f01..da744e965a 100644 --- a/packages/cli/src/rpc/explain/index-snippets.ts +++ b/packages/cli/src/rpc/explain/index-snippets.ts @@ -1,3 +1,5 @@ +import sqlite3 from 'node-sqlite3-wasm'; + import { buildSnippetIndex, FileSearchResult, @@ -6,7 +8,6 @@ import { readFileSafe, SnippetIndex, } from '@appland/search'; -import sqlite3 from 'better-sqlite3'; export default async function indexSnippets( db: sqlite3.Database, diff --git a/packages/cli/src/rpc/explain/index/appmap-file-index.ts b/packages/cli/src/rpc/explain/index/appmap-file-index.ts index aaf6aa1bd6..bc152e3c09 100644 --- a/packages/cli/src/rpc/explain/index/appmap-file-index.ts +++ b/packages/cli/src/rpc/explain/index/appmap-file-index.ts @@ -1,4 +1,4 @@ -import sqlite3 from 'better-sqlite3'; +import sqlite3 from 'node-sqlite3-wasm'; import { FileIndex, SessionId } from '@appland/search'; @@ -10,7 +10,7 @@ export async function buildAppMapFileIndex( appmapDirectories: string[] ): Promise> { return await buildIndexInTempDir('appmaps', async (indexFile) => { - const db = new sqlite3(indexFile); + const db = new sqlite3.Database(indexFile); const fileIndex = new FileIndex(db); await buildAppMapIndex(fileIndex, appmapDirectories); return fileIndex; diff --git a/packages/cli/src/rpc/explain/index/project-file-index.ts b/packages/cli/src/rpc/explain/index/project-file-index.ts index c9b44c2b65..235a0e2d84 100644 --- a/packages/cli/src/rpc/explain/index/project-file-index.ts +++ b/packages/cli/src/rpc/explain/index/project-file-index.ts @@ -1,5 +1,5 @@ -import sqlite3 from 'better-sqlite3'; import makeDebug from 'debug'; +import sqlite3 from 'node-sqlite3-wasm'; import { buildFileIndex, @@ -71,7 +71,7 @@ export async function buildProjectFileIndex( excludePatterns: RegExp[] | undefined ): Promise> { return await buildIndexInTempDir('files', async (indexFile) => { - const db = new sqlite3(indexFile); + const db = new sqlite3.Database(indexFile); return await indexFiles(db, sourceDirectories, includePatterns, excludePatterns); }); } diff --git a/packages/cli/src/rpc/explain/index/project-file-snippet-index.ts b/packages/cli/src/rpc/explain/index/project-file-snippet-index.ts index 8b97cdaac1..9fdfbc5d89 100644 --- a/packages/cli/src/rpc/explain/index/project-file-snippet-index.ts +++ b/packages/cli/src/rpc/explain/index/project-file-snippet-index.ts @@ -1,4 +1,5 @@ -import sqlite3 from 'better-sqlite3'; +import sqlite3 from 'node-sqlite3-wasm'; + import { FileSearchResult, SessionId, SnippetIndex } from '@appland/search'; import buildIndexInTempDir, { CloseableIndex } from './build-index-in-temp-dir'; @@ -76,7 +77,7 @@ export async function buildProjectFileSnippetIndex( fileSearchResults: FileSearchResult[] ): Promise { const snippetIndex = await buildIndexInTempDir('snippets', async (indexFile) => { - const db = new sqlite3(indexFile); + const db = new sqlite3.Database(indexFile); return await indexSnippets(db, fileSearchResults); }); diff --git a/packages/cli/src/rpc/search/search.ts b/packages/cli/src/rpc/search/search.ts index cbeb63c1e3..7fe32dfcbb 100644 --- a/packages/cli/src/rpc/search/search.ts +++ b/packages/cli/src/rpc/search/search.ts @@ -1,5 +1,7 @@ import { isAbsolute, join } from 'path'; -import sqlite3 from 'better-sqlite3'; + +import sqlite3 from 'node-sqlite3-wasm'; + import { FileIndex, generateSessionId } from '@appland/search'; import { SearchRpc } from '@appland/rpc'; @@ -62,7 +64,7 @@ export async function handler( // Search across all AppMaps, creating a map from AppMap id to AppMapSearchResult const maxResults = options.maxDiagrams || options.maxResults || DEFAULT_MAX_DIAGRAMS; const index = await buildIndexInTempDir('appmaps', async (indexFile) => { - const db = new sqlite3(indexFile); + const db = new sqlite3.Database(indexFile); const fileIndex = new FileIndex(db); await buildAppMapIndex( fileIndex, diff --git a/packages/search/package.json b/packages/search/package.json index cd5fe365ee..7a2c4c2913 100644 --- a/packages/search/package.json +++ b/packages/search/package.json @@ -21,7 +21,6 @@ "author": "AppLand, Inc", "license": "Commons Clause + MIT", "devDependencies": { - "@types/better-sqlite3": "^7.6.11", "@types/jest": "^29.5.4", "@types/node": "^16", "eslint": "^9", @@ -39,8 +38,8 @@ "typescript-eslint": "^8.11.0" }, "dependencies": { - "better-sqlite3": "^11.5.0", "isbinaryfile": "^5.0.4", + "node-sqlite3-wasm": "^0.8.34", "yargs": "^17.7.2" } } diff --git a/packages/search/src/cli.ts b/packages/search/src/cli.ts index 0d3fb45dd8..14597e40c7 100644 --- a/packages/search/src/cli.ts +++ b/packages/search/src/cli.ts @@ -1,7 +1,7 @@ import yargs from 'yargs'; import { hideBin } from 'yargs/helpers'; -import sqlite3 from 'better-sqlite3'; import makeDebug from 'debug'; +import sqlite3 from 'node-sqlite3-wasm'; import { fileTokens } from './tokenize'; import FileIndex from './file-index'; @@ -62,7 +62,7 @@ const cli = yargs(hideBin(process.argv)) return !filterRE.test(path); }; - const db = new sqlite3(':memory:'); + const db = new sqlite3.Database(':memory:'); const fileIndex = new FileIndex(db); const sessionId = generateSessionId(); diff --git a/packages/search/src/file-index.ts b/packages/search/src/file-index.ts index 9f0c63cf14..1b9253c256 100644 --- a/packages/search/src/file-index.ts +++ b/packages/search/src/file-index.ts @@ -1,4 +1,5 @@ -import sqlite3 from 'better-sqlite3'; +import type sqlite3 from 'node-sqlite3-wasm'; + import { SessionId } from './session-id'; const CREATE_TABLE_SQL = `CREATE VIRTUAL TABLE file_content USING fts5( @@ -74,14 +75,14 @@ export type FileSearchResult = { export default class FileIndex { #insert: sqlite3.Statement; #updateBoost: sqlite3.Statement; - #deleteSession: sqlite3.Statement; - #search: sqlite3.Statement<[string, string, number]>; + #deleteSession: sqlite3.Statement; + #search: sqlite3.Statement; constructor(public database: sqlite3.Database) { this.database.exec(CREATE_TABLE_SQL); this.database.exec(CREATE_BOOST_TABLE_SQL); - this.database.pragma('journal_mode = OFF'); - this.database.pragma('synchronous = OFF'); + this.database.exec('PRAGMA journal_mode = OFF'); + this.database.exec('PRAGMA synchronous = OFF'); this.#insert = this.database.prepare(INSERT_SQL); this.#updateBoost = this.database.prepare(UPDATE_BOOST_SQL); this.#deleteSession = this.database.prepare(DELETE_SESSION_SQL); @@ -89,7 +90,7 @@ export default class FileIndex { } indexFile(directory: string, filePath: string, symbols: string, words: string): void { - this.#insert.run(directory, filePath, symbols, words); + this.#insert.run([directory, filePath, symbols, words]); } /** @@ -99,7 +100,7 @@ export default class FileIndex { * @param boostFactor - The factor by which to boost the file's relevance. */ boostFile(sessionId: SessionId, filePath: string, boostFactor: number): void { - this.#updateBoost.run(sessionId, filePath, boostFactor); + this.#updateBoost.run([sessionId, filePath, boostFactor]); } /** @@ -118,7 +119,7 @@ export default class FileIndex { * @returns An array of search results with directory, file path, and score. */ search(sessionId: SessionId, query: string, limit = 10): FileSearchResult[] { - const rows = this.#search.all(sessionId, query, limit) as FileIndexRow[]; + const rows = this.#search.all([sessionId, query, limit]) as FileIndexRow[]; return rows.map((row) => ({ directory: row.directory, filePath: row.file_path, diff --git a/packages/search/src/git.ts b/packages/search/src/git.ts index 79c4cfbe53..091af3325f 100644 --- a/packages/search/src/git.ts +++ b/packages/search/src/git.ts @@ -1,4 +1,4 @@ -import { exec as execCallback, spawn } from 'child_process'; +import { exec as execCallback, spawn } from 'node:child_process'; import { PathLike } from 'fs'; import { promisify } from 'util'; diff --git a/packages/search/src/snippet-index.ts b/packages/search/src/snippet-index.ts index bc13e33676..40f9f0d82b 100644 --- a/packages/search/src/snippet-index.ts +++ b/packages/search/src/snippet-index.ts @@ -1,5 +1,7 @@ import assert from 'assert'; -import sqlite3 from 'better-sqlite3'; + +import type sqlite3 from 'node-sqlite3-wasm'; + import { SessionId } from './session-id'; const CREATE_SNIPPET_CONTENT_TABLE_SQL = `CREATE VIRTUAL TABLE snippet_content USING fts5( @@ -115,14 +117,14 @@ type SnippetSearchRow = { export default class SnippetIndex { #insertSnippet: sqlite3.Statement; #updateSnippetBoost: sqlite3.Statement; - #deleteSession: sqlite3.Statement<[string]>; - #searchSnippet: sqlite3.Statement<[string, string, number]>; + #deleteSession: sqlite3.Statement; + #searchSnippet: sqlite3.Statement; constructor(public database: sqlite3.Database) { this.database.exec(CREATE_SNIPPET_CONTENT_TABLE_SQL); this.database.exec(CREATE_SNIPPET_BOOST_TABLE_SQL); - this.database.pragma('journal_mode = OFF'); - this.database.pragma('synchronous = OFF'); + this.database.exec('PRAGMA journal_mode = OFF'); + this.database.exec('PRAGMA synchronous = OFF'); this.#insertSnippet = this.database.prepare(INSERT_SNIPPET_SQL); this.#deleteSession = this.database.prepare(DELETE_SESSION_SQL); this.#updateSnippetBoost = this.database.prepare(UPDATE_SNIPPET_BOOST_SQL); @@ -152,7 +154,7 @@ export default class SnippetIndex { words: string, content: string ): void { - this.#insertSnippet.run(encodeSnippetId(snippetId), directory, symbols, words, content); + this.#insertSnippet.run([encodeSnippetId(snippetId), directory, symbols, words, content]); } /** @@ -162,11 +164,11 @@ export default class SnippetIndex { * @param boostFactor - The factor by which to boost the snippet's relevance. */ boostSnippet(sessionId: SessionId, snippetId: SnippetId, boostFactor: number): void { - this.#updateSnippetBoost.run(sessionId, encodeSnippetId(snippetId), boostFactor); + this.#updateSnippetBoost.run([sessionId, encodeSnippetId(snippetId), boostFactor]); } searchSnippets(sessionId: SessionId, query: string, limit = 10): SnippetSearchResult[] { - const rows = this.#searchSnippet.all(sessionId, query, limit) as SnippetSearchRow[]; + const rows = this.#searchSnippet.all([sessionId, query, limit]) as SnippetSearchRow[]; return rows.map((row) => ({ directory: row.directory, snippetId: parseSnippetId(row.snippet_id), diff --git a/packages/search/test/file-index.spec.ts b/packages/search/test/file-index.spec.ts index cff7369b1c..2b6664610c 100644 --- a/packages/search/test/file-index.spec.ts +++ b/packages/search/test/file-index.spec.ts @@ -1,5 +1,7 @@ import { strict as assert } from 'assert'; -import sqlite3 from 'better-sqlite3'; + +import sqlite3 from 'node-sqlite3-wasm'; + import FileIndex, { FileSearchResult } from '../src/file-index'; import { generateSessionId, SessionId } from '../src/session-id'; @@ -16,7 +18,7 @@ describe('FileIndex', () => { const directory = 'src'; beforeEach(() => { - db = new sqlite3(':memory:'); + db = new sqlite3.Database(':memory:'); index = new FileIndex(db); sessionId = generateSessionId(); }); diff --git a/packages/search/test/snippet-index.spec.ts b/packages/search/test/snippet-index.spec.ts index 352d6a2098..71bd102f42 100644 --- a/packages/search/test/snippet-index.spec.ts +++ b/packages/search/test/snippet-index.spec.ts @@ -1,5 +1,6 @@ import { strict as assert } from 'assert'; -import sqlite3 from 'better-sqlite3'; + +import sqlite3 from 'node-sqlite3-wasm'; import SnippetIndex, { fileChunkSnippetId, @@ -26,7 +27,7 @@ describe('SnippetIndex', () => { const snippet4: SnippetId = { type: 'code-snippet', id: 'test4.txt:31' }; beforeEach(() => { - db = new sqlite3(':memory:'); + db = new sqlite3.Database(':memory:'); index = new SnippetIndex(db); sessionId = generateSessionId(); }); diff --git a/yarn.lock b/yarn.lock index 83184c190e..c42659cbf5 100644 --- a/yarn.lock +++ b/yarn.lock @@ -160,7 +160,6 @@ __metadata: "@octokit/types": ^11.1.0 "@sidvind/better-ajv-errors": ^0.9.1 "@swc/core": ^1.3.76 - "@types/better-sqlite3": ^7.6.11 "@types/cors": ^2.8.17 "@types/fs-extra": ^9.0.13 "@types/gitconfiglocal": ^2.0.1 @@ -182,7 +181,6 @@ __metadata: applicationinsights: ^2.1.4 async: ^3.2.4 axios: ^0.27.2 - better-sqlite3: ^11.5.0 body-parser: ^1.20.2 boxen: ^5.0.1 chalk: ^4.1.2 @@ -221,6 +219,7 @@ __metadata: minimatch: ^5.1.2 moo: ^0.5.1 node-fetch: 2.6.7 + node-sqlite3-wasm: ^0.8.34 open: ^8.2.1 openapi-diff: ^0.23.6 openapi-types: ^12.1.3 @@ -592,10 +591,8 @@ __metadata: version: 0.0.0-use.local resolution: "@appland/search@workspace:packages/search" dependencies: - "@types/better-sqlite3": ^7.6.11 "@types/jest": ^29.5.4 "@types/node": ^16 - better-sqlite3: ^11.5.0 eslint: ^9 eslint-config-prettier: ^9 eslint-plugin-eslint-comments: ^3.2.0 @@ -605,6 +602,7 @@ __metadata: eslint-plugin-promise: ^7.1.0 isbinaryfile: ^5.0.4 jest: ^29.7.0 + node-sqlite3-wasm: ^0.8.34 prettier: ^3.3.3 ts-jest: ^29.2.5 tsc: ^2.0.4 @@ -10670,15 +10668,6 @@ __metadata: languageName: node linkType: hard -"@types/better-sqlite3@npm:^7.6.11": - version: 7.6.11 - resolution: "@types/better-sqlite3@npm:7.6.11" - dependencies: - "@types/node": "*" - checksum: 981740c78f97961bf1d8e78ed8bc841792cb25695afd9d920769d5b890d40ddf3724ace8a4db1adbf634c2f05c6efe3491ce9bf66818bacd93629ea060493546 - languageName: node - linkType: hard - "@types/body-parser@npm:*": version: 1.19.2 resolution: "@types/body-parser@npm:1.19.2" @@ -15901,17 +15890,6 @@ __metadata: languageName: node linkType: hard -"better-sqlite3@npm:^11.5.0": - version: 11.5.0 - resolution: "better-sqlite3@npm:11.5.0" - dependencies: - bindings: ^1.5.0 - node-gyp: latest - prebuild-install: ^7.1.1 - checksum: 37acef8d4272ad57fe211aa5a2e177f95443eafb794c7db6c2d0dae0a4fe4c2ef508b8b970d82f1d9744d009677d82067279f45e27d4155a5e68a578f5c5c051 - languageName: node - linkType: hard - "bfj@npm:^6.1.1": version: 6.1.2 resolution: "bfj@npm:6.1.2" @@ -32664,6 +32642,13 @@ __metadata: languageName: node linkType: hard +"node-sqlite3-wasm@npm:^0.8.34": + version: 0.8.34 + resolution: "node-sqlite3-wasm@npm:0.8.34" + checksum: c6c9b2c75d714badaa41ac66a9ed6a30a718252d9259268bb96d899308aa9024c8766a0995c10e5225e7c57cc8c1483ef9279c3b6e97ddd45b6dba3af3e6e551 + languageName: node + linkType: hard + "non-layered-tidy-tree-layout@npm:^2.0.2": version: 2.0.2 resolution: "non-layered-tidy-tree-layout@npm:2.0.2" @@ -35376,28 +35361,6 @@ __metadata: languageName: node linkType: hard -"prebuild-install@npm:^7.1.1": - version: 7.1.2 - resolution: "prebuild-install@npm:7.1.2" - dependencies: - detect-libc: ^2.0.0 - expand-template: ^2.0.3 - github-from-package: 0.0.0 - minimist: ^1.2.3 - mkdirp-classic: ^0.5.3 - napi-build-utils: ^1.0.1 - node-abi: ^3.3.0 - pump: ^3.0.0 - rc: ^1.2.7 - simple-get: ^4.0.0 - tar-fs: ^2.0.0 - tunnel-agent: ^0.6.0 - bin: - prebuild-install: bin.js - checksum: 543dadf8c60e004ae9529e6013ca0cbeac8ef38b5f5ba5518cb0b622fe7f8758b34e4b5cb1a791db3cdc9d2281766302df6088bd1a225f206925d6fee17d6c5c - languageName: node - linkType: hard - "prelude-ls@npm:^1.2.1": version: 1.2.1 resolution: "prelude-ls@npm:1.2.1"