From ca5d297661b0fc91268059f72b98a7d4cb690890 Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 16 Jul 2025 17:59:12 +0000 Subject: [PATCH 001/270] try to test btrfs integration using github actions --- .github/workflows/make-and-test.yml | 8 ++------ src/package.json | 2 +- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/.github/workflows/make-and-test.yml b/.github/workflows/make-and-test.yml index b233bf2421..933bd6e328 100644 --- a/.github/workflows/make-and-test.yml +++ b/.github/workflows/make-and-test.yml @@ -91,12 +91,8 @@ jobs: # cache: "pnpm" # cache-dependency-path: "src/packages/pnpm-lock.yaml" - - name: Download and install Valkey - run: | - VALKEY_VERSION=8.1.2 - curl -LO https://download.valkey.io/releases/valkey-${VALKEY_VERSION}-jammy-x86_64.tar.gz - tar -xzf valkey-${VALKEY_VERSION}-jammy-x86_64.tar.gz - sudo cp valkey-${VALKEY_VERSION}-jammy-x86_64/bin/valkey-server /usr/local/bin/ + - name: Install btrfs-progs + run: sudo apt-get update && sudo apt-get install -y btrfs-progs - name: Set up Python venv and Jupyter kernel run: | diff --git a/src/package.json b/src/package.json index d0f0701277..efa416dcd0 100644 --- a/src/package.json +++ b/src/package.json @@ -18,7 +18,7 @@ "version-check": "pip3 install typing_extensions mypy || pip3 install --break-system-packages typing_extensions mypy && ./workspaces.py version-check && mypy scripts/check_npm_packages.py", "test-parallel": "unset DEBUG && pnpm run version-check && cd packages && pnpm run -r --parallel test", "test": "unset DEBUG && pnpm run depcheck && pnpm run version-check && ./workspaces.py test", - "test-github-ci": "unset DEBUG && pnpm run depcheck && pnpm run version-check && ./workspaces.py test --exclude=jupyter,file-server --retries=1", + "test-github-ci": "unset DEBUG && pnpm run depcheck && pnpm run version-check && ./workspaces.py test --exclude=jupyter --retries=1", "depcheck": "cd packages && pnpm run -r --parallel depcheck", "prettier-all": "cd packages/", "local-ci": "./scripts/ci.sh", From f4ad609ff0bd5ad0990cae2efa3521f049f8ff35 Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 16 Jul 2025 18:15:01 +0000 Subject: [PATCH 002/270] also need bup for file-server --- .github/workflows/make-and-test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/make-and-test.yml b/.github/workflows/make-and-test.yml index 933bd6e328..19b2feeb82 100644 --- a/.github/workflows/make-and-test.yml +++ b/.github/workflows/make-and-test.yml @@ -91,8 +91,8 @@ jobs: # cache: "pnpm" # cache-dependency-path: "src/packages/pnpm-lock.yaml" - - name: Install btrfs-progs - run: sudo apt-get update && sudo apt-get install -y btrfs-progs + - name: Install btrfs-progs and bup for @cocalc/file-server + run: sudo apt-get update && sudo apt-get install -y btrfs-progs bup - name: Set up Python venv and Jupyter kernel run: | From adbb4ebcc9acf8b2cd76566897286ce808ea59d4 Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 16 Jul 2025 18:15:09 +0000 Subject: [PATCH 003/270] fix a run script --- src/scripts/g-tmux.sh | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/src/scripts/g-tmux.sh b/src/scripts/g-tmux.sh index f75aaf865a..386479673a 100755 --- a/src/scripts/g-tmux.sh +++ b/src/scripts/g-tmux.sh @@ -12,14 +12,8 @@ sleep 2 tmux send-keys -t mysession:1 'pnpm database' C-m if [ -n "$NO_RSPACK_DEV_SERVER" ]; then -sleep 2 -tmux send-keys -t mysession:2 'pnpm rspack' C-m - -else - -# no longer needed, due to using rspack for nextjs -#sleep 2 -#tmux send-keys -t mysession:2 '$PWD/scripts/memory_monitor.py' C-m + sleep 2 + tmux send-keys -t mysession:2 'pnpm rspack' C-m fi tmux attach -t mysession:1 From 4cd8f3b44f1e47aed94272adf3f58dda48c6c936 Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 16 Jul 2025 23:48:49 +0000 Subject: [PATCH 004/270] file-server: refactor fs sandbox module --- src/packages/file-server/btrfs/subvolume.ts | 6 +- .../{btrfs/subvolume-fs.ts => fs/index.ts} | 153 ++++++++++-------- 2 files changed, 89 insertions(+), 70 deletions(-) rename src/packages/file-server/{btrfs/subvolume-fs.ts => fs/index.ts} (51%) diff --git a/src/packages/file-server/btrfs/subvolume.ts b/src/packages/file-server/btrfs/subvolume.ts index 3f77abd9c4..6926fc8e57 100644 --- a/src/packages/file-server/btrfs/subvolume.ts +++ b/src/packages/file-server/btrfs/subvolume.ts @@ -6,10 +6,10 @@ import { type Filesystem, DEFAULT_SUBVOLUME_SIZE } from "./filesystem"; import refCache from "@cocalc/util/refcache"; import { sudo } from "./util"; import { join, normalize } from "path"; -import { SubvolumeFilesystem } from "./subvolume-fs"; import { SubvolumeBup } from "./subvolume-bup"; import { SubvolumeSnapshots } from "./subvolume-snapshots"; import { SubvolumeQuota } from "./subvolume-quota"; +import { SandboxedFilesystem } from "../fs"; import { exists } from "@cocalc/backend/misc/async-utils-node"; import getLogger from "@cocalc/backend/logger"; @@ -26,7 +26,7 @@ export class Subvolume { public readonly filesystem: Filesystem; public readonly path: string; - public readonly fs: SubvolumeFilesystem; + public readonly fs: SandboxedFilesystem; public readonly bup: SubvolumeBup; public readonly snapshots: SubvolumeSnapshots; public readonly quota: SubvolumeQuota; @@ -35,7 +35,7 @@ export class Subvolume { this.filesystem = filesystem; this.name = name; this.path = join(filesystem.opts.mount, name); - this.fs = new SubvolumeFilesystem(this); + this.fs = new SandboxedFilesystem(this.path); this.bup = new SubvolumeBup(this); this.snapshots = new SubvolumeSnapshots(this); this.quota = new SubvolumeQuota(this); diff --git a/src/packages/file-server/btrfs/subvolume-fs.ts b/src/packages/file-server/fs/index.ts similarity index 51% rename from src/packages/file-server/btrfs/subvolume-fs.ts rename to src/packages/file-server/fs/index.ts index f1f2dd3677..487327c17e 100644 --- a/src/packages/file-server/btrfs/subvolume-fs.ts +++ b/src/packages/file-server/fs/index.ts @@ -1,3 +1,9 @@ +/* +Given a path to a folder on the filesystem, this provides +a wrapper class with an API very similar to the fs/promises modules, +but which only allows access to files in that folder. +*/ + import { appendFile, chmod, @@ -21,111 +27,100 @@ import { import { exists } from "@cocalc/backend/misc/async-utils-node"; import { type DirectoryListingEntry } from "@cocalc/util/types"; import getListing from "@cocalc/backend/get-listing"; -import { type Subvolume } from "./subvolume"; -import { isdir, sudo } from "./util"; - -export class SubvolumeFilesystem { - constructor(private subvolume: Subvolume) {} - - private normalize = this.subvolume.normalize; +import { isdir, sudo } from "../btrfs/util"; +import { join, resolve } from "path"; - ls = async ( - path: string, - { hidden, limit }: { hidden?: boolean; limit?: number } = {}, - ): Promise => { - return await getListing(this.normalize(path), hidden, { - limit, - home: "/", - }); - }; +export class SandboxedFilesystem { + // path should be the path to a FOLDER on the filesystem (not a file) + constructor(public readonly path: string) {} - readFile = async (path: string, encoding?: any): Promise => { - return await readFile(this.normalize(path), encoding); + private safeAbsPath = (path: string) => { + if (typeof path != "string") { + throw Error(`path must be a string but is of type ${typeof path}`); + } + return join(this.path, resolve("/", path)); }; - writeFile = async (path: string, data: string | Buffer) => { - return await writeFile(this.normalize(path), data); + appendFile = async (path: string, data: string | Buffer, encoding?) => { + return await appendFile(this.safeAbsPath(path), data, encoding); }; - appendFile = async (path: string, data: string | Buffer, encoding?) => { - return await appendFile(this.normalize(path), data, encoding); + chmod = async (path: string, mode: string | number) => { + await chmod(this.safeAbsPath(path), mode); }; - unlink = async (path: string) => { - await unlink(this.normalize(path)); + copyFile = async (src: string, dest: string) => { + await copyFile(this.safeAbsPath(src), this.safeAbsPath(dest)); }; - stat = async (path: string) => { - return await stat(this.normalize(path)); + cp = async (src: string, dest: string, options?) => { + await cp(this.safeAbsPath(src), this.safeAbsPath(dest), options); }; exists = async (path: string) => { - return await exists(this.normalize(path)); + return await exists(this.safeAbsPath(path)); }; // hard link link = async (existingPath: string, newPath: string) => { - return await link(this.normalize(existingPath), this.normalize(newPath)); - }; - - symlink = async (target: string, path: string) => { - return await symlink(this.normalize(target), this.normalize(path)); - }; - - realpath = async (path: string) => { - const x = await realpath(this.normalize(path)); - return x.slice(this.subvolume.path.length + 1); - }; - - rename = async (oldPath: string, newPath: string) => { - await rename(this.normalize(oldPath), this.normalize(newPath)); + return await link( + this.safeAbsPath(existingPath), + this.safeAbsPath(newPath), + ); }; - utimes = async ( + ls = async ( path: string, - atime: number | string | Date, - mtime: number | string | Date, - ) => { - await utimes(this.normalize(path), atime, mtime); + { hidden, limit }: { hidden?: boolean; limit?: number } = {}, + ): Promise => { + return await getListing(this.safeAbsPath(path), hidden, { + limit, + home: "/", + }); }; - watch = (filename: string, options?) => { - return watch(this.normalize(filename), options); + mkdir = async (path: string, options?) => { + await mkdir(this.safeAbsPath(path), options); }; - truncate = async (path: string, len?: number) => { - await truncate(this.normalize(path), len); + readFile = async (path: string, encoding?: any): Promise => { + return await readFile(this.safeAbsPath(path), encoding); }; - copyFile = async (src: string, dest: string) => { - await copyFile(this.normalize(src), this.normalize(dest)); + realpath = async (path: string) => { + const x = await realpath(this.safeAbsPath(path)); + return x.slice(this.path.length + 1); }; - cp = async (src: string, dest: string, options?) => { - await cp(this.normalize(src), this.normalize(dest), options); + rename = async (oldPath: string, newPath: string) => { + await rename(this.safeAbsPath(oldPath), this.safeAbsPath(newPath)); }; - chmod = async (path: string, mode: string | number) => { - await chmod(this.normalize(path), mode); + rm = async (path: string, options?) => { + await rm(this.safeAbsPath(path), options); }; - mkdir = async (path: string, options?) => { - await mkdir(this.normalize(path), options); + rmdir = async (path: string, options?) => { + await rmdir(this.safeAbsPath(path), options); }; rsync = async ({ src, target, - args = ["-axH"], timeout = 5 * 60 * 1000, }: { src: string; target: string; - args?: string[]; timeout?: number; }): Promise<{ stdout: string; stderr: string; exit_code: number }> => { - let srcPath = this.normalize(src); - let targetPath = this.normalize(target); + let srcPath = this.safeAbsPath(src); + let targetPath = this.safeAbsPath(target); + if (src.endsWith("/")) { + srcPath += "/"; + } + if (target.endsWith("/")) { + targetPath += "/"; + } if (!srcPath.endsWith("/") && (await isdir(srcPath))) { srcPath += "/"; if (!targetPath.endsWith("/")) { @@ -134,17 +129,41 @@ export class SubvolumeFilesystem { } return await sudo({ command: "rsync", - args: [...args, srcPath, targetPath], + args: [srcPath, targetPath], err_on_exit: false, timeout: timeout / 1000, }); }; - rmdir = async (path: string, options?) => { - await rmdir(this.normalize(path), options); + stat = async (path: string) => { + return await stat(this.safeAbsPath(path)); }; - rm = async (path: string, options?) => { - await rm(this.normalize(path), options); + symlink = async (target: string, path: string) => { + return await symlink(this.safeAbsPath(target), this.safeAbsPath(path)); + }; + + truncate = async (path: string, len?: number) => { + await truncate(this.safeAbsPath(path), len); + }; + + unlink = async (path: string) => { + await unlink(this.safeAbsPath(path)); + }; + + utimes = async ( + path: string, + atime: number | string | Date, + mtime: number | string | Date, + ) => { + await utimes(this.safeAbsPath(path), atime, mtime); + }; + + watch = (filename: string, options?) => { + return watch(this.safeAbsPath(filename), options); + }; + + writeFile = async (path: string, data: string | Buffer) => { + return await writeFile(this.safeAbsPath(path), data); }; } From d03d6f9be68773dfcffbd564fc26e15cfde7f16c Mon Sep 17 00:00:00 2001 From: William Stein Date: Thu, 17 Jul 2025 01:56:45 +0000 Subject: [PATCH 005/270] working on adding conat fileserver support --- src/packages/conat/files/fs.ts | 134 ++++++++++++++++++ .../file-server/btrfs/subvolume-bup.ts | 6 +- src/packages/file-server/btrfs/subvolume.ts | 40 ++++-- src/packages/file-server/conat/local-path.ts | 44 ++++++ .../file-server/conat/test/local-path.test.ts | 3 + src/packages/file-server/fs/sandbox.test.ts | 59 ++++++++ .../file-server/fs/{index.ts => sandbox.ts} | 50 ++----- src/packages/file-server/package.json | 17 +-- src/packages/pnpm-lock.yaml | 7 +- 9 files changed, 301 insertions(+), 59 deletions(-) create mode 100644 src/packages/conat/files/fs.ts create mode 100644 src/packages/file-server/conat/local-path.ts create mode 100644 src/packages/file-server/conat/test/local-path.test.ts create mode 100644 src/packages/file-server/fs/sandbox.test.ts rename src/packages/file-server/fs/{index.ts => sandbox.ts} (78%) diff --git a/src/packages/conat/files/fs.ts b/src/packages/conat/files/fs.ts new file mode 100644 index 0000000000..048a491f91 --- /dev/null +++ b/src/packages/conat/files/fs.ts @@ -0,0 +1,134 @@ +import { type Client } from "@cocalc/conat/core/client"; +import { conat } from "@cocalc/conat/client"; + +export interface Filesystem { + appendFile: (path: string, data: string | Buffer, encoding?) => Promise; + chmod: (path: string, mode: string | number) => Promise; + copyFile: (src: string, dest: string) => Promise; + cp: (src: string, dest: string, options?) => Promise; + exists: (path: string) => Promise; + link: (existingPath: string, newPath: string) => Promise; + mkdir: (path: string, options?) => Promise; + readFile: (path: string, encoding?: any) => Promise; + readdir: (path: string) => Promise; + realpath: (path: string) => Promise; + rename: (oldPath: string, newPath: string) => Promise; + rm: (path: string, options?) => Promise; + rmdir: (path: string, options?) => Promise; + stat: (path: string) => Promise; + symlink: (target: string, path: string) => Promise; + truncate: (path: string, len?: number) => Promise; + unlink: (path: string) => Promise; + utimes: ( + path: string, + atime: number | string | Date, + mtime: number | string | Date, + ) => Promise; + writeFile: (path: string, data: string | Buffer) => Promise; +} + +export interface Stats { + dev: number; + ino: number; + mode: number; + nlink: number; + uid: number; + gid: number; + rdev: number; + size: number; + blksize: number; + blocks: number; + atimeMs: number; + mtimeMs: number; + ctimeMs: number; + birthtimeMs: number; + atime: Date; + mtime: Date; + ctime: Date; + birthtime: Date; +} + +interface Options { + service: string; + client?: Client; + fs: (subject?: string) => Promise; +} + +export async function fsServer({ service, fs, client }: Options) { + return await (client ?? conat()).service( + `${service}.*`, + { + async appendFile(path: string, data: string | Buffer, encoding?) { + await (await fs(this.subject)).appendFile(path, data, encoding); + }, + async chmod(path: string, mode: string | number) { + await (await fs(this.subject)).chmod(path, mode); + }, + async copyFile(src: string, dest: string) { + await (await fs(this.subject)).copyFile(src, dest); + }, + async cp(src: string, dest: string, options?) { + await (await fs(this.subject)).cp(src, dest, options); + }, + async exists(path: string) { + await (await fs(this.subject)).exists(path); + }, + async link(existingPath: string, newPath: string) { + await (await fs(this.subject)).link(existingPath, newPath); + }, + async mkdir(path: string, options?) { + await (await fs(this.subject)).mkdir(path, options); + }, + async readFile(path: string, encoding?) { + return await (await fs(this.subject)).readFile(path, encoding); + }, + async readdir(path: string) { + return await (await fs(this.subject)).readdir(path); + }, + async realpath(path: string) { + return await (await fs(this.subject)).realpath(path); + }, + async rename(oldPath: string, newPath: string) { + return await (await fs(this.subject)).rename(oldPath, newPath); + }, + async rm(path: string, options?) { + return await (await fs(this.subject)).rm(path, options); + }, + async rmdir(path: string, options?) { + return await (await fs(this.subject)).rmdir(path, options); + }, + async stat(path: string): Promise { + return await (await fs(this.subject)).stat(path); + }, + async symlink(target: string, path: string) { + return await (await fs(this.subject)).symlink(target, path); + }, + async truncate(path: string, len?: number) { + return await (await fs(this.subject)).truncate(path, len); + }, + async unlink(path: string) { + return await (await fs(this.subject)).unlink(path); + }, + async utimes( + path: string, + atime: number | string | Date, + mtime: number | string | Date, + ) { + return await (await fs(this.subject)).utimes(path, atime, mtime); + }, + async writeFile(path: string, data: string | Buffer) { + return await (await fs(this.subject)).writeFile(path, data); + }, + }, + ); +} + +export function fsClient({ + client, + subject, +}: { + client?: Client; + subject: string; +}) { + return (client ?? conat()).call(subject); +} diff --git a/src/packages/file-server/btrfs/subvolume-bup.ts b/src/packages/file-server/btrfs/subvolume-bup.ts index 21cbbff364..3849b49379 100644 --- a/src/packages/file-server/btrfs/subvolume-bup.ts +++ b/src/packages/file-server/btrfs/subvolume-bup.ts @@ -46,7 +46,7 @@ export class SubvolumeBup { `createBackup: creating ${BUP_SNAPSHOT} to get a consistent backup`, ); await this.subvolume.snapshots.create(BUP_SNAPSHOT); - const target = this.subvolume.normalize( + const target = this.subvolume.fs.safeAbsPath( this.subvolume.snapshots.path(BUP_SNAPSHOT), ); @@ -133,7 +133,9 @@ export class SubvolumeBup { return v; } - path = normalize(path); + path = this.subvolume.fs + .safeAbsPath(path) + .slice(this.subvolume.path.length); const { stdout } = await sudo({ command: "bup", args: [ diff --git a/src/packages/file-server/btrfs/subvolume.ts b/src/packages/file-server/btrfs/subvolume.ts index 6926fc8e57..72c3dd7d54 100644 --- a/src/packages/file-server/btrfs/subvolume.ts +++ b/src/packages/file-server/btrfs/subvolume.ts @@ -4,12 +4,12 @@ A subvolume import { type Filesystem, DEFAULT_SUBVOLUME_SIZE } from "./filesystem"; import refCache from "@cocalc/util/refcache"; -import { sudo } from "./util"; -import { join, normalize } from "path"; +import { isdir, sudo } from "./util"; +import { join } from "path"; import { SubvolumeBup } from "./subvolume-bup"; import { SubvolumeSnapshots } from "./subvolume-snapshots"; import { SubvolumeQuota } from "./subvolume-quota"; -import { SandboxedFilesystem } from "../fs"; +import { SandboxedFilesystem } from "../fs/sandbox"; import { exists } from "@cocalc/backend/misc/async-utils-node"; import getLogger from "@cocalc/backend/logger"; @@ -77,11 +77,35 @@ export class Subvolume { }); }; - // this should provide a path that is guaranteed to be - // inside this.path on the filesystem or throw error - // [ ] TODO: not sure if the code here is sufficient!! - normalize = (path: string) => { - return join(this.path, normalize(path)); + rsync = async ({ + src, + target, + timeout = 5 * 60 * 1000, + }: { + src: string; + target: string; + timeout?: number; + }): Promise<{ stdout: string; stderr: string; exit_code: number }> => { + let srcPath = this.fs.safeAbsPath(src); + let targetPath = this.fs.safeAbsPath(target); + if (src.endsWith("/")) { + srcPath += "/"; + } + if (target.endsWith("/")) { + targetPath += "/"; + } + if (!srcPath.endsWith("/") && (await isdir(srcPath))) { + srcPath += "/"; + if (!targetPath.endsWith("/")) { + targetPath += "/"; + } + } + return await sudo({ + command: "rsync", + args: [srcPath, targetPath], + err_on_exit: false, + timeout: timeout / 1000, + }); }; } diff --git a/src/packages/file-server/conat/local-path.ts b/src/packages/file-server/conat/local-path.ts new file mode 100644 index 0000000000..e8450f3adf --- /dev/null +++ b/src/packages/file-server/conat/local-path.ts @@ -0,0 +1,44 @@ +import { fsServer } from "@cocalc/conat/files/fs"; +import { conat } from "@cocalc/backend/conat"; +import { SandboxedFilesystem } from "@cocalc/file-server/fs/sandbox"; +import { mkdir } from "fs/promises"; +import { join } from "path"; +import { isValidUUID } from "@cocalc/util/misc"; + +export function localPathFileserver({ + service, + path, +}: { + service: string; + path: string; +}) { + const client = conat(); + const server = fsServer({ + service, + client, + fs: async (subject: string) => { + const project_id = getProjectId(subject); + const p = join(path, project_id); + try { + await mkdir(p); + } catch {} + return new SandboxedFilesystem(p); + }, + }); + return server; +} + +function getProjectId(subject: string) { + const v = subject.split("."); + if (v.length != 2) { + throw Error("subject must have 2 segments"); + } + if (!v[1].startsWith("project-")) { + throw Error("second segment of subject must start with 'project-'"); + } + const project_id = v[1].slice("project-".length); + if (!isValidUUID(project_id)) { + throw Error("not a valid project id"); + } + return project_id; +} diff --git a/src/packages/file-server/conat/test/local-path.test.ts b/src/packages/file-server/conat/test/local-path.test.ts new file mode 100644 index 0000000000..4aa561af08 --- /dev/null +++ b/src/packages/file-server/conat/test/local-path.test.ts @@ -0,0 +1,3 @@ +describe("use the simple fileserver", () => { + it("does nothing", async () => {}); +}); diff --git a/src/packages/file-server/fs/sandbox.test.ts b/src/packages/file-server/fs/sandbox.test.ts new file mode 100644 index 0000000000..73bd45b171 --- /dev/null +++ b/src/packages/file-server/fs/sandbox.test.ts @@ -0,0 +1,59 @@ +import { SandboxedFilesystem } from "./sandbox"; +import { mkdtemp, mkdir, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "path"; + +let tempDir; +beforeAll(async () => { + tempDir = await mkdtemp(join(tmpdir(), "cocalc")); +}); + +describe("test using the filesystem sandbox to do a few standard things", () => { + let fs; + it("creates and reads file", async () => { + await mkdir(join(tempDir, "test-1")); + fs = new SandboxedFilesystem(join(tempDir, "test-1")); + await fs.writeFile("a", "hi"); + const r = await fs.readFile("a", "utf8"); + expect(r).toEqual("hi"); + }); + + it("truncate file", async () => { + await fs.writeFile("b", "hello"); + await fs.truncate("b", 4); + const r = await fs.readFile("b", "utf8"); + expect(r).toEqual("hell"); + }); +}); + +describe("make various attempts to break out of the sandbox", () => { + let fs; + it("creates sandbox", async () => { + await mkdir(join(tempDir, "test-2")); + fs = new SandboxedFilesystem(join(tempDir, "test-2")); + await fs.writeFile("x", "hi"); + }); + + it("obvious first attempt to escape fails", async () => { + const v = await fs.readdir(".."); + expect(v).toEqual(["x"]); + }); + + it("obvious first attempt to escape fails", async () => { + const v = await fs.readdir("a/../.."); + expect(v).toEqual(["x"]); + }); + + it("another attempt", async () => { + await fs.copyFile("/x", "/tmp"); + const v = await fs.readdir("a/../.."); + expect(v).toEqual(["tmp", "x"]); + + const r = await fs.readFile("tmp", "utf8"); + expect(r).toEqual("hi"); + }); +}); + +afterAll(async () => { + await rm(tempDir, { force: true, recursive: true }); +}); diff --git a/src/packages/file-server/fs/index.ts b/src/packages/file-server/fs/sandbox.ts similarity index 78% rename from src/packages/file-server/fs/index.ts rename to src/packages/file-server/fs/sandbox.ts index 487327c17e..dabd277ec2 100644 --- a/src/packages/file-server/fs/index.ts +++ b/src/packages/file-server/fs/sandbox.ts @@ -1,7 +1,14 @@ /* Given a path to a folder on the filesystem, this provides -a wrapper class with an API very similar to the fs/promises modules, +a wrapper class with an API similar to the fs/promises modules, but which only allows access to files in that folder. +It's a bit simpler with return data that is always +serializable. + +Absolute and relative paths are considered as relative to the input folder path. + +REFERENCE: We don't use https://github.com/metarhia/sandboxed-fs, but did +look at the code. */ import { @@ -10,6 +17,7 @@ import { cp, copyFile, link, + readdir, readFile, realpath, rename, @@ -27,14 +35,13 @@ import { import { exists } from "@cocalc/backend/misc/async-utils-node"; import { type DirectoryListingEntry } from "@cocalc/util/types"; import getListing from "@cocalc/backend/get-listing"; -import { isdir, sudo } from "../btrfs/util"; import { join, resolve } from "path"; export class SandboxedFilesystem { // path should be the path to a FOLDER on the filesystem (not a file) constructor(public readonly path: string) {} - private safeAbsPath = (path: string) => { + safeAbsPath = (path: string) => { if (typeof path != "string") { throw Error(`path must be a string but is of type ${typeof path}`); } @@ -87,7 +94,11 @@ export class SandboxedFilesystem { return await readFile(this.safeAbsPath(path), encoding); }; - realpath = async (path: string) => { + readdir = async (path: string): Promise => { + return await readdir(this.safeAbsPath(path)); + }; + + realpath = async (path: string): Promise => { const x = await realpath(this.safeAbsPath(path)); return x.slice(this.path.length + 1); }; @@ -104,37 +115,6 @@ export class SandboxedFilesystem { await rmdir(this.safeAbsPath(path), options); }; - rsync = async ({ - src, - target, - timeout = 5 * 60 * 1000, - }: { - src: string; - target: string; - timeout?: number; - }): Promise<{ stdout: string; stderr: string; exit_code: number }> => { - let srcPath = this.safeAbsPath(src); - let targetPath = this.safeAbsPath(target); - if (src.endsWith("/")) { - srcPath += "/"; - } - if (target.endsWith("/")) { - targetPath += "/"; - } - if (!srcPath.endsWith("/") && (await isdir(srcPath))) { - srcPath += "/"; - if (!targetPath.endsWith("/")) { - targetPath += "/"; - } - } - return await sudo({ - command: "rsync", - args: [srcPath, targetPath], - err_on_exit: false, - timeout: timeout / 1000, - }); - }; - stat = async (path: string) => { return await stat(this.safeAbsPath(path)); }; diff --git a/src/packages/file-server/package.json b/src/packages/file-server/package.json index 58285c4bf6..c72ed98314 100644 --- a/src/packages/file-server/package.json +++ b/src/packages/file-server/package.json @@ -4,7 +4,9 @@ "description": "CoCalc File Server", "exports": { "./btrfs": "./dist/btrfs/index.js", - "./btrfs/*": "./dist/btrfs/*.js" + "./btrfs/*": "./dist/btrfs/*.js", + "./conat/*": "./dist/conat/*.js", + "./fs/*": "./dist/fs/*.js" }, "scripts": { "preinstall": "npx only-allow pnpm", @@ -14,20 +16,13 @@ "depcheck": "pnpx depcheck", "clean": "rm -rf node_modules dist" }, - "files": [ - "dist/**", - "README.md", - "package.json" - ], + "files": ["dist/**", "README.md", "package.json"], "author": "SageMath, Inc.", - "keywords": [ - "utilities", - "btrfs", - "cocalc" - ], + "keywords": ["utilities", "btrfs", "cocalc"], "license": "SEE LICENSE.md", "dependencies": { "@cocalc/backend": "workspace:*", + "@cocalc/conat": "workspace:*", "@cocalc/file-server": "workspace:*", "@cocalc/util": "workspace:*", "awaiting": "^3.0.0" diff --git a/src/packages/pnpm-lock.yaml b/src/packages/pnpm-lock.yaml index 1fe4499fe0..ac022d9c8f 100644 --- a/src/packages/pnpm-lock.yaml +++ b/src/packages/pnpm-lock.yaml @@ -289,6 +289,9 @@ importers: '@cocalc/backend': specifier: workspace:* version: link:../backend + '@cocalc/conat': + specifier: workspace:* + version: link:../conat '@cocalc/file-server': specifier: workspace:* version: 'link:' @@ -15321,7 +15324,7 @@ snapshots: axios@1.10.0: dependencies: - follow-redirects: 1.15.9 + follow-redirects: 1.15.9(debug@4.4.1) form-data: 4.0.3 proxy-from-env: 1.1.0 transitivePeerDependencies: @@ -17457,8 +17460,6 @@ snapshots: dependencies: dtype: 2.0.0 - follow-redirects@1.15.9: {} - follow-redirects@1.15.9(debug@4.4.1): optionalDependencies: debug: 4.4.1 From cbe88607c4a5351d0b2f62941a3cc520e9ba3b56 Mon Sep 17 00:00:00 2001 From: William Stein Date: Thu, 17 Jul 2025 03:27:37 +0000 Subject: [PATCH 006/270] start writing unit tests for local-path fs --- .../file-server/conat/test/local-path.test.ts | 42 ++++++++++++++++++- src/packages/file-server/package.json | 2 +- 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/src/packages/file-server/conat/test/local-path.test.ts b/src/packages/file-server/conat/test/local-path.test.ts index 4aa561af08..682cec7719 100644 --- a/src/packages/file-server/conat/test/local-path.test.ts +++ b/src/packages/file-server/conat/test/local-path.test.ts @@ -1,3 +1,43 @@ +import { localPathFileserver } from "../local-path"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "path"; +import { fsClient } from "@cocalc/conat/files/fs"; + +let tempDir; +beforeAll(async () => { + tempDir = await mkdtemp(join(tmpdir(), "cocalc-local-path")); +}); + describe("use the simple fileserver", () => { - it("does nothing", async () => {}); + let service; + it("creates the simple fileserver service", async () => { + service = await localPathFileserver({ service: "fs", path: tempDir }); + }); + + const project_id = "6b851643-360e-435e-b87e-f9a6ab64a8b1"; + let fs; + it("create a client", () => { + fs = fsClient({ subject: `fs.project-${project_id}` }); + }); + + it("checks appendFile works", async () => { + await fs.appendFile("a", "foo"); + expect(await fs.readFile("a", "utf8")).toEqual("foo"); + }); + + it("checks chmod works", async () => { + await fs.writeFile("b", "hi"); + await fs.chmod("b", 0o755); + const s = await fs.stat("b"); + expect(s.mode.toString(8)).toBe("100755"); + }); + + it("closes the service", () => { + service.close(); + }); +}); + +afterAll(async () => { + await rm(tempDir, { force: true, recursive: true }); }); diff --git a/src/packages/file-server/package.json b/src/packages/file-server/package.json index c72ed98314..55d9d9980a 100644 --- a/src/packages/file-server/package.json +++ b/src/packages/file-server/package.json @@ -12,7 +12,7 @@ "preinstall": "npx only-allow pnpm", "build": "pnpm exec tsc --build", "tsc": "pnpm exec tsc --watch --pretty --preserveWatchOutput", - "test": "pnpm exec jest", + "test": "pnpm exec jest --forceExit", "depcheck": "pnpx depcheck", "clean": "rm -rf node_modules dist" }, From 3e57982ebebf9fef87c44b4f0c55d9efd3e3b6e2 Mon Sep 17 00:00:00 2001 From: William Stein Date: Thu, 17 Jul 2025 04:41:39 +0000 Subject: [PATCH 007/270] file-server: implement more of the fs api and unit tests, and even stats.isDirectory(), etc. --- src/packages/conat/core/client.ts | 6 +- src/packages/conat/files/fs.ts | 101 +++++++++++++++--- .../file-server/conat/test/local-path.test.ts | 76 +++++++++++-- src/packages/file-server/fs/sandbox.ts | 10 ++ src/packages/frontend/conat/client.ts | 3 + 5 files changed, 176 insertions(+), 20 deletions(-) diff --git a/src/packages/conat/core/client.ts b/src/packages/conat/core/client.ts index 27e8f0bb4e..c0f541c91f 100644 --- a/src/packages/conat/core/client.ts +++ b/src/packages/conat/core/client.ts @@ -1133,7 +1133,11 @@ export class Client extends EventEmitter { return new Proxy( {}, { - get: (_, name) => { + get: (target, name) => { + const s = target[String(name)]; + if (s !== undefined) { + return s; + } if (typeof name !== "string") { return undefined; } diff --git a/src/packages/conat/files/fs.ts b/src/packages/conat/files/fs.ts index 048a491f91..e33d745dbe 100644 --- a/src/packages/conat/files/fs.ts +++ b/src/packages/conat/files/fs.ts @@ -4,10 +4,12 @@ import { conat } from "@cocalc/conat/client"; export interface Filesystem { appendFile: (path: string, data: string | Buffer, encoding?) => Promise; chmod: (path: string, mode: string | number) => Promise; + constants: () => Promise<{ [key: string]: number }>; copyFile: (src: string, dest: string) => Promise; cp: (src: string, dest: string, options?) => Promise; exists: (path: string) => Promise; link: (existingPath: string, newPath: string) => Promise; + lstat: (path: string) => Promise; mkdir: (path: string, options?) => Promise; readFile: (path: string, encoding?: any) => Promise; readdir: (path: string) => Promise; @@ -15,7 +17,7 @@ export interface Filesystem { rename: (oldPath: string, newPath: string) => Promise; rm: (path: string, options?) => Promise; rmdir: (path: string, options?) => Promise; - stat: (path: string) => Promise; + stat: (path: string) => Promise; symlink: (target: string, path: string) => Promise; truncate: (path: string, len?: number) => Promise; unlink: (path: string) => Promise; @@ -27,7 +29,7 @@ export interface Filesystem { writeFile: (path: string, data: string | Buffer) => Promise; } -export interface Stats { +interface IStats { dev: number; ino: number; mode: number; @@ -48,6 +50,48 @@ export interface Stats { birthtime: Date; } +class Stats { + dev: number; + ino: number; + mode: number; + nlink: number; + uid: number; + gid: number; + rdev: number; + size: number; + blksize: number; + blocks: number; + atimeMs: number; + mtimeMs: number; + ctimeMs: number; + birthtimeMs: number; + atime: Date; + mtime: Date; + ctime: Date; + birthtime: Date; + + constructor(private constants: { [key: string]: number }) {} + + isSymbolicLink = () => + (this.mode & this.constants.S_IFMT) === this.constants.S_IFLNK; + + isFile = () => (this.mode & this.constants.S_IFMT) === this.constants.S_IFREG; + + isDirectory = () => + (this.mode & this.constants.S_IFMT) === this.constants.S_IFDIR; + + isBlockDevice = () => + (this.mode & this.constants.S_IFMT) === this.constants.S_IFBLK; + + isCharacterDevice = () => + (this.mode & this.constants.S_IFMT) === this.constants.S_IFCHR; + + isFIFO = () => (this.mode & this.constants.S_IFMT) === this.constants.S_IFIFO; + + isSocket = () => + (this.mode & this.constants.S_IFMT) === this.constants.S_IFSOCK; +} + interface Options { service: string; client?: Client; @@ -64,6 +108,9 @@ export async function fsServer({ service, fs, client }: Options) { async chmod(path: string, mode: string | number) { await (await fs(this.subject)).chmod(path, mode); }, + async constants(): Promise<{ [key: string]: number }> { + return await (await fs(this.subject)).constants(); + }, async copyFile(src: string, dest: string) { await (await fs(this.subject)).copyFile(src, dest); }, @@ -71,11 +118,14 @@ export async function fsServer({ service, fs, client }: Options) { await (await fs(this.subject)).cp(src, dest, options); }, async exists(path: string) { - await (await fs(this.subject)).exists(path); + return await (await fs(this.subject)).exists(path); }, async link(existingPath: string, newPath: string) { await (await fs(this.subject)).link(existingPath, newPath); }, + async lstat(path: string): Promise { + return await (await fs(this.subject)).lstat(path); + }, async mkdir(path: string, options?) { await (await fs(this.subject)).mkdir(path, options); }, @@ -89,35 +139,35 @@ export async function fsServer({ service, fs, client }: Options) { return await (await fs(this.subject)).realpath(path); }, async rename(oldPath: string, newPath: string) { - return await (await fs(this.subject)).rename(oldPath, newPath); + await (await fs(this.subject)).rename(oldPath, newPath); }, async rm(path: string, options?) { - return await (await fs(this.subject)).rm(path, options); + await (await fs(this.subject)).rm(path, options); }, async rmdir(path: string, options?) { - return await (await fs(this.subject)).rmdir(path, options); + await (await fs(this.subject)).rmdir(path, options); }, - async stat(path: string): Promise { + async stat(path: string): Promise { return await (await fs(this.subject)).stat(path); }, async symlink(target: string, path: string) { - return await (await fs(this.subject)).symlink(target, path); + await (await fs(this.subject)).symlink(target, path); }, async truncate(path: string, len?: number) { - return await (await fs(this.subject)).truncate(path, len); + await (await fs(this.subject)).truncate(path, len); }, async unlink(path: string) { - return await (await fs(this.subject)).unlink(path); + await (await fs(this.subject)).unlink(path); }, async utimes( path: string, atime: number | string | Date, mtime: number | string | Date, ) { - return await (await fs(this.subject)).utimes(path, atime, mtime); + await (await fs(this.subject)).utimes(path, atime, mtime); }, async writeFile(path: string, data: string | Buffer) { - return await (await fs(this.subject)).writeFile(path, data); + await (await fs(this.subject)).writeFile(path, data); }, }, ); @@ -130,5 +180,30 @@ export function fsClient({ client?: Client; subject: string; }) { - return (client ?? conat()).call(subject); + let call = (client ?? conat()).call(subject); + + let constants: any = null; + const stat0 = call.stat.bind(call); + call.stat = async (path: string) => { + const s = await stat0(path); + constants = constants ?? (await call.constants()); + const stats = new Stats(constants); + for (const k in s) { + stats[k] = s[k]; + } + return stats; + }; + + const lstat0 = call.lstat.bind(call); + call.lstat = async (path: string) => { + const s = await lstat0(path); + constants = constants ?? (await call.constants()); + const stats = new Stats(constants); + for (const k in s) { + stats[k] = s[k]; + } + return stats; + }; + + return call; } diff --git a/src/packages/file-server/conat/test/local-path.test.ts b/src/packages/file-server/conat/test/local-path.test.ts index 682cec7719..875cbc34f0 100644 --- a/src/packages/file-server/conat/test/local-path.test.ts +++ b/src/packages/file-server/conat/test/local-path.test.ts @@ -3,6 +3,7 @@ import { mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "path"; import { fsClient } from "@cocalc/conat/files/fs"; +import { randomId } from "@cocalc/conat/names"; let tempDir; beforeAll(async () => { @@ -10,31 +11,94 @@ beforeAll(async () => { }); describe("use the simple fileserver", () => { - let service; + const service = `fs-${randomId()}`; + let server; it("creates the simple fileserver service", async () => { - service = await localPathFileserver({ service: "fs", path: tempDir }); + server = await localPathFileserver({ service, path: tempDir }); }); const project_id = "6b851643-360e-435e-b87e-f9a6ab64a8b1"; let fs; it("create a client", () => { - fs = fsClient({ subject: `fs.project-${project_id}` }); + fs = fsClient({ subject: `${service}.project-${project_id}` }); }); - it("checks appendFile works", async () => { + it("appendFile works", async () => { + await fs.writeFile("a", ""); await fs.appendFile("a", "foo"); expect(await fs.readFile("a", "utf8")).toEqual("foo"); }); - it("checks chmod works", async () => { + it("chmod works", async () => { await fs.writeFile("b", "hi"); await fs.chmod("b", 0o755); const s = await fs.stat("b"); expect(s.mode.toString(8)).toBe("100755"); }); + it("constants work", async () => { + const constants = await fs.constants(); + expect(constants.O_RDONLY).toBe(0); + expect(constants.O_WRONLY).toBe(1); + expect(constants.O_RDWR).toBe(2); + }); + + it("copyFile works", async () => { + await fs.writeFile("c", "hello"); + await fs.copyFile("c", "d.txt"); + expect(await fs.readFile("d.txt", "utf8")).toEqual("hello"); + }); + + it("cp works on a directory", async () => { + await fs.mkdir("folder"); + await fs.writeFile("folder/a.txt", "hello"); + await fs.cp("folder", "folder2", { recursive: true }); + expect(await fs.readFile("folder2/a.txt", "utf8")).toEqual("hello"); + }); + + it("exists works", async () => { + expect(await fs.exists("does-not-exist")).toBe(false); + await fs.writeFile("does-exist", ""); + expect(await fs.exists("does-exist")).toBe(true); + }); + + it("creating a hard link works", async () => { + await fs.writeFile("source", "the source"); + await fs.link("source", "target"); + expect(await fs.readFile("target", "utf8")).toEqual("the source"); + // hard link, not symlink + expect(await fs.realpath("target")).toBe("target"); + + await fs.appendFile("source", " and more"); + expect(await fs.readFile("target", "utf8")).toEqual("the source and more"); + }); + + it("mkdir works", async () => { + await fs.mkdir("xyz"); + const s = await fs.stat("xyz"); + expect(s.isDirectory()).toBe(true); + }); + + it("creating a symlink works", async () => { + await fs.writeFile("source1", "the source"); + await fs.symlink("source1", "target1"); + expect(await fs.readFile("target1", "utf8")).toEqual("the source"); + // symlink, not hard + expect(await fs.realpath("target1")).toBe("source1"); + await fs.appendFile("source1", " and more"); + expect(await fs.readFile("target1", "utf8")).toEqual("the source and more"); + const stats = await fs.stat("target1"); + expect(stats.isSymbolicLink()).toBe(false); + + const lstats = await fs.lstat("target1"); + expect(lstats.isSymbolicLink()).toBe(true); + + const stats0 = await fs.stat("source1"); + expect(stats0.isSymbolicLink()).toBe(false); + }); + it("closes the service", () => { - service.close(); + server.close(); }); }); diff --git a/src/packages/file-server/fs/sandbox.ts b/src/packages/file-server/fs/sandbox.ts index dabd277ec2..4c5d7de83b 100644 --- a/src/packages/file-server/fs/sandbox.ts +++ b/src/packages/file-server/fs/sandbox.ts @@ -15,8 +15,10 @@ import { appendFile, chmod, cp, + constants, copyFile, link, + lstat, readdir, readFile, realpath, @@ -56,6 +58,10 @@ export class SandboxedFilesystem { await chmod(this.safeAbsPath(path), mode); }; + constants = async (): Promise<{ [key: string]: number }> => { + return constants; + }; + copyFile = async (src: string, dest: string) => { await copyFile(this.safeAbsPath(src), this.safeAbsPath(dest)); }; @@ -86,6 +92,10 @@ export class SandboxedFilesystem { }); }; + lstat = async (path: string) => { + return await lstat(this.safeAbsPath(path)); + }; + mkdir = async (path: string, options?) => { await mkdir(this.safeAbsPath(path), options); }; diff --git a/src/packages/frontend/conat/client.ts b/src/packages/frontend/conat/client.ts index 628522f8d2..bf7af5088d 100644 --- a/src/packages/frontend/conat/client.ts +++ b/src/packages/frontend/conat/client.ts @@ -46,6 +46,7 @@ import { deleteRememberMe, setRememberMe, } from "@cocalc/frontend/misc/remember-me"; +import { fsClient } from "@cocalc/conat/files/fs"; export interface ConatConnectionStatus { state: "connected" | "disconnected"; @@ -515,6 +516,8 @@ export class ConatClient extends EventEmitter { }; refCacheInfo = () => refCacheInfo(); + + fsClient = (subject: string) => fsClient({ subject, client: this.conat() }); } function setDeleted({ project_id, path, deleted }) { From e146dd18579cecbc391f5b1ce128fd0adcadf088 Mon Sep 17 00:00:00 2001 From: William Stein Date: Thu, 17 Jul 2025 18:26:51 +0000 Subject: [PATCH 008/270] btrfs: enable and fix some disabled tests --- src/packages/file-server/btrfs/subvolume-snapshots.ts | 2 +- src/packages/file-server/btrfs/test/subvolume.test.ts | 4 ++-- src/packages/file-server/conat/test/local-path.test.ts | 3 ++- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/packages/file-server/btrfs/subvolume-snapshots.ts b/src/packages/file-server/btrfs/subvolume-snapshots.ts index ffe71fe6fc..9dcd4f30ad 100644 --- a/src/packages/file-server/btrfs/subvolume-snapshots.ts +++ b/src/packages/file-server/btrfs/subvolume-snapshots.ts @@ -27,7 +27,7 @@ export class SubvolumeSnapshots { return; } await this.subvolume.fs.mkdir(SNAPSHOTS); - await this.subvolume.fs.chmod(SNAPSHOTS, "0550"); + await this.subvolume.fs.chmod(SNAPSHOTS, "0700"); }; create = async (name?: string) => { diff --git a/src/packages/file-server/btrfs/test/subvolume.test.ts b/src/packages/file-server/btrfs/test/subvolume.test.ts index e586cc656b..72f72c4dfd 100644 --- a/src/packages/file-server/btrfs/test/subvolume.test.ts +++ b/src/packages/file-server/btrfs/test/subvolume.test.ts @@ -226,7 +226,7 @@ describe("test snapshots", () => { }); }); -describe.only("test bup backups", () => { +describe("test bup backups", () => { let vol: Subvolume; it("creates a volume", async () => { vol = await fs.subvolumes.get("bup-test"); @@ -273,7 +273,7 @@ describe.only("test bup backups", () => { { name: "mydir", size: 0, mtime: x[1].mtime, isdir: true }, ]); expect(Math.abs((x[0].mtime ?? 0) * 1000 - Date.now())).toBeLessThan( - 60_000, + 5 * 60_000, ); }); diff --git a/src/packages/file-server/conat/test/local-path.test.ts b/src/packages/file-server/conat/test/local-path.test.ts index 875cbc34f0..4a66847a60 100644 --- a/src/packages/file-server/conat/test/local-path.test.ts +++ b/src/packages/file-server/conat/test/local-path.test.ts @@ -77,9 +77,10 @@ describe("use the simple fileserver", () => { await fs.mkdir("xyz"); const s = await fs.stat("xyz"); expect(s.isDirectory()).toBe(true); + expect(s.isFile()).toBe(false); }); - it("creating a symlink works", async () => { + it("creating a symlink works (and using lstat)", async () => { await fs.writeFile("source1", "the source"); await fs.symlink("source1", "target1"); expect(await fs.readFile("target1", "utf8")).toEqual("the source"); From a7a0d95c0e960bb0f33928517c1aa2b6a4068c4a Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 18 Jul 2025 14:29:29 +0000 Subject: [PATCH 009/270] fix file-server conat test to work with self-contained testing conat server --- src/packages/file-server/conat/local-path.ts | 6 ++++-- src/packages/file-server/conat/test/local-path.test.ts | 5 ++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/packages/file-server/conat/local-path.ts b/src/packages/file-server/conat/local-path.ts index e8450f3adf..53edc7443b 100644 --- a/src/packages/file-server/conat/local-path.ts +++ b/src/packages/file-server/conat/local-path.ts @@ -1,18 +1,20 @@ import { fsServer } from "@cocalc/conat/files/fs"; -import { conat } from "@cocalc/backend/conat"; import { SandboxedFilesystem } from "@cocalc/file-server/fs/sandbox"; import { mkdir } from "fs/promises"; import { join } from "path"; import { isValidUUID } from "@cocalc/util/misc"; +import { type Client, getClient } from "@cocalc/conat/core/client"; export function localPathFileserver({ service, path, + client, }: { service: string; path: string; + client?: Client; }) { - const client = conat(); + client ??= getClient(); const server = fsServer({ service, client, diff --git a/src/packages/file-server/conat/test/local-path.test.ts b/src/packages/file-server/conat/test/local-path.test.ts index 4a66847a60..0cd4a92718 100644 --- a/src/packages/file-server/conat/test/local-path.test.ts +++ b/src/packages/file-server/conat/test/local-path.test.ts @@ -4,9 +4,11 @@ import { tmpdir } from "node:os"; import { join } from "path"; import { fsClient } from "@cocalc/conat/files/fs"; import { randomId } from "@cocalc/conat/names"; +import { before, after, client } from "@cocalc/backend/conat/test/setup"; let tempDir; beforeAll(async () => { + await before(); tempDir = await mkdtemp(join(tmpdir(), "cocalc-local-path")); }); @@ -14,7 +16,7 @@ describe("use the simple fileserver", () => { const service = `fs-${randomId()}`; let server; it("creates the simple fileserver service", async () => { - server = await localPathFileserver({ service, path: tempDir }); + server = await localPathFileserver({ client, service, path: tempDir }); }); const project_id = "6b851643-360e-435e-b87e-f9a6ab64a8b1"; @@ -104,5 +106,6 @@ describe("use the simple fileserver", () => { }); afterAll(async () => { + await after(); await rm(tempDir, { force: true, recursive: true }); }); From 732ee7289ebda1db64668e3df5ba1101739c1361 Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 18 Jul 2025 16:59:50 +0000 Subject: [PATCH 010/270] btrfs: add sync command in hopes of getting tests to pass on gitub CI --- src/packages/file-server/btrfs/filesystem.ts | 4 ++++ .../file-server/btrfs/test/filesystem-stress.test.ts | 8 ++++++++ src/packages/file-server/conat/test/local-path.test.ts | 2 ++ 3 files changed, 14 insertions(+) diff --git a/src/packages/file-server/btrfs/filesystem.ts b/src/packages/file-server/btrfs/filesystem.ts index 927fb23548..194988e036 100644 --- a/src/packages/file-server/btrfs/filesystem.ts +++ b/src/packages/file-server/btrfs/filesystem.ts @@ -71,6 +71,10 @@ export class Filesystem { await this.initBup(); }; + sync = async () => { + await btrfs({ args: ["filesystem", "sync", this.opts.mount] }); + }; + unmount = async () => { await sudo({ command: "umount", diff --git a/src/packages/file-server/btrfs/test/filesystem-stress.test.ts b/src/packages/file-server/btrfs/test/filesystem-stress.test.ts index a2e1de9531..354f261801 100644 --- a/src/packages/file-server/btrfs/test/filesystem-stress.test.ts +++ b/src/packages/file-server/btrfs/test/filesystem-stress.test.ts @@ -48,6 +48,7 @@ describe("stress operations with subvolumes", () => { }); it("clone the first group in serial", async () => { + await fs.sync(); // needed on github actions const t = Date.now(); for (let i = 0; i < count1; i++) { await fs.subvolumes.clone(`${i}`, `clone-of-${i}`); @@ -58,6 +59,7 @@ describe("stress operations with subvolumes", () => { }); it("clone the second group in parallel", async () => { + await fs.sync(); // needed on github actions const t = Date.now(); const v: any[] = []; for (let i = 0; i < count2; i++) { @@ -90,6 +92,12 @@ describe("stress operations with subvolumes", () => { `deleted ${Math.round((count2 / (Date.now() - t)) * 1000)} subvolumes per second in parallel`, ); }); + + it("everything should be gone except the clones", async () => { + await fs.sync(); + const v = await fs.subvolumes.list(); + expect(v.length).toBe(count1 + count2); + }); }); afterAll(after); diff --git a/src/packages/file-server/conat/test/local-path.test.ts b/src/packages/file-server/conat/test/local-path.test.ts index 0cd4a92718..44b717cca8 100644 --- a/src/packages/file-server/conat/test/local-path.test.ts +++ b/src/packages/file-server/conat/test/local-path.test.ts @@ -99,6 +99,8 @@ describe("use the simple fileserver", () => { const stats0 = await fs.stat("source1"); expect(stats0.isSymbolicLink()).toBe(false); }); + + it("closes the service", () => { server.close(); From 4644d6a308dab8c5519014c8bd1b14fa7e1e892a Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 18 Jul 2025 19:02:41 +0000 Subject: [PATCH 011/270] address obvious symlink issue with fs sandbox; also make tests hopefully work on github --- src/package.json | 2 +- src/packages/file-server/btrfs/filesystem.ts | 1 + .../file-server/btrfs/subvolume-bup.ts | 8 +- src/packages/file-server/btrfs/subvolume.ts | 4 +- .../btrfs/test/filesystem-stress.test.ts | 3 - .../file-server/btrfs/test/subvolume.test.ts | 2 +- .../file-server/conat/test/local-path.test.ts | 90 +++++++++++++- src/packages/file-server/fs/sandbox.ts | 113 ++++++++++++++---- src/packages/file-server/package.json | 1 + src/workspaces.py | 27 +++-- 10 files changed, 200 insertions(+), 51 deletions(-) diff --git a/src/package.json b/src/package.json index efa416dcd0..8f4ce63e09 100644 --- a/src/package.json +++ b/src/package.json @@ -18,7 +18,7 @@ "version-check": "pip3 install typing_extensions mypy || pip3 install --break-system-packages typing_extensions mypy && ./workspaces.py version-check && mypy scripts/check_npm_packages.py", "test-parallel": "unset DEBUG && pnpm run version-check && cd packages && pnpm run -r --parallel test", "test": "unset DEBUG && pnpm run depcheck && pnpm run version-check && ./workspaces.py test", - "test-github-ci": "unset DEBUG && pnpm run depcheck && pnpm run version-check && ./workspaces.py test --exclude=jupyter --retries=1", + "test-github-ci": "unset DEBUG && pnpm run depcheck && pnpm run version-check && ./workspaces.py test --test-github-ci --exclude=jupyter --retries=1", "depcheck": "cd packages && pnpm run -r --parallel depcheck", "prettier-all": "cd packages/", "local-ci": "./scripts/ci.sh", diff --git a/src/packages/file-server/btrfs/filesystem.ts b/src/packages/file-server/btrfs/filesystem.ts index 194988e036..248e086630 100644 --- a/src/packages/file-server/btrfs/filesystem.ts +++ b/src/packages/file-server/btrfs/filesystem.ts @@ -69,6 +69,7 @@ export class Filesystem { args: ["quota", "enable", "--simple", this.opts.mount], }); await this.initBup(); + await this.sync(); }; sync = async () => { diff --git a/src/packages/file-server/btrfs/subvolume-bup.ts b/src/packages/file-server/btrfs/subvolume-bup.ts index 3849b49379..b4d64c9b5a 100644 --- a/src/packages/file-server/btrfs/subvolume-bup.ts +++ b/src/packages/file-server/btrfs/subvolume-bup.ts @@ -46,7 +46,7 @@ export class SubvolumeBup { `createBackup: creating ${BUP_SNAPSHOT} to get a consistent backup`, ); await this.subvolume.snapshots.create(BUP_SNAPSHOT); - const target = this.subvolume.fs.safeAbsPath( + const target = await this.subvolume.fs.safeAbsPath( this.subvolume.snapshots.path(BUP_SNAPSHOT), ); @@ -133,9 +133,9 @@ export class SubvolumeBup { return v; } - path = this.subvolume.fs - .safeAbsPath(path) - .slice(this.subvolume.path.length); + path = (await this.subvolume.fs.safeAbsPath(path)).slice( + this.subvolume.path.length, + ); const { stdout } = await sudo({ command: "bup", args: [ diff --git a/src/packages/file-server/btrfs/subvolume.ts b/src/packages/file-server/btrfs/subvolume.ts index 72c3dd7d54..7eb7808518 100644 --- a/src/packages/file-server/btrfs/subvolume.ts +++ b/src/packages/file-server/btrfs/subvolume.ts @@ -86,8 +86,8 @@ export class Subvolume { target: string; timeout?: number; }): Promise<{ stdout: string; stderr: string; exit_code: number }> => { - let srcPath = this.fs.safeAbsPath(src); - let targetPath = this.fs.safeAbsPath(target); + let srcPath = await this.fs.safeAbsPath(src); + let targetPath = await this.fs.safeAbsPath(target); if (src.endsWith("/")) { srcPath += "/"; } diff --git a/src/packages/file-server/btrfs/test/filesystem-stress.test.ts b/src/packages/file-server/btrfs/test/filesystem-stress.test.ts index 354f261801..2302785d14 100644 --- a/src/packages/file-server/btrfs/test/filesystem-stress.test.ts +++ b/src/packages/file-server/btrfs/test/filesystem-stress.test.ts @@ -48,7 +48,6 @@ describe("stress operations with subvolumes", () => { }); it("clone the first group in serial", async () => { - await fs.sync(); // needed on github actions const t = Date.now(); for (let i = 0; i < count1; i++) { await fs.subvolumes.clone(`${i}`, `clone-of-${i}`); @@ -59,7 +58,6 @@ describe("stress operations with subvolumes", () => { }); it("clone the second group in parallel", async () => { - await fs.sync(); // needed on github actions const t = Date.now(); const v: any[] = []; for (let i = 0; i < count2; i++) { @@ -94,7 +92,6 @@ describe("stress operations with subvolumes", () => { }); it("everything should be gone except the clones", async () => { - await fs.sync(); const v = await fs.subvolumes.list(); expect(v.length).toBe(count1 + count2); }); diff --git a/src/packages/file-server/btrfs/test/subvolume.test.ts b/src/packages/file-server/btrfs/test/subvolume.test.ts index 72f72c4dfd..d7e4bdface 100644 --- a/src/packages/file-server/btrfs/test/subvolume.test.ts +++ b/src/packages/file-server/btrfs/test/subvolume.test.ts @@ -143,7 +143,7 @@ describe("the filesystem operations", () => { await vol.fs.writeFile("w.txt", "hi"); const ac = new AbortController(); const { signal } = ac; - const watcher = vol.fs.watch("w.txt", { signal }); + const watcher = await vol.fs.watch("w.txt", { signal }); vol.fs.appendFile("w.txt", " there"); // @ts-ignore const { value, done } = await watcher.next(); diff --git a/src/packages/file-server/conat/test/local-path.test.ts b/src/packages/file-server/conat/test/local-path.test.ts index 44b717cca8..a1a9013a4c 100644 --- a/src/packages/file-server/conat/test/local-path.test.ts +++ b/src/packages/file-server/conat/test/local-path.test.ts @@ -1,25 +1,28 @@ import { localPathFileserver } from "../local-path"; -import { mkdtemp, rm } from "node:fs/promises"; +import { mkdtemp, readFile, rm, symlink } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "path"; import { fsClient } from "@cocalc/conat/files/fs"; import { randomId } from "@cocalc/conat/names"; import { before, after, client } from "@cocalc/backend/conat/test/setup"; +import { uuid } from "@cocalc/util/misc"; let tempDir; +let tempDir2; beforeAll(async () => { await before(); tempDir = await mkdtemp(join(tmpdir(), "cocalc-local-path")); + tempDir2 = await mkdtemp(join(tmpdir(), "cocalc-local-path-2")); }); -describe("use the simple fileserver", () => { +describe("use all the standard api functions of fs", () => { const service = `fs-${randomId()}`; let server; it("creates the simple fileserver service", async () => { server = await localPathFileserver({ client, service, path: tempDir }); }); - const project_id = "6b851643-360e-435e-b87e-f9a6ab64a8b1"; + const project_id = uuid(); let fs; it("create a client", () => { fs = fsClient({ subject: `${service}.project-${project_id}` }); @@ -82,6 +85,27 @@ describe("use the simple fileserver", () => { expect(s.isFile()).toBe(false); }); + it("readFile works", async () => { + await fs.writeFile("a", Buffer.from([1, 2, 3])); + const s = await fs.readFile("a"); + expect(s).toEqual(Buffer.from([1, 2, 3])); + + await fs.writeFile("b.txt", "conat"); + const t = await fs.readFile("b.txt", "utf8"); + expect(t).toEqual("conat"); + }); + + it("readdir works", async () => { + await fs.mkdir("dirtest"); + for (let i = 0; i < 5; i++) { + await fs.writeFile(`dirtest/${i}`, `${i}`); + } + const fire = "🔥.txt"; + await fs.writeFile(join("dirtest", fire), "this is ️‍🔥!"); + const v = await fs.readdir("dirtest"); + expect(v).toEqual(["0", "1", "2", "3", "4", fire]); + }); + it("creating a symlink works (and using lstat)", async () => { await fs.writeFile("source1", "the source"); await fs.symlink("source1", "target1"); @@ -99,15 +123,71 @@ describe("use the simple fileserver", () => { const stats0 = await fs.stat("source1"); expect(stats0.isSymbolicLink()).toBe(false); }); - - it("closes the service", () => { server.close(); }); }); +describe("security: dangerous symlinks can't be followed", () => { + const service = `fs-${randomId()}`; + let server; + it("creates the simple fileserver service", async () => { + server = await localPathFileserver({ client, service, path: tempDir2 }); + }); + + const project_id = uuid(); + const project_id2 = uuid(); + let fs, fs2; + it("create two clients", () => { + fs = fsClient({ subject: `${service}.project-${project_id}` }); + fs2 = fsClient({ subject: `${service}.project-${project_id2}` }); + }); + + it("create a secret in one", async () => { + await fs.writeFile("password", "s3cr3t"); + await fs2.writeFile("a", "init"); + }); + + // This is setup bypassing security and is part of our threat model, due to users + // having full access internally to their sandbox fs. + it("directly create a file that is a symlink outside of the sandbox -- this should work", async () => { + await symlink( + join(tempDir2, project_id, "password"), + join(tempDir2, project_id2, "link"), + ); + const s = await readFile(join(tempDir2, project_id2, "link"), "utf8"); + expect(s).toBe("s3cr3t"); + }); + + it("fails to read the symlink content via the api", async () => { + await expect(async () => { + await fs2.readFile("link", "utf8"); + }).rejects.toThrow("outside of sandbox"); + }); + + it("directly create a relative symlink ", async () => { + await symlink( + join("..", project_id, "password"), + join(tempDir2, project_id2, "link2"), + ); + const s = await readFile(join(tempDir2, project_id2, "link2"), "utf8"); + expect(s).toBe("s3cr3t"); + }); + + it("fails to read the relative symlink content via the api", async () => { + await expect(async () => { + await fs2.readFile("link2", "utf8"); + }).rejects.toThrow("outside of sandbox"); + }); + + it("closes the server", () => { + server.close(); + }); +}); + afterAll(async () => { await after(); await rm(tempDir, { force: true, recursive: true }); + // await rm(tempDir2, { force: true, recursive: true }); }); diff --git a/src/packages/file-server/fs/sandbox.ts b/src/packages/file-server/fs/sandbox.ts index 4c5d7de83b..f3795937c0 100644 --- a/src/packages/file-server/fs/sandbox.ts +++ b/src/packages/file-server/fs/sandbox.ts @@ -9,6 +9,36 @@ Absolute and relative paths are considered as relative to the input folder path. REFERENCE: We don't use https://github.com/metarhia/sandboxed-fs, but did look at the code. + + + +SECURITY: + +The following could be a big problem -- user somehow create or change path to +be a dangerous symlink *after* the realpath check below, but before we do an fs *read* +operation. If they did that, then we would end up reading the target of the +symlink. I.e., if they could somehow create the file *as an unsafe symlink* +right after we confirm that it does not exist and before we read from it. This +would only happen via something not involving this sandbox, e.g., the filesystem +mounted into a container some other way. + +In short, I'm worried about: + +1. Request to read a file named "link" which is just a normal file. We confirm this using realpath + in safeAbsPath. +2. Somehow delete "link" and replace it by a new file that is a symlink to "../{project_id}/.ssh/id_ed25519" +3. Read the file "link" and get the contents of "../{project_id}/.ssh/id_ed25519". + +The problem is that 1 and 3 happen microseconds apart as separate calls to the filesystem. + +**[ ] TODO -- NOT IMPLEMENTED YET: This is why we have to uses file descriptors!** + +1. User requests to read a file named "link" which is just a normal file. +2. We wet file descriptor fd for whatever "link" is. Then confirm this is OK using realpath in safeAbsPath. +3. user somehow deletes "link" and replace it by a new file that is a symlink to "../{project_id}/.ssh/id_ed25519" +4. We read from the file descriptor fd and get the contents of original "link" (or error). + + */ import { @@ -43,19 +73,40 @@ export class SandboxedFilesystem { // path should be the path to a FOLDER on the filesystem (not a file) constructor(public readonly path: string) {} - safeAbsPath = (path: string) => { + safeAbsPath = async (path: string): Promise => { if (typeof path != "string") { throw Error(`path must be a string but is of type ${typeof path}`); } - return join(this.path, resolve("/", path)); + // pathInSandbox is *definitely* a path in the sandbox: + const pathInSandbox = join(this.path, resolve("/", path)); + // However, there is still one threat, which is that it could + // be a path to an existing link that goes out of the sandbox. So + // we resolve to the realpath: + try { + const p = await realpath(pathInSandbox); + if (p != this.path && !p.startsWith(this.path + "/")) { + throw Error( + `realpath of '${path}' resolves to a path outside of sandbox`, + ); + } + // don't return the result of calling realpath -- what's important is + // their path's realpath is in the sandbox. + return pathInSandbox; + } catch (err) { + if (err.code == "ENOENT") { + return pathInSandbox; + } else { + throw err; + } + } }; appendFile = async (path: string, data: string | Buffer, encoding?) => { - return await appendFile(this.safeAbsPath(path), data, encoding); + return await appendFile(await this.safeAbsPath(path), data, encoding); }; chmod = async (path: string, mode: string | number) => { - await chmod(this.safeAbsPath(path), mode); + await chmod(await this.safeAbsPath(path), mode); }; constants = async (): Promise<{ [key: string]: number }> => { @@ -63,22 +114,26 @@ export class SandboxedFilesystem { }; copyFile = async (src: string, dest: string) => { - await copyFile(this.safeAbsPath(src), this.safeAbsPath(dest)); + await copyFile(await this.safeAbsPath(src), await this.safeAbsPath(dest)); }; cp = async (src: string, dest: string, options?) => { - await cp(this.safeAbsPath(src), this.safeAbsPath(dest), options); + await cp( + await this.safeAbsPath(src), + await this.safeAbsPath(dest), + options, + ); }; exists = async (path: string) => { - return await exists(this.safeAbsPath(path)); + return await exists(await this.safeAbsPath(path)); }; // hard link link = async (existingPath: string, newPath: string) => { return await link( - this.safeAbsPath(existingPath), - this.safeAbsPath(newPath), + await this.safeAbsPath(existingPath), + await this.safeAbsPath(newPath), ); }; @@ -86,59 +141,65 @@ export class SandboxedFilesystem { path: string, { hidden, limit }: { hidden?: boolean; limit?: number } = {}, ): Promise => { - return await getListing(this.safeAbsPath(path), hidden, { + return await getListing(await this.safeAbsPath(path), hidden, { limit, home: "/", }); }; lstat = async (path: string) => { - return await lstat(this.safeAbsPath(path)); + return await lstat(await this.safeAbsPath(path)); }; mkdir = async (path: string, options?) => { - await mkdir(this.safeAbsPath(path), options); + await mkdir(await this.safeAbsPath(path), options); }; readFile = async (path: string, encoding?: any): Promise => { - return await readFile(this.safeAbsPath(path), encoding); + return await readFile(await this.safeAbsPath(path), encoding); }; readdir = async (path: string): Promise => { - return await readdir(this.safeAbsPath(path)); + return await readdir(await this.safeAbsPath(path)); }; realpath = async (path: string): Promise => { - const x = await realpath(this.safeAbsPath(path)); + const x = await realpath(await this.safeAbsPath(path)); return x.slice(this.path.length + 1); }; rename = async (oldPath: string, newPath: string) => { - await rename(this.safeAbsPath(oldPath), this.safeAbsPath(newPath)); + await rename( + await this.safeAbsPath(oldPath), + await this.safeAbsPath(newPath), + ); }; rm = async (path: string, options?) => { - await rm(this.safeAbsPath(path), options); + await rm(await this.safeAbsPath(path), options); }; rmdir = async (path: string, options?) => { - await rmdir(this.safeAbsPath(path), options); + await rmdir(await this.safeAbsPath(path), options); }; stat = async (path: string) => { - return await stat(this.safeAbsPath(path)); + return await stat(await this.safeAbsPath(path)); }; symlink = async (target: string, path: string) => { - return await symlink(this.safeAbsPath(target), this.safeAbsPath(path)); + return await symlink( + await this.safeAbsPath(target), + await this.safeAbsPath(path), + ); }; truncate = async (path: string, len?: number) => { - await truncate(this.safeAbsPath(path), len); + await truncate(await this.safeAbsPath(path), len); }; unlink = async (path: string) => { - await unlink(this.safeAbsPath(path)); + await unlink(await this.safeAbsPath(path)); }; utimes = async ( @@ -146,14 +207,14 @@ export class SandboxedFilesystem { atime: number | string | Date, mtime: number | string | Date, ) => { - await utimes(this.safeAbsPath(path), atime, mtime); + await utimes(await this.safeAbsPath(path), atime, mtime); }; - watch = (filename: string, options?) => { - return watch(this.safeAbsPath(filename), options); + watch = async (filename: string, options?) => { + return watch(await this.safeAbsPath(filename), options); }; writeFile = async (path: string, data: string | Buffer) => { - return await writeFile(this.safeAbsPath(path), data); + return await writeFile(await this.safeAbsPath(path), data); }; } diff --git a/src/packages/file-server/package.json b/src/packages/file-server/package.json index 55d9d9980a..e12191d181 100644 --- a/src/packages/file-server/package.json +++ b/src/packages/file-server/package.json @@ -13,6 +13,7 @@ "build": "pnpm exec tsc --build", "tsc": "pnpm exec tsc --watch --pretty --preserveWatchOutput", "test": "pnpm exec jest --forceExit", + "test-github-ci": "pnpm exec jest --maxWorkers=1 --forceExit", "depcheck": "pnpx depcheck", "clean": "rm -rf node_modules dist" }, diff --git a/src/workspaces.py b/src/workspaces.py index da60e90a8d..57574af6d5 100755 --- a/src/workspaces.py +++ b/src/workspaces.py @@ -284,11 +284,7 @@ def test(args) -> None: success = [] def status(): - print("Status: ", { - "flaky": flaky, - "fails": fails, - "success": success - }) + print("Status: ", {"flaky": flaky, "fails": fails, "success": success}) v = packages(args) v.sort() @@ -307,11 +303,15 @@ def f(): print(f"TESTING {n}/{len(v)}: {path}") print("*") print("*" * 40) - cmd("pnpm run --if-present test", package_path) + if args.test_github_ci and 'test-github-ci' in open( + os.path.join(package_path, 'package.json')).read(): + cmd("pnpm run test-github-ci", package_path) + else: + cmd("pnpm run --if-present test", package_path) success.append(path) worked = False - for i in range(args.retries+1): + for i in range(args.retries + 1): try: f() worked = True @@ -325,7 +325,9 @@ def f(): flaky.append(path) print(f"ERROR testing {path}") if args.retries - i >= 1: - print(f"Trying {path} again at most {args.retries - i} more times") + print( + f"Trying {path} again at most {args.retries - i} more times" + ) if not worked: fails.append(path) @@ -577,7 +579,14 @@ def packages_arg(parser): "--retries", type=int, default=2, - help="how many times to retry a failed test suite before giving up; set to 0 to NOT retry") + help= + "how many times to retry a failed test suite before giving up; set to 0 to NOT retry" + ) + subparser.add_argument( + '--test-github-ci', + const=True, + action="store_const", + help="run 'pnpm test-github-ci' if available instead of 'pnpm test'") packages_arg(subparser) subparser.set_defaults(func=test) From 02cc5f2214b17e037410dcb5d10a0c1cdc14d21b Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 18 Jul 2025 22:44:51 +0000 Subject: [PATCH 012/270] finish testing conat fs interface and fix some subtle issues with stat found when testing --- src/packages/conat/files/fs.ts | 11 +- .../file-server/btrfs/test/subvolume.test.ts | 2 + .../file-server/conat/test/local-path.test.ts | 163 ++++++++++++++++-- 3 files changed, 164 insertions(+), 12 deletions(-) diff --git a/src/packages/conat/files/fs.ts b/src/packages/conat/files/fs.ts index e33d745dbe..506f3823b5 100644 --- a/src/packages/conat/files/fs.ts +++ b/src/packages/conat/files/fs.ts @@ -148,7 +148,16 @@ export async function fsServer({ service, fs, client }: Options) { await (await fs(this.subject)).rmdir(path, options); }, async stat(path: string): Promise { - return await (await fs(this.subject)).stat(path); + const s = await (await fs(this.subject)).stat(path); + return { + ...s, + // for some reason these times get corrupted on transport from the nodejs datastructure, + // so we make them standard Date objects. + atime: new Date(s.atime), + mtime: new Date(s.mtime), + ctime: new Date(s.ctime), + birthtime: new Date(s.birthtime), + }; }, async symlink(target: string, path: string) { await (await fs(this.subject)).symlink(target, path); diff --git a/src/packages/file-server/btrfs/test/subvolume.test.ts b/src/packages/file-server/btrfs/test/subvolume.test.ts index d7e4bdface..ecc5ed9756 100644 --- a/src/packages/file-server/btrfs/test/subvolume.test.ts +++ b/src/packages/file-server/btrfs/test/subvolume.test.ts @@ -126,6 +126,7 @@ describe("the filesystem operations", () => { it("make a file readonly, then change it back", async () => { await vol.fs.writeFile("c.txt", "hi"); await vol.fs.chmod("c.txt", "440"); + await fs.sync(); expect(async () => { await vol.fs.appendFile("c.txt", " there"); }).rejects.toThrow("EACCES"); @@ -219,6 +220,7 @@ describe("test snapshots", () => { }); it("unlock our snapshot and delete it", async () => { + await fs.sync(); await vol.snapshots.unlock("snap1"); await vol.snapshots.delete("snap1"); expect(await vol.snapshots.exists("snap1")).toBe(false); diff --git a/src/packages/file-server/conat/test/local-path.test.ts b/src/packages/file-server/conat/test/local-path.test.ts index a1a9013a4c..d53f72c0f6 100644 --- a/src/packages/file-server/conat/test/local-path.test.ts +++ b/src/packages/file-server/conat/test/local-path.test.ts @@ -1,5 +1,5 @@ import { localPathFileserver } from "../local-path"; -import { mkdtemp, readFile, rm, symlink } from "node:fs/promises"; +import { link, mkdtemp, readFile, rm, symlink } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "path"; import { fsClient } from "@cocalc/conat/files/fs"; @@ -106,7 +106,131 @@ describe("use all the standard api functions of fs", () => { expect(v).toEqual(["0", "1", "2", "3", "4", fire]); }); - it("creating a symlink works (and using lstat)", async () => { + it("realpath works", async () => { + await fs.writeFile("file0", "file0"); + await fs.symlink("file0", "file1"); + expect(await fs.readFile("file1", "utf8")).toBe("file0"); + const r = await fs.realpath("file1"); + expect(r).toBe("file0"); + + await fs.writeFile("file2", "file2"); + await fs.link("file2", "file3"); + expect(await fs.readFile("file3", "utf8")).toBe("file2"); + const r3 = await fs.realpath("file3"); + expect(r3).toBe("file3"); + }); + + it("rename a file", async () => { + await fs.writeFile("bella", "poo"); + await fs.rename("bella", "bells"); + expect(await fs.readFile("bells", "utf8")).toBe("poo"); + await fs.mkdir("x"); + await fs.rename("bells", "x/belltown"); + }); + + it("rm a file", async () => { + await fs.writeFile("bella-to-rm", "poo"); + await fs.rm("bella-to-rm"); + expect(await fs.exists("bella-to-rm")).toBe(false); + }); + + it("rm a directory", async () => { + await fs.mkdir("rm-dir"); + expect(async () => { + await fs.rm("rm-dir"); + }).rejects.toThrow("Path is a directory"); + await fs.rm("rm-dir", { recursive: true }); + expect(await fs.exists("rm-dir")).toBe(false); + }); + + it("rm a nonempty directory", async () => { + await fs.mkdir("rm-dir2"); + await fs.writeFile("rm-dir2/a", "a"); + await fs.rm("rm-dir2", { recursive: true }); + expect(await fs.exists("rm-dir2")).toBe(false); + }); + + it("rmdir empty directory", async () => { + await fs.mkdir("rm-dir3"); + await fs.rmdir("rm-dir3"); + expect(await fs.exists("rm-dir3")).toBe(false); + }); + + it("stat not existing path", async () => { + expect(async () => { + await fs.stat(randomId()); + }).rejects.toThrow("no such file or directory"); + }); + + it("stat a file", async () => { + await fs.writeFile("abc.txt", "hi"); + const stat = await fs.stat("abc.txt"); + expect(stat.size).toBe(2); + expect(stat.isFile()).toBe(true); + expect(stat.isSymbolicLink()).toBe(false); + expect(stat.isDirectory()).toBe(false); + expect(stat.isBlockDevice()).toBe(false); + expect(stat.isCharacterDevice()).toBe(false); + expect(stat.isSymbolicLink()).toBe(false); + expect(stat.isFIFO()).toBe(false); + expect(stat.isSocket()).toBe(false); + }); + + it("stat a directory", async () => { + await fs.mkdir("my-stat-dir"); + const stat = await fs.stat("my-stat-dir"); + expect(stat.isFile()).toBe(false); + expect(stat.isSymbolicLink()).toBe(false); + expect(stat.isDirectory()).toBe(true); + expect(stat.isBlockDevice()).toBe(false); + expect(stat.isCharacterDevice()).toBe(false); + expect(stat.isSymbolicLink()).toBe(false); + expect(stat.isFIFO()).toBe(false); + expect(stat.isSocket()).toBe(false); + }); + + it("stat a symlink", async () => { + await fs.writeFile("sl2", "the source"); + await fs.symlink("sl2", "target-sl2"); + const stat = await fs.stat("target-sl2"); + // this is how stat works! + expect(stat.isFile()).toBe(true); + expect(stat.isSymbolicLink()).toBe(false); + // so use lstat + const lstat = await fs.lstat("target-sl2"); + expect(lstat.isFile()).toBe(false); + expect(lstat.isSymbolicLink()).toBe(true); + }); + + it("truncate a file", async () => { + await fs.writeFile("t", ""); + await fs.truncate("t", 10); + const s = await fs.stat("t"); + expect(s.size).toBe(10); + }); + + it("delete a file with unlink", async () => { + await fs.writeFile("to-unlink", ""); + await fs.unlink("to-unlink"); + expect(await fs.exists("to-unlink")).toBe(false); + }); + + it("sets times of a file", async () => { + await fs.writeFile("my-times", ""); + const statsBefore = await fs.stat("my-times"); + const atime = Date.now() - 100_000; + const mtime = Date.now() - 10_000_000; + // NOTE: fs.utimes in nodejs takes *seconds*, not ms, hence + // dividing by 1000 here: + await fs.utimes("my-times", atime / 1000, mtime / 1000); + const s = await fs.stat("my-times"); + expect(s.atimeMs).toBeCloseTo(atime); + expect(s.mtimeMs).toBeCloseTo(mtime); + expect(s.atime.valueOf()).toBeCloseTo(atime); + expect(s.mtime.valueOf()).toBeCloseTo(mtime); + }); + + it("creating a symlink works (as does using lstat)", async () => { await fs.writeFile("source1", "the source"); await fs.symlink("source1", "target1"); expect(await fs.readFile("target1", "utf8")).toEqual("the source"); @@ -151,36 +275,53 @@ describe("security: dangerous symlinks can't be followed", () => { // This is setup bypassing security and is part of our threat model, due to users // having full access internally to their sandbox fs. - it("directly create a file that is a symlink outside of the sandbox -- this should work", async () => { + it("directly create a dangerous file that is a symlink outside of the sandbox -- this should work", async () => { await symlink( join(tempDir2, project_id, "password"), - join(tempDir2, project_id2, "link"), + join(tempDir2, project_id2, "danger"), ); - const s = await readFile(join(tempDir2, project_id2, "link"), "utf8"); + const s = await readFile(join(tempDir2, project_id2, "danger"), "utf8"); expect(s).toBe("s3cr3t"); }); it("fails to read the symlink content via the api", async () => { await expect(async () => { - await fs2.readFile("link", "utf8"); + await fs2.readFile("danger", "utf8"); }).rejects.toThrow("outside of sandbox"); }); - it("directly create a relative symlink ", async () => { + it("directly create a dangerous relative symlink ", async () => { await symlink( join("..", project_id, "password"), - join(tempDir2, project_id2, "link2"), + join(tempDir2, project_id2, "danger2"), ); - const s = await readFile(join(tempDir2, project_id2, "link2"), "utf8"); + const s = await readFile(join(tempDir2, project_id2, "danger2"), "utf8"); expect(s).toBe("s3cr3t"); }); it("fails to read the relative symlink content via the api", async () => { await expect(async () => { - await fs2.readFile("link2", "utf8"); + await fs2.readFile("danger2", "utf8"); }).rejects.toThrow("outside of sandbox"); }); + // This is not a vulnerability, because there's no way for the user + // to create a hard link like this from within an nfs mount (say) + // of their own folder. + it("directly create a hard link", async () => { + await link( + join(tempDir2, project_id, "password"), + join(tempDir2, project_id2, "danger3"), + ); + const s = await readFile(join(tempDir2, project_id2, "danger3"), "utf8"); + expect(s).toBe("s3cr3t"); + }); + + it("a hardlink *can* get outside the sandbox", async () => { + const s = await fs2.readFile("danger3", "utf8"); + expect(s).toBe("s3cr3t"); + }); + it("closes the server", () => { server.close(); }); @@ -189,5 +330,5 @@ describe("security: dangerous symlinks can't be followed", () => { afterAll(async () => { await after(); await rm(tempDir, { force: true, recursive: true }); - // await rm(tempDir2, { force: true, recursive: true }); + await rm(tempDir2, { force: true, recursive: true }); }); From 5901190a18af1dd7615d7ea0560873e97aeb4def Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 18 Jul 2025 23:39:05 +0000 Subject: [PATCH 013/270] fix a typescript error --- src/packages/file-server/conat/test/local-path.test.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/packages/file-server/conat/test/local-path.test.ts b/src/packages/file-server/conat/test/local-path.test.ts index d53f72c0f6..8ccb4b293b 100644 --- a/src/packages/file-server/conat/test/local-path.test.ts +++ b/src/packages/file-server/conat/test/local-path.test.ts @@ -217,7 +217,6 @@ describe("use all the standard api functions of fs", () => { it("sets times of a file", async () => { await fs.writeFile("my-times", ""); - const statsBefore = await fs.stat("my-times"); const atime = Date.now() - 100_000; const mtime = Date.now() - 10_000_000; // NOTE: fs.utimes in nodejs takes *seconds*, not ms, hence From 11a21fa9bc60a4bfb94b22d8dbe3fa392292c17d Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 19 Jul 2025 00:21:39 +0000 Subject: [PATCH 014/270] tiny steps to make sync-doc more flexible in various ways --- src/packages/backend/conat/sync-doc/client.ts | 185 ++++++++++++++++++ .../backend/conat/sync-doc/syncstring.ts | 21 ++ .../conat/sync-doc/test/syncstring.test.ts | 35 ++++ src/packages/backend/package.json | 2 + src/packages/pnpm-lock.yaml | 3 + src/packages/sync/client/sync-client.ts | 3 + src/packages/sync/editor/generic/sync-doc.ts | 8 + 7 files changed, 257 insertions(+) create mode 100644 src/packages/backend/conat/sync-doc/client.ts create mode 100644 src/packages/backend/conat/sync-doc/syncstring.ts create mode 100644 src/packages/backend/conat/sync-doc/test/syncstring.test.ts diff --git a/src/packages/backend/conat/sync-doc/client.ts b/src/packages/backend/conat/sync-doc/client.ts new file mode 100644 index 0000000000..bf6742a550 --- /dev/null +++ b/src/packages/backend/conat/sync-doc/client.ts @@ -0,0 +1,185 @@ +/* + * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. + * License: MS-RSL – see LICENSE.md for details + */ + +import { EventEmitter } from "events"; +import { bind_methods, keys } from "@cocalc/util/misc"; +import { + Client as Client0, + FileWatcher as FileWatcher0, +} from "@cocalc/sync/editor/generic/types"; +import { SyncTable } from "@cocalc/sync/table/synctable"; +import { ExecuteCodeOptionsWithCallback } from "@cocalc/util/types/execute-code"; +import { once } from "@cocalc/util/async-utils"; + +export class FileWatcher extends EventEmitter implements FileWatcher0 { + private path: string; + constructor(path: string) { + super(); + this.path = path; + console.log("FileWatcher", this.path); + } + public close(): void {} +} + +export class Client extends EventEmitter implements Client0 { + private _client_id: string; + private initial_get_query: { [table: string]: any[] }; + public set_queries: any[] = []; + + constructor( + initial_get_query: { [table: string]: any[] }, + client_id: string, + ) { + super(); + this._client_id = client_id; + this.initial_get_query = initial_get_query; + bind_methods(this, ["query", "dbg", "query_cancel"]); + } + + public server_time(): Date { + return new Date(); + } + + isTestClient = () => { + return true; + }; + + public is_project(): boolean { + return false; + } + + public is_browser(): boolean { + return true; + } + + public is_compute_server(): boolean { + return false; + } + + public dbg(_f: string): Function { + // return (...args) => { + // console.log(_f, ...args); + // }; + return (..._) => {}; + } + + public mark_file(_opts: { + project_id: string; + path: string; + action: string; + ttl: number; + }): void { + //console.log("mark_file", opts); + } + + public log_error(opts: { + project_id: string; + path: string; + string_id: string; + error: any; + }): void { + console.log("log_error", opts); + } + + public query(opts): void { + if (opts.options && opts.options.length === 1 && opts.options[0].set) { + // set query + this.set_queries.push(opts); + opts.cb(); + } else { + // get query -- returns predetermined result + const table = keys(opts.query)[0]; + let result = this.initial_get_query[table]; + if (result == null) { + result = []; + } + //console.log("GET QUERY ", table, result); + opts.cb(undefined, { query: { [table]: result } }); + } + } + + path_access(opts: { path: string; mode: string; cb: Function }): void { + console.log("path_access", opts.path, opts.mode); + opts.cb(true); + } + path_exists(opts: { path: string; cb: Function }): void { + console.log("path_access", opts.path); + opts.cb(true); + } + path_stat(opts: { path: string; cb: Function }): void { + console.log("path_state", opts.path); + opts.cb(true); + } + async path_read(opts: { + path: string; + maxsize_MB?: number; + cb: Function; + }): Promise { + console.log("path_ready", opts.path); + opts.cb(true); + } + async write_file(opts: { + path: string; + data: string; + cb: Function; + }): Promise { + console.log("write_file", opts.path, opts.data); + opts.cb(true); + } + watch_file(opts: { path: string }): FileWatcher { + return new FileWatcher(opts.path); + } + + public is_connected(): boolean { + return true; + } + + public is_signed_in(): boolean { + return true; + } + + public touch_project(_): void {} + + public query_cancel(_): void {} + + public alert_message(_): void {} + + public is_deleted(_filename: string, _project_id?: string): boolean { + return false; + } + + public set_deleted(_filename: string, _project_id?: string): void {} + + async synctable_ephemeral( + _project_id: string, + query: any, + options: any, + throttle_changes?: number, + ): Promise { + const s = new SyncTable(query, options, this, throttle_changes); + await once(s, "connected"); + return s; + } + + async synctable_conat(_query: any): Promise { + throw Error("synctable_conat: not implemented"); + } + async pubsub_conat(_query: any): Promise { + throw Error("pubsub_conat: not implemented"); + } + + // account_id or project_id + public client_id(): string { + return this._client_id; + } + + public sage_session({ path }): void { + console.log(`sage_session: path=${path}`); + } + + public shell(opts: ExecuteCodeOptionsWithCallback): void { + console.log(`shell: opts=${JSON.stringify(opts)}`); + } +} diff --git a/src/packages/backend/conat/sync-doc/syncstring.ts b/src/packages/backend/conat/sync-doc/syncstring.ts new file mode 100644 index 0000000000..f0454b7fdf --- /dev/null +++ b/src/packages/backend/conat/sync-doc/syncstring.ts @@ -0,0 +1,21 @@ +import { Client } from "./client"; +import { SyncString } from "@cocalc/sync/editor/string/sync"; +import { a_txt } from "@cocalc/sync/editor/string/test/data"; +import { once } from "@cocalc/util/async-utils"; + +export default async function ephemeralSyncstring() { + const { client_id, project_id, path, init_queries } = a_txt(); + const client = new Client(init_queries, client_id); + const syncstring = new SyncString({ + project_id, + path, + client, + ephemeral: true, + }); + // replace save to disk, since otherwise unless string is empty, + // this will hang forever... and it is called on close. + // @ts-ignore + syncstring.save_to_disk = async () => Promise; + await once(syncstring, "ready"); + return syncstring; +} diff --git a/src/packages/backend/conat/sync-doc/test/syncstring.test.ts b/src/packages/backend/conat/sync-doc/test/syncstring.test.ts new file mode 100644 index 0000000000..62b9875bb5 --- /dev/null +++ b/src/packages/backend/conat/sync-doc/test/syncstring.test.ts @@ -0,0 +1,35 @@ +import syncstring from "@cocalc/backend/conat/sync-doc/syncstring"; + +describe("basic tests of a syncstring", () => { + let s; + + it("creates a syncstring", async () => { + s = await syncstring(); + }); + + it("initially it is empty", () => { + expect(s.to_str()).toBe(""); + expect(s.versions().length).toBe(0); + }); + + it("set the value", () => { + s.from_str("test"); + expect(s.to_str()).toBe("test"); + expect(s.versions().length).toBe(0); + }); + + it("commit the value", () => { + s.commit(); + expect(s.versions().length).toBe(1); + }); + + it("change the value and commit a second time", () => { + s.from_str("bar"); + s.commit(); + expect(s.versions().length).toBe(2); + }); + + it("get first version", () => { + expect(s.version(s.versions()[0]).to_str()).toBe("test"); + }); +}); diff --git a/src/packages/backend/package.json b/src/packages/backend/package.json index ebcc660430..f190097f05 100644 --- a/src/packages/backend/package.json +++ b/src/packages/backend/package.json @@ -6,6 +6,7 @@ "./*": "./dist/*.js", "./database": "./dist/database/index.js", "./conat": "./dist/conat/index.js", + "./conat/sync/*": "./dist/conat/sync/*.js", "./server-settings": "./dist/server-settings/index.js", "./auth/*": "./dist/auth/*.js", "./auth/tokens/*": "./dist/auth/tokens/*.js" @@ -34,6 +35,7 @@ "dependencies": { "@cocalc/backend": "workspace:*", "@cocalc/conat": "workspace:*", + "@cocalc/sync": "workspace:*", "@cocalc/util": "workspace:*", "@types/debug": "^4.1.12", "@types/jest": "^29.5.14", diff --git a/src/packages/pnpm-lock.yaml b/src/packages/pnpm-lock.yaml index ac022d9c8f..930789121f 100644 --- a/src/packages/pnpm-lock.yaml +++ b/src/packages/pnpm-lock.yaml @@ -81,6 +81,9 @@ importers: '@cocalc/conat': specifier: workspace:* version: link:../conat + '@cocalc/sync': + specifier: workspace:* + version: link:../sync '@cocalc/util': specifier: workspace:* version: link:../util diff --git a/src/packages/sync/client/sync-client.ts b/src/packages/sync/client/sync-client.ts index 56db603ed7..d3287e9295 100644 --- a/src/packages/sync/client/sync-client.ts +++ b/src/packages/sync/client/sync-client.ts @@ -99,6 +99,7 @@ export class SyncClient { data_server: undefined, client: this.client, ephemeral: false, + fs: undefined, }); return new SyncString(opts0); } @@ -122,6 +123,8 @@ export class SyncClient { client: this.client, ephemeral: false, + + fs: undefined, }); return new SyncDB(opts0); } diff --git a/src/packages/sync/editor/generic/sync-doc.ts b/src/packages/sync/editor/generic/sync-doc.ts index 213775e8d3..9466089037 100644 --- a/src/packages/sync/editor/generic/sync-doc.ts +++ b/src/packages/sync/editor/generic/sync-doc.ts @@ -125,6 +125,11 @@ const DEBUG = false; export type State = "init" | "ready" | "closed"; export type DataServer = "project" | "database"; +export interface SyncDocFilesystem { + readFile: (path: string, encoding?: any) => Promise; + writeFile: (path: string, data: string | Buffer) => Promise; +} + export interface SyncOpts0 { project_id: string; path: string; @@ -151,6 +156,9 @@ export interface SyncOpts0 { // which data/changefeed server to use data_server?: DataServer; + + // optional filesystem interface. + fs?: SyncDocFilesystem; } export interface SyncOpts extends SyncOpts0 { From 1f9060f4d118cb1e9b8aa7ec6d3fa23841adb9b6 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 19 Jul 2025 00:40:00 +0000 Subject: [PATCH 015/270] move fs sandbox code to @cocalc/backend, since it fits with other things there and is very lightweight --- .../{file-server/conat => backend/conat/files}/local-path.ts | 2 +- .../conat => backend/conat/files}/test/local-path.test.ts | 0 src/packages/backend/package.json | 1 + .../{file-server/fs/sandbox.ts => backend/sandbox/index.ts} | 0 .../{file-server/fs => backend/sandbox}/sandbox.test.ts | 2 +- 5 files changed, 3 insertions(+), 2 deletions(-) rename src/packages/{file-server/conat => backend/conat/files}/local-path.ts (94%) rename src/packages/{file-server/conat => backend/conat/files}/test/local-path.test.ts (100%) rename src/packages/{file-server/fs/sandbox.ts => backend/sandbox/index.ts} (100%) rename src/packages/{file-server/fs => backend/sandbox}/sandbox.test.ts (96%) diff --git a/src/packages/file-server/conat/local-path.ts b/src/packages/backend/conat/files/local-path.ts similarity index 94% rename from src/packages/file-server/conat/local-path.ts rename to src/packages/backend/conat/files/local-path.ts index 53edc7443b..1c50dcdc80 100644 --- a/src/packages/file-server/conat/local-path.ts +++ b/src/packages/backend/conat/files/local-path.ts @@ -1,5 +1,5 @@ import { fsServer } from "@cocalc/conat/files/fs"; -import { SandboxedFilesystem } from "@cocalc/file-server/fs/sandbox"; +import { SandboxedFilesystem } from "@cocalc/backend/sandbox"; import { mkdir } from "fs/promises"; import { join } from "path"; import { isValidUUID } from "@cocalc/util/misc"; diff --git a/src/packages/file-server/conat/test/local-path.test.ts b/src/packages/backend/conat/files/test/local-path.test.ts similarity index 100% rename from src/packages/file-server/conat/test/local-path.test.ts rename to src/packages/backend/conat/files/test/local-path.test.ts diff --git a/src/packages/backend/package.json b/src/packages/backend/package.json index f190097f05..e31ec57b16 100644 --- a/src/packages/backend/package.json +++ b/src/packages/backend/package.json @@ -6,6 +6,7 @@ "./*": "./dist/*.js", "./database": "./dist/database/index.js", "./conat": "./dist/conat/index.js", + "./sandbox": "./dist/sandbox/index.js", "./conat/sync/*": "./dist/conat/sync/*.js", "./server-settings": "./dist/server-settings/index.js", "./auth/*": "./dist/auth/*.js", diff --git a/src/packages/file-server/fs/sandbox.ts b/src/packages/backend/sandbox/index.ts similarity index 100% rename from src/packages/file-server/fs/sandbox.ts rename to src/packages/backend/sandbox/index.ts diff --git a/src/packages/file-server/fs/sandbox.test.ts b/src/packages/backend/sandbox/sandbox.test.ts similarity index 96% rename from src/packages/file-server/fs/sandbox.test.ts rename to src/packages/backend/sandbox/sandbox.test.ts index 73bd45b171..882676e76e 100644 --- a/src/packages/file-server/fs/sandbox.test.ts +++ b/src/packages/backend/sandbox/sandbox.test.ts @@ -1,4 +1,4 @@ -import { SandboxedFilesystem } from "./sandbox"; +import { SandboxedFilesystem } from "@cocalc/backend/sandbox"; import { mkdtemp, mkdir, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "path"; From 4b37a56046ecd0a4b86acca5b0fb3ac441dcf5ee Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 19 Jul 2025 01:05:16 +0000 Subject: [PATCH 016/270] file sandbox: refactor some unit testing code --- .../backend/conat/files/local-path.ts | 6 +-- .../conat/files/test/local-path.test.ts | 52 +++++++------------ src/packages/backend/conat/files/test/util.ts | 30 +++++++++++ 3 files changed, 53 insertions(+), 35 deletions(-) create mode 100644 src/packages/backend/conat/files/test/util.ts diff --git a/src/packages/backend/conat/files/local-path.ts b/src/packages/backend/conat/files/local-path.ts index 1c50dcdc80..fe52d3aedb 100644 --- a/src/packages/backend/conat/files/local-path.ts +++ b/src/packages/backend/conat/files/local-path.ts @@ -5,7 +5,7 @@ import { join } from "path"; import { isValidUUID } from "@cocalc/util/misc"; import { type Client, getClient } from "@cocalc/conat/core/client"; -export function localPathFileserver({ +export async function localPathFileserver({ service, path, client, @@ -15,7 +15,7 @@ export function localPathFileserver({ client?: Client; }) { client ??= getClient(); - const server = fsServer({ + const server = await fsServer({ service, client, fs: async (subject: string) => { @@ -27,7 +27,7 @@ export function localPathFileserver({ return new SandboxedFilesystem(p); }, }); - return server; + return { server, client, path, service, close: () => server.end() }; } function getProjectId(subject: string) { diff --git a/src/packages/backend/conat/files/test/local-path.test.ts b/src/packages/backend/conat/files/test/local-path.test.ts index 8ccb4b293b..65430a8c2a 100644 --- a/src/packages/backend/conat/files/test/local-path.test.ts +++ b/src/packages/backend/conat/files/test/local-path.test.ts @@ -1,31 +1,23 @@ -import { localPathFileserver } from "../local-path"; -import { link, mkdtemp, readFile, rm, symlink } from "node:fs/promises"; -import { tmpdir } from "node:os"; +import { link, readFile, symlink } from "node:fs/promises"; import { join } from "path"; import { fsClient } from "@cocalc/conat/files/fs"; import { randomId } from "@cocalc/conat/names"; -import { before, after, client } from "@cocalc/backend/conat/test/setup"; +import { before, after } from "@cocalc/backend/conat/test/setup"; import { uuid } from "@cocalc/util/misc"; +import { createPathFileserver, cleanupFileservers } from "./util"; -let tempDir; -let tempDir2; -beforeAll(async () => { - await before(); - tempDir = await mkdtemp(join(tmpdir(), "cocalc-local-path")); - tempDir2 = await mkdtemp(join(tmpdir(), "cocalc-local-path-2")); -}); +beforeAll(before); describe("use all the standard api functions of fs", () => { - const service = `fs-${randomId()}`; let server; it("creates the simple fileserver service", async () => { - server = await localPathFileserver({ client, service, path: tempDir }); + server = await createPathFileserver(); }); const project_id = uuid(); let fs; it("create a client", () => { - fs = fsClient({ subject: `${service}.project-${project_id}` }); + fs = fsClient({ subject: `${server.service}.project-${project_id}` }); }); it("appendFile works", async () => { @@ -246,25 +238,22 @@ describe("use all the standard api functions of fs", () => { const stats0 = await fs.stat("source1"); expect(stats0.isSymbolicLink()).toBe(false); }); - - it("closes the service", () => { - server.close(); - }); }); describe("security: dangerous symlinks can't be followed", () => { - const service = `fs-${randomId()}`; let server; + let tempDir; it("creates the simple fileserver service", async () => { - server = await localPathFileserver({ client, service, path: tempDir2 }); + server = await createPathFileserver(); + tempDir = server.path; }); const project_id = uuid(); const project_id2 = uuid(); let fs, fs2; it("create two clients", () => { - fs = fsClient({ subject: `${service}.project-${project_id}` }); - fs2 = fsClient({ subject: `${service}.project-${project_id2}` }); + fs = fsClient({ subject: `${server.service}.project-${project_id}` }); + fs2 = fsClient({ subject: `${server.service}.project-${project_id2}` }); }); it("create a secret in one", async () => { @@ -276,10 +265,10 @@ describe("security: dangerous symlinks can't be followed", () => { // having full access internally to their sandbox fs. it("directly create a dangerous file that is a symlink outside of the sandbox -- this should work", async () => { await symlink( - join(tempDir2, project_id, "password"), - join(tempDir2, project_id2, "danger"), + join(tempDir, project_id, "password"), + join(tempDir, project_id2, "danger"), ); - const s = await readFile(join(tempDir2, project_id2, "danger"), "utf8"); + const s = await readFile(join(tempDir, project_id2, "danger"), "utf8"); expect(s).toBe("s3cr3t"); }); @@ -292,9 +281,9 @@ describe("security: dangerous symlinks can't be followed", () => { it("directly create a dangerous relative symlink ", async () => { await symlink( join("..", project_id, "password"), - join(tempDir2, project_id2, "danger2"), + join(tempDir, project_id2, "danger2"), ); - const s = await readFile(join(tempDir2, project_id2, "danger2"), "utf8"); + const s = await readFile(join(tempDir, project_id2, "danger2"), "utf8"); expect(s).toBe("s3cr3t"); }); @@ -309,10 +298,10 @@ describe("security: dangerous symlinks can't be followed", () => { // of their own folder. it("directly create a hard link", async () => { await link( - join(tempDir2, project_id, "password"), - join(tempDir2, project_id2, "danger3"), + join(tempDir, project_id, "password"), + join(tempDir, project_id2, "danger3"), ); - const s = await readFile(join(tempDir2, project_id2, "danger3"), "utf8"); + const s = await readFile(join(tempDir, project_id2, "danger3"), "utf8"); expect(s).toBe("s3cr3t"); }); @@ -328,6 +317,5 @@ describe("security: dangerous symlinks can't be followed", () => { afterAll(async () => { await after(); - await rm(tempDir, { force: true, recursive: true }); - await rm(tempDir2, { force: true, recursive: true }); + await cleanupFileservers(); }); diff --git a/src/packages/backend/conat/files/test/util.ts b/src/packages/backend/conat/files/test/util.ts new file mode 100644 index 0000000000..2de57463ea --- /dev/null +++ b/src/packages/backend/conat/files/test/util.ts @@ -0,0 +1,30 @@ +import { localPathFileserver } from "../local-path"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "path"; +import { client } from "@cocalc/backend/conat/test/setup"; +import { randomId } from "@cocalc/conat/names"; + +const tempDirs: string[] = []; +const servers: any[] = []; +export async function createPathFileserver({ + service = `fs-${randomId()}`, +}: { service?: string } = {}) { + const tempDir = await mkdtemp(join(tmpdir(), `cocalc-${randomId()}`)); + tempDirs.push(tempDir); + const server = await localPathFileserver({ client, service, path: tempDir }); + servers.push(server); + return server; +} + +// clean up any +export async function cleanupFileservers() { + for (const server of servers) { + server.close(); + } + for (const tempDir of tempDirs) { + try { + await rm(tempDir, { force: true, recursive: true }); + } catch {} + } +} From e6c6c1e0e07a3ec03ed830d82910530340fdd1f6 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 19 Jul 2025 05:12:10 +0000 Subject: [PATCH 017/270] sync-doc: start using the new fs interface --- .../conat/files/test/local-path.test.ts | 16 +++ .../backend/conat/sync-doc/syncstring.ts | 14 +- .../backend/conat/sync-doc/test/setup.ts | 27 ++++ .../conat/sync-doc/test/syncstring.test.ts | 35 ++++- src/packages/backend/sandbox/index.ts | 24 +++- src/packages/conat/core/client.ts | 23 +++- src/packages/conat/files/fs.ts | 2 +- src/packages/sync/editor/generic/sync-doc.ts | 120 ++++++++++++++---- 8 files changed, 222 insertions(+), 39 deletions(-) create mode 100644 src/packages/backend/conat/sync-doc/test/setup.ts diff --git a/src/packages/backend/conat/files/test/local-path.test.ts b/src/packages/backend/conat/files/test/local-path.test.ts index 65430a8c2a..5eb9e9fadd 100644 --- a/src/packages/backend/conat/files/test/local-path.test.ts +++ b/src/packages/backend/conat/files/test/local-path.test.ts @@ -87,6 +87,22 @@ describe("use all the standard api functions of fs", () => { expect(t).toEqual("conat"); }); + it("the full error message structure is preserved exactly as in the nodejs library", async () => { + const path = randomId(); + try { + await fs.readFile(path); + } catch (err) { + expect(err.message).toEqual( + `ENOENT: no such file or directory, open '${path}'`, + ); + expect(err.message).toContain(path); + expect(err.code).toEqual("ENOENT"); + expect(err.errno).toEqual(-2); + expect(err.path).toEqual(path); + expect(err.syscall).toEqual("open"); + } + }); + it("readdir works", async () => { await fs.mkdir("dirtest"); for (let i = 0; i < 5; i++) { diff --git a/src/packages/backend/conat/sync-doc/syncstring.ts b/src/packages/backend/conat/sync-doc/syncstring.ts index f0454b7fdf..c7cee121b5 100644 --- a/src/packages/backend/conat/sync-doc/syncstring.ts +++ b/src/packages/backend/conat/sync-doc/syncstring.ts @@ -2,15 +2,25 @@ import { Client } from "./client"; import { SyncString } from "@cocalc/sync/editor/string/sync"; import { a_txt } from "@cocalc/sync/editor/string/test/data"; import { once } from "@cocalc/util/async-utils"; +import { type SyncDocFilesystem } from "@cocalc/sync/editor/generic/sync-doc"; -export default async function ephemeralSyncstring() { - const { client_id, project_id, path, init_queries } = a_txt(); +export default async function syncstring({ + fs, + project_id, + path, +}: { + fs: SyncDocFilesystem; + project_id: string; + path: string; +}) { + const { client_id, init_queries } = a_txt(); const client = new Client(init_queries, client_id); const syncstring = new SyncString({ project_id, path, client, ephemeral: true, + fs, }); // replace save to disk, since otherwise unless string is empty, // this will hang forever... and it is called on close. diff --git a/src/packages/backend/conat/sync-doc/test/setup.ts b/src/packages/backend/conat/sync-doc/test/setup.ts new file mode 100644 index 0000000000..bcef527ac6 --- /dev/null +++ b/src/packages/backend/conat/sync-doc/test/setup.ts @@ -0,0 +1,27 @@ +import { + before as before0, + after as after0, +} from "@cocalc/backend/conat/test/setup"; +import { + createPathFileserver, + cleanupFileservers, +} from "@cocalc/backend/conat/files/test/util"; +import { type Filesystem } from "@cocalc/conat/files/fs"; +export { uuid } from "@cocalc/util/misc"; +import { fsClient } from "@cocalc/conat/files/fs"; + +export let server; + +export async function before() { + await before0(); + server = await createPathFileserver(); +} + +export function getFS(project_id: string): Filesystem { + return fsClient({ subject: `${server.service}.project-${project_id}` }); +} + +export async function after() { + await cleanupFileservers(); + await after0(); +} diff --git a/src/packages/backend/conat/sync-doc/test/syncstring.test.ts b/src/packages/backend/conat/sync-doc/test/syncstring.test.ts index 62b9875bb5..96bf2d1643 100644 --- a/src/packages/backend/conat/sync-doc/test/syncstring.test.ts +++ b/src/packages/backend/conat/sync-doc/test/syncstring.test.ts @@ -1,35 +1,56 @@ import syncstring from "@cocalc/backend/conat/sync-doc/syncstring"; +import { before, after, getFS, uuid } from "./setup"; + +beforeAll(before); +afterAll(after); describe("basic tests of a syncstring", () => { let s; + const project_id = uuid(); + let fs; - it("creates a syncstring", async () => { - s = await syncstring(); + it("creates the fs client", () => { + fs = getFS(project_id); }); - it("initially it is empty", () => { + it("a syncstring associated to a file that does not exist on disk is initialized to the empty string", async () => { + s = await syncstring({ fs, project_id, path: "new.txt" }); expect(s.to_str()).toBe(""); expect(s.versions().length).toBe(0); + s.close(); + }); + + it("a syncstring for editing a file that already exists on disk is initialized to that file", async () => { + fs = getFS(project_id); + await fs.writeFile("a.txt", "hello"); + s = await syncstring({ fs, project_id, path: "a.txt" }); + expect(s.fs).not.toEqual(undefined); + }); + + it("initially it is 'hello'", () => { + expect(s.to_str()).toBe("hello"); + expect(s.versions().length).toBe(1); }); it("set the value", () => { s.from_str("test"); expect(s.to_str()).toBe("test"); - expect(s.versions().length).toBe(0); + expect(s.versions().length).toBe(1); }); it("commit the value", () => { s.commit(); - expect(s.versions().length).toBe(1); + expect(s.versions().length).toBe(2); }); it("change the value and commit a second time", () => { s.from_str("bar"); s.commit(); - expect(s.versions().length).toBe(2); + expect(s.versions().length).toBe(3); }); it("get first version", () => { - expect(s.version(s.versions()[0]).to_str()).toBe("test"); + expect(s.version(s.versions()[0]).to_str()).toBe("hello"); + expect(s.version(s.versions()[1]).to_str()).toBe("test"); }); }); diff --git a/src/packages/backend/sandbox/index.ts b/src/packages/backend/sandbox/index.ts index f3795937c0..09146f7883 100644 --- a/src/packages/backend/sandbox/index.ts +++ b/src/packages/backend/sandbox/index.ts @@ -38,7 +38,6 @@ The problem is that 1 and 3 happen microseconds apart as separate calls to the f 3. user somehow deletes "link" and replace it by a new file that is a symlink to "../{project_id}/.ssh/id_ed25519" 4. We read from the file descriptor fd and get the contents of original "link" (or error). - */ import { @@ -68,10 +67,31 @@ import { exists } from "@cocalc/backend/misc/async-utils-node"; import { type DirectoryListingEntry } from "@cocalc/util/types"; import getListing from "@cocalc/backend/get-listing"; import { join, resolve } from "path"; +import { replace_all } from "@cocalc/util/misc"; export class SandboxedFilesystem { // path should be the path to a FOLDER on the filesystem (not a file) - constructor(public readonly path: string) {} + constructor(public readonly path: string) { + for (const f in this) { + if (f == "safeAbsPath" || f == "constructor" || f == "path") { + continue; + } + const orig = this[f]; + // @ts-ignore + this[f] = async (...args) => { + try { + // @ts-ignore + return await orig(...args); + } catch (err) { + if (err.path) { + err.path = err.path.slice(this.path.length + 1); + } + err.message = replace_all(err.message, this.path + "/", ""); + throw err; + } + }; + } + } safeAbsPath = async (path: string): Promise => { if (typeof path != "string") { diff --git a/src/packages/conat/core/client.ts b/src/packages/conat/core/client.ts index 4ba04b56b1..e09c94cd85 100644 --- a/src/packages/conat/core/client.ts +++ b/src/packages/conat/core/client.ts @@ -1106,9 +1106,16 @@ export class Client extends EventEmitter { // good for services. await mesg.respond(result); } catch (err) { + let error = err.message; + if (!error) { + error = `${err}`.slice("Error: ".length); + } await mesg.respond(null, { - noThrow: true, // we're not catching this one - headers: { error: `${err}` }, + noThrow: true, // we're not catching this respond + headers: { + error, + error_attrs: JSON.parse(JSON.stringify(err)), + }, }); } }; @@ -1127,7 +1134,7 @@ export class Client extends EventEmitter { const call = async (name: string, args: any[]) => { const resp = await this.request(subject, [name, args], opts); if (resp.headers?.error) { - throw Error(`${resp.headers.error}`); + throw headerToError(resp.headers); } else { return resp.data; } @@ -1944,3 +1951,13 @@ function toConatError(socketIoError) { }); } } + +export function headerToError(headers) { + const err = Error(headers.error); + if (headers.error_attrs) { + for (const field in headers.error_attrs) { + err[field] = headers.error_attrs[field]; + } + } + return err; +} diff --git a/src/packages/conat/files/fs.ts b/src/packages/conat/files/fs.ts index 506f3823b5..f0ce9e1f8e 100644 --- a/src/packages/conat/files/fs.ts +++ b/src/packages/conat/files/fs.ts @@ -188,7 +188,7 @@ export function fsClient({ }: { client?: Client; subject: string; -}) { +}): Filesystem { let call = (client ?? conat()).call(subject); let constants: any = null; diff --git a/src/packages/sync/editor/generic/sync-doc.ts b/src/packages/sync/editor/generic/sync-doc.ts index 9466089037..498f319c14 100644 --- a/src/packages/sync/editor/generic/sync-doc.ts +++ b/src/packages/sync/editor/generic/sync-doc.ts @@ -128,6 +128,7 @@ export type DataServer = "project" | "database"; export interface SyncDocFilesystem { readFile: (path: string, encoding?: any) => Promise; writeFile: (path: string, data: string | Buffer) => Promise; + stat: (path: string) => Promise; // todo } export interface SyncOpts0 { @@ -272,6 +273,8 @@ export class SyncDoc extends EventEmitter { private useConat: boolean; legacy: LegacyHistory; + private fs?: SyncDocFilesystem; + constructor(opts: SyncOpts) { super(); if (opts.string_id === undefined) { @@ -293,6 +296,7 @@ export class SyncDoc extends EventEmitter { "persistent", "data_server", "ephemeral", + "fs", ]) { if (opts[field] != undefined) { this[field] = opts[field]; @@ -1505,31 +1509,34 @@ export class SyncDoc extends EventEmitter { log("file_use_interval"); this.init_file_use_interval(); - - if (await this.isFileServer()) { - log("load_from_disk"); - // This sets initialized, which is needed to be fully ready. - // We keep trying this load from disk until sync-doc is closed - // or it succeeds. It may fail if, e.g., the file is too - // large or is not readable by the user. They are informed to - // fix the problem... and once they do (and wait up to 10s), - // this will finish. - // if (!this.client.is_browser() && !this.client.is_project()) { - // // FAKE DELAY!!! Just to simulate flakiness / slow network!!!! - // await delay(3000); - // } - await retry_until_success({ - f: this.init_load_from_disk, - max_delay: 10000, - desc: "syncdoc -- load_from_disk", - }); - log("done loading from disk"); + if (this.fs != null) { + await this.fsLoadFromDisk(); } else { - if (this.patch_list!.count() == 0) { - await Promise.race([ - this.waitUntilFullyReady(), - once(this.patch_list!, "change"), - ]); + if (await this.isFileServer()) { + log("load_from_disk"); + // This sets initialized, which is needed to be fully ready. + // We keep trying this load from disk until sync-doc is closed + // or it succeeds. It may fail if, e.g., the file is too + // large or is not readable by the user. They are informed to + // fix the problem... and once they do (and wait up to 10s), + // this will finish. + // if (!this.client.is_browser() && !this.client.is_project()) { + // // FAKE DELAY!!! Just to simulate flakiness / slow network!!!! + // await delay(3000); + // } + await retry_until_success({ + f: this.init_load_from_disk, + max_delay: 10000, + desc: "syncdoc -- load_from_disk", + }); + log("done loading from disk"); + } else { + if (this.patch_list!.count() == 0) { + await Promise.race([ + this.waitUntilFullyReady(), + once(this.patch_list!, "change"), + ]); + } } } this.assert_not_closed("initAll -- load from disk"); @@ -1757,7 +1764,46 @@ export class SyncDoc extends EventEmitter { } }; + private fsLoadFromDiskIfNewer = async (): Promise => { + // [ ] TODO: readonly handling... + if (this.fs == null) throw Error("bug"); + const dbg = this.dbg("fsLoadFromDiskIfNewer"); + let stats; + try { + stats = await this.fs.stat(this.path); + } catch (err) { + if (err.code == "ENOENT") { + // path does not exist -- nothing further to do + return false; + } else { + // no clue + return true; + } + } + dbg("path exists"); + const lastChanged = new Date(this.last_changed()); + const firstLoad = this.versions().length == 0; + if (firstLoad || stats.ctime > lastChanged) { + dbg( + `disk file changed more recently than edits, so loading ${stats.ctime} > ${lastChanged}; firstLoad=${firstLoad}`, + ); + await this.fsLoadFromDisk(); + if (firstLoad) { + dbg("emitting first-load event"); + // this event is emited the first time the document is ever loaded from disk. + this.emit("first-load"); + } + dbg("loaded"); + } else { + dbg("stick with sync version"); + } + return false; + }; + private load_from_disk_if_newer = async (): Promise => { + if (this.fs != null) { + return await this.fsLoadFromDiskIfNewer(); + } const last_changed = new Date(this.last_changed()); const firstLoad = this.versions().length == 0; const dbg = this.dbg("load_from_disk_if_newer"); @@ -2938,6 +2984,32 @@ export class SyncDoc extends EventEmitter { this.close(); }; + private fsLoadFromDisk = async (): Promise => { + if (this.fs == null) throw Error("bug"); + const dbg = this.dbg("fsLoadFromDisk"); + + let size: number; + let contents; + try { + contents = await this.fs.readFile(this.path, "utf8"); + dbg("file exists"); + size = contents.length; + this.from_str(contents); + } catch (err) { + if (err.code == "ENOENT") { + dbg("file no longer exists -- setting to blank"); + size = 0; + this.from_str(""); + } else { + throw err; + } + } + // save new version to stream, which we just set via from_str + this.commit(); + await this.save(); + return size; + }; + private load_from_disk = async (): Promise => { const path = this.path; const dbg = this.dbg("load_from_disk"); From d872a972fec9da9109a4985f32f86dde5893d2d3 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 19 Jul 2025 05:20:43 +0000 Subject: [PATCH 018/270] sync-doc: use fs interface to save to disk --- src/packages/backend/conat/sync-doc/syncstring.ts | 4 ---- .../backend/conat/sync-doc/test/syncstring.test.ts | 6 ++++++ src/packages/sync/editor/generic/sync-doc.ts | 14 ++++++++++++++ 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/src/packages/backend/conat/sync-doc/syncstring.ts b/src/packages/backend/conat/sync-doc/syncstring.ts index c7cee121b5..64990e42c9 100644 --- a/src/packages/backend/conat/sync-doc/syncstring.ts +++ b/src/packages/backend/conat/sync-doc/syncstring.ts @@ -22,10 +22,6 @@ export default async function syncstring({ ephemeral: true, fs, }); - // replace save to disk, since otherwise unless string is empty, - // this will hang forever... and it is called on close. - // @ts-ignore - syncstring.save_to_disk = async () => Promise; await once(syncstring, "ready"); return syncstring; } diff --git a/src/packages/backend/conat/sync-doc/test/syncstring.test.ts b/src/packages/backend/conat/sync-doc/test/syncstring.test.ts index 96bf2d1643..fa637f28d0 100644 --- a/src/packages/backend/conat/sync-doc/test/syncstring.test.ts +++ b/src/packages/backend/conat/sync-doc/test/syncstring.test.ts @@ -38,6 +38,12 @@ describe("basic tests of a syncstring", () => { expect(s.versions().length).toBe(1); }); + it("save value to disk", async () => { + await s.save_to_disk(); + const disk = await fs.readFile("a.txt", "utf8"); + expect(disk).toEqual("test"); + }); + it("commit the value", () => { s.commit(); expect(s.versions().length).toBe(2); diff --git a/src/packages/sync/editor/generic/sync-doc.ts b/src/packages/sync/editor/generic/sync-doc.ts index 498f319c14..b4c019f57c 100644 --- a/src/packages/sync/editor/generic/sync-doc.ts +++ b/src/packages/sync/editor/generic/sync-doc.ts @@ -3188,6 +3188,17 @@ export class SyncDoc extends EventEmitter { return true; }; + fsSaveToDisk = async () => { + const dbg = this.dbg("fsSaveToDisk"); + if (this.client.is_deleted(this.path, this.project_id)) { + dbg("not saving to disk because deleted"); + return; + } + dbg(); + if (this.fs == null) throw Error("bug"); + await this.fs.writeFile(this.path, this.to_str()); + }; + /* Initiates a save of file to disk, then waits for the state to change. */ save_to_disk = async (): Promise => { @@ -3201,6 +3212,9 @@ export class SyncDoc extends EventEmitter { // properly. return; } + if (this.fs != null) { + return await this.fsSaveToDisk(); + } const dbg = this.dbg("save_to_disk"); if (this.client.is_deleted(this.path, this.project_id)) { dbg("not saving to disk because deleted"); From 89dc29087545999a71776cf6bbd1cf21a36a102a Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 19 Jul 2025 05:32:32 +0000 Subject: [PATCH 019/270] sync-doc: start marking fs related client functions as only required for legacy clients --- src/packages/backend/conat/sync-doc/client.ts | 4 ---- src/packages/sync/editor/generic/sync-doc.ts | 19 +++++++++++++++++++ src/packages/sync/editor/generic/types.ts | 2 +- 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/src/packages/backend/conat/sync-doc/client.ts b/src/packages/backend/conat/sync-doc/client.ts index bf6742a550..19813b772d 100644 --- a/src/packages/backend/conat/sync-doc/client.ts +++ b/src/packages/backend/conat/sync-doc/client.ts @@ -104,10 +104,6 @@ export class Client extends EventEmitter implements Client0 { console.log("path_access", opts.path, opts.mode); opts.cb(true); } - path_exists(opts: { path: string; cb: Function }): void { - console.log("path_access", opts.path); - opts.cb(true); - } path_stat(opts: { path: string; cb: Function }): void { console.log("path_state", opts.path); opts.cb(true); diff --git a/src/packages/sync/editor/generic/sync-doc.ts b/src/packages/sync/editor/generic/sync-doc.ts index b4c019f57c..cb971876cd 100644 --- a/src/packages/sync/editor/generic/sync-doc.ts +++ b/src/packages/sync/editor/generic/sync-doc.ts @@ -1714,6 +1714,13 @@ export class SyncDoc extends EventEmitter { }; private pathExistsAndIsReadOnly = async (path): Promise => { + if (this.client.path_exists == null) { + throw Error("legacy clients must define path_exists"); + } + if (this.client.path_access == null) { + throw Error("legacy clients must define path_access"); + } + try { await callback2(this.client.path_access, { path, @@ -1804,6 +1811,9 @@ export class SyncDoc extends EventEmitter { if (this.fs != null) { return await this.fsLoadFromDiskIfNewer(); } + if (this.client.path_exists == null) { + throw Error("legacy clients must define path_exists"); + } const last_changed = new Date(this.last_changed()); const firstLoad = this.versions().length == 0; const dbg = this.dbg("load_from_disk_if_newer"); @@ -2890,6 +2900,12 @@ export class SyncDoc extends EventEmitter { }; private update_watch_path = async (path?: string): Promise => { + if (this.fs != null) { + return; + } + if (this.client.path_exists == null) { + throw Error("legacy clients must define path_exists"); + } const dbg = this.dbg("update_watch_path"); if (this.file_watcher != null) { // clean up @@ -3011,6 +3027,9 @@ export class SyncDoc extends EventEmitter { }; private load_from_disk = async (): Promise => { + if (this.client.path_exists == null) { + throw Error("legacy clients must define path_exists"); + } const path = this.path; const dbg = this.dbg("load_from_disk"); dbg(); diff --git a/src/packages/sync/editor/generic/types.ts b/src/packages/sync/editor/generic/types.ts index 4cf3519e94..9222636e92 100644 --- a/src/packages/sync/editor/generic/types.ts +++ b/src/packages/sync/editor/generic/types.ts @@ -102,7 +102,7 @@ export interface ProjectClient extends EventEmitter { // Only required to work on project client. path_access: (opts: { path: string; mode: string; cb: Function }) => void; - path_exists: (opts: { path: string; cb: Function }) => void; + path_exists?: (opts: { path: string; cb: Function }) => void; path_stat: (opts: { path: string; cb: Function }) => void; From 45e094348ad665eac4ed2060bd0470d357ed0405 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 19 Jul 2025 14:23:01 +0000 Subject: [PATCH 020/270] fix some tests --- src/packages/backend/conat/files/local-path.ts | 2 +- src/packages/backend/{ => files}/sandbox/index.ts | 0 src/packages/backend/{ => files}/sandbox/sandbox.test.ts | 2 +- src/packages/backend/package.json | 3 ++- src/packages/file-server/btrfs/subvolume.ts | 2 +- 5 files changed, 5 insertions(+), 4 deletions(-) rename src/packages/backend/{ => files}/sandbox/index.ts (100%) rename src/packages/backend/{ => files}/sandbox/sandbox.test.ts (95%) diff --git a/src/packages/backend/conat/files/local-path.ts b/src/packages/backend/conat/files/local-path.ts index fe52d3aedb..1d0aa43a8e 100644 --- a/src/packages/backend/conat/files/local-path.ts +++ b/src/packages/backend/conat/files/local-path.ts @@ -1,5 +1,5 @@ import { fsServer } from "@cocalc/conat/files/fs"; -import { SandboxedFilesystem } from "@cocalc/backend/sandbox"; +import { SandboxedFilesystem } from "@cocalc/backend/files/sandbox"; import { mkdir } from "fs/promises"; import { join } from "path"; import { isValidUUID } from "@cocalc/util/misc"; diff --git a/src/packages/backend/sandbox/index.ts b/src/packages/backend/files/sandbox/index.ts similarity index 100% rename from src/packages/backend/sandbox/index.ts rename to src/packages/backend/files/sandbox/index.ts diff --git a/src/packages/backend/sandbox/sandbox.test.ts b/src/packages/backend/files/sandbox/sandbox.test.ts similarity index 95% rename from src/packages/backend/sandbox/sandbox.test.ts rename to src/packages/backend/files/sandbox/sandbox.test.ts index 882676e76e..5e445b42c0 100644 --- a/src/packages/backend/sandbox/sandbox.test.ts +++ b/src/packages/backend/files/sandbox/sandbox.test.ts @@ -1,4 +1,4 @@ -import { SandboxedFilesystem } from "@cocalc/backend/sandbox"; +import { SandboxedFilesystem } from "@cocalc/backend/files/sandbox"; import { mkdtemp, mkdir, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "path"; diff --git a/src/packages/backend/package.json b/src/packages/backend/package.json index e31ec57b16..1ceb59e9c6 100644 --- a/src/packages/backend/package.json +++ b/src/packages/backend/package.json @@ -6,7 +6,8 @@ "./*": "./dist/*.js", "./database": "./dist/database/index.js", "./conat": "./dist/conat/index.js", - "./sandbox": "./dist/sandbox/index.js", + "./files/*": "./dist/files/*.js", + "./files/sandbox": "./dist/files/sandbox/index.js", "./conat/sync/*": "./dist/conat/sync/*.js", "./server-settings": "./dist/server-settings/index.js", "./auth/*": "./dist/auth/*.js", diff --git a/src/packages/file-server/btrfs/subvolume.ts b/src/packages/file-server/btrfs/subvolume.ts index 7eb7808518..f10e9f6182 100644 --- a/src/packages/file-server/btrfs/subvolume.ts +++ b/src/packages/file-server/btrfs/subvolume.ts @@ -9,7 +9,7 @@ import { join } from "path"; import { SubvolumeBup } from "./subvolume-bup"; import { SubvolumeSnapshots } from "./subvolume-snapshots"; import { SubvolumeQuota } from "./subvolume-quota"; -import { SandboxedFilesystem } from "../fs/sandbox"; +import { SandboxedFilesystem } from "@cocalc/backend/files/sandbox"; import { exists } from "@cocalc/backend/misc/async-utils-node"; import getLogger from "@cocalc/backend/logger"; From 6921d7e226046a46570a90bccb17a093289be51d Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 19 Jul 2025 15:35:46 +0000 Subject: [PATCH 021/270] rewrite nodejs watch async iterator since it is broken until node v24 and we don't want to require node24 yet --- src/packages/backend/files/sandbox/index.ts | 14 ++++++++++++-- .../backend/files/sandbox/sandbox.test.ts | 19 +++++++++++++++++++ 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/src/packages/backend/files/sandbox/index.ts b/src/packages/backend/files/sandbox/index.ts index 09146f7883..446a9e0813 100644 --- a/src/packages/backend/files/sandbox/index.ts +++ b/src/packages/backend/files/sandbox/index.ts @@ -61,13 +61,14 @@ import { writeFile, unlink, utimes, - watch, } from "node:fs/promises"; +import { watch } from "node:fs"; import { exists } from "@cocalc/backend/misc/async-utils-node"; import { type DirectoryListingEntry } from "@cocalc/util/types"; import getListing from "@cocalc/backend/get-listing"; import { join, resolve } from "path"; import { replace_all } from "@cocalc/util/misc"; +import { EventIterator } from "@cocalc/util/event-iterator"; export class SandboxedFilesystem { // path should be the path to a FOLDER on the filesystem (not a file) @@ -231,7 +232,16 @@ export class SandboxedFilesystem { }; watch = async (filename: string, options?) => { - return watch(await this.safeAbsPath(filename), options); + // NOTE: in node v24 they fixed the fs/promises watch to have a queue, but previous + // versions were clearly badly implemented so we reimplement it from scratch + // using the non-promise watch. + const watcher = watch(await this.safeAbsPath(filename), options); + return new EventIterator(watcher, "change", { + map: (args) => { + // exact same api as new fs/promises watch + return { eventType: args[0], filename: args[1] }; + }, + }); }; writeFile = async (path: string, data: string | Buffer) => { diff --git a/src/packages/backend/files/sandbox/sandbox.test.ts b/src/packages/backend/files/sandbox/sandbox.test.ts index 5e445b42c0..44bcfc9114 100644 --- a/src/packages/backend/files/sandbox/sandbox.test.ts +++ b/src/packages/backend/files/sandbox/sandbox.test.ts @@ -54,6 +54,25 @@ describe("make various attempts to break out of the sandbox", () => { }); }); +describe.only("test watching a file and a folder in the sandbox", () => { + let fs; + it("creates sandbox", async () => { + await mkdir(join(tempDir, "test-watch")); + fs = new SandboxedFilesystem(join(tempDir, "test-watch")); + await fs.writeFile("x", "hi"); + }); + + it("watches the file x for changes", async () => { + const w = await fs.watch("x"); + await fs.appendFile("x", " there"); + const x = await w.next(); + expect(x).toEqual({ + value: { eventType: "change", filename: "x" }, + done: false, + }); + }); +}); + afterAll(async () => { await rm(tempDir, { force: true, recursive: true }); }); From 1363b5365969c7911f2d44554ea8c9e5e77f8673 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 19 Jul 2025 16:00:11 +0000 Subject: [PATCH 022/270] add maxQueue support to watch (and our EventIterator) --- src/packages/backend/files/sandbox/index.ts | 4 +++ .../backend/files/sandbox/sandbox.test.ts | 28 ++++++++++++++++++- src/packages/util/event-iterator.ts | 17 +++++++++-- 3 files changed, 46 insertions(+), 3 deletions(-) diff --git a/src/packages/backend/files/sandbox/index.ts b/src/packages/backend/files/sandbox/index.ts index 446a9e0813..1cbb53616a 100644 --- a/src/packages/backend/files/sandbox/index.ts +++ b/src/packages/backend/files/sandbox/index.ts @@ -237,10 +237,14 @@ export class SandboxedFilesystem { // using the non-promise watch. const watcher = watch(await this.safeAbsPath(filename), options); return new EventIterator(watcher, "change", { + maxQueue: options?.maxQueue ?? 2048, map: (args) => { // exact same api as new fs/promises watch return { eventType: args[0], filename: args[1] }; }, + onEnd: () => { + watcher.close(); + }, }); }; diff --git a/src/packages/backend/files/sandbox/sandbox.test.ts b/src/packages/backend/files/sandbox/sandbox.test.ts index 44bcfc9114..b541dae579 100644 --- a/src/packages/backend/files/sandbox/sandbox.test.ts +++ b/src/packages/backend/files/sandbox/sandbox.test.ts @@ -54,7 +54,7 @@ describe("make various attempts to break out of the sandbox", () => { }); }); -describe.only("test watching a file and a folder in the sandbox", () => { +describe("test watching a file and a folder in the sandbox", () => { let fs; it("creates sandbox", async () => { await mkdir(join(tempDir, "test-watch")); @@ -70,6 +70,32 @@ describe.only("test watching a file and a folder in the sandbox", () => { value: { eventType: "change", filename: "x" }, done: false, }); + w.end(); + }); + + it("the maxQueue parameter limits the number of queue events", async () => { + const w = await fs.watch("x", { maxQueue: 2 }); + expect(w.queueSize()).toBe(0); + // make many changes + await fs.appendFile("x", "0"); + await fs.appendFile("x", "0"); + await fs.appendFile("x", "0"); + await fs.appendFile("x", "0"); + // there will only be 2 available: + expect(w.queueSize()).toBe(2); + const x0 = await w.next(); + expect(x0).toEqual({ + value: { eventType: "change", filename: "x" }, + done: false, + }); + const x1 = await w.next(); + expect(x1).toEqual({ + value: { eventType: "change", filename: "x" }, + done: false, + }); + // one more next would hang... + expect(w.queueSize()).toBe(0); + w.end(); }); }); diff --git a/src/packages/util/event-iterator.ts b/src/packages/util/event-iterator.ts index c3ca053301..7353d2991c 100644 --- a/src/packages/util/event-iterator.ts +++ b/src/packages/util/event-iterator.ts @@ -1,7 +1,7 @@ /* LICENSE: MIT -This is a slight fork of +This is a slight fork of https://github.com/sapphiredev/utilities/tree/main/packages/event-iterator @@ -10,7 +10,7 @@ agree with the docs. I can see why. Upstream would capture ['arg1','arg2']] for an event emitter doing this emitter.emit('foo', 'arg1', 'arg2') - + But for our application we only want 'arg1'. I thus added a map option, which makes it easy to do what we want. */ @@ -46,6 +46,9 @@ export interface EventIteratorOptions { // called when iterator ends -- use to do cleanup. onEnd?: (iter?: EventIterator) => void; + + // Specifies the number of events to queue between iterations of the returned. + maxQueue?: number; } /** @@ -100,6 +103,8 @@ export class EventIterator */ readonly #limit: number; + readonly #maxQueue: number; + /** * The timer to track when this will idle out. */ @@ -124,6 +129,7 @@ export class EventIterator this.event = event; this.map = options.map ?? ((args) => args); this.#limit = options.limit ?? Infinity; + this.#maxQueue = options.maxQueue ?? Infinity; this.#idle = options.idle; this.filter = options.filter ?? ((): boolean => true); this.onEnd = options.onEnd; @@ -263,10 +269,17 @@ export class EventIterator try { const value = this.map(args); this.#queue.push(value); + while (this.#queue.length > this.#maxQueue && this.#queue.length > 0) { + this.#queue.shift(); + } } catch (err) { this.err = err; // fake event to trigger handling of err this.emitter.emit(this.event); } } + + public queueSize(): number { + return this.#queue.length; + } } From f3e40430e174afc5223a2b2e07bedb360a925382 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 19 Jul 2025 17:00:16 +0000 Subject: [PATCH 023/270] support EventIterator overflow options (like node fs watch); add ability to EventIterator next to handle if it is ended while waiting (which prevents a major deadlock/race that upstream had and could hang things); rewrite llm server to ensure messages get through, which avoids race condition if client makes llm request immediately on login (highly unlikely, but let's do it right). --- src/packages/backend/conat/test/llm.test.ts | 4 +- src/packages/backend/files/sandbox/index.ts | 22 ++++++++-- .../backend/files/sandbox/sandbox.test.ts | 29 +++++++++++++ src/packages/conat/llm/server.ts | 34 +++++++++++---- src/packages/util/event-iterator.ts | 41 ++++++++++++++----- 5 files changed, 105 insertions(+), 25 deletions(-) diff --git a/src/packages/backend/conat/test/llm.test.ts b/src/packages/backend/conat/test/llm.test.ts index 9ac69af07f..ab0eeb8ece 100644 --- a/src/packages/backend/conat/test/llm.test.ts +++ b/src/packages/backend/conat/test/llm.test.ts @@ -18,7 +18,7 @@ beforeAll(before); describe("create an llm server, client, and stub evaluator, and run an evaluation", () => { // define trivial evaluate - const OUTPUT = "Thanks for asing about "; + const OUTPUT = "Thanks for asking about "; async function evaluate({ input, stream }) { stream(OUTPUT); stream(input); @@ -52,7 +52,7 @@ describe("create an llm server, client, and stub evaluator, and run an evaluatio describe("test an evaluate that throws an error half way through", () => { // define trivial evaluate - const OUTPUT = "Thanks for asing about "; + const OUTPUT = "Thanks for asking about "; const ERROR = "I give up"; async function evaluate({ stream }) { stream(OUTPUT); diff --git a/src/packages/backend/files/sandbox/index.ts b/src/packages/backend/files/sandbox/index.ts index 1cbb53616a..2888e3a480 100644 --- a/src/packages/backend/files/sandbox/index.ts +++ b/src/packages/backend/files/sandbox/index.ts @@ -231,13 +231,24 @@ export class SandboxedFilesystem { await utimes(await this.safeAbsPath(path), atime, mtime); }; - watch = async (filename: string, options?) => { + watch = async ( + filename: string, + options?: { + persistent?: boolean; + recursive?: boolean; + encoding?: string; + signal?: AbortSignal; + maxQueue?: number; + overflow?: "ignore" | "throw"; + }, + ) => { // NOTE: in node v24 they fixed the fs/promises watch to have a queue, but previous // versions were clearly badly implemented so we reimplement it from scratch // using the non-promise watch. - const watcher = watch(await this.safeAbsPath(filename), options); - return new EventIterator(watcher, "change", { + const watcher = watch(await this.safeAbsPath(filename), options as any); + const iter = new EventIterator(watcher, "change", { maxQueue: options?.maxQueue ?? 2048, + overflow: options?.overflow, map: (args) => { // exact same api as new fs/promises watch return { eventType: args[0], filename: args[1] }; @@ -246,6 +257,11 @@ export class SandboxedFilesystem { watcher.close(); }, }); + // AbortController signal can cause this + watcher.once("close", () => { + iter.end(); + }); + return iter; }; writeFile = async (path: string, data: string | Buffer) => { diff --git a/src/packages/backend/files/sandbox/sandbox.test.ts b/src/packages/backend/files/sandbox/sandbox.test.ts index b541dae579..1f29ae4b53 100644 --- a/src/packages/backend/files/sandbox/sandbox.test.ts +++ b/src/packages/backend/files/sandbox/sandbox.test.ts @@ -63,6 +63,7 @@ describe("test watching a file and a folder in the sandbox", () => { }); it("watches the file x for changes", async () => { + await fs.writeFile("x", "hi"); const w = await fs.watch("x"); await fs.appendFile("x", " there"); const x = await w.next(); @@ -74,6 +75,7 @@ describe("test watching a file and a folder in the sandbox", () => { }); it("the maxQueue parameter limits the number of queue events", async () => { + await fs.writeFile("x", "hi"); const w = await fs.watch("x", { maxQueue: 2 }); expect(w.queueSize()).toBe(0); // make many changes @@ -97,6 +99,33 @@ describe("test watching a file and a folder in the sandbox", () => { expect(w.queueSize()).toBe(0); w.end(); }); + + it("maxQueue with overflow throw", async () => { + await fs.writeFile("x", "hi"); + const w = await fs.watch("x", { maxQueue: 2, overflow: "throw" }); + await fs.appendFile("x", "0"); + await fs.appendFile("x", "0"); + await fs.appendFile("x", "0"); + expect(async () => { + await w.next(); + }).rejects.toThrow("maxQueue overflow"); + w.end(); + }); + + it("AbortController works", async () => { + const ac = new AbortController(); + const { signal } = ac; + await fs.writeFile("x", "hi"); + const w = await fs.watch("x", { signal }); + await fs.appendFile("x", "0"); + const e = await w.next(); + expect(e.done).toBe(false); + + // now abort + ac.abort(); + const { done } = await w.next(); + expect(done).toBe(true); + }); }); afterAll(async () => { diff --git a/src/packages/conat/llm/server.ts b/src/packages/conat/llm/server.ts index 77afd7d86d..e04571cbcc 100644 --- a/src/packages/conat/llm/server.ts +++ b/src/packages/conat/llm/server.ts @@ -14,6 +14,9 @@ how paying for that would work. import { conat } from "@cocalc/conat/client"; import { isValidUUID } from "@cocalc/util/misc"; import type { Subscription } from "@cocalc/conat/core/client"; +import { getLogger } from "@cocalc/conat/client"; + +const logger = getLogger("conat:llm:server"); export const SUBJECT = process.env.COCALC_TEST_MODE ? "llm-test" : "llm"; @@ -61,7 +64,7 @@ export async function close() { if (sub == null) { return; } - sub.drain(); + sub.close(); sub = null; } @@ -77,24 +80,37 @@ async function listen(evaluate) { async function handleMessage(mesg, evaluate) { const options = mesg.data; - let seq = 0; - const respond = ({ text, error }: { text?: string; error?: string }) => { - mesg.respondSync({ text, error, seq }); + let seq = -1; + const respond = async ({ + text, + error, + }: { + text?: string; + error?: string; + }) => { seq += 1; + try { + // mesg.respondSync({ text, error, seq }); + const { count } = await mesg.respond({ text, error, seq }); + } catch (err) { + logger.debug("WARNING: error sending response -- ", err); + end(); + } }; let done = false; - const end = () => { + const end = async () => { if (done) return; done = true; - // end response stream with null payload. - mesg.respondSync(null); + // end response stream with null payload -- send sync, or it could + // get sent before the responses above, which would cancel them out! + await mesg.respond(null, { noThrow: true }); }; - const stream = (text?) => { + const stream = async (text?) => { if (done) return; if (text != null) { - respond({ text }); + await respond({ text }); } else { end(); } diff --git a/src/packages/util/event-iterator.ts b/src/packages/util/event-iterator.ts index 7353d2991c..8dcca43803 100644 --- a/src/packages/util/event-iterator.ts +++ b/src/packages/util/event-iterator.ts @@ -49,6 +49,11 @@ export interface EventIteratorOptions { // Specifies the number of events to queue between iterations of the returned. maxQueue?: number; + + // Either 'ignore' or 'throw' when there are more events to be queued than maxQueue allows. + // 'ignore' means overflow events are dropped and a warning is emitted, while + // 'throw' means to throw an exception. Default: 'ignore'. + overflow?: "ignore" | "throw"; } /** @@ -104,6 +109,9 @@ export class EventIterator readonly #limit: number; readonly #maxQueue: number; + readonly #overflow?: "ignore" | "throw"; + + private resolveNext?: Function; /** * The timer to track when this will idle out. @@ -130,6 +138,7 @@ export class EventIterator this.map = options.map ?? ((args) => args); this.#limit = options.limit ?? Infinity; this.#maxQueue = options.maxQueue ?? Infinity; + this.#overflow = options.overflow ?? "ignore"; this.#idle = options.idle; this.filter = options.filter ?? ((): boolean => true); this.onEnd = options.onEnd; @@ -158,6 +167,7 @@ export class EventIterator */ public end(): void { if (this.#ended) return; + this.resolveNext?.(); this.#ended = true; this.#queue = []; @@ -171,14 +181,9 @@ export class EventIterator // aliases to match usage in NATS and CoCalc. close = this.end; stop = this.end; - - drain(): void { - // just immediately end - this.end(); - // [ ] TODO: for compat. I'm not sure what this should be - // or if it matters... - // console.log("WARNING: TODO -- event-iterator drain not implemented"); - } + // TODO/worry: drain doesn't do anything special to address outstanding + // requests like NATS did. Probably this isn't the place for it... + drain = this.end; /** * The next value that's received from the EventEmitter. @@ -232,10 +237,18 @@ export class EventIterator // Once it has received at least one value, we will clear the timer (if defined), // and resolve with the new value: - this.emitter.once(this.event, () => { - if (idleTimer) clearTimeout(idleTimer); + const handleEvent = () => { + delete this.resolveNext; + if (idleTimer) { + clearTimeout(idleTimer); + } resolve(this.next()); - }); + }; + this.emitter.once(this.event, handleEvent); + this.resolveNext = () => { + this.emitter.removeListener(this.event, handleEvent); + resolve({ done: true, value: undefined }); + }; }); } @@ -266,10 +279,16 @@ export class EventIterator * Pushes a value into the queue. */ protected push(...args): void { + if (this.err) { + return; + } try { const value = this.map(args); this.#queue.push(value); while (this.#queue.length > this.#maxQueue && this.#queue.length > 0) { + if (this.#overflow == "throw") { + throw Error("maxQueue overflow"); + } this.#queue.shift(); } } catch (err) { From 0c48df61f3b05a26eb7f3c30977fe949411c8096 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 19 Jul 2025 17:14:46 +0000 Subject: [PATCH 024/270] more fs.watch unit tests --- .../backend/files/sandbox/sandbox.test.ts | 38 +++++++++++++++++++ src/packages/conat/llm/server.ts | 3 +- 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/src/packages/backend/files/sandbox/sandbox.test.ts b/src/packages/backend/files/sandbox/sandbox.test.ts index 1f29ae4b53..ebabcb5dba 100644 --- a/src/packages/backend/files/sandbox/sandbox.test.ts +++ b/src/packages/backend/files/sandbox/sandbox.test.ts @@ -126,6 +126,44 @@ describe("test watching a file and a folder in the sandbox", () => { const { done } = await w.next(); expect(done).toBe(true); }); + + it("watches a directory", async () => { + await fs.mkdir("folder"); + const w = await fs.watch("folder"); + + await fs.writeFile("folder/x", "hi"); + expect(await w.next()).toEqual({ + done: false, + value: { eventType: "rename", filename: "x" }, + }); + expect(await w.next()).toEqual({ + done: false, + value: { eventType: "change", filename: "x" }, + }); + + await fs.appendFile("folder/x", "xxx"); + expect(await w.next()).toEqual({ + done: false, + value: { eventType: "change", filename: "x" }, + }); + + await fs.writeFile("folder/z", "there"); + expect(await w.next()).toEqual({ + done: false, + value: { eventType: "rename", filename: "z" }, + }); + expect(await w.next()).toEqual({ + done: false, + value: { eventType: "change", filename: "z" }, + }); + + // this is correct -- from the node docs "On most platforms, 'rename' is emitted whenever a filename appears or disappears in the directory." + await fs.unlink("folder/z"); + expect(await w.next()).toEqual({ + done: false, + value: { eventType: "rename", filename: "z" }, + }); + }); }); afterAll(async () => { diff --git a/src/packages/conat/llm/server.ts b/src/packages/conat/llm/server.ts index e04571cbcc..c13ef765d2 100644 --- a/src/packages/conat/llm/server.ts +++ b/src/packages/conat/llm/server.ts @@ -90,8 +90,7 @@ async function handleMessage(mesg, evaluate) { }) => { seq += 1; try { - // mesg.respondSync({ text, error, seq }); - const { count } = await mesg.respond({ text, error, seq }); + await mesg.respond({ text, error, seq }); } catch (err) { logger.debug("WARNING: error sending response -- ", err); end(); From 41c3cd44318349a0155080e67bf3a9dd97220957 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 19 Jul 2025 17:49:34 +0000 Subject: [PATCH 025/270] add fs watch core --- .../backend/conat/files/test/watch.test.ts | 61 +++++++++++++ src/packages/conat/files/watch.ts | 90 +++++++++++++++++++ src/packages/conat/persist/server.ts | 2 +- 3 files changed, 152 insertions(+), 1 deletion(-) create mode 100644 src/packages/backend/conat/files/test/watch.test.ts create mode 100644 src/packages/conat/files/watch.ts diff --git a/src/packages/backend/conat/files/test/watch.test.ts b/src/packages/backend/conat/files/test/watch.test.ts new file mode 100644 index 0000000000..860c3f99d1 --- /dev/null +++ b/src/packages/backend/conat/files/test/watch.test.ts @@ -0,0 +1,61 @@ +import { before, after, client, wait } from "@cocalc/backend/conat/test/setup"; +import { watchServer, watchClient } from "@cocalc/conat/files/watch"; +import { SandboxedFilesystem } from "@cocalc/backend/files/sandbox"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "path"; +import { randomId } from "@cocalc/conat/names"; + +let tmp; +beforeAll(async () => { + await before(); + tmp = await mkdtemp(join(tmpdir(), `cocalc-${randomId()}`)); +}); +afterAll(async () => { + await after(); + try { + await rm(tmp, { force: true, recursive: true }); + } catch {} +}); + +describe("basic core of the async path watch functionality", () => { + let fs; + it("creates sandboxed filesystem", () => { + fs = new SandboxedFilesystem(tmp); + }); + + let server; + it("create watch server", () => { + server = watchServer({ client, subject: "foo", watch: fs.watch }); + }); + + it("create a file", async () => { + await fs.writeFile("a.txt", "hi"); + }); + + let w; + it("create a watcher client", async () => { + w = await watchClient({ client, subject: "foo", path: "a.txt" }); + }); + + it("observe watch works", async () => { + await fs.appendFile("a.txt", "foo"); + expect(await w.next()).toEqual({ + done: false, + value: [{ eventType: "change", filename: "a.txt" }, {}], + }); + + await fs.appendFile("a.txt", "bar"); + expect(await w.next()).toEqual({ + done: false, + value: [{ eventType: "change", filename: "a.txt" }, {}], + }); + }); + + it("close the watcher client frees up a server socket", async () => { + expect(Object.keys(server.sockets).length).toEqual(1); + w.close(); + await wait({ until: () => Object.keys(server.sockets).length == 0 }); + expect(Object.keys(server.sockets).length).toEqual(0); + }); +}); diff --git a/src/packages/conat/files/watch.ts b/src/packages/conat/files/watch.ts new file mode 100644 index 0000000000..97d0b3dd4a --- /dev/null +++ b/src/packages/conat/files/watch.ts @@ -0,0 +1,90 @@ +/* +Remotely proxying a fs.watch AsyncIterator over a Conat Socket. +*/ + +import { type Client as ConatClient } from "@cocalc/conat/core/client"; +import { + type ConatSocketServer, + type ServerSocket, +} from "@cocalc/conat/socket"; +import { EventIterator } from "@cocalc/util/event-iterator"; + +import { getLogger } from "@cocalc/conat/client"; + +const logger = getLogger("conat:files:watch"); + +// (path:string, options:WatchOptions) => AsyncIterator +type AsyncWatchFunction = any; +type WatchOptions = any; + +export function watchServer({ + client, + subject, + watch, +}: { + client: ConatClient; + subject: string; + watch: AsyncWatchFunction; +}) { + const server: ConatSocketServer = client.socket.listen(subject); + logger.debug("server: listening on ", { subject }); + + server.on("connection", (socket: ServerSocket) => { + logger.debug("server: got new connection", { + id: socket.id, + subject: socket.subject, + }); + let w: undefined | ReturnType = undefined; + socket.on("closed", () => { + w?.close(); + w = undefined; + }); + + socket.on("request", async (mesg) => { + try { + const { path, options } = mesg.data; + logger.debug("got request", { path, options }); + if (w != null) { + w.close(); + w = undefined; + } + w = await watch(path, options); + await mesg.respond(); + for await (const event of w) { + socket.write(event); + } + } catch (err) { + mesg.respondSync(null, { + headers: { error: `${err}`, code: err.code }, + }); + } + }); + }); + + return server; +} + +export async function watchClient({ + client, + subject, + path, + options, +}: { + client: ConatClient; + subject: string; + path: string; + options: WatchOptions; +}) { + const socket = await client.socket.connect(subject); + const iter = new EventIterator(socket, "data", { + onEnd: () => { + socket.close(); + }, + }); + socket.on("closed", () => { + iter.end(); + }); + // tell it what to watch + await socket.request({ path, options }); + return iter; +} diff --git a/src/packages/conat/persist/server.ts b/src/packages/conat/persist/server.ts index 5a270630e7..8ad17e4715 100644 --- a/src/packages/conat/persist/server.ts +++ b/src/packages/conat/persist/server.ts @@ -59,7 +59,6 @@ import { type ConatSocketServer, type ServerSocket, } from "@cocalc/conat/socket"; -import { getLogger } from "@cocalc/conat/client"; import type { StoredMessage, PersistentStream, @@ -70,6 +69,7 @@ import { throttle } from "lodash"; import { type SetOptions } from "./client"; import { once } from "@cocalc/util/async-utils"; import { UsageMonitor } from "@cocalc/conat/monitor/usage"; +import { getLogger } from "@cocalc/conat/client"; const logger = getLogger("persist:server"); From dd29234b94810491f8161416a45f751dc10c807f Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 19 Jul 2025 18:13:24 +0000 Subject: [PATCH 026/270] conat rpc fs.watch implemented and working here - obviously, still worried about leaks, typing, etc. But this works. --- .../backend/conat/files/local-path.ts | 2 +- .../conat/files/test/local-path.test.ts | 49 +++++ .../backend/conat/files/test/watch.test.ts | 4 +- src/packages/conat/files/fs.ts | 195 ++++++++++-------- src/packages/conat/files/watch.ts | 3 +- 5 files changed, 167 insertions(+), 86 deletions(-) diff --git a/src/packages/backend/conat/files/local-path.ts b/src/packages/backend/conat/files/local-path.ts index 1d0aa43a8e..3ba4fb5dad 100644 --- a/src/packages/backend/conat/files/local-path.ts +++ b/src/packages/backend/conat/files/local-path.ts @@ -27,7 +27,7 @@ export async function localPathFileserver({ return new SandboxedFilesystem(p); }, }); - return { server, client, path, service, close: () => server.end() }; + return { server, client, path, service, close: () => server.close() }; } function getProjectId(subject: string) { diff --git a/src/packages/backend/conat/files/test/local-path.test.ts b/src/packages/backend/conat/files/test/local-path.test.ts index 5eb9e9fadd..30c8d04476 100644 --- a/src/packages/backend/conat/files/test/local-path.test.ts +++ b/src/packages/backend/conat/files/test/local-path.test.ts @@ -254,6 +254,55 @@ describe("use all the standard api functions of fs", () => { const stats0 = await fs.stat("source1"); expect(stats0.isSymbolicLink()).toBe(false); }); + + it("watch a file", async () => { + await fs.writeFile("a.txt", "hi"); + const w = await fs.watch("a.txt"); + await fs.appendFile("a.txt", " there"); + expect(await w.next()).toEqual({ + done: false, + value: { eventType: "change", filename: "a.txt" }, + }); + }); + + it("watch a directory", async () => { + const FOLDER = randomId(); + await fs.mkdir(FOLDER); + const w = await fs.watch(FOLDER); + + await fs.writeFile(join(FOLDER, "x"), "hi"); + expect(await w.next()).toEqual({ + done: false, + value: { eventType: "rename", filename: "x" }, + }); + expect(await w.next()).toEqual({ + done: false, + value: { eventType: "change", filename: "x" }, + }); + + await fs.appendFile(join(FOLDER, "x"), "xxx"); + expect(await w.next()).toEqual({ + done: false, + value: { eventType: "change", filename: "x" }, + }); + + await fs.writeFile(join(FOLDER, "z"), "there"); + expect(await w.next()).toEqual({ + done: false, + value: { eventType: "rename", filename: "z" }, + }); + expect(await w.next()).toEqual({ + done: false, + value: { eventType: "change", filename: "z" }, + }); + + // this is correct -- from the node docs "On most platforms, 'rename' is emitted whenever a filename appears or disappears in the directory." + await fs.unlink(join(FOLDER, "z")); + expect(await w.next()).toEqual({ + done: false, + value: { eventType: "rename", filename: "z" }, + }); + }); }); describe("security: dangerous symlinks can't be followed", () => { diff --git a/src/packages/backend/conat/files/test/watch.test.ts b/src/packages/backend/conat/files/test/watch.test.ts index 860c3f99d1..fdcd760751 100644 --- a/src/packages/backend/conat/files/test/watch.test.ts +++ b/src/packages/backend/conat/files/test/watch.test.ts @@ -42,13 +42,13 @@ describe("basic core of the async path watch functionality", () => { await fs.appendFile("a.txt", "foo"); expect(await w.next()).toEqual({ done: false, - value: [{ eventType: "change", filename: "a.txt" }, {}], + value: { eventType: "change", filename: "a.txt" }, }); await fs.appendFile("a.txt", "bar"); expect(await w.next()).toEqual({ done: false, - value: [{ eventType: "change", filename: "a.txt" }, {}], + value: { eventType: "change", filename: "a.txt" }, }); }); diff --git a/src/packages/conat/files/fs.ts b/src/packages/conat/files/fs.ts index f0ce9e1f8e..afa7826f66 100644 --- a/src/packages/conat/files/fs.ts +++ b/src/packages/conat/files/fs.ts @@ -1,5 +1,6 @@ import { type Client } from "@cocalc/conat/core/client"; import { conat } from "@cocalc/conat/client"; +import { watchServer, watchClient } from "@cocalc/conat/files/watch"; export interface Filesystem { appendFile: (path: string, data: string | Buffer, encoding?) => Promise; @@ -27,6 +28,8 @@ export interface Filesystem { mtime: number | string | Date, ) => Promise; writeFile: (path: string, data: string | Buffer) => Promise; + // todo: typing + watch: (path: string, options?) => Promise; } interface IStats { @@ -99,87 +102,108 @@ interface Options { } export async function fsServer({ service, fs, client }: Options) { - return await (client ?? conat()).service( - `${service}.*`, - { - async appendFile(path: string, data: string | Buffer, encoding?) { - await (await fs(this.subject)).appendFile(path, data, encoding); - }, - async chmod(path: string, mode: string | number) { - await (await fs(this.subject)).chmod(path, mode); - }, - async constants(): Promise<{ [key: string]: number }> { - return await (await fs(this.subject)).constants(); - }, - async copyFile(src: string, dest: string) { - await (await fs(this.subject)).copyFile(src, dest); - }, - async cp(src: string, dest: string, options?) { - await (await fs(this.subject)).cp(src, dest, options); - }, - async exists(path: string) { - return await (await fs(this.subject)).exists(path); - }, - async link(existingPath: string, newPath: string) { - await (await fs(this.subject)).link(existingPath, newPath); - }, - async lstat(path: string): Promise { - return await (await fs(this.subject)).lstat(path); - }, - async mkdir(path: string, options?) { - await (await fs(this.subject)).mkdir(path, options); - }, - async readFile(path: string, encoding?) { - return await (await fs(this.subject)).readFile(path, encoding); - }, - async readdir(path: string) { - return await (await fs(this.subject)).readdir(path); - }, - async realpath(path: string) { - return await (await fs(this.subject)).realpath(path); - }, - async rename(oldPath: string, newPath: string) { - await (await fs(this.subject)).rename(oldPath, newPath); - }, - async rm(path: string, options?) { - await (await fs(this.subject)).rm(path, options); - }, - async rmdir(path: string, options?) { - await (await fs(this.subject)).rmdir(path, options); - }, - async stat(path: string): Promise { - const s = await (await fs(this.subject)).stat(path); - return { - ...s, - // for some reason these times get corrupted on transport from the nodejs datastructure, - // so we make them standard Date objects. - atime: new Date(s.atime), - mtime: new Date(s.mtime), - ctime: new Date(s.ctime), - birthtime: new Date(s.birthtime), - }; - }, - async symlink(target: string, path: string) { - await (await fs(this.subject)).symlink(target, path); - }, - async truncate(path: string, len?: number) { - await (await fs(this.subject)).truncate(path, len); - }, - async unlink(path: string) { - await (await fs(this.subject)).unlink(path); - }, - async utimes( - path: string, - atime: number | string | Date, - mtime: number | string | Date, - ) { - await (await fs(this.subject)).utimes(path, atime, mtime); - }, - async writeFile(path: string, data: string | Buffer) { - await (await fs(this.subject)).writeFile(path, data); - }, - }, - ); + client ??= conat(); + const subject = `${service}.*`; + const watches: { [subject: string]: any } = {}; + const sub = await client.service(subject, { + async appendFile(path: string, data: string | Buffer, encoding?) { + await (await fs(this.subject)).appendFile(path, data, encoding); + }, + async chmod(path: string, mode: string | number) { + await (await fs(this.subject)).chmod(path, mode); + }, + async constants(): Promise<{ [key: string]: number }> { + return await (await fs(this.subject)).constants(); + }, + async copyFile(src: string, dest: string) { + await (await fs(this.subject)).copyFile(src, dest); + }, + async cp(src: string, dest: string, options?) { + await (await fs(this.subject)).cp(src, dest, options); + }, + async exists(path: string) { + return await (await fs(this.subject)).exists(path); + }, + async link(existingPath: string, newPath: string) { + await (await fs(this.subject)).link(existingPath, newPath); + }, + async lstat(path: string): Promise { + return await (await fs(this.subject)).lstat(path); + }, + async mkdir(path: string, options?) { + await (await fs(this.subject)).mkdir(path, options); + }, + async readFile(path: string, encoding?) { + return await (await fs(this.subject)).readFile(path, encoding); + }, + async readdir(path: string) { + return await (await fs(this.subject)).readdir(path); + }, + async realpath(path: string) { + return await (await fs(this.subject)).realpath(path); + }, + async rename(oldPath: string, newPath: string) { + await (await fs(this.subject)).rename(oldPath, newPath); + }, + async rm(path: string, options?) { + await (await fs(this.subject)).rm(path, options); + }, + async rmdir(path: string, options?) { + await (await fs(this.subject)).rmdir(path, options); + }, + async stat(path: string): Promise { + const s = await (await fs(this.subject)).stat(path); + return { + ...s, + // for some reason these times get corrupted on transport from the nodejs datastructure, + // so we make them standard Date objects. + atime: new Date(s.atime), + mtime: new Date(s.mtime), + ctime: new Date(s.ctime), + birthtime: new Date(s.birthtime), + }; + }, + async symlink(target: string, path: string) { + await (await fs(this.subject)).symlink(target, path); + }, + async truncate(path: string, len?: number) { + await (await fs(this.subject)).truncate(path, len); + }, + async unlink(path: string) { + await (await fs(this.subject)).unlink(path); + }, + async utimes( + path: string, + atime: number | string | Date, + mtime: number | string | Date, + ) { + await (await fs(this.subject)).utimes(path, atime, mtime); + }, + async writeFile(path: string, data: string | Buffer) { + await (await fs(this.subject)).writeFile(path, data); + }, + async watch() { + const subject = this.subject!; + if (watches[subject] != null) { + return; + } + const f = await fs(subject); + watches[subject] = watchServer({ + client, + subject: subject!, + watch: f.watch, + }); + }, + }); + return { + close: () => { + for (const subject in watches) { + watches[subject].close(); + delete watches[subject]; + } + sub.close(); + }, + }; } export function fsClient({ @@ -189,7 +213,8 @@ export function fsClient({ client?: Client; subject: string; }): Filesystem { - let call = (client ?? conat()).call(subject); + client ??= conat(); + let call = client.call(subject); let constants: any = null; const stat0 = call.stat.bind(call); @@ -214,5 +239,11 @@ export function fsClient({ return stats; }; + const ensureWatchServerExists = call.watch.bind(call); + call.watch = async (path: string, options?) => { + await ensureWatchServerExists(path, options); + return await watchClient({ client, subject, path, options }); + }; + return call; } diff --git a/src/packages/conat/files/watch.ts b/src/packages/conat/files/watch.ts index 97d0b3dd4a..47dd65d939 100644 --- a/src/packages/conat/files/watch.ts +++ b/src/packages/conat/files/watch.ts @@ -73,10 +73,11 @@ export async function watchClient({ client: ConatClient; subject: string; path: string; - options: WatchOptions; + options?: WatchOptions; }) { const socket = await client.socket.connect(subject); const iter = new EventIterator(socket, "data", { + map: (args) => args[0], onEnd: () => { socket.close(); }, From 99656fd9e6257199f9f536c03950c4f9a82baa93 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 19 Jul 2025 20:00:01 +0000 Subject: [PATCH 027/270] broken work in progress creating a more generic and unit testable sync-doc --- src/packages/backend/conat/sync-doc/client.ts | 181 ++++++------------ .../backend/conat/sync-doc/syncstring.ts | 7 +- .../backend/conat/sync-doc/test/setup.ts | 1 + .../conat/sync-doc/test/syncstring.test.ts | 42 +++- src/packages/backend/conat/test/util.ts | 4 +- src/packages/conat/core/client.ts | 4 + 6 files changed, 113 insertions(+), 126 deletions(-) diff --git a/src/packages/backend/conat/sync-doc/client.ts b/src/packages/backend/conat/sync-doc/client.ts index 19813b772d..ee53ab1e4c 100644 --- a/src/packages/backend/conat/sync-doc/client.ts +++ b/src/packages/backend/conat/sync-doc/client.ts @@ -4,14 +4,15 @@ */ import { EventEmitter } from "events"; -import { bind_methods, keys } from "@cocalc/util/misc"; import { Client as Client0, FileWatcher as FileWatcher0, } from "@cocalc/sync/editor/generic/types"; -import { SyncTable } from "@cocalc/sync/table/synctable"; -import { ExecuteCodeOptionsWithCallback } from "@cocalc/util/types/execute-code"; -import { once } from "@cocalc/util/async-utils"; +import { conat as conat0 } from "@cocalc/backend/conat/conat"; +import { parseQueryWithOptions } from "@cocalc/sync/table/util"; +import { PubSub } from "@cocalc/conat/sync/pubsub"; +import { type Client as ConatClient } from "@cocalc/conat/core/client"; +import { type ConatSyncTable } from "@cocalc/conat/sync/synctable"; export class FileWatcher extends EventEmitter implements FileWatcher0 { private path: string; @@ -20,94 +21,78 @@ export class FileWatcher extends EventEmitter implements FileWatcher0 { this.path = path; console.log("FileWatcher", this.path); } - public close(): void {} + close(): void {} } export class Client extends EventEmitter implements Client0 { - private _client_id: string; - private initial_get_query: { [table: string]: any[] }; - public set_queries: any[] = []; - - constructor( - initial_get_query: { [table: string]: any[] }, - client_id: string, - ) { + private conat: ConatClient; + constructor(conat?: ConatClient) { super(); - this._client_id = client_id; - this.initial_get_query = initial_get_query; - bind_methods(this, ["query", "dbg", "query_cancel"]); + this.conat = conat ?? conat0(); } - public server_time(): Date { - return new Date(); - } + is_project = (): boolean => false; + is_browser = (): boolean => true; + is_compute_server = (): boolean => false; - isTestClient = () => { - return true; + dbg = (_f: string) => { + return (..._) => {}; }; - public is_project(): boolean { - return false; - } + is_connected = (): boolean => { + return this.conat.isConnected(); + }; - public is_browser(): boolean { - return true; - } + is_signed_in = (): boolean => { + return this.conat.isSignedIn(); + }; + + touch_project = (_): void => {}; - public is_compute_server(): boolean { + is_deleted = (_filename: string, _project_id?: string): boolean => { return false; - } + }; - public dbg(_f: string): Function { - // return (...args) => { - // console.log(_f, ...args); - // }; - return (..._) => {}; - } + set_deleted = (_filename: string, _project_id?: string): void => {}; - public mark_file(_opts: { - project_id: string; - path: string; - action: string; - ttl: number; - }): void { - //console.log("mark_file", opts); - } + synctable_conat = async (query0, options?): Promise => { + const { query } = parseQueryWithOptions(query0, options); + return await this.conat.sync.synctable({ + ...options, + query, + }); + }; - public log_error(opts: { - project_id: string; - path: string; - string_id: string; - error: any; - }): void { - console.log("log_error", opts); - } + pubsub_conat = async (opts): Promise => { + return new PubSub({ client: this.conat, ...opts }); + }; - public query(opts): void { - if (opts.options && opts.options.length === 1 && opts.options[0].set) { - // set query - this.set_queries.push(opts); - opts.cb(); - } else { - // get query -- returns predetermined result - const table = keys(opts.query)[0]; - let result = this.initial_get_query[table]; - if (result == null) { - result = []; - } - //console.log("GET QUERY ", table, result); - opts.cb(undefined, { query: { [table]: result } }); - } - } + // account_id or project_id + client_id = (): string => this.conat.id; + + server_time = (): Date => { + return new Date(); + }; + + ///////////////////////////////// + // EVERYTHING BELOW: TO REMOVE? + mark_file = (_): void => {}; + + alert_message = (_): void => {}; + + sage_session = (_): void => {}; - path_access(opts: { path: string; mode: string; cb: Function }): void { + shell = (_): void => {}; + + path_access = (opts: { path: string; mode: string; cb: Function }): void => { console.log("path_access", opts.path, opts.mode); opts.cb(true); - } - path_stat(opts: { path: string; cb: Function }): void { + }; + path_stat = (opts: { path: string; cb: Function }): void => { console.log("path_state", opts.path); opts.cb(true); - } + }; + async path_read(opts: { path: string; maxsize_MB?: number; @@ -128,54 +113,10 @@ export class Client extends EventEmitter implements Client0 { return new FileWatcher(opts.path); } - public is_connected(): boolean { - return true; - } - - public is_signed_in(): boolean { - return true; - } - - public touch_project(_): void {} - - public query_cancel(_): void {} + log_error = (_): void => {}; - public alert_message(_): void {} - - public is_deleted(_filename: string, _project_id?: string): boolean { - return false; - } - - public set_deleted(_filename: string, _project_id?: string): void {} - - async synctable_ephemeral( - _project_id: string, - query: any, - options: any, - throttle_changes?: number, - ): Promise { - const s = new SyncTable(query, options, this, throttle_changes); - await once(s, "connected"); - return s; - } - - async synctable_conat(_query: any): Promise { - throw Error("synctable_conat: not implemented"); - } - async pubsub_conat(_query: any): Promise { - throw Error("pubsub_conat: not implemented"); - } - - // account_id or project_id - public client_id(): string { - return this._client_id; - } - - public sage_session({ path }): void { - console.log(`sage_session: path=${path}`); - } - - public shell(opts: ExecuteCodeOptionsWithCallback): void { - console.log(`shell: opts=${JSON.stringify(opts)}`); - } + query = (_): void => { + throw Error("not implemented"); + }; + query_cancel = (_): void => {}; } diff --git a/src/packages/backend/conat/sync-doc/syncstring.ts b/src/packages/backend/conat/sync-doc/syncstring.ts index 64990e42c9..0d5a6e28f2 100644 --- a/src/packages/backend/conat/sync-doc/syncstring.ts +++ b/src/packages/backend/conat/sync-doc/syncstring.ts @@ -1,20 +1,21 @@ import { Client } from "./client"; import { SyncString } from "@cocalc/sync/editor/string/sync"; -import { a_txt } from "@cocalc/sync/editor/string/test/data"; import { once } from "@cocalc/util/async-utils"; import { type SyncDocFilesystem } from "@cocalc/sync/editor/generic/sync-doc"; +import { type Client as ConatClient } from "@cocalc/conat/core/client"; export default async function syncstring({ fs, project_id, path, + conat, }: { fs: SyncDocFilesystem; project_id: string; path: string; + conat?: ConatClient; }) { - const { client_id, init_queries } = a_txt(); - const client = new Client(init_queries, client_id); + const client = new Client(conat); const syncstring = new SyncString({ project_id, path, diff --git a/src/packages/backend/conat/sync-doc/test/setup.ts b/src/packages/backend/conat/sync-doc/test/setup.ts index bcef527ac6..dd976fb36a 100644 --- a/src/packages/backend/conat/sync-doc/test/setup.ts +++ b/src/packages/backend/conat/sync-doc/test/setup.ts @@ -2,6 +2,7 @@ import { before as before0, after as after0, } from "@cocalc/backend/conat/test/setup"; +export { connect, wait } from "@cocalc/backend/conat/test/setup"; import { createPathFileserver, cleanupFileservers, diff --git a/src/packages/backend/conat/sync-doc/test/syncstring.test.ts b/src/packages/backend/conat/sync-doc/test/syncstring.test.ts index fa637f28d0..06d3cb07ff 100644 --- a/src/packages/backend/conat/sync-doc/test/syncstring.test.ts +++ b/src/packages/backend/conat/sync-doc/test/syncstring.test.ts @@ -1,10 +1,10 @@ import syncstring from "@cocalc/backend/conat/sync-doc/syncstring"; -import { before, after, getFS, uuid } from "./setup"; +import { before, after, getFS, uuid, wait, connect } from "./setup"; beforeAll(before); afterAll(after); -describe("basic tests of a syncstring", () => { +describe("loading/saving syncstring to disk and setting values", () => { let s; const project_id = uuid(); let fs; @@ -60,3 +60,41 @@ describe("basic tests of a syncstring", () => { expect(s.version(s.versions()[1]).to_str()).toBe("test"); }); }); + +describe.only("sync with two copies of a syncstring", () => { + const project_id = uuid(); + let s1, s2, fs; + + it("creates the fs client and two copies of a syncstring", async () => { + fs = getFS(project_id); + await fs.writeFile("a.txt", "hello"); + s1 = await syncstring({ fs, project_id, path: "a.txt" }); + s2 = await syncstring({ fs, project_id, path: "a.txt", conat: connect() }); + expect(s1.to_str()).toBe("hello"); + expect(s2.to_str()).toBe("hello"); + expect(s1 === s2).toBe(false); + }); + + it("change one, commit and save, and see change reflected in the other", async () => { + s1.from_str("hello world"); + s1.commit(); + await s1.save(); + await wait({ + until: () => { + console.log(s1.to_str(), s2.to_str()); + console.log(s1.patches_table.dstream?.name, s2.patches_table.dstream?.name); + console.log(s1.patches_table.get(), s2.patches_table.get()); + console.log(s1.patch_list.patches, s2.patch_list.patches); + return s2.to_str() == "hello world"; + }, + min: 2000, + }); + }); + + it.skip("change second and see change reflected in first", async () => { + s2.from_str("hello world!"); + s2.commit(); + await s2.save(); + await wait({ until: () => s1.to_str() == "hello world!" }); + }); +}); diff --git a/src/packages/backend/conat/test/util.ts b/src/packages/backend/conat/test/util.ts index a6a9adb437..d3fa39ac63 100644 --- a/src/packages/backend/conat/test/util.ts +++ b/src/packages/backend/conat/test/util.ts @@ -4,12 +4,14 @@ export async function wait({ until: f, start = 5, decay = 1.2, + min = 5, max = 300, timeout = 10000, }: { until: Function; start?: number; decay?: number; + min?: number; max?: number; timeout?: number; }) { @@ -25,7 +27,7 @@ export async function wait({ start, decay, max, - min: 5, + min, timeout, }, ); diff --git a/src/packages/conat/core/client.ts b/src/packages/conat/core/client.ts index e09c94cd85..6bba37d09b 100644 --- a/src/packages/conat/core/client.ts +++ b/src/packages/conat/core/client.ts @@ -561,6 +561,10 @@ export class Client extends EventEmitter { setTimeout(() => this.conn.io.disconnect(), 1); }; + isConnected = () => this.state == "connected"; + + isSignedIn = () => !!(this.info?.user && !this.info?.user?.error); + // this has NO timeout by default waitUntilSignedIn = reuseInFlight( async ({ timeout }: { timeout?: number } = {}) => { From 346b9ef5099f1be3d2f1c1764632653e3abb5768 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 19 Jul 2025 22:02:43 +0000 Subject: [PATCH 028/270] sync-doc: integrating with conat --- src/packages/backend/conat/files/test/util.ts | 2 +- .../backend/conat/files/test/watch.test.ts | 2 +- src/packages/backend/conat/sync-doc/client.ts | 43 +++----------- .../backend/conat/sync-doc/syncstring.ts | 3 +- .../backend/conat/sync-doc/test/setup.ts | 10 +++- .../conat/sync-doc/test/syncstring.test.ts | 57 ++++++++++++------- src/packages/frontend/conat/client.ts | 5 +- src/packages/sync/table/util.ts | 3 + 8 files changed, 61 insertions(+), 64 deletions(-) diff --git a/src/packages/backend/conat/files/test/util.ts b/src/packages/backend/conat/files/test/util.ts index 2de57463ea..b50a114c5d 100644 --- a/src/packages/backend/conat/files/test/util.ts +++ b/src/packages/backend/conat/files/test/util.ts @@ -10,7 +10,7 @@ const servers: any[] = []; export async function createPathFileserver({ service = `fs-${randomId()}`, }: { service?: string } = {}) { - const tempDir = await mkdtemp(join(tmpdir(), `cocalc-${randomId()}`)); + const tempDir = await mkdtemp(join(tmpdir(), `cocalc-${randomId()}0`)); tempDirs.push(tempDir); const server = await localPathFileserver({ client, service, path: tempDir }); servers.push(server); diff --git a/src/packages/backend/conat/files/test/watch.test.ts b/src/packages/backend/conat/files/test/watch.test.ts index fdcd760751..0c3e61c376 100644 --- a/src/packages/backend/conat/files/test/watch.test.ts +++ b/src/packages/backend/conat/files/test/watch.test.ts @@ -9,7 +9,7 @@ import { randomId } from "@cocalc/conat/names"; let tmp; beforeAll(async () => { await before(); - tmp = await mkdtemp(join(tmpdir(), `cocalc-${randomId()}`)); + tmp = await mkdtemp(join(tmpdir(), `cocalc-${randomId()}0`)); }); afterAll(async () => { await after(); diff --git a/src/packages/backend/conat/sync-doc/client.ts b/src/packages/backend/conat/sync-doc/client.ts index ee53ab1e4c..98fd947a57 100644 --- a/src/packages/backend/conat/sync-doc/client.ts +++ b/src/packages/backend/conat/sync-doc/client.ts @@ -4,31 +4,17 @@ */ import { EventEmitter } from "events"; -import { - Client as Client0, - FileWatcher as FileWatcher0, -} from "@cocalc/sync/editor/generic/types"; -import { conat as conat0 } from "@cocalc/backend/conat/conat"; +import { Client as Client0 } from "@cocalc/sync/editor/generic/types"; import { parseQueryWithOptions } from "@cocalc/sync/table/util"; import { PubSub } from "@cocalc/conat/sync/pubsub"; import { type Client as ConatClient } from "@cocalc/conat/core/client"; import { type ConatSyncTable } from "@cocalc/conat/sync/synctable"; -export class FileWatcher extends EventEmitter implements FileWatcher0 { - private path: string; - constructor(path: string) { - super(); - this.path = path; - console.log("FileWatcher", this.path); - } - close(): void {} -} - export class Client extends EventEmitter implements Client0 { private conat: ConatClient; - constructor(conat?: ConatClient) { + constructor(conat: ConatClient) { super(); - this.conat = conat ?? conat0(); + this.conat = conat; } is_project = (): boolean => false; @@ -84,34 +70,21 @@ export class Client extends EventEmitter implements Client0 { shell = (_): void => {}; - path_access = (opts: { path: string; mode: string; cb: Function }): void => { - console.log("path_access", opts.path, opts.mode); + path_access = (opts): void => { opts.cb(true); }; - path_stat = (opts: { path: string; cb: Function }): void => { + path_stat = (opts): void => { console.log("path_state", opts.path); opts.cb(true); }; - async path_read(opts: { - path: string; - maxsize_MB?: number; - cb: Function; - }): Promise { - console.log("path_ready", opts.path); + async path_read(opts): Promise { opts.cb(true); } - async write_file(opts: { - path: string; - data: string; - cb: Function; - }): Promise { - console.log("write_file", opts.path, opts.data); + async write_file(opts): Promise { opts.cb(true); } - watch_file(opts: { path: string }): FileWatcher { - return new FileWatcher(opts.path); - } + watch_file(_): any {} log_error = (_): void => {}; diff --git a/src/packages/backend/conat/sync-doc/syncstring.ts b/src/packages/backend/conat/sync-doc/syncstring.ts index 0d5a6e28f2..ae1d17b335 100644 --- a/src/packages/backend/conat/sync-doc/syncstring.ts +++ b/src/packages/backend/conat/sync-doc/syncstring.ts @@ -13,14 +13,13 @@ export default async function syncstring({ fs: SyncDocFilesystem; project_id: string; path: string; - conat?: ConatClient; + conat: ConatClient; }) { const client = new Client(conat); const syncstring = new SyncString({ project_id, path, client, - ephemeral: true, fs, }); await once(syncstring, "ready"); diff --git a/src/packages/backend/conat/sync-doc/test/setup.ts b/src/packages/backend/conat/sync-doc/test/setup.ts index dd976fb36a..24c6f1e477 100644 --- a/src/packages/backend/conat/sync-doc/test/setup.ts +++ b/src/packages/backend/conat/sync-doc/test/setup.ts @@ -1,6 +1,7 @@ import { before as before0, after as after0, + client as client0, } from "@cocalc/backend/conat/test/setup"; export { connect, wait } from "@cocalc/backend/conat/test/setup"; import { @@ -11,6 +12,8 @@ import { type Filesystem } from "@cocalc/conat/files/fs"; export { uuid } from "@cocalc/util/misc"; import { fsClient } from "@cocalc/conat/files/fs"; +export { client0 as client }; + export let server; export async function before() { @@ -18,8 +21,11 @@ export async function before() { server = await createPathFileserver(); } -export function getFS(project_id: string): Filesystem { - return fsClient({ subject: `${server.service}.project-${project_id}` }); +export function getFS(project_id: string, client): Filesystem { + return fsClient({ + subject: `${server.service}.project-${project_id}`, + client, + }); } export async function after() { diff --git a/src/packages/backend/conat/sync-doc/test/syncstring.test.ts b/src/packages/backend/conat/sync-doc/test/syncstring.test.ts index 06d3cb07ff..f78a4116f1 100644 --- a/src/packages/backend/conat/sync-doc/test/syncstring.test.ts +++ b/src/packages/backend/conat/sync-doc/test/syncstring.test.ts @@ -7,23 +7,24 @@ afterAll(after); describe("loading/saving syncstring to disk and setting values", () => { let s; const project_id = uuid(); - let fs; + let fs, conat; it("creates the fs client", () => { - fs = getFS(project_id); + conat = connect(); + fs = getFS(project_id, conat); }); it("a syncstring associated to a file that does not exist on disk is initialized to the empty string", async () => { - s = await syncstring({ fs, project_id, path: "new.txt" }); + s = await syncstring({ fs, project_id, path: "new.txt", conat }); expect(s.to_str()).toBe(""); expect(s.versions().length).toBe(0); s.close(); }); it("a syncstring for editing a file that already exists on disk is initialized to that file", async () => { - fs = getFS(project_id); + fs = getFS(project_id, conat); await fs.writeFile("a.txt", "hello"); - s = await syncstring({ fs, project_id, path: "a.txt" }); + s = await syncstring({ fs, project_id, path: "a.txt", conat }); expect(s.fs).not.toEqual(undefined); }); @@ -61,17 +62,29 @@ describe("loading/saving syncstring to disk and setting values", () => { }); }); -describe.only("sync with two copies of a syncstring", () => { +describe("synchronized editing with two copies of a syncstring", () => { const project_id = uuid(); - let s1, s2, fs; + let s1, s2, fs1, fs2, client1, client2; it("creates the fs client and two copies of a syncstring", async () => { - fs = getFS(project_id); - await fs.writeFile("a.txt", "hello"); - s1 = await syncstring({ fs, project_id, path: "a.txt" }); - s2 = await syncstring({ fs, project_id, path: "a.txt", conat: connect() }); - expect(s1.to_str()).toBe("hello"); - expect(s2.to_str()).toBe("hello"); + client1 = connect(); + client2 = connect(); + fs1 = getFS(project_id, client1); + s1 = await syncstring({ + fs: fs1, + project_id, + path: "a.txt", + conat: client1, + }); + fs2 = getFS(project_id, client2); + s2 = await syncstring({ + fs: fs2, + project_id, + path: "a.txt", + conat: client2, + }); + expect(s1.to_str()).toBe(""); + expect(s2.to_str()).toBe(""); expect(s1 === s2).toBe(false); }); @@ -81,20 +94,26 @@ describe.only("sync with two copies of a syncstring", () => { await s1.save(); await wait({ until: () => { - console.log(s1.to_str(), s2.to_str()); - console.log(s1.patches_table.dstream?.name, s2.patches_table.dstream?.name); - console.log(s1.patches_table.get(), s2.patches_table.get()); - console.log(s1.patch_list.patches, s2.patch_list.patches); return s2.to_str() == "hello world"; }, - min: 2000, }); }); - it.skip("change second and see change reflected in first", async () => { + it("change second and see change reflected in first", async () => { s2.from_str("hello world!"); s2.commit(); await s2.save(); await wait({ until: () => s1.to_str() == "hello world!" }); }); + + it("view the history from each", async () => { + expect(s1.versions().length).toEqual(2); + expect(s2.versions().length).toEqual(2); + + const v1: string[] = [], + v2: string[] = []; + s1.show_history({ log: (x) => v1.push(x) }); + s2.show_history({ log: (x) => v2.push(x) }); + expect(v1).toEqual(v2); + }); }); diff --git a/src/packages/frontend/conat/client.ts b/src/packages/frontend/conat/client.ts index bf7af5088d..878ef21973 100644 --- a/src/packages/frontend/conat/client.ts +++ b/src/packages/frontend/conat/client.ts @@ -393,10 +393,7 @@ export class ConatClient extends EventEmitter { query0, options?, ): Promise => { - const { query, table } = parseQueryWithOptions(query0, options); - if (options?.project_id != null && query[table][0]["project_id"] === null) { - query[table][0]["project_id"] = options.project_id; - } + const { query } = parseQueryWithOptions(query0, options); return await this.conat().sync.synctable({ ...options, query, diff --git a/src/packages/sync/table/util.ts b/src/packages/sync/table/util.ts index 6815f3052d..dac8c0befd 100644 --- a/src/packages/sync/table/util.ts +++ b/src/packages/sync/table/util.ts @@ -46,6 +46,9 @@ export function parseQueryWithOptions(query, options) { query[table][0][k] = obj[k]; } } + if (options?.project_id != null && query[table][0]["project_id"] === null) { + query[table][0]["project_id"] = options.project_id; + } return { query, table }; } From ec86176a064b98aec34bc8376fe7ff354268dd56 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 19 Jul 2025 22:15:21 +0000 Subject: [PATCH 029/270] sync-doc conat: minor refactoring --- .../sync-doc/{client.ts => sync-client.ts} | 21 +++++++++------- .../backend/conat/sync-doc/syncstring.ts | 21 ++++++++++------ .../backend/conat/sync-doc/test/setup.ts | 11 +++++--- .../conat/sync-doc/test/syncstring.test.ts | 25 ++++++++----------- 4 files changed, 43 insertions(+), 35 deletions(-) rename src/packages/backend/conat/sync-doc/{client.ts => sync-client.ts} (80%) diff --git a/src/packages/backend/conat/sync-doc/client.ts b/src/packages/backend/conat/sync-doc/sync-client.ts similarity index 80% rename from src/packages/backend/conat/sync-doc/client.ts rename to src/packages/backend/conat/sync-doc/sync-client.ts index 98fd947a57..ebc6377f8e 100644 --- a/src/packages/backend/conat/sync-doc/client.ts +++ b/src/packages/backend/conat/sync-doc/sync-client.ts @@ -10,11 +10,14 @@ import { PubSub } from "@cocalc/conat/sync/pubsub"; import { type Client as ConatClient } from "@cocalc/conat/core/client"; import { type ConatSyncTable } from "@cocalc/conat/sync/synctable"; -export class Client extends EventEmitter implements Client0 { - private conat: ConatClient; - constructor(conat: ConatClient) { +export class SyncClient extends EventEmitter implements Client0 { + private client: ConatClient; + constructor(client: ConatClient) { super(); - this.conat = conat; + if (client == null) { + throw Error("client must be specified"); + } + this.client = client; } is_project = (): boolean => false; @@ -26,11 +29,11 @@ export class Client extends EventEmitter implements Client0 { }; is_connected = (): boolean => { - return this.conat.isConnected(); + return this.client.isConnected(); }; is_signed_in = (): boolean => { - return this.conat.isSignedIn(); + return this.client.isSignedIn(); }; touch_project = (_): void => {}; @@ -43,18 +46,18 @@ export class Client extends EventEmitter implements Client0 { synctable_conat = async (query0, options?): Promise => { const { query } = parseQueryWithOptions(query0, options); - return await this.conat.sync.synctable({ + return await this.client.sync.synctable({ ...options, query, }); }; pubsub_conat = async (opts): Promise => { - return new PubSub({ client: this.conat, ...opts }); + return new PubSub({ client: this.client, ...opts }); }; // account_id or project_id - client_id = (): string => this.conat.id; + client_id = (): string => this.client.id; server_time = (): Date => { return new Date(); diff --git a/src/packages/backend/conat/sync-doc/syncstring.ts b/src/packages/backend/conat/sync-doc/syncstring.ts index ae1d17b335..ff6837b58a 100644 --- a/src/packages/backend/conat/sync-doc/syncstring.ts +++ b/src/packages/backend/conat/sync-doc/syncstring.ts @@ -1,25 +1,30 @@ -import { Client } from "./client"; +import { SyncClient } from "./sync-client"; import { SyncString } from "@cocalc/sync/editor/string/sync"; import { once } from "@cocalc/util/async-utils"; -import { type SyncDocFilesystem } from "@cocalc/sync/editor/generic/sync-doc"; import { type Client as ConatClient } from "@cocalc/conat/core/client"; +import { fsClient } from "@cocalc/conat/files/fs"; export default async function syncstring({ - fs, project_id, path, - conat, + client, + // name of the file server that hosts this document: + service, }: { - fs: SyncDocFilesystem; project_id: string; path: string; - conat: ConatClient; + client: ConatClient; + service?: string; }) { - const client = new Client(conat); + const fs = fsClient({ + subject: `${service}.project-${project_id}`, + client, + }); + const syncClient = new SyncClient(client); const syncstring = new SyncString({ project_id, path, - client, + client: syncClient, fs, }); await once(syncstring, "ready"); diff --git a/src/packages/backend/conat/sync-doc/test/setup.ts b/src/packages/backend/conat/sync-doc/test/setup.ts index 24c6f1e477..122b4200f7 100644 --- a/src/packages/backend/conat/sync-doc/test/setup.ts +++ b/src/packages/backend/conat/sync-doc/test/setup.ts @@ -11,23 +11,28 @@ import { import { type Filesystem } from "@cocalc/conat/files/fs"; export { uuid } from "@cocalc/util/misc"; import { fsClient } from "@cocalc/conat/files/fs"; +import syncstring0 from "@cocalc/backend/conat/sync-doc/syncstring"; export { client0 as client }; -export let server; +export let server, fs; export async function before() { await before0(); server = await createPathFileserver(); } -export function getFS(project_id: string, client): Filesystem { +export function getFS(project_id: string, client?): Filesystem { return fsClient({ subject: `${server.service}.project-${project_id}`, - client, + client: client ?? client0, }); } +export async function syncstring(opts) { + return await syncstring0({ ...opts, service: server.service }); +} + export async function after() { await cleanupFileservers(); await after0(); diff --git a/src/packages/backend/conat/sync-doc/test/syncstring.test.ts b/src/packages/backend/conat/sync-doc/test/syncstring.test.ts index f78a4116f1..20ee35f1c4 100644 --- a/src/packages/backend/conat/sync-doc/test/syncstring.test.ts +++ b/src/packages/backend/conat/sync-doc/test/syncstring.test.ts @@ -1,5 +1,4 @@ -import syncstring from "@cocalc/backend/conat/sync-doc/syncstring"; -import { before, after, getFS, uuid, wait, connect } from "./setup"; +import { before, after, uuid, wait, connect, syncstring, getFS } from "./setup"; beforeAll(before); afterAll(after); @@ -7,24 +6,24 @@ afterAll(after); describe("loading/saving syncstring to disk and setting values", () => { let s; const project_id = uuid(); - let fs, conat; + let client; it("creates the fs client", () => { - conat = connect(); - fs = getFS(project_id, conat); + client = connect(); }); it("a syncstring associated to a file that does not exist on disk is initialized to the empty string", async () => { - s = await syncstring({ fs, project_id, path: "new.txt", conat }); + s = await syncstring({ project_id, path: "new.txt", client }); expect(s.to_str()).toBe(""); expect(s.versions().length).toBe(0); s.close(); }); + let fs; it("a syncstring for editing a file that already exists on disk is initialized to that file", async () => { - fs = getFS(project_id, conat); + fs = getFS(project_id, client); await fs.writeFile("a.txt", "hello"); - s = await syncstring({ fs, project_id, path: "a.txt", conat }); + s = await syncstring({ project_id, path: "a.txt", client }); expect(s.fs).not.toEqual(undefined); }); @@ -64,24 +63,20 @@ describe("loading/saving syncstring to disk and setting values", () => { describe("synchronized editing with two copies of a syncstring", () => { const project_id = uuid(); - let s1, s2, fs1, fs2, client1, client2; + let s1, s2, client1, client2; it("creates the fs client and two copies of a syncstring", async () => { client1 = connect(); client2 = connect(); - fs1 = getFS(project_id, client1); s1 = await syncstring({ - fs: fs1, project_id, path: "a.txt", - conat: client1, + client: client1, }); - fs2 = getFS(project_id, client2); s2 = await syncstring({ - fs: fs2, project_id, path: "a.txt", - conat: client2, + client: client2, }); expect(s1.to_str()).toBe(""); expect(s2.to_str()).toBe(""); From 366ca14731598ba160985dc1a17c321075e1130f Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 19 Jul 2025 23:11:09 +0000 Subject: [PATCH 030/270] make SyncString available from conat core client --- .../backend/conat/files/local-path.ts | 11 ++++---- .../{sync-doc/test => test/sync-doc}/setup.ts | 5 ++-- .../test => test/sync-doc}/syncstring.test.ts | 24 +++++++++++------ src/packages/conat/core/client.ts | 22 ++++++++++++++++ src/packages/conat/files/fs.ts | 1 + src/packages/conat/package.json | 16 ++++-------- .../conat/sync-doc/sync-client.ts | 0 .../conat/sync-doc/syncstring.ts | 26 +++++++++---------- src/packages/frontend/conat/client.ts | 3 --- src/packages/jupyter/package.json | 1 - src/packages/pnpm-lock.yaml | 20 +++----------- src/packages/sync/editor/generic/sync-doc.ts | 6 ++--- src/packages/util/package.json | 1 - 13 files changed, 72 insertions(+), 64 deletions(-) rename src/packages/backend/conat/{sync-doc/test => test/sync-doc}/setup.ts (81%) rename src/packages/backend/conat/{sync-doc/test => test/sync-doc}/syncstring.test.ts (84%) rename src/packages/{backend => }/conat/sync-doc/sync-client.ts (100%) rename src/packages/{backend => }/conat/sync-doc/syncstring.ts (75%) diff --git a/src/packages/backend/conat/files/local-path.ts b/src/packages/backend/conat/files/local-path.ts index 3ba4fb5dad..bc905561a6 100644 --- a/src/packages/backend/conat/files/local-path.ts +++ b/src/packages/backend/conat/files/local-path.ts @@ -1,20 +1,21 @@ -import { fsServer } from "@cocalc/conat/files/fs"; +import { fsServer, DEFAULT_FILE_SERVICE } from "@cocalc/conat/files/fs"; import { SandboxedFilesystem } from "@cocalc/backend/files/sandbox"; import { mkdir } from "fs/promises"; import { join } from "path"; import { isValidUUID } from "@cocalc/util/misc"; -import { type Client, getClient } from "@cocalc/conat/core/client"; +import { type Client } from "@cocalc/conat/core/client"; +import { conat } from "@cocalc/backend/conat/conat"; export async function localPathFileserver({ - service, path, + service = DEFAULT_FILE_SERVICE, client, }: { - service: string; path: string; + service?: string; client?: Client; }) { - client ??= getClient(); + client ??= conat(); const server = await fsServer({ service, client, diff --git a/src/packages/backend/conat/sync-doc/test/setup.ts b/src/packages/backend/conat/test/sync-doc/setup.ts similarity index 81% rename from src/packages/backend/conat/sync-doc/test/setup.ts rename to src/packages/backend/conat/test/sync-doc/setup.ts index 122b4200f7..d89109ff7a 100644 --- a/src/packages/backend/conat/sync-doc/test/setup.ts +++ b/src/packages/backend/conat/test/sync-doc/setup.ts @@ -11,7 +11,8 @@ import { import { type Filesystem } from "@cocalc/conat/files/fs"; export { uuid } from "@cocalc/util/misc"; import { fsClient } from "@cocalc/conat/files/fs"; -import syncstring0 from "@cocalc/backend/conat/sync-doc/syncstring"; +import { syncstring as syncstring0 } from "@cocalc/conat/sync-doc/syncstring"; +import { SyncString } from "@cocalc/sync/editor/string/sync"; export { client0 as client }; @@ -29,7 +30,7 @@ export function getFS(project_id: string, client?): Filesystem { }); } -export async function syncstring(opts) { +export async function syncstring(opts): Promise { return await syncstring0({ ...opts, service: server.service }); } diff --git a/src/packages/backend/conat/sync-doc/test/syncstring.test.ts b/src/packages/backend/conat/test/sync-doc/syncstring.test.ts similarity index 84% rename from src/packages/backend/conat/sync-doc/test/syncstring.test.ts rename to src/packages/backend/conat/test/sync-doc/syncstring.test.ts index 20ee35f1c4..c02c2caed8 100644 --- a/src/packages/backend/conat/sync-doc/test/syncstring.test.ts +++ b/src/packages/backend/conat/test/sync-doc/syncstring.test.ts @@ -1,4 +1,4 @@ -import { before, after, uuid, wait, connect, syncstring, getFS } from "./setup"; +import { before, after, uuid, wait, connect, server } from "./setup"; beforeAll(before); afterAll(after); @@ -13,7 +13,11 @@ describe("loading/saving syncstring to disk and setting values", () => { }); it("a syncstring associated to a file that does not exist on disk is initialized to the empty string", async () => { - s = await syncstring({ project_id, path: "new.txt", client }); + s = await client.sync.string({ + project_id, + path: "new.txt", + service: server.service, + }); expect(s.to_str()).toBe(""); expect(s.versions().length).toBe(0); s.close(); @@ -21,9 +25,13 @@ describe("loading/saving syncstring to disk and setting values", () => { let fs; it("a syncstring for editing a file that already exists on disk is initialized to that file", async () => { - fs = getFS(project_id, client); + fs = client.fs({ project_id, service: server.service }); await fs.writeFile("a.txt", "hello"); - s = await syncstring({ project_id, path: "a.txt", client }); + s = await client.sync.string({ + project_id, + path: "a.txt", + service: server.service, + }); expect(s.fs).not.toEqual(undefined); }); @@ -68,15 +76,15 @@ describe("synchronized editing with two copies of a syncstring", () => { it("creates the fs client and two copies of a syncstring", async () => { client1 = connect(); client2 = connect(); - s1 = await syncstring({ + s1 = await client1.sync.string({ project_id, path: "a.txt", - client: client1, + service: server.service, }); - s2 = await syncstring({ + s2 = await client2.sync.string({ project_id, path: "a.txt", - client: client2, + service: server.service, }); expect(s1.to_str()).toBe(""); expect(s2.to_str()).toBe(""); diff --git a/src/packages/conat/core/client.ts b/src/packages/conat/core/client.ts index 6bba37d09b..eedf71e197 100644 --- a/src/packages/conat/core/client.ts +++ b/src/packages/conat/core/client.ts @@ -244,6 +244,12 @@ import { } from "@cocalc/conat/sync/dstream"; import { akv, type AKV } from "@cocalc/conat/sync/akv"; import { astream, type AStream } from "@cocalc/conat/sync/astream"; +import { + syncstring, + type SyncString, + type SyncStringOptions, +} from "@cocalc/conat/sync-doc/syncstring"; +import { fsClient, DEFAULT_FILE_SERVICE } from "@cocalc/conat/files/fs"; import TTL from "@isaacs/ttlcache"; import { ConatSocketServer, @@ -1462,6 +1468,19 @@ export class Client extends EventEmitter { return sub; }; + fs = ({ + project_id, + service = DEFAULT_FILE_SERVICE, + }: { + project_id: string; + service?: string; + }) => { + return fsClient({ + subject: `${service}.project-${project_id}`, + client: this, + }); + }; + sync = { dkv: async (opts: DKVOptions): Promise> => await dkv({ ...opts, client: this }), @@ -1475,6 +1494,9 @@ export class Client extends EventEmitter { await astream({ ...opts, client: this }), synctable: async (opts: SyncTableOptions): Promise => await createSyncTable({ ...opts, client: this }), + string: async ( + opts: Omit, + ): Promise => await syncstring({ ...opts, client: this }), }; socket = { diff --git a/src/packages/conat/files/fs.ts b/src/packages/conat/files/fs.ts index afa7826f66..a06baa02bb 100644 --- a/src/packages/conat/files/fs.ts +++ b/src/packages/conat/files/fs.ts @@ -1,6 +1,7 @@ import { type Client } from "@cocalc/conat/core/client"; import { conat } from "@cocalc/conat/client"; import { watchServer, watchClient } from "@cocalc/conat/files/watch"; +export const DEFAULT_FILE_SERVICE = "fs"; export interface Filesystem { appendFile: (path: string, data: string | Buffer, encoding?) => Promise; diff --git a/src/packages/conat/package.json b/src/packages/conat/package.json index 79bfe77778..1d8dd0f6ca 100644 --- a/src/packages/conat/package.json +++ b/src/packages/conat/package.json @@ -7,6 +7,8 @@ "./llm/*": "./dist/llm/*.js", "./socket": "./dist/socket/index.js", "./socket/*": "./dist/socket/*.js", + "./sync-doc": "./dist/sync-doc/index.js", + "./sync-doc/*": "./dist/sync-doc/*.js", "./hub/changefeeds": "./dist/hub/changefeeds/index.js", "./hub/api": "./dist/hub/api/index.js", "./hub/api/*": "./dist/hub/api/*.js", @@ -24,21 +26,14 @@ "test": "pnpm exec jest", "depcheck": "pnpx depcheck --ignores events" }, - "files": [ - "dist/**", - "README.md", - "package.json" - ], + "files": ["dist/**", "README.md", "package.json"], "author": "SageMath, Inc.", - "keywords": [ - "utilities", - "conat", - "cocalc" - ], + "keywords": ["utilities", "conat", "cocalc"], "license": "SEE LICENSE.md", "dependencies": { "@cocalc/comm": "workspace:*", "@cocalc/conat": "workspace:*", + "@cocalc/sync": "workspace:*", "@cocalc/util": "workspace:*", "@isaacs/ttlcache": "^1.4.1", "@msgpack/msgpack": "^3.1.1", @@ -56,7 +51,6 @@ }, "devDependencies": { "@types/better-sqlite3": "^7.6.13", - "@types/json-stable-stringify": "^1.0.32", "@types/lodash": "^4.14.202", "@types/node": "^18.16.14" }, diff --git a/src/packages/backend/conat/sync-doc/sync-client.ts b/src/packages/conat/sync-doc/sync-client.ts similarity index 100% rename from src/packages/backend/conat/sync-doc/sync-client.ts rename to src/packages/conat/sync-doc/sync-client.ts diff --git a/src/packages/backend/conat/sync-doc/syncstring.ts b/src/packages/conat/sync-doc/syncstring.ts similarity index 75% rename from src/packages/backend/conat/sync-doc/syncstring.ts rename to src/packages/conat/sync-doc/syncstring.ts index ff6837b58a..2c7fcb795e 100644 --- a/src/packages/backend/conat/sync-doc/syncstring.ts +++ b/src/packages/conat/sync-doc/syncstring.ts @@ -2,24 +2,24 @@ import { SyncClient } from "./sync-client"; import { SyncString } from "@cocalc/sync/editor/string/sync"; import { once } from "@cocalc/util/async-utils"; import { type Client as ConatClient } from "@cocalc/conat/core/client"; -import { fsClient } from "@cocalc/conat/files/fs"; -export default async function syncstring({ - project_id, - path, - client, - // name of the file server that hosts this document: - service, -}: { +export interface SyncStringOptions { project_id: string; path: string; client: ConatClient; + // name of the file server that hosts this document: service?: string; -}) { - const fs = fsClient({ - subject: `${service}.project-${project_id}`, - client, - }); +} + +export type { SyncString }; + +export async function syncstring({ + project_id, + path, + client, + service, +}: SyncStringOptions): Promise { + const fs = client.fs({ service, project_id }); const syncClient = new SyncClient(client); const syncstring = new SyncString({ project_id, diff --git a/src/packages/frontend/conat/client.ts b/src/packages/frontend/conat/client.ts index 878ef21973..dedb2d4533 100644 --- a/src/packages/frontend/conat/client.ts +++ b/src/packages/frontend/conat/client.ts @@ -46,7 +46,6 @@ import { deleteRememberMe, setRememberMe, } from "@cocalc/frontend/misc/remember-me"; -import { fsClient } from "@cocalc/conat/files/fs"; export interface ConatConnectionStatus { state: "connected" | "disconnected"; @@ -513,8 +512,6 @@ export class ConatClient extends EventEmitter { }; refCacheInfo = () => refCacheInfo(); - - fsClient = (subject: string) => fsClient({ subject, client: this.conat() }); } function setDeleted({ project_id, path, deleted }) { diff --git a/src/packages/jupyter/package.json b/src/packages/jupyter/package.json index d1e3f3ab14..a109440ca8 100644 --- a/src/packages/jupyter/package.json +++ b/src/packages/jupyter/package.json @@ -63,7 +63,6 @@ "zeromq": "^6.4.2" }, "devDependencies": { - "@types/json-stable-stringify": "^1.0.32", "@types/node": "^18.16.14", "@types/node-cleanup": "^2.1.2" }, diff --git a/src/packages/pnpm-lock.yaml b/src/packages/pnpm-lock.yaml index 930789121f..135d788520 100644 --- a/src/packages/pnpm-lock.yaml +++ b/src/packages/pnpm-lock.yaml @@ -170,6 +170,9 @@ importers: '@cocalc/conat': specifier: workspace:* version: 'link:' + '@cocalc/sync': + specifier: workspace:* + version: link:../sync '@cocalc/util': specifier: workspace:* version: link:../util @@ -216,9 +219,6 @@ importers: '@types/better-sqlite3': specifier: ^7.6.13 version: 7.6.13 - '@types/json-stable-stringify': - specifier: ^1.0.32 - version: 1.2.0 '@types/lodash': specifier: ^4.14.202 version: 4.17.20 @@ -899,9 +899,6 @@ importers: specifier: ^6.4.2 version: 6.5.0 devDependencies: - '@types/json-stable-stringify': - specifier: ^1.0.32 - version: 1.2.0 '@types/node': specifier: ^18.16.14 version: 18.19.118 @@ -1849,9 +1846,6 @@ importers: specifier: ^1.3.0 version: 1.3.0 devDependencies: - '@types/json-stable-stringify': - specifier: ^1.0.32 - version: 1.2.0 '@types/lodash': specifier: ^4.14.202 version: 4.17.20 @@ -4269,10 +4263,6 @@ packages: '@types/json-schema@7.0.15': resolution: {integrity: sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==} - '@types/json-stable-stringify@1.2.0': - resolution: {integrity: sha512-PEHY3ohqolHqAzDyB1+31tFaAMnoLN7x/JgdcGmNZ2uvtEJ6rlFCUYNQc0Xe754xxCYLNGZbLUGydSE6tS4S9A==} - deprecated: This is a stub types definition. json-stable-stringify provides its own type definitions, so you do not need this installed. - '@types/katex@0.16.7': resolution: {integrity: sha512-HMwFiRujE5PjrgwHQ25+bsLJgowjGjm5Z8FVSf0N6PwgJrwxH0QxzHYDcKsTfV3wva0vzrpqMTJS2jXPr5BMEQ==} @@ -14546,10 +14536,6 @@ snapshots: '@types/json-schema@7.0.15': {} - '@types/json-stable-stringify@1.2.0': - dependencies: - json-stable-stringify: 1.3.0 - '@types/katex@0.16.7': {} '@types/keyv@3.1.4': diff --git a/src/packages/sync/editor/generic/sync-doc.ts b/src/packages/sync/editor/generic/sync-doc.ts index cb971876cd..2fed66dd7a 100644 --- a/src/packages/sync/editor/generic/sync-doc.ts +++ b/src/packages/sync/editor/generic/sync-doc.ts @@ -2903,9 +2903,6 @@ export class SyncDoc extends EventEmitter { if (this.fs != null) { return; } - if (this.client.path_exists == null) { - throw Error("legacy clients must define path_exists"); - } const dbg = this.dbg("update_watch_path"); if (this.file_watcher != null) { // clean up @@ -2932,6 +2929,9 @@ export class SyncDoc extends EventEmitter { if (this.state === "closed") { throw Error("must not be closed"); } + if (this.client.path_exists == null) { + throw Error("legacy clients must define path_exists"); + } this.watch_path = path; try { if (!(await callback2(this.client.path_exists, { path }))) { diff --git a/src/packages/util/package.json b/src/packages/util/package.json index e9656ac1e3..62d8367572 100644 --- a/src/packages/util/package.json +++ b/src/packages/util/package.json @@ -71,7 +71,6 @@ }, "homepage": "https://github.com/sagemathinc/cocalc/tree/master/src/packages/util", "devDependencies": { - "@types/json-stable-stringify": "^1.0.32", "@types/lodash": "^4.14.202", "@types/node": "^18.16.14", "@types/seedrandom": "^3.0.8", From 205f14db5730889369c479e4b8b4ac998fb9308b Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 19 Jul 2025 23:29:25 +0000 Subject: [PATCH 031/270] conat sync/fileserver: integrate with hub --- src/packages/hub/hub.ts | 11 ++++++++++- src/packages/server/conat/index.ts | 16 +++++++++++++++- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/src/packages/hub/hub.ts b/src/packages/hub/hub.ts index 5e5b84d4f1..500b48ad3d 100644 --- a/src/packages/hub/hub.ts +++ b/src/packages/hub/hub.ts @@ -44,6 +44,7 @@ import { initConatChangefeedServer, initConatApi, initConatPersist, + initConatFileserver, } from "@cocalc/server/conat"; import { initConatServer } from "@cocalc/server/conat/socketio"; @@ -182,6 +183,10 @@ async function startServer(): Promise { await initConatServer({ kucalc: program.mode == "kucalc" }); } + if (program.conatFileserver || program.conatServer) { + await initConatFileserver(); + } + if (program.conatApi || program.conatServer) { await initConatApi(); await initConatChangefeedServer(); @@ -318,12 +323,16 @@ async function main(): Promise { ) .option( "--conat-server", - "run a hub that provides a single-core conat server (i.e., conat-router but integrated with the http server), api, and persistence, along with an http server. This is for dev and small deployments of cocalc (and if given, do not bother with --conat-[core|api|persist] below.)", + "run a hub that provides a single-core conat server (i.e., conat-router but integrated with the http server), api, and persistence, fileserver, along with an http server. This is for dev and small deployments of cocalc (and if given, do not bother with --conat-[core|api|persist] below.)", ) .option( "--conat-router", "run a hub that provides the core conat communication layer server over a websocket (but not http server).", ) + .option( + "--conat-fileserver", + "run a hub that provides a fileserver conat service", + ) .option( "--conat-api", "run a hub that connect to conat-router and provides the standard conat API services, e.g., basic api, LLM's, changefeeds, http file upload/download, etc. There must be at least one of these. You can increase or decrease the number of these servers with no coordination needed.", diff --git a/src/packages/server/conat/index.ts b/src/packages/server/conat/index.ts index 56e08e45b5..49e3d4b142 100644 --- a/src/packages/server/conat/index.ts +++ b/src/packages/server/conat/index.ts @@ -5,7 +5,8 @@ import { init as initLLM } from "./llm"; import { loadConatConfiguration } from "./configuration"; import { createTimeService } from "@cocalc/conat/service/time"; export { initConatPersist } from "./persist"; -import { conatApiCount } from "@cocalc/backend/data"; +import { conatApiCount, projects } from "@cocalc/backend/data"; +import { localPathFileserver } from "@cocalc/backend/conat/files/local-path"; export { loadConatConfiguration }; @@ -30,3 +31,16 @@ export async function initConatApi() { initLLM(); createTimeService(); } + +export async function initConatFileserver() { + await loadConatConfiguration(); + const i = projects.indexOf("/[project_id]"); + if (i == -1) { + throw Error( + `projects must be a template containing /[project_id] -- ${projects}`, + ); + } + const path = projects.slice(0, i); + logger.debug("initFileserver", { path }); + localPathFileserver({ path }); +} From 67a3d1d4cbef72ce06d130d29d49448d48cbb3bd Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 19 Jul 2025 23:46:49 +0000 Subject: [PATCH 032/270] conat sync-doc: make sync.string not async to match existing api usage - i might change this everywhere; not sure --- .../backend/conat/test/sync-doc/setup.ts | 17 +---------------- .../conat/test/sync-doc/syncstring.test.ts | 15 ++++++++++----- src/packages/conat/core/client.ts | 5 ++--- src/packages/conat/sync-doc/syncstring.ts | 9 +++------ .../frontend/frame-editors/generic/client.ts | 1 + 5 files changed, 17 insertions(+), 30 deletions(-) diff --git a/src/packages/backend/conat/test/sync-doc/setup.ts b/src/packages/backend/conat/test/sync-doc/setup.ts index d89109ff7a..0155bb46f4 100644 --- a/src/packages/backend/conat/test/sync-doc/setup.ts +++ b/src/packages/backend/conat/test/sync-doc/setup.ts @@ -3,16 +3,12 @@ import { after as after0, client as client0, } from "@cocalc/backend/conat/test/setup"; -export { connect, wait } from "@cocalc/backend/conat/test/setup"; +export { connect, wait, once } from "@cocalc/backend/conat/test/setup"; import { createPathFileserver, cleanupFileservers, } from "@cocalc/backend/conat/files/test/util"; -import { type Filesystem } from "@cocalc/conat/files/fs"; export { uuid } from "@cocalc/util/misc"; -import { fsClient } from "@cocalc/conat/files/fs"; -import { syncstring as syncstring0 } from "@cocalc/conat/sync-doc/syncstring"; -import { SyncString } from "@cocalc/sync/editor/string/sync"; export { client0 as client }; @@ -23,17 +19,6 @@ export async function before() { server = await createPathFileserver(); } -export function getFS(project_id: string, client?): Filesystem { - return fsClient({ - subject: `${server.service}.project-${project_id}`, - client: client ?? client0, - }); -} - -export async function syncstring(opts): Promise { - return await syncstring0({ ...opts, service: server.service }); -} - export async function after() { await cleanupFileservers(); await after0(); diff --git a/src/packages/backend/conat/test/sync-doc/syncstring.test.ts b/src/packages/backend/conat/test/sync-doc/syncstring.test.ts index c02c2caed8..d1d73332a7 100644 --- a/src/packages/backend/conat/test/sync-doc/syncstring.test.ts +++ b/src/packages/backend/conat/test/sync-doc/syncstring.test.ts @@ -1,4 +1,4 @@ -import { before, after, uuid, wait, connect, server } from "./setup"; +import { before, after, uuid, wait, connect, server, once } from "./setup"; beforeAll(before); afterAll(after); @@ -13,11 +13,12 @@ describe("loading/saving syncstring to disk and setting values", () => { }); it("a syncstring associated to a file that does not exist on disk is initialized to the empty string", async () => { - s = await client.sync.string({ + s = client.sync.string({ project_id, path: "new.txt", service: server.service, }); + await once(s, "ready"); expect(s.to_str()).toBe(""); expect(s.versions().length).toBe(0); s.close(); @@ -27,11 +28,12 @@ describe("loading/saving syncstring to disk and setting values", () => { it("a syncstring for editing a file that already exists on disk is initialized to that file", async () => { fs = client.fs({ project_id, service: server.service }); await fs.writeFile("a.txt", "hello"); - s = await client.sync.string({ + s = client.sync.string({ project_id, path: "a.txt", service: server.service, }); + await once(s, "ready"); expect(s.fs).not.toEqual(undefined); }); @@ -76,16 +78,19 @@ describe("synchronized editing with two copies of a syncstring", () => { it("creates the fs client and two copies of a syncstring", async () => { client1 = connect(); client2 = connect(); - s1 = await client1.sync.string({ + s1 = client1.sync.string({ project_id, path: "a.txt", service: server.service, }); - s2 = await client2.sync.string({ + await once(s1, "ready"); + + s2 = client2.sync.string({ project_id, path: "a.txt", service: server.service, }); + await once(s2, "ready"); expect(s1.to_str()).toBe(""); expect(s2.to_str()).toBe(""); expect(s1 === s2).toBe(false); diff --git a/src/packages/conat/core/client.ts b/src/packages/conat/core/client.ts index eedf71e197..9a3d8d3dc4 100644 --- a/src/packages/conat/core/client.ts +++ b/src/packages/conat/core/client.ts @@ -1494,9 +1494,8 @@ export class Client extends EventEmitter { await astream({ ...opts, client: this }), synctable: async (opts: SyncTableOptions): Promise => await createSyncTable({ ...opts, client: this }), - string: async ( - opts: Omit, - ): Promise => await syncstring({ ...opts, client: this }), + string: (opts: Omit): SyncString => + syncstring({ ...opts, client: this }), }; socket = { diff --git a/src/packages/conat/sync-doc/syncstring.ts b/src/packages/conat/sync-doc/syncstring.ts index 2c7fcb795e..474acd8488 100644 --- a/src/packages/conat/sync-doc/syncstring.ts +++ b/src/packages/conat/sync-doc/syncstring.ts @@ -1,6 +1,5 @@ import { SyncClient } from "./sync-client"; import { SyncString } from "@cocalc/sync/editor/string/sync"; -import { once } from "@cocalc/util/async-utils"; import { type Client as ConatClient } from "@cocalc/conat/core/client"; export interface SyncStringOptions { @@ -13,20 +12,18 @@ export interface SyncStringOptions { export type { SyncString }; -export async function syncstring({ +export function syncstring({ project_id, path, client, service, -}: SyncStringOptions): Promise { +}: SyncStringOptions): SyncString { const fs = client.fs({ service, project_id }); const syncClient = new SyncClient(client); - const syncstring = new SyncString({ + return new SyncString({ project_id, path, client: syncClient, fs, }); - await once(syncstring, "ready"); - return syncstring; } diff --git a/src/packages/frontend/frame-editors/generic/client.ts b/src/packages/frontend/frame-editors/generic/client.ts index 7c2bdbfe19..db1c708a39 100644 --- a/src/packages/frontend/frame-editors/generic/client.ts +++ b/src/packages/frontend/frame-editors/generic/client.ts @@ -218,6 +218,7 @@ interface SyncstringOpts2 { } export function syncstring2(opts: SyncstringOpts2): SyncString { + // return webapp_client.conat_client.conat().sync.string(opts); const opts1: any = opts; opts1.client = webapp_client; return webapp_client.sync_client.sync_string(opts1); From 088dbab45ca09c5a6e1f52746d1bed9d50cfcb38 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 20 Jul 2025 03:12:09 +0000 Subject: [PATCH 033/270] syncdb: implement similar to syncstring --- .../backend/conat/test/sync-doc/setup.ts | 2 +- .../conat/test/sync-doc/syncdb.test.ts | 99 +++++++++++++++++++ .../conat/test/sync-doc/syncstring.test.ts | 2 +- src/packages/conat/core/client.ts | 7 ++ src/packages/conat/sync-doc/syncdb.ts | 17 ++++ src/packages/conat/sync-doc/syncstring.ts | 26 ++--- .../frontend/frame-editors/generic/client.ts | 20 ++-- src/packages/sync/editor/string/index.ts | 1 + src/packages/sync/editor/string/sync.ts | 2 + 9 files changed, 149 insertions(+), 27 deletions(-) create mode 100644 src/packages/backend/conat/test/sync-doc/syncdb.test.ts create mode 100644 src/packages/conat/sync-doc/syncdb.ts diff --git a/src/packages/backend/conat/test/sync-doc/setup.ts b/src/packages/backend/conat/test/sync-doc/setup.ts index 0155bb46f4..5af527bf53 100644 --- a/src/packages/backend/conat/test/sync-doc/setup.ts +++ b/src/packages/backend/conat/test/sync-doc/setup.ts @@ -3,7 +3,7 @@ import { after as after0, client as client0, } from "@cocalc/backend/conat/test/setup"; -export { connect, wait, once } from "@cocalc/backend/conat/test/setup"; +export { connect, wait, once, delay } from "@cocalc/backend/conat/test/setup"; import { createPathFileserver, cleanupFileservers, diff --git a/src/packages/backend/conat/test/sync-doc/syncdb.test.ts b/src/packages/backend/conat/test/sync-doc/syncdb.test.ts new file mode 100644 index 0000000000..24e2706f52 --- /dev/null +++ b/src/packages/backend/conat/test/sync-doc/syncdb.test.ts @@ -0,0 +1,99 @@ +import { + before, + after, + uuid, + wait, + connect, + server, + once, +} from "./setup"; + +beforeAll(before); +afterAll(after); + +describe("loading/saving syncstring to disk and setting values", () => { + let s; + const project_id = uuid(); + let client; + + it("creates a client", () => { + client = connect(); + }); + + it("a syncdb associated to a file that does not exist on disk is initialized to empty", async () => { + s = client.sync.db({ + project_id, + path: "new.syncdb", + service: server.service, + primary_keys: ["name"], + }); + await once(s, "ready"); + expect(s.to_str()).toBe(""); + expect(s.versions().length).toBe(0); + }); + + it("store a record", async () => { + s.set({ name: "cocalc", value: 10 }); + expect(s.to_str()).toBe('{"name":"cocalc","value":10}'); + const t = s.get_one({ name: "cocalc" }).toJS(); + expect(t).toEqual({ name: "cocalc", value: 10 }); + await s.commit(); + await s.save(); + // [ ] TODO: this save to disk definitely should NOT be needed + await s.save_to_disk(); + }); + + let client2, s2; + it("connect another client", async () => { + client2 = connect(); + // [ ] loading this resets the state if we do not save above. + s2 = client2.sync.db({ + project_id, + path: "new.syncdb", + service: server.service, + primary_keys: ["name"], + }); + await once(s2, "ready"); + expect(s2).not.toBe(s); + expect(s2.to_str()).toBe('{"name":"cocalc","value":10}'); + const t = s2.get_one({ name: "cocalc" }).toJS(); + expect(t).toEqual({ name: "cocalc", value: 10 }); + + s2.set({ name: "conat", date: new Date() }); + s2.commit(); + await s2.save(); + }); + + it("verifies the change on s2 is seen by s (and also that Date objects do NOT work)", async () => { + await wait({ until: () => s.get_one({ name: "conat" }) != null }); + const t = s.get_one({ name: "conat" }).toJS(); + expect(t).toEqual({ name: "conat", date: t.date }); + // They don't work because we're storing syncdb's in jsonl format, + // so json is used. We should have a new format called + // msgpackl and start using that. + expect(t.date instanceof Date).toBe(false); + }); + + const count = 1000; + it(`store ${count} records`, async () => { + const before = s.get().size; + for (let i = 0; i < count; i++) { + s.set({ name: i }); + } + s.commit(); + await s.save(); + expect(s.get().size).toBe(count + before); + }); + + it("confirm file saves to disk with many lines", async () => { + await s.save_to_disk(); + const v = (await s.fs.readFile("new.syncdb", "utf8")).split("\n"); + expect(v.length).toBe(s.get().size); + }); + + it("verifies lookups are not too slow (there is an index)", () => { + for (let i = 0; i < count; i++) { + expect(s.get_one({ name: i }).get("name")).toEqual(i); + } + }); +}); diff --git a/src/packages/backend/conat/test/sync-doc/syncstring.test.ts b/src/packages/backend/conat/test/sync-doc/syncstring.test.ts index d1d73332a7..e67361ddfb 100644 --- a/src/packages/backend/conat/test/sync-doc/syncstring.test.ts +++ b/src/packages/backend/conat/test/sync-doc/syncstring.test.ts @@ -8,7 +8,7 @@ describe("loading/saving syncstring to disk and setting values", () => { const project_id = uuid(); let client; - it("creates the fs client", () => { + it("creates the client", () => { client = connect(); }); diff --git a/src/packages/conat/core/client.ts b/src/packages/conat/core/client.ts index 9a3d8d3dc4..bfa92256f6 100644 --- a/src/packages/conat/core/client.ts +++ b/src/packages/conat/core/client.ts @@ -249,6 +249,11 @@ import { type SyncString, type SyncStringOptions, } from "@cocalc/conat/sync-doc/syncstring"; +import { + syncdb, + type SyncDB, + type SyncDBOptions, +} from "@cocalc/conat/sync-doc/syncdb"; import { fsClient, DEFAULT_FILE_SERVICE } from "@cocalc/conat/files/fs"; import TTL from "@isaacs/ttlcache"; import { @@ -1496,6 +1501,8 @@ export class Client extends EventEmitter { await createSyncTable({ ...opts, client: this }), string: (opts: Omit): SyncString => syncstring({ ...opts, client: this }), + db: (opts: Omit): SyncDB => + syncdb({ ...opts, client: this }), }; socket = { diff --git a/src/packages/conat/sync-doc/syncdb.ts b/src/packages/conat/sync-doc/syncdb.ts new file mode 100644 index 0000000000..1ef7b135c0 --- /dev/null +++ b/src/packages/conat/sync-doc/syncdb.ts @@ -0,0 +1,17 @@ +import { SyncClient } from "./sync-client"; +import { SyncDB, type SyncDBOpts } from "@cocalc/sync/editor/db"; +import { type Client as ConatClient } from "@cocalc/conat/core/client"; + +export interface SyncDBOptions extends Omit { + client: ConatClient; + // name of the file service that hosts this file: + service?: string; +} + +export type { SyncDB }; + +export function syncdb({ client, service, ...opts }: SyncDBOptions): SyncDB { + const fs = client.fs({ service, project_id: opts.project_id }); + const syncClient = new SyncClient(client); + return new SyncDB({ ...opts, fs, client: syncClient }); +} diff --git a/src/packages/conat/sync-doc/syncstring.ts b/src/packages/conat/sync-doc/syncstring.ts index 474acd8488..5fb6d2e2fb 100644 --- a/src/packages/conat/sync-doc/syncstring.ts +++ b/src/packages/conat/sync-doc/syncstring.ts @@ -1,10 +1,11 @@ import { SyncClient } from "./sync-client"; -import { SyncString } from "@cocalc/sync/editor/string/sync"; +import { + SyncString, + type SyncStringOpts, +} from "@cocalc/sync/editor/string/sync"; import { type Client as ConatClient } from "@cocalc/conat/core/client"; -export interface SyncStringOptions { - project_id: string; - path: string; +export interface SyncStringOptions extends Omit { client: ConatClient; // name of the file server that hosts this document: service?: string; @@ -12,18 +13,9 @@ export interface SyncStringOptions { export type { SyncString }; -export function syncstring({ - project_id, - path, - client, - service, -}: SyncStringOptions): SyncString { - const fs = client.fs({ service, project_id }); +export function syncstring({ client, service, ...opts }: SyncStringOptions): SyncString { + const fs = client.fs({ service, project_id: opts.project_id }); const syncClient = new SyncClient(client); - return new SyncString({ - project_id, - path, - client: syncClient, - fs, - }); + return new SyncString({ ...opts, fs, client: syncClient }); } + diff --git a/src/packages/frontend/frame-editors/generic/client.ts b/src/packages/frontend/frame-editors/generic/client.ts index db1c708a39..67d143f53a 100644 --- a/src/packages/frontend/frame-editors/generic/client.ts +++ b/src/packages/frontend/frame-editors/generic/client.ts @@ -200,7 +200,8 @@ export function syncstring(opts: SyncstringOpts): any { delete opts.fake; } opts1.id = schema.client_db.sha1(opts.project_id, opts.path); - return webapp_client.sync_string(opts1); + return webapp_client.conat_client.conat().sync.string(opts1); + // return webapp_client.sync_string(opts1); } import { DataServer } from "@cocalc/sync/editor/generic/sync-doc"; @@ -218,10 +219,10 @@ interface SyncstringOpts2 { } export function syncstring2(opts: SyncstringOpts2): SyncString { - // return webapp_client.conat_client.conat().sync.string(opts); - const opts1: any = opts; - opts1.client = webapp_client; - return webapp_client.sync_client.sync_string(opts1); + return webapp_client.conat_client.conat().sync.string(opts); + // const opts1: any = opts; + // opts1.client = webapp_client; + // return webapp_client.sync_client.sync_string(opts1); } export interface SyncDBOpts { @@ -239,8 +240,10 @@ export interface SyncDBOpts { } export function syncdb(opts: SyncDBOpts): any { - const opts1: any = opts; - return webapp_client.sync_db(opts1); + return webapp_client.conat_client.conat().sync.db(opts); + + // const opts1: any = opts; + // return webapp_client.sync_db(opts1); } import type { SyncDB } from "@cocalc/sync/editor/db/sync"; @@ -251,7 +254,8 @@ export function syncdb2(opts: SyncDBOpts): SyncDB { } const opts1: any = opts; opts1.client = webapp_client; - return webapp_client.sync_client.sync_db(opts1); + return webapp_client.conat_client.conat().sync.db(opts1); + // return webapp_client.sync_client.sync_db(opts1); } interface QueryOpts { diff --git a/src/packages/sync/editor/string/index.ts b/src/packages/sync/editor/string/index.ts index c1fed4e3a5..5a43e5f755 100644 --- a/src/packages/sync/editor/string/index.ts +++ b/src/packages/sync/editor/string/index.ts @@ -1 +1,2 @@ export { SyncString } from "./sync"; +export { type SyncStringOpts } from "./sync"; diff --git a/src/packages/sync/editor/string/sync.ts b/src/packages/sync/editor/string/sync.ts index 780c066370..95494d9f1e 100644 --- a/src/packages/sync/editor/string/sync.ts +++ b/src/packages/sync/editor/string/sync.ts @@ -6,6 +6,8 @@ import { SyncDoc, SyncOpts0, SyncOpts } from "../generic/sync-doc"; import { StringDocument } from "./doc"; +export type SyncStringOpts = SyncOpts0; + export class SyncString extends SyncDoc { constructor(opts: SyncOpts0) { // TS question -- What is the right way to do this? From 1304de4c77b567828dcaf44f1293f5a0ccf89caf Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 20 Jul 2025 03:12:09 +0000 Subject: [PATCH 034/270] syncdb: implement similar to syncstring --- .../backend/conat/test/sync-doc/setup.ts | 2 +- .../conat/test/sync-doc/syncdb.test.ts | 99 +++++++++++++++++++ .../conat/test/sync-doc/syncstring.test.ts | 2 +- src/packages/conat/core/client.ts | 7 ++ src/packages/conat/sync-doc/syncdb.ts | 17 ++++ src/packages/conat/sync-doc/syncstring.ts | 26 ++--- .../frontend/frame-editors/generic/client.ts | 20 ++-- src/packages/sync/editor/db/sync.ts | 6 +- src/packages/sync/editor/string/index.ts | 1 + src/packages/sync/editor/string/sync.ts | 2 + 10 files changed, 152 insertions(+), 30 deletions(-) create mode 100644 src/packages/backend/conat/test/sync-doc/syncdb.test.ts create mode 100644 src/packages/conat/sync-doc/syncdb.ts diff --git a/src/packages/backend/conat/test/sync-doc/setup.ts b/src/packages/backend/conat/test/sync-doc/setup.ts index 0155bb46f4..5af527bf53 100644 --- a/src/packages/backend/conat/test/sync-doc/setup.ts +++ b/src/packages/backend/conat/test/sync-doc/setup.ts @@ -3,7 +3,7 @@ import { after as after0, client as client0, } from "@cocalc/backend/conat/test/setup"; -export { connect, wait, once } from "@cocalc/backend/conat/test/setup"; +export { connect, wait, once, delay } from "@cocalc/backend/conat/test/setup"; import { createPathFileserver, cleanupFileservers, diff --git a/src/packages/backend/conat/test/sync-doc/syncdb.test.ts b/src/packages/backend/conat/test/sync-doc/syncdb.test.ts new file mode 100644 index 0000000000..24e2706f52 --- /dev/null +++ b/src/packages/backend/conat/test/sync-doc/syncdb.test.ts @@ -0,0 +1,99 @@ +import { + before, + after, + uuid, + wait, + connect, + server, + once, +} from "./setup"; + +beforeAll(before); +afterAll(after); + +describe("loading/saving syncstring to disk and setting values", () => { + let s; + const project_id = uuid(); + let client; + + it("creates a client", () => { + client = connect(); + }); + + it("a syncdb associated to a file that does not exist on disk is initialized to empty", async () => { + s = client.sync.db({ + project_id, + path: "new.syncdb", + service: server.service, + primary_keys: ["name"], + }); + await once(s, "ready"); + expect(s.to_str()).toBe(""); + expect(s.versions().length).toBe(0); + }); + + it("store a record", async () => { + s.set({ name: "cocalc", value: 10 }); + expect(s.to_str()).toBe('{"name":"cocalc","value":10}'); + const t = s.get_one({ name: "cocalc" }).toJS(); + expect(t).toEqual({ name: "cocalc", value: 10 }); + await s.commit(); + await s.save(); + // [ ] TODO: this save to disk definitely should NOT be needed + await s.save_to_disk(); + }); + + let client2, s2; + it("connect another client", async () => { + client2 = connect(); + // [ ] loading this resets the state if we do not save above. + s2 = client2.sync.db({ + project_id, + path: "new.syncdb", + service: server.service, + primary_keys: ["name"], + }); + await once(s2, "ready"); + expect(s2).not.toBe(s); + expect(s2.to_str()).toBe('{"name":"cocalc","value":10}'); + const t = s2.get_one({ name: "cocalc" }).toJS(); + expect(t).toEqual({ name: "cocalc", value: 10 }); + + s2.set({ name: "conat", date: new Date() }); + s2.commit(); + await s2.save(); + }); + + it("verifies the change on s2 is seen by s (and also that Date objects do NOT work)", async () => { + await wait({ until: () => s.get_one({ name: "conat" }) != null }); + const t = s.get_one({ name: "conat" }).toJS(); + expect(t).toEqual({ name: "conat", date: t.date }); + // They don't work because we're storing syncdb's in jsonl format, + // so json is used. We should have a new format called + // msgpackl and start using that. + expect(t.date instanceof Date).toBe(false); + }); + + const count = 1000; + it(`store ${count} records`, async () => { + const before = s.get().size; + for (let i = 0; i < count; i++) { + s.set({ name: i }); + } + s.commit(); + await s.save(); + expect(s.get().size).toBe(count + before); + }); + + it("confirm file saves to disk with many lines", async () => { + await s.save_to_disk(); + const v = (await s.fs.readFile("new.syncdb", "utf8")).split("\n"); + expect(v.length).toBe(s.get().size); + }); + + it("verifies lookups are not too slow (there is an index)", () => { + for (let i = 0; i < count; i++) { + expect(s.get_one({ name: i }).get("name")).toEqual(i); + } + }); +}); diff --git a/src/packages/backend/conat/test/sync-doc/syncstring.test.ts b/src/packages/backend/conat/test/sync-doc/syncstring.test.ts index d1d73332a7..e67361ddfb 100644 --- a/src/packages/backend/conat/test/sync-doc/syncstring.test.ts +++ b/src/packages/backend/conat/test/sync-doc/syncstring.test.ts @@ -8,7 +8,7 @@ describe("loading/saving syncstring to disk and setting values", () => { const project_id = uuid(); let client; - it("creates the fs client", () => { + it("creates the client", () => { client = connect(); }); diff --git a/src/packages/conat/core/client.ts b/src/packages/conat/core/client.ts index 9a3d8d3dc4..bfa92256f6 100644 --- a/src/packages/conat/core/client.ts +++ b/src/packages/conat/core/client.ts @@ -249,6 +249,11 @@ import { type SyncString, type SyncStringOptions, } from "@cocalc/conat/sync-doc/syncstring"; +import { + syncdb, + type SyncDB, + type SyncDBOptions, +} from "@cocalc/conat/sync-doc/syncdb"; import { fsClient, DEFAULT_FILE_SERVICE } from "@cocalc/conat/files/fs"; import TTL from "@isaacs/ttlcache"; import { @@ -1496,6 +1501,8 @@ export class Client extends EventEmitter { await createSyncTable({ ...opts, client: this }), string: (opts: Omit): SyncString => syncstring({ ...opts, client: this }), + db: (opts: Omit): SyncDB => + syncdb({ ...opts, client: this }), }; socket = { diff --git a/src/packages/conat/sync-doc/syncdb.ts b/src/packages/conat/sync-doc/syncdb.ts new file mode 100644 index 0000000000..a4437d7459 --- /dev/null +++ b/src/packages/conat/sync-doc/syncdb.ts @@ -0,0 +1,17 @@ +import { SyncClient } from "./sync-client"; +import { SyncDB, type SyncDBOpts0 } from "@cocalc/sync/editor/db"; +import { type Client as ConatClient } from "@cocalc/conat/core/client"; + +export interface SyncDBOptions extends Omit { + client: ConatClient; + // name of the file service that hosts this file: + service?: string; +} + +export type { SyncDB }; + +export function syncdb({ client, service, ...opts }: SyncDBOptions): SyncDB { + const fs = client.fs({ service, project_id: opts.project_id }); + const syncClient = new SyncClient(client); + return new SyncDB({ ...opts, fs, client: syncClient }); +} diff --git a/src/packages/conat/sync-doc/syncstring.ts b/src/packages/conat/sync-doc/syncstring.ts index 474acd8488..5fb6d2e2fb 100644 --- a/src/packages/conat/sync-doc/syncstring.ts +++ b/src/packages/conat/sync-doc/syncstring.ts @@ -1,10 +1,11 @@ import { SyncClient } from "./sync-client"; -import { SyncString } from "@cocalc/sync/editor/string/sync"; +import { + SyncString, + type SyncStringOpts, +} from "@cocalc/sync/editor/string/sync"; import { type Client as ConatClient } from "@cocalc/conat/core/client"; -export interface SyncStringOptions { - project_id: string; - path: string; +export interface SyncStringOptions extends Omit { client: ConatClient; // name of the file server that hosts this document: service?: string; @@ -12,18 +13,9 @@ export interface SyncStringOptions { export type { SyncString }; -export function syncstring({ - project_id, - path, - client, - service, -}: SyncStringOptions): SyncString { - const fs = client.fs({ service, project_id }); +export function syncstring({ client, service, ...opts }: SyncStringOptions): SyncString { + const fs = client.fs({ service, project_id: opts.project_id }); const syncClient = new SyncClient(client); - return new SyncString({ - project_id, - path, - client: syncClient, - fs, - }); + return new SyncString({ ...opts, fs, client: syncClient }); } + diff --git a/src/packages/frontend/frame-editors/generic/client.ts b/src/packages/frontend/frame-editors/generic/client.ts index db1c708a39..67d143f53a 100644 --- a/src/packages/frontend/frame-editors/generic/client.ts +++ b/src/packages/frontend/frame-editors/generic/client.ts @@ -200,7 +200,8 @@ export function syncstring(opts: SyncstringOpts): any { delete opts.fake; } opts1.id = schema.client_db.sha1(opts.project_id, opts.path); - return webapp_client.sync_string(opts1); + return webapp_client.conat_client.conat().sync.string(opts1); + // return webapp_client.sync_string(opts1); } import { DataServer } from "@cocalc/sync/editor/generic/sync-doc"; @@ -218,10 +219,10 @@ interface SyncstringOpts2 { } export function syncstring2(opts: SyncstringOpts2): SyncString { - // return webapp_client.conat_client.conat().sync.string(opts); - const opts1: any = opts; - opts1.client = webapp_client; - return webapp_client.sync_client.sync_string(opts1); + return webapp_client.conat_client.conat().sync.string(opts); + // const opts1: any = opts; + // opts1.client = webapp_client; + // return webapp_client.sync_client.sync_string(opts1); } export interface SyncDBOpts { @@ -239,8 +240,10 @@ export interface SyncDBOpts { } export function syncdb(opts: SyncDBOpts): any { - const opts1: any = opts; - return webapp_client.sync_db(opts1); + return webapp_client.conat_client.conat().sync.db(opts); + + // const opts1: any = opts; + // return webapp_client.sync_db(opts1); } import type { SyncDB } from "@cocalc/sync/editor/db/sync"; @@ -251,7 +254,8 @@ export function syncdb2(opts: SyncDBOpts): SyncDB { } const opts1: any = opts; opts1.client = webapp_client; - return webapp_client.sync_client.sync_db(opts1); + return webapp_client.conat_client.conat().sync.db(opts1); + // return webapp_client.sync_client.sync_db(opts1); } interface QueryOpts { diff --git a/src/packages/sync/editor/db/sync.ts b/src/packages/sync/editor/db/sync.ts index db970235b2..b8db32366c 100644 --- a/src/packages/sync/editor/db/sync.ts +++ b/src/packages/sync/editor/db/sync.ts @@ -9,7 +9,7 @@ import { Document, DocType } from "../generic/types"; export interface SyncDBOpts0 extends SyncOpts0 { primary_keys: string[]; - string_cols: string[]; + string_cols?: string[]; } export interface SyncDBOpts extends SyncDBOpts0 { @@ -25,13 +25,13 @@ export class SyncDB extends SyncDoc { throw Error("primary_keys must have length at least 1"); } opts1.from_str = (str) => - from_str(str, opts1.primary_keys, opts1.string_cols); + from_str(str, opts1.primary_keys, opts1.string_cols ?? []); opts1.doctype = { type: "db", patch_format: 1, opts: { primary_keys: opts1.primary_keys, - string_cols: opts1.string_cols, + string_cols: opts1.string_cols ?? [], }, }; super(opts1 as SyncOpts); diff --git a/src/packages/sync/editor/string/index.ts b/src/packages/sync/editor/string/index.ts index c1fed4e3a5..5a43e5f755 100644 --- a/src/packages/sync/editor/string/index.ts +++ b/src/packages/sync/editor/string/index.ts @@ -1 +1,2 @@ export { SyncString } from "./sync"; +export { type SyncStringOpts } from "./sync"; diff --git a/src/packages/sync/editor/string/sync.ts b/src/packages/sync/editor/string/sync.ts index 780c066370..95494d9f1e 100644 --- a/src/packages/sync/editor/string/sync.ts +++ b/src/packages/sync/editor/string/sync.ts @@ -6,6 +6,8 @@ import { SyncDoc, SyncOpts0, SyncOpts } from "../generic/sync-doc"; import { StringDocument } from "./doc"; +export type SyncStringOpts = SyncOpts0; + export class SyncString extends SyncDoc { constructor(opts: SyncOpts0) { // TS question -- What is the right way to do this? From 592915fd136e80f324e02c3f36f7c0cdfb92f796 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 19 Jul 2025 20:27:05 -0700 Subject: [PATCH 035/270] fix some depcheck issues --- src/packages/backend/package.json | 13 ++++++++++--- src/packages/conat/tsconfig.json | 2 +- src/packages/file-server/package.json | 13 ++++++++++--- src/packages/pnpm-lock.yaml | 6 ------ 4 files changed, 21 insertions(+), 13 deletions(-) diff --git a/src/packages/backend/package.json b/src/packages/backend/package.json index 1ceb59e9c6..4a71ea0a22 100644 --- a/src/packages/backend/package.json +++ b/src/packages/backend/package.json @@ -13,7 +13,10 @@ "./auth/*": "./dist/auth/*.js", "./auth/tokens/*": "./dist/auth/tokens/*.js" }, - "keywords": ["utilities", "cocalc"], + "keywords": [ + "utilities", + "cocalc" + ], "scripts": { "preinstall": "npx only-allow pnpm", "clean": "rm -rf dist node_modules", @@ -31,13 +34,17 @@ "conat-persist": "DEBUG=cocalc:* node ./bin/conat-persist.cjs", "conat-test-server": "node ./bin/conat-test-server.cjs" }, - "files": ["dist/**", "bin/**", "README.md", "package.json"], + "files": [ + "dist/**", + "bin/**", + "README.md", + "package.json" + ], "author": "SageMath, Inc.", "license": "SEE LICENSE.md", "dependencies": { "@cocalc/backend": "workspace:*", "@cocalc/conat": "workspace:*", - "@cocalc/sync": "workspace:*", "@cocalc/util": "workspace:*", "@types/debug": "^4.1.12", "@types/jest": "^29.5.14", diff --git a/src/packages/conat/tsconfig.json b/src/packages/conat/tsconfig.json index 687201523d..5a8dd8655a 100644 --- a/src/packages/conat/tsconfig.json +++ b/src/packages/conat/tsconfig.json @@ -6,5 +6,5 @@ }, "exclude": ["node_modules", "dist", "test"], "references_comment": "Do not define path:../comm because that causes a circular references.", - "references": [{ "path": "../util" }] + "references": [{ "path": "../util", "path": "../sync" }] } diff --git a/src/packages/file-server/package.json b/src/packages/file-server/package.json index e12191d181..1a952b2369 100644 --- a/src/packages/file-server/package.json +++ b/src/packages/file-server/package.json @@ -17,13 +17,20 @@ "depcheck": "pnpx depcheck", "clean": "rm -rf node_modules dist" }, - "files": ["dist/**", "README.md", "package.json"], + "files": [ + "dist/**", + "README.md", + "package.json" + ], "author": "SageMath, Inc.", - "keywords": ["utilities", "btrfs", "cocalc"], + "keywords": [ + "utilities", + "btrfs", + "cocalc" + ], "license": "SEE LICENSE.md", "dependencies": { "@cocalc/backend": "workspace:*", - "@cocalc/conat": "workspace:*", "@cocalc/file-server": "workspace:*", "@cocalc/util": "workspace:*", "awaiting": "^3.0.0" diff --git a/src/packages/pnpm-lock.yaml b/src/packages/pnpm-lock.yaml index 135d788520..74e8838273 100644 --- a/src/packages/pnpm-lock.yaml +++ b/src/packages/pnpm-lock.yaml @@ -81,9 +81,6 @@ importers: '@cocalc/conat': specifier: workspace:* version: link:../conat - '@cocalc/sync': - specifier: workspace:* - version: link:../sync '@cocalc/util': specifier: workspace:* version: link:../util @@ -292,9 +289,6 @@ importers: '@cocalc/backend': specifier: workspace:* version: link:../backend - '@cocalc/conat': - specifier: workspace:* - version: link:../conat '@cocalc/file-server': specifier: workspace:* version: 'link:' From f1885802e234a459ef44fb75cf546073219f7d5b Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 20 Jul 2025 03:30:30 +0000 Subject: [PATCH 036/270] fix circular ref --- src/packages/sync/tsconfig.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/packages/sync/tsconfig.json b/src/packages/sync/tsconfig.json index 0bdfd8b3a4..6cdb913e19 100644 --- a/src/packages/sync/tsconfig.json +++ b/src/packages/sync/tsconfig.json @@ -5,5 +5,5 @@ "outDir": "dist" }, "exclude": ["node_modules", "dist", "test"], - "references": [{ "path": "../util" }, { "path": "../conat" }] + "references": [{ "path": "../util" }] } From 24766743b9f8bed0436facabd110895e7a7dde5a Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 20 Jul 2025 03:33:43 +0000 Subject: [PATCH 037/270] fix a test now that I made EventIterator work much better --- src/packages/file-server/btrfs/test/subvolume.test.ts | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/packages/file-server/btrfs/test/subvolume.test.ts b/src/packages/file-server/btrfs/test/subvolume.test.ts index ecc5ed9756..456282d8b3 100644 --- a/src/packages/file-server/btrfs/test/subvolume.test.ts +++ b/src/packages/file-server/btrfs/test/subvolume.test.ts @@ -151,11 +151,10 @@ describe("the filesystem operations", () => { expect(done).toBe(false); expect(value).toEqual({ eventType: "change", filename: "w.txt" }); ac.abort(); - - expect(async () => { - // @ts-ignore - await watcher.next(); - }).rejects.toThrow("aborted"); + { + const { done } = await watcher.next(); + expect(done).toBe(true); + } }); it("rename a file", async () => { From 3eaf09128a42cb79b18c8bd39338b73b45c63e6d Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 20 Jul 2025 15:30:35 +0000 Subject: [PATCH 038/270] writing some sync-doc tests --- .../backend/conat/test/sync-doc/merge.test.ts | 95 +++++++++++++++++++ .../test/sync-doc/syncstring-bench.test.ts | 32 +++++++ 2 files changed, 127 insertions(+) create mode 100644 src/packages/backend/conat/test/sync-doc/merge.test.ts create mode 100644 src/packages/backend/conat/test/sync-doc/syncstring-bench.test.ts diff --git a/src/packages/backend/conat/test/sync-doc/merge.test.ts b/src/packages/backend/conat/test/sync-doc/merge.test.ts new file mode 100644 index 0000000000..bea90510dd --- /dev/null +++ b/src/packages/backend/conat/test/sync-doc/merge.test.ts @@ -0,0 +1,95 @@ +/* +Illustrate and test behavior when there is conflict. +*/ + +import { + before, + after, + uuid, + wait, + connect, + server, + once, + delay, +} from "./setup"; + +beforeAll(before); +afterAll(after); + +describe("synchronized editing with branching and merging", () => { + const project_id = uuid(); + let s1, s2, client1, client2; + + it("creates two clients", async () => { + client1 = connect(); + client2 = connect(); + s1 = client1.sync.string({ + project_id, + path: "a.txt", + service: server.service, + }); + await once(s1, "ready"); + + s2 = client2.sync.string({ + project_id, + path: "a.txt", + service: server.service, + }); + await once(s2, "ready"); + expect(s1.to_str()).toBe(""); + expect(s2.to_str()).toBe(""); + expect(s1 === s2).toBe(false); + }); + + it("both clients set the first version independently and inconsistently", async () => { + s1.from_str("x"); + s2.from_str("y"); + s1.commit(); + s2.commit(); + await s1.save(); + await s2.save(); + }); + + it("wait until both clients see two heads", async () => { + let heads1, heads2; + await wait({ + until: () => { + heads1 = s1.patch_list.getHeads(); + heads2 = s2.patch_list.getHeads(); + return heads1.length == 2 && heads2.length == 2; + }, + }); + expect(heads1.length).toBe(2); + expect(heads2.length).toBe(2); + expect(heads1).toEqual(heads2); + }); + + it("get the current value, which is a merge", () => { + const v1 = s1.to_str(); + const v2 = s2.to_str(); + expect(v1).toEqual("xy"); + expect(v2).toEqual("xy"); + }); + + // this is broken already: + it.skip("commit current value and see that there is a new single head that both share, thus resolving the merge in this way", async () => { + s1.commit(); + await s1.save(); + s1.show_history(); + let heads1, heads2; + await wait({ + until: () => { + heads1 = s1.patch_list.getHeads(); + heads2 = s2.patch_list.getHeads(); + console.log({ heads1, heads2 }); + return heads1.length == 1 && heads2.length == 1; + }, + }); + expect(heads1.length).toBe(1); + expect(heads2.length).toBe(1); + expect(heads1).toEqual(heads2); + }); + + // set values inconsistently again and explicitly resolve the merge conflict + // in a way that is different than the default. +}); diff --git a/src/packages/backend/conat/test/sync-doc/syncstring-bench.test.ts b/src/packages/backend/conat/test/sync-doc/syncstring-bench.test.ts new file mode 100644 index 0000000000..5bbf634377 --- /dev/null +++ b/src/packages/backend/conat/test/sync-doc/syncstring-bench.test.ts @@ -0,0 +1,32 @@ +import { before, after, uuid, client, server, once } from "./setup"; + +beforeAll(before); +afterAll(after); + +const log = console.log; + +describe("loading/saving syncstring to disk and setting values", () => { + let s; + const project_id = uuid(); + let fs; + it("time opening a syncstring for editing a file that already exists on disk", async () => { + fs = client.fs({ project_id, service: server.service }); + await fs.writeFile("a.txt", "hello"); + + const t0 = Date.now(); + await fs.readFile("a.txt", "utf8"); + console.log("lower bound: time to read file", Date.now() - t0, "ms"); + + const start = Date.now(); + s = client.sync.string({ + project_id, + path: "a.txt", + service: server.service, + }); + await once(s, "ready"); + const total = Date.now() - start; + log("time to open", total); + + expect(s.to_str()).toBe("hello"); + }); +}); From ad13e3b23e0f632aecc8a9c6e32f926434f2aa0f Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 20 Jul 2025 17:20:35 +0000 Subject: [PATCH 039/270] write some merge conflict related tests - these need noAutosave support for syncdocs, which doesn't work properly yet --- .../conat/test/sync-doc/conflict.test.ts | 202 ++++++++++++++++++ .../backend/conat/test/sync-doc/merge.test.ts | 95 -------- .../backend/conat/test/sync-doc/setup.ts | 18 ++ .../test/sync-doc/syncstring-bench.test.ts | 6 +- src/packages/conat/sync/dstream.ts | 1 + src/packages/conat/sync/synctable-kv.ts | 6 + src/packages/conat/sync/synctable-stream.ts | 5 + src/packages/conat/sync/synctable.ts | 1 + .../sync/editor/generic/sorted-patch-list.ts | 10 +- src/packages/sync/editor/generic/sync-doc.ts | 45 +++- 10 files changed, 278 insertions(+), 111 deletions(-) create mode 100644 src/packages/backend/conat/test/sync-doc/conflict.test.ts delete mode 100644 src/packages/backend/conat/test/sync-doc/merge.test.ts diff --git a/src/packages/backend/conat/test/sync-doc/conflict.test.ts b/src/packages/backend/conat/test/sync-doc/conflict.test.ts new file mode 100644 index 0000000000..4c5c521d02 --- /dev/null +++ b/src/packages/backend/conat/test/sync-doc/conflict.test.ts @@ -0,0 +1,202 @@ +/* +Illustrate and test behavior when there is conflict. + +TODO: we must get noAutosave to fully work so we can make +the tests of conflicts, etc., better. + +E.g, the test below WILL RANDOMLY FAIL right now due to autosave randomness... +*/ + +import { + before, + after, + uuid, + connect, + server, + once, + delay, + waitUntilSynced, +} from "./setup"; + +beforeAll(before); +afterAll(after); + +describe("synchronized editing with branching and merging", () => { + const project_id = uuid(); + let s1, s2, client1, client2; + + it("creates two clients", async () => { + client1 = connect(); + client2 = connect(); + s1 = client1.sync.string({ + project_id, + path: "a.txt", + service: server.service, + }); + await once(s1, "ready"); + + s2 = client2.sync.string({ + project_id, + path: "a.txt", + service: server.service, + }); + await once(s2, "ready"); + expect(s1.to_str()).toBe(""); + expect(s2.to_str()).toBe(""); + expect(s1 === s2).toBe(false); + }); + + it("both clients set the first version independently and inconsistently", async () => { + s1.from_str("x"); + s2.from_str("y"); + s1.commit(); + // delay so s2's time is always bigger than s1's so our unit test + // is well defined + await delay(1); + s2.commit(); + await s1.save(); + await s2.save(); + }); + + it("wait until both clients see two heads", async () => { + await waitUntilSynced([s1, s2]); + const heads1 = s1.patch_list.getHeads(); + const heads2 = s2.patch_list.getHeads(); + expect(heads1.length).toBe(2); + expect(heads2.length).toBe(2); + expect(heads1).toEqual(heads2); + }); + + it("get the current value, which is a merge", () => { + const v1 = s1.to_str(); + const v2 = s2.to_str(); + expect(v1).toEqual("xy"); + expect(v2).toEqual("xy"); + }); + + it("commit current value and see that there is a new single head that both share, thus resolving the merge in this way", async () => { + s1.commit(); + await s1.save(); + await waitUntilSynced([s1, s2]); + const heads1 = s1.patch_list.getHeads(); + const heads2 = s2.patch_list.getHeads(); + expect(heads1.length).toBe(1); + expect(heads2.length).toBe(1); + expect(heads1).toEqual(heads2); + }); + + it("set values inconsistently again and explicitly resolve the merge conflict in a way that is different than the default", async () => { + s1.from_str("xy1"); + s1.commit(); + await delay(1); + s2.from_str("xy2"); + s2.commit(); + await s1.save(); + await s2.save(); + + await waitUntilSynced([s1, s2]); + expect(s1.to_str()).toEqual("xy12"); + expect(s2.to_str()).toEqual("xy12"); + + // how we resolve the conflict + s1.from_str("xy3"); + s1.commit(); + await waitUntilSynced([s1, s2]); + + // everybody has this state now + expect(s1.to_str()).toEqual("xy3"); + expect(s2.to_str()).toEqual("xy3"); + }); +}); + +describe.only("do the example in the blog post 'Lies I was Told About Collaborative Editing, Part 1: Algorithms for offline editing' -- https://www.moment.dev/blog/lies-i-was-told-pt-1", () => { + const project_id = uuid(); + let client1, client2; + + async function getInitialState(path: string) { + client1 ??= connect(); + client2 ??= connect(); + client1 + .fs({ project_id, service: server.service }) + .writeFile(path, "The Color of Pomegranates"); + const alice = client1.sync.string({ + project_id, + path, + service: server.service, + }); + await once(alice, "ready"); + + const bob = client2.sync.string({ + project_id, + path, + service: server.service, + }); + await once(bob, "ready"); + return { alice, bob }; + } + + let alice, bob; + it("creates two clients", async () => { + ({ alice, bob } = await getInitialState("first.txt")); + expect(alice.to_str()).toEqual("The Color of Pomegranates"); + expect(bob.to_str()).toEqual("The Color of Pomegranates"); + }); + + it("Bob changes the spelling of Color to the British Colour and unaware Alice deletes all of the text.", async () => { + bob.from_str("The Colour of Pomegranates"); + bob.commit(); + alice.from_str(""); + alice.commit(); + }); + + it("Both come back online -- the resolution is the empty (with either order above) string because the **best effort** application of inserting the u (with context) to either is a no-op.", async () => { + await bob.save(); + await alice.save(); + await waitUntilSynced([bob, alice]); + expect(alice.to_str()).toEqual(""); + expect(bob.to_str()).toEqual(""); + }); + + it("the important thing about the cocalc approach is that a consistent history is saved, so everybody knows precisely what happened. **I.e., the fact that at one point Bob adding a British u is not lost to either party!**", () => { + const v = alice.versions(); + const x = v.map((t) => alice.version(t).to_str()); + expect(new Set(x)).toEqual( + new Set(["The Color of Pomegranates", "The Colour of Pomegranates", ""]), + ); + + const w = alice.versions(); + const y = w.map((t) => bob.version(t).to_str()); + expect(y).toEqual(x); + }); + + it("reset -- create alicea and bob again", async () => { + ({ alice, bob } = await getInitialState("second.txt")); + }); + + // opposite order this time + it("Bob changes the spelling of Color to the British Colour and unaware Alice deletes all of the text.", async () => { + alice.from_str(""); + alice.commit(); + bob.from_str("The Colour of Pomegranates"); + bob.commit(); + }); + + it("both empty again", async () => { + await bob.save(); + await alice.save(); + await waitUntilSynced([bob, alice]); + expect(alice.to_str()).toEqual(""); + expect(bob.to_str()).toEqual(""); + }); + + it("There are in fact two heads right now, and either party can resolve the merge conflict however they want.", async () => { + expect(alice.patch_list.getHeads().length).toBe(2); + expect(bob.patch_list.getHeads().length).toBe(2); + bob.from_str("The Colour of Pomegranates"); + bob.commit(); + + await waitUntilSynced([bob, alice]); + expect(alice.to_str()).toEqual("The Colour of Pomegranates"); + expect(bob.to_str()).toEqual("The Colour of Pomegranates"); + }); +}); diff --git a/src/packages/backend/conat/test/sync-doc/merge.test.ts b/src/packages/backend/conat/test/sync-doc/merge.test.ts deleted file mode 100644 index bea90510dd..0000000000 --- a/src/packages/backend/conat/test/sync-doc/merge.test.ts +++ /dev/null @@ -1,95 +0,0 @@ -/* -Illustrate and test behavior when there is conflict. -*/ - -import { - before, - after, - uuid, - wait, - connect, - server, - once, - delay, -} from "./setup"; - -beforeAll(before); -afterAll(after); - -describe("synchronized editing with branching and merging", () => { - const project_id = uuid(); - let s1, s2, client1, client2; - - it("creates two clients", async () => { - client1 = connect(); - client2 = connect(); - s1 = client1.sync.string({ - project_id, - path: "a.txt", - service: server.service, - }); - await once(s1, "ready"); - - s2 = client2.sync.string({ - project_id, - path: "a.txt", - service: server.service, - }); - await once(s2, "ready"); - expect(s1.to_str()).toBe(""); - expect(s2.to_str()).toBe(""); - expect(s1 === s2).toBe(false); - }); - - it("both clients set the first version independently and inconsistently", async () => { - s1.from_str("x"); - s2.from_str("y"); - s1.commit(); - s2.commit(); - await s1.save(); - await s2.save(); - }); - - it("wait until both clients see two heads", async () => { - let heads1, heads2; - await wait({ - until: () => { - heads1 = s1.patch_list.getHeads(); - heads2 = s2.patch_list.getHeads(); - return heads1.length == 2 && heads2.length == 2; - }, - }); - expect(heads1.length).toBe(2); - expect(heads2.length).toBe(2); - expect(heads1).toEqual(heads2); - }); - - it("get the current value, which is a merge", () => { - const v1 = s1.to_str(); - const v2 = s2.to_str(); - expect(v1).toEqual("xy"); - expect(v2).toEqual("xy"); - }); - - // this is broken already: - it.skip("commit current value and see that there is a new single head that both share, thus resolving the merge in this way", async () => { - s1.commit(); - await s1.save(); - s1.show_history(); - let heads1, heads2; - await wait({ - until: () => { - heads1 = s1.patch_list.getHeads(); - heads2 = s2.patch_list.getHeads(); - console.log({ heads1, heads2 }); - return heads1.length == 1 && heads2.length == 1; - }, - }); - expect(heads1.length).toBe(1); - expect(heads2.length).toBe(1); - expect(heads1).toEqual(heads2); - }); - - // set values inconsistently again and explicitly resolve the merge conflict - // in a way that is different than the default. -}); diff --git a/src/packages/backend/conat/test/sync-doc/setup.ts b/src/packages/backend/conat/test/sync-doc/setup.ts index 5af527bf53..b6ce38042d 100644 --- a/src/packages/backend/conat/test/sync-doc/setup.ts +++ b/src/packages/backend/conat/test/sync-doc/setup.ts @@ -2,6 +2,7 @@ import { before as before0, after as after0, client as client0, + wait, } from "@cocalc/backend/conat/test/setup"; export { connect, wait, once, delay } from "@cocalc/backend/conat/test/setup"; import { @@ -23,3 +24,20 @@ export async function after() { await cleanupFileservers(); await after0(); } + +// wait until the state of several syncdocs all have same heads- they may have multiple +// heads, but they all have the same heads +export async function waitUntilSynced(syncdocs: any[]) { + await wait({ + until: () => { + const X = new Set(); + for (const s of syncdocs) { + X.add(JSON.stringify(s.patch_list.getHeads()?.sort())); + if (X.size > 1) { + return false; + } + } + return true; + }, + }); +} diff --git a/src/packages/backend/conat/test/sync-doc/syncstring-bench.test.ts b/src/packages/backend/conat/test/sync-doc/syncstring-bench.test.ts index 5bbf634377..2e5a64ef2a 100644 --- a/src/packages/backend/conat/test/sync-doc/syncstring-bench.test.ts +++ b/src/packages/backend/conat/test/sync-doc/syncstring-bench.test.ts @@ -3,7 +3,7 @@ import { before, after, uuid, client, server, once } from "./setup"; beforeAll(before); afterAll(after); -const log = console.log; +const log = process.env.BENCH ? console.log : (..._args) => {}; describe("loading/saving syncstring to disk and setting values", () => { let s; @@ -15,7 +15,7 @@ describe("loading/saving syncstring to disk and setting values", () => { const t0 = Date.now(); await fs.readFile("a.txt", "utf8"); - console.log("lower bound: time to read file", Date.now() - t0, "ms"); + log("lower bound: time to read file", Date.now() - t0, "ms"); const start = Date.now(); s = client.sync.string({ @@ -25,7 +25,7 @@ describe("loading/saving syncstring to disk and setting values", () => { }); await once(s, "ready"); const total = Date.now() - start; - log("time to open", total); + log("actual time to open sync document", total); expect(s.to_str()).toBe("hello"); }); diff --git a/src/packages/conat/sync/dstream.ts b/src/packages/conat/sync/dstream.ts index 4bc6121c26..26d0c997f9 100644 --- a/src/packages/conat/sync/dstream.ts +++ b/src/packages/conat/sync/dstream.ts @@ -290,6 +290,7 @@ export class DStream extends EventEmitter { }; save = reuseInFlight(async () => { + //console.log("save", this.noAutosave); await until( async () => { if (this.isClosed()) { diff --git a/src/packages/conat/sync/synctable-kv.ts b/src/packages/conat/sync/synctable-kv.ts index 7950305ed4..9e56393fff 100644 --- a/src/packages/conat/sync/synctable-kv.ts +++ b/src/packages/conat/sync/synctable-kv.ts @@ -33,6 +33,7 @@ export class SyncTableKV extends EventEmitter { private config?: Partial; private desc?: JSONValue; private ephemeral?: boolean; + private noAutosave?: boolean; constructor({ query, @@ -44,6 +45,7 @@ export class SyncTableKV extends EventEmitter { config, desc, ephemeral, + noAutosave, }: { query; client: Client; @@ -54,6 +56,7 @@ export class SyncTableKV extends EventEmitter { config?: Partial; desc?: JSONValue; ephemeral?: boolean; + noAutosave?: boolean; }) { super(); this.setMaxListeners(1000); @@ -64,6 +67,7 @@ export class SyncTableKV extends EventEmitter { this.client = client; this.desc = desc; this.ephemeral = ephemeral; + this.noAutosave = noAutosave; this.table = keys(query)[0]; if (query[this.table][0].string_id && query[this.table][0].project_id) { this.project_id = query[this.table][0].project_id; @@ -126,6 +130,7 @@ export class SyncTableKV extends EventEmitter { config: this.config, desc: this.desc, ephemeral: this.ephemeral, + noAutosave: this.noAutosave, }); } else { this.dkv = await createDko({ @@ -136,6 +141,7 @@ export class SyncTableKV extends EventEmitter { config: this.config, desc: this.desc, ephemeral: this.ephemeral, + noAutosave: this.noAutosave, }); } // For some reason this one line confuses typescript and break building the compute server package (nothing else similar happens). diff --git a/src/packages/conat/sync/synctable-stream.ts b/src/packages/conat/sync/synctable-stream.ts index 36b1b3e27e..597cfc8d95 100644 --- a/src/packages/conat/sync/synctable-stream.ts +++ b/src/packages/conat/sync/synctable-stream.ts @@ -45,6 +45,7 @@ export class SyncTableStream extends EventEmitter { private config?: Partial; private start_seq?: number; private noInventory?: boolean; + private noAutosave?: boolean; private ephemeral?: boolean; constructor({ @@ -57,6 +58,7 @@ export class SyncTableStream extends EventEmitter { start_seq, noInventory, ephemeral, + noAutosave, }: { query; client: Client; @@ -67,10 +69,12 @@ export class SyncTableStream extends EventEmitter { start_seq?: number; noInventory?: boolean; ephemeral?: boolean; + noAutosave?: boolean; }) { super(); this.client = client; this.noInventory = noInventory; + this.noAutosave = noAutosave; this.ephemeral = ephemeral; this.setMaxListeners(1000); this.getHook = immutable ? fromJS : (x) => x; @@ -107,6 +111,7 @@ export class SyncTableStream extends EventEmitter { start_seq: this.start_seq, noInventory: this.noInventory, ephemeral: this.ephemeral, + noAutosave: this.noAutosave, }); this.dstream.on("change", (mesg) => { this.handle(mesg, true); diff --git a/src/packages/conat/sync/synctable.ts b/src/packages/conat/sync/synctable.ts index 8f69000eee..95048b86d2 100644 --- a/src/packages/conat/sync/synctable.ts +++ b/src/packages/conat/sync/synctable.ts @@ -43,6 +43,7 @@ export interface SyncTableOptions { start_seq?: number; noInventory?: boolean; ephemeral?: boolean; + noAutosave?: boolean; } export const createSyncTable = refCache({ diff --git a/src/packages/sync/editor/generic/sorted-patch-list.ts b/src/packages/sync/editor/generic/sorted-patch-list.ts index 5d35c08bec..6a2be95f95 100644 --- a/src/packages/sync/editor/generic/sorted-patch-list.ts +++ b/src/packages/sync/editor/generic/sorted-patch-list.ts @@ -102,7 +102,7 @@ export class SortedPatchList extends EventEmitter { }; /* Choose the next available time in ms that is congruent to - m modulo n and is larger than any current times. + m modulo n and is larger than any current times. This is a LOGICAL TIME; it does not have to equal the actual wall clock. The key is that it is increasing. The congruence condition is so that any time @@ -134,9 +134,13 @@ export class SortedPatchList extends EventEmitter { if (n <= 0) { n = 1; } - let a = m - (time % n); + // we add 50 to the modulus so that if a bunch of new users are joining at the exact same moment, + // they don't have to be instantly aware of each other for this to keep working. Basically, we + // give ourself a buffer of 10 + const modulus = n + 10; + let a = m - (time % modulus); if (a < 0) { - a += n; + a += modulus; } time += a; // now time = m (mod n) // There is also no possibility of a conflict with a known time diff --git a/src/packages/sync/editor/generic/sync-doc.ts b/src/packages/sync/editor/generic/sync-doc.ts index 2fed66dd7a..33475c3920 100644 --- a/src/packages/sync/editor/generic/sync-doc.ts +++ b/src/packages/sync/editor/generic/sync-doc.ts @@ -160,6 +160,10 @@ export interface SyncOpts0 { // optional filesystem interface. fs?: SyncDocFilesystem; + + // if true, do not implicitly save on commit. This is very + // useful for unit testing to easily simulate offline state. + noAutosave?: boolean; } export interface SyncOpts extends SyncOpts0 { @@ -275,6 +279,8 @@ export class SyncDoc extends EventEmitter { private fs?: SyncDocFilesystem; + private noAutosave?: boolean; + constructor(opts: SyncOpts) { super(); if (opts.string_id === undefined) { @@ -297,6 +303,7 @@ export class SyncDoc extends EventEmitter { "data_server", "ephemeral", "fs", + "noAutosave", ]) { if (opts[field] != undefined) { this[field] = opts[field]; @@ -1289,6 +1296,7 @@ export class SyncDoc extends EventEmitter { desc: { path: this.path }, start_seq: this.last_seq, ephemeral, + noAutosave: this.noAutosave, }); if (this.last_seq) { @@ -1308,6 +1316,7 @@ export class SyncDoc extends EventEmitter { atomic: true, desc: { path: this.path }, ephemeral, + noAutosave: this.noAutosave, }); // also find the correct last_seq: @@ -1355,6 +1364,7 @@ export class SyncDoc extends EventEmitter { immutable: true, desc: { path: this.path }, ephemeral, + noAutosave: this.noAutosave, }); } else if (this.useConat && query.ipywidgets) { synctable = await this.client.synctable_conat(query, { @@ -1370,6 +1380,7 @@ export class SyncDoc extends EventEmitter { config: { max_age: 1000 * 60 * 60 * 24 }, desc: { path: this.path }, ephemeral: true, // ipywidgets state always ephemeral + noAutosave: this.noAutosave, }); } else if (this.useConat && (query.eval_inputs || query.eval_outputs)) { synctable = await this.client.synctable_conat(query, { @@ -1383,6 +1394,7 @@ export class SyncDoc extends EventEmitter { config: { max_age: 5 * 60 * 1000 }, desc: { path: this.path }, ephemeral: true, // eval state (for sagews) is always ephemeral + noAutosave: this.noAutosave, }); } else if (this.useConat) { synctable = await this.client.synctable_conat(query, { @@ -1395,6 +1407,7 @@ export class SyncDoc extends EventEmitter { immutable: true, desc: { path: this.path }, ephemeral, + noAutosave: this.noAutosave, }); } else { // only used for unit tests and the ephemeral messaging composer @@ -3181,10 +3194,15 @@ export class SyncDoc extends EventEmitter { // fine offline, and does not wait until anything // is saved to the network, etc. commit = (emitChangeImmediately = false): boolean => { - if (this.last == null || this.doc == null || this.last.is_equal(this.doc)) { + if ( + this.last == null || + this.doc == null || + (this.last.is_equal(this.doc) && + (this.patch_list?.getHeads().length ?? 0) <= 1) + ) { return false; } - // console.trace('commit'); + // console.trace("commit"); if (emitChangeImmediately) { // used for local clients. NOTE: don't do this without explicit @@ -3197,12 +3215,14 @@ export class SyncDoc extends EventEmitter { // Now save to backend as a new patch: this.emit("user-change"); - const patch = this.last.make_patch(this.doc); // must be nontrivial + const patch = this.last.make_patch(this.doc); this.last = this.doc; // ... and save that to patches table const time = this.next_patch_time(); this.commit_patch(time, patch); - this.save(); // so eventually also gets sent out. + if (!this.noAutosave) { + this.save(); // so eventually also gets sync'd out to other clients + } this.touchProject(); return true; }; @@ -3626,10 +3646,13 @@ export class SyncDoc extends EventEmitter { return; } - // Critical to save what we have now so it doesn't get overwritten during - // before-change or setting this.doc below. This caused - // https://github.com/sagemathinc/cocalc/issues/5871 - this.commit(); + if (!this.last.is_equal(this.doc)) { + // If live versions differs from last commit (or merge of heads), it is + // commit what we have now so it doesn't get overwritten during + // before-change or setting this.doc below. This caused + // https://github.com/sagemathinc/cocalc/issues/5871 + this.commit(); + } if (upstreamPatches && this.state == "ready") { // First save any unsaved changes from the live document, which this @@ -3637,10 +3660,12 @@ export class SyncDoc extends EventEmitter { // rapidly changing live editor with changes not yet saved here. this.emit("before-change"); // As a result of the emit in the previous line, all kinds of - // nontrivial listener code probably just ran, and it should + // nontrivial listener code may have just ran, and it could // have updated this.doc. We commit this.doc, so that the // upstream patches get applied against the correct live this.doc. - this.commit(); + if (!this.last.is_equal(this.doc)) { + this.commit(); + } } // Compute the global current state of the document, From 0ce82c72cc3ae9bf20c388a14272d4b67fc61a44 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 20 Jul 2025 20:22:00 +0000 Subject: [PATCH 040/270] sync-doc: tests using noAutosave --- .../conat/test/sync-doc/conflict.test.ts | 18 +++- .../conat/test/sync-doc/no-autosave.test.ts | 88 +++++++++++++++++++ .../conat/test/sync-doc/syncstring.test.ts | 2 +- src/packages/conat/sync-doc/sync-client.ts | 9 +- src/packages/sync/editor/generic/sync-doc.ts | 16 ++-- src/scripts/runoo | 32 +++++++ 6 files changed, 151 insertions(+), 14 deletions(-) create mode 100644 src/packages/backend/conat/test/sync-doc/no-autosave.test.ts create mode 100755 src/scripts/runoo diff --git a/src/packages/backend/conat/test/sync-doc/conflict.test.ts b/src/packages/backend/conat/test/sync-doc/conflict.test.ts index 4c5c521d02..0533878fe5 100644 --- a/src/packages/backend/conat/test/sync-doc/conflict.test.ts +++ b/src/packages/backend/conat/test/sync-doc/conflict.test.ts @@ -32,6 +32,7 @@ describe("synchronized editing with branching and merging", () => { project_id, path: "a.txt", service: server.service, + noAutosave: true, }); await once(s1, "ready"); @@ -39,6 +40,7 @@ describe("synchronized editing with branching and merging", () => { project_id, path: "a.txt", service: server.service, + noAutosave: true, }); await once(s2, "ready"); expect(s1.to_str()).toBe(""); @@ -52,7 +54,7 @@ describe("synchronized editing with branching and merging", () => { s1.commit(); // delay so s2's time is always bigger than s1's so our unit test // is well defined - await delay(1); + await delay(75); s2.commit(); await s1.save(); await s2.save(); @@ -88,7 +90,7 @@ describe("synchronized editing with branching and merging", () => { it("set values inconsistently again and explicitly resolve the merge conflict in a way that is different than the default", async () => { s1.from_str("xy1"); s1.commit(); - await delay(1); + await delay(75); s2.from_str("xy2"); s2.commit(); await s1.save(); @@ -101,6 +103,7 @@ describe("synchronized editing with branching and merging", () => { // how we resolve the conflict s1.from_str("xy3"); s1.commit(); + await s1.save(); await waitUntilSynced([s1, s2]); // everybody has this state now @@ -109,7 +112,7 @@ describe("synchronized editing with branching and merging", () => { }); }); -describe.only("do the example in the blog post 'Lies I was Told About Collaborative Editing, Part 1: Algorithms for offline editing' -- https://www.moment.dev/blog/lies-i-was-told-pt-1", () => { +describe("do the example in the blog post 'Lies I was Told About Collaborative Editing, Part 1: Algorithms for offline editing' -- https://www.moment.dev/blog/lies-i-was-told-pt-1", () => { const project_id = uuid(); let client1, client2; @@ -123,15 +126,21 @@ describe.only("do the example in the blog post 'Lies I was Told About Collaborat project_id, path, service: server.service, + noAutosave: true, }); await once(alice, "ready"); + await alice.save(); const bob = client2.sync.string({ project_id, path, service: server.service, + noAutosave: true, }); await once(bob, "ready"); + await bob.save(); + await waitUntilSynced([bob, alice]); + return { alice, bob }; } @@ -189,11 +198,12 @@ describe.only("do the example in the blog post 'Lies I was Told About Collaborat expect(bob.to_str()).toEqual(""); }); - it("There are in fact two heads right now, and either party can resolve the merge conflict however they want.", async () => { + it("There are two heads; either client can resolve the merge conflict.", async () => { expect(alice.patch_list.getHeads().length).toBe(2); expect(bob.patch_list.getHeads().length).toBe(2); bob.from_str("The Colour of Pomegranates"); bob.commit(); + await bob.save(); await waitUntilSynced([bob, alice]); expect(alice.to_str()).toEqual("The Colour of Pomegranates"); diff --git a/src/packages/backend/conat/test/sync-doc/no-autosave.test.ts b/src/packages/backend/conat/test/sync-doc/no-autosave.test.ts new file mode 100644 index 0000000000..032526f381 --- /dev/null +++ b/src/packages/backend/conat/test/sync-doc/no-autosave.test.ts @@ -0,0 +1,88 @@ +import { + before, + after, + uuid, + connect, + server, + once, + delay, + waitUntilSynced, +} from "./setup"; + +beforeAll(before); +afterAll(after); + +describe("confirm noAutosave works", () => { + const project_id = uuid(); + const path = "a.txt"; + let client1, client2, s1, s2; + + it("creates two clients with noAutosave enabled", async () => { + client1 = connect(); + client2 = connect(); + await client1 + .fs({ project_id, service: server.service }) + .writeFile(path, ""); + s1 = client1.sync.string({ + project_id, + path, + service: server.service, + noAutosave: true, + }); + + await once(s1, "ready"); + + s2 = client2.sync.string({ + project_id, + path, + service: server.service, + noAutosave: true, + }); + await once(s2, "ready"); + expect(s1.noAutosave).toEqual(true); + expect(s2.noAutosave).toEqual(true); + }); + + const howLong = 750; + it(`write a change to s1 and commit it, but observe s2 does not see it even after ${howLong}ms (which should be plenty of time)`, async () => { + s1.from_str("new-ver"); + s1.commit(); + + expect(s2.to_str()).toEqual(""); + await delay(howLong); + expect(s2.to_str()).toEqual(""); + }); + + it("explicitly save and see s2 does get the change", async () => { + await s1.save(); + await waitUntilSynced([s1, s2]); + expect(s2.to_str()).toEqual("new-ver"); + }); + + it("make a change resulting in two heads", async () => { + s2.from_str("new-ver-1"); + s2.commit(); + // no background saving happening: + await delay(100); + s1.from_str("new-ver-2"); + s1.commit(); + await Promise.all([s1.save(), s2.save()]); + }); + + it("there are two heads and value is merged", async () => { + await waitUntilSynced([s1, s2]); + expect(s1.to_str()).toEqual("new-ver-1-2"); + expect(s2.to_str()).toEqual("new-ver-1-2"); + expect(s1.patch_list.getHeads().length).toBe(2); + expect(s2.patch_list.getHeads().length).toBe(2); + }); + + it("string state info matches", async () => { + const a1 = s1.syncstring_table_get_one().toJS(); + const a2 = s2.syncstring_table_get_one().toJS(); + expect(a1).toEqual(a2); + expect(new Set(a1.users)).toEqual( + new Set([s1.client.client_id(), s2.client.client_id()]), + ); + }); +}); diff --git a/src/packages/backend/conat/test/sync-doc/syncstring.test.ts b/src/packages/backend/conat/test/sync-doc/syncstring.test.ts index e67361ddfb..3990baaa55 100644 --- a/src/packages/backend/conat/test/sync-doc/syncstring.test.ts +++ b/src/packages/backend/conat/test/sync-doc/syncstring.test.ts @@ -124,4 +124,4 @@ describe("synchronized editing with two copies of a syncstring", () => { s2.show_history({ log: (x) => v2.push(x) }); expect(v1).toEqual(v2); }); -}); +}); \ No newline at end of file diff --git a/src/packages/conat/sync-doc/sync-client.ts b/src/packages/conat/sync-doc/sync-client.ts index ebc6377f8e..9609dce5a3 100644 --- a/src/packages/conat/sync-doc/sync-client.ts +++ b/src/packages/conat/sync-doc/sync-client.ts @@ -56,8 +56,13 @@ export class SyncClient extends EventEmitter implements Client0 { return new PubSub({ client: this.client, ...opts }); }; - // account_id or project_id - client_id = (): string => this.client.id; + // account_id or project_id or hub_id or fallback client.id + client_id = (): string => { + const user = this.client.info?.user; + return ( + user?.account_id ?? user?.project_id ?? user?.hub_id ?? this.client.id + ); + }; server_time = (): Date => { return new Date(); diff --git a/src/packages/sync/editor/generic/sync-doc.ts b/src/packages/sync/editor/generic/sync-doc.ts index 33475c3920..66f0939626 100644 --- a/src/packages/sync/editor/generic/sync-doc.ts +++ b/src/packages/sync/editor/generic/sync-doc.ts @@ -1626,7 +1626,7 @@ export class SyncDoc extends EventEmitter { // normally this only happens in a later event loop, // so force it now. dbg("handling patch update queue since", this.patch_list.count()); - await this.handle_patch_update_queue(); + await this.handle_patch_update_queue(true); assertDefined(this.patch_list); dbg("done handling, now ", this.patch_list.count()); if (this.patch_list.count() === 0) { @@ -1636,7 +1636,7 @@ export class SyncDoc extends EventEmitter { // This is the root cause of https://github.com/sagemathinc/cocalc/issues/2382 await once(this.patches_table, "change"); dbg("got patches_table change"); - await this.handle_patch_update_queue(); + await this.handle_patch_update_queue(true); dbg("handled update queue"); } } @@ -2269,7 +2269,7 @@ export class SyncDoc extends EventEmitter { // above async waits could have resulted in state change. return; } - await this.handle_patch_update_queue(); + await this.handle_patch_update_queue(true); if (this.state != "ready") { return; } @@ -3539,7 +3539,7 @@ export class SyncDoc extends EventEmitter { Whenever new patches are added to this.patches_table, their timestamp gets added to this.patch_update_queue. */ - private handle_patch_update_queue = async (): Promise => { + private handle_patch_update_queue = async (save = false): Promise => { const dbg = this.dbg("handle_patch_update_queue"); try { this.handle_patch_update_queue_running = true; @@ -3574,9 +3574,11 @@ export class SyncDoc extends EventEmitter { dbg("waiting for remote and doc to sync..."); this.sync_remote_and_doc(v.length > 0); - await this.patches_table.save(); - if (this.state === ("closed" as State)) return; // closed during await; nothing further to do - dbg("remote and doc now synced"); + if (save || !this.noAutosave) { + await this.patches_table.save(); + if (this.state === ("closed" as State)) return; // closed during await; nothing further to do + dbg("remote and doc now synced"); + } if (this.patch_update_queue.length > 0) { // It is very important that next loop happen in a later diff --git a/src/scripts/runoo b/src/scripts/runoo new file mode 100755 index 0000000000..eb8a32fe8b --- /dev/null +++ b/src/scripts/runoo @@ -0,0 +1,32 @@ +#!/usr/bin/env python3 + +""" +This is just meant to be a quick and dirty python script so I can run other +scripts (e.g., a unit test runner) to get a sense if they are flaky or not.: + +runoo 20 python test.py # runs "python test.py" 20 times in parallel, ncpus at once + +""" + +import sys, os, time +from concurrent.futures import ProcessPoolExecutor, as_completed +import multiprocessing + +def run_cmd(cmd, i): + print('\n'*10) + print(f'Loop: {i+1} , Time: {round(time.time() - start)}, Command: {cmd}') + ret = os.system(cmd) + if ret: + raise RuntimeError(f'Command failed on run {i+1}') + return ret + +if __name__ == '__main__': + n = int(sys.argv[1]) + cmd = ' '.join(sys.argv[2:]) + k = multiprocessing.cpu_count() + start = time.time() + + with ProcessPoolExecutor(max_workers=k) as executor: + futures = [executor.submit(run_cmd, cmd, i) for i in range(n)] + for f in as_completed(futures): + f.result() # Raises exception if failed \ No newline at end of file From 69ea3a1f29ea4bc70fefea194b87b39a754caa1c Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 20 Jul 2025 20:37:15 +0000 Subject: [PATCH 041/270] improve runoo --- .../conat/test/sync-doc/conflict.test.ts | 10 ++++--- src/scripts/runoo | 26 ++++++++++++++----- 2 files changed, 25 insertions(+), 11 deletions(-) diff --git a/src/packages/backend/conat/test/sync-doc/conflict.test.ts b/src/packages/backend/conat/test/sync-doc/conflict.test.ts index 0533878fe5..00b590f8fe 100644 --- a/src/packages/backend/conat/test/sync-doc/conflict.test.ts +++ b/src/packages/backend/conat/test/sync-doc/conflict.test.ts @@ -21,6 +21,8 @@ import { beforeAll(before); afterAll(after); +const GAP_DELAY = 50; + describe("synchronized editing with branching and merging", () => { const project_id = uuid(); let s1, s2, client1, client2; @@ -49,12 +51,12 @@ describe("synchronized editing with branching and merging", () => { }); it("both clients set the first version independently and inconsistently", async () => { - s1.from_str("x"); s2.from_str("y"); + s1.from_str("x"); s1.commit(); // delay so s2's time is always bigger than s1's so our unit test // is well defined - await delay(75); + await delay(GAP_DELAY); s2.commit(); await s1.save(); await s2.save(); @@ -90,7 +92,7 @@ describe("synchronized editing with branching and merging", () => { it("set values inconsistently again and explicitly resolve the merge conflict in a way that is different than the default", async () => { s1.from_str("xy1"); s1.commit(); - await delay(75); + await delay(GAP_DELAY); s2.from_str("xy2"); s2.commit(); await s1.save(); @@ -100,7 +102,7 @@ describe("synchronized editing with branching and merging", () => { expect(s1.to_str()).toEqual("xy12"); expect(s2.to_str()).toEqual("xy12"); - // how we resolve the conflict + // resolve the conflict in our own way s1.from_str("xy3"); s1.commit(); await s1.save(); diff --git a/src/scripts/runoo b/src/scripts/runoo index eb8a32fe8b..4445904c26 100755 --- a/src/scripts/runoo +++ b/src/scripts/runoo @@ -12,13 +12,23 @@ import sys, os, time from concurrent.futures import ProcessPoolExecutor, as_completed import multiprocessing +failed = False def run_cmd(cmd, i): - print('\n'*10) - print(f'Loop: {i+1} , Time: {round(time.time() - start)}, Command: {cmd}') - ret = os.system(cmd) - if ret: - raise RuntimeError(f'Command failed on run {i+1}') - return ret + global failed + if failed: + return + print('\n'*5) + print('*'*60) + print(f'* Loop: {i+1}/{n} , Time: {round(time.time() - start)}, Command: {cmd}') + print('*'*60) + if os.system(cmd): + failed = True + print('\n'*5) + print('*'*60) + print(f'* Command failed on run {i+1}**') + print('*'*60) + print('\n'*5) + sys.exit(1); if __name__ == '__main__': n = int(sys.argv[1]) @@ -29,4 +39,6 @@ if __name__ == '__main__': with ProcessPoolExecutor(max_workers=k) as executor: futures = [executor.submit(run_cmd, cmd, i) for i in range(n)] for f in as_completed(futures): - f.result() # Raises exception if failed \ No newline at end of file + f.result() # Raises exception if failed + + print(f"successfully ran {n} times") \ No newline at end of file From c17234b5386371f0e8803581e46f1a13581d9275 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 20 Jul 2025 20:57:44 +0000 Subject: [PATCH 042/270] syncdoc: unit test involving merging many heads --- .../conat/test/sync-doc/conflict.test.ts | 63 +++++++++++++++++++ src/packages/conat/sync/dko.ts | 3 +- src/scripts/runoo | 6 +- 3 files changed, 70 insertions(+), 2 deletions(-) diff --git a/src/packages/backend/conat/test/sync-doc/conflict.test.ts b/src/packages/backend/conat/test/sync-doc/conflict.test.ts index 00b590f8fe..34797cc53e 100644 --- a/src/packages/backend/conat/test/sync-doc/conflict.test.ts +++ b/src/packages/backend/conat/test/sync-doc/conflict.test.ts @@ -17,6 +17,7 @@ import { delay, waitUntilSynced, } from "./setup"; +import { split } from "@cocalc/util/misc"; beforeAll(before); afterAll(after); @@ -212,3 +213,65 @@ describe("do the example in the blog post 'Lies I was Told About Collaborative E expect(bob.to_str()).toEqual("The Colour of Pomegranates"); }); }); + +const numHeads = 15; +describe.only(`create editing conflict with ${numHeads} heads`, () => { + const project_id = uuid(); + let docs: any[] = [], + clients: any[] = []; + + it(`create ${numHeads} clients`, async () => { + const v: any[] = []; + for (let i = 0; i < numHeads; i++) { + const client = connect(); + clients.push(client); + const doc = client.sync.string({ + project_id, + path: "a.txt", + service: server.service, + noAutosave: true, + }); + docs.push(doc); + v.push(once(doc, "ready")); + } + await Promise.all(v); + }); + + it("every client writes a different value all at once", async () => { + for (let i = 0; i < numHeads; i++) { + docs[i].from_str(`${i} `); + docs[i].commit(); + docs[i].save(); + } + await waitUntilSynced(docs); + const heads = docs[0].patch_list.getHeads(); + expect(heads.length).toBe(docs.length); + }); + + it("merge -- order is random, but value is consistent", async () => { + const value = docs[0].to_str(); + let v = new Set(); + for (let i = 0; i < numHeads; i++) { + v.add(`${i}`); + expect(docs[i].to_str()).toEqual(value); + } + const t = new Set(split(docs[0].to_str())); + expect(t).toEqual(v); + }); + + it(`resolve the merge conflict -- all ${numHeads} clients then see the resolution`, async () => { + let r = ""; + for (let i = 0; i < numHeads; i++) { + r += `${i} `; + } + docs[0].from_str(r); + docs[0].commit(); + await docs[0].save(); + + await waitUntilSynced(docs); + for (let i = 0; i < numHeads; i++) { + expect(docs[i].to_str()).toEqual(r); + } + // docs[0].show_history(); + }); +}); diff --git a/src/packages/conat/sync/dko.ts b/src/packages/conat/sync/dko.ts index f4e773a90f..c0c04c1b06 100644 --- a/src/packages/conat/sync/dko.ts +++ b/src/packages/conat/sync/dko.ts @@ -1,7 +1,7 @@ /* Distributed eventually consistent key:object store, where changes propogate sparsely. -The "values" MUST be objects and no keys or fields of objects can container the +The "values" MUST be objects and no keys or fields of objects can container the sep character, which is '|' by default. NOTE: Whenever you do a set, the lodash isEqual function is used to see which fields @@ -38,6 +38,7 @@ export class DKO extends EventEmitter { constructor(private opts: DKVOptions) { super(); + this.setMaxListeners(1000); return new Proxy(this, { deleteProperty(target, prop) { if (typeof prop == "string") { diff --git a/src/scripts/runoo b/src/scripts/runoo index 4445904c26..4c0f8e93cc 100755 --- a/src/scripts/runoo +++ b/src/scripts/runoo @@ -41,4 +41,8 @@ if __name__ == '__main__': for f in as_completed(futures): f.result() # Raises exception if failed - print(f"successfully ran {n} times") \ No newline at end of file + print('\n'*5) + print('*'*60) + print(f"Successfully ran {n} times") + print('*'*60) + \ No newline at end of file From 8da8ef57ba93fc532876afaccbf7e70555eb14c2 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 20 Jul 2025 21:10:51 +0000 Subject: [PATCH 043/270] sync-doc: make the "read from disk" command "readFile" and be public; start writing unit tests for watch (not implemented) --- .../conat/test/sync-doc/watch-file.test.ts | 39 +++++ src/packages/sync/editor/generic/sync-doc.ts | 139 +++++++++--------- 2 files changed, 107 insertions(+), 71 deletions(-) create mode 100644 src/packages/backend/conat/test/sync-doc/watch-file.test.ts diff --git a/src/packages/backend/conat/test/sync-doc/watch-file.test.ts b/src/packages/backend/conat/test/sync-doc/watch-file.test.ts new file mode 100644 index 0000000000..8e0a2dd92d --- /dev/null +++ b/src/packages/backend/conat/test/sync-doc/watch-file.test.ts @@ -0,0 +1,39 @@ +import { before, after, uuid, connect, server, once, wait } from "./setup"; + +beforeAll(before); +afterAll(after); + +describe("basic watching of file on disk happens automatically", () => { + const project_id = uuid(); + const path = "a.txt"; + let client, s, fs; + + it("creates two clients with noAutosave enabled", async () => { + client = connect(); + fs = client.fs({ project_id, service: server.service }); + await fs.writeFile(path, "init"); + s = client.sync.string({ + project_id, + path, + service: server.service, + }); + await once(s, "ready"); + expect(s.to_str()).toEqual("init"); + }); + + it("changes the file on disk and call readFile to immediately update", async () => { + await fs.writeFile(path, "modified"); + await s.readFile(); + expect(s.to_str()).toEqual("modified"); + }); + + // this is not implemented yet + it.skip("changes the file on disk and the watcher automatically updates with no explicit call needed", async () => { + await fs.writeFile(path, "changed again!"); + await wait({ + until: () => { + return s.to_str() == "changed again!"; + }, + }); + }); +}); diff --git a/src/packages/sync/editor/generic/sync-doc.ts b/src/packages/sync/editor/generic/sync-doc.ts index 66f0939626..acc042b628 100644 --- a/src/packages/sync/editor/generic/sync-doc.ts +++ b/src/packages/sync/editor/generic/sync-doc.ts @@ -81,7 +81,6 @@ import { cancel_scheduled, once, retry_until_success, - reuse_in_flight_methods, until, } from "@cocalc/util/async-utils"; import { wait } from "@cocalc/util/async-wait"; @@ -342,13 +341,6 @@ export class SyncDoc extends EventEmitter { // to update this). this.cursor_last_time = this.client?.server_time(); - reuse_in_flight_methods(this, [ - "save", - "save_to_disk", - "load_from_disk", - "handle_patch_update_queue", - ]); - if (this.change_throttle) { this.emit_change = throttle(this.emit_change, this.change_throttle); } @@ -555,7 +547,7 @@ export class SyncDoc extends EventEmitter { await this.update_watch_path(); } else { // load our state from the disk - await this.load_from_disk(); + await this.readFile(); // we were not acting as the file server, but now we need. Let's // step up to the plate. // start watching filesystem @@ -1845,7 +1837,7 @@ export class SyncDoc extends EventEmitter { dbg( `disk file changed more recently than edits (or first load), so loading, ${stats.ctime} > ${last_changed}; firstLoad=${firstLoad}`, ); - size = await this.load_from_disk(); + size = await this.readFile(); if (firstLoad) { dbg("emitting first-load event"); // this event is emited the first time the document is ever loaded from disk. @@ -3000,7 +2992,7 @@ export class SyncDoc extends EventEmitter { // to save was at least RECENT_SAVE_TO_DISK_MS ago, and it finished, // so definitely this change event was not caused by it. dbg("load_from_disk since no recent save to disk"); - await this.load_from_disk(); + await this.readFile(); return; } }; @@ -3039,7 +3031,10 @@ export class SyncDoc extends EventEmitter { return size; }; - private load_from_disk = async (): Promise => { + readFile = reuseInFlight(async (): Promise => { + if (this.fs != null) { + return await this.fsLoadFromDisk(); + } if (this.client.path_exists == null) { throw Error("legacy clients must define path_exists"); } @@ -3075,7 +3070,7 @@ export class SyncDoc extends EventEmitter { // save new version to database, which we just set via from_str. await this.save(); return size; - }; + }); private set_save = async (save: { state: string; @@ -3240,7 +3235,7 @@ export class SyncDoc extends EventEmitter { /* Initiates a save of file to disk, then waits for the state to change. */ - save_to_disk = async (): Promise => { + save_to_disk = reuseInFlight(async (): Promise => { if (this.state != "ready") { // We just make save_to_disk a successful // no operation, if the document is either @@ -3309,7 +3304,7 @@ export class SyncDoc extends EventEmitter { await this.wait_for_save_to_disk_done(); } this.update_has_unsaved_changes(); - }; + }); /* Export the (currently loaded) history of editing of this document to a simple JSON-able object. */ @@ -3539,68 +3534,70 @@ export class SyncDoc extends EventEmitter { Whenever new patches are added to this.patches_table, their timestamp gets added to this.patch_update_queue. */ - private handle_patch_update_queue = async (save = false): Promise => { - const dbg = this.dbg("handle_patch_update_queue"); - try { - this.handle_patch_update_queue_running = true; - while (this.state != "closed" && this.patch_update_queue.length > 0) { - dbg("queue size = ", this.patch_update_queue.length); - const v: Patch[] = []; - for (const key of this.patch_update_queue) { - let x = this.patches_table.get(key); - if (x == null) { - continue; + private handle_patch_update_queue = reuseInFlight( + async (save = false): Promise => { + const dbg = this.dbg("handle_patch_update_queue"); + try { + this.handle_patch_update_queue_running = true; + while (this.state != "closed" && this.patch_update_queue.length > 0) { + dbg("queue size = ", this.patch_update_queue.length); + const v: Patch[] = []; + for (const key of this.patch_update_queue) { + let x = this.patches_table.get(key); + if (x == null) { + continue; + } + if (!Map.isMap(x)) { + // TODO: my NATS synctable-stream doesn't convert to immutable on get. + x = fromJS(x); + } + // may be null, e.g., when deleted. + const t = x.get("time"); + // Optimization: only need to process patches that we didn't + // create ourselves during this session. + if (t && !this.my_patches[t.valueOf()]) { + const p = this.processPatch({ x }); + //dbg(`patch=${JSON.stringify(p)}`); + if (p != null) { + v.push(p); + } + } } - if (!Map.isMap(x)) { - // TODO: my NATS synctable-stream doesn't convert to immutable on get. - x = fromJS(x); + this.patch_update_queue = []; + this.emit("patch-update-queue-empty"); + assertDefined(this.patch_list); + this.patch_list.add(v); + + dbg("waiting for remote and doc to sync..."); + this.sync_remote_and_doc(v.length > 0); + if (save || !this.noAutosave) { + await this.patches_table.save(); + if (this.state === ("closed" as State)) return; // closed during await; nothing further to do + dbg("remote and doc now synced"); } - // may be null, e.g., when deleted. - const t = x.get("time"); - // Optimization: only need to process patches that we didn't - // create ourselves during this session. - if (t && !this.my_patches[t.valueOf()]) { - const p = this.processPatch({ x }); - //dbg(`patch=${JSON.stringify(p)}`); - if (p != null) { - v.push(p); - } + + if (this.patch_update_queue.length > 0) { + // It is very important that next loop happen in a later + // event loop to avoid the this.sync_remote_and_doc call + // in this.handle_patch_update_queue above from causing + // sync_remote_and_doc to get called from within itself, + // due to synctable changes being emited on save. + dbg("wait for next event loop"); + await delay(1); } } - this.patch_update_queue = []; - this.emit("patch-update-queue-empty"); - assertDefined(this.patch_list); - this.patch_list.add(v); - - dbg("waiting for remote and doc to sync..."); - this.sync_remote_and_doc(v.length > 0); - if (save || !this.noAutosave) { - await this.patches_table.save(); - if (this.state === ("closed" as State)) return; // closed during await; nothing further to do - dbg("remote and doc now synced"); - } + } finally { + if (this.state == "closed") return; // got closed, so nothing further to do - if (this.patch_update_queue.length > 0) { - // It is very important that next loop happen in a later - // event loop to avoid the this.sync_remote_and_doc call - // in this.handle_patch_update_queue above from causing - // sync_remote_and_doc to get called from within itself, - // due to synctable changes being emited on save. - dbg("wait for next event loop"); - await delay(1); - } + // OK, done and nothing in the queue + // Notify save() to try again -- it may have + // paused waiting for this to clear. + dbg("done"); + this.handle_patch_update_queue_running = false; + this.emit("handle_patch_update_queue_done"); } - } finally { - if (this.state == "closed") return; // got closed, so nothing further to do - - // OK, done and nothing in the queue - // Notify save() to try again -- it may have - // paused waiting for this to clear. - dbg("done"); - this.handle_patch_update_queue_running = false; - this.emit("handle_patch_update_queue_done"); - } - }; + }, + ); /* Disable and enable sync. When disabled we still collect patches from upstream (but do not apply them From 681e2a2820e9aa14819fae59861931c80ec0c1df Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 20 Jul 2025 22:27:09 +0000 Subject: [PATCH 044/270] sync-doc: first steps toward client-only fs listener --- .../conat/test/sync-doc/watch-file.test.ts | 13 +++- src/packages/sync/editor/generic/sync-doc.ts | 64 +++++++++++++++---- 2 files changed, 62 insertions(+), 15 deletions(-) diff --git a/src/packages/backend/conat/test/sync-doc/watch-file.test.ts b/src/packages/backend/conat/test/sync-doc/watch-file.test.ts index 8e0a2dd92d..873b81c886 100644 --- a/src/packages/backend/conat/test/sync-doc/watch-file.test.ts +++ b/src/packages/backend/conat/test/sync-doc/watch-file.test.ts @@ -28,7 +28,7 @@ describe("basic watching of file on disk happens automatically", () => { }); // this is not implemented yet - it.skip("changes the file on disk and the watcher automatically updates with no explicit call needed", async () => { + it("change file on disk and it automatically updates with no explicit call needed", async () => { await fs.writeFile(path, "changed again!"); await wait({ until: () => { @@ -37,3 +37,14 @@ describe("basic watching of file on disk happens automatically", () => { }); }); }); + +/* +watching of file with multiple clients + +-- only one does the actual file load + +-- when one writes file to disk, another doesn't try to load it + +(various ways to do that: sticky fs server would mean only one is +writing backend can ignore the resulting change event) +*/ diff --git a/src/packages/sync/editor/generic/sync-doc.ts b/src/packages/sync/editor/generic/sync-doc.ts index acc042b628..538145d147 100644 --- a/src/packages/sync/editor/generic/sync-doc.ts +++ b/src/packages/sync/editor/generic/sync-doc.ts @@ -117,6 +117,7 @@ import { CONAT_OPEN_FILE_TOUCH_INTERVAL } from "@cocalc/util/conat"; import mergeDeep from "@cocalc/util/immutable-deep-merge"; import { JUPYTER_SYNCDB_EXTENSIONS } from "@cocalc/util/jupyter/names"; import { LegacyHistory } from "./legacy"; +import { type Filesystem } from "@cocalc/conat/files/fs"; import { getLogger } from "@cocalc/conat/client"; const DEBUG = false; @@ -124,12 +125,6 @@ const DEBUG = false; export type State = "init" | "ready" | "closed"; export type DataServer = "project" | "database"; -export interface SyncDocFilesystem { - readFile: (path: string, encoding?: any) => Promise; - writeFile: (path: string, data: string | Buffer) => Promise; - stat: (path: string) => Promise; // todo -} - export interface SyncOpts0 { project_id: string; path: string; @@ -157,8 +152,8 @@ export interface SyncOpts0 { // which data/changefeed server to use data_server?: DataServer; - // optional filesystem interface. - fs?: SyncDocFilesystem; + // filesystem interface. + fs?: Filesystem; // if true, do not implicitly save on commit. This is very // useful for unit testing to easily simulate offline state. @@ -276,7 +271,7 @@ export class SyncDoc extends EventEmitter { private useConat: boolean; legacy: LegacyHistory; - private fs?: SyncDocFilesystem; + private fs?: Filesystem; private noAutosave?: boolean; @@ -396,7 +391,6 @@ export class SyncDoc extends EventEmitter { // Success -- everything initialized with no issues. this.set_state("ready"); - this.init_watch(); this.emit_change(); // from nothing to something. }; @@ -1174,6 +1168,8 @@ export class SyncDoc extends EventEmitter { // a file change in its current state. this.update_watch_path(); // no input = closes it, if open + this.fsCloseFileWatcher(); + if (this.patch_list != null) { // not async -- just a data structure in memory this.patch_list.close(); @@ -1494,6 +1490,7 @@ export class SyncDoc extends EventEmitter { this.init_cursors(), this.init_evaluator(), this.init_ipywidgets(), + this.initFileWatcher(), ]); this.assert_not_closed( "initAll -- successful init patch_list, cursors, evaluator, and ipywidgets", @@ -2871,7 +2868,11 @@ export class SyncDoc extends EventEmitter { this.emit("metadata-change"); }; - private init_watch = async (): Promise => { + private initFileWatcher = async (): Promise => { + if (this.fs != null) { + return await this.fsInitFileWatcher(); + } + if (!(await this.isFileServer())) { // ensures we are NOT watching anything await this.update_watch_path(); @@ -3026,8 +3027,9 @@ export class SyncDoc extends EventEmitter { } } // save new version to stream, which we just set via from_str - this.commit(); + this.commit(true); await this.save(); + this.emit("after-change"); return size; }; @@ -3229,8 +3231,12 @@ export class SyncDoc extends EventEmitter { return; } dbg(); - if (this.fs == null) throw Error("bug"); - await this.fs.writeFile(this.path, this.to_str()); + if (this.fs == null) { + throw Error("bug"); + } + const value = this.to_str(); + console.log("fs.writeFile", this.path); + await this.fs.writeFile(this.path, value); }; /* Initiates a save of file to disk, then waits for the @@ -3754,6 +3760,36 @@ export class SyncDoc extends EventEmitter { }, ); }; + + private fsFileWatcher?: any; + private fsInitFileWatcher = async () => { + if (this.fs == null) { + throw Error("this.fs must be defined"); + } + console.log("watching for changes"); + // use this.fs interface to watch path for changes. + this.fsFileWatcher = await this.fs.watch(this.path); + (async () => { + for await (const { eventType } of this.fsFileWatcher) { + console.log("got change", eventType); + if (eventType == "change" || eventType == "rename") { + await this.fsLoadFromDisk(); + } + if (eventType == "rename") { + this.fsFileWatcher.close(); + // start a new watcher since file descriptor changed + this.fsInitFileWatcher(); + return; + } + } + console.log("done watching"); + })(); + }; + + private fsCloseFileWatcher = () => { + this.fsFileWatcher?.close(); + delete this.fsFileWatcher; + }; } function isCompletePatchStream(dstream) { From f804b01b61be119b3a83570c3c5916169589ba9d Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 20 Jul 2025 23:16:29 +0000 Subject: [PATCH 045/270] sync-doc: working on multiple clients efficiently watching for fs changes without interfering with each other --- .../conat/test/sync-doc/watch-file.test.ts | 25 +++++++- src/packages/backend/files/sandbox/index.ts | 13 +--- src/packages/conat/files/watch.ts | 62 ++++++++++++++----- src/packages/sync/editor/generic/sync-doc.ts | 25 ++++++-- 4 files changed, 93 insertions(+), 32 deletions(-) diff --git a/src/packages/backend/conat/test/sync-doc/watch-file.test.ts b/src/packages/backend/conat/test/sync-doc/watch-file.test.ts index 873b81c886..fc345b0947 100644 --- a/src/packages/backend/conat/test/sync-doc/watch-file.test.ts +++ b/src/packages/backend/conat/test/sync-doc/watch-file.test.ts @@ -27,7 +27,6 @@ describe("basic watching of file on disk happens automatically", () => { expect(s.to_str()).toEqual("modified"); }); - // this is not implemented yet it("change file on disk and it automatically updates with no explicit call needed", async () => { await fs.writeFile(path, "changed again!"); await wait({ @@ -36,6 +35,30 @@ describe("basic watching of file on disk happens automatically", () => { }, }); }); + + let client2, s2; + it("file watching also works if there are multiple clients, with only one handling the change", async () => { + client2 = connect(); + s2 = client2.sync.string({ + project_id, + path, + service: server.service, + }); + let c = 0, + c2 = 0; + s.on("after-change", () => c++); + s2.on("after-change", () => c2++); + + await fs.writeFile(path, "version3"); + await wait({ + until: () => { + return s2.to_str() == "version3" && s.to_str() == "version3"; + }, + }); + expect(s.to_str()).toEqual("version3"); + expect(s2.to_str()).toEqual("version3"); + expect(c + c2).toBe(1); + }); }); /* diff --git a/src/packages/backend/files/sandbox/index.ts b/src/packages/backend/files/sandbox/index.ts index 2888e3a480..27b4a800ad 100644 --- a/src/packages/backend/files/sandbox/index.ts +++ b/src/packages/backend/files/sandbox/index.ts @@ -69,6 +69,7 @@ import getListing from "@cocalc/backend/get-listing"; import { join, resolve } from "path"; import { replace_all } from "@cocalc/util/misc"; import { EventIterator } from "@cocalc/util/event-iterator"; +import { type WatchOptions } from "@cocalc/conat/files/watch"; export class SandboxedFilesystem { // path should be the path to a FOLDER on the filesystem (not a file) @@ -231,17 +232,7 @@ export class SandboxedFilesystem { await utimes(await this.safeAbsPath(path), atime, mtime); }; - watch = async ( - filename: string, - options?: { - persistent?: boolean; - recursive?: boolean; - encoding?: string; - signal?: AbortSignal; - maxQueue?: number; - overflow?: "ignore" | "throw"; - }, - ) => { + watch = async (filename: string, options?: WatchOptions) => { // NOTE: in node v24 they fixed the fs/promises watch to have a queue, but previous // versions were clearly badly implemented so we reimplement it from scratch // using the non-promise watch. diff --git a/src/packages/conat/files/watch.ts b/src/packages/conat/files/watch.ts index 47dd65d939..942d3116b5 100644 --- a/src/packages/conat/files/watch.ts +++ b/src/packages/conat/files/watch.ts @@ -15,7 +15,21 @@ const logger = getLogger("conat:files:watch"); // (path:string, options:WatchOptions) => AsyncIterator type AsyncWatchFunction = any; -type WatchOptions = any; + +// see https://nodejs.org/docs/latest/api/fs.html#fspromiseswatchfilename-options +export interface WatchOptions { + persistent?: boolean; + recursive?: boolean; + encoding?: string; + signal?: AbortSignal; + maxQueue?: number; + overflow?: "ignore" | "throw"; + + // if more than one client is actively watching the same path and has unique set, only one + // will receive updates. Also, if there are multiple clients with unique set, the options + // of all but the first are ignored. + unique?: boolean; +} export function watchServer({ client, @@ -29,29 +43,47 @@ export function watchServer({ const server: ConatSocketServer = client.socket.listen(subject); logger.debug("server: listening on ", { subject }); + //const unique; + async function handleUnique({ mesg, socket, path, options }) { + const w = await watch(path, options); + socket.once("closed", () => { + w.close(); + }); + await mesg.respond(); + for await (const event of w) { + socket.write(event); + } + } + + async function handleNonUnique({ mesg, socket, path, options }) { + const w = await watch(path, options); + socket.once("closed", () => { + w.close(); + }); + await mesg.respond(); + for await (const event of w) { + socket.write(event); + } + } + server.on("connection", (socket: ServerSocket) => { logger.debug("server: got new connection", { id: socket.id, subject: socket.subject, }); - let w: undefined | ReturnType = undefined; - socket.on("closed", () => { - w?.close(); - w = undefined; - }); - + let initialized = false; socket.on("request", async (mesg) => { try { + if (initialized) { + throw Error("already initialized"); + } + initialized = true; const { path, options } = mesg.data; logger.debug("got request", { path, options }); - if (w != null) { - w.close(); - w = undefined; - } - w = await watch(path, options); - await mesg.respond(); - for await (const event of w) { - socket.write(event); + if (options?.unique) { + await handleUnique({ mesg, socket, path, options }); + } else { + await handleNonUnique({ mesg, socket, path, options }); } } catch (err) { mesg.respondSync(null, { diff --git a/src/packages/sync/editor/generic/sync-doc.ts b/src/packages/sync/editor/generic/sync-doc.ts index 538145d147..79c65b3739 100644 --- a/src/packages/sync/editor/generic/sync-doc.ts +++ b/src/packages/sync/editor/generic/sync-doc.ts @@ -82,6 +82,7 @@ import { once, retry_until_success, until, + asyncDebounce, } from "@cocalc/util/async-utils"; import { wait } from "@cocalc/util/async-wait"; import { @@ -3018,6 +3019,7 @@ export class SyncDoc extends EventEmitter { size = contents.length; this.from_str(contents); } catch (err) { + console.log(err); if (err.code == "ENOENT") { dbg("file no longer exists -- setting to blank"); size = 0; @@ -3761,19 +3763,32 @@ export class SyncDoc extends EventEmitter { ); }; + private fsLoadFromDiskDebounced = asyncDebounce( + async () => { + try { + await this.fsLoadFromDisk(); + } catch {} + }, + 50, + { + leading: false, + trailing: true, + }, + ); + private fsFileWatcher?: any; private fsInitFileWatcher = async () => { if (this.fs == null) { throw Error("this.fs must be defined"); } - console.log("watching for changes"); + // console.log("watching for changes"); // use this.fs interface to watch path for changes. - this.fsFileWatcher = await this.fs.watch(this.path); + this.fsFileWatcher = await this.fs.watch(this.path, { unique: true }); (async () => { for await (const { eventType } of this.fsFileWatcher) { - console.log("got change", eventType); + // console.log("got change", eventType); if (eventType == "change" || eventType == "rename") { - await this.fsLoadFromDisk(); + this.fsLoadFromDiskDebounced(); } if (eventType == "rename") { this.fsFileWatcher.close(); @@ -3782,7 +3797,7 @@ export class SyncDoc extends EventEmitter { return; } } - console.log("done watching"); + //console.log("done watching"); })(); }; From e64eee33aa7a8201cb84b7e98b4b5333e47f9576 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 20 Jul 2025 23:54:38 +0000 Subject: [PATCH 046/270] sync-doc: implement a pretty nice approach to filesystem watching for sync editing --- .../conat/test/sync-doc/watch-file.test.ts | 37 ++++++++++++++++++- src/packages/conat/files/watch.ts | 34 ++++++++++++++--- src/packages/sync/editor/generic/sync-doc.ts | 9 +++-- 3 files changed, 68 insertions(+), 12 deletions(-) diff --git a/src/packages/backend/conat/test/sync-doc/watch-file.test.ts b/src/packages/backend/conat/test/sync-doc/watch-file.test.ts index fc345b0947..a1ad595960 100644 --- a/src/packages/backend/conat/test/sync-doc/watch-file.test.ts +++ b/src/packages/backend/conat/test/sync-doc/watch-file.test.ts @@ -44,10 +44,11 @@ describe("basic watching of file on disk happens automatically", () => { path, service: server.service, }); + await once(s2, "ready"); let c = 0, c2 = 0; - s.on("after-change", () => c++); - s2.on("after-change", () => c2++); + s.on("handle-file-change", () => c++); + s2.on("handle-file-change", () => c2++); await fs.writeFile(path, "version3"); await wait({ @@ -59,6 +60,38 @@ describe("basic watching of file on disk happens automatically", () => { expect(s2.to_str()).toEqual("version3"); expect(c + c2).toBe(1); }); + + it("file watching must still work if either client is closed", async () => { + s.close(); + await fs.writeFile(path, "version4"); + await wait({ + until: () => { + return s2.to_str() == "version4"; + }, + }); + expect(s2.to_str()).toEqual("version4"); + }); + + let client3, s3; + it("add a third client and close client2 and have file watching still work", async () => { + client3 = connect(); + s3 = client3.sync.string({ + project_id, + path, + service: server.service, + }); + await once(s3, "ready"); + s2.close(); + + await fs.writeFile(path, "version5"); + + await wait({ + until: () => { + return s3.to_str() == "version5"; + }, + }); + expect(s3.to_str()).toEqual("version5"); + }); }); /* diff --git a/src/packages/conat/files/watch.ts b/src/packages/conat/files/watch.ts index 942d3116b5..03a05c57ae 100644 --- a/src/packages/conat/files/watch.ts +++ b/src/packages/conat/files/watch.ts @@ -43,15 +43,37 @@ export function watchServer({ const server: ConatSocketServer = client.socket.listen(subject); logger.debug("server: listening on ", { subject }); - //const unique; + const unique: { [path: string]: ServerSocket[] } = {}; async function handleUnique({ mesg, socket, path, options }) { - const w = await watch(path, options); + let w: any = undefined; + socket.once("closed", () => { - w.close(); + // when this socket closes, remove it from recipient list + unique[path] = unique[path]?.filter((x) => x.id != socket.id); + if (unique[path] != null && unique[path].length == 0) { + // nobody listening + w?.close(); + w = undefined; + delete unique[path]; + } }); - await mesg.respond(); - for await (const event of w) { - socket.write(event); + + if (unique[path] == null) { + // set it up + unique[path] = [socket]; + w = await watch(path, options); + await mesg.respond(); + for await (const event of w) { + for (const s of unique[path]) { + if (s.state == "ready") { + s.write(event); + break; + } + } + } + } else { + unique[path].push(socket); + await mesg.respond(); } } diff --git a/src/packages/sync/editor/generic/sync-doc.ts b/src/packages/sync/editor/generic/sync-doc.ts index 79c65b3739..3d89526e1d 100644 --- a/src/packages/sync/editor/generic/sync-doc.ts +++ b/src/packages/sync/editor/generic/sync-doc.ts @@ -61,6 +61,8 @@ const CURSOR_THROTTLE_NATS_MS = 150; // Ignore file changes for this long after save to disk. const RECENT_SAVE_TO_DISK_MS = 2000; +const WATCH_DEBOUNCE = 250; + const PARALLEL_INIT = true; import { @@ -3019,7 +3021,6 @@ export class SyncDoc extends EventEmitter { size = contents.length; this.from_str(contents); } catch (err) { - console.log(err); if (err.code == "ENOENT") { dbg("file no longer exists -- setting to blank"); size = 0; @@ -3237,7 +3238,6 @@ export class SyncDoc extends EventEmitter { throw Error("bug"); } const value = this.to_str(); - console.log("fs.writeFile", this.path); await this.fs.writeFile(this.path, value); }; @@ -3766,10 +3766,11 @@ export class SyncDoc extends EventEmitter { private fsLoadFromDiskDebounced = asyncDebounce( async () => { try { + this.emit("handle-file-change"); await this.fsLoadFromDisk(); } catch {} }, - 50, + WATCH_DEBOUNCE, { leading: false, trailing: true, @@ -3786,7 +3787,7 @@ export class SyncDoc extends EventEmitter { this.fsFileWatcher = await this.fs.watch(this.path, { unique: true }); (async () => { for await (const { eventType } of this.fsFileWatcher) { - // console.log("got change", eventType); + //console.log("got change", eventType); if (eventType == "change" || eventType == "rename") { this.fsLoadFromDiskDebounced(); } From c4c96322ade03914ff75b62ace463e50a643ea48 Mon Sep 17 00:00:00 2001 From: William Stein Date: Mon, 21 Jul 2025 01:07:59 +0000 Subject: [PATCH 047/270] syncdoc: working but hacky approach to saving to disk without causing a load --- .../conat/test/sync-doc/watch-file.test.ts | 28 ++++++++++++++++++- src/packages/conat/files/watch.ts | 28 ++++++++++++++++++- src/packages/sync/editor/generic/sync-doc.ts | 4 +++ 3 files changed, 58 insertions(+), 2 deletions(-) diff --git a/src/packages/backend/conat/test/sync-doc/watch-file.test.ts b/src/packages/backend/conat/test/sync-doc/watch-file.test.ts index a1ad595960..42da5aeb10 100644 --- a/src/packages/backend/conat/test/sync-doc/watch-file.test.ts +++ b/src/packages/backend/conat/test/sync-doc/watch-file.test.ts @@ -1,4 +1,13 @@ -import { before, after, uuid, connect, server, once, wait } from "./setup"; +import { + before, + after, + uuid, + connect, + server, + once, + wait, + delay, +} from "./setup"; beforeAll(before); afterAll(after); @@ -36,6 +45,23 @@ describe("basic watching of file on disk happens automatically", () => { }); }); + it("change file on disk should not trigger a load from disk", async () => { + const orig = s.fsLoadFromDiskDebounced; + let c = 0; + s.fsLoadFromDiskDebounced = () => { + c += 1; + }; + s.from_str("a different value"); + await s.save_to_disk(); + expect(c).toBe(0); + await delay(100); + expect(c).toBe(0); + s.fsLoadFromDiskDebounced = orig; + // disable the ignore that happens as part of save_to_disk, + // or the tests below won't work + await s.fsFileWatcher?.ignore(0); + }); + let client2, s2; it("file watching also works if there are multiple clients, with only one handling the change", async () => { client2 = connect(); diff --git a/src/packages/conat/files/watch.ts b/src/packages/conat/files/watch.ts index 03a05c57ae..6032f2b023 100644 --- a/src/packages/conat/files/watch.ts +++ b/src/packages/conat/files/watch.ts @@ -64,6 +64,17 @@ export function watchServer({ w = await watch(path, options); await mesg.respond(); for await (const event of w) { + const now = Date.now(); + let doIgnore = false; + for (const s of unique[path]) { + if (s.ignoreUntil != null && s.ignoreUntil > now) { + doIgnore = true; + break; + } + } + if (doIgnore) { + continue; + } for (const s of unique[path]) { if (s.state == "ready") { s.write(event); @@ -84,6 +95,9 @@ export function watchServer({ }); await mesg.respond(); for await (const event of w) { + if ((socket.ignoreUntil ?? 0) >= Date.now()) { + continue; + } socket.write(event); } } @@ -95,12 +109,18 @@ export function watchServer({ }); let initialized = false; socket.on("request", async (mesg) => { + const data = mesg.data; + if (data.ignore != null) { + socket.ignoreUntil = Date.now() + data.ignore; + await mesg.respond(null, { noThrow: true }); + return; + } try { if (initialized) { throw Error("already initialized"); } initialized = true; - const { path, options } = mesg.data; + const { path, options } = data; logger.debug("got request", { path, options }); if (options?.unique) { await handleUnique({ mesg, socket, path, options }); @@ -141,5 +161,11 @@ export async function watchClient({ }); // tell it what to watch await socket.request({ path, options }); + + // ignore events for ignore ms. + iter.ignore = async (ignore: number) => { + await socket.request({ ignore }); + }; + return iter; } diff --git a/src/packages/sync/editor/generic/sync-doc.ts b/src/packages/sync/editor/generic/sync-doc.ts index 3d89526e1d..286f3d3507 100644 --- a/src/packages/sync/editor/generic/sync-doc.ts +++ b/src/packages/sync/editor/generic/sync-doc.ts @@ -3238,6 +3238,10 @@ export class SyncDoc extends EventEmitter { throw Error("bug"); } const value = this.to_str(); + // tell watcher not to fire any change events for a little time, + // so no clients waste resources loading in response to us saving + // to disk. + await this.fsFileWatcher?.ignore(2000); await this.fs.writeFile(this.path, value); }; From 6b52a5a4240b46d99f9a53da52c690734dc99292 Mon Sep 17 00:00:00 2001 From: William Stein Date: Mon, 21 Jul 2025 01:20:48 +0000 Subject: [PATCH 048/270] file watch ignore when saving -- more ts friendly implementation --- src/packages/conat/files/fs.ts | 9 ++++++-- src/packages/conat/files/watch.ts | 38 ++++++++++++++++++++++--------- 2 files changed, 34 insertions(+), 13 deletions(-) diff --git a/src/packages/conat/files/fs.ts b/src/packages/conat/files/fs.ts index a06baa02bb..80b96ecccd 100644 --- a/src/packages/conat/files/fs.ts +++ b/src/packages/conat/files/fs.ts @@ -1,6 +1,10 @@ import { type Client } from "@cocalc/conat/core/client"; import { conat } from "@cocalc/conat/client"; -import { watchServer, watchClient } from "@cocalc/conat/files/watch"; +import { + watchServer, + watchClient, + type WatchIterator, +} from "@cocalc/conat/files/watch"; export const DEFAULT_FILE_SERVICE = "fs"; export interface Filesystem { @@ -30,7 +34,7 @@ export interface Filesystem { ) => Promise; writeFile: (path: string, data: string | Buffer) => Promise; // todo: typing - watch: (path: string, options?) => Promise; + watch: (path: string, options?) => Promise; } interface IStats { @@ -183,6 +187,7 @@ export async function fsServer({ service, fs, client }: Options) { async writeFile(path: string, data: string | Buffer) { await (await fs(this.subject)).writeFile(path, data); }, + // @ts-ignore async watch() { const subject = this.subject!; if (watches[subject] != null) { diff --git a/src/packages/conat/files/watch.ts b/src/packages/conat/files/watch.ts index 6032f2b023..3372da2279 100644 --- a/src/packages/conat/files/watch.ts +++ b/src/packages/conat/files/watch.ts @@ -44,7 +44,8 @@ export function watchServer({ logger.debug("server: listening on ", { subject }); const unique: { [path: string]: ServerSocket[] } = {}; - async function handleUnique({ mesg, socket, path, options }) { + const ignores: { [path: string]: { ignoreUntil: number }[] } = {}; + async function handleUnique({ mesg, socket, path, options, ignore }) { let w: any = undefined; socket.once("closed", () => { @@ -55,19 +56,21 @@ export function watchServer({ w?.close(); w = undefined; delete unique[path]; + delete ignores[path]; } }); if (unique[path] == null) { // set it up unique[path] = [socket]; + ignores[path] = [ignore]; w = await watch(path, options); await mesg.respond(); for await (const event of w) { const now = Date.now(); let doIgnore = false; - for (const s of unique[path]) { - if (s.ignoreUntil != null && s.ignoreUntil > now) { + for (const { ignoreUntil } of ignores[path]) { + if (ignoreUntil > now) { doIgnore = true; break; } @@ -84,18 +87,19 @@ export function watchServer({ } } else { unique[path].push(socket); + ignores[path].push(ignore); await mesg.respond(); } } - async function handleNonUnique({ mesg, socket, path, options }) { + async function handleNonUnique({ mesg, socket, path, options, ignore }) { const w = await watch(path, options); socket.once("closed", () => { w.close(); }); await mesg.respond(); for await (const event of w) { - if ((socket.ignoreUntil ?? 0) >= Date.now()) { + if (ignore.ignoreUntil >= Date.now()) { continue; } socket.write(event); @@ -108,10 +112,11 @@ export function watchServer({ subject: socket.subject, }); let initialized = false; + const ignore = { ignoreUntil: 0 }; socket.on("request", async (mesg) => { const data = mesg.data; if (data.ignore != null) { - socket.ignoreUntil = Date.now() + data.ignore; + ignore.ignoreUntil = data.ignore > 0 ? Date.now() + data.ignore : 0; await mesg.respond(null, { noThrow: true }); return; } @@ -123,9 +128,9 @@ export function watchServer({ const { path, options } = data; logger.debug("got request", { path, options }); if (options?.unique) { - await handleUnique({ mesg, socket, path, options }); + await handleUnique({ mesg, socket, path, options, ignore }); } else { - await handleNonUnique({ mesg, socket, path, options }); + await handleNonUnique({ mesg, socket, path, options, ignore }); } } catch (err) { mesg.respondSync(null, { @@ -138,6 +143,15 @@ export function watchServer({ return server; } +export type WatchIterator = EventIterator & { + ignore?: (ignore: number) => Promise; +}; + +export interface ChangeEvent { + eventType: "change" | "rename"; + filename: string; +} + export async function watchClient({ client, subject, @@ -148,7 +162,7 @@ export async function watchClient({ subject: string; path: string; options?: WatchOptions; -}) { +}): Promise { const socket = await client.socket.connect(subject); const iter = new EventIterator(socket, "data", { map: (args) => args[0], @@ -162,10 +176,12 @@ export async function watchClient({ // tell it what to watch await socket.request({ path, options }); + const iter2 = iter as WatchIterator; + // ignore events for ignore ms. - iter.ignore = async (ignore: number) => { + iter2.ignore = async (ignore: number) => { await socket.request({ ignore }); }; - return iter; + return iter2; } From e39cd45380c1be3b14ee47d761b38369f7c5f58d Mon Sep 17 00:00:00 2001 From: William Stein Date: Mon, 21 Jul 2025 01:24:06 +0000 Subject: [PATCH 049/270] ... --- src/packages/sync/editor/generic/sync-doc.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/packages/sync/editor/generic/sync-doc.ts b/src/packages/sync/editor/generic/sync-doc.ts index 286f3d3507..99caf07f8e 100644 --- a/src/packages/sync/editor/generic/sync-doc.ts +++ b/src/packages/sync/editor/generic/sync-doc.ts @@ -3242,6 +3242,7 @@ export class SyncDoc extends EventEmitter { // so no clients waste resources loading in response to us saving // to disk. await this.fsFileWatcher?.ignore(2000); + if(this.isClosed()) return; await this.fs.writeFile(this.path, value); }; From 87fb723c0837e393b0d881a3d243909095103987 Mon Sep 17 00:00:00 2001 From: William Stein Date: Mon, 21 Jul 2025 04:06:35 +0000 Subject: [PATCH 050/270] sync-doc: implement extremely simple version of "has unsaved changes" for saved-to-disk - this isn't as "amazing" as the full on hash-on-disk comparison one, but it's vastly simpler and very fast, and should be as good for most realistic purposes. It's probably much closer to what people expect anyways and has the advantage of rarely being wrong. --- .../conat/test/sync-doc/watch-file.test.ts | 51 ++++++++++++++++++- src/packages/conat/files/watch.ts | 22 ++++---- src/packages/sync/editor/generic/sync-doc.ts | 27 ++++++++-- 3 files changed, 85 insertions(+), 15 deletions(-) diff --git a/src/packages/backend/conat/test/sync-doc/watch-file.test.ts b/src/packages/backend/conat/test/sync-doc/watch-file.test.ts index 42da5aeb10..2a1acb48ea 100644 --- a/src/packages/backend/conat/test/sync-doc/watch-file.test.ts +++ b/src/packages/backend/conat/test/sync-doc/watch-file.test.ts @@ -7,6 +7,7 @@ import { once, wait, delay, + waitUntilSynced, } from "./setup"; beforeAll(before); @@ -17,7 +18,7 @@ describe("basic watching of file on disk happens automatically", () => { const path = "a.txt"; let client, s, fs; - it("creates two clients with noAutosave enabled", async () => { + it("creates client", async () => { client = connect(); fs = client.fs({ project_id, service: server.service }); await fs.writeFile(path, "init"); @@ -120,6 +121,54 @@ describe("basic watching of file on disk happens automatically", () => { }); }); +describe.only("has unsaved changes", () => { + const project_id = uuid(); + let s1, s2, client1, client2; + + it("creates two clients", async () => { + client1 = connect(); + client2 = connect(); + s1 = client1.sync.string({ + project_id, + path: "a.txt", + service: server.service, + }); + await once(s1, "ready"); + // definitely has unsaved changes, since it doesn't even exist + expect(s1.has_unsaved_changes()).toBe(true); + + s2 = client2.sync.string({ + project_id, + path: "a.txt", + service: server.service, + }); + await once(s2, "ready"); + expect(s1.to_str()).toBe(""); + expect(s2.to_str()).toBe(""); + expect(s1 === s2).toBe(false); + expect(s2.has_unsaved_changes()).toBe(true); + }); + + it("save empty file to disk -- now no unsaved changes", async () => { + await s1.save_to_disk(); + expect(s1.has_unsaved_changes()).toBe(false); + // but s2 doesn't know anything + expect(s2.has_unsaved_changes()).toBe(true); + }); + + it("make a change via s2 and save", async () => { + s2.from_str("i am s2"); + await s2.save_to_disk(); + expect(s2.has_unsaved_changes()).toBe(false); + }); + + it("as soon as s1 learns that there was a change to the file on disk, it doesn't know", async () => { + await waitUntilSynced([s1, s2]); + expect(s1.has_unsaved_changes()).toBe(true); + expect(s1.to_str()).toEqual("i am s2"); + }); +}); + /* watching of file with multiple clients diff --git a/src/packages/conat/files/watch.ts b/src/packages/conat/files/watch.ts index 3372da2279..531dc82d9f 100644 --- a/src/packages/conat/files/watch.ts +++ b/src/packages/conat/files/watch.ts @@ -25,9 +25,9 @@ export interface WatchOptions { maxQueue?: number; overflow?: "ignore" | "throw"; - // if more than one client is actively watching the same path and has unique set, only one - // will receive updates. Also, if there are multiple clients with unique set, the options - // of all but the first are ignored. + // if more than one client is actively watching the same path and has unique set, all but one may receive + // the extra field ignore:true in the update. Also, if there are multiple clients with unique set, the + // other options of all but the first are ignored. unique?: boolean; } @@ -68,20 +68,22 @@ export function watchServer({ await mesg.respond(); for await (const event of w) { const now = Date.now(); - let doIgnore = false; + let ignore = false; for (const { ignoreUntil } of ignores[path]) { if (ignoreUntil > now) { - doIgnore = true; + // every client is told to ignore this change, i.e., not load based on it happening + ignore = true; break; } } - if (doIgnore) { - continue; - } for (const s of unique[path]) { if (s.state == "ready") { - s.write(event); - break; + if (ignore) { + s.write({ ...event, ignore: true }); + } else { + s.write(event); + ignore = true; + } } } } diff --git a/src/packages/sync/editor/generic/sync-doc.ts b/src/packages/sync/editor/generic/sync-doc.ts index 99caf07f8e..20f9bc1da3 100644 --- a/src/packages/sync/editor/generic/sync-doc.ts +++ b/src/packages/sync/editor/generic/sync-doc.ts @@ -1784,6 +1784,7 @@ export class SyncDoc extends EventEmitter { try { stats = await this.fs.stat(this.path); } catch (err) { + this.lastDiskValue = undefined; // nonexistent or don't know if (err.code == "ENOENT") { // path does not exist -- nothing further to do return false; @@ -3017,6 +3018,7 @@ export class SyncDoc extends EventEmitter { let contents; try { contents = await this.fs.readFile(this.path, "utf8"); + this.lastDiskValue = contents; dbg("file exists"); size = contents.length; this.from_str(contents); @@ -3139,6 +3141,9 @@ export class SyncDoc extends EventEmitter { if (this.state !== "ready") { return; } + if (this.fs != null) { + return this.fsHasUnsavedChanges(); + } const dbg = this.dbg("has_unsaved_changes"); try { return this.hash_of_saved_version() !== this.hash_of_live_version(); @@ -3227,6 +3232,11 @@ export class SyncDoc extends EventEmitter { return true; }; + private lastDiskValue: string | undefined = undefined; + fsHasUnsavedChanges = (): boolean => { + return this.lastDiskValue != this.to_str(); + }; + fsSaveToDisk = async () => { const dbg = this.dbg("fsSaveToDisk"); if (this.client.is_deleted(this.path, this.project_id)) { @@ -3242,8 +3252,9 @@ export class SyncDoc extends EventEmitter { // so no clients waste resources loading in response to us saving // to disk. await this.fsFileWatcher?.ignore(2000); - if(this.isClosed()) return; + if (this.isClosed()) return; await this.fs.writeFile(this.path, value); + this.lastDiskValue = value; }; /* Initiates a save of file to disk, then waits for the @@ -3259,9 +3270,14 @@ export class SyncDoc extends EventEmitter { // properly. return; } + if (this.fs != null) { - return await this.fsSaveToDisk(); + this.commit(); + await this.fsSaveToDisk(); + this.update_has_unsaved_changes(); + return; } + const dbg = this.dbg("save_to_disk"); if (this.client.is_deleted(this.path, this.project_id)) { dbg("not saving to disk because deleted"); @@ -3791,12 +3807,15 @@ export class SyncDoc extends EventEmitter { // use this.fs interface to watch path for changes. this.fsFileWatcher = await this.fs.watch(this.path, { unique: true }); (async () => { - for await (const { eventType } of this.fsFileWatcher) { + for await (const { eventType, ignore } of this.fsFileWatcher) { + // we don't know what's on disk anymore, + this.lastDiskValue = undefined; //console.log("got change", eventType); - if (eventType == "change" || eventType == "rename") { + if (!ignore) { this.fsLoadFromDiskDebounced(); } if (eventType == "rename") { + // always have to recreate in case of a rename this.fsFileWatcher.close(); // start a new watcher since file descriptor changed this.fsInitFileWatcher(); From 611ee19dd5bb3147a08ef051fae20b6c3f440b7f Mon Sep 17 00:00:00 2001 From: William Stein Date: Mon, 21 Jul 2025 23:09:50 +0000 Subject: [PATCH 051/270] rewrite code formatting to not use backend state or service -- just a simple stateless api call - this will make it easier/transparent to just have a backend code formatter, e.g., maybe as part of conat-api, which is important for making editing fast/easier without having to start a project --- .../conat/test/sync-doc/watch-file.test.ts | 11 ---- src/packages/conat/core/client.ts | 1 + src/packages/conat/project/api/editor.ts | 2 +- src/packages/frontend/client/project.ts | 7 ++- .../frame-editors/code-editor/actions.ts | 54 ++++++++++--------- src/packages/project/conat/open-files.ts | 9 ++++ .../project/formatters/format.test.ts | 36 +++++++++++++ src/packages/project/formatters/index.ts | 31 +++++------ .../project/formatters/prettier-lib.ts | 15 ------ src/packages/project/package.json | 2 +- src/packages/sync/editor/db/sync.ts | 5 +- src/packages/sync/editor/generic/util.ts | 1 + 12 files changed, 100 insertions(+), 74 deletions(-) create mode 100644 src/packages/project/formatters/format.test.ts delete mode 100644 src/packages/project/formatters/prettier-lib.ts diff --git a/src/packages/backend/conat/test/sync-doc/watch-file.test.ts b/src/packages/backend/conat/test/sync-doc/watch-file.test.ts index 2a1acb48ea..e1f3cb830f 100644 --- a/src/packages/backend/conat/test/sync-doc/watch-file.test.ts +++ b/src/packages/backend/conat/test/sync-doc/watch-file.test.ts @@ -168,14 +168,3 @@ describe.only("has unsaved changes", () => { expect(s1.to_str()).toEqual("i am s2"); }); }); - -/* -watching of file with multiple clients - --- only one does the actual file load - --- when one writes file to disk, another doesn't try to load it - -(various ways to do that: sticky fs server would mean only one is -writing backend can ignore the resulting change event) -*/ diff --git a/src/packages/conat/core/client.ts b/src/packages/conat/core/client.ts index bfa92256f6..58bc516679 100644 --- a/src/packages/conat/core/client.ts +++ b/src/packages/conat/core/client.ts @@ -273,6 +273,7 @@ export const MAX_INTEREST_TIMEOUT = 90_000; const DEFAULT_WAIT_FOR_INTEREST_TIMEOUT = 30_000; +// WARNING: do NOT change MSGPACK_ENCODER_OPTIONS unless you know what you're doing! const MSGPACK_ENCODER_OPTIONS = { // ignoreUndefined is critical so database queries work properly, and // also we have a lot of api calls with tons of wasted undefined values. diff --git a/src/packages/conat/project/api/editor.ts b/src/packages/conat/project/api/editor.ts index cd70dc973d..37e9d27b5a 100644 --- a/src/packages/conat/project/api/editor.ts +++ b/src/packages/conat/project/api/editor.ts @@ -46,7 +46,7 @@ export interface Editor { jupyterKernels: (opts?: { noCache?: boolean }) => Promise; - // returns a patch to transform str into formatted form. + // returns formatted version of str. formatterString: (opts: { str: string; options: FormatterOptions; diff --git a/src/packages/frontend/client/project.ts b/src/packages/frontend/client/project.ts index 79ed25713f..6d621d4d84 100644 --- a/src/packages/frontend/client/project.ts +++ b/src/packages/frontend/client/project.ts @@ -50,8 +50,11 @@ export class ProjectClient { this.client = client; } - private conatApi = (project_id: string) => { - return this.client.conat_client.projectApi({ project_id }); + conatApi = (project_id: string, compute_server_id = 0) => { + return this.client.conat_client.projectApi({ + project_id, + compute_server_id, + }); }; // This can write small text files in one message. diff --git a/src/packages/frontend/frame-editors/code-editor/actions.ts b/src/packages/frontend/frame-editors/code-editor/actions.ts index 17fc8b7c8a..ded1541198 100644 --- a/src/packages/frontend/frame-editors/code-editor/actions.ts +++ b/src/packages/frontend/frame-editors/code-editor/actions.ts @@ -52,11 +52,11 @@ import { } from "@cocalc/frontend/misc/local-storage"; import { AvailableFeatures } from "@cocalc/frontend/project_configuration"; import { SyncDB } from "@cocalc/sync/editor/db"; -import { apply_patch } from "@cocalc/sync/editor/generic/util"; +import { apply_patch, make_patch } from "@cocalc/sync/editor/generic/util"; import type { SyncString } from "@cocalc/sync/editor/string/sync"; import { once } from "@cocalc/util/async-utils"; import { - Config as FormatterConfig, + Options as FormatterOptions, Exts as FormatterExts, Syntax as FormatterSyntax, Tool as FormatterTool, @@ -89,7 +89,6 @@ import { SetMap, } from "../frame-tree/types"; import { - formatter, get_default_font_size, log_error, syncdb2, @@ -112,6 +111,7 @@ import { misspelled_words } from "./spell-check"; import { log_opened_time } from "@cocalc/frontend/project/open-file"; import { ensure_project_running } from "@cocalc/frontend/project/project-start-warning"; import { alert_message } from "@cocalc/frontend/alerts"; +import { webapp_client } from "@cocalc/frontend/webapp-client"; interface gutterMarkerParams { line: number; @@ -2259,10 +2259,6 @@ export class Actions< const cm = this._get_cm(id); if (!cm) return; - if (!(await this.ensure_latest_changes_are_saved())) { - return; - } - // Important: this function may be called even if there is no format support, // because it can be called via a keyboard shortcut. That's why we gracefully // handle this case -- see https://github.com/sagemathinc/cocalc/issues/4180 @@ -2270,36 +2266,44 @@ export class Actions< if (s == null) { return; } - // TODO: Using any here since TypeMap is just not working right... - if (!this.has_format_support(id, s.get("available_features"))) { - return; - } // Definitely have format support cm.focus(); const ext = filename_extension(this.path).toLowerCase() as FormatterExts; const syntax: FormatterSyntax = ext2syntax[ext]; - const config: FormatterConfig = { - syntax, + const parser = syntax2tool[syntax]; + if (!parser || !this.has_format_support(id, s.get("available_features"))) { + return; + } + const options: FormatterOptions = { + parser, tabWidth: cm.getOption("tabSize") as number, useTabs: cm.getOption("indentWithTabs") as boolean, lastChanged: this._syncstring.last_changed(), }; - this.set_status("Running code formatter..."); + const api = webapp_client.project_client.conatApi(this.project_id); + const str = cm.getValue(); + try { - const patch = await formatter(this.project_id, this.path, config); - if (patch != null) { - // Apply the patch. - // NOTE: old backends that haven't restarted just return {status:'ok'} - // and directly make the change. Delete this comment in a month or so. - // See https://github.com/sagemathinc/cocalc/issues/4335 - this.set_syncstring_to_codemirror(); - const new_val = apply_patch(patch, this._syncstring.to_str())[0]; - this._syncstring.from_str(new_val); - this._syncstring.commit(); - this.set_codemirror_to_syncstring(); + this.set_status("Running code formatter..."); + let formatted = await api.editor.formatterString({ + str, + options, + path: this.path, + }); + if (formatted == str) { + // nothing to do + return; + } + const str2 = cm.getValue(); + if (str2 != str) { + // user made edits *during* formatting, so we "3-way merge" it in, rather + // than breaking what they did: + const patch = make_patch(str, formatted); + formatted = apply_patch(patch, str2)[0]; } + cm.setValueNoJump(formatted); this.setFormatError(""); } catch (err) { this.setFormatError(`${err}`, this._syncstring?.to_str()); diff --git a/src/packages/project/conat/open-files.ts b/src/packages/project/conat/open-files.ts index 10c96d1ede..e5114c6993 100644 --- a/src/packages/project/conat/open-files.ts +++ b/src/packages/project/conat/open-files.ts @@ -221,6 +221,12 @@ function computeServerId(path: string): number { return computeServers?.get(path) ?? 0; } +function hasBackendState(path) { + return ( + path.endsWith("." + JUPYTER_SYNCDB_EXTENSIONS) || path.endsWith(".sagews") + ); +} + async function handleChange({ path, time, @@ -229,6 +235,9 @@ async function handleChange({ doctype, id, }: OpenFileEntry & { id?: number }) { + if (!hasBackendState(path)) { + return; + } try { if (id == null) { id = computeServerId(path); diff --git a/src/packages/project/formatters/format.test.ts b/src/packages/project/formatters/format.test.ts new file mode 100644 index 0000000000..e538da02af --- /dev/null +++ b/src/packages/project/formatters/format.test.ts @@ -0,0 +1,36 @@ +import { run_formatter_string as formatString } from "./index"; + +describe("format some strings", () => { + it("formats markdown with math", async () => { + const s = await formatString({ + str: "# foo\n\n- $\\int x^2$\n- blah", + options: { parser: "markdown" }, + }); + expect(s).toEqual("# foo\n\n- $\\int x^2$\n- blah\n"); + }); + + it("formats some python", async () => { + const s = await formatString({ + str: "def f( n = 0):\n print( n )", + options: { parser: "python" }, + }); + expect(s).toEqual("def f(n=0):\n print(n)\n"); + }); + + it("format some typescript", async () => { + const s = await formatString({ + str: "function f( n = 0) { console.log( n ) }", + options: { parser: "typescript" }, + }); + expect(s).toEqual("function f(n = 0) {\n console.log(n);\n}\n"); + }); + + it("formatting invalid typescript throws an error", async () => { + await expect(async () => { + await formatString({ + str: "function f( n = 0) { console.log( n ) ", + options: { parser: "typescript" }, + }); + }).rejects.toThrow("'}' expected"); + }); +}); diff --git a/src/packages/project/formatters/index.ts b/src/packages/project/formatters/index.ts index e9bff1e08c..ca2c26434e 100644 --- a/src/packages/project/formatters/index.ts +++ b/src/packages/project/formatters/index.ts @@ -13,8 +13,6 @@ Also, by doing this on the backend we don't add 5MB (!) to the webpack frontend something that is not supported on the frontend anyway. */ -declare let require: any; - import { make_patch } from "@cocalc/sync/editor/generic/util"; import { math_escape, math_unescape } from "@cocalc/util/markdown-utils"; import { filename_extension } from "@cocalc/util/misc"; @@ -30,7 +28,6 @@ import { xml_format } from "./xml-format"; // mathjax-utils is from upstream project Jupyter import { once } from "@cocalc/util/async-utils"; import { remove_math, replace_math } from "@cocalc/util/mathjax-utils"; -import { get_prettier } from "./prettier-lib"; import type { Syntax as FormatterSyntax, Config, @@ -91,20 +88,14 @@ export async function run_formatter({ } } const doc = syncstring.get_doc(); - let formatted, math, input0; + let formatted, input0; let input = (input0 = doc.to_str()); - if (options.parser === "markdown") { - [input, math] = remove_math(math_escape(input)); - } try { formatted = await run_formatter_string({ path, str: input, options }); } catch (err) { logger.debug(`run_formatter error: ${err.message}`); return { status: "error", phase: "format", error: err.message }; } - if (options.parser === "markdown") { - formatted = math_unescape(replace_math(formatted, math)); - } // NOTE: the code used to make the change here on the backend. // See https://github.com/sagemathinc/cocalc/issues/4335 for why // that leads to confusion. @@ -121,9 +112,13 @@ export async function run_formatter_string({ options: Options; path?: string; // only used for CLANG }): Promise { - let formatted; - const input = str; + let formatted, math; + let input = str; logger.debug(`run_formatter options.parser: "${options.parser}"`); + if (options.parser === "markdown") { + [input, math] = remove_math(math_escape(input)); + } + switch (options.parser) { case "latex": case "latexindent": @@ -163,12 +158,12 @@ export async function run_formatter_string({ formatted = await rust_format(input, options, logger); break; default: - const prettier = get_prettier(); - if (prettier != null) { - formatted = prettier.format(input, options); - } else { - throw Error("Could not load 'prettier'"); - } + const prettier = await import("prettier"); + formatted = await prettier.format(input, options as any); + } + + if (options.parser === "markdown") { + formatted = math_unescape(replace_math(formatted, math)); } return formatted; } diff --git a/src/packages/project/formatters/prettier-lib.ts b/src/packages/project/formatters/prettier-lib.ts deleted file mode 100644 index 413569cb42..0000000000 --- a/src/packages/project/formatters/prettier-lib.ts +++ /dev/null @@ -1,15 +0,0 @@ -/* - * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. - * License: MS-RSL – see LICENSE.md for details - */ - -// The whole purpose of this is to only load prettier if we really need it – this saves a few MB of project memory usage - -let instance: { format: Function } | null = null; - -export function get_prettier() { - if (instance == null) { - instance = require("prettier"); - } - return instance; -} diff --git a/src/packages/project/package.json b/src/packages/project/package.json index 19f3fda579..320802b629 100644 --- a/src/packages/project/package.json +++ b/src/packages/project/package.json @@ -68,7 +68,7 @@ "start": "NODE_OPTIONS='--trace-warnings --unhandled-rejections=strict --enable-source-maps' pnpm cocalc-project", "build": "../node_modules/.bin/tsc --build", "tsc": "../node_modules/.bin/tsc --watch --pretty --preserveWatchOutput", - "test": "COCALC_PROJECT_ID=812abe34-a382-4bd1-9071-29b6f4334f03 COCALC_USERNAME=user pnpm exec jest", + "test": "COCALC_PROJECT_ID=812abe34-a382-4bd1-9071-29b6f4334f03 COCALC_USERNAME=user NODE_OPTIONS='--experimental-vm-modules' pnpm exec jest", "depcheck": "pnpx depcheck", "prepublishOnly": "pnpm test", "clean": "rm -rf dist" diff --git a/src/packages/sync/editor/db/sync.ts b/src/packages/sync/editor/db/sync.ts index b8db32366c..2ba806511c 100644 --- a/src/packages/sync/editor/db/sync.ts +++ b/src/packages/sync/editor/db/sync.ts @@ -10,6 +10,9 @@ import { Document, DocType } from "../generic/types"; export interface SyncDBOpts0 extends SyncOpts0 { primary_keys: string[]; string_cols?: string[]; + // format = what format to store the underlying file using: json or msgpack + // The default is json unless otherwise specified. + format?: "json" | "msgpack"; } export interface SyncDBOpts extends SyncDBOpts0 { @@ -37,7 +40,7 @@ export class SyncDB extends SyncDoc { super(opts1 as SyncOpts); } - get_one(arg?) : any { + get_one(arg?): any { // I know it is really of type DBDocument. return (this.get_doc() as DBDocument).get_one(arg); } diff --git a/src/packages/sync/editor/generic/util.ts b/src/packages/sync/editor/generic/util.ts index fd666b09a1..2c66ee9729 100644 --- a/src/packages/sync/editor/generic/util.ts +++ b/src/packages/sync/editor/generic/util.ts @@ -68,6 +68,7 @@ export function make_patch(s0: string, s1: string): CompressedPatch { } // apply a compressed patch to a string. +// Returns the result *and* whether or not the patch applied cleanly. export function apply_patch( patch: CompressedPatch, s: string, From dd945012bc53ec0d739bcdb82eb7286615278094 Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 22 Jul 2025 00:30:11 +0000 Subject: [PATCH 052/270] delete all the syncstring-based formatter code (just format strings); implement simple approach to file deletion detection --- src/packages/conat/service/formatter.ts | 46 ------------------------- src/packages/project/conat/formatter.ts | 25 -------------- 2 files changed, 71 deletions(-) delete mode 100644 src/packages/conat/service/formatter.ts delete mode 100644 src/packages/project/conat/formatter.ts diff --git a/src/packages/conat/service/formatter.ts b/src/packages/conat/service/formatter.ts deleted file mode 100644 index 4fa5b5a490..0000000000 --- a/src/packages/conat/service/formatter.ts +++ /dev/null @@ -1,46 +0,0 @@ -/* -Formatting services in a project. -*/ - -import { createServiceClient, createServiceHandler } from "./typed"; - -import type { - Options as FormatterOptions, - FormatResult, -} from "@cocalc/util/code-formatter"; - -// TODO: we may change it to NOT take compute server and have this listening from -// project and all compute servers... and have only the one with the file open -// actually reply. -interface FormatterApi { - formatter: (opts: { - path: string; - options: FormatterOptions; - }) => Promise; -} - -export function formatterClient({ compute_server_id = 0, project_id }) { - return createServiceClient({ - project_id, - compute_server_id, - service: "formatter", - }); -} - -export async function createFormatterService({ - compute_server_id = 0, - project_id, - impl, -}: { - project_id: string; - compute_server_id?: number; - impl: FormatterApi; -}) { - return await createServiceHandler({ - project_id, - compute_server_id, - service: "formatter", - description: "Code formatter API", - impl, - }); -} diff --git a/src/packages/project/conat/formatter.ts b/src/packages/project/conat/formatter.ts deleted file mode 100644 index aa593e89bc..0000000000 --- a/src/packages/project/conat/formatter.ts +++ /dev/null @@ -1,25 +0,0 @@ -/* -File formatting service. -*/ - -import { run_formatter, type Options } from "../formatters"; -import { createFormatterService as create } from "@cocalc/conat/service/formatter"; -import { compute_server_id, project_id } from "@cocalc/project/data"; - -interface Message { - path: string; - options: Options; -} - -export async function createFormatterService({ openSyncDocs }) { - const impl = { - formatter: async (opts: Message) => { - const syncstring = openSyncDocs[opts.path]; - if (syncstring == null) { - throw Error(`"${opts.path}" is not opened`); - } - return await run_formatter({ ...opts, syncstring }); - }, - }; - return await create({ compute_server_id, project_id, impl }); -} From fee5a14b58b0e37e4e96088cf4df479b5133f85f Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 22 Jul 2025 00:43:20 +0000 Subject: [PATCH 053/270] surpress an antd react19 warning --- src/packages/static/src/rspack.config.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/packages/static/src/rspack.config.ts b/src/packages/static/src/rspack.config.ts index c79f2282ba..dd973ef7d5 100644 --- a/src/packages/static/src/rspack.config.ts +++ b/src/packages/static/src/rspack.config.ts @@ -168,7 +168,10 @@ export default function getConfig({ middleware }: Options = {}): Configuration { const config: Configuration = { // this makes things 10x slower: //cache: RSPACK_DEV_SERVER || PRODMODE ? false : true, - ignoreWarnings: [/Failed to parse source map/], + ignoreWarnings: [ + /Failed to parse source map/, + /formItemNode = ReactDOM.findDOMNode/, + ], devtool: PRODMODE ? undefined : "eval-cheap-module-source-map", mode: PRODMODE ? ("production" as "production") From 581414681eb2bb126e2d1d3a347ff667ec530de3 Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 22 Jul 2025 00:43:38 +0000 Subject: [PATCH 054/270] remove old formatter; implement file delete --- src/packages/conat/project/api/editor.ts | 4 +- .../frame-editors/code-editor/actions.ts | 28 ++++--- .../frontend/frame-editors/generic/client.ts | 26 ------ .../frontend/project/websocket/api.ts | 24 +----- src/packages/project/browser-websocket/api.ts | 8 +- src/packages/project/conat/api/editor.ts | 2 +- src/packages/project/conat/open-files.ts | 12 +-- .../project/formatters/format.test.ts | 2 +- src/packages/project/formatters/index.ts | 79 +------------------ src/packages/sync/editor/generic/sync-doc.ts | 51 ++++++++++++ 10 files changed, 80 insertions(+), 156 deletions(-) diff --git a/src/packages/conat/project/api/editor.ts b/src/packages/conat/project/api/editor.ts index 37e9d27b5a..9db2b45471 100644 --- a/src/packages/conat/project/api/editor.ts +++ b/src/packages/conat/project/api/editor.ts @@ -10,7 +10,7 @@ export const editor = { jupyterRunNotebook: true, jupyterKernelLogo: true, jupyterKernels: true, - formatterString: true, + formatString: true, printSageWS: true, createTerminalService: true, }; @@ -47,7 +47,7 @@ export interface Editor { jupyterKernels: (opts?: { noCache?: boolean }) => Promise; // returns formatted version of str. - formatterString: (opts: { + formatString: (opts: { str: string; options: FormatterOptions; path?: string; // only used for CLANG diff --git a/src/packages/frontend/frame-editors/code-editor/actions.ts b/src/packages/frontend/frame-editors/code-editor/actions.ts index ded1541198..28f025e153 100644 --- a/src/packages/frontend/frame-editors/code-editor/actions.ts +++ b/src/packages/frontend/frame-editors/code-editor/actions.ts @@ -350,6 +350,12 @@ export class Actions< throw Error(`invalid doctype="${this.doctype}"`); } + this._syncstring.once("deleted", () => { + // the file was deleted + this._syncstring.close(); + this._get_project_actions().close_file(this.path); + }); + this._syncstring.once("ready", (err) => { if (this.doctype != "none") { // doctype = 'none' must be handled elsewhere, e.g., terminals. @@ -2287,23 +2293,21 @@ export class Actions< try { this.set_status("Running code formatter..."); - let formatted = await api.editor.formatterString({ + let formatted = await api.editor.formatString({ str, options, path: this.path, }); - if (formatted == str) { - // nothing to do - return; - } - const str2 = cm.getValue(); - if (str2 != str) { - // user made edits *during* formatting, so we "3-way merge" it in, rather - // than breaking what they did: - const patch = make_patch(str, formatted); - formatted = apply_patch(patch, str2)[0]; + if (formatted != str) { + const str2 = cm.getValue(); + if (str2 != str) { + // user made edits *during* formatting, so we "3-way merge" it in, rather + // than breaking what they did: + const patch = make_patch(str, formatted); + formatted = apply_patch(patch, str2)[0]; + } + cm.setValueNoJump(formatted); } - cm.setValueNoJump(formatted); this.setFormatError(""); } catch (err) { this.setFormatError(`${err}`, this._syncstring?.to_str()); diff --git a/src/packages/frontend/frame-editors/generic/client.ts b/src/packages/frontend/frame-editors/generic/client.ts index 67d143f53a..ef26b16dfb 100644 --- a/src/packages/frontend/frame-editors/generic/client.ts +++ b/src/packages/frontend/frame-editors/generic/client.ts @@ -10,9 +10,7 @@ Typescript async/await rewrite of @cocalc/util/client.coffee... import { Map } from "immutable"; import { redux } from "@cocalc/frontend/app-framework"; import { webapp_client } from "@cocalc/frontend/webapp-client"; -import { CompressedPatch } from "@cocalc/sync/editor/generic/types"; import { callback2 } from "@cocalc/util/async-utils"; -import { Config as FormatterConfig } from "@cocalc/util/code-formatter"; import { FakeSyncstring } from "./syncstring-fake"; import { type UserSearchResult as User } from "@cocalc/util/db-schema/accounts"; export { type User }; @@ -154,30 +152,6 @@ export async function write_text_file_to_project( await webapp_client.project_client.write_text_file(opts); } -export async function formatter( - project_id: string, - path: string, - config: FormatterConfig, -): Promise { - const api = await webapp_client.project_client.api(project_id); - const resp = await api.formatter(path, config); - - if (resp.status === "error") { - const loc = resp.error?.loc; - if (loc && loc.start) { - throw Error( - `Syntax error prevented formatting code (possibly on line ${loc.start.line} column ${loc.start.column}) -- fix and run again.`, - ); - } else if (resp.error) { - throw Error(resp.error); - } else { - throw Error("Syntax error prevented formatting code."); - } - } else { - return resp.patch; - } -} - export function log_error(error: string | object): void { webapp_client.tracking_client.log_error(error); } diff --git a/src/packages/frontend/project/websocket/api.ts b/src/packages/frontend/project/websocket/api.ts index 00ed94d1a6..31dced4922 100644 --- a/src/packages/frontend/project/websocket/api.ts +++ b/src/packages/frontend/project/websocket/api.ts @@ -19,7 +19,6 @@ import { import type { Config as FormatterConfig, Options as FormatterOptions, - FormatResult, } from "@cocalc/util/code-formatter"; import { syntax2tool } from "@cocalc/util/code-formatter"; import { DirectoryListingEntry } from "@cocalc/util/types"; @@ -35,7 +34,6 @@ import type { ExecuteCodeOutput, ExecuteCodeOptions, } from "@cocalc/util/types/execute-code"; -import { formatterClient } from "@cocalc/conat/service/formatter"; import { syncFsClientClient } from "@cocalc/conat/service/syncfs-client"; const log = (...args) => { @@ -259,33 +257,15 @@ export class API { } }; - // Returns { status: "ok", patch:... the patch} or - // { status: "error", phase: "format", error: err.message }. - // We return a patch rather than the entire file, since often - // the file is very large, but the formatting is tiny. This is purely - // a data compression technique. - formatter = async ( - path: string, - config: FormatterConfig, - compute_server_id?: number, - ): Promise => { - const options: FormatterOptions = this.check_formatter_available(config); - const client = formatterClient({ - project_id: this.project_id, - compute_server_id: compute_server_id ?? this.getComputeServerId(path), - }); - return await client.formatter({ path, options }); - }; - formatter_string = async ( str: string, config: FormatterConfig, timeout: number = 15000, compute_server_id?: number, - ): Promise => { + ): Promise => { const options: FormatterOptions = this.check_formatter_available(config); const api = this.getApi({ compute_server_id, timeout }); - return await api.editor.formatterString({ str, options }); + return await api.editor.formatString({ str, options }); }; exec = async (opts: ExecuteCodeOptions): Promise => { diff --git a/src/packages/project/browser-websocket/api.ts b/src/packages/project/browser-websocket/api.ts index 8ca6e9387b..edccfdb87f 100644 --- a/src/packages/project/browser-websocket/api.ts +++ b/src/packages/project/browser-websocket/api.ts @@ -16,7 +16,7 @@ import { getClient } from "@cocalc/project/client"; import { get_configuration } from "../configuration"; -import { run_formatter, run_formatter_string } from "../formatters"; +import { formatString } from "../formatters"; import { nbconvert as jupyter_nbconvert } from "../jupyter/convert"; import { jupyter_strip_notebook } from "@cocalc/jupyter/nbgrader/jupyter-parse"; import { jupyter_run_notebook } from "@cocalc/jupyter/nbgrader/jupyter-run"; @@ -131,12 +131,8 @@ export async function handleApiCall({ return await canonical_paths(data.paths); case "configuration": return await get_configuration(data.aspect, data.no_cache); - case "prettier": // deprecated - case "formatter": - return await run_formatter(data); - case "prettier_string": // deprecated case "formatter_string": - return await run_formatter_string(data); + return await formatString(data); case "exec": if (data.opts == null) { throw Error("opts must not be null"); diff --git a/src/packages/project/conat/api/editor.ts b/src/packages/project/conat/api/editor.ts index 32358e614d..c8fde5365a 100644 --- a/src/packages/project/conat/api/editor.ts +++ b/src/packages/project/conat/api/editor.ts @@ -1,7 +1,7 @@ export { jupyter_strip_notebook as jupyterStripNotebook } from "@cocalc/jupyter/nbgrader/jupyter-parse"; export { jupyter_run_notebook as jupyterRunNotebook } from "@cocalc/jupyter/nbgrader/jupyter-run"; export { nbconvert as jupyterNbconvert } from "../../jupyter/convert"; -export { run_formatter_string as formatterString } from "../../formatters"; +export { formatString } from "../../formatters"; export { logo as jupyterKernelLogo } from "@cocalc/jupyter/kernel/logo"; export { get_kernel_data as jupyterKernels } from "@cocalc/jupyter/kernel/kernel-data"; export { newFile } from "@cocalc/backend/misc/new-file"; diff --git a/src/packages/project/conat/open-files.ts b/src/packages/project/conat/open-files.ts index e5114c6993..11dab2fe09 100644 --- a/src/packages/project/conat/open-files.ts +++ b/src/packages/project/conat/open-files.ts @@ -17,7 +17,7 @@ DEBUG_CONSOLE=yes DEBUG=cocalc:debug:project:conat:* node x = await require("@cocalc/project/conat/open-files").init(); Object.keys(x) -[ 'openFiles', 'openDocs', 'formatter', 'terminate', 'computeServers', 'cc' ] +[ 'openFiles', 'openDocs', 'terminate', 'computeServers', 'cc' ] > x.openFiles.getAll(); @@ -67,7 +67,7 @@ doing this! Then: Welcome to Node.js v20.19.0. Type ".help" for more information. > x = await require("@cocalc/project/conat/open-files").init(); Object.keys(x) -[ 'openFiles', 'openDocs', 'formatter', 'terminate', 'computeServers' ] +[ 'openFiles', 'openDocs', 'terminate', 'computeServers' ] > @@ -89,7 +89,6 @@ import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; import { delay } from "awaiting"; import { initJupyterRedux, removeJupyterRedux } from "@cocalc/jupyter/kernel"; import { filename_extension, original_path } from "@cocalc/util/misc"; -import { createFormatterService } from "./formatter"; import { type ConatService } from "@cocalc/conat/service/service"; import { exists } from "@cocalc/backend/misc/async-utils-node"; import { map as awaitMap } from "awaiting"; @@ -124,7 +123,6 @@ const FILE_DELETION_GRACE_PERIOD = 2000; const FILE_DELETION_INITIAL_DELAY = 15000; let openFiles: OpenFiles | null = null; -let formatter: any = null; const openDocs: { [path: string]: SyncDoc | ConatService } = {}; let computeServers: ComputeServerManager | null = null; const openTimes: { [path: string]: number } = {}; @@ -185,13 +183,10 @@ export async function init() { } }); - formatter = await createFormatterService({ openSyncDocs: openDocs }); - // useful for development return { openFiles, openDocs, - formatter, terminate, computeServers, cc: connectToConat(), @@ -206,9 +201,6 @@ export function terminate() { openFiles?.close(); openFiles = null; - formatter?.close(); - formatter = null; - computeServers?.close(); computeServers = null; } diff --git a/src/packages/project/formatters/format.test.ts b/src/packages/project/formatters/format.test.ts index e538da02af..34ce38cca8 100644 --- a/src/packages/project/formatters/format.test.ts +++ b/src/packages/project/formatters/format.test.ts @@ -1,4 +1,4 @@ -import { run_formatter_string as formatString } from "./index"; +import { formatString } from "./index"; describe("format some strings", () => { it("formats markdown with math", async () => { diff --git a/src/packages/project/formatters/index.ts b/src/packages/project/formatters/index.ts index ca2c26434e..62fdcaa16a 100644 --- a/src/packages/project/formatters/index.ts +++ b/src/packages/project/formatters/index.ts @@ -4,16 +4,9 @@ */ /* -Use a formatter like prettier to reformat a syncstring. - -This very nicely use the in-memory node module to prettyify code, by simply modifying the syncstring -on the backend. This avoids having to send the whole file back and forth, worrying about multiple users -and their cursors, file state etc. -- it just merges in the prettification at a point in time. -Also, by doing this on the backend we don't add 5MB (!) to the webpack frontend bundle, to install -something that is not supported on the frontend anyway. +Use a formatter like prettier to format a string of code. */ -import { make_patch } from "@cocalc/sync/editor/generic/util"; import { math_escape, math_unescape } from "@cocalc/util/markdown-utils"; import { filename_extension } from "@cocalc/util/misc"; import { bib_format } from "./bib-format"; @@ -26,90 +19,24 @@ import { r_format } from "./r-format"; import { rust_format } from "./rust-format"; import { xml_format } from "./xml-format"; // mathjax-utils is from upstream project Jupyter -import { once } from "@cocalc/util/async-utils"; import { remove_math, replace_math } from "@cocalc/util/mathjax-utils"; import type { Syntax as FormatterSyntax, Config, Options, - FormatResult, } from "@cocalc/util/code-formatter"; export type { Config, Options, FormatterSyntax }; import { getLogger } from "@cocalc/backend/logger"; -import { getClient } from "@cocalc/project/client"; - -// don't wait too long, since the entire api call likely times out after 5s. -const MAX_WAIT_FOR_SYNC = 3000; const logger = getLogger("project:formatters"); -export async function run_formatter({ - path, - options, - syncstring, -}: { - path: string; - options: Options; - syncstring?; -}): Promise { - const client = getClient(); - // What we do is edit the syncstring with the given path to be "prettier" if possible... - if (syncstring == null) { - syncstring = client.syncdoc({ path }); - } - if (syncstring == null || syncstring.get_state() == "closed") { - return { - status: "error", - error: "document not fully opened", - phase: "format", - }; - } - if (syncstring.get_state() != "ready") { - await once(syncstring, "ready"); - } - if (options.lastChanged) { - // wait within reason until syncstring's last change is this new. - // (It's not a huge problem if this fails for some reason.) - const start = Date.now(); - const waitUntil = new Date(options.lastChanged); - while ( - Date.now() - start < MAX_WAIT_FOR_SYNC && - syncstring.last_changed() < waitUntil - ) { - try { - await once( - syncstring, - "change", - MAX_WAIT_FOR_SYNC - (Date.now() - start), - ); - } catch { - break; - } - } - } - const doc = syncstring.get_doc(); - let formatted, input0; - let input = (input0 = doc.to_str()); - try { - formatted = await run_formatter_string({ path, str: input, options }); - } catch (err) { - logger.debug(`run_formatter error: ${err.message}`); - return { status: "error", phase: "format", error: err.message }; - } - // NOTE: the code used to make the change here on the backend. - // See https://github.com/sagemathinc/cocalc/issues/4335 for why - // that leads to confusion. - const patch = make_patch(input0, formatted); - return { status: "ok", patch }; -} - -export async function run_formatter_string({ +export async function formatString({ options, str, path, }: { str: string; - options: Options; + options: Options; // e.g., {parser:'python'} path?: string; // only used for CLANG }): Promise { let formatted, math; diff --git a/src/packages/sync/editor/generic/sync-doc.ts b/src/packages/sync/editor/generic/sync-doc.ts index 20f9bc1da3..ac91aa7a3b 100644 --- a/src/packages/sync/editor/generic/sync-doc.ts +++ b/src/packages/sync/editor/generic/sync-doc.ts @@ -61,6 +61,10 @@ const CURSOR_THROTTLE_NATS_MS = 150; // Ignore file changes for this long after save to disk. const RECENT_SAVE_TO_DISK_MS = 2000; +// If file does not exist for this long, then we close. +const CLOSE_WHEN_DELETED_MS = 2000; +const CLOSE_CHECK_INTERVAL_MS = 500; + const WATCH_DEBOUNCE = 250; const PARALLEL_INIT = true; @@ -3756,6 +3760,9 @@ export class SyncDoc extends EventEmitter { }, 60000); private initInterestLoop = async () => { + if (this.fs != null) { + return; + } if (!this.client.is_browser()) { // only browser clients -- so actual humans return; @@ -3819,6 +3826,8 @@ export class SyncDoc extends EventEmitter { this.fsFileWatcher.close(); // start a new watcher since file descriptor changed this.fsInitFileWatcher(); + // also check if file was deleted, in which case we'll just close + this.fsCloseIfFileDeleted(); return; } } @@ -3830,6 +3839,48 @@ export class SyncDoc extends EventEmitter { this.fsFileWatcher?.close(); delete this.fsFileWatcher; }; + + // returns true if file definitely exists right now, + // false if it definitely does not, and throws exception otherwise, + // e.g., network error. + private fsFileExists = async (): Promise => { + if (this.fs == null) { + throw Error("bug -- fs must be defined"); + } + try { + await this.fs.stat(this.path); + return true; + } catch (err) { + if (err.code == "ENOENT") { + // file not there now. + return false; + } + throw err; + } + }; + + private fsCloseIfFileDeleted = async () => { + if (this.fs == null) { + throw Error("bug -- fs must be defined"); + } + const start = Date.now(); + while (Date.now() - start < CLOSE_WHEN_DELETED_MS) { + try { + if (await this.fsFileExists()) { + // file definitely exists right now. + return; + } + // file definitely does NOT exist right now. + } catch { + // network not working or project off -- no way to know. + return; + } + await delay(CLOSE_CHECK_INTERVAL_MS); + } + // file still doesn't exist -- consider it deleted -- browsers + // should close the tab and possibly notify user. + this.emit("deleted"); + }; } function isCompletePatchStream(dstream) { From 476e9c35bc4ed1ea54ee59b86e562ff84e43358f Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 22 Jul 2025 01:15:09 +0000 Subject: [PATCH 055/270] fix building latex on save --- src/packages/sync/editor/generic/sync-doc.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/packages/sync/editor/generic/sync-doc.ts b/src/packages/sync/editor/generic/sync-doc.ts index ac91aa7a3b..aafea609b9 100644 --- a/src/packages/sync/editor/generic/sync-doc.ts +++ b/src/packages/sync/editor/generic/sync-doc.ts @@ -3257,6 +3257,7 @@ export class SyncDoc extends EventEmitter { // to disk. await this.fsFileWatcher?.ignore(2000); if (this.isClosed()) return; + this.last_save_to_disk_time = new Date(); await this.fs.writeFile(this.path, value); this.lastDiskValue = value; }; From dffe85edc0e469ec64a6bd8dea084d48f0de9e8b Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 22 Jul 2025 01:25:44 +0000 Subject: [PATCH 056/270] sync: eliminate non-conate syncdb and syncstring usage in the frontend --- src/packages/frontend/chat/register.ts | 2 +- src/packages/frontend/course/sync.ts | 2 +- src/packages/frontend/frame-editors/generic/client.ts | 7 ------- src/packages/frontend/syncdoc.coffee | 4 ++-- 4 files changed, 4 insertions(+), 11 deletions(-) diff --git a/src/packages/frontend/chat/register.ts b/src/packages/frontend/chat/register.ts index 7ff96d84ca..c28991c167 100644 --- a/src/packages/frontend/chat/register.ts +++ b/src/packages/frontend/chat/register.ts @@ -26,7 +26,7 @@ export function initChat(project_id: string, path: string): ChatActions { redux.getProjectActions(project_id)?.setNotDeleted(path); } - const syncdb = webapp_client.sync_client.sync_db({ + const syncdb = webapp_client.conat_client.conat().sync.db({ project_id, path, primary_keys: ["date", "sender_id", "event"], diff --git a/src/packages/frontend/course/sync.ts b/src/packages/frontend/course/sync.ts index 69a470e1f8..6fbfe3df11 100644 --- a/src/packages/frontend/course/sync.ts +++ b/src/packages/frontend/course/sync.ts @@ -31,7 +31,7 @@ export function create_sync_db( const path = store.get("course_filename"); actions.setState({ loading: true }); - const syncdb = webapp_client.sync_client.sync_db({ + const syncdb = webapp_client.conat_client.conat().sync.db({ project_id, path, primary_keys: ["table", "handout_id", "student_id", "assignment_id"], diff --git a/src/packages/frontend/frame-editors/generic/client.ts b/src/packages/frontend/frame-editors/generic/client.ts index ef26b16dfb..33a8159f04 100644 --- a/src/packages/frontend/frame-editors/generic/client.ts +++ b/src/packages/frontend/frame-editors/generic/client.ts @@ -194,9 +194,6 @@ interface SyncstringOpts2 { export function syncstring2(opts: SyncstringOpts2): SyncString { return webapp_client.conat_client.conat().sync.string(opts); - // const opts1: any = opts; - // opts1.client = webapp_client; - // return webapp_client.sync_client.sync_string(opts1); } export interface SyncDBOpts { @@ -215,9 +212,6 @@ export interface SyncDBOpts { export function syncdb(opts: SyncDBOpts): any { return webapp_client.conat_client.conat().sync.db(opts); - - // const opts1: any = opts; - // return webapp_client.sync_db(opts1); } import type { SyncDB } from "@cocalc/sync/editor/db/sync"; @@ -229,7 +223,6 @@ export function syncdb2(opts: SyncDBOpts): SyncDB { const opts1: any = opts; opts1.client = webapp_client; return webapp_client.conat_client.conat().sync.db(opts1); - // return webapp_client.sync_client.sync_db(opts1); } interface QueryOpts { diff --git a/src/packages/frontend/syncdoc.coffee b/src/packages/frontend/syncdoc.coffee index ae8ab1995c..535a38c5c6 100644 --- a/src/packages/frontend/syncdoc.coffee +++ b/src/packages/frontend/syncdoc.coffee @@ -66,7 +66,7 @@ class SynchronizedString extends AbstractSynchronizedDoc @project_id = @opts.project_id @filename = @opts.filename @connect = @_connect - @_syncstring = webapp_client.sync_client.sync_string + @_syncstring = webapp_client.conat_client.conat().sync.string project_id : @project_id path : @filename cursors : opts.cursors @@ -209,7 +209,7 @@ class SynchronizedDocument2 extends SynchronizedDocument @filename = '.smc/root' + @filename id = require('@cocalc/util/schema').client_db.sha1(@project_id, @filename) - @_syncstring = webapp_client.sync_client.sync_string + @_syncstring = webapp_client.conat_client.conat().sync.string id : id project_id : @project_id path : @filename From 91b79d3469b7a03d33e276fa9e997d452b43ab35 Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 22 Jul 2025 14:26:48 +0000 Subject: [PATCH 057/270] sync-doc: add unit test of file deletion --- .../conat/test/sync-doc/delete.test.ts | 53 +++++++++++++++ src/packages/conat/files/watch.ts | 9 ++- src/packages/conat/socket/client.ts | 2 +- src/packages/conat/sync-doc/sync-client.ts | 7 ++ src/packages/conat/sync-doc/syncdb.ts | 2 +- src/packages/conat/sync-doc/syncstring.ts | 9 ++- src/packages/sync/editor/generic/sync-doc.ts | 65 +++++++++++++++---- src/packages/util/async-utils.ts | 14 ++++ 8 files changed, 141 insertions(+), 20 deletions(-) create mode 100644 src/packages/backend/conat/test/sync-doc/delete.test.ts diff --git a/src/packages/backend/conat/test/sync-doc/delete.test.ts b/src/packages/backend/conat/test/sync-doc/delete.test.ts new file mode 100644 index 0000000000..c34b57d0b0 --- /dev/null +++ b/src/packages/backend/conat/test/sync-doc/delete.test.ts @@ -0,0 +1,53 @@ +import { + before, + after, + uuid, + connect, + server, + once, + delay, + waitUntilSynced, +} from "./setup"; + +beforeAll(before); +afterAll(after); + +describe("deleting a file that is open as a syncdoc", () => { + const project_id = uuid(); + const path = "a.txt"; + let client1, client2, s1, s2, fs; + const deletedThreshold = 50; // make test faster + + it("creates two clients editing 'a.txt'", async () => { + client1 = connect(); + client2 = connect(); + fs = client1.fs({ project_id, service: server.service }); + await fs.writeFile(path, "my existing file"); + s1 = client1.sync.string({ + project_id, + path, + fs, + deletedThreshold, + }); + + await once(s1, "ready"); + + s2 = client2.sync.string({ + project_id, + path, + service: server.service, + deletedThreshold, + }); + await once(s2, "ready"); + }); + + it("delete 'a.txt' from disk and both clients emit 'deleted' event in about ${deletedThreshold}ms", async () => { + const start = Date.now(); + const d1 = once(s1, "deleted"); + const d2 = once(s2, "deleted"); + await fs.unlink("a.txt"); + await d1; + await d2; + expect(Date.now() - start).toBeLessThan(deletedThreshold + 1000); + }); +}); diff --git a/src/packages/conat/files/watch.ts b/src/packages/conat/files/watch.ts index 531dc82d9f..fea473a8a8 100644 --- a/src/packages/conat/files/watch.ts +++ b/src/packages/conat/files/watch.ts @@ -8,7 +8,6 @@ import { type ServerSocket, } from "@cocalc/conat/socket"; import { EventIterator } from "@cocalc/util/event-iterator"; - import { getLogger } from "@cocalc/conat/client"; const logger = getLogger("conat:files:watch"); @@ -165,7 +164,8 @@ export async function watchClient({ path: string; options?: WatchOptions; }): Promise { - const socket = await client.socket.connect(subject); + await client.waitForInterest(subject); + const socket = client.socket.connect(subject); const iter = new EventIterator(socket, "data", { map: (args) => args[0], onEnd: () => { @@ -176,7 +176,10 @@ export async function watchClient({ iter.end(); }); // tell it what to watch - await socket.request({ path, options }); + await socket.request({ + path, + options, + }); const iter2 = iter as WatchIterator; diff --git a/src/packages/conat/socket/client.ts b/src/packages/conat/socket/client.ts index bb107bdaeb..db8598a0e2 100644 --- a/src/packages/conat/socket/client.ts +++ b/src/packages/conat/socket/client.ts @@ -225,7 +225,7 @@ export class ConatSocketClient extends ConatSocketBase { if (this.state == "closed") { throw Error("closed"); } - // console.log("sending request from client ", { subject, data, options }); + //console.log("sending request from client ", { subject, data, options }); return await this.client.request(subject, data, options); }; diff --git a/src/packages/conat/sync-doc/sync-client.ts b/src/packages/conat/sync-doc/sync-client.ts index 9609dce5a3..6c284b2421 100644 --- a/src/packages/conat/sync-doc/sync-client.ts +++ b/src/packages/conat/sync-doc/sync-client.ts @@ -18,8 +18,15 @@ export class SyncClient extends EventEmitter implements Client0 { throw Error("client must be specified"); } this.client = client; + this.client.once("closed", this.close); } + close = () => { + this.emit("closed"); + // @ts-ignore + delete this.client; + }; + is_project = (): boolean => false; is_browser = (): boolean => true; is_compute_server = (): boolean => false; diff --git a/src/packages/conat/sync-doc/syncdb.ts b/src/packages/conat/sync-doc/syncdb.ts index a4437d7459..b67add9dea 100644 --- a/src/packages/conat/sync-doc/syncdb.ts +++ b/src/packages/conat/sync-doc/syncdb.ts @@ -11,7 +11,7 @@ export interface SyncDBOptions extends Omit { export type { SyncDB }; export function syncdb({ client, service, ...opts }: SyncDBOptions): SyncDB { - const fs = client.fs({ service, project_id: opts.project_id }); + const fs = opts.fs ?? client.fs({ service, project_id: opts.project_id }); const syncClient = new SyncClient(client); return new SyncDB({ ...opts, fs, client: syncClient }); } diff --git a/src/packages/conat/sync-doc/syncstring.ts b/src/packages/conat/sync-doc/syncstring.ts index 5fb6d2e2fb..37cf1185af 100644 --- a/src/packages/conat/sync-doc/syncstring.ts +++ b/src/packages/conat/sync-doc/syncstring.ts @@ -13,9 +13,12 @@ export interface SyncStringOptions extends Omit { export type { SyncString }; -export function syncstring({ client, service, ...opts }: SyncStringOptions): SyncString { - const fs = client.fs({ service, project_id: opts.project_id }); +export function syncstring({ + client, + service, + ...opts +}: SyncStringOptions): SyncString { + const fs = opts.fs ?? client.fs({ service, project_id: opts.project_id }); const syncClient = new SyncClient(client); return new SyncString({ ...opts, fs, client: syncClient }); } - diff --git a/src/packages/sync/editor/generic/sync-doc.ts b/src/packages/sync/editor/generic/sync-doc.ts index aafea609b9..3063ce80a9 100644 --- a/src/packages/sync/editor/generic/sync-doc.ts +++ b/src/packages/sync/editor/generic/sync-doc.ts @@ -61,9 +61,9 @@ const CURSOR_THROTTLE_NATS_MS = 150; // Ignore file changes for this long after save to disk. const RECENT_SAVE_TO_DISK_MS = 2000; -// If file does not exist for this long, then we close. -const CLOSE_WHEN_DELETED_MS = 2000; -const CLOSE_CHECK_INTERVAL_MS = 500; +// If file does not exist for this long, then syncdoc emits a 'deleted' event. +const DELETED_THRESHOLD = 2000; +const DELETED_CHECK_INTERVAL = 750; const WATCH_DEBOUNCE = 250; @@ -165,6 +165,10 @@ export interface SyncOpts0 { // if true, do not implicitly save on commit. This is very // useful for unit testing to easily simulate offline state. noAutosave?: boolean; + + // optional timeout for how long to wait from when a file is + // deleted until emiting a 'deleted' event. + deletedThreshold?: number; } export interface SyncOpts extends SyncOpts0 { @@ -282,8 +286,11 @@ export class SyncDoc extends EventEmitter { private noAutosave?: boolean; + private deletedThreshold?: number; + constructor(opts: SyncOpts) { super(); + if (opts.string_id === undefined) { this.string_id = schema.client_db.sha1(opts.project_id, opts.path); } else { @@ -305,12 +312,15 @@ export class SyncDoc extends EventEmitter { "ephemeral", "fs", "noAutosave", + "deletedThreshold", ]) { if (opts[field] != undefined) { this[field] = opts[field]; } } + this.client.once("closed", this.close); + this.legacy = new LegacyHistory({ project_id: this.project_id, path: this.path, @@ -395,6 +405,7 @@ export class SyncDoc extends EventEmitter { }, { start: 3000, max: 15000, decay: 1.3 }, ); + if (this.isClosed()) return; // Success -- everything initialized with no issues. this.set_state("ready"); @@ -3813,9 +3824,16 @@ export class SyncDoc extends EventEmitter { } // console.log("watching for changes"); // use this.fs interface to watch path for changes. - this.fsFileWatcher = await this.fs.watch(this.path, { unique: true }); + try { + this.fsFileWatcher = await this.fs.watch(this.path, { unique: true }); + } catch (err) { + if (this.isClosed()) return; + throw err; + } + if (this.isClosed()) return; (async () => { for await (const { eventType, ignore } of this.fsFileWatcher) { + if (this.isClosed()) return; // we don't know what's on disk anymore, this.lastDiskValue = undefined; //console.log("got change", eventType); @@ -3823,12 +3841,28 @@ export class SyncDoc extends EventEmitter { this.fsLoadFromDiskDebounced(); } if (eventType == "rename") { + // check if file was deleted + this.fsCloseIfFileDeleted(); // always have to recreate in case of a rename this.fsFileWatcher.close(); // start a new watcher since file descriptor changed - this.fsInitFileWatcher(); - // also check if file was deleted, in which case we'll just close - this.fsCloseIfFileDeleted(); + await until( + async () => { + if (this.isClosed()) return true; + try { + await this.fsInitFileWatcher(); + return true; + } catch (err) { + // console.warn( + // "sync-doc WARNING: issue creating file watch", + // this.path, + // err, + // ); + return false; + } + }, + { min: 3000 }, + ); return; } } @@ -3865,7 +3899,8 @@ export class SyncDoc extends EventEmitter { throw Error("bug -- fs must be defined"); } const start = Date.now(); - while (Date.now() - start < CLOSE_WHEN_DELETED_MS) { + const threshold = this.deletedThreshold ?? DELETED_THRESHOLD; + while (true) { try { if (await this.fsFileExists()) { // file definitely exists right now. @@ -3876,11 +3911,17 @@ export class SyncDoc extends EventEmitter { // network not working or project off -- no way to know. return; } - await delay(CLOSE_CHECK_INTERVAL_MS); + const elapsed = Date.now() - start; + if (elapsed > threshold) { + // out of time to appear again, and definitely concluded + // it does not exist above + // file still doesn't exist -- consider it deleted -- browsers + // should close the tab and possibly notify user. + this.emit("deleted"); + return; + } + await delay(Math.min(DELETED_CHECK_INTERVAL, threshold - elapsed)); } - // file still doesn't exist -- consider it deleted -- browsers - // should close the tab and possibly notify user. - this.emit("deleted"); }; } diff --git a/src/packages/util/async-utils.ts b/src/packages/util/async-utils.ts index f0fed46ef4..ac40a124e1 100644 --- a/src/packages/util/async-utils.ts +++ b/src/packages/util/async-utils.ts @@ -168,6 +168,12 @@ export class TimeoutError extends Error { } } +function captureStackWithoutPrinting() { + const obj = {} as any; + Error.captureStackTrace(obj, captureStackWithoutPrinting); + return obj.stack; +} + /* Wait for an event emitter to emit any event at all once. Returns array of args emitted by that event. If timeout_ms is 0 (the default) this can wait an unbounded @@ -178,12 +184,14 @@ export class TimeoutError extends Error { If the obj throws 'closed' before the event is emitted, then this throws an error, since clearly event can never be emitted. */ +const DEBUG_ONCE = false; // log a better stack trace in some cases export async function once( obj: EventEmitter, event: string, timeout_ms: number | undefined = 0, ): Promise { if (obj == null) throw Error("once -- obj is undefined"); + const stack = DEBUG_ONCE ? captureStackWithoutPrinting() : undefined; if (timeout_ms == null) { // clients might explicitly pass in undefined, but below we expect 0 to mean "no timeout" timeout_ms = 0; @@ -207,11 +215,17 @@ export async function once( function onClosed() { cleanup(); + if (DEBUG_ONCE) { + console.log(stack); + } reject(new TimeoutError(`once: "${event}" not emitted before "closed"`)); } function onTimeout() { cleanup(); + if (DEBUG_ONCE) { + console.log(stack); + } reject( new TimeoutError( `once: timeout of ${timeout_ms}ms waiting for "${event}"`, From eac0e2fda4f734a7bcf89349b182048571dd8a8a Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 22 Jul 2025 15:28:54 +0000 Subject: [PATCH 058/270] fix broken test and some ts --- .../backend/conat/test/sync-doc/delete.test.ts | 11 +---------- src/packages/conat/files/watch.ts | 1 - 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/src/packages/backend/conat/test/sync-doc/delete.test.ts b/src/packages/backend/conat/test/sync-doc/delete.test.ts index c34b57d0b0..823fba9932 100644 --- a/src/packages/backend/conat/test/sync-doc/delete.test.ts +++ b/src/packages/backend/conat/test/sync-doc/delete.test.ts @@ -1,13 +1,4 @@ -import { - before, - after, - uuid, - connect, - server, - once, - delay, - waitUntilSynced, -} from "./setup"; +import { before, after, uuid, connect, server, once } from "./setup"; beforeAll(before); afterAll(after); diff --git a/src/packages/conat/files/watch.ts b/src/packages/conat/files/watch.ts index fea473a8a8..0e9027b33b 100644 --- a/src/packages/conat/files/watch.ts +++ b/src/packages/conat/files/watch.ts @@ -164,7 +164,6 @@ export async function watchClient({ path: string; options?: WatchOptions; }): Promise { - await client.waitForInterest(subject); const socket = client.socket.connect(subject); const iter = new EventIterator(socket, "data", { map: (args) => args[0], From 0120c3ec798a251761653831cb2821210320ca1d Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 22 Jul 2025 17:53:55 +0000 Subject: [PATCH 059/270] sync-doc: improving behavior based on unit testing --- .../backend/conat/files/test/watch.test.ts | 16 +- .../conat/test/sync-doc/conflict.test.ts | 2 +- .../conat/test/sync-doc/delete.test.ts | 79 ++++- .../conat/test/sync-doc/watch-file.test.ts | 4 +- src/packages/conat/core/client.ts | 7 + src/packages/conat/files/watch.ts | 11 +- src/packages/frontend/conat/client.ts | 2 +- src/packages/sync/editor/generic/sync-doc.ts | 320 ++++-------------- 8 files changed, 176 insertions(+), 265 deletions(-) diff --git a/src/packages/backend/conat/files/test/watch.test.ts b/src/packages/backend/conat/files/test/watch.test.ts index 0c3e61c376..3cefad5843 100644 --- a/src/packages/backend/conat/files/test/watch.test.ts +++ b/src/packages/backend/conat/files/test/watch.test.ts @@ -34,7 +34,7 @@ describe("basic core of the async path watch functionality", () => { }); let w; - it("create a watcher client", async () => { + it("create a watcher client for 'a.txt'", async () => { w = await watchClient({ client, subject: "foo", path: "a.txt" }); }); @@ -58,4 +58,18 @@ describe("basic core of the async path watch functionality", () => { await wait({ until: () => Object.keys(server.sockets).length == 0 }); expect(Object.keys(server.sockets).length).toEqual(0); }); + + it("trying to watch file that does not exist throws error", async () => { + await expect(async () => { + await watchClient({ client, subject: "foo", path: "b.txt" }); + }).rejects.toThrow( + "Error: ENOENT: no such file or directory, watch 'b.txt'", + ); + + try { + await watchClient({ client, subject: "foo", path: "b.txt" }); + } catch (err) { + expect(err.code).toEqual("ENOENT"); + } + }); }); diff --git a/src/packages/backend/conat/test/sync-doc/conflict.test.ts b/src/packages/backend/conat/test/sync-doc/conflict.test.ts index 34797cc53e..739b33da7e 100644 --- a/src/packages/backend/conat/test/sync-doc/conflict.test.ts +++ b/src/packages/backend/conat/test/sync-doc/conflict.test.ts @@ -215,7 +215,7 @@ describe("do the example in the blog post 'Lies I was Told About Collaborative E }); const numHeads = 15; -describe.only(`create editing conflict with ${numHeads} heads`, () => { +describe(`create editing conflict with ${numHeads} heads`, () => { const project_id = uuid(); let docs: any[] = [], clients: any[] = []; diff --git a/src/packages/backend/conat/test/sync-doc/delete.test.ts b/src/packages/backend/conat/test/sync-doc/delete.test.ts index 823fba9932..0d1854be74 100644 --- a/src/packages/backend/conat/test/sync-doc/delete.test.ts +++ b/src/packages/backend/conat/test/sync-doc/delete.test.ts @@ -1,4 +1,4 @@ -import { before, after, uuid, connect, server, once } from "./setup"; +import { before, after, uuid, connect, server, once, delay } from "./setup"; beforeAll(before); afterAll(after); @@ -8,6 +8,7 @@ describe("deleting a file that is open as a syncdoc", () => { const path = "a.txt"; let client1, client2, s1, s2, fs; const deletedThreshold = 50; // make test faster + const watchRecreateWait = 100; it("creates two clients editing 'a.txt'", async () => { client1 = connect(); @@ -19,6 +20,7 @@ describe("deleting a file that is open as a syncdoc", () => { path, fs, deletedThreshold, + watchRecreateWait, }); await once(s1, "ready"); @@ -28,11 +30,12 @@ describe("deleting a file that is open as a syncdoc", () => { path, service: server.service, deletedThreshold, + watchRecreateWait, }); await once(s2, "ready"); }); - it("delete 'a.txt' from disk and both clients emit 'deleted' event in about ${deletedThreshold}ms", async () => { + it(`delete 'a.txt' from disk and both clients emit 'deleted' event in about ${deletedThreshold}ms`, async () => { const start = Date.now(); const d1 = once(s1, "deleted"); const d2 = once(s2, "deleted"); @@ -41,4 +44,76 @@ describe("deleting a file that is open as a syncdoc", () => { await d2; expect(Date.now() - start).toBeLessThan(deletedThreshold + 1000); }); + + it("clients still work (clients can ignore 'deleted' if they want)", async () => { + expect(s1.isClosed()).toBe(false); + expect(s2.isClosed()).toBe(false); + s1.from_str("back"); + const d1 = once(s1, "watching"); + const d2 = once(s2, "watching"); + await s1.save_to_disk(); + expect(await fs.readFile("a.txt", "utf8")).toEqual("back"); + await d1; + await d2; + }); + + it(`deleting 'a.txt' again -- still triggers deleted events`, async () => { + const start = Date.now(); + const d1 = once(s1, "deleted"); + const d2 = once(s2, "deleted"); + await fs.unlink("a.txt"); + await d1; + await d2; + expect(Date.now() - start).toBeLessThan(deletedThreshold + 1000); + }); + + // it("disconnect one client, delete file, then reconnect client", async () => { + // console.log(1); + // client2.disconnect(); + // const d1 = once(s1, "deleted"); + // const d2 = once(s2, "deleted"); + // console.log(2); + // await fs.unlink("a.txt"); + // console.log(3); + // client2.connect(); + // console.log(4); + // await d1; + // console.log(5); + // await d2; + // expect(Date.now() - start).toBeLessThan(deletedThreshold + 1000); + // }); +}); + +describe("deleting a file then recreate it quickly does NOT trigger a 'deleted' event", () => { + const project_id = uuid(); + const path = "a.txt"; + let client1, s1, fs; + const deletedThreshold = 250; + + it("creates two clients editing 'a.txt'", async () => { + client1 = connect(); + fs = client1.fs({ project_id, service: server.service }); + await fs.writeFile(path, "my existing file"); + s1 = client1.sync.string({ + project_id, + path, + fs, + service: server.service, + deletedThreshold, + }); + + await once(s1, "ready"); + }); + + it(`delete 'a.txt' from disk and both clients emit 'deleted' event in about ${deletedThreshold}ms`, async () => { + let c1 = 0; + s1.once("deleted", () => { + c1++; + }); + await fs.unlink("a.txt"); + await delay(deletedThreshold - 100); + await fs.writeFile(path, "I'm back!"); + await delay(deletedThreshold); + expect(c1).toBe(0); + }); }); diff --git a/src/packages/backend/conat/test/sync-doc/watch-file.test.ts b/src/packages/backend/conat/test/sync-doc/watch-file.test.ts index e1f3cb830f..85d11a5d1f 100644 --- a/src/packages/backend/conat/test/sync-doc/watch-file.test.ts +++ b/src/packages/backend/conat/test/sync-doc/watch-file.test.ts @@ -121,11 +121,11 @@ describe("basic watching of file on disk happens automatically", () => { }); }); -describe.only("has unsaved changes", () => { +describe("has unsaved changes", () => { const project_id = uuid(); let s1, s2, client1, client2; - it("creates two clients", async () => { + it("creates two clients and opens a new file (does not exist on disk yet)", async () => { client1 = connect(); client2 = connect(); s1 = client1.sync.string({ diff --git a/src/packages/conat/core/client.ts b/src/packages/conat/core/client.ts index 58bc516679..e11b2ec0a7 100644 --- a/src/packages/conat/core/client.ts +++ b/src/packages/conat/core/client.ts @@ -573,6 +573,10 @@ export class Client extends EventEmitter { setTimeout(() => this.conn.io.disconnect(), 1); }; + connect = () => { + this.conn.io.connect(); + }; + isConnected = () => this.state == "connected"; isSignedIn = () => !!(this.info?.user && !this.info?.user?.error); @@ -1992,5 +1996,8 @@ export function headerToError(headers) { err[field] = headers.error_attrs[field]; } } + if (err['code'] === undefined && headers.code) { + err['code'] = headers.code; + } return err; } diff --git a/src/packages/conat/files/watch.ts b/src/packages/conat/files/watch.ts index 0e9027b33b..4c1beade5b 100644 --- a/src/packages/conat/files/watch.ts +++ b/src/packages/conat/files/watch.ts @@ -2,7 +2,10 @@ Remotely proxying a fs.watch AsyncIterator over a Conat Socket. */ -import { type Client as ConatClient } from "@cocalc/conat/core/client"; +import { + type Client as ConatClient, + headerToError, +} from "@cocalc/conat/core/client"; import { type ConatSocketServer, type ServerSocket, @@ -173,12 +176,16 @@ export async function watchClient({ }); socket.on("closed", () => { iter.end(); + delete iter2.ignore; }); // tell it what to watch - await socket.request({ + const resp = await socket.request({ path, options, }); + if (resp.headers?.error) { + throw headerToError(resp.headers); + } const iter2 = iter as WatchIterator; diff --git a/src/packages/frontend/conat/client.ts b/src/packages/frontend/conat/client.ts index dedb2d4533..674aa05ac5 100644 --- a/src/packages/frontend/conat/client.ts +++ b/src/packages/frontend/conat/client.ts @@ -262,7 +262,7 @@ export class ConatClient extends EventEmitter { console.log( `Connecting to ${this._conatClient?.options.address}: attempts ${attempts}`, ); - this._conatClient?.conn.io.connect(); + this._conatClient?.connect(); return false; }, { min: 3000, max: 15000 }, diff --git a/src/packages/sync/editor/generic/sync-doc.ts b/src/packages/sync/editor/generic/sync-doc.ts index 3063ce80a9..73cf96966a 100644 --- a/src/packages/sync/editor/generic/sync-doc.ts +++ b/src/packages/sync/editor/generic/sync-doc.ts @@ -64,11 +64,10 @@ const RECENT_SAVE_TO_DISK_MS = 2000; // If file does not exist for this long, then syncdoc emits a 'deleted' event. const DELETED_THRESHOLD = 2000; const DELETED_CHECK_INTERVAL = 750; +const WATCH_RECREATE_WAIT = 3000; const WATCH_DEBOUNCE = 250; -const PARALLEL_INIT = true; - import { COMPUTE_THRESH_MS, COMPUTER_SERVER_CURSOR_TYPE, @@ -169,6 +168,10 @@ export interface SyncOpts0 { // optional timeout for how long to wait from when a file is // deleted until emiting a 'deleted' event. deletedThreshold?: number; + // how long to wait before trying to recreate a watch -- this mainly + // matters in cases when the file is deleted and the client ignores + // the 'deleted' event. + watchRecreateWait?: number; } export interface SyncOpts extends SyncOpts0 { @@ -287,6 +290,7 @@ export class SyncDoc extends EventEmitter { private noAutosave?: boolean; private deletedThreshold?: number; + private watchRecreateWait?: number; constructor(opts: SyncOpts) { super(); @@ -313,6 +317,7 @@ export class SyncDoc extends EventEmitter { "fs", "noAutosave", "deletedThreshold", + "watchRecreateWait", ]) { if (opts[field] != undefined) { this[field] = opts[field]; @@ -1037,26 +1042,6 @@ export class SyncDoc extends EventEmitter { return t; }; - /* The project calls set_initialized once it has checked for - the file on disk; this way the frontend knows that the - syncstring has been initialized in the database, and also - if there was an error doing the check. - */ - private set_initialized = async ( - error: string, - read_only: boolean, - size: number, - ): Promise => { - this.assert_table_is_ready("syncstring"); - this.dbg("set_initialized")({ error, read_only, size }); - const init = { time: this.client.server_time(), size, error }; - await this.set_syncstring_table({ - init, - read_only, - last_active: this.client.server_time(), - }); - }; - /* List of logical timestamps of the versions of this string in the sync table that we opened to start editing (so starts with what was the most recent snapshot when we started). The list of timestamps @@ -1502,153 +1487,32 @@ export class SyncDoc extends EventEmitter { this.assert_not_closed( "initAll -- before init patch_list, cursors, evaluator, ipywidgets", ); - if (PARALLEL_INIT) { - await Promise.all([ - this.init_patch_list(), - this.init_cursors(), - this.init_evaluator(), - this.init_ipywidgets(), - this.initFileWatcher(), - ]); - this.assert_not_closed( - "initAll -- successful init patch_list, cursors, evaluator, and ipywidgets", - ); - } else { - await this.init_patch_list(); - this.assert_not_closed("initAll -- successful init_patch_list"); - await this.init_cursors(); - this.assert_not_closed("initAll -- successful init_patch_cursors"); - await this.init_evaluator(); - this.assert_not_closed("initAll -- successful init_evaluator"); - await this.init_ipywidgets(); - this.assert_not_closed("initAll -- successful init_ipywidgets"); - } + await Promise.all([ + this.init_patch_list(), + this.init_cursors(), + this.init_evaluator(), + this.init_ipywidgets(), + this.initFileWatcher(), + ]); + this.assert_not_closed( + "initAll -- successful init patch_list, cursors, evaluator, and ipywidgets", + ); this.init_table_close_handlers(); this.assert_not_closed("initAll -- successful init_table_close_handlers"); log("file_use_interval"); this.init_file_use_interval(); - if (this.fs != null) { - await this.fsLoadFromDisk(); - } else { - if (await this.isFileServer()) { - log("load_from_disk"); - // This sets initialized, which is needed to be fully ready. - // We keep trying this load from disk until sync-doc is closed - // or it succeeds. It may fail if, e.g., the file is too - // large or is not readable by the user. They are informed to - // fix the problem... and once they do (and wait up to 10s), - // this will finish. - // if (!this.client.is_browser() && !this.client.is_project()) { - // // FAKE DELAY!!! Just to simulate flakiness / slow network!!!! - // await delay(3000); - // } - await retry_until_success({ - f: this.init_load_from_disk, - max_delay: 10000, - desc: "syncdoc -- load_from_disk", - }); - log("done loading from disk"); - } else { - if (this.patch_list!.count() == 0) { - await Promise.race([ - this.waitUntilFullyReady(), - once(this.patch_list!, "change"), - ]); - } - } - } - this.assert_not_closed("initAll -- load from disk"); - this.emit("init"); + await this.fsLoadFromDiskIfNewer(); + + this.emit("init"); this.assert_not_closed("initAll -- after waiting until fully ready"); - if (await this.isFileServer()) { - log("init file autosave"); - this.init_file_autosave(); - } this.update_has_unsaved_changes(); log("done"); }; - private init_error = (): string | undefined => { - let x; - try { - x = this.syncstring_table.get_one(); - } catch (_err) { - // if the table hasn't been initialized yet, - // it can't be in error state. - return undefined; - } - return x?.get("init")?.get("error"); - }; - - // wait until the syncstring table is ready to be - // used (so extracted from archive, etc.), - private waitUntilFullyReady = async (): Promise => { - this.assert_not_closed("wait_until_fully_ready"); - const dbg = this.dbg("wait_until_fully_ready"); - dbg(); - - if (this.client.is_browser() && this.init_error()) { - // init is set and is in error state. Give the backend a few seconds - // to try to fix this error before giving up. The browser client - // can close and open the file to retry this (as instructed). - try { - await this.syncstring_table.wait(() => !this.init_error(), 5); - } catch (err) { - // fine -- let the code below deal with this problem... - } - } - - let init; - const is_init = (t: SyncTable) => { - this.assert_not_closed("is_init"); - const tbl = t.get_one(); - if (tbl == null) { - dbg("null"); - return false; - } - init = tbl.get("init")?.toJS(); - return init != null; - }; - dbg("waiting for init..."); - await this.syncstring_table.wait(is_init, 0); - dbg("init done"); - if (init.error) { - throw Error(init.error); - } - assertDefined(this.patch_list); - if (init.size == null) { - // don't crash but warn at least. - console.warn("SYNC BUG -- init.size must be defined", { init }); - } - if ( - !this.client.is_project() && - this.patch_list.count() === 0 && - init.size - ) { - dbg("waiting for patches for nontrivial file"); - // normally this only happens in a later event loop, - // so force it now. - dbg("handling patch update queue since", this.patch_list.count()); - await this.handle_patch_update_queue(true); - assertDefined(this.patch_list); - dbg("done handling, now ", this.patch_list.count()); - if (this.patch_list.count() === 0) { - // wait for a change -- i.e., project loading the file from - // disk and making available... Because init.size > 0, we know that - // there must be SOMETHING in the patches table once initialization is done. - // This is the root cause of https://github.com/sagemathinc/cocalc/issues/2382 - await once(this.patches_table, "change"); - dbg("got patches_table change"); - await this.handle_patch_update_queue(true); - dbg("handled update queue"); - } - } - }; - private assert_table_is_ready = (table: string): void => { const t = this[table + "_table"]; // not using string template only because it breaks codemirror! if (t == null || t.get_state() != "connected") { @@ -1780,17 +1644,6 @@ export class SyncDoc extends EventEmitter { this.set_read_only(read_only); }; - private init_load_from_disk = async (): Promise => { - if (this.state == "closed") { - // stop trying, no error -- this is assumed - // in a retry_until_success elsewhere. - return; - } - if (await this.load_from_disk_if_newer()) { - throw Error("failed to load from disk"); - } - }; - private fsLoadFromDiskIfNewer = async (): Promise => { // [ ] TODO: readonly handling... if (this.fs == null) throw Error("bug"); @@ -1828,54 +1681,6 @@ export class SyncDoc extends EventEmitter { return false; }; - private load_from_disk_if_newer = async (): Promise => { - if (this.fs != null) { - return await this.fsLoadFromDiskIfNewer(); - } - if (this.client.path_exists == null) { - throw Error("legacy clients must define path_exists"); - } - const last_changed = new Date(this.last_changed()); - const firstLoad = this.versions().length == 0; - const dbg = this.dbg("load_from_disk_if_newer"); - let is_read_only: boolean = false; - let size: number = 0; - let error: string = ""; - try { - dbg("check if path exists"); - if (await callback2(this.client.path_exists, { path: this.path })) { - // the path exists - dbg("path exists -- stat file"); - const stats = await callback2(this.client.path_stat, { - path: this.path, - }); - if (firstLoad || stats.ctime > last_changed) { - dbg( - `disk file changed more recently than edits (or first load), so loading, ${stats.ctime} > ${last_changed}; firstLoad=${firstLoad}`, - ); - size = await this.readFile(); - if (firstLoad) { - dbg("emitting first-load event"); - // this event is emited the first time the document is ever loaded from disk. - this.emit("first-load"); - } - dbg("loaded"); - } else { - dbg("stick with database version"); - } - dbg("checking if read only"); - is_read_only = await this.file_is_read_only(); - dbg("read_only", is_read_only); - } - } catch (err) { - error = `${err}`; - } - - await this.set_initialized(error, is_read_only, size); - dbg("done"); - return !!error; - }; - private patch_table_query = (cutoff?: number) => { const query = { string_id: this.string_id, @@ -3266,7 +3071,12 @@ export class SyncDoc extends EventEmitter { // tell watcher not to fire any change events for a little time, // so no clients waste resources loading in response to us saving // to disk. - await this.fsFileWatcher?.ignore(2000); + try { + await this.fsFileWatcher?.ignore(2000); + } catch { + // not a big problem if we can't ignore (e.g., this happens potentially + // after deleting the file or if file doesn't exist) + } if (this.isClosed()) return; this.last_save_to_disk_time = new Date(); await this.fs.writeFile(this.path, value); @@ -3822,51 +3632,48 @@ export class SyncDoc extends EventEmitter { if (this.fs == null) { throw Error("this.fs must be defined"); } - // console.log("watching for changes"); - // use this.fs interface to watch path for changes. + // use this.fs interface to watch path for changes -- we try once: try { this.fsFileWatcher = await this.fs.watch(this.path, { unique: true }); - } catch (err) { - if (this.isClosed()) return; - throw err; - } + } catch {} if (this.isClosed()) return; + + // not closed -- so if above succeeds we start watching. + // if not, we loop waiting for file to be created so we can watch it (async () => { - for await (const { eventType, ignore } of this.fsFileWatcher) { - if (this.isClosed()) return; - // we don't know what's on disk anymore, - this.lastDiskValue = undefined; - //console.log("got change", eventType); - if (!ignore) { - this.fsLoadFromDiskDebounced(); - } - if (eventType == "rename") { - // check if file was deleted - this.fsCloseIfFileDeleted(); - // always have to recreate in case of a rename - this.fsFileWatcher.close(); - // start a new watcher since file descriptor changed - await until( - async () => { - if (this.isClosed()) return true; - try { - await this.fsInitFileWatcher(); - return true; - } catch (err) { - // console.warn( - // "sync-doc WARNING: issue creating file watch", - // this.path, - // err, - // ); - return false; - } - }, - { min: 3000 }, - ); - return; + if (this.fsFileWatcher != null) { + this.emit("watching"); + for await (const { eventType, ignore } of this.fsFileWatcher) { + if (this.isClosed()) return; + // we don't know what's on disk anymore, + this.lastDiskValue = undefined; + //console.log("got change", eventType); + if (!ignore) { + this.fsLoadFromDiskDebounced(); + } + if (eventType == "rename") { + break; + } } - } - //console.log("done watching"); + // check if file was deleted + this.fsCloseIfFileDeleted(); + this.fsFileWatcher?.close(); + delete this.fsFileWatcher; + } + // start a new watcher since file descriptor probably changed or maybe file deleted + await delay(this.watchRecreateWait ?? WATCH_RECREATE_WAIT); + await until( + async () => { + if (this.isClosed()) return true; + try { + await this.fsInitFileWatcher(); + return true; + } catch { + return false; + } + }, + { min: this.watchRecreateWait ?? WATCH_RECREATE_WAIT }, + ); })(); }; @@ -3895,6 +3702,7 @@ export class SyncDoc extends EventEmitter { }; private fsCloseIfFileDeleted = async () => { + if (this.isClosed()) return; if (this.fs == null) { throw Error("bug -- fs must be defined"); } From 156bef6d01e3fdbdc1394ec63ec49a6a56c70c4f Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 22 Jul 2025 18:48:08 +0000 Subject: [PATCH 060/270] syncdoc: a little bit of org and opt; frontend -- delay spinners and progress --- .../frontend/components/fake-progress.tsx | 1 - src/packages/frontend/components/loading.tsx | 14 +- .../frame-editors/frame-tree/editor.tsx | 4 +- .../frontend/project/new/new-file-page.tsx | 4 +- src/packages/sync/editor/generic/sync-doc.ts | 122 ++---------------- 5 files changed, 24 insertions(+), 121 deletions(-) diff --git a/src/packages/frontend/components/fake-progress.tsx b/src/packages/frontend/components/fake-progress.tsx index d634e14ad5..1b0f10ef85 100644 --- a/src/packages/frontend/components/fake-progress.tsx +++ b/src/packages/frontend/components/fake-progress.tsx @@ -23,7 +23,6 @@ export default function FakeProgress({ time }) { return ( null} percent={percent} strokeColor={{ "0%": "#108ee9", "100%": "#87d068" }} diff --git a/src/packages/frontend/components/loading.tsx b/src/packages/frontend/components/loading.tsx index 8e0dac0769..5b1578de9b 100644 --- a/src/packages/frontend/components/loading.tsx +++ b/src/packages/frontend/components/loading.tsx @@ -19,9 +19,9 @@ export const Estimate = null; // webpack + TS es2020 modules need this interface Props { style?: CSSProperties; text?: string; - estimate?: Estimate; + estimate?: Estimate | number; theme?: "medium" | undefined; - delay?: number; // if given, don't show anything until after delay milliseconds. The component could easily unmount by then, and hence never annoyingly flicker on screen. + delay?: number; // (default:1000) don't show anything until after delay milliseconds. The component could easily unmount by then, and hence never annoyingly flicker on screen. transparent?: boolean; } @@ -40,7 +40,7 @@ export function Loading({ text, estimate, theme, - delay, + delay = 1000, transparent = false, }: Props) { const intl = useIntl(); @@ -64,7 +64,13 @@ export function Loading({ {estimate != undefined && (
- +
)} diff --git a/src/packages/frontend/frame-editors/frame-tree/editor.tsx b/src/packages/frontend/frame-editors/frame-tree/editor.tsx index 3cd75739f8..a689300108 100644 --- a/src/packages/frontend/frame-editors/frame-tree/editor.tsx +++ b/src/packages/frontend/frame-editors/frame-tree/editor.tsx @@ -18,7 +18,7 @@ import { import { ErrorDisplay, Loading, - LoadingEstimate, + type LoadingEstimate, } from "@cocalc/frontend/components"; import { AvailableFeatures } from "@cocalc/frontend/project_configuration"; import { is_different } from "@cocalc/util/misc"; @@ -194,7 +194,7 @@ const FrameTreeEditor: React.FC = React.memo( if (is_loaded) return; return (
- +
); } diff --git a/src/packages/frontend/project/new/new-file-page.tsx b/src/packages/frontend/project/new/new-file-page.tsx index 08df80c968..015c97b069 100644 --- a/src/packages/frontend/project/new/new-file-page.tsx +++ b/src/packages/frontend/project/new/new-file-page.tsx @@ -6,7 +6,6 @@ import { Button, Input, Modal, Space } from "antd"; import { useEffect, useRef, useState } from "react"; import { defineMessage, FormattedMessage, useIntl } from "react-intl"; - import { default_filename } from "@cocalc/frontend/account"; import { Alert, Col, Row } from "@cocalc/frontend/antd-bootstrap"; import { @@ -25,7 +24,6 @@ import { SettingBox, Tip, } from "@cocalc/frontend/components"; -import FakeProgress from "@cocalc/frontend/components/fake-progress"; import ComputeServer from "@cocalc/frontend/compute/inline"; import { filenameIcon } from "@cocalc/frontend/file-associations"; import { FileUpload, UploadLink } from "@cocalc/frontend/file-upload"; @@ -430,7 +428,7 @@ export default function NewFilePage(props: Props) { footer={<>} >
- +
diff --git a/src/packages/sync/editor/generic/sync-doc.ts b/src/packages/sync/editor/generic/sync-doc.ts index 73cf96966a..6a477b6768 100644 --- a/src/packages/sync/editor/generic/sync-doc.ts +++ b/src/packages/sync/editor/generic/sync-doc.ts @@ -119,7 +119,6 @@ import type { Patch, } from "./types"; import { isTestClient, patch_cmp } from "./util"; -import { CONAT_OPEN_FILE_TOUCH_INTERVAL } from "@cocalc/util/conat"; import mergeDeep from "@cocalc/util/immutable-deep-merge"; import { JUPYTER_SYNCDB_EXTENSIONS } from "@cocalc/util/jupyter/names"; import { LegacyHistory } from "./legacy"; @@ -399,7 +398,7 @@ export class SyncDoc extends EventEmitter { } const m = `WARNING: problem initializing ${this.path} -- ${err}`; log(m); - if (DEBUG) { + if (DEBUG || true) { console.trace(err); } // log always @@ -1200,59 +1199,6 @@ export class SyncDoc extends EventEmitter { this.ipywidgets_state?.close(); }; - // TODO: We **have** to do this on the client, since the backend - // **security model** for accessing the patches table only - // knows the string_id, but not the project_id/path. Thus - // there is no way currently to know whether or not the client - // has access to the patches, and hence the patches table - // query fails. This costs significant time -- a roundtrip - // and write to the database -- whenever the user opens a file. - // This fix should be to change the patches schema somehow - // to have the user also provide the project_id and path, thus - // proving they have access to the sha1 hash (string_id), but - // don't actually use the project_id and path as columns in - // the table. This requires some new idea I guess of virtual - // fields.... - // Also, this also establishes the correct doctype. - - // Since this MUST succeed before doing anything else. This is critical - // because the patches table can't be opened anywhere if the syncstring - // object doesn't exist, due to how our security works, *AND* that the - // patches table uses the string_id, which is a SHA1 hash. - private ensure_syncstring_exists_in_db = async (): Promise => { - const dbg = this.dbg("ensure_syncstring_exists_in_db"); - if (this.useConat) { - dbg("skipping -- no database"); - return; - } - - if (!this.client.is_connected()) { - dbg("wait until connected...", this.client.is_connected()); - await once(this.client, "connected"); - } - - if (this.client.is_browser() && !this.client.is_signed_in()) { - // the browser has to sign in, unlike the project (and compute servers) - await once(this.client, "signed_in"); - } - - if (this.state == ("closed" as State)) return; - - dbg("do syncstring write query..."); - - await callback2(this.client.query, { - query: { - syncstrings: { - string_id: this.string_id, - project_id: this.project_id, - path: this.path, - doctype: JSON.stringify(this.doctype), - }, - }, - }); - dbg("wrote syncstring to db - done."); - }; - private synctable = async ( query, options: any[], @@ -1445,15 +1391,10 @@ export class SyncDoc extends EventEmitter { dbg("getting table..."); this.syncstring_table = await this.synctable(query, []); - if (this.ephemeral && this.client.is_project()) { - await this.set_syncstring_table({ - doctype: JSON.stringify(this.doctype), - }); - } else { - dbg("handling the first update..."); - this.handle_syncstring_update(); - } + dbg("handling the first update..."); + this.handle_syncstring_update(); this.syncstring_table.on("change", this.handle_syncstring_update); + this.syncstring_table.on("change", this.update_has_unsaved_changes); }; // Used for internal debug logging @@ -1468,26 +1409,18 @@ export class SyncDoc extends EventEmitter { }; private initAll = async (): Promise => { + //const t0 = Date.now(); if (this.state !== "init") { throw Error("connect can only be called in init state"); } const log = this.dbg("initAll"); - log("update interest"); - this.initInterestLoop(); - - log("ensure syncstring exists"); - this.assert_not_closed("initAll -- before ensuring syncstring exists"); - await this.ensure_syncstring_exists_in_db(); - - await this.init_syncstring_table(); - this.assert_not_closed("initAll -- successful init_syncstring_table"); - log("patch_list, cursors, evaluator, ipywidgets"); this.assert_not_closed( "initAll -- before init patch_list, cursors, evaluator, ipywidgets", ); await Promise.all([ + this.init_syncstring_table(), this.init_patch_list(), this.init_cursors(), this.init_evaluator(), @@ -1511,6 +1444,7 @@ export class SyncDoc extends EventEmitter { this.update_has_unsaved_changes(); log("done"); + //console.log("initAll: done", Date.now() - t0); }; private assert_table_is_ready = (table: string): void => { @@ -1662,9 +1596,9 @@ export class SyncDoc extends EventEmitter { } } dbg("path exists"); - const lastChanged = new Date(this.last_changed()); + const lastChanged = this.last_changed(); const firstLoad = this.versions().length == 0; - if (firstLoad || stats.ctime > lastChanged) { + if (firstLoad || stats.mtime.valueOf() > lastChanged) { dbg( `disk file changed more recently than edits, so loading ${stats.ctime} > ${lastChanged}; firstLoad=${firstLoad}`, ); @@ -1747,10 +1681,6 @@ export class SyncDoc extends EventEmitter { update_has_unsaved_changes(); }); - this.syncstring_table.on("change", () => { - update_has_unsaved_changes(); - }); - dbg("adding all known patches"); patch_list.add(this.get_patches()); @@ -3080,6 +3010,8 @@ export class SyncDoc extends EventEmitter { if (this.isClosed()) return; this.last_save_to_disk_time = new Date(); await this.fs.writeFile(this.path, value); + const lastChanged = this.last_changed(); + await this.fs.utimes(this.path, lastChanged / 1000, lastChanged / 1000); this.lastDiskValue = value; }; @@ -3581,38 +3513,6 @@ export class SyncDoc extends EventEmitter { } }, 60000); - private initInterestLoop = async () => { - if (this.fs != null) { - return; - } - if (!this.client.is_browser()) { - // only browser clients -- so actual humans - return; - } - const touch = async () => { - if (this.state == "closed" || this.client?.touchOpenFile == null) return; - await this.client.touchOpenFile({ - path: this.path, - project_id: this.project_id, - doctype: this.doctype, - }); - }; - // then every CONAT_OPEN_FILE_TOUCH_INTERVAL (30 seconds). - await until( - async () => { - if (this.state == "closed") { - return true; - } - await touch(); - return false; - }, - { - start: CONAT_OPEN_FILE_TOUCH_INTERVAL, - max: CONAT_OPEN_FILE_TOUCH_INTERVAL, - }, - ); - }; - private fsLoadFromDiskDebounced = asyncDebounce( async () => { try { From c78bc6266aba1bfddfdd92b2cfe0b974da23f61a Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 22 Jul 2025 19:01:20 +0000 Subject: [PATCH 061/270] preload background file tabs --- src/packages/frontend/project/open-file.ts | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/packages/frontend/project/open-file.ts b/src/packages/frontend/project/open-file.ts index 2f9a49912b..c905f6914b 100644 --- a/src/packages/frontend/project/open-file.ts +++ b/src/packages/frontend/project/open-file.ts @@ -28,6 +28,14 @@ import { syncdbPath as ipynbSyncdbPath } from "@cocalc/util/jupyter/names"; import { termPath } from "@cocalc/util/terminal/names"; import { excludeFromComputeServer } from "@cocalc/frontend/file-associations"; +// if true, PRELOAD_BACKGROUND_TABS makes it so all tabs have their file editing +// preloaded, even background tabs. This can make the UI much more responsive, +// since after refreshing your browser or opening a project that had tabs open, +// all files are ready to edit instantly. It uses more browser memory (of course), +// and increases server load. Most users have very few files open at once, +// so this is probably a major win for power users and has little impact on load. +const PRELOAD_BACKGROUND_TABS = true; + export interface OpenFileOpts { path: string; ext?: string; // if given, use editor for this extension instead of whatever extension path has. @@ -339,6 +347,10 @@ export async function open_file( return; } + if (PRELOAD_BACKGROUND_TABS) { + await actions.initFileRedux(opts.path); + } + if (opts.foreground) { actions.foreground_project(opts.change_history); const tab = path_to_tab(opts.path); From 687f713bb4b2872f5569845614cbe0a2f95a14ee Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 22 Jul 2025 20:56:34 +0000 Subject: [PATCH 062/270] syncdoc -- delete the old approach and embrace this.fs --- .../conat/test/sync-doc/watch-file.test.ts | 9 +- src/packages/sync/editor/generic/sync-doc.ts | 948 +----------------- 2 files changed, 54 insertions(+), 903 deletions(-) diff --git a/src/packages/backend/conat/test/sync-doc/watch-file.test.ts b/src/packages/backend/conat/test/sync-doc/watch-file.test.ts index 85d11a5d1f..8f1238dbc0 100644 --- a/src/packages/backend/conat/test/sync-doc/watch-file.test.ts +++ b/src/packages/backend/conat/test/sync-doc/watch-file.test.ts @@ -47,9 +47,9 @@ describe("basic watching of file on disk happens automatically", () => { }); it("change file on disk should not trigger a load from disk", async () => { - const orig = s.fsLoadFromDiskDebounced; + const orig = s.readFileDebounced; let c = 0; - s.fsLoadFromDiskDebounced = () => { + s.readFileDebounced = () => { c += 1; }; s.from_str("a different value"); @@ -57,10 +57,10 @@ describe("basic watching of file on disk happens automatically", () => { expect(c).toBe(0); await delay(100); expect(c).toBe(0); - s.fsLoadFromDiskDebounced = orig; + s.readFileDebounced = orig; // disable the ignore that happens as part of save_to_disk, // or the tests below won't work - await s.fsFileWatcher?.ignore(0); + await s.fileWatcher?.ignore(0); }); let client2, s2; @@ -78,6 +78,7 @@ describe("basic watching of file on disk happens automatically", () => { s2.on("handle-file-change", () => c2++); await fs.writeFile(path, "version3"); + expect(await fs.readFile(path, "utf8")).toEqual("version3"); await wait({ until: () => { return s2.to_str() == "version3" && s.to_str() == "version3"; diff --git a/src/packages/sync/editor/generic/sync-doc.ts b/src/packages/sync/editor/generic/sync-doc.ts index 6a477b6768..68e780ef0a 100644 --- a/src/packages/sync/editor/generic/sync-doc.ts +++ b/src/packages/sync/editor/generic/sync-doc.ts @@ -19,47 +19,13 @@ EVENTS: - ... TODO */ -const USE_CONAT = true; - -/* OFFLINE_THRESH_S - If the client becomes disconnected from - the backend for more than this long then---on reconnect---do - extra work to ensure that all snapshots are up to date (in - case snapshots were made when we were offline), and mark the - sent field of patches that weren't saved. I.e., we rebase - all offline changes. */ -// const OFFLINE_THRESH_S = 5 * 60; // 5 minutes. - -/* How often the local hub will autosave this file to disk if - it has it open and there are unsaved changes. This is very - important since it ensures that a user that edits a file but - doesn't click "Save" and closes their browser (right after - their edits have gone to the database), still has their - file saved to disk soon. This is important, e.g., for homework - getting collected and not missing the last few changes. It turns - out this is what people expect. - Set to 0 to disable. (But don't do that.) */ -const FILE_SERVER_AUTOSAVE_S = 45; -// const FILE_SERVER_AUTOSAVE_S = 5; - // How big of files we allow users to open using syncstrings. const MAX_FILE_SIZE_MB = 32; -// How frequently to check if file is or is not read only. -// The filesystem watcher is NOT sufficient for this, because -// it is NOT triggered on permissions changes. Thus we must -// poll for read only status periodically, unfortunately. -const READ_ONLY_CHECK_INTERVAL_MS = 7500; - // This parameter determines throttling when broadcasting cursor position // updates. Make this larger to reduce bandwidth at the expense of making // cursors less responsive. -const CURSOR_THROTTLE_MS = 750; - -// NATS is much faster and can handle load, and cursors only uses pub/sub -const CURSOR_THROTTLE_NATS_MS = 150; - -// Ignore file changes for this long after save to disk. -const RECENT_SAVE_TO_DISK_MS = 2000; +const CURSOR_THROTTLE_MS = 150; // If file does not exist for this long, then syncdoc emits a 'deleted' event. const DELETED_THRESHOLD = 2000; @@ -72,7 +38,6 @@ import { COMPUTE_THRESH_MS, COMPUTER_SERVER_CURSOR_TYPE, decodeUUIDtoNum, - SYNCDB_PARAMS as COMPUTE_SERVE_MANAGER_SYNCDB_PARAMS, } from "@cocalc/util/compute/manager"; import { DEFAULT_SNAPSHOT_INTERVAL } from "@cocalc/util/db-schema/syncstring-schema"; @@ -85,16 +50,13 @@ import { callback2, cancel_scheduled, once, - retry_until_success, until, asyncDebounce, } from "@cocalc/util/async-utils"; import { wait } from "@cocalc/util/async-wait"; import { - auxFileToOriginal, assertDefined, close, - endswith, field_cmp, filename_extension, hash_string, @@ -115,7 +77,6 @@ import type { CompressedPatch, DocType, Document, - FileWatcher, Patch, } from "./types"; import { isTestClient, patch_cmp } from "./util"; @@ -203,12 +164,6 @@ export class SyncDoc extends EventEmitter { // Throttling of incoming upstream patches from project to client. private patch_interval: number = 250; - // This is what's actually output by setInterval -- it's - // not an amount of time. - private fileserver_autosave_timer: number = 0; - - private read_only_timer: number = 0; - // throttling of change events -- e.g., is useful for course // editor where we have hundreds of changes and the UI gets // overloaded unless we throttle and group them. @@ -253,22 +208,14 @@ export class SyncDoc extends EventEmitter { private settings: Map = Map(); - private syncstring_save_state: string = ""; - // patches that this client made during this editing session. private my_patches: { [time: string]: XPatch } = {}; - private watch_path?: string; - private file_watcher?: FileWatcher; - private handle_patch_update_queue_running: boolean; private patch_update_queue: string[] = []; private undo_state: UndoState | undefined; - private save_to_disk_start_ctime: number | undefined; - private save_to_disk_end_ctime: number | undefined; - private persistent: boolean = false; private last_has_unsaved_changes?: boolean = undefined; @@ -278,9 +225,6 @@ export class SyncDoc extends EventEmitter { private sync_is_disabled: boolean = false; private delay_sync_timer: any; - // static because we want exactly one across all docs! - private static computeServerManagerDoc?: SyncDoc; - private useConat: boolean; legacy: LegacyHistory; @@ -334,7 +278,7 @@ export class SyncDoc extends EventEmitter { // NOTE: Do not use conat in test mode, since there we use a minimal // "fake" client that does all communication internally and doesn't // use conat. We also use this for the messages composer. - this.useConat = USE_CONAT && !isTestClient(opts.client); + this.useConat = !isTestClient(opts.client); if (this.ephemeral) { // So the doctype written to the database reflects the // ephemeral state. Here ephemeral determines whether @@ -416,164 +360,6 @@ export class SyncDoc extends EventEmitter { this.emit_change(); // from nothing to something. }; - // True if this client is responsible for managing - // the state of this document with respect to - // the file system. By default, the project is responsible, - // but it could be something else (e.g., a compute server!). It's - // important that whatever algorithm determines this, it is - // a function of state that is eventually consistent. - // IMPORTANT: whether or not we are the file server can - // change over time, so if you call isFileServer and - // set something up (e.g., autosave or a watcher), based - // on the result, you need to clear it when the state - // changes. See the function handleComputeServerManagerChange. - private isFileServer = reuseInFlight(async () => { - if (this.state == "closed") return; - if (this.client == null || this.client.is_browser()) { - // browser is never the file server (yet), and doesn't need to do - // anything related to watching for changes in state. - // Someday via webassembly or browsers making users files availabl, - // etc., we will have this. Not today. - return false; - } - const computeServerManagerDoc = this.getComputeServerManagerDoc(); - const log = this.dbg("isFileServer"); - if (computeServerManagerDoc == null) { - log("not using compute server manager for this doc"); - return this.client.is_project(); - } - - const state = computeServerManagerDoc.get_state(); - log("compute server manager doc state: ", state); - if (state == "closed") { - log("compute server manager is closed"); - // something really messed up - return this.client.is_project(); - } - if (state != "ready") { - try { - log( - "waiting for compute server manager doc to be ready; current state=", - state, - ); - await once(computeServerManagerDoc, "ready", 15000); - log("compute server manager is ready"); - } catch (err) { - log( - "WARNING -- failed to initialize computeServerManagerDoc -- err=", - err, - ); - return this.client.is_project(); - } - } - - // id of who the user *wants* to be the file server. - const path = this.getFileServerPath(); - const fileServerId = - computeServerManagerDoc.get_one({ path })?.get("id") ?? 0; - if (this.client.is_project()) { - log( - "we are project, so we are fileserver if fileServerId=0 and it is ", - fileServerId, - ); - return fileServerId == 0; - } - // at this point we have to be a compute server - const computeServerId = decodeUUIDtoNum(this.client.client_id()); - // this is usually true -- but might not be if we are switching - // directly from one compute server to another. - log("we are compute server and ", { fileServerId, computeServerId }); - return fileServerId == computeServerId; - }); - - private getFileServerPath = () => { - if (this.path?.endsWith("." + JUPYTER_SYNCDB_EXTENSIONS)) { - // treating jupyter as a weird special case here. - return auxFileToOriginal(this.path); - } - return this.path; - }; - - private getComputeServerManagerDoc = () => { - if (this.path == COMPUTE_SERVE_MANAGER_SYNCDB_PARAMS.path) { - // don't want to recursively explode! - return null; - } - if (SyncDoc.computeServerManagerDoc == null) { - if (this.client.is_project()) { - // @ts-ignore: TODO! - SyncDoc.computeServerManagerDoc = this.client.syncdoc({ - path: COMPUTE_SERVE_MANAGER_SYNCDB_PARAMS.path, - }); - } else { - // @ts-ignore: TODO! - SyncDoc.computeServerManagerDoc = this.client.sync_client.sync_db({ - project_id: this.project_id, - ...COMPUTE_SERVE_MANAGER_SYNCDB_PARAMS, - }); - } - if ( - SyncDoc.computeServerManagerDoc != null && - !this.client.is_browser() - ) { - // start watching for state changes - SyncDoc.computeServerManagerDoc.on( - "change", - this.handleComputeServerManagerChange, - ); - } - } - return SyncDoc.computeServerManagerDoc; - }; - - private handleComputeServerManagerChange = async (keys) => { - if (SyncDoc.computeServerManagerDoc == null) { - return; - } - let relevant = false; - for (const key of keys ?? []) { - if (key.get("path") == this.path) { - relevant = true; - break; - } - } - if (!relevant) { - return; - } - const path = this.getFileServerPath(); - const fileServerId = - SyncDoc.computeServerManagerDoc.get_one({ path })?.get("id") ?? 0; - const ourId = this.client.is_project() - ? 0 - : decodeUUIDtoNum(this.client.client_id()); - // we are considering ourself the file server already if we have - // either a watcher or autosave on. - const thinkWeAreFileServer = - this.file_watcher != null || this.fileserver_autosave_timer; - const weAreFileServer = fileServerId == ourId; - if (thinkWeAreFileServer != weAreFileServer) { - // life has changed! Let's adapt. - if (thinkWeAreFileServer) { - // we were acting as the file server, but now we are not. - await this.save_to_disk_filesystem_owner(); - // Stop doing things we are no longer supposed to do. - clearInterval(this.fileserver_autosave_timer as any); - this.fileserver_autosave_timer = 0; - // stop watching filesystem - await this.update_watch_path(); - } else { - // load our state from the disk - await this.readFile(); - // we were not acting as the file server, but now we need. Let's - // step up to the plate. - // start watching filesystem - await this.update_watch_path(this.path); - // enable autosave - await this.init_file_autosave(); - } - } - }; - // Return id of ACTIVE remote compute server, if one is connected and pinging, or 0 // if none is connected. This is used by Jupyter to determine who // should evaluate code. @@ -627,7 +413,7 @@ export class SyncDoc extends EventEmitter { locs: any, side_effect: boolean = false, ) => { - if (this.state != "ready") { + if (!this.isReady()) { return; } if (this.cursors_table == null) { @@ -680,7 +466,7 @@ export class SyncDoc extends EventEmitter { set_cursor_locs: typeof this.setCursorLocsNoThrottle = throttle( this.setCursorLocsNoThrottle, - USE_CONAT ? CURSOR_THROTTLE_NATS_MS : CURSOR_THROTTLE_MS, + CURSOR_THROTTLE_MS, { leading: true, trailing: true, @@ -729,6 +515,7 @@ export class SyncDoc extends EventEmitter { }; isClosed = () => (this.state ?? "closed") == "closed"; + isReady = () => this.state == "ready"; private set_state = (state: State): void => { this.state = state; @@ -978,42 +765,6 @@ export class SyncDoc extends EventEmitter { return this.undo_state; }; - private save_to_disk_autosave = async (): Promise => { - if (this.state !== "ready") { - return; - } - const dbg = this.dbg("save_to_disk_autosave"); - dbg(); - try { - await this.save_to_disk(); - } catch (err) { - dbg(`failed -- ${err}`); - } - }; - - /* Make it so the local hub project will automatically save - the file to disk periodically. */ - private init_file_autosave = async () => { - // Do not autosave sagews until we resolve - // https://github.com/sagemathinc/cocalc/issues/974 - // Similarly, do not autosave ipynb because of - // https://github.com/sagemathinc/cocalc/issues/5216 - if ( - !FILE_SERVER_AUTOSAVE_S || - !(await this.isFileServer()) || - this.fileserver_autosave_timer || - endswith(this.path, ".sagews") || - endswith(this.path, "." + JUPYTER_SYNCDB_EXTENSIONS) - ) { - return; - } - - // Explicit cast due to node vs browser typings. - this.fileserver_autosave_timer = ( - setInterval(this.save_to_disk_autosave, FILE_SERVER_AUTOSAVE_S * 1000) - ); - }; - // account_id of the user who made the edit at // the given point in time. account_id = (time: number): string => { @@ -1119,10 +870,6 @@ export class SyncDoc extends EventEmitter { const dbg = this.dbg("close"); dbg("close"); - SyncDoc.computeServerManagerDoc?.removeListener( - "change", - this.handleComputeServerManagerChange, - ); // // SYNC STUFF // @@ -1152,25 +899,13 @@ export class SyncDoc extends EventEmitter { cancel_scheduled(this.emit_change); } - if (this.fileserver_autosave_timer) { - clearInterval(this.fileserver_autosave_timer as any); - this.fileserver_autosave_timer = 0; - } - - if (this.read_only_timer) { - clearInterval(this.read_only_timer as any); - this.read_only_timer = 0; - } - this.patch_update_queue = []; // Stop watching for file changes. It's important to // do this *before* all the await's below, since // this syncdoc can't do anything in response to a // a file change in its current state. - this.update_watch_path(); // no input = closes it, if open - - this.fsCloseFileWatcher(); + this.closeFileWatcher(); if (this.patch_list != null) { // not async -- just a data structure in memory @@ -1437,7 +1172,7 @@ export class SyncDoc extends EventEmitter { log("file_use_interval"); this.init_file_use_interval(); - await this.fsLoadFromDiskIfNewer(); + await this.loadFromDiskIfNewer(); this.emit("init"); this.assert_not_closed("initAll -- after waiting until fully ready"); @@ -1457,7 +1192,7 @@ export class SyncDoc extends EventEmitter { }; assert_is_ready = (desc: string): void => { - if (this.state != "ready") { + if (!this.isReady()) { throw Error(`must be ready -- ${desc}`); } }; @@ -1531,57 +1266,10 @@ export class SyncDoc extends EventEmitter { await Promise.all(v); }; - private pathExistsAndIsReadOnly = async (path): Promise => { - if (this.client.path_exists == null) { - throw Error("legacy clients must define path_exists"); - } - if (this.client.path_access == null) { - throw Error("legacy clients must define path_access"); - } - - try { - await callback2(this.client.path_access, { - path, - mode: "w", - }); - // clearly exists and is NOT read only: - return false; - } catch (err) { - // either it doesn't exist or it is read only - if (await callback2(this.client.path_exists, { path })) { - // it exists, so is read only and exists - return true; - } - // doesn't exist - return false; - } - }; - - private file_is_read_only = async (): Promise => { - if (await this.pathExistsAndIsReadOnly(this.path)) { - return true; - } - const path = this.getFileServerPath(); - if (path != this.path) { - if (await this.pathExistsAndIsReadOnly(path)) { - return true; - } - } - return false; - }; - - private update_if_file_is_read_only = async (): Promise => { - const read_only = await this.file_is_read_only(); - if (this.state == "closed") { - return; - } - this.set_read_only(read_only); - }; - - private fsLoadFromDiskIfNewer = async (): Promise => { + private loadFromDiskIfNewer = async (): Promise => { // [ ] TODO: readonly handling... if (this.fs == null) throw Error("bug"); - const dbg = this.dbg("fsLoadFromDiskIfNewer"); + const dbg = this.dbg("loadFromDiskIfNewer"); let stats; try { stats = await this.fs.stat(this.path); @@ -1602,7 +1290,7 @@ export class SyncDoc extends EventEmitter { dbg( `disk file changed more recently than edits, so loading ${stats.ctime} > ${lastChanged}; firstLoad=${firstLoad}`, ); - await this.fsLoadFromDisk(); + await this.readFile(); if (firstLoad) { dbg("emitting first-load event"); // this event is emited the first time the document is ever loaded from disk. @@ -2008,12 +1696,12 @@ export class SyncDoc extends EventEmitter { break; } } - if (this.state != "ready") { + if (!this.isReady()) { // above async waits could have resulted in state change. return; } await this.handle_patch_update_queue(true); - if (this.state != "ready") { + if (!this.isReady()) { return; } @@ -2174,7 +1862,7 @@ export class SyncDoc extends EventEmitter { this.patch_list.add([obj]); this.patches_table.set(obj); await this.patches_table.save(); - if (this.state != "ready") { + if (!this.isReady()) { return; } @@ -2449,77 +2137,6 @@ export class SyncDoc extends EventEmitter { return this.last_save_to_disk_time; }; - private handle_syncstring_save_state = async ( - state: string, - time: Date, - ): Promise => { - // Called when the save state changes. - - /* this.syncstring_save_state is used to make it possible to emit a - 'save-to-disk' event, whenever the state changes - to indicate a save completed. - - NOTE: it is intentional that this.syncstring_save_state is not defined - the first time this function is called, so that save-to-disk - with last save time gets emitted on initial load (which, e.g., triggers - latex compilation properly in case of a .tex file). - */ - if (state === "done" && this.syncstring_save_state !== "done") { - this.last_save_to_disk_time = time; - this.emit("save-to-disk", time); - } - const dbg = this.dbg("handle_syncstring_save_state"); - dbg( - `state='${state}', this.syncstring_save_state='${this.syncstring_save_state}', this.state='${this.state}'`, - ); - if ( - this.state === "ready" && - (await this.isFileServer()) && - this.syncstring_save_state !== "requested" && - state === "requested" - ) { - this.syncstring_save_state = state; // only used in the if above - dbg("requesting save to disk -- calling save_to_disk"); - // state just changed to requesting a save to disk... - // so we do it (unless of course syncstring is still - // being initialized). - try { - // Uncomment the following to test simulating a - // random failure in save_to_disk: - // if (Math.random() < 0.5) throw Error("CHAOS MONKEY!"); // FOR TESTING ONLY. - await this.save_to_disk(); - } catch (err) { - // CRITICAL: we must unset this.syncstring_save_state (and set the save state); - // otherwise, it stays as "requested" and this if statement would never get - // run again, thus completely breaking saving this doc to disk. - // It is normal behavior that *sometimes* this.save_to_disk might - // throw an exception, e.g., if the file is temporarily deleted - // or save it called before everything is initialized, or file - // is temporarily set readonly, or maybe there is a file system error. - // Of course, the finally below will also take care of this. However, - // it's nice to record the error here. - this.syncstring_save_state = "done"; - await this.set_save({ state: "done", error: `${err}` }); - dbg(`ERROR saving to disk in handle_syncstring_save_state-- ${err}`); - } finally { - // No matter what, after the above code is run, - // the save state in the table better be "done". - // We triple check that here, though of course - // we believe the logic in save_to_disk and above - // should always accomplish this. - dbg("had to set the state to done in finally block"); - if ( - this.state === "ready" && - (this.syncstring_save_state != "done" || - this.syncstring_table_get_one().getIn(["save", "state"]) != "done") - ) { - this.syncstring_save_state = "done"; - await this.set_save({ state: "done", error: "" }); - } - } - } - }; - private handle_syncstring_update = async (): Promise => { if (this.state === "closed") { return; @@ -2530,10 +2147,6 @@ export class SyncDoc extends EventEmitter { const data = this.syncstring_table_get_one(); const x: any = data != null ? data.toJS() : undefined; - if (x != null && x.save != null) { - this.handle_syncstring_save_state(x.save.state, x.save.time); - } - dbg(JSON.stringify(x)); if (x == null || x.users == null) { dbg("new_document"); @@ -2622,147 +2235,9 @@ export class SyncDoc extends EventEmitter { this.emit("metadata-change"); }; - private initFileWatcher = async (): Promise => { - if (this.fs != null) { - return await this.fsInitFileWatcher(); - } - - if (!(await this.isFileServer())) { - // ensures we are NOT watching anything - await this.update_watch_path(); - return; - } - - // If path isn't being properly watched, make it so. - if (this.watch_path !== this.path) { - await this.update_watch_path(this.path); - } - - await this.pending_save_to_disk(); - }; - - private pending_save_to_disk = async (): Promise => { - this.assert_table_is_ready("syncstring"); - if (!(await this.isFileServer())) { - return; - } - - const x = this.syncstring_table.get_one(); - // Check if there is a pending save-to-disk that is needed. - if (x != null && x.getIn(["save", "state"]) === "requested") { - try { - await this.save_to_disk(); - } catch (err) { - const dbg = this.dbg("pending_save_to_disk"); - dbg(`ERROR saving to disk in pending_save_to_disk -- ${err}`); - } - } - }; - - private update_watch_path = async (path?: string): Promise => { - if (this.fs != null) { - return; - } - const dbg = this.dbg("update_watch_path"); - if (this.file_watcher != null) { - // clean up - dbg("close"); - this.file_watcher.close(); - delete this.file_watcher; - delete this.watch_path; - } - if (path != null && this.client.is_deleted(path, this.project_id)) { - dbg(`not setting up watching since "${path}" is explicitly deleted`); - return; - } - if (path == null) { - dbg("not opening another watcher since path is null"); - this.watch_path = path; - return; - } - if (this.watch_path != null) { - // this case is impossible since we deleted it above if it is was defined. - dbg("watch_path already defined"); - return; - } - dbg("opening watcher..."); - if (this.state === "closed") { - throw Error("must not be closed"); - } - if (this.client.path_exists == null) { - throw Error("legacy clients must define path_exists"); - } - this.watch_path = path; - try { - if (!(await callback2(this.client.path_exists, { path }))) { - if (this.client.is_deleted(path, this.project_id)) { - dbg(`not setting up watching since "${path}" is explicitly deleted`); - return; - } - // path does not exist - dbg( - `write '${path}' to disk from syncstring in-memory database version`, - ); - const data = this.to_str(); - await callback2(this.client.write_file, { path, data }); - dbg(`wrote '${path}' to disk`); - } - } catch (err) { - // This can happen, e.g, if path is read only. - dbg(`could NOT write '${path}' to disk -- ${err}`); - await this.update_if_file_is_read_only(); - // In this case, can't really setup a file watcher. - return; - } - - dbg("now requesting to watch file"); - this.file_watcher = this.client.watch_file({ path }); - this.file_watcher.on("change", this.handle_file_watcher_change); - this.file_watcher.on("delete", this.handle_file_watcher_delete); - this.setupReadOnlyTimer(); - }; - - private setupReadOnlyTimer = () => { - if (this.read_only_timer) { - clearInterval(this.read_only_timer as any); - this.read_only_timer = 0; - } - this.read_only_timer = ( - setInterval(this.update_if_file_is_read_only, READ_ONLY_CHECK_INTERVAL_MS) - ); - }; - - private handle_file_watcher_change = async (ctime: Date): Promise => { - const dbg = this.dbg("handle_file_watcher_change"); - const time: number = ctime.valueOf(); - dbg( - `file_watcher: change, ctime=${time}, this.save_to_disk_start_ctime=${this.save_to_disk_start_ctime}, this.save_to_disk_end_ctime=${this.save_to_disk_end_ctime}`, - ); - if ( - this.save_to_disk_start_ctime == null || - (this.save_to_disk_end_ctime != null && - time - this.save_to_disk_end_ctime >= RECENT_SAVE_TO_DISK_MS) - ) { - // Either we never saved to disk, or the last attempt - // to save was at least RECENT_SAVE_TO_DISK_MS ago, and it finished, - // so definitely this change event was not caused by it. - dbg("load_from_disk since no recent save to disk"); - await this.readFile(); - return; - } - }; - - private handle_file_watcher_delete = async (): Promise => { - this.assert_is_ready("handle_file_watcher_delete"); - const dbg = this.dbg("handle_file_watcher_delete"); - dbg("delete: set_deleted and closing"); - await this.client.set_deleted(this.path, this.project_id); - this.close(); - }; - - private fsLoadFromDisk = async (): Promise => { + readFile = reuseInFlight(async (): Promise => { if (this.fs == null) throw Error("bug"); - const dbg = this.dbg("fsLoadFromDisk"); + const dbg = this.dbg("readFile"); let size: number; let contents; @@ -2786,98 +2261,15 @@ export class SyncDoc extends EventEmitter { await this.save(); this.emit("after-change"); return size; - }; - - readFile = reuseInFlight(async (): Promise => { - if (this.fs != null) { - return await this.fsLoadFromDisk(); - } - if (this.client.path_exists == null) { - throw Error("legacy clients must define path_exists"); - } - const path = this.path; - const dbg = this.dbg("load_from_disk"); - dbg(); - const exists: boolean = await callback2(this.client.path_exists, { path }); - let size: number; - if (!exists) { - dbg("file no longer exists -- setting to blank"); - size = 0; - this.from_str(""); - } else { - dbg("file exists"); - await this.update_if_file_is_read_only(); - - const data = await callback2(this.client.path_read, { - path, - maxsize_MB: MAX_FILE_SIZE_MB, - }); - - size = data.length; - dbg(`got it -- length=${size}`); - this.from_str(data); - this.commit(); - // we also know that this is the version on disk, so we update the hash - await this.set_save({ - state: "done", - error: "", - hash: hash_string(data), - }); - } - // save new version to database, which we just set via from_str. - await this.save(); - return size; }); - private set_save = async (save: { - state: string; - error: string; - hash?: number; - expected_hash?: number; - time?: number; - }): Promise => { - this.assert_table_is_ready("syncstring"); - // set timestamp of when the save happened; this can be useful - // for coordinating running code, etc.... and is just generally useful. - const cur = this.syncstring_table_get_one().toJS()?.save; - if (cur != null) { - if ( - cur.state == save.state && - cur.error == save.error && - cur.hash == (save.hash ?? cur.hash) && - cur.expected_hash == (save.expected_hash ?? cur.expected_hash) && - cur.time == (save.time ?? cur.time) - ) { - // no genuine change, so no point in wasting cycles on updating. - return; - } - } - if (!save.time) { - save.time = Date.now(); - } - await this.set_syncstring_table({ save }); - }; - - private set_read_only = async (read_only: boolean): Promise => { - this.assert_table_is_ready("syncstring"); - await this.set_syncstring_table({ read_only }); - }; - is_read_only = (): boolean => { - this.assert_table_is_ready("syncstring"); + // [ ] TODO return this.syncstring_table_get_one().get("read_only"); }; wait_until_read_only_known = async (): Promise => { - await this.wait_until_ready(); - function read_only_defined(t: SyncTable): boolean { - const x = t.get_one(); - if (x == null) { - return false; - } - return x.get("read_only") != null; - } - await this.syncstring_table.wait(read_only_defined, 5 * 60); + // [ ] TODO }; /* Returns true if the current live version of this document has @@ -2888,30 +2280,15 @@ export class SyncDoc extends EventEmitter { commited to the database yet. Returns *undefined* if initialization not even done yet. */ has_unsaved_changes = (): boolean | undefined => { - if (this.state !== "ready") { - return; - } - if (this.fs != null) { - return this.fsHasUnsavedChanges(); - } - const dbg = this.dbg("has_unsaved_changes"); - try { - return this.hash_of_saved_version() !== this.hash_of_live_version(); - } catch (err) { - dbg( - "exception computing hash_of_saved_version and hash_of_live_version", - err, - ); - // This could happen, e.g. when syncstring_table isn't connected - // in some edge case. Better to just say we don't know then crash - // everything. See https://github.com/sagemathinc/cocalc/issues/3577 + if (!this.isReady()) { return; } + return this.hasUnsavedChanges(); }; // Returns hash of last version saved to disk (as far as we know). hash_of_saved_version = (): number | undefined => { - if (this.state !== "ready") { + if (!this.isReady()) { return; } return this.syncstring_table_get_one().getIn(["save", "hash"]) as @@ -2924,7 +2301,7 @@ export class SyncDoc extends EventEmitter { (TODO: write faster version of this for syncdb, which avoids converting to a string, which is a waste of time.) */ hash_of_live_version = (): number | undefined => { - if (this.state !== "ready") { + if (!this.isReady()) { return; } return hash_string(this.doc.to_str()); @@ -2937,7 +2314,7 @@ export class SyncDoc extends EventEmitter { the user to close their browser. */ has_uncommitted_changes = (): boolean => { - if (this.state !== "ready") { + if (!this.isReady()) { return false; } return this.patches_table.has_uncommitted_changes(); @@ -2983,12 +2360,12 @@ export class SyncDoc extends EventEmitter { }; private lastDiskValue: string | undefined = undefined; - fsHasUnsavedChanges = (): boolean => { + private hasUnsavedChanges = (): boolean => { return this.lastDiskValue != this.to_str(); }; - fsSaveToDisk = async () => { - const dbg = this.dbg("fsSaveToDisk"); + writeFile = async () => { + const dbg = this.dbg("writeFile"); if (this.client.is_deleted(this.path, this.project_id)) { dbg("not saving to disk because deleted"); return; @@ -2998,11 +2375,11 @@ export class SyncDoc extends EventEmitter { throw Error("bug"); } const value = this.to_str(); - // tell watcher not to fire any change events for a little time, + // include {ignore:true} with events for this long, // so no clients waste resources loading in response to us saving // to disk. try { - await this.fsFileWatcher?.ignore(2000); + await this.fileWatcher?.ignore(2000); } catch { // not a big problem if we can't ignore (e.g., this happens potentially // after deleting the file or if file doesn't exist) @@ -3018,7 +2395,7 @@ export class SyncDoc extends EventEmitter { /* Initiates a save of file to disk, then waits for the state to change. */ save_to_disk = reuseInFlight(async (): Promise => { - if (this.state != "ready") { + if (!this.isReady()) { // We just make save_to_disk a successful // no operation, if the document is either // closed or hasn't finished opening, since @@ -3029,67 +2406,8 @@ export class SyncDoc extends EventEmitter { return; } - if (this.fs != null) { - this.commit(); - await this.fsSaveToDisk(); - this.update_has_unsaved_changes(); - return; - } - - const dbg = this.dbg("save_to_disk"); - if (this.client.is_deleted(this.path, this.project_id)) { - dbg("not saving to disk because deleted"); - await this.set_save({ state: "done", error: "" }); - return; - } - - // Make sure to include changes to the live document. - // A side effect of save if we didn't do this is potentially - // discarding them, which is obviously not good. this.commit(); - - dbg("initiating the save"); - if (!this.has_unsaved_changes()) { - dbg("no unsaved changes, so don't save"); - // CRITICAL: this optimization is assumed by - // autosave, etc. - await this.set_save({ state: "done", error: "" }); - return; - } - - if (this.is_read_only()) { - dbg("read only, so can't save to disk"); - // save should fail if file is read only and there are changes - throw Error("can't save readonly file with changes to disk"); - } - - // First make sure any changes are saved to the database. - // One subtle case where this matters is that loading a file - // with \r's into codemirror changes them to \n... - if (!(await this.isFileServer())) { - dbg("browser client -- sending any changes over network"); - await this.save(); - dbg("save done; now do actual save to the *disk*."); - this.assert_is_ready("save_to_disk - after save"); - } - - try { - await this.save_to_disk_aux(); - } catch (err) { - if (this.state != "ready") return; - const error = `save to disk failed -- ${err}`; - dbg(error); - if (await this.isFileServer()) { - this.set_save({ error, state: "done" }); - } - } - if (this.state != "ready") return; - - if (!(await this.isFileServer())) { - dbg("now wait for the save to disk to finish"); - this.assert_is_ready("save_to_disk - waiting to finish"); - await this.wait_for_save_to_disk_done(); - } + await this.writeFile(); this.update_has_unsaved_changes(); }); @@ -3107,7 +2425,7 @@ export class SyncDoc extends EventEmitter { }; private update_has_unsaved_changes = (): void => { - if (this.state != "ready") { + if (!this.isReady()) { // This can happen, since this is called by a debounced function. // Make it a no-op in case we're not ready. // See https://github.com/sagemathinc/cocalc/issues/3577 @@ -3120,174 +2438,6 @@ export class SyncDoc extends EventEmitter { } }; - // wait for save.state to change state. - private wait_for_save_to_disk_done = async (): Promise => { - const dbg = this.dbg("wait_for_save_to_disk_done"); - dbg(); - function until(table): boolean { - const done = table.get_one().getIn(["save", "state"]) === "done"; - dbg("checking... done=", done); - return done; - } - - let last_err: string | undefined = undefined; - const f = async () => { - dbg("f"); - if ( - this.state != "ready" || - this.client.is_deleted(this.path, this.project_id) - ) { - dbg("not ready or deleted - no longer trying to save."); - return; - } - try { - dbg("waiting until done..."); - await this.syncstring_table.wait(until, 15); - } catch (err) { - dbg("timed out after 15s"); - throw Error("timed out"); - } - if ( - this.state != "ready" || - this.client.is_deleted(this.path, this.project_id) - ) { - dbg("not ready or deleted - no longer trying to save."); - return; - } - const err = this.syncstring_table_get_one().getIn(["save", "error"]) as - | string - | undefined; - if (err) { - dbg("error", err); - last_err = err; - throw Error(err); - } - dbg("done, with no error."); - last_err = undefined; - return; - }; - await retry_until_success({ - f, - max_tries: 8, - desc: "wait_for_save_to_disk_done", - }); - if ( - this.state != "ready" || - this.client.is_deleted(this.path, this.project_id) - ) { - return; - } - if (last_err && typeof this.client.log_error != null) { - this.client.log_error?.({ - string_id: this.string_id, - path: this.path, - project_id: this.project_id, - error: `Error saving file -- ${last_err}`, - }); - } - }; - - /* Auxiliary function 2 for saving to disk: - If this is associated with - a project and has a filename. - A user (web browsers) sets the save state to requested. - The project sets the state to saving, does the save - to disk, then sets the state to done. - */ - private save_to_disk_aux = async (): Promise => { - this.assert_is_ready("save_to_disk_aux"); - - if (!(await this.isFileServer())) { - return await this.save_to_disk_non_filesystem_owner(); - } - - try { - return await this.save_to_disk_filesystem_owner(); - } catch (err) { - this.emit("save_to_disk_filesystem_owner", err); - throw err; - } - }; - - private save_to_disk_non_filesystem_owner = async (): Promise => { - this.assert_is_ready("save_to_disk_non_filesystem_owner"); - - if (!this.has_unsaved_changes()) { - /* Browser client has no unsaved changes, - so don't need to save -- - CRITICAL: this optimization is assumed by autosave. - */ - return; - } - const x = this.syncstring_table.get_one(); - if (x != null && x.getIn(["save", "state"]) === "requested") { - // Nothing to do -- save already requested, which is - // all the browser client has to do. - return; - } - - // string version of this doc - const data: string = this.to_str(); - const expected_hash = hash_string(data); - await this.set_save({ state: "requested", error: "", expected_hash }); - }; - - private save_to_disk_filesystem_owner = async (): Promise => { - this.assert_is_ready("save_to_disk_filesystem_owner"); - const dbg = this.dbg("save_to_disk_filesystem_owner"); - - // check if on-disk version is same as in memory, in - // which case no save is needed. - const data = this.to_str(); // string version of this doc - const hash = hash_string(data); - dbg("hash = ", hash); - - /* - // TODO: put this consistency check back in (?). - const expected_hash = this.syncstring_table - .get_one() - .getIn(["save", "expected_hash"]); - */ - - if (hash === this.hash_of_saved_version()) { - // No actual save to disk needed; still we better - // record this fact in table in case it - // isn't already recorded - this.set_save({ state: "done", error: "", hash }); - return; - } - - const path = this.path; - if (!path) { - const err = "cannot save without path"; - this.set_save({ state: "done", error: err }); - throw Error(err); - } - - dbg("project - write to disk file", path); - // set window to slightly earlier to account for clock - // imprecision. - // Over an sshfs mount, all stats info is **rounded down - // to the nearest second**, which this also takes care of. - this.save_to_disk_start_ctime = Date.now() - 1500; - this.save_to_disk_end_ctime = undefined; - try { - await callback2(this.client.write_file, { path, data }); - this.assert_is_ready("save_to_disk_filesystem_owner -- after write_file"); - const stat = await callback2(this.client.path_stat, { path }); - this.assert_is_ready("save_to_disk_filesystem_owner -- after path_state"); - this.save_to_disk_end_ctime = stat.ctime.valueOf() + 1500; - this.set_save({ - state: "done", - error: "", - hash: hash_string(data), - }); - } catch (err) { - this.set_save({ state: "done", error: JSON.stringify(err) }); - throw err; - } - }; - /* When the underlying synctable that defines the state of the document changes due to new remote patches, this @@ -3513,11 +2663,11 @@ export class SyncDoc extends EventEmitter { } }, 60000); - private fsLoadFromDiskDebounced = asyncDebounce( + private readFileDebounced = asyncDebounce( async () => { try { this.emit("handle-file-change"); - await this.fsLoadFromDisk(); + await this.readFile(); } catch {} }, WATCH_DEBOUNCE, @@ -3527,38 +2677,38 @@ export class SyncDoc extends EventEmitter { }, ); - private fsFileWatcher?: any; - private fsInitFileWatcher = async () => { + private fileWatcher?: any; + private initFileWatcher = async () => { if (this.fs == null) { throw Error("this.fs must be defined"); } // use this.fs interface to watch path for changes -- we try once: try { - this.fsFileWatcher = await this.fs.watch(this.path, { unique: true }); + this.fileWatcher = await this.fs.watch(this.path, { unique: true }); } catch {} if (this.isClosed()) return; // not closed -- so if above succeeds we start watching. // if not, we loop waiting for file to be created so we can watch it (async () => { - if (this.fsFileWatcher != null) { + if (this.fileWatcher != null) { this.emit("watching"); - for await (const { eventType, ignore } of this.fsFileWatcher) { + for await (const { eventType, ignore } of this.fileWatcher) { if (this.isClosed()) return; // we don't know what's on disk anymore, this.lastDiskValue = undefined; //console.log("got change", eventType); if (!ignore) { - this.fsLoadFromDiskDebounced(); + this.readFileDebounced(); } if (eventType == "rename") { break; } } // check if file was deleted - this.fsCloseIfFileDeleted(); - this.fsFileWatcher?.close(); - delete this.fsFileWatcher; + this.closeIfFileDeleted(); + this.fileWatcher?.close(); + delete this.fileWatcher; } // start a new watcher since file descriptor probably changed or maybe file deleted await delay(this.watchRecreateWait ?? WATCH_RECREATE_WAIT); @@ -3566,7 +2716,7 @@ export class SyncDoc extends EventEmitter { async () => { if (this.isClosed()) return true; try { - await this.fsInitFileWatcher(); + await this.initFileWatcher(); return true; } catch { return false; @@ -3577,15 +2727,15 @@ export class SyncDoc extends EventEmitter { })(); }; - private fsCloseFileWatcher = () => { - this.fsFileWatcher?.close(); - delete this.fsFileWatcher; + private closeFileWatcher = () => { + this.fileWatcher?.close(); + delete this.fileWatcher; }; // returns true if file definitely exists right now, // false if it definitely does not, and throws exception otherwise, // e.g., network error. - private fsFileExists = async (): Promise => { + private fileExists = async (): Promise => { if (this.fs == null) { throw Error("bug -- fs must be defined"); } @@ -3601,7 +2751,7 @@ export class SyncDoc extends EventEmitter { } }; - private fsCloseIfFileDeleted = async () => { + private closeIfFileDeleted = async () => { if (this.isClosed()) return; if (this.fs == null) { throw Error("bug -- fs must be defined"); @@ -3610,7 +2760,7 @@ export class SyncDoc extends EventEmitter { const threshold = this.deletedThreshold ?? DELETED_THRESHOLD; while (true) { try { - if (await this.fsFileExists()) { + if (await this.fileExists()) { // file definitely exists right now. return; } From 1dc407e01a8d25dc17ca8116c3c0f59084908630 Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 22 Jul 2025 22:47:12 +0000 Subject: [PATCH 063/270] add find command to fs api - this is one thing that node's fs does not have, but this is done in a highly flexible, but safe and minimal way. --- .../conat/files/test/local-path.test.ts | 10 ++- .../backend/files/sandbox/find.test.ts | 42 ++++++++++ src/packages/backend/files/sandbox/find.ts | 83 +++++++++++++++++++ src/packages/backend/files/sandbox/index.ts | 25 +++--- src/packages/conat/files/fs.ts | 12 +++ 5 files changed, 159 insertions(+), 13 deletions(-) create mode 100644 src/packages/backend/files/sandbox/find.test.ts create mode 100644 src/packages/backend/files/sandbox/find.ts diff --git a/src/packages/backend/conat/files/test/local-path.test.ts b/src/packages/backend/conat/files/test/local-path.test.ts index 30c8d04476..2c49add752 100644 --- a/src/packages/backend/conat/files/test/local-path.test.ts +++ b/src/packages/backend/conat/files/test/local-path.test.ts @@ -103,17 +103,25 @@ describe("use all the standard api functions of fs", () => { } }); + let fire; it("readdir works", async () => { await fs.mkdir("dirtest"); for (let i = 0; i < 5; i++) { await fs.writeFile(`dirtest/${i}`, `${i}`); } - const fire = "🔥.txt"; + fire = "🔥.txt"; await fs.writeFile(join("dirtest", fire), "this is ️‍🔥!"); const v = await fs.readdir("dirtest"); expect(v).toEqual(["0", "1", "2", "3", "4", fire]); }); + it("use the find command instead of readdir", async () => { + const { stdout } = await fs.find("dirtest", "%f\n"); + const v = stdout.toString().trim().split("\n"); + // output of find is NOT in alphabetical order: + expect(new Set(v)).toEqual(new Set(["0", "1", "2", "3", "4", fire])); + }); + it("realpath works", async () => { await fs.writeFile("file0", "file0"); await fs.symlink("file0", "file1"); diff --git a/src/packages/backend/files/sandbox/find.test.ts b/src/packages/backend/files/sandbox/find.test.ts new file mode 100644 index 0000000000..ca725e3569 --- /dev/null +++ b/src/packages/backend/files/sandbox/find.test.ts @@ -0,0 +1,42 @@ +import find from "./find"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +let tempDir; +beforeAll(async () => { + tempDir = await mkdtemp(join(tmpdir(), "cocalc")); +}); +afterAll(async () => { + await rm(tempDir, { force: true, recursive: true }); +}); + +describe("find files", () => { + it("directory starts empty", async () => { + const { stdout, truncated } = await find(tempDir, "%f\n"); + expect(stdout.length).toBe(0); + expect(truncated).toBe(false); + }); + + it("create a file and see it appears in find", async () => { + await writeFile(join(tempDir, "a.txt"), "hello"); + const { stdout, truncated } = await find(tempDir, "%f\n"); + expect(truncated).toBe(false); + expect(stdout.toString()).toEqual("a.txt\n"); + }); + + // this is NOT a great test, unfortunately. + const count = 10000; + it(`hopefully exceed the timeout by creating ${count} files`, async () => { + for (let i = 0; i < count; i++) { + await writeFile(join(tempDir, `${i}`), ""); + } + const t = Date.now(); + const { stdout, truncated } = await find(tempDir, "%f\n", 2); + expect(truncated).toBe(true); + expect(Date.now() - t).toBeGreaterThan(1); + + const { stdout: stdout2 } = await find(tempDir, "%f\n"); + expect(stdout2.length).toBeGreaterThan(stdout.length); + }); +}); diff --git a/src/packages/backend/files/sandbox/find.ts b/src/packages/backend/files/sandbox/find.ts new file mode 100644 index 0000000000..edc7047c37 --- /dev/null +++ b/src/packages/backend/files/sandbox/find.ts @@ -0,0 +1,83 @@ +import { spawn } from "node:child_process"; + +export default async function find( + path: string, + printf: string, + timeout?: number, +): Promise<{ + // the output as a Buffer (not utf8, since it could have arbitrary file names!) + stdout: Buffer; + // truncated is true if the timeout gets hit. + truncated: boolean; +}> { + if (!path) { + throw Error("path must be specified"); + } + if (!printf) { + throw Error("printf must be specified"); + } + return new Promise((resolve, reject) => { + const chunks: Buffer[] = []; + let truncated = false; + + const args = [ + "-P", // Never follow symlinks (security) + path, // Search path + "-maxdepth", + "1", + "-mindepth", + "1", + "-printf", + printf, + ]; + + // Spawn find with minimal, fixed arguments + const child = spawn("find", args, { + stdio: ["ignore", "pipe", "pipe"], + env: {}, // Empty environment (security) + shell: false, // No shell interpretation (security) + }); + + let timer; + if (timeout) { + timer = setTimeout(() => { + if (!truncated) { + truncated = true; + child.kill("SIGTERM"); + } + }, timeout); + } else { + timer = null; + } + + child.stdout.on("data", (chunk: Buffer) => { + chunks.push(chunk); + }); + + let stderr = ""; + child.stderr.on("data", (chunk: Buffer) => { + stderr += chunk.toString(); + }); + + // Handle completion + child.on("error", (error) => { + if (timer) { + clearTimeout(timer); + } + reject(error); + }); + + child.on("exit", (code) => { + if (timer) { + clearTimeout(timer); + } + + if (code !== 0 && !truncated) { + reject(new Error(`find exited with code ${code}: ${stderr}`)); + return; + } + + resolve({ stdout: Buffer.concat(chunks), truncated }); + }); + }); +} diff --git a/src/packages/backend/files/sandbox/index.ts b/src/packages/backend/files/sandbox/index.ts index 27b4a800ad..979aca2a4c 100644 --- a/src/packages/backend/files/sandbox/index.ts +++ b/src/packages/backend/files/sandbox/index.ts @@ -64,12 +64,16 @@ import { } from "node:fs/promises"; import { watch } from "node:fs"; import { exists } from "@cocalc/backend/misc/async-utils-node"; -import { type DirectoryListingEntry } from "@cocalc/util/types"; -import getListing from "@cocalc/backend/get-listing"; import { join, resolve } from "path"; import { replace_all } from "@cocalc/util/misc"; import { EventIterator } from "@cocalc/util/event-iterator"; import { type WatchOptions } from "@cocalc/conat/files/watch"; +import find from "./find"; + +// max time a user find request can run -- this can cause excessive +// load on a server if there were a directory with a massive number of files, +// so must be limited. +const FIND_TIMEOUT = 3000; export class SandboxedFilesystem { // path should be the path to a FOLDER on the filesystem (not a file) @@ -151,6 +155,13 @@ export class SandboxedFilesystem { return await exists(await this.safeAbsPath(path)); }; + find = async ( + path: string, + printf: string, + ): Promise<{ stdout: Buffer; truncated: boolean }> => { + return await find(await this.safeAbsPath(path), printf, FIND_TIMEOUT); + }; + // hard link link = async (existingPath: string, newPath: string) => { return await link( @@ -159,16 +170,6 @@ export class SandboxedFilesystem { ); }; - ls = async ( - path: string, - { hidden, limit }: { hidden?: boolean; limit?: number } = {}, - ): Promise => { - return await getListing(await this.safeAbsPath(path), hidden, { - limit, - home: "/", - }); - }; - lstat = async (path: string) => { return await lstat(await this.safeAbsPath(path)); }; diff --git a/src/packages/conat/files/fs.ts b/src/packages/conat/files/fs.ts index 80b96ecccd..0f6746ebec 100644 --- a/src/packages/conat/files/fs.ts +++ b/src/packages/conat/files/fs.ts @@ -35,6 +35,15 @@ export interface Filesystem { writeFile: (path: string, data: string | Buffer) => Promise; // todo: typing watch: (path: string, options?) => Promise; + + // We add very little to the Filesystem api, but we have to add + // a sandboxed "find" command, since it is a 1-call way to get + // arbitrary directory listing info. + // find -P {path} -maxdepth 1 -mindepth 1 -printf {printf} + find: ( + path: string, + printf: string, + ) => Promise<{ stdout: Buffer; truncated: boolean }>; } interface IStats { @@ -129,6 +138,9 @@ export async function fsServer({ service, fs, client }: Options) { async exists(path: string) { return await (await fs(this.subject)).exists(path); }, + async find(path: string, printf: string) { + return await (await fs(this.subject)).find(path, printf); + }, async link(existingPath: string, newPath: string) { await (await fs(this.subject)).link(existingPath, newPath); }, From c4382823dcedc819f61496e0ecacc94e3cfd480d Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 22 Jul 2025 23:46:01 +0000 Subject: [PATCH 064/270] use reuseInFlight instead of 1000's of once listeners at once. - so we don't get that scary eventemitter warning. - I'm not technically sure this is actually better, but probably... --- src/packages/conat/core/cluster.ts | 3 +-- src/packages/conat/core/patterns.ts | 9 ++++++++- src/packages/conat/core/server.ts | 6 +++--- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/src/packages/conat/core/cluster.ts b/src/packages/conat/core/cluster.ts index 7db83c03c6..863f2cd943 100644 --- a/src/packages/conat/core/cluster.ts +++ b/src/packages/conat/core/cluster.ts @@ -7,7 +7,6 @@ import { type StickyUpdate, } from "@cocalc/conat/core/server"; import type { DStream } from "@cocalc/conat/sync/dstream"; -import { once } from "@cocalc/util/async-utils"; import { server as createPersistServer } from "@cocalc/conat/persist/server"; import { getLogger } from "@cocalc/conat/client"; import { hash_string } from "@cocalc/util/misc"; @@ -165,7 +164,7 @@ class ClusterLink { if (Date.now() - start >= timeout) { throw Error("timeout"); } - await once(this.interest, "change"); + await this.interest.waitForChange(); if ((this.state as any) == "closed" || signal?.aborted) { return false; } diff --git a/src/packages/conat/core/patterns.ts b/src/packages/conat/core/patterns.ts index 79eada9e5e..708df2afde 100644 --- a/src/packages/conat/core/patterns.ts +++ b/src/packages/conat/core/patterns.ts @@ -2,6 +2,8 @@ import { isEqual } from "lodash"; import { getLogger } from "@cocalc/conat/client"; import { EventEmitter } from "events"; import { hash_string } from "@cocalc/util/misc"; +import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; +import { once } from "@cocalc/util/async-utils"; type Index = { [pattern: string]: Index | string }; @@ -13,9 +15,14 @@ export class Patterns extends EventEmitter { constructor() { super(); - this.setMaxListeners(1000); + this.setMaxListeners(100); } + // wait until one single change event fires. Throws an error if this gets closed first. + waitForChange = reuseInFlight(async (timeout?) => { + await once(this, "change", timeout); + }); + close = () => { this.emit("closed"); this.patterns = {}; diff --git a/src/packages/conat/core/server.ts b/src/packages/conat/core/server.ts index dc87dcf191..e41135d2cf 100644 --- a/src/packages/conat/core/server.ts +++ b/src/packages/conat/core/server.ts @@ -53,7 +53,7 @@ import { import { Patterns } from "./patterns"; import { is_array } from "@cocalc/util/misc"; import { UsageMonitor } from "@cocalc/conat/monitor/usage"; -import { once, until } from "@cocalc/util/async-utils"; +import { until } from "@cocalc/util/async-utils"; import { clusterLink, type ClusterLink, @@ -1703,8 +1703,8 @@ export class ConatServer extends EventEmitter { throw Error("timeout"); } try { - // if signal is set only wait for the change for up to 1 second. - await once(this.interest, "change", signal != null ? 1000 : undefined); + // if signal is set, only wait for the change for up to 1 second. + await this.interest.waitForChange(signal != null ? 1000 : undefined); } catch { continue; } From 80224e933a472116189c39aec0d16d0007f385d3 Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 23 Jul 2025 00:17:27 +0000 Subject: [PATCH 065/270] fs sandbox: support for readonly and unsafe; ability to find files matching patterns --- .../conat/files/test/local-path.test.ts | 2 +- .../backend/files/sandbox/find.test.ts | 26 +++++- src/packages/backend/files/sandbox/find.ts | 88 ++++++++++++++++-- src/packages/backend/files/sandbox/index.ts | 89 +++++++++++++++++-- .../backend/files/sandbox/sandbox.test.ts | 88 +++++++++++++++++- src/packages/conat/files/fs.ts | 30 ++++++- 6 files changed, 304 insertions(+), 19 deletions(-) diff --git a/src/packages/backend/conat/files/test/local-path.test.ts b/src/packages/backend/conat/files/test/local-path.test.ts index 2c49add752..a05a7b297a 100644 --- a/src/packages/backend/conat/files/test/local-path.test.ts +++ b/src/packages/backend/conat/files/test/local-path.test.ts @@ -313,7 +313,7 @@ describe("use all the standard api functions of fs", () => { }); }); -describe("security: dangerous symlinks can't be followed", () => { +describe("SECURITY CHECKS: dangerous symlinks can't be followed", () => { let server; let tempDir; it("creates the simple fileserver service", async () => { diff --git a/src/packages/backend/files/sandbox/find.test.ts b/src/packages/backend/files/sandbox/find.test.ts index ca725e3569..f57ff4cd48 100644 --- a/src/packages/backend/files/sandbox/find.test.ts +++ b/src/packages/backend/files/sandbox/find.test.ts @@ -1,5 +1,5 @@ import find from "./find"; -import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -25,14 +25,34 @@ describe("find files", () => { expect(stdout.toString()).toEqual("a.txt\n"); }); + it("find files matching a given pattern", async () => { + await writeFile(join(tempDir, "pattern"), ""); + await mkdir(join(tempDir, "blue")); + await writeFile(join(tempDir, "blue", "Patton"), ""); + const { stdout } = await find(tempDir, "%f\n", { + expression: { type: "iname", pattern: "patt*" }, + }); + const v = stdout.toString().trim().split("\n"); + expect(new Set(v)).toEqual(new Set(["pattern"])); + }); + + it("find file in a subdirectory too", async () => { + const { stdout } = await find(tempDir, "%P\n", { + recursive: true, + expression: { type: "iname", pattern: "patt*" }, + }); + const w = stdout.toString().trim().split("\n"); + expect(new Set(w)).toEqual(new Set(["pattern", "blue/Patton"])); + }); + // this is NOT a great test, unfortunately. - const count = 10000; + const count = 5000; it(`hopefully exceed the timeout by creating ${count} files`, async () => { for (let i = 0; i < count; i++) { await writeFile(join(tempDir, `${i}`), ""); } const t = Date.now(); - const { stdout, truncated } = await find(tempDir, "%f\n", 2); + const { stdout, truncated } = await find(tempDir, "%f\n", { timeout: 0.1 }); expect(truncated).toBe(true); expect(Date.now() - t).toBeGreaterThan(1); diff --git a/src/packages/backend/files/sandbox/find.ts b/src/packages/backend/files/sandbox/find.ts index edc7047c37..90cd67c9bc 100644 --- a/src/packages/backend/files/sandbox/find.ts +++ b/src/packages/backend/files/sandbox/find.ts @@ -1,9 +1,11 @@ import { spawn } from "node:child_process"; +import type { FindOptions, FindExpression } from "@cocalc/conat/files/fs"; +export type { FindOptions, FindExpression }; export default async function find( path: string, printf: string, - timeout?: number, + { timeout = 0, recursive, expression }: FindOptions = {}, ): Promise<{ // the output as a Buffer (not utf8, since it could have arbitrary file names!) stdout: Buffer; @@ -23,13 +25,25 @@ export default async function find( const args = [ "-P", // Never follow symlinks (security) path, // Search path - "-maxdepth", - "1", "-mindepth", "1", - "-printf", - printf, ]; + if (!recursive) { + args.push("-maxdepth", "1"); + } + + // Add expression if provided + if (expression) { + try { + args.push(...buildFindArgs(expression)); + } catch (error) { + reject(error); + return; + } + } + args.push("-printf", printf); + + //console.log(`find ${args.join(" ")}`); // Spawn find with minimal, fixed arguments const child = spawn("find", args, { @@ -81,3 +95,67 @@ export default async function find( }); }); } + +function buildFindArgs(expr: FindExpression): string[] { + switch (expr.type) { + case "name": + // Validate pattern has no path separators + if (expr.pattern.includes("/")) { + throw new Error("Path separators not allowed in name patterns"); + } + return ["-name", expr.pattern]; + + case "iname": + if (expr.pattern.includes("/")) { + throw new Error("Path separators not allowed in name patterns"); + } + return ["-iname", expr.pattern]; + + case "type": + return ["-type", expr.value]; + + case "size": + // Validate size format (e.g., "10M", "1G", "500k") + if (!/^[0-9]+[kMGTP]?$/.test(expr.value)) { + throw new Error("Invalid size format"); + } + return ["-size", expr.operator + expr.value]; + + case "mtime": + if (!Number.isInteger(expr.days) || expr.days < 0) { + throw new Error("Invalid mtime days"); + } + return ["-mtime", expr.operator + expr.days]; + + case "newer": + // This is risky - would need to validate file path is within sandbox + if (expr.file.includes("..") || expr.file.startsWith("/")) { + throw new Error("Invalid reference file path"); + } + return ["-newer", expr.file]; + + case "and": + return [ + "(", + ...buildFindArgs(expr.left), + "-a", + ...buildFindArgs(expr.right), + ")", + ]; + + case "or": + return [ + "(", + ...buildFindArgs(expr.left), + "-o", + ...buildFindArgs(expr.right), + ")", + ]; + + case "not": + return ["!", ...buildFindArgs(expr.expr)]; + + default: + throw new Error("Unsupported expression type"); + } +} diff --git a/src/packages/backend/files/sandbox/index.ts b/src/packages/backend/files/sandbox/index.ts index 979aca2a4c..b65ba1ff3f 100644 --- a/src/packages/backend/files/sandbox/index.ts +++ b/src/packages/backend/files/sandbox/index.ts @@ -68,18 +68,44 @@ import { join, resolve } from "path"; import { replace_all } from "@cocalc/util/misc"; import { EventIterator } from "@cocalc/util/event-iterator"; import { type WatchOptions } from "@cocalc/conat/files/watch"; -import find from "./find"; +import find, { type FindOptions } from "./find"; // max time a user find request can run -- this can cause excessive // load on a server if there were a directory with a massive number of files, // so must be limited. -const FIND_TIMEOUT = 3000; +const MAX_FIND_TIMEOUT = 3000; + +interface Options { + // unsafeMode -- if true, assume security model where user is running this + // themself, e.g., in a project, so no security is needed at all. + unsafeMode?: boolean; + // readonly -- only allow operations that don't change files + readonly?: boolean; +} + +// If you add any methods below that are NOT for the public api +// be sure to exclude them here! +const INTERNAL_METHODS = new Set([ + "safeAbsPath", + "constructor", + "path", + "unsafeMode", + "readonly", + "assertWritable", +]); export class SandboxedFilesystem { - // path should be the path to a FOLDER on the filesystem (not a file) - constructor(public readonly path: string) { + public readonly unsafeMode: boolean; + public readonly readonly: boolean; + constructor( + // path should be the path to a FOLDER on the filesystem (not a file) + public readonly path: string, + { unsafeMode = false, readonly = false }: Options = {}, + ) { + this.unsafeMode = !!unsafeMode; + this.readonly = !!readonly; for (const f in this) { - if (f == "safeAbsPath" || f == "constructor" || f == "path") { + if (INTERNAL_METHODS.has(f)) { continue; } const orig = this[f]; @@ -99,12 +125,26 @@ export class SandboxedFilesystem { } } + private assertWritable = (path: string) => { + if (this.readonly) { + throw new SandboxError( + `EACCES: permission denied -- read only filesystem, open '${path}'`, + { errno: -13, code: "EACCES", syscall: "open", path }, + ); + } + }; + safeAbsPath = async (path: string): Promise => { if (typeof path != "string") { throw Error(`path must be a string but is of type ${typeof path}`); } // pathInSandbox is *definitely* a path in the sandbox: const pathInSandbox = join(this.path, resolve("/", path)); + + if (this.unsafeMode) { + // not secure -- just convenient. + return pathInSandbox; + } // However, there is still one threat, which is that it could // be a path to an existing link that goes out of the sandbox. So // we resolve to the realpath: @@ -128,10 +168,12 @@ export class SandboxedFilesystem { }; appendFile = async (path: string, data: string | Buffer, encoding?) => { + this.assertWritable(path); return await appendFile(await this.safeAbsPath(path), data, encoding); }; chmod = async (path: string, mode: string | number) => { + this.assertWritable(path); await chmod(await this.safeAbsPath(path), mode); }; @@ -140,10 +182,12 @@ export class SandboxedFilesystem { }; copyFile = async (src: string, dest: string) => { + this.assertWritable(dest); await copyFile(await this.safeAbsPath(src), await this.safeAbsPath(dest)); }; cp = async (src: string, dest: string, options?) => { + this.assertWritable(dest); await cp( await this.safeAbsPath(src), await this.safeAbsPath(dest), @@ -158,12 +202,22 @@ export class SandboxedFilesystem { find = async ( path: string, printf: string, + options?: FindOptions, ): Promise<{ stdout: Buffer; truncated: boolean }> => { - return await find(await this.safeAbsPath(path), printf, FIND_TIMEOUT); + options = { ...options }; + if ( + !this.unsafeMode && + (!options.timeout || options.timeout > MAX_FIND_TIMEOUT) + ) { + options.timeout = MAX_FIND_TIMEOUT; + } + + return await find(await this.safeAbsPath(path), printf, options); }; // hard link link = async (existingPath: string, newPath: string) => { + this.assertWritable(newPath); return await link( await this.safeAbsPath(existingPath), await this.safeAbsPath(newPath), @@ -175,6 +229,7 @@ export class SandboxedFilesystem { }; mkdir = async (path: string, options?) => { + this.assertWritable(path); await mkdir(await this.safeAbsPath(path), options); }; @@ -192,6 +247,7 @@ export class SandboxedFilesystem { }; rename = async (oldPath: string, newPath: string) => { + this.assertWritable(oldPath); await rename( await this.safeAbsPath(oldPath), await this.safeAbsPath(newPath), @@ -199,10 +255,12 @@ export class SandboxedFilesystem { }; rm = async (path: string, options?) => { + this.assertWritable(path); await rm(await this.safeAbsPath(path), options); }; rmdir = async (path: string, options?) => { + this.assertWritable(path); await rmdir(await this.safeAbsPath(path), options); }; @@ -211,6 +269,7 @@ export class SandboxedFilesystem { }; symlink = async (target: string, path: string) => { + this.assertWritable(target); return await symlink( await this.safeAbsPath(target), await this.safeAbsPath(path), @@ -218,10 +277,12 @@ export class SandboxedFilesystem { }; truncate = async (path: string, len?: number) => { + this.assertWritable(path); await truncate(await this.safeAbsPath(path), len); }; unlink = async (path: string) => { + this.assertWritable(path); await unlink(await this.safeAbsPath(path)); }; @@ -230,6 +291,7 @@ export class SandboxedFilesystem { atime: number | string | Date, mtime: number | string | Date, ) => { + this.assertWritable(path); await utimes(await this.safeAbsPath(path), atime, mtime); }; @@ -257,6 +319,21 @@ export class SandboxedFilesystem { }; writeFile = async (path: string, data: string | Buffer) => { + this.assertWritable(path); return await writeFile(await this.safeAbsPath(path), data); }; } + +export class SandboxError extends Error { + code: string; + errno: number; + syscall: string; + path: string; + constructor(mesg: string, { code, errno, syscall, path }) { + super(mesg); + this.code = code; + this.errno = errno; + this.syscall = syscall; + this.path = path; + } +} diff --git a/src/packages/backend/files/sandbox/sandbox.test.ts b/src/packages/backend/files/sandbox/sandbox.test.ts index ebabcb5dba..2676f5bbe4 100644 --- a/src/packages/backend/files/sandbox/sandbox.test.ts +++ b/src/packages/backend/files/sandbox/sandbox.test.ts @@ -1,5 +1,12 @@ import { SandboxedFilesystem } from "@cocalc/backend/files/sandbox"; -import { mkdtemp, mkdir, rm } from "node:fs/promises"; +import { + mkdtemp, + mkdir, + rm, + readFile, + symlink, + writeFile, +} from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "path"; @@ -16,6 +23,7 @@ describe("test using the filesystem sandbox to do a few standard things", () => await fs.writeFile("a", "hi"); const r = await fs.readFile("a", "utf8"); expect(r).toEqual("hi"); + expect(fs.unsafeMode).toBe(false); }); it("truncate file", async () => { @@ -166,6 +174,84 @@ describe("test watching a file and a folder in the sandbox", () => { }); }); +describe("unsafe mode sandbox", () => { + let fs; + it("creates and reads file", async () => { + await mkdir(join(tempDir, "test-unsafe")); + fs = new SandboxedFilesystem(join(tempDir, "test-unsafe"), { + unsafeMode: true, + }); + expect(fs.unsafeMode).toBe(true); + await fs.writeFile("a", "hi"); + const r = await fs.readFile("a", "utf8"); + expect(r).toEqual("hi"); + }); + + it("directly create a dangerous file that is a symlink outside of the sandbox -- this should work", async () => { + await writeFile(join(tempDir, "password"), "s3cr3t"); + await symlink( + join(tempDir, "password"), + join(tempDir, "test-unsafe", "danger"), + ); + const s = await readFile(join(tempDir, "test-unsafe", "danger"), "utf8"); + expect(s).toBe("s3cr3t"); + }); + + it("can **UNSAFELY** read the symlink content via the api", async () => { + expect(await fs.readFile("danger", "utf8")).toBe("s3cr3t"); + }); +}); + +describe("safe mode sandbox", () => { + let fs; + it("creates and reads file", async () => { + await mkdir(join(tempDir, "test-safe")); + fs = new SandboxedFilesystem(join(tempDir, "test-safe"), { + unsafeMode: false, + }); + expect(fs.unsafeMode).toBe(false); + expect(fs.readonly).toBe(false); + await fs.writeFile("a", "hi"); + const r = await fs.readFile("a", "utf8"); + expect(r).toEqual("hi"); + }); + + it("directly create a dangerous file that is a symlink outside of the sandbox -- this should work", async () => { + await writeFile(join(tempDir, "password"), "s3cr3t"); + await symlink( + join(tempDir, "password"), + join(tempDir, "test-safe", "danger"), + ); + const s = await readFile(join(tempDir, "test-safe", "danger"), "utf8"); + expect(s).toBe("s3cr3t"); + }); + + it("cannot read the symlink content via the api", async () => { + await expect(async () => { + await fs.readFile("danger", "utf8"); + }).rejects.toThrow("outside of sandbox"); + }); +}); + +describe("read only sandbox", () => { + let fs; + it("creates and reads file", async () => { + await mkdir(join(tempDir, "test-ro")); + fs = new SandboxedFilesystem(join(tempDir, "test-ro"), { + readonly: true, + }); + expect(fs.readonly).toBe(true); + await expect(async () => { + await fs.writeFile("a", "hi"); + }).rejects.toThrow("permission denied -- read only filesystem"); + try { + await fs.writeFile("a", "hi"); + } catch (err) { + expect(err.code).toEqual("EACCES"); + } + }); +}); + afterAll(async () => { await rm(tempDir, { force: true, recursive: true }); }); diff --git a/src/packages/conat/files/fs.ts b/src/packages/conat/files/fs.ts index 0f6746ebec..c820c6dc1b 100644 --- a/src/packages/conat/files/fs.ts +++ b/src/packages/conat/files/fs.ts @@ -7,6 +7,28 @@ import { } from "@cocalc/conat/files/watch"; export const DEFAULT_FILE_SERVICE = "fs"; +export interface FindOptions { + // timeout is very limited (e.g., 3s?) if fs is running on file + // server (not in own project) + timeout?: number; + // recursive is false by default (unlike actual find command) + recursive?: boolean; + // see typing below -- we can't just pass arbitrary args since + // that would not be secure. + expression?: FindExpression; +} + +export type FindExpression = + | { type: "name"; pattern: string } + | { type: "iname"; pattern: string } + | { type: "type"; value: "f" | "d" | "l" } + | { type: "size"; operator: "+" | "-"; value: string } + | { type: "mtime"; operator: "+" | "-"; days: number } + | { type: "newer"; file: string } + | { type: "and"; left: FindExpression; right: FindExpression } + | { type: "or"; left: FindExpression; right: FindExpression } + | { type: "not"; expr: FindExpression }; + export interface Filesystem { appendFile: (path: string, data: string | Buffer, encoding?) => Promise; chmod: (path: string, mode: string | number) => Promise; @@ -38,11 +60,13 @@ export interface Filesystem { // We add very little to the Filesystem api, but we have to add // a sandboxed "find" command, since it is a 1-call way to get - // arbitrary directory listing info. + // arbitrary directory listing info, which is just not possible + // with the fs API, but required in any serious application. // find -P {path} -maxdepth 1 -mindepth 1 -printf {printf} find: ( path: string, printf: string, + options?: FindOptions, ) => Promise<{ stdout: Buffer; truncated: boolean }>; } @@ -138,8 +162,8 @@ export async function fsServer({ service, fs, client }: Options) { async exists(path: string) { return await (await fs(this.subject)).exists(path); }, - async find(path: string, printf: string) { - return await (await fs(this.subject)).find(path, printf); + async find(path: string, printf: string, options?: FindOptions) { + return await (await fs(this.subject)).find(path, printf, options); }, async link(existingPath: string, newPath: string) { await (await fs(this.subject)).link(existingPath, newPath); From 06ddce723290a880b1f3be08a4ddd65bc47d8cb6 Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 23 Jul 2025 01:33:10 +0000 Subject: [PATCH 066/270] update file-server to not use fs.ls --- src/packages/file-server/btrfs/snapshots.ts | 6 +-- .../file-server/btrfs/subvolume-snapshots.ts | 13 +++--- .../file-server/btrfs/test/filesystem.test.ts | 4 +- .../btrfs/test/subvolume-stress.test.ts | 16 ++++---- .../file-server/btrfs/test/subvolume.test.ts | 40 +++++++++---------- 5 files changed, 38 insertions(+), 41 deletions(-) diff --git a/src/packages/file-server/btrfs/snapshots.ts b/src/packages/file-server/btrfs/snapshots.ts index daf8e09212..99d0856245 100644 --- a/src/packages/file-server/btrfs/snapshots.ts +++ b/src/packages/file-server/btrfs/snapshots.ts @@ -50,9 +50,9 @@ export async function updateRollingSnapshots({ } // get exactly the iso timestamp snapshot names: - const snapshotNames = (await snapshots.ls()) - .map((x) => x.name) - .filter((name) => DATE_REGEXP.test(name)); + const snapshotNames = (await snapshots.readdir()).filter((name) => + DATE_REGEXP.test(name), + ); snapshotNames.sort(); if (snapshotNames.length > 0) { const age = Date.now() - new Date(snapshotNames.slice(-1)[0]).valueOf(); diff --git a/src/packages/file-server/btrfs/subvolume-snapshots.ts b/src/packages/file-server/btrfs/subvolume-snapshots.ts index 9dcd4f30ad..ddcc3ca2e1 100644 --- a/src/packages/file-server/btrfs/subvolume-snapshots.ts +++ b/src/packages/file-server/btrfs/subvolume-snapshots.ts @@ -2,7 +2,6 @@ import { type Subvolume } from "./subvolume"; import { btrfs } from "./util"; import getLogger from "@cocalc/backend/logger"; import { join } from "path"; -import { type DirectoryListingEntry } from "@cocalc/util/types"; import { SnapshotCounts, updateRollingSnapshots } from "./snapshots"; export const SNAPSHOTS = ".snapshots"; @@ -48,9 +47,9 @@ export class SubvolumeSnapshots { }); }; - ls = async (): Promise => { + readdir = async (): Promise => { await this.makeSnapshotsDir(); - return await this.subvolume.fs.ls(SNAPSHOTS, { hidden: false }); + return await this.subvolume.fs.readdir(SNAPSHOTS); }; lock = async (name: string) => { @@ -85,18 +84,18 @@ export class SubvolumeSnapshots { // has newly written changes since last snapshot hasUnsavedChanges = async (): Promise => { - const s = await this.ls(); + const s = await this.readdir(); if (s.length == 0) { // more than just the SNAPSHOTS directory? - const v = await this.subvolume.fs.ls("", { hidden: true }); - if (v.length == 0 || (v.length == 1 && v[0].name == SNAPSHOTS)) { + const v = await this.subvolume.fs.readdir(""); + if (v.length == 0 || (v.length == 1 && v[0] == SNAPSHOTS)) { return false; } return true; } const pathGen = await getGeneration(this.subvolume.path); const snapGen = await getGeneration( - join(this.snapshotsDir, s[s.length - 1].name), + join(this.snapshotsDir, s[s.length - 1]), ); return snapGen < pathGen; }; diff --git a/src/packages/file-server/btrfs/test/filesystem.test.ts b/src/packages/file-server/btrfs/test/filesystem.test.ts index 673980af89..e0693b16bd 100644 --- a/src/packages/file-server/btrfs/test/filesystem.test.ts +++ b/src/packages/file-server/btrfs/test/filesystem.test.ts @@ -30,7 +30,7 @@ describe("operations with subvolumes", () => { const vol = await fs.subvolumes.get("cocalc"); expect(vol.name).toBe("cocalc"); // it has no snapshots - expect(await vol.snapshots.ls()).toEqual([]); + expect(await vol.snapshots.readdir()).toEqual([]); }); it("our subvolume is in the list", async () => { @@ -97,7 +97,7 @@ describe("clone of a subvolume with snapshots should have no snapshots", () => { it("clone has no snapshots", async () => { const clone = await fs.subvolumes.get("my-clone"); expect(await clone.fs.readFile("abc.txt", "utf8")).toEqual("hi"); - expect(await clone.snapshots.ls()).toEqual([]); + expect(await clone.snapshots.readdir()).toEqual([]); await clone.snapshots.create("my-clone-snap"); }); }); diff --git a/src/packages/file-server/btrfs/test/subvolume-stress.test.ts b/src/packages/file-server/btrfs/test/subvolume-stress.test.ts index 29bc048e69..69279d9b04 100644 --- a/src/packages/file-server/btrfs/test/subvolume-stress.test.ts +++ b/src/packages/file-server/btrfs/test/subvolume-stress.test.ts @@ -29,16 +29,16 @@ describe(`stress test creating ${numSnapshots} snapshots`, () => { `created ${Math.round((numSnapshots / (Date.now() - start)) * 1000)} snapshots per second in serial`, ); snaps.sort(); - expect((await vol.snapshots.ls()).map(({ name }) => name).sort()).toEqual( - snaps.sort(), - ); + expect( + (await vol.snapshots.readdir()).filter((x) => !x.startsWith(".")).sort(), + ).toEqual(snaps.sort()); }); it(`delete our ${numSnapshots} snapshots`, async () => { for (let i = 0; i < numSnapshots; i++) { await vol.snapshots.delete(`snap${i}`); } - expect(await vol.snapshots.ls()).toEqual([]); + expect(await vol.snapshots.readdir()).toEqual([]); }); }); @@ -58,9 +58,8 @@ describe(`create ${numFiles} files`, () => { log( `created ${Math.round((numFiles / (Date.now() - start)) * 1000)} files per second in serial`, ); - const v = await vol.fs.ls(""); - const w = v.map(({ name }) => name); - expect(w.sort()).toEqual(names.sort()); + const v = await vol.fs.readdir(""); + expect(v.sort()).toEqual(names.sort()); }); it(`creates ${numFiles} files in parallel`, async () => { @@ -77,9 +76,8 @@ describe(`create ${numFiles} files`, () => { `created ${Math.round((numFiles / (Date.now() - start)) * 1000)} files per second in parallel`, ); const t0 = Date.now(); - const v = await vol.fs.ls("p"); + const w = await vol.fs.readdir("p"); log("get listing of files took", Date.now() - t0, "ms"); - const w = v.map(({ name }) => name); expect(w.sort()).toEqual(names.sort()); }); }); diff --git a/src/packages/file-server/btrfs/test/subvolume.test.ts b/src/packages/file-server/btrfs/test/subvolume.test.ts index 456282d8b3..1736f17c4f 100644 --- a/src/packages/file-server/btrfs/test/subvolume.test.ts +++ b/src/packages/file-server/btrfs/test/subvolume.test.ts @@ -19,7 +19,7 @@ describe("setting and getting quota of a subvolume", () => { }); it("get directory listing", async () => { - const v = await vol.fs.ls(""); + const v = await vol.fs.readdir(""); expect(v).toEqual([]); }); @@ -36,9 +36,8 @@ describe("setting and getting quota of a subvolume", () => { const { used } = await vol.quota.usage(); expect(used).toBeGreaterThan(0); - const v = await vol.fs.ls(""); - // size is potentially random, reflecting compression - expect(v).toEqual([{ name: "buf", mtime: v[0].mtime, size: v[0].size }]); + const v = await vol.fs.readdir(""); + expect(v).toEqual(["buf"]); }); it("fail to write a 50MB file (due to quota)", async () => { @@ -54,20 +53,20 @@ describe("the filesystem operations", () => { it("creates a volume and get empty listing", async () => { vol = await fs.subvolumes.get("fs"); - expect(await vol.fs.ls("")).toEqual([]); + expect(await vol.fs.readdir("")).toEqual([]); }); it("error listing non-existent path", async () => { vol = await fs.subvolumes.get("fs"); expect(async () => { - await vol.fs.ls("no-such-path"); + await vol.fs.readdir("no-such-path"); }).rejects.toThrow("ENOENT"); }); it("creates a text file to it", async () => { await vol.fs.writeFile("a.txt", "hello"); - const ls = await vol.fs.ls(""); - expect(ls).toEqual([{ name: "a.txt", mtime: ls[0].mtime, size: 5 }]); + const ls = await vol.fs.readdir(""); + expect(ls).toEqual(["a.txt"]); }); it("read the file we just created as utf8", async () => { @@ -87,17 +86,18 @@ describe("the filesystem operations", () => { let origStat; it("snapshot filesystem and see file is in snapshot", async () => { await vol.snapshots.create("snap"); - const s = await vol.fs.ls(vol.snapshots.path("snap")); - expect(s).toEqual([{ name: "a.txt", mtime: s[0].mtime, size: 5 }]); + const s = await vol.fs.readdir(vol.snapshots.path("snap")); + expect(s).toContain("a.txt"); - const stat = await vol.fs.stat("a.txt"); - origStat = stat; - expect(stat.mtimeMs / 1000).toBeCloseTo(s[0].mtime ?? 0); + const stat0 = await vol.fs.stat(vol.snapshots.path("snap")); + const stat1 = await vol.fs.stat("a.txt"); + origStat = stat1; + expect(stat1.mtimeMs).toBeCloseTo(stat0.mtimeMs, -2); }); it("unlink (delete) our file", async () => { await vol.fs.unlink("a.txt"); - expect(await vol.fs.ls("")).toEqual([]); + expect(await vol.fs.readdir("")).toEqual([".snapshots"]); }); it("snapshot still exists", async () => { @@ -185,9 +185,9 @@ describe("test snapshots", () => { }); it("snapshot the volume", async () => { - expect(await vol.snapshots.ls()).toEqual([]); + expect(await vol.snapshots.readdir()).toEqual([]); await vol.snapshots.create("snap1"); - expect((await vol.snapshots.ls()).map((x) => x.name)).toEqual(["snap1"]); + expect(await vol.snapshots.readdir()).toEqual(["snap1"]); expect(await vol.snapshots.hasUnsavedChanges()).toBe(false); }); @@ -223,7 +223,7 @@ describe("test snapshots", () => { await vol.snapshots.unlock("snap1"); await vol.snapshots.delete("snap1"); expect(await vol.snapshots.exists("snap1")).toBe(false); - expect(await vol.snapshots.ls()).toEqual([]); + expect(await vol.snapshots.readdir()).toEqual([]); }); }); @@ -266,7 +266,7 @@ describe("test bup backups", () => { it("add a directory and back up", async () => { await mkdir(join(vol.path, "mydir")); await vol.fs.writeFile(join("mydir", "file.txt"), "hello3"); - expect((await vol.fs.ls("mydir"))[0].name).toBe("file.txt"); + expect((await vol.fs.readdir("mydir"))[0]).toBe("file.txt"); await vol.bup.save(); const x = await vol.bup.ls("latest"); expect(x).toEqual([ @@ -287,8 +287,8 @@ describe("test bup backups", () => { }); it("most recent snapshot has a backup before the restore", async () => { - const s = await vol.snapshots.ls(); - const recent = s.slice(-1)[0].name; + const s = await vol.snapshots.readdir(); + const recent = s.slice(-1)[0]; const p = vol.snapshots.path(recent, "mydir", "file.txt"); expect(await vol.fs.readFile(p, "utf8")).toEqual("changed"); }); From c32f46984873cdb1f0bf84dcaac16dc040cd99a7 Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 23 Jul 2025 01:40:39 +0000 Subject: [PATCH 067/270] fix the packages/sync tests --- .../sync/editor/string/test/README.md | 5 ++++ .../sync/editor/string/test/client-test.ts | 8 ++++++ .../string/test/ephemeral-syncstring.ts | 3 ++- .../sync/editor/string/test/sync.0.test.ts | 4 +-- .../sync/editor/string/test/sync.1.test.ts | 25 ++++++------------- 5 files changed, 25 insertions(+), 20 deletions(-) create mode 100644 src/packages/sync/editor/string/test/README.md diff --git a/src/packages/sync/editor/string/test/README.md b/src/packages/sync/editor/string/test/README.md new file mode 100644 index 0000000000..0a064bf05c --- /dev/null +++ b/src/packages/sync/editor/string/test/README.md @@ -0,0 +1,5 @@ +There is additional _integration_ testing of the sync code in: + +``` +packages/backend/conat/test/sync-doc +``` \ No newline at end of file diff --git a/src/packages/sync/editor/string/test/client-test.ts b/src/packages/sync/editor/string/test/client-test.ts index b9d43e012a..0c28027621 100644 --- a/src/packages/sync/editor/string/test/client-test.ts +++ b/src/packages/sync/editor/string/test/client-test.ts @@ -187,3 +187,11 @@ export class Client extends EventEmitter implements Client0 { console.log(`shell: opts=${JSON.stringify(opts)}`); } } + +class Filesystem { + readFile = () => ""; + writeFile = () => {}; + utimes = () => {}; +} + +export const fs = new Filesystem() as any; diff --git a/src/packages/sync/editor/string/test/ephemeral-syncstring.ts b/src/packages/sync/editor/string/test/ephemeral-syncstring.ts index 81cb8001f0..086fa9a154 100644 --- a/src/packages/sync/editor/string/test/ephemeral-syncstring.ts +++ b/src/packages/sync/editor/string/test/ephemeral-syncstring.ts @@ -3,7 +3,7 @@ This is useful not just for testing, but also for implementing undo/redo for editing a text document when there is no actual file or project involved. */ -import { Client } from "./client-test"; +import { Client, fs } from "./client-test"; import { SyncString } from "../sync"; import { a_txt } from "./data"; import { once } from "@cocalc/util/async-utils"; @@ -16,6 +16,7 @@ export default async function ephemeralSyncstring() { path, client, ephemeral: true, + fs, }); // replace save to disk, since otherwise unless string is empty, // this will hang forever... and it is called on close. diff --git a/src/packages/sync/editor/string/test/sync.0.test.ts b/src/packages/sync/editor/string/test/sync.0.test.ts index 9d5289dbd1..82f080dac9 100644 --- a/src/packages/sync/editor/string/test/sync.0.test.ts +++ b/src/packages/sync/editor/string/test/sync.0.test.ts @@ -11,7 +11,7 @@ pnpm test sync.0.test.ts */ -import { Client } from "./client-test"; +import { Client, fs } from "./client-test"; import { SyncString } from "../sync"; import { a_txt } from "./data"; import { once } from "@cocalc/util/async-utils"; @@ -23,7 +23,7 @@ describe("create a blank minimal string SyncDoc and call public methods on it", let syncstring: SyncString; it("creates the syncstring and wait for it to be ready", async () => { - syncstring = new SyncString({ project_id, path, client }); + syncstring = new SyncString({ project_id, path, client, fs }); expect(syncstring.get_state()).toBe("init"); await once(syncstring, "ready"); expect(syncstring.get_state()).toBe("ready"); diff --git a/src/packages/sync/editor/string/test/sync.1.test.ts b/src/packages/sync/editor/string/test/sync.1.test.ts index a58fa9d3a4..842c85c610 100644 --- a/src/packages/sync/editor/string/test/sync.1.test.ts +++ b/src/packages/sync/editor/string/test/sync.1.test.ts @@ -12,7 +12,7 @@ pnpm test sync.1.test.ts */ -import { Client } from "./client-test"; +import { Client, fs } from "./client-test"; import { SyncString } from "../sync"; import { once } from "@cocalc/util/async-utils"; import { a_txt } from "./data"; @@ -28,7 +28,13 @@ describe("create syncstring and test doing some edits", () => { ]; it("creates the syncstring and wait until ready", async () => { - syncstring = new SyncString({ project_id, path, client, cursors: true }); + syncstring = new SyncString({ + project_id, + path, + client, + cursors: true, + fs, + }); expect(syncstring.get_state()).toBe("init"); await once(syncstring, "ready"); }); @@ -97,21 +103,6 @@ describe("create syncstring and test doing some edits", () => { expect(syncstring.is_read_only()).toBe(false); }); - it("save to disk", async () => { - expect(syncstring.has_unsaved_changes()).toBe(true); - const promise = syncstring.save_to_disk(); - // Mock: we set save to done in the syncstring - // table, otherwise the promise will never resolve. - (syncstring as any).set_save({ - state: "done", - error: "", - hash: syncstring.hash_of_live_version(), - }); - (syncstring as any).syncstring_table.emit("change-no-throttle"); - await promise; - expect(syncstring.has_unsaved_changes()).toBe(false); - }); - it("close and clean up", async () => { await syncstring.close(); expect(syncstring.get_state()).toBe("closed"); From 17077cf8ff515c9c414750ac2ab4c67dea6028a0 Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 23 Jul 2025 05:07:02 +0000 Subject: [PATCH 068/270] implement core of new listings --- .../backend/conat/files/test/listing.test.ts | 92 +++++++++++++ src/packages/conat/files/fs.ts | 7 + src/packages/conat/files/listing.ts | 126 ++++++++++++++++++ 3 files changed, 225 insertions(+) create mode 100644 src/packages/backend/conat/files/test/listing.test.ts create mode 100644 src/packages/conat/files/listing.ts diff --git a/src/packages/backend/conat/files/test/listing.test.ts b/src/packages/backend/conat/files/test/listing.test.ts new file mode 100644 index 0000000000..d19b459d94 --- /dev/null +++ b/src/packages/backend/conat/files/test/listing.test.ts @@ -0,0 +1,92 @@ +import { SandboxedFilesystem } from "@cocalc/backend/files/sandbox"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "path"; +import { randomId } from "@cocalc/conat/names"; +import listing from "@cocalc/conat/files/listing"; + +let tmp; +beforeAll(async () => { + tmp = await mkdtemp(join(tmpdir(), `cocalc-${randomId()}0`)); +}); + +afterAll(async () => { + try { + await rm(tmp, { force: true, recursive: true }); + } catch {} +}); + +describe("creating a listing monitor starting with an empty directory", () => { + let fs, dir; + it("creates sandboxed filesystem", async () => { + fs = new SandboxedFilesystem(tmp); + dir = await listing({ path: "", fs }); + }); + + it("initial listing is empty", () => { + expect(Object.keys(dir.files)).toEqual([]); + }); + + let iter; + it("create a file and get an update", async () => { + iter = dir.iter(); + await fs.writeFile("a.txt", "hello"); + let { value } = await iter.next(); + expect(value).toEqual({ + mtime: value.mtime, + name: "a.txt", + size: value.size, + }); + // it's possible that the file isn't written completely above. + if (value.size != 5) { + ({ value } = await iter.next()); + } + const stat = await fs.stat("a.txt"); + expect(stat.mtimeMs).toEqual(value.mtime); + expect(dir.files["a.txt"]).toEqual({ mtime: value.mtime, size: 5 }); + }); + + it("modify the file and get two updates -- one when it starts and another when done", async () => { + await fs.appendFile("a.txt", " there"); + const { value } = await iter.next(); + expect(value).toEqual({ mtime: value.mtime, name: "a.txt", size: 5 }); + const { value: value2 } = await iter.next(); + expect(value2).toEqual({ mtime: value2.mtime, name: "a.txt", size: 11 }); + const stat = await fs.stat("a.txt"); + expect(stat.mtimeMs).toEqual(value2.mtime); + expect(dir.files["a.txt"]).toEqual({ mtime: value2.mtime, size: 11 }); + }); + + it("create another monitor starting with the now nonempty directory", async () => { + const dir2 = await listing({ path: "", fs }); + expect(Object.keys(dir.files)).toEqual(["a.txt"]); + dir2.close(); + }); + + const count = 500; + it(`creates ${count} files and see they are found`, async () => { + const n = Object.keys(dir.files).length; + + for (let i = 0; i < count; i++) { + await fs.writeFile(`${i}`, ""); + } + const values: string[] = []; + while (true) { + const { value } = await iter.next(); + if (value == "a.txt") { + continue; + } + values.push(value); + if (value.name == `${count - 1}`) { + break; + } + } + expect(new Set(values).size).toEqual(count); + + expect(Object.keys(dir.files).length).toEqual(n + count); + }); + + it("cleans up", () => { + dir.close(); + }); +}); diff --git a/src/packages/conat/files/fs.ts b/src/packages/conat/files/fs.ts index c820c6dc1b..2c30db142f 100644 --- a/src/packages/conat/files/fs.ts +++ b/src/packages/conat/files/fs.ts @@ -5,6 +5,8 @@ import { watchClient, type WatchIterator, } from "@cocalc/conat/files/watch"; +import listing, { type Listing } from "./listing"; + export const DEFAULT_FILE_SERVICE = "fs"; export interface FindOptions { @@ -68,6 +70,8 @@ export interface Filesystem { printf: string, options?: FindOptions, ) => Promise<{ stdout: Buffer; truncated: boolean }>; + + listing?: (path: string) => Promise; } interface IStats { @@ -286,6 +290,9 @@ export function fsClient({ await ensureWatchServerExists(path, options); return await watchClient({ client, subject, path, options }); }; + call.listing = async (path: string) => { + return await listing({ fs: call, path }); + }; return call; } diff --git a/src/packages/conat/files/listing.ts b/src/packages/conat/files/listing.ts new file mode 100644 index 0000000000..a82c7afd20 --- /dev/null +++ b/src/packages/conat/files/listing.ts @@ -0,0 +1,126 @@ +/* +Directory Listing + +Tests in packages/backend/conat/files/test/listing.test.ts +*/ + +import { EventEmitter } from "events"; +import { join } from "path"; +import { type Filesystem } from "./fs"; +import { EventIterator } from "@cocalc/util/event-iterator"; + +interface FileData { + mtime: number; + size: number; +} + +type Files = { [name: string]: FileData }; + +interface Options { + path: string; + fs: Filesystem; +} + +export default async function listing(opts: Options): Promise { + const listing = new Listing(opts); + await listing.init(); + return listing; +} + +export class Listing extends EventEmitter { + public files?: Files = {}; + public truncated?: boolean; + private watch?; + private iters: EventIterator[] = []; + constructor(public readonly opts: Options) { + super(); + } + + iter = () => { + const iter = new EventIterator(this, "change", { + map: (args) => { + return { name: args[0], ...args[1] }; + }, + }); + this.iters.push(iter); + return iter; + }; + + close = () => { + this.emit("closed"); + this.iters.map((iter) => iter.end()); + this.iters.length = 0; + this.watch?.close(); + delete this.files; + delete this.watch; + }; + + init = async () => { + const { fs, path } = this.opts; + this.watch = await fs.watch(path); + const { files, truncated } = await getListing(fs, path); + this.files = files; + this.truncated = truncated; + this.emit("ready"); + this.handleUpdates(); + }; + + private handleUpdates = async () => { + for await (const { filename } of this.watch) { + if (this.files == null) { + return; + } + this.update(filename); + } + }; + + private update = async (filename: string) => { + if (this.files == null) { + // closed or not initialized yet + return; + } + try { + const stats = await this.opts.fs.stat(join(this.opts.path, filename)); + if (this.files == null) { + return; + } + this.files[filename] = { mtime: stats.mtimeMs, size: stats.size }; + } catch (err) { + if (err.code == "ENOENT") { + // file deleted + delete this.files[filename]; + } else { + console.warn("WARNING:", err); + // TODO: some other error -- e.g., network down or permissions, so we don't know anything. + // Should we retry (?). + return; + } + } + this.emit("change", filename, this.files[filename]); + }; +} + +async function getListing( + fs: Filesystem, + path: string, +): Promise<{ files: Files; truncated: boolean }> { + const { stdout, truncated } = await fs.find(path, "%f\\0%T@\\0%s\n"); + const buf = Buffer.from(stdout); + const files: Files = {}; + // todo -- what about non-utf8...? + + const s = buf.toString().trim(); + if (!s) { + return { files, truncated }; + } + for (const line of s.split("\n")) { + try { + const v = line.split("\0"); + const name = v[0]; + const mtime = parseFloat(v[1]) * 1000; + const size = parseInt(v[2]); + files[name] = { mtime, size }; + } catch {} + } + return { files, truncated }; +} From 7fbde56e14cd42efcc4d95861d6833846e98e5e7 Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 23 Jul 2025 13:46:35 +0000 Subject: [PATCH 069/270] timetravel -- implement opening timetravel doc in a much more straightforward explicit way: basically, you need to open the main document - it was implicit and complicated before; now just open the main file as a background tab. Simple and much easier to get right. --- .../frame-editors/code-editor/actions.ts | 2 +- .../time-travel-editor/actions.ts | 73 +++++---- src/packages/sync/client/sync-client.ts | 147 +----------------- 3 files changed, 52 insertions(+), 170 deletions(-) diff --git a/src/packages/frontend/frame-editors/code-editor/actions.ts b/src/packages/frontend/frame-editors/code-editor/actions.ts index 28f025e153..744008bf83 100644 --- a/src/packages/frontend/frame-editors/code-editor/actions.ts +++ b/src/packages/frontend/frame-editors/code-editor/actions.ts @@ -445,7 +445,7 @@ export class Actions< // Flag that there is activity (causes icon to turn orange). private activity = (): void => { - this._get_project_actions().flag_file_activity(this.path); + this._get_project_actions()?.flag_file_activity(this.path); }; // This is currently NOT used in this base class. It's used in other diff --git a/src/packages/frontend/frame-editors/time-travel-editor/actions.ts b/src/packages/frontend/frame-editors/time-travel-editor/actions.ts index edbe640391..1448177aad 100644 --- a/src/packages/frontend/frame-editors/time-travel-editor/actions.ts +++ b/src/packages/frontend/frame-editors/time-travel-editor/actions.ts @@ -21,7 +21,6 @@ import { List } from "immutable"; import { once } from "@cocalc/util/async-utils"; import { filename_extension, path_split } from "@cocalc/util/misc"; import { SyncDoc } from "@cocalc/sync/editor/generic/sync-doc"; -import { webapp_client } from "../../webapp-client"; import { exec } from "@cocalc/frontend/frame-editors/generic/client"; import { ViewDocument } from "./view-document"; import { @@ -32,8 +31,8 @@ import { FrameTree } from "../frame-tree/types"; import { export_to_json } from "./export-to-json"; import type { Document } from "@cocalc/sync/editor/generic/types"; import LRUCache from "lru-cache"; -import { syncdbPath } from "@cocalc/util/jupyter/names"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; +import { until } from "@cocalc/util/async-utils"; const EXTENSION = ".time-travel"; @@ -81,7 +80,6 @@ export class TimeTravelActions extends CodeEditorActions { protected doctype: string = "none"; // actual document is managed elsewhere private docpath: string; private docext: string; - private syncpath: string; syncdoc?: SyncDoc; private first_load: boolean = true; ambient_actions?: CodeEditorActions; @@ -96,11 +94,7 @@ export class TimeTravelActions extends CodeEditorActions { this.docpath = head + "/" + this.docpath; } // log("init", { path: this.path }); - this.syncpath = this.docpath; this.docext = filename_extension(this.docpath); - if (this.docext == "ipynb") { - this.syncpath = syncdbPath(this.docpath); - } this.setState({ versions: List([]), loading: true, @@ -118,27 +112,50 @@ export class TimeTravelActions extends CodeEditorActions { init_frame_tree = () => {}; - close = (): void => { - if (this.syncdoc != null) { - this.syncdoc.close(); - delete this.syncdoc; - } - super.close(); - }; - set_error = (error) => { this.setState({ error }); }; private init_syncdoc = async (): Promise => { - const persistent = this.docext == "ipynb" || this.docext == "sagews"; // ugly for now (?) - this.syncdoc = await webapp_client.sync_client.open_existing_sync_document({ - project_id: this.project_id, - path: this.syncpath, - persistent, + let mainFileActions: any = null; + await until(async () => { + if (this.isClosed()) { + return true; + } + mainFileActions = this.redux.getEditorActions( + this.project_id, + this.docpath, + ); + console.log("mainFileActions", mainFileActions != null); + if (mainFileActions == null) { + console.log("opening file"); + // open the file that we're showing timetravel for, so that the + // actions are available + try { + await this.open_file({ foreground: false, explicit: false }); + } catch (err) { + console.warn(err); + } + // will try again above in the next loop + return false; + } else { + const doc = mainFileActions._syncstring; + if (doc == null || doc.get_state() == "closed") { + // file is closing + return false; + } + // got it! + return true; + } }); - if (this.syncdoc == null) return; - this.syncdoc.on("change", debounce(this.syncdoc_changed, 1000)); + if (this.isClosed() || mainFileActions == null) { + return; + } + this.syncdoc = mainFileActions._syncstring; + + if (this.syncdoc == null || this.syncdoc.get_state() == "closed") { + return; + } if (this.syncdoc.get_state() != "ready") { try { await once(this.syncdoc, "ready"); @@ -146,14 +163,16 @@ export class TimeTravelActions extends CodeEditorActions { return; } } - if (this.syncdoc == null) return; + this.syncdoc.on("change", debounce(this.syncdoc_changed, 750)); // cause initial load -- we could be plugging into an already loaded syncdoc, // so there wouldn't be any change event, so we have to trigger this. this.syncdoc_changed(); this.syncdoc.on("close", () => { - // in our code we don't check if the state is closed, but instead - // that this.syncdoc is not null. + console.log("in timetravel, syncdoc was closed"); + // in the actions in this file, we don't check if the state is closed, but instead + // that this.syncdoc is not null: delete this.syncdoc; + this.init_syncdoc(); }); this.setState({ @@ -289,10 +308,10 @@ export class TimeTravelActions extends CodeEditorActions { } }; - open_file = async (): Promise => { + open_file = async (opts?): Promise => { // log("open_file"); const actions = this.redux.getProjectActions(this.project_id); - await actions.open_file({ path: this.docpath, foreground: true }); + await actions.open_file({ path: this.docpath, foreground: true, ...opts }); }; // Revert the live version of the document to a specific version */ diff --git a/src/packages/sync/client/sync-client.ts b/src/packages/sync/client/sync-client.ts index d3287e9295..2edd1eb008 100644 --- a/src/packages/sync/client/sync-client.ts +++ b/src/packages/sync/client/sync-client.ts @@ -8,9 +8,8 @@ Functionality related to Sync. */ import { once } from "@cocalc/util/async-utils"; -import { defaults, required } from "@cocalc/util/misc"; import { SyncDoc, SyncOpts0 } from "@cocalc/sync/editor/generic/sync-doc"; -import { SyncDB, SyncDBOpts0 } from "@cocalc/sync/editor/db"; +import { SyncDBOpts0 } from "@cocalc/sync/editor/db"; import { SyncString } from "@cocalc/sync/editor/string/sync"; import { synctable, @@ -20,7 +19,6 @@ import { synctable_no_changefeed, } from "@cocalc/sync/table"; import type { AppClient } from "./types"; -import { getSyncDocType } from "@cocalc/conat/sync/syncdoc-info"; interface SyncOpts extends Omit { noCache?: boolean; @@ -71,146 +69,11 @@ export class SyncClient { ); } - // These are not working properly, e.g., if you close and open - // a LARGE jupyter notebook quickly (so save to disk takes a while), - // then it gets broken until browser refresh. The problem is that - // the doc is still closing right when a new one starts being created. - // So for now we just revert to the non-cached-here approach. - // There is other caching elsewhere. - - // public sync_string(opts: SyncOpts): SyncString { - // return syncstringCache({ ...opts, client: this.client }); - // } - - // public sync_db(opts: SyncDBOpts): SyncDB { - // return syncdbCache({ ...opts, client: this.client }); - // } - - public sync_string(opts: SyncOpts): SyncString { - const opts0: SyncOpts0 = defaults(opts, { - id: undefined, - project_id: required, - path: required, - file_use_interval: "default", - cursors: false, - patch_interval: 1000, - save_interval: 2000, - persistent: false, - data_server: undefined, - client: this.client, - ephemeral: false, - fs: undefined, - }); - return new SyncString(opts0); - } - - public sync_db(opts: SyncDBOpts): SyncDoc { - const opts0: SyncDBOpts0 = defaults(opts, { - id: undefined, - project_id: required, - path: required, - file_use_interval: "default", - cursors: false, - patch_interval: 1000, - save_interval: 2000, - change_throttle: undefined, - persistent: false, - data_server: undefined, - - primary_keys: required, - string_cols: [], - - client: this.client, - - ephemeral: false, - - fs: undefined, - }); - return new SyncDB(opts0); + public sync_string(_opts: SyncOpts): SyncString { + throw Error("deprecated"); } - public async open_existing_sync_document({ - project_id, - path, - data_server, - persistent, - }: { - project_id: string; - path: string; - data_server?: string; - persistent?: boolean; - }): Promise { - const doctype = await getSyncDocType({ - project_id, - path, - client: this.client, - }); - const { type } = doctype; - const f = `sync_${type}`; - return (this as any)[f]({ - project_id, - path, - data_server, - persistent, - ...doctype.opts, - }); + public sync_db(_opts: SyncDBOpts): SyncDoc { + throw Error("deprecated"); } } - -/* -const syncdbCache = refCacheSync({ - name: "syncdb", - - createKey: ({ project_id, path }: SyncDBOpts) => { - return JSON.stringify({ project_id, path }); - }, - - createObject: (opts: SyncDBOpts) => { - const opts0: SyncDBOpts0 = defaults(opts, { - id: undefined, - project_id: required, - path: required, - file_use_interval: "default", - cursors: false, - patch_interval: 1000, - save_interval: 2000, - change_throttle: undefined, - persistent: false, - data_server: undefined, - - primary_keys: required, - string_cols: [], - - client: required, - - ephemeral: false, - }); - return new SyncDB(opts0); - }, -}); - -const syncstringCache = refCacheSync({ - name: "syncstring", - createKey: ({ project_id, path }: SyncOpts) => { - const key = JSON.stringify({ project_id, path }); - return key; - }, - - createObject: (opts: SyncOpts) => { - const opts0: SyncOpts0 = defaults(opts, { - id: undefined, - project_id: required, - path: required, - file_use_interval: "default", - cursors: false, - patch_interval: 1000, - save_interval: 2000, - persistent: false, - data_server: undefined, - client: required, - ephemeral: false, - }); - return new SyncString(opts0); - }, -}); -*/ From b011fdfe9053db4572617574d458e066d6fde3bd Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 23 Jul 2025 14:51:37 +0000 Subject: [PATCH 070/270] file opening in frontend -- fix leak I had just caused by opening background tabs --- src/packages/frontend/file-editors.ts | 4 +- .../time-travel-editor/actions.ts | 3 -- src/packages/frontend/project/open-file.ts | 6 +-- src/packages/frontend/project_actions.ts | 49 ++++++++++--------- 4 files changed, 31 insertions(+), 31 deletions(-) diff --git a/src/packages/frontend/file-editors.ts b/src/packages/frontend/file-editors.ts index ed692f555f..aed28b2fbc 100644 --- a/src/packages/frontend/file-editors.ts +++ b/src/packages/frontend/file-editors.ts @@ -259,9 +259,9 @@ export async function remove( save(path, redux, project_id, is_public); } - if (!is_public) { + if (!is_public && project_id) { // Also free the corresponding side chat, if it was created. - require("./chat/register").remove( + (await import("./chat/register")).remove( meta_file(path, "chat"), redux, project_id, diff --git a/src/packages/frontend/frame-editors/time-travel-editor/actions.ts b/src/packages/frontend/frame-editors/time-travel-editor/actions.ts index 1448177aad..2ef79fc2c6 100644 --- a/src/packages/frontend/frame-editors/time-travel-editor/actions.ts +++ b/src/packages/frontend/frame-editors/time-travel-editor/actions.ts @@ -126,9 +126,7 @@ export class TimeTravelActions extends CodeEditorActions { this.project_id, this.docpath, ); - console.log("mainFileActions", mainFileActions != null); if (mainFileActions == null) { - console.log("opening file"); // open the file that we're showing timetravel for, so that the // actions are available try { @@ -168,7 +166,6 @@ export class TimeTravelActions extends CodeEditorActions { // so there wouldn't be any change event, so we have to trigger this. this.syncdoc_changed(); this.syncdoc.on("close", () => { - console.log("in timetravel, syncdoc was closed"); // in the actions in this file, we don't check if the state is closed, but instead // that this.syncdoc is not null: delete this.syncdoc; diff --git a/src/packages/frontend/project/open-file.ts b/src/packages/frontend/project/open-file.ts index c905f6914b..d5d20be4b2 100644 --- a/src/packages/frontend/project/open-file.ts +++ b/src/packages/frontend/project/open-file.ts @@ -347,16 +347,14 @@ export async function open_file( return; } - if (PRELOAD_BACKGROUND_TABS) { - await actions.initFileRedux(opts.path); - } - if (opts.foreground) { actions.foreground_project(opts.change_history); const tab = path_to_tab(opts.path); actions.set_active_tab(tab, { change_history: opts.change_history, }); + } else if (PRELOAD_BACKGROUND_TABS) { + await actions.initFileRedux(opts.path); } if (alreadyOpened && opts.fragmentId) { diff --git a/src/packages/frontend/project_actions.ts b/src/packages/frontend/project_actions.ts index c6ca0fed22..803fbde4a0 100644 --- a/src/packages/frontend/project_actions.ts +++ b/src/packages/frontend/project_actions.ts @@ -12,7 +12,6 @@ import { List, Map, fromJS, Set as immutableSet } from "immutable"; import { isEqual, throttle } from "lodash"; import { join } from "path"; import { defineMessage } from "react-intl"; - import { computeServerManager, type ComputeServerManager, @@ -1061,28 +1060,34 @@ export class ProjectActions extends Actions { }; /* Initialize the redux store and react component for editing - a particular file. + a particular file, if necessary. */ - initFileRedux = async ( - path: string, - is_public: boolean = false, - ext?: string, // use this extension even instead of path's extension. - ): Promise => { - // LAZY IMPORT, so that editors are only available - // when you are going to use them. Helps with code splitting. - await import("./editors/register-all"); - - // Initialize the file's store and actions - const name = await project_file.initializeAsync( - path, - this.redux, - this.project_id, - is_public, - undefined, - ext, - ); - return name; - }; + initFileRedux = reuseInFlight( + async ( + path: string, + is_public: boolean = false, + ext?: string, // use this extension even instead of path's extension. + ): Promise => { + const cur = redux.getEditorActions(this.project_id, path); + if (cur != null) { + return cur.name; + } + // LAZY IMPORT, so that editors are only available + // when you are going to use them. Helps with code splitting. + await import("./editors/register-all"); + + // Initialize the file's store and actions + const name = await project_file.initializeAsync( + path, + this.redux, + this.project_id, + is_public, + undefined, + ext, + ); + return name; + }, + ); private init_file_react_redux = async ( path: string, From 3bd88b8fb85841caaadd9be0ca40198f94029208 Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 23 Jul 2025 14:57:16 +0000 Subject: [PATCH 071/270] github ci: fix test failing due to missing yapf --- .github/workflows/make-and-test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/make-and-test.yml b/.github/workflows/make-and-test.yml index 1498f67653..325c8e5031 100644 --- a/.github/workflows/make-and-test.yml +++ b/.github/workflows/make-and-test.yml @@ -39,7 +39,7 @@ jobs: detached: true - uses: actions/checkout@v4 - name: Install python3 requests - run: sudo apt-get install python3-requests + run: sudo apt-get install python3-requests python3-yapf - name: Check doc links run: cd src/scripts && python3 check_doc_urls.py || sleep 5 || python3 check_doc_urls.py @@ -99,7 +99,7 @@ jobs: python3 -m pip install --upgrade pip virtualenv python3 -m virtualenv venv source venv/bin/activate - pip install ipykernel + pip install ipykernel yapf python -m ipykernel install --prefix=./jupyter-local --name python3-local --display-name "Python 3 (Local)" From cabf024a86737d5162e2d227ec74d40fcb5139bc Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 23 Jul 2025 16:46:48 +0000 Subject: [PATCH 072/270] create a new frontend integration test package and one tiny little test; starting work in listings that can be used from the frontend --- .../conat/files/test/local-path.test.ts | 5 +- src/packages/backend/package.json | 17 +--- src/packages/conat/files/fs.ts | 8 +- src/packages/conat/files/listing.ts | 2 +- .../frontend/project/listing/use-listing.ts | 86 +++++++++++++++++++ src/packages/pnpm-lock.yaml | 46 ++++++++-- src/packages/test/jest.config.js | 13 +++ src/packages/test/package.json | 37 ++++++++ src/packages/test/test/setup.js | 5 ++ src/packages/test/tsconfig.json | 22 +++++ src/packages/test/use-listing.test.ts | 33 +++++++ src/packages/util/async-utils.ts | 6 +- src/workspaces.py | 1 + 13 files changed, 257 insertions(+), 24 deletions(-) create mode 100644 src/packages/frontend/project/listing/use-listing.ts create mode 100644 src/packages/test/jest.config.js create mode 100644 src/packages/test/package.json create mode 100644 src/packages/test/test/setup.js create mode 100644 src/packages/test/tsconfig.json create mode 100644 src/packages/test/use-listing.test.ts diff --git a/src/packages/backend/conat/files/test/local-path.test.ts b/src/packages/backend/conat/files/test/local-path.test.ts index a05a7b297a..43825d45bc 100644 --- a/src/packages/backend/conat/files/test/local-path.test.ts +++ b/src/packages/backend/conat/files/test/local-path.test.ts @@ -4,7 +4,10 @@ import { fsClient } from "@cocalc/conat/files/fs"; import { randomId } from "@cocalc/conat/names"; import { before, after } from "@cocalc/backend/conat/test/setup"; import { uuid } from "@cocalc/util/misc"; -import { createPathFileserver, cleanupFileservers } from "./util"; +import { + createPathFileserver, + cleanupFileservers, +} from "@cocalc/backend/conat/files/test/util"; beforeAll(before); diff --git a/src/packages/backend/package.json b/src/packages/backend/package.json index 4a71ea0a22..48c602b8a1 100644 --- a/src/packages/backend/package.json +++ b/src/packages/backend/package.json @@ -13,10 +13,7 @@ "./auth/*": "./dist/auth/*.js", "./auth/tokens/*": "./dist/auth/tokens/*.js" }, - "keywords": [ - "utilities", - "cocalc" - ], + "keywords": ["utilities", "cocalc"], "scripts": { "preinstall": "npx only-allow pnpm", "clean": "rm -rf dist node_modules", @@ -24,7 +21,6 @@ "tsc": "pnpm exec tsc --watch --pretty --preserveWatchOutput", "test": "pnpm exec jest --forceExit", "test-conat": " pnpm exec jest --forceExit conat", - "testp": "pnpm exec jest --forceExit", "depcheck": "pnpx depcheck --ignores events", "prepublishOnly": "pnpm test", "conat-watch": "node ./bin/conat-watch.cjs", @@ -34,20 +30,13 @@ "conat-persist": "DEBUG=cocalc:* node ./bin/conat-persist.cjs", "conat-test-server": "node ./bin/conat-test-server.cjs" }, - "files": [ - "dist/**", - "bin/**", - "README.md", - "package.json" - ], + "files": ["dist/**", "bin/**", "README.md", "package.json"], "author": "SageMath, Inc.", "license": "SEE LICENSE.md", "dependencies": { "@cocalc/backend": "workspace:*", "@cocalc/conat": "workspace:*", "@cocalc/util": "workspace:*", - "@types/debug": "^4.1.12", - "@types/jest": "^29.5.14", "awaiting": "^3.0.0", "better-sqlite3": "^11.10.0", "chokidar": "^3.6.0", @@ -68,6 +57,8 @@ }, "homepage": "https://github.com/sagemathinc/cocalc/tree/master/src/packages/backend", "devDependencies": { + "@types/debug": "^4.1.12", + "@types/jest": "^29.5.14", "@types/node": "^18.16.14" } } diff --git a/src/packages/conat/files/fs.ts b/src/packages/conat/files/fs.ts index 2c30db142f..c522b29b24 100644 --- a/src/packages/conat/files/fs.ts +++ b/src/packages/conat/files/fs.ts @@ -252,15 +252,19 @@ export async function fsServer({ service, fs, client }: Options) { }; } +export type FilesystemClient = Filesystem & { + listing: (path: string) => Promise; +}; + export function fsClient({ client, subject, }: { client?: Client; subject: string; -}): Filesystem { +}): FilesystemClient { client ??= conat(); - let call = client.call(subject); + let call = client.call(subject); let constants: any = null; const stat0 = call.stat.bind(call); diff --git a/src/packages/conat/files/listing.ts b/src/packages/conat/files/listing.ts index a82c7afd20..c2d8313fdb 100644 --- a/src/packages/conat/files/listing.ts +++ b/src/packages/conat/files/listing.ts @@ -14,7 +14,7 @@ interface FileData { size: number; } -type Files = { [name: string]: FileData }; +export type Files = { [name: string]: FileData }; interface Options { path: string; diff --git a/src/packages/frontend/project/listing/use-listing.ts b/src/packages/frontend/project/listing/use-listing.ts new file mode 100644 index 0000000000..e29318d872 --- /dev/null +++ b/src/packages/frontend/project/listing/use-listing.ts @@ -0,0 +1,86 @@ +/* +A directory listing hook. +*/ + +import { useMemo, useState } from "react"; +import { DirectoryListingEntry } from "@cocalc/util/types"; +import useAsyncEffect from "use-async-effect"; +import { throttle } from "lodash"; +import { field_cmp } from "@cocalc/util/misc"; +import { type Files } from "@cocalc/conat/files/listing"; +import { type FilesystemClient } from "@cocalc/conat/files/fs"; + +const DEFAULT_THROTTLE_FILE_UPDATE = 500; + +type SortField = "name" | "mtime" | "size"; +type SortDirection = "inc" | "dec"; + +export default function useListing({ + fs, + path, + sortField = "name", + sortDirection = "inc", +}: { + fs: FilesystemClient; + path: string; + sortField?: SortField; + sortDirection?: SortDirection; +}): { listing: null | DirectoryListingEntry[]; error } { + const { files, error } = useFiles({ fs, path }); + + const listing = useMemo(() => { + if (files == null) { + return null; + } + const v: DirectoryListingEntry[] = []; + for (const name in files) { + v.push({ name, ...files[name] }); + } + v.sort(field_cmp("name")); + if (sortDirection == "dec") { + v.reverse(); + } + return v; + }, [sortField, sortDirection, files]); + + return { listing, error }; +} + +export function useFiles({ + fs, + path, + throttleUpdate = DEFAULT_THROTTLE_FILE_UPDATE, +}: { + fs: FilesystemClient; + path: string; + throttleUpdate?: number; +}): { files: Files | null; error } { + const [files, setFiles] = useState(null); + const [error, setError] = useState(null); + + useAsyncEffect(async () => { + let listing; + try { + listing = await fs.listing(path); + } catch (err) { + setError(err); + return; + } + + const update = () => { + setFiles({ ...listing.files }); + }; + update(); + + listing.on( + "change", + throttle(update, throttleUpdate, { leading: true, trailing: true }), + ); + + return () => { + listing.close(); + }; + }, [fs, path]); + + return { files, error }; +} diff --git a/src/packages/pnpm-lock.yaml b/src/packages/pnpm-lock.yaml index db43c158d4..901cc56769 100644 --- a/src/packages/pnpm-lock.yaml +++ b/src/packages/pnpm-lock.yaml @@ -87,12 +87,6 @@ importers: '@cocalc/util': specifier: workspace:* version: link:../util - '@types/debug': - specifier: ^4.1.12 - version: 4.1.12 - '@types/jest': - specifier: ^29.5.14 - version: 29.5.14 awaiting: specifier: ^3.0.0 version: 3.0.0 @@ -133,6 +127,12 @@ importers: specifier: ^0.0.10 version: 0.0.10 devDependencies: + '@types/debug': + specifier: ^4.1.12 + version: 4.1.12 + '@types/jest': + specifier: ^29.5.14 + version: 29.5.14 '@types/node': specifier: ^18.16.14 version: 18.19.118 @@ -1765,6 +1765,40 @@ importers: specifier: ^18.16.14 version: 18.19.118 + test: + dependencies: + '@cocalc/backend': + specifier: workspace:* + version: link:../backend + '@cocalc/conat': + specifier: workspace:* + version: link:../conat + '@cocalc/frontend': + specifier: workspace:* + version: link:../frontend + '@cocalc/util': + specifier: workspace:* + version: link:../util + devDependencies: + '@testing-library/jest-dom': + specifier: ^6.6.3 + version: 6.6.3 + '@testing-library/react': + specifier: ^16.3.0 + version: 16.3.0(@testing-library/dom@10.4.0)(@types/react-dom@19.1.6(@types/react@19.1.8))(@types/react@19.1.8)(react-dom@19.1.0(react@19.1.0))(react@19.1.0) + '@types/debug': + specifier: ^4.1.12 + version: 4.1.12 + '@types/jest': + specifier: ^29.5.14 + version: 29.5.14 + '@types/node': + specifier: ^18.16.14 + version: 18.19.118 + jest-environment-jsdom: + specifier: ^30.0.2 + version: 30.0.4 + util: dependencies: '@ant-design/colors': diff --git a/src/packages/test/jest.config.js b/src/packages/test/jest.config.js new file mode 100644 index 0000000000..3e9e290535 --- /dev/null +++ b/src/packages/test/jest.config.js @@ -0,0 +1,13 @@ +/** @type {import('ts-jest').JestConfigWithTsJest} */ +module.exports = { + preset: "ts-jest", + testEnvironment: "jsdom", + testEnvironmentOptions: { + // needed or jest imports the ts directly rather than the compiled + // dist exported from our package.json. Without this imports won't work. + // See https://jestjs.io/docs/configuration#testenvironment-string + customExportConditions: ["node", "node-addons"], + }, + testMatch: ["**/?(*.)+(spec|test).ts?(x)"], + setupFilesAfterEnv: ["./test/setup.js"], +}; diff --git a/src/packages/test/package.json b/src/packages/test/package.json new file mode 100644 index 0000000000..bed4234e0c --- /dev/null +++ b/src/packages/test/package.json @@ -0,0 +1,37 @@ +{ + "name": "@cocalc/test", + "version": "1.0.0", + "description": "CoCalc Integration Testing", + "exports": { + "./*": "./dist/*.js" + }, + "keywords": ["test", "cocalc"], + "scripts": { + "preinstall": "npx only-allow pnpm", + "clean": "rm -rf dist node_modules", + "build": "pnpm exec tsc --build", + "tsc": "pnpm exec tsc --watch --pretty --preserveWatchOutput", + "test": "pnpm exec jest --forceExit", + "depcheck": "pnpx depcheck --ignores events" + }, + "author": "SageMath, Inc.", + "license": "SEE LICENSE.md", + "dependencies": { + "@cocalc/backend": "workspace:*", + "@cocalc/conat": "workspace:*", + "@cocalc/frontend": "workspace:*", + "@cocalc/util": "workspace:*" + }, + "repository": { + "type": "git", + "url": "https://github.com/sagemathinc/cocalc" + }, + "homepage": "https://github.com/sagemathinc/cocalc/tree/master/src/packages/test", + "devDependencies": { + "@types/node": "^18.16.14", + "@types/debug": "^4.1.12", + "@types/jest": "^29.5.14", + "@testing-library/jest-dom": "^6.6.3", + "jest-environment-jsdom": "^30.0.2" + } +} diff --git a/src/packages/test/test/setup.js b/src/packages/test/test/setup.js new file mode 100644 index 0000000000..fb85d307c6 --- /dev/null +++ b/src/packages/test/test/setup.js @@ -0,0 +1,5 @@ +require("@testing-library/jest-dom"); +process.env.COCALC_TEST_MODE = true; + +global.TextEncoder = require("util").TextEncoder; +global.TextDecoder = require("util").TextDecoder; diff --git a/src/packages/test/tsconfig.json b/src/packages/test/tsconfig.json new file mode 100644 index 0000000000..6e3f1bef81 --- /dev/null +++ b/src/packages/test/tsconfig.json @@ -0,0 +1,22 @@ +{ + "extends": "../tsconfig.json", + "compilerOptions": { + "composite": true, + "incremental": true, + "rootDir": "./", + "outDir": "dist", + "sourceMap": true, + "lib": ["es5", "es6", "es2017", "dom"], + "declaration": true + }, + "exclude": ["dist", "node_modules"], + "types": ["jest", "@testing-library/jest-dom"], + "references": [ + { + "path": "../util", + "path": "../conat", + "path": "../backend", + "path": "../frontend" + } + ] +} diff --git a/src/packages/test/use-listing.test.ts b/src/packages/test/use-listing.test.ts new file mode 100644 index 0000000000..465d4d088a --- /dev/null +++ b/src/packages/test/use-listing.test.ts @@ -0,0 +1,33 @@ +import { renderHook } from "@testing-library/react"; +import { fsClient } from "@cocalc/conat/files/fs"; +import { before, after } from "@cocalc/backend/conat/test/setup"; +import { uuid } from "@cocalc/util/misc"; +import { + createPathFileserver, + cleanupFileservers, +} from "@cocalc/backend/conat/files/test/util"; +import { useFiles } from "@cocalc/frontend/project/listing/use-listing"; + +beforeAll(before); + +describe("use all the standard api functions of fs", () => { + const project_id = uuid(); + let fs, server; + it("creates fileserver service and fs client", async () => { + server = await createPathFileserver(); + fs = fsClient({ subject: `${server.service}.project-${project_id}` }); + }); + + it("test useFiles", async () => { + const f = () => { + return useFiles({ fs, path: "", throttleUpdate: 0 }); + }; + const { result } = renderHook(f); + expect(result.current).toEqual({ files: null, error: null }); + }); +}); + +afterAll(async () => { + await after(); + await cleanupFileservers(); +}); diff --git a/src/packages/util/async-utils.ts b/src/packages/util/async-utils.ts index ac40a124e1..d5142926e4 100644 --- a/src/packages/util/async-utils.ts +++ b/src/packages/util/async-utils.ts @@ -184,7 +184,11 @@ function captureStackWithoutPrinting() { If the obj throws 'closed' before the event is emitted, then this throws an error, since clearly event can never be emitted. */ -const DEBUG_ONCE = false; // log a better stack trace in some cases + +// Set DEBUG_ONCE to true and see a MUCH better stack trace about what +// caused once to throw in some cases! Do not leave this on though, +// since it uses extra time and memory grabbing a stack trace on every call. +const DEBUG_ONCE = false; export async function once( obj: EventEmitter, event: string, diff --git a/src/workspaces.py b/src/workspaces.py index 738782d291..53011ac7f6 100755 --- a/src/workspaces.py +++ b/src/workspaces.py @@ -126,6 +126,7 @@ def all_packages() -> List[str]: 'packages/file-server', 'packages/next', 'packages/hub', # hub won't build if next isn't already built + 'packages/test' ] for x in os.listdir('packages'): path = os.path.join("packages", x) From 8a226398f7301a8d4688e4425cd9756ebe54b637 Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 23 Jul 2025 17:40:50 +0000 Subject: [PATCH 073/270] test: actually unit testing the useFiles hook --- src/packages/conat/files/listing.ts | 5 + .../frontend/project/listing/use-listing.ts | 19 ++-- src/packages/test/use-listing.test.ts | 98 +++++++++++++++++-- 3 files changed, 107 insertions(+), 15 deletions(-) diff --git a/src/packages/conat/files/listing.ts b/src/packages/conat/files/listing.ts index c2d8313fdb..63d9bab6c9 100644 --- a/src/packages/conat/files/listing.ts +++ b/src/packages/conat/files/listing.ts @@ -86,13 +86,18 @@ export class Listing extends EventEmitter { } this.files[filename] = { mtime: stats.mtimeMs, size: stats.size }; } catch (err) { + if (this.files == null) { + return; + } if (err.code == "ENOENT") { // file deleted delete this.files[filename]; } else { + //if (!process.env.COCALC_TEST_MODE) { console.warn("WARNING:", err); // TODO: some other error -- e.g., network down or permissions, so we don't know anything. // Should we retry (?). + //} return; } } diff --git a/src/packages/frontend/project/listing/use-listing.ts b/src/packages/frontend/project/listing/use-listing.ts index e29318d872..d36c676e11 100644 --- a/src/packages/frontend/project/listing/use-listing.ts +++ b/src/packages/frontend/project/listing/use-listing.ts @@ -25,8 +25,12 @@ export default function useListing({ path: string; sortField?: SortField; sortDirection?: SortDirection; -}): { listing: null | DirectoryListingEntry[]; error } { - const { files, error } = useFiles({ fs, path }); +}): { + listing: null | DirectoryListingEntry[]; + error: null | Error; + refresh: () => void; +} { + const { files, error, refresh } = useFiles({ fs, path }); const listing = useMemo(() => { if (files == null) { @@ -43,7 +47,7 @@ export default function useListing({ return v; }, [sortField, sortDirection, files]); - return { listing, error }; + return { listing, error, refresh }; } export function useFiles({ @@ -54,16 +58,19 @@ export function useFiles({ fs: FilesystemClient; path: string; throttleUpdate?: number; -}): { files: Files | null; error } { +}): { files: Files | null; error: null | Error; refresh: () => void } { const [files, setFiles] = useState(null); const [error, setError] = useState(null); + const [counter, setCounter] = useState(0); useAsyncEffect(async () => { let listing; try { listing = await fs.listing(path); + setError(null); } catch (err) { setError(err); + setFiles(null); return; } @@ -80,7 +87,7 @@ export function useFiles({ return () => { listing.close(); }; - }, [fs, path]); + }, [fs, path, counter]); - return { files, error }; + return { files, error, refresh: () => setCounter(counter + 1) }; } diff --git a/src/packages/test/use-listing.test.ts b/src/packages/test/use-listing.test.ts index 465d4d088a..831a0fd692 100644 --- a/src/packages/test/use-listing.test.ts +++ b/src/packages/test/use-listing.test.ts @@ -1,6 +1,6 @@ -import { renderHook } from "@testing-library/react"; +import { act, renderHook, waitFor } from "@testing-library/react"; import { fsClient } from "@cocalc/conat/files/fs"; -import { before, after } from "@cocalc/backend/conat/test/setup"; +import { before, after, wait } from "@cocalc/backend/conat/test/setup"; import { uuid } from "@cocalc/util/misc"; import { createPathFileserver, @@ -10,7 +10,7 @@ import { useFiles } from "@cocalc/frontend/project/listing/use-listing"; beforeAll(before); -describe("use all the standard api functions of fs", () => { +describe("the useFiles hook", () => { const project_id = uuid(); let fs, server; it("creates fileserver service and fs client", async () => { @@ -18,12 +18,92 @@ describe("use all the standard api functions of fs", () => { fs = fsClient({ subject: `${server.service}.project-${project_id}` }); }); - it("test useFiles", async () => { - const f = () => { - return useFiles({ fs, path: "", throttleUpdate: 0 }); - }; - const { result } = renderHook(f); - expect(result.current).toEqual({ files: null, error: null }); + it("test useFiles and file creation", async () => { + let path = "", + fs2 = fs; + const { result, rerender } = renderHook(() => + useFiles({ fs: fs2, path, throttleUpdate: 0 }), + ); + + expect(result.current).toEqual({ + files: null, + error: null, + refresh: expect.any(Function), + }); + + // eventually it will be initialized to not be null + await waitFor(() => { + expect(result.current.files).not.toBeNull(); + }); + expect(result.current).toEqual({ + files: {}, + error: null, + refresh: expect.any(Function), + }); + + // now write a file + await act(async () => { + await fs.writeFile("hello.txt", "world"); + }); + + await waitFor(() => { + expect(result.current.files["hello.txt"]).toBeDefined(); + }); + + expect(result.current).toEqual({ + files: { + "hello.txt": { + size: 5, + mtime: expect.any(Number), + }, + }, + error: null, + refresh: expect.any(Function), + }); + + // change the path to one that does not exist and rerender, + // resulting in an ENOENT error + path = "scratch"; + rerender(); + await waitFor(() => { + expect(result.current.files?.["hello.txt"]).not.toBeDefined(); + }); + expect(result.current.error.code).toBe("ENOENT"); + + await act(async () => { + // create the path, a file in there, refresh and it works + await fs.mkdir(path); + await fs.writeFile("scratch/b.txt", "hi"); + result.current.refresh(); + }); + + await waitFor(() => { + expect(result.current).toEqual({ + files: { + "b.txt": { + size: 2, + mtime: expect.any(Number), + }, + }, + error: null, + refresh: expect.any(Function), + }); + }); + + // change fs and see the hook update + const project_id2 = uuid(); + fs2 = fsClient({ + subject: `${server.service}.project-${project_id2}`, + }); + path = ""; + rerender(); + await waitFor(() => { + expect(result.current).toEqual({ + files: {}, + error: null, + refresh: expect.any(Function), + }); + }); }); }); From db04858ddae2796c7a96863097b258b51b82b566 Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 23 Jul 2025 17:58:15 +0000 Subject: [PATCH 074/270] add basic test of use-listing --- src/packages/conat/core/client.ts | 8 +- .../frontend/project/listing/use-files.ts | 58 +++++++++ .../frontend/project/listing/use-listing.ts | 59 ++-------- .../listing/use-files.test.ts} | 10 +- .../test/project/listing/use-listing.test.ts | 110 ++++++++++++++++++ 5 files changed, 186 insertions(+), 59 deletions(-) create mode 100644 src/packages/frontend/project/listing/use-files.ts rename src/packages/test/{use-listing.test.ts => project/listing/use-files.test.ts} (90%) create mode 100644 src/packages/test/project/listing/use-listing.test.ts diff --git a/src/packages/conat/core/client.ts b/src/packages/conat/core/client.ts index e11b2ec0a7..adb360d216 100644 --- a/src/packages/conat/core/client.ts +++ b/src/packages/conat/core/client.ts @@ -1963,7 +1963,7 @@ export function messageData( export type Subscription = EventIterator; export class ConatError extends Error { - code: string | number; + code?: string | number; constructor(mesg: string, { code }) { super(mesg); this.code = code; @@ -1989,15 +1989,15 @@ function toConatError(socketIoError) { } } -export function headerToError(headers) { +export function headerToError(headers): ConatError { const err = Error(headers.error); if (headers.error_attrs) { for (const field in headers.error_attrs) { err[field] = headers.error_attrs[field]; } } - if (err['code'] === undefined && headers.code) { - err['code'] = headers.code; + if (err["code"] === undefined && headers.code) { + err["code"] = headers.code; } return err; } diff --git a/src/packages/frontend/project/listing/use-files.ts b/src/packages/frontend/project/listing/use-files.ts new file mode 100644 index 0000000000..f8c1b96aab --- /dev/null +++ b/src/packages/frontend/project/listing/use-files.ts @@ -0,0 +1,58 @@ +/* +Hook that provides all files in a directory via a Conat FilesystemClient. +This automatically updates when files change. + +TESTS: See packages/test/project/listing/ + +*/ + +import useAsyncEffect from "use-async-effect"; +import { useState } from "react"; +import { throttle } from "lodash"; +import { type Files } from "@cocalc/conat/files/listing"; +import { type FilesystemClient } from "@cocalc/conat/files/fs"; +import { type ConatError } from "@cocalc/conat/core/client"; + +const DEFAULT_THROTTLE_FILE_UPDATE = 500; + +export default function useFiles({ + fs, + path, + throttleUpdate = DEFAULT_THROTTLE_FILE_UPDATE, +}: { + fs: FilesystemClient; + path: string; + throttleUpdate?: number; +}): { files: Files | null; error: null | ConatError; refresh: () => void } { + const [files, setFiles] = useState(null); + const [error, setError] = useState(null); + const [counter, setCounter] = useState(0); + + useAsyncEffect(async () => { + let listing; + try { + listing = await fs.listing(path); + setError(null); + } catch (err) { + setError(err); + setFiles(null); + return; + } + + const update = () => { + setFiles({ ...listing.files }); + }; + update(); + + listing.on( + "change", + throttle(update, throttleUpdate, { leading: true, trailing: true }), + ); + + return () => { + listing.close(); + }; + }, [fs, path, counter]); + + return { files, error, refresh: () => setCounter(counter + 1) }; +} diff --git a/src/packages/frontend/project/listing/use-listing.ts b/src/packages/frontend/project/listing/use-listing.ts index d36c676e11..43cfd4207d 100644 --- a/src/packages/frontend/project/listing/use-listing.ts +++ b/src/packages/frontend/project/listing/use-listing.ts @@ -1,16 +1,15 @@ /* A directory listing hook. + +TESTS: See packages/test/project/listing/ */ -import { useMemo, useState } from "react"; +import { useMemo } from "react"; import { DirectoryListingEntry } from "@cocalc/util/types"; -import useAsyncEffect from "use-async-effect"; -import { throttle } from "lodash"; import { field_cmp } from "@cocalc/util/misc"; -import { type Files } from "@cocalc/conat/files/listing"; +import useFiles from "./use-files"; import { type FilesystemClient } from "@cocalc/conat/files/fs"; - -const DEFAULT_THROTTLE_FILE_UPDATE = 500; +import { type ConatError } from "@cocalc/conat/core/client"; type SortField = "name" | "mtime" | "size"; type SortDirection = "inc" | "dec"; @@ -20,17 +19,19 @@ export default function useListing({ path, sortField = "name", sortDirection = "inc", + throttleUpdate, }: { fs: FilesystemClient; path: string; sortField?: SortField; sortDirection?: SortDirection; + throttleUpdate?: number; }): { listing: null | DirectoryListingEntry[]; - error: null | Error; + error: null | ConatError; refresh: () => void; } { - const { files, error, refresh } = useFiles({ fs, path }); + const { files, error, refresh } = useFiles({ fs, path, throttleUpdate }); const listing = useMemo(() => { if (files == null) { @@ -49,45 +50,3 @@ export default function useListing({ return { listing, error, refresh }; } - -export function useFiles({ - fs, - path, - throttleUpdate = DEFAULT_THROTTLE_FILE_UPDATE, -}: { - fs: FilesystemClient; - path: string; - throttleUpdate?: number; -}): { files: Files | null; error: null | Error; refresh: () => void } { - const [files, setFiles] = useState(null); - const [error, setError] = useState(null); - const [counter, setCounter] = useState(0); - - useAsyncEffect(async () => { - let listing; - try { - listing = await fs.listing(path); - setError(null); - } catch (err) { - setError(err); - setFiles(null); - return; - } - - const update = () => { - setFiles({ ...listing.files }); - }; - update(); - - listing.on( - "change", - throttle(update, throttleUpdate, { leading: true, trailing: true }), - ); - - return () => { - listing.close(); - }; - }, [fs, path, counter]); - - return { files, error, refresh: () => setCounter(counter + 1) }; -} diff --git a/src/packages/test/use-listing.test.ts b/src/packages/test/project/listing/use-files.test.ts similarity index 90% rename from src/packages/test/use-listing.test.ts rename to src/packages/test/project/listing/use-files.test.ts index 831a0fd692..4dc27fe789 100644 --- a/src/packages/test/use-listing.test.ts +++ b/src/packages/test/project/listing/use-files.test.ts @@ -1,12 +1,12 @@ import { act, renderHook, waitFor } from "@testing-library/react"; import { fsClient } from "@cocalc/conat/files/fs"; -import { before, after, wait } from "@cocalc/backend/conat/test/setup"; +import { before, after } from "@cocalc/backend/conat/test/setup"; import { uuid } from "@cocalc/util/misc"; import { createPathFileserver, cleanupFileservers, } from "@cocalc/backend/conat/files/test/util"; -import { useFiles } from "@cocalc/frontend/project/listing/use-listing"; +import useFiles from "@cocalc/frontend/project/listing/use-files"; beforeAll(before); @@ -41,13 +41,13 @@ describe("the useFiles hook", () => { refresh: expect.any(Function), }); - // now write a file + // now create a file await act(async () => { await fs.writeFile("hello.txt", "world"); }); await waitFor(() => { - expect(result.current.files["hello.txt"]).toBeDefined(); + expect(result.current.files?.["hello.txt"]).toBeDefined(); }); expect(result.current).toEqual({ @@ -68,7 +68,7 @@ describe("the useFiles hook", () => { await waitFor(() => { expect(result.current.files?.["hello.txt"]).not.toBeDefined(); }); - expect(result.current.error.code).toBe("ENOENT"); + expect(result.current.error?.code).toBe("ENOENT"); await act(async () => { // create the path, a file in there, refresh and it works diff --git a/src/packages/test/project/listing/use-listing.test.ts b/src/packages/test/project/listing/use-listing.test.ts new file mode 100644 index 0000000000..bb0b6d2117 --- /dev/null +++ b/src/packages/test/project/listing/use-listing.test.ts @@ -0,0 +1,110 @@ +import { act, renderHook, waitFor } from "@testing-library/react"; +import { fsClient } from "@cocalc/conat/files/fs"; +import { before, after } from "@cocalc/backend/conat/test/setup"; +import { uuid } from "@cocalc/util/misc"; +import { + createPathFileserver, + cleanupFileservers, +} from "@cocalc/backend/conat/files/test/util"; +import useListing from "@cocalc/frontend/project/listing/use-listing"; + +beforeAll(before); + +describe("the useListing hook", () => { + const project_id = uuid(); + let fs, server; + it("creates fileserver service and fs client", async () => { + server = await createPathFileserver(); + fs = fsClient({ subject: `${server.service}.project-${project_id}` }); + }); + + it("test useListing and file creation", async () => { + let path = "", + fs2 = fs; + const { result, rerender } = renderHook(() => + useListing({ fs: fs2, path, throttleUpdate: 0 }), + ); + + expect(result.current).toEqual({ + listing: null, + error: null, + refresh: expect.any(Function), + }); + + // eventually it will be initialized to not be null + await waitFor(() => { + expect(result.current.listing).not.toBeNull(); + }); + + expect(result.current).toEqual({ + listing: [], + error: null, + refresh: expect.any(Function), + }); + + // now create a file + await act(async () => { + await fs.writeFile("hello.txt", "world"); + }); + + await waitFor(() => { + expect(result.current.listing?.length).toEqual(1); + }); + + expect(result.current).toEqual({ + listing: [{ name: "hello.txt", size: 5, mtime: expect.any(Number) }], + error: null, + refresh: expect.any(Function), + }); + + // change the path to one that does not exist and rerender, + // resulting in an ENOENT error + path = "scratch"; + rerender(); + await waitFor(() => { + expect(result.current.listing).toBeNull(); + expect(result.current.error?.code).toBe("ENOENT"); + }); + + await act(async () => { + // create the path, a file in there, refresh and it works + await fs.mkdir(path); + await fs.writeFile("scratch/b.txt", "hi"); + result.current.refresh(); + }); + + await waitFor(() => { + expect(result.current).toEqual({ + listing: [ + { + name: "b.txt", + size: 2, + mtime: expect.any(Number), + }, + ], + error: null, + refresh: expect.any(Function), + }); + }); + + // change fs and see the hook update + const project_id2 = uuid(); + fs2 = fsClient({ + subject: `${server.service}.project-${project_id2}`, + }); + path = ""; + rerender(); + await waitFor(() => { + expect(result.current).toEqual({ + listing: [], + error: null, + refresh: expect.any(Function), + }); + }); + }); +}); + +afterAll(async () => { + await after(); + await cleanupFileservers(); +}); From 73b70800834d43b491aa644385242060c8359e2a Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 23 Jul 2025 18:15:33 +0000 Subject: [PATCH 075/270] unit tests of use-listing that involve sorting --- .../frontend/project/listing/use-listing.ts | 14 ++- .../test/project/listing/use-listing.test.ts | 95 +++++++++++++++++++ 2 files changed, 105 insertions(+), 4 deletions(-) diff --git a/src/packages/frontend/project/listing/use-listing.ts b/src/packages/frontend/project/listing/use-listing.ts index 43cfd4207d..5418f50cba 100644 --- a/src/packages/frontend/project/listing/use-listing.ts +++ b/src/packages/frontend/project/listing/use-listing.ts @@ -12,13 +12,13 @@ import { type FilesystemClient } from "@cocalc/conat/files/fs"; import { type ConatError } from "@cocalc/conat/core/client"; type SortField = "name" | "mtime" | "size"; -type SortDirection = "inc" | "dec"; +type SortDirection = "asc" | "desc"; export default function useListing({ fs, path, sortField = "name", - sortDirection = "inc", + sortDirection = "asc", throttleUpdate, }: { fs: FilesystemClient; @@ -41,9 +41,15 @@ export default function useListing({ for (const name in files) { v.push({ name, ...files[name] }); } - v.sort(field_cmp("name")); - if (sortDirection == "dec") { + if (sortField != "name" && sortField != "mtime" && sortField != "size") { + console.warn(`invalid sort field: '${sortField}'`); + } + v.sort(field_cmp(sortField)); + if (sortDirection == "desc") { v.reverse(); + } else if (sortDirection == "asc") { + } else { + console.warn(`invalid sort direction: '${sortDirection}'`); } return v; }, [sortField, sortDirection, files]); diff --git a/src/packages/test/project/listing/use-listing.test.ts b/src/packages/test/project/listing/use-listing.test.ts index bb0b6d2117..c6d86dbeeb 100644 --- a/src/packages/test/project/listing/use-listing.test.ts +++ b/src/packages/test/project/listing/use-listing.test.ts @@ -104,6 +104,101 @@ describe("the useListing hook", () => { }); }); +describe("test sorting many files with useListing", () => { + const project_id = uuid(); + let fs, server; + it("creates fileserver service and fs client", async () => { + server = await createPathFileserver(); + fs = fsClient({ subject: `${server.service}.project-${project_id}` }); + }); + + it("create some files", async () => { + await fs.writeFile("a.txt", "abc"); + await fs.writeFile("b.txt", "b"); + await fs.writeFile("huge.txt", "b".repeat(1000)); + + // make b.txt old + await fs.utimes( + "b.txt", + (Date.now() - 60_000) / 1000, + (Date.now() - 60_000) / 1000, + ); + }); + + it("test useListing with many files and sorting", async () => { + let path = "", + sortField = "name", + sortDirection = "asc"; + const { result, rerender } = renderHook(() => + useListing({ fs, path, throttleUpdate: 0, sortField, sortDirection }), + ); + + await waitFor(() => { + expect(result.current.listing?.length).toEqual(3); + }); + expect(result.current.listing.map(({ name }) => name)).toEqual([ + "a.txt", + "b.txt", + "huge.txt", + ]); + + sortDirection = "desc"; + sortField = "name"; + rerender(); + await waitFor(() => { + expect(result.current.listing.map(({ name }) => name)).toEqual([ + "huge.txt", + "b.txt", + "a.txt", + ]); + }); + + sortDirection = "asc"; + sortField = "mtime"; + rerender(); + await waitFor(() => { + expect(result.current.listing.map(({ name }) => name)).toEqual([ + "b.txt", + "a.txt", + "huge.txt", + ]); + }); + + sortDirection = "desc"; + sortField = "mtime"; + rerender(); + await waitFor(() => { + expect(result.current.listing.map(({ name }) => name)).toEqual([ + "huge.txt", + "a.txt", + "b.txt", + ]); + }); + + sortDirection = "asc"; + sortField = "size"; + rerender(); + await waitFor(() => { + expect(result.current.listing.map(({ name }) => name)).toEqual([ + "b.txt", + "a.txt", + "huge.txt", + ]); + }); + + sortDirection = "desc"; + sortField = "size"; + rerender(); + await waitFor(() => { + expect(result.current.listing.map(({ name }) => name)).toEqual([ + "huge.txt", + "a.txt", + "b.txt", + ]); + }); + }); +}); + afterAll(async () => { await after(); await cleanupFileservers(); From 7b3c4dd81ae424a54d3069a8866b8804a308027c Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 23 Jul 2025 22:10:14 +0000 Subject: [PATCH 076/270] frontend listings: first pass at using the new fs interface and hooks (not done and the old stuff is still being computed) --- src/packages/backend/files/sandbox/index.ts | 5 + src/packages/conat/files/fs.ts | 30 +++- src/packages/conat/files/listing.ts | 65 ++++++- .../frontend/project/explorer/explorer.tsx | 165 +++++------------- .../explorer/file-listing/file-listing.tsx | 113 ++++++------ .../frontend/project/listing/use-files.ts | 74 ++++---- .../frontend/project/listing/use-fs.ts | 20 +++ .../frontend/project/listing/use-listing.ts | 14 +- .../test/project/listing/use-files.test.ts | 2 + .../test/project/listing/use-listing.test.ts | 34 ++-- 10 files changed, 289 insertions(+), 233 deletions(-) create mode 100644 src/packages/frontend/project/listing/use-fs.ts diff --git a/src/packages/backend/files/sandbox/index.ts b/src/packages/backend/files/sandbox/index.ts index b65ba1ff3f..aa09b12c25 100644 --- a/src/packages/backend/files/sandbox/index.ts +++ b/src/packages/backend/files/sandbox/index.ts @@ -50,6 +50,7 @@ import { lstat, readdir, readFile, + readlink, realpath, rename, rm, @@ -241,6 +242,10 @@ export class SandboxedFilesystem { return await readdir(await this.safeAbsPath(path)); }; + readlink = async (path: string): Promise => { + return await readlink(await this.safeAbsPath(path)); + }; + realpath = async (path: string): Promise => { const x = await realpath(await this.safeAbsPath(path)); return x.slice(this.path.length + 1); diff --git a/src/packages/conat/files/fs.ts b/src/packages/conat/files/fs.ts index c522b29b24..bd60f4763c 100644 --- a/src/packages/conat/files/fs.ts +++ b/src/packages/conat/files/fs.ts @@ -5,7 +5,7 @@ import { watchClient, type WatchIterator, } from "@cocalc/conat/files/watch"; -import listing, { type Listing } from "./listing"; +import listing, { type Listing, type FileTypeLabel } from "./listing"; export const DEFAULT_FILE_SERVICE = "fs"; @@ -43,6 +43,7 @@ export interface Filesystem { mkdir: (path: string, options?) => Promise; readFile: (path: string, encoding?: any) => Promise; readdir: (path: string) => Promise; + readlink: (path: string) => Promise; realpath: (path: string) => Promise; rename: (oldPath: string, newPath: string) => Promise; rm: (path: string, options?) => Promise; @@ -135,6 +136,26 @@ class Stats { isSocket = () => (this.mode & this.constants.S_IFMT) === this.constants.S_IFSOCK; + + get type(): FileTypeLabel { + switch (this.mode & this.constants.S_IFMT) { + case this.constants.S_IFLNK: + return "l"; + case this.constants.S_IFREG: + return "f"; + case this.constants.S_IFDIR: + return "d"; + case this.constants.S_IFBLK: + return "b"; + case this.constants.S_IFCHR: + return "c"; + case this.constants.S_IFSOCK: + return "s"; + case this.constants.S_IFIFO: + return "p"; + } + return "f"; + } } interface Options { @@ -184,6 +205,9 @@ export async function fsServer({ service, fs, client }: Options) { async readdir(path: string) { return await (await fs(this.subject)).readdir(path); }, + async readlink(path: string) { + return await (await fs(this.subject)).readlink(path); + }, async realpath(path: string) { return await (await fs(this.subject)).realpath(path); }, @@ -252,8 +276,10 @@ export async function fsServer({ service, fs, client }: Options) { }; } -export type FilesystemClient = Filesystem & { +export type FilesystemClient = Omit, "lstat"> & { listing: (path: string) => Promise; + stat: (path: string) => Promise; + lstat: (path: string) => Promise; }; export function fsClient({ diff --git a/src/packages/conat/files/listing.ts b/src/packages/conat/files/listing.ts index 63d9bab6c9..d45bec5401 100644 --- a/src/packages/conat/files/listing.ts +++ b/src/packages/conat/files/listing.ts @@ -2,23 +2,44 @@ Directory Listing Tests in packages/backend/conat/files/test/listing.test.ts + + */ import { EventEmitter } from "events"; import { join } from "path"; -import { type Filesystem } from "./fs"; +import { type FilesystemClient } from "./fs"; import { EventIterator } from "@cocalc/util/event-iterator"; +export type FileTypeLabel = "f" | "d" | "l" | "b" | "c" | "s" | "p"; + +export const typeDescription = { + f: "regular file", + d: "directory", + l: "symlink", + b: "block device", + c: "character device", + s: "socket", + p: "fifo", +}; + interface FileData { mtime: number; size: number; + // isdir = mainly for backward compat: + isdir?: boolean; + // issymlink = mainly for backward compat: + issymlink?: boolean; + link_target?: string; + // see typeDescription above. + type?: FileTypeLabel; } export type Files = { [name: string]: FileData }; interface Options { path: string; - fs: Filesystem; + fs: FilesystemClient; } export default async function listing(opts: Options): Promise { @@ -48,6 +69,7 @@ export class Listing extends EventEmitter { close = () => { this.emit("closed"); + this.removeAllListeners(); this.iters.map((iter) => iter.end()); this.iters.length = 0; this.watch?.close(); @@ -80,11 +102,26 @@ export class Listing extends EventEmitter { return; } try { - const stats = await this.opts.fs.stat(join(this.opts.path, filename)); + const stats = await this.opts.fs.lstat(join(this.opts.path, filename)); if (this.files == null) { return; } - this.files[filename] = { mtime: stats.mtimeMs, size: stats.size }; + const data: FileData = { + mtime: stats.mtimeMs / 1000, + size: stats.size, + type: stats.type, + }; + if (stats.isSymbolicLink()) { + // resolve target. + data.link_target = await this.opts.fs.readlink( + join(this.opts.path, filename), + ); + data.issymlink = true; + } + if (stats.isDirectory()) { + data.isdir = true; + } + this.files[filename] = data; } catch (err) { if (this.files == null) { return; @@ -106,10 +143,13 @@ export class Listing extends EventEmitter { } async function getListing( - fs: Filesystem, + fs: FilesystemClient, path: string, ): Promise<{ files: Files; truncated: boolean }> { - const { stdout, truncated } = await fs.find(path, "%f\\0%T@\\0%s\n"); + const { stdout, truncated } = await fs.find( + path, + "%f\\0%T@\\0%s\\0%y\\0%l\n", + ); const buf = Buffer.from(stdout); const files: Files = {}; // todo -- what about non-utf8...? @@ -122,9 +162,18 @@ async function getListing( try { const v = line.split("\0"); const name = v[0]; - const mtime = parseFloat(v[1]) * 1000; + const mtime = parseFloat(v[1]); const size = parseInt(v[2]); - files[name] = { mtime, size }; + files[name] = { mtime, size, type: v[3] as FileTypeLabel }; + if (v[3] == "l") { + files[name].issymlink = true; + } + if (v[3] == "d") { + files[name].isdir = true; + } + if (v[4]) { + files[name].link_target = v[4]; + } } catch {} } return { files, truncated }; diff --git a/src/packages/frontend/project/explorer/explorer.tsx b/src/packages/frontend/project/explorer/explorer.tsx index 6ea7f84b1b..e4818639cd 100644 --- a/src/packages/frontend/project/explorer/explorer.tsx +++ b/src/packages/frontend/project/explorer/explorer.tsx @@ -3,11 +3,9 @@ * License: MS-RSL – see LICENSE.md for details */ -import { Alert } from "antd"; import * as immutable from "immutable"; import * as _ from "lodash"; import React from "react"; -import { FormattedMessage } from "react-intl"; import { UsersViewing } from "@cocalc/frontend/account/avatar/users-viewing"; import { Button, ButtonGroup, Col, Row } from "@cocalc/frontend/antd-bootstrap"; import { @@ -24,7 +22,6 @@ import { ErrorDisplay, Icon, Loading, - Paragraph, SettingBox, } from "@cocalc/frontend/components"; import { ComputeServerDocStatus } from "@cocalc/frontend/compute/doc-status"; @@ -45,7 +42,6 @@ import { useProjectContext } from "../context"; import { AccessErrors } from "./access-errors"; import { ActionBar } from "./action-bar"; import { ActionBox } from "./action-box"; -import { FetchDirectoryErrors } from "./fetch-directory-errors"; import { FileListing } from "./file-listing"; import { default_ext } from "./file-listing/utils"; import { MiscSideButtons } from "./misc-side-buttons"; @@ -432,117 +428,45 @@ const Explorer0 = rclass( return ; } - render_file_listing( - listing: ListingItem[] | undefined, - file_map, - fetch_directory_error: any, - project_is_running: boolean, - ) { - if (fetch_directory_error) { - return ( -
- -
- -
- ); - } else if (listing != undefined) { - return ( - this.props.actions.fetch_directory_listing(), + }} + config={{ clickable: ".upload-button" }} + style={{ + flex: "1 0 auto", + display: "flex", + flexDirection: "column", + }} + className="smc-vfill" + > + this.props.actions.fetch_directory_listing(), - }} - config={{ clickable: ".upload-button" }} - style={{ - flex: "1 0 auto", - display: "flex", - flexDirection: "column", - }} - className="smc-vfill" - > - - - ); - } else { - if (project_is_running) { - // ensure directory listing starts getting computed. - redux.getProjectStore(this.props.project_id).get_listings(); - return ( -
- -
- ); - } else { - return ( - } - style={{ textAlign: "center" }} - showIcon - description={ - - start this project.`} - values={{ - a: (c) => ( - { - redux - .getActions("projects") - .start_project(this.props.project_id); - }} - > - {c} - - ), - }} - /> - - } - /> - ); - } - } + shift_is_down={this.state.shift_is_down} + sort_by={this.props.actions.set_sorted_file_column} + other_settings={this.props.other_settings} + library={this.props.library} + redux={redux} + last_scroll_top={this.props.file_listing_scroll_top} + configuration_main={this.props.configuration?.get("main")} + show_hidden={this.props.show_hidden} + show_masked={this.props.show_masked} + /> + + ); } file_listing_page_size() { @@ -714,7 +638,6 @@ const Explorer0 = rclass( const displayed_listing = this.props.displayed_listing; const { listing, file_map } = displayed_listing; - const directory_error = displayed_listing.error; const file_listing_page_size = this.file_listing_page_size(); if (listing != undefined) { @@ -786,17 +709,7 @@ const Explorer0 = rclass( padding: "0 5px 5px 5px", }} > - {this.render_file_listing( - visible_listing, - file_map, - directory_error, - project_is_running, - )} - {listing != undefined - ? this.render_paging_buttons( - Math.ceil(listing.length / file_listing_page_size), - ) - : undefined} + {this.render_file_listing()} ; listing: any[]; - file_map: object; file_search: string; checked_files: immutable.Set; current_path: string; @@ -55,6 +58,11 @@ interface Props { last_scroll_top?: number; configuration_main?: MainConfiguration; isRunning?: boolean; // true if this project is running + + show_hidden?: boolean; + show_masked?: boolean; + + stale?: boolean; } export function watchFiles({ actions, current_path }): void { @@ -67,13 +75,48 @@ export function watchFiles({ actions, current_path }): void { } } -export const FileListing: React.FC = ({ +function sortDesc(active_file_sort?): { + sortField: SortField; + sortDirection: "asc" | "desc"; +} { + const { column_name, is_descending } = active_file_sort?.toJS() ?? { + column_name: "name", + is_descending: false, + }; + if (column_name == "time") { + return { + sortField: "mtime", + sortDirection: is_descending ? "asc" : "desc", + }; + } + return { + sortField: column_name, + sortDirection: is_descending ? "desc" : "asc", + }; +} + +export function FileListing(props) { + const fs = useFs({ project_id: props.project_id }); + const { listing, error } = useListing({ + fs, + path: props.current_path, + ...sortDesc(props.active_file_sort), + }); + if (error) { + return ; + } + if (listing == null) { + return ; + } + return ; +} + +function FileListing0({ actions, redux, name, active_file_sort, listing, - file_map, checked_files, current_path, create_folder, @@ -85,8 +128,16 @@ export const FileListing: React.FC = ({ configuration_main, file_search = "", isRunning, -}: Props) => { - const [starting, setStarting] = useState(false); + show_hidden, + stale, + // show_masked, +}: Props) { + if (!show_hidden) { + listing = listing.filter((x) => !x.name.startsWith(".")); + } + if (file_search) { + listing = listing.filter((x) => x.name.includes(file_search)); + } const prev_current_path = usePrevious(current_path); @@ -135,7 +186,6 @@ export const FileListing: React.FC = ({ ): Rendered { const checked = checked_files.has(misc.path_to_file(current_path, name)); const color = misc.rowBackground({ index, checked }); - const { is_public } = file_map[name]; return ( = ({ name={name} display_name={display_name} time={time} - size={size} + size={isdir ? undefined : size} issymlink={issymlink} color={color} selected={ @@ -151,7 +201,7 @@ export const FileListing: React.FC = ({ } mask={mask} public_data={public_data} - is_public={is_public} + is_public={false} checked={checked} key={index} current_path={current_path} @@ -188,7 +238,7 @@ export const FileListing: React.FC = ({ return ( { const a = listing[index]; @@ -236,44 +286,9 @@ export const FileListing: React.FC = ({ ); } - if (!isRunning && listing.length == 0) { - return ( - - { - if (starting) return; - try { - setStarting(true); - await actions.fetch_directory_listing_directly( - current_path, - true, - ); - } finally { - setStarting(false); - } - }} - > - Start this project to see your files. - {starting && } - - - } - /> - ); - } - return ( <> - {!isRunning && listing.length > 0 && ( + {stale && (
@@ -318,4 +333,4 @@ export const FileListing: React.FC = ({ ); -}; +} diff --git a/src/packages/frontend/project/listing/use-files.ts b/src/packages/frontend/project/listing/use-files.ts index f8c1b96aab..925a284894 100644 --- a/src/packages/frontend/project/listing/use-files.ts +++ b/src/packages/frontend/project/listing/use-files.ts @@ -7,11 +7,12 @@ TESTS: See packages/test/project/listing/ */ import useAsyncEffect from "use-async-effect"; -import { useState } from "react"; +import { useRef, useState } from "react"; import { throttle } from "lodash"; import { type Files } from "@cocalc/conat/files/listing"; import { type FilesystemClient } from "@cocalc/conat/files/fs"; import { type ConatError } from "@cocalc/conat/core/client"; +import useCounter from "@cocalc/frontend/app-framework/counter-hook"; const DEFAULT_THROTTLE_FILE_UPDATE = 500; @@ -20,39 +21,50 @@ export default function useFiles({ path, throttleUpdate = DEFAULT_THROTTLE_FILE_UPDATE, }: { - fs: FilesystemClient; + // fs = undefined is supported and just waits until you provide a fs that is defined + fs?: FilesystemClient | null; path: string; throttleUpdate?: number; }): { files: Files | null; error: null | ConatError; refresh: () => void } { const [files, setFiles] = useState(null); const [error, setError] = useState(null); - const [counter, setCounter] = useState(0); - - useAsyncEffect(async () => { - let listing; - try { - listing = await fs.listing(path); - setError(null); - } catch (err) { - setError(err); - setFiles(null); - return; - } - - const update = () => { - setFiles({ ...listing.files }); - }; - update(); - - listing.on( - "change", - throttle(update, throttleUpdate, { leading: true, trailing: true }), - ); - - return () => { - listing.close(); - }; - }, [fs, path, counter]); - - return { files, error, refresh: () => setCounter(counter + 1) }; + const { val: counter, inc: refresh } = useCounter(); + const listingRef = useRef(null); + + useAsyncEffect( + async () => { + if (fs == null) { + setError(null); + setFiles(null); + return; + } + let listing; + try { + listing = await fs.listing(path); + listingRef.current = listing; + setError(null); + } catch (err) { + setError(err); + setFiles(null); + return; + } + + const update = () => { + setFiles({ ...listing.files }); + }; + update(); + + listing.on( + "change", + throttle(update, throttleUpdate, { leading: true, trailing: true }), + ); + }, + () => { + listingRef.current?.close(); + delete listingRef.current; + }, + [fs, path, counter], + ); + + return { files, error, refresh }; } diff --git a/src/packages/frontend/project/listing/use-fs.ts b/src/packages/frontend/project/listing/use-fs.ts new file mode 100644 index 0000000000..ff76d79801 --- /dev/null +++ b/src/packages/frontend/project/listing/use-fs.ts @@ -0,0 +1,20 @@ +/* +Hook for getting a FilesystemClient. +*/ +import { webapp_client } from "@cocalc/frontend/webapp-client"; +import { type FilesystemClient } from "@cocalc/conat/files/fs"; +import { useState } from "react"; + +// this will probably get more complicated temporarily when we +// are transitioning between filesystems (hence why we return null in +// the typing for now) +export default function useFs({ + project_id, +}: { + project_id: string; +}): FilesystemClient | null { + const [fs] = useState(() => + webapp_client.conat_client.conat().fs({ project_id }), + ); + return fs; +} diff --git a/src/packages/frontend/project/listing/use-listing.ts b/src/packages/frontend/project/listing/use-listing.ts index 5418f50cba..75ffc37f40 100644 --- a/src/packages/frontend/project/listing/use-listing.ts +++ b/src/packages/frontend/project/listing/use-listing.ts @@ -11,8 +11,8 @@ import useFiles from "./use-files"; import { type FilesystemClient } from "@cocalc/conat/files/fs"; import { type ConatError } from "@cocalc/conat/core/client"; -type SortField = "name" | "mtime" | "size"; -type SortDirection = "asc" | "desc"; +export type SortField = "name" | "mtime" | "size" | "type"; +export type SortDirection = "asc" | "desc"; export default function useListing({ fs, @@ -21,7 +21,8 @@ export default function useListing({ sortDirection = "asc", throttleUpdate, }: { - fs: FilesystemClient; + // fs = undefined is supported and just waits until you provide a fs that is defined + fs?: FilesystemClient | null; path: string; sortField?: SortField; sortDirection?: SortDirection; @@ -41,7 +42,12 @@ export default function useListing({ for (const name in files) { v.push({ name, ...files[name] }); } - if (sortField != "name" && sortField != "mtime" && sortField != "size") { + if ( + sortField != "name" && + sortField != "mtime" && + sortField != "size" && + sortField != "type" + ) { console.warn(`invalid sort field: '${sortField}'`); } v.sort(field_cmp(sortField)); diff --git a/src/packages/test/project/listing/use-files.test.ts b/src/packages/test/project/listing/use-files.test.ts index 4dc27fe789..f36ad76040 100644 --- a/src/packages/test/project/listing/use-files.test.ts +++ b/src/packages/test/project/listing/use-files.test.ts @@ -55,6 +55,7 @@ describe("the useFiles hook", () => { "hello.txt": { size: 5, mtime: expect.any(Number), + type: "f", }, }, error: null, @@ -83,6 +84,7 @@ describe("the useFiles hook", () => { "b.txt": { size: 2, mtime: expect.any(Number), + type: "f", }, }, error: null, diff --git a/src/packages/test/project/listing/use-listing.test.ts b/src/packages/test/project/listing/use-listing.test.ts index c6d86dbeeb..37807219dc 100644 --- a/src/packages/test/project/listing/use-listing.test.ts +++ b/src/packages/test/project/listing/use-listing.test.ts @@ -6,7 +6,11 @@ import { createPathFileserver, cleanupFileservers, } from "@cocalc/backend/conat/files/test/util"; -import useListing from "@cocalc/frontend/project/listing/use-listing"; +import useListing, { + type SortField, + type SortDirection, +} from "@cocalc/frontend/project/listing/use-listing"; +import { type FilesystemClient } from "@cocalc/conat/files/fs"; beforeAll(before); @@ -20,18 +24,19 @@ describe("the useListing hook", () => { it("test useListing and file creation", async () => { let path = "", - fs2 = fs; + fs2: FilesystemClient | undefined = undefined; const { result, rerender } = renderHook(() => useListing({ fs: fs2, path, throttleUpdate: 0 }), ); - expect(result.current).toEqual({ listing: null, error: null, refresh: expect.any(Function), }); + fs2 = fs; + rerender(); - // eventually it will be initialized to not be null + // now that fs2 is set, eventually it will be initialized to not be null await waitFor(() => { expect(result.current.listing).not.toBeNull(); }); @@ -52,7 +57,9 @@ describe("the useListing hook", () => { }); expect(result.current).toEqual({ - listing: [{ name: "hello.txt", size: 5, mtime: expect.any(Number) }], + listing: [ + { name: "hello.txt", size: 5, mtime: expect.any(Number), type: "f" }, + ], error: null, refresh: expect.any(Function), }); @@ -79,6 +86,7 @@ describe("the useListing hook", () => { { name: "b.txt", size: 2, + type: "f", mtime: expect.any(Number), }, ], @@ -127,8 +135,8 @@ describe("test sorting many files with useListing", () => { it("test useListing with many files and sorting", async () => { let path = "", - sortField = "name", - sortDirection = "asc"; + sortField: SortField = "name", + sortDirection: SortDirection = "asc"; const { result, rerender } = renderHook(() => useListing({ fs, path, throttleUpdate: 0, sortField, sortDirection }), ); @@ -136,7 +144,7 @@ describe("test sorting many files with useListing", () => { await waitFor(() => { expect(result.current.listing?.length).toEqual(3); }); - expect(result.current.listing.map(({ name }) => name)).toEqual([ + expect(result.current.listing?.map(({ name }) => name)).toEqual([ "a.txt", "b.txt", "huge.txt", @@ -146,7 +154,7 @@ describe("test sorting many files with useListing", () => { sortField = "name"; rerender(); await waitFor(() => { - expect(result.current.listing.map(({ name }) => name)).toEqual([ + expect(result.current.listing?.map(({ name }) => name)).toEqual([ "huge.txt", "b.txt", "a.txt", @@ -157,7 +165,7 @@ describe("test sorting many files with useListing", () => { sortField = "mtime"; rerender(); await waitFor(() => { - expect(result.current.listing.map(({ name }) => name)).toEqual([ + expect(result.current.listing?.map(({ name }) => name)).toEqual([ "b.txt", "a.txt", "huge.txt", @@ -168,7 +176,7 @@ describe("test sorting many files with useListing", () => { sortField = "mtime"; rerender(); await waitFor(() => { - expect(result.current.listing.map(({ name }) => name)).toEqual([ + expect(result.current.listing?.map(({ name }) => name)).toEqual([ "huge.txt", "a.txt", "b.txt", @@ -179,7 +187,7 @@ describe("test sorting many files with useListing", () => { sortField = "size"; rerender(); await waitFor(() => { - expect(result.current.listing.map(({ name }) => name)).toEqual([ + expect(result.current.listing?.map(({ name }) => name)).toEqual([ "b.txt", "a.txt", "huge.txt", @@ -190,7 +198,7 @@ describe("test sorting many files with useListing", () => { sortField = "size"; rerender(); await waitFor(() => { - expect(result.current.listing.map(({ name }) => name)).toEqual([ + expect(result.current.listing?.map(({ name }) => name)).toEqual([ "huge.txt", "a.txt", "b.txt", From 5bdfca9451f90ed33169902e1001b3e20156c098 Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 23 Jul 2025 22:23:12 +0000 Subject: [PATCH 077/270] remove some not-necessary default styling; increase react virtuoso default viewport --- .../frontend/components/virtuoso-scroll-hook.ts | 5 +++-- .../explorer/file-listing/file-listing.tsx | 16 ++++------------ 2 files changed, 7 insertions(+), 14 deletions(-) diff --git a/src/packages/frontend/components/virtuoso-scroll-hook.ts b/src/packages/frontend/components/virtuoso-scroll-hook.ts index fab8ed41c9..8972a24333 100644 --- a/src/packages/frontend/components/virtuoso-scroll-hook.ts +++ b/src/packages/frontend/components/virtuoso-scroll-hook.ts @@ -46,7 +46,7 @@ export default function useVirtuosoScrollHook({ }, []); if (disabled) return {}; const lastScrollRef = useRef( - initialState ?? { index: 0, offset: 0 } + initialState ?? { index: 0, offset: 0 }, ); const recordScrollState = useMemo(() => { return (state: ScrollState) => { @@ -64,8 +64,9 @@ export default function useVirtuosoScrollHook({ }, [onScroll, cacheId]); return { + increaseViewportBy: 2000 /* a lot better default than 0 */, initialTopMostItemIndex: - (cacheId ? cache.get(cacheId) ?? initialState : initialState) ?? 0, + (cacheId ? (cache.get(cacheId) ?? initialState) : initialState) ?? 0, scrollerRef: handleScrollerRef, onScroll: () => { const scrollTop = scrollerRef.current?.scrollTop; diff --git a/src/packages/frontend/project/explorer/file-listing/file-listing.tsx b/src/packages/frontend/project/explorer/file-listing/file-listing.tsx index 01c73c66be..8c10a2987c 100644 --- a/src/packages/frontend/project/explorer/file-listing/file-listing.tsx +++ b/src/packages/frontend/project/explorer/file-listing/file-listing.tsx @@ -13,7 +13,6 @@ import { useEffect, useRef, useState } from "react"; import { useInterval } from "react-interval-hook"; import { FormattedMessage } from "react-intl"; import { Virtuoso, VirtuosoHandle } from "react-virtuoso"; -import { Col, Row } from "@cocalc/frontend/antd-bootstrap"; import { AppRedux, Rendered, @@ -312,25 +311,18 @@ function FileListing0({ />
)} - - {listing.length > 0 && ( - - )} - {listing.length > 0 && {render_rows()}} + + {render_rows()} {render_no_files()} - + ); } From 247a6a06c070f3d58f81223acf213d6b3f358bfe Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 23 Jul 2025 22:58:08 +0000 Subject: [PATCH 078/270] show file creation buttons even if you don't search (hsy suggested this in a meeting) --- .../explorer/file-listing/file-listing.tsx | 11 +--- .../explorer/file-listing/no-files.tsx | 55 ++++++++++++------- src/packages/frontend/project_actions.ts | 6 -- 3 files changed, 37 insertions(+), 35 deletions(-) diff --git a/src/packages/frontend/project/explorer/file-listing/file-listing.tsx b/src/packages/frontend/project/explorer/file-listing/file-listing.tsx index 8c10a2987c..e85b0c754a 100644 --- a/src/packages/frontend/project/explorer/file-listing/file-listing.tsx +++ b/src/packages/frontend/project/explorer/file-listing/file-listing.tsx @@ -46,12 +46,10 @@ interface Props { file_search: string; checked_files: immutable.Set; current_path: string; - create_folder: (switch_over?: boolean) => void; // TODO: should be action! - create_file: (ext?: string, switch_over?: boolean) => void; // TODO: should be action! selected_file_index?: number; project_id: string; shift_is_down: boolean; - sort_by: (heading: string) => void; // TODO: should be data + sort_by: (heading: string) => void; library?: object; other_settings?: immutable.Map; last_scroll_top?: number; @@ -118,8 +116,6 @@ function FileListing0({ listing, checked_files, current_path, - create_folder, - create_file, selected_file_index, project_id, shift_is_down, @@ -277,8 +273,6 @@ function FileListing0({ current_path={current_path} actions={actions} file_search={file_search} - create_folder={create_folder} - create_file={create_file} project_id={project_id} configuration_main={configuration_main} /> @@ -320,8 +314,7 @@ function FileListing0({ }} > - {render_rows()} - {render_no_files()} + {listing.length > 0 ? render_rows() : render_no_files()} ); diff --git a/src/packages/frontend/project/explorer/file-listing/no-files.tsx b/src/packages/frontend/project/explorer/file-listing/no-files.tsx index ac6f58f6d2..a4c42fbf2b 100644 --- a/src/packages/frontend/project/explorer/file-listing/no-files.tsx +++ b/src/packages/frontend/project/explorer/file-listing/no-files.tsx @@ -6,7 +6,6 @@ import { Button } from "antd"; import { useMemo } from "react"; import { FormattedMessage, useIntl } from "react-intl"; - import { useTypedRedux } from "@cocalc/frontend/app-framework"; import { Paragraph, Text } from "@cocalc/frontend/components"; import { Icon } from "@cocalc/frontend/components/icon"; @@ -20,12 +19,12 @@ import { capitalize } from "@cocalc/util/misc"; import { COLORS } from "@cocalc/util/theme"; import { HelpAlert } from "./help-alert"; import { full_path_text } from "./utils"; +import { default_filename } from "@cocalc/frontend/account"; +import { join } from "path"; interface Props { name: string; actions: ProjectActions; - create_folder: () => void; - create_file: () => void; file_search: string; current_path?: string; project_id: string; @@ -34,9 +33,8 @@ interface Props { export default function NoFiles({ actions, - create_folder, - create_file, file_search = "", + current_path, project_id, configuration_main, }: Props) { @@ -109,12 +107,16 @@ export default function NoFiles({ padding: "30px", }} onClick={(): void => { - if (file_search.length === 0) { + if (!file_search?.trim()) { actions.set_active_tab("new"); } else if (file_search[file_search.length - 1] === "/") { - create_folder(); + actions.create_folder({ + name: join(current_path ?? "", file_search), + }); } else { - create_file(); + actions.create_file({ + name: join(current_path ?? "", actualNewFilename), + }); } }} > @@ -137,18 +139,31 @@ export default function NoFiles({ file_search={file_search} actual_new_filename={actualNewFilename} /> - {file_search.length > 0 && ( -
-

Or Select a File Type

- -
- )} +
+

Select a File Type

+ { + ext = ext ? ext : "ipynb"; + const filename = file_search.trim() + ? file_search + "." + ext + : default_filename(ext, project_id); + actions.create_file({ + name: join(current_path ?? "", filename), + }); + }} + create_folder={() => { + const filename = default_filename(undefined, project_id); + actions.create_folder({ + name: file_search.trim() + ? file_search + : join(current_path ?? "", filename), + }); + }} + projectActions={actions} + availableFeatures={availableFeatures} + filename={actualNewFilename} + /> +
); } diff --git a/src/packages/frontend/project_actions.ts b/src/packages/frontend/project_actions.ts index 803fbde4a0..7a27ec3cea 100644 --- a/src/packages/frontend/project_actions.ts +++ b/src/packages/frontend/project_actions.ts @@ -620,9 +620,6 @@ export class ProjectActions extends Actions { if (opts.change_history) { this.set_url_to_path(store.get("current_path") ?? "", ""); } - if (opts.update_file_listing) { - this.fetch_directory_listing(); - } break; case "new": @@ -1911,7 +1908,6 @@ export class ProjectActions extends Actions { // returns a function that takes the err and output and // does the right activity logging stuff. return (err?, output?) => { - this.fetch_directory_listing(); if (err) { this.set_activity({ id, error: err }); } else if ( @@ -1962,7 +1958,6 @@ export class ProjectActions extends Actions { throw err; } finally { this.set_activity({ id, stop: "" }); - this.fetch_directory_listing(); } }; @@ -2752,7 +2747,6 @@ export class ProjectActions extends Actions { }); return; } - this.fetch_directory_listing({ path: p, compute_server_id }); if (switch_over) { this.open_directory(p); } From 4c41caa9955993bed30dd5ca97529c7ab8ff4c95 Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 23 Jul 2025 23:11:06 +0000 Subject: [PATCH 079/270] fix package.json issue --- src/packages/test/package.json | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/packages/test/package.json b/src/packages/test/package.json index bed4234e0c..94c8d6ef9e 100644 --- a/src/packages/test/package.json +++ b/src/packages/test/package.json @@ -5,7 +5,10 @@ "exports": { "./*": "./dist/*.js" }, - "keywords": ["test", "cocalc"], + "keywords": [ + "test", + "cocalc" + ], "scripts": { "preinstall": "npx only-allow pnpm", "clean": "rm -rf dist node_modules", @@ -28,10 +31,11 @@ }, "homepage": "https://github.com/sagemathinc/cocalc/tree/master/src/packages/test", "devDependencies": { - "@types/node": "^18.16.14", + "@testing-library/jest-dom": "^6.6.3", + "@testing-library/react": "^16.3.0", "@types/debug": "^4.1.12", "@types/jest": "^29.5.14", - "@testing-library/jest-dom": "^6.6.3", + "@types/node": "^18.16.14", "jest-environment-jsdom": "^30.0.2" } } From 95c705475c0e3b4c55a73b185294178564560f47 Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 23 Jul 2025 23:13:19 +0000 Subject: [PATCH 080/270] fix depcheck issue with new test package --- src/packages/test/package.json | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/packages/test/package.json b/src/packages/test/package.json index 94c8d6ef9e..820229b57e 100644 --- a/src/packages/test/package.json +++ b/src/packages/test/package.json @@ -5,17 +5,14 @@ "exports": { "./*": "./dist/*.js" }, - "keywords": [ - "test", - "cocalc" - ], + "keywords": ["test", "cocalc"], "scripts": { "preinstall": "npx only-allow pnpm", "clean": "rm -rf dist node_modules", "build": "pnpm exec tsc --build", "tsc": "pnpm exec tsc --watch --pretty --preserveWatchOutput", "test": "pnpm exec jest --forceExit", - "depcheck": "pnpx depcheck --ignores events" + "depcheck": "pnpx depcheck --ignores @types/debug,@types/jest,@types/node,jest-environment-jsdom" }, "author": "SageMath, Inc.", "license": "SEE LICENSE.md", From d332bb6d05cb5fbc0450902b5f23c50c39c22ce6 Mon Sep 17 00:00:00 2001 From: William Stein Date: Thu, 24 Jul 2025 01:54:04 +0000 Subject: [PATCH 081/270] refactor code for filtering the listing --- .../frontend/project/explorer/action-bar.tsx | 6 ++--- .../explorer/file-listing/file-listing.tsx | 18 +++++++-------- .../project/listing/filter-listing.ts | 23 +++++++++++++++++++ 3 files changed, 34 insertions(+), 13 deletions(-) create mode 100644 src/packages/frontend/project/listing/filter-listing.ts diff --git a/src/packages/frontend/project/explorer/action-bar.tsx b/src/packages/frontend/project/explorer/action-bar.tsx index 7cfc6f58d5..eec0f43eda 100644 --- a/src/packages/frontend/project/explorer/action-bar.tsx +++ b/src/packages/frontend/project/explorer/action-bar.tsx @@ -8,7 +8,6 @@ import * as immutable from "immutable"; import { throttle } from "lodash"; import React, { useEffect, useRef, useState } from "react"; import { FormattedMessage, useIntl } from "react-intl"; - import { Button, ButtonToolbar } from "@cocalc/frontend/antd-bootstrap"; import { Gap, Icon } from "@cocalc/frontend/components"; import { useStudentProjectFunctionality } from "@cocalc/frontend/course"; @@ -19,7 +18,6 @@ import { labels } from "@cocalc/frontend/i18n"; import { file_actions, ProjectActions } from "@cocalc/frontend/project_store"; import * as misc from "@cocalc/util/misc"; import { COLORS } from "@cocalc/util/theme"; - import { useProjectContext } from "../context"; const ROW_INFO_STYLE = { @@ -43,7 +41,7 @@ interface Props { project_is_running?: boolean; } -export const ActionBar: React.FC = (props: Props) => { +export function ActionBar(props: Props) { const intl = useIntl(); const [showLabels, setShowLabels] = useState(true); const { mainWidthPx } = useProjectContext(); @@ -350,7 +348,7 @@ export const ActionBar: React.FC = (props: Props) => { ); -}; +} export const ACTION_BUTTONS_DIR = [ "download", diff --git a/src/packages/frontend/project/explorer/file-listing/file-listing.tsx b/src/packages/frontend/project/explorer/file-listing/file-listing.tsx index e85b0c754a..d5ed3dbb9f 100644 --- a/src/packages/frontend/project/explorer/file-listing/file-listing.tsx +++ b/src/packages/frontend/project/explorer/file-listing/file-listing.tsx @@ -34,6 +34,7 @@ import useFs from "@cocalc/frontend/project/listing/use-fs"; import useListing, { type SortField, } from "@cocalc/frontend/project/listing/use-listing"; +import filterListing from "@cocalc/frontend/project/listing/filter-listing"; interface Props { // TODO: everything but actions/redux should be immutable JS data, and use shouldComponentUpdate @@ -94,7 +95,7 @@ function sortDesc(active_file_sort?): { export function FileListing(props) { const fs = useFs({ project_id: props.project_id }); - const { listing, error } = useListing({ + let { listing, error } = useListing({ fs, path: props.current_path, ...sortDesc(props.active_file_sort), @@ -102,6 +103,13 @@ export function FileListing(props) { if (error) { return ; } + + listing = filterListing({ + listing, + search: props.file_search, + showHidden: props.show_hidden, + }); + if (listing == null) { return ; } @@ -123,17 +131,9 @@ function FileListing0({ configuration_main, file_search = "", isRunning, - show_hidden, stale, // show_masked, }: Props) { - if (!show_hidden) { - listing = listing.filter((x) => !x.name.startsWith(".")); - } - if (file_search) { - listing = listing.filter((x) => x.name.includes(file_search)); - } - const prev_current_path = usePrevious(current_path); function watch() { diff --git a/src/packages/frontend/project/listing/filter-listing.ts b/src/packages/frontend/project/listing/filter-listing.ts new file mode 100644 index 0000000000..53e296da7f --- /dev/null +++ b/src/packages/frontend/project/listing/filter-listing.ts @@ -0,0 +1,23 @@ +import { DirectoryListingEntry } from "@cocalc/util/types"; + +export default function filterListing({ + listing, + search, + showHidden, +}: { + listing?: DirectoryListingEntry[] | null; + search?: string; + showHidden?: boolean; +}): DirectoryListingEntry[] | null { + if (listing == null) { + return null; + } + if (!showHidden) { + listing = listing.filter((x) => !x.name.startsWith(".")); + } + search = search?.trim()?.toLowerCase(); + if (!search || search.startsWith("/")) { + return listing; + } + return listing.filter((x) => x.name.toLowerCase().includes(search)); +} From a7987bd590dc62947fc4e3a2dd30982b6fb18cc0 Mon Sep 17 00:00:00 2001 From: William Stein Date: Thu, 24 Jul 2025 04:57:29 +0000 Subject: [PATCH 082/270] ripping out the old listings related code and rewriting/replacing it -- this is in the middle --- src/packages/frontend/file-upload.tsx | 8 +- .../frontend/project/directory-listing.ts | 168 ----------- .../frontend/project/explorer/explorer.tsx | 92 +----- .../explorer/file-listing/file-listing.tsx | 52 +--- .../project/fetch-directory-listing.ts | 127 -------- .../frontend/project/listing/use-files.ts | 35 ++- .../frontend/project/listing/use-listing.ts | 88 ++++-- src/packages/frontend/project/utils.ts | 2 +- src/packages/frontend/project_actions.ts | 141 +++------ src/packages/frontend/project_store.ts | 278 ------------------ 10 files changed, 162 insertions(+), 829 deletions(-) delete mode 100644 src/packages/frontend/project/directory-listing.ts delete mode 100644 src/packages/frontend/project/fetch-directory-listing.ts diff --git a/src/packages/frontend/file-upload.tsx b/src/packages/frontend/file-upload.tsx index 9e1d1a81f8..171280e5c0 100644 --- a/src/packages/frontend/file-upload.tsx +++ b/src/packages/frontend/file-upload.tsx @@ -192,7 +192,7 @@ interface FileUploadWrapperProps { project_id: string; // The project to upload files to dest_path: string; // The path for files to be sent config?: object; // All supported dropzone.js config options - event_handlers: { + event_handlers?: { complete?: Function | Function[]; sending?: Function | Function[]; removedfile?: Function | Function[]; @@ -245,7 +245,7 @@ export function FileUploadWrapper({ previewTemplate: ReactDOMServer.renderToStaticMarkup( preview_template?.() ?? , ), - addRemoveLinks: event_handlers.removedfile != null, + addRemoveLinks: event_handlers?.removedfile != null, ...UPLOAD_OPTIONS, }, true, @@ -309,7 +309,7 @@ export function FileUploadWrapper({ // from the dropzone. This is true by default if there is // no "removedfile" handler, and false otherwise. function close_preview( - remove_all: boolean = event_handlers.removedfile == null, + remove_all: boolean = event_handlers?.removedfile == null, ) { if (typeof on_close === "function") { on_close(); @@ -381,7 +381,7 @@ export function FileUploadWrapper({ } function set_up_events(): void { - if (dropzone.current == null) { + if (dropzone.current == null || event_handlers == null) { return; } diff --git a/src/packages/frontend/project/directory-listing.ts b/src/packages/frontend/project/directory-listing.ts deleted file mode 100644 index a636d49617..0000000000 --- a/src/packages/frontend/project/directory-listing.ts +++ /dev/null @@ -1,168 +0,0 @@ -/* - * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. - * License: MS-RSL – see LICENSE.md for details - */ - -import { server_time } from "@cocalc/util/misc"; -import { once, retry_until_success } from "@cocalc/util/async-utils"; -import { webapp_client } from "../webapp-client"; -import { redux } from "../app-framework"; -import { dirname } from "path"; - -//const log = (...args) => console.log("directory-listing", ...args); -const log = (..._args) => {}; - -interface ListingOpts { - project_id: string; - path: string; - hidden: boolean; - max_time_s: number; - group: string; - trigger_start_project?: boolean; - compute_server_id: number; -} - -// This makes an api call directly to the project to get a directory listing. - -export async function get_directory_listing(opts: ListingOpts) { - log("get_directory_listing", opts); - - let method, state, time0, timeout; - - if (["owner", "collaborator", "admin"].indexOf(opts.group) != -1) { - method = webapp_client.project_client.directory_listing; - // Also, make sure project starts running, in case it isn't. - state = (redux.getStore("projects") as any).getIn([ - "project_map", - opts.project_id, - "state", - "state", - ]); - if (state != null && state !== "running") { - timeout = 0.5; - time0 = server_time(); - if (opts.trigger_start_project === false) { - return { files: [], noRunning: true }; - } - redux.getActions("projects").start_project(opts.project_id); - } else { - timeout = 1; - } - } else { - throw Error("you do not have access to this project"); - } - - let listing_err: Error | undefined; - method = method.bind(webapp_client.project_client); - async function f(): Promise { - try { - return await method({ - project_id: opts.project_id, - path: opts.path, - hidden: opts.hidden, - compute_server_id: opts.compute_server_id, - timeout, - }); - } catch (err) { - if (err.message != null) { - if (err.message.indexOf("ENOENT") != -1) { - listing_err = Error("no_dir"); - return; - } else if (err.message.indexOf("ENOTDIR") != -1) { - listing_err = Error("not_a_dir"); - return; - } - } - if (timeout < 5) { - timeout *= 1.3; - } - throw err; - } - } - - let listing; - try { - listing = await retry_until_success({ - f, - max_time: opts.max_time_s * 1000, - start_delay: 100, - max_delay: 1000, - }); - } catch (err) { - listing_err = err; - } finally { - // no error, but `listing` has no value, too - // https://github.com/sagemathinc/cocalc/issues/3223 - if (!listing_err && listing == null) { - listing_err = Error("no_dir"); - } - if (time0 && state !== "running" && !listing_err) { - // successfully opened, started, and got directory listing - redux.getProjectActions(opts.project_id).log({ - event: "start_project", - time: server_time().valueOf() - time0.valueOf(), - }); - } - - if (listing_err) { - throw listing_err; - } else { - return listing; - } - } -} - -import { Listings } from "@cocalc/frontend/conat/listings"; - -export async function get_directory_listing2(opts: ListingOpts): Promise { - log("get_directory_listing2", opts); - const start = Date.now(); - const store = redux.getProjectStore(opts.project_id); - const compute_server_id = - opts.compute_server_id ?? store.get("compute_server_id"); - const listings: Listings = await store.get_listings(compute_server_id); - listings.watch(opts.path); - if (opts.path) { - listings.watch(dirname(opts.path)); - } - while (Date.now() - start < opts.max_time_s * 1000) { - if (listings.getMissing(opts.path)) { - if ( - store.getIn(["directory_listings", compute_server_id, opts.path]) != - null - ) { - // just update an already loaded listing: - try { - const files = await listings.getListingDirectly( - opts.path, - opts.trigger_start_project, - ); - return { files }; - } catch (err) { - console.log( - `WARNING: temporary problem getting directory listing -- ${err}`, - ); - } - } else { - // ensure all listing entries get loaded soon. - redux - .getProjectActions(opts.project_id) - ?.fetch_directory_listing_directly( - opts.path, - opts.trigger_start_project, - compute_server_id, - ); - } - } - // return what we have now, if anything. - const files = await listings.get(opts.path, opts.trigger_start_project); - if (files != null) { - return { files }; - } - await once( - listings, - "change", - opts.max_time_s * 1000 - (Date.now() - start), - ); - } -} diff --git a/src/packages/frontend/project/explorer/explorer.tsx b/src/packages/frontend/project/explorer/explorer.tsx index e4818639cd..ccaddf8ead 100644 --- a/src/packages/frontend/project/explorer/explorer.tsx +++ b/src/packages/frontend/project/explorer/explorer.tsx @@ -7,7 +7,7 @@ import * as immutable from "immutable"; import * as _ from "lodash"; import React from "react"; import { UsersViewing } from "@cocalc/frontend/account/avatar/users-viewing"; -import { Button, ButtonGroup, Col, Row } from "@cocalc/frontend/antd-bootstrap"; +import { Col, Row } from "@cocalc/frontend/antd-bootstrap"; import { project_redux_name, rclass, @@ -20,7 +20,6 @@ import { A, ActivityDisplay, ErrorDisplay, - Icon, Loading, SettingBox, } from "@cocalc/frontend/components"; @@ -49,12 +48,6 @@ import { NewButton } from "./new-button"; import { PathNavigator } from "./path-navigator"; import { SearchBar } from "./search-bar"; import ExplorerTour from "./tour/tour"; -import { ListingItem } from "./types"; - -function pager_range(page_size, page_number) { - const start_index = page_size * page_number; - return { start_index, end_index: start_index + page_size }; -} export type Configuration = ShallowTypedMap<{ main: MainConfiguration }>; @@ -105,11 +98,6 @@ interface ReduxProps { selected_file_index?: number; file_creation_error?: string; ext_selection?: string; - displayed_listing: { - listing: ListingItem[]; - error: any; - file_map: Map; - }; new_name?: string; library?: object; show_library?: boolean; @@ -184,7 +172,6 @@ const Explorer0 = rclass( selected_file_index: rtypes.number, file_creation_error: rtypes.string, ext_selection: rtypes.string, - displayed_listing: rtypes.object, new_name: rtypes.string, library: rtypes.object, show_library: rtypes.bool, @@ -291,49 +278,16 @@ const Explorer0 = rclass( this.props.actions.setState({ file_search: "", page_number: 0 }); }; - render_paging_buttons(num_pages: number): React.JSX.Element | undefined { - if (num_pages > 1) { - return ( - - - - - - - - - - ); - } - } - - render_files_action_box(file_map?) { - if (file_map == undefined) { - return; - } + render_files_action_box() { return ( @@ -366,15 +320,15 @@ const Explorer0 = rclass( ); } - render_files_actions(listing, project_is_running) { + render_files_actions(project_is_running) { return ( this.props.actions.fetch_directory_listing(), - }} config={{ clickable: ".upload-button" }} style={{ flex: "1 0 auto", @@ -476,9 +427,7 @@ const Explorer0 = rclass( ); } - render_control_row( - visible_listing: ListingItem[] | undefined, - ): React.JSX.Element { + render_control_row(): React.JSX.Element { return (
{this.render_error()} {this.render_activity()} - {this.render_control_row(visible_listing)} + {this.render_control_row()} {this.props.ext_selection != null && ( )} @@ -680,9 +614,7 @@ const Explorer0 = rclass( minWidth: "20em", }} > - {listing != undefined - ? this.render_files_actions(listing, project_is_running) - : undefined} + {this.render_files_actions(project_is_running)}
{this.render_project_files_buttons()} @@ -695,7 +627,7 @@ const Explorer0 = rclass( {this.props.checked_files.size > 0 && this.props.file_action != undefined ? ( - {this.render_files_action_box(file_map)} + {this.render_files_action_box()} ) : undefined} @@ -732,7 +664,6 @@ const SearchTerminalBar = React.forwardRef( current_path, file_search, actions, - visible_listing, selected_file_index, file_creation_error, create_file, @@ -742,7 +673,6 @@ const SearchTerminalBar = React.forwardRef( current_path: string; file_search: string; actions: ProjectActions; - visible_listing: ListingItem[] | undefined; selected_file_index?: number; file_creation_error?: string; create_file: (ext?: string, switch_over?: boolean) => void; @@ -750,6 +680,8 @@ const SearchTerminalBar = React.forwardRef( }, ref: React.LegacyRef | undefined, ) => { + // [ ] TODO + const visible_listing = []; return (
; @@ -130,41 +118,9 @@ function FileListing0({ sort_by, configuration_main, file_search = "", - isRunning, stale, // show_masked, }: Props) { - const prev_current_path = usePrevious(current_path); - - function watch() { - watchFiles({ actions, current_path }); - } - - // once after mounting, when changing paths, and in regular intervals call watch() - useEffect(() => { - watch(); - }, []); - - useEffect(() => { - if (current_path != prev_current_path) watch(); - }, [current_path, prev_current_path]); - - useInterval(watch, WATCH_THROTTLE_MS); - - const [missing, setMissing] = useState(0); - - useEffect(() => { - if (isRunning) return; - if (listing.length == 0) return; - (async () => { - const missing = await redux - .getProjectStore(project_id) - .get_listings() - .getMissingUsingDatabase(current_path); - setMissing(missing ?? 0); - })(); - }, [current_path, isRunning]); - const computeServerId = useTypedRedux({ project_id }, "compute_server_id"); function render_row( @@ -287,11 +243,9 @@ function FileListing0({ > missing {missing} files} other {}}. + defaultMessage={`Showing stale directory listing. To update the directory listing start this project.`} values={{ - is_missing: missing > 0, - missing, a: (c) => ( { diff --git a/src/packages/frontend/project/fetch-directory-listing.ts b/src/packages/frontend/project/fetch-directory-listing.ts deleted file mode 100644 index 026de8cbf3..0000000000 --- a/src/packages/frontend/project/fetch-directory-listing.ts +++ /dev/null @@ -1,127 +0,0 @@ -import { is_running_or_starting } from "./project-start-warning"; -import type { ProjectActions } from "@cocalc/frontend/project_actions"; -import { trunc_middle, uuid } from "@cocalc/util/misc"; -import { get_directory_listing } from "./directory-listing"; -import { fromJS, Map } from "immutable"; -import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; - -//const log = (...args) => console.log("fetchDirectoryListing", ...args); -const log = (..._args) => {}; - -interface FetchDirectoryListingOpts { - path?: string; - // WARNING: THINK VERY HARD BEFORE YOU USE force=true, due to efficiency! - force?: boolean; - // can be explicit here; otherwise will fall back to store.get('compute_server_id') - compute_server_id?: number; -} - -function getPath( - actions, - opts?: FetchDirectoryListingOpts, -): string | undefined { - return opts?.path ?? actions.get_store()?.get("current_path"); -} - -function getComputeServerId(actions, opts): number { - return ( - opts?.compute_server_id ?? - actions.get_store()?.get("compute_server_id") ?? - 0 - ); -} - -const fetchDirectoryListing = reuseInFlight( - async ( - actions: ProjectActions, - opts: FetchDirectoryListingOpts = {}, - ): Promise => { - let status; - let store = actions.get_store(); - if (store == null) { - return; - } - const { force } = opts; - const path = getPath(actions, opts); - const compute_server_id = getComputeServerId(actions, opts); - - if (force && path != null) { - // update our interest. - store.get_listings().watch(path, true); - } - log({ force, path, compute_server_id }); - - if (path == null) { - // nothing to do if path isn't defined -- there is no current path -- - // see https://github.com/sagemathinc/cocalc/issues/818 - return; - } - - const id = uuid(); - if (path) { - status = `Loading file list - ${trunc_middle(path, 30)}`; - } else { - status = "Loading file list"; - } - - let error = ""; - try { - // only show actions indicator, if the project is running or starting - // if it is stopped, we get a stale listing from the database, which is fine. - if (is_running_or_starting(actions.project_id)) { - log("show activity"); - actions.set_activity({ id, status }); - } - - log("make sure user is fully signed in"); - await actions.redux.getStore("account").async_wait({ - until: (s) => s.get("is_logged_in") && s.get("account_id"), - }); - - log("getting listing"); - const listing = await get_directory_listing({ - project_id: actions.project_id, - path, - hidden: true, - max_time_s: 15, - trigger_start_project: false, - group: "collaborator", // nothing else is implemented - compute_server_id, - }); - log("got ", listing.files); - const value = fromJS(listing.files); - log("saving result"); - store = actions.get_store(); - if (store == null) { - return; - } - const directory_listings = store.get("directory_listings"); - let listing2 = directory_listings.get(compute_server_id) ?? Map(); - if (listing.noRunning && (listing2.get(path)?.size ?? 0) > 0) { - // do not change it - return; - } - listing2 = listing2.set(path, value); - actions.setState({ - directory_listings: directory_listings.set(compute_server_id, listing2), - }); - } catch (err) { - log("error", err); - error = `${err}`; - } finally { - actions.set_activity({ id, stop: "", error }); - } - }, - { - createKey: (args) => { - const actions = args[0]; - // reuse in flight on the project id, compute server id and path - return `${actions.project_id}-${getComputeServerId( - actions, - args[1], - )}-${getPath(actions, args[1])}`; - }, - }, -); - -export default fetchDirectoryListing; diff --git a/src/packages/frontend/project/listing/use-files.ts b/src/packages/frontend/project/listing/use-files.ts index 925a284894..8c6bc7f130 100644 --- a/src/packages/frontend/project/listing/use-files.ts +++ b/src/packages/frontend/project/listing/use-files.ts @@ -13,20 +13,45 @@ import { type Files } from "@cocalc/conat/files/listing"; import { type FilesystemClient } from "@cocalc/conat/files/fs"; import { type ConatError } from "@cocalc/conat/core/client"; import useCounter from "@cocalc/frontend/app-framework/counter-hook"; +import LRU from "lru-cache"; +import type { JSONValue } from "@cocalc/util/types"; +export { Files }; const DEFAULT_THROTTLE_FILE_UPDATE = 500; +const CACHE_SIZE = 100; + +const cache = new LRU({ max: CACHE_SIZE }); + +export function getFiles({ + cacheId, + path, +}: { + cacheId?: JSONValue; + path: string; +}): Files | null { + if (cacheId == null) { + return null; + } + return cache.get(key(cacheId, path)) ?? null; +} + export default function useFiles({ fs, path, throttleUpdate = DEFAULT_THROTTLE_FILE_UPDATE, + cacheId, }: { // fs = undefined is supported and just waits until you provide a fs that is defined fs?: FilesystemClient | null; path: string; throttleUpdate?: number; + // cacheId -- if given, save most recently loaded Files for a path in an in-memory LRU cache. + // An example cacheId could be {project_id, compute_server_id}. + // This is used to speed up the first load, and can also be fetched synchronously. + cacheId?: JSONValue; }): { files: Files | null; error: null | ConatError; refresh: () => void } { - const [files, setFiles] = useState(null); + const [files, setFiles] = useState(getFiles({ cacheId, path })); const [error, setError] = useState(null); const { val: counter, inc: refresh } = useCounter(); const listingRef = useRef(null); @@ -48,7 +73,9 @@ export default function useFiles({ setFiles(null); return; } - + if (cacheId != null) { + cache.set(key(cacheId, path), listing.files); + } const update = () => { setFiles({ ...listing.files }); }; @@ -68,3 +95,7 @@ export default function useFiles({ return { files, error, refresh }; } + +function key(cacheId: JSONValue, path: string) { + return JSON.stringify({ cacheId, path }); +} diff --git a/src/packages/frontend/project/listing/use-listing.ts b/src/packages/frontend/project/listing/use-listing.ts index 75ffc37f40..da5a1870ac 100644 --- a/src/packages/frontend/project/listing/use-listing.ts +++ b/src/packages/frontend/project/listing/use-listing.ts @@ -10,16 +10,35 @@ import { field_cmp } from "@cocalc/util/misc"; import useFiles from "./use-files"; import { type FilesystemClient } from "@cocalc/conat/files/fs"; import { type ConatError } from "@cocalc/conat/core/client"; +import type { JSONValue } from "@cocalc/util/types"; +import { getFiles, type Files } from "./use-files"; export type SortField = "name" | "mtime" | "size" | "type"; export type SortDirection = "asc" | "desc"; +export function getListing({ + path, + cacheId, + sortField, + sortDirection, +}: { + path; + string; + cacheId?: JSONValue; + sortField?: SortField; + sortDirection?: SortDirection; +}): null | DirectoryListingEntry[] { + const files = getFiles({ cacheId, path }); + return filesToListing({ files, sortField, sortDirection }); +} + export default function useListing({ fs, path, sortField = "name", sortDirection = "asc", throttleUpdate, + cacheId, }: { // fs = undefined is supported and just waits until you provide a fs that is defined fs?: FilesystemClient | null; @@ -27,38 +46,59 @@ export default function useListing({ sortField?: SortField; sortDirection?: SortDirection; throttleUpdate?: number; + cacheId?: JSONValue; }): { listing: null | DirectoryListingEntry[]; error: null | ConatError; refresh: () => void; } { - const { files, error, refresh } = useFiles({ fs, path, throttleUpdate }); + const { files, error, refresh } = useFiles({ + fs, + path, + throttleUpdate, + cacheId, + }); const listing = useMemo(() => { - if (files == null) { - return null; - } - const v: DirectoryListingEntry[] = []; - for (const name in files) { - v.push({ name, ...files[name] }); - } - if ( - sortField != "name" && - sortField != "mtime" && - sortField != "size" && - sortField != "type" - ) { - console.warn(`invalid sort field: '${sortField}'`); - } - v.sort(field_cmp(sortField)); - if (sortDirection == "desc") { - v.reverse(); - } else if (sortDirection == "asc") { - } else { - console.warn(`invalid sort direction: '${sortDirection}'`); - } - return v; + return filesToListing({ files, sortField, sortDirection }); }, [sortField, sortDirection, files]); return { listing, error, refresh }; } + +function filesToListing({ + files, + sortField = "name", + sortDirection = "asc", +}: { + files?: Files | null; + sortField?: SortField; + sortDirection?: SortDirection; +}): null | DirectoryListingEntry[] { + if (files == null) { + return null; + } + if (files == null) { + return null; + } + const v: DirectoryListingEntry[] = []; + for (const name in files) { + v.push({ name, ...files[name] }); + } + if ( + sortField != "name" && + sortField != "mtime" && + sortField != "size" && + sortField != "type" + ) { + console.warn(`invalid sort field: '${sortField}'`); + } + v.sort(field_cmp(sortField)); + if (sortDirection == "desc") { + v.reverse(); + } else if (sortDirection == "asc") { + } else { + console.warn(`invalid sort direction: '${sortDirection}'`); + } + return v; +} diff --git a/src/packages/frontend/project/utils.ts b/src/packages/frontend/project/utils.ts index e1bfb9cbaf..d0f2b5a1c1 100644 --- a/src/packages/frontend/project/utils.ts +++ b/src/packages/frontend/project/utils.ts @@ -78,7 +78,7 @@ export class NewFilenames { // generate a new filename, by optionally avoiding the keys in the dictionary public gen( type?: NewFilenameTypes, - avoid?: { [name: string]: boolean }, + avoid?: { [name: string]: any } | null, ): string { type = this.sanitize_type(type); // reset the enumeration if type changes diff --git a/src/packages/frontend/project_actions.ts b/src/packages/frontend/project_actions.ts index 7a27ec3cea..d76df13f18 100644 --- a/src/packages/frontend/project_actions.ts +++ b/src/packages/frontend/project_actions.ts @@ -49,7 +49,6 @@ import { import Fragment, { FragmentId } from "@cocalc/frontend/misc/fragment-id"; import * as project_file from "@cocalc/frontend/project-file"; import { delete_files } from "@cocalc/frontend/project/delete-files"; -import fetchDirectoryListing from "@cocalc/frontend/project/fetch-directory-listing"; import { ProjectEvent, SoftwareEnvironmentEvent, @@ -108,6 +107,11 @@ import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; import { MARKERS } from "@cocalc/util/sagews"; import { client_db } from "@cocalc/util/schema"; import { get_editor } from "./editors/react-wrapper"; +import { type FilesystemClient } from "@cocalc/conat/files/fs"; +import { + getFiles, + type Files, +} from "@cocalc/frontend/project/listing/use-files"; const { defaults, required } = misc; @@ -1498,8 +1502,6 @@ export class ProjectActions extends Actions { page_number: 0, most_recent_file_click: undefined, }); - - store.get_listings().watch(path, true); }; setComputeServerId = (compute_server_id: number) => { @@ -1559,33 +1561,14 @@ export class ProjectActions extends Actions { // Update the directory listing cache for the given path. // Uses current path if path not provided. - fetch_directory_listing = async (opts?): Promise => { - await fetchDirectoryListing(this, opts); + fetch_directory_listing = async (_opts?): Promise => { + console.log("TODO: eliminate code that uses fetch_directory_listing"); }; - public async fetch_directory_listing_directly( - path: string, - trigger_start_project?: boolean, - compute_server_id?: number, - ): Promise { - const store = this.get_store(); - if (store == null) return; - compute_server_id = this.getComputeServerId(compute_server_id); - const listings = store.get_listings(compute_server_id); - try { - const files = await listings.getListingDirectly( - path, - trigger_start_project, - ); - const directory_listings = store.get("directory_listings"); - let listing = directory_listings.get(compute_server_id) ?? Map(); - listing = listing.set(path, files); - this.setState({ - directory_listings: directory_listings.set(compute_server_id, listing), - }); - } catch (err) { - console.warn(`Unable to fetch directory listing -- "${err}"`); - } + public async fetch_directory_listing_directly(): Promise { + console.log( + "TODO: eliminate code that uses fetch_directory_listing_directly", + ); } // Sets the active file_sort to next_column_name @@ -1755,40 +1738,6 @@ export class ProjectActions extends Actions { }); } - // this isn't really an action, but very helpful! - public get_filenames_in_current_dir(): - | { [name: string]: boolean } - | undefined { - const store = this.get_store(); - if (store == undefined) { - return; - } - - const files_in_dir = {}; - // This will set files_in_dir to our current view of the files in the current - // directory (at least the visible ones) or do nothing in case we don't know - // anything about files (highly unlikely). Unfortunately (for this), our - // directory listings are stored as (immutable) lists, so we have to make - // a map out of them. - const compute_server_id = store.get("compute_server_id"); - const listing = store.getIn([ - "directory_listings", - compute_server_id, - store.get("current_path"), - ]); - - if (typeof listing === "string") { - // must be an error - return undefined; // simple fallback - } - if (listing != null) { - listing.map(function (x) { - files_in_dir[x.get("name")] = true; - }); - } - return files_in_dir; - } - suggestDuplicateFilenameInCurrentDirectory = ( name: string, ): string | undefined => { @@ -2496,21 +2445,40 @@ export class ProjectActions extends Actions { } } + private _filesystem: FilesystemClient; + fs = (): FilesystemClient => { + this._filesystem ??= webapp_client.conat_client + .conat() + .fs({ project_id: this.project_id }); + return this._filesystem; + }; + + // if available in cache, this returns the filenames in the current directory, + // which is often useful, or null if they are not known. This is sync, so it + // can't query the backend. (Here Files is a map from path names to data about them.) + get_filenames_in_current_dir = (): Files | null => { + const store = this.get_store(); + if (store == undefined) { + return null; + } + const path = store.get("current_path"); + if (path == null) { + return null; + } + // todo: compute_server_id here and in place that does useListing! + return getFiles({ cacheId: { project_id: this.project_id }, path }); + }; + // return true if exists and is a directory - private async isdir(path: string): Promise { + isdir = async (path: string): Promise => { if (path == "") return true; // easy special case try { - await webapp_client.project_client.exec({ - project_id: this.project_id, - command: "test", - args: ["-d", path], - err_on_exit: true, - }); - return true; + const stats = await this.fs().stat(path); + return stats.isDirectory(); } catch (_) { return false; } - } + }; public async move_files(opts: { src: string[]; @@ -3235,17 +3203,16 @@ export class ProjectActions extends Actions { // log // settings // search - async load_target( + load_target = async ( target, foreground = true, ignore_kiosk = false, change_history = true, fragmentId?: FragmentId, - ): Promise { + ): Promise => { const segments = target.split("/"); const full_path = segments.slice(1).join("/"); const parent_path = segments.slice(1, segments.length - 1).join("/"); - const last = segments.slice(-1).join(); const main_segment = segments[0] as FixedTab | "home"; switch (main_segment) { case "active": @@ -3264,27 +3231,9 @@ export class ProjectActions extends Actions { if (store == null) { return; // project closed already } - // We check whether the path is a directory or not, first by checking if - // we have a recent directory listing in our cache, and if not, by calling - // isdir, which is a single exec. - let isdir; - let { item, err } = store.get_item_in_path(last, parent_path); - if (item == null || err) { - try { - isdir = await webapp_client.project_client.isdir({ - project_id: this.project_id, - path: normalize(full_path), - }); - } catch (err) { - // TODO: e.g., project is not running? - // I've seen this, e.g., when trying to open a file when not running, and it just - // gets retried and works. - console.log(`Error opening '${target}' -- ${err}`); - return; - } - } else { - isdir = item.get("isdir"); - } + + // We check whether the path is a directory or not: + const isdir = await this.isdir(full_path); if (isdir) { this.open_directory(full_path, change_history); } else { @@ -3343,7 +3292,7 @@ export class ProjectActions extends Actions { misc.unreachable(main_segment); console.warn(`project/load_target: don't know segment ${main_segment}`); } - } + }; set_compute_image = async (compute_image: string): Promise => { const projects_store = this.redux.getStore("projects"); diff --git a/src/packages/frontend/project_store.ts b/src/packages/frontend/project_store.ts index 3b06e820c6..9ce25746ec 100644 --- a/src/packages/frontend/project_store.ts +++ b/src/packages/frontend/project_store.ts @@ -25,7 +25,6 @@ import { Table, TypedMap, } from "@cocalc/frontend/app-framework"; -import { Listings, listings } from "@cocalc/frontend/conat/listings"; import { fileURL } from "@cocalc/frontend/lib/cocalc-urls"; import { get_local_storage } from "@cocalc/frontend/misc"; import { QueryParams } from "@cocalc/frontend/misc/query-params"; @@ -42,8 +41,6 @@ import { ProjectConfiguration, } from "@cocalc/frontend/project_configuration"; import * as misc from "@cocalc/util/misc"; -import { compute_file_masks } from "./project/explorer/compute-file-masks"; -import { DirectoryListing } from "./project/explorer/types"; import { FixedTab } from "./project/page/file-tab"; import { FlyoutActiveMode, @@ -76,8 +73,6 @@ export interface ProjectStoreState { just_closed_files: immutable.List; public_paths?: immutable.Map>; - // directory_listings is a map from compute_server_id to {path:[listing for that path on the compute server]} - directory_listings: immutable.Map; show_upload: boolean; create_file_alert: boolean; displayed_listing?: any; // computed(object), @@ -181,7 +176,6 @@ export interface ProjectStoreState { export class ProjectStore extends Store { public project_id: string; private previous_runstate: string | undefined; - private listings: { [compute_server_id: number]: Listings } = {}; public readonly computeServerIdLocalStorageKey: string; // Function to call to initialize one of the tables in this store. @@ -226,10 +220,6 @@ export class ProjectStore extends Store { if (projects_store !== undefined) { projects_store.removeListener("change", this._projects_store_change); } - for (const id in this.listings) { - this.listings[id].close(); - delete this.listings[id]; - } // close any open file tabs, properly cleaning up editor state: const open = this.get("open_files")?.toJS(); if (open != null) { @@ -296,7 +286,6 @@ export class ProjectStore extends Store { open_files: immutable.Map>({}), open_files_order: immutable.List([]), just_closed_files: immutable.List([]), - directory_listings: immutable.Map(), // immutable, show_upload: false, create_file_alert: false, displayed_listing: undefined, // computed(object), @@ -371,156 +360,6 @@ export class ProjectStore extends Store { }, }, - // cached pre-processed file listing, which should always be up to date when - // called, and properly depends on dependencies. - displayed_listing: { - dependencies: [ - "active_file_sort", - "current_path", - "directory_listings", - "stripped_public_paths", - "file_search", - "other_settings", - "show_hidden", - "show_masked", - "compute_server_id", - ] as const, - fn: () => { - const search_escape_char = "/"; - const listingStored = this.getIn([ - "directory_listings", - this.get("compute_server_id"), - this.get("current_path"), - ]); - if (typeof listingStored === "string") { - if ( - listingStored.indexOf("ECONNREFUSED") !== -1 || - listingStored.indexOf("ENOTFOUND") !== -1 - ) { - return { error: "no_instance" }; // the host VM is down - } else if (listingStored.indexOf("o such path") !== -1) { - return { error: "no_dir" }; - } else if (listingStored.indexOf("ot a directory") !== -1) { - return { error: "not_a_dir" }; - } else if (listingStored.indexOf("not running") !== -1) { - // yes, no underscore. - return { error: "not_running" }; - } else { - return { error: listingStored }; - } - } - if (listingStored == null) { - return {}; - } - try { - if (listingStored?.errno) { - return { error: misc.to_json(listingStored) }; - } - } catch (err) { - return { - error: "Error getting directory listing - please try again.", - }; - } - - if (listingStored?.toJS == null) { - return { - error: "Unable to get directory listing - please try again.", - }; - } - - // We can proceed and get the listing as a JS object. - let listing: DirectoryListing = listingStored.toJS(); - - if (this.get("other_settings").get("mask_files")) { - compute_file_masks(listing); - } - - if (this.get("current_path") === ".snapshots") { - compute_snapshot_display_names(listing); - } - - const search = this.get("file_search"); - if (search && search[0] !== search_escape_char) { - listing = _matched_files(search.toLowerCase(), listing); - } - - const sorter = (() => { - switch (this.get("active_file_sort").get("column_name")) { - case "name": - return _sort_on_string_field("name"); - case "time": - return _sort_on_numerical_field("mtime", -1); - case "size": - return _sort_on_numerical_field("size"); - case "type": - return (a, b) => { - if (a.isdir && !b.isdir) { - return -1; - } else if (b.isdir && !a.isdir) { - return 1; - } else { - return misc.cmp_array( - a.name.split(".").reverse(), - b.name.split(".").reverse(), - ); - } - }; - } - })(); - - listing.sort(sorter); - - if (this.get("active_file_sort").get("is_descending")) { - listing.reverse(); - } - - if (!this.get("show_hidden")) { - listing = (() => { - const result: DirectoryListing = []; - for (const l of listing) { - if (!l.name.startsWith(".")) { - result.push(l); - } - } - return result; - })(); - } - - if (!this.get("show_masked", true)) { - // if we do not gray out files (and hence haven't computed the file mask yet) - // we do it now! - if (!this.get("other_settings").get("mask_files")) { - compute_file_masks(listing); - } - - const filtered: DirectoryListing = []; - for (const f of listing) { - if (!f.mask) filtered.push(f); - } - listing = filtered; - } - - const file_map = {}; - for (const v of listing) { - file_map[v.name] = v; - } - - const data = { - listing, - public: {}, - path: this.get("current_path"), - file_map, - }; - - mutate_data_to_compute_public_files( - data, - this.get("stripped_public_paths"), - this.get("current_path"), - ); - - return data; - }, - }, stripped_public_paths: { dependencies: ["public_paths"] as const, @@ -565,20 +404,6 @@ export class ProjectStore extends Store { return this.getIn(["open_files", path]) != null; }; - get_item_in_path = (name, path) => { - const listing = this.get("directory_listings").get(path); - if (typeof listing === "string") { - // must be an error - return { err: listing }; - } - return { - item: - listing != null - ? listing.find((val) => val.get("name") === name) - : undefined, - }; - }; - fileURL = (path, compute_server_id?: number) => { return fileURL({ project_id: this.project_id, @@ -605,89 +430,6 @@ export class ProjectStore extends Store { // note that component is NOT an immutable.js object: return this.getIn(["open_files", path, "component"])?.Editor != null; } - - public get_listings(compute_server_id: number | null = null): Listings { - const computeServerId = compute_server_id ?? this.get("compute_server_id"); - if (this.listings[computeServerId] == null) { - const listingsTable = listings(this.project_id, computeServerId); - this.listings[computeServerId] = listingsTable; - listingsTable.watch(this.get("current_path") ?? "", true); - listingsTable.on("change", async (paths) => { - let directory_listings_for_server = - this.getIn(["directory_listings", computeServerId]) ?? - immutable.Map(); - - const missing: string[] = []; - for (const path of paths) { - if (listingsTable.getMissing(path)) { - missing.push(path); - } - const files = await listingsTable.getForStore(path); - directory_listings_for_server = directory_listings_for_server.set( - path, - files, - ); - } - const f = () => { - const actions = redux.getProjectActions(this.project_id); - const directory_listings = this.get("directory_listings").set( - computeServerId, - directory_listings_for_server, - ); - actions.setState({ directory_listings }); - }; - f(); - - if (missing.length > 0) { - for (const path of missing) { - try { - const files = immutable.fromJS( - await listingsTable.getListingDirectly(path), - ); - directory_listings_for_server = directory_listings_for_server.set( - path, - files, - ); - } catch { - // happens if e.g., the project is not running - continue; - } - } - f(); - } - }); - } - if (this.listings[computeServerId] == null) { - throw Error("bug"); - } - return this.listings[computeServerId]; - } -} - -function _matched_files(search: string, listing: DirectoryListing) { - if (listing == null) { - return []; - } - const words = misc.search_split(search); - const v: DirectoryListing = []; - for (const x of listing) { - const name = (x.display_name ?? x.name ?? "").toLowerCase(); - if ( - misc.search_match(name, words) || - (x.isdir && misc.search_match(name + "/", words)) - ) { - v.push(x); - } - } - return v; -} - -function compute_snapshot_display_names(listing): void { - for (const item of listing) { - const tm = misc.parse_bup_timestamp(item.name); - item.display_name = `${tm}`; - item.mtime = tm.valueOf() / 1000; - } } // Mutates data to include info on public paths. @@ -723,26 +465,6 @@ export function mutate_data_to_compute_public_files( } } -function _sort_on_string_field(field) { - return function (a, b) { - return misc.cmp( - a[field] !== undefined ? a[field].toLowerCase() : "", - b[field] !== undefined ? b[field].toLowerCase() : "", - ); - }; -} - -function _sort_on_numerical_field(field, factor = 1) { - return (a, b) => { - const c = misc.cmp( - (a[field] != null ? a[field] : -1) * factor, - (b[field] != null ? b[field] : -1) * factor, - ); - if (c) return c; - // break ties using the name, so well defined. - return misc.cmp(a.name, b.name) * factor; - }; -} export function init(project_id: string, redux: AppRedux): ProjectStore { const name = project_redux_name(project_id); From bf6f1824dddcf2a6d280b3f199ff4c23837456c9 Mon Sep 17 00:00:00 2001 From: William Stein Date: Thu, 24 Jul 2025 05:11:20 +0000 Subject: [PATCH 083/270] trivial --- src/packages/frontend/components/virtuoso-scroll-hook.ts | 4 +++- .../frontend/project/explorer/file-listing/file-listing.tsx | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/packages/frontend/components/virtuoso-scroll-hook.ts b/src/packages/frontend/components/virtuoso-scroll-hook.ts index 8972a24333..43a2050623 100644 --- a/src/packages/frontend/components/virtuoso-scroll-hook.ts +++ b/src/packages/frontend/components/virtuoso-scroll-hook.ts @@ -15,6 +15,8 @@ the upstream Virtuoso project: https://github.com/petyosi/react-virtuoso/blob/m import LRU from "lru-cache"; import { useCallback, useMemo, useRef } from "react"; +const DEFAULT_VIEWPORT = 1000; + export interface ScrollState { index: number; offset: number; @@ -64,7 +66,7 @@ export default function useVirtuosoScrollHook({ }, [onScroll, cacheId]); return { - increaseViewportBy: 2000 /* a lot better default than 0 */, + increaseViewportBy: DEFAULT_VIEWPORT, initialTopMostItemIndex: (cacheId ? (cache.get(cacheId) ?? initialState) : initialState) ?? 0, scrollerRef: handleScrollerRef, diff --git a/src/packages/frontend/project/explorer/file-listing/file-listing.tsx b/src/packages/frontend/project/explorer/file-listing/file-listing.tsx index f7986dc500..4ef83fc7b7 100644 --- a/src/packages/frontend/project/explorer/file-listing/file-listing.tsx +++ b/src/packages/frontend/project/explorer/file-listing/file-listing.tsx @@ -81,10 +81,11 @@ function sortDesc(active_file_sort?): { } export function FileListing(props) { + const path = props.current_path; const fs = useFs({ project_id: props.project_id }); let { listing, error } = useListing({ fs, - path: props.current_path, + path, ...sortDesc(props.active_file_sort), cacheId: { project_id: props.project_id }, }); From e3a31bdfd7641a227464d6807419bfebdd47db8a Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 26 Jul 2025 16:00:14 +0000 Subject: [PATCH 084/270] conat fs.readdir -- implement options --- .../conat/files/test/local-path.test.ts | 58 ++++++++++++++++ src/packages/backend/files/sandbox/index.ts | 21 +++++- src/packages/conat/core/client.ts | 4 +- src/packages/conat/files/fs.ts | 67 ++++++++++++++++++- src/packages/conat/sync-doc/syncdb.ts | 6 +- src/packages/conat/sync-doc/syncstring.ts | 4 +- .../frontend/course/assignments/actions.ts | 8 +-- src/packages/sync/editor/generic/sync-doc.ts | 4 +- 8 files changed, 155 insertions(+), 17 deletions(-) diff --git a/src/packages/backend/conat/files/test/local-path.test.ts b/src/packages/backend/conat/files/test/local-path.test.ts index 43825d45bc..527012f559 100644 --- a/src/packages/backend/conat/files/test/local-path.test.ts +++ b/src/packages/backend/conat/files/test/local-path.test.ts @@ -118,6 +118,64 @@ describe("use all the standard api functions of fs", () => { expect(v).toEqual(["0", "1", "2", "3", "4", fire]); }); + it("readdir with the withFileTypes option", async () => { + const path = "readdir-1"; + await fs.mkdir(path); + expect(await fs.readdir(path, { withFileTypes: true })).toEqual([]); + await fs.writeFile(join(path, "a.txt"), ""); + + { + const v = await fs.readdir(path, { withFileTypes: true }); + expect(v.map(({ name }) => name)).toEqual(["a.txt"]); + expect(v.map((x) => x.isFile())).toEqual([true]); + } + { + await fs.mkdir(join(path, "co")); + const v = await fs.readdir(path, { withFileTypes: true }); + expect(v.map(({ name }) => name)).toEqual(["a.txt", "co"]); + expect(v.map((x) => x.isFile())).toEqual([true, false]); + expect(v.map((x) => x.isDirectory())).toEqual([false, true]); + } + + { + await fs.symlink(join(path, "a.txt"), join(path, "link")); + const v = await fs.readdir(path, { withFileTypes: true }); + expect(v.map(({ name }) => name)).toEqual(["a.txt", "co", "link"]); + expect(v[2].isSymbolicLink()).toEqual(true); + } + }); + + it("readdir with the recursive option", async () => { + const path = "readdir-2"; + await fs.mkdir(path); + expect(await fs.readdir(path, { recursive: true })).toEqual([]); + await fs.mkdir(join(path, "subdir")); + await fs.writeFile(join(path, "subdir", "b.txt"), "x"); + const v = await fs.readdir(path, { recursive: true }); + expect(v).toEqual(["subdir", "subdir/b.txt"]); + + // and withFileTypes + const w = await fs.readdir(path, { recursive: true, withFileTypes: true }); + expect(w.map(({ name }) => name)).toEqual(["subdir", "b.txt"]); + expect(w[0]).toEqual( + expect.objectContaining({ + name: "subdir", + parentPath: path, + path, + }), + ); + expect(w[0].isDirectory()).toBe(true); + expect(w[1]).toEqual( + expect.objectContaining({ + name: "b.txt", + parentPath: join(path, "subdir"), + path: join(path, "subdir"), + }), + ); + expect(w[1].isFile()).toBe(true); + expect(await fs.readFile(join(w[1].path, w[1].name), "utf8")).toEqual("x"); + }); + it("use the find command instead of readdir", async () => { const { stdout } = await fs.find("dirtest", "%f\n"); const v = stdout.toString().trim().split("\n"); diff --git a/src/packages/backend/files/sandbox/index.ts b/src/packages/backend/files/sandbox/index.ts index aa09b12c25..52749163fc 100644 --- a/src/packages/backend/files/sandbox/index.ts +++ b/src/packages/backend/files/sandbox/index.ts @@ -141,7 +141,6 @@ export class SandboxedFilesystem { } // pathInSandbox is *definitely* a path in the sandbox: const pathInSandbox = join(this.path, resolve("/", path)); - if (this.unsafeMode) { // not secure -- just convenient. return pathInSandbox; @@ -238,8 +237,24 @@ export class SandboxedFilesystem { return await readFile(await this.safeAbsPath(path), encoding); }; - readdir = async (path: string): Promise => { - return await readdir(await this.safeAbsPath(path)); + readdir = async (path: string, options?) => { + const x = (await readdir(await this.safeAbsPath(path), options)) as any[]; + if (options?.withFileTypes) { + // each entry in x has a path and parentPath field, which refers to the + // absolute paths to the directory that contains x or the target of x (if + // it is a link). This is an absolute path on the fileserver, which we try + // not to expose from the sandbox, hence we modify them all if possible. + for (const a of x) { + if (a.path.startsWith(this.path)) { + a.path = a.path.slice(this.path.length + 1); + } + if (a.parentPath.startsWith(this.path)) { + a.parentPath = a.parentPath.slice(this.path.length + 1); + } + } + } + + return x; }; readlink = async (path: string): Promise => { diff --git a/src/packages/conat/core/client.ts b/src/packages/conat/core/client.ts index adb360d216..d4349349b3 100644 --- a/src/packages/conat/core/client.ts +++ b/src/packages/conat/core/client.ts @@ -1504,9 +1504,9 @@ export class Client extends EventEmitter { await astream({ ...opts, client: this }), synctable: async (opts: SyncTableOptions): Promise => await createSyncTable({ ...opts, client: this }), - string: (opts: Omit): SyncString => + string: (opts: Omit, "fs">): SyncString => syncstring({ ...opts, client: this }), - db: (opts: Omit): SyncDB => + db: (opts: Omit, "fs">): SyncDB => syncdb({ ...opts, client: this }), }; diff --git a/src/packages/conat/files/fs.ts b/src/packages/conat/files/fs.ts index bd60f4763c..dc197a6ca1 100644 --- a/src/packages/conat/files/fs.ts +++ b/src/packages/conat/files/fs.ts @@ -1,3 +1,10 @@ +/* +Tests are in + +packages/backend/conat/files/test/local-path.test.ts + +*/ + import { type Client } from "@cocalc/conat/core/client"; import { conat } from "@cocalc/conat/client"; import { @@ -42,7 +49,9 @@ export interface Filesystem { lstat: (path: string) => Promise; mkdir: (path: string, options?) => Promise; readFile: (path: string, encoding?: any) => Promise; - readdir: (path: string) => Promise; + readdir(path: string, options?): Promise; + readdir(path: string, options: { withFileTypes?: false }): Promise; + readdir(path: string, options: { withFileTypes: true }): Promise; readlink: (path: string) => Promise; realpath: (path: string) => Promise; rename: (oldPath: string, newPath: string) => Promise; @@ -75,6 +84,40 @@ export interface Filesystem { listing?: (path: string) => Promise; } +interface IDirent { + name: string; + parentPath: string; + path: string; + type?: number; +} + +const DIRENT_TYPES = { + 0: "UNKNOWN", + 1: "FILE", + 2: "DIR", + 3: "LINK", + 4: "FIFO", + 5: "SOCKET", + 6: "CHAR", + 7: "BLOCK", +}; + +class Dirent { + constructor( + public name: string, + public parentPath: string, + public path: string, + public type: number, + ) {} + isFile = () => DIRENT_TYPES[this.type] == "FILE"; + isDirectory = () => DIRENT_TYPES[this.type] == "DIR"; + isSymbolicLink = () => DIRENT_TYPES[this.type] == "LINK"; + isFIFO = () => DIRENT_TYPES[this.type] == "FIFO"; + isSocket = () => DIRENT_TYPES[this.type] == "SOCKET"; + isCharacterDevice = () => DIRENT_TYPES[this.type] == "CHAR"; + isBlockDevice = () => DIRENT_TYPES[this.type] == "BLOCK"; +} + interface IStats { dev: number; ino: number; @@ -202,8 +245,16 @@ export async function fsServer({ service, fs, client }: Options) { async readFile(path: string, encoding?) { return await (await fs(this.subject)).readFile(path, encoding); }, - async readdir(path: string) { - return await (await fs(this.subject)).readdir(path); + async readdir(path: string, options?) { + const files = await (await fs(this.subject)).readdir(path, options); + if (!options?.withFileTypes) { + return files; + } + // Dirent - change the [Symbol(type)] field to something serializable so client can use this: + return files.map((x) => { + // @ts-ignore + return { ...x, type: x[Object.getOwnPropertySymbols(x)[0]] }; + }); }, async readlink(path: string) { return await (await fs(this.subject)).readlink(path); @@ -292,6 +343,16 @@ export function fsClient({ client ??= conat(); let call = client.call(subject); + const readdir0 = call.readdir.bind(call); + call.readdir = async (path: string, options?) => { + const files = await readdir0(path, options); + if (options?.withFileTypes) { + return files.map((x) => new Dirent(x.name, x.parentPath, x.path, x.type)); + } else { + return files; + } + }; + let constants: any = null; const stat0 = call.stat.bind(call); call.stat = async (path: string) => { diff --git a/src/packages/conat/sync-doc/syncdb.ts b/src/packages/conat/sync-doc/syncdb.ts index b67add9dea..4211ad8477 100644 --- a/src/packages/conat/sync-doc/syncdb.ts +++ b/src/packages/conat/sync-doc/syncdb.ts @@ -2,7 +2,11 @@ import { SyncClient } from "./sync-client"; import { SyncDB, type SyncDBOpts0 } from "@cocalc/sync/editor/db"; import { type Client as ConatClient } from "@cocalc/conat/core/client"; -export interface SyncDBOptions extends Omit { +export type MakeOptional = Omit & + Partial>; + +export interface SyncDBOptions + extends MakeOptional, "fs"> { client: ConatClient; // name of the file service that hosts this file: service?: string; diff --git a/src/packages/conat/sync-doc/syncstring.ts b/src/packages/conat/sync-doc/syncstring.ts index 37cf1185af..322a2621ed 100644 --- a/src/packages/conat/sync-doc/syncstring.ts +++ b/src/packages/conat/sync-doc/syncstring.ts @@ -4,8 +4,10 @@ import { type SyncStringOpts, } from "@cocalc/sync/editor/string/sync"; import { type Client as ConatClient } from "@cocalc/conat/core/client"; +import { type MakeOptional } from "./syncdb"; -export interface SyncStringOptions extends Omit { +export interface SyncStringOptions + extends MakeOptional, "fs"> { client: ConatClient; // name of the file server that hosts this document: service?: string; diff --git a/src/packages/frontend/course/assignments/actions.ts b/src/packages/frontend/course/assignments/actions.ts index a50934da00..2be16c86fe 100644 --- a/src/packages/frontend/course/assignments/actions.ts +++ b/src/packages/frontend/course/assignments/actions.ts @@ -1641,10 +1641,8 @@ ${details} const project_id = store.get("course_project_id"); let files; try { - files = await redux - .getProjectStore(project_id) - .get_listings() - .getListingDirectly(path); + const { fs } = this.course_actions.syncdb; + files = await fs.readdir(path, { withFileTypes: true }); } catch (err) { // This happens, e.g., if the instructor moves the directory // that contains their version of the ipynb file. @@ -1658,7 +1656,7 @@ ${details} if (this.course_actions.is_closed()) return result; const to_read = files - .filter((entry) => !entry.isdir && endswith(entry.name, ".ipynb")) + .filter((entry) => entry.isFile() && endswith(entry.name, ".ipynb")) .map((entry) => entry.name); const f: (file: string) => Promise = async (file) => { diff --git a/src/packages/sync/editor/generic/sync-doc.ts b/src/packages/sync/editor/generic/sync-doc.ts index 68e780ef0a..2295693231 100644 --- a/src/packages/sync/editor/generic/sync-doc.ts +++ b/src/packages/sync/editor/generic/sync-doc.ts @@ -119,7 +119,7 @@ export interface SyncOpts0 { data_server?: DataServer; // filesystem interface. - fs?: Filesystem; + fs: Filesystem; // if true, do not implicitly save on commit. This is very // useful for unit testing to easily simulate offline state. @@ -228,7 +228,7 @@ export class SyncDoc extends EventEmitter { private useConat: boolean; legacy: LegacyHistory; - private fs?: Filesystem; + public readonly fs: Filesystem; private noAutosave?: boolean; From f039e0d246dc2e8100d805d98674f4e2aae457bc Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 26 Jul 2025 16:17:51 +0000 Subject: [PATCH 085/270] add test of non-utf8 Buffer readdir --- .../conat/files/test/local-path.test.ts | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/src/packages/backend/conat/files/test/local-path.test.ts b/src/packages/backend/conat/files/test/local-path.test.ts index 527012f559..fa8d0f5acd 100644 --- a/src/packages/backend/conat/files/test/local-path.test.ts +++ b/src/packages/backend/conat/files/test/local-path.test.ts @@ -1,4 +1,4 @@ -import { link, readFile, symlink } from "node:fs/promises"; +import { link, readFile, stat, symlink, writeFile } from "node:fs/promises"; import { join } from "path"; import { fsClient } from "@cocalc/conat/files/fs"; import { randomId } from "@cocalc/conat/names"; @@ -8,6 +8,7 @@ import { createPathFileserver, cleanupFileservers, } from "@cocalc/backend/conat/files/test/util"; +import { TextDecoder } from "node:util"; beforeAll(before); @@ -176,6 +177,24 @@ describe("use all the standard api functions of fs", () => { expect(await fs.readFile(join(w[1].path, w[1].name), "utf8")).toEqual("x"); }); + it("readdir works with non-utf8 filenames in the path", async () => { + // this test uses internal implementation details (kind of crappy) + const path = "readdir-3"; + await fs.mkdir(path); + const fullPath = join(server.path, project_id, path); + + process.chdir(fullPath); + + const buf = Buffer.from([0xff, 0xfe, 0xfd]); + expect(() => { + const decoder = new TextDecoder("utf-8", { fatal: true }); + decoder.decode(buf); + }).toThrow("not valid"); + await writeFile(buf, "hi"); + const w = await fs.readdir(path, { encoding: "buffer" }); + expect(w[0]).toEqual(buf); + }); + it("use the find command instead of readdir", async () => { const { stdout } = await fs.find("dirtest", "%f\n"); const v = stdout.toString().trim().split("\n"); From e6b1f6bbdd8555f554e526ccdd996a711d6be666 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 26 Jul 2025 16:35:15 +0000 Subject: [PATCH 086/270] change mtime for listings, etc., to be in ms --- src/packages/backend/conat/files/test/listing.test.ts | 3 ++- src/packages/backend/conat/files/test/local-path.test.ts | 7 ++++--- src/packages/conat/files/listing.ts | 5 +++-- src/packages/conat/persist/storage.ts | 2 +- .../frontend/project/explorer/file-listing/file-row.tsx | 2 +- .../frontend/project/page/flyouts/files-controls.tsx | 4 ++-- src/packages/frontend/project/page/flyouts/files.tsx | 4 ++-- 7 files changed, 15 insertions(+), 12 deletions(-) diff --git a/src/packages/backend/conat/files/test/listing.test.ts b/src/packages/backend/conat/files/test/listing.test.ts index d19b459d94..1427f0b440 100644 --- a/src/packages/backend/conat/files/test/listing.test.ts +++ b/src/packages/backend/conat/files/test/listing.test.ts @@ -59,7 +59,8 @@ describe("creating a listing monitor starting with an empty directory", () => { it("create another monitor starting with the now nonempty directory", async () => { const dir2 = await listing({ path: "", fs }); - expect(Object.keys(dir.files)).toEqual(["a.txt"]); + expect(Object.keys(dir2.files!)).toEqual(["a.txt"]); + expect(dir.files["a.txt"].mtime).toBeCloseTo(dir2.files!["a.txt"].mtime); dir2.close(); }); diff --git a/src/packages/backend/conat/files/test/local-path.test.ts b/src/packages/backend/conat/files/test/local-path.test.ts index fa8d0f5acd..b7fdfc156d 100644 --- a/src/packages/backend/conat/files/test/local-path.test.ts +++ b/src/packages/backend/conat/files/test/local-path.test.ts @@ -1,4 +1,4 @@ -import { link, readFile, stat, symlink, writeFile } from "node:fs/promises"; +import { link, readFile, symlink, writeFile } from "node:fs/promises"; import { join } from "path"; import { fsClient } from "@cocalc/conat/files/fs"; import { randomId } from "@cocalc/conat/names"; @@ -183,14 +183,15 @@ describe("use all the standard api functions of fs", () => { await fs.mkdir(path); const fullPath = join(server.path, project_id, path); - process.chdir(fullPath); - const buf = Buffer.from([0xff, 0xfe, 0xfd]); expect(() => { const decoder = new TextDecoder("utf-8", { fatal: true }); decoder.decode(buf); }).toThrow("not valid"); + const c = process.cwd(); + process.chdir(fullPath); await writeFile(buf, "hi"); + process.chdir(c); const w = await fs.readdir(path, { encoding: "buffer" }); expect(w[0]).toEqual(buf); }); diff --git a/src/packages/conat/files/listing.ts b/src/packages/conat/files/listing.ts index d45bec5401..912363c0ab 100644 --- a/src/packages/conat/files/listing.ts +++ b/src/packages/conat/files/listing.ts @@ -24,6 +24,7 @@ export const typeDescription = { }; interface FileData { + // last modification time as time since epoch in **milliseconds** (as is usual for javascript) mtime: number; size: number; // isdir = mainly for backward compat: @@ -107,7 +108,7 @@ export class Listing extends EventEmitter { return; } const data: FileData = { - mtime: stats.mtimeMs / 1000, + mtime: stats.mtimeMs, size: stats.size, type: stats.type, }; @@ -162,7 +163,7 @@ async function getListing( try { const v = line.split("\0"); const name = v[0]; - const mtime = parseFloat(v[1]); + const mtime = parseFloat(v[1]) * 1000; const size = parseInt(v[2]); files[name] = { mtime, size, type: v[3] as FileTypeLabel }; if (v[3] == "l") { diff --git a/src/packages/conat/persist/storage.ts b/src/packages/conat/persist/storage.ts index 53971163ad..b12e849daa 100644 --- a/src/packages/conat/persist/storage.ts +++ b/src/packages/conat/persist/storage.ts @@ -367,7 +367,7 @@ export class PersistentStream extends EventEmitter { try { await this.db.backup(path); } catch (err) { - console.log(err); + // console.log(err); logger.debug("WARNING: error creating a backup", path, err); } }); diff --git a/src/packages/frontend/project/explorer/file-listing/file-row.tsx b/src/packages/frontend/project/explorer/file-listing/file-row.tsx index ca95d8258d..869d99b776 100644 --- a/src/packages/frontend/project/explorer/file-listing/file-row.tsx +++ b/src/packages/frontend/project/explorer/file-listing/file-row.tsx @@ -234,7 +234,7 @@ export const FileRow: React.FC = React.memo((props) => { try { return ( ); diff --git a/src/packages/frontend/project/page/flyouts/files-controls.tsx b/src/packages/frontend/project/page/flyouts/files-controls.tsx index 9669e1126b..18c2b5db2a 100644 --- a/src/packages/frontend/project/page/flyouts/files-controls.tsx +++ b/src/packages/frontend/project/page/flyouts/files-controls.tsx @@ -101,7 +101,7 @@ export function FilesSelectedControls({ function renderFileInfoBottom() { if (singleFile != null) { const { size, mtime, isdir } = singleFile; - const age = typeof mtime === "number" ? 1000 * mtime : null; + const age = typeof mtime === "number" ? mtime : null; return ( {age ? ( @@ -144,7 +144,7 @@ export function FilesSelectedControls({ const { size = 0, mtime, isdir } = file; totSize += isdir ? 0 : size; if (typeof mtime === "number") { - const dt = new Date(1000 * mtime); + const dt = new Date(mtime); if (startDT.getTime() === 0 || dt < startDT) startDT = dt; if (endDT.getTime() === 0 || dt > endDT) endDT = dt; } diff --git a/src/packages/frontend/project/page/flyouts/files.tsx b/src/packages/frontend/project/page/flyouts/files.tsx index aca83cbf65..b8ebdcc743 100644 --- a/src/packages/frontend/project/page/flyouts/files.tsx +++ b/src/packages/frontend/project/page/flyouts/files.tsx @@ -527,7 +527,7 @@ export function FilesFlyout({ if (typeof mtime === "number") { return ( @@ -570,7 +570,7 @@ export function FilesFlyout({ function renderListItem(index: number, item: DirectoryListingEntry) { const { mtime, mask = false } = item; - const age = typeof mtime === "number" ? 1000 * mtime : null; + const age = typeof mtime === "number" ? mtime : null; // either select by scrolling (and only scrolling!) or by clicks const isSelected = scrollIdx != null From 256affd9c6cc10d2a3ddb14446053b3fb0f72112 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 26 Jul 2025 18:43:43 +0000 Subject: [PATCH 087/270] filesystem rewrite frontend integration progress --- src/packages/conat/core/client.ts | 12 +- src/packages/conat/files/fs.ts | 30 +- .../frontend/project/directory-selector.tsx | 285 ++++++------------ .../frontend/project/listing/use-fs.ts | 11 +- src/packages/frontend/project_actions.ts | 20 +- src/packages/frontend/projects/actions.ts | 1 - 6 files changed, 154 insertions(+), 205 deletions(-) diff --git a/src/packages/conat/core/client.ts b/src/packages/conat/core/client.ts index d4349349b3..cdf1a9f89e 100644 --- a/src/packages/conat/core/client.ts +++ b/src/packages/conat/core/client.ts @@ -254,7 +254,7 @@ import { type SyncDB, type SyncDBOptions, } from "@cocalc/conat/sync-doc/syncdb"; -import { fsClient, DEFAULT_FILE_SERVICE } from "@cocalc/conat/files/fs"; +import { fsClient, fsSubject } from "@cocalc/conat/files/fs"; import TTL from "@isaacs/ttlcache"; import { ConatSocketServer, @@ -1168,7 +1168,7 @@ export class Client extends EventEmitter { if (s !== undefined) { return s; } - if (typeof name !== "string") { + if (typeof name !== "string" || name == "then") { return undefined; } return async (...args) => await call(name, args); @@ -1478,15 +1478,13 @@ export class Client extends EventEmitter { return sub; }; - fs = ({ - project_id, - service = DEFAULT_FILE_SERVICE, - }: { + fs = (opts: { project_id: string; + compute_server_id?: number; service?: string; }) => { return fsClient({ - subject: `${service}.project-${project_id}`, + subject: fsSubject(opts), client: this, }); }; diff --git a/src/packages/conat/files/fs.ts b/src/packages/conat/files/fs.ts index dc197a6ca1..bd29b7a394 100644 --- a/src/packages/conat/files/fs.ts +++ b/src/packages/conat/files/fs.ts @@ -13,6 +13,7 @@ import { type WatchIterator, } from "@cocalc/conat/files/watch"; import listing, { type Listing, type FileTypeLabel } from "./listing"; +import { isValidUUID } from "@cocalc/util/misc"; export const DEFAULT_FILE_SERVICE = "fs"; @@ -44,7 +45,7 @@ export interface Filesystem { constants: () => Promise<{ [key: string]: number }>; copyFile: (src: string, dest: string) => Promise; cp: (src: string, dest: string, options?) => Promise; - exists: (path: string) => Promise; + exists: (path: string) => Promise; link: (existingPath: string, newPath: string) => Promise; lstat: (path: string) => Promise; mkdir: (path: string, options?) => Promise; @@ -227,7 +228,7 @@ export async function fsServer({ service, fs, client }: Options) { async cp(src: string, dest: string, options?) { await (await fs(this.subject)).cp(src, dest, options); }, - async exists(path: string) { + async exists(path: string): Promise { return await (await fs(this.subject)).exists(path); }, async find(path: string, printf: string, options?: FindOptions) { @@ -333,6 +334,31 @@ export type FilesystemClient = Omit, "lstat"> & { lstat: (path: string) => Promise; }; +export function fsSubject({ + project_id, + compute_server_id = 0, + service = DEFAULT_FILE_SERVICE, +}: { + project_id: string; + compute_server_id?: number; + service?: string; +}) { + if (!isValidUUID(project_id)) { + throw Error(`project_id must be a valid uuid -- ${project_id}`); + } + if (typeof compute_server_id != "number") { + throw Error("compute_server_id must be a number"); + } + if (typeof service != "string") { + throw Error("service must be a string"); + } + if (compute_server_id) { + return `${service}/${compute_server_id}.project-${project_id}`; + } else { + return `${service}.project-${project_id}`; + } +} + export function fsClient({ client, subject, diff --git a/src/packages/frontend/project/directory-selector.tsx b/src/packages/frontend/project/directory-selector.tsx index 6ec742ccd9..bc10ca9fdb 100644 --- a/src/packages/frontend/project/directory-selector.tsx +++ b/src/packages/frontend/project/directory-selector.tsx @@ -21,13 +21,12 @@ import { } from "react"; import { Icon, Loading } from "@cocalc/frontend/components"; import { path_split } from "@cocalc/util/misc"; -import { exec } from "@cocalc/frontend/frame-editors/generic/client"; import { alert_message } from "@cocalc/frontend/alerts"; -import { delay } from "awaiting"; import { redux, useTypedRedux } from "@cocalc/frontend/app-framework"; -import useIsMountedRef from "@cocalc/frontend/app-framework/is-mounted-hook"; import { useFrameContext } from "@cocalc/frontend/frame-editors/frame-tree/frame-context"; import ShowError from "@cocalc/frontend/components/error"; +import useFs from "@cocalc/frontend/project/listing/use-fs"; +import useFiles from "@cocalc/frontend/project/listing/use-files"; const NEW_FOLDER = "New Folder"; @@ -74,11 +73,6 @@ export default function DirectorySelector({ "compute_server_id", ); const computeServerId = compute_server_id ?? fallbackComputeServerId; - const directoryListings = useTypedRedux( - { project_id }, - "directory_listings", - )?.get(computeServerId); - const isMountedRef = useIsMountedRef(); const [expandedPaths, setExpandedPaths] = useState>(() => { const expandedPaths: string[] = [""]; if (startingPath == null) { @@ -123,77 +117,6 @@ export default function DirectorySelector({ [selectedPaths, multi], ); - useEffect(() => { - // Run the loop below every 30s until project_id or expandedPaths changes (or unmount) - // in which case loop stops. If not unmount, then get new loops for new values. - if (!project_id) return; - const state = { loop: true }; - (async () => { - while (state.loop && isMountedRef.current) { - // Component is mounted, so call watch on all expanded paths. - const listings = redux - .getProjectStore(project_id) - .get_listings(computeServerId); - for (const path of expandedPaths) { - listings.watch(path); - } - await delay(30000); - } - })(); - return () => { - state.loop = false; - }; - }, [project_id, expandedPaths, computeServerId]); - - let body; - if (directoryListings == null) { - (async () => { - await delay(0); - // Ensure store gets initialized before redux - // E.g., for copy between projects you make this - // directory selector before even opening the project. - redux.getProjectStore(project_id); - })(); - body = ; - } else { - body = ( - <> - {}} - /> - - { - setShowHidden(!showHidden); - }} - > - Show hidden - - - ); - } - return ( - {body} + {}} + /> + + { + setShowHidden(!showHidden); + }} + > + Show hidden + ); } @@ -245,14 +198,10 @@ function SelectablePath({ return; // no-op } try { - await exec({ - command: "mv", - project_id, - path: path_split(path).head, - args: [tail, editedTail], - compute_server_id: computeServerId, - filesystem: true, - }); + const actions = redux.getProjectActions(project_id); + const fs = actions.fs(computeServerId); + const { head } = path_split(path); + await fs.rename(join(head, tail), join(head, editedTail)); setEditedTail(null); } catch (err) { alert_message({ type: "error", message: err.toString() }); @@ -415,83 +364,68 @@ function Directory(props) { } } +// Show the directories in path function Subdirs(props) { const { computeServerId, - directoryListings, path, project_id, showHidden, style, toggleSelection, } = props; - const x = directoryListings?.get(path); - const v = x?.toJS?.(); - if (v == null) { - (async () => { - // Must happen in a different render loop, hence the delay, because - // fetch can actually update the store in the same render loop. - await delay(0); - redux.getProjectActions(project_id)?.fetch_directory_listing({ path }); - })(); + const fs = useFs({ project_id, computeServerId }); + const { files, error, refresh } = useFiles({ + fs, + path, + cacheId: { project_id }, + }); + if (error) { + return ; + } + if (files == null) { return ; - } else { - const w: React.JSX.Element[] = []; - const base = !path ? "" : path + "/"; - const paths: string[] = []; - const newPaths: string[] = []; - for (const x of v) { - if (x?.isdir) { - if (x.name.startsWith(".") && !showHidden) continue; - if (x.name.startsWith(NEW_FOLDER)) { - newPaths.push(x.name); - } else { - paths.push(x.name); - } - } - } - paths.sort(); - newPaths.sort(); - const createProps = { - project_id, - path, - computeServerId, - directoryListings, - toggleSelection, - }; - w.push(); - for (const name of paths.concat(newPaths)) { - w.push(); - } - if (w.length > 10) { - w.push(); + } + + const w: React.JSX.Element[] = []; + const base = !path ? "" : path + "/"; + const paths: string[] = []; + const newPaths: string[] = []; + for (const name in files) { + if (!files[name].isdir) continue; + if (name.startsWith(".") && !showHidden) continue; + if (name.startsWith(NEW_FOLDER)) { + newPaths.push(name); + } else { + paths.push(name); } - return ( -
- {w} -
- ); } + paths.sort(); + newPaths.sort(); + const createProps = { + project_id, + path, + computeServerId, + toggleSelection, + }; + w.push(); + for (const name of paths.concat(newPaths)) { + w.push(); + } + if (w.length > 10) { + w.push(); + } + return ( +
+ {w} +
+ ); } -async function getValidPath( - project_id, - target, - directoryListings, - computeServerId, -) { - if ( - await pathExists(project_id, target, directoryListings, computeServerId) - ) { +async function getValidPath(project_id, target, computeServerId) { + if (await pathExists(project_id, target, computeServerId)) { let i: number = 1; - while ( - await pathExists( - project_id, - target + ` (${i})`, - directoryListings, - computeServerId, - ) - ) { + while (await pathExists(project_id, target + ` (${i})`, computeServerId)) { i += 1; } target += ` (${i})`; @@ -503,7 +437,6 @@ function CreateDirectory({ computeServerId, project_id, path, - directoryListings, toggleSelection, }) { const [error, setError] = useState(""); @@ -518,12 +451,7 @@ function CreateDirectory({ const target = path + (path != "" ? "/" : "") + value; (async () => { try { - const path1 = await getValidPath( - project_id, - target, - directoryListings, - computeServerId, - ); + const path1 = await getValidPath(project_id, target, computeServerId); setValue(path_split(path1).tail); setTimeout(() => { input_ref.current?.select(); @@ -536,18 +464,16 @@ function CreateDirectory({ const createFolder = async () => { setOpen(false); + if (!value?.trim()) return; try { - await exec({ - command: "mkdir", - args: ["-p", value], - project_id, - path, - compute_server_id: computeServerId, - filesystem: true, - }); + const actions = redux.getProjectActions(project_id); + const fs = actions.fs(computeServerId); + await fs.mkdir(join(path, value)); toggleSelection(value); } catch (err) { setError(`${err}`); + } finally { + setValue(NEW_FOLDER); } }; @@ -556,7 +482,8 @@ function CreateDirectory({ - New Folder + New + Folder } open={open} @@ -569,18 +496,19 @@ function CreateDirectory({ style={{ marginTop: "30px" }} value={value} onChange={(e) => setValue(e.target.value)} - onPressEnter={createFolder} + onPressEnter={() => createFolder()} autoFocus />
@@ -590,24 +518,9 @@ function CreateDirectory({ export async function pathExists( project_id: string, path: string, - directoryListings?, computeServerId?, ): Promise { - const { head, tail } = path_split(path); - let known = directoryListings?.get(head); - if (known == null) { - const actions = redux.getProjectActions(project_id); - await actions.fetch_directory_listing({ - path: head, - compute_server_id: computeServerId, - }); - } - known = directoryListings?.get(head); - if (known == null) { - return false; - } - for (const x of known) { - if (x.get("name") == tail) return true; - } - return false; + const actions = redux.getProjectActions(project_id); + const fs = actions.fs(computeServerId); + return await fs.exists(path); } diff --git a/src/packages/frontend/project/listing/use-fs.ts b/src/packages/frontend/project/listing/use-fs.ts index ff76d79801..1bf01ab762 100644 --- a/src/packages/frontend/project/listing/use-fs.ts +++ b/src/packages/frontend/project/listing/use-fs.ts @@ -10,11 +10,20 @@ import { useState } from "react"; // the typing for now) export default function useFs({ project_id, + compute_server_id, + computeServerId, }: { project_id: string; + compute_server_id?: number; + computeServerId?: number; }): FilesystemClient | null { const [fs] = useState(() => - webapp_client.conat_client.conat().fs({ project_id }), + webapp_client.conat_client + .conat() + .fs({ + project_id, + compute_server_id: compute_server_id ?? computeServerId, + }), ); return fs; } diff --git a/src/packages/frontend/project_actions.ts b/src/packages/frontend/project_actions.ts index d76df13f18..4647f056b0 100644 --- a/src/packages/frontend/project_actions.ts +++ b/src/packages/frontend/project_actions.ts @@ -399,6 +399,7 @@ export class ProjectActions extends Actions { this.open_files?.close(); delete this.open_files; this.state = "closed"; + this._filesystem = {}; }; private save_session(): void { @@ -1562,12 +1563,12 @@ export class ProjectActions extends Actions { // Update the directory listing cache for the given path. // Uses current path if path not provided. fetch_directory_listing = async (_opts?): Promise => { - console.log("TODO: eliminate code that uses fetch_directory_listing"); + console.trace("TODO: rewrite code that uses fetch_directory_listing"); }; public async fetch_directory_listing_directly(): Promise { - console.log( - "TODO: eliminate code that uses fetch_directory_listing_directly", + console.trace( + "TODO: rewrite code that uses fetch_directory_listing_directly", ); } @@ -2445,12 +2446,15 @@ export class ProjectActions extends Actions { } } - private _filesystem: FilesystemClient; - fs = (): FilesystemClient => { - this._filesystem ??= webapp_client.conat_client + // note: there is no need to explicitly close or await what is returned by + // fs(...) since it's just a lightweight wrapper object to format appropriate RPC calls. + private _filesystem: { [compute_server_id: number]: FilesystemClient } = {}; + fs = (compute_server_id?: number): FilesystemClient => { + compute_server_id ??= this.get_store()?.get("compute_server_id") ?? 0; + this._filesystem[compute_server_id] ??= webapp_client.conat_client .conat() - .fs({ project_id: this.project_id }); - return this._filesystem; + .fs({ project_id: this.project_id, compute_server_id }); + return this._filesystem[compute_server_id]; }; // if available in cache, this returns the filenames in the current directory, diff --git a/src/packages/frontend/projects/actions.ts b/src/packages/frontend/projects/actions.ts index 76aa6bb3c0..3a1fcf4716 100644 --- a/src/packages/frontend/projects/actions.ts +++ b/src/packages/frontend/projects/actions.ts @@ -464,7 +464,6 @@ export class ProjectsActions extends Actions { if (relation == null || ["public", "admin"].includes(relation)) { this.fetch_public_project_title(opts.project_id); } - project_actions.fetch_directory_listing(); if (opts.switch_to) { redux .getActions("page") From c52224bedf1868df4e1cacde3291c2b3ccc1ad52 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 26 Jul 2025 19:41:51 +0000 Subject: [PATCH 088/270] rewrite flyout panel directory listing to use new fs api. - wow, it's incredible how much this simplifies and cleans up the code! --- .../configuration/configuration-copying.tsx | 25 +--- .../frame-editors/code-editor/actions.ts | 11 ++ .../frame-editors/qmd-editor/actions.ts | 47 +------ .../frame-editors/rmd-editor/actions.ts | 45 +------ .../frame-editors/rmd-editor/utils.ts | 29 +++++ .../project/page/flyouts/file-list-item.tsx | 29 +++-- .../frontend/project/page/flyouts/files.tsx | 115 +++++------------- 7 files changed, 101 insertions(+), 200 deletions(-) diff --git a/src/packages/frontend/course/configuration/configuration-copying.tsx b/src/packages/frontend/course/configuration/configuration-copying.tsx index 4510ad0f23..41df5121e6 100644 --- a/src/packages/frontend/course/configuration/configuration-copying.tsx +++ b/src/packages/frontend/course/configuration/configuration-copying.tsx @@ -35,19 +35,15 @@ import { } from "antd"; import { useMemo, useState } from "react"; import { FormattedMessage, useIntl } from "react-intl"; - import { labels } from "@cocalc/frontend/i18n"; import { redux, useFrameContext, - useTypedRedux, } from "@cocalc/frontend/app-framework"; import { Icon } from "@cocalc/frontend/components"; import ShowError from "@cocalc/frontend/components/error"; import { COMMANDS } from "@cocalc/frontend/course/commands"; -import { exec } from "@cocalc/frontend/frame-editors/generic/client"; import { IntlMessage } from "@cocalc/frontend/i18n"; -import { pathExists } from "@cocalc/frontend/project/directory-selector"; import { ProjectTitle } from "@cocalc/frontend/projects/project-title"; import { isIntlMessage } from "@cocalc/util/i18n"; import { plural } from "@cocalc/util/misc"; @@ -446,10 +442,6 @@ function AddTarget({ settings, actions, project_id }) { const [path, setPath] = useState(""); const [error, setError] = useState(""); const [create, setCreate] = useState(""); - const directoryListings = useTypedRedux( - { project_id }, - "directory_listings", - )?.get(0); const add = async () => { try { @@ -458,19 +450,14 @@ function AddTarget({ settings, actions, project_id }) { throw Error(`'${path} is the current course'`); } setLoading(true); - const exists = await pathExists(project_id, path, directoryListings); - if (!exists) { + const projectActions = redux.getProjectActions(project_id); + const fs = projectActions.fs(); + if (!(await fs.exists(path))) { if (create) { - await exec({ - command: "touch", - args: [path], - project_id, - filesystem: true, - }); - } else { - setCreate(path); - return; + await fs.writeFile(path, ""); } + } else { + setCreate(path); } const copy_config_targets = getTargets(settings); copy_config_targets[`${project_id}/${path}`] = true; diff --git a/src/packages/frontend/frame-editors/code-editor/actions.ts b/src/packages/frontend/frame-editors/code-editor/actions.ts index 744008bf83..38cac66727 100644 --- a/src/packages/frontend/frame-editors/code-editor/actions.ts +++ b/src/packages/frontend/frame-editors/code-editor/actions.ts @@ -3195,4 +3195,15 @@ export class Actions< }); actions?.foldAllThreads(); } + + getComputeServerId = (): number | undefined => { + return this.redux + .getProjectActions(this.project_id) + .getComputeServerIdForFile(this.path); + }; + + fs = () => { + const a = this.redux.getProjectActions(this.project_id); + return a.fs(a.getComputeServerIdForFile(this.path)); + }; } diff --git a/src/packages/frontend/frame-editors/qmd-editor/actions.ts b/src/packages/frontend/frame-editors/qmd-editor/actions.ts index 50920aa7cc..17cea7239c 100644 --- a/src/packages/frontend/frame-editors/qmd-editor/actions.ts +++ b/src/packages/frontend/frame-editors/qmd-editor/actions.ts @@ -7,12 +7,8 @@ Quarto Editor Actions */ -import { Set } from "immutable"; import { debounce } from "lodash"; - -import { redux } from "@cocalc/frontend/app-framework"; import { markdown_to_html_frontmatter } from "@cocalc/frontend/markdown"; -import { path_split } from "@cocalc/util/misc"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; import { Actions as BaseActions, @@ -21,7 +17,7 @@ import { import { FrameTree } from "../frame-tree/types"; import { ExecOutput } from "../generic/client"; import { Actions as MarkdownActions } from "../markdown-editor/actions"; -import { derive_rmd_output_filename } from "../rmd-editor/utils"; +import { checkProducedFiles } from "../rmd-editor/utils"; import { convert } from "./qmd-converter"; const custom_pdf_error_message: string = ` @@ -112,44 +108,7 @@ export class Actions extends MarkdownActions { } async _check_produced_files(): Promise { - const project_actions = redux.getProjectActions(this.project_id); - if (project_actions == undefined) { - return; - } - const path = path_split(this.path).head; - await project_actions.fetch_directory_listing({ path }); - - const project_store = project_actions.get_store(); - if (project_store == undefined) { - return; - } - // TODO: change the 0 to the compute server when/if we ever support QMD on a compute server (which we don't) - const dir_listings = project_store.getIn(["directory_listings", 0]); - if (dir_listings == undefined) { - return; - } - const listing = dir_listings.get(path); - if (listing == undefined) { - return; - } - - let existing = Set(); - for (const ext of ["pdf", "html", "nb.html"]) { - // full path – basename might change - const expected_fn = derive_rmd_output_filename(this.path, ext); - const fn_exists = listing.some((entry) => { - const name = entry.get("name"); - return name === path_split(expected_fn).tail; - }); - if (fn_exists) { - existing = existing.add(ext); - } - } - - // console.log("setting derived_file_types to", existing.toJS()); - this.setState({ - derived_file_types: existing as any, - }); + await checkProducedFiles(this); } private set_log(output?: ExecOutput | undefined): void { @@ -248,4 +207,4 @@ export class Actions extends MarkdownActions { this.build(); } } -} +} \ No newline at end of file diff --git a/src/packages/frontend/frame-editors/rmd-editor/actions.ts b/src/packages/frontend/frame-editors/rmd-editor/actions.ts index 2b92b449a0..517cef0558 100644 --- a/src/packages/frontend/frame-editors/rmd-editor/actions.ts +++ b/src/packages/frontend/frame-editors/rmd-editor/actions.ts @@ -7,13 +7,9 @@ R Markdown Editor Actions */ -import { Set } from "immutable"; import { debounce } from "lodash"; - -import { redux } from "@cocalc/frontend/app-framework"; import { markdown_to_html_frontmatter } from "@cocalc/frontend/markdown"; import { open_new_tab } from "@cocalc/frontend/misc"; -import { path_split } from "@cocalc/util/misc"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; import { Actions as BaseActions, @@ -23,7 +19,7 @@ import { FrameTree } from "../frame-tree/types"; import { ExecOutput } from "../generic/client"; import { Actions as MarkdownActions } from "../markdown-editor/actions"; import { convert } from "./rmd-converter"; -import { derive_rmd_output_filename } from "./utils"; +import { checkProducedFiles } from "./utils"; const HELP_URL = "https://doc.cocalc.com/frame-editor.html#edit-rmd"; const MINIMAL = `--- @@ -139,44 +135,7 @@ export class Actions extends MarkdownActions { } async _check_produced_files(): Promise { - const project_actions = redux.getProjectActions(this.project_id); - if (project_actions == undefined) { - return; - } - const path = path_split(this.path).head; - await project_actions.fetch_directory_listing({ path }); - - const project_store = project_actions.get_store(); - if (project_store == undefined) { - return; - } - // TODO: change the 0 to the compute server when/if we ever support RMD on a compute server (which we don't) - const dir_listings = project_store.getIn(["directory_listings", 0]); - if (dir_listings == undefined) { - return; - } - const listing = dir_listings.get(path); - if (listing == undefined) { - return; - } - - let existing = Set(); - for (const ext of ["pdf", "html", "nb.html"]) { - // full path – basename might change - const expected_fn = derive_rmd_output_filename(this.path, ext); - const fn_exists = listing.some((entry) => { - const name = entry.get("name"); - return name === path_split(expected_fn).tail; - }); - if (fn_exists) { - existing = existing.add(ext); - } - } - - // console.log("setting derived_file_types to", existing.toJS()); - this.setState({ - derived_file_types: existing as any, - }); + await checkProducedFiles(this); } private set_log(output?: ExecOutput | undefined): void { diff --git a/src/packages/frontend/frame-editors/rmd-editor/utils.ts b/src/packages/frontend/frame-editors/rmd-editor/utils.ts index d4c129650a..63dfce78f1 100644 --- a/src/packages/frontend/frame-editors/rmd-editor/utils.ts +++ b/src/packages/frontend/frame-editors/rmd-editor/utils.ts @@ -5,6 +5,7 @@ import { change_filename_extension, path_split } from "@cocalc/util/misc"; import { join } from "path"; +import { Set } from "immutable"; // something in the rmarkdown source code replaces all spaces by dashes // [hsy] I think this is because of calling pandoc. @@ -17,3 +18,31 @@ export function derive_rmd_output_filename(path, ext) { // avoid a leading / if it's just a filename (i.e. head = '') return join(head, fn); } + +export async function checkProducedFiles(codeEditorActions) { + const project_actions = codeEditorActions.redux.getProjectActions( + codeEditorActions.project_id, + ); + if (project_actions == null) { + return; + } + + let existing = Set(); + const fs = codeEditorActions.fs(); + const f = async (ext: string) => { + const expectedFilename = derive_rmd_output_filename( + codeEditorActions.path, + ext, + ); + if (await fs.exists(expectedFilename)) { + existing = existing.add(ext); + } + }; + const v = ["pdf", "html", "nb.html"].map(f); + await Promise.all(v); + + // console.log("setting derived_file_types to", existing.toJS()); + codeEditorActions.setState({ + derived_file_types: existing as any, + }); +} diff --git a/src/packages/frontend/project/page/flyouts/file-list-item.tsx b/src/packages/frontend/project/page/flyouts/file-list-item.tsx index 348ab4dd12..341435e9d2 100644 --- a/src/packages/frontend/project/page/flyouts/file-list-item.tsx +++ b/src/packages/frontend/project/page/flyouts/file-list-item.tsx @@ -6,7 +6,6 @@ import { Button, Dropdown, MenuProps, Tooltip } from "antd"; import immutable from "immutable"; import { useIntl } from "react-intl"; - import { CSS, React, @@ -115,6 +114,7 @@ interface Item { name: string; size?: number; mask?: boolean; + link_target?: string; } interface FileListItemProps { @@ -219,7 +219,6 @@ export const FileListItem = React.memo((props: Readonly) => { function renderName(): React.JSX.Element { const name = item.name; - const path = isActive ? path_split(name).tail : name; const { name: basename, ext } = item.isdir ? { name: path, ext: "" } @@ -230,8 +229,8 @@ export const FileListItem = React.memo((props: Readonly) => { ? item.isopen ? { fontWeight: "bold" } : item.isdir - ? undefined - : { color: COLORS.FILE_EXT } + ? undefined + : { color: COLORS.FILE_EXT } : undefined; return ( @@ -252,6 +251,12 @@ export const FileListItem = React.memo((props: Readonly) => { ) ) : undefined} + {!!item.link_target && ( + <> + + {item.link_target} + + )} ); } @@ -279,8 +284,8 @@ export const FileListItem = React.memo((props: Readonly) => { ? "check-square" : "square" : item.isdir - ? "folder-open" - : file_options(item.name)?.icon ?? "file"); + ? "folder-open" + : (file_options(item.name)?.icon ?? "file")); return ( ) => { const actionNames = multiple ? ACTION_BUTTONS_MULTI : isdir - ? ACTION_BUTTONS_DIR - : ACTION_BUTTONS_FILE; + ? ACTION_BUTTONS_DIR + : ACTION_BUTTONS_FILE; for (const key of actionNames) { if (key === "download" && !item.isdir) continue; const disabled = @@ -527,10 +532,10 @@ export const FileListItem = React.memo((props: Readonly) => { ? FILE_ITEM_ACTIVE_STYLE_2 : {} : item.isopen - ? item.isactive - ? FILE_ITEM_ACTIVE_STYLE - : FILE_ITEM_OPENED_STYLE - : {}; + ? item.isactive + ? FILE_ITEM_ACTIVE_STYLE + : FILE_ITEM_OPENED_STYLE + : {}; return ( diff --git a/src/packages/frontend/project/page/flyouts/files.tsx b/src/packages/frontend/project/page/flyouts/files.tsx index b8ebdcc743..87555832a5 100644 --- a/src/packages/frontend/project/page/flyouts/files.tsx +++ b/src/packages/frontend/project/page/flyouts/files.tsx @@ -4,17 +4,14 @@ */ import { Alert, InputRef } from "antd"; -import { delay } from "awaiting"; -import { List, Map } from "immutable"; +import { List } from "immutable"; import { debounce, fromPairs } from "lodash"; import { Virtuoso, VirtuosoHandle } from "react-virtuoso"; - import { React, TypedMap, redux, useEffect, - useIsMountedRef, useLayoutEffect, useMemo, usePrevious, @@ -35,7 +32,6 @@ import { DirectoryListingEntry, FileMap, } from "@cocalc/frontend/project/explorer/types"; -import { WATCH_THROTTLE_MS } from "@cocalc/frontend/conat/listings"; import { mutate_data_to_compute_public_files } from "@cocalc/frontend/project_store"; import track from "@cocalc/frontend/user-tracking"; import { @@ -44,8 +40,6 @@ import { human_readable_size, path_split, path_to_file, - search_match, - search_split, separate_file_extension, tab_to_path, unreachable, @@ -59,6 +53,9 @@ import { FileListItem } from "./file-list-item"; import { FilesBottom } from "./files-bottom"; import { FilesHeader } from "./files-header"; import { fileItemStyle } from "./utils"; +import useFs from "@cocalc/frontend/project/listing/use-fs"; +import useListing from "@cocalc/frontend/project/listing/use-listing"; +import ShowError from "@cocalc/frontend/components/error"; type PartialClickEvent = Pick< React.MouseEvent | React.KeyboardEvent, @@ -100,7 +97,6 @@ export function FilesFlyout({ project_id, actions, } = useProjectContext(); - const isMountedRef = useIsMountedRef(); const rootRef = useRef(null as any); const refInput = useRef(null as any); const [rootHeightPx, setRootHeightPx] = useState(0); @@ -110,12 +106,6 @@ export function FilesFlyout({ const current_path = useTypedRedux({ project_id }, "current_path"); const strippedPublicPaths = useStrippedPublicPaths(project_id); const compute_server_id = useTypedRedux({ project_id }, "compute_server_id"); - const directoryListings: Map< - string, - TypedMap | null - > | null = useTypedRedux({ project_id }, "directory_listings")?.get( - compute_server_id, - ); const activeTab = useTypedRedux({ project_id }, "active_project_tab"); const activeFileSort: ActiveFileSort = useTypedRedux( { project_id }, @@ -143,25 +133,6 @@ export function FilesFlyout({ return tab_to_path(activeTab); }, [activeTab]); - // copied roughly from directory-selector.tsx - useEffect(() => { - // Run the loop below every 30s until project_id or current_path changes (or unmount) - // in which case loop stops. If not unmount, then get new loops for new values. - if (!project_id) return; - const state = { loop: true }; - (async () => { - while (state.loop && isMountedRef.current) { - // Component is mounted, so call watch on all expanded paths. - const listings = redux.getProjectStore(project_id).get_listings(); - listings.watch(current_path); - await delay(WATCH_THROTTLE_MS); - } - })(); - return () => { - state.loop = false; - }; - }, [project_id, current_path]); - // selecting files switches over to "select" mode or back to "open" useEffect(() => { if (mode === "open" && checked_files.size > 0) { @@ -172,6 +143,16 @@ export function FilesFlyout({ } }, [checked_files]); + const fs = useFs({ project_id, compute_server_id }); + const { + listing: directoryListing, + error: listingError, + refresh, + } = useListing({ + fs, + path: current_path, + }); + // active file: current editor is the file in the listing // empty: either no files, or just the ".." for the parent dir const [directoryFiles, fileMap, activeFile, isEmpty] = useMemo((): [ @@ -180,28 +161,19 @@ export function FilesFlyout({ DirectoryListingEntry | null, boolean, ] => { - if (directoryListings == null) return EMPTY_LISTING; - const filesStore = directoryListings.get(current_path); - if (filesStore == null) return EMPTY_LISTING; - - // TODO this is an error, process it - if (typeof filesStore === "string") return EMPTY_LISTING; - - const files: DirectoryListing | null = filesStore.toJS?.(); + const files = directoryListing; if (files == null) return EMPTY_LISTING; let activeFile: DirectoryListingEntry | null = null; compute_file_masks(files); - const searchWords = search_split(file_search.trim().toLowerCase()); + const searchWords = file_search.trim().toLowerCase(); - const procFiles = files + const processedFiles : DirectoryListingEntry[] = files .filter((file: DirectoryListingEntry) => { - file.name ??= ""; // sanitization - if (file_search === "") return true; - const fName = file.name.toLowerCase(); + const filename = file.name.toLowerCase(); return ( - search_match(fName, searchWords) || - ((file.isdir ?? false) && search_match(`${fName}/`, searchWords)) + filename.includes(searchWords) || + (file.isdir && `${filename}/`.includes(searchWords)) ); }) .filter( @@ -211,17 +183,17 @@ export function FilesFlyout({ (file: DirectoryListingEntry) => hidden || !file.name.startsWith("."), ); - // this shares the logic with what's in project_store.js + // this shares the logic with what's in project_store.ts mutate_data_to_compute_public_files( { - listing: procFiles, + listing: processedFiles, public: {}, }, strippedPublicPaths, current_path, ); - procFiles.sort((a, b) => { + processedFiles.sort((a, b) => { // This replicated what project_store is doing const col = activeFileSort.get("column_name"); switch (col) { @@ -245,7 +217,7 @@ export function FilesFlyout({ } }); - for (const file of procFiles) { + for (const file of processedFiles) { const fullPath = path_to_file(current_path, file.name); if (openFiles.some((path) => path == fullPath)) { file.isopen = true; @@ -257,26 +229,26 @@ export function FilesFlyout({ } if (activeFileSort.get("is_descending")) { - procFiles.reverse(); // inplace op + processedFiles.reverse(); // inplace op } - const isEmpty = procFiles.length === 0; + const isEmpty = processedFiles.length === 0; // the ".." dir does not change the isEmpty state // hide ".." if there is a search -- https://github.com/sagemathinc/cocalc/issues/6877 if (file_search === "" && current_path != "") { - procFiles.unshift({ + processedFiles.unshift({ name: "..", isdir: true, }); } // map each filename to it's entry in the directory listing - const fileMap = fromPairs(procFiles.map((file) => [file.name, file])); + const fileMap = fromPairs(processedFiles.map((file) => [file.name, file])); - return [procFiles, fileMap, activeFile, isEmpty]; + return [processedFiles, fileMap, activeFile, isEmpty]; }, [ - directoryListings, + directoryListing, activeFileSort, hidden, file_search, @@ -309,7 +281,7 @@ export function FilesFlyout({ useEffect(() => { setShowCheckboxIndex(null); - }, [directoryListings, current_path]); + }, [directoryListing, current_path]); const triggerRootResize = debounce( () => setRootHeightPx(rootRef.current?.clientHeight ?? 0), @@ -353,27 +325,6 @@ export function FilesFlyout({ return fileMap[basename]; } - if (directoryListings == null) { - (async () => { - await delay(0); - // Ensure store gets initialized before redux - // E.g., for copy between projects you make this - // directory selector before even opening the project. - redux.getProjectStore(project_id); - })(); - } - - if (directoryListings?.get(current_path) == null) { - (async () => { - // Must happen in a different render loop, hence the delay, because - // fetch can actually update the store in the same render loop. - await delay(0); - redux - .getProjectActions(project_id) - ?.fetch_directory_listing({ path: current_path }); - })(); - } - function open( e: PartialClickEvent, index: number, @@ -639,8 +590,7 @@ export function FilesFlyout({ } function renderListing(): React.JSX.Element { - const files = directoryListings?.get(current_path); - if (files == null) { + if (directoryListing == null) { return renderLoadingOrStartProject(); } @@ -683,6 +633,7 @@ export function FilesFlyout({ ref={rootRef} style={{ flex: "1 0 auto", flexDirection: "column", display: "flex" }} > + Date: Sat, 26 Jul 2025 20:25:04 +0000 Subject: [PATCH 089/270] update guide and fix some major bugs/issues - the biggest bug is cwd wasn't being updated which made it kind of useless (I think I caused this bug recently). I fixed this. - i updated the file stats thing to be much better (using our human_readable_size function) and simpler cleaner code - We should remove this or write something modern. See https://github.com/sagemathinc/cocalc/issues/8462 In particular: - the buttons look weird -- they are totally different than anywhere else in cocalc - clicking a button in there instantly does the thing, no matter how dangerous. This makes me a little scared to use this. - The files section isn't searchable and can be huge -- do I want to sort through 50 pages to find something - But of course the real thing that makes this so dates is that LLM's exist, and if there is anything they are good it, it's basic bash terminal stuff. And they have a VASTLY larger range of options for what they can do, with vastly better reasoning, than what is here. Even the cheapest old models... --- .../frame-editors/code-editor/actions.ts | 4 +- .../terminal-editor/commands-guide.tsx | 148 +++++++----------- .../terminal-editor/conat-terminal.ts | 2 +- .../terminal-editor/connected-terminal.ts | 23 ++- 4 files changed, 74 insertions(+), 103 deletions(-) diff --git a/src/packages/frontend/frame-editors/code-editor/actions.ts b/src/packages/frontend/frame-editors/code-editor/actions.ts index 38cac66727..8a7eb2c93e 100644 --- a/src/packages/frontend/frame-editors/code-editor/actions.ts +++ b/src/packages/frontend/frame-editors/code-editor/actions.ts @@ -1500,7 +1500,7 @@ export class Actions< return this.terminals.get_terminal(id, parent); } - public set_terminal_cwd(id: string, cwd: string): void { + set_terminal_cwd(id: string, cwd: string): void { this.save_editor_state(id, { cwd }); } @@ -3204,6 +3204,6 @@ export class Actions< fs = () => { const a = this.redux.getProjectActions(this.project_id); - return a.fs(a.getComputeServerIdForFile(this.path)); + return a.fs(a.getComputeServerIdForFile({ path: this.path })); }; } diff --git a/src/packages/frontend/frame-editors/terminal-editor/commands-guide.tsx b/src/packages/frontend/frame-editors/terminal-editor/commands-guide.tsx index 3f99d6a579..ba18b638f5 100644 --- a/src/packages/frontend/frame-editors/terminal-editor/commands-guide.tsx +++ b/src/packages/frontend/frame-editors/terminal-editor/commands-guide.tsx @@ -11,8 +11,7 @@ import { Table, Typography, } from "antd"; -import { List, Map } from "immutable"; - +import { Map } from "immutable"; import { ControlOutlined, FileOutlined, @@ -20,82 +19,45 @@ import { InfoCircleOutlined, QuestionCircleOutlined, } from "@ant-design/icons"; -import { - CSS, - React, - TypedMap, - useActions, - useEffect, - useState, - useTypedRedux, -} from "@cocalc/frontend/app-framework"; +import { createContext, useEffect, useState } from "react"; import { Icon } from "@cocalc/frontend/components"; -import { plural, round1 } from "@cocalc/util/misc"; -import { DirectoryListingEntry } from "../../project/explorer/types"; +import { human_readable_size, plural } from "@cocalc/util/misc"; import { TerminalActions } from "./actions"; import { Command, SelectFile } from "./commands-guide-components"; +import useFiles from "@cocalc/frontend/project/listing/use-files"; +import ShowError from "@cocalc/frontend/components/error"; const { Panel } = Collapse; interface Props { - font_size: number; - project_id: string; actions: TerminalActions; local_view_state: Map; } -export const TerminalActionsContext = React.createContext< +export const TerminalActionsContext = createContext< TerminalActions | undefined >(undefined); const ListingStatsInit = { - total: 0, num_files: 0, num_dirs: 0, - size_mib: 0, + size: 0, }; const info = "info"; -type ListingImm = List>; - -function listing2names(listing?): string[] { - if (listing == null) { - return []; - } else { - return listing - .map((val) => val.get("name")) - .sort() - .toJS(); - } -} - function cwd2path(cwd: string): string { return cwd.charAt(0) === "/" ? ".smc/root" + cwd : cwd; } -export const CommandsGuide: React.FC = React.memo((props: Props) => { - const { /*font_size,*/ actions, local_view_state, project_id } = props; - - const project_actions = useActions({ project_id }); - // TODO: for now just assuming in the project (not a compute server) -- problem - // is that the guide is general to the whole terminal not a particular frame, - // and each frame can be on a different compute server! Not worth solving if - // nobody is using either the guide or compute servers. - const directory_listings = useTypedRedux( - { project_id }, - "directory_listings", - )?.get(0); - - const [terminal_id, set_terminal_id] = useState(); +export function CommandsGuide({ actions, local_view_state }: Props) { + const [terminal_id, setTerminalId] = useState(); const [cwd, set_cwd] = useState(""); // default home directory const [hidden, set_hidden] = useState(false); // hidden files // empty immutable js list - const [listing, set_listing] = useState(List([])); - const [listing_stats, set_listing_stats] = useState(ListingStatsInit); - const [directorynames, set_directorynames] = useState([]); + const [directoryNames, set_directoryNames] = useState([]); const [filenames, set_filenames] = useState([]); // directory and filenames const [dir1, set_dir1] = useState(undefined); @@ -103,13 +65,17 @@ export const CommandsGuide: React.FC = React.memo((props: Props) => { const [fn2, set_fn2] = useState(undefined); useEffect(() => { - const tid = actions._get_most_recent_active_frame_id_of_type("terminal"); - if (tid == null) return; - if (terminal_id != tid) set_terminal_id(tid); + const terminalId = + actions._get_most_recent_active_frame_id_of_type("terminal"); + if (terminalId == null) { + return; + } + if (terminal_id != terminalId) { + setTerminalId(terminalId); + } }, [local_view_state]); useEffect(() => { - //const terminal = actions.get_terminal(tid); const next_cwd = local_view_state.getIn([ "editor_state", terminal_id, @@ -117,50 +83,45 @@ export const CommandsGuide: React.FC = React.memo((props: Props) => { ]) as string | undefined; if (next_cwd != null && cwd != next_cwd) { set_cwd(next_cwd); - project_actions?.fetch_directory_listing({ path: cwd2path(next_cwd) }); } }, [terminal_id, local_view_state]); - // if the working directory changes or the listing itself, recompute the listing we base the files on - useEffect(() => { - if (cwd == null) return; - set_listing(directory_listings?.get(cwd2path(cwd))); - }, [directory_listings, cwd]); + const { files, error, refresh } = useFiles({ + fs: actions.fs(), + path: cwd2path(cwd), + }); // finally, if the listing really did change – or show/hide hidden files toggled – recalculate everything useEffect(() => { - // a user reported a crash "Uncaught TypeError: listing.filter is not a function". - // This was because directory_listings is a map from path to either an immutable - // listing **or** an error, as you can see where it is set in the file frontend/project_actions.ts - // The typescript there just has an "any", because it's code that was partly converted from coffeescript. - // Fixing this by just doing listing?.filter==null instead of listing==null here, since dealing with - // an error isn't necessary for this command guide. - if ( - listing == null || - typeof listing == "string" || - listing?.filter == null - ) + if (files == null) { return; - const all_files = hidden - ? listing - : listing.filter((val) => !val.get("name").startsWith(".")); - const grouped = all_files.groupBy((val) => !!val.get("isdir")); - const dirnames = [".", "..", ...listing2names(grouped.get(true))]; - const filenames = listing2names(grouped.get(false)); - set_directorynames(dirnames); + } + const dirnames: string[] = [".", ".."]; + const filenames: string[] = []; + for (const name in files) { + if (!hidden && name.startsWith(".")) { + continue; + } + if (files[name].isdir) { + dirnames.push(name); + } else { + filenames.push(name); + } + } + dirnames.sort(); + filenames.sort(); + + set_directoryNames(dirnames); set_filenames(filenames); - const total = all_files.size; - const size_red = grouped - .get(false) - ?.reduce((cur, val) => cur + val.get("size", 0), 0); - const size = (size_red ?? 0) / (1024 * 1024); + const size = filenames + .map((name) => files[name].size ?? 0) + .reduce((a, b) => a + b, 0); set_listing_stats({ - total, num_files: filenames.length, - num_dirs: dirnames.length, - size_mib: size, + num_dirs: dirnames.length - 2, + size, }); - }, [listing, hidden]); + }, [files, hidden]); // we also clear selected files if they no longer exist useEffect(() => { @@ -170,16 +131,16 @@ export const CommandsGuide: React.FC = React.memo((props: Props) => { if (fn2 != null && !filenames.includes(fn2)) { set_fn2(undefined); } - if (dir1 != null && !directorynames.includes(dir1)) { + if (dir1 != null && !directoryNames.includes(dir1)) { set_dir1(undefined); } - }, [directorynames, filenames]); + }, [directoryNames, filenames]); function render_files() { - const dirs = directorynames.map((v) => ({ key: v, name: v, type: "dir" })); + const dirs = directoryNames.map((v) => ({ key: v, name: v, type: "dir" })); const fns = filenames.map((v) => ({ key: v, name: v, type: "file" })); const data = [...dirs, ...fns]; - const style: CSS = { cursor: "pointer" }; + const style = { cursor: "pointer" } as const; const columns = [ { title: "Name", @@ -306,7 +267,7 @@ export const CommandsGuide: React.FC = React.memo((props: Props) => { {plural(listing_stats.num_files, "file")},{" "} {listing_stats.num_dirs}{" "} {plural(listing_stats.num_dirs, "directory", "directories")},{" "} - {round1(listing_stats.size_mib)} MiB + {human_readable_size(listing_stats.size)} @@ -316,7 +277,7 @@ export const CommandsGuide: React.FC = React.memo((props: Props) => { - + @@ -430,7 +391,7 @@ export const CommandsGuide: React.FC = React.memo((props: Props) => { } function render() { - const style: CSS = { overflowY: "auto" }; + const style = { overflowY: "auto" } as const; return ( } key={info}> @@ -501,7 +462,8 @@ export const CommandsGuide: React.FC = React.memo((props: Props) => { return ( + {render()} ); -}); +} diff --git a/src/packages/frontend/frame-editors/terminal-editor/conat-terminal.ts b/src/packages/frontend/frame-editors/terminal-editor/conat-terminal.ts index 5bb2c6a5bc..58e0e1d51e 100644 --- a/src/packages/frontend/frame-editors/terminal-editor/conat-terminal.ts +++ b/src/packages/frontend/frame-editors/terminal-editor/conat-terminal.ts @@ -24,7 +24,7 @@ export class ConatTerminal extends EventEmitter { private terminalResize; private openPaths; private closePaths; - private api: TerminalServiceApi; + public readonly api: TerminalServiceApi; private service?; private options?; private writeQueue: string = ""; diff --git a/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts b/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts index 95e9bf16fc..cfb7437e96 100644 --- a/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts +++ b/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts @@ -313,9 +313,6 @@ export class Terminal { this.conn = conn as any; conn.on("close", this.connect); conn.on("kick", this.close_request); - conn.on("cwd", (cwd) => { - this.actions.set_terminal_cwd(this.id, cwd); - }); conn.on("data", this.handleDataFromProject); conn.on("init", this.render); conn.once("ready", () => { @@ -438,7 +435,7 @@ export class Terminal { this.terminal.onTitleChange((title) => { if (title != null) { this.actions.set_title(this.id, title); - this.ask_for_cwd(); + this.update_cwd(); } }); }; @@ -720,9 +717,21 @@ export class Terminal { this.render_buffer = ""; }; - ask_for_cwd = debounce((): void => { - this.conn_write({ cmd: "cwd" }); - }); + update_cwd = debounce( + async () => { + let cwd; + try { + cwd = await this.conn?.api.cwd(); + } catch { + return; + } + if (cwd != null) { + this.actions.set_terminal_cwd(this.id, cwd); + } + }, + 1000, + { leading: true, trailing: true }, + ); kick_other_users_out(): void { // @ts-ignore From 4b36fa9bfd1b60d9e0ad2ffe8b95f3daf7c02ca3 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 26 Jul 2025 21:22:19 +0000 Subject: [PATCH 090/270] switch terminal to be ephemeral by default --- .../frame-editors/terminal-editor/connected-terminal.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts b/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts index cfb7437e96..34e73fb720 100644 --- a/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts +++ b/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts @@ -50,6 +50,11 @@ const MAX_DELAY = 15000; const ENABLE_WEBGL = false; + +// ephemeral = faster, less load on servers, but if project and browser all +// close, the history is gone... which may be good and less confusing. +const EPHEMERAL = true; + interface Path { file?: string; directory?: string; @@ -309,6 +314,7 @@ export class Terminal { cwd: this.workingDir, env: this.actions.get_term_env(), }, + ephemeral: EPHEMERAL, }); this.conn = conn as any; conn.on("close", this.connect); From 39ed974cb540c451b01f1909b2291dac31bfba8c Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 26 Jul 2025 21:29:39 +0000 Subject: [PATCH 091/270] no need to alert about opening the target of a symlink --- src/packages/frontend/project/open-file.ts | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/packages/frontend/project/open-file.ts b/src/packages/frontend/project/open-file.ts index d5d20be4b2..cb10653459 100644 --- a/src/packages/frontend/project/open-file.ts +++ b/src/packages/frontend/project/open-file.ts @@ -169,11 +169,6 @@ export async function open_file( } if (opts.path != realpath) { if (!actions.open_files) return; // closed - alert_message({ - type: "info", - message: `Opening normalized real path "${realpath}"`, - timeout: 10, - }); actions.open_files.delete(opts.path); opts.path = realpath; actions.open_files.set(opts.path, "component", {}); From d36bb978a0f1ab24240da337c75c9dc075836de5 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 26 Jul 2025 22:41:53 +0000 Subject: [PATCH 092/270] fix a case where I guess trying to edit a readonly sagews would crash (and generally seems good to be more careful here) --- .../frontend/project/page/file-tabs.tsx | 23 +++++++++++-------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/src/packages/frontend/project/page/file-tabs.tsx b/src/packages/frontend/project/page/file-tabs.tsx index 0e1eaab893..c593217ef2 100644 --- a/src/packages/frontend/project/page/file-tabs.tsx +++ b/src/packages/frontend/project/page/file-tabs.tsx @@ -100,9 +100,11 @@ export default function FileTabs({ openFiles, project_id, activeTab }) { if (action == "add") { actions.set_active_tab("files"); } else { - const path = keyToPath(key); - // close given file - actions.close_tab(path); + if (key) { + const path = keyToPath(key); + // close given file + actions.close_tab(path); + } } }; @@ -135,11 +137,14 @@ export default function FileTabs({ openFiles, project_id, activeTab }) { function onDragStart(event) { if (actions == null) return; if (event?.active?.id != activeKey) { - actions.set_active_tab(path_to_tab(keyToPath(event?.active?.id)), { - // noFocus -- critical to not focus when dragging or codemirror focus breaks on end of drag. - // See https://github.com/sagemathinc/cocalc/issues/7029 - noFocus: true, - }); + const key = event?.active?.id; + if (key) { + actions.set_active_tab(path_to_tab(keyToPath(key)), { + // noFocus -- critical to not focus when dragging or codemirror focus breaks on end of drag. + // See https://github.com/sagemathinc/cocalc/issues/7029 + noFocus: true, + }); + } } } @@ -160,7 +165,7 @@ export default function FileTabs({ openFiles, project_id, activeTab }) { activeKey={activeKey} type={"editable-card"} onChange={(key) => { - if (actions == null) return; + if (actions == null || !key) return; actions.set_active_tab(path_to_tab(keyToPath(key))); }} popupClassName={"cocalc-files-tabs-more"} From 3ec160a75822f7e11fd770115975d346e0180220 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 27 Jul 2025 01:19:24 +0000 Subject: [PATCH 093/270] jupyter: first steps on getting notebooks to start running just using an rpc call --- .../backend/conat/files/local-path.ts | 27 +++++++-- src/packages/conat/files/fs.ts | 26 +++++--- src/packages/conat/project/api/editor.ts | 14 +++++ src/packages/frontend/components/time-ago.tsx | 2 +- .../frontend/jupyter/browser-actions.ts | 47 +++++++++++++-- .../frontend/project/explorer/action-box.tsx | 1 - .../project/explorer/create-archive.tsx | 1 - .../frontend/project/explorer/download.tsx | 3 - .../frontend/project/explorer/rename-file.tsx | 1 - .../frontend/project/new/new-file-page.tsx | 11 +--- .../project/page/flyouts/files-header.tsx | 3 - .../frontend/project/page/flyouts/files.tsx | 5 +- src/packages/frontend/project_actions.ts | 16 ----- src/packages/jupyter/control.ts | 60 +++++++++++++++++++ src/packages/jupyter/kernel/kernel.ts | 13 ++-- src/packages/jupyter/redux/actions.ts | 25 ++------ src/packages/jupyter/redux/project-actions.ts | 2 +- src/packages/jupyter/zmq/index.ts | 10 ++-- src/packages/project/conat/api/editor.ts | 18 ++++++ src/packages/project/conat/files/fs.ts | 30 ++++++++++ src/packages/project/sagews/control.ts | 10 ++++ src/packages/util/redux/Actions.ts | 10 +--- 22 files changed, 237 insertions(+), 98 deletions(-) create mode 100644 src/packages/jupyter/control.ts create mode 100644 src/packages/project/conat/files/fs.ts create mode 100644 src/packages/project/sagews/control.ts diff --git a/src/packages/backend/conat/files/local-path.ts b/src/packages/backend/conat/files/local-path.ts index bc905561a6..3fa1c4a83a 100644 --- a/src/packages/backend/conat/files/local-path.ts +++ b/src/packages/backend/conat/files/local-path.ts @@ -10,22 +10,37 @@ export async function localPathFileserver({ path, service = DEFAULT_FILE_SERVICE, client, + project_id, + unsafeMode, }: { path: string; service?: string; client?: Client; + // if project_id is specified, use single project mode. + project_id?: string; + unsafeMode?: boolean; }) { client ??= conat(); + + const singleProjectFilesystem = project_id + ? new SandboxedFilesystem(path, { unsafeMode }) + : undefined; + const server = await fsServer({ service, client, + project_id, fs: async (subject: string) => { - const project_id = getProjectId(subject); - const p = join(path, project_id); - try { - await mkdir(p); - } catch {} - return new SandboxedFilesystem(p); + if (project_id) { + return singleProjectFilesystem!; + } else { + const project_id = getProjectId(subject); + const p = join(path, project_id); + try { + await mkdir(p); + } catch {} + return new SandboxedFilesystem(p, { unsafeMode }); + } }, }); return { server, client, path, service, close: () => server.close() }; diff --git a/src/packages/conat/files/fs.ts b/src/packages/conat/files/fs.ts index bd29b7a394..c7ce7280f8 100644 --- a/src/packages/conat/files/fs.ts +++ b/src/packages/conat/files/fs.ts @@ -206,11 +206,17 @@ interface Options { service: string; client?: Client; fs: (subject?: string) => Promise; + // project-id: if given, ONLY serve files for this one project, and the + // path must be the home of the project + // If not given, + project_id?: string; } -export async function fsServer({ service, fs, client }: Options) { +export async function fsServer({ service, fs, client, project_id }: Options) { client ??= conat(); - const subject = `${service}.*`; + const subject = project_id + ? `${service}.project-${project_id}` + : `${service}.*`; const watches: { [subject: string]: any } = {}; const sub = await client.service(subject, { async appendFile(path: string, data: string | Buffer, encoding?) { @@ -334,6 +340,16 @@ export type FilesystemClient = Omit, "lstat"> & { lstat: (path: string) => Promise; }; +export function getService({ + compute_server_id, + service = DEFAULT_FILE_SERVICE, +}: { + compute_server_id?: number; + service?: string; +}) { + return compute_server_id ? `${service}/${compute_server_id}` : service; +} + export function fsSubject({ project_id, compute_server_id = 0, @@ -352,11 +368,7 @@ export function fsSubject({ if (typeof service != "string") { throw Error("service must be a string"); } - if (compute_server_id) { - return `${service}/${compute_server_id}.project-${project_id}`; - } else { - return `${service}.project-${project_id}`; - } + return `${getService({ service, compute_server_id })}.project-${project_id}`; } export function fsClient({ diff --git a/src/packages/conat/project/api/editor.ts b/src/packages/conat/project/api/editor.ts index 9db2b45471..e17d597e8a 100644 --- a/src/packages/conat/project/api/editor.ts +++ b/src/packages/conat/project/api/editor.ts @@ -5,13 +5,21 @@ import type { KernelSpec } from "@cocalc/util/jupyter/types"; export const editor = { newFile: true, + + jupyterStart: true, + jupyterStop: true, jupyterStripNotebook: true, jupyterNbconvert: true, jupyterRunNotebook: true, jupyterKernelLogo: true, jupyterKernels: true, + formatString: true, + printSageWS: true, + sagewsStart: true, + sagewsStop: true, + createTerminalService: true, }; @@ -35,6 +43,10 @@ export interface Editor { jupyterStripNotebook: (path_ipynb: string) => Promise; + // path = the syncdb path (not *.ipynb) + jupyterStart: (path: string) => Promise; + jupyterStop: (path: string) => Promise; + jupyterNbconvert: (opts: NbconvertParams) => Promise; jupyterRunNotebook: (opts: RunNotebookOptions) => Promise; @@ -54,6 +66,8 @@ export interface Editor { }) => Promise; printSageWS: (opts) => Promise; + sagewsStart: (path_sagews: string) => Promise; + sagewsStop: (path_sagews: string) => Promise; createTerminalService: ( termPath: string, diff --git a/src/packages/frontend/components/time-ago.tsx b/src/packages/frontend/components/time-ago.tsx index 7259a5a75a..784747c0d3 100644 --- a/src/packages/frontend/components/time-ago.tsx +++ b/src/packages/frontend/components/time-ago.tsx @@ -203,7 +203,7 @@ export const TimeAgo: React.FC = React.memo( }: TimeAgoElementProps) => { const { timeAgoAbsolute } = useAppContext(); - if (date == null) { + if (!date || date.valueOf()) { return <>; } diff --git a/src/packages/frontend/jupyter/browser-actions.ts b/src/packages/frontend/jupyter/browser-actions.ts index 97f2e1f341..2779fccf87 100644 --- a/src/packages/frontend/jupyter/browser-actions.ts +++ b/src/packages/frontend/jupyter/browser-actions.ts @@ -56,6 +56,7 @@ import { syncdbPath } from "@cocalc/util/jupyter/names"; import getKernelSpec from "@cocalc/frontend/jupyter/kernelspecs"; import { get as getUsageInfo } from "@cocalc/conat/project/usage-info"; import { delay } from "awaiting"; +import { until } from "@cocalc/util/async-utils"; // local cache: map project_id (string) -> kernels (immutable) let jupyter_kernels = Map(); @@ -70,6 +71,7 @@ export class JupyterActions extends JupyterActions0 { protected init2(): void { this.syncdbPath = syncdbPath(this.path); + this.initBackend(); this.update_contents = debounce(this.update_contents.bind(this), 2000); this.setState({ toolbar: !this.get_local_storage("hide_toolbar"), @@ -172,6 +174,44 @@ export class JupyterActions extends JupyterActions0 { } } + // if the project or compute server is running and listening, this call + // tells them to open this jupyter notebook, so it can provide the compute + // functionality. + + private conatApi = async () => { + const compute_server_id = await this.getComputeServerId(); + const api = webapp_client.project_client.conatApi( + this.project_id, + compute_server_id, + ); + return api; + }; + + initBackend = async () => { + await until( + async () => { + if (this.is_closed()) { + return true; + } + try { + const api = await this.conatApi(); + await api.editor.jupyterStart(this.syncdbPath); + console.log("initialized ", this.path); + return true; + } catch (err) { + console.log("failed to initialize ", this.path, err); + return false; + } + }, + { min: 3000 }, + ); + }; + + stopBackend = async () => { + const api = await this.conatApi(); + await api.editor.jupyterStop(this.syncdbPath); + }; + initOpenLog = () => { // Put an entry in the project log once the jupyter notebook gets opened and // shows cells. @@ -354,10 +394,9 @@ export class JupyterActions extends JupyterActions0 { }; protected close_client_only(): void { - const account = this.redux.getStore("account"); - if (account != null) { - account.removeListener("change", this.account_change); - } + const account = this.redux + ?.getStore("account") + ?.removeListener("change", this.account_change); } private syncdb_cursor_activity = (): void => { diff --git a/src/packages/frontend/project/explorer/action-box.tsx b/src/packages/frontend/project/explorer/action-box.tsx index 949a87c17b..d52c5e9e18 100644 --- a/src/packages/frontend/project/explorer/action-box.tsx +++ b/src/packages/frontend/project/explorer/action-box.tsx @@ -122,7 +122,6 @@ export function ActionBox(props: ReactProps) { props.actions.delete_files({ paths }); props.actions.set_file_action(); props.actions.set_all_files_unchecked(); - props.actions.fetch_directory_listing(); } function render_delete_warning(): React.JSX.Element | undefined { diff --git a/src/packages/frontend/project/explorer/create-archive.tsx b/src/packages/frontend/project/explorer/create-archive.tsx index a18c2d9fa0..b599de0761 100644 --- a/src/packages/frontend/project/explorer/create-archive.tsx +++ b/src/packages/frontend/project/explorer/create-archive.tsx @@ -46,7 +46,6 @@ export default function CreateArchive({}) { dest: target + ".zip", path, }); - await actions.fetch_directory_listing({ path }); } catch (err) { setLoading(false); setError(err); diff --git a/src/packages/frontend/project/explorer/download.tsx b/src/packages/frontend/project/explorer/download.tsx index 24654ec1d1..ddaa75c793 100644 --- a/src/packages/frontend/project/explorer/download.tsx +++ b/src/packages/frontend/project/explorer/download.tsx @@ -89,9 +89,6 @@ export default function Download({}) { dest = files[0]; } actions.download_file({ path: dest, log: files }); - await actions.fetch_directory_listing({ - path: store.get("current_path"), - }); } catch (err) { console.log(err); setLoading(false); diff --git a/src/packages/frontend/project/explorer/rename-file.tsx b/src/packages/frontend/project/explorer/rename-file.tsx index 2b819ba9d5..327427126f 100644 --- a/src/packages/frontend/project/explorer/rename-file.tsx +++ b/src/packages/frontend/project/explorer/rename-file.tsx @@ -79,7 +79,6 @@ export default function RenameFile({ duplicate }: Props) { } else { await actions.rename_file(opts); } - await actions.fetch_directory_listing({ path: renameDir }); } catch (err) { setLoading(false); setError(err); diff --git a/src/packages/frontend/project/new/new-file-page.tsx b/src/packages/frontend/project/new/new-file-page.tsx index 015c97b069..d8f9ace744 100644 --- a/src/packages/frontend/project/new/new-file-page.tsx +++ b/src/packages/frontend/project/new/new-file-page.tsx @@ -250,11 +250,6 @@ export default function NewFilePage(props: Props) { { - getActions().fetch_directory_listing(); - }, - }} project_id={project_id} current_path={current_path} show_header={false} @@ -355,11 +350,7 @@ export default function NewFilePage(props: Props) { } values={{ upload: ( - getActions().fetch_directory_listing()} - /> + ), folder: (txt) => ( ): React.JSX.Element { actions?.fetch_directory_listing(), - }} config={{ clickable: `.${uploadClassName}` }} className="smc-vfill" > diff --git a/src/packages/frontend/project/page/flyouts/files.tsx b/src/packages/frontend/project/page/flyouts/files.tsx index 87555832a5..4ed6d1f2a7 100644 --- a/src/packages/frontend/project/page/flyouts/files.tsx +++ b/src/packages/frontend/project/page/flyouts/files.tsx @@ -167,7 +167,7 @@ export function FilesFlyout({ compute_file_masks(files); const searchWords = file_search.trim().toLowerCase(); - const processedFiles : DirectoryListingEntry[] = files + const processedFiles: DirectoryListingEntry[] = files .filter((file: DirectoryListingEntry) => { if (file_search === "") return true; const filename = file.name.toLowerCase(); @@ -661,9 +661,6 @@ export function FilesFlyout({ actions?.fetch_directory_listing(), - }} style={{ flex: "1 0 auto", display: "flex", diff --git a/src/packages/frontend/project_actions.ts b/src/packages/frontend/project_actions.ts index 4647f056b0..f038821e6a 100644 --- a/src/packages/frontend/project_actions.ts +++ b/src/packages/frontend/project_actions.ts @@ -1519,7 +1519,6 @@ export class ProjectActions extends Actions { compute_server_id, checked_files: store.get("checked_files").clear(), // always clear on compute_server_id change }); - this.fetch_directory_listing({ compute_server_id }); set_local_storage( store.computeServerIdLocalStorageKey, `${compute_server_id}`, @@ -1560,18 +1559,6 @@ export class ProjectActions extends Actions { }); } - // Update the directory listing cache for the given path. - // Uses current path if path not provided. - fetch_directory_listing = async (_opts?): Promise => { - console.trace("TODO: rewrite code that uses fetch_directory_listing"); - }; - - public async fetch_directory_listing_directly(): Promise { - console.trace( - "TODO: rewrite code that uses fetch_directory_listing_directly", - ); - } - // Sets the active file_sort to next_column_name set_sorted_file_column(column_name): void { let is_descending; @@ -2819,8 +2806,6 @@ export class ProjectActions extends Actions { explicit: true, compute_server_id, }); - } else { - this.fetch_directory_listing(); } }; @@ -2846,7 +2831,6 @@ export class ProjectActions extends Actions { alert: true, }); } finally { - this.fetch_directory_listing(); this.set_activity({ id, stop: "" }); this.setState({ downloading_file: false }); this.set_active_tab("files", { update_file_listing: false }); diff --git a/src/packages/jupyter/control.ts b/src/packages/jupyter/control.ts new file mode 100644 index 0000000000..49f5d8e1d2 --- /dev/null +++ b/src/packages/jupyter/control.ts @@ -0,0 +1,60 @@ +import { SyncDB } from "@cocalc/sync/editor/db/sync"; +import { SYNCDB_OPTIONS } from "@cocalc/jupyter/redux/sync"; +import { type Filesystem } from "@cocalc/conat/files/fs"; +import { getLogger } from "@cocalc/backend/logger"; +import { initJupyterRedux, removeJupyterRedux } from "@cocalc/jupyter/kernel"; +import { original_path } from "@cocalc/util/misc"; + +const logger = getLogger("jupyter:control"); + +const sessions: { [path: string]: SyncDB } = {}; +let project_id: string = ""; + +export function jupyterStart({ + path, + client, + project_id: project_id0, + fs, +}: { + path: string; + client; + project_id: string; + fs: Filesystem; +}) { + project_id = project_id0; + if (sessions[path] != null) { + logger.debug("jupyterStart: ", path, " - already running"); + return; + } + logger.debug("jupyterStart: ", path, " - starting it"); + const syncdb = new SyncDB({ + ...SYNCDB_OPTIONS, + project_id, + path, + client, + fs, + }); + sessions[path] = syncdb; + // [ ] TODO: some way to convey this to clients (?) + syncdb.on("error", (err) => { + logger.debug(`syncdb error -- ${err}`, path); + jupyterStop({ path }); + }); + syncdb.on("close", () => { + jupyterStop({ path }); + }); + initJupyterRedux(syncdb, client); +} + +export function jupyterStop({ path }: { path: string }) { + const syncdb = sessions[path]; + if (syncdb == null) { + logger.debug("jupyterStop: ", path, " - not running"); + } else { + logger.debug("jupyterStop: ", path, " - stopping it"); + syncdb.close(); + delete sessions[path]; + const path_ipynb = original_path(path); + removeJupyterRedux(path_ipynb, project_id); + } +} diff --git a/src/packages/jupyter/kernel/kernel.ts b/src/packages/jupyter/kernel/kernel.ts index 9f12677ccd..d22e206c50 100644 --- a/src/packages/jupyter/kernel/kernel.ts +++ b/src/packages/jupyter/kernel/kernel.ts @@ -121,7 +121,7 @@ const SAGE_JUPYTER_ENV = merge(copy(process.env), { // the ipynb file, and this function creates the corresponding // actions and store, which make it possible to work with this // notebook. -export async function initJupyterRedux(syncdb: SyncDB, client: Client) { +export function initJupyterRedux(syncdb: SyncDB, client: Client) { const project_id = syncdb.project_id; if (project_id == null) { throw Error("project_id must be defined"); @@ -147,7 +147,7 @@ export async function initJupyterRedux(syncdb: SyncDB, client: Client) { // Having two at once basically results in things feeling hung. // This should never happen, but we ensure it // See https://github.com/sagemathinc/cocalc/issues/4331 - await removeJupyterRedux(path, project_id); + removeJupyterRedux(path, project_id); } const store = redux.createStore(name, JupyterStore); const actions = redux.createActions(name, JupyterActions); @@ -171,14 +171,11 @@ export async function getJupyterRedux(syncdb: SyncDB) { // Remove the store/actions for a given Jupyter notebook, // and also close the kernel if it is running. -export async function removeJupyterRedux( - path: string, - project_id: string, -): Promise { +export function removeJupyterRedux(path: string, project_id: string): void { logger.debug("removeJupyterRedux", path); // if there is a kernel, close it try { - await kernels.get(path)?.close(); + kernels.get(path)?.close(); } catch (_err) { // ignore } @@ -186,7 +183,7 @@ export async function removeJupyterRedux( const actions = redux.getActions(name); if (actions != null) { try { - await actions.close(); + actions.close(); } catch (err) { logger.debug( "removeJupyterRedux", diff --git a/src/packages/jupyter/redux/actions.ts b/src/packages/jupyter/redux/actions.ts index be7a437acf..2c8975435a 100644 --- a/src/packages/jupyter/redux/actions.ts +++ b/src/packages/jupyter/redux/actions.ts @@ -213,30 +213,15 @@ export abstract class JupyterActions extends Actions { } public is_closed(): boolean { - return this._state === "closed" || this._state === undefined; + return (this._state ?? "closed") === "closed"; } - public async close({ noSave }: { noSave?: boolean } = {}): Promise { + public close() { if (this.is_closed()) { return; } - // ensure save to disk happens: - // - it will automatically happen for the sync-doc file, but - // we also need it for the ipynb file... as ipynb is unique - // in having two formats. - if (!noSave) { - await this.save(); - } - if (this.is_closed()) { - return; - } - - if (this.syncdb != null) { - this.syncdb.close(); - } - if (this._file_watcher != null) { - this._file_watcher.close(); - } + this.syncdb?.close(); + this._file_watcher?.close(); if (this.is_project || this.is_compute_server) { this.close_project_only(); } else { @@ -246,7 +231,7 @@ export abstract class JupyterActions extends Actions { // since otherwise this.redux and this.name are gone, // which makes destroying the actions properly impossible. this.destroy(); - this.store.destroy(); + this.store?.destroy(); close(this); this._state = "closed"; } diff --git a/src/packages/jupyter/redux/project-actions.ts b/src/packages/jupyter/redux/project-actions.ts index ee1300deec..87eca214ed 100644 --- a/src/packages/jupyter/redux/project-actions.ts +++ b/src/packages/jupyter/redux/project-actions.ts @@ -1303,7 +1303,7 @@ export class JupyterActions extends JupyterActions0 { // we should terminate and clean up everything. if (this.isDeleted()) { dbg("ipynb file is deleted, so NOT saving to disk and closing"); - this.close({ noSave: true }); + this.close(); return; } diff --git a/src/packages/jupyter/zmq/index.ts b/src/packages/jupyter/zmq/index.ts index 3a18b162ba..632a3e5ff0 100644 --- a/src/packages/jupyter/zmq/index.ts +++ b/src/packages/jupyter/zmq/index.ts @@ -1,10 +1,10 @@ import { EventEmitter } from "events"; import { Dealer, Subscriber } from "zeromq"; import { Message } from "./message"; -import { getLogger } from "@cocalc/backend/logger"; import type { JupyterMessage } from "./types"; -const logger = getLogger("jupyter:zmq"); +//import { getLogger } from "@cocalc/backend/logger"; +//const logger = getLogger("jupyter:zmq"); type JupyterSocketName = "iopub" | "shell" | "stdin" | "control"; @@ -76,7 +76,7 @@ export class JupyterSockets extends EventEmitter { throw Error(`invalid socket name '${name}'`); } - logger.debug("send message", message); + //logger.debug("send message", message); const jMessage = new Message(message); socket.send( jMessage._encode( @@ -119,9 +119,9 @@ export class JupyterSockets extends EventEmitter { private listen = async (name: JupyterSocketName, socket) => { if (ZMQ_TYPE[name] == "sub") { - // subscribe to everything -- + // subscribe to everything -- // https://zeromq.github.io/zeromq.js/classes/Subscriber.html#subscribe - socket.subscribe(); + socket.subscribe(); } for await (const data of socket) { const mesg = Message._decode( diff --git a/src/packages/project/conat/api/editor.ts b/src/packages/project/conat/api/editor.ts index c8fde5365a..22cad3d183 100644 --- a/src/packages/project/conat/api/editor.ts +++ b/src/packages/project/conat/api/editor.ts @@ -7,6 +7,8 @@ export { get_kernel_data as jupyterKernels } from "@cocalc/jupyter/kernel/kernel export { newFile } from "@cocalc/backend/misc/new-file"; import { printSageWS as printSageWS0 } from "@cocalc/project/print_to_pdf"; +export { sagewsStart, sagewsStop } from "@cocalc/project/sagews/control"; + import { filename_extension } from "@cocalc/util/misc"; export async function printSageWS(opts): Promise { let pdf; @@ -32,3 +34,19 @@ export async function printSageWS(opts): Promise { } export { createTerminalService } from "@cocalc/project/conat/terminal"; + +import { getClient } from "@cocalc/project/client"; +import { project_id } from "@cocalc/project/data"; +import * as control from "@cocalc/jupyter/control"; +import { SandboxedFilesystem } from "@cocalc/backend/files/sandbox"; + +export async function jupyterStart(path: string) { + const fs = new SandboxedFilesystem(process.env.HOME ?? "/tmp", { + unsafeMode: true, + }); + await control.jupyterStart({ project_id, path, client: getClient(), fs }); +} + +export async function jupyterStop(path: string) { + await control.jupyterStop({ path }); +} diff --git a/src/packages/project/conat/files/fs.ts b/src/packages/project/conat/files/fs.ts new file mode 100644 index 0000000000..718d011ace --- /dev/null +++ b/src/packages/project/conat/files/fs.ts @@ -0,0 +1,30 @@ +/* +Fileserver with all safety off for the project. This is run inside the project by the project, +so the security is off. +*/ + +import { localPathFileserver } from "@cocalc/backend/conat/files/local-path"; +import { getService } from "@cocalc/conat/files/fs"; +import { compute_server_id, project_id } from "@cocalc/project/data"; +import { connectToConat } from "@cocalc/project/conat/connection"; + +let server: any = undefined; +export async function init() { + if (server) { + return; + } + const client = connectToConat(); + const service = getService({ compute_server_id }); + server = await localPathFileserver({ + client, + service, + path: process.env.HOME ?? "/tmp", + unsafeMode: true, + project_id, + }); +} + +export function close() { + server?.close(); + server = undefined; +} diff --git a/src/packages/project/sagews/control.ts b/src/packages/project/sagews/control.ts new file mode 100644 index 0000000000..fa31dd09f8 --- /dev/null +++ b/src/packages/project/sagews/control.ts @@ -0,0 +1,10 @@ +import { getLogger } from "@cocalc/backend/logger"; +const logger = getLogger("project:sagews:control"); + +export async function sagewsStart(path_ipynb: string) { + logger.debug("sagewsStart: ", path_ipynb); +} + +export async function sagewsStop(path_ipynb: string) { + logger.debug("sagewsStop: ", path_ipynb); +} diff --git a/src/packages/util/redux/Actions.ts b/src/packages/util/redux/Actions.ts index 27d9ffc098..032303919e 100644 --- a/src/packages/util/redux/Actions.ts +++ b/src/packages/util/redux/Actions.ts @@ -39,13 +39,9 @@ export class Actions { }; destroy = (): void => { - if (this.name == null) { - throw Error("unable to destroy actions because this.name is not defined"); - } - if (this.redux == null) { - throw Error( - `unable to destroy actions '${this.name}' since this.redux is not defined`, - ); + if (this.name == null || this.redux == null) { + // already closed + return; } // On the share server this.redux can be undefined at this point. this.redux.removeActions(this.name); From 986f664ec067ff2c91b3620c0ff0e7210c3a1c84 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 27 Jul 2025 02:17:24 +0000 Subject: [PATCH 094/270] wire up proof of concept to show we can do code evaluation in the ping time --- src/packages/conat/project/api/editor.ts | 2 + .../frontend/jupyter/browser-actions.ts | 13 ++++- src/packages/jupyter/control.ts | 52 +++++++++++++++++-- src/packages/jupyter/kernel/kernel.ts | 2 + src/packages/jupyter/redux/sync.ts | 4 +- src/packages/project/conat/api/editor.ts | 5 ++ 6 files changed, 70 insertions(+), 8 deletions(-) diff --git a/src/packages/conat/project/api/editor.ts b/src/packages/conat/project/api/editor.ts index e17d597e8a..2771f927b9 100644 --- a/src/packages/conat/project/api/editor.ts +++ b/src/packages/conat/project/api/editor.ts @@ -13,6 +13,7 @@ export const editor = { jupyterRunNotebook: true, jupyterKernelLogo: true, jupyterKernels: true, + jupyterRun: true, formatString: true, @@ -46,6 +47,7 @@ export interface Editor { // path = the syncdb path (not *.ipynb) jupyterStart: (path: string) => Promise; jupyterStop: (path: string) => Promise; + jupyterRun: (path: string, ids: string[]) => Promise; jupyterNbconvert: (opts: NbconvertParams) => Promise; diff --git a/src/packages/frontend/jupyter/browser-actions.ts b/src/packages/frontend/jupyter/browser-actions.ts index 2779fccf87..89be281203 100644 --- a/src/packages/frontend/jupyter/browser-actions.ts +++ b/src/packages/frontend/jupyter/browser-actions.ts @@ -212,6 +212,17 @@ export class JupyterActions extends JupyterActions0 { await api.editor.jupyterStop(this.syncdbPath); }; + // temporary proof of concept + runCell = async (id: string) => { + const api = await this.conatApi(); + const resp = await api.editor.jupyterRun(this.syncdbPath, [id]); + const output = {}; + for (let i = 0; i < resp.length; i++) { + output[i] = resp[i]; + } + this.syncdb.set({ id, type: "cell", output }); + }; + initOpenLog = () => { // Put an entry in the project log once the jupyter notebook gets opened and // shows cells. @@ -394,7 +405,7 @@ export class JupyterActions extends JupyterActions0 { }; protected close_client_only(): void { - const account = this.redux + this.redux ?.getStore("account") ?.removeListener("change", this.account_change); } diff --git a/src/packages/jupyter/control.ts b/src/packages/jupyter/control.ts index 49f5d8e1d2..d610d3807e 100644 --- a/src/packages/jupyter/control.ts +++ b/src/packages/jupyter/control.ts @@ -4,10 +4,11 @@ import { type Filesystem } from "@cocalc/conat/files/fs"; import { getLogger } from "@cocalc/backend/logger"; import { initJupyterRedux, removeJupyterRedux } from "@cocalc/jupyter/kernel"; import { original_path } from "@cocalc/util/misc"; +import { once } from "@cocalc/util/async-utils"; const logger = getLogger("jupyter:control"); -const sessions: { [path: string]: SyncDB } = {}; +const sessions: { [path: string]: { syncdb: SyncDB; actions; store } } = {}; let project_id: string = ""; export function jupyterStart({ @@ -34,7 +35,6 @@ export function jupyterStart({ client, fs, }); - sessions[path] = syncdb; // [ ] TODO: some way to convey this to clients (?) syncdb.on("error", (err) => { logger.debug(`syncdb error -- ${err}`, path); @@ -43,14 +43,56 @@ export function jupyterStart({ syncdb.on("close", () => { jupyterStop({ path }); }); - initJupyterRedux(syncdb, client); + const { actions, store } = initJupyterRedux(syncdb, client); + sessions[path] = { syncdb, actions, store }; +} + +// run the cells with given id... +export async function jupyterRun({ + path, + ids, +}: { + path: string; + ids: string[]; +}) { + logger.debug("jupyterRun", { path, ids }); + const session = sessions[path]; + if (session == null) { + throw Error(`${path} not running`); + } + const { syncdb, actions, store } = session; + if (syncdb.isClosed()) { + // shouldn't be possible + throw Error("syncdb is closed"); + } + if (!syncdb.isReady()) { + logger.debug("jupyterRun: waiting until ready"); + await once(syncdb, "ready"); + } + // for (let i = 0; i < ids.length; i++) { + // actions.run_cell(ids[i], false); + // } + logger.debug("jupyterRun: running"); + if (ids.length == 1) { + const code = store.get("cells").get(ids[0])?.get("input")?.trim(); + if (code) { + const result: any[] = []; + for (const x of await actions.jupyter_kernel.execute_code_now({ code })) { + if (x.msg_type == "execute_result") { + result.push(x.content); + } + } + return result; + } + } } export function jupyterStop({ path }: { path: string }) { - const syncdb = sessions[path]; - if (syncdb == null) { + const session = sessions[path]; + if (session == null) { logger.debug("jupyterStop: ", path, " - not running"); } else { + const { syncdb } = session; logger.debug("jupyterStop: ", path, " - stopping it"); syncdb.close(); delete sessions[path]; diff --git a/src/packages/jupyter/kernel/kernel.ts b/src/packages/jupyter/kernel/kernel.ts index d22e206c50..a3e67c6e9a 100644 --- a/src/packages/jupyter/kernel/kernel.ts +++ b/src/packages/jupyter/kernel/kernel.ts @@ -160,6 +160,8 @@ export function initJupyterRedux(syncdb: SyncDB, client: Client) { syncdb.once("ready", () => logger.debug("initJupyterRedux", path, "syncdb ready"), ); + + return { actions, store }; } export async function getJupyterRedux(syncdb: SyncDB) { diff --git a/src/packages/jupyter/redux/sync.ts b/src/packages/jupyter/redux/sync.ts index d36ef1a971..b94935c196 100644 --- a/src/packages/jupyter/redux/sync.ts +++ b/src/packages/jupyter/redux/sync.ts @@ -1,6 +1,6 @@ export const SYNCDB_OPTIONS = { - change_throttle: 50, // our UI/React can handle more rapid updates; plus we want output FAST. - patch_interval: 50, + change_throttle: 25, + patch_interval: 25, primary_keys: ["type", "id"], string_cols: ["input"], cursors: true, diff --git a/src/packages/project/conat/api/editor.ts b/src/packages/project/conat/api/editor.ts index 22cad3d183..f1ec6b82fa 100644 --- a/src/packages/project/conat/api/editor.ts +++ b/src/packages/project/conat/api/editor.ts @@ -47,6 +47,11 @@ export async function jupyterStart(path: string) { await control.jupyterStart({ project_id, path, client: getClient(), fs }); } +export async function jupyterRun(path: string, ids: string[]) { + await jupyterStart(path); + return await control.jupyterRun({ path, ids }); +} + export async function jupyterStop(path: string) { await control.jupyterStop({ path }); } From 80e70f4f5da75b71665f373f3acef3d23a989149 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 27 Jul 2025 05:20:01 +0000 Subject: [PATCH 095/270] sometimes err is a string so we can't do err.message in that case --- src/packages/frontend/conat/client.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/packages/frontend/conat/client.ts b/src/packages/frontend/conat/client.ts index 674aa05ac5..516c138143 100644 --- a/src/packages/frontend/conat/client.ts +++ b/src/packages/frontend/conat/client.ts @@ -314,7 +314,13 @@ export class ConatClient extends EventEmitter { const resp = await cn.request(subject, data, { timeout }); return resp.data; } catch (err) { - err.message = `${err.message} - callHub: subject='${subject}', name='${name}', `; + try { + err.message = `${err.message} - callHub: subject='${subject}', name='${name}', `; + } catch { + err = new Error( + `${err.message} - callHub: subject='${subject}', name='${name}', `, + ); + } throw err; } }; From bef4fff56baa7cfcbf20477f9c8e3e1426d76aca Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 27 Jul 2025 05:38:35 +0000 Subject: [PATCH 096/270] add better typing to the jupyter output handler --- .../jupyter/execute/output-handler.ts | 86 +++++++++++-------- src/packages/jupyter/ipynb/export-to-ipynb.ts | 10 ++- 2 files changed, 58 insertions(+), 38 deletions(-) diff --git a/src/packages/jupyter/execute/output-handler.ts b/src/packages/jupyter/execute/output-handler.ts index 80312d1e6b..61536a3f77 100644 --- a/src/packages/jupyter/execute/output-handler.ts +++ b/src/packages/jupyter/execute/output-handler.ts @@ -28,49 +28,65 @@ OutputHandler emits these events: import { callback } from "awaiting"; import { EventEmitter } from "events"; -import { - close, - defaults, - required, - server_time, - len, - to_json, - is_object, -} from "@cocalc/util/misc"; +import { close, server_time, len, to_json, is_object } from "@cocalc/util/misc"; +import { type TypedMap } from "@cocalc/util/types/typed-map"; const now = () => server_time().valueOf() - 0; const MIN_SAVE_INTERVAL_MS = 500; const MAX_SAVE_INTERVAL_MS = 45000; +import { type Cell } from "@cocalc/jupyter/ipynb/export-to-ipynb"; + +interface Message { + execution_state?; + execution_count?: number; + exec_count?: number | null; + code?: string; + status?; + source?; + name?: string; + opts?; + more_output?: boolean; + text?: string; + data?: { [mimeType: string]: any }; +} + +interface Options { + // object; the cell whose output (etc.) will get mutated + cell: Cell; + // If given, used to truncate, discard output messages; extra + // messages are saved and made available. + max_output_length?: number; + max_output_messages?: number; + // If no messages for this many ms, then we update via set to indicate + // that cell is being run. + report_started_ms?: number; + dbg?; +} + +type State = "ready" | "closed"; + export class OutputHandler extends EventEmitter { - private _opts: any; + private _opts: Options; private _n: number; private _clear_before_next_output: boolean; private _output_length: number; - private _in_more_output_mode: any; - private _state: any; - private _stdin_cb: any; + private _in_more_output_mode: boolean; + private _state: State; + private _stdin_cb?: Function; - // Never commit output to send to the frontend more frequently than this.saveIntervalMs + // Never commit output to send to the frontend more frequently + // than this.saveIntervalMs // Otherwise, we'll end up with a large number of patches. // We start out with MIN_SAVE_INTERVAL_MS and exponentially back it off to // MAX_SAVE_INTERVAL_MS. private lastSave: number = 0; private saveIntervalMs = MIN_SAVE_INTERVAL_MS; - constructor(opts: any) { + constructor(opts: Options) { super(); - this._opts = defaults(opts, { - cell: required, // object; the cell whose output (etc.) will get mutated - // If given, used to truncate, discard output messages; extra - // messages are saved and made available. - max_output_length: undefined, - max_output_messages: undefined, - report_started_ms: undefined, // If no messages for this many ms, then we update via set to indicate - // that cell is being run. - dbg: undefined, - }); + this._opts = opts; const { cell } = this._opts; cell.output = null; cell.exec_count = null; @@ -177,7 +193,7 @@ export class OutputHandler extends EventEmitter { this._clear_output(); }; - _clean_mesg = (mesg: any): void => { + _clean_mesg = (mesg: Message): void => { delete mesg.execution_state; delete mesg.code; delete mesg.status; @@ -190,7 +206,7 @@ export class OutputHandler extends EventEmitter { } }; - private _push_mesg = (mesg: any, save?: boolean): void => { + private _push_mesg = (mesg: Message, save?: boolean): void => { if (this._state === "closed") { return; } @@ -209,7 +225,7 @@ export class OutputHandler extends EventEmitter { this.lastSave = now(); } - if (this._opts.cell.output === null) { + if (this._opts.cell.output == null) { this._opts.cell.output = {}; } this._opts.cell.output[`${this._n}`] = mesg; @@ -217,7 +233,7 @@ export class OutputHandler extends EventEmitter { this.emit("change", save); }; - set_input = (input: any, save = true): void => { + set_input = (input: string, save = true): void => { if (this._state === "closed") { return; } @@ -226,8 +242,8 @@ export class OutputHandler extends EventEmitter { }; // Process incoming messages. This may mutate mesg. - message = (mesg: any): void => { - let has_exec_count: any; + message = (mesg: Message): void => { + let has_exec_count: boolean; if (this._state === "closed") { return; } @@ -299,7 +315,7 @@ export class OutputHandler extends EventEmitter { }; async stdin(prompt: string, password: boolean): Promise { - // See docs for stdin option to execute_code in backend jupyter.coffee + // See docs for stdin option to execute_code in backend. this._push_mesg({ name: "input", opts: { prompt, password } }); // Now we wait until the output message we just included has its // value set. Then we call cb with that value. @@ -310,14 +326,14 @@ export class OutputHandler extends EventEmitter { } // Call this when the cell changes; only used for stdin right now. - cell_changed = (cell: any, get_password: any): void => { + cell_changed = (cell: TypedMap, get_password: () => string): void => { if (this._state === "closed") { return; } if (this._stdin_cb == null) { return; } - const output = cell != null ? cell.get("output") : undefined; + const output = cell?.get("output"); if (output == null) { return; } @@ -346,7 +362,7 @@ export class OutputHandler extends EventEmitter { } }; - payload = (payload: any): void => { + payload = (payload: { source?; text: string }): void => { if (this._state === "closed") { return; } diff --git a/src/packages/jupyter/ipynb/export-to-ipynb.ts b/src/packages/jupyter/ipynb/export-to-ipynb.ts index 8c7ec57886..ebd57e53d1 100644 --- a/src/packages/jupyter/ipynb/export-to-ipynb.ts +++ b/src/packages/jupyter/ipynb/export-to-ipynb.ts @@ -13,7 +13,7 @@ type CellType = "code" | "markdown" | "raw"; type Tags = { [key: string]: boolean }; -interface Cell { +export interface Cell { cell_type?: CellType; input?: string; collapsed?: boolean; @@ -21,9 +21,13 @@ interface Cell { slide?; attachments?; tags?: Tags; - output?: { [n: string]: OutputMessage }; + output?: { [n: string]: OutputMessage } | null; metadata?: Metadata; - exec_count?: number; + exec_count?: number | null; + + start?: number | null; + end?: number | null; + state?: "done" | "busy" | "run"; } type OutputMessage = any; From 1ccb0997b5d3b4b06ecee917dbba40e4e9e53a73 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 27 Jul 2025 06:22:28 +0000 Subject: [PATCH 097/270] jupyter: faster execution proof of concept --- src/packages/conat/project/api/editor.ts | 5 +- .../frontend/jupyter/browser-actions.ts | 16 ++--- src/packages/frontend/jupyter/run-cell.ts | 30 +++++++++ src/packages/jupyter/control.ts | 28 +++------ .../jupyter/execute/output-handler.ts | 62 ++++++++++++++++--- src/packages/jupyter/package.json | 1 + src/packages/jupyter/redux/project-actions.ts | 49 +-------------- src/packages/pnpm-lock.yaml | 3 + src/packages/project/conat/api/editor.ts | 7 ++- 9 files changed, 116 insertions(+), 85 deletions(-) create mode 100644 src/packages/frontend/jupyter/run-cell.ts diff --git a/src/packages/conat/project/api/editor.ts b/src/packages/conat/project/api/editor.ts index 2771f927b9..c18d7d00f8 100644 --- a/src/packages/conat/project/api/editor.ts +++ b/src/packages/conat/project/api/editor.ts @@ -47,7 +47,10 @@ export interface Editor { // path = the syncdb path (not *.ipynb) jupyterStart: (path: string) => Promise; jupyterStop: (path: string) => Promise; - jupyterRun: (path: string, ids: string[]) => Promise; + jupyterRun: ( + path: string, + cells: { id: string; input: string }[], + ) => Promise; jupyterNbconvert: (opts: NbconvertParams) => Promise; diff --git a/src/packages/frontend/jupyter/browser-actions.ts b/src/packages/frontend/jupyter/browser-actions.ts index 89be281203..932bee24a3 100644 --- a/src/packages/frontend/jupyter/browser-actions.ts +++ b/src/packages/frontend/jupyter/browser-actions.ts @@ -57,6 +57,7 @@ import getKernelSpec from "@cocalc/frontend/jupyter/kernelspecs"; import { get as getUsageInfo } from "@cocalc/conat/project/usage-info"; import { delay } from "awaiting"; import { until } from "@cocalc/util/async-utils"; +import { runCell } from "./run-cell"; // local cache: map project_id (string) -> kernels (immutable) let jupyter_kernels = Map(); @@ -67,7 +68,7 @@ export class JupyterActions extends JupyterActions0 { private cursor_manager: CursorManager; private account_change_editor_settings: any; private update_keyboard_shortcuts: any; - private syncdbPath: string; + public syncdbPath: string; protected init2(): void { this.syncdbPath = syncdbPath(this.path); @@ -178,7 +179,7 @@ export class JupyterActions extends JupyterActions0 { // tells them to open this jupyter notebook, so it can provide the compute // functionality. - private conatApi = async () => { + conatApi = async () => { const compute_server_id = await this.getComputeServerId(); const api = webapp_client.project_client.conatApi( this.project_id, @@ -214,13 +215,7 @@ export class JupyterActions extends JupyterActions0 { // temporary proof of concept runCell = async (id: string) => { - const api = await this.conatApi(); - const resp = await api.editor.jupyterRun(this.syncdbPath, [id]); - const output = {}; - for (let i = 0; i < resp.length; i++) { - output[i] = resp[i]; - } - this.syncdb.set({ id, type: "cell", output }); + await runCell({ actions: this, id }); }; initOpenLog = () => { @@ -278,7 +273,8 @@ export class JupyterActions extends JupyterActions0 { this.clear_cell(id, save); return; } - this.run_code_cell(id, save, no_halt); + this.runCell(id); + //this.run_code_cell(id, save, no_halt); if (save) { this.save_asap(); } diff --git a/src/packages/frontend/jupyter/run-cell.ts b/src/packages/frontend/jupyter/run-cell.ts new file mode 100644 index 0000000000..6c818e5582 --- /dev/null +++ b/src/packages/frontend/jupyter/run-cell.ts @@ -0,0 +1,30 @@ +import { OutputHandler } from "@cocalc/jupyter/execute/output-handler"; +import { type JupyterActions } from "./browser-actions"; + +export async function runCell({ + actions, + id, +}: { + actions: JupyterActions; + id: string; +}) { + const cell = actions.store.getIn(["cells", id])?.toJS(); + if (cell == null) { + // nothing to do + return; + } + cell.output = null; + actions._set(cell); + const handler = new OutputHandler({ cell }); + const api = await actions.conatApi(); + const mesgs = await api.editor.jupyterRun(actions.syncdbPath, [ + { id: cell.id, input: cell.input }, + ]); + console.log(mesgs); + for (const mesg of mesgs) { + handler.process(mesg); + } + cell.state = "done"; + console.log(cell); + actions._set(cell); +} diff --git a/src/packages/jupyter/control.ts b/src/packages/jupyter/control.ts index d610d3807e..499e830945 100644 --- a/src/packages/jupyter/control.ts +++ b/src/packages/jupyter/control.ts @@ -50,17 +50,17 @@ export function jupyterStart({ // run the cells with given id... export async function jupyterRun({ path, - ids, + cells, }: { path: string; - ids: string[]; + cells: { id: string; input: string }[]; }) { - logger.debug("jupyterRun", { path, ids }); + logger.debug("jupyterRun", { path, cells }); const session = sessions[path]; if (session == null) { throw Error(`${path} not running`); } - const { syncdb, actions, store } = session; + const { syncdb, actions } = session; if (syncdb.isClosed()) { // shouldn't be possible throw Error("syncdb is closed"); @@ -69,22 +69,14 @@ export async function jupyterRun({ logger.debug("jupyterRun: waiting until ready"); await once(syncdb, "ready"); } - // for (let i = 0; i < ids.length; i++) { - // actions.run_cell(ids[i], false); - // } logger.debug("jupyterRun: running"); - if (ids.length == 1) { - const code = store.get("cells").get(ids[0])?.get("input")?.trim(); - if (code) { - const result: any[] = []; - for (const x of await actions.jupyter_kernel.execute_code_now({ code })) { - if (x.msg_type == "execute_result") { - result.push(x.content); - } - } - return result; - } + let v = []; + for (const cell of cells) { + v = v.concat( + await actions.jupyter_kernel.execute_code_now({ code: cell.input }), + ); } + return v; } export function jupyterStop({ path }: { path: string }) { diff --git a/src/packages/jupyter/execute/output-handler.ts b/src/packages/jupyter/execute/output-handler.ts index 61536a3f77..0d6bec0dea 100644 --- a/src/packages/jupyter/execute/output-handler.ts +++ b/src/packages/jupyter/execute/output-handler.ts @@ -28,7 +28,7 @@ OutputHandler emits these events: import { callback } from "awaiting"; import { EventEmitter } from "events"; -import { close, server_time, len, to_json, is_object } from "@cocalc/util/misc"; +import { close, server_time, len, is_object } from "@cocalc/util/misc"; import { type TypedMap } from "@cocalc/util/types/typed-map"; const now = () => server_time().valueOf() - 0; @@ -62,7 +62,6 @@ interface Options { // If no messages for this many ms, then we update via set to indicate // that cell is being run. report_started_ms?: number; - dbg?; } type State = "ready" | "closed"; @@ -107,6 +106,57 @@ export class OutputHandler extends EventEmitter { this.stdin = this.stdin.bind(this); } + // mesg = from the kernel + process = (mesg) => { + if (mesg == null) { + // can't possibly happen, + return; + } + if (mesg.done) { + // done is a special internal cocalc message. + this.done(); + return; + } + if (mesg.content?.transient?.display_id != null) { + //this.handleTransientUpdate(mesg); + if (mesg.msg_type == "update_display_data") { + // don't also create a new output + return; + } + } + + if (mesg.msg_type === "clear_output") { + this.clear(mesg.content.wait); + return; + } + + if (mesg.content.comm_id != null) { + // ignore any comm/widget related messages here + return; + } + + if (mesg.content.execution_state === "busy") { + this.start(); + } + + if (mesg.content.payload != null) { + if (mesg.content.payload.length > 0) { + // payload shell message: + // Despite https://ipython.org/ipython-doc/3/development/messaging.html#payloads saying + // ""Payloads are considered deprecated, though their replacement is not yet implemented." + // we fully have to implement them, since they are used to implement (crazy, IMHO) + // things like %load in the python2 kernel! + for (const p of mesg.content.payload) { + this.payload(p); + } + return; + } + } else { + // Normal iopub output message + this.message(mesg.content); + } + }; + close = (): void => { if (this._state == "closed") return; this._state = "closed"; @@ -241,7 +291,8 @@ export class OutputHandler extends EventEmitter { this.emit("change", save); }; - // Process incoming messages. This may mutate mesg. + // Process incoming messages. **This may mutate mesg** and + // definitely mutates this.cell. message = (mesg: Message): void => { let has_exec_count: boolean; if (this._state === "closed") { @@ -375,10 +426,7 @@ export class OutputHandler extends EventEmitter { // https://github.com/sagemathinc/cocalc/issues/1933 this.message(payload); } else { - // No idea what to do with this... - if (typeof this._opts.dbg === "function") { - this._opts.dbg(`Unknown PAYLOAD: ${to_json(payload)}`); - } + // TODO: No idea what to do with this... } }; } diff --git a/src/packages/jupyter/package.json b/src/packages/jupyter/package.json index a109440ca8..7c4bd52173 100644 --- a/src/packages/jupyter/package.json +++ b/src/packages/jupyter/package.json @@ -47,6 +47,7 @@ "@cocalc/util": "workspace:*", "awaiting": "^3.0.0", "debug": "^4.4.0", + "events": "3.3.0", "expect": "^26.6.2", "he": "^1.2.0", "immutable": "^4.3.0", diff --git a/src/packages/jupyter/redux/project-actions.ts b/src/packages/jupyter/redux/project-actions.ts index 87eca214ed..92d28879b6 100644 --- a/src/packages/jupyter/redux/project-actions.ts +++ b/src/packages/jupyter/redux/project-actions.ts @@ -718,7 +718,6 @@ export class JupyterActions extends JupyterActions0 { max_output_length: this.store.get("max_output_length"), max_output_messages: MAX_OUTPUT_MESSAGES, report_started_ms: 250, - dbg, }); dbg("setting up jupyter_kernel.once('closed', ...) handler"); @@ -885,63 +884,19 @@ export class JupyterActions extends JupyterActions0 { exec.on("output", (mesg) => { // uncomment only for specific low level debugging -- see https://github.com/sagemathinc/cocalc/issues/7022 // dbg(`got mesg='${JSON.stringify(mesg)}'`); // !!!☡ ☡ ☡ -- EXTREME DANGER ☡ ☡ ☡ !!!! - - if (mesg == null) { - // can't possibly happen, of course. - const err = "empty mesg"; - dbg(`got error='${err}'`); - handler.error(err); - return; - } - if (mesg.done) { - // done is a special internal cocalc message. - handler.done(); - return; - } if (mesg.content?.transient?.display_id != null) { // See https://github.com/sagemathinc/cocalc/issues/2132 // We find any other outputs in the document with // the same transient.display_id, and set their output to // this mesg's output. this.handleTransientUpdate(mesg); - if (mesg.msg_type == "update_display_data") { - // don't also create a new output - return; - } } - - if (mesg.msg_type === "clear_output") { - handler.clear(mesg.content.wait); - return; - } - - if (mesg.content.comm_id != null) { - // ignore any comm/widget related messages - return; - } - if (mesg.content.execution_state === "idle") { this.store.removeListener("cell_change", cell_change); return; } - if (mesg.content.execution_state === "busy") { - handler.start(); - } - if (mesg.content.payload != null) { - if (mesg.content.payload.length > 0) { - // payload shell message: - // Despite https://ipython.org/ipython-doc/3/development/messaging.html#payloads saying - // ""Payloads are considered deprecated, though their replacement is not yet implemented." - // we fully have to implement them, since they are used to implement (crazy, IMHO) - // things like %load in the python2 kernel! - mesg.content.payload.map((p) => handler.payload(p)); - return; - } - } else { - // Normal iopub output message - handler.message(mesg.content); - return; - } + + handler.process(mesg); }); exec.on("error", (err) => { diff --git a/src/packages/pnpm-lock.yaml b/src/packages/pnpm-lock.yaml index 901cc56769..20bdac5535 100644 --- a/src/packages/pnpm-lock.yaml +++ b/src/packages/pnpm-lock.yaml @@ -853,6 +853,9 @@ importers: debug: specifier: ^4.4.0 version: 4.4.1 + events: + specifier: 3.3.0 + version: 3.3.0 expect: specifier: ^26.6.2 version: 26.6.2 diff --git a/src/packages/project/conat/api/editor.ts b/src/packages/project/conat/api/editor.ts index f1ec6b82fa..5783ddae67 100644 --- a/src/packages/project/conat/api/editor.ts +++ b/src/packages/project/conat/api/editor.ts @@ -47,9 +47,12 @@ export async function jupyterStart(path: string) { await control.jupyterStart({ project_id, path, client: getClient(), fs }); } -export async function jupyterRun(path: string, ids: string[]) { +export async function jupyterRun( + path: string, + cells: { id: string; input: string }[], +) { await jupyterStart(path); - return await control.jupyterRun({ path, ids }); + return await control.jupyterRun({ path, cells }); } export async function jupyterStop(path: string) { From 979b272d2104f71640b2fff672d1c24ed90c7198 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 27 Jul 2025 14:49:05 +0000 Subject: [PATCH 098/270] conat: basic framework for running jupyter code --- .../test/project/jupyter/run-code.test.ts | 122 ++++++++++++++++ .../conat/project/jupyter/run-code.ts | 135 ++++++++++++++++++ 2 files changed, 257 insertions(+) create mode 100644 src/packages/backend/conat/test/project/jupyter/run-code.test.ts create mode 100644 src/packages/conat/project/jupyter/run-code.ts diff --git a/src/packages/backend/conat/test/project/jupyter/run-code.test.ts b/src/packages/backend/conat/test/project/jupyter/run-code.test.ts new file mode 100644 index 0000000000..6dbb84f2e7 --- /dev/null +++ b/src/packages/backend/conat/test/project/jupyter/run-code.test.ts @@ -0,0 +1,122 @@ +/* + +DEVELOPMENT: + +pnpm test `pwd`/run-code.test.ts + +*/ + +import { before, after, connect } from "@cocalc/backend/conat/test/setup"; +import { + jupyterClient, + jupyterServer, +} from "@cocalc/conat/project/jupyter/run-code"; +import { uuid } from "@cocalc/util/misc"; + +beforeAll(before); + +describe("create very simple mocked jupyter runner and test evaluating code", () => { + let client1, client2; + it("create two clients", async () => { + client1 = connect(); + client2 = connect(); + }); + + let server; + const project_id = uuid(); + it("create jupyter code run server", () => { + // running code with this just results in two responses: the path and the cells + async function jupyterRun({ path, cells }) { + async function* runner() { + yield [{ path }]; + yield [{ cells }]; + } + return runner(); + } + + server = jupyterServer({ client: client1, project_id, jupyterRun }); + }); + + let client; + const path = "a.ipynb"; + const cells = [{ id: "a", input: "2+3" }]; + it("create a jupyter client, then run some code", async () => { + client = jupyterClient({ path, project_id, client: client2 }); + const iter = await client.run(cells); + const v: any[] = []; + for await (const output of iter) { + v.push(output); + } + expect(v).toEqual([[{ path }], [{ cells }]]); + }); + + const count = 100; + it(`run ${count} evaluations to ensure that the speed is reasonable (and also everything is kept properly ordered, etc.)`, async () => { + const start = Date.now(); + for (let i = 0; i < count; i++) { + const v: any[] = []; + const cells = [{ id: `${i}`, input: `${i} + ${i}` }]; + for await (const output of await client.run(cells)) { + v.push(output); + } + expect(v).toEqual([[{ path }], [{ cells }]]); + } + const evalsPerSecond = Math.floor((1000 * count) / (Date.now() - start)); + if (process.env.BENCH) { + console.log({ evalsPerSecond }); + } + expect(evalsPerSecond).toBeGreaterThan(25); + }); + + it("cleans up", () => { + server.close(); + client.close(); + }); +}); + +describe("create simple mocked jupyter runner that does actually eval an expression", () => { + let client1, client2; + it("create two clients", async () => { + client1 = connect(); + client2 = connect(); + }); + + let server; + const project_id = uuid(); + it("create jupyter code run server", () => { + // running code with this just results in two responses: the path and the cells + async function jupyterRun({ cells }) { + async function* runner() { + for (const { id, input } of cells) { + yield [{ id, output: eval(input) }]; + } + } + return runner(); + } + + server = jupyterServer({ client: client1, project_id, jupyterRun }); + }); + + let client; + const path = "b.ipynb"; + const cells = [ + { id: "a", input: "2+3" }, + { id: "b", input: "3**5" }, + ]; + it("create a jupyter client, then run some code", async () => { + client = jupyterClient({ path, project_id, client: client2 }); + const iter = await client.run(cells); + const v: any[] = []; + for await (const output of iter) { + v.push(output); + } + expect(v).toEqual([[{ id: "a", output: 5 }], [{ id: "b", output: 243 }]]); + }); + + it("cleans up", () => { + server.close(); + client.close(); + }); +}); + +afterAll(after); diff --git a/src/packages/conat/project/jupyter/run-code.ts b/src/packages/conat/project/jupyter/run-code.ts new file mode 100644 index 0000000000..1cc87d3873 --- /dev/null +++ b/src/packages/conat/project/jupyter/run-code.ts @@ -0,0 +1,135 @@ +/* +A conat socket server that takes as input + +Tests are in + +packages/backend/conat/test/juypter/run-code.test.s + +*/ + +import { type Client as ConatClient } from "@cocalc/conat/core/client"; +import { + type ConatSocketServer, + type ServerSocket, +} from "@cocalc/conat/socket"; +import { EventIterator } from "@cocalc/util/event-iterator"; +import { getLogger } from "@cocalc/conat/client"; + +const logger = getLogger("conat:project:jupyter:run-code"); + +function getSubject({ + project_id, + compute_server_id = 0, +}: { + project_id: string; + compute_server_id?: number; +}) { + return `jupyter.project-${project_id}.${compute_server_id}`; +} + +interface InputCell { + id: string; + input: string; +} + +type OutputMessage = any; + +type JupyterCodeRunner = (opts: { + // syncdb path + path: string; + // array of input cells to run + cells: InputCell[]; +}) => Promise>; + +export function jupyterServer({ + client, + project_id, + compute_server_id = 0, + jupyterRun, +}: { + client: ConatClient; + project_id: string; + compute_server_id?: number; + jupyterRun: JupyterCodeRunner; +}) { + const subject = getSubject({ project_id, compute_server_id }); + const server: ConatSocketServer = client.socket.listen(subject); + logger.debug("server: listening on ", { subject }); + + server.on("connection", (socket: ServerSocket) => { + logger.debug("server: got new connection", { + id: socket.id, + subject: socket.subject, + }); + + socket.on("request", async (mesg) => { + const { path, cells } = mesg.data; + try { + mesg.respondSync(null); + await handleRequest({ socket, jupyterRun, path, cells }); + } catch (err) { + console.log(err); + logger.debug("server: failed response -- ", err); + } + }); + }); + + return server; +} + +async function handleRequest({ socket, jupyterRun, path, cells }) { + const runner = await jupyterRun({ path, cells }); + for await (const result of runner) { + socket.write(result); + } + socket.write(null); +} + +class JupyterClient { + private iter?: EventIterator; + private socket; + constructor( + private client: ConatClient, + private subject: string, + private path: string, + ) { + this.socket = this.client.socket.connect(this.subject); + this.socket.once("close", () => this.iter?.end()); + } + + close = () => { + this.iter?.end(); + delete this.iter; + this.socket.close(); + }; + + run = async (cells: InputCell[]) => { + if (this.iter) { + // one evaluation at a time. + this.iter.end(); + delete this.iter; + } + this.iter = new EventIterator(this.socket, "data", { + map: (args) => { + if (args[0] == null) { + this.iter?.end(); + return null; + } else { + return args[0]; + } + }, + }); + await this.socket.request({ path: this.path, cells }); + return this.iter; + }; +} + +export function jupyterClient(opts: { + path: string; + project_id: string; + compute_server_id?: number; + client: ConatClient; +}) { + const subject = getSubject(opts); + return new JupyterClient(opts.client, subject, opts.path); +} From 4f6bb4140d346a881087e90406e6db31dbec5b0b Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 27 Jul 2025 16:22:25 +0000 Subject: [PATCH 099/270] conat jupyter code runner: streaming output now working for one cell --- .../test/project/jupyter/run-code.test.ts | 39 +++++++++++--- src/packages/conat/project/api/editor.ts | 5 -- .../conat/project/jupyter/run-code.ts | 13 +++-- .../frontend/jupyter/browser-actions.ts | 1 + src/packages/frontend/jupyter/run-cell.ts | 35 +++++++++---- src/packages/jupyter/control.ts | 52 +++++++++++-------- src/packages/project/conat/api/editor.ts | 15 +++--- src/packages/project/conat/index.ts | 2 + src/packages/project/conat/jupyter.ts | 20 +++++++ 9 files changed, 131 insertions(+), 51 deletions(-) create mode 100644 src/packages/project/conat/jupyter.ts diff --git a/src/packages/backend/conat/test/project/jupyter/run-code.test.ts b/src/packages/backend/conat/test/project/jupyter/run-code.test.ts index 6dbb84f2e7..f13e0249cb 100644 --- a/src/packages/backend/conat/test/project/jupyter/run-code.test.ts +++ b/src/packages/backend/conat/test/project/jupyter/run-code.test.ts @@ -13,6 +13,9 @@ import { } from "@cocalc/conat/project/jupyter/run-code"; import { uuid } from "@cocalc/util/misc"; +// it's really 100+, but tests fails if less than this. +const MIN_EVALS_PER_SECOND = 10; + beforeAll(before); describe("create very simple mocked jupyter runner and test evaluating code", () => { @@ -28,8 +31,8 @@ describe("create very simple mocked jupyter runner and test evaluating code", () // running code with this just results in two responses: the path and the cells async function jupyterRun({ path, cells }) { async function* runner() { - yield [{ path }]; - yield [{ cells }]; + yield { path }; + yield { cells }; } return runner(); } @@ -65,7 +68,7 @@ describe("create very simple mocked jupyter runner and test evaluating code", () if (process.env.BENCH) { console.log({ evalsPerSecond }); } - expect(evalsPerSecond).toBeGreaterThan(25); + expect(evalsPerSecond).toBeGreaterThan(MIN_EVALS_PER_SECOND); }); it("cleans up", () => { @@ -83,18 +86,24 @@ describe("create simple mocked jupyter runner that does actually eval an expres let server; const project_id = uuid(); + const compute_server_id = 3; it("create jupyter code run server", () => { // running code with this just results in two responses: the path and the cells async function jupyterRun({ cells }) { async function* runner() { for (const { id, input } of cells) { - yield [{ id, output: eval(input) }]; + yield { id, output: eval(input) }; } } return runner(); } - server = jupyterServer({ client: client1, project_id, jupyterRun }); + server = jupyterServer({ + client: client1, + project_id, + jupyterRun, + compute_server_id, + }); }); let client; @@ -104,7 +113,12 @@ describe("create simple mocked jupyter runner that does actually eval an expres { id: "b", input: "3**5" }, ]; it("create a jupyter client, then run some code", async () => { - client = jupyterClient({ path, project_id, client: client2 }); + client = jupyterClient({ + path, + project_id, + client: client2, + compute_server_id, + }); const iter = await client.run(cells); const v: any[] = []; for await (const output of iter) { @@ -113,6 +127,19 @@ describe("create simple mocked jupyter runner that does actually eval an expres expect(v).toEqual([[{ id: "a", output: 5 }], [{ id: "b", output: 243 }]]); }); + it("run code that FAILS and see error is visible to client properly", async () => { + const iter = await client.run([ + { id: "a", input: "2+3" }, + { id: "b", input: "2+invalid" }, + ]); + try { + for await (const _ of iter) { + } + } catch (err) { + expect(`${err}`).toContain("ReferenceError: invalid is not defined"); + } + }); + it("cleans up", () => { server.close(); client.close(); diff --git a/src/packages/conat/project/api/editor.ts b/src/packages/conat/project/api/editor.ts index c18d7d00f8..e17d597e8a 100644 --- a/src/packages/conat/project/api/editor.ts +++ b/src/packages/conat/project/api/editor.ts @@ -13,7 +13,6 @@ export const editor = { jupyterRunNotebook: true, jupyterKernelLogo: true, jupyterKernels: true, - jupyterRun: true, formatString: true, @@ -47,10 +46,6 @@ export interface Editor { // path = the syncdb path (not *.ipynb) jupyterStart: (path: string) => Promise; jupyterStop: (path: string) => Promise; - jupyterRun: ( - path: string, - cells: { id: string; input: string }[], - ) => Promise; jupyterNbconvert: (opts: NbconvertParams) => Promise; diff --git a/src/packages/conat/project/jupyter/run-code.ts b/src/packages/conat/project/jupyter/run-code.ts index 1cc87d3873..98e1844282 100644 --- a/src/packages/conat/project/jupyter/run-code.ts +++ b/src/packages/conat/project/jupyter/run-code.ts @@ -39,7 +39,7 @@ type JupyterCodeRunner = (opts: { path: string; // array of input cells to run cells: InputCell[]; -}) => Promise>; +}) => Promise>; export function jupyterServer({ client, @@ -68,8 +68,9 @@ export function jupyterServer({ mesg.respondSync(null); await handleRequest({ socket, jupyterRun, path, cells }); } catch (err) { - console.log(err); + //console.log(err); logger.debug("server: failed response -- ", err); + socket.write(null, { headers: { error: `${err}` } }); } }); }); @@ -79,8 +80,8 @@ export function jupyterServer({ async function handleRequest({ socket, jupyterRun, path, cells }) { const runner = await jupyterRun({ path, cells }); - for await (const result of runner) { - socket.write(result); + for await (const mesg of runner) { + socket.write([mesg]); } socket.write(null); } @@ -111,6 +112,10 @@ class JupyterClient { } this.iter = new EventIterator(this.socket, "data", { map: (args) => { + if (args[1]?.error) { + this.iter?.throw(Error(args[1].error)); + return; + } if (args[0] == null) { this.iter?.end(); return null; diff --git a/src/packages/frontend/jupyter/browser-actions.ts b/src/packages/frontend/jupyter/browser-actions.ts index 932bee24a3..02f806b221 100644 --- a/src/packages/frontend/jupyter/browser-actions.ts +++ b/src/packages/frontend/jupyter/browser-actions.ts @@ -214,6 +214,7 @@ export class JupyterActions extends JupyterActions0 { }; // temporary proof of concept + public jupyterClient?; runCell = async (id: string) => { await runCell({ actions: this, id }); }; diff --git a/src/packages/frontend/jupyter/run-cell.ts b/src/packages/frontend/jupyter/run-cell.ts index 6c818e5582..977533e5f1 100644 --- a/src/packages/frontend/jupyter/run-cell.ts +++ b/src/packages/frontend/jupyter/run-cell.ts @@ -1,5 +1,7 @@ import { OutputHandler } from "@cocalc/jupyter/execute/output-handler"; import { type JupyterActions } from "./browser-actions"; +import { jupyterClient } from "@cocalc/conat/project/jupyter/run-code"; +import { webapp_client } from "@cocalc/frontend/webapp-client"; export async function runCell({ actions, @@ -13,18 +15,33 @@ export async function runCell({ // nothing to do return; } + + if (actions.jupyterClient == null) { + // [ ] **TODO: Must invalidate this when compute server changes!!!!!** + // and + const compute_server_id = await actions.getComputeServerId(); + actions.jupyterClient = jupyterClient({ + path: actions.syncdbPath, + client: webapp_client.conat_client.conat(), + project_id: actions.project_id, + compute_server_id, + }); + } + const client = actions.jupyterClient; + if (client == null) { + throw Error("bug"); + } + cell.output = null; actions._set(cell); const handler = new OutputHandler({ cell }); - const api = await actions.conatApi(); - const mesgs = await api.editor.jupyterRun(actions.syncdbPath, [ - { id: cell.id, input: cell.input }, - ]); - console.log(mesgs); - for (const mesg of mesgs) { - handler.process(mesg); + const runner = await client.run([cell]); + for await (const mesgs of runner) { + for (const mesg of mesgs) { + handler.process(mesg); + actions._set(cell, false); + } } cell.state = "done"; - console.log(cell); - actions._set(cell); + actions._set(cell, true); } diff --git a/src/packages/jupyter/control.ts b/src/packages/jupyter/control.ts index 499e830945..27726da352 100644 --- a/src/packages/jupyter/control.ts +++ b/src/packages/jupyter/control.ts @@ -47,7 +47,21 @@ export function jupyterStart({ sessions[path] = { syncdb, actions, store }; } -// run the cells with given id... +export function jupyterStop({ path }: { path: string }) { + const session = sessions[path]; + if (session == null) { + logger.debug("jupyterStop: ", path, " - not running"); + } else { + const { syncdb } = session; + logger.debug("jupyterStop: ", path, " - stopping it"); + syncdb.close(); + delete sessions[path]; + const path_ipynb = original_path(path); + removeJupyterRedux(path_ipynb, project_id); + } +} + +// Returns async iterator over outputs export async function jupyterRun({ path, cells, @@ -56,6 +70,7 @@ export async function jupyterRun({ cells: { id: string; input: string }[]; }) { logger.debug("jupyterRun", { path, cells }); + const session = sessions[path]; if (session == null) { throw Error(`${path} not running`); @@ -70,25 +85,20 @@ export async function jupyterRun({ await once(syncdb, "ready"); } logger.debug("jupyterRun: running"); - let v = []; - for (const cell of cells) { - v = v.concat( - await actions.jupyter_kernel.execute_code_now({ code: cell.input }), - ); - } - return v; -} - -export function jupyterStop({ path }: { path: string }) { - const session = sessions[path]; - if (session == null) { - logger.debug("jupyterStop: ", path, " - not running"); - } else { - const { syncdb } = session; - logger.debug("jupyterStop: ", path, " - stopping it"); - syncdb.close(); - delete sessions[path]; - const path_ipynb = original_path(path); - removeJupyterRedux(path_ipynb, project_id); + async function* run() { + for (const cell of cells) { + const output = actions.jupyter_kernel.execute_code({ + halt_on_error: true, + code: cell.input, + }); + for await (const mesg of output.iter()) { + yield mesg; + } + if (actions.jupyter_kernel.failedError) { + // kernel failed during call + throw Error(actions.jupyter_kernel.failedError); + } + } } + return await run(); } diff --git a/src/packages/project/conat/api/editor.ts b/src/packages/project/conat/api/editor.ts index 5783ddae67..c858c7d124 100644 --- a/src/packages/project/conat/api/editor.ts +++ b/src/packages/project/conat/api/editor.ts @@ -47,12 +47,15 @@ export async function jupyterStart(path: string) { await control.jupyterStart({ project_id, path, client: getClient(), fs }); } -export async function jupyterRun( - path: string, - cells: { id: string; input: string }[], -) { - await jupyterStart(path); - return await control.jupyterRun({ path, cells }); +// IMPORTANT: jupyterRun is NOT used directly by the API, but instead by packages/project/conat/jupyter.ts +// It is convenient to have it here so it can call jupyterStart above, etc. The reason is because +// this returns an async iterator managed using a dedicated socket, and the api is request/response.. +export async function jupyterRun(opts: { + path: string; + cells: { id: string; input: string }[]; +}) { + await jupyterStart(opts.path); + return await control.jupyterRun(opts); } export async function jupyterStop(path: string) { diff --git a/src/packages/project/conat/index.ts b/src/packages/project/conat/index.ts index 056c9c8c7b..a93de46d88 100644 --- a/src/packages/project/conat/index.ts +++ b/src/packages/project/conat/index.ts @@ -17,12 +17,14 @@ import { init as initRead } from "./files/read"; import { init as initWrite } from "./files/write"; import { init as initProjectStatus } from "@cocalc/project/project-status/server"; import { init as initUsageInfo } from "@cocalc/project/usage-info"; +import { init as initJupyter } from "./jupyter"; const logger = getLogger("project:conat:index"); export default async function init() { logger.debug("starting Conat project services"); await initAPI(); + await initJupyter(); await initOpenFiles(); initWebsocketApi(); await initListings(); diff --git a/src/packages/project/conat/jupyter.ts b/src/packages/project/conat/jupyter.ts new file mode 100644 index 0000000000..f64859571b --- /dev/null +++ b/src/packages/project/conat/jupyter.ts @@ -0,0 +1,20 @@ +import { jupyterRun } from "@cocalc/project/conat/api/editor"; +import { jupyterServer } from "@cocalc/conat/project/jupyter/run-code"; +import { connectToConat } from "@cocalc/project/conat/connection"; +import { compute_server_id, project_id } from "@cocalc/project/data"; +import { getLogger } from "@cocalc/project/logger"; + +const logger = getLogger("project:conat:jupyter"); + +let server: any = null; +export function init() { + logger.debug("initializing jupyter run server"); + const client = connectToConat(); + server = jupyterServer({ client, project_id, compute_server_id, jupyterRun }); +} + +export function close() { + logger.debug("closing jupyter run server"); + server?.close(); + server = null; +} From 16b799406b767d7d0ee82d2552780630f7b71031 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 27 Jul 2025 16:27:21 +0000 Subject: [PATCH 100/270] depcheck fix --- src/packages/jupyter/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/packages/jupyter/package.json b/src/packages/jupyter/package.json index 7c4bd52173..24c0d73b8f 100644 --- a/src/packages/jupyter/package.json +++ b/src/packages/jupyter/package.json @@ -24,7 +24,7 @@ "build": "../node_modules/.bin/tsc --build", "clean": "rm -rf node_modules dist", "test": "pnpm exec jest --forceExit --maxWorkers=1", - "depcheck": "pnpx depcheck", + "depcheck": "pnpx depcheck --ignores events", "tsc": "../node_modules/.bin/tsc --watch --pretty --preserveWatchOutput" }, "files": [ From e02db38fbb1388ba3965221ad16fb4e6c385372d Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 27 Jul 2025 17:13:00 +0000 Subject: [PATCH 101/270] ts --- src/packages/frontend/jupyter/browser-actions.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/packages/frontend/jupyter/browser-actions.ts b/src/packages/frontend/jupyter/browser-actions.ts index 02f806b221..2e34745151 100644 --- a/src/packages/frontend/jupyter/browser-actions.ts +++ b/src/packages/frontend/jupyter/browser-actions.ts @@ -215,7 +215,7 @@ export class JupyterActions extends JupyterActions0 { // temporary proof of concept public jupyterClient?; - runCell = async (id: string) => { + runCell = async (id: string, _noHalt) => { await runCell({ actions: this, id }); }; @@ -274,7 +274,7 @@ export class JupyterActions extends JupyterActions0 { this.clear_cell(id, save); return; } - this.runCell(id); + this.runCell(id, no_halt); //this.run_code_cell(id, save, no_halt); if (save) { this.save_asap(); From a43db52701c7729b8e26349a78b97def2c77918e Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 27 Jul 2025 18:21:55 +0000 Subject: [PATCH 102/270] jupyter eval: organizing code; removing flicker --- .../conat/project/jupyter/run-code.ts | 15 +- src/packages/conat/socket/server-socket.ts | 10 +- .../frontend/jupyter/browser-actions.ts | 135 ++++++++++++------ src/packages/frontend/jupyter/cell-list.tsx | 2 +- src/packages/frontend/jupyter/run-cell.ts | 47 ------ src/packages/jupyter/control.ts | 2 +- 6 files changed, 113 insertions(+), 98 deletions(-) delete mode 100644 src/packages/frontend/jupyter/run-cell.ts diff --git a/src/packages/conat/project/jupyter/run-code.ts b/src/packages/conat/project/jupyter/run-code.ts index 98e1844282..4df926c29d 100644 --- a/src/packages/conat/project/jupyter/run-code.ts +++ b/src/packages/conat/project/jupyter/run-code.ts @@ -53,7 +53,10 @@ export function jupyterServer({ jupyterRun: JupyterCodeRunner; }) { const subject = getSubject({ project_id, compute_server_id }); - const server: ConatSocketServer = client.socket.listen(subject); + const server: ConatSocketServer = client.socket.listen(subject, { + keepAlive: 5000, + keepAliveTimeout: 5000, + }); logger.debug("server: listening on ", { subject }); server.on("connection", (socket: ServerSocket) => { @@ -73,6 +76,10 @@ export function jupyterServer({ socket.write(null, { headers: { error: `${err}` } }); } }); + + socket.on("closed", () => { + logger.debug("socket closed", { id: socket.id }); + }); }); return server; @@ -81,7 +88,11 @@ export function jupyterServer({ async function handleRequest({ socket, jupyterRun, path, cells }) { const runner = await jupyterRun({ path, cells }); for await (const mesg of runner) { - socket.write([mesg]); + if (socket.state == "closed") { + logger.debug("socket closed -- server is now handling output!", mesg); + } else { + socket.write([mesg]); + } } socket.write(null); } diff --git a/src/packages/conat/socket/server-socket.ts b/src/packages/conat/socket/server-socket.ts index edfdc225a1..531e151949 100644 --- a/src/packages/conat/socket/server-socket.ts +++ b/src/packages/conat/socket/server-socket.ts @@ -51,6 +51,7 @@ export class ServerSocket extends EventEmitter { this.initKeepAlive(); } + private firstPing = true; private initKeepAlive = () => { this.alive?.close(); this.alive = keepAlive({ @@ -59,10 +60,15 @@ export class ServerSocket extends EventEmitter { await this.request(null, { headers: { [SOCKET_HEADER_CMD]: "ping" }, timeout: this.conatSocket.keepAliveTimeout, - // waitForInterest is very important in a cluster -- also, obviously + // waitForInterest for the *first ping* is very important + // in a cluster -- also, obviously // if somebody just opened a socket, they probably exist. - waitForInterest: true, + // However, after the first ping, we want to fail + // very quickly if the client disappears (and hence no + // more interest). + waitForInterest: this.firstPing, }); + this.firstPing = false; }, disconnect: this.close, keepAlive: this.conatSocket.keepAlive, diff --git a/src/packages/frontend/jupyter/browser-actions.ts b/src/packages/frontend/jupyter/browser-actions.ts index 2e34745151..0cc92b136e 100644 --- a/src/packages/frontend/jupyter/browser-actions.ts +++ b/src/packages/frontend/jupyter/browser-actions.ts @@ -57,7 +57,9 @@ import getKernelSpec from "@cocalc/frontend/jupyter/kernelspecs"; import { get as getUsageInfo } from "@cocalc/conat/project/usage-info"; import { delay } from "awaiting"; import { until } from "@cocalc/util/async-utils"; -import { runCell } from "./run-cell"; +import { jupyterClient } from "@cocalc/conat/project/jupyter/run-code"; +import { OutputHandler } from "@cocalc/jupyter/execute/output-handler"; +import { throttle } from "lodash"; // local cache: map project_id (string) -> kernels (immutable) let jupyter_kernels = Map(); @@ -175,50 +177,6 @@ export class JupyterActions extends JupyterActions0 { } } - // if the project or compute server is running and listening, this call - // tells them to open this jupyter notebook, so it can provide the compute - // functionality. - - conatApi = async () => { - const compute_server_id = await this.getComputeServerId(); - const api = webapp_client.project_client.conatApi( - this.project_id, - compute_server_id, - ); - return api; - }; - - initBackend = async () => { - await until( - async () => { - if (this.is_closed()) { - return true; - } - try { - const api = await this.conatApi(); - await api.editor.jupyterStart(this.syncdbPath); - console.log("initialized ", this.path); - return true; - } catch (err) { - console.log("failed to initialize ", this.path, err); - return false; - } - }, - { min: 3000 }, - ); - }; - - stopBackend = async () => { - const api = await this.conatApi(); - await api.editor.jupyterStop(this.syncdbPath); - }; - - // temporary proof of concept - public jupyterClient?; - runCell = async (id: string, _noHalt) => { - await runCell({ actions: this, id }); - }; - initOpenLog = () => { // Put an entry in the project log once the jupyter notebook gets opened and // shows cells. @@ -377,6 +335,7 @@ export class JupyterActions extends JupyterActions0 { public async close(): Promise { if (this.is_closed()) return; + this.jupyterClient?.close(); await super.close(); } @@ -1494,4 +1453,90 @@ export class JupyterActions extends JupyterActions0 { } return; }; + + // if the project or compute server is running and listening, this call + // tells them to open this jupyter notebook, so it can provide the compute + // functionality. + + conatApi = async () => { + const compute_server_id = await this.getComputeServerId(); + const api = webapp_client.project_client.conatApi( + this.project_id, + compute_server_id, + ); + return api; + }; + + initBackend = async () => { + await until( + async () => { + if (this.is_closed()) { + return true; + } + try { + const api = await this.conatApi(); + await api.editor.jupyterStart(this.syncdbPath); + return true; + } catch (err) { + console.log("failed to initialize ", this.path, err); + return false; + } + }, + { min: 3000 }, + ); + }; + + stopBackend = async () => { + const api = await this.conatApi(); + await api.editor.jupyterStop(this.syncdbPath); + }; + + // temporary proof of concept + private jupyterClient?; + runCell = async (id: string, _noHalt) => { + const cell = this.store.getIn(["cells", id])?.toJS(); + if (cell == null) { + // nothing to do + return; + } + + if (this.jupyterClient == null) { + // [ ] **TODO: Must invalidate this when compute server changes!!!!!** + // and + const compute_server_id = await this.getComputeServerId(); + this.jupyterClient = jupyterClient({ + path: this.syncdbPath, + client: webapp_client.conat_client.conat(), + project_id: this.project_id, + compute_server_id, + }); + } + const client = this.jupyterClient; + if (client == null) { + throw Error("bug"); + } + + if (cell.output) { + // trick to avoid flicker + for (const n in cell.output) { + if (n == "0") continue; + cell.output[n] = null; + } + this._set(cell, false); + } + const handler = new OutputHandler({ cell }); + const f = throttle(() => this._set(cell, false), 1000 / 24, { + leading: false, + trailing: true, + }); + handler.on("change", f); + const runner = await client.run([cell]); + for await (const mesgs of runner) { + for (const mesg of mesgs) { + handler.process(mesg); + } + } + handler.done(); + this._set(cell, true); + }; } diff --git a/src/packages/frontend/jupyter/cell-list.tsx b/src/packages/frontend/jupyter/cell-list.tsx index d8a717943e..3812b636b0 100644 --- a/src/packages/frontend/jupyter/cell-list.tsx +++ b/src/packages/frontend/jupyter/cell-list.tsx @@ -434,7 +434,7 @@ export const CellList: React.FC = (props: CellListProps) => { if (index == null) { index = cell_list.indexOf(id) ?? 0; } - const dragHandle = actions?.store.is_cell_editable(id) ? ( + const dragHandle = actions?.store?.is_cell_editable(id) ? ( Date: Sun, 27 Jul 2025 19:38:35 +0000 Subject: [PATCH 103/270] jupyter: evaluation when client vanishes (with one cell) is working --- .../conat/project/jupyter/run-code.ts | 59 +++++++++++++++++-- src/packages/jupyter/control.ts | 37 +++++++++--- .../jupyter/execute/output-handler.ts | 2 + src/packages/jupyter/ipynb/export-to-ipynb.ts | 2 + src/packages/project/conat/jupyter.ts | 9 ++- 5 files changed, 94 insertions(+), 15 deletions(-) diff --git a/src/packages/conat/project/jupyter/run-code.ts b/src/packages/conat/project/jupyter/run-code.ts index 4df926c29d..f9b29fa207 100644 --- a/src/packages/conat/project/jupyter/run-code.ts +++ b/src/packages/conat/project/jupyter/run-code.ts @@ -34,23 +34,44 @@ interface InputCell { type OutputMessage = any; -type JupyterCodeRunner = (opts: { +export interface RunOptions { // syncdb path path: string; // array of input cells to run cells: InputCell[]; -}) => Promise>; +} + +type JupyterCodeRunner = ( + opts: RunOptions, +) => Promise>; + +interface OutputHandler { + process: (mesg: OutputMessage) => void; + done: () => void; +} +type CreateOutputHandler = (opts: { + path: string; + cells: InputCell[]; +}) => OutputHandler; export function jupyterServer({ client, project_id, compute_server_id = 0, + // jupyterRun takes a path and cells to run and returns an async iterator + // over the outputs. jupyterRun, + // outputHandler takes a path and returns an OutputHandler, which can be + // used to process the output and include it in the notebook. It is used + // as a fallback in case the client that initiated running cells is + // disconnected, so output won't be lost. + outputHandler, }: { client: ConatClient; project_id: string; compute_server_id?: number; jupyterRun: JupyterCodeRunner; + outputHandler?: CreateOutputHandler; }) { const subject = getSubject({ project_id, compute_server_id }); const server: ConatSocketServer = client.socket.listen(subject, { @@ -69,11 +90,13 @@ export function jupyterServer({ const { path, cells } = mesg.data; try { mesg.respondSync(null); - await handleRequest({ socket, jupyterRun, path, cells }); + await handleRequest({ socket, jupyterRun, outputHandler, path, cells }); } catch (err) { //console.log(err); logger.debug("server: failed response -- ", err); - socket.write(null, { headers: { error: `${err}` } }); + if (socket.state != "closed") { + socket.write(null, { headers: { error: `${err}` } }); + } } }); @@ -85,15 +108,39 @@ export function jupyterServer({ return server; } -async function handleRequest({ socket, jupyterRun, path, cells }) { +async function handleRequest({ + socket, + jupyterRun, + outputHandler, + path, + cells, +}) { const runner = await jupyterRun({ path, cells }); + const output: OutputMessage[] = []; + let handler: OutputHandler | null = null; for await (const mesg of runner) { if (socket.state == "closed") { - logger.debug("socket closed -- server is now handling output!", mesg); + if (handler == null) { + logger.debug("socket closed -- server must handle output"); + if (outputHandler == null) { + throw Error("no output handler available"); + } + handler = outputHandler({ path, cells }); + if (handler == null) { + throw Error("bug -- outputHandler must return a handler"); + } + for (const prev of output) { + handler.process(prev); + } + output.length = 0; + } + handler.process(mesg); } else { + output.push(mesg); socket.write([mesg]); } } + handler?.done(); socket.write(null); } diff --git a/src/packages/jupyter/control.ts b/src/packages/jupyter/control.ts index 2641a5fd24..9b4ac09fbe 100644 --- a/src/packages/jupyter/control.ts +++ b/src/packages/jupyter/control.ts @@ -5,6 +5,9 @@ import { getLogger } from "@cocalc/backend/logger"; import { initJupyterRedux, removeJupyterRedux } from "@cocalc/jupyter/kernel"; import { original_path } from "@cocalc/util/misc"; import { once } from "@cocalc/util/async-utils"; +import { OutputHandler } from "@cocalc/jupyter/execute/output-handler"; +import { throttle } from "lodash"; +import { type RunOptions } from "@cocalc/conat/project/jupyter/run-code"; const logger = getLogger("jupyter:control"); @@ -62,14 +65,8 @@ export function jupyterStop({ path }: { path: string }) { } // Returns async iterator over outputs -export async function jupyterRun({ - path, - cells, -}: { - path: string; - cells: { id: string; input: string }[]; -}) { - logger.debug("jupyterRun", { path }) // , cells }); +export async function jupyterRun({ path, cells }: RunOptions) { + logger.debug("jupyterRun", { path }); // , cells }); const session = sessions[path]; if (session == null) { @@ -102,3 +99,27 @@ export async function jupyterRun({ } return await run(); } + +const BACKEND_OUTPUT_FPS = 8; +export function outputHandler({ path, cells }: RunOptions) { + if (sessions[path] == null) { + throw Error(`session '${path}' not available`); + } + const { actions } = sessions[path]; + // todo: need to handle multiple cells + const cell = { type: "cell" as "cell", ...cells[0] }; + const handler = new OutputHandler({ cell }); + const f = throttle( + () => { + logger.debug("outputHandler", path, cell); + actions._set(cell, true); + }, + 1000 / BACKEND_OUTPUT_FPS, + { + leading: false, + trailing: true, + }, + ); + handler.on("change", f); + return handler; +} diff --git a/src/packages/jupyter/execute/output-handler.ts b/src/packages/jupyter/execute/output-handler.ts index 0d6bec0dea..f1ef5a8eff 100644 --- a/src/packages/jupyter/execute/output-handler.ts +++ b/src/packages/jupyter/execute/output-handler.ts @@ -38,6 +38,8 @@ const MAX_SAVE_INTERVAL_MS = 45000; import { type Cell } from "@cocalc/jupyter/ipynb/export-to-ipynb"; +export { type Cell }; + interface Message { execution_state?; execution_count?: number; diff --git a/src/packages/jupyter/ipynb/export-to-ipynb.ts b/src/packages/jupyter/ipynb/export-to-ipynb.ts index ebd57e53d1..f3dd354351 100644 --- a/src/packages/jupyter/ipynb/export-to-ipynb.ts +++ b/src/packages/jupyter/ipynb/export-to-ipynb.ts @@ -14,6 +14,8 @@ type CellType = "code" | "markdown" | "raw"; type Tags = { [key: string]: boolean }; export interface Cell { + type?: "cell"; + id?: string; cell_type?: CellType; input?: string; collapsed?: boolean; diff --git a/src/packages/project/conat/jupyter.ts b/src/packages/project/conat/jupyter.ts index f64859571b..1405684198 100644 --- a/src/packages/project/conat/jupyter.ts +++ b/src/packages/project/conat/jupyter.ts @@ -1,4 +1,5 @@ import { jupyterRun } from "@cocalc/project/conat/api/editor"; +import { outputHandler } from "@cocalc/jupyter/control"; import { jupyterServer } from "@cocalc/conat/project/jupyter/run-code"; import { connectToConat } from "@cocalc/project/conat/connection"; import { compute_server_id, project_id } from "@cocalc/project/data"; @@ -10,7 +11,13 @@ let server: any = null; export function init() { logger.debug("initializing jupyter run server"); const client = connectToConat(); - server = jupyterServer({ client, project_id, compute_server_id, jupyterRun }); + server = jupyterServer({ + client, + project_id, + compute_server_id, + jupyterRun, + outputHandler, + }); } export function close() { From 61a82f1db261273f58c822a3583fa6a82e73d84a Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 27 Jul 2025 20:42:13 +0000 Subject: [PATCH 104/270] jupyter run: add unit test for "client closes connection" --- .../test/project/jupyter/run-code.test.ts | 94 ++++++++++++++++++- .../conat/project/jupyter/run-code.ts | 1 + 2 files changed, 93 insertions(+), 2 deletions(-) diff --git a/src/packages/backend/conat/test/project/jupyter/run-code.test.ts b/src/packages/backend/conat/test/project/jupyter/run-code.test.ts index f13e0249cb..39c3fb0035 100644 --- a/src/packages/backend/conat/test/project/jupyter/run-code.test.ts +++ b/src/packages/backend/conat/test/project/jupyter/run-code.test.ts @@ -6,12 +6,13 @@ pnpm test `pwd`/run-code.test.ts */ -import { before, after, connect } from "@cocalc/backend/conat/test/setup"; +import { before, after, connect, wait } from "@cocalc/backend/conat/test/setup"; import { jupyterClient, jupyterServer, } from "@cocalc/conat/project/jupyter/run-code"; import { uuid } from "@cocalc/util/misc"; +import { delay } from "awaiting"; // it's really 100+, but tests fails if less than this. const MIN_EVALS_PER_SECOND = 10; @@ -77,7 +78,7 @@ describe("create very simple mocked jupyter runner and test evaluating code", () }); }); -describe("create simple mocked jupyter runner that does actually eval an expression", () => { +describe("create simple mocked jupyter runner that does actually eval an expression", () => { let client1, client2; it("create two clients", async () => { client1 = connect(); @@ -146,4 +147,93 @@ describe("create simple mocked jupyter runner that does actually eval an expres }); }); +describe("create mocked jupyter runner that does failover to backend output management when client disconnects", () => { + let client1, client2; + it("create two clients", async () => { + client1 = connect(); + client2 = connect(); + }); + + const path = "b.ipynb"; + const cells = [ + { id: "a", input: "10*(2+3)" }, + { id: "b", input: "100" }, + ]; + let server; + const project_id = uuid(); + let handler: any = null; + + it("create jupyter code run server that also takes as long as the output to run", () => { + async function jupyterRun({ cells }) { + async function* runner() { + for (const { id, input } of cells) { + const output = eval(input); + await delay(output); + yield { id, output }; + } + } + return runner(); + } + + class OutputHandler { + messages: any[] = []; + + constructor(public cells) {} + + process = (mesg: any) => { + this.messages.push(mesg); + }; + done = () => { + this.messages.push({ done: true }); + }; + } + + function outputHandler({ path: path0, cells }) { + if (path0 != path) { + throw Error(`path must be ${path}`); + } + handler = new OutputHandler(cells); + return handler; + } + + server = jupyterServer({ + client: client1, + project_id, + jupyterRun, + outputHandler, + }); + }); + + let client; + it("create a jupyter client, then run some code (doesn't use output handler)", async () => { + client = jupyterClient({ + path, + project_id, + client: client2, + }); + const iter = await client.run(cells); + const v: any[] = []; + for await (const output of iter) { + v.push(output); + } + expect(v).toEqual([[{ id: "a", output: 50 }], [{ id: "b", output: 100 }]]); + }); + + it("starts code running then closes the client, which causes output to have to be placed in the handler instead.", async () => { + const iter = await client.run(cells); + client.close(); + await wait({ until: () => handler.messages.length >= 3 }); + expect(handler.messages).toEqual([ + { id: "a", output: 50 }, + { id: "b", output: 100 }, + { done: true }, + ]); + }); + + it("cleans up", () => { + server.close(); + client.close(); + }); +}); + afterAll(after); diff --git a/src/packages/conat/project/jupyter/run-code.ts b/src/packages/conat/project/jupyter/run-code.ts index f9b29fa207..f7fdbf8076 100644 --- a/src/packages/conat/project/jupyter/run-code.ts +++ b/src/packages/conat/project/jupyter/run-code.ts @@ -49,6 +49,7 @@ interface OutputHandler { process: (mesg: OutputMessage) => void; done: () => void; } + type CreateOutputHandler = (opts: { path: string; cells: InputCell[]; From 235d983270394667afc0840e44314a738bcbbb4f Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 27 Jul 2025 21:53:16 +0000 Subject: [PATCH 105/270] jupyter cell runner: handle multiple input cells --- .../test/project/jupyter/run-code.test.ts | 10 +-- .../conat/project/jupyter/run-code.ts | 18 ++++- .../frontend/jupyter/browser-actions.ts | 70 ++++++++++++------- src/packages/jupyter/control.ts | 52 +++++++++----- .../jupyter/execute/output-handler.ts | 10 ++- 5 files changed, 111 insertions(+), 49 deletions(-) diff --git a/src/packages/backend/conat/test/project/jupyter/run-code.test.ts b/src/packages/backend/conat/test/project/jupyter/run-code.test.ts index 39c3fb0035..fdb960d326 100644 --- a/src/packages/backend/conat/test/project/jupyter/run-code.test.ts +++ b/src/packages/backend/conat/test/project/jupyter/run-code.test.ts @@ -32,8 +32,8 @@ describe("create very simple mocked jupyter runner and test evaluating code", () // running code with this just results in two responses: the path and the cells async function jupyterRun({ path, cells }) { async function* runner() { - yield { path }; - yield { cells }; + yield { path, id: "0" }; + yield { cells, id: "0" }; } return runner(); } @@ -51,7 +51,7 @@ describe("create very simple mocked jupyter runner and test evaluating code", () for await (const output of iter) { v.push(output); } - expect(v).toEqual([[{ path }], [{ cells }]]); + expect(v).toEqual([[{ path, id: "0" }], [{ cells, id: "0" }]]); }); const count = 100; @@ -63,7 +63,7 @@ describe("create very simple mocked jupyter runner and test evaluating code", () for await (const output of await client.run(cells)) { v.push(output); } - expect(v).toEqual([[{ path }], [{ cells }]]); + expect(v).toEqual([[{ path, id: "0" }], [{ cells, id: "0" }]]); } const evalsPerSecond = Math.floor((1000 * count) / (Date.now() - start)); if (process.env.BENCH) { @@ -220,7 +220,7 @@ describe("create mocked jupyter runner that does failover to backend output mana }); it("starts code running then closes the client, which causes output to have to be placed in the handler instead.", async () => { - const iter = await client.run(cells); + await client.run(cells); client.close(); await wait({ until: () => handler.messages.length >= 3 }); expect(handler.messages).toEqual([ diff --git a/src/packages/conat/project/jupyter/run-code.ts b/src/packages/conat/project/jupyter/run-code.ts index f7fdbf8076..ed9f0cbe6b 100644 --- a/src/packages/conat/project/jupyter/run-code.ts +++ b/src/packages/conat/project/jupyter/run-code.ts @@ -32,7 +32,16 @@ interface InputCell { input: string; } -type OutputMessage = any; +export interface OutputMessage { + // id = id of the cell + id: string; + // everything below is exactly from Jupyter + metadata?; + content?; + buffers?; + msg_type?: string; + done?: boolean; +} export interface RunOptions { // syncdb path @@ -183,7 +192,12 @@ class JupyterClient { } }, }); - await this.socket.request({ path: this.path, cells }); + // get rid of any fields except id and input from the cells, since, e.g., + // if there is a lot of output in a cell, there is no need to send that to the backend. + const cells1 = cells.map(({ id, input }) => { + return { id, input }; + }); + await this.socket.request({ path: this.path, cells: cells1 }); return this.iter; }; } diff --git a/src/packages/frontend/jupyter/browser-actions.ts b/src/packages/frontend/jupyter/browser-actions.ts index 0cc92b136e..6ddf7e0d9b 100644 --- a/src/packages/frontend/jupyter/browser-actions.ts +++ b/src/packages/frontend/jupyter/browser-actions.ts @@ -61,6 +61,8 @@ import { jupyterClient } from "@cocalc/conat/project/jupyter/run-code"; import { OutputHandler } from "@cocalc/jupyter/execute/output-handler"; import { throttle } from "lodash"; +const OUTPUT_FPS = 29; + // local cache: map project_id (string) -> kernels (immutable) let jupyter_kernels = Map(); @@ -232,7 +234,7 @@ export class JupyterActions extends JupyterActions0 { this.clear_cell(id, save); return; } - this.runCell(id, no_halt); + this.runCells([id], no_halt); //this.run_code_cell(id, save, no_halt); if (save) { this.save_asap(); @@ -1491,15 +1493,18 @@ export class JupyterActions extends JupyterActions0 { await api.editor.jupyterStop(this.syncdbPath); }; - // temporary proof of concept - private jupyterClient?; - runCell = async (id: string, _noHalt) => { - const cell = this.store.getIn(["cells", id])?.toJS(); - if (cell == null) { - // nothing to do - return; - } + getOutputHandler = (cell) => { + const handler = new OutputHandler({ cell }); + const f = throttle(() => this._set(cell, false), 1000 / OUTPUT_FPS, { + leading: false, + trailing: true, + }); + handler.on("change", f); + return handler; + }; + private jupyterClient?; + runCells = async (ids: string[], _noHalt) => { if (this.jupyterClient == null) { // [ ] **TODO: Must invalidate this when compute server changes!!!!!** // and @@ -1515,28 +1520,43 @@ export class JupyterActions extends JupyterActions0 { if (client == null) { throw Error("bug"); } - - if (cell.output) { - // trick to avoid flicker - for (const n in cell.output) { - if (n == "0") continue; - cell.output[n] = null; + const cells: any[] = []; + for (const id of ids) { + const cell = this.store.getIn(["cells", id])?.toJS(); + if (!cell?.input?.trim()) { + // nothing to do + continue; + } + if (cell.output) { + // trick to avoid flicker + for (const n in cell.output) { + if (n == "0") continue; + cell.output[n] = null; + } + this._set(cell, false); } - this._set(cell, false); + cells.push(cell); } - const handler = new OutputHandler({ cell }); - const f = throttle(() => this._set(cell, false), 1000 / 24, { - leading: false, - trailing: true, - }); - handler.on("change", f); - const runner = await client.run([cell]); + + const runner = await client.run(cells); + let handler: null | OutputHandler = null; + let id: null | string = null; for await (const mesgs of runner) { for (const mesg of mesgs) { + if (mesg.id !== id || handler == null) { + id = mesg.id; + let cell = this.store.getIn(["cells", mesg.id])?.toJS(); + if (cell == null) { + // cell removed? + cell = { id }; + } + handler?.done(); + handler = this.getOutputHandler(cell); + } handler.process(mesg); } } - handler.done(); - this._set(cell, true); + handler?.done(); + this.save_asap(); }; } diff --git a/src/packages/jupyter/control.ts b/src/packages/jupyter/control.ts index 9b4ac09fbe..8b3dffba0c 100644 --- a/src/packages/jupyter/control.ts +++ b/src/packages/jupyter/control.ts @@ -89,6 +89,7 @@ export async function jupyterRun({ path, cells }: RunOptions) { code: cell.input, }); for await (const mesg of output.iter()) { + mesg.id = cell.id; yield mesg; } if (actions.jupyter_kernel.failedError) { @@ -100,26 +101,45 @@ export async function jupyterRun({ path, cells }: RunOptions) { return await run(); } +class MulticellOutputHandler { + private id: string | null = null; + private handler: OutputHandler | null = null; + + constructor( + private cells: RunOptions["cells"], + private actions, + ) {} + + process = (mesg) => { + if (mesg.id !== this.id || this.handler == null) { + this.id = mesg.id; + let cell = this.cells[mesg.id] ?? { id: mesg.id }; + this.handler?.done(); + this.handler = new OutputHandler({ cell }); + const f = throttle( + () => this.actions._set({ ...cell, type: "cell" }, true), + 1000 / BACKEND_OUTPUT_FPS, + { + leading: true, + trailing: true, + }, + ); + this.handler.on("change", f); + } + this.handler!.process(mesg); + }; + + done = () => { + this.handler?.done(); + this.handler = null; + }; +} + const BACKEND_OUTPUT_FPS = 8; export function outputHandler({ path, cells }: RunOptions) { if (sessions[path] == null) { throw Error(`session '${path}' not available`); } const { actions } = sessions[path]; - // todo: need to handle multiple cells - const cell = { type: "cell" as "cell", ...cells[0] }; - const handler = new OutputHandler({ cell }); - const f = throttle( - () => { - logger.debug("outputHandler", path, cell); - actions._set(cell, true); - }, - 1000 / BACKEND_OUTPUT_FPS, - { - leading: false, - trailing: true, - }, - ); - handler.on("change", f); - return handler; + return new MulticellOutputHandler(cells, actions); } diff --git a/src/packages/jupyter/execute/output-handler.ts b/src/packages/jupyter/execute/output-handler.ts index f1ef5a8eff..f26b075cd8 100644 --- a/src/packages/jupyter/execute/output-handler.ts +++ b/src/packages/jupyter/execute/output-handler.ts @@ -54,6 +54,14 @@ interface Message { data?: { [mimeType: string]: any }; } +interface JupyterMessage { + metadata?; + content?; + buffers?; + msg_type?: string; + done?: boolean; +} + interface Options { // object; the cell whose output (etc.) will get mutated cell: Cell; @@ -109,7 +117,7 @@ export class OutputHandler extends EventEmitter { } // mesg = from the kernel - process = (mesg) => { + process = (mesg: JupyterMessage) => { if (mesg == null) { // can't possibly happen, return; From 8a5911b171163b1d1e674ec6975b00d060a12c5b Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 27 Jul 2025 22:02:32 +0000 Subject: [PATCH 106/270] ensure cells run in order --- src/packages/frontend/jupyter/browser-actions.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/packages/frontend/jupyter/browser-actions.ts b/src/packages/frontend/jupyter/browser-actions.ts index 6ddf7e0d9b..e5a3f9ab7f 100644 --- a/src/packages/frontend/jupyter/browser-actions.ts +++ b/src/packages/frontend/jupyter/browser-actions.ts @@ -27,6 +27,7 @@ import { base64ToBuffer, bufferToBase64 } from "@cocalc/util/base64"; import { Config as FormatterConfig, Syntax } from "@cocalc/util/code-formatter"; import { closest_kernel_match, + field_cmp, from_json, history_path, merge_copy, @@ -1537,6 +1538,8 @@ export class JupyterActions extends JupyterActions0 { } cells.push(cell); } + // ensures cells run in order: + cells.sort(field_cmp("pos")); const runner = await client.run(cells); let handler: null | OutputHandler = null; From e6fc7176258d420fe55c2d597cb92a7db762199c Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 27 Jul 2025 22:35:10 +0000 Subject: [PATCH 107/270] jupyter eval: support noHalt --- .../conat/project/jupyter/run-code.ts | 24 +++++++++++++++---- src/packages/frontend/components/time-ago.tsx | 2 +- .../frontend/jupyter/browser-actions.ts | 8 +++---- .../frontend/jupyter/cell-output-time.tsx | 16 ++++++++++--- src/packages/jupyter/control.ts | 10 +++++--- .../jupyter/execute/output-handler.ts | 4 ++-- 6 files changed, 46 insertions(+), 18 deletions(-) diff --git a/src/packages/conat/project/jupyter/run-code.ts b/src/packages/conat/project/jupyter/run-code.ts index ed9f0cbe6b..6efe51fbdf 100644 --- a/src/packages/conat/project/jupyter/run-code.ts +++ b/src/packages/conat/project/jupyter/run-code.ts @@ -48,6 +48,8 @@ export interface RunOptions { path: string; // array of input cells to run cells: InputCell[]; + // if true do not halt running the cells, even if one fails with an error + noHalt?: boolean; } type JupyterCodeRunner = ( @@ -97,10 +99,17 @@ export function jupyterServer({ }); socket.on("request", async (mesg) => { - const { path, cells } = mesg.data; + const { path, cells, noHalt } = mesg.data; try { mesg.respondSync(null); - await handleRequest({ socket, jupyterRun, outputHandler, path, cells }); + await handleRequest({ + socket, + jupyterRun, + outputHandler, + path, + cells, + noHalt, + }); } catch (err) { //console.log(err); logger.debug("server: failed response -- ", err); @@ -124,8 +133,9 @@ async function handleRequest({ outputHandler, path, cells, + noHalt, }) { - const runner = await jupyterRun({ path, cells }); + const runner = await jupyterRun({ path, cells, noHalt }); const output: OutputMessage[] = []; let handler: OutputHandler | null = null; for await (const mesg of runner) { @@ -172,7 +182,7 @@ class JupyterClient { this.socket.close(); }; - run = async (cells: InputCell[]) => { + run = async (cells: InputCell[], opts: { noHalt?: boolean } = {}) => { if (this.iter) { // one evaluation at a time. this.iter.end(); @@ -197,7 +207,11 @@ class JupyterClient { const cells1 = cells.map(({ id, input }) => { return { id, input }; }); - await this.socket.request({ path: this.path, cells: cells1 }); + await this.socket.request({ + path: this.path, + cells: cells1, + noHalt: opts.noHalt, + }); return this.iter; }; } diff --git a/src/packages/frontend/components/time-ago.tsx b/src/packages/frontend/components/time-ago.tsx index 784747c0d3..58010616ab 100644 --- a/src/packages/frontend/components/time-ago.tsx +++ b/src/packages/frontend/components/time-ago.tsx @@ -203,7 +203,7 @@ export const TimeAgo: React.FC = React.memo( }: TimeAgoElementProps) => { const { timeAgoAbsolute } = useAppContext(); - if (!date || date.valueOf()) { + if (!date?.valueOf()) { return <>; } diff --git a/src/packages/frontend/jupyter/browser-actions.ts b/src/packages/frontend/jupyter/browser-actions.ts index e5a3f9ab7f..f8000170ac 100644 --- a/src/packages/frontend/jupyter/browser-actions.ts +++ b/src/packages/frontend/jupyter/browser-actions.ts @@ -219,7 +219,7 @@ export class JupyterActions extends JupyterActions0 { public run_cell( id: string, save: boolean = true, - no_halt: boolean = false, + noHalt: boolean = false, ): void { if (this.store.get("read_only")) return; const cell = this.store.getIn(["cells", id]); @@ -235,7 +235,7 @@ export class JupyterActions extends JupyterActions0 { this.clear_cell(id, save); return; } - this.runCells([id], no_halt); + this.runCells([id], { noHalt }); //this.run_code_cell(id, save, no_halt); if (save) { this.save_asap(); @@ -1505,7 +1505,7 @@ export class JupyterActions extends JupyterActions0 { }; private jupyterClient?; - runCells = async (ids: string[], _noHalt) => { + runCells = async (ids: string[], opts: { noHalt?: boolean } = {}) => { if (this.jupyterClient == null) { // [ ] **TODO: Must invalidate this when compute server changes!!!!!** // and @@ -1541,7 +1541,7 @@ export class JupyterActions extends JupyterActions0 { // ensures cells run in order: cells.sort(field_cmp("pos")); - const runner = await client.run(cells); + const runner = await client.run(cells, opts); let handler: null | OutputHandler = null; let id: null | string = null; for await (const mesgs of runner) { diff --git a/src/packages/frontend/jupyter/cell-output-time.tsx b/src/packages/frontend/jupyter/cell-output-time.tsx index e4e7c031dc..333e3cde11 100644 --- a/src/packages/frontend/jupyter/cell-output-time.tsx +++ b/src/packages/frontend/jupyter/cell-output-time.tsx @@ -23,6 +23,14 @@ interface CellTimingProps { // make this small so smooth. const DELAY_MS = 100; +function humanReadableSeconds(s) { + if (s >= 0.9) { + return seconds2hms(s, true); + } else { + return `${Math.round(s * 1000)} ms`; + } +} + export default function CellTiming({ start, end, @@ -53,10 +61,12 @@ export default function CellTiming({ - Evaluated using {capitalize(kernel)} and took - about {seconds2hms(ms / 1000, true)}. + Took about {humanReadableSeconds(ms / 1000)}. Evaluated{" "} + + {kernel ? " using " : ""} + {capitalize(kernel)}. {last != null ? ( - <> Previous run took {seconds2hms(last / 1000, true)}. + <> Previous run took {humanReadableSeconds(last / 1000)}. ) : undefined} } diff --git a/src/packages/jupyter/control.ts b/src/packages/jupyter/control.ts index 8b3dffba0c..b581dd9a1f 100644 --- a/src/packages/jupyter/control.ts +++ b/src/packages/jupyter/control.ts @@ -65,8 +65,8 @@ export function jupyterStop({ path }: { path: string }) { } // Returns async iterator over outputs -export async function jupyterRun({ path, cells }: RunOptions) { - logger.debug("jupyterRun", { path }); // , cells }); +export async function jupyterRun({ path, cells, noHalt }: RunOptions) { + logger.debug("jupyterRun", { path, noHalt }); const session = sessions[path]; if (session == null) { @@ -85,12 +85,16 @@ export async function jupyterRun({ path, cells }: RunOptions) { async function* run() { for (const cell of cells) { const output = actions.jupyter_kernel.execute_code({ - halt_on_error: true, + halt_on_error: !noHalt, code: cell.input, }); for await (const mesg of output.iter()) { mesg.id = cell.id; yield mesg; + if (!noHalt && mesg.msg_type == "error") { + // done running code because there was an error. + return; + } } if (actions.jupyter_kernel.failedError) { // kernel failed during call diff --git a/src/packages/jupyter/execute/output-handler.ts b/src/packages/jupyter/execute/output-handler.ts index f26b075cd8..47974f325a 100644 --- a/src/packages/jupyter/execute/output-handler.ts +++ b/src/packages/jupyter/execute/output-handler.ts @@ -28,10 +28,10 @@ OutputHandler emits these events: import { callback } from "awaiting"; import { EventEmitter } from "events"; -import { close, server_time, len, is_object } from "@cocalc/util/misc"; +import { close, len, is_object } from "@cocalc/util/misc"; import { type TypedMap } from "@cocalc/util/types/typed-map"; -const now = () => server_time().valueOf() - 0; +const now = () => Date.now(); const MIN_SAVE_INTERVAL_MS = 500; const MAX_SAVE_INTERVAL_MS = 45000; From 3ca8aa96035f359f65295aa0d3f68d59c3ce6918 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 27 Jul 2025 22:41:22 +0000 Subject: [PATCH 108/270] jupyter runner: include the kernel (not bothering with this if exec moves to backend, since that should be rare, and this isn't super important functionality) --- src/packages/frontend/jupyter/browser-actions.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/packages/frontend/jupyter/browser-actions.ts b/src/packages/frontend/jupyter/browser-actions.ts index f8000170ac..a7f90d9c13 100644 --- a/src/packages/frontend/jupyter/browser-actions.ts +++ b/src/packages/frontend/jupyter/browser-actions.ts @@ -1522,6 +1522,7 @@ export class JupyterActions extends JupyterActions0 { throw Error("bug"); } const cells: any[] = []; + const kernel = this.store.get("kernel"); for (const id of ids) { const cell = this.store.getIn(["cells", id])?.toJS(); if (!cell?.input?.trim()) { @@ -1553,6 +1554,7 @@ export class JupyterActions extends JupyterActions0 { // cell removed? cell = { id }; } + cell.kernel = kernel; handler?.done(); handler = this.getOutputHandler(cell); } From 198f0e2584f4bca657c2e6005692b0ddd4601cf6 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 27 Jul 2025 23:44:25 +0000 Subject: [PATCH 109/270] jupyter run: record last evaluation time --- src/packages/frontend/jupyter/browser-actions.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/packages/frontend/jupyter/browser-actions.ts b/src/packages/frontend/jupyter/browser-actions.ts index a7f90d9c13..1660b7b8d7 100644 --- a/src/packages/frontend/jupyter/browser-actions.ts +++ b/src/packages/frontend/jupyter/browser-actions.ts @@ -1535,6 +1535,8 @@ export class JupyterActions extends JupyterActions0 { if (n == "0") continue; cell.output[n] = null; } + // time last evaluation took + cell.last = cell.start && cell.end ? cell.end - cell.start : null; this._set(cell, false); } cells.push(cell); From d908e2663bd8a503550d08a359e65e6f248b2334 Mon Sep 17 00:00:00 2001 From: William Stein Date: Mon, 28 Jul 2025 01:15:24 +0000 Subject: [PATCH 110/270] jupyter run: queuing up cells to run --- .../frontend/jupyter/browser-actions.ts | 178 ++++++++++++------ src/packages/frontend/jupyter/cell-input.tsx | 4 +- src/packages/frontend/jupyter/cell-list.tsx | 3 + src/packages/frontend/jupyter/cell.tsx | 7 +- src/packages/frontend/jupyter/main.tsx | 10 +- .../frontend/jupyter/prompt/input.tsx | 1 - src/packages/jupyter/redux/actions.ts | 5 + src/packages/jupyter/redux/store.ts | 5 + 8 files changed, 152 insertions(+), 61 deletions(-) diff --git a/src/packages/frontend/jupyter/browser-actions.ts b/src/packages/frontend/jupyter/browser-actions.ts index 1660b7b8d7..a4475db41d 100644 --- a/src/packages/frontend/jupyter/browser-actions.ts +++ b/src/packages/frontend/jupyter/browser-actions.ts @@ -8,7 +8,7 @@ browser-actions: additional actions that are only available in the web browser frontend. */ import * as awaiting from "awaiting"; -import { fromJS, Map } from "immutable"; +import { fromJS, Map, Set as iSet } from "immutable"; import { debounce, isEqual } from "lodash"; import { jupyter, labels } from "@cocalc/frontend/i18n"; import { getIntl } from "@cocalc/frontend/i18n/get-intl"; @@ -1496,74 +1496,140 @@ export class JupyterActions extends JupyterActions0 { getOutputHandler = (cell) => { const handler = new OutputHandler({ cell }); - const f = throttle(() => this._set(cell, false), 1000 / OUTPUT_FPS, { - leading: false, - trailing: true, - }); + let first = true; + const f = throttle( + () => { + // save first so that other clients know this cell is running. + this._set(cell, first); + first = false; + }, + 1000 / OUTPUT_FPS, + { + leading: false, + trailing: true, + }, + ); handler.on("change", f); return handler; }; - private jupyterClient?; - runCells = async (ids: string[], opts: { noHalt?: boolean } = {}) => { - if (this.jupyterClient == null) { - // [ ] **TODO: Must invalidate this when compute server changes!!!!!** - // and - const compute_server_id = await this.getComputeServerId(); - this.jupyterClient = jupyterClient({ - path: this.syncdbPath, - client: webapp_client.conat_client.conat(), - project_id: this.project_id, - compute_server_id, - }); + private addPendingCells = (ids: string[]) => { + let pendingCells = this.store.get("pendingCells") ?? iSet(); + for (const id of ids) { + pendingCells = pendingCells.add(id); } - const client = this.jupyterClient; - if (client == null) { - throw Error("bug"); + this.store.setState({ pendingCells }); + }; + private deletePendingCells = (ids: string[]) => { + let pendingCells = this.store.get("pendingCells"); + if (pendingCells == null) { + return; } - const cells: any[] = []; - const kernel = this.store.get("kernel"); for (const id of ids) { - const cell = this.store.getIn(["cells", id])?.toJS(); - if (!cell?.input?.trim()) { - // nothing to do - continue; + pendingCells = pendingCells.delete(id); + } + this.store.setState({ pendingCells }); + }; + + // uses inheritence so NOT arrow function + protected clearRunQueue() { + this.store?.setState({ pendingCells: iSet() }); + this.runQueue.length = 0; + } + + private jupyterClient?; + private runQueue: any[] = []; + private runningNow = false; + runCells = async (ids: string[], opts: { noHalt?: boolean } = {}) => { + if (this.runningNow) { + this.runQueue.push([ids, opts]); + this.addPendingCells(ids); + return; + } + try { + this.runningNow = true; + if (this.jupyterClient == null) { + // [ ] **TODO: Must invalidate this when compute server changes!!!!!** + // and + const compute_server_id = await this.getComputeServerId(); + this.jupyterClient = jupyterClient({ + path: this.syncdbPath, + client: webapp_client.conat_client.conat(), + project_id: this.project_id, + compute_server_id, + }); + } + const client = this.jupyterClient; + if (client == null) { + throw Error("bug"); } - if (cell.output) { - // trick to avoid flicker - for (const n in cell.output) { - if (n == "0") continue; - cell.output[n] = null; + const cells: any[] = []; + const kernel = this.store.get("kernel"); + + for (const id of ids) { + const cell = this.store.getIn(["cells", id])?.toJS(); + if (!cell?.input?.trim()) { + // nothing to do + continue; + } + if (!kernel) { + this._set({ type: "cell", id, state: "done" }); + continue; + } + if (cell.output) { + // trick to avoid flicker + for (const n in cell.output) { + if (n == "0") continue; + cell.output[n] = null; + } + // time last evaluation took + cell.last = cell.start && cell.end ? cell.end - cell.start : null; + this._set(cell, false); } - // time last evaluation took - cell.last = cell.start && cell.end ? cell.end - cell.start : null; - this._set(cell, false); + cells.push(cell); } - cells.push(cell); - } - // ensures cells run in order: - cells.sort(field_cmp("pos")); - - const runner = await client.run(cells, opts); - let handler: null | OutputHandler = null; - let id: null | string = null; - for await (const mesgs of runner) { - for (const mesg of mesgs) { - if (mesg.id !== id || handler == null) { - id = mesg.id; - let cell = this.store.getIn(["cells", mesg.id])?.toJS(); - if (cell == null) { - // cell removed? - cell = { id }; + this.addPendingCells(ids); + + // ensures cells run in order: + cells.sort(field_cmp("pos")); + + const runner = await client.run(cells, opts); + let handler: null | OutputHandler = null; + let id: null | string = null; + for await (const mesgs of runner) { + for (const mesg of mesgs) { + if (!opts.noHalt && mesg.msg_type == "error") { + this.clearRunQueue(); + } + if (mesg.id !== id || handler == null) { + id = mesg.id; + if (id == null) { + continue; + } + this.deletePendingCells([id]); + let cell = this.store.getIn(["cells", mesg.id])?.toJS(); + if (cell == null) { + // cell removed? + cell = { id }; + } + cell.kernel = kernel; + handler?.done(); + handler = this.getOutputHandler(cell); } - cell.kernel = kernel; - handler?.done(); - handler = this.getOutputHandler(cell); + handler.process(mesg); } - handler.process(mesg); + } + console.log("exited the runner loop"); + handler?.done(); + this.save_asap(); + } catch (err) { + console.log("runCells", err); + } finally { + this.runningNow = false; + if (this.runQueue.length > 0) { + const [ids, opts] = this.runQueue.shift(); + this.runCells(ids, opts); } } - handler?.done(); - this.save_asap(); }; } diff --git a/src/packages/frontend/jupyter/cell-input.tsx b/src/packages/frontend/jupyter/cell-input.tsx index d3f740ec5c..25f0409df8 100644 --- a/src/packages/frontend/jupyter/cell-input.tsx +++ b/src/packages/frontend/jupyter/cell-input.tsx @@ -73,6 +73,7 @@ export interface CellInputProps { computeServerId?: number; setShowAICellGen?: (show: Position) => void; dragHandle?: React.JSX.Element; + isPending?: boolean; } export const CellInput: React.FC = React.memo( @@ -96,7 +97,7 @@ export const CellInput: React.FC = React.memo( = React.memo( next.index !== cur.index || next.computeServerId != cur.computeServerId || next.dragHandle !== cur.dragHandle || + next.isPending !== cur.isPending || (next.cell_toolbar === "slideshow" && next.cell.get("slide") !== cur.cell.get("slide")) ), diff --git a/src/packages/frontend/jupyter/cell-list.tsx b/src/packages/frontend/jupyter/cell-list.tsx index 3812b636b0..e6342f78a2 100644 --- a/src/packages/frontend/jupyter/cell-list.tsx +++ b/src/packages/frontend/jupyter/cell-list.tsx @@ -91,6 +91,7 @@ interface CellListProps { llmTools?: LLMTools; computeServerId?: number; read_only?: boolean; + pendingCells?: immutable.Set; } export const CellList: React.FC = (props: CellListProps) => { @@ -121,6 +122,7 @@ export const CellList: React.FC = (props: CellListProps) => { llmTools, computeServerId, read_only, + pendingCells, } = props; const cellListDivRef = useRef(null); @@ -478,6 +480,7 @@ export const CellList: React.FC = (props: CellListProps) => { dragHandle={dragHandle} read_only={read_only} isDragging={isDragging} + isPending={pendingCells?.has(id)} /> ); diff --git a/src/packages/frontend/jupyter/cell.tsx b/src/packages/frontend/jupyter/cell.tsx index 3d31309d07..251f407db7 100644 --- a/src/packages/frontend/jupyter/cell.tsx +++ b/src/packages/frontend/jupyter/cell.tsx @@ -37,7 +37,6 @@ interface Props { font_size: number; id?: string; // redundant, since it's in the cell. actions?: JupyterActions; - name?: string; index?: number; // position of cell in the list of all cells; just used to optimize rendering and for no other reason. is_current?: boolean; is_selected?: boolean; @@ -62,6 +61,8 @@ interface Props { dragHandle?: React.JSX.Element; read_only?: boolean; isDragging?: boolean; + isPending?: boolean; + name?: string; } function areEqual(props: Props, nextProps: Props): boolean { @@ -91,7 +92,8 @@ function areEqual(props: Props, nextProps: Props): boolean { (nextProps.is_current || props.is_current)) || nextProps.dragHandle !== props.dragHandle || nextProps.read_only !== props.read_only || - nextProps.isDragging !== props.isDragging + nextProps.isDragging !== props.isDragging || + nextProps.isPending !== props.isPending ); } @@ -138,6 +140,7 @@ export const Cell: React.FC = React.memo((props: Props) => { computeServerId={props.computeServerId} setShowAICellGen={setShowAICellGen} dragHandle={props.dragHandle} + isPending={props.isPending} /> ); } diff --git a/src/packages/frontend/jupyter/main.tsx b/src/packages/frontend/jupyter/main.tsx index 819b519e69..aca7b02f86 100644 --- a/src/packages/frontend/jupyter/main.tsx +++ b/src/packages/frontend/jupyter/main.tsx @@ -183,6 +183,10 @@ export const JupyterEditor: React.FC = React.memo((props: Props) => { name, "check_select_kernel_init", ]); + const pendingCells: undefined | immutable.Set = useRedux([ + name, + "pendingCells", + ]); const computeServerId = path ? useTypedRedux({ project_id }, "compute_server_ids")?.get(syncdbPath(path)) @@ -318,6 +322,7 @@ export const JupyterEditor: React.FC = React.memo((props: Props) => { use_windowed_list={useWindowedListRef.current} llmTools={llmTools} computeServerId={computeServerId} + pendingCells={pendingCells} /> ); } @@ -451,7 +456,10 @@ export const JupyterEditor: React.FC = React.memo((props: Props) => { overflowY: "hidden", }} > - + {!read_only && } {render_error()} {render_modals()} diff --git a/src/packages/frontend/jupyter/prompt/input.tsx b/src/packages/frontend/jupyter/prompt/input.tsx index fb055aa8f5..7993720633 100644 --- a/src/packages/frontend/jupyter/prompt/input.tsx +++ b/src/packages/frontend/jupyter/prompt/input.tsx @@ -13,7 +13,6 @@ src/packages/frontend/frame-editors/whiteboard-editor/elements/code/input-prompt */ import React from "react"; - import { Icon } from "@cocalc/frontend/components/icon"; import { TimeAgo } from "@cocalc/frontend/components/time-ago"; import { Tip } from "@cocalc/frontend/components/tip"; diff --git a/src/packages/jupyter/redux/actions.ts b/src/packages/jupyter/redux/actions.ts index 2c8975435a..29f931273e 100644 --- a/src/packages/jupyter/redux/actions.ts +++ b/src/packages/jupyter/redux/actions.ts @@ -1049,7 +1049,12 @@ export abstract class JupyterActions extends Actions { this.save_asap(); }; + protected clearRunQueue() { + // implemented in frontend browser actions + } + clear_all_cell_run_state = (): void => { + this.clearRunQueue(); const { store } = this; if (!store) { return; diff --git a/src/packages/jupyter/redux/store.ts b/src/packages/jupyter/redux/store.ts index 1d62316bdf..e611b59eec 100644 --- a/src/packages/jupyter/redux/store.ts +++ b/src/packages/jupyter/redux/store.ts @@ -103,6 +103,11 @@ export interface JupyterStoreState { // run progress = Percent (0-100) of runnable cells that have been run since the last // kernel restart. (Thus markdown and empty cells are excluded.) runProgress?: number; + + // cells that this particular client has queued up to run. This is + // only known to this client, goes away on browser refresh, and is used + // only visually for the user to see. + pendingCells: Set; } export const initial_jupyter_store_state: { From a95ed82c7725fb07e9413990733ccd979383e83a Mon Sep 17 00:00:00 2001 From: William Stein Date: Mon, 28 Jul 2025 01:22:27 +0000 Subject: [PATCH 111/270] delete jupyter exec_state sync, since it's no longer possible --- src/packages/jupyter/redux/project-actions.ts | 52 ------------------- 1 file changed, 52 deletions(-) diff --git a/src/packages/jupyter/redux/project-actions.ts b/src/packages/jupyter/redux/project-actions.ts index 92d28879b6..c6abcde4ba 100644 --- a/src/packages/jupyter/redux/project-actions.ts +++ b/src/packages/jupyter/redux/project-actions.ts @@ -181,7 +181,6 @@ export class JupyterActions extends JupyterActions0 { dbg("initializing blob store"); await this.initBlobStore(); - this.sync_exec_state = debounce(this.sync_exec_state, 2000); this._throttled_ensure_positions_are_unique = debounce( this.ensure_positions_are_unique, 5000, @@ -318,7 +317,6 @@ export class JupyterActions extends JupyterActions0 { this.ensure_there_is_a_cell(); this._throttled_ensure_positions_are_unique(); - this.sync_exec_state(); }; // ensure_backend_kernel_setup ensures that we have a connection @@ -562,56 +560,6 @@ export class JupyterActions extends JupyterActions0 { } } - // Ensure that the cells listed as running *are* exactly the - // ones actually running or queued up to run. - sync_exec_state = () => { - // sync_exec_state is debounced, so it is *expected* to get called - // after actions have been closed. - if (this.store == null || this._state !== "ready") { - // not initialized, so we better not - // mess with cell state (that is somebody else's responsibility). - return; - } - - const dbg = this.dbg("sync_exec_state"); - let change = false; - const cells = this.store.get("cells"); - // First verify that all actual cells that are said to be running - // (according to the store) are in fact running. - if (cells != null) { - cells.forEach((cell, id) => { - const state = cell.get("state"); - if ( - state != null && - state != "done" && - state != "start" && // regarding "start", see https://github.com/sagemathinc/cocalc/issues/5467 - !this._running_cells?.[id] - ) { - dbg(`set cell ${id} with state "${state}" to done`); - this._set({ type: "cell", id, state: "done" }, false); - change = true; - } - }); - } - if (this._running_cells != null) { - const cells = this.store.get("cells"); - // Next verify that every cell actually running is still in the document - // and listed as running. TimeTravel, deleting cells, etc., can - // certainly lead to this being necessary. - for (const id in this._running_cells) { - const state = cells.getIn([id, "state"]); - if (state == null || state === "done") { - // cell no longer exists or isn't in a running state - dbg(`tell kernel to not run ${id}`); - this._cancel_run(id); - } - } - } - if (change) { - return this._sync(); - } - }; - _cancel_run = (id: any) => { const dbg = this.dbg(`_cancel_run ${id}`); // All these checks are so we only cancel if it is actually running From 65ffcb0b80e05b46b61b424225ebc02d0b4c0be2 Mon Sep 17 00:00:00 2001 From: William Stein Date: Mon, 28 Jul 2025 01:38:54 +0000 Subject: [PATCH 112/270] jupyter: fix opening timetravel --- .../frame-editors/code-editor/actions.ts | 95 ++++++++++--------- .../frame-editors/generic/syncstring-fake.ts | 2 + .../frame-editors/jupyter-editor/actions.ts | 7 +- .../time-travel-editor/actions.ts | 7 +- 4 files changed, 63 insertions(+), 48 deletions(-) diff --git a/src/packages/frontend/frame-editors/code-editor/actions.ts b/src/packages/frontend/frame-editors/code-editor/actions.ts index 8a7eb2c93e..e0adf71f0e 100644 --- a/src/packages/frontend/frame-editors/code-editor/actions.ts +++ b/src/packages/frontend/frame-editors/code-editor/actions.ts @@ -299,55 +299,58 @@ export class Actions< } protected _init_syncstring(): void { - if (this.doctype == "none") { - this._syncstring = syncstring({ - project_id: this.project_id, - path: this.path, - cursors: !this.disable_cursors, - before_change_hook: () => this.set_syncstring_to_codemirror(), - after_change_hook: () => this.set_codemirror_to_syncstring(), - fake: true, - patch_interval: 500, - }) as SyncString; - } else if (this.doctype == "syncstring") { - this._syncstring = syncstring2({ - project_id: this.project_id, - path: this.path, - cursors: !this.disable_cursors, - }); - } else if (this.doctype == "syncdb") { - if ( - this.primary_keys == null || - this.primary_keys.length == null || - this.primary_keys.length <= 0 - ) { - throw Error("primary_keys must be array of positive length"); - } - this._syncstring = syncdb2({ - project_id: this.project_id, - path: this.path, - primary_keys: this.primary_keys, - string_cols: this.string_cols, - cursors: !this.disable_cursors, - }); - if (this.searchEmbeddings != null) { - if (!this.primary_keys.includes(this.searchEmbeddings.primaryKey)) { - throw Error( - `search embedding primaryKey must be in ${JSON.stringify( - this.primary_keys, - )}`, - ); + if (this._syncstring == null) { + // this._syncstring wasn't set in derived class so we set it here + if (this.doctype == "none") { + this._syncstring = syncstring({ + project_id: this.project_id, + path: this.path, + cursors: !this.disable_cursors, + before_change_hook: () => this.set_syncstring_to_codemirror(), + after_change_hook: () => this.set_codemirror_to_syncstring(), + fake: true, + patch_interval: 500, + }) as SyncString; + } else if (this.doctype == "syncstring") { + this._syncstring = syncstring2({ + project_id: this.project_id, + path: this.path, + cursors: !this.disable_cursors, + }); + } else if (this.doctype == "syncdb") { + if ( + this.primary_keys == null || + this.primary_keys.length == null || + this.primary_keys.length <= 0 + ) { + throw Error("primary_keys must be array of positive length"); } - if (!this.string_cols.includes(this.searchEmbeddings.textColumn)) { - throw Error( - `search embedding textColumn must be in ${JSON.stringify( - this.string_cols, - )}`, - ); + this._syncstring = syncdb2({ + project_id: this.project_id, + path: this.path, + primary_keys: this.primary_keys, + string_cols: this.string_cols, + cursors: !this.disable_cursors, + }); + if (this.searchEmbeddings != null) { + if (!this.primary_keys.includes(this.searchEmbeddings.primaryKey)) { + throw Error( + `search embedding primaryKey must be in ${JSON.stringify( + this.primary_keys, + )}`, + ); + } + if (!this.string_cols.includes(this.searchEmbeddings.textColumn)) { + throw Error( + `search embedding textColumn must be in ${JSON.stringify( + this.string_cols, + )}`, + ); + } } + } else { + throw Error(`invalid doctype="${this.doctype}"`); } - } else { - throw Error(`invalid doctype="${this.doctype}"`); } this._syncstring.once("deleted", () => { diff --git a/src/packages/frontend/frame-editors/generic/syncstring-fake.ts b/src/packages/frontend/frame-editors/generic/syncstring-fake.ts index 3e5b4e87cf..f22ef04532 100644 --- a/src/packages/frontend/frame-editors/generic/syncstring-fake.ts +++ b/src/packages/frontend/frame-editors/generic/syncstring-fake.ts @@ -25,6 +25,8 @@ export class FakeSyncstring extends EventEmitter { this.emit("ready"); } + hasFullHistory = () => true; + close() {} from_str() {} diff --git a/src/packages/frontend/frame-editors/jupyter-editor/actions.ts b/src/packages/frontend/frame-editors/jupyter-editor/actions.ts index 4d5f394c2f..e15e797f68 100644 --- a/src/packages/frontend/frame-editors/jupyter-editor/actions.ts +++ b/src/packages/frontend/frame-editors/jupyter-editor/actions.ts @@ -42,8 +42,13 @@ export class JupyterEditorActions extends BaseActions { return { type: "jupyter_cell_notebook" }; } - _init2(): void { + protected _init_syncstring(): void { this.create_jupyter_actions(); + this._syncstring = this.jupyter_actions.syncdb; + super._init_syncstring(); + } + + _init2(): void { this.init_new_frame(); this.init_changes_state(); diff --git a/src/packages/frontend/frame-editors/time-travel-editor/actions.ts b/src/packages/frontend/frame-editors/time-travel-editor/actions.ts index 2ef79fc2c6..cf13516678 100644 --- a/src/packages/frontend/frame-editors/time-travel-editor/actions.ts +++ b/src/packages/frontend/frame-editors/time-travel-editor/actions.ts @@ -151,7 +151,12 @@ export class TimeTravelActions extends CodeEditorActions { } this.syncdoc = mainFileActions._syncstring; - if (this.syncdoc == null || this.syncdoc.get_state() == "closed") { + if ( + this.syncdoc == null || + this.syncdoc.get_state() == "closed" || + // @ts-ignore + this.syncdoc.is_fake + ) { return; } if (this.syncdoc.get_state() != "ready") { From 0afbd270f458bd436b4e201d024b018c77554772 Mon Sep 17 00:00:00 2001 From: William Stein Date: Mon, 28 Jul 2025 02:59:24 +0000 Subject: [PATCH 113/270] disabling/deleting a lot of jupyter backend code that will no longer be needed --- .../frontend/jupyter/browser-actions.ts | 4 +- src/packages/jupyter/control.ts | 1 + .../jupyter/execute/output-handler.ts | 2 + src/packages/jupyter/redux/actions.ts | 66 +--------------- src/packages/jupyter/redux/project-actions.ts | 75 +------------------ src/packages/project/conat/api/index.ts | 5 ++ src/packages/project/conat/open-files.ts | 3 + 7 files changed, 21 insertions(+), 135 deletions(-) diff --git a/src/packages/frontend/jupyter/browser-actions.ts b/src/packages/frontend/jupyter/browser-actions.ts index a4475db41d..c3fe57d307 100644 --- a/src/packages/frontend/jupyter/browser-actions.ts +++ b/src/packages/frontend/jupyter/browser-actions.ts @@ -1541,6 +1541,9 @@ export class JupyterActions extends JupyterActions0 { private runQueue: any[] = []; private runningNow = false; runCells = async (ids: string[], opts: { noHalt?: boolean } = {}) => { + if (this.store?.get("read_only")) { + return; + } if (this.runningNow) { this.runQueue.push([ids, opts]); this.addPendingCells(ids); @@ -1619,7 +1622,6 @@ export class JupyterActions extends JupyterActions0 { handler.process(mesg); } } - console.log("exited the runner loop"); handler?.done(); this.save_asap(); } catch (err) { diff --git a/src/packages/jupyter/control.ts b/src/packages/jupyter/control.ts index b581dd9a1f..6072920581 100644 --- a/src/packages/jupyter/control.ts +++ b/src/packages/jupyter/control.ts @@ -84,6 +84,7 @@ export async function jupyterRun({ path, cells, noHalt }: RunOptions) { logger.debug("jupyterRun: running"); async function* run() { for (const cell of cells) { + actions.ensure_backend_kernel_setup(); const output = actions.jupyter_kernel.execute_code({ halt_on_error: !noHalt, code: cell.input, diff --git a/src/packages/jupyter/execute/output-handler.ts b/src/packages/jupyter/execute/output-handler.ts index 47974f325a..94d5ae0cd0 100644 --- a/src/packages/jupyter/execute/output-handler.ts +++ b/src/packages/jupyter/execute/output-handler.ts @@ -99,6 +99,8 @@ export class OutputHandler extends EventEmitter { const { cell } = this._opts; cell.output = null; cell.exec_count = null; + // running a cell always de-collapses it: + cell.collapsed = false; cell.state = "run"; cell.start = null; cell.end = null; diff --git a/src/packages/jupyter/redux/actions.ts b/src/packages/jupyter/redux/actions.ts index 29f931273e..8cc0eac611 100644 --- a/src/packages/jupyter/redux/actions.ts +++ b/src/packages/jupyter/redux/actions.ts @@ -69,7 +69,6 @@ export abstract class JupyterActions extends Actions { public is_compute_server?: boolean; readonly path: string; readonly project_id: string; - private _last_start?: number; public jupyter_kernel?: JupyterKernelInterface; private last_cursor_move_time: Date = new Date(0); private _cursor_locs?: any; @@ -526,7 +525,6 @@ export abstract class JupyterActions extends Actions { } } - this.onCellChange(id, new_cell, old_cell); this.store.emit("cell_change", id, new_cell, old_cell); return cell_list_needs_recompute; @@ -698,11 +696,6 @@ export abstract class JupyterActions extends Actions { // things in project, browser, etc. } - protected onCellChange(_id: string, _new_cell: any, _old_cell: any) { - // no-op in base class. This is a hook though - // for potentially doing things when any cell changes. - } - ensure_backend_kernel_setup() { // nontrivial in the project, but not in client or here. } @@ -741,7 +734,6 @@ export abstract class JupyterActions extends Actions { } } } - //@dbg("_set")("obj=#{misc.to_json(obj)}") this.syncdb.set(obj); if (save) { this.syncdb.commit(); @@ -957,61 +949,11 @@ export abstract class JupyterActions extends Actions { } public run_code_cell( - id: string, - save: boolean = true, - no_halt: boolean = false, + _id: string, + _save: boolean = true, + _no_halt: boolean = false, ): void { - const cell = this.store.getIn(["cells", id]); - if (cell == null) { - // it is trivial to run a cell that does not exist -- nothing needs to be done. - return; - } - const kernel = this.store.get("kernel"); - if (kernel == null || kernel === "") { - // just in case, we clear any "running" indicators - this._set({ type: "cell", id, state: "done" }); - // don't attempt to run a code-cell if there is no kernel defined - this.set_error( - "No kernel set for running cells. Therefore it is not possible to run a code cell. You have to select a kernel!", - ); - return; - } - - if (cell.get("state", "done") != "done") { - // already running -- stop it first somehow if you want to run it again... - return; - } - - // We mark the start timestamp uniquely, so that the backend can sort - // multiple cells with a simultaneous time to start request. - - let start: number = this._client.server_time().valueOf(); - if (this._last_start != null && start <= this._last_start) { - start = this._last_start + 1; - } - this._last_start = start; - this.set_jupyter_metadata(id, "outputs_hidden", undefined, false); - - this._set( - { - type: "cell", - id, - state: "start", - start, - end: null, - // time last evaluation took - last: - cell.get("start") != null && cell.get("end") != null - ? cell.get("end") - cell.get("start") - : cell.get("last"), - output: null, - exec_count: null, - collapsed: null, - no_halt: no_halt ? no_halt : null, - }, - save, - ); - this.set_trust_notebook(true, save); + console.log("run_code_cell: deprecated"); } clear_cell = (id: string, save = true) => { diff --git a/src/packages/jupyter/redux/project-actions.ts b/src/packages/jupyter/redux/project-actions.ts index c6abcde4ba..7cf4d0085c 100644 --- a/src/packages/jupyter/redux/project-actions.ts +++ b/src/packages/jupyter/redux/project-actions.ts @@ -73,25 +73,7 @@ export class JupyterActions extends JupyterActions0 { save: boolean = true, no_halt: boolean = false, ): void { - if (this.store.get("read_only")) { - return; - } - const cell = this.store.getIn(["cells", id]); - if (cell == null) { - // it is trivial to run a cell that does not exist -- nothing needs to be done. - return; - } - const cell_type = cell.get("cell_type", "code"); - if (cell_type == "code") { - // when the backend is running code, just don't worry about - // trying to parse things like "foo?" out. We can't do - // it without CodeMirror, and it isn't worth it for that - // application. - this.run_code_cell(id, save, no_halt); - } - if (save) { - this.save_asap(); - } + console.log("run_cell: DEPRECATED"); } private set_backend_state(backend_state: BackendState): void { @@ -389,19 +371,6 @@ export class JupyterActions extends JupyterActions0 { this.set_kernel_error(error); }); - // Since we just made a new kernel, clearly no cells are running on the backend: - this._running_cells = {}; - - const toStart: string[] = []; - this.store?.get_cell_list().forEach((id) => { - if (this.store.getIn(["cells", id, "state"]) == "start") { - toStart.push(id); - } - }); - - dbg("clear cell run state"); - this.clear_all_cell_run_state(); - this.restartKernelOnClose = () => { // When the kernel closes, make sure a new kernel gets setup. if (this.store == null || this._state !== "ready") { @@ -426,11 +395,8 @@ export class JupyterActions extends JupyterActions0 { case "off": case "closed": // things went wrong. - this._running_cells = {}; - this.clear_all_cell_run_state(); this.set_backend_state("ready"); this.jupyter_kernel?.close(); - this.running_manager_run_cell_process_queue = false; delete this.jupyter_kernel; return; case "spawning": @@ -446,15 +412,6 @@ export class JupyterActions extends JupyterActions0 { this.handle_all_cell_attachments(); dbg("ready"); this.set_backend_state("ready"); - - // Run cells that the user explicitly set to be running before the - // kernel actually had finished starting up. - // This must be done after the state is ready. - if (toStart.length > 0) { - for (const id of toStart) { - this.run_cell(id); - } - } }; set_connection_file = () => { @@ -510,34 +467,6 @@ export class JupyterActions extends JupyterActions0 { await this.syncdb.wait(is_running, 60); } - // onCellChange is called after a cell change has been - // incorporated into the store after the syncdb change event. - // - If we are responsible for running cells, then it ensures - // that cell gets computed. - // - We also handle attachments for markdown cells. - protected onCellChange(id: string, new_cell: any, old_cell: any) { - const dbg = this.dbg(`onCellChange(id='${id}')`); - dbg(); - // this logging could be expensive due to toJS, so only uncomment - // if really needed - // dbg("new_cell=", new_cell?.toJS(), "old_cell", old_cell?.toJS()); - - if ( - new_cell?.get("state") === "start" && - old_cell?.get("state") !== "start" - ) { - this.manager_run_cell_enqueue(id); - // attachments below only happen for markdown cells, which don't get run, - // we can return here: - return; - } - - const attachments = new_cell?.get("attachments"); - if (attachments != null && attachments !== old_cell?.get("attachments")) { - this.handle_cell_attachments(new_cell); - } - } - protected __syncdb_change_post_hook(doInit: boolean) { if (doInit) { // Since just opening the actions in the project, definitely the kernel @@ -704,6 +633,8 @@ export class JupyterActions extends JupyterActions0 { manager_run_cell = (id: string) => { const dbg = this.dbg(`manager_run_cell(id='${id}')`); + console.log("manager_run_cell: DEPRECATED"); + return; dbg(JSON.stringify(misc.keys(this._running_cells))); if (this._running_cells == null) { diff --git a/src/packages/project/conat/api/index.ts b/src/packages/project/conat/api/index.ts index 7ce9561c64..a523d594d0 100644 --- a/src/packages/project/conat/api/index.ts +++ b/src/packages/project/conat/api/index.ts @@ -58,6 +58,7 @@ import { close as closeListings } from "@cocalc/project/conat/listings"; import { project_id } from "@cocalc/project/data"; import { close as closeFilesRead } from "@cocalc/project/conat/files/read"; import { close as closeFilesWrite } from "@cocalc/project/conat/files/write"; +import { close as closeJupyter } from "@cocalc/project/conat/jupyter"; import { getLogger } from "@cocalc/project/logger"; const logger = getLogger("conat:api"); @@ -109,6 +110,10 @@ async function handleMessage(api, subject, mesg) { closeListings(); await mesg.respond({ status: "terminated", service }); return; + } else if (service == "jupyter") { + closeJupyter(); + await mesg.respond({ status: "terminated", service }); + return; } else if (service == "files:read") { await closeFilesRead(); await mesg.respond({ status: "terminated", service }); diff --git a/src/packages/project/conat/open-files.ts b/src/packages/project/conat/open-files.ts index 11dab2fe09..1b93b8c3e5 100644 --- a/src/packages/project/conat/open-files.ts +++ b/src/packages/project/conat/open-files.ts @@ -227,6 +227,8 @@ async function handleChange({ doctype, id, }: OpenFileEntry & { id?: number }) { + // DEPRECATED! + return; if (!hasBackendState(path)) { return; } @@ -262,6 +264,7 @@ async function handleChange({ } } + // @ts-ignore if (time != null && time >= getCutoff()) { if (!isOpenHere) { logger.debug("handleChange: opening", { path }); From 9924e931c7d5c684c3766f75da403a57ca16664f Mon Sep 17 00:00:00 2001 From: William Stein Date: Mon, 28 Jul 2025 04:20:05 +0000 Subject: [PATCH 114/270] ts --- src/packages/jupyter/redux/project-actions.ts | 160 +----------------- 1 file changed, 4 insertions(+), 156 deletions(-) diff --git a/src/packages/jupyter/redux/project-actions.ts b/src/packages/jupyter/redux/project-actions.ts index 7cf4d0085c..742f4d6315 100644 --- a/src/packages/jupyter/redux/project-actions.ts +++ b/src/packages/jupyter/redux/project-actions.ts @@ -35,9 +35,6 @@ import { type DKV, dkv } from "@cocalc/conat/sync/dkv"; import { computeServerManager } from "@cocalc/conat/compute/manager"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; -// see https://github.com/sagemathinc/cocalc/issues/8060 -const MAX_OUTPUT_SAVE_DELAY = 30000; - // refuse to open an ipynb that is bigger than this: const MAX_SIZE_IPYNB_MB = 150; @@ -69,9 +66,9 @@ export class JupyterActions extends JupyterActions0 { // } public run_cell( - id: string, - save: boolean = true, - no_halt: boolean = false, + _id: string, + _save: boolean = true, + _no_halt: boolean = false, ): void { console.log("run_cell: DEPRECATED"); } @@ -631,157 +628,8 @@ export class JupyterActions extends JupyterActions0 { return handler; } - manager_run_cell = (id: string) => { - const dbg = this.dbg(`manager_run_cell(id='${id}')`); + manager_run_cell = (_id: string) => { console.log("manager_run_cell: DEPRECATED"); - return; - dbg(JSON.stringify(misc.keys(this._running_cells))); - - if (this._running_cells == null) { - this._running_cells = {}; - } - - if (this._running_cells[id]) { - dbg("cell already queued to run in kernel"); - return; - } - - // It's important to set this._running_cells[id] to be true so that - // sync_exec_state doesn't declare this cell done. The kernel identity - // will get set properly below in case it changes. - this._running_cells[id] = this.jupyter_kernel?.identity ?? "none"; - - const orig_cell = this.store.get("cells").get(id); - if (orig_cell == null) { - // nothing to do -- cell deleted - return; - } - - let input: string | undefined = orig_cell.get("input", ""); - if (input == null) { - input = ""; - } else { - input = input.trim(); - } - - const halt_on_error: boolean = !orig_cell.get("no_halt", false); - - if (this.jupyter_kernel == null) { - throw Error("bug -- this is guaranteed by the above"); - } - this._running_cells[id] = this.jupyter_kernel.identity; - - const cell: any = { - id, - type: "cell", - kernel: this.store.get("kernel"), - }; - - dbg(`using max_output_length=${this.store.get("max_output_length")}`); - const handler = this._output_handler(cell); - - // exponentiallyThrottledSaved calls this.syncdb?.save, but - // it throttles the calls, and does so using exponential backoff - // up to MAX_OUTPUT_SAVE_DELAY milliseconds. Basically every - // time exponentiallyThrottledSaved is called it increases the - // interval used for throttling by multiplying saveThrottleMs by 1.3 - // until saveThrottleMs gets to MAX_OUTPUT_SAVE_DELAY. There is no - // need at all to do a trailing call, since other code handles that. - let saveThrottleMs = 1; - let lastCall = 0; - const exponentiallyThrottledSaved = () => { - const now = Date.now(); - if (now - lastCall < saveThrottleMs) { - return; - } - lastCall = now; - saveThrottleMs = Math.min(1.3 * saveThrottleMs, MAX_OUTPUT_SAVE_DELAY); - this.syncdb?.save(); - }; - - handler.on("change", (save) => { - if (!this.store.getIn(["cells", id])) { - // The cell was deleted, but we just got some output - // NOTE: client shouldn't allow deleting running or queued - // cells, but we still want to do something useful/sensible. - // We put cell back where it was with same input. - cell.input = orig_cell.get("input"); - cell.pos = orig_cell.get("pos"); - } - this.syncdb.set(cell); - // This is potentially very verbose -- don't due it unless - // doing low level debugging: - //dbg(`change (save=${save}): cell='${JSON.stringify(cell)}'`); - if (save) { - exponentiallyThrottledSaved(); - } - }); - - handler.once("done", () => { - dbg("handler is done"); - this.store.removeListener("cell_change", cell_change); - exec.close(); - if (this._running_cells != null) { - delete this._running_cells[id]; - } - this.syncdb?.save(); - setTimeout(() => this.syncdb?.save(), 100); - }); - - if (this.jupyter_kernel == null) { - handler.error("Unable to start Jupyter"); - return; - } - - const get_password = (): string => { - if (this.jupyter_kernel == null) { - dbg("get_password", id, "no kernel"); - return ""; - } - const password = this.jupyter_kernel.store.get(id); - dbg("get_password", id, password); - this.jupyter_kernel.store.delete(id); - return password; - }; - - // This is used only for stdin right now. - const cell_change = (cell_id, new_cell) => { - if (id === cell_id) { - dbg("cell_change"); - handler.cell_changed(new_cell, get_password); - } - }; - this.store.on("cell_change", cell_change); - - const exec = this.jupyter_kernel.execute_code({ - code: input, - id, - stdin: handler.stdin, - halt_on_error, - }); - - exec.on("output", (mesg) => { - // uncomment only for specific low level debugging -- see https://github.com/sagemathinc/cocalc/issues/7022 - // dbg(`got mesg='${JSON.stringify(mesg)}'`); // !!!☡ ☡ ☡ -- EXTREME DANGER ☡ ☡ ☡ !!!! - if (mesg.content?.transient?.display_id != null) { - // See https://github.com/sagemathinc/cocalc/issues/2132 - // We find any other outputs in the document with - // the same transient.display_id, and set their output to - // this mesg's output. - this.handleTransientUpdate(mesg); - } - if (mesg.content.execution_state === "idle") { - this.store.removeListener("cell_change", cell_change); - return; - } - - handler.process(mesg); - }); - - exec.on("error", (err) => { - dbg(`got error='${err}'`); - handler.error(err); - }); }; reset_more_output = (id: string) => { From b4f060503f9febcaf2592890caa0909e7dab87d1 Mon Sep 17 00:00:00 2001 From: William Stein Date: Mon, 28 Jul 2025 05:53:43 +0000 Subject: [PATCH 115/270] jupyter run: run all cells and also deleting old code --- .../jupyter-editor/cell-notebook/actions.ts | 37 ++- .../frontend/jupyter/browser-actions.ts | 38 +-- .../frontend/jupyter/cell-buttonbar.tsx | 2 +- .../jupyter/output-messages/ipywidget.tsx | 2 +- src/packages/jupyter/redux/actions.ts | 17 +- src/packages/jupyter/redux/project-actions.ts | 279 +----------------- 6 files changed, 40 insertions(+), 335 deletions(-) diff --git a/src/packages/frontend/frame-editors/jupyter-editor/cell-notebook/actions.ts b/src/packages/frontend/frame-editors/jupyter-editor/cell-notebook/actions.ts index 74cfc894fd..8158f8cb40 100644 --- a/src/packages/frontend/frame-editors/jupyter-editor/cell-notebook/actions.ts +++ b/src/packages/frontend/frame-editors/jupyter-editor/cell-notebook/actions.ts @@ -324,36 +324,41 @@ export class NotebookFrameActions { } } - public run_selected_cells(v?: string[]): void { + public run_selected_cells(ids?: string[]): void { this.save_input_editor(); - if (v === undefined) { - v = this.store.get_selected_cell_ids_list(); + if (ids == null) { + ids = this.store.get_selected_cell_ids_list(); } // for whatever reason, any running of a cell deselects // in official jupyter this.unselect_all_cells(); - for (const id of v) { - const save = id === v[v.length - 1]; // save only last one. - this.run_cell(id, save); - } + this.runCells(ids); + } + + run_cell(id: string) { + this.runCells([id]); } // This is here since it depends on knowing the edit state // of markdown cells. - public run_cell(id: string, save: boolean = true): void { - const type = this.jupyter_actions.store.get_cell_type(id); - if (type === "markdown") { - if (this.store.get("md_edit_ids", Set()).contains(id)) { - this.set_md_cell_not_editing(id); + public runCells(ids: string[]): void { + const v: string[] = []; + for (const id of ids) { + const type = this.jupyter_actions.store.get_cell_type(id); + if (type === "markdown") { + if (this.store.get("md_edit_ids", Set()).contains(id)) { + this.set_md_cell_not_editing(id); + } + } else if (type === "code") { + v.push(id); } - return; + // running is a no-op for raw cells. } - if (type === "code") { - this.jupyter_actions.run_cell(id, save); + if (v.length > 0) { + this.jupyter_actions.runCells(v); } - // running is a no-op for raw cells. } /*** diff --git a/src/packages/frontend/jupyter/browser-actions.ts b/src/packages/frontend/jupyter/browser-actions.ts index c3fe57d307..8dcd87cdcc 100644 --- a/src/packages/frontend/jupyter/browser-actions.ts +++ b/src/packages/frontend/jupyter/browser-actions.ts @@ -216,33 +216,6 @@ export class JupyterActions extends JupyterActions0 { } }; - public run_cell( - id: string, - save: boolean = true, - noHalt: boolean = false, - ): void { - if (this.store.get("read_only")) return; - const cell = this.store.getIn(["cells", id]); - if (cell == null) { - // it is trivial to run a cell that does not exist -- nothing needs to be done. - return; - } - - const cell_type = cell.get("cell_type", "code"); - if (cell_type == "code") { - const code = this.get_cell_input(id).trim(); - if (!code) { - this.clear_cell(id, save); - return; - } - this.runCells([id], { noHalt }); - //this.run_code_cell(id, save, no_halt); - if (save) { - this.save_asap(); - } - } - } - private async api_call_formatter( str: string, config: FormatterConfig, @@ -1540,7 +1513,7 @@ export class JupyterActions extends JupyterActions0 { private jupyterClient?; private runQueue: any[] = []; private runningNow = false; - runCells = async (ids: string[], opts: { noHalt?: boolean } = {}) => { + async runCells(ids: string[], opts: { noHalt?: boolean } = {}) { if (this.store?.get("read_only")) { return; } @@ -1571,6 +1544,9 @@ export class JupyterActions extends JupyterActions0 { for (const id of ids) { const cell = this.store.getIn(["cells", id])?.toJS(); + if (cell?.cell_type != "code") { + continue; + } if (!cell?.input?.trim()) { // nothing to do continue; @@ -1591,7 +1567,7 @@ export class JupyterActions extends JupyterActions0 { } cells.push(cell); } - this.addPendingCells(ids); + this.addPendingCells(cells.map(({ id }) => id)); // ensures cells run in order: cells.sort(field_cmp("pos")); @@ -1625,7 +1601,7 @@ export class JupyterActions extends JupyterActions0 { handler?.done(); this.save_asap(); } catch (err) { - console.log("runCells", err); + console.warn("runCells", err); } finally { this.runningNow = false; if (this.runQueue.length > 0) { @@ -1633,5 +1609,5 @@ export class JupyterActions extends JupyterActions0 { this.runCells(ids, opts); } } - }; + } } diff --git a/src/packages/frontend/jupyter/cell-buttonbar.tsx b/src/packages/frontend/jupyter/cell-buttonbar.tsx index 3d6c91afd4..47afd9f011 100644 --- a/src/packages/frontend/jupyter/cell-buttonbar.tsx +++ b/src/packages/frontend/jupyter/cell-buttonbar.tsx @@ -103,7 +103,7 @@ export const CellButtonBar: React.FC = React.memo( tooltip: "Run this cell", label: "Run", icon: "step-forward", - onClick: () => actions?.run_cell(id), + onClick: () => actions?.runCells([id]), }; } } diff --git a/src/packages/frontend/jupyter/output-messages/ipywidget.tsx b/src/packages/frontend/jupyter/output-messages/ipywidget.tsx index b0c1a6f50b..d0815ffcad 100644 --- a/src/packages/frontend/jupyter/output-messages/ipywidget.tsx +++ b/src/packages/frontend/jupyter/output-messages/ipywidget.tsx @@ -146,7 +146,7 @@ ax.plot(x, y) - ); - case "clear": - return ( - - ); - } - } - function render_currently_selected(): React.JSX.Element | undefined { if (props.listing.length === 0) { return; @@ -235,7 +177,6 @@ export function ActionBar(props: Props) { )} - {render_select_entire_directory()} ); } diff --git a/src/packages/frontend/project/explorer/explorer.tsx b/src/packages/frontend/project/explorer/explorer.tsx index 41bfcd0313..e200d507d2 100644 --- a/src/packages/frontend/project/explorer/explorer.tsx +++ b/src/packages/frontend/project/explorer/explorer.tsx @@ -19,7 +19,6 @@ import { ShallowTypedMap } from "@cocalc/frontend/app-framework/ShallowTypedMap" import { A, ActivityDisplay, - ErrorDisplay, Loading, SettingBox, } from "@cocalc/frontend/components"; @@ -38,7 +37,6 @@ import { ProjectActions } from "@cocalc/frontend/project_store"; import { ProjectMap, ProjectStatus } from "@cocalc/frontend/todo-types"; import AskNewFilename from "../ask-filename"; import { useProjectContext } from "../context"; -import { AccessErrors } from "./access-errors"; import { ActionBar } from "./action-bar"; import { ActionBox } from "./action-box"; import { FileListing } from "./file-listing"; @@ -48,6 +46,7 @@ import { NewButton } from "./new-button"; import { PathNavigator } from "./path-navigator"; import { SearchBar } from "./search-bar"; import ExplorerTour from "./tour/tour"; +import ShowError from "@cocalc/frontend/components/error"; export type Configuration = ShallowTypedMap<{ main: MainConfiguration }>; @@ -79,7 +78,6 @@ interface ReduxProps { current_path: string; history_path: string; activity?: object; - page_number: number; file_action?: | "compress" | "delete" @@ -162,7 +160,6 @@ const Explorer0 = rclass( current_path: rtypes.string, history_path: rtypes.string, activity: rtypes.object, - page_number: rtypes.number.isRequired, file_action: rtypes.string, file_search: rtypes.string, show_hidden: rtypes.bool, @@ -186,7 +183,6 @@ const Explorer0 = rclass( }; static defaultProps = { - page_number: 0, file_search: "", new_name: "", redux, @@ -225,20 +221,6 @@ const Explorer0 = rclass( } }; - previous_page = () => { - if (this.props.page_number > 0) { - this.props.actions.setState({ - page_number: this.props.page_number - 1, - }); - } - }; - - next_page = () => { - this.props.actions.setState({ - page_number: this.props.page_number + 1, - }); - }; - create_file = (ext, switch_over) => { if (switch_over == undefined) { switch_over = true; @@ -265,7 +247,7 @@ const Explorer0 = rclass( current_path: this.props.current_path, switch_over, }); - this.props.actions.setState({ file_search: "", page_number: 0 }); + this.props.actions.setState({ file_search: "" }); }; create_folder = (switch_over = true): void => { @@ -274,151 +256,9 @@ const Explorer0 = rclass( current_path: this.props.current_path, switch_over, }); - this.props.actions.setState({ file_search: "", page_number: 0 }); + this.props.actions.setState({ file_search: "" }); }; - render_files_action_box() { - return ( - - - - ); - } - - render_library() { - return ( - - - - Library{" "} - - (help...) - - - } - close={() => this.props.actions.toggle_library(false)} - > - this.props.actions.toggle_library(false)} - /> - - - - ); - } - - render_files_actions(project_is_running) { - return ( - - ); - } - - render_new_file() { - return ( -
- -
- ); - } - - render_activity() { - return ( - this.props.actions.clear_all_activity()} - style={{ top: "100px" }} - /> - ); - } - - render_error() { - if (this.props.error) { - return ( - this.props.actions.setState({ error: "" })} - /> - ); - } - } - - render_access_error() { - return ; - } - - render_file_listing() { - return ( - - - - ); - } - file_listing_page_size() { return ( this.props.other_settings && @@ -426,135 +266,6 @@ const Explorer0 = rclass( ); } - render_control_row(): React.JSX.Element { - return ( -
-
-
- -
- -
-
- {!!this.props.compute_server_id && ( -
- -
- )} -
- {!IS_MOBILE && ( -
- {this.render_new_file()} -
- )} - {!IS_MOBILE && ( - - )} -
- -
-
- ); - } - - render_project_files_buttons(): React.JSX.Element { - return ( -
- -
- ); - } - - render_custom_software_reset() { - if (!this.props.show_custom_software_reset) { - return undefined; - } - // also don't show this box, if any files are selected - if (this.props.checked_files.size > 0) { - return undefined; - } - return ( - - ); - } - render() { let project_is_running: boolean, project_state: ProjectStatus | undefined; @@ -599,9 +310,109 @@ const Explorer0 = rclass( padding: "2px 2px 0 2px", }} > - {this.render_error()} - {this.render_activity()} - {this.render_control_row()} + this.props.actions.setState({ error })} + /> + this.props.actions.clear_all_activity()} + style={{ top: "100px" }} + /> +
+
+
+ +
+ +
+
+ {!!this.props.compute_server_id && ( +
+ +
+ )} +
+ {!IS_MOBILE && ( +
+
+ +
+
+ )} + {!IS_MOBILE && ( +
+ +
+ )} +
+ +
+
+ {this.props.ext_selection != null && ( )} @@ -613,20 +424,101 @@ const Explorer0 = rclass( minWidth: "20em", }} > - {this.render_files_actions(project_is_running)} + + +
+
- {this.render_project_files_buttons()} - {project_is_running - ? this.render_custom_software_reset() - : undefined} - - {this.props.show_library ? this.render_library() : undefined} + {project_is_running && + this.props.show_custom_software_reset && + this.props.checked_files.size == 0 && ( + + )} + + {this.props.show_library && ( + + + + Library{" "} + + (help...) + + + } + close={() => this.props.actions.toggle_library(false)} + > + this.props.actions.toggle_library(false)} + /> + + + + )} {this.props.checked_files.size > 0 && this.props.file_action != undefined ? ( - {this.render_files_action_box()} + + + + + ) : undefined} @@ -640,7 +532,39 @@ const Explorer0 = rclass( padding: "0 5px 5px 5px", }} > - {this.render_file_listing()} + + + ; - current_path: string; - file_search: string; - actions: ProjectActions; - file_creation_error?: string; - create_file: (ext?: string, switch_over?: boolean) => void; - create_folder: (switch_over?: boolean) => void; - listingRef; - }, - ref: React.LegacyRef | undefined, - ) => { - return ( -
- -
- ); - }, -); diff --git a/src/packages/frontend/project/listing/use-files.ts b/src/packages/frontend/project/listing/use-files.ts index fa7dd3f9d8..6e925e817d 100644 --- a/src/packages/frontend/project/listing/use-files.ts +++ b/src/packages/frontend/project/listing/use-files.ts @@ -170,7 +170,6 @@ async function cacheNeighbors({ v.push(parent); } } - const t = Date.now(); const f = async (path: string) => { await ensureCached({ cacheId, fs, path }); }; @@ -178,5 +177,4 @@ async function cacheNeighbors({ // grab up to MAX_SUBDIR_CACHE missing listings in parallel v = v.slice(0, MAX_SUBDIR_CACHE); await Promise.all(v.map(f)); - console.log(Date.now() - t, v); } diff --git a/src/packages/frontend/project_actions.ts b/src/packages/frontend/project_actions.ts index d13880bd2a..285e6e4f06 100644 --- a/src/packages/frontend/project_actions.ts +++ b/src/packages/frontend/project_actions.ts @@ -1500,7 +1500,6 @@ export class ProjectActions extends Actions { this.setState({ current_path: path, history_path, - page_number: 0, most_recent_file_click: undefined, }); }; @@ -1552,7 +1551,6 @@ export class ProjectActions extends Actions { set_file_search(search): void { this.setState({ file_search: search, - page_number: 0, file_action: undefined, most_recent_file_click: undefined, create_file_alert: false, diff --git a/src/packages/frontend/project_store.ts b/src/packages/frontend/project_store.ts index 6499c91853..388a9fb74a 100644 --- a/src/packages/frontend/project_store.ts +++ b/src/packages/frontend/project_store.ts @@ -93,7 +93,6 @@ export interface ProjectStoreState { // Project Files activity: any; // immutable, active_file_sort: TypedMap<{ column_name: string; is_descending: boolean }>; - page_number: number; file_action?: string; // undefineds is meaningfully none here file_search?: string; show_hidden?: boolean; @@ -310,7 +309,6 @@ export class ProjectStore extends Store { // Project Files activity: undefined, - page_number: 0, checked_files: immutable.Set(), show_library: false, file_listing_scroll_top: undefined, From 397dde1ae1d9b47fba55baeaea0307a609550570 Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 29 Jul 2025 00:19:45 +0000 Subject: [PATCH 122/270] move explorer keyboard handling to the explorer itself - fixes bug where if the filter looses focus you can't navigate or select - better approach anyways (more direct) --- .../frontend/project/explorer/explorer.tsx | 38 +++++++++++++++---- .../frontend/project/explorer/search-bar.tsx | 32 ---------------- 2 files changed, 30 insertions(+), 40 deletions(-) diff --git a/src/packages/frontend/project/explorer/explorer.tsx b/src/packages/frontend/project/explorer/explorer.tsx index e200d507d2..b3976a60d2 100644 --- a/src/packages/frontend/project/explorer/explorer.tsx +++ b/src/packages/frontend/project/explorer/explorer.tsx @@ -47,6 +47,14 @@ import { PathNavigator } from "./path-navigator"; import { SearchBar } from "./search-bar"; import ExplorerTour from "./tour/tour"; import ShowError from "@cocalc/frontend/components/error"; +import { join } from "path"; + +const FLEX_ROW_STYLE = { + display: "flex", + flexFlow: "row wrap", + justifyContent: "space-between", + alignItems: "stretch", +} as const; export type Configuration = ShallowTypedMap<{ main: MainConfiguration }>; @@ -105,6 +113,7 @@ interface ReduxProps { show_custom_software_reset?: boolean; explorerTour?: boolean; compute_server_id: number; + selected_file_index?: number; } interface State { @@ -178,6 +187,7 @@ const Explorer0 = rclass( show_custom_software_reset: rtypes.bool, explorerTour: rtypes.bool, compute_server_id: rtypes.number, + selected_file_index: rtypes.number, }, }; }; @@ -212,6 +222,26 @@ const Explorer0 = rclass( handle_files_key_down = (e): void => { if (e.key === "Shift") { this.setState({ shift_is_down: true }); + } else if (e.key == "ArrowUp") { + this.props.actions.decrement_selected_file_index(); + } else if (e.key == "ArrowDown") { + this.props.actions.increment_selected_file_index(); + } else if (e.key == "Enter") { + const x = + this.listingRef.current?.[this.props.selected_file_index ?? 0]; + if (x != null) { + const { isdir, name } = x; + const path = join(this.props.current_path, name); + if (isdir) { + this.props.actions.open_directory(path); + } else { + this.props.actions.open_file({ path, foreground: !e.ctrlKey }); + } + if (!e.ctrlKey) { + this.props.actions.set_file_search(""); + this.props.actions.clear_selected_file_index(); + } + } } }; @@ -292,13 +322,6 @@ const Explorer0 = rclass( project_is_running = false; } - const FLEX_ROW_STYLE = { - display: "flex", - flexFlow: "row wrap", - justifyContent: "space-between", - alignItems: "stretch", - }; - // be careful with adding height:'100%'. it could cause flex to miscalculate. see #3904 return (
@@ -400,7 +423,6 @@ const Explorer0 = rclass( file_creation_error={this.props.file_creation_error} create_file={this.create_file} create_folder={this.create_folder} - listingRef={this.listingRef} />
)} diff --git a/src/packages/frontend/project/explorer/search-bar.tsx b/src/packages/frontend/project/explorer/search-bar.tsx index ebe746a307..7070669954 100644 --- a/src/packages/frontend/project/explorer/search-bar.tsx +++ b/src/packages/frontend/project/explorer/search-bar.tsx @@ -14,7 +14,6 @@ import { useProjectContext } from "../context"; import { TERM_MODE_CHAR } from "./file-listing"; import { TerminalModeDisplay } from "@cocalc/frontend/project/explorer/file-listing/terminal-mode-display"; import { useTypedRedux } from "@cocalc/frontend/app-framework"; -import { join } from "path"; const HelpStyle = { wordWrap: "break-word", @@ -50,7 +49,6 @@ interface Props { file_creation_error?: string; disabled?: boolean; ext_selection?: string; - listingRef; } // Commands such as CD throw a setState error. @@ -65,14 +63,11 @@ export const SearchBar = memo( file_creation_error, disabled = false, ext_selection, - listingRef, }: Props) => { const intl = useIntl(); const { project_id } = useProjectContext(); const numDisplayedFiles = useTypedRedux({ project_id }, "numDisplayedFiles") ?? 0; - const selected_file_index = - useTypedRedux({ project_id }, "selected_file_index") ?? 0; // edit → run → edit // TODO use "state" to show a progress spinner while a command is running @@ -241,19 +236,6 @@ export const SearchBar = memo( if (value.startsWith(TERM_MODE_CHAR)) { const command = value.slice(1, value.length); execute_command(command); - } else if (listingRef.current?.[selected_file_index] != null) { - const { isdir, name } = listingRef.current[selected_file_index]; - const path = join(current_path, name); - if (isdir) { - actions.open_directory(path); - } else { - actions.open_file({ path, foreground: !ctrl_down }); - } - if (!ctrl_down) { - actions.set_file_search(""); - actions.clear_selected_file_index(); - } - return; } else if (file_search.length > 0 && shift_down) { // only create a file, if shift is pressed as well to avoid creating // jupyter notebooks (default file-type) by accident. @@ -266,18 +248,6 @@ export const SearchBar = memo( } } - function on_up_press(): void { - if (selected_file_index > 0) { - actions.decrement_selected_file_index(); - } - } - - function on_down_press(): void { - if (selected_file_index < numDisplayedFiles - 1) { - actions.increment_selected_file_index(); - } - } - function on_change(search: string): void { actions.zero_selected_file_index(); actions.set_file_search(search); @@ -302,8 +272,6 @@ export const SearchBar = memo( value={file_search} on_change={on_change} on_submit={search_submit} - on_up={on_up_press} - on_down={on_down_press} on_clear={on_clear} disabled={disabled || !!ext_selection} /> From ecc14de58934aafcf01bccd4f43773a1d5e50365 Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 29 Jul 2025 00:25:03 +0000 Subject: [PATCH 123/270] explorer: support [*]-up arrow to move to parent directory - noticed this is missing and obviously expected (and trivial) --- src/packages/frontend/project/explorer/explorer.tsx | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/packages/frontend/project/explorer/explorer.tsx b/src/packages/frontend/project/explorer/explorer.tsx index b3976a60d2..58857e1f8e 100644 --- a/src/packages/frontend/project/explorer/explorer.tsx +++ b/src/packages/frontend/project/explorer/explorer.tsx @@ -47,7 +47,7 @@ import { PathNavigator } from "./path-navigator"; import { SearchBar } from "./search-bar"; import ExplorerTour from "./tour/tour"; import ShowError from "@cocalc/frontend/components/error"; -import { join } from "path"; +import { dirname, join } from "path"; const FLEX_ROW_STYLE = { display: "flex", @@ -223,7 +223,12 @@ const Explorer0 = rclass( if (e.key === "Shift") { this.setState({ shift_is_down: true }); } else if (e.key == "ArrowUp") { - this.props.actions.decrement_selected_file_index(); + if (e.shiftKey || e.ctrlKey || e.metaKey) { + const path = dirname(this.props.current_path); + this.props.actions.open_directory(path == "." ? "" : path); + } else { + this.props.actions.decrement_selected_file_index(); + } } else if (e.key == "ArrowDown") { this.props.actions.increment_selected_file_index(); } else if (e.key == "Enter") { From 145ce9a615094e6b94b8c42da1f91728c0568e2c Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 29 Jul 2025 01:45:12 +0000 Subject: [PATCH 124/270] rewrite file explorer to be a functional component --- .../frontend/account/other-settings.tsx | 19 - .../frontend/custom-software/reset-bar.tsx | 10 +- .../frontend/project/explorer/action-bar.tsx | 6 +- .../frontend/project/explorer/action-box.tsx | 113 +-- .../frontend/project/explorer/explorer.tsx | 872 +++++++----------- .../explorer/file-listing/file-checkbox.tsx | 42 +- .../explorer/file-listing/file-listing.tsx | 30 +- .../explorer/file-listing/listing-header.tsx | 8 +- .../project/explorer/file-listing/utils.ts | 6 +- .../project/explorer/misc-side-buttons.tsx | 69 +- .../frontend/project/explorer/new-button.tsx | 22 +- .../project/explorer/path-navigator.tsx | 13 +- .../project/explorer/path-segment-link.tsx | 22 +- .../frontend/project/explorer/tour/tour.tsx | 5 +- .../project/listing/filter-listing.ts | 5 + 15 files changed, 520 insertions(+), 722 deletions(-) diff --git a/src/packages/frontend/account/other-settings.tsx b/src/packages/frontend/account/other-settings.tsx index bdf6fcb0ca..dba71ce509 100644 --- a/src/packages/frontend/account/other-settings.tsx +++ b/src/packages/frontend/account/other-settings.tsx @@ -320,24 +320,6 @@ export function OtherSettings(props: Readonly): React.JSX.Element { ); } - function render_page_size(): Rendered { - return ( - - on_change("page_size", n)} - min={1} - max={10000} - number={props.other_settings.get("page_size")} - /> - - ); - } - function render_no_free_warnings(): Rendered { let extra; if (!props.is_stripe_customer) { @@ -714,7 +696,6 @@ export function OtherSettings(props: Readonly): React.JSX.Element { {render_vertical_fixed_bar_options()} {render_new_filenames()} {render_default_file_sort()} - {render_page_size()} {render_standby_timeout()}
diff --git a/src/packages/frontend/custom-software/reset-bar.tsx b/src/packages/frontend/custom-software/reset-bar.tsx index 2442a38bbe..445b947900 100644 --- a/src/packages/frontend/custom-software/reset-bar.tsx +++ b/src/packages/frontend/custom-software/reset-bar.tsx @@ -6,15 +6,15 @@ import { Button as AntdButton, Card } from "antd"; import React from "react"; import { FormattedMessage, useIntl } from "react-intl"; - import { A, Icon } from "@cocalc/frontend/components"; import { labels } from "@cocalc/frontend/i18n"; import { ProjectMap } from "@cocalc/frontend/todo-types"; import { COLORS, SITE_NAME } from "@cocalc/util/theme"; - import { Available as AvailableFeatures } from "../project_configuration"; import { ComputeImages } from "./init"; import { props2img, RESET_ICON } from "./util"; +import { useTypedRedux } from "@cocalc/frontend/app-framework"; +import { type ProjectActions } from "@cocalc/frontend/project_store"; const doc_snap = "https://doc.cocalc.com/project-files.html#snapshots"; const doc_tt = "https://doc.cocalc.com/time-travel.html"; @@ -36,13 +36,13 @@ interface Props { project_id: string; images: ComputeImages; project_map?: ProjectMap; - actions: any; + actions: ProjectActions; available_features?: AvailableFeatures; - site_name?: string; } export const CustomSoftwareReset: React.FC = (props: Props) => { - const { actions, site_name } = props; + const { actions } = props; + const site_name = useTypedRedux("customize", "site_name"); const intl = useIntl(); diff --git a/src/packages/frontend/project/explorer/action-bar.tsx b/src/packages/frontend/project/explorer/action-bar.tsx index 7c7407e713..fa82fc17df 100644 --- a/src/packages/frontend/project/explorer/action-bar.tsx +++ b/src/packages/frontend/project/explorer/action-bar.tsx @@ -12,10 +12,10 @@ import { Button, ButtonToolbar } from "@cocalc/frontend/antd-bootstrap"; import { Gap, Icon } from "@cocalc/frontend/components"; import { useStudentProjectFunctionality } from "@cocalc/frontend/course"; import { CustomSoftwareInfo } from "@cocalc/frontend/custom-software/info-bar"; -import { ComputeImages } from "@cocalc/frontend/custom-software/init"; +import { type ComputeImages } from "@cocalc/frontend/custom-software/init"; import { IS_MOBILE } from "@cocalc/frontend/feature"; import { labels } from "@cocalc/frontend/i18n"; -import { file_actions, ProjectActions } from "@cocalc/frontend/project_store"; +import { file_actions, type ProjectActions } from "@cocalc/frontend/project_store"; import * as misc from "@cocalc/util/misc"; import { COLORS } from "@cocalc/util/theme"; import { useProjectContext } from "../context"; @@ -31,7 +31,7 @@ interface Props { checked_files: immutable.Set; listing: { name: string; isdir: boolean }[]; current_path?: string; - project_map?: immutable.Map; + project_map?; images?: ComputeImages; actions: ProjectActions; available_features?; diff --git a/src/packages/frontend/project/explorer/action-box.tsx b/src/packages/frontend/project/explorer/action-box.tsx index d52c5e9e18..b5a5df0f19 100644 --- a/src/packages/frontend/project/explorer/action-box.tsx +++ b/src/packages/frontend/project/explorer/action-box.tsx @@ -9,7 +9,6 @@ import { Button as AntdButton, Radio, Space } from "antd"; import * as immutable from "immutable"; import { useState } from "react"; import { useIntl } from "react-intl"; - import { Alert, Button, @@ -28,7 +27,6 @@ import { SelectProject } from "@cocalc/frontend/projects/select-project"; import ConfigureShare from "@cocalc/frontend/share/config"; import * as misc from "@cocalc/util/misc"; import { COLORS } from "@cocalc/util/theme"; -import { useProjectContext } from "../context"; import DirectorySelector from "../directory-selector"; import { in_snapshot_path } from "../utils"; import CreateArchive from "./create-archive"; @@ -56,13 +54,17 @@ interface ReactProps { file_map: object; actions: ProjectActions; displayed_listing?: object; - //new_name?: string; - name: string; } -export function ActionBox(props: ReactProps) { +export function ActionBox({ + checked_files, + file_action, + current_path, + project_id, + file_map, + actions, +}: ReactProps) { const intl = useIntl(); - const { project_id } = useProjectContext(); const runQuota = useRunQuota(project_id, null); const get_user_type: () => string = useRedux("account", "get_user_type"); const compute_server_id = useTypedRedux({ project_id }, "compute_server_id"); @@ -74,7 +76,6 @@ export function ActionBox(props: ReactProps) { const [copy_from_compute_server_to, set_copy_from_compute_server_to] = useState<"compute-server" | "project">("compute-server"); const [move_destination, set_move_destination] = useState(""); - //const [new_name, set_new_name] = useState(props.new_name ?? ""); const [show_different_project, set_show_different_project] = useState(false); const [overwrite_newer, set_overwrite_newer] = useState(); @@ -84,7 +85,7 @@ export function ActionBox(props: ReactProps) { ); function cancel_action(): void { - props.actions.set_file_action(); + actions.set_file_action(); } function action_key(e): void { @@ -93,7 +94,7 @@ export function ActionBox(props: ReactProps) { cancel_action(); break; case 13: - switch (props.file_action) { + switch (file_action) { case "move": submit_action_move(); break; @@ -107,7 +108,7 @@ export function ActionBox(props: ReactProps) { function render_selected_files_list(): React.JSX.Element { return ( 
-        {props.checked_files.toArray().map((name) => (
+        {checked_files.toArray().map((name) => (
           
{misc.path_split(name).tail}

         ))}
@@ -115,17 +116,17 @@ export function ActionBox(props: ReactProps) { } function delete_click(): void { - const paths = props.checked_files.toArray(); + const paths = checked_files.toArray(); for (const path of paths) { - props.actions.close_tab(path); + actions.close_tab(path); } - props.actions.delete_files({ paths }); - props.actions.set_file_action(); - props.actions.set_all_files_unchecked(); + actions.delete_files({ paths }); + actions.set_file_action(); + actions.set_all_files_unchecked(); } function render_delete_warning(): React.JSX.Element | undefined { - if (props.current_path === ".trash") { + if (current_path === ".trash") { return ( @@ -140,7 +141,7 @@ export function ActionBox(props: ReactProps) { } function render_delete(): React.JSX.Element | undefined { - const { size } = props.checked_files; + const { size } = checked_files; return (
@@ -161,7 +162,7 @@ export function ActionBox(props: ReactProps) { href="" onClick={(e) => { e.preventDefault(); - props.actions.open_directory(".snapshots"); + actions.open_directory(".snapshots"); }} > ~/.snapshots @@ -178,7 +179,7 @@ export function ActionBox(props: ReactProps) { Delete {size} {misc.plural(size, "Item")} @@ -190,16 +191,16 @@ export function ActionBox(props: ReactProps) { } function move_click(): void { - props.actions.move_files({ - src: props.checked_files.toArray(), + actions.move_files({ + src: checked_files.toArray(), dest: move_destination, }); - props.actions.set_file_action(); - props.actions.set_all_files_unchecked(); + actions.set_file_action(); + actions.set_all_files_unchecked(); } function valid_move_input(): boolean { - const src_path = misc.path_split(props.checked_files.first()).head; + const src_path = misc.path_split(checked_files.first()).head; let dest = move_destination.trim(); if (dest === src_path) { return false; @@ -210,11 +211,11 @@ export function ActionBox(props: ReactProps) { if (dest.charAt(dest.length - 1) === "/") { dest = dest.slice(0, dest.length - 1); } - return dest !== props.current_path; + return dest !== current_path; } function render_move(): React.JSX.Element { - const { size } = props.checked_files; + const { size } = checked_files; return (
@@ -243,9 +244,9 @@ export function ActionBox(props: ReactProps) { onSelect={(move_destination: string) => set_move_destination(move_destination) } - project_id={props.project_id} - startingPath={props.current_path} - isExcluded={(path) => props.checked_files.has(path)} + project_id={project_id} + startingPath={current_path} + isExcluded={(path) => checked_files.has(path)} style={{ width: "100%" }} bodyStyle={{ maxHeight: "250px" }} /> @@ -267,7 +268,7 @@ export function ActionBox(props: ReactProps) {

Target Project

set_copy_destination_project_id(copy_destination_project_id) @@ -279,8 +280,10 @@ export function ActionBox(props: ReactProps) { } } - function render_copy_different_project_options(): React.JSX.Element | undefined { - if (props.project_id !== copy_destination_project_id) { + function render_copy_different_project_options(): + | React.JSX.Element + | undefined { + if (project_id !== copy_destination_project_id) { return (
@@ -430,7 +433,7 @@ export function ActionBox(props: ReactProps) { } function render_copy(): React.JSX.Element { - const { size } = props.checked_files; + const { size } = checked_files; const signed_in = get_user_type() === "signed_in"; if (!signed_in) { return ( @@ -498,7 +501,7 @@ export function ActionBox(props: ReactProps) {
set_dest_compute_server_id(dest_compute_server_id) @@ -513,7 +516,7 @@ export function ActionBox(props: ReactProps) { set_copy_destination_directory(value) } key="copy_destination_directory" - startingPath={props.current_path} + startingPath={current_path} project_id={copy_destination_project_id} style={{ width: "100%" }} bodyStyle={{ maxHeight: "250px" }} @@ -540,18 +543,18 @@ export function ActionBox(props: ReactProps) { function render_share(): React.JSX.Element { // currently only works for a single selected file - const path: string = props.checked_files.first() ?? ""; + const path: string = checked_files.first() ?? ""; if (!path) { return <>; } - const public_data = props.file_map[misc.path_split(path).tail]; + const public_data = file_map[misc.path_split(path).tail]; if (public_data == undefined) { // directory listing not loaded yet... (will get re-rendered when loaded) return ; } return ( props.actions.set_public_path(path, opts)} + set_public_path={(opts) => actions.set_public_path(path, opts)} has_network_access={!!runQuota.network} /> ); } - function render_action_box(action: FileAction): React.JSX.Element | undefined { + function render_action_box( + action: FileAction, + ): React.JSX.Element | undefined { switch (action) { case "compress": return ; @@ -590,12 +595,12 @@ export function ActionBox(props: ReactProps) { } } - const action = props.file_action; + const action = file_action; const action_button = file_actions[action || "undefined"]; if (action_button == undefined) { return
Undefined action
; } - if (props.file_map == undefined) { + if (file_map == undefined) { return ; } else { return ( diff --git a/src/packages/frontend/project/explorer/explorer.tsx b/src/packages/frontend/project/explorer/explorer.tsx index 58857e1f8e..50402c84dc 100644 --- a/src/packages/frontend/project/explorer/explorer.tsx +++ b/src/packages/frontend/project/explorer/explorer.tsx @@ -3,40 +3,26 @@ * License: MS-RSL – see LICENSE.md for details */ -import * as immutable from "immutable"; import * as _ from "lodash"; -import React from "react"; import { UsersViewing } from "@cocalc/frontend/account/avatar/users-viewing"; import { Col, Row } from "@cocalc/frontend/antd-bootstrap"; -import { - project_redux_name, - rclass, - redux, - rtypes, - TypedMap, -} from "@cocalc/frontend/app-framework"; -import { ShallowTypedMap } from "@cocalc/frontend/app-framework/ShallowTypedMap"; +import { type CSSProperties, useEffect, useRef, useState } from "react"; import { A, ActivityDisplay, + ErrorDisplay, Loading, SettingBox, } from "@cocalc/frontend/components"; import { ComputeServerDocStatus } from "@cocalc/frontend/compute/doc-status"; import SelectComputeServerForFileExplorer from "@cocalc/frontend/compute/select-server-for-explorer"; -import { ComputeImages } from "@cocalc/frontend/custom-software/init"; import { CustomSoftwareReset } from "@cocalc/frontend/custom-software/reset-bar"; import { IS_MOBILE } from "@cocalc/frontend/feature"; import { FileUploadWrapper } from "@cocalc/frontend/file-upload"; import { Library } from "@cocalc/frontend/library"; -import { - Available, - MainConfiguration, -} from "@cocalc/frontend/project_configuration"; -import { ProjectActions } from "@cocalc/frontend/project_store"; -import { ProjectMap, ProjectStatus } from "@cocalc/frontend/todo-types"; +import { ProjectStatus } from "@cocalc/frontend/todo-types"; import AskNewFilename from "../ask-filename"; -import { useProjectContext } from "../context"; +import { useProjectContext } from "@cocalc/frontend/project/context"; import { ActionBar } from "./action-bar"; import { ActionBox } from "./action-box"; import { FileListing } from "./file-listing"; @@ -46,8 +32,8 @@ import { NewButton } from "./new-button"; import { PathNavigator } from "./path-navigator"; import { SearchBar } from "./search-bar"; import ExplorerTour from "./tour/tour"; -import ShowError from "@cocalc/frontend/components/error"; import { dirname, join } from "path"; +import { redux, useTypedRedux } from "@cocalc/frontend/app-framework"; const FLEX_ROW_STYLE = { display: "flex", @@ -56,9 +42,7 @@ const FLEX_ROW_STYLE = { alignItems: "stretch", } as const; -export type Configuration = ShallowTypedMap<{ main: MainConfiguration }>; - -const error_style: React.CSSProperties = { +const ERROR_STYLE: CSSProperties = { marginRight: "1ex", whiteSpace: "pre-line", position: "absolute", @@ -67,543 +51,393 @@ const error_style: React.CSSProperties = { boxShadow: "5px 5px 5px grey", } as const; -interface ReactProps { - project_id: string; - actions: ProjectActions; - name: string; -} - -interface ReduxProps { - project_map?: ProjectMap; - get_my_group: (project_id: string) => "admin" | "public"; - get_total_project_quotas: (project_id: string) => { member_host: boolean }; - other_settings?: immutable.Map; - is_logged_in?: boolean; - kucalc?: string; - site_name?: string; - images: ComputeImages; - active_file_sort: TypedMap<{ column_name: string; is_descending: boolean }>; - current_path: string; - history_path: string; - activity?: object; - file_action?: - | "compress" - | "delete" - | "rename" - | "duplicate" - | "move" - | "copy" - | "share" - | "download" - | "upload"; - file_search: string; - show_hidden?: boolean; - show_masked?: boolean; - error?: string; - checked_files: immutable.Set; - file_creation_error?: string; - ext_selection?: string; - new_name?: string; - library?: object; - show_library?: boolean; - public_paths?: immutable.List; // used only to trigger table init - configuration?: Configuration; - available_features?: Available; - file_listing_scroll_top?: number; - show_custom_software_reset?: boolean; - explorerTour?: boolean; - compute_server_id: number; - selected_file_index?: number; -} - -interface State { - shift_is_down: boolean; -} - export function Explorer() { - const { project_id } = useProjectContext(); - return ( - + const { actions, project_id } = useProjectContext(); + const newFileRef = useRef(null); + const searchAndTerminalBar = useRef(null); + const fileListingRef = useRef(null); + const currentDirectoryRef = useRef(null); + const miscButtonsRef = useRef(null); + const listingRef = useRef(null); + + const activity = useTypedRedux({ project_id }, "activity")?.toJS(); + const available_features = useTypedRedux( + { project_id }, + "available_features", + )?.toJS(); + const checked_files = useTypedRedux({ project_id }, "checked_files"); + const compute_server_id = useTypedRedux({ project_id }, "compute_server_id"); + const configuration = useTypedRedux({ project_id }, "configuration"); + const current_path = useTypedRedux({ project_id }, "current_path"); + const error = useTypedRedux({ project_id }, "error"); + const ext_selection = useTypedRedux({ project_id }, "ext_selection"); + const file_action = useTypedRedux({ project_id }, "file_action"); + const file_creation_error = useTypedRedux( + { project_id }, + "file_creation_error", ); -} - -// TODO: change/rewrite Explorer to not have any rtypes.objects and -// add a shouldComponentUpdate!! -const Explorer0 = rclass( - class Explorer extends React.Component { - newFileRef = React.createRef(); - searchAndTerminalBar = React.createRef(); - fileListingRef = React.createRef(); - currentDirectoryRef = React.createRef(); - miscButtonsRef = React.createRef(); - listingRef = React.createRef(); - - static reduxProps = ({ name }) => { - return { - projects: { - project_map: rtypes.immutable.Map, - get_my_group: rtypes.func.isRequired, - get_total_project_quotas: rtypes.func.isRequired, - }, - - account: { - other_settings: rtypes.immutable.Map, - is_logged_in: rtypes.bool, - }, - - customize: { - kucalc: rtypes.string, - site_name: rtypes.string, - }, - - compute_images: { - images: rtypes.immutable.Map, - }, - - [name]: { - active_file_sort: rtypes.immutable.Map, - current_path: rtypes.string, - history_path: rtypes.string, - activity: rtypes.object, - file_action: rtypes.string, - file_search: rtypes.string, - show_hidden: rtypes.bool, - show_masked: rtypes.bool, - error: rtypes.string, - checked_files: rtypes.immutable, - file_creation_error: rtypes.string, - ext_selection: rtypes.string, - new_name: rtypes.string, - library: rtypes.object, - show_library: rtypes.bool, - public_paths: rtypes.immutable, // used only to trigger table init - configuration: rtypes.immutable, - available_features: rtypes.object, - file_listing_scroll_top: rtypes.number, - show_custom_software_reset: rtypes.bool, - explorerTour: rtypes.bool, - compute_server_id: rtypes.number, - selected_file_index: rtypes.number, - }, - }; - }; - - static defaultProps = { - file_search: "", - new_name: "", - redux, - }; - - constructor(props) { - super(props); - this.state = { - shift_is_down: false, - }; - } + const file_search = useTypedRedux({ project_id }, "file_search"); + const selected_file_index = useTypedRedux( + { project_id }, + "selected_file_index", + ); + const show_custom_software_reset = useTypedRedux( + { project_id }, + "show_custom_software_reset", + ); + const show_library = useTypedRedux({ project_id }, "show_library"); + const [shiftIsDown, setShiftIsDown] = useState(false); - componentDidMount() { - // Update AFTER react draws everything - // Should probably be moved elsewhere - // Prevents cascading changes which impact responsiveness - // https://github.com/sagemathinc/cocalc/pull/3705#discussion_r268263750 - $(window).on("keydown", this.handle_files_key_down); - $(window).on("keyup", this.handle_files_key_up); - } + const project_map = useTypedRedux("projects", "project_map"); - componentWillUnmount() { - $(window).off("keydown", this.handle_files_key_down); - $(window).off("keyup", this.handle_files_key_up); - } + const images = useTypedRedux("compute_images", "images"); - handle_files_key_down = (e): void => { - if (e.key === "Shift") { - this.setState({ shift_is_down: true }); - } else if (e.key == "ArrowUp") { - if (e.shiftKey || e.ctrlKey || e.metaKey) { - const path = dirname(this.props.current_path); - this.props.actions.open_directory(path == "." ? "" : path); - } else { - this.props.actions.decrement_selected_file_index(); - } - } else if (e.key == "ArrowDown") { - this.props.actions.increment_selected_file_index(); - } else if (e.key == "Enter") { - const x = - this.listingRef.current?.[this.props.selected_file_index ?? 0]; - if (x != null) { - const { isdir, name } = x; - const path = join(this.props.current_path, name); - if (isdir) { - this.props.actions.open_directory(path); - } else { - this.props.actions.open_file({ path, foreground: !e.ctrlKey }); - } - if (!e.ctrlKey) { - this.props.actions.set_file_search(""); - this.props.actions.clear_selected_file_index(); - } - } - } - }; + if (actions == null || project_map == null) { + return ; + } - handle_files_key_up = (e): void => { - if (e.key === "Shift") { - this.setState({ shift_is_down: false }); - } + useEffect(() => { + $(window).on("keydown", handle_files_key_down); + $(window).on("keyup", handle_files_key_up); + return () => { + $(window).off("keydown", handle_files_key_down); + $(window).off("keyup", handle_files_key_up); }; - - create_file = (ext, switch_over) => { - if (switch_over == undefined) { - switch_over = true; + }, []); + + const handle_files_key_down = (e): void => { + if (e.key === "Shift") { + setShiftIsDown(true); + } else if (e.key == "ArrowUp") { + if (e.shiftKey || e.ctrlKey || e.metaKey) { + const path = dirname(current_path); + actions.open_directory(path == "." ? "" : path); + } else { + actions.decrement_selected_file_index(); } - const { file_search } = this.props; - if ( - ext == undefined && - file_search.lastIndexOf(".") <= file_search.lastIndexOf("/") - ) { - let disabled_ext; - if (this.props.configuration != undefined) { - ({ disabled_ext } = this.props.configuration.get("main", { - disabled_ext: [], - })); + } else if (e.key == "ArrowDown") { + actions.increment_selected_file_index(); + } else if (e.key == "Enter") { + const x = listingRef.current?.[selected_file_index ?? 0]; + if (x != null) { + const { isdir, name } = x; + const path = join(current_path, name); + if (isdir) { + actions.open_directory(path); } else { - disabled_ext = []; + actions.open_file({ path, foreground: !e.ctrlKey }); + } + if (!e.ctrlKey) { + actions.set_file_search(""); + actions.clear_selected_file_index(); } - ext = default_ext(disabled_ext); } - - this.props.actions.create_file({ - name: file_search, - ext, - current_path: this.props.current_path, - switch_over, - }); - this.props.actions.setState({ file_search: "" }); - }; - - create_folder = (switch_over = true): void => { - this.props.actions.create_folder({ - name: this.props.file_search, - current_path: this.props.current_path, - switch_over, - }); - this.props.actions.setState({ file_search: "" }); - }; - - file_listing_page_size() { - return ( - this.props.other_settings && - this.props.other_settings.get("page_size", 50) - ); } + }; - render() { - let project_is_running: boolean, project_state: ProjectStatus | undefined; - - if (this.props.checked_files == undefined) { - // hasn't loaded/initialized at all - return ; - } - - const my_group = this.props.get_my_group(this.props.project_id); + const handle_files_key_up = (e): void => { + if (e.key === "Shift") { + setShiftIsDown(false); + } + }; - // regardless of consequences, for admins a project is always running - // see https://github.com/sagemathinc/cocalc/issues/3863 - if (my_group === "admin") { - project_state = new ProjectStatus({ state: "running" }); - project_is_running = true; - // next, we check if this is a common user (not public) - } else if (my_group !== "public") { - project_state = this.props.project_map?.getIn([ - this.props.project_id, - "state", - ]) as any; - project_is_running = project_state?.get("state") == "running"; - } else { - project_is_running = false; - } + const create_file = (ext, switch_over) => { + if (switch_over == undefined) { + switch_over = true; + } + if ( + ext == undefined && + file_search != null && + file_search.lastIndexOf(".") <= file_search.lastIndexOf("/") + ) { + const disabled_ext = // @ts-ignore + configuration?.getIn(["main", "disabled_ext"])?.toJS() ?? []; + ext = default_ext(disabled_ext); + } - // be careful with adding height:'100%'. it could cause flex to miscalculate. see #3904 - return ( -
+ actions.create_file({ + name: file_search ?? "", + ext, + current_path: current_path, + switch_over, + }); + actions.setState({ file_search: "" }); + }; + + const create_folder = (switch_over = true): void => { + actions.create_folder({ + name: file_search ?? "", + current_path: current_path, + switch_over, + }); + actions.setState({ file_search: "" }); + }; + + let project_is_running: boolean, project_state: ProjectStatus | undefined; + + if (checked_files == undefined) { + // hasn't loaded/initialized at all + return ; + } + + const my_group = redux.getStore("projects").get_my_group(project_id); + + // regardless of consequences, for admins a project is always running + // see https://github.com/sagemathinc/cocalc/issues/3863 + if (my_group === "admin") { + project_state = new ProjectStatus({ state: "running" }); + project_is_running = true; + // next, we check if this is a common user (not public) + } else if (my_group !== "public") { + project_state = project_map?.getIn([project_id, "state"]) as any; + project_is_running = project_state?.get("state") == "running"; + } else { + project_is_running = false; + } + + // be careful with adding height:'100%'. it could cause flex to miscalculate. see #3904 + return ( +
+
+ {error && ( + actions.setState({ error: "" })} + /> + )} + actions.clear_all_activity()} + style={{ top: "100px" }} + /> +
- this.props.actions.setState({ error })} - /> - this.props.actions.clear_all_activity()} - style={{ top: "100px" }} - /> -
-
-
- -
- -
-
- {!!this.props.compute_server_id && ( -
- -
- )} -
- {!IS_MOBILE && ( -
-
- -
-
- )} - {!IS_MOBILE && ( -
- -
- )} +
+
- +
- - {this.props.ext_selection != null && ( - - )} -
+ {!!compute_server_id && (
-
-
- + {!IS_MOBILE && ( +
+
+
- - {project_is_running && - this.props.show_custom_software_reset && - this.props.checked_files.size == 0 && ( - - )} - - {this.props.show_library && ( - - - - Library{" "} - - (help...) - - - } - close={() => this.props.actions.toggle_library(false)} - > - this.props.actions.toggle_library(false)} - /> - - - - )} - - {this.props.checked_files.size > 0 && - this.props.file_action != undefined ? ( - - - - - - ) : undefined} + )} + {!IS_MOBILE && ( +
+ +
+ )} +
+
+
+ {ext_selection != null && } +
- - - + +
+
+
-
- ); - } - }, -); + + {project_is_running && + show_custom_software_reset && + checked_files.size == 0 && + images != null && ( + + )} + + {show_library && ( + + + + Library{" "} + + (help...) + + + } + close={() => actions.toggle_library(false)} + > + actions.toggle_library(false)} + /> + + + + )} + + {checked_files.size > 0 && file_action != undefined ? ( + + + + + + ) : undefined} +
+ +
+ + + +
+ +
+ ); +} diff --git a/src/packages/frontend/project/explorer/file-listing/file-checkbox.tsx b/src/packages/frontend/project/explorer/file-listing/file-checkbox.tsx index 79481771c9..723fcc5b3d 100644 --- a/src/packages/frontend/project/explorer/file-listing/file-checkbox.tsx +++ b/src/packages/frontend/project/explorer/file-listing/file-checkbox.tsx @@ -17,26 +17,26 @@ interface Props { style?: React.CSSProperties; } -export const FileCheckbox: React.FC = React.memo((props: Props) => { - const { name, checked, actions, current_path, style } = props; - - function handle_click(e) { - e.stopPropagation(); // so we don't open the file - const full_name = path_to_file(current_path, name); - if (e.shiftKey) { - actions.set_selected_file_range(full_name, !checked); - } else { - actions.set_file_checked(full_name, !checked); +export const FileCheckbox: React.FC = React.memo( + ({ name, checked, actions, current_path, style }: Props) => { + function handle_click(e) { + e.stopPropagation(); // so we don't open the file + const full_name = path_to_file(current_path, name); + if (e.shiftKey) { + actions.set_selected_file_range(full_name, !checked); + } else { + actions.set_file_checked(full_name, !checked); + } + actions.set_most_recent_file_click(full_name); } - actions.set_most_recent_file_click(full_name); - } - return ( - - - - ); -}); + return ( + + + + ); + }, +); diff --git a/src/packages/frontend/project/explorer/file-listing/file-listing.tsx b/src/packages/frontend/project/explorer/file-listing/file-listing.tsx index a46c23d765..c76c733b7d 100644 --- a/src/packages/frontend/project/explorer/file-listing/file-listing.tsx +++ b/src/packages/frontend/project/explorer/file-listing/file-listing.tsx @@ -13,10 +13,10 @@ import { useEffect, useRef } from "react"; import { FormattedMessage } from "react-intl"; import { Virtuoso, VirtuosoHandle } from "react-virtuoso"; import { - AppRedux, Rendered, TypedMap, useTypedRedux, + redux, } from "@cocalc/frontend/app-framework"; import useVirtuosoScrollHook from "@cocalc/frontend/components/virtuoso-scroll-hook"; import { ProjectActions } from "@cocalc/frontend/project_actions"; @@ -36,7 +36,6 @@ import filterListing from "@cocalc/frontend/project/listing/filter-listing"; interface Props { // TODO: everything but actions/redux should be immutable JS data, and use shouldComponentUpdate actions: ProjectActions; - redux: AppRedux; name: string; active_file_sort: TypedMap<{ column_name: string; is_descending: boolean }>; @@ -46,16 +45,9 @@ interface Props { current_path: string; project_id: string; shift_is_down: boolean; - sort_by: (heading: string) => void; - library?: object; - other_settings?: immutable.Map; - last_scroll_top?: number; configuration_main?: MainConfiguration; isRunning?: boolean; // true if this project is running - show_hidden?: boolean; - show_masked?: boolean; - stale?: boolean; listingRef; @@ -82,21 +74,26 @@ function sortDesc(active_file_sort?): { } export function FileListing(props) { + const { project_id } = props; + const active_file_sort = useTypedRedux({ project_id }, "active_file_sort"); const path = props.current_path; - const fs = useFs({ project_id: props.project_id }); + const fs = useFs({ project_id }); let { listing, error } = useListing({ fs, path, ...sortDesc(props.active_file_sort), - cacheId: { project_id: props.project_id }, + cacheId: { project_id }, }); + const showHidden = useTypedRedux({ project_id }, "show_hidden"); + const showMasked = useTypedRedux({ project_id }, "show_masked"); props.listingRef.current = listing = error ? null : filterListing({ listing, search: props.file_search, - showHidden: props.show_hidden, + showHidden, + showMasked, }); useEffect(() => { @@ -109,12 +106,11 @@ export function FileListing(props) { if (listing == null) { return ; } - return ; + return ; } function FileListing0({ actions, - redux, name, active_file_sort, listing, @@ -122,7 +118,6 @@ function FileListing0({ current_path, project_id, shift_is_down, - sort_by, configuration_main, file_search = "", stale, @@ -276,7 +271,10 @@ function FileListing0({ flexDirection: "column", }} > - + {listing.length > 0 ? render_rows() : render_no_files()}
diff --git a/src/packages/frontend/project/explorer/file-listing/listing-header.tsx b/src/packages/frontend/project/explorer/file-listing/listing-header.tsx index 7d40cd5ee5..978153bf02 100644 --- a/src/packages/frontend/project/explorer/file-listing/listing-header.tsx +++ b/src/packages/frontend/project/explorer/file-listing/listing-header.tsx @@ -24,11 +24,7 @@ const row_style: React.CSSProperties = { const inner_icon_style = { marginRight: "10px" }; -// TODO: Something should uniformly describe how sorted table headers work. -// 5/8/2017 We have 3 right now, Course students, assignments panel and this one. -export const ListingHeader: React.FC = (props: Props) => { - const { active_file_sort, sort_by } = props; - +export function ListingHeader({ active_file_sort, sort_by }: Props) { function render_sort_link( column_name: string, display_name: string, @@ -81,4 +77,4 @@ export const ListingHeader: React.FC = (props: Props) => { ); -}; +} diff --git a/src/packages/frontend/project/explorer/file-listing/utils.ts b/src/packages/frontend/project/explorer/file-listing/utils.ts index 9745e51556..3fd6015077 100644 --- a/src/packages/frontend/project/explorer/file-listing/utils.ts +++ b/src/packages/frontend/project/explorer/file-listing/utils.ts @@ -48,9 +48,7 @@ export const EXTs: ReadonlyArray = Object.freeze([ "sage-chat", ]); -export function default_ext( - disabled_ext: { includes: (s: string) => boolean } | undefined -): Extension { +export function default_ext(disabled_ext: string[] | undefined): Extension { if (disabled_ext != null) { for (const ext of EXTs) { if (disabled_ext.includes(ext)) continue; @@ -79,7 +77,7 @@ export function full_path_text(file_search: string, disabled_ext: string[]) { export function generate_click_for( file_action_name: string, full_path: string, - project_actions: ProjectActions + project_actions: ProjectActions, ) { return (e) => { e.preventDefault(); diff --git a/src/packages/frontend/project/explorer/misc-side-buttons.tsx b/src/packages/frontend/project/explorer/misc-side-buttons.tsx index 73772b9272..b4444c0c18 100644 --- a/src/packages/frontend/project/explorer/misc-side-buttons.tsx +++ b/src/packages/frontend/project/explorer/misc-side-buttons.tsx @@ -5,7 +5,6 @@ import { Space } from "antd"; import { join } from "path"; -import React from "react"; import { defineMessage, useIntl } from "react-intl"; import { Button, ButtonToolbar } from "@cocalc/frontend/antd-bootstrap"; import { Icon, Tip, VisibleLG } from "@cocalc/frontend/components"; @@ -13,10 +12,11 @@ import LinkRetry from "@cocalc/frontend/components/link-retry"; import { useStudentProjectFunctionality } from "@cocalc/frontend/course"; import { labels } from "@cocalc/frontend/i18n"; import { serverURL, SPEC } from "@cocalc/frontend/project/named-server-panel"; -import { Available } from "@cocalc/frontend/project_configuration"; -import { ProjectActions } from "@cocalc/frontend/project_store"; import track from "@cocalc/frontend/user-tracking"; import { KUCALC_COCALC_COM } from "@cocalc/util/db-schema/site-defaults"; +import { useProjectContext } from "@cocalc/frontend/project/context"; +import { useTypedRedux } from "@cocalc/frontend/app-framework"; +import { type JSX, type MouseEvent } from "react"; const SHOW_SERVER_LAUNCHERS = false; @@ -27,53 +27,42 @@ const OPEN_MSG = defineMessage({ defaultMessage: `Opens the current directory in a {name} server instance, running inside this project.`, }); -interface Props { - actions: ProjectActions; - available_features?: Available; - current_path: string; - kucalc?: string; - project_id: string; - show_hidden?: boolean; - show_masked?: boolean; -} - -export const MiscSideButtons: React.FC = (props) => { - const { - actions, - available_features, - current_path, - kucalc, - project_id, - show_hidden, - show_masked, - } = props; - +export function MiscSideButtons() { + const { actions, project_id } = useProjectContext(); + const show_hidden = useTypedRedux({ project_id }, "show_hidden"); + const show_masked = useTypedRedux({ project_id }, "show_masked"); + const current_path = useTypedRedux({ project_id }, "current_path"); + const available_features = useTypedRedux( + { project_id }, + "available_features", + )?.toJS(); + const kucalc = useTypedRedux("customize", "kucalc"); const intl = useIntl(); const student_project_functionality = useStudentProjectFunctionality(project_id); - const handle_hidden_toggle = (e: React.MouseEvent): void => { + const handle_hidden_toggle = (e: MouseEvent): void => { e.preventDefault(); - return actions.setState({ + return actions?.setState({ show_hidden: !show_hidden, }); }; - const handle_masked_toggle = (e: React.MouseEvent): void => { + const handle_masked_toggle = (e: MouseEvent): void => { e.preventDefault(); - actions.setState({ + actions?.setState({ show_masked: !show_masked, }); }; - const handle_backup = (e: React.MouseEvent): void => { + const handle_backup = (e: MouseEvent): void => { e.preventDefault(); - actions.open_directory(".snapshots"); + actions?.open_directory(".snapshots"); track("snapshots", { action: "open", where: "explorer" }); }; - function render_hidden_toggle(): React.JSX.Element { + function render_hidden_toggle(): JSX.Element { const icon = show_hidden ? "eye" : "eye-slash"; return (
); -}; +} diff --git a/src/packages/frontend/project/explorer/new-button.tsx b/src/packages/frontend/project/explorer/new-button.tsx index 437821485f..a9e37355c7 100644 --- a/src/packages/frontend/project/explorer/new-button.tsx +++ b/src/packages/frontend/project/explorer/new-button.tsx @@ -10,7 +10,6 @@ import { DropdownMenu, Icon } from "@cocalc/frontend/components"; import { labels } from "@cocalc/frontend/i18n"; import { ProjectActions } from "@cocalc/frontend/project_store"; import { COLORS } from "@cocalc/util/theme"; -import { Configuration } from "./explorer"; import { EXTs as ALL_FILE_BUTTON_TYPES } from "./file-listing/utils"; const { file_options } = require("@cocalc/frontend/editor"); @@ -21,21 +20,18 @@ interface Props { actions: ProjectActions; create_folder: (switch_over?: boolean) => void; create_file: (ext?: string, switch_over?: boolean) => void; - configuration?: Configuration; + configuration?; disabled: boolean; } -export const NewButton: React.FC = (props: Props) => { - const { - file_search = "", - /*current_path,*/ - actions, - create_folder, - create_file, - configuration, - disabled, - } = props; - +export const NewButton: React.FC = ({ + file_search = "", + actions, + create_folder, + create_file, + configuration, + disabled, +}: Props) => { const intl = useIntl(); function new_file_button_types() { diff --git a/src/packages/frontend/project/explorer/path-navigator.tsx b/src/packages/frontend/project/explorer/path-navigator.tsx index 52cdd0eb7f..835e082e44 100644 --- a/src/packages/frontend/project/explorer/path-navigator.tsx +++ b/src/packages/frontend/project/explorer/path-navigator.tsx @@ -23,13 +23,12 @@ interface Props { // This path consists of several PathSegmentLinks export const PathNavigator: React.FC = React.memo( - (props: Readonly) => { - const { - project_id, - style, - className = "cc-path-navigator", - mode = "files", - } = props; + ({ + project_id, + style, + className = "cc-path-navigator", + mode = "files", + }: Readonly) => { const current_path = useTypedRedux({ project_id }, "current_path"); const history_path = useTypedRedux({ project_id }, "history_path"); const actions = useActions({ project_id }); diff --git a/src/packages/frontend/project/explorer/path-segment-link.tsx b/src/packages/frontend/project/explorer/path-segment-link.tsx index de1fbb2cea..63c4d56dd5 100644 --- a/src/packages/frontend/project/explorer/path-segment-link.tsx +++ b/src/packages/frontend/project/explorer/path-segment-link.tsx @@ -27,18 +27,16 @@ export interface PathSegmentItem { } // One segment of the directory links at the top of the files listing. -export function createPathSegmentLink(props: Readonly): PathSegmentItem { - const { - path = "", - display, - on_click, - full_name, - history, - active = false, - key, - style, - } = props; - +export function createPathSegmentLink({ + path = "", + display, + on_click, + full_name, + history, + active = false, + key, + style, +}: Readonly): PathSegmentItem { function render_content(): React.JSX.Element | string | undefined { if (full_name && full_name !== display) { return ( diff --git a/src/packages/frontend/project/explorer/tour/tour.tsx b/src/packages/frontend/project/explorer/tour/tour.tsx index 8a8a0e41c1..6b7d4bd412 100644 --- a/src/packages/frontend/project/explorer/tour/tour.tsx +++ b/src/packages/frontend/project/explorer/tour/tour.tsx @@ -1,14 +1,12 @@ import type { TourProps } from "antd"; import { Checkbox, Tour } from "antd"; - -import { redux } from "@cocalc/frontend/app-framework"; +import { redux, useTypedRedux } from "@cocalc/frontend/app-framework"; import { Paragraph, Text } from "@cocalc/frontend/components"; import { A } from "@cocalc/frontend/components/A"; import { Icon } from "@cocalc/frontend/components/icon"; import actionsImage from "./actions.png"; export default function ExplorerTour({ - open, project_id, newFileRef, searchAndTerminalBar, @@ -16,6 +14,7 @@ export default function ExplorerTour({ currentDirectoryRef, miscButtonsRef, }) { + const open = useTypedRedux({ project_id }, "explorerTour"); const steps: TourProps["steps"] = [ { title: ( diff --git a/src/packages/frontend/project/listing/filter-listing.ts b/src/packages/frontend/project/listing/filter-listing.ts index 53e296da7f..a5b03edb69 100644 --- a/src/packages/frontend/project/listing/filter-listing.ts +++ b/src/packages/frontend/project/listing/filter-listing.ts @@ -4,11 +4,16 @@ export default function filterListing({ listing, search, showHidden, + showMasked, }: { listing?: DirectoryListingEntry[] | null; search?: string; showHidden?: boolean; + showMasked?: boolean; }): DirectoryListingEntry[] | null { + if (!showMasked) { + console.log("TODO: show masked"); + } if (listing == null) { return null; } From 489eaf1419cbee4b160f081af1de0c96c85e8a43 Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 29 Jul 2025 02:54:15 +0000 Subject: [PATCH 125/270] fix bugs in new file listings --- .../frontend/project/explorer/explorer.tsx | 84 +++++++++---------- .../explorer/file-listing/file-listing.tsx | 6 +- 2 files changed, 44 insertions(+), 46 deletions(-) diff --git a/src/packages/frontend/project/explorer/explorer.tsx b/src/packages/frontend/project/explorer/explorer.tsx index 50402c84dc..d81087b553 100644 --- a/src/packages/frontend/project/explorer/explorer.tsx +++ b/src/packages/frontend/project/explorer/explorer.tsx @@ -77,10 +77,6 @@ export function Explorer() { "file_creation_error", ); const file_search = useTypedRedux({ project_id }, "file_search"); - const selected_file_index = useTypedRedux( - { project_id }, - "selected_file_index", - ); const show_custom_software_reset = useTypedRedux( { project_id }, "show_custom_software_reset", @@ -97,49 +93,51 @@ export function Explorer() { } useEffect(() => { + const handle_files_key_down = (e): void => { + if (e.key === "Shift") { + setShiftIsDown(true); + } else if (e.key == "ArrowUp") { + if (e.shiftKey || e.ctrlKey || e.metaKey) { + const path = dirname(current_path); + actions.open_directory(path == "." ? "" : path); + } else { + actions.decrement_selected_file_index(); + } + } else if (e.key == "ArrowDown") { + actions.increment_selected_file_index(); + } else if (e.key == "Enter") { + const n = + redux.getProjectStore(project_id).get("selected_file_index") ?? 0; + const x = listingRef.current?.[n]; + if (x != null) { + const { isdir, name } = x; + const path = join(current_path, name); + if (isdir) { + actions.open_directory(path); + } else { + actions.open_file({ path, foreground: !e.ctrlKey }); + } + if (!e.ctrlKey) { + actions.set_file_search(""); + actions.clear_selected_file_index(); + } + } + } + }; + + const handle_files_key_up = (e): void => { + if (e.key === "Shift") { + setShiftIsDown(false); + } + }; + $(window).on("keydown", handle_files_key_down); $(window).on("keyup", handle_files_key_up); return () => { $(window).off("keydown", handle_files_key_down); $(window).off("keyup", handle_files_key_up); }; - }, []); - - const handle_files_key_down = (e): void => { - if (e.key === "Shift") { - setShiftIsDown(true); - } else if (e.key == "ArrowUp") { - if (e.shiftKey || e.ctrlKey || e.metaKey) { - const path = dirname(current_path); - actions.open_directory(path == "." ? "" : path); - } else { - actions.decrement_selected_file_index(); - } - } else if (e.key == "ArrowDown") { - actions.increment_selected_file_index(); - } else if (e.key == "Enter") { - const x = listingRef.current?.[selected_file_index ?? 0]; - if (x != null) { - const { isdir, name } = x; - const path = join(current_path, name); - if (isdir) { - actions.open_directory(path); - } else { - actions.open_file({ path, foreground: !e.ctrlKey }); - } - if (!e.ctrlKey) { - actions.set_file_search(""); - actions.clear_selected_file_index(); - } - } - } - }; - - const handle_files_key_up = (e): void => { - if (e.key === "Shift") { - setShiftIsDown(false); - } - }; + }, [project_id, current_path]); const create_file = (ext, switch_over) => { if (switch_over == undefined) { @@ -151,7 +149,7 @@ export function Explorer() { file_search.lastIndexOf(".") <= file_search.lastIndexOf("/") ) { const disabled_ext = // @ts-ignore - configuration?.getIn(["main", "disabled_ext"])?.toJS() ?? []; + configuration?.getIn(["main", "disabled_ext"])?.toJS?.() ?? []; ext = default_ext(disabled_ext); } @@ -425,7 +423,7 @@ export function Explorer() { create_file={create_file} create_folder={create_folder} project_id={project_id} - shift_is_down={shiftIsDown} + shiftIsDown={shiftIsDown} configuration_main={configuration?.get("main")} /> diff --git a/src/packages/frontend/project/explorer/file-listing/file-listing.tsx b/src/packages/frontend/project/explorer/file-listing/file-listing.tsx index c76c733b7d..422fb0ace4 100644 --- a/src/packages/frontend/project/explorer/file-listing/file-listing.tsx +++ b/src/packages/frontend/project/explorer/file-listing/file-listing.tsx @@ -44,7 +44,7 @@ interface Props { checked_files: immutable.Set; current_path: string; project_id: string; - shift_is_down: boolean; + shiftIsDown: boolean; configuration_main?: MainConfiguration; isRunning?: boolean; // true if this project is running @@ -117,7 +117,7 @@ function FileListing0({ checked_files, current_path, project_id, - shift_is_down, + shiftIsDown, configuration_main, file_search = "", stale, @@ -161,7 +161,7 @@ function FileListing0({ key={index} current_path={current_path} actions={actions} - no_select={shift_is_down} + no_select={shiftIsDown} link_target={link_target} computeServerId={computeServerId} /> From adf444f577a214250399511527b9ab9e205998fe Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 29 Jul 2025 05:14:59 +0000 Subject: [PATCH 126/270] files: select a range of files --- .../frontend/project/directory-selector.tsx | 6 +- .../frontend/project/explorer/action-box.tsx | 5 +- .../frontend/project/explorer/download.tsx | 10 +- .../explorer/file-listing/file-checkbox.tsx | 50 +++--- .../explorer/file-listing/file-listing.tsx | 4 +- .../explorer/file-listing/file-row.tsx | 2 + .../frontend/project/listing/use-files.ts | 10 ++ src/packages/frontend/project_actions.ts | 162 ++++++++++++------ src/packages/frontend/project_store.ts | 2 - 9 files changed, 159 insertions(+), 92 deletions(-) diff --git a/src/packages/frontend/project/directory-selector.tsx b/src/packages/frontend/project/directory-selector.tsx index bc10ca9fdb..bb1a474595 100644 --- a/src/packages/frontend/project/directory-selector.tsx +++ b/src/packages/frontend/project/directory-selector.tsx @@ -26,7 +26,9 @@ import { redux, useTypedRedux } from "@cocalc/frontend/app-framework"; import { useFrameContext } from "@cocalc/frontend/frame-editors/frame-tree/frame-context"; import ShowError from "@cocalc/frontend/components/error"; import useFs from "@cocalc/frontend/project/listing/use-fs"; -import useFiles from "@cocalc/frontend/project/listing/use-files"; +import useFiles, { + getCacheId, +} from "@cocalc/frontend/project/listing/use-files"; const NEW_FOLDER = "New Folder"; @@ -378,7 +380,7 @@ function Subdirs(props) { const { files, error, refresh } = useFiles({ fs, path, - cacheId: { project_id }, + cacheId: getCacheId({ project_id, compute_server_id: computeServerId }), }); if (error) { return ; diff --git a/src/packages/frontend/project/explorer/action-box.tsx b/src/packages/frontend/project/explorer/action-box.tsx index b5a5df0f19..f39af2a2b9 100644 --- a/src/packages/frontend/project/explorer/action-box.tsx +++ b/src/packages/frontend/project/explorer/action-box.tsx @@ -46,14 +46,13 @@ export const PRE_STYLE = { type FileAction = undefined | keyof typeof file_actions; -interface ReactProps { +interface Props { checked_files: immutable.Set; file_action: FileAction; current_path: string; project_id: string; file_map: object; actions: ProjectActions; - displayed_listing?: object; } export function ActionBox({ @@ -63,7 +62,7 @@ export function ActionBox({ project_id, file_map, actions, -}: ReactProps) { +}: Props) { const intl = useIntl(); const runQuota = useRunQuota(project_id, null); const get_user_type: () => string = useRedux("account", "get_user_type"); diff --git a/src/packages/frontend/project/explorer/download.tsx b/src/packages/frontend/project/explorer/download.tsx index ddaa75c793..17f7740a76 100644 --- a/src/packages/frontend/project/explorer/download.tsx +++ b/src/packages/frontend/project/explorer/download.tsx @@ -3,7 +3,7 @@ import { useEffect, useRef, useState } from "react"; import { useIntl } from "react-intl"; import { default_filename } from "@cocalc/frontend/account"; -import { redux, useRedux } from "@cocalc/frontend/app-framework"; +import { useRedux } from "@cocalc/frontend/app-framework"; import ShowError from "@cocalc/frontend/components/error"; import { Icon } from "@cocalc/frontend/components/icon"; import { labels } from "@cocalc/frontend/i18n"; @@ -12,7 +12,7 @@ import { path_split, path_to_file, plural } from "@cocalc/util/misc"; import { PRE_STYLE } from "./action-box"; import CheckedFiles from "./checked-files"; -export default function Download({}) { +export default function Download() { const intl = useIntl(); const inputRef = useRef(null); const { actions } = useProjectContext(); @@ -33,6 +33,9 @@ export default function Download({}) { ); useEffect(() => { + if (actions == null) { + return; + } if (checked_files == null) { return; } @@ -41,8 +44,7 @@ export default function Download({}) { return; } const file = checked_files.first(); - const isdir = redux.getProjectStore(project_id).get("displayed_listing") - ?.file_map?.[path_split(file).tail]?.isdir; + const isdir = !!actions.isDirViaCache(file); setArchiveMode(!!isdir); if (!isdir) { const store = actions?.get_store(); diff --git a/src/packages/frontend/project/explorer/file-listing/file-checkbox.tsx b/src/packages/frontend/project/explorer/file-listing/file-checkbox.tsx index 723fcc5b3d..9dd9c62301 100644 --- a/src/packages/frontend/project/explorer/file-listing/file-checkbox.tsx +++ b/src/packages/frontend/project/explorer/file-listing/file-checkbox.tsx @@ -3,8 +3,6 @@ * License: MS-RSL – see LICENSE.md for details */ -import React from "react"; - import { ProjectActions } from "@cocalc/frontend/project_actions"; import { Icon } from "@cocalc/frontend/components"; import { path_to_file } from "@cocalc/util/misc"; @@ -15,28 +13,34 @@ interface Props { actions: ProjectActions; current_path: string; style?: React.CSSProperties; + listing; } -export const FileCheckbox: React.FC = React.memo( - ({ name, checked, actions, current_path, style }: Props) => { - function handle_click(e) { - e.stopPropagation(); // so we don't open the file - const full_name = path_to_file(current_path, name); - if (e.shiftKey) { - actions.set_selected_file_range(full_name, !checked); - } else { - actions.set_file_checked(full_name, !checked); - } - actions.set_most_recent_file_click(full_name); +export function FileCheckbox({ + name, + checked, + actions, + current_path, + style, + listing, +}: Props) { + function handle_click(e) { + e.stopPropagation(); // so we don't open the file + const full_name = path_to_file(current_path, name); + if (e.shiftKey) { + actions.set_selected_file_range(full_name, !checked, listing); + } else { + actions.set_file_checked(full_name, !checked); } + actions.set_most_recent_file_click(full_name); + } - return ( - - - - ); - }, -); + return ( + + + + ); +} diff --git a/src/packages/frontend/project/explorer/file-listing/file-listing.tsx b/src/packages/frontend/project/explorer/file-listing/file-listing.tsx index 422fb0ace4..86b89dd6a0 100644 --- a/src/packages/frontend/project/explorer/file-listing/file-listing.tsx +++ b/src/packages/frontend/project/explorer/file-listing/file-listing.tsx @@ -34,7 +34,6 @@ import useListing, { import filterListing from "@cocalc/frontend/project/listing/filter-listing"; interface Props { - // TODO: everything but actions/redux should be immutable JS data, and use shouldComponentUpdate actions: ProjectActions; name: string; @@ -82,7 +81,7 @@ export function FileListing(props) { fs, path, ...sortDesc(props.active_file_sort), - cacheId: { project_id }, + cacheId: props.actions.getCacheId(), }); const showHidden = useTypedRedux({ project_id }, "show_hidden"); const showMasked = useTypedRedux({ project_id }, "show_masked"); @@ -164,6 +163,7 @@ function FileListing0({ no_select={shiftIsDown} link_target={link_target} computeServerId={computeServerId} + listing={listing} /> ); } diff --git a/src/packages/frontend/project/explorer/file-listing/file-row.tsx b/src/packages/frontend/project/explorer/file-listing/file-row.tsx index 869d99b776..f5217d7bd4 100644 --- a/src/packages/frontend/project/explorer/file-listing/file-row.tsx +++ b/src/packages/frontend/project/explorer/file-listing/file-row.tsx @@ -55,6 +55,7 @@ interface Props { // if given, include a little 'server' tag in this color, and tooltip etc using id. // Also important for download and preview links! computeServerId?: number; + listing; } export const FileRow: React.FC = React.memo((props) => { @@ -356,6 +357,7 @@ export const FileRow: React.FC = React.memo((props) => { current_path={props.current_path} actions={props.actions} style={{ verticalAlign: "sub", color: "#888" }} + listing={props.listing} /> )} diff --git a/src/packages/frontend/project/listing/use-files.ts b/src/packages/frontend/project/listing/use-files.ts index 6e925e817d..7075c98b4c 100644 --- a/src/packages/frontend/project/listing/use-files.ts +++ b/src/packages/frontend/project/listing/use-files.ts @@ -178,3 +178,13 @@ async function cacheNeighbors({ v = v.slice(0, MAX_SUBDIR_CACHE); await Promise.all(v.map(f)); } + +export function getCacheId({ + project_id, + compute_server_id = 0, +}: { + project_id: string; + compute_server_id?: number; +}) { + return { project_id, compute_server_id }; +} diff --git a/src/packages/frontend/project_actions.ts b/src/packages/frontend/project_actions.ts index 285e6e4f06..296678b9cb 100644 --- a/src/packages/frontend/project_actions.ts +++ b/src/packages/frontend/project_actions.ts @@ -109,9 +109,11 @@ import { client_db } from "@cocalc/util/schema"; import { get_editor } from "./editors/react-wrapper"; import { type FilesystemClient } from "@cocalc/conat/files/fs"; import { + getCacheId, getFiles, type Files, } from "@cocalc/frontend/project/listing/use-files"; +import { map as awaitMap } from "awaiting"; const { defaults, required } = misc; @@ -1618,7 +1620,7 @@ export class ProjectActions extends Actions { } // Set the selected state of all files between the most_recent_file_click and the given file - set_selected_file_range(file: string, checked: boolean): void { + set_selected_file_range(file: string, checked: boolean, listing): void { let range; const store = this.get_store(); if (store == undefined) { @@ -1631,9 +1633,9 @@ export class ProjectActions extends Actions { } else { // get the range of files const current_path = store.get("current_path"); - const names = store - .get("displayed_listing") - .listing.map((a) => misc.path_to_file(current_path, a.name)); + const names = listing.map(({ name }) => + misc.path_to_file(current_path, name), + ); range = misc.get_array_range(names, most_recent, file); } @@ -1898,22 +1900,29 @@ export class ProjectActions extends Actions { } }; - // DANGER: ASSUMES PATH IS IN THE DISPLAYED LISTING - private _convert_to_displayed_path(path): string { - if (path.slice(-1) === "/") { - return path; - } else { - const store = this.get_store(); - const file_name = misc.path_split(path).tail; - if (store !== undefined && store.get("displayed_listing")) { - const file_data = store.get("displayed_listing").file_map[file_name]; - if (file_data !== undefined && file_data.isdir) { - return path + "/"; - } + private appendSlashToDirectoryPaths = async ( + paths: string[], + compute_server_id?: number, + ): Promise => { + const f = async (path: string) => { + if (path.endsWith("/")) { + return path; } - return path; - } - } + const isdir = this.isDirViaCache(path, compute_server_id); + if (isdir === false) { + return path; + } + if (isdir === true) { + return path + "/"; + } + if (await this.isdir(path, compute_server_id)) { + return path + "/"; + } else { + return path; + } + }; + return await Promise.all(paths.map(f)); + }; // this is called in "projects.cjsx" (more then once) // in turn, it is calling init methods just once, though @@ -2234,12 +2243,15 @@ export class ProjectActions extends Actions { dest_compute_server_id: this.get_store()?.get("compute_server_id") ?? 0, }); // true for duplicating files - const with_slashes = opts.src.map(this._convert_to_displayed_path); + const withSlashes = await this.appendSlashToDirectoryPaths( + opts.src, + opts.src_compute_server_id, + ); this.log({ event: "file_action", action: "copied", - files: with_slashes.slice(0, 3), + files: withSlashes.slice(0, 3), count: opts.src.length > 3 ? opts.src.length : undefined, dest: opts.dest + (opts.only_contents ? "" : "/"), ...(opts.src_compute_server_id != opts.dest_compute_server_id @@ -2253,7 +2265,7 @@ export class ProjectActions extends Actions { }); if (opts.only_contents) { - opts.src = with_slashes; + opts.src = withSlashes; } // If files start with a -, make them interpretable by rsync (see https://github.com/sagemathinc/cocalc/issues/516) @@ -2341,17 +2353,23 @@ export class ProjectActions extends Actions { }); }; - copy_paths_between_projects(opts) { - opts = defaults(opts, { - public: false, - src_project_id: required, // id of source project - src: required, // list of relative paths of directories or files in the source project - target_project_id: required, // id of target project - target_path: undefined, // defaults to src_path - overwrite_newer: false, // overwrite newer versions of file at destination (destructive) - delete_missing: false, // delete files in dest that are missing from source (destructive) - backup: false, // make ~ backup files instead of overwriting changed files - }); + copy_paths_between_projects = async (opts: { + public: boolean; + // id of source project + src_project_id: string; + // list of relative paths of directories or files in the source project + src: string[]; + // id of target project + target_project_id: string; + // defaults to src_path + target_path?: string; + // overwrite newer versions of file at destination (destructive) + overwrite_newer?: boolean; + // delete files in dest that are missing from source (destructive) + delete_missing?: boolean; + // make ~ backup files instead of overwriting changed files + backup?: boolean; + }) => { const id = misc.uuid(); this.set_activity({ id, @@ -2361,8 +2379,7 @@ export class ProjectActions extends Actions { )} to a project`, }); const { src } = opts; - delete opts.src; - const with_slashes = src.map(this._convert_to_displayed_path); + const withSlashes = await this.appendSlashToDirectoryPaths(src); let dest: string | undefined = undefined; if (opts.target_path != null) { dest = opts.target_path; @@ -2374,13 +2391,13 @@ export class ProjectActions extends Actions { event: "file_action", action: "copied", dest, - files: with_slashes.slice(0, 3), + files: withSlashes.slice(0, 3), count: src.length > 3 ? src.length : undefined, project: opts.target_project_id, }); - const f = async (src_path, cb) => { - const opts0 = misc.copy(opts); - delete opts0.cb; + const f = async (src_path) => { + const opts0: any = misc.copy(opts); + delete opts0.src; opts0.src_path = src_path; // we do this for consistent semantics with file copy opts0.target_path = misc.path_to_file( @@ -2388,15 +2405,11 @@ export class ProjectActions extends Actions { misc.path_split(src_path).tail, ); opts0.timeout = 90 * 1000; - try { - await webapp_client.project_client.copy_path_between_projects(opts0); - cb(); - } catch (err) { - cb(err); - } + await webapp_client.project_client.copy_path_between_projects(opts0); }; - async.mapLimit(src, 3, f, this._finish_exec(id, opts.cb)); - } + await awaitMap(src, 5, f); + this._finish_exec(id); + }; public async rename_file(opts: { src: string; @@ -2456,19 +2469,56 @@ export class ProjectActions extends Actions { if (path == null) { return null; } - // todo: compute_server_id here and in place that does useListing! - return getFiles({ cacheId: { project_id: this.project_id }, path }); + return this.getFilesCache(path); + }; + + private getCacheId = (compute_server_id?: number) => { + return getCacheId({ + project_id: this.project_id, + compute_server_id: + compute_server_id ?? this.get_store()?.get("compute_server_id") ?? 0, + }); + }; + + private getFilesCache = ( + path: string, + compute_server_id?: number, + ): Files | null => { + return getFiles({ + cacheId: this.getCacheId(compute_server_id), + path, + }); + }; + + // using listings cache, attempt to tell if path is a directory; + // undefined if no data about path in the cache. + isDirViaCache = ( + path: string, + compute_server_id?: number, + ): boolean | undefined => { + if (!path) { + return true; + } + const { head: dir, tail: base } = misc.path_split(path); + const files = this.getFilesCache(dir, compute_server_id); + const data = files?.[base]; + if (data == null) { + return undefined; + } else { + return !!data.isdir; + } }; // return true if exists and is a directory - isdir = async (path: string): Promise => { + // error if doesn't exist or can't find out. + // Use isDirViaCache for more of a fast hint. + isdir = async ( + path: string, + compute_server_id?: number, + ): Promise => { if (path == "") return true; // easy special case - try { - const stats = await this.fs().stat(path); - return stats.isDirectory(); - } catch (_) { - return false; - } + const stats = await this.fs(compute_server_id).stat(path); + return stats.isDirectory(); }; public async move_files(opts: { diff --git a/src/packages/frontend/project_store.ts b/src/packages/frontend/project_store.ts index 388a9fb74a..78fb2e0aea 100644 --- a/src/packages/frontend/project_store.ts +++ b/src/packages/frontend/project_store.ts @@ -75,7 +75,6 @@ export interface ProjectStoreState { show_upload: boolean; create_file_alert: boolean; - displayed_listing?: any; // computed(object), configuration?: ProjectConfiguration; configuration_loading: boolean; // for UI feedback available_features?: TypedMap; @@ -292,7 +291,6 @@ export class ProjectStore extends Store { just_closed_files: immutable.List([]), show_upload: false, create_file_alert: false, - displayed_listing: undefined, // computed(object), show_masked: true, configuration: undefined, configuration_loading: false, // for UI feedback From 57934c7d0c9616359d0b25f720a6d63fca838e49 Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 29 Jul 2025 16:33:29 +0000 Subject: [PATCH 127/270] refactor explorer listings so that action bar works again --- .../frontend/project/explorer/action-bar.tsx | 3 +- .../frontend/project/explorer/explorer.tsx | 119 +++++++++++++----- .../explorer/file-listing/file-listing.tsx | 87 ++----------- .../explorer/file-listing/file-row.tsx | 7 +- src/packages/frontend/project_actions.ts | 2 +- src/packages/util/types/directory-listing.ts | 7 +- 6 files changed, 111 insertions(+), 114 deletions(-) diff --git a/src/packages/frontend/project/explorer/action-bar.tsx b/src/packages/frontend/project/explorer/action-bar.tsx index fa82fc17df..f427ff1864 100644 --- a/src/packages/frontend/project/explorer/action-bar.tsx +++ b/src/packages/frontend/project/explorer/action-bar.tsx @@ -19,6 +19,7 @@ import { file_actions, type ProjectActions } from "@cocalc/frontend/project_stor import * as misc from "@cocalc/util/misc"; import { COLORS } from "@cocalc/util/theme"; import { useProjectContext } from "../context"; +import { DirectoryListingEntry } from "@cocalc/util/types"; const ROW_INFO_STYLE = { color: COLORS.GRAY, @@ -29,7 +30,7 @@ const ROW_INFO_STYLE = { interface Props { project_id?: string; checked_files: immutable.Set; - listing: { name: string; isdir: boolean }[]; + listing: DirectoryListingEntry[]; current_path?: string; project_map?; images?: ComputeImages; diff --git a/src/packages/frontend/project/explorer/explorer.tsx b/src/packages/frontend/project/explorer/explorer.tsx index d81087b553..9a0d7bf19b 100644 --- a/src/packages/frontend/project/explorer/explorer.tsx +++ b/src/packages/frontend/project/explorer/explorer.tsx @@ -34,6 +34,13 @@ import { SearchBar } from "./search-bar"; import ExplorerTour from "./tour/tour"; import { dirname, join } from "path"; import { redux, useTypedRedux } from "@cocalc/frontend/app-framework"; +import useFs from "@cocalc/frontend/project/listing/use-fs"; +import useListing, { + type SortField, +} from "@cocalc/frontend/project/listing/use-listing"; +import filterListing from "@cocalc/frontend/project/listing/filter-listing"; +import ShowError from "@cocalc/frontend/components/error"; +import { MainConfiguration } from "@cocalc/frontend/project_configuration"; const FLEX_ROW_STYLE = { display: "flex", @@ -51,14 +58,34 @@ const ERROR_STYLE: CSSProperties = { boxShadow: "5px 5px 5px grey", } as const; +function sortDesc(active_file_sort?): { + sortField: SortField; + sortDirection: "asc" | "desc"; +} { + const { column_name, is_descending } = active_file_sort?.toJS() ?? { + column_name: "name", + is_descending: false, + }; + if (column_name == "time") { + return { + sortField: "mtime", + sortDirection: is_descending ? "asc" : "desc", + }; + } + return { + sortField: column_name, + sortDirection: is_descending ? "desc" : "asc", + }; +} + export function Explorer() { const { actions, project_id } = useProjectContext(); + const newFileRef = useRef(null); const searchAndTerminalBar = useRef(null); const fileListingRef = useRef(null); const currentDirectoryRef = useRef(null); const miscButtonsRef = useRef(null); - const listingRef = useRef(null); const activity = useTypedRedux({ project_id }, "activity")?.toJS(); const available_features = useTypedRedux( @@ -76,7 +103,7 @@ export function Explorer() { { project_id }, "file_creation_error", ); - const file_search = useTypedRedux({ project_id }, "file_search"); + const file_search = useTypedRedux({ project_id }, "file_search") ?? ""; const show_custom_software_reset = useTypedRedux( { project_id }, "show_custom_software_reset", @@ -88,6 +115,34 @@ export function Explorer() { const images = useTypedRedux("compute_images", "images"); + const active_file_sort = useTypedRedux({ project_id }, "active_file_sort"); + const fs = useFs({ project_id }); + let { listing, error: listingError } = useListing({ + fs, + path: current_path, + ...sortDesc(active_file_sort), + cacheId: actions?.getCacheId(compute_server_id), + }); + const showHidden = useTypedRedux({ project_id }, "show_hidden"); + const showMasked = useTypedRedux({ project_id }, "show_masked"); + + listing = listingError + ? null + : filterListing({ + listing, + search: file_search, + showHidden, + showMasked, + }); + + useEffect(() => { + actions?.setState({ numDisplayedFiles: listing?.length ?? 0 }); + }, [listing?.length]); + + if (listingError) { + return ; + } + if (actions == null || project_map == null) { return ; } @@ -108,7 +163,7 @@ export function Explorer() { } else if (e.key == "Enter") { const n = redux.getProjectStore(project_id).get("selected_file_index") ?? 0; - const x = listingRef.current?.[n]; + const x = listing?.[n]; if (x != null) { const { isdir, name } = x; const path = join(current_path, name); @@ -118,7 +173,7 @@ export function Explorer() { actions.open_file({ path, foreground: !e.ctrlKey }); } if (!e.ctrlKey) { - actions.set_file_search(""); + setTimeout(() => actions.set_file_search(""), 10); actions.clear_selected_file_index(); } } @@ -314,18 +369,20 @@ export function Explorer() { minWidth: "20em", }} > - + {listing != null && ( + + )}
- + {listing == null ? ( + + ) : ( + + )}
; - listing: any[]; + listing: DirectoryListingEntry[]; file_search: string; checked_files: immutable.Set; current_path: string; @@ -46,72 +39,11 @@ interface Props { shiftIsDown: boolean; configuration_main?: MainConfiguration; isRunning?: boolean; // true if this project is running - stale?: boolean; - - listingRef; -} - -function sortDesc(active_file_sort?): { - sortField: SortField; - sortDirection: "asc" | "desc"; -} { - const { column_name, is_descending } = active_file_sort?.toJS() ?? { - column_name: "name", - is_descending: false, - }; - if (column_name == "time") { - return { - sortField: "mtime", - sortDirection: is_descending ? "asc" : "desc", - }; - } - return { - sortField: column_name, - sortDirection: is_descending ? "desc" : "asc", - }; } -export function FileListing(props) { - const { project_id } = props; - const active_file_sort = useTypedRedux({ project_id }, "active_file_sort"); - const path = props.current_path; - const fs = useFs({ project_id }); - let { listing, error } = useListing({ - fs, - path, - ...sortDesc(props.active_file_sort), - cacheId: props.actions.getCacheId(), - }); - const showHidden = useTypedRedux({ project_id }, "show_hidden"); - const showMasked = useTypedRedux({ project_id }, "show_masked"); - - props.listingRef.current = listing = error - ? null - : filterListing({ - listing, - search: props.file_search, - showHidden, - showMasked, - }); - - useEffect(() => { - props.actions.setState({ numDisplayedFiles: listing?.length ?? 0 }); - }, [listing?.length]); - - if (error) { - return ; - } - if (listing == null) { - return ; - } - return ; -} - -function FileListing0({ +export function FileListing({ actions, - name, - active_file_sort, listing, checked_files, current_path, @@ -120,11 +52,12 @@ function FileListing0({ configuration_main, file_search = "", stale, - // show_masked, }: Props) { + const active_file_sort = useTypedRedux({ project_id }, "active_file_sort"); const computeServerId = useTypedRedux({ project_id }, "compute_server_id"); const selected_file_index = useTypedRedux({ project_id }, "selected_file_index") ?? 0; + const name = actions.name; function render_row( name, @@ -132,8 +65,6 @@ function FileListing0({ time, mask, isdir, - display_name, - public_data, issymlink, index: number, link_target?: string, // if given, is a known symlink to this file @@ -145,7 +76,6 @@ function FileListing0({ ; + } + function render_rows(): Rendered { return ( = [ interface Props { isdir: boolean; name: string; - display_name: string; // if given, will display this, and will show true filename in popover + display_name?: string; // if given, will display this, and will show true filename in popover size: number; // sometimes is NOT known! time: number; issymlink: boolean; @@ -46,7 +46,6 @@ interface Props { selected: boolean; color: string; mask: boolean; - public_data: object; is_public: boolean; current_path: string; actions: ProjectActions; @@ -211,7 +210,9 @@ export const FileRow: React.FC = React.memo((props) => { explicit: true, }); if (foreground) { - props.actions.set_file_search(""); + // delay slightly since it looks weird to see the full listing right when you click on a file + const actions = props.actions; + setTimeout(() => actions.set_file_search(""), 10); } } } diff --git a/src/packages/frontend/project_actions.ts b/src/packages/frontend/project_actions.ts index 296678b9cb..316cc52280 100644 --- a/src/packages/frontend/project_actions.ts +++ b/src/packages/frontend/project_actions.ts @@ -2472,7 +2472,7 @@ export class ProjectActions extends Actions { return this.getFilesCache(path); }; - private getCacheId = (compute_server_id?: number) => { + getCacheId = (compute_server_id?: number) => { return getCacheId({ project_id: this.project_id, compute_server_id: diff --git a/src/packages/util/types/directory-listing.ts b/src/packages/util/types/directory-listing.ts index 4b9f58da66..3352a99268 100644 --- a/src/packages/util/types/directory-listing.ts +++ b/src/packages/util/types/directory-listing.ts @@ -2,8 +2,11 @@ export interface DirectoryListingEntry { name: string; isdir?: boolean; issymlink?: boolean; - link_target?: string; // set if issymlink is true and we're able to determine the target of the link - size?: number; // bytes for file, number of entries for directory (*including* . and ..). + // set if issymlink is true and we're able to determine the target of the link + link_target?: string; + // bytes for file, number of entries for directory (*including* . and ..). + size?: number; mtime?: number; error?: string; + mask?: boolean; } From df6f9db0a739451c6e1bee2c76475278fb324559 Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 29 Jul 2025 19:12:32 +0000 Subject: [PATCH 128/270] fix crash in socket reconnect after close --- src/packages/conat/socket/base.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/packages/conat/socket/base.ts b/src/packages/conat/socket/base.ts index 24d57ac148..3f98df515f 100644 --- a/src/packages/conat/socket/base.ts +++ b/src/packages/conat/socket/base.ts @@ -116,14 +116,16 @@ export abstract class ConatSocketBase extends EventEmitter { } if (this.reconnection) { setTimeout(() => { - this.connect(); + if (this.state != "closed") { + this.connect(); + } }, RECONNECT_DELAY); } }; connect = async () => { - if (this.state != "disconnected") { - // already connected + if (this.state != "disconnected" || !this.client) { + // already connected or closed return; } this.setState("connecting"); From ecc2f494e29926dba757ca484a9e5d5d1b5a7ded Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 29 Jul 2025 19:29:36 +0000 Subject: [PATCH 129/270] action bar -- show even if project is not running --- .../frontend/project/explorer/action-bar.tsx | 105 ++++++++++-------- 1 file changed, 57 insertions(+), 48 deletions(-) diff --git a/src/packages/frontend/project/explorer/action-bar.tsx b/src/packages/frontend/project/explorer/action-bar.tsx index f427ff1864..98d967f393 100644 --- a/src/packages/frontend/project/explorer/action-bar.tsx +++ b/src/packages/frontend/project/explorer/action-bar.tsx @@ -15,7 +15,10 @@ import { CustomSoftwareInfo } from "@cocalc/frontend/custom-software/info-bar"; import { type ComputeImages } from "@cocalc/frontend/custom-software/init"; import { IS_MOBILE } from "@cocalc/frontend/feature"; import { labels } from "@cocalc/frontend/i18n"; -import { file_actions, type ProjectActions } from "@cocalc/frontend/project_store"; +import { + file_actions, + type ProjectActions, +} from "@cocalc/frontend/project_store"; import * as misc from "@cocalc/util/misc"; import { COLORS } from "@cocalc/util/theme"; import { useProjectContext } from "../context"; @@ -40,22 +43,33 @@ interface Props { project_is_running?: boolean; } -export function ActionBar(props: Props) { +export function ActionBar({ + project_id, + checked_files, + listing, + current_path, + project_map, + images, + actions, + available_features, + show_custom_software_reset, + project_is_running, +}: Props) { const intl = useIntl(); const [showLabels, setShowLabels] = useState(true); const { mainWidthPx } = useProjectContext(); const buttonRef = useRef(null); - const widthThld = useRef(0); + const tableHeaderWidth = useRef(0); const student_project_functionality = useStudentProjectFunctionality( - props.actions.project_id, + actions.project_id, ); if (student_project_functionality.disableActions) { return
; } useEffect(() => { - const btnbar = buttonRef.current; - if (btnbar == null) return; + const buttonBar = buttonRef.current; + if (buttonBar == null) return; const resizeObserver = new ResizeObserver( throttle( (entries) => { @@ -65,8 +79,8 @@ export function ActionBar(props: Props) { // (e.g. german buttons were cutoff all the time), but could need more tweaking if (showLabels && width > mainWidthPx + 100) { setShowLabels(false); - widthThld.current = width; - } else if (!showLabels && width < widthThld.current - 1) { + tableHeaderWidth.current = width; + } else if (!showLabels && width < tableHeaderWidth.current - 1) { setShowLabels(true); } } @@ -75,22 +89,20 @@ export function ActionBar(props: Props) { { leading: false, trailing: true }, ), ); - resizeObserver.observe(btnbar); + resizeObserver.observe(buttonBar); return () => { resizeObserver.disconnect(); }; }, [mainWidthPx, buttonRef.current]); function clear_selection(): void { - props.actions.set_all_files_unchecked(); + actions.set_all_files_unchecked(); } function check_all_click_handler(): void { - if (props.checked_files.size === 0) { - props.actions.set_file_list_checked( - props.listing.map((file) => - misc.path_to_file(props.current_path ?? "", file.name), - ), + if (checked_files.size === 0) { + actions.set_file_list_checked( + listing.map((file) => misc.path_to_file(current_path ?? "", file.name)), ); } else { clear_selection(); @@ -98,11 +110,11 @@ export function ActionBar(props: Props) { } function render_check_all_button(): React.JSX.Element | undefined { - if (props.listing.length === 0) { + if (listing.length === 0) { return; } - const checked = props.checked_files.size > 0; + const checked = checked_files.size > 0; const button_text = intl.formatMessage( { id: "project.explorer.action-bar.check_all.button", @@ -114,10 +126,10 @@ export function ActionBar(props: Props) { ); let button_icon; - if (props.checked_files.size === 0) { + if (checked_files.size === 0) { button_icon = "square-o"; } else { - if (props.checked_files.size >= props.listing.length) { + if (checked_files.size >= listing.length) { button_icon = "check-square-o"; } else { button_icon = "minus-square-o"; @@ -136,11 +148,11 @@ export function ActionBar(props: Props) { } function render_currently_selected(): React.JSX.Element | undefined { - if (props.listing.length === 0) { + if (listing.length === 0) { return; } - const checked = props.checked_files.size; - const total = props.listing.length; + const checked = checked_files.size; + const total = listing.length; const style = ROW_INFO_STYLE; if (checked === 0) { @@ -186,12 +198,12 @@ export function ActionBar(props: Props) { function render_action_button(name: string): React.JSX.Element { const disabled = isDisabledSnapshots(name) && - (props.current_path != null - ? props.current_path.startsWith(".snapshots") + (current_path != null + ? current_path.startsWith(".snapshots") : undefined); const obj = file_actions[name]; const handle_click = (_e: React.MouseEvent) => { - props.actions.set_file_action(name); + actions.set_file_action(name); }; return ( @@ -213,16 +225,13 @@ export function ActionBar(props: Props) { | "copy" | "share" )[]; - if (!props.project_is_running) { + if (checked_files.size === 0) { return; - } - if (props.checked_files.size === 0) { - return; - } else if (props.checked_files.size === 1) { + } else if (checked_files.size === 1) { let isdir; - const item = props.checked_files.first(); - for (const file of props.listing) { - if (misc.path_to_file(props.current_path ?? "", file.name) === item) { + const item = checked_files.first(); + for (const file of listing) { + if (misc.path_to_file(current_path ?? "", file.name) === item) { ({ isdir } = file); } } @@ -246,25 +255,25 @@ export function ActionBar(props: Props) { } function render_button_area(): React.JSX.Element | undefined { - if (props.checked_files.size === 0) { + if (checked_files.size === 0) { if ( - props.project_id == null || - props.images == null || - props.project_map == null || - props.available_features == null + project_id == null || + images == null || + project_map == null || + available_features == null ) { return; } return ( ); @@ -272,7 +281,7 @@ export function ActionBar(props: Props) { return render_action_buttons(); } } - if (props.checked_files.size === 0 && IS_MOBILE) { + if (checked_files.size === 0 && IS_MOBILE) { return null; } return ( @@ -280,13 +289,13 @@ export function ActionBar(props: Props) {
- {props.project_is_running ? render_check_all_button() : undefined} + {render_check_all_button()} {render_button_area()}
- {props.project_is_running ? render_currently_selected() : undefined} + {render_currently_selected()}
); From b297394beea6f4ac7b0905f859c7502cd2e4044d Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 29 Jul 2025 21:08:48 +0000 Subject: [PATCH 130/270] ts --- .../frontend/project/explorer/explorer.tsx | 23 +++++++++++-------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/src/packages/frontend/project/explorer/explorer.tsx b/src/packages/frontend/project/explorer/explorer.tsx index 9a0d7bf19b..0289ce991f 100644 --- a/src/packages/frontend/project/explorer/explorer.tsx +++ b/src/packages/frontend/project/explorer/explorer.tsx @@ -139,16 +139,11 @@ export function Explorer() { actions?.setState({ numDisplayedFiles: listing?.length ?? 0 }); }, [listing?.length]); - if (listingError) { - return ; - } - - if (actions == null || project_map == null) { - return ; - } - useEffect(() => { const handle_files_key_down = (e): void => { + if (actions == null) { + return; + } if (e.key === "Shift") { setShiftIsDown(true); } else if (e.key == "ArrowUp") { @@ -194,6 +189,14 @@ export function Explorer() { }; }, [project_id, current_path]); + if (listingError) { + return ; + } + + if (actions == null || project_map == null) { + return ; + } + const create_file = (ext, switch_over) => { if (switch_over == undefined) { switch_over = true; @@ -471,7 +474,9 @@ export function Explorer() { className="smc-vfill" > {listing == null ? ( - +
+ +
) : ( Date: Wed, 30 Jul 2025 01:07:38 +0000 Subject: [PATCH 131/270] typo --- src/packages/conat/core/cluster.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/packages/conat/core/cluster.ts b/src/packages/conat/core/cluster.ts index 863f2cd943..e4e8c8efbe 100644 --- a/src/packages/conat/core/cluster.ts +++ b/src/packages/conat/core/cluster.ts @@ -272,8 +272,8 @@ export async function trimClusterStreams( minAge: number, ): Promise<{ seqsInterest: number[]; seqsSticky: number[] }> { const { interest, sticky } = streams; - // First deal with interst - // we iterate over the interest stream checking for subjects + // First deal with interest. + // We iterate over the interest stream checking for subjects // with no current interest at all; in such cases it is safe // to purge them entirely from the stream. const seqs: number[] = []; From 47699d88080a4448ce449cfd538300551a2e8709 Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 30 Jul 2025 03:11:12 +0000 Subject: [PATCH 132/270] sync: fix hash_of_saved_version --- src/packages/sync/editor/generic/sync-doc.ts | 31 ++++++++++++-------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/src/packages/sync/editor/generic/sync-doc.ts b/src/packages/sync/editor/generic/sync-doc.ts index 2295693231..a69ae7a801 100644 --- a/src/packages/sync/editor/generic/sync-doc.ts +++ b/src/packages/sync/editor/generic/sync-doc.ts @@ -1274,7 +1274,7 @@ export class SyncDoc extends EventEmitter { try { stats = await this.fs.stat(this.path); } catch (err) { - this.lastDiskValue = undefined; // nonexistent or don't know + this.valueOnDisk = undefined; // nonexistent or don't know if (err.code == "ENOENT") { // path does not exist -- nothing further to do return false; @@ -2243,7 +2243,7 @@ export class SyncDoc extends EventEmitter { let contents; try { contents = await this.fs.readFile(this.path, "utf8"); - this.lastDiskValue = contents; + this.valueOnDisk = contents; dbg("file exists"); size = contents.length; this.from_str(contents); @@ -2286,14 +2286,15 @@ export class SyncDoc extends EventEmitter { return this.hasUnsavedChanges(); }; - // Returns hash of last version saved to disk (as far as we know). + // Returns hash of last version that we saved to disk or undefined + // if we haven't saved yet. + // NOTE: this does not take into account saving by another client + // anymore; it used to, but that made things much more complicated. hash_of_saved_version = (): number | undefined => { - if (!this.isReady()) { + if (!this.isReady() || this.valueOnDisk == null) { return; } - return this.syncstring_table_get_one().getIn(["save", "hash"]) as - | number - | undefined; + return hash_string(this.valueOnDisk); }; /* Return hash of the live version of the document, @@ -2359,9 +2360,13 @@ export class SyncDoc extends EventEmitter { return true; }; - private lastDiskValue: string | undefined = undefined; + // valueOnDisk = value of the file on disk, if known. If there's an + // event indicating what was on disk may have changed, this + // this.valueOnDisk is deleted until the new version is loaded. + private valueOnDisk: string | undefined = undefined; + private hasUnsavedChanges = (): boolean => { - return this.lastDiskValue != this.to_str(); + return this.valueOnDisk != this.to_str(); }; writeFile = async () => { @@ -2389,7 +2394,7 @@ export class SyncDoc extends EventEmitter { await this.fs.writeFile(this.path, value); const lastChanged = this.last_changed(); await this.fs.utimes(this.path, lastChanged / 1000, lastChanged / 1000); - this.lastDiskValue = value; + this.valueOnDisk = value; }; /* Initiates a save of file to disk, then waits for the @@ -2695,10 +2700,10 @@ export class SyncDoc extends EventEmitter { this.emit("watching"); for await (const { eventType, ignore } of this.fileWatcher) { if (this.isClosed()) return; - // we don't know what's on disk anymore, - this.lastDiskValue = undefined; - //console.log("got change", eventType); if (!ignore) { + // we don't know what's on disk anymore, + this.valueOnDisk = undefined; + // and we should find out! this.readFileDebounced(); } if (eventType == "rename") { From 8752002e0c9b0f34d70fc7a031a08e325dec4b6d Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 30 Jul 2025 03:28:42 +0000 Subject: [PATCH 133/270] sync: missing save-to-disk event --- src/packages/sync/editor/generic/sync-doc.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/packages/sync/editor/generic/sync-doc.ts b/src/packages/sync/editor/generic/sync-doc.ts index a69ae7a801..abc9515e91 100644 --- a/src/packages/sync/editor/generic/sync-doc.ts +++ b/src/packages/sync/editor/generic/sync-doc.ts @@ -2395,6 +2395,7 @@ export class SyncDoc extends EventEmitter { const lastChanged = this.last_changed(); await this.fs.utimes(this.path, lastChanged / 1000, lastChanged / 1000); this.valueOnDisk = value; + this.emit("save-to-disk"); }; /* Initiates a save of file to disk, then waits for the From b8eb20c7354290b51984e458c604f4f414f77078 Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 30 Jul 2025 04:08:14 +0000 Subject: [PATCH 134/270] sync: reimplement read-write versus read-only handling --- src/packages/conat/files/fs.ts | 2 +- .../frame-editors/frame-tree/save-button.tsx | 4 +- .../frame-editors/frame-tree/title-bar.tsx | 4 +- src/packages/sync/editor/generic/sync-doc.ts | 90 ++++++++++++++----- 4 files changed, 74 insertions(+), 26 deletions(-) diff --git a/src/packages/conat/files/fs.ts b/src/packages/conat/files/fs.ts index c7ce7280f8..ea7f567f10 100644 --- a/src/packages/conat/files/fs.ts +++ b/src/packages/conat/files/fs.ts @@ -140,7 +140,7 @@ interface IStats { birthtime: Date; } -class Stats { +export class Stats { dev: number; ino: number; mode: number; diff --git a/src/packages/frontend/frame-editors/frame-tree/save-button.tsx b/src/packages/frontend/frame-editors/frame-tree/save-button.tsx index 4874a04c2b..0a9585fe4d 100644 --- a/src/packages/frontend/frame-editors/frame-tree/save-button.tsx +++ b/src/packages/frontend/frame-editors/frame-tree/save-button.tsx @@ -47,7 +47,7 @@ export function SaveButton({ const intl = useIntl(); const label = useMemo(() => { - if (!no_labels) { + if (!no_labels || read_only) { return intl.formatMessage(labels.frame_editors_title_bar_save_label, { type: is_public ? "is_public" : read_only ? "read_only" : "save", }); @@ -67,7 +67,7 @@ export function SaveButton({ ); function renderLabel() { - if (!no_labels && label) { + if (label) { return {` ${label}`}; } } diff --git a/src/packages/frontend/frame-editors/frame-tree/title-bar.tsx b/src/packages/frontend/frame-editors/frame-tree/title-bar.tsx index 67bb8337b8..eed5a2bdce 100644 --- a/src/packages/frontend/frame-editors/frame-tree/title-bar.tsx +++ b/src/packages/frontend/frame-editors/frame-tree/title-bar.tsx @@ -717,8 +717,8 @@ export function FrameTitleBar(props: FrameTitleBarProps) { label === APPLICATION_MENU ? manageCommands.applicationMenuTitle() : isIntlMessage(label) - ? intl.formatMessage(label) - : label + ? intl.formatMessage(label) + : label } items={v} /> diff --git a/src/packages/sync/editor/generic/sync-doc.ts b/src/packages/sync/editor/generic/sync-doc.ts index abc9515e91..2fe3749193 100644 --- a/src/packages/sync/editor/generic/sync-doc.ts +++ b/src/packages/sync/editor/generic/sync-doc.ts @@ -40,6 +40,8 @@ import { decodeUUIDtoNum, } from "@cocalc/util/compute/manager"; +const STAT_DEBOUNCE = 10000; + import { DEFAULT_SNAPSHOT_INTERVAL } from "@cocalc/util/db-schema/syncstring-schema"; type XPatch = any; @@ -83,7 +85,7 @@ import { isTestClient, patch_cmp } from "./util"; import mergeDeep from "@cocalc/util/immutable-deep-merge"; import { JUPYTER_SYNCDB_EXTENSIONS } from "@cocalc/util/jupyter/names"; import { LegacyHistory } from "./legacy"; -import { type Filesystem } from "@cocalc/conat/files/fs"; +import { type Filesystem, type Stats } from "@cocalc/conat/files/fs"; import { getLogger } from "@cocalc/conat/client"; const DEBUG = false; @@ -1267,12 +1269,10 @@ export class SyncDoc extends EventEmitter { }; private loadFromDiskIfNewer = async (): Promise => { - // [ ] TODO: readonly handling... - if (this.fs == null) throw Error("bug"); const dbg = this.dbg("loadFromDiskIfNewer"); let stats; try { - stats = await this.fs.stat(this.path); + stats = await this.stat(); } catch (err) { this.valueOnDisk = undefined; // nonexistent or don't know if (err.code == "ENOENT") { @@ -2236,7 +2236,6 @@ export class SyncDoc extends EventEmitter { }; readFile = reuseInFlight(async (): Promise => { - if (this.fs == null) throw Error("bug"); const dbg = this.dbg("readFile"); let size: number; @@ -2264,12 +2263,47 @@ export class SyncDoc extends EventEmitter { }); is_read_only = (): boolean => { - // [ ] TODO - return this.syncstring_table_get_one().get("read_only"); + if (this.stats) { + return isReadOnlyForOwner(this.stats); + } else { + return false; + } + }; + + private stats?: Stats; + stat = async (): Promise => { + const prevStats = this.stats; + this.stats = (await this.fs.stat(this.path)) as Stats; + if (prevStats?.mode != this.stats.mode) { + this.emit("metadata-change"); + } + return this.stats; }; + debouncedStat = debounce( + async () => { + try { + await this.stat(); + } catch {} + }, + STAT_DEBOUNCE, + { leading: true, trailing: true }, + ); + wait_until_read_only_known = async (): Promise => { - // [ ] TODO + await until(async () => { + if (this.isClosed()) { + return true; + } + if (this.stats != null) { + return true; + } + try { + await this.stat(); + return true; + } catch {} + return false; + }); }; /* Returns true if the current live version of this document has @@ -2376,9 +2410,14 @@ export class SyncDoc extends EventEmitter { return; } dbg(); - if (this.fs == null) { - throw Error("bug"); + if (this.is_read_only()) { + await this.stat(); + if (this.is_read_only()) { + // it is definitely still read only. + return; + } } + const value = this.to_str(); // include {ignore:true} with events for this long, // so no clients waste resources loading in response to us saving @@ -2391,7 +2430,17 @@ export class SyncDoc extends EventEmitter { } if (this.isClosed()) return; this.last_save_to_disk_time = new Date(); - await this.fs.writeFile(this.path, value); + try { + await this.fs.writeFile(this.path, value); + } catch (err) { + if (err.code == "EACCES") { + try { + // update read only knowledge -- that may have caused save error. + await this.stat(); + } catch {} + } + throw err; + } const lastChanged = this.last_changed(); await this.fs.utimes(this.path, lastChanged / 1000, lastChanged / 1000); this.valueOnDisk = value; @@ -2674,6 +2723,7 @@ export class SyncDoc extends EventEmitter { try { this.emit("handle-file-change"); await this.readFile(); + await this.stat(); } catch {} }, WATCH_DEBOUNCE, @@ -2685,9 +2735,6 @@ export class SyncDoc extends EventEmitter { private fileWatcher?: any; private initFileWatcher = async () => { - if (this.fs == null) { - throw Error("this.fs must be defined"); - } // use this.fs interface to watch path for changes -- we try once: try { this.fileWatcher = await this.fs.watch(this.path, { unique: true }); @@ -2706,6 +2753,8 @@ export class SyncDoc extends EventEmitter { this.valueOnDisk = undefined; // and we should find out! this.readFileDebounced(); + } else { + this.debouncedStat(); } if (eventType == "rename") { break; @@ -2742,11 +2791,8 @@ export class SyncDoc extends EventEmitter { // false if it definitely does not, and throws exception otherwise, // e.g., network error. private fileExists = async (): Promise => { - if (this.fs == null) { - throw Error("bug -- fs must be defined"); - } try { - await this.fs.stat(this.path); + await this.stat(); return true; } catch (err) { if (err.code == "ENOENT") { @@ -2759,9 +2805,6 @@ export class SyncDoc extends EventEmitter { private closeIfFileDeleted = async () => { if (this.isClosed()) return; - if (this.fs == null) { - throw Error("bug -- fs must be defined"); - } const start = Date.now(); const threshold = this.deletedThreshold ?? DELETED_THRESHOLD; while (true) { @@ -2808,3 +2851,8 @@ function isCompletePatchStream(dstream) { } return false; } + +function isReadOnlyForOwner(stats): boolean { + // 0o200 is owner write permission + return (stats.mode & 0o200) === 0; +} From 3520607562d98b2d6ec2fc137bc0e9bb5b39e58c Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 30 Jul 2025 04:19:34 +0000 Subject: [PATCH 135/270] copy between projects -- hide activity indicator when done --- src/packages/frontend/project_actions.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/packages/frontend/project_actions.ts b/src/packages/frontend/project_actions.ts index 316cc52280..683b63c1ec 100644 --- a/src/packages/frontend/project_actions.ts +++ b/src/packages/frontend/project_actions.ts @@ -2407,8 +2407,8 @@ export class ProjectActions extends Actions { opts0.timeout = 90 * 1000; await webapp_client.project_client.copy_path_between_projects(opts0); }; - await awaitMap(src, 5, f); - this._finish_exec(id); + await awaitMap(withSlashes, 5, f); + this.set_activity({ id, stop: "" }); }; public async rename_file(opts: { From 78ed4917d3319f327f2c2e0d91e3c92d201e0759 Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 30 Jul 2025 18:18:39 +0000 Subject: [PATCH 136/270] state issue with listing --- src/packages/frontend/project/explorer/explorer.tsx | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/packages/frontend/project/explorer/explorer.tsx b/src/packages/frontend/project/explorer/explorer.tsx index 0289ce991f..50fc1b46da 100644 --- a/src/packages/frontend/project/explorer/explorer.tsx +++ b/src/packages/frontend/project/explorer/explorer.tsx @@ -140,6 +140,9 @@ export function Explorer() { }, [listing?.length]); useEffect(() => { + if (listing == null) { + return; + } const handle_files_key_down = (e): void => { if (actions == null) { return; @@ -187,7 +190,7 @@ export function Explorer() { $(window).off("keydown", handle_files_key_down); $(window).off("keyup", handle_files_key_up); }; - }, [project_id, current_path]); + }, [project_id, current_path, listing]); if (listingError) { return ; From a12ea7c32dbfb266ce126ae52fa17ff89b79a1e8 Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 30 Jul 2025 19:00:34 +0000 Subject: [PATCH 137/270] focus explorer filter on directory change --- .../frontend/components/search-input.tsx | 80 +++++++++++-------- .../frontend/project/explorer/explorer.tsx | 4 + .../frontend/project/explorer/search-bar.tsx | 1 + 3 files changed, 50 insertions(+), 35 deletions(-) diff --git a/src/packages/frontend/components/search-input.tsx b/src/packages/frontend/components/search-input.tsx index e3839ca547..446c788d96 100644 --- a/src/packages/frontend/components/search-input.tsx +++ b/src/packages/frontend/components/search-input.tsx @@ -10,13 +10,7 @@ */ import { Input, InputRef } from "antd"; - -import { - React, - useEffect, - useRef, - useState, -} from "@cocalc/frontend/app-framework"; +import { CSSProperties, useEffect, useRef, useState } from "react"; interface Props { size?; @@ -31,21 +25,37 @@ interface Props { on_down?: () => void; on_up?: () => void; on_escape?: (value: string) => void; - style?: React.CSSProperties; - input_class?: string; + style?: CSSProperties; autoFocus?: boolean; autoSelect?: boolean; placeholder?: string; - focus?: number; // if this changes, focus the search box. + focus?; // if this changes, focus the search box. status?: "warning" | "error"; } -export const SearchInput: React.FC = React.memo((props) => { - const [value, setValue] = useState( - props.value ?? props.default_value ?? "", - ); +export function SearchInput({ + size, + default_value, + value: value0, + on_change, + on_clear, + on_submit, + buttonAfter, + disabled, + clear_on_submit, + on_down, + on_up, + on_escape, + style, + autoFocus, + autoSelect, + placeholder, + focus, + status, +}: Props) { + const [value, setValue] = useState(value0 ?? default_value ?? ""); // if value changes, we update as well! - useEffect(() => setValue(props.value ?? ""), [props.value]); + useEffect(() => setValue(value ?? ""), [value]); const [ctrl_down, set_ctrl_down] = useState(false); const [shift_down, set_shift_down] = useState(false); @@ -53,7 +63,7 @@ export const SearchInput: React.FC = React.memo((props) => { const input_ref = useRef(null); useEffect(() => { - if (props.autoSelect && input_ref.current) { + if (autoSelect && input_ref.current) { try { input_ref.current?.select(); } catch (_) {} @@ -71,20 +81,20 @@ export const SearchInput: React.FC = React.memo((props) => { function clear_value(): void { setValue(""); - props.on_change?.("", get_opts()); - props.on_clear?.(); + on_change?.("", get_opts()); + on_clear?.(); } function submit(e?): void { if (e != null) { e.preventDefault(); } - if (typeof props.on_submit === "function") { - props.on_submit(value, get_opts()); + if (typeof on_submit === "function") { + on_submit(value, get_opts()); } - if (props.clear_on_submit) { + if (clear_on_submit) { clear_value(); - props.on_change?.(value, get_opts()); + on_change?.(value, get_opts()); } } @@ -94,10 +104,10 @@ export const SearchInput: React.FC = React.memo((props) => { escape(); break; case 40: - props.on_down?.(); + on_down?.(); break; case 38: - props.on_up?.(); + on_up?.(); break; case 17: set_ctrl_down(true); @@ -120,35 +130,35 @@ export const SearchInput: React.FC = React.memo((props) => { } function escape(): void { - if (typeof props.on_escape === "function") { - props.on_escape(value); + if (typeof on_escape === "function") { + on_escape(value); } clear_value(); } return ( { e.preventDefault(); const value = e.target?.value ?? ""; setValue(value); - props.on_change?.(value, get_opts()); + on_change?.(value, get_opts()); if (!value) clear_value(); }} onKeyDown={key_down} onKeyUp={key_up} - disabled={props.disabled} - enterButton={props.buttonAfter} - status={props.status} + disabled={disabled} + enterButton={buttonAfter} + status={status} /> ); -}); +} diff --git a/src/packages/frontend/project/explorer/explorer.tsx b/src/packages/frontend/project/explorer/explorer.tsx index 50fc1b46da..e682bfa1f4 100644 --- a/src/packages/frontend/project/explorer/explorer.tsx +++ b/src/packages/frontend/project/explorer/explorer.tsx @@ -159,6 +159,10 @@ export function Explorer() { } else if (e.key == "ArrowDown") { actions.increment_selected_file_index(); } else if (e.key == "Enter") { + if (file_search.startsWith("/")) { + // running a terminal command + return; + } const n = redux.getProjectStore(project_id).get("selected_file_index") ?? 0; const x = listing?.[n]; diff --git a/src/packages/frontend/project/explorer/search-bar.tsx b/src/packages/frontend/project/explorer/search-bar.tsx index 7070669954..535e426536 100644 --- a/src/packages/frontend/project/explorer/search-bar.tsx +++ b/src/packages/frontend/project/explorer/search-bar.tsx @@ -274,6 +274,7 @@ export const SearchBar = memo( on_submit={search_submit} on_clear={on_clear} disabled={disabled || !!ext_selection} + focus={current_path} /> {render_file_creation_error()} {render_help_info()} From 850ce5d4e3fb81fbf9683042ce4a565946f93293 Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 30 Jul 2025 19:37:07 +0000 Subject: [PATCH 138/270] masked files -- make that work again --- .../project/explorer/compute-file-masks.ts | 20 +++++++++---------- .../frontend/project/explorer/explorer.tsx | 1 + .../frontend/project/listing/use-listing.ts | 10 +++++++++- .../frontend/project/page/flyouts/files.tsx | 4 ++-- 4 files changed, 22 insertions(+), 13 deletions(-) diff --git a/src/packages/frontend/project/explorer/compute-file-masks.ts b/src/packages/frontend/project/explorer/compute-file-masks.ts index 6b8e4d95d0..f7162818f5 100644 --- a/src/packages/frontend/project/explorer/compute-file-masks.ts +++ b/src/packages/frontend/project/explorer/compute-file-masks.ts @@ -54,7 +54,7 @@ const MASKED_FILE_EXTENSIONS = { * the general outcome of this function is to set for some file entry objects * in "listing" the attribute .mask=true */ -export function compute_file_masks(listing: DirectoryListing): void { +export function computeFileMasks(listing: DirectoryListing): void { // map filename to file for easier lookup const filename_map: { [name: string]: DirectoryListingEntry } = dict( listing.map((item) => [item.name, item]), @@ -75,29 +75,29 @@ export function compute_file_masks(listing: DirectoryListing): void { for (let mask_ext of MASKED_FILE_EXTENSIONS[ext] ?? []) { // check each possible compiled extension - let bn; // derived basename + let derivedBasename; // some uppercase-strings have special meaning if (startswith(mask_ext, "NODOT")) { - bn = basename.slice(0, -1); // exclude the trailing dot + derivedBasename = basename.slice(0, -1); // exclude the trailing dot mask_ext = mask_ext.slice("NODOT".length); } else if (mask_ext.indexOf("FILENAME") >= 0) { - bn = mask_ext.replace("FILENAME", filename); + derivedBasename = mask_ext.replace("FILENAME", filename); mask_ext = ""; } else if (mask_ext.indexOf("BASENAME") >= 0) { - bn = mask_ext.replace("BASENAME", basename.slice(0, -1)); + derivedBasename = mask_ext.replace("BASENAME", basename.slice(0, -1)); mask_ext = ""; } else if (mask_ext.indexOf("BASEDASHNAME") >= 0) { // BASEDASHNAME is like BASENAME, but replaces spaces by dashes // https://github.com/sagemathinc/cocalc/issues/3229 const fragment = basename.slice(0, -1).replace(/ /g, "-"); - bn = mask_ext.replace("BASEDASHNAME", fragment); + derivedBasename = mask_ext.replace("BASEDASHNAME", fragment); mask_ext = ""; } else { - bn = basename; + derivedBasename = basename; } - const mask_fn = `${bn}${mask_ext}`; - if (filename_map[mask_fn] != null) { - filename_map[mask_fn].mask = true; + const maskFilename = `${derivedBasename}${mask_ext}`; + if (filename_map[maskFilename] != null) { + filename_map[maskFilename].mask = true; } } } diff --git a/src/packages/frontend/project/explorer/explorer.tsx b/src/packages/frontend/project/explorer/explorer.tsx index e682bfa1f4..edb7cf3528 100644 --- a/src/packages/frontend/project/explorer/explorer.tsx +++ b/src/packages/frontend/project/explorer/explorer.tsx @@ -122,6 +122,7 @@ export function Explorer() { path: current_path, ...sortDesc(active_file_sort), cacheId: actions?.getCacheId(compute_server_id), + mask: true, }); const showHidden = useTypedRedux({ project_id }, "show_hidden"); const showMasked = useTypedRedux({ project_id }, "show_masked"); diff --git a/src/packages/frontend/project/listing/use-listing.ts b/src/packages/frontend/project/listing/use-listing.ts index da5a1870ac..a4c481c52f 100644 --- a/src/packages/frontend/project/listing/use-listing.ts +++ b/src/packages/frontend/project/listing/use-listing.ts @@ -12,6 +12,7 @@ import { type FilesystemClient } from "@cocalc/conat/files/fs"; import { type ConatError } from "@cocalc/conat/core/client"; import type { JSONValue } from "@cocalc/util/types"; import { getFiles, type Files } from "./use-files"; +import { computeFileMasks } from "@cocalc/frontend/project/explorer/compute-file-masks"; export type SortField = "name" | "mtime" | "size" | "type"; export type SortDirection = "asc" | "desc"; @@ -39,6 +40,7 @@ export default function useListing({ sortDirection = "asc", throttleUpdate, cacheId, + mask, }: { // fs = undefined is supported and just waits until you provide a fs that is defined fs?: FilesystemClient | null; @@ -47,6 +49,7 @@ export default function useListing({ sortDirection?: SortDirection; throttleUpdate?: number; cacheId?: JSONValue; + mask?: boolean; }): { listing: null | DirectoryListingEntry[]; error: null | ConatError; @@ -60,7 +63,7 @@ export default function useListing({ }); const listing = useMemo(() => { - return filesToListing({ files, sortField, sortDirection }); + return filesToListing({ files, sortField, sortDirection, mask }); }, [sortField, sortDirection, files]); return { listing, error, refresh }; @@ -70,10 +73,12 @@ function filesToListing({ files, sortField = "name", sortDirection = "asc", + mask, }: { files?: Files | null; sortField?: SortField; sortDirection?: SortDirection; + mask?: boolean; }): null | DirectoryListingEntry[] { if (files == null) { return null; @@ -100,5 +105,8 @@ function filesToListing({ } else { console.warn(`invalid sort direction: '${sortDirection}'`); } + if (mask) { + computeFileMasks(v); + } return v; } diff --git a/src/packages/frontend/project/page/flyouts/files.tsx b/src/packages/frontend/project/page/flyouts/files.tsx index 4ed6d1f2a7..55e6a5d2cf 100644 --- a/src/packages/frontend/project/page/flyouts/files.tsx +++ b/src/packages/frontend/project/page/flyouts/files.tsx @@ -26,7 +26,7 @@ import { file_options } from "@cocalc/frontend/editor-tmp"; import { FileUploadWrapper } from "@cocalc/frontend/file-upload"; import { should_open_in_foreground } from "@cocalc/frontend/lib/should-open-in-foreground"; import { useProjectContext } from "@cocalc/frontend/project/context"; -import { compute_file_masks } from "@cocalc/frontend/project/explorer/compute-file-masks"; +import { computeFileMasks } from "@cocalc/frontend/project/explorer/compute-file-masks"; import { DirectoryListing, DirectoryListingEntry, @@ -164,7 +164,7 @@ export function FilesFlyout({ const files = directoryListing; if (files == null) return EMPTY_LISTING; let activeFile: DirectoryListingEntry | null = null; - compute_file_masks(files); + computeFileMasks(files); const searchWords = file_search.trim().toLowerCase(); const processedFiles: DirectoryListingEntry[] = files From 631502c0bc60e96e9ccbc8623eaddac2d2e72d50 Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 30 Jul 2025 19:47:31 +0000 Subject: [PATCH 139/270] fix little bug I just introduced in SearchInput --- src/packages/frontend/components/search-input.tsx | 4 +++- src/packages/frontend/project/explorer/search-bar.tsx | 4 ++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/packages/frontend/components/search-input.tsx b/src/packages/frontend/components/search-input.tsx index 446c788d96..88d017315a 100644 --- a/src/packages/frontend/components/search-input.tsx +++ b/src/packages/frontend/components/search-input.tsx @@ -55,7 +55,9 @@ export function SearchInput({ }: Props) { const [value, setValue] = useState(value0 ?? default_value ?? ""); // if value changes, we update as well! - useEffect(() => setValue(value ?? ""), [value]); + useEffect(() => { + setValue(value0 ?? ""); + }, [value0]); const [ctrl_down, set_ctrl_down] = useState(false); const [shift_down, set_shift_down] = useState(false); diff --git a/src/packages/frontend/project/explorer/search-bar.tsx b/src/packages/frontend/project/explorer/search-bar.tsx index 535e426536..0ede8f79d6 100644 --- a/src/packages/frontend/project/explorer/search-bar.tsx +++ b/src/packages/frontend/project/explorer/search-bar.tsx @@ -81,6 +81,10 @@ export const SearchBar = memo( undefined, ); + useEffect(() => { + actions.set_file_search(""); + }, [current_path]); + useEffect(() => { if (cmd == null) return; const { input, id } = cmd; From ecdf99252a1ff3dceb4c945d4091832412cee882 Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 30 Jul 2025 20:02:43 +0000 Subject: [PATCH 140/270] get rid of file mask toggle button for full page file explorer since it's advanced/complicated (leave it in for flyout panels) in for --- .../project/explorer/compute-file-masks.ts | 19 +++++++------ .../frontend/project/explorer/explorer.tsx | 5 ++-- .../project/explorer/misc-side-buttons.tsx | 28 ------------------- .../project/listing/filter-listing.ts | 5 ---- .../frontend/project/listing/use-listing.ts | 1 + src/packages/util/db-schema/accounts.ts | 7 ++++- 6 files changed, 19 insertions(+), 46 deletions(-) diff --git a/src/packages/frontend/project/explorer/compute-file-masks.ts b/src/packages/frontend/project/explorer/compute-file-masks.ts index f7162818f5..50e6d94b67 100644 --- a/src/packages/frontend/project/explorer/compute-file-masks.ts +++ b/src/packages/frontend/project/explorer/compute-file-masks.ts @@ -4,10 +4,10 @@ */ import { derive_rmd_output_filename } from "@cocalc/frontend/frame-editors/rmd-editor/utils"; -import { dict, filename_extension, startswith } from "@cocalc/util/misc"; +import { dict, filename_extension } from "@cocalc/util/misc"; import { DirectoryListing, DirectoryListingEntry } from "./types"; -const MASKED_FILENAMES = ["__pycache__"] as const; +const MASKED_FILENAMES = new Set(["__pycache__"]); const MASKED_FILE_EXTENSIONS = { py: ["pyc"], @@ -60,12 +60,13 @@ export function computeFileMasks(listing: DirectoryListing): void { listing.map((item) => [item.name, item]), ); for (const file of listing) { - // mask certain known directories - if (MASKED_FILENAMES.indexOf(file.name as any) >= 0) { + // mask certain known paths + if (MASKED_FILENAMES.has(file.name as any)) { filename_map[file.name].mask = true; } - // note: never skip already masked files, because of rnw/rtex->tex + // NOTE: never skip already masked files, because of rnw/rtex->tex + const ext = filename_extension(file.name).toLowerCase(); // some extensions like Rmd modify the basename during compilation @@ -77,16 +78,16 @@ export function computeFileMasks(listing: DirectoryListing): void { // check each possible compiled extension let derivedBasename; // some uppercase-strings have special meaning - if (startswith(mask_ext, "NODOT")) { + if (mask_ext.startsWith("NODOT")) { derivedBasename = basename.slice(0, -1); // exclude the trailing dot mask_ext = mask_ext.slice("NODOT".length); - } else if (mask_ext.indexOf("FILENAME") >= 0) { + } else if (mask_ext.includes("FILENAME")) { derivedBasename = mask_ext.replace("FILENAME", filename); mask_ext = ""; - } else if (mask_ext.indexOf("BASENAME") >= 0) { + } else if (mask_ext.includes("BASENAME")) { derivedBasename = mask_ext.replace("BASENAME", basename.slice(0, -1)); mask_ext = ""; - } else if (mask_ext.indexOf("BASEDASHNAME") >= 0) { + } else if (mask_ext.includes("BASEDASHNAME")) { // BASEDASHNAME is like BASENAME, but replaces spaces by dashes // https://github.com/sagemathinc/cocalc/issues/3229 const fragment = basename.slice(0, -1).replace(/ /g, "-"); diff --git a/src/packages/frontend/project/explorer/explorer.tsx b/src/packages/frontend/project/explorer/explorer.tsx index edb7cf3528..5e1f523a6a 100644 --- a/src/packages/frontend/project/explorer/explorer.tsx +++ b/src/packages/frontend/project/explorer/explorer.tsx @@ -114,6 +114,7 @@ export function Explorer() { const project_map = useTypedRedux("projects", "project_map"); const images = useTypedRedux("compute_images", "images"); + const mask = useTypedRedux("account", "other_settings")?.get("mask_files"); const active_file_sort = useTypedRedux({ project_id }, "active_file_sort"); const fs = useFs({ project_id }); @@ -122,10 +123,9 @@ export function Explorer() { path: current_path, ...sortDesc(active_file_sort), cacheId: actions?.getCacheId(compute_server_id), - mask: true, + mask, }); const showHidden = useTypedRedux({ project_id }, "show_hidden"); - const showMasked = useTypedRedux({ project_id }, "show_masked"); listing = listingError ? null @@ -133,7 +133,6 @@ export function Explorer() { listing, search: file_search, showHidden, - showMasked, }); useEffect(() => { diff --git a/src/packages/frontend/project/explorer/misc-side-buttons.tsx b/src/packages/frontend/project/explorer/misc-side-buttons.tsx index b4444c0c18..5d9fd3d090 100644 --- a/src/packages/frontend/project/explorer/misc-side-buttons.tsx +++ b/src/packages/frontend/project/explorer/misc-side-buttons.tsx @@ -30,7 +30,6 @@ const OPEN_MSG = defineMessage({ export function MiscSideButtons() { const { actions, project_id } = useProjectContext(); const show_hidden = useTypedRedux({ project_id }, "show_hidden"); - const show_masked = useTypedRedux({ project_id }, "show_masked"); const current_path = useTypedRedux({ project_id }, "current_path"); const available_features = useTypedRedux( { project_id }, @@ -49,13 +48,6 @@ export function MiscSideButtons() { }); }; - const handle_masked_toggle = (e: MouseEvent): void => { - e.preventDefault(); - actions?.setState({ - show_masked: !show_masked, - }); - }; - const handle_backup = (e: MouseEvent): void => { e.preventDefault(); actions?.open_directory(".snapshots"); @@ -78,25 +70,6 @@ export function MiscSideButtons() { ); } - function render_masked_toggle(): JSX.Element { - return ( - - ); - } - function render_backup(): JSX.Element | undefined { // NOTE -- snapshots aren't available except in "kucalc" version // -- they are complicated nontrivial thing that isn't usually setup... @@ -211,7 +184,6 @@ export function MiscSideButtons() {
{render_hidden_toggle()} - {render_masked_toggle()} {render_backup()} diff --git a/src/packages/frontend/project/listing/filter-listing.ts b/src/packages/frontend/project/listing/filter-listing.ts index a5b03edb69..53e296da7f 100644 --- a/src/packages/frontend/project/listing/filter-listing.ts +++ b/src/packages/frontend/project/listing/filter-listing.ts @@ -4,16 +4,11 @@ export default function filterListing({ listing, search, showHidden, - showMasked, }: { listing?: DirectoryListingEntry[] | null; search?: string; showHidden?: boolean; - showMasked?: boolean; }): DirectoryListingEntry[] | null { - if (!showMasked) { - console.log("TODO: show masked"); - } if (listing == null) { return null; } diff --git a/src/packages/frontend/project/listing/use-listing.ts b/src/packages/frontend/project/listing/use-listing.ts index a4c481c52f..764d602108 100644 --- a/src/packages/frontend/project/listing/use-listing.ts +++ b/src/packages/frontend/project/listing/use-listing.ts @@ -106,6 +106,7 @@ function filesToListing({ console.warn(`invalid sort direction: '${sortDirection}'`); } if (mask) { + // note -- this masking is as much time as everything above computeFileMasks(v); } return v; diff --git a/src/packages/util/db-schema/accounts.ts b/src/packages/util/db-schema/accounts.ts index fc7774d8d5..27d8ad9a2d 100644 --- a/src/packages/util/db-schema/accounts.ts +++ b/src/packages/util/db-schema/accounts.ts @@ -482,7 +482,12 @@ Table({ other_settings: { katex: true, confirm_close: false, - mask_files: true, + // mask_files -- note that there is a performance cost to this, e.g., 5ms if you have 10K files in + // a directory (basically it doubles the processing costs). + // It's also confusing and can be subtly wrong. Finally, it's almost never necessary due to us changing the defaults + // for running latex to put all the temp files in /tmp -- in general we should always put temp files in tmp anyways + // with all build processes. So mask_files is off by default if not explicitly selected. + mask_files: false, page_size: 500, standby_timeout_m: 5, default_file_sort: "name", From 0e420a55496c5866ad4323d6d275fed698e4e94b Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 30 Jul 2025 21:56:47 +0000 Subject: [PATCH 141/270] isdir --> isDir; isopen --> isOpen, etc. -- consistent naming. Also much more use of {...} : Props so we can always identify unused props quickly. --- src/packages/backend/get-listing.ts | 18 +- src/packages/conat/files/listing.ts | 20 +- .../database/postgres-server-queries.coffee | 2 +- .../file-server/btrfs/subvolume-bup.ts | 13 +- src/packages/file-server/btrfs/subvolume.ts | 4 +- src/packages/file-server/btrfs/subvolumes.ts | 4 +- .../file-server/btrfs/test/subvolume.test.ts | 6 +- src/packages/file-server/btrfs/util.ts | 2 +- src/packages/frontend/client/project.ts | 2 +- .../frontend/course/assignments/actions.ts | 2 +- .../course/export/export-assignment.ts | 2 +- src/packages/frontend/cspell.json | 1 + .../terminal-editor/commands-guide.tsx | 2 +- .../frontend/project/directory-selector.tsx | 2 +- .../frontend/project/explorer/action-bar.tsx | 6 +- .../frontend/project/explorer/action-box.tsx | 34 +--- .../frontend/project/explorer/download.tsx | 6 +- .../frontend/project/explorer/explorer.tsx | 4 +- .../explorer/file-listing/file-listing.tsx | 50 +---- .../explorer/file-listing/file-row.tsx | 172 ++++++++---------- .../frontend/project/explorer/types.ts | 32 +--- .../frontend/project/listing/use-files.ts | 2 +- .../frontend/project/listing/use-listing.ts | 2 +- .../project/page/flyouts/active-group.tsx | 10 +- .../frontend/project/page/flyouts/active.tsx | 14 +- .../project/page/flyouts/file-list-item.tsx | 56 +++--- .../project/page/flyouts/files-bottom.tsx | 10 +- .../project/page/flyouts/files-controls.tsx | 23 ++- .../frontend/project/page/flyouts/files.tsx | 24 +-- .../frontend/project/page/flyouts/log.tsx | 4 +- src/packages/frontend/project_actions.ts | 20 +- src/packages/frontend/share/config.tsx | 105 +++++------ src/packages/util/types/directory-listing.ts | 17 +- 33 files changed, 305 insertions(+), 366 deletions(-) diff --git a/src/packages/backend/get-listing.ts b/src/packages/backend/get-listing.ts index 02bcdbc835..90c66c3c8d 100644 --- a/src/packages/backend/get-listing.ts +++ b/src/packages/backend/get-listing.ts @@ -3,10 +3,12 @@ * License: MS-RSL – see LICENSE.md for details */ +// DEPRECATABLE? + /* This is used by backends to serve directory listings to clients: -{files:[..., {size:?,name:?,mtime:?,isdir:?}]} +{files:[..., {size:?,name:?,mtime:?,isDir:?}]} where mtime is integer SECONDS since epoch, size is in bytes, and isdir is only there if true. @@ -46,7 +48,7 @@ const getListing = reuseInFlight( if (!hidden && file.name[0] === ".") { continue; } - let entry: DirectoryListingEntry; + let entry: Partial; try { // I don't actually know if file.name can fail to be JSON-able with node.js -- is there // even a string in Node.js that cannot be dumped to JSON? With python @@ -61,14 +63,14 @@ const getListing = reuseInFlight( try { let stats: Stats; if (file.isSymbolicLink()) { - // Optimization: don't explicitly set issymlink if it is false - entry.issymlink = true; + // Optimization: don't explicitly set isSymLink if it is false + entry.isSymLink = true; } - if (entry.issymlink) { + if (entry.isSymLink) { // at least right now we only use this symlink stuff to display // information to the user in a listing, and nothing else. try { - entry.link_target = await readlink(dir + "/" + entry.name); + entry.linkTarget = await readlink(dir + "/" + entry.name); } catch (err) { // If we don't know the link target for some reason; just ignore this. } @@ -81,7 +83,7 @@ const getListing = reuseInFlight( } entry.mtime = stats.mtime.valueOf() / 1000; if (stats.isDirectory()) { - entry.isdir = true; + entry.isDir = true; const v = await readdir(dir + "/" + entry.name); if (hidden) { entry.size = v.length; @@ -100,7 +102,7 @@ const getListing = reuseInFlight( } catch (err) { entry.error = `${entry.error ? entry.error : ""}${err}`; } - files.push(entry); + files.push(entry as DirectoryListingEntry); } return files; }, diff --git a/src/packages/conat/files/listing.ts b/src/packages/conat/files/listing.ts index 912363c0ab..e1836510d3 100644 --- a/src/packages/conat/files/listing.ts +++ b/src/packages/conat/files/listing.ts @@ -27,11 +27,11 @@ interface FileData { // last modification time as time since epoch in **milliseconds** (as is usual for javascript) mtime: number; size: number; - // isdir = mainly for backward compat: - isdir?: boolean; + // isDir = mainly for backward compat: + isDir?: boolean; // issymlink = mainly for backward compat: - issymlink?: boolean; - link_target?: string; + isSymLink?: boolean; + linkTarget?: string; // see typeDescription above. type?: FileTypeLabel; } @@ -114,13 +114,13 @@ export class Listing extends EventEmitter { }; if (stats.isSymbolicLink()) { // resolve target. - data.link_target = await this.opts.fs.readlink( + data.linkTarget = await this.opts.fs.readlink( join(this.opts.path, filename), ); - data.issymlink = true; + data.isSymLink = true; } if (stats.isDirectory()) { - data.isdir = true; + data.isDir = true; } this.files[filename] = data; } catch (err) { @@ -167,13 +167,13 @@ async function getListing( const size = parseInt(v[2]); files[name] = { mtime, size, type: v[3] as FileTypeLabel }; if (v[3] == "l") { - files[name].issymlink = true; + files[name].isSymLink = true; } if (v[3] == "d") { - files[name].isdir = true; + files[name].isDir = true; } if (v[4]) { - files[name].link_target = v[4]; + files[name].linkTarget = v[4]; } } catch {} } diff --git a/src/packages/database/postgres-server-queries.coffee b/src/packages/database/postgres-server-queries.coffee index 423fedbc84..e3f0eccb42 100644 --- a/src/packages/database/postgres-server-queries.coffee +++ b/src/packages/database/postgres-server-queries.coffee @@ -1517,7 +1517,7 @@ exports.extend_PostgreSQL = (ext) -> class PostgreSQL extends ext opts = defaults opts, project_id : required path : required - listing : required # files in path [{name:..., isdir:boolean, ....}, ...] + listing : required # files in path [{name:..., isDir:boolean, ....}, ...] cb : required # Get all public paths for the given project_id, then check if path is "in" one according # to the definition in misc. diff --git a/src/packages/file-server/btrfs/subvolume-bup.ts b/src/packages/file-server/btrfs/subvolume-bup.ts index b4d64c9b5a..3237e32dba 100644 --- a/src/packages/file-server/btrfs/subvolume-bup.ts +++ b/src/packages/file-server/btrfs/subvolume-bup.ts @@ -110,6 +110,7 @@ export class SubvolumeBup { }); }; + // [ ] TODO: remove this ls and instead rely only on the fs sandbox code. ls = async (path: string = ""): Promise => { if (!path) { const { stdout } = await sudo({ @@ -125,10 +126,10 @@ export class SubvolumeBup { } const mtime = parseBupTime(name).valueOf() / 1000; newest = Math.max(mtime, newest); - v.push({ name, isdir: true, mtime }); + v.push({ name, isDir: true, mtime }); } if (v.length > 0) { - v.push({ name: "latest", isdir: true, mtime: newest }); + v.push({ name: "latest", isDir: true, mtime: newest }); } return v; } @@ -153,20 +154,20 @@ export class SubvolumeBup { // [-rw-------","6b851643360e435eb87ef9a6ab64a8b1/6b851643360e435eb87ef9a6ab64a8b1","5","2025-07-15","06:12","a.txt"] const w = x.split(/\s+/); if (w.length >= 6) { - let isdir, name; + let isDir, name; if (w[5].endsWith("@") || w[5].endsWith("=") || w[5].endsWith("|")) { w[5] = w[5].slice(0, -1); } if (w[5].endsWith("/")) { - isdir = true; + isDir = true; name = w[5].slice(0, -1); } else { name = w[5]; - isdir = false; + isDir = false; } const size = parseInt(w[2]); const mtime = new Date(w[3] + "T" + w[4]).valueOf() / 1000; - v.push({ name, size, mtime, isdir }); + v.push({ name, size, mtime, isDir }); } } return v; diff --git a/src/packages/file-server/btrfs/subvolume.ts b/src/packages/file-server/btrfs/subvolume.ts index f10e9f6182..c29d393994 100644 --- a/src/packages/file-server/btrfs/subvolume.ts +++ b/src/packages/file-server/btrfs/subvolume.ts @@ -4,7 +4,7 @@ A subvolume import { type Filesystem, DEFAULT_SUBVOLUME_SIZE } from "./filesystem"; import refCache from "@cocalc/util/refcache"; -import { isdir, sudo } from "./util"; +import { isDir, sudo } from "./util"; import { join } from "path"; import { SubvolumeBup } from "./subvolume-bup"; import { SubvolumeSnapshots } from "./subvolume-snapshots"; @@ -94,7 +94,7 @@ export class Subvolume { if (target.endsWith("/")) { targetPath += "/"; } - if (!srcPath.endsWith("/") && (await isdir(srcPath))) { + if (!srcPath.endsWith("/") && (await isDir(srcPath))) { srcPath += "/"; if (!targetPath.endsWith("/")) { targetPath += "/"; diff --git a/src/packages/file-server/btrfs/subvolumes.ts b/src/packages/file-server/btrfs/subvolumes.ts index 0f8da1468f..e1a6c418be 100644 --- a/src/packages/file-server/btrfs/subvolumes.ts +++ b/src/packages/file-server/btrfs/subvolumes.ts @@ -4,7 +4,7 @@ import getLogger from "@cocalc/backend/logger"; import { SNAPSHOTS } from "./subvolume-snapshots"; import { exists } from "@cocalc/backend/misc/async-utils-node"; import { join, normalize } from "path"; -import { btrfs, isdir } from "./util"; +import { btrfs, isDir } from "./util"; import { chmod, rename, rm } from "node:fs/promises"; import { executeCode } from "@cocalc/backend/execute-code"; @@ -99,7 +99,7 @@ export class Subvolumes { if (!targetPath.startsWith(this.filesystem.opts.mount)) { throw Error("suspicious target"); } - if (!srcPath.endsWith("/") && (await isdir(srcPath))) { + if (!srcPath.endsWith("/") && (await isDir(srcPath))) { srcPath += "/"; if (!targetPath.endsWith("/")) { targetPath += "/"; diff --git a/src/packages/file-server/btrfs/test/subvolume.test.ts b/src/packages/file-server/btrfs/test/subvolume.test.ts index 1736f17c4f..fe9286935e 100644 --- a/src/packages/file-server/btrfs/test/subvolume.test.ts +++ b/src/packages/file-server/btrfs/test/subvolume.test.ts @@ -248,7 +248,7 @@ describe("test bup backups", () => { it("confirm a.txt is in our backup", async () => { const x = await vol.bup.ls("latest"); expect(x).toEqual([ - { name: "a.txt", size: 5, mtime: x[0].mtime, isdir: false }, + { name: "a.txt", size: 5, mtime: x[0].mtime, isDir: false }, ]); }); @@ -270,8 +270,8 @@ describe("test bup backups", () => { await vol.bup.save(); const x = await vol.bup.ls("latest"); expect(x).toEqual([ - { name: "a.txt", size: 5, mtime: x[0].mtime, isdir: false }, - { name: "mydir", size: 0, mtime: x[1].mtime, isdir: true }, + { name: "a.txt", size: 5, mtime: x[0].mtime, isDir: false }, + { name: "mydir", size: 0, mtime: x[1].mtime, isDir: true }, ]); expect(Math.abs((x[0].mtime ?? 0) * 1000 - Date.now())).toBeLessThan( 5 * 60_000, diff --git a/src/packages/file-server/btrfs/util.ts b/src/packages/file-server/btrfs/util.ts index b6408f8440..ee71b79eac 100644 --- a/src/packages/file-server/btrfs/util.ts +++ b/src/packages/file-server/btrfs/util.ts @@ -44,7 +44,7 @@ export async function btrfs( return await sudo({ ...opts, command: "btrfs" }); } -export async function isdir(path: string) { +export async function isDir(path: string) { return (await stat(path)).isDirectory(); } diff --git a/src/packages/frontend/client/project.ts b/src/packages/frontend/client/project.ts index 6d621d4d84..8938d29aa4 100644 --- a/src/packages/frontend/client/project.ts +++ b/src/packages/frontend/client/project.ts @@ -496,7 +496,7 @@ export class ProjectClient { return (await this.api(opts.project_id)).realpath(opts.path); }; - isdir = async ({ + isDir = async ({ project_id, path, }: { diff --git a/src/packages/frontend/course/assignments/actions.ts b/src/packages/frontend/course/assignments/actions.ts index 2be16c86fe..f53ba0d227 100644 --- a/src/packages/frontend/course/assignments/actions.ts +++ b/src/packages/frontend/course/assignments/actions.ts @@ -1598,7 +1598,7 @@ ${details} let has_student_subdir: boolean = false; for (const entry of listing) { - if (entry.isdir && entry.name == STUDENT_SUBDIR) { + if (entry.isDir && entry.name == STUDENT_SUBDIR) { has_student_subdir = true; break; } diff --git a/src/packages/frontend/course/export/export-assignment.ts b/src/packages/frontend/course/export/export-assignment.ts index 0ab46b502e..35e14ea09b 100644 --- a/src/packages/frontend/course/export/export-assignment.ts +++ b/src/packages/frontend/course/export/export-assignment.ts @@ -82,7 +82,7 @@ async function export_one_directory( let x: any; const timeout = 60; // 60 seconds for (x of listing) { - if (x.isdir) continue; // we ignore subdirectories... + if (x.isDir) continue; // we ignore subdirectories... const { name } = x; if (startswith(name, "STUDENT")) continue; if (startswith(name, ".")) continue; diff --git a/src/packages/frontend/cspell.json b/src/packages/frontend/cspell.json index d742413365..2a9d09d1bf 100644 --- a/src/packages/frontend/cspell.json +++ b/src/packages/frontend/cspell.json @@ -50,6 +50,7 @@ "immutablejs", "ipynb", "isdir", + "isDir", "kernelspec", "LLM", "LLMs", diff --git a/src/packages/frontend/frame-editors/terminal-editor/commands-guide.tsx b/src/packages/frontend/frame-editors/terminal-editor/commands-guide.tsx index ba18b638f5..ad338c2faf 100644 --- a/src/packages/frontend/frame-editors/terminal-editor/commands-guide.tsx +++ b/src/packages/frontend/frame-editors/terminal-editor/commands-guide.tsx @@ -102,7 +102,7 @@ export function CommandsGuide({ actions, local_view_state }: Props) { if (!hidden && name.startsWith(".")) { continue; } - if (files[name].isdir) { + if (files[name].isDir) { dirnames.push(name); } else { filenames.push(name); diff --git a/src/packages/frontend/project/directory-selector.tsx b/src/packages/frontend/project/directory-selector.tsx index bb1a474595..1463cbbbe3 100644 --- a/src/packages/frontend/project/directory-selector.tsx +++ b/src/packages/frontend/project/directory-selector.tsx @@ -394,7 +394,7 @@ function Subdirs(props) { const paths: string[] = []; const newPaths: string[] = []; for (const name in files) { - if (!files[name].isdir) continue; + if (!files[name].isDir) continue; if (name.startsWith(".") && !showHidden) continue; if (name.startsWith(NEW_FOLDER)) { newPaths.push(name); diff --git a/src/packages/frontend/project/explorer/action-bar.tsx b/src/packages/frontend/project/explorer/action-bar.tsx index 98d967f393..80977d97dd 100644 --- a/src/packages/frontend/project/explorer/action-bar.tsx +++ b/src/packages/frontend/project/explorer/action-bar.tsx @@ -228,15 +228,15 @@ export function ActionBar({ if (checked_files.size === 0) { return; } else if (checked_files.size === 1) { - let isdir; + let isDir; const item = checked_files.first(); for (const file of listing) { if (misc.path_to_file(current_path ?? "", file.name) === item) { - ({ isdir } = file); + ({ isDir } = file); } } - if (isdir) { + if (isDir) { // one directory selected action_buttons = [...ACTION_BUTTONS_DIR]; } else { diff --git a/src/packages/frontend/project/explorer/action-box.tsx b/src/packages/frontend/project/explorer/action-box.tsx index f39af2a2b9..1e05857215 100644 --- a/src/packages/frontend/project/explorer/action-box.tsx +++ b/src/packages/frontend/project/explorer/action-box.tsx @@ -104,7 +104,7 @@ export function ActionBox({ } } - function render_selected_files_list(): React.JSX.Element { + function render_selected_files_list() { return ( 
         {checked_files.toArray().map((name) => (
@@ -124,7 +124,7 @@ export function ActionBox({
     actions.set_all_files_unchecked();
   }
 
-  function render_delete_warning(): React.JSX.Element | undefined {
+  function render_delete_warning() {
     if (current_path === ".trash") {
       return (
         
@@ -139,7 +139,7 @@ export function ActionBox({
     }
   }
 
-  function render_delete(): React.JSX.Element | undefined {
+  function render_delete() {
     const { size } = checked_files;
     return (
       

@@ -213,7 +213,7 @@ export function ActionBox({
     return dest !== current_path;
   }
 
-  function render_move(): React.JSX.Element {
+  function render_move() {
     const { size } = checked_files;
     return (
       

@@ -261,7 +261,7 @@ export function ActionBox({
     }
   }
 
-  function render_different_project_dialog(): React.JSX.Element | undefined {
+  function render_different_project_dialog() {
     if (show_different_project) {
       return (
         
@@ -279,9 +279,7 @@ export function ActionBox({
     }
   }
 
-  function render_copy_different_project_options():
-    | React.JSX.Element
-    | undefined {
+  function render_copy_different_project_options() {
     if (project_id !== copy_destination_project_id) {
       return (
         

@@ -431,7 +429,7 @@ export function ActionBox({
     );
   }
 
-  function render_copy(): React.JSX.Element {
+  function render_copy() {
     const { size } = checked_files;
     const signed_in = get_user_type() === "signed_in";
     if (!signed_in) {
@@ -540,27 +538,17 @@ export function ActionBox({
     }
   }
 
-  function render_share(): React.JSX.Element {
+  function render_share() {
     // currently only works for a single selected file
     const path: string = checked_files.first() ?? "";
     if (!path) {
-      return <>;
-    }
-    const public_data = file_map[misc.path_split(path).tail];
-    if (public_data == undefined) {
-      // directory listing not loaded yet... (will get re-rendered when loaded)
-      return ;
+      return null;
     }
     return (
        actions.set_public_path(path, opts)}
@@ -569,9 +557,7 @@ export function ActionBox({
     );
   }
 
-  function render_action_box(
-    action: FileAction,
-  ): React.JSX.Element | undefined {
+  function render_action_box(action: FileAction) {
     switch (action) {
       case "compress":
         return ;
diff --git a/src/packages/frontend/project/explorer/download.tsx b/src/packages/frontend/project/explorer/download.tsx
index 17f7740a76..0bf5c11a42 100644
--- a/src/packages/frontend/project/explorer/download.tsx
+++ b/src/packages/frontend/project/explorer/download.tsx
@@ -44,9 +44,9 @@ export default function Download() {
       return;
     }
     const file = checked_files.first();
-    const isdir = !!actions.isDirViaCache(file);
-    setArchiveMode(!!isdir);
-    if (!isdir) {
+    const isDir = !!actions.isDirViaCache(file);
+    setArchiveMode(!!isDir);
+    if (!isDir) {
       const store = actions?.get_store();
       setUrl(store?.fileURL(file) ?? "");
     }
diff --git a/src/packages/frontend/project/explorer/explorer.tsx b/src/packages/frontend/project/explorer/explorer.tsx
index 5e1f523a6a..3ec833a4aa 100644
--- a/src/packages/frontend/project/explorer/explorer.tsx
+++ b/src/packages/frontend/project/explorer/explorer.tsx
@@ -167,9 +167,9 @@ export function Explorer() {
           redux.getProjectStore(project_id).get("selected_file_index") ?? 0;
         const x = listing?.[n];
         if (x != null) {
-          const { isdir, name } = x;
+          const { isDir, name } = x;
           const path = join(current_path, name);
-          if (isdir) {
+          if (isDir) {
             actions.open_directory(path);
           } else {
             actions.open_file({ path, foreground: !e.ctrlKey });
diff --git a/src/packages/frontend/project/explorer/file-listing/file-listing.tsx b/src/packages/frontend/project/explorer/file-listing/file-listing.tsx
index 52f7b6f275..4663119389 100644
--- a/src/packages/frontend/project/explorer/file-listing/file-listing.tsx
+++ b/src/packages/frontend/project/explorer/file-listing/file-listing.tsx
@@ -12,12 +12,7 @@ import * as immutable from "immutable";
 import { useEffect, useRef } from "react";
 import { FormattedMessage } from "react-intl";
 import { Virtuoso, VirtuosoHandle } from "react-virtuoso";
-import {
-  Rendered,
-  TypedMap,
-  useTypedRedux,
-  redux,
-} from "@cocalc/frontend/app-framework";
+import { TypedMap, useTypedRedux, redux } from "@cocalc/frontend/app-framework";
 import useVirtuosoScrollHook from "@cocalc/frontend/components/virtuoso-scroll-hook";
 import { ProjectActions } from "@cocalc/frontend/project_actions";
 import { MainConfiguration } from "@cocalc/frontend/project_configuration";
@@ -26,7 +21,7 @@ import { FileRow } from "./file-row";
 import { ListingHeader } from "./listing-header";
 import NoFiles from "./no-files";
 import { TERM_MODE_CHAR } from "./utils";
-import { DirectoryListingEntry } from "@cocalc/util/types";
+import { type DirectoryListingEntry } from "@cocalc/frontend/project/explorer/types";
 
 interface Props {
   actions: ProjectActions;
@@ -59,38 +54,22 @@ export function FileListing({
     useTypedRedux({ project_id }, "selected_file_index") ?? 0;
   const name = actions.name;
 
-  function render_row(
-    name,
-    size,
-    time,
-    mask,
-    isdir,
-    issymlink,
-    index: number,
-    link_target?: string, // if given, is a known symlink to this file
-  ): Rendered {
+  function renderRow(index, file) {
     const checked = checked_files.has(misc.path_to_file(current_path, name));
     const color = misc.rowBackground({ index, checked });
 
     return (
       
@@ -122,28 +101,19 @@ export function FileListing({
     return ;
   }
 
-  function render_rows(): Rendered {
+  function renderRows() {
     return (
        {
-          const a = listing[index];
-          if (a == null) {
+          const file = listing[index];
+          if (file == null) {
             // shouldn't happen
             return 
;
           }
-          return render_row(
-            a.name,
-            a.size,
-            a.mtime,
-            a.mask,
-            a.isdir,
-            a.issymlink,
-            index,
-            a.link_target,
-          );
+          return renderRow(index, file);
         }}
         {...virtuosoScroll}
       />
@@ -206,7 +176,7 @@ export function FileListing({
           active_file_sort={active_file_sort}
           sort_by={actions.set_sorted_file_column}
         />
-        {listing.length > 0 ? render_rows() : render_no_files()}
+        {listing.length > 0 ? renderRows() : render_no_files()}
       

     
   );
diff --git a/src/packages/frontend/project/explorer/file-listing/file-row.tsx b/src/packages/frontend/project/explorer/file-listing/file-row.tsx
index 92308958be..881f449b16 100644
--- a/src/packages/frontend/project/explorer/file-listing/file-row.tsx
+++ b/src/packages/frontend/project/explorer/file-listing/file-row.tsx
@@ -25,6 +25,7 @@ import { url_href } from "@cocalc/frontend/project/utils";
 import { FileCheckbox } from "./file-checkbox";
 import { PublicButton } from "./public-button";
 import { generate_click_for } from "./utils";
+import { type DirectoryListing } from "@cocalc/frontend/project/explorer/types";
 
 export const VIEWABLE_FILE_EXT: Readonly = [
   "md",
@@ -36,41 +37,59 @@ export const VIEWABLE_FILE_EXT: Readonly = [
 ] as const;
 
 interface Props {
-  isdir: boolean;
+  isDir: boolean;
   name: string;
-  display_name?: string; // if given, will display this, and will show true filename in popover
-  size: number; // sometimes is NOT known!
-  time: number;
-  issymlink: boolean;
+  // if given, will display this, and will show true filename in popover
+  display_name?: string;
+  size: number;
+  mtime: number;
+  isSymLink: boolean;
   checked: boolean;
   selected: boolean;
   color: string;
   mask: boolean;
-  is_public: boolean;
+  isPublic: boolean;
   current_path: string;
   actions: ProjectActions;
   no_select: boolean;
-  link_target?: string;
+  linkTarget?: string;
   // if given, include a little 'server' tag in this color, and tooltip etc using id.
   // Also important for download and preview links!
   computeServerId?: number;
-  listing;
+  listing: DirectoryListing;
 }
 
-export const FileRow: React.FC = React.memo((props) => {
+export function FileRow({
+  isDir,
+  name,
+  display_name,
+  size,
+  mtime,
+  checked,
+  selected,
+  color,
+  mask,
+  isPublic,
+  current_path,
+  actions,
+  no_select,
+  linkTarget,
+  computeServerId,
+  listing,
+}: Props) {
   const student_project_functionality = useStudentProjectFunctionality(
-    props.actions.project_id,
+    actions.project_id,
   );
   const [selection_at_last_mouse_down, set_selection_at_last_mouse_down] =
     useState(undefined);
 
   function render_icon() {
     const style: React.CSSProperties = {
-      color: props.mask ? "#bbbbbb" : COLORS.FILE_ICON,
+      color: mask ? "#bbbbbb" : COLORS.FILE_ICON,
       verticalAlign: "sub",
     } as const;
     let body: React.JSX.Element;
-    if (props.isdir) {
+    if (isDir) {
       body = (
         <>
            = React.memo((props) => {
       );
     } else {
       // get the file_associations[ext] just like it is defined in the editor
-      let name: IconName;
-      const info = file_options(props.name);
-      if (info != null) {
-        name = info.icon;
-      } else {
-        name = "file";
-      }
+      const info = file_options(name);
+      const iconName: IconName = info?.icon ?? "file";
 
-      body = ;
+      body = ;
     }
 
     return {body};
   }
 
-  function render_link_target() {
-    if (props.link_target == null || props.link_target == props.name) return;
-    return (
-      <>
-        {" "}
-        {" "}
-        {props.link_target}{" "}
-      
-    );
-  }
-
   function render_name_link(styles, name, ext) {
     return (
       
         {misc.trunc_middle(name, 50)}
-        
+        
           {ext === "" ? "" : `.${ext}`}
         
-        {render_link_target()}
+        {linkTarget != null && linkTarget != name && (
+          <>
+            {" "}
+            {" "}
+            {linkTarget}{" "}
+          
+        )}
       
     );
   }
 
   function render_name() {
-    let name = props.display_name ?? props.name;
+    let name0 = display_name ?? name;
     let ext: string;
-    if (props.isdir) {
+    if (isDir) {
       ext = "";
     } else {
-      const name_and_ext = misc.separate_file_extension(name);
-      ({ name, ext } = name_and_ext);
+      const name_and_ext = misc.separate_file_extension(name0);
+      ({ name: name0, ext } = name_and_ext);
     }
 
     const show_tip =
-      (props.display_name != undefined && props.name !== props.display_name) ||
-      name.length > 50;
+      (display_name != undefined && name0 !== display_name) ||
+      name0.length > 50;
 
     const styles = {
       whiteSpace: "pre-wrap",
       wordWrap: "break-word",
       overflowWrap: "break-word",
       verticalAlign: "middle",
-      color: props.mask ? "#bbbbbb" : COLORS.TAB,
+      color: mask ? "#bbbbbb" : COLORS.TAB,
     };
 
     if (show_tip) {
       return (
         
-          {render_name_link(styles, name, ext)}
+          {render_name_link(styles, name0, ext)}
         
       );
     } else {
-      return render_name_link(styles, name, ext);
+      return render_name_link(styles, name0, ext);
     }
   }
 
   const generate_on_share_click = memoizeOne((full_path: string) => {
-    return generate_click_for("share", full_path, props.actions);
+    return generate_click_for("share", full_path, actions);
   });
 
   function render_public_file_info() {
-    if (props.is_public) {
+    if (isPublic) {
       return ;
     }
   }
 
   function full_path() {
-    return misc.path_to_file(props.current_path, props.name);
+    return misc.path_to_file(current_path, name);
   }
 
   function handle_mouse_down() {
@@ -193,25 +202,24 @@ export const FileRow: React.FC = React.memo((props) => {
       // the click to do the selection triggering opening of the file.
       return;
     }
-    if (props.isdir) {
-      props.actions.open_directory(full_path());
-      props.actions.set_file_search("");
+    if (isDir) {
+      actions.open_directory(full_path());
+      actions.set_file_search("");
     } else {
       const foreground = should_open_in_foreground(e);
       const path = full_path();
       track("open-file", {
-        project_id: props.actions.project_id,
+        project_id: actions.project_id,
         path,
         how: "click-on-listing",
       });
-      props.actions.open_file({
+      actions.open_file({
         path,
         foreground,
         explicit: true,
       });
       if (foreground) {
         // delay slightly since it looks weird to see the full listing right when you click on a file
-        const actions = props.actions;
         setTimeout(() => actions.set_file_search(""), 10);
       }
     }
@@ -220,7 +228,7 @@ export const FileRow: React.FC = React.memo((props) => {
   function handle_download_click(e) {
     e.preventDefault();
     e.stopPropagation();
-    props.actions.download_file({
+    actions.download_file({
       path: full_path(),
       log: true,
     });
@@ -236,7 +244,7 @@ export const FileRow: React.FC = React.memo((props) => {
     try {
       return (
         
       );
@@ -295,7 +303,8 @@ export const FileRow: React.FC = React.memo((props) => {
 
   function render_download_button(url) {
     if (student_project_functionality.disableActions) return;
-    const size = misc.human_readable_size(props.size);
+    if (isDir) return;
+    const displaySize = misc.human_readable_size(size);
     // TODO: This really should not be in the size column...
     return (
        = React.memo((props) => {
         }
         content={
           <>
-            Download this {size} file
+            Download this {displaySize} file
             

             to your computer.
           
@@ -320,7 +329,7 @@ export const FileRow: React.FC = React.memo((props) => {
           onClick={handle_download_click}
           style={{ color: COLORS.GRAY, padding: 0 }}
         >
-          {size}
+          {displaySize}
           
         
       
@@ -330,35 +339,31 @@ export const FileRow: React.FC = React.memo((props) => {
   const row_styles: CSS = {
     cursor: "pointer",
     borderRadius: "4px",
-    backgroundColor: props.color,
+    backgroundColor: color,
     borderStyle: "solid",
-    borderColor: props.selected ? "#08c" : "transparent",
+    borderColor: selected ? "#08c" : "transparent",
     margin: "1px 1px 1px 1px",
   } as const;
 
   // See https://github.com/sagemathinc/cocalc/issues/1020
   // support right-click → copy url for the download button
-  const url = url_href(
-    props.actions.project_id,
-    full_path(),
-    props.computeServerId,
-  );
+  const url = url_href(actions.project_id, full_path(), computeServerId);
 
   return (
     
       
         {!student_project_functionality.disableActions && (
           
         )}
       
@@ -386,34 +391,13 @@ export const FileRow: React.FC = React.memo((props) => {
           
         
         {render_timestamp()}
-        {props.isdir ? (
-          <>
-            
-          
-        ) : (
+        {!isDir && (
           
             {render_download_button(url)}
-            {render_view_button(url, props.name)}
+            {render_view_button(url, name)}
           
         )}
       
     
   );
-});
-
-const directory_size_style: React.CSSProperties = {
-  color: COLORS.GRAY,
-  marginRight: "3em",
-} as const;
-
-function DirectorySize({ size }) {
-  if (size == undefined) {
-    return null;
-  }
-
-  return (
-    
-      {size} {misc.plural(size, "item")}
-    
-  );
 }
diff --git a/src/packages/frontend/project/explorer/types.ts b/src/packages/frontend/project/explorer/types.ts
index 9b85a77ad0..62b0f1b64b 100644
--- a/src/packages/frontend/project/explorer/types.ts
+++ b/src/packages/frontend/project/explorer/types.ts
@@ -3,30 +3,18 @@
  *  License: MS-RSL – see LICENSE.md for details
  */
 
-// NOTE(hsy): I don't know if these two types are the same, maybe they should be merged.
+import type { DirectoryListingEntry as DirectoryListingEntry0 } from "@cocalc/util/types";
 
-export interface ListingItem {
-  name: string;
-  isdir: boolean;
-  isopen?: boolean;
-  mtime?: number;
-  size?: number; // bytes
-}
-
-// NOTE: there is also @cocalc/util/types/directory-listing::DirectoryListingEntry
-// but ATM the relation ship to this one is unclear. Don't mix them up!
-// This type here is used in the frontend, e.g. in Explorer and Flyout Files.
-export interface DirectoryListingEntry {
-  display_name?: string; // unclear, if this even exists
-  name: string;
-  size?: number;
-  mtime?: number;
-  isdir?: boolean;
+// fill in extra info used in the frontend, mainly for the UI
+export interface DirectoryListingEntry extends DirectoryListingEntry0 {
+  // whether or not mask this file in the UI
   mask?: boolean;
-  isopen?: boolean; // opened in an editor
-  isactive?: boolean; // opeend in the currently active editor
-  is_public?: boolean; // a shared file
-  public?: any; // some data about the shared file (TODO type?)
+  // a publicly shared file
+  isPublic?: boolean;
+
+  // used in flyout panels
+  isOpen?: boolean;
+  isActive?: boolean;
 }
 
 export type DirectoryListing = DirectoryListingEntry[];
diff --git a/src/packages/frontend/project/listing/use-files.ts b/src/packages/frontend/project/listing/use-files.ts
index 7075c98b4c..4d681b5d0c 100644
--- a/src/packages/frontend/project/listing/use-files.ts
+++ b/src/packages/frontend/project/listing/use-files.ts
@@ -152,7 +152,7 @@ async function cacheNeighbors({
 }) {
   let v: string[] = [];
   for (const dir in files) {
-    if (!dir.startsWith(".") && files[dir].isdir) {
+    if (!dir.startsWith(".") && files[dir].isDir) {
       const full = join(path, dir);
       const k = key(cacheId, full);
       if (!cache.has(k) && !failed.has(k)) {
diff --git a/src/packages/frontend/project/listing/use-listing.ts b/src/packages/frontend/project/listing/use-listing.ts
index 764d602108..e49cdfafcd 100644
--- a/src/packages/frontend/project/listing/use-listing.ts
+++ b/src/packages/frontend/project/listing/use-listing.ts
@@ -5,7 +5,7 @@ TESTS: See packages/test/project/listing/
 */
 
 import { useMemo } from "react";
-import { DirectoryListingEntry } from "@cocalc/util/types";
+import { type DirectoryListingEntry } from "@cocalc/frontend/project/explorer/types";
 import { field_cmp } from "@cocalc/util/misc";
 import useFiles from "./use-files";
 import { type FilesystemClient } from "@cocalc/conat/files/fs";
diff --git a/src/packages/frontend/project/page/flyouts/active-group.tsx b/src/packages/frontend/project/page/flyouts/active-group.tsx
index e284164888..763e512e97 100644
--- a/src/packages/frontend/project/page/flyouts/active-group.tsx
+++ b/src/packages/frontend/project/page/flyouts/active-group.tsx
@@ -75,7 +75,7 @@ export function Group({
     const fileType = file_options(`foo.${group}`);
     return {
       iconName:
-        group === "" ? UNKNOWN_FILE_TYPE_ICON : fileType?.icon ?? "file",
+        group === "" ? UNKNOWN_FILE_TYPE_ICON : (fileType?.icon ?? "file"),
       display: (group === "" ? "No extension" : fileType?.name) || group,
     };
   }
@@ -83,7 +83,7 @@ export function Group({
   switch (mode) {
     case "folder":
       const isHome = group === "";
-      const isopen = openFilesGrouped[group].some((path) =>
+      const isOpen = openFilesGrouped[group].some((path) =>
         openFiles.includes(path),
       );
       return (
@@ -93,9 +93,9 @@ export function Group({
           mode="active"
           item={{
             name: group,
-            isdir: true,
-            isopen,
-            isactive: current_path === group && activeTab === "files",
+            isDir: true,
+            isOpen,
+            isActive: current_path === group && activeTab === "files",
           }}
           multiline={false}
           displayedNameOverride={displayed}
diff --git a/src/packages/frontend/project/page/flyouts/active.tsx b/src/packages/frontend/project/page/flyouts/active.tsx
index d8b81fa76c..8436d1cde9 100644
--- a/src/packages/frontend/project/page/flyouts/active.tsx
+++ b/src/packages/frontend/project/page/flyouts/active.tsx
@@ -194,7 +194,7 @@ export function ActiveFlyout(props: Readonly): React.JSX.Element {
     filteredFiles.forEach((path) => {
       const { head, tail } = path_split(path);
       const group =
-        mode === "folder" ? head : filename_extension_notilde(tail) ?? "";
+        mode === "folder" ? head : (filename_extension_notilde(tail) ?? "");
       if (grouped[group] == null) grouped[group] = [];
       grouped[group].push(path);
     });
@@ -258,7 +258,7 @@ export function ActiveFlyout(props: Readonly): React.JSX.Element {
     group?: string,
     isLast?: boolean,
   ): React.JSX.Element {
-    const isactive: boolean = activePath === path;
+    const isActive: boolean = activePath === path;
     const style =
       group != null
         ? {
@@ -267,12 +267,12 @@ export function ActiveFlyout(props: Readonly): React.JSX.Element {
           }
         : undefined;
 
-    const isdir = path.endsWith("/");
-    const isopen = openFiles.includes(path);
+    const isDir = path.endsWith("/");
+    const isOpen = openFiles.includes(path);
 
     // if it is a directory, remove the trailing slash
     // and if it starts with ".smc/root/", replace that by a "/"
-    const display = isdir
+    const display = isDir
       ? path.slice(0, -1).replace(/^\.smc\/root\//, "/")
       : undefined;
 
@@ -280,7 +280,7 @@ export function ActiveFlyout(props: Readonly): React.JSX.Element {
       ): React.JSX.Element {
           // we only toggle star, if it is currently opened!
           // otherwise, when closed and accidentally clicking on the star
           // the file unstarred and just vanishes
-          if (isopen) {
+          if (isOpen) {
             setStarredPath(path, starState);
           } else {
             handleFileClick(undefined, path, "star");
diff --git a/src/packages/frontend/project/page/flyouts/file-list-item.tsx b/src/packages/frontend/project/page/flyouts/file-list-item.tsx
index 341435e9d2..45755b57e1 100644
--- a/src/packages/frontend/project/page/flyouts/file-list-item.tsx
+++ b/src/packages/frontend/project/page/flyouts/file-list-item.tsx
@@ -107,14 +107,14 @@ const CLOSE_ICON_STYLE: CSS = {
 };
 
 interface Item {
-  isopen?: boolean;
-  isdir?: boolean;
-  isactive?: boolean;
-  is_public?: boolean;
+  isOpen?: boolean;
+  isDir?: boolean;
+  isActive?: boolean;
+  isPublic?: boolean;
   name: string;
   size?: number;
   mask?: boolean;
-  link_target?: string;
+  linkTarget?: string;
 }
 
 interface FileListItemProps {
@@ -184,7 +184,7 @@ export const FileListItem = React.memo((props: Readonly) => {
   const bodyRef = useRef(null);
 
   function renderCloseItem(item: Item): React.JSX.Element | null {
-    if (onClose == null || !item.isopen) return null;
+    if (onClose == null || !item.isOpen) return null;
 
     const { name } = item;
     return (
@@ -200,7 +200,7 @@ export const FileListItem = React.memo((props: Readonly) => {
   }
 
   function renderPublishedIcon(): React.JSX.Element | undefined {
-    if (!showPublish || !item.is_public) return undefined;
+    if (!showPublish || !item.isPublic) return undefined;
     return (
       
         

@@ -283,7 +283,7 @@ export const FileListItem = React.memo((props: Readonly) => {
         ? selected
           ? "check-square"
           : "square"
-        : item.isdir
+        : item.isDir
           ? "folder-open"
           : (file_options(item.name)?.icon ?? "file"));
 
@@ -315,7 +315,7 @@ export const FileListItem = React.memo((props: Readonly) => {
         name={icon}
         style={{
           ...ICON_STYLE,
-          color: isStarred && item.isopen ? COLORS.STAR : COLORS.GRAY_L,
+          color: isStarred && item.isOpen ? COLORS.STAR : COLORS.GRAY_L,
         }}
         onClick={(e: React.MouseEvent) => {
           e?.stopPropagation();
@@ -329,8 +329,8 @@ export const FileListItem = React.memo((props: Readonly) => {
     const currentExtra = type === 1 ? extra : extra2;
     if (currentExtra == null) return;
     // calculate extra margin to align the columns. if there is no "onClose", no margin
-    const closeMargin = onClose != null ? (item.isopen ? 0 : 18) : 0;
-    const publishMargin = showPublish ? (item.is_public ? 0 : 20) : 0;
+    const closeMargin = onClose != null ? (item.isOpen ? 0 : 18) : 0;
+    const publishMargin = showPublish ? (item.isPublic ? 0 : 20) : 0;
     const marginRight = type === 1 ? publishMargin + closeMargin : undefined;
     const widthPx = FLYOUT_DEFAULT_WIDTH_PX * 0.33;
     // if the 2nd extra shows up, fix the width to align the columns
@@ -405,14 +405,14 @@ export const FileListItem = React.memo((props: Readonly) => {
     item: Item,
     multiple: boolean,
   ) {
-    const { isdir, name: fileName } = item;
+    const { isDir, name: fileName } = item;
     const actionNames = multiple
       ? ACTION_BUTTONS_MULTI
-      : isdir
+      : isDir
         ? ACTION_BUTTONS_DIR
         : ACTION_BUTTONS_FILE;
     for (const key of actionNames) {
-      if (key === "download" && !item.isdir) continue;
+      if (key === "download" && !item.isDir) continue;
       const disabled =
         isDisabledSnapshots(key) &&
         (current_path?.startsWith(".snapshots") ?? false);
@@ -446,13 +446,13 @@ export const FileListItem = React.memo((props: Readonly) => {
   }
 
   function getContextMenu(): MenuProps["items"] {
-    const { name, isdir, is_public, size } = item;
+    const { name, isDir, isPublic, size } = item;
     const n = checked_files?.size ?? 0;
     const multiple = n > 1;
 
     const sizeStr = size ? human_readable_size(size) : "";
     const nameStr = trunc_middle(item.name, 30);
-    const typeStr = isdir ? "Folder" : "File";
+    const typeStr = isDir ? "Folder" : "File";
 
     const ctx: NonNullable = [];
 
@@ -466,7 +466,7 @@ export const FileListItem = React.memo((props: Readonly) => {
     } else {
       ctx.push({
         key: "header",
-        icon: ,
+        icon: ,
         label: `${typeStr} ${nameStr}${sizeStr ? ` (${sizeStr})` : ""}`,
         title: `${name}`,
         style: { fontWeight: "bold" },
@@ -474,14 +474,14 @@ export const FileListItem = React.memo((props: Readonly) => {
       ctx.push({
         key: "open",
         icon: ,
-        label: isdir ? "Open folder" : "Open file",
+        label: isDir ? "Open folder" : "Open file",
         onClick: () => onClick?.(),
       });
     }
 
     ctx.push({ key: "divider-header", type: "divider" });
 
-    if (is_public && typeof onPublic === "function") {
+    if (isPublic && typeof onPublic === "function") {
       ctx.push({
         key: "public",
         label: "Item is published",
@@ -495,7 +495,7 @@ export const FileListItem = React.memo((props: Readonly) => {
 
     // view/download buttons at the bottom
     const showDownload = !student_project_functionality.disableActions;
-    if (name !== ".." && !isdir && showDownload && !multiple) {
+    if (name !== ".." && !isDir && showDownload && !multiple) {
       const full_path = path_to_file(current_path, name);
       const ext = (filename_extension(name) ?? "").toLowerCase();
       const showView = VIEWABLE_FILE_EXT.includes(ext);
@@ -528,11 +528,11 @@ export const FileListItem = React.memo((props: Readonly) => {
   // because all those files are opened
   const activeStyle: CSS =
     mode === "active"
-      ? item.isactive
+      ? item.isActive
         ? FILE_ITEM_ACTIVE_STYLE_2
         : {}
-      : item.isopen
-        ? item.isactive
+      : item.isOpen
+        ? item.isActive
           ? FILE_ITEM_ACTIVE_STYLE
           : FILE_ITEM_OPENED_STYLE
         : {};
diff --git a/src/packages/frontend/project/page/flyouts/files-bottom.tsx b/src/packages/frontend/project/page/flyouts/files-bottom.tsx
index ea186390a4..7d075bc9e2 100644
--- a/src/packages/frontend/project/page/flyouts/files-bottom.tsx
+++ b/src/packages/frontend/project/page/flyouts/files-bottom.tsx
@@ -185,8 +185,8 @@ export function FilesBottom({
 
   function renderDownloadView() {
     if (!singleFile) return;
-    const { name, isdir, size = 0 } = singleFile;
-    if (isdir) return;
+    const { name, isDir, size = 0 } = singleFile;
+    if (isDir) return;
     const full_path = path_to_file(current_path, name);
     const ext = (filename_extension(name) ?? "").toLowerCase();
     const showView = VIEWABLE_FILE_EXT.includes(ext);
@@ -276,7 +276,7 @@ export function FilesBottom({
     if (checked_files.size === 0) {
       let totSize = 0;
       for (const f of directoryFiles) {
-        if (!f.isdir) totSize += f.size ?? 0;
+        if (!f.isDir) totSize += f.size ?? 0;
       }
       return (
         

@@ -292,7 +292,7 @@ export function FilesBottom({
     if (checked_files.size === 0) {
       let [nFiles, nDirs] = [0, 0];
       for (const f of directoryFiles) {
-        if (f.isdir) {
+        if (f.isDir) {
           nDirs++;
         } else {
           nFiles++;
@@ -307,7 +307,7 @@ export function FilesBottom({
       );
     } else if (singleFile) {
       const name = singleFile.name;
-      const iconName = singleFile.isdir
+      const iconName = singleFile.isDir
         ? "folder"
         : file_options(name)?.icon ?? "file";
       return (
diff --git a/src/packages/frontend/project/page/flyouts/files-controls.tsx b/src/packages/frontend/project/page/flyouts/files-controls.tsx
index 18c2b5db2a..5f150df239 100644
--- a/src/packages/frontend/project/page/flyouts/files-controls.tsx
+++ b/src/packages/frontend/project/page/flyouts/files-controls.tsx
@@ -6,7 +6,6 @@
 import { Button, Descriptions, Space, Tooltip } from "antd";
 import immutable from "immutable";
 import { useIntl } from "react-intl";
-
 import { useActions, useTypedRedux } from "@cocalc/frontend/app-framework";
 import { Icon, TimeAgo } from "@cocalc/frontend/components";
 import {
@@ -15,7 +14,7 @@ import {
   ACTION_BUTTONS_MULTI,
   isDisabledSnapshots,
 } from "@cocalc/frontend/project/explorer/action-bar";
-import {
+import type {
   DirectoryListing,
   DirectoryListingEntry,
 } from "@cocalc/frontend/project/explorer/types";
@@ -68,7 +67,7 @@ export function FilesSelectedControls({
       const basename = path_split(file).tail;
       const index = directoryFiles.findIndex((f) => f.name === basename);
       // skipping directories, because it makes no sense to flip through them rapidly
-      if (skipDirs && getFile(file)?.isdir) {
+      if (skipDirs && getFile(file)?.isDir) {
         open(e, index, true);
         continue;
       }
@@ -83,7 +82,7 @@ export function FilesSelectedControls({
 
     let [nFiles, nDirs] = [0, 0];
     for (const f of directoryFiles) {
-      if (f.isdir) {
+      if (f.isDir) {
         nDirs++;
       } else {
         nFiles++;
@@ -100,7 +99,7 @@ export function FilesSelectedControls({
 
   function renderFileInfoBottom() {
     if (singleFile != null) {
-      const { size, mtime, isdir } = singleFile;
+      const { size, mtime, isDir } = singleFile;
       const age = typeof mtime === "number" ? mtime : null;
       return (
         
@@ -109,7 +108,7 @@ export function FilesSelectedControls({
               
             
           ) : undefined}
-          {isdir ? (
+          {isDir ? (
             
               {size} {plural(size, "item")}
             
@@ -118,7 +117,7 @@ export function FilesSelectedControls({
               {human_readable_size(size)}
             
           )}
-          {singleFile.is_public ? (
+          {singleFile.isPublic ? (
             
               
     );
@@ -253,12 +260,10 @@ export default function Configure(props: Props) {
       
         <a
           onClick={() => {
-            redux
-              .getProjectActions(props.project_id)
-              ?.load_target("files/" + props.path);
+            redux.getProjectActions(project_id)?.load_target("files/" + path);
           }}
         >
-          {trunc_middle(props.path, 128)}
+          {trunc_middle(path, 128)}
         </a>
         <span style={{ float: "right" }}>{renderFinishedButton()}</span>
       
@@ -282,7 +287,7 @@ export default function Configure(props: Props) {
       
       
         
-          {!parent_is_public && (
+          {!parentIsPublic && (
             <>
               
                 
                       
                       {STATES.public_listed} - on the{" "}
                       
                         public search engine indexed server.{" "}
-                        {!props.has_network_access && (
+                        {!has_network_access && (
                           
                             (This project must be upgraded to have Internet
                             access.)
@@ -349,16 +354,16 @@ export default function Configure(props: Props) {
               
             
           )}
-          {parent_is_public && props.public != null && (
+          {parentIsPublic && publicInfo != null && (
             
-                  This {props.isdir ? "directory" : "file"} is public because it
-                  is in the public folder "{props.public.path}". Adjust the
-                  sharing configuration of that folder instead.
+                  This is public because it is in the public folder "
+                  {publicInfo.path}". Adjust the sharing configuration of that
+                  folder instead.
                 
               }
             />
@@ -393,11 +398,11 @@ export default function Configure(props: Props) {
                   style={{ paddingTop: "5px", margin: "15px 0" }}
                   value={description}
                   onChange={(e) => setDescription(e.target.value)}
-                  disabled={parent_is_public}
+                  disabled={parentIsPublic}
                   placeholder="Describe what you are sharing.  You can change this at any time."
-                  onKeyUp={props.action_key}
+                  onKeyUp={action_key}
                   onBlur={() => {
-                    props.set_public_path({ description });
+                    set_public_path({ description });
                   }}
                 />
               

@@ -415,11 +420,9 @@ export default function Configure(props: Props) {
                 
                 
                   
-                      props.set_public_path({ license })
-                    }
+                    set_license={(license) => set_public_path({ license })}
                   />
                 
 
@@ -431,7 +434,7 @@ export default function Configure(props: Props) {
                     licenseId={licenseId}
                     setLicenseId={(licenseId) => {
                       setLicenseId(licenseId);
-                      props.set_public_path({ site_license_id: licenseId });
+                      set_public_path({ site_license_id: licenseId });
                     }}
                   />
                   
@@ -446,10 +449,10 @@ export default function Configure(props: Props) {
                 
               

                {
-                  props.set_public_path({ jupyter_api });
+                  set_public_path({ jupyter_api });
                 }}
               />
             
@@ -477,12 +480,12 @@ export default function Configure(props: Props) {
{ - props.set_public_path({ redirect }); + set_public_path({ redirect }); }} - disabled={parent_is_public} + disabled={parentIsPublic} /> diff --git a/src/packages/util/types/directory-listing.ts b/src/packages/util/types/directory-listing.ts index 3352a99268..11e82d5472 100644 --- a/src/packages/util/types/directory-listing.ts +++ b/src/packages/util/types/directory-listing.ts @@ -1,12 +1,15 @@ export interface DirectoryListingEntry { + // relative path (to containing directory) name: string; - isdir?: boolean; - issymlink?: boolean; + // number of *bytes* used to store this path. + size: number; + // last modification time in ms of this file + mtime: number; + // true if it is a directory + isDir?: boolean; + // true if it is a symlink + isSymLink?: boolean; // set if issymlink is true and we're able to determine the target of the link - link_target?: string; - // bytes for file, number of entries for directory (*including* . and ..). - size?: number; - mtime?: number; + linkTarget?: string; error?: string; - mask?: boolean; } From 322fad2a82f997bc5ca89df1183d5a0767ad351c Mon Sep 17 00:00:00 2001 From: William Stein Date: Wed, 30 Jul 2025 23:50:08 +0000 Subject: [PATCH 142/270] mainly fixing publish state view and editing - also remove all use of the abbrevation 'dflt' from our codebase for variable names; this is more consistent, since we basically don't use any abbreviations. --- .../file-server/btrfs/subvolume-bup.ts | 4 +- .../admin/site-settings/row-entry.tsx | 5 +- .../frontend/course/shared-project/actions.ts | 6 +- .../course/student-projects/actions.ts | 8 +- .../frontend/custom-software/selector.tsx | 20 ++-- .../frontend/project/explorer/action-box.tsx | 9 +- .../frontend/project/explorer/explorer.tsx | 23 +++- .../explorer/file-listing/file-listing.tsx | 7 +- .../explorer/file-listing/file-row.tsx | 3 +- .../frontend/project/explorer/types.ts | 10 +- .../project/page/flyouts/files-bottom.tsx | 5 +- .../project/page/flyouts/files-controls.tsx | 4 +- .../project/page/flyouts/files-header.tsx | 7 +- .../frontend/project/page/flyouts/files.tsx | 45 +++----- .../project/settings/software-env-info.tsx | 2 +- src/packages/frontend/project_actions.ts | 4 +- src/packages/frontend/project_store.ts | 105 ++++++++---------- src/packages/frontend/share/config.tsx | 94 +++++++--------- src/packages/frontend/share/license.tsx | 9 +- src/packages/jupyter/redux/store.ts | 4 +- .../next/components/store/quota-config.tsx | 7 +- .../components/store/quota-query-params.ts | 4 +- .../next/components/store/site-license.tsx | 12 +- src/packages/util/db-schema/llm-utils.ts | 6 +- src/packages/util/sanitize-software-envs.ts | 12 +- src/packages/util/upgrades/consts.ts | 14 +-- src/packages/util/upgrades/quota.ts | 24 ++-- .../smc_sagews/tests/test_sagews_modes.py | 24 ++-- 28 files changed, 239 insertions(+), 238 deletions(-) diff --git a/src/packages/file-server/btrfs/subvolume-bup.ts b/src/packages/file-server/btrfs/subvolume-bup.ts index 3237e32dba..a8ec09a945 100644 --- a/src/packages/file-server/btrfs/subvolume-bup.ts +++ b/src/packages/file-server/btrfs/subvolume-bup.ts @@ -126,10 +126,10 @@ export class SubvolumeBup { } const mtime = parseBupTime(name).valueOf() / 1000; newest = Math.max(mtime, newest); - v.push({ name, isDir: true, mtime }); + v.push({ name, isDir: true, mtime, size: -1 }); } if (v.length > 0) { - v.push({ name: "latest", isDir: true, mtime: newest }); + v.push({ name: "latest", isDir: true, mtime: newest, size: -1 }); } return v; } diff --git a/src/packages/frontend/admin/site-settings/row-entry.tsx b/src/packages/frontend/admin/site-settings/row-entry.tsx index 8c964e00ca..22e5ed4858 100644 --- a/src/packages/frontend/admin/site-settings/row-entry.tsx +++ b/src/packages/frontend/admin/site-settings/row-entry.tsx @@ -168,9 +168,8 @@ function VersionHint({ value }: { value: string }) { // The production site works differently. // TODO: make this a more sophisticated data editor. function JsonEntry({ name, data, readonly, onJsonEntryChange }) { - const jval = JSON.parse(data ?? "{}") ?? {}; - const dflt = FIELD_DEFAULTS[name]; - const quotas = { ...dflt, ...jval }; + const jsonValue = JSON.parse(data ?? "{}") ?? {}; + const quotas = { ...FIELD_DEFAULTS[name], ...jsonValue }; const value = JSON.stringify(quotas); return ( => { diff --git a/src/packages/frontend/course/student-projects/actions.ts b/src/packages/frontend/course/student-projects/actions.ts index aeb9ea37a3..a173dda6c0 100644 --- a/src/packages/frontend/course/student-projects/actions.ts +++ b/src/packages/frontend/course/student-projects/actions.ts @@ -66,13 +66,13 @@ export class StudentProjectsActions { const id = this.course_actions.set_activity({ desc: `Create project for ${store.get_student_name(student_id)}.`, }); - const dflt_img = await redux.getStore("customize").getDefaultComputeImage(); + const defaultImage = await redux.getStore("customize").getDefaultComputeImage(); let project_id: string; try { project_id = await redux.getActions("projects").create_project({ title: store.get("settings").get("title"), description: store.get("settings").get("description"), - image: store.get("settings").get("custom_image") ?? dflt_img, + image: store.get("settings").get("custom_image") ?? defaultImage, noPool: true, // student is unlikely to use the project right *now* }); } catch (err) { @@ -607,8 +607,8 @@ export class StudentProjectsActions { ): Promise => { const store = this.get_store(); if (store == null) return; - const dflt_img = await redux.getStore("customize").getDefaultComputeImage(); - const img_id = store.get("settings").get("custom_image") ?? dflt_img; + const defaultImage = await redux.getStore("customize").getDefaultComputeImage(); + const img_id = store.get("settings").get("custom_image") ?? defaultImage; const actions = redux.getProjectActions(student_project_id); await actions.set_compute_image(img_id); }; diff --git a/src/packages/frontend/custom-software/selector.tsx b/src/packages/frontend/custom-software/selector.tsx index 7e2f716369..5e7c9309d5 100644 --- a/src/packages/frontend/custom-software/selector.tsx +++ b/src/packages/frontend/custom-software/selector.tsx @@ -3,7 +3,7 @@ * License: MS-RSL – see LICENSE.md for details */ -// cSpell:ignore descr disp dflt +// cSpell:ignore descr disp import { Col, Form } from "antd"; import { FormattedMessage, useIntl } from "react-intl"; @@ -43,11 +43,11 @@ export async function derive_project_img_name( custom_software: SoftwareEnvironmentState, ): Promise { const { image_type, image_selected } = custom_software; - const dflt_software_img = await redux + const defaultSoftwareImage = await redux .getStore("customize") .getDefaultComputeImage(); if (image_selected == null || image_type == null) { - return dflt_software_img; + return defaultSoftwareImage; } switch (image_type) { case "custom": @@ -56,7 +56,7 @@ export async function derive_project_img_name( return image_selected; default: unreachable(image_type); - return dflt_software_img; // make TS happy + return defaultSoftwareImage; // make TS happy } } @@ -77,7 +77,7 @@ export function SoftwareEnvironment(props: Props) { const onCoCalcCom = customize_kucalc === KUCALC_COCALC_COM; const customize_software = useTypedRedux("customize", "software"); const organization_name = useTypedRedux("customize", "organization_name"); - const dflt_software_img = customize_software.get("default"); + const defaultSoftwareImage = customize_software.get("default"); const software_images = customize_software.get("environments"); const haveSoftwareImages: boolean = useMemo( @@ -109,7 +109,7 @@ export function SoftwareEnvironment(props: Props) { // initialize selection, if there is a default image set React.useEffect(() => { - if (default_image == null || default_image === dflt_software_img) { + if (default_image == null || default_image === defaultSoftwareImage) { // do nothing, that's the initial state already! } else if (is_custom_image(default_image)) { if (images == null) return; @@ -123,7 +123,7 @@ export function SoftwareEnvironment(props: Props) { } else { // must be standard image const img = software_images.get(default_image); - const display = img != null ? img.get("title") ?? "" : ""; + const display = img != null ? (img.get("title") ?? "") : ""; setState(default_image, display, "standard"); } }, []); @@ -170,7 +170,7 @@ export function SoftwareEnvironment(props: Props) { } function render_onprem() { - const selected = image_selected ?? dflt_software_img; + const selected = image_selected ?? defaultSoftwareImage; return ( <> @@ -219,7 +219,7 @@ export function SoftwareEnvironment(props: Props) { } function render_standard_image_selector() { - const isCustom = is_custom_image(image_selected ?? dflt_software_img); + const isCustom = is_custom_image(image_selected ?? defaultSoftwareImage); return ( <> @@ -230,7 +230,7 @@ export function SoftwareEnvironment(props: Props) { > { diff --git a/src/packages/frontend/project/explorer/action-box.tsx b/src/packages/frontend/project/explorer/action-box.tsx index 1e05857215..2a25f09bfd 100644 --- a/src/packages/frontend/project/explorer/action-box.tsx +++ b/src/packages/frontend/project/explorer/action-box.tsx @@ -22,7 +22,10 @@ import { Icon, Loading, LoginLink } from "@cocalc/frontend/components"; import SelectServer from "@cocalc/frontend/compute/select-server"; import ComputeServerTag from "@cocalc/frontend/compute/server-tag"; import { useRunQuota } from "@cocalc/frontend/project/settings/run-quota/hooks"; -import { file_actions, ProjectActions } from "@cocalc/frontend/project_store"; +import { + file_actions, + type ProjectActions, +} from "@cocalc/frontend/project_store"; import { SelectProject } from "@cocalc/frontend/projects/select-project"; import ConfigureShare from "@cocalc/frontend/share/config"; import * as misc from "@cocalc/util/misc"; @@ -550,8 +553,8 @@ export function ActionBox({ path={path} compute_server_id={compute_server_id} close={cancel_action} - action_key={action_key} - set_public_path={(opts) => actions.set_public_path(path, opts)} + onKeyUp={action_key} + actions={actions} has_network_access={!!runQuota.network} /> ); diff --git a/src/packages/frontend/project/explorer/explorer.tsx b/src/packages/frontend/project/explorer/explorer.tsx index 3ec833a4aa..47349a95c4 100644 --- a/src/packages/frontend/project/explorer/explorer.tsx +++ b/src/packages/frontend/project/explorer/explorer.tsx @@ -6,7 +6,13 @@ import * as _ from "lodash"; import { UsersViewing } from "@cocalc/frontend/account/avatar/users-viewing"; import { Col, Row } from "@cocalc/frontend/antd-bootstrap"; -import { type CSSProperties, useEffect, useRef, useState } from "react"; +import { + type CSSProperties, + useEffect, + useMemo, + useRef, + useState, +} from "react"; import { A, ActivityDisplay, @@ -41,6 +47,10 @@ import useListing, { import filterListing from "@cocalc/frontend/project/listing/filter-listing"; import ShowError from "@cocalc/frontend/components/error"; import { MainConfiguration } from "@cocalc/frontend/project_configuration"; +import { + getPublicFiles, + useStrippedPublicPaths, +} from "@cocalc/frontend/project_store"; const FLEX_ROW_STYLE = { display: "flex", @@ -109,6 +119,7 @@ export function Explorer() { "show_custom_software_reset", ); const show_library = useTypedRedux({ project_id }, "show_library"); + const [shiftIsDown, setShiftIsDown] = useState(false); const project_map = useTypedRedux("projects", "project_map"); @@ -139,6 +150,15 @@ export function Explorer() { actions?.setState({ numDisplayedFiles: listing?.length ?? 0 }); }, [listing?.length]); + // ensure that listing entries have isPublic set: + const strippedPublicPaths = useStrippedPublicPaths(project_id); + const publicFiles: Set = useMemo(() => { + if (listing == null) { + return new Set(); + } + return getPublicFiles(listing, strippedPublicPaths, current_path); + }, [listing, current_path, strippedPublicPaths]); + useEffect(() => { if (listing == null) { return; @@ -497,6 +517,7 @@ export function Explorer() { configuration_main={ configuration?.get("main") as MainConfiguration | undefined } + publicFiles={publicFiles} /> )} diff --git a/src/packages/frontend/project/explorer/file-listing/file-listing.tsx b/src/packages/frontend/project/explorer/file-listing/file-listing.tsx index 4663119389..1430f51eca 100644 --- a/src/packages/frontend/project/explorer/file-listing/file-listing.tsx +++ b/src/packages/frontend/project/explorer/file-listing/file-listing.tsx @@ -35,6 +35,7 @@ interface Props { configuration_main?: MainConfiguration; isRunning?: boolean; // true if this project is running stale?: boolean; + publicFiles: Set; } export function FileListing({ @@ -47,6 +48,7 @@ export function FileListing({ configuration_main, file_search = "", stale, + publicFiles, }: Props) { const active_file_sort = useTypedRedux({ project_id }, "active_file_sort"); const computeServerId = useTypedRedux({ project_id }, "compute_server_id"); @@ -55,12 +57,15 @@ export function FileListing({ const name = actions.name; function renderRow(index, file) { - const checked = checked_files.has(misc.path_to_file(current_path, name)); + const checked = checked_files.has( + misc.path_to_file(current_path, file.name), + ); const color = misc.rowBackground({ index, checked }); return ( void; selectAllFiles: () => void; getFile: (path: string) => DirectoryListingEntry | undefined; + publicFiles: Set; } export function FilesBottom({ @@ -78,6 +79,7 @@ export function FilesBottom({ showFileSharingDialog, getFile, directoryFiles, + publicFiles, }: FilesBottomProps) { const [mode, setMode] = modeState; const current_path = useTypedRedux({ project_id }, "current_path"); @@ -268,6 +270,7 @@ export function FilesBottom({ getFile={getFile} mode="bottom" activeFile={activeFile} + publicFiles={publicFiles} /> ); } @@ -309,7 +312,7 @@ export function FilesBottom({ const name = singleFile.name; const iconName = singleFile.isDir ? "folder" - : file_options(name)?.icon ?? "file"; + : (file_options(name)?.icon ?? "file"); return ( ); diff --git a/src/packages/frontend/project/settings/software-env-info.tsx b/src/packages/frontend/project/settings/software-env-info.tsx index 25ba23a6c8..15616a227c 100644 --- a/src/packages/frontend/project/settings/software-env-info.tsx +++ b/src/packages/frontend/project/settings/software-env-info.tsx @@ -3,7 +3,7 @@ * License: MS-RSL – see LICENSE.md for details */ -// cSpell:ignore descr disp dflt +// cSpell:ignore descr disp import { join } from "path"; import { FormattedMessage } from "react-intl"; diff --git a/src/packages/frontend/project_actions.ts b/src/packages/frontend/project_actions.ts index 3b8d507b0c..177ed7c9c6 100644 --- a/src/packages/frontend/project_actions.ts +++ b/src/packages/frontend/project_actions.ts @@ -2922,13 +2922,13 @@ export class ProjectActions extends Actions { const id = client_db.sha1(project_id, path); const projects_store = redux.getStore("projects"); - const dflt_compute_img = await redux + const defaultComputeImage = await redux .getStore("customize") .getDefaultComputeImage(); const compute_image: string = projects_store.getIn(["project_map", project_id, "compute_image"]) ?? - dflt_compute_img; + defaultComputeImage; const table = this.redux.getProjectTable(project_id, "public_paths"); let obj: undefined | Map = table._table.get(id); diff --git a/src/packages/frontend/project_store.ts b/src/packages/frontend/project_store.ts index 78fb2e0aea..4e92521389 100644 --- a/src/packages/frontend/project_store.ts +++ b/src/packages/frontend/project_store.ts @@ -16,7 +16,6 @@ if (typeof window !== "undefined" && window !== null) { } import * as immutable from "immutable"; - import { AppRedux, project_redux_name, @@ -24,7 +23,9 @@ import { Store, Table, TypedMap, + useTypedRedux, } from "@cocalc/frontend/app-framework"; +import { useMemo } from "react"; import { fileURL } from "@cocalc/frontend/lib/cocalc-urls"; import { get_local_storage } from "@cocalc/frontend/misc"; import { QueryParams } from "@cocalc/frontend/misc/query-params"; @@ -40,7 +41,7 @@ import { isMainConfiguration, ProjectConfiguration, } from "@cocalc/frontend/project_configuration"; -import * as misc from "@cocalc/util/misc"; +import { containing_public_path, deep_copy } from "@cocalc/util/misc"; import { FixedTab } from "./project/page/file-tab"; import { FlyoutActiveMode, @@ -54,7 +55,8 @@ import { FLYOUT_LOG_FILTER_DEFAULT, FlyoutLogFilter, } from "./project/page/flyouts/utils"; - +import { type PublicPath } from "@cocalc/util/db-schema/public-paths"; +import { DirectoryListing } from "@cocalc/frontend/project/explorer/types"; export { FILE_ACTIONS as file_actions, ProjectActions }; export type ModalInfo = TypedMap<{ @@ -71,7 +73,7 @@ export interface ProjectStoreState { open_files: immutable.Map>; open_files_order: immutable.List; just_closed_files: immutable.List; - public_paths?: immutable.Map>; + public_paths?: immutable.Map>; show_upload: boolean; create_file_alert: boolean; @@ -145,7 +147,6 @@ export interface ProjectStoreState { // Project Settings get_public_path_id?: (path: string) => any; - stripped_public_paths: any; //computed(immutable.List) // Project Info show_project_info_explanation?: boolean; @@ -330,8 +331,6 @@ export class ProjectStore extends Store { most_recent_path: "", // Project Settings - stripped_public_paths: this.selectors.stripped_public_paths.fn, - other_settings: undefined, compute_server_id, @@ -360,26 +359,6 @@ export class ProjectStore extends Store { }; }, }, - - stripped_public_paths: { - dependencies: ["public_paths"] as const, - fn: () => { - const public_paths = this.get("public_paths"); - if (public_paths != null) { - return immutable.fromJS( - (() => { - const result: any[] = []; - const object = public_paths.toJS(); - for (const id in object) { - const x = object[id]; - result.push(misc.copy_without(x, ["id", "project_id"])); - } - return result; - })(), - ); - } - }, - }, }; // Returns the cursor positions for the given project_id/path, if that @@ -432,37 +411,41 @@ export class ProjectStore extends Store { } } -// Mutates data to include info on public paths. -export function mutate_data_to_compute_public_files( - data, - public_paths, - current_path, -) { - const { listing } = data; - const pub = data.public; - if (public_paths != null && public_paths.size > 0) { - const head = current_path ? current_path + "/" : ""; - const paths: string[] = []; - const public_path_data = {}; - for (const x of public_paths.toJS()) { - if (x.disabled) { - // Do not include disabled paths. Otherwise, it causes this confusing bug: - // https://github.com/sagemathinc/cocalc/issues/6159 - continue; - } - public_path_data[x.path] = x; - paths.push(x.path); - } - for (const x of listing) { - const full = head + x.name; - const p = misc.containing_public_path(full, paths); - if (p != null) { - x.public = public_path_data[p]; - x.is_public = !x.public.disabled; - pub[x.name] = public_path_data[p]; - } +// Returns set of paths that are public in the given +// listing, because they are in a public folder or are themselves public. +// This is used entirely to put an extra "public" label in the row of the file, +// when displaying it in a listing. +export function getPublicFiles( + listing: DirectoryListing, + public_paths: PublicPath[], + current_path: string, +): Set { + if ((public_paths?.length ?? 0) == 0) { + return new Set(); + } + const paths = public_paths + .filter(({ disabled }) => !disabled) + .map(({ path }) => path); + + if (paths.length == 0) { + return new Set(); + } + + const head = current_path ? current_path + "/" : ""; + if (containing_public_path(current_path, paths)) { + // fast special case: *every* file is public + return new Set(listing.map(({ name }) => name)); + } + + // maybe some files are public? + const X = new Set(); + for (const file of listing) { + const full = head + file.name; + if (containing_public_path(full, paths) != null) { + X.add(file.name); } } + return X; } export function init(project_id: string, redux: AppRedux): ProjectStore { @@ -486,7 +469,7 @@ export function init(project_id: string, redux: AppRedux): ProjectStore { actions.project_id = project_id; // so actions can assume this is available on the object store._init(); - const queries = misc.deep_copy(QUERIES); + const queries = deep_copy(QUERIES); const create_table = function (table_name, q) { //console.log("create_table", table_name) @@ -543,3 +526,11 @@ export function init(project_id: string, redux: AppRedux): ProjectStore { return store; } + +export function useStrippedPublicPaths(project_id: string): PublicPath[] { + const public_paths = useTypedRedux({ project_id }, "public_paths"); + return useMemo(() => { + const rows = public_paths?.valueSeq()?.toJS() ?? []; + return rows as unknown as PublicPath[]; + }, [public_paths]); +} diff --git a/src/packages/frontend/share/config.tsx b/src/packages/frontend/share/config.tsx index 527aa01524..48b0a407a8 100644 --- a/src/packages/frontend/share/config.tsx +++ b/src/packages/frontend/share/config.tsx @@ -33,8 +33,7 @@ import { Row, Space, } from "antd"; -import { useEffect, useState } from "react"; - +import { useEffect, useMemo, useState } from "react"; import { CSS, redux, useTypedRedux } from "@cocalc/frontend/app-framework"; import { A, @@ -60,45 +59,19 @@ import { COLORS } from "@cocalc/util/theme"; import { ConfigureName } from "./configure-name"; import { License } from "./license"; import { publicShareUrl, shareServerUrl } from "./util"; +import { containing_public_path } from "@cocalc/util/misc"; +import { type PublicPath } from "@cocalc/util/db-schema/public-paths"; +import { type ProjectActions } from "@cocalc/frontend/project_store"; // https://ant.design/components/grid/ const GUTTER: [number, number] = [20, 30]; -interface PublicInfo { - created: Date; - description: string; - disabled: boolean; - last_edited: Date; - path: string; - unlisted: boolean; - authenticated?: boolean; - license?: string; - name?: string; - site_license_id?: string; - redirect?: string; - jupyter_api?: boolean; -} - interface Props { project_id: string; path: string; - size: number; - mtime: number; - isPublic?: boolean; - publicInfo?: PublicInfo; close: (event: any) => void; - action_key: (event: any) => void; - site_license_id?: string; - set_public_path: (options: { - description?: string; - unlisted?: boolean; - license?: string; - disabled?: boolean; - authenticated?: boolean; - site_license_id?: string | null; - redirect?: string; - jupyter_api?: boolean; - }) => void; + onKeyUp?: (event: any) => void; + actions: ProjectActions; has_network_access?: boolean; compute_server_id?: number; } @@ -123,26 +96,40 @@ function SC({ children }) { export default function Configure({ project_id, path, - isPublic, - publicInfo, close, - action_key, - set_public_path, + onKeyUp, + actions, has_network_access, compute_server_id, }: Props) { + const publicPaths = useTypedRedux({ project_id }, "public_paths"); + const publicInfo: null | PublicPath = useMemo(() => { + for (const x of publicPaths?.valueSeq() ?? []) { + if ( + !x.get("disabled") && + containing_public_path(path, [x.get("path")]) != null + ) { + return x.toJS(); + } + } + return null; + }, [publicPaths]); + const student = useStudentProjectFunctionality(project_id); const [description, setDescription] = useState( publicInfo?.description ?? "", ); const [sharingOptionsState, setSharingOptionsState] = useState(() => { - if (isPublic && publicInfo?.unlisted) { + if (publicInfo == null) { + return "private"; + } + if (publicInfo?.unlisted) { return "public_unlisted"; } - if (isPublic && publicInfo?.authenticated) { + if (publicInfo?.authenticated) { return "authenticated"; } - if (isPublic && !publicInfo?.unlisted) { + if (!publicInfo?.unlisted) { return "public_listed"; } return "private"; @@ -176,17 +163,17 @@ export default function Configure({ setSharingOptionsState(state); switch (state) { case "private": - set_public_path(SHARE_FLAGS.DISABLED); + actions.set_public_path(path, SHARE_FLAGS.DISABLED); break; case "public_listed": // public is suppose to work in this state - set_public_path(SHARE_FLAGS.LISTED); + actions.set_public_path(path, SHARE_FLAGS.LISTED); break; case "public_unlisted": - set_public_path(SHARE_FLAGS.UNLISTED); + actions.set_public_path(path, SHARE_FLAGS.UNLISTED); break; case "authenticated": - set_public_path(SHARE_FLAGS.AUTHENTICATED); + actions.set_public_path(path, SHARE_FLAGS.AUTHENTICATED); break; default: unreachable(state); @@ -196,8 +183,7 @@ export default function Configure({ const license = publicInfo?.license ?? ""; // This path is public because some parent folder is public. - const parentIsPublic = - !!isPublic && publicInfo != null && publicInfo.path != path; + const parentIsPublic = publicInfo != null && publicInfo.path != path; const url = publicShareUrl( project_id, @@ -400,9 +386,9 @@ export default function Configure({ onChange={(e) => setDescription(e.target.value)} disabled={parentIsPublic} placeholder="Describe what you are sharing. You can change this at any time." - onKeyUp={action_key} + onKeyUp={onKeyUp} onBlur={() => { - set_public_path({ description }); + actions.set_public_path(path, { description }); }} />
{trunc_middle(name, 20)} diff --git a/src/packages/frontend/project/page/flyouts/files-controls.tsx b/src/packages/frontend/project/page/flyouts/files-controls.tsx index 5f150df239..7246a73666 100644 --- a/src/packages/frontend/project/page/flyouts/files-controls.tsx +++ b/src/packages/frontend/project/page/flyouts/files-controls.tsx @@ -37,6 +37,7 @@ interface FilesSelectedControlsProps { skip?: boolean, ) => void; activeFile: DirectoryListingEntry | null; + publicFiles: Set; } export function FilesSelectedControls({ @@ -48,6 +49,7 @@ export function FilesSelectedControls({ project_id, showFileSharingDialog, activeFile, + publicFiles, }: FilesSelectedControlsProps) { const intl = useIntl(); const current_path = useTypedRedux({ project_id }, "current_path"); @@ -117,7 +119,7 @@ export function FilesSelectedControls({ {human_readable_size(size)} )} - {singleFile.isPublic ? ( + {publicFiles.has(singleFile.name) ? (
@@ -422,7 +408,9 @@ export default function Configure({ set_public_path({ license })} + set_license={(license) => + actions.set_public_path(path, { license }) + } /> @@ -434,7 +422,9 @@ export default function Configure({ licenseId={licenseId} setLicenseId={(licenseId) => { setLicenseId(licenseId); - set_public_path({ site_license_id: licenseId }); + actions.set_public_path(path, { + site_license_id: licenseId, + }); }} /> @@ -452,7 +442,7 @@ export default function Configure({ disabled={parentIsPublic} jupyter_api={publicInfo?.jupyter_api} saveJupyterApi={(jupyter_api) => { - set_public_path({ jupyter_api }); + actions.set_public_path(path, { jupyter_api }); }} /> @@ -483,7 +473,7 @@ export default function Configure({ project_id={project_id} path={publicInfo?.path ?? path} saveRedirect={(redirect) => { - set_public_path({ redirect }); + actions.set_public_path(path, { redirect }); }} disabled={parentIsPublic} /> diff --git a/src/packages/frontend/share/license.tsx b/src/packages/frontend/share/license.tsx index 784e08d6b8..819c9f1e96 100644 --- a/src/packages/frontend/share/license.tsx +++ b/src/packages/frontend/share/license.tsx @@ -12,8 +12,7 @@ between them. I think this is acceptable, since it is unlikely for people to do that. */ -import { FC, memo, useMemo, useState } from "react"; - +import { useMemo, useState } from "react"; import { DropdownMenu } from "@cocalc/frontend/components"; import { MenuItems } from "../components/dropdown-menu"; import { LICENSES } from "./licenses"; @@ -24,9 +23,7 @@ interface Props { disabled?: boolean; } -export const License: FC = memo((props: Props) => { - const { license, set_license, disabled = false } = props; - +export function License({ license, set_license, disabled = false }: Props) { const [sel_license, set_sel_license] = useState(license); function select(license: string): void { @@ -65,4 +62,4 @@ export const License: FC = memo((props: Props) => { items={items} /> ); -}); +} diff --git a/src/packages/jupyter/redux/store.ts b/src/packages/jupyter/redux/store.ts index e611b59eec..a334be7de2 100644 --- a/src/packages/jupyter/redux/store.ts +++ b/src/packages/jupyter/redux/store.ts @@ -451,10 +451,10 @@ export class JupyterStore extends Store { // (??) return `${project_id}-${computeServerId}-default`; } - const dflt_img = await customize.getDefaultComputeImage(); + const defaultImage = await customize.getDefaultComputeImage(); const compute_image = projects_store.getIn( ["project_map", project_id, "compute_image"], - dflt_img, + defaultImage, ); const key = [project_id, `${computeServerId}`, compute_image].join("::"); // console.log("jupyter store / jupyter_kernel_key", key); diff --git a/src/packages/next/components/store/quota-config.tsx b/src/packages/next/components/store/quota-config.tsx index 56a65c62c5..91da4eeaf1 100644 --- a/src/packages/next/components/store/quota-config.tsx +++ b/src/packages/next/components/store/quota-config.tsx @@ -17,7 +17,6 @@ import { Typography, } from "antd"; import { useEffect, useRef, useState, type JSX } from "react"; - import { HelpIcon } from "@cocalc/frontend/components/help-icon"; import { Icon } from "@cocalc/frontend/components/icon"; import { displaySiteLicense } from "@cocalc/util/consts/site-license"; @@ -193,7 +192,7 @@ export const QuotaConfig: React.FC = (props: Props) => { = (props: Props) => { = (props: Props) => { diff --git a/src/packages/next/components/store/quota-query-params.ts b/src/packages/next/components/store/quota-query-params.ts index 92e85f30aa..c492089e6f 100644 --- a/src/packages/next/components/store/quota-query-params.ts +++ b/src/packages/next/components/store/quota-query-params.ts @@ -138,10 +138,10 @@ function decodeValue(val): boolean | number | string | DateRange { function fixNumVal( val: any, - param: { min: number; max: number; dflt: number }, + param: { min: number; max: number; default: number }, ): number { if (typeof val !== "number") { - return param.dflt; + return param.default; } else { return clamp(val, param.min, param.max); } diff --git a/src/packages/next/components/store/site-license.tsx b/src/packages/next/components/store/site-license.tsx index 1b1f99f858..4f46d568d8 100644 --- a/src/packages/next/components/store/site-license.tsx +++ b/src/packages/next/components/store/site-license.tsx @@ -292,14 +292,14 @@ function CreateSiteLicense({ })(); } else { const vals = decodeFormValues(router, "regular"); - const dflt = presets[DEFAULT_PRESET]; + const defaultPreset = presets[DEFAULT_PRESET]; // Only use the configuration fields from the default preset, not the entire object const defaultConfig = { - cpu: dflt.cpu, - ram: dflt.ram, - disk: dflt.disk, - uptime: dflt.uptime, - member: dflt.member, + cpu: defaultPreset.cpu, + ram: defaultPreset.ram, + disk: defaultPreset.disk, + uptime: defaultPreset.uptime, + member: defaultPreset.member, // Add other form fields that might be needed period: source === "course" ? "range" : "monthly", user: source === "course" ? "academic" : "business", diff --git a/src/packages/util/db-schema/llm-utils.ts b/src/packages/util/db-schema/llm-utils.ts index 0542461cfb..092c81681f 100644 --- a/src/packages/util/db-schema/llm-utils.ts +++ b/src/packages/util/db-schema/llm-utils.ts @@ -428,15 +428,15 @@ export function getValidLanguageModelName({ } for (const free of [true, false]) { - const dflt = getDefaultLLM( + const defaultModel = getDefaultLLM( selectable_llms, filter, ollama, custom_openai, free, ); - if (dflt != null) { - return dflt; + if (defaultModel != null) { + return defaultModel; } } return DEFAULT_MODEL; diff --git a/src/packages/util/sanitize-software-envs.ts b/src/packages/util/sanitize-software-envs.ts index 074ba1ac4d..8893d8b851 100644 --- a/src/packages/util/sanitize-software-envs.ts +++ b/src/packages/util/sanitize-software-envs.ts @@ -118,19 +118,19 @@ export function sanitizeSoftwareEnv( return null; } - const swDflt = software["default"]; + const swDefault = software["default"]; // we check that the default is a string and that it exists in envs - const dflt = - typeof swDflt === "string" && envs[swDflt] != null - ? swDflt + const defaultSoftware = + typeof swDefault === "string" && envs[swDefault] != null + ? swDefault : Object.keys(envs)[0]; // this is a fallback entry, when projects were created before the software env was configured if (envs[DEFAULT_COMPUTE_IMAGE] == null) { - envs[DEFAULT_COMPUTE_IMAGE] = { ...envs[dflt], hidden: true }; + envs[DEFAULT_COMPUTE_IMAGE] = { ...envs[defaultSoftware], hidden: true }; } - return { groups, default: dflt, environments: envs }; + return { groups, default: defaultSoftware, environments: envs }; } function fallback(a: any, b: any, c?: string): any { diff --git a/src/packages/util/upgrades/consts.ts b/src/packages/util/upgrades/consts.ts index 6e6ae8d409..90b4d248ea 100644 --- a/src/packages/util/upgrades/consts.ts +++ b/src/packages/util/upgrades/consts.ts @@ -22,7 +22,7 @@ export const MIN_DISK_GB = DISK_DEFAULT_GB; interface Values { min: number; - dflt: number; + default: number; max: number; } @@ -35,25 +35,25 @@ interface Limits { export const REGULAR: Limits = { cpu: { min: 1, - dflt: DEFAULT_CPU, + default: DEFAULT_CPU, max: MAX_CPU, }, ram: { min: 4, - dflt: RAM_DEFAULT_GB, + default: RAM_DEFAULT_GB, max: MAX_RAM_GB, }, disk: { min: MIN_DISK_GB, - dflt: DISK_DEFAULT_GB, + default: DISK_DEFAULT_GB, max: MAX_DISK_GB, }, } as const; export const BOOST: Limits = { - cpu: { min: 0, dflt: 0, max: MAX_CPU - 1 }, - ram: { min: 0, dflt: 0, max: MAX_RAM_GB - 1 }, - disk: { min: 0, dflt: 0, max: MAX_DISK_GB - 1 * DISK_DEFAULT_GB }, + cpu: { min: 0, default: 0, max: MAX_CPU - 1 }, + ram: { min: 0, default: 0, max: MAX_RAM_GB - 1 }, + disk: { min: 0, default: 0, max: MAX_DISK_GB - 1 * DISK_DEFAULT_GB }, } as const; // on-prem: this dedicated VM machine name is only used for cocalc-onprem diff --git a/src/packages/util/upgrades/quota.ts b/src/packages/util/upgrades/quota.ts index 8e532e3887..a7980a6428 100644 --- a/src/packages/util/upgrades/quota.ts +++ b/src/packages/util/upgrades/quota.ts @@ -637,21 +637,21 @@ function calcSiteLicenseQuotaIdleTimeout( // there is an old schema, inherited from SageMathCloud, etc. and newer iterations. // this helps by going from one schema to the newer one function upgrade2quota(up: Partial): RQuota { - const dflt_false = (x) => + const defaultFalse = (x) => x != null ? (typeof x === "boolean" ? x : to_int(x) >= 1) : false; - const dflt_num = (x) => + const defaultNumber = (x) => x != null ? (typeof x === "number" ? x : to_float(x)) : 0; return { - network: dflt_false(up.network), - member_host: dflt_false(up.member_host), - always_running: dflt_false(up.always_running), - disk_quota: dflt_num(up.disk_quota), - memory_limit: dflt_num(up.memory), - memory_request: dflt_num(up.memory_request), - cpu_limit: dflt_num(up.cores), - cpu_request: dflt_num(up.cpu_shares) / 1024, - privileged: dflt_false(up.privileged), - idle_timeout: dflt_num(up.mintime), + network: defaultFalse(up.network), + member_host: defaultFalse(up.member_host), + always_running: defaultFalse(up.always_running), + disk_quota: defaultNumber(up.disk_quota), + memory_limit: defaultNumber(up.memory), + memory_request: defaultNumber(up.memory_request), + cpu_limit: defaultNumber(up.cores), + cpu_request: defaultNumber(up.cpu_shares) / 1024, + privileged: defaultFalse(up.privileged), + idle_timeout: defaultNumber(up.mintime), dedicated_vm: false, // old schema has no dedicated_vm upgrades dedicated_disks: [] as DedicatedDisk[], // old schema has no dedicated_disk upgrades ext_rw: false, diff --git a/src/smc_sagews/smc_sagews/tests/test_sagews_modes.py b/src/smc_sagews/smc_sagews/tests/test_sagews_modes.py index 0f87827740..54ffd9a4f3 100644 --- a/src/smc_sagews/smc_sagews/tests/test_sagews_modes.py +++ b/src/smc_sagews/smc_sagews/tests/test_sagews_modes.py @@ -200,34 +200,34 @@ def test_bad_command(self, exec2): class TestShDefaultMode: - def test_start_sh_dflt(self, exec2): + def test_start_sh_default(self, exec2): exec2("%default_mode sh") - def test_multiline_dflt(self, exec2): + def test_multiline_default(self, exec2): exec2("FOO=hello\necho $FOO", pattern="^hello") def test_date(self, exec2): exec2("date +%Y-%m-%d", pattern=r'^\d{4}-\d{2}-\d{2}') - def test_capture_sh_01_dflt(self, exec2): + def test_capture_sh_01_default(self, exec2): exec2("%capture(stdout='output')\nuptime") - def test_capture_sh_02_dflt(self, exec2): + def test_capture_sh_02_default(self, exec2): exec2("%sage\noutput", pattern="up.*user.*load average") - def test_remember_settings_01_dflt(self, exec2): + def test_remember_settings_01_default(self, exec2): exec2("FOO='testing123'") - def test_remember_settings_02_dflt(self, exec2): + def test_remember_settings_02_default(self, exec2): exec2("echo $FOO", pattern=r"^testing123\s+") - def test_sh_display_dflt(self, execblob, image_file): + def test_sh_display_default(self, execblob, image_file): execblob("display < " + str(image_file), want_html=False) - def test_sh_autocomplete_01_dflt(self, exec2): + def test_sh_autocomplete_01_default(self, exec2): exec2("TESTVAR29=xyz") - def test_sh_autocomplete_02_dflt(self, execintrospect): + def test_sh_autocomplete_02_default(self, execintrospect): execintrospect('echo $TESTV', ["AR29"], '$TESTV') @@ -249,13 +249,13 @@ class TestRDefaultMode: def test_set_r_mode(self, exec2): exec2("%default_mode r") - def test_rdflt_assignment(self, exec2): + def test_rdefault_assignment(self, exec2): exec2("xx <- c(4,7,13)\nmean(xx)", html_pattern="^8$") - def test_dflt_capture_r_01(self, exec2): + def test_default_capture_r_01(self, exec2): exec2("%capture(stdout='output')\nsum(xx)") - def test_dflt_capture_r_02(self, exec2): + def test_default_capture_r_02(self, exec2): exec2("%sage\nprint(output)", "24\n") From 1f04e770b6061990ee937d6bbc78c5503d41598e Mon Sep 17 00:00:00 2001 From: William Stein Date: Thu, 31 Jul 2025 00:30:04 +0000 Subject: [PATCH 143/270] fix isActive and isOpen for file listings to get updated; also make isOpen work for the main explorer --- .../explorer/file-listing/file-listing.tsx | 12 ++++--- .../explorer/file-listing/file-row.tsx | 11 +++++-- .../frontend/project/explorer/types.ts | 8 ----- .../project/page/flyouts/file-list-item.tsx | 2 +- .../frontend/project/page/flyouts/files.tsx | 33 +++++++++---------- 5 files changed, 32 insertions(+), 34 deletions(-) diff --git a/src/packages/frontend/project/explorer/file-listing/file-listing.tsx b/src/packages/frontend/project/explorer/file-listing/file-listing.tsx index 1430f51eca..536bef1a62 100644 --- a/src/packages/frontend/project/explorer/file-listing/file-listing.tsx +++ b/src/packages/frontend/project/explorer/file-listing/file-listing.tsx @@ -16,7 +16,7 @@ import { TypedMap, useTypedRedux, redux } from "@cocalc/frontend/app-framework"; import useVirtuosoScrollHook from "@cocalc/frontend/components/virtuoso-scroll-hook"; import { ProjectActions } from "@cocalc/frontend/project_actions"; import { MainConfiguration } from "@cocalc/frontend/project_configuration"; -import * as misc from "@cocalc/util/misc"; +import { path_to_file, rowBackground } from "@cocalc/util/misc"; import { FileRow } from "./file-row"; import { ListingHeader } from "./listing-header"; import NoFiles from "./no-files"; @@ -55,16 +55,18 @@ export function FileListing({ const selected_file_index = useTypedRedux({ project_id }, "selected_file_index") ?? 0; const name = actions.name; + const openFiles = new Set( + useTypedRedux({ project_id }, "open_files_order")?.toJS() ?? [], + ); function renderRow(index, file) { - const checked = checked_files.has( - misc.path_to_file(current_path, file.name), - ); - const color = misc.rowBackground({ index, checked }); + const checked = checked_files.has(path_to_file(current_path, file.name)); + const color = rowBackground({ index, checked }); return ( = [ "md", @@ -49,6 +50,7 @@ interface Props { color: string; mask: boolean; isPublic: boolean; + isOpen: boolean; current_path: string; actions: ProjectActions; no_select: boolean; @@ -70,6 +72,7 @@ export function FileRow({ color, mask, isPublic, + isOpen, current_path, actions, no_select, @@ -149,12 +152,14 @@ export function FileRow({ (display_name != undefined && name0 !== display_name) || name0.length > 50; - const styles = { + const style = { whiteSpace: "pre-wrap", wordWrap: "break-word", overflowWrap: "break-word", verticalAlign: "middle", color: mask ? "#bbbbbb" : COLORS.TAB, + ...(isOpen ? FILE_ITEM_OPENED_STYLE : undefined), + backgroundColor: undefined, }; if (show_tip) { @@ -167,11 +172,11 @@ export function FileRow({ } tip={name0} > - {render_name_link(styles, name0, ext)} + {render_name_link(style, name0, ext)} ); } else { - return render_name_link(styles, name0, ext); + return render_name_link(style, name0, ext); } } diff --git a/src/packages/frontend/project/explorer/types.ts b/src/packages/frontend/project/explorer/types.ts index 563f12b4ab..cbab305a19 100644 --- a/src/packages/frontend/project/explorer/types.ts +++ b/src/packages/frontend/project/explorer/types.ts @@ -9,14 +9,6 @@ import type { DirectoryListingEntry as DirectoryListingEntry0 } from "@cocalc/ut export interface DirectoryListingEntry extends DirectoryListingEntry0 { // whether or not to mask this file in the UI mask?: boolean; - - // This is used in flyout panels. TODO: Mutating listings based on status info - // like this that randomly changes (unlik mask above) will lead to subtle state bugs or requiring - // inefficient frequent rerenders. Instead one should make a separate - // Set of the paths of open files and active files and use that in the UI. I'm not fixing - // this now since it is only used in the flyout panels and not the main explorer. - isOpen?: boolean; - isActive?: boolean; } export type DirectoryListing = DirectoryListingEntry[]; diff --git a/src/packages/frontend/project/page/flyouts/file-list-item.tsx b/src/packages/frontend/project/page/flyouts/file-list-item.tsx index 45755b57e1..44c74426ce 100644 --- a/src/packages/frontend/project/page/flyouts/file-list-item.tsx +++ b/src/packages/frontend/project/page/flyouts/file-list-item.tsx @@ -42,7 +42,7 @@ const FILE_ITEM_SELECTED_STYLE: CSS = { backgroundColor: COLORS.BLUE_LLL, // bit darker than .cc-project-flyout-file-item:hover } as const; -const FILE_ITEM_OPENED_STYLE: CSS = { +export const FILE_ITEM_OPENED_STYLE: CSS = { fontWeight: "bold", backgroundColor: COLORS.GRAY_LL, color: COLORS.PROJECT.FIXED_LEFT_ACTIVE, diff --git a/src/packages/frontend/project/page/flyouts/files.tsx b/src/packages/frontend/project/page/flyouts/files.tsx index 2b0e73bec7..5834d2ec0c 100644 --- a/src/packages/frontend/project/page/flyouts/files.tsx +++ b/src/packages/frontend/project/page/flyouts/files.tsx @@ -100,7 +100,9 @@ export function FilesFlyout({ const show_masked = useTypedRedux({ project_id }, "show_masked"); const hidden = useTypedRedux({ project_id }, "show_hidden"); const checked_files = useTypedRedux({ project_id }, "checked_files"); - const openFiles = useTypedRedux({ project_id }, "open_files_order"); + const openFiles = new Set( + useTypedRedux({ project_id }, "open_files_order")?.toJS() ?? [], + ); // mainly controls what a single click does, plus additional UI elements const [mode, setMode] = useState<"open" | "select">("open"); const [prevSelected, setPrevSelected] = useState(null); @@ -192,17 +194,6 @@ export function FilesFlyout({ } }); - for (const file of processedFiles) { - const fullPath = path_to_file(current_path, file.name); - if (openFiles.some((path) => path == fullPath)) { - file.isOpen = true; - } - if (activePath === fullPath) { - file.isActive = true; - activeFile = file; - } - } - if (activeFileSort.get("is_descending")) { processedFiles.reverse(); // inplace op } @@ -229,12 +220,15 @@ export function FilesFlyout({ activeFileSort, hidden, file_search, - openFiles, show_masked, current_path, strippedPublicPaths, ]); + const isOpen = (file) => openFiles.has(path_to_file(current_path, file.name)); + const isActive = (file) => + activePath == path_to_file(current_path, file.name); + const publicFiles = getPublicFiles( directoryFiles, strippedPublicPaths, @@ -393,7 +387,7 @@ export function FilesFlyout({ } // similar, if in open mode and already opened, just switch to it as well - if (mode === "open" && file.isOpen && !e.shiftKey && !e.ctrlKey) { + if (mode === "open" && isOpen(file) && !e.shiftKey && !e.ctrlKey) { setPrevSelected(index); open(e, index); return; @@ -457,13 +451,13 @@ export function FilesFlyout({ } function renderTimeAgo(item: DirectoryListingEntry) { - const { mtime, isOpen = false } = item; + const { mtime } = item; if (typeof mtime === "number") { return ( ); } @@ -515,7 +509,12 @@ export function FilesFlyout({ return ( Date: Thu, 31 Jul 2025 00:34:27 +0000 Subject: [PATCH 144/270] fix a test --- src/packages/test/project/listing/use-files.test.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/packages/test/project/listing/use-files.test.ts b/src/packages/test/project/listing/use-files.test.ts index f36ad76040..ca113bbbb8 100644 --- a/src/packages/test/project/listing/use-files.test.ts +++ b/src/packages/test/project/listing/use-files.test.ts @@ -69,6 +69,9 @@ describe("the useFiles hook", () => { await waitFor(() => { expect(result.current.files?.["hello.txt"]).not.toBeDefined(); }); + await waitFor(() => { + expect(result.current.error).not.toBe(null); + }); expect(result.current.error?.code).toBe("ENOENT"); await act(async () => { From 0e23bab2b56b50e1b1a5f1d3380c0bd473f03a03 Mon Sep 17 00:00:00 2001 From: William Stein Date: Thu, 31 Jul 2025 00:44:14 +0000 Subject: [PATCH 145/270] change tests for how we changed hash of saved version and is read only --- src/packages/sync/editor/generic/sync-doc.ts | 29 ++++++++++--------- .../sync/editor/string/test/sync.0.test.ts | 3 +- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/src/packages/sync/editor/generic/sync-doc.ts b/src/packages/sync/editor/generic/sync-doc.ts index 2fe3749193..b949d1b529 100644 --- a/src/packages/sync/editor/generic/sync-doc.ts +++ b/src/packages/sync/editor/generic/sync-doc.ts @@ -2291,19 +2291,22 @@ export class SyncDoc extends EventEmitter { ); wait_until_read_only_known = async (): Promise => { - await until(async () => { - if (this.isClosed()) { - return true; - } - if (this.stats != null) { - return true; - } - try { - await this.stat(); - return true; - } catch {} - return false; - }); + await until( + async () => { + if (this.isClosed()) { + return true; + } + if (this.stats != null) { + return true; + } + try { + await this.stat(); + return true; + } catch {} + return false; + }, + { min: 3000 }, + ); }; /* Returns true if the current live version of this document has diff --git a/src/packages/sync/editor/string/test/sync.0.test.ts b/src/packages/sync/editor/string/test/sync.0.test.ts index 82f080dac9..0dc3607e5f 100644 --- a/src/packages/sync/editor/string/test/sync.0.test.ts +++ b/src/packages/sync/editor/string/test/sync.0.test.ts @@ -142,12 +142,11 @@ describe("create a blank minimal string SyncDoc and call public methods on it", }); it("read only checks", async () => { - await syncstring.wait_until_read_only_known(); // no-op expect(syncstring.is_read_only()).toBe(false); }); it("hashes of versions", () => { - expect(syncstring.hash_of_saved_version()).toBe(0); + expect(syncstring.hash_of_saved_version()).toBe(undefined); expect(syncstring.hash_of_live_version()).toBe(0); expect(syncstring.has_uncommitted_changes()).toBe(false); }); From 102cae04da58850a504acc47822e3f6655e7381f Mon Sep 17 00:00:00 2001 From: William Stein Date: Thu, 31 Jul 2025 04:25:45 +0000 Subject: [PATCH 146/270] remove the entire open-files implementation, front to back - we have chosen a different path --- src/packages/backend/conat/sync.ts | 4 - .../conat/test/sync/open-files.test.ts | 132 ----- src/packages/conat/sync/open-files.ts | 302 ---------- src/packages/frontend/client/client.ts | 20 - src/packages/frontend/conat/client.ts | 54 -- .../terminal-editor/conat-terminal.ts | 21 - src/packages/frontend/project_actions.ts | 10 - src/packages/project/client.ts | 11 - src/packages/project/conat/api/index.ts | 7 +- src/packages/project/conat/index.ts | 4 +- src/packages/project/conat/open-files.ts | 516 ------------------ src/packages/project/conat/sync.ts | 11 +- src/packages/sync/editor/generic/types.ts | 6 - 13 files changed, 4 insertions(+), 1094 deletions(-) delete mode 100644 src/packages/backend/conat/test/sync/open-files.test.ts delete mode 100644 src/packages/conat/sync/open-files.ts delete mode 100644 src/packages/project/conat/open-files.ts diff --git a/src/packages/backend/conat/sync.ts b/src/packages/backend/conat/sync.ts index 3bccd54978..50963ac696 100644 --- a/src/packages/backend/conat/sync.ts +++ b/src/packages/backend/conat/sync.ts @@ -7,7 +7,6 @@ import { dkv as createDKV, type DKV, type DKVOptions } from "@cocalc/conat/sync/ import { dko as createDKO, type DKO } from "@cocalc/conat/sync/dko"; import { akv as createAKV, type AKV } from "@cocalc/conat/sync/akv"; import { astream as createAStream, type AStream } from "@cocalc/conat/sync/astream"; -import { createOpenFiles, type OpenFiles } from "@cocalc/conat/sync/open-files"; export { inventory } from "@cocalc/conat/sync/inventory"; import "./index"; @@ -35,6 +34,3 @@ export async function dko(opts: DKVOptions): Promise> { return await createDKO(opts); } -export async function openFiles(project_id: string, opts?): Promise { - return await createOpenFiles({ project_id, ...opts }); -} diff --git a/src/packages/backend/conat/test/sync/open-files.test.ts b/src/packages/backend/conat/test/sync/open-files.test.ts deleted file mode 100644 index 371ba6a476..0000000000 --- a/src/packages/backend/conat/test/sync/open-files.test.ts +++ /dev/null @@ -1,132 +0,0 @@ -/* -Unit test basic functionality of the openFiles distributed key:value -store. Projects and compute servers use this to know what files -to open so they can fulfill their backend responsibilities: - - computation - - save to disk - - load from disk when file changes - -DEVELOPMENT: - -pnpm test ./open-files.test.ts - -*/ - -import { openFiles as createOpenFiles } from "@cocalc/backend/conat/sync"; -import { once } from "@cocalc/util/async-utils"; -import { delay } from "awaiting"; -import { before, after, wait } from "@cocalc/backend/conat/test/setup"; - -beforeAll(before); - -const project_id = "00000000-0000-4000-8000-000000000000"; -async function create() { - return await createOpenFiles(project_id, { noAutosave: true, noCache: true }); -} - -describe("create open file tracker and do some basic operations", () => { - let o1, o2; - let file1 = `${Math.random()}.txt`; - let file2 = `${Math.random()}.txt`; - - it("creates two open files trackers (tracking same project) and clear them", async () => { - o1 = await create(); - o2 = await create(); - // ensure caching disabled so our sync tests are real - expect(o1.getKv() === o2.getKv()).toBe(false); - o1.clear(); - await o1.save(); - expect(o1.hasUnsavedChanges()).toBe(false); - o2.clear(); - while (o2.hasUnsavedChanges() || o1.hasUnsavedChanges()) { - try { - // expected due to merge conflict and autosave being disabled. - await o2.save(); - } catch { - await delay(10); - } - } - }); - - it("confirm they are cleared", async () => { - expect(o1.getAll()).toEqual([]); - expect(o2.getAll()).toEqual([]); - }); - - it("touch file in one and observe change and timestamp getting assigned by server", async () => { - o1.touch(file1); - expect(o1.get(file1).time).toBeCloseTo(Date.now(), -3); - }); - - it("touches file in one and observes change by OTHER", async () => { - o1.touch(file2); - expect(o1.get(file2)?.path).toBe(file2); - expect(o2.get(file2)).toBe(undefined); - await o1.save(); - if (o2.get(file2) == null) { - await once(o2, "change", 250); - expect(o2.get(file2).path).toBe(file2); - expect(o2.get(file2).time == null).toBe(false); - } - }); - - it("get all in o2 sees both file1 and file2", async () => { - const v = o2.getAll(); - expect(v[0].path).toBe(file1); - expect(v[1].path).toBe(file2); - expect(v.length).toBe(2); - }); - - it("delete file1 and verify fact that it is deleted is sync'd", async () => { - o1.delete(file1); - expect(o1.get(file1)).toBe(undefined); - expect(o1.getAll().length).toBe(1); - await o1.save(); - - // verify file is gone in o2, at least after waiting (if necessary) - await wait({ - until: () => { - return o2.getAll().length == 1; - }, - }); - expect(o2.get(file1)).toBe(undefined); - // should be 1 due to file2 still being there: - expect(o2.getAll().length).toBe(1); - - // Also confirm file1 is gone in a newly opened one: - const o3 = await create(); - expect(o3.get(file1)).toBe(undefined); - // should be 1 due to file2 still being there, but not file1. - expect(o3.getAll().length).toBe(1); - o3.close(); - }); - - it("sets an error", async () => { - o2.setError(file2, Error("test error")); - expect(o2.get(file2).error.error).toBe("Error: test error"); - expect(typeof o2.get(file2).error.time == "number").toBe(true); - expect(Math.abs(Date.now() - o2.get(file2).error.time)).toBeLessThan(10000); - try { - // get a conflict due to above so resolve it... - await o2.save(); - } catch { - await o2.save(); - } - if (!o1.get(file2).error) { - await once(o1, "change", 250); - } - expect(o1.get(file2).error.error).toBe("Error: test error"); - }); - - it("clears the error", async () => { - o1.setError(file2); - expect(o1.get(file2).error).toBe(undefined); - await o1.save(); - if (o2.get(file2).error) { - await once(o2, "change", 250); - } - expect(o2.get(file2).error).toBe(undefined); - }); -}); - -afterAll(after); diff --git a/src/packages/conat/sync/open-files.ts b/src/packages/conat/sync/open-files.ts deleted file mode 100644 index b82afd8578..0000000000 --- a/src/packages/conat/sync/open-files.ts +++ /dev/null @@ -1,302 +0,0 @@ -/* -Keep track of open files. - -We use the "dko" distributed key:value store because of the potential of merge -conflicts, e.g,. one client changes the compute server id and another changes -whether a file is deleted. By using dko, only the field that changed is sync'd -out, so we get last-write-wins on the level of fields. - -WARNINGS: -An old version use dkv with merge conflict resolution, but with multiple clients -and the project, feedback loops or something happened and it would start getting -slow -- basically, merge conflicts could take a few seconds to resolve, which would -make opening a file start to be slow. Instead we use DKO data type, where fields -are treated separately atomically by the storage system. A *subtle issue* is -that when you set an object, this is NOT treated atomically. E.g., if you -set 2 fields in a set operation, then 2 distinct changes are emitted as the -two fields get set. - -DEVELOPMENT: - -Change to packages/backend, since packages/conat doesn't have a way to connect: - -~/cocalc/src/packages/backend$ node - -> z = await require('@cocalc/backend/conat/sync').openFiles({project_id:cc.current().project_id}) -> z.touch({path:'a.txt'}) -> z.get({path:'a.txt'}) -{ open: true, count: 1, time:2025-02-09T16:37:20.713Z } -> z.touch({path:'a.txt'}) -> z.get({path:'a.txt'}) -{ open: true, count: 2 } -> z.time({path:'a.txt'}) -2025-02-09T16:36:58.510Z -> z.touch({path:'foo/b.md',id:0}) -> z.getAll() -{ - 'a.txt': { open: true, count: 3 }, - 'foo/b.md': { open: true, count: 1 } - -Frontend Dev in browser: - -z = await cc.client.conat_client.openFiles({project_id:cc.current().project_id)) -z.getAll() -} -*/ - -import { type State } from "@cocalc/conat/types"; -import { dko, type DKO } from "@cocalc/conat/sync/dko"; -import { EventEmitter } from "events"; -import getTime, { getSkew } from "@cocalc/conat/time"; - -// info about interest in open files (and also what was explicitly deleted) older -// than this is automatically purged. -const MAX_AGE_MS = 1000 * 60 * 60 * 24; - -interface Deleted { - // what deleted state is - deleted: boolean; - // when deleted state set - time: number; -} - -interface Backend { - // who has it opened -- the compute_server_id (0 for project) - id: number; - // when they last reported having it opened - time: number; -} - -export interface KVEntry { - // a web browser has the file open at this point in time (in ms) - time?: number; - // if the file was removed from disk (and not immmediately written back), - // then deleted gets set to the time when this happened (in ms since epoch) - // and the file is closed on the backend. It won't be re-opened until - // either (1) the file is created on disk again, or (2) deleted is cleared. - // Note: the actual time here isn't really important -- what matter is the number - // is nonzero. It's just used for a display to the user. - // We store the deleted state *and* when this was set, so that in case - // of merge conflict we can do something sensible. - deleted?: Deleted; - - // if file is actively opened on a compute server/project, then it sets - // this entry. Right when it closes the file, it clears this. - // If it gets killed/broken and doesn't have a chance to clear it, then - // backend.time can be used to decide this isn't valid. - backend?: Backend; - - // optional information - doctype?; -} - -export interface Entry extends KVEntry { - // path to file relative to HOME - path: string; -} - -interface Options { - project_id: string; - noAutosave?: boolean; - noCache?: boolean; -} - -export async function createOpenFiles(opts: Options) { - const openFiles = new OpenFiles(opts); - await openFiles.init(); - return openFiles; -} - -export class OpenFiles extends EventEmitter { - private project_id: string; - private noCache?: boolean; - private noAutosave?: boolean; - private kv?: DKO; - public state: "disconnected" | "connected" | "closed" = "disconnected"; - - constructor({ project_id, noAutosave, noCache }: Options) { - super(); - if (!project_id) { - throw Error("project_id must be specified"); - } - this.project_id = project_id; - this.noAutosave = noAutosave; - this.noCache = noCache; - } - - private setState = (state: State) => { - this.state = state; - this.emit(state); - }; - - private initialized = false; - init = async () => { - if (this.initialized) { - throw Error("init can only be called once"); - } - this.initialized = true; - const d = await dko({ - name: "open-files", - project_id: this.project_id, - config: { - max_age: MAX_AGE_MS, - }, - noAutosave: this.noAutosave, - noCache: this.noCache, - noInventory: true, - }); - this.kv = d; - d.on("change", this.handleChange); - // ensure clock is synchronized - await getSkew(); - this.setState("connected"); - }; - - private handleChange = ({ key: path }) => { - const entry = this.get(path); - if (entry != null) { - // not deleted and timestamp is set: - this.emit("change", entry as Entry); - } - }; - - close = () => { - if (this.kv == null) { - return; - } - this.setState("closed"); - this.removeAllListeners(); - this.kv.removeListener("change", this.handleChange); - this.kv.close(); - delete this.kv; - // @ts-ignore - delete this.project_id; - }; - - private getKv = () => { - const { kv } = this; - if (kv == null) { - throw Error("closed"); - } - return kv; - }; - - private set = (path, entry: KVEntry) => { - this.getKv().set(path, entry); - }; - - // When a client has a file open, they should periodically - // touch it to indicate that it is open. - // updates timestamp and ensures open=true. - touch = (path: string, doctype?) => { - if (!path) { - throw Error("path must be specified"); - } - const kv = this.getKv(); - const cur = kv.get(path); - const time = getTime(); - if (doctype) { - this.set(path, { - ...cur, - time, - doctype, - }); - } else { - this.set(path, { - ...cur, - time, - }); - } - }; - - setError = (path: string, err?: any) => { - const kv = this.getKv(); - if (!err) { - const current = { ...kv.get(path) }; - delete current.error; - this.set(path, current); - } else { - const current = { ...kv.get(path) }; - current.error = { time: Date.now(), error: `${err}` }; - this.set(path, current); - } - }; - - setDeleted = (path: string) => { - const kv = this.getKv(); - this.set(path, { - ...kv.get(path), - deleted: { deleted: true, time: getTime() }, - }); - }; - - isDeleted = (path: string) => { - return !!this.getKv().get(path)?.deleted?.deleted; - }; - - setNotDeleted = (path: string) => { - const kv = this.getKv(); - this.set(path, { - ...kv.get(path), - deleted: { deleted: false, time: getTime() }, - }); - }; - - // set that id is the backend with the file open. - // This should be called by that backend periodically - // when it has the file opened. - setBackend = (path: string, id: number) => { - const kv = this.getKv(); - this.set(path, { - ...kv.get(path), - backend: { id, time: getTime() }, - }); - }; - - // get current backend that has file opened. - getBackend = (path: string): Backend | undefined => { - return this.getKv().get(path)?.backend; - }; - - // ONLY if backend for path is currently set to id, then clear - // the backend field. - setNotBackend = (path: string, id: number) => { - const kv = this.getKv(); - const cur = { ...kv.get(path) }; - if (cur?.backend?.id == id) { - delete cur.backend; - this.set(path, cur); - } - }; - - getAll = (): Entry[] => { - const x = this.getKv().getAll(); - return Object.keys(x).map((path) => { - return { ...x[path], path }; - }); - }; - - get = (path: string): Entry | undefined => { - const x = this.getKv().get(path); - if (x == null) { - return x; - } - return { ...x, path }; - }; - - delete = (path) => { - this.getKv().delete(path); - }; - - clear = () => { - this.getKv().clear(); - }; - - save = async () => { - await this.getKv().save(); - }; - - hasUnsavedChanges = () => { - return this.getKv().hasUnsavedChanges(); - }; -} diff --git a/src/packages/frontend/client/client.ts b/src/packages/frontend/client/client.ts index 08dec77e57..786f57ef0e 100644 --- a/src/packages/frontend/client/client.ts +++ b/src/packages/frontend/client/client.ts @@ -317,26 +317,6 @@ class Client extends EventEmitter implements WebappClient { public set_deleted(): void { throw Error("not implemented for frontend"); } - - touchOpenFile = async ({ - project_id, - path, - setNotDeleted, - doctype, - }: { - project_id: string; - path: string; - id?: number; - doctype?; - // if file is deleted, this explicitly undeletes it. - setNotDeleted?: boolean; - }) => { - const x = await this.conat_client.openFiles(project_id); - if (setNotDeleted) { - x.setNotDeleted(path); - } - x.touch(path, doctype); - }; } export const webapp_client = new Client(); diff --git a/src/packages/frontend/conat/client.ts b/src/packages/frontend/conat/client.ts index 516c138143..eb9c7ca613 100644 --- a/src/packages/frontend/conat/client.ts +++ b/src/packages/frontend/conat/client.ts @@ -11,7 +11,6 @@ import { parseQueryWithOptions } from "@cocalc/sync/table/util"; import { type HubApi, initHubApi } from "@cocalc/conat/hub/api"; import { type ProjectApi, initProjectApi } from "@cocalc/conat/project/api"; import { isValidUUID } from "@cocalc/util/misc"; -import { createOpenFiles, OpenFiles } from "@cocalc/conat/sync/open-files"; import { PubSub } from "@cocalc/conat/sync/pubsub"; import type { ChatOptions } from "@cocalc/util/types/llm"; import { dkv } from "@cocalc/conat/sync/dkv"; @@ -62,7 +61,6 @@ export class ConatClient extends EventEmitter { client: WebappClient; public hub: HubApi; public sessionId = randomId(); - private openFilesCache: { [project_id: string]: OpenFiles } = {}; private clientWithState: ClientWithState; private _conatClient: null | ReturnType; public numConnectionAttempts = 0; @@ -428,42 +426,6 @@ export class ConatClient extends EventEmitter { }); }; - openFiles = reuseInFlight(async (project_id: string) => { - if (this.openFilesCache[project_id] == null) { - const openFiles = await createOpenFiles({ - project_id, - }); - this.openFilesCache[project_id] = openFiles; - openFiles.on("closed", () => { - delete this.openFilesCache[project_id]; - }); - openFiles.on("change", (entry) => { - if (entry.deleted?.deleted) { - setDeleted({ - project_id, - path: entry.path, - deleted: entry.deleted.time, - }); - } else { - setNotDeleted({ project_id, path: entry.path }); - } - }); - const recentlyDeletedPaths: any = {}; - for (const { path, deleted } of openFiles.getAll()) { - if (deleted?.deleted) { - recentlyDeletedPaths[path] = deleted.time; - } - } - const store = redux.getProjectStore(project_id); - store.setState({ recentlyDeletedPaths }); - } - return this.openFilesCache[project_id]!; - }); - - closeOpenFiles = (project_id) => { - this.openFilesCache[project_id]?.close(); - }; - pubsub = async ({ project_id, path, @@ -520,22 +482,6 @@ export class ConatClient extends EventEmitter { refCacheInfo = () => refCacheInfo(); } -function setDeleted({ project_id, path, deleted }) { - if (!redux.hasProjectStore(project_id)) { - return; - } - const actions = redux.getProjectActions(project_id); - actions.setRecentlyDeleted(path, deleted); -} - -function setNotDeleted({ project_id, path }) { - if (!redux.hasProjectStore(project_id)) { - return; - } - const actions = redux.getProjectActions(project_id); - actions?.setRecentlyDeleted(path, 0); -} - async function waitForOnline(): Promise { if (navigator.onLine) return; await new Promise((resolve) => { diff --git a/src/packages/frontend/frame-editors/terminal-editor/conat-terminal.ts b/src/packages/frontend/frame-editors/terminal-editor/conat-terminal.ts index 58e0e1d51e..3734b56808 100644 --- a/src/packages/frontend/frame-editors/terminal-editor/conat-terminal.ts +++ b/src/packages/frontend/frame-editors/terminal-editor/conat-terminal.ts @@ -10,7 +10,6 @@ import { SIZE_TIMEOUT_MS, createBrowserClient, } from "@cocalc/conat/service/terminal"; -import { CONAT_OPEN_FILE_TOUCH_INTERVAL } from "@cocalc/util/conat"; import { until } from "@cocalc/util/async-utils"; type State = "disconnected" | "init" | "running" | "closed"; @@ -58,7 +57,6 @@ export class ConatTerminal extends EventEmitter { this.path = path; this.termPath = termPath; this.options = options; - this.touchLoop({ project_id, path: termPath }); this.sizeLoop(measureSize); this.api = createTerminalClient({ project_id, termPath }); this.createBrowserService(); @@ -143,25 +141,6 @@ export class ConatTerminal extends EventEmitter { } }; - touchLoop = async ({ project_id, path }) => { - while (this.state != ("closed" as State)) { - try { - // this marks the path as being of interest for editing and starts - // the service; it doesn't actually create a file on disk. - await webapp_client.touchOpenFile({ - project_id, - path, - }); - } catch (err) { - console.warn(err); - } - if (this.state == ("closed" as State)) { - break; - } - await delay(CONAT_OPEN_FILE_TOUCH_INTERVAL); - } - }; - sizeLoop = async (measureSize) => { while (this.state != ("closed" as State)) { measureSize(); diff --git a/src/packages/frontend/project_actions.ts b/src/packages/frontend/project_actions.ts index 177ed7c9c6..4a4b91d80f 100644 --- a/src/packages/frontend/project_actions.ts +++ b/src/packages/frontend/project_actions.ts @@ -382,8 +382,6 @@ export class ProjectActions extends Actions { this.remove_table(table); } - webapp_client.conat_client.closeOpenFiles(this.project_id); - const store = this.get_store(); store?.close_all_tables(); }; @@ -3539,14 +3537,6 @@ export class ProjectActions extends Actions { const store = this.get_store(); if (store == null) return; this.setRecentlyDeleted(path, 0); - (async () => { - try { - const o = await webapp_client.conat_client.openFiles(this.project_id); - o.setNotDeleted(path); - } catch (err) { - console.log("WARNING: issue undeleting file", err); - } - })(); }; private initProjectStatus = async () => { diff --git a/src/packages/project/client.ts b/src/packages/project/client.ts index ea631ddc47..9b1fa744e7 100644 --- a/src/packages/project/client.ts +++ b/src/packages/project/client.ts @@ -27,7 +27,6 @@ import { join } from "node:path"; import { FileSystemClient } from "@cocalc/sync-client/lib/client-fs"; import { execute_code, uuidsha1 } from "@cocalc/backend/misc_node"; import { CoCalcSocket } from "@cocalc/backend/tcp/enable-messaging-protocol"; -import type { SyncDoc } from "@cocalc/sync/editor/generic/sync-doc"; import type { ProjectClient as ProjectClientInterface } from "@cocalc/sync/editor/generic/types"; import { SyncString } from "@cocalc/sync/editor/string/sync"; import * as synctable2 from "@cocalc/sync/table"; @@ -54,7 +53,6 @@ import { type CreateConatServiceFunction, } from "@cocalc/conat/service"; import { connectToConat } from "./conat/connection"; -import { getSyncDoc } from "@cocalc/project/conat/open-files"; import { isDeleted } from "@cocalc/project/conat/listings"; const winston = getLogger("client"); @@ -520,15 +518,6 @@ export class Client extends EventEmitter implements ProjectClientInterface { }); }; - // WARNING: making two of the exact same sync_string or sync_db will definitely - // lead to corruption! - - // Get the synchronized doc with the given path. Returns undefined - // if currently no such sync-doc. - syncdoc = ({ path }: { path: string }): SyncDoc | undefined => { - return getSyncDoc(path); - }; - public path_access(opts: { path: string; mode: string; cb: CB }): void { // mode: sub-sequence of 'rwxf' -- see https://nodejs.org/api/fs.html#fs_class_fs_stats // cb(err); err = if any access fails; err=undefined if all access is OK diff --git a/src/packages/project/conat/api/index.ts b/src/packages/project/conat/api/index.ts index a523d594d0..ef45cb1eb8 100644 --- a/src/packages/project/conat/api/index.ts +++ b/src/packages/project/conat/api/index.ts @@ -53,7 +53,6 @@ Remember, if you don't set API_KEY, then the project MUST be running so that the import { type ProjectApi } from "@cocalc/conat/project/api"; import { connectToConat } from "@cocalc/project/conat/connection"; import { getSubject } from "../names"; -import { terminate as terminateOpenFiles } from "@cocalc/project/conat/open-files"; import { close as closeListings } from "@cocalc/project/conat/listings"; import { project_id } from "@cocalc/project/data"; import { close as closeFilesRead } from "@cocalc/project/conat/files/read"; @@ -102,11 +101,7 @@ async function handleMessage(api, subject, mesg) { // TODO: should be part of handleApiRequest below, but done differently because // one case halts this loop const { service } = request.args[0] ?? {}; - if (service == "open-files") { - terminateOpenFiles(); - await mesg.respond({ status: "terminated", service }); - return; - } else if (service == "listings") { + if (service == "listings") { closeListings(); await mesg.respond({ status: "terminated", service }); return; diff --git a/src/packages/project/conat/index.ts b/src/packages/project/conat/index.ts index a93de46d88..2c2d9121eb 100644 --- a/src/packages/project/conat/index.ts +++ b/src/packages/project/conat/index.ts @@ -9,7 +9,7 @@ Start the NATS servers: import "./connection"; import { getLogger } from "@cocalc/project/logger"; import { init as initAPI } from "./api"; -import { init as initOpenFiles } from "./open-files"; +// import { init as initOpenFiles } from "./open-files"; // TODO: initWebsocketApi is temporary import { init as initWebsocketApi } from "./browser-websocket-api"; import { init as initListings } from "./listings"; @@ -25,7 +25,7 @@ export default async function init() { logger.debug("starting Conat project services"); await initAPI(); await initJupyter(); - await initOpenFiles(); + // await initOpenFiles(); initWebsocketApi(); await initListings(); await initRead(); diff --git a/src/packages/project/conat/open-files.ts b/src/packages/project/conat/open-files.ts deleted file mode 100644 index 1b93b8c3e5..0000000000 --- a/src/packages/project/conat/open-files.ts +++ /dev/null @@ -1,516 +0,0 @@ -/* -Handle opening files in a project to save/load from disk and also enable compute capabilities. - -DEVELOPMENT: - -0. From the browser with the project opened, terminate the open-files api service: - - - await cc.client.conat_client.projectApi(cc.current()).system.terminate({service:'open-files'}) - - - -Set env variables as in a project (see api/index.ts ), then in nodejs: - -DEBUG_CONSOLE=yes DEBUG=cocalc:debug:project:conat:* node - - x = await require("@cocalc/project/conat/open-files").init(); Object.keys(x) - - -[ 'openFiles', 'openDocs', 'terminate', 'computeServers', 'cc' ] - -> x.openFiles.getAll(); - -> Object.keys(x.openDocs) - -> s = x.openDocs['z4.tasks'] -// now you can directly work with the syncdoc for a given file, -// but from the perspective of the project, not the browser! -// -// - -OR: - - echo "require('@cocalc/project/conat/open-files').init(); require('@cocalc/project/bug-counter').init()" | node - -COMPUTE SERVER: - -To simulate a compute server, do exactly as above, but also set the environment -variable COMPUTE_SERVER_ID to the *global* (not project specific) id of the compute -server: - - COMPUTE_SERVER_ID=84 node - -In this case, you aso don't need to use the terminate command if the compute -server isn't actually running. To terminate a compute server open files service though: - - (TODO) - - -EDITOR ACTIONS: - -Stop the open-files server and define x as above in a terminal. You can -then get the actions or store in a nodejs terminal for a particular document -as follows: - -project_id = '00847397-d6a8-4cb0-96a8-6ef64ac3e6cf'; path = '2025-03-21-100921.ipynb'; -redux = require("@cocalc/jupyter/redux/app").redux; a = redux.getEditorActions(project_id, path); s = redux.getEditorStore(project_id, path); 0; - - -IN A LIVE RUNNING PROJECT IN KUCALC: - -Ssh in to the project itself. You can use a terminal because that very terminal will be broken by -doing this! Then: - -/cocalc/github/src/packages/project$ . /cocalc/nvm/nvm.sh -/cocalc/github/src/packages/project$ COCALC_PROJECT_ID=... COCALC_SECRET_TOKEN="/secrets/secret-token/token" CONAT_SERVER=hub-conat node # not sure about CONAT_SERVER -Welcome to Node.js v20.19.0. -Type ".help" for more information. -> x = await require("@cocalc/project/conat/open-files").init(); Object.keys(x) -[ 'openFiles', 'openDocs', 'terminate', 'computeServers' ] -> - - -*/ - -import { - openFiles as createOpenFiles, - type OpenFiles, - type OpenFileEntry, -} from "@cocalc/project/conat/sync"; -import { CONAT_OPEN_FILE_TOUCH_INTERVAL } from "@cocalc/util/conat"; -import { compute_server_id, project_id } from "@cocalc/project/data"; -import type { SyncDoc } from "@cocalc/sync/editor/generic/sync-doc"; -import { getClient } from "@cocalc/project/client"; -import { SyncString } from "@cocalc/sync/editor/string/sync"; -import { SyncDB } from "@cocalc/sync/editor/db/sync"; -import getLogger from "@cocalc/backend/logger"; -import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; -import { delay } from "awaiting"; -import { initJupyterRedux, removeJupyterRedux } from "@cocalc/jupyter/kernel"; -import { filename_extension, original_path } from "@cocalc/util/misc"; -import { type ConatService } from "@cocalc/conat/service/service"; -import { exists } from "@cocalc/backend/misc/async-utils-node"; -import { map as awaitMap } from "awaiting"; -import { unlink } from "fs/promises"; -import { join } from "path"; -import { - computeServerManager, - ComputeServerManager, -} from "@cocalc/conat/compute/manager"; -import { JUPYTER_SYNCDB_EXTENSIONS } from "@cocalc/util/jupyter/names"; -import { connectToConat } from "@cocalc/project/conat/connection"; - -// ensure conat connection stuff is initialized -import "@cocalc/project/conat/env"; -import { chdir } from "node:process"; - -const logger = getLogger("project:conat:open-files"); - -// we check all files we are currently managing this frequently to -// see if they exist on the filesystem: -const FILE_DELETION_CHECK_INTERVAL = 5000; - -// once we determine that a file does not exist for some reason, we -// wait this long and check *again* just to be sure. If it is still missing, -// then we close the file in memory and set the file as deleted in the -// shared openfile state. -const FILE_DELETION_GRACE_PERIOD = 2000; - -// We NEVER check a file for deletion for this long after first opening it. -// This is VERY important, since some documents, e.g., jupyter notebooks, -// can take a while to get created on disk the first time. -const FILE_DELETION_INITIAL_DELAY = 15000; - -let openFiles: OpenFiles | null = null; -const openDocs: { [path: string]: SyncDoc | ConatService } = {}; -let computeServers: ComputeServerManager | null = null; -const openTimes: { [path: string]: number } = {}; - -export function getSyncDoc(path: string): SyncDoc | undefined { - const doc = openDocs[path]; - if (doc instanceof SyncString || doc instanceof SyncDB) { - return doc; - } - return undefined; -} - -export async function init() { - logger.debug("init"); - - if (process.env.HOME) { - chdir(process.env.HOME); - } - - openFiles = await createOpenFiles(); - - computeServers = computeServerManager({ project_id }); - await computeServers.waitUntilReady(); - computeServers.on("change", async ({ path, id }) => { - if (openFiles == null) { - return; - } - const entry = openFiles?.get(path); - if (entry != null) { - await handleChange({ ...entry, id }); - } else { - await closeDoc(path); - } - }); - - // initialize - for (const entry of openFiles.getAll()) { - handleChange(entry); - } - - // start loop to watch for and close files that aren't touched frequently: - closeIgnoredFilesLoop(); - - // periodically update timestamp on backend for files we have open - touchOpenFilesLoop(); - // watch if any file that is currently opened on this host gets deleted, - // and if so, mark it as such, and set it to closed. - watchForFileDeletionLoop(); - - // handle changes - openFiles.on("change", (entry) => { - // we ONLY actually try to open the file here if there - // is a doctype set. When it is first being created, - // the doctype won't be the first field set, and we don't - // want to launch this until it is set. - if (entry.doctype) { - handleChange(entry); - } - }); - - // useful for development - return { - openFiles, - openDocs, - terminate, - computeServers, - cc: connectToConat(), - }; -} - -export function terminate() { - logger.debug("terminating open-files service"); - for (const path in openDocs) { - closeDoc(path); - } - openFiles?.close(); - openFiles = null; - - computeServers?.close(); - computeServers = null; -} - -function getCutoff(): number { - return Date.now() - 2.5 * CONAT_OPEN_FILE_TOUCH_INTERVAL; -} - -function computeServerId(path: string): number { - return computeServers?.get(path) ?? 0; -} - -function hasBackendState(path) { - return ( - path.endsWith("." + JUPYTER_SYNCDB_EXTENSIONS) || path.endsWith(".sagews") - ); -} - -async function handleChange({ - path, - time, - deleted, - backend, - doctype, - id, -}: OpenFileEntry & { id?: number }) { - // DEPRECATED! - return; - if (!hasBackendState(path)) { - return; - } - try { - if (id == null) { - id = computeServerId(path); - } - logger.debug("handleChange", { path, time, deleted, backend, doctype, id }); - const syncDoc = openDocs[path]; - const isOpenHere = syncDoc != null; - - if (id != compute_server_id) { - if (backend?.id == compute_server_id) { - // we are definitely not the backend right now. - openFiles?.setNotBackend(path, compute_server_id); - } - // only thing we should do is close it if it is open. - if (isOpenHere) { - await closeDoc(path); - } - return; - } - - if (deleted?.deleted) { - if (await exists(path)) { - // it's back - openFiles?.setNotDeleted(path); - } else { - if (isOpenHere) { - await closeDoc(path); - } - return; - } - } - - // @ts-ignore - if (time != null && time >= getCutoff()) { - if (!isOpenHere) { - logger.debug("handleChange: opening", { path }); - // users actively care about this file being opened HERE, but it isn't - await openDoc(path); - } - return; - } - } catch (err) { - console.trace(err); - logger.debug(`handleChange: WARNING - error opening ${path} -- ${err}`); - } -} - -function supportAutoclose(path: string): boolean { - // this feels way too "hard coded"; alternatively, maybe we make the kernel or whatever - // actually update the interest? or something else... - if ( - path.endsWith("." + JUPYTER_SYNCDB_EXTENSIONS) || - path.endsWith(".sagews") || - path.endsWith(".term") - ) { - return false; - } - return true; -} - -async function closeIgnoredFilesLoop() { - while (openFiles?.state == "connected") { - await delay(CONAT_OPEN_FILE_TOUCH_INTERVAL); - if (openFiles?.state != "connected") { - return; - } - const paths = Object.keys(openDocs); - if (paths.length == 0) { - logger.debug("closeIgnoredFiles: no paths currently open"); - continue; - } - logger.debug( - "closeIgnoredFiles: checking", - paths.length, - "currently open paths...", - ); - const cutoff = getCutoff(); - for (const entry of openFiles.getAll()) { - if ( - entry != null && - entry.time != null && - openDocs[entry.path] != null && - entry.time <= cutoff && - supportAutoclose(entry.path) - ) { - logger.debug("closeIgnoredFiles: closing due to inactivity", entry); - closeDoc(entry.path); - } - } - } -} - -async function touchOpenFilesLoop() { - while (openFiles?.state == "connected" && openDocs != null) { - for (const path in openDocs) { - openFiles.setBackend(path, compute_server_id); - } - await delay(CONAT_OPEN_FILE_TOUCH_INTERVAL); - } -} - -async function checkForFileDeletion(path: string) { - if (openFiles == null) { - return; - } - if (Date.now() - (openTimes[path] ?? 0) <= FILE_DELETION_INITIAL_DELAY) { - return; - } - const id = computeServerId(path); - if (id != compute_server_id) { - // not our concern - return; - } - - if (path.endsWith(".term")) { - // term files are exempt -- we don't save data in them and often - // don't actually make the hidden ones for each frame in the - // filesystem at all. - return; - } - const entry = openFiles.get(path); - if (entry == null) { - return; - } - if (entry.deleted?.deleted) { - // already set as deleted -- shouldn't still be opened - await closeDoc(entry.path); - } else { - if (!process.env.HOME) { - // too dangerous - return; - } - const fullPath = join(process.env.HOME, entry.path); - // if file doesn't exist and still doesn't exist in a while, - // mark deleted, which also causes a close. - if (await exists(fullPath)) { - return; - } - // still doesn't exist? - // We must give things a reasonable amount of time, e.g., otherwise - // creating a file (e.g., jupyter notebook) might take too long and - // we randomly think it is deleted before we even make it! - await delay(FILE_DELETION_GRACE_PERIOD); - if (await exists(fullPath)) { - return; - } - // still doesn't exist - if (openFiles != null) { - logger.debug("checkForFileDeletion: marking as deleted -- ", entry); - openFiles.setDeleted(entry.path); - await closeDoc(fullPath); - // closing a file may cause it to try to save to disk the last version, - // so we delete it if that happens. - // TODO: add an option to close everywhere to not do this, and/or make - // it not save on close if the file doesn't exist. - try { - if (await exists(fullPath)) { - await unlink(fullPath); - } - } catch {} - } - } -} - -async function watchForFileDeletionLoop() { - while (openFiles != null && openFiles.state == "connected") { - await delay(FILE_DELETION_CHECK_INTERVAL); - if (openFiles?.state != "connected") { - return; - } - const paths = Object.keys(openDocs); - if (paths.length == 0) { - // logger.debug("watchForFileDeletionLoop: no paths currently open"); - continue; - } - // logger.debug( - // "watchForFileDeletionLoop: checking", - // paths.length, - // "currently open paths to see if any were deleted", - // ); - await awaitMap(paths, 20, checkForFileDeletion); - } -} - -const closeDoc = reuseInFlight(async (path: string) => { - logger.debug("close", { path }); - try { - const doc = openDocs[path]; - if (doc == null) { - return; - } - delete openDocs[path]; - delete openTimes[path]; - try { - await doc.close(); - } catch (err) { - logger.debug(`WARNING -- issue closing doc -- ${err}`); - openFiles?.setError(path, err); - } - } finally { - if (openDocs[path] == null) { - openFiles?.setNotBackend(path, compute_server_id); - } - } -}); - -const openDoc = reuseInFlight(async (path: string) => { - logger.debug("openDoc", { path }); - try { - const doc = openDocs[path]; - if (doc != null) { - return; - } - openTimes[path] = Date.now(); - - if (path.endsWith(".term")) { - // terminals are handled directly by the project api -- also since - // doctype probably not set for them, they won't end up here. - // (this could change though, e.g., we might use doctype to - // set the terminal command). - return; - } - - const client = getClient(); - let doctype: any = openFiles?.get(path)?.doctype; - logger.debug("openDoc: open files table knows ", openFiles?.get(path), { - path, - }); - if (doctype == null) { - logger.debug("openDoc: doctype must be set but isn't, so bailing", { - path, - }); - } else { - logger.debug("openDoc: got doctype from openFiles table", { - path, - doctype, - }); - } - - let syncdoc; - if (doctype.type == "string") { - syncdoc = new SyncString({ - ...doctype.opts, - project_id, - path, - client, - }); - } else { - syncdoc = new SyncDB({ - ...doctype.opts, - project_id, - path, - client, - }); - } - openDocs[path] = syncdoc; - - syncdoc.on("error", (err) => { - closeDoc(path); - openFiles?.setError(path, err); - logger.debug(`syncdoc error -- ${err}`, path); - }); - - // Extra backend support in some cases, e.g., Jupyter, Sage, etc. - const ext = filename_extension(path); - switch (ext) { - case JUPYTER_SYNCDB_EXTENSIONS: - logger.debug("initializing Jupyter backend for ", path); - await initJupyterRedux(syncdoc, client); - const path1 = original_path(syncdoc.get_path()); - syncdoc.on("closed", async () => { - logger.debug("removing Jupyter backend for ", path1); - await removeJupyterRedux(path1, project_id); - }); - break; - } - } finally { - if (openDocs[path] != null) { - openFiles?.setBackend(path, compute_server_id); - } - } -}); diff --git a/src/packages/project/conat/sync.ts b/src/packages/project/conat/sync.ts index ca2c343501..b705b4b3ee 100644 --- a/src/packages/project/conat/sync.ts +++ b/src/packages/project/conat/sync.ts @@ -10,11 +10,6 @@ import { } from "@cocalc/conat/sync/dkv"; import { dko as createDKO, type DKO } from "@cocalc/conat/sync/dko"; import { project_id } from "@cocalc/project/data"; -import { - createOpenFiles, - type OpenFiles, - Entry as OpenFileEntry, -} from "@cocalc/conat/sync/open-files"; import { inventory as createInventory, type Inventory, @@ -26,7 +21,7 @@ import { type AStream, } from "@cocalc/conat/sync/astream"; -export type { DStream, DKV, OpenFiles, OpenFileEntry }; +export type { DStream, DKV }; export async function dstream( opts: DStreamOptions, @@ -50,10 +45,6 @@ export async function dko(opts: DKVOptions): Promise> { return await createDKO({ project_id, ...opts }); } -export async function openFiles(): Promise { - return await createOpenFiles({ project_id }); -} - export async function inventory(): Promise { return await createInventory({ project_id }); } diff --git a/src/packages/sync/editor/generic/types.ts b/src/packages/sync/editor/generic/types.ts index 9222636e92..f840cec3a1 100644 --- a/src/packages/sync/editor/generic/types.ts +++ b/src/packages/sync/editor/generic/types.ts @@ -169,12 +169,6 @@ export interface Client extends ProjectClient { sage_session: (opts: { path: string }) => any; - touchOpenFile?: (opts: { - project_id: string; - path: string; - doctype?; - }) => Promise; - touch_project?: (path: string) => void; } From 192627a53ac6df41f1195bab4806c27169aba9a8 Mon Sep 17 00:00:00 2001 From: William Stein Date: Thu, 31 Jul 2025 05:09:12 +0000 Subject: [PATCH 147/270] jupyter: when evaluating a cell don't have the input get reset as new output appears --- src/packages/conat/project/jupyter/run-code.ts | 5 +++++ src/packages/frontend/jupyter/browser-actions.ts | 13 +++++++++---- src/packages/jupyter/control.ts | 8 +++++++- 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/src/packages/conat/project/jupyter/run-code.ts b/src/packages/conat/project/jupyter/run-code.ts index 6efe51fbdf..04c7eeb53f 100644 --- a/src/packages/conat/project/jupyter/run-code.ts +++ b/src/packages/conat/project/jupyter/run-code.ts @@ -30,6 +30,11 @@ function getSubject({ interface InputCell { id: string; input: string; + output?: { [n: string]: OutputMessage } | null; + state?: "done" | "busy" | "run"; + exec_count?: number | null; + start?: number | null; + end?: number | null; } export interface OutputMessage { diff --git a/src/packages/frontend/jupyter/browser-actions.ts b/src/packages/frontend/jupyter/browser-actions.ts index f325724127..34e2626133 100644 --- a/src/packages/frontend/jupyter/browser-actions.ts +++ b/src/packages/frontend/jupyter/browser-actions.ts @@ -1469,11 +1469,16 @@ export class JupyterActions extends JupyterActions0 { getOutputHandler = (cell) => { const handler = new OutputHandler({ cell }); + + // save first time, so that other clients know this cell is running. let first = true; const f = throttle( () => { - // save first so that other clients know this cell is running. - this._set(cell, first); + // we ONLY set certain fields; e.g., setting the input would be + // extremely annoying since the user can edit the input while the + // cell is running. + const { id, state, output, start, end, exec_count } = cell; + this._set({ id, state, output, start, end, exec_count }, first); first = false; }, 1000 / OUTPUT_FPS, @@ -1575,8 +1580,8 @@ export class JupyterActions extends JupyterActions0 { cell.output[n] = null; } // time last evaluation took - cell.last = cell.start && cell.end ? cell.end - cell.start : null; - this._set(cell, false); + const last = cell.start && cell.end ? cell.end - cell.start : null; + this._set({ id: cell.id, last, output: cell.output }, false); } cells.push(cell); } diff --git a/src/packages/jupyter/control.ts b/src/packages/jupyter/control.ts index 457efe81ec..58cf958bfa 100644 --- a/src/packages/jupyter/control.ts +++ b/src/packages/jupyter/control.ts @@ -122,7 +122,13 @@ class MulticellOutputHandler { this.handler?.done(); this.handler = new OutputHandler({ cell }); const f = throttle( - () => this.actions._set({ ...cell, type: "cell" }, true), + () => { + const { id, state, output, start, end, exec_count } = cell; + this.actions._set( + { type:"cell", id, state, output, start, end, exec_count }, + true, + ); + }, 1000 / BACKEND_OUTPUT_FPS, { leading: true, From 22c35e3873606af9b6b14fc96b538f445be5e3c3 Mon Sep 17 00:00:00 2001 From: William Stein Date: Thu, 31 Jul 2025 20:12:11 +0000 Subject: [PATCH 148/270] surprise subtle improvement to EventIterator --- .../test/project/jupyter/run-code.test.ts | 27 +++++++++++++++++-- .../conat/project/jupyter/run-code.ts | 11 +++++--- .../frontend/jupyter/browser-actions.ts | 9 ++++--- src/packages/util/event-iterator.ts | 16 ++++++++++- 4 files changed, 53 insertions(+), 10 deletions(-) diff --git a/src/packages/backend/conat/test/project/jupyter/run-code.test.ts b/src/packages/backend/conat/test/project/jupyter/run-code.test.ts index fdb960d326..9cc2535124 100644 --- a/src/packages/backend/conat/test/project/jupyter/run-code.test.ts +++ b/src/packages/backend/conat/test/project/jupyter/run-code.test.ts @@ -6,13 +6,18 @@ pnpm test `pwd`/run-code.test.ts */ -import { before, after, connect, wait } from "@cocalc/backend/conat/test/setup"; +import { + before, + after, + connect, + delay, + wait, +} from "@cocalc/backend/conat/test/setup"; import { jupyterClient, jupyterServer, } from "@cocalc/conat/project/jupyter/run-code"; import { uuid } from "@cocalc/util/misc"; -import { delay } from "awaiting"; // it's really 100+, but tests fails if less than this. const MIN_EVALS_PER_SECOND = 10; @@ -54,6 +59,24 @@ describe("create very simple mocked jupyter runner and test evaluating code", () expect(v).toEqual([[{ path, id: "0" }], [{ cells, id: "0" }]]); }); + it("start iterating over the output after waiting", async () => { + // this is the same as the previous test, except we insert a + // delay from when we create the iterator, and when we start + // reading values out of it. This is important to test, because + // it was broken in my first implementation, and is a common mistake + // when implementing async iterators. + client.verbose = true; + const iter = await client.run(cells); + iter.verbose = true; + const v: any[] = []; + await delay(500); + for await (const output of iter) { + v.push(output); + } + client.verbose = false; + expect(v).toEqual([[{ path, id: "0" }], [{ cells, id: "0" }]]); + }); + const count = 100; it(`run ${count} evaluations to ensure that the speed is reasonable (and also everything is kept properly ordered, etc.)`, async () => { const start = Date.now(); diff --git a/src/packages/conat/project/jupyter/run-code.ts b/src/packages/conat/project/jupyter/run-code.ts index 04c7eeb53f..483d52d0c2 100644 --- a/src/packages/conat/project/jupyter/run-code.ts +++ b/src/packages/conat/project/jupyter/run-code.ts @@ -27,14 +27,15 @@ function getSubject({ return `jupyter.project-${project_id}.${compute_server_id}`; } -interface InputCell { +export interface InputCell { id: string; input: string; - output?: { [n: string]: OutputMessage } | null; + output?: { [n: string]: OutputMessage | null } | null; state?: "done" | "busy" | "run"; exec_count?: number | null; start?: number | null; end?: number | null; + cell_type?: "code"; } export interface OutputMessage { @@ -189,7 +190,9 @@ class JupyterClient { run = async (cells: InputCell[], opts: { noHalt?: boolean } = {}) => { if (this.iter) { - // one evaluation at a time. + // one evaluation at a time -- starting a new one ends the previous one. + // Each client browser has a separate instance of JupyterClient, so + // a properly implemented frontend client would never hit this. this.iter.end(); delete this.iter; } @@ -201,7 +204,7 @@ class JupyterClient { } if (args[0] == null) { this.iter?.end(); - return null; + return; } else { return args[0]; } diff --git a/src/packages/frontend/jupyter/browser-actions.ts b/src/packages/frontend/jupyter/browser-actions.ts index 34e2626133..8092f5e326 100644 --- a/src/packages/frontend/jupyter/browser-actions.ts +++ b/src/packages/frontend/jupyter/browser-actions.ts @@ -58,7 +58,10 @@ import getKernelSpec from "@cocalc/frontend/jupyter/kernelspecs"; import { get as getUsageInfo } from "@cocalc/conat/project/usage-info"; import { delay } from "awaiting"; import { until } from "@cocalc/util/async-utils"; -import { jupyterClient } from "@cocalc/conat/project/jupyter/run-code"; +import { + jupyterClient, + type InputCell, +} from "@cocalc/conat/project/jupyter/run-code"; import { OutputHandler } from "@cocalc/jupyter/execute/output-handler"; import { throttle } from "lodash"; @@ -1556,11 +1559,11 @@ export class JupyterActions extends JupyterActions0 { if (client == null) { throw Error("bug"); } - const cells: any[] = []; + const cells: InputCell[] = []; const kernel = this.store.get("kernel"); for (const id of ids) { - const cell = this.store.getIn(["cells", id])?.toJS(); + const cell = this.store.getIn(["cells", id])?.toJS() as InputCell; if ((cell?.cell_type ?? "code") != "code") { // code is the default type continue; diff --git a/src/packages/util/event-iterator.ts b/src/packages/util/event-iterator.ts index 8dcca43803..2d9ec62210 100644 --- a/src/packages/util/event-iterator.ts +++ b/src/packages/util/event-iterator.ts @@ -169,7 +169,6 @@ export class EventIterator if (this.#ended) return; this.resolveNext?.(); this.#ended = true; - this.#queue = []; this.emitter.off(this.event, this.#push); const maxListeners = this.emitter.getMaxListeners(); @@ -189,6 +188,9 @@ export class EventIterator * The next value that's received from the EventEmitter. */ public async next(): Promise> { + // if (this.verbose) { + // console.log("next", this.#queue); + // } if (this.err) { const err = this.err; delete this.err; @@ -279,11 +281,23 @@ export class EventIterator * Pushes a value into the queue. */ protected push(...args): void { + // if (this.verbose) { + // console.log("push", args, this.#queue); + // } if (this.err) { return; } try { const value = this.map(args); + if (this.#ended) { + // the this.map... call could have decided to end + // the iterator, by calling this.end() instead of returning a value. + if (value !== undefined) { + // not undefined so at least give the user the opportunity to get this final value. + this.#queue.push(value); + } + return; + } this.#queue.push(value); while (this.#queue.length > this.#maxQueue && this.#queue.length > 0) { if (this.#overflow == "throw") { From c22f8d44b7d4f62c0ab12a3818a49996f9a42bd8 Mon Sep 17 00:00:00 2001 From: William Stein Date: Thu, 31 Jul 2025 20:54:56 +0000 Subject: [PATCH 149/270] jupyter run -- handle tcp buffer issue --- .../backend/conat/test/socket/basic.test.ts | 2 +- .../conat/project/jupyter/run-code.ts | 19 +++++++++++++++++-- src/packages/conat/socket/client.ts | 4 ++-- src/packages/conat/socket/server-socket.ts | 4 ++-- src/packages/conat/socket/tcp.ts | 2 +- src/packages/conat/socket/util.ts | 5 +++-- .../jupyter/output-messages/message.tsx | 7 ++++--- 7 files changed, 30 insertions(+), 13 deletions(-) diff --git a/src/packages/backend/conat/test/socket/basic.test.ts b/src/packages/backend/conat/test/socket/basic.test.ts index 1ae6e364b2..36e4467b2a 100644 --- a/src/packages/backend/conat/test/socket/basic.test.ts +++ b/src/packages/backend/conat/test/socket/basic.test.ts @@ -168,7 +168,7 @@ describe("create a client first and write more messages than the queue size resu }); it("wait for client to drain; then we can now send another message without an error", async () => { - await client.waitUntilDrain(); + await client.drain(); client.write("foo"); }); diff --git a/src/packages/conat/project/jupyter/run-code.ts b/src/packages/conat/project/jupyter/run-code.ts index 483d52d0c2..866f0c9aab 100644 --- a/src/packages/conat/project/jupyter/run-code.ts +++ b/src/packages/conat/project/jupyter/run-code.ts @@ -120,7 +120,11 @@ export function jupyterServer({ //console.log(err); logger.debug("server: failed response -- ", err); if (socket.state != "closed") { - socket.write(null, { headers: { error: `${err}` } }); + try { + socket.write(null, { headers: { error: `${err}` } }); + } catch { + // an error trying to report an error shouldn't crash everything + } } } }); @@ -146,6 +150,7 @@ async function handleRequest({ let handler: OutputHandler | null = null; for await (const mesg of runner) { if (socket.state == "closed") { + // client socket has closed -- the backend server must take over! if (handler == null) { logger.debug("socket closed -- server must handle output"); if (outputHandler == null) { @@ -163,7 +168,17 @@ async function handleRequest({ handler.process(mesg); } else { output.push(mesg); - socket.write([mesg]); + try { + socket.write([mesg]); + } catch (err) { + if (err.code == "ENOBUFS") { + // wait for the over-filled socket to finish writing out data. + await socket.drain(); + socket.write([mesg]); + } else { + throw err; + } + } } } handler?.done(); diff --git a/src/packages/conat/socket/client.ts b/src/packages/conat/socket/client.ts index db8598a0e2..caa1a45fb9 100644 --- a/src/packages/conat/socket/client.ts +++ b/src/packages/conat/socket/client.ts @@ -91,8 +91,8 @@ export class ConatSocketClient extends ConatSocketBase { }); } - waitUntilDrain = async () => { - await this.tcp?.send.waitUntilDrain(); + drain = async () => { + await this.tcp?.send.drain(); }; private sendCommandToServer = async ( diff --git a/src/packages/conat/socket/server-socket.ts b/src/packages/conat/socket/server-socket.ts index 531e151949..0b678b05db 100644 --- a/src/packages/conat/socket/server-socket.ts +++ b/src/packages/conat/socket/server-socket.ts @@ -238,7 +238,7 @@ export class ServerSocket extends EventEmitter { } }); - waitUntilDrain = async () => { - await this.tcp?.send.waitUntilDrain(); + drain = async () => { + await this.tcp?.send.drain(); }; } diff --git a/src/packages/conat/socket/tcp.ts b/src/packages/conat/socket/tcp.ts index 7f0e2b4994..da55e76a6a 100644 --- a/src/packages/conat/socket/tcp.ts +++ b/src/packages/conat/socket/tcp.ts @@ -275,7 +275,7 @@ export class Sender extends EventEmitter { } }; - waitUntilDrain = reuseInFlight(async () => { + drain = reuseInFlight(async () => { if (this.unsent == 0) { return; } diff --git a/src/packages/conat/socket/util.ts b/src/packages/conat/socket/util.ts index 9e81f08439..2baef9fbea 100644 --- a/src/packages/conat/socket/util.ts +++ b/src/packages/conat/socket/util.ts @@ -20,8 +20,9 @@ export const PING_PONG_INTERVAL = 90000; // NOTE: in nodejs the default for exactly this is "infinite=use up all RAM", so // maybe we should make this even larger (?). // Also note that this is just the *number* of messages, and a message can have -// any size. -export const DEFAULT_MAX_QUEUE_SIZE = 1000; +// any size. But determining message size is very difficult without serializing the +// message, which costs. +export const DEFAULT_MAX_QUEUE_SIZE = 10_000; export let DEFAULT_COMMAND_TIMEOUT = 10_000; export let DEFAULT_KEEP_ALIVE = 25_000; diff --git a/src/packages/frontend/jupyter/output-messages/message.tsx b/src/packages/frontend/jupyter/output-messages/message.tsx index 69596b4623..27d59f0081 100644 --- a/src/packages/frontend/jupyter/output-messages/message.tsx +++ b/src/packages/frontend/jupyter/output-messages/message.tsx @@ -10,7 +10,6 @@ Handling of output messages. import Anser from "anser"; import type { Map } from "immutable"; import React from "react"; - import type { JupyterActions } from "@cocalc/jupyter/redux/actions"; import { LLMTools } from "@cocalc/jupyter/types"; import { Input } from "./input"; @@ -130,8 +129,10 @@ export const CellOutputMessages: React.FC = React.memo( const mesg = obj[n]; if (mesg != null) { if (mesg.get("traceback")) { - hasError = true; - traceback += mesg.get("traceback").join("\n") + "\n"; + const t = mesg.get("traceback").join("\n"); + // if user clicks "Stop" there is a traceback, but it's not an error to fix with AI. + hasError = !t.includes("KeyboardInterrupt"); + traceback += t + "\n"; } if (scrolled && !hasIframes && mesg.getIn(["data", "iframe"])) { hasIframes = true; From 14fc7f8f0a66f9f6ae7380251f906771f12117b4 Mon Sep 17 00:00:00 2001 From: William Stein Date: Thu, 31 Jul 2025 22:00:51 +0000 Subject: [PATCH 150/270] jupyter run -- buffer the jupyter output messages --- .../test/project/jupyter/run-code.test.ts | 30 ++++--- .../conat/project/jupyter/run-code.ts | 82 ++++++++++++------- src/packages/conat/socket/util.ts | 4 +- .../project/conat/terminal/session.ts | 6 +- src/packages/util/throttle.test.ts | 74 +++++++++++++++++ src/packages/util/throttle.ts | 32 ++++++-- 6 files changed, 176 insertions(+), 52 deletions(-) create mode 100644 src/packages/util/throttle.test.ts diff --git a/src/packages/backend/conat/test/project/jupyter/run-code.test.ts b/src/packages/backend/conat/test/project/jupyter/run-code.test.ts index 9cc2535124..95dcb1400e 100644 --- a/src/packages/backend/conat/test/project/jupyter/run-code.test.ts +++ b/src/packages/backend/conat/test/project/jupyter/run-code.test.ts @@ -54,9 +54,12 @@ describe("create very simple mocked jupyter runner and test evaluating code", () const iter = await client.run(cells); const v: any[] = []; for await (const output of iter) { - v.push(output); + v.push(...output); } - expect(v).toEqual([[{ path, id: "0" }], [{ cells, id: "0" }]]); + expect(v).toEqual([ + { path, id: "0" }, + { cells, id: "0" }, + ]); }); it("start iterating over the output after waiting", async () => { @@ -67,14 +70,15 @@ describe("create very simple mocked jupyter runner and test evaluating code", () // when implementing async iterators. client.verbose = true; const iter = await client.run(cells); - iter.verbose = true; const v: any[] = []; await delay(500); for await (const output of iter) { - v.push(output); + v.push(...output); } - client.verbose = false; - expect(v).toEqual([[{ path, id: "0" }], [{ cells, id: "0" }]]); + expect(v).toEqual([ + { path, id: "0" }, + { cells, id: "0" }, + ]); }); const count = 100; @@ -84,9 +88,12 @@ describe("create very simple mocked jupyter runner and test evaluating code", () const v: any[] = []; const cells = [{ id: `${i}`, input: `${i} + ${i}` }]; for await (const output of await client.run(cells)) { - v.push(output); + v.push(...output); } - expect(v).toEqual([[{ path, id: "0" }], [{ cells, id: "0" }]]); + expect(v).toEqual([ + { path, id: "0" }, + { cells, id: "0" }, + ]); } const evalsPerSecond = Math.floor((1000 * count) / (Date.now() - start)); if (process.env.BENCH) { @@ -146,9 +153,12 @@ describe("create simple mocked jupyter runner that does actually eval an express const iter = await client.run(cells); const v: any[] = []; for await (const output of iter) { - v.push(output); + v.push(...output); } - expect(v).toEqual([[{ id: "a", output: 5 }], [{ id: "b", output: 243 }]]); + expect(v).toEqual([ + { id: "a", output: 5 }, + { id: "b", output: 243 }, + ]); }); it("run code that FAILS and see error is visible to client properly", async () => { diff --git a/src/packages/conat/project/jupyter/run-code.ts b/src/packages/conat/project/jupyter/run-code.ts index 866f0c9aab..0417350eef 100644 --- a/src/packages/conat/project/jupyter/run-code.ts +++ b/src/packages/conat/project/jupyter/run-code.ts @@ -14,7 +14,10 @@ import { } from "@cocalc/conat/socket"; import { EventIterator } from "@cocalc/util/event-iterator"; import { getLogger } from "@cocalc/conat/client"; - +import { Throttle } from "@cocalc/util/throttle"; +const MAX_MSGS_PER_SECOND = parseInt( + process.env.COCALC_JUPYTER_MAX_MSGS_PER_SECOND ?? "20", +); const logger = getLogger("conat:project:jupyter:run-code"); function getSubject({ @@ -147,42 +150,59 @@ async function handleRequest({ }) { const runner = await jupyterRun({ path, cells, noHalt }); const output: OutputMessage[] = []; - let handler: OutputHandler | null = null; - for await (const mesg of runner) { - if (socket.state == "closed") { - // client socket has closed -- the backend server must take over! - if (handler == null) { - logger.debug("socket closed -- server must handle output"); - if (outputHandler == null) { - throw Error("no output handler available"); - } - handler = outputHandler({ path, cells }); + + const throttle = new Throttle(MAX_MSGS_PER_SECOND); + let unhandledClientWriteError: any = undefined; + throttle.on("data", async (mesgs) => { + try { + socket.write(mesgs); + } catch (err) { + if (err.code == "ENOBUFS") { + // wait for the over-filled socket to finish writing out data. + await socket.drain(); + socket.write(mesgs); + } else { + unhandledClientWriteError = err; + } + } + }); + + try { + let handler: OutputHandler | null = null; + for await (const mesg of runner) { + if (socket.state == "closed") { + // client socket has closed -- the backend server must take over! if (handler == null) { - throw Error("bug -- outputHandler must return a handler"); - } - for (const prev of output) { - handler.process(prev); + logger.debug("socket closed -- server must handle output"); + if (outputHandler == null) { + throw Error("no output handler available"); + } + handler = outputHandler({ path, cells }); + if (handler == null) { + throw Error("bug -- outputHandler must return a handler"); + } + for (const prev of output) { + handler.process(prev); + } + output.length = 0; } - output.length = 0; - } - handler.process(mesg); - } else { - output.push(mesg); - try { - socket.write([mesg]); - } catch (err) { - if (err.code == "ENOBUFS") { - // wait for the over-filled socket to finish writing out data. - await socket.drain(); - socket.write([mesg]); - } else { - throw err; + handler.process(mesg); + } else { + if (unhandledClientWriteError) { + throw unhandledClientWriteError; } + output.push(mesg); + throttle.write(mesg); } } + handler?.done(); + } finally { + if (socket.state != "closed" && !unhandledClientWriteError) { + throttle.flush(); + socket.write(null); + } + throttle.close(); } - handler?.done(); - socket.write(null); } class JupyterClient { diff --git a/src/packages/conat/socket/util.ts b/src/packages/conat/socket/util.ts index 2baef9fbea..5274ba1206 100644 --- a/src/packages/conat/socket/util.ts +++ b/src/packages/conat/socket/util.ts @@ -13,7 +13,7 @@ export type Role = "client" | "server"; // socketio and use those to manage things. This ping // is entirely a "just in case" backup if some event // were missed (e.g., a kill -9'd process...) -export const PING_PONG_INTERVAL = 90000; +export const PING_PONG_INTERVAL = 90_000; // We queue up unsent writes, but only up to a point (to not have a huge memory issue). // Any write beyond this size result in an exception. @@ -22,7 +22,7 @@ export const PING_PONG_INTERVAL = 90000; // Also note that this is just the *number* of messages, and a message can have // any size. But determining message size is very difficult without serializing the // message, which costs. -export const DEFAULT_MAX_QUEUE_SIZE = 10_000; +export const DEFAULT_MAX_QUEUE_SIZE = 1_000; export let DEFAULT_COMMAND_TIMEOUT = 10_000; export let DEFAULT_KEEP_ALIVE = 25_000; diff --git a/src/packages/project/conat/terminal/session.ts b/src/packages/project/conat/terminal/session.ts index 63a0097857..8afd87b1c9 100644 --- a/src/packages/project/conat/terminal/session.ts +++ b/src/packages/project/conat/terminal/session.ts @@ -12,7 +12,7 @@ import { } from "@cocalc/conat/service/terminal"; import { project_id, compute_server_id } from "@cocalc/project/data"; import { throttle } from "lodash"; -import { ThrottleString as Throttle } from "@cocalc/util/throttle"; +import { ThrottleString } from "@cocalc/util/throttle"; import { join } from "path"; import type { CreateTerminalOptions } from "@cocalc/conat/project/api/editor"; import { delay } from "awaiting"; @@ -55,7 +55,7 @@ const MAX_BYTES_PER_SECOND = parseInt( // having to discard writes. This is basically the "frame rate" // we are supporting for users. const MAX_MSGS_PER_SECOND = parseInt( - process.env.COCALC_TERMINAL_MAX_MSGS_PER_SECOND ?? "24", + process.env.COCALC_TERMINAL_MAX_MSGS_PER_SECOND ?? "20", ); type State = "running" | "off" | "closed"; @@ -236,7 +236,7 @@ export class Session { // use slighlty less than MAX_MSGS_PER_SECOND to avoid reject // due to being *slightly* off. - const throttle = new Throttle(1000 / (MAX_MSGS_PER_SECOND - 3)); + const throttle = new ThrottleString(MAX_MSGS_PER_SECOND - 3); throttle.on("data", (data: string) => { // logger.debug("got data out of pty"); this.handleBackendMessages(data); diff --git a/src/packages/util/throttle.test.ts b/src/packages/util/throttle.test.ts new file mode 100644 index 0000000000..ac357c544f --- /dev/null +++ b/src/packages/util/throttle.test.ts @@ -0,0 +1,74 @@ +import { ThrottleString, Throttle } from "./throttle"; +import { delay } from "awaiting"; + +describe("a throttled string", () => { + let t; + let output = ""; + it("creates a throttled string", () => { + // emits 10 times a second or once very 100ms. + t = new ThrottleString(10); + t.on("data", (data) => { + output += data; + }); + }); + + it("write 3 times and wait 50ms and get nothing, then 70 more ms and get all", async () => { + t.write("a"); + t.write("b"); + t.write("c"); + await delay(50); + expect(output).toBe(""); + // this "d" also gets included -- it makes it in before the cutoff. + t.write("d"); + await delay(70); + expect(output).toBe("abcd"); + }); + + it("do the same again", async () => { + t.write("a"); + t.write("b"); + t.write("c"); + await delay(50); + expect(output).toBe("abcd"); + t.write("d"); + await delay(70); + expect(output).toBe("abcdabcd"); + }); +}); + +describe("a throttled list of objects", () => { + let t; + let output: any[] = []; + + it("creates a throttled any[]", () => { + // emits 10 times a second or once very 100ms. + t = new Throttle(10); + t.on("data", (data: any[]) => { + output = output.concat(data); + }); + }); + + it("write 3 times and wait 50ms and get nothing, then 70 more ms and get all", async () => { + t.write("a"); + t.write("b"); + t.write("c"); + await delay(50); + expect(output).toEqual([]); + // this "d" also gets included -- it makes it in before the cutoff. + t.write("d"); + await delay(70); + expect(output).toEqual(["a", "b", "c", "d"]); + }); + + it("do it again", async () => { + t.write("a"); + t.write("b"); + t.write("c"); + await delay(50); + expect(output).toEqual(["a", "b", "c", "d"]); + // this "d" also gets included -- it makes it in before the cutoff. + t.write("d"); + await delay(70); + expect(output).toEqual(["a", "b", "c", "d", "a", "b", "c", "d"]); + }); +}); diff --git a/src/packages/util/throttle.ts b/src/packages/util/throttle.ts index 2fff3d2596..92e64dfab9 100644 --- a/src/packages/util/throttle.ts +++ b/src/packages/util/throttle.ts @@ -3,15 +3,22 @@ This is a really simple but incredibly useful little class. See packages/project/conat/terminal.ts for how to use it to make it so the terminal sends output at a rate of say "24 frames per second". + +This could also be called "buffering"... */ import { EventEmitter } from "events"; +const DEFAULT_MESSAGES_PER_SECOND = 24; + +// Throttling a string where use "+" to add more to our buffer export class ThrottleString extends EventEmitter { private buf: string = ""; private last = Date.now(); + private interval: number; - constructor(private interval: number) { + constructor(messagesPerSecond: number = DEFAULT_MESSAGES_PER_SECOND) { super(); + this.interval = 1000 / messagesPerSecond; } write = (data: string) => { @@ -34,20 +41,33 @@ export class ThrottleString extends EventEmitter { }; } -export class ThrottleAny extends EventEmitter { - private buf: any[] = []; +// Throttle a list of objects, where push them into an array to add more to our buffer. +export class Throttle extends EventEmitter { + private buf: T[] = []; private last = Date.now(); + private interval: number; - constructor(private interval: number) { + constructor(messagesPerSecond: number = DEFAULT_MESSAGES_PER_SECOND) { super(); + this.interval = 1000 / messagesPerSecond; } - write = (data: any) => { + // if you want data to be sent be sure to flush before closing + close = () => { + this.removeAllListeners(); + this.buf.length = 0; + }; + + write = (data: T) => { this.buf.push(data); + this.update(); + }; + + private update = () => { const now = Date.now(); const timeUntilEmit = this.interval - (now - this.last); if (timeUntilEmit > 0) { - setTimeout(() => this.write([]), timeUntilEmit); + setTimeout(() => this.update(), timeUntilEmit); } else { this.flush(); } From f18e917931bead026c582d88ef35231b11236a1e Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 1 Aug 2025 05:01:15 +0000 Subject: [PATCH 151/270] mainly fix using File --> Duplicate, etc.; did this by being more careful about timing using async instead of callbacks more. improved typings as well. --- .../course/assignments/assignment.tsx | 4 +- .../frontend/editors/file-info-dropdown.tsx | 6 +- .../frame-tree/commands/generic-commands.tsx | 3 +- .../frontend/project/explorer/action-bar.tsx | 3 +- .../frontend/project/explorer/action-box.tsx | 80 ++++---- .../frontend/project/explorer/explorer.tsx | 1 - .../project/explorer/file-listing/utils.ts | 4 +- src/packages/frontend/project/open-file.ts | 3 +- .../frontend/project/page/share-indicator.tsx | 2 +- src/packages/frontend/project_actions.ts | 188 +++++++++--------- src/packages/frontend/project_store.ts | 5 +- src/packages/frontend/projects/store.ts | 12 +- 12 files changed, 156 insertions(+), 155 deletions(-) diff --git a/src/packages/frontend/course/assignments/assignment.tsx b/src/packages/frontend/course/assignments/assignment.tsx index 276aaa1364..9849c2592e 100644 --- a/src/packages/frontend/course/assignments/assignment.tsx +++ b/src/packages/frontend/course/assignments/assignment.tsx @@ -407,7 +407,7 @@ export function Assignment({ ); } - function open_assignment_path(): void { + async function open_assignment_path() { if (assignment.get("listing")?.size == 0) { // there are no files yet, so we *close* the assignment // details panel. This is just **a hack** so that the user @@ -421,7 +421,7 @@ export function Assignment({ assignment.get("assignment_id"), ); } - return redux + await redux .getProjectActions(project_id) .open_directory(assignment.get("path")); } diff --git a/src/packages/frontend/editors/file-info-dropdown.tsx b/src/packages/frontend/editors/file-info-dropdown.tsx index bcbf3530a2..e25de0188d 100644 --- a/src/packages/frontend/editors/file-info-dropdown.tsx +++ b/src/packages/frontend/editors/file-info-dropdown.tsx @@ -11,7 +11,7 @@ import { CSS, React, useActions } from "@cocalc/frontend/app-framework"; import { DropdownMenu, Icon, IconName } from "@cocalc/frontend/components"; import { MenuItems } from "@cocalc/frontend/components/dropdown-menu"; import { useStudentProjectFunctionality } from "@cocalc/frontend/course"; -import { file_actions } from "@cocalc/frontend/project_store"; +import { file_actions, type FileAction } from "@cocalc/frontend/project_store"; import { capitalize, filename_extension } from "@cocalc/util/misc"; interface Props { @@ -57,9 +57,9 @@ const EditorFileInfoDropdown: React.FC = React.memo( } for (const key in file_actions) { if (key === name) { - actions.show_file_action_panel({ + actions.showFileActionPanel({ path: filename, - action: key, + action: key as FileAction, }); break; } diff --git a/src/packages/frontend/frame-editors/frame-tree/commands/generic-commands.tsx b/src/packages/frontend/frame-editors/frame-tree/commands/generic-commands.tsx index 1de09b438a..0bb95fecce 100644 --- a/src/packages/frontend/frame-editors/frame-tree/commands/generic-commands.tsx +++ b/src/packages/frontend/frame-editors/frame-tree/commands/generic-commands.tsx @@ -7,7 +7,6 @@ import { Input } from "antd"; import { debounce } from "lodash"; import { useEffect, useRef } from "react"; import { defineMessage, IntlShape, useIntl } from "react-intl"; - import { set_account_table } from "@cocalc/frontend/account/util"; import { redux } from "@cocalc/frontend/app-framework"; import { Icon } from "@cocalc/frontend/components"; @@ -1467,7 +1466,7 @@ function fileAction(action) { alwaysShow: true, onClick: ({ props }) => { const actions = redux.getProjectActions(props.project_id); - actions.show_file_action_panel({ + actions.showFileActionPanel({ path: props.path, action, }); diff --git a/src/packages/frontend/project/explorer/action-bar.tsx b/src/packages/frontend/project/explorer/action-bar.tsx index 80977d97dd..58a30dbfe2 100644 --- a/src/packages/frontend/project/explorer/action-bar.tsx +++ b/src/packages/frontend/project/explorer/action-bar.tsx @@ -18,6 +18,7 @@ import { labels } from "@cocalc/frontend/i18n"; import { file_actions, type ProjectActions, + type FileAction, } from "@cocalc/frontend/project_store"; import * as misc from "@cocalc/util/misc"; import { COLORS } from "@cocalc/util/theme"; @@ -195,7 +196,7 @@ export function ActionBar({ } } - function render_action_button(name: string): React.JSX.Element { + function render_action_button(name: FileAction): React.JSX.Element { const disabled = isDisabledSnapshots(name) && (current_path != null diff --git a/src/packages/frontend/project/explorer/action-box.tsx b/src/packages/frontend/project/explorer/action-box.tsx index 2a25f09bfd..110bdf2e5b 100644 --- a/src/packages/frontend/project/explorer/action-box.tsx +++ b/src/packages/frontend/project/explorer/action-box.tsx @@ -18,7 +18,7 @@ import { Well, } from "@cocalc/frontend/antd-bootstrap"; import { useRedux, useTypedRedux } from "@cocalc/frontend/app-framework"; -import { Icon, Loading, LoginLink } from "@cocalc/frontend/components"; +import { Icon, LoginLink } from "@cocalc/frontend/components"; import SelectServer from "@cocalc/frontend/compute/select-server"; import ComputeServerTag from "@cocalc/frontend/compute/server-tag"; import { useRunQuota } from "@cocalc/frontend/project/settings/run-quota/hooks"; @@ -54,7 +54,6 @@ interface Props { file_action: FileAction; current_path: string; project_id: string; - file_map: object; actions: ProjectActions; } @@ -63,7 +62,6 @@ export function ActionBox({ file_action, current_path, project_id, - file_map, actions, }: Props) { const intl = useIntl(); @@ -588,44 +586,40 @@ export function ActionBox({ if (action_button == undefined) { return
Undefined action
; } - if (file_map == undefined) { - return ; - } else { - return ( - - - - {" "} - {intl.formatMessage(action_button.name)} -
- - - -
- {!!compute_server_id && ( - - )} - - {render_action_box(action)} - - - ); - } + return ( + + + + {" "} + {intl.formatMessage(action_button.name)} +
+ + + +
+ {!!compute_server_id && ( + + )} + + {render_action_box(action)} + + + ); } diff --git a/src/packages/frontend/project/explorer/explorer.tsx b/src/packages/frontend/project/explorer/explorer.tsx index 47349a95c4..4694e6ac5d 100644 --- a/src/packages/frontend/project/explorer/explorer.tsx +++ b/src/packages/frontend/project/explorer/explorer.tsx @@ -467,7 +467,6 @@ export function Explorer() { = React.memo( -
-
- ); - }); + const v: JSX.Element[] = []; + const attachments = cell.get("attachments"); + if (attachments) { + attachments.forEach((_, name) => { + if (v.length > 0) { + v.push(
); } - } - if (v.length === 0) { - return ( - - There are no attachments. To attach images, use Edit → Insert - Image. - + return v.push( +
+
{name}
+
+ +
+
, ); - } - return v; + }); + } + if (v.length === 0) { + return ( + + There are no attachments. To attach images, use Edit → Insert + Image. + + ); + } + + function close() { + actions.setState({ edit_attachments: undefined }); + actions.focus(true); } return ( @@ -79,7 +78,7 @@ export function EditAttachments({ actions, cell }: EditAttachmentsProps) { } > - {renderAttachments()} + {v} ); } diff --git a/src/packages/frontend/jupyter/main.tsx b/src/packages/frontend/jupyter/main.tsx index aca7b02f86..77a96a80c4 100644 --- a/src/packages/frontend/jupyter/main.tsx +++ b/src/packages/frontend/jupyter/main.tsx @@ -12,11 +12,10 @@ import { CSS, React, redux, - Rendered, useRedux, - useRef, useTypedRedux, } from "@cocalc/frontend/app-framework"; +import { useRef } from "react"; // Support for all the MIME types import { Button, Tooltip } from "antd"; import "./output-messages/mime-types/init-frontend"; @@ -117,7 +116,6 @@ export const JupyterEditor: React.FC = React.memo((props: Props) => { "show_kernel_selector", ]); // string name of the kernel - const kernels: undefined | KernelsType = useRedux([name, "kernels"]); const kernelspec = useRedux([name, "kernel_info"]); const error: undefined | KernelsType = useRedux([name, "error"]); // settings for all the codemirror editors @@ -261,25 +259,11 @@ export const JupyterEditor: React.FC = React.memo((props: Props) => { ); } - function render_loading(): Rendered { - return ( - - ); - } - function render_cells() { if ( cell_list == null || font_size == null || cm_options == null || - kernels == null || cells == null ) { return ( @@ -327,95 +311,20 @@ export const JupyterEditor: React.FC = React.memo((props: Props) => { ); } - function render_about() { - return ( - - ); - } - - function render_nbconvert() { - if (path == null || project_id == null) return; - return ( - - ); - } - - function render_edit_attachments() { - if (edit_attachments == null || cells == null) { - return; - } - const cell = cells.get(edit_attachments); - if (cell == null) { - return; - } - return ; - } - - function render_edit_cell_metadata() { - if (edit_cell_metadata == null) { - return; - } - return ( - - ); - } - - function render_find_and_replace() { - if (cells == null || cur_id == null) { - return; - } - return ( - - ); - } - - function render_confirm_dialog() { - if (confirm_dialog == null || actions == null) return; - return ; - } - function render_select_kernel() { return ; } - function render_keyboard_shortcuts() { - if (actions == null) return; - return ( - - ); - } - function render_main() { if (!check_select_kernel_init) { - return render_loading(); + ; } else if (show_kernel_selector) { return render_select_kernel(); } else { @@ -427,13 +336,58 @@ export const JupyterEditor: React.FC = React.memo((props: Props) => { if (!is_focused) return; return ( <> - {render_about()} - {render_nbconvert()} - {render_edit_attachments()} - {render_edit_cell_metadata()} - {render_find_and_replace()} - {render_keyboard_shortcuts()} - {render_confirm_dialog()} + + {path != null && project_id != null && ( + + )} + {edit_attachments != null && ( + + )} + {edit_cell_metadata != null && ( + + )} + {cells != null && cur_id != null && ( + + )} + {actions != null && ( + + )} + {actions != null && confirm_dialog != null && ( + + )} ); } diff --git a/src/packages/jupyter/redux/actions.ts b/src/packages/jupyter/redux/actions.ts index debebb2a4f..41615bac30 100644 --- a/src/packages/jupyter/redux/actions.ts +++ b/src/packages/jupyter/redux/actions.ts @@ -688,6 +688,7 @@ export abstract class JupyterActions extends Actions { this.set_cell_list(); } + this.ensureThereIsACell(); this.__syncdb_change_post_hook(doInit); }; @@ -2505,6 +2506,25 @@ export abstract class JupyterActions extends Actions { } this.setState({ runProgress: total > 0 ? (100 * ran) / total : 100 }); }; + + ensureThereIsACell = () => { + if (this._state !== "ready") { + return; + } + const cells = this.store.get("cells"); + if (cells == null || cells.size === 0) { + this._set({ + type: "cell", + // by using the same id across clients we solve the problem of multiple + // clients creating a cell at the same time. + id: "alpha", + pos: 0, + input: "", + }); + // We are obviously contributing content to this (empty!) notebook. + return this.set_trust_notebook(true); + } + }; } function extractLabel(content: string): string { diff --git a/src/packages/jupyter/redux/project-actions.ts b/src/packages/jupyter/redux/project-actions.ts index b30b01d779..a334f5f3a9 100644 --- a/src/packages/jupyter/redux/project-actions.ts +++ b/src/packages/jupyter/redux/project-actions.ts @@ -287,7 +287,6 @@ export class JupyterActions extends JupyterActions0 { }); } - this.ensure_there_is_a_cell(); this._throttled_ensure_positions_are_unique(); }; @@ -784,23 +783,6 @@ export class JupyterActions extends JupyterActions0 { } }; - ensure_there_is_a_cell = () => { - if (this._state !== "ready") { - return; - } - const cells = this.store.get("cells"); - if (cells == null || cells.size === 0) { - this._set({ - type: "cell", - id: this.new_id(), - pos: 0, - input: "", - }); - // We are obviously contributing content to this (empty!) notebook. - return this.set_trust_notebook(true); - } - }; - private handle_all_cell_attachments() { // Check if any cell attachments need to be loaded. const cells = this.store.get("cells"); From 84077c318afa20ea3c9ff96adedbe4e6a5cc3c98 Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 1 Aug 2025 17:19:32 +0000 Subject: [PATCH 155/270] delete the old project actions entirely -- starting from scratch with something much more direct --- src/packages/jupyter/control.ts | 4 +- src/packages/jupyter/kernel/kernel.ts | 3 +- src/packages/jupyter/redux/actions.ts | 31 +- src/packages/jupyter/redux/project-actions.ts | 981 +----------------- src/packages/project/conat/jupyter.ts | 19 + src/packages/sync/editor/generic/sync-doc.ts | 4 +- 6 files changed, 52 insertions(+), 990 deletions(-) diff --git a/src/packages/jupyter/control.ts b/src/packages/jupyter/control.ts index 58cf958bfa..1020c94ec7 100644 --- a/src/packages/jupyter/control.ts +++ b/src/packages/jupyter/control.ts @@ -84,7 +84,7 @@ export async function jupyterRun({ path, cells, noHalt }: RunOptions) { logger.debug("jupyterRun: running"); async function* run() { for (const cell of cells) { - actions.ensure_backend_kernel_setup(); + actions.initKernel(); const output = actions.jupyter_kernel.execute_code({ halt_on_error: !noHalt, code: cell.input, @@ -125,7 +125,7 @@ class MulticellOutputHandler { () => { const { id, state, output, start, end, exec_count } = cell; this.actions._set( - { type:"cell", id, state, output, start, end, exec_count }, + { type: "cell", id, state, output, start, end, exec_count }, true, ); }, diff --git a/src/packages/jupyter/kernel/kernel.ts b/src/packages/jupyter/kernel/kernel.ts index a3e67c6e9a..f8ef7ac1fa 100644 --- a/src/packages/jupyter/kernel/kernel.ts +++ b/src/packages/jupyter/kernel/kernel.ts @@ -36,7 +36,6 @@ import { jupyterSockets, type JupyterSockets } from "@cocalc/jupyter/zmq"; import { EventEmitter } from "node:events"; import { unlink } from "@cocalc/backend/misc/async-utils-node"; import { remove_redundant_reps } from "@cocalc/jupyter/ipynb/import-from-ipynb"; -import { JupyterActions } from "@cocalc/jupyter/redux/project-actions"; import { type BlobStoreInterface, CodeExecutionEmitterInterface, @@ -44,6 +43,7 @@ import { JupyterKernelInterface, KernelInfo, } from "@cocalc/jupyter/types/project-interface"; +import { JupyterActions } from "@cocalc/jupyter/redux/project-actions"; import { JupyterStore } from "@cocalc/jupyter/redux/store"; import { JUPYTER_MIMETYPES } from "@cocalc/jupyter/util/misc"; import type { SyncDB } from "@cocalc/sync/editor/db/sync"; @@ -151,7 +151,6 @@ export function initJupyterRedux(syncdb: SyncDB, client: Client) { } const store = redux.createStore(name, JupyterStore); const actions = redux.createActions(name, JupyterActions); - actions._init(project_id, path, syncdb, store, client); syncdb.once("error", (err) => diff --git a/src/packages/jupyter/redux/actions.ts b/src/packages/jupyter/redux/actions.ts index 41615bac30..7c3c9f85ba 100644 --- a/src/packages/jupyter/redux/actions.ts +++ b/src/packages/jupyter/redux/actions.ts @@ -64,7 +64,7 @@ const CellDeleteProtectedException = new Error("CellDeleteProtectedException"); type State = "init" | "load" | "ready" | "closed"; -export abstract class JupyterActions extends Actions { +export class JupyterActions extends Actions { public is_project: boolean; public is_compute_server?: boolean; readonly path: string; @@ -95,6 +95,7 @@ export abstract class JupyterActions extends Actions { store: any, client: Client, ): void { + console.log("jupyter actions: _init", { path }); this._client = client; const dbg = this.dbg("_init"); dbg("Initializing Jupyter Actions"); @@ -113,20 +114,17 @@ export abstract class JupyterActions extends Actions { this.path = path; store.syncdb = syncdb; this.syncdb = syncdb; - // the project client is designated to manage execution/conflict, etc. this.is_project = client.is_project(); - if (this.is_project) { - this.syncdb.on("first-load", () => { - dbg("handling first load of syncdb in project"); - // Clear settings the first time the syncdb is ever - // loaded, since it has settings like "ipynb last save" - // and trust, which shouldn't be initialized to - // what they were before. Not doing this caused - // https://github.com/sagemathinc/cocalc/issues/7074 - this.syncdb.delete({ type: "settings" }); - this.syncdb.commit(); - }); - } + this.syncdb.on("first-load", () => { + dbg("handling first load of syncdb"); + // Clear settings the first time the syncdb is ever + // loaded, since it has settings like "ipynb last save" + // and trust, which shouldn't be initialized to + // what they were before. Not doing this caused + // https://github.com/sagemathinc/cocalc/issues/7074 + this.syncdb.delete({ type: "settings" }); + this.syncdb.commit(); + }); this.is_compute_server = client.is_compute_server(); let directory: any; @@ -983,7 +981,10 @@ export abstract class JupyterActions extends Actions { this.deprecated("run_selected_cells"); }; - abstract runCells(ids: string[], opts?: { noHalt?: boolean }): Promise; + runCells(_ids: string[], _opts?: { noHalt?: boolean }): Promise { + // defined in derived class (e.g., frontend browser). + throw Error("DEPRECATED"); + } run_all_cells = (no_halt: boolean = false): void => { this.runCells(this.store.get_cell_list().toJS(), { noHalt: no_halt }); diff --git a/src/packages/jupyter/redux/project-actions.ts b/src/packages/jupyter/redux/project-actions.ts index a334f5f3a9..8befaacec2 100644 --- a/src/packages/jupyter/redux/project-actions.ts +++ b/src/packages/jupyter/redux/project-actions.ts @@ -14,993 +14,34 @@ fully unit test it via mocking of components. NOTE: this is also now the actions used by remote compute servers as well. */ -import { get_kernel_data } from "@cocalc/jupyter/kernel/kernel-data"; -import * as immutable from "immutable"; -import json_stable from "json-stable-stringify"; -import { debounce } from "lodash"; import { JupyterActions as JupyterActions0 } from "@cocalc/jupyter/redux/actions"; -import { callback2, once } from "@cocalc/util/async-utils"; -import * as misc from "@cocalc/util/misc"; -import { RunAllLoop } from "./run-all-loop"; -import nbconvertChange from "./handle-nbconvert-change"; -import type { ClientFs } from "@cocalc/sync/client/types"; import { kernel as createJupyterKernel } from "@cocalc/jupyter/kernel"; -import { removeJupyterRedux } from "@cocalc/jupyter/kernel"; -import { initConatService } from "@cocalc/jupyter/kernel/conat-service"; -import { type DKV, dkv } from "@cocalc/conat/sync/dkv"; -import { computeServerManager } from "@cocalc/conat/compute/manager"; -import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; - -// refuse to open an ipynb that is bigger than this: -const MAX_SIZE_IPYNB_MB = 150; - -type BackendState = "init" | "ready" | "spawning" | "starting" | "running"; export class JupyterActions extends JupyterActions0 { - private _backend_state: BackendState = "init"; - private lastSavedBackendState?: BackendState; - private _initialize_manager_already_done: any; - private _kernel_state: any; - private _running_cells: { [id: string]: string }; - private _throttled_ensure_positions_are_unique: any; - private run_all_loop?: RunAllLoop; - private clear_kernel_error?: any; - private last_ipynb_save: number = 0; - protected _client: ClientFs; // this has filesystem access, etc. - public blobs: DKV; - private computeServers?; - - private initBlobStore = async () => { - this.blobs = await dkv(this.blobStoreOptions()); - }; - - // uncomment for verbose logging of everything here to the console. - // dbg(f: string) { - // return (...args) => console.log(f, args); - // } - - async runCells( - _ids: string[], - _opts: { noHalt?: boolean } = {}, - ): Promise { - throw Error("DEPRECATED"); - } - - private set_backend_state(backend_state: BackendState): void { - this.dbg("set_backend_state")(backend_state); - - /* - The backend states, which are put in the syncdb so clients - can display this: - - - 'init' -- the backend is checking the file on disk, etc. - - 'ready' -- the backend is setup and ready to use; kernel isn't running though - - 'starting' -- the kernel itself is actived and currently starting up (e.g., Sage is starting up) - - 'running' -- the kernel is running and ready to evaluate code - - - 'init' --> 'ready' --> 'spawning' --> 'starting' --> 'running' - /|\ | - |-----------------------------------------| - - Going from ready to starting happens first when a code execution is requested. - */ - - // Check just in case Typescript doesn't catch something: - if ( - ["init", "ready", "spawning", "starting", "running"].indexOf( - backend_state, - ) === -1 - ) { - throw Error(`invalid backend state '${backend_state}'`); - } - if (backend_state == "init" && this._backend_state != "init") { - // Do NOT allow changing the state to init from any other state. - throw Error( - `illegal state change '${this._backend_state}' --> '${backend_state}'`, - ); - } - this._backend_state = backend_state; - - if (this.lastSavedBackendState != backend_state) { - this._set({ - type: "settings", - backend_state, - last_backend_state: Date.now(), - }); - this.save_asap(); - this.lastSavedBackendState = backend_state; - } - - // The following is to clear kernel_error if things are working only. - if (backend_state == "running") { - // clear kernel error if kernel successfully starts and stays - // in running state for a while. - this.clear_kernel_error = setTimeout(() => { - this._set({ - type: "settings", - kernel_error: "", - }); - }, 3000); - } else { - // change to a different state; cancel attempt to clear kernel error - if (this.clear_kernel_error) { - clearTimeout(this.clear_kernel_error); - delete this.clear_kernel_error; - } - } - } - - set_kernel_state = (state: any, save = false) => { - this._kernel_state = state; - this._set({ type: "settings", kernel_state: state }, save); - }; - - // Called exactly once when the manager first starts up after the store is initialized. - // Here we ensure everything is in a consistent state so that we can react - // to changes later. - async initialize_manager() { - if (this._initialize_manager_already_done) { - return; - } - const dbg = this.dbg("initialize_manager"); - dbg(); - this._initialize_manager_already_done = true; - - dbg("initialize Jupyter Conat api handler"); - await this.initConatApi(); - - dbg("initializing blob store"); - await this.initBlobStore(); - - this._throttled_ensure_positions_are_unique = debounce( - this.ensure_positions_are_unique, - 5000, - ); - // Listen for changes... - this.syncdb.on("change", this.backendSyncdbChange); - - this.setState({ - // used by the kernel_info function of this.jupyter_kernel - start_time: this._client.server_time().valueOf(), - }); - - // clear nbconvert start on init, since no nbconvert can be running yet - this.syncdb.delete({ type: "nbconvert" }); - - // Initialize info about available kernels, which is used e.g., for - // saving to ipynb format. - this.init_kernel_info(); - - // We try once to load from disk. If it fails, then - // a record with type:'fatal' - // is created in the database; if it succeeds, that record is deleted. - // Try again only when the file changes. - await this._first_load(); - - // Listen for model state changes... - if (this.syncdb.ipywidgets_state == null) { - throw Error("syncdb's ipywidgets_state must be defined!"); - } - this.syncdb.ipywidgets_state.on( - "change", - this.handle_ipywidgets_state_change, - ); - } - - private conatService?; - private initConatApi = reuseInFlight(async () => { - if (this.conatService != null) { - this.conatService.close(); - this.conatService = null; - } - const service = (this.conatService = await initConatService({ - project_id: this.project_id, - path: this.path, - })); - this.syncdb.on("closed", () => { - service.close(); - }); - }); - - private _first_load = async () => { - const dbg = this.dbg("_first_load"); - dbg("doing load"); - if (this.is_closed()) { - throw Error("actions must not be closed"); - } - try { - await this.loadFromDiskIfNewer(); - } catch (err) { - dbg(`load failed -- ${err}; wait for file change and try again`); - const path = this.store.get("path"); - const watcher = this._client.watch_file({ path }); - await once(watcher, "change"); - dbg("file changed"); - watcher.close(); - await this._first_load(); - return; - } - dbg("loading worked"); - this._init_after_first_load(); - }; - - private _init_after_first_load = () => { - const dbg = this.dbg("_init_after_first_load"); - - dbg("initializing"); - // this may change the syncdb. - this.ensure_backend_kernel_setup(); - - this.init_file_watcher(); - - this._state = "ready"; + public blobs = { + set: (_k, _v) => {}, + get: (_k): any => {}, }; + save_ipynb_file = async (_opts?) => {}; + capture_output_message = (_opts) => {}; + process_comm_message_from_kernel = (_mesg) => {}; - private backendSyncdbChange = (changes: any) => { - if (this.is_closed()) { - return; - } - const dbg = this.dbg("backendSyncdbChange"); - if (changes != null) { - changes.forEach((key) => { - switch (key.get("type")) { - case "settings": - dbg("settings change"); - var record = this.syncdb.get_one(key); - if (record != null) { - // ensure kernel is properly configured - this.ensure_backend_kernel_setup(); - // only the backend should change kernel and backend state; - // however, our security model allows otherwise (e.g., via TimeTravel). - if ( - record.get("kernel_state") !== this._kernel_state && - this._kernel_state != null - ) { - this.set_kernel_state(this._kernel_state, true); - } - if (record.get("backend_state") !== this._backend_state) { - this.set_backend_state(this._backend_state); - } - - if (record.get("run_all_loop_s")) { - if (this.run_all_loop == null) { - this.run_all_loop = new RunAllLoop( - this, - record.get("run_all_loop_s"), - ); - } else { - // ensure interval is correct - this.run_all_loop.set_interval(record.get("run_all_loop_s")); - } - } else if ( - !record.get("run_all_loop_s") && - this.run_all_loop != null - ) { - // stop it. - this.run_all_loop.close(); - delete this.run_all_loop; - } - } - break; - } - }); - } - - this._throttled_ensure_positions_are_unique(); - }; - - // ensure_backend_kernel_setup ensures that we have a connection - // to the selected Jupyter kernel, if any. - ensure_backend_kernel_setup = () => { - const dbg = this.dbg("ensure_backend_kernel_setup"); - if (this.isDeleted()) { - dbg("file is deleted"); + initKernel = () => { + if (this.jupyter_kernel != null) { return; } - const kernel = this.store.get("kernel"); - dbg("ensure_backend_kernel_setup", { kernel }); - - let current: string | undefined = undefined; - if (this.jupyter_kernel != null) { - current = this.jupyter_kernel.name; - if (current == kernel) { - const state = this.jupyter_kernel.get_state(); - if (state == "error") { - dbg("kernel is broken"); - // nothing to do -- let user ponder the error they should see. - return; - } - if (state != "closed") { - dbg("everything is properly setup and working"); - return; - } - } - } - - dbg(`kernel='${kernel}', current='${current}'`); - if ( - this.jupyter_kernel != null && - this.jupyter_kernel.get_state() != "closed" - ) { - if (current != kernel) { - dbg("kernel changed -- kill running kernel to trigger switch"); - this.jupyter_kernel.close(); - return; - } else { - dbg("nothing to do"); - return; - } - } - - dbg("make a new kernel"); - + console.log("initKernel", { kernel, path: this.path }); // No kernel wrapper object setup at all. Make one. this.jupyter_kernel = createJupyterKernel({ name: kernel, - path: this.store.get("path"), + path: this.path, actions: this, }); - - if (this.syncdb.ipywidgets_state == null) { - throw Error("syncdb's ipywidgets_state must be defined!"); - } - this.syncdb.ipywidgets_state.clear(); - - if (this.jupyter_kernel == null) { - // to satisfy typescript. - throw Error("jupyter_kernel must be defined"); - } - dbg("kernel created -- installing handlers"); - - // save so gets reported to frontend, and surfaced to user: - // https://github.com/sagemathinc/cocalc/issues/4847 - this.jupyter_kernel.on("kernel_error", (error) => { - this.set_kernel_error(error); - }); - - this.restartKernelOnClose = () => { - // When the kernel closes, make sure a new kernel gets setup. - if (this.store == null || this._state !== "ready") { - // This event can also happen when this actions is being closed, - // in which case obviously we shouldn't make a new kernel. - return; - } - dbg("kernel closed -- make new one."); - this.ensure_backend_kernel_setup(); - }; - - this.jupyter_kernel.once("closed", this.restartKernelOnClose); - - // Track backend state changes other than closing, so they - // are visible to user etc. - // TODO: Maybe all these need to move to ephemeral table? - // There's a good argument that recording these is useful though, so when - // looking at time travel or debugging, you know what was going on. - this.jupyter_kernel.on("state", (state) => { - dbg("jupyter_kernel state --> ", state); - switch (state) { - case "off": - case "closed": - // things went wrong. - this.set_backend_state("ready"); - this.jupyter_kernel?.close(); - delete this.jupyter_kernel; - return; - case "spawning": - case "starting": - this.set_connection_file(); // yes, fall through - case "running": - this.set_backend_state(state); - } - }); - - this.jupyter_kernel.on("execution_state", this.set_kernel_state); - - this.handle_all_cell_attachments(); - dbg("ready"); - this.set_backend_state("ready"); - }; - - set_connection_file = () => { - const connection_file = this.jupyter_kernel?.get_connection_file() ?? ""; - this._set({ - type: "settings", - connection_file, - }); }; - init_kernel_info = async () => { - let kernels0 = this.store.get("kernels"); - if (kernels0 != null) { - return; - } - const dbg = this.dbg("init_kernel_info"); - dbg("getting"); - let kernels; - try { - kernels = await get_kernel_data(); - dbg("success"); - } catch (err) { - dbg(`FAILED to get kernel info: ${err}`); - // TODO: what to do?? Saving will be broken... - return; - } - this.setState({ - kernels: immutable.fromJS(kernels), - }); - }; - - async ensure_backend_kernel_is_running() { - const dbg = this.dbg("ensure_backend_kernel_is_running"); - if (this._backend_state == "ready") { - dbg("in state 'ready', so kick it into gear"); - await this.set_backend_kernel_info(); - dbg("done getting kernel info"); - } - const is_running = (s): boolean => { - if (this._state === "closed") { - return true; - } - const t = s.get_one({ type: "settings" }); - if (t == null) { - dbg("no settings"); - return false; - } else { - const state = t.get("backend_state"); - dbg(`state = ${state}`); - return state == "running"; - } - }; - await this.syncdb.wait(is_running, 60); - } - - protected __syncdb_change_post_hook(doInit: boolean) { - if (doInit) { - // Since just opening the actions in the project, definitely the kernel - // isn't running so set this fact in the shared database. It will make - // things always be in the right initial state. - this.syncdb.set({ - type: "settings", - backend_state: "init", - kernel_state: "idle", - kernel_usage: { memory: 0, cpu: 0 }, - }); - this.syncdb.commit(); - - // Also initialize the execution manager, which runs cells that have been - // requested to run. - this.initialize_manager(); - } - } - - _cancel_run = (id: any) => { - const dbg = this.dbg(`_cancel_run ${id}`); - // All these checks are so we only cancel if it is actually running - // with the current kernel... - if (this._running_cells == null || this.jupyter_kernel == null) return; - const identity = this._running_cells[id]; - if (identity == null) return; - if (this.jupyter_kernel.identity == identity) { - dbg("canceling"); - this.jupyter_kernel.cancel_execute(id); - } else { - dbg("not canceling since wrong identity"); - } - }; - - private init_file_watcher = () => { - const dbg = this.dbg("file_watcher"); - dbg(); - this._file_watcher = this._client.watch_file({ - path: this.store.get("path"), - debounce: 1000, - }); - - this._file_watcher.on("change", async () => { - dbg("change"); - try { - await this.loadFromDiskIfNewer(); - } catch (err) { - dbg("failed to load on change", err); - } - }); - }; - - // Load file from disk if it is newer than - // the last we saved to disk. - private loadFromDiskIfNewer = async () => { - const dbg = this.dbg("loadFromDiskIfNewer"); - // Get mtime of last .ipynb file that we explicitly saved. - - // TODO: breaking the syncdb typescript data hiding. The - // right fix will be to move - // this info to a new ephemeral state table. - const last_ipynb_save = await this.get_last_ipynb_save(); - dbg(`syncdb last_ipynb_save=${last_ipynb_save}`); - let file_changed; - if (last_ipynb_save == 0) { - // we MUST load from file the first time, of course. - file_changed = true; - dbg("file changed because FIRST TIME"); - } else { - const path = this.store.get("path"); - let stats; - try { - stats = await callback2(this._client.path_stat, { path }); - dbg(`stats.mtime = ${stats.mtime}`); - } catch (err) { - // This err just means the file doesn't exist. - // We set the 'last load' to now in this case, since - // the frontend clients need to know that we - // have already scanned the disk. - this.set_last_load(); - return; - } - const mtime = stats.mtime.getTime(); - file_changed = mtime > last_ipynb_save; - dbg({ mtime, last_ipynb_save }); - } - if (file_changed) { - dbg(".ipynb disk file changed ==> loading state from disk"); - try { - await this.load_ipynb_file(); - } catch (err) { - dbg("failed to load on change", err); - } - } else { - dbg("disk file NOT changed: NOT loading"); - } - }; - - // if also set load is true, we also set the "last_ipynb_save" time. - set_last_load = (alsoSetLoad: boolean = false) => { - const last_load = new Date().getTime(); - this.syncdb.set({ - type: "file", - last_load, - }); - if (alsoSetLoad) { - // yes, load v save is inconsistent! - this.syncdb.set({ type: "settings", last_ipynb_save: last_load }); - } - this.syncdb.commit(); - }; - - /* Determine timestamp of aux .ipynb file, and record it here, - so we know that we do not have to load exactly that file - back from disk. */ - private set_last_ipynb_save = async () => { - let stats; - try { - stats = await callback2(this._client.path_stat, { - path: this.store.get("path"), - }); - } catch (err) { - // no-op -- nothing to do. - this.dbg("set_last_ipynb_save")(`WARNING -- issue in path_stat ${err}`); - return; - } - - // This is ugly (i.e., how we get access), but I need to get this done. - // This is the RIGHT place to save the info though. - // TODO: move this state info to new ephemeral table. - try { - const last_ipynb_save = stats.mtime.getTime(); - this.last_ipynb_save = last_ipynb_save; - this._set({ - type: "settings", - last_ipynb_save, - }); - this.dbg("stats.mtime.getTime()")( - `set_last_ipynb_save = ${last_ipynb_save}`, - ); - } catch (err) { - this.dbg("set_last_ipynb_save")( - `WARNING -- issue in set_last_ipynb_save ${err}`, - ); - return; - } - }; - - private get_last_ipynb_save = async () => { - const x = - this.syncdb.get_one({ type: "settings" })?.get("last_ipynb_save") ?? 0; - return Math.max(x, this.last_ipynb_save); - }; - - load_ipynb_file = async () => { - /* - Read the ipynb file from disk. Fully use the ipynb file to - set the syncdb's state. We do this when opening a new file, or when - the file changes on disk (e.g., a git checkout or something). - */ - const dbg = this.dbg(`load_ipynb_file`); - dbg("reading file"); - const path = this.store.get("path"); - let content: string; - try { - content = await callback2(this._client.path_read, { - path, - maxsize_MB: MAX_SIZE_IPYNB_MB, - }); - } catch (err) { - // possibly file doesn't exist -- set notebook to empty. - const exists = await callback2(this._client.path_exists, { - path, - }); - if (!exists) { - content = ""; - } else { - // It would be better to have a button to push instead of - // suggesting running a command in the terminal, but - // adding that took 1 second. Better than both would be - // making it possible to edit huge files :-). - const error = `Error reading ipynb file '${path}': ${err.toString()}. Fix this to continue. You can delete all output by typing cc-jupyter-no-output [filename].ipynb in a terminal.`; - this.syncdb.set({ type: "fatal", error }); - throw Error(error); - } - } - if (content.length === 0) { - // Blank file, e.g., when creating in CoCalc. - // This is good, works, etc. -- just clear state, including error. - this.syncdb.delete(); - this.set_last_load(true); - return; - } - - // File is nontrivial -- parse and load. - let parsed_content; - try { - parsed_content = JSON.parse(content); - } catch (err) { - const error = `Error parsing the ipynb file '${path}': ${err}. You must fix the ipynb file somehow before continuing, or use TimeTravel to revert to a recent version.`; - dbg(error); - this.syncdb.set({ type: "fatal", error }); - throw Error(error); - } - this.syncdb.delete({ type: "fatal" }); - await this.set_to_ipynb(parsed_content); - this.set_last_load(true); - }; - - private fetch_jupyter_kernels = async () => { - const data = await get_kernel_data(); - const kernels = immutable.fromJS(data as any); - this.setState({ kernels }); - }; - - save_ipynb_file = async ({ - version = 0, - timeout = 15000, - }: { - // if version is given, waits (up to timeout ms) for syncdb to - // contain that exact version before writing the ipynb to disk. - // This may be needed to ensure that ipynb saved to disk - // reflects given frontend state. This comes up, e.g., in - // generating the nbgrader version of a document. - version?: number; - timeout?: number; - } = {}) => { - const dbg = this.dbg("save_ipynb_file"); - if (version && !this.syncdb.hasVersion(version)) { - dbg(`frontend needs ${version}, which we do not yet have`); - const start = Date.now(); - while (true) { - if (this.is_closed()) { - return; - } - if (Date.now() - start >= timeout) { - dbg("timed out waiting"); - break; - } - try { - dbg(`waiting for version ${version}`); - await once(this.syncdb, "change", timeout - (Date.now() - start)); - } catch { - dbg("timed out waiting"); - break; - } - if (this.syncdb.hasVersion(version)) { - dbg("now have the version"); - break; - } - } - } - if (this.is_closed()) { - return; - } - dbg("saving to file"); - - // Check first if file was deleted, in which case instead of saving to disk, - // we should terminate and clean up everything. - if (this.isDeleted()) { - dbg("ipynb file is deleted, so NOT saving to disk and closing"); - this.close(); - return; - } - - if (this.jupyter_kernel == null) { - // The kernel is needed to get access to the blob store, which - // may be needed to save to disk. - this.ensure_backend_kernel_setup(); - if (this.jupyter_kernel == null) { - // still not null? This would happen if no kernel is set at all, - // in which case it's OK that saving isn't possible. - throw Error("no kernel so cannot save"); - } - } - if (this.store.get("kernels") == null) { - await this.init_kernel_info(); - if (this.store.get("kernels") == null) { - // This should never happen, but maybe could in case of a very - // messed up compute environment where the kernelspecs can't be listed. - throw Error( - "kernel info not known and can't be determined, so can't save", - ); - } - } - dbg("going to try to save: getting ipynb object..."); - const blob_store = this.jupyter_kernel.get_blob_store(); - let ipynb = this.store.get_ipynb(blob_store); - if (this.store.get("kernel")) { - // if a kernel is set, check that it was sufficiently known that - // we can fill in data about it -- - // see https://github.com/sagemathinc/cocalc/issues/7286 - if (ipynb?.metadata?.kernelspec?.name == null) { - dbg("kernelspec not known -- try loading kernels again"); - await this.fetch_jupyter_kernels(); - // and again grab the ipynb - ipynb = this.store.get_ipynb(blob_store); - if (ipynb?.metadata?.kernelspec?.name == null) { - dbg("kernelspec STILL not known: metadata will be incomplete"); - } - } - } - dbg("got ipynb object"); - // We use json_stable (and indent 1) to be more diff friendly to user, - // and more consistent with official Jupyter. - const data = json_stable(ipynb, { space: 1 }); - if (data == null) { - dbg("failed -- ipynb not defined yet"); - throw Error("ipynb not defined yet; can't save"); - } - dbg("converted ipynb to stable JSON string", data?.length); - //dbg(`got string version '${data}'`) - try { - dbg("writing to disk..."); - await callback2(this._client.write_file, { - path: this.store.get("path"), - data, - }); - dbg("succeeded at saving"); - await this.set_last_ipynb_save(); - } catch (err) { - const e = `error writing file: ${err}`; - dbg(e); - throw Error(e); - } - }; - - private handle_all_cell_attachments() { - // Check if any cell attachments need to be loaded. - const cells = this.store.get("cells"); - cells?.forEach((cell) => { - this.handle_cell_attachments(cell); - }); - } - - private handle_cell_attachments(cell) { - if (this.jupyter_kernel == null) { - // can't do anything - return; - } - const dbg = this.dbg(`handle_cell_attachments(id=${cell.get("id")})`); - dbg(); - - const attachments = cell.get("attachments"); - if (attachments == null) return; // nothing to do - attachments.forEach(async (x, name) => { - if (x == null) return; - if (x.get("type") === "load") { - if (this.jupyter_kernel == null) return; // try later - // need to load from disk - this.set_cell_attachment(cell.get("id"), name, { - type: "loading", - value: null, - }); - let sha1: string; - try { - sha1 = await this.jupyter_kernel.load_attachment(x.get("value")); - } catch (err) { - this.set_cell_attachment(cell.get("id"), name, { - type: "error", - value: `${err}`, - }); - return; - } - this.set_cell_attachment(cell.get("id"), name, { - type: "sha1", - value: sha1, - }); - } - }); - } - - // handle_ipywidgets_state_change is called when the project ipywidgets_state - // object changes, e.g., in response to a user moving a slider in the browser. - // It crafts a comm message that is sent to the running Jupyter kernel telling - // it about this change by calling send_comm_message_to_kernel. - private handle_ipywidgets_state_change = (keys): void => { - if (this.is_closed()) { - return; - } - const dbg = this.dbg("handle_ipywidgets_state_change"); - dbg(keys); - if (this.jupyter_kernel == null) { - dbg("no kernel, so ignoring changes to ipywidgets"); - return; - } - if (this.syncdb.ipywidgets_state == null) { - throw Error("syncdb's ipywidgets_state must be defined!"); - } - for (const key of keys) { - const [, model_id, type] = JSON.parse(key); - dbg({ key, model_id, type }); - let data: any; - if (type === "value") { - const state = this.syncdb.ipywidgets_state.get_model_value(model_id); - // Saving the buffers on change is critical since otherwise this breaks: - // https://ipywidgets.readthedocs.io/en/latest/examples/Widget%20List.html#file-upload - // Note that stupidly the buffer (e.g., image upload) gets sent to the kernel twice. - // But it does work robustly, and the kernel and nodejs server processes next to each - // other so this isn't so bad. - const { buffer_paths, buffers } = - this.syncdb.ipywidgets_state.getKnownBuffers(model_id); - data = { method: "update", state, buffer_paths }; - this.jupyter_kernel.send_comm_message_to_kernel({ - msg_id: misc.uuid(), - target_name: "jupyter.widget", - comm_id: model_id, - data, - buffers, - }); - } else if (type === "buffers") { - // TODO: we MIGHT need implement this... but MAYBE NOT. An example where this seems like it might be - // required is by the file upload widget, but actually that just uses the value type above, since - // we explicitly fill in the widgets there; also there is an explicit comm upload message that - // the widget sends out that updates the buffer, and in send_comm_message_to_kernel in jupyter/kernel/kernel.ts - // when processing that message, we saves those buffers and make sure they are set in the - // value case above (otherwise they would get removed). - // https://ipywidgets.readthedocs.io/en/latest/examples/Widget%20List.html#file-upload - // which creates a buffer from the content of the file, then sends it to the backend, - // which sees a change and has to write that buffer to the kernel (here) so that - // the running python process can actually do something with the file contents (e.g., - // process data, save file to disk, etc). - // We need to be careful though to not send buffers to the kernel that the kernel sent us, - // since that would be a waste. - } else if (type === "state") { - // TODO: currently ignoring this, since it seems chatty and pointless, - // and could lead to race conditions probably with multiple users, etc. - // It happens right when the widget is created. - /* - const state = this.syncdb.ipywidgets_state.getModelSerializedState(model_id); - data = { method: "update", state }; - this.jupyter_kernel.send_comm_message_to_kernel( - misc.uuid(), - model_id, - data - ); - */ - } else { - const m = `Jupyter: unknown type '${type}'`; - console.warn(m); - dbg(m); - } - } - }; - - async process_comm_message_from_kernel(mesg: any): Promise { - const dbg = this.dbg("process_comm_message_from_kernel"); - // serializing the full message could cause enormous load on the server, since - // the mesg may contain large buffers. Only do for low level debugging! - // dbg(mesg); // EXTREME DANGER! - // This should be safe: - dbg(JSON.stringify(mesg.header)); - if (this.syncdb.ipywidgets_state == null) { - throw Error("syncdb's ipywidgets_state must be defined!"); - } - await this.syncdb.ipywidgets_state.process_comm_message_from_kernel(mesg); - } - - capture_output_message(mesg: any): boolean { - if (this.syncdb.ipywidgets_state == null) { - throw Error("syncdb's ipywidgets_state must be defined!"); - } - return this.syncdb.ipywidgets_state.capture_output_message(mesg); - } - - close_project_only() { - const dbg = this.dbg("close_project_only"); - dbg(); - if (this.run_all_loop) { - this.run_all_loop.close(); - delete this.run_all_loop; - } - // this stops the kernel and cleans everything up - // so no resources are wasted and next time starting - // is clean - (async () => { - try { - await removeJupyterRedux(this.store.get("path"), this.project_id); - } catch (err) { - dbg("WARNING -- issue removing jupyter redux", err); - } - })(); - - this.blobs?.close(); - } - // not actually async... - async signal(signal = "SIGINT"): Promise { + signal = async (signal = "SIGINT"): Promise => { this.jupyter_kernel?.signal(signal); - } - - handle_nbconvert_change(oldVal, newVal): void { - nbconvertChange(this, oldVal?.toJS(), newVal?.toJS()); - } - - // Handle transient cell messages. - handleTransientUpdate = (mesg) => { - const display_id = mesg.content?.transient?.display_id; - if (!display_id) { - return false; - } - - let matched = false; - // are there any transient outputs in the entire document that - // have this display_id? search to find them. - // TODO: we could use a clever data structure to make - // this faster and more likely to have bugs. - const cells = this.syncdb.get({ type: "cell" }); - for (let cell of cells) { - let output = cell.get("output"); - if (output != null) { - for (const [n, val] of output) { - if (val.getIn(["transient", "display_id"]) == display_id) { - // found a match -- replace it - output = output.set(n, immutable.fromJS(mesg.content)); - this.syncdb.set({ type: "cell", id: cell.get("id"), output }); - matched = true; - } - } - } - } - if (matched) { - this.syncdb.commit(); - } - }; - - getComputeServers = () => { - // we don't bother worrying about freeing this since it is only - // run in the project or compute server, which needs the underlying - // dkv for its entire lifetime anyways. - if (this.computeServers == null) { - this.computeServers = computeServerManager({ - project_id: this.project_id, - }); - } - return this.computeServers; - }; - - getComputeServerIdSync = (): number => { - const c = this.getComputeServers(); - return c.get(this.syncdb.path) ?? 0; - }; - - getComputeServerId = async (): Promise => { - const c = this.getComputeServers(); - return (await c.getServerIdForPath(this.syncdb.path)) ?? 0; }; } diff --git a/src/packages/project/conat/jupyter.ts b/src/packages/project/conat/jupyter.ts index 1405684198..80f8482acc 100644 --- a/src/packages/project/conat/jupyter.ts +++ b/src/packages/project/conat/jupyter.ts @@ -1,3 +1,22 @@ +/* + +To run just this for a project in a console, from the browser, terminate the jupyter server by running this +in your browser with the project open: + + await cc.client.conat_client.projectApi(cc.current()).system.terminate({service:'jupyter'}) + +As explained in packages/project/conat/api/index.ts setup your environment as for the project. + +Then run this code in nodejs: + + require("@cocalc/project/conat/jupyter").init() + + + + +*/ + + import { jupyterRun } from "@cocalc/project/conat/api/editor"; import { outputHandler } from "@cocalc/jupyter/control"; import { jupyterServer } from "@cocalc/conat/project/jupyter/run-code"; diff --git a/src/packages/sync/editor/generic/sync-doc.ts b/src/packages/sync/editor/generic/sync-doc.ts index 868eecdf52..22bd8bedfd 100644 --- a/src/packages/sync/editor/generic/sync-doc.ts +++ b/src/packages/sync/editor/generic/sync-doc.ts @@ -1299,7 +1299,9 @@ export class SyncDoc extends EventEmitter { await this.readFile(); if (firstLoad) { dbg("emitting first-load event"); - // this event is emited the first time the document is ever loaded from disk. + // this event is emited the first time the document is ever + // loaded from disk. It's used, e.g., for notebook "trust" state, + // so important from a security POV. this.emit("first-load"); } dbg("loaded"); From 7cca2c1a75253a9d63b24c50b080d1e08f5baa23 Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 1 Aug 2025 18:00:58 +0000 Subject: [PATCH 156/270] refactor: move the jupyter functionality from project api.editor to new api.jupyter --- .../test/project/jupyter/run-code.test.ts | 12 +++---- src/packages/conat/project/api/editor.ts | 28 ---------------- src/packages/conat/project/api/index.ts | 3 ++ src/packages/conat/project/api/jupyter.ts | 32 +++++++++++++++++++ .../conat/project/jupyter/run-code.ts | 12 +++---- .../frontend/jupyter/browser-actions.ts | 4 +-- src/packages/frontend/jupyter/kernelspecs.ts | 2 +- src/packages/frontend/jupyter/logo.tsx | 2 +- .../frontend/project/websocket/api.ts | 6 ++-- src/packages/jupyter/control.ts | 20 ++++++------ src/packages/project/conat/api/editor.ts | 32 ------------------- src/packages/project/conat/api/index.ts | 2 ++ src/packages/project/conat/jupyter.ts | 5 ++- 13 files changed, 68 insertions(+), 92 deletions(-) create mode 100644 src/packages/conat/project/api/jupyter.ts diff --git a/src/packages/backend/conat/test/project/jupyter/run-code.test.ts b/src/packages/backend/conat/test/project/jupyter/run-code.test.ts index 95dcb1400e..460442324b 100644 --- a/src/packages/backend/conat/test/project/jupyter/run-code.test.ts +++ b/src/packages/backend/conat/test/project/jupyter/run-code.test.ts @@ -35,7 +35,7 @@ describe("create very simple mocked jupyter runner and test evaluating code", () const project_id = uuid(); it("create jupyter code run server", () => { // running code with this just results in two responses: the path and the cells - async function jupyterRun({ path, cells }) { + async function run({ path, cells }) { async function* runner() { yield { path, id: "0" }; yield { cells, id: "0" }; @@ -43,7 +43,7 @@ describe("create very simple mocked jupyter runner and test evaluating code", () return runner(); } - server = jupyterServer({ client: client1, project_id, jupyterRun }); + server = jupyterServer({ client: client1, project_id, run }); }); let client; @@ -120,7 +120,7 @@ describe("create simple mocked jupyter runner that does actually eval an express const compute_server_id = 3; it("create jupyter code run server", () => { // running code with this just results in two responses: the path and the cells - async function jupyterRun({ cells }) { + async function run({ cells }) { async function* runner() { for (const { id, input } of cells) { yield { id, output: eval(input) }; @@ -132,7 +132,7 @@ describe("create simple mocked jupyter runner that does actually eval an express server = jupyterServer({ client: client1, project_id, - jupyterRun, + run, compute_server_id, }); }); @@ -197,7 +197,7 @@ describe("create mocked jupyter runner that does failover to backend output mana let handler: any = null; it("create jupyter code run server that also takes as long as the output to run", () => { - async function jupyterRun({ cells }) { + async function run({ cells }) { async function* runner() { for (const { id, input } of cells) { const output = eval(input); @@ -232,7 +232,7 @@ describe("create mocked jupyter runner that does failover to backend output mana server = jupyterServer({ client: client1, project_id, - jupyterRun, + run, outputHandler, }); }); diff --git a/src/packages/conat/project/api/editor.ts b/src/packages/conat/project/api/editor.ts index e17d597e8a..66750e2c51 100644 --- a/src/packages/conat/project/api/editor.ts +++ b/src/packages/conat/project/api/editor.ts @@ -1,19 +1,8 @@ -import type { NbconvertParams } from "@cocalc/util/jupyter/types"; -import type { RunNotebookOptions } from "@cocalc/util/jupyter/nbgrader-types"; import type { Options as FormatterOptions } from "@cocalc/util/code-formatter"; -import type { KernelSpec } from "@cocalc/util/jupyter/types"; export const editor = { newFile: true, - jupyterStart: true, - jupyterStop: true, - jupyterStripNotebook: true, - jupyterNbconvert: true, - jupyterRunNotebook: true, - jupyterKernelLogo: true, - jupyterKernels: true, - formatString: true, printSageWS: true, @@ -41,23 +30,6 @@ export interface Editor { // context of our editors. newFile: (path: string) => Promise; - jupyterStripNotebook: (path_ipynb: string) => Promise; - - // path = the syncdb path (not *.ipynb) - jupyterStart: (path: string) => Promise; - jupyterStop: (path: string) => Promise; - - jupyterNbconvert: (opts: NbconvertParams) => Promise; - - jupyterRunNotebook: (opts: RunNotebookOptions) => Promise; - - jupyterKernelLogo: ( - kernelName: string, - opts?: { noCache?: boolean }, - ) => Promise<{ filename: string; base64: string }>; - - jupyterKernels: (opts?: { noCache?: boolean }) => Promise; - // returns formatted version of str. formatString: (opts: { str: string; diff --git a/src/packages/conat/project/api/index.ts b/src/packages/conat/project/api/index.ts index 694229b2bc..96febf9172 100644 --- a/src/packages/conat/project/api/index.ts +++ b/src/packages/conat/project/api/index.ts @@ -1,17 +1,20 @@ import { type System, system } from "./system"; import { type Editor, editor } from "./editor"; +import { type Jupyter, jupyter } from "./jupyter"; import { type Sync, sync } from "./sync"; import { handleErrorMessage } from "@cocalc/conat/util"; export interface ProjectApi { system: System; editor: Editor; + jupyter: Jupyter; sync: Sync; } const ProjectApiStructure = { system, editor, + jupyter, sync, } as const; diff --git a/src/packages/conat/project/api/jupyter.ts b/src/packages/conat/project/api/jupyter.ts new file mode 100644 index 0000000000..20ec07b4bf --- /dev/null +++ b/src/packages/conat/project/api/jupyter.ts @@ -0,0 +1,32 @@ +import type { NbconvertParams } from "@cocalc/util/jupyter/types"; +import type { RunNotebookOptions } from "@cocalc/util/jupyter/nbgrader-types"; +import type { KernelSpec } from "@cocalc/util/jupyter/types"; + +export const jupyter = { + start: true, + stop: true, + stripNotebook: true, + nbconvert: true, + runNotebook: true, + kernelLogo: true, + kernels: true, +}; + +export interface Jupyter { + stripNotebook: (path_ipynb: string) => Promise; + + // path = the syncdb path (not *.ipynb) + start: (path: string) => Promise; + stop: (path: string) => Promise; + + nbconvert: (opts: NbconvertParams) => Promise; + + runNotebook: (opts: RunNotebookOptions) => Promise; + + kernelLogo: ( + kernelName: string, + opts?: { noCache?: boolean }, + ) => Promise<{ filename: string; base64: string }>; + + kernels: (opts?: { noCache?: boolean }) => Promise; +} diff --git a/src/packages/conat/project/jupyter/run-code.ts b/src/packages/conat/project/jupyter/run-code.ts index 0417350eef..ef78773bf9 100644 --- a/src/packages/conat/project/jupyter/run-code.ts +++ b/src/packages/conat/project/jupyter/run-code.ts @@ -79,9 +79,9 @@ export function jupyterServer({ client, project_id, compute_server_id = 0, - // jupyterRun takes a path and cells to run and returns an async iterator + // run takes a path and cells to run and returns an async iterator // over the outputs. - jupyterRun, + run, // outputHandler takes a path and returns an OutputHandler, which can be // used to process the output and include it in the notebook. It is used // as a fallback in case the client that initiated running cells is @@ -91,7 +91,7 @@ export function jupyterServer({ client: ConatClient; project_id: string; compute_server_id?: number; - jupyterRun: JupyterCodeRunner; + run: JupyterCodeRunner; outputHandler?: CreateOutputHandler; }) { const subject = getSubject({ project_id, compute_server_id }); @@ -113,7 +113,7 @@ export function jupyterServer({ mesg.respondSync(null); await handleRequest({ socket, - jupyterRun, + run, outputHandler, path, cells, @@ -142,13 +142,13 @@ export function jupyterServer({ async function handleRequest({ socket, - jupyterRun, + run, outputHandler, path, cells, noHalt, }) { - const runner = await jupyterRun({ path, cells, noHalt }); + const runner = await run({ path, cells, noHalt }); const output: OutputMessage[] = []; const throttle = new Throttle(MAX_MSGS_PER_SECOND); diff --git a/src/packages/frontend/jupyter/browser-actions.ts b/src/packages/frontend/jupyter/browser-actions.ts index 0415bd3d37..a2341cf7f6 100644 --- a/src/packages/frontend/jupyter/browser-actions.ts +++ b/src/packages/frontend/jupyter/browser-actions.ts @@ -1464,7 +1464,7 @@ export class JupyterActions extends JupyterActions0 { } try { const api = await this.conatApi(); - await api.editor.jupyterStart(this.syncdbPath); + await api.jupyter.start(this.syncdbPath); return true; } catch (err) { console.log("failed to initialize ", this.path, err); @@ -1477,7 +1477,7 @@ export class JupyterActions extends JupyterActions0 { stopBackend = async () => { const api = await this.conatApi(); - await api.editor.jupyterStop(this.syncdbPath); + await api.jupyter.stop(this.syncdbPath); }; getOutputHandler = (cell) => { diff --git a/src/packages/frontend/jupyter/kernelspecs.ts b/src/packages/frontend/jupyter/kernelspecs.ts index 596e55b57a..5a1db32a2b 100644 --- a/src/packages/frontend/jupyter/kernelspecs.ts +++ b/src/packages/frontend/jupyter/kernelspecs.ts @@ -38,7 +38,7 @@ const getKernelSpec = reuseInFlight( compute_server_id, timeout: 7500, }); - const spec = await api.editor.jupyterKernels(); + const spec = await api.jupyter.kernels(); cache.set(key, spec); return spec; }, diff --git a/src/packages/frontend/jupyter/logo.tsx b/src/packages/frontend/jupyter/logo.tsx index fe26b5f15e..963a47650e 100644 --- a/src/packages/frontend/jupyter/logo.tsx +++ b/src/packages/frontend/jupyter/logo.tsx @@ -113,7 +113,7 @@ async function getLogo({ return cache[key]; } const api = client.conat_client.projectApi({ project_id }); - const { filename, base64 } = await api.editor.jupyterKernelLogo(kernel, { + const { filename, base64 } = await api.jupyter.kernelLogo(kernel, { noCache, }); if (!filename || !base64) { diff --git a/src/packages/frontend/project/websocket/api.ts b/src/packages/frontend/project/websocket/api.ts index 31dced4922..4fad962002 100644 --- a/src/packages/frontend/project/websocket/api.ts +++ b/src/packages/frontend/project/websocket/api.ts @@ -306,7 +306,7 @@ export class API { compute_server_id ?? this.getComputeServerId(opts.args[0]), timeout: (opts.timeout ?? 60) * 1000 + 5000, }); - return await api.editor.jupyterNbconvert(opts); + return await api.jupyter.nbconvert(opts); }; // Get contents of an ipynb file, but with output and attachments removed (to save space) @@ -318,7 +318,7 @@ export class API { compute_server_id: compute_server_id ?? this.getComputeServerId(ipynb_path), }); - return await api.editor.jupyterStripNotebook(ipynb_path); + return await api.jupyter.stripNotebook(ipynb_path); }; // Run the notebook filling in the output of all cells, then return the @@ -338,7 +338,7 @@ export class API { compute_server_id, timeout: 60 + 2 * max_total_time_ms, }); - return await api.editor.jupyterRunNotebook(opts); + return await api.jupyter.runNotebook(opts); }; // Get the x11 *channel* for the given '.x11' path. diff --git a/src/packages/jupyter/control.ts b/src/packages/jupyter/control.ts index 1020c94ec7..bd80e795b4 100644 --- a/src/packages/jupyter/control.ts +++ b/src/packages/jupyter/control.ts @@ -14,7 +14,7 @@ const logger = getLogger("jupyter:control"); const sessions: { [path: string]: { syncdb: SyncDB; actions; store } } = {}; let project_id: string = ""; -export function jupyterStart({ +export function start({ path, client, project_id: project_id0, @@ -27,10 +27,10 @@ export function jupyterStart({ }) { project_id = project_id0; if (sessions[path] != null) { - logger.debug("jupyterStart: ", path, " - already running"); + logger.debug("start: ", path, " - already running"); return; } - logger.debug("jupyterStart: ", path, " - starting it"); + logger.debug("start: ", path, " - starting it"); const syncdb = new SyncDB({ ...SYNCDB_OPTIONS, project_id, @@ -41,22 +41,22 @@ export function jupyterStart({ // [ ] TODO: some way to convey this to clients (?) syncdb.on("error", (err) => { logger.debug(`syncdb error -- ${err}`, path); - jupyterStop({ path }); + stop({ path }); }); syncdb.on("close", () => { - jupyterStop({ path }); + stop({ path }); }); const { actions, store } = initJupyterRedux(syncdb, client); sessions[path] = { syncdb, actions, store }; } -export function jupyterStop({ path }: { path: string }) { +export function stop({ path }: { path: string }) { const session = sessions[path]; if (session == null) { - logger.debug("jupyterStop: ", path, " - not running"); + logger.debug("stop: ", path, " - not running"); } else { const { syncdb } = session; - logger.debug("jupyterStop: ", path, " - stopping it"); + logger.debug("stop: ", path, " - stopping it"); syncdb.close(); delete sessions[path]; const path_ipynb = original_path(path); @@ -65,8 +65,8 @@ export function jupyterStop({ path }: { path: string }) { } // Returns async iterator over outputs -export async function jupyterRun({ path, cells, noHalt }: RunOptions) { - logger.debug("jupyterRun", { path, noHalt }); +export async function run({ path, cells, noHalt }: RunOptions) { + logger.debug("run:", { path, noHalt }); const session = sessions[path]; if (session == null) { diff --git a/src/packages/project/conat/api/editor.ts b/src/packages/project/conat/api/editor.ts index c858c7d124..b03f0c4ea1 100644 --- a/src/packages/project/conat/api/editor.ts +++ b/src/packages/project/conat/api/editor.ts @@ -1,9 +1,4 @@ -export { jupyter_strip_notebook as jupyterStripNotebook } from "@cocalc/jupyter/nbgrader/jupyter-parse"; -export { jupyter_run_notebook as jupyterRunNotebook } from "@cocalc/jupyter/nbgrader/jupyter-run"; -export { nbconvert as jupyterNbconvert } from "../../jupyter/convert"; export { formatString } from "../../formatters"; -export { logo as jupyterKernelLogo } from "@cocalc/jupyter/kernel/logo"; -export { get_kernel_data as jupyterKernels } from "@cocalc/jupyter/kernel/kernel-data"; export { newFile } from "@cocalc/backend/misc/new-file"; import { printSageWS as printSageWS0 } from "@cocalc/project/print_to_pdf"; @@ -34,30 +29,3 @@ export async function printSageWS(opts): Promise { } export { createTerminalService } from "@cocalc/project/conat/terminal"; - -import { getClient } from "@cocalc/project/client"; -import { project_id } from "@cocalc/project/data"; -import * as control from "@cocalc/jupyter/control"; -import { SandboxedFilesystem } from "@cocalc/backend/files/sandbox"; - -export async function jupyterStart(path: string) { - const fs = new SandboxedFilesystem(process.env.HOME ?? "/tmp", { - unsafeMode: true, - }); - await control.jupyterStart({ project_id, path, client: getClient(), fs }); -} - -// IMPORTANT: jupyterRun is NOT used directly by the API, but instead by packages/project/conat/jupyter.ts -// It is convenient to have it here so it can call jupyterStart above, etc. The reason is because -// this returns an async iterator managed using a dedicated socket, and the api is request/response.. -export async function jupyterRun(opts: { - path: string; - cells: { id: string; input: string }[]; -}) { - await jupyterStart(opts.path); - return await control.jupyterRun(opts); -} - -export async function jupyterStop(path: string) { - await control.jupyterStop({ path }); -} diff --git a/src/packages/project/conat/api/index.ts b/src/packages/project/conat/api/index.ts index ef45cb1eb8..7133583609 100644 --- a/src/packages/project/conat/api/index.ts +++ b/src/packages/project/conat/api/index.ts @@ -152,11 +152,13 @@ async function handleApiRequest(request, mesg) { import * as system from "./system"; import * as editor from "./editor"; +import * as jupyter from "./jupyter"; import * as sync from "./sync"; export const projectApi: ProjectApi = { system, editor, + jupyter, sync, }; diff --git a/src/packages/project/conat/jupyter.ts b/src/packages/project/conat/jupyter.ts index 80f8482acc..82a04312d3 100644 --- a/src/packages/project/conat/jupyter.ts +++ b/src/packages/project/conat/jupyter.ts @@ -16,8 +16,7 @@ Then run this code in nodejs: */ - -import { jupyterRun } from "@cocalc/project/conat/api/editor"; +import { run } from "@cocalc/project/conat/api/jupyter"; import { outputHandler } from "@cocalc/jupyter/control"; import { jupyterServer } from "@cocalc/conat/project/jupyter/run-code"; import { connectToConat } from "@cocalc/project/conat/connection"; @@ -34,7 +33,7 @@ export function init() { client, project_id, compute_server_id, - jupyterRun, + run, outputHandler, }); } From d3c1d2dc8d5d3b96b6c37032b80e82c7c4a2fe98 Mon Sep 17 00:00:00 2001 From: William Stein Date: Fri, 1 Aug 2025 20:01:45 +0000 Subject: [PATCH 157/270] jupyter: rewriting to move more control/state to frontend --- src/packages/conat/project/api/jupyter.ts | 13 + .../frontend/jupyter/browser-actions.ts | 360 ++++++++++++++++-- src/packages/jupyter/control.ts | 98 +++-- src/packages/jupyter/kernel/kernel.ts | 21 +- src/packages/jupyter/pool/pool.ts | 4 +- src/packages/jupyter/redux/actions.ts | 311 +-------------- src/packages/jupyter/redux/project-actions.ts | 8 +- src/packages/jupyter/redux/run-all-loop.ts | 63 --- .../jupyter/types/project-interface.ts | 4 +- src/packages/project/conat/api/jupyter.ts | 50 +++ src/packages/util/jupyter/names.ts | 23 +- 11 files changed, 501 insertions(+), 454 deletions(-) delete mode 100644 src/packages/jupyter/redux/run-all-loop.ts create mode 100644 src/packages/project/conat/api/jupyter.ts diff --git a/src/packages/conat/project/api/jupyter.ts b/src/packages/conat/project/api/jupyter.ts index 20ec07b4bf..3defeb5b57 100644 --- a/src/packages/conat/project/api/jupyter.ts +++ b/src/packages/conat/project/api/jupyter.ts @@ -10,8 +10,12 @@ export const jupyter = { runNotebook: true, kernelLogo: true, kernels: true, + introspect: true, + signal: true, }; +// In the functions below path can be either the .ipynb or the .sage-jupyter2 path, and +// the correct backend kernel will get found/created automatically. export interface Jupyter { stripNotebook: (path_ipynb: string) => Promise; @@ -29,4 +33,13 @@ export interface Jupyter { ) => Promise<{ filename: string; base64: string }>; kernels: (opts?: { noCache?: boolean }) => Promise; + + introspect: (opts: { + path: string; + code: string; + cursor_pos: number; + detail_level: 0 | 1; + }) => Promise; + + signal: (opts: { path: string; signal: string }) => Promise; } diff --git a/src/packages/frontend/jupyter/browser-actions.ts b/src/packages/frontend/jupyter/browser-actions.ts index a2341cf7f6..7601684e60 100644 --- a/src/packages/frontend/jupyter/browser-actions.ts +++ b/src/packages/frontend/jupyter/browser-actions.ts @@ -18,7 +18,6 @@ import { get_local_storage, set_local_storage, } from "@cocalc/frontend/misc/local-storage"; -import track from "@cocalc/frontend/user-tracking"; import { webapp_client } from "@cocalc/frontend/webapp-client"; import { JupyterActions as JupyterActions0 } from "@cocalc/jupyter/redux/actions"; import { CellToolbarName } from "@cocalc/jupyter/types"; @@ -27,6 +26,7 @@ import { base64ToBuffer, bufferToBase64 } from "@cocalc/util/base64"; import { Config as FormatterConfig, Syntax } from "@cocalc/util/code-formatter"; import { closest_kernel_match, + cmp, field_cmp, from_json, history_path, @@ -64,6 +64,11 @@ import { } from "@cocalc/conat/project/jupyter/run-code"; import { OutputHandler } from "@cocalc/jupyter/execute/output-handler"; import { throttle } from "lodash"; +import { + char_idx_to_js_idx, + codemirror_to_jupyter_pos, + js_idx_to_char_idx, +} from "@cocalc/jupyter/util/misc"; const OUTPUT_FPS = 29; @@ -77,6 +82,8 @@ export class JupyterActions extends JupyterActions0 { private account_change_editor_settings: any; private update_keyboard_shortcuts: any; public syncdbPath: string; + private last_cursor_move_time: Date = new Date(0); + private _introspect_request?: any; protected init2(): void { this.syncdbPath = syncdbPath(this.path); @@ -125,15 +132,15 @@ export class JupyterActions extends JupyterActions0 { this.fetch_jupyter_kernels(); // Load kernel (once ipynb file loads). - (async () => { - await this.set_kernel_after_load(); - if (!this.store) return; - track("jupyter", { - kernel: this.store.get("kernel"), - project_id: this.project_id, - path: this.path, - }); - })(); + // (async () => { + // await this.set_kernel_after_load(); + // if (!this.store) return; + // track("jupyter", { + // kernel: this.store.get("kernel"), + // project_id: this.project_id, + // path: this.path, + // }); + // })(); // nbgrader support this.nbgrader_actions = new NBGraderActions(this, this.redux); @@ -1132,22 +1139,22 @@ export class JupyterActions extends JupyterActions0 { }); }; - private set_kernel_after_load = async (): Promise => { - // Browser Client: Wait until the .ipynb file has actually been parsed into - // the (hidden, e.g. .a.ipynb.sage-jupyter2) syncdb file, - // then set the kernel, if necessary. - try { - await this.syncdb.wait((s) => !!s.get_one({ type: "file" }), 600); - } catch (err) { - if (this._state != "ready") { - // Probably user just closed the notebook before it finished - // loading, so we don't need to set the kernel. - return; - } - throw Error("error waiting for ipynb file to load"); - } - this._syncdb_init_kernel(); - }; + // private set_kernel_after_load = async (): Promise => { + // // Browser Client: Wait until the .ipynb file has actually been parsed into + // // the (hidden, e.g. .a.ipynb.sage-jupyter2) syncdb file, + // // then set the kernel, if necessary. + // try { + // await this.syncdb.wait((s) => !!s.get_one({ type: "file" }), 600); + // } catch (err) { + // if (this._state != "ready") { + // // Probably user just closed the notebook before it finished + // // loading, so we don't need to set the kernel. + // return; + // } + // throw Error("error waiting for ipynb file to load"); + // } + // this._syncdb_init_kernel(); + // }; private _syncdb_init_kernel = (): void => { // console.log("jupyter::_syncdb_init_kernel", this.store.get("kernel")); @@ -1534,7 +1541,7 @@ export class JupyterActions extends JupyterActions0 { private jupyterClient?; private runQueue: any[] = []; private runningNow = false; - async runCells(ids: string[], opts: { noHalt?: boolean } = {}) { + runCells = async (ids: string[], opts: { noHalt?: boolean } = {}) => { if (this.store?.get("read_only")) { return; } @@ -1552,6 +1559,7 @@ export class JupyterActions extends JupyterActions0 { // [ ] **TODO: Must invalidate this when compute server changes!!!!!** // and const compute_server_id = await this.getComputeServerId(); + if (this.isClosed()) return; this.jupyterClient = jupyterClient({ path: this.syncdbPath, client: webapp_client.conat_client.conat(), @@ -1604,9 +1612,11 @@ export class JupyterActions extends JupyterActions0 { cells.sort(field_cmp("pos")); const runner = await client.run(cells, opts); + if (this.isClosed()) return; let handler: null | OutputHandler = null; let id: null | string = null; for await (const mesgs of runner) { + if (this.isClosed()) return; for (const mesg of mesgs) { if (!opts.noHalt && mesg.msg_type == "error") { this.clearRunQueue(); @@ -1630,15 +1640,309 @@ export class JupyterActions extends JupyterActions0 { } } handler?.done(); - this.save_asap(); + this.syncdb.save(); + setTimeout(() => { + if (!this.isClosed()) { + this.syncdb.save(); + } + }, 1000); } catch (err) { console.warn("runCells", err); } finally { + if (this.isClosed()) return; this.runningNow = false; if (this.runQueue.length > 0) { const [ids, opts] = this.runQueue.shift(); this.runCells(ids, opts); } } + }; + + is_introspecting(): boolean { + const actions = this.getFrameActions(); + return actions?.store?.get("introspect") != null; + } + + introspect_close = () => { + if (this.is_introspecting()) { + this.getFrameActions()?.setState({ introspect: undefined }); + } + }; + + introspect_at_pos = async ( + code: string, + detail_level: 0 | 1 = 0, + pos: { ch: number; line: number }, + ): Promise => { + if (code === "") return; // no-op if there is no code (should never happen) + await this.introspect( + code, + detail_level, + codemirror_to_jupyter_pos(code, pos), + ); + }; + + introspect = async ( + code: string, + detail_level: 0 | 1, + cursor_pos?: number, + ): Promise | undefined> => { + const req = (this._introspect_request = + (this._introspect_request != null ? this._introspect_request : 0) + 1); + + if (cursor_pos == null) { + cursor_pos = code.length; + } + cursor_pos = js_idx_to_char_idx(cursor_pos, code); + + let introspect; + try { + const api = await this.conatApi(); + introspect = await api.jupyter.introspect({ + path: this.path, + code, + cursor_pos, + detail_level, + }); + if (introspect.status !== "ok") { + introspect = { error: "completion failed" }; + } + delete introspect.status; + } catch (err) { + introspect = { error: err }; + } + if (this._introspect_request > req) return; + const i = fromJS(introspect); + this.getFrameActions()?.setState({ + introspect: i, + }); + return introspect; // convenient / useful, e.g., for use by whiteboard. + }; + + clear_introspect = (): void => { + this._introspect_request = + (this._introspect_request != null ? this._introspect_request : 0) + 1; + this.getFrameActions()?.setState({ introspect: undefined }); + }; + + // Attempt to fetch completions for give code and cursor_pos + // If successful, the completions are put in store.get('completions') and looks like + // this (as an immutable map): + // cursor_end : 2 + // cursor_start : 0 + // matches : ['the', 'completions', ...] + // status : "ok" + // code : code + // cursor_pos : cursor_pos + // + // If not successful, result is: + // status : "error" + // code : code + // cursor_pos : cursor_pos + // error : 'an error message' + // + // Only the most recent fetch has any impact, and calling + // clear_complete() ensures any fetch made before that + // is ignored. + + // Returns true if a dialog with options appears, and false otherwise. + complete = async ( + code: string, + pos?: { line: number; ch: number } | number, + id?: string, + offset?: any, + ): Promise => { + let cursor_pos; + const req = (this._complete_request = + (this._complete_request != null ? this._complete_request : 0) + 1); + + this.setState({ complete: undefined }); + + // pos can be either a {line:?, ch:?} object as in codemirror, + // or a number. + if (pos == null || typeof pos == "number") { + cursor_pos = pos; + } else { + cursor_pos = codemirror_to_jupyter_pos(code, pos); + } + cursor_pos = js_idx_to_char_idx(cursor_pos, code); + + const start = new Date(); + let complete; + try { + complete = await this.api().complete({ + code, + cursor_pos, + }); + } catch (err) { + if (this._complete_request > req) return false; + this.setState({ complete: { error: err } }); + // no op for now... + throw Error(`ignore -- ${err}`); + //return false; + } + + if (this.last_cursor_move_time >= start) { + // see https://github.com/sagemathinc/cocalc/issues/3611 + throw Error("ignore"); + //return false; + } + if (this._complete_request > req) { + // future completion or clear happened; so ignore this result. + throw Error("ignore"); + //return false; + } + + if (complete.status !== "ok") { + this.setState({ + complete: { + error: complete.error ? complete.error : "completion failed", + }, + }); + return false; + } + + if (complete.matches == 0) { + return false; + } + + delete complete.status; + complete.base = code; + complete.code = code; + complete.pos = char_idx_to_js_idx(cursor_pos, code); + complete.cursor_start = char_idx_to_js_idx(complete.cursor_start, code); + complete.cursor_end = char_idx_to_js_idx(complete.cursor_end, code); + complete.id = id; + // Set the result so the UI can then react to the change. + if (offset != null) { + complete.offset = offset; + } + // For some reason, sometimes complete.matches are not unique, which is annoying/confusing, + // and breaks an assumption in our react code too. + // I think the reason is e.g., a filename and a variable could be the same. We're not + // worrying about that now. + complete.matches = Array.from(new Set(complete.matches)); + // sort in a way that matches how JupyterLab sorts completions, which + // is case insensitive with % magics at the bottom + complete.matches.sort((x, y) => { + const c = cmp(getCompletionGroup(x), getCompletionGroup(y)); + if (c) { + return c; + } + return cmp(x.toLowerCase(), y.toLowerCase()); + }); + const i_complete = fromJS(complete); + if (complete.matches && complete.matches.length === 1 && id != null) { + // special case -- a unique completion and we know id of cell in which completing is given. + this.select_complete(id, complete.matches[0], i_complete); + return false; + } else { + this.setState({ complete: i_complete }); + return true; + } + }; + + clear_complete = (): void => { + this._complete_request = + (this._complete_request != null ? this._complete_request : 0) + 1; + this.setState({ complete: undefined }); + }; + + public select_complete( + id: string, + item: string, + complete?: Map, + ): void { + if (complete == null) { + complete = this.store.get("complete"); + } + this.clear_complete(); + if (complete == null) { + return; + } + const input = complete.get("code"); + if (input != null && complete.get("error") == null) { + const starting = input.slice(0, complete.get("cursor_start")); + const ending = input.slice(complete.get("cursor_end")); + const new_input = starting + item + ending; + const base = complete.get("base"); + this.complete_cell(id, base, new_input); + } + } + + complete_cell(id: string, base: string, new_input: string): void { + this.merge_cell_input(id, base, new_input); + } + + public set_cursor_locs(locs: any[] = [], side_effect: boolean = false): void { + this.last_cursor_move_time = new Date(); + if (this.syncdb == null) { + // syncdb not always set -- https://github.com/sagemathinc/cocalc/issues/2107 + return; + } + if (locs.length === 0) { + // don't remove on blur -- cursor will fade out just fine + return; + } + this._cursor_locs = locs; // remember our own cursors for splitting cell + this.syncdb.set_cursor_locs(locs, side_effect); + } + + async signal(signal = "SIGINT"): Promise { + const api = await this.conatApi(); + try { + await api.jupyter.signal({ path: this.path, signal }); + } catch (err) { + this.set_error(err); + } + } + + // Kill the running kernel and does NOT start it up again. + halt = reuseInFlight(async (): Promise => { + if (this.restartKernelOnClose != null && this.jupyter_kernel != null) { + this.jupyter_kernel.removeListener("closed", this.restartKernelOnClose); + delete this.restartKernelOnClose; + } + this.clear_all_cell_run_state(); + await this.signal("SIGKILL"); + // Wait a little, since SIGKILL has to really happen on backend, + // and server has to respond and change state. + const not_running = (s): boolean => { + if (this._state === "closed") return true; + const t = s.get_one({ type: "settings" }); + return t != null && t.get("backend_state") != "running"; + }; + try { + await this.syncdb.wait(not_running, 30); + // worked -- and also no need to show "kernel got killed" message since this was intentional. + this.set_error(""); + } catch (err) { + // failed + this.set_error(err); + } + }); + + restart = reuseInFlight(async (): Promise => { + await this.halt(); + if (this.is_closed()) return; + this.clear_all_cell_run_state(); + }); + + shutdown = reuseInFlight(async (): Promise => { + if (this.is_closed()) return; + await this.signal("SIGKILL"); + if (this.is_closed()) return; + this.clear_all_cell_run_state(); + }); +} + +function getCompletionGroup(x: string): number { + switch (x[0]) { + case "_": + return 1; + case "%": + return 2; + default: + return 0; } } diff --git a/src/packages/jupyter/control.ts b/src/packages/jupyter/control.ts index bd80e795b4..8efeddd0ac 100644 --- a/src/packages/jupyter/control.ts +++ b/src/packages/jupyter/control.ts @@ -3,21 +3,27 @@ import { SYNCDB_OPTIONS } from "@cocalc/jupyter/redux/sync"; import { type Filesystem } from "@cocalc/conat/files/fs"; import { getLogger } from "@cocalc/backend/logger"; import { initJupyterRedux, removeJupyterRedux } from "@cocalc/jupyter/kernel"; -import { original_path } from "@cocalc/util/misc"; +import { syncdbPath, ipynbPath } from "@cocalc/util/jupyter/names"; import { once } from "@cocalc/util/async-utils"; import { OutputHandler } from "@cocalc/jupyter/execute/output-handler"; import { throttle } from "lodash"; import { type RunOptions } from "@cocalc/conat/project/jupyter/run-code"; +import { type JupyterActions } from "@cocalc/jupyter/redux/project-actions"; const logger = getLogger("jupyter:control"); -const sessions: { [path: string]: { syncdb: SyncDB; actions; store } } = {}; +const jupyterActions: { [ipynbPath: string]: JupyterActions } = {}; + +export function isRunning(path): boolean { + return jupyterActions[ipynbPath(path)] != null; +} + let project_id: string = ""; export function start({ path, - client, project_id: project_id0, + client, fs, }: { path: string; @@ -25,42 +31,40 @@ export function start({ project_id: string; fs: Filesystem; }) { - project_id = project_id0; - if (sessions[path] != null) { - logger.debug("start: ", path, " - already running"); + if (isRunning(path)) { return; } + project_id = project_id0; logger.debug("start: ", path, " - starting it"); const syncdb = new SyncDB({ ...SYNCDB_OPTIONS, project_id, - path, + path: syncdbPath(path), client, fs, }); - // [ ] TODO: some way to convey this to clients (?) syncdb.on("error", (err) => { + // [ ] TODO: some way to convey this to clients (?) logger.debug(`syncdb error -- ${err}`, path); stop({ path }); }); - syncdb.on("close", () => { + syncdb.once("closed", () => { stop({ path }); }); - const { actions, store } = initJupyterRedux(syncdb, client); - sessions[path] = { syncdb, actions, store }; + const { actions } = initJupyterRedux(syncdb, client); + jupyterActions[ipynbPath(path)] = actions; } export function stop({ path }: { path: string }) { - const session = sessions[path]; - if (session == null) { + const actions = jupyterActions[ipynbPath(path)]; + if (actions == null) { logger.debug("stop: ", path, " - not running"); } else { - const { syncdb } = session; + delete jupyterActions[ipynbPath(path)]; + const { syncdb } = actions; logger.debug("stop: ", path, " - stopping it"); syncdb.close(); - delete sessions[path]; - const path_ipynb = original_path(path); - removeJupyterRedux(path_ipynb, project_id); + removeJupyterRedux(ipynbPath(path), project_id); } } @@ -68,42 +72,42 @@ export function stop({ path }: { path: string }) { export async function run({ path, cells, noHalt }: RunOptions) { logger.debug("run:", { path, noHalt }); - const session = sessions[path]; - if (session == null) { - throw Error(`${path} not running`); + const actions = jupyterActions[ipynbPath(path)]; + if (actions == null) { + throw Error(`${ipynbPath(path)} not running`); } - const { syncdb, actions } = session; - if (syncdb.isClosed()) { + if (actions.syncdb.isClosed()) { // shouldn't be possible throw Error("syncdb is closed"); } - if (!syncdb.isReady()) { + if (!actions.syncdb.isReady()) { logger.debug("jupyterRun: waiting until ready"); - await once(syncdb, "ready"); + await once(actions.syncdb, "ready"); } logger.debug("jupyterRun: running"); - async function* run() { + async function* runCells() { for (const cell of cells) { - actions.initKernel(); - const output = actions.jupyter_kernel.execute_code({ + actions.ensureKernelIsReady(); + const kernel = actions.jupyter_kernel!; + const output = kernel.execute_code({ halt_on_error: !noHalt, code: cell.input, }); - for await (const mesg of output.iter()) { - mesg.id = cell.id; + for await (const mesg0 of output.iter()) { + const mesg = { ...mesg0, id: cell.id }; yield mesg; if (!noHalt && mesg.msg_type == "error") { // done running code because there was an error. return; } } - if (actions.jupyter_kernel.failedError) { + if (kernel.failedError) { // kernel failed during call - throw Error(actions.jupyter_kernel.failedError); + throw Error(kernel.failedError); } } } - return await run(); + return await runCells(); } class MulticellOutputHandler { @@ -156,9 +160,33 @@ class MulticellOutputHandler { const BACKEND_OUTPUT_FPS = 8; export function outputHandler({ path, cells }: RunOptions) { - if (sessions[path] == null) { - throw Error(`session '${path}' not available`); + if (jupyterActions[ipynbPath(path)] == null) { + throw Error(`session '${ipynbPath(path)}' not available`); } - const { actions } = sessions[path]; + const actions = jupyterActions[ipynbPath(path)]; return new MulticellOutputHandler(cells, actions); } + +function getKernel(path: string) { + const actions = jupyterActions[ipynbPath(path)]; + if (actions == null) { + throw Error(`${ipynbPath(path)} not running`); + } + actions.ensureKernelIsReady(); + return actions.jupyter_kernel!; +} + +export async function introspect(opts: { + path: string; + code: string; + cursor_pos: number; + detail_level: 0 | 1; +}) { + const kernel = getKernel(opts.path); + return await kernel.introspect(opts); +} + +export async function signal(opts: { path: string; signal: string }) { + const kernel = getKernel(opts.path); + await kernel.signal(opts.signal); +} diff --git a/src/packages/jupyter/kernel/kernel.ts b/src/packages/jupyter/kernel/kernel.ts index f8ef7ac1fa..d41b1253a1 100644 --- a/src/packages/jupyter/kernel/kernel.ts +++ b/src/packages/jupyter/kernel/kernel.ts @@ -250,7 +250,7 @@ export class JupyterKernel public _execute_code_queue: CodeExecutionEmitter[] = []; public sockets?: JupyterSockets; private has_ensured_running: boolean = false; - private failedError: string = ""; + public failedError: string = ""; constructor( name: string | undefined, @@ -284,6 +284,8 @@ export class JupyterKernel dbg("done"); } + isClosed = () => this._state == "closed"; + get_path = () => { return this._path; }; @@ -388,7 +390,7 @@ export class JupyterKernel } catch (err) { dbg(`ERROR spawning kernel - ${err}, ${err.stack}`); // @ts-ignore - if (this._state == "closed") { + if (this.isClosed()) { throw Error("closed"); } // console.trace(err); @@ -554,6 +556,7 @@ export class JupyterKernel // Signal should be a string like "SIGINT", "SIGKILL". // See https://nodejs.org/api/process.html#process_process_kill_pid_signal + // this does NOT raise an error. signal = (signal: string): void => { const dbg = this.dbg("signal"); const pid = this.pid(); @@ -564,9 +567,7 @@ export class JupyterKernel try { process.kill(-pid, signal); // negative to signal the process group this.clear_execute_code_queue(); - } catch (err) { - dbg(`error: ${err}`); - } + } catch {} }; close = (): void => { @@ -628,7 +629,7 @@ export class JupyterKernel ensure_running = reuseInFlight(async (): Promise => { const dbg = this.dbg("ensure_running"); dbg(this._state); - if (this._state == "closed") { + if (this.isClosed()) { throw Error("closed so not possible to ensure running"); } if (this._state == "running") { @@ -659,7 +660,7 @@ export class JupyterKernel opts.halt_on_error = true; } if (this._state === "closed") { - throw Error("closed -- kernel -- execute_code"); + throw Error("execute_code: jupyter kernel is closed"); } const code = new CodeExecutionEmitter(this, opts); if (skipToFront) { @@ -740,7 +741,7 @@ export class JupyterKernel const dbg = this.dbg("_clear_execute_code_queue"); // ensure no future queued up evaluation occurs (currently running // one will complete and new executions could happen) - if (this._state === "closed") { + if (this.isClosed()) { dbg("no op since state is closed"); return; } @@ -762,7 +763,7 @@ export class JupyterKernel // the terminal and nbgrader and the stateless api. execute_code_now = async (opts: ExecOpts): Promise => { this.dbg("execute_code_now")(); - if (this._state == "closed") { + if (this.isClosed()) { throw Error("closed"); } if (this.failedError) { @@ -854,7 +855,7 @@ export class JupyterKernel await this.ensure_running(); } // Do a paranoid double check anyways... - if (this.sockets == null || this._state == "closed") { + if (this.sockets == null || this.isClosed()) { throw Error("not running, so can't call"); } diff --git a/src/packages/jupyter/pool/pool.ts b/src/packages/jupyter/pool/pool.ts index 8e74263a2c..da5fc3091c 100644 --- a/src/packages/jupyter/pool/pool.ts +++ b/src/packages/jupyter/pool/pool.ts @@ -267,8 +267,8 @@ export async function killKernel(kernel: SpawnedKernel) { log.debug("killKernel pid=", kernel.spawn.pid); try { process.kill(-kernel.spawn.pid, "SIGTERM"); - } catch (error) { - log.error("Failed to send SIGTERM to Jupyter kernel", error); + } catch { + //log.error("Failed to send SIGTERM to Jupyter kernel", error); } } kernel.spawn?.close?.(); diff --git a/src/packages/jupyter/redux/actions.ts b/src/packages/jupyter/redux/actions.ts index 7c3c9f85ba..872f472826 100644 --- a/src/packages/jupyter/redux/actions.ts +++ b/src/packages/jupyter/redux/actions.ts @@ -39,11 +39,6 @@ import { JupyterStore, JupyterStoreState } from "@cocalc/jupyter/redux/store"; import { Cell, KernelInfo } from "@cocalc/jupyter/types"; import { IPynbImporter } from "@cocalc/jupyter/ipynb/import-from-ipynb"; import type { JupyterKernelInterface } from "@cocalc/jupyter/types/project-interface"; -import { - char_idx_to_js_idx, - codemirror_to_jupyter_pos, - js_idx_to_char_idx, -} from "@cocalc/jupyter/util/misc"; import { SyncDB } from "@cocalc/sync/editor/db/sync"; import type { Client } from "@cocalc/sync/client/types"; import latexEnvs from "@cocalc/util/latex-envs"; @@ -70,9 +65,7 @@ export class JupyterActions extends Actions { readonly path: string; readonly project_id: string; public jupyter_kernel?: JupyterKernelInterface; - private last_cursor_move_time: Date = new Date(0); - private _cursor_locs?: any; - private _introspect_request?: any; + public _cursor_locs?: any; protected set_save_status: any; protected _client: Client; protected _file_watcher: any; @@ -95,7 +88,6 @@ export class JupyterActions extends Actions { store: any, client: Client, ): void { - console.log("jupyter actions: _init", { path }); this._client = client; const dbg = this.dbg("_init"); dbg("Initializing Jupyter Actions"); @@ -209,11 +201,10 @@ export class JupyterActions extends Actions { // an account_change listener. } - public is_closed(): boolean { - return (this._state ?? "closed") === "closed"; - } + isClosed = () => (this._state ?? "closed") == "closed"; + is_closed = () => (this._state ?? "closed") == "closed"; - public close() { + close() { if (this.is_closed()) { return; } @@ -1031,20 +1022,6 @@ export class JupyterActions extends Actions { this.runCells(v.slice(i)); } - public set_cursor_locs(locs: any[] = [], side_effect: boolean = false): void { - this.last_cursor_move_time = new Date(); - if (this.syncdb == null) { - // syncdb not always set -- https://github.com/sagemathinc/cocalc/issues/2107 - return; - } - if (locs.length === 0) { - // don't remove on blur -- cursor will fade out just fine - return; - } - this._cursor_locs = locs; // remember our own cursors for splitting cell - this.syncdb.set_cursor_locs(locs, side_effect); - } - public split_cell(id: string, cursor: { line: number; ch: number }): void { if (this.check_edit_protection(id, "splitting cell")) { return; @@ -1410,155 +1387,6 @@ export class JupyterActions extends Actions { return this.store.getIn(["cells", id, "input"], ""); } - // Attempt to fetch completions for give code and cursor_pos - // If successful, the completions are put in store.get('completions') and looks like - // this (as an immutable map): - // cursor_end : 2 - // cursor_start : 0 - // matches : ['the', 'completions', ...] - // status : "ok" - // code : code - // cursor_pos : cursor_pos - // - // If not successful, result is: - // status : "error" - // code : code - // cursor_pos : cursor_pos - // error : 'an error message' - // - // Only the most recent fetch has any impact, and calling - // clear_complete() ensures any fetch made before that - // is ignored. - - // Returns true if a dialog with options appears, and false otherwise. - public async complete( - code: string, - pos?: { line: number; ch: number } | number, - id?: string, - offset?: any, - ): Promise { - let cursor_pos; - const req = (this._complete_request = - (this._complete_request != null ? this._complete_request : 0) + 1); - - this.setState({ complete: undefined }); - - // pos can be either a {line:?, ch:?} object as in codemirror, - // or a number. - if (pos == null || typeof pos == "number") { - cursor_pos = pos; - } else { - cursor_pos = codemirror_to_jupyter_pos(code, pos); - } - cursor_pos = js_idx_to_char_idx(cursor_pos, code); - - const start = new Date(); - let complete; - try { - complete = await this.api().complete({ - code, - cursor_pos, - }); - } catch (err) { - if (this._complete_request > req) return false; - this.setState({ complete: { error: err } }); - // no op for now... - throw Error(`ignore -- ${err}`); - //return false; - } - - if (this.last_cursor_move_time >= start) { - // see https://github.com/sagemathinc/cocalc/issues/3611 - throw Error("ignore"); - //return false; - } - if (this._complete_request > req) { - // future completion or clear happened; so ignore this result. - throw Error("ignore"); - //return false; - } - - if (complete.status !== "ok") { - this.setState({ - complete: { - error: complete.error ? complete.error : "completion failed", - }, - }); - return false; - } - - if (complete.matches == 0) { - return false; - } - - delete complete.status; - complete.base = code; - complete.code = code; - complete.pos = char_idx_to_js_idx(cursor_pos, code); - complete.cursor_start = char_idx_to_js_idx(complete.cursor_start, code); - complete.cursor_end = char_idx_to_js_idx(complete.cursor_end, code); - complete.id = id; - // Set the result so the UI can then react to the change. - if (offset != null) { - complete.offset = offset; - } - // For some reason, sometimes complete.matches are not unique, which is annoying/confusing, - // and breaks an assumption in our react code too. - // I think the reason is e.g., a filename and a variable could be the same. We're not - // worrying about that now. - complete.matches = Array.from(new Set(complete.matches)); - // sort in a way that matches how JupyterLab sorts completions, which - // is case insensitive with % magics at the bottom - complete.matches.sort((x, y) => { - const c = misc.cmp(getCompletionGroup(x), getCompletionGroup(y)); - if (c) { - return c; - } - return misc.cmp(x.toLowerCase(), y.toLowerCase()); - }); - const i_complete = immutable.fromJS(complete); - if (complete.matches && complete.matches.length === 1 && id != null) { - // special case -- a unique completion and we know id of cell in which completing is given. - this.select_complete(id, complete.matches[0], i_complete); - return false; - } else { - this.setState({ complete: i_complete }); - return true; - } - } - - clear_complete = (): void => { - this._complete_request = - (this._complete_request != null ? this._complete_request : 0) + 1; - this.setState({ complete: undefined }); - }; - - public select_complete( - id: string, - item: string, - complete?: immutable.Map, - ): void { - if (complete == null) { - complete = this.store.get("complete"); - } - this.clear_complete(); - if (complete == null) { - return; - } - const input = complete.get("code"); - if (input != null && complete.get("error") == null) { - const starting = input.slice(0, complete.get("cursor_start")); - const ending = input.slice(complete.get("cursor_end")); - const new_input = starting + item + ending; - const base = complete.get("base"); - this.complete_cell(id, base, new_input); - } - } - - complete_cell(id: string, base: string, new_input: string): void { - this.merge_cell_input(id, base, new_input); - } - merge_cell_input( id: string, base: string, @@ -1577,126 +1405,6 @@ export class JupyterActions extends Actions { this.set_cell_input(id, new_input, save); } - is_introspecting(): boolean { - const actions = this.getFrameActions() as any; - return actions?.store?.get("introspect") != null; - } - - introspect_close = () => { - if (this.is_introspecting()) { - this.getFrameActions()?.setState({ introspect: undefined }); - } - }; - - introspect_at_pos = async ( - code: string, - level: 0 | 1 = 0, - pos: { ch: number; line: number }, - ): Promise => { - if (code === "") return; // no-op if there is no code (should never happen) - await this.introspect(code, level, codemirror_to_jupyter_pos(code, pos)); - }; - - introspect = async ( - code: string, - level: 0 | 1, - cursor_pos?: number, - ): Promise | undefined> => { - const req = (this._introspect_request = - (this._introspect_request != null ? this._introspect_request : 0) + 1); - - if (cursor_pos == null) { - cursor_pos = code.length; - } - cursor_pos = js_idx_to_char_idx(cursor_pos, code); - - let introspect; - try { - introspect = await this.api().introspect({ - code, - cursor_pos, - level, - }); - if (introspect.status !== "ok") { - introspect = { error: "completion failed" }; - } - delete introspect.status; - } catch (err) { - introspect = { error: err }; - } - if (this._introspect_request > req) return; - const i = immutable.fromJS(introspect); - this.getFrameActions()?.setState({ - introspect: i, - }); - return i; // convenient / useful, e.g., for use by whiteboard. - }; - - clear_introspect = (): void => { - this._introspect_request = - (this._introspect_request != null ? this._introspect_request : 0) + 1; - this.getFrameActions()?.setState({ introspect: undefined }); - }; - - public async signal(signal = "SIGINT"): Promise { - const api = this.api({ timeout: 5000 }); - try { - await api.signal(signal); - } catch (err) { - this.set_error(err); - } - } - - // Kill the running kernel and does NOT start it up again. - halt = reuseInFlight(async (): Promise => { - if (this.restartKernelOnClose != null && this.jupyter_kernel != null) { - this.jupyter_kernel.removeListener("closed", this.restartKernelOnClose); - delete this.restartKernelOnClose; - } - this.clear_all_cell_run_state(); - await this.signal("SIGKILL"); - // Wait a little, since SIGKILL has to really happen on backend, - // and server has to respond and change state. - const not_running = (s): boolean => { - if (this._state === "closed") return true; - const t = s.get_one({ type: "settings" }); - return t != null && t.get("backend_state") != "running"; - }; - try { - await this.syncdb.wait(not_running, 30); - // worked -- and also no need to show "kernel got killed" message since this was intentional. - this.set_error(""); - } catch (err) { - // failed - this.set_error(err); - } - }); - - restart = reuseInFlight(async (): Promise => { - await this.halt(); - if (this._state === "closed") return; - this.clear_all_cell_run_state(); - // Actually start it running again (rather than waiting for - // user to do something), since this is called "restart". - try { - await this.set_backend_kernel_info(); // causes kernel to start - } catch (err) { - this.set_error(err); - } - }); - - public shutdown = reuseInFlight(async (): Promise => { - if (this._state === ("closed" as State)) { - return; - } - await this.signal("SIGKILL"); - if (this._state === ("closed" as State)) { - return; - } - this.clear_all_cell_run_state(); - await this.save_asap(); - }); - set_backend_kernel_info = async (): Promise => { if (this._state === "closed" || this.syncdb.is_read_only()) { return; @@ -2550,14 +2258,3 @@ function bounded_integer(n: any, min: any, max: any, def: any) { } return n; } - -function getCompletionGroup(x: string): number { - switch (x[0]) { - case "_": - return 1; - case "%": - return 2; - default: - return 0; - } -} diff --git a/src/packages/jupyter/redux/project-actions.ts b/src/packages/jupyter/redux/project-actions.ts index 8befaacec2..01f1e0d537 100644 --- a/src/packages/jupyter/redux/project-actions.ts +++ b/src/packages/jupyter/redux/project-actions.ts @@ -26,9 +26,13 @@ export class JupyterActions extends JupyterActions0 { capture_output_message = (_opts) => {}; process_comm_message_from_kernel = (_mesg) => {}; - initKernel = () => { + ensureKernelIsReady = () => { if (this.jupyter_kernel != null) { - return; + if (this.jupyter_kernel.isClosed()) { + delete this.jupyter_kernel; + } else { + return; + } } const kernel = this.store.get("kernel"); console.log("initKernel", { kernel, path: this.path }); diff --git a/src/packages/jupyter/redux/run-all-loop.ts b/src/packages/jupyter/redux/run-all-loop.ts deleted file mode 100644 index 472aa42ab3..0000000000 --- a/src/packages/jupyter/redux/run-all-loop.ts +++ /dev/null @@ -1,63 +0,0 @@ -/* - * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. - * License: MS-RSL – see LICENSE.md for details - */ - -import { delay } from "awaiting"; - -import { close } from "@cocalc/util/misc"; -import { JupyterActions } from "./project-actions"; - -export class RunAllLoop { - private actions: JupyterActions; - public interval_s: number; - private closed: boolean = false; - private dbg: Function; - - constructor(actions, interval_s) { - this.actions = actions; - this.interval_s = interval_s; - this.dbg = actions.dbg("RunAllLoop"); - this.dbg(`interval_s=${interval_s}`); - this.loop(); - } - - public set_interval(interval_s: number): void { - if (this.closed) { - throw Error("should not call set_interval if RunAllLoop is closed"); - } - if (this.interval_s == interval_s) return; - this.dbg(`.set_interval: interval_s=${interval_s}`); - this.interval_s = interval_s; - } - - private async loop(): Promise { - this.dbg("starting loop..."); - while (true) { - if (this.closed) break; - try { - this.dbg("loop: restart"); - await this.actions.restart(); - } catch (err) { - this.dbg(`restart failed (will try run-all anyways) - ${err}`); - } - if (this.closed) break; - try { - this.dbg("loop: run_all_cells"); - await this.actions.run_all_cells(true); - } catch (err) { - this.dbg(`run_all_cells failed - ${err}`); - } - if (this.closed) break; - this.dbg(`loop: waiting ${this.interval_s} seconds`); - await delay(this.interval_s * 1000); - } - this.dbg("terminating loop..."); - } - - public close() { - this.dbg("close"); - close(this); - this.closed = true; - } -} diff --git a/src/packages/jupyter/types/project-interface.ts b/src/packages/jupyter/types/project-interface.ts index 009f4b5226..df1898833f 100644 --- a/src/packages/jupyter/types/project-interface.ts +++ b/src/packages/jupyter/types/project-interface.ts @@ -76,7 +76,7 @@ export interface ExecOpts { timeout_ms?: number; } -export type OutputMessage = object; // todo +export type OutputMessage = any; // todo export interface CodeExecutionEmitterInterface extends EventEmitterInterface { emit_output(result: OutputMessage): void; @@ -97,7 +97,9 @@ export interface JupyterKernelInterface extends EventEmitterInterface { name: string | undefined; // name = undefined implies it is not spawnable. It's a notebook with no actual jupyter kernel process. store: any; readonly identity: string; + failedError: string; + isClosed(): boolean; get_state(): string; signal(signal: string): void; close(): void; diff --git a/src/packages/project/conat/api/jupyter.ts b/src/packages/project/conat/api/jupyter.ts new file mode 100644 index 0000000000..8d6260ed71 --- /dev/null +++ b/src/packages/project/conat/api/jupyter.ts @@ -0,0 +1,50 @@ +export { jupyter_strip_notebook as stripNotebook } from "@cocalc/jupyter/nbgrader/jupyter-parse"; +export { jupyter_run_notebook as runNotebook } from "@cocalc/jupyter/nbgrader/jupyter-run"; +export { nbconvert } from "../../jupyter/convert"; +export { formatString } from "../../formatters"; +export { logo as kernelLogo } from "@cocalc/jupyter/kernel/logo"; +export { get_kernel_data as kernels } from "@cocalc/jupyter/kernel/kernel-data"; +export { newFile } from "@cocalc/backend/misc/new-file"; +import { getClient } from "@cocalc/project/client"; +import { project_id } from "@cocalc/project/data"; +import * as control from "@cocalc/jupyter/control"; +import { SandboxedFilesystem } from "@cocalc/backend/files/sandbox"; + +let fs: SandboxedFilesystem | null = null; +export async function start(path: string) { + if (control.isRunning(path)) { + return; + } + fs ??= new SandboxedFilesystem(process.env.HOME ?? "/tmp", { + unsafeMode: true, + }); + await control.start({ project_id, path, client: getClient(), fs }); +} + +// IMPORTANT: run is NOT used directly by the API, but instead by packages/project/conat/jupyter.ts +// It is convenient to have it here so it can call start above, etc. The reason is because +// this returns an async iterator managed using a dedicated socket, and the api is request/response, +// so it can't just be part of the normal api. +export async function run(opts: { + path: string; + cells: { id: string; input: string }[]; +}) { + await start(opts.path); + return await control.run(opts); +} + +export async function stop(path: string) { + await control.stop({ path }); +} + +export async function introspect(opts) { + await start(opts.path); + return await control.introspect(opts); +} + +export async function signal(opts) { + if (!control.isRunning(opts.path)) { + return; + } + await control.signal(opts); +} diff --git a/src/packages/util/jupyter/names.ts b/src/packages/util/jupyter/names.ts index e7d578494d..f16106a862 100644 --- a/src/packages/util/jupyter/names.ts +++ b/src/packages/util/jupyter/names.ts @@ -1,12 +1,23 @@ -import { meta_file } from "@cocalc/util/misc"; +import { meta_file, original_path } from "@cocalc/util/misc"; export const JUPYTER_POSTFIX = "jupyter2"; export const JUPYTER_SYNCDB_EXTENSIONS = `sage-${JUPYTER_POSTFIX}`; -// a.ipynb --> ".a.ipynb.sage-jupyter2" -export function syncdbPath(ipynbPath: string) { - if (!ipynbPath.endsWith(".ipynb")) { - throw Error(`ipynbPath must end with .ipynb but it is "${ipynbPath}"`); +// a.ipynb or .a.ipynb.sage-jupyter2 --> .a.ipynb.sage-jupyter2 +export function syncdbPath(path: string) { + if (path.endsWith(JUPYTER_POSTFIX)) { + return path; } - return meta_file(ipynbPath, JUPYTER_POSTFIX); + if (!path.endsWith(".ipynb")) { + throw Error(`must end with .ipynb but it is "${ipynbPath}"`); + } + return meta_file(path, JUPYTER_POSTFIX); +} + +// a.ipynb or .a.ipynb.sage-jupyter2 --> a.ipynb +export function ipynbPath(path: string) { + if (path.endsWith(".ipynb")) { + return path; + } + return original_path(path); } From 9dc18999af073ff7001486cfe0de4a1a9846d67e Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 2 Aug 2025 01:17:36 +0000 Subject: [PATCH 158/270] frontend terminal -- noticed some cases where it might try to do something while closed, so cleaned that up --- .../terminal-editor/connected-terminal.ts | 32 +++++++++++-------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts b/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts index 34e73fb720..eaf068aac6 100644 --- a/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts +++ b/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts @@ -50,7 +50,6 @@ const MAX_DELAY = 15000; const ENABLE_WEBGL = false; - // ephemeral = faster, less load on servers, but if project and browser all // close, the history is gone... which may be good and less confusing. const EPHEMERAL = true; @@ -60,8 +59,10 @@ interface Path { directory?: string; } +type State = "ready" | "closed"; + export class Terminal { - private state: string = "ready"; + private state: State = "ready"; private actions: Actions | ConnectedTerminalInterface; private account_store: any; private project_actions: ProjectActions; @@ -196,6 +197,8 @@ export class Terminal { // this.terminal_resize = debounce(this.terminal_resize, 2000); } + isClosed = () => (this.state ?? "closed") === "closed"; + private get_xtermjs_options = (): any => { const rendererType = this.rendererType; const settings = this.account_store.get("terminal"); @@ -221,13 +224,13 @@ export class Terminal { }; private assert_not_closed = (): void => { - if (this.state === "closed") { + if (this.isClosed()) { throw Error("BUG -- Terminal is closed."); } }; close = (): void => { - if (this.state === "closed") { + if (this.isClosed()) { return; } this.set_connection_status("disconnected"); @@ -398,10 +401,9 @@ export class Terminal { }; private render = async (data: string): Promise => { - if (data == null) { + if (data == null || this.isClosed()) { return; } - this.assert_not_closed(); this.history += data; if (this.history.length > MAX_HISTORY_LENGTH) { this.history = this.history.slice( @@ -423,7 +425,7 @@ export class Terminal { await delay(0); this.ignoreData--; } - if (this.state == "done") return; + if (this.isClosed()) return; // tell anyone who waited for output coming back about this while (this.render_done.length > 0) { this.render_done.pop()?.(); @@ -453,7 +455,7 @@ export class Terminal { }; touch = async () => { - if (this.state === "closed") return; + if (this.isClosed()) return; if (Date.now() - this.last_active < 70000) { if (this.project_actions.isTabClosed()) { return; @@ -467,7 +469,7 @@ export class Terminal { }; init_keyhandler = (): void => { - if (this.state === "closed") { + if (this.isClosed()) { return; } if (this.keyhandler_initialized) { @@ -578,7 +580,7 @@ export class Terminal { // Stop ignoring terminal data... but ONLY once // the render buffer is also empty. no_ignore = async (): Promise => { - if (this.state === "closed") { + if (this.isClosed()) { return; } const g = (cb) => { @@ -590,7 +592,7 @@ export class Terminal { } // cause render to actually appear now. await delay(0); - if (this.state === "closed") { + if (this.isClosed()) { return; } try { @@ -725,12 +727,14 @@ export class Terminal { update_cwd = debounce( async () => { + if (this.isClosed()) return; let cwd; try { cwd = await this.conn?.api.cwd(); } catch { return; } + if (this.isClosed()) return; if (cwd != null) { this.actions.set_terminal_cwd(this.id, cwd); } @@ -775,14 +779,14 @@ export class Terminal { } focus(): void { - if (this.state === "closed") { + if (this.isClosed()) { return; } this.terminal.focus(); } refresh(): void { - if (this.state === "closed") { + if (this.isClosed()) { return; } this.terminal.refresh(0, this.terminal.rows - 1); @@ -792,7 +796,7 @@ export class Terminal { try { await open_init_file(this.actions._get_project_actions(), this.termPath); } catch (err) { - if (this.state === "closed") { + if (this.isClosed()) { return; } this.actions.set_error(`Problem opening init file -- ${err}`); From 73a9e318a8321b2950f1e7a93c0d9aa68bef4d7f Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 2 Aug 2025 02:10:43 +0000 Subject: [PATCH 159/270] jupyter: fix situation where errors during execution not properly reported to browser --- src/packages/conat/project/jupyter/run-code.ts | 17 +++++++++++------ .../frontend/jupyter/browser-actions.ts | 2 ++ src/packages/jupyter/execute/execute-code.ts | 8 +++++++- 3 files changed, 20 insertions(+), 7 deletions(-) diff --git a/src/packages/conat/project/jupyter/run-code.ts b/src/packages/conat/project/jupyter/run-code.ts index ef78773bf9..3a0dc32919 100644 --- a/src/packages/conat/project/jupyter/run-code.ts +++ b/src/packages/conat/project/jupyter/run-code.ts @@ -120,13 +120,16 @@ export function jupyterServer({ noHalt, }); } catch (err) { - //console.log(err); - logger.debug("server: failed response -- ", err); + logger.debug("server: failed to handle execute request -- ", err); if (socket.state != "closed") { try { - socket.write(null, { headers: { error: `${err}` } }); - } catch { + logger.debug("sending to client: ", { + headers: { error: `${err}` }, + }); + socket.write(null, { headers: { foo: "bar", error: `${err}` } }); + } catch (err) { // an error trying to report an error shouldn't crash everything + logger.debug("WARNING: unable to send error to client", err); } } } @@ -195,12 +198,14 @@ async function handleRequest({ throttle.write(mesg); } } + // no errors happened, so close up and flush and + // remaining data immediately: handler?.done(); - } finally { - if (socket.state != "closed" && !unhandledClientWriteError) { + if (socket.state != "closed") { throttle.flush(); socket.write(null); } + } finally { throttle.close(); } } diff --git a/src/packages/frontend/jupyter/browser-actions.ts b/src/packages/frontend/jupyter/browser-actions.ts index 7601684e60..4c5dd69d10 100644 --- a/src/packages/frontend/jupyter/browser-actions.ts +++ b/src/packages/frontend/jupyter/browser-actions.ts @@ -1648,6 +1648,8 @@ export class JupyterActions extends JupyterActions0 { }, 1000); } catch (err) { console.warn("runCells", err); + this.clearRunQueue(); + this.set_error(err); } finally { if (this.isClosed()) return; this.runningNow = false; diff --git a/src/packages/jupyter/execute/execute-code.ts b/src/packages/jupyter/execute/execute-code.ts index 73744b34a7..df396ebbe8 100644 --- a/src/packages/jupyter/execute/execute-code.ts +++ b/src/packages/jupyter/execute/execute-code.ts @@ -150,7 +150,13 @@ export class CodeExecutionEmitter }; throw_error = (err): void => { - this.emit("error", err); + if (this._iter != null) { + // using the iter, so we can use that to report the error + this._iter.throw(err); + } else { + // no iter so make error known via error event + this.emit("error", err); + } this.close(); }; From 3f029952400632860c399d6ee132af9560c0ee32 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 2 Aug 2025 02:43:44 +0000 Subject: [PATCH 160/270] jupyter: tab completion --- src/packages/conat/project/api/jupyter.ts | 7 + src/packages/frontend/client/project.ts | 3 +- .../frontend/jupyter/browser-actions.ts | 124 +++++++++--------- src/packages/jupyter/control.ts | 9 ++ src/packages/jupyter/redux/actions.ts | 1 - src/packages/project/conat/api/index.ts | 4 +- src/packages/project/conat/api/jupyter.ts | 5 + 7 files changed, 87 insertions(+), 66 deletions(-) diff --git a/src/packages/conat/project/api/jupyter.ts b/src/packages/conat/project/api/jupyter.ts index 3defeb5b57..1ac6b39ed3 100644 --- a/src/packages/conat/project/api/jupyter.ts +++ b/src/packages/conat/project/api/jupyter.ts @@ -11,6 +11,7 @@ export const jupyter = { kernelLogo: true, kernels: true, introspect: true, + complete: true, signal: true, }; @@ -41,5 +42,11 @@ export interface Jupyter { detail_level: 0 | 1; }) => Promise; + complete: (opts: { + path: string; + code: string; + cursor_pos: number; + }) => Promise; + signal: (opts: { path: string; signal: string }) => Promise; } diff --git a/src/packages/frontend/client/project.ts b/src/packages/frontend/client/project.ts index 8938d29aa4..55ee2b9d80 100644 --- a/src/packages/frontend/client/project.ts +++ b/src/packages/frontend/client/project.ts @@ -41,6 +41,7 @@ import { WebappClient } from "./client"; import { throttle } from "lodash"; import { writeFile, type WriteFileOptions } from "@cocalc/conat/files/write"; import { readFile, type ReadFileOptions } from "@cocalc/conat/files/read"; +import { type ProjectApi } from "@cocalc/conat/project/api"; export class ProjectClient { private client: WebappClient; @@ -50,7 +51,7 @@ export class ProjectClient { this.client = client; } - conatApi = (project_id: string, compute_server_id = 0) => { + conatApi = (project_id: string, compute_server_id = 0): ProjectApi => { return this.client.conat_client.projectApi({ project_id, compute_server_id, diff --git a/src/packages/frontend/jupyter/browser-actions.ts b/src/packages/frontend/jupyter/browser-actions.ts index 4c5dd69d10..9f4af24b10 100644 --- a/src/packages/frontend/jupyter/browser-actions.ts +++ b/src/packages/frontend/jupyter/browser-actions.ts @@ -82,8 +82,7 @@ export class JupyterActions extends JupyterActions0 { private account_change_editor_settings: any; private update_keyboard_shortcuts: any; public syncdbPath: string; - private last_cursor_move_time: Date = new Date(0); - private _introspect_request?: any; + private lastCursorMoveTime: number = 0; protected init2(): void { this.syncdbPath = syncdbPath(this.path); @@ -1454,13 +1453,13 @@ export class JupyterActions extends JupyterActions0 { // tells them to open this jupyter notebook, so it can provide the compute // functionality. - conatApi = async () => { + private jupyterApi = async () => { const compute_server_id = await this.getComputeServerId(); const api = webapp_client.project_client.conatApi( this.project_id, compute_server_id, ); - return api; + return api.jupyter; }; initBackend = async () => { @@ -1470,8 +1469,8 @@ export class JupyterActions extends JupyterActions0 { return true; } try { - const api = await this.conatApi(); - await api.jupyter.start(this.syncdbPath); + const api = await this.jupyterApi(); + await api.start(this.syncdbPath); return true; } catch (err) { console.log("failed to initialize ", this.path, err); @@ -1483,8 +1482,8 @@ export class JupyterActions extends JupyterActions0 { }; stopBackend = async () => { - const api = await this.conatApi(); - await api.jupyter.stop(this.syncdbPath); + const api = await this.jupyterApi(); + await api.stop(this.syncdbPath); }; getOutputHandler = (cell) => { @@ -1684,14 +1683,14 @@ export class JupyterActions extends JupyterActions0 { ); }; + private introspectRequest: number = 0; introspect = async ( code: string, detail_level: 0 | 1, cursor_pos?: number, ): Promise | undefined> => { - const req = (this._introspect_request = - (this._introspect_request != null ? this._introspect_request : 0) + 1); - + this.introspectRequest++; + const req = this.introspectRequest; if (cursor_pos == null) { cursor_pos = code.length; } @@ -1699,8 +1698,8 @@ export class JupyterActions extends JupyterActions0 { let introspect; try { - const api = await this.conatApi(); - introspect = await api.jupyter.introspect({ + const api = await this.jupyterApi(); + introspect = await api.introspect({ path: this.path, code, cursor_pos, @@ -1713,55 +1712,56 @@ export class JupyterActions extends JupyterActions0 { } catch (err) { introspect = { error: err }; } - if (this._introspect_request > req) return; - const i = fromJS(introspect); - this.getFrameActions()?.setState({ - introspect: i, - }); + if (this.introspectRequest > req) return; + this.getFrameActions()?.setState({ introspect }); return introspect; // convenient / useful, e.g., for use by whiteboard. }; clear_introspect = (): void => { - this._introspect_request = - (this._introspect_request != null ? this._introspect_request : 0) + 1; + this.introspectRequest = + (this.introspectRequest != null ? this.introspectRequest : 0) + 1; this.getFrameActions()?.setState({ introspect: undefined }); }; - // Attempt to fetch completions for give code and cursor_pos - // If successful, the completions are put in store.get('completions') and looks like - // this (as an immutable map): - // cursor_end : 2 - // cursor_start : 0 - // matches : ['the', 'completions', ...] - // status : "ok" - // code : code - // cursor_pos : cursor_pos - // - // If not successful, result is: - // status : "error" - // code : code - // cursor_pos : cursor_pos - // error : 'an error message' - // - // Only the most recent fetch has any impact, and calling - // clear_complete() ensures any fetch made before that - // is ignored. + /* + complete: + + Attempt to fetch completions for give code and cursor_pos + If successful, the completions are put in store.get('completions') and looks + like this (as an immutable map): + cursor_end : 2 + cursor_start : 0 + matches : ['the', 'completions', ...] + status : "ok" + code : code + cursor_pos : cursor_pos + + If not successful, result is: + status : "error" + code : code + cursor_pos : cursor_pos + error : 'an error message' + + Only the most recent fetch has any impact, and calling + clear_complete() ensures any fetch made before that + is ignored. // Returns true if a dialog with options appears, and false otherwise. + */ + private completeRequest = 0; complete = async ( code: string, pos?: { line: number; ch: number } | number, id?: string, offset?: any, ): Promise => { - let cursor_pos; - const req = (this._complete_request = - (this._complete_request != null ? this._complete_request : 0) + 1); - + this.completeRequest++; + const req = this.completeRequest; this.setState({ complete: undefined }); // pos can be either a {line:?, ch:?} object as in codemirror, // or a number. + let cursor_pos; if (pos == null || typeof pos == "number") { cursor_pos = pos; } else { @@ -1769,30 +1769,28 @@ export class JupyterActions extends JupyterActions0 { } cursor_pos = js_idx_to_char_idx(cursor_pos, code); - const start = new Date(); + const start = Date.now(); let complete; try { - complete = await this.api().complete({ + const api = await this.jupyterApi(); + complete = await api.complete({ + path: this.path, code, cursor_pos, }); } catch (err) { - if (this._complete_request > req) return false; + if (this.completeRequest > req) return false; this.setState({ complete: { error: err } }); - // no op for now... throw Error(`ignore -- ${err}`); - //return false; } - if (this.last_cursor_move_time >= start) { + if (this.lastCursorMoveTime >= start) { // see https://github.com/sagemathinc/cocalc/issues/3611 throw Error("ignore"); - //return false; } - if (this._complete_request > req) { + if (this.completeRequest > req) { // future completion or clear happened; so ignore this result. throw Error("ignore"); - //return false; } if (complete.status !== "ok") { @@ -1845,8 +1843,8 @@ export class JupyterActions extends JupyterActions0 { }; clear_complete = (): void => { - this._complete_request = - (this._complete_request != null ? this._complete_request : 0) + 1; + this.completeRequest = + (this.completeRequest != null ? this.completeRequest : 0) + 1; this.setState({ complete: undefined }); }; @@ -1872,12 +1870,12 @@ export class JupyterActions extends JupyterActions0 { } } - complete_cell(id: string, base: string, new_input: string): void { + complete_cell = (id: string, base: string, new_input: string): void => { this.merge_cell_input(id, base, new_input); - } + }; - public set_cursor_locs(locs: any[] = [], side_effect: boolean = false): void { - this.last_cursor_move_time = new Date(); + set_cursor_locs = (locs: any[] = [], side_effect: boolean = false): void => { + this.lastCursorMoveTime = Date.now(); if (this.syncdb == null) { // syncdb not always set -- https://github.com/sagemathinc/cocalc/issues/2107 return; @@ -1888,16 +1886,16 @@ export class JupyterActions extends JupyterActions0 { } this._cursor_locs = locs; // remember our own cursors for splitting cell this.syncdb.set_cursor_locs(locs, side_effect); - } + }; - async signal(signal = "SIGINT"): Promise { - const api = await this.conatApi(); + signal = async (signal = "SIGINT"): Promise => { + const api = await this.jupyterApi(); try { - await api.jupyter.signal({ path: this.path, signal }); + await api.signal({ path: this.path, signal }); } catch (err) { this.set_error(err); } - } + }; // Kill the running kernel and does NOT start it up again. halt = reuseInFlight(async (): Promise => { diff --git a/src/packages/jupyter/control.ts b/src/packages/jupyter/control.ts index 8efeddd0ac..78674b6edc 100644 --- a/src/packages/jupyter/control.ts +++ b/src/packages/jupyter/control.ts @@ -186,6 +186,15 @@ export async function introspect(opts: { return await kernel.introspect(opts); } +export async function complete(opts: { + path: string; + code: string; + cursor_pos: number; +}) { + const kernel = getKernel(opts.path); + return await kernel.complete(opts); +} + export async function signal(opts: { path: string; signal: string }) { const kernel = getKernel(opts.path); await kernel.signal(opts.signal); diff --git a/src/packages/jupyter/redux/actions.ts b/src/packages/jupyter/redux/actions.ts index 872f472826..2022d935eb 100644 --- a/src/packages/jupyter/redux/actions.ts +++ b/src/packages/jupyter/redux/actions.ts @@ -73,7 +73,6 @@ export class JupyterActions extends Actions { protected restartKernelOnClose?: (...args: any[]) => void; public asyncBlobStore: AKV; - public _complete_request?: number; public store: JupyterStore; public syncdb: SyncDB; private labels?: { diff --git a/src/packages/project/conat/api/index.ts b/src/packages/project/conat/api/index.ts index 7133583609..a6b32a86ce 100644 --- a/src/packages/project/conat/api/index.ts +++ b/src/packages/project/conat/api/index.ts @@ -166,7 +166,9 @@ async function getResponse({ name, args }) { const [group, functionName] = name.split("."); const f = projectApi[group]?.[functionName]; if (f == null) { - throw Error(`unknown function '${name}'`); + throw Error( + `unknown function '${name}' -- available functions are ${JSON.stringify(Object.keys(projectApi[group]))}`, + ); } return await f(...args); } diff --git a/src/packages/project/conat/api/jupyter.ts b/src/packages/project/conat/api/jupyter.ts index 8d6260ed71..0afe0a9a93 100644 --- a/src/packages/project/conat/api/jupyter.ts +++ b/src/packages/project/conat/api/jupyter.ts @@ -42,6 +42,11 @@ export async function introspect(opts) { return await control.introspect(opts); } +export async function complete(opts) { + await start(opts.path); + return await control.complete(opts); +} + export async function signal(opts) { if (!control.isRunning(opts.path)) { return; From 9e7e214b7778c944c038c0ff872b12346ec3f414 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 2 Aug 2025 03:03:12 +0000 Subject: [PATCH 161/270] add jupyter api getConnectionFile and use that instead of redux/sync --- src/packages/conat/project/api/jupyter.ts | 3 +++ .../frame-editors/jupyter-editor/actions.ts | 24 ++++--------------- .../frontend/jupyter/browser-actions.ts | 9 +++++++ src/packages/jupyter/control.ts | 10 ++++++++ src/packages/jupyter/kernel/kernel.ts | 2 +- src/packages/jupyter/redux/actions.ts | 1 - src/packages/jupyter/redux/store.ts | 1 - .../jupyter/types/project-interface.ts | 2 +- src/packages/project/conat/api/jupyter.ts | 5 ++++ 9 files changed, 33 insertions(+), 24 deletions(-) diff --git a/src/packages/conat/project/api/jupyter.ts b/src/packages/conat/project/api/jupyter.ts index 1ac6b39ed3..22f7e46211 100644 --- a/src/packages/conat/project/api/jupyter.ts +++ b/src/packages/conat/project/api/jupyter.ts @@ -13,6 +13,7 @@ export const jupyter = { introspect: true, complete: true, signal: true, + getConnectionFile: true, }; // In the functions below path can be either the .ipynb or the .sage-jupyter2 path, and @@ -48,5 +49,7 @@ export interface Jupyter { cursor_pos: number; }) => Promise; + getConnectionFile: (opts: { path: string }) => Promise; + signal: (opts: { path: string; signal: string }) => Promise; } diff --git a/src/packages/frontend/frame-editors/jupyter-editor/actions.ts b/src/packages/frontend/frame-editors/jupyter-editor/actions.ts index e15e797f68..ed8f115c65 100644 --- a/src/packages/frontend/frame-editors/jupyter-editor/actions.ts +++ b/src/packages/frontend/frame-editors/jupyter-editor/actions.ts @@ -117,26 +117,12 @@ export class JupyterEditorActions extends BaseActions { private watchJupyterStore = (): void => { const store = this.jupyter_actions.store; - let connection_file = store.get("connection_file"); store.on("change", () => { // sync read only state -- source of true is jupyter_actions.store.get('read_only') const read_only = store.get("read_only"); if (read_only != this.store.get("read_only")) { this.setState({ read_only }); } - // sync connection file - const c = store.get("connection_file"); - if (c == connection_file) { - return; - } - connection_file = c; - const id = this._get_most_recent_shell_id("jupyter"); - if (id == null) { - // There is no Jupyter console open right now... - return; - } - // This will update the connection file - this.shell(id, true); }); }; @@ -295,14 +281,12 @@ export class JupyterEditorActions extends BaseActions { } protected async get_shell_spec( - id: string, - ): Promise { - id = id; // not used - const connection_file = this.jupyter_actions.store.get("connection_file"); - if (connection_file == null) return; + _id: string, + ): Promise<{ command: string; args: string[] }> { + const connectionFile = await this.jupyter_actions.getConnectionFile(); return { command: "jupyter", - args: ["console", "--existing", connection_file], + args: ["console", "--existing", connectionFile], }; } diff --git a/src/packages/frontend/jupyter/browser-actions.ts b/src/packages/frontend/jupyter/browser-actions.ts index 9f4af24b10..01ddede80b 100644 --- a/src/packages/frontend/jupyter/browser-actions.ts +++ b/src/packages/frontend/jupyter/browser-actions.ts @@ -1336,6 +1336,7 @@ export class JupyterActions extends JupyterActions0 { }; private saveIpynb = async () => { + if(this.isClosed()) return; const ipynb = await this.toIpynb(); const serialize = JSON.stringify(ipynb, undefined, 2); this.syncdb.fs.writeFile(this.path, serialize); @@ -1639,6 +1640,9 @@ export class JupyterActions extends JupyterActions0 { } } handler?.done(); + if (this.isClosed()) { + return; + } this.syncdb.save(); setTimeout(() => { if (!this.isClosed()) { @@ -1934,6 +1938,11 @@ export class JupyterActions extends JupyterActions0 { if (this.is_closed()) return; this.clear_all_cell_run_state(); }); + + getConnectionFile = async (): Promise => { + const api = await this.jupyterApi(); + return await api.getConnectionFile({ path: this.path }); + }; } function getCompletionGroup(x: string): number { diff --git a/src/packages/jupyter/control.ts b/src/packages/jupyter/control.ts index 78674b6edc..2c7bc900cf 100644 --- a/src/packages/jupyter/control.ts +++ b/src/packages/jupyter/control.ts @@ -195,6 +195,16 @@ export async function complete(opts: { return await kernel.complete(opts); } +export async function getConnectionFile(opts: { path }) { + const kernel = getKernel(opts.path); + await kernel.ensure_running(); + const c = kernel.getConnectionFile(); + if (c == null) { + throw Error("unable to start kernel"); + } + return c; +} + export async function signal(opts: { path: string; signal: string }) { const kernel = getKernel(opts.path); await kernel.signal(opts.signal); diff --git a/src/packages/jupyter/kernel/kernel.ts b/src/packages/jupyter/kernel/kernel.ts index d41b1253a1..b83594c80e 100644 --- a/src/packages/jupyter/kernel/kernel.ts +++ b/src/packages/jupyter/kernel/kernel.ts @@ -404,7 +404,7 @@ export class JupyterKernel return this._kernel; }; - get_connection_file = (): string | undefined => { + getConnectionFile = (): string | undefined => { return this._kernel?.connectionFile; }; diff --git a/src/packages/jupyter/redux/actions.ts b/src/packages/jupyter/redux/actions.ts index 2022d935eb..3cbc2c992d 100644 --- a/src/packages/jupyter/redux/actions.ts +++ b/src/packages/jupyter/redux/actions.ts @@ -641,7 +641,6 @@ export class JupyterActions extends Actions { kernel_state: record.get("kernel_state"), kernel_error: record.get("kernel_error"), metadata: record.get("metadata"), // extra custom user-specified metadata - connection_file: record.get("connection_file") ?? "", max_output_length: bounded_integer( record.get("max_output_length"), 100, diff --git a/src/packages/jupyter/redux/store.ts b/src/packages/jupyter/redux/store.ts index a334be7de2..1109054ecd 100644 --- a/src/packages/jupyter/redux/store.ts +++ b/src/packages/jupyter/redux/store.ts @@ -60,7 +60,6 @@ export interface JupyterStoreState { cm_options: any; complete: any; confirm_dialog: any; - connection_file?: string; contents?: List>; // optional global contents info (about sections, problems, etc.) default_kernel?: string; directory: string; diff --git a/src/packages/jupyter/types/project-interface.ts b/src/packages/jupyter/types/project-interface.ts index df1898833f..986d16cd28 100644 --- a/src/packages/jupyter/types/project-interface.ts +++ b/src/packages/jupyter/types/project-interface.ts @@ -127,7 +127,7 @@ export interface JupyterKernelInterface extends EventEmitterInterface { buffers?: any[]; buffers64?: any[]; }): void; - get_connection_file(): string | undefined; + getConnectionFile(): string | undefined; _execute_code_queue: CodeExecutionEmitterInterface[]; clear_execute_code_queue(): void; diff --git a/src/packages/project/conat/api/jupyter.ts b/src/packages/project/conat/api/jupyter.ts index 0afe0a9a93..849d81071f 100644 --- a/src/packages/project/conat/api/jupyter.ts +++ b/src/packages/project/conat/api/jupyter.ts @@ -47,6 +47,11 @@ export async function complete(opts) { return await control.complete(opts); } +export async function getConnectionFile(opts) { + await start(opts.path); + return await control.getConnectionFile(opts); +} + export async function signal(opts) { if (!control.isRunning(opts.path)) { return; From ca84275fb5b61e26998180accf9101357f01dc7a Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 2 Aug 2025 04:34:45 +0000 Subject: [PATCH 162/270] jupyter: ensure positions are unique --- .../frontend/jupyter/browser-actions.ts | 19 +++++++++------- src/packages/jupyter/redux/actions.ts | 22 ++++++++----------- src/packages/jupyter/util/cell-utils.ts | 14 ++++++------ 3 files changed, 27 insertions(+), 28 deletions(-) diff --git a/src/packages/frontend/jupyter/browser-actions.ts b/src/packages/frontend/jupyter/browser-actions.ts index 01ddede80b..5570db0696 100644 --- a/src/packages/frontend/jupyter/browser-actions.ts +++ b/src/packages/frontend/jupyter/browser-actions.ts @@ -86,7 +86,6 @@ export class JupyterActions extends JupyterActions0 { protected init2(): void { this.syncdbPath = syncdbPath(this.path); - this.update_contents = debounce(this.update_contents.bind(this), 2000); this.setState({ toolbar: !this.get_local_storage("hide_toolbar"), cell_toolbar: this.get_local_storage("cell_toolbar"), @@ -116,13 +115,16 @@ export class JupyterActions extends JupyterActions0 { this.syncdb.on("connected", this.sync_read_only); // first update - this.syncdb.once("change", this.updateContentsNow); - this.syncdb.once("change", this.updateRunProgress); + this.syncdb.once("change", () => { + this.updateContentsNow(); + this.updateRunProgress(); + this.ensurePositionsAreUnique(); + }); this.syncdb.on("change", () => { // And activity indicator this.activity(); - // Update table of contents + // Update table of contents -- this is debounced this.update_contents(); // run progress this.updateRunProgress(); @@ -319,7 +321,7 @@ export class JupyterActions extends JupyterActions0 { } public async close(): Promise { - if (this.is_closed()) return; + if (this.isClosed()) return; this.jupyterClient?.close(); await super.close(); } @@ -987,9 +989,10 @@ export class JupyterActions extends JupyterActions0 { this.setState({ contents }); }; - public update_contents(): void { + update_contents = debounce(() => { + if (this.isClosed()) return; this.updateContentsNow(); - } + }, 2000); protected __syncdb_change_post_hook(_doInit: boolean) { if (this._state === "init") { @@ -1336,7 +1339,7 @@ export class JupyterActions extends JupyterActions0 { }; private saveIpynb = async () => { - if(this.isClosed()) return; + if (this.isClosed()) return; const ipynb = await this.toIpynb(); const serialize = JSON.stringify(ipynb, undefined, 2); this.syncdb.fs.writeFile(this.path, serialize); diff --git a/src/packages/jupyter/redux/actions.ts b/src/packages/jupyter/redux/actions.ts index 3cbc2c992d..b66bddb166 100644 --- a/src/packages/jupyter/redux/actions.ts +++ b/src/packages/jupyter/redux/actions.ts @@ -1300,7 +1300,7 @@ export class JupyterActions extends Actions { cell = cell.set("pos", i); this._set(cell, false); }); - this.ensure_positions_are_unique(); + this.ensurePositionsAreUnique(); this._sync(); return; } @@ -1731,7 +1731,7 @@ export class JupyterActions extends Actions { this._state = "ready"; }; - public set_cell_slide(id: string, value: any): void { + set_cell_slide = (id: string, value: any) => { if (!value) { value = null; // delete } @@ -1743,15 +1743,11 @@ export class JupyterActions extends Actions { id, slide: value, }); - } + }; - public ensure_positions_are_unique(): void { - if (this._state != "ready" || this.store == null) { - // because of debouncing, this ensure_positions_are_unique can - // be called after jupyter actions are closed. - return; - } - const changes = cell_utils.ensure_positions_are_unique( + ensurePositionsAreUnique = () => { + if (this.isClosed()) return; + const changes = cell_utils.ensurePositionsAreUnique( this.store.get("cells"), ); if (changes != null) { @@ -1761,9 +1757,9 @@ export class JupyterActions extends Actions { } } this._sync(); - } + }; - public set_default_kernel(kernel?: string): void { + set_default_kernel = (kernel?: string) => { if (kernel == null || kernel === "") return; // doesn't make sense for project (right now at least) if (this.is_project || this.is_compute_server) return; @@ -1780,7 +1776,7 @@ export class JupyterActions extends Actions { (this.redux.getTable("account") as any).set({ editor_settings: { jupyter: cur }, }); - } + }; edit_attachments = (id: string): void => { this.setState({ edit_attachments: id }); diff --git a/src/packages/jupyter/util/cell-utils.ts b/src/packages/jupyter/util/cell-utils.ts index ecc59eef87..e6d68ac9b3 100644 --- a/src/packages/jupyter/util/cell-utils.ts +++ b/src/packages/jupyter/util/cell-utils.ts @@ -13,7 +13,7 @@ import { field_cmp, len } from "@cocalc/util/misc"; export function positions_between( before_pos: number | undefined, after_pos: number | undefined, - num: number + num: number, ) { // Return an array of num equally spaced positions starting after // before_pos and ending before after_pos, so @@ -66,22 +66,22 @@ export function sorted_cell_list(cells: Map): List { .toList(); } -export function ensure_positions_are_unique(cells?: Map) { +export function ensurePositionsAreUnique(cells?: Map) { // Verify that pos's of cells are distinct. If not // return map from id's to new unique positions. if (cells == null) { return; } - const v: any = {}; + const v = new Set(); let all_unique = true; cells.forEach((cell) => { const pos = cell.get("pos"); - if (pos == null || v[pos]) { + if (pos == null || v.has(pos)) { // dup! (or not defined) all_unique = false; return false; } - v[pos] = true; + v.add(pos); }); if (all_unique) { return; @@ -99,7 +99,7 @@ export function new_cell_pos( cells: Map, cell_list: List, cur_id: string, - delta: -1 | 1 + delta: -1 | 1, ): number { /* Returns pos for a new cell whose position @@ -145,7 +145,7 @@ export function new_cell_pos( export function move_selected_cells( v?: string[], selected?: { [id: string]: true }, - delta?: number + delta?: number, ) { /* - v = ordered js array of all cell id's From ccc1c25d64332814cad9fb27870bf85f5294041e Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 2 Aug 2025 05:12:15 +0000 Subject: [PATCH 163/270] file explorer -- fix bug when hitting enter in action box --- .../frontend/project/explorer/action-box.tsx | 25 +++++++++++-------- .../project/explorer/create-archive.tsx | 5 ++-- .../frontend/project/explorer/download.tsx | 6 ++--- .../frontend/project/explorer/explorer.tsx | 12 +++++++-- .../frontend/project/explorer/rename-file.tsx | 6 ++--- src/packages/frontend/project_actions.ts | 2 +- 6 files changed, 34 insertions(+), 22 deletions(-) diff --git a/src/packages/frontend/project/explorer/action-box.tsx b/src/packages/frontend/project/explorer/action-box.tsx index 110bdf2e5b..14aec4cfb6 100644 --- a/src/packages/frontend/project/explorer/action-box.tsx +++ b/src/packages/frontend/project/explorer/action-box.tsx @@ -84,8 +84,15 @@ export function ActionBox({ compute_server_id ?? 0, ); + function clear() { + actions.set_all_files_unchecked(); + setTimeout(() => { + actions.set_file_action(); + }, 1); + } + function cancel_action(): void { - actions.set_file_action(); + clear(); } function action_key(e): void { @@ -121,8 +128,7 @@ export function ActionBox({ actions.close_tab(path); } actions.delete_files({ paths }); - actions.set_file_action(); - actions.set_all_files_unchecked(); + clear(); } function render_delete_warning() { @@ -195,8 +201,7 @@ export function ActionBox({ src: checked_files.toArray(), dest: move_destination, }); - actions.set_file_action(); - actions.set_all_files_unchecked(); + clear(); } function valid_move_input(): boolean { @@ -340,7 +345,7 @@ export function ActionBox({ } } - actions.set_file_action(); + clear(); } function valid_copy_input(): boolean { @@ -561,17 +566,17 @@ export function ActionBox({ function render_action_box(action: FileAction) { switch (action) { case "compress": - return ; + return ; case "copy": return render_copy(); case "delete": return render_delete(); case "download": - return ; + return ; case "rename": - return ; + return ; case "duplicate": - return ; + return ; case "move": return render_move(); case "share": diff --git a/src/packages/frontend/project/explorer/create-archive.tsx b/src/packages/frontend/project/explorer/create-archive.tsx index b599de0761..1ae72fb230 100644 --- a/src/packages/frontend/project/explorer/create-archive.tsx +++ b/src/packages/frontend/project/explorer/create-archive.tsx @@ -9,7 +9,7 @@ import { useProjectContext } from "@cocalc/frontend/project/context"; import { path_split, plural } from "@cocalc/util/misc"; import CheckedFiles from "./checked-files"; -export default function CreateArchive({}) { +export default function CreateArchive({ clear }) { const intl = useIntl(); const inputRef = useRef(null); const { actions } = useProjectContext(); @@ -53,8 +53,7 @@ export default function CreateArchive({}) { setLoading(false); } - actions.set_all_files_unchecked(); - actions.set_file_action(); + clear(); }; if (actions == null) { diff --git a/src/packages/frontend/project/explorer/download.tsx b/src/packages/frontend/project/explorer/download.tsx index 0bf5c11a42..2c213e2e1f 100644 --- a/src/packages/frontend/project/explorer/download.tsx +++ b/src/packages/frontend/project/explorer/download.tsx @@ -12,7 +12,7 @@ import { path_split, path_to_file, plural } from "@cocalc/util/misc"; import { PRE_STYLE } from "./action-box"; import CheckedFiles from "./checked-files"; -export default function Download() { +export default function Download({ clear }) { const intl = useIntl(); const inputRef = useRef(null); const { actions } = useProjectContext(); @@ -98,8 +98,8 @@ export default function Download() { } finally { setLoading(false); } - actions.set_all_files_unchecked(); - actions.set_file_action(); + + clear(); }; if (actions == null) { diff --git a/src/packages/frontend/project/explorer/explorer.tsx b/src/packages/frontend/project/explorer/explorer.tsx index 4694e6ac5d..83b65339d0 100644 --- a/src/packages/frontend/project/explorer/explorer.tsx +++ b/src/packages/frontend/project/explorer/explorer.tsx @@ -160,10 +160,12 @@ export function Explorer() { }, [listing, current_path, strippedPublicPaths]); useEffect(() => { - if (listing == null) { + if (listing == null || file_action) { return; } + console.log("enabling key handler"); const handle_files_key_down = (e): void => { + console.log("key down", e.key); if (actions == null) { return; } @@ -179,6 +181,11 @@ export function Explorer() { } else if (e.key == "ArrowDown") { actions.increment_selected_file_index(); } else if (e.key == "Enter") { + console.log("Enter key", checked_files.size, file_action); + if (checked_files.size > 0 && file_action != undefined) { + // using the action box. + return; + } if (file_search.startsWith("/")) { // running a terminal command return; @@ -211,10 +218,11 @@ export function Explorer() { $(window).on("keydown", handle_files_key_down); $(window).on("keyup", handle_files_key_up); return () => { + console.log("disabling key handler"); $(window).off("keydown", handle_files_key_down); $(window).off("keyup", handle_files_key_up); }; - }, [project_id, current_path, listing]); + }, [project_id, current_path, listing, file_action]); if (listingError) { return ; diff --git a/src/packages/frontend/project/explorer/rename-file.tsx b/src/packages/frontend/project/explorer/rename-file.tsx index 327427126f..4bcc06bcbb 100644 --- a/src/packages/frontend/project/explorer/rename-file.tsx +++ b/src/packages/frontend/project/explorer/rename-file.tsx @@ -18,9 +18,10 @@ const MAX_FILENAME_LENGTH = 4095; interface Props { duplicate?: boolean; + clear: () => void; } -export default function RenameFile({ duplicate }: Props) { +export default function RenameFile({ duplicate, clear }: Props) { const intl = useIntl(); const inputRef = useRef(null); const { actions } = useProjectContext(); @@ -85,8 +86,7 @@ export default function RenameFile({ duplicate }: Props) { } finally { setLoading(false); } - actions.set_all_files_unchecked(); - actions.set_file_action(); + clear(); }; if (actions == null) { diff --git a/src/packages/frontend/project_actions.ts b/src/packages/frontend/project_actions.ts index d16ddbf3f6..6e18614b23 100644 --- a/src/packages/frontend/project_actions.ts +++ b/src/packages/frontend/project_actions.ts @@ -1722,7 +1722,6 @@ export class ProjectActions extends Actions { } this.setState({ checked_files: store.get("checked_files").clear(), - file_action: undefined, }); } @@ -1747,6 +1746,7 @@ export class ProjectActions extends Actions { }; set_file_action = (action?: FileAction): void => { + console.trace("set_file_action", action); const store = this.get_store(); if (store == null) { return; From bcc49258330aa0feb2e77e3c9ff72eb5e4a3d5d9 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 2 Aug 2025 15:01:21 +0000 Subject: [PATCH 164/270] remove debug messages --- src/packages/frontend/project/explorer/explorer.tsx | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/packages/frontend/project/explorer/explorer.tsx b/src/packages/frontend/project/explorer/explorer.tsx index 83b65339d0..2c2968b0eb 100644 --- a/src/packages/frontend/project/explorer/explorer.tsx +++ b/src/packages/frontend/project/explorer/explorer.tsx @@ -163,9 +163,7 @@ export function Explorer() { if (listing == null || file_action) { return; } - console.log("enabling key handler"); const handle_files_key_down = (e): void => { - console.log("key down", e.key); if (actions == null) { return; } @@ -181,7 +179,6 @@ export function Explorer() { } else if (e.key == "ArrowDown") { actions.increment_selected_file_index(); } else if (e.key == "Enter") { - console.log("Enter key", checked_files.size, file_action); if (checked_files.size > 0 && file_action != undefined) { // using the action box. return; @@ -218,7 +215,6 @@ export function Explorer() { $(window).on("keydown", handle_files_key_down); $(window).on("keyup", handle_files_key_up); return () => { - console.log("disabling key handler"); $(window).off("keydown", handle_files_key_down); $(window).off("keyup", handle_files_key_up); }; From 2f8f7f074a5ccfbf41caae8c65a7f15ec1184ad4 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 2 Aug 2025 16:12:57 +0000 Subject: [PATCH 165/270] add ripgrep support to the fs module (not used by frontend yet) --- src/packages/backend/files/sandbox/index.ts | 59 ++- src/packages/backend/files/sandbox/ripgrep.ts | 344 ++++++++++++++++++ src/packages/backend/package.json | 4 +- src/packages/conat/files/fs.ts | 23 ++ src/packages/package.json | 4 +- src/packages/pnpm-lock.yaml | 39 ++ 6 files changed, 461 insertions(+), 12 deletions(-) create mode 100644 src/packages/backend/files/sandbox/ripgrep.ts diff --git a/src/packages/backend/files/sandbox/index.ts b/src/packages/backend/files/sandbox/index.ts index 52749163fc..61003ced40 100644 --- a/src/packages/backend/files/sandbox/index.ts +++ b/src/packages/backend/files/sandbox/index.ts @@ -70,12 +70,16 @@ import { replace_all } from "@cocalc/util/misc"; import { EventIterator } from "@cocalc/util/event-iterator"; import { type WatchOptions } from "@cocalc/conat/files/watch"; import find, { type FindOptions } from "./find"; +import ripgrep, { type RipgrepOptions } from "./ripgrep"; -// max time a user find request can run -- this can cause excessive +// max time a user find request can run (in safe mode) -- this can cause excessive // load on a server if there were a directory with a massive number of files, // so must be limited. const MAX_FIND_TIMEOUT = 3000; +// max time a user ripgrep can run (when in safe mode) +const MAX_RIPGREP_TIMEOUT = 3000; + interface Options { // unsafeMode -- if true, assume security model where user is running this // themself, e.g., in a project, so no security is needed at all. @@ -204,17 +208,42 @@ export class SandboxedFilesystem { printf: string, options?: FindOptions, ): Promise<{ stdout: Buffer; truncated: boolean }> => { - options = { ...options }; - if ( - !this.unsafeMode && - (!options.timeout || options.timeout > MAX_FIND_TIMEOUT) - ) { - options.timeout = MAX_FIND_TIMEOUT; - } - + options = { + ...options, + timeout: capTimeout(options?.timeout, MAX_FIND_TIMEOUT), + }; return await find(await this.safeAbsPath(path), printf, options); }; + ripgrep = async ( + path: string, + regexp: string, + options?: RipgrepOptions, + ): Promise<{ + stdout: Buffer; + stderr: Buffer; + code: number | null; + truncated: boolean; + }> => { + if (this.unsafeMode) { + // unsafeMode = slightly less locked down... + return await ripgrep(path, regexp, { + timeout: options?.timeout, + options: options?.options, + allowedBasePath: "/", + }); + } + options = { + ...options, + timeout: capTimeout(options?.timeout, MAX_RIPGREP_TIMEOUT), + }; + return await ripgrep(await this.safeAbsPath(path), regexp, { + timeout: capTimeout(options?.timeout, MAX_RIPGREP_TIMEOUT), + options: options?.options, + allowedBasePath: this.path, + }); + }; + // hard link link = async (existingPath: string, newPath: string) => { this.assertWritable(newPath); @@ -357,3 +386,15 @@ export class SandboxError extends Error { this.path = path; } } + +function capTimeout(timeout: any, max: number): number { + try { + timeout = parseFloat(timeout); + } catch { + return max; + } + if (!isFinite(timeout)) { + return max; + } + return Math.min(timeout, max); +} diff --git a/src/packages/backend/files/sandbox/ripgrep.ts b/src/packages/backend/files/sandbox/ripgrep.ts new file mode 100644 index 0000000000..ab5c04824f --- /dev/null +++ b/src/packages/backend/files/sandbox/ripgrep.ts @@ -0,0 +1,344 @@ +import { spawn } from "node:child_process"; +import { realpath } from "node:fs/promises"; +import * as path from "node:path"; +import type { RipgrepOptions } from "@cocalc/conat/files/fs"; +export type { RipgrepOptions }; + +const MAX_OUTPUT_SIZE = 10 * 1024 * 1024; // 10MB limit + +// Safely allowed options that don't pose security risks +const SAFE_OPTIONS = new Set([ + // Search behavior + "--case-sensitive", + "-s", + "--ignore-case", + "-i", + "--word-regexp", + "-w", + "--line-number", + "-n", + "--count", + "-c", + "--files-with-matches", + "-l", + "--files-without-match", + "--fixed-strings", + "-F", + "--invert-match", + "-v", + + // Output format + "--heading", + "--no-heading", + "--column", + "--pretty", + "--color", + "--no-line-number", + "-N", + + // Context lines (safe as long as we control the path) + "--context", + "-C", + "--before-context", + "-B", + "--after-context", + "-A", + + // Performance/filtering + "--max-count", + "-m", + "--max-depth", + "--max-filesize", + "--type", + "-t", + "--type-not", + "-T", + "--glob", + "-g", + "--iglob", + + // File selection + "--no-ignore", + "--hidden", + "--one-file-system", + "--null-data", + "--multiline", + "-U", + "--multiline-dotall", + "--crlf", + "--encoding", + "-E", + "--no-encoding", +]); + +// Options that take values - need special validation +const OPTIONS_WITH_VALUES = new Set([ + "--max-count", + "-m", + "--max-depth", + "--max-filesize", + "--type", + "-t", + "--type-not", + "-T", + "--glob", + "-g", + "--iglob", + "--context", + "-C", + "--before-context", + "-B", + "--after-context", + "-A", + "--encoding", + "-E", + "--color", +]); + +interface ExtendedRipgrepOptions extends RipgrepOptions { + options?: string[]; + allowedBasePath?: string; // The base path users are allowed to search within +} + +function validateGlobPattern(pattern: string): boolean { + // Reject patterns that could escape directory + if (pattern.includes("../") || pattern.includes("..\\")) { + return false; + } + // Reject absolute paths + if (path.isAbsolute(pattern)) { + return false; + } + return true; +} + +function validateNumber(value: string): boolean { + return /^\d+$/.test(value); +} + +function validateEncoding(value: string): boolean { + // Allow only safe encodings + const safeEncodings = [ + "utf-8", + "utf-16", + "utf-16le", + "utf-16be", + "ascii", + "latin-1", + ]; + return safeEncodings.includes(value.toLowerCase()); +} + +function parseAndValidateOptions(options: string[]): string[] { + const validatedOptions: string[] = []; + let i = 0; + + while (i < options.length) { + const opt = options[i]; + + // Check if this is a safe option + if (!SAFE_OPTIONS.has(opt)) { + throw new Error(`Disallowed option: ${opt}`); + } + + validatedOptions.push(opt); + + // Handle options that take values + if (OPTIONS_WITH_VALUES.has(opt)) { + i++; + if (i >= options.length) { + throw new Error(`Option ${opt} requires a value`); + } + + const value = options[i]; + + // Validate based on option type + if (opt === "--glob" || opt === "-g" || opt === "--iglob") { + if (!validateGlobPattern(value)) { + throw new Error(`Invalid glob pattern: ${value}`); + } + } else if ( + opt === "--max-count" || + opt === "-m" || + opt === "--max-depth" || + opt === "--context" || + opt === "-C" || + opt === "--before-context" || + opt === "-B" || + opt === "--after-context" || + opt === "-A" + ) { + if (!validateNumber(value)) { + throw new Error(`Invalid number for ${opt}: ${value}`); + } + } else if (opt === "--encoding" || opt === "-E") { + if (!validateEncoding(value)) { + throw new Error(`Invalid encoding: ${value}`); + } + } else if (opt === "--color") { + if (!["never", "auto", "always", "ansi"].includes(value)) { + throw new Error(`Invalid color option: ${value}`); + } + } + + validatedOptions.push(value); + } + + i++; + } + + return validatedOptions; +} + +export default async function ripgrep( + searchPath: string, + regexp: string, + { timeout = 0, options = [], allowedBasePath }: ExtendedRipgrepOptions = {}, +): Promise<{ + stdout: Buffer; + stderr: Buffer; + code: number | null; + truncated: boolean; +}> { + if (!searchPath) { + throw Error("path must be specified"); + } + if (!regexp) { + throw Error("regexp must be specified"); + } + + // Validate and normalize the search path + let normalizedPath: string; + try { + // Resolve to real path (follows symlinks to get actual path) + normalizedPath = await realpath(searchPath); + } catch (err) { + // If path doesn't exist, use normalize to check it + normalizedPath = path.normalize(searchPath); + } + + // Security check: ensure path is within allowed base path + if (allowedBasePath) { + const normalizedBase = await realpath(allowedBasePath); + const relative = path.relative(normalizedBase, normalizedPath); + + // If relative path starts with .. or is absolute, it's outside allowed path + if (relative.startsWith("..") || path.isAbsolute(relative)) { + throw new Error("Search path is outside allowed directory"); + } + } + + // Validate regexp doesn't contain null bytes (command injection protection) + if (regexp.includes("\0")) { + throw new Error("Invalid regexp: contains null bytes"); + } + + // Build arguments array with security flags first + const args = [ + "--no-follow", // Don't follow symlinks + "--no-config", // Ignore config files + "--no-ignore-global", // Don't use global gitignore + "--no-require-git", // Don't require git repo + "--no-messages", // Suppress error messages that might leak info + ]; + + // Add validated user options + if (options.length > 0) { + const validatedOptions = parseAndValidateOptions(options); + args.push(...validatedOptions); + } + + // Add the search pattern and path last + args.push("--", regexp, normalizedPath); // -- prevents regexp from being treated as option + + return new Promise((resolve, reject) => { + const stdoutChunks: Buffer[] = []; + const stderrChunks: Buffer[] = []; + let truncated = false; + let stdoutSize = 0; + let stderrSize = 0; + + const child = spawn("rg", args, { + stdio: ["ignore", "pipe", "pipe"], + env: { + // Minimal environment - only what ripgrep needs + PATH: process.env.PATH, + HOME: "/tmp", // Prevent access to user's home + RIPGREP_CONFIG_PATH: "/dev/null", // Explicitly disable config + }, + cwd: allowedBasePath || process.cwd(), // Restrict working directory + }); + + let timeoutHandle: NodeJS.Timeout | null = null; + + if (timeout > 0) { + timeoutHandle = setTimeout(() => { + truncated = true; + child.kill("SIGTERM"); + // Force kill after grace period + setTimeout(() => { + if (!child.killed) { + child.kill("SIGKILL"); + } + }, 1000); + }, timeout); + } + + child.stdout.on("data", (chunk: Buffer) => { + stdoutSize += chunk.length; + if (stdoutSize > MAX_OUTPUT_SIZE) { + truncated = true; + child.kill("SIGTERM"); + return; + } + stdoutChunks.push(chunk); + }); + + child.stderr.on("data", (chunk: Buffer) => { + stderrSize += chunk.length; + if (stderrSize > MAX_OUTPUT_SIZE) { + truncated = true; + child.kill("SIGTERM"); + return; + } + stderrChunks.push(chunk); + }); + + child.on("error", (err) => { + if (timeoutHandle) clearTimeout(timeoutHandle); + reject(err); + }); + + child.on("close", (code) => { + if (timeoutHandle) clearTimeout(timeoutHandle); + + const stdout = Buffer.concat(stdoutChunks); + const stderr = Buffer.concat(stderrChunks); + + // Truncate output if it's too large + const finalStdout = + stdout.length > MAX_OUTPUT_SIZE + ? stdout.slice(0, MAX_OUTPUT_SIZE) + : stdout; + const finalStderr = + stderr.length > MAX_OUTPUT_SIZE + ? stderr.slice(0, MAX_OUTPUT_SIZE) + : stderr; + + resolve({ + stdout: finalStdout, + stderr: finalStderr, + code, + truncated, + }); + }); + }); +} + +// Export utility functions for testing +export const _internal = { + validateGlobPattern, + validateNumber, + validateEncoding, + parseAndValidateOptions, +}; diff --git a/src/packages/backend/package.json b/src/packages/backend/package.json index 48c602b8a1..754945c3b2 100644 --- a/src/packages/backend/package.json +++ b/src/packages/backend/package.json @@ -17,7 +17,8 @@ "scripts": { "preinstall": "npx only-allow pnpm", "clean": "rm -rf dist node_modules", - "build": "pnpm exec tsc --build", + "install-ripgrep": "echo 'require(\"@vscode/ripgrep/lib/postinstall\")' | node", + "build": "pnpm install-ripgrep && pnpm exec tsc --build", "tsc": "pnpm exec tsc --watch --pretty --preserveWatchOutput", "test": "pnpm exec jest --forceExit", "test-conat": " pnpm exec jest --forceExit conat", @@ -37,6 +38,7 @@ "@cocalc/backend": "workspace:*", "@cocalc/conat": "workspace:*", "@cocalc/util": "workspace:*", + "@vscode/ripgrep": "^1.15.14", "awaiting": "^3.0.0", "better-sqlite3": "^11.10.0", "chokidar": "^3.6.0", diff --git a/src/packages/conat/files/fs.ts b/src/packages/conat/files/fs.ts index ea7f567f10..284593b795 100644 --- a/src/packages/conat/files/fs.ts +++ b/src/packages/conat/files/fs.ts @@ -17,6 +17,11 @@ import { isValidUUID } from "@cocalc/util/misc"; export const DEFAULT_FILE_SERVICE = "fs"; +export interface RipgrepOptions { + timeout?: number; + options?: string[]; +} + export interface FindOptions { // timeout is very limited (e.g., 3s?) if fs is running on file // server (not in own project) @@ -76,13 +81,28 @@ export interface Filesystem { // arbitrary directory listing info, which is just not possible // with the fs API, but required in any serious application. // find -P {path} -maxdepth 1 -mindepth 1 -printf {printf} + // For security reasons, this does not support all find arguments, + // and can only use limited resources. find: ( path: string, printf: string, options?: FindOptions, ) => Promise<{ stdout: Buffer; truncated: boolean }>; + // Convenience function that uses the find and stat support to + // provide all files in a directory by using tricky options to find, + // and ensuring they are used by stat in a consistent way for updates. listing?: (path: string) => Promise; + + // We add ripgrep, as a 1-call way to very efficiently search in files + // directly on whatever is serving files. + // For security reasons, this does not support all ripgrep arguments, + // and can only use limited resources. + ripgrep: ( + path: string, + regexp: string, + options?: RipgrepOptions, + ) => Promise<{ stdout: Buffer; stderr: Buffer; truncated: boolean }>; } interface IDirent { @@ -272,6 +292,9 @@ export async function fsServer({ service, fs, client, project_id }: Options) { async rename(oldPath: string, newPath: string) { await (await fs(this.subject)).rename(oldPath, newPath); }, + async ripgrep(path: string, regexp: string, options?: RipgrepOptions) { + return await (await fs(this.subject)).ripgrep(path, regexp, options); + }, async rm(path: string, options?) { await (await fs(this.subject)).rm(path, options); }, diff --git a/src/packages/package.json b/src/packages/package.json index ec02a0d467..7dc7a7d386 100644 --- a/src/packages/package.json +++ b/src/packages/package.json @@ -31,10 +31,10 @@ "tar-fs@3.0.8": "3.0.9" }, "onlyBuiltDependencies": [ + "@vscode/ripgrep", + "better-sqlite3", "websocket-sftp", "websocketfs", - "zeromq", - "better-sqlite3", "zstd-napi" ] } diff --git a/src/packages/pnpm-lock.yaml b/src/packages/pnpm-lock.yaml index 7628085fb0..b6d5458ae6 100644 --- a/src/packages/pnpm-lock.yaml +++ b/src/packages/pnpm-lock.yaml @@ -87,6 +87,9 @@ importers: '@cocalc/util': specifier: workspace:* version: link:../util + '@vscode/ripgrep': + specifier: ^1.15.14 + version: 1.15.14 awaiting: specifier: ^3.0.0 version: 3.0.0 @@ -4568,6 +4571,9 @@ packages: peerDependencies: react: '>= 16.8.0' + '@vscode/ripgrep@1.15.14': + resolution: {integrity: sha512-/G1UJPYlm+trBWQ6cMO3sv6b8D1+G16WaJH1/DSqw32JOVlzgZbLkDxRyzIpTpv30AcYGMkCf5tUqGlW6HbDWw==} + '@vscode/vscode-languagedetection@1.0.22': resolution: {integrity: sha512-rQ/BgMyLuIXSmbA0MSkIPHtcOw14QkeDbAq19sjvaS9LTRr905yij0S8lsyqN5JgOsbtIx7pAcyOxFMzPmqhZQ==} hasBin: true @@ -5137,6 +5143,9 @@ packages: bser@2.1.1: resolution: {integrity: sha512-gQxTNE/GAfIIrmHLUE3oJyp5FO6HRBfhjnw4/wMmA63ZGDJnWBmgY/lyQBpnDUkGmAhbSe39tx2d/iTOAfglwQ==} + buffer-crc32@0.2.13: + resolution: {integrity: sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==} + buffer-equal-constant-time@1.0.1: resolution: {integrity: sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==} @@ -6710,6 +6719,9 @@ packages: fb-watchman@2.0.2: resolution: {integrity: sha512-p5161BqbuCaSnB8jIbzQHOlpgsPmK5rJVDfDKO91Axs5NC1uu3HRQm6wt9cd9/+GtQQIO53JdGXXoyDpTAsgYA==} + fd-slicer@1.1.0: + resolution: {integrity: sha512-cE1qsB/VwyQozZ+q1dGxR8LBYNZeofhEdUNGSMbQD3Gw2lAzX9Zb3uIU6Ebc/Fmyjo9AWWfnn0AUCHqtevs/8g==} + fdir@6.4.6: resolution: {integrity: sha512-hiFoqpyZcfNm1yc4u8oWCf9A2c4D3QjCrks3zmoVKVxpQRzmPNar1hUJcBG2RQHvEVGDN+Jm81ZheVLAQMK6+w==} peerDependencies: @@ -9263,6 +9275,9 @@ packages: resolution: {integrity: sha512-/Y3fcFrXEAsMjJXeL9J8+ZG9U01LbuWaYypvDW2ycW1jL269L3js3DVBjDJ0Up9Np1uqDXsDrRihHANhZOlwdQ==} engines: {node: '>=20'} + pend@1.2.0: + resolution: {integrity: sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==} + performance-now@2.1.0: resolution: {integrity: sha512-7EAHlyLHI56VEIdK57uwHdHKIaAGbnXPiw0yWbarQZOKaKpvUIgW0jWRVLiatnM+XXlSwsanIBH/hzGMJulMow==} @@ -11734,6 +11749,9 @@ packages: resolution: {integrity: sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==} engines: {node: '>=12'} + yauzl@2.10.0: + resolution: {integrity: sha512-p4a9I6X6nu6IhoGmBqAcbJy1mlC4j27vEPZX9F4L4/vZT3Lyq1VkFHw/V/PUcB9Buo+DG3iHkT0x3Qya58zc3g==} + yjs@13.6.27: resolution: {integrity: sha512-OIDwaflOaq4wC6YlPBy2L6ceKeKuF7DeTxx+jPzv1FHn9tCZ0ZwSRnUBxD05E3yed46fv/FWJbvR+Ud7x0L7zw==} engines: {node: '>=16.0.0', npm: '>=8.0.0'} @@ -14904,6 +14922,14 @@ snapshots: '@use-gesture/core': 10.3.1 react: 19.1.0 + '@vscode/ripgrep@1.15.14': + dependencies: + https-proxy-agent: 7.0.6 + proxy-from-env: 1.1.0 + yauzl: 2.10.0 + transitivePeerDependencies: + - supports-color + '@vscode/vscode-languagedetection@1.0.22': {} '@webassemblyjs/ast@1.14.1': @@ -15571,6 +15597,8 @@ snapshots: dependencies: node-int64: 0.4.0 + buffer-crc32@0.2.13: {} + buffer-equal-constant-time@1.0.1: {} buffer-from@1.1.2: {} @@ -17423,6 +17451,10 @@ snapshots: dependencies: bser: 2.1.1 + fd-slicer@1.1.0: + dependencies: + pend: 1.2.0 + fdir@6.4.6(picomatch@4.0.2): optionalDependencies: picomatch: 4.0.2 @@ -20524,6 +20556,8 @@ snapshots: optionalDependencies: '@napi-rs/canvas': 0.1.73 + pend@1.2.0: {} + performance-now@2.1.0: {} pg-cloudflare@1.2.7: @@ -23457,6 +23491,11 @@ snapshots: y18n: 5.0.8 yargs-parser: 21.1.1 + yauzl@2.10.0: + dependencies: + buffer-crc32: 0.2.13 + fd-slicer: 1.1.0 + yjs@13.6.27: dependencies: lib0: 0.2.109 From ec056d89b5c97d1f8926471013ae93e2a984abbb Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 2 Aug 2025 17:19:23 +0000 Subject: [PATCH 166/270] ripgrep: automate installing it properly (not using the vscode version) --- .../backend/files/sandbox/install-ripgrep.ts | 136 ++++++++++++++++++ src/packages/backend/files/sandbox/ripgrep.ts | 13 +- src/packages/backend/package.json | 4 +- 3 files changed, 145 insertions(+), 8 deletions(-) create mode 100644 src/packages/backend/files/sandbox/install-ripgrep.ts diff --git a/src/packages/backend/files/sandbox/install-ripgrep.ts b/src/packages/backend/files/sandbox/install-ripgrep.ts new file mode 100644 index 0000000000..506dbd9dc6 --- /dev/null +++ b/src/packages/backend/files/sandbox/install-ripgrep.ts @@ -0,0 +1,136 @@ +/* +Download a ripgrep binary. + +This supports: + +- x86_64 Linux +- aarch64 Linux +- arm64 macos + +This assumes tar is installed. + +NOTE: There are several npm modules that purport to install ripgrep. We do not use +https://www.npmjs.com/package/@vscode/ripgrep because it is not properly maintained, +e.g., + - security vulnerabilities: https://github.com/microsoft/ripgrep-prebuilt/issues/48 + - not updated to a major new release without a good reason: https://github.com/microsoft/ripgrep-prebuilt/issues/38 +*/ + +import { arch, platform } from "os"; +import { exists } from "@cocalc/backend/misc/async-utils-node"; +import { execFileSync } from "child_process"; +import { writeFile, unlink, chmod } from "fs/promises"; +import { join } from "path"; + +// See https://github.com/BurntSushi/ripgrep/releases +const VERSION = "14.1.1"; +const BASE = "https://github.com/BurntSushi/ripgrep/releases/download"; + +export const rgPath = join(__dirname, "rg"); + +export async function install() { + if (await exists(rgPath)) { + return; + } + const url = getUrl(); + // - 1. Fetch the tarball from the github url (using the fetch library) + const response = await downloadFromGithub(url); + const tarballBuffer = Buffer.from(await response.arrayBuffer()); + + // - 2. Extract the file "rg" from the tarball to ${__dirname}/rg + // The tarball contains this one file "rg" at the top level, i.e., for + // ripgrep-14.1.1-x86_64-unknown-linux-musl.tar.gz + // we have "tar tvf ripgrep-14.1.1-x86_64-unknown-linux-musl.tar.gz" outputs + // ... + // ripgrep-14.1.1-x86_64-unknown-linux-musl/rg + const tmpFile = join(__dirname, `ripgrep-${VERSION}.tar.gz`); + await writeFile(tmpFile, tarballBuffer); + // sync is fine since this is run at *build time*. + execFileSync("tar", [ + "xzf", + tmpFile, + "--strip-components=1", + `-C`, + __dirname, + `ripgrep-${VERSION}-${getName()}/rg`, + ]); + await unlink(tmpFile); + + // - 3. Make the file rg executable + await chmod(rgPath, 0o755); +} + +// Download from github, but aware of rate limits, the retry-after header, etc. +async function downloadFromGithub(url: string) { + const maxRetries = 10; + const baseDelay = 1000; // 1 second + + for (let attempt = 1; attempt <= maxRetries; attempt++) { + try { + const res = await fetch(url); + + if (res.status === 429) { + // Rate limit error + if (attempt === maxRetries) { + throw new Error("Rate limit exceeded after max retries"); + } + + const retryAfter = res.headers.get("retry-after"); + const delay = retryAfter + ? parseInt(retryAfter) * 1000 + : baseDelay * Math.pow(2, attempt - 1); // Exponential backoff + + console.log( + `Rate limited. Retrying in ${delay}ms (attempt ${attempt}/${maxRetries})`, + ); + await new Promise((resolve) => setTimeout(resolve, delay)); + continue; + } + + if (!res.ok) { + throw new Error(`HTTP ${res.status}: ${res.statusText}`); + } + return res; + } catch (error) { + if (attempt === maxRetries) { + throw error; + } + + const delay = baseDelay * Math.pow(2, attempt - 1); + console.log( + `Fetch failed. Retrying in ${delay}ms (attempt ${attempt}/${maxRetries})`, + ); + await new Promise((resolve) => setTimeout(resolve, delay)); + } + } + throw new Error("Should not reach here"); +} + +function getUrl() { + return `${BASE}/${VERSION}/ripgrep-${VERSION}-${getName()}.tar.gz`; +} + +function getName() { + switch (platform()) { + case "linux": + switch (arch()) { + case "x64": + return "x86_64-unknown-linux-musl"; + case "arm64": + return "aarch64-unknown-linux-gnu"; + default: + throw Error(`unsupported arch '${arch()}'`); + } + case "darwin": + switch (arch()) { + case "x64": + return "x86_64-apple-darwin"; + case "arm64": + return "aarch64-apple-darwin"; + default: + throw Error(`unsupported arch '${arch()}'`); + } + default: + throw Error(`unsupported platform '${platform()}'`); + } +} diff --git a/src/packages/backend/files/sandbox/ripgrep.ts b/src/packages/backend/files/sandbox/ripgrep.ts index ab5c04824f..5e587a2c2f 100644 --- a/src/packages/backend/files/sandbox/ripgrep.ts +++ b/src/packages/backend/files/sandbox/ripgrep.ts @@ -3,6 +3,7 @@ import { realpath } from "node:fs/promises"; import * as path from "node:path"; import type { RipgrepOptions } from "@cocalc/conat/files/fs"; export type { RipgrepOptions }; +import { rgPath } from "./install-ripgrep"; const MAX_OUTPUT_SIZE = 10 * 1024 * 1024; // 10MB limit @@ -69,6 +70,9 @@ const SAFE_OPTIONS = new Set([ "--encoding", "-E", "--no-encoding", + + // basic info + "--version", ]); // Options that take values - need special validation @@ -180,13 +184,10 @@ function parseAndValidateOptions(options: string[]): string[] { throw new Error(`Invalid color option: ${value}`); } } - validatedOptions.push(value); } - i++; } - return validatedOptions; } @@ -200,10 +201,10 @@ export default async function ripgrep( code: number | null; truncated: boolean; }> { - if (!searchPath) { + if (searchPath == null) { throw Error("path must be specified"); } - if (!regexp) { + if (regexp == null) { throw Error("regexp must be specified"); } @@ -258,7 +259,7 @@ export default async function ripgrep( let stdoutSize = 0; let stderrSize = 0; - const child = spawn("rg", args, { + const child = spawn(rgPath, args, { stdio: ["ignore", "pipe", "pipe"], env: { // Minimal environment - only what ripgrep needs diff --git a/src/packages/backend/package.json b/src/packages/backend/package.json index 754945c3b2..3cf22ef11b 100644 --- a/src/packages/backend/package.json +++ b/src/packages/backend/package.json @@ -17,8 +17,8 @@ "scripts": { "preinstall": "npx only-allow pnpm", "clean": "rm -rf dist node_modules", - "install-ripgrep": "echo 'require(\"@vscode/ripgrep/lib/postinstall\")' | node", - "build": "pnpm install-ripgrep && pnpm exec tsc --build", + "install-ripgrep": "echo 'require(\"@cocalc/backend/files/sandbox/install-ripgrep\").install()' | node", + "build": "pnpm exec tsc --build && pnpm install-ripgrep", "tsc": "pnpm exec tsc --watch --pretty --preserveWatchOutput", "test": "pnpm exec jest --forceExit", "test-conat": " pnpm exec jest --forceExit conat", From bbd52a0fec380f00f339337aa4b107b032824804 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 2 Aug 2025 10:38:43 -0700 Subject: [PATCH 167/270] update better-sqlite3 and zstd-napi (both broke building on macos) --- src/packages/backend/package.json | 14 +++++++++++--- src/packages/pnpm-lock.yaml | 11 ++++++----- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/src/packages/backend/package.json b/src/packages/backend/package.json index 3cf22ef11b..861ad94254 100644 --- a/src/packages/backend/package.json +++ b/src/packages/backend/package.json @@ -13,7 +13,10 @@ "./auth/*": "./dist/auth/*.js", "./auth/tokens/*": "./dist/auth/tokens/*.js" }, - "keywords": ["utilities", "cocalc"], + "keywords": [ + "utilities", + "cocalc" + ], "scripts": { "preinstall": "npx only-allow pnpm", "clean": "rm -rf dist node_modules", @@ -31,7 +34,12 @@ "conat-persist": "DEBUG=cocalc:* node ./bin/conat-persist.cjs", "conat-test-server": "node ./bin/conat-test-server.cjs" }, - "files": ["dist/**", "bin/**", "README.md", "package.json"], + "files": [ + "dist/**", + "bin/**", + "README.md", + "package.json" + ], "author": "SageMath, Inc.", "license": "SEE LICENSE.md", "dependencies": { @@ -40,7 +48,7 @@ "@cocalc/util": "workspace:*", "@vscode/ripgrep": "^1.15.14", "awaiting": "^3.0.0", - "better-sqlite3": "^11.10.0", + "better-sqlite3": "^12.2.0", "chokidar": "^3.6.0", "debug": "^4.4.0", "fs-extra": "^11.2.0", diff --git a/src/packages/pnpm-lock.yaml b/src/packages/pnpm-lock.yaml index b6d5458ae6..8781b7180e 100644 --- a/src/packages/pnpm-lock.yaml +++ b/src/packages/pnpm-lock.yaml @@ -94,8 +94,8 @@ importers: specifier: ^3.0.0 version: 3.0.0 better-sqlite3: - specifier: ^11.10.0 - version: 11.10.0 + specifier: ^12.2.0 + version: 12.2.0 chokidar: specifier: ^3.6.0 version: 3.6.0 @@ -5056,8 +5056,9 @@ packages: batch@0.6.1: resolution: {integrity: sha512-x+VAiMRL6UPkx+kudNvxTl6hB2XNNCG2r+7wixVfIYwu/2HKRXimwQyaumLjMveWvT2Hkd/cAJw+QBMfJ/EKVw==} - better-sqlite3@11.10.0: - resolution: {integrity: sha512-EwhOpyXiOEL/lKzHz9AW1msWFNzGc/z+LzeB3/jnFJpxu+th2yqvzsSWas1v9jgs9+xiXJcD5A8CJxAG2TaghQ==} + better-sqlite3@12.2.0: + resolution: {integrity: sha512-eGbYq2CT+tos1fBwLQ/tkBt9J5M3JEHjku4hbvQUePCckkvVf14xWj+1m7dGoK81M/fOjFT7yM9UMeKT/+vFLQ==} + engines: {node: 20.x || 22.x || 23.x || 24.x} big.js@3.2.0: resolution: {integrity: sha512-+hN/Zh2D08Mx65pZ/4g5bsmNiZUuChDiQfTUQ7qJr4/kuopCr88xZsAXv6mBoZEsUI4OuGHlX59qE94K2mMW8Q==} @@ -15497,7 +15498,7 @@ snapshots: batch@0.6.1: {} - better-sqlite3@11.10.0: + better-sqlite3@12.2.0: dependencies: bindings: 1.5.0 prebuild-install: 7.1.3 From 144f647dd59f7e3654592469d7df623ae5d6a2cb Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 2 Aug 2025 17:42:03 +0000 Subject: [PATCH 168/270] ripgrep: whitelist searching binary files --- src/packages/backend/files/sandbox/ripgrep.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/packages/backend/files/sandbox/ripgrep.ts b/src/packages/backend/files/sandbox/ripgrep.ts index 5e587a2c2f..5c7859f824 100644 --- a/src/packages/backend/files/sandbox/ripgrep.ts +++ b/src/packages/backend/files/sandbox/ripgrep.ts @@ -73,6 +73,11 @@ const SAFE_OPTIONS = new Set([ // basic info "--version", + + // allow searching in binary files (files that contain NUL) + // this should be safe, since we're capturing output to buffers. + "--text", + "-a", ]); // Options that take values - need special validation From 79dcc3c823900420e92d90ca4119ff0b8b562a51 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 2 Aug 2025 22:25:10 +0000 Subject: [PATCH 169/270] files sandbox: create generic exec command for running command with various sandboxing dimensions, but keeping api as similar to upstream as possible; use for the fd command. --- src/packages/backend/files/sandbox/exec.ts | 155 ++++++++++++++++++ src/packages/backend/files/sandbox/fd.ts | 134 +++++++++++++++ src/packages/backend/files/sandbox/find.ts | 11 ++ src/packages/backend/files/sandbox/index.ts | 36 ++-- src/packages/backend/files/sandbox/ripgrep.ts | 7 + src/packages/conat/files/fs.ts | 25 +++ 6 files changed, 355 insertions(+), 13 deletions(-) create mode 100644 src/packages/backend/files/sandbox/exec.ts create mode 100644 src/packages/backend/files/sandbox/fd.ts diff --git a/src/packages/backend/files/sandbox/exec.ts b/src/packages/backend/files/sandbox/exec.ts new file mode 100644 index 0000000000..8a829a1771 --- /dev/null +++ b/src/packages/backend/files/sandbox/exec.ts @@ -0,0 +1,155 @@ +import { spawn } from "node:child_process"; +import { arch } from "node:os"; +import { type ExecOutput } from "@cocalc/conat/files/fs"; +export { type ExecOutput }; + +const DEFAULT_TIMEOUT = 3_000; +const DEFAULT_MAX_SIZE = 10_000_000; + +export interface Options { + // the path to the command + cmd: string; + // positional arguments; these are not checked in any way, so are given after '--' for safety + positionalArgs?: string[]; + // whitelisted args flags; these are checked according to the whitelist specified below + options?: string[]; + // if given, use these options when os.arch()=='darwin' (i.e., macOS) + darwin?: string[]; + // if given, use these options when os.arch()=='linux' + linux?: string[]; + // when total size of stdout and stderr hits this amount, command is terminated, and + // truncated is set. The total amount of output may thus be slightly larger than maxOutput + maxSize?: number; + // command is terminated after this many ms + timeout?: number; + // each command line option that is explicitly whitelisted + // should be a key in the following whitelist map. + // The value can be either: + // - true: in which case the option does not take a argument, or + // - a function: in which the option takes exactly one argument; the function should validate that argument + // and throw an error if the argument is not allowed. + whitelist?: { [option: string]: true | ValidateFunction }; + // where to launch command + cwd?: string; +} + +type ValidateFunction = (value: string) => void; + +export default async function exec({ + cmd, + positionalArgs = [], + options = [], + linux = [], + darwin = [], + maxSize = DEFAULT_MAX_SIZE, + timeout = DEFAULT_TIMEOUT, + whitelist = {}, + cwd, +}: Options): Promise { + if (arch() == "darwin") { + options = options.concat(darwin); + } else if (arch() == "linux") { + options = options.concat(linux); + } + options = parseAndValidateOptions(options, whitelist); + + return new Promise((resolve, reject) => { + const stdoutChunks: Buffer[] = []; + const stderrChunks: Buffer[] = []; + let truncated = false; + let stdoutSize = 0; + let stderrSize = 0; + + const args = options.concat(["--"]).concat(positionalArgs); + const child = spawn(cmd, args, { + stdio: ["ignore", "pipe", "pipe"], + env: {}, + cwd, + }); + + let timeoutHandle: NodeJS.Timeout | null = null; + + if (timeout > 0) { + timeoutHandle = setTimeout(() => { + truncated = true; + child.kill("SIGTERM"); + // Force kill after grace period + setTimeout(() => { + if (!child.killed) { + child.kill("SIGKILL"); + } + }, 1000); + }, timeout); + } + + child.stdout.on("data", (chunk: Buffer) => { + stdoutSize += chunk.length; + if (stdoutSize + stderrSize >= maxSize) { + truncated = true; + child.kill("SIGTERM"); + return; + } + stdoutChunks.push(chunk); + }); + + child.stderr.on("data", (chunk: Buffer) => { + stderrSize += chunk.length; + if (stdoutSize + stderrSize > maxSize) { + truncated = true; + child.kill("SIGTERM"); + return; + } + stderrChunks.push(chunk); + }); + + child.on("error", (err) => { + if (timeoutHandle) { + clearTimeout(timeoutHandle); + } + reject(err); + }); + + child.once("close", (code) => { + if (timeoutHandle) { + clearTimeout(timeoutHandle); + } + + resolve({ + stdout: Buffer.concat(stdoutChunks), + stderr: Buffer.concat(stderrChunks), + code, + truncated, + }); + }); + }); +} + +function parseAndValidateOptions(options: string[], whitelist): string[] { + const validatedOptions: string[] = []; + let i = 0; + + while (i < options.length) { + const opt = options[i]; + + // Check if this is a safe option + const validate = whitelist[opt]; + if (!validate) { + throw new Error(`Disallowed option: ${opt}`); + } + validatedOptions.push(opt); + + // Handle options that take values + if (validate !== true) { + i++; + if (i >= options.length) { + throw new Error(`Option ${opt} requires a value`); + } + const value = options[i]; + validate(value); + // didn't throw, so good to go + validatedOptions.push(value); + } + i++; + } + return validatedOptions; +} diff --git a/src/packages/backend/files/sandbox/fd.ts b/src/packages/backend/files/sandbox/fd.ts new file mode 100644 index 0000000000..14551a9091 --- /dev/null +++ b/src/packages/backend/files/sandbox/fd.ts @@ -0,0 +1,134 @@ +import exec, { type ExecOutput } from "./exec"; +import { type FdOptions } from "@cocalc/conat/files/fs"; +export { type FdOptions }; + +export default async function fd( + path: string, + { options, darwin, linux, pattern, timeout, maxSize }: FdOptions, +): Promise { + if (path == null) { + throw Error("path must be specified"); + } + + return await exec({ + cmd: "/projects/6b851643-360e-435e-b87e-f9a6ab64a8b1/bin/fd", + cwd: path, + positionalArgs: pattern ? [pattern] : [], + options, + darwin, + linux, + maxSize, + timeout, + whitelist, + }); +} + +const whitelist = { + "-H": true, + "--hidden": true, + + "-I": true, + "--no-ignore": true, + + "-u": true, + "--unrestricted": true, + + "--no-ignore-vcs": true, + "--no-require-git": true, + + "-s": true, + "--case-sensitive": true, + + "-i": true, + "--ignore-case": true, + + "-g": true, + "--glob": true, + + "--regex": true, + + "-F": true, + "--fixed-strings": true, + + "--and": anyValue, + + "-l": true, + "--list-details": true, + + "-p": true, + "--full-path": true, + + "-0": true, + "--print0": true, + + "--max-results": validateInt, + + "-1": true, + + "-q": true, + "--quite": true, + + "--show-errors": true, + + "--strip-cwd-prefix": validateEnum(["never", "always", "auto"]), + + "--one-file-system": true, + "--mount": true, + "--xdev": true, + + "-h": true, + "--help": true, + + "-V": true, + "--version": true, + + "-d": validateInt, + "--max-depth": validateInt, + + "--min-depth": validateInt, + + "--exact-depth": validateInt, + + "--prune": true, + + "--type": anyValue, + + "-e": anyValue, + "--extension": anyValue, + + "-E": anyValue, + "--exclude": anyValue, + + "--ignore-file": anyValue, + + "-c": validateEnum(["never", "always", "auto"]), + "--color": validateEnum(["never", "always", "auto"]), + + "-S": anyValue, + "--size": anyValue, + + "--changed-within": anyValue, + "--changed-before": anyValue, + + "-o": anyValue, + "--owner": anyValue, + + "--format": anyValue, +} as const; + +function anyValue() {} + +function validateEnum(allowed: string[]) { + return (value: string) => { + if (!allowed.includes(value)) { + throw Error("invalid value"); + } + }; +} + +function validateInt(value: string) { + const count = parseInt(value); + if (!isFinite(count)) { + throw Error("argument must be a number"); + } +} diff --git a/src/packages/backend/files/sandbox/find.ts b/src/packages/backend/files/sandbox/find.ts index 90cd67c9bc..2f6e614ce3 100644 --- a/src/packages/backend/files/sandbox/find.ts +++ b/src/packages/backend/files/sandbox/find.ts @@ -1,3 +1,14 @@ +/* + +NOTE: there is a program https://github.com/sharkdp/fd that is a very fast +parallel rust program for finding files matching a pattern. It is complementary +to find here though, because we mainly use find to compute directory +listing info (e.g., file size, mtime, etc.), and fd does NOT do that; it can +exec ls, but that is slower than using find. So both find and fd are useful +for different tasks -- find is *better* for directory listings and fd is better +for finding filesnames in a directory tree that match a pattern. +*/ + import { spawn } from "node:child_process"; import type { FindOptions, FindExpression } from "@cocalc/conat/files/fs"; export type { FindOptions, FindExpression }; diff --git a/src/packages/backend/files/sandbox/index.ts b/src/packages/backend/files/sandbox/index.ts index 61003ced40..cf8874cdce 100644 --- a/src/packages/backend/files/sandbox/index.ts +++ b/src/packages/backend/files/sandbox/index.ts @@ -71,6 +71,8 @@ import { EventIterator } from "@cocalc/util/event-iterator"; import { type WatchOptions } from "@cocalc/conat/files/watch"; import find, { type FindOptions } from "./find"; import ripgrep, { type RipgrepOptions } from "./ripgrep"; +import fd, { type FdOptions } from "./fd"; +import { type ExecOutput } from "./exec"; // max time a user find request can run (in safe mode) -- this can cause excessive // load on a server if there were a directory with a massive number of files, @@ -80,6 +82,8 @@ const MAX_FIND_TIMEOUT = 3000; // max time a user ripgrep can run (when in safe mode) const MAX_RIPGREP_TIMEOUT = 3000; +const MAX_FD_TIMEOUT = 3000; + interface Options { // unsafeMode -- if true, assume security model where user is running this // themself, e.g., in a project, so no security is needed at all. @@ -208,13 +212,17 @@ export class SandboxedFilesystem { printf: string, options?: FindOptions, ): Promise<{ stdout: Buffer; truncated: boolean }> => { - options = { - ...options, - timeout: capTimeout(options?.timeout, MAX_FIND_TIMEOUT), - }; + options = capTimeout(options, MAX_FIND_TIMEOUT); return await find(await this.safeAbsPath(path), printf, options); }; + fd = async (path: string, options?: FdOptions): Promise => { + return await fd( + await this.safeAbsPath(path), + capTimeout(options, MAX_FD_TIMEOUT), + ); + }; + ripgrep = async ( path: string, regexp: string, @@ -233,10 +241,7 @@ export class SandboxedFilesystem { allowedBasePath: "/", }); } - options = { - ...options, - timeout: capTimeout(options?.timeout, MAX_RIPGREP_TIMEOUT), - }; + options = capTimeout(options, MAX_RIPGREP_TIMEOUT); return await ripgrep(await this.safeAbsPath(path), regexp, { timeout: capTimeout(options?.timeout, MAX_RIPGREP_TIMEOUT), options: options?.options, @@ -387,14 +392,19 @@ export class SandboxError extends Error { } } -function capTimeout(timeout: any, max: number): number { +function capTimeout(options, max: number) { + if (options == null) { + return { timeout: max }; + } + + let timeout; try { - timeout = parseFloat(timeout); + timeout = parseFloat(options.timeout); } catch { - return max; + return { ...options, timeout: max }; } if (!isFinite(timeout)) { - return max; + return { ...options, timeout: max }; } - return Math.min(timeout, max); + return { ...options, timeout: Math.min(timeout, max) }; } diff --git a/src/packages/backend/files/sandbox/ripgrep.ts b/src/packages/backend/files/sandbox/ripgrep.ts index 5c7859f824..f2ff751661 100644 --- a/src/packages/backend/files/sandbox/ripgrep.ts +++ b/src/packages/backend/files/sandbox/ripgrep.ts @@ -78,6 +78,13 @@ const SAFE_OPTIONS = new Set([ // this should be safe, since we're capturing output to buffers. "--text", "-a", + + // this ignores gitignore, hidden, and binary restrictions, which we allow + // above, so should be safe. + "--unrestricted", + "-u", + + "--debug", ]); // Options that take values - need special validation diff --git a/src/packages/conat/files/fs.ts b/src/packages/conat/files/fs.ts index 284593b795..b3ee87786f 100644 --- a/src/packages/conat/files/fs.ts +++ b/src/packages/conat/files/fs.ts @@ -17,6 +17,14 @@ import { isValidUUID } from "@cocalc/util/misc"; export const DEFAULT_FILE_SERVICE = "fs"; +export interface ExecOutput { + stdout: Buffer; + stderr: Buffer; + code: number | null; + // true if terminated early due to output size or time + truncated?: boolean; +} + export interface RipgrepOptions { timeout?: number; options?: string[]; @@ -44,6 +52,15 @@ export type FindExpression = | { type: "or"; left: FindExpression; right: FindExpression } | { type: "not"; expr: FindExpression }; +export interface FdOptions { + pattern?: string; + options?: string[]; + darwin?: string[]; + linux?: string[]; + timeout?: number; + maxSize?: number; +} + export interface Filesystem { appendFile: (path: string, data: string | Buffer, encoding?) => Promise; chmod: (path: string, mode: string | number) => Promise; @@ -94,6 +111,11 @@ export interface Filesystem { // and ensuring they are used by stat in a consistent way for updates. listing?: (path: string) => Promise; + // fd is a rust rewrite of find that is extremely fast at finding + // files that match an expression, e.g., + // options: { type: "name", pattern:"^\.DS_Store$" } + fd: (path: string, options?: FdOptions) => Promise; + // We add ripgrep, as a 1-call way to very efficiently search in files // directly on whatever is serving files. // For security reasons, this does not support all ripgrep arguments, @@ -257,6 +279,9 @@ export async function fsServer({ service, fs, client, project_id }: Options) { async exists(path: string): Promise { return await (await fs(this.subject)).exists(path); }, + async fd(path: string, options?: FdOptions) { + return await (await fs(this.subject)).fd(path, options); + }, async find(path: string, printf: string, options?: FindOptions) { return await (await fs(this.subject)).find(path, printf, options); }, From 284f91dd547a8bce8a23e6371cf146b23d27307c Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 3 Aug 2025 04:23:58 +0000 Subject: [PATCH 170/270] automate install of fd and ripgrep in a more robust way --- src/packages/backend/files/sandbox/exec.ts | 52 +- src/packages/backend/files/sandbox/fd.test.ts | 27 + src/packages/backend/files/sandbox/fd.ts | 69 +-- .../backend/files/sandbox/find.test.ts | 36 +- src/packages/backend/files/sandbox/find.ts | 257 ++++---- src/packages/backend/files/sandbox/index.ts | 40 +- .../{install-ripgrep.ts => install.ts} | 117 ++-- .../backend/files/sandbox/ripgrep.test.ts | 27 + src/packages/backend/files/sandbox/ripgrep.ts | 548 +++++++----------- src/packages/backend/package.json | 2 +- src/packages/conat/files/fs.ts | 46 +- src/packages/conat/files/listing.ts | 16 +- src/packages/frontend/project/search/body.tsx | 17 +- 13 files changed, 593 insertions(+), 661 deletions(-) create mode 100644 src/packages/backend/files/sandbox/fd.test.ts rename src/packages/backend/files/sandbox/{install-ripgrep.ts => install.ts} (57%) create mode 100644 src/packages/backend/files/sandbox/ripgrep.test.ts diff --git a/src/packages/backend/files/sandbox/exec.ts b/src/packages/backend/files/sandbox/exec.ts index 8a829a1771..b0d477b622 100644 --- a/src/packages/backend/files/sandbox/exec.ts +++ b/src/packages/backend/files/sandbox/exec.ts @@ -2,6 +2,9 @@ import { spawn } from "node:child_process"; import { arch } from "node:os"; import { type ExecOutput } from "@cocalc/conat/files/fs"; export { type ExecOutput }; +import getLogger from "@cocalc/backend/logger"; + +const logger = getLogger("files:sandbox:exec"); const DEFAULT_TIMEOUT = 3_000; const DEFAULT_MAX_SIZE = 10_000_000; @@ -9,13 +12,15 @@ const DEFAULT_MAX_SIZE = 10_000_000; export interface Options { // the path to the command cmd: string; - // positional arguments; these are not checked in any way, so are given after '--' for safety + // position args *before* any options; these are not sanitized + prefixArgs?: string[]; + // positional arguments; these are not sanitized, but are given after '--' for safety positionalArgs?: string[]; // whitelisted args flags; these are checked according to the whitelist specified below options?: string[]; - // if given, use these options when os.arch()=='darwin' (i.e., macOS) + // if given, use these options when os.arch()=='darwin' (i.e., macOS); these must match whitelist darwin?: string[]; - // if given, use these options when os.arch()=='linux' + // if given, use these options when os.arch()=='linux'; these must match whitelist linux?: string[]; // when total size of stdout and stderr hits this amount, command is terminated, and // truncated is set. The total amount of output may thus be slightly larger than maxOutput @@ -31,6 +36,9 @@ export interface Options { whitelist?: { [option: string]: true | ValidateFunction }; // where to launch command cwd?: string; + + // options that are always included first for safety and need NOT match whitelist + safety?: string[]; } type ValidateFunction = (value: string) => void; @@ -38,9 +46,11 @@ type ValidateFunction = (value: string) => void; export default async function exec({ cmd, positionalArgs = [], + prefixArgs = [], options = [], linux = [], darwin = [], + safety = [], maxSize = DEFAULT_MAX_SIZE, timeout = DEFAULT_TIMEOUT, whitelist = {}, @@ -51,7 +61,7 @@ export default async function exec({ } else if (arch() == "linux") { options = options.concat(linux); } - options = parseAndValidateOptions(options, whitelist); + options = safety.concat(parseAndValidateOptions(options, whitelist)); return new Promise((resolve, reject) => { const stdoutChunks: Buffer[] = []; @@ -60,7 +70,13 @@ export default async function exec({ let stdoutSize = 0; let stderrSize = 0; - const args = options.concat(["--"]).concat(positionalArgs); + const args = prefixArgs.concat(options); + if (positionalArgs.length > 0) { + args.push("--", ...positionalArgs); + } + + // console.log(`${cmd} ${args.join(" ")}`); + logger.debug({ cmd, args }); const child = spawn(cmd, args, { stdio: ["ignore", "pipe", "pipe"], env: {}, @@ -144,7 +160,7 @@ function parseAndValidateOptions(options: string[], whitelist): string[] { if (i >= options.length) { throw new Error(`Option ${opt} requires a value`); } - const value = options[i]; + const value = String(options[i]); validate(value); // didn't throw, so good to go validatedOptions.push(value); @@ -153,3 +169,27 @@ function parseAndValidateOptions(options: string[], whitelist): string[] { } return validatedOptions; } + +export const validate = { + str: () => {}, + set: (allowed) => { + allowed = new Set(allowed); + return (value: string) => { + if (!allowed.includes(value)) { + throw Error("invalid value"); + } + }; + }, + int: (value: string) => { + const x = parseInt(value); + if (!isFinite(x)) { + throw Error("argument must be a number"); + } + }, + float: (value: string) => { + const x = parseFloat(value); + if (!isFinite(x)) { + throw Error("argument must be a number"); + } + }, +}; diff --git a/src/packages/backend/files/sandbox/fd.test.ts b/src/packages/backend/files/sandbox/fd.test.ts new file mode 100644 index 0000000000..f17132d972 --- /dev/null +++ b/src/packages/backend/files/sandbox/fd.test.ts @@ -0,0 +1,27 @@ +import fd from "./fd"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +let tempDir; +beforeAll(async () => { + tempDir = await mkdtemp(join(tmpdir(), "cocalc")); +}); +afterAll(async () => { + await rm(tempDir, { force: true, recursive: true }); +}); + +describe("fd files", () => { + it("directory starts empty", async () => { + const { stdout, truncated } = await fd(tempDir); + expect(stdout.length).toBe(0); + expect(truncated).toBe(false); + }); + + it("create a file and see it appears via fd", async () => { + await writeFile(join(tempDir, "a.txt"), "hello"); + const { stdout, truncated } = await fd(tempDir); + expect(truncated).toBe(false); + expect(stdout.toString()).toEqual("a.txt\n"); + }); +}); diff --git a/src/packages/backend/files/sandbox/fd.ts b/src/packages/backend/files/sandbox/fd.ts index 14551a9091..dfc92e8e1e 100644 --- a/src/packages/backend/files/sandbox/fd.ts +++ b/src/packages/backend/files/sandbox/fd.ts @@ -1,22 +1,24 @@ -import exec, { type ExecOutput } from "./exec"; +import exec, { type ExecOutput, validate } from "./exec"; import { type FdOptions } from "@cocalc/conat/files/fs"; export { type FdOptions }; +import { fd as fdPath } from "./install"; export default async function fd( path: string, - { options, darwin, linux, pattern, timeout, maxSize }: FdOptions, + { options, darwin, linux, pattern, timeout, maxSize }: FdOptions = {}, ): Promise { if (path == null) { throw Error("path must be specified"); } return await exec({ - cmd: "/projects/6b851643-360e-435e-b87e-f9a6ab64a8b1/bin/fd", + cmd: fdPath, cwd: path, positionalArgs: pattern ? [pattern] : [], options, darwin, linux, + safety: ["--no-follow"], maxSize, timeout, whitelist, @@ -50,7 +52,7 @@ const whitelist = { "-F": true, "--fixed-strings": true, - "--and": anyValue, + "--and": validate.str, "-l": true, "--list-details": true, @@ -61,7 +63,7 @@ const whitelist = { "-0": true, "--print0": true, - "--max-results": validateInt, + "--max-results": validate.int, "-1": true, @@ -70,7 +72,7 @@ const whitelist = { "--show-errors": true, - "--strip-cwd-prefix": validateEnum(["never", "always", "auto"]), + "--strip-cwd-prefix": validate.set(["never", "always", "auto"]), "--one-file-system": true, "--mount": true, @@ -82,53 +84,36 @@ const whitelist = { "-V": true, "--version": true, - "-d": validateInt, - "--max-depth": validateInt, + "-d": validate.int, + "--max-depth": validate.int, - "--min-depth": validateInt, + "--min-depth": validate.int, - "--exact-depth": validateInt, + "--exact-depth": validate.int, "--prune": true, - "--type": anyValue, + "--type": validate.str, - "-e": anyValue, - "--extension": anyValue, + "-e": validate.str, + "--extension": validate.str, - "-E": anyValue, - "--exclude": anyValue, + "-E": validate.str, + "--exclude": validate.str, - "--ignore-file": anyValue, + "--ignore-file": validate.str, - "-c": validateEnum(["never", "always", "auto"]), - "--color": validateEnum(["never", "always", "auto"]), + "-c": validate.set(["never", "always", "auto"]), + "--color": validate.set(["never", "always", "auto"]), - "-S": anyValue, - "--size": anyValue, + "-S": validate.str, + "--size": validate.str, - "--changed-within": anyValue, - "--changed-before": anyValue, + "--changed-within": validate.str, + "--changed-before": validate.str, - "-o": anyValue, - "--owner": anyValue, + "-o": validate.str, + "--owner": validate.str, - "--format": anyValue, + "--format": validate.str, } as const; - -function anyValue() {} - -function validateEnum(allowed: string[]) { - return (value: string) => { - if (!allowed.includes(value)) { - throw Error("invalid value"); - } - }; -} - -function validateInt(value: string) { - const count = parseInt(value); - if (!isFinite(count)) { - throw Error("argument must be a number"); - } -} diff --git a/src/packages/backend/files/sandbox/find.test.ts b/src/packages/backend/files/sandbox/find.test.ts index f57ff4cd48..66574f5cac 100644 --- a/src/packages/backend/files/sandbox/find.test.ts +++ b/src/packages/backend/files/sandbox/find.test.ts @@ -13,14 +13,18 @@ afterAll(async () => { describe("find files", () => { it("directory starts empty", async () => { - const { stdout, truncated } = await find(tempDir, "%f\n"); + const { stdout, truncated } = await find(tempDir, { + options: ["-maxdepth", "1", "-mindepth", "1", "-printf", "%f\n"], + }); expect(stdout.length).toBe(0); expect(truncated).toBe(false); }); it("create a file and see it appears in find", async () => { await writeFile(join(tempDir, "a.txt"), "hello"); - const { stdout, truncated } = await find(tempDir, "%f\n"); + const { stdout, truncated } = await find(tempDir, { + options: ["-maxdepth", "1", "-mindepth", "1", "-printf", "%f\n"], + }); expect(truncated).toBe(false); expect(stdout.toString()).toEqual("a.txt\n"); }); @@ -29,17 +33,25 @@ describe("find files", () => { await writeFile(join(tempDir, "pattern"), ""); await mkdir(join(tempDir, "blue")); await writeFile(join(tempDir, "blue", "Patton"), ""); - const { stdout } = await find(tempDir, "%f\n", { - expression: { type: "iname", pattern: "patt*" }, + const { stdout } = await find(tempDir, { + options: [ + "-maxdepth", + "1", + "-mindepth", + "1", + "-iname", + "patt*", + "-printf", + "%f\n", + ], }); const v = stdout.toString().trim().split("\n"); expect(new Set(v)).toEqual(new Set(["pattern"])); }); it("find file in a subdirectory too", async () => { - const { stdout } = await find(tempDir, "%P\n", { - recursive: true, - expression: { type: "iname", pattern: "patt*" }, + const { stdout } = await find(tempDir, { + options: ["-iname", "patt*", "-printf", "%P\n"], }); const w = stdout.toString().trim().split("\n"); expect(new Set(w)).toEqual(new Set(["pattern", "blue/Patton"])); @@ -52,11 +64,17 @@ describe("find files", () => { await writeFile(join(tempDir, `${i}`), ""); } const t = Date.now(); - const { stdout, truncated } = await find(tempDir, "%f\n", { timeout: 0.1 }); + const { stdout, truncated } = await find(tempDir, { + options: ["-printf", "%f\n"], + timeout: 0.1, + }); + expect(truncated).toBe(true); expect(Date.now() - t).toBeGreaterThan(1); - const { stdout: stdout2 } = await find(tempDir, "%f\n"); + const { stdout: stdout2 } = await find(tempDir, { + options: ["-maxdepth", "1", "-mindepth", "1", "-printf", "%f\n"], + }); expect(stdout2.length).toBeGreaterThan(stdout.length); }); }); diff --git a/src/packages/backend/files/sandbox/find.ts b/src/packages/backend/files/sandbox/find.ts index 2f6e614ce3..8c1929ce70 100644 --- a/src/packages/backend/files/sandbox/find.ts +++ b/src/packages/backend/files/sandbox/find.ts @@ -1,172 +1,117 @@ /* -NOTE: there is a program https://github.com/sharkdp/fd that is a very fast -parallel rust program for finding files matching a pattern. It is complementary -to find here though, because we mainly use find to compute directory +NOTE: fd is a very fast parallel rust program for finding files matching +a pattern. It is complementary to find here though, because we mainly +use find to compute directory listing info (e.g., file size, mtime, etc.), and fd does NOT do that; it can exec ls, but that is slower than using find. So both find and fd are useful for different tasks -- find is *better* for directory listings and fd is better for finding filesnames in a directory tree that match a pattern. */ -import { spawn } from "node:child_process"; -import type { FindOptions, FindExpression } from "@cocalc/conat/files/fs"; -export type { FindOptions, FindExpression }; +import type { FindOptions } from "@cocalc/conat/files/fs"; +export type { FindOptions }; +import exec, { type ExecOutput, validate } from "./exec"; export default async function find( path: string, - printf: string, - { timeout = 0, recursive, expression }: FindOptions = {}, -): Promise<{ - // the output as a Buffer (not utf8, since it could have arbitrary file names!) - stdout: Buffer; - // truncated is true if the timeout gets hit. - truncated: boolean; -}> { - if (!path) { + { options, darwin, linux, timeout, maxSize }: FindOptions, +): Promise { + if (path == null) { throw Error("path must be specified"); } - if (!printf) { - throw Error("printf must be specified"); - } - return new Promise((resolve, reject) => { - const chunks: Buffer[] = []; - let truncated = false; - - const args = [ - "-P", // Never follow symlinks (security) - path, // Search path - "-mindepth", - "1", - ]; - if (!recursive) { - args.push("-maxdepth", "1"); - } - - // Add expression if provided - if (expression) { - try { - args.push(...buildFindArgs(expression)); - } catch (error) { - reject(error); - return; - } - } - args.push("-printf", printf); - - //console.log(`find ${args.join(" ")}`); - - // Spawn find with minimal, fixed arguments - const child = spawn("find", args, { - stdio: ["ignore", "pipe", "pipe"], - env: {}, // Empty environment (security) - shell: false, // No shell interpretation (security) - }); - - let timer; - if (timeout) { - timer = setTimeout(() => { - if (!truncated) { - truncated = true; - child.kill("SIGTERM"); - } - }, timeout); - } else { - timer = null; - } - - child.stdout.on("data", (chunk: Buffer) => { - chunks.push(chunk); - }); - - let stderr = ""; - child.stderr.on("data", (chunk: Buffer) => { - stderr += chunk.toString(); - }); - - // Handle completion - child.on("error", (error) => { - if (timer) { - clearTimeout(timer); - } - reject(error); - }); - - child.on("exit", (code) => { - if (timer) { - clearTimeout(timer); - } - - if (code !== 0 && !truncated) { - reject(new Error(`find exited with code ${code}: ${stderr}`)); - return; - } - - resolve({ stdout: Buffer.concat(chunks), truncated }); - }); + return await exec({ + cmd: "find", + cwd: path, + prefixArgs: [path ? path : "."], + options, + darwin, + linux, + maxSize, + timeout, + whitelist, + safety: [], }); } -function buildFindArgs(expr: FindExpression): string[] { - switch (expr.type) { - case "name": - // Validate pattern has no path separators - if (expr.pattern.includes("/")) { - throw new Error("Path separators not allowed in name patterns"); - } - return ["-name", expr.pattern]; - - case "iname": - if (expr.pattern.includes("/")) { - throw new Error("Path separators not allowed in name patterns"); - } - return ["-iname", expr.pattern]; - - case "type": - return ["-type", expr.value]; - - case "size": - // Validate size format (e.g., "10M", "1G", "500k") - if (!/^[0-9]+[kMGTP]?$/.test(expr.value)) { - throw new Error("Invalid size format"); - } - return ["-size", expr.operator + expr.value]; - - case "mtime": - if (!Number.isInteger(expr.days) || expr.days < 0) { - throw new Error("Invalid mtime days"); - } - return ["-mtime", expr.operator + expr.days]; - - case "newer": - // This is risky - would need to validate file path is within sandbox - if (expr.file.includes("..") || expr.file.startsWith("/")) { - throw new Error("Invalid reference file path"); - } - return ["-newer", expr.file]; - - case "and": - return [ - "(", - ...buildFindArgs(expr.left), - "-a", - ...buildFindArgs(expr.right), - ")", - ]; - - case "or": - return [ - "(", - ...buildFindArgs(expr.left), - "-o", - ...buildFindArgs(expr.right), - ")", - ]; - - case "not": - return ["!", ...buildFindArgs(expr.expr)]; - - default: - throw new Error("Unsupported expression type"); - } -} +const whitelist = { + // POSITIONAL OPTIONS + "-daystart": true, + "-regextype": validate.str, + "-warn": true, + "-nowarn": true, + + // GLOBAL OPTIONS + "-d": true, + "-depth": true, + "--help": true, + "-ignore_readdir_race": true, + "-maxdepth": validate.int, + "-mindepth": validate.int, + "-mount": true, + "-noignore_readdir_race": true, + "--version": true, + "-xdev": true, + + // TESTS + "-amin": validate.float, + "-anewer": validate.str, + "-atime": validate.float, + "-cmin": validate.float, + "-cnewer": validate.str, + "-ctime": validate.float, + "-empty": true, + "-executable": true, + "-fstype": validate.str, + "-gid": validate.int, + "-group": validate.str, + "-ilname": validate.str, + "-iname": validate.str, + "-inum": validate.int, + "-ipath": validate.str, + "-iregex": validate.str, + "-iwholename": validate.str, + "-links": validate.int, + "-lname": validate.str, + "-mmin": validate.int, + "-mtime": validate.int, + "-name": validate.str, + "-newer": validate.str, + "-newerXY": validate.str, + "-nogroup": true, + "-nouser": true, + "-path": validate.str, + "-perm": validate.str, + "-readable": true, + "-regex": validate.str, + "-samefile": validate.str, + "-size": validate.str, + "-true": true, + "-type": validate.str, + "-uid": validate.int, + "-used": validate.float, + "-user": validate.str, + "-wholename": validate.str, + "-writable": true, + "-xtype": validate.str, + "-context": validate.str, + + // ACTIONS: obviously many are not whitelisted! + "-ls": true, + "-print": true, + "-print0": true, + "-printf": validate.str, + "-prune": true, + "-quit": true, + + // OPERATORS + "(": true, + ")": true, + "!": true, + "-not": true, + "-a": true, + "-and": true, + "-o": true, + "-or": true, + ",": true, +} as const; diff --git a/src/packages/backend/files/sandbox/index.ts b/src/packages/backend/files/sandbox/index.ts index cf8874cdce..7bb111943a 100644 --- a/src/packages/backend/files/sandbox/index.ts +++ b/src/packages/backend/files/sandbox/index.ts @@ -207,13 +207,11 @@ export class SandboxedFilesystem { return await exists(await this.safeAbsPath(path)); }; - find = async ( - path: string, - printf: string, - options?: FindOptions, - ): Promise<{ stdout: Buffer; truncated: boolean }> => { - options = capTimeout(options, MAX_FIND_TIMEOUT); - return await find(await this.safeAbsPath(path), printf, options); + find = async (path: string, options?: FindOptions): Promise => { + return await find( + await this.safeAbsPath(path), + capTimeout(options, MAX_FIND_TIMEOUT), + ); }; fd = async (path: string, options?: FdOptions): Promise => { @@ -225,28 +223,14 @@ export class SandboxedFilesystem { ripgrep = async ( path: string, - regexp: string, + pattern: string, options?: RipgrepOptions, - ): Promise<{ - stdout: Buffer; - stderr: Buffer; - code: number | null; - truncated: boolean; - }> => { - if (this.unsafeMode) { - // unsafeMode = slightly less locked down... - return await ripgrep(path, regexp, { - timeout: options?.timeout, - options: options?.options, - allowedBasePath: "/", - }); - } - options = capTimeout(options, MAX_RIPGREP_TIMEOUT); - return await ripgrep(await this.safeAbsPath(path), regexp, { - timeout: capTimeout(options?.timeout, MAX_RIPGREP_TIMEOUT), - options: options?.options, - allowedBasePath: this.path, - }); + ): Promise => { + return await ripgrep( + await this.safeAbsPath(path), + pattern, + capTimeout(options, MAX_RIPGREP_TIMEOUT), + ); }; // hard link diff --git a/src/packages/backend/files/sandbox/install-ripgrep.ts b/src/packages/backend/files/sandbox/install.ts similarity index 57% rename from src/packages/backend/files/sandbox/install-ripgrep.ts rename to src/packages/backend/files/sandbox/install.ts index 506dbd9dc6..7f5fe1030b 100644 --- a/src/packages/backend/files/sandbox/install-ripgrep.ts +++ b/src/packages/backend/files/sandbox/install.ts @@ -1,11 +1,7 @@ /* -Download a ripgrep binary. +Download a ripgrep or fd binary for the operating system -This supports: - -- x86_64 Linux -- aarch64 Linux -- arm64 macos +This supports x86_64/arm64 linux & macos This assumes tar is installed. @@ -17,22 +13,61 @@ e.g., */ import { arch, platform } from "os"; -import { exists } from "@cocalc/backend/misc/async-utils-node"; import { execFileSync } from "child_process"; -import { writeFile, unlink, chmod } from "fs/promises"; +import { writeFile, stat, unlink, mkdir, chmod } from "fs/promises"; import { join } from "path"; -// See https://github.com/BurntSushi/ripgrep/releases -const VERSION = "14.1.1"; -const BASE = "https://github.com/BurntSushi/ripgrep/releases/download"; - -export const rgPath = join(__dirname, "rg"); +const i = __dirname.lastIndexOf("packages/backend"); +const binPath = join( + __dirname.slice(0, i + "packages/backend".length), + "node_modules/.bin", +); +export const ripgrep = join(binPath, "rg"); +export const fd = join(binPath, "fd"); + +const SPEC = { + ripgrep: { + // See https://github.com/BurntSushi/ripgrep/releases + VERSION: "14.1.1", + BASE: "https://github.com/BurntSushi/ripgrep/releases/download", + binary: "rg", + path: join(binPath, "rg"), + }, + fd: { + // See https://github.com/sharkdp/fd/releases + VERSION: "v10.2.0", + BASE: "https://github.com/sharkdp/fd/releases/download", + binary: "fd", + path: join(binPath, "fd"), + }, +} as const; + +type App = keyof typeof SPEC; + +// https://github.com/sharkdp/fd/releases/download/v10.2.0/fd-v10.2.0-x86_64-unknown-linux-musl.tar.gz +// https://github.com/BurntSushi/ripgrep/releases/download/14.1.1/ripgrep-14.1.1-x86_64-unknown-linux-musl.tar.gz + +async function exists(path: string) { + try { + await stat(path); + return true; + } catch { + return false; + } +} -export async function install() { - if (await exists(rgPath)) { +export async function install(app?: App) { + if (app == null) { + await Promise.all([install("ripgrep"), install("fd")]); + return; + } + if (app == "ripgrep" && (await exists(ripgrep))) { + return; + } + if (app == "fd" && (await exists(fd))) { return; } - const url = getUrl(); + const url = getUrl(app); // - 1. Fetch the tarball from the github url (using the fetch library) const response = await downloadFromGithub(url); const tarballBuffer = Buffer.from(await response.arrayBuffer()); @@ -43,21 +78,34 @@ export async function install() { // we have "tar tvf ripgrep-14.1.1-x86_64-unknown-linux-musl.tar.gz" outputs // ... // ripgrep-14.1.1-x86_64-unknown-linux-musl/rg - const tmpFile = join(__dirname, `ripgrep-${VERSION}.tar.gz`); - await writeFile(tmpFile, tarballBuffer); - // sync is fine since this is run at *build time*. - execFileSync("tar", [ - "xzf", - tmpFile, - "--strip-components=1", - `-C`, - __dirname, - `ripgrep-${VERSION}-${getName()}/rg`, - ]); - await unlink(tmpFile); - - // - 3. Make the file rg executable - await chmod(rgPath, 0o755); + + const { VERSION, binary, path } = SPEC[app]; + + const tmpFile = join(__dirname, `${app}-${VERSION}.tar.gz`); + try { + try { + if (!(await exists(binPath))) { + await mkdir(binPath); + } + } catch {} + await writeFile(tmpFile, tarballBuffer); + // sync is fine since this is run at *build time*. + execFileSync("tar", [ + "xzf", + tmpFile, + "--strip-components=1", + `-C`, + binPath, + `${app}-${VERSION}-${getOS()}/${binary}`, + ]); + + // - 3. Make the file rg executable + await chmod(path, 0o755); + } finally { + try { + await unlink(tmpFile); + } catch {} + } } // Download from github, but aware of rate limits, the retry-after header, etc. @@ -106,11 +154,12 @@ async function downloadFromGithub(url: string) { throw new Error("Should not reach here"); } -function getUrl() { - return `${BASE}/${VERSION}/ripgrep-${VERSION}-${getName()}.tar.gz`; +function getUrl(app: App) { + const { BASE, VERSION } = SPEC[app]; + return `${BASE}/${VERSION}/${app}-${VERSION}-${getOS()}.tar.gz`; } -function getName() { +function getOS() { switch (platform()) { case "linux": switch (arch()) { diff --git a/src/packages/backend/files/sandbox/ripgrep.test.ts b/src/packages/backend/files/sandbox/ripgrep.test.ts new file mode 100644 index 0000000000..34ba5e0d67 --- /dev/null +++ b/src/packages/backend/files/sandbox/ripgrep.test.ts @@ -0,0 +1,27 @@ +import ripgrep from "./ripgrep"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +let tempDir; +beforeAll(async () => { + tempDir = await mkdtemp(join(tmpdir(), "cocalc")); +}); +afterAll(async () => { + await rm(tempDir, { force: true, recursive: true }); +}); + +describe("ripgrep files", () => { + it("directory starts empty - no results", async () => { + const { stdout, truncated } = await ripgrep(tempDir, ""); + expect(stdout.length).toBe(0); + expect(truncated).toBe(false); + }); + + it("create a file and see it appears in the rigrep result", async () => { + await writeFile(join(tempDir, "a.txt"), "hello"); + const { stdout, truncated } = await ripgrep(tempDir, "he"); + expect(truncated).toBe(false); + expect(stdout.toString()).toEqual("a.txt:hello\n"); + }); +}); diff --git a/src/packages/backend/files/sandbox/ripgrep.ts b/src/packages/backend/files/sandbox/ripgrep.ts index f2ff751661..9fbb362816 100644 --- a/src/packages/backend/files/sandbox/ripgrep.ts +++ b/src/packages/backend/files/sandbox/ripgrep.ts @@ -1,357 +1,237 @@ -import { spawn } from "node:child_process"; -import { realpath } from "node:fs/promises"; -import * as path from "node:path"; +import exec, { type ExecOutput, validate } from "./exec"; import type { RipgrepOptions } from "@cocalc/conat/files/fs"; export type { RipgrepOptions }; -import { rgPath } from "./install-ripgrep"; - -const MAX_OUTPUT_SIZE = 10 * 1024 * 1024; // 10MB limit - -// Safely allowed options that don't pose security risks -const SAFE_OPTIONS = new Set([ - // Search behavior - "--case-sensitive", - "-s", - "--ignore-case", - "-i", - "--word-regexp", - "-w", - "--line-number", - "-n", - "--count", - "-c", - "--files-with-matches", - "-l", - "--files-without-match", - "--fixed-strings", - "-F", - "--invert-match", - "-v", - - // Output format - "--heading", - "--no-heading", - "--column", - "--pretty", - "--color", - "--no-line-number", - "-N", - - // Context lines (safe as long as we control the path) - "--context", - "-C", - "--before-context", - "-B", - "--after-context", - "-A", - - // Performance/filtering - "--max-count", - "-m", - "--max-depth", - "--max-filesize", - "--type", - "-t", - "--type-not", - "-T", - "--glob", - "-g", - "--iglob", - - // File selection - "--no-ignore", - "--hidden", - "--one-file-system", - "--null-data", - "--multiline", - "-U", - "--multiline-dotall", - "--crlf", - "--encoding", - "-E", - "--no-encoding", - - // basic info - "--version", - - // allow searching in binary files (files that contain NUL) - // this should be safe, since we're capturing output to buffers. - "--text", - "-a", - - // this ignores gitignore, hidden, and binary restrictions, which we allow - // above, so should be safe. - "--unrestricted", - "-u", - - "--debug", -]); - -// Options that take values - need special validation -const OPTIONS_WITH_VALUES = new Set([ - "--max-count", - "-m", - "--max-depth", - "--max-filesize", - "--type", - "-t", - "--type-not", - "-T", - "--glob", - "-g", - "--iglob", - "--context", - "-C", - "--before-context", - "-B", - "--after-context", - "-A", - "--encoding", - "-E", - "--color", -]); - -interface ExtendedRipgrepOptions extends RipgrepOptions { - options?: string[]; - allowedBasePath?: string; // The base path users are allowed to search within -} +import { ripgrep as ripgrepPath } from "./install"; -function validateGlobPattern(pattern: string): boolean { - // Reject patterns that could escape directory - if (pattern.includes("../") || pattern.includes("..\\")) { - return false; +export default async function ripgrep( + path: string, + pattern: string, + { options, darwin, linux, timeout, maxSize }: RipgrepOptions = {}, +): Promise { + if (path == null) { + throw Error("path must be specified"); } - // Reject absolute paths - if (path.isAbsolute(pattern)) { - return false; + if (pattern == null) { + throw Error("pattern must be specified"); } - return true; -} -function validateNumber(value: string): boolean { - return /^\d+$/.test(value); + return await exec({ + cmd: ripgrepPath, + cwd: path, + positionalArgs: [pattern], + options, + darwin, + linux, + maxSize, + timeout, + whitelist, + // if large memory usage is an issue, it might be caused by parallel interleaving; using + // -j1 below will prevent that, but will make ripgrep much slower (since not in parallel). + // See the ripgrep man page. + safety: ["--no-follow", "--block-buffered", "--no-config" /* "-j1"*/], + }); } -function validateEncoding(value: string): boolean { - // Allow only safe encodings - const safeEncodings = [ +const whitelist = { + "-e": validate.str, + + "-s": true, + "--case-sensitive": true, + + "--crlf": true, + + "-E": validate.set([ "utf-8", "utf-16", "utf-16le", "utf-16be", "ascii", "latin-1", - ]; - return safeEncodings.includes(value.toLowerCase()); -} + ]), + "--encoding": validate.set([ + "utf-8", + "utf-16", + "utf-16le", + "utf-16be", + "ascii", + "latin-1", + ]), -function parseAndValidateOptions(options: string[]): string[] { - const validatedOptions: string[] = []; - let i = 0; - - while (i < options.length) { - const opt = options[i]; - - // Check if this is a safe option - if (!SAFE_OPTIONS.has(opt)) { - throw new Error(`Disallowed option: ${opt}`); - } - - validatedOptions.push(opt); - - // Handle options that take values - if (OPTIONS_WITH_VALUES.has(opt)) { - i++; - if (i >= options.length) { - throw new Error(`Option ${opt} requires a value`); - } - - const value = options[i]; - - // Validate based on option type - if (opt === "--glob" || opt === "-g" || opt === "--iglob") { - if (!validateGlobPattern(value)) { - throw new Error(`Invalid glob pattern: ${value}`); - } - } else if ( - opt === "--max-count" || - opt === "-m" || - opt === "--max-depth" || - opt === "--context" || - opt === "-C" || - opt === "--before-context" || - opt === "-B" || - opt === "--after-context" || - opt === "-A" - ) { - if (!validateNumber(value)) { - throw new Error(`Invalid number for ${opt}: ${value}`); - } - } else if (opt === "--encoding" || opt === "-E") { - if (!validateEncoding(value)) { - throw new Error(`Invalid encoding: ${value}`); - } - } else if (opt === "--color") { - if (!["never", "auto", "always", "ansi"].includes(value)) { - throw new Error(`Invalid color option: ${value}`); - } - } - validatedOptions.push(value); - } - i++; - } - return validatedOptions; -} + "--engine": validate.set(["default", "pcre2", "auto"]), -export default async function ripgrep( - searchPath: string, - regexp: string, - { timeout = 0, options = [], allowedBasePath }: ExtendedRipgrepOptions = {}, -): Promise<{ - stdout: Buffer; - stderr: Buffer; - code: number | null; - truncated: boolean; -}> { - if (searchPath == null) { - throw Error("path must be specified"); - } - if (regexp == null) { - throw Error("regexp must be specified"); - } + "-F": true, + "--fixed-strings": true, - // Validate and normalize the search path - let normalizedPath: string; - try { - // Resolve to real path (follows symlinks to get actual path) - normalizedPath = await realpath(searchPath); - } catch (err) { - // If path doesn't exist, use normalize to check it - normalizedPath = path.normalize(searchPath); - } + "-i": true, + "--ignore-case": true, - // Security check: ensure path is within allowed base path - if (allowedBasePath) { - const normalizedBase = await realpath(allowedBasePath); - const relative = path.relative(normalizedBase, normalizedPath); + "-v": true, + "--invert-match": true, - // If relative path starts with .. or is absolute, it's outside allowed path - if (relative.startsWith("..") || path.isAbsolute(relative)) { - throw new Error("Search path is outside allowed directory"); - } - } + "-x": true, + "--line-regexp": true, - // Validate regexp doesn't contain null bytes (command injection protection) - if (regexp.includes("\0")) { - throw new Error("Invalid regexp: contains null bytes"); - } + "-m": validate.int, + "--max-count": validate.int, - // Build arguments array with security flags first - const args = [ - "--no-follow", // Don't follow symlinks - "--no-config", // Ignore config files - "--no-ignore-global", // Don't use global gitignore - "--no-require-git", // Don't require git repo - "--no-messages", // Suppress error messages that might leak info - ]; - - // Add validated user options - if (options.length > 0) { - const validatedOptions = parseAndValidateOptions(options); - args.push(...validatedOptions); - } + "-U": true, + "--multiline": true, - // Add the search pattern and path last - args.push("--", regexp, normalizedPath); // -- prevents regexp from being treated as option - - return new Promise((resolve, reject) => { - const stdoutChunks: Buffer[] = []; - const stderrChunks: Buffer[] = []; - let truncated = false; - let stdoutSize = 0; - let stderrSize = 0; - - const child = spawn(rgPath, args, { - stdio: ["ignore", "pipe", "pipe"], - env: { - // Minimal environment - only what ripgrep needs - PATH: process.env.PATH, - HOME: "/tmp", // Prevent access to user's home - RIPGREP_CONFIG_PATH: "/dev/null", // Explicitly disable config - }, - cwd: allowedBasePath || process.cwd(), // Restrict working directory - }); - - let timeoutHandle: NodeJS.Timeout | null = null; - - if (timeout > 0) { - timeoutHandle = setTimeout(() => { - truncated = true; - child.kill("SIGTERM"); - // Force kill after grace period - setTimeout(() => { - if (!child.killed) { - child.kill("SIGKILL"); - } - }, 1000); - }, timeout); - } - - child.stdout.on("data", (chunk: Buffer) => { - stdoutSize += chunk.length; - if (stdoutSize > MAX_OUTPUT_SIZE) { - truncated = true; - child.kill("SIGTERM"); - return; - } - stdoutChunks.push(chunk); - }); - - child.stderr.on("data", (chunk: Buffer) => { - stderrSize += chunk.length; - if (stderrSize > MAX_OUTPUT_SIZE) { - truncated = true; - child.kill("SIGTERM"); - return; - } - stderrChunks.push(chunk); - }); - - child.on("error", (err) => { - if (timeoutHandle) clearTimeout(timeoutHandle); - reject(err); - }); - - child.on("close", (code) => { - if (timeoutHandle) clearTimeout(timeoutHandle); - - const stdout = Buffer.concat(stdoutChunks); - const stderr = Buffer.concat(stderrChunks); - - // Truncate output if it's too large - const finalStdout = - stdout.length > MAX_OUTPUT_SIZE - ? stdout.slice(0, MAX_OUTPUT_SIZE) - : stdout; - const finalStderr = - stderr.length > MAX_OUTPUT_SIZE - ? stderr.slice(0, MAX_OUTPUT_SIZE) - : stderr; - - resolve({ - stdout: finalStdout, - stderr: finalStderr, - code, - truncated, - }); - }); - }); -} + "--multiline-dotall": true, + + "--no-unicode": true, + + "--null-data": true, + + "-P": true, + "--pcre2": true, + + "-S": true, + "--smart-case": true, + + "--stop-on-nonmatch": true, + + // this allows searching in binary files -- there is some danger of this + // using a lot more memory. Hence we do not allow it. + // "-a": true, + // "--text": true, + + "-w": true, + "--word-regexp": true, + + "--binary": true, + + "-g": validate.str, + "--glob": validate.str, + "--glob-case-insensitive": true, + + "-.": true, + "--hidden": true, + + "--iglob": validate.str, + + "--ignore-file-case-insensitive": true, + + "-d": validate.int, + "--max-depth": validate.int, + + "--max-filesize": validate.str, + + "--no-ignore": true, + "--no-ignore-dot": true, + "--no-ignore-exclude": true, + "--no-ignore-files": true, + "--no-ignore-global": true, + "--no-ignore-parent": true, + "--no-ignore-vcs": true, + "--no-require-git": true, + "--one-file-system": true, + + "-t": validate.str, + "--type": validate.str, + "-T": validate.str, + "--type-not": validate.str, + "--type-add": validate.str, + "--type-list": true, + "--type-clear": validate.str, + + "-u": true, + "--unrestricted": true, + + "-A": validate.int, + "--after-context": validate.int, + "-B": validate.int, + "--before-context": validate.int, + + "-b": true, + "--byte-offset": true, + + "--color": validate.set(["never", "auto", "always", "ansi"]), + "--colors": validate.str, + + "--column": true, + "-C": validate.int, + "--context": validate.int, + + "--context-separator": validate.str, + "--field-context-separator": validate.str, + "--field-match-separator": validate.str, + + "--heading": true, + "--no-heading": true, + + "-h": true, + "--help": true, + + "--include-zero": true, + + "-n": true, + "--line-number": true, + "-N": true, + "--no-line-number": true, + + "-M": validate.int, + "--max-columns": validate.int, + + "--max-columns-preview": validate.int, + + "-O": true, + "--null": true, + + "--passthru": true, + + "-p": true, + "--pretty": true, + + "-q": true, + "--quiet": true, + + // From the docs: "Neither this flag nor any other ripgrep flag will modify your files." + "-r": validate.str, + "--replace": validate.str, + + "--sort": validate.set(["none", "path", "modified", "accessed", "created"]), + "--sortr": validate.set(["none", "path", "modified", "accessed", "created"]), + + "--trim": true, + "--no-trim": true, + + "--vimgrep": true, + + "-H": true, + "--with-filename": true, + + "-I": true, + "--no-filename": true, + + "-c": true, + "--count": true, + + "--count-matches": true, + "-l": true, + "--files-with-matches": true, + "--files-without-match": true, + "--json": true, + + "--debug": true, + "--no-ignore-messages": true, + "--no-messages": true, + + "--stats": true, + + "--trace": true, + + "--files": true, + + "--generate": validate.set([ + "man", + "complete-bash", + "complete-zsh", + "complete-fish", + "complete-powershell", + ]), -// Export utility functions for testing -export const _internal = { - validateGlobPattern, - validateNumber, - validateEncoding, - parseAndValidateOptions, -}; + "--pcre2-version": true, + "-V": true, + "--version": true, +} as const; diff --git a/src/packages/backend/package.json b/src/packages/backend/package.json index 861ad94254..3be77288a7 100644 --- a/src/packages/backend/package.json +++ b/src/packages/backend/package.json @@ -20,7 +20,7 @@ "scripts": { "preinstall": "npx only-allow pnpm", "clean": "rm -rf dist node_modules", - "install-ripgrep": "echo 'require(\"@cocalc/backend/files/sandbox/install-ripgrep\").install()' | node", + "install-ripgrep": "echo 'require(\"@cocalc/backend/files/sandbox/install\").install()' | node", "build": "pnpm exec tsc --build && pnpm install-ripgrep", "tsc": "pnpm exec tsc --watch --pretty --preserveWatchOutput", "test": "pnpm exec jest --forceExit", diff --git a/src/packages/conat/files/fs.ts b/src/packages/conat/files/fs.ts index b3ee87786f..90bb566836 100644 --- a/src/packages/conat/files/fs.ts +++ b/src/packages/conat/files/fs.ts @@ -26,32 +26,22 @@ export interface ExecOutput { } export interface RipgrepOptions { - timeout?: number; options?: string[]; + darwin?: string[]; + linux?: string[]; + timeout?: number; + maxSize?: number; } export interface FindOptions { - // timeout is very limited (e.g., 3s?) if fs is running on file - // server (not in own project) timeout?: number; - // recursive is false by default (unlike actual find command) - recursive?: boolean; - // see typing below -- we can't just pass arbitrary args since - // that would not be secure. - expression?: FindExpression; + // all safe whitelisted options to the find command + options?: string[]; + darwin?: string[]; + linux?: string[]; + maxSize?: number; } -export type FindExpression = - | { type: "name"; pattern: string } - | { type: "iname"; pattern: string } - | { type: "type"; value: "f" | "d" | "l" } - | { type: "size"; operator: "+" | "-"; value: string } - | { type: "mtime"; operator: "+" | "-"; days: number } - | { type: "newer"; file: string } - | { type: "and"; left: FindExpression; right: FindExpression } - | { type: "or"; left: FindExpression; right: FindExpression } - | { type: "not"; expr: FindExpression }; - export interface FdOptions { pattern?: string; options?: string[]; @@ -100,11 +90,7 @@ export interface Filesystem { // find -P {path} -maxdepth 1 -mindepth 1 -printf {printf} // For security reasons, this does not support all find arguments, // and can only use limited resources. - find: ( - path: string, - printf: string, - options?: FindOptions, - ) => Promise<{ stdout: Buffer; truncated: boolean }>; + find: (path: string, options?: FindOptions) => Promise; // Convenience function that uses the find and stat support to // provide all files in a directory by using tricky options to find, @@ -122,9 +108,9 @@ export interface Filesystem { // and can only use limited resources. ripgrep: ( path: string, - regexp: string, + pattern: string, options?: RipgrepOptions, - ) => Promise<{ stdout: Buffer; stderr: Buffer; truncated: boolean }>; + ) => Promise; } interface IDirent { @@ -282,8 +268,8 @@ export async function fsServer({ service, fs, client, project_id }: Options) { async fd(path: string, options?: FdOptions) { return await (await fs(this.subject)).fd(path, options); }, - async find(path: string, printf: string, options?: FindOptions) { - return await (await fs(this.subject)).find(path, printf, options); + async find(path: string, options?: FindOptions) { + return await (await fs(this.subject)).find(path, options); }, async link(existingPath: string, newPath: string) { await (await fs(this.subject)).link(existingPath, newPath); @@ -317,8 +303,8 @@ export async function fsServer({ service, fs, client, project_id }: Options) { async rename(oldPath: string, newPath: string) { await (await fs(this.subject)).rename(oldPath, newPath); }, - async ripgrep(path: string, regexp: string, options?: RipgrepOptions) { - return await (await fs(this.subject)).ripgrep(path, regexp, options); + async ripgrep(path: string, pattern: string, options?: RipgrepOptions) { + return await (await fs(this.subject)).ripgrep(path, pattern, options); }, async rm(path: string, options?) { await (await fs(this.subject)).rm(path, options); diff --git a/src/packages/conat/files/listing.ts b/src/packages/conat/files/listing.ts index e1836510d3..ddfd133346 100644 --- a/src/packages/conat/files/listing.ts +++ b/src/packages/conat/files/listing.ts @@ -146,11 +146,17 @@ export class Listing extends EventEmitter { async function getListing( fs: FilesystemClient, path: string, -): Promise<{ files: Files; truncated: boolean }> { - const { stdout, truncated } = await fs.find( - path, - "%f\\0%T@\\0%s\\0%y\\0%l\n", - ); +): Promise<{ files: Files; truncated?: boolean }> { + const { stdout, truncated } = await fs.find(path, { + options: [ + "-maxdepth", + "1", + "-mindepth", + "1", + "-printf", + "%f\\0%T@\\0%s\\0%y\\0%l\n", + ], + }); const buf = Buffer.from(stdout); const files: Files = {}; // todo -- what about non-utf8...? diff --git a/src/packages/frontend/project/search/body.tsx b/src/packages/frontend/project/search/body.tsx index dbf142f546..7b98eb0c7f 100644 --- a/src/packages/frontend/project/search/body.tsx +++ b/src/packages/frontend/project/search/body.tsx @@ -298,22 +298,7 @@ function ProjectSearchInput({ type="primary" onClick={() => actions?.search()} > - {neural ? ( - <> - - {small ? "" : " Neural Search"} - - ) : git ? ( - <> - - {small ? "" : " Git Grep Search"} - - ) : ( - <> - - {small ? "" : " Grep Search"} - - )} + Search } /> From 348f005276223718a3e3aafee9b541310263f345 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 3 Aug 2025 05:06:38 +0000 Subject: [PATCH 171/270] refactor search action function -- first step before rewriting it to use ripgrep, etc. --- src/packages/frontend/project/search/body.tsx | 21 +- src/packages/frontend/project/search/run.ts | 186 +++++++++++++++++ src/packages/frontend/project_actions.ts | 193 +++--------------- 3 files changed, 220 insertions(+), 180 deletions(-) create mode 100644 src/packages/frontend/project/search/run.ts diff --git a/src/packages/frontend/project/search/body.tsx b/src/packages/frontend/project/search/body.tsx index 7b98eb0c7f..ef3a2b66e4 100644 --- a/src/packages/frontend/project/search/body.tsx +++ b/src/packages/frontend/project/search/body.tsx @@ -84,11 +84,7 @@ export const ProjectSearchBody: React.FC<{ return ( - + {mode != "flyout" ? ( ) : undefined} @@ -151,12 +147,7 @@ export const ProjectSearchBody: React.FC<{ function renderHeaderFlyout() { return (
- + actions?.setState({ user_input: value })} on_submit={() => actions?.search()} on_clear={() => diff --git a/src/packages/frontend/project/search/run.ts b/src/packages/frontend/project/search/run.ts new file mode 100644 index 0000000000..18eef83742 --- /dev/null +++ b/src/packages/frontend/project/search/run.ts @@ -0,0 +1,186 @@ +import { type FilesystemClient } from "@cocalc/conat/files/fs"; +import { MARKERS } from "@cocalc/util/sagews"; +import { webapp_client } from "@cocalc/frontend/webapp-client"; + +export async function search({ + query, + path, + setState, + fs: _fs, + options = {}, + project_id, + compute_server_id, +}: { + query: string; + path: string; + setState: (any) => void; + fs: FilesystemClient; + options: { + case_sensitive?: boolean; + git_grep?: boolean; + subdirectories?: boolean; + hidden_files?: boolean; + }; + project_id: string; + compute_server_id: number; +}) { + if (!query) { + return; + } + + query = query.trim().replace(/"/g, '\\"'); + if (query === "") { + return; + } + const search_query = `"${query}"`; + setState({ + search_results: undefined, + search_error: undefined, + most_recent_search: query, + most_recent_path: path, + too_many_results: false, + }); + + // generate the grep command for the given query with the given flags + let cmd, ins; + if (options.case_sensitive) { + ins = ""; + } else { + ins = " -i "; + } + + if (options.git_grep) { + let max_depth; + if (options.subdirectories) { + max_depth = ""; + } else { + max_depth = "--max-depth=0"; + } + // The || true is so that if git rev-parse has exit code 0, + // but "git grep" finds nothing (hence has exit code 1), we don't + // fall back to normal git (the other side of the ||). See + // https://github.com/sagemathinc/cocalc/issues/4276 + cmd = `git rev-parse --is-inside-work-tree && (git grep -n -I -H ${ins} ${max_depth} ${search_query} || true) || `; + } else { + cmd = ""; + } + if (options.subdirectories) { + if (options.hidden_files) { + cmd += `rgrep -n -I -H --exclude-dir=.smc --exclude-dir=.snapshots ${ins} ${search_query} -- *`; + } else { + cmd += `rgrep -n -I -H --exclude-dir='.*' --exclude='.*' ${ins} ${search_query} -- *`; + } + } else { + if (options.hidden_files) { + cmd += `grep -n -I -H ${ins} ${search_query} -- .* *`; + } else { + cmd += `grep -n -I -H ${ins} ${search_query} -- *`; + } + } + + cmd += ` | grep -v ${MARKERS.cell}`; + const max_results = 1000; + const max_output = 110 * max_results; // just in case + + setState({ + command: cmd, + }); + + let output; + try { + output = await webapp_client.exec({ + project_id, + command: cmd + " | cut -c 1-256", // truncate horizontal line length (imagine a binary file that is one very long line) + timeout: 20, // how long grep runs on client + max_output, + bash: true, + err_on_exit: true, + compute_server_id, + filesystem: true, + path, + }); + } catch (err) { + processResults({ err, setState }); + return; + } + processResults({ + output, + max_results, + max_output, + setState, + }); +} + +function processResults({ + err, + output, + max_results, + max_output, + setState, +}: { + err?; + output?; + max_results?; + max_output?; + setState; +}) { + if (err) { + err = `${err}`; + } + if ((err && output == null) || (output != null && output.stdout == null)) { + setState({ search_error: err }); + return; + } + + const results = output.stdout.split("\n"); + const too_many_results = !!( + output.stdout.length >= max_output || + results.length > max_results || + err + ); + let num_results = 0; + const search_results: {}[] = []; + for (const line of results) { + if (line.trim() === "") { + continue; + } + let i = line.indexOf(":"); + num_results += 1; + if (i !== -1) { + // all valid lines have a ':', the last line may have been truncated too early + let filename = line.slice(0, i); + if (filename.slice(0, 2) === "./") { + filename = filename.slice(2); + } + let context = line.slice(i + 1); + // strip codes in worksheet output + if (context.length > 0 && context[0] === MARKERS.output) { + i = context.slice(1).indexOf(MARKERS.output); + context = context.slice(i + 2, context.length - 1); + } + + const m = /^(\d+):/.exec(context); + let line_number: number | undefined; + if (m != null) { + try { + line_number = parseInt(m[1]); + } catch (e) {} + } + + search_results.push({ + filename, + description: context, + line_number, + filter: `${filename.toLowerCase()} ${context.toLowerCase()}`, + }); + } + if (num_results >= max_results) { + break; + } + } + + setState({ + too_many_results, + search_results, + }); +} diff --git a/src/packages/frontend/project_actions.ts b/src/packages/frontend/project_actions.ts index 6e18614b23..abe12a9d31 100644 --- a/src/packages/frontend/project_actions.ts +++ b/src/packages/frontend/project_actions.ts @@ -104,7 +104,6 @@ import { DEFAULT_NEW_FILENAMES, NEW_FILENAMES } from "@cocalc/util/db-schema"; import * as misc from "@cocalc/util/misc"; import { reduxNameToProjectId } from "@cocalc/util/redux/name"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; -import { MARKERS } from "@cocalc/util/sagews"; import { client_db } from "@cocalc/util/schema"; import { get_editor } from "./editors/react-wrapper"; import { type FilesystemClient } from "@cocalc/conat/files/fs"; @@ -114,6 +113,7 @@ import { type Files, } from "@cocalc/frontend/project/listing/use-files"; import { map as awaitMap } from "awaiting"; +import { search } from "@cocalc/frontend/project/search/run"; const { defaults, required } = misc; @@ -3074,167 +3074,6 @@ export class ProjectActions extends Actions { redux.getActions("account")?.set_other_settings("find_git_grep", git_grep); } - process_search_results(err, output, max_results, max_output, cmd) { - const store = this.get_store(); - if (store == undefined) { - return; - } - if (err) { - err = misc.to_user_string(err); - } - if ((err && output == null) || (output != null && output.stdout == null)) { - this.setState({ search_error: err }); - return; - } - - const results = output.stdout.split("\n"); - const too_many_results = !!( - output.stdout.length >= max_output || - results.length > max_results || - err - ); - let num_results = 0; - const search_results: {}[] = []; - for (const line of results) { - if (line.trim() === "") { - continue; - } - let i = line.indexOf(":"); - num_results += 1; - if (i !== -1) { - // all valid lines have a ':', the last line may have been truncated too early - let filename = line.slice(0, i); - if (filename.slice(0, 2) === "./") { - filename = filename.slice(2); - } - let context = line.slice(i + 1); - // strip codes in worksheet output - if (context.length > 0 && context[0] === MARKERS.output) { - i = context.slice(1).indexOf(MARKERS.output); - context = context.slice(i + 2, context.length - 1); - } - - const m = /^(\d+):/.exec(context); - let line_number: number | undefined; - if (m != null) { - try { - line_number = parseInt(m[1]); - } catch (e) {} - } - - search_results.push({ - filename, - description: context, - line_number, - filter: `${filename.toLowerCase()} ${context.toLowerCase()}`, - }); - } - if (num_results >= max_results) { - break; - } - } - - if (store.get("command") === cmd) { - // only update the state if the results are from the most recent command - this.setState({ - too_many_results, - search_results, - }); - } - } - - search = () => { - let cmd, ins; - const store = this.get_store(); - if (store == undefined) { - return; - } - - const query = store.get("user_input").trim().replace(/"/g, '\\"'); - if (query === "") { - return; - } - const search_query = `"${query}"`; - this.setState({ - search_results: undefined, - search_error: undefined, - most_recent_search: query, - most_recent_path: store.get("current_path"), - too_many_results: false, - }); - const path = store.get("current_path"); - - track("search", { - project_id: this.project_id, - path, - query, - neural_search: store.get("neural_search"), - subdirectories: store.get("subdirectories"), - hidden_files: store.get("hidden_files"), - git_grep: store.get("git_grep"), - }); - - // generate the grep command for the given query with the given flags - if (store.get("case_sensitive")) { - ins = ""; - } else { - ins = " -i "; - } - - if (store.get("git_grep")) { - let max_depth; - if (store.get("subdirectories")) { - max_depth = ""; - } else { - max_depth = "--max-depth=0"; - } - // The || true is so that if git rev-parse has exit code 0, - // but "git grep" finds nothing (hence has exit code 1), we don't - // fall back to normal git (the other side of the ||). See - // https://github.com/sagemathinc/cocalc/issues/4276 - cmd = `git rev-parse --is-inside-work-tree && (git grep -n -I -H ${ins} ${max_depth} ${search_query} || true) || `; - } else { - cmd = ""; - } - if (store.get("subdirectories")) { - if (store.get("hidden_files")) { - cmd += `rgrep -n -I -H --exclude-dir=.smc --exclude-dir=.snapshots ${ins} ${search_query} -- *`; - } else { - cmd += `rgrep -n -I -H --exclude-dir='.*' --exclude='.*' ${ins} ${search_query} -- *`; - } - } else { - if (store.get("hidden_files")) { - cmd += `grep -n -I -H ${ins} ${search_query} -- .* *`; - } else { - cmd += `grep -n -I -H ${ins} ${search_query} -- *`; - } - } - - cmd += ` | grep -v ${MARKERS.cell}`; - const max_results = 1000; - const max_output = 110 * max_results; // just in case - - this.setState({ - command: cmd, - }); - - const compute_server_id = this.getComputeServerId(); - webapp_client.exec({ - project_id: this.project_id, - command: cmd + " | cut -c 1-256", // truncate horizontal line length (imagine a binary file that is one very long line) - timeout: 20, // how long grep runs on client - max_output, - bash: true, - err_on_exit: true, - compute_server_id, - filesystem: true, - path: store.get("current_path"), - cb: (err, output) => { - this.process_search_results(err, output, max_results, max_output, cmd); - }, - }); - }; - set_file_listing_scroll(scroll_top) { this.setState({ file_listing_scroll_top: scroll_top }); } @@ -3703,4 +3542,34 @@ export class ProjectActions extends Actions { project_id: this.project_id, }); }; + + private searchId = 0; + search = async () => { + const store = this.get_store(); + if (!store) { + return; + } + const searchId = ++this.searchId; + const setState = (x) => { + if (this.searchId != searchId) { + // there's a newer search + return; + } + this.setState(x); + }; + await search({ + setState, + fs: this.fs(), + query: store.get("user_input").trim(), + path: store.get("current_path"), + project_id: this.project_id, + compute_server_id: this.getComputeServerId(), + options: { + case_sensitive: store.get("case_sensitive"), + git_grep: store.get("git_grep"), + subdirectories: store.get("subdirectories"), + hidden_files: store.get("hidden_files"), + }, + }); + }; } From 5cbff75ab6624215d4431a0f799590c154259568 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 3 Aug 2025 05:19:53 +0000 Subject: [PATCH 172/270] add dust -- still need to add whitelisted commands... --- src/packages/backend/files/sandbox/dust.ts | 29 +++++++++++++++++++ src/packages/backend/files/sandbox/index.ts | 26 +++++++++-------- src/packages/backend/files/sandbox/install.ts | 25 +++++++++++----- src/packages/conat/files/fs.ts | 14 +++++++++ 4 files changed, 75 insertions(+), 19 deletions(-) create mode 100644 src/packages/backend/files/sandbox/dust.ts diff --git a/src/packages/backend/files/sandbox/dust.ts b/src/packages/backend/files/sandbox/dust.ts new file mode 100644 index 0000000000..e5be132bd2 --- /dev/null +++ b/src/packages/backend/files/sandbox/dust.ts @@ -0,0 +1,29 @@ +import exec, { type ExecOutput /* validate */ } from "./exec"; +import { type DustOptions } from "@cocalc/conat/files/fs"; +export { type DustOptions }; +import { dust as dustPath } from "./install"; + +export default async function dust( + path: string, + { options, darwin, linux, timeout, maxSize }: DustOptions = {}, +): Promise { + if (path == null) { + throw Error("path must be specified"); + } + + return await exec({ + cmd: dustPath, + cwd: path, + positionalArgs: [path], + options, + darwin, + linux, + maxSize, + timeout, + whitelist, + }); +} + +const whitelist = { + "-j": true, +} as const; diff --git a/src/packages/backend/files/sandbox/index.ts b/src/packages/backend/files/sandbox/index.ts index 7bb111943a..2788f29017 100644 --- a/src/packages/backend/files/sandbox/index.ts +++ b/src/packages/backend/files/sandbox/index.ts @@ -72,17 +72,12 @@ import { type WatchOptions } from "@cocalc/conat/files/watch"; import find, { type FindOptions } from "./find"; import ripgrep, { type RipgrepOptions } from "./ripgrep"; import fd, { type FdOptions } from "./fd"; +import dust, { type DustOptions } from "./dust"; import { type ExecOutput } from "./exec"; -// max time a user find request can run (in safe mode) -- this can cause excessive -// load on a server if there were a directory with a massive number of files, -// so must be limited. -const MAX_FIND_TIMEOUT = 3000; - -// max time a user ripgrep can run (when in safe mode) -const MAX_RIPGREP_TIMEOUT = 3000; - -const MAX_FD_TIMEOUT = 3000; +// max time code can run (in safe mode), e.g., for find, +// ripgrep, fd, and dust. +const MAX_TIMEOUT = 5000; interface Options { // unsafeMode -- if true, assume security model where user is running this @@ -210,14 +205,21 @@ export class SandboxedFilesystem { find = async (path: string, options?: FindOptions): Promise => { return await find( await this.safeAbsPath(path), - capTimeout(options, MAX_FIND_TIMEOUT), + capTimeout(options, MAX_TIMEOUT), ); }; fd = async (path: string, options?: FdOptions): Promise => { return await fd( await this.safeAbsPath(path), - capTimeout(options, MAX_FD_TIMEOUT), + capTimeout(options, MAX_TIMEOUT), + ); + }; + + dust = async (path: string, options?: DustOptions): Promise => { + return await dust( + await this.safeAbsPath(path), + capTimeout(options, MAX_TIMEOUT), ); }; @@ -229,7 +231,7 @@ export class SandboxedFilesystem { return await ripgrep( await this.safeAbsPath(path), pattern, - capTimeout(options, MAX_RIPGREP_TIMEOUT), + capTimeout(options, MAX_TIMEOUT), ); }; diff --git a/src/packages/backend/files/sandbox/install.ts b/src/packages/backend/files/sandbox/install.ts index 7f5fe1030b..dfd396c46d 100644 --- a/src/packages/backend/files/sandbox/install.ts +++ b/src/packages/backend/files/sandbox/install.ts @@ -22,8 +22,6 @@ const binPath = join( __dirname.slice(0, i + "packages/backend".length), "node_modules/.bin", ); -export const ripgrep = join(binPath, "rg"); -export const fd = join(binPath, "fd"); const SPEC = { ripgrep: { @@ -40,8 +38,19 @@ const SPEC = { binary: "fd", path: join(binPath, "fd"), }, + dust: { + // See https://github.com/bootandy/dust/releases + VERSION: "v1.2.3", + BASE: "https://github.com/bootandy/dust/releases/download", + binary: "dust", + path: join(binPath, "dust"), + }, } as const; +export const ripgrep = SPEC.ripgrep.path; +export const fd = SPEC.fd.path; +export const dust = SPEC.dust.path; + type App = keyof typeof SPEC; // https://github.com/sharkdp/fd/releases/download/v10.2.0/fd-v10.2.0-x86_64-unknown-linux-musl.tar.gz @@ -56,15 +65,17 @@ async function exists(path: string) { } } +async function alreadyInstalled(app: App) { + return await exists(SPEC[app].path); +} + export async function install(app?: App) { if (app == null) { - await Promise.all([install("ripgrep"), install("fd")]); - return; - } - if (app == "ripgrep" && (await exists(ripgrep))) { + // @ts-ignore + await Promise.all(Object.keys(SPEC).map(install)); return; } - if (app == "fd" && (await exists(fd))) { + if (await alreadyInstalled(app)) { return; } const url = getUrl(app); diff --git a/src/packages/conat/files/fs.ts b/src/packages/conat/files/fs.ts index 90bb566836..a5f9d3c4b0 100644 --- a/src/packages/conat/files/fs.ts +++ b/src/packages/conat/files/fs.ts @@ -51,6 +51,14 @@ export interface FdOptions { maxSize?: number; } +export interface DustOptions { + options?: string[]; + darwin?: string[]; + linux?: string[]; + timeout?: number; + maxSize?: number; +} + export interface Filesystem { appendFile: (path: string, data: string | Buffer, encoding?) => Promise; chmod: (path: string, mode: string | number) => Promise; @@ -102,6 +110,9 @@ export interface Filesystem { // options: { type: "name", pattern:"^\.DS_Store$" } fd: (path: string, options?: FdOptions) => Promise; + // dust is an amazing disk space tool + dust: (path: string, options?: DustOptions) => Promise; + // We add ripgrep, as a 1-call way to very efficiently search in files // directly on whatever is serving files. // For security reasons, this does not support all ripgrep arguments, @@ -262,6 +273,9 @@ export async function fsServer({ service, fs, client, project_id }: Options) { async cp(src: string, dest: string, options?) { await (await fs(this.subject)).cp(src, dest, options); }, + async dust(path: string, options?: DustOptions) { + return await (await fs(this.subject)).dust(path, options); + }, async exists(path: string): Promise { return await (await fs(this.subject)).exists(path); }, From 7283c139f66e8bece46342367b5fd19e728f876c Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 3 Aug 2025 15:54:19 +0000 Subject: [PATCH 173/270] add dust whitelist and test --- .../backend/files/sandbox/dust.test.ts | 39 ++++++++ src/packages/backend/files/sandbox/dust.ts | 94 ++++++++++++++++++- src/packages/backend/files/sandbox/index.ts | 4 +- 3 files changed, 135 insertions(+), 2 deletions(-) create mode 100644 src/packages/backend/files/sandbox/dust.test.ts diff --git a/src/packages/backend/files/sandbox/dust.test.ts b/src/packages/backend/files/sandbox/dust.test.ts new file mode 100644 index 0000000000..cd69a56951 --- /dev/null +++ b/src/packages/backend/files/sandbox/dust.test.ts @@ -0,0 +1,39 @@ +import dust from "./dust"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +let tempDir; +beforeAll(async () => { + tempDir = await mkdtemp(join(tmpdir(), "cocalc")); +}); +afterAll(async () => { + await rm(tempDir, { force: true, recursive: true }); +}); + +describe("dust works", () => { + it("directory starts empty - no results", async () => { + const { stdout, truncated } = await dust(tempDir, { options: ["-j"] }); + const s = JSON.parse(Buffer.from(stdout).toString()); + expect(s).toEqual({ children: [], name: tempDir, size: s.size }); + expect(truncated).toBe(false); + }); + + it("create a file and see it appears in the dust result", async () => { + await writeFile(join(tempDir, "a.txt"), "hello"); + const { stdout, truncated } = await dust(tempDir, { options: ["-j"] }); + const s = JSON.parse(Buffer.from(stdout).toString()); + expect(s).toEqual({ + size: s.size, + name: tempDir, + children: [ + { + size: s.children[0].size, + name: join(tempDir, "a.txt"), + children: [], + }, + ], + }); + expect(truncated).toBe(false); + }); +}); diff --git a/src/packages/backend/files/sandbox/dust.ts b/src/packages/backend/files/sandbox/dust.ts index e5be132bd2..1e5043ccc1 100644 --- a/src/packages/backend/files/sandbox/dust.ts +++ b/src/packages/backend/files/sandbox/dust.ts @@ -1,4 +1,4 @@ -import exec, { type ExecOutput /* validate */ } from "./exec"; +import exec, { type ExecOutput, validate } from "./exec"; import { type DustOptions } from "@cocalc/conat/files/fs"; export { type DustOptions }; import { dust as dustPath } from "./install"; @@ -25,5 +25,97 @@ export default async function dust( } const whitelist = { + "-d": validate.int, + "--depth": validate.int, + + "-n": validate.int, + "--number-of-lines": validate.int, + + "-p": true, + "--full-paths": true, + + "-X": validate.str, + "--ignore-directory": validate.str, + + "-x": true, + "--limit-filesystem": true, + + "-s": true, + "--apparent-size": true, + + "-r": true, + "--reverse": true, + + "-c": true, + "--no-colors": true, + "-C": true, + "--force-colors": true, + + "-b": true, + "--no-percent-bars": true, + + "-B": true, + "--bars-on-right": true, + + "-z": validate.str, + "--min-size": validate.str, + + "-R": true, + "--screen-reader": true, + + "--skip-total": true, + + "-f": true, + "--filecount": true, + + "-i": true, + "--ignore-hidden": true, + + "-v": validate.str, + "--invert-filter": validate.str, + + "-e": validate.str, + "--filter": validate.str, + + "-t": validate.str, + "--file-types": validate.str, + + "-w": validate.int, + "--terminal-width": validate.int, + + "-P": true, + "--no-progress": true, + + "--print-errors": true, + + "-D": true, + "--only-dir": true, + + "-F": true, + "--only-file": true, + + "-o": validate.str, + "--output-format": validate.str, + "-j": true, + "--output-json": true, + + "-M": validate.str, + "--mtime": validate.str, + + "-A": validate.str, + "--atime": validate.str, + + "-y": validate.str, + "--ctime": validate.str, + + "--collapse": validate.str, + + "-m": validate.set(["a", "c", "m"]), + "--filetime": validate.set(["a", "c", "m"]), + + "-h": true, + "--help": true, + "-V": true, + "--version": true, } as const; diff --git a/src/packages/backend/files/sandbox/index.ts b/src/packages/backend/files/sandbox/index.ts index 2788f29017..b80ba3b53b 100644 --- a/src/packages/backend/files/sandbox/index.ts +++ b/src/packages/backend/files/sandbox/index.ts @@ -219,7 +219,9 @@ export class SandboxedFilesystem { dust = async (path: string, options?: DustOptions): Promise => { return await dust( await this.safeAbsPath(path), - capTimeout(options, MAX_TIMEOUT), + // dust reasonably takes longer than the other commands and is used less, + // so for now we give it more breathing room. + capTimeout(options, 4*MAX_TIMEOUT), ); }; From 17fa2ca39fdffe78813db3427fb7d00837efc17f Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 3 Aug 2025 20:21:47 +0000 Subject: [PATCH 174/270] no longer using vscode/ripgrep --- src/packages/backend/package.json | 1 - src/packages/pnpm-lock.yaml | 39 ------------------------------- 2 files changed, 40 deletions(-) diff --git a/src/packages/backend/package.json b/src/packages/backend/package.json index 3be77288a7..c74cf851ab 100644 --- a/src/packages/backend/package.json +++ b/src/packages/backend/package.json @@ -46,7 +46,6 @@ "@cocalc/backend": "workspace:*", "@cocalc/conat": "workspace:*", "@cocalc/util": "workspace:*", - "@vscode/ripgrep": "^1.15.14", "awaiting": "^3.0.0", "better-sqlite3": "^12.2.0", "chokidar": "^3.6.0", diff --git a/src/packages/pnpm-lock.yaml b/src/packages/pnpm-lock.yaml index 8781b7180e..00ebbf5056 100644 --- a/src/packages/pnpm-lock.yaml +++ b/src/packages/pnpm-lock.yaml @@ -87,9 +87,6 @@ importers: '@cocalc/util': specifier: workspace:* version: link:../util - '@vscode/ripgrep': - specifier: ^1.15.14 - version: 1.15.14 awaiting: specifier: ^3.0.0 version: 3.0.0 @@ -4571,9 +4568,6 @@ packages: peerDependencies: react: '>= 16.8.0' - '@vscode/ripgrep@1.15.14': - resolution: {integrity: sha512-/G1UJPYlm+trBWQ6cMO3sv6b8D1+G16WaJH1/DSqw32JOVlzgZbLkDxRyzIpTpv30AcYGMkCf5tUqGlW6HbDWw==} - '@vscode/vscode-languagedetection@1.0.22': resolution: {integrity: sha512-rQ/BgMyLuIXSmbA0MSkIPHtcOw14QkeDbAq19sjvaS9LTRr905yij0S8lsyqN5JgOsbtIx7pAcyOxFMzPmqhZQ==} hasBin: true @@ -5144,9 +5138,6 @@ packages: bser@2.1.1: resolution: {integrity: sha512-gQxTNE/GAfIIrmHLUE3oJyp5FO6HRBfhjnw4/wMmA63ZGDJnWBmgY/lyQBpnDUkGmAhbSe39tx2d/iTOAfglwQ==} - buffer-crc32@0.2.13: - resolution: {integrity: sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==} - buffer-equal-constant-time@1.0.1: resolution: {integrity: sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==} @@ -6720,9 +6711,6 @@ packages: fb-watchman@2.0.2: resolution: {integrity: sha512-p5161BqbuCaSnB8jIbzQHOlpgsPmK5rJVDfDKO91Axs5NC1uu3HRQm6wt9cd9/+GtQQIO53JdGXXoyDpTAsgYA==} - fd-slicer@1.1.0: - resolution: {integrity: sha512-cE1qsB/VwyQozZ+q1dGxR8LBYNZeofhEdUNGSMbQD3Gw2lAzX9Zb3uIU6Ebc/Fmyjo9AWWfnn0AUCHqtevs/8g==} - fdir@6.4.6: resolution: {integrity: sha512-hiFoqpyZcfNm1yc4u8oWCf9A2c4D3QjCrks3zmoVKVxpQRzmPNar1hUJcBG2RQHvEVGDN+Jm81ZheVLAQMK6+w==} peerDependencies: @@ -9276,9 +9264,6 @@ packages: resolution: {integrity: sha512-/Y3fcFrXEAsMjJXeL9J8+ZG9U01LbuWaYypvDW2ycW1jL269L3js3DVBjDJ0Up9Np1uqDXsDrRihHANhZOlwdQ==} engines: {node: '>=20'} - pend@1.2.0: - resolution: {integrity: sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==} - performance-now@2.1.0: resolution: {integrity: sha512-7EAHlyLHI56VEIdK57uwHdHKIaAGbnXPiw0yWbarQZOKaKpvUIgW0jWRVLiatnM+XXlSwsanIBH/hzGMJulMow==} @@ -11750,9 +11735,6 @@ packages: resolution: {integrity: sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==} engines: {node: '>=12'} - yauzl@2.10.0: - resolution: {integrity: sha512-p4a9I6X6nu6IhoGmBqAcbJy1mlC4j27vEPZX9F4L4/vZT3Lyq1VkFHw/V/PUcB9Buo+DG3iHkT0x3Qya58zc3g==} - yjs@13.6.27: resolution: {integrity: sha512-OIDwaflOaq4wC6YlPBy2L6ceKeKuF7DeTxx+jPzv1FHn9tCZ0ZwSRnUBxD05E3yed46fv/FWJbvR+Ud7x0L7zw==} engines: {node: '>=16.0.0', npm: '>=8.0.0'} @@ -14923,14 +14905,6 @@ snapshots: '@use-gesture/core': 10.3.1 react: 19.1.0 - '@vscode/ripgrep@1.15.14': - dependencies: - https-proxy-agent: 7.0.6 - proxy-from-env: 1.1.0 - yauzl: 2.10.0 - transitivePeerDependencies: - - supports-color - '@vscode/vscode-languagedetection@1.0.22': {} '@webassemblyjs/ast@1.14.1': @@ -15598,8 +15572,6 @@ snapshots: dependencies: node-int64: 0.4.0 - buffer-crc32@0.2.13: {} - buffer-equal-constant-time@1.0.1: {} buffer-from@1.1.2: {} @@ -17452,10 +17424,6 @@ snapshots: dependencies: bser: 2.1.1 - fd-slicer@1.1.0: - dependencies: - pend: 1.2.0 - fdir@6.4.6(picomatch@4.0.2): optionalDependencies: picomatch: 4.0.2 @@ -20557,8 +20525,6 @@ snapshots: optionalDependencies: '@napi-rs/canvas': 0.1.73 - pend@1.2.0: {} - performance-now@2.1.0: {} pg-cloudflare@1.2.7: @@ -23492,11 +23458,6 @@ snapshots: y18n: 5.0.8 yargs-parser: 21.1.1 - yauzl@2.10.0: - dependencies: - buffer-crc32: 0.2.13 - fd-slicer: 1.1.0 - yjs@13.6.27: dependencies: lib0: 0.2.109 From 61eda29753a2680ccfafb770705e7658f858919a Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 3 Aug 2025 20:31:01 +0000 Subject: [PATCH 175/270] update find unit test --- src/packages/backend/conat/files/test/local-path.test.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/packages/backend/conat/files/test/local-path.test.ts b/src/packages/backend/conat/files/test/local-path.test.ts index b7fdfc156d..88bbeaf783 100644 --- a/src/packages/backend/conat/files/test/local-path.test.ts +++ b/src/packages/backend/conat/files/test/local-path.test.ts @@ -197,7 +197,9 @@ describe("use all the standard api functions of fs", () => { }); it("use the find command instead of readdir", async () => { - const { stdout } = await fs.find("dirtest", "%f\n"); + const { stdout } = await fs.find("dirtest", { + options: ["-maxdepth", "1", "-mindepth", "1", "-printf", "%f\n"], + }); const v = stdout.toString().trim().split("\n"); // output of find is NOT in alphabetical order: expect(new Set(v)).toEqual(new Set(["0", "1", "2", "3", "4", fire])); From e1fc9f96b1284d147b117686596c39e1d7e58edd Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 3 Aug 2025 23:03:30 +0000 Subject: [PATCH 176/270] unsafe proof of concept rustic integration --- src/packages/backend/data.ts | 2 + src/packages/backend/files/sandbox/index.ts | 15 ++++++- src/packages/backend/files/sandbox/install.ts | 33 ++++++++++++-- src/packages/backend/files/sandbox/rustic.ts | 43 +++++++++++++++++++ src/packages/conat/files/fs.ts | 5 +++ 5 files changed, 93 insertions(+), 5 deletions(-) create mode 100644 src/packages/backend/files/sandbox/rustic.ts diff --git a/src/packages/backend/data.ts b/src/packages/backend/data.ts index a95c19dd7f..507c7b9291 100644 --- a/src/packages/backend/data.ts +++ b/src/packages/backend/data.ts @@ -180,6 +180,8 @@ export const pgdatabase: string = export const projects: string = process.env.PROJECTS ?? join(data, "projects", "[project_id]"); export const secrets: string = process.env.SECRETS ?? join(data, "secrets"); +export const rusticRepo: string = + process.env.RUSTIC_REPO ?? join(data, "rustic"); // Where the sqlite database files used for sync are stored. // The idea is there is one very fast *ephemeral* directory diff --git a/src/packages/backend/files/sandbox/index.ts b/src/packages/backend/files/sandbox/index.ts index b80ba3b53b..90946e7c62 100644 --- a/src/packages/backend/files/sandbox/index.ts +++ b/src/packages/backend/files/sandbox/index.ts @@ -73,7 +73,9 @@ import find, { type FindOptions } from "./find"; import ripgrep, { type RipgrepOptions } from "./ripgrep"; import fd, { type FdOptions } from "./fd"; import dust, { type DustOptions } from "./dust"; +import rustic from "./rustic"; import { type ExecOutput } from "./exec"; +import { rusticRepo } from "@cocalc/backend/data"; // max time code can run (in safe mode), e.g., for find, // ripgrep, fd, and dust. @@ -96,11 +98,13 @@ const INTERNAL_METHODS = new Set([ "unsafeMode", "readonly", "assertWritable", + "rusticRepo", ]); export class SandboxedFilesystem { public readonly unsafeMode: boolean; public readonly readonly: boolean; + private readonly rusticRepo: string = rusticRepo; constructor( // path should be the path to a FOLDER on the filesystem (not a file) public readonly path: string, @@ -221,10 +225,19 @@ export class SandboxedFilesystem { await this.safeAbsPath(path), // dust reasonably takes longer than the other commands and is used less, // so for now we give it more breathing room. - capTimeout(options, 4*MAX_TIMEOUT), + capTimeout(options, 4 * MAX_TIMEOUT), ); }; + rustic = async (args: string[]): Promise => { + return await rustic(args, { + repo: this.rusticRepo, + safeAbsPath: this.safeAbsPath, + timeout: 120_000, + maxSize: 10_000, + }); + }; + ripgrep = async ( path: string, pattern: string, diff --git a/src/packages/backend/files/sandbox/install.ts b/src/packages/backend/files/sandbox/install.ts index dfd396c46d..7b5bd947c8 100644 --- a/src/packages/backend/files/sandbox/install.ts +++ b/src/packages/backend/files/sandbox/install.ts @@ -23,6 +23,15 @@ const binPath = join( "node_modules/.bin", ); +interface Spec { + VERSION: string; + BASE: string; + binary: string; + path: string; + stripComponents?: number; + pathInArchive?: string; +} + const SPEC = { ripgrep: { // See https://github.com/BurntSushi/ripgrep/releases @@ -45,11 +54,21 @@ const SPEC = { binary: "dust", path: join(binPath, "dust"), }, -} as const; + rustic: { + // See https://github.com/rustic-rs/rustic/releases + VERSION: "v0.9.5", + BASE: "https://github.com/rustic-rs/rustic/releases/download", + binary: "rustic", + path: join(binPath, "rustic"), + stripComponents: 0, + pathInArchive: "rustic", + }, +}; export const ripgrep = SPEC.ripgrep.path; export const fd = SPEC.fd.path; export const dust = SPEC.dust.path; +export const rustic = SPEC.rustic.path; type App = keyof typeof SPEC; @@ -90,7 +109,13 @@ export async function install(app?: App) { // ... // ripgrep-14.1.1-x86_64-unknown-linux-musl/rg - const { VERSION, binary, path } = SPEC[app]; + const { + VERSION, + binary, + path, + stripComponents = 1, + pathInArchive = `${app}-${VERSION}-${getOS()}/${binary}`, + } = SPEC[app] as Spec; const tmpFile = join(__dirname, `${app}-${VERSION}.tar.gz`); try { @@ -104,10 +129,10 @@ export async function install(app?: App) { execFileSync("tar", [ "xzf", tmpFile, - "--strip-components=1", + `--strip-components=${stripComponents}`, `-C`, binPath, - `${app}-${VERSION}-${getOS()}/${binary}`, + pathInArchive, ]); // - 3. Make the file rg executable diff --git a/src/packages/backend/files/sandbox/rustic.ts b/src/packages/backend/files/sandbox/rustic.ts new file mode 100644 index 0000000000..a079692723 --- /dev/null +++ b/src/packages/backend/files/sandbox/rustic.ts @@ -0,0 +1,43 @@ +import exec, { type ExecOutput /*, validate*/ } from "./exec"; +import { rustic as rusticPath } from "./install"; +import { exists } from "@cocalc/backend/misc/async-utils-node"; +import { join } from "path"; + +export interface RusticOptions { + repo: string; + timeout?: number; + maxSize?: number; + safeAbsPath?: (path: string) => Promise; +} + +export default async function rustic( + args: string[], + options: RusticOptions, +): Promise { + const { timeout, maxSize, repo, safeAbsPath } = options; + + await ensureInitialized(repo); + + return await exec({ + cmd: rusticPath, + cwd: safeAbsPath ? await safeAbsPath("") : undefined, + safety: ["--password", "", "-r", repo, ...args], + maxSize, + timeout, + }); +} + +async function ensureInitialized(repo: string) { + if (!(await exists(join(repo, "config")))) { + await exec({ + cmd: rusticPath, + safety: ["--password", "", "-r", repo, "init"], + }); + } +} + +// const whitelist = { +// backup: {}, +// restore: {}, +// snapshots: {}, +// } as const; diff --git a/src/packages/conat/files/fs.ts b/src/packages/conat/files/fs.ts index a5f9d3c4b0..c02ad3f984 100644 --- a/src/packages/conat/files/fs.ts +++ b/src/packages/conat/files/fs.ts @@ -122,6 +122,8 @@ export interface Filesystem { pattern: string, options?: RipgrepOptions, ) => Promise; + + rustic: (args: string[]) => Promise; } interface IDirent { @@ -320,6 +322,9 @@ export async function fsServer({ service, fs, client, project_id }: Options) { async ripgrep(path: string, pattern: string, options?: RipgrepOptions) { return await (await fs(this.subject)).ripgrep(path, pattern, options); }, + async rustic(args: string[]) { + return await (await fs(this.subject)).rustic(args); + }, async rm(path: string, options?) { await (await fs(this.subject)).rm(path, options); }, From b237f858d8d296036b039be06b247cbc993fa6ad Mon Sep 17 00:00:00 2001 From: William Stein Date: Mon, 4 Aug 2025 00:53:25 +0000 Subject: [PATCH 177/270] working on rustic whitelist options --- .../backend/conat/files/local-path.ts | 2 +- src/packages/backend/files/sandbox/exec.ts | 2 +- src/packages/backend/files/sandbox/index.ts | 7 +- src/packages/backend/files/sandbox/rustic.ts | 225 ++++++++++++++++-- 4 files changed, 217 insertions(+), 19 deletions(-) diff --git a/src/packages/backend/conat/files/local-path.ts b/src/packages/backend/conat/files/local-path.ts index 3fa1c4a83a..0272996739 100644 --- a/src/packages/backend/conat/files/local-path.ts +++ b/src/packages/backend/conat/files/local-path.ts @@ -39,7 +39,7 @@ export async function localPathFileserver({ try { await mkdir(p); } catch {} - return new SandboxedFilesystem(p, { unsafeMode }); + return new SandboxedFilesystem(p, { unsafeMode, project_id }); } }, }); diff --git a/src/packages/backend/files/sandbox/exec.ts b/src/packages/backend/files/sandbox/exec.ts index b0d477b622..b1c3a4d474 100644 --- a/src/packages/backend/files/sandbox/exec.ts +++ b/src/packages/backend/files/sandbox/exec.ts @@ -140,7 +140,7 @@ export default async function exec({ }); } -function parseAndValidateOptions(options: string[], whitelist): string[] { +export function parseAndValidateOptions(options: string[], whitelist): string[] { const validatedOptions: string[] = []; let i = 0; diff --git a/src/packages/backend/files/sandbox/index.ts b/src/packages/backend/files/sandbox/index.ts index 90946e7c62..00a698a5b8 100644 --- a/src/packages/backend/files/sandbox/index.ts +++ b/src/packages/backend/files/sandbox/index.ts @@ -87,6 +87,7 @@ interface Options { unsafeMode?: boolean; // readonly -- only allow operations that don't change files readonly?: boolean; + project_id?: string; } // If you add any methods below that are NOT for the public api @@ -99,19 +100,22 @@ const INTERNAL_METHODS = new Set([ "readonly", "assertWritable", "rusticRepo", + "project_id", ]); export class SandboxedFilesystem { public readonly unsafeMode: boolean; public readonly readonly: boolean; private readonly rusticRepo: string = rusticRepo; + private project_id?: string; constructor( // path should be the path to a FOLDER on the filesystem (not a file) public readonly path: string, - { unsafeMode = false, readonly = false }: Options = {}, + { unsafeMode = false, readonly = false, project_id }: Options = {}, ) { this.unsafeMode = !!unsafeMode; this.readonly = !!readonly; + this.project_id = project_id; for (const f in this) { if (INTERNAL_METHODS.has(f)) { continue; @@ -235,6 +239,7 @@ export class SandboxedFilesystem { safeAbsPath: this.safeAbsPath, timeout: 120_000, maxSize: 10_000, + host: this.project_id ?? "global", }); }; diff --git a/src/packages/backend/files/sandbox/rustic.ts b/src/packages/backend/files/sandbox/rustic.ts index a079692723..9c991dcc9d 100644 --- a/src/packages/backend/files/sandbox/rustic.ts +++ b/src/packages/backend/files/sandbox/rustic.ts @@ -1,30 +1,130 @@ -import exec, { type ExecOutput /*, validate*/ } from "./exec"; +/* +Whitelist: + +The idea is that + - the client can only work with snapshots with exactly the given host. + - any snapshots they create have that host + - snapshots are only of data in their sandbox + - snapshots can only be restored to their sandbox + +The subcommands with some whitelisted support are: + + - backup + - snapshots + - ls + - restore + - find + - forget + +The source options are relative paths and the command is run from the +root of the sandbox_path. + + rustic backup --host=sandbox_path [whitelisted options]... [source]... + + rustic snapshots --filter-host=... [whitelisted options]... + + +Here the snapshot id will be checked to have the right host before +the command is run. Destination is relative to sandbox_path. + + rustic restore [whitelisted options] + + +Dump is used for viewing a version of a file via timetravel: + + rustic dump + +Find is used for getting info about all versions of a file that are backed up: + + rustic find --filter-host=... + + rustic find --filter-host=... --glob='foo/x.txt' -h + + +Delete snapshots: + +- delete snapshot with specific id, which must have the specified host. + + rustic forget [id] + +- + + +*/ + +import exec, { + type ExecOutput, + parseAndValidateOptions, + validate, +} from "./exec"; import { rustic as rusticPath } from "./install"; import { exists } from "@cocalc/backend/misc/async-utils-node"; import { join } from "path"; +import { rusticRepo } from "@cocalc/backend/data"; +import LRU from "lru-cache"; export interface RusticOptions { - repo: string; + repo?: string; timeout?: number; maxSize?: number; - safeAbsPath?: (path: string) => Promise; + safeAbsPath: (path: string) => Promise; + host: string; } export default async function rustic( args: string[], options: RusticOptions, ): Promise { - const { timeout, maxSize, repo, safeAbsPath } = options; + const { timeout, maxSize, repo = rusticRepo, safeAbsPath, host } = options; await ensureInitialized(repo); + const base = await safeAbsPath(""); - return await exec({ - cmd: rusticPath, - cwd: safeAbsPath ? await safeAbsPath("") : undefined, - safety: ["--password", "", "-r", repo, ...args], - maxSize, - timeout, - }); + const common = ["--password", "", "-r", repo]; + + const run = async (sanitizedArgs: string[]) => { + return await exec({ + cmd: rusticPath, + cwd: base, + safety: [...common, ...sanitizedArgs], + maxSize, + timeout, + }); + }; + + if (args[0] == "backup") { + if (args.length == 1) { + throw Error("missing backup source"); + } + const source = (await safeAbsPath(args.slice(-1)[0])).slice(base.length); + const options = parseAndValidateOptions( + args.slice(1, -1), + whitelist.backup, + ); + + return await run([ + "backup", + ...options, + "--no-scan", + "--host", + host, + "--", + source, + ]); + } else if (args[0] == "snapshots") { + const options = parseAndValidateOptions(args.slice(1), whitelist.snapshots); + return await run(["snapshots", ...options, "--filter-host", host]); + } else if (args[0] == "ls") { + if (args.length <= 1) { + throw Error("missing "); + } + const snapshot = args.slice(-1)[0]; // + await assertValidSnapshot({ snapshot, host, repo }); + const options = parseAndValidateOptions(args.slice(1, -1), whitelist.ls); + return await run(["ls", ...options, snapshot]); + } else { + throw Error(`subcommand not allowed: ${args[0]}`); + } } async function ensureInitialized(repo: string) { @@ -36,8 +136,101 @@ async function ensureInitialized(repo: string) { } } -// const whitelist = { -// backup: {}, -// restore: {}, -// snapshots: {}, -// } as const; +const whitelist = { + backup: { + "--label": validate.str, + "--tag": validate.str, + "--description": validate.str, + "--time": validate.str, + "--delete-after": validate.str, + "--as-path": validate.str, + "--with-atime": true, + "--ignore-devid": true, + "--json": true, + "--long": true, + "--quiet": true, + "-h": true, + "--help": true, + "--glob": validate.str, + "--iglob": validate.str, + "--git-ignore": true, + "--no-require-git": true, + "-x": true, + "--one-file-system": true, + "--exclude-larger-than": validate.str, + }, + snapshots: { + "-g": validate.str, + "--group-by": validate.str, + "--long": true, + "--json": true, + "--all": true, + "-h": true, + "--help": true, + "--filter-label": validate.str, + "--filter-paths": validate.str, + "--filter-paths-exact": validate.str, + "--filter-after": validate.str, + "--filter-before": validate.str, + "--filter-size": validate.str, + "--filter-size-added": validate.str, + "--filter-jq": validate.str, + }, + restore: {}, + ls: { + "-s": true, + "--summary": true, + "-l": true, + "--long": true, + "--json": true, + "--numeric-uid-gid": true, + "--recursive": true, + "-h": true, + "--help": true, + "--glob": validate.str, + "--iglob": validate.str, + }, +} as const; + +async function assertValidSnapshot({ snapshot, host, repo }) { + const id = snapshot.split(":")[0]; + if (id == "latest") { + // possible race condition so do not allow + throw Error("latest is not allowed"); + } + const actualHost = await getHost({ id, repo }); + if (actualHost != host) { + throw Error( + `host for snapshot with id ${id} must be '${host}' but it is ${actualHost}`, + ); + } +} + +// we do not allow changing host so this is safe to cache. +const hostCache = new LRU({ + max: 10000, +}); + +export async function getHost(opts) { + if (hostCache.has(opts.id)) { + return hostCache.get(opts.id); + } + const info = await getSnapshot(opts); + const hostname = info[0][1][0]["hostname"]; + hostCache.set(opts.id, hostname); + return hostname; +} + +export async function getSnapshot({ + id, + repo = rusticRepo, +}: { + id: string; + repo?: string; +}) { + const { stdout } = await exec({ + cmd: rusticPath, + safety: ["--password", "", "-r", repo, "snapshots", "--json", id], + }); + return JSON.parse(stdout.toString()); +} From e99b4fba75d64a24f9625c0cf93c000a60e9d301 Mon Sep 17 00:00:00 2001 From: William Stein Date: Mon, 4 Aug 2025 01:14:57 +0000 Subject: [PATCH 178/270] finished rustic whitelisting --- src/packages/backend/files/sandbox/rustic.ts | 116 ++++++++++++++++--- 1 file changed, 101 insertions(+), 15 deletions(-) diff --git a/src/packages/backend/files/sandbox/rustic.ts b/src/packages/backend/files/sandbox/rustic.ts index 9c991dcc9d..5118b1548f 100644 --- a/src/packages/backend/files/sandbox/rustic.ts +++ b/src/packages/backend/files/sandbox/rustic.ts @@ -86,7 +86,7 @@ export default async function rustic( return await exec({ cmd: rusticPath, cwd: base, - safety: [...common, ...sanitizedArgs], + safety: [...common, args[0], ...sanitizedArgs], maxSize, timeout, }); @@ -102,18 +102,10 @@ export default async function rustic( whitelist.backup, ); - return await run([ - "backup", - ...options, - "--no-scan", - "--host", - host, - "--", - source, - ]); + return await run([...options, "--no-scan", "--host", host, "--", source]); } else if (args[0] == "snapshots") { const options = parseAndValidateOptions(args.slice(1), whitelist.snapshots); - return await run(["snapshots", ...options, "--filter-host", host]); + return await run([args[0], ...options, "--filter-host", host]); } else if (args[0] == "ls") { if (args.length <= 1) { throw Error("missing "); @@ -121,7 +113,32 @@ export default async function rustic( const snapshot = args.slice(-1)[0]; // await assertValidSnapshot({ snapshot, host, repo }); const options = parseAndValidateOptions(args.slice(1, -1), whitelist.ls); - return await run(["ls", ...options, snapshot]); + return await run([...options, snapshot]); + } else if (args[0] == "restore") { + if (args.length <= 2) { + throw Error("missing "); + } + const snapshot = args.slice(-2)[0]; // + await assertValidSnapshot({ snapshot, host, repo }); + const destination = await safeAbsPath(args.slice(-1)[0]); // + const options = parseAndValidateOptions( + args.slice(1, -2), + whitelist.restore, + ); + return await run([...options, snapshot, destination]); + } else if (args[0] == "find") { + const options = parseAndValidateOptions(args.slice(1), whitelist.find); + return await run([...options, "--filter-host", host]); + } else if (args[0] == "forget") { + if (args.length == 2 && !args[1].startsWith("-")) { + // delete exactly id + const snapshot = args[1]; + await assertValidSnapshot({ snapshot, host, repo }); + return await run([snapshot]); + } + // delete several defined by rules. + const options = parseAndValidateOptions(args.slice(1), whitelist.forget); + return await run([...options, "--filter-host", host]); } else { throw Error(`subcommand not allowed: ${args[0]}`); } @@ -131,7 +148,7 @@ async function ensureInitialized(repo: string) { if (!(await exists(join(repo, "config")))) { await exec({ cmd: rusticPath, - safety: ["--password", "", "-r", repo, "init"], + safety: ["--no-progress", "--password", "", "-r", repo, "init"], }); } } @@ -176,20 +193,89 @@ const whitelist = { "--filter-size-added": validate.str, "--filter-jq": validate.str, }, - restore: {}, + restore: { + "--delete": true, + "--verify-existing": true, + "--recursive": true, + "-h": true, + "--help": true, + "--glob": validate.str, + "--iglob": validate.str, + }, ls: { "-s": true, "--summary": true, "-l": true, "--long": true, "--json": true, - "--numeric-uid-gid": true, "--recursive": true, "-h": true, "--help": true, "--glob": validate.str, "--iglob": validate.str, }, + find: { + "--glob": validate.str, + "--iglob": validate.str, + "--path": validate.str, + "-g": validate.str, + "--group-by": validate.str, + "--all": true, + "--show-misses": true, + "-h": true, + "--help": true, + "--filter-label": validate.str, + "--filter-paths": validate.str, + "--filter-paths-exact": validate.str, + "--filter-after": validate.str, + "--filter-before": validate.str, + "--filter-size": validate.str, + "--filter-size-added": validate.str, + "--filter-jq": validate.str, + }, + forget: { + "--json": true, + "-g": validate.str, + "--group-by": validate.str, + "-h": true, + "--help": true, + "--filter-label": validate.str, + "--filter-paths": validate.str, + "--filter-paths-exact": validate.str, + "--filter-after": validate.str, + "--filter-before": validate.str, + "--filter-size": validate.str, + "--filter-size-added": validate.str, + "--filter-jq": validate.str, + "--keep-tags": validate.str, + "--keep-id": validate.str, + "-l": validate.int, + "--keep-last": validate.int, + "-M": validate.int, + "--keep-minutely": validate.int, + "-H": validate.int, + "--keep-hourly": validate.int, + "-d": validate.int, + "--keep-daily": validate.int, + "-w": validate.int, + "--keep-weekly": validate.int, + "-m": validate.int, + "--keep-monthly": validate.int, + "--keep-quarter-yearly": validate.int, + "--keep-half-yearly": validate.int, + "-y": validate.int, + "--keep-yearly": validate.int, + "--keep-within": validate.str, + "--keep-within-minutely": validate.str, + "--keep-within-hourly": validate.str, + "--keep-within-daily": validate.str, + "--keep-within-weekly": validate.str, + "--keep-within-monthly": validate.str, + "--keep-within-quarter-yearly": validate.str, + "--keep-within-half-yearly": validate.str, + "--keep-within-yearly": validate.str, + "--keep-none": validate.str, + }, } as const; async function assertValidSnapshot({ snapshot, host, repo }) { From dde1d5c2741e8c01181aebddb208cba260a9a2ac Mon Sep 17 00:00:00 2001 From: William Stein Date: Mon, 4 Aug 2025 03:56:11 +0000 Subject: [PATCH 179/270] add a test of rustic --- src/packages/backend/files/sandbox/exec.ts | 7 +- src/packages/backend/files/sandbox/install.ts | 4 + .../backend/files/sandbox/rustic.test.ts | 60 +++++++++ src/packages/backend/files/sandbox/rustic.ts | 117 ++++++++++-------- src/packages/backend/package.json | 16 +-- 5 files changed, 137 insertions(+), 67 deletions(-) create mode 100644 src/packages/backend/files/sandbox/rustic.test.ts diff --git a/src/packages/backend/files/sandbox/exec.ts b/src/packages/backend/files/sandbox/exec.ts index b1c3a4d474..be17c9b676 100644 --- a/src/packages/backend/files/sandbox/exec.ts +++ b/src/packages/backend/files/sandbox/exec.ts @@ -75,7 +75,7 @@ export default async function exec({ args.push("--", ...positionalArgs); } - // console.log(`${cmd} ${args.join(" ")}`); + // console.log(`${cmd} ${args.join(" ")}`, { cmd, args }); logger.debug({ cmd, args }); const child = spawn(cmd, args, { stdio: ["ignore", "pipe", "pipe"], @@ -140,7 +140,10 @@ export default async function exec({ }); } -export function parseAndValidateOptions(options: string[], whitelist): string[] { +export function parseAndValidateOptions( + options: string[], + whitelist, +): string[] { const validatedOptions: string[] = []; let i = 0; diff --git a/src/packages/backend/files/sandbox/install.ts b/src/packages/backend/files/sandbox/install.ts index 7b5bd947c8..d22f1d3634 100644 --- a/src/packages/backend/files/sandbox/install.ts +++ b/src/packages/backend/files/sandbox/install.ts @@ -16,6 +16,9 @@ import { arch, platform } from "os"; import { execFileSync } from "child_process"; import { writeFile, stat, unlink, mkdir, chmod } from "fs/promises"; import { join } from "path"; +import getLogger from "@cocalc/backend/logger"; + +const logger = getLogger("files:sandbox:install"); const i = __dirname.lastIndexOf("packages/backend"); const binPath = join( @@ -98,6 +101,7 @@ export async function install(app?: App) { return; } const url = getUrl(app); + logger.debug("install", { app, url }); // - 1. Fetch the tarball from the github url (using the fetch library) const response = await downloadFromGithub(url); const tarballBuffer = Buffer.from(await response.arrayBuffer()); diff --git a/src/packages/backend/files/sandbox/rustic.test.ts b/src/packages/backend/files/sandbox/rustic.test.ts new file mode 100644 index 0000000000..db1c9ae1f9 --- /dev/null +++ b/src/packages/backend/files/sandbox/rustic.test.ts @@ -0,0 +1,60 @@ +/* +Test the rustic backup api. + +https://github.com/rustic-rs/rustic +*/ + +import rustic from "./rustic"; +import { mkdtemp, mkdir, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; + +let tempDir, options; +beforeAll(async () => { + tempDir = await mkdtemp(join(tmpdir(), "cocalc")); + const repo = join(tempDir, "repo"); + const home = join(tempDir, "home"); + await mkdir(home); + const safeAbsPath = (path: string) => join(home, resolve("/", path)); + options = { + host: "my-host", + repo, + safeAbsPath, + }; +}); +afterAll(async () => { + await rm(tempDir, { force: true, recursive: true }); +}); + +describe("rustic does something", () => { + it("there are initially no backups", async () => { + const { stdout, truncated } = await rustic( + ["snapshots", "--json"], + options, + ); + const s = JSON.parse(Buffer.from(stdout).toString()); + expect(s).toEqual([]); + expect(truncated).toBe(false); + }); + + it("create a file and back it up", async () => { + await writeFile(join(tempDir, "a.txt"), "hello"); + const { stdout, truncated } = await rustic( + ["backup", "--json", "a.txt"], + options, + ); + const s = JSON.parse(Buffer.from(stdout).toString()); + expect(s.paths).toEqual(["a.txt"]); + expect(truncated).toBe(false); + }); + + // it("it appears in the snapshots list", async () => { + // const { stdout, truncated } = await rustic( + // ["snapshots", "--json"], + // options, + // ); + // const s = JSON.parse(Buffer.from(stdout).toString()); + // expect(s).toEqual([]); + // expect(truncated).toBe(false); + // }); +}); diff --git a/src/packages/backend/files/sandbox/rustic.ts b/src/packages/backend/files/sandbox/rustic.ts index 5118b1548f..03d9240cc7 100644 --- a/src/packages/backend/files/sandbox/rustic.ts +++ b/src/packages/backend/files/sandbox/rustic.ts @@ -92,64 +92,65 @@ export default async function rustic( }); }; - if (args[0] == "backup") { - if (args.length == 1) { - throw Error("missing backup source"); + switch (args[0]) { + case "backup": { + if (args.length == 1) { + throw Error("missing backup source"); + } + const source = (await safeAbsPath(args.slice(-1)[0])).slice(base.length); + const options = parseAndValidateOptions( + args.slice(1, -1), + whitelist.backup, + ); + + return await run([...options, "--no-scan", "--host", host, "--", source]); } - const source = (await safeAbsPath(args.slice(-1)[0])).slice(base.length); - const options = parseAndValidateOptions( - args.slice(1, -1), - whitelist.backup, - ); - - return await run([...options, "--no-scan", "--host", host, "--", source]); - } else if (args[0] == "snapshots") { - const options = parseAndValidateOptions(args.slice(1), whitelist.snapshots); - return await run([args[0], ...options, "--filter-host", host]); - } else if (args[0] == "ls") { - if (args.length <= 1) { - throw Error("missing "); + case "snapshots": { + const options = parseAndValidateOptions( + args.slice(1), + whitelist.snapshots, + ); + return await run([...options, "--filter-host", host]); } - const snapshot = args.slice(-1)[0]; // - await assertValidSnapshot({ snapshot, host, repo }); - const options = parseAndValidateOptions(args.slice(1, -1), whitelist.ls); - return await run([...options, snapshot]); - } else if (args[0] == "restore") { - if (args.length <= 2) { - throw Error("missing "); + case "ls": { + if (args.length <= 1) { + throw Error("missing "); + } + const snapshot = args.slice(-1)[0]; // + await assertValidSnapshot({ snapshot, host, repo }); + const options = parseAndValidateOptions(args.slice(1, -1), whitelist.ls); + return await run([...options, snapshot]); } - const snapshot = args.slice(-2)[0]; // - await assertValidSnapshot({ snapshot, host, repo }); - const destination = await safeAbsPath(args.slice(-1)[0]); // - const options = parseAndValidateOptions( - args.slice(1, -2), - whitelist.restore, - ); - return await run([...options, snapshot, destination]); - } else if (args[0] == "find") { - const options = parseAndValidateOptions(args.slice(1), whitelist.find); - return await run([...options, "--filter-host", host]); - } else if (args[0] == "forget") { - if (args.length == 2 && !args[1].startsWith("-")) { - // delete exactly id - const snapshot = args[1]; + case "restore": { + if (args.length <= 2) { + throw Error("missing "); + } + const snapshot = args.slice(-2)[0]; // await assertValidSnapshot({ snapshot, host, repo }); - return await run([snapshot]); + const destination = await safeAbsPath(args.slice(-1)[0]); // + const options = parseAndValidateOptions( + args.slice(1, -2), + whitelist.restore, + ); + return await run([...options, snapshot, destination]); } - // delete several defined by rules. - const options = parseAndValidateOptions(args.slice(1), whitelist.forget); - return await run([...options, "--filter-host", host]); - } else { - throw Error(`subcommand not allowed: ${args[0]}`); - } -} - -async function ensureInitialized(repo: string) { - if (!(await exists(join(repo, "config")))) { - await exec({ - cmd: rusticPath, - safety: ["--no-progress", "--password", "", "-r", repo, "init"], - }); + case "find": { + const options = parseAndValidateOptions(args.slice(1), whitelist.find); + return await run([...options, "--filter-host", host]); + } + case "forget": { + if (args.length == 2 && !args[1].startsWith("-")) { + // delete exactly id + const snapshot = args[1]; + await assertValidSnapshot({ snapshot, host, repo }); + return await run([snapshot]); + } + // delete several defined by rules. + const options = parseAndValidateOptions(args.slice(1), whitelist.forget); + return await run([...options, "--filter-host", host]); + } + default: + throw Error(`subcommand not allowed: ${args[0]}`); } } @@ -278,6 +279,16 @@ const whitelist = { }, } as const; +async function ensureInitialized(repo: string) { + const config = join(repo, "config"); + if (!(await exists(config))) { + await exec({ + cmd: rusticPath, + safety: ["--no-progress", "--password", "", "-r", repo, "init"], + }); + } +} + async function assertValidSnapshot({ snapshot, host, repo }) { const id = snapshot.split(":")[0]; if (id == "latest") { diff --git a/src/packages/backend/package.json b/src/packages/backend/package.json index c74cf851ab..2c45ae00dd 100644 --- a/src/packages/backend/package.json +++ b/src/packages/backend/package.json @@ -13,15 +13,12 @@ "./auth/*": "./dist/auth/*.js", "./auth/tokens/*": "./dist/auth/tokens/*.js" }, - "keywords": [ - "utilities", - "cocalc" - ], + "keywords": ["utilities", "cocalc"], "scripts": { "preinstall": "npx only-allow pnpm", "clean": "rm -rf dist node_modules", - "install-ripgrep": "echo 'require(\"@cocalc/backend/files/sandbox/install\").install()' | node", - "build": "pnpm exec tsc --build && pnpm install-ripgrep", + "install-sandbox-tools": "echo 'require(\"@cocalc/backend/files/sandbox/install\").install()' | node", + "build": "pnpm exec tsc --build && pnpm install-sandbox-tools", "tsc": "pnpm exec tsc --watch --pretty --preserveWatchOutput", "test": "pnpm exec jest --forceExit", "test-conat": " pnpm exec jest --forceExit conat", @@ -34,12 +31,7 @@ "conat-persist": "DEBUG=cocalc:* node ./bin/conat-persist.cjs", "conat-test-server": "node ./bin/conat-test-server.cjs" }, - "files": [ - "dist/**", - "bin/**", - "README.md", - "package.json" - ], + "files": ["dist/**", "bin/**", "README.md", "package.json"], "author": "SageMath, Inc.", "license": "SEE LICENSE.md", "dependencies": { From a999dcf5c206da7b2250a3c85aaa1d4f72f70a30 Mon Sep 17 00:00:00 2001 From: William Stein Date: Mon, 4 Aug 2025 21:57:37 +0000 Subject: [PATCH 180/270] search -- switch to using fs ripgrep, first version --- src/packages/frontend/project/search/body.tsx | 4 +- src/packages/frontend/project/search/run.ts | 180 ++++-------------- src/packages/frontend/project_actions.ts | 2 - 3 files changed, 38 insertions(+), 148 deletions(-) diff --git a/src/packages/frontend/project/search/body.tsx b/src/packages/frontend/project/search/body.tsx index ef3a2b66e4..667c042855 100644 --- a/src/packages/frontend/project/search/body.tsx +++ b/src/packages/frontend/project/search/body.tsx @@ -120,8 +120,8 @@ export const ProjectSearchBody: React.FC<{ checked={git_grep} onChange={() => actions?.toggle_search_checkbox_git_grep()} > - Git search: in GIT repo, use "git grep" - to only search files in the git repo. + .gitignore aware: exclude files via + .gitignore and similar rules. {neural_search_enabled && ( = max_output || - results.length > max_results || - err - ); - let num_results = 0; - const search_results: {}[] = []; - for (const line of results) { - if (line.trim() === "") { + const search_results: SearchResult[] = []; + for (const line of lines) { + let result; + try { + result = JSON.parse(line); + } catch { continue; } - let i = line.indexOf(":"); - num_results += 1; - if (i !== -1) { - // all valid lines have a ':', the last line may have been truncated too early - let filename = line.slice(0, i); - if (filename.slice(0, 2) === "./") { - filename = filename.slice(2); - } - let context = line.slice(i + 1); - // strip codes in worksheet output - if (context.length > 0 && context[0] === MARKERS.output) { - i = context.slice(1).indexOf(MARKERS.output); - context = context.slice(i + 2, context.length - 1); - } - - const m = /^(\d+):/.exec(context); - let line_number: number | undefined; - if (m != null) { - try { - line_number = parseInt(m[1]); - } catch (e) {} - } - + if (result.type == "match") { + const { line_number, lines, path } = result.data; search_results.push({ - filename, - description: context, + filename: path?.text ?? "-", + description: `${(line_number.toString() + ":").padEnd(8, " ")}${lines.text}`, + filter: `${path?.text?.toLowerCase?.() ?? ""} ${lines.text.toLowerCase()}`, line_number, - filter: `${filename.toLowerCase()} ${context.toLowerCase()}`, }); } - if (num_results >= max_results) { - break; - } } setState({ - too_many_results, + too_many_results: truncated, search_results, + most_recent_search: query, + most_recent_path: path, }); } diff --git a/src/packages/frontend/project_actions.ts b/src/packages/frontend/project_actions.ts index abe12a9d31..cbbd0fd8bc 100644 --- a/src/packages/frontend/project_actions.ts +++ b/src/packages/frontend/project_actions.ts @@ -3562,8 +3562,6 @@ export class ProjectActions extends Actions { fs: this.fs(), query: store.get("user_input").trim(), path: store.get("current_path"), - project_id: this.project_id, - compute_server_id: this.getComputeServerId(), options: { case_sensitive: store.get("case_sensitive"), git_grep: store.get("git_grep"), From c60a5113bc2ee84e483fdf4a52407c0c99fa5e0e Mon Sep 17 00:00:00 2001 From: William Stein Date: Mon, 4 Aug 2025 23:25:00 +0000 Subject: [PATCH 181/270] search: make consistent with flyout; use virtuoso so displaying a large number is very fast --- src/packages/frontend/chat/chat-log.tsx | 3 +- .../frontend/project/page/flyouts/body.tsx | 3 +- .../frontend/project/page/flyouts/header.tsx | 1 - .../frontend/project/page/flyouts/search.tsx | 4 +- src/packages/frontend/project/search/body.tsx | 317 ++++++------------ src/packages/frontend/project/search/run.ts | 15 +- .../frontend/project/search/search.tsx | 2 +- 7 files changed, 122 insertions(+), 223 deletions(-) diff --git a/src/packages/frontend/chat/chat-log.tsx b/src/packages/frontend/chat/chat-log.tsx index e1f25a778b..37f0d29cf1 100644 --- a/src/packages/frontend/chat/chat-log.tsx +++ b/src/packages/frontend/chat/chat-log.tsx @@ -13,11 +13,10 @@ import { Alert, Button } from "antd"; import { Set as immutableSet } from "immutable"; import { MutableRefObject, useEffect, useMemo, useRef } from "react"; import { Virtuoso, VirtuosoHandle } from "react-virtuoso"; - +import useVirtuosoScrollHook from "@cocalc/frontend/components/virtuoso-scroll-hook"; import { chatBotName, isChatBot } from "@cocalc/frontend/account/chatbot"; import { useRedux, useTypedRedux } from "@cocalc/frontend/app-framework"; import { Icon } from "@cocalc/frontend/components"; -import useVirtuosoScrollHook from "@cocalc/frontend/components/virtuoso-scroll-hook"; import { HashtagBar } from "@cocalc/frontend/editors/task-editor/hashtag-bar"; import { DivTempHeight } from "@cocalc/frontend/jupyter/cell-list"; import { diff --git a/src/packages/frontend/project/page/flyouts/body.tsx b/src/packages/frontend/project/page/flyouts/body.tsx index d71a3c2c3d..cc7883ddda 100644 --- a/src/packages/frontend/project/page/flyouts/body.tsx +++ b/src/packages/frontend/project/page/flyouts/body.tsx @@ -4,7 +4,6 @@ */ import { debounce } from "lodash"; - import { CSS, redux, @@ -97,7 +96,7 @@ export function FlyoutBody({ flyout, flyoutWidth }: FlyoutBodyProps) { style={style} onFocus={() => { // Remove any active key handler that is next to this side chat. - // E.g, this is critical for taks lists... + // E.g, this is critical for task lists... redux.getActions("page").erase_active_key_handler(); }} > diff --git a/src/packages/frontend/project/page/flyouts/header.tsx b/src/packages/frontend/project/page/flyouts/header.tsx index bacbf7573e..ad40f53e1a 100644 --- a/src/packages/frontend/project/page/flyouts/header.tsx +++ b/src/packages/frontend/project/page/flyouts/header.tsx @@ -6,7 +6,6 @@ import { Button, Tooltip } from "antd"; import { useEffect, useState } from "react"; import { FormattedMessage, useIntl } from "react-intl"; - import { TourName } from "@cocalc/frontend/account/tours"; import { redux, useTypedRedux } from "@cocalc/frontend/app-framework"; import { Icon } from "@cocalc/frontend/components"; diff --git a/src/packages/frontend/project/page/flyouts/search.tsx b/src/packages/frontend/project/page/flyouts/search.tsx index fadb01c57d..18b09784c4 100644 --- a/src/packages/frontend/project/page/flyouts/search.tsx +++ b/src/packages/frontend/project/page/flyouts/search.tsx @@ -5,6 +5,6 @@ import { ProjectSearchBody } from "@cocalc/frontend/project/search/body"; -export function SearchFlyout({ wrap }) { - return ; +export function SearchFlyout() { + return ; } diff --git a/src/packages/frontend/project/search/body.tsx b/src/packages/frontend/project/search/body.tsx index 667c042855..8af738422a 100644 --- a/src/packages/frontend/project/search/body.tsx +++ b/src/packages/frontend/project/search/body.tsx @@ -10,13 +10,12 @@ of course, a disaster waiting to happen. They all need to be in a single namespace somehow...! */ -import { Button, Card, Col, Input, Row, Space, Tag } from "antd"; -import { useEffect, useMemo, useState } from "react"; +import { Button, Card, Col, Input, Row, Tag } from "antd"; +import { useMemo, useState } from "react"; import { useProjectContext } from "@cocalc/frontend/project/context"; -import { Alert, Checkbox, Well } from "@cocalc/frontend/antd-bootstrap"; +import { Alert, Checkbox } from "@cocalc/frontend/antd-bootstrap"; import { useActions, useTypedRedux } from "@cocalc/frontend/app-framework"; import { - Gap, HelpIcon, Icon, Loading, @@ -34,21 +33,19 @@ import { filename_extension, path_split, path_to_file, + plural, search_match, search_split, unreachable, } from "@cocalc/util/misc"; import { COLORS } from "@cocalc/util/theme"; import SelectComputeServerForFileExplorer from "@cocalc/frontend/compute/select-server-for-explorer"; - -const RESULTS_WELL_STYLE: React.CSSProperties = { - backgroundColor: "white", -} as const; +import { Virtuoso } from "react-virtuoso"; +import useVirtuosoScrollHook from "@cocalc/frontend/components/virtuoso-scroll-hook"; export const ProjectSearchBody: React.FC<{ mode: "project" | "flyout"; - wrap?: Function; -}> = ({ mode = "project", wrap }) => { +}> = ({ mode = "project" }) => { const { project_id } = useProjectContext(); const subdirectories = useTypedRedux({ project_id }, "subdirectories"); const case_sensitive = useTypedRedux({ project_id }, "case_sensitive"); @@ -62,28 +59,17 @@ export const ProjectSearchBody: React.FC<{ const actions = useActions({ project_id }); - const isFlyout = mode === "flyout"; - function renderResultList() { - if (isFlyout) { - return ( - - ); - } else { - return ( - - - - - - ); - } + return ; } function renderHeaderProject() { return ( - + {mode != "flyout" ? ( @@ -120,8 +106,8 @@ export const ProjectSearchBody: React.FC<{ checked={git_grep} onChange={() => actions?.toggle_search_checkbox_git_grep()} > - .gitignore aware: exclude files via - .gitignore and similar rules. + Git aware: exclude files via .gitignore + and similar rules. {neural_search_enabled && ( - {renderHeader()} - {renderResultList()} -
- ); - } - - if (isFlyout) { - return ( -
- {renderContent()} -
- ); - } else { - return {renderContent()}; - } + return ( +
+ {renderHeader()} + {renderResultList()} +
+ ); }; interface ProjectSearchInputProps { @@ -298,12 +256,10 @@ interface ProjectSearchOutputProps { function ProjectSearchOutput({ project_id, - wrap, mode = "project", }: ProjectSearchOutputProps) { const [filter, setFilter] = useState(""); const [currentFilter, setCurrentFilter] = useState(""); - const isFlyout = mode === "flyout"; const most_recent_search = useTypedRedux( { project_id }, "most_recent_search", @@ -316,10 +272,9 @@ function ProjectSearchOutput({ const search_error = useTypedRedux({ project_id }, "search_error"); const too_many_results = useTypedRedux({ project_id }, "too_many_results"); - useEffect(() => { - setFilter(""); - setCurrentFilter(""); - }, [unfiltered_search_results]); + const virtuosoScroll = useVirtuosoScrollHook({ + cacheId: `search-${project_id}`, + }); const search_results = useMemo(() => { const f = filter?.trim(); @@ -357,148 +312,99 @@ function ProjectSearchOutput({ if (search_results?.size == 0) { return ( - There were no results for your search. + There are no results for your search. ); } - const v: React.JSX.Element[] = []; - let i = 0; - for (const result of search_results) { - v.push( - , - ); - i += 1; - } - return v; + return ( + { + const result = search_results.get(index); + return ( + + ); + }} + {...virtuosoScroll} + /> + ); } function renderResultList() { - if (isFlyout) { - return wrap?.( - - {render_get_results()} - , - { borderTop: `1px solid ${COLORS.GRAY_L}` }, - ); - } else { - return {render_get_results()}; - } + return
{render_get_results()}
; } return ( - <> +
setCurrentFilter(e.target.value)} - placeholder="Filter... (regexp in / /)" + placeholder="Filter results... (regexp in / /)" onSearch={setFilter} enterButton="Filter" style={{ width: "350px", maxWidth: "100%", marginBottom: "15px" }} /> {too_many_results && ( + + {search_results.size} {plural(search_results.size, "Result")}: + {" "} There were more results than displayed below. Try making your search more specific. )} + {!too_many_results && ( + + + {search_results.size} {plural(search_results.size, "Result")} + + + )} {renderResultList()} - +
); } function ProjectSearchOutputHeader({ project_id }: { project_id: string }) { const actions = useActions({ project_id }); - const info_visible = useTypedRedux({ project_id }, "info_visible"); - const search_results = useTypedRedux({ project_id }, "search_results"); - const command = useTypedRedux({ project_id }, "command"); const most_recent_search = useTypedRedux( { project_id }, "most_recent_search", ); const most_recent_path = useTypedRedux({ project_id }, "most_recent_path"); - function output_path() { - return !most_recent_path ? : most_recent_path; - } - - function render_get_info() { - return ( - -
    -
  • - Search command (in a terminal): 
    {command}
    -
    • -
  • - Number of results:{" "} - {search_results ? search_results?.size : } -
    • -
- - ); - } - if (most_recent_search == null || most_recent_path == null) { return ; } return ( -
- - -

- Results of searching in {output_path()} for "{most_recent_search}" - - -

- - {info_visible && render_get_info()} + ); } -const DESC_STYLE: React.CSSProperties = { - color: COLORS.GRAY_M, - marginBottom: "5px", - border: "1px solid #eee", - borderRadius: "5px", - maxHeight: "300px", - padding: "15px", - overflowY: "auto", -} as const; - interface ProjectSearchResultLineProps { project_id: string; filename: string; @@ -509,17 +415,14 @@ interface ProjectSearchResultLineProps { mode?: "project" | "flyout"; } -function ProjectSearchResultLine(_: Readonly) { - const { - project_id, - filename, - description, - line_number, - fragment_id, - most_recent_path, - mode = "project", - } = _; - const isFlyout = mode === "flyout"; +function ProjectSearchResultLine({ + project_id, + filename, + description, + line_number, + fragment_id, + most_recent_path, +}: Readonly) { const actions = useActions({ project_id }); const ext = filename_extension(filename); const icon = file_associations[ext]?.icon ?? "file"; @@ -565,40 +468,32 @@ function ProjectSearchResultLine(_: Readonly) { ); } - if (isFlyout) { - return ( - - } - > - - - ); - } else { - return ( -
- {renderFileLink()} -
- -
-
- ); - } + } + > + + + ); } const MARKDOWN_EXTS = ["tasks", "slides", "board", "sage-chat"] as const; diff --git a/src/packages/frontend/project/search/run.ts b/src/packages/frontend/project/search/run.ts index 840d2f7d30..317783574d 100644 --- a/src/packages/frontend/project/search/run.ts +++ b/src/packages/frontend/project/search/run.ts @@ -1,4 +1,10 @@ import { type FilesystemClient } from "@cocalc/conat/files/fs"; +import { trunc } from "@cocalc/util/misc"; + +// we get about this many bytes of results from the filesystem, then stop... +const MAX_SIZE = 1_000_000; + +const MAX_LINE_LENGTH = 256; interface SearchResult { filename: string; @@ -30,7 +36,7 @@ export async function search({ return; } - const rgOptions = ["--json", "-M", "256"]; + const rgOptions = ["--json"]; // note that -M doesn't seem to combine with --json, so can't do -M {MAX_LINE_LENGTH} if (!options.subdirectories) { rgOptions.push("-d", "1"); } @@ -46,7 +52,7 @@ export async function search({ const { stdout, truncated } = await fs.ripgrep(path, query, { options: rgOptions, - maxSize: 100_000, + maxSize: MAX_SIZE, }); const lines = Buffer.from(stdout).toString().split("\n"); @@ -60,10 +66,11 @@ export async function search({ } if (result.type == "match") { const { line_number, lines, path } = result.data; + const description = trunc(lines?.text ?? "", MAX_LINE_LENGTH); search_results.push({ filename: path?.text ?? "-", - description: `${(line_number.toString() + ":").padEnd(8, " ")}${lines.text}`, - filter: `${path?.text?.toLowerCase?.() ?? ""} ${lines.text.toLowerCase()}`, + description: `${(line_number.toString() + ":").padEnd(8, " ")}${description}`, + filter: `${path?.text?.toLowerCase?.() ?? ""} ${description.toLowerCase()}`, line_number, }); } diff --git a/src/packages/frontend/project/search/search.tsx b/src/packages/frontend/project/search/search.tsx index c9b6a3d53d..19bfa377aa 100644 --- a/src/packages/frontend/project/search/search.tsx +++ b/src/packages/frontend/project/search/search.tsx @@ -4,7 +4,7 @@ import { ProjectSearchHeader } from "./header"; export const ProjectSearch: React.FC = () => { return ( -
+
From 96c1ad48b6c88131d94ff728d11331abe4dfba13 Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 5 Aug 2025 00:24:57 +0000 Subject: [PATCH 182/270] add archiver to sandbox for easily making zip/tar files. --- .../backend/files/sandbox/archiver.ts | 51 +++++++ src/packages/backend/files/sandbox/index.ts | 16 ++ src/packages/backend/package.json | 13 +- src/packages/conat/files/fs.ts | 27 ++++ src/packages/pnpm-lock.yaml | 144 ++++++++++++++++++ 5 files changed, 249 insertions(+), 2 deletions(-) create mode 100644 src/packages/backend/files/sandbox/archiver.ts diff --git a/src/packages/backend/files/sandbox/archiver.ts b/src/packages/backend/files/sandbox/archiver.ts new file mode 100644 index 0000000000..73f447a4e7 --- /dev/null +++ b/src/packages/backend/files/sandbox/archiver.ts @@ -0,0 +1,51 @@ +import Archiver from "archiver"; +import { createWriteStream } from "fs"; +import { stat } from "fs/promises"; +import { once } from "@cocalc/util/async-utils"; +import { type ArchiverOptions } from "@cocalc/conat/files/fs"; +export { type ArchiverOptions }; + +export default async function archiver( + path: string, + paths: string[] | string, + options?: ArchiverOptions, +) { + if (options == null) { + if (path.endsWith(".zip")) { + options = { format: "zip" }; + } else if (path.endsWith(".tar")) { + options = { format: "tar" }; + } else if (path.endsWith(".tar.gz")) { + options = { format: "tar", gzip: true }; + } + } + if (typeof paths == "string") { + paths = [paths]; + } + const archive = new Archiver(options!.format, options); + const output = createWriteStream(path); + // docs say to listen before calling finalize + const closed = once(output, "close"); + archive.pipe(output); + + let error: any = undefined; + archive.on("error", (err) => { + error = err; + }); + + const isDir = async (path) => (await stat(path)).isDirectory(); + const v = await Promise.all(paths.map(isDir)); + for (let i = 0; i < paths.length; i++) { + if (v[i]) { + archive.directory(paths[i]); + } else { + archive.file(paths[i]); + } + } + + await archive.finalize(); + await closed; + if (error) { + throw error; + } +} diff --git a/src/packages/backend/files/sandbox/index.ts b/src/packages/backend/files/sandbox/index.ts index 00a698a5b8..5302f546f0 100644 --- a/src/packages/backend/files/sandbox/index.ts +++ b/src/packages/backend/files/sandbox/index.ts @@ -76,6 +76,7 @@ import dust, { type DustOptions } from "./dust"; import rustic from "./rustic"; import { type ExecOutput } from "./exec"; import { rusticRepo } from "@cocalc/backend/data"; +import archiver, { type ArchiverOptions } from "./archiver"; // max time code can run (in safe mode), e.g., for find, // ripgrep, fd, and dust. @@ -243,6 +244,21 @@ export class SandboxedFilesystem { }); }; + archiver = async ( + path: string, + paths: string[] | string, + options?: ArchiverOptions, + ): Promise => { + if (typeof paths == "string") { + paths = [paths]; + } + await archiver( + await this.safeAbsPath(path), + await Promise.all(paths.map(this.safeAbsPath)), + options, + ); + }; + ripgrep = async ( path: string, pattern: string, diff --git a/src/packages/backend/package.json b/src/packages/backend/package.json index 2c45ae00dd..056ce72824 100644 --- a/src/packages/backend/package.json +++ b/src/packages/backend/package.json @@ -13,7 +13,10 @@ "./auth/*": "./dist/auth/*.js", "./auth/tokens/*": "./dist/auth/tokens/*.js" }, - "keywords": ["utilities", "cocalc"], + "keywords": [ + "utilities", + "cocalc" + ], "scripts": { "preinstall": "npx only-allow pnpm", "clean": "rm -rf dist node_modules", @@ -31,13 +34,19 @@ "conat-persist": "DEBUG=cocalc:* node ./bin/conat-persist.cjs", "conat-test-server": "node ./bin/conat-test-server.cjs" }, - "files": ["dist/**", "bin/**", "README.md", "package.json"], + "files": [ + "dist/**", + "bin/**", + "README.md", + "package.json" + ], "author": "SageMath, Inc.", "license": "SEE LICENSE.md", "dependencies": { "@cocalc/backend": "workspace:*", "@cocalc/conat": "workspace:*", "@cocalc/util": "workspace:*", + "archiver": "^7.0.1", "awaiting": "^3.0.0", "better-sqlite3": "^12.2.0", "chokidar": "^3.6.0", diff --git a/src/packages/conat/files/fs.ts b/src/packages/conat/files/fs.ts index c02ad3f984..f675a08ecd 100644 --- a/src/packages/conat/files/fs.ts +++ b/src/packages/conat/files/fs.ts @@ -59,6 +59,24 @@ export interface DustOptions { maxSize?: number; } +interface ZipOptions { + format: "zip"; + comment?: string; + forceLocalTime?: boolean; + forceZip64?: boolean; + namePrependSlash?: boolean; + store?: boolean; + zlib?: object; +} + +interface TarOptions { + format: "tar"; + gzip?: boolean; + gzipOPtions?: object; +} + +export type ArchiverOptions = ZipOptions | TarOptions; + export interface Filesystem { appendFile: (path: string, data: string | Buffer, encoding?) => Promise; chmod: (path: string, mode: string | number) => Promise; @@ -91,6 +109,12 @@ export interface Filesystem { // todo: typing watch: (path: string, options?) => Promise; + archiver: ( + path: string, // archive to create, e.g., a.zip, a.tar or a.tar.gz + paths: string, // paths to include in archive -- can be files or directories in the sandbox. + options?: ArchiverOptions, // options -- see https://www.archiverjs.com/ + ) => Promise; + // We add very little to the Filesystem api, but we have to add // a sandboxed "find" command, since it is a 1-call way to get // arbitrary directory listing info, which is just not possible @@ -263,6 +287,9 @@ export async function fsServer({ service, fs, client, project_id }: Options) { async appendFile(path: string, data: string | Buffer, encoding?) { await (await fs(this.subject)).appendFile(path, data, encoding); }, + async archiver(path: string, paths: string, options?: ArchiverOptions) { + return await (await fs(this.subject)).archiver(path, paths, options); + }, async chmod(path: string, mode: string | number) { await (await fs(this.subject)).chmod(path, mode); }, diff --git a/src/packages/pnpm-lock.yaml b/src/packages/pnpm-lock.yaml index 00ebbf5056..4d90e0f376 100644 --- a/src/packages/pnpm-lock.yaml +++ b/src/packages/pnpm-lock.yaml @@ -87,6 +87,9 @@ importers: '@cocalc/util': specifier: workspace:* version: link:../util + archiver: + specifier: ^7.0.1 + version: 7.0.1 awaiting: specifier: ^3.0.0 version: 3.0.0 @@ -4804,6 +4807,14 @@ packages: resolution: {integrity: sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==} engines: {node: '>= 8'} + archiver-utils@5.0.2: + resolution: {integrity: sha512-wuLJMmIBQYCsGZgYLTy5FIB2pF6Lfb6cXMSF8Qywwk3t20zWnAi7zLcQFdKQmIB8wyZpY5ER38x08GbwtR2cLA==} + engines: {node: '>= 14'} + + archiver@7.0.1: + resolution: {integrity: sha512-ZcbTaIqJOfCc03QwD468Unz/5Ir8ATtvAHsK+FdXbDIbGfihqh9mrvdcYunQzqn4HrvWWaFyaxJhGZagaJJpPQ==} + engines: {node: '>= 14'} + argparse@1.0.10: resolution: {integrity: sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==} @@ -4970,6 +4981,9 @@ packages: axios@1.11.0: resolution: {integrity: sha512-1Lx3WLFQWm3ooKDYZD1eXmoGO9fxYQjrycfHFC8P0sCfQVXyROp0p9PFWBehewBOdCwHc+f/b8I0fMto5eSfwA==} + b4a@1.6.7: + resolution: {integrity: sha512-OnAYlL5b7LEkALw87fUVafQw5rVR9RjwGd4KUwNQ6DrrNmaVaUCgLipfVlzrPQ4tWOR9P0IXGNOx50jYCCdSJg==} + babel-jest@29.7.0: resolution: {integrity: sha512-BrvGY3xZSwEcCzKvKsCi2GgHqDqsYkOP4/by5xCgIwGXQxIEh+8ew3gmrE1y7XRR6LHZIj6yLYnUi/mm2KXKBg==} engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} @@ -5014,6 +5028,9 @@ packages: balanced-match@1.0.2: resolution: {integrity: sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==} + bare-events@2.6.0: + resolution: {integrity: sha512-EKZ5BTXYExaNqi3I3f9RtEsaI/xBSGjE0XZCZilPzFAV/goswFHuPd9jEZlPIZ/iNZJwDSao9qRiScySz7MbQg==} + base-64@1.0.0: resolution: {integrity: sha512-kwDPIFCGx0NZHog36dj+tHiwP4QMzsZ3AgMViUBKI0+V5n4U0ufTCUMhnQ04diaRI8EX/QcPfql7zlhZ7j4zgg==} @@ -5138,6 +5155,10 @@ packages: bser@2.1.1: resolution: {integrity: sha512-gQxTNE/GAfIIrmHLUE3oJyp5FO6HRBfhjnw4/wMmA63ZGDJnWBmgY/lyQBpnDUkGmAhbSe39tx2d/iTOAfglwQ==} + buffer-crc32@1.0.0: + resolution: {integrity: sha512-Db1SbgBS/fg/392AblrMJk97KggmvYhr4pB5ZIMTWtaivCPMWLkmb7m21cJvpvgK+J3nsU2CmmixNBZx4vFj/w==} + engines: {node: '>=8.0.0'} + buffer-equal-constant-time@1.0.1: resolution: {integrity: sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==} @@ -5500,6 +5521,10 @@ packages: compare-versions@4.1.4: resolution: {integrity: sha512-FemMreK9xNyL8gQevsdRMrvO4lFCkQP7qbuktn1q8ndcNk1+0mz7lgE7b/sNvbhVgY4w6tMN1FDp6aADjqw2rw==} + compress-commons@6.0.2: + resolution: {integrity: sha512-6FqVXeETqWPoGcfzrXb37E50NP0LXT8kAMu5ooZayhWWdgEY4lBEEcbQNXtkuKQsGduxiIcI4gOTsxTmuq/bSg==} + engines: {node: '>= 14'} + compressible@2.0.18: resolution: {integrity: sha512-AF3r7P5dWxL8MxyITRMlORQNaOA2IkAFaTr4k7BUumjPtRpGDTZpl0Pb1XCO6JeDCBdp126Cgs9sMxqSjgYyRg==} engines: {node: '>= 0.6'} @@ -5629,6 +5654,15 @@ packages: country-regex@1.1.0: resolution: {integrity: sha512-iSPlClZP8vX7MC3/u6s3lrDuoQyhQukh5LyABJ3hvfzbQ3Yyayd4fp04zjLnfi267B/B2FkumcWWgrbban7sSA==} + crc-32@1.2.2: + resolution: {integrity: sha512-ROmzCKrTnOwybPcJApAA6WBWij23HVfGVNKqqrZpuyZOHqK2CwHSvpGuyt/UNNvaIjEd8X5IFGp4Mh+Ie1IHJQ==} + engines: {node: '>=0.8'} + hasBin: true + + crc32-stream@6.0.0: + resolution: {integrity: sha512-piICUB6ei4IlTv1+653yq5+KoqfBYmj9bw6LqXoOneTMDXk5nM1qt12mFW1caG3LlJXEKW1Bp0WggEmIfQB34g==} + engines: {node: '>= 14'} + create-error-class@3.0.2: resolution: {integrity: sha512-gYTKKexFO3kh200H1Nit76sRwRtOY32vQd3jpAQKpLtZqyNsSQNfI4N7o3eP2wUjV35pTWKRYqFUDBvUha/Pkw==} engines: {node: '>=0.10.0'} @@ -6671,6 +6705,9 @@ packages: resolution: {integrity: sha512-V7/RktU11J3I36Nwq2JnZEM7tNm17eBJz+u25qdxBZeCKiX6BkVSZQjwWIr+IobgnZy+ag73tTZgZi7tr0LrBw==} engines: {node: '>=6.0.0'} + fast-fifo@1.3.2: + resolution: {integrity: sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==} + fast-glob@3.3.3: resolution: {integrity: sha512-7MptL8U0cqcFdzIzwOTHoilX9x5BrNqye7Z/LuC7kCMRio1EMSyqRK3BEAUD7sXRq4iT4AzTVuZdhgQ2TCvYLg==} engines: {node: '>=8.6.0'} @@ -8259,6 +8296,10 @@ packages: layout-base@2.0.1: resolution: {integrity: sha512-dp3s92+uNI1hWIpPGH3jK2kxE2lMjdXdr+DH8ynZHpd6PUlH6x6cbuXnoMmiNumznqaNO31xu9e79F0uuZ0JFg==} + lazystream@1.0.1: + resolution: {integrity: sha512-b94GiNHQNy6JNTrt5w6zNyffMrNkXZb3KTkCZJb2V1xaEGCk093vkZ2jk3tpaeP33/OiXC+WvK9AxUebnf5nbw==} + engines: {node: '>= 0.6.3'} + ldap-filter@0.3.3: resolution: {integrity: sha512-/tFkx5WIn4HuO+6w9lsfxq4FN3O+fDZeO9Mek8dCD8rTUpqzRa766BOBO7BcGkn3X86m5+cBm1/2S/Shzz7gMg==} engines: {node: '>=0.8'} @@ -10044,6 +10085,13 @@ packages: resolution: {integrity: sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==} engines: {node: '>= 6'} + readable-stream@4.7.0: + resolution: {integrity: sha512-oIGGmcpTLwPga8Bn6/Z75SVaH1z5dUut2ibSyAMVhmUggWpmDn2dapB0n7f8nwaSiRtepAsfJyfXIO5DCVAODg==} + engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} + + readdir-glob@1.1.3: + resolution: {integrity: sha512-v05I2k7xN8zXvPD9N+z/uhXPaj0sUFCe2rcWZIpBsqxfP7xXFQ0tipAd/wjj1YxWyWtUS5IDJpOG82JKt2EAVA==} + readdirp@3.6.0: resolution: {integrity: sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==} engines: {node: '>=8.10.0'} @@ -10603,6 +10651,9 @@ packages: resolution: {integrity: sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg==} engines: {node: '>=10.0.0'} + streamx@2.22.1: + resolution: {integrity: sha512-znKXEBxfatz2GBNK02kRnCXjV+AA4kjZIUxeWSr3UGirZMJfTE9uiwKHobnbgxWyL/JWro8tTq+vOqAK1/qbSA==} + string-convert@0.2.1: resolution: {integrity: sha512-u/1tdPl4yQnPBjnVrmdLo9gtuLvELKsAoRapekWggdiQNvvvum+jYF329d84NAa660KQw7pB2n36KrIKVoXa3A==} @@ -10795,6 +10846,9 @@ packages: resolution: {integrity: sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==} engines: {node: '>=6'} + tar-stream@3.1.7: + resolution: {integrity: sha512-qJj60CXt7IU1Ffyc3NJMjh6EkuCFej46zUqJ4J7pqYlThyd9bO0XBTmcOIhSzZJVWfsLks0+nle/j538YAW9RQ==} + tdigest@0.1.2: resolution: {integrity: sha512-+G0LLgjjo9BZX2MfdvPfH+MKLCrxlXSYec5DaPYP1fe6Iyhf0/fSmJ0bFiZ1F8BT6cGXl2LpltQptzjXKWEkKA==} @@ -10836,6 +10890,9 @@ packages: resolution: {integrity: sha512-cAGWPIyOHU6zlmg88jwm7VRyXnMN7iV68OGAbYDk/Mh/xC/pzVPlQtY6ngoIH/5/tciuhGfvESU8GrHrcxD56w==} engines: {node: '>=8'} + text-decoder@1.2.3: + resolution: {integrity: sha512-3/o9z3X0X0fTupwsYvR03pJ/DjWuqqrfwBgTQzdWDiQSm9KitAyz/9WqsT2JQW7KV2m+bC2ol/zqpW37NHxLaA==} + text-hex@1.0.0: resolution: {integrity: sha512-uuVGNWzgJ4yhRaNSiubPY7OjISw4sw4E5Uv0wbjp+OzcbmVU/rsT8ujgcXJhn9ypzsgr5vlzpPqP+MBBKcGvbg==} @@ -11747,6 +11804,10 @@ packages: resolution: {integrity: sha512-vWOrt19lvcXTxu5tiHXfEGQuldSlU+qZn2TT+4EbRQzaciWGwNZ99QQTolQOmcwVgZLodv+1QfC6UZs2PX/6pQ==} engines: {node: '>= 12'} + zip-stream@6.0.1: + resolution: {integrity: sha512-zK7YHHz4ZXpW89AHXUPbQVGKI7uvkd3hzusTdotCg1UxyaVtg0zFJSTfW/Dq5f7OBBVnq6cZIaC8Ti4hb6dtCA==} + engines: {node: '>= 14'} + zlibjs@0.3.1: resolution: {integrity: sha512-+J9RrgTKOmlxFSDHo0pI1xM6BLVUv+o0ZT9ANtCxGkjIVCCUdx9alUF8Gm+dGLKbkkkidWIHFDZHDMpfITt4+w==} @@ -15195,6 +15256,26 @@ snapshots: normalize-path: 3.0.0 picomatch: 2.3.1 + archiver-utils@5.0.2: + dependencies: + glob: 10.4.5 + graceful-fs: 4.2.11 + is-stream: 2.0.1 + lazystream: 1.0.1 + lodash: 4.17.21 + normalize-path: 3.0.0 + readable-stream: 4.7.0 + + archiver@7.0.1: + dependencies: + archiver-utils: 5.0.2 + async: 3.2.6 + buffer-crc32: 1.0.0 + readable-stream: 4.7.0 + readdir-glob: 1.1.3 + tar-stream: 3.1.7 + zip-stream: 6.0.1 + argparse@1.0.10: dependencies: sprintf-js: 1.0.3 @@ -15374,6 +15455,8 @@ snapshots: transitivePeerDependencies: - debug + b4a@1.6.7: {} + babel-jest@29.7.0(@babel/core@7.28.0): dependencies: '@babel/core': 7.28.0 @@ -15450,6 +15533,9 @@ snapshots: balanced-match@1.0.2: {} + bare-events@2.6.0: + optional: true + base-64@1.0.0: {} base16@1.0.0: {} @@ -15572,6 +15658,8 @@ snapshots: dependencies: node-int64: 0.4.0 + buffer-crc32@1.0.0: {} + buffer-equal-constant-time@1.0.1: {} buffer-from@1.1.2: {} @@ -15965,6 +16053,14 @@ snapshots: compare-versions@4.1.4: {} + compress-commons@6.0.2: + dependencies: + crc-32: 1.2.2 + crc32-stream: 6.0.0 + is-stream: 2.0.1 + normalize-path: 3.0.0 + readable-stream: 4.7.0 + compressible@2.0.18: dependencies: mime-db: 1.54.0 @@ -16097,6 +16193,13 @@ snapshots: country-regex@1.1.0: {} + crc-32@1.2.2: {} + + crc32-stream@6.0.0: + dependencies: + crc-32: 1.2.2 + readable-stream: 4.7.0 + create-error-class@3.0.2: dependencies: capture-stack-trace: 1.0.2 @@ -17384,6 +17487,8 @@ snapshots: fast-equals@5.2.2: {} + fast-fifo@1.3.2: {} + fast-glob@3.3.3: dependencies: '@nodelib/fs.stat': 2.0.5 @@ -19399,6 +19504,10 @@ snapshots: layout-base@2.0.1: {} + lazystream@1.0.1: + dependencies: + readable-stream: 2.3.8 + ldap-filter@0.3.3: dependencies: assert-plus: 1.0.0 @@ -21516,6 +21625,18 @@ snapshots: string_decoder: 1.3.0 util-deprecate: 1.0.2 + readable-stream@4.7.0: + dependencies: + abort-controller: 3.0.0 + buffer: 6.0.3 + events: 3.3.0 + process: 0.11.10 + string_decoder: 1.3.0 + + readdir-glob@1.1.3: + dependencies: + minimatch: 5.1.6 + readdirp@3.6.0: dependencies: picomatch: 2.3.1 @@ -22241,6 +22362,13 @@ snapshots: streamsearch@1.1.0: {} + streamx@2.22.1: + dependencies: + fast-fifo: 1.3.2 + text-decoder: 1.2.3 + optionalDependencies: + bare-events: 2.6.0 + string-convert@0.2.1: {} string-length@4.0.2: @@ -22464,6 +22592,12 @@ snapshots: inherits: 2.0.4 readable-stream: 3.6.2 + tar-stream@3.1.7: + dependencies: + b4a: 1.6.7 + fast-fifo: 1.3.2 + streamx: 2.22.1 + tdigest@0.1.2: dependencies: bintrees: 1.0.2 @@ -22524,6 +22658,10 @@ snapshots: glob: 7.2.3 minimatch: 3.1.2 + text-decoder@1.2.3: + dependencies: + b4a: 1.6.7 + text-hex@1.0.0: {} text-table@0.2.0: {} @@ -23469,6 +23607,12 @@ snapshots: cmake-ts: 1.0.2 node-addon-api: 8.4.0 + zip-stream@6.0.1: + dependencies: + archiver-utils: 5.0.2 + compress-commons: 6.0.2 + readable-stream: 4.7.0 + zlibjs@0.3.1: {} zod-to-json-schema@3.21.4(zod@3.25.76): From 4f788e4eadaf7900271a80145436c20d7bcf0ea4 Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 5 Aug 2025 05:38:45 +0000 Subject: [PATCH 183/270] ouch -- rust based compression --- .../backend/files/sandbox/archiver.ts | 56 ++++++++++++------- src/packages/backend/files/sandbox/index.ts | 36 +++++++++--- src/packages/backend/files/sandbox/install.ts | 34 +++++++++-- .../backend/files/sandbox/ouch.test.ts | 56 +++++++++++++++++++ src/packages/backend/files/sandbox/ouch.ts | 52 +++++++++++++++++ src/packages/conat/files/fs.ts | 33 +++++++++-- 6 files changed, 229 insertions(+), 38 deletions(-) create mode 100644 src/packages/backend/files/sandbox/ouch.test.ts create mode 100644 src/packages/backend/files/sandbox/ouch.ts diff --git a/src/packages/backend/files/sandbox/archiver.ts b/src/packages/backend/files/sandbox/archiver.ts index 73f447a4e7..18ca316ca0 100644 --- a/src/packages/backend/files/sandbox/archiver.ts +++ b/src/packages/backend/files/sandbox/archiver.ts @@ -1,5 +1,5 @@ import Archiver from "archiver"; -import { createWriteStream } from "fs"; +import { createReadStream, createWriteStream } from "fs"; import { stat } from "fs/promises"; import { once } from "@cocalc/util/async-utils"; import { type ArchiverOptions } from "@cocalc/conat/files/fs"; @@ -7,45 +7,61 @@ export { type ArchiverOptions }; export default async function archiver( path: string, - paths: string[] | string, + // map from absolute path to path as it should appear in the archive + pathMap: { [absolutePath: string]: string | null }, options?: ArchiverOptions, ) { - if (options == null) { - if (path.endsWith(".zip")) { - options = { format: "zip" }; - } else if (path.endsWith(".tar")) { - options = { format: "tar" }; - } else if (path.endsWith(".tar.gz")) { - options = { format: "tar", gzip: true }; - } - } - if (typeof paths == "string") { - paths = [paths]; - } - const archive = new Archiver(options!.format, options); + options = { ...options }; + const format = getFormat(path, options); + const archive = new Archiver(format, options); + + let error: any = undefined; + + const timer = options.timeout + ? setTimeout(() => { + error = `Timeout after ${options.timeout} ms`; + archive.abort(); + }, options.timeout) + : undefined; + const output = createWriteStream(path); // docs say to listen before calling finalize const closed = once(output, "close"); archive.pipe(output); - let error: any = undefined; archive.on("error", (err) => { error = err; }); + const paths = Object.keys(pathMap); const isDir = async (path) => (await stat(path)).isDirectory(); const v = await Promise.all(paths.map(isDir)); for (let i = 0; i < paths.length; i++) { + const name = pathMap[paths[i]]; if (v[i]) { - archive.directory(paths[i]); + archive.directory(paths[i], name); } else { - archive.file(paths[i]); + archive.append(createReadStream(paths[i]), { name }); } } - await archive.finalize(); + archive.finalize(); await closed; + if (timer) { + clearTimeout(timer); + } if (error) { - throw error; + throw Error(error); + } +} + +function getFormat(path: string, options) { + if (path.endsWith(".zip")) { + return "zip"; + } else if (path.endsWith(".tar")) { + return "tar"; + } else if (path.endsWith(".tar.gz")) { + options.gzip = true; + return "tar"; } } diff --git a/src/packages/backend/files/sandbox/index.ts b/src/packages/backend/files/sandbox/index.ts index 5302f546f0..1f361dbc36 100644 --- a/src/packages/backend/files/sandbox/index.ts +++ b/src/packages/backend/files/sandbox/index.ts @@ -76,6 +76,7 @@ import dust, { type DustOptions } from "./dust"; import rustic from "./rustic"; import { type ExecOutput } from "./exec"; import { rusticRepo } from "@cocalc/backend/data"; +import ouch, { type OuchOptions } from "./ouch"; import archiver, { type ArchiverOptions } from "./archiver"; // max time code can run (in safe mode), e.g., for find, @@ -218,6 +219,7 @@ export class SandboxedFilesystem { ); }; + // find files fd = async (path: string, options?: FdOptions): Promise => { return await fd( await this.safeAbsPath(path), @@ -225,6 +227,7 @@ export class SandboxedFilesystem { ); }; + // disk usage dust = async (path: string, options?: DustOptions): Promise => { return await dust( await this.safeAbsPath(path), @@ -234,6 +237,19 @@ export class SandboxedFilesystem { ); }; + // compression + ouch = async (args: string[], options?: OuchOptions): Promise => { + options = { ...options }; + if (options.cwd) { + options.cwd = await this.safeAbsPath(options.cwd); + } + return await ouch( + [args[0]].concat(await Promise.all(args.slice(1).map(this.safeAbsPath))), + capTimeout(options, 6 * MAX_TIMEOUT), + ); + }; + + // backups rustic = async (args: string[]): Promise => { return await rustic(args, { repo: this.rusticRepo, @@ -246,17 +262,21 @@ export class SandboxedFilesystem { archiver = async ( path: string, - paths: string[] | string, + // map from path relative to sandbox to the name that path should get in the archive. + pathMap0: { [path: string]: string | null }, options?: ArchiverOptions, ): Promise => { - if (typeof paths == "string") { - paths = [paths]; + const pathMap: { [absPath: string]: string } = {}; + const v = Object.keys(pathMap0); + const absPaths = await Promise.all(v.map(this.safeAbsPath)); + for (let i = 0; i < v.length; i++) { + pathMap[absPaths[i]] = + pathMap0[v[i]] ?? absPaths[i].slice(this.path.length + 1); } - await archiver( - await this.safeAbsPath(path), - await Promise.all(paths.map(this.safeAbsPath)), - options, - ); + await archiver(await this.safeAbsPath(path), pathMap, { + timeout: 4 * MAX_TIMEOUT, + ...options, + }); }; ripgrep = async ( diff --git a/src/packages/backend/files/sandbox/install.ts b/src/packages/backend/files/sandbox/install.ts index d22f1d3634..9a66d895f7 100644 --- a/src/packages/backend/files/sandbox/install.ts +++ b/src/packages/backend/files/sandbox/install.ts @@ -33,6 +33,7 @@ interface Spec { path: string; stripComponents?: number; pathInArchive?: string; + skip?: string[]; } const SPEC = { @@ -57,6 +58,16 @@ const SPEC = { binary: "dust", path: join(binPath, "dust"), }, + ouch: { + // See https://github.com/ouch-org/ouch/releases + VERSION: "0.6.1", + BASE: "https://github.com/ouch-org/ouch/releases/download", + binary: "ouch", + path: join(binPath, "ouch"), + // See https://github.com/ouch-org/ouch/issues/45; note that ouch is in home brew + // for this platform. + skip: ["aarch64-apple-darwin"], + }, rustic: { // See https://github.com/rustic-rs/rustic/releases VERSION: "v0.9.5", @@ -72,6 +83,7 @@ export const ripgrep = SPEC.ripgrep.path; export const fd = SPEC.fd.path; export const dust = SPEC.dust.path; export const rustic = SPEC.rustic.path; +export const ouch = SPEC.ouch.path; type App = keyof typeof SPEC; @@ -101,6 +113,10 @@ export async function install(app?: App) { return; } const url = getUrl(app); + if (!url) { + logger.debug("install: skipping ", app); + return; + } logger.debug("install", { app, url }); // - 1. Fetch the tarball from the github url (using the fetch library) const response = await downloadFromGithub(url); @@ -118,7 +134,9 @@ export async function install(app?: App) { binary, path, stripComponents = 1, - pathInArchive = `${app}-${VERSION}-${getOS()}/${binary}`, + pathInArchive = app == "ouch" + ? `${app}-${getOS()}/${binary}` + : `${app}-${VERSION}-${getOS()}/${binary}`, } = SPEC[app] as Spec; const tmpFile = join(__dirname, `${app}-${VERSION}.tar.gz`); @@ -186,7 +204,7 @@ async function downloadFromGithub(url: string) { const delay = baseDelay * Math.pow(2, attempt - 1); console.log( - `Fetch failed. Retrying in ${delay}ms (attempt ${attempt}/${maxRetries})`, + `Fetch ${url} failed. Retrying in ${delay}ms (attempt ${attempt}/${maxRetries})`, ); await new Promise((resolve) => setTimeout(resolve, delay)); } @@ -195,8 +213,16 @@ async function downloadFromGithub(url: string) { } function getUrl(app: App) { - const { BASE, VERSION } = SPEC[app]; - return `${BASE}/${VERSION}/${app}-${VERSION}-${getOS()}.tar.gz`; + const { BASE, VERSION, skip } = SPEC[app] as Spec; + const os = getOS(); + if (skip?.includes(os)) { + return ""; + } + if (app == "ouch") { + return `${BASE}/${VERSION}/${app}-${os}.tar.gz`; + } else { + return `${BASE}/${VERSION}/${app}-${VERSION}-${os}.tar.gz`; + } } function getOS() { diff --git a/src/packages/backend/files/sandbox/ouch.test.ts b/src/packages/backend/files/sandbox/ouch.test.ts new file mode 100644 index 0000000000..5d9ded0afb --- /dev/null +++ b/src/packages/backend/files/sandbox/ouch.test.ts @@ -0,0 +1,56 @@ +/* +Test the ouch compression api. +*/ + +import ouch from "./ouch"; +import { mkdtemp, mkdir, rm, readFile, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { exists } from "@cocalc/backend/misc/async-utils-node"; + +let tempDir, options; +beforeAll(async () => { + tempDir = await mkdtemp(join(tmpdir(), "cocalc")); + options = { cwd: tempDir }; +}); +afterAll(async () => { + await rm(tempDir, { force: true, recursive: true }); +}); + +describe("ouch works on a little file", () => { + for (const ext of [ + "zip", + "7z", + "tar.gz", + "tar.xz", + "tar.bz", + "tar.bz2", + "tar.bz3", + "tar.lz4", + "tar.sz", + "tar.zst", + "tar.br", + ]) { + it(`create file and compress it up using ${ext}`, async () => { + await writeFile(join(tempDir, "a.txt"), "hello"); + const { truncated, code } = await ouch( + ["compress", "a.txt", `a.${ext}`], + options, + ); + expect(code).toBe(0); + expect(truncated).toBe(false); + expect(await exists(join(tempDir, `a.${ext}`))).toBe(true); + }); + + it(`extract ${ext} in subdirectory`, async () => { + await mkdir(join(tempDir, `target-${ext}`)); + const { code } = await ouch(["decompress", join(tempDir, `a.${ext}`)], { + cwd: join(tempDir, `target-${ext}`), + }); + expect(code).toBe(0); + expect( + (await readFile(join(tempDir, `target-${ext}`, "a.txt"))).toString(), + ).toEqual("hello"); + }); + } +}); diff --git a/src/packages/backend/files/sandbox/ouch.ts b/src/packages/backend/files/sandbox/ouch.ts new file mode 100644 index 0000000000..51b94a8270 --- /dev/null +++ b/src/packages/backend/files/sandbox/ouch.ts @@ -0,0 +1,52 @@ +/* + +https://github.com/ouch-org/ouch + +ouch stands for Obvious Unified Compression Helper. + +The .tar.gz support in 'ouch' is excellent -- super fast and memory efficient, +since it is fully parallel. + +*/ + +import exec, { type ExecOutput, validate } from "./exec"; +import { type OuchOptions } from "@cocalc/conat/files/fs"; +export { type OuchOptions }; +import { ouch as ouchPath } from "./install"; + +export default async function ouch( + args: string[], + { timeout, options, cwd }: OuchOptions = {}, +): Promise { + const command = args[0]; + if (!commands.includes(command)) { + throw Error(`first argument must be one of ${commands.join(", ")}`); + } + + return await exec({ + cmd: ouchPath, + cwd, + positionalArgs: args.slice(1), + safety: [command, "-y", "-q"], + timeout, + options, + whitelist, + }); +} + +const commands = ["compress", "c", "decompress", "d", "list", "l", "ls"]; + +const whitelist = { + "-H": true, + "--hidden": true, + g: true, + "--gitignore": true, + "-f": validate.str, + "--format": validate.str, + "-p": validate.str, + "--password": validate.str, + "-h": true, + "--help": true, + "-V": true, + "--version": true, +} as const; diff --git a/src/packages/conat/files/fs.ts b/src/packages/conat/files/fs.ts index f675a08ecd..e3a1670651 100644 --- a/src/packages/conat/files/fs.ts +++ b/src/packages/conat/files/fs.ts @@ -59,20 +59,26 @@ export interface DustOptions { maxSize?: number; } +export interface OuchOptions { + cwd?: string; + options?: string[]; + timeout?: number; +} + interface ZipOptions { - format: "zip"; comment?: string; forceLocalTime?: boolean; forceZip64?: boolean; namePrependSlash?: boolean; store?: boolean; zlib?: object; + timeout?: number; } interface TarOptions { - format: "tar"; gzip?: boolean; gzipOPtions?: object; + timeout?: number; } export type ArchiverOptions = ZipOptions | TarOptions; @@ -111,10 +117,14 @@ export interface Filesystem { archiver: ( path: string, // archive to create, e.g., a.zip, a.tar or a.tar.gz - paths: string, // paths to include in archive -- can be files or directories in the sandbox. + // map from path relative to sandbox to the name that path should get in the archive. + pathMap: { [path: string]: string | null }, options?: ArchiverOptions, // options -- see https://www.archiverjs.com/ ) => Promise; + // compression + ouch: (args: string[], options?: OuchOptions) => Promise; + // We add very little to the Filesystem api, but we have to add // a sandboxed "find" command, since it is a 1-call way to get // arbitrary directory listing info, which is just not possible @@ -287,8 +297,12 @@ export async function fsServer({ service, fs, client, project_id }: Options) { async appendFile(path: string, data: string | Buffer, encoding?) { await (await fs(this.subject)).appendFile(path, data, encoding); }, - async archiver(path: string, paths: string, options?: ArchiverOptions) { - return await (await fs(this.subject)).archiver(path, paths, options); + async archiver( + path: string, + pathMap: { [path: string]: string | null }, + options?: ArchiverOptions, + ) { + return await (await fs(this.subject)).archiver(path, pathMap, options); }, async chmod(path: string, mode: string | number) { await (await fs(this.subject)).chmod(path, mode); @@ -323,6 +337,9 @@ export async function fsServer({ service, fs, client, project_id }: Options) { async mkdir(path: string, options?) { await (await fs(this.subject)).mkdir(path, options); }, + async ouch(args: string[], options?: OuchOptions) { + return await (await fs(this.subject)).ouch(args, options); + }, async readFile(path: string, encoding?) { return await (await fs(this.subject)).readFile(path, encoding); }, @@ -451,15 +468,19 @@ export function fsSubject({ return `${getService({ service, compute_server_id })}.project-${project_id}`; } +const DEFAULT_FS_CALL_TIMEOUT = 5 * 60_000; + export function fsClient({ client, subject, + timeout = DEFAULT_FS_CALL_TIMEOUT, }: { client?: Client; subject: string; + timeout?: number; }): FilesystemClient { client ??= conat(); - let call = client.call(subject); + let call = client.call(subject, { timeout }); const readdir0 = call.readdir.bind(call); call.readdir = async (path: string, options?) => { From 606a5978b8b23783ff14aae32cde5f9befad4275 Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 5 Aug 2025 05:45:04 +0000 Subject: [PATCH 184/270] remove archiver integration completely -- it's just not fast enough to be worth it --- .../backend/files/sandbox/archiver.ts | 67 ------------------- src/packages/backend/files/sandbox/index.ts | 20 ------ .../backend/files/sandbox/ouch.test.ts | 2 +- src/packages/conat/files/fs.ts | 32 --------- 4 files changed, 1 insertion(+), 120 deletions(-) delete mode 100644 src/packages/backend/files/sandbox/archiver.ts diff --git a/src/packages/backend/files/sandbox/archiver.ts b/src/packages/backend/files/sandbox/archiver.ts deleted file mode 100644 index 18ca316ca0..0000000000 --- a/src/packages/backend/files/sandbox/archiver.ts +++ /dev/null @@ -1,67 +0,0 @@ -import Archiver from "archiver"; -import { createReadStream, createWriteStream } from "fs"; -import { stat } from "fs/promises"; -import { once } from "@cocalc/util/async-utils"; -import { type ArchiverOptions } from "@cocalc/conat/files/fs"; -export { type ArchiverOptions }; - -export default async function archiver( - path: string, - // map from absolute path to path as it should appear in the archive - pathMap: { [absolutePath: string]: string | null }, - options?: ArchiverOptions, -) { - options = { ...options }; - const format = getFormat(path, options); - const archive = new Archiver(format, options); - - let error: any = undefined; - - const timer = options.timeout - ? setTimeout(() => { - error = `Timeout after ${options.timeout} ms`; - archive.abort(); - }, options.timeout) - : undefined; - - const output = createWriteStream(path); - // docs say to listen before calling finalize - const closed = once(output, "close"); - archive.pipe(output); - - archive.on("error", (err) => { - error = err; - }); - - const paths = Object.keys(pathMap); - const isDir = async (path) => (await stat(path)).isDirectory(); - const v = await Promise.all(paths.map(isDir)); - for (let i = 0; i < paths.length; i++) { - const name = pathMap[paths[i]]; - if (v[i]) { - archive.directory(paths[i], name); - } else { - archive.append(createReadStream(paths[i]), { name }); - } - } - - archive.finalize(); - await closed; - if (timer) { - clearTimeout(timer); - } - if (error) { - throw Error(error); - } -} - -function getFormat(path: string, options) { - if (path.endsWith(".zip")) { - return "zip"; - } else if (path.endsWith(".tar")) { - return "tar"; - } else if (path.endsWith(".tar.gz")) { - options.gzip = true; - return "tar"; - } -} diff --git a/src/packages/backend/files/sandbox/index.ts b/src/packages/backend/files/sandbox/index.ts index 1f361dbc36..0994d207fd 100644 --- a/src/packages/backend/files/sandbox/index.ts +++ b/src/packages/backend/files/sandbox/index.ts @@ -77,7 +77,6 @@ import rustic from "./rustic"; import { type ExecOutput } from "./exec"; import { rusticRepo } from "@cocalc/backend/data"; import ouch, { type OuchOptions } from "./ouch"; -import archiver, { type ArchiverOptions } from "./archiver"; // max time code can run (in safe mode), e.g., for find, // ripgrep, fd, and dust. @@ -260,25 +259,6 @@ export class SandboxedFilesystem { }); }; - archiver = async ( - path: string, - // map from path relative to sandbox to the name that path should get in the archive. - pathMap0: { [path: string]: string | null }, - options?: ArchiverOptions, - ): Promise => { - const pathMap: { [absPath: string]: string } = {}; - const v = Object.keys(pathMap0); - const absPaths = await Promise.all(v.map(this.safeAbsPath)); - for (let i = 0; i < v.length; i++) { - pathMap[absPaths[i]] = - pathMap0[v[i]] ?? absPaths[i].slice(this.path.length + 1); - } - await archiver(await this.safeAbsPath(path), pathMap, { - timeout: 4 * MAX_TIMEOUT, - ...options, - }); - }; - ripgrep = async ( path: string, pattern: string, diff --git a/src/packages/backend/files/sandbox/ouch.test.ts b/src/packages/backend/files/sandbox/ouch.test.ts index 5d9ded0afb..96ae332833 100644 --- a/src/packages/backend/files/sandbox/ouch.test.ts +++ b/src/packages/backend/files/sandbox/ouch.test.ts @@ -5,7 +5,7 @@ Test the ouch compression api. import ouch from "./ouch"; import { mkdtemp, mkdir, rm, readFile, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; -import { join, resolve } from "node:path"; +import { join } from "node:path"; import { exists } from "@cocalc/backend/misc/async-utils-node"; let tempDir, options; diff --git a/src/packages/conat/files/fs.ts b/src/packages/conat/files/fs.ts index e3a1670651..67e397176f 100644 --- a/src/packages/conat/files/fs.ts +++ b/src/packages/conat/files/fs.ts @@ -65,24 +65,6 @@ export interface OuchOptions { timeout?: number; } -interface ZipOptions { - comment?: string; - forceLocalTime?: boolean; - forceZip64?: boolean; - namePrependSlash?: boolean; - store?: boolean; - zlib?: object; - timeout?: number; -} - -interface TarOptions { - gzip?: boolean; - gzipOPtions?: object; - timeout?: number; -} - -export type ArchiverOptions = ZipOptions | TarOptions; - export interface Filesystem { appendFile: (path: string, data: string | Buffer, encoding?) => Promise; chmod: (path: string, mode: string | number) => Promise; @@ -115,13 +97,6 @@ export interface Filesystem { // todo: typing watch: (path: string, options?) => Promise; - archiver: ( - path: string, // archive to create, e.g., a.zip, a.tar or a.tar.gz - // map from path relative to sandbox to the name that path should get in the archive. - pathMap: { [path: string]: string | null }, - options?: ArchiverOptions, // options -- see https://www.archiverjs.com/ - ) => Promise; - // compression ouch: (args: string[], options?: OuchOptions) => Promise; @@ -297,13 +272,6 @@ export async function fsServer({ service, fs, client, project_id }: Options) { async appendFile(path: string, data: string | Buffer, encoding?) { await (await fs(this.subject)).appendFile(path, data, encoding); }, - async archiver( - path: string, - pathMap: { [path: string]: string | null }, - options?: ArchiverOptions, - ) { - return await (await fs(this.subject)).archiver(path, pathMap, options); - }, async chmod(path: string, mode: string | number) { await (await fs(this.subject)).chmod(path, mode); }, From 31b040a31913f164767d56289c84de42e580503b Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 5 Aug 2025 05:56:33 +0000 Subject: [PATCH 185/270] ability to compress directory using new fs api --- .../project/explorer/create-archive.tsx | 34 ++++++++++++------- src/packages/frontend/project_actions.ts | 1 - 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/src/packages/frontend/project/explorer/create-archive.tsx b/src/packages/frontend/project/explorer/create-archive.tsx index 1ae72fb230..2318a45e24 100644 --- a/src/packages/frontend/project/explorer/create-archive.tsx +++ b/src/packages/frontend/project/explorer/create-archive.tsx @@ -8,6 +8,9 @@ import { labels } from "@cocalc/frontend/i18n"; import { useProjectContext } from "@cocalc/frontend/project/context"; import { path_split, plural } from "@cocalc/util/misc"; import CheckedFiles from "./checked-files"; +import { join } from "path"; + +const FORMAT = ".tar.gz"; export default function CreateArchive({ clear }) { const intl = useIntl(); @@ -41,19 +44,22 @@ export default function CreateArchive({ clear }) { setLoading(true); const files = checked_files.toArray(); const path = store.get("current_path"); - await actions.zip_files({ - src: path ? files.map((x) => x.slice(path.length + 1)) : files, - dest: target + ".zip", - path, - }); + const fs = actions.fs(); + const { code, stderr } = await fs.ouch([ + "compress", + ...files, + join(path, target + FORMAT), + ]); + if (code) { + throw Error(Buffer.from(stderr).toString()); + } + clear(); } catch (err) { setLoading(false); setError(err); } finally { setLoading(false); } - - clear(); }; if (actions == null) { @@ -63,8 +69,8 @@ export default function CreateArchive({ clear }) { return ( - Create a zip file from the following {checked_files?.size} selected{" "} - {plural(checked_files?.size, "item")} + Create a downloadable {FORMAT} archive from the following{" "} + {checked_files?.size} selected {plural(checked_files?.size, "item")} > @@ -74,9 +80,9 @@ export default function CreateArchive({ clear }) { autoFocus onChange={(e) => setTarget(e.target.value)} value={target} - placeholder="Name of zip archive..." + placeholder="Name of archive..." onPressEnter={doCompress} - suffix=".zip" + suffix={FORMAT} />
- + ); } diff --git a/src/packages/frontend/project_actions.ts b/src/packages/frontend/project_actions.ts index cbbd0fd8bc..df812641e0 100644 --- a/src/packages/frontend/project_actions.ts +++ b/src/packages/frontend/project_actions.ts @@ -1746,7 +1746,6 @@ export class ProjectActions extends Actions { }; set_file_action = (action?: FileAction): void => { - console.trace("set_file_action", action); const store = this.get_store(); if (store == null) { return; From 7eafe94b1a098f6b9014bfde616305af5811846c Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 5 Aug 2025 15:16:56 +0000 Subject: [PATCH 186/270] ouch: add some missing options --- .../backend/files/sandbox/exec.test.ts | 30 +++++++++++++++++++ src/packages/backend/files/sandbox/exec.ts | 28 ++++++++++++++++- src/packages/backend/files/sandbox/ouch.ts | 17 +++++++++++ 3 files changed, 74 insertions(+), 1 deletion(-) create mode 100644 src/packages/backend/files/sandbox/exec.test.ts diff --git a/src/packages/backend/files/sandbox/exec.test.ts b/src/packages/backend/files/sandbox/exec.test.ts new file mode 100644 index 0000000000..8dcbc38332 --- /dev/null +++ b/src/packages/backend/files/sandbox/exec.test.ts @@ -0,0 +1,30 @@ +/* +Test the exec command. +*/ + +import exec from "./exec"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +let tempDir; +beforeAll(async () => { + tempDir = await mkdtemp(join(tmpdir(), "cocalc")); +}); +afterAll(async () => { + await rm(tempDir, { force: true, recursive: true }); +}); + +describe("exec works", () => { + it(`create file and run ls command`, async () => { + await writeFile(join(tempDir, "a.txt"), "hello"); + const { stderr, stdout, truncated, code } = await exec({ + cmd: "ls", + cwd: tempDir, + }); + expect(code).toBe(0); + expect(truncated).toBe(false); + expect(Buffer.from(stdout).toString()).toEqual("a.txt\n"); + expect(Buffer.from(stderr).toString()).toEqual(""); + }); +}); diff --git a/src/packages/backend/files/sandbox/exec.ts b/src/packages/backend/files/sandbox/exec.ts index be17c9b676..d84dd688a6 100644 --- a/src/packages/backend/files/sandbox/exec.ts +++ b/src/packages/backend/files/sandbox/exec.ts @@ -1,4 +1,4 @@ -import { spawn } from "node:child_process"; +import { execFile, spawn } from "node:child_process"; import { arch } from "node:os"; import { type ExecOutput } from "@cocalc/conat/files/fs"; export { type ExecOutput }; @@ -39,6 +39,10 @@ export interface Options { // options that are always included first for safety and need NOT match whitelist safety?: string[]; + + // if nodejs is running as root and give this username, then cmd runs as this + // user instead. + username?: string; } type ValidateFunction = (value: string) => void; @@ -55,6 +59,7 @@ export default async function exec({ timeout = DEFAULT_TIMEOUT, whitelist = {}, cwd, + username, }: Options): Promise { if (arch() == "darwin") { options = options.concat(darwin); @@ -62,6 +67,7 @@ export default async function exec({ options = options.concat(linux); } options = safety.concat(parseAndValidateOptions(options, whitelist)); + const userId = username ? await getUserIds(username) : undefined; return new Promise((resolve, reject) => { const stdoutChunks: Buffer[] = []; @@ -81,6 +87,7 @@ export default async function exec({ stdio: ["ignore", "pipe", "pipe"], env: {}, cwd, + ...userId, }); let timeoutHandle: NodeJS.Timeout | null = null; @@ -196,3 +203,22 @@ export const validate = { } }, }; + +async function getUserIds( + username: string, +): Promise<{ uid: number; gid: number }> { + return Promise.all([ + new Promise((resolve, reject) => { + execFile("id", ["-u", username], (err, stdout) => { + if (err) return reject(err); + resolve(parseInt(stdout.trim(), 10)); + }); + }), + new Promise((resolve, reject) => { + execFile("id", ["-g", username], (err, stdout) => { + if (err) return reject(err); + resolve(parseInt(stdout.trim(), 10)); + }); + }), + ]).then(([uid, gid]) => ({ uid, gid })); +} diff --git a/src/packages/backend/files/sandbox/ouch.ts b/src/packages/backend/files/sandbox/ouch.ts index 51b94a8270..69754330b4 100644 --- a/src/packages/backend/files/sandbox/ouch.ts +++ b/src/packages/backend/files/sandbox/ouch.ts @@ -37,6 +37,7 @@ export default async function ouch( const commands = ["compress", "c", "decompress", "d", "list", "l", "ls"]; const whitelist = { + // general options, "-H": true, "--hidden": true, g: true, @@ -49,4 +50,20 @@ const whitelist = { "--help": true, "-V": true, "--version": true, + + // compression-specific options + // do NOT enable '-S, --follow-symlinks' as that could escape the sandbox! + // It's off by default. + + "-l": validate.str, + "--level": validate.str, + "--fast": true, + "--slow": true, + + // decompress specific options + "-d": validate.str, + "--dir": validate.str, + r: true, + "--remove": true, + "--no-smart-unpack": true, } as const; From 2036af7a90d934388d1a9363ca9389004181ca08 Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 5 Aug 2025 15:29:51 +0000 Subject: [PATCH 187/270] compress -- allow user to select format --- src/packages/conat/files/fs.ts | 14 ++++++++++ .../project/explorer/create-archive.tsx | 27 +++++++++++++++---- 2 files changed, 36 insertions(+), 5 deletions(-) diff --git a/src/packages/conat/files/fs.ts b/src/packages/conat/files/fs.ts index 67e397176f..e86ccbfcad 100644 --- a/src/packages/conat/files/fs.ts +++ b/src/packages/conat/files/fs.ts @@ -65,6 +65,20 @@ export interface OuchOptions { timeout?: number; } +export const OUCH_FORMATS = [ + "zip", + "7z", + "tar.gz", + "tar.xz", + "tar.bz", + "tar.bz2", + "tar.bz3", + "tar.lz4", + "tar.sz", + "tar.zst", + "tar.br", +]; + export interface Filesystem { appendFile: (path: string, data: string | Buffer, encoding?) => Promise; chmod: (path: string, mode: string | number) => Promise; diff --git a/src/packages/frontend/project/explorer/create-archive.tsx b/src/packages/frontend/project/explorer/create-archive.tsx index 2318a45e24..74bcf4550e 100644 --- a/src/packages/frontend/project/explorer/create-archive.tsx +++ b/src/packages/frontend/project/explorer/create-archive.tsx @@ -1,4 +1,4 @@ -import { Button, Card, Input, Space, Spin } from "antd"; +import { Button, Card, Input, Select, Space, Spin } from "antd"; import { useEffect, useRef, useState } from "react"; import { useIntl } from "react-intl"; import { default_filename } from "@cocalc/frontend/account"; @@ -9,10 +9,16 @@ import { useProjectContext } from "@cocalc/frontend/project/context"; import { path_split, plural } from "@cocalc/util/misc"; import CheckedFiles from "./checked-files"; import { join } from "path"; +import { OUCH_FORMATS } from "@cocalc/conat/files/fs"; -const FORMAT = ".tar.gz"; +const defaultFormat = OUCH_FORMATS.includes("tar.gz") + ? "tar.gz" + : OUCH_FORMATS[0]; export default function CreateArchive({ clear }) { + const [format, setFormat] = useState( + localStorage.defaultCompressionFormat ?? defaultFormat, + ); const intl = useIntl(); const inputRef = useRef(null); const { actions } = useProjectContext(); @@ -48,7 +54,7 @@ export default function CreateArchive({ clear }) { const { code, stderr } = await fs.ouch([ "compress", ...files, - join(path, target + FORMAT), + join(path, target + "." + format), ]); if (code) { throw Error(Buffer.from(stderr).toString()); @@ -69,7 +75,7 @@ export default function CreateArchive({ clear }) { return ( - Create a downloadable {FORMAT} archive from the following{" "} + Create a downloadable {format} archive from the following{" "} {checked_files?.size} selected {plural(checked_files?.size, "item")} > @@ -82,7 +88,7 @@ export default function CreateArchive({ clear }) { value={target} placeholder="Name of archive..." onPressEnter={doCompress} - suffix={FORMAT} + suffix={"." + format} />
+ { - return { value }; - })} - onChange={(format) => { - setFormat(format); - localStorage.defaultCompressionFormat = format; - }} - /> + ); } + +export async function createArchive({ path, files, target, format, actions }) { + const fs = actions.fs(); + const { code, stderr } = await fs.ouch([ + "compress", + ...files, + join(path, target + "." + format), + ]); + if (code) { + throw Error(Buffer.from(stderr).toString()); + } +} + +export function SelectFormat({ format, setFormat }) { + useEffect(() => { + if (!OUCH_FORMATS.includes(format)) { + if (OUCH_FORMATS.includes(localStorage.defaultCompressionFormat)) { + setFormat(localStorage.defaultCompressionFormat); + } else { + setFormat(defaultFormat); + } + } + }, [format]); + + return ( + { + setTitle(e.target.value); + titleRef.current = e.target.value; + }} + /> +
+ ); +} diff --git a/src/packages/frontend/project/explorer/misc-side-buttons.tsx b/src/packages/frontend/project/explorer/misc-side-buttons.tsx index 5d9fd3d090..c8c8d36530 100644 --- a/src/packages/frontend/project/explorer/misc-side-buttons.tsx +++ b/src/packages/frontend/project/explorer/misc-side-buttons.tsx @@ -21,6 +21,7 @@ import { type JSX, type MouseEvent } from "react"; const SHOW_SERVER_LAUNCHERS = false; import TourButton from "./tour/button"; +import ForkProject from "./fork"; const OPEN_MSG = defineMessage({ id: "project.explorer.misc-side-buttons.open_dir.tooltip", @@ -185,6 +186,7 @@ export function MiscSideButtons() { {render_hidden_toggle()} {render_backup()} +
diff --git a/src/packages/frontend/project/page/flyouts/files-header.tsx b/src/packages/frontend/project/page/flyouts/files-header.tsx index 19874e28dd..02f384b0e7 100644 --- a/src/packages/frontend/project/page/flyouts/files-header.tsx +++ b/src/packages/frontend/project/page/flyouts/files-header.tsx @@ -7,7 +7,6 @@ import { Alert, Button, Input, InputRef, Radio, Space, Tooltip } from "antd"; import immutable from "immutable"; import { FormattedMessage, useIntl } from "react-intl"; import { VirtuosoHandle } from "react-virtuoso"; - import { Button as BootstrapButton } from "@cocalc/frontend/antd-bootstrap"; import { CSS, @@ -36,6 +35,7 @@ import { ActiveFileSort } from "./files"; import { FilesSelectedControls } from "./files-controls"; import { FilesSelectButtons } from "./files-select-extra"; import { FlyoutClearFilter, FlyoutFilterWarning } from "./filter-warning"; +import ForkProject from "@cocalc/frontend/project/explorer/fork"; function searchToFilename(search: string): string { if (search.endsWith(" ")) { @@ -479,6 +479,7 @@ export function FilesHeader(props: Readonly): React.JSX.Element { } icon={} /> + ) : undefined}
diff --git a/src/packages/frontend/projects/actions.ts b/src/packages/frontend/projects/actions.ts index 5fb8e793e0..ba281d4465 100644 --- a/src/packages/frontend/projects/actions.ts +++ b/src/packages/frontend/projects/actions.ts @@ -417,7 +417,7 @@ export class ProjectsActions extends Actions { } // Open the given project - public async open_project(opts: { + open_project = async (opts: { project_id: string; // id of the project to open target?: string; // The file path to open fragmentId?: FragmentId; // if given, an uri fragment in the editor that is opened. @@ -425,7 +425,7 @@ export class ProjectsActions extends Actions { ignore_kiosk?: boolean; // Ignore ?fullscreen=kiosk change_history?: boolean; // (default: true) Whether or not to alter browser history restore_session?: boolean; // (default: true) Opens up previously closed editor tabs - }) { + }) => { opts = defaults(opts, { project_id: undefined, target: undefined, @@ -486,7 +486,7 @@ export class ProjectsActions extends Actions { } // initialize project project_actions.init(); - } + }; // tab at old_index taken out and then inserted into the resulting array's new index public move_project_tab({ diff --git a/src/packages/frontend/projects/project-row.tsx b/src/packages/frontend/projects/project-row.tsx index 5631972a8d..08c03de776 100644 --- a/src/packages/frontend/projects/project-row.tsx +++ b/src/packages/frontend/projects/project-row.tsx @@ -34,7 +34,7 @@ import track from "@cocalc/frontend/user-tracking"; import { DEFAULT_COMPUTE_IMAGE } from "@cocalc/util/db-schema"; import { KUCALC_COCALC_COM } from "@cocalc/util/db-schema/site-defaults"; import { COLORS } from "@cocalc/util/theme"; -import { Avatar } from "antd"; +import { Avatar, Button, Tooltip } from "antd"; import { CSSProperties, useEffect } from "react"; import { ProjectUsers } from "./project-users"; @@ -215,7 +215,7 @@ export const ProjectRow: React.FC = ({ project_id, index }: Props) => { = ({ project_id, index }: Props) => { /> )} + + {!is_anonymous && ( + + + + )} + ); From e0ac383ba0a8e5a0a427329a8a76c31148cd9031 Mon Sep 17 00:00:00 2001 From: William Stein Date: Mon, 11 Aug 2025 04:15:20 +0000 Subject: [PATCH 259/270] file-server -- start adding rustic support --- .../backend/conat/files/local-path.ts | 2 +- src/packages/backend/sandbox/exec.ts | 2 +- src/packages/backend/sandbox/index.ts | 35 +++++-- src/packages/backend/sandbox/rustic.ts | 40 ++++++-- src/packages/conat/hub/api/projects.ts | 8 ++ src/packages/file-server/btrfs/filesystem.ts | 12 +++ .../file-server/btrfs/subvolume-rustic.ts | 92 +++++++++++++++++++ src/packages/file-server/btrfs/subvolume.ts | 8 +- src/packages/file-server/btrfs/subvolumes.ts | 1 - .../file-server/btrfs/test/subvolume.test.ts | 22 ++++- src/packages/frontend/compute/clone.tsx | 2 +- .../frontend/project/explorer/fork.tsx | 7 +- src/packages/server/conat/api/projects.ts | 14 +++ .../server/conat/file-server/index.ts | 4 +- 14 files changed, 223 insertions(+), 26 deletions(-) create mode 100644 src/packages/file-server/btrfs/subvolume-rustic.ts diff --git a/src/packages/backend/conat/files/local-path.ts b/src/packages/backend/conat/files/local-path.ts index 6ad630ad51..17857d27fb 100644 --- a/src/packages/backend/conat/files/local-path.ts +++ b/src/packages/backend/conat/files/local-path.ts @@ -36,7 +36,7 @@ export async function localPathFileserver({ const project_id = getProjectId(subject); const fsclient = createFileClient({ client }); const { path } = await fsclient.mount({ project_id }); - return new SandboxedFilesystem(path, { unsafeMode, project_id }); + return new SandboxedFilesystem(path, { unsafeMode, host: project_id }); } }, }); diff --git a/src/packages/backend/sandbox/exec.ts b/src/packages/backend/sandbox/exec.ts index fd558c92ee..cfbb06421c 100644 --- a/src/packages/backend/sandbox/exec.ts +++ b/src/packages/backend/sandbox/exec.ts @@ -93,7 +93,7 @@ export default async function exec({ cmd = nsjailPath; } - // console.log(`${cmd} ${args.join(" ")}`); + // console.log(`${cmd} ${args.join(" ")}`, { cwd }); const child = spawn(cmd, args, { stdio: ["ignore", "pipe", "pipe"], env: {}, diff --git a/src/packages/backend/sandbox/index.ts b/src/packages/backend/sandbox/index.ts index 8400e53c54..6a87e40514 100644 --- a/src/packages/backend/sandbox/index.ts +++ b/src/packages/backend/sandbox/index.ts @@ -92,7 +92,8 @@ interface Options { unsafeMode?: boolean; // readonly -- only allow operations that don't change files readonly?: boolean; - project_id?: string; + host?: string; + rusticRepo?: string; } // If you add any methods below that are NOT for the public api @@ -106,22 +107,28 @@ const INTERNAL_METHODS = new Set([ "readonly", "assertWritable", "rusticRepo", - "project_id", + "host", ]); export class SandboxedFilesystem { public readonly unsafeMode: boolean; public readonly readonly: boolean; - private readonly rusticRepo: string = rusticRepo; - private project_id?: string; + public rusticRepo: string; + private host?: string; constructor( // path should be the path to a FOLDER on the filesystem (not a file) public readonly path: string, - { unsafeMode = false, readonly = false, project_id }: Options = {}, + { + unsafeMode = false, + readonly = false, + host = "global", + rusticRepo: repo, + }: Options = {}, ) { this.unsafeMode = !!unsafeMode; this.readonly = !!readonly; - this.project_id = project_id; + this.host = host; + this.rusticRepo = repo ?? rusticRepo; for (const f in this) { if (INTERNAL_METHODS.has(f)) { continue; @@ -285,13 +292,21 @@ export class SandboxedFilesystem { }; // backups - rustic = async (args: string[]): Promise => { + rustic = async ( + args: string[], + { + timeout = 120_000, + maxSize = 10_000, + cwd, + }: { timeout?: number; maxSize?: number; cwd?: string } = {}, + ): Promise => { return await rustic(args, { repo: this.rusticRepo, safeAbsPath: this.safeAbsPath, - timeout: 120_000, - maxSize: 10_000, - host: this.project_id ?? "global", + timeout, + maxSize, + host: this.host, + cwd, }); }; diff --git a/src/packages/backend/sandbox/rustic.ts b/src/packages/backend/sandbox/rustic.ts index 03d9240cc7..28bb7e4a96 100644 --- a/src/packages/backend/sandbox/rustic.ts +++ b/src/packages/backend/sandbox/rustic.ts @@ -67,25 +67,32 @@ export interface RusticOptions { repo?: string; timeout?: number; maxSize?: number; - safeAbsPath: (path: string) => Promise; - host: string; + safeAbsPath?: (path: string) => Promise; + host?: string; + cwd?: string; } export default async function rustic( args: string[], options: RusticOptions, ): Promise { - const { timeout, maxSize, repo = rusticRepo, safeAbsPath, host } = options; + const { + timeout, + maxSize, + repo = rusticRepo, + safeAbsPath, + host = "host", + } = options; await ensureInitialized(repo); - const base = await safeAbsPath(""); + const cwd = await safeAbsPath?.(options.cwd ?? ""); const common = ["--password", "", "-r", repo]; const run = async (sanitizedArgs: string[]) => { return await exec({ cmd: rusticPath, - cwd: base, + cwd, safety: [...common, args[0], ...sanitizedArgs], maxSize, timeout, @@ -93,17 +100,33 @@ export default async function rustic( }; switch (args[0]) { + case "init": { + if (safeAbsPath != null) { + throw Error("init not allowed"); + } + return await run([]); + } case "backup": { + if (safeAbsPath == null || cwd == null) { + throw Error("safeAbsPath must be specified when making a backup"); + } if (args.length == 1) { throw Error("missing backup source"); } - const source = (await safeAbsPath(args.slice(-1)[0])).slice(base.length); + const source = (await safeAbsPath(args.slice(-1)[0])).slice(cwd.length); const options = parseAndValidateOptions( args.slice(1, -1), whitelist.backup, ); - return await run([...options, "--no-scan", "--host", host, "--", source]); + return await run([ + ...options, + "--no-scan", + "--host", + host, + "--", + source ? source : ".", + ]); } case "snapshots": { const options = parseAndValidateOptions( @@ -125,6 +148,9 @@ export default async function rustic( if (args.length <= 2) { throw Error("missing "); } + if (safeAbsPath == null) { + throw Error("safeAbsPath must be specified when restoring"); + } const snapshot = args.slice(-2)[0]; // await assertValidSnapshot({ snapshot, host, repo }); const destination = await safeAbsPath(args.slice(-1)[0]); // diff --git a/src/packages/conat/hub/api/projects.ts b/src/packages/conat/hub/api/projects.ts index 95d8efd1b5..fa79963587 100644 --- a/src/packages/conat/hub/api/projects.ts +++ b/src/packages/conat/hub/api/projects.ts @@ -10,6 +10,8 @@ export const projects = { inviteCollaborator: authFirstRequireAccount, inviteCollaboratorWithoutAccount: authFirstRequireAccount, setQuotas: authFirstRequireAccount, + + getDiskQuota: authFirstRequireAccount, }; export type AddCollaborator = @@ -89,6 +91,7 @@ export interface Projects { }; }) => Promise; + // for admins only! setQuotas: (opts: { account_id?: string; project_id: string; @@ -102,4 +105,9 @@ export interface Projects { member_host?: number; always_running?: number; }) => Promise; + + getDiskQuota: (opts: { + account_id?: string; + project_id: string; + }) => Promise<{ used: number; size: number }>; } diff --git a/src/packages/file-server/btrfs/filesystem.ts b/src/packages/file-server/btrfs/filesystem.ts index 248e086630..e29de633cb 100644 --- a/src/packages/file-server/btrfs/filesystem.ts +++ b/src/packages/file-server/btrfs/filesystem.ts @@ -18,6 +18,7 @@ import { Subvolumes } from "./subvolumes"; import { mkdir } from "fs/promises"; import { exists } from "@cocalc/backend/misc/async-utils-node"; import { executeCode } from "@cocalc/backend/execute-code"; +import rustic from "@cocalc/backend/sandbox/rustic"; // default size of btrfs filesystem if creating an image file. const DEFAULT_FILESYSTEM_SIZE = "10G"; @@ -48,6 +49,7 @@ export interface Options { export class Filesystem { public readonly opts: Options; public readonly bup: string; + public readonly rustic: string; public readonly subvolumes: Subvolumes; constructor(opts: Options) { @@ -58,6 +60,7 @@ export class Filesystem { }; this.opts = opts; this.bup = join(this.opts.mount, "bup"); + this.rustic = join(this.opts.mount, "rustic"); this.subvolumes = new Subvolumes(this); } @@ -69,6 +72,7 @@ export class Filesystem { args: ["quota", "enable", "--simple", this.opts.mount], }); await this.initBup(); + await this.initRustic(); await this.sync(); }; @@ -186,6 +190,14 @@ export class Filesystem { env: { BUP_DIR: this.bup }, }); }; + + private initRustic = async () => { + if (await exists(this.rustic)) { + return; + } + await mkdir(this.rustic); + await rustic(["init"], { repo: this.rustic }); + }; } function isImageFile(name: string) { diff --git a/src/packages/file-server/btrfs/subvolume-rustic.ts b/src/packages/file-server/btrfs/subvolume-rustic.ts new file mode 100644 index 0000000000..0a0e0a79ca --- /dev/null +++ b/src/packages/file-server/btrfs/subvolume-rustic.ts @@ -0,0 +1,92 @@ +/* +Rustic Architecture: + +The minimal option is a single global repo stored in the btrfs filesystem. +Obviously, admins should rsync this regularly to a separate location as a +genuine backup strategy. It's better to configure repo on separate +storage. Rustic has a very wide range of options. + +Instead of using btrfs send/recv for backups, we use Rustic because: + - much easier to check backups are valid + - globally compressed and dedup'd! btrfs send/recv is NOT globally dedupd + - decoupled from any btrfs issues + - rustic has full support for using cloud buckets as hot/cold storage + - not tied to any specific filesystem at all + - easier to offsite via incremental rsync + - much more space efficient with *global* dedup and compression + - rustic "is" restic, which is very mature and proven + - rustic is VERY fast, being parallel and in rust. +*/ + +import { type Subvolume } from "./subvolume"; +import getLogger from "@cocalc/backend/logger"; + +const RUSTIC_SNAPSHOT = "temp-rustic-snapshot"; + +const logger = getLogger("file-server:btrfs:subvolume-rustic"); + +interface Snapshot { + id: string; + time: Date; +} + +export class SubvolumeRustic { + constructor(private subvolume: Subvolume) {} + + // create a new rustic backup + backup = async ({ timeout = 30 * 60 * 1000 }: { timeout?: number } = {}) => { + if (await this.subvolume.snapshots.exists(RUSTIC_SNAPSHOT)) { + logger.debug(`backup: deleting existing ${RUSTIC_SNAPSHOT}`); + await this.subvolume.snapshots.delete(RUSTIC_SNAPSHOT); + } + const target = this.subvolume.snapshots.path(RUSTIC_SNAPSHOT); + try { + logger.debug( + `backup: creating ${RUSTIC_SNAPSHOT} to get a consistent backup`, + ); + await this.subvolume.snapshots.create(RUSTIC_SNAPSHOT); + logger.debug(`backup: backing up ${RUSTIC_SNAPSHOT} using rustic`); + const { stderr, code } = await this.subvolume.fs.rustic( + ["backup", "-x", "."], + { + timeout, + cwd: target, + }, + ); + if (code) { + throw Error(stderr.toString()); + } + } finally { + logger.debug(`backup: deleting temporary ${RUSTIC_SNAPSHOT}`); + await this.subvolume.snapshots.delete(RUSTIC_SNAPSHOT); + } + }; + + snapshots = async (): Promise => { + const { stdout, stderr, code } = await this.subvolume.fs.rustic([ + "snapshots", + "--json", + ]); + if (code) { + throw Error(stderr.toString()); + } else { + const x = JSON.parse(stdout.toString()); + return x[0][1].map(({ time, id }) => { + return { time: new Date(time), id }; + }); + } + }; + + ls = async (id: string) => { + const { stdout, stderr, code } = await this.subvolume.fs.rustic([ + "ls", + "--json", + id, + ]); + if (code) { + throw Error(stderr.toString()); + } else { + return JSON.parse(stdout.toString()); + } + }; +} diff --git a/src/packages/file-server/btrfs/subvolume.ts b/src/packages/file-server/btrfs/subvolume.ts index a02be9d7cd..7eedfc0806 100644 --- a/src/packages/file-server/btrfs/subvolume.ts +++ b/src/packages/file-server/btrfs/subvolume.ts @@ -7,6 +7,7 @@ import refCache from "@cocalc/util/refcache"; import { sudo } from "./util"; import { join } from "path"; import { SubvolumeBup } from "./subvolume-bup"; +import { SubvolumeRustic } from "./subvolume-rustic"; import { SubvolumeSnapshots } from "./subvolume-snapshots"; import { SubvolumeQuota } from "./subvolume-quota"; import { SandboxedFilesystem } from "@cocalc/backend/sandbox"; @@ -27,6 +28,7 @@ export class Subvolume { public readonly path: string; public readonly fs: SandboxedFilesystem; public readonly bup: SubvolumeBup; + public readonly rustic: SubvolumeRustic; public readonly snapshots: SubvolumeSnapshots; public readonly quota: SubvolumeQuota; @@ -34,8 +36,12 @@ export class Subvolume { this.filesystem = filesystem; this.name = name; this.path = join(filesystem.opts.mount, name); - this.fs = new SandboxedFilesystem(this.path); + this.fs = new SandboxedFilesystem(this.path, { + rusticRepo: filesystem.rustic, + host: this.name, + }); this.bup = new SubvolumeBup(this); + this.rustic = new SubvolumeRustic(this); this.snapshots = new SubvolumeSnapshots(this); this.quota = new SubvolumeQuota(this); } diff --git a/src/packages/file-server/btrfs/subvolumes.ts b/src/packages/file-server/btrfs/subvolumes.ts index 63a6ee992f..e4f10191b9 100644 --- a/src/packages/file-server/btrfs/subvolumes.ts +++ b/src/packages/file-server/btrfs/subvolumes.ts @@ -26,7 +26,6 @@ export class Subvolumes { return await subvolume({ filesystem: this.filesystem, name }); }; - // create a subvolume by cloning an existing one. clone = async (source: string, dest: string) => { logger.debug("clone ", { source, dest }); if (RESERVED.has(dest)) { diff --git a/src/packages/file-server/btrfs/test/subvolume.test.ts b/src/packages/file-server/btrfs/test/subvolume.test.ts index fe9286935e..21a46c6f07 100644 --- a/src/packages/file-server/btrfs/test/subvolume.test.ts +++ b/src/packages/file-server/btrfs/test/subvolume.test.ts @@ -227,7 +227,27 @@ describe("test snapshots", () => { }); }); -describe("test bup backups", () => { +describe("test rustic backups", () => { + let vol: Subvolume; + it("creates a volume", async () => { + vol = await fs.subvolumes.get("rustic-test"); + await vol.fs.writeFile("a.txt", "hello"); + }); + + it("create a rustic backup", async () => { + await vol.rustic.backup(); + }); + + it("confirm a.txt is in our backup", async () => { + const v = await vol.rustic.snapshots(); + expect(v.length == 1); + expect(Math.abs(Date.now() - v[0].time.valueOf())).toBeLessThan(10000); + const w = await vol.rustic.ls(v[0].id); + expect(w).toEqual([".snapshots", "a.txt"]); + }); +}); + +describe.skip("test bup backups", () => { let vol: Subvolume; it("creates a volume", async () => { vol = await fs.subvolumes.get("bup-test"); diff --git a/src/packages/frontend/compute/clone.tsx b/src/packages/frontend/compute/clone.tsx index 6c7c281526..c2c5dcc696 100644 --- a/src/packages/frontend/compute/clone.tsx +++ b/src/packages/frontend/compute/clone.tsx @@ -78,7 +78,7 @@ export async function cloneConfiguration({ const server = servers[0] as ComputeServerUserInfo; if (!noChange) { let n = 1; - let title = `Clone of ${server.title}`; + let title = `Fork of ${server.title}`; const titles = new Set(servers.map((x) => x.title)); if (titles.has(title)) { while (titles.has(title + ` (${n})`)) { diff --git a/src/packages/frontend/project/explorer/fork.tsx b/src/packages/frontend/project/explorer/fork.tsx index f3af68fb42..240f5cd047 100644 --- a/src/packages/frontend/project/explorer/fork.tsx +++ b/src/packages/frontend/project/explorer/fork.tsx @@ -1,4 +1,4 @@ -import { useRef, useState } from "react"; +import { useEffect, useRef, useState } from "react"; import { Button, Input, Popconfirm, Tooltip } from "antd"; import { Icon } from "@cocalc/frontend/components/icon"; import { ProjectTitle } from "@cocalc/frontend/projects/project-title"; @@ -59,11 +59,14 @@ export default function ForkProject({ project_id, flyout }: Props) { function Description({ project_id, titleRef }) { const [title, setTitle] = useState( - `Clone of ${ + `Fork of ${ redux.getStore("projects").getIn(["project_map", project_id, "title"]) ?? "project" }`, ); + useEffect(() => { + titleRef.current = title; + }, []); return (
A fork is an exact copy of a project. Forking a project allows you to diff --git a/src/packages/server/conat/api/projects.ts b/src/packages/server/conat/api/projects.ts index 3e79bc7420..86070a4106 100644 --- a/src/packages/server/conat/api/projects.ts +++ b/src/packages/server/conat/api/projects.ts @@ -62,3 +62,17 @@ export async function setQuotas(opts: { // @ts-ignore await project?.setAllQuotas(); } + +export async function getDiskQuota({ + account_id, + project_id, +}: { + account_id: string; + project_id: string; +}): Promise<{ used: number; size: number }> { + if (!(await isCollaborator({ account_id, project_id }))) { + throw Error("user must be a collaborator on project to get quota"); + } + const client = filesystemClient(); + return await client.getQuota({ project_id }); +} diff --git a/src/packages/server/conat/file-server/index.ts b/src/packages/server/conat/file-server/index.ts index 995dc6792a..ce5ace5ee3 100644 --- a/src/packages/server/conat/file-server/index.ts +++ b/src/packages/server/conat/file-server/index.ts @@ -173,6 +173,8 @@ export function close() { server = null; } +let cachedClient: null | Fileserver = null; export function client(): Fileserver { - return createFileClient({ client: conat() }); + cachedClient ??= createFileClient({ client: conat() }); + return cachedClient!; } From d8c1d5f25f621c65cbcb54693ab5f1a6cc508011 Mon Sep 17 00:00:00 2001 From: William Stein Date: Mon, 11 Aug 2025 20:00:06 +0000 Subject: [PATCH 260/270] implement more rustic backup and testing --- src/packages/backend/sandbox/exec.ts | 13 ++++ .../file-server/btrfs/subvolume-rustic.ts | 65 +++++++++++------ .../btrfs/test/rustic-stress.test.ts | 45 ++++++++++++ .../file-server/btrfs/test/rustic.test.ts | 73 +++++++++++++++++++ .../file-server/btrfs/test/subvolume.test.ts | 20 ----- 5 files changed, 173 insertions(+), 43 deletions(-) create mode 100644 src/packages/file-server/btrfs/test/rustic-stress.test.ts create mode 100644 src/packages/file-server/btrfs/test/rustic.test.ts diff --git a/src/packages/backend/sandbox/exec.ts b/src/packages/backend/sandbox/exec.ts index cfbb06421c..a966474db9 100644 --- a/src/packages/backend/sandbox/exec.ts +++ b/src/packages/backend/sandbox/exec.ts @@ -233,3 +233,16 @@ async function getUserIds( }), ]).then(([uid, gid]) => ({ uid, gid })); } + +// take the output of exec and convert stdout, stderr to strings. If code is nonzero, +// instead throw an error with message stderr. +export function parseOutput({ stdout, stderr, code, truncated }: ExecOutput) { + if (code) { + throw new Error(Buffer.from(stderr).toString()); + } + return { + stdout: Buffer.from(stdout).toString(), + stderr: Buffer.from(stderr).toString(), + truncated, + }; +} diff --git a/src/packages/file-server/btrfs/subvolume-rustic.ts b/src/packages/file-server/btrfs/subvolume-rustic.ts index 0a0e0a79ca..6200a42f0c 100644 --- a/src/packages/file-server/btrfs/subvolume-rustic.ts +++ b/src/packages/file-server/btrfs/subvolume-rustic.ts @@ -20,6 +20,7 @@ Instead of using btrfs send/recv for backups, we use Rustic because: import { type Subvolume } from "./subvolume"; import getLogger from "@cocalc/backend/logger"; +import { parseOutput } from "@cocalc/backend/sandbox/exec"; const RUSTIC_SNAPSHOT = "temp-rustic-snapshot"; @@ -62,31 +63,49 @@ export class SubvolumeRustic { } }; + restore = async ({ + id, + path = "", + dest, + timeout = 30 * 60 * 1000, + }: { + id: string; + path?: string; + dest?: string; + timeout?: number; + }) => { + dest ??= path; + const { stdout } = parseOutput( + await this.subvolume.fs.rustic( + ["restore", `${id}${path != null ? ":" + path : ""}`, dest], + { timeout }, + ), + ); + return stdout; + }; + snapshots = async (): Promise => { - const { stdout, stderr, code } = await this.subvolume.fs.rustic([ - "snapshots", - "--json", - ]); - if (code) { - throw Error(stderr.toString()); - } else { - const x = JSON.parse(stdout.toString()); - return x[0][1].map(({ time, id }) => { - return { time: new Date(time), id }; - }); - } + const { stdout } = parseOutput( + await this.subvolume.fs.rustic(["snapshots", "--json"]), + ); + const x = JSON.parse(stdout); + return x[0][1].map(({ time, id }) => { + return { time: new Date(time), id }; + }); }; - ls = async (id: string) => { - const { stdout, stderr, code } = await this.subvolume.fs.rustic([ - "ls", - "--json", - id, - ]); - if (code) { - throw Error(stderr.toString()); - } else { - return JSON.parse(stdout.toString()); - } + ls = async ({ id }: { id: string }) => { + const { stdout } = parseOutput( + await this.subvolume.fs.rustic(["ls", "--json", id]), + ); + return JSON.parse(stdout); + }; + + // (this doesn't actually clean up disk space -- purge must be done separately) + forget = async ({ id }: { id: string }) => { + const { stdout } = parseOutput( + await this.subvolume.fs.rustic(["forget", id]), + ); + return stdout; }; } diff --git a/src/packages/file-server/btrfs/test/rustic-stress.test.ts b/src/packages/file-server/btrfs/test/rustic-stress.test.ts new file mode 100644 index 0000000000..2eb0514df7 --- /dev/null +++ b/src/packages/file-server/btrfs/test/rustic-stress.test.ts @@ -0,0 +1,45 @@ +import { before, after, fs } from "./setup"; +import { type Subvolume } from "../subvolume"; + +beforeAll(before); + +const count = 10; +describe(`make backups of ${count} different volumes at the same time`, () => { + const vols: Subvolume[] = []; + it(`creates ${count} volumes`, async () => { + for (let i = 0; i < count; i++) { + const vol = await fs.subvolumes.get(`rustic-multi-${i}`); + await vol.fs.writeFile(`a-${i}.txt`, `hello-${i}`); + vols.push(vol); + } + }); + + it(`create ${count} rustic backup in parallel`, async () => { + await Promise.all(vols.map((vol) => vol.rustic.backup())); + }); + + it("delete file from each volume, then restore them all in parallel and confirm restore worked", async () => { + const snapshots = await Promise.all( + vols.map((vol) => vol.rustic.snapshots()), + ); + const ids = snapshots.map((x) => x[0].id); + for (let i = 0; i < count; i++) { + await vols[i].fs.unlink(`a-${i}.txt`); + } + + const v: any[] = []; + for (let i = 0; i < count; i++) { + v.push(vols[i].rustic.restore({ id: ids[i] })); + } + await Promise.all(v); + + for (let i = 0; i < count; i++) { + const vol = vols[i]; + expect((await vol.fs.readFile(`a-${i}.txt`)).toString("utf8")).toEqual( + `hello-${i}`, + ); + } + }); +}); + +afterAll(after); diff --git a/src/packages/file-server/btrfs/test/rustic.test.ts b/src/packages/file-server/btrfs/test/rustic.test.ts new file mode 100644 index 0000000000..feb36ce5a5 --- /dev/null +++ b/src/packages/file-server/btrfs/test/rustic.test.ts @@ -0,0 +1,73 @@ +import { before, after, fs } from "./setup"; +import { type Subvolume } from "../subvolume"; + +beforeAll(before); + +describe("test rustic backups", () => { + let vol: Subvolume; + it("creates a volume", async () => { + vol = await fs.subvolumes.get("rustic-test"); + await vol.fs.writeFile("a.txt", "hello"); + }); + + it("create a rustic backup", async () => { + await vol.rustic.backup(); + }); + + it("confirm a.txt is in our backup", async () => { + const v = await vol.rustic.snapshots(); + expect(v.length == 1); + expect(Math.abs(Date.now() - v[0].time.valueOf())).toBeLessThan(10000); + const { id } = v[0]; + const w = await vol.rustic.ls({ id }); + expect(w).toEqual([".snapshots", "a.txt"]); + }); + + it("delete a.txt, then restore it from the backup", async () => { + await vol.fs.unlink("a.txt"); + const { id } = (await vol.rustic.snapshots())[0]; + await vol.rustic.restore({ id }); + expect((await vol.fs.readFile("a.txt")).toString("utf8")).toEqual("hello"); + }); + + it("create a directory, make second backup, delete directory, then restore it from backup, and also restore just one file", async () => { + await vol.fs.mkdir("my-dir"); + await vol.fs.writeFile("my-dir/file.txt", "hello"); + await vol.fs.writeFile("my-dir/file2.txt", "hello2"); + await vol.rustic.backup(); + const v = await vol.rustic.snapshots(); + expect(v.length == 2); + const { id } = v[1]; + const w = await vol.rustic.ls({ id }); + expect(w).toEqual([ + ".snapshots", + "a.txt", + "my-dir", + "my-dir/file.txt", + "my-dir/file2.txt", + ]); + await vol.fs.rm("my-dir", { recursive: true }); + + // rustic all, including the path we just deleted + await vol.rustic.restore({ id }); + expect((await vol.fs.readFile("my-dir/file.txt")).toString("utf8")).toEqual( + "hello", + ); + + // restore just one specific file overwriting current version + await vol.fs.unlink("my-dir/file2.txt"); + await vol.fs.writeFile("my-dir/file.txt", "changed"); + await vol.rustic.restore({ id, path: "my-dir/file2.txt" }); + expect( + (await vol.fs.readFile("my-dir/file2.txt")).toString("utf8"), + ).toEqual("hello2"); + + // forget the second snapshot + const z = await vol.rustic.forget({ id }); + const v2 = await vol.rustic.snapshots(); + expect(v2.length).toBe(1); + expect(v2[0].id).not.toEqual(id); + }); +}); + +afterAll(after); diff --git a/src/packages/file-server/btrfs/test/subvolume.test.ts b/src/packages/file-server/btrfs/test/subvolume.test.ts index 21a46c6f07..fcdb878f7d 100644 --- a/src/packages/file-server/btrfs/test/subvolume.test.ts +++ b/src/packages/file-server/btrfs/test/subvolume.test.ts @@ -227,26 +227,6 @@ describe("test snapshots", () => { }); }); -describe("test rustic backups", () => { - let vol: Subvolume; - it("creates a volume", async () => { - vol = await fs.subvolumes.get("rustic-test"); - await vol.fs.writeFile("a.txt", "hello"); - }); - - it("create a rustic backup", async () => { - await vol.rustic.backup(); - }); - - it("confirm a.txt is in our backup", async () => { - const v = await vol.rustic.snapshots(); - expect(v.length == 1); - expect(Math.abs(Date.now() - v[0].time.valueOf())).toBeLessThan(10000); - const w = await vol.rustic.ls(v[0].id); - expect(w).toEqual([".snapshots", "a.txt"]); - }); -}); - describe.skip("test bup backups", () => { let vol: Subvolume; it("creates a volume", async () => { From aca121e13af1aa302195af7734a0635ae5083cce Mon Sep 17 00:00:00 2001 From: William Stein Date: Mon, 11 Aug 2025 21:31:11 +0000 Subject: [PATCH 261/270] delete bup from file-server codebase -- we're going with rustic --- src/packages/file-server/btrfs/filesystem.ts | 70 ++----- .../file-server/btrfs/subvolume-bup.ts | 195 ------------------ .../file-server/btrfs/subvolume-rustic.ts | 2 + src/packages/file-server/btrfs/subvolume.ts | 10 +- src/packages/file-server/btrfs/subvolumes.ts | 3 +- .../file-server/btrfs/test/filesystem.test.ts | 3 +- .../file-server/btrfs/test/rustic.test.ts | 2 +- src/packages/file-server/btrfs/test/setup.ts | 2 +- .../file-server/btrfs/test/subvolume.test.ts | 69 ------- .../server/conat/file-server/index.ts | 3 +- 10 files changed, 26 insertions(+), 333 deletions(-) delete mode 100644 src/packages/file-server/btrfs/subvolume-bup.ts diff --git a/src/packages/file-server/btrfs/filesystem.ts b/src/packages/file-server/btrfs/filesystem.ts index e29de633cb..d675c863b7 100644 --- a/src/packages/file-server/btrfs/filesystem.ts +++ b/src/packages/file-server/btrfs/filesystem.ts @@ -17,50 +17,30 @@ import { join } from "path"; import { Subvolumes } from "./subvolumes"; import { mkdir } from "fs/promises"; import { exists } from "@cocalc/backend/misc/async-utils-node"; -import { executeCode } from "@cocalc/backend/execute-code"; import rustic from "@cocalc/backend/sandbox/rustic"; - -// default size of btrfs filesystem if creating an image file. -const DEFAULT_FILESYSTEM_SIZE = "10G"; - -// default for newly created subvolumes -export const DEFAULT_SUBVOLUME_SIZE = "1G"; - -const MOUNT_ERROR = "wrong fs type, bad option, bad superblock"; +import { RUSTIC } from "./subvolume-rustic"; export interface Options { // the underlying block device. // If this is a file (or filename) ending in .img, then it's a sparse file mounted as a loopback device. // If this starts with "/dev" then it is a raw block device. device: string; - // formatIfNeeded -- DANGEROUS! if true, format the device or image, - // if it doesn't mount with an error containing "wrong fs type, - // bad option, bad superblock". Never use this in production. Useful - // for testing and dev. - formatIfNeeded?: boolean; - // where the btrfs filesystem is mounted + // where to mount the btrfs filesystem mount: string; - - // default size of newly created subvolumes - defaultSize?: string | number; - defaultFilesystemSize?: string | number; + // size -- if true and 'device' is a path to a .img file that DOES NOT EXIST, create device + // as a sparse image file of the given size. If img already exists, it will not be touched + // in any way, and it is up to you to mkfs.btrfs it, etc. + size?: string | number; } export class Filesystem { public readonly opts: Options; - public readonly bup: string; public readonly rustic: string; public readonly subvolumes: Subvolumes; constructor(opts: Options) { - opts = { - defaultSize: DEFAULT_SUBVOLUME_SIZE, - defaultFilesystemSize: DEFAULT_FILESYSTEM_SIZE, - ...opts, - }; this.opts = opts; - this.bup = join(this.opts.mount, "bup"); - this.rustic = join(this.opts.mount, "rustic"); + this.rustic = join(this.opts.mount, RUSTIC); this.subvolumes = new Subvolumes(this); } @@ -71,7 +51,6 @@ export class Filesystem { await btrfs({ args: ["quota", "enable", "--simple", this.opts.mount], }); - await this.initBup(); await this.initRustic(); await this.sync(); }; @@ -96,10 +75,17 @@ export class Filesystem { return; } if (!(await exists(this.opts.device))) { + if (!this.opts.size) { + throw Error( + "you must specify the size of the btrfs sparse image file, or explicitly create and format it", + ); + } + // we create and format the sparse image await sudo({ command: "truncate", - args: ["-s", `${this.opts.defaultFilesystemSize}`, this.opts.device], + args: ["-s", `${this.opts.size}`, this.opts.device], }); + await sudo({ command: "mkfs.btrfs", args: [this.opts.device] }); } }; @@ -124,25 +110,10 @@ export class Filesystem { } catch {} const { stderr, exit_code } = await this._mountFilesystem(); if (exit_code) { - if (stderr.includes(MOUNT_ERROR)) { - if (this.opts.formatIfNeeded) { - await this.formatDevice(); - const { stderr, exit_code } = await this._mountFilesystem(); - if (exit_code) { - throw Error(stderr); - } else { - return; - } - } - } throw Error(stderr); } }; - private formatDevice = async () => { - await sudo({ command: "mkfs.btrfs", args: [this.opts.device] }); - }; - private _mountFilesystem = async () => { const args: string[] = isImageFile(this.opts.device) ? ["-o", "loop"] : []; args.push( @@ -180,17 +151,6 @@ export class Filesystem { return { stderr, exit_code }; }; - private initBup = async () => { - if (!(await exists(this.bup))) { - await mkdir(this.bup); - } - await executeCode({ - command: "bup", - args: ["init"], - env: { BUP_DIR: this.bup }, - }); - }; - private initRustic = async () => { if (await exists(this.rustic)) { return; diff --git a/src/packages/file-server/btrfs/subvolume-bup.ts b/src/packages/file-server/btrfs/subvolume-bup.ts deleted file mode 100644 index a8ec09a945..0000000000 --- a/src/packages/file-server/btrfs/subvolume-bup.ts +++ /dev/null @@ -1,195 +0,0 @@ -/* - -BUP Architecture: - -There is a single global dedup'd backup archive stored in the btrfs filesystem. -Obviously, admins should rsync this regularly to a separate location as a genuine -backup strategy. - -NOTE: we use bup instead of btrfs send/recv ! - -Not used. Instead we will rely on bup (and snapshots of the underlying disk) for backups, since: - - much easier to check they are valid - - decoupled from any btrfs issues - - not tied to any specific filesystem at all - - easier to offsite via incremental rsync - - much more space efficient with *global* dedup and compression - - bup is really just git, which is much more proven than even btrfs - -The drawback is speed, but that can be managed. -*/ - -import { type DirectoryListingEntry } from "@cocalc/util/types"; -import { type Subvolume } from "./subvolume"; -import { sudo, parseBupTime } from "./util"; -import { join, normalize } from "path"; -import getLogger from "@cocalc/backend/logger"; - -const BUP_SNAPSHOT = "temp-bup-snapshot"; - -const logger = getLogger("file-server:btrfs:subvolume-bup"); - -export class SubvolumeBup { - constructor(private subvolume: Subvolume) {} - - // create a new bup backup - save = async ({ - // timeout used for bup index and bup save commands - timeout = 30 * 60 * 1000, - }: { timeout?: number } = {}) => { - if (await this.subvolume.snapshots.exists(BUP_SNAPSHOT)) { - logger.debug(`createBupBackup: deleting existing ${BUP_SNAPSHOT}`); - await this.subvolume.snapshots.delete(BUP_SNAPSHOT); - } - try { - logger.debug( - `createBackup: creating ${BUP_SNAPSHOT} to get a consistent backup`, - ); - await this.subvolume.snapshots.create(BUP_SNAPSHOT); - const target = await this.subvolume.fs.safeAbsPath( - this.subvolume.snapshots.path(BUP_SNAPSHOT), - ); - - logger.debug(`createBupBackup: indexing ${BUP_SNAPSHOT}`); - await sudo({ - command: "bup", - args: [ - "-d", - this.subvolume.filesystem.bup, - "index", - "--exclude", - join(target, ".snapshots"), - "-x", - target, - ], - timeout, - }); - - logger.debug(`createBackup: saving ${BUP_SNAPSHOT}`); - await sudo({ - command: "bup", - args: [ - "-d", - this.subvolume.filesystem.bup, - "save", - "--strip", - "-n", - this.subvolume.name, - target, - ], - timeout, - }); - } finally { - logger.debug(`createBupBackup: deleting temporary ${BUP_SNAPSHOT}`); - await this.subvolume.snapshots.delete(BUP_SNAPSHOT); - } - }; - - restore = async (path: string) => { - // path -- branch/revision/path/to/dir - if (path.startsWith("/")) { - path = path.slice(1); - } - path = normalize(path); - // ... but to avoid potential data loss, we make a snapshot before deleting it. - await this.subvolume.snapshots.create(); - const i = path.indexOf("/"); // remove the commit name - // remove the target we're about to restore - await this.subvolume.fs.rm(path.slice(i + 1), { recursive: true }); - await sudo({ - command: "bup", - args: [ - "-d", - this.subvolume.filesystem.bup, - "restore", - "-C", - this.subvolume.path, - join(`/${this.subvolume.name}`, path), - "--quiet", - ], - }); - }; - - // [ ] TODO: remove this ls and instead rely only on the fs sandbox code. - ls = async (path: string = ""): Promise => { - if (!path) { - const { stdout } = await sudo({ - command: "bup", - args: ["-d", this.subvolume.filesystem.bup, "ls", this.subvolume.name], - }); - const v: DirectoryListingEntry[] = []; - let newest = 0; - for (const x of stdout.trim().split("\n")) { - const name = x.split(" ").slice(-1)[0]; - if (name == "latest") { - continue; - } - const mtime = parseBupTime(name).valueOf() / 1000; - newest = Math.max(mtime, newest); - v.push({ name, isDir: true, mtime, size: -1 }); - } - if (v.length > 0) { - v.push({ name: "latest", isDir: true, mtime: newest, size: -1 }); - } - return v; - } - - path = (await this.subvolume.fs.safeAbsPath(path)).slice( - this.subvolume.path.length, - ); - const { stdout } = await sudo({ - command: "bup", - args: [ - "-d", - this.subvolume.filesystem.bup, - "ls", - "--almost-all", - "--file-type", - "-l", - join(`/${this.subvolume.name}`, path), - ], - }); - const v: DirectoryListingEntry[] = []; - for (const x of stdout.split("\n")) { - // [-rw-------","6b851643360e435eb87ef9a6ab64a8b1/6b851643360e435eb87ef9a6ab64a8b1","5","2025-07-15","06:12","a.txt"] - const w = x.split(/\s+/); - if (w.length >= 6) { - let isDir, name; - if (w[5].endsWith("@") || w[5].endsWith("=") || w[5].endsWith("|")) { - w[5] = w[5].slice(0, -1); - } - if (w[5].endsWith("/")) { - isDir = true; - name = w[5].slice(0, -1); - } else { - name = w[5]; - isDir = false; - } - const size = parseInt(w[2]); - const mtime = new Date(w[3] + "T" + w[4]).valueOf() / 1000; - v.push({ name, size, mtime, isDir }); - } - } - return v; - }; - - prune = async ({ - dailies = "1w", - monthlies = "4m", - all = "3d", - }: { dailies?: string; monthlies?: string; all?: string } = {}) => { - await sudo({ - command: "bup", - args: [ - "-d", - this.subvolume.filesystem.bup, - "prune-older", - `--keep-dailies-for=${dailies}`, - `--keep-monthlies-for=${monthlies}`, - `--keep-all-for=${all}`, - "--unsafe", - this.subvolume.name, - ], - }); - }; -} diff --git a/src/packages/file-server/btrfs/subvolume-rustic.ts b/src/packages/file-server/btrfs/subvolume-rustic.ts index 6200a42f0c..545ca41e7b 100644 --- a/src/packages/file-server/btrfs/subvolume-rustic.ts +++ b/src/packages/file-server/btrfs/subvolume-rustic.ts @@ -22,6 +22,8 @@ import { type Subvolume } from "./subvolume"; import getLogger from "@cocalc/backend/logger"; import { parseOutput } from "@cocalc/backend/sandbox/exec"; +export const RUSTIC = "rustic"; + const RUSTIC_SNAPSHOT = "temp-rustic-snapshot"; const logger = getLogger("file-server:btrfs:subvolume-rustic"); diff --git a/src/packages/file-server/btrfs/subvolume.ts b/src/packages/file-server/btrfs/subvolume.ts index 7eedfc0806..75b4836539 100644 --- a/src/packages/file-server/btrfs/subvolume.ts +++ b/src/packages/file-server/btrfs/subvolume.ts @@ -2,11 +2,10 @@ A subvolume */ -import { type Filesystem, DEFAULT_SUBVOLUME_SIZE } from "./filesystem"; +import { type Filesystem } from "./filesystem"; import refCache from "@cocalc/util/refcache"; import { sudo } from "./util"; import { join } from "path"; -import { SubvolumeBup } from "./subvolume-bup"; import { SubvolumeRustic } from "./subvolume-rustic"; import { SubvolumeSnapshots } from "./subvolume-snapshots"; import { SubvolumeQuota } from "./subvolume-quota"; @@ -27,7 +26,6 @@ export class Subvolume { public readonly filesystem: Filesystem; public readonly path: string; public readonly fs: SandboxedFilesystem; - public readonly bup: SubvolumeBup; public readonly rustic: SubvolumeRustic; public readonly snapshots: SubvolumeSnapshots; public readonly quota: SubvolumeQuota; @@ -40,7 +38,6 @@ export class Subvolume { rusticRepo: filesystem.rustic, host: this.name, }); - this.bup = new SubvolumeBup(this); this.rustic = new SubvolumeRustic(this); this.snapshots = new SubvolumeSnapshots(this); this.quota = new SubvolumeQuota(this); @@ -54,9 +51,6 @@ export class Subvolume { args: ["subvolume", "create", this.path], }); await this.chown(this.path); - await this.quota.set( - this.filesystem.opts.defaultSize ?? DEFAULT_SUBVOLUME_SIZE, - ); } }; @@ -69,7 +63,7 @@ export class Subvolume { delete this.path; // @ts-ignore delete this.snapshotsDir; - for (const sub of ["fs", "bup", "snapshots", "quota"]) { + for (const sub of ["fs", "rustic", "snapshots", "quota"]) { this[sub].close?.(); delete this[sub]; } diff --git a/src/packages/file-server/btrfs/subvolumes.ts b/src/packages/file-server/btrfs/subvolumes.ts index e4f10191b9..a193b37a1a 100644 --- a/src/packages/file-server/btrfs/subvolumes.ts +++ b/src/packages/file-server/btrfs/subvolumes.ts @@ -7,8 +7,9 @@ import { join } from "path"; import { btrfs } from "./util"; import { chmod, rename, rm } from "node:fs/promises"; import { SandboxedFilesystem } from "@cocalc/backend/sandbox"; +import { RUSTIC } from "./subvolume-rustic"; -const RESERVED = new Set(["bup", SNAPSHOTS]); +const RESERVED = new Set([RUSTIC, SNAPSHOTS]); const logger = getLogger("file-server:btrfs:subvolumes"); diff --git a/src/packages/file-server/btrfs/test/filesystem.test.ts b/src/packages/file-server/btrfs/test/filesystem.test.ts index fc5dbc1c04..c4eea6992a 100644 --- a/src/packages/file-server/btrfs/test/filesystem.test.ts +++ b/src/packages/file-server/btrfs/test/filesystem.test.ts @@ -1,5 +1,6 @@ import { before, after, fs } from "./setup"; import { isValidUUID } from "@cocalc/util/misc"; +import { RUSTIC } from "@cocalc/file-server/btrfs/subvolume-rustic"; beforeAll(before); @@ -22,7 +23,7 @@ describe("some basic tests", () => { describe("operations with subvolumes", () => { it("can't use a reserved subvolume name", async () => { expect(async () => { - await fs.subvolumes.get("bup"); + await fs.subvolumes.get(RUSTIC); }).rejects.toThrow("is reserved"); }); diff --git a/src/packages/file-server/btrfs/test/rustic.test.ts b/src/packages/file-server/btrfs/test/rustic.test.ts index feb36ce5a5..674cd1e6ed 100644 --- a/src/packages/file-server/btrfs/test/rustic.test.ts +++ b/src/packages/file-server/btrfs/test/rustic.test.ts @@ -63,7 +63,7 @@ describe("test rustic backups", () => { ).toEqual("hello2"); // forget the second snapshot - const z = await vol.rustic.forget({ id }); + await vol.rustic.forget({ id }); const v2 = await vol.rustic.snapshots(); expect(v2.length).toBe(1); expect(v2[0].id).not.toEqual(id); diff --git a/src/packages/file-server/btrfs/test/setup.ts b/src/packages/file-server/btrfs/test/setup.ts index c9736c8b79..6ee6d8460b 100644 --- a/src/packages/file-server/btrfs/test/setup.ts +++ b/src/packages/file-server/btrfs/test/setup.ts @@ -50,7 +50,7 @@ export async function before() { await chmod(mount, 0o777); fs = await filesystem({ device: join(tempDir, "btrfs.img"), - formatIfNeeded: true, + size: "1G", mount: join(tempDir, "mnt"), }); } diff --git a/src/packages/file-server/btrfs/test/subvolume.test.ts b/src/packages/file-server/btrfs/test/subvolume.test.ts index fcdb878f7d..7f1a03050b 100644 --- a/src/packages/file-server/btrfs/test/subvolume.test.ts +++ b/src/packages/file-server/btrfs/test/subvolume.test.ts @@ -1,6 +1,4 @@ import { before, after, fs, sudo } from "./setup"; -import { mkdir } from "fs/promises"; -import { join } from "path"; import { wait } from "@cocalc/backend/conat/test/util"; import { randomBytes } from "crypto"; import { type Subvolume } from "../subvolume"; @@ -227,71 +225,4 @@ describe("test snapshots", () => { }); }); -describe.skip("test bup backups", () => { - let vol: Subvolume; - it("creates a volume", async () => { - vol = await fs.subvolumes.get("bup-test"); - await vol.fs.writeFile("a.txt", "hello"); - }); - - it("create a bup backup", async () => { - await vol.bup.save(); - }); - - it("list bup backups of this vol -- there are 2, one for the date and 'latest'", async () => { - const v = await vol.bup.ls(); - expect(v.length).toBe(2); - const t = (v[0].mtime ?? 0) * 1000; - expect(Math.abs(t.valueOf() - Date.now())).toBeLessThan(10_000); - }); - - it("confirm a.txt is in our backup", async () => { - const x = await vol.bup.ls("latest"); - expect(x).toEqual([ - { name: "a.txt", size: 5, mtime: x[0].mtime, isDir: false }, - ]); - }); - - it("restore a.txt from our backup", async () => { - await vol.fs.writeFile("a.txt", "hello2"); - await vol.bup.restore("latest/a.txt"); - expect(await vol.fs.readFile("a.txt", "utf8")).toEqual("hello"); - }); - - it("prune bup backups does nothing since we have so few", async () => { - await vol.bup.prune(); - expect((await vol.bup.ls()).length).toBe(2); - }); - - it("add a directory and back up", async () => { - await mkdir(join(vol.path, "mydir")); - await vol.fs.writeFile(join("mydir", "file.txt"), "hello3"); - expect((await vol.fs.readdir("mydir"))[0]).toBe("file.txt"); - await vol.bup.save(); - const x = await vol.bup.ls("latest"); - expect(x).toEqual([ - { name: "a.txt", size: 5, mtime: x[0].mtime, isDir: false }, - { name: "mydir", size: 0, mtime: x[1].mtime, isDir: true }, - ]); - expect(Math.abs((x[0].mtime ?? 0) * 1000 - Date.now())).toBeLessThan( - 5 * 60_000, - ); - }); - - it("change file in the directory, then restore from backup whole dir", async () => { - await vol.fs.writeFile(join("mydir", "file.txt"), "changed"); - await vol.bup.restore("latest/mydir"); - expect(await vol.fs.readFile(join("mydir", "file.txt"), "utf8")).toEqual( - "hello3", - ); - }); - - it("most recent snapshot has a backup before the restore", async () => { - const s = await vol.snapshots.readdir(); - const recent = s.slice(-1)[0]; - const p = vol.snapshots.path(recent, "mydir", "file.txt"); - expect(await vol.fs.readFile(p, "utf8")).toEqual("changed"); - }); -}); - afterAll(after); diff --git a/src/packages/server/conat/file-server/index.ts b/src/packages/server/conat/file-server/index.ts index ce5ace5ee3..61f49bd004 100644 --- a/src/packages/server/conat/file-server/index.ts +++ b/src/packages/server/conat/file-server/index.ts @@ -152,9 +152,8 @@ export async function init() { fs = await filesystem({ device: btrfsDevice, - formatIfNeeded: true, + size: "25G", mount: mountPoint, - defaultFilesystemSize: "25G", }); server = await createFileServer({ From 7b026e2a731e28deb93d6da63c753b2a23159ae0 Mon Sep 17 00:00:00 2001 From: William Stein Date: Mon, 11 Aug 2025 23:19:06 +0000 Subject: [PATCH 262/270] rustic: work in progress exposing backups of file-server over conat --- src/packages/conat/files/file-server.ts | 32 ++++++++ src/packages/conat/hub/api/projects.ts | 2 + .../file-server/btrfs/subvolume-rustic.ts | 16 ++-- .../file-server/btrfs/test/rustic.test.ts | 4 +- .../server/conat/file-server/index.ts | 8 +- .../server/conat/file-server/rustic.ts | 76 +++++++++++++++++++ 6 files changed, 128 insertions(+), 10 deletions(-) create mode 100644 src/packages/server/conat/file-server/rustic.ts diff --git a/src/packages/conat/files/file-server.ts b/src/packages/conat/files/file-server.ts index f6bd47ff89..dbbd59787b 100644 --- a/src/packages/conat/files/file-server.ts +++ b/src/packages/conat/files/file-server.ts @@ -58,6 +58,38 @@ export interface Fileserver { dest: { project_id: string; path: string }; options?: CopyOptions; }) => Promise; + + // create new complete backup of the project; this first snapshots the + // project, makes a backup of the snapshot, then deletes the snapshot, so the + // backup is guranteed to be consistent. + backup: (opts: { project_id: string }) => Promise<{ time: Date; id: string }>; + + // restore the given path in the backup to the given dest. The default + // path is '' (the whole project) and the default destination is the + // same as the path. + restore: (opts: { + project_id: string; + id: string; + path?: string; + dest?: string; + }) => Promise; + + // delete the given backup + deleteBackup: (opts: { project_id: string; id: string }) => Promise; + + // Return list of id's and timestamps of all backups of this project. + getBackups: (opts: { project_id: string }) => Promise< + { + id: string; + time: Date; + }[] + >; + + // Return list of all files in the given backup. + getBackupFiles: (opts: { + project_id: string; + id: string; + }) => Promise; } export interface Options extends Fileserver { diff --git a/src/packages/conat/hub/api/projects.ts b/src/packages/conat/hub/api/projects.ts index fa79963587..c089a3ea25 100644 --- a/src/packages/conat/hub/api/projects.ts +++ b/src/packages/conat/hub/api/projects.ts @@ -110,4 +110,6 @@ export interface Projects { account_id?: string; project_id: string; }) => Promise<{ used: number; size: number }>; + + } diff --git a/src/packages/file-server/btrfs/subvolume-rustic.ts b/src/packages/file-server/btrfs/subvolume-rustic.ts index 545ca41e7b..040c91cc90 100644 --- a/src/packages/file-server/btrfs/subvolume-rustic.ts +++ b/src/packages/file-server/btrfs/subvolume-rustic.ts @@ -37,7 +37,9 @@ export class SubvolumeRustic { constructor(private subvolume: Subvolume) {} // create a new rustic backup - backup = async ({ timeout = 30 * 60 * 1000 }: { timeout?: number } = {}) => { + backup = async ({ + timeout = 30 * 60 * 1000, + }: { timeout?: number } = {}): Promise => { if (await this.subvolume.snapshots.exists(RUSTIC_SNAPSHOT)) { logger.debug(`backup: deleting existing ${RUSTIC_SNAPSHOT}`); await this.subvolume.snapshots.delete(RUSTIC_SNAPSHOT); @@ -49,16 +51,14 @@ export class SubvolumeRustic { ); await this.subvolume.snapshots.create(RUSTIC_SNAPSHOT); logger.debug(`backup: backing up ${RUSTIC_SNAPSHOT} using rustic`); - const { stderr, code } = await this.subvolume.fs.rustic( - ["backup", "-x", "."], - { + const { stdout } = parseOutput( + await this.subvolume.fs.rustic(["backup", "-x", "--json", "."], { timeout, cwd: target, - }, + }), ); - if (code) { - throw Error(stderr.toString()); - } + const { time, id } = JSON.parse(stdout); + return { time: new Date(time), id }; } finally { logger.debug(`backup: deleting temporary ${RUSTIC_SNAPSHOT}`); await this.subvolume.snapshots.delete(RUSTIC_SNAPSHOT); diff --git a/src/packages/file-server/btrfs/test/rustic.test.ts b/src/packages/file-server/btrfs/test/rustic.test.ts index 674cd1e6ed..5c75e46dde 100644 --- a/src/packages/file-server/btrfs/test/rustic.test.ts +++ b/src/packages/file-server/btrfs/test/rustic.test.ts @@ -10,13 +10,15 @@ describe("test rustic backups", () => { await vol.fs.writeFile("a.txt", "hello"); }); + let x; it("create a rustic backup", async () => { - await vol.rustic.backup(); + x = await vol.rustic.backup(); }); it("confirm a.txt is in our backup", async () => { const v = await vol.rustic.snapshots(); expect(v.length == 1); + expect(v[0]).toEqual(x); expect(Math.abs(Date.now() - v[0].time.valueOf())).toBeLessThan(10000); const { id } = v[0]; const w = await vol.rustic.ls({ id }); diff --git a/src/packages/server/conat/file-server/index.ts b/src/packages/server/conat/file-server/index.ts index 61f49bd004..d2f78cb76c 100644 --- a/src/packages/server/conat/file-server/index.ts +++ b/src/packages/server/conat/file-server/index.ts @@ -26,6 +26,7 @@ import { join } from "node:path"; import { mkdir } from "fs/promises"; import { filesystem, type Filesystem } from "@cocalc/file-server/btrfs"; import { exists } from "@cocalc/backend/misc/async-utils-node"; +import * as rustic from "./rustic"; const logger = getLogger("server:conat:file-server"); @@ -33,7 +34,7 @@ function name(project_id: string) { return `project-${project_id}`; } -async function getVolume(project_id) { +export async function getVolume(project_id) { if (fs == null) { throw Error("file server not initialized"); } @@ -164,6 +165,11 @@ export async function init() { getQuota: reuseInFlight(getQuota), setQuota, cp, + backup: reuseInFlight(rustic.backup), + restore: rustic.restore, + deleteBackup: rustic.deleteBackup, + getBackups: reuseInFlight(rustic.getBackups), + getBackupFiles: reuseInFlight(rustic.getBackupFiles), }); } diff --git a/src/packages/server/conat/file-server/rustic.ts b/src/packages/server/conat/file-server/rustic.ts new file mode 100644 index 0000000000..b89bcff4eb --- /dev/null +++ b/src/packages/server/conat/file-server/rustic.ts @@ -0,0 +1,76 @@ +import { getVolume } from "./index"; +import getLogger from "@cocalc/backend/logger"; + +const logger = getLogger("server:conat:file-server:rustic"); + +// create new complete backup of the project; this first snapshots the +// project, makes a backup of the snapshot, then deletes the snapshot, so the +// backup is guranteed to be consistent. +export async function backup({ + project_id, +}: { + project_id: string; +}): Promise<{ time: Date; id: string }> { + logger.debug("backup", { project_id }); + const vol = await getVolume(project_id); + return await vol.rustic.backup(); +} + +// restore the given path in the backup to the given dest. The default +// path is '' (the whole project) and the default destination is the +// same as the path. +export async function restore({ + project_id, + id, + path, + dest, +}: { + project_id: string; + id: string; + path?: string; + dest?: string; +}): Promise { + logger.debug("restore", { project_id, id, path, dest }); + const vol = await getVolume(project_id); + await vol.rustic.restore({ id, path, dest }); +} + +export async function deleteBackup({ + project_id, + id, +}: { + project_id: string; + id: string; +}): Promise { + logger.debug("deleteBackup", { project_id, id }); + const vol = await getVolume(project_id); + await vol.rustic.forget({ id }); +} + +// Return list of id's and timestamps of all backups of this project. +export async function getBackups({ + project_id, +}: { + project_id: string; +}): Promise< + { + id: string; + time: Date; + }[] +> { + logger.debug("getBackups", { project_id }); + const vol = await getVolume(project_id); + return await vol.rustic.snapshots(); +} +// Return list of all files in the given backup. +export async function getBackupFiles({ + project_id, + id, +}: { + project_id: string; + id: string; +}): Promise { + logger.debug("getBackupFiles", { project_id, id }); + const vol = await getVolume(project_id); + return await vol.rustic.ls({ id }); +} From f9ef536e75701e5e10a1279b640be6277cfdcac0 Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 12 Aug 2025 00:48:27 +0000 Subject: [PATCH 263/270] rustic: support toml config for much more sophisticated use of rustic backups; and move backup out of btrfs fs. --- .../conat/test/files/file-server.test.ts | 36 ++++++++++ src/packages/backend/sandbox/rustic.test.ts | 24 ++++++- src/packages/backend/sandbox/rustic.ts | 13 +++- src/packages/file-server/btrfs/filesystem.ts | 66 +++++++++---------- src/packages/file-server/btrfs/test/setup.ts | 2 +- .../server/conat/file-server/index.ts | 3 +- 6 files changed, 104 insertions(+), 40 deletions(-) diff --git a/src/packages/backend/conat/test/files/file-server.test.ts b/src/packages/backend/conat/test/files/file-server.test.ts index da5c93ad1c..e31e835479 100644 --- a/src/packages/backend/conat/test/files/file-server.test.ts +++ b/src/packages/backend/conat/test/files/file-server.test.ts @@ -68,6 +68,42 @@ describe("create basic mocked file server and test it out", () => { dest: { project_id: string; path: string }; options?; }): Promise => {}, + + backup: async (_opts: { + project_id: string; + }): Promise<{ time: Date; id: string }> => { + return { time: new Date(), id: "0" }; + }, + + restore: async (_opts: { + project_id: string; + id: string; + path?: string; + dest?: string; + }): Promise => {}, + + deleteBackup: async (_opts: { + project_id: string; + id: string; + }): Promise => {}, + + getBackups: async (_opts: { + project_id: string; + }): Promise< + { + id: string; + time: Date; + }[] + > => { + return []; + }, + + getBackupFiles: async (_opts: { + project_id: string; + id: string; + }): Promise => { + return []; + }, }); }); diff --git a/src/packages/backend/sandbox/rustic.test.ts b/src/packages/backend/sandbox/rustic.test.ts index db1c9ae1f9..0e9561c27c 100644 --- a/src/packages/backend/sandbox/rustic.test.ts +++ b/src/packages/backend/sandbox/rustic.test.ts @@ -8,12 +8,13 @@ import rustic from "./rustic"; import { mkdtemp, mkdir, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join, resolve } from "node:path"; +import { parseOutput } from "./exec"; -let tempDir, options; +let tempDir, options, home; beforeAll(async () => { tempDir = await mkdtemp(join(tmpdir(), "cocalc")); const repo = join(tempDir, "repo"); - const home = join(tempDir, "home"); + home = join(tempDir, "home"); await mkdir(home); const safeAbsPath = (path: string) => join(home, resolve("/", path)); options = { @@ -48,6 +49,25 @@ describe("rustic does something", () => { expect(truncated).toBe(false); }); + it("use a .toml file instead of explicitly passing in a repo", async () => { + await mkdir(join(home, "x")); + await writeFile( + join(home, "x/a.toml"), + ` +[repository] +repository = "${options.repo}" +password = "" +`, + ); + const options2 = { ...options, repo: join(home, "x/a.toml") }; + const { stdout } = parseOutput( + await rustic(["snapshots", "--json"], options2), + ); + const s = JSON.parse(stdout); + expect(s.length).toEqual(1); + expect(s[0][0].hostname).toEqual("my-host"); + }); + // it("it appears in the snapshots list", async () => { // const { stdout, truncated } = await rustic( // ["snapshots", "--json"], diff --git a/src/packages/backend/sandbox/rustic.ts b/src/packages/backend/sandbox/rustic.ts index 28bb7e4a96..eb4d2d7d5e 100644 --- a/src/packages/backend/sandbox/rustic.ts +++ b/src/packages/backend/sandbox/rustic.ts @@ -84,11 +84,16 @@ export default async function rustic( host = "host", } = options; + let common; + if (repo.endsWith(".toml")) { + common = ["-P", repo.slice(0, -".toml".length)]; + } else { + common = ["--password", "", "-r", repo]; + } + await ensureInitialized(repo); const cwd = await safeAbsPath?.(options.cwd ?? ""); - const common = ["--password", "", "-r", repo]; - const run = async (sanitizedArgs: string[]) => { return await exec({ cmd: rusticPath, @@ -306,6 +311,10 @@ const whitelist = { } as const; async function ensureInitialized(repo: string) { + if (repo.endsWith(".toml")) { + // nothing to do + return; + } const config = join(repo, "config"); if (!(await exists(config))) { await exec({ diff --git a/src/packages/file-server/btrfs/filesystem.ts b/src/packages/file-server/btrfs/filesystem.ts index d675c863b7..e3c46a5036 100644 --- a/src/packages/file-server/btrfs/filesystem.ts +++ b/src/packages/file-server/btrfs/filesystem.ts @@ -13,24 +13,31 @@ a = require('@cocalc/file-server/btrfs'); fs = await a.filesystem({device:'/tmp/ import refCache from "@cocalc/util/refcache"; import { mkdirp, btrfs, sudo } from "./util"; -import { join } from "path"; import { Subvolumes } from "./subvolumes"; import { mkdir } from "fs/promises"; import { exists } from "@cocalc/backend/misc/async-utils-node"; import rustic from "@cocalc/backend/sandbox/rustic"; -import { RUSTIC } from "./subvolume-rustic"; export interface Options { - // the underlying block device. - // If this is a file (or filename) ending in .img, then it's a sparse file mounted as a loopback device. - // If this starts with "/dev" then it is a raw block device. - device: string; - // where to mount the btrfs filesystem + // mount = root mountpoint of the btrfs filesystem. If you specify the image + // path below, then a btrfs filesystem will get automatically created (via sudo + // and a loopback device). mount: string; - // size -- if true and 'device' is a path to a .img file that DOES NOT EXIST, create device - // as a sparse image file of the given size. If img already exists, it will not be touched - // in any way, and it is up to you to mkfs.btrfs it, etc. + + // image = optioanlly use a image file at this location for the btrfs filesystem. + // This is used for development and in Docker. It will be created as a sparse image file + // with given size, and mounted at opts.mount if it does not exist. If you create + // it be sure to use mkfs.btrfs to format it. + image?: string; size?: string | number; + + // rustic = the rustic backups path. + // If this path ends in .toml, it is the configuration file for rustic, e.g., you can + // configure rustic however you want by pointing this at a toml cofig file. + // Otherwise, if this path does not exist, it will be created a new rustic repo + // initialized here. + // If not given, then backups will throw an error. + rustic?: string; } export class Filesystem { @@ -40,7 +47,6 @@ export class Filesystem { constructor(opts: Options) { this.opts = opts; - this.rustic = join(this.opts.mount, RUSTIC); this.subvolumes = new Subvolumes(this); } @@ -51,7 +57,9 @@ export class Filesystem { await btrfs({ args: ["quota", "enable", "--simple", this.opts.mount], }); - await this.initRustic(); + if (this.opts.rustic) { + await this.initRustic(); + } await this.sync(); }; @@ -70,22 +78,16 @@ export class Filesystem { close = () => {}; private initDevice = async () => { - if (!isImageFile(this.opts.device)) { - // raw block device -- nothing to do + if (!this.opts.image) { return; } - if (!(await exists(this.opts.device))) { - if (!this.opts.size) { - throw Error( - "you must specify the size of the btrfs sparse image file, or explicitly create and format it", - ); - } + if (!(await exists(this.opts.image))) { // we create and format the sparse image await sudo({ command: "truncate", - args: ["-s", `${this.opts.size}`, this.opts.device], + args: ["-s", `${this.opts.size ?? "10G"}`, this.opts.image], }); - await sudo({ command: "mkfs.btrfs", args: [this.opts.device] }); + await sudo({ command: "mkfs.btrfs", args: [this.opts.image] }); } }; @@ -115,7 +117,10 @@ export class Filesystem { }; private _mountFilesystem = async () => { - const args: string[] = isImageFile(this.opts.device) ? ["-o", "loop"] : []; + if (!this.opts.image) { + throw Error(`there must be a btrfs filesystem at ${this.opts.mount}`); + } + const args: string[] = ["-o", "loop"]; args.push( "-o", "compress=zstd", @@ -125,7 +130,7 @@ export class Filesystem { "space_cache=v2", "-o", "autodefrag", - this.opts.device, + this.opts.image, "-t", "btrfs", this.opts.mount, @@ -152,22 +157,17 @@ export class Filesystem { }; private initRustic = async () => { - if (await exists(this.rustic)) { + if (!this.rustic || (await exists(this.rustic))) { return; } + if (this.rustic.endsWith(".toml")) { + throw Error(`file not found: ${this.rustic}`); + } await mkdir(this.rustic); await rustic(["init"], { repo: this.rustic }); }; } -function isImageFile(name: string) { - if (name.startsWith("/dev")) { - return false; - } - // TODO: could probably check os for a device with given name? - return name.endsWith(".img"); -} - const cache = refCache({ name: "btrfs-filesystems", createObject: async (options: Options) => { diff --git a/src/packages/file-server/btrfs/test/setup.ts b/src/packages/file-server/btrfs/test/setup.ts index 6ee6d8460b..b6acd1bf6a 100644 --- a/src/packages/file-server/btrfs/test/setup.ts +++ b/src/packages/file-server/btrfs/test/setup.ts @@ -49,7 +49,7 @@ export async function before() { await mkdir(mount); await chmod(mount, 0o777); fs = await filesystem({ - device: join(tempDir, "btrfs.img"), + image: join(tempDir, "btrfs.img"), size: "1G", mount: join(tempDir, "mnt"), }); diff --git a/src/packages/server/conat/file-server/index.ts b/src/packages/server/conat/file-server/index.ts index d2f78cb76c..3ef3628efb 100644 --- a/src/packages/server/conat/file-server/index.ts +++ b/src/packages/server/conat/file-server/index.ts @@ -145,14 +145,13 @@ export async function init() { if (!(await exists(image))) { await mkdir(image, { recursive: true }); } - const btrfsDevice = join(image, "btrfs.img"); const mountPoint = join(data, "btrfs", "mnt"); if (!(await exists(mountPoint))) { await mkdir(mountPoint, { recursive: true }); } fs = await filesystem({ - device: btrfsDevice, + image: join(image, "btrfs.img"), size: "25G", mount: mountPoint, }); From 80363877298fa258942e5b20037ba52648b09078 Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 12 Aug 2025 01:20:09 +0000 Subject: [PATCH 264/270] size --> disk --- src/packages/file-server/btrfs/filesystem.ts | 2 +- src/packages/server/conat/project/load-balancer.ts | 4 ++-- src/packages/server/conat/project/run.ts | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/packages/file-server/btrfs/filesystem.ts b/src/packages/file-server/btrfs/filesystem.ts index e3c46a5036..c1e4947bd9 100644 --- a/src/packages/file-server/btrfs/filesystem.ts +++ b/src/packages/file-server/btrfs/filesystem.ts @@ -7,7 +7,7 @@ Start node, then: DEBUG="cocalc:*file-server*" DEBUG_CONSOLE=yes node -a = require('@cocalc/file-server/btrfs'); fs = await a.filesystem({device:'/tmp/btrfs.img', formatIfNeeded:true, mount:'/mnt/btrfs', uid:293597964}) +a = require('@cocalc/file-server/btrfs'); fs = await a.filesystem({image:'/tmp/btrfs.img', mount:'/mnt/btrfs', size:'2G'}) */ diff --git a/src/packages/server/conat/project/load-balancer.ts b/src/packages/server/conat/project/load-balancer.ts index ba7ee15dc8..6c2dbed152 100644 --- a/src/packages/server/conat/project/load-balancer.ts +++ b/src/packages/server/conat/project/load-balancer.ts @@ -38,7 +38,7 @@ async function getConfig({ project_id }) { throw Error(`no project with id ${project_id}`); } if (rows[0].settings?.admin) { - return { admin: true, size: "10G" }; + return { admin: true, disk: "25G" }; } else { // some defaults, mainly for testing return { @@ -46,7 +46,7 @@ async function getConfig({ project_id }) { memory: "8Gi", pids: 10000, swap: "5000Gi", - size: "1000M", + disk: "1G", }; } } diff --git a/src/packages/server/conat/project/run.ts b/src/packages/server/conat/project/run.ts index e844cfe573..398ad67761 100644 --- a/src/packages/server/conat/project/run.ts +++ b/src/packages/server/conat/project/run.ts @@ -152,11 +152,11 @@ async function start({ await setupDataPath(home, uid); await writeSecretToken(home, await getProjectSecretToken(project_id), uid); - if (config?.size) { + if (config?.disk) { // TODO: maybe this should be done in parallel with other things // to make startup time slightly faster (?) -- could also be incorporated // into mount. - await setQuota(project_id, config.size); + await setQuota(project_id, config.disk); } let script: string, From ee87b547a65a7da4fa40ea797154a4d6c2d879b9 Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 12 Aug 2025 01:31:19 +0000 Subject: [PATCH 265/270] ts and pnpm --- src/packages/pnpm-lock.yaml | 132 ++++++--------------- src/packages/server/conat/project/types.ts | 2 +- 2 files changed, 39 insertions(+), 95 deletions(-) diff --git a/src/packages/pnpm-lock.yaml b/src/packages/pnpm-lock.yaml index a5a2d3de87..2d91207588 100644 --- a/src/packages/pnpm-lock.yaml +++ b/src/packages/pnpm-lock.yaml @@ -1212,22 +1212,22 @@ importers: version: 1.4.1 '@langchain/anthropic': specifier: ^0.3.26 - version: 0.3.26(@langchain/core@0.3.68(@opentelemetry/api@1.9.0)(openai@5.12.1(ws@8.18.3)(zod@3.25.76))) + version: 0.3.26(@langchain/core@0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3(utf-8-validate@6.0.5))(zod@3.25.76))) '@langchain/core': specifier: ^0.3.68 - version: 0.3.68(@opentelemetry/api@1.9.0)(openai@5.12.1(ws@8.18.3)(zod@3.25.76)) + version: 0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3(utf-8-validate@6.0.5))(zod@3.25.76)) '@langchain/google-genai': specifier: ^0.2.16 - version: 0.2.16(@langchain/core@0.3.68(@opentelemetry/api@1.9.0)(openai@5.12.1(ws@8.18.3)(zod@3.25.76))) + version: 0.2.16(@langchain/core@0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3(utf-8-validate@6.0.5))(zod@3.25.76))) '@langchain/mistralai': specifier: ^0.2.1 - version: 0.2.1(@langchain/core@0.3.68(@opentelemetry/api@1.9.0)(openai@5.12.1(ws@8.18.3)(zod@3.25.76)))(zod@3.25.76) + version: 0.2.1(@langchain/core@0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3(utf-8-validate@6.0.5))(zod@3.25.76)))(zod@3.25.76) '@langchain/ollama': specifier: ^0.2.3 - version: 0.2.3(@langchain/core@0.3.68(@opentelemetry/api@1.9.0)(openai@5.12.1(ws@8.18.3)(zod@3.25.76))) + version: 0.2.3(@langchain/core@0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3(utf-8-validate@6.0.5))(zod@3.25.76))) '@langchain/openai': specifier: ^0.6.6 - version: 0.6.6(@langchain/core@0.3.68(@opentelemetry/api@1.9.0)(openai@5.12.1(ws@8.18.3)(zod@3.25.76)))(ws@8.18.3) + version: 0.6.7(@langchain/core@0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3(utf-8-validate@6.0.5))(zod@3.25.76)))(ws@8.18.3(utf-8-validate@6.0.5)) '@node-saml/passport-saml': specifier: ^5.1.0 version: 5.1.0 @@ -1338,7 +1338,7 @@ importers: version: 6.10.1 openai: specifier: ^5.12.1 - version: 5.12.1(ws@8.18.3)(zod@3.25.76) + version: 5.12.2(ws@8.18.3(utf-8-validate@6.0.5))(zod@3.25.76) parse-domain: specifier: ^5.0.0 version: 5.0.0(encoding@0.1.13) @@ -3060,8 +3060,8 @@ packages: peerDependencies: '@langchain/core': ^0.3.68 - '@langchain/core@0.3.68': - resolution: {integrity: sha512-dWPT1h9ObG1TK9uivFTk/pgBULZ6/tBmq8czGUjZjR+1xh9jB4tm/D5FY6o5FklXcEpnAI9peNq2x17Kl9wbMg==} + '@langchain/core@0.3.69': + resolution: {integrity: sha512-N6ZmgcnoMnGw+hQuS8w8FrNUm/5FuvtB868Jr1i1+4pASngLLVUyjeAQbKBBFMFH+WY5ga9LSvaQegUe3TLF8g==} engines: {node: '>=18'} '@langchain/google-genai@0.2.16': @@ -3082,8 +3082,8 @@ packages: peerDependencies: '@langchain/core': ^0.3.68 - '@langchain/openai@0.6.6': - resolution: {integrity: sha512-0fxSg290WTCTEM0PECDGfst2QYUiULKhzyydaOPLMQ5pvWHjJkzBudx+CyHkeQ8DvGXysJteSmZzAMjRCj4duQ==} + '@langchain/openai@0.6.7': + resolution: {integrity: sha512-mNT9AdfEvDjlWU76hEl1HgTFkgk7yFKdIRgQz3KXKZhEERXhAwYJNgPFq8+HIpgxYSnc12akZ1uo8WPS98ErPQ==} engines: {node: '>=18'} peerDependencies: '@langchain/core': ^0.3.68 @@ -4349,8 +4349,6 @@ packages: '@types/node-cleanup@2.1.5': resolution: {integrity: sha512-+82RAk5uYiqiMoEv2fPeh03AL4pB5d3TL+Pf+hz31Mme6ECFI1kRlgmxYjdSlHzDbJ9yLorTnKi4Op5FA54kQQ==} - '@types/node-fetch@2.6.13': - resolution: {integrity: sha512-QGpRVpzSaUs30JBSGPjOg4Uveu384erbHBoT1zeONvyCfwQxIkUshLAOqN/k9EjGviPRmWTTe6aH2qySWKTVSw==} '@types/node-forge@1.3.13': resolution: {integrity: sha512-zePQJSW5QkwSHKRApqWCVKeKoSOt4xvEnLENZPjyvm9Ezdf/EyDeJM7jqLzOwjVICQQzvLZ63T55MKdJB5H6ww==} @@ -4360,9 +4358,6 @@ packages: '@types/node@18.19.118': resolution: {integrity: sha512-hIPK0hSrrcaoAu/gJMzN3QClXE4QdCdFvaenJ0JsjIbExP1JFFVH+RHcBt25c9n8bx5dkIfqKE+uw6BmBns7ug==} - '@types/node@24.2.1': - resolution: {integrity: sha512-DRh5K+ka5eJic8CjH7td8QpYEV6Zo10gfRkjHCO3weqZHWDtAaSTFtl4+VMqOJ4N5jcuhZ9/l+yy8rVgw7BQeQ==} - '@types/nodemailer@6.4.17': resolution: {integrity: sha512-I9CCaIp6DTldEg7vyUTZi8+9Vo0hi1/T8gv3C89yk1rSAAzoKQ8H8ki/jBYJSFoH/BisgLP8tkZMlQ91CIquww==} @@ -5132,11 +5127,6 @@ packages: engines: {node: ^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7} hasBin: true - browserslist@4.25.2: - resolution: {integrity: sha512-0si2SJK3ooGzIawRu61ZdPCO1IncZwS8IzuX73sPZsXW6EQ/w/DAfPyKI8l1ETTCr2MnvqWitmlCUxgdul45jA==} - engines: {node: ^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7} - hasBin: true - bs-logger@0.2.6: resolution: {integrity: sha512-pd8DCoxmbgc7hyPKOvxtqNcjYoOsABPQdcCUjGp3d42VR2CX1ORhk2A87oqqu5R1kk+76nsxZupkmyd+MVtCog==} engines: {node: '>= 6'} @@ -5210,9 +5200,6 @@ packages: caniuse-lite@1.0.30001727: resolution: {integrity: sha512-pB68nIHmbN6L/4C6MH1DokyR3bYqFwjaSs/sWDHGj4CTcFtQUQMuJftVwWkXq7mNWOybD3KhUv3oWHoGxgP14Q==} - caniuse-lite@1.0.30001733: - resolution: {integrity: sha512-e4QKw/O2Kavj2VQTKZWrwzkt3IxOmIlU6ajRb6LP64LHpBo1J67k2Hi4Vu/TgJWsNtynurfS0uK3MaUTCPfu5Q==} - canvas-fit@1.5.0: resolution: {integrity: sha512-onIcjRpz69/Hx5bB5HGbYKUF2uC6QT6Gp+pfpGm3A7mPfcluSLV5v4Zu+oflDUwLdUw0rLIBhUbi0v8hM4FJQQ==} @@ -6321,9 +6308,6 @@ packages: electron-to-chromium@1.5.182: resolution: {integrity: sha512-Lv65Btwv9W4J9pyODI6EWpdnhfvrve/us5h1WspW8B2Fb0366REPtY3hX7ounk1CkV/TBjWCEvCBBbYbmV0qCA==} - electron-to-chromium@1.5.199: - resolution: {integrity: sha512-3gl0S7zQd88kCAZRO/DnxtBKuhMO4h0EaQIN3YgZfV6+pW+5+bf2AdQeHNESCoaQqo/gjGVYEf2YM4O5HJQqpQ==} - element-size@1.1.1: resolution: {integrity: sha512-eaN+GMOq/Q+BIWy0ybsgpcYImjGIdNLyjLFJU4XsLHXYQao5jCNb36GyN6C2qwmDDYSfIBmKpPpr4VnBdLCsPQ==} @@ -6382,8 +6366,8 @@ packages: resolution: {integrity: sha512-ZCkIjSYNDyGn0R6ewHDtXgns/Zre/NT6Agvq1/WobF7JXgFff4SeDroKiCO3fNJreU9YG429Sc81o4w5ok/W5g==} engines: {node: '>=10.2.0'} - enhanced-resolve@5.18.3: - resolution: {integrity: sha512-d4lC8xfavMeBjzGr2vECC3fsGXziXZQyJxD868h2M/mBI3PwAuODxAkLkq5HYuvrPYcUtiLzsTo8U3PgX3Ocww==} + enhanced-resolve@5.18.2: + resolution: {integrity: sha512-6Jw4sE1maoRJo3q8MsSIn2onJFbLTOjY9hlx4DZXmOKvLRd1Ok2kXmAGXaafL2+ijsJZ1ClYbl/pmqr9+k4iUQ==} engines: {node: '>=10.13.0'} entities@2.2.0: @@ -6815,9 +6799,6 @@ packages: resolution: {integrity: sha512-gIXjKqtFuWEgzFRJA9WCQeSJLZDjgJUOMCMzxtvFq/37KojM1BFGufqsCy0r4qSQmYLsZYMeyRqzIWOMup03sw==} engines: {node: '>=14'} - form-data-encoder@1.7.2: - resolution: {integrity: sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A==} - form-data@2.5.5: resolution: {integrity: sha512-jqdObeR2rxZZbPSGL+3VckHMYtu+f9//KXBsVny6JSX/pa38Fy+bGjuG8eW/H6USNQWhLi8Num++cU2yOCNz4A==} engines: {node: '>= 0.12'} @@ -6950,9 +6931,6 @@ packages: gl-matrix@3.4.3: resolution: {integrity: sha512-wcCp8vu8FT22BnvKVPjXa/ICBWRq/zjFfdofZy1WSpQZpphblv12/bOQLBC1rMM7SGOFS9ltVmKOHil5+Ml7gA==} - gl-matrix@3.4.4: - resolution: {integrity: sha512-latSnyDNt/8zYUB6VIJ6PCh2jBjJX6gnDsoCZ7LyW7GkqrD51EWwa9qCoGixj8YqBtETQK/xY7OmpTF8xz1DdQ==} - gl-text@1.4.0: resolution: {integrity: sha512-o47+XBqLCj1efmuNyCHt7/UEJmB9l66ql7pnobD6p+sgmBUdzfMZXIF0zD2+KRfpd99DJN+QXdvTFAGCKCVSmQ==} @@ -8979,8 +8957,8 @@ packages: resolution: {integrity: sha512-cxN6aIDPz6rm8hbebcP7vrQNhvRcveZoJU72Y7vskh4oIm+BZwBECnx5nTmrlres1Qapvx27Qo1Auukpf8PKXw==} engines: {node: '>=18'} - openai@5.12.1: - resolution: {integrity: sha512-26s536j4Fi7P3iUma1S9H33WRrw0Qu8pJ2nYJHffrlKHPU0JK4d0r3NcMgqEcAeTdNLGYNyoFsqN4g4YE9vutg==} + openai@5.12.2: + resolution: {integrity: sha512-xqzHHQch5Tws5PcKR2xsZGX9xtch+JQFz5zb14dGqlshmmDAFBFEWmeIpf7wVqWV+w7Emj7jRgkNJakyKE0tYQ==} hasBin: true peerDependencies: ws: ^8.18.0 @@ -11149,9 +11127,6 @@ packages: undici-types@5.26.5: resolution: {integrity: sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==} - undici-types@7.10.0: - resolution: {integrity: sha512-t5Fy/nfn+14LuOc2KNYg75vZqClpAiqscVvMygNnlsHBFpSXdJaYtXMcdNLpl/Qvc3P2cB3s6lOV51nqsFq4ag==} - unicorn-magic@0.3.0: resolution: {integrity: sha512-+QBBXBCvifc56fsbuxZQ6Sic3wqqc3WWaqxs58gvJrcOuN83HGTCwz3oS5phzU9LthRNE9VrJCFCLUgHeeFnfA==} engines: {node: '>=18'} @@ -13140,20 +13115,20 @@ snapshots: '@lumino/properties': 2.0.3 '@lumino/signaling': 2.1.4 - '@langchain/anthropic@0.3.26(@langchain/core@0.3.68(@opentelemetry/api@1.9.0)(openai@5.12.1(ws@8.18.3)(zod@3.25.76)))': + '@langchain/anthropic@0.3.26(@langchain/core@0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3(utf-8-validate@6.0.5))(zod@3.25.76)))': dependencies: '@anthropic-ai/sdk': 0.56.0 - '@langchain/core': 0.3.68(@opentelemetry/api@1.9.0)(openai@5.12.1(ws@8.18.3)(zod@3.25.76)) + '@langchain/core': 0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3(utf-8-validate@6.0.5))(zod@3.25.76)) fast-xml-parser: 4.5.3 - '@langchain/core@0.3.68(@opentelemetry/api@1.9.0)(openai@5.12.1(ws@8.18.3)(zod@3.25.76))': + '@langchain/core@0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3(utf-8-validate@6.0.5))(zod@3.25.76))': dependencies: '@cfworker/json-schema': 4.1.1 ansi-styles: 5.2.0 camelcase: 6.3.0 decamelize: 1.2.0 js-tiktoken: 1.0.20 - langsmith: 0.3.46(@opentelemetry/api@1.9.0)(openai@5.12.1(ws@8.18.3)(zod@3.25.76)) + langsmith: 0.3.46(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3(utf-8-validate@6.0.5))(zod@3.25.76)) mustache: 4.2.0 p-queue: 6.6.2 p-retry: 4.6.2 @@ -13166,31 +13141,31 @@ snapshots: - '@opentelemetry/sdk-trace-base' - openai - '@langchain/google-genai@0.2.16(@langchain/core@0.3.68(@opentelemetry/api@1.9.0)(openai@5.12.1(ws@8.18.3)(zod@3.25.76)))': + '@langchain/google-genai@0.2.16(@langchain/core@0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3(utf-8-validate@6.0.5))(zod@3.25.76)))': dependencies: '@google/generative-ai': 0.24.1 - '@langchain/core': 0.3.68(@opentelemetry/api@1.9.0)(openai@5.12.1(ws@8.18.3)(zod@3.25.76)) + '@langchain/core': 0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3(utf-8-validate@6.0.5))(zod@3.25.76)) uuid: 11.1.0 - '@langchain/mistralai@0.2.1(@langchain/core@0.3.68(@opentelemetry/api@1.9.0)(openai@5.12.1(ws@8.18.3)(zod@3.25.76)))(zod@3.25.76)': + '@langchain/mistralai@0.2.1(@langchain/core@0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3(utf-8-validate@6.0.5))(zod@3.25.76)))(zod@3.25.76)': dependencies: - '@langchain/core': 0.3.68(@opentelemetry/api@1.9.0)(openai@5.12.1(ws@8.18.3)(zod@3.25.76)) + '@langchain/core': 0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3(utf-8-validate@6.0.5))(zod@3.25.76)) '@mistralai/mistralai': 1.7.4(zod@3.25.76) uuid: 10.0.0 transitivePeerDependencies: - zod - '@langchain/ollama@0.2.3(@langchain/core@0.3.68(@opentelemetry/api@1.9.0)(openai@5.12.1(ws@8.18.3)(zod@3.25.76)))': + '@langchain/ollama@0.2.3(@langchain/core@0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3(utf-8-validate@6.0.5))(zod@3.25.76)))': dependencies: - '@langchain/core': 0.3.68(@opentelemetry/api@1.9.0)(openai@5.12.1(ws@8.18.3)(zod@3.25.76)) + '@langchain/core': 0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3(utf-8-validate@6.0.5))(zod@3.25.76)) ollama: 0.5.16 uuid: 10.0.0 - '@langchain/openai@0.6.6(@langchain/core@0.3.68(@opentelemetry/api@1.9.0)(openai@5.12.1(ws@8.18.3)(zod@3.25.76)))(ws@8.18.3)': + '@langchain/openai@0.6.7(@langchain/core@0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3(utf-8-validate@6.0.5))(zod@3.25.76)))(ws@8.18.3(utf-8-validate@6.0.5))': dependencies: - '@langchain/core': 0.3.68(@opentelemetry/api@1.9.0)(openai@5.12.1(ws@8.18.3)(zod@3.25.76)) + '@langchain/core': 0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3(utf-8-validate@6.0.5))(zod@3.25.76)) js-tiktoken: 1.0.20 - openai: 5.12.1(ws@8.18.3)(zod@3.25.76) + openai: 5.12.2(ws@8.18.3(utf-8-validate@6.0.5))(zod@3.25.76) zod: 3.25.76 transitivePeerDependencies: - ws @@ -14630,10 +14605,6 @@ snapshots: '@types/node-cleanup@2.1.5': {} - '@types/node-fetch@2.6.13': - dependencies: - '@types/node': 18.19.118 - form-data: 4.0.4 '@types/node-forge@1.3.13': dependencies: '@types/node': 18.19.118 @@ -14646,10 +14617,6 @@ snapshots: dependencies: undici-types: 5.26.5 - '@types/node@24.2.1': - dependencies: - undici-types: 7.10.0 - '@types/nodemailer@6.4.17': dependencies: '@types/node': 18.19.118 @@ -15557,13 +15524,6 @@ snapshots: node-releases: 2.0.19 update-browserslist-db: 1.1.3(browserslist@4.25.1) - browserslist@4.25.2: - dependencies: - caniuse-lite: 1.0.30001733 - electron-to-chromium: 1.5.199 - node-releases: 2.0.19 - update-browserslist-db: 1.1.3(browserslist@4.25.2) - bs-logger@0.2.6: dependencies: fast-json-stable-stringify: 2.1.0 @@ -15645,8 +15605,6 @@ snapshots: caniuse-lite@1.0.30001727: {} - caniuse-lite@1.0.30001733: {} - canvas-fit@1.5.0: dependencies: element-size: 1.1.1 @@ -16869,8 +16827,6 @@ snapshots: electron-to-chromium@1.5.182: {} - electron-to-chromium@1.5.199: {} - element-size@1.1.1: {} elementary-circuits-directed-graph@1.3.1: @@ -16937,7 +16893,7 @@ snapshots: - supports-color - utf-8-validate - enhanced-resolve@5.18.3: + enhanced-resolve@5.18.2: dependencies: graceful-fs: 4.2.11 tapable: 2.2.2 @@ -17529,8 +17485,6 @@ snapshots: cross-spawn: 7.0.6 signal-exit: 4.1.0 - form-data@4.0.3: - form-data-encoder@1.7.2: {} form-data@2.5.5: dependencies: asynckit: 0.4.0 @@ -17684,8 +17638,6 @@ snapshots: gl-matrix@3.4.3: {} - gl-matrix@3.4.4: {} - gl-text@1.4.0: dependencies: bit-twiddle: 1.0.2 @@ -19098,7 +19050,7 @@ snapshots: jest-worker@27.5.1: dependencies: - '@types/node': 24.2.1 + '@types/node': 18.19.118 merge-stream: 2.0.0 supports-color: 8.1.1 @@ -19373,7 +19325,7 @@ snapshots: langs@2.0.0: {} - langsmith@0.3.46(@opentelemetry/api@1.9.0)(openai@5.12.1(ws@8.18.3)(zod@3.25.76)): + langsmith@0.3.46(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3(utf-8-validate@6.0.5))(zod@3.25.76)): dependencies: '@types/uuid': 10.0.0 chalk: 4.1.2 @@ -19384,7 +19336,7 @@ snapshots: uuid: 10.0.0 optionalDependencies: '@opentelemetry/api': 1.9.0 - openai: 5.12.1(ws@8.18.3)(zod@3.25.76) + openai: 5.12.2(ws@8.18.3(utf-8-validate@6.0.5))(zod@3.25.76) launch-editor@2.10.0: dependencies: @@ -19618,7 +19570,7 @@ snapshots: csscolorparser: 1.0.3 earcut: 2.2.4 geojson-vt: 3.2.1 - gl-matrix: 3.4.4 + gl-matrix: 3.4.3 grid-index: 1.1.0 murmurhash-js: 1.0.0 pbf: 3.3.0 @@ -20198,7 +20150,7 @@ snapshots: is-inside-container: 1.0.0 is-wsl: 3.1.0 - openai@5.12.1(ws@8.18.3)(zod@3.25.76): + openai@5.12.2(ws@8.18.3(utf-8-validate@6.0.5))(zod@3.25.76): optionalDependencies: ws: 8.18.3(utf-8-validate@6.0.5) zod: 3.25.76 @@ -22774,8 +22726,6 @@ snapshots: undici-types@5.26.5: {} - undici-types@7.10.0: {} - unicorn-magic@0.3.0: {} unified@11.0.5: @@ -22848,12 +22798,6 @@ snapshots: escalade: 3.2.0 picocolors: 1.1.1 - update-browserslist-db@1.1.3(browserslist@4.25.2): - dependencies: - browserslist: 4.25.2 - escalade: 3.2.0 - picocolors: 1.1.1 - update-diff@1.1.0: {} uri-js@4.4.1: @@ -23179,9 +23123,9 @@ snapshots: '@webassemblyjs/wasm-parser': 1.14.1 acorn: 8.15.0 acorn-import-phases: 1.0.4(acorn@8.15.0) - browserslist: 4.25.2 + browserslist: 4.25.1 chrome-trace-event: 1.0.4 - enhanced-resolve: 5.18.3 + enhanced-resolve: 5.18.2 es-module-lexer: 1.7.0 eslint-scope: 5.1.1 events: 3.3.0 @@ -23211,9 +23155,9 @@ snapshots: '@webassemblyjs/wasm-parser': 1.14.1 acorn: 8.15.0 acorn-import-phases: 1.0.4(acorn@8.15.0) - browserslist: 4.25.2 + browserslist: 4.25.1 chrome-trace-event: 1.0.4 - enhanced-resolve: 5.18.3 + enhanced-resolve: 5.18.2 es-module-lexer: 1.7.0 eslint-scope: 5.1.1 events: 3.3.0 diff --git a/src/packages/server/conat/project/types.ts b/src/packages/server/conat/project/types.ts index 1c29cdab9f..15ae892e85 100644 --- a/src/packages/server/conat/project/types.ts +++ b/src/packages/server/conat/project/types.ts @@ -9,5 +9,5 @@ export interface Configuration { // pid limit pids?: number | string; // disk size - size?: number | string; + disk?: number | string; } From 179242f77fa5d8673d469e83a584f1b2acaaf1ea Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 12 Aug 2025 01:50:40 +0000 Subject: [PATCH 266/270] add todo for copyPath in next api --- src/packages/next/pages/api/v2/projects/copy-path.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/packages/next/pages/api/v2/projects/copy-path.ts b/src/packages/next/pages/api/v2/projects/copy-path.ts index 6d9390e1bb..d38a53b655 100644 --- a/src/packages/next/pages/api/v2/projects/copy-path.ts +++ b/src/packages/next/pages/api/v2/projects/copy-path.ts @@ -60,8 +60,10 @@ export default async function handle(req, res) { throw Error("must be a collaborator on source project"); } } + throw Error("TODO: reimplement copyPath"); const project = getProject(src_project_id); - await project.copyPath({ + console.log({ + project, path, target_project_id, target_path, From f4e7a14fdd814f8490a36f6fd167a08d9b777026 Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 12 Aug 2025 01:58:02 +0000 Subject: [PATCH 267/270] ts issue --- src/packages/backend/sandbox/exec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/packages/backend/sandbox/exec.ts b/src/packages/backend/sandbox/exec.ts index a966474db9..062292dc78 100644 --- a/src/packages/backend/sandbox/exec.ts +++ b/src/packages/backend/sandbox/exec.ts @@ -96,7 +96,7 @@ export default async function exec({ // console.log(`${cmd} ${args.join(" ")}`, { cwd }); const child = spawn(cmd, args, { stdio: ["ignore", "pipe", "pipe"], - env: {}, + env: {} as any, // sometimes next complains about this cwd, ...userId, }); From 582110afadebf286a09efbf4bfc2e7f78ad39856 Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 12 Aug 2025 02:07:16 +0000 Subject: [PATCH 268/270] clearly deprecate copy-path in hub --- src/packages/hub/copy-path.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/packages/hub/copy-path.ts b/src/packages/hub/copy-path.ts index 67c0f3b7a0..fe92300beb 100644 --- a/src/packages/hub/copy-path.ts +++ b/src/packages/hub/copy-path.ts @@ -3,6 +3,7 @@ * License: MS-RSL – see LICENSE.md for details */ +// DEPRECATED // Copy Operations Provider // Used in the "Client" @@ -148,6 +149,8 @@ export class CopyPath { const project = projectControl(mesg.src_project_id); // do the copy + throw Error("DEPRECATED"); + // @ts-ignore const copy_id = await project.copyPath({ path: mesg.src_path, target_project_id: mesg.target_project_id, From d8bc9605483f3edeca7abf8cfb82644bc8d289fb Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 12 Aug 2025 04:00:13 +0000 Subject: [PATCH 269/270] ... --- src/scripts/g.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/src/scripts/g.sh b/src/scripts/g.sh index 68c84d070c..ca41f3058b 100755 --- a/src/scripts/g.sh +++ b/src/scripts/g.sh @@ -3,7 +3,6 @@ mkdir -p `pwd`/logs export LOGS=`pwd`/logs rm -f $LOGS/log unset INIT_CWD -unset PGHOST export DEBUG="cocalc:*,-cocalc:silly:*" export DEBUG_CONSOLE="no" From f551b1d92cb531e02c665a28e02c837079627a88 Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 12 Aug 2025 04:33:33 +0000 Subject: [PATCH 270/270] remove no-longer-used dep --- src/packages/pnpm-lock.yaml | 15 +++------------ src/packages/server/package.json | 8 +++++--- 2 files changed, 8 insertions(+), 15 deletions(-) diff --git a/src/packages/pnpm-lock.yaml b/src/packages/pnpm-lock.yaml index 2d91207588..21c7e107b4 100644 --- a/src/packages/pnpm-lock.yaml +++ b/src/packages/pnpm-lock.yaml @@ -1258,9 +1258,6 @@ importers: async: specifier: ^1.5.2 version: 1.5.2 - await-spawn: - specifier: ^4.0.2 - version: 4.0.2 awaiting: specifier: ^3.0.0 version: 3.0.0 @@ -4955,10 +4952,6 @@ packages: resolution: {integrity: sha512-wvUjBtSGN7+7SjNpq/9M2Tg350UZD3q62IFZLbRAR1bSMlCo1ZaeW+BJ+D090e4hIIZLBcTDWe4Mh4jvUDajzQ==} engines: {node: '>= 0.4'} - await-spawn@4.0.2: - resolution: {integrity: sha512-GdADmeLJiMvGKJD3xWBcX40DMn07JNH1sqJYgYJZH7NTGJ3B1qDjKBKzxhhyR1hjIcnUGFUmE/+4D1HcHAJBAA==} - engines: {node: '>=10'} - awaiting@3.0.0: resolution: {integrity: sha512-19i4G7Hjxj9idgMlAM0BTRII8HfvsOdlr4D9cf3Dm1MZhvcKjBpzY8AMNEyIKyi+L9TIK15xZatmdcPG003yww==} engines: {node: '>=7.6.x'} @@ -15320,15 +15313,11 @@ snapshots: dependencies: possible-typed-array-names: 1.1.0 - await-spawn@4.0.2: - dependencies: - bl: 4.1.0 - awaiting@3.0.0: {} axios@1.11.0: dependencies: - follow-redirects: 1.15.9(debug@4.4.1) + follow-redirects: 1.15.9 form-data: 4.0.4 proxy-from-env: 1.1.0 transitivePeerDependencies: @@ -17464,6 +17453,8 @@ snapshots: dependencies: dtype: 2.0.0 + follow-redirects@1.15.9: {} + follow-redirects@1.15.9(debug@4.4.1): optionalDependencies: debug: 4.4.1 diff --git a/src/packages/server/package.json b/src/packages/server/package.json index 79d7f3e63a..0f982c8d81 100644 --- a/src/packages/server/package.json +++ b/src/packages/server/package.json @@ -28,7 +28,10 @@ "./settings": "./dist/settings/index.js", "./settings/*": "./dist/settings/*.js" }, - "keywords": ["utilities", "cocalc"], + "keywords": [ + "utilities", + "cocalc" + ], "scripts": { "preinstall": "npx only-allow pnpm", "clean": "rm -rf node_modules dist", @@ -44,8 +47,8 @@ "@cocalc/backend": "workspace:*", "@cocalc/conat": "workspace:*", "@cocalc/database": "workspace:*", - "@cocalc/gcloud-pricing-calculator": "^1.17.0", "@cocalc/file-server": "workspace:*", + "@cocalc/gcloud-pricing-calculator": "^1.17.0", "@cocalc/server": "workspace:*", "@cocalc/util": "workspace:*", "@google-cloud/bigquery": "^7.8.0", @@ -71,7 +74,6 @@ "@zxcvbn-ts/language-common": "^3.0.4", "@zxcvbn-ts/language-en": "^3.0.2", "async": "^1.5.2", - "await-spawn": "^4.0.2", "awaiting": "^3.0.0", "axios": "^1.11.0", "base62": "^2.0.1",