new architecture

.github/workflows/make-and-test.yml

@@ -39,7 +39,7 @@ jobs:
           detached: true
       - uses: actions/checkout@v4
       - name: Install python3 requests
-        run: sudo apt-get install python3-requests
+        run: sudo apt-get install python3-requests python3-yapf
       - name: Check doc links
         run: cd src/scripts && python3 check_doc_urls.py || sleep 5 || python3 check_doc_urls.py
 
@@ -91,19 +91,15 @@ jobs:
           # cache: "pnpm"
           # cache-dependency-path: "src/packages/pnpm-lock.yaml"
 
-      - name: Download and install Valkey
-        run: |
-          VALKEY_VERSION=8.1.2
-          curl -LOq https://download.valkey.io/releases/valkey-${VALKEY_VERSION}-jammy-x86_64.tar.gz
-          tar -xzf valkey-${VALKEY_VERSION}-jammy-x86_64.tar.gz
-          sudo cp valkey-${VALKEY_VERSION}-jammy-x86_64/bin/valkey-server /usr/local/bin/
+      - name: Install btrfs-progs and bup for @cocalc/file-server
+        run: sudo apt-get update && sudo apt-get install -y btrfs-progs bup
 
       - name: Set up Python venv and Jupyter kernel
         run: |
           python3 -m pip install --upgrade pip virtualenv
           python3 -m virtualenv venv
           source venv/bin/activate
-          pip install ipykernel
+          pip install ipykernel yapf
           python -m ipykernel install --prefix=./jupyter-local --name python3-local --display-name "Python 3 (Local)"
 
 
@@ -128,30 +124,30 @@ jobs:
           name: "test-results-node-${{ matrix.node-version }}-pg-${{ matrix.pg-version }}"
           path: 'src/packages/*/junit.xml'
 
-  report:
-    runs-on: ubuntu-latest
+#   report:
+#     runs-on: ubuntu-latest
 
-    needs: [test]
+#     needs: [test]
 
-    if: ${{ !cancelled() }}
+#     if: ${{ !cancelled() }}
 
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
+#     steps:
+#       - name: Checkout code
+#         uses: actions/checkout@v4
 
-      - name: Download all test artifacts
-        uses: actions/download-artifact@v4
-        with:
-          pattern: "test-results-*"
-          merge-multiple: true
-          path: test-results/
+#       - name: Download all test artifacts
+#         uses: actions/download-artifact@v4
+#         with:
+#           pattern: "test-results-*"
+#           merge-multiple: true
+#           path: test-results/
 
-      - name: Test Report
-        uses: dorny/test-reporter@v2
-        with:
-          name: CoCalc Jest Tests
-          path: 'test-results/**/junit.xml'
-          reporter: jest-junit
-          use-actions-summary: 'true'
-          fail-on-error: false
+#       - name: Test Report
+#         uses: dorny/test-reporter@v2
+#         with:
+#           name: CoCalc Jest Tests
+#           path: 'test-results/**/junit.xml'
+#           reporter: jest-junit
+#           use-actions-summary: 'true'
+#           fail-on-error: false
 

src/package.json

@@ -18,7 +18,7 @@
     "version-check": "pip3 install typing_extensions mypy  || pip3 install --break-system-packages typing_extensions mypy && ./workspaces.py version-check && mypy scripts/check_npm_packages.py",
     "test-parallel": "unset DEBUG && pnpm run version-check && cd packages && pnpm run -r --parallel test",
     "test": "unset DEBUG && pnpm run depcheck && pnpm run version-check && ./workspaces.py test",
-    "test-github-ci": "unset DEBUG && pnpm run depcheck && pnpm run version-check && ./workspaces.py test --exclude=jupyter,file-server --retries=1",
+    "test-github-ci": "unset DEBUG && pnpm run depcheck && pnpm run version-check && ./workspaces.py test --test-github-ci --exclude=jupyter --retries=1",
     "depcheck": "cd packages && pnpm run -r --parallel depcheck",
     "prettier-all": "cd packages/",
     "local-ci": "./scripts/ci.sh",

src/packages/backend/conat/files/local-path.ts

@@ -0,0 +1,59 @@
+import { fsServer, DEFAULT_FILE_SERVICE } from "@cocalc/conat/files/fs";
+import { SandboxedFilesystem } from "@cocalc/backend/sandbox";
+import { isValidUUID } from "@cocalc/util/misc";
+import { type Client } from "@cocalc/conat/core/client";
+import { conat } from "@cocalc/backend/conat/conat";
+import { client as createFileClient } from "@cocalc/conat/files/file-server";
+
+export async function localPathFileserver({
+  path,
+  service = DEFAULT_FILE_SERVICE,
+  client,
+  project_id,
+  unsafeMode,
+}: {
+  service?: string;
+  client?: Client;
+  // if project_id is specified, use single project mode.
+  project_id?: string;
+  // only used in single project mode
+  path?: string;
+  unsafeMode?: boolean;
+} = {}) {
+  client ??= conat();
+
+  const server = await fsServer({
+    service,
+    client,
+    project_id,
+    fs: async (subject: string) => {
+      if (project_id) {
+        if (path == null) {
+          throw Error("path must be specified");
+        }
+        return new SandboxedFilesystem(path, { unsafeMode });
+      } else {
+        const project_id = getProjectId(subject);
+        const fsclient = createFileClient({ client });
+        const { path } = await fsclient.mount({ project_id });
+        return new SandboxedFilesystem(path, { unsafeMode, host: project_id });
+      }
+    },
+  });
+  return { server, client, path, service, close: () => server.close() };
+}
+
+function getProjectId(subject: string) {
+  const v = subject.split(".");
+  if (v.length != 2) {
+    throw Error("subject must have 2 segments");
+  }
+  if (!v[1].startsWith("project-")) {
+    throw Error("second segment of subject must start with 'project-'");
+  }
+  const project_id = v[1].slice("project-".length);
+  if (!isValidUUID(project_id)) {
+    throw Error("not a valid project id");
+  }
+  return project_id;
+}

src/packages/backend/conat/files/test/listing.test.ts

@@ -0,0 +1,93 @@
+import { SandboxedFilesystem } from "@cocalc/backend/sandbox";
+import { mkdtemp, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "path";
+import { randomId } from "@cocalc/conat/names";
+import listing from "@cocalc/conat/files/listing";
+
+let tmp;
+beforeAll(async () => {
+  tmp = await mkdtemp(join(tmpdir(), `cocalc-${randomId()}0`));
+});
+
+afterAll(async () => {
+  try {
+    await rm(tmp, { force: true, recursive: true });
+  } catch {}
+});
+
+describe("creating a listing monitor starting with an empty directory", () => {
+  let fs, dir;
+  it("creates sandboxed filesystem", async () => {
+    fs = new SandboxedFilesystem(tmp);
+    dir = await listing({ path: "", fs });
+  });
+
+  it("initial listing is empty", () => {
+    expect(Object.keys(dir.files)).toEqual([]);
+  });
+
+  let iter;
+  it("create a file and get an update", async () => {
+    iter = dir.iter();
+    await fs.writeFile("a.txt", "hello");
+    let { value } = await iter.next();
+    expect(value).toEqual({
+      mtime: value.mtime,
+      name: "a.txt",
+      size: value.size,
+    });
+    // it's possible that the file isn't written completely above.
+    if (value.size != 5) {
+      ({ value } = await iter.next());
+    }
+    const stat = await fs.stat("a.txt");
+    expect(stat.mtimeMs).toEqual(value.mtime);
+    expect(dir.files["a.txt"]).toEqual({ mtime: value.mtime, size: 5 });
+  });
+
+  it("modify the file and get two updates -- one when it starts and another when done", async () => {
+    await fs.appendFile("a.txt", " there");
+    const { value } = await iter.next();
+    expect(value).toEqual({ mtime: value.mtime, name: "a.txt", size: 5 });
+    const { value: value2 } = await iter.next();
+    expect(value2).toEqual({ mtime: value2.mtime, name: "a.txt", size: 11 });
+    const stat = await fs.stat("a.txt");
+    expect(stat.mtimeMs).toEqual(value2.mtime);
+    expect(dir.files["a.txt"]).toEqual({ mtime: value2.mtime, size: 11 });
+  });
+
+  it("create another monitor starting with the now nonempty directory", async () => {
+    const dir2 = await listing({ path: "", fs });
+    expect(Object.keys(dir2.files!)).toEqual(["a.txt"]);
+    expect(dir.files["a.txt"].mtime).toBeCloseTo(dir2.files!["a.txt"].mtime);
+    dir2.close();
+  });
+
+  const count = 500;
+  it(`creates ${count} files and see they are found`, async () => {
+    const n = Object.keys(dir.files).length;
+
+    for (let i = 0; i < count; i++) {
+      await fs.writeFile(`${i}`, "");
+    }
+    const values: string[] = [];
+    while (true) {
+      const { value } = await iter.next();
+      if (value == "a.txt") {
+        continue;
+      }
+      values.push(value);
+      if (value.name == `${count - 1}`) {
+        break;
+      }
+    }
+    expect(new Set(values).size).toEqual(count);
+
+    expect(Object.keys(dir.files).length).toEqual(n + count);
+  });
+
+  it("cleans up", () => {
+    dir.close();
+  });
+});