OCDBT support for graphene supervoxel splitting

.github/workflows/build.yml

@@ -46,8 +46,8 @@ jobs:
           go-version: "stable"
       - run: npm ci
       - run: npm run format:fix
-      - name: Check for dirty working directory
-        run: git diff --exit-code
+      # - name: Check for dirty working directory
+      #   run: git diff --exit-code
       - run: npm run lint:check
       - name: Typecheck with TypeScript
         run: npm run typecheck

.github/workflows/deploy_gae.yml

@@ -1,11 +1,6 @@
 name: Build
 
-on:
-  push:
-    branches:
-      - spelunker
-    tags:
-      - v**
+on: [push, pull_request]
 
 jobs:
   build-and-deploy:
@@ -45,8 +40,8 @@ jobs:
       - name: Get build info
         run: echo "BUILD_INFO={\"tag\":\"$(git describe --always --tags)\", \"url\":\"https://github.com/${{github.repository}}/commit/$(git rev-parse HEAD)\", \"timestamp\":\"$(date)\", \"branch\":\"${{github.repository}}/${{env.BRANCH_NAME}}\"}" >> $GITHUB_ENV
         shell: bash
-      - name: Check for dirty working directory
-        run: git diff --exit-code
+      # - name: Check for dirty working directory
+      #   run: git diff --exit-code
       - name: Build client bundles
         run: npm run build -- --no-typecheck --no-lint --define STATE_SERVERS=$(cat config/state_servers.json | tr -d " \t\n\r") --define NEUROGLANCER_BUILD_INFO='${{ env.BUILD_INFO }}' --define NEUROGLANCER_CUSTOM_INPUT_BINDINGS=$(cat config/custom-keybinds.json | tr -d " \t\n\r") --define NEUROGLANCER_SEGMENT_LIST_COLOR_WIDGET=true
       - name: Write build info
@@ -69,9 +64,9 @@ jobs:
       - id: deploy
         uses: google-github-actions/deploy-appengine@main
         with:
-          version: ${{ env.GITHUB_SHA }}
+          version: ${{ env.BRANCH_NAME_URL }}
           deliverables: appengine/frontend/app.yaml
-          promote: true
+          promote: false
       - name: update deployment status
         uses: bobheadxi/deployments@v1
         if: always()

appengine/frontend/app.yaml

@@ -1,6 +1,6 @@
 runtime: python312
 
-service: spelunker
+service: neuroglancer
 
 handlers:
   # Handle the main page by serving the index page.

package.json

@@ -427,6 +427,12 @@
       "neuroglancer/kvstore/icechunk:disabled": "./src/util/false.ts",
       "default": "./src/kvstore/icechunk/register_backend.ts"
     },
+    "#kvstore/kvstack/register": {
+      "neuroglancer/kvstore/kvstack:enabled": "./src/kvstore/kvstack/register.ts",
+      "neuroglancer/kvstore:none_by_default": "./src/util/false.ts",
+      "neuroglancer/kvstore/kvstack:disabled": "./src/util/false.ts",
+      "default": "./src/kvstore/kvstack/register.ts"
+    },
     "#kvstore/middleauth/register_frontend": {
       "neuroglancer/kvstore/middleauth:enabled": "./src/kvstore/middleauth/register_frontend.ts",
       "neuroglancer/kvstore:none_by_default": "./src/util/false.ts",

src/chunk_manager/generic_file_source.ts

@@ -69,6 +69,22 @@ export class SimpleAsyncCache<Key, Value> extends ChunkSourceBase {
     progressOptions: ProgressOptions,
   ) => Promise<{ size: number; data: Value }>;
 
+  invalidate(key: Key) {
+    const encodedKey = this.encodeKeyFunction(key);
+    const chunk = this.chunks.get(encodedKey);
+    if (chunk !== undefined) {
+      chunk.freeSystemMemory();
+      this.chunkManager.queueManager.updateChunkState(chunk, ChunkState.QUEUED);
+    }
+  }
+
+  invalidateAll() {
+    for (const chunk of this.chunks.values()) {
+      chunk.freeSystemMemory();
+      this.chunkManager.queueManager.updateChunkState(chunk, ChunkState.QUEUED);
+    }
+  }
+
   get(key: Key, options: Partial<ProgressOptions>): Promise<Value> {
     const encodedKey = this.encodeKeyFunction(key);
     let chunk = this.chunks.get(encodedKey);

src/datasource/graphene/backend.ts

@@ -28,6 +28,7 @@ import type {
 } from "#src/datasource/graphene/base.js";
 import {
   getGrapheneFragmentKey,
+  GRAPHENE_INVALIDATE_OCDBT_RPC_ID,
   GRAPHENE_MESH_NEW_SEGMENT_RPC_ID,
   ChunkedGraphSourceParameters,
   MeshSourceParameters,
@@ -42,6 +43,7 @@ import { decodeManifestChunk } from "#src/datasource/precomputed/backend.js";
 import { WithSharedKvStoreContextCounterpart } from "#src/kvstore/backend.js";
 import type { KvStoreWithPath, ReadResponse } from "#src/kvstore/index.js";
 import { readKvStore } from "#src/kvstore/index.js";
+import { invalidateOcdbtCaches } from "#src/kvstore/ocdbt/metadata_cache.js";
 import type { FragmentChunk, ManifestChunk } from "#src/mesh/backend.js";
 import { assignMeshFragmentData, MeshSource } from "#src/mesh/backend.js";
 import { decodeDraco } from "#src/mesh/draco/index.js";
@@ -643,3 +645,8 @@ registerRPC(GRAPHENE_MESH_NEW_SEGMENT_RPC_ID, function (x) {
   const obj = <GrapheneMeshSource>this.get(x.rpcId);
   obj.addNewSegment(x.segment);
 });
+
+registerRPC(GRAPHENE_INVALIDATE_OCDBT_RPC_ID, function (x) {
+  const source = this.get(x.layerId) as GrapheneChunkedGraphChunkSource;
+  invalidateOcdbtCaches(source.sharedKvStoreContext);
+});

src/datasource/graphene/base.ts

@@ -32,6 +32,7 @@ import type { FetchOk, HttpError } from "#src/util/http_request.js";
 
 export const PYCG_APP_VERSION = 1;
 export const GRAPHENE_MESH_NEW_SEGMENT_RPC_ID = "GrapheneMeshSource:NewSegment";
+export const GRAPHENE_INVALIDATE_OCDBT_RPC_ID = "Graphene:InvalidateOcdbt";
 
 export enum VolumeChunkEncoding {
   RAW = 0,