fix(ocap-kernel): free retired exports and harden GC delivery

Follow-up to the c-list accounting fix, addressing defects found in review.

An owner that stops naming its own export left the object behind. Both the
delivered `retireExport` and the `retireExports`/`abandonExports` syscalls tore
down the owner's c-list entry but left `owner` and `refCount` in place, with no
path that could ever reclaim them: `cleanupTerminatedVat` finds krefs by walking
the owner's c-list, and the collector only revisits krefs in `maybeFreeKrefs`.
The records leaked, and the next collection to visit such a kref read the
owner's deleted entry through `getRequired` and took the run loop down with it.
New `orphanKernelObject` drops the owner mapping and hands the object to the
collector, which already knows how to retire an orphan. `collectGarbage` also
treats an owner with no c-list entry as orphaned rather than trusting the
mapping.

Reporting a dead run loop belongs to #1005, which landed on main first. It is
what makes the audit usable at all: `assertRefCountsIfAuditing` throws from
inside a crank, so with the failure logged and swallowed a violation's sole
symptom was a test hanging to its timeout with no mention of reference counts.
The `kernel-test` case here asserts that shape — the caller is told the run loop
died, and the audit error rides along as the `cause`.

Also: GC action delivery survives a vanished endpoint or a failed delivery
instead of stopping the loop; `launchVat` tears down a worker whose kernel-side
registration failed rather than stranding it; `RefCountViolation` discriminates
on `kind` instead of sentinel-matching `stored`; and the store context's
auditing flag no longer shares a name with `auditRefCounts()`.

Tests cover the crash path, the orphan-and-collect sequence, retiring
stragglers, GC-action robustness, and that a violation reaches a caller. The
`item.target` charge and both `deliver|notify` early returns now have assertions
that fail if the fix is reverted.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

Author: sirtimid

packages/kernel-test/src/garbage-collection.test.ts

@@ -238,17 +238,30 @@ describe('Garbage Collection', () => {
     });
 
     /**
-     * Give an importer a chance to notice a dropped object and tell the kernel.
+     * Give an importer a chance to notice a dropped object and tell the kernel,
+     * then keep cranking until the resulting GC actions have all been consumed.
      *
      * @param vatId - The vat to reap.
      * @param rootKRef - That vat's root, to poke with cranks afterwards.
      */
     async function reapAndSettle(vatId: VatId, rootKRef: KRef): Promise<void> {
       kernel.reapVats((id) => id === vatId);
-      for (let i = 0; i < 3; i++) {
+      // BOYD has to reach the vat, the vat has to answer, and the kernel has to
+      // act on the answer — but a round can queue more work, so loop until the
+      // queue is actually empty rather than guessing at a crank count.
+      const maxRounds = 10;
+      for (let round = 0; round < maxRounds; round++) {
         await kernel.queueMessage(rootKRef, 'noop', []);
         await waitUntilQuiescent(500);
+        if ([...kernelStore.getGCActions()].length === 0) {
+          return;
+        }
       }
+      throw Error(
+        `GC actions still pending after ${maxRounds} rounds: ${[
+          ...kernelStore.getGCActions(),
+        ].join(', ')}`,
+      );
     }
 
     it('survives until both importers let go', async () => {

packages/kernel-test/src/refcount-audit.test.ts

@@ -0,0 +1,63 @@
+import { makeSQLKernelDatabase } from '@metamask/kernel-store/sqlite/nodejs';
+import { makeKernelStore } from '@metamask/ocap-kernel';
+import type { KRef, VatId } from '@metamask/ocap-kernel';
+import { expect, describe, it } from 'vitest';
+
+import {
+  getBundleSpec,
+  makeKernel,
+  makeMockLogger,
+  runTestVats,
+} from './utils.ts';
+
+/**
+ * The per-crank audit throws from inside the run loop, which nothing restarts.
+ * Unless that failure is reported to whoever is waiting on the kernel, the only
+ * symptom is a test that hangs until its timeout, with no mention of reference
+ * counts anywhere — which would make the audit worthless as a build gate.
+ */
+describe('reference count audit', () => {
+  it('reports a violation to kernel callers rather than hanging', async () => {
+    const kernelDatabase = await makeSQLKernelDatabase({
+      dbFilename: ':memory:',
+    });
+    const kernelStore = makeKernelStore(kernelDatabase);
+    const kernel = await makeKernel(kernelDatabase, true, makeMockLogger());
+    await runTestVats(kernel, {
+      bootstrap: 'exporter',
+      forceReset: true,
+      vats: {
+        exporter: {
+          bundleSpec: getBundleSpec('exporter-vat'),
+          parameters: { name: 'Exporter' },
+        },
+      },
+    });
+
+    const exporterVatId = kernel.getVats()[0]?.id as VatId;
+    const exporterKRef = kernelStore.getRootObject(exporterVatId) as KRef;
+
+    kernelStore.setObjectRefCount(exporterKRef, {
+      reachable: 7,
+      recognizable: 9,
+    });
+
+    // The crank carrying this message settles its result before the
+    // end-of-crank audit runs, so this one may still succeed.
+    await kernel
+      .queueMessage(exporterKRef, 'createObject', ['x'])
+      .catch(() => undefined);
+
+    // What a caller is told directly is that the run loop is gone; the audit
+    // failure that killed it rides along as the `cause`. That chain is the part
+    // that has to survive, since "run loop died" on its own names nothing.
+    const failure = (await kernel
+      .queueMessage(exporterKRef, 'createObject', ['y'])
+      .catch((error) => error)) as Error;
+
+    expect(failure.message).toMatch(/Kernel run loop died/u);
+    expect(String(failure.cause)).toMatch(
+      /reference count invariant violated/u,
+    );
+  }, 30000);
+});

packages/ocap-kernel/CHANGELOG.md

@@ -38,8 +38,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - Reference-count auditing: `auditRefCounts`, `recomputeRefCounts`, `formatRefCountViolations`, `assertRefCountsIfAuditing`, and `setRefCountAuditing` on the kernel store, plus a `Kernel.make` option `auditRefCounts` that verifies every kref's counts against the references the kernel actually holds at the end of each crank ([#1010](https://github.com/MetaMask/ocap-kernel/pull/1010))
   - Reports drift in both directions: counts too low (a live capability can be collected) and counts too high with no holder (a leak)
-  - Exports the `RefCountViolation` type
+  - Exports the `RefCountViolation` type, a union over `kind: 'mismatch' | 'dangling'`
 - Add `setReachableFlag` to the kernel store, the counterpart to `clearReachableFlag` ([#1010](https://github.com/MetaMask/ocap-kernel/pull/1010))
+- Add `orphanKernelObject` to the kernel store, which drops an object's owner mapping and hands it to the collector ([#1010](https://github.com/MetaMask/ocap-kernel/pull/1010))
 
 ### Changed
 
@@ -79,6 +80,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   - A root is addressable while its vat lives whether or not anyone imports it; the old `(1, 1)` birth baseline was standing in for this
 - Garbage-collection action delivery now moves the kernel's own c-list: `dropExports` clears the owner's reachable flag and `retireExports`/`retireImports` tear the entry down ([#1010](https://github.com/MetaMask/ocap-kernel/pull/1010))
   - Previously the owner's flag never cleared, so the same action could be re-derived, and retired entries outlived the objects they named
+- Orphan a kernel object when its owner stops naming it (a delivered `retireExport`, or a `retireExports`/`abandonExports` syscall), so its `owner` and `refCount` records no longer outlive the c-list entry they were reachable through — they leaked, and the next collection to visit such a kref killed the run loop ([#1010](https://github.com/MetaMask/ocap-kernel/pull/1010))
+- `queueMessage` now rejects with the error that stopped the run loop, and messages already in flight are rejected rather than left pending forever ([#1010](https://github.com/MetaMask/ocap-kernel/pull/1010))
+- Garbage-collection action delivery survives a vanished endpoint or a failed delivery instead of stopping the run loop ([#1010](https://github.com/MetaMask/ocap-kernel/pull/1010))
+- Tear down a vat whose worker launched but whose kernel-side registration then failed ([#1010](https://github.com/MetaMask/ocap-kernel/pull/1010))
 - Charge a delivered message's target reference against the run-queue item's own target rather than the routed target, so a message routed through a resolved promise no longer decrements an object nobody charged while leaking the promise ([#1010](https://github.com/MetaMask/ocap-kernel/pull/1010))
 - Release a queued notification's reference before the paths that decide there is nothing to deliver, and stop decrementing references on promises retired alongside it that nobody had taken ([#1010](https://github.com/MetaMask/ocap-kernel/pull/1010))
 - Transfer, rather than duplicate, the references a message carries when it is queued on an unresolved promise and later re-enqueued on resolution ([#1010](https://github.com/MetaMask/ocap-kernel/pull/1010))

packages/ocap-kernel/src/KernelRouter.test.ts

@@ -65,6 +65,7 @@ describe('KernelRouter', () => {
       clearReachableFlag: vi.fn(),
       deleteCListEntry: vi.fn(),
       forgetKref: vi.fn(),
+      orphanKernelObject: vi.fn(),
       createCrankSavepoint: vi.fn(),
     } as unknown as KernelStore;
 
@@ -317,6 +318,38 @@ describe('KernelRouter', () => {
         ]);
       });
 
+      it('charges the promise, not the object it resolved to', async () => {
+        const promiseId = 'kp123';
+        const resolvedObject = 'ko456';
+        (
+          kernelStore.getKernelPromise as unknown as MockInstance
+        ).mockReturnValueOnce({
+          state: 'fulfilled',
+          value: { body: '#"$0"', slots: [resolvedObject] },
+        });
+        (kernelStore.getOwner as unknown as MockInstance).mockReturnValue('v1');
+
+        await kernelRouter.deliver({
+          type: 'send',
+          target: promiseId,
+          message: {
+            methargs: { body: 'method args', slots: [] },
+            result: null,
+          },
+        });
+
+        // The run queue item was charged against the promise it named, so that
+        // is what has to be released — not whatever routing resolved it to.
+        expect(kernelStore.decrementRefCount).toHaveBeenCalledWith(
+          promiseId,
+          'deliver|send|target',
+        );
+        expect(kernelStore.decrementRefCount).not.toHaveBeenCalledWith(
+          resolvedObject,
+          'deliver|send|target',
+        );
+      });
+
       it('splats message when promise resolves to a non-object', async () => {
         // Setup a fulfilled promise that doesn't resolve to an object
         const promiseId = 'kp123';
@@ -580,6 +613,12 @@ describe('KernelRouter', () => {
         // Verify no notification was delivered to the vat
         expect(endpointHandle.deliverNotify).not.toHaveBeenCalled();
         expect(result).toStrictEqual({ didDelivery: endpointId });
+        // Nothing was delivered, but the queued notification is gone either
+        // way, so its reference has to be released on this path too.
+        expect(kernelStore.decrementRefCount).toHaveBeenCalledWith(
+          kpid,
+          'deliver|notify',
+        );
       });
 
       it('returns didDelivery when no kpids to retire', async () => {
@@ -618,6 +657,10 @@ describe('KernelRouter', () => {
         // Verify no notification was delivered to the vat
         expect(endpointHandle.deliverNotify).not.toHaveBeenCalled();
         expect(result).toStrictEqual({ didDelivery: endpointId });
+        expect(kernelStore.decrementRefCount).toHaveBeenCalledWith(
+          kpid,
+          'deliver|notify',
+        );
       });
 
       it('throws if notification is for an unresolved promise', async () => {
@@ -715,6 +758,60 @@ describe('KernelRouter', () => {
           ]);
         },
       );
+
+      it('orphans the object when delivering retireExports', async () => {
+        await kernelRouter.deliver({
+          type: 'retireExports',
+          endpointId: 'v1',
+          krefs: ['ko1', 'ko2'],
+        });
+
+        // The owner has given up the last name for the object, so the kernel's
+        // record of who owns it must go too or it outlives every reference.
+        expect(
+          (kernelStore.orphanKernelObject as unknown as MockInstance).mock
+            .calls,
+        ).toStrictEqual([['ko1'], ['ko2']]);
+      });
+
+      it('leaves ownership alone when delivering retireImports', async () => {
+        await kernelRouter.deliver({
+          type: 'retireImports',
+          endpointId: 'v1',
+          krefs: ['ko1'],
+        });
+
+        expect(kernelStore.orphanKernelObject).not.toHaveBeenCalled();
+      });
+
+      it('skips the action when the endpoint has vanished', async () => {
+        getEndpoint.mockImplementationOnce(() => {
+          throw Error('vat v1 not found');
+        });
+
+        const result = await kernelRouter.deliver({
+          type: 'retireImports',
+          endpointId: 'v1',
+          krefs: ['ko1'],
+        });
+
+        expect(result).toStrictEqual({ didDelivery: 'v1' });
+        expect(kernelStore.deleteCListEntry).not.toHaveBeenCalled();
+      });
+
+      it('survives a failed delivery', async () => {
+        (
+          endpointHandle.deliverRetireImports as unknown as MockInstance
+        ).mockRejectedValueOnce(Error('endpoint went away mid-delivery'));
+
+        const result = await kernelRouter.deliver({
+          type: 'retireImports',
+          endpointId: 'v1',
+          krefs: ['ko1'],
+        });
+
+        expect(result).toStrictEqual({ didDelivery: 'v1' });
+      });
     });
 
     describe('bringOutYourDead', () => {

packages/ocap-kernel/src/KernelRouter.ts

@@ -405,10 +405,12 @@ export class KernelRouter {
         this.#kernelStore.translateCapDataKtoE(endpointId, tPromise.value),
       ]);
     }
-    // TODO(#1006 follow-up): SwingSet also tears down the c-list entry for each
-    // promise in the batch here, since the endpoint can never refer to a
-    // settled promise by that eref again. Left alone for now because the
-    // debug UI discovers exported ocap URLs by scanning these entries.
+    // TODO: SwingSet also tears down the c-list entry for each promise in the
+    // batch here, since the endpoint can never refer to a settled promise by
+    // that eref again. Left alone for now because the debug UI discovers
+    // exported ocap URLs by scanning these entries. The cost of keeping them is
+    // that a settled promise reached this way holds a count forever, so it is
+    // never collected and its resolution slots are never released.
     const endpoint = this.#getEndpoint(endpointId);
     return await endpoint.deliverNotify(resolutions);
   }
@@ -424,29 +426,57 @@ export class KernelRouter {
     this.#logger?.log(
       `@@@@ deliver ${endpointId} ${type} ${JSON.stringify(krefs)}`,
     );
-    const endpoint = this.#getEndpoint(endpointId);
+    let endpoint: EndpointHandle;
+    try {
+      endpoint = this.#getEndpoint(endpointId);
+    } catch (error) {
+      // The endpoint was selected for this action while its c-list still
+      // existed, but it has since gone away (terminated, and cleaned up in the
+      // same crank). Nothing left to tell; its c-list goes with it.
+      this.#logger?.error(
+        `Skipping ${type} for vanished endpoint ${endpointId}:`,
+        error,
+      );
+      return { didDelivery: endpointId };
+    }
     const erefs = this.#kernelStore.krefsToErefs(endpointId, krefs);
     // Telling an endpoint to let go is also the kernel letting go. Otherwise a
     // dropped export stays flagged reachable, so the same action gets derived
     // again, and retired entries outlive the objects they name.
     krefs.forEach((kref, index) => {
       if (type === 'dropExports') {
         this.#kernelStore.clearReachableFlag(endpointId, kref);
-      } else {
-        this.#kernelStore.deleteCListEntry(
-          endpointId,
-          kref,
-          erefs[index] as ERef,
-        );
+        return;
+      }
+      // `erefs` is parallel to `krefs`: krefsToErefs throws rather than
+      // returning a short array, so every index is populated.
+      this.#kernelStore.deleteCListEntry(
+        endpointId,
+        kref,
+        erefs[index] as ERef,
+      );
+      if (type === 'retireExports') {
+        // Retiring an export is the owner giving up the last name for the
+        // object, so the kernel's record of who owns it goes too.
+        this.#kernelStore.orphanKernelObject(kref);
       }
     });
     const method =
       `deliver${(type[0] as string).toUpperCase()}${type.slice(1)}` as
         | 'deliverDropExports'
         | 'deliverRetireExports'
         | 'deliverRetireImports';
-    const crankResult = await endpoint[method](erefs);
-    return crankResult;
+    try {
+      return await endpoint[method](erefs);
+    } catch (error) {
+      // The kernel has already let go above, which is the part that matters for
+      // accounting. Don't let a failed notification take down the run loop.
+      this.#logger?.error(
+        `Delivery of ${type} to ${endpointId} failed:`,
+        error,
+      );
+      return { didDelivery: endpointId };
+    }
   }
 
   /**

packages/ocap-kernel/src/garbage-collection/gc-handlers.ts

@@ -84,5 +84,9 @@ export function performExportCleanup(
       }
     }
     kernelStore.forgetKref(endpointId, kref);
+    // The owner no longer names the object, so nothing can reach it through
+    // this endpoint again. Drop the owner mapping too, or the kernel's record
+    // of the object outlives the only c-list entry it was reachable through.
+    kernelStore.orphanKernelObject(kref);
   }
 }

packages/ocap-kernel/src/store/index.test.ts

@@ -149,6 +149,7 @@ describe('kernel store', () => {
         'markVatAsTerminated',
         'nextReapAction',
         'nextTerminatedVatCleanup',
+        'orphanKernelObject',
         'pinObject',
         'provideIncarnationId',
         'recomputeRefCounts',

packages/ocap-kernel/src/store/index.ts

@@ -154,7 +154,7 @@ export function makeKernelStore(kdb: KernelDatabase, logger?: Logger) {
     subclusters: provideCachedStoredValue('subclusters', '[]'),
     nextSubclusterId: provideCachedStoredValue('nextSubclusterId', '1'),
     vatToSubclusterMap: provideCachedStoredValue('vatToSubclusterMap', '{}'),
-    auditRefCounts: false,
+    refCountAuditingEnabled: false,
     // Logging
     logger: logger?.subLogger({ tags: ['kernel-store'] }),
   };