From 0c34a2693fada5159dbda207f6766a03d46742ec Mon Sep 17 00:00:00 2001 From: Bob Webb Date: Wed, 12 Feb 2025 13:43:11 +0100 Subject: [PATCH 01/36] add MessageType and Updates types --- .../src/eip712/Secp256k1DelegateSigner.ts | 16 +++++--- .../chain-ethereum/src/siwf/SIWFSigner.ts | 12 +++++- packages/cli/src/commands/import.ts | 4 +- packages/core/src/Canvas.ts | 38 +++++++++++++------ packages/core/src/ExecutionContext.ts | 5 ++- packages/core/src/runtime/AbstractRuntime.ts | 22 +++++------ packages/core/src/schema.ts | 23 +++++++++-- packages/core/src/targets/interface.ts | 6 +-- packages/core/src/utils.ts | 20 +++++----- packages/interfaces/src/MessageType.ts | 6 +++ packages/interfaces/src/SessionSigner.ts | 13 +++---- packages/interfaces/src/Updates.ts | 10 +++++ packages/interfaces/src/index.ts | 2 + .../signatures/src/AbstractSessionSigner.ts | 13 +++---- 14 files changed, 122 insertions(+), 68 deletions(-) create mode 100644 packages/interfaces/src/MessageType.ts create mode 100644 packages/interfaces/src/Updates.ts diff --git a/packages/chain-ethereum/src/eip712/Secp256k1DelegateSigner.ts b/packages/chain-ethereum/src/eip712/Secp256k1DelegateSigner.ts index 9cd906e87..961b95b06 100644 --- a/packages/chain-ethereum/src/eip712/Secp256k1DelegateSigner.ts +++ b/packages/chain-ethereum/src/eip712/Secp256k1DelegateSigner.ts @@ -11,7 +11,7 @@ import { import { AbiCoder } from "ethers/abi" -import type { Action, Message, Session, Snapshot, Signature, SignatureScheme, Signer } from "@canvas-js/interfaces" +import type { Message, Signature, SignatureScheme, Signer, MessageType } from "@canvas-js/interfaces" import { decodeURI, encodeURI } from "@canvas-js/signatures" import { assert, prepareMessage, signalInvalidType } from "@canvas-js/utils" @@ -28,7 +28,7 @@ export const codecs = { * - canvas-action-eip712 * - canvas-session-eip712 */ -export class Secp256k1DelegateSigner implements Signer | Snapshot> { +export class Secp256k1DelegateSigner implements Signer> { public static eip712ActionTypes = { Message: [ { name: "topic", type: "string" }, @@ -63,7 +63,7 @@ export class Secp256k1DelegateSigner implements Signer - public readonly scheme: SignatureScheme | Snapshot> = Secp256k1SignatureScheme + public readonly scheme: SignatureScheme> = Secp256k1SignatureScheme public readonly publicKey: string readonly #wallet: BaseWallet @@ -80,7 +80,7 @@ export class Secp256k1DelegateSigner implements Signer | Snapshot>): Promise { + public async sign(message: Message>): Promise { const { topic, clock, parents, payload } = prepareMessage(message) if (payload.type === "action") { @@ -128,6 +128,8 @@ export class Secp256k1DelegateSigner implements Signer | Snapshot> = { +export const Secp256k1SignatureScheme: SignatureScheme> = { type: "secp256k1", codecs: [codecs.action, codecs.session], - verify(signature: Signature, message: Message | Snapshot>) { + verify(signature: Signature, message: Message>) { const { type, publicKey } = decodeURI(signature.publicKey) assert(type === Secp256k1SignatureScheme.type) @@ -244,6 +246,8 @@ export const Secp256k1SignatureScheme: SignatureScheme { authorizationData: SIWFSessionData, timestamp: number, privateKey: Uint8Array, - ): Promise<{ payload: Session; signer: Signer | Snapshot> }> { + ): Promise<{ payload: Session; signer: Signer> }> { const signer = this.scheme.create({ type: ed25519.type, privateKey }) const did = await this.getDid() diff --git a/packages/cli/src/commands/import.ts b/packages/cli/src/commands/import.ts index 1eab3e324..ce5314d0b 100644 --- a/packages/cli/src/commands/import.ts +++ b/packages/cli/src/commands/import.ts @@ -5,7 +5,7 @@ import type { Argv } from "yargs" import chalk from "chalk" import * as json from "@ipld/dag-json" -import type { Action, Message, Session, Signature, Snapshot } from "@canvas-js/interfaces" +import type { Action, Message, MessageType, Session, Signature, Snapshot } from "@canvas-js/interfaces" import { Canvas } from "@canvas-js/core" import { getContractLocation } from "../utils.js" @@ -52,7 +52,7 @@ export async function handler(args: Args) { const { id, signature, message } = json.parse<{ id: string signature: Signature - message: Message + message: Message }>(line) try { diff --git a/packages/core/src/Canvas.ts b/packages/core/src/Canvas.ts index f62b42380..750e7fa2a 100644 --- a/packages/core/src/Canvas.ts +++ b/packages/core/src/Canvas.ts @@ -4,13 +4,23 @@ import { logger } from "@libp2p/logger" import type pg from "pg" import type { SqlStorage } from "@cloudflare/workers-types" -import { Signature, Action, Session, Message, Snapshot, SessionSigner, SignerCache } from "@canvas-js/interfaces" +import { + Signature, + Action, + Session, + Message, + MessageType, + Snapshot, + SessionSigner, + SignerCache, + Updates, +} from "@canvas-js/interfaces" import { AbstractModelDB, Model, ModelSchema, Effect } from "@canvas-js/modeldb" import { SIWESigner } from "@canvas-js/chain-ethereum" import { AbstractGossipLog, GossipLogEvents, SignedMessage } from "@canvas-js/gossiplog" import type { ServiceMap, NetworkConfig } from "@canvas-js/gossiplog/libp2p" -import { assert, mapValues } from "@canvas-js/utils" +import { assert, mapValues, signalInvalidType } from "@canvas-js/utils" import target from "#target" @@ -49,14 +59,14 @@ export type ActionResult = { id: string; signature: Signature; mes export type ActionAPI = any, Result = any> = (...args: Args) => Promise> -export interface CanvasEvents extends GossipLogEvents { +export interface CanvasEvents extends GossipLogEvents { stop: Event } export type CanvasLogEvent = CustomEvent<{ id: string signature: unknown - message: Message + message: Message }> export type ApplicationData = { @@ -88,7 +98,7 @@ export class Canvas< const signers = new SignerCache(initSigners.length === 0 ? [new SIWESigner()] : initSigners) - const verifySignature = (signature: Signature, message: Message) => { + const verifySignature = (signature: Signature, message: Message) => { const signer = signers.getAll().find((signer) => signer.scheme.codecs.includes(signature.codec)) assert(signer !== undefined, "no matching signer found") return signer.scheme.verify(signature, message) @@ -149,9 +159,9 @@ export class Canvas< let resultCount: number let start: string | undefined = undefined do { - const results: { id: string; message: Message }[] = await db.query<{ + const results: { id: string; message: Message }[] = await db.query<{ id: string - message: Message + message: Message }>("$messages", { limit, select: { id: true, message: true }, @@ -176,6 +186,10 @@ export class Canvas< app.log("indexing user %s (did: %s)", publicKey, did) const record = { did } effects.push({ operation: "set", model: "$dids", value: record }) + } else if (message.payload.type === "updates") { + // TODO: handle updates + } else { + signalInvalidType(message.payload) } start = id } @@ -216,7 +230,7 @@ export class Canvas< private constructor( public readonly signers: SignerCache, - public readonly messageLog: AbstractGossipLog, + public readonly messageLog: AbstractGossipLog, private readonly runtime: Runtime, ) { super() @@ -302,7 +316,7 @@ export class Canvas< await target.listen(this, port, options) } - public async startLibp2p(config: NetworkConfig): Promise>> { + public async startLibp2p(config: NetworkConfig): Promise>> { this.networkConfig = config return await this.messageLog.startLibp2p(config) } @@ -382,7 +396,7 @@ export class Canvas< * Low-level utility method for internal and debugging use. * The normal way to apply actions is to use the `Canvas.actions[name](...)` functions. */ - public async insert(signature: Signature, message: Message): Promise<{ id: string }> { + public async insert(signature: Signature, message: Message): Promise<{ id: string }> { assert(message.topic === this.topic, "invalid message topic") const signedMessage = this.messageLog.encode(signature, message) @@ -390,7 +404,7 @@ export class Canvas< return { id: signedMessage.id } } - public async getMessage(id: string): Promise | null> { + public async getMessage(id: string): Promise | null> { return await this.messageLog.get(id) } @@ -398,7 +412,7 @@ export class Canvas< lowerBound: { id: string; inclusive: boolean } | null = null, upperBound: { id: string; inclusive: boolean } | null = null, options: { reverse?: boolean } = {}, - ): AsyncIterable> { + ): AsyncIterable> { const range: { lt?: string; lte?: string; gt?: string; gte?: string; reverse?: boolean; limit?: number } = {} if (lowerBound) { if (lowerBound.inclusive) range.gte = lowerBound.id diff --git a/packages/core/src/ExecutionContext.ts b/packages/core/src/ExecutionContext.ts index b9d452ea9..5b7f1f3a2 100644 --- a/packages/core/src/ExecutionContext.ts +++ b/packages/core/src/ExecutionContext.ts @@ -2,7 +2,7 @@ import * as cbor from "@ipld/dag-cbor" import { blake3 } from "@noble/hashes/blake3" import { bytesToHex } from "@noble/hashes/utils" -import type { Action, Session, Snapshot } from "@canvas-js/interfaces" +import type { Action, MessageType } from "@canvas-js/interfaces" import { ModelValue, PropertyValue, validateModelValue, updateModelValues, mergeModelValues } from "@canvas-js/modeldb" import { AbstractGossipLog, SignedMessage, MessageId } from "@canvas-js/gossiplog" @@ -21,7 +21,8 @@ export class ExecutionContext { public readonly root: MessageId[] constructor( - public readonly messageLog: AbstractGossipLog, + public readonly messageLog: AbstractGossipLog, + // TODO: why is this just action? should it be `SignedMessage`? public readonly signedMessage: SignedMessage, public readonly address: string, ) { diff --git a/packages/core/src/runtime/AbstractRuntime.ts b/packages/core/src/runtime/AbstractRuntime.ts index dd73ca9b1..2e87f5002 100644 --- a/packages/core/src/runtime/AbstractRuntime.ts +++ b/packages/core/src/runtime/AbstractRuntime.ts @@ -1,14 +1,14 @@ import * as cbor from "@ipld/dag-cbor" import { logger } from "@libp2p/logger" -import type { Action, Session, Snapshot, SignerCache, Awaitable } from "@canvas-js/interfaces" +import type { Action, Session, Snapshot, SignerCache, Awaitable, MessageType } from "@canvas-js/interfaces" import { AbstractModelDB, Effect, ModelSchema } from "@canvas-js/modeldb" import { GossipLogConsumer, MAX_MESSAGE_ID, AbstractGossipLog, SignedMessage } from "@canvas-js/gossiplog" -import { assert } from "@canvas-js/utils" +import { assert, signalInvalidType } from "@canvas-js/utils" import { ExecutionContext, getKeyHash } from "../ExecutionContext.js" -import { isAction, isSession, isSnapshot } from "../utils.js" +import { isAction, isSession, isSnapshot, isUpdates } from "../utils.js" export type EffectRecord = { key: string; value: Uint8Array | null; branch: number; clock: number } @@ -95,28 +95,27 @@ export abstract class AbstractRuntime { this.#db = db } - public getConsumer(): GossipLogConsumer { + public getConsumer(): GossipLogConsumer { const handleSession = this.handleSession.bind(this) const handleAction = this.handleAction.bind(this) const handleSnapshot = this.handleSnapshot.bind(this) - return async function (this: AbstractGossipLog, signedMessage) { + return async function (this: AbstractGossipLog, signedMessage) { if (isSession(signedMessage)) { return await handleSession(signedMessage) } else if (isAction(signedMessage)) { return await handleAction(signedMessage, this) } else if (isSnapshot(signedMessage)) { return await handleSnapshot(signedMessage, this) + } else if (isUpdates(signedMessage)) { + // TODO: handle updates } else { throw new Error("invalid message payload type") } } } - private async handleSnapshot( - signedMessage: SignedMessage, - messageLog: AbstractGossipLog, - ) { + private async handleSnapshot(signedMessage: SignedMessage, messageLog: AbstractGossipLog) { const { models, effects } = signedMessage.message.payload const messages = await messageLog.getMessages() @@ -167,10 +166,7 @@ export abstract class AbstractRuntime { await this.db.apply(effects) } - private async handleAction( - signedMessage: SignedMessage, - messageLog: AbstractGossipLog, - ) { + private async handleAction(signedMessage: SignedMessage, messageLog: AbstractGossipLog) { const { id, signature, message } = signedMessage const { did, name, context } = message.payload diff --git a/packages/core/src/schema.ts b/packages/core/src/schema.ts index 922fcf20a..f21aba577 100644 --- a/packages/core/src/schema.ts +++ b/packages/core/src/schema.ts @@ -1,7 +1,7 @@ import { fromDSL } from "@ipld/schema/from-dsl.js" import { create } from "@ipld/schema/typed.js" -import type { Action, Session, Snapshot } from "@canvas-js/interfaces" +import type { Action, MessageType, Session, Snapshot, Updates } from "@canvas-js/interfaces" const schema = ` type ActionContext struct { @@ -39,10 +39,22 @@ type Snapshot struct { effects [SnapshotEffect] } +type Update = { + model: string + key: string + diff: Uint8Array +} + +type Updates = { + type: "updates" + updates: Update[] +} + type Payload union { | Action "action" | Session "session" | Snapshot "snapshot" + | Updates "updates" } representation inline { discriminantKey "type" } @@ -50,7 +62,12 @@ type Payload union { const { toTyped } = create(fromDSL(schema), "Payload") -export function validatePayload(payload: unknown): payload is Action | Session | Snapshot { - const result = toTyped(payload) as { Action: Omit } | { Session: Omit } | undefined +export function validatePayload(payload: unknown): payload is MessageType { + const result = toTyped(payload) as + | { Action: Omit } + | { Session: Omit } + | { Snapshot: Omit } + | { Updates: Omit } + | undefined return result !== undefined } diff --git a/packages/core/src/targets/interface.ts b/packages/core/src/targets/interface.ts index c68ebeacb..d09954036 100644 --- a/packages/core/src/targets/interface.ts +++ b/packages/core/src/targets/interface.ts @@ -1,6 +1,6 @@ import type pg from "pg" -import type { Action, Session, Snapshot } from "@canvas-js/interfaces" +import type { MessageType } from "@canvas-js/interfaces" import type { AbstractGossipLog, GossipLogInit } from "@canvas-js/gossiplog" import type { Canvas } from "@canvas-js/core" import type { SqlStorage } from "@cloudflare/workers-types" @@ -8,8 +8,8 @@ import type { SqlStorage } from "@cloudflare/workers-types" export interface PlatformTarget { openGossipLog: ( location: { path: string | pg.ConnectionConfig | SqlStorage | null; topic: string; clear?: boolean }, - init: GossipLogInit, - ) => Promise> + init: GossipLogInit, + ) => Promise> listen: (app: Canvas, port: number, options?: { signal?: AbortSignal }) => Promise } diff --git a/packages/core/src/utils.ts b/packages/core/src/utils.ts index c362bbc10..9c12a4ea9 100644 --- a/packages/core/src/utils.ts +++ b/packages/core/src/utils.ts @@ -4,21 +4,21 @@ import { blake3 } from "@noble/hashes/blake3" import { utf8ToBytes } from "@noble/hashes/utils" import { base64 } from "multiformats/bases/base64" -import { Action, Session, Snapshot } from "@canvas-js/interfaces" +import { Action, MessageType, Session, Snapshot, Updates } from "@canvas-js/interfaces" import { SignedMessage } from "@canvas-js/gossiplog" import { PrimaryKeyValue } from "@canvas-js/modeldb" -export const isAction = ( - signedMessage: SignedMessage, -): signedMessage is SignedMessage => signedMessage.message.payload.type === "action" +export const isAction = (signedMessage: SignedMessage): signedMessage is SignedMessage => + signedMessage.message.payload.type === "action" -export const isSession = ( - signedMessage: SignedMessage, -): signedMessage is SignedMessage => signedMessage.message.payload.type === "session" +export const isSession = (signedMessage: SignedMessage): signedMessage is SignedMessage => + signedMessage.message.payload.type === "session" -export const isSnapshot = ( - signedMessage: SignedMessage, -): signedMessage is SignedMessage => signedMessage.message.payload.type === "snapshot" +export const isSnapshot = (signedMessage: SignedMessage): signedMessage is SignedMessage => + signedMessage.message.payload.type === "snapshot" + +export const isUpdates = (signedMessage: SignedMessage): signedMessage is SignedMessage => + signedMessage.message.payload.type === "updates" export const topicPattern = /^[a-zA-Z0-9.-]+$/ diff --git a/packages/interfaces/src/MessageType.ts b/packages/interfaces/src/MessageType.ts new file mode 100644 index 000000000..3e47cf939 --- /dev/null +++ b/packages/interfaces/src/MessageType.ts @@ -0,0 +1,6 @@ +import { Action } from "./Action.js" +import { Session } from "./Session.js" +import { Snapshot } from "./Snapshot.js" +import { Updates } from "./Updates.js" + +export type MessageType = Action | Session | Snapshot | Updates diff --git a/packages/interfaces/src/SessionSigner.ts b/packages/interfaces/src/SessionSigner.ts index e627c8785..570b66ef7 100644 --- a/packages/interfaces/src/SessionSigner.ts +++ b/packages/interfaces/src/SessionSigner.ts @@ -1,8 +1,7 @@ import type { SignatureScheme, Signer } from "./Signer.js" import type { Session } from "./Session.js" -import type { Action } from "./Action.js" -import type { Snapshot } from "./Snapshot.js" import type { Awaitable } from "./Awaitable.js" +import { MessageType } from "./MessageType.js" export type DidIdentifier = `did:${string}` @@ -17,7 +16,7 @@ export interface AbstractSessionData { } export interface SessionSigner { - scheme: SignatureScheme | Snapshot> + scheme: SignatureScheme> match: (did: DidIdentifier) => boolean getDid: () => Awaitable @@ -30,13 +29,11 @@ export interface SessionSigner { options?: { did?: DidIdentifier } | { address: string }, ) => Awaitable<{ payload: Session - signer: Signer | Snapshot> + signer: Signer> } | null> - newSession: ( - topic: string, - ) => Awaitable<{ + newSession: (topic: string) => Awaitable<{ payload: Session - signer: Signer | Snapshot> + signer: Signer> }> /** diff --git a/packages/interfaces/src/Updates.ts b/packages/interfaces/src/Updates.ts new file mode 100644 index 000000000..c7b1ac963 --- /dev/null +++ b/packages/interfaces/src/Updates.ts @@ -0,0 +1,10 @@ +type Update = { + model: string + key: string + diff: Uint8Array +} + +export type Updates = { + type: "updates" + updates: Update[] +} diff --git a/packages/interfaces/src/index.ts b/packages/interfaces/src/index.ts index 75b2e49e5..1e6d125db 100644 --- a/packages/interfaces/src/index.ts +++ b/packages/interfaces/src/index.ts @@ -7,3 +7,5 @@ export * from "./Message.js" export * from "./Session.js" export * from "./Snapshot.js" export * from "./Awaitable.js" +export * from "./Updates.js" +export * from "./MessageType.js" diff --git a/packages/signatures/src/AbstractSessionSigner.ts b/packages/signatures/src/AbstractSessionSigner.ts index ee6d80b94..14134b472 100644 --- a/packages/signatures/src/AbstractSessionSigner.ts +++ b/packages/signatures/src/AbstractSessionSigner.ts @@ -3,15 +3,14 @@ import { Logger, logger } from "@libp2p/logger" import { randomBytes, bytesToHex } from "@noble/hashes/utils" import type { - Action, Session, - Snapshot, Signer, Awaitable, SignatureScheme, AbstractSessionData, SessionSigner, DidIdentifier, + MessageType, } from "@canvas-js/interfaces" import target from "#target" @@ -32,11 +31,11 @@ export abstract class AbstractSessionSigner< protected readonly log: Logger protected readonly privkeySeed: string - #cache = new Map | Snapshot> }>() + #cache = new Map> }>() public constructor( public readonly key: string, - public readonly scheme: SignatureScheme | Snapshot>, + public readonly scheme: SignatureScheme>, options: AbstractSessionSignerOptions = {}, ) { this.log = logger(`canvas:signer:${key}`) @@ -71,8 +70,8 @@ export abstract class AbstractSessionSigner< * or by using a provided AuthorizationData and timestamp (for services like Farcaster). */ public async newSession( - topic: string - ): Promise<{ payload: Session; signer: Signer | Snapshot> }> { + topic: string, + ): Promise<{ payload: Session; signer: Signer> }> { const signer = this.scheme.create() const did = await this.getDid() @@ -131,7 +130,7 @@ export abstract class AbstractSessionSigner< options: { did?: string; address?: string } = {}, ): Promise<{ payload: Session - signer: Signer | Snapshot> + signer: Signer> } | null> { let did if (options.address) { From 944181bb0307c7a6b62b5f1247a8231b02eaa61b Mon Sep 17 00:00:00 2001 From: Bob Webb Date: Wed, 12 Feb 2025 15:39:54 +0100 Subject: [PATCH 02/36] fix typo --- packages/core/src/Canvas.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/core/src/Canvas.ts b/packages/core/src/Canvas.ts index 750e7fa2a..ad66d6de7 100644 --- a/packages/core/src/Canvas.ts +++ b/packages/core/src/Canvas.ts @@ -230,7 +230,7 @@ export class Canvas< private constructor( public readonly signers: SignerCache, - public readonly messageLog: AbstractGossipLog, + public readonly messageLog: AbstractGossipLog, private readonly runtime: Runtime, ) { super() From 61475f1eeab4dcace807b7dcccfc5576a394526a Mon Sep 17 00:00:00 2001 From: Bob Webb Date: Wed, 12 Feb 2025 16:35:39 +0100 Subject: [PATCH 03/36] add yjsInsert and yjsDelete methods to runtime, when these are called compute diffs on the yjs document --- packages/core/src/ExecutionContext.ts | 27 ++++++++++ packages/core/src/runtime/AbstractRuntime.ts | 54 +++++++++++++++++++- packages/core/src/runtime/ContractRuntime.ts | 19 +++++++ packages/core/src/runtime/FunctionRuntime.ts | 10 ++++ packages/core/src/types.ts | 20 +++++++- 5 files changed, 126 insertions(+), 4 deletions(-) diff --git a/packages/core/src/ExecutionContext.ts b/packages/core/src/ExecutionContext.ts index 5b7f1f3a2..2e3ca2df4 100644 --- a/packages/core/src/ExecutionContext.ts +++ b/packages/core/src/ExecutionContext.ts @@ -1,6 +1,7 @@ import * as cbor from "@ipld/dag-cbor" import { blake3 } from "@noble/hashes/blake3" import { bytesToHex } from "@noble/hashes/utils" +import * as Y from "yjs" import type { Action, MessageType } from "@canvas-js/interfaces" @@ -9,6 +10,17 @@ import { AbstractGossipLog, SignedMessage, MessageId } from "@canvas-js/gossiplo import { assert, mapValues } from "@canvas-js/utils" export const getKeyHash = (key: string) => bytesToHex(blake3(key, { dkLen: 16 })) +type YjsCallInsert = { + call: "insert" + index: number + content: string +} +type YjsCallDelete = { + call: "delete" + index: number + length: number +} +export type YjsCall = YjsCallInsert | YjsCallDelete export class ExecutionContext { // // recordId -> { version, value } @@ -18,6 +30,7 @@ export class ExecutionContext { // public readonly writes: Record = {} public readonly modelEntries: Record> + public readonly yjsCalls: Record> = {} public readonly root: MessageId[] constructor( @@ -138,4 +151,18 @@ export class ExecutionContext { validateModelValue(this.db.models[model], result) this.modelEntries[model][key] = result } + + public async getYDoc(modelName: string, id: string): Promise { + const existingStateEntries = await this.db.query<{ id: string; content: Uint8Array }>(`${modelName}:state`, { + where: { id: id }, + limit: 1, + }) + if (existingStateEntries.length > 0) { + const doc = new Y.Doc() + Y.applyUpdate(doc, existingStateEntries[0].content) + return doc + } else { + return null + } + } } diff --git a/packages/core/src/runtime/AbstractRuntime.ts b/packages/core/src/runtime/AbstractRuntime.ts index 2e87f5002..a7c8689d7 100644 --- a/packages/core/src/runtime/AbstractRuntime.ts +++ b/packages/core/src/runtime/AbstractRuntime.ts @@ -1,11 +1,12 @@ import * as cbor from "@ipld/dag-cbor" import { logger } from "@libp2p/logger" +import * as Y from "yjs" import type { Action, Session, Snapshot, SignerCache, Awaitable, MessageType } from "@canvas-js/interfaces" import { AbstractModelDB, Effect, ModelSchema } from "@canvas-js/modeldb" import { GossipLogConsumer, MAX_MESSAGE_ID, AbstractGossipLog, SignedMessage } from "@canvas-js/gossiplog" -import { assert, signalInvalidType } from "@canvas-js/utils" +import { assert } from "@canvas-js/utils" import { ExecutionContext, getKeyHash } from "../ExecutionContext.js" import { isAction, isSession, isSnapshot, isUpdates } from "../utils.js" @@ -63,8 +64,32 @@ export abstract class AbstractRuntime { } satisfies ModelSchema protected static getModelSchema(schema: ModelSchema): ModelSchema { + const outputSchema: ModelSchema = {} + for (const [modelName, modelSchema] of Object.entries(schema)) { + // @ts-ignore + if (modelSchema.content === "yjs-doc") { + if ( + Object.entries(modelSchema).length !== 2 && + // @ts-ignore + modelSchema.id !== "primary" + ) { + // not valid + throw new Error("yjs-doc tables must have two columns, one of which is 'id'") + } else { + // this table stores the current state of the Yjs document + // we just need one entry per document because updates are commutative + outputSchema[`${modelName}:state`] = { + id: "primary", + content: "bytes", + } + } + } else { + outputSchema[modelName] = modelSchema + } + } + return { - ...schema, + ...outputSchema, ...AbstractRuntime.sessionsModel, ...AbstractRuntime.actionsModel, ...AbstractRuntime.effectsModel, @@ -238,6 +263,31 @@ export abstract class AbstractRuntime { } } + const diffs: { model: string; key: string; diff: Uint8Array }[] = [] + for (const [model, modelCalls] of Object.entries(executionContext.yjsCalls)) { + for (const [key, calls] of Object.entries(modelCalls)) { + const doc = (await executionContext.getYDoc(model, key)) || new Y.Doc() + // get the initial state of the document + const beforeState = Y.encodeStateAsUpdate(doc) + for (const call of calls) { + if (call.call === "insert") { + doc.getText().insert(call.index, call.content) + } else if (call.call === "delete") { + doc.getText().delete(call.index, call.length) + } else { + throw new Error("unexpected call type") + } + } + // diff the document with the initial state + const afterState = Y.encodeStateAsUpdate(doc) + const diff = Y.diffUpdate(afterState, Y.encodeStateVectorFromUpdate(beforeState)) + diffs.push({ model, key, diff }) + } + } + + // TODO: we want the diff to trigger another message - how should we do this? + // add another return value to the consumer + this.log("applying effects %O", effects) try { diff --git a/packages/core/src/runtime/ContractRuntime.ts b/packages/core/src/runtime/ContractRuntime.ts index 8ceeacfc4..fd245a38a 100644 --- a/packages/core/src/runtime/ContractRuntime.ts +++ b/packages/core/src/runtime/ContractRuntime.ts @@ -134,6 +134,25 @@ export class ContractRuntime extends AbstractRuntime { const key = vm.context.getString(keyHandle) this.context.deleteModelValue(model, key) }), + + yjsInsert: vm.context.newFunction("yjsInsert", (modelHandle, keyHandle, indexHandle, contentHandle) => { + const model = vm.context.getString(modelHandle) + const key = vm.context.getString(keyHandle) + const index = vm.context.getNumber(indexHandle) + const content = vm.context.getString(contentHandle) + this.context.yjsCalls[model] ||= {} + this.context.yjsCalls[model][key] ||= [] + this.context.yjsCalls[model][key].push({ call: "insert", index, content }) + }), + yjsDelete: vm.context.newFunction("yjsDelete", (modelHandle, keyHandle, indexHandle, lengthHandle) => { + const model = vm.context.getString(modelHandle) + const key = vm.context.getString(keyHandle) + const index = vm.context.getNumber(indexHandle) + const length = vm.context.getNumber(lengthHandle) + this.context.yjsCalls[model] ||= {} + this.context.yjsCalls[model][key] ||= [] + this.context.yjsCalls[model][key].push({ call: "delete", index, length }) + }), }) .consume(vm.cache) } diff --git a/packages/core/src/runtime/FunctionRuntime.ts b/packages/core/src/runtime/FunctionRuntime.ts index 7306e2a6b..87f7121da 100644 --- a/packages/core/src/runtime/FunctionRuntime.ts +++ b/packages/core/src/runtime/FunctionRuntime.ts @@ -173,6 +173,16 @@ export class FunctionRuntime extends AbstractRuntim this.releaseLock() } }, + yjsInsert: async (model: string, key: string, index: number, content: string) => { + this.context.yjsCalls[model] ||= {} + this.context.yjsCalls[model][key] ||= [] + this.context.yjsCalls[model][key].push({ call: "insert", index, content }) + }, + yjsDelete: async (model: string, key: string, index: number, length: number) => { + this.context.yjsCalls[model] ||= {} + this.context.yjsCalls[model][key] ||= [] + this.context.yjsCalls[model][key].push({ call: "delete", index, length }) + }, } } diff --git a/packages/core/src/types.ts b/packages/core/src/types.ts index 24d3b3acc..3b14acb82 100644 --- a/packages/core/src/types.ts +++ b/packages/core/src/types.ts @@ -17,14 +17,18 @@ export type ActionImplementation< ModelsT extends ModelSchema = ModelSchema, Args extends Array = any, Result = any, -> = (this: ActionContext>, db: ModelAPI>, ...args: Args) => Awaitable +> = ( + this: ActionContext>, + db: ModelAPI>, + ...args: Args +) => Awaitable export type Chainable> = Promise & { link: ( model: T, primaryKey: string, through?: { through: string }, - ) => Promise, + ) => Promise unlink: ( model: T, primaryKey: string, @@ -40,6 +44,18 @@ export type ModelAPI> = { update: (model: T, value: Partial) => Chainable merge: (model: T, value: Partial) => Chainable delete: (model: T, key: string) => Promise + yjsInsert: ( + model: T, + key: string, + index: number, + content: string, + ) => Promise + yjsDelete: ( + model: T, + key: string, + index: number, + length: number, + ) => Promise } export type ActionContext> = { From e10e39ef1eda0499d8819778d7dbcc97a5a05ed3 Mon Sep 17 00:00:00 2001 From: Bob Webb Date: Wed, 12 Feb 2025 18:13:09 +0100 Subject: [PATCH 04/36] remove signalInvalidType call --- packages/core/src/Canvas.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/core/src/Canvas.ts b/packages/core/src/Canvas.ts index 03ae57c09..025f44e09 100644 --- a/packages/core/src/Canvas.ts +++ b/packages/core/src/Canvas.ts @@ -10,7 +10,7 @@ import { SIWESigner } from "@canvas-js/chain-ethereum" import { AbstractGossipLog, GossipLogEvents, SignedMessage } from "@canvas-js/gossiplog" import type { ServiceMap, NetworkConfig } from "@canvas-js/gossiplog/libp2p" -import { assert, mapValues, signalInvalidType } from "@canvas-js/utils" +import { assert, mapValues } from "@canvas-js/utils" import target from "#target" @@ -178,8 +178,6 @@ export class Canvas< effects.push({ operation: "set", model: "$dids", value: record }) } else if (message.payload.type === "updates") { // TODO: handle updates - } else { - signalInvalidType(message.payload) } start = id } From 7c70cf68b52458d96fbd37f4247a29d716d5db1e Mon Sep 17 00:00:00 2001 From: Bob Webb Date: Thu, 13 Feb 2025 13:46:25 +0100 Subject: [PATCH 05/36] return an extra 'additionalMessages' value from the gossiplog apply method --- packages/core/src/ExecutionContext.ts | 25 ++++++++++++ packages/core/src/runtime/AbstractRuntime.ts | 40 +++++--------------- packages/gossiplog/src/AbstractGossipLog.ts | 15 +++++--- packages/gossiplog/test/result.test.ts | 2 +- 4 files changed, 45 insertions(+), 37 deletions(-) diff --git a/packages/core/src/ExecutionContext.ts b/packages/core/src/ExecutionContext.ts index 12a347c51..c561f6521 100644 --- a/packages/core/src/ExecutionContext.ts +++ b/packages/core/src/ExecutionContext.ts @@ -164,4 +164,29 @@ export class ExecutionContext { return null } } + + public async generateAdditionalMessages(): Promise { + const updates = [] + for (const [model, modelCalls] of Object.entries(this.yjsCalls)) { + for (const [key, calls] of Object.entries(modelCalls)) { + const doc = (await this.getYDoc(model, key)) || new Y.Doc() + // get the initial state of the document + const beforeState = Y.encodeStateAsUpdate(doc) + for (const call of calls) { + if (call.call === "insert") { + doc.getText().insert(call.index, call.content) + } else if (call.call === "delete") { + doc.getText().delete(call.index, call.length) + } else { + throw new Error("unexpected call type") + } + } + // diff the document with the initial state + const afterState = Y.encodeStateAsUpdate(doc) + const diff = Y.diffUpdate(afterState, Y.encodeStateVectorFromUpdate(beforeState)) + updates.push({ model, key, diff }) + } + } + return [{ type: "updates", updates }] + } } diff --git a/packages/core/src/runtime/AbstractRuntime.ts b/packages/core/src/runtime/AbstractRuntime.ts index a7c8689d7..de2cc927c 100644 --- a/packages/core/src/runtime/AbstractRuntime.ts +++ b/packages/core/src/runtime/AbstractRuntime.ts @@ -1,6 +1,5 @@ import * as cbor from "@ipld/dag-cbor" import { logger } from "@libp2p/logger" -import * as Y from "yjs" import type { Action, Session, Snapshot, SignerCache, Awaitable, MessageType } from "@canvas-js/interfaces" @@ -125,11 +124,11 @@ export abstract class AbstractRuntime { const handleAction = this.handleAction.bind(this) const handleSnapshot = this.handleSnapshot.bind(this) - return async function (this: AbstractGossipLog, signedMessage) { + return async function (this: AbstractGossipLog, signedMessage, isAppend) { if (isSession(signedMessage)) { return await handleSession(signedMessage) } else if (isAction(signedMessage)) { - return await handleAction(signedMessage, this) + return await handleAction(signedMessage, isAppend, this) } else if (isSnapshot(signedMessage)) { return await handleSnapshot(signedMessage, this) } else if (isUpdates(signedMessage)) { @@ -191,7 +190,11 @@ export abstract class AbstractRuntime { await this.db.apply(effects) } - private async handleAction(signedMessage: SignedMessage, messageLog: AbstractGossipLog) { + private async handleAction( + signedMessage: SignedMessage, + isAppend: boolean, + messageLog: AbstractGossipLog, + ) { const { id, signature, message } = signedMessage const { did, name, context } = message.payload @@ -263,31 +266,6 @@ export abstract class AbstractRuntime { } } - const diffs: { model: string; key: string; diff: Uint8Array }[] = [] - for (const [model, modelCalls] of Object.entries(executionContext.yjsCalls)) { - for (const [key, calls] of Object.entries(modelCalls)) { - const doc = (await executionContext.getYDoc(model, key)) || new Y.Doc() - // get the initial state of the document - const beforeState = Y.encodeStateAsUpdate(doc) - for (const call of calls) { - if (call.call === "insert") { - doc.getText().insert(call.index, call.content) - } else if (call.call === "delete") { - doc.getText().delete(call.index, call.length) - } else { - throw new Error("unexpected call type") - } - } - // diff the document with the initial state - const afterState = Y.encodeStateAsUpdate(doc) - const diff = Y.diffUpdate(afterState, Y.encodeStateVectorFromUpdate(beforeState)) - diffs.push({ model, key, diff }) - } - } - - // TODO: we want the diff to trigger another message - how should we do this? - // add another return value to the consumer - this.log("applying effects %O", effects) try { @@ -299,6 +277,8 @@ export abstract class AbstractRuntime { throw err } - return result + const additionalMessages = isAppend ? await executionContext.generateAdditionalMessages() : [] + + return { result, additionalMessages } } } diff --git a/packages/gossiplog/src/AbstractGossipLog.ts b/packages/gossiplog/src/AbstractGossipLog.ts index 902a8c9e6..fc968b3d4 100644 --- a/packages/gossiplog/src/AbstractGossipLog.ts +++ b/packages/gossiplog/src/AbstractGossipLog.ts @@ -25,7 +25,8 @@ import { gossiplogTopicPattern } from "./utils.js" export type GossipLogConsumer = ( this: AbstractGossipLog, signedMessage: SignedMessage, -) => Awaitable + isAppend: boolean, +) => Awaitable<{ result: Result; additionalMessages?: Payload[] } | void> export interface GossipLogInit { topic: string @@ -127,7 +128,7 @@ export abstract class AbstractGossipLog extends const { signature, message, branch } = record const signedMessage = this.encode(signature, message, { branch }) assert(signedMessage.id === id) - await this.#apply.apply(this, [signedMessage]) + await this.#apply.apply(this, [signedMessage, false]) } }) } @@ -253,7 +254,7 @@ export abstract class AbstractGossipLog extends const signedMessage = this.encode(signature, message) this.log("appending message %s at clock %d with parents %o", signedMessage.id, clock, parents) - const applyResult = await this.apply(txn, signedMessage) + const applyResult = await this.apply(txn, signedMessage, true) root = applyResult.root heads = applyResult.heads @@ -291,7 +292,7 @@ export abstract class AbstractGossipLog extends return null } - return await this.apply(txn, signedMessage) + return await this.apply(txn, signedMessage, false) }) if (result !== null) { @@ -304,6 +305,7 @@ export abstract class AbstractGossipLog extends private async apply( txn: ReadWriteTransaction, signedMessage: SignedMessage, + isAppend: boolean, ): Promise<{ root: Node; heads: string[]; result: Result }> { const { id, signature, message, key, value } = signedMessage this.log.trace("applying %s %O", id, message) @@ -321,7 +323,7 @@ export abstract class AbstractGossipLog extends const branch = await this.getBranch(id, parentMessageRecords) signedMessage.branch = branch - const result = await this.#apply.apply(this, [signedMessage]) + const result = await this.#apply.apply(this, [signedMessage, isAppend]) const hash = toString(hashEntry(key, value), "hex") @@ -359,7 +361,8 @@ export abstract class AbstractGossipLog extends this.dispatchEvent(new CustomEvent("message", { detail: signedMessage })) const root = txn.getRoot() - return { root, heads, result } + // TODO: more elegant handling of `Result` type + return { root, heads, result: result ? result.result : (undefined as any) } } private async newBranch() { diff --git a/packages/gossiplog/test/result.test.ts b/packages/gossiplog/test/result.test.ts index fa5700c09..d3d1592d9 100644 --- a/packages/gossiplog/test/result.test.ts +++ b/packages/gossiplog/test/result.test.ts @@ -7,7 +7,7 @@ import { ed25519 } from "@canvas-js/signatures" import type { GossipLogConsumer } from "@canvas-js/gossiplog" import { testPlatforms } from "./utils.js" -const apply: GossipLogConsumer = ({ message: { payload } }) => bytesToHex(sha256(payload)) +const apply: GossipLogConsumer = ({ message: { payload } }) => ({ result: bytesToHex(sha256(payload)) }) testPlatforms( "get apply result", From 8b35b93596f32d5da33b05e7776e32d2cc838989 Mon Sep 17 00:00:00 2001 From: Bob Webb Date: Thu, 13 Feb 2025 14:22:57 +0100 Subject: [PATCH 06/36] append additional messages if they have been returned by apply --- packages/gossiplog/src/AbstractGossipLog.ts | 32 ++++++++++----------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/packages/gossiplog/src/AbstractGossipLog.ts b/packages/gossiplog/src/AbstractGossipLog.ts index fc968b3d4..b83b9761f 100644 --- a/packages/gossiplog/src/AbstractGossipLog.ts +++ b/packages/gossiplog/src/AbstractGossipLog.ts @@ -235,11 +235,7 @@ export abstract class AbstractGossipLog extends payload: T, { signer = this.signer }: { signer?: Signer } = {}, ): Promise & { result: Result }> { - let root: Node | null = null - let heads: string[] | null = null - let result: Result | undefined = undefined - - const signedMessage = await this.tree.write(async (txn) => { + const { signedMessage, applyResult } = await this.tree.write(async (txn) => { const [clock, parents] = await this.getClock() const message: Message = { @@ -256,17 +252,22 @@ export abstract class AbstractGossipLog extends const applyResult = await this.apply(txn, signedMessage, true) - root = applyResult.root - heads = applyResult.heads - result = applyResult.result - - return signedMessage + return { signedMessage, applyResult } }) - assert(root !== null && heads !== null, "failed to commit transaction") - this.dispatchEvent(new CustomEvent("commit", { detail: { root, heads } })) + assert(applyResult.root !== null && applyResult.heads !== null, "failed to commit transaction") + this.dispatchEvent(new CustomEvent("commit", { detail: { root: applyResult.root, heads: applyResult.heads } })) + + signedMessage.result = applyResult.result?.result + + // append the additional messages + const additionalMessages = applyResult.result?.additionalMessages + if (additionalMessages) { + for (const additionalMessage of additionalMessages) { + await this.append(additionalMessage, { signer }) + } + } - signedMessage.result = result return signedMessage as SignedMessage & { result: Result } } @@ -306,7 +307,7 @@ export abstract class AbstractGossipLog extends txn: ReadWriteTransaction, signedMessage: SignedMessage, isAppend: boolean, - ): Promise<{ root: Node; heads: string[]; result: Result }> { + ): Promise<{ root: Node; heads: string[]; result: { result: Result; additionalMessages?: Payload[] } | void }> { const { id, signature, message, key, value } = signedMessage this.log.trace("applying %s %O", id, message) @@ -361,8 +362,7 @@ export abstract class AbstractGossipLog extends this.dispatchEvent(new CustomEvent("message", { detail: signedMessage })) const root = txn.getRoot() - // TODO: more elegant handling of `Result` type - return { root, heads, result: result ? result.result : (undefined as any) } + return { root, heads, result } } private async newBranch() { From b2a87bd6370bb34c34fd8109e2beea7caca83b87 Mon Sep 17 00:00:00 2001 From: Bob Webb Date: Thu, 13 Feb 2025 15:05:06 +0100 Subject: [PATCH 07/36] implement handleUpdates --- packages/core/src/runtime/AbstractRuntime.ts | 29 ++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/packages/core/src/runtime/AbstractRuntime.ts b/packages/core/src/runtime/AbstractRuntime.ts index de2cc927c..b9d660eca 100644 --- a/packages/core/src/runtime/AbstractRuntime.ts +++ b/packages/core/src/runtime/AbstractRuntime.ts @@ -1,7 +1,8 @@ import * as cbor from "@ipld/dag-cbor" import { logger } from "@libp2p/logger" +import * as Y from "yjs" -import type { Action, Session, Snapshot, SignerCache, Awaitable, MessageType } from "@canvas-js/interfaces" +import type { Action, Session, Snapshot, SignerCache, Awaitable, MessageType, Updates } from "@canvas-js/interfaces" import { AbstractModelDB, Effect, ModelSchema } from "@canvas-js/modeldb" import { GossipLogConsumer, MAX_MESSAGE_ID, AbstractGossipLog, SignedMessage } from "@canvas-js/gossiplog" @@ -123,6 +124,7 @@ export abstract class AbstractRuntime { const handleSession = this.handleSession.bind(this) const handleAction = this.handleAction.bind(this) const handleSnapshot = this.handleSnapshot.bind(this) + const handleUpdates = this.handleUpdates.bind(this) return async function (this: AbstractGossipLog, signedMessage, isAppend) { if (isSession(signedMessage)) { @@ -132,7 +134,7 @@ export abstract class AbstractRuntime { } else if (isSnapshot(signedMessage)) { return await handleSnapshot(signedMessage, this) } else if (isUpdates(signedMessage)) { - // TODO: handle updates + return await handleUpdates(signedMessage, this) } else { throw new Error("invalid message payload type") } @@ -281,4 +283,27 @@ export abstract class AbstractRuntime { return { result, additionalMessages } } + + private async handleUpdates(signedMessage: SignedMessage, messageLog: AbstractGossipLog) { + const effects: Effect[] = [] + for (const { model, key, diff } of signedMessage.message.payload.updates) { + const existingStateEntries = await this.db.query<{ id: string; content: Uint8Array }>(`${model}:state`, { + where: { id: key }, + limit: 1, + }) + + // apply the diff to the doc + const doc = new Y.Doc() + Y.applyUpdate(doc, existingStateEntries[0].content) + Y.applyUpdate(doc, diff) + const newContent = Y.encodeStateAsUpdate(doc) + + effects.push({ + model: `${model}:state`, + operation: "set", + value: { id: key, content: newContent }, + }) + } + await this.db.apply(effects) + } } From e863949fc48ca93c5d8d04a4b56177c6fc9d79b3 Mon Sep 17 00:00:00 2001 From: Bob Webb Date: Mon, 3 Mar 2025 15:15:24 +0100 Subject: [PATCH 08/36] don't return additional updates in the gossiplog apply function, store them in a field on the AbstractRuntime object --- packages/core/src/Canvas.ts | 10 +++++++- packages/core/src/ExecutionContext.ts | 4 ++-- packages/core/src/runtime/AbstractRuntime.ts | 16 ++++++------- packages/gossiplog/src/AbstractGossipLog.ts | 24 +++++--------------- packages/gossiplog/test/result.test.ts | 2 +- 5 files changed, 25 insertions(+), 31 deletions(-) diff --git a/packages/core/src/Canvas.ts b/packages/core/src/Canvas.ts index 4dffdcc5b..927fd9996 100644 --- a/packages/core/src/Canvas.ts +++ b/packages/core/src/Canvas.ts @@ -286,6 +286,14 @@ export class Canvas< this.log("applied action %s", id) + for (const additionalUpdate of this.runtime.additionalUpdates.get(id) || []) { + await this.messageLog.append(additionalUpdate) + } + + this.runtime.additionalUpdates.delete(id) + + this.log("applied additional updates for action %s", id) + return { id, signature, message, result } } @@ -370,7 +378,7 @@ export class Canvas< public async getApplicationData(): Promise { const models = Object.fromEntries(Object.entries(this.db.models).filter(([name]) => !name.startsWith("$"))) - const root = (await this.messageLog.tree.read((txn) => txn.getRoot())) + const root = await this.messageLog.tree.read((txn) => txn.getRoot()) const heads = await this.db.query<{ id: string }>("$heads").then((records) => records.map((record) => record.id)) return { diff --git a/packages/core/src/ExecutionContext.ts b/packages/core/src/ExecutionContext.ts index c561f6521..c312497ba 100644 --- a/packages/core/src/ExecutionContext.ts +++ b/packages/core/src/ExecutionContext.ts @@ -3,7 +3,7 @@ import { blake3 } from "@noble/hashes/blake3" import { bytesToHex } from "@noble/hashes/utils" import * as Y from "yjs" -import type { Action, MessageType } from "@canvas-js/interfaces" +import type { Action, MessageType, Updates } from "@canvas-js/interfaces" import { ModelValue, PropertyValue, validateModelValue, updateModelValues, mergeModelValues } from "@canvas-js/modeldb" import { AbstractGossipLog, SignedMessage, MessageId } from "@canvas-js/gossiplog" @@ -165,7 +165,7 @@ export class ExecutionContext { } } - public async generateAdditionalMessages(): Promise { + public async generateAdditionalUpdates(): Promise { const updates = [] for (const [model, modelCalls] of Object.entries(this.yjsCalls)) { for (const [key, calls] of Object.entries(modelCalls)) { diff --git a/packages/core/src/runtime/AbstractRuntime.ts b/packages/core/src/runtime/AbstractRuntime.ts index b9d660eca..af0f41e22 100644 --- a/packages/core/src/runtime/AbstractRuntime.ts +++ b/packages/core/src/runtime/AbstractRuntime.ts @@ -102,6 +102,8 @@ export abstract class AbstractRuntime { public abstract readonly schema: ModelSchema public abstract readonly actionNames: string[] + public readonly additionalUpdates = new Map() + protected readonly log = logger("canvas:runtime") #db: AbstractModelDB | null = null @@ -126,11 +128,11 @@ export abstract class AbstractRuntime { const handleSnapshot = this.handleSnapshot.bind(this) const handleUpdates = this.handleUpdates.bind(this) - return async function (this: AbstractGossipLog, signedMessage, isAppend) { + return async function (this: AbstractGossipLog, signedMessage) { if (isSession(signedMessage)) { return await handleSession(signedMessage) } else if (isAction(signedMessage)) { - return await handleAction(signedMessage, isAppend, this) + return await handleAction(signedMessage, this) } else if (isSnapshot(signedMessage)) { return await handleSnapshot(signedMessage, this) } else if (isUpdates(signedMessage)) { @@ -192,11 +194,7 @@ export abstract class AbstractRuntime { await this.db.apply(effects) } - private async handleAction( - signedMessage: SignedMessage, - isAppend: boolean, - messageLog: AbstractGossipLog, - ) { + private async handleAction(signedMessage: SignedMessage, messageLog: AbstractGossipLog) { const { id, signature, message } = signedMessage const { did, name, context } = message.payload @@ -279,9 +277,9 @@ export abstract class AbstractRuntime { throw err } - const additionalMessages = isAppend ? await executionContext.generateAdditionalMessages() : [] + this.additionalUpdates.set(id, await executionContext.generateAdditionalUpdates()) - return { result, additionalMessages } + return result } private async handleUpdates(signedMessage: SignedMessage, messageLog: AbstractGossipLog) { diff --git a/packages/gossiplog/src/AbstractGossipLog.ts b/packages/gossiplog/src/AbstractGossipLog.ts index 1312b6a13..31bbec5be 100644 --- a/packages/gossiplog/src/AbstractGossipLog.ts +++ b/packages/gossiplog/src/AbstractGossipLog.ts @@ -26,8 +26,7 @@ import { gossiplogTopicPattern } from "./utils.js" export type GossipLogConsumer = ( this: AbstractGossipLog, signedMessage: SignedMessage, - isAppend: boolean, -) => Awaitable<{ result: Result; additionalMessages?: Payload[] } | void> +) => Awaitable export interface GossipLogInit { topic: string @@ -132,7 +131,7 @@ export abstract class AbstractGossipLog extends const { signature, message, branch } = record const signedMessage = this.encode(signature, message, { branch }) assert(signedMessage.id === id) - await this.#apply.apply(this, [signedMessage, false]) + await this.#apply.apply(this, [signedMessage]) } }) } @@ -257,7 +256,7 @@ export abstract class AbstractGossipLog extends const signedMessage = this.encode(signature, message) this.log("appending message %s at clock %d with parents %o", signedMessage.id, clock, parents) - const applyResult = await this.apply(txn, signedMessage, true) + const applyResult = await this.apply(txn, signedMessage) return { signedMessage, applyResult } }) @@ -265,16 +264,6 @@ export abstract class AbstractGossipLog extends assert(applyResult.root !== null && applyResult.heads !== null, "failed to commit transaction") this.dispatchEvent(new CustomEvent("commit", { detail: { root: applyResult.root, heads: applyResult.heads } })) - signedMessage.result = applyResult.result?.result - - // append the additional messages - const additionalMessages = applyResult.result?.additionalMessages - if (additionalMessages) { - for (const additionalMessage of additionalMessages) { - await this.append(additionalMessage, { signer }) - } - } - return signedMessage as SignedMessage & { result: Result } } @@ -300,7 +289,7 @@ export abstract class AbstractGossipLog extends return null } - return await this.apply(txn, signedMessage, false) + return await this.apply(txn, signedMessage) }) if (result !== null) { @@ -313,8 +302,7 @@ export abstract class AbstractGossipLog extends private async apply( txn: ReadWriteTransaction, signedMessage: SignedMessage, - isAppend: boolean, - ): Promise<{ root: Node; heads: string[]; result: { result: Result; additionalMessages?: Payload[] } | void }> { + ): Promise<{ root: Node; heads: string[]; result: Result | void }> { const { id, signature, message, key, value } = signedMessage this.log.trace("applying %s %O", id, message) @@ -331,7 +319,7 @@ export abstract class AbstractGossipLog extends const branch = await this.getBranch(id, parentMessageRecords) signedMessage.branch = branch - const result = await this.#apply.apply(this, [signedMessage, isAppend]) + const result = await this.#apply.apply(this, [signedMessage]) const hash = toString(hashEntry(key, value), "hex") diff --git a/packages/gossiplog/test/result.test.ts b/packages/gossiplog/test/result.test.ts index d3d1592d9..fa5700c09 100644 --- a/packages/gossiplog/test/result.test.ts +++ b/packages/gossiplog/test/result.test.ts @@ -7,7 +7,7 @@ import { ed25519 } from "@canvas-js/signatures" import type { GossipLogConsumer } from "@canvas-js/gossiplog" import { testPlatforms } from "./utils.js" -const apply: GossipLogConsumer = ({ message: { payload } }) => ({ result: bytesToHex(sha256(payload)) }) +const apply: GossipLogConsumer = ({ message: { payload } }) => bytesToHex(sha256(payload)) testPlatforms( "get apply result", From 95c6beaee83eed577fb9cce63135c15b07dac5f5 Mon Sep 17 00:00:00 2001 From: Bob Webb Date: Mon, 3 Mar 2025 15:30:52 +0100 Subject: [PATCH 09/36] add yjs dependency to core --- package-lock.json | 48 +++++++++++++++++++++++++++++++++++++- packages/core/package.json | 3 ++- 2 files changed, 49 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 6754cb6af..7021ff63f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -21826,6 +21826,15 @@ "ws": "*" } }, + "node_modules/isomorphic.js": { + "version": "0.2.5", + "resolved": "https://registry.npmjs.org/isomorphic.js/-/isomorphic.js-0.2.5.tgz", + "integrity": "sha512-PIeMbHqMt4DnUP3MA/Flc0HElYjMXArsw1qwJZcm9sqR8mq3l8NYizFMty0pWwE/tzIGH3EKK5+jes5mAr85yw==", + "funding": { + "type": "GitHub Sponsors ❤", + "url": "https://github.com/sponsors/dmonad" + } + }, "node_modules/isows": { "version": "1.0.6", "funding": [ @@ -23483,6 +23492,26 @@ "node": ">= 0.8.0" } }, + "node_modules/lib0": { + "version": "0.2.99", + "resolved": "https://registry.npmjs.org/lib0/-/lib0-0.2.99.tgz", + "integrity": "sha512-vwztYuUf1uf/1zQxfzRfO5yzfNKhTtgOByCruuiQQxWQXnPb8Itaube5ylofcV0oM0aKal9Mv+S1s1Ky0UYP1w==", + "dependencies": { + "isomorphic.js": "^0.2.4" + }, + "bin": { + "0ecdsa-generate-keypair": "bin/0ecdsa-generate-keypair.js", + "0gentesthtml": "bin/gentesthtml.js", + "0serve": "bin/0serve.js" + }, + "engines": { + "node": ">=16" + }, + "funding": { + "type": "GitHub Sponsors ❤", + "url": "https://github.com/sponsors/dmonad" + } + }, "node_modules/libp2p": { "version": "2.7.4", "resolved": "https://registry.npmjs.org/libp2p/-/libp2p-2.7.4.tgz", @@ -34966,6 +34995,22 @@ "fd-slicer": "~1.1.0" } }, + "node_modules/yjs": { + "version": "13.6.23", + "resolved": "https://registry.npmjs.org/yjs/-/yjs-13.6.23.tgz", + "integrity": "sha512-ExtnT5WIOVpkL56bhLeisG/N5c4fmzKn4k0ROVfJa5TY2QHbH7F0Wu2T5ZhR7ErsFWQEFafyrnSI8TPKVF9Few==", + "dependencies": { + "lib0": "^0.2.99" + }, + "engines": { + "node": ">=16.0.0", + "npm": ">=8.0.0" + }, + "funding": { + "type": "GitHub Sponsors ❤", + "url": "https://github.com/sponsors/dmonad" + } + }, "node_modules/yn": { "version": "3.1.1", "devOptional": true, @@ -35482,7 +35527,8 @@ "quickjs-emscripten": "^0.31.0", "uint8arraylist": "^2.4.8", "uint8arrays": "^5.1.0", - "ws": "^8.18.0" + "ws": "^8.18.0", + "yjs": "^13.6.23" }, "devDependencies": { "@canvas-js/chain-cosmos": "0.14.0-next.1", diff --git a/packages/core/package.json b/packages/core/package.json index dab879b26..d43036c20 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -69,7 +69,8 @@ "quickjs-emscripten": "^0.31.0", "uint8arraylist": "^2.4.8", "uint8arrays": "^5.1.0", - "ws": "^8.18.0" + "ws": "^8.18.0", + "yjs": "^13.6.23" }, "devDependencies": { "@canvas-js/chain-cosmos": "0.14.0-next.1", From ff6715a2ac3cf2b2445fc48e2475598cebe4238e Mon Sep 17 00:00:00 2001 From: Bob Webb Date: Mon, 3 Mar 2025 15:38:19 +0100 Subject: [PATCH 10/36] reset change to gossiplog --- packages/gossiplog/src/AbstractGossipLog.ts | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/packages/gossiplog/src/AbstractGossipLog.ts b/packages/gossiplog/src/AbstractGossipLog.ts index 31bbec5be..b6dca1424 100644 --- a/packages/gossiplog/src/AbstractGossipLog.ts +++ b/packages/gossiplog/src/AbstractGossipLog.ts @@ -241,7 +241,11 @@ export abstract class AbstractGossipLog extends payload: T, { signer = this.signer }: { signer?: Signer } = {}, ): Promise & { result: Result }> { - const { signedMessage, applyResult } = await this.tree.write(async (txn) => { + let root: Node | null = null + let heads: string[] | null = null + let result: Result | undefined = undefined + + const signedMessage = await this.tree.write(async (txn) => { const [clock, parents] = await this.getClock() const message: Message = { @@ -258,12 +262,17 @@ export abstract class AbstractGossipLog extends const applyResult = await this.apply(txn, signedMessage) - return { signedMessage, applyResult } + root = applyResult.root + heads = applyResult.heads + result = applyResult.result + + return signedMessage }) - assert(applyResult.root !== null && applyResult.heads !== null, "failed to commit transaction") - this.dispatchEvent(new CustomEvent("commit", { detail: { root: applyResult.root, heads: applyResult.heads } })) + assert(root !== null && heads !== null, "failed to commit transaction") + this.dispatchEvent(new CustomEvent("commit", { detail: { root, heads } })) + signedMessage.result = result return signedMessage as SignedMessage & { result: Result } } @@ -302,7 +311,7 @@ export abstract class AbstractGossipLog extends private async apply( txn: ReadWriteTransaction, signedMessage: SignedMessage, - ): Promise<{ root: Node; heads: string[]; result: Result | void }> { + ): Promise<{ root: Node; heads: string[]; result: Result }> { const { id, signature, message, key, value } = signedMessage this.log.trace("applying %s %O", id, message) From c0f894d6d303ff4a8b837e90983d0769e5543a14 Mon Sep 17 00:00:00 2001 From: Bob Webb Date: Mon, 3 Mar 2025 16:04:17 +0100 Subject: [PATCH 11/36] fix schema definition --- packages/core/src/schema.ts | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/packages/core/src/schema.ts b/packages/core/src/schema.ts index f21aba577..fce4e89cd 100644 --- a/packages/core/src/schema.ts +++ b/packages/core/src/schema.ts @@ -39,15 +39,14 @@ type Snapshot struct { effects [SnapshotEffect] } -type Update = { - model: string - key: string - diff: Uint8Array +type Update struct { + model String + key String + diff Bytes } -type Updates = { - type: "updates" - updates: Update[] +type Updates struct { + updates [Update] } type Payload union { From 34f943179726a4242d18bd83527de44579a922f9 Mon Sep 17 00:00:00 2001 From: Bob Webb Date: Mon, 3 Mar 2025 16:15:25 +0100 Subject: [PATCH 12/36] only append updates if there are any updates --- packages/core/src/ExecutionContext.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/core/src/ExecutionContext.ts b/packages/core/src/ExecutionContext.ts index c312497ba..3560b1db8 100644 --- a/packages/core/src/ExecutionContext.ts +++ b/packages/core/src/ExecutionContext.ts @@ -187,6 +187,10 @@ export class ExecutionContext { updates.push({ model, key, diff }) } } - return [{ type: "updates", updates }] + if (updates.length > 0) { + return [{ type: "updates", updates }] + } else { + return [] + } } } From 7b044d1a730173fea3398d4ccdca59e105dc3890 Mon Sep 17 00:00:00 2001 From: Bob Webb Date: Mon, 3 Mar 2025 16:57:26 +0100 Subject: [PATCH 13/36] add test to assert that documents converge --- packages/core/src/runtime/AbstractRuntime.ts | 4 +- packages/core/test/yjs.test.ts | 81 ++++++++++++++++++++ 2 files changed, 84 insertions(+), 1 deletion(-) create mode 100644 packages/core/test/yjs.test.ts diff --git a/packages/core/src/runtime/AbstractRuntime.ts b/packages/core/src/runtime/AbstractRuntime.ts index af0f41e22..7392134b0 100644 --- a/packages/core/src/runtime/AbstractRuntime.ts +++ b/packages/core/src/runtime/AbstractRuntime.ts @@ -292,7 +292,9 @@ export abstract class AbstractRuntime { // apply the diff to the doc const doc = new Y.Doc() - Y.applyUpdate(doc, existingStateEntries[0].content) + if (existingStateEntries.length > 0) { + Y.applyUpdate(doc, existingStateEntries[0].content) + } Y.applyUpdate(doc, diff) const newContent = Y.encodeStateAsUpdate(doc) diff --git a/packages/core/test/yjs.test.ts b/packages/core/test/yjs.test.ts new file mode 100644 index 000000000..a0fb49f6c --- /dev/null +++ b/packages/core/test/yjs.test.ts @@ -0,0 +1,81 @@ +import { SIWESigner } from "@canvas-js/chain-ethereum" +import test, { ExecutionContext } from "ava" +import { Canvas } from "@canvas-js/core" +import * as Y from "yjs" + +const contract = ` +export const models = { + articles: { + id: "primary", + content: "yjs-doc" + } +}; +export const actions = { + async createNewArticle(db) { + const { id } = this + await db.yjsInsert("articles", id, 0, "") + }, + async insertIntoDoc(db, key, index, text) { + await db.yjsInsert("articles", key, index, text) + }, + async deleteFromDoc(db, key, index, length) { + await db.yjsDelete("articles", key, index, length) + } +}; +` + +async function stringifyDoc(app: Canvas, key: string) { + const doc = new Y.Doc() + Y.applyUpdate(doc, (await app.db.get("articles:state", key))!.content) + return doc.getText().toJSON() +} + +const init = async (t: ExecutionContext) => { + const signer = new SIWESigner() + const app = await Canvas.initialize({ + contract, + topic: "com.example.app", + reset: true, + signers: [signer], + }) + + t.teardown(() => app.stop()) + return { app, signer } +} + +test("apply an action and read a record from the database", async (t) => { + const { app: app1 } = await init(t) + + const { id } = await app1.actions.createNewArticle() + + t.log(`applied action ${id}`) + + await app1.actions.insertIntoDoc(id, 0, "Hello, world") + t.is(await stringifyDoc(app1, id), "Hello, world") + + // create another app + const { app: app2 } = await init(t) + + // sync the apps + await app1.messageLog.serve((s) => app2.messageLog.sync(s)) + t.is(await stringifyDoc(app2, id), "Hello, world") + + // insert ! into app1 + await app1.actions.insertIntoDoc(id, 12, "!") + t.is(await stringifyDoc(app1, id), "Hello, world!") + + // insert ? into app2 + await app2.actions.insertIntoDoc(id, 12, "?") + t.is(await stringifyDoc(app2, id), "Hello, world?") + + // sync app2 -> app1 + await app2.messageLog.serve((s) => app1.messageLog.sync(s)) + const app1MergedText = await stringifyDoc(app1, id) + + // sync app1 -> app2 + await app1.messageLog.serve((s) => app2.messageLog.sync(s)) + const app2MergedText = await stringifyDoc(app2, id) + + // both apps should now have converged + t.is(app1MergedText, app2MergedText) +}) From 35074177d7b7a6d50bebf0f8c32ba67d8a2bd1b4 Mon Sep 17 00:00:00 2001 From: Bob Webb Date: Tue, 4 Mar 2025 11:57:38 +0100 Subject: [PATCH 14/36] implement snapshot for updates --- packages/core/src/Canvas.ts | 6 ++- packages/core/src/runtime/AbstractRuntime.ts | 42 ++++++++++++++------ 2 files changed, 34 insertions(+), 14 deletions(-) diff --git a/packages/core/src/Canvas.ts b/packages/core/src/Canvas.ts index 927fd9996..3d496f6ae 100644 --- a/packages/core/src/Canvas.ts +++ b/packages/core/src/Canvas.ts @@ -17,7 +17,7 @@ import target from "#target" import type { Contract, Actions, ActionImplementation, ModelAPI, DeriveModelTypes } from "./types.js" import { Runtime, createRuntime } from "./runtime/index.js" -import { ActionRecord } from "./runtime/AbstractRuntime.js" +import { ActionRecord, updatesToEffects } from "./runtime/AbstractRuntime.js" import { validatePayload } from "./schema.js" import { createSnapshot, hashSnapshot } from "./snapshot.js" import { topicPattern } from "./utils.js" @@ -184,7 +184,9 @@ export class Canvas< const record = { did } effects.push({ operation: "set", model: "$dids", value: record }) } else if (message.payload.type === "updates") { - // TODO: handle updates + for (const effect of await updatesToEffects(message.payload, db)) { + effects.push(effect) + } } start = id } diff --git a/packages/core/src/runtime/AbstractRuntime.ts b/packages/core/src/runtime/AbstractRuntime.ts index 7392134b0..6e379ef40 100644 --- a/packages/core/src/runtime/AbstractRuntime.ts +++ b/packages/core/src/runtime/AbstractRuntime.ts @@ -136,7 +136,7 @@ export abstract class AbstractRuntime { } else if (isSnapshot(signedMessage)) { return await handleSnapshot(signedMessage, this) } else if (isUpdates(signedMessage)) { - return await handleUpdates(signedMessage, this) + return await handleUpdates(signedMessage) } else { throw new Error("invalid message payload type") } @@ -282,28 +282,46 @@ export abstract class AbstractRuntime { return result } - private async handleUpdates(signedMessage: SignedMessage, messageLog: AbstractGossipLog) { - const effects: Effect[] = [] - for (const { model, key, diff } of signedMessage.message.payload.updates) { - const existingStateEntries = await this.db.query<{ id: string; content: Uint8Array }>(`${model}:state`, { + private async handleUpdates(signedMessage: SignedMessage) { + const effects = await updatesToEffects(signedMessage.message.payload, this.db) + await this.db.apply(effects) + } +} + +export const updatesToEffects = async (payload: Updates, db: AbstractModelDB) => { + const updatedEntries: Record> = {} + + for (const { model, key, diff } of payload.updates) { + let doc = (updatedEntries[model] || {})[key] + + if (!doc) { + const existingStateEntries = await db.query<{ id: string; content: Uint8Array }>(`${model}:state`, { where: { id: key }, limit: 1, }) - - // apply the diff to the doc - const doc = new Y.Doc() + doc = new Y.Doc() if (existingStateEntries.length > 0) { Y.applyUpdate(doc, existingStateEntries[0].content) } - Y.applyUpdate(doc, diff) - const newContent = Y.encodeStateAsUpdate(doc) + } + + // apply the diff to the doc + Y.applyUpdate(doc, diff) + + updatedEntries[model] = { ...updatedEntries[model], [key]: doc } + } + + const effects: Effect[] = [] + for (const [model, entries] of Object.entries(updatedEntries)) { + for (const [key, doc] of Object.entries(entries)) { + const diff = Y.encodeStateAsUpdate(doc) effects.push({ model: `${model}:state`, operation: "set", - value: { id: key, content: newContent }, + value: { id: key, content: diff }, }) } - await this.db.apply(effects) } + return effects } From 0f2ffec424b5b4746bd78d51831dc3af8d745f9c Mon Sep 17 00:00:00 2001 From: Bob Webb Date: Tue, 4 Mar 2025 14:09:05 +0100 Subject: [PATCH 15/36] add yjs to snapshot test --- packages/core/test/snapshot.test.ts | 31 +++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/packages/core/test/snapshot.test.ts b/packages/core/test/snapshot.test.ts index 76149bc0d..76c8374e6 100644 --- a/packages/core/test/snapshot.test.ts +++ b/packages/core/test/snapshot.test.ts @@ -1,4 +1,5 @@ import test from "ava" +import * as Y from "yjs" import { Canvas, Config } from "@canvas-js/core" @@ -11,6 +12,10 @@ test("snapshot persists data across apps", async (t) => { id: "primary", content: "string", }, + documents: { + id: "primary", + content: "yjs-doc", + }, }, actions: { async createPost(db, { id, content }: { id: string; content: string }) { @@ -19,6 +24,12 @@ test("snapshot persists data across apps", async (t) => { async deletePost(db, { id }: { id: string }) { await db.delete("posts", id) }, + async insertIntoDocument(db, key, index, text) { + await db.yjsInsert("documents", key, index, text) + }, + async deleteFromDocument(db, key, index, length) { + await db.yjsDelete("documents", key, index, length) + }, }, }, } @@ -35,9 +46,11 @@ test("snapshot persists data across apps", async (t) => { await app.actions.createPost({ id: "d", content: "baz" }) await app.actions.deletePost({ id: "b" }) await app.actions.deletePost({ id: "d" }) + await app.actions.insertIntoDocument("e", 0, "Hello, world") + await app.actions.deleteFromDocument("e", 5, 7) const [clock, parents] = await app.messageLog.getClock() - t.is(clock, 8) // one session, six actions + t.is(clock, 12) // one session, eight actions, two "updates" messages t.is(parents.length, 1) // snapshot and add some more actions @@ -52,13 +65,19 @@ test("snapshot persists data across apps", async (t) => { t.is(await app2.db.get("posts", "d"), null) t.is(await app2.db.get("posts", "e"), null) + const docDiff1 = await app2.db.get("documents:state", "e") + const doc1 = new Y.Doc() + Y.applyUpdate(doc1, docDiff1!.content) + t.is(doc1.getText().toJSON(), "Hello") + await app2.actions.createPost({ id: "a", content: "1" }) await app2.actions.createPost({ id: "b", content: "2" }) await app2.actions.createPost({ id: "e", content: "3" }) await app2.actions.createPost({ id: "f", content: "4" }) + await app2.actions.insertIntoDocument("e", 6, "?") const [clock2, parents2] = await app2.messageLog.getClock() - t.is(clock2, 7) // one snapshot, one session, four actions + t.is(clock2, 9) // one snapshot, one session, four actions t.is(parents2.length, 1) t.is((await app2.db.get("posts", "a"))?.content, "1") @@ -67,6 +86,10 @@ test("snapshot persists data across apps", async (t) => { t.is(await app2.db.get("posts", "d"), null) t.is((await app2.db.get("posts", "e"))?.content, "3") t.is((await app2.db.get("posts", "f"))?.content, "4") + const docDiff2 = await app2.db.get("documents:state", "e") + const doc2 = new Y.Doc() + Y.applyUpdate(doc2, docDiff2!.content) + t.is(doc2.getText().toJSON(), "Hello?") // snapshot a second time const snapshot2 = await app2.createSnapshot() @@ -79,6 +102,10 @@ test("snapshot persists data across apps", async (t) => { t.is((await app3.db.get("posts", "e"))?.content, "3") t.is((await app3.db.get("posts", "f"))?.content, "4") t.is(await app3.db.get("posts", "g"), null) + const docDiff3 = await app3.db.get("documents:state", "e") + const doc3 = new Y.Doc() + Y.applyUpdate(doc3, docDiff3!.content) + t.is(doc3.getText().toJSON(), "Hello?") const [clock3] = await app3.messageLog.getClock() t.is(clock3, 2) // one snapshot From ad01488ff3d2ea36cbcb55b444c4a4c692f0a00a Mon Sep 17 00:00:00 2001 From: Bob Webb Date: Wed, 5 Mar 2025 13:32:04 +0100 Subject: [PATCH 16/36] add contract function to apply a quill delta to the yjs document --- package-lock.json | 66 ++++++++++++++++++++ packages/core/src/ExecutionContext.ts | 14 ++++- packages/core/src/runtime/ContractRuntime.ts | 14 +++-- packages/core/src/runtime/FunctionRuntime.ts | 11 ++-- packages/core/src/types.ts | 1 + tsconfig.json | 1 + 6 files changed, 94 insertions(+), 13 deletions(-) diff --git a/package-lock.json b/package-lock.json index 7021ff63f..e912151ef 100644 --- a/package-lock.json +++ b/package-lock.json @@ -152,6 +152,68 @@ "url": "https://paulmillr.com/funding/" } }, + "examples/document": { + "version": "0.14.0-next.1", + "dependencies": { + "@canvas-js/chain-atp": "0.14.0-next.1", + "@canvas-js/chain-cosmos": "0.14.0-next.1", + "@canvas-js/chain-ethereum": "0.14.0-next.1", + "@canvas-js/chain-ethereum-viem": "0.14.0-next.1", + "@canvas-js/chain-solana": "0.14.0-next.1", + "@canvas-js/chain-substrate": "0.14.0-next.1", + "@canvas-js/cli": "0.14.0-next.1", + "@canvas-js/core": "0.14.0-next.1", + "@canvas-js/gossiplog": "0.14.0-next.1", + "@canvas-js/hooks": "0.14.0-next.1", + "@canvas-js/interfaces": "0.14.0-next.1", + "@canvas-js/modeldb-sqlite-wasm": "0.14.0-next.1", + "@cosmjs/encoding": "^0.32.3", + "@farcaster/auth-kit": "^0.6.0", + "@farcaster/frame-sdk": "^0.0.26", + "@keplr-wallet/types": "^0.11.64", + "@libp2p/interface": "^2.5.0", + "@magic-ext/auth": "^4.3.2", + "@metamask/providers": "^14.0.2", + "@multiformats/multiaddr": "^12.3.5", + "@noble/hashes": "^1.7.1", + "@polkadot/extension-dapp": "^0.46.9", + "@terra-money/feather.js": "^2.0.4", + "@types/react": "^18.3.9", + "@types/react-dom": "^18.3.0", + "buffer": "^6.0.3", + "comlink": "^4.4.1", + "ethers": "^6.13.5", + "idb": "^8.0.2", + "magic-sdk": "^21.5.0", + "multiformats": "^13.3.2", + "near-api-js": "^2.1.4", + "process": "^0.11.10", + "react": "^18.3.1", + "react-dom": "^18.3.1", + "viem": "^2.22.21", + "vite-plugin-wasm": "^3.3.0", + "web3": "^4.10.0" + }, + "devDependencies": { + "autoprefixer": "^10.4.19", + "postcss": "^8.4.38", + "tailwindcss": "^3.4.4", + "typescript": "~5.6.0", + "vite": "^5.4.8", + "vite-plugin-node-polyfills": "^0.22.0" + } + }, + "examples/document/node_modules/@noble/hashes": { + "version": "1.7.1", + "resolved": "https://registry.npmjs.org/@noble/hashes/-/hashes-1.7.1.tgz", + "integrity": "sha512-B8XBPsn4vT/KJAGqDzbwztd+6Yte3P4V7iafm24bxgDe/mlRuK6xmWPuCNrKt2vDafZ8MfJLlchDG/vYafQEjQ==", + "engines": { + "node": "^14.21.3 || >=16" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, "examples/encrypted-chat": { "name": "@canvas-js/example-chat-encrypted", "version": "0.14.0-next.1", @@ -2762,6 +2824,10 @@ "resolved": "packages/core", "link": true }, + "node_modules/@canvas-js/document": { + "resolved": "examples/document", + "link": true + }, "node_modules/@canvas-js/ethereum-contracts": { "resolved": "packages/ethereum-contracts", "link": true diff --git a/packages/core/src/ExecutionContext.ts b/packages/core/src/ExecutionContext.ts index 3560b1db8..8c767cf67 100644 --- a/packages/core/src/ExecutionContext.ts +++ b/packages/core/src/ExecutionContext.ts @@ -20,7 +20,13 @@ type YjsCallDelete = { index: number length: number } -export type YjsCall = YjsCallInsert | YjsCallDelete +type YjsCallApplyDelta = { + call: "applyDelta" + // we don't have a declared type for Quill Deltas + delta: any +} + +export type YjsCall = YjsCallInsert | YjsCallDelete | YjsCallApplyDelta export class ExecutionContext { // // recordId -> { version, value } @@ -151,6 +157,12 @@ export class ExecutionContext { this.modelEntries[model][key] = result } + public pushYjsCall(modelName: string, id: string, call: YjsCall) { + this.yjsCalls[modelName] ||= {} + this.yjsCalls[modelName][id] ||= [] + this.yjsCalls[modelName][id].push(call) + } + public async getYDoc(modelName: string, id: string): Promise { const existingStateEntries = await this.db.query<{ id: string; content: Uint8Array }>(`${modelName}:state`, { where: { id: id }, diff --git a/packages/core/src/runtime/ContractRuntime.ts b/packages/core/src/runtime/ContractRuntime.ts index fd245a38a..967478770 100644 --- a/packages/core/src/runtime/ContractRuntime.ts +++ b/packages/core/src/runtime/ContractRuntime.ts @@ -140,18 +140,20 @@ export class ContractRuntime extends AbstractRuntime { const key = vm.context.getString(keyHandle) const index = vm.context.getNumber(indexHandle) const content = vm.context.getString(contentHandle) - this.context.yjsCalls[model] ||= {} - this.context.yjsCalls[model][key] ||= [] - this.context.yjsCalls[model][key].push({ call: "insert", index, content }) + this.context.pushYjsCall(model, key, { call: "insert", index, content }) }), yjsDelete: vm.context.newFunction("yjsDelete", (modelHandle, keyHandle, indexHandle, lengthHandle) => { const model = vm.context.getString(modelHandle) const key = vm.context.getString(keyHandle) const index = vm.context.getNumber(indexHandle) const length = vm.context.getNumber(lengthHandle) - this.context.yjsCalls[model] ||= {} - this.context.yjsCalls[model][key] ||= [] - this.context.yjsCalls[model][key].push({ call: "delete", index, length }) + this.context.pushYjsCall(model, key, { call: "delete", index, length }) + }), + yjsApplyDelta: vm.context.newFunction("yjsApplyDelta", (modelHandle, keyHandle, deltaHandle) => { + const model = vm.context.getString(modelHandle) + const key = vm.context.getString(keyHandle) + const delta = vm.unwrapValue(deltaHandle) + this.context.pushYjsCall(model, key, { call: "applyDelta", delta }) }), }) .consume(vm.cache) diff --git a/packages/core/src/runtime/FunctionRuntime.ts b/packages/core/src/runtime/FunctionRuntime.ts index 87f7121da..8285bc223 100644 --- a/packages/core/src/runtime/FunctionRuntime.ts +++ b/packages/core/src/runtime/FunctionRuntime.ts @@ -174,14 +174,13 @@ export class FunctionRuntime extends AbstractRuntim } }, yjsInsert: async (model: string, key: string, index: number, content: string) => { - this.context.yjsCalls[model] ||= {} - this.context.yjsCalls[model][key] ||= [] - this.context.yjsCalls[model][key].push({ call: "insert", index, content }) + this.context.pushYjsCall(model, key, { call: "insert", index, content }) }, yjsDelete: async (model: string, key: string, index: number, length: number) => { - this.context.yjsCalls[model] ||= {} - this.context.yjsCalls[model][key] ||= [] - this.context.yjsCalls[model][key].push({ call: "delete", index, length }) + this.context.pushYjsCall(model, key, { call: "delete", index, length }) + }, + yjsApplyDelta: async (model: string, key: string, delta: any) => { + this.context.pushYjsCall(model, key, { call: "applyDelta", delta }) }, } } diff --git a/packages/core/src/types.ts b/packages/core/src/types.ts index 3b14acb82..def4e6236 100644 --- a/packages/core/src/types.ts +++ b/packages/core/src/types.ts @@ -56,6 +56,7 @@ export type ModelAPI> = { index: number, length: number, ) => Promise + yjsApplyDelta: (model: T, key: string, delta: any) => Promise } export type ActionContext> = { diff --git a/tsconfig.json b/tsconfig.json index c8ebabe5b..7657e4021 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -44,6 +44,7 @@ { "path": "./packages/utils/test" }, { "path": "./examples/chat" }, { "path": "./examples/chat-next" }, + { "path": "./examples/document" }, { "path": "./examples/encrypted-chat" }, { "path": "./examples/snake" } ] From 1b803196a40f5b736ea9dfa12d99c08fb81cac43 Mon Sep 17 00:00:00 2001 From: Bob Webb Date: Wed, 5 Mar 2025 14:08:07 +0100 Subject: [PATCH 17/36] Add yjs-doc to model schema expected column types --- packages/modeldb/src/types.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/modeldb/src/types.ts b/packages/modeldb/src/types.ts index e686b4f7f..154214f02 100644 --- a/packages/modeldb/src/types.ts +++ b/packages/modeldb/src/types.ts @@ -9,6 +9,7 @@ export type NullablePrimitiveType = `${PrimitiveType}?` export type ReferenceType = `@${string}` export type NullableReferenceType = `@${string}?` export type RelationType = `@${string}[]` +export type YjsDocType = "yjs-doc" export type PropertyType = | PrimaryKeyType @@ -17,6 +18,7 @@ export type PropertyType = | ReferenceType | NullableReferenceType | RelationType + | YjsDocType /** property name, or property names joined by slashes */ export type IndexInit = string From 547848bbe44558e398d07db6ebe12cf53ef6344e Mon Sep 17 00:00:00 2001 From: Bob Webb Date: Wed, 5 Mar 2025 14:43:04 +0100 Subject: [PATCH 18/36] more fixes for applyDelta --- packages/core/src/ExecutionContext.ts | 2 ++ packages/core/src/runtime/ContractRuntime.ts | 4 ++-- packages/core/src/runtime/FunctionRuntime.ts | 2 +- packages/core/src/types.ts | 2 +- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/packages/core/src/ExecutionContext.ts b/packages/core/src/ExecutionContext.ts index 8c767cf67..5c1d6cef2 100644 --- a/packages/core/src/ExecutionContext.ts +++ b/packages/core/src/ExecutionContext.ts @@ -189,6 +189,8 @@ export class ExecutionContext { doc.getText().insert(call.index, call.content) } else if (call.call === "delete") { doc.getText().delete(call.index, call.length) + } else if (call.call === "applyDelta") { + doc.getText().applyDelta(call.delta) } else { throw new Error("unexpected call type") } diff --git a/packages/core/src/runtime/ContractRuntime.ts b/packages/core/src/runtime/ContractRuntime.ts index 967478770..29561e42e 100644 --- a/packages/core/src/runtime/ContractRuntime.ts +++ b/packages/core/src/runtime/ContractRuntime.ts @@ -152,8 +152,8 @@ export class ContractRuntime extends AbstractRuntime { yjsApplyDelta: vm.context.newFunction("yjsApplyDelta", (modelHandle, keyHandle, deltaHandle) => { const model = vm.context.getString(modelHandle) const key = vm.context.getString(keyHandle) - const delta = vm.unwrapValue(deltaHandle) - this.context.pushYjsCall(model, key, { call: "applyDelta", delta }) + const delta = vm.context.getString(deltaHandle) + this.context.pushYjsCall(model, key, { call: "applyDelta", delta: JSON.parse(delta) }) }), }) .consume(vm.cache) diff --git a/packages/core/src/runtime/FunctionRuntime.ts b/packages/core/src/runtime/FunctionRuntime.ts index 8285bc223..56d90eba8 100644 --- a/packages/core/src/runtime/FunctionRuntime.ts +++ b/packages/core/src/runtime/FunctionRuntime.ts @@ -180,7 +180,7 @@ export class FunctionRuntime extends AbstractRuntim this.context.pushYjsCall(model, key, { call: "delete", index, length }) }, yjsApplyDelta: async (model: string, key: string, delta: any) => { - this.context.pushYjsCall(model, key, { call: "applyDelta", delta }) + this.context.pushYjsCall(model, key, { call: "applyDelta", delta: JSON.parse(delta) }) }, } } diff --git a/packages/core/src/types.ts b/packages/core/src/types.ts index def4e6236..16630c33f 100644 --- a/packages/core/src/types.ts +++ b/packages/core/src/types.ts @@ -56,7 +56,7 @@ export type ModelAPI> = { index: number, length: number, ) => Promise - yjsApplyDelta: (model: T, key: string, delta: any) => Promise + yjsApplyDelta: (model: T, key: string, delta: string) => Promise } export type ActionContext> = { From c87bdd3a3e5818f773fc4c257ead8323f316f425 Mon Sep 17 00:00:00 2001 From: Bob Webb Date: Wed, 5 Mar 2025 16:11:14 +0100 Subject: [PATCH 19/36] call apply delta with ops --- packages/core/src/ExecutionContext.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/core/src/ExecutionContext.ts b/packages/core/src/ExecutionContext.ts index 5c1d6cef2..c740c7546 100644 --- a/packages/core/src/ExecutionContext.ts +++ b/packages/core/src/ExecutionContext.ts @@ -190,7 +190,7 @@ export class ExecutionContext { } else if (call.call === "delete") { doc.getText().delete(call.index, call.length) } else if (call.call === "applyDelta") { - doc.getText().applyDelta(call.delta) + doc.getText().applyDelta(call.delta.ops) } else { throw new Error("unexpected call type") } From 303edd82524d53a3df3adff112619364ba9e07e2 Mon Sep 17 00:00:00 2001 From: Bob Webb Date: Wed, 5 Mar 2025 16:11:27 +0100 Subject: [PATCH 20/36] WIP basic collaborative editor demo --- examples/document/.gitignore | 1 + examples/document/README.md | 54 ++++++++ examples/document/index.html | 31 +++++ examples/document/package.json | 46 +++++++ examples/document/postcss.config.js | 6 + examples/document/src/App.tsx | 113 +++++++++++++++++ examples/document/src/AppContext.ts | 28 +++++ examples/document/src/ConnectionStatus.tsx | 90 ++++++++++++++ examples/document/src/ControlPanel.tsx | 115 ++++++++++++++++++ examples/document/src/Editor.tsx | 60 +++++++++ examples/document/src/LogStatus.tsx | 67 ++++++++++ examples/document/src/SessionStatus.tsx | 85 +++++++++++++ .../document/src/components/AddressView.tsx | 16 +++ .../document/src/components/MultiaddrView.tsx | 22 ++++ .../document/src/components/PeerIdView.tsx | 19 +++ .../src/connect/ConnectEIP712Burner.tsx | 62 ++++++++++ examples/document/src/contract.ts | 14 +++ examples/document/src/index.tsx | 14 +++ examples/document/styles.css | 17 +++ examples/document/tailwind.config.js | 4 + examples/document/tsconfig.json | 19 +++ examples/document/vite.config.js | 28 +++++ package-lock.json | 67 ++++++---- 23 files changed, 951 insertions(+), 27 deletions(-) create mode 100644 examples/document/.gitignore create mode 100644 examples/document/README.md create mode 100644 examples/document/index.html create mode 100644 examples/document/package.json create mode 100644 examples/document/postcss.config.js create mode 100644 examples/document/src/App.tsx create mode 100644 examples/document/src/AppContext.ts create mode 100644 examples/document/src/ConnectionStatus.tsx create mode 100644 examples/document/src/ControlPanel.tsx create mode 100644 examples/document/src/Editor.tsx create mode 100644 examples/document/src/LogStatus.tsx create mode 100644 examples/document/src/SessionStatus.tsx create mode 100644 examples/document/src/components/AddressView.tsx create mode 100644 examples/document/src/components/MultiaddrView.tsx create mode 100644 examples/document/src/components/PeerIdView.tsx create mode 100644 examples/document/src/connect/ConnectEIP712Burner.tsx create mode 100644 examples/document/src/contract.ts create mode 100644 examples/document/src/index.tsx create mode 100644 examples/document/styles.css create mode 100644 examples/document/tailwind.config.js create mode 100644 examples/document/tsconfig.json create mode 100644 examples/document/vite.config.js diff --git a/examples/document/.gitignore b/examples/document/.gitignore new file mode 100644 index 000000000..8fce60300 --- /dev/null +++ b/examples/document/.gitignore @@ -0,0 +1 @@ +data/ diff --git a/examples/document/README.md b/examples/document/README.md new file mode 100644 index 000000000..827150480 --- /dev/null +++ b/examples/document/README.md @@ -0,0 +1,54 @@ +# Document Editor Example + +[Github](https://github.com/canvasxyz/canvas/tree/main/examples/document) + +This example app demonstrates collaborative document editing using the CRDT provided by [Yjs](https://github.com/yjs/yjs). It allows users to collaboratively edit a single text document with persistence over libp2p. + +```ts +export const models = { + documents: { + id: "primary", + content: "yjs-doc", + }, +} + +export const actions = { + async applyDeltaToDoc(db, index, text) { + await db.yjsApplyDelta("documents", "0", index, text) + }, +} +``` + +## Server + +Run `npm run dev:server` to start a temporary in-memory server, or +`npm run start:server` to persist data to a `.cache` directory. + +To deploy the replication server: + +``` +$ cd server +$ fly deploy +``` + +If you are forking this example, you should change: + +- the Fly app name +- the `ANNOUNCE` environment variable to match your Fly app name + +## Running the Docker container locally + +Mount a volume to `/data`. Set the `PORT`, `LISTEN`, `ANNOUNCE`, and +`BOOTSTRAP_LIST` environment variables if appropriate. + +## Deploying to Railway + +Create a Railway space based on the root of this Github workspace (e.g. canvasxyz/canvas). + +- Custom build command: `npm run build && VITE_CANVAS_WS_URL=wss://chat-example.canvas.xyz npm run build --workspace=@canvas-js/example-chat` +- Custom start command: `./install-prod.sh && canvas run /tmp/canvas-example-chat --port 8080 --static examples/chat/dist --topic chat-example.canvas.xyz --init examples/chat/src/contract.ts` +- Watch paths: `/examples/chat/**` +- Public networking: + - Add a service domain for port 8080. + - Add a service domain for port 4444. +- Watch path: `/examples/chat/**`. (Only build when chat code is updated, or a chat package is updated.) diff --git a/examples/document/index.html b/examples/document/index.html new file mode 100644 index 000000000..6ee9fc7fc --- /dev/null +++ b/examples/document/index.html @@ -0,0 +1,31 @@ + + + + + + + Canvas Chat + + + + + +
+ + diff --git a/examples/document/package.json b/examples/document/package.json new file mode 100644 index 000000000..1af344a48 --- /dev/null +++ b/examples/document/package.json @@ -0,0 +1,46 @@ +{ + "private": true, + "name": "@canvas-js/document", + "version": "0.14.0-next.1", + "type": "module", + "scripts": { + "build": "npx vite build", + "dev": "npx vite", + "dev:server": "canvas run --port 8080 --static dist --network-explorer --topic document-example.canvas.xyz src/contract.ts" + }, + "dependencies": { + "@canvas-js/chain-atp": "0.14.0-next.1", + "@canvas-js/chain-cosmos": "0.14.0-next.1", + "@canvas-js/chain-ethereum": "0.14.0-next.1", + "@canvas-js/chain-ethereum-viem": "0.14.0-next.1", + "@canvas-js/chain-solana": "0.14.0-next.1", + "@canvas-js/chain-substrate": "0.14.0-next.1", + "@canvas-js/cli": "0.14.0-next.1", + "@canvas-js/core": "0.14.0-next.1", + "@canvas-js/gossiplog": "0.14.0-next.1", + "@canvas-js/hooks": "0.14.0-next.1", + "@canvas-js/interfaces": "0.14.0-next.1", + "@canvas-js/modeldb-sqlite-wasm": "0.14.0-next.1", + "@types/react": "^18.3.9", + "@types/react-dom": "^18.3.0", + "buffer": "^6.0.3", + "comlink": "^4.4.1", + "ethers": "^6.13.5", + "idb": "^8.0.2", + "multiformats": "^13.3.2", + "near-api-js": "^2.1.4", + "process": "^0.11.10", + "quill": "^2.0.3", + "react": "^18.3.1", + "react-dom": "^18.3.1", + "vite-plugin-wasm": "^3.3.0" + }, + "devDependencies": { + "autoprefixer": "^10.4.19", + "postcss": "^8.4.38", + "tailwindcss": "^3.4.4", + "typescript": "~5.6.0", + "vite": "^5.4.8", + "vite-plugin-node-polyfills": "^0.22.0" + } +} diff --git a/examples/document/postcss.config.js b/examples/document/postcss.config.js new file mode 100644 index 000000000..1a5262473 --- /dev/null +++ b/examples/document/postcss.config.js @@ -0,0 +1,6 @@ +export default { + plugins: { + tailwindcss: {}, + autoprefixer: {}, + }, +} diff --git a/examples/document/src/App.tsx b/examples/document/src/App.tsx new file mode 100644 index 000000000..2df30350b --- /dev/null +++ b/examples/document/src/App.tsx @@ -0,0 +1,113 @@ +import React, { useEffect, useRef, useState } from "react" + +import type { SessionSigner } from "@canvas-js/interfaces" +import { Eip712Signer, SIWESigner, SIWFSigner } from "@canvas-js/chain-ethereum" +import { ATPSigner } from "@canvas-js/chain-atp" +import { CosmosSigner } from "@canvas-js/chain-cosmos" +import { SolanaSigner } from "@canvas-js/chain-solana" +import { SubstrateSigner } from "@canvas-js/chain-substrate" + +import { useCanvas, useLiveQuery } from "@canvas-js/hooks" + +import { AuthKitProvider } from "@farcaster/auth-kit" +import { JsonRpcProvider } from "ethers" +import Quill from "quill/core" +import * as Y from "yjs" + +import { AppContext } from "./AppContext.js" +import { ControlPanel } from "./ControlPanel.js" +import { SessionStatus } from "./SessionStatus.js" +import { ConnectEIP712Burner } from "./connect/ConnectEIP712Burner.js" +import { ConnectionStatus } from "./ConnectionStatus.js" +import { LogStatus } from "./LogStatus.js" +import * as contract from "./contract.js" +import { Editor } from "./Editor.js" + +export const topic = "document-example.canvas.xyz" + +const wsURL = import.meta.env.VITE_CANVAS_WS_URL ?? null +console.log("websocket API URL:", wsURL) + +const config = { + // For a production app, replace this with an Optimism Mainnet + // RPC URL from a provider like Alchemy or Infura. + relay: "https://relay.farcaster.xyz", + rpcUrl: "https://mainnet.optimism.io", + domain: "document-example.canvas.xyz", + siweUri: "https://document-example.canvas.xyz", + provider: new JsonRpcProvider(undefined, 10), +} + +export const App: React.FC<{}> = ({}) => { + const [sessionSigner, setSessionSigner] = useState(null) + const [address, setAddress] = useState(null) + + const quillRef = useRef() + + const topicRef = useRef(topic) + + const { app, ws } = useCanvas(wsURL, { + topic: topicRef.current, + contract, + signers: [ + new SIWESigner(), + new Eip712Signer(), + new SIWFSigner(), + new ATPSigner(), + new CosmosSigner(), + new SubstrateSigner({}), + new SolanaSigner(), + ], + }) + + const existingStateEntries = useLiveQuery(app, "documents:state", { + where: { id: "0" }, + limit: 1, + }) as { id: string; content: Uint8Array }[] + + useEffect(() => { + if (existingStateEntries && existingStateEntries.length > 0) { + const doc = new Y.Doc() + Y.applyUpdate(doc, existingStateEntries[0].content) + // doc.getText().toJSON() + console.log("doc.getText().toJSON():", doc.getText().toJSON()) + quillRef.current?.setText(doc.getText().toJSON(), "silent") + } + }, [existingStateEntries]) + + return ( + + + {app && ws ? ( +
+
+
+ {}} + onTextChange={async (delta) => { + await app.actions.applyDeltaToDoc(JSON.stringify(delta)) + }} + /> +
+
+ + + + + +
+
+
+ ) : ( +
Connecting to {wsURL}...
+ )} + + + ) +} diff --git a/examples/document/src/AppContext.ts b/examples/document/src/AppContext.ts new file mode 100644 index 000000000..2e4ec4586 --- /dev/null +++ b/examples/document/src/AppContext.ts @@ -0,0 +1,28 @@ +import { createContext } from "react" + +import type { SessionSigner } from "@canvas-js/interfaces" +import { Canvas } from "@canvas-js/core" + +export type AppContext = { + app: Canvas | null + + address: string | null + setAddress: (address: string | null) => void + + sessionSigner: SessionSigner | null + setSessionSigner: (signer: SessionSigner | null) => void +} + +export const AppContext = createContext({ + app: null, + + address: null, + setAddress: (address: string | null) => { + throw new Error("AppContext.Provider not found") + }, + + sessionSigner: null, + setSessionSigner: (signer) => { + throw new Error("AppContext.Provider not found") + }, +}) diff --git a/examples/document/src/ConnectionStatus.tsx b/examples/document/src/ConnectionStatus.tsx new file mode 100644 index 000000000..08aba6381 --- /dev/null +++ b/examples/document/src/ConnectionStatus.tsx @@ -0,0 +1,90 @@ +import React, { useContext, useEffect, useState } from "react" + +import type { Canvas, NetworkClient } from "@canvas-js/core" + +import { AppContext } from "./AppContext.js" + +export interface ConnectionStatusProps { + topic: string + ws: NetworkClient +} + +export const ConnectionStatus: React.FC = ({ topic, ws }) => { + const { app } = useContext(AppContext) + + const [, setTick] = useState(0) + useEffect(() => { + const timer = setInterval(() => { + setTick(t => t + 1) + }, 1000) + return () => clearInterval(timer) + }, []) + + if (app === null) { + return null + } + + return ( +
+
+ Topic +
+
+ {topic} +
+ +
+
+ Connection +
+
+ {import.meta.env.VITE_CANVAS_WS_URL} + ({ws.isConnected() ? 'Connected' : 'Disconnected'}) +
+
+ ) +} + +interface ConnectionListProps { + app: Canvas +} + +// const ConnectionList: React.FC = ({ app }) => { +// const [peers, setPeers] = useState([]) + +// useEffect(() => { +// if (app === null) { +// return +// } + +// const handleConnectionOpen = ({ detail: { peer } }: CustomEvent<{ peer: string }>) => +// void setPeers((peers) => [...peers, peer]) + +// const handleConnectionClose = ({ detail: { peer } }: CustomEvent<{ peer: string }>) => +// void setPeers((peers) => peers.filter((id) => id !== peer)) + +// app.messageLog.addEventListener("connect", handleConnectionOpen) +// app.messageLog.addEventListener("disconnect", handleConnectionClose) + +// return () => { +// app.messageLog.removeEventListener("connect", handleConnectionOpen) +// app.messageLog.removeEventListener("disconnect", handleConnectionClose) +// } +// }, [app]) + +// if (peers.length === 0) { +// return
No connections
+// } else { +// return ( +//
    +// {peers.map((peer) => { +// return ( +//
  • +// {peer} +//
    • +// ) +// })} +//
+// ) +// } +// } diff --git a/examples/document/src/ControlPanel.tsx b/examples/document/src/ControlPanel.tsx new file mode 100644 index 000000000..3efbec545 --- /dev/null +++ b/examples/document/src/ControlPanel.tsx @@ -0,0 +1,115 @@ +import React, { useCallback, useContext, useState } from "react" +import { deleteDB } from "idb" +import { bytesToHex, randomBytes } from "@noble/hashes/utils" + +import { AppContext } from "./AppContext.js" + +export interface ControlPanelProps {} + +export const ControlPanel: React.FC = ({}) => { + const { app, sessionSigner } = useContext(AppContext) + + const [isStarted, setIsStarted] = useState(false) + + const start = useCallback(async () => { + if (app === null) { + return + } + + // try { + // await app.libp2p.start() + // setIsStarted(true) + // } catch (err) { + // console.error(err) + // } + }, [app]) + + const stop = useCallback(async () => { + if (app === null) { + return + } + + // try { + // await app.libp2p.stop() + // setIsStarted(false) + // } catch (err) { + // console.error(err) + // } + }, [app]) + + const clear = useCallback(async () => { + if (app === null) { + return + } + + await app.stop() + + console.log("deleting database") + await deleteDB(`canvas/v1/${app.topic}`, {}) + + console.log("clearing session signer data", sessionSigner) + await sessionSigner?.clear?.(app.topic) + + window.location.reload() + }, [app, sessionSigner]) + + const spam = useCallback(async () => { + if (app === null || sessionSigner === null) { + return + } + + for (let i = 0; i < 100; i++) { + const content = bytesToHex(randomBytes(8)) + await app.as(sessionSigner).createMessage(content) + } + }, [app, sessionSigner]) + + const button = `p-2 border rounded flex` + const disabled = `bg-gray-100 text-gray-500 hover:cursor-not-allowed` + const enabledGreen = `bg-green-100 active:bg-green-300 hover:cursor-pointer hover:bg-green-200` + const enabledRed = `bg-red-100 active:bg-red-300 hover:cursor-pointer hover:bg-red-200` + const enabledYellow = `bg-yellow-100 active:bg-yellow-300 hover:cursor-pointer hover:bg-yellow-200` + + if (app === null) { + return ( +
+ + +
+ ) + } else if (isStarted) { + return ( +
+ + +
+ ) + } else { + return ( +
+ + +
+ ) + } +} diff --git a/examples/document/src/Editor.tsx b/examples/document/src/Editor.tsx new file mode 100644 index 000000000..df161d176 --- /dev/null +++ b/examples/document/src/Editor.tsx @@ -0,0 +1,60 @@ +import Quill, { EmitterSource } from "quill" +import "quill/dist/quill.snow.css" +import React, { forwardRef, useEffect, useLayoutEffect, useRef } from "react" + +type EditorProps = { + readOnly: boolean + defaultValue: any + onTextChange: (delta: any, oldDelta: any, source: EmitterSource) => void + onSelectionChange: (delta: any, oldDelta: any, source: EmitterSource) => void +} + +// Editor is an uncontrolled React component +export const Editor = forwardRef(({ readOnly, defaultValue, onTextChange, onSelectionChange }: EditorProps, ref) => { + const containerRef = useRef(null) + const defaultValueRef = useRef(defaultValue) + const onTextChangeRef = useRef(onTextChange) + const onSelectionChangeRef = useRef(onSelectionChange) + + useLayoutEffect(() => { + onTextChangeRef.current = onTextChange + onSelectionChangeRef.current = onSelectionChange + }) + + useEffect(() => { + // @ts-ignore + ref.current?.enable(!readOnly) + }, [ref, readOnly]) + + useEffect(() => { + const container = containerRef.current + if (!container || !ref) return + const editorContainer = container.appendChild(container.ownerDocument.createElement("div")) + const quill = new Quill(editorContainer, { + theme: "snow", + }) + + // @ts-ignore + ref.current = quill + + if (defaultValueRef.current) { + quill.setContents(defaultValueRef.current) + } + + quill.on(Quill.events.TEXT_CHANGE, (...args) => { + onTextChangeRef.current?.(...args) + }) + + quill.on(Quill.events.SELECTION_CHANGE, (...args) => { + onSelectionChangeRef.current?.(...args) + }) + + return () => { + // @ts-ignore + ref.current = undefined + container.innerHTML = "" + } + }, [ref]) + + return
+}) diff --git a/examples/document/src/LogStatus.tsx b/examples/document/src/LogStatus.tsx new file mode 100644 index 000000000..811350d24 --- /dev/null +++ b/examples/document/src/LogStatus.tsx @@ -0,0 +1,67 @@ +import React, { useContext, useEffect, useState } from "react" +import { bytesToHex } from "@noble/hashes/utils" + +import type { CanvasEvents } from "@canvas-js/core" +import { MessageId } from "@canvas-js/gossiplog" + +import { AppContext } from "./AppContext.js" + +export interface LogStatusProps {} + +export const LogStatus: React.FC = ({}) => { + const { app } = useContext(AppContext) + + const [root, setRoot] = useState(null) + const [heads, setHeads] = useState(null) + useEffect(() => { + if (app === null) { + return + } + + app.messageLog.tree.read((txn) => txn.getRoot()).then((root) => setRoot(`${root.level}:${bytesToHex(root.hash)}`)) + app.db + .getAll<{ id: string }>("$heads") + .then((records) => setHeads(records.map((record) => MessageId.encode(record.id)))) + + const handleCommit = ({ detail: { root, heads } }: CanvasEvents["commit"]) => { + const rootValue = `${root.level}:${bytesToHex(root.hash)}` + setRoot(rootValue) + setHeads(heads.map(MessageId.encode)) + } + + app.addEventListener("commit", handleCommit) + return () => app.removeEventListener("commit", handleCommit) + }, [app]) + + if (app === null) { + return null + } + + return ( +
+
+ Merkle root +
+
+ {root !== null ? {root} : none} +
+
+ Message heads +
+
+ {heads !== null ? ( +
    + {heads.map((head) => ( +
  • + {head.id} + (clock: {head.clock}) +
    • + ))} +
+ ) : ( + none + )} +
+
+ ) +} diff --git a/examples/document/src/SessionStatus.tsx b/examples/document/src/SessionStatus.tsx new file mode 100644 index 000000000..189484c19 --- /dev/null +++ b/examples/document/src/SessionStatus.tsx @@ -0,0 +1,85 @@ +import React, { useContext, useMemo } from "react" +import { useLiveQuery } from "@canvas-js/hooks" +import { DeriveModelTypes } from "@canvas-js/modeldb" + +import { AppContext } from "./AppContext.js" +import { AddressView } from "./components/AddressView.js" + +const sessionSchema = { + $sessions: { + message_id: "primary", + did: "string", + public_key: "string", + address: "string", + expiration: "integer?", + // $indexes: [["did"], ["public_key"]], + }, +} as const + +export interface SessionStatusProps {} + +export const SessionStatus: React.FC = ({}) => { + const { address } = useContext(AppContext) + if (address === null) { + return null + } + + return ( +
+
+ Address +
+
+ +
+
+
+ Sessions +
+ +
+ ) +} + +interface SessionListProps { + address: string +} + +const SessionList: React.FC = ({ address }) => { + const { app } = useContext(AppContext) + + const timestamp = useMemo(() => Date.now(), []) + + const results = useLiveQuery(app, "$sessions", { + where: { address, expiration: { gt: timestamp } }, + }) + + if (results === null) { + return null + } else if (results.length === 0) { + return
No sessions
+ } else { + return ( +
    + {results.map((session) => { + return ( +
  • +
    + {session.public_key} +
    + {session.expiration && session.expiration < Number.MAX_SAFE_INTEGER ? ( +
    + Expires {new Date(session.expiration).toLocaleString()} +
    + ) : ( +
    + No expiration +
    + )} +
    • + ) + })} +
+ ) + } +} diff --git a/examples/document/src/components/AddressView.tsx b/examples/document/src/components/AddressView.tsx new file mode 100644 index 000000000..e051a4a91 --- /dev/null +++ b/examples/document/src/components/AddressView.tsx @@ -0,0 +1,16 @@ +import React from "react" + +export interface AddressViewProps { + className?: string + address: string +} + +export const AddressView: React.FC = (props) => { + const className = props.className ?? "text-sm" + return ( + + {/* {props.address.slice(0, 6)}…{props.address.slice(-4)} */} + {props.address} + + ) +} diff --git a/examples/document/src/components/MultiaddrView.tsx b/examples/document/src/components/MultiaddrView.tsx new file mode 100644 index 000000000..30a7073ae --- /dev/null +++ b/examples/document/src/components/MultiaddrView.tsx @@ -0,0 +1,22 @@ +import React from "react" + +import type { PeerId } from "@libp2p/interface" +import type { Multiaddr } from "@multiformats/multiaddr" + +export interface MultiaddrViewProps { + addr: Multiaddr + peerId?: PeerId +} + +export const MultiaddrView: React.FC = (props) => { + let address = props.addr.toString() + if (props.peerId && address.endsWith(`/p2p/${props.peerId}`)) { + address = address.slice(0, address.lastIndexOf("/p2p/")) + } + + if (address.endsWith("/p2p-circuit/webrtc")) { + return /webrtc + } else { + return {address} + } +} diff --git a/examples/document/src/components/PeerIdView.tsx b/examples/document/src/components/PeerIdView.tsx new file mode 100644 index 000000000..1d523e773 --- /dev/null +++ b/examples/document/src/components/PeerIdView.tsx @@ -0,0 +1,19 @@ +import React from "react" + +import type { PeerId } from "@libp2p/interface" + +export interface PeerIdViewProps { + className?: string + peerId: PeerId +} + +export const PeerIdView: React.FC = (props) => { + const className = props.className ?? "text-sm" + const id = props.peerId.toString() + return ( + + {/* {id.slice(0, 12)}…{id.slice(-4)} */} + {id} + + ) +} diff --git a/examples/document/src/connect/ConnectEIP712Burner.tsx b/examples/document/src/connect/ConnectEIP712Burner.tsx new file mode 100644 index 000000000..2e5cd987f --- /dev/null +++ b/examples/document/src/connect/ConnectEIP712Burner.tsx @@ -0,0 +1,62 @@ +import React, { useCallback, useContext, useState } from "react" +import { Eip1193Provider, EventEmitterable } from "ethers" + +import { Eip712Signer } from "@canvas-js/chain-ethereum" + +import { AppContext } from "../AppContext.js" + +declare global { + // eslint-disable-next-line no-var + var ethereum: undefined | null | (Eip1193Provider & EventEmitterable<"accountsChanged" | "chainChanged">) +} + +export interface ConnectEIP712BurnerProps {} + +export const ConnectEIP712Burner: React.FC = ({}) => { + const { app, sessionSigner, setSessionSigner, address, setAddress } = useContext(AppContext) + + const [error, setError] = useState(null) + + const connect = useCallback(async () => { + if (app === null) { + setError(new Error("app not initialized")) + return + } + + const signer = new Eip712Signer() + const address = await signer.getDid() + setAddress(address) + setSessionSigner(signer) + }, [app]) + + const disconnect = useCallback(async () => { + setAddress(null) + setSessionSigner(null) + }, [sessionSigner]) + + if (error !== null) { + return ( +
+ {error.message} +
+ ) + } else if (address !== null && sessionSigner instanceof Eip712Signer) { + return ( + + ) + } else { + return ( + + ) + } +} diff --git a/examples/document/src/contract.ts b/examples/document/src/contract.ts new file mode 100644 index 000000000..6ca2daee5 --- /dev/null +++ b/examples/document/src/contract.ts @@ -0,0 +1,14 @@ +import type { Actions, ModelSchema } from "@canvas-js/core" + +export const models = { + documents: { + id: "primary", + content: "yjs-doc", + }, +} satisfies ModelSchema + +export const actions = { + async applyDeltaToDoc(db, delta) { + await db.yjsApplyDelta("documents", "0", delta) + }, +} satisfies Actions diff --git a/examples/document/src/index.tsx b/examples/document/src/index.tsx new file mode 100644 index 000000000..a2a1b0c67 --- /dev/null +++ b/examples/document/src/index.tsx @@ -0,0 +1,14 @@ +import React from "react" +import ReactDOM from "react-dom/client" + +import "../styles.css" + +import { App } from "./App.js" + +const root = ReactDOM.createRoot(document.getElementById("root")!) + +root.render( + + + , +) diff --git a/examples/document/styles.css b/examples/document/styles.css new file mode 100644 index 000000000..fd0d1983e --- /dev/null +++ b/examples/document/styles.css @@ -0,0 +1,17 @@ +@tailwind base; +@tailwind components; +@tailwind utilities; + +html, +body, +#root { + margin: 0; + width: 100%; + height: 100%; +} + +main { + height: 100%; + max-width: 960px; + padding: 2em; +} diff --git a/examples/document/tailwind.config.js b/examples/document/tailwind.config.js new file mode 100644 index 000000000..719ea4b3d --- /dev/null +++ b/examples/document/tailwind.config.js @@ -0,0 +1,4 @@ +export default { + content: ["./src/**/*.{html,js,tsx}"], + plugins: [], +} diff --git a/examples/document/tsconfig.json b/examples/document/tsconfig.json new file mode 100644 index 000000000..d21902116 --- /dev/null +++ b/examples/document/tsconfig.json @@ -0,0 +1,19 @@ +{ + "extends": "../../tsconfig.base.json", + "include": ["src"], + "compilerOptions": { + "types": ["vite/client"], + "rootDir": "src", + "outDir": "lib", + "esModuleInterop": true, + "jsx": "react" + }, + "references": [ + { "path": "../../packages/core" }, + { "path": "../../packages/chain-ethereum" }, + { "path": "../../packages/gossiplog" }, + { "path": "../../packages/hooks" }, + { "path": "../../packages/interfaces" }, + { "path": "../../packages/modeldb" } + ] +} diff --git a/examples/document/vite.config.js b/examples/document/vite.config.js new file mode 100644 index 000000000..8cafc7b4d --- /dev/null +++ b/examples/document/vite.config.js @@ -0,0 +1,28 @@ +import { defineConfig } from "vite" +import wasm from "vite-plugin-wasm" +import { nodePolyfills } from "vite-plugin-node-polyfills" + +export default defineConfig({ + // ...other config settings + plugins: [nodePolyfills({ globals: { Buffer: true } }), wasm()], + server: { + headers: { + "Cross-Origin-Opener-Policy": "same-origin", + "Cross-Origin-Embedder-Policy": "require-corp", + }, + }, + build: { + minify: false, + target: "es2022", + }, + optimizeDeps: { + exclude: ["@sqlite.org/sqlite-wasm", "quickjs-emscripten"], + esbuildOptions: { + // Node.js global to browser globalThis + define: { + global: "globalThis", + }, + }, + }, + publicDir: 'public' +}) diff --git a/package-lock.json b/package-lock.json index e912151ef..9ba7b9e95 100644 --- a/package-lock.json +++ b/package-lock.json @@ -153,6 +153,7 @@ } }, "examples/document": { + "name": "@canvas-js/document", "version": "0.14.0-next.1", "dependencies": { "@canvas-js/chain-atp": "0.14.0-next.1", @@ -167,32 +168,19 @@ "@canvas-js/hooks": "0.14.0-next.1", "@canvas-js/interfaces": "0.14.0-next.1", "@canvas-js/modeldb-sqlite-wasm": "0.14.0-next.1", - "@cosmjs/encoding": "^0.32.3", - "@farcaster/auth-kit": "^0.6.0", - "@farcaster/frame-sdk": "^0.0.26", - "@keplr-wallet/types": "^0.11.64", - "@libp2p/interface": "^2.5.0", - "@magic-ext/auth": "^4.3.2", - "@metamask/providers": "^14.0.2", - "@multiformats/multiaddr": "^12.3.5", - "@noble/hashes": "^1.7.1", - "@polkadot/extension-dapp": "^0.46.9", - "@terra-money/feather.js": "^2.0.4", "@types/react": "^18.3.9", "@types/react-dom": "^18.3.0", "buffer": "^6.0.3", "comlink": "^4.4.1", "ethers": "^6.13.5", "idb": "^8.0.2", - "magic-sdk": "^21.5.0", "multiformats": "^13.3.2", "near-api-js": "^2.1.4", "process": "^0.11.10", + "quill": "^2.0.3", "react": "^18.3.1", "react-dom": "^18.3.1", - "viem": "^2.22.21", - "vite-plugin-wasm": "^3.3.0", - "web3": "^4.10.0" + "vite-plugin-wasm": "^3.3.0" }, "devDependencies": { "autoprefixer": "^10.4.19", @@ -203,17 +191,6 @@ "vite-plugin-node-polyfills": "^0.22.0" } }, - "examples/document/node_modules/@noble/hashes": { - "version": "1.7.1", - "resolved": "https://registry.npmjs.org/@noble/hashes/-/hashes-1.7.1.tgz", - "integrity": "sha512-B8XBPsn4vT/KJAGqDzbwztd+6Yte3P4V7iafm24bxgDe/mlRuK6xmWPuCNrKt2vDafZ8MfJLlchDG/vYafQEjQ==", - "engines": { - "node": "^14.21.3 || >=16" - }, - "funding": { - "url": "https://paulmillr.com/funding/" - } - }, "examples/encrypted-chat": { "name": "@canvas-js/example-chat-encrypted", "version": "0.14.0-next.1", @@ -19275,7 +19252,6 @@ }, "node_modules/fast-diff": { "version": "1.3.0", - "dev": true, "license": "Apache-2.0" }, "node_modules/fast-fifo": { @@ -23793,6 +23769,11 @@ "dev": true, "license": "MIT" }, + "node_modules/lodash.clonedeep": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.clonedeep/-/lodash.clonedeep-4.5.0.tgz", + "integrity": "sha512-H5ZhCF25riFd9uB5UCkVKo61m3S/xZk1x4wA6yp/L3RFP6Z/eHH1ymQcGLo7J3GMPfm0V/7m1tryHuGVxpqEBQ==" + }, "node_modules/lodash.debounce": { "version": "4.0.8", "license": "MIT", @@ -26633,6 +26614,11 @@ "dev": true, "license": "(MIT AND Zlib)" }, + "node_modules/parchment": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/parchment/-/parchment-3.0.0.tgz", + "integrity": "sha512-HUrJFQ/StvgmXRcQ1ftY6VEZUq3jA2t9ncFN4F84J/vN0/FPpQF+8FKXb3l6fLces6q0uOHj6NJn+2xvZnxO6A==" + }, "node_modules/parent-module": { "version": "1.0.1", "license": "MIT", @@ -28089,6 +28075,33 @@ "@jitl/quickjs-ffi-types": "0.31.0" } }, + "node_modules/quill": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/quill/-/quill-2.0.3.tgz", + "integrity": "sha512-xEYQBqfYx/sfb33VJiKnSJp8ehloavImQ2A6564GAbqG55PGw1dAWUn1MUbQB62t0azawUS2CZZhWCjO8gRvTw==", + "dependencies": { + "eventemitter3": "^5.0.1", + "lodash-es": "^4.17.21", + "parchment": "^3.0.0", + "quill-delta": "^5.1.0" + }, + "engines": { + "npm": ">=8.2.3" + } + }, + "node_modules/quill-delta": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/quill-delta/-/quill-delta-5.1.0.tgz", + "integrity": "sha512-X74oCeRI4/p0ucjb5Ma8adTXd9Scumz367kkMK5V/IatcX6A0vlgLgKbzXWy5nZmCGeNJm2oQX0d2Eqj+ZIlCA==", + "dependencies": { + "fast-diff": "^1.3.0", + "lodash.clonedeep": "^4.5.0", + "lodash.isequal": "^4.5.0" + }, + "engines": { + "node": ">= 12.0.0" + } + }, "node_modules/race-event": { "version": "1.3.0", "license": "Apache-2.0 OR MIT" From 63c6c6c3146d64cb27d795459100ea246fc9eef8 Mon Sep 17 00:00:00 2001 From: Bob Webb Date: Wed, 5 Mar 2025 18:07:55 +0100 Subject: [PATCH 21/36] autoformatting index.html --- examples/document/index.html | 40 ++++++++++++++++++------------------ 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/examples/document/index.html b/examples/document/index.html index 6ee9fc7fc..0aa3300d3 100644 --- a/examples/document/index.html +++ b/examples/document/index.html @@ -1,31 +1,31 @@ - - - - - Canvas Chat - - - - -
- + }' /> + + + +
+ + From c51d70663861172285be5fbc6a2a1df8e8b65154 Mon Sep 17 00:00:00 2001 From: Bob Webb Date: Mon, 10 Mar 2025 16:28:50 +0100 Subject: [PATCH 22/36] don't store ydoc states in modeldb, pass isAppend to message handlers, update tests --- packages/core/src/Canvas.ts | 21 +++- packages/core/src/ExecutionContext.ts | 31 ------ packages/core/src/runtime/AbstractRuntime.ts | 109 ++++++++++--------- packages/core/test/snapshot.test.ts | 13 +-- packages/core/test/yjs.test.ts | 19 +--- packages/gossiplog/src/AbstractGossipLog.ts | 11 +- 6 files changed, 91 insertions(+), 113 deletions(-) diff --git a/packages/core/src/Canvas.ts b/packages/core/src/Canvas.ts index 3d496f6ae..41d3f5755 100644 --- a/packages/core/src/Canvas.ts +++ b/packages/core/src/Canvas.ts @@ -5,7 +5,16 @@ import type pg from "pg" import type { SqlStorage } from "@cloudflare/workers-types" import { bytesToHex } from "@noble/hashes/utils" -import { Signature, Action, Message, Snapshot, SessionSigner, SignerCache, MessageType } from "@canvas-js/interfaces" +import { + Signature, + Action, + Message, + Snapshot, + SessionSigner, + SignerCache, + MessageType, + Updates, +} from "@canvas-js/interfaces" import { AbstractModelDB, Model, ModelSchema, Effect } from "@canvas-js/modeldb" import { SIWESigner } from "@canvas-js/chain-ethereum" import { AbstractGossipLog, GossipLogEvents, NetworkClient, SignedMessage } from "@canvas-js/gossiplog" @@ -17,7 +26,7 @@ import target from "#target" import type { Contract, Actions, ActionImplementation, ModelAPI, DeriveModelTypes } from "./types.js" import { Runtime, createRuntime } from "./runtime/index.js" -import { ActionRecord, updatesToEffects } from "./runtime/AbstractRuntime.js" +import { ActionRecord } from "./runtime/AbstractRuntime.js" import { validatePayload } from "./schema.js" import { createSnapshot, hashSnapshot } from "./snapshot.js" import { topicPattern } from "./utils.js" @@ -184,9 +193,7 @@ export class Canvas< const record = { did } effects.push({ operation: "set", model: "$dids", value: record }) } else if (message.payload.type === "updates") { - for (const effect of await updatesToEffects(message.payload, db)) { - effects.push(effect) - } + runtime.handleUpdates(message as Message, true) } start = id } @@ -459,6 +466,10 @@ export class Canvas< } } + public getYDoc(model: string, key: string) { + return this.runtime.getYDoc(model, key) + } + public async createSnapshot(): Promise { return createSnapshot(this) } diff --git a/packages/core/src/ExecutionContext.ts b/packages/core/src/ExecutionContext.ts index c740c7546..937b9c83a 100644 --- a/packages/core/src/ExecutionContext.ts +++ b/packages/core/src/ExecutionContext.ts @@ -176,35 +176,4 @@ export class ExecutionContext { return null } } - - public async generateAdditionalUpdates(): Promise { - const updates = [] - for (const [model, modelCalls] of Object.entries(this.yjsCalls)) { - for (const [key, calls] of Object.entries(modelCalls)) { - const doc = (await this.getYDoc(model, key)) || new Y.Doc() - // get the initial state of the document - const beforeState = Y.encodeStateAsUpdate(doc) - for (const call of calls) { - if (call.call === "insert") { - doc.getText().insert(call.index, call.content) - } else if (call.call === "delete") { - doc.getText().delete(call.index, call.length) - } else if (call.call === "applyDelta") { - doc.getText().applyDelta(call.delta.ops) - } else { - throw new Error("unexpected call type") - } - } - // diff the document with the initial state - const afterState = Y.encodeStateAsUpdate(doc) - const diff = Y.diffUpdate(afterState, Y.encodeStateVectorFromUpdate(beforeState)) - updates.push({ model, key, diff }) - } - } - if (updates.length > 0) { - return [{ type: "updates", updates }] - } else { - return [] - } - } } diff --git a/packages/core/src/runtime/AbstractRuntime.ts b/packages/core/src/runtime/AbstractRuntime.ts index 6e379ef40..7075f78a0 100644 --- a/packages/core/src/runtime/AbstractRuntime.ts +++ b/packages/core/src/runtime/AbstractRuntime.ts @@ -2,7 +2,16 @@ import * as cbor from "@ipld/dag-cbor" import { logger } from "@libp2p/logger" import * as Y from "yjs" -import type { Action, Session, Snapshot, SignerCache, Awaitable, MessageType, Updates } from "@canvas-js/interfaces" +import type { + Action, + Session, + Snapshot, + SignerCache, + Awaitable, + MessageType, + Updates, + Message, +} from "@canvas-js/interfaces" import { AbstractModelDB, Effect, ModelSchema } from "@canvas-js/modeldb" import { GossipLogConsumer, MAX_MESSAGE_ID, AbstractGossipLog, SignedMessage } from "@canvas-js/gossiplog" @@ -75,13 +84,6 @@ export abstract class AbstractRuntime { ) { // not valid throw new Error("yjs-doc tables must have two columns, one of which is 'id'") - } else { - // this table stores the current state of the Yjs document - // we just need one entry per document because updates are commutative - outputSchema[`${modelName}:state`] = { - id: "primary", - content: "bytes", - } } } else { outputSchema[modelName] = modelSchema @@ -104,6 +106,8 @@ export abstract class AbstractRuntime { public readonly additionalUpdates = new Map() + private readonly docs: Record> = {} + protected readonly log = logger("canvas:runtime") #db: AbstractModelDB | null = null @@ -128,15 +132,15 @@ export abstract class AbstractRuntime { const handleSnapshot = this.handleSnapshot.bind(this) const handleUpdates = this.handleUpdates.bind(this) - return async function (this: AbstractGossipLog, signedMessage) { + return async function (this: AbstractGossipLog, signedMessage, isAppend: boolean) { if (isSession(signedMessage)) { return await handleSession(signedMessage) } else if (isAction(signedMessage)) { - return await handleAction(signedMessage, this) + return await handleAction(signedMessage, this, isAppend) } else if (isSnapshot(signedMessage)) { return await handleSnapshot(signedMessage, this) } else if (isUpdates(signedMessage)) { - return await handleUpdates(signedMessage) + return handleUpdates(signedMessage.message, isAppend) } else { throw new Error("invalid message payload type") } @@ -194,7 +198,11 @@ export abstract class AbstractRuntime { await this.db.apply(effects) } - private async handleAction(signedMessage: SignedMessage, messageLog: AbstractGossipLog) { + private async handleAction( + signedMessage: SignedMessage, + messageLog: AbstractGossipLog, + isAppend: boolean, + ) { const { id, signature, message } = signedMessage const { did, name, context } = message.payload @@ -277,51 +285,52 @@ export abstract class AbstractRuntime { throw err } - this.additionalUpdates.set(id, await executionContext.generateAdditionalUpdates()) + if (isAppend) { + const updates = [] + for (const [model, modelCalls] of Object.entries(executionContext.yjsCalls)) { + for (const [key, calls] of Object.entries(modelCalls)) { + const doc = this.getYDoc(model, key) + // get the initial state of the document + const beforeState = Y.encodeStateAsUpdate(doc) + for (const call of calls) { + if (call.call === "insert") { + doc.getText().insert(call.index, call.content) + } else if (call.call === "delete") { + doc.getText().delete(call.index, call.length) + } else if (call.call === "applyDelta") { + doc.getText().applyDelta(call.delta.ops) + } else { + throw new Error("unexpected call type") + } + } + // diff the document with the initial state + const afterState = Y.encodeStateAsUpdate(doc) + const diff = Y.diffUpdate(afterState, Y.encodeStateVectorFromUpdate(beforeState)) + updates.push({ model, key, diff }) + } + } + if (updates.length > 0) { + this.additionalUpdates.set(id, [{ type: "updates", updates }]) + } else { + this.additionalUpdates.set(id, []) + } + } return result } - private async handleUpdates(signedMessage: SignedMessage) { - const effects = await updatesToEffects(signedMessage.message.payload, this.db) - await this.db.apply(effects) + public getYDoc(model: string, key: string) { + this.docs[model] ||= {} + this.docs[model][key] ||= new Y.Doc() + return this.docs[model][key] } -} - -export const updatesToEffects = async (payload: Updates, db: AbstractModelDB) => { - const updatedEntries: Record> = {} - - for (const { model, key, diff } of payload.updates) { - let doc = (updatedEntries[model] || {})[key] - if (!doc) { - const existingStateEntries = await db.query<{ id: string; content: Uint8Array }>(`${model}:state`, { - where: { id: key }, - limit: 1, - }) - doc = new Y.Doc() - if (existingStateEntries.length > 0) { - Y.applyUpdate(doc, existingStateEntries[0].content) + public handleUpdates(message: Message, isAppend: boolean) { + if (!isAppend) { + for (const { model, key, diff } of message.payload.updates) { + // apply the diff to the doc + Y.applyUpdate(this.getYDoc(model, key), diff) } } - - // apply the diff to the doc - Y.applyUpdate(doc, diff) - - updatedEntries[model] = { ...updatedEntries[model], [key]: doc } - } - - const effects: Effect[] = [] - for (const [model, entries] of Object.entries(updatedEntries)) { - for (const [key, doc] of Object.entries(entries)) { - const diff = Y.encodeStateAsUpdate(doc) - - effects.push({ - model: `${model}:state`, - operation: "set", - value: { id: key, content: diff }, - }) - } } - return effects } diff --git a/packages/core/test/snapshot.test.ts b/packages/core/test/snapshot.test.ts index 76c8374e6..897797556 100644 --- a/packages/core/test/snapshot.test.ts +++ b/packages/core/test/snapshot.test.ts @@ -1,5 +1,4 @@ import test from "ava" -import * as Y from "yjs" import { Canvas, Config } from "@canvas-js/core" @@ -65,9 +64,7 @@ test("snapshot persists data across apps", async (t) => { t.is(await app2.db.get("posts", "d"), null) t.is(await app2.db.get("posts", "e"), null) - const docDiff1 = await app2.db.get("documents:state", "e") - const doc1 = new Y.Doc() - Y.applyUpdate(doc1, docDiff1!.content) + const doc1 = app2.getYDoc("documents", "e") t.is(doc1.getText().toJSON(), "Hello") await app2.actions.createPost({ id: "a", content: "1" }) @@ -86,9 +83,7 @@ test("snapshot persists data across apps", async (t) => { t.is(await app2.db.get("posts", "d"), null) t.is((await app2.db.get("posts", "e"))?.content, "3") t.is((await app2.db.get("posts", "f"))?.content, "4") - const docDiff2 = await app2.db.get("documents:state", "e") - const doc2 = new Y.Doc() - Y.applyUpdate(doc2, docDiff2!.content) + const doc2 = app2.getYDoc("documents", "e") t.is(doc2.getText().toJSON(), "Hello?") // snapshot a second time @@ -102,9 +97,7 @@ test("snapshot persists data across apps", async (t) => { t.is((await app3.db.get("posts", "e"))?.content, "3") t.is((await app3.db.get("posts", "f"))?.content, "4") t.is(await app3.db.get("posts", "g"), null) - const docDiff3 = await app3.db.get("documents:state", "e") - const doc3 = new Y.Doc() - Y.applyUpdate(doc3, docDiff3!.content) + const doc3 = app3.getYDoc("documents", "e") t.is(doc3.getText().toJSON(), "Hello?") const [clock3] = await app3.messageLog.getClock() diff --git a/packages/core/test/yjs.test.ts b/packages/core/test/yjs.test.ts index a0fb49f6c..30e1cda3d 100644 --- a/packages/core/test/yjs.test.ts +++ b/packages/core/test/yjs.test.ts @@ -1,7 +1,6 @@ import { SIWESigner } from "@canvas-js/chain-ethereum" import test, { ExecutionContext } from "ava" import { Canvas } from "@canvas-js/core" -import * as Y from "yjs" const contract = ` export const models = { @@ -24,12 +23,6 @@ export const actions = { }; ` -async function stringifyDoc(app: Canvas, key: string) { - const doc = new Y.Doc() - Y.applyUpdate(doc, (await app.db.get("articles:state", key))!.content) - return doc.getText().toJSON() -} - const init = async (t: ExecutionContext) => { const signer = new SIWESigner() const app = await Canvas.initialize({ @@ -51,30 +44,30 @@ test("apply an action and read a record from the database", async (t) => { t.log(`applied action ${id}`) await app1.actions.insertIntoDoc(id, 0, "Hello, world") - t.is(await stringifyDoc(app1, id), "Hello, world") + t.is(app1.getYDoc("articles", id).getText().toJSON(), "Hello, world") // create another app const { app: app2 } = await init(t) // sync the apps await app1.messageLog.serve((s) => app2.messageLog.sync(s)) - t.is(await stringifyDoc(app2, id), "Hello, world") + t.is(app2.getYDoc("articles", id).getText().toJSON(), "Hello, world") // insert ! into app1 await app1.actions.insertIntoDoc(id, 12, "!") - t.is(await stringifyDoc(app1, id), "Hello, world!") + t.is(app1.getYDoc("articles", id).getText().toJSON(), "Hello, world!") // insert ? into app2 await app2.actions.insertIntoDoc(id, 12, "?") - t.is(await stringifyDoc(app2, id), "Hello, world?") + t.is(app2.getYDoc("articles", id).getText().toJSON(), "Hello, world?") // sync app2 -> app1 await app2.messageLog.serve((s) => app1.messageLog.sync(s)) - const app1MergedText = await stringifyDoc(app1, id) + const app1MergedText = app1.getYDoc("articles", id).getText().toJSON() // sync app1 -> app2 await app1.messageLog.serve((s) => app2.messageLog.sync(s)) - const app2MergedText = await stringifyDoc(app2, id) + const app2MergedText = app2.getYDoc("articles", id).getText().toJSON() // both apps should now have converged t.is(app1MergedText, app2MergedText) diff --git a/packages/gossiplog/src/AbstractGossipLog.ts b/packages/gossiplog/src/AbstractGossipLog.ts index b6dca1424..919e3caf0 100644 --- a/packages/gossiplog/src/AbstractGossipLog.ts +++ b/packages/gossiplog/src/AbstractGossipLog.ts @@ -26,6 +26,7 @@ import { gossiplogTopicPattern } from "./utils.js" export type GossipLogConsumer = ( this: AbstractGossipLog, signedMessage: SignedMessage, + isAppend: boolean, ) => Awaitable export interface GossipLogInit { @@ -131,7 +132,8 @@ export abstract class AbstractGossipLog extends const { signature, message, branch } = record const signedMessage = this.encode(signature, message, { branch }) assert(signedMessage.id === id) - await this.#apply.apply(this, [signedMessage]) + // TODO: should isAppend be true or false? + await this.#apply.apply(this, [signedMessage, true]) } }) } @@ -260,7 +262,7 @@ export abstract class AbstractGossipLog extends const signedMessage = this.encode(signature, message) this.log("appending message %s at clock %d with parents %o", signedMessage.id, clock, parents) - const applyResult = await this.apply(txn, signedMessage) + const applyResult = await this.apply(txn, signedMessage, true) root = applyResult.root heads = applyResult.heads @@ -298,7 +300,7 @@ export abstract class AbstractGossipLog extends return null } - return await this.apply(txn, signedMessage) + return await this.apply(txn, signedMessage, false) }) if (result !== null) { @@ -311,6 +313,7 @@ export abstract class AbstractGossipLog extends private async apply( txn: ReadWriteTransaction, signedMessage: SignedMessage, + isAppend: boolean, ): Promise<{ root: Node; heads: string[]; result: Result }> { const { id, signature, message, key, value } = signedMessage this.log.trace("applying %s %O", id, message) @@ -328,7 +331,7 @@ export abstract class AbstractGossipLog extends const branch = await this.getBranch(id, parentMessageRecords) signedMessage.branch = branch - const result = await this.#apply.apply(this, [signedMessage]) + const result = await this.#apply.apply(this, [signedMessage, isAppend]) const hash = toString(hashEntry(key, value), "hex") From 6ef479dde30e80da97ec6928fdb4451a0a79c2fc Mon Sep 17 00:00:00 2001 From: Bob Webb Date: Tue, 11 Mar 2025 15:28:12 +0100 Subject: [PATCH 23/36] create a class to manage the stored yjs documents, don't apply a quill delta that has already been applied --- examples/document/src/App.tsx | 31 +++--- packages/core/src/Canvas.ts | 5 +- packages/core/src/DocumentStore.ts | 106 +++++++++++++++++++ packages/core/src/runtime/AbstractRuntime.ts | 42 +++----- packages/core/src/runtime/ContractRuntime.ts | 1 + 5 files changed, 140 insertions(+), 45 deletions(-) create mode 100644 packages/core/src/DocumentStore.ts diff --git a/examples/document/src/App.tsx b/examples/document/src/App.tsx index 2df30350b..c3c0ab464 100644 --- a/examples/document/src/App.tsx +++ b/examples/document/src/App.tsx @@ -12,7 +12,6 @@ import { useCanvas, useLiveQuery } from "@canvas-js/hooks" import { AuthKitProvider } from "@farcaster/auth-kit" import { JsonRpcProvider } from "ethers" import Quill from "quill/core" -import * as Y from "yjs" import { AppContext } from "./AppContext.js" import { ControlPanel } from "./ControlPanel.js" @@ -42,6 +41,8 @@ export const App: React.FC<{}> = ({}) => { const [sessionSigner, setSessionSigner] = useState(null) const [address, setAddress] = useState(null) + const [cursor, setCursor] = useState("documents/0/") + const quillRef = useRef() const topicRef = useRef(topic) @@ -60,20 +61,20 @@ export const App: React.FC<{}> = ({}) => { ], }) - const existingStateEntries = useLiveQuery(app, "documents:state", { - where: { id: "0" }, - limit: 1, - }) as { id: string; content: Uint8Array }[] + // TODO: encapsulate this in a library function to ship with canvas-js/hooks + const results = useLiveQuery(app, "$document_operations", { + where: { id: { gt: cursor, lt: "documents/1" }, isAppend: false }, + }) as { id: string; key: string; data: string }[] | null useEffect(() => { - if (existingStateEntries && existingStateEntries.length > 0) { - const doc = new Y.Doc() - Y.applyUpdate(doc, existingStateEntries[0].content) - // doc.getText().toJSON() - console.log("doc.getText().toJSON():", doc.getText().toJSON()) - quillRef.current?.setText(doc.getText().toJSON(), "silent") + if (!results) return + for (const message of results) { + const { data } = message + quillRef.current?.updateContents(data) } - }, [existingStateEntries]) + // set the cursor to the id of the last item + if (results.length > 0) setCursor(results[results.length - 1].id) + }, [results]) return ( @@ -90,8 +91,10 @@ export const App: React.FC<{}> = ({}) => { readOnly={false} defaultValue={null} onSelectionChange={() => {}} - onTextChange={async (delta) => { - await app.actions.applyDeltaToDoc(JSON.stringify(delta)) + onTextChange={async (delta, oldContents, source) => { + if (source === "user") { + await app.actions.applyDeltaToDoc(JSON.stringify(delta)) + } }} /> diff --git a/packages/core/src/Canvas.ts b/packages/core/src/Canvas.ts index 41d3f5755..556e47bae 100644 --- a/packages/core/src/Canvas.ts +++ b/packages/core/src/Canvas.ts @@ -30,6 +30,7 @@ import { ActionRecord } from "./runtime/AbstractRuntime.js" import { validatePayload } from "./schema.js" import { createSnapshot, hashSnapshot } from "./snapshot.js" import { topicPattern } from "./utils.js" +import { YTextEvent } from "yjs" export type { Model } from "@canvas-js/modeldb" export type { PeerId } from "@libp2p/interface" @@ -152,6 +153,8 @@ export class Canvas< await messageLog.append(config.snapshot) } + await runtime.loadSavedDocuments() + const app = new Canvas(signers, messageLog, runtime) // Check to see if the $actions table is empty and populate it if necessary @@ -192,8 +195,6 @@ export class Canvas< app.log("indexing user %s (did: %s)", publicKey, did) const record = { did } effects.push({ operation: "set", model: "$dids", value: record }) - } else if (message.payload.type === "updates") { - runtime.handleUpdates(message as Message, true) } start = id } diff --git a/packages/core/src/DocumentStore.ts b/packages/core/src/DocumentStore.ts new file mode 100644 index 000000000..2c88c87bd --- /dev/null +++ b/packages/core/src/DocumentStore.ts @@ -0,0 +1,106 @@ +import { Message, Updates } from "@canvas-js/interfaces" +import { AbstractModelDB, ModelSchema } from "@canvas-js/modeldb" +import * as Y from "yjs" +import { YjsCall } from "./ExecutionContext.js" + +type Delta = Y.YTextEvent["changes"]["delta"] + +function getDeltaForYText(ytext: Y.Text, fn: () => void): Delta { + let delta: Delta | null = null + + const handler = (event: Y.YTextEvent) => { + delta = event.changes.delta + } + + ytext.observe(handler) + fn() + ytext.unobserve(handler) + return delta || [] +} + +export class DocumentStore { + private documents: Record> = {} + + public static schema = { + $document_operations: { + // ${model}/${key}/${messageId} + id: "primary", + // applyDelta, insert or delete + data: "json", + // yjs document diff + diff: "bytes", + isAppend: "boolean", + }, + } satisfies ModelSchema + + public getYDoc(model: string, key: string) { + this.documents[model] ||= {} + this.documents[model][key] ||= new Y.Doc() + return this.documents[model][key] + } + + public async loadSavedDocuments(db: AbstractModelDB) { + // iterate over the past document operations + // and create the yjs documents + for await (const operation of db.iterate("$document_operations")) { + const [model, key, _messageId] = operation.id.split("/") + const doc = this.getYDoc(model, key) + Y.applyUpdate(doc, operation.diff) + } + } + + public async applyYjsCalls(db: AbstractModelDB, model: string, key: string, messageId: string, calls: YjsCall[]) { + const doc = this.getYDoc(model, key) + + // get the initial state of the document + const beforeState = Y.encodeStateAsUpdate(doc) + + const delta = getDeltaForYText(doc.getText(), () => { + for (const call of calls) { + if (call.call === "insert") { + doc.getText().insert(call.index, call.content) + } else if (call.call === "delete") { + doc.getText().delete(call.index, call.length) + } else if (call.call === "applyDelta") { + // TODO: do we actually need to call sanitize here? + doc.getText().applyDelta(call.delta.ops, { sanitize: true }) + } else { + throw new Error("unexpected call type") + } + } + }) + + // diff the document with the initial state + const afterState = Y.encodeStateAsUpdate(doc) + const diff = Y.diffUpdate(afterState, Y.encodeStateVectorFromUpdate(beforeState)) + + await db.set(`$document_operations`, { + id: `${model}/${key}/${messageId}`, + key, + data: delta || [], + diff, + isAppend: true, + }) + + return { model, key, diff } + } + + public async consumeUpdatesMessage(db: AbstractModelDB, message: Message, id: string) { + for (const { model, key, diff } of message.payload.updates) { + // apply the diff to the doc + const doc = this.getYDoc(model, key) + const delta = getDeltaForYText(doc.getText(), () => { + Y.applyUpdate(doc, diff) + }) + + // save the observed update to the db + await db.set(`$document_operations`, { + id: `${model}/${key}/${id}`, + key, + data: delta, + diff, + isAppend: false, + }) + } + } +} diff --git a/packages/core/src/runtime/AbstractRuntime.ts b/packages/core/src/runtime/AbstractRuntime.ts index 7075f78a0..68d9c03a2 100644 --- a/packages/core/src/runtime/AbstractRuntime.ts +++ b/packages/core/src/runtime/AbstractRuntime.ts @@ -19,6 +19,7 @@ import { assert } from "@canvas-js/utils" import { ExecutionContext, getKeyHash } from "../ExecutionContext.js" import { isAction, isSession, isSnapshot, isUpdates } from "../utils.js" +import { DocumentStore } from "../DocumentStore.js" export type EffectRecord = { key: string; value: Uint8Array | null; branch: number; clock: number } @@ -96,6 +97,7 @@ export abstract class AbstractRuntime { ...AbstractRuntime.actionsModel, ...AbstractRuntime.effectsModel, ...AbstractRuntime.usersModel, + ...DocumentStore.schema, } } @@ -106,10 +108,9 @@ export abstract class AbstractRuntime { public readonly additionalUpdates = new Map() - private readonly docs: Record> = {} - protected readonly log = logger("canvas:runtime") #db: AbstractModelDB | null = null + #documentStore = new DocumentStore() protected constructor() {} @@ -140,7 +141,7 @@ export abstract class AbstractRuntime { } else if (isSnapshot(signedMessage)) { return await handleSnapshot(signedMessage, this) } else if (isUpdates(signedMessage)) { - return handleUpdates(signedMessage.message, isAppend) + return await handleUpdates(signedMessage.id, signedMessage.message, isAppend) } else { throw new Error("invalid message payload type") } @@ -289,24 +290,8 @@ export abstract class AbstractRuntime { const updates = [] for (const [model, modelCalls] of Object.entries(executionContext.yjsCalls)) { for (const [key, calls] of Object.entries(modelCalls)) { - const doc = this.getYDoc(model, key) - // get the initial state of the document - const beforeState = Y.encodeStateAsUpdate(doc) - for (const call of calls) { - if (call.call === "insert") { - doc.getText().insert(call.index, call.content) - } else if (call.call === "delete") { - doc.getText().delete(call.index, call.length) - } else if (call.call === "applyDelta") { - doc.getText().applyDelta(call.delta.ops) - } else { - throw new Error("unexpected call type") - } - } - // diff the document with the initial state - const afterState = Y.encodeStateAsUpdate(doc) - const diff = Y.diffUpdate(afterState, Y.encodeStateVectorFromUpdate(beforeState)) - updates.push({ model, key, diff }) + const update = await this.#documentStore.applyYjsCalls(this.db, model, key, id, calls) + updates.push(update) } } if (updates.length > 0) { @@ -320,17 +305,16 @@ export abstract class AbstractRuntime { } public getYDoc(model: string, key: string) { - this.docs[model] ||= {} - this.docs[model][key] ||= new Y.Doc() - return this.docs[model][key] + return this.#documentStore.getYDoc(model, key) + } + + public async loadSavedDocuments() { + await this.#documentStore.loadSavedDocuments(this.db) } - public handleUpdates(message: Message, isAppend: boolean) { + public async handleUpdates(id: string, message: Message, isAppend: boolean) { if (!isAppend) { - for (const { model, key, diff } of message.payload.updates) { - // apply the diff to the doc - Y.applyUpdate(this.getYDoc(model, key), diff) - } + return await this.#documentStore.consumeUpdatesMessage(this.db, message, id) } } } diff --git a/packages/core/src/runtime/ContractRuntime.ts b/packages/core/src/runtime/ContractRuntime.ts index 29561e42e..8f3a8694e 100644 --- a/packages/core/src/runtime/ContractRuntime.ts +++ b/packages/core/src/runtime/ContractRuntime.ts @@ -7,6 +7,7 @@ import { assert, mapValues } from "@canvas-js/utils" import { ExecutionContext } from "../ExecutionContext.js" import { AbstractRuntime } from "./AbstractRuntime.js" +import { DocumentStore } from "../DocumentStore.js" export class ContractRuntime extends AbstractRuntime { public static async init( From 3e67b8f283653f112cf4841b8f4842a76d234b6d Mon Sep 17 00:00:00 2001 From: Bob Webb Date: Tue, 11 Mar 2025 16:04:52 +0100 Subject: [PATCH 24/36] create reusable useDelta hook --- examples/document/src/App.tsx | 24 ++++++------------------ examples/document/src/useDelta.ts | 29 +++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 18 deletions(-) create mode 100644 examples/document/src/useDelta.ts diff --git a/examples/document/src/App.tsx b/examples/document/src/App.tsx index c3c0ab464..b65203cfa 100644 --- a/examples/document/src/App.tsx +++ b/examples/document/src/App.tsx @@ -1,4 +1,4 @@ -import React, { useEffect, useRef, useState } from "react" +import React, { useRef, useState } from "react" import type { SessionSigner } from "@canvas-js/interfaces" import { Eip712Signer, SIWESigner, SIWFSigner } from "@canvas-js/chain-ethereum" @@ -7,7 +7,7 @@ import { CosmosSigner } from "@canvas-js/chain-cosmos" import { SolanaSigner } from "@canvas-js/chain-solana" import { SubstrateSigner } from "@canvas-js/chain-substrate" -import { useCanvas, useLiveQuery } from "@canvas-js/hooks" +import { useCanvas } from "@canvas-js/hooks" import { AuthKitProvider } from "@farcaster/auth-kit" import { JsonRpcProvider } from "ethers" @@ -21,6 +21,7 @@ import { ConnectionStatus } from "./ConnectionStatus.js" import { LogStatus } from "./LogStatus.js" import * as contract from "./contract.js" import { Editor } from "./Editor.js" +import { useDelta } from "./useDelta.js" export const topic = "document-example.canvas.xyz" @@ -41,8 +42,6 @@ export const App: React.FC<{}> = ({}) => { const [sessionSigner, setSessionSigner] = useState(null) const [address, setAddress] = useState(null) - const [cursor, setCursor] = useState("documents/0/") - const quillRef = useRef() const topicRef = useRef(topic) @@ -61,20 +60,9 @@ export const App: React.FC<{}> = ({}) => { ], }) - // TODO: encapsulate this in a library function to ship with canvas-js/hooks - const results = useLiveQuery(app, "$document_operations", { - where: { id: { gt: cursor, lt: "documents/1" }, isAppend: false }, - }) as { id: string; key: string; data: string }[] | null - - useEffect(() => { - if (!results) return - for (const message of results) { - const { data } = message - quillRef.current?.updateContents(data) - } - // set the cursor to the id of the last item - if (results.length > 0) setCursor(results[results.length - 1].id) - }, [results]) + useDelta<(typeof contract)["models"]>(app, "documents", "0", (deltas) => { + quillRef.current?.updateContents(deltas) + }) return ( diff --git a/examples/document/src/useDelta.ts b/examples/document/src/useDelta.ts new file mode 100644 index 000000000..123b3f048 --- /dev/null +++ b/examples/document/src/useDelta.ts @@ -0,0 +1,29 @@ +import { Canvas, ModelSchema } from "@canvas-js/core" +import { MAX_MESSAGE_ID, MIN_MESSAGE_ID } from "@canvas-js/gossiplog" +import { useLiveQuery } from "@canvas-js/hooks" +import { Op } from "quill" +import { useEffect, useState } from "react" + +export const useDelta = ( + app: Canvas | undefined, + modelName: string, + key: string, + apply: (deltas: Op[]) => void, +) => { + const [cursor, setCursor] = useState(`${modelName}/${key}/${MIN_MESSAGE_ID}`) + + // TODO: is there a cleaner way to type calls to `useLiveQuery` that use internal tables? + const results = useLiveQuery(app as any, "$document_operations", { + where: { id: { gt: cursor, lt: `${modelName}/${key}/${MAX_MESSAGE_ID}` }, isAppend: false }, + }) as { id: string; key: string; data: Op[] }[] | null + + useEffect(() => { + if (!results) return + for (const message of results) { + const { data } = message + apply(data) + } + // set the cursor to the id of the last item + if (results.length > 0) setCursor(results[results.length - 1].id) + }, [results]) +} From f4cbc6be5d14a872ae1d79f7d8cbbc511d70fde4 Mon Sep 17 00:00:00 2001 From: Bob Webb Date: Tue, 11 Mar 2025 16:19:55 +0100 Subject: [PATCH 25/36] remove unused imports --- packages/core/src/Canvas.ts | 12 +----------- packages/core/src/runtime/AbstractRuntime.ts | 1 - 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/packages/core/src/Canvas.ts b/packages/core/src/Canvas.ts index 0a2d4c06c..f7ef99bd1 100644 --- a/packages/core/src/Canvas.ts +++ b/packages/core/src/Canvas.ts @@ -5,16 +5,7 @@ import type pg from "pg" import type { SqlStorage } from "@cloudflare/workers-types" import { bytesToHex } from "@noble/hashes/utils" -import { - Signature, - Action, - Message, - Snapshot, - SessionSigner, - SignerCache, - MessageType, - Updates, -} from "@canvas-js/interfaces" +import { Signature, Action, Message, Snapshot, SessionSigner, SignerCache, MessageType } from "@canvas-js/interfaces" import { AbstractModelDB, Model, ModelSchema, Effect } from "@canvas-js/modeldb" import { SIWESigner } from "@canvas-js/chain-ethereum" import { AbstractGossipLog, GossipLogEvents, NetworkClient, SignedMessage } from "@canvas-js/gossiplog" @@ -30,7 +21,6 @@ import { ActionRecord } from "./runtime/AbstractRuntime.js" import { validatePayload } from "./schema.js" import { createSnapshot, hashSnapshot } from "./snapshot.js" import { topicPattern } from "./utils.js" -import { YTextEvent } from "yjs" export type { Model } from "@canvas-js/modeldb" export type { PeerId } from "@libp2p/interface" diff --git a/packages/core/src/runtime/AbstractRuntime.ts b/packages/core/src/runtime/AbstractRuntime.ts index e3f9cb479..3f1a402eb 100644 --- a/packages/core/src/runtime/AbstractRuntime.ts +++ b/packages/core/src/runtime/AbstractRuntime.ts @@ -1,6 +1,5 @@ import * as cbor from "@ipld/dag-cbor" import { logger } from "@libp2p/logger" -import * as Y from "yjs" import type { Action, From 427c9e4b7b041176686984c40f1dac39dbba7181 Mon Sep 17 00:00:00 2001 From: Bob Webb Date: Wed, 12 Mar 2025 16:15:24 +0100 Subject: [PATCH 26/36] comment out spam button --- examples/document/src/ControlPanel.tsx | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/examples/document/src/ControlPanel.tsx b/examples/document/src/ControlPanel.tsx index 3efbec545..4fef9b076 100644 --- a/examples/document/src/ControlPanel.tsx +++ b/examples/document/src/ControlPanel.tsx @@ -73,9 +73,9 @@ export const ControlPanel: React.FC = ({}) => { if (app === null) { return (
- + */} @@ -84,13 +84,13 @@ export const ControlPanel: React.FC = ({}) => { } else if (isStarted) { return (
- + */} @@ -99,13 +99,13 @@ export const ControlPanel: React.FC = ({}) => { } else { return (
- + */} From a2b5638f0ee965ab862e8bca481b7b5a8b12b295 Mon Sep 17 00:00:00 2001 From: Bob Webb Date: Mon, 17 Mar 2025 17:02:36 +0100 Subject: [PATCH 27/36] apply updates directly to quill ref object --- examples/document/src/App.tsx | 11 ++-- examples/document/src/Editor.tsx | 1 + examples/document/src/useDelta.ts | 29 ---------- examples/document/src/useQuill.ts | 88 +++++++++++++++++++++++++++++++ 4 files changed, 92 insertions(+), 37 deletions(-) delete mode 100644 examples/document/src/useDelta.ts create mode 100644 examples/document/src/useQuill.ts diff --git a/examples/document/src/App.tsx b/examples/document/src/App.tsx index b65203cfa..15e78d1d7 100644 --- a/examples/document/src/App.tsx +++ b/examples/document/src/App.tsx @@ -11,7 +11,6 @@ import { useCanvas } from "@canvas-js/hooks" import { AuthKitProvider } from "@farcaster/auth-kit" import { JsonRpcProvider } from "ethers" -import Quill from "quill/core" import { AppContext } from "./AppContext.js" import { ControlPanel } from "./ControlPanel.js" @@ -21,7 +20,7 @@ import { ConnectionStatus } from "./ConnectionStatus.js" import { LogStatus } from "./LogStatus.js" import * as contract from "./contract.js" import { Editor } from "./Editor.js" -import { useDelta } from "./useDelta.js" +import { useQuill } from "./useQuill.js" export const topic = "document-example.canvas.xyz" @@ -42,8 +41,6 @@ export const App: React.FC<{}> = ({}) => { const [sessionSigner, setSessionSigner] = useState(null) const [address, setAddress] = useState(null) - const quillRef = useRef() - const topicRef = useRef(topic) const { app, ws } = useCanvas(wsURL, { @@ -60,9 +57,7 @@ export const App: React.FC<{}> = ({}) => { ], }) - useDelta<(typeof contract)["models"]>(app, "documents", "0", (deltas) => { - quillRef.current?.updateContents(deltas) - }) + const quillRef = useQuill({ modelName: "documents", modelKey: "0", app }) return ( @@ -77,7 +72,7 @@ export const App: React.FC<{}> = ({}) => { {}} onTextChange={async (delta, oldContents, source) => { if (source === "user") { diff --git a/examples/document/src/Editor.tsx b/examples/document/src/Editor.tsx index df161d176..6cf9648be 100644 --- a/examples/document/src/Editor.tsx +++ b/examples/document/src/Editor.tsx @@ -1,3 +1,4 @@ +import { Canvas } from "@canvas-js/core" import Quill, { EmitterSource } from "quill" import "quill/dist/quill.snow.css" import React, { forwardRef, useEffect, useLayoutEffect, useRef } from "react" diff --git a/examples/document/src/useDelta.ts b/examples/document/src/useDelta.ts deleted file mode 100644 index 123b3f048..000000000 --- a/examples/document/src/useDelta.ts +++ /dev/null @@ -1,29 +0,0 @@ -import { Canvas, ModelSchema } from "@canvas-js/core" -import { MAX_MESSAGE_ID, MIN_MESSAGE_ID } from "@canvas-js/gossiplog" -import { useLiveQuery } from "@canvas-js/hooks" -import { Op } from "quill" -import { useEffect, useState } from "react" - -export const useDelta = ( - app: Canvas | undefined, - modelName: string, - key: string, - apply: (deltas: Op[]) => void, -) => { - const [cursor, setCursor] = useState(`${modelName}/${key}/${MIN_MESSAGE_ID}`) - - // TODO: is there a cleaner way to type calls to `useLiveQuery` that use internal tables? - const results = useLiveQuery(app as any, "$document_operations", { - where: { id: { gt: cursor, lt: `${modelName}/${key}/${MAX_MESSAGE_ID}` }, isAppend: false }, - }) as { id: string; key: string; data: Op[] }[] | null - - useEffect(() => { - if (!results) return - for (const message of results) { - const { data } = message - apply(data) - } - // set the cursor to the id of the last item - if (results.length > 0) setCursor(results[results.length - 1].id) - }, [results]) -} diff --git a/examples/document/src/useQuill.ts b/examples/document/src/useQuill.ts new file mode 100644 index 000000000..9843ce289 --- /dev/null +++ b/examples/document/src/useQuill.ts @@ -0,0 +1,88 @@ +import { Canvas } from "@canvas-js/core" +import { MAX_MESSAGE_ID, MIN_MESSAGE_ID } from "@canvas-js/gossiplog" +import { AbstractModelDB } from "@canvas-js/modeldb" +import Quill from "quill" +import { useEffect, useRef } from "react" + +export const useQuill = ({ + modelName, + modelKey, + app, +}: { + modelName: string + modelKey: string + // db: AbstractModelDB | undefined + app: Canvas | undefined +}) => { + const db = app?.db + const dbRef = useRef(db ?? null) + const subscriptionRef = useRef(null) + const quillRef = useRef() + + const seenCursorsRef = useRef(new Set()) + + useEffect(() => { + // Unsubscribe from the cached database handle, if necessary + if ( + !app || + db === null || + modelName === null || + db === undefined || + modelName === undefined || + modelKey === undefined + ) { + if (dbRef.current !== null) { + if (subscriptionRef.current !== null) { + dbRef.current.unsubscribe(subscriptionRef.current) + subscriptionRef.current = null + } + } + + dbRef.current = db ?? null + console.log("exit1") + return + } + + if (dbRef.current === db && subscriptionRef.current !== null) { + console.log("exit2") + return + } + + if (dbRef.current !== null && subscriptionRef.current !== null) { + db.unsubscribe(subscriptionRef.current) + } + + // set the initial value + const initialContents = app.getYDoc(modelName, modelKey).getText().toDelta() + quillRef.current?.updateContents(initialContents) + + const query = { + where: { + id: { gt: `${modelName}/${modelKey}/${MIN_MESSAGE_ID}`, lt: `${modelName}/${modelKey}/${MAX_MESSAGE_ID}` }, + isAppend: false, + }, + } + const { id } = db.subscribe("$document_operations", query, (results) => { + for (const result of results) { + const resultId = result.id as string + if (!seenCursorsRef.current.has(resultId)) { + console.log(result.data) + seenCursorsRef.current.add(resultId) + quillRef.current?.updateContents(result.data) + } + } + }) + dbRef.current = db + + subscriptionRef.current = id + + console.log("subscribed to", modelName, modelKey, id) + return () => { + if (dbRef.current !== null && subscriptionRef.current !== null) dbRef.current.unsubscribe(subscriptionRef.current) + dbRef.current = null + subscriptionRef.current = null + } + }, [(db as any)?.isProxy ? null : db, modelKey, modelName]) + + return quillRef +} From c0ac328af7bbeda4a4fdc9e99e0f58b38c83a931 Mon Sep 17 00:00:00 2001 From: Bob Webb Date: Tue, 18 Mar 2025 11:16:13 +0100 Subject: [PATCH 28/36] add deleteDB to global for deleting the database during development --- examples/document/src/ControlPanel.tsx | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/examples/document/src/ControlPanel.tsx b/examples/document/src/ControlPanel.tsx index 4fef9b076..4d2d8ecc9 100644 --- a/examples/document/src/ControlPanel.tsx +++ b/examples/document/src/ControlPanel.tsx @@ -6,6 +6,11 @@ import { AppContext } from "./AppContext.js" export interface ControlPanelProps {} +// this is used for debugging in development +// when the database schema changes between page loads +// @ts-ignore +window.deleteDB = deleteDB + export const ControlPanel: React.FC = ({}) => { const { app, sessionSigner } = useContext(AppContext) From 334346acb4024f88320cfc2400d1c629eab13c4e Mon Sep 17 00:00:00 2001 From: Bob Webb Date: Tue, 18 Mar 2025 11:16:38 +0100 Subject: [PATCH 29/36] use an integer cursor so that we don't apply the same delta twice --- examples/document/src/useQuill.ts | 24 ++++--- packages/core/src/Canvas.ts | 4 ++ packages/core/src/DocumentStore.ts | 75 ++++++++++++++------ packages/core/src/runtime/AbstractRuntime.ts | 12 ++-- 4 files changed, 78 insertions(+), 37 deletions(-) diff --git a/examples/document/src/useQuill.ts b/examples/document/src/useQuill.ts index 9843ce289..2e7e9e6da 100644 --- a/examples/document/src/useQuill.ts +++ b/examples/document/src/useQuill.ts @@ -1,5 +1,4 @@ import { Canvas } from "@canvas-js/core" -import { MAX_MESSAGE_ID, MIN_MESSAGE_ID } from "@canvas-js/gossiplog" import { AbstractModelDB } from "@canvas-js/modeldb" import Quill from "quill" import { useEffect, useRef } from "react" @@ -11,7 +10,6 @@ export const useQuill = ({ }: { modelName: string modelKey: string - // db: AbstractModelDB | undefined app: Canvas | undefined }) => { const db = app?.db @@ -19,7 +17,7 @@ export const useQuill = ({ const subscriptionRef = useRef(null) const quillRef = useRef() - const seenCursorsRef = useRef(new Set()) + const cursorRef = useRef(-1) useEffect(() => { // Unsubscribe from the cached database handle, if necessary @@ -39,12 +37,10 @@ export const useQuill = ({ } dbRef.current = db ?? null - console.log("exit1") return } if (dbRef.current === db && subscriptionRef.current !== null) { - console.log("exit2") return } @@ -56,18 +52,24 @@ export const useQuill = ({ const initialContents = app.getYDoc(modelName, modelKey).getText().toDelta() quillRef.current?.updateContents(initialContents) + // get the initial value for cursorRef + const startId = app.getYDocId(modelName, modelKey) const query = { where: { - id: { gt: `${modelName}/${modelKey}/${MIN_MESSAGE_ID}`, lt: `${modelName}/${modelKey}/${MAX_MESSAGE_ID}` }, + model: modelName, + key: modelKey, isAppend: false, + id: { gt: startId }, }, + limit: 1, + orderBy: { id: "desc" as const }, } - const { id } = db.subscribe("$document_operations", query, (results) => { + + const { id } = db.subscribe("$document_updates", query, (results) => { for (const result of results) { - const resultId = result.id as string - if (!seenCursorsRef.current.has(resultId)) { - console.log(result.data) - seenCursorsRef.current.add(resultId) + const resultId = result.id as number + if (cursorRef.current < resultId) { + cursorRef.current = resultId quillRef.current?.updateContents(result.data) } } diff --git a/packages/core/src/Canvas.ts b/packages/core/src/Canvas.ts index f7ef99bd1..0106f2c50 100644 --- a/packages/core/src/Canvas.ts +++ b/packages/core/src/Canvas.ts @@ -479,6 +479,10 @@ export class Canvas< return this.runtime.getYDoc(model, key) } + public getYDocId(model: string, key: string) { + return this.runtime.getYDocId(model, key) + } + public async createSnapshot(): Promise { return createSnapshot(this) } diff --git a/packages/core/src/DocumentStore.ts b/packages/core/src/DocumentStore.ts index 2c88c87bd..8723f48f1 100644 --- a/packages/core/src/DocumentStore.ts +++ b/packages/core/src/DocumentStore.ts @@ -20,16 +20,20 @@ function getDeltaForYText(ytext: Y.Text, fn: () => void): Delta { export class DocumentStore { private documents: Record> = {} + private documentIds: Record> = {} public static schema = { - $document_operations: { - // ${model}/${key}/${messageId} - id: "primary", + $document_updates: { + primary: "primary", + model: "string", + key: "string", + id: "number", // applyDelta, insert or delete data: "json", // yjs document diff diff: "bytes", isAppend: "boolean", + $indexes: ["id"], }, } satisfies ModelSchema @@ -42,14 +46,31 @@ export class DocumentStore { public async loadSavedDocuments(db: AbstractModelDB) { // iterate over the past document operations // and create the yjs documents - for await (const operation of db.iterate("$document_operations")) { - const [model, key, _messageId] = operation.id.split("/") - const doc = this.getYDoc(model, key) + for await (const operation of db.iterate("$document_updates")) { + const doc = this.getYDoc(operation.model, operation.key) Y.applyUpdate(doc, operation.diff) + const existingId = this.getId(operation.model, operation.key) + if (operation.id > existingId) { + this.setId(operation.model, operation.key, operation.id) + } } } - public async applyYjsCalls(db: AbstractModelDB, model: string, key: string, messageId: string, calls: YjsCall[]) { + public getId(model: string, key: string) { + this.documentIds[model] ||= {} + return this.documentIds[model][key] ?? -1 + } + + private setId(model: string, key: string, id: number) { + this.documentIds[model] ||= {} + this.documentIds[model][key] = id + } + + private getNextId(model: string, key: string) { + return this.getId(model, key) + 1 + } + + public async applyYjsCalls(db: AbstractModelDB, model: string, key: string, calls: YjsCall[]) { const doc = this.getYDoc(model, key) // get the initial state of the document @@ -74,18 +95,12 @@ export class DocumentStore { const afterState = Y.encodeStateAsUpdate(doc) const diff = Y.diffUpdate(afterState, Y.encodeStateVectorFromUpdate(beforeState)) - await db.set(`$document_operations`, { - id: `${model}/${key}/${messageId}`, - key, - data: delta || [], - diff, - isAppend: true, - }) + await this.writeDocumentUpdate(db, model, key, delta || [], diff, true) return { model, key, diff } } - public async consumeUpdatesMessage(db: AbstractModelDB, message: Message, id: string) { + public async consumeUpdatesMessage(db: AbstractModelDB, message: Message) { for (const { model, key, diff } of message.payload.updates) { // apply the diff to the doc const doc = this.getYDoc(model, key) @@ -94,13 +109,29 @@ export class DocumentStore { }) // save the observed update to the db - await db.set(`$document_operations`, { - id: `${model}/${key}/${id}`, - key, - data: delta, - diff, - isAppend: false, - }) + await this.writeDocumentUpdate(db, model, key, delta, diff, false) } } + + private async writeDocumentUpdate( + db: AbstractModelDB, + model: string, + key: string, + data: Delta, + diff: Uint8Array, + isAppend: boolean, + ) { + const id = this.getNextId(model, key) + this.setId(model, key, id) + + await db.set(`$document_updates`, { + primary: `${model}/${key}/${id}`, + model, + key, + id, + data, + diff, + isAppend, + }) + } } diff --git a/packages/core/src/runtime/AbstractRuntime.ts b/packages/core/src/runtime/AbstractRuntime.ts index 3f1a402eb..e90760e21 100644 --- a/packages/core/src/runtime/AbstractRuntime.ts +++ b/packages/core/src/runtime/AbstractRuntime.ts @@ -142,7 +142,7 @@ export abstract class AbstractRuntime { } else if (isSnapshot(signedMessage)) { return await handleSnapshot(signedMessage, this) } else if (isUpdates(signedMessage)) { - return await handleUpdates(signedMessage.id, signedMessage.message, isAppend) + return await handleUpdates(signedMessage.message, isAppend) } else { throw new Error("invalid message payload type") } @@ -291,7 +291,7 @@ export abstract class AbstractRuntime { const updates = [] for (const [model, modelCalls] of Object.entries(executionContext.yjsCalls)) { for (const [key, calls] of Object.entries(modelCalls)) { - const update = await this.#documentStore.applyYjsCalls(this.db, model, key, id, calls) + const update = await this.#documentStore.applyYjsCalls(this.db, model, key, calls) updates.push(update) } } @@ -309,13 +309,17 @@ export abstract class AbstractRuntime { return this.#documentStore.getYDoc(model, key) } + public getYDocId(model: string, key: string) { + return this.#documentStore.getId(model, key) + } + public async loadSavedDocuments() { await this.#documentStore.loadSavedDocuments(this.db) } - public async handleUpdates(id: string, message: Message, isAppend: boolean) { + public async handleUpdates(message: Message, isAppend: boolean) { if (!isAppend) { - return await this.#documentStore.consumeUpdatesMessage(this.db, message, id) + return await this.#documentStore.consumeUpdatesMessage(this.db, message) } } } From 724ad1b3a71a3778954c025546a49b194d7e5d60 Mon Sep 17 00:00:00 2001 From: Bob Webb Date: Tue, 18 Mar 2025 13:13:32 +0100 Subject: [PATCH 30/36] remove print statement --- examples/document/src/useQuill.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/document/src/useQuill.ts b/examples/document/src/useQuill.ts index 2e7e9e6da..7fc72a4ae 100644 --- a/examples/document/src/useQuill.ts +++ b/examples/document/src/useQuill.ts @@ -78,7 +78,6 @@ export const useQuill = ({ subscriptionRef.current = id - console.log("subscribed to", modelName, modelKey, id) return () => { if (dbRef.current !== null && subscriptionRef.current !== null) dbRef.current.unsubscribe(subscriptionRef.current) dbRef.current = null From 86d74b4ac2afac6b9ce22747bce5d24e2c202880 Mon Sep 17 00:00:00 2001 From: Bob Webb Date: Tue, 18 Mar 2025 16:35:54 +0100 Subject: [PATCH 31/36] set db field on DocumentStore --- packages/core/src/DocumentStore.ts | 41 +++++++++++++------- packages/core/src/index.ts | 1 + packages/core/src/runtime/AbstractRuntime.ts | 7 ++-- 3 files changed, 31 insertions(+), 18 deletions(-) diff --git a/packages/core/src/DocumentStore.ts b/packages/core/src/DocumentStore.ts index 8723f48f1..5f7cc7473 100644 --- a/packages/core/src/DocumentStore.ts +++ b/packages/core/src/DocumentStore.ts @@ -2,6 +2,7 @@ import { Message, Updates } from "@canvas-js/interfaces" import { AbstractModelDB, ModelSchema } from "@canvas-js/modeldb" import * as Y from "yjs" import { YjsCall } from "./ExecutionContext.js" +import { assert } from "@canvas-js/utils" type Delta = Y.YTextEvent["changes"]["delta"] @@ -37,16 +38,28 @@ export class DocumentStore { }, } satisfies ModelSchema + #db: AbstractModelDB | null = null + + public get db() { + assert(this.#db !== null, "internal error - expected this.#db !== null") + return this.#db + } + + public set db(db: AbstractModelDB) { + this.#db = db + } + public getYDoc(model: string, key: string) { this.documents[model] ||= {} this.documents[model][key] ||= new Y.Doc() return this.documents[model][key] } - public async loadSavedDocuments(db: AbstractModelDB) { + public async loadSavedDocuments() { + assert(this.#db !== null, "internal error - expected this.#db !== null") // iterate over the past document operations // and create the yjs documents - for await (const operation of db.iterate("$document_updates")) { + for await (const operation of this.#db.iterate("$document_updates")) { const doc = this.getYDoc(operation.model, operation.key) Y.applyUpdate(doc, operation.diff) const existingId = this.getId(operation.model, operation.key) @@ -70,7 +83,9 @@ export class DocumentStore { return this.getId(model, key) + 1 } - public async applyYjsCalls(db: AbstractModelDB, model: string, key: string, calls: YjsCall[]) { + public async applyYjsCalls(model: string, key: string, calls: YjsCall[]) { + assert(this.#db !== null, "internal error - expected this.#db !== null") + const doc = this.getYDoc(model, key) // get the initial state of the document @@ -95,12 +110,14 @@ export class DocumentStore { const afterState = Y.encodeStateAsUpdate(doc) const diff = Y.diffUpdate(afterState, Y.encodeStateVectorFromUpdate(beforeState)) - await this.writeDocumentUpdate(db, model, key, delta || [], diff, true) + await this.writeDocumentUpdate(model, key, delta || [], diff, true) return { model, key, diff } } - public async consumeUpdatesMessage(db: AbstractModelDB, message: Message) { + public async consumeUpdatesMessage(message: Message) { + assert(this.#db !== null, "internal error - expected this.#db !== null") + for (const { model, key, diff } of message.payload.updates) { // apply the diff to the doc const doc = this.getYDoc(model, key) @@ -109,22 +126,16 @@ export class DocumentStore { }) // save the observed update to the db - await this.writeDocumentUpdate(db, model, key, delta, diff, false) + await this.writeDocumentUpdate(model, key, delta, diff, false) } } - private async writeDocumentUpdate( - db: AbstractModelDB, - model: string, - key: string, - data: Delta, - diff: Uint8Array, - isAppend: boolean, - ) { + private async writeDocumentUpdate(model: string, key: string, data: Delta, diff: Uint8Array, isAppend: boolean) { + assert(this.#db !== null, "internal error - expected this.#db !== null") const id = this.getNextId(model, key) this.setId(model, key, id) - await db.set(`$document_updates`, { + await this.#db.set(`$document_updates`, { primary: `${model}/${key}/${id}`, model, key, diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index ec6890370..326442227 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -1,6 +1,7 @@ export * from "./Canvas.js" export * from "./CanvasLoadable.js" export * from "./types.js" +export * from "./DocumentStore.js" export { hashContract } from "./snapshot.js" export { Action, Session, Snapshot } from "@canvas-js/interfaces" export { NetworkClient } from "@canvas-js/gossiplog" diff --git a/packages/core/src/runtime/AbstractRuntime.ts b/packages/core/src/runtime/AbstractRuntime.ts index e90760e21..701c0f94b 100644 --- a/packages/core/src/runtime/AbstractRuntime.ts +++ b/packages/core/src/runtime/AbstractRuntime.ts @@ -126,6 +126,7 @@ export abstract class AbstractRuntime { public set db(db: AbstractModelDB) { this.#db = db + this.#documentStore.db = db } public getConsumer(): GossipLogConsumer { @@ -291,7 +292,7 @@ export abstract class AbstractRuntime { const updates = [] for (const [model, modelCalls] of Object.entries(executionContext.yjsCalls)) { for (const [key, calls] of Object.entries(modelCalls)) { - const update = await this.#documentStore.applyYjsCalls(this.db, model, key, calls) + const update = await this.#documentStore.applyYjsCalls(model, key, calls) updates.push(update) } } @@ -314,12 +315,12 @@ export abstract class AbstractRuntime { } public async loadSavedDocuments() { - await this.#documentStore.loadSavedDocuments(this.db) + await this.#documentStore.loadSavedDocuments() } public async handleUpdates(message: Message, isAppend: boolean) { if (!isAppend) { - return await this.#documentStore.consumeUpdatesMessage(this.db, message) + return await this.#documentStore.consumeUpdatesMessage(message) } } } From 533c3a65204f3bba811367ced0bc4b7465c84f4a Mon Sep 17 00:00:00 2001 From: Bob Webb Date: Tue, 18 Mar 2025 16:58:29 +0100 Subject: [PATCH 32/36] add basic test to save/load a document --- packages/core/test/documentStore.test.ts | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 packages/core/test/documentStore.test.ts diff --git a/packages/core/test/documentStore.test.ts b/packages/core/test/documentStore.test.ts new file mode 100644 index 000000000..377b137c2 --- /dev/null +++ b/packages/core/test/documentStore.test.ts @@ -0,0 +1,20 @@ +import test from "ava" + +import { DocumentStore } from "@canvas-js/core" +import { ModelDB } from "@canvas-js/modeldb-sqlite" + +test("save and load a document using the document store", async (t) => { + const db = await ModelDB.open(null, { models: DocumentStore.schema }) + const ds = new DocumentStore() + ds.db = db + + const delta = { ops: [{ insert: "hello world" }] } + await ds.applyYjsCalls("documents", "0", [{ call: "applyDelta", delta }]) + + const ds2 = new DocumentStore() + ds2.db = db + await ds2.loadSavedDocuments() + const doc = ds2.getYDoc("documents", "0") + + t.deepEqual(doc.getText().toDelta(), delta.ops) +}) From a948b26a9059de99ee9f58ad6617320715f5eb2f Mon Sep 17 00:00:00 2001 From: Bob Webb Date: Tue, 18 Mar 2025 16:59:54 +0100 Subject: [PATCH 33/36] code for getting/setting id --- packages/core/src/DocumentStore.ts | 4 ++-- packages/core/test/documentStore.test.ts | 9 +++++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/packages/core/src/DocumentStore.ts b/packages/core/src/DocumentStore.ts index 5f7cc7473..5aa7fd5bf 100644 --- a/packages/core/src/DocumentStore.ts +++ b/packages/core/src/DocumentStore.ts @@ -74,12 +74,12 @@ export class DocumentStore { return this.documentIds[model][key] ?? -1 } - private setId(model: string, key: string, id: number) { + public setId(model: string, key: string, id: number) { this.documentIds[model] ||= {} this.documentIds[model][key] = id } - private getNextId(model: string, key: string) { + public getNextId(model: string, key: string) { return this.getId(model, key) + 1 } diff --git a/packages/core/test/documentStore.test.ts b/packages/core/test/documentStore.test.ts index 377b137c2..496ee4158 100644 --- a/packages/core/test/documentStore.test.ts +++ b/packages/core/test/documentStore.test.ts @@ -18,3 +18,12 @@ test("save and load a document using the document store", async (t) => { t.deepEqual(doc.getText().toDelta(), delta.ops) }) + +test("get and set id", async (t) => { + const ds = new DocumentStore() + + t.is(ds.getId("documents", "0"), -1) + ds.setId("documents", "0", 42) + t.is(ds.getId("documents", "0"), 42) + t.is(ds.getNextId("documents", "0"), 43) +}) From 758942f4365d81edc33618e68823540996ede9e6 Mon Sep 17 00:00:00 2001 From: Bob Webb Date: Tue, 18 Mar 2025 17:54:54 +0100 Subject: [PATCH 34/36] add missing dependencies --- examples/document/package.json | 5 +++++ package-lock.json | 16 ++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/examples/document/package.json b/examples/document/package.json index 1af344a48..1769190b2 100644 --- a/examples/document/package.json +++ b/examples/document/package.json @@ -21,6 +21,11 @@ "@canvas-js/hooks": "0.14.0-next.1", "@canvas-js/interfaces": "0.14.0-next.1", "@canvas-js/modeldb-sqlite-wasm": "0.14.0-next.1", + "@farcaster/auth-kit": "^0.6.0", + "@farcaster/frame-sdk": "^0.0.26", + "@libp2p/interface": "^2.7.0", + "@multiformats/multiaddr": "^12.3.5", + "@noble/hashes": "^1.7.1", "@types/react": "^18.3.9", "@types/react-dom": "^18.3.0", "buffer": "^6.0.3", diff --git a/package-lock.json b/package-lock.json index 240b5ec4d..c8b2696be 100644 --- a/package-lock.json +++ b/package-lock.json @@ -169,6 +169,11 @@ "@canvas-js/hooks": "0.14.0-next.1", "@canvas-js/interfaces": "0.14.0-next.1", "@canvas-js/modeldb-sqlite-wasm": "0.14.0-next.1", + "@farcaster/auth-kit": "^0.6.0", + "@farcaster/frame-sdk": "^0.0.26", + "@libp2p/interface": "^2.7.0", + "@multiformats/multiaddr": "^12.3.5", + "@noble/hashes": "^1.7.1", "@types/react": "^18.3.9", "@types/react-dom": "^18.3.0", "buffer": "^6.0.3", @@ -192,6 +197,17 @@ "vite-plugin-node-polyfills": "^0.22.0" } }, + "examples/document/node_modules/@noble/hashes": { + "version": "1.7.1", + "resolved": "https://registry.npmjs.org/@noble/hashes/-/hashes-1.7.1.tgz", + "integrity": "sha512-B8XBPsn4vT/KJAGqDzbwztd+6Yte3P4V7iafm24bxgDe/mlRuK6xmWPuCNrKt2vDafZ8MfJLlchDG/vYafQEjQ==", + "engines": { + "node": "^14.21.3 || >=16" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, "examples/encrypted-chat": { "name": "@canvas-js/example-chat-encrypted", "version": "0.14.0-next.1", From b3fc76ba34fe13d69e570b66d7303c9f2aa7a29f Mon Sep 17 00:00:00 2001 From: Bob Webb Date: Tue, 18 Mar 2025 19:28:05 +0100 Subject: [PATCH 35/36] store squashed document updates in snapshot --- packages/core/src/DocumentStore.ts | 10 ++++++++++ packages/core/src/snapshot.ts | 21 +++++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/packages/core/src/DocumentStore.ts b/packages/core/src/DocumentStore.ts index 5aa7fd5bf..b78ff9c73 100644 --- a/packages/core/src/DocumentStore.ts +++ b/packages/core/src/DocumentStore.ts @@ -19,6 +19,16 @@ function getDeltaForYText(ytext: Y.Text, fn: () => void): Delta { return delta || [] } +export type DocumentUpdateRecord = { + primary: string + model: string + key: string + id: number + data: Delta + diff: Uint8Array + isAppend: boolean +} + export class DocumentStore { private documents: Record> = {} private documentIds: Record> = {} diff --git a/packages/core/src/snapshot.ts b/packages/core/src/snapshot.ts index 11388d8fb..790752170 100644 --- a/packages/core/src/snapshot.ts +++ b/packages/core/src/snapshot.ts @@ -1,6 +1,7 @@ import { sha256 } from "@noble/hashes/sha256" import { bytesToHex } from "@noble/hashes/utils" import * as cbor from "@ipld/dag-cbor" +import * as Y from "yjs" import { MIN_MESSAGE_ID } from "@canvas-js/gossiplog" import { Snapshot, SnapshotEffect } from "@canvas-js/interfaces" @@ -9,6 +10,7 @@ import type { PropertyType } from "@canvas-js/modeldb" import { Canvas } from "./Canvas.js" import { Contract } from "./types.js" import { EffectRecord } from "./runtime/AbstractRuntime.js" +import { DocumentUpdateRecord } from "./DocumentStore.js" // typeguards export const isIndexInit = (value: unknown): value is string[] => Array.isArray(value) @@ -60,6 +62,25 @@ export async function createSnapshot>(app: Canvas): Prom effectsMap.set(recordId, { key: effectKey, value, branch, clock }) } } + + // iterate over the document updates table + const documentsMap = new Map() + for await (const row of app.db.iterate("$document_updates")) { + const { primary, diff } = row + + const doc = documentsMap.get(primary) || new Y.Doc() + Y.applyUpdate(doc, diff) + documentsMap.set(primary, doc) + } + + modelData[`$document_updates`] = [] + for (const [primary, doc] of documentsMap.entries()) { + const diff = Y.encodeStateAsUpdate(doc) + const [model, key, id_] = primary.split("/") + const row = { primary, model, key, id: 0, data: doc.getText().toDelta(), diff, isAppend: false } + modelData[`$document_updates`].push(cbor.encode(row)) + } + const effects = Array.from(effectsMap.values()).map(({ key, value }: SnapshotEffect) => ({ key, value })) return { From e61a164756170add78024eceb6f2bc63797b2857 Mon Sep 17 00:00:00 2001 From: Bob Webb Date: Wed, 19 Mar 2025 14:07:30 +0100 Subject: [PATCH 36/36] rename ytext functions --- examples/document/README.md | 2 +- examples/document/src/contract.ts | 2 +- packages/core/src/runtime/ContractRuntime.ts | 41 ++++++++++---------- packages/core/src/runtime/FunctionRuntime.ts | 18 +++++---- packages/core/src/types.ts | 23 +++++------ packages/core/test/snapshot.test.ts | 4 +- packages/core/test/yjs.test.ts | 6 +-- 7 files changed, 48 insertions(+), 48 deletions(-) diff --git a/examples/document/README.md b/examples/document/README.md index 827150480..a8e358c81 100644 --- a/examples/document/README.md +++ b/examples/document/README.md @@ -14,7 +14,7 @@ export const models = { export const actions = { async applyDeltaToDoc(db, index, text) { - await db.yjsApplyDelta("documents", "0", index, text) + await db.ytext.applyDelta("documents", "0", index, text) }, } ``` diff --git a/examples/document/src/contract.ts b/examples/document/src/contract.ts index 6ca2daee5..5d63ed928 100644 --- a/examples/document/src/contract.ts +++ b/examples/document/src/contract.ts @@ -9,6 +9,6 @@ export const models = { export const actions = { async applyDeltaToDoc(db, delta) { - await db.yjsApplyDelta("documents", "0", delta) + await db.ytext.applyDelta("documents", "0", delta) }, } satisfies Actions diff --git a/packages/core/src/runtime/ContractRuntime.ts b/packages/core/src/runtime/ContractRuntime.ts index 98f84871a..4364388d7 100644 --- a/packages/core/src/runtime/ContractRuntime.ts +++ b/packages/core/src/runtime/ContractRuntime.ts @@ -137,26 +137,27 @@ export class ContractRuntime extends AbstractRuntime { const key = vm.context.getString(keyHandle) this.context.deleteModelValue(model, key) }), - - yjsInsert: vm.context.newFunction("yjsInsert", (modelHandle, keyHandle, indexHandle, contentHandle) => { - const model = vm.context.getString(modelHandle) - const key = vm.context.getString(keyHandle) - const index = vm.context.getNumber(indexHandle) - const content = vm.context.getString(contentHandle) - this.context.pushYjsCall(model, key, { call: "insert", index, content }) - }), - yjsDelete: vm.context.newFunction("yjsDelete", (modelHandle, keyHandle, indexHandle, lengthHandle) => { - const model = vm.context.getString(modelHandle) - const key = vm.context.getString(keyHandle) - const index = vm.context.getNumber(indexHandle) - const length = vm.context.getNumber(lengthHandle) - this.context.pushYjsCall(model, key, { call: "delete", index, length }) - }), - yjsApplyDelta: vm.context.newFunction("yjsApplyDelta", (modelHandle, keyHandle, deltaHandle) => { - const model = vm.context.getString(modelHandle) - const key = vm.context.getString(keyHandle) - const delta = vm.context.getString(deltaHandle) - this.context.pushYjsCall(model, key, { call: "applyDelta", delta: JSON.parse(delta) }) + ytext: vm.wrapObject({ + insert: vm.context.newFunction("insert", (modelHandle, keyHandle, indexHandle, contentHandle) => { + const model = vm.context.getString(modelHandle) + const key = vm.context.getString(keyHandle) + const index = vm.context.getNumber(indexHandle) + const content = vm.context.getString(contentHandle) + this.context.pushYjsCall(model, key, { call: "insert", index, content }) + }), + delete: vm.context.newFunction("delete", (modelHandle, keyHandle, indexHandle, lengthHandle) => { + const model = vm.context.getString(modelHandle) + const key = vm.context.getString(keyHandle) + const index = vm.context.getNumber(indexHandle) + const length = vm.context.getNumber(lengthHandle) + this.context.pushYjsCall(model, key, { call: "delete", index, length }) + }), + applyDelta: vm.context.newFunction("applyDelta", (modelHandle, keyHandle, deltaHandle) => { + const model = vm.context.getString(modelHandle) + const key = vm.context.getString(keyHandle) + const delta = vm.context.getString(deltaHandle) + this.context.pushYjsCall(model, key, { call: "applyDelta", delta: JSON.parse(delta) }) + }), }), }) .consume(vm.cache) diff --git a/packages/core/src/runtime/FunctionRuntime.ts b/packages/core/src/runtime/FunctionRuntime.ts index 1272b348b..6d79cf176 100644 --- a/packages/core/src/runtime/FunctionRuntime.ts +++ b/packages/core/src/runtime/FunctionRuntime.ts @@ -174,14 +174,16 @@ export class FunctionRuntime extends AbstractRuntim this.releaseLock() } }, - yjsInsert: async (model: string, key: string, index: number, content: string) => { - this.context.pushYjsCall(model, key, { call: "insert", index, content }) - }, - yjsDelete: async (model: string, key: string, index: number, length: number) => { - this.context.pushYjsCall(model, key, { call: "delete", index, length }) - }, - yjsApplyDelta: async (model: string, key: string, delta: any) => { - this.context.pushYjsCall(model, key, { call: "applyDelta", delta: JSON.parse(delta) }) + ytext: { + insert: async (model: string, key: string, index: number, content: string) => { + this.context.pushYjsCall(model, key, { call: "insert", index, content }) + }, + delete: async (model: string, key: string, index: number, length: number) => { + this.context.pushYjsCall(model, key, { call: "delete", index, length }) + }, + applyDelta: async (model: string, key: string, delta: any) => { + this.context.pushYjsCall(model, key, { call: "applyDelta", delta: JSON.parse(delta) }) + }, }, } } diff --git a/packages/core/src/types.ts b/packages/core/src/types.ts index 16630c33f..4fb57822f 100644 --- a/packages/core/src/types.ts +++ b/packages/core/src/types.ts @@ -44,19 +44,16 @@ export type ModelAPI> = { update: (model: T, value: Partial) => Chainable merge: (model: T, value: Partial) => Chainable delete: (model: T, key: string) => Promise - yjsInsert: ( - model: T, - key: string, - index: number, - content: string, - ) => Promise - yjsDelete: ( - model: T, - key: string, - index: number, - length: number, - ) => Promise - yjsApplyDelta: (model: T, key: string, delta: string) => Promise + ytext: { + insert: ( + model: T, + key: string, + index: number, + content: string, + ) => Promise + delete: (model: T, key: string, index: number, length: number) => Promise + applyDelta: (model: T, key: string, delta: string) => Promise + } } export type ActionContext> = { diff --git a/packages/core/test/snapshot.test.ts b/packages/core/test/snapshot.test.ts index 897797556..3e1c459b3 100644 --- a/packages/core/test/snapshot.test.ts +++ b/packages/core/test/snapshot.test.ts @@ -24,10 +24,10 @@ test("snapshot persists data across apps", async (t) => { await db.delete("posts", id) }, async insertIntoDocument(db, key, index, text) { - await db.yjsInsert("documents", key, index, text) + await db.ytext.insert("documents", key, index, text) }, async deleteFromDocument(db, key, index, length) { - await db.yjsDelete("documents", key, index, length) + await db.ytext.delete("documents", key, index, length) }, }, }, diff --git a/packages/core/test/yjs.test.ts b/packages/core/test/yjs.test.ts index 30e1cda3d..09b8a023b 100644 --- a/packages/core/test/yjs.test.ts +++ b/packages/core/test/yjs.test.ts @@ -12,13 +12,13 @@ export const models = { export const actions = { async createNewArticle(db) { const { id } = this - await db.yjsInsert("articles", id, 0, "") + await db.ytext.insert("articles", id, 0, "") }, async insertIntoDoc(db, key, index, text) { - await db.yjsInsert("articles", key, index, text) + await db.ytext.insert("articles", key, index, text) }, async deleteFromDoc(db, key, index, length) { - await db.yjsDelete("articles", key, index, length) + await db.ytext.delete("articles", key, index, length) } }; `