import { WavingDBDriver } from "./WavingDBDriver"; import { WavingMemoryDB } from "./WavingMemoryDB"; import { v4 as uuid } from "uuid"; import throttle from "lodash.throttle"; import SocketIO from "socket.io"; import * as Y from "yjs"; import { MessageCode } from "./types"; import { WavingSyncDriver } from "./WavingSyncDriver"; import { squashYjsState } from "./util"; import { WavingPermissionDriverConstructor, IWavingPermissionDriver, } from "./WavingPermissionDriver"; import { WavingPermissionYesMan } from "./WavingPermissionYesMan"; import { string } from "lib0"; interface Awareness { id: number; user: { name: string; }; cursor: { anchor: any; head: any; }; } interface WavingOpts { dbDriver?: WavingDBDriver; io: SocketIO.Server; sync?: WavingSyncDriver; PermissionDriver?: WavingPermissionDriverConstructor; } export class Waving { private db: WavingDBDriver; private io: SocketIO.Server; private sync?: WavingSyncDriver; private PermissionDriver: WavingPermissionDriverConstructor; private docs = new Map>(); private awarenessCache = new Map>(); private socketsXId = new Map(); public setDoc: ( docId: string, newDoc: Y.Doc, force?: boolean, immediate?: boolean ) => Promise; private setDocThrottled: ( docId: string, newDoc: Y.Doc, force?: boolean ) => Promise; constructor({ dbDriver, io, sync, PermissionDriver }: WavingOpts) { this.db = dbDriver || new WavingMemoryDB({}); this.io = io; this.sync = sync; this.PermissionDriver = PermissionDriver || WavingPermissionYesMan; if (this.sync) { this.sync.onAwareness = this.onAwareness.bind(this); this.sync.onUpdateVector = this.onUpdateVector.bind(this); } const throttleMs = 1000; this.setDoc = ( docId: string, newDoc: Y.Doc, force = false, immediate = false ) => { if (immediate) { return this._setDoc(docId, newDoc, force); } else { return this.setDocThrottled(docId, newDoc, force); } }; this.setDocThrottled = throttle( (docId: string, newDoc: Y.Doc, force = false) => { this._setDoc(docId, newDoc, force); }, throttleMs, { leading: false, trailing: true } ); if (this.sync) { setInterval(() => { for (const [docId] of this.docs.keys()) { this.sync!.publishLoadedDoc(docId); } }, 1000 * 30); } process.on("exit", () => { // Commit all docs from memory to DB // for each of docs.conns, close connections // close db }); } public async getDoc(docId: string) { const cachedDocP = this.docs.get(docId); if (cachedDocP) { const cachedDoc = await cachedDocP; if (cachedDoc) { return cachedDoc; } } const existingDocPromise = this.db.getDoc(docId); if (!docId.startsWith("ns:")) { this.docs.set(docId, existingDocPromise); } const existingDoc = await existingDocPromise; if (!existingDoc) { throw new Error("Document does not exist"); } this.sync?.publishLoadedDoc(docId); existingDoc.on("update", (vector, origin) => this.onDocChange(docId, vector, origin) ); return existingDoc; } private async _setDoc(docId: string, doc: Y.Doc, force = false) { if (!docId.startsWith("ns:")) { const oldDoc = await this.getDoc(docId); const docWritePromise = this.db.setDoc(docId, doc, oldDoc, force); this.docs.set( docId, docWritePromise.then(() => doc) ); } } public registerSocket(socket: SocketIO.Socket) { let originalContent: Uint8Array; const socketId = uuid(); this.socketsXId.set(socketId, socket); let permissions: { [docId: string]: IWavingPermissionDriver } = {}; socket.on("auth", (token) => { (socket as any).token = token; }); socket.on("unsubscribe", async (docId, callback) => { delete permissions[docId]; this.unsubscribeFromDoc(docId, socket); delete (socket as any).docId; callback(); }); socket.on("subscribe", async (docId, remoteState, callback) => { if (callback) { permissions[docId] = new this.PermissionDriver( docId, (socket as any).token ); await permissions[docId].init(); if (!(await permissions[docId].canView())) return; const doc = await this.getDoc(docId); for (const key of Object.keys(socket.rooms)) { if (key.startsWith("doc:")) { socket.leave(key); } } socket.join("doc:" + docId); (socket as any).docId = docId; const docAwareness = this.awarenessCache.get(docId); const awarenessItems = docAwareness ? Array.from(docAwareness.values()) : []; // TODO: Y.encodeStateAsUpdate(doc, encodedStateVector) // const vector = Y.encodeStateAsUpdate(doc, Uint8Array.from(remoteState)); const vector = Y.encodeStateAsUpdate(doc); callback(docId, Array.from(vector), awarenessItems); // socket.emit(MessageCode.updateVector, docId, Array.from(vector)); try { (socket as any).numEdits = 0; (socket as any).originalContent = originalContent = vector; } catch {} } }); socket.on("disconnecting", async (reason) => { this.socketsXId.delete(socketId); let docs = Object.keys(socket.rooms).filter((key) => key.startsWith("doc:") ); for (const roomId of docs) { const docId = roomId.replace("doc:", ""); this.unsubscribeFromDoc(docId, socket); } }); socket.on( MessageCode.updateVector, async (docId, updateVectors: number[][]) => { if (!permissions[docId]) { (socket as any).pendingUpdates = (socket as any).pendingUpdates || []; (socket as any).pendingUpdates.push(updateVectors); } else { const canEdit = await permissions[docId].canEdit(); if (!canEdit) return; if ((socket as any).pendingUpdates) { for (const vector of (socket as any).pendingUpdates) { this.onUpdateVector(docId, Uint8Array.from(vector), socketId); this.sync?.publishUpdateVector(docId, Uint8Array.from(vector)); } (socket as any).pendingUpdates = []; } for (const vector of updateVectors) { this.onUpdateVector(docId, Uint8Array.from(vector), socketId); this.sync?.publishUpdateVector(docId, Uint8Array.from(vector)); } } } ); socket.on(MessageCode.awareness, async (docId, awareness) => { (socket as any).awarenessId = awareness.id; this.onAwareness(docId, awareness, socket); this.sync?.publishAwareness(docId, awareness); }); } async unsubscribeFromDoc(docId: string, socket) { if ((socket as any).originalContent && (socket as any).numEdits) { await this.db.setSessionRevision( (socket as any).docId, (socket as any).token, socket.originalContent ); } const aw = { id: (socket as any).awarenessId } as any; const roomId = `doc:${docId}`; socket.leave(); this.onAwareness(docId, aw, socket); this.sync?.publishAwareness(docId, aw); const docRoom = this.io.sockets.adapter.rooms[roomId]; if (!docRoom) { this.unloadDoc(docId); } } async onDocChange(docId: string, updateVector, socketId?: string) { const socket = socketId && this.socketsXId.get(socketId); const from = socket ? socket.to(`doc:${docId}`) : this.io.in(`doc:${docId}`); (from as any).emit(MessageCode.updateVector, docId, [updateVector]); if (socket || !socketId) { console.log("onchange, persisting"); if (socket) { // If this node received the message first (socket as any).numEdits += 1; } const doc = await this.getDoc(docId); this.setDoc(docId, doc); } } async onUpdateVector(docId, updateVector: Uint8Array, socketId?: string) { const doc = await this.getDoc(docId); if (doc) { // don't load unloaded docs Y.applyUpdate(doc, updateVector, socketId); } } async onAwareness(docId, awareness: Awareness, socket?) { const from = socket ? socket.to(`doc:${docId}`) : this.io.in(`doc:${docId}`); from.emit(MessageCode.awareness, docId, awareness); const isAwarenessEnd = !awareness.cursor || !(awareness.cursor.head || awareness.cursor.anchor); let docAwareness = this.awarenessCache.get(docId); if (!docAwareness) { docAwareness = new Map(); this.awarenessCache.set(docId, docAwareness); } if (isAwarenessEnd) { docAwareness?.delete(awareness.id); } else { docAwareness?.set(awareness.id, awareness); } } async unloadDoc(docId: string) { this.docs.delete(docId); let shouldSquashDoc = true; if (this.sync) { await this.sync.publishUnloadedDoc(docId); shouldSquashDoc = !(await this.sync.isDocLoaded(docId)); } if (shouldSquashDoc) { await this.squashDoc(docId); } } async squashDoc(docId: string) { // console.warn(`Squashing [${docId}] - This is dangerous if the document is still loaded by another node`); // // WARNING, THIS MUST ONLY BE CALLED WHEN NO NODES HAVE THE DOCUMENT LOADED // const newDoc = new Y.Doc(); // const contentDoc = await this.db.getDoc(docId); // const newUpdate = squashYjsState( // Y.encodeStateAsUpdate(contentDoc) // ); // Y.applyUpdate(newDoc, newUpdate); // await this.db.setDoc(docId, newDoc, undefined, true); } } function fancyStringify(obj: any) { return JSON.stringify(obj, (key, value) => { if (value instanceof Map) { return { __map: true, entries: Array.from(value.entries()) }; } else if (value instanceof Set) { return { __set: true, entries: Array.from(value) }; } else { return value; } }); }