import * as dagJson from '@ipld/dag-json' import { Logger } from 'besonders-logger' import { CID } from 'multiformats/cid' import stringify from 'safe-stable-stringify' import { ensureTsPvAndFinalizeApplog } from '../applog/applog-helpers.ts' import type { Applog, ApplogArrayMaybeEncrypted, ApplogArrayMaybeEncryptedRO, ApplogArrayNoCIDMaybeEncryptedRO, ApplogEnc, ApplogEncNoCid, CidString, } from '../applog/datom-types.ts' import { BlockStoreish, DecodedCar, getDecodedBlock, makeCarBlob } from '../ipfs/car.ts' import { encodeBlockOriginal, prepareForPub } from '../ipfs/ipfs-utils.ts' import { lastWriteWins } from './../query/basic.ts' import { anyOf } from '../query/matchers.ts' import { ApplogsOrThread, getLogsFromThread, Thread } from '../thread.ts' import { rollingFilter } from '../thread/filters.ts' import { keepTruthy } from '../utils.ts' import type { AppAgent, IShare, SnapBlockChunks, SnapBlockLogs, SnapBlockLogsOrChunks } from './pubsub-types.ts' const { WARN, LOG, DEBUG, VERBOSE, ERROR } = Logger.setup(Logger.INFO) // eslint-disable-line no-unused-vars // export const neverEncryptAttrs = [ // 'agent/jwkd', // 'agent/appAgent', // 'pub/encryptedFor', // 'pub/sharedKey', // ] // export interface WovinPublicationInfo { // id: string // } export async function prepareSnapshotForPush( agent: AppAgent, appThread: Thread, threadToPublish: ApplogsOrThread, share: IShare, prevSnapCID: CID | null, prevCounter: number | null, ) { if (prevCounter !== null && !prevSnapCID) { throw ERROR('[prepareSnapshotForPush] prevCounter provided without prevSnapCID') } // TODO prevent publish if there is no new info let logsToPublish = getLogsFromThread(threadToPublish) // const logsFromLastPush = await getLogsFromPub(publication) // logsToPublish = logsToPublish.filter(eachLog => !logsFromLastPush.includes(eachLog.cid)) // TODO deep compare includes // const prevPushCIDs = [ // publication.lastCID, // //TODO add this one and update the publication data after push // ] // const includedLogCIDs = [ // 'full array of CIDS from all previous pushes' // ] DEBUG(`[preparePubForPush] Collected ${logsToPublish.length} logs :`, { logsToPublish, threadOrLogsCount: (threadToPublish as any).nameAndSizeUntracked || (`[${(threadToPublish as any).length}]`), }) const { sharedAgents, sharedKeyMap, sharedKey, pubCounter } = share ?? {} const getExistingOrNewLog = (thread: Thread, share: IShare, ag: string, at: string, vl) => { let logInQuestion = rollingFilter(lastWriteWins(thread), { en: share.id, at }).latestLog if (!logInQuestion && vl !== undefined) { logInQuestion = ensureTsPvAndFinalizeApplog({ ag, en: share.id, at, vl }, thread) } return logInQuestion // can be undefined if the passed vl is undefined and the log is not found } const shareNameLog = getExistingOrNewLog(appThread, share, agent.ag, 'share/name', share.name) // ? using did as it is derived from the same ecdh in note3 and part of the minimal AppAgent type required here in wovin core const shareCounterLog = getExistingOrNewLog(appThread, share, agent.ag, 'share/counter', `${agent.did}<::>${pubCounter}`) // ? discuss if this works to bind the counter to a specific derivation key - did is not necessarily derived from the same key by all lib users const encryptApplog = async (applog: Applog, keyToUse: CryptoKey): Promise => { const { log: eachLog, cid } = prepareForPub(applog) // without cid const enc = new TextEncoder() const stringified = stringify(eachLog) const stringifiedEncodedAppLogPayload = enc.encode(stringified) // TODO: consider encodeToDagJson instead VERBOSE('[odd]', { eachLog, stringified, stringifiedEncodedAppLogPayload }) try { // @ts-expect-error const encPayload = await agent.crypto?.aes.encrypt(stringifiedEncodedAppLogPayload, keyToUse, 'AES-GCM') // TODO get rid of odd down here VERBOSE('[odd] encrypted length:', stringifiedEncodedAppLogPayload.length, { encPayload }) return encPayload } catch (err) { throw ERROR('FAILED TO ENC payload length:', stringifiedEncodedAppLogPayload.length, { err }) } // const decrypted = await decryptWithAesSharedKey(encPayload, keyToUse, 'string') } let maybeEncryptedApplogs: ApplogEncNoCid[] | readonly Applog[] const encryptedApplogs = [] as { enc: Uint8Array }[] const agentSharedKeyLogs = [] if (sharedAgents) { // encrypt all Applogs if (!sharedKey || !sharedKeyMap) { throw ERROR('sharedAgents but no Keys/Map', { sharedAgents, sharedKeyMap, sharedKey }) } VERBOSE('encrypting', { sharedAgents, sharedKeyMap }) for (const [eachAgent, eachEncKey] of Array.from(sharedKeyMap.entries())) { VERBOSE('adding key', { eachAgent, eachEncKey }) agentSharedKeyLogs.push({ ag: agent.ag, en: eachAgent, at: 'share/sharedKey', vl: eachEncKey, // these are encrypted with the derived key from the local agent private and remote agent public keys }) } // const encryptedForLogs = await insertApplogsInAppDB(agentSharedKeyLogs) // DEBUG(`[publish] adding agentSharedKeyLogs:`, encryptedForLogs) const CIDlist: { cid: CidString; encCID?: CidString }[] = [] const pubCIDmap: Record = {} // TODO ensure that all needed keys are in for (const eachLog of logsToPublish) { VERBOSE('[crypto] encrypting ', { eachLog, sharedKey }) // try { const encPayload = await encryptApplog(eachLog, sharedKey) DEBUG('[crypto] encrypted ', { eachLog, encPayload, sharedKey }) // } catch (err) { // // its already traced in encryptAndTestDecrypt // // continue // } encryptedApplogs.push({ enc: encPayload }) } maybeEncryptedApplogs = encryptedApplogs } else { maybeEncryptedApplogs = logsToPublish // publish nonEncrypted } DEBUG('adding all agent info and shareAtoms', { share, agent, logsToPublish, // threadToPublish, - very verbose agentSharedKeyLogs, }) const infoLogs = [ ...rollingFilter(lastWriteWins(appThread), { // TODO: use static filter for performance en: agent.ag, at: anyOf('agent/ecdh', 'agent/jwkd', 'agent/appAgent'), }).applogs, ...(shareNameLog ? [shareNameLog] : []), ...(shareCounterLog ? [shareCounterLog] : []), ...agentSharedKeyLogs, ] DEBUG(`[prepareSnapshotForPush] info logs:`, infoLogs) if (!infoLogs.find(({ at }) => at === 'agent/appAgent')) throw ERROR(`[prepareSnapshotForPush] appThread missing agent/appAgent log`) const applogsToEncode = keepTruthy(maybeEncryptedApplogs) const infologsToEncode = keepTruthy(infoLogs) if (!applogsToEncode.length) { throw ERROR('no valid applogs', { agent, maybeEncryptedApplogs, infoLogs, applogsToEncode, infologsToEncode, prevSnapCID }) } if (!infologsToEncode.length) { throw ERROR('no valid infologs', { agent, maybeEncryptedApplogs, infoLogs, applogsToEncode, infologsToEncode, prevSnapCID }) } const encodedSnapshot = await encodeSnapshotAsCar(agent, applogsToEncode, infologsToEncode, prevSnapCID, prevCounter) DEBUG('inPrepareSnapshotForPush', { encodedSnapshot }) return encodedSnapshot } /** * @param applogs Encrypted or plain applogs * @returns Car file */ export async function encodeSnapshotAsCar( agent: AppAgent, applogs: ApplogArrayNoCIDMaybeEncryptedRO, infoLogs: readonly Applog[], prevSnapCID: CID | null, prevCounter: number | null, ) { DEBUG(`[encodeSnapshotAsCar] encoding`, { agent, applogs, infoLogs }) const { cids: infoLogCids, encodedApplogs: encodedInfoLogs } = await encodeApplogsAsIPLD(infoLogs) const { cids: applogCids, encodedApplogs } = await encodeApplogsAsIPLD(applogs) let blocks = encodedApplogs.concat(encodedInfoLogs) // We need to wrap the array to get a CID const infoLogsWrap = await encodeBlockOriginal({ logs: infoLogCids }) blocks.push(infoLogsWrap) const { rootCID: chunkRootCID, blocks: chunkBlocks } = await chunkApplogs(applogCids) blocks = blocks.concat(chunkBlocks) // (i) concat bc. https://stackoverflow.com/a/51860949 const infoSignature = await agent.sign(infoLogsWrap.cid.bytes) const applogsSignature = await agent.sign(chunkRootCID.bytes) const root = { info: infoLogsWrap.cid, applogs: chunkRootCID, infoSignature, applogsSignature, prev: prevSnapCID, prevCounter: !prevSnapCID ? 0 : prevCounter !== null ? prevCounter + 1 : null, } DEBUG('[encodeSnapshotAsCar] encoding root', { root, logCids: applogCids, infoLogCids }) const encodedRoot = await encodeBlockOriginal(root) blocks.push(encodedRoot) DEBUG('[encodeSnapshotAsCar] => root', { encodedRoot }) return { cid: encodedRoot.cid, blob: await makeCarBlob(encodedRoot.cid, blocks), // TODO: create CarBuilder (incl .encodeAndAdd({...})) blocks, infoLogCids, applogCids, } } /** (i) IPFS has a block size limit of 1MB - which is about 15K CIDs */ export async function chunkApplogs(applogCids: CID[], size = 10000) { if (!applogCids.length) throw ERROR(`[chunkApplogs] called with empty array`) const chunks = [] // TODO: chunk by stable btree based on size or something like that for (let i = 0; i < applogCids.length; i += size) { const chunk = await encodeBlockOriginal({ logs: applogCids.slice(i, Math.min(i + applogCids.length, i + size)) }) chunks.push(chunk) } if (chunks.length === 1) return { rootCID: chunks[0].cid, blocks: chunks } const root = await encodeBlockOriginal({ chunks: chunks.map(chunk => chunk.cid) }) const blocks = [root, ...chunks] DEBUG(`[chunkApplogs] ${applogCids.length} logs chunked into ${chunks.length}`, { applogCids, root, blocks, chunks, dagJson }) return { rootCID: root.cid, blocks, chunks } } export async function unchunkApplogsBlock(block: SnapBlockLogsOrChunks | null | undefined, blockStore: BlockStoreish): Promise { if (!block) return [] if (isSnapBlockChunks(block)) { return (await Promise.all( block.chunks.map(async (chunkCid) => { const block = (await getDecodedBlock(blockStore, chunkCid)) as SnapBlockLogs if (!block?.logs) throw ERROR(`Weird chunk`, block) return block.logs }), )).flat() } else { return block.logs ?? [] } } export function isSnapBlockChunks(block: SnapBlockLogsOrChunks | null | undefined): block is SnapBlockChunks { return !!block && 'chunks' in block } /** * @param applogs Encrypted or plain applogs * @returns Car file */ export async function encodeSnapshotApplogsAsCar( applogs: ApplogArrayMaybeEncryptedRO, ) { const encoded = await encodeApplogsAsIPLD(applogs) if (!encoded) throw ERROR('invalid applogs cannot continue', { applogs, encoded }) const { cids, encodedApplogs } = encoded const root = { applogs: cids } const encodedRoot = await encodeBlockOriginal(root) DEBUG('[encodeSnapshotApplogsAsCar] encoded root', { cids, encodedRoot }) return await makeCarBlob(encodedRoot.cid, [encodedRoot, ...encodedApplogs]) } async function encodeApplogsAsIPLD(applogs: ApplogArrayNoCIDMaybeEncryptedRO) { DEBUG({ applogs }) const validApplogs = applogs.filter(eachLog => !!eachLog) DEBUG({ validApplogs }) if (!validApplogs.length) throw ERROR('no valid applogs') const preppedLogs = validApplogs.map(log => prepareForPub(log as Applog).log) const encodedApplogs = await Promise.all(preppedLogs.map(encodeBlockOriginal)) DEBUG('[encodeApplogsAsIpld] encoded applogs', { preppedLogs, encodedApplogs }) const cids = encodedApplogs.map(b => { if (!b.cid) throw ERROR(`[publish] no cid for encoded log:`, b) return b.cid }) return { cids, encodedApplogs } }