{
  "version": 3,
  "sources": ["../../src/modules/diviner-payload-generic/Diviner.ts"],
  "sourcesContent": ["import type { EventListener, Hex } from '@ariestools/sdk'\nimport {\n  assertEx, filterAs, forget,\n  zodAsFactory, zodIsFactory, zodToFactory,\n} from '@ariestools/sdk'\nimport { PayloadBuilder } from '@xyo-network/sdk-protocol'\nimport {\n  asSchema,\n  type Payload, type Schema,\n  type Sequence,\n  type WithStorageMeta,\n} from '@xyo-network/sdk-protocol'\nimport { Mutex } from 'async-mutex'\nimport * as z from 'zod/mini'\n\nimport type { ArchivistInstance, ArchivistModuleEventData } from '#archivist-model'\nimport type { DivinerInstance, DivinerModuleEventData } from '#diviner-model'\nimport { PayloadDiviner } from '#diviner-payload-abstract'\nimport type {\n  Order,\n  PayloadDivinerConfig,\n  PayloadDivinerParams,\n  PayloadDivinerQueryPayload,\n} from '#diviner-payload-model'\nimport { asPayloadDivinerQueryPayload, PayloadDivinerConfigZod } from '#diviner-payload-model'\nimport { creatableModule } from '#module-model'\n\nconst DEFAULT_INDEX_BATCH_SIZE = 100 as const\nconst DEFAULT_MAX_INDEX_SIZE = 8000 as const\n\nexport const GenericPayloadDivinerConfigSchema: 'network.xyo.diviner.payload.generic.config' & {\n  readonly __schema: true\n} = asSchema('network.xyo.diviner.payload.generic.config', true)\nexport type GenericPayloadDivinerConfigSchema = typeof GenericPayloadDivinerConfigSchema\n\nexport const GenericPayloadDivinerConfigZod: z.ZodMiniObject<\n  Omit<(typeof PayloadDivinerConfigZod)['shape'], 'schema'> & {\n    indexes: z.ZodMiniOptional<z.ZodMiniArray<z.ZodMiniString<string>>>\n    schema: z.ZodMiniLiteral<GenericPayloadDivinerConfigSchema>\n  },\n  z.core.$strip\n> = z.extend(PayloadDivinerConfigZod, {\n  indexes: z.optional(z.array(z.string())),\n  schema: z.literal(GenericPayloadDivinerConfigSchema),\n})\n\nexport type GenericPayloadDivinerConfigBase = z.infer<typeof GenericPayloadDivinerConfigZod>\n\nexport const isGenericPayloadDivinerConfig: ReturnType<typeof zodIsFactory<GenericPayloadDivinerConfigBase>> = zodIsFactory(GenericPayloadDivinerConfigZod)\nexport const asGenericPayloadDivinerConfig: ReturnType<typeof zodAsFactory<GenericPayloadDivinerConfigBase>> = zodAsFactory(GenericPayloadDivinerConfigZod, 'asGenericPayloadDivinerConfig')\nexport const toGenericPayloadDivinerConfig: ReturnType<typeof zodToFactory<GenericPayloadDivinerConfigBase>> = zodToFactory(GenericPayloadDivinerConfigZod, 'toGenericPayloadDivinerConfig')\n\nexport type GenericPayloadDivinerConfig = PayloadDivinerConfig<\n  {\n    indexes?: string[]\n  },\n  GenericPayloadDivinerConfigSchema\n>\n\n@creatableModule()\nexport class GenericPayloadDiviner<\n  TParams extends PayloadDivinerParams<GenericPayloadDivinerConfig> = PayloadDivinerParams<GenericPayloadDivinerConfig>,\n  TIn extends PayloadDivinerQueryPayload = PayloadDivinerQueryPayload,\n  TOut extends WithStorageMeta<Payload> = WithStorageMeta<Payload>,\n  TEventData extends DivinerModuleEventData<DivinerInstance<TParams, TIn, TOut>, TIn, TOut> = DivinerModuleEventData<\n    DivinerInstance<TParams, TIn, TOut>,\n    TIn,\n    TOut\n  >,\n> extends PayloadDiviner<TParams, TIn, TOut, TEventData> {\n  static override readonly configSchemas: Schema[] = [...super.configSchemas, GenericPayloadDivinerConfigSchema]\n  static override readonly defaultConfigSchema: Schema = GenericPayloadDivinerConfigSchema\n\n  protected indexMaps: Record<string, WithStorageMeta<TOut>[]> = {}\n  protected payloadsWithMeta: WithStorageMeta<TOut>[] = []\n\n  private _archivistInstance?: ArchivistInstance\n  private _cursor?: Sequence\n  private _updatePayloadPairsMutex = new Mutex()\n\n  protected get indexBatchSize(): number {\n    return this.config.indexBatchSize ?? DEFAULT_INDEX_BATCH_SIZE\n  }\n\n  protected get indexes(): string[] {\n    return ['schema', ...(this.config.indexes ?? [])]\n  }\n\n  protected get maxIndexSize(): number {\n    return this.config.maxIndexSize ?? DEFAULT_MAX_INDEX_SIZE\n  }\n\n  protected all(order: Order = 'desc', cursor?: Hex): WithStorageMeta<TOut>[] {\n    const payloads = this.payloadsWithMeta.toSorted(PayloadBuilder.compareStorageMeta)\n    if (order === 'desc') payloads.reverse()\n    const startIndex = (cursor ? (payloads.findIndex(payload => payload._sequence === cursor) ?? -1) : -1) + 1\n    return payloads.slice(startIndex)\n  }\n\n  protected override async archivistInstance(): Promise<ArchivistInstance | undefined>\n  protected override async archivistInstance(required: true): Promise<ArchivistInstance>\n  protected override async archivistInstance(required = false): Promise<ArchivistInstance | undefined> {\n    if (!this._archivistInstance) {\n      const archivist = await super.archivistInstance()\n      if (required && !archivist) {\n        throw new Error('Failed to find archivist')\n      }\n      archivist?.on('inserted', this.onArchivistInserted)\n      archivist?.on('cleared', this.onArchivistCleared)\n      archivist?.on('deleted', this.onArchivistDeleted)\n      this._archivistInstance = archivist\n    }\n    return this._archivistInstance\n  }\n\n  protected async clearIndex(): Promise<void> {\n    await this._updatePayloadPairsMutex.runExclusive(() => {\n      this._cursor = undefined\n      this.payloadsWithMeta = []\n      this.indexMaps = {}\n    })\n  }\n\n  protected override async divineHandler(payloads?: TIn[]): Promise<TOut[]> {\n    const filters = filterAs(payloads ?? [], asPayloadDivinerQueryPayload)\n    assertEx(filters.length < 2, () => 'Multiple PayloadDivinerQuery payloads may not be specified')\n    const filter = assertEx(filters.shift(), () => 'No PayloadDivinerQuery specified')\n    await this.updateIndex()\n\n    const {\n\n      schema, schemas, order, limit, cursor, ...props\n    } = filter\n    let all: TOut[] = this.all(order, cursor)\n    if (all) {\n      if (schemas?.length) all = all.filter(payload => schemas.includes(payload.schema))\n      if (Object.keys(props).length > 0) {\n        const additionalFilterCriteria = Object.entries(props)\n        for (const [prop, filter] of additionalFilterCriteria) {\n          const property = prop as keyof TOut\n          all\n            = Array.isArray(filter)\n              ? all.filter(payload =>\n                  filter.every((value) => {\n                    const prop = payload?.[property]\n                    // TODO: This seems to be written just to check arrays, and now that $meta is there, need to check type?\n                    return Array.isArray(prop) && prop.includes?.(value)\n                  }))\n              : all.filter(payload => payload?.[property] === filter)\n        }\n      }\n      return limit ? all.slice(0, limit) : all\n    } else {\n      throw new Error('Archivist does not support \"all\"')\n    }\n  }\n\n  protected onArchivistCleared: EventListener<ArchivistModuleEventData['cleared']> = () => {\n    forget(\n      (async () => {\n        await this.clearIndex()\n        await this.updateIndex()\n      })(),\n    )\n  }\n\n  // we are just rebuilding the entire index at this point on delete since large archivists do not support delete\n  protected onArchivistDeleted: EventListener<ArchivistModuleEventData['deleted']> = () => {\n    forget(\n      (async () => {\n        await this.clearIndex()\n        await this.updateIndex()\n      })(),\n    )\n  }\n\n  protected onArchivistInserted: EventListener<ArchivistModuleEventData['inserted']> = () => {\n    forget(this.updateIndex())\n  }\n\n  protected override async stopHandler(_timeout?: number): Promise<void> {\n    await super.stopHandler()\n    const archivist = await this.archivistInstance(true)\n    archivist.off('inserted', this.onArchivistInserted)\n    archivist.off('deleted', this.onArchivistDeleted)\n    archivist.off('cleared', this.onArchivistCleared)\n  }\n\n  // index any new payloads\n  protected async updateIndex(): Promise<void> {\n    await this._updatePayloadPairsMutex.runExclusive(async () => {\n      const archivist = await this.archivistInstance(true)\n      let newPayloads = await archivist.next({ limit: 100, cursor: this._cursor }) as TOut[]\n      while (newPayloads.length > 0) {\n        this._cursor = newPayloads.at(-1)?._sequence\n        assertEx(this.payloadsWithMeta.length + newPayloads.length <= this.maxIndexSize, () => 'maxIndexSize exceeded')\n        this.indexPayloads(newPayloads)\n        newPayloads = await archivist.next({ limit: 100, cursor: this._cursor }) as TOut[]\n      }\n    })\n  }\n\n  private indexPayloads(payloads: WithStorageMeta<TOut>[]): Hex {\n    this.payloadsWithMeta.push(...payloads)\n\n    // update the custom indexes\n    for (const index of this.indexes ?? []) {\n      this.indexMaps[index] = this.indexMaps[index] ?? []\n      for (const payload of payloads) {\n        this.indexMaps[index].push(payload)\n      }\n    }\n    return assertEx(payloads.at(-1), () => 'No payloads to index')._sequence\n  }\n}\n"],
  "mappings": ";;;;;;;;;;;;;;;;;AACA;AAAA,EACE;AAAA,EAAU;AAAA,EAAU;AAAA,EACpB;AAAA,EAAc;AAAA,EAAc;AAAA,OACvB;AACP,SAAS,sBAAsB;AAC/B;AAAA,EACE;AAAA,OAIK;AACP,SAAS,aAAa;AACtB,YAAY,OAAO;AAInB,SAAS,sBAAsB;AAO/B,SAAS,8BAA8B,+BAA+B;AACtE,SAAS,uBAAuB;AAEhC,IAAM,2BAA2B;AACjC,IAAM,yBAAyB;AAExB,IAAM,oCAET,SAAS,8CAA8C,IAAI;AAGxD,IAAM,iCAMP,SAAO,yBAAyB;AAAA,EACpC,SAAW,WAAW,QAAQ,SAAO,CAAC,CAAC;AAAA,EACvC,QAAU,UAAQ,iCAAiC;AACrD,CAAC;AAIM,IAAM,gCAAkG,aAAa,8BAA8B;AACnJ,IAAM,gCAAkG,aAAa,gCAAgC,+BAA+B;AACpL,IAAM,gCAAkG,aAAa,gCAAgC,+BAA+B;AAUpL,IAAM,wBAAN,cASG,eAA+C;AAAA,EAI7C,YAAqD,CAAC;AAAA,EACtD,mBAA4C,CAAC;AAAA,EAE/C;AAAA,EACA;AAAA,EACA,2BAA2B,IAAI,MAAM;AAAA,EAE7C,IAAc,iBAAyB;AACrC,WAAO,KAAK,OAAO,kBAAkB;AAAA,EACvC;AAAA,EAEA,IAAc,UAAoB;AAChC,WAAO,CAAC,UAAU,GAAI,KAAK,OAAO,WAAW,CAAC,CAAE;AAAA,EAClD;AAAA,EAEA,IAAc,eAAuB;AACnC,WAAO,KAAK,OAAO,gBAAgB;AAAA,EACrC;AAAA,EAEU,IAAI,QAAe,QAAQ,QAAuC;AAC1E,UAAM,WAAW,KAAK,iBAAiB,SAAS,eAAe,kBAAkB;AACjF,QAAI,UAAU,OAAQ,UAAS,QAAQ;AACvC,UAAM,cAAc,SAAU,SAAS,UAAU,aAAW,QAAQ,cAAc,MAAM,KAAK,KAAM,MAAM;AACzG,WAAO,SAAS,MAAM,UAAU;AAAA,EAClC;AAAA,EAIA,MAAyB,kBAAkB,WAAW,OAA+C;AACnG,QAAI,CAAC,KAAK,oBAAoB;AAC5B,YAAM,YAAY,MAAM,MAAM,kBAAkB;AAChD,UAAI,YAAY,CAAC,WAAW;AAC1B,cAAM,IAAI,MAAM,0BAA0B;AAAA,MAC5C;AACA,iBAAW,GAAG,YAAY,KAAK,mBAAmB;AAClD,iBAAW,GAAG,WAAW,KAAK,kBAAkB;AAChD,iBAAW,GAAG,WAAW,KAAK,kBAAkB;AAChD,WAAK,qBAAqB;AAAA,IAC5B;AACA,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,MAAgB,aAA4B;AAC1C,UAAM,KAAK,yBAAyB,aAAa,MAAM;AACrD,WAAK,UAAU;AACf,WAAK,mBAAmB,CAAC;AACzB,WAAK,YAAY,CAAC;AAAA,IACpB,CAAC;AAAA,EACH;AAAA,EAEA,MAAyB,cAAc,UAAmC;AACxE,UAAM,UAAU,SAAS,YAAY,CAAC,GAAG,4BAA4B;AACrE,aAAS,QAAQ,SAAS,GAAG,MAAM,4DAA4D;AAC/F,UAAM,SAAS,SAAS,QAAQ,MAAM,GAAG,MAAM,kCAAkC;AACjF,UAAM,KAAK,YAAY;AAEvB,UAAM;AAAA,MAEJ;AAAA,MAAQ;AAAA,MAAS;AAAA,MAAO;AAAA,MAAO;AAAA,MAAQ,GAAG;AAAA,IAC5C,IAAI;AACJ,QAAI,MAAc,KAAK,IAAI,OAAO,MAAM;AACxC,QAAI,KAAK;AACP,UAAI,SAAS,OAAQ,OAAM,IAAI,OAAO,aAAW,QAAQ,SAAS,QAAQ,MAAM,CAAC;AACjF,UAAI,OAAO,KAAK,KAAK,EAAE,SAAS,GAAG;AACjC,cAAM,2BAA2B,OAAO,QAAQ,KAAK;AACrD,mBAAW,CAAC,MAAMA,OAAM,KAAK,0BAA0B;AACrD,gBAAM,WAAW;AACjB,gBACI,MAAM,QAAQA,OAAM,IAClB,IAAI,OAAO,aACTA,QAAO,MAAM,CAAC,UAAU;AACtB,kBAAMC,QAAO,UAAU,QAAQ;AAE/B,mBAAO,MAAM,QAAQA,KAAI,KAAKA,MAAK,WAAW,KAAK;AAAA,UACrD,CAAC,CAAC,IACJ,IAAI,OAAO,aAAW,UAAU,QAAQ,MAAMD,OAAM;AAAA,QAC5D;AAAA,MACF;AACA,aAAO,QAAQ,IAAI,MAAM,GAAG,KAAK,IAAI;AAAA,IACvC,OAAO;AACL,YAAM,IAAI,MAAM,kCAAkC;AAAA,IACpD;AAAA,EACF;AAAA,EAEU,qBAAyE,MAAM;AACvF;AAAA,OACG,YAAY;AACX,cAAM,KAAK,WAAW;AACtB,cAAM,KAAK,YAAY;AAAA,MACzB,GAAG;AAAA,IACL;AAAA,EACF;AAAA;AAAA,EAGU,qBAAyE,MAAM;AACvF;AAAA,OACG,YAAY;AACX,cAAM,KAAK,WAAW;AACtB,cAAM,KAAK,YAAY;AAAA,MACzB,GAAG;AAAA,IACL;AAAA,EACF;AAAA,EAEU,sBAA2E,MAAM;AACzF,WAAO,KAAK,YAAY,CAAC;AAAA,EAC3B;AAAA,EAEA,MAAyB,YAAY,UAAkC;AACrE,UAAM,MAAM,YAAY;AACxB,UAAM,YAAY,MAAM,KAAK,kBAAkB,IAAI;AACnD,cAAU,IAAI,YAAY,KAAK,mBAAmB;AAClD,cAAU,IAAI,WAAW,KAAK,kBAAkB;AAChD,cAAU,IAAI,WAAW,KAAK,kBAAkB;AAAA,EAClD;AAAA;AAAA,EAGA,MAAgB,cAA6B;AAC3C,UAAM,KAAK,yBAAyB,aAAa,YAAY;AAC3D,YAAM,YAAY,MAAM,KAAK,kBAAkB,IAAI;AACnD,UAAI,cAAc,MAAM,UAAU,KAAK,EAAE,OAAO,KAAK,QAAQ,KAAK,QAAQ,CAAC;AAC3E,aAAO,YAAY,SAAS,GAAG;AAC7B,aAAK,UAAU,YAAY,GAAG,EAAE,GAAG;AACnC,iBAAS,KAAK,iBAAiB,SAAS,YAAY,UAAU,KAAK,cAAc,MAAM,uBAAuB;AAC9G,aAAK,cAAc,WAAW;AAC9B,sBAAc,MAAM,UAAU,KAAK,EAAE,OAAO,KAAK,QAAQ,KAAK,QAAQ,CAAC;AAAA,MACzE;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEQ,cAAc,UAAwC;AAC5D,SAAK,iBAAiB,KAAK,GAAG,QAAQ;AAGtC,eAAW,SAAS,KAAK,WAAW,CAAC,GAAG;AACtC,WAAK,UAAU,KAAK,IAAI,KAAK,UAAU,KAAK,KAAK,CAAC;AAClD,iBAAW,WAAW,UAAU;AAC9B,aAAK,UAAU,KAAK,EAAE,KAAK,OAAO;AAAA,MACpC;AAAA,IACF;AACA,WAAO,SAAS,SAAS,GAAG,EAAE,GAAG,MAAM,sBAAsB,EAAE;AAAA,EACjE;AACF;AAhJE,cAVW,uBAUc,iBAA0B,CAAC,GAAG,yDAAM,kBAAe,iCAAiC;AAC7G,cAXW,uBAWc,uBAA8B;AAX5C,wBAAN;AAAA,EADN,gBAAgB;AAAA,GACJ;",
  "names": ["filter", "prop"]
}