import { Channel, Deferred } from "@nostr-fetch/kernel/channel"; import { verifyEventSig } from "@nostr-fetch/kernel/crypto"; import { DebugLogger } from "@nostr-fetch/kernel/debugLogger"; import { type EnsureRelaysOptions, type FetchTillEoseOptions, type NostrFetcherBackend, type NostrFetcherBackendInitializer, type NostrFetcherCommonOptions, defaultFetcherCommonOptions, isFetchTillEoseAbortedSignal, isFetchTillEoseFailedSignal, } from "@nostr-fetch/kernel/fetcherBackend"; import { type EventVerifier, type NostrEvent, isValidTagQueryKey } from "@nostr-fetch/kernel/nostr"; import { abbreviate, currUnixtimeSec, normalizeRelayUrlSet } from "@nostr-fetch/kernel/utils"; import { DefaultFetcherBackend, type DefaultFetcherBackendOptions, defaultDefaultFetcherBackendOptions, } from "./fetcherBackend"; import { EventBuckets, FetchStatsManager, KeyRelayMatrix, ProgressTracker, type RelayCapCheckerInitializer, type RelayCapabilityChecker, assertReq, checkIfNonEmpty, checkIfTagQueriesAreValid, checkIfTimeRangeIsValid, checkIfTrue, compareNostrEvents, getKeysOfEvent, initDefaultRelayCapChecker, initSeenEvents, makeBreakableSignal, } from "./fetcherHelper"; import { type FetchFilter, type FetchFilterKeyElem, type FetchFilterKeyName, type FetchStatsListener, type FetchTimeRangeFilter, NostrFetchError, } from "./types"; const MAX_LIMIT_PER_REQ = 5000; const MAX_LIMIT_PER_REQ_IN_BACKPRESSURE = 500; const MIN_HIGH_WATER_MARK = 5000; /** * Nostr event with extra fields. */ export type NostrEventExt = NostrEvent & { seenOn: SeenOn extends true ? string[] : undefined; }; /** * Pair of the "key" of events and list of events which have that key. * * It is the type of elements of `AsyncIterable` returned from {@linkcode NostrFetcher.fetchLatestEventsPerKey}. */ export type NostrEventListWithKey = { key: FetchFilterKeyElem; events: NostrEventExt[]; }; /** * Pair of the "key" of an event and the event which has that key. If no event found matching the key, it will be `undefined`. * * It is the type of elements of `AsyncIterable` returned from {@linkcode NostrFetcher.fetchLastEventPerKey}. */ export type NostrEventWithKey = { key: FetchFilterKeyElem; event: NostrEventExt | undefined; }; /** * Pair of the pubkey of event author and list of events from that author. * * It is the type of elements of `AsyncIterable` returned from {@linkcode NostrFetcher.fetchLatestEventsPerAuthor}. */ export type NostrEventListWithAuthor = { author: string; events: NostrEventExt[]; }; /** * Pair of the pubkye of event author and an event from that author. If no event found from the author, it will be `undefined`. * * It is the type of elements of `AsyncIterable` returned from {@linkcode NostrFetcher.fetchLastEventPerAuthor}. */ export type NostrEventWithAuthor = { author: string; event: NostrEventExt | undefined; }; /** * Common options for all the fetch methods. */ export type FetchOptions = { /** * If specified, the fetcher uses the given function as an event signature verifier instead of the default one. * * The function must return `true` if the event signature is valid. Otherwise, it should return `false`. * * @example * // How to use nostr-wasm's verifyEvent() with nostr-fetch * import { NostrFetcher, type NostrEvent } from "nostr-fetch"; * import { initNostrWasm } from "nostr-wasm"; * * const nw = await initNostrWasm(); * const nwVerifyEvent = (ev: NostrEvent) => { * try { * nw.verifyEvent(ev); * return true; * } catch { * return false; * } * }; * const fetcher = NostrFetcher.init({ * eventVerifier: nwVerifyEvent, * }); */ eventVerifier?: EventVerifier; /** * If true, the fetcher skips event signature verification. * * Note: This option has no effect under some relay pool adapters. * Check the document of the relay pool adapter you want to use. * * @default false * * @deprecated This option will be removed in nostr-fetch v1. Set the `noopVerifier` to the `eventVerifier` option instead of turning this on. */ skipVerification?: boolean; /** * If true, the fetcher skips a check that events from relays are certainly match with filters in REQuest. * * By default, fetchers perform the check to protect clients from malicious relays. * * @default false */ skipFilterMatching?: boolean; /** * If true, `seenOn` property is appended to every returned events. * The value of `seenOn` is array of relay URLs on which the event have been seen. * * @default false */ withSeenOn?: SeenOn; /** * The function for listening fetch statistics. * * @default undefined */ statsListener?: FetchStatsListener | undefined; /** * How often fetch statistics is notified to the listener (specified via `statsListener`), in milliseconds. * * @default 1000 */ statsNotifIntervalMs?: number; /** * The maximum amount of time allowed to attempt to connect to relays, in milliseconds. * * @default 5000 */ connectTimeoutMs?: number; /** * The `AbortSignal` used to abort an event fetching. * * @default undefined */ signal?: AbortSignal | undefined; /** * The `AbortSignal` used to abort an event fetching. * * @default undefined * * @deprecated Set `signal` option instead. */ abortSignal?: AbortSignal | undefined; /** * The maximum amount of time to wait for events from relay before a subscription is automatically aborted before EOSE, in milliseconds. * * @default 10000 */ abortSubBeforeEoseTimeoutMs?: number; /** * `limit` value to be used in internal subscriptions. * You may want to lower this value if relays you use have limit on value of `limit`. * * @default 5000 */ limitPerReq?: number; }; const defaultFetchOptions: Required = { eventVerifier: verifyEventSig, skipVerification: false, skipFilterMatching: false, withSeenOn: false, statsListener: undefined, statsNotifIntervalMs: 1000, connectTimeoutMs: 5000, signal: undefined, abortSignal: undefined, abortSubBeforeEoseTimeoutMs: 10000, limitPerReq: MAX_LIMIT_PER_REQ, }; /** * Options for {@linkcode NostrFetcher.allEventsIterator}. */ export type AllEventsIterOptions = FetchOptions & { /** * If true, the backpressure mode is enabled. * * In the backpressure mode, a fetcher is automatically slowed down when the consumer of events is slower than the fetcher (producer of events). * * This feature may be useful for jobs like transferring events from relays to other relays. * * @default false */ enableBackpressure?: boolean; }; const defaultAllEventsIterOptions: Required = { ...defaultFetchOptions, enableBackpressure: false, }; /** * Options for {@linkcode NostrFetcher.fetchAllEvents}. */ export type FetchAllOptions = FetchOptions & { /** * If true, resulting events are sorted in "newest to oldest" order. * * @default false */ sort?: boolean; }; const defaultFetchAllOptions: Required = { ...defaultFetchOptions, sort: false, }; /** * Options for "fetch latest N events" kind of fetchers, such as {@linkcode NostrFetcher.fetchLatestEvents}. */ export type FetchLatestOptions = FetchOptions & { /** * Takes unixtime in second. If specified, fetch latest events **as of the time**. * * Note: it is useful only for fetching *regular* events. Using this for replaceable events will result in an unexpected behavior. */ asOf?: number | undefined; /** * If true, the "reduced verification" mode is enabled. * * In the reduced verification mode, event signature verification is performed only to minimum amount of events enough to ensure validity. * * @default false * * @deprecated This option will be removed in nostr-fetch v1. Fetch with `eventVerifier: noopVerifier` first, then verify fetched events by yourself. */ reduceVerification?: boolean; }; const defaultFetchLatestOptions: Required = { ...defaultFetchOptions, asOf: undefined, reduceVerification: false, }; /** * Type of the first argument of {@linkcode NostrFetcher.fetchLatestEventsPerAuthor}/{@linkcode NostrFetcher.fetchLastEventPerAuthor}. */ export type KeysAndRelays = | { keys: FetchFilterKeyElem[]; relayUrls: string[]; } | Iterable<[key: FetchFilterKeyElem, relayUrls: string[]]>; /** * Use same relay set for all authors */ type RelaySetForAllKeys = { keys: FetchFilterKeyElem[]; relayUrls: string[]; }; /** * Use saperate relay set for each author. Typically `Map` */ type RelaySetsPerKey = Iterable< [key: FetchFilterKeyElem, relayUrls: string[]] >; const isRelaySetForAllKeys = ( kr: KeysAndRelays, ): kr is RelaySetForAllKeys => { return "relayUrls" in kr && "keys" in kr; }; const isRelaySetsPerKey = ( kr: KeysAndRelays, ): kr is RelaySetsPerKey => { return Symbol.iterator in Object(kr); }; /** * Type of the first argument of {@linkcode NostrFetcher.fetchLatestEventsPerAuthor}/{@linkcode NostrFetcher.fetchLastEventPerAuthor}. */ export type AuthorsAndRelays = RelaySetForAllAuthors | RelaySetsPerAuthor; /** * Use same relay set for all authors */ type RelaySetForAllAuthors = { authors: string[]; relayUrls: string[]; }; /** * Use saperate relay set for each author. Typically `Map` */ type RelaySetsPerAuthor = Iterable<[author: string, relayUrls: string[]]>; const isRelaySetForAllAuthors = (a2rs: AuthorsAndRelays): a2rs is RelaySetForAllAuthors => { return "relayUrls" in a2rs && "authors" in a2rs; }; const isRelaySetsPerAuthor = (a2rs: AuthorsAndRelays): a2rs is RelaySetsPerAuthor => { return Symbol.iterator in Object(a2rs); }; const adaptAuthorsAndRelays = (ar: AuthorsAndRelays): KeysAndRelays<"authors"> => { if (isRelaySetForAllAuthors(ar)) { return { keys: ar.authors, relayUrls: ar.relayUrls }; } if (isRelaySetsPerAuthor(ar)) { return ar; } throw Error("adaptAuthorsAndRelays: unreachable"); }; /** * The entry point of the Nostr event fetching. * * It sits on top of a Nostr relay pool implementation which manages connections to Nostr relays. It is recommended to reuse single `NostrFetcher` instance in entire app. * * You must instantiate `NostrFetcher` with static methods like {@linkcode NostrFetcher.init} or {@linkcode NostrFetcher.withCustomPool} instead of the constructor. */ export class NostrFetcher { #backend: NostrFetcherBackend; #relayCapChecker: RelayCapabilityChecker; #debugLogger: DebugLogger | undefined; private constructor( backend: NostrFetcherBackend, relayCapChecker: RelayCapabilityChecker, initOpts: Required, ) { this.#backend = backend; this.#relayCapChecker = relayCapChecker; if (initOpts.minLogLevel !== "none") { this.#debugLogger = new DebugLogger(initOpts.minLogLevel); } } /** * Initializes {@linkcode NostrFetcher} with the default relay pool implementation. * * If you are on an runtime that doesn't have a native WebSocket implementation (e.g. Node.js < v22), * you may want to set custom `WebSocket` constructor imported from an external package as follows: * * ```ts * import { NostrFetcher } from "nostr-fetch"; * import WebSocket from "ws"; * * const fetcher = NostrFetcher.init({ * webSocketConstructor: WebSocket * }); * ``` */ public static init( options: NostrFetcherCommonOptions & DefaultFetcherBackendOptions = {}, initRelayCapChecker: RelayCapCheckerInitializer = initDefaultRelayCapChecker, ): NostrFetcher { const finalOpts = { ...defaultFetcherCommonOptions, ...defaultDefaultFetcherBackendOptions, ...options, }; if (finalOpts.webSocketConstructor === undefined) { throw Error( "No WebSocket implementation found. Please explicitly set a custom `WebSocket` constructor to the `webSocketConstructor` option.", ); } const backend = new DefaultFetcherBackend(finalOpts); const relayCapChecker = initRelayCapChecker(finalOpts); return new NostrFetcher(backend, relayCapChecker, finalOpts); } /** * Initializes {@linkcode NostrFetcher} with the given adapted custom relay pool implementation. * * @example * ```ts * const pool = new SimplePool(); * const fetcher = NostrFetcher.withCustomPool(simplePoolAdapter(pool)); * ``` */ public static withCustomPool( poolAdapter: NostrFetcherBackendInitializer, options: NostrFetcherCommonOptions = {}, initRelayCapChecker: RelayCapCheckerInitializer = initDefaultRelayCapChecker, ): NostrFetcher { const finalOpts = { ...defaultFetcherCommonOptions, ...options }; const relayCapChecker = initRelayCapChecker(finalOpts); return new NostrFetcher(poolAdapter(finalOpts), relayCapChecker, finalOpts); } async #ensureRelaysWithCapCheck( relayUrls: string[], opts: EnsureRelaysOptions, requiredNips: number[], ): Promise { const connectedRelays = await this.#backend.ensureRelays(relayUrls, opts); if (requiredNips.length === 0) { // if capability check is not needed, return early return connectedRelays; } this.#debugLogger?.log("info", `required NIPs: ${requiredNips}`); const res: string[] = []; await Promise.all( connectedRelays.map(async (rurl) => { if (await this.#relayCapChecker.relaySupportsNips(rurl, requiredNips)) { res.push(rurl); } }), ); this.#debugLogger?.log("info", `eligible relays: ${res}`); return res; } #calcRequiredNips(filter: { search?: string }): number[] { const res: number[] = []; if ("search" in filter) { res.push(50); // NIP-50: Search Capability } return res; } /** * Returns an async iterable of all events matching the filter from Nostr relays specified by the array of URLs. * * You can iterate over events using `for-await-of` loop. * * Note: there are no guarantees about the order of returned events. * * Throws {@linkcode NostrFetchError} if `timeRangeFilter` is invalid (`since` > `until`). */ public allEventsIterator( relayUrls: string[], filter: FetchFilter, timeRangeFilter: FetchTimeRangeFilter, options: AllEventsIterOptions = {}, ): AsyncIterable> { assertReq( { relayUrls, filter, timeRangeFilter }, [ checkIfNonEmpty((r) => r.relayUrls, "warn", "Specify at least 1 relay URL"), checkIfTagQueriesAreValid((r) => r.filter, "error"), checkIfTimeRangeIsValid((r) => r.timeRangeFilter, "error"), ], this.#debugLogger, ); const filledOpts = { ...defaultAllEventsIterOptions, ...options, } as Required>; const finalOpts: Required> = { ...filledOpts, // if only `abortSignal` is specified, copy it to `signal` signal: filledOpts.signal ?? filledOpts.abortSignal, // use smaller limit if backpressure is enabled limitPerReq: filledOpts.enableBackpressure ? Math.min(filledOpts.limitPerReq, MAX_LIMIT_PER_REQ_IN_BACKPRESSURE) : filledOpts.limitPerReq, }; this.#debugLogger?.log("verbose", "finalOpts=%O", finalOpts); return this.#allEventsIterWithCleanupOnBreak(relayUrls, filter, timeRangeFilter, finalOpts); } async *#allEventsIterWithCleanupOnBreak( relayUrls: string[], filter: FetchFilter, timeRangeFilter: FetchTimeRangeFilter, options: Required>, ): AsyncIterable> { const [breakableSig, breakSignal] = makeBreakableSignal(options.signal); try { yield* this.#allEventsIterBody(relayUrls, filter, timeRangeFilter, { ...options, signal: breakableSig, }); } finally { // this block will be executed when: // - iteration is finished normally (whether throws or not) // - iteration is terminated early by break/return in a for-await-of loop // // in second case, breakSignal() causes the "cleanup" (abortion of event fetching by backend). breakSignal(); } } async *#allEventsIterBody( relayUrls: string[], filter: FetchFilter, timeRangeFilter: FetchTimeRangeFilter, options: Required>, ): AsyncIterable> { const statsMngr = FetchStatsManager.init(options.statsListener, options.statsNotifIntervalMs); const reqNips = this.#calcRequiredNips(filter); const eligibleRelayUrls = await this.#ensureRelaysWithCapCheck(relayUrls, options, reqNips); const highWaterMark = options.enableBackpressure ? Math.max(options.limitPerReq * eligibleRelayUrls.length, MIN_HIGH_WATER_MARK) : undefined; const [tx, chIter] = Channel.make>({ highWaterMark }); const globalSeenEvents = initSeenEvents(options.withSeenOn); const initialUntil = timeRangeFilter.until ?? currUnixtimeSec(); // codes for tracking progress // if `since` is undefined, duration is infinite (represented by undefined) const timeRangeDur = timeRangeFilter.since !== undefined ? Math.max(initialUntil - timeRangeFilter.since + 1, 1) : undefined; const progTracker = new ProgressTracker(eligibleRelayUrls); statsMngr?.setProgressMax(eligibleRelayUrls.length); statsMngr?.initRelayStats(relayUrls, eligibleRelayUrls, initialUntil); // fetch events from each relay Promise.all( eligibleRelayUrls.map(async (rurl) => { // repeat subscription until one of the following conditions is met: // 1. the relay didn't return new event // 2. aborted by AbortController // E. an error occurred while fetching events const logger = this.#debugLogger?.subLogger(rurl); let nextUntil = initialUntil; const localSeenEventIds = new Set(); while (true) { const refinedFilter = { ...timeRangeFilter, ...filter, until: nextUntil, // relays are supposed to return *latest* events by specifying `limit` explicitly (cf. [NIP-01](https://github.com/nostr-protocol/nips/blob/master/01.md)). // nostream doesn't accept a filter which has `limit` grater than 5000, so limit `limit` to this threshold or less. limit: Math.min(options.limitPerReq, MAX_LIMIT_PER_REQ), }; logger?.log("verbose", "refinedFilter=%O", refinedFilter); let gotNewEvent = false; let oldestCreatedAt = Number.MAX_SAFE_INTEGER; let isAboutToAbort = false; try { statsMngr?.subOpened(); for await (const e of this.#backend.fetchTillEose(rurl, refinedFilter, options)) { // eliminate duplicated events if (!localSeenEventIds.has(e.id)) { // hasn't seen the event on this relay gotNewEvent = true; localSeenEventIds.add(e.id); if (e.created_at < oldestCreatedAt) { oldestCreatedAt = e.created_at; } const { hasSeen, seenOn } = globalSeenEvents.report(e, rurl); // `withSeenOn`: true -> send the event even if it has already been seen in order to update seenOn // `withSeenOn`: false -> send the event only if it hasn't been seen yet if (options.withSeenOn || !hasSeen) { tx.send({ ...e, seenOn }); } statsMngr?.eventFetched(rurl); statsMngr?.setNumBufferedEvents(tx.numBufferedItems()); } } } catch (err) { if (isFetchTillEoseFailedSignal(err)) { // an error occurred while fetching events logger?.log("error", err); statsMngr?.setRelayStatus(rurl, "failed"); break; } if (isFetchTillEoseAbortedSignal(err)) { // fetch aborted logger?.log("info", err.message); isAboutToAbort = true; } else { logger?.log("error", "unexpected error:", err); statsMngr?.setRelayStatus(rurl, "failed"); break; } } finally { statsMngr?.subClosed(); } if (!gotNewEvent) { // termination contidion 1 logger?.log("info", `got ${localSeenEventIds.size} events`); statsMngr?.setRelayStatus(rurl, isAboutToAbort ? "aborted" : "completed"); break; } // set next `until` to `created_at` of the oldest event returned in this time. nextUntil = oldestCreatedAt; statsMngr?.setRelayFrontier(rurl, oldestCreatedAt); // update progress if (timeRangeDur !== undefined) { progTracker.setProgress(rurl, (initialUntil - oldestCreatedAt) / timeRangeDur); statsMngr?.setCurrentProgress(progTracker.calcTotalProgress()); } if (options.signal?.aborted) { // termination contidion 2 logger?.log("info", "aborted"); statsMngr?.setRelayStatus(rurl, "aborted"); break; } // receive backpressure: wait until the channel is drained enough await tx.waitUntilDrained(); } // subscription loop for the relay terminated progTracker.setProgress(rurl, 1); statsMngr?.setCurrentProgress(progTracker.calcTotalProgress()); }), ).then(() => { // all subscription loops have been terminated tx.close(); statsMngr?.stop(); }); yield* chIter; } /** * Fetches all events matching the filter from Nostr relays specified by the array of URLs, * and collect them into an array. * * Note: there are no guarantees about the order of returned events if `sort` options is not specified. * * Throws {@linkcode NostrFetchError} if `timeRangeFilter` is invalid (`since` > `until`). */ public async fetchAllEvents( relayUrls: string[], filter: FetchFilter, timeRangeFilter: FetchTimeRangeFilter, options: FetchAllOptions = {}, ): Promise[]> { assertReq( { relayUrls, filter, timeRangeFilter }, [ checkIfNonEmpty((r) => r.relayUrls, "warn", "Specify at least 1 relay URL"), checkIfTagQueriesAreValid((r) => r.filter, "error"), checkIfTimeRangeIsValid((r) => r.timeRangeFilter, "error"), ], this.#debugLogger, ); const finalOpts = { ...defaultFetchAllOptions, ...options, } as Required>; const allEvents = this.allEventsIterator(relayUrls, filter, timeRangeFilter, { ...finalOpts, enableBackpressure: false, }); // collect events const res = await (async () => { if (finalOpts.withSeenOn) { const evs: Map> = new Map(); for await (const ev of allEvents) { evs.set(ev.id, ev); } return [...evs.values()]; } const evs: NostrEventExt[] = []; for await (const ev of allEvents) { evs.push(ev); } return evs; })(); // sort events in "newest to oldest" order if `sort` options is specified if (finalOpts.sort) { res.sort(compareNostrEvents); } return res; } /** * Fetches latest events matching the filter from Nostr relays specified by the array of URLs. * * Events are sorted in "newest to oldest" order. * * Throws {@linkcode NostrFetchError} if `limit` is a non-positive number. */ public async fetchLatestEvents( relayUrls: string[], filter: FetchFilter, limit: number, options: FetchLatestOptions = {}, ): Promise[]> { assertReq( { relayUrls, filter, limit }, [ checkIfNonEmpty((r) => r.relayUrls, "warn", "Specify at least 1 relay URL"), checkIfTagQueriesAreValid((r) => r.filter, "error"), checkIfTrue((r) => r.limit > 0, "error", '"limit" should be positive number'), ], this.#debugLogger, ); const finalOpts = { ...defaultFetchLatestOptions, ...options, // if only `abortSignal` is specified, copy it to `signal` signal: options.signal ?? options.abortSignal, } as Required>; this.#debugLogger?.log("verbose", "finalOpts=%O", finalOpts); // options for subscription const subOpts: FetchTillEoseOptions = { ...finalOpts, // skip "full" verification if `reduceVerification` is enabled skipVerification: finalOpts.skipVerification || finalOpts.reduceVerification, }; const statsMngr = FetchStatsManager.init( finalOpts.statsListener, finalOpts.statsNotifIntervalMs, ); const reqNips = this.#calcRequiredNips(filter); const eligibleRelayUrls = await this.#ensureRelaysWithCapCheck(relayUrls, finalOpts, reqNips); const [tx, chIter] = Channel.make(); const globalSeenEvents = initSeenEvents(finalOpts.withSeenOn); const initialUntil = finalOpts.asOf ?? currUnixtimeSec(); const progTracker = new ProgressTracker(eligibleRelayUrls); statsMngr?.setProgressMax(eligibleRelayUrls.length * limit); statsMngr?.initRelayStats(relayUrls, eligibleRelayUrls, initialUntil); // fetch at most `limit` events from each relay Promise.all( eligibleRelayUrls.map(async (rurl) => { // repeat subscription until one of the following conditions is met: // 1. got enough amount of events // 2. the relay didn't return new event // 3. aborted by AbortController // E. an error occurred while fetching events const logger = this.#debugLogger?.subLogger(rurl); let nextUntil = initialUntil; let remainingLimit = limit; const localSeenEventIds = new Set(); while (true) { const refinedFilter = { ...filter, until: nextUntil, // relays are supposed to return *latest* events by specifying `limit` explicitly (cf. [NIP-01](https://github.com/nostr-protocol/nips/blob/master/01.md)). // nostream doesn't accept a filter which has `limit` grater than 5000, so limit `limit` to this threshold or less. limit: Math.min(remainingLimit, MAX_LIMIT_PER_REQ), }; logger?.log("verbose", "refinedFilter=%O", refinedFilter); let numNewEvents = 0; let oldestCreatedAt = Number.MAX_SAFE_INTEGER; let isAboutToAbort = false; try { statsMngr?.subOpened(); for await (const e of this.#backend.fetchTillEose(rurl, refinedFilter, subOpts)) { // eliminate duplicated events if (!localSeenEventIds.has(e.id)) { // hasn't seen the event on this relay numNewEvents++; localSeenEventIds.add(e.id); if (e.created_at < oldestCreatedAt) { oldestCreatedAt = e.created_at; } const { hasSeen } = globalSeenEvents.report(e, rurl); if (!hasSeen) { tx.send(e); } statsMngr?.eventFetched(rurl); statsMngr?.setNumBufferedEvents(tx.numBufferedItems()); } } } catch (err) { if (isFetchTillEoseFailedSignal(err)) { // an error occurred while fetching events logger?.log("error", err); statsMngr?.setRelayStatus(rurl, "failed"); break; } if (isFetchTillEoseAbortedSignal(err)) { // fetch aborted logger?.log("info", err.message); isAboutToAbort = true; } else { logger?.log("error", "unexpected error:", err); statsMngr?.setRelayStatus(rurl, "failed"); break; } } finally { statsMngr?.subClosed(); } progTracker.addProgress(rurl, Math.min(numNewEvents, remainingLimit)); statsMngr?.setCurrentProgress(progTracker.calcTotalProgress()); remainingLimit -= numNewEvents; if (numNewEvents === 0 || remainingLimit <= 0) { // termination condition 1, 2 logger?.log("info", `got ${localSeenEventIds.size} events`); statsMngr?.setRelayStatus(rurl, isAboutToAbort ? "aborted" : "completed"); break; } if (finalOpts.signal?.aborted) { // termination condition 3 logger?.log("info", "aborted"); statsMngr?.setRelayStatus(rurl, "aborted"); break; } // set next `until` to `created_at` of the oldest event returned in this time. nextUntil = oldestCreatedAt; statsMngr?.setRelayFrontier(rurl, oldestCreatedAt); } // subscription loop for the relay terminated progTracker.setProgress(rurl, limit); statsMngr?.setCurrentProgress(progTracker.calcTotalProgress()); }), ).then(() => { // all subnscription loops have been terminated tx.close(); statsMngr?.stop(); }); // collect events from relays. events are already deduped const evs: NostrEvent[] = []; for await (const ev of chIter) { evs.push(ev); } evs.sort(compareNostrEvents); // take latest events const res = (async () => { // return latest `limit` events if not "reduced verification mode" if (finalOpts.skipVerification || !finalOpts.reduceVerification) { return evs.slice(0, limit); } // reduced verification: return latest `limit` events whose signature is valid const verified: NostrEvent[] = []; for (const ev of evs) { if (await finalOpts.eventVerifier(ev)) { verified.push(ev); if (verified.length >= limit) { break; } } } return verified; })(); if (!finalOpts.withSeenOn) { return (await res) as NostrEventExt[]; } // append "seen on" data to events if `withSeenOn` is true. return (await res).map((e) => { return { ...e, seenOn: globalSeenEvents.getSeenOn(e.id) }; }) as NostrEventExt[]; } /** * Fetches the last event matching the filter from Nostr relays specified by the array of URLs. * * Returns `undefined` if no event matching the filter exists in any relay. */ public async fetchLastEvent( relayUrls: string[], filter: FetchFilter, options: FetchLatestOptions = {}, ): Promise | undefined> { const finalOpts = { ...defaultFetchLatestOptions, ...{ // override default value of `abortSubBeforeEoseTimeoutMs` (10000 -> 1000) abortSubBeforeEoseTimeoutMs: 1000, ...options, }, } as Required>; const latest1 = await this.fetchLatestEvents(relayUrls, filter, 1, finalOpts); return latest1[0]; } // creates mapping of available relays to keys. // returns triple: the mapping, array of all keys, and array of all relay URLs. // these arrays are guaranteed to be deduped. async #mapAvailableRelayToKeys( kr: KeysAndRelays, ensureOpts: EnsureRelaysOptions, reqNips: number[], ): Promise< [ relayToKeys: Map[]>, allKeys: FetchFilterKeyElem[], allRelays: string[], ] > { if (isRelaySetForAllKeys(kr)) { assertReq( kr, [ checkIfNonEmpty((r) => r.relayUrls, "warn", "Specify at least 1 relay URL"), checkIfNonEmpty((r) => r.keys as unknown[], "warn", "Specify at least 1 key"), ], this.#debugLogger, ); const dedupedKeys = [...new Set(kr.keys)]; const eligibleRelays = await this.#ensureRelaysWithCapCheck( kr.relayUrls, ensureOpts, reqNips, ); return [ new Map(eligibleRelays.map((rurl) => [rurl, dedupedKeys])), dedupedKeys, eligibleRelays, ]; } if (isRelaySetsPerKey(kr)) { const krArr = [...kr]; assertReq( krArr, [ checkIfNonEmpty((kr) => kr, "warn", "Specify at least 1 key"), checkIfTrue( (kr) => kr.every(([, relays]) => relays.length > 0), "warn", "Specify at least 1 relay URL for all keys", ), ], this.#debugLogger, ); const dedupedKeys = [...new Set(krArr.map(([key]) => key))]; // transpose: key to rurls -> rurl to keys const rurl2keys = new Map>>(); for (const [key, rurls] of krArr) { const normalized = normalizeRelayUrlSet(rurls); for (const rurl of normalized) { const keys = rurl2keys.get(rurl); rurl2keys.set(rurl, keys ? keys.add(key) : new Set([key])); } } const allRelays = [...rurl2keys.keys()]; const eligibleRelays = await this.#ensureRelaysWithCapCheck(allRelays, ensureOpts, reqNips); // retain eligible relays only return [ // biome-ignore lint/style/noNonNullAssertion: rurl2keys should have all relay URLs in eligibleRelays new Map(eligibleRelays.map((rurl) => [rurl, [...rurl2keys.get(rurl)!]])), dedupedKeys, eligibleRelays, ]; } throw new NostrFetchError( "malformed first argument for fetchLatestEventsPerKey/fetchLastEventPerKey", ); } /** * Fetches latest up to `limit` events **for each key specified by `keyName` and `keysAndRelays`**. * * `keysAndRelays` can be either of two types: * * - `{ keys: K[], relayUrls: string[] }`: The fetcher will use the same relay set (`relayUrls`) for all `keys` to fetch events. * - `Map`: Key must be the key of event and value must be relay set for that key. The fetcher will use separate relay set for each key to fetch events. * * Result is an async iterable of `{ key: , events: }` pairs. * * Each array of events in the result are sorted in "newest to oldest" order. * * Throws {@linkcode NostrFetchError} if `limit` is a non-positive number. */ public fetchLatestEventsPerKey( keyName: K, keysAndRelays: KeysAndRelays, otherFilter: FetchFilter, limit: number, options: FetchLatestOptions = {}, ): AsyncIterable> { assertReq( { limit, keyName, otherFilter }, [ checkIfTrue((r) => r.limit > 0, "error", '"limit" should be positive number'), checkIfTrue( (r) => !r.keyName.startsWith("#") || isValidTagQueryKey(r.keyName), "error", `Specified key '${keyName}' is invalid tag query key`, ), checkIfTagQueriesAreValid((r) => r.otherFilter, "error"), checkIfTrue( ({ keyName, otherFilter }) => !(keyName in otherFilter), "warn", `'${keyName}' field in "otherFilter" will be ignored because it is specified as main key`, ), ], this.#debugLogger, ); const filledOpts = { ...defaultFetchLatestOptions, ...options, } as Required>; this.#debugLogger?.log("verbose", "finalOpts=%O", filledOpts); // options for subscription const finalOpts = { ...filledOpts, // skip "full" verification if `reduceVerification` is enabled skipVerification: filledOpts.skipVerification || filledOpts.reduceVerification, // if only `abortSignal` is specified, copy it to `signal` signal: filledOpts.signal ?? filledOpts.abortSignal, }; return this.#fetchLatestEventPerKeyWithCleanupOnBreak( keyName, keysAndRelays, otherFilter, limit, finalOpts, ); } async *#fetchLatestEventPerKeyWithCleanupOnBreak< K extends FetchFilterKeyName, SeenOn extends boolean = false, >( keyName: K, keysAndRelays: KeysAndRelays, otherFilter: FetchFilter, limit: number, options: Required>, ): AsyncIterable> { const [breakableSig, breakSignal] = makeBreakableSignal(options.signal); try { yield* this.#fetchLatestEventPerKeyBody(keyName, keysAndRelays, otherFilter, limit, { ...options, signal: breakableSig, }); } finally { // this block will be executed when: // - iteration is finished normally (whether throws or not) // - iteration is terminated early by break/return in a for-await-of loop // // in second case, breakSignal() causes the "cleanup" (abortion of event fetching by backend). breakSignal(); } } async *#fetchLatestEventPerKeyBody( keyName: K, keysAndRelays: KeysAndRelays, otherFilter: FetchFilter, limit: number, options: Required>, ): AsyncIterable> { const statsMngr = FetchStatsManager.init(options.statsListener, options.statsNotifIntervalMs); // get mapping of available relay to keys and list of all keys const reqNips = this.#calcRequiredNips(otherFilter); const [relayToKeys, allKeys, allRelays] = await this.#mapAvailableRelayToKeys( keysAndRelays, options, reqNips, ); this.#debugLogger?.log("verbose", "relayToKeys=%O", relayToKeys); const [tx, chIter] = Channel.make>(); const globalSeenEvents = initSeenEvents(options.withSeenOn); const initialUntil = options.asOf ?? currUnixtimeSec(); statsMngr?.setProgressMax(allKeys.length); statsMngr?.initRelayStats(allRelays, [...relayToKeys.keys()], initialUntil); // for each pair of key and relay URL, create a promise that act as "latch", so that the "merger" can wait for a subscription to complete const latches = new KeyRelayMatrix(relayToKeys, () => new Deferred()); // the "fetcher" fetches events from each relay Promise.all( [...relayToKeys].map(async ([rurl, keys]) => { // repeat subscription until one of the following conditions is met: // 1. have fetched required number of events for all keys // 2. the relay didn't return new event // 3. aborted by AbortController // E. an error occurred while fetching events const logger = this.#debugLogger?.subLogger(rurl); let nextUntil = initialUntil; const evBucketsPerKey = new EventBuckets(keys, limit); const localSeenEventIds = new Set(); // procedure to complete the subscription in the middle, resolving all remaining promises. // resolve() is called even if a promise is already resolved, but it's not a problem. const resolveAllOnEarlyBreak = () => { logger?.log("verbose", "resolving bucket on early return"); for (const pk of keys) { latches.get(pk, rurl)?.resolve(evBucketsPerKey.getBucket(pk) ?? []); } }; while (true) { const { keys: nextKeys, limit: nextLimit } = evBucketsPerKey.calcKeysAndLimitForNextReq(); if (nextKeys.length === 0) { // termination condition 1 logger?.log("verbose", "fulfilled buckets for all keys"); statsMngr?.setRelayStatus(rurl, "completed"); break; } const refinedFilter = { ...otherFilter, [keyName]: nextKeys, until: nextUntil, limit: Math.min(nextLimit, MAX_LIMIT_PER_REQ), }; logger?.log("verbose", "refinedFilter=%O", refinedFilter); let gotNewEvent = false; let oldestCreatedAt = Number.MAX_SAFE_INTEGER; let isAboutToAbort = false; try { statsMngr?.subOpened(); for await (const e of this.#backend.fetchTillEose(rurl, refinedFilter, options)) { if (!localSeenEventIds.has(e.id)) { // hasn't seen the event on this relay gotNewEvent = true; localSeenEventIds.add(e.id); if (e.created_at < oldestCreatedAt) { oldestCreatedAt = e.created_at; } globalSeenEvents.report(e, rurl); // add the event to the bucket for the keys for (const evKey of getKeysOfEvent(keyName, e)) { const addRes = evBucketsPerKey.add(evKey, e); if (addRes.state === "fulfilled") { // notify that event fetching is completed for the key at this relay // by resolveing the Promise corresponds to the key and the relay latches.get(evKey, rurl)?.resolve(addRes.events); logger?.log("verbose", `fulfilled a bucket for key=${evKey}`); } } statsMngr?.eventFetched(rurl); statsMngr?.setNumBufferedEvents(tx.numBufferedItems()); } } } catch (err) { if (isFetchTillEoseFailedSignal(err)) { // an error occurred while fetching events logger?.log("error", err); statsMngr?.setRelayStatus(rurl, "failed"); resolveAllOnEarlyBreak(); break; } if (isFetchTillEoseAbortedSignal(err)) { // fetch aborted logger?.log("info", err.message); isAboutToAbort = true; } else { logger?.log("error", "unexpected error:", err); statsMngr?.setRelayStatus(rurl, "failed"); resolveAllOnEarlyBreak(); break; } } finally { statsMngr?.subClosed(); } if (!gotNewEvent) { // termination condition 2 logger?.log("info", `got ${localSeenEventIds.size} events`); statsMngr?.setRelayStatus(rurl, isAboutToAbort ? "aborted" : "completed"); resolveAllOnEarlyBreak(); break; } if (options.signal?.aborted) { // termination condition 3 logger?.log("info", "aborted"); statsMngr?.setRelayStatus(rurl, isAboutToAbort ? "aborted" : "completed"); resolveAllOnEarlyBreak(); break; } // set next `until` to `created_at` of the oldest event returned in this time. nextUntil = oldestCreatedAt; statsMngr?.setRelayFrontier(rurl, oldestCreatedAt); } }), ); // the "merger". // for each key: merges result from relays, sorts events, takes latest events and sends it to the result channel. Promise.all( allKeys.map(async (key) => { const logger = this.#debugLogger?.subLogger(abbreviate(String(key), 6)); // wait for all the buckets for the key to fulfilled const evsPerRelay = await Promise.all(latches.itemsByKey(key)?.map((d) => d.promise) ?? []); logger?.log("verbose", "fulfilled all buckets for this key"); // merge and sort const evsDeduped = (() => { const res = []; const seenIds = new Set(); for (const evs of evsPerRelay) { for (const ev of evs) { if (!seenIds.has(ev.id)) { res.push(ev); seenIds.add(ev.id); } } } return res; })(); evsDeduped.sort(compareNostrEvents); // take latest events const res = (async () => { // return latest `limit` events if not "reduced verification mode" if (options.skipVerification || !options.reduceVerification) { return evsDeduped.slice(0, limit); } // reduced verification: return latest `limit` events whose signature is valid const verified = []; for (const ev of evsDeduped) { if (await options.eventVerifier(ev)) { verified.push(ev); if (verified.length >= limit) { break; } } } return verified; })(); // send result if (options.withSeenOn) { // append "seen on" data to events if `withSeenOn` is true. tx.send({ key, events: (await res).map((e) => { return { ...e, seenOn: globalSeenEvents.getSeenOn(e.id) }; }) as NostrEventExt[], }); } else { tx.send({ key, events: (await res) as NostrEventExt[] }); } statsMngr?.addProgress(1); }), ).then(() => { // finished to fetch events for all keys tx.close(); statsMngr?.stop(); }); yield* chIter; } /** * Fetches the last event **for each key specified by `keysAndRelays`**. * * `keysAndRelays` can be either of two types: * * - `{ keys: K[], relayUrls: string[] }`: The fetcher will use the same relay set (`relayUrls`) for all `keys` to fetch events. * - `Map`: Key must be key of the event and value must be relay set for that key. The fetcher will use separate relay set for each key to fetch events. * * Result is an async iterable of `{ key: , event: }` pairs. * * `event` in result will be `undefined` if no event matching the filter exists in any relay. */ public async *fetchLastEventPerKey( keyName: K, keysAndRelays: KeysAndRelays, otherFilter: FetchFilter, options: FetchLatestOptions = {}, ): AsyncIterable> { const finalOpts = { ...defaultFetchLatestOptions, ...{ // override default value of `abortSubBeforeEoseTimeoutMs` (10000 -> 1000) abortSubBeforeEoseTimeoutMs: 1000, ...options, }, } as Required>; const latest1Iter = this.fetchLatestEventsPerKey( keyName, keysAndRelays, otherFilter, 1, finalOpts, ); for await (const { key, events } of latest1Iter) { yield { key, event: events[0] }; } } /** * Fetches latest up to `limit` events **for each author specified by `authorsAndRelays`**. * * `authorsAndRelays` can be either of two types: * * - `{ authors: string[], relayUrls: string[] }`: The fetcher will use the same relay set (`relayUrls`) for all `authors` to fetch events. * - `Map`: Key must be author's pubkey and value must be relay set for that author. The fetcher will use separate relay set for each author to fetch events. * * Result is an async iterable of `{ author: , events: }` pairs. * * Each array of events in the result are sorted in "newest to oldest" order. * * Throws {@linkcode NostrFetchError} if `limit` is a non-positive number. * * Note: it's just an wrapper of `fetchLatestEventsPerKey`. */ public async *fetchLatestEventsPerAuthor( authorsAndRelays: AuthorsAndRelays, otherFilter: Omit, limit: number, options: FetchLatestOptions = {}, ): AsyncIterable> { for await (const { key, events } of this.fetchLatestEventsPerKey( "authors", adaptAuthorsAndRelays(authorsAndRelays), otherFilter, limit, options, )) { yield { author: key, events }; } } /** * Fetches the last event **for each author specified by `authorsAndRelays`**. * * `authorsAndRelays` can be either of two types: * * - `{ authors: string[], relayUrls: string[] }`: The fetcher will use the same relay set (`relayUrls`) for all `authors` to fetch events. * - `Map`: Key must be author's pubkey and value must be relay set for that author. The fetcher will use separate relay set for each author to fetch events. * * Result is an async iterable of `{ author: , event: }` pairs. * * `event` in result will be `undefined` if no event matching the filter for the author exists in any relay. * * Note: it's just a wrapper of `fetchLastEventPerKey`. */ public async *fetchLastEventPerAuthor( authorsAndRelays: AuthorsAndRelays, otherFilter: Omit, options: FetchLatestOptions = {}, ): AsyncIterable> { for await (const { key, event } of this.fetchLastEventPerKey( "authors", adaptAuthorsAndRelays(authorsAndRelays), otherFilter, options, )) { yield { author: key, event }; } } /** * Cleans up all the internal states of the fetcher. */ public shutdown() { this.#backend.shutdown(); } /** * Enables [explicit resource management](https://www.typescriptlang.org/docs/handbook/release-notes/typescript-5-2.html#using-declarations-and-explicit-resource-management) for `NostrFetcher` instances. * * If you bind a `NostrFetcher` instance to a variable with `using` keyword, it will be automatically shut down when the scope of the variable ends. */ public [Symbol.dispose]() { this.shutdown(); } }