import { SearchQuery, IndexableResultSet, DocumentCollatorFactory, DocumentDecoratorFactory, DocumentTypeInfo, IndexableDocument } from '@backstage/plugin-search-common'; import { LoggerService, SchedulerServiceTaskFunction, SchedulerServiceTaskRunner, BackstageCredentials, UrlReaderService } from '@backstage/backend-plugin-api'; import { Writable, Readable, Transform } from 'node:stream'; import { Config } from '@backstage/config'; import { Permission } from '@backstage/plugin-permission-common'; import lunr from 'lunr'; /** * ScheduleTaskParameters * @public */ type ScheduleTaskParameters = { id: string; task: SchedulerServiceTaskFunction; scheduledRunner: SchedulerServiceTaskRunner; }; /** * Scheduler responsible for all search tasks. * @public */ declare class Scheduler { private logger; private schedule; private abortControllers; private isRunning; constructor(options: { logger: LoggerService; }); /** * Adds each task and interval to the schedule. * When running the tasks, the scheduler waits at least for the time specified * in the interval once the task was completed, before running it again. */ addToSchedule(options: ScheduleTaskParameters): void; /** * Starts the scheduling process for each task */ start(): void; /** * Stop all scheduled tasks. */ stop(): void; } /** * Options required to instantiate the index builder. * @public */ type IndexBuilderOptions = { searchEngine: SearchEngine; logger: LoggerService; }; /** * Parameters required to register a collator. * @public */ interface RegisterCollatorParameters { /** * The schedule for which the provided collator will be called, commonly the result of * {@link @backstage/backend-plugin-api#SchedulerService.createScheduledTaskRunner} */ schedule: SchedulerServiceTaskRunner; /** * The class responsible for returning the document collator of the given type. */ factory: DocumentCollatorFactory; } /** * Parameters required to register a decorator * @public */ interface RegisterDecoratorParameters { /** * The class responsible for returning the decorator which appends, modifies, or filters documents. */ factory: DocumentDecoratorFactory; } /** * A type of function responsible for translating an abstract search query into * a concrete query relevant to a particular search engine. * @public */ type QueryTranslator = (query: SearchQuery) => unknown; /** * Options when querying a search engine. * @public */ type QueryRequestOptions = { /** @deprecated use the `credentials` option instead. */ token?: string; } | { credentials: BackstageCredentials; }; /** * Interface that must be implemented by specific search engines, responsible * for performing indexing and querying and translating abstract queries into * concrete, search engine-specific queries. * @public */ interface SearchEngine { /** * Override the default translator provided by the SearchEngine. */ setTranslator(translator: QueryTranslator): void; /** * Factory method for getting a search engine indexer for a given document * type. * * @param type - The type or name of the document set for which an indexer * should be retrieved. This corresponds to the `type` property on the * document collator/decorator factories and will most often be used to * identify an index or group to which documents should be written. */ getIndexer(type: string): Promise; /** * Perform a search query against the SearchEngine. */ query(query: SearchQuery, options?: QueryRequestOptions): Promise; } /** * Used for adding collators, decorators and compile them into tasks which are added to a scheduler returned to the caller. * @public */ declare class IndexBuilder { private collators; private decorators; private documentTypes; private searchEngine; private logger; constructor(options: IndexBuilderOptions); /** * Responsible for returning the registered search engine. */ getSearchEngine(): SearchEngine; /** * Responsible for returning the registered document types. */ getDocumentTypes(): Record; /** * Makes the index builder aware of a collator that should be executed at the * given refresh interval. */ addCollator(options: RegisterCollatorParameters): void; /** * Makes the index builder aware of a decorator. If no types are provided on * the decorator, it will be applied to documents from all known collators, * otherwise it will only be applied to documents of the given types. */ addDecorator(options: RegisterDecoratorParameters): void; /** * Compiles collators and decorators into tasks, which are added to a * scheduler returned to the caller. */ build(): Promise<{ scheduler: Scheduler; }>; } /** * Options for instantiate NewlineDelimitedJsonCollatorFactory * @public */ type NewlineDelimitedJsonCollatorFactoryOptions = { type: string; searchPattern: string; reader: UrlReaderService; logger: LoggerService; visibilityPermission?: Permission; }; /** * Factory class producing a collator that can be used to index documents * sourced from the latest newline delimited JSON file matching a given search * pattern. "Latest" is determined by the name of the file (last alphabetically * is considered latest). * * @remarks * The reader provided must implement the `search()` method as well as the * `readUrl` method whose response includes the `stream()` method. Naturally, * the reader must also be configured to understand the given search pattern. * * @example * Here's an example configuration using Google Cloud Storage, which would * return the latest file under the `bucket` GCS bucket with files like * `xyz-2021.ndjson` or `xyz-2022.ndjson`. * ```ts * indexBuilder.addCollator({ * schedule, * factory: NewlineDelimitedJsonCollatorFactory.fromConfig(env.config, { * type: 'techdocs', * searchPattern: 'https://storage.cloud.google.com/bucket/xyz-*', * reader: env.reader, * logger: env.logger, * }) * }); * ``` * * @public */ declare class NewlineDelimitedJsonCollatorFactory implements DocumentCollatorFactory { readonly type: string; readonly visibilityPermission: Permission | undefined; private readonly searchPattern; private readonly reader; private readonly logger; private constructor(); /** * Returns a NewlineDelimitedJsonCollatorFactory instance from configuration * and a set of options. */ static fromConfig(_config: Config, options: NewlineDelimitedJsonCollatorFactoryOptions): NewlineDelimitedJsonCollatorFactory; /** * Returns the "latest" URL for the given search pattern (e.g. the one at the * end of the list, sorted alphabetically). */ private lastUrl; getCollator(): Promise; } /** * Options for {@link BatchSearchEngineIndexer} * @public */ type BatchSearchEngineOptions = { batchSize: number; }; /** * Base class encapsulating batch-based stream processing. Useful as a base * class for search engine indexers. * @public */ declare abstract class BatchSearchEngineIndexer extends Writable { private batchSize; private currentBatch; constructor(options: BatchSearchEngineOptions); /** * Receives an array of indexable documents (of size this.batchSize) which * should be written to the search engine. This method won't be called again * at least until it resolves. */ abstract index(documents: IndexableDocument[]): Promise; /** * Any asynchronous setup tasks can be performed here. */ abstract initialize(): Promise; /** * Any asynchronous teardown tasks can be performed here. */ abstract finalize(): Promise; } /** * Base class encapsulating simple async transformations. Useful as a base * class for Backstage search decorators. * @public */ declare abstract class DecoratorBase extends Transform { constructor(); /** * Any asynchronous setup tasks can be performed here. */ abstract initialize(): Promise; /** * Receives a single indexable document. In your decorate method, you can: * * - Resolve `undefined` to indicate the record should be omitted. * - Resolve a single modified document, which could contain new fields, * edited fields, or removed fields. * - Resolve an array of indexable documents, if the purpose if the decorator * is to convert one document into multiple derivative documents. */ abstract decorate(document: IndexableDocument): Promise; /** * Any asynchronous teardown tasks can be performed here. */ abstract finalize(): Promise; } /** * Lunr specific search engine indexer * @public */ declare class LunrSearchEngineIndexer extends BatchSearchEngineIndexer { private schemaInitialized; private builder; private docStore; constructor(); initialize(): Promise; finalize(): Promise; index(documents: IndexableDocument[]): Promise; buildIndex(): lunr.Index; getDocumentStore(): Record; } /** * Type of translated query for the Lunr Search Engine. * @public */ type ConcreteLunrQuery = { lunrQueryBuilder: lunr.Index.QueryBuilder; documentTypes?: string[]; pageSize: number; }; /** * Translator responsible for translating search term and filters to a query that the Lunr Search Engine understands. * @public */ type LunrQueryTranslator = (query: SearchQuery) => ConcreteLunrQuery; /** * Lunr specific search engine implementation. * @public */ declare class LunrSearchEngine implements SearchEngine { protected lunrIndices: Record; protected docStore: Record; protected logger: LoggerService; protected highlightPreTag: string; protected highlightPostTag: string; constructor(options: { logger: LoggerService; }); protected translator: QueryTranslator; setTranslator(translator: LunrQueryTranslator): void; getIndexer(type: string): Promise; query(query: SearchQuery): Promise; } /** * Failed to query documents for index that does not exist. * @public */ declare class MissingIndexError extends Error { /** * An inner error that caused this error to be thrown, if any. */ readonly cause?: Error | undefined; constructor(message?: string, cause?: Error | unknown); } /** * Object resolved after a test pipeline is executed. * @public */ type TestPipelineResult = { /** * If an error was emitted by the pipeline, it will be set here. */ error: unknown; /** * A list of documents collected at the end of the pipeline. If the subject * under test is an indexer, this will be an empty array (because your * indexer should have received the documents instead). */ documents: IndexableDocument[]; }; /** * Test utility for Backstage Search collators, decorators, and indexers. * * @example * An example test checking that a collator provides expected documents. * ``` * it('provides expected documents', async () => { * const testSubject = await yourCollatorFactory.getCollator(); * const pipeline = TestPipeline.fromCollator(testSubject); * * const { documents } = await pipeline.execute(); * * expect(documents).toHaveLength(2); * }) * ``` * * @example * An example test checking that a decorator behaves as expected. * ``` * it('filters private documents', async () => { * const testSubject = await yourDecoratorFactory.getDecorator(); * const pipeline = TestPipeline * .fromDecorator(testSubject) * .withDocuments([{ title: 'Private', location: '/private', text: '' }]); * * const { documents } = await pipeline.execute(); * * expect(documents).toHaveLength(0); * }) * ``` * * @public */ declare class TestPipeline { private collator?; private decorator?; private indexer?; private constructor(); /** * Provide the collator, decorator, or indexer to be tested. * * @deprecated Use `fromCollator`, `fromDecorator` or `fromIndexer` static * methods to create a test pipeline instead. */ static withSubject(subject: Readable | Transform | Writable): TestPipeline; /** * Create a test pipeline given a collator you want to test. */ static fromCollator(collator: Readable): TestPipeline; /** * Add a collator to the test pipeline. */ withCollator(collator: Readable): this; /** * Create a test pipeline given a decorator you want to test. */ static fromDecorator(decorator: Transform): TestPipeline; /** * Add a decorator to the test pipeline. */ withDecorator(decorator: Transform): this; /** * Create a test pipeline given an indexer you want to test. */ static fromIndexer(indexer: Writable): TestPipeline; /** * Add an indexer to the test pipeline. */ withIndexer(indexer: Writable): this; /** * Provide documents for testing decorators and indexers. */ withDocuments(documents: IndexableDocument[]): TestPipeline; /** * Execute the test pipeline so that you can make assertions about the result * or behavior of the given test subject. */ execute(): Promise; } export { BatchSearchEngineIndexer, DecoratorBase, IndexBuilder, LunrSearchEngine, LunrSearchEngineIndexer, MissingIndexError, NewlineDelimitedJsonCollatorFactory, Scheduler, TestPipeline }; export type { BatchSearchEngineOptions, ConcreteLunrQuery, IndexBuilderOptions, LunrQueryTranslator, NewlineDelimitedJsonCollatorFactoryOptions, QueryRequestOptions, QueryTranslator, RegisterCollatorParameters, RegisterDecoratorParameters, ScheduleTaskParameters, SearchEngine, TestPipelineResult };