import * as Chunk from "effect/Chunk"; import * as Data from "effect/Data"; import * as Match from "effect/Match"; import * as Option from "effect/Option"; import * as Result from "effect/Result"; import * as Schema from "effect/Schema"; import type * as QueryStreamKey from "./QueryStreamKey"; import * as QueryStreamKeyValues from "./QueryStreamKeyValues"; const PageSize = Schema.Natural.pipe( Schema.brand("~@confect/server/QueryStreamPagination/PageSize"), ); type PageSize = typeof PageSize.Type; /** * @experimental */ export class InvalidPageSizeError extends Data.TaggedError( "InvalidPageSizeError", )<{ readonly numItems: number; }> { override get message(): string { return `QueryStream.paginate: numItems must be a nonnegative integer (received ${this.numItems})`; } } /** * @experimental */ export class EmptyInitialPageError extends Data.TaggedError( "EmptyInitialPageError", ) { override get message(): string { return "QueryStream.paginate: numItems of 0 with a null cursor is not supported"; } } /** * @experimental */ export type Range = Data.TaggedEnum<{ Unpinned: {}; ThroughKey: { readonly key: QueryStreamKey.Complete }; ThroughEnd: {}; }>; /** * @experimental */ export const Range = Data.taggedEnum(); /** * @experimental */ export type Start = Data.TaggedEnum<{ Beginning: {}; After: { readonly key: QueryStreamKey.Complete; readonly cursor: string; }; }>; /** * @experimental */ export const Start = Data.taggedEnum(); /** * @experimental */ export interface ScanRequest { readonly numItems: PageSize; readonly after: Option.Option; readonly range: Range; } /** * @experimental */ export type Request = Data.TaggedEnum<{ Unchanged: { readonly cursor: string }; Scan: ScanRequest; }>; const Request = Data.taggedEnum(); /** * @experimental */ export const parseRequest = ( numItems: number, start: Start, range: Range, ): Result.Result => Result.flatMap( Schema.decodeResult(PageSize)(numItems).pipe( Result.mapError(() => new InvalidPageSizeError({ numItems })), ), (size): Result.Result => size === 0 ? Start.$match(start, { Beginning: () => Result.fail(new EmptyInitialPageError()), After: ({ cursor }) => Result.succeed(Request.Unchanged({ cursor })), }) : Result.succeed( Request.Scan({ numItems: size, after: Start.$match(start, { Beginning: () => Option.none(), After: ({ key }) => Option.some(key), }), range, }), ), ); /** * A nonempty scan, including filtered markers. */ type Progress = Chunk.NonEmptyChunk; const append = ( progress: Option.Option, key: QueryStreamKey.Complete, ): Progress => Option.match(progress, { onNone: () => Chunk.of(key), onSome: (keys) => Chunk.append(keys, key), }); const midpoint = (progress: Progress): QueryStreamKey.Complete => Option.getOrElse( Chunk.get(progress, Math.floor((Chunk.size(progress) - 1) / 2)), () => Chunk.headNonEmpty(progress), ); interface Stopped { readonly page: Chunk.Chunk; readonly progress: Progress; } /** * @experimental */ export type QueryStreamPagination = Data.TaggedEnum<{ Reading: { readonly page: Chunk.Chunk; readonly progress: Option.Option; }; ItemLimit: Stopped; ReadLimit: Stopped; }>; interface PaginationDefinition extends Data.TaggedEnum.WithGenerics<1> { readonly taggedEnum: QueryStreamPagination; } const QueryStreamPagination = Data.taggedEnum(); /** * @experimental */ export const initial = (): QueryStreamPagination => QueryStreamPagination.Reading({ page: Chunk.empty(), progress: Option.none(), }); /** * @experimental */ export const record = ( request: ScanRequest, self: QueryStreamPagination, doc: Option.Option, key: QueryStreamKey.Complete, readLimit: boolean, ): QueryStreamPagination => Match.value(self).pipe( Match.tagsExhaustive({ Reading: (reading) => { const progress = append(reading.progress, key); const page = Option.match(doc, { onNone: () => reading.page, onSome: (value) => Chunk.append(reading.page, value), }); const continueReading = () => QueryStreamPagination.Reading({ page, progress: Option.some(progress), }); return readLimit ? QueryStreamPagination.ReadLimit({ page, progress }) : Range.$match(request.range, { Unpinned: () => Chunk.size(page) >= request.numItems ? QueryStreamPagination.ItemLimit({ page, progress }) : continueReading(), ThroughKey: continueReading, ThroughEnd: continueReading, }); }, ItemLimit: () => self, ReadLimit: () => self, }), ); /** * @experimental */ export type Continuation = Data.TaggedEnum<{ End: {}; Key: { readonly key: QueryStreamKey.Complete }; }>; const Continuation = Data.taggedEnum(); interface Split { readonly page: ReadonlyArray; readonly continuation: Continuation; readonly splitKey: QueryStreamKey.Complete; } /** * @experimental */ export type Outcome = Data.TaggedEnum<{ Done: { readonly page: ReadonlyArray }; Continue: { readonly page: ReadonlyArray; readonly key: QueryStreamKey.Complete; }; SplitRequired: Split; SplitRecommended: Split; }>; /** * @experimental */ export class UnsafePageBoundaryError extends Data.TaggedError( "UnsafePageBoundaryError", )<{ readonly reason: "NoProgress" | "NoInteriorSplit"; }> { override get message(): string { return Match.value(this.reason).pipe( Match.when( "NoProgress", () => "The read budget stopped the page before any key was scanned", ), Match.when( "NoInteriorSplit", () => "The read budget stopped the page without an interior split key", ), Match.exhaustive, ); } } const SOFT_MAX_SCAN_LENGTH = 16000; /** * Decide the page result without reading a stream or serializing a cursor. * * @experimental */ export const finish = ( request: ScanRequest, self: QueryStreamPagination, upstreamStopped: boolean, ): Result.Result, UnsafePageBoundaryError> => { const page = Chunk.toArray(self.page); const stopped = ( progress: Option.Option, readLimit: boolean, ): Result.Result, UnsafePageBoundaryError> => Result.flatMap( Result.fromOption( progress, () => new UnsafePageBoundaryError({ reason: "NoProgress" }), ), (scanned): Result.Result, UnsafePageBoundaryError> => { const splitKey = midpoint(scanned); const atEnd = Range.$match(request.range, { Unpinned: () => false, ThroughEnd: () => false, ThroughKey: ({ key }) => QueryStreamKeyValues.Order("asc")(splitKey.values, key.values) === 0, }); if (readLimit && atEnd) { return Result.fail( new UnsafePageBoundaryError({ reason: "NoInteriorSplit" }), ); } const key = Chunk.lastNonEmpty(scanned); if (readLimit) return Result.succeed({ _tag: "SplitRequired", page, continuation: Continuation.Key({ key }), splitKey, }); return Result.succeed( Chunk.size(scanned) >= SOFT_MAX_SCAN_LENGTH ? { _tag: "SplitRecommended", page, continuation: Continuation.Key({ key }), splitKey, } : { _tag: "Continue", page, key }, ); }, ); const exhausted = (progress: Option.Option): Outcome => { const finishRange = ( continuation: Continuation, pinned: boolean, ): Outcome => { const split = Option.filter( progress, (scanned) => pinned && (Chunk.size(scanned) >= SOFT_MAX_SCAN_LENGTH || page.length > request.numItems + 1), ); return Option.match(split, { onSome: (scanned): Outcome => ({ _tag: "SplitRecommended", page, continuation, splitKey: midpoint(scanned), }), onNone: () => Continuation.$match(continuation, { End: (): Outcome => ({ _tag: "Done", page }), Key: ({ key }): Outcome => ({ _tag: "Continue", page, key, }), }), }); }; return Range.$match(request.range, { Unpinned: () => finishRange(Continuation.End(), false), ThroughEnd: () => finishRange(Continuation.End(), true), ThroughKey: ({ key }) => finishRange(Continuation.Key({ key }), true), }); }; return Match.value(self).pipe( Match.tagsExhaustive({ Reading: ({ progress }) => upstreamStopped ? stopped(progress, true) : Result.succeed(exhausted(progress)), ItemLimit: ({ progress }) => stopped(Option.some(progress), upstreamStopped), ReadLimit: ({ progress }) => stopped(Option.some(progress), true), }), ); };