/** * @since 1.0.0 */ import type * as Cause from "effect/Cause" import * as Channel from "effect/Channel" import * as Chunk from "effect/Chunk" import * as Effect from "effect/Effect" import { dual, identity } from "effect/Function" import type { ParseError } from "effect/ParseResult" import type * as Schema from "effect/Schema" import * as ChannelSchema from "./ChannelSchema.js" import { TypeIdError } from "./Error.js" /** * @since 1.0.0 * @category type ids */ export const ErrorTypeId: unique symbol = Symbol.for("@effect/platform/Ndjson/NdjsonError") /** * @since 1.0.0 * @category type ids */ export type NdjsonErrorTypeId = typeof ErrorTypeId const encoder = new TextEncoder() /** * @since 1.0.0 * @category errors */ export class NdjsonError extends TypeIdError(ErrorTypeId, "NdjsonError")<{ readonly reason: "Pack" | "Unpack" readonly cause: unknown }> { get message() { return this.reason } } /** * Represents a set of options which can be used to control how the newline * delimited JSON is handled. * * @since 1.0.0 * @category models */ export interface NdjsonOptions { /** * Whether or not the newline delimited JSON parser should ignore empty lines. * * Defaults to `false`. * * From the [newline delimited JSON spec](https://github.com/ndjson/ndjson-spec): * ```text * The parser MAY silently ignore empty lines, e.g. \n\n. This behavior MUST * be documented and SHOULD be configurable by the user of the parser. * ``` * * @since 1.0.0 */ readonly ignoreEmptyLines?: boolean } /** * @since 1.0.0 * @category constructors */ export const packString = (): Channel.Channel< Chunk.Chunk, Chunk.Chunk, IE | NdjsonError, IE, Done, Done > => { const loop: Channel.Channel< Chunk.Chunk, Chunk.Chunk, IE | NdjsonError, IE, Done, Done > = Channel.readWithCause({ onInput: (input: Chunk.Chunk) => Channel.zipRight( Channel.flatMap( Effect.try({ try: () => Chunk.of(Chunk.toReadonlyArray(input).map((_) => JSON.stringify(_)).join("\n") + "\n"), catch: (cause) => new NdjsonError({ reason: "Pack", cause }) }), Channel.write ), loop ), onFailure: Channel.failCause, onDone: Channel.succeed }) return loop } /** * @since 1.0.0 * @category constructors */ export const pack = (): Channel.Channel< Chunk.Chunk, Chunk.Chunk, IE | NdjsonError, IE, Done, Done > => Channel.mapOut(packString(), Chunk.map((_) => encoder.encode(_))) /** * @since 1.0.0 * @category constructors */ export const packSchema = ( schema: Schema.Schema ) => (): Channel.Channel< Chunk.Chunk, Chunk.Chunk, IE | NdjsonError | ParseError, IE, Done, Done, R > => Channel.pipeTo(ChannelSchema.encode(schema)(), pack()) /** * @since 1.0.0 * @category constructors */ export const packSchemaString = ( schema: Schema.Schema ) => (): Channel.Channel< Chunk.Chunk, Chunk.Chunk, IE | NdjsonError | ParseError, IE, Done, Done, R > => Channel.pipeTo(ChannelSchema.encode(schema)(), packString()) const filterEmpty = Chunk.filter((line) => line.length > 0) const filterEmptyChannel = () => { const loop: Channel.Channel< Chunk.Chunk, Chunk.Chunk, IE, IE, Done, Done, never > = Channel.readWithCause({ onInput(input: Chunk.Chunk) { const filtered = filterEmpty(input) return Channel.zipRight(Chunk.isEmpty(filtered) ? Channel.void : Channel.write(filtered), loop) }, onFailure(cause: Cause.Cause) { return Channel.failCause(cause) }, onDone(done: Done) { return Channel.succeed(done) } }) return loop } /** * @since 1.0.0 * @category constructors */ export const unpackString = (options?: NdjsonOptions): Channel.Channel< Chunk.Chunk, Chunk.Chunk, IE | NdjsonError, IE, Done, Done > => { const lines = Channel.splitLines().pipe( options?.ignoreEmptyLines === true ? Channel.pipeTo(filterEmptyChannel()) : identity ) return Channel.mapOutEffect(lines, (chunk) => Effect.try({ try: () => Chunk.map(chunk, (_) => JSON.parse(_)), catch: (cause) => new NdjsonError({ reason: "Unpack", cause }) })) } const decodeString = () => { const decoder = new TextDecoder() const loop: Channel.Channel< Chunk.Chunk, Chunk.Chunk, IE, IE, Done, Done, never > = Channel.readWithCause({ onInput: (input) => Channel.zipRight( Channel.write(Chunk.map(input, (_) => decoder.decode(_))), loop ), onFailure: Channel.failCause, onDone: Channel.succeed }) return loop } /** * @since 1.0.0 * @category constructors */ export const unpack = (options?: NdjsonOptions): Channel.Channel< Chunk.Chunk, Chunk.Chunk, IE | NdjsonError, IE, Done, Done > => { return Channel.pipeTo(decodeString(), unpackString(options)) } /** * @since 1.0.0 * @category constructors */ export const unpackSchema = ( schema: Schema.Schema ) => (options?: NdjsonOptions): Channel.Channel< Chunk.Chunk, Chunk.Chunk, NdjsonError | ParseError | IE, IE, Done, Done, R > => Channel.pipeTo(unpack(options), ChannelSchema.decodeUnknown(schema)()) /** * @since 1.0.0 * @category constructors */ export const unpackSchemaString = ( schema: Schema.Schema ) => (options?: NdjsonOptions): Channel.Channel< Chunk.Chunk, Chunk.Chunk, NdjsonError | ParseError | IE, IE, Done, Done, R > => Channel.pipeTo(unpackString(options), ChannelSchema.decodeUnknown(schema)()) /** * @since 1.0.0 * @category combinators */ export const duplex: { /** * @since 1.0.0 * @category combinators */ (options?: NdjsonOptions): ( self: Channel.Channel, Chunk.Chunk, OE, IE | NdjsonError, OutDone, InDone, R> ) => Channel.Channel, Chunk.Chunk, NdjsonError | OE, IE, OutDone, InDone, R> /** * @since 1.0.0 * @category combinators */ ( self: Channel.Channel, Chunk.Chunk, OE, IE | NdjsonError, OutDone, InDone, R>, options?: NdjsonOptions ): Channel.Channel, Chunk.Chunk, NdjsonError | OE, IE, OutDone, InDone, R> } = dual((args) => Channel.isChannel(args[0]), ( self: Channel.Channel, Chunk.Chunk, OE, IE | NdjsonError, OutDone, InDone, R>, options?: NdjsonOptions ): Channel.Channel, Chunk.Chunk, NdjsonError | OE, IE, OutDone, InDone, R> => Channel.pipeTo( Channel.pipeTo(pack(), self), unpack(options) )) /** * @since 1.0.0 * @category combinators */ export const duplexString: { /** * @since 1.0.0 * @category combinators */ (options?: NdjsonOptions): ( self: Channel.Channel, Chunk.Chunk, OE, IE | NdjsonError, OutDone, InDone, R> ) => Channel.Channel, Chunk.Chunk, NdjsonError | OE, IE, OutDone, InDone, R> /** * @since 1.0.0 * @category combinators */ ( self: Channel.Channel, Chunk.Chunk, OE, IE | NdjsonError, OutDone, InDone, R>, options?: NdjsonOptions ): Channel.Channel, Chunk.Chunk, NdjsonError | OE, IE, OutDone, InDone, R> } = dual((args) => Channel.isChannel(args[0]), ( self: Channel.Channel, Chunk.Chunk, OE, IE | NdjsonError, OutDone, InDone, R>, options?: NdjsonOptions ): Channel.Channel, Chunk.Chunk, NdjsonError | OE, IE, OutDone, InDone, R> => Channel.pipeTo( Channel.pipeTo(packString(), self), unpackString(options) )) /** * @since 1.0.0 * @category combinators */ export const duplexSchema: { /** * @since 1.0.0 * @category combinators */ ( options: Partial & { readonly inputSchema: Schema.Schema readonly outputSchema: Schema.Schema } ): ( self: Channel.Channel< Chunk.Chunk, Chunk.Chunk, OutErr, NdjsonError | ParseError | InErr, OutDone, InDone, R > ) => Channel.Channel< Chunk.Chunk, Chunk.Chunk, NdjsonError | ParseError | OutErr, InErr, OutDone, InDone, R | IR | OR > /** * @since 1.0.0 * @category combinators */ ( self: Channel.Channel< Chunk.Chunk, Chunk.Chunk, OutErr, NdjsonError | ParseError | InErr, OutDone, InDone, R >, options: Partial & { readonly inputSchema: Schema.Schema readonly outputSchema: Schema.Schema } ): Channel.Channel< Chunk.Chunk, Chunk.Chunk, NdjsonError | ParseError | OutErr, InErr, OutDone, InDone, R | IR | OR > } = dual(2, ( self: Channel.Channel< Chunk.Chunk, Chunk.Chunk, OutErr, NdjsonError | ParseError | InErr, OutDone, InDone, R >, options: Partial & { readonly inputSchema: Schema.Schema readonly outputSchema: Schema.Schema } ): Channel.Channel< Chunk.Chunk, Chunk.Chunk, NdjsonError | ParseError | OutErr, InErr, OutDone, InDone, R | IR | OR > => ChannelSchema.duplexUnknown(duplex(self, options), options)) /** * @since 1.0.0 * @category combinators */ export const duplexSchemaString: { /** * @since 1.0.0 * @category combinators */ ( options: Partial & { readonly inputSchema: Schema.Schema readonly outputSchema: Schema.Schema } ): ( self: Channel.Channel< Chunk.Chunk, Chunk.Chunk, OutErr, NdjsonError | ParseError | InErr, OutDone, InDone, R > ) => Channel.Channel< Chunk.Chunk, Chunk.Chunk, NdjsonError | ParseError | OutErr, InErr, OutDone, InDone, R | IR | OR > /** * @since 1.0.0 * @category combinators */ ( self: Channel.Channel< Chunk.Chunk, Chunk.Chunk, OutErr, NdjsonError | ParseError | InErr, OutDone, InDone, R >, options: Partial & { readonly inputSchema: Schema.Schema readonly outputSchema: Schema.Schema } ): Channel.Channel< Chunk.Chunk, Chunk.Chunk, NdjsonError | ParseError | OutErr, InErr, OutDone, InDone, R | IR | OR > } = dual(2, ( self: Channel.Channel< Chunk.Chunk, Chunk.Chunk, OutErr, NdjsonError | ParseError | InErr, OutDone, InDone, R >, options: Partial & { readonly inputSchema: Schema.Schema readonly outputSchema: Schema.Schema } ): Channel.Channel< Chunk.Chunk, Chunk.Chunk, NdjsonError | ParseError | OutErr, InErr, OutDone, InDone, R | IR | OR > => ChannelSchema.duplexUnknown(duplexString(self, options), options))