import { Stream, TypeToFields } from "../data/Stream"; import { BlobType, BooleanType, DateTimeType, DictType, EastFunction, EastType, EastTypeOf, IntegerType, Nullable, PatchType, StringType, StructType, Value, ValueTypeOf, Variable } from "../east"; import { Builder, ModulePath, Template } from '../template'; import { Assertion } from '../task'; import { ElaraSourceTaskDescription, PatchSourceTaskDescription, WriteableDefaultConfig } from './Source'; import { ModuleBuilder } from '../template/ModuleBuilder'; /** * A helper class to build a data {@link Source}, which populates a data {@link Stream} * with data. Data sources can range from simple clocks to complex database queries or API requests. * * A corresponding {@link Template} can be created using `.toTemplate()`. * * @remarks See [Ingest Structured Data](https://learn.elaraai.com/topics/03_work_with_data/get_data_in_and_out/03_define_a_datasource) for a related learning module. * * @category Source * * @example * ```typescript * // Create a datastream with an integer value that can be provided by users * export default new SourceBuilder("Hourly") * .writeable(IntegerType) * .toTemplate() * ``` */ export declare class SourceBuilder = {}> { private name; private inputs; private input_streams; private input_assertions; private module; /** * Construct a `SourceBuilder` to build a data {@link Source}, which populates a data {@link Stream} * with data. Data sources can range from simple clocks to complex database queries or API requests. * * In an EDK project, a data source will usually be exported using `.toTemplate()`. * * @param name the name of the {@link Source} * * @category Source * * @example * ```typescript * // Create a datastream which is updated to the current time at the beginning of every hour * export default new SourceBuilder("Hourly") * .writeable(IntegerType) * .toTemplate() * ``` */ constructor(name: string, module?: ModuleBuilder | ModulePath); /** @internal */ constructor(name: string, module: ModulePath, inputs: Inputs, input_streams: Record, input_assertions: Assertion[]); /** * Add a stream as input to the `SourceBuilder`, which can be used in * expressions to configure how the {@link Source} will access the data (such as a URL or password), * the output data, or any data assertions (preconditions and postconditions). * * @param config the configuration for the `input` * @returns a new {@link SourceBuilder} * * @category Source * * @example * ```typescript * const other = Stream("Other", FloatType) * const source = new SourceBuilder("Source") * .input({ name: "other", stream: other }) * .clock({ cron: "0 * * * *" }) * .toSource() * ``` */ input(config: { /** the name to give the input {@link Variable} */ name: Name & Name extends (keyof Inputs | "input") ? never : Name; /** the input stream (the stream and associated preconditions) */ stream: Stream; }): SourceBuilder; }>; /** * Check the source inputs for validity and identify any errors. When the `if` predicate returns `true` * the source will be terminated with an error message and output data will not be produced. * * @param config the error message and predicate * @returns a new {@link SourceBuilder} * * @category Source * * @example * ```typescript * const source = new SourceBuilder("Source") * .input({ name: "other", stream: other }) * // ensure that other is not negative * .error({ * if: ({ other }) => Less(other, 0), * message: () => Const("other is negative") * }) * .clock({ cron: "0 * * * *" }) * .toSource() * ``` */ error(config: { /** If `true` an error will be created */ if: (inputs: Inputs) => EastFunction; /** The message in the case that an error is created */ message: ((inputs: Inputs) => EastFunction) | string; }): SourceBuilder; /** * Check the source inputs for validity and raise warnings as necessary. When the `if` predicate returns `true` * the source will register a warning with a message, but will proceed to proceed to produce output data. * * @param config the warning message and predicate * @returns a new {@link SourceBuilder} * * @category Source * * @example * ```typescript * const source = new SourceBuilder("Source") * .input({ name: "other", stream: other }) * // warn whenever other is not negative * .warn({ * if: ({ other }) => Less(other, 0), * message: () => Const("other is negative") * }) * .clock({ cron: "0 * * * *" }) * .toSource() * ``` */ warn(config: { /** If true a warning will be produced */ if: (inputs: Inputs) => EastFunction; /** The message in the case that a warning is created */ message: ((inputs: Inputs) => EastFunction) | string; }): SourceBuilder; /** * Produce log messages based on the source inputs. When the `if` predicate returns `true` * the source will register a log message and proceed to proceed to produce output data. * * @param config the log message and optional predicate * @returns a new {@link SourceBuilder} * * @category Source * * @example * ```typescript * const source = new SourceBuilder("Source") * .input({ name: "other", stream: other }) * // log a message whenever other is negative * .log({ * if: ({ other }) => Less(other, 0), * message: () => Const("other is negative") * }) * .clock({ cron: "0 * * * *" }) * .toSource() * ``` */ log(config: { /** If true a log message will be produced (optional) */ if?: (inputs: Inputs) => EastFunction; /** The log message */ message: ((inputs: Inputs) => EastFunction) | string; }): SourceBuilder; /** * Build a writeable source, consisting of a datastream that end-users can write values to at runtime. * * @param type The {@link EastType} of the value that can be written to the stream. * @returns a new {@link SourceBuilder} * * @example * ```typescript * const source = new SourceBuilder("FloatSource") * .writeable(FloatType) * ``` */ writeable>(type: T): TabularWriteableSourceBuilder; writeable(type: T): GenericWriteableSourceBuilder; /** * Build a writeable source of type `BlobType`, consisting of a blob datastream populated with a file on your local disk (which end-users can overwrite at runtime). * * @param config The configutation of the `file` datasource. * @returns a new {@link SourceBuilder} * * @example * ```typescript * const source = new SourceBuilder("FileSource") * .file({ * path: "./my_big_csv_file.csv", * }) * ``` */ file(config: { path: string; }): GenericWriteableSourceBuilder; /** * Build a writeable source, consisting of a datastream with an initial default value (which end-users can overwrite at runtime). * * @param config The configutation of the `file` datasource. * @returns a new {@link SourceBuilder} * * @example * ```typescript * const source = new SourceBuilder("StringSource") * .value({ * value: "a string" * }) * ``` */ value(config: { /** The {@link EastType} of the value to be written to the stream. **/ type: T; /** The value to be written to the stream. **/ value: ValueTypeOf; }): WriteableSourceBuilder; value(config: { /** The value to be written to the stream. **/ value: V; }): WriteableSourceBuilder, Inputs>; /** * Build a patch source that modify or update a "base" stream. * * @param base_stream the "base" stream to apply a patch to * @returns a new {@link SourceBuilder} * * @category Source * * @example * ```typescript * // Create a datastream which whose value is a (writeable) patch applied to a base stream * export default new SourceBuilder("Patch") * .patch(base_stream) * .toTemplate() * ``` */ patch(base_stream: Stream): PatchSourceBuilder; /** * Build a clock source that runs based on a [cron](https://en.wikipedia.org/wiki/Cron) string. * * @param config the configuration of the clock source * @returns a new {@link SourceBuilder} * * @category Source * * @example * ```typescript * // Create a datastream which is updated to the current time at the beginning of every hour * export default new SourceBuilder("Hourly") * .clock({ * cron: "0 * * * *", * value: datetime => StringJoin`The current time is ${datetime}` * }) * .toTemplate() * ``` */ clock, inputs: Inputs) => EastFunction>(config: { /** The [cron](https://en.wikipedia.org/wiki/Cron) string used to determine when to trigger the clock source */ cron: string; /** A function to generate an {@link EastFunction} for the output value based on the current time (optional) */ value: V; }): ClockSourceBuilder["type"], Inputs>; clock(config: { /** The [cron](https://en.wikipedia.org/wiki/Cron) string used to determine when to trigger the clock source */ cron: string; }): ClockSourceBuilder; /** * Build a source that pulls data from a datastream in a different workspace in the same tenant. * * @param config an object containing the workspace name, stream name and type * @returns a new {@link SourceBuilder} * * @category Source * * @example * ```typescript * // Create a datastream which whose value comes from the output of pipeline `my_pipeline` in another workspace called `my_other_workspace`. * export default new SourceBuilder("Elara") * .elara({ * workspace: "my_other_workspace", * stream: "Pipeline.my_pipeline", * type: IntegerType, * }) * .toTemplate() * ``` */ elara(config: { workspace: string; stream: string; type: T; }): ElaraSourceBuilder; /** * Build source from a http get request * * @param request the configuration of the http request * @returns a new {@link SourceBuilder} * * @category Source * * @example * ```typescript * // Create a blob datastream containing the response from a http get request, updated daily * export default new SourceBuilder("Hourly") * .api({ * cron: "0 8 * * *", * url: () => Const("https://covid.ourworldindata.org/data/owid-covid-data.json"), * retry: (_, attempts) => Less(attempts, 5n) * }) * .toTemplate() * ``` */ api>; headers: Variable>>; status_code: Variable; failure: Variable; }, attempts: Variable, inputs: Inputs) => EastFunction, RB extends (prev_response: { body: Variable>; headers: Variable>>; status_code: Variable; failure: Variable; }, attempts: Variable, inputs: Inputs) => EastFunction, RH extends (prev_response: { body: Variable>; headers: Variable>>; status_code: Variable; failure: Variable; }, attempts: Variable, inputs: Inputs) => EastFunction>, RR extends (prev_response: { body: Variable>; headers: Variable>>; status_code: Variable; failure: Variable; }, attempts: Variable, inputs: Inputs) => EastFunction, RD extends (prev_response: { body: Variable>; headers: Variable>>; status_code: Variable; failure: Variable; }, attempts: Variable, inputs: Inputs) => EastFunction>(request: { /** The [cron](https://en.wikipedia.org/wiki/Cron) string used to determine when to trigger the clock source */ cron?: string; /** An {@link EastFunction} for the request url */ url: RU; /** An {@link EastFunction} for the request body */ body?: RB; /** An {@link EastFunction} for the request headers */ headers?: RH; /** An {@link EastFunction} to determine if the request should be retired */ retry?: RR; /** An {@link EastFunction} for the request url */ delay?: RD; }): ApiSourceBuilder; /** * Build source from a file at an ftp or sftp uri * * @param config the configuration of the ftp file * @returns a new {@link SourceBuilder} * * @category Source * * @example * ```typescript * // Create a blob datastream containing the data in an ftp file, updated daily * export default new SourceBuilder("Hourly") * .ftp({ * cron: "0 8 * * *", * uri: () => Const("ftp://ftp.bom.gov.au/anon/gen/clim_data/IDCKWCDEA0/tables/qld/applethorpe/applethorpe-200902.csv"), * }) * .toTemplate() * ``` */ ftp EastFunction, P extends (inputs: Inputs) => EastFunction>(config: { /** The [cron](https://en.wikipedia.org/wiki/Cron) string used to determine when to trigger the clock source */ cron?: string; /** An {@link EastFunction} for the ftp uri */ uri: U; /** An {@link EastFunction} for the ftp username */ username?: U; /** An {@link EastFunction} for the ftp password */ password?: U; /** An {@link EastFunction} for the ftp key */ key?: U; /** An {@link EastFunction} for the ftp port */ port?: P; }): FtpSourceBuilder; /** * Build source from a file at an s3 path * * @param config the configuration of the s3 blob * @returns a new {@link SourceBuilder} * * @category Source * * @example * ```typescript * // Create an s3 datastream * export default new SourceBuilder("My S3 Blob") * .s3({ * region: (inputs) => Const("__REGION__"), * bucket: (inputs) => Const("__BUCKET__"), * key: (inputs) => Const("__KEY__"), * access_key_id: (inputs) => Const("__ACCESS_KEY_ID__"), * secret_access_key: (inputs) => Const("__SECRET_KEY__") * }) * .toTemplate() * ``` */ s3 EastFunction>(config: { /** The [cron](https://en.wikipedia.org/wiki/Cron) string used to determine when to trigger the clock source */ cron?: string; /** An {@link EastFunction} with the aws region */ region: U; /** An {@link EastFunction} with a bucket name containing the object */ bucket: U; /** An {@link EastFunction} with the key of the object to get. */ key: U; /** An {@link EastFunction} with the AWS access key ID */ access_key_id: U; /** An {@link EastFunction} with the AWS secret access key */ secret_access_key: U; }): S3SourceBuilder; } /** @internal */ declare abstract class AbstractSourceBuilder> extends Builder { protected type: T; protected inputs: Inputs; protected input_streams: Record; protected input_assertions: Assertion[]; protected output_assertions: Assertion[]; /** @internal */ constructor(name: string, module: ModulePath, type: T, inputs: Inputs, input_streams: Record, input_assertions: Assertion[], output_assertions: Assertion[]); /** * Check the source outputs for validity and identify any errors. When the `if` predicate returns `true` * the source will be terminated with an error message and output data will not be produced. * * @param config the error message and predicate * @returns a new {@link SourceBuilder} * * @category Source * * @example * ```typescript * const source = new SourceBuilder("FloatSource") * .input({ name: "max", stream: max_stream }) * .writeable(FloatType) * .error({ * if: (value, { max }) => Greater(value, max), * message: (value) => StringJoin`The value ${value} exceeds maximum` * }) * ``` */ error(config: { /** If true an error will be created */ if: (value: Variable, inputs: Inputs) => EastFunction; /** The message in the case that an error is created */ message: ((value: Variable, inputs: Inputs) => EastFunction) | string; }): this; /** * Check the source outputs for validity and produce warnings as necessary. When the `if` predicate returns `true` * the source will register a warning with a message, but will proceed to proceed to produce output data. * * @param config the warning message and predicate * @returns a new {@link SourceBuilder} * * @category Source * * @example * ```typescript * const source = new SourceBuilder("FloatSource") * .input({ name: "max", stream: max_stream }) * .writeable(FloatType) * .warn({ * if: (value, { max }) => Greater(value, max), * message: (value) => StringJoin`The value ${value} exceeds maximum` * }) * ``` */ warn(config: { /** If true a warning will be produced */ if: (value: Variable, inputs: Inputs) => EastFunction; /** The message in the case that a warning is created */ message: ((value: Variable, inputs: Inputs) => EastFunction) | string; }): this; /** * Produce log messages based on the source outputs and inputs. When the `if` predicate returns `true` * source will register a warning with a message, but will proceed to proceed to produce output data. * * @param config the log message and optional predicate * @returns a new {@link SourceBuilder} * * @category Source * * @example * ```typescript * const source = new SourceBuilder("FloatSource") * .input({ name: "max", stream: max_stream }) * .writeable(FloatType) * .log({ * if: (value, { max }) => Greater(value, max), * message: (value) => StringJoin`The value ${value} exceeds maximum` * }) * ``` */ log(config: { /** If true a log message will be produced (optional) */ if?: (value: Variable, inputs: Inputs) => EastFunction; /** The message in the case that a warning is created */ message: ((value: Variable, inputs: Inputs) => EastFunction) | string; }): this; /** * Return the {@link Stream} containing the output of the data source. * * @category Source * * @example * ```typescript * // Create a datastream which is updated to the current time at the beginning of every hour * const hourly = new SourceBuilder("Hourly") * .clock({ cron: "0 * * * *" }) * .outputStream() * ``` */ abstract outputStream(): Stream; /** * Return a {@link Template} fragment containing the {@link Stream}s and {@link TaskDescription}s built by this builder. * * @category Source * * @example * ```typescript * // Create a datastream which is updated to the current time at the beginning of every hour * export default new SourceBuilder("Hourly") * .clock({ cron: "0 * * * *" }) * .toTemplate() * ``` */ abstract toTemplate(): Template; } /** @internal */ type WriteableSourceBuilder> = T extends DictType ? TabularWriteableSourceBuilder : GenericWriteableSourceBuilder; /** * {@inheritDoc SourceBuilder} * * @category Source * */ export declare class GenericWriteableSourceBuilder> extends AbstractSourceBuilder { private default_value; /** @internal */ constructor(name: string, module: ModulePath, type: T, inputs: Inputs, input_streams: Record, input_assertions: Assertion[], output_assertions: Assertion[], default_value: WriteableDefaultConfig); /** * Add a stream as input to the `SourceBuilder`, which can be used in * expressions to configure how the {@link Source} will access the data (such as a URL or password), * the output data, or any data assertions (preconditions and postconditions). * * @param config the configuration for the `input` * @returns a new {@link SourceBuilder} * * @category Source * * @example * ```typescript * const other = Stream("Other", FloatType) * const source = new SourceBuilder("Source") * .input({ name: "other", stream: other }) * .clock({ cron: "0 * * * *" }) * .toSource() * ``` */ input(config: { /** the name to give the input {@link Variable} */ name: Name & Name extends (keyof Inputs | "input") ? never : Name; /** the input stream (the stream and associated preconditions) */ stream: Stream; }): GenericWriteableSourceBuilder; }>; /** internal */ private isBareStream; /** * Return the {@link Stream} containing the stream end-users can write to. * * @category Source * * @example * ```typescript * // Create a datastream where a password can be written by end-user at runtime. * const hourly = new SourceBuilder("DatabasePassword") * .writeable(StringType) * .outputStream() * ``` */ writeableStream(): Stream; /** * Return the {@link Stream} containing the output of the data source. * * @category Source * * @example * ```typescript * // Create a datastream where a password can be written by end-user at runtime. * const hourly = new SourceBuilder("DatabasePassword") * .writeable(StringType) * .outputStream() * ``` */ outputStream(): Stream; /** * Return a {@link Template} fragment containing the {@link Stream}s and {@link TaskDescription}s built by this builder. * * @category Source * * @example * ```typescript * // Create a datastream which is updated to the current time at the beginning of every hour * export default new SourceBuilder("Hourly") * .clock({ cron: "0 * * * *" }) * .toTemplate() * ``` */ toTemplate(): Template; } /** * {@inheritDoc SourceBuilder} * * @category Source * */ export declare class TabularWriteableSourceBuilder, Inputs extends Record> extends AbstractSourceBuilder { private default_value; /** @internal */ constructor(name: string, module: ModulePath, type: T, inputs: Inputs, input_streams: Record, input_assertions: Assertion[], output_assertions: Assertion[], default_value: WriteableDefaultConfig); /** * Add a stream as input to the `SourceBuilder`, which can be used in * expressions to configure how the {@link Source} will access the data (such as a URL or password), * the output data, or any data assertions (preconditions and postconditions). * * @param config the configuration for the `input` * @returns a new {@link SourceBuilder} * * @category Source * * @example * ```typescript * const other = Stream("Other", FloatType) * const source = new SourceBuilder("Source") * .input({ name: "other", stream: other }) * .clock({ cron: "0 * * * *" }) * .toSource() * ``` */ input(config: { /** the name to give the input {@link Variable} */ name: Name & Name extends (keyof Inputs | "input") ? never : Name; /** the input stream (the stream and associated preconditions) */ stream: Stream; }): TabularWriteableSourceBuilder; }>; /** @internal */ private isBareStream; /** * Add an assertion on every row of the source output to identify errors. For each row when the `if` predicate returns `true` * the source will be terminated with an error message and output data will not be produced. * * @param config the error message and predicate * @returns a new {@link SourceBuilder} * * @category Source * * @example * ```typescript * const maximumValue = Stream("MaximumValue", FloatType) * * const tableSource = new SourceBuilder("TableSource") * .input({ name: "max", stream: other }) * .writeable(DictType(StringType, StructType({ id: IntegerType, value: FloatType, }))) * .errorEvery({ * if: ({ value }, { max }) => Greater(value, max), * message: ({ id }) => StringJoin`The value for row with id ${id} exceeds maximum` * }) * ``` */ errorEvery(config: { /** If true an error will be created */ if: (fields: TypeToFields, key: Variable, inputs: Inputs) => EastFunction; /** The message in the case that an error is created */ message: ((fields: TypeToFields, key: Variable, inputs: Inputs) => EastFunction) | string; }): TabularWriteableSourceBuilder; /** * Add an warning on every row of the source output to identify errors. For each row when the `if` predicate returns `true` * the source will register a warning with a message, but will proceed to proceed to produce output data. * * @param config the warning message and predicate * @returns a new {@link SourceBuilder} * * @category Source * * @example * ```typescript * const maximumValue = Stream("MaximumValue", FloatType) * * const tableSource = new SourceBuilder("TableSource") * .input({ name: "max", stream: other }) * .writeable(DictType(StringType, StructType({ id: IntegerType, value: FloatType, }))) * .warnEvery({ * if: ({ value }, { max }) => Greater(value, max), * message: ({ id }) => StringJoin`The value for row with id ${id} exceeds maximum` * }) * ``` */ warnEvery(config: { /** If true a warning will be produced */ if: (fields: TypeToFields, key: Variable, inputs: Inputs) => EastFunction; /** The message in the case that a warning is create */ message: ((fields: TypeToFields, key: Variable, inputs: Inputs) => EastFunction) | string; }): TabularWriteableSourceBuilder; /** * Produce a long message for every row of the table. For each row when the `if` predicate returns `true` * the source will register a log message and proceed to proceed to produce output data. * * @param config the log message and optional predicate * @returns a new {@link SourceBuilder} * * @category Source * * @example * ```typescript * const maximumValue = Stream("MaximumValue", FloatType) * * const tableSource = new SourceBuilder("TableSource") * .input({ name: "max", stream: other }) * .writeable(DictType(StringType, StructType({ id: IntegerType, value: FloatType, }))) * .logEvery({ * if: ({ value }, { max }) => Greater(value, max), * message: ({ id }) => StringJoin`The value for row with id ${id} exceeds maximum` * }) * ``` */ logEvery(config: { /** If true a log message will be produced */ if?: (fields: TypeToFields, key: Variable, inputs: Inputs) => EastFunction; /** The message in the case that a warning is create */ message: ((fields: TypeToFields, key: Variable, inputs: Inputs) => EastFunction) | string; }): TabularWriteableSourceBuilder; /** * Return the {@link Stream} containing the stream end-users can write to. * * @category Source * * @example * ```typescript * // Create a datastream where a password can be written by end-user at runtime. * const hourly = new SourceBuilder("DatabasePassword") * .writeable(StringType) * .outputStream() * ``` */ writeableStream(): Stream; /** * Return the {@link Stream} containing the output of the data source. * * @category Source * * @example * ```typescript * // Create a datastream where a password can be written by end-user at runtime. * const hourly = new SourceBuilder("DatabasePassword") * .writeable(StringType) * .outputStream() * ``` */ outputStream(): Stream; /** * Return a {@link Template} fragment containing the {@link Stream}s and {@link TaskDescription}s built by this builder. * * @category Source * * @example * ```typescript * // Create a datastream which is updated to the current time at the beginning of every hour * export default new SourceBuilder("Hourly") * .clock({ cron: "0 * * * *" }) * .toTemplate() * ``` */ toTemplate(): Template; } /** * A helper class to build a patch source that modify or update a "base" stream. * * @category Source * * @example * ```typescript * // Create a datastream which whose value is a (writeable) patch applied to a base stream * export default new SourceBuilder("Patch") * .patch(base_stream) * .toTemplate() * ``` */ export declare class PatchSourceBuilder extends Builder { private type; private base; private patch; /** @internal */ constructor(name: string, module: ModulePath, type: T, base: Stream, patch: GenericWriteableSourceBuilder, {}>); /** * Initialize the patch stream with a value. * * @param value a default patch to apply * @returns a new {@link PatchSourceBuilder} * * @category Source * * @example * ```typescript * // Create a datastream which whose value is a base stream patched with an additional inserted key-value pair * const source = SourceBuilder("Patch") * .patch(base_stream) * .value(new Map([["key", variant("Insert", "value")]])) * ``` */ value(value: ValueTypeOf>): this; writeableStream(): Stream>; /** * Return the {@link Stream} containing any conflicts that occurred when attempting to apply the patch. * * Conflicts can occur when: * * 1. a key is inserted when it already exists in the base * 2. a key is updated that doesn't exist in the base, or has an unexpected value in the base * 3. a key is deleted when it doesn't exist in the base * * The stream will be empty when there are no conflicts * * @category Source * * @example * ```typescript * // Return the datastream containing any patch conflicts * const source = new SourceBuilder("Patch") * .patch(base_stream) * .conflictStream() * ``` */ conflictStream(): Stream>; /** * Return the {@link Stream} containing the patched output. * * @category Source * * @example * ```typescript * // Return the datastream which whose value is a (writeable) patch applied to a base stream * const source = new SourceBuilder("Patch") * .patch(base_stream) * .outputStream() * ``` */ outputStream(): Stream; /** * Return a {@link TaskDescription}s built by this builder. * * @category Source * * @example * ```typescript * // Create a datastream which whose value is a (writeable) patch applied to a base stream * const task = new SourceBuilder("Patch") * .patch(base_stream) * .toTask() * ``` */ toTask(): PatchSourceTaskDescription & { task_type: "source"; }; /** * Return a {@link Template} fragment containing the {@link Stream}s and {@link TaskDescription}s built by this builder. * * @category Source * * @example * ```typescript * // Create a datastream which whose value is a (writeable) patch applied to a base stream * const template = new SourceBuilder("Patch") * .patch(base_stream) * .toTemplate() * ``` */ toTemplate(): Template; } /** * {@inheritDoc SourceBuilder} * * @category Source * */ export declare class ClockSourceBuilder> extends AbstractSourceBuilder { cron: string; value: EastFunction; /** @internal */ constructor(name: string, module: ModulePath, type: T, inputs: Inputs, input_streams: Record, input_assertions: Assertion[], output_assertions: Assertion[], cron: string, value: EastFunction); /** * Add a stream as input to the `SourceBuilder`, which can be used in * expressions to configure how the {@link Source} will access the data (such as a URL or password), * the output data, or any data assertions (preconditions and postconditions). * * @param config the configuration for the `input` * @returns a new {@link SourceBuilder} * * @category Source * * @example * ```typescript * const other = Stream("Other", FloatType) * const source = new SourceBuilder("Source") * .input({ name: "other", stream: other }) * .clock({ cron: "0 * * * *" }) * .toSource() * ``` */ input(config: { /** the name to give the input {@link Variable} */ name: Name & Name extends (keyof Inputs | "input") ? never : Name; /** the input stream (the stream and associated preconditions) */ stream: Stream; }): ClockSourceBuilder; }>; /** * Return the {@link Stream} containing the output of the datasource. * * @category Source * * @example * ```typescript * // Create a datastream which is updated to the current time at the beginning of every hour * const hourly = new SourceBuilder("Hourly") * .clock({ cron: "0 * * * *" }) * .outputStream() * ``` */ outputStream(): Stream; /** * Return a {@link Template} fragment containing the {@link Stream}s and {@link TaskDescription}s built by this builder. * * @category Source * * @example * ```typescript * // Create a datastream which is updated to the current time at the beginning of every hour * export default new SourceBuilder("Hourly") * .clock({ cron: "0 * * * *" }) * .toTemplate() * ``` */ toTemplate(): Template; } /** * A helper class to build a Elara source that fetches data from another workspace in the same tenant. * * @category Source * * @example * ```typescript * // Create a datastream which whose value comes from the output of pipeline `my_pipeline` in another workspace called `my_other_workspace`. * export default new SourceBuilder("Elara") * .elara({ * workspace: "my_other_workspace", * stream: "Pipeline.my_pipeline", * type: IntegerType, * }) * .toTemplate() * ``` */ export declare class ElaraSourceBuilder extends Builder { private workspace; private stream; private type; /** @internal */ constructor(name: string, module: ModulePath, workspace: string, stream: string, type: T); /** * Return the {@link Stream} containing the patched output. * * @category Source * * @example * ```typescript * // Return a datastream which whose value comes from the output of pipeline `my_pipeline` in another workspace called `my_other_workspace`. * export default new SourceBuilder("Elara") * .elara({ * workspace: "my_other_workspace", * stream: "Pipeline.my_pipeline", * type: IntegerType, * }) * .outputStream() */ outputStream(): Stream; /** * Return a {@link TaskDescription}s built by this builder. * * @category Source * * @example * ```typescript * // Create a datastream which whose value comes from the output of pipeline `my_pipeline` in another workspace called `my_other_workspace`. * export default new SourceBuilder("Elara") * .elara({ * workspace: "my_other_workspace", * stream: "Pipeline.my_pipeline", * type: IntegerType, * }) * .toTask() * ``` */ toTask(): ElaraSourceTaskDescription; /** * Return a {@link Template} fragment containing the {@link Stream}s and {@link TaskDescription}s built by this builder. * * @category Source * * @example * ```typescript * // Create a datastream which whose value comes from the output of pipeline `my_pipeline` in another workspace called `my_other_workspace`. * export default new SourceBuilder("Elara") * .elara({ * workspace: "my_other_workspace", * stream: "Pipeline.my_pipeline", * type: IntegerType, * }) * .toTemplate() * ``` */ toTemplate(): Template; } /** * {@inheritDoc SourceBuilder} * * @category Source * */ export declare class ApiSourceBuilder> extends AbstractSourceBuilder { protected request_url: EastFunction; protected request_body?: EastFunction | undefined; protected request_headers?: EastFunction> | undefined; protected request_retry?: EastFunction | undefined; protected request_delay?: EastFunction | undefined; cron?: string | undefined; /** @internal */ constructor(name: string, module: ModulePath, type: B, inputs: Inputs, input_streams: Record, input_assertions: Assertion[], output_assertions: Assertion[], request_url: EastFunction, request_body?: EastFunction | undefined, request_headers?: EastFunction> | undefined, request_retry?: EastFunction | undefined, request_delay?: EastFunction | undefined, cron?: string | undefined); /** * Return the {@link Stream} containing the output of the datasource. * * @category Source * * @example * ```typescript * // Create a datastream which is updated to the current time at the beginning of every hour * const hourly = new SourceBuilder("Hourly") * .api({ * cron: "0 8 * * *", * url: () => Const("https://covid.ourworldindata.org/data/owid-covid-data.json"), * retry: (_, attempts) => Less(attempts, 5n) * }) * .outputStream() * ``` */ outputStream(): Stream; /** * Return a {@link Template} fragment containing the {@link Stream}s and {@link TaskDescription}s built by this builder. * * @category Source * * @example * ```typescript * // Create a datastream which is updated to the current time at the beginning of every hour * export default new SourceBuilder("Hourly") * .api({ * cron: "0 8 * * *", * url: () => Const("https://covid.ourworldindata.org/data/owid-covid-data.json"), * retry: (_, attempts) => Less(attempts, 5n) * }) * .toTemplate() * ``` */ toTemplate(): Template; } /** * {@inheritDoc SourceBuilder} * * @category Source * */ export declare class FtpSourceBuilder> extends AbstractSourceBuilder { protected ftp_uri: EastFunction; protected ftp_username?: EastFunction | undefined; protected ftp_password?: EastFunction | undefined; protected ftp_key?: EastFunction | undefined; protected ftp_port?: EastFunction | undefined; cron?: string | undefined; /** @internal */ constructor(name: string, module: ModulePath, type: B, inputs: Inputs, input_streams: Record, input_assertions: Assertion[], output_assertions: Assertion[], ftp_uri: EastFunction, ftp_username?: EastFunction | undefined, ftp_password?: EastFunction | undefined, ftp_key?: EastFunction | undefined, ftp_port?: EastFunction | undefined, cron?: string | undefined); /** * Return the {@link Stream} containing the output of the datasource. * * @category Source * * @example * ```typescript * // Create an ftp datastream which is updated at the beginning of every hour * const hourly = new SourceBuilder("Hourly") * .ftp({ * cron: "0 8 * * *", * uri: () => Const("ftp://ftp.bom.gov.au/anon/gen/clim_data/IDCKWCDEA0/tables/qld/applethorpe/applethorpe-200902.csv"), * }) * .outputStream() * ``` */ outputStream(): Stream; /** * Return a {@link Template} fragment containing the {@link Stream}s and {@link TaskDescription}s built by this builder. * * @category Source * * @example * ```typescript * // Create a datastream which is updated to the current time at the beginning of every hour * export default new SourceBuilder("Hourly") * .ftp({ * cron: "0 8 * * *", * uri: () => Const("ftp://ftp.bom.gov.au/anon/gen/clim_data/IDCKWCDEA0/tables/qld/applethorpe/applethorpe-200902.csv"), * }) * .toTemplate() * ``` */ toTemplate(): Template; } /** * {@inheritDoc SourceBuilder} * * @category Source * */ export declare class S3SourceBuilder> extends AbstractSourceBuilder { protected s3_region: EastFunction; protected s3_bucket: EastFunction; protected s3_key: EastFunction; protected s3_access_key_id: EastFunction; protected s3_secret_access_key: EastFunction; cron?: string | undefined; /** @internal */ constructor(name: string, module: ModulePath, type: B, inputs: Inputs, input_streams: Record, input_assertions: Assertion[], output_assertions: Assertion[], s3_region: EastFunction, s3_bucket: EastFunction, s3_key: EastFunction, s3_access_key_id: EastFunction, s3_secret_access_key: EastFunction, cron?: string | undefined); /** * Return the {@link Stream} containing the output of the datasource. * * @category Source * * @example * ```typescript * // Create an s3 datastream * const source = new SourceBuilder("My S3 Blob") * .s3({ * region: (inputs) => Const("__REGION__"), * bucket: (inputs) => Const("__BUCKET__"), * key: (inputs) => Const("__KEY__"), * access_key_id: (inputs) => Const("__ACCESS_KEY_ID__"), * secret_access_key: (inputs) => Const("__SECRET_KEY__") * }) * .outputStream() * ``` */ outputStream(): Stream; /** * Return a {@link Template} fragment containing the {@link Stream}s and {@link TaskDescription}s built by this builder. * * @category Source * * @example * ```typescript * // Create an s3 datastream * export default new SourceBuilder("My S3 Blob") * .s3({ * region: (inputs) => Const("__REGION__"), * bucket: (inputs) => Const("__BUCKET__"), * key: (inputs) => Const("__KEY__"), * access_key_id: (inputs) => Const("__ACCESS_KEY_ID__"), * secret_access_key: (inputs) => Const("__SECRET_KEY__") * }) * .toTemplate() * ``` */ toTemplate(): Template; } export {};