import { Stream, StreamIO } from '../data/Stream'; import { BlobType, EastFunction, EastType, StringType, Variable } from '../east'; import { Builder, ModulePath } from '../template'; import { ModuleBuilder } from '../template/ModuleBuilder'; /** * A helper class to build a custom task to run on the ELARA platform. * * The custom task executes a command inside a dockerized sandbox. It is given some input * files as blob data, and produces some output files as blob data. The custom task can run * any linux-compatible binary program (which can be provided as one of the input files) * inside our default Ubuntu container, or any docker image you upload to ELARA via a blob * {@link Stream}. * * A corresponding {@link Template} can be created using `.toTemplate()`. * * @category Task * * @example * ```typescript * // Create a custom task which uses a user-provided program to transform a JSON file into another JSON file * const my_task = new CustomTaskBuilder("MyTask") * .input("my_program", my_program_stream, true) * .input("input_data.json", input_data_stream, false, data => Utf8Encode(ToJson(data))) * .output("output_data.json", blob => FromJson(MyOutputType, Utf8Decode(blob))) * .shell("./my_program input_data.json output_data.json") * ``` */ export declare class CustomTaskBuilder = {}, Results extends Record = {}> extends Builder { private inputs; private outputs; private results; private inputFiles; private outputFiles; private image; private task_command; private mount_dir; private log_stdout; private log_stderr; private envs; /** * Construct a `CustomTaskBuilder` to build a custom task, which executes a command * given some input files as blob {@link Stream}s, and produces some output files as * blob streams. The custom task can run any linux-compatible binary program, which can * be provided as one of the input files. * * @param name the name of the custom task * * @category Task * * @example * ```typescript * // Create a custom task which uses a user-provided program to transform a JSON file into another JSON file * const my_task = new CustomTaskBuilder("MyTask") * .input("my_program", my_program_stream, true) * .input("input_data.json", input_data_stream, false, data => Utf8Encode(ToJson(data))) * .output("output_data.json", blob => FromJson(MyOutputType, Utf8Decode(blob))) * .shell("./my_program input_data.json output_data.json") * ``` */ constructor(name: string, module?: ModuleBuilder | ModulePath); /** @internal */ constructor(name: string, module: ModulePath, inputs: Record, outputs: Record, results: Record, inputFiles: { input: string; path: string; executable: boolean; toBlob: EastFunction; }[], outputFiles: { path: string; fromBlob: EastFunction; }[], image: string | null, task_command: string[] | null, mount_dir: string, logStdOut: boolean, logStdErr: boolean, envs: Record>); /** * Add an input to the custom task. A file with the provided `filename` is created from * the data stored in a blob {@link Stream} before the custom task command is launched. * * Note that the input could be a linux-compatible program to execute. * * @param filename the filename of the input * @param stream a blob stream contain the data to place in the file * @param executable set to `true` if the file is an executable program (default `false`) * @param toBlob an optional function to create an {@link Expression} to serialize the input data into a raw file (blob) * * @category Task * * @example * ```typescript * // Create a custom task which uses a user-provided program to transform a JSON file into another JSON file * const my_task = new CustomTaskBuilder("MyTask") * .input("my_program", my_program, true) * .input("input_data.json", input_data_stream, false, data => Utf8Encode(ToJson(data))) * .output("output_data.json", blob => FromJson(MyOutputType, Utf8Decode(blob))) * .shell("./my_program input_data.json output_data.json") * ``` */ input(filename: string, stream: Stream, executable?: boolean, toBlob?: (input: Variable) => EastFunction): CustomTaskBuilder; /** * Add an output to the custom task. A file with the provided `filename` is expected to * exist after the custom task command is completed. The output data is then copied to a * {@link Stream}. * * @param filename the filename of the output * @param fromBlob an optional function to create an {@link Expression} to deserialize (or parse) the output data from a raw file (blob) * @param result whether this file is written to a datastream (default `true`) * * @category Task * * @example * ```typescript * // Create a custom task which uses a user-provided program to transform a JSON file into another JSON file * const my_task = new CustomTaskBuilder("MyTask") * .input("my_program", my_program_stream, true) * .input("input_data.json", input_data_stream, false, data => Utf8Encode(ToJson(data))) * .output("output_data.json", blob => FromJson(MyOutputType, Utf8Decode(blob))) * .shell("./my_program input_data.json output_data.json") * ``` */ output(filename: Name, fromBlob?: undefined, result?: true): CustomTaskBuilder; }, Results & { [K in Name]: BlobType; }>; output(filename: Name, fromBlob: undefined, result: false): CustomTaskBuilder; }>; output) => EastFunction>(filename: Name, fromBlob: F, result?: true): CustomTaskBuilder["type"]>; }, Results & { [K in Name]: ReturnType["type"]; }>; output) => EastFunction>(filename: Name, fromBlob: F, result: false): CustomTaskBuilder["type"]>; }, Results>; /** * Add a result to the custom task which is computed from the output files. * * @param name the name of the computed result * @param result an function to create an {@link EastFunction} computing the desired result * * @category Task * * @example * ```typescript * // Create a custom task which uses a user-provided program to transform a JSON file into another JSON file, and get a field from the result * const my_task = new CustomTaskBuilder("MyTask") * .input("my_program", my_program_stream, true) * .input("input_data.json", input_data_stream, false, data => Utf8Encode(ToJson(data))) * .output("output_data.json", blob => FromJson(MyOutputType, Utf8Decode(blob))) * .result("field_a", outputs => GetField(outputs["output_data.json"], "a")) * .shell("./my_program input_data.json output_data.json") * ``` */ result; }) => EastFunction>(name: Name, result: F): CustomTaskBuilder["type"]; }>; /** * Provide the directory where the input and output data is located. * * This directory is a docker "volume mount" - if this directory already exists in the image, files inside will not be visible inside the container. If necessary, copy files into the data directory from elsewhere in the image via a {@link shell} command. * * Note: the directory is also set as the current working directory before the task command is executed. * * @param dir the directory to mount the data inside the docker container * * @category CustomScenario * * @example * ```typescript * // Optimize a custom task which uses a user-provided program to transform a JSON file into another JSON file * const my_scenario = new CustomScenarioBuilder("MyScearnio") * .dataDir("/my/custom/dir") * .input("my_program", my_program_stream, file => file, true) * .input("input.json", input_data_stream, data => Utf8Encode(ToJson(data))) * .output("output.json", blob => FromJson(MyOutputType, Utf8Decode(blob))) * .shell("./my_program input.json output.json") * .objective(outputs => outputs["output.json"]) * .optimizeEvery("input.json", "amount", { min: () => Const(0), max: () => Const(100) }) * ``` */ dataDir(dir: string): CustomTaskBuilder; /** * Provide the command used to execute the custom task in shell. This command will be executed * by the linux `bash` shell in a sandboxed environment. * * @param command the command to be executed * * @category Task * * @example * ```typescript * // Create a custom task which uses a user-provided program to transform a JSON file into another JSON file * const my_task = new CustomTaskBuilder("MyTask") * .input("my_program", my_program_stream, true) * .input("input_data.json", input_data_stream, false, data => Utf8Encode(ToJson(data))) * .output("output_data.json", blob => FromJson(MyOutputType, Utf8Decode(blob))) * .shell("./my_program input_data.json output_data.json") * ``` */ shell(command: string): CustomTaskBuilder; /** * Provide the command used to execute the custom task with raw arguemnts. This command * will be executed in a sandbox. As the command is not be interpretted by any shell, * it is not necessary to quote the arguments (e.g. filenames containing spaces, etc). * * @param command the filename of the program to be executed * @param args the command-line arguments given to the program * * @category Task * * @example * ```typescript * // Create a custom task which uses a user-provided program to transform a JSON file into another JSON file * const my_task = new CustomTaskBuilder("MyTask") * .input("my_program", my_program_stream, true) * .input("input data.json", input_data_stream, false, data => Utf8Encode(ToJson(data))) * .output("output data.json", blob => FromJson(MyOutputType, Utf8Decode(blob))) * .exec("./my_program", "input data.json", "output data.json") * ``` */ exec(command: string, ...args: string[]): CustomTaskBuilder; /** * Provide a user-defined docker image to execute the custom task. The docker image is * provided via a blob {@link Stream}, containing an OCI container in a .tar, .tar.gz, * .tar.bz2 file format as generated by docker via `docker image save ...` or similar methods. * * If no docker image is provided, then an Ubuntu LTS image is used by default. If no command * is provided the image's default command is used. * * @param stream the stream containing the docker image * * @category Task * * @example * ```typescript * // Create a custom task which uses a user-provided docker image to transform a JSON file into another JSON file * const my_task = new CustomTaskBuilder("MyTask") * .dockerImage(my_docker_image_stream) * .input("input_data.json", input_data_stream, false, data => Utf8Encode(ToJson(data))) * .output("output_data.json", blob => FromJson(MyOutputType, Utf8Decode(blob))) * ``` */ dockerImage(stream: Stream): CustomTaskBuilder; /** * Set an environment variable for the custom task * * @param name the name of the environment variable * @param value the value of the environment variable * * @category Task * * @example * ```typescript * // Create a custom task which uses a user-provided program to transform a JSON file into another JSON file * const my_task = new CustomTaskBuilder("MyTask") * .input("my_program", my_program_stream, true) * .input("input_data.json", input_data_stream, false, data => Utf8Encode(ToJson(data))) * .output("output_data.json", blob => FromJson(MyOutputType, Utf8Decode(blob))) * .env("MY_PROGRAM_SETTING", "TRUE") * .shell("./my_program input_data.json output_data.json") * ``` */ env(name: string, value: string): CustomTaskBuilder; /** * Enable or disable logging of stdout from the custom task. * * @param enabled whether logging is enabled (default `true`) * * @category Task * * @example * ```typescript * // Create a custom task which uses a user-provided program to transform a JSON file into another JSON file * const my_task = new CustomTaskBuilder("MyTask") * .input("my_program", my_program_stream, true) * .input("input_data.json", input_data_stream, false, data => Utf8Encode(ToJson(data))) * .output("output_data.json", blob => FromJson(MyOutputType, Utf8Decode(blob))) * .shell("./my_program input_data.json output_data.json") * .logStdOut(false) * ``` */ logStdOut(enabled: boolean): CustomTaskBuilder; /** * Enable or disable logging of stderr from the custom task. * * @param enabled whether logging is enabled (default `true`) * * @category Task * * @example * ```typescript * // Create a custom task which uses a user-provided program to transform a JSON file into another JSON file * const my_task = new CustomTaskBuilder("MyTask") * .input("my_program", my_program_stream, true) * .input("input_data.json", input_data_stream, false, data => Utf8Encode(ToJson(data))) * .output("output_data.json", blob => FromJson(MyOutputType, Utf8Decode(blob))) * .shell("./my_program input_data.json output_data.json") * .logStdErr(false) * ``` */ logStdErr(enabled: boolean): CustomTaskBuilder; /** * Fetch the output streams of the custom task. This returns a record of data streams, * where the keys are the expected filenames. * * @category Task * * @example * ```typescript * // Create a custom task which uses a user-provided program to transform a JSON file into another JSON file * const my_task = new CustomTaskBuilder("MyTask") * .input("my_program", my_program_stream, true) * .input("input_data.json", input_data_stream, false, data => Utf8Encode(ToJson(data))) * .output("output_data.json", blob => FromJson(MyOutputType, Utf8Decode(blob))) * .shell("./my_program input_data.json output_data.json") * * // Get the decoded output data stream * const output_data = my_task.outputStreams()["output_data.json"] * ``` */ outputStreams(): { [K in keyof Results]: Stream; }; /** * Return a {@link Template} fragment containing the {@link Stream}s and {@link TaskDescription}s built by this builder. * * @category Task * * @example * ```typescript * // Create a custom task which uses a user-provided program to transform a JSON file into another JSON file * export default new CustomTaskBuilder("MyTask") * .input("my_program", my_program_stream, true) * .input("input_data.json", input_data_stream, false, data => Utf8Encode(ToJson(data))) * .output("output_data.json", blob => FromJson(MyOutputType, Utf8Decode(blob))) * .shell("./my_program input_data.json output_data.json") * .toTemplate() * ``` */ toTemplate(): { streams: Record; tasks: { [x: string]: { task_type: "custom"; module: ModulePath; name: string; inputs: Record>; outputs: Record>; results: Record; inputFiles: { input: string; path: string; executable: boolean; toBlob: EastFunction; }[]; outputFiles: { path: string; fromBlob: EastFunction; }[]; image: string | null; command: string[] | null; mountDir: string; logStdOut: boolean; logStdErr: boolean; envs: Record>; }; }; }; }