import { AUTOMATION_WEB_SCRAPE_ENTRYPOINT_PREFIX, platformWebScrapeInputSchema, platformWebScrapeStartSchema, webScrapeCompletionSchema, webScrapeInputSchema, } from "@automate.ax/api-contract/runtime" import { automationInvokedTriggerDefinition } from "@automate.ax/catalog/triggers/core-invocation" import * as z from "zod" import { defineAction, type ActionObjectInput } from "../../automation/actions" import { getCurrentHookScopePath } from "../../automation/runtime" import { correlate, getCurrentContextPrerequisite, getCurrentSignalPrerequisites, group, scope, withoutSignalPrerequisites, withContextPrerequisite, withPrerequisites, } from "../../automation/signal-operators" import { FailedSignalError, transform, type Signal, } from "../../automation/signal-protocol" import { createSubscription } from "../../automation/subscription" const startPlatformWebScrapeAction = defineAction("Scrape web page") .input(platformWebScrapeInputSchema) .output(platformWebScrapeStartSchema) .retry({ replaySafety: "safe" }) .handler(async ({ input, runtime }) => await runtime.startWebScrape(input)) type WebScrapeActionInput = ActionObjectInput /** Browser controls accepted by the built-in web-scraping action. */ export type ScrapeWebOptions = Omit< WebScrapeActionInput, "cacheTtl" | "html" | "url" > & ( | { html: Exclude url?: never } | { html?: never url: Exclude } ) /** * Renders one URL or HTML document and returns its content as Markdown. * * The platform supplies the browser account and counts actual browser time * against the organization's monthly web-scraping allowance. * * @param options - Page source, request controls, browser settings, and waits. */ export function scrapeWeb(options: ScrapeWebOptions): Signal { const prerequisites = getCurrentSignalPrerequisites() const contextPrerequisite = getCurrentContextPrerequisite() return group({ name: "Scrape web page", presentation: "hidden" }, () => withoutSignalPrerequisites(() => { const entrypoint = `${AUTOMATION_WEB_SCRAPE_ENTRYPOINT_PREFIX}${getCurrentHookScopePath().join(".")}` const completed = createSubscription< z.output >(automationInvokedTriggerDefinition, { entrypoint }, undefined, { inferActionBoundary: false, }) const start = () => startPlatformWebScrapeAction({ ...options, entrypoint, }) const startWithContext = () => contextPrerequisite ? withContextPrerequisite(contextPrerequisite, start) : start() return transform( [ correlate([ (prerequisites ? scope(() => withPrerequisites(prerequisites, startWithContext)) : startWithContext() ).keyBy(({ key }) => key), completed.keyBy(({ key }) => key), ]), completed, ], (_matched, { outcome }) => { if (outcome.status === "failed") { throw new FailedSignalError(outcome.failure) } return outcome.result }, ) }), ) }