/** * MIT License * * Copyright (c) 2020-present, Elastic NV * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * */ import { join } from 'path'; import { mkdir, rm, writeFile } from 'fs/promises'; import { Journey } from '../dsl/journey'; import { Step } from '../dsl/step'; import { reporters, Reporter } from '../reporters'; import { CACHE_PATH, monotonicTimeInSeconds, getTimestamp, runParallel, generateUniqueId, } from '../helpers'; import { HooksCallback, HooksArgs, Driver, Screenshot, RunOptions, JourneyResult, StepResult, PushOptions, } from '../common_types'; import { PerformanceManager, filterBrowserMessages } from '../plugins'; import { Gatherer } from './gatherer'; import { log } from './logger'; import { Monitor, MonitorConfig } from '../dsl/monitor'; import { parseSpaces } from '../push/monitor'; type HookType = 'beforeAll' | 'afterAll'; export type SuiteHooks = Record>; type RunResult = Record; export interface RunnerInfo { /** * Processed configuration from the CLI args and the config file */ readonly config: RunOptions; /** * Currently active journey */ readonly currentJourney: Journey | undefined; /** * All registerd journeys */ readonly journeys: Journey[]; } export default class Runner implements RunnerInfo { #active = false; #reporter: Reporter; #currentJourney?: Journey = null; #journeys: Journey[] = []; #hooks: SuiteHooks = { beforeAll: [], afterAll: [] }; #screenshotPath = join(CACHE_PATH, 'screenshots'); #driver?: Driver; #browserDelay = -1; #hookError: Error | undefined; #monitor?: Monitor; config: RunOptions; get currentJourney() { return this.#currentJourney; } get journeys() { return this.#journeys; } get hooks() { return this.#hooks; } private async captureScreenshot(page: Driver['page'], step: Step) { try { const buffer = await page.screenshot({ type: 'jpeg', quality: 80, timeout: 5000, }); /** * Write the screenshot image buffer with additional details (step * information) which could be extracted at the end of * each journey without impacting the step timing information */ const fileName = `${generateUniqueId()}.json`; const screenshot: Screenshot = { step, timestamp: getTimestamp(), data: buffer.toString('base64'), }; await writeFile( join(this.#screenshotPath, fileName), JSON.stringify(screenshot) ); log(`Runner: captured screenshot for (${step.name})`); } catch (_) { // Screenshot may fail sometimes, log and continue. log(`Runner: failed to capture screenshot for (${step.name})`); } } _addHook(type: HookType, callback: HooksCallback) { this.#hooks[type].push(callback); } /** * @deprecated Since version Please do not rely on the internal methods. * Alias _addHook for backwards compatibility */ addHook(type: HookType, callback: HooksCallback) { this._addHook(type, callback); } private buildHookArgs() { return { env: this.config.environment, params: this.config.params, info: this as RunnerInfo, }; } _updateMonitor(config: MonitorConfig) { if (!this.#monitor) { this.#monitor = new Monitor(config); return; } this.#monitor.update(config); } /** * @deprecated Since version Please do not rely on the internal methods. * Alias _addJourney for backwards compatibility */ updateMonitor(config: MonitorConfig) { this._updateMonitor(config); } _addJourney(journey: Journey) { this.#journeys.push(journey); this.#currentJourney = journey; } /** * @deprecated Since version 1.17.0. Please do not rely on the internal methods. * Alias _addJourney for backwards compatibility */ addJourney(journey: Journey) { this._addJourney(journey); } private setReporter(options: RunOptions) { /** * Set up the corresponding reporter and fallback * to default reporter if not provided */ const { reporter, outfd, dryRun } = options; const Reporter = typeof reporter === 'function' ? reporter : reporters[reporter] || reporters['default']; this.#reporter = new Reporter({ fd: outfd, dryRun }); } async #runBeforeAllHook(args: HooksArgs) { log(`Runner: beforeAll hooks`); await runParallel(this.#hooks.beforeAll, args); } async #runAfterAllHook(args: HooksArgs) { log(`Runner: afterAll hooks`); await runParallel(this.#hooks.afterAll, args); } async #runBeforeHook(journey: Journey, args: HooksArgs) { log(`Runner: before hooks for (${journey.name})`); await runParallel(journey._getHook('before'), args); } async #runAfterHook(journey: Journey, args: HooksArgs) { log(`Runner: after hooks for (${journey.name})`); await runParallel(journey._getHook('after'), args); } async #runStep(step: Step, options: RunOptions): Promise { log(`Runner: start step (${step.name})`); const { metrics, screenshots, filmstrips, trace } = options; /** * URL needs to be the first navigation request of any step * Listening for request solves the case where `about:blank` would be * reported for failed navigations */ const captureUrl = req => { if (!step.url && req.isNavigationRequest()) { step.url = req.url(); } this.#driver.context.off('request', captureUrl); }; this.#driver.context.on('request', captureUrl); const data: StepResult = {}; const traceEnabled = trace || filmstrips; try { /** * Set up plugin manager context and also register * step level plugins */ Gatherer.pluginManager.onStep(step); traceEnabled && (await Gatherer.pluginManager.start('trace')); // invoke the step callback by extracting to a variable to get better stack trace const cb = step.cb; await cb(); step.status = 'succeeded'; } catch (error) { step.status = 'failed'; step.error = error; } finally { /** * Collect all step level metrics and trace events */ if (metrics) { data.pagemetrics = await ( Gatherer.pluginManager.get('performance') as PerformanceManager ).getMetrics(); } if (traceEnabled) { const traceOutput = await Gatherer.pluginManager.stop('trace'); Object.assign(data, traceOutput); } /** * Capture screenshot for the newly created pages * via popup or new windows/tabs * * Last open page will get us the correct screenshot */ const pages = this.#driver.context.pages(); const page = pages[pages.length - 1]; if (page) { step.url ??= page.url(); if (screenshots && screenshots !== 'off') { await this.captureScreenshot(page, step); } } } log(`Runner: end step (${step.name})`); return data; } async #runSteps(journey: Journey, options: RunOptions) { const results: Array = []; const isOnlyExists = journey.steps.filter(s => s.only).length > 0; let skipStep = false; for (const step of journey.steps) { step._startTime = monotonicTimeInSeconds(); this.#reporter?.onStepStart?.(journey, step); let data: StepResult = {}; /** * Skip the step * - if the step is marked as skip * - if the previous step fails and the current step is not marked as soft * - if the step is not marked as only and there are steps marked as only */ if ( step.skip || (skipStep && !step.only) || (isOnlyExists && !step.only) ) { step.status = 'skipped'; } else { data = await this.#runStep(step, options); /** * skip next steps if the previous step returns error */ if (step.error && !step.soft) skipStep = true; } step.duration = monotonicTimeInSeconds() - step._startTime; this.#reporter?.onStepEnd?.(journey, step, data); if (options.pauseOnError && step.error) { await new Promise(r => process.stdin.on('data', r)); } results.push(data); } return results; } async #startJourney(journey: Journey, options: RunOptions) { journey._startTime = monotonicTimeInSeconds(); this.#driver = await Gatherer.setupDriver(options); await Gatherer.beginRecording(this.#driver, options); /** * For each journey we create the screenshots folder for * caching all screenshots and clear them at end of each journey */ await mkdir(this.#screenshotPath, { recursive: true }); const params = options.params; this.#reporter?.onJourneyStart?.(journey, { timestamp: getTimestamp(), params, }); /** * Exeucute the journey callback which registers the steps for current journey */ journey.cb({ ...this.#driver, params, info: this }); } async #endJourney( journey: Journey, result: JourneyResult, options: RunOptions ) { // Enhance the journey results const pOutput = await Gatherer.pluginManager.output(); const bConsole = filterBrowserMessages( pOutput.browserconsole, journey.status ); await this.#reporter?.onJourneyEnd?.(journey, { browserDelay: this.#browserDelay, timestamp: getTimestamp(), options, networkinfo: pOutput.networkinfo, browserconsole: bConsole, }); await Gatherer.endRecording(); await Gatherer.dispose(this.#driver); // clear screenshots cache after each journey await rm(this.#screenshotPath, { recursive: true, force: true }); return Object.assign(result, { networkinfo: pOutput.networkinfo, browserconsole: bConsole, ...journey, }); } /** * Simulate a journey run to capture errors in the beforeAll hook */ async #runFakeJourney(journey: Journey, options: RunOptions) { const start = monotonicTimeInSeconds(); this.#reporter.onJourneyStart?.(journey, { timestamp: getTimestamp(), params: options.params, }); // Mark the journey as failed and report the hook error as journey error journey.status = 'failed'; journey.error = this.#hookError; journey.duration = monotonicTimeInSeconds() - start; await this.#reporter.onJourneyEnd?.(journey, { timestamp: getTimestamp(), options, browserDelay: this.#browserDelay, }); return journey; } async _runJourney(journey: Journey, options: RunOptions) { this.#currentJourney = journey; log(`Runner: start journey (${journey.name})`); let result: JourneyResult = {}; const hookArgs = { env: options.environment, params: options.params, info: this, }; try { await this.#startJourney(journey, options); await this.#runBeforeHook(journey, hookArgs); const stepResults = await this.#runSteps(journey, options); journey.status = 'succeeded'; // Mark journey as failed if any one of the step fails for (const step of journey.steps) { if (step.status === 'failed') { journey.status = step.status; journey.error = step.error; } } result.stepsresults = stepResults; } catch (e) { journey.status = 'failed'; journey.error = e; } finally { journey.duration = monotonicTimeInSeconds() - journey._startTime; // Run after hook on journey failure and capture the uncaught error as // journey error, hook is purposely run before to capture errors during reporting await this.#runAfterHook(journey, hookArgs).catch(e => { journey.status = 'failed'; journey.error = e; }); result = await this.#endJourney(journey, result, options); } log(`Runner: end journey (${journey.name})`); return result; } /** * @deprecated Since version 1.17.0. Please do not rely on the internal methods. * Alias _runJourney for backwards compatibility */ runJourney(journey: Journey, options: RunOptions) { this._runJourney(journey, options); } _buildMonitors(options: PushOptions) { /** * Update the global monitor configuration required for setting defaults */ this._updateMonitor({ throttling: options.throttling, schedule: options.schedule, locations: options.locations, privateLocations: options.privateLocations, params: options.params, playwrightOptions: options.playwrightOptions, screenshot: options.screenshots, tags: options.tags, alert: options.alert, retestOnFailure: options.retestOnFailure, enabled: options.enabled, fields: options.fields, spaces: Array.from(new Set([...(options.spaces ?? []), options.space])), namespace: options.namespace, maintenanceWindows: options.maintenanceWindows, }); const monitors: Monitor[] = []; for (const journey of this.#journeys) { this.#currentJourney = journey; if (journey.skip) { throw new Error( `Journey ${journey.name} is skipped. Please remove the journey.skip annotation and try again.` ); } /** * Before pushing a browser monitor, three things need to be done: * * - execute callback `monitor.use` in particular to get monitor configurations * - update the monitor config with global configuration * - filter out monitors based on matched tags and name after applying both * global and local monitor configurations */ // TODO: Fix backwards compatibility with 1.14 and prior (journey.cb ?? journey.callback)({ params: options.params } as any); const monitor = journey.monitor ?? journey?._getMonitor(); monitor.update({ ...this.#monitor?.config, ...parseSpaces(monitor.config, options), }); if (!monitor.isMatch(options.grepOpts?.match, options.grepOpts?.tags)) { continue; } monitor.validate(); monitors.push(monitor); } return monitors; } /** * @deprecated Since version 1.17.0. Please do not rely on the internal methods. * Alias _buildMonitors for backwards compatibility */ buildMonitors(options: PushOptions) { this._buildMonitors(options); } async #init(options: RunOptions) { this.setReporter(options); this.#reporter.onStart?.({ numJourneys: this.#journeys.length, networkConditions: options.networkConditions, }); /** * Set up the directory for caching screenshots */ await mkdir(CACHE_PATH, { recursive: true }); } async _run(options: RunOptions): Promise { this.config = options; let result: RunResult = {}; if (this.#active) { return result; } this.#active = true; log(`Runner: run ${this.#journeys.length} journeys`); this.#init(options); const hookArgs = this.buildHookArgs(); await this.#runBeforeAllHook(hookArgs).catch(e => (this.#hookError = e)); const { dryRun, grepOpts } = options; // collect all journeys with `.only` annotation and skip the rest const onlyJournerys = this.#journeys.filter(j => j.only); if (onlyJournerys.length > 0) { this.#journeys = onlyJournerys; } else { // filter journeys based on tags and skip annotations this.#journeys = this.#journeys.filter( j => j._isMatch(grepOpts?.match, grepOpts?.tags) && !j.skip ); } // Used by heartbeat to gather all registered journeys if (dryRun) { this.#journeys.forEach(journey => this.#reporter.onJourneyRegister?.(journey) ); } else if (this.#journeys.length > 0) { result = await this._runJourneys(options); } await this.#runAfterAllHook(hookArgs).catch( async () => await this._reset() ); await this._reset(); return result; } /** * @deprecated Since version 1.17.0. Please do not rely on the internal methods. * Alias _run for backwards compatibility */ run(options: RunOptions): Promise { return this._run(options); } async _runJourneys(options: RunOptions) { const result: RunResult = {}; const browserStart = monotonicTimeInSeconds(); await Gatherer.launchBrowser(options); this.#browserDelay = monotonicTimeInSeconds() - browserStart; for (const journey of this.#journeys) { const journeyResult: JourneyResult = this.#hookError ? await this.#runFakeJourney(journey, options) : await this._runJourney(journey, options); result[journey.name] = journeyResult; } await Gatherer.stop(); return result; } /** * @deprecated Since version 1.17.0. Please do not rely on the internal methods. * Alias _runJourneys for backwards compatibility */ runJourneys(options: RunOptions): Promise { return this._runJourneys(options); } async _reset() { this.#currentJourney = null; this.#journeys = []; this.#active = false; /** * Clear all cache data stored for post processing by * the current synthetic agent run */ await rm(CACHE_PATH, { recursive: true, force: true }); await this.#reporter?.onEnd?.(); } /** * @deprecated Since version 1.17.0. Please do not rely on the internal methods. * Alias _reset for backwards compatibility */ reset() { return this._reset(); } }