/** * MIT License * * Copyright (c) 2020-present, Elastic NV * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * */ import { mkdir, rm, readFile } from 'fs/promises'; import { extname, join } from 'path'; import { LineCounter, parseDocument, Document, YAMLSeq, YAMLMap } from 'yaml'; import { bold, red } from 'kleur/colors'; import { Bundler } from './bundler'; import NodeBuffer from 'node:buffer'; import { SYNTHETICS_PATH, totalist, indent, warn } from '../helpers'; import { LocationsMap } from '../locations/public-locations'; import { AlertConfig, ALLOWED_SCHEDULES, Monitor, MonitorConfig, } from '../dsl/monitor'; import { PushOptions } from '../common_types'; import { isParamOptionSupported, normalizeMonitorName } from './utils'; // Allowed extensions for lightweight monitor files const ALLOWED_LW_EXTENSIONS = ['.yml', '.yaml']; // 1500kB Max Gzipped limit for bundled monitor code to be pushed as Kibana project monitors. const SIZE_LIMIT_KB = 1500; export type MonitorSchema = Omit & { locations: string[]; content?: string; filter?: Monitor['filter']; hash?: string; size?: number; }; // Abbreviated monitor info, as often returned by the API, // just the journey ID and hash export type MonitorHashID = { journey_id?: string; hash?: string; }; function translateLocation(locations?: MonitorConfig['locations']) { if (!locations) return []; return locations.map(loc => LocationsMap[loc] || loc).filter(Boolean); } class RemoteDiffResult { // The set of monitor IDs that have been added newIDs = new Set(); // Monitor IDs that are different locally than remotely changedIDs = new Set(); // Monitor IDs that are no longer present locally removedIDs = new Set(); // Monitor IDs that are identical on the remote server unchangedIDs = new Set(); } export function diffMonitors( local: MonitorHashID[], remote: MonitorHashID[] ): RemoteDiffResult { const result = new RemoteDiffResult(); const localMonitorsIDToHash = new Map(); for (const hashID of local) { localMonitorsIDToHash.set(hashID.journey_id, hashID.hash); } const remoteMonitorsIDToHash = new Map(); for (const hashID of remote) { remoteMonitorsIDToHash.set(hashID.journey_id, hashID.hash); } // Compare local to remote for (const [localID, localHash] of localMonitorsIDToHash) { // Hash is reset to '' when a monitor is edited on the UI if (!remoteMonitorsIDToHash.has(localID)) { result.newIDs.add(localID); } else { const remoteHash = remoteMonitorsIDToHash.get(localID); if (remoteHash != localHash) { result.changedIDs.add(localID); } else if (remoteHash === localHash) { result.unchangedIDs.add(localID); } } // We no longer need to process this ID, removing it here // reduces the numbers considered in the next phase remoteMonitorsIDToHash.delete(localID); } for (const [id] of remoteMonitorsIDToHash) { result.removedIDs.add(id); } return result; } export function getLocalMonitors(schemas: MonitorSchema[]) { const data: MonitorHashID[] = []; for (const schema of schemas) { data.push({ journey_id: schema.id, hash: schema.hash, }); } return data; } export async function buildMonitorSchema(monitors: Monitor[], isV2: boolean) { /** * Set up the bundle artifacts path which can be used to * create the bundles required for uploading journeys */ const bundlePath = join(SYNTHETICS_PATH, 'bundles'); await mkdir(bundlePath, { recursive: true }); const bundler = new Bundler(); const schemas: MonitorSchema[] = []; const sizes: Map = new Map(); for (const monitor of monitors) { const { source, config, filter, type } = monitor; const schema: MonitorSchema = { ...config, locations: translateLocation(config.locations), }; if (type === 'browser') { const outPath = join( bundlePath, normalizeMonitorName(config.name) + '.zip' ); const content = await bundler.build(source.file, outPath); monitor.setContent(content); Object.assign(schema, { content, filter }); } const size = monitor.size(); const sizeKB = Math.round(size / 1000); if (sizeKB > SIZE_LIMIT_KB) { let outer = bold( `Aborted: Bundled code ${sizeKB}kB exceeds the recommended ${SIZE_LIMIT_KB}kB limit. Please check the dependencies imported.\n` ); const inner = `* ${config.id} - ${source.file}:${source.line}:${source.column}\n`; outer += indent(inner); throw red(outer); } sizes.set(config.id, size); /** * Generate hash only after the bundled content is created * to capture code changes in imported files */ if (isV2) { schema.hash = monitor.hash(); } schemas.push(schema); } await rm(bundlePath, { recursive: true }); return { schemas, sizes }; } export async function createLightweightMonitors( workDir: string, options: PushOptions ) { const lwFiles = new Set(); // Filter monitor files based on the provided pattern const pattern = options.grepOpts?.pattern ? new RegExp(options.grepOpts?.pattern, 'i') : /.(yml|yaml)$/; const ignore = /(node_modules|.github)/; await totalist(workDir, (rel, abs) => { if ( !ignore.test(rel) && pattern.test(rel) && ALLOWED_LW_EXTENSIONS.includes(extname(abs)) ) { lwFiles.add(abs); } }); let warnOnce = false; const monitors: Monitor[] = []; for (const file of lwFiles.values()) { // First check encoding and warn if any files are not the correct encoding. const bufferContent = await readFile(file); const isUtf8 = NodeBuffer.isUtf8(bufferContent); if (!isUtf8) { warn(`${file} is not UTF-8 encoded. Monitors might be skipped.`); } const content = bufferContent.toString('utf-8'); const lineCounter = new LineCounter(); const parsedDoc = parseDocument(content, { lineCounter, merge: true, keepSourceTokens: true, }) as Document.Parsed; // Skip other yml files that are not relevant const monitorSeq = parsedDoc.get('heartbeat.monitors') as YAMLSeq; if (!monitorSeq) { continue; } // Warn users about schedule that are less than 60 seconds if (!warnOnce) { warn( 'Lightweight monitor schedules will be adjusted to their nearest frequency supported by our synthetics infrastructure.' ); warnOnce = true; } // Store the offsets of each monitor in the sequence to construct the source // location later for capturing the error const offsets = []; for (const monNode of monitorSeq.items) { offsets.push(monNode.srcToken.offset); } const mergedConfig = parsedDoc.toJS()[ 'heartbeat.monitors' ] as Array; for (let i = 0; i < mergedConfig.length; i++) { const monitor = mergedConfig[i]; // Skip browser monitors from the YML files if (monitor['type'] === 'browser') { warn(`Browser monitors from ${file} are skipped.`); continue; } const { line, col } = lineCounter.linePos(offsets[i]); try { /** * Build the monitor object from the yaml config along with global configuration * and perform the match based on the provided filters */ const mon = buildMonitorFromYaml(monitor, options); if (!mon.isMatch(options.grepOpts?.match, options.grepOpts?.tags)) { continue; } mon.setSource({ file, line, column: col }); monitors.push(mon); } catch (e) { let outer = bold(`Aborted: ${e}\n`); outer += indent( `* ${monitor.id || monitor.name} - ${file}:${line}:${col}\n` ); throw red(outer); } } } return monitors; } const REQUIRED_MONITOR_FIELDS = ['id', 'name']; export function buildMonitorFromYaml( config: MonitorConfig, options: PushOptions ) { // Validate required fields for (const field of REQUIRED_MONITOR_FIELDS) { if (!config[field]) { throw `Monitor ${field} is required`; } } const schedule = config.schedule && parseSchedule(String(config.schedule)); const privateLocations = config['private_locations'] || config.privateLocations || options.privateLocations; const retestOnFailure = config['retest_on_failure'] ?? options.retestOnFailure; const maintenanceWindows = (config['maintenance_windows'] || config.maintenanceWindows) ?? options.maintenanceWindows; const alertConfig = parseAlertConfig(config, options.alert); const mon = new Monitor({ namespace: config.namespace ?? options.namespace, enabled: config.enabled ?? options.enabled, locations: options.locations, tags: options.tags, fields: parseFields(config, options.fields), ...normalizeConfig(config), ...parseSpaces(config, options), retestOnFailure, privateLocations, schedule: (schedule as typeof ALLOWED_SCHEDULES[number]) || options.schedule, alert: alertConfig, maintenanceWindows, }); /** * Params support is only available for lightweight monitors * post 8.7.2 stack */ if (isParamOptionSupported(options.kibanaVersion)) { mon.config.params = options.params; } return mon; } // Deletes unnecessary fields from the lightweight monitor config // that is not supported by the Kibana API function normalizeConfig(config: MonitorConfig) { delete config['private_locations']; delete config['retest_on_failure']; delete config['maintenance_windows']; return config; } export const parseAlertConfig = ( config: MonitorConfig, gConfig?: AlertConfig ) => { // If the user has provided a global alert config, merge it with the monitor alert config const status = getAlertKeyValue('status', config, gConfig); const tls = getAlertKeyValue('tls', config, gConfig); const result = {}; if (status) { result['status'] = status; } if (tls) { result['tls'] = tls; } return Object.keys(result).length > 0 ? result : undefined; }; export const parseSpaces = (config: MonitorConfig, options: PushOptions) => { const baseSpaces = [...(config.spaces ?? []), ...(options.spaces ?? [])]; if (!baseSpaces.length) { // If no spaces are defined, we return an empty object return {}; } // If the user has provided a global space, merge it with the monitor spaces const spaces = Array.from( options.space ? new Set([options.space, ...baseSpaces]) : new Set(baseSpaces) ); if (spaces.includes('*')) { // If the user has provided a wildcard space, we should not override it with the global space return { spaces: ['*'] }; } return { spaces }; }; export const parseFields = ( config: MonitorConfig, gFields?: Record ) => { // get all keys starting with `label.` const keys = Object.keys(config).filter(key => key.startsWith('fields.')); const fields = {}; for (const key of keys) { fields[key.replace('fields.', '')] = config[key]; delete config[key]; } if (gFields) { for (const key of Object.keys(gFields)) { fields[key] = gFields[key]; } } return Object.keys(fields).length > 0 ? fields : undefined; }; export function getAlertKeyValue( key: 'status' | 'tls', config: MonitorConfig, alertConfig?: AlertConfig ): { enabled: boolean } { const value = config.alert; if (value?.[key]?.enabled !== undefined) { return { enabled: value[key].enabled, }; } if (value?.[`${key}.enabled`] !== undefined) { const val = value?.[`${key}.enabled`]; delete value?.[`${key}.enabled`]; if (Object.keys(value).length === 0) { delete config.alert; } return { enabled: val, }; } const rootKey = `alert.${key}.enabled`; if (config[rootKey] !== undefined) { const enabled = config[rootKey]; delete config[rootKey]; return { enabled, }; } return alertConfig?.[key]; } export function parseSchedule(schedule: string) { const EVERY_SYNTAX = '@every'; if (!(schedule + '').startsWith(EVERY_SYNTAX)) { throw `Monitor schedule format(${schedule}) not supported: use '@every' syntax instead`; } const duration = schedule.substring(EVERY_SYNTAX.length + 1); // split between non-digit (\D) and a digit (\d) const durations = duration.split(/(?<=\D)(?=\d)/g); let minutes = 0; let seconds = 0; for (const dur of durations) { // split between a digit and non-digit const [value, format] = dur.split(/(?<=\d)(?=\D)/g); // Calculate based on the duration symbol const scheduleValue = parseInt(value, 10); switch (format) { case 's': if (scheduleValue < 60) { seconds += scheduleValue; } else { minutes += Math.round(scheduleValue / 60); } break; case 'm': minutes += scheduleValue; break; case 'h': minutes += scheduleValue * 60; break; case 'd': minutes += scheduleValue * 24 * 60; break; } } return nearestSchedule(minutes, seconds); } // Find the nearest schedule that is supported by the platform // from the parsed schedule value function nearestSchedule(minutes: number, seconds: number) { if (seconds > 0 && minutes === 0) { // we allow only 10 and 30 seconds, return the nearest one return seconds < 20 ? '10s' : '30s'; } let nearest: typeof ALLOWED_SCHEDULES[number] = ALLOWED_SCHEDULES[0]; let prev = Math.abs(nearest - minutes); for (let i = 1; i < ALLOWED_SCHEDULES.length; i++) { const curr = Math.abs(ALLOWED_SCHEDULES[i] - minutes); if (curr <= prev) { nearest = ALLOWED_SCHEDULES[i]; prev = curr; } } return nearest; }