import * as _ from 'lodash'; import * as path from 'path'; import config from '@uifabric/build/config'; // TODO: check false positive potential regression reports in fluent ui repo and fix type Reporter = { markdown: (markdown: string) => void; warn: (message: string) => void; }; export function getFluentPerfRegressions() { const output: string[] = []; const markdown = (text: string) => { output.push(text); }; const warn = (text: string) => { output.push(`:warning: ${text}\n`); }; checkPerfRegressions({ markdown, warn }); return output.join('\n'); } function linkToFlamegraph(value: string, filename: string) { const urlForDeployPath = process.env.BUILD_SOURCEBRANCH ? `http://fabricweb.z5.web.core.windows.net/pr-deploy-site/${process.env.BUILD_SOURCEBRANCH}/perf-test/fluentui` : 'file://' + config.paths.packageDist('perf-test'); return `[${value}](${urlForDeployPath}/${path.basename(filename)})`; } function fluentFabricComparison(perfCounts: any, reporter: Reporter) { const results = _.mapValues( _.pickBy(perfCounts, (value, key) => key.endsWith('.Fluent')), stats => { const fluentTpi = _.get(stats, 'extended.tpi'); const fabricTpi = _.get(stats, 'extended.fabricTpi'); return { numTicks: stats.analysis.numTicks, iterations: stats.extended.iterations, fluentTpi, fabricTpi, fluentToFabric: Math.round((fluentTpi / fabricTpi) * 100) / 100, fluentFlamegraphFile: _.get(stats, 'processed.output.flamegraphFile'), }; }, ); const getStatus: (arg: number) => string = fluentToFabric => fluentToFabric > 1 ? '🔧' : fluentToFabric >= 0.7 ? '🎯' : '🦄'; reporter.markdown( [ '
Perf comparison', '', 'Status | Scenario | Fluent TPI | Fabric TPI | Ratio | Iterations | Ticks', ':---: | :--- | ---:| ---:| ---:| ---:| ---:', ..._.map(results, (value, key) => [ getStatus(value.fluentToFabric), key, linkToFlamegraph(value.fluentTpi, value.fluentFlamegraphFile), value.fabricTpi, `${value.fluentToFabric}:1`, value.iterations, value.numTicks, ].join(' | '), ), '>🔧 Needs work     🎯 On target     🦄 Amazing', '', '
', ].join('\n'), ); } function reportResults(perfCounts: any, reporter: Reporter) { const results = _.map( _.pickBy(perfCounts, value => _.has(value, 'analysis.regression')), (stats, name) => { const currentTicks = _.get(stats, 'analysis.numTicks'); const baselineTicks = _.get(stats, 'analysis.baseline.numTicks'); return { name, numTicks: currentTicks, flamegraphFile: _.get(stats, 'processed.output.flamegraphFile'), baseline: { numTicks: baselineTicks, flamegraphFile: _.get(stats, 'processed.baseline.output.flamegraphFile'), }, isRegression: _.get(stats, 'analysis.regression.isRegression'), regressionFile: _.get(stats, 'analysis.regression.regressionFile'), currentToBaseline: Math.round((currentTicks / baselineTicks) * 100) / 100, }; }, ); const regressions = _.sortBy(_.filter(results, 'isRegression'), stats => stats.currentToBaseline * -1); if (regressions.length > 0) { reporter.warn(`${regressions.length} potential perf regressions detected`); reporter.markdown( [ '### Potential regressions comparing to master', '', 'Scenario | Current PR Ticks | Baseline Ticks | Ratio | Regression Analysis', ':--- | ---:| ---:| ---: | ---: ', ..._.map(regressions, (value, key) => [ value.name, linkToFlamegraph(value.numTicks, value.flamegraphFile), linkToFlamegraph(value.baseline.numTicks, value.baseline.flamegraphFile), `${value.currentToBaseline}:1`, linkToFlamegraph('analysis', value.regressionFile), ].join(' | '), ), ].join('\n'), ); } fluentFabricComparison(perfCounts, reporter); const noRegressions = _.sortBy( _.filter(results, stats => !stats.isRegression), stats => stats.currentToBaseline * -1, ); reporter.markdown( [ '
Perf tests with no regressions', '', 'Scenario | Current PR Ticks | Baseline Ticks | Ratio', ':--- | ---:| ---:| ---:', ..._.map(noRegressions, (value, key) => [ value.name, linkToFlamegraph(value.numTicks, value.flamegraphFile), linkToFlamegraph(value.baseline.numTicks, value.baseline.flamegraphFile), `${value.currentToBaseline}:1`, ].join(' | '), ), '', '
', ].join('\n'), ); } const checkPerfRegressions = (reporter: Reporter) => { let perfCounts; reporter.markdown('## Perf Analysis (Fluent)'); try { perfCounts = require(config.paths.packageDist('perf-test', 'perfCounts.json')); } catch { reporter.warn('No perf measurements available'); return; } reportResults(perfCounts, reporter); };