import fs from 'fs'; import path from 'path'; import _ from 'lodash'; import flamegrill, { CookResult, CookResults, ScenarioConfig, Scenarios } from 'flamegrill'; import { generateUrl } from '@fluentui/digest'; type ExtendedCookResult = CookResult & { extended: { kind: string; story: string; iterations: number; tpi?: number; fabricTpi?: number; filename?: number; }; }; type ExtendedCookResults = Record; // !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! // TODO: // // As much of this file should be absorbed into flamegrill as possible. // Flamegrill knows all possible kinds and stories from digest. Could default to running tests against all. // Embed iterations in stories as well as scenarios. That way they would apply for static tests as well. // // !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! // TODO: We can't do CI, measure baseline or do regression analysis until master & PR files are deployed and publicly accessible. // TODO: Fluent reporting is outside of this script so this code will probably be moved entirely on perf-test consolidation. // const urlForDeployPath = process.env.BUILD_SOURCEBRANCH // ? `http://fabricweb.z5.web.core.windows.net/pr-deploy-site/${process.env.BUILD_SOURCEBRANCH}/perf-test` // : `file://${path.resolve(__dirname, '../dist/')}`; const urlForDeployPath = `file://${path.resolve(__dirname, '../dist/')}`; // const urlForMaster = process.env.SYSTEM_PULLREQUEST_TARGETBRANCH // ? `http://fabricweb.z5.web.core.windows.net/pr-deploy-site/refs/heads/${process.env.SYSTEM_PULLREQUEST_TARGETBRANCH}/perf-test/index.html` // : 'http://fabricweb.z5.web.core.windows.net/pr-deploy-site/refs/heads/master/perf-test/index.html'; const urlForDeploy = `${urlForDeployPath}/index.html`; const defaultIterations = 1; const outDir = path.join(__dirname, '../dist'); const tempDir = path.join(__dirname, '../logfiles'); console.log(`__dirname: ${__dirname}`); export default async function getPerfRegressions(baselineOnly: boolean = false) { let urlForMaster; if (!baselineOnly) { urlForMaster = process.env.SYSTEM_PULLREQUEST_TARGETBRANCH ? `http://fabricweb.z5.web.core.windows.net/pr-deploy-site/refs/heads/${process.env.SYSTEM_PULLREQUEST_TARGETBRANCH}/perf-test/fluentui/index.html` : 'http://fabricweb.z5.web.core.windows.net/pr-deploy-site/refs/heads/master/perf-test/fluentui/index.html'; } // TODO: support iteration/kind/story via commandline as in other perf-test script // TODO: can do this now that we have story information // TODO: align flamegrill terminology with CSF (story vs. scenario) const scenarios: Scenarios = {}; const scenarioList: string[] = []; // TODO: can this get typing somehow? can't be imported since file is only available after build const test = require('../dist/stories.js'); const { stories } = test.default; console.log('stories:'); console.dir(stories, { depth: null }); Object.keys(stories).forEach(kindKey => { Object.keys(stories[kindKey]) .filter(storyKey => typeof stories[kindKey][storyKey] === 'function') .forEach(storyKey => { const scenarioName = `${kindKey}.${storyKey}`; scenarioList.push(scenarioName); scenarios[scenarioName] = { scenario: generateUrl(urlForDeploy, kindKey, storyKey, getIterations(stories, kindKey, storyKey)), ...(!baselineOnly && storyKey !== 'Fabric' && { // Optimization: skip baseline comparision for Fabric baseline: generateUrl(urlForMaster, kindKey, storyKey, getIterations(stories, kindKey, storyKey)), }), }; }); }); console.log(`\nRunning scenarios: ${scenarioList}\n`); if (!fs.existsSync(tempDir)) { console.log(`Making temp directory ${tempDir}...`); fs.mkdirSync(tempDir); } const tempContents = fs.readdirSync(tempDir); if (tempContents.length > 0) { console.log(`Unexpected files already present in ${tempDir}`); tempContents.forEach(logFile => { const logFilePath = path.join(tempDir, logFile); console.log(`Deleting ${logFilePath}`); fs.unlinkSync(logFilePath); }); } const scenarioConfig: ScenarioConfig = { outDir, tempDir, pageActions: async (page, options) => { // Occasionally during our CI, page takes unexpected amount of time to navigate (unsure about the root cause). // Removing the timeout to avoid perf-test failures but be cautious about long test runs. page.setDefaultTimeout(0); await page.goto(options.url); }, }; const scenarioResults = await flamegrill.cook(scenarios, scenarioConfig); const extendedCookResults = extendCookResults(stories, scenarioResults); fs.writeFileSync(path.join(outDir, 'perfCounts.json'), JSON.stringify(extendedCookResults, null, 2)); const comment = createReport(stories, extendedCookResults); // TODO: determine status according to perf numbers const status = 'success'; console.log(`Perf evaluation status: ${status}`); // Write results to file fs.writeFileSync(path.join(outDir, 'perfCounts.html'), comment); console.log(`##vso[task.setvariable variable=PerfCommentFilePath;]apps/perf-test/dist/perfCounts.html`); console.log(`##vso[task.setvariable variable=PerfCommentStatus;]${status}`); } function extendCookResults(stories, testResults: CookResults): ExtendedCookResults { return _.mapValues(testResults, (testResult, resultKey) => { const kind = getKindKey(resultKey); const story = getStoryKey(resultKey); const iterations = getIterations(stories, kind, story); const tpi = getTpiResult(testResults, stories, kind, story); // || 'n/a' const fabricTpi = getTpiResult(testResults, stories, kind, 'Fabric'); // || '' return { ...testResult, extended: { kind, story, iterations, tpi, fabricTpi, filename: stories[kind][story].filename, }, }; }); } /** * Create test summary based on test results. * * @param {CookResults} testResults * @returns {string} */ function createReport(stories, testResults: ExtendedCookResults): string { const report = '' // TODO: We can't do CI, measure baseline or do regression analysis until master & PR files are deployed and publicly accessible. // TODO: Fluent reporting is outside of this script so this code will probably be moved entirely on perf-test consolidation. // // Show only significant changes by default. // .concat(createScenarioTable(testResults, false)) // // Show all results in a collapsible table. // .concat('
All results

') // .concat(createScenarioTable(testResults, true)) // .concat('

'); .concat(createScenarioTable(stories, testResults, true)); return report; } /** * Create a table of scenario results. * * @param {CookResults} testResults * @param {boolean} showAll Show only significant results by default. * @returns {string} */ function createScenarioTable(stories, testResults: ExtendedCookResults, showAll: boolean): string { const resultsToDisplay = Object.keys(testResults) .filter( key => showAll || (testResults[key].analysis && testResults[key].analysis.regression && testResults[key].analysis.regression.isRegression), ) .filter(testResultKey => getStoryKey(testResultKey) !== 'Fabric') .sort(); if (resultsToDisplay.length === 0) { return '

No significant results to display.

'; } // TODO: We can't do CI, measure baseline or do regression analysis until master & PR files are deployed and publicly accessible. // TODO: Fluent reporting is outside of this script so this code will probably be moved entirely on perf-test consolidation. // const result = ` // // // // // // // `.concat( // resultsToDisplay // .map(key => { // const testResult = testResults[key]; // return ` // // ${getCell(testResult, true)} // ${getCell(testResult, false)} // ${getRegression(testResult)} // `; // }) // .join('\n') // .concat(`
Scenario // Master Ticks // // PR Ticks // Status
${scenarioNames[key] || key}
`), // ); // TODO: add iterations column (and maybe ticks per iteration) const result = ` `.concat( resultsToDisplay .map(resultKey => { const testResult = testResults[resultKey]; const tpi = testResult.extended.tpi ? linkifyResult( testResult, testResult.extended.tpi.toLocaleString('en', { maximumSignificantDigits: 2 }), false, ) : 'n/a'; const fabricTpi = testResult.extended.fabricTpi ? linkifyResult( testResult, testResult.extended.fabricTpi.toLocaleString('en', { maximumSignificantDigits: 2 }), false, ) : ''; return ``; }) .join('\n') .concat(`
Kind Story Fabric TPI TPI Iterations PR Ticks
${testResult.extended.kind} ${testResult.extended.story} ${fabricTpi} ${tpi} ${testResult.extended.iterations} ${getTicksResult(testResult, false)}
`), ); return result; } function getKindKey(resultKey: string): string { const [kind] = resultKey.split('.'); return kind; } function getStoryKey(resultKey: string): string { const [, story] = resultKey.split('.'); return story; } function getTpiResult(testResults, stories, kind, story): number | undefined { let tpi = undefined; if (stories[kind][story]) { const resultKey = `${kind}.${story}`; const testResult = testResults[resultKey]; const ticks = getTicks(testResult); const iterations = getIterations(stories, kind, story); tpi = ticks && iterations && Math.round((ticks / iterations) * 100) / 100; } return tpi; } function getIterations(stories, kind, story): number { // Give highest priority to most localized definition of iterations. Story => kind => default. return ( stories[kind][story].iterations || (stories[kind].default && stories[kind].default.iterations) || defaultIterations ); } function getTicks(testResult: CookResult): number | undefined { return testResult.analysis && testResult.analysis.numTicks; } function linkifyResult(testResult, resultContent, getBaseline) { let flamegraphFile = testResult.processed.output && testResult.processed.output.flamegraphFile; let errorFile = testResult.processed.error && testResult.processed.error.errorFile; if (getBaseline) { const processedBaseline = testResult.processed.baseline; flamegraphFile = processedBaseline && processedBaseline.output && processedBaseline.output.flamegraphFile; errorFile = processedBaseline && processedBaseline.error && processedBaseline.error.errorFile; } const cell = errorFile ? `err` : flamegraphFile ? `${resultContent}` : `n/a`; return cell; } /** * Helper that renders an output cell based on a test result. * * @param {CookResult} testResult * @param {boolean} getBaseline * @returns {string} */ function getTicksResult(testResult: CookResult, getBaseline: boolean): string { let numTicks = testResult.analysis && testResult.analysis.numTicks; if (getBaseline) { numTicks = testResult.analysis && testResult.analysis.baseline && testResult.analysis.baseline.numTicks; } return linkifyResult(testResult, numTicks, getBaseline); } /** * Helper that renders an output cell based on a test result. * * @param {CookResult} testResult * @returns {string} */ // TODO: We can't do CI, measure baseline or do regression analysis until master & PR files are deployed and publicly accessible. // TODO: Fluent reporting is outside of this script so this code will probably be moved entirely on perf-test consolidation. // function getRegression(testResult: CookResult): string { // const cell = testResult.analysis && testResult.analysis.regression && testResult.analysis.regression.isRegression // ? testResult.analysis.regression.regressionFile // ? `Possible regression` // : '' // : ''; // return `${cell}`; // }