/** * PAVI resources内存数据库 * @author guxiang * 整个页面共享一个内存数据库,原始resources对象统一存储在Map里面,用iter做tag,相同的iter传多个直接忽略不计。meta对象应该包含的是{ iter: number, time: number } * 最后的查询顺序是 resourceType (scalar, feature, histogram, heatmap) -> taskID -> tag -> iter -> value。 * TODO: option使用immutable.js进行优化,尽可能减轻数据处理的代价 * * 最终页面上需要用到的有3种x轴,其中iter应该使用value,time和relativeTime应该使用time类型 */ import { smoothData } from './delayFilterSmooth'; import echarts from 'echarts'; import { xAxisFormatter } from '../utils/formatterHelper'; import { uniqBy } from 'lodash'; import { TrainingInfoSchema } from '../types'; import { getRelativeTime } from '../utils/timeHelper'; import dayjs from 'dayjs'; import ScalarStore from '../stores/ScalarStore'; export type DBValue = { value: number; time: number; iter: number }; export type XAxisType = 'relative' | 'time' | 'iter'; export type SeriesData = [number, number, DBValue, TrainingInfoSchema, string]; // x, y, dbvalue, task, tag interface ScalarSchema { iteration: number; value: number; created: number; } export interface WSResponse { Type: string; Timestamp: number; // 秒为单位 Content: T; } export interface TagSchema { dataKey: string; color?: string; title: string; multiplier?: number; } const lineStyle = { width: 2, opacity: 1, }; const unhighlightLineStyle = { width: 2, opacity: 1, }; const highlightLineStyle = { width: 4, opacity: 1, }; export type ScalarDBOnChangeFunc = (event: { type: 'init' | 'append'; taskID: string; tag: string; values: DBValue[]; }) => void; export const getSeriesID = (taskID: string, tag: string) => `${taskID}_${tag}`; /** * 针对PAVI四种相似类型的数据设计的一个资源数据库 * echarts dataset 的 source 我们使用k-v式的声明,{ y1: v1, y2: v2, ... } */ export class ScalarDB { /** * 一类资源的所有数据都存储在这里,退出页面销毁 * 最终的结果里存一下iter,虽然是冗余的 */ private _dataMap: { [taskID: string]: { [tag: string]: { [iter: number]: DBValue } }; } = {}; /** * 不同task的的开始时间,用于计算relativeTime */ private _startTime: { [taskID: string]: { [tag: string]: number } } = {}; public get startTime() { return this._startTime; } private _endTime: { [taskID: string]: { [tag: string]: number } } = {}; public get endTime() { return this._endTime; } private _startIter: { [taskID: string]: { [tag: string]: number } } = {}; public get startIter() { return this._startIter; } private _endIter: { [taskID: string]: { [tag: string]: number } } = {}; public get endIter() { return this._endIter; } /** * 最小值的指针集合 */ private _min: { [taskID: string]: { [tag: string]: DBValue } } = {}; public get min() { return this._min; } /** * 最大值的指针集合 */ private _max: { [taskID: string]: { [tag: string]: DBValue } } = {}; public get max() { return this._max; } public getYAxis(isMetric): echarts.EChartOption.YAxis { return { name: 'Value', nameLocation: 'center', nameGap: 60, type: 'value', scale: true, splitLine: { show: false, }, axisLine: { onZero: false }, min: function (value) { return value.min; }, max: function (value) { return value.max; }, axisLabel: { formatter: (value: number, index: any) => { return isMetric ? `${value.toFixed(0)}%` : value.toFixed(3); }, }, }; } public getXAxis( xAxis: XAxisType, oneIter: boolean, isMetric, ): echarts.EChartOption.XAxis { return { name: isMetric ? 'Epoch' : 'Iteration', nameLocation: 'center', nameGap: 30, type: xAxis === 'iter' ? (oneIter ? 'category' : 'time') : 'time', splitLine: { show: false, }, axisLabel: { formatter: xAxisFormatter(xAxis, false), }, axisLine: { onZero: false }, min: function (value) { return value.min; }, max: function (value) { return value.max; }, }; } public getSeries( tasks: { id: string; name: string }[], tags: TagSchema[], xAxis: XAxisType, smoothWeight: number = 0, dataZoom: Parameters[2], scalarStore: typeof ScalarStore, isMetric: boolean, ) { const { colorStore, highlightStore } = scalarStore; const { currentHover } = highlightStore; const series: echarts.EChartOption.SeriesLine[] = []; const indexMap = new Map(); // 用于给同名的任务加个后缀 tasks.forEach((task) => { let index = indexMap.get(task.name); if (index === undefined) { indexMap.set(task.name, 0); index = 0; } else { index += 1; indexMap.set(task.name, index); } tags.forEach((tag) => { const { dataKey, title, color } = tag; if (!this._dataMap[task.id] || !this._dataMap[task.id][dataKey]) return; const data = uniqBy( Object.values(this._dataMap[task.id][dataKey]).map( (o) => [ xAxis === 'iter' ? o.iter : xAxis === 'time' ? o.time : o.time - this._startTime[task.id][dataKey], o.value, o, task, dataKey, ] as SeriesData, ), (o) => o[0], ); const seriesName = title || dataKey; // 线的名字 const colorName = `${dataKey}_${task.name}${index ? `_${index}` : ''}`; colorStore.addNameToColor(colorName); const seriesItem = { id: getSeriesID(task.id, dataKey), // 添加自定义的数据tagName,给Summary拼tooltip的name用 tagName: tag, colorName: colorName, markPoint: {}, name: seriesName, type: 'line', sampling: 'average', // 点太多的时候做一个均值采样 // animation: false, // 取消动画 connectNulls: true, // 这个是为了缩放到很小的时候不要出现散点 symbol: 'circle', smooth: false, // 关闭echarts默认的smooth选项,原因是这样可以看折线信息不会弯 data: smoothData(data, smoothWeight, dataZoom), itemStyle: { color: color || colorStore.getColorByName(colorName), }, lineStyle: currentHover != '' ? currentHover == task.id ? highlightLineStyle : unhighlightLineStyle : lineStyle, }; seriesItem.markPoint = { data: [ { name: 'Max value', type: 'max', label: { formatter: ({ value }: { value: number }) => { return isMetric ? `${value.toFixed(3)}%` : value.toFixed(3); }, }, }, { name: 'Min value', type: 'min', label: { formatter: ({ value }: { value: number }) => { return isMetric ? `${value.toFixed(3)}%` : value.toFixed(3); }, }, }, ] as any, }; series.push(seriesItem); }); }); return series; } public getDataRange( tasks: { id: string; name: string }[], tags: string[], xAxis: XAxisType, ) { // todo : 改成从数据里取值 const dataRange = { x: { startValue: undefined as number | undefined, endValue: undefined as number | undefined, }, y: { startValue: undefined as number | undefined, endValue: undefined as number | undefined, }, }; tasks.forEach((task) => { tags.forEach((tag) => { if (!this._dataMap[task.id] || !this._dataMap[task.id][tag]) return; // 计算x轴的最大范围 if (xAxis === 'iter') { if ( dataRange.x.startValue === undefined || dataRange.x.startValue > this._startIter[task.id][tag] ) dataRange.x.startValue = this._startIter[task.id][tag]; if ( dataRange.x.endValue === undefined || dataRange.x.endValue < this._endIter[task.id][tag] ) dataRange.x.endValue = this._endIter[task.id][tag]; } else if (xAxis === 'time') { if ( dataRange.x.startValue === undefined || dataRange.x.startValue > this._startTime[task.id][tag] ) dataRange.x.startValue = this._startTime[task.id][tag]; if ( dataRange.x.endValue === undefined || dataRange.x.endValue < this._endTime[task.id][tag] ) dataRange.x.endValue = this._endTime[task.id][tag]; } else { dataRange.x.startValue = 0; if ( dataRange.x.endValue === undefined || dataRange.x.endValue < this._endTime[task.id][tag] - this._startTime[task.id][tag] ) dataRange.x.endValue = this._endTime[task.id][tag] - this._startTime[task.id][tag]; } // 计算y轴的最大范围 if ( dataRange.y.startValue === undefined || dataRange.y.startValue > this._min[task.id][tag].value ) dataRange.y.startValue = this._min[task.id][tag].value; if ( dataRange.y.endValue === undefined || dataRange.y.endValue < this._max[task.id][tag].value ) dataRange.y.endValue = this._max[task.id][tag].value; }); }); return dataRange; } /** * Event Handler 用于告知外部数据发生了变化 */ private _onChange: { [taskID: string]: { [tag: string]: ScalarDBOnChangeFunc[] }; } = {}; /** * 一次性向数据库里更新数据,重复就替换 * @param taskID * @param tag * @param data */ public updateData(taskID: string, tag: string, payload: DBValue[]) { // 初始化 if (!this._dataMap[taskID]) this._dataMap[taskID] = {}; this._dataMap[taskID][tag] = {}; payload?.forEach((item) => { this.appendData(taskID, tag, item); }); } /** * 向数据库里增加数据 * @param taskID * @param tag * @param data */ public appendData(taskID: string, tag: string, payload: DBValue) { // 如果取不到值就初始化,最后赋值 if (!this._dataMap[taskID]) this._dataMap[taskID] = {}; if (!this._dataMap[taskID][tag]) this._dataMap[taskID][tag] = {}; const preValue = this._dataMap[taskID][tag][payload.iter]; // p.s. 如果iter相同,旧值时间早于新值,那么就覆盖之前的结果,华滨需求 if (!preValue || preValue.time < payload.time) this._dataMap[taskID][tag][payload.iter] = payload; // 记录开始时间 if (!this._startTime[taskID]) this._startTime[taskID] = {}; if ( this._startTime[taskID][tag] === undefined || this._startTime[taskID][tag] > payload.time ) this._startTime[taskID][tag] = payload.time; if (!this._endTime[taskID]) this._endTime[taskID] = {}; // 记录结束时间 if (!this._endTime[taskID]) this._endTime[taskID] = {}; if ( this._endTime[taskID][tag] === undefined || this._endTime[taskID][tag] < payload.time ) this._endTime[taskID][tag] = payload.time; // 记录开始iter if (!this._startIter[taskID]) this._startIter[taskID] = {}; if ( this._startIter[taskID][tag] === undefined || this._startIter[taskID][tag] > payload.iter ) this._startIter[taskID][tag] = payload.iter; if (!this._endIter[taskID]) this._endIter[taskID] = {}; // 记录结束时间 if (!this._endIter[taskID]) this._endIter[taskID] = {}; if ( this._endIter[taskID][tag] === undefined || this._endIter[taskID][tag] < payload.iter ) this._endIter[taskID][tag] = payload.iter; // 记录最小的 if (!this._min[taskID]) this._min[taskID] = {}; if ( this._min[taskID][tag] === undefined || this._min[taskID][tag].value > payload.value ) this._min[taskID][tag] = payload; // 记录最大的 if (!this._max[taskID]) this._max[taskID] = {}; if ( this._max[taskID][tag] === undefined || this._max[taskID][tag].value < payload.value ) this._max[taskID][tag] = payload; } /** * 获取当前dataset的所有tags信息 */ public get tags() { const tags = new Set(); Object.entries(this._onChange).forEach(([taskID, v1]) => { Object.entries(v1).forEach(([tag, v2]) => { if (this._onChange[taskID][tag] && this._onChange[taskID][tag].length) tags.add(tag); }); }); return Array.from(tags); } /** * 根据x轴的值和x轴类型返回一个slice的summary信息,用于tooltip表格的展示 * @param axisValue * @param axisType */ static Summary( scalardb: ScalarDB, series: echarts.EChartOption.SeriesLine[], axisValue: number, scalarStore: typeof ScalarStore, // colors: string[], ): { key: string; color: string; name: string; iter: number; time: string; relative: string; value: number; min: string; max: string; }[] { const maybeFindExactItem = (line: echarts.EChartOption.SeriesLine) => { // 因为不可能没有点,所以这里就写简略了一点,没有考虑找不到item的情况 const data = (line.data || []) as SeriesData[]; const nearest = data.reduce((pre, cur) => { if (!pre) return cur; else if (Math.abs(pre[0] - axisValue) > Math.abs(cur[0] - axisValue)) return cur; return pre; }, data[0]); return nearest; }; return series.map((line, index) => { const item = maybeFindExactItem(line); const max = scalardb.max[item[3].id][item[4]]; const min = scalardb.min[item[3].id][item[4]]; return { key: line.id!, // @ts-ignore color: line.itemStyle.color as string, // @ts-ignore name: line.tagAlias ? // @ts-ignore `${line.tagAlias}(${line.tagName})` : // @ts-ignore line.name, iter: item[2].iter, time: dayjs(item[2].time).format('YYYY-MM-DD HH:mm:ss'), relative: getRelativeTime( item[2].time - scalardb.startTime[item[3].id][item[4]], ), value: item[2].value, min: `${min.value} | ${min.iter}`, max: `${max.value} | ${max.iter}`, }; }); } } /** * scalardb全局单例 */ // export const scalardb = new ScalarDB(); // @ts-ignore // window['scalardb'] = scalardb;