// (C) 2019-2020 GoodData Corporation import { ExecuteAFM as AFM, Execution } from "@gooddata/typings"; import { isEmpty, isEqual, uniq, chunk, flattenDeep } from "lodash"; import { DimensionHeaders, Headers, RandomData } from "./dataResult"; import { isILocalIdentifierQualifier } from "./execution"; import { CachedResponse, IErrorResponse, isErrorResponse } from "../executeAfm"; import { getSizeFromHeaders } from "./getDataSize"; import IResultHeaderItem = Execution.IResultHeaderItem; import IResultMeasureHeaderItem = Execution.IResultMeasureHeaderItem; import isMeasureHeaderItem = Execution.isMeasureHeaderItem; import IExecutionResultWrapper = Execution.IExecutionResultWrapper; import IExecution = AFM.IExecution; import isMeasureValueFilter = AFM.isMeasureValueFilter; import IMeasureValueFilter = AFM.IMeasureValueFilter; import ILocalIdentifierQualifier = AFM.ILocalIdentifierQualifier; import IComparisonCondition = AFM.IComparisonCondition; import IRangeCondition = AFM.IRangeCondition; import MeasureValueFilterCondition = AFM.MeasureValueFilterCondition; import IMeasure = AFM.IMeasure; import CompatibilityFilter = AFM.CompatibilityFilter; const REMOVED_VALUE_PLACEHOLDER = "#REMOVED_VALUE#"; export interface IDataResult { headerItems: Headers; data: RandomData; } function findLocalMeasureValueFilters(filters: CompatibilityFilter[]): IMeasureValueFilter[] { if (isEmpty(filters)) { return []; } return filters.filter( filter => isMeasureValueFilter(filter) && isILocalIdentifierQualifier(filter.measureValueFilter.measure), ) as IMeasureValueFilter[]; } function findIndexOfMeasuresDimension(headerItems: Headers): number { return headerItems.findIndex((dimension: DimensionHeaders) => dimension.some((headers: IResultHeaderItem[]) => headers.some((header: IResultHeaderItem) => isMeasureHeaderItem(header)), ), ); } function findMeasureResultHeaders(measureDimension: DimensionHeaders): IResultMeasureHeaderItem[] { return measureDimension.find( (headers: IResultHeaderItem[]) => headers.length > 0 && isMeasureHeaderItem(headers[0]), ) as IResultMeasureHeaderItem[]; } function findReferencedMeasureAfmIndex(filter: IMeasureValueFilter, measures: IMeasure[]) { const measureQualifier = filter.measureValueFilter.measure as ILocalIdentifierQualifier; return measures.findIndex(measure => measure.localIdentifier === measureQualifier.localIdentifier); } /** * Apply measure value filters from provided execution to provided dataResult generated for the execution. * The function filters the row or column measure data and removes attribute headers that no longer * have values. * * @param execution * @param dataResult * @return filtered execution result if matching filters were found in execution */ function filterDataResult(execution: IExecution, dataResult: IDataResult): IDataResult | IErrorResponse { const { execution: { afm }, } = execution; const { measures, filters } = afm; if (isEmpty(measures) || isEmpty(filters)) { return dataResult; } const measureValueFilters = findLocalMeasureValueFilters(filters); if (measureValueFilters.length === 0) { return dataResult; } const errorResponse = validateExecution(execution); if (errorResponse) { return errorResponse; } const { headerItems, data } = dataResult; const measureDimensionIndex = findIndexOfMeasuresDimension(headerItems); const measuresAreInRow = measureDimensionIndex === 0; const attributeDimensionIndex = measuresAreInRow ? 1 : 0; const measureHeaders = findMeasureResultHeaders(headerItems[measureDimensionIndex]); const getRemovedValuesIfSomeValueIsRemoved = (data: string[]) => data.includes(REMOVED_VALUE_PLACEHOLDER) ? new Array(data.length).fill(REMOVED_VALUE_PLACEHOLDER) : data; const filterDataRow = (dataRow: string[], filter: IMeasureValueFilter, filteredMeasureIndex: number) => { // split values into groups per repeated measures ([m1, m2, m1, m2] => [[m1, m2], [m1, m2]]) const filteredMeasureGroups = chunk(dataRow, measures.length).map((measureGroupValues: string[]) => { // replace values that does not match filter with removed value placeholder in every group const processedGroup = measureGroupValues.map((value: string, columnIndex: number) => { const currentMeasureIndex = measureHeaders[columnIndex].measureHeaderItem.order; return currentMeasureIndex === filteredMeasureIndex ? getMatchingValueOrRemovedValuePlaceholder(value, filter) : value; }); // replace all values in group with removed value placeholder if at least one value is removed value placeholder (was filtered out) return getRemovedValuesIfSomeValueIsRemoved(processedGroup); }); // put processed groups to the original shape ([[m1, m2], [m1, m2]] => [m1, m2, m1, m2]) return flattenDeep(filteredMeasureGroups); }; // rotate the data to have measures in columns and attributes in rows if it is not the case const columnData = measuresAreInRow ? rotateMatrix(data) : data; // replace values not matching the filters with removed value placeholder const validDataResultValues = measureValueFilters.reduce( (dataResult: string[][], filter: IMeasureValueFilter) => { const measureIndex = findReferencedMeasureAfmIndex(filter, measures); return dataResult.map((dataRow: string[]) => filterDataRow(dataRow, filter, measureIndex)); }, columnData, ); // remove row attribute headers that have every data value set as removed value placeholder in the row const keepRow = (_: any, rowIndex: number) => !validDataResultValues[rowIndex].every( (columnValue: string) => columnValue === REMOVED_VALUE_PLACEHOLDER, ); const filteredAttributesHeaders = headerItems[attributeDimensionIndex].map(rowAttributes => rowAttributes.filter(keepRow), ); // remove column attribute headers that have every data value set as removed value placeholder in the column const keepColumn = (_: any, columnIndex: number) => !validDataResultValues.every( (dataRow: string[]) => dataRow[columnIndex] === REMOVED_VALUE_PLACEHOLDER, ); const filteredMeasureHeaders = headerItems[measureDimensionIndex].map(columnAttributes => columnAttributes.filter(keepColumn), ); // remove data rows and columns that contain only removed value placeholder const filteredDataRows = validDataResultValues .filter(keepRow) .map((dataRow: string[]) => dataRow.filter(keepColumn)); const replaceRemovedValuesByNull = (value: string) => value === REMOVED_VALUE_PLACEHOLDER ? null : value; // replace removed value placeholder by null values const filteredDataRowsWithNullValues = filteredDataRows.map((dataRow: string[]) => dataRow.map(replaceRemovedValuesByNull), ); // rebuild header item dimensions const filteredHeaderItems = []; filteredHeaderItems[attributeDimensionIndex] = filteredAttributesHeaders; filteredHeaderItems[measureDimensionIndex] = filteredMeasureHeaders; return { headerItems: filteredHeaderItems, data: measuresAreInRow ? rotateMatrix(filteredDataRowsWithNullValues) // rotate data back to its original shape : filteredDataRowsWithNullValues, }; } function rotateMatrix(matrix: string[][]): string[][] { return matrix.length === 0 ? matrix : matrix[0].map((_, index) => matrix.map(row => row[index])); } function isComparisonCondition(condition: MeasureValueFilterCondition): condition is IComparisonCondition { return !isEmpty(condition) && (condition as IComparisonCondition).comparison !== undefined; } function isRangeCondition(condition: MeasureValueFilterCondition): condition is IRangeCondition { return !isEmpty(condition) && (condition as IRangeCondition).range !== undefined; } function getMatchingValueOrRemovedValuePlaceholder( afmResultValue: string, filter: IMeasureValueFilter, ): string { return isMatchingFilter(afmResultValue, filter) ? afmResultValue : REMOVED_VALUE_PLACEHOLDER; } function transformNullValues(testedValue: number, treatNullValuesAs: number): number { if (treatNullValuesAs === undefined || testedValue !== null) { return testedValue; } return treatNullValuesAs; } function isMatchingFilter(afmResultValue: string, filter: IMeasureValueFilter): boolean { const { measureValueFilter: { condition }, } = filter; let testedValue = afmResultValue === null ? null : Number(afmResultValue); if (isComparisonCondition(condition)) { const { comparison: { operator, value, treatNullValuesAs }, } = condition; testedValue = transformNullValues(testedValue, treatNullValuesAs); if (testedValue === null) { return false; } switch (operator) { case "GREATER_THAN": return testedValue > value; case "GREATER_THAN_OR_EQUAL_TO": return testedValue >= value; case "LESS_THAN": return testedValue < value; case "LESS_THAN_OR_EQUAL_TO": return testedValue <= value; case "EQUAL_TO": return testedValue === value; case "NOT_EQUAL_TO": return testedValue !== value; default: return true; } } if (isRangeCondition(condition)) { const { range: { operator, from, to, treatNullValuesAs }, } = condition; testedValue = transformNullValues(testedValue, treatNullValuesAs); if (testedValue === null) { return false; } switch (operator) { case "BETWEEN": return from <= testedValue && testedValue <= to; case "NOT_BETWEEN": return testedValue < from || to < testedValue; default: return true; } } return true; } function buildBadRequestResponse(message: string): IErrorResponse { return { error: { statusCode: 400, message, }, }; } function eachMeasureValueFilterReferencesValidLocalMeasure( measures: IMeasure[], filters: IMeasureValueFilter[], ): boolean { return filters.every(filter => findReferencedMeasureAfmIndex(filter, measures) >= 0); } function eachMeasureHasMaximumOneMeasureValueFilter(filters: IMeasureValueFilter[]): boolean { const referencedMeasureIdentifiers = filters.map( filter => (filter.measureValueFilter.measure as ILocalIdentifierQualifier).localIdentifier, ); return referencedMeasureIdentifiers.length === uniq(referencedMeasureIdentifiers).length; } function validateExecution(execution: IExecution): IErrorResponse { const { execution: { afm }, } = execution; if (isEmpty(afm.attributes)) { return buildBadRequestResponse( "The insight is filtered by a measure value but it is not sliced by any attribute!", ); } const measureValueFilters = findLocalMeasureValueFilters(afm.filters); if (!eachMeasureValueFilterReferencesValidLocalMeasure(afm.measures, measureValueFilters)) { return buildBadRequestResponse( "Some measure value filters reference a measure that was not found in AFM!", ); } if (!eachMeasureHasMaximumOneMeasureValueFilter(measureValueFilters)) { return buildBadRequestResponse("Some measure is referenced by multiple measure value filters!"); } } function isExecutionResult(result: CachedResponse): result is IExecutionResultWrapper { return !isEmpty(result) && (result as IExecutionResultWrapper).executionResult !== undefined; } function buildDataResult(result: IExecutionResultWrapper): IDataResult { const { executionResult: { headerItems, data }, } = result; return { headerItems, data, }; } function buildExecutionResult(originalResponse: IExecutionResultWrapper, dataResult: IDataResult) { const { executionResult } = originalResponse; const { headerItems, data } = dataResult; const total = getSizeFromHeaders(headerItems); return { executionResult: { ...executionResult, headerItems, data, paging: { ...executionResult.paging, count: total, total, }, }, }; } /** * Apply measure value filters from provided execution to provided execution result stored for execution. * The function filters the row or column measure data and removes attribute headers that no longer * have values. It also updates the paging information. * * @param execution * @param response * @return filtered execution result if matching filters were found in execution */ function filterExecutionResult( execution: IExecution, response: CachedResponse | IErrorResponse, ): CachedResponse | IErrorResponse { if (isErrorResponse(response) || !isExecutionResult(response)) { return response; } const dataResult = filterDataResult(execution, buildDataResult(response)); if (isErrorResponse(dataResult)) { return dataResult; } return isEqual(dataResult.data, response.executionResult.data) ? response : buildExecutionResult(response, dataResult); } export default { filterDataResult, filterExecutionResult, };