import CryptoJS from 'crypto-js' import { stringify } from 'flatted' import { chunk, every, isNumber, isObject, isString, omit, set } from 'lodash' import { ElementDataWrapper, instanceOfLocalElement, } from '../../interfaces/element.interface' import { CriteriumKey, DateQuery, FileAttributeCriteria, IListingConfig, IListingFilter, IListingPreset, IQueryOptions, ListingHeaders, ListingOptions, ListingQuery, ListingRequestData, ListingService, MetaAttributeKey, OnElementCriteria, QueryCriteria, RawCriteria, RawCriterium, RawListingQuery, SerializableListingParams, } from '../../interfaces/listing.interface' import { ELEMENT_NAME, META_ATTRIBUTES } from '../../models/constants' import { Element } from '../../models/element' import { isArrayLike } from '../../util/isArrayLike' import { DynamicContext, parseDynamicValue, parseTimeRange, } from '../../util/parseDynamicValue' import { Api, ApiResponse } from '../api' import { ENDPOINTS } from '.' export const LISTING_PER_PAGE = 50 export class Listing implements ListingService { api: Api constructor(api: Api) { this.api = api } private static isMetaKey(key: string): key is MetaAttributeKey { return META_ATTRIBUTES.includes(key) } private static isValueKey(key: string): key is `${string}.value` { return key.endsWith('.value') } private static isFileKey(key: string): key is `${string}.file` { return key.endsWith('.file') } private static setValue( obj: RawCriteria, key: `${string}.value` | MetaAttributeKey, value: RawCriterium, ) { ;(obj as unknown as Record)[key] = value } private static setFile( obj: RawCriteria, key: `${string}.file`, value: FileAttributeCriteria, ) { ;(obj as unknown as Record)[key] = value } private static setOr(obj: RawCriteria, value: RawCriteria[]) { ;(obj as unknown as Record)['$or'] = value } parseCriteria( criteria: QueryCriteria, context?: DynamicContext, initial: RawCriteria = {}, ) { return Object.entries(criteria).reduce((acc: RawCriteria, [key, value]) => { const criteriaKey: CriteriumKey = Listing.isValueKey(key) || Listing.isFileKey(key) || key === '$or' || Listing.isMetaKey(key) ? (key as CriteriumKey) : `${key}.value` if (value === null) { // ignore empty fields return acc } if (Array.isArray(value) && every(value, (e) => e instanceof Element)) { // TODO: HANDLE ATTRIBUTE MULTIPLE if (Listing.isValueKey(criteriaKey) || Listing.isMetaKey(criteriaKey)) { Listing.setValue( acc, criteriaKey as `${string}.value`, (value as Element[])[0].hash, ) } } else if ( isObject(value) && (Object.prototype.hasOwnProperty.call(value, '$gte') || Object.prototype.hasOwnProperty.call(value, '$lte')) ) { const query: DateQuery = {} const dateRange = value as DateQuery if (dateRange.$gte) { query.$gte = dateRange.$gte } if (dateRange.$lte) { query.$lte = dateRange.$lte } if (Listing.isValueKey(criteriaKey) || Listing.isMetaKey(criteriaKey)) { Listing.setValue(acc, criteriaKey as `${string}.value`, query) } } else if (value instanceof Element) { if (Listing.isValueKey(criteriaKey) || Listing.isMetaKey(criteriaKey)) { Listing.setValue(acc, criteriaKey as `${string}.value`, value.hash) } } else if ( isObject(value) && Object.prototype.hasOwnProperty.call(value, 'criteria') ) { const onElement = value as OnElementCriteria const parsedSubcriteria = this.parseCriteria( onElement.criteria, context, ) if (Listing.isValueKey(criteriaKey) || Listing.isMetaKey(criteriaKey)) { Listing.setValue( acc, criteriaKey as `${string}.value`, { collection: onElement.collection, type: onElement.type, criteria: parsedSubcriteria, } as RawCriterium, ) } } else if (criteriaKey === '$or' && Array.isArray(value)) { Listing.setOr( acc, (value as QueryCriteria[]).map((crit) => this.parseCriteria(crit, context), ), ) } else if ( isObject(value) && '$not' in value && typeof value.$not === 'string' ) { if (Listing.isValueKey(criteriaKey) || Listing.isMetaKey(criteriaKey)) { Listing.setValue( acc, criteriaKey as `${string}.value`, { $not: typeof value.$not === 'string' ? this.parseDynamicFilter(value.$not, context) : value.$not, } as RawCriterium, ) } } else { const dynamic = this.parseDynamicFilter(value, context) if (dynamic) { if ( Listing.isValueKey(criteriaKey) || Listing.isMetaKey(criteriaKey) ) { Listing.setValue(acc, criteriaKey as `${string}.value`, dynamic) } else if (Listing.isFileKey(criteriaKey) && isObject(dynamic)) { Listing.setFile(acc, criteriaKey, dynamic as FileAttributeCriteria) } } } return acc }, initial) } getListingRequestHash( params: SerializableListingParams, organization: string, context: DynamicContext, ) { const isLocalElement = instanceOfLocalElement(context?.element) const elementContextHash = isLocalElement ? undefined : context?.element?.hash const userElementContextHash = context?.userElement?.hash const dataString = stringify([ ...params, organization, elementContextHash, userElementContextHash, ]) const dataHash = CryptoJS.MD5(dataString).toString() return dataHash } // TODO: support nested filters ($or) static parsePresetConfig(preset: IListingPreset): IListingConfig { const cleanFilters = preset.filter?.filter( (filter) => !isArrayLike(filter), ) as IListingFilter[] const basicCriteria = Listing.formatCriteria( preset.filter as IListingFilter[], ) const projection = cleanFilters .filter((filter) => !filter.value) .map((filter) => filter.key) const relationCriteria = preset.relationFilters ? { [`${preset.relationFilters.attribute}.value`]: { collection: preset.relationFilters.collection, type: preset.relationFilters.type || 'relation', criteria: this.formatCriteria( preset.relationFilters.criteria || [], ), }, } : {} const combinedCriteria: QueryCriteria = { ...basicCriteria, ...relationCriteria, } return { type: preset.type, criteria: combinedCriteria as QueryCriteria, projection: [ELEMENT_NAME, ...projection], page: preset.pagination?.page || 1, limit: preset.pagination?.limit || LISTING_PER_PAGE, sort: preset.sortBy ? { [preset.sortBy]: preset.sortReverse || 1, } : { [ELEMENT_NAME]: 1, }, } } static formatCriteria(criteria: IListingFilter[]) { return criteria.reduce((resultCriteria, criterion) => { let formattedCriterion if (Array.isArray(criterion)) { formattedCriterion = { $or: criterion.reduce((resultC, c) => { const formattedC = Listing.formatCriterionReducer(c) if (formattedC) { resultC.push(formattedC) } return resultC }, []), } } else { formattedCriterion = Listing.formatCriterionReducer(criterion) } return formattedCriterion ? { ...resultCriteria, ...formattedCriterion } : resultCriteria }, {} as QueryCriteria) } private static deconstructCriterion(criterion: IListingFilter | string): { key: string type?: string value: IListingFilter['value'] logicOperation?: IListingFilter['logicOperation'] } { if (typeof criterion === 'string') { const parsedFilter = criterion.split(':') const [ key, value = '', // eslint-disable-next-line @typescript-eslint/no-unused-vars _notUsedFilterBlock, logicOperation = 'CONTAINSWORD', ] = parsedFilter return { key, value, type: 'Text', logicOperation: logicOperation as IListingFilter['logicOperation'], } } else { return criterion } } private static formatCriterionReducer(criterion: IListingFilter | string) { const { key, value, type, logicOperation } = Listing.deconstructCriterion(criterion) const criterionName = Listing.getCriterionName(key) const criterionValue = value || value === 0 || (logicOperation && ['EXISTS', 'NOTEXIST'].includes(logicOperation)) ? Listing.getCriterionValue({ key, value, type, logicOperation, }).getCriterion() : null return criterionName && (criterionValue || criterionValue === 0) ? set({}, [criterionName], criterionValue) : undefined } public static getCriterionName = (key: string) => { return META_ATTRIBUTES.includes(key) ? key : `${key}.value` } public static getDefaultLogicOperation(attributeType: string | undefined) { const comparatorConfig = Listing.getComparatorConfig({ key: '', value: '' }) return attributeType && comparatorConfig['CONTAINSWORD'].attrTypes.includes(attributeType) ? 'CONTAINSWORD' : 'EQUAL' } private static getCriterionValue(criterion: IListingFilter) { return Listing.getComparatorConfig(criterion)[ criterion.logicOperation || Listing.getDefaultLogicOperation(criterion.type) ] } private static isValidDate( val: unknown, type: string, ): val is [string, string] { const isNonEmptyString = (s: unknown): s is string => isString(s) && s.length > 0 const isArrayOf2 = (a: unknown): a is [unknown, unknown] => isArrayLike(a) && a.length === 2 return ( ['Date', 'DateTime'].includes(type) && (isNonEmptyString(val) || isArrayOf2(val)) ) } private static getComparatorConfig(criterion: IListingFilter) { const { value, type = '', key } = criterion return { CONTAINSWORD: { attrTypes: ['Text', 'Textarea', 'Dropdown'], getCriterion: () => Listing.isValidDate(value, type) ? Listing.getDateRangeCriterion(value) : isNumber(value) || (key && META_ATTRIBUTES.includes(key)) || isObject(value) ? value : `=${value}`, }, EQUAL: { attrTypes: [ 'Text', 'Number', 'User', 'Element', 'Dropdown', 'Date', 'DateTime', ], getCriterion: () => Listing.isValidDate(value, type) ? Listing.getDateRangeCriterion(value) : isNumber(value) || (key && META_ATTRIBUTES.includes(key)) || isObject(value) ? value : `=${value}`, }, NOTEQUAL: { attrTypes: [ 'Text', 'Number', 'User', 'Element', 'Dropdown', 'Date', 'DateTime', ], getCriterion: () => ({ $not: value }), }, IN: { attrTypes: [ 'Text', 'Number', 'User', 'Element', 'Dropdown', 'Date', 'DateTime', ], getCriterion: () => ['Element', 'Text', 'Dropdown', 'User'].includes(type) && isArrayLike(value) ? { $in: value.map((stringOrHash) => `=${stringOrHash}`), } : { $in: [`=${value}`] }, }, NOTIN: { attrTypes: [ 'Text', 'Number', 'User', 'Element', 'Dropdown', 'Date', 'DateTime', ], getCriterion: () => ({ $not: { $in: isArrayLike(value) ? value.map((stringOrHash) => `=${stringOrHash}`) : [`=${value}`], }, }), }, '≥': { attrTypes: ['Number', 'Date', 'DateTime'], getCriterion: () => ({ $gte: value, }), }, '>': { attrTypes: ['Number', 'Date', 'DateTime'], getCriterion: () => ({ $gt: value, }), }, '<': { attrTypes: ['Number', 'Date', 'DateTime'], getCriterion: () => ({ $lt: value, }), }, '≤': { attrTypes: ['Number', 'Date', 'DateTime'], getCriterion: () => ({ $lte: value, }), }, EXISTS: { attrTypes: [ 'Text', 'Number', 'User', 'Element', 'Dropdown', 'Date', 'DateTime', ], getCriterion: () => ({ $exists: true }), }, NOTEXIST: { attrTypes: [ 'Text', 'Number', 'User', 'Element', 'Dropdown', 'Date', 'DateTime', ], getCriterion: () => ({ $exists: false }), }, } } private static getDateRangeCriterion( criterionValue: string | [string, string], ): DateQuery { const dateRanges: [string, string] = isString(criterionValue) ? (criterionValue.split(',') as [string, string]) : criterionValue // Convert secs to milisecs to create Date object // and then parse ISO string out of it return { $gte: new Date(+dateRanges[0] * 1000).toISOString(), $lt: new Date(+dateRanges[1] * 1000).toISOString(), } } async getRawQuery(query: RawListingQuery) { const { type, criteria, projection } = getListingConfigFromMongoQuery(query) return this.get( type, criteria as RawCriteria, projection, query.sort, query.paginate?.page || 1, query.paginate?.['per-page'] || 100, ) } async get( type: string | string[] | null, criteria: RawCriteria, projection: string[], sort?: Record, page = 1, limit = 100, atTime?: string, queryParams?: Record, options?: ListingOptions, preventDefaultArchive?: boolean, ): Promise> { const projectionBase: RawListingQuery['projection'] = { [ELEMENT_NAME]: `$${ELEMENT_NAME}.value`, } const projectionObject = projection?.reduce((acc, attribute) => { if (!attribute?.length) { return acc } acc[attribute] = `$${attribute}.value` return acc }, projectionBase) || projectionBase const paginateObject = { page: page, 'per-page': limit, } const query: RawListingQuery = { collection: type, criteria: criteria, projection: projectionObject, paginate: paginateObject, sort: sort || { timestamp: -1, }, atTime, preventDefaultArchive, } const data: ListingRequestData = { query: query, element: {}, } const headers: ListingHeaders = {} if (options?.allowAsyncListing !== undefined) { headers['x-allow-async-listing'] = options.allowAsyncListing ? 'true' : 'false' } if (options?.syncChildren !== undefined) { headers['x-sync-children'] = options.syncChildren ? 'true' : 'false' } return this.api.apisauce.post( `${ENDPOINTS.MEMBRANE}/search/query?links=false`, data, { params: queryParams, headers, }, ) } /// Needs to be refactored - too many arguments and assumes knowing total count, Epick async loadAllElements( totalCount: number, criteria: RawCriteria, type: string | string[] | null, projection: string[], options: IQueryOptions, onLoadCurrentPage?: ( rows: ElementDataWrapper[], pagesToLeft?: number, ) => Promise, queryParams?: Record, ): Promise { const MAX_TOTAL_COUNT = 25000 const trimmedTotalCount = totalCount > MAX_TOTAL_COUNT ? MAX_TOTAL_COUNT : totalCount const pagesToLoadCount = Math.ceil(trimmedTotalCount / +options.paginate['per-page']) - options.paginate.page + 1 const pagesToLoad = numberRange(pagesToLoadCount, 1) if (totalCount > MAX_TOTAL_COUNT) { console.error('too many elements') } const loadedRowsReducer = async ( previousRows: ElementDataWrapper[], currentPageRows: ElementDataWrapper[], pagesToLeft?: number, ) => { const allRows = previousRows.concat(currentPageRows) if (onLoadCurrentPage) await onLoadCurrentPage(currentPageRows, pagesToLeft) return allRows } const sequencedReduce = await sequencedReducePromises< number, ElementDataWrapper[], ElementDataWrapper[] >( pagesToLoad, // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-ignore async (page) => { const getOnePage = async (): Promise< ElementDataWrapper[] | undefined > => { const resp = await this.get( type, criteria, projection, options.sort, page, options.paginate['per-page'], options.atTime, queryParams, ) if (!resp.ok) { return getOnePage() } return resp.data } const resp = await getOnePage() return resp }, loadedRowsReducer, Promise.resolve([]), 3, // number of parallel calls ) // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-ignore return sequencedReduce } parseDynamicFilter( rawValue: unknown, context?: DynamicContext, ): RawCriterium { if (!isString(rawValue) || !context) { return rawValue as RawCriterium } const dynamicValue = parseDynamicValue(rawValue, context, { allowObjectNotation: true, allowUnparsedOutput: true, }) return parseTimeRange(dynamicValue, context) } } export function getListingConfigFromMongoQuery( query: RawListingQuery | ListingQuery, ): IListingConfig { const criteria = query.criteria const projection = Object.keys(omit(query.projection, ['_name'])) const page = query.paginate?.page const limit = query.paginate?.['per-page'] projection.push('element_name') return { ...omit(query, ['collection']), type: query.collection ?? null, criteria: criteria as unknown as QueryCriteria, projection, limit, page, } } function sequencedReducePromises( inputArray: Array, fn: PromiseReturningFn, reducerFn: Reducer, initialValue: Promise, parallelCalls = 1, ) { const inputChunks = chunk(inputArray, parallelCalls) let pagesToLeft = inputArray.length // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-ignore: moved from old FE return inputChunks.reduce(async (promise, inputs) => { const results = await promise const thisChunkResults = await Promise.all(inputs.map(fn)) pagesToLeft = pagesToLeft - parallelCalls // Reduce results of currently resolved parallel chunk return thisChunkResults.reduce( async (results, nextResult) => // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-ignore: moved from old FE reducerFn(await results, nextResult, pagesToLeft), Promise.resolve(results), ) }, Promise.resolve(initialValue)) } type PromiseReturningFn = (input: TInput) => Promise type Reducer = ( result: T, element: TElement, currentPage?: TNumber, ) => Promise /** * Creates array range of numbers with given offset from 0 for the first element. * @param number Number of elements in the range * @param offset value of first item (0 by default) */ function numberRange(number: number, offset = 0) { return Array.from(Array(number), (_, i) => i + offset) }