/* eslint-disable max-len */
/* eslint-disable @typescript-eslint/no-explicit-any */
/* eslint-disable @typescript-eslint/no-non-null-assertion */
import stableStringify from 'fast-json-stable-stringify';
import { CalculationSpec, DataSourceSpec, FieldSpec, LookUpRefSpec, LookUpSpec, ScopeSpec, WebApiSpec } from '../schema/WebApiSchema.js';
import { Config } from '../types/Config.js';
import { PagingState } from '../types/PagingState.js';
import { Plugin } from '../types/Plugin.js';
import { PluginContext } from '../types/PluginContext.js';
import { DataSource } from '../types/ReadDataSourceEvent.js';
import { SqupContext } from '../types/SqupContext.js';
import { Timeframe } from '../types/Timeframe.js';
import { CallApi } from './ApiFetcher.js';
import { PerformCalculation } from './Calculations.js';
import { CallLookUp } from './CallLookup.js';
import { EvaluateCondition } from './EvaluateCondition.js';
import { getArray } from './GetArray.js';
import { referenceData } from './Render.js';
import { CompressTimeSeries } from './TimeSeries.js';
import { FormatTimeframe, handleSimpleField } from './Util.js';

export async function ReadDataSource(
    plugin: Plugin,
    webapiSpec: WebApiSpec,
    config: Config,
    dataSource: DataSource,
    dataSourceConfig: Record<string, any>,
    targetNodes: any[] | undefined,
    timeframe: Timeframe,
    squpContext: SqupContext
): Promise<any|any[]> {
    squpContext.log.debug(`ReadDataSource starts: dataSource='${JSON.stringify(dataSource, null, 4)}', webapiSpec='${JSON.stringify(webapiSpec, null, 4)}', dataSourceConfig='${JSON.stringify(dataSourceConfig, null, 4)}', `);

    // Set up plugin context
    const lookUps: LookUpSpec[] = Array.isArray(webapiSpec.lookUps) ? webapiSpec.lookUps : [];
    const apiDefinitions = (Array.isArray(webapiSpec.api) ? webapiSpec.api : []).reduce(
        (acc, val) => acc.set(val.name, val),
        new Map()
    );
    const pluginContext: PluginContext = {
        plugin,
        config,
        apiDefinitions,
        lookUps: lookUps.reduce((acc, val) => acc.set(val.name, val), new Map()),
        lookUpCache: new Map(),
        dataTransforms: webapiSpec.dataTransforms ?? []
    };

    // Find the definition for this data source in the webapi definition
    const dataSourceSpec = webapiSpec.dataSources.find((d) => d.name === dataSource.name);
    if (!dataSourceSpec) {
        throw new Error(`No WebAPI spec found for data source ${dataSource.name}`);
    }
    if (dataSourceSpec.supportedScope !== dataSource.supportedScope) {
        throw new Error(`WebAPI spec for data source ${dataSource.name} has supportedScope = '${dataSourceSpec.supportedScope}' - the data source has '${dataSource.supportedScope}'`);
    }

    const startTime = new Date();
    const startTimeMS = startTime.getTime();
    let replacements: Record<string, any> = {
        dataSourceConfig,
        now: {
            mSecs: startTimeMS,
            Unix: Math.floor(startTimeMS / 1000),
            ISO: startTime.toISOString()
        }
    };

    // Set replacements for timeframe
    replacements.timeframe = FormatTimeframe(timeframe, dataSourceSpec.timeframe);

    // Handle the distinct scope types...
    switch (dataSource.supportedScope) {
        case 'single': {
            if (!Array.isArray(targetNodes) || targetNodes.length !== 1) {
                throw new Error(`Data Source ${dataSource.name} has supportedScope=${dataSource.supportedScope} but received bad targetNodes`);
            }
            replacements.targetNode = targetNodes[0];
            replacements = await PerformPreCalcs(pluginContext, dataSourceSpec, replacements, squpContext);
            replacements.targetNodeIdsBySourceId = new Map([[targetNodes[0].sourceId[0], targetNodes[0].id]]);
            break;
        }
        case 'list': {
            if (!Array.isArray(targetNodes)) {
                throw new Error(`Data Source ${dataSource.name} has supportedScope=${dataSource.supportedScope} but received bad targetNodes`);
            }

            // Group target nodes by sourceType
            const targetNodesBySourceType = targetNodes.reduce(
                (acc: Map<string, any[]>, val: any) => acc.set(val.sourceType[0], [val, ...(acc.get(val.sourceType[0]) ?? [])]),
                new Map<string, any[]>());

            // Fire off the pre-calcs for each group of target nodes
            const promises = Array.from(targetNodesBySourceType.entries()).map(([key, val]) => {
                const repl = {
                    ...replacements,
                    scope: getReplacementsForListScope(dataSourceSpec.scope, val, squpContext)
                };
                return PerformPreCalcs(pluginContext, dataSourceSpec, repl, squpContext);
            });

            const settled = await Promise.allSettled([promises].flat());
            const failures = settled.filter((p) => p.status === 'rejected');
            if (failures.length > 0) {
                squpContext.report.error(`Data stream read failed: '${JSON.stringify(failures.map((f: any) => f.reason))}'`);
            }
        
            for (const r of settled.filter((p) => p.status === 'fulfilled')) {
                const updatedReplacements = (r as any).value as Record<string, any>;
                replacements = mergeReplacements(replacements, updatedReplacements);
            }

            // The main API gets the whole (possibly heterogenous) targetNodes array
            replacements.targetNodes = getReplacementsForListScope(dataSourceSpec.scope, targetNodes, squpContext);
            replacements.targetNodeIdsBySourceId = targetNodes.reduce((acc, targetNode) => acc.set(targetNode.sourceId[0], targetNode.id), new Map());
            break;
        }
        default: {
            replacements = await PerformPreCalcs(pluginContext, dataSourceSpec, replacements, squpContext);
            break;
        }
    }

    let rows: Record<string, any>[] = [];
    const pagingState: PagingState = { complete: false, recordCount: 0 };

    let timestampProperty = '';
    if (dataSourceSpec.api) {
        // Timeframe handling
        let outOfRangeCount = 0;
        if (dataSourceSpec.timeframe?.format) {
            // See if there's a nominated column for timestamp (in which case we'll filter out out-of-range rows)
            const timestampField = dataSourceSpec.fields.find((f: FieldSpec) => f.isTimestamp);
            if (timestampField) {
                timestampProperty = timestampField.name;
            }
        }

        let lowestTimestamp = Number.MAX_SAFE_INTEGER;
        let highestTimestamp = Number.MIN_SAFE_INTEGER;

        // Distinct col handling
        const distinctValuesSeen = new Set<string>();
        const distinctColNames = Array.isArray(dataSourceSpec.fields)
            ? dataSourceSpec.fields.filter((f) => f.distinct).map((f) => f.name)
            : [];

        squpContext.log.debug(`distinctColNames: ${JSON.stringify(distinctColNames)}`);
        const isDup = (row: Record<string, any>) => {
            if (distinctColNames.length <= 0) {
                // Never a dup if no distinct cols
                return false;
            }
            const val: Record<string, any> = {};
            for (const distinctColName of distinctColNames) {
                val[distinctColName] = row[distinctColName];
            }
            const valKey = stableStringify(val);
            if (distinctValuesSeen.has(valKey)) {
                return true;
            }
            distinctValuesSeen.add(valKey);
            return false;
        };

        const pushResult = (row: Record<string, any>) => {
            if (timestampProperty) {
                const timestamp = row[timestampProperty];
                if (isNaN(timestamp)) {
                    squpContext.log.debug('Row had bad timestamp', { row });
                    outOfRangeCount++;
                } else {
                    lowestTimestamp = Math.min(lowestTimestamp, timestamp);
                    highestTimestamp = Math.max(highestTimestamp, timestamp);
                    if (timestamp < timeframe.unixStart || timestamp > timeframe.unixEnd) {
                        outOfRangeCount++;
                    } else {
                        if (!isDup(row)) {
                            rows.push(row);
                        }
                    }
                }
            } else {
                if (!isDup(row)) {
                    rows.push(row);
                }
            }
        };

        while (
            !pagingState.complete &&
            (
                typeof squpContext.canContinue !== 'function' ||
                squpContext.canContinue()
            )
        ) {
            const { data } = await CallApi(pluginContext, dataSourceSpec.api, pagingState, squpContext, replacements);

            if (dataSourceSpec.api.isScalar) {
                if (Array.isArray(data)) {
                    throw new Error('Got array when expecting scalar');
                }
                const row = await CreateRow(pluginContext, dataSourceSpec, squpContext, data, replacements);
                if (row) {
                    pushResult(row);
                }
            } else {
                const dataArray = Array.isArray(data) ? data : [data];
                for (const item of dataArray) {
                    const row = await CreateRow(pluginContext, dataSourceSpec, squpContext, item, replacements);
                    if (row) {
                        pushResult(row);
                    }
                }
            }
        }
        if (!pagingState.complete) {
            squpContext.report.warning('This data set is incomplete');
        }

        squpContext.log.debug(`timeframe is ${timeframe.unixStart}-${timeframe.unixEnd}, lowestTimestamp is ${lowestTimestamp}, highestTimestamp is ${highestTimestamp}`);
        if (outOfRangeCount > 0) {
            squpContext.log.debug(`API returned ${outOfRangeCount} rows outside requested date range`);
        }
    } else {
        // Create single row from lookup data only
        const row = await CreateRow(pluginContext, dataSourceSpec, squpContext, {}, replacements);
        if (row) {
            rows.push(row);
        }
    }

    // Check for result filtering
    if (Object.hasOwnProperty.call(dataSourceSpec, 'filterResults')) {
        rows = rows.filter((row) => {
            const keep = EvaluateCondition(pluginContext, squpContext, dataSourceSpec.filterResults!.condition, { ...replacements, row });
            return keep;
        });
    }

    if (rows.length > 0 && timestampProperty && dataSourceSpec.timeframe &&
        Array.isArray(dataSourceSpec.timeframe.summarizeColumns) &&
        dataSourceSpec.timeframe.summarizeColumns.length > 0) {
        const summarizeColumns = dataSourceSpec.timeframe.summarizeColumns.map((c) => referenceData(c, pluginContext, replacements));
        const dateFormat = dataSourceSpec.timeframe.format === 'Unix' ? 'Unix' : 'ISO-8601';
        rows = CompressTimeSeries(
            rows,
            timestampProperty,
            squpContext,
            {
                interval: timeframe.interval,
                dateFormat,
                valueColumn: summarizeColumns
            }
        );
    }

    // Check for dynamic column metadata
    const metadataFieldNames = new Set(['displayName', 'visible', 'include', 'shape', 'role' ]);
    if (dataSourceSpec.fields.some((f) => Object.keys(f).some((k) => metadataFieldNames.has(k)))) {
        const metadata = dataSourceSpec.fields.reduce((acc, val) => {
            acc.push({
                name: val.name,
                displayName: val.displayName,
                visible: val.visible,
                include: val.include,
                shape: val.shape,
                role: val.role
            });
            return acc;
        }, [] as any[]);

        // return the results in the special format the platform uses to accept dynamic column metadata
        return {
            result: 'rawData',
            data: rows,
            metadata
        };
    } else {

        squpContext.log.debug(`ReadDataSource returns: ${JSON.stringify(rows, null, 4)}`);
        // return vanilla results with no column metadata
        return rows;
    }
}

async function PerformPreCalcs(
    pluginContext: PluginContext,
    dataSourceSpec: DataSourceSpec,
    replacements: Record<string, any>,
    squpContext: SqupContext
): Promise<Record<string, any>> {
    
    if (Array.isArray(dataSourceSpec.preCalc)) {
        for (const preStep of dataSourceSpec.preCalc) {
            if (typeof squpContext.canContinue === 'function' && !squpContext.canContinue()) {
                break;
            }
            if (preStep.condition) {
                const r = EvaluateCondition(pluginContext, squpContext, preStep.condition, replacements);
                if (!r) {
                    continue;
                }
            }
            if ((preStep as any).lookUp) {
                const lookUpRef = preStep as any as LookUpRefSpec;
                replacements[lookUpRef.name] = await CallLookUp(pluginContext, squpContext, lookUpRef, replacements);

            } else {
                const calculation = preStep as any as CalculationSpec;
                replacements[calculation.name] = PerformCalculation(pluginContext, squpContext, calculation.expression, replacements);
            }
        }
    }
    return replacements;
}

/**
 * Merges an replacements object potentially updated by sourceType-specific pre-calcs into the
 * main replacements object to be passed to the data stream main API.
 * 
 * @param replacements 
 * @param updatedReplacements 
 */
function mergeReplacements(
    replacements: Record<string, any>,
    updatedReplacements: Record<string, any>
): Record<string, any> {
    const result = { ...replacements };
    for (const [key, val] of Array.from(Object.entries(updatedReplacements))) {
        if (['dataSourceConfig', 'targetNodes', 'timeframe'].includes(key)) {
            // Leave our standard replacement values alone
            continue;
        }
        if (Object.hasOwn(result, key)) {
            if (Array.isArray(result[key])) {
                const merged = new Set(result[key]);
                for (const item of (Array.isArray(val) ? val : [val])) {
                    merged.add(item);
                }
                result[key] = Array.from(merged);

            } else {
                // We aren't clever with non-array values, it's a case of last wins!
                result[key] = val;
            }

        } else {
            result[key] = val;
        }
    }
    return result;
}

async function CreateRow(
    pluginContext: PluginContext,
    dataSourceSpec: DataSourceSpec,
    squpContext: SqupContext,
    object: Record<string, any> = {},
    replacements: Record<string,any> = {}): Promise<Record<string, any> | undefined> {
    const result = {};

    const repl: Record<string,any> = {
        ...replacements,
        object
    };

    for (const field of dataSourceSpec.fields) {
        if (Object.hasOwnProperty.call(field, 'condition') && field.condition) {
            const condition = EvaluateCondition(pluginContext, squpContext, field.condition, repl);
            if (condition === false) {
                continue;
            }
        }

        if (Array.isArray(field.calc)) {
            for (const step of field.calc) {
                if (step.condition) {
                    const r = EvaluateCondition(pluginContext, squpContext, step.condition, repl);
                    if (!r) {
                        continue;
                    }
                }    
                if ((step as any).lookUp) {
                    const lookUpRef = step as any as LookUpRefSpec;
                    repl[lookUpRef.name] = await CallLookUp(pluginContext, squpContext, lookUpRef, repl);
    
                } else {
                    const calculation = step as any as CalculationSpec;
                    repl[calculation.name] = PerformCalculation(pluginContext, squpContext, calculation.expression, repl);
                }
            }
        }
    
        handleSimpleField(pluginContext, field, result, repl);
    }
    return result;
}


function getReplacementsForListScope(
    scope: ScopeSpec | undefined,
    targetNodes: any[],
    _squpContext: SqupContext
): Record<string, any> {
    const result: Record<string, any> = {};

    result.idArray = targetNodes.map((tn) => getArray(tn, `${scope?.idPath ?? 'sourceId'}.0`));
    result.idArrayString = result.idArray.join(scope?.idSeparator ?? ',');
    result.targetNodes = [...targetNodes];
    if (targetNodes.length > 0) {
        result.sourceType = targetNodes[0].sourceType[0];
        
        // Target nodes updated with a first and a last flag to allow some tricks with mustache
        result.targetNodes[0] = {
            ...targetNodes[0],
            first: true
        };
        result.targetNodes[targetNodes.length - 1] = {
            ...targetNodes[targetNodes.length - 1],
            last: true
        };
    }
    return result;
}