{
    "$schema": "../meta-schema.json",
    "name": "complex-data-pipeline",
    "description": "Demonstrates a complex data processing pipeline combining multiple operations",
    "context": {
        "minConfidence": 0.8,
        "batchSize": 50
    },
    "steps": [
        {
            "name": "getRawData",
            "request": {
                "method": "data.fetch",
                "params": {
                    "source": "sensors",
                    "timeRange": "1h"
                }
            }
        },
        {
            "name": "cleanData",
            "transform": {
                "input": "${getRawData.result}",
                "operations": [
                    {
                        "type": "filter",
                        "using": "${item.confidence} >= ${context.minConfidence}"
                    },
                    {
                        "type": "map",
                        "using": "{...${item}, timestamp: ${item.timestamp}}"
                    }
                ]
            }
        },
        {
            "name": "processBatches",
            "transform": {
                "input": "${cleanData.result}",
                "operations": [
                    {
                        "type": "reduce",
                        "using": "[...${acc}, ${item}]",
                        "initial": []
                    }
                ]
            }
        },
        {
            "name": "analyzeBatches",
            "loop": {
                "over": "${processBatches.result}",
                "as": "batch",
                "step": {
                    "name": "analyzeBatch",
                    "request": {
                        "method": "analysis.process",
                        "params": {
                            "data": "${batch}",
                            "type": "timeseries"
                        }
                    }
                }
            }
        },
        {
            "name": "aggregateResults",
            "transform": {
                "input": "${analyzeBatches.result.value}",
                "operations": [
                    {
                        "type": "filter",
                        "using": "${item} != null"
                    },
                    {
                        "type": "reduce",
                        "using": "[...${acc}, ...${item.result}]",
                        "initial": []
                    }
                ]
            }
        },
        {
            "name": "storeResults",
            "request": {
                "method": "results.store",
                "params": {
                    "data": "${aggregateResults}",
                    "metadata": {
                        "processedAt": "Today",
                        "recordCount": "${aggregateResults.result.length}"
                    }
                }
            }
        }
    ]
}