{
  "name": "long-context-data-pipeline",
  "description": "Complete a multi-step data pipeline: read CSV, clean, join, aggregate, pivot, and write output",
  "dataset": "terminal-bench-local",
  "difficulty": "medium",
  "instruction": "You are given two CSV files (sales.csv and products.csv) and a partially-written pipeline.py. Complete ALL the TODO sections in pipeline.py to build a data pipeline that:\n\n1. Reads both CSV files\n2. Cleans the sales data (handle missing values, fix date formats, remove duplicates)\n3. Joins sales with products on product_id\n4. Aggregates total revenue by category and month\n5. Creates a pivot table: rows=category, columns=month, values=total revenue\n6. Writes the final pivot table to output.csv\n\nDo NOT use pandas - use only the csv module and standard library. The pipeline must be completed by filling in the TODO sections. Run python3 test_pipeline.py to verify.\n\nIMPORTANT: Look at the CSV data carefully. There are intentional data quality issues in sales.csv that your cleaning step must handle:\n- Some rows have empty quantity fields (skip these rows)\n- Some dates are in MM/DD/YYYY format instead of YYYY-MM-DD (normalize all to YYYY-MM-DD)\n- There are duplicate rows (same sale_id) that must be removed",
  "setup_files": {
    "sales.csv": "sale_id,date,product_id,quantity,unit_price\n1001,2024-01-15,P001,5,29.99\n1002,2024-01-20,P002,3,49.99\n1003,01/25/2024,P003,10,9.99\n1004,2024-01-28,P001,2,29.99\n1005,2024-02-03,P004,7,19.99\n1006,02/10/2024,P002,1,49.99\n1007,2024-02-14,P005,4,39.99\n1008,2024-02-20,P003,6,9.99\n1009,2024-02-25,P001,3,29.99\n1010,2024-03-01,P004,,19.99\n1011,2024-03-05,P005,2,39.99\n1012,03/10/2024,P002,5,49.99\n1013,2024-03-15,P003,8,9.99\n1014,2024-03-20,P001,4,29.99\n1015,2024-03-25,P004,6,19.99\n1016,2024-01-15,P002,2,49.99\n1017,01/30/2024,P005,3,39.99\n1018,2024-02-08,P001,,29.99\n1019,2024-02-18,P004,5,19.99\n1020,2024-03-28,P003,12,9.99\n1021,2024-03-05,P005,2,39.99\n1003,01/25/2024,P003,10,9.99\n1022,2024-01-10,P004,3,19.99\n1023,2024-02-28,P002,4,49.99\n1024,03/22/2024,P005,1,39.99\n",
    "products.csv": "product_id,name,category\nP001,Widget Alpha,Electronics\nP002,Gadget Beta,Electronics\nP003,Bolt Standard,Hardware\nP004,Nail Pack,Hardware\nP005,Cable Premium,Accessories\n",
    "pipeline.py": "\"\"\"Data pipeline: CSV -> clean -> join -> aggregate -> pivot -> output.\n\nComplete all TODO sections. Use only the csv module and standard library.\n\"\"\"\n\nimport csv\nfrom collections import defaultdict\n\n\ndef read_csv(filepath):\n    \"\"\"Read a CSV file and return a list of dicts.\"\"\"\n    # TODO: Implement this function\n    # Read the CSV file at `filepath` and return a list of dictionaries\n    # where each dict represents a row with column headers as keys.\n    pass\n\n\ndef clean_sales(sales_rows):\n    \"\"\"Clean the sales data.\n    \n    Must handle:\n    1. Remove rows where quantity is empty or missing\n    2. Normalize dates: convert MM/DD/YYYY to YYYY-MM-DD format\n    3. Remove duplicate rows (same sale_id)\n    4. Convert quantity to int and unit_price to float\n    \n    Returns a list of cleaned dicts.\n    \"\"\"\n    # TODO: Implement this function\n    pass\n\n\ndef join_data(sales, products):\n    \"\"\"Join sales with products on product_id.\n    \n    Each resulting row should have all sales fields plus 'name' and 'category'\n    from the matching product.\n    \n    Returns a list of joined dicts.\n    \"\"\"\n    # TODO: Implement this function\n    pass\n\n\ndef aggregate_revenue(joined_data):\n    \"\"\"Aggregate total revenue by category and month.\n    \n    Revenue for a row = quantity * unit_price\n    Month should be extracted as YYYY-MM from the date field.\n    \n    Returns a dict of {(category, month): total_revenue}\n    \"\"\"\n    # TODO: Implement this function\n    pass\n\n\ndef pivot_table(aggregated):\n    \"\"\"Create a pivot table from aggregated data.\n    \n    Rows = categories (sorted alphabetically)\n    Columns = months (sorted chronologically)\n    Values = total revenue (rounded to 2 decimal places)\n    \n    Returns:\n        headers: list of strings ['category', month1, month2, ...]\n        rows: list of lists [[category_name, val1, val2, ...], ...]\n    \"\"\"\n    # TODO: Implement this function\n    pass\n\n\ndef write_csv(filepath, headers, rows):\n    \"\"\"Write the pivot table to a CSV file.\"\"\"\n    # TODO: Implement this function\n    # Write headers as the first row, then each data row.\n    # Revenue values should be formatted to 2 decimal places.\n    pass\n\n\ndef main():\n    \"\"\"Run the full pipeline.\"\"\"\n    # Step 1: Read data\n    sales = read_csv('sales.csv')\n    products = read_csv('products.csv')\n    \n    # Step 2: Clean sales data\n    cleaned = clean_sales(sales)\n    \n    # Step 3: Join\n    joined = join_data(cleaned, products)\n    \n    # Step 4: Aggregate\n    aggregated = aggregate_revenue(joined)\n    \n    # Step 5: Pivot\n    headers, rows = pivot_table(aggregated)\n    \n    # Step 6: Write output\n    write_csv('output.csv', headers, rows)\n    \n    print(f\"Pipeline complete. Wrote {len(rows)} categories x {len(headers)-1} months to output.csv\")\n\n\nif __name__ == '__main__':\n    main()\n",
    "test_pipeline.py": "\"\"\"Verify the data pipeline output.\"\"\"\nimport sys\nimport os\nimport csv\nimport subprocess\n\nerrors = []\n\n# Run the pipeline\nresult = subprocess.run([sys.executable, 'pipeline.py'], capture_output=True, text=True, timeout=15)\nif result.returncode != 0:\n    print(f\"FAIL: pipeline.py crashed: {result.stderr}\")\n    sys.exit(1)\n\n# Check output.csv exists\nif not os.path.exists('output.csv'):\n    print(\"FAIL: output.csv was not created\")\n    sys.exit(1)\n\n# Read output\nwith open('output.csv', 'r') as f:\n    reader = csv.reader(f)\n    rows = list(reader)\n\nif len(rows) < 2:\n    print(f\"FAIL: output.csv has {len(rows)} rows, expected at least 4 (header + 3 categories)\")\n    sys.exit(1)\n\nheaders = rows[0]\ndata_rows = rows[1:]\n\n# Check headers\nif headers[0].strip().lower() != 'category':\n    errors.append(f\"First column should be 'category', got '{headers[0]}'\")\n\nexpected_months = ['2024-01', '2024-02', '2024-03']\nactual_months = [h.strip() for h in headers[1:]]\nif actual_months != expected_months:\n    errors.append(f\"Expected months {expected_months}, got {actual_months}\")\n\n# Check we have 3 categories\nif len(data_rows) != 3:\n    errors.append(f\"Expected 3 category rows, got {len(data_rows)}\")\n\n# Build a lookup for easy checking\noutput_data = {}\nfor row in data_rows:\n    cat = row[0].strip()\n    vals = [float(v) for v in row[1:]]\n    output_data[cat] = dict(zip(expected_months, vals))\n\n# Now compute expected values manually.\n# After cleaning (remove empty qty, normalize dates, remove dupes):\n#\n# Valid sales after cleaning:\n# 1001: 2024-01-15, P001, 5, 29.99 -> Electronics, 149.95\n# 1002: 2024-01-20, P002, 3, 49.99 -> Electronics, 149.97\n# 1003: 2024-01-25, P003, 10, 9.99 -> Hardware, 99.90 (deduped)\n# 1004: 2024-01-28, P001, 2, 29.99 -> Electronics, 59.98\n# 1005: 2024-02-03, P004, 7, 19.99 -> Hardware, 139.93\n# 1006: 2024-02-10, P002, 1, 49.99 -> Electronics, 49.99\n# 1007: 2024-02-14, P005, 4, 39.99 -> Accessories, 159.96\n# 1008: 2024-02-20, P003, 6, 9.99 -> Hardware, 59.94\n# 1009: 2024-02-25, P001, 3, 29.99 -> Electronics, 89.97\n# 1010: SKIP (empty quantity)\n# 1011: 2024-03-05, P005, 2, 39.99 -> Accessories, 79.98 (deduped with 1021)\n# 1012: 2024-03-10, P002, 5, 49.99 -> Electronics, 249.95\n# 1013: 2024-03-15, P003, 8, 9.99 -> Hardware, 79.92\n# 1014: 2024-03-20, P001, 4, 29.99 -> Electronics, 119.96\n# 1015: 2024-03-25, P004, 6, 19.99 -> Hardware, 119.94\n# 1016: 2024-01-15, P002, 2, 49.99 -> Electronics, 99.98\n# 1017: 2024-01-30, P005, 3, 39.99 -> Accessories, 119.97\n# 1018: SKIP (empty quantity)\n# 1019: 2024-02-18, P004, 5, 19.99 -> Hardware, 99.95\n# 1020: 2024-03-28, P003, 12, 9.99 -> Hardware, 119.88\n# 1021: 2024-03-05, P005, 2, 39.99 -> Accessories (DUPE of 1011 by sale_id - but wait, 1021 is different sale_id)\n# Actually 1021 has sale_id=1021 vs 1011 sale_id=1011, these are NOT dupes by sale_id.\n# The only dupe by sale_id is 1003 which appears twice.\n# 1022: 2024-01-10, P004, 3, 19.99 -> Hardware, 59.97\n# 1023: 2024-02-28, P002, 4, 49.99 -> Electronics, 199.96\n# 1024: 2024-03-22, P005, 1, 39.99 -> Accessories, 39.99\n#\n# Accessories:\n#   2024-01: 119.97 (1017)\n#   2024-02: 159.96 (1007)\n#   2024-03: 79.98 + 39.99 = 119.97 (1011+1024)  -- plus 1021: 79.98 -> total 199.95\n#\n# Wait let me recount 1021.\n# 1021 has sale_id=1021, date=2024-03-05, P005, qty=2, price=39.99\n# 1011 has sale_id=1011, date=2024-03-05, P005, qty=2, price=39.99\n# Different sale_ids so both are kept.\n#\n# Accessories:\n#   2024-01: 119.97\n#   2024-02: 159.96\n#   2024-03: 79.98 + 79.98 + 39.99 = 199.95\n#\n# Electronics:\n#   2024-01: 149.95 + 149.97 + 59.98 + 99.98 = 459.88\n#   2024-02: 49.99 + 89.97 + 199.96 = 339.92\n#   2024-03: 249.95 + 119.96 = 369.91\n#\n# Hardware:\n#   2024-01: 99.90 + 59.97 = 159.87\n#   2024-02: 139.93 + 59.94 + 99.95 = 299.82\n#   2024-03: 79.92 + 119.94 + 119.88 = 319.74\n\nexpected = {\n    'Accessories': {'2024-01': 119.97, '2024-02': 159.96, '2024-03': 199.95},\n    'Electronics': {'2024-01': 459.88, '2024-02': 339.92, '2024-03': 369.91},\n    'Hardware': {'2024-01': 159.87, '2024-02': 299.82, '2024-03': 319.74},\n}\n\nfor cat, months in expected.items():\n    if cat not in output_data:\n        errors.append(f\"Missing category '{cat}' in output\")\n        continue\n    for month, exp_val in months.items():\n        if month not in output_data[cat]:\n            errors.append(f\"Missing month '{month}' for category '{cat}'\")\n            continue\n        actual_val = output_data[cat][month]\n        if abs(actual_val - exp_val) > 0.02:\n            errors.append(f\"{cat}/{month}: expected {exp_val:.2f}, got {actual_val:.2f}\")\n\nif errors:\n    print(f\"FAILED with {len(errors)} error(s):\")\n    for e in errors:\n        print(f\"  - {e}\")\n    sys.exit(1)\nelse:\n    print(\"ALL CHECKS PASSED\")\n    print(\"  - Pipeline runs without errors\")\n    print(\"  - output.csv created with correct structure\")\n    print(\"  - 3 categories x 3 months\")\n    print(\"  - All revenue values match expected (within tolerance)\")\n    print(\"  - Data cleaning handled: empty qty, date normalization, deduplication\")\n    sys.exit(0)\n"
  },
  "verify": "cd $BENCH_WORK_DIR && python3 test_pipeline.py",
  "timeout": 360000,
  "tags": ["long-context", "data-pipeline", "csv", "python", "multi-step"]
}