{
  "id": "databricks-lakeflow-pipeline-engineering-agent",
  "name": "Databricks Lakeflow Pipeline Engineering Agent",
  "version": "0.1.0",
  "type": "agent",
  "provider": "databricks",
  "harnesses": [
    "codex",
    "copilot",
    "claude-code",
    "cursor",
    "gemini",
    "kiro"
  ],
  "summary": "Static review of Lakeflow Spark Declarative Pipelines (formerly Delta Live Tables) design: medallion layering strategy, Lakeflow Jobs orchestration and task dependencies, Delta table layout (liquid clustering vs Z-order vs partitioning, deletion vectors, Predictive Optimization), Auto Loader versus COPY INTO choice, schema evolution and `_rescued_data` handling, materialized view versus streaming table selection, and backfill strategy. Reads pipeline source, table metadata, and job definitions only.",
  "source_type": "original",
  "official_docs": [
    "https://docs.databricks.com/aws/en/ldp",
    "https://docs.databricks.com/aws/en/ldp/concepts/where-is-dlt",
    "https://docs.databricks.com/aws/en/delta-live-tables/develop",
    "https://docs.databricks.com/aws/en/delta-live-tables/configure-pipeline",
    "https://docs.databricks.com/aws/en/tables/clustering",
    "https://docs.databricks.com/aws/en/delta/optimize",
    "https://docs.databricks.com/aws/en/optimizations/predictive-optimization",
    "https://docs.databricks.com/aws/en/ingestion/cloud-object-storage/auto-loader/",
    "https://docs.databricks.com/aws/en/lakehouse/medallion"
  ],
  "security_notes": "Static review only — reads pipeline source files, table metadata, job definitions, and medallion architecture design; never executes pipelines, never runs queries, never accesses customer data or production workspaces, and never requests credentials, tokens, storage keys, or workspace URLs. A claim about a feature's availability (Liquid Clustering, Predictive Optimization on a given tier) is verified against the official docs, not assumed from product marketing.",
  "last_verified": "2026-08-17",
  "path": "agents/databricks/databricks-lakeflow-pipeline-engineering-agent/",
  "harness_variants": {
    "codex": "agents/databricks/databricks-lakeflow-pipeline-engineering-agent/harnesses/codex.toml",
    "copilot": "agents/databricks/databricks-lakeflow-pipeline-engineering-agent/harnesses/copilot.agent.md",
    "claude-code": "agents/databricks/databricks-lakeflow-pipeline-engineering-agent/harnesses/claude-code.agent.md",
    "cursor": "agents/databricks/databricks-lakeflow-pipeline-engineering-agent/harnesses/cursor.agent.md",
    "gemini": "agents/databricks/databricks-lakeflow-pipeline-engineering-agent/harnesses/gemini.agent.md",
    "kiro-ide": "agents/databricks/databricks-lakeflow-pipeline-engineering-agent/harnesses/kiro-ide.agent.md",
    "kiro-cli": "agents/databricks/databricks-lakeflow-pipeline-engineering-agent/harnesses/kiro-cli.agent.json"
  },
  "companion_skills": [
    "databricks-lakeflow-pipeline-engineering"
  ],
  "execution_tier": "static-review",
  "lifecycle": "experimental",
  "author": "github: VincentChuWaiChow",
  "routing_keywords": [
    "lakeflow",
    "declarative pipeline",
    "delta live tables",
    "medallion",
    "auto loader",
    "copy into",
    "schema evolution",
    "liquid clustering",
    "z-order",
    "partitioning",
    "materialized view",
    "streaming table",
    "predictive optimization",
    "deletion vectors",
    "backfill"
  ]
}