{
  "id": "gcp-data-pipeline-engineer",
  "name": "GCP Data Pipeline Engineer",
  "type": "skill",
  "provider": "gcp",
  "harnesses": [
    "codex",
    "claude-code",
    "cursor",
    "gemini",
    "kiro",
    "other"
  ],
  "summary": "Design and troubleshoot data pipelines using Dataflow (Apache Beam), Pub/Sub messaging, Dataproc (Spark/Hadoop), Cloud Composer (Apache Airflow), and Dataplex data governance.",
  "source_type": "original",
  "official_docs": [
    "https://cloud.google.com/dataflow/docs/overview",
    "https://cloud.google.com/pubsub/docs/overview",
    "https://cloud.google.com/dataproc/docs/overview",
    "https://cloud.google.com/composer/docs/concepts/overview",
    "https://cloud.google.com/dataplex/docs/introduction"
  ],
  "security_notes": "Dead letter topics are critical for any production Pub/Sub pipeline. Use ephemeral Dataproc clusters for cost efficiency. Pub/Sub delivers at-least-once — design consumers for idempotency.",
  "last_verified": "2026-05-08",
  "path": "skills/gcp/gcp-data-pipeline-engineer",
  "author": "github: VincentChuWaiChow",
  "version": "0.1.0"
}