import { useState } from 'react'; import { motion, AnimatePresence } from 'framer-motion'; import { Icon } from '../shared/Icon'; interface Incident { title: string; severity: 'P1' | 'P2' | 'P3'; symptoms: string[]; diagnosis: string[]; resolution: string[]; escalation: string; } const SEVERITY_COLORS = { P1: { color: '#ef4444', bg: 'rgba(239,68,68,0.1)', border: 'rgba(239,68,68,0.2)' }, P2: { color: '#f59e0b', bg: 'rgba(245,158,11,0.1)', border: 'rgba(245,158,11,0.2)' }, P3: { color: '#60a5fa', bg: 'rgba(96,165,250,0.1)', border: 'rgba(96,165,250,0.2)' }, }; const INCIDENTS: Incident[] = [ { title: 'Optimistic Lock Failures Spike', severity: 'P2', symptoms: [ 'High rate of modifiedCount: 0 in logs', 'Booking statuses appear stuck or outdated', 'Users report stale booking UI', ], diagnosis: [ 'Check for concurrent request patterns (multiple webhooks firing simultaneously)', 'Verify MongoDB connection pool is not exhausted', 'Check if any deployment introduced a new middleware that reads/writes booking_status', ], resolution: [ 'The system is designed to skip silently on lock failures — this is safe by design', 'If persistent: check for infinite retry loops in upstream services', 'Monitor BullMQ job queue for backed-up jobs that might be causing contention', ], escalation: 'Backend team lead → Staff engineer if rate exceeds 5% of requests', }, { title: 'BullMQ Jobs Stuck / Not Processing', severity: 'P1', symptoms: [ 'Riders not receiving confirmation notifications', 'Timeouts not firing — expired bookings remain active', 'BullMQ dashboard shows jobs in "waiting" state indefinitely', ], diagnosis: [ 'Check Redis connectivity (BullMQ depends on Redis)', 'Verify worker processes are running (pm2 list or k8s pod status)', 'Check ENABLE_BOOKING_JOBS flag is true in the environment', ], resolution: [ 'Restart BullMQ worker pods/processes', 'If Redis is down: restore from backup, jobs will auto-retry on reconnect', 'For stuck individual jobs: use BullMQ admin to manually remove and reschedule', ], escalation: 'Immediate: on-call backend engineer → infrastructure team if Redis related', }, { title: 'DAC Card Orphans', severity: 'P3', symptoms: [ 'Driver app shows stale action cards for cancelled/completed trips', 'DAC delete Pub/Sub messages failing silently', ], diagnosis: [ 'Check GCP Pub/Sub topic health and subscription backlog', 'Verify PUB_SUB_DRIVER_ACTIONS_TOPIC environment variable', 'Check DAC service logs for processing errors', ], resolution: [ 'DAC operations are fire-and-forget — orphans resolve when driver refreshes app', 'For bulk cleanup: run a script that queries terminal-status trips and publishes delete messages', 'Long-term: add a TTL-based auto-cleanup in DAC service', ], escalation: 'Driver platform team if affecting > 100 drivers', }, { title: 'Notification Delivery Failures', severity: 'P2', symptoms: [ 'Riders/drivers not receiving push notifications', 'sendBookingNotificationWithConfig errors in logs', 'FCM/APNS error codes in notification service logs', ], diagnosis: [ 'Check FCM/APNS service status', 'Verify device tokens are valid and not expired', 'Check notification service rate limits', ], resolution: [ 'Notifications are fire-and-forget — core booking flow continues regardless', 'For critical notifications: trigger manual re-send through ops dashboard', 'If FCM key expired: rotate in GCP console and redeploy', ], escalation: 'Notifications team → Mobile platform team if device token issues', }, ]; export function CriticalRunbookSection() { const [expandedIndex, setExpandedIndex] = useState(null); return (
{INCIDENTS.map((incident, i) => { const isExpanded = expandedIndex === i; const sc = SEVERITY_COLORS[incident.severity]; return (
{isExpanded && (
{[ { title: 'Symptoms', items: incident.symptoms, icon: 'warning', color: '#fbbf24' }, { title: 'Diagnosis', items: incident.diagnosis, icon: 'search', color: '#60a5fa' }, { title: 'Resolution', items: incident.resolution, icon: 'build', color: '#22c55e' }, ].map((section) => (
{section.title}
    {section.items.map((item, j) => (
  • - {item}
  • ))}
))}
Escalation: {incident.escalation}
)}
); })}
); }