/**
 * streaming-media.ts — Live media streams (camera/mic/screen) with registry.
 *
 * Agent opens a stream once → gets an ID → refers to it from UI panels or
 * stream_to_transformers loops. Central registry lives on window.__carelessStreams
 * so UIPanelBubble's panel.js can bind <video> elements to streams by id.
 *
 * Also: a frame grabber for pulling periodic snapshots from any stream,
 * and a loop runner that pipes snapshots through a transformers_js pipeline.
 */
import { tool } from '@strands-agents/sdk'
import { z } from 'zod'
import { TASK_REGISTRY, TASK_ALIASES } from './transformers-js'


/* ──────────────────────────────────────────────────────────────────────────
 * Global stream registry on window (so panels can bind by id).
 * ──────────────────────────────────────────────────────────────────────── */

type StreamEntry = {
  id: string
  kind: 'camera' | 'mic' | 'screen' | 'canvas' | 'video-url'
  stream: MediaStream | null
  videoEl?: HTMLVideoElement
  audioEl?: HTMLAudioElement
  url?: string
  createdAt: number
  lastFrameTs: number
  meta?: Record<string, any>
}

declare global {
  interface Window {
    __carelessStreams?: Map<string, StreamEntry>
    __carelessStreamLoops?: Map<string, { stop: () => void }>
    __carelessOverlayCanvases?: Map<string, HTMLCanvasElement>
  }
}

function registry(): Map<string, StreamEntry> {
  if (!window.__carelessStreams) window.__carelessStreams = new Map()
  return window.__carelessStreams
}
function loopRegistry(): Map<string, { stop: () => void }> {
  if (!window.__carelessStreamLoops) window.__carelessStreamLoops = new Map()
  return window.__carelessStreamLoops
}
function overlayRegistry(): Map<string, HTMLCanvasElement> {
  if (!window.__carelessOverlayCanvases) window.__carelessOverlayCanvases = new Map()
  return window.__carelessOverlayCanvases
}

function newId(kind: string): string {
  return `${kind}-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 6)}`
}

async function ensureVideoElement(entry: StreamEntry): Promise<HTMLVideoElement> {
  if (entry.videoEl && entry.videoEl.isConnected) return entry.videoEl
  const v = document.createElement('video')
  v.autoplay = true
  v.muted = true
  v.playsInline = true
  v.style.display = 'none'
  document.body.appendChild(v)
  if (entry.stream) v.srcObject = entry.stream
  else if (entry.url) v.src = entry.url
  await new Promise<void>((r) => {
    if (v.readyState >= 2) return r()
    v.onloadeddata = () => r()
    setTimeout(r, 3000)
  })
  entry.videoEl = v
  return v
}

function captureFrameDataURL(entry: StreamEntry, quality = 0.85, type = 'image/jpeg'): Promise<string> {
  return new Promise(async (resolve, reject) => {
    try {
      const v = await ensureVideoElement(entry)
      const canvas = document.createElement('canvas')
      canvas.width = v.videoWidth || 640
      canvas.height = v.videoHeight || 480
      const ctx = canvas.getContext('2d')
      if (!ctx) return reject(new Error('2D context unavailable'))
      ctx.drawImage(v, 0, 0, canvas.width, canvas.height)
      entry.lastFrameTs = Date.now()
      resolve(canvas.toDataURL(type, quality))
    } catch (e) { reject(e) }
  })
}

/* ──────────────────────────────────────────────────────────────────────────
 * Tools
 * ──────────────────────────────────────────────────────────────────────── */

export const openStreamTool = tool({
  name: 'stream_open',
  description:
    'Open a media stream (camera/mic/screen/video URL) and register it globally. ' +
    'Returns a stream id you can pass to render_video_panel, stream_capture_frame, ' +
    'or stream_to_transformers. Getting camera/mic/screen requires a user gesture — fire ' +
    'this from the click/message handler path.',
  inputSchema: z.object({
    kind: z.enum(['camera', 'mic', 'screen', 'video-url']),
    facingMode: z.enum(['user', 'environment']).optional().describe('camera: "user" (front) or "environment" (rear)'),
    width: z.number().optional(),
    height: z.number().optional(),
    frameRate: z.number().optional(),
    audio: z.boolean().optional().describe('For camera: include audio track. Default false.'),
    url: z.string().optional().describe('For video-url: the video source URL'),
  }),
  callback: async (input) => {
    try {
      const id = newId(input.kind)
      let stream: MediaStream | null = null
      if (input.kind === 'camera') {
        stream = await navigator.mediaDevices.getUserMedia({
          video: {
            facingMode: input.facingMode || 'user',
            width: input.width ? { ideal: input.width } : undefined,
            height: input.height ? { ideal: input.height } : undefined,
            frameRate: input.frameRate ? { ideal: input.frameRate } : undefined,
          },
          audio: input.audio ?? false,
        })
      } else if (input.kind === 'mic') {
        stream = await navigator.mediaDevices.getUserMedia({ audio: true, video: false })
      } else if (input.kind === 'screen') {
        // getDisplayMedia — user prompted to pick a screen/window/tab
        stream = await (navigator.mediaDevices as any).getDisplayMedia({ video: true, audio: false })
      } else if (input.kind === 'video-url') {
        if (!input.url) throw new Error('url required for kind="video-url"')
      }
      const entry: StreamEntry = {
        id,
        kind: input.kind,
        stream,
        url: input.url,
        createdAt: Date.now(),
        lastFrameTs: 0,
      }
      registry().set(id, entry)

      // Provide some metadata about the stream
      const tracks = stream?.getTracks().map((t) => ({
        kind: t.kind, label: t.label, enabled: t.enabled, readyState: t.readyState,
      })) || []
      return JSON.stringify({
        status: 'success',
        id,
        kind: input.kind,
        tracks,
        hint: `Use stream_id="${id}" with render_video_panel / stream_capture_frame / stream_to_transformers.`,
      })
    } catch (err: unknown) {
      return JSON.stringify({ status: 'error', error: (err as Error).message })
    }
  },
})

export const listStreamsTool = tool({
  name: 'stream_list',
  description: 'List all open media streams with their ids, kinds, and activity.',
  inputSchema: z.object({}),
  callback: () => {
    const now = Date.now()
    const streams = Array.from(registry().values()).map((e) => ({
      id: e.id,
      kind: e.kind,
      url: e.url,
      tracks: e.stream?.getTracks().map((t) => ({ kind: t.kind, label: t.label })) || [],
      age_sec: Math.round((now - e.createdAt) / 1000),
      last_frame_ago_sec: e.lastFrameTs ? Math.round((now - e.lastFrameTs) / 1000) : null,
    }))
    const loops = Array.from(loopRegistry().keys())
    return JSON.stringify({
      status: 'success',
      count: streams.length,
      streams,
      active_loops: loops,
    })
  },
})

export const closeStreamTool = tool({
  name: 'stream_close',
  description: 'Stop a media stream and remove it from the registry. Also stops any associated inference loops.',
  inputSchema: z.object({
    id: z.string().optional().describe('Stream id. Omit with all=true to close everything.'),
    all: z.boolean().optional(),
  }),
  callback: (input) => {
    const reg = registry()
    const loops = loopRegistry()
    const closed: string[] = []
    const ids = input.all ? Array.from(reg.keys()) : input.id ? [input.id] : []
    for (const id of ids) {
      const entry = reg.get(id)
      if (!entry) continue
      try {
        entry.stream?.getTracks().forEach((t) => t.stop())
        entry.videoEl?.remove()
        reg.delete(id)
        closed.push(id)
        // Stop any loops targeting this stream
        for (const [lid, lh] of Array.from(loops.entries())) {
          if (lid.includes(id)) { lh.stop(); loops.delete(lid) }
        }
      } catch {}
    }
    return JSON.stringify({ status: 'success', closed, remaining: reg.size })
  },
})

export const captureFrameTool = tool({
  name: 'stream_capture_frame',
  description: 'Capture a single frame from a stream as a JPEG/PNG data URL. For video-url streams, seeks time-based if currentTime set.',
  inputSchema: z.object({
    stream_id: z.string(),
    quality: z.number().optional().describe('JPEG quality 0-1. Default 0.85.'),
    type: z.enum(['image/jpeg', 'image/png']).optional(),
  }),
  callback: async (input) => {
    try {
      const entry = registry().get(input.stream_id)
      if (!entry) throw new Error(`Stream not found: ${input.stream_id}`)
      const dataUrl = await captureFrameDataURL(entry, input.quality ?? 0.85, input.type ?? 'image/jpeg')
      return JSON.stringify({
        status: 'success',
        stream_id: input.stream_id,
        dataUrl,
        size_kb: Math.round(dataUrl.length * 0.75 / 1024),
      })
    } catch (err: unknown) {
      return JSON.stringify({ status: 'error', error: (err as Error).message })
    }
  },
})

/* ──────────────────────────────────────────────────────────────────────────
 * Inline video panel — leverages render_ui under the hood (same event system).
 * Emits careless:ui-render so UIPanelBubble picks it up. The panel JS wires
 * <video> to the registered MediaStream by id, and optionally attaches an
 * overlay canvas.
 * ──────────────────────────────────────────────────────────────────────── */

export const renderVideoPanelTool = tool({
  name: 'render_video_panel',
  description:
    'Render an inline video panel bound to an open stream. Shows live MediaStream (camera/screen/mic) or video URL. ' +
    'Optionally adds a transparent overlay canvas on top for drawing bboxes / masks / depth maps from transformers output. ' +
    'Panel renders inline in the current message via the same system as render_ui.',
  inputSchema: z.object({
    stream_id: z.string().describe('From stream_open'),
    title: z.string().optional(),
    width: z.number().optional().describe('Panel width in px (default 480)'),
    height: z.number().optional().describe('Panel height in px (default auto / aspect-fit)'),
    overlay: z.boolean().optional().describe('Attach overlay canvas (registered with same id) — default true for camera/screen/video'),
    muted: z.boolean().optional().describe('Default true'),
    controls: z.boolean().optional().describe('Show native <video> controls. Default false.'),
  }),
  callback: (input) => {
    try {
      const reg = registry().get(input.stream_id)
      if (!reg) throw new Error(`Stream not found: ${input.stream_id}`)
      const id = `video-${input.stream_id}`
      const title = input.title || `🎥 ${reg.kind} (${input.stream_id})`
      const width = input.width ?? 480
      const hasOverlay = input.overlay ?? (reg.kind !== 'mic')
      const muted = input.muted ?? true
      const controls = input.controls ?? false

      // HTML structure: relatively-positioned container; <video> fills it;
      // overlay <canvas> is absolutely positioned on top with pointer-events: none.
      const html = `
<div class="video-panel" data-stream="${input.stream_id}" style="position:relative;width:${width}px;max-width:100%;">
  ${reg.kind === 'mic' ? `
  <div class="mic-indicator" style="padding:16px;background:#111;border-radius:6px;text-align:center;">
    <div style="font-size:12px;opacity:.7;margin-bottom:8px;">🎤 Microphone: ${input.stream_id}</div>
    <canvas class="audio-viz" width="${width}" height="60" style="width:100%;border-radius:4px;background:#0a0a0a;"></canvas>
  </div>
  ` : `
  <video class="stream-video" ${muted ? 'muted' : ''} ${controls ? 'controls' : ''} autoplay playsinline style="width:100%;height:${input.height ? input.height + 'px' : 'auto'};display:block;background:#000;border-radius:6px;"></video>
  ${hasOverlay ? `<canvas class="overlay" style="position:absolute;top:0;left:0;width:100%;height:100%;pointer-events:none;"></canvas>` : ''}
  `}
  <div class="video-meta" style="margin-top:6px;font-size:10px;font-family:ui-monospace,monospace;opacity:.55;display:flex;gap:10px;">
    <span class="status">connecting…</span>
    <span class="fps" style="margin-left:auto;"></span>
  </div>
</div>
`
      // JS runs inside UIPanelBubble with `root` as the content div.
      // root is the <div> containing our HTML above.
      const js = `
(async () => {
  const streamId = ${JSON.stringify(input.stream_id)}
  const reg = window.__carelessStreams && window.__carelessStreams.get(streamId)
  if (!reg) { root.querySelector('.status').textContent = 'stream not found'; return }
  const statusEl = root.querySelector('.status')
  const fpsEl = root.querySelector('.fps')

  if (reg.kind === 'mic') {
    // Audio visualization via AudioContext AnalyserNode
    const canvas = root.querySelector('.audio-viz')
    const ctx2d = canvas.getContext('2d')
    const AC = window.AudioContext || window.webkitAudioContext
    const ac = new AC()
    const src = ac.createMediaStreamSource(reg.stream)
    const analyser = ac.createAnalyser()
    analyser.fftSize = 256
    src.connect(analyser)
    const data = new Uint8Array(analyser.frequencyBinCount)
    statusEl.textContent = 'recording'
    let raf = 0
    const draw = () => {
      analyser.getByteFrequencyData(data)
      ctx2d.fillStyle = '#0a0a0a'
      ctx2d.fillRect(0, 0, canvas.width, canvas.height)
      const barW = canvas.width / data.length
      for (let i = 0; i < data.length; i++) {
        const h = (data[i] / 255) * canvas.height
        ctx2d.fillStyle = 'hsl(' + (200 + (data[i]/2)) + ',70%,55%)'
        ctx2d.fillRect(i * barW, canvas.height - h, barW - 1, h)
      }
      raf = requestAnimationFrame(draw)
    }
    draw()
    // cleanup when panel closes (root removed from DOM)
    const mo = new MutationObserver(() => {
      if (!root.isConnected) { cancelAnimationFrame(raf); try { ac.close() } catch (e) {}; mo.disconnect() }
    })
    mo.observe(document.body, { childList: true, subtree: true })
    return
  }

  const video = root.querySelector('.stream-video')
  if (!video) return
  if (reg.stream) video.srcObject = reg.stream
  else if (reg.url) video.src = reg.url
  await video.play().catch(() => {})
  statusEl.textContent = 'live'

  // Register overlay canvas so draw tools can find it
  const overlay = root.querySelector('.overlay')
  if (overlay) {
    const setSize = () => {
      overlay.width = video.videoWidth || video.clientWidth
      overlay.height = video.videoHeight || video.clientHeight
    }
    if (video.readyState >= 2) setSize()
    else video.onloadeddata = setSize
    if (!window.__carelessOverlayCanvases) window.__carelessOverlayCanvases = new Map()
    window.__carelessOverlayCanvases.set(streamId, overlay)
  }

  // FPS counter
  let frames = 0
  let lastTick = performance.now()
  let raf = 0
  const tick = () => {
    frames++
    const now = performance.now()
    if (now - lastTick > 1000) {
      fpsEl.textContent = Math.round(frames * 1000 / (now - lastTick)) + ' fps'
      frames = 0; lastTick = now
    }
    raf = requestAnimationFrame(tick)
  }
  tick()
  const mo = new MutationObserver(() => {
    if (!root.isConnected) {
      cancelAnimationFrame(raf)
      if (window.__carelessOverlayCanvases) window.__carelessOverlayCanvases.delete(streamId)
      mo.disconnect()
    }
  })
  mo.observe(document.body, { childList: true, subtree: true })
})().catch(e => { try { root.querySelector('.status').textContent = 'error: ' + e.message } catch(_){} })
`
      if (typeof window !== 'undefined') {
        window.dispatchEvent(new CustomEvent('careless:ui-render', {
          detail: { id, title, html, css: '', js, createdAt: Date.now() },
        }))
      }
      return JSON.stringify({
        status: 'rendered',
        id,
        stream_id: input.stream_id,
        has_overlay: hasOverlay,
        overlay_canvas_id: hasOverlay ? input.stream_id : null,
      })
    } catch (err: unknown) {
      return JSON.stringify({ status: 'error', error: (err as Error).message })
    }
  },
})

/* ──────────────────────────────────────────────────────────────────────────
 * Overlay drawing — the agent paints detections/masks/labels on top of video.
 * ──────────────────────────────────────────────────────────────────────── */

export const drawOverlayTool = tool({
  name: 'stream_draw_overlay',
  description:
    'Draw shapes/text on the overlay canvas of a streamed video. Supports bboxes, labels, masks (as dataURL), heatmaps, dots. ' +
    'Use after render_video_panel (with overlay:true). Coordinates are in VIDEO pixel space (not CSS pixels).',
  inputSchema: z.object({
    stream_id: z.string(),
    clear: z.boolean().optional().describe('Clear before drawing. Default true.'),
    draw: z.array(z.object({
      type: z.enum(['bbox', 'label', 'mask', 'heatmap', 'circle', 'line', 'text', 'clear']),
      x: z.number().optional(),
      y: z.number().optional(),
      w: z.number().optional(),
      h: z.number().optional(),
      x2: z.number().optional(),
      y2: z.number().optional(),
      r: z.number().optional(),
      text: z.string().optional(),
      color: z.string().optional(),
      lineWidth: z.number().optional(),
      font: z.string().optional(),
      fill: z.boolean().optional(),
      alpha: z.number().optional(),
      maskDataUrl: z.string().optional().describe('For type:mask — a data URL image whose non-transparent pixels overlay'),
    })).describe('Ordered list of draw operations'),
  }),
  callback: async (input) => {
    try {
      const overlay = overlayRegistry().get(input.stream_id)
      if (!overlay) throw new Error(`No overlay for stream ${input.stream_id}. Did you render_video_panel with overlay:true?`)
      const ctx = overlay.getContext('2d')
      if (!ctx) throw new Error('Overlay 2d context unavailable')
      if (input.clear !== false) ctx.clearRect(0, 0, overlay.width, overlay.height)

      for (const op of input.draw) {
        ctx.save()
        if (op.alpha !== undefined) ctx.globalAlpha = op.alpha
        const color = op.color || '#00ff99'
        ctx.strokeStyle = color
        ctx.fillStyle = color
        ctx.lineWidth = op.lineWidth ?? 2
        ctx.font = op.font || '14px ui-sans-serif'

        switch (op.type) {
          case 'clear': ctx.clearRect(op.x ?? 0, op.y ?? 0, op.w ?? overlay.width, op.h ?? overlay.height); break
          case 'bbox':
            if (op.w !== undefined && op.h !== undefined) {
              ctx.strokeRect(op.x ?? 0, op.y ?? 0, op.w, op.h)
              if (op.fill) ctx.fillRect(op.x ?? 0, op.y ?? 0, op.w, op.h)
              if (op.text) {
                const pad = 3
                const tw = ctx.measureText(op.text).width + pad * 2
                ctx.fillStyle = color
                ctx.fillRect(op.x ?? 0, (op.y ?? 0) - 16, tw, 16)
                ctx.fillStyle = '#000'
                ctx.fillText(op.text, (op.x ?? 0) + pad, (op.y ?? 0) - 4)
              }
            }
            break
          case 'circle':
            ctx.beginPath(); ctx.arc(op.x ?? 0, op.y ?? 0, op.r ?? 4, 0, Math.PI * 2)
            if (op.fill) ctx.fill(); else ctx.stroke()
            break
          case 'line':
            ctx.beginPath(); ctx.moveTo(op.x ?? 0, op.y ?? 0); ctx.lineTo(op.x2 ?? 0, op.y2 ?? 0); ctx.stroke()
            break
          case 'text':
            if (op.text) ctx.fillText(op.text, op.x ?? 0, op.y ?? 0)
            break
          case 'label':
            if (op.text) {
              const pad = 4
              const tw = ctx.measureText(op.text).width + pad * 2
              ctx.fillRect((op.x ?? 0) - pad, (op.y ?? 0) - 14, tw, 18)
              ctx.fillStyle = '#000'
              ctx.fillText(op.text, op.x ?? 0, op.y ?? 0)
            }
            break
          case 'mask':
            if (op.maskDataUrl) {
              // Load mask image and composite
              const img = new Image()
              await new Promise<void>((r) => { img.onload = () => r(); img.onerror = () => r(); img.src = op.maskDataUrl! })
              ctx.drawImage(img, op.x ?? 0, op.y ?? 0, op.w ?? overlay.width, op.h ?? overlay.height)
            }
            break
          case 'heatmap':
            // Expect op.text to be a JSON array of {x,y,v} points normalized 0-1; we convert to radial gradients
            if (op.text) {
              try {
                const pts = JSON.parse(op.text) as { x: number; y: number; v: number }[]
                for (const p of pts) {
                  const g = ctx.createRadialGradient(p.x * overlay.width, p.y * overlay.height, 0, p.x * overlay.width, p.y * overlay.height, 30)
                  g.addColorStop(0, `rgba(255,0,0,${p.v})`)
                  g.addColorStop(1, 'rgba(255,0,0,0)')
                  ctx.fillStyle = g
                  ctx.fillRect(0, 0, overlay.width, overlay.height)
                }
              } catch {}
            }
            break
        }
        ctx.restore()
      }
      return JSON.stringify({ status: 'success', stream_id: input.stream_id, drawn: input.draw.length })
    } catch (err: unknown) {
      return JSON.stringify({ status: 'error', error: (err as Error).message })
    }
  },
})

/* ──────────────────────────────────────────────────────────────────────────
 * The big one: continuous stream → transformers pipeline loop.
 * ──────────────────────────────────────────────────────────────────────── */

export const streamToTransformersTool = tool({
  name: 'stream_to_transformers',
  description:
    'Continuously pipe frames from a stream through a Transformers.js pipeline and emit results. ' +
    'Example: camera → object-detection → draws bboxes on overlay. Or mic-chunks → whisper → updates caption. ' +
    'Results fire as careless:stream-inference events AND (if auto_draw) paint on the overlay canvas. ' +
    'Use stream_stop_loop to end.',
  inputSchema: z.object({
    stream_id: z.string(),
    task: z.string().describe('Transformers.js task (e.g. object-detection, depth-estimation, image-to-text, zero-shot-object-detection, image-classification, image-segmentation)'),
    model: z.string().optional(),
    device: z.enum(['wasm', 'webgpu']).optional(),
    dtype: z.string().optional(),
    fps: z.number().optional().describe('Target frames per second. Default 1. Higher needs WebGPU.'),
    options: z.record(z.string(), z.any()).optional().describe('Pipeline options (e.g. candidate_labels, threshold)'),
    auto_draw: z.boolean().optional().describe('Auto-paint results on the overlay canvas. Default true for detection tasks.'),
    max_iterations: z.number().optional().describe('Stop after N iterations. Default: infinite.'),
    loop_id: z.string().optional().describe('Custom loop id. Default: auto.'),
  }),
  callback: async (input) => {
    try {
      const entry = registry().get(input.stream_id)
      if (!entry) throw new Error(`Stream not found: ${input.stream_id}`)

      // Lazy-load transformers (550 KB) only when user actually runs a stream loop
      const { pipeline: hfPipeline } = await import('@huggingface/transformers')
      const resolvedTask = TASK_ALIASES[input.task] ?? input.task
      const info = TASK_REGISTRY[resolvedTask]
      const model = input.model ?? info?.defaultModel
      const device = input.device ?? 'wasm'
      const pipeOpts: any = { device }

      // For seq2seq/encoder-decoder/vision-seq tasks, default to all-sessions fp32
      // to dodge the Firefox ORT "Missing required scale" bug on quantized weights.
      const allFp32 = {
        model: 'fp32', encoder_model: 'fp32', decoder_model_merged: 'fp32',
        decoder_model: 'fp32', embed_tokens: 'fp32', vision_encoder: 'fp32',
        audio_encoder: 'fp32', prepare_inputs_embeds: 'fp32',
      }
      const isDecoderOnlyLLM = resolvedTask === 'text-generation' ||
        (model && /qwen|llama|phi|gemma|smollm/i.test(model))
      if (input.dtype) {
        pipeOpts.dtype = input.dtype
      } else if (!isDecoderOnlyLLM) {
        pipeOpts.dtype = allFp32
      } else if (info?.defaultDtype) {
        pipeOpts.dtype = info.defaultDtype
      }

      let pipe: any
      try {
        pipe = await hfPipeline(resolvedTask as any, model, pipeOpts)
      } catch (err: unknown) {
        const msg = (err as Error).message || ''
        if ((msg.includes('weight_merged_0_scale') || msg.includes('Missing required scale')) && !input.dtype) {
          // Retry with all-fp32
          pipe = await hfPipeline(resolvedTask as any, model, { ...pipeOpts, dtype: allFp32 })
        } else {
          throw err
        }
      }

      const loopId = input.loop_id ?? `loop-${input.stream_id}-${Date.now().toString(36)}`
      const fps = input.fps ?? 1
      const intervalMs = 1000 / fps
      const autoDraw = input.auto_draw ?? ['object-detection', 'zero-shot-object-detection', 'image-segmentation', 'depth-estimation'].includes(resolvedTask)

      let stopped = false
      let iterations = 0

      const tick = async () => {
        if (stopped) return
        if (input.max_iterations && iterations >= input.max_iterations) {
          stop(); return
        }
        try {
          const dataUrl = await captureFrameDataURL(entry, 0.85, 'image/jpeg')
          const result = await pipe(dataUrl, input.options ?? {})
          iterations++

          // Emit event for UI / other tools to consume
          window.dispatchEvent(new CustomEvent('careless:stream-inference', {
            detail: {
              loop_id: loopId, stream_id: input.stream_id,
              task: resolvedTask, iteration: iterations, ts: Date.now(), result,
            },
          }))

          // Auto-draw for detection tasks
          if (autoDraw) {
            const overlay = overlayRegistry().get(input.stream_id)
            if (overlay) {
              const ctx = overlay.getContext('2d')
              if (ctx) {
                ctx.clearRect(0, 0, overlay.width, overlay.height)
                const colorFor = (label: string) => {
                  let h = 0
                  for (let i = 0; i < label.length; i++) h = (h * 31 + label.charCodeAt(i)) & 0xfff
                  return `hsl(${h % 360}, 80%, 60%)`
                }
                if (Array.isArray(result)) {
                  for (const det of result) {
                    // object-detection: { score, label, box:{xmin,ymin,xmax,ymax} }
                    if (det.box) {
                      const { xmin, ymin, xmax, ymax } = det.box
                      const color = colorFor(det.label ?? 'obj')
                      ctx.strokeStyle = color
                      ctx.lineWidth = 2
                      ctx.strokeRect(xmin, ymin, xmax - xmin, ymax - ymin)
                      const caption = `${det.label ?? ''} ${(det.score ? ' ' + det.score.toFixed(2) : '')}`.trim()
                      if (caption) {
                        ctx.font = '13px ui-sans-serif'
                        const tw = ctx.measureText(caption).width + 8
                        ctx.fillStyle = color
                        ctx.fillRect(xmin, Math.max(0, ymin - 16), tw, 16)
                        ctx.fillStyle = '#000'
                        ctx.fillText(caption, xmin + 4, Math.max(12, ymin - 4))
                      }
                    }
                    // segmentation: { label, mask, score }
                    if (det.mask) {
                      try {
                        // mask is a RawImage; convert to <img> via putImageData
                        const mw = det.mask.width
                        const mh = det.mask.height
                        const scaleX = overlay.width / mw
                        const scaleY = overlay.height / mh
                        const tmp = document.createElement('canvas')
                        tmp.width = mw; tmp.height = mh
                        const tctx = tmp.getContext('2d')
                        if (tctx) {
                          const imgData = tctx.createImageData(mw, mh)
                          for (let i = 0; i < mw * mh; i++) {
                            const v = det.mask.data[i]
                            if (v > 128) {
                              const c = colorFor(det.label ?? '').match(/\d+/g)!
                              imgData.data[i*4] = parseInt(c[0])
                              imgData.data[i*4+1] = parseInt(c[1]) * 2.55
                              imgData.data[i*4+2] = parseInt(c[2]) * 2.55
                              imgData.data[i*4+3] = 100
                            }
                          }
                          tctx.putImageData(imgData, 0, 0)
                          ctx.globalAlpha = 0.5
                          ctx.drawImage(tmp, 0, 0, overlay.width, overlay.height)
                          ctx.globalAlpha = 1
                        }
                      } catch {}
                    }
                  }
                } else if (result && result.predicted_depth) {
                  // depth-estimation: { predicted_depth: Tensor, depth: RawImage }
                  try {
                    const depth = result.depth
                    if (depth && depth.data) {
                      const dw = depth.width; const dh = depth.height
                      const tmp = document.createElement('canvas')
                      tmp.width = dw; tmp.height = dh
                      const tctx = tmp.getContext('2d')
                      if (tctx) {
                        const imgData = tctx.createImageData(dw, dh)
                        for (let i = 0; i < dw * dh; i++) {
                          const v = depth.data[i]
                          imgData.data[i*4] = v
                          imgData.data[i*4+1] = v
                          imgData.data[i*4+2] = 255 - v
                          imgData.data[i*4+3] = 200
                        }
                        tctx.putImageData(imgData, 0, 0)
                        ctx.globalAlpha = 0.6
                        ctx.drawImage(tmp, 0, 0, overlay.width, overlay.height)
                        ctx.globalAlpha = 1
                      }
                    }
                  } catch {}
                }
              }
            }
          }
        } catch (e) {
          window.dispatchEvent(new CustomEvent('careless:stream-inference', {
            detail: { loop_id: loopId, stream_id: input.stream_id, error: (e as Error).message, ts: Date.now() },
          }))
        }
        if (!stopped) setTimeout(tick, intervalMs)
      }

      const stop = () => {
        stopped = true
        loopRegistry().delete(loopId)
        try { pipe?.dispose?.() } catch {}
      }
      loopRegistry().set(loopId, { stop })
      tick() // fire-and-forget

      return JSON.stringify({
        status: 'started',
        loop_id: loopId,
        stream_id: input.stream_id,
        task: resolvedTask,
        model,
        fps,
        auto_draw: autoDraw,
        hint: `Listen for careless:stream-inference events, or call stream_stop_loop({loop_id: "${loopId}"})`,
      })
    } catch (err: unknown) {
      return JSON.stringify({ status: 'error', error: (err as Error).message })
    }
  },
})

export const stopStreamLoopTool = tool({
  name: 'stream_stop_loop',
  description: 'Stop a running stream→transformers inference loop.',
  inputSchema: z.object({
    loop_id: z.string().optional(),
    all: z.boolean().optional(),
  }),
  callback: (input) => {
    const loops = loopRegistry()
    const stopped: string[] = []
    const ids = input.all ? Array.from(loops.keys()) : input.loop_id ? [input.loop_id] : []
    for (const id of ids) {
      const handle = loops.get(id)
      if (handle) { handle.stop(); loops.delete(id); stopped.push(id) }
    }
    return JSON.stringify({ status: 'success', stopped, remaining: loops.size })
  },
})

/* ──────────────────────────────────────────────────────────────────────────
 * Audio streaming chunks for Whisper (continuous STT).
 * Records rolling chunks of audio from a mic stream and transcribes them.
 * ──────────────────────────────────────────────────────────────────────── */

export const streamTranscribeTool = tool({
  name: 'stream_transcribe',
  description:
    'Continuously transcribe a microphone stream via Whisper. Records rolling chunks (default 5s) and fires ' +
    'careless:stream-inference events with partial transcripts. Requires an active mic stream (stream_open kind="mic").',
  inputSchema: z.object({
    stream_id: z.string().describe('Must be a mic stream'),
    model: z.string().optional().describe('Default: Xenova/whisper-tiny.en'),
    device: z.enum(['wasm', 'webgpu']).optional(),
    chunk_sec: z.number().optional().describe('Rolling window length. Default 5.'),
    language: z.string().optional(),
    max_iterations: z.number().optional(),
    loop_id: z.string().optional(),
  }),
  callback: async (input) => {
    try {
      const entry = registry().get(input.stream_id)
      if (!entry || entry.kind !== 'mic') throw new Error('Requires a mic stream')
      if (!entry.stream) throw new Error('No MediaStream on entry')

      const { pipeline: hfPipeline } = await import('@huggingface/transformers')
      const device = input.device ?? 'wasm'
      const model = input.model ?? TASK_REGISTRY['automatic-speech-recognition'].defaultModel
      const pipeOpts: any = { device }
      // Always all-fp32 for Whisper — safest across Firefox/Chrome/Safari.
      pipeOpts.dtype = {
        model: 'fp32', encoder_model: 'fp32', decoder_model_merged: 'fp32',
      }
      let pipe: any
      try {
        pipe = await hfPipeline('automatic-speech-recognition', model, pipeOpts)
      } catch (err: unknown) {
        // If fp32 fails, the model probably doesn't ship fp32 weights; retry with
        // a minimal config that lets transformers.js pick defaults.
        delete pipeOpts.dtype
        pipe = await hfPipeline('automatic-speech-recognition', model, pipeOpts)
      }

      const chunkSec = input.chunk_sec ?? 5
      const loopId = input.loop_id ?? `transcribe-${input.stream_id}-${Date.now().toString(36)}`
      let stopped = false
      let iterations = 0

      const recorder = new MediaRecorder(entry.stream, { mimeType: 'audio/webm' })
      const chunks: Blob[] = []
      recorder.ondataavailable = async (e) => {
        if (e.data.size > 0) chunks.push(e.data)
      }
      recorder.onstop = async () => {
        const blob = new Blob(chunks.splice(0), { type: 'audio/webm' })
        try {
          const url = URL.createObjectURL(blob)
          const opts: any = {}
          if (input.language) opts.language = input.language
          const result = await pipe(url, opts)
          URL.revokeObjectURL(url)
          iterations++
          window.dispatchEvent(new CustomEvent('careless:stream-inference', {
            detail: { loop_id: loopId, stream_id: input.stream_id, task: 'asr', iteration: iterations, result, ts: Date.now() },
          }))
        } catch (e) {
          window.dispatchEvent(new CustomEvent('careless:stream-inference', {
            detail: { loop_id: loopId, error: (e as Error).message },
          }))
        }
        if (!stopped && (!input.max_iterations || iterations < input.max_iterations)) {
          try { recorder.start(); setTimeout(() => { if (!stopped && recorder.state === 'recording') recorder.stop() }, chunkSec * 1000) } catch {}
        } else {
          stop()
        }
      }

      const stop = () => {
        stopped = true
        try { if (recorder.state !== 'inactive') recorder.stop() } catch {}
        loopRegistry().delete(loopId)
        try { pipe?.dispose?.() } catch {}
      }
      loopRegistry().set(loopId, { stop })
      recorder.start()
      setTimeout(() => { if (!stopped && recorder.state === 'recording') recorder.stop() }, chunkSec * 1000)

      return JSON.stringify({
        status: 'started',
        loop_id: loopId,
        stream_id: input.stream_id,
        chunk_sec: chunkSec,
        hint: `Listen for careless:stream-inference events with result.text`,
      })
    } catch (err: unknown) {
      return JSON.stringify({ status: 'error', error: (err as Error).message })
    }
  },
})

export const STREAMING_TOOLS = [
  openStreamTool,
  listStreamsTool,
  closeStreamTool,
  captureFrameTool,
  renderVideoPanelTool,
  drawOverlayTool,
  streamToTransformersTool,
  stopStreamLoopTool,
  streamTranscribeTool,
]