/**
 * @module custom_cil-doc
 * @audience installer
 * @layer api-handler
 * @stability experimental
 *
 * Pass 4 of the CIL ingestion pipeline: AI documentation generation.
 *
 * For each cil_chunk whose contract fields (macro, micro, input_spec,
 * output_spec, side_effects) are blank, this function calls the LLM to
 * fill them in, then writes them back to the item's data field.
 *
 * Critically, it exposes a **cost estimator** action that returns the
 * projected token usage and USD cost BEFORE any LLM calls are made,
 * so humans can review before running the expensive pass.
 *
 * Actions:
 *   GET  ?action=estimate    { project_id, model? }         — pre-flight cost estimate
 *   POST ?action=document    { project_id, chunk_ids?, model?, dry_run? } — generate docs
 *   POST ?action=doc_chunk   { chunk_id, model?, force? }   — document a single chunk
 *   GET  ?action=gap_report  { project_id }                 — list undocumented chunks
 */

import { createHandler } from './_shared/middleware'
import { adminDb } from './_shared/db'
import * as fs from 'fs'
import * as path from 'path'

// ─── CONSTANTS ────────────────────────────────────────────────────────────────

const DEFAULT_MODEL = 'gpt-4o-mini'

// Token pricing per 1M tokens (USD) — used for cost estimation
// Values as of mid-2025; override via model_prices in request body
const MODEL_PRICES: Record<string, { input_per_1m: number; output_per_1m: number }> = {
  'gpt-4o-mini':   { input_per_1m: 0.15,  output_per_1m: 0.60  },
  'gpt-4o':        { input_per_1m: 2.50,  output_per_1m: 10.00 },
  'gpt-4-turbo':   { input_per_1m: 10.00, output_per_1m: 30.00 },
  'gpt-3.5-turbo': { input_per_1m: 0.50,  output_per_1m: 1.50  },
}

// Conservative output estimate: the JSON contract reply per chunk
const ESTIMATED_OUTPUT_TOKENS_PER_CHUNK = 200
// System prompt + instruction overhead tokens per call
const SYSTEM_PROMPT_OVERHEAD_TOKENS = 350

// ─── UTILITIES ────────────────────────────────────────────────────────────────

function estimateTokens(text: string): number {
  return Math.ceil(text.length / 4)
}

function needsDocumentation(chunk: any): boolean {
  const d = chunk.data || {}
  // Chunks with validation errors need to be re-documented
  if (d.doc_validation_errors && d.doc_validation_errors.length > 0) return true
  // Re-document source or AI-generated chunks that are missing the new rich fields
  if (d.doc_confidence === 'from_source' || d.doc_confidence === 'ai_generated') {
    const required = d.chunk_type === 'module_header'
      ? REQUIRED_MODULE_FIELDS
      : REQUIRED_CHUNK_FIELDS
    return required.some(f => !d[f] || String(d[f]).trim() === '')
  }
  return !d.macro && !d.micro && !d.input_spec && !d.output_spec
}

// Supabase caps unbounded .select() at 1000 rows by default. Load in pages to be safe.
async function loadAllChunks(projectId: string, filePath?: string): Promise<any[]> {
  const PAGE = 1000
  const all: any[] = []
  for (let offset = 0; ; offset += PAGE) {
    let q = adminDb
      .from('items')
      .select('id, title, data')
      .filter('data->>kb_type', 'eq', 'cil_chunk')
      .filter('data->>project_id', 'eq', projectId)
      .eq('is_active', true)
      .order('id')
      .range(offset, offset + PAGE - 1)
    if (filePath) q = q.filter('data->>file_path', 'eq', filePath)
    const { data, error } = await q
    if (error) throw new Error(`Failed to load chunks: ${error.message}`)
    if (!data || data.length === 0) break
    all.push(...data)
    if (data.length < PAGE) break
  }
  return all
}

// ───────────────────────────────────────────────────────────────────────────────
// CANONICAL JSDOC FORMAT
//
// Module headers (__module_header__) use:
//   /**
//    * @module <file-name>
//    * @audience internal | external | both
//    * @layer shared-core | app-specific | infrastructure | test
//    * @stability stable | experimental | deprecated
//    *
//    * <description>
//    *
//    * @seeAlso <file> (<purpose>)
//    * @cil-generated
//    */
//
// All other chunks use:
//   /**
//    * <description>
//    *
//    * @macro       <one sentence purpose>
//    * @micro       <one sentence implementation>
//    * @inputSpec
//    *   - <name>: <type> (required|optional) — <description>
//    * @outputSpec  <output shape>
//    * @sideEffects <side effects or 'none'>
//    * @stability   stable | experimental | deprecated
//    * @calledBy    <callers>
//    * @calls       <callees>
//    * @tags        <comma-separated tags>
//    * @complexity  simple | moderate | complex
//    * @errorHandling <error handling behavior>
//    * @cil-generated
//    */
//
// Required fields:
//   - module_header: @module, @audience, @layer, @stability, description
//   - chunk: @macro, @inputSpec, @outputSpec, @sideEffects, @stability, description
// ───────────────────────────────────────────────────────────────────────────────

const REQUIRED_MODULE_FIELDS = ['module', 'audience', 'layer', 'stability', 'description']
const REQUIRED_CHUNK_FIELDS = ['description', 'macro', 'inputSpec', 'outputSpec', 'sideEffects', 'stability']

// Extract a concise file name from a relative path for the @module tag.
function moduleNameFromPath(relPath: string): string {
  return path.basename(relPath, path.extname(relPath))
}

// Build a JSDoc comment block from the chunk's AI-generated documentation.
// Replaces the simple legacy format with the rich canonical format.
function buildJsDoc(chunk: any): string {
  const d = chunk.data || {}
  const isModuleHeader = d.chunk_type === 'module_header'

  const lines: string[] = []
  const description = (d.description || d.macro || '').trim()

  if (description) lines.push(description)
  if (d.micro && d.micro !== description) lines.push(d.micro)

  const body: string[] = []

  if (isModuleHeader) {
    if (d.module) body.push(`@module ${d.module}`)
    if (d.audience) body.push(`@audience ${d.audience}`)
    if (d.layer) body.push(`@layer ${d.layer}`)
  }

  if (!isModuleHeader && d.macro) body.push(`@macro       ${d.macro}`)
  if (!isModuleHeader && d.micro) body.push(`@micro       ${d.micro}`)

  if (d.inputSpec) {
    body.push('@inputSpec')
    const entries = d.inputSpec.split(/\n/).map((s: string) => s.trim()).filter(Boolean)
    for (const entry of entries) body.push(`  ${entry}`)
  }

  if (d.outputSpec) body.push(`@outputSpec  ${d.outputSpec}`)
  if (d.throws) body.push(`@throws     ${d.throws}`)
  if (d.sideEffects) body.push(`@sideEffects ${d.sideEffects}`)
  if (d.stability) body.push(`@stability   ${d.stability}`)

  if (!isModuleHeader && d.calledBy) body.push(`@calledBy    ${d.calledBy}`)
  if (!isModuleHeader && d.calls) body.push(`@calls       ${d.calls}`)
  if (!isModuleHeader && d.tags) body.push(`@tags        ${d.tags}`)
  if (!isModuleHeader && d.complexity) body.push(`@complexity  ${d.complexity}`)
  if (!isModuleHeader && d.errorHandling) body.push(`@errorHandling ${d.errorHandling}`)

  if (Array.isArray(d.depends_on) && d.depends_on.length > 0) {
    body.push(`@dependsOn  ${d.depends_on.join(', ')}`)
  }
  if (Array.isArray(d.depended_by) && d.depended_by.length > 0) {
    body.push(`@dependedBy ${d.depended_by.join(', ')}`)
  }

  if (d.seeAlso) {
    const refs = d.seeAlso.split(/\n/).map((s: string) => s.trim()).filter(Boolean)
    for (const ref of refs) body.push(`@seeAlso    ${ref}`)
  }

  body.push('@cil-generated')

  if (body.length === 0 || !description) return ''

  const docLines = [
    '/**',
    ...lines.map(l => ` * ${l}`),
    ...(lines.length > 0 && body.length > 0 ? [' *'] : []),
    ...body.map(l => ` * ${l}`),
    ' */',
  ]
  return docLines.join('\n')
}

// Validate that a chunk's documentation contains all required fields and that
// every parameter is documented. Returns a list of validation errors.
function validateChunkDocs(chunk: any): string[] {
  const d = chunk.data || {}
  const errors: string[] = []
  const isModuleHeader = d.chunk_type === 'module_header'

  const required = isModuleHeader ? REQUIRED_MODULE_FIELDS : REQUIRED_CHUNK_FIELDS
  for (const field of required) {
    if (!d[field] || String(d[field]).trim() === '') {
      errors.push(`missing ${field}`)
    }
  }

  if (!isModuleHeader && d.inputSpec && d.inputSpec !== 'none') {
    const params = d.inputSpec.split(/\n/).map((s: string) => s.trim()).filter(Boolean)
    const paramNames = params.map(p => {
      const m = p.match(/^[-\s]*([a-zA-Z0-9_]+)/)
      return m ? m[1] : ''
    }).filter(Boolean)

    const code = d.code_content || ''
    const signatureMatch = code.match(/(?:function|const|let|var)\s+\w+\s*\(([^)]*)\)/)
    if (signatureMatch) {
      const rawParams = signatureMatch[1]
      const declaredNames = rawParams
        .split(',')
        .map(s => s.trim())
        .filter(s => s)
        .map(s => s.split(/[:=]/)[0].trim().replace(/\?/, ''))
        .filter(Boolean)
      for (const declared of declaredNames) {
        if (!paramNames.includes(declared)) {
          errors.push(`undocumented parameter: ${declared}`)
        }
      }
    }
  }

  return errors
}

// Find an existing CIL-generated JSDoc block immediately BELOW the given marker.
// Returns { start, end } line indices, or null if none found.
function findExistingCilJsDoc(lines: string[], markerIdx: number): { start: number; end: number } | null {
  let startIdx = -1
  for (let i = markerIdx + 1; i < lines.length; i++) {
    const trimmed = lines[i].trim()
    if (trimmed === '') continue
    if (trimmed.startsWith('/**')) { startIdx = i; break }
    break
  }
  if (startIdx === -1) return null

  let endIdx = -1
  let hasCilTag = false
  for (let i = startIdx; i < lines.length; i++) {
    const trimmed = lines[i].trim()
    if (trimmed.includes('@cil-generated')) hasCilTag = true
    if (trimmed === '*/' || trimmed.endsWith('*/')) { endIdx = i; break }
  }
  if (endIdx === -1 || !hasCilTag) return null
  return { start: startIdx, end: endIdx }
}

// Write JSDoc for a chunk into the source file, immediately BELOW the CHUNK_START marker.
// Replaces an existing CIL JSDoc block so documentation stays current.
function writeJsDocToSource(chunk: any, repoPath: string): void {
  const d = chunk.data || {}
  const relPath = d.file_path
  const lineStart = d.line_start
  if (!repoPath || !relPath || !lineStart) return

  const ext = path.extname(relPath).toLowerCase()
  if (!['.ts', '.tsx', '.js', '.jsx'].includes(ext)) return

  const jsDoc = buildJsDoc(chunk)
  if (!jsDoc) return

  const absPath = path.join(repoPath, relPath)
  if (!fs.existsSync(absPath)) return

  const content = fs.readFileSync(absPath, 'utf-8')
  const lines = content.split('\n')

  const markerRe = new RegExp(`CHUNK_START:\\s*${escapeRegExp(d.chunk_name || chunk.title)}`)
  let markerIdx = -1
  const searchStart = Math.max(0, lineStart - 5)
  const searchEnd = Math.min(lines.length, lineStart + 5)
  for (let i = searchStart; i < searchEnd; i++) {
    if (markerRe.test(lines[i] || '')) { markerIdx = i; break }
  }
  if (markerIdx === -1) return

  const existing = findExistingCilJsDoc(lines, markerIdx)
  const newLines = [...lines]
  if (existing) {
    newLines.splice(existing.start, existing.end - existing.start + 1, jsDoc)
  } else {
    newLines.splice(markerIdx + 1, 0, jsDoc)
  }

  fs.writeFileSync(absPath, newLines.join('\n'), 'utf-8')
}

function escapeRegExp(s: string): string {
  return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
}

function buildDocPrompt(chunk: any): string {
  const d = chunk.data || {}
  const code = (d.code_content || '').slice(0, 3000)
  const location = `${d.file_path || 'unknown'}:${d.line_start || 0}-${d.line_end || 0}`
  const isModuleHeader = d.chunk_type === 'module_header'

  const headerInstructions = isModuleHeader
    ? `This is the module header for file "${d.file_path || 'unknown'}". Generate a file-level JSDoc block. Include @module, @audience, @layer, @stability, a concise description, and @seeAlso references to related files if obvious.`
    : `Analyse this ${d.chunk_type || 'function'} named "${d.chunk_name || 'unknown'}" located at ${location}.`

  return `${headerInstructions}

\`\`\`typescript
${code}
\`\`\`

Respond ONLY with a JSON object with these exact keys (no markdown, no commentary):
{
  "description": "<human-readable summary — one to three sentences>",
  "macro": "<one sentence — what this chunk IS and its primary responsibility>",
  "micro": "<one sentence — HOW it achieves that responsibility; key algorithm or pattern>",
  "inputSpec": "<newline-separated list of: - name: Type (required|optional) — description/constraint>",
  "outputSpec": "<return type and shape; include Promise<T> if async>",
  "throws": "<comma-separated list of: ErrorType — condition that triggers it, or 'none'>",
  "side_effects": "<comma-separated list of observable side effects (DB writes, API calls, file I/O), or 'none'>",
  "stability": "<one of: stable | experimental | deprecated>",
  "calledBy": "<comma-separated list of likely callers/contexts, or 'unknown'>",
  "calls": "<comma-separated list of functions/modules this chunk calls, or 'unknown'>",
  "tags": "<comma-separated relevant tags>",
  "complexity": "<one of: simple | moderate | complex>",
  "errorHandling": "<how errors are handled: throws | returns null | graceful | etc>",
  "module": "<for module headers only: concise file name without extension>",
  "audience": "<for module headers only: internal | external | both>",
  "layer": "<for module headers only: shared-core | app-specific | infrastructure | test>",
  "seeAlso": "<for module headers only: newline-separated list of related files>",
}`
}

// ─── LLM CALLER ──────────────────────────────────────────────────────────────

async function callLLM(prompt: string, model: string): Promise<string> {
  const apiKey = process.env.OPENAI_API_KEY || process.env.LLM_API_KEY
  const baseUrl = process.env.OPENAI_BASE_URL || 'https://api.openai.com/v1'
  if (!apiKey) throw new Error('OPENAI_API_KEY not configured')

  const res = await fetch(`${baseUrl}/chat/completions`, {
    method: 'POST',
    headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${apiKey}` },
    body: JSON.stringify({
      model,
      temperature: 0.2,
      max_tokens: 800,
      response_format: { type: 'json_object' },
      messages: [
        {
          role: 'system',
          content: 'You are a code documentation assistant. You extract structured contracts from source code. Always respond with valid JSON only.',
        },
        { role: 'user', content: prompt },
      ],
    }),
  })

  if (!res.ok) {
    const body = await res.text()
    throw new Error(`LLM API error ${res.status}: ${body.slice(0, 200)}`)
  }

  const j: any = await res.json()
  return j.choices?.[0]?.message?.content?.trim() || '{}'
}

// ─── PARSE LLM RESPONSE ───────────────────────────────────────────────────────

function parseDocResponse(raw: string): Record<string, string> {
  try {
    const parsed = JSON.parse(raw)
    return {
      description: String(parsed.description || parsed.macro || '').slice(0, 1000),
      macro:       String(parsed.macro       || '').slice(0, 500),
      micro:       String(parsed.micro       || '').slice(0, 500),
      input_spec:  String(parsed.inputSpec   || parsed.input_spec  || '').slice(0, 2000),
      output_spec: String(parsed.outputSpec  || parsed.output_spec || '').slice(0, 500),
      throws:      String(parsed.throws      || '').slice(0, 500),
      side_effects:String(parsed.sideEffects || parsed.side_effects|| '').slice(0, 500),
      stability:   ['stable', 'internal', 'experimental', 'deprecated'].includes(parsed.stability)
        ? parsed.stability : 'internal',
      called_by:   String(parsed.calledBy    || parsed.called_by  || '').slice(0, 500),
      calls:       String(parsed.calls        || '').slice(0, 500),
      tags:        String(parsed.tags          || '').slice(0, 500),
      complexity:  ['simple', 'moderate', 'complex'].includes(parsed.complexity)
        ? parsed.complexity : 'moderate',
      error_handling: String(parsed.errorHandling || parsed.error_handling || '').slice(0, 500),
      module:      String(parsed.module        || '').slice(0, 100),
      audience:    ['internal', 'external', 'both'].includes(parsed.audience)
        ? parsed.audience : 'both',
      layer:       ['shared-core', 'app-specific', 'infrastructure', 'test'].includes(parsed.layer)
        ? parsed.layer : 'app-specific',
      see_also:    String(parsed.seeAlso       || parsed.see_also    || '').slice(0, 1000),
    }
  } catch {
    return {}
  }
}

// ─── DOCUMENT ONE CHUNK ───────────────────────────────────────────────────────

async function documentChunk(
  chunk: any,
  model: string,
  force: boolean,
  repoPath?: string
): Promise<{ success: boolean; fields_set: string[]; error?: string; skipped?: boolean; updatedChunk?: any; validation_errors?: string[] }> {
  if (!force && !needsDocumentation(chunk)) {
    return { success: true, fields_set: [], skipped: true, updatedChunk: chunk }
  }

  const d = chunk.data || {}
  if (!d.code_content && !d.chunk_name) {
    return { success: false, fields_set: [], error: 'No code content to document' }
  }

  const prompt = buildDocPrompt(chunk)

  let raw: string
  try {
    raw = await callLLM(prompt, model)
  } catch (err) {
    return { success: false, fields_set: [], error: err instanceof Error ? err.message : String(err) }
  }

  const fields = parseDocResponse(raw)
  if (!Object.keys(fields).length) {
    return { success: false, fields_set: [], error: 'LLM returned unparseable response' }
  }

  // Merge module name from path if not provided
  if (d.chunk_type === 'module_header' && !fields.module && d.file_path) {
    fields.module = moduleNameFromPath(d.file_path)
  }

  // Only write fields that are currently blank (unless force=true)
  const toWrite: Record<string, string> = {}
  for (const [k, v] of Object.entries(fields)) {
    if (force || !d[k]) toWrite[k] = v
  }

  if (!Object.keys(toWrite).length) {
    return { success: true, fields_set: [], skipped: true, updatedChunk: chunk }
  }

  const updatedData = { ...d, ...toWrite, doc_confidence: 'ai_generated' }
  const updatedChunk = { ...chunk, data: updatedData }

  // Validate before persisting
  const validationErrors = validateChunkDocs(updatedChunk)
  if (validationErrors.length > 0) {
    return { success: false, fields_set: [], error: `Validation failed: ${validationErrors.join(', ')}`, validation_errors: validationErrors, updatedChunk }
  }

  const { error: updateError } = await adminDb
    .from('items')
    .update({
      data: updatedData,
      updated_at: new Date().toISOString(),
    })
    .eq('id', chunk.id)

  if (updateError) {
    return { success: false, fields_set: [], error: updateError.message }
  }

  // Write inline JSDoc to the source file if we have a repo path
  if (repoPath) {
    writeJsDocToSource(updatedChunk, repoPath)
  }

  return { success: true, fields_set: Object.keys(toWrite), updatedChunk }
}

// ─── HANDLER ──────────────────────────────────────────────────────────────────

export const handler = createHandler(async (ctx, body) => {
  const action = ctx.query?.action

  // ── GAP REPORT — list chunks missing contract fields ─────────────────────────
  if (action === 'gap_report') {
    const projectId = ctx.query?.project_id || body?.project_id
    if (!projectId) throw new Error('project_id is required')

    const chunks = await loadAllChunks(projectId)

    const undocumented = chunks.filter(needsDocumentation)
    const documented   = chunks.filter(c => !needsDocumentation(c))
    const aiGenerated  = documented.filter(c => c.data?.doc_confidence === 'ai_generated')
    const humanVerified= documented.filter(c => c.data?.doc_confidence === 'human_verified')

    return {
      project_id: projectId,
      total_chunks: chunks.length,
      undocumented_count: undocumented.length,
      documented_count: documented.length,
      ai_generated_count: aiGenerated.length,
      human_verified_count: humanVerified.length,
      coverage_pct: chunks.length
        ? Math.round((documented.length / chunks.length) * 100) : 0,
      undocumented: undocumented.slice(0, 100).map(c => ({
        id: c.id,
        chunk_name: c.data?.chunk_name || c.title,
        file_path: c.data?.file_path || '',
        chunk_type: c.data?.chunk_type || '',
        line_count: (c.data?.line_end || 0) - (c.data?.line_start || 0) + 1,
        token_count: c.data?.token_count || 0,
      })),
    }
  }

  // ── COST ESTIMATE — pre-flight token + USD cost projection ───────────────────
  if (action === 'estimate') {
    const projectId = ctx.query?.project_id || body?.project_id
    const model = ctx.query?.model || body?.model || DEFAULT_MODEL
    const modelPrices = body?.model_prices || MODEL_PRICES[model] || MODEL_PRICES[DEFAULT_MODEL]
    const filePath = ctx.query?.file_path || body?.file_path

    if (!projectId) throw new Error('project_id is required')

    const chunks = await loadAllChunks(projectId, filePath)
    const toDocument = chunks.filter(needsDocumentation)

    let totalInputTokens = 0
    const breakdown: any[] = []

    for (const chunk of toDocument) {
      const prompt = buildDocPrompt(chunk)
      const inputTokens = estimateTokens(prompt) + SYSTEM_PROMPT_OVERHEAD_TOKENS
      totalInputTokens += inputTokens
      breakdown.push({
        chunk_name: chunk.data?.chunk_name || chunk.title,
        file_path: chunk.data?.file_path || '',
        input_tokens: inputTokens,
      })
    }

    const totalOutputTokens = toDocument.length * ESTIMATED_OUTPUT_TOKENS_PER_CHUNK
    const totalTokens = totalInputTokens + totalOutputTokens

    const inputCostUsd  = (totalInputTokens  / 1_000_000) * modelPrices.input_per_1m
    const outputCostUsd = (totalOutputTokens / 1_000_000) * modelPrices.output_per_1m
    const totalCostUsd  = inputCostUsd + outputCostUsd

    return {
      project_id: projectId,
      model,
      file_path: filePath || null,
      model_prices: modelPrices,
      chunks_to_document: toDocument.length,
      total_chunks: chunks.length,
      already_documented: chunks.length - toDocument.length,
      estimated_input_tokens: totalInputTokens,
      estimated_output_tokens: totalOutputTokens,
      estimated_total_tokens: totalTokens,
      estimated_cost_usd: parseFloat(totalCostUsd.toFixed(4)),
      estimated_cost_usd_breakdown: {
        input:  parseFloat(inputCostUsd.toFixed(4)),
        output: parseFloat(outputCostUsd.toFixed(4)),
      },
      // Top 20 most expensive chunks by input token count
      top_chunks_by_cost: breakdown
        .sort((a, b) => b.input_tokens - a.input_tokens)
        .slice(0, 20),
    }
  }

  // ── DOCUMENT ONE CHUNK ────────────────────────────────────────────────────────
  if (action === 'doc_chunk') {
    const chunkId = body?.chunk_id || ctx.query?.chunk_id
    const model = body?.model || ctx.query?.model || DEFAULT_MODEL
    const force = body?.force ?? false
    if (!chunkId) throw new Error('chunk_id is required')

    const { data: chunk, error } = await adminDb
      .from('items')
      .select('id, title, data')
      .eq('id', chunkId)
      .maybeSingle()

    if (error || !chunk) throw new Error(`Chunk not found: ${chunkId}`)

    const projectId = chunk.data?.project_id
    let repoPath = ''
    if (projectId) {
      const { data: projectItem } = await adminDb.from('items').select('data').eq('id', projectId).maybeSingle()
      repoPath = projectItem?.data?.repo_path || ''
    }

    const result = await documentChunk(chunk, model, force, repoPath)
    return { chunk_id: chunkId, chunk_name: chunk.data?.chunk_name || chunk.title, ...result }
  }

  // ── VALIDATE DOCUMENTATION ───────────────────────────────────────────────────
  // Lists all chunks that fail the canonical documentation contract.
  // Optional file_path filter. Returns per-chunk validation errors.
  if (action === 'validate') {
    const projectId = ctx.query?.project_id || body?.project_id
    const filePath = ctx.query?.file_path || body?.file_path
    if (!projectId) throw new Error('project_id is required')

    const chunks = await loadAllChunks(projectId, filePath)
    const invalid: any[] = []
    for (const chunk of chunks) {
      const errors = validateChunkDocs(chunk)
      if (errors.length > 0) {
        invalid.push({
          chunk_id: chunk.id,
          chunk_name: chunk.data?.chunk_name || chunk.title,
          file_path: chunk.data?.file_path || '',
          chunk_type: chunk.data?.chunk_type || '',
          errors,
        })
      }
    }

    return {
      project_id: projectId,
      file_path: filePath || null,
      total_chunks: chunks.length,
      valid_chunks: chunks.length - invalid.length,
      invalid_chunks: invalid.length,
      invalid,
    }
  }

  // ── DOCUMENT ALL UNDOCUMENTED CHUNKS IN A PROJECT (paginated, parallel) ────────
  //
  // Supports limit + offset so the UI can call this in pages:
  //   POST { project_id, limit: 50, offset: 0 }  → processes chunks 0–49
  //   POST { project_id, limit: 50, offset: 50 } → processes chunks 50–99
  //   ... repeat until done_total === total_to_document
  //
  // Also supports a single-file path:
  //   POST { project_id, file_path: '.framework/functions/_shared/principal.ts' }
  //
  // Within each page, all LLM calls run in parallel (Promise.all), then
  // DB writes are batched to avoid N+1 round-trips.
  if (action === 'document') {
    const projectId = body?.project_id
    if (!projectId) throw new Error('project_id is required')

    const model     = body?.model     || DEFAULT_MODEL
    const dryRun    = body?.dry_run   ?? false
    const force     = body?.force     ?? false
    const chunkIds: string[] | undefined = body?.chunk_ids
    const filePath: string | undefined = body?.file_path
    const pageLimit = Math.min(parseInt(body?.limit || '50', 10), 100) // max 100 per page
    const pageOffset= parseInt(body?.offset || '0', 10)
    const concurrency = Math.min(parseInt(body?.concurrency || '20', 10), 30)

    // ── 1. Load chunks for this page ───────────────────────────────────────────
    // Use the paged helper to avoid Supabase's default 1000-row select limit.
    let stableList: any[]
    if (chunkIds?.length) {
      let q = adminDb
        .from('items')
        .select('id, title, data')
        .filter('data->>kb_type', 'eq', 'cil_chunk')
        .filter('data->>project_id', 'eq', projectId)
        .eq('is_active', true)
        .in('id', chunkIds)
        .order('id')
      if (filePath) q = q.filter('data->>file_path', 'eq', filePath)
      const { data, error } = await q
      if (error) throw new Error(`Failed to load chunks: ${error.message}`)
      stableList = data || []
    } else {
      stableList = await loadAllChunks(projectId, filePath)
    }

    const totalChunks = stableList.length
    const pageChunks  = stableList.slice(pageOffset, pageOffset + pageLimit)

    // Within the page, skip already-documented ones (unless force=true)
    const allToDocument = force ? pageChunks : pageChunks.filter(needsDocumentation)
    // total_to_document is reported as total undocumented across ALL chunks for the estimate banner
    const totalToDocument = force
      ? stableList.length
      : stableList.filter(needsDocumentation).length

    if (dryRun) {
      return {
        dry_run: true,
        project_id: projectId,
        model,
        total_to_document: totalToDocument,
        page_offset: pageOffset,
        page_limit: pageLimit,
        page_chunks: pageChunks.length,
        chunk_names: pageChunks.map(c => c.data?.chunk_name || c.title),
      }
    }

    // ── 2. Call LLM for chunks in this page that need documenting ────────────────
    // Process in sub-batches of `concurrency` to respect rate limits
    const results: Array<{ chunk: any; fields: Record<string, string> | null; error?: string }> = []

    for (let i = 0; i < allToDocument.length; i += concurrency) {
      const subBatch = allToDocument.slice(i, i + concurrency)
      const subResults = await Promise.all(subBatch.map(async chunk => {
        const d = chunk.data || {}
        if (!d.code_content && !d.chunk_name) {
          return { chunk, fields: null, error: 'No code content' }
        }
        const prompt = buildDocPrompt(chunk)
        try {
          const raw = await callLLM(prompt, model)
          const fields = parseDocResponse(raw)
          return { chunk, fields: Object.keys(fields).length ? fields : null, error: Object.keys(fields).length ? undefined : 'Unparseable LLM response' }
        } catch (err) {
          return { chunk, fields: null, error: err instanceof Error ? err.message : String(err) }
        }
      }))
      results.push(...subResults)
    }

    // ── 3. Validate and batch DB writes — one update per successful chunk ─────
    let documented = 0, validated = 0, skipped = 0, failed = 0
    const errors: string[] = []
    const updatedChunks: any[] = []

    await Promise.all(results.map(async ({ chunk, fields, error: llmError }) => {
      if (!fields) {
        if (llmError) { failed++; errors.push(`${chunk.data?.chunk_name || chunk.id}: ${llmError}`) }
        else skipped++
        return
      }
      const d = chunk.data || {}
      const toWrite: Record<string, string> = {}
      for (const [k, v] of Object.entries(fields)) {
        if (force || !d[k]) toWrite[k] = v
      }
      if (!Object.keys(toWrite).length) { skipped++; return }

      const updatedData = { ...d, ...toWrite, doc_confidence: 'ai_generated' }
      const updatedChunk = { ...chunk, data: updatedData }

      // Validate before persisting
      const validationErrors = validateChunkDocs(updatedChunk)
      if (validationErrors.length > 0) {
        failed++
        errors.push(`${chunk.data?.chunk_name || chunk.id}: ${validationErrors.join(', ')}`)
        await adminDb
          .from('items')
          .update({ data: { ...updatedData, doc_validation_errors: validationErrors }, updated_at: new Date().toISOString() })
          .eq('id', chunk.id)
        return
      }

      const { error: updateError } = await adminDb
        .from('items')
        .update({ data: updatedData, updated_at: new Date().toISOString() })
        .eq('id', chunk.id)

      if (updateError) {
        failed++; errors.push(`${chunk.data?.chunk_name || chunk.id}: ${updateError.message}`)
      } else {
        documented++; validated++; updatedChunks.push(updatedChunk)
      }
    }))

    // ── 4. Write JSDoc comments back to source files ───────────────────────────
    // Group by file so we can rewrite each file once (avoids parallel write races)
    if (updatedChunks.length > 0) {
      const { data: projectItem } = await adminDb.from('items').select('id, data').eq('id', projectId).maybeSingle()
      const repoPath = projectItem?.data?.repo_path || ''
      if (repoPath) {
        const byFile = new Map<string, any[]>()
        for (const chunk of updatedChunks) {
          const relPath = chunk.data?.file_path
          if (!relPath) continue
          if (!byFile.has(relPath)) byFile.set(relPath, [])
          byFile.get(relPath)!.push(chunk)
        }
        for (const [relPath, chunks] of byFile) {
          const absPath = path.join(repoPath, relPath)
          if (!fs.existsSync(absPath)) continue
          const content = fs.readFileSync(absPath, 'utf-8')
          const fileLines = content.split('\n')
          const newLines = [...fileLines]
          // Sort by marker position descending so inserting doesn't shift earlier indices
          const sorted = [...chunks].sort((a, b) => (b.data?.line_start || 0) - (a.data?.line_start || 0))
          for (const chunk of sorted) {
            const markerRe = new RegExp(`CHUNK_START:\\s*${escapeRegExp(chunk.data?.chunk_name || chunk.title)}`)
            let markerIdx = -1
            const searchStart = Math.max(0, (chunk.data?.line_start || 1) - 5)
            const searchEnd = Math.min(newLines.length, (chunk.data?.line_start || 1) + 5)
            for (let i = searchStart; i < searchEnd; i++) {
              if (markerRe.test(newLines[i] || '')) { markerIdx = i; break }
            }
            if (markerIdx === -1) continue
            const jsDoc = buildJsDoc(chunk)
            if (!jsDoc) continue
            const existing = findExistingCilJsDoc(newLines, markerIdx)
            if (existing) {
              newLines.splice(existing.start, existing.end - existing.start + 1, jsDoc)
            } else {
              // Insert JSDoc immediately BELOW the CHUNK_START marker
              newLines.splice(markerIdx + 1, 0, jsDoc)
            }
          }
          fs.writeFileSync(absPath, newLines.join('\n'), 'utf-8')
        }
      }
    }

    // ── 5. Update project cost on the last page ────────────────────────────────
    const nextOffset = pageOffset + pageLimit
    const isLastPage = nextOffset >= totalChunks
    if (isLastPage && documented > 0) {
      const { data: projectItem } = await adminDb.from('items').select('id, data').eq('id', projectId).maybeSingle()
      if (projectItem) {
        const prices = MODEL_PRICES[model] || MODEL_PRICES[DEFAULT_MODEL]
        const approxCost = (documented * (ESTIMATED_OUTPUT_TOKENS_PER_CHUNK + 800) / 1_000_000) * ((prices.input_per_1m + prices.output_per_1m) / 2)
        await adminDb.from('items').update({
          data: { ...projectItem.data, total_ingestion_cost_usd: (projectItem.data?.total_ingestion_cost_usd || 0) + approxCost, last_ingested_at: new Date().toISOString() },
          updated_at: new Date().toISOString(),
        }).eq('id', projectId)
      }
    }

    return {
      project_id: projectId,
      model,
      total_to_document: totalToDocument,
      total_chunks: totalChunks,
      page_offset: pageOffset,
      page_limit: pageLimit,
      documented,
      validated,
      skipped,
      failed,
      errors: errors.slice(0, 20),
      // Pagination is against the stable full-chunk list, not the filtered undocumented list
      has_more: nextOffset < totalChunks,
      next_offset: nextOffset < totalChunks ? nextOffset : null,
      done_total: pageOffset + pageChunks.length,
    }
  }

  throw new Error(`Unknown action: ${action}. Valid: estimate, gap_report, document, doc_chunk, validate`)
})