{"version":3,"file":"sources-KBGEA_E6.mjs","names":[],"sources":["../src/cli/commands/memory/sources.ts"],"sourcesContent":["/**\n * `pai memory sources` — what the indexer is actually taking in.\n *\n * The index had grown to ~1.5M chunks against a vault of ~2,700 notes, and\n * nothing in the CLI could show why. `memory status` refuses to report at all\n * when the backend is not SQLite, and `daemon status` gives three totals with no\n * composition — so answering \"what is it indexing, and why is it re-indexing\"\n * meant hand-writing aggregate SQL against the container.\n *\n * The cause, when it was finally measured, was visible in one breakdown: the\n * vault indexer follows symlinks out of the vault into cloud-synced trees, and\n * those trees have mtimes rewritten by the sync client on content that has not\n * changed. Each rewrite re-chunks the file, and re-chunking assigns new chunk\n * ids, which discards the embeddings — so the embedder was re-doing work\n * indefinitely while never catching up.\n *\n * Hence the four sections below. Each one exists because it was needed:\n *   composition — the vault dwarfing everything else is the first clue\n *   roots       — where content enters from, which is how symlink leakage shows\n *   heaviest    — single files contributing thousands of chunks (attachments)\n *   churn       — chunks rewritten per day, which is what distinguishes a\n *                 backlog that will finish from a treadmill that never will\n */\n\nimport { ok, warn, dim, bold, header, renderTable } from \"../../utils.js\";\nimport type { StorageBackend } from \"../../../storage/interface.js\";\n\ninterface Row {\n  label: string;\n  chunks: number;\n  embedded: number;\n}\n\ninterface ChurnRow {\n  day: string;\n  chunks: number;\n  embedded: number;\n}\n\n/** A backend that can answer aggregate SQL. Both concrete backends can. */\ninterface Queryable {\n  backendType: \"sqlite\" | \"postgres\";\n  getPool?: () => { query: (sql: string) => Promise<{ rows: unknown[] }> };\n  getSqliteDb?: () => { prepare: (sql: string) => { all: () => unknown[] } };\n}\n\n/**\n * Run one aggregate query against whichever backend is configured.\n *\n * The two dialects differ only in the epoch conversion, so the SQL is passed in\n * per-dialect rather than abstracted — an abstraction over four reporting\n * queries would cost more than it saves.\n */\nasync function queryRows(\n  backend: Queryable,\n  pgSql: string,\n  sqliteSql: string\n): Promise<Record<string, unknown>[]> {\n  if (backend.backendType === \"postgres\" && backend.getPool) {\n    const res = await backend.getPool().query(pgSql);\n    return res.rows as Record<string, unknown>[];\n  }\n  if (backend.getSqliteDb) {\n    return backend.getSqliteDb().prepare(sqliteSql).all() as Record<string, unknown>[];\n  }\n  return [];\n}\n\nconst num = (v: unknown): number => Number(v ?? 0);\nconst pct = (part: number, whole: number): string =>\n  whole === 0 ? \"—\" : `${Math.round((part / whole) * 100)}%`;\n\n/** Group the leading path segments, which is where content enters the index. */\nexport function rootOf(path: string, depth = 2): string {\n  const parts = path.split(\"/\").filter(Boolean);\n  if (parts.length <= depth) return path;\n  return parts.slice(0, depth).join(\"/\") + \"/…\";\n}\n\nexport async function cmdMemorySources(\n  backend: StorageBackend,\n  opts: { limit?: number } = {}\n): Promise<void> {\n  const q = backend as unknown as Queryable;\n  const limit = opts.limit ?? 8;\n\n  // ---- composition -------------------------------------------------------\n  const comp = (await queryRows(\n    q,\n    `SELECT source, tier, COUNT(*) AS chunks, COUNT(embedding) AS embedded\n       FROM pai_chunks GROUP BY source, tier ORDER BY COUNT(*) DESC`,\n    `SELECT source, tier, COUNT(*) AS chunks,\n            SUM(CASE WHEN embedding IS NOT NULL THEN 1 ELSE 0 END) AS embedded\n       FROM memory_chunks GROUP BY source, tier ORDER BY COUNT(*) DESC`\n  )) as Array<{ source: string; tier: string; chunks: unknown; embedded: unknown }>;\n\n  if (comp.length === 0) {\n    console.log();\n    console.log(warn(`  Nothing indexed yet, or the backend is unreachable.`));\n    console.log(dim(`  Backend: ${q.backendType}`));\n    console.log();\n    return;\n  }\n\n  const totalChunks = comp.reduce((s, r) => s + num(r.chunks), 0);\n  const totalEmbedded = comp.reduce((s, r) => s + num(r.embedded), 0);\n\n  console.log();\n  console.log(header(`What the indexer has taken in`));\n  console.log();\n  console.log(\n    `  ${bold(totalChunks.toLocaleString())} chunks   ` +\n      `${totalEmbedded.toLocaleString()} embedded (${pct(totalEmbedded, totalChunks)})   ` +\n      dim(`backend: ${q.backendType}`)\n  );\n  console.log();\n\n  console.log(\n    renderTable(\n      [\"source / tier\", \"chunks\", \"share\", \"embedded\"],\n      comp.map((r) => [\n        `${r.source} / ${r.tier}`,\n        num(r.chunks).toLocaleString(),\n        pct(num(r.chunks), totalChunks),\n        `${num(r.embedded).toLocaleString()} (${pct(num(r.embedded), num(r.chunks))})`,\n      ])\n    )\n  );\n\n  // ---- where it enters from ---------------------------------------------\n  const paths = (await queryRows(\n    q,\n    `SELECT path, COUNT(*) AS chunks FROM pai_chunks GROUP BY path`,\n    `SELECT path, COUNT(*) AS chunks FROM memory_chunks GROUP BY path`\n  )) as Array<{ path: string; chunks: unknown }>;\n\n  const byRoot = new Map<string, number>();\n  for (const r of paths) {\n    const k = rootOf(String(r.path ?? \"\"));\n    byRoot.set(k, (byRoot.get(k) ?? 0) + num(r.chunks));\n  }\n  const roots = [...byRoot.entries()].sort((a, b) => b[1] - a[1]).slice(0, limit);\n\n  console.log();\n  console.log(header(`Where it comes from`));\n  console.log(dim(`  A root you did not expect here is usually a symlink leading out of the vault.`));\n  console.log();\n  console.log(\n    renderTable(\n      [\"root\", \"chunks\", \"share\"],\n      roots.map(([k, v]) => [k, v.toLocaleString(), pct(v, totalChunks)])\n    )\n  );\n\n  // ---- heaviest single files -------------------------------------------\n  const heaviest = [...paths]\n    .sort((a, b) => num(b.chunks) - num(a.chunks))\n    .slice(0, limit);\n\n  console.log();\n  console.log(header(`Heaviest single files`));\n  console.log();\n  console.log(\n    renderTable(\n      [\"chunks\", \"path\"],\n      heaviest.map((r) => [num(r.chunks).toLocaleString(), tail(String(r.path ?? \"\"), 62)])\n    )\n  );\n\n  // ---- churn ------------------------------------------------------------\n  // The section that distinguishes a backlog from a treadmill. A day with a\n  // large chunk count and a small embedded count means those chunks were\n  // rewritten and their embeddings thrown away.\n  const churn = (await queryRows(\n    q,\n    `SELECT to_char(to_timestamp(updated_at/1000),'YYYY-MM-DD') AS day,\n            COUNT(*) AS chunks, COUNT(embedding) AS embedded\n       FROM pai_chunks GROUP BY day ORDER BY day DESC LIMIT 10`,\n    `SELECT date(updated_at/1000,'unixepoch') AS day, COUNT(*) AS chunks,\n            SUM(CASE WHEN embedding IS NOT NULL THEN 1 ELSE 0 END) AS embedded\n       FROM memory_chunks GROUP BY day ORDER BY day DESC LIMIT 10`\n  )) as unknown as ChurnRow[];\n\n  console.log();\n  console.log(header(`Rewritten per day`));\n  console.log(\n    dim(`  Many chunks with few embedded means they were re-chunked and their`)\n  );\n  console.log(dim(`  embeddings discarded — work the embedder has to redo.`));\n  console.log();\n  console.log(\n    renderTable(\n      [\"day\", \"chunks touched\", \"of those embedded\"],\n      churn.map((r) => [\n        String(r.day),\n        num(r.chunks).toLocaleString(),\n        `${num(r.embedded).toLocaleString()} (${pct(num(r.embedded), num(r.chunks))})`,\n      ])\n    )\n  );\n\n  const missing = totalChunks - totalEmbedded;\n  console.log();\n  if (missing > 0) {\n    console.log(`  ${bold(missing.toLocaleString())} chunks still need embedding.`);\n  } else {\n    console.log(ok(`  Everything indexed is embedded.`));\n  }\n  console.log();\n}\n\n/** Keep the informative end of a long path. */\nfunction tail(s: string, n: number): string {\n  return s.length <= n ? s : \"…\" + s.slice(-(n - 1));\n}\n\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAqDA,eAAe,UACb,SACA,OACA,WACoC;AACpC,KAAI,QAAQ,gBAAgB,cAAc,QAAQ,QAEhD,SADY,MAAM,QAAQ,SAAS,CAAC,MAAM,MAAM,EACrC;AAEb,KAAI,QAAQ,YACV,QAAO,QAAQ,aAAa,CAAC,QAAQ,UAAU,CAAC,KAAK;AAEvD,QAAO,EAAE;;AAGX,MAAM,OAAO,MAAuB,OAAO,KAAK,EAAE;AAClD,MAAM,OAAO,MAAc,UACzB,UAAU,IAAI,MAAM,GAAG,KAAK,MAAO,OAAO,QAAS,IAAI,CAAC;;AAG1D,SAAgB,OAAO,MAAc,QAAQ,GAAW;CACtD,MAAM,QAAQ,KAAK,MAAM,IAAI,CAAC,OAAO,QAAQ;AAC7C,KAAI,MAAM,UAAU,MAAO,QAAO;AAClC,QAAO,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,IAAI,GAAG;;AAG3C,eAAsB,iBACpB,SACA,OAA2B,EAAE,EACd;CACf,MAAM,IAAI;CACV,MAAM,QAAQ,KAAK,SAAS;CAG5B,MAAM,OAAQ,MAAM,UAClB,GACA;sEAEA;;wEAGD;AAED,KAAI,KAAK,WAAW,GAAG;AACrB,UAAQ,KAAK;AACb,UAAQ,IAAI,KAAK,wDAAwD,CAAC;AAC1E,UAAQ,IAAI,IAAI,cAAc,EAAE,cAAc,CAAC;AAC/C,UAAQ,KAAK;AACb;;CAGF,MAAM,cAAc,KAAK,QAAQ,GAAG,MAAM,IAAI,IAAI,EAAE,OAAO,EAAE,EAAE;CAC/D,MAAM,gBAAgB,KAAK,QAAQ,GAAG,MAAM,IAAI,IAAI,EAAE,SAAS,EAAE,EAAE;AAEnE,SAAQ,KAAK;AACb,SAAQ,IAAI,OAAO,gCAAgC,CAAC;AACpD,SAAQ,KAAK;AACb,SAAQ,IACN,KAAK,KAAK,YAAY,gBAAgB,CAAC,CAAC,YACnC,cAAc,gBAAgB,CAAC,aAAa,IAAI,eAAe,YAAY,CAAC,QAC/E,IAAI,YAAY,EAAE,cAAc,CACnC;AACD,SAAQ,KAAK;AAEb,SAAQ,IACN,YACE;EAAC;EAAiB;EAAU;EAAS;EAAW,EAChD,KAAK,KAAK,MAAM;EACd,GAAG,EAAE,OAAO,KAAK,EAAE;EACnB,IAAI,EAAE,OAAO,CAAC,gBAAgB;EAC9B,IAAI,IAAI,EAAE,OAAO,EAAE,YAAY;EAC/B,GAAG,IAAI,EAAE,SAAS,CAAC,gBAAgB,CAAC,IAAI,IAAI,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;EAC7E,CAAC,CACH,CACF;CAGD,MAAM,QAAS,MAAM,UACnB,GACA,iEACA,mEACD;CAED,MAAM,yBAAS,IAAI,KAAqB;AACxC,MAAK,MAAM,KAAK,OAAO;EACrB,MAAM,IAAI,OAAO,OAAO,EAAE,QAAQ,GAAG,CAAC;AACtC,SAAO,IAAI,IAAI,OAAO,IAAI,EAAE,IAAI,KAAK,IAAI,EAAE,OAAO,CAAC;;CAErD,MAAM,QAAQ,CAAC,GAAG,OAAO,SAAS,CAAC,CAAC,MAAM,GAAG,MAAM,EAAE,KAAK,EAAE,GAAG,CAAC,MAAM,GAAG,MAAM;AAE/E,SAAQ,KAAK;AACb,SAAQ,IAAI,OAAO,sBAAsB,CAAC;AAC1C,SAAQ,IAAI,IAAI,kFAAkF,CAAC;AACnG,SAAQ,KAAK;AACb,SAAQ,IACN,YACE;EAAC;EAAQ;EAAU;EAAQ,EAC3B,MAAM,KAAK,CAAC,GAAG,OAAO;EAAC;EAAG,EAAE,gBAAgB;EAAE,IAAI,GAAG,YAAY;EAAC,CAAC,CACpE,CACF;CAGD,MAAM,WAAW,CAAC,GAAG,MAAM,CACxB,MAAM,GAAG,MAAM,IAAI,EAAE,OAAO,GAAG,IAAI,EAAE,OAAO,CAAC,CAC7C,MAAM,GAAG,MAAM;AAElB,SAAQ,KAAK;AACb,SAAQ,IAAI,OAAO,wBAAwB,CAAC;AAC5C,SAAQ,KAAK;AACb,SAAQ,IACN,YACE,CAAC,UAAU,OAAO,EAClB,SAAS,KAAK,MAAM,CAAC,IAAI,EAAE,OAAO,CAAC,gBAAgB,EAAE,KAAK,OAAO,EAAE,QAAQ,GAAG,EAAE,GAAG,CAAC,CAAC,CACtF,CACF;CAMD,MAAM,QAAS,MAAM,UACnB,GACA;;iEAGA;;mEAGD;AAED,SAAQ,KAAK;AACb,SAAQ,IAAI,OAAO,oBAAoB,CAAC;AACxC,SAAQ,IACN,IAAI,uEAAuE,CAC5E;AACD,SAAQ,IAAI,IAAI,0DAA0D,CAAC;AAC3E,SAAQ,KAAK;AACb,SAAQ,IACN,YACE;EAAC;EAAO;EAAkB;EAAoB,EAC9C,MAAM,KAAK,MAAM;EACf,OAAO,EAAE,IAAI;EACb,IAAI,EAAE,OAAO,CAAC,gBAAgB;EAC9B,GAAG,IAAI,EAAE,SAAS,CAAC,gBAAgB,CAAC,IAAI,IAAI,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;EAC7E,CAAC,CACH,CACF;CAED,MAAM,UAAU,cAAc;AAC9B,SAAQ,KAAK;AACb,KAAI,UAAU,EACZ,SAAQ,IAAI,KAAK,KAAK,QAAQ,gBAAgB,CAAC,CAAC,+BAA+B;KAE/E,SAAQ,IAAI,GAAG,oCAAoC,CAAC;AAEtD,SAAQ,KAAK;;;AAIf,SAAS,KAAK,GAAW,GAAmB;AAC1C,QAAO,EAAE,UAAU,IAAI,IAAI,MAAM,EAAE,MAAM,EAAE,IAAI,GAAG"}