import { mkdir, writeFile } from 'node:fs/promises' import { join } from 'node:path' import type { MetabaseClient } from '../client.js' import { Resolver } from '../resolver.js' import type { MetabaseCard, MetabaseCollection, MetabaseDashboard, MetabaseDashcard, StateFile, } from '../types.js' import { upsertState } from '../state.js' const RESOURCES_DIR = '.' function sanitizeFilename(name: string): string { return name .toLowerCase() .replace(/[^a-z0-9]+/g, '-') .replace(/^-|-$/g, '') } function escapeTs(str: string): string { return str .replace(/\\/g, '\\\\') .replace(/'/g, "\\'") .replace(/`/g, '\\`') .replace(/\$/g, '\\$') } export async function pull(client: MetabaseClient, outputDir?: string, importBase?: string): Promise { const baseDir = outputDir ?? RESOURCES_DIR const typesImport = importBase ?? 'metabase-iac/types' const resolver = new Resolver() await resolver.init(client) const state: StateFile = [] // Create directories await mkdir(join(baseDir, 'collections'), { recursive: true }) await mkdir(join(baseDir, 'questions'), { recursive: true }) await mkdir(join(baseDir, 'dashboards'), { recursive: true }) // 1. Pull collections console.log('Pulling collections...') const collections = await client.getCollections() const activeCollections = collections.filter((c) => !c.archived && c.id !== 0) // Build parent lookup const collectionsById = new Map() for (const c of activeCollections) { collectionsById.set(c.id, c) } for (const c of activeCollections) { // Skip system collections if (c.name === 'Our analytics' && !c.parent_id) continue const parentName = c.parent_id ? collectionsById.get(c.parent_id)?.name : undefined const filename = sanitizeFilename(c.name) const content = buildCollectionFile(c, parentName, typesImport) await writeFile(join(baseDir, 'collections', `${filename}.ts`), content) upsertState(state, { kind: 'collection', name: c.name, metabaseId: c.id }) console.log(` collection: ${c.name} (id: ${c.id}, slug: ${c.slug})`) } // 2. Pull cards (questions) console.log('Pulling questions...') const cards = await client.getCards() const activeCards = cards.filter((c) => !c.archived) for (const card of activeCards) { const collectionName = resolver.getCollectionName(card.collection_id) const dbName = resolver.getDatabaseName(card.database_id) ?? `database-${card.database_id}` const filename = sanitizeFilename(card.name) // Detect native queries: type field may be at dataset_query.type or inferred from native key const dq = card.dataset_query const isNative = dq.type === 'native' || (dq.native && dq.native.query) if (isNative && dq.native) { // Write SQL/MongoDB query as separate file + meta const query = dq.native.query const ext = guessQueryExtension(resolver, dbName) await writeFile(join(baseDir, 'questions', `${filename}.${ext}`), query) const meta: Record = { id: card.id, name: card.name, collection: collectionName, database: dbName, display: card.display, description: card.description, visualizationSettings: simplifyVizSettings(card.visualization_settings), } if (card.type && card.type !== 'question') { meta.cardType = card.type } if (dq.native['template-tags'] && Object.keys(dq.native['template-tags']).length > 0) { meta.templateTags = dq.native['template-tags'] } await writeFile( join(baseDir, 'questions', `${filename}.meta.json`), JSON.stringify(meta, null, 2) + '\n', ) console.log(` native question: ${card.name} (id: ${card.id}, ${ext})`) } else { // Structured query → TypeScript file // Capture the full dataset_query including stages-based (MLv2) queries const content = buildStructuredQuestionFile(card, collectionName, dbName, typesImport) await writeFile(join(baseDir, 'questions', `${filename}.ts`), content) console.log(` structured question: ${card.name} (id: ${card.id})`) } upsertState(state, { kind: 'question', name: card.name, metabaseId: card.id, collection: collectionName, }) } // 3. Pull dashboards console.log('Pulling dashboards...') const dashboardsList = await client.getDashboards() const activeDashboards = dashboardsList.filter((d) => !d.archived) for (const dashSummary of activeDashboards) { // Get full dashboard with cards const dash = await client.getDashboard(dashSummary.id) const collectionName = resolver.getCollectionName(dash.collection_id) const filename = sanitizeFilename(dash.name) const content = buildDashboardFile(dash, collectionName, activeCards, typesImport) await writeFile(join(baseDir, 'dashboards', `${filename}.ts`), content) upsertState(state, { kind: 'dashboard', name: dash.name, metabaseId: dash.id, collection: collectionName, }) console.log(` dashboard: ${dash.name} (id: ${dash.id})`) } // 4. Pull snippets console.log('Pulling snippets...') try { const snippets = await client.getSnippets() const activeSnippets = snippets.filter((s) => !s.archived) if (activeSnippets.length > 0) { await mkdir(join(baseDir, 'snippets'), { recursive: true }) for (const s of activeSnippets) { const filename = sanitizeFilename(s.name) await writeFile(join(baseDir, 'snippets', `${filename}.sql`), s.content) const meta: Record = { id: s.id, name: s.name, description: s.description, } await writeFile( join(baseDir, 'snippets', `${filename}.meta.json`), JSON.stringify(meta, null, 2) + '\n', ) upsertState(state, { kind: 'snippet', name: s.name, metabaseId: s.id }) console.log(` snippet: ${s.name} (id: ${s.id})`) } } } catch (err) { console.log(' (snippets not available, skipping)') } return state } function guessQueryExtension(resolver: Resolver, dbName: string): string { const engine = resolver.getDatabaseEngine(dbName) if (engine === 'mongo') return 'mongodb' return 'sql' } function buildCollectionFile(c: MetabaseCollection, parentName: string | undefined, typesImport: string): string { const lines = [`import { collection } from '${typesImport}'`, '', 'export default collection({'] lines.push(` id: ${c.id},`) if (c.slug) { lines.push(` slug: '${escapeTs(c.slug)}',`) } lines.push(` name: '${escapeTs(c.name)}',`) if (c.description) { lines.push(` description: '${escapeTs(c.description)}',`) } if (parentName) { lines.push(` parent: '${escapeTs(parentName)}',`) } lines.push('})', '') return lines.join('\n') } function buildStructuredQuestionFile(card: MetabaseCard, collectionName: string, dbName: string, typesImport: string): string { const dq = card.dataset_query // Extract the query payload: could be legacy .query or newer .stages const query = dq.query ?? {} const stages = (dq as any).stages const vizSettings = simplifyVizSettings(card.visualization_settings) const lines = [`import { structuredQuestion } from '${typesImport}'`, '', 'export default structuredQuestion({'] lines.push(` id: ${card.id},`) lines.push(` name: '${escapeTs(card.name)}',`) if (card.description) { lines.push(` description: '${escapeTs(card.description)}',`) } lines.push(` collection: '${escapeTs(collectionName)}',`) lines.push(` database: '${escapeTs(dbName)}',`) lines.push(` display: '${card.display}',`) if (card.type && card.type !== 'question') { lines.push(` cardType: '${card.type}',`) } if (stages && Array.isArray(stages) && stages.length > 0) { // MLv2 stages-based query — store the full dataset_query for fidelity const cleanDq = cleanDatasetQuery(dq) lines.push(` datasetQuery: ${indentJson(cleanDq)},`) } else if (Object.keys(query).length > 0) { lines.push(` query: ${indentJson(query)},`) } else { // Fallback: store the full dataset_query const cleanDq = cleanDatasetQuery(dq) lines.push(` datasetQuery: ${indentJson(cleanDq)},`) } if (Object.keys(vizSettings).length > 0) { lines.push(` visualizationSettings: ${indentJson(vizSettings)},`) } lines.push('})', '') return lines.join('\n') } function buildDashboardFile( dash: MetabaseDashboard, collectionName: string, allCards: MetabaseCard[], typesImport: string, ): string { const cardsById = new Map() for (const c of allCards) cardsById.set(c.id, c) const lines = [`import { dashboard } from '${typesImport}'`, '', 'export default dashboard({'] lines.push(` id: ${dash.id},`) lines.push(` name: '${escapeTs(dash.name)}',`) if (dash.description) { lines.push(` description: '${escapeTs(dash.description)}',`) } lines.push(` collection: '${escapeTs(collectionName)}',`) // Cards lines.push(' cards: [') for (const dc of dash.dashcards ?? []) { if (dc.card_id && cardsById.has(dc.card_id)) { const card = cardsById.get(dc.card_id)! lines.push(` ${serializeQuestionCard(card, dc)},`) } else if (dc.card_id && dc.card) { const card = dc.card lines.push(` ${serializeQuestionCard(card, dc)},`) } else if ( dc.visualization_settings && (dc.visualization_settings as any).text && (dc.visualization_settings as any).virtual_card?.display === 'text' ) { const text = escapeTs((dc.visualization_settings as any).text) const extras = serializeDashcardExtras(dc) lines.push(` { text: \`${text}\`, width: ${dc.size_x}, height: ${dc.size_y}, row: ${dc.row}, col: ${dc.col}${extras} },`) } else if (dc.visualization_settings && (dc.visualization_settings as any).virtual_card) { // Heading, link, action, or other virtual card const vizSettings = simplifyVizSettings(dc.visualization_settings) const extras = serializeDashcardExtras(dc) lines.push(` { virtualCard: ${indentJson((dc.visualization_settings as any).virtual_card)}, visualizationSettings: ${indentJson(vizSettings)}, width: ${dc.size_x}, height: ${dc.size_y}, row: ${dc.row}, col: ${dc.col}${extras} },`) } } lines.push(' ],') // Parameters if (dash.parameters && dash.parameters.length > 0) { lines.push(` parameters: ${indentJson(dash.parameters)},`) } lines.push('})', '') return lines.join('\n') } function serializeQuestionCard(card: MetabaseCard, dc: MetabaseDashcard): string { const vizSettings = simplifyVizSettings(dc.visualization_settings) const hasViz = Object.keys(vizSettings).length > 0 const extras = serializeDashcardExtras(dc) const vizPart = hasViz ? `, visualizationSettings: ${indentJson(vizSettings)}` : '' return `{ question: '${escapeTs(card.name)}', questionId: ${card.id}, width: ${dc.size_x}, height: ${dc.size_y}, row: ${dc.row}, col: ${dc.col}${vizPart}${extras} }` } function serializeDashcardExtras(dc: MetabaseDashcard): string { let extras = '' if (dc.parameter_mappings && dc.parameter_mappings.length > 0) { extras += `, parameterMappings: ${indentJson(dc.parameter_mappings)}` } return extras } function cleanValue(value: unknown): unknown { if (Array.isArray(value)) { return value.map((item) => cleanValue(item)) } if (typeof value === 'object' && value !== null) { return cleanDatasetQuery(value as Record) } return value } function cleanDatasetQuery(dq: Record): Record { // Remove internal lib/* keys that are Metabase internals, keep the query semantics const cleaned: Record = {} for (const [key, value] of Object.entries(dq)) { if (key.startsWith('lib/') || key.startsWith('lib.')) continue cleaned[key] = cleanValue(value) } return cleaned } function indentJson(obj: unknown): string { return JSON.stringify(obj, null, 4).replace(/\n/g, '\n ') } function simplifyVizSettings(settings: Record): Record { const cleaned: Record = {} for (const [key, value] of Object.entries(settings)) { if (value !== null && value !== undefined && value !== '' && JSON.stringify(value) !== '{}') { cleaned[key] = value } } return cleaned }