/** * brand-dna.ts — the brand-extract stage. * * Turns a website URL into a `brand-dna.json` artifact: * 1. scrape the page (injectable plain `fetch` + regex extraction — no * headless browser, no new deps), * 2. compute the colour palette DETERMINISTICALLY from the scraped colours, * 3. run ONE strict-JSON Gemini text pass (via the shared key-pool) for the * brand-voice / audience / messaging fields, * 4. merge the deterministic palette over the model guess. * * This is the ONLY LLM step in the brief->storyboard->story-bible chain; it is * isolated in its own stage with its own artifact so downstream stages stay * deterministic. The brief consumes this artifact only behind an explicit * `--from-brand-dna` opt-in (see brandDnaBriefSeed + handleVideoBrief). * * Concepts (not code) adapted from the MIT-licensed Open-Pomelli project. * * Content-filter rule: brand-dna records logo URL / colours / text only. Scraped * photoreal faces (founder headshots, og:image people) are NEVER registered or * passed downstream as reference images — they trip the ARK/Seedance real-person * filter and don't lock identity anyway. */ import { mkdir, readFile, writeFile } from 'node:fs/promises'; import { join } from 'node:path'; import { fetchGeminiWithPool } from './gemini-key-pool.js'; import type { VideoProjectWorkspace } from './workspace.js'; const DEFAULT_GEMINI_ENDPOINT = 'https://generativelanguage.googleapis.com/v1beta/models/gemini-3.5-flash:generateContent'; /** Max characters of body text sent to the model (mirrors Open-Pomelli's 8k cap). */ const BODY_TEXT_LIMIT = 8000; /** Hard ceiling on a scrape fetch so an unattended/overnight run can't hang. */ const DEFAULT_SCRAPE_TIMEOUT_MS = 20_000; export const IMAGERY_STYLES = [ 'professional', 'casual', 'illustrated', 'cinematic', 'minimalist', 'editorial', ] as const; export type ImageryStyle = (typeof IMAGERY_STYLES)[number]; export const LAYOUT_STYLES = ['modern', 'classic', 'minimalist', 'bold', 'editorial'] as const; export type LayoutStyle = (typeof LAYOUT_STYLES)[number]; export interface ScrapedSite { url: string; title: string; description: string; bodyText: string; ogImage: string | null; favicon: string | null; logoCandidates: string[]; rawColors: string[]; fonts: string[]; } export interface BrandDnaContent { brandName: string; industry: string; tagline: string; valueProposition: string; toneOfVoice: string[]; brandPersonality: string[]; targetAudience: string; keyMessages: string[]; primaryColors: string[]; secondaryColors: string[]; fonts: string[]; logoUrl: string | null; imageryStyle: ImageryStyle; layoutStyle: LayoutStyle; sourceUrl: string; } export interface BrandDnaArtifact extends BrandDnaContent { schemaVersion: 1; kind: 'brand-dna'; createdAt: string; projectSlug: string; } /** Injectable scraper so tests pass a fake (mirrors the FetchLike DI in native-*.ts). */ export type ScrapeRunner = (url: string, fetcher?: typeof fetch) => Promise; // -------------------------------------------------------------------------- // Scraping (plain fetch + regex — deterministic, unit-testable) // -------------------------------------------------------------------------- function absoluteUrl(href: string, base: string): string | null { try { return new URL(href, base).href; } catch { return null; } } function decodeEntities(text: string): string { return text .replace(/&/g, '&') .replace(/</g, '<') .replace(/>/g, '>') .replace(/"/g, '"') .replace(/�?39;|'/g, "'") .replace(/ /g, ' '); } function firstMatch(html: string, re: RegExp): string | null { const m = re.exec(html); return m && m[1] !== undefined ? m[1].trim() : null; } function metaContent(html: string, attr: 'name' | 'property', key: string): string | null { const a = firstMatch( html, new RegExp(`]+${attr}=["']${key}["'][^>]*content=["']([^"']*)["']`, 'i'), ); if (a) return decodeEntities(a); const b = firstMatch( html, new RegExp(`]+content=["']([^"']*)["'][^>]*${attr}=["']${key}["']`, 'i'), ); return b ? decodeEntities(b) : null; } function extractBodyText(html: string): string { const bodyMatch = /]*>([\s\S]*?)<\/body>/i.exec(html); const body = bodyMatch ? bodyMatch[1] : html; const stripped = body .replace(//gi, ' ') .replace(//gi, ' ') .replace(//gi, ' ') .replace(/<[^>]+>/g, ' '); return decodeEntities(stripped).replace(/\s+/g, ' ').trim().slice(0, BODY_TEXT_LIMIT); } function extractRawColors(html: string): string[] { const colors: string[] = []; const seen = new Set(); const push = (value: string) => { const key = value.toLowerCase(); if (!seen.has(key)) { seen.add(key); colors.push(value); } }; const theme = metaContent(html, 'name', 'theme-color'); if (theme) push(theme.trim()); for (const m of html.matchAll(/#[0-9a-fA-F]{6}\b|#[0-9a-fA-F]{3}\b/g)) push(m[0]); for (const m of html.matchAll(/rgba?\(\s*\d{1,3}\s*,\s*\d{1,3}\s*,\s*\d{1,3}\s*(?:,\s*[\d.]+\s*)?\)/gi)) { push(m[0]); } return colors.slice(0, 60); } function extractFonts(html: string): string[] { const fonts: string[] = []; const seen = new Set(); for (const m of html.matchAll(/font-family\s*:\s*([^;}]+)/gi)) { for (const raw of m[1].split(',')) { const font = raw.trim().replace(/^['"]|['"]$/g, '').replace(/['"].*$/, '').trim(); const key = font.toLowerCase(); if (font && !key.startsWith('var(') && !seen.has(key)) { seen.add(key); fonts.push(font); } } } return fonts.slice(0, 20); } function extractLogoCandidates(html: string, base: string): string[] { const out: string[] = []; const seen = new Set(); for (const m of html.matchAll(/]*>/gi)) { const tag = m[0]; if (!/logo/i.test(tag)) continue; const src = firstMatch(tag, /\bsrc=["']([^"']+)["']/i); if (!src) continue; const abs = absoluteUrl(src, base); if (abs && !seen.has(abs)) { seen.add(abs); out.push(abs); } if (out.length >= 5) break; } return out; } /** Default scraper: plain fetch + regex. Injectable `fetcher` for tests. */ export const scrapeSite: ScrapeRunner = async (url, fetcher = fetch) => { let parsed: URL; try { parsed = new URL(url); } catch { throw new Error(`brand-extract: invalid URL ${JSON.stringify(url)}`); } if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') { throw new Error(`brand-extract: URL must be http(s) (got ${parsed.protocol})`); } const controller = new AbortController(); const timer = setTimeout(() => controller.abort(), DEFAULT_SCRAPE_TIMEOUT_MS); let html: string; try { const response = await fetcher(url, { signal: controller.signal, headers: { 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36', Accept: 'text/html,application/xhtml+xml', }, }); if (!response.ok) { throw new Error(`brand-extract: fetch ${url} failed with HTTP ${response.status}`); } html = await response.text(); } catch (error) { if ((error as Error)?.name === 'AbortError') { throw new Error(`brand-extract: fetch ${url} timed out after ${DEFAULT_SCRAPE_TIMEOUT_MS}ms`); } throw error instanceof Error ? error : new Error(String(error)); } finally { clearTimeout(timer); } const titleRaw = firstMatch(html, /]*>([\s\S]*?)<\/title>/i); const faviconHref = firstMatch(html, /]+rel=["'][^"']*icon[^"']*["'][^>]*href=["']([^"']+)["']/i); const ogImage = metaContent(html, 'property', 'og:image'); return { url, title: titleRaw ? decodeEntities(titleRaw).replace(/\s+/g, ' ').trim() : '', description: metaContent(html, 'name', 'description') ?? metaContent(html, 'property', 'og:description') ?? '', bodyText: extractBodyText(html), ogImage: ogImage ? absoluteUrl(ogImage, url) : null, favicon: faviconHref ? absoluteUrl(faviconHref, url) : null, logoCandidates: extractLogoCandidates(html, url), rawColors: extractRawColors(html), fonts: extractFonts(html), }; }; // -------------------------------------------------------------------------- // Deterministic palette // -------------------------------------------------------------------------- function normalizeHex(value: string): string | null { const v = value.trim().toLowerCase(); const short = /^#([0-9a-f])([0-9a-f])([0-9a-f])$/.exec(v); if (short) return `#${short[1]}${short[1]}${short[2]}${short[2]}${short[3]}${short[3]}`; if (/^#[0-9a-f]{6}$/.test(v)) return v; const rgb = /^rgba?\(\s*(\d{1,3})\s*,\s*(\d{1,3})\s*,\s*(\d{1,3})/i.exec(v); if (rgb) { const [r, g, b] = [rgb[1], rgb[2], rgb[3]].map((n) => Math.min(255, Number(n))); return `#${[r, g, b].map((n) => n.toString(16).padStart(2, '0')).join('')}`; } return null; } function isNeutral(hex: string): boolean { const r = parseInt(hex.slice(1, 3), 16); const g = parseInt(hex.slice(3, 5), 16); const b = parseInt(hex.slice(5, 7), 16); const max = Math.max(r, g, b); const min = Math.min(r, g, b); if (max >= 245 && min >= 245) return true; if (max <= 16) return true; return max - min < 12; } /** rgb/hex -> hex, drop white/black/greys, frequency-rank: top-3 primary, next-5 secondary. */ export function pickPalette(rawColors: string[]): { primary: string[]; secondary: string[] } { const counts = new Map(); for (const raw of rawColors) { const hex = normalizeHex(raw); if (!hex || isNeutral(hex)) continue; counts.set(hex, (counts.get(hex) ?? 0) + 1); } const ranked = [...counts.entries()] .sort((a, b) => b[1] - a[1] || (a[0] < b[0] ? -1 : 1)) .map(([hex]) => hex); return { primary: ranked.slice(0, 3), secondary: ranked.slice(3, 8) }; } // -------------------------------------------------------------------------- // Gemini text pass (mirrors gemini-analyze.ts) // -------------------------------------------------------------------------- export function buildBrandDnaPrompt(scraped: ScrapedSite): string { return `You are a brand analyst. From the website content below, extract the brand DNA. Return ONLY valid JSON with this exact shape (no markdown, no commentary): { "brandName": "string", "industry": "string", "tagline": "string", "valueProposition": "string", "toneOfVoice": ["3-5 trait words"], "brandPersonality": ["3-5 trait words"], "targetAudience": "one sentence", "keyMessages": ["3-5 short messages"], "imageryStyle": "${IMAGERY_STYLES.join(' | ')}", "layoutStyle": "${LAYOUT_STYLES.join(' | ')}" } Rules: - Base every field on the supplied content; do not invent products or claims. - Keep trait words and key messages short and reusable. - Pick imageryStyle and layoutStyle from the allowed values only. WEBSITE TITLE: ${scraped.title || 'Untitled'} META DESCRIPTION: ${scraped.description || 'n/a'} BODY TEXT (truncated): ${scraped.bodyText || 'n/a'}`; } function parseGeminiText(payload: unknown): string { const candidates = (payload as { candidates?: Array<{ content?: { parts?: Array<{ text?: string }> } }> }) .candidates; const text = candidates?.[0]?.content?.parts ?.map((part) => part.text ?? '') .join('\n') .trim(); if (!text) { throw new Error('brand-extract: Gemini response did not contain text output.'); } return text; } function coerceEnum(value: unknown, allowed: readonly T[], fallback: T): T { return typeof value === 'string' && (allowed as readonly string[]).includes(value) ? (value as T) : fallback; } function coerceStringArray(value: unknown): string[] { return Array.isArray(value) ? value.map((v) => (typeof v === 'string' ? v.trim() : '')).filter(Boolean) : []; } function parseBrandDnaJson( text: string, ): Omit { const cleaned = text .replace(/^```json\s*/i, '') .replace(/^```\s*/i, '') .replace(/\s*```$/i, '') .trim(); const parsed = JSON.parse(cleaned) as Record; return { brandName: typeof parsed.brandName === 'string' ? parsed.brandName.trim() : '', industry: typeof parsed.industry === 'string' ? parsed.industry.trim() : '', tagline: typeof parsed.tagline === 'string' ? parsed.tagline.trim() : '', valueProposition: typeof parsed.valueProposition === 'string' ? parsed.valueProposition.trim() : '', toneOfVoice: coerceStringArray(parsed.toneOfVoice), brandPersonality: coerceStringArray(parsed.brandPersonality), targetAudience: typeof parsed.targetAudience === 'string' ? parsed.targetAudience.trim() : '', keyMessages: coerceStringArray(parsed.keyMessages), imageryStyle: coerceEnum(parsed.imageryStyle, IMAGERY_STYLES, 'professional'), layoutStyle: coerceEnum(parsed.layoutStyle, LAYOUT_STYLES, 'modern'), }; } export interface RunBrandExtractionOptions { endpoint?: string; fetcher?: typeof fetch; } /** The strict-JSON Gemini text pass. Merges the deterministic palette over the model. */ export async function runBrandExtraction( scraped: ScrapedSite, options: RunBrandExtractionOptions = {}, ): Promise { const endpoint = options.endpoint ?? process.env.VCLAW_GEMINI_API_ENDPOINT ?? DEFAULT_GEMINI_ENDPOINT; const response = await fetchGeminiWithPool( (key) => `${endpoint}${endpoint.includes('?') ? '&' : '?'}key=${encodeURIComponent(key)}`, { method: 'POST', headers: { 'Content-Type': 'application/json', Connection: 'close' }, body: JSON.stringify({ contents: [{ parts: [{ text: buildBrandDnaPrompt(scraped) }] }], generationConfig: { temperature: 0.2, maxOutputTokens: 1000, responseMimeType: 'application/json' }, }), }, { ...(options.fetcher ? { fetcher: options.fetcher } : {}), onRetry: (label, status) => { process.stderr.write(`[brand-extract/gemini] ${label} returned HTTP ${status}; rotating key\n`); }, }, ); if (!response.ok) { throw new Error(`brand-extract: Gemini request failed with HTTP ${response.status}`); } const fields = parseBrandDnaJson(parseGeminiText(await response.json())); const palette = pickPalette(scraped.rawColors); return { ...fields, primaryColors: palette.primary, secondaryColors: palette.secondary, fonts: scraped.fonts, logoUrl: scraped.logoCandidates[0] ?? scraped.ogImage ?? scraped.favicon ?? null, sourceUrl: scraped.url, }; } // -------------------------------------------------------------------------- // Artifact I/O (self-contained fs; no dependency on the typed artifact-store // name union so this stage compiles without touching VideoStageArtifactName) // -------------------------------------------------------------------------- export function createBrandDnaArtifact(input: { projectSlug: string; content: BrandDnaContent; generatedAt?: string; }): BrandDnaArtifact { return { schemaVersion: 1, kind: 'brand-dna', createdAt: input.generatedAt ?? new Date().toISOString(), projectSlug: input.projectSlug, ...input.content, }; } const BRAND_DNA_ARTIFACT_FILE = 'brand-dna.json'; export async function writeBrandDnaArtifact( workspace: VideoProjectWorkspace, artifact: BrandDnaArtifact, ): Promise { const path = join(workspace.artifactsDir, BRAND_DNA_ARTIFACT_FILE); await mkdir(workspace.artifactsDir, { recursive: true }); await writeFile(path, `${JSON.stringify(artifact, null, 2)}\n`, 'utf-8'); return path; } export async function readBrandDnaArtifact( workspace: VideoProjectWorkspace, ): Promise { const path = join(workspace.artifactsDir, BRAND_DNA_ARTIFACT_FILE); try { return JSON.parse(await readFile(path, 'utf-8')) as BrandDnaArtifact; } catch (error) { if ((error as NodeJS.ErrnoException).code === 'ENOENT') return null; throw error; } } /** * Brand-derived seed for the brief stage, consumed only behind `--from-brand-dna`. * The brief artifact is intentionally minimal — { title, intent, productionMode, * metadata } — so brand-dna fills title/intent (when the operator omits them) and * parks the richer brand fields under metadata.brandDna for downstream stages. * Explicit operator flags always win; this only supplies fallbacks. */ export interface BrandBriefSeed { title: string; intent: string; metadata: { brandDna: BrandDnaContent }; } export function brandDnaBriefSeed(dna: BrandDnaContent): BrandBriefSeed { return { title: dna.brandName, intent: dna.valueProposition, metadata: { brandDna: { ...dna } }, }; }