import type { Database } from "bun:sqlite"; import chalk from "chalk"; import pLimit from "p-limit"; import { collectWorksFromListing, fetchWorkDetail } from "@/crawlers/fc2"; import { extractMagnets, searchInDb } from "@/crawlers/sukebei.db"; import * as authorsQ from "@/db/queries/authors"; import * as priceHistoryQ from "@/db/queries/price-history"; import * as sukebeiMagnetsQ from "@/db/queries/sukebei-magnets"; import * as syncRunsQ from "@/db/queries/sync-runs"; import * as worksQ from "@/db/queries/works"; import type { FC2Work, SyncRunStatus } from "@/interfaces"; import { nowMs } from "@/utils/dates"; export interface SyncOptions { author: string; concurrency: number; delayMs: number; withSukebei: boolean; withDetail: boolean; db: Database; } export interface SyncSummary { author: string; runId: number; totalWorks: number; newWorks: number; priceChanges: number; newMagnets: number; warnings: string[]; status: SyncRunStatus; } const wait = (ms: number): Promise => new Promise((resolve) => setTimeout(resolve, ms)); export const syncAuthor = async (opts: SyncOptions): Promise => { const { author, db, withDetail, withSukebei, concurrency, delayMs } = opts; const warnings: string[] = []; const runId = syncRunsQ.startRun(db, author, { withDetail, withSukebei }); try { const listing = await collectWorksFromListing(author); console.info(chalk.green(`FC2 listing: ${listing.length} work(s)`)); // Detail-page enrichment is opt-in via --with-detail. Implicit enrichment // on missing fields is too expensive on a fresh author (N sequential // HTTP requests with no visible progress looks like a hang); users who // want backfill pass --with-detail explicitly. const detailMap = new Map>>(); if (withDetail && listing.length > 0) { console.info(chalk.blue(`Enriching ${listing.length} work(s) from detail pages...`)); const limit = pLimit(Math.max(1, concurrency)); let done = 0; await Promise.all( listing.map((w) => limit(async () => { if (delayMs > 0) await wait(delayMs); try { const detail = await fetchWorkDetail(w.id); detailMap.set(w.id, detail); } catch (err) { const msg = err instanceof Error ? err.message : String(err); warnings.push(`detail ${w.id}: ${msg}`); } finally { done++; if (done % 10 === 0 || done === listing.length) { console.info(chalk.dim(` detail ${done}/${listing.length}`)); } } }) ) ); } let newWorks = 0; let priceChanges = 0; db.transaction(() => { for (const w of listing) { const detail = detailMap.get(w.id); const merged: FC2Work = detail ? { ...w, title: detail.title ?? w.title, publishedAt: detail.publishedAt ?? w.publishedAt, priceYen: detail.priceYen ?? w.priceYen, onPromo: detail.priceYen !== null ? detail.onPromo : w.onPromo, source: detail.priceYen !== null || detail.publishedAt !== null ? "detail" : w.source } : w; const upsertRes = worksQ.upsertWork(db, merged); if (upsertRes.inserted) newWorks++; if (detail) { db.query(`UPDATE works SET detail_fetched_at = ? WHERE id = ?`).run( merged.observedAt, merged.id ); } if (merged.priceYen !== null) { const res = priceHistoryQ.recordPriceIfChanged( db, { id: merged.id, priceYen: merged.priceYen, onPromo: merged.onPromo, observedAt: merged.observedAt }, merged.source ); if (res.changed) priceChanges++; } } authorsQ.touchAuthorLastSynced(db, author, nowMs()); })(); // Sukebei cross-check is opt-in via --with-sukebei. One request per work; // failures are per-work warnings only, not a run-level degradation. let newMagnets = 0; if (withSukebei && listing.length > 0) { console.info(chalk.blue(`Checking sukebei for ${listing.length} work(s)...`)); const limit = pLimit(Math.max(1, concurrency)); let done = 0; await Promise.all( listing.map((w) => limit(async () => { if (delayMs > 0) await wait(delayMs); try { const result = await searchInDb({ id: w.id, title: w.title, url: "", duration: w.duration ?? "00:00" }); if (result.code === 0) { warnings.push(`sukebei ${w.id}: request failed`); return; } if (result.code !== 200) { warnings.push(`sukebei ${w.id}: HTTP ${result.code}`); return; } const magnets = extractMagnets(result); const out = sukebeiMagnetsQ.upsertMagnets(db, w.id, magnets); newMagnets += out.newMagnetIds.length; } catch (err) { const msg = err instanceof Error ? err.message : String(err); warnings.push(`sukebei ${w.id}: ${msg}`); } finally { done++; if (done % 10 === 0 || done === listing.length) { console.info(chalk.dim(` sukebei ${done}/${listing.length}`)); } } }) ) ); } // Hard degradations would downgrade the whole run to "partial" (exit 4). // Per-work detail / sukebei errors stay as warnings only; exit stays 0. const hardDegradations: string[] = []; const status: SyncRunStatus = hardDegradations.length > 0 ? "partial" : "ok"; syncRunsQ.finishRun(db, runId, { status, counts: { newWorks, priceChanges, newMagnets } }); return { author, runId, totalWorks: listing.length, newWorks, priceChanges, newMagnets, warnings, status }; } catch (err) { const msg = err instanceof Error ? err.message : String(err); syncRunsQ.finishRun(db, runId, { status: "error", errorMessage: msg }); throw err; } };