import chalk from 'chalk'; import type { Command } from 'commander'; import { inArray } from 'drizzle-orm'; import { getDb } from '../../db'; import { articles } from '../../db/schema'; import { fetchArticlesContent } from '../../fetchers/article'; import { fetchAllFeeds, fetchFeed, getFeed } from '../../fetchers/feed'; import { DEFAULT_ARTICLE_LIMIT, DEFAULT_CONCURRENCY, DEFAULT_TIMEOUT, MAX_ARTICLE_LIMIT, MAX_CONCURRENCY, MAX_TIMEOUT, MIN_CONCURRENCY, MIN_TIMEOUT, } from '../../utils/constants'; import { formatFetchSummary } from '../../utils/feed-formatter'; import { sanitizeTerminalText } from '../../utils/terminal'; import { validateIntRange, validatePositiveInt } from '../../utils/validation'; import { fail, requireValue } from '../error-handler'; import { runCommand } from '../run-command'; export function registerFetchCommands(program: Command): void { const fetchCmd = program.command('fetch').description('Fetch feeds and articles'); // fetch all fetchCmd .command('all') .description('Fetch all feeds') .option('-l, --limit ', 'Limit number of feeds to fetch') .option('-f, --force', 'Force fetch even if not modified') .option('--timeout ', 'Request timeout in ms', String(DEFAULT_TIMEOUT)) .option('-c, --concurrency ', 'Number of concurrent fetches', String(DEFAULT_CONCURRENCY)) .action(async (options: { limit?: string; force?: boolean; timeout: string; concurrency: string }) => { await runCommand(program, async (output) => { const timeout = requireValue(validateIntRange(options.timeout, 'Timeout', MIN_TIMEOUT, MAX_TIMEOUT)); const concurrency = requireValue( validateIntRange(options.concurrency, 'Concurrency', MIN_CONCURRENCY, MAX_CONCURRENCY), ); let limit: number | undefined; if (options.limit) { limit = requireValue(validatePositiveInt(options.limit, 'Limit')); } const progress = output.progress('Fetching feeds...'); const results = await fetchAllFeeds( { force: options.force, timeout, concurrency, limit, }, ({ current, total, feed, status }) => { progress.update( `[${current}/${total}] ${sanitizeTerminalText(feed?.title ?? 'Unknown')} — ${sanitizeTerminalText(status)}`, ); }, ); progress.succeed(`Fetched ${results.length} feeds`); output.result(results, formatFetchSummary); }); }); // fetch feed fetchCmd .command('feed ') .description('Fetch a single feed') .option('-f, --force', 'Force fetch even if not modified') .option('--timeout ', 'Request timeout in ms', String(DEFAULT_TIMEOUT)) .action(async (id: string, options: { force?: boolean; timeout: string }) => { await runCommand(program, async (output) => { const feedId = requireValue(validatePositiveInt(id, 'Feed ID')); const timeout = requireValue(validateIntRange(options.timeout, 'Timeout', MIN_TIMEOUT, MAX_TIMEOUT)); const feed = await getFeed(feedId); if (!feed) { fail('Feed not found'); } const safeTitle = sanitizeTerminalText(feed.title); const progress = output.progress(`Fetching ${safeTitle}...`); const result = await fetchFeed(feed, { force: options.force, timeout, }); if (result.status === 'success') { progress.succeed(`Fetched ${safeTitle}: ${result.newArticles} new articles`); } else if (result.status === 'not_modified') { progress.warn(`${safeTitle}: Not modified`); } else { progress.fail(`${safeTitle}: ${sanitizeTerminalText(result.error ?? 'Unknown error')}`); } output.result(result, (r) => formatFetchSummary([r])); }); }); // fetch content fetchCmd .command('content [ids...]') .description('Fetch article content (automatically skips known broken articles)') .option('-l, --limit ', 'Maximum articles to fetch', String(DEFAULT_ARTICLE_LIMIT)) .option('--fix-broken', 'Try to fix broken links') .option('--no-skip-broken', 'Re-check articles with broken links') .option('--flag-broken', 'Flag definitively missing (404/410) links in the database') .option('--allow-private-network', 'Allow Article requests to private/LAN addresses (metadata remains blocked)') .option('--timeout ', 'Request timeout in ms', String(DEFAULT_TIMEOUT)) .option('-c, --concurrency ', 'Number of concurrent fetches', String(DEFAULT_CONCURRENCY)) .action( async ( ids: string[], options: { limit: string; fixBroken?: boolean; skipBroken: boolean; flagBroken?: boolean; allowPrivateNetwork?: boolean; timeout: string; concurrency: string; }, cmd: Command, ) => { await runCommand(program, async (output) => { const limit = requireValue(validateIntRange(options.limit, 'Limit', 1, MAX_ARTICLE_LIMIT)); const timeout = requireValue(validateIntRange(options.timeout, 'Timeout', MIN_TIMEOUT, MAX_TIMEOUT)); const concurrency = requireValue( validateIntRange(options.concurrency, 'Concurrency', MIN_CONCURRENCY, MAX_CONCURRENCY), ); const articleIds: number[] = []; for (const idStr of ids) { articleIds.push(requireValue(validatePositiveInt(idStr, 'Article ID'))); } if (articleIds.length > 0) { // Keep each IN list below SQLite's conservative bind-variable // ceiling; explicit ID batches are allowed to exceed one chunk. const existing: Array<{ id: number }> = []; const db = getDb(); for (let offset = 0; offset < articleIds.length; offset += 400) { existing.push( ...(await db .select({ id: articles.id }) .from(articles) .where(inArray(articles.id, articleIds.slice(offset, offset + 400)))), ); } const existingIds = new Set(existing.map((article) => article.id)); const missingIds = [...new Set(articleIds)].filter((id) => !existingIds.has(id)); if (missingIds.length > 0) { fail(`Article not found: ${missingIds.join(', ')}`); } } // When explicit article IDs are given, the default --limit must not // silently truncate the list. Only forward a limit when the user set // one explicitly (or when no IDs were passed, so the limit caps the // open-ended scan). const limitWasExplicit = cmd.getOptionValueSource('limit') !== 'default'; const forwardLimit = articleIds.length === 0 || limitWasExplicit ? limit : undefined; const progress = output.progress('Fetching article content...'); const results = await fetchArticlesContent(articleIds.length > 0 ? articleIds : undefined, { limit: forwardLimit, fixBroken: options.fixBroken, skipBroken: options.skipBroken, flagBroken: options.flagBroken, allowPrivateNetwork: options.allowPrivateNetwork, timeout, concurrency, }); progress.succeed(`Fetched content for ${results.length} articles`); output.result(results, (rs) => { const success = rs.filter((r) => r.status === 'success').length; const errors = rs.filter((r) => r.status === 'error').length; const skipped = rs.filter((r) => r.status === 'skipped').length; const lines: string[] = []; lines.push(chalk.green(` Success: ${success}`)); if (errors > 0) lines.push(chalk.red(` Errors: ${errors}`)); if (skipped > 0) lines.push(chalk.gray(` Skipped: ${skipped}`)); return lines.join('\n'); }); }); }, ); }