import { yahooSecurityName } from "../../../sources/yahoo-finance/names"; import { resolveCurrencyUnit } from "../../../utils/currency-units"; import { createThrottledFetch, type ThrottledFetchTransport } from "../../../utils/throttled-fetch"; import type { PluginPersistence } from "../../../types/plugin"; import { apiClient } from "../../../api-client"; import type { CloudMarketResponse, CloudMarketScreenerCategory, CloudMarketScreenerItem, CloudMarketScreenerPayload, } from "../../../api-client/types"; import { hasProAccess } from "../shared/plan-access"; const YAHOO_FINANCE_HOSTS = [ "query2.finance.yahoo.com", "query1.finance.yahoo.com", ] as const; const CACHE_KIND = "yahoo-screener"; const CACHE_SOURCE = "yahoo-finance"; const CACHE_SCHEMA_VERSION = 2; const CACHE_POLICY = { staleMs: 5 * 60 * 1000, expireMs: 60 * 60 * 1000, } as const; const YAHOO_METADATA_WAIT_MS = 1_500; const YAHOO_FINANCE_HEADERS = { "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36", Accept: "application/json,text/plain,*/*", "Accept-Language": "en-US,en;q=0.9", Referer: "https://finance.yahoo.com/", }; export interface YahooScreenerApi { fetchJson(path: string, params: Record): Promise; } export interface FetchCacheOptions { cache?: boolean; forceRefresh?: boolean; } let marketMoversPersistence: PluginPersistence | null = null; const activeFetches = new Map>(); const failedFetches = new Set(); export function attachMarketMoversPersistence(persistence: PluginPersistence): void { marketMoversPersistence = persistence; } export function resetMarketMoversPersistence(): void { marketMoversPersistence = null; activeFetches.clear(); failedFetches.clear(); } export function createYahooScreenerApi(transport?: ThrottledFetchTransport): YahooScreenerApi { const client = createThrottledFetch({ requestsPerMinute: 15, maxRetries: 2, timeoutMs: 10_000, defaultHeaders: YAHOO_FINANCE_HEADERS, transport, }); return { async fetchJson(path: string, params: Record): Promise { let lastError: unknown; for (const host of YAHOO_FINANCE_HOSTS) { const url = new URL(`https://${host}${path}`); for (const [key, value] of Object.entries(params)) { url.searchParams.set(key, String(value)); } try { return await client.fetchJson(url.toString()); } catch (error) { lastError = error; } } throw lastError instanceof Error ? lastError : new Error("Yahoo Finance screener request failed"); }, }; } const screenerApi = createYahooScreenerApi(); function shouldUseCache(api: YahooScreenerApi, options?: FetchCacheOptions): boolean { return options?.cache === true || (options?.cache !== false && api === screenerApi); } function readCache(key: string, options?: { allowExpired?: boolean }): { data: T; stale: boolean } | null { const record = marketMoversPersistence?.getResource(CACHE_KIND, key, { sourceKey: CACHE_SOURCE, schemaVersion: CACHE_SCHEMA_VERSION, allowExpired: options?.allowExpired, }); if (!record) return null; return { data: record.value, stale: !!record.stale || failedFetches.has(key), }; } function writeCache(key: string, data: T): void { marketMoversPersistence?.setResource(CACHE_KIND, key, data, { sourceKey: CACHE_SOURCE, schemaVersion: CACHE_SCHEMA_VERSION, cachePolicy: CACHE_POLICY, }); } /** A cached value plus whether it is a fallback rather than a fresh load. */ export interface CachedResult { data: T; stale: boolean; } async function loadCached( key: string, fetcher: () => Promise, options?: FetchCacheOptions, ): Promise> { if (options?.cache === false) return { data: await fetcher(), stale: false }; const cached = readCache(key); if (!options?.forceRefresh && cached && !cached.stale) return { data: cached.data, stale: false }; const activeFetch = activeFetches.get(key) as Promise> | undefined; if (activeFetch) return activeFetch; const fallback = cached ?? readCache(key, { allowExpired: true }); const fetchPromise = fetcher() .then((data) => { writeCache(key, data); failedFetches.delete(key); return { data, stale: false }; }) .catch((error) => { // Serving the expired copy is right; hiding that it is expired is not. if (fallback) { failedFetches.add(key); return { data: fallback.data, stale: true }; } throw error; }) .finally(() => { if (activeFetches.get(key) === fetchPromise) { activeFetches.delete(key); } }); activeFetches.set(key, fetchPromise); return fetchPromise; } export type ScreenerCategory = "day_gainers" | "day_losers" | "most_actives"; export interface ScreenerQuote { symbol: string; name: string; price: number | null; change: number | null; changePercent: number | null; volume: number | null; avgVolume: number | null; volumeRatio: number | null; // volume / avgVolume marketCap: number | undefined; currency: string; fiftyTwoWeekHigh: number | undefined; fiftyTwoWeekLow: number | undefined; dayHigh: number | undefined; dayLow: number | undefined; exchange: string; lastUpdated?: number; /** Session-fixed, so the price column reads its decimals from it rather than from each tick. */ previousClose?: number; } export type MarketMoversDataSource = "cloud" | "yahoo"; export interface MarketMoversResult { quotes: ScreenerQuote[]; source: MarketMoversDataSource; stale: boolean; } export interface PreferredMarketMoverSources { isCloudEligible(): boolean; fetchCloud( category: CloudMarketScreenerCategory, count: number, mode: "cache-first" | "refresh", ): Promise>; fetchYahoo( category: ScreenerCategory, count: number, options?: FetchCacheOptions, ): Promise>; } export interface TrendingSymbol { symbol: string; } export interface MarketSummaryQuote { symbol: string; name: string; price: number; change: number; changePercent: number; } /** Unknown source fields stay unavailable; zero is a reported observation. */ export function screenerNumber(value: unknown): number | null { return typeof value === "number" && Number.isFinite(value) ? value : null; } export function screenerVolume(value: unknown): number | null { const number = screenerNumber(value); return number != null && number >= 0 ? number : null; } export function screenerVolumeRatio(volume: number | null, average: number | null): number | null { return volume != null && average != null && average > 0 ? volume / average : null; } /** Cross-source endpoints require explicit compatible price units. */ export function convertScreenerPriceUnit(value: number | undefined, from: string, to: string): number | undefined { const source = resolveCurrencyUnit(from); const target = resolveCurrencyUnit(to); return value != null && Number.isFinite(value) && source.currency && source.currency === target.currency ? value / source.divisor * target.divisor : undefined; } const textField = (value: unknown) => typeof value === "string" ? value : undefined; export function parseScreenerResponse(data: any): ScreenerQuote[] { const quotes = data?.finance?.result?.[0]?.quotes; if (data?.finance?.error != null || !Array.isArray(quotes)) { throw new Error("Invalid market movers response"); } return quotes.flatMap((q): ScreenerQuote[] => { if (!q || typeof q.symbol !== "string" || !q.symbol.trim()) return []; const volume = screenerVolume(q.regularMarketVolume); const avgVolume = screenerVolume(q.averageDailyVolume3Month) ?? screenerVolume(q.averageDailyVolume10Day); return [{ symbol: q.symbol.trim(), name: yahooSecurityName(textField(q.shortName), textField(q.longName)) ?? q.symbol.trim(), price: screenerNumber(q.regularMarketPrice), change: screenerNumber(q.regularMarketChange), changePercent: screenerNumber(q.regularMarketChangePercent), volume, avgVolume, volumeRatio: screenerVolumeRatio(volume, avgVolume), marketCap: screenerVolume(q.marketCap) ?? undefined, currency: typeof q.currency === "string" ? q.currency.trim() : "", fiftyTwoWeekHigh: screenerNumber(q.fiftyTwoWeekHigh) ?? undefined, fiftyTwoWeekLow: screenerNumber(q.fiftyTwoWeekLow) ?? undefined, dayHigh: screenerNumber(q.regularMarketDayHigh) ?? undefined, dayLow: screenerNumber(q.regularMarketDayLow) ?? undefined, previousClose: screenerNumber(q.regularMarketPreviousClose) ?? undefined, exchange: typeof (q.fullExchangeName ?? q.exchange) === "string" ? q.fullExchangeName ?? q.exchange : "", lastUpdated: typeof q.regularMarketTime === "number" && Number.isFinite(q.regularMarketTime) && q.regularMarketTime > 0 ? q.regularMarketTime * 1000 : undefined, }]; }); } export async function fetchScreenerResult( category: ScreenerCategory, count = 25, api: YahooScreenerApi = screenerApi, options?: FetchCacheOptions, ): Promise> { const load = async () => { const data = await api.fetchJson("/v1/finance/screener/predefined/saved", { formatted: "false", lang: "en-US", region: "US", scrIds: category, count, }); return parseScreenerResponse(data); }; if (!shouldUseCache(api, options)) return { data: await load(), stale: false }; return loadCached(`screener:${category}:count=${count}`, load, options); } export async function fetchScreener( category: ScreenerCategory, count = 25, api: YahooScreenerApi = screenerApi, options?: FetchCacheOptions, ): Promise { return (await fetchScreenerResult(category, count, api, options)).data; } function cloudScreenerCategory(category: ScreenerCategory): CloudMarketScreenerCategory { if (category === "day_gainers") return "gainers"; if (category === "day_losers") return "losers"; return "most-active"; } function isCloudScreenerEligible(): boolean { const user = apiClient.getCurrentUser(); return apiClient.isVerified() && hasProAccess(user); } const defaultPreferredMarketMoverSources: PreferredMarketMoverSources = { isCloudEligible: isCloudScreenerEligible, fetchCloud: (category, count, mode) => apiClient.getCloudMarketScreener(category, count, mode), fetchYahoo: (category, count, options) => fetchScreenerResult(category, count, undefined, options), }; function mergeCloudScreenerItem( item: CloudMarketScreenerItem, metadata?: ScreenerQuote, ): ScreenerQuote { const currency = typeof item.currency === "string" ? item.currency.trim() : ""; const metadataPrice = (value: number | undefined) => convertScreenerPriceUnit(value, metadata?.currency ?? "", currency); return { symbol: item.symbol, name: item.name && item.name !== item.symbol ? item.name : metadata?.name ?? item.symbol, price: screenerNumber(item.price), change: screenerNumber(item.change), changePercent: screenerNumber(item.changePercent), volume: screenerVolume(item.volume), avgVolume: metadata?.avgVolume ?? null, volumeRatio: screenerVolumeRatio(screenerVolume(item.volume), metadata?.avgVolume ?? null), marketCap: metadata?.marketCap, currency, fiftyTwoWeekHigh: screenerNumber(item.high52w) ?? metadataPrice(metadata?.fiftyTwoWeekHigh), fiftyTwoWeekLow: screenerNumber(item.low52w) ?? metadataPrice(metadata?.fiftyTwoWeekLow), dayHigh: screenerNumber(item.dayHigh) ?? metadataPrice(metadata?.dayHigh), dayLow: screenerNumber(item.dayLow) ?? metadataPrice(metadata?.dayLow), exchange: item.exchange || metadata?.exchange || "", lastUpdated: item.lastUpdated, }; } async function bestEffortYahooMetadata( request: Promise>, ): Promise { let timeout: ReturnType | undefined; try { return await Promise.race([ request.then((result) => result.data).catch(() => []), new Promise((resolve) => { timeout = setTimeout(() => resolve([]), YAHOO_METADATA_WAIT_MS); }), ]); } finally { if (timeout) clearTimeout(timeout); } } /** * Screeners rank on a snapshot older than the quote fields they return, so a * list is re-ranked on the metric it displays. Unknown values sort last. */ export function rankScreenerQuotes(category: ScreenerCategory, quotes: ScreenerQuote[]): ScreenerQuote[] { const metric = (quote: ScreenerQuote): number | null => { const value = category === "most_actives" ? quote.volume : quote.changePercent; if (value == null || !Number.isFinite(value)) return null; return category === "day_losers" ? -value : value; }; return [...quotes].sort((left, right) => { const a = metric(left); const b = metric(right); if (a == null || b == null) return a == null ? (b == null ? 0 : 1) : -1; return b - a; }); } export async function fetchPreferredMarketMovers( category: ScreenerCategory, count = 25, options?: FetchCacheOptions, sources: PreferredMarketMoverSources = defaultPreferredMarketMoverSources, ): Promise { if (!sources.isCloudEligible()) { const result = await sources.fetchYahoo(category, count, options); return { quotes: rankScreenerQuotes(category, result.data), source: "yahoo", stale: result.stale }; } const yahooMetadata = sources.fetchYahoo( category, Math.max(count, 50), options, ); try { const response = await sources.fetchCloud( cloudScreenerCategory(category), count, options?.forceRefresh ? "refresh" : "cache-first", ); if ( (response.status === "success" || response.status === "partial") && response.data && response.data.items.length > 0 ) { const metadata = await bestEffortYahooMetadata(yahooMetadata); const metadataBySymbol = new Map(metadata.map((quote) => [quote.symbol, quote])); return { quotes: rankScreenerQuotes(category, response.data.items.map((item) => ( mergeCloudScreenerItem(item, metadataBySymbol.get(item.symbol)) ))), source: "cloud", stale: response.stale === true || response.data.stale === true, }; } } catch { // Yahoo remains the resilient fallback when Cloud is unavailable. } const fallback = await yahooMetadata; return { quotes: rankScreenerQuotes(category, fallback.data), source: "yahoo", stale: fallback.stale, }; } export function parseTrendingResponse(data: any): TrendingSymbol[] { try { const quotes = data?.finance?.result?.[0]?.quotes; if (!Array.isArray(quotes)) return []; const result: TrendingSymbol[] = []; for (const q of quotes) { if (!q || typeof q.symbol !== "string") continue; result.push({ symbol: q.symbol }); } return result; } catch { return []; } } export async function fetchTrending( count = 25, api: YahooScreenerApi = screenerApi, options?: FetchCacheOptions, ): Promise { const load = async () => { const data = await api.fetchJson("/v1/finance/trending/US", { count, }); return parseTrendingResponse(data); }; if (!shouldUseCache(api, options)) return load(); return (await loadCached(`trending:US:count=${count}`, load, options)).data; } export const MARKET_SUMMARY_SYMBOLS = ["^GSPC", "^DJI", "^IXIC", "^RUT"] as const;