import { RedisClient, RedisService } from "./redis.service"; import ConsumerSetQueue from "../utils/consumerSetQueue"; import { ICategoryProvider, ICategoryValue, CategoryProviderType, BrandSafetySeverity, EmptyCategoryValue, CategoryRating, } from "../categories/categoryProvider"; import CategoryProviderMap from "../categories/categoryProviderMap"; import CategoryRedisWriterConsumer from "../categories/categoryRedisWriterConsumer"; import { createHash } from "crypto"; import logger from "../utils/logger"; import MetricService from "./metric.service"; import { ChoiceRequest } from "../models/request.model"; import { sanitizeURL } from "../utils/url-sanitization"; import { getRequestSource, RequestSource } from "../utils/RequestUtils"; export type CategoryRequest = { device: boolean; urlOrBundle?: string; ip?: string; userAgent?: string; provider: CategoryProviderType; requestSource: RequestSource; }; export default class CategoryService { refreshTTLrandomness: number = 30 * 60 * 60 * 1000; private static instance: CategoryService; private NB_CONSUMERS: number = 2; private metricService: MetricService; private CATEGORY_READ_SECTION: string = "cat_read"; private redisClient: RedisClient; private consumerSetQueue: ConsumerSetQueue; public static getInstance(): CategoryService { return this.instance || (this.instance = new CategoryService()); } constructor() { this.redisClient = RedisService.getClient("cat", true); this.consumerSetQueue = new ConsumerSetQueue(); this.createConsumers(); this.metricService = MetricService.getInstance(); this.metricService.addSection(this.CATEGORY_READ_SECTION, [ "provider", "device", "bs_low_result", "bs_medium_result", "bs_high_result", "device_result", "available_data", "stale_data", ]); } close(): void { this.consumerSetQueue.close(); this.redisClient.disconnect(); } private createConsumers() { for (let i = 0; i < this.NB_CONSUMERS; i++) { this.consumerSetQueue.addConsumer( new CategoryRedisWriterConsumer(RedisService.getClient("cat", false), this) ); } } async isKeyStale(key: string, redisClient: RedisClient): Promise { const value = await redisClient.get(key); if (value == null) { return true; } const providerType = this.getProviderTypeFromKey(key); if (!providerType) { throw new Error("Unknown provider type"); } const provider: ICategoryProvider = CategoryProviderMap[providerType]; const decoded: ICategoryValue = provider.decode(value); const ttl: number = decoded.getRefreshTTL(); return Date.now() > ttl; } private isStale(ttl: number): boolean { ttl += Math.floor(Math.random() * this.refreshTTLrandomness); return Date.now() > ttl; } async getPublisherCategoryValue( request: ChoiceRequest, providerType: CategoryProviderType ): Promise { if (request.categories && request.categories.get(providerType)) { return request.categories.get(providerType) as ICategoryValue; } const requestSource = getRequestSource(request); let urlOrBundle: string; if (requestSource.environmentType === "web") { urlOrBundle = request.site?.page ?? ""; } else if (requestSource.deviceOS === "android") { urlOrBundle = request.app?.bundle_id ?? ""; } else { urlOrBundle = request.app?.app_id ?? ""; } return this.getCategoryValueFromRequest( { device: false, urlOrBundle: urlOrBundle, requestSource: requestSource, provider: providerType, }, request ); } async getDeviceCategoryValue( request: ChoiceRequest, providerType: CategoryProviderType ): Promise { const requestSource = getRequestSource(request); return this.getCategoryValueFromRequest( { device: true, ip: request.device?.ip ?? undefined, userAgent: request.device?.ua ?? undefined, provider: providerType, requestSource: requestSource, }, request ); } private async getCategoryValueFromRequest( req: CategoryRequest, request: ChoiceRequest ): Promise { const end = this.metricService.startSection(this.CATEGORY_READ_SECTION); logger.debug( () => `[CATEGORY_SERVICE] checking for ${req.urlOrBundle} source ${req.requestSource}` ); let returnValue: ICategoryValue; let availableData: boolean = false; let staleData: boolean = false; const provider = CategoryProviderMap[req.provider]; if (!provider.isRequestValid(req)) { returnValue = new EmptyCategoryValue(CategoryRating.BadRequest); } else { const value = await this.redisClient.get(this.encodeRedisKey(req, provider)); if (value == null) { this.consumerSetQueue.push(req); staleData = true; returnValue = new EmptyCategoryValue(); } else { availableData = true; returnValue = provider.decode(value); if (req.device) { if (request.deviceCategories === undefined) { request.deviceCategories = new Map(); } request.deviceCategories.set(req.provider, returnValue); } else { if (request.categories === undefined) { request.categories = new Map(); } request.categories.set(req.provider, returnValue); } if (this.isStale(returnValue.getRefreshTTL())) { this.consumerSetQueue.push(req); staleData = true; } } } logger.debug( () => `[CATEGORY_SERVICE] ${returnValue} value in cache for ${req.urlOrBundle} source ${req.requestSource}` ); end(true, { provider: req.provider, device: String(req.device), bs_low_result: returnValue.getBrandSafety(BrandSafetySeverity.Low), bs_medium_result: returnValue.getBrandSafety(BrandSafetySeverity.Medium), bs_high_result: returnValue.getBrandSafety(BrandSafetySeverity.High), device_result: returnValue.getDeviceFraud(), available_data: String(availableData), stale_data: String(staleData), }); return returnValue; } async getBrandSafetyDebugPurpose( urlOrBundle: string, providerType: CategoryProviderType = CategoryProviderType.DV, fromCache: boolean = false, requestSource: RequestSource = { environmentType: "web", deviceOS: "android" } ): Promise { const req: CategoryRequest = { device: false, urlOrBundle: urlOrBundle, provider: providerType, requestSource: requestSource, }; if (fromCache) { const value = await this.redisClient.get( this.encodeRedisKey(req, CategoryProviderMap[providerType]) ); if (value == null) { return new EmptyCategoryValue(); } return CategoryProviderMap[providerType].decode(value); } else { const provider: ICategoryProvider = CategoryProviderMap[providerType]; return provider.decode((await provider.getCategories(req)).encodedValue); } } private getProviderTypeFromKey(key: string): CategoryProviderType | undefined { if (key.startsWith("cat:")) { return key.split(":")[1] as CategoryProviderType; } return undefined; } encodeRedisKey(req: CategoryRequest, provider: ICategoryProvider): string { let payload: string; if (req.urlOrBundle) { if (req.requestSource.environmentType === "web") payload = sanitizeURL(req.urlOrBundle); else payload = req.urlOrBundle; } else { //DeviceFraud payload = req.ip!; if (provider.needUserAgent()) payload += ":" + req.userAgent; } let result = "cat:" + req.provider + ":" + this.sha256(payload); if (req.requestSource.environmentType !== "web") { result = result + ":" + req.requestSource.deviceOS; } return result; } private sha256(input: string) { const hash = createHash("sha256"); hash.update(input); return hash.digest("hex"); } }