import axios, { AxiosInstance, AxiosResponse } from "axios"; import { createHash } from "crypto"; import http from "http"; import https from "https"; import { sanitizeURL } from "../utils/url-sanitization"; import { CategoryRequest } from "../services/category.service"; import MetricService, { BucketSet } from "../services/metric.service"; import { config } from "../utils/config"; import { CategoryProviderValue, ICategoryProvider, ICategoryValue, CategoryRating, BrandSafetySeverity, FraudSeverity, } from "./categoryProvider"; export default class DVCategoryProvider implements ICategoryProvider { private dvBaseUrl: string; private dvPath: string; private salt: string; private partnerId: string; private defaultExpireTTL: number = 5 * 24 * 3600 * 1000; private noDataExpireTTL: number = 24 * 3600 * 1000; private noDataRefreshTTL: number = 15 * 60; // refresh TTL is in seconds private noDataPayload = "80010999,80310999,80020999,80220999,80120999"; // all categories are unknown private metricService: MetricService; private CATEGORY_SECTION: string = "cat_dv"; private BRAND_SAFETY_PRODUCT_SET: string = "80"; private instance: AxiosInstance; constructor() { this.dvBaseUrl = config["DV_CAT_BASEURL"] ?? "http://ogury.iq.doubleverify.com"; this.dvPath = config["CAT_DV_PATH"] ?? "/dv-iqc"; this.salt = config["DV_CAT_SALT"] ?? "dvhashed"; this.partnerId = config["DV_CAT_PARTNERID"] ?? "1234"; this.metricService = MetricService.getInstance(); this.metricService.addSection(this.CATEGORY_SECTION, [], BucketSet.zeroTo600ms); this.instance = axios.create({ baseURL: this.dvBaseUrl, timeout: 60000, httpAgent: new http.Agent({ keepAlive: true }), httpsAgent: new https.Agent({ keepAlive: true }), }); } private encodePayload(payload: string, ttl: number): string { const values: string[] = payload.split(","); const result: string[] = new Array(); result.push(ttl.toString(16)); values.forEach((elt: string) => { if (elt.startsWith(this.BRAND_SAFETY_PRODUCT_SET)) { result.push(parseInt(elt).toString(16)); } }); return result.join("|"); } decode(payload: string): ICategoryValue { return new DVCategoryValue(payload); } needUserAgent(): boolean { return true; } isRequestValid(categoryRequest: CategoryRequest): boolean { if ( !categoryRequest.device && categoryRequest.urlOrBundle && categoryRequest.urlOrBundle.length > 0 ) { return true; } return false; } async getCategories(categoryRequest: CategoryRequest): Promise { const request = this.generateURL(categoryRequest.urlOrBundle!); const endMetric = this.metricService.startSection(this.CATEGORY_SECTION); try { const response: AxiosResponse = await this.instance.get(request); const payload: string = response.data; const cc: string = response.headers["cache-control"]; let ttl: number = 86400; if (cc != null) { const arr = cc.split("="); if (arr.length === 2) { ttl = parseInt(arr[1]); } } endMetric(true); return { encodedValue: this.encodePayload(payload, Date.now() / 1000 + ttl), ttl: this.defaultExpireTTL, }; } catch (error: any) { endMetric(false); return { encodedValue: this.encodePayload(this.noDataPayload, this.noDataRefreshTTL), ttl: this.noDataExpireTTL, }; } } private generateURL(urlToRate: string): string { return this.authentify( this.dvPath + "?" + new URLSearchParams({ partnerid: this.partnerId, url: sanitizeURL(urlToRate) }), this.salt ); } private authentify(inputUrl: string, salt: string): string { const hash = createHash("sha256"); hash.update(inputUrl + salt); const digest: string = hash.digest("hex"); return inputUrl + "&hv=" + digest; } } export class DVCategoryValue implements ICategoryValue { private segments: number[]; private refreshTTL: number; private brandSafetyResult: Map = new Map< BrandSafetySeverity, CategoryRating >(); private highBSDataGroup: number[] = [1, 31]; // all high risk DataGroups private mediumBSDataGroup: number[] = [22]; // all medium risk BSDataGroups private lowBSDataGroup: number[] = [12]; // all low risk BSDataGroups constructor(value: string) { const values = value.split("|"); this.refreshTTL = parseInt(values.shift() ?? "", 16) * 1000; this.segments = values.map((stringValue) => parseInt(stringValue, 16)); this.parseData(); } toSegmentList(): string[] { return this.segments.map((value) => String(value)); } getPublisherFraud(_level: FraudSeverity): CategoryRating { return CategoryRating.Unknown; } getDeviceFraud(): CategoryRating { return CategoryRating.Unknown; } getBrandSafety(severity: BrandSafetySeverity): CategoryRating { return this.brandSafetyResult.get(severity)!; } getRefreshTTL(): number { return this.refreshTTL; } isBlockedBySegment(stringSegment: string): boolean { const filterSegment = parseInt(stringSegment); return this.segments.includes(filterSegment); } isBrandSuitable(_stringSegment: string, _passUnknown: boolean): boolean { return true; } private parseData() { this.brandSafetyResult.set( BrandSafetySeverity.High, this.getBrandSafetyValue(this.highBSDataGroup) ); this.brandSafetyResult.set( BrandSafetySeverity.Medium, this.getBrandSafetyValue(this.mediumBSDataGroup) ); this.brandSafetyResult.set( BrandSafetySeverity.Low, this.getBrandSafetyValue(this.lowBSDataGroup) ); } private getBrandSafetyValue(filterGroup: number[]): CategoryRating { let result: CategoryRating = CategoryRating.Safe; for (const segment of this.segments) { const dataGroup = this.getDataGroup(segment); if (filterGroup.includes(dataGroup)) { if (this.getSegmentType(segment) === 0) { result = CategoryRating.Unknown; } else { return CategoryRating.Unsafe; } } } return result; } private getDataGroup(segment: number): number { return this.getDigits(segment, 2, 2); } private getSegmentType(segment: number): number { return this.getDigits(segment, 4, 1); } private getDigits(segment: number, pos: number, size: number): number { segment = Math.floor(segment / Math.pow(10, 8 - (pos + size))); return segment % Math.pow(10, size); } }