import axios, { AxiosInstance, AxiosResponse } from "axios"; import http from "http"; import https from "https"; import { RequestSource, decodeRequestSource, encodeRequestSource } from "../utils/RequestUtils"; import MetricService, { BucketSet } from "../services/metric.service"; import { config } from "../utils/config"; import { CategoryProviderValue, ICategoryProvider, ICategoryValue, CategoryRating, BrandSafetySeverity, FraudSeverity, } from "./categoryProvider"; import { CategoryRequest } from "../services/category.service"; import logger from "../utils/logger"; export default class IASCategoryProvider implements ICategoryProvider { private iasPath: string; private authToken: string = ""; private authExpire: number = 0; private defaultExpireTTL: number = 5 * 24 * 3600 * 1000; private noDataExpireTTL: number = 24 * 3600 * 1000; private noDataRefreshTTL: number = 15 * 60; // refresh TTL is in seconds private tokenExpireMargin: number = 5; private noDataPayload = '{"segment_ids":[]}'; private metricService: MetricService; private CATEGORY_SECTION: string = "cat_ias"; private webInstance: AxiosInstance; private appInstance: AxiosInstance; private authInstance: AxiosInstance; constructor() { const iasWebBaseUrl = config["IAS_CAT_WEB_URL"] ?? "https://api.adsafeprotected.com"; const iasAppBaseUrl = config["IAS_CAT_APP_URL"] ?? "https://appapi.adsafeprotected.com"; const iasAuthBaseUrl = config["IAS_CAT_AUTH_URL"] ?? "https://auth.adsafeprotected.com"; const clientId = config["IAS_CAT_CLIENTID"] ?? "1234"; const login = config["IAS_CAT_LOGIN"] ?? "mylogin"; const pass = config["IAS_CAT_PASS"] ?? "mypass"; this.iasPath = `/db2/client/${clientId}/segt`; this.metricService = MetricService.getInstance(); this.metricService.addSection(this.CATEGORY_SECTION, ["device"], BucketSet.zeroTo600ms); this.webInstance = axios.create({ baseURL: iasWebBaseUrl, timeout: 60000, httpAgent: new http.Agent({ keepAlive: true }), httpsAgent: new https.Agent({ keepAlive: true }), }); this.appInstance = axios.create({ baseURL: iasAppBaseUrl, timeout: 60000, httpAgent: new http.Agent({ keepAlive: true }), httpsAgent: new https.Agent({ keepAlive: true }), }); this.authInstance = axios.create({ baseURL: iasAuthBaseUrl, auth: { username: login, password: pass, }, params: { grant_type: "client_credentials", }, timeout: 60000, httpAgent: new http.Agent({ keepAlive: true }), httpsAgent: new https.Agent({ keepAlive: true }), }); } private encodePayload(payload: any, ttl: number, requestSource: RequestSource): string { const values: string[] = payload.segment_ids || []; const result: string[] = new Array(); result.push(encodeRequestSource(requestSource)); result.push(ttl.toString(16)); values.forEach((elt: string) => { result.push(parseInt(elt).toString()); }); return result.join("|"); } decode(payload: string): ICategoryValue { return new IASCategoryValue(payload); } async getToken(): Promise { if (this.authToken && Date.now() < this.authExpire + this.tokenExpireMargin) { return this.authToken; } const response: AxiosResponse = await this.authInstance.post("/uaa/oauth/token"); const payload: any = response.data as any; const authToken = payload.access_token; const expires = payload.expires_in; if (expires > this.tokenExpireMargin) { this.authToken = authToken; this.authExpire = Date.now() + expires * 1000; } return authToken; } isRequestValid(categoryRequest: CategoryRequest): boolean { if ( !categoryRequest.device && categoryRequest.urlOrBundle && categoryRequest.urlOrBundle.length > 0 ) { return true; } if (categoryRequest.device && categoryRequest.ip && categoryRequest.ip.length > 0) { return true; } return false; } async getCategories(categoryRequest: CategoryRequest): Promise { const request = this.generateURL(categoryRequest); const endMetric = this.metricService.startSection(this.CATEGORY_SECTION); try { const token = await this.getToken(); const instance = categoryRequest.requestSource.environmentType === "web" || categoryRequest.urlOrBundle === undefined ? this.webInstance : this.appInstance; const response: AxiosResponse = await instance.request({ url: request, headers: { Authorization: "bearer " + token }, }); const payload = response.data; const cc: string = response.headers["cache-control"]; let ttl: number = this.noDataRefreshTTL; let hasTtl = false; if (cc != null) { const arr = cc.split("="); if (arr.length === 2) { if (arr[1].toLowerCase() === "no cache") { ttl = 0; } else { ttl = parseInt(arr[1]); } hasTtl = true; } } endMetric(true, { device: String(categoryRequest.device) }); return { encodedValue: this.encodePayload( payload, Date.now() / 1000 + ttl, categoryRequest.requestSource ), ttl: hasTtl ? this.defaultExpireTTL : this.noDataExpireTTL, }; } catch (error: any) { endMetric(false, { device: String(categoryRequest.device) }); if (Math.random() <= 0.05) { logger.error(error, { message: "IAS request error", responseData: error.response.data, errCode: error.response.status, }); } return { encodedValue: this.encodePayload( this.noDataPayload, Date.now() / 1000 + this.noDataRefreshTTL, categoryRequest.requestSource ), ttl: this.noDataExpireTTL, }; } } private generateURL(categoryRequest: CategoryRequest): string { if (!categoryRequest.device) { if (categoryRequest.requestSource.environmentType === "web") { return ( this.iasPath + "?" + new URLSearchParams({ adsafe_url: categoryRequest.urlOrBundle! }) ); } else { return ( this.iasPath + "?" + new URLSearchParams({ bundle_id: categoryRequest.urlOrBundle!, app_store: categoryRequest.requestSource.deviceOS === "android" ? "android" : "apple", }) ); } } else { return this.iasPath + "?" + new URLSearchParams({ adsafe_ip: categoryRequest.ip! }); } } needUserAgent(): boolean { return false; } } export class IASCategoryValue implements ICategoryValue { private segments: number[]; private refreshTTL: number; private requestSource: RequestSource; private publisherFraud: Map = new Map(); private brandSafetyResult: Map = new Map< BrandSafetySeverity, CategoryRating >(); private deviceFraud: CategoryRating = CategoryRating.Unknown; private unknownBrandSuitability = 1000075; private dataGroups = { web: { brandSafety: { high: [501, 535, 503, 537, 507, 541, 505, 539, 509, 543, 511, 545, 531, 533], moderate: [502, 536, 504, 538, 508, 542, 506, 540, 510, 544, 512, 546, 532, 534], unclassified: [513, 551], }, publisherFraud: { high: [401, 404], moderate: [402, 407], unclassified: [400, 410], }, deviceFraud: [411], }, app: { brandSafety: { high: [ 4000, 4002, 4004, 4006, 4008, 4010, 4012, 4018, 8000, 8002, 8006, 8004, 8008, 8010, 8012, 8018, ], moderate: [ 4001, 4003, 4005, 4007, 4009, 4011, 4013, 4019, 8001, 8003, 8005, 8007, 8009, 8011, 8013, 8019, ], unclassified: [4014, 8014], }, publisherFraud: { high: [4016, 8016], moderate: [8017, 4017], unclassified: [4015, 8015], }, deviceFraud: [411], }, }; private brandSuitabilityNegativeFilteringRanges = [ // segments in this range are negative: if present the campaign should not be displayed { min: 1000002, max: 1999999 }, { min: 4000000, max: 4999999 }, ]; constructor(value: string) { const values = value.split("|"); this.requestSource = decodeRequestSource(values.shift() ?? ""); this.refreshTTL = parseInt(values.shift() ?? "", 16) * 1000; this.segments = values.map((stringValue) => parseInt(stringValue)); this.parseData(); } toSegmentList(): string[] { return this.segments.map((value) => String(value)); } getBrandSafety(severity: BrandSafetySeverity): CategoryRating { return this.brandSafetyResult.get(severity)!; } getPublisherFraud(severity: FraudSeverity): CategoryRating { return this.publisherFraud.get(severity)!; } getDeviceFraud(): CategoryRating { return this.deviceFraud; } getRefreshTTL(): number { return this.refreshTTL; } isBrandSuitable(stringSegment: string, passUnknown: boolean): boolean { if (!passUnknown && this.segments.includes(this.unknownBrandSuitability)) { return false; } return !this.isBlockedBySegment(stringSegment); } isBlockedBySegment(stringSegment: string): boolean { const filterSegment = parseInt(stringSegment); if (isNaN(filterSegment)) { return false; } let negative = false; for (const range of this.brandSuitabilityNegativeFilteringRanges) { if (filterSegment >= range.min && filterSegment <= range.max) { negative = true; break; } } return negative === this.segments.includes(filterSegment); } private parseData() { let v; if (this.requestSource.environmentType === "web") v = this.dataGroups.web; else v = this.dataGroups.app; this.brandSafetyResult.set( BrandSafetySeverity.High, this.getBrandSafetyValue(v.brandSafety.high, v.brandSafety.unclassified) ); const medium = this.getBrandSafetyValue(v.brandSafety.moderate, v.brandSafety.unclassified); this.brandSafetyResult.set(BrandSafetySeverity.Medium, medium); this.brandSafetyResult.set(BrandSafetySeverity.Low, medium); this.publisherFraud.set( FraudSeverity.High, this.getBrandSafetyValue(v.publisherFraud.high, v.publisherFraud.unclassified) ); this.publisherFraud.set( FraudSeverity.Medium, this.getBrandSafetyValue(v.publisherFraud.moderate, v.publisherFraud.unclassified) ); this.deviceFraud = this.containsAll(this.dataGroups.web.deviceFraud) ? CategoryRating.Safe : CategoryRating.Unsafe; } private getBrandSafetyValue(filterGroup: number[], unknownGroup: number[]): CategoryRating { if (this.containsAll(filterGroup)) { return CategoryRating.Safe; } if (this.containsAll(unknownGroup)) { return CategoryRating.Unsafe; } return CategoryRating.Unknown; } private containsAll(filterGroup: number[]): boolean { for (const segment of filterGroup) { if (!this.segments.includes(segment)) { return false; } } return true; } }