import MetricService, { BucketSet } from "../services/metric.service"; import S3Service from "./s3.service"; import moment from "moment"; import logger from "../utils/logger"; import { CampaignModel } from "../models/CampaignModel"; import { config } from "../utils/config"; const EXTERNAL_BLACKLIST_RELOAD_SECTION = "blacklist_external_reload"; const EXTERNAL_BLACKLIST_ENTRY_GAUGE = "blacklist_external_entries_total"; export class ExternalBlacklistService { blacklist: Set; reloadTime: number; metricService: MetricService; constructor( private readonly s3service: S3Service, private readonly campaignModel: CampaignModel, reloadTime: number, private readonly bucket: string, private readonly fileName: string ) { this.metricService = MetricService.getInstance(); this.metricService.addSection(EXTERNAL_BLACKLIST_RELOAD_SECTION, [], BucketSet.zeroTo600ms); this.metricService.addGauge( EXTERNAL_BLACKLIST_ENTRY_GAUGE, "Number of entries in blacklist", [] ); this.reloadTime = reloadTime * 60000; this.blacklist = new Set(); if (config.OGURY_ENV !== "test") { this.start(); logger.info( `[BLACKLIST_EXTERNAL] S3 file [bucket: '${this.bucket}' key: '${this.fileName}'] will be reloaded every ${this.reloadTime} ms` ); } } isBlacklisted(campaignId: string): boolean { return this.blacklist.has(campaignId); } start() { this.refresh(); setInterval(() => { this.refresh(); }, this.reloadTime); } async refresh() { logger.info("[BLACKLIST_EXTERNAL] Reloading file"); const end = this.metricService.startSection(EXTERNAL_BLACKLIST_RELOAD_SECTION); try { const campaignIds = await this.loadBlacklistFile(); const filteredIds = await this.removeForcedExternal(campaignIds); this.blacklist = new Set(filteredIds); end(true); } catch (err: any) { logger.error(err, `[BLACKLIST_EXTERNAL] Reload error`); this.blacklist = new Set(); end(false); } finally { this.metricService.setGauge(EXTERNAL_BLACKLIST_ENTRY_GAUGE, this.blacklist.size); } } async loadBlacklistFile(): Promise { const { file, lastModified } = await this.s3service.loadFile(this.bucket, this.fileName); if (!moment.utc().isSame(lastModified, "day")) { logger.info("[BLACKLIST_EXTERNAL] File outdated, aborting reload"); return []; } const entries = file .toString("utf-8") .split("\n") .map((line) => line.trim()) .filter((line) => !!line); logger.info(`[BLACKLIST_EXTERNAL] File reloaded: ${entries.length} entries`); return entries; } // exclude campaigns that must deliver on external anyway async removeForcedExternal(campaignIds: string[]): Promise { const result = []; const campaignMap = await this.campaignModel.findByMultipleId(campaignIds); for (const cId of campaignIds) { try { const isForcedExternal = this.campaignModel.isExternalForced(campaignMap[cId]!); if (!isForcedExternal) { result.push(cId); } } catch (err: any) { logger.error(err, `[BLACKLIST_EXTERNAL] Could not load campaign ${cId}`); continue; } } return result; } }