import MetricService, { BucketSet, EndFunctionWithReq } from "../services/metric.service"; import { CampaignGroup, ChoiceRequest, RequestInventorySource } from "../models"; import { KafkaConnector } from "../models/connector/KafkaProducer"; import { BidRequestService } from "../services/BidRequestService"; import { bid } from "../beeswax-protos/beeswax"; import { hasCampaign, initCampaigns, initPromLabels } from "../utils/RequestUtils"; import { Step } from "../steps/step"; import ChoicesTopicService from "../services/choicesTopic.service"; import { executeStep } from "../steps/step.util"; import logger from "../utils/logger"; import constants from "../utils/constants"; import { toChoiceRequest, toOpenRtbResponse } from "../mappers/openRtb.mapper"; import { CampaignModel } from "../models/CampaignModel"; import { Request, Response } from "express"; import { validate } from "../validation/validation"; import { externalOpenRtbRequestSchema } from "../controllers/validation/ExternalRequestValidation"; import DeliveryMonitoringService from "../services/deliveryMonitoringService"; import { StepFactory } from "../steps/stepFactory"; import { FeatureConfigService } from "../services/FeatureConfig.service"; import Funnel from "./funnel"; const NORMALIZATION_PROTO_READ = "request_normalization_proto_read"; const NORMALIZATION_ADAPT = "request_normalization_adapt"; const PREPARE_RESPONSE_SECTION = "prepare_response"; export class ExternalInventoryService { private metricService: MetricService; private choicesTopicService: ChoicesTopicService; private deliveryTopicService: DeliveryMonitoringService; private featureConfigService: FeatureConfigService; constructor( private readonly campaignModel: CampaignModel, private readonly bidRequestService: BidRequestService, private readonly kafkaInternalRequestProducer: KafkaConnector, private readonly stepFactory: StepFactory ) { this.metricService = MetricService.getInstance(); this.choicesTopicService = ChoicesTopicService.getInstance(); this.deliveryTopicService = DeliveryMonitoringService.getInstance(); this.featureConfigService = FeatureConfigService.getInstance(); this.metricService.addSection(PREPARE_RESPONSE_SECTION, [], BucketSet.zeroTo1ms); this.metricService.addHistogram( NORMALIZATION_PROTO_READ, "duration of normalization proto read" ); this.metricService.addHistogram( NORMALIZATION_ADAPT, "duration of normalization request adaptation" ); } public async handle(req: Request, res: Response): Promise { const end = this.metricService.startSectionWithDelayedReq(constants.CHOICES_REQUEST_SECTION); let maybeRequest: ChoiceRequest | null = null; try { const decodedRequest = this.decode(req); const shouldLog = this.shouldLog(); Funnel.logRequest(RequestInventorySource.EXTERNAL, decodedRequest, shouldLog); const validationError = validate( decodedRequest, externalOpenRtbRequestSchema, RequestInventorySource.EXTERNAL ); if (validationError) { return this.respondWithValidationError(res, decodedRequest, shouldLog); } const normalizedRequest = await this.normalize(decodedRequest); maybeRequest = normalizedRequest; await this.runSteps( [ this.stepFactory.getAllowedCampaignGroupsStep(CampaignGroup.Managed), this.stepFactory.dvFraudStep, this.stepFactory.deviceFraudIasStep, this.stepFactory.publisherFraudIasStep, this.stepFactory.brandSafetyDvStep, this.stepFactory.brandSafetyIasStep, this.stepFactory.brandSuitabilityIasStep, this.stepFactory.externalBlacklistStep as Step, this.stepFactory.orderingStep, this.stepFactory.bidFloorStep, this.stepFactory.speedThresholdStep, this.stepFactory.priorityStep, this.stepFactory.campaignSizeStep, ], normalizedRequest ); // Prepare and send response return this.respond(req, res, decodedRequest, normalizedRequest, end, shouldLog); } catch (e: any) { end(false, maybeRequest, { inventory_source: RequestInventorySource.EXTERNAL, }); logger.error(e, `[${e.section}] Unable to handle external request`); } } public decode(req: Request): bid.BidAgentRequest { const endRead = this.metricService.startTimer(NORMALIZATION_PROTO_READ); const decodedRequest: bid.BidAgentRequest = bid.BidAgentRequest.decode(req.body); endRead({ inventory_source: RequestInventorySource.EXTERNAL }); return decodedRequest; } public async normalize(decodedRequest: bid.BidAgentRequest): Promise { const endAdapt = this.metricService.startTimer(NORMALIZATION_ADAPT); const request: ChoiceRequest = toChoiceRequest(decodedRequest); request.size = 1; await initCampaigns(request, this.campaignModel); await initPromLabels(request); endAdapt({ inventory_source: RequestInventorySource.EXTERNAL }); return request; } /** * Respond to beeswax. * @param res * @param req * @param decodedRequest * @param normalizedRequest * @param endTimerFunction * @return {Promise} */ private async respond( req: Request, res: Response, decodedRequest: bid.BidAgentRequest, normalizedRequest: ChoiceRequest, endTimerFunction: EndFunctionWithReq, shouldLog: boolean ) { const { rtbResponse, binary } = await this.prepareResponse(decodedRequest, normalizedRequest); Funnel.logResponse(RequestInventorySource.EXTERNAL, rtbResponse, shouldLog); res.writeHead(200, { "Content-Type": "application/x-protobuf" }); res.end(binary); setImmediate(() => { this.choicesTopicService.sendReport(normalizedRequest); this.deliveryTopicService.sendMessage(normalizedRequest); this.bidRequestService.logBid(req, normalizedRequest, decodedRequest, rtbResponse); }); endTimerFunction(true, normalizedRequest, { fill: hasCampaign(normalizedRequest).toString(), ...normalizedRequest.globalRequestLabels, }); } private async respondWithValidationError( res: Response, decodedRequest: bid.BidAgentRequest, shouldLog: boolean ) { const { rtbResponse, binary } = await this.prepareResponse(decodedRequest, null); Funnel.logResponse(RequestInventorySource.EXTERNAL, rtbResponse, shouldLog); res.writeHead(200, { "Content-Type": "application/x-protobuf" }); res.end(binary); } /** * Step 5 : prepare response * @param decodedRequest * @param normalizedRequest * @return {Promise<*>} */ private async prepareResponse( decodedRequest: bid.BidAgentRequest, normalizedRequest: ChoiceRequest | null ) { const end = this.metricService.startSectionWithReq(normalizedRequest, PREPARE_RESPONSE_SECTION); try { const rtbResponse = toOpenRtbResponse(decodedRequest, normalizedRequest); const binary = bid.BidAgentResponse.encode(rtbResponse).finish(); end(true); return { rtbResponse: rtbResponse, binary: binary, }; } catch (e: any) { end(false); e.section = PREPARE_RESPONSE_SECTION; throw e; } } protected async runSteps(steps: Step[], request: ChoiceRequest): Promise { for (const step of steps) { if (await executeStep(step, request)) { return true; } } return false; } private shouldLog(): boolean { return ( Math.random() < this.featureConfigService.getFloatValue(null, "ms-choices.log-request-pct", 0) ); } }