{ "annotations": { "list": [ { "builtIn": 1, "datasource": "-- Grafana --", "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "target": { "limit": 100, "matchAny": false, "tags": [], "type": "dashboard" }, "type": "dashboard" } ] }, "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, "id": 101045, "iteration": 1651588745452, "links": [], "liveNow": false, "panels": [ { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "id": 45, "panels": [], "title": "Profiling global", "type": "row" }, { "aliasColors": { "INTERNAL error": "red" }, "bars": false, "dashLength": 10, "dashes": false, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 1 }, "hiddenSeries": false, "id": 111, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.1", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "sum(choices_request_duration_seconds_count:rate2m{platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=~\"$inventory_source\",source_type=~\"$source_type\",error=\"false\"}) by (inventory_source)", "interval": "", "legendFormat": "{{inventory_source}} success", "refId": "A" }, { "expr": "sum(choices_request_duration_seconds_count:rate2m{platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=~\"$inventory_source\",source_type=~\"$source_type\",error=\"true\"}) by (inventory_source)", "interval": "", "legendFormat": "{{inventory_source}} error", "refId": "B" } ], "thresholds": [], "timeRegions": [], "title": "requests/s", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:352", "format": "none", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:353", "format": "none", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": { "INTERNAL error": "red" }, "bars": false, "dashLength": 10, "dashes": false, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 1 }, "hiddenSeries": false, "id": 300, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.1", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "exemplar": true, "expr": "histogram_quantile(0.50, sum(choices_request_duration_seconds_bucket:rate2m{platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=~\"$inventory_source\",source_type=~\"$source_type\",error=\"false\"}) by (le))", "interval": "", "legendFormat": "0.5p", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "exemplar": true, "expr": "histogram_quantile(0.9, sum(choices_request_duration_seconds_bucket:rate2m{platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=~\"$inventory_source\",source_type=~\"$source_type\",error=\"false\"}) by (le))", "hide": false, "interval": "", "legendFormat": "0.9p", "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "expr": "histogram_quantile(0.99, sum(choices_request_duration_seconds_bucket:rate2m{platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=~\"$inventory_source\",source_type=~\"$source_type\",error=\"false\"}) by (le))", "hide": false, "interval": "", "legendFormat": "0.99p", "refId": "C" } ], "thresholds": [], "timeRegions": [], "title": "response time", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:352", "format": "s", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:353", "format": "none", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 9 }, "hiddenSeries": false, "id": 193, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.1", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "exemplar": true, "expr": "sum(nginx_ingress_controller_requests:rate2m{region=~\"$region\",platform=~\"$env\",namespace=~\"ms-choices\",ingress=~\"ms-choices-int-private\"}) / sum(kube_pod_container_resource_requests{region=~\"$region\", platform=~\"$env\", namespace=~\"ms-choices\", pod=~\"ms-choices-int-.*\", container!=\"POD\",container!=\"\",pod!=\"\", job=\"kube-state-metrics\",unit=\"core\"})", "format": "heatmap", "interval": "", "legendFormat": "Internal", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "exemplar": true, "expr": "sum(nginx_ingress_controller_requests:rate2m{region=~\"$region\",platform=~\"$env\",namespace=~\"ms-choices\",ingress=~\"ms-choices-ext-public\"}) / sum(kube_pod_container_resource_requests{region=~\"$region\", platform=~\"$env\", namespace=~\"ms-choices\", pod=~\"ms-choices-ext-.*\", container!=\"POD\",container!=\"\",pod!=\"\", job=\"kube-state-metrics\",unit=\"core\"})", "interval": "", "legendFormat": "External", "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "exemplar": true, "expr": "sum(kube_pod_container_resource_requests{region=\"$region\", platform=\"$env\", namespace=~\"ms-choices\", pod=~\"ms-choices-int-.*\", container!=\"POD\",container!=\"\",pod!=\"\", job=\"kube-state-metrics\",unit=\"core\"})", "hide": false, "interval": "", "legendFormat": "dsfsd", "refId": "C" } ], "thresholds": [], "timeRegions": [], "title": "QPS / core", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:234", "format": "short", "logBase": 1, "show": true }, { "$$hashKey": "object:235", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": { "INTERNAL error": "red" }, "bars": false, "dashLength": 10, "dashes": false, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 9 }, "hiddenSeries": false, "id": 311, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.1", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "exemplar": true, "expr": "sum(rate(nginx_ingress_controller_ingress_upstream_latency_seconds_sum{region=\"eu-west-1\",platform=\"$env\",stack=~\"$stack\",namespace=\"ms-choices\"}[1m])) / sum(rate(nginx_ingress_controller_ingress_upstream_latency_seconds_count{region=\"eu-west-1\",platform=\"$env\",stack=~\"$stack\",namespace=\"ms-choices\"}[1m]))", "interval": "", "legendFormat": "eu-west-1", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "exemplar": true, "expr": "sum(rate(nginx_ingress_controller_ingress_upstream_latency_seconds_sum{region=\"us-east-1\",platform=\"$env\",stack=~\"$stack\",namespace=\"ms-choices\"}[1m])) / sum(rate(nginx_ingress_controller_ingress_upstream_latency_seconds_count{region=\"us-east-1\",platform=\"$env\",stack=~\"$stack\",namespace=\"ms-choices\"}[1m]))", "hide": false, "interval": "", "legendFormat": "us-east-1", "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "exemplar": true, "expr": "sum(rate(nginx_ingress_controller_ingress_upstream_latency_seconds_sum{region=\"us-west-2\",platform=\"$env\",stack=~\"$stack\",namespace=\"ms-choices\"}[1m])) / sum(rate(nginx_ingress_controller_ingress_upstream_latency_seconds_count{region=\"us-west-2\",platform=\"$env\",stack=~\"$stack\",namespace=\"ms-choices\"}[1m]))", "hide": false, "interval": "", "legendFormat": "us-west-2", "refId": "C" }, { "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "exemplar": true, "expr": "sum(rate(nginx_ingress_controller_ingress_upstream_latency_seconds_sum{region=\"ap-northeast-1\",platform=\"$env\",stack=~\"$stack\",namespace=\"ms-choices\"}[1m])) / sum(rate(nginx_ingress_controller_ingress_upstream_latency_seconds_count{region=\"ap-northeast-1\",platform=\"$env\",stack=~\"$stack\",namespace=\"ms-choices\"}[1m]))", "hide": false, "interval": "", "legendFormat": "ap-northeast-1", "refId": "D" } ], "thresholds": [], "timeRegions": [], "title": "average response time (1min)", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:352", "format": "s", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:353", "format": "none", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": { "INTERNAL error": "red" }, "bars": false, "dashLength": 10, "dashes": false, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 17 }, "hiddenSeries": false, "id": 312, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.1", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "exemplar": true, "expr": "sum(rate(nginx_ingress_controller_requests{region=\"eu-west-1\",platform=\"$env\",stack=~\"$stack\",namespace=\"ms-choices\",status!~\"5.*\"}[1m])) / sum(rate(nginx_ingress_controller_requests{region=\"eu-west-1\",platform=\"$env\",stack=~\"$stack\",namespace=\"ms-choices\"}[1m]))", "interval": "", "legendFormat": "eu-west-1", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "exemplar": true, "expr": "sum(rate(nginx_ingress_controller_requests{region=\"us-east-1\",platform=\"$env\",stack=~\"$stack\",namespace=\"ms-choices\",status!~\"5.*\"}[1m])) / sum(rate(nginx_ingress_controller_requests{region=\"us-east-1\",platform=\"$env\",stack=~\"$stack\",namespace=\"ms-choices\"}[1m]))", "hide": false, "interval": "", "legendFormat": "us-east-1", "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "exemplar": true, "expr": "sum(rate(nginx_ingress_controller_requests{region=\"us-west-2\",platform=\"$env\",stack=~\"$stack\",namespace=\"ms-choices\",status!~\"5.*\"}[1m])) / sum(rate(nginx_ingress_controller_requests{region=\"us-west-2\",platform=\"$env\",stack=~\"$stack\",namespace=\"ms-choices\"}[1m]))", "hide": false, "interval": "", "legendFormat": "us-west-2", "refId": "C" }, { "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "exemplar": true, "expr": "sum(rate(nginx_ingress_controller_requests{region=\"ap-northeast-1\",platform=\"$env\",stack=~\"$stack\",namespace=\"ms-choices\",status!~\"5.*\"}[1m])) / sum(rate(nginx_ingress_controller_requests{region=\"ap-northeast-1\",platform=\"$env\",stack=~\"$stack\",namespace=\"ms-choices\"}[1m]))", "hide": false, "interval": "", "legendFormat": "ap-northeast-1", "refId": "D" } ], "thresholds": [], "timeRegions": [], "title": "average success rate (1min)", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:352", "format": "percentunit", "logBase": 1, "show": true }, { "$$hashKey": "object:353", "format": "none", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 25 }, "id": 205, "panels": [ { "aliasColors": { "INTERNAL error": "red" }, "bars": false, "dashLength": 10, "dashes": false, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 2 }, "hiddenSeries": false, "id": 202, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.1", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(choices_request_duration_seconds_count:rate2m{platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=\"INTERNAL\",source_type=\"ANDROID\",error=\"false\",fill=\"true\"}) / sum(choices_request_duration_seconds_count:rate2m{platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=\"INTERNAL\",source_type=\"ANDROID\",error=\"false\"})", "interval": "", "legendFormat": "INTERNAL android", "refId": "A" }, { "expr": "sum(choices_request_duration_seconds_count:rate2m{platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=\"INTERNAL\",source_type=\"IOS\",error=\"false\",fill=\"true\"}) / sum(choices_request_duration_seconds_count:rate2m{platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=\"INTERNAL\",source_type=\"IOS\",error=\"false\"})", "interval": "", "legendFormat": "INTERNAL ios", "refId": "B" }, { "expr": "sum(choices_request_duration_seconds_count:rate2m{platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=\"INTERNAL\",source_type=\"WEBMOBILE\",error=\"false\",fill=\"true\"}) / sum(choices_request_duration_seconds_count:rate2m{platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=\"INTERNAL\",source_type=\"WEBMOBILE\",error=\"false\"})", "interval": "", "legendFormat": "INTERNAL web", "refId": "C" }, { "expr": "sum(choices_request_duration_seconds_count:rate2m{platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=\"HEADER_BIDDING\",source_type=\"ANDROID\",error=\"false\",fill=\"true\"}) / sum(choices_request_duration_seconds_count:rate2m{platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=\"HEADER_BIDDING\",source_type=\"ANDROID\",error=\"false\"})", "interval": "", "legendFormat": "HEADER_BIDDING android", "refId": "D" }, { "expr": "sum(choices_request_duration_seconds_count:rate2m{platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=\"HEADER_BIDDING\",source_type=\"IOS\",error=\"false\",fill=\"true\"}) / sum(choices_request_duration_seconds_count:rate2m{platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=\"HEADER_BIDDING\",source_type=\"IOS\",error=\"false\"})", "interval": "", "legendFormat": "HEADER_BIDDING ios", "refId": "E" }, { "expr": "sum(choices_request_duration_seconds_count:rate2m{platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=\"HEADER_BIDDING\",source_type=\"WEBMOBILE\",error=\"false\",fill=\"true\"}) / sum(choices_request_duration_seconds_count:rate2m{platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=\"HEADER_BIDDING\",source_type=\"WEBMOBILE\",error=\"false\"})", "interval": "", "legendFormat": "HEADER_BIDDING web", "refId": "F" }, { "expr": "sum(choices_request_duration_seconds_count:rate2m{platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=\"EXTERNAL\",source_type=\"ANDROID\",error=\"false\",fill=\"true\"}) / sum(choices_request_duration_seconds_count:rate2m{platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=\"EXTERNAL\",source_type=\"ANDROID\",error=\"false\"})", "interval": "", "legendFormat": "EXTERNAL android", "refId": "G" }, { "expr": "sum(choices_request_duration_seconds_count:rate2m{platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=\"EXTERNAL\",source_type=\"IOS\",error=\"false\",fill=\"true\"}) / sum(choices_request_duration_seconds_count:rate2m{platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=\"EXTERNAL\",source_type=\"IOS\",error=\"false\"})", "interval": "", "legendFormat": "EXTERNAL ios", "refId": "H" } ], "thresholds": [], "timeRegions": [], "title": "fill rate", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "format": "percentunit", "logBase": 1, "min": "0", "show": true }, { "format": "none", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": { "INTERNAL error": "red" }, "bars": false, "dashLength": 10, "dashes": false, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 2 }, "hiddenSeries": false, "id": 203, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.1", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(choices_request_duration_seconds_count:rate2m{platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=~\"$inventory_source\",source_type=~\"$source_type\",error=\"false\",fill=\"true\"})", "interval": "", "legendFormat": "fill", "refId": "A" }, { "expr": "sum(choices_request_duration_seconds_count:rate2m{platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=~\"$inventory_source\",source_type=~\"$source_type\",error=\"false\",fill=\"false\"})", "interval": "", "legendFormat": "no fill", "refId": "B" } ], "thresholds": [], "timeRegions": [], "title": "fill rate", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "format": "none", "logBase": 1, "min": "0", "show": true }, { "format": "none", "logBase": 1, "show": true } ], "yaxis": { "align": false } } ], "title": "Fill Rate", "type": "row" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 26 }, "id": 308, "panels": [ { "aliasColors": { "INTERNAL error": "red" }, "bars": false, "dashLength": 10, "dashes": false, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 3 }, "hiddenSeries": false, "id": 309, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.1", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "exemplar": true, "expr": "sum(rate(request_validation_duration_seconds_count{platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=~\"$inventory_source\",skipped=\"true\"}[2m]))", "interval": "", "legendFormat": "skipped", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "exemplar": true, "expr": "sum(rate(request_validation_duration_seconds_count{platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=~\"$inventory_source\",skipped=\"false\",error=\"false\"}[2m]))", "hide": false, "interval": "", "legendFormat": "pass", "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "exemplar": true, "expr": "sum(rate(request_validation_duration_seconds_count{platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=~\"$inventory_source\",skipped=\"false\",error=\"true\"}[2m]))", "hide": false, "interval": "", "legendFormat": "error", "refId": "C" } ], "thresholds": [], "timeRegions": [], "title": "request validation", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:142", "format": "none", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:143", "format": "none", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": { "INTERNAL error": "red" }, "bars": false, "dashLength": 10, "dashes": false, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 3 }, "hiddenSeries": false, "id": 310, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.1", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "exemplar": true, "expr": "histogram_quantile(0.50, sum(request_validation_duration_seconds_bucket:rate2m{platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=~\"$inventory_source\",source_type=~\"$source_type\",skipped=\"false\"}) by (le))", "interval": "", "legendFormat": "0.5p", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "exemplar": true, "expr": "histogram_quantile(0.90, sum(request_validation_duration_seconds_bucket:rate2m{platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=~\"$inventory_source\",source_type=~\"$source_type\",error=\"false\"}) by (le))", "hide": false, "interval": "", "legendFormat": "0.9p", "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "exemplar": true, "expr": "histogram_quantile(0.99, sum(request_validation_duration_seconds_bucket:rate2m{platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=~\"$inventory_source\",source_type=~\"$source_type\",error=\"false\"}) by (le))", "hide": false, "interval": "", "legendFormat": "0.99p", "refId": "C" } ], "thresholds": [], "timeRegions": [], "title": "validation duration", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:352", "format": "s", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:353", "format": "none", "logBase": 1, "show": true } ], "yaxis": { "align": false } } ], "title": "Request Validation", "type": "row" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 27 }, "id": 188, "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 24, "x": 0, "y": 4 }, "hiddenSeries": false, "id": 189, "legend": { "alignAsTable": true, "avg": false, "current": false, "hideEmpty": false, "hideZero": false, "max": false, "min": false, "rightSide": true, "show": true, "sideWidth": 300, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.1", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.95, sum(dv_fraud_duration_seconds_bucket:rate2m{platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=~\"$inventory_source\",source_type=~\"$source_type\",error=\"false\"}) by (le))", "hide": false, "interval": "", "legendFormat": "dv_fraud", "refId": "G" }, { "expr": "histogram_quantile(0.95, sum(yield_duration_seconds_bucket:rate2m{platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=~\"$inventory_source\",source_type=~\"$source_type\",error=\"false\"}) by (le))", "hide": false, "interval": "", "legendFormat": "yield", "refId": "A" }, { "expr": "histogram_quantile(0.95, sum(ordering_duration_seconds_bucket:rate2m{platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=~\"$inventory_source\",source_type=~\"$source_type\",error=\"false\"}) by (le))", "hide": false, "interval": "", "legendFormat": "ordering", "refId": "C" }, { "expr": "histogram_quantile(0.95, sum(speed_duration_seconds_bucket:rate2m{platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=~\"$inventory_source\",source_type=~\"$source_type\",error=\"false\"}) by (le))", "hide": false, "interval": "", "legendFormat": "speed", "refId": "D" }, { "expr": "histogram_quantile(0.95, sum(threshold_duration_seconds_bucket:rate2m{platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=~\"$inventory_source\",source_type=~\"$source_type\",error=\"false\"}) by (le))", "hide": false, "interval": "", "legendFormat": "threshold", "refId": "E" }, { "expr": "histogram_quantile(0.95, sum(speed_threshold_duration_seconds_bucket:rate2m{platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=~\"$inventory_source\",source_type=~\"$source_type\",error=\"false\"}) by (le))", "hide": false, "interval": "", "legendFormat": "speed_threshold", "refId": "F" }, { "expr": "histogram_quantile(0.95, sum(multi_formats_candidates_selection_duration_seconds_bucket:rate2m{platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=~\"$inventory_source\",source_type=~\"$source_type\",error=\"false\"}) by (le))", "hide": false, "interval": "", "legendFormat": "multi_formats_candidates_selection", "refId": "H" }, { "expr": "histogram_quantile(0.95, sum(priority_duration_seconds_bucket:rate2m{platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=~\"$inventory_source\",source_type=~\"$source_type\",error=\"false\"}) by (le))", "hide": false, "interval": "", "legendFormat": "priority", "refId": "I" }, { "expr": "histogram_quantile(0.95, sum(rate(campaign_size_duration_seconds_bucket{platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=~\"$inventory_source\",source_type=~\"$source_type\",error=\"false\"}[2m])) by (le))", "hide": false, "interval": "", "legendFormat": "campaign_size", "refId": "J" }, { "expr": "histogram_quantile(0.95, sum(programmatic_ivt_duration_seconds_bucket:rate2m{platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=~\"$inventory_source\",source_type=~\"$source_type\",error=\"false\"}) by (le))", "hide": false, "interval": "", "legendFormat": "programmatic_ivt", "refId": "K" }, { "expr": "histogram_quantile(0.95, sum(best_programmatic_duration_seconds_bucket:rate2m{platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=~\"$inventory_source\",source_type=~\"$source_type\",error=\"false\"}) by (le))", "hide": false, "interval": "", "legendFormat": "best_programmatic", "refId": "L" }, { "expr": "histogram_quantile(0.95, sum(programmatic_response_duration_seconds_bucket:rate2m{platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=~\"$inventory_source\",source_type=~\"$source_type\"}) by (le))", "hide": false, "interval": "", "legendFormat": "programmatic_response", "refId": "M" }, { "expr": "histogram_quantile(0.95,sum(brand_safety_duration_seconds_bucket:rate2m{platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=~\"$inventory_source\",source_type=~\"$source_type\"}) by (le))", "interval": "", "legendFormat": "brand_safety", "refId": "B" } ], "thresholds": [], "timeRegions": [], "title": "steps duration p95", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:435", "format": "s", "label": "", "logBase": 1, "show": true }, { "$$hashKey": "object:436", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 24, "x": 0, "y": 12 }, "hiddenSeries": false, "id": 230, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "sideWidth": 300, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.1", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "expr": "sum(dv_fraud_duration_seconds_bucket:rate2m{kubernetes_namespace=\"ms-choices\",platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=~\"$inventory_source\",source_type=~\"$source_type\"})", "interval": "", "legendFormat": "dv fraud", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "expr": "sum(brand_safety_duration_seconds_bucket:rate2m{kubernetes_namespace=\"ms-choices\",platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=~\"$inventory_source\",source_type=~\"$source_type\"})", "interval": "", "legendFormat": "brand safety", "refId": "I" }, { "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "expr": "sum(yield_duration_seconds_bucket:rate2m{kubernetes_namespace=\"ms-choices\",platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=~\"$inventory_source\",source_type=~\"$source_type\"})", "interval": "", "legendFormat": "yield", "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "expr": "sum(ordering_duration_seconds_bucket:rate2m{kubernetes_namespace=\"ms-choices\",platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=~\"$inventory_source\",source_type=~\"$source_type\"})", "interval": "", "legendFormat": "ordering", "refId": "C" }, { "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "expr": "sum(multi_formats_candidates_selection_duration_seconds_bucket:rate2m{kubernetes_namespace=\"ms-choices\",platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=~\"$inventory_source\",source_type=~\"$source_type\"})", "interval": "", "legendFormat": "multi formats candiates sleection", "refId": "E" }, { "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "expr": "sum(speed_threshold_duration_seconds_bucket:rate2m{kubernetes_namespace=\"ms-choices\",platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=~\"$inventory_source\",source_type=~\"$source_type\"})", "interval": "", "legendFormat": "speed threshold", "refId": "D" }, { "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "expr": "sum(priority_duration_seconds_bucket:rate2m{kubernetes_namespace=\"ms-choices\",platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=~\"$inventory_source\",source_type=~\"$source_type\"})", "interval": "", "legendFormat": "priority", "refId": "F" }, { "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "expr": "sum(programmatic_ivt_duration_seconds_bucket:rate2m{kubernetes_namespace=\"ms-choices\",platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=~\"$inventory_source\",source_type=~\"$source_type\"})", "interval": "", "legendFormat": "programmatic ivt", "refId": "G" }, { "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "expr": "sum(programmatic_response_duration_seconds_bucket:rate2m{kubernetes_namespace=\"ms-choices\",platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=~\"$inventory_source\",source_type=~\"$source_type\"})", "interval": "", "legendFormat": "programmatic response", "refId": "H" } ], "thresholds": [], "timeRegions": [], "title": "steps executions", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:517", "format": "short", "logBase": 1, "show": true }, { "$$hashKey": "object:518", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 24, "x": 0, "y": 20 }, "hiddenSeries": false, "id": 234, "legend": { "alignAsTable": true, "avg": false, "current": false, "hideEmpty": false, "hideZero": false, "max": false, "min": false, "rightSide": true, "show": true, "sideWidth": 300, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.1", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.95, sum(ordering_stage_duration_seconds_bucket:rate2m{platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=~\"$inventory_source\",source_type=~\"$source_type\"}) by (le, stage))", "hide": false, "interval": "", "legendFormat": "{{ stage }}", "refId": "G" } ], "thresholds": [], "timeRegions": [], "title": "Ordering step profiling", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:435", "format": "s", "label": "", "logBase": 1, "show": true }, { "$$hashKey": "object:436", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } } ], "title": "Steps", "type": "row" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 28 }, "id": 245, "panels": [ { "fieldConfig": { "defaults": { "custom": {}, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 4 }, "id": 247, "options": { "displayMode": "gradient", "orientation": "horizontal", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showUnfilled": true }, "pluginVersion": "7.2.0", "repeat": "step_for_campaign_num", "repeatDirection": "v", "targets": [ { "expr": "sum(step_campaign_num_bucket:sum{kubernetes_namespace=\"ms-choices\",platform=~\"$env\",region=~\"$region\",inventory_source=~\"$inventory_source\",source_type=~\"$source_type\",step=~\"$step_for_campaign_num\",stage=\"in\"}) by (le)", "format": "heatmap", "interval": "", "legendFormat": "{{le}}", "refId": "A" } ], "title": "$step_for_campaign_num IN", "type": "bargauge" }, { "fieldConfig": { "defaults": { "custom": {}, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 4 }, "id": 248, "options": { "displayMode": "gradient", "orientation": "horizontal", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showUnfilled": true }, "pluginVersion": "7.2.0", "repeat": "step_for_campaign_num", "repeatDirection": "v", "targets": [ { "expr": "sum(step_campaign_num_bucket:sum{kubernetes_namespace=\"ms-choices\",platform=~\"$env\",region=~\"$region\",inventory_source=~\"$inventory_source\",source_type=~\"$source_type\",step=~\"$step_for_campaign_num\",stage=\"out\"}) by (le)", "format": "heatmap", "interval": "", "legendFormat": "{{le}}", "refId": "A" } ], "title": "$step_for_campaign_num OUT", "type": "bargauge" } ], "title": "Campaigns in steps", "type": "row" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 29 }, "id": 299, "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 0, "y": 6 }, "hiddenSeries": false, "id": 29, "legend": { "alignAsTable": true, "avg": false, "current": false, "hideEmpty": false, "hideZero": false, "max": false, "min": false, "rightSide": true, "show": true, "sideWidth": 300, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.1", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(split_campaign_creative_total:rate2m{platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=~\"$inventory_source\",source_type=~\"$source_type\"}) by (type)", "hide": false, "interval": "", "legendFormat": "{{type}}", "refId": "G" } ], "thresholds": [], "timeRegions": [], "title": "campaign creative creation", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:84", "format": "none", "label": "", "logBase": 1, "show": true }, { "$$hashKey": "object:85", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "description": "", "fieldConfig": { "defaults": { "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 9, "w": 6, "x": 12, "y": 6 }, "id": 301, "options": { "displayMode": "gradient", "orientation": "horizontal", "reduceOptions": { "calcs": [ "mean" ], "fields": "", "values": false }, "showUnfilled": true, "text": {} }, "pluginVersion": "8.3.1", "repeatDirection": "v", "targets": [ { "expr": "sum(creative_count_total_bucket:sum{kubernetes_namespace=\"ms-choices\",platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=~\"$inventory_source\",source_type=~\"$source_type\",format=\"header_ad\"}) by (le)", "format": "heatmap", "interval": "", "legendFormat": "{{le}}", "refId": "C" } ], "title": "creative candidates: header_ad", "type": "bargauge" }, { "description": "", "fieldConfig": { "defaults": { "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 9, "w": 6, "x": 18, "y": 6 }, "id": 273, "options": { "displayMode": "gradient", "orientation": "horizontal", "reduceOptions": { "calcs": [ "mean" ], "fields": "", "values": false }, "showUnfilled": true, "text": {} }, "pluginVersion": "8.3.1", "repeat": "format", "repeatDirection": "v", "targets": [ { "expr": "sum(creative_count_total_bucket:sum{kubernetes_namespace=\"ms-choices\",platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=~\"$inventory_source\",source_type=~\"$source_type\",format=\"overlay_thumbnail\"}) by (le)", "format": "heatmap", "interval": "", "legendFormat": "{{le}}", "refId": "C" } ], "title": "creative candidates: overlay_thumbnail", "type": "bargauge" } ], "title": "Creatives", "type": "row" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 30 }, "id": 99, "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 7 }, "hiddenSeries": false, "id": 100, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.1", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.95, sum(rate(request_normalization_duration_seconds_bucket{kubernetes_pod_name=~\"ms-choices.*\",platform=~\"$env\",region=~\"$region\",stack=~\"$stack\"}[10m])) by (le))", "interval": "", "legendFormat": "total duration", "refId": "A" }, { "expr": "histogram_quantile(0.95, sum(rate(request_normalization_proto_read_bucket{kubernetes_pod_name=~\"ms-choices.*\",platform=~\"$env\",region=~\"$region\",stack=~\"$stack\"}[10m])) by (le))", "interval": "", "legendFormat": "proto read duration", "refId": "B" }, { "expr": "histogram_quantile(0.95, sum(rate(request_normalization_adapt_bucket{kubernetes_pod_name=~\"ms-choices-.*\",platform=~\"$env\",region=~\"$region\",stack=~\"$stack\"}[10m])) by (le))", "interval": "", "legendFormat": "adapt request duration", "refId": "C" }, { "expr": "histogram_quantile(0.95, sum(rate(normalization_init_bucket{kubernetes_pod_name=~\"ms-choices-.*\",platform=~\"$env\",region=~\"$region\",stack=~\"$stack\"}[10m])) by (le))", "interval": "", "legendFormat": "adapt: init", "refId": "D" }, { "expr": "histogram_quantile(0.95, sum(rate(normalization_extract_lituus_bucket{kubernetes_pod_name=~\"ms-choices-.*\",platform=~\"$env\",region=~\"$region\",stack=~\"$stack\"}[10m])) by (le))", "interval": "", "legendFormat": "adapt: extract lituus", "refId": "E" } ], "thresholds": [], "timeRegions": [], "title": "Request normalization duration", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:161", "format": "s", "logBase": 1, "show": true }, { "$$hashKey": "object:162", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 7 }, "hiddenSeries": false, "id": 243, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.1", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.95, sum(rate(prepare_response_duration_seconds_bucket{kubernetes_pod_name=~\"ms-choices.*\",platform=~\"$env\",region=~\"$region\",stack=~\"$stack\"}[10m])) by (le))", "interval": "", "legendFormat": "total duration", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "prepare response duration", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:161", "format": "s", "logBase": 1, "show": true }, { "$$hashKey": "object:162", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "alert": { "alertRuleTags": {}, "conditions": [ { "evaluator": { "params": [ 5 ], "type": "gt" }, "operator": { "type": "and" }, "query": { "params": [ "B", "5m", "now" ] }, "reducer": { "params": [], "type": "avg" }, "type": "query" } ], "executionErrorState": "alerting", "for": "5m", "frequency": "1m", "handler": 1, "message": "Error in request normalisation.\nVerify logs in Datadog to ensure it related to decoding Beeswax bid request.\nhttps://app.datadoghq.eu/logs?agg_m=count&agg_t=count&cols=host%2Cservice%2C%40stack_trace%2C%40ogy_campaign&event&from_ts=1623939061076&index=main&live=true&messageDisplay=inline&query=service%3Ams-choices-ext+status%3Aerror+%40ogy_env%3Aprod&stream_sort=desc&to_ts=1623942661076\n\nPlease contact Beeswax: support@beeswax.com", "name": "Request normalization errors alert (ms-choices-ext)", "noDataState": "ok", "notifications": [ { "uid": "sTLSyNrZk" } ] }, "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 15 }, "hiddenSeries": false, "id": 97, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.1", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "expr": "sum(rate(request_normalization_duration_seconds_count{kubernetes_pod_name=~\"ms-choices.*\",platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",error=\"true\"}[5m]))", "interval": "", "legendFormat": "", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "exemplar": true, "expr": "sum(rate(request_normalization_duration_seconds_count{kubernetes_pod_name=~\"ms-choices-.*\",platform=\"prod\",error=\"true\"}[5m]))", "hide": true, "interval": "", "legendFormat": "", "refId": "B" } ], "thresholds": [ { "colorMode": "critical", "fill": true, "line": true, "op": "gt", "value": 5, "visible": true } ], "timeRegions": [], "title": "Request normalization errors", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:68", "format": "short", "logBase": 1, "show": true }, { "$$hashKey": "object:69", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } } ], "title": "Request Normalization", "type": "row" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 31 }, "id": 314, "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 0, "y": 8 }, "hiddenSeries": false, "id": 315, "legend": { "alignAsTable": true, "avg": false, "current": false, "hideEmpty": false, "hideZero": false, "max": false, "min": false, "rightSide": true, "show": true, "sideWidth": 300, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.1", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "exemplar": true, "expr": "sum(rate(get_composite_score_total{platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=~\"$inventory_source\",source_type=~\"$source_type\"}[2m])) by (strategy)", "hide": false, "interval": "", "legendFormat": "{{strategy}}", "refId": "G" } ], "thresholds": [], "timeRegions": [], "title": "get composite score /s", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:84", "format": "none", "label": "", "logBase": 1, "show": true }, { "$$hashKey": "object:85", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 12, "y": 8 }, "hiddenSeries": false, "id": 316, "legend": { "alignAsTable": true, "avg": false, "current": false, "hideEmpty": false, "hideZero": false, "max": false, "min": false, "rightSide": true, "show": true, "sideWidth": 300, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.1", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "exemplar": true, "expr": "sum(rate(sort_campaigns_total{platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=~\"$inventory_source\",source_type=~\"$source_type\"}[2m])) by (strategy)", "hide": false, "interval": "", "legendFormat": "{{strategy}}", "refId": "G" } ], "thresholds": [], "timeRegions": [], "title": "campaigns sort /s", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:84", "format": "none", "label": "", "logBase": 1, "show": true }, { "$$hashKey": "object:85", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } } ], "title": "Choices strategies", "type": "row" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 32 }, "id": 107, "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 17 }, "hiddenSeries": false, "id": 161, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.1", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(bidfloor_duration_seconds_bucket:rate2m{kubernetes_namespace=\"ms-choices\",platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=~\"$inventory_source\",source_type=~\"$source_type\"}) by (le)) * 1000", "interval": "", "legendFormat": "p 99", "refId": "A" }, { "expr": "histogram_quantile(0.95, sum(bidfloor_duration_seconds_bucket:rate2m{kubernetes_namespace=\"ms-choices\",platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=~\"$inventory_source\",source_type=~\"$source_type\"}) by (le)) * 1000", "hide": false, "interval": "", "legendFormat": "p 95", "refId": "B" }, { "expr": "histogram_quantile(0.9, sum(bidfloor_duration_seconds_bucket:rate2m{kubernetes_namespace=\"ms-choices\",platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=~\"$inventory_source\",source_type=~\"$source_type\"}) by (le)) * 1000", "hide": false, "interval": "", "legendFormat": "p 90", "refId": "C" }, { "expr": "histogram_quantile(0.5, sum(bidfloor_duration_seconds_bucket:rate2m{kubernetes_namespace=\"ms-choices\",platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=~\"$inventory_source\",source_type=~\"$source_type\"}) by (le)) * 1000", "interval": "", "legendFormat": "p 50", "refId": "D" } ], "thresholds": [], "timeRegions": [], "title": "bidfloor step duration", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "format": "ms", "logBase": 1, "show": true }, { "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 17 }, "hiddenSeries": false, "id": 162, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.1", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(bidfloor_duration_seconds_count:rate2m{kubernetes_namespace=\"ms-choices\",platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=~\"$inventory_source\",source_type=~\"$source_type\",error=\"true\"})", "hide": false, "interval": "", "legendFormat": "error", "refId": "A" }, { "expr": "sum(bidfloor_duration_seconds_count:rate2m{kubernetes_namespace=\"ms-choices\",platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=~\"$inventory_source\",source_type=~\"$source_type\",error=\"false\"})", "hide": false, "interval": "", "legendFormat": "success", "refId": "B" } ], "thresholds": [], "timeRegions": [], "title": "bidfloor executions", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "format": "short", "logBase": 1, "show": true }, { "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 25 }, "hiddenSeries": false, "id": 109, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.1", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(bidfloor_duration_seconds_count:rate2m{kubernetes_namespace=\"ms-choices\",platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=~\"$inventory_source\",source_type=~\"$source_type\"}) by (has_bidfloor)", "interval": "", "legendFormat": "has_bidfloor: {{has_bidfloor}}", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "bidfloor presence", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:530", "format": "none", "logBase": 1, "show": true }, { "$$hashKey": "object:531", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 25 }, "hiddenSeries": false, "id": 110, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.1", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(bidfloor_duration_seconds_count:rate2m{kubernetes_namespace=\"ms-choices\",platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=~\"$inventory_source\",source_type=~\"$source_type\",has_bidfloor=\"true\"}) / sum(bidfloor_duration_seconds_count:rate2m{kubernetes_namespace=\"ms-choices\",platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=~\"$inventory_source\",source_type=~\"$source_type\"})", "interval": "", "legendFormat": "has_bidfloor: {{has_bidfloor}}", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "bidfloor presence percentage", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "format": "percentunit", "logBase": 1, "show": true }, { "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } } ], "title": "Profiling bidfloor", "type": "row" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 33 }, "id": 61, "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "fieldConfig": { "defaults": { "custom": {}, "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 21 }, "hiddenSeries": false, "id": 96, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.2.0", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(external_blacklist_duration_seconds_bucket:rate2m{kubernetes_namespace=\"ms-choices\",platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=~\"$inventory_source\",source_type=~\"$source_type\"}) by (le))", "hide": false, "interval": "", "legendFormat": "p 99", "refId": "A" }, { "expr": "histogram_quantile(0.95, sum(external_blacklist_duration_seconds_bucket:rate2m{kubernetes_namespace=\"ms-choices\",platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=~\"$inventory_source\",source_type=~\"$source_type\"}) by (le))", "interval": "", "legendFormat": "p 95", "refId": "B" } ], "thresholds": [], "timeRegions": [], "title": "External blacklist step duration", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "format": "s", "logBase": 1, "show": true }, { "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "fieldConfig": { "defaults": { "custom": {}, "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 21 }, "hiddenSeries": false, "id": 63, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.2.0", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(blacklist_external_reload_duration_seconds_count:rate2m{kubernetes_namespace=\"ms-choices\",platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=~\"$inventory_source\",source_type=~\"$source_type\",error=\"false\"})", "interval": "", "legendFormat": "success", "refId": "A" }, { "expr": "sum(blacklist_external_reload_duration_seconds_count:rate2m{kubernetes_namespace=\"ms-choices\",platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=~\"$inventory_source\",source_type=~\"$source_type\",error=\"true\"})", "interval": "", "legendFormat": "error", "refId": "B" } ], "thresholds": [], "timeRegions": [], "title": "External blacklist reloads", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "format": "short", "logBase": 1, "min": "0", "show": true }, { "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "fieldConfig": { "defaults": { "custom": {}, "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 29 }, "hiddenSeries": false, "id": 62, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.2.0", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(blacklist_external_reload_duration_seconds_bucket:rate2m{kubernetes_namespace=\"ms-choices\",platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=~\"$inventory_source\",source_type=~\"$source_type\"}) by (le))", "hide": false, "interval": "", "legendFormat": "p 99", "refId": "A" }, { "expr": "histogram_quantile(0.95, sum(blacklist_external_reload_duration_seconds_bucket:rate2m{kubernetes_namespace=\"ms-choices\",platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=~\"$inventory_source\",source_type=~\"$source_type\"}) by (le))", "interval": "", "legendFormat": "p 95", "refId": "B" }, { "expr": "histogram_quantile(0.5, sum(blacklist_external_reload_duration_seconds_bucket:rate2m{kubernetes_namespace=\"ms-choices\",platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=~\"$inventory_source\",source_type=~\"$source_type\"}) by (le))", "interval": "", "legendFormat": "p 50", "refId": "C" } ], "thresholds": [], "timeRegions": [], "title": "External blacklist reload duration", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "format": "s", "logBase": 1, "show": true }, { "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } } ], "title": "Profiling external blacklist (external only)", "type": "row" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 34 }, "id": 224, "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 68 }, "hiddenSeries": false, "id": 228, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.1", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(speed_threshold_duration_seconds_bucket:rate2m{kubernetes_namespace=\"ms-choices\",platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=~\"$inventory_source\",source_type=~\"$source_type\"}) by (le)) * 1000", "interval": "", "legendFormat": "p99", "refId": "A" }, { "expr": "histogram_quantile(0.95, sum(speed_threshold_duration_seconds_bucket:rate2m{kubernetes_namespace=\"ms-choices\",platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=~\"$inventory_source\",source_type=~\"$source_type\"}) by (le)) * 1000", "interval": "", "legendFormat": "p95", "refId": "B" }, { "expr": "histogram_quantile(0.90, sum(speed_threshold_duration_seconds_bucket:rate2m{kubernetes_namespace=\"ms-choices\",platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=~\"$inventory_source\",source_type=~\"$source_type\"}) by (le)) * 1000", "interval": "", "legendFormat": "p90", "refId": "C" }, { "expr": "histogram_quantile(0.50, sum(speed_threshold_duration_seconds_bucket:rate2m{kubernetes_namespace=\"ms-choices\",platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=~\"$inventory_source\",source_type=~\"$source_type\"}) by (le)) * 1000", "interval": "", "legendFormat": "p50", "refId": "D" } ], "thresholds": [], "timeRegions": [], "title": "Speed-threshold step duration", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:355", "format": "short", "logBase": 1, "show": true }, { "$$hashKey": "object:356", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 68 }, "hiddenSeries": false, "id": 226, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": true, "pluginVersion": "8.3.1", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(speed_threshold_duration_seconds_count:rate2m{kubernetes_namespace=\"ms-choices\",platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=~\"$inventory_source\",source_type=~\"$source_type\",error=\"true\"})", "interval": "", "legendFormat": "failures", "refId": "A" }, { "expr": "sum(speed_threshold_duration_seconds_count:rate2m{kubernetes_namespace=\"ms-choices\",platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=~\"$inventory_source\",source_type=~\"$source_type\",error=\"false\"})", "interval": "", "legendFormat": "success", "refId": "B" } ], "thresholds": [], "timeRegions": [], "title": "Speed-threshold executions", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:167", "format": "short", "logBase": 1, "show": true }, { "$$hashKey": "object:168", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } } ], "title": "Profiling Speed Threshold", "type": "row" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 35 }, "id": 113, "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 12 }, "hiddenSeries": false, "id": 115, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.1", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.95, sum(rate(context_hashmap_reload_duration_seconds_bucket{kubernetes_namespace=\"ms-choices\",platform=~\"$env\",region=~\"$region\",stack=~\"$stack\"}[10m])) by (le))", "interval": "", "legendFormat": "", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Reload time", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:746", "format": "s", "logBase": 1, "show": true }, { "$$hashKey": "object:747", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 12 }, "hiddenSeries": false, "id": 117, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.1", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(context_hashmap_reload_duration_seconds_count{kubernetes_namespace=\"ms-choices\",platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",error=\"true\"}[5m]))", "interval": "", "legendFormat": "", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Reload error", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "format": "short", "logBase": 1, "show": true }, { "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 20 }, "hiddenSeries": false, "id": 140, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.1", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.90, sum(audience_hashmap_refresh_bucket:rate20m{kubernetes_namespace=\"ms-choices\",platform=~\"$env\",region=~\"$region\",stack=~\"$stack\"}) by (le))", "interval": "", "legendFormat": "percentile 90", "refId": "A" }, { "expr": "histogram_quantile(0.5, sum(audience_hashmap_refresh_bucket:rate20m{kubernetes_namespace=\"ms-choices\",platform=~\"$env\",region=~\"$region\",stack=~\"$stack\"}) by (le))", "interval": "", "legendFormat": "percentile 50", "refId": "B" } ], "thresholds": [], "timeRegions": [], "title": "Refresh time", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "format": "s", "logBase": 1, "show": true }, { "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 20 }, "hiddenSeries": false, "id": 129, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.1", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.95, sum(rate(context_internal_hashmap_duration_seconds_bucket{kubernetes_namespace=\"ms-choices\",platform=~\"$env\",region=~\"$region\",stack=~\"$stack\"}[10m])) by (le))", "interval": "", "legendFormat": "", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Internal Targeting get scores for campaigns time", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "format": "s", "logBase": 1, "show": true }, { "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "description": "", "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 28 }, "hiddenSeries": false, "id": 190, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": true, "pluginVersion": "8.3.1", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": true, "steppedLine": false, "targets": [ { "expr": "sum by (app_type) (rate(audience_hashmap_usage{kubernetes_namespace=\"ms-choices\",platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=~\"$inventory_source\",source_type=~\"$source_type\",app_type=~\".+\"}[10m]))", "interval": "", "legendFormat": "{{app_type}}", "refId": "C" } ], "thresholds": [], "timeRegions": [], "title": "app_type per request", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "format": "none", "logBase": 1, "min": "0", "show": true }, { "format": "none", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 28 }, "hiddenSeries": false, "id": 120, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.1", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.95, sum(rate(context_hashmap_get_score_duration_seconds_bucket{kubernetes_namespace=\"ms-choices\",platform=~\"$env\",region=~\"$region\",stack=~\"$stack\"}[10m])) by (le))", "interval": "", "legendFormat": "", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Single get score time", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "format": "s", "logBase": 1, "show": true }, { "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "alert": { "alertRuleTags": {}, "conditions": [ { "evaluator": { "params": [ 0.9 ], "type": "lt" }, "operator": { "type": "and" }, "query": { "params": [ "B", "5m", "now" ] }, "reducer": { "params": [], "type": "avg" }, "type": "query" } ], "executionErrorState": "alerting", "for": "5m", "frequency": "1m", "handler": 1, "message": "Acc-Rate Score coverage alert", "name": "Acc-Rate Score coverage alert", "noDataState": "no_data", "notifications": [ { "uid": "sTLSyNrZk" } ] }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 5, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 36 }, "id": 332, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "single" } }, "pluginVersion": "8.3.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "exemplar": true, "expr": "sum(rate(score_presence_total{kubernetes_namespace=\"ms-choices\",platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",present=\"true\",score_name=\"context_score\"}[5m])) / sum(rate(score_presence_total{kubernetes_namespace=\"ms-choices\",platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",score_name=\"context_score\"}[5m]))", "interval": "", "legendFormat": "Score present %", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "exemplar": true, "expr": "sum(rate(score_presence_total{kubernetes_namespace=\"ms-choices\",platform=~\"prod\",present=\"true\",score_name=\"context_score\"}[5m])) / sum(rate(score_presence_total{kubernetes_namespace=\"ms-choices\",platform=~\"prod\",score_name=\"context_score\"}[5m]))", "hide": true, "interval": "", "legendFormat": "", "refId": "B" } ], "thresholds": [ { "colorMode": "critical", "op": "lt", "value": 0.9, "visible": true } ], "title": "Score coverage", "type": "timeseries" }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 36 }, "hiddenSeries": false, "id": 158, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.1", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "1 - \n(\nsum by (region) (rate(cta_hashmap_campain_ssp_fallback{kubernetes_namespace=\"ms-choices\",platform=~\"$env\",region=~\"$region\",stack=~\"$stack\"}[10m])) \n)\n/ \n(\nsum by (region) (rate(cta_hashmap_get_score_count{kubernetes_namespace=\"ms-choices\",platform=~\"$env\",region=~\"$region\",stack=~\"$stack\"}[10m]))\n)", "interval": "", "legendFormat": "", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "coverage", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "format": "percentunit", "logBase": 1, "show": true }, { "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } } ], "repeat": "env", "title": "Profiling Context hashmap", "type": "row" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 36 }, "id": 138, "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 69 }, "hiddenSeries": false, "id": 142, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.1", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.95, sum(rate(audience_hashmap_get_score_bucket{kubernetes_namespace=\"ms-choices\",platform=~\"$env\",region=~\"$region\",stack=~\"$stack\"}[10m])) by (le))", "interval": "", "legendFormat": "percentile 95", "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(audience_hashmap_get_score_bucket{kubernetes_namespace=\"ms-choices\",platform=~\"$env\",region=~\"$region\",stack=~\"$stack\"}[10m])) by (le))", "interval": "", "legendFormat": "percentile 50", "refId": "B" } ], "thresholds": [], "timeRegions": [], "title": "Get Score time", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "format": "s", "logBase": 1, "show": true }, { "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "description": "", "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 69 }, "hiddenSeries": false, "id": 191, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.1", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum by (score_type) (rate(audience_hashmap_usage{kubernetes_namespace=\"ms-choices\",platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",score_type=~\".+\"}[10m])) / ignoring(score_type) group_left\nsum(rate(audience_hashmap_get_score_count{kubernetes_namespace=\"ms-choices\",platform=~\"$env\",region=~\"$region\",stack=~\"$stack\"}[10m]))", "interval": "", "legendFormat": "{{score_type}}", "refId": "C" } ], "thresholds": [], "timeRegions": [], "title": "score_type per campaign", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "format": "percentunit", "logBase": 1, "max": "1", "min": "0", "show": true }, { "format": "none", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "description": "", "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 77 }, "hiddenSeries": false, "id": 146, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.1", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "max(audience_hashmap_similarities_size{kubernetes_namespace=\"ms-choices\",platform=~\"$env\",region=~\"$region\",stack=~\"$stack\"})", "interval": "", "legendFormat": "number of similarities", "refId": "C" }, { "expr": "", "interval": "", "legendFormat": "calls with non-briefed campaigns ({{region}})", "refId": "B" } ], "thresholds": [], "timeRegions": [], "title": "Number of similarities", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "format": "none", "logBase": 1, "show": true }, { "format": "none", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 77 }, "hiddenSeries": false, "id": 141, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.1", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.95, sum(rate(audience_hashmap_add_scores_bucket{kubernetes_namespace=\"ms-choices\",platform=~\"$env\",region=~\"$region\",stack=~\"$stack\"}[10m])) by (le))", "interval": "", "legendFormat": "percentile 95", "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(audience_hashmap_add_scores_bucket{kubernetes_namespace=\"ms-choices\",platform=~\"$env\",region=~\"$region\",stack=~\"$stack\"}[10m])) by (le))", "interval": "", "legendFormat": "percentile 50", "refId": "B" } ], "thresholds": [], "timeRegions": [], "title": "Add Scores time", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "format": "s", "logBase": 1, "show": true }, { "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 85 }, "hiddenSeries": false, "id": 333, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.1", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "exemplar": true, "expr": "sum(rate(score_presence_total{kubernetes_namespace=\"ms-choices\",platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",present=\"true\",score_name=\"audience_score\"}[5m])) / sum(rate(score_presence_total{kubernetes_namespace=\"ms-choices\",platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",score_name=\"audience_score\"}[5m]))", "interval": "", "legendFormat": "Score present %", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Score coverage", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:104", "format": "percentunit", "logBase": 1, "show": true }, { "$$hashKey": "object:105", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } } ], "repeat": "env", "title": "Profiling Audience hashmap", "type": "row" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 37 }, "id": 152, "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 46 }, "hiddenSeries": false, "id": 150, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.1", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.90, sum(rate(cta_hashmap_refresh_duration_seconds_bucket{kubernetes_namespace=\"ms-choices\",platform=~\"$env\",region=~\"$region\",stack=~\"$stack\"}[20m])) by (le))", "interval": "", "legendFormat": "percentile 90", "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(cta_hashmap_refresh_duration_seconds_bucket{kubernetes_namespace=\"ms-choices\",platform=~\"$env\",region=~\"$region\",stack=~\"$stack\"}[20m])) by (le))", "interval": "", "legendFormat": "percentile 50", "refId": "B" } ], "thresholds": [], "timeRegions": [], "title": "refresh time", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "format": "s", "logBase": 1, "show": true }, { "format": "s", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 46 }, "hiddenSeries": false, "id": 156, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.1", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.95, sum(rate(cta_hashmap_add_scores_bucket{kubernetes_namespace=\"ms-choices\",platform=~\"$env\",region=~\"$region\",stack=~\"$stack\"}[10m])) by (le))", "interval": "", "legendFormat": "percentile 95", "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(cta_hashmap_add_scores_bucket{kubernetes_namespace=\"ms-choices\",platform=~\"$env\",region=~\"$region\",stack=~\"$stack\"}[10m])) by (le))", "interval": "", "legendFormat": "percentile 50", "refId": "B" } ], "thresholds": [], "timeRegions": [], "title": "add scores", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "format": "s", "logBase": 1, "show": true }, { "format": "s", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 54 }, "hiddenSeries": false, "id": 154, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.1", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.95, sum(rate(cta_hashmap_get_score_bucket{kubernetes_namespace=\"ms-choices\",platform=~\"$env\",region=~\"$region\",stack=~\"$stack\"}[10m])) by (le))", "interval": "", "legendFormat": "percentile 95", "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(cta_hashmap_get_score_bucket{kubernetes_namespace=\"ms-choices\",platform=~\"$env\",region=~\"$region\",stack=~\"$stack\"}[10m])) by (le))", "interval": "", "legendFormat": "percentile 50", "refId": "B" } ], "thresholds": [], "timeRegions": [], "title": "get score", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "format": "s", "logBase": 1, "show": true }, { "format": "s", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "alert": { "alertRuleTags": {}, "conditions": [ { "evaluator": { "params": [ 0.4 ], "type": "lt" }, "operator": { "type": "and" }, "query": { "params": [ "B", "5m", "now" ] }, "reducer": { "params": [], "type": "avg" }, "type": "query" } ], "executionErrorState": "alerting", "for": "5m", "frequency": "1m", "handler": 1, "message": "CTA Score coverage alert", "name": "CTA Score coverage alert", "noDataState": "no_data", "notifications": [ { "uid": "sTLSyNrZk" } ] }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 62 }, "id": 334, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "single" } }, "pluginVersion": "8.3.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "exemplar": true, "expr": "sum(rate(score_presence_total{kubernetes_namespace=\"ms-choices\",platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",present=\"true\",score_name=\"cta_score\"}[5m])) / sum(rate(score_presence_total{kubernetes_namespace=\"ms-choices\",platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",score_name=\"cta_score\"}[5m]))", "interval": "", "legendFormat": "Score present %", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "exemplar": true, "expr": "sum(rate(score_presence_total{kubernetes_namespace=\"ms-choices\",platform=~\"prod\",present=\"true\",score_name=\"cta_score\"}[5m])) / sum(rate(score_presence_total{kubernetes_namespace=\"ms-choices\",platform=~\"prod\",score_name=\"cta_score\"}[5m]))", "hide": true, "interval": "", "legendFormat": "", "refId": "B" } ], "thresholds": [ { "colorMode": "critical", "op": "lt", "value": 0.4, "visible": true } ], "title": "Score coverage", "type": "timeseries" } ], "title": "Profiling CTA HashMap", "type": "row" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 38 }, "id": 88, "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 13 }, "hiddenSeries": false, "id": 163, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.1", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(dv_fraud_duration_seconds_bucket:rate2m{kubernetes_namespace=\"ms-choices\",platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=~\"$inventory_source\",source_type=~\"$source_type\"}) by (le)) * 1000", "interval": "", "legendFormat": "p 99", "refId": "A" }, { "expr": "histogram_quantile(0.95, sum(dv_fraud_duration_seconds_bucket:rate2m{kubernetes_namespace=\"ms-choices\",platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=~\"$inventory_source\",source_type=~\"$source_type\"}) by (le)) * 1000", "hide": false, "interval": "", "legendFormat": "p 95", "refId": "B" }, { "expr": "histogram_quantile(0.9, sum(dv_fraud_duration_seconds_bucket:rate2m{kubernetes_namespace=\"ms-choices\",platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=~\"$inventory_source\",source_type=~\"$source_type\"}) by (le)) * 1000", "hide": false, "interval": "", "legendFormat": "p 90", "refId": "C" }, { "expr": "histogram_quantile(0.5, sum(dv_fraud_duration_seconds_bucket:rate2m{kubernetes_namespace=\"ms-choices\",platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=~\"$inventory_source\",source_type=~\"$source_type\"}) by (le)) * 1000", "interval": "", "legendFormat": "p 50", "refId": "D" } ], "thresholds": [], "timeRegions": [], "title": "dvfraud step duration", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "format": "ms", "logBase": 1, "show": true }, { "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 13 }, "hiddenSeries": false, "id": 164, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.1", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(dv_fraud_duration_seconds_count:rate2m{kubernetes_namespace=\"ms-choices\",platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=~\"$inventory_source\",source_type=~\"$source_type\",error=\"true\"})", "interval": "", "legendFormat": "failures", "refId": "A" }, { "expr": "sum(dv_fraud_duration_seconds_count:rate2m{kubernetes_namespace=\"ms-choices\",platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=~\"$inventory_source\",source_type=~\"$source_type\",error=\"false\", is_fraud=\"true\"})", "interval": "", "legendFormat": "fraud", "refId": "B" }, { "expr": "sum(dv_fraud_duration_seconds_count:rate2m{kubernetes_namespace=\"ms-choices\",platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=~\"$inventory_source\",source_type=~\"$source_type\",error=\"false\",is_fraud=\"false\"})", "interval": "", "legendFormat": "ok", "refId": "C" }, { "expr": "sum(dv_fraud_duration_seconds_count:rate2m{kubernetes_namespace=\"ms-choices\",platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=~\"$inventory_source\",source_type=~\"$source_type\",error=\"false\",bypass=\"true\"})", "interval": "", "legendFormat": "bypass", "refId": "D" } ], "thresholds": [], "timeRegions": [], "title": "DV fraud rate", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "format": "short", "logBase": 1, "show": true }, { "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 21 }, "hiddenSeries": false, "id": 93, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.1", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "(sum by (fraud_type,source_type) (dv_fraud_duration_seconds_count:rate2m{kubernetes_namespace=\"ms-choices\",platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=~\"$inventory_source\",source_type=~\"$source_type\",is_fraud=\"true\"})) / ignoring(fraud_type,source_type) group_left sum(dv_fraud_duration_seconds_count:rate2m{kubernetes_namespace=\"ms-choices\",platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=~\"$inventory_source\",source_type=~\"$source_type\",is_fraud!=\"\"})", "format": "time_series", "hide": false, "interval": "", "legendFormat": "device: {{source_type}}, fraud_type: {{fraud_type}}", "refId": "B" }, { "expr": "sum(dv_fraud_duration_seconds_count:rate2m{kubernetes_namespace=\"ms-choices\",platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=~\"$inventory_source\",source_type=~\"$source_type\",is_fraud=\"true\"}) / sum(dv_fraud_duration_seconds_count:rate2m{kubernetes_namespace=\"ms-choices\",platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=~\"$inventory_source\",source_type=~\"$source_type\",is_fraud!=\"\"})", "hide": false, "interval": "", "legendFormat": "global", "refId": "C" } ], "thresholds": [], "timeRegions": [], "title": "Fraud types", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "decimals": 2, "format": "percentunit", "logBase": 1, "min": "0", "show": true }, { "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 21 }, "hiddenSeries": false, "id": 305, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.1", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "exemplar": true, "expr": "sum(rate(dv_strong_missing_user_agent{kubernetes_namespace=\"ms-choices\",platform=~\"$env\",region=~\"$region\",stack=~\"$stack\"}[5m])) by (inventory_source, source_type, sdk_version)", "hide": true, "interval": "", "legendFormat": "{{inventory_source}} | {{source_type}} | {{sdk_version}}", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "exemplar": true, "expr": "sum(dv_strong_missing_user_agent{kubernetes_namespace=\"ms-choices\",platform=~\"$env\",region=~\"$region\",stack=~\"$stack\"}) by (inventory_source, source_type, sdk_version)", "hide": false, "interval": "", "legendFormat": "{{inventory_source}} | {{source_type}} | {{sdk_version}}", "refId": "B" } ], "thresholds": [], "timeRegions": [], "title": "missing user agent", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:168", "format": "short", "logBase": 1, "show": true }, { "$$hashKey": "object:169", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 0, "y": 29 }, "hiddenSeries": false, "id": 94, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.1", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(redis_request_duration_seconds_count{kubernetes_namespace=~\"ms-choices\",platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",client_name!=\"\"}[10m])) by (client_name)", "interval": "", "legendFormat": "{{client_name}}", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Redis error count", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "format": "short", "logBase": 1, "min": "0", "show": true }, { "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 29 }, "hiddenSeries": false, "id": 89, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.1", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.5, sum(dv_fraud_duration_seconds_bucket:rate2m{kubernetes_namespace=\"ms-choices\",platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=~\"$inventory_source\",source_type=~\"$source_type\",is_fraud=\"false\"}) by (le))", "hide": false, "interval": "", "legendFormat": "ok", "refId": "A" }, { "expr": "histogram_quantile(0.5, sum(dv_fraud_duration_seconds_bucket:rate2m{kubernetes_namespace=\"ms-choices\",platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=~\"$inventory_source\",source_type=~\"$source_type\",is_fraud=\"true\"}) by (le, fraud_type))", "hide": false, "interval": "", "legendFormat": "fraud: {{fraud_type}}", "refId": "B" } ], "thresholds": [], "timeRegions": [], "title": "Fraud type duration (p95)", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "format": "s", "logBase": 1, "show": true }, { "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 12, "y": 37 }, "hiddenSeries": false, "id": 95, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.1", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.95, sum(rate(redis_request_duration_seconds_bucket{kubernetes_namespace=\"ms-choices\", platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",command=\"hexists\"}[10m])) by (le, client_name))", "interval": "", "legendFormat": "{{client_name}} - p95", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Redis HEXISTS duration", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:445", "format": "s", "logBase": 1, "show": true }, { "$$hashKey": "object:446", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "alert": { "alertRuleTags": {}, "conditions": [ { "evaluator": { "params": [ 100 ], "type": "gt" }, "operator": { "type": "and" }, "query": { "params": [ "A", "5m", "now" ] }, "reducer": { "params": [], "type": "avg" }, "type": "query" } ], "executionErrorState": "keep_state", "for": "5m", "frequency": "1m", "handler": 1, "message": "Redis errors in ms-choices for DV", "name": "DV Redis errors", "noDataState": "ok", "notifications": [ { "uid": "sTLSyNrZk" } ] }, "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 38 }, "hiddenSeries": false, "id": 104, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.1", "pointradius": 2, "points": false, "renderer": "flot", "repeat": "region", "repeatDirection": "v", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(redis_request_duration_seconds_count{platform=~\"prod\",client_name!=\"\",error=\"true\"}[10m])) by (client_name)", "interval": "", "legendFormat": "[PROD] Redis Errors {{client_name}}", "refId": "A" } ], "thresholds": [ { "colorMode": "critical", "fill": true, "line": true, "op": "gt", "value": 100, "visible": true } ], "timeRegions": [], "title": "DV Redis Alerting", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "format": "short", "logBase": 1, "min": "0", "show": true }, { "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } } ], "title": "Profiling DV fraud", "type": "row" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 39 }, "id": 318, "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 40 }, "hiddenSeries": false, "id": 319, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.1", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "exemplar": true, "expr": "histogram_quantile(0.5, sum(rate(speed_get_thresholds_duration_seconds_bucket{kubernetes_pod_name=~\"ms-choices-.*\",region=~\"$region\"}[60s])) by (le)) * 1000", "interval": "", "legendFormat": "0.5p", "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "exemplar": true, "expr": "histogram_quantile(0.9, sum(rate(speed_get_thresholds_duration_seconds_bucket{kubernetes_pod_name=~\"ms-choices-.*\",region=~\"$region\"}[60s])) by (le)) * 1000", "interval": "", "legendFormat": "0.9p", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "exemplar": true, "expr": "histogram_quantile(0.99, sum(rate(speed_get_thresholds_duration_seconds_bucket{kubernetes_pod_name=~\"ms-choices-.*\",region=~\"$region\"}[60s])) by (le)) * 1000", "interval": "", "legendFormat": "0.99p", "refId": "C" } ], "thresholds": [], "timeRegions": [], "title": "ms-speed response time - threshold", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:144", "format": "none", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:145", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 40 }, "hiddenSeries": false, "id": 320, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.1", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "exemplar": true, "expr": "histogram_quantile(0.5, sum(rate(speed_get_prio_scores_duration_seconds_bucket{kubernetes_pod_name=~\"ms-choices-.*\",region=~\"$region\"}[60s])) by (le)) * 1000", "interval": "", "legendFormat": "0.5p", "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "exemplar": true, "expr": "histogram_quantile(0.9, sum(rate(speed_get_prio_scores_duration_seconds_bucket{kubernetes_pod_name=~\"ms-choices-.*\",region=~\"$region\"}[60s])) by (le)) * 1000", "interval": "", "legendFormat": "0.9p", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "exemplar": true, "expr": "histogram_quantile(0.99, sum(rate(speed_get_prio_scores_duration_seconds_bucket{kubernetes_pod_name=~\"ms-choices-.*\",region=~\"$region\"}[60s])) by (le)) * 1000", "interval": "", "legendFormat": "0.99p", "refId": "C" } ], "thresholds": [], "timeRegions": [], "title": "ms-speed response time - priority score", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:144", "format": "none", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:145", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 47 }, "hiddenSeries": false, "id": 324, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.1", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "exemplar": true, "expr": "sum(rate(speed_get_thresholds_duration_seconds_count{kubernetes_namespace=~\"ms-choices\",stack=~\"$stack\",error=\"false\"}[2m]))", "interval": "", "legendFormat": "threshold - success", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "exemplar": true, "expr": "sum(rate(speed_get_thresholds_duration_seconds_count{kubernetes_namespace=~\"ms-choices\",stack=~\"$stack\",error=\"true\"}[2m]))", "hide": false, "interval": "", "legendFormat": "threshold - error", "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "exemplar": true, "expr": "sum(rate(speed_get_prio_scores_duration_seconds_count{kubernetes_namespace=~\"ms-choices\",stack=~\"$stack\",error=\"false\"}[2m]))", "hide": false, "interval": "", "legendFormat": "prio score - success", "refId": "C" }, { "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "exemplar": true, "expr": "sum(rate(speed_get_prio_scores_duration_seconds_count{kubernetes_namespace=~\"ms-choices\",stack=~\"$stack\",error=\"true\"}[2m]))", "hide": false, "interval": "", "legendFormat": "prio score - error", "refId": "D" } ], "thresholds": [], "timeRegions": [], "title": "ms-speed executions", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:560", "format": "short", "logBase": 1, "show": true }, { "$$hashKey": "object:561", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": { "INTERNAL error": "red" }, "bars": false, "dashLength": 10, "dashes": false, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 47 }, "hiddenSeries": false, "id": 322, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.1", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "exemplar": true, "expr": "max(speed_cache_size_total{platform=~\"$env\",region=~\"$region\",stack=~\"$stack\"}) by (score_type, isolated)", "interval": "", "legendFormat": "{{ score_type }} - isolated: {{ isolated }}", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Cache size", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:318", "format": "none", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:319", "format": "none", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": { "INTERNAL error": "red" }, "bars": false, "dashLength": 10, "dashes": false, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 55 }, "hiddenSeries": false, "id": 321, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.1", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": true, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "exemplar": true, "expr": "sum(rate(speed_cache_requests_total{platform=~\"$env\",region=~\"$region\",stack=~\"$stack\", score_type=\"campaign_threshold\", response_type=\"miss\", isolated=\"false\"}[2m]))", "hide": false, "interval": "", "legendFormat": "miss", "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "exemplar": true, "expr": "sum(rate(speed_cache_requests_total{platform=~\"$env\",region=~\"$region\",stack=~\"$stack\", score_type=\"campaign_threshold\", response_type=\"stale\", isolated=\"false\"}[2m]))", "hide": false, "interval": "", "legendFormat": "stale", "refId": "C" }, { "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "exemplar": true, "expr": "sum(rate(speed_cache_requests_total{platform=~\"$env\",region=~\"$region\",stack=~\"$stack\", score_type=\"campaign_threshold\", response_type=\"hit\", isolated=\"false\"}[2m]))", "interval": "", "legendFormat": "hit", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Cache requests - threshold", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:318", "format": "none", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:319", "format": "none", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": { "INTERNAL error": "red" }, "bars": false, "dashLength": 10, "dashes": false, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 55 }, "hiddenSeries": false, "id": 326, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.1", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": true, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "exemplar": true, "expr": "sum(rate(speed_cache_requests_total{platform=~\"$env\",region=~\"$region\",stack=~\"$stack\", score_type=\"campaign_threshold\", response_type=\"miss\", isolated=\"true\"}[2m]))", "hide": false, "interval": "", "legendFormat": "miss", "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "exemplar": true, "expr": "sum(rate(speed_cache_requests_total{platform=~\"$env\",region=~\"$region\",stack=~\"$stack\", score_type=\"campaign_threshold\", response_type=\"stale\", isolated=\"true\"}[2m]))", "hide": false, "interval": "", "legendFormat": "stale", "refId": "C" }, { "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "exemplar": true, "expr": "sum(rate(speed_cache_requests_total{platform=~\"$env\",region=~\"$region\",stack=~\"$stack\", score_type=\"campaign_threshold\", response_type=\"hit\", isolated=\"true\"}[2m]))", "interval": "", "legendFormat": "hit", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Cache requests - threshold isolated", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:318", "format": "none", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:319", "format": "none", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": { "INTERNAL error": "red" }, "bars": false, "dashLength": 10, "dashes": false, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 63 }, "hiddenSeries": false, "id": 325, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.1", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": true, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "exemplar": true, "expr": "sum(rate(speed_cache_requests_total{platform=~\"$env\",region=~\"$region\",stack=~\"$stack\", score_type=\"supply_prio_score\", response_type=\"miss\", isolated=\"false\"}[2m]))", "hide": false, "interval": "", "legendFormat": "miss", "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "exemplar": true, "expr": "sum(rate(speed_cache_requests_total{platform=~\"$env\",region=~\"$region\",stack=~\"$stack\", score_type=\"supply_prio_score\", response_type=\"stale\", isolated=\"false\"}[2m]))", "hide": false, "interval": "", "legendFormat": "stale", "refId": "C" }, { "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "exemplar": true, "expr": "sum(rate(speed_cache_requests_total{platform=~\"$env\",region=~\"$region\",stack=~\"$stack\", score_type=\"supply_prio_score\", response_type=\"hit\", isolated=\"false\"}[2m]))", "interval": "", "legendFormat": "hit", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Cache requests - priority score", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:318", "format": "none", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:319", "format": "none", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": { "INTERNAL error": "red" }, "bars": false, "dashLength": 10, "dashes": false, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 63 }, "hiddenSeries": false, "id": 327, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.1", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": true, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "exemplar": true, "expr": "sum(rate(speed_cache_requests_total{platform=~\"$env\",region=~\"$region\",stack=~\"$stack\", score_type=\"supply_prio_score\", response_type=\"miss\", isolated=\"true\"}[2m]))", "hide": false, "interval": "", "legendFormat": "miss", "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "exemplar": true, "expr": "sum(rate(speed_cache_requests_total{platform=~\"$env\",region=~\"$region\",stack=~\"$stack\", score_type=\"supply_prio_score\", response_type=\"stale\", isolated=\"true\"}[2m]))", "hide": false, "interval": "", "legendFormat": "stale", "refId": "C" }, { "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "exemplar": true, "expr": "sum(rate(speed_cache_requests_total{platform=~\"$env\",region=~\"$region\",stack=~\"$stack\", score_type=\"supply_prio_score\", response_type=\"hit\", isolated=\"true\"}[2m]))", "interval": "", "legendFormat": "hit", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Cache requests - priority score isolated", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:318", "format": "none", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:319", "format": "none", "logBase": 1, "show": true } ], "yaxis": { "align": false } } ], "title": "Profiling Speed", "type": "row" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 40 }, "id": 51, "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 82 }, "hiddenSeries": false, "id": 159, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.1", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(yield_duration_seconds_bucket:rate2m{kubernetes_namespace=~\"ms-choices\",inventory_source=~\"$inventory_source\",source_type=~\"$source_type\",platform=~\"$env\",region=~\"$region\",stack=~\"$stack\"}) by (le)) * 1000", "interval": "", "legendFormat": "p 99", "refId": "A" }, { "expr": "histogram_quantile(0.95, sum(yield_duration_seconds_bucket:rate2m{kubernetes_namespace=~\"ms-choices\",inventory_source=~\"$inventory_source\",source_type=~\"$source_type\",platform=~\"$env\",region=~\"$region\",stack=~\"$stack\"}) by (le)) * 1000", "hide": false, "interval": "", "legendFormat": "p 95", "refId": "B" }, { "expr": "histogram_quantile(0.9, sum(yield_duration_seconds_bucket:rate2m{kubernetes_namespace=~\"ms-choices\",inventory_source=~\"$inventory_source\",source_type=~\"$source_type\",platform=~\"$env\",region=~\"$region\",stack=~\"$stack\"}) by (le)) * 1000", "hide": false, "interval": "", "legendFormat": "p 90", "refId": "C" }, { "expr": "histogram_quantile(0.5, sum(yield_duration_seconds_bucket:rate2m{kubernetes_namespace=~\"ms-choices\",inventory_source=~\"$inventory_source\",source_type=~\"$source_type\",platform=~\"$env\",region=~\"$region\",stack=~\"$stack\"}) by (le)) * 1000", "interval": "", "legendFormat": "p 50", "refId": "D" } ], "thresholds": [], "timeRegions": [], "title": "yield step duration", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "format": "ms", "logBase": 1, "show": true }, { "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 82 }, "hiddenSeries": false, "id": 160, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.1", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(yield_duration_seconds_count:rate2m{kubernetes_namespace=~\"ms-choices\",inventory_source=~\"$inventory_source\",source_type=~\"$source_type\",platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",error=\"true\"})", "interval": "", "legendFormat": "error", "refId": "A" }, { "expr": "sum(yield_duration_seconds_count:rate2m{kubernetes_namespace=~\"ms-choices\",inventory_source=~\"$inventory_source\",source_type=~\"$source_type\",platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",error=\"false\"})", "interval": "", "legendFormat": "success", "refId": "B" } ], "thresholds": [], "timeRegions": [], "title": "yield executions", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "format": "short", "logBase": 1, "show": true }, { "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 90 }, "hiddenSeries": false, "id": 134, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.1", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.5, sum(rate(ms_yield_response_time_duration_seconds_bucket{kubernetes_pod_name=~\"ms-choices-.*\",inventory_source=~\"$inventory_source\",source_type=~\"$source_type\",platform=~\"$env\",region=~\"$region\"}[60s])) by (le)) * 1000", "interval": "", "legendFormat": "0.5p", "refId": "B" }, { "expr": "histogram_quantile(0.9, sum(rate(ms_yield_response_time_duration_seconds_bucket{kubernetes_pod_name=~\"ms-choices-.*\",inventory_source=~\"$inventory_source\",source_type=~\"$source_type\",platform=~\"$env\",region=~\"$region\"}[60s])) by (le)) * 1000", "interval": "", "legendFormat": "0.9p", "refId": "A" }, { "expr": "histogram_quantile(0.99, sum(rate(ms_yield_response_time_duration_seconds_bucket{kubernetes_pod_name=~\"ms-choices-.*\",inventory_source=~\"$inventory_source\",source_type=~\"$source_type\",platform=~\"$env\",region=~\"$region\"}[60s])) by (le)) * 1000", "interval": "", "legendFormat": "0.99p", "refId": "C" } ], "thresholds": [], "timeRegions": [], "title": "ms-yield response", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "format": "none", "logBase": 1, "min": "0", "show": true }, { "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 90 }, "hiddenSeries": false, "id": 135, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.1", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(ms_yield_response_time_duration_seconds_count{kubernetes_pod_name=~\"ms-choices-.*\",inventory_source=~\"$inventory_source\",source_type=~\"$source_type\",platform=~\"$env\",region=~\"$region\",error_code=~\".+\"}[60s])) by (error_code)", "interval": "", "legendFormat": "{{error_code}}", "refId": "B" } ], "thresholds": [], "timeRegions": [], "title": "ms-yield errors", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "format": "none", "logBase": 1, "min": "0", "show": true }, { "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } } ], "title": "Profiling Yield (internal only)", "type": "row" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 41 }, "id": 55, "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 19 }, "hiddenSeries": false, "id": 165, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.1", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "exemplar": true, "expr": "histogram_quantile(0.95, sum(auction_calls_duration_seconds_bucket:rate2m{kubernetes_namespace=\"ms-choices\",platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=~\"$inventory_source\",source_type=~\"$source_type\"}) by (le, auction_type, has_aaid))", "interval": "", "legendFormat": "{{auction_type}}", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "ms-auction duration", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:54", "format": "ms", "logBase": 1, "show": true }, { "$$hashKey": "object:55", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 19 }, "hiddenSeries": false, "id": 166, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.1", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "exemplar": true, "expr": "sum(auction_calls_duration_seconds_count:rate2m{kubernetes_namespace=\"ms-choices\",platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=~\"$inventory_source\",source_type=~\"$source_type\",error=\"true\"}) by (auction_type, has_aaid)", "interval": "", "legendFormat": "auction_type: {{auction_type}}, has_aaid: {{has_aaid}} -> error", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "exemplar": true, "expr": "sum(auction_calls_duration_seconds_count:rate2m{kubernetes_namespace=\"ms-choices\",platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=~\"$inventory_source\",source_type=~\"$source_type\",error=\"false\"}) by (has_aaid)", "interval": "", "legendFormat": "auction_type: {{auction_type}}, has_aaid: {{has_aaid}} -> success", "refId": "B" } ], "thresholds": [], "timeRegions": [], "title": "ms-auction executions", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:106", "format": "short", "logBase": 1, "show": true }, { "$$hashKey": "object:107", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } } ], "title": "Profiling Auction", "type": "row" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 42 }, "id": 57, "panels": [ { "description": "", "fieldConfig": { "defaults": { "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 9, "w": 6, "x": 0, "y": 19 }, "id": 121, "options": { "displayMode": "gradient", "orientation": "horizontal", "reduceOptions": { "calcs": [ "mean" ], "fields": "", "values": false }, "showUnfilled": true }, "pluginVersion": "8.3.1", "targets": [ { "expr": "sum(marketplace_ogury_bidfloor_bucket:rate2m{kubernetes_namespace=\"ms-choices\",platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=~\"$inventory_source\",source_type=~\"$source_type\",country=\"FR\"}) by (le)", "format": "heatmap", "interval": "", "legendFormat": "{{le}}", "refId": "C" } ], "title": "ogury bidfloor distribution FR", "type": "bargauge" }, { "description": "", "fieldConfig": { "defaults": { "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 9, "w": 6, "x": 6, "y": 19 }, "id": 122, "options": { "displayMode": "gradient", "orientation": "horizontal", "reduceOptions": { "calcs": [ "mean" ], "fields": "", "values": false }, "showUnfilled": true }, "pluginVersion": "8.3.1", "targets": [ { "expr": "sum(marketplace_ogury_bidfloor_bucket:rate2m{kubernetes_namespace=\"ms-choices\",platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=~\"$inventory_source\",source_type=~\"$source_type\",country=\"IT\"}) by (le)", "format": "heatmap", "interval": "", "legendFormat": "{{le}}", "refId": "C" } ], "title": "ogury bidfloor distribution IT", "type": "bargauge" }, { "description": "", "fieldConfig": { "defaults": { "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 9, "w": 6, "x": 12, "y": 19 }, "id": 123, "options": { "displayMode": "gradient", "orientation": "horizontal", "reduceOptions": { "calcs": [ "mean" ], "fields": "", "values": false }, "showUnfilled": true }, "pluginVersion": "8.3.1", "targets": [ { "expr": "sum(marketplace_ogury_bidfloor_bucket:rate2m{kubernetes_namespace=\"ms-choices\",platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=~\"$inventory_source\",source_type=~\"$source_type\",country=\"GB\"}) by (le)", "format": "heatmap", "interval": "", "legendFormat": "{{le}}", "refId": "C" } ], "title": "ogury bidfloor distribution GB", "type": "bargauge" }, { "description": "", "fieldConfig": { "defaults": { "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 9, "w": 6, "x": 18, "y": 19 }, "id": 124, "options": { "displayMode": "gradient", "orientation": "horizontal", "reduceOptions": { "calcs": [ "mean" ], "fields": "", "values": false }, "showUnfilled": true }, "pluginVersion": "8.3.1", "targets": [ { "expr": "sum(marketplace_ogury_bidfloor_bucket:rate2m{kubernetes_namespace=\"ms-choices\",platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=~\"$inventory_source\",source_type=~\"$source_type\",country=\"ES\"}) by (le)", "format": "heatmap", "interval": "", "legendFormat": "{{le}}", "refId": "C" } ], "title": "ogury bidfloor distribution ES", "type": "bargauge" }, { "description": "", "fieldConfig": { "defaults": { "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 9, "w": 6, "x": 0, "y": 28 }, "id": 125, "options": { "displayMode": "gradient", "orientation": "horizontal", "reduceOptions": { "calcs": [ "mean" ], "fields": "", "values": false }, "showUnfilled": true }, "pluginVersion": "8.3.1", "targets": [ { "expr": "sum(marketplace_ogury_bidfloor_bucket:rate2m{kubernetes_namespace=\"ms-choices\",platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=~\"$inventory_source\",source_type=~\"$source_type\",country=\"DE\"}) by (le)", "format": "heatmap", "interval": "", "legendFormat": "{{le}}", "refId": "C" } ], "title": "ogury bidfloor distribution DE", "type": "bargauge" }, { "description": "", "fieldConfig": { "defaults": { "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 9, "w": 6, "x": 6, "y": 28 }, "id": 126, "options": { "displayMode": "gradient", "orientation": "horizontal", "reduceOptions": { "calcs": [ "mean" ], "fields": "", "values": false }, "showUnfilled": true }, "pluginVersion": "8.3.1", "targets": [ { "expr": "sum(marketplace_ogury_bidfloor_bucket:rate2m{kubernetes_namespace=\"ms-choices\",platform=~\"$env\",region=~\"$region\",stack=~\"$stack\",inventory_source=~\"$inventory_source\",source_type=~\"$source_type\",country=\"US\"}) by (le)", "format": "heatmap", "interval": "", "legendFormat": "{{le}}", "refId": "C" } ], "title": "ogury bidfloor distribution US", "type": "bargauge" } ], "title": "Profiling Marketplace (internal only)", "type": "row" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 43 }, "id": 102, "panels": [ { "aliasColors": { "INTERNAL error": "red" }, "bars": false, "dashLength": 10, "dashes": false, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 45 }, "hiddenSeries": false, "id": 219, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.1", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(choices_request_duration_seconds_count:rate2m{platform=\"prod\",region=\"eu-west-1\",inventory_source=\"INTERNAL\",source_type=\"ANDROID\",error=\"false\",fill=\"true\"}) / sum(choices_request_duration_seconds_count:rate2m{platform=\"prod\",region=\"eu-west-1\",inventory_source=\"INTERNAL\",source_type=\"ANDROID\",error=\"false\"})", "interval": "", "legendFormat": "eu-west-1 | INTERNAL | android", "refId": "A" }, { "expr": "sum(choices_request_duration_seconds_count:rate2m{platform=\"prod\",region=\"eu-west-1\",inventory_source=\"INTERNAL\",source_type=\"IOS\",error=\"false\",fill=\"true\"}) / sum(choices_request_duration_seconds_count:rate2m{platform=\"prod\",region=\"eu-west-1\",inventory_source=\"INTERNAL\",source_type=\"IOS\",error=\"false\"})", "interval": "", "legendFormat": "eu-west-1 | INTERNAL | ios", "refId": "B" }, { "expr": "sum(choices_request_duration_seconds_count:rate2m{platform=\"prod\",region=\"eu-west-1\",inventory_source=\"INTERNAL\",source_type=\"WEBMOBILE\",error=\"false\",fill=\"true\"}) / sum(choices_request_duration_seconds_count:rate2m{platform=\"prod\",region=\"eu-west-1\",inventory_source=\"INTERNAL\",source_type=\"WEBMOBILE\",error=\"false\"})", "interval": "", "legendFormat": "eu-west-1 | INTERNAL | web", "refId": "C" }, { "expr": "sum(choices_request_duration_seconds_count:rate2m{platform=\"prod\",region=\"us-east-1\",inventory_source=\"INTERNAL\",source_type=\"ANDROID\",error=\"false\",fill=\"true\"}) / sum(choices_request_duration_seconds_count:rate2m{platform=\"prod\",region=\"us-east-1\",inventory_source=\"INTERNAL\",source_type=\"ANDROID\",error=\"false\"})", "interval": "", "legendFormat": "us-east-1 | INTERNAL | android", "refId": "D" }, { "expr": "sum(choices_request_duration_seconds_count:rate2m{platform=\"prod\",region=\"us-east-1\",inventory_source=\"INTERNAL\",source_type=\"IOS\",error=\"false\",fill=\"true\"}) / sum(choices_request_duration_seconds_count:rate2m{platform=\"prod\",region=\"us-east-1\",inventory_source=\"INTERNAL\",source_type=\"IOS\",error=\"false\"})", "interval": "", "legendFormat": "us-east-1 | INTERNAL | ios", "refId": "E" }, { "expr": "sum(choices_request_duration_seconds_count:rate2m{platform=\"prod\",region=\"us-east-1\",inventory_source=\"INTERNAL\",source_type=\"WEBMOBILE\",error=\"false\",fill=\"true\"}) / sum(choices_request_duration_seconds_count:rate2m{platform=\"prod\",region=\"us-east-1\",inventory_source=\"INTERNAL\",source_type=\"WEBMOBILE\",error=\"false\"})", "interval": "", "legendFormat": "us-east-1 | INTERNAL | web", "refId": "F" }, { "expr": "sum(choices_request_duration_seconds_count:rate2m{platform=\"prod\",region=\"us-west-2\",inventory_source=\"INTERNAL\",source_type=\"ANDROID\",error=\"false\",fill=\"true\"}) / sum(choices_request_duration_seconds_count:rate2m{platform=\"prod\",region=\"us-west-2\",inventory_source=\"INTERNAL\",source_type=\"ANDROID\",error=\"false\"})", "interval": "", "legendFormat": "us-west-2 | INTERNAL | android", "refId": "D" }, { "expr": "sum(choices_request_duration_seconds_count:rate2m{platform=\"prod\",region=\"us-west-2\",inventory_source=\"INTERNAL\",source_type=\"IOS\",error=\"false\",fill=\"true\"}) / sum(choices_request_duration_seconds_count:rate2m{platform=\"prod\",region=\"us-west-2\",inventory_source=\"INTERNAL\",source_type=\"IOS\",error=\"false\"})", "interval": "", "legendFormat": "us-west-2 | INTERNAL | ios", "refId": "E" }, { "expr": "sum(choices_request_duration_seconds_count:rate2m{platform=\"prod\",region=\"us-west-2\",inventory_source=\"INTERNAL\",source_type=\"WEBMOBILE\",error=\"false\",fill=\"true\"}) / sum(choices_request_duration_seconds_count:rate2m{platform=\"prod\",region=\"us-west-2\",inventory_source=\"INTERNAL\",source_type=\"WEBMOBILE\",error=\"false\"})", "interval": "", "legendFormat": "us-west-2 | INTERNAL | web", "refId": "F" }, { "expr": "sum(choices_request_duration_seconds_count:rate2m{platform=\"prod\",region=\"ap-northeast-1\",inventory_source=\"INTERNAL\",source_type=\"ANDROID\",error=\"false\",fill=\"true\"}) / sum(choices_request_duration_seconds_count:rate2m{platform=\"prod\",region=\"ap-northeast-1\",inventory_source=\"INTERNAL\",source_type=\"ANDROID\",error=\"false\"})", "interval": "", "legendFormat": "ap-northeast-1 | INTERNAL | android", "refId": "D" }, { "expr": "sum(choices_request_duration_seconds_count:rate2m{platform=\"prod\",region=\"ap-northeast-1\",inventory_source=\"INTERNAL\",source_type=\"IOS\",error=\"false\",fill=\"true\"}) / sum(choices_request_duration_seconds_count:rate2m{platform=\"prod\",region=\"ap-northeast-1\",inventory_source=\"INTERNAL\",source_type=\"IOS\",error=\"false\"})", "interval": "", "legendFormat": "ap-northeast-1 | INTERNAL | ios", "refId": "E" }, { "expr": "sum(choices_request_duration_seconds_count:rate2m{platform=\"prod\",region=\"ap-northeast-1\",inventory_source=\"INTERNAL\",source_type=\"WEBMOBILE\",error=\"false\",fill=\"true\"}) / sum(choices_request_duration_seconds_count:rate2m{platform=\"prod\",region=\"ap-northeast-1\",inventory_source=\"INTERNAL\",source_type=\"WEBMOBILE\",error=\"false\"})", "interval": "", "legendFormat": "ap-northeast-1 | INTERNAL | web", "refId": "F" } ], "thresholds": [], "timeRegions": [], "title": "fill rate INTERNAL", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:324", "format": "percentunit", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:325", "format": "none", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "alert": { "alertRuleTags": {}, "conditions": [ { "evaluator": { "params": [ 1 ], "type": "gt" }, "operator": { "type": "and" }, "query": { "params": [ "A", "5m", "now" ] }, "reducer": { "params": [], "type": "avg" }, "type": "query" } ], "executionErrorState": "keep_state", "for": "5m", "frequency": "1m", "handler": 1, "message": "One or more programmatic campaigns are candidate in ms-choices external.", "name": "external programmatic campaign candidate alert", "noDataState": "ok", "notifications": [ { "uid": "sTLSyNrZk" } ] }, "aliasColors": { "INTERNAL error": "red" }, "bars": false, "dashLength": 10, "dashes": false, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 45 }, "hiddenSeries": false, "id": 186, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.1", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(external_prog_campaigns_total{kubernetes_pod_name=~\"ms-choices-.*\",platform=\"prod\"}[60s]))", "interval": "", "legendFormat": "", "refId": "A" } ], "thresholds": [ { "colorMode": "critical", "fill": true, "line": true, "op": "gt", "value": 1, "visible": true } ], "timeRegions": [], "title": "external programmatic campaign candidate", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:410", "format": "none", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:411", "format": "none", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "alert": { "alertRuleTags": {}, "conditions": [ { "evaluator": { "params": [ 0.35 ], "type": "lt" }, "operator": { "type": "and" }, "query": { "params": [ "A", "5m", "now" ] }, "reducer": { "params": [], "type": "avg" }, "type": "query" }, { "evaluator": { "params": [ 0.7 ], "type": "gt" }, "operator": { "type": "or" }, "query": { "params": [ "B", "5m", "now" ] }, "reducer": { "params": [], "type": "avg" }, "type": "query" } ], "executionErrorState": "keep_state", "for": "5m", "frequency": "1m", "handler": 1, "message": "Score coverage too low!", "name": "Score coverage alert", "noDataState": "alerting", "notifications": [ { "uid": "sTLSyNrZk" } ] }, "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 53 }, "hiddenSeries": false, "id": 335, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.1", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "exemplar": true, "expr": "sum(rate(score_presence_total{kubernetes_namespace=\"ms-choices\",platform=\"prod\",present=\"true\",score_name=\"cta_score\"}[5m])) / sum(rate(score_presence_total{kubernetes_namespace=\"ms-choices\",platform=\"prod\",score_name=\"cta_score\"}[5m]))", "interval": "", "legendFormat": "Score present %", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "exemplar": true, "expr": "sum(rate(score_presence_total{kubernetes_namespace=\"ms-choices\",platform=\"prod\",present=\"true\",score_name=\"context_score\"}[5m])) / sum(rate(score_presence_total{kubernetes_namespace=\"ms-choices\",platform=\"prod\",score_name=\"context_score\"}[5m]))", "hide": false, "interval": "", "legendFormat": "Score present %", "refId": "B" } ], "thresholds": [ { "colorMode": "critical", "fill": true, "line": true, "op": "lt", "value": 0.35, "visible": true } ], "timeRegions": [], "title": "Score coverage", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:104", "format": "percentunit", "logBase": 1, "show": true }, { "$$hashKey": "object:105", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } } ], "title": "Alerting", "type": "row" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 44 }, "id": 329, "panels": [ { "aliasColors": { "INTERNAL error": "red" }, "bars": false, "dashLength": 10, "dashes": false, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 77 }, "hiddenSeries": false, "id": 330, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.1", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "exemplar": true, "expr": "max(dynamo_cache_size_total{platform=~\"$env\",region=~\"$region\",stack=~\"$stack\"}) by (table_name)", "interval": "", "legendFormat": "{{ table_name }}", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Cache size", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:318", "format": "none", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:319", "format": "none", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": { "INTERNAL error": "red" }, "bars": false, "dashLength": 10, "dashes": false, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 77 }, "hiddenSeries": false, "id": 331, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.3.1", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": true, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "exemplar": true, "expr": "sum(rate(dynamo_cache_requests_total{platform=~\"$env\",region=~\"$region\",stack=~\"$stack\", table_name=\"data-prod\", response_type=\"miss\"}[2m]))", "hide": false, "interval": "", "legendFormat": "miss", "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "exemplar": true, "expr": "sum(rate(dynamo_cache_requests_total{platform=~\"$env\",region=~\"$region\",stack=~\"$stack\", table_name=\"data-prod\", response_type=\"stale\"}[2m]))", "hide": false, "interval": "", "legendFormat": "stale", "refId": "C" }, { "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "exemplar": true, "expr": "sum(rate(dynamo_cache_requests_total{platform=~\"$env\",region=~\"$region\",stack=~\"$stack\", table_name=\"data-prod\", response_type=\"hit\"}[2m]))", "interval": "", "legendFormat": "hit", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Cache requests - data", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:318", "format": "none", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:319", "format": "none", "logBase": 1, "show": true } ], "yaxis": { "align": false } } ], "title": "Dynamo cache", "type": "row" } ], "refresh": "", "schemaVersion": 33, "style": "dark", "tags": [ "Adserver" ], "templating": { "list": [ { "current": { "selected": false, "text": "prod", "value": "prod" }, "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "definition": "label_values(kube_endpoint_info{region=~\"$region\"}, platform)", "hide": 0, "includeAll": false, "multi": false, "name": "env", "options": [], "query": { "query": "label_values(kube_endpoint_info{region=~\"$region\"}, platform)", "refId": "prometheus-env-Variable-Query" }, "refresh": 1, "regex": "", "skipUrlSync": false, "sort": 1, "tagValuesQuery": "", "tagsQuery": "", "type": "query", "useTags": false }, { "current": { "selected": false, "text": [ "eu-west-1" ], "value": [ "eu-west-1" ] }, "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "definition": "label_values(kube_endpoint_info, region)", "hide": 0, "includeAll": true, "multi": true, "name": "region", "options": [], "query": { "query": "label_values(kube_endpoint_info, region)", "refId": "prometheus-region-Variable-Query" }, "refresh": 1, "regex": "", "skipUrlSync": false, "sort": 0, "tagValuesQuery": "", "tagsQuery": "", "type": "query", "useTags": false }, { "current": { "selected": false, "text": "None", "value": "" }, "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "definition": "", "hide": 2, "includeAll": false, "multi": false, "name": "datasource", "options": [], "query": { "query": "", "refId": "prometheus-datasource-Variable-Query" }, "refresh": 1, "regex": "", "skipUrlSync": false, "sort": 0, "tagValuesQuery": "", "tagsQuery": "", "type": "query", "useTags": false }, { "current": { "selected": false, "text": [ "All" ], "value": [ "$__all" ] }, "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "definition": "label_values(kube_endpoint_info{region=~\"$region\",platform=~\"$env\"}, stack)", "hide": 0, "includeAll": true, "multi": true, "name": "stack", "options": [], "query": { "query": "label_values(kube_endpoint_info{region=~\"$region\",platform=~\"$env\"}, stack)", "refId": "prometheus-stack-Variable-Query" }, "refresh": 1, "regex": "", "skipUrlSync": false, "sort": 0, "tagValuesQuery": "", "tagsQuery": "", "type": "query", "useTags": false }, { "allValue": ".*", "current": { "selected": false, "text": "All", "value": "$__all" }, "hide": 0, "includeAll": true, "multi": false, "name": "inventory_source", "options": [ { "selected": true, "text": "All", "value": "$__all" }, { "selected": false, "text": "INTERNAL", "value": "INTERNAL" }, { "selected": false, "text": "EXTERNAL", "value": "EXTERNAL" }, { "selected": false, "text": "HEADER_BIDDING", "value": "HEADER_BIDDING" } ], "query": "INTERNAL,EXTERNAL,HEADER_BIDDING", "queryValue": "", "skipUrlSync": false, "type": "custom" }, { "allValue": ".*", "current": { "selected": false, "text": "All", "value": "$__all" }, "hide": 0, "includeAll": true, "multi": false, "name": "source_type", "options": [ { "selected": true, "text": "All", "value": "$__all" }, { "selected": false, "text": "ANDROID", "value": "ANDROID" }, { "selected": false, "text": "IOS", "value": "IOS" }, { "selected": false, "text": "WEBMOBILE", "value": "WEBMOBILE" } ], "query": "ANDROID,IOS,WEBMOBILE", "queryValue": "", "skipUrlSync": false, "type": "custom" }, { "current": { "selected": false, "text": "All", "value": "$__all" }, "datasource": { "type": "prometheus", "uid": "P1809F7CD0C75ACF3" }, "definition": "label_values(step_campaign_num_bucket:sum{kubernetes_namespace=\"ms-choices\"}, step)", "hide": 2, "includeAll": true, "multi": false, "name": "step_for_campaign_num", "options": [], "query": { "query": "label_values(step_campaign_num_bucket:sum{kubernetes_namespace=\"ms-choices\"}, step)", "refId": "prometheus-step_for_campaign_num-Variable-Query" }, "refresh": 1, "regex": "", "skipUrlSync": false, "sort": 0, "tagValuesQuery": "", "tagsQuery": "", "type": "query", "useTags": false }, { "current": { "selected": false, "text": [ "All" ], "value": [ "$__all" ] }, "hide": 0, "includeAll": true, "multi": true, "name": "environment_type", "options": [ { "selected": true, "text": "All", "value": "$__all" }, { "selected": false, "text": "in-app", "value": "in-app" }, { "selected": false, "text": "web", "value": "web" } ], "query": "in-app,web", "queryValue": "", "skipUrlSync": false, "type": "custom" }, { "current": { "selected": false, "text": [ "All" ], "value": [ "$__all" ] }, "hide": 0, "includeAll": true, "multi": true, "name": "device_os", "options": [ { "selected": true, "text": "All", "value": "$__all" }, { "selected": false, "text": "android", "value": "android" }, { "selected": false, "text": "ios", "value": "ios" } ], "query": "android,ios", "queryValue": "", "skipUrlSync": false, "type": "custom" } ] }, "time": { "from": "now-1h", "to": "now" }, "timepicker": { "nowDelay": "", "refresh_intervals": [ "10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d" ] }, "timezone": "", "title": "ms-choices server kube", "uid": "o0hlvMgGz", "version": 354, "weekStart": "" }