{
  "task": "preflight",
  "modelType": "lr",
  "trainRows": 1640,
  "rawDomainRows": 3787,
  "reviewedDomainRows": 2910,
  "rawMultitaskRows": 609,
  "reviewedMultitaskRows": 606,
  "queryRows": 176,
  "multitaskQueryRows": 606,
  "cappedDomainRows": 1809,
  "maxDomainRowsPerLabel": 600,
  "domainHoldoutRows": 126,
  "queryHoldoutRows": 24,
  "targets": {
    "domain": {
      "threshold": 0.9984900951385498,
      "labels": [
        "ai-ml",
        "changelog",
        "cloud-docs",
        "ecommerce",
        "finance",
        "forums",
        "github",
        "legal",
        "medical",
        "news-current-events",
        "package-registry",
        "papers",
        "quantum",
        "security",
        "shopify",
        "specs",
        "standards",
        "trading",
        "vendor-status",
        "web"
      ]
    },
    "query_shape": {
      "threshold": 0.9901491403579712,
      "labels": [
        "academic_review",
        "ambiguous_factoid",
        "comparison",
        "current_or_version_sensitive",
        "explanation",
        "howto",
        "legal_medical_finance_sensitive",
        "shopping_or_ecommerce",
        "short_fact",
        "troubleshooting"
      ]
    },
    "answer_shape": {
      "threshold": 0.5767588019371033,
      "labels": [
        "citation_heavy",
        "comparison_table",
        "list",
        "long_explanation",
        "short_answer",
        "step_by_step"
      ]
    },
    "source_family": {
      "threshold": 0.98759925365448,
      "labels": [
        "academic",
        "community",
        "encyclopedia",
        "general_web",
        "government_or_legal",
        "official_docs",
        "primary_source",
        "product_or_ecommerce",
        "recent_news"
      ]
    },
    "recency_need": {
      "threshold": 0.901154637336731,
      "labels": [
        "helpful",
        "none",
        "required"
      ]
    },
    "ambiguity": {
      "threshold": 0.5972715020179749,
      "labels": [
        "high",
        "low",
        "medium"
      ]
    }
  },
  "summary": {
    "domain_macro_f1": 0.4622100122100123,
    "query_understanding_macro_f1": 0.5310928448007556
  },
  "targetMetrics": {
    "domain": {
      "train_support": {
        "web": 698,
        "papers": 80,
        "github": 158,
        "security": 41,
        "specs": 47,
        "changelog": 46,
        "package-registry": 78,
        "vendor-status": 40,
        "ai-ml": 107,
        "forums": 37,
        "ecommerce": 7,
        "standards": 1,
        "shopify": 27,
        "legal": 25,
        "cloud-docs": 2,
        "trading": 4,
        "finance": 25,
        "news-current-events": 20,
        "medical": 20,
        "quantum": 1
      },
      "train_rows": 1464,
      "resampled_train_rows": 13960,
      "labels": [
        "ai-ml",
        "changelog",
        "cloud-docs",
        "ecommerce",
        "finance",
        "forums",
        "github",
        "legal",
        "medical",
        "news-current-events",
        "package-registry",
        "papers",
        "quantum",
        "security",
        "shopify",
        "specs",
        "standards",
        "trading",
        "vendor-status",
        "web"
      ],
      "raw": {
        "accuracy": 0.5634920634920635,
        "macro_f1": 0.4622100122100123,
        "support": 126
      },
      "accepted": {
        "accuracy": 0.015873015873015872,
        "macro_f1": 0.025974025974025972,
        "support": 126
      },
      "threshold": {
        "threshold": 0.9984900951385498,
        "coverage": 0.015873015873015872,
        "accuracy": 1.0
      },
      "holdout_support": {
        "security": 6,
        "vendor-status": 6,
        "package-registry": 6,
        "github": 6,
        "changelog": 6,
        "papers": 6,
        "specs": 6,
        "forums": 6,
        "legal": 6,
        "medical": 6,
        "finance": 6,
        "cloud-docs": 6,
        "ai-ml": 6,
        "ecommerce": 6,
        "trading": 6,
        "quantum": 6,
        "shopify": 6,
        "standards": 6,
        "news-current-events": 6,
        "local-howto": 6,
        "web": 6
      }
    },
    "query_shape": {
      "train_support": {
        "short_fact": 509,
        "legal_medical_finance_sensitive": 15,
        "current_or_version_sensitive": 123,
        "explanation": 67,
        "ambiguous_factoid": 32,
        "comparison": 10,
        "howto": 10,
        "troubleshooting": 7,
        "academic_review": 6,
        "shopping_or_ecommerce": 3
      },
      "train_rows": 782,
      "resampled_train_rows": 5090,
      "labels": [
        "academic_review",
        "ambiguous_factoid",
        "comparison",
        "current_or_version_sensitive",
        "explanation",
        "howto",
        "legal_medical_finance_sensitive",
        "shopping_or_ecommerce",
        "short_fact",
        "troubleshooting"
      ],
      "raw": {
        "accuracy": 0.5416666666666666,
        "macro_f1": 0.5382539682539682,
        "support": 24
      },
      "accepted": {
        "accuracy": 0.125,
        "macro_f1": 0.18181818181818182,
        "support": 24
      },
      "threshold": {
        "threshold": 0.9901491403579712,
        "coverage": 0.125,
        "accuracy": 1.0
      },
      "holdout_support": {
        "short_fact": 2,
        "explanation": 6,
        "comparison": 2,
        "howto": 1,
        "troubleshooting": 1,
        "current_or_version_sensitive": 4,
        "academic_review": 2,
        "shopping_or_ecommerce": 2,
        "legal_medical_finance_sensitive": 2,
        "ambiguous_factoid": 2
      }
    },
    "answer_shape": {
      "train_support": {
        "short_answer": 540,
        "citation_heavy": 8,
        "list": 73,
        "long_explanation": 139,
        "comparison_table": 6,
        "step_by_step": 16
      },
      "train_rows": 782,
      "resampled_train_rows": 3240,
      "labels": [
        "citation_heavy",
        "comparison_table",
        "list",
        "long_explanation",
        "short_answer",
        "step_by_step"
      ],
      "raw": {
        "accuracy": 0.7916666666666666,
        "macro_f1": 0.7761904761904762,
        "support": 24
      },
      "accepted": {
        "accuracy": 0.6666666666666666,
        "macro_f1": 0.5961538461538461,
        "support": 24
      },
      "threshold": {
        "threshold": 0.5767588019371033,
        "coverage": 0.75,
        "accuracy": 0.8888888888888888
      },
      "holdout_support": {
        "short_answer": 5,
        "long_explanation": 7,
        "comparison_table": 3,
        "step_by_step": 2,
        "citation_heavy": 4,
        "list": 3
      }
    },
    "source_family": {
      "train_support": {
        "government_or_legal": 40,
        "primary_source": 17,
        "recent_news": 14,
        "academic": 9,
        "encyclopedia": 458,
        "official_docs": 26,
        "general_web": 209,
        "community": 6,
        "product_or_ecommerce": 3
      },
      "train_rows": 782,
      "resampled_train_rows": 4122,
      "labels": [
        "academic",
        "community",
        "encyclopedia",
        "general_web",
        "government_or_legal",
        "official_docs",
        "primary_source",
        "product_or_ecommerce",
        "recent_news"
      ],
      "raw": {
        "accuracy": 0.375,
        "macro_f1": 0.32275132275132273,
        "support": 24
      },
      "accepted": {
        "accuracy": 0.08333333333333333,
        "macro_f1": 0.09427609427609426,
        "support": 24
      },
      "threshold": {
        "threshold": 0.98759925365448,
        "coverage": 0.08333333333333333,
        "accuracy": 1.0
      },
      "holdout_support": {
        "encyclopedia": 3,
        "official_docs": 10,
        "academic": 2,
        "product_or_ecommerce": 2,
        "primary_source": 3,
        "government_or_legal": 1,
        "general_web": 2,
        "community": 1
      }
    },
    "recency_need": {
      "train_support": {
        "none": 587,
        "helpful": 131,
        "required": 64
      },
      "train_rows": 782,
      "resampled_train_rows": 1761,
      "labels": [
        "helpful",
        "none",
        "required"
      ],
      "raw": {
        "accuracy": 0.7083333333333334,
        "macro_f1": 0.7167133520074698,
        "support": 24
      },
      "accepted": {
        "accuracy": 0.25,
        "macro_f1": 0.3345238095238095,
        "support": 24
      },
      "threshold": {
        "threshold": 0.901154637336731,
        "coverage": 0.25,
        "accuracy": 1.0
      },
      "holdout_support": {
        "none": 6,
        "helpful": 13,
        "required": 5
      }
    },
    "ambiguity": {
      "train_support": {
        "low": 612,
        "medium": 159,
        "high": 11
      },
      "train_rows": 782,
      "resampled_train_rows": 1836,
      "labels": [
        "high",
        "low",
        "medium"
      ],
      "raw": {
        "accuracy": 0.4583333333333333,
        "macro_f1": 0.30155510480054093,
        "support": 24
      },
      "accepted": {
        "accuracy": 0.4166666666666667,
        "macro_f1": 0.22962962962962963,
        "support": 24
      },
      "threshold": {
        "threshold": 0.5972715020179749,
        "coverage": 0.8333333333333334,
        "accuracy": 0.5
      },
      "holdout_support": {
        "low": 18,
        "medium": 4,
        "high": 2
      }
    }
  }
}