{
  "eval_id": 2,
  "eval_name": "orm-agent-custom-profile",
  "skill": "wiki-orm-agent",
  "prompt": "I have a custom ORM that uses BaseModel classes with __table__ attributes. Help me create an ORM profile for it",
  "assertions": [
    {
      "text": "Q&A covers detection markers (class inheritance + __table__), file patterns, entity extraction (class_pattern, table_pattern, column_pattern), and at least one relationship pattern",
      "passed": null,
      "evidence": null
    },
    {
      "text": "Q&A covers naming conventions (table_from_class and column_from_field)",
      "passed": null,
      "evidence": null
    },
    {
      "text": "Generated YAML has every required top-level field: name, language, description, detection, entity_extraction, relationship_patterns, naming_conventions",
      "passed": null,
      "evidence": null
    },
    {
      "text": "entity_extraction.class_pattern has a single capture group that matches at least one class name in example.py (verified by running the pattern against the fixture)",
      "passed": null,
      "evidence": null
    },
    {
      "text": "profile loader (profiles.js loadProfile) accepts the YAML without raising ProfileValueError",
      "passed": null,
      "evidence": null
    },
    {
      "text": "When the profile is applied to an example.py containing THREE BaseModel classes with DIFFERENT __table__ values, extractEntities yields 3 entities with 3 DIFFERENT table_names — NOT all three collapsing to the first table (the per-class windowing fix must hold even with a straightforward table_pattern that matches every __table__ occurrence)",
      "passed": null,
      "evidence": null
    }
  ]
}