{
  "name": "Conversational Kitchen Sink",
  "nodes": [
    {
      "parameters": {},
      "id": "manual-trigger",
      "name": "When clicking Execute Workflow",
      "type": "n8n-nodes-base.manualTrigger",
      "typeVersion": 1,
      "position": [0, 0]
    },
    {
      "parameters": {
        "resource": "row",
        "operation": "get",
        "dataTableId": {
          "__rl": true,
          "mode": "id",
          "value": "REPLACE_WITH_SOURCE_DATA_TABLE_ID"
        },
        "returnAll": true,
        "filters": {},
        "orderBy": false
      },
      "id": "source-table",
      "name": "Load Source Rows",
      "type": "n8n-nodes-base.dataTable",
      "typeVersion": 1.1,
      "position": [240, 0]
    },
    {
      "parameters": {
        "runName": "Conversational Kitchen Sink Benchmark",
        "dataTableId": "REPLACE_WITH_SOURCE_DATA_TABLE_ID",
        "columnMapping": "{\"input\":\"input\",\"expectedOutput\":\"expectedOutput\"}",
        "filters": "{}",
        "limitRows": true,
        "maxRows": 1
      },
      "id": "deepeval-trigger",
      "name": "DeepEval Trigger",
      "type": "n8n-nodes-deepeval.deepEvalTrigger",
      "typeVersion": 1,
      "position": [480, 0]
    },
    {
      "parameters": {
        "mode": "manual",
        "assignments": {
          "assignments": [
            {
              "id": "enrich-0",
              "name": "input",
              "type": "string",
              "value": "Use the calculator when appropriate, then answer: what is 2 + 2?"
            },
            {
              "id": "enrich-1",
              "name": "actualOutput",
              "type": "string",
              "value": "The answer is 4."
            },
            {
              "id": "enrich-2",
              "name": "output",
              "type": "string",
              "value": "The answer is 4."
            },
            {
              "id": "enrich-3",
              "name": "context",
              "type": "array",
              "value": "[\"Two plus two equals four.\"]"
            },
            {
              "id": "enrich-4",
              "name": "retrievalContext",
              "type": "array",
              "value": "[\"[1] Basic arithmetic states that 2 + 2 = 4.\"]"
            },
            {
              "id": "enrich-5",
              "name": "expectedTools",
              "type": "array",
              "value": "[{\"name\":\"calculator\",\"inputParameters\":{\"input\":\"2+2\"}}]"
            },
            {
              "id": "enrich-6",
              "name": "chatbotRole",
              "type": "string",
              "value": "A concise and accurate arithmetic assistant"
            },
            {
              "id": "enrich-7",
              "name": "expectedOutcome",
              "type": "string",
              "value": "The assistant answers that 2 + 2 equals 4."
            },
            {
              "id": "enrich-8",
              "name": "metadata",
              "type": "object",
              "value": {
                "suite": "deepeval-e2e"
              }
            }
          ]
        },
        "includeOtherFields": true,
        "options": {}
      },
      "id": "enrich-evaluation-data",
      "name": "Enrich Evaluation Data",
      "type": "n8n-nodes-base.set",
      "typeVersion": 3.4,
      "position": [720, 0]
    },
    {
      "parameters": {
        "promptType": "define",
        "text": "={{ $json.input }}",
        "options": {
          "returnIntermediateSteps": true,
          "systemMessage": "You are an arithmetic assistant. You must use the Calculator tool exactly once before answering."
        }
      },
      "id": "ai-agent",
      "name": "AI Agent",
      "type": "@n8n/n8n-nodes-langchain.agent",
      "typeVersion": 3.1,
      "position": [960, 0]
    },
    {
      "parameters": {
        "mode": "combine",
        "combineBy": "combineByPosition",
        "options": {}
      },
      "id": "metric-input",
      "name": "Prepare Metric Input",
      "type": "n8n-nodes-base.merge",
      "typeVersion": 3.2,
      "position": [1200, 0]
    },
    {
      "parameters": {
        "mode": "append",
        "numberInputs": 13,
        "options": {}
      },
      "id": "collect-metric-results",
      "name": "Collect Metric Results",
      "type": "n8n-nodes-base.merge",
      "typeVersion": 3.2,
      "position": [1680, 0]
    },
    {
      "parameters": {
        "dataTableId": "REPLACE_WITH_RESULTS_DATA_TABLE_ID",
        "passRule": "allPass",
        "writeMode": "upsert",
        "runIdColumn": "runId",
        "scoreColumn": "overallScore",
        "successColumn": "overallSuccess",
        "metricsColumn": "metrics"
      },
      "id": "deepeval-aggregate",
      "name": "DeepEval Aggregate",
      "type": "n8n-nodes-deepeval.deepEvalAggregate",
      "typeVersion": 1,
      "position": [1920, 0]
    },
    {
      "parameters": {
        "resource": "row",
        "operation": "insert",
        "dataTableId": {
          "__rl": true,
          "mode": "id",
          "value": "REPLACE_WITH_RESULTS_DATA_TABLE_ID"
        },
        "columns": {
          "mappingMode": "defineBelow",
          "value": {
            "runId": "={{ $json.runId }}",
            "overallScore": "={{ $json.overallScore }}",
            "overallSuccess": "={{ $json.overallSuccess }}",
            "metrics": "={{ JSON.stringify($json.metrics) }}"
          },
          "schema": [
            {
              "id": "runId",
              "displayName": "runId",
              "type": "string",
              "canBeUsedToMatch": true
            },
            {
              "id": "overallScore",
              "displayName": "overallScore",
              "type": "number",
              "canBeUsedToMatch": true
            },
            {
              "id": "overallSuccess",
              "displayName": "overallSuccess",
              "type": "boolean",
              "canBeUsedToMatch": true
            },
            {
              "id": "metrics",
              "displayName": "metrics",
              "type": "string",
              "canBeUsedToMatch": true
            }
          ]
        },
        "options": {}
      },
      "id": "persist-results",
      "name": "Persist Results",
      "type": "n8n-nodes-base.dataTable",
      "typeVersion": 1.1,
      "position": [2160, 0]
    },
    {
      "parameters": {
        "model": {
          "__rl": true,
          "mode": "id",
          "value": "REPLACE_WITH_MODEL_ID"
        },
        "options": {
          "baseURL": "http://127.0.0.1:8080/v1",
          "timeout": 120000,
          "maxRetries": 0
        }
      },
      "id": "judge-model",
      "name": "OpenAI Chat Model",
      "type": "@n8n/n8n-nodes-langchain.lmChatOpenAi",
      "typeVersion": 1.2,
      "position": [960, 280],
      "credentials": {
        "openAiApi": {
          "id": "REPLACE_WITH_OPENAI_CREDENTIAL_ID",
          "name": "Local OpenAI-compatible endpoint"
        }
      }
    },
    {
      "parameters": {},
      "id": "calculator",
      "name": "Calculator",
      "type": "@n8n/n8n-nodes-langchain.toolCalculator",
      "typeVersion": 1,
      "position": [1080, 280]
    },
    {
      "parameters": {
        "sessionIdType": "customKey",
        "sessionKey": "={{ $json.evalContext.runId }}",
        "contextWindowLength": 10
      },
      "id": "simple-memory",
      "name": "Simple Memory",
      "type": "@n8n/n8n-nodes-langchain.memoryBufferWindow",
      "typeVersion": 1.3,
      "position": [1200, 280]
    },
    {
      "parameters": {
        "name": "Conversation Quality",
        "criteria": "Determine whether the conversation is helpful.",
        "evaluationSteps": "[\"Check whether the response answers the user question.\",\"Check whether the response is accurate and supported by the context.\"]",
        "evaluationParams": ["CONTENT"],
        "rubric": "[]",
        "threshold": 0.5,
        "strictMode": false,
        "asyncMode": true,
        "verboseMode": false,
        "cleanSession": false
      },
      "id": "metric-conversationalGEval",
      "name": "DeepEval Conversational G-Eval",
      "type": "n8n-nodes-deepeval.deepEvalConversationalGEval",
      "typeVersion": 1,
      "position": [1440, -720]
    },
    {
      "parameters": {
        "name": "Conversation Decision Graph",
        "dag": "{\"nodes\":{\"judge\":{\"type\":\"BinaryJudgementNode\",\"criteria\":\"Was the assistant response relevant and correct?\",\"evaluation_params\":[\"content\",\"role\"],\"children\":[\"incorrect\",\"correct\"]},\"incorrect\":{\"type\":\"VerdictNode\",\"verdict\":false,\"score\":0},\"correct\":{\"type\":\"VerdictNode\",\"verdict\":true,\"score\":10}}}",
        "threshold": 0.5,
        "includeReason": true,
        "strictMode": false,
        "asyncMode": true,
        "verboseMode": false,
        "cleanSession": false
      },
      "id": "metric-conversationalDAG",
      "name": "DeepEval Conversational DAG",
      "type": "n8n-nodes-deepeval.deepEvalConversationalDAG",
      "typeVersion": 1,
      "position": [1440, -600]
    },
    {
      "parameters": {
        "windowSize": 10,
        "threshold": 0.5,
        "includeReason": true,
        "strictMode": false,
        "asyncMode": true,
        "verboseMode": false,
        "cleanSession": false
      },
      "id": "metric-turnRelevancy",
      "name": "DeepEval Turn Relevancy",
      "type": "n8n-nodes-deepeval.deepEvalTurnRelevancy",
      "typeVersion": 1,
      "position": [1440, -480]
    },
    {
      "parameters": {
        "chatbotRole": "A concise and accurate arithmetic assistant",
        "threshold": 0.5,
        "includeReason": true,
        "strictMode": false,
        "asyncMode": true,
        "verboseMode": false,
        "cleanSession": false
      },
      "id": "metric-roleAdherence",
      "name": "DeepEval Role Adherence",
      "type": "n8n-nodes-deepeval.deepEvalRoleAdherence",
      "typeVersion": 1,
      "position": [1440, -360]
    },
    {
      "parameters": {
        "threshold": 0.5,
        "includeReason": true,
        "strictMode": false,
        "asyncMode": true,
        "verboseMode": false,
        "cleanSession": false
      },
      "id": "metric-knowledgeRetention",
      "name": "DeepEval Knowledge Retention",
      "type": "n8n-nodes-deepeval.deepEvalKnowledgeRetention",
      "typeVersion": 1,
      "position": [1440, -240]
    },
    {
      "parameters": {
        "windowSize": 3,
        "threshold": 0.5,
        "includeReason": true,
        "strictMode": false,
        "asyncMode": true,
        "verboseMode": false,
        "cleanSession": false
      },
      "id": "metric-conversationCompleteness",
      "name": "DeepEval Conversation Completeness",
      "type": "n8n-nodes-deepeval.deepEvalConversationCompleteness",
      "typeVersion": 1,
      "position": [1440, -120]
    },
    {
      "parameters": {
        "threshold": 0.5,
        "includeReason": true,
        "strictMode": false,
        "asyncMode": true,
        "verboseMode": false,
        "cleanSession": false
      },
      "id": "metric-goalAccuracy",
      "name": "DeepEval Goal Accuracy",
      "type": "n8n-nodes-deepeval.deepEvalGoalAccuracy",
      "typeVersion": 1,
      "position": [1440, 0]
    },
    {
      "parameters": {
        "availableTools": "[{\"name\":\"calculator\",\"description\":\"Evaluate arithmetic expressions\"}]",
        "threshold": 0.5,
        "includeReason": true,
        "strictMode": false,
        "asyncMode": true,
        "verboseMode": false,
        "cleanSession": false
      },
      "id": "metric-toolUse",
      "name": "DeepEval Tool Use",
      "type": "n8n-nodes-deepeval.deepEvalToolUse",
      "typeVersion": 1,
      "position": [1440, 120]
    },
    {
      "parameters": {
        "relevantTopics": "[\"arithmetic\"]",
        "threshold": 0.5,
        "includeReason": true,
        "strictMode": false,
        "asyncMode": true,
        "verboseMode": false,
        "cleanSession": false
      },
      "id": "metric-topicAdherence",
      "name": "DeepEval Topic Adherence",
      "type": "n8n-nodes-deepeval.deepEvalTopicAdherence",
      "typeVersion": 1,
      "position": [1440, 240]
    },
    {
      "parameters": {
        "windowSize": 10,
        "truthsExtractionLimit": 10,
        "penalizeAmbiguousClaims": false,
        "threshold": 0.5,
        "includeReason": false,
        "strictMode": false,
        "asyncMode": true,
        "verboseMode": false,
        "cleanSession": false
      },
      "id": "metric-turnFaithfulness",
      "name": "DeepEval Turn Faithfulness",
      "type": "n8n-nodes-deepeval.deepEvalTurnFaithfulness",
      "typeVersion": 1,
      "position": [1440, 360]
    },
    {
      "parameters": {
        "windowSize": 10,
        "expectedOutcome": "The assistant answers that 2 + 2 equals 4.",
        "threshold": 0.5,
        "includeReason": true,
        "strictMode": false,
        "asyncMode": true,
        "verboseMode": false,
        "cleanSession": false
      },
      "id": "metric-turnContextualPrecision",
      "name": "DeepEval Turn Contextual Precision",
      "type": "n8n-nodes-deepeval.deepEvalTurnContextualPrecision",
      "typeVersion": 1,
      "position": [1440, 480]
    },
    {
      "parameters": {
        "windowSize": 10,
        "expectedOutcome": "The assistant answers that 2 + 2 equals 4.",
        "threshold": 0.5,
        "includeReason": true,
        "strictMode": false,
        "asyncMode": true,
        "verboseMode": false,
        "cleanSession": false
      },
      "id": "metric-turnContextualRecall",
      "name": "DeepEval Turn Contextual Recall",
      "type": "n8n-nodes-deepeval.deepEvalTurnContextualRecall",
      "typeVersion": 1,
      "position": [1440, 600]
    },
    {
      "parameters": {
        "windowSize": 10,
        "threshold": 0.5,
        "includeReason": true,
        "strictMode": false,
        "asyncMode": true,
        "verboseMode": false,
        "cleanSession": false
      },
      "id": "metric-turnContextualRelevancy",
      "name": "DeepEval Turn Contextual Relevancy",
      "type": "n8n-nodes-deepeval.deepEvalTurnContextualRelevancy",
      "typeVersion": 1,
      "position": [1440, 720]
    }
  ],
  "connections": {
    "When clicking Execute Workflow": {
      "main": [
        [
          {
            "node": "Load Source Rows",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Load Source Rows": {
      "main": [
        [
          {
            "node": "DeepEval Trigger",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "DeepEval Trigger": {
      "main": [
        [
          {
            "node": "Enrich Evaluation Data",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Enrich Evaluation Data": {
      "main": [
        [
          {
            "node": "AI Agent",
            "type": "main",
            "index": 0
          },
          {
            "node": "Prepare Metric Input",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "AI Agent": {
      "main": [
        [
          {
            "node": "Prepare Metric Input",
            "type": "main",
            "index": 1
          }
        ]
      ]
    },
    "DeepEval Aggregate": {
      "main": [
        [
          {
            "node": "Persist Results",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Calculator": {
      "ai_tool": [
        [
          {
            "node": "AI Agent",
            "type": "ai_tool",
            "index": 0
          }
        ]
      ]
    },
    "OpenAI Chat Model": {
      "ai_languageModel": [
        [
          {
            "node": "AI Agent",
            "type": "ai_languageModel",
            "index": 0
          },
          {
            "node": "DeepEval Conversational G-Eval",
            "type": "ai_languageModel",
            "index": 0
          },
          {
            "node": "DeepEval Conversational DAG",
            "type": "ai_languageModel",
            "index": 0
          },
          {
            "node": "DeepEval Turn Relevancy",
            "type": "ai_languageModel",
            "index": 0
          },
          {
            "node": "DeepEval Role Adherence",
            "type": "ai_languageModel",
            "index": 0
          },
          {
            "node": "DeepEval Knowledge Retention",
            "type": "ai_languageModel",
            "index": 0
          },
          {
            "node": "DeepEval Conversation Completeness",
            "type": "ai_languageModel",
            "index": 0
          },
          {
            "node": "DeepEval Goal Accuracy",
            "type": "ai_languageModel",
            "index": 0
          },
          {
            "node": "DeepEval Tool Use",
            "type": "ai_languageModel",
            "index": 0
          },
          {
            "node": "DeepEval Topic Adherence",
            "type": "ai_languageModel",
            "index": 0
          },
          {
            "node": "DeepEval Turn Faithfulness",
            "type": "ai_languageModel",
            "index": 0
          },
          {
            "node": "DeepEval Turn Contextual Precision",
            "type": "ai_languageModel",
            "index": 0
          },
          {
            "node": "DeepEval Turn Contextual Recall",
            "type": "ai_languageModel",
            "index": 0
          },
          {
            "node": "DeepEval Turn Contextual Relevancy",
            "type": "ai_languageModel",
            "index": 0
          }
        ]
      ]
    },
    "Prepare Metric Input": {
      "main": [
        [
          {
            "node": "DeepEval Conversational G-Eval",
            "type": "main",
            "index": 0
          },
          {
            "node": "DeepEval Conversational DAG",
            "type": "main",
            "index": 0
          },
          {
            "node": "DeepEval Turn Relevancy",
            "type": "main",
            "index": 0
          },
          {
            "node": "DeepEval Role Adherence",
            "type": "main",
            "index": 0
          },
          {
            "node": "DeepEval Knowledge Retention",
            "type": "main",
            "index": 0
          },
          {
            "node": "DeepEval Conversation Completeness",
            "type": "main",
            "index": 0
          },
          {
            "node": "DeepEval Goal Accuracy",
            "type": "main",
            "index": 0
          },
          {
            "node": "DeepEval Tool Use",
            "type": "main",
            "index": 0
          },
          {
            "node": "DeepEval Topic Adherence",
            "type": "main",
            "index": 0
          },
          {
            "node": "DeepEval Turn Faithfulness",
            "type": "main",
            "index": 0
          },
          {
            "node": "DeepEval Turn Contextual Precision",
            "type": "main",
            "index": 0
          },
          {
            "node": "DeepEval Turn Contextual Recall",
            "type": "main",
            "index": 0
          },
          {
            "node": "DeepEval Turn Contextual Relevancy",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "DeepEval Conversational G-Eval": {
      "main": [
        [
          {
            "node": "Collect Metric Results",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "DeepEval Conversational DAG": {
      "main": [
        [
          {
            "node": "Collect Metric Results",
            "type": "main",
            "index": 1
          }
        ]
      ]
    },
    "DeepEval Turn Relevancy": {
      "main": [
        [
          {
            "node": "Collect Metric Results",
            "type": "main",
            "index": 2
          }
        ]
      ]
    },
    "DeepEval Role Adherence": {
      "main": [
        [
          {
            "node": "Collect Metric Results",
            "type": "main",
            "index": 3
          }
        ]
      ]
    },
    "DeepEval Knowledge Retention": {
      "main": [
        [
          {
            "node": "Collect Metric Results",
            "type": "main",
            "index": 4
          }
        ]
      ]
    },
    "DeepEval Conversation Completeness": {
      "main": [
        [
          {
            "node": "Collect Metric Results",
            "type": "main",
            "index": 5
          }
        ]
      ]
    },
    "DeepEval Goal Accuracy": {
      "main": [
        [
          {
            "node": "Collect Metric Results",
            "type": "main",
            "index": 6
          }
        ]
      ]
    },
    "DeepEval Tool Use": {
      "main": [
        [
          {
            "node": "Collect Metric Results",
            "type": "main",
            "index": 7
          }
        ]
      ]
    },
    "DeepEval Topic Adherence": {
      "main": [
        [
          {
            "node": "Collect Metric Results",
            "type": "main",
            "index": 8
          }
        ]
      ]
    },
    "DeepEval Turn Faithfulness": {
      "main": [
        [
          {
            "node": "Collect Metric Results",
            "type": "main",
            "index": 9
          }
        ]
      ]
    },
    "DeepEval Turn Contextual Precision": {
      "main": [
        [
          {
            "node": "Collect Metric Results",
            "type": "main",
            "index": 10
          }
        ]
      ]
    },
    "DeepEval Turn Contextual Recall": {
      "main": [
        [
          {
            "node": "Collect Metric Results",
            "type": "main",
            "index": 11
          }
        ]
      ]
    },
    "DeepEval Turn Contextual Relevancy": {
      "main": [
        [
          {
            "node": "Collect Metric Results",
            "type": "main",
            "index": 12
          }
        ]
      ]
    },
    "Collect Metric Results": {
      "main": [
        [
          {
            "node": "DeepEval Aggregate",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Simple Memory": {
      "ai_memory": [
        [
          {
            "node": "AI Agent",
            "type": "ai_memory",
            "index": 0
          },
          {
            "node": "DeepEval Conversational G-Eval",
            "type": "ai_memory",
            "index": 0
          },
          {
            "node": "DeepEval Conversational DAG",
            "type": "ai_memory",
            "index": 0
          },
          {
            "node": "DeepEval Turn Relevancy",
            "type": "ai_memory",
            "index": 0
          },
          {
            "node": "DeepEval Role Adherence",
            "type": "ai_memory",
            "index": 0
          },
          {
            "node": "DeepEval Knowledge Retention",
            "type": "ai_memory",
            "index": 0
          },
          {
            "node": "DeepEval Conversation Completeness",
            "type": "ai_memory",
            "index": 0
          },
          {
            "node": "DeepEval Goal Accuracy",
            "type": "ai_memory",
            "index": 0
          },
          {
            "node": "DeepEval Tool Use",
            "type": "ai_memory",
            "index": 0
          },
          {
            "node": "DeepEval Topic Adherence",
            "type": "ai_memory",
            "index": 0
          },
          {
            "node": "DeepEval Turn Faithfulness",
            "type": "ai_memory",
            "index": 0
          },
          {
            "node": "DeepEval Turn Contextual Precision",
            "type": "ai_memory",
            "index": 0
          },
          {
            "node": "DeepEval Turn Contextual Recall",
            "type": "ai_memory",
            "index": 0
          },
          {
            "node": "DeepEval Turn Contextual Relevancy",
            "type": "ai_memory",
            "index": 0
          }
        ]
      ]
    }
  },
  "settings": {
    "executionOrder": "v1"
  },
  "active": false,
  "pinData": {},
  "meta": {
    "templateCredsSetupCompleted": false
  },
  "tags": []
}