import { z } from "zod";
import { LLM } from "../llms/base";
import { MemoryItem, SearchResult } from "../types";
import { HistoryManager } from "../storage/base";

// Schema for agent decision making
export const SearchAgentDecisionSchema = z.object({
  needsAdditionalSteps: z.boolean().describe("Whether additional steps are needed to fully answer the query"),
  reasoning: z.string().nullable().default(null).describe("Brief explanation of why additional steps are needed or not"),
  needsAnotherHop: z.boolean().nullable().default(false).describe("Whether another complete analysis hop is needed after these steps"),
  hopReasoning: z.string().nullable().default(null).describe("Brief explanation of why another hop is needed or not"),
  nextSteps: z.array(
    z.object({
      action: z.enum([
        "get_next_messages",
        "get_prev_messages", 
        "get_entity_memories",
        "get_time_based_memories",
        "semantic_research",
        "get_message_details",
        "get_conversation_context",
        "search_all_messages",
        "get_messages_by_pattern",
        "deduce_memory_date",
        "finished"
      ]).describe("The type of action to take"),
      parameters: z.object({
        messageId: z.string().nullable().default(null).describe("Message ID for context operations"),
        memoryId: z.string().nullable().default(null).describe("Memory ID for getting detailed message content"),
        entityId: z.string().nullable().default(null).describe("Entity ID for entity-specific searches"),
        timeRange: z.union([
          z.string(),
          z.object({
            start: z.string().nullable().default(null),
            end: z.string().nullable().default(null)
          })
        ]).nullable().default(null).describe("Time range for time-based searches (string format 'start to end' or object with start/end)"),
        count: z.number().nullable().default(null).describe("Number of items to retrieve"),
        researchQuery: z.string().nullable().default(null).describe("Alternative query for semantic research or message search"),
        researchType: z.string().nullable().default(null).describe("Type of research: 'related', 'context', 'deeper', 'broader'"),
        includeMessages: z.boolean().nullable().default(false).describe("Whether to include full message details in semantic research (default: false)"),
        exact: z.boolean().nullable().default(false).describe("Whether to use exact match for pattern searches (default: false)"),
        dateContext: z.string().nullable().default(null).describe("Context about what date/time period to focus on for date deduction"),
      }).nullable().default({}).describe("Parameters for the action"),
      priority: z.number().min(1).max(5).nullable().default(3).describe("Priority level (1-5, 5 being highest)"),
    })
  ).nullable().default([]).describe("List of additional steps to take"),
});

export interface SearchAgentOptions {
  userId?: string;
  agentId?: string;
  runId?: string;
  maxSteps?: number;
  enableMultihop?: boolean; // Enable multihop analysis
  maxHops?: number; // Maximum number of analysis hops
  reasoningCustomInstructions?: string; // Custom instructions for reasoning about search completeness
}

export interface SearchFunction {
  (query: string, options?: any): Promise<SearchResult>;
}

// Global agent statistics tracking
export interface AgentStatistics {
  totalHops: number;
  totalAgentCalls: number;
  agentCallsByName: Record<string, number>;
  totalStepsExecuted: number;
  successfulSteps: number;
  additionalResultsFound: number;
  startTime: number;
  endTime?: number;
}

export class SearchAgents {
  // IMPORTANT: Search agents use reasoningCustomInstructions for determining search completeness
  // but remain objective during search execution. Custom instructions guide the reasoning process
  // about whether the current results are sufficient to answer the user's query
  private statistics: AgentStatistics = {
    totalHops: 0,
    totalAgentCalls: 0,
    agentCallsByName: {},
    totalStepsExecuted: 0,
    successfulSteps: 0,
    additionalResultsFound: 0,
    startTime: Date.now()
  };

  constructor(
    private llm: LLM,
    private db: HistoryManager,
    private searchFunction?: SearchFunction,
  ) {}

  private trackAgentCall(agentName: string): void {
    this.statistics.totalAgentCalls++;
    this.statistics.agentCallsByName[agentName] = (this.statistics.agentCallsByName[agentName] || 0) + 1;
  }

  private logSummary(): void {
    this.statistics.endTime = Date.now();
    const duration = this.statistics.endTime - this.statistics.startTime;
    
    console.log(`\n🎯 [Search Agents Summary]`);
    console.log(`   📊 Total Hops: ${this.statistics.totalHops}`);
    console.log(`   🤖 Total Agent Calls: ${this.statistics.totalAgentCalls}`);
    console.log(`   ⚡ Steps Executed: ${this.statistics.totalStepsExecuted} (${this.statistics.successfulSteps} successful)`);
    console.log(`   📈 Additional Results Found: ${this.statistics.additionalResultsFound}`);
    console.log(`   ⏱️  Processing Time: ${duration}ms`);
    
    if (Object.keys(this.statistics.agentCallsByName).length > 0) {
      console.log(`   🔧 Agent Calls by Type:`);
      Object.entries(this.statistics.agentCallsByName)
        .sort(([,a], [,b]) => b - a)
        .forEach(([name, count]) => {
          console.log(`      • ${name}: ${count} time(s)`);
        });
    }
  }

  public finalizeSingleHopAnalysis(): void {
    // Set hop count to 1 for single-hop analysis
    this.statistics.totalHops = 1;
    this.logSummary();
  }

  // Always add get_messages_by_pattern for enhanced context when we have messageIds
  private addPatternSearchIfNeeded(
    existingSteps: Array<{ action: string; parameters: Record<string, any>; priority: number; }>,
    searchResult: SearchResult,
    options: SearchAgentOptions
  ): Array<{ action: string; parameters: Record<string, any>; priority: number; }> {
    // Check if get_messages_by_pattern is already included
    const hasPatternSearch = existingSteps.some(step => step.action === 'get_messages_by_pattern');
    
    if (hasPatternSearch) {
      return existingSteps;
    }

    // Find any messageId from the search results
    const messageIds = searchResult.results
      .map(result => result.metadata?.messageId)
      .filter(Boolean);

    if (messageIds.length > 0) {
      // Use the first available messageId for pattern search
      const patternStep = {
        action: 'get_messages_by_pattern' as const,
        parameters: {
          messageId: messageIds[0],
          exact: false,
          count: 10, // Get more context messages
        },
        priority: 3 // Medium priority for enhanced context
      };

      console.log(`🔄 Auto-adding get_messages_by_pattern for enhanced context (messageId: ${messageIds[0]?.substring(0, 8)}...)`);
      
      // Add to existing steps, maintaining priority order
      return [...existingSteps, patternStep].sort((a, b) => b.priority - a.priority);
    }

    return existingSteps;
  }

  async analyzeSearchNeed(
    query: string,
    searchResult: SearchResult,
    options: SearchAgentOptions,
    hopNumber: number = 1,
  ): Promise<SearchAgentDecision> {
    console.log(`🤖 Analyzing search needs for: "${query}" (${searchResult.results.length} initial results)`);
    
    this.trackAgentCall('analyzeSearchNeed');

    let basePrompt = `You are an intelligent search agent analyzing whether additional information gathering is needed to provide a COMPLETE, OBJECTIVE, and SPECIFIC answer to the user's query. Your mission is to transform vague answers into precise, factual responses.

## CORE OBJECTIVES:
1. **SPECIFICITY OVER GENERALITY**: Convert vague terms to concrete details
2. **OBJECTIVE FACTS**: Find exact dates, names, locations, quantities instead of relative terms
3. **COMPLETE CONTEXT**: Ensure no important details are missing from the answer
4. **RESOLUTION OF AMBIGUITY**: Clarify unclear references using available context

## CRITICAL FOCUS AREAS:
- **Temporal Vagueness**: "last year" → exact year/date, "recently" → specific timeframe
- **Entity Vagueness**: "home country" → specific country name, "that person" → actual name
- **Action Vagueness**: "loves doing that" → specific activity/hobby
- **Location Vagueness**: "over there" → specific place/address
- **Quantity Vagueness**: "a lot" → specific numbers/amounts`;

    // Add reasoning custom instructions if provided
    if (options.reasoningCustomInstructions) {
      basePrompt += `\n\n## Enhanced Custom Reasoning Instructions
${options.reasoningCustomInstructions}

IMPORTANT: These instructions should be combined with the core objectives above. Focus on finding SPECIFIC, OBJECTIVE answers rather than accepting vague or general responses.`;
    }

    const prompt = basePrompt + `

## Analysis Context
- Current hop: ${hopNumber}
- Max hops allowed: ${options.maxHops || 3}
- Query: "${query}"
- Current search results: ${searchResult.results.length} items found

## Current Search Results Summary
${searchResult.results.length === 0 ? 
  `🚨 NO MEMORIES FOUND - This is a perfect case for search_all_messages!
   
   When no memories exist, you should:
   1. Use search_all_messages with a broad query to find ALL user messages
   2. For queries like "everything the user said", use "" (empty string) or ".*" to match all content
   3. The system will filter by userId automatically to get all messages from that user
   4. This finds casual conversations that never became memories!` 
  : 
  searchResult.results.slice(0, 5).map((item, idx) => {
    const hasMessageId = item.metadata?.messageId;
    const hasEntityIds = item.entityIds && item.entityIds.length > 0;
    const availableActions = [];
    if (hasMessageId) availableActions.push('get_prev_messages', 'get_next_messages', 'get_conversation_context');
    if (item.id) availableActions.push('get_message_details');
    if (hasEntityIds) availableActions.push('get_entity_memories');
    
    return `${idx + 1}. ${item.memory} (Score: ${item.score?.toFixed(3)})
       📊 Memory ID: ${item.id?.substring(0, 8) || 'unknown'}...
       ${hasMessageId ? `🔗 Message ID: ${item.metadata!.messageId?.substring(0, 8) || 'unknown'}...` : '❌ No message ID'}
       ${hasEntityIds ? `🏷️ Entities: ${item.entityIds!.join(', ')}` : '❌ No entities'}
       ⚡ Available: ${availableActions.length > 0 ? availableActions.join(', ') : 'semantic_research only'}`;
  }).join('\n\n')}

${searchResult.results.length > 5 ? `\n... and ${searchResult.results.length - 5} more results` : ''}

## Action Selection Strategy
Choose actions based on the TYPE of information gap you've identified:

### 🕐 TEMPORAL GAPS (when information spans time)
- **get_prev_messages**: When you need context that happened BEFORE a specific moment
  - Use when: "What led up to this?" or "What was discussed earlier?"
  - Requires: messageId from search results
- **get_next_messages**: When you need context that happened AFTER a specific moment  
  - Use when: "What happened next?" or "How did this resolve?"
  - Requires: messageId from search results

### 🔍 DETAIL GAPS (when you need more specifics)
- **get_message_details**: When search results seem relevant but lack detail
  - Use when: Results mention something important but content is truncated
  - Requires: memoryId from search results
- **get_conversation_context**: When you need the full conversation around a memory
  - Use when: Need to understand the discussion flow around a specific topic
  - Requires: memoryId from search results

### 🏷️ ENTITY/TOPIC GAPS (when information is scattered)
- **get_entity_memories**: When query involves specific people, places, or organizations
  - Use when: Looking for all information about a specific entity
  - Requires: entityId from existing entities
- **semantic_research**: When you need alternative perspectives or related topics
  - Use when: Current results are too narrow, need broader or deeper context
  - LAST RESORT: Try other actions first

### 🗂️ COMPREHENSIVE GAPS (when normal search misses things)
- **search_all_messages**: Search through ALL stored messages (not just memories)
  - **CRITICAL USE CASES:**
    - **NO MEMORIES FOUND**: When search returns 0 results, always try this first!
    - Query asks for "everything [person] said" or "everything the user said"
    - Query asks for "all messages/conversations from [person]"
    - Query asks to "list everything" about someone
    - Few memory results but query seems conversational
    - Looking for casual mentions that didn't become memories
  - **Parameters for "everything user said" queries**: 
    - Use researchQuery: "" (empty string) or ".*" to match ALL content
    - The system will automatically filter by userId to get all user messages
    - Set priority: 5 (highest) for comprehensive user queries
  - **Parameters for person-specific queries**: 
    - Use researchQuery: "[person's name]" (the actual name)
    - Set priority: 4-5 depending on importance
- **get_messages_by_pattern**: Complete session reconstruction from message IDs
  - Use when: Need to see everything from a specific conversation session
  - Requires: priority ≥ 4, messageId pattern

### 📅 TEMPORAL GAPS (when you need specific dates/timing)
- **deduce_memory_date**: Analyze memory content and messages to extract specific dates
  - **USE CASES:**
    - Query asks "when did [event] happen?" or "what date was [event]?"
    - Found relevant memory but lacks specific date information
    - Need to determine exact timing from conversational context
    - Memory has createdAt but actual event happened at different time
  - **How it works:**
    - Analyzes memory content for temporal references ("yesterday", "last week", "on Monday")
    - Examines associated messages for date clues
    - Compares with memory metadata (createdAt, updatedAt, happenedAt)
    - Uses LLM to deduce actual event date from context
  - **Parameters:**
    - memoryId: The memory to analyze for date information
    - dateContext: Optional context about what timeframe to focus on
    - priority: 4-5 for date-specific queries

## Your Enhanced Analysis Process
1. **Evaluate Answer Completeness**: Does the current information provide SPECIFIC, OBJECTIVE answers?
2. **Identify Vagueness Gaps**: What vague terms need to be converted to concrete details?
3. **Check for Missing Context**: What additional information could clarify ambiguous references?
4. **Match gap to action type**: Use the strategy guide above, prioritizing specificity
5. **Validate Parameters**: Ensure you have required messageId, memoryId, entityId, etc.
6. **Prioritize by Specificity Impact**: How much will this action improve answer precision?

## VAGUENESS DETECTION CHECKLIST:
❌ **REJECT these vague patterns in current results:**
- Temporal: "recently", "last year", "a while ago", "yesterday" (without context)
- Entities: "home country", "that person", "the company", "that place"  
- Actions: "doing that", "the usual", "that thing", "the activity"
- Quantities: "a lot", "some", "many", "few"
- Locations: "over there", "that place", "the location"

✅ **SEEK these specific alternatives:**
- Exact dates, years, timeframes
- Proper names, specific titles, company names
- Detailed activity descriptions, specific actions
- Precise numbers, measurements, quantities
- Full addresses, geographic locations

## Enhanced Decision Examples

### SPECIFICITY-FOCUSED DECISIONS:
- **Answer says "home country" but doesn't specify** → get_message_details + search_all_messages (find "India")
- **Answer says "last year" without specific year** → deduce_memory_date (convert to 2023/2024)
- **Answer says "loves doing that" without naming activity** → get_conversation_context (find specific hobby)
- **Answer mentions "the company" without name** → get_entity_memories (find actual company name)
- **Answer says "recently" without timeframe** → deduce_memory_date + get_prev_messages (get exact dates)

### TRADITIONAL PATTERN DECISIONS:
- Missing temporal context → get_prev_messages/get_next_messages
- Results too brief → get_message_details  
- Need entity-specific info → get_entity_memories
- Need conversation flow → get_conversation_context
- Results completely off-topic → semantic_research
- **Query asks for "everything [person] said" → search_all_messages (HIGH PRIORITY)**
- **Query asks for "all messages from [person]" → search_all_messages (HIGH PRIORITY)**
- **Query asks for "list all/everything" → search_all_messages (HIGH PRIORITY)**
- **Few/no memory results but query seems conversational → search_all_messages**
- **Query asks "when did [event] happen?" → deduce_memory_date (HIGH PRIORITY)**
- **Query asks "what date was [event]?" → deduce_memory_date (HIGH PRIORITY)**
- **Found memory but need specific timing → deduce_memory_date**
- **Need to convert relative dates ("yesterday") to actual dates → deduce_memory_date**

### VAGUENESS RESOLUTION PRIORITY:
🔥 **ALWAYS PRIORITY 5 (CRITICAL):**
- Converting relative dates to specific dates
- Finding specific names for vague entity references
- Clarifying "that thing/activity" to exact descriptions

🔶 **PRIORITY 4 (HIGH):**
- Getting additional context for ambiguous terms
- Finding supporting details for incomplete answers
- Cross-referencing entities for complete information

## Important Guidelines
- **DIVERSIFY your actions** - don't just use semantic_research for everything
- **Use available messageIds and memoryIds** from current results when possible
- **Match action to the specific information gap** you've identified
- Be specific about WHY you're choosing each action
- Prioritize by potential impact (1-5 scale, where 5 = critical)
- Maximum ${options.maxSteps || 3} actions per hop

## 🚨 IMPORTANT: Message vs Memory Search Patterns
**Use search_all_messages for these query patterns (BUT FIRST MAKE SURE THAT YOU HAVE USED THE ENTITY SEARCH FIRST):**
- "everything that [name] said"
- "all messages from [name]"  
- "list everything [name] mentioned"
- "show me all conversations with [name]"
- "what did [name] say" (when memory results are sparse)
- Any query requesting comprehensive/complete information about a person's messages

ALSO REMEMBER THAT SEARCH_ALL_MESSAGES IS THE MOST EXPENSIVE ACTION TO TAKE. AS THIS ADDS A LOT OF MESSAGES TO THE SEARCH RESULT. WHICH IS WHY YOU SHOULD USE IT AS LAST RESORT.
YOU CAN ALSO USE THIS IN CASES WHEN A USER ASKS FOR SOMETHING SPECIFIC AND YOU HAVE NO MEMORIES TO SHOW THEM.

**Memory search misses casual conversations that didn't create memories!**

## Your Response
Analyze the search results and determine if additional steps are needed. If so, select the most appropriate diverse actions based on the available information and the specific gaps you identify.`;

    try {
      const response = await this.llm.generateResponse(
        [{ role: "user", content: prompt }],
        undefined, // No response_format needed
        undefined, // No tools
        SearchAgentDecisionSchema // Pass the zod schema
      );

      // Handle structured output response
      if (typeof response === 'object' && response.parsed) {
        const validated = response.parsed;
        
        // Always add get_messages_by_pattern for enhanced context if we have messageIds
        const enhancedSteps = this.addPatternSearchIfNeeded(
          (validated.nextSteps || []).map((step: any) => ({
            action: step.action,
            parameters: step.parameters || {},
            priority: step.priority || 3
          })),
          searchResult,
          options
        );
        
        return {
          needsAdditionalSteps: validated.needsAdditionalSteps || enhancedSteps.length > 0,
          reasoning: validated.reasoning || "Analysis completed",
          needsAnotherHop: options.enableMultihop ? (validated.needsAnotherHop || false) : false,
          hopReasoning: validated.hopReasoning || "No hop reasoning provided",
          nextSteps: enhancedSteps
        };
      }

      // Fallback for non-structured response (shouldn't happen with structured output)
      const cleanResponse = removeCodeBlocks(response as string);
      
      let parsed: any;
      try {
        parsed = JSON.parse(cleanResponse);
      } catch (parseError) {
        console.error("❌ Failed to parse LLM analysis response");
        // Return safe fallback
        return {
          needsAdditionalSteps: false,
          reasoning: "Failed to parse LLM analysis response",
          needsAnotherHop: false,
          hopReasoning: "Parser error occurred",
          nextSteps: []
        };
      }
      
      // Validate against schema with better error handling
      let validated: any;
      try {
        validated = SearchAgentDecisionSchema.parse(parsed);
      } catch (schemaError) {
        console.error("⚠️ Schema validation failed, using fallback");
        // Extract what we can from the parsed response and provide defaults
        let sanitizedSteps: any[] = [];
        if (Array.isArray(parsed.nextSteps)) {
          sanitizedSteps = parsed.nextSteps
            .filter((step: any) => step && typeof step === 'object')
            .map((step: any) => ({
              action: ['get_next_messages', 'get_prev_messages', 'get_entity_memories', 'get_time_based_memories', 'semantic_research', 'finished'].includes(step.action) 
                ? step.action 
                : 'finished',
              parameters: typeof step.parameters === 'object' ? step.parameters : {},
              priority: typeof step.priority === 'number' && step.priority >= 1 && step.priority <= 5 
                ? step.priority 
                : 3
            }))
            .slice(0, 5); // Limit to 5 steps max
        }
        
        // Always add get_messages_by_pattern for enhanced context if we have messageIds
        const enhancedSteps = this.addPatternSearchIfNeeded(sanitizedSteps, searchResult, options);
        
        return {
          needsAdditionalSteps: Boolean(parsed.needsAdditionalSteps) || enhancedSteps.length > 0,
          reasoning: String(parsed.reasoning || "Analysis completed"),
          needsAnotherHop: options.enableMultihop ? Boolean(parsed.needsAnotherHop) : false,
          hopReasoning: String(parsed.hopReasoning || "No hop reasoning provided"),
          nextSteps: enhancedSteps
        };
      }
      
      // Always add get_messages_by_pattern for enhanced context if we have messageIds
      const enhancedSteps = this.addPatternSearchIfNeeded(validated.nextSteps || [], searchResult, options);
      
      return {
        needsAdditionalSteps: validated.needsAdditionalSteps || enhancedSteps.length > 0,
        reasoning: validated.reasoning || "Analysis completed",
        needsAnotherHop: options.enableMultihop ? (validated.needsAnotherHop || false) : false,
        hopReasoning: validated.hopReasoning || "No hop reasoning provided",
        nextSteps: enhancedSteps
      };
    } catch (error) {
      console.error("❌ Error in search agent analysis:", error);
      return {
        needsAdditionalSteps: false,
        reasoning: "Error occurred during analysis",
        needsAnotherHop: false,
        hopReasoning: "Error prevented hop analysis",
        nextSteps: []
      };
    }
  }

  async executeMultihopAnalysis(
    query: string,
    initialSearchResult: SearchResult,
    options: SearchAgentOptions
  ): Promise<{
    finalResult: SearchResult;
    hopResults: Array<{
      hopNumber: number;
      decision: SearchAgentDecision;
      stepResults: Array<{ action: string; results: any; success: boolean; }>;
      additionalResults: any[];
    }>;
  }> {
    console.log(`🚀 Starting multihop analysis (max ${options.maxHops || 3} hops)`);
    
    let currentResult = initialSearchResult;
    const hopResults = [];
    const maxHops = options.maxHops || 3;
    
    for (let hop = 1; hop <= maxHops; hop++) {
      this.statistics.totalHops = hop;
      console.log(`🔄 Hop ${hop}/${maxHops}...`);
      
      // Analyze current results
      const decision = await this.analyzeSearchNeed(query, currentResult, options, hop);
      
      let stepResults: Array<{ action: string; results: any; success: boolean; }> = [];
      let additionalResults: any[] = [];
      
      // Execute additional steps if needed
      if (decision.needsAdditionalSteps) {
        console.log(`⚡ Executing ${decision.nextSteps.length} additional steps...`);
        const stepResult = await this.executeSteps(decision, options, false, query); // Don't log summary here, will be logged at end
        stepResults = stepResult.stepResults;
        additionalResults = stepResult.additionalResults;
        
        // Merge additional results into current result
        currentResult = {
          ...currentResult,
          results: [...currentResult.results, ...additionalResults]
        };
        
        this.statistics.additionalResultsFound += additionalResults.length;
      } else {
        console.log(`✅ No additional steps needed for hop ${hop}`);
      }
      
      hopResults.push({
        hopNumber: hop,
        decision,
        stepResults,
        additionalResults
      });
      
      // Check if we need another hop
      if (!decision.needsAnotherHop || !options.enableMultihop) {
        break;
      }
      
      if (hop === maxHops) {
        console.log(`⚠️ Reached maximum hops (${maxHops})`);
        break;
      }
      
      // Add delay between hops
      await new Promise(resolve => setTimeout(resolve, 200));
    }
    
    // Log the summary at the end
    this.logSummary();
    
    return {
      finalResult: currentResult,
      hopResults
    };
  }

  async executeSteps(
    decision: SearchAgentDecision,
    options: SearchAgentOptions,
    shouldLogSummary: boolean = false,
    query: string = "search" // Add query for filtering context
  ): Promise<{
    stepResults: Array<{ action: string; results: any; success: boolean; }>;
    additionalResults: any[];
  }> {
    const stepResults: Array<{ action: string; results: any; success: boolean; }> = [];
    const additionalResults: any[] = [];
    
    // Sort steps by priority (highest first)
    const sortedSteps = decision.nextSteps.sort((a, b) => b.priority - a.priority);
    const maxSteps = Math.min(sortedSteps.length, options.maxSteps || 3);
    
    // Encourage action diversity - track action types used
    const actionTypesUsed = new Set<string>();
    const actionCategories = {
      temporal: ['get_prev_messages', 'get_next_messages'],
      detail: ['get_message_details', 'get_conversation_context'],
      entity: ['get_entity_memories'],
      semantic: ['semantic_research'],
      comprehensive: ['search_all_messages', 'get_messages_by_pattern']
    };
    
    for (let i = 0; i < maxSteps; i++) {
      const step = sortedSteps[i];
      this.statistics.totalStepsExecuted++;
      this.trackAgentCall(step.action);
      
      // Log action selection rationale
      const category = Object.entries(actionCategories).find(([_, actions]) => 
        actions.includes(step.action)
      )?.[0] || 'other';
      
      // Track diversity
      actionTypesUsed.add(category);
      
      try {
        let results: any[] = [];
        let success = false;
        
        switch (step.action) {
          case "get_next_messages":
            if (step.parameters.messageId) {
              results = await this.db.getMessageContext(
                step.parameters.messageId,
                {
                  beforeCount: 0,
                  afterCount: step.parameters.count || 5,
                  userId: options.userId,
                  runId: options.runId
                }
              );
              success = true;
            }
            break;
            
          case "get_prev_messages":
            if (step.parameters.messageId) {
              results = await this.db.getMessageContext(
                step.parameters.messageId,
                {
                  beforeCount: step.parameters.count || 5,
                  afterCount: 0,
                  userId: options.userId,
                  runId: options.runId
                }
              );
              success = true;
            }
            break;
            
          case "semantic_research":
            if (this.searchFunction && step.parameters.researchQuery) {
              try {
                const researchResult = await this.searchFunction(
                  step.parameters.researchQuery,
                  {
                    userId: options.userId,
                    limit: step.parameters.count || 5,
                    useSearchAgents: false, // Prevent recursive agent calls
                  }
                );
                
                // Return concise results by default, full details only if explicitly requested
                const includeMessages = step.parameters.includeMessages === true;
                
                results = researchResult.results.map(r => {
                  const baseResult = {
                  type: 'semantic_research',
                  query: step.parameters.researchQuery,
                  researchType: step.parameters.researchType || 'related',
                    id: r.id,
                    memory: r.memory,
                    score: r.score,
                    createdAt: r.createdAt,
                    updatedAt: r.updatedAt,
                    entityIds: r.entityIds || [],
                    userId: r.userId,
                    metadata: {
                      messageId: r.metadata?.messageId,
                      // Only include essential metadata
                      ...(r.metadata && Object.fromEntries(
                        Object.entries(r.metadata).filter(([key]) => 
                          !['messageId'].includes(key) && typeof r.metadata![key] !== 'object'
                        )
                      ))
                    }
                  };
                  
                  // Only include full messages and context if explicitly requested
                  if (includeMessages) {
                    return {
                      ...baseResult,
                      messages: r.messages,
                      context: r.context
                    };
                  }
                  
                  return baseResult;
                });
                
                success = researchResult.results.length > 0;
              } catch (error) {
                success = false;
              }
            }
            break;
            
          case "get_entity_memories":
            if (step.parameters.entityId) {
              try {
                // First get the entity to verify it exists
                const entity = await this.db.getEntity(step.parameters.entityId);
                if (entity) {
                  // Get all entities for this user/run to find related memories
                  const allEntities = await this.db.getAllEntities({
                    userId: options.userId,
                    runId: options.runId
                  });
                  
                  // Find entities with the same entityId
                  const matchingEntities = allEntities.filter(e => 
                    e.entity_id === step.parameters.entityId
                  );
                  
                  results = matchingEntities.map(entity => ({
                    type: 'entity_memory',
                    entityId: entity.entity_id,
                    label: entity.label,
                    entityType: entity.type,
                    userId: entity.user_id,
                    runId: entity.run_id,
                    createdAt: entity.created_at,
                    updatedAt: entity.updated_at
                  }));
                  
                  success = results.length > 0;
                } else {
                  success = false;
                }
              } catch (error) {
                success = false;
              }
            } else {
            success = false;
            }
            break;
            
          case "get_time_based_memories":
            try {
              // This can search messages by time range
              let timeRangeParam: { start?: string; end?: string } | undefined;
              if (step.parameters.timeRange) {
                if (typeof step.parameters.timeRange === 'string') {
                  const parts = step.parameters.timeRange.split(' to ');
                  timeRangeParam = {
                    start: parts[0]?.trim(),
                    end: parts[1]?.trim()
                  };
                } else {
                  timeRangeParam = step.parameters.timeRange;
                }
                
                // Use searchMessages with a broad query to get time-filtered results
                const timeFilteredMessages = await this.db.searchMessages(
                  step.parameters.researchQuery || ".*", // Use regex to match all if no specific query
                  {
                    userId: options.userId,
                    runId: options.runId,
                    limit: step.parameters.count || 10,
                    timeRange: timeRangeParam
                  }
                );
                
                results = timeFilteredMessages.map(msg => ({
                  type: 'time_based_memory',
                  messageId: msg.message_id,
                  content: msg.content,
                  role: msg.role,
                  userId: msg.user_id,
                  runId: msg.run_id,
                  happenedAt: msg.happened_at,
                  messageIndex: msg.message_index,
                  timeRange: timeRangeParam ? `${timeRangeParam.start || 'start'} to ${timeRangeParam.end || 'end'}` : 'all time'
                }));
                
                success = results.length > 0;
              } else {
                success = false;
              }
            } catch (error) {
              success = false;
            }
            break;
            
          case "get_message_details":
            if (step.parameters.memoryId) {
              results = await this.db.getMessageDetails(step.parameters.memoryId);
              success = results.length > 0;
            }
            break;
            
          case "get_conversation_context":
            if (step.parameters.memoryId) {
              results = await this.db.getMessageConversationContext(step.parameters.memoryId);
              success = results.length > 0;
            }
            break;
            
          case "search_all_messages":
            // Allow search_all_messages for any query with researchQuery, with flexible priority
            if (step.parameters.researchQuery !== undefined) {
              try {
                // Handle timeRange parameter - can be string or object
                let timeRangeParam;
                if (step.parameters.timeRange) {
                  if (typeof step.parameters.timeRange === 'string') {
                    const parts = step.parameters.timeRange.split(' to ');
                    timeRangeParam = {
                      start: parts[0]?.trim(),
                      end: parts[1]?.trim()
                    };
                  } else {
                    timeRangeParam = step.parameters.timeRange;
                  }
                }

                // Smart query detection: if the research query looks like a person's name or userId,
                // and we're asking for "everything they said", search by userId instead of content
                const searchQuery = step.parameters.researchQuery;
                const isPersonQuery = /^[a-zA-Z_][a-zA-Z0-9_]*$/.test(searchQuery) && searchQuery.length < 30; // Looks like a username/name
                
                let searchResults = [];
                
                if (isPersonQuery && options.userId) {
                  // If it looks like a person query and we have a userId context, try both approaches
                  // Strategy 1: Search by exact userId match (case-insensitive)
                  const userIdVariants = [searchQuery, searchQuery.toLowerCase(), searchQuery.toUpperCase()];
                  let userIdMatched = false;
                  
                  for (const variant of userIdVariants) {
                    if (variant === options.userId) {
                      userIdMatched = true;
                      
                      // Get ALL messages from this user by using getAllMessages and filtering
                      try {
                        const allUserMessages = await this.db.getAllMessages({
                          userId: options.userId,
                          runId: options.runId,
                          limit: step.parameters.count || 50
                        });
                        
                        // Filter for user messages only (role: "user")
                        searchResults = allUserMessages
                          .filter(msg => msg.role === 'user')
                          .map(msg => ({
                            message_id: msg.message_id || msg.id,
                            content: msg.content || msg.new_value || msg.previous_value,
                            role: msg.role,
                            user_id: msg.user_id || msg.userId,
                            run_id: msg.run_id || msg.runId,
                            happened_at: msg.happened_at || msg.happenedAt || msg.created_at,
                            message_index: msg.message_index || 0
                          }));
                      } catch (error) {
                        // Fallback: try a broad search query
                        searchResults = await this.db.searchMessages(
                          "", // Empty string might work better than ".*"
                          {
                            userId: options.userId,
                            runId: options.runId,
                            limit: step.parameters.count || 50,
                            timeRange: timeRangeParam
                          }
                        );
                      }
                      break;
                    }
                  }
                  
                  // Strategy 2: If no userId match, search content for the name
                  if (!userIdMatched) {
                    searchResults = await this.db.searchMessages(
                      searchQuery,
                      {
                        userId: options.userId,
                        runId: options.runId,
                        limit: step.parameters.count || 20,
                        timeRange: timeRangeParam
                      }
                    );
                  }
                } else {
                  // Regular content search for non-person queries
                  searchResults = await this.db.searchMessages(
                    searchQuery,
                    {
                      userId: options.userId,
                      runId: options.runId,
                      limit: step.parameters.count || 20,
                      timeRange: timeRangeParam
                    }
                  );
                }
                
                results = searchResults.map(msg => ({
                  type: 'all_messages_search',
                  messageId: msg.message_id,
                  content: msg.content,
                  role: msg.role,
                  userId: msg.user_id,
                  runId: msg.run_id,
                  happenedAt: msg.happened_at,
                  messageIndex: msg.message_index,
                  searchQuery: step.parameters.researchQuery // Include the search query used
                }));
                success = searchResults.length > 0;
              } catch (error) {
                success = false;
              }
            } else {
              success = false;
            }
            break;
            
          case "get_messages_by_pattern":
            // Only allow if we have a message ID and high priority (this should be rare)
            if (step.parameters.messageId && (step.priority || 1) >= 4) {
              try {
                const patternResults = await this.db.getMessagesByPattern(
                  step.parameters.messageId,
                  {
                    userId: options.userId,
                    runId: options.runId,
                    exact: step.parameters.exact || false
                  }
                );
                results = patternResults.map(msg => ({
                  type: 'message_pattern_search',
                  messageId: msg.message_id,
                  content: msg.content,
                  role: msg.role,
                  userId: msg.user_id,
                  runId: msg.run_id,
                  happenedAt: msg.happened_at,
                  messageIndex: msg.message_index
                }));
                success = patternResults.length > 0;
              } catch (error) {
                success = false;
              }
            } else {
              success = false;
            }
            break;

          case "deduce_memory_date":
            if (step.parameters.memoryId) {
              try {
                // Get the memory details and analyze for date information
                const dateAnalysis = await this.deduceMemoryDate(
                  step.parameters.memoryId,
                  {
                    userId: options.userId,
                    runId: options.runId,
                    dateContext: step.parameters.dateContext
                  }
                );
                
                if (dateAnalysis) {
                  results = [dateAnalysis];
                  success = true;
                } else {
                  success = false;
                }
              } catch (error) {
                console.error("Date deduction failed:", error);
                success = false;
              }
            } else {
              success = false;
            }
            break;
            
          case "finished":
            success = true;
            break;
            
          default:
            break;
        }
        
        if (success) {
          this.statistics.successfulSteps++;
        }
        
        stepResults.push({
          action: step.action,
          results,
          success
        });
        
        if (success && results.length > 0) {
          const newResults = results.map(r => {
            // For search_all_messages results, format them as proper memory-like items
            if (r.type === 'all_messages_search') {
              return {
                id: `msg-${r.messageId}`, // Unique ID for message
                memory: r.content, // The actual message content
                score: 1.0, // High score since it's an exact match
                type: 'raw_message', // Mark as raw message, not memory
                role: r.role,
                messageId: r.messageId,
                userId: r.userId,
                runId: r.runId,
                happenedAt: r.happenedAt,
                messageIndex: r.messageIndex,
                searchQuery: r.searchQuery,
                metadata: {
                  source: 'all_messages_search',
                  messageId: r.messageId,
                  searchQuery: r.searchQuery
                }
              };
            }

            // For date analysis results, format them with temporal information
            if (r.type === 'date_analysis') {
              return {
                id: `date-analysis-${r.memoryId}`,
                memory: `Date Analysis: ${r.originalMemory}`,
                score: 1.0,
                type: 'date_analysis',
                memoryId: r.memoryId,
                dateAnalysis: r.analysis,
                metadata: {
                  source: 'date_deduction_agent',
                  actualDate: r.analysis.actualDate,
                  confidence: r.analysis.confidence,
                  reasoning: r.analysis.reasoning,
                  evidence: r.analysis.evidence,
                  ...r.metadata
                }
              };
            }
            
            // For other types, use the original format
            return {
              type: step.action,
              content: r.new_value || r.previous_value || r.content || JSON.stringify(r),
              metadata: r
            };
          });
          
          // Filter additional results to prevent overwhelming context
          const filteredNewResults = await this.filterAdditionalResults(
            query, 
            additionalResults, 
            newResults, 
            5
          );
          
          additionalResults.push(...filteredNewResults);
        }
        
      } catch (error) {
        stepResults.push({
          action: step.action,
          results: [],
          success: false
        });
      }
    }
    
    // Log summary if requested (for single-hop scenarios)
    if (shouldLogSummary) {
      this.logSummary();
    }
    
    return { stepResults, additionalResults };
  }

  // Deduce specific dates from memory content and associated messages
  private async deduceMemoryDate(
    memoryId: string,
    options: {
      userId?: string;
      runId?: string;
      dateContext?: string;
    }
  ): Promise<any | null> {
    try {
      this.trackAgentCall('deduceMemoryDate');

      // Get memory details including messages
      const memoryDetails = await this.db.getMessageDetails(memoryId);
      if (!memoryDetails || memoryDetails.length === 0) {
        return null;
      }

      // Get the memory conversation context for additional date clues
      const conversationContext = await this.db.getMessageConversationContext(memoryId);
      
      // Combine all available information for date analysis
      const memoryInfo = memoryDetails[0];
      const allMessages = [...memoryDetails, ...conversationContext];

      // Build context for LLM analysis
      const dateAnalysisPrompt = `You are a temporal analysis expert. Analyze the provided memory and its context to deduce the most accurate date when the described event actually occurred.

## Memory Information
**Memory ID**: ${memoryId}
**Content**: ${memoryInfo.new_value || memoryInfo.content || 'N/A'}
**Created At**: ${memoryInfo.created_at || 'Unknown'}
**Happened At**: ${memoryInfo.happened_at || 'Unknown'}
**Previous Value**: ${memoryInfo.previous_value || 'N/A'}

## Associated Messages & Context
${allMessages.length > 0 ? allMessages.map((msg, idx) => 
  `${idx + 1}. [${msg.happened_at || msg.created_at || 'Unknown time'}] ${msg.new_value || msg.content || msg.previous_value || 'No content'}`
).join('\n') : 'No additional messages found'}

${options.dateContext ? `\n## Additional Context\n${options.dateContext}` : ''}

## Your Task
Analyze all the temporal information to determine:
1. **Actual Event Date**: When did the event described in the memory actually happen?
2. **Confidence Level**: How confident are you in this date (High/Medium/Low)?
3. **Date Source**: What evidence led to this date conclusion?
4. **Time Period**: If exact date unknown, what's the likely time range?

## Analysis Guidelines
- Look for explicit dates ("on March 15th", "yesterday", "last Tuesday")
- Consider relative time references in context of message timestamps
- Compare memory creation time vs. when event likely occurred
- Account for time zones and potential delays between event and recording
- If multiple dates are mentioned, identify which refers to the main event

## Response Format
Respond with a JSON object:
{
  "actualDate": "YYYY-MM-DD" or "YYYY-MM" or "YYYY" or null,
  "timeRange": {
    "start": "YYYY-MM-DD" or null,
    "end": "YYYY-MM-DD" or null
  },
  "confidence": "High" | "Medium" | "Low",
  "reasoning": "Detailed explanation of how you arrived at this date",
  "evidence": ["List of specific clues that supported the date deduction"],
  "relativeReferences": ["Any relative time phrases found like 'yesterday', 'last week'"],
  "alternativeDates": ["Other possible dates if multiple were found"]
}`;

      const response = await this.llm.generateResponse(
        [{ role: "user", content: dateAnalysisPrompt }],
        { type: "json_object" }
      );

      const cleanResponse = removeCodeBlocks(response as string);
      let analysis: any;
      
      try {
        analysis = JSON.parse(cleanResponse);
      } catch (parseError) {
        console.error("Failed to parse date analysis response:", parseError);
        return null;
      }

      // Return structured date analysis result
      return {
        type: 'date_analysis',
        memoryId,
        originalMemory: memoryInfo.new_value || memoryInfo.content,
        analysis: {
          actualDate: analysis.actualDate || null,
          timeRange: analysis.timeRange || null,
          confidence: analysis.confidence || 'Low',
          reasoning: analysis.reasoning || 'No reasoning provided',
          evidence: analysis.evidence || [],
          relativeReferences: analysis.relativeReferences || [],
          alternativeDates: analysis.alternativeDates || []
        },
        metadata: {
          memoryCreatedAt: memoryInfo.created_at,
          memoryHappenedAt: memoryInfo.happened_at,
          contextMessagesCount: allMessages.length,
          analysisTimestamp: new Date().toISOString()
        }
      };

    } catch (error) {
      console.error("Error in date deduction:", error);
      return null;
    }
  }

  // Filter and focus additional results to prevent overwhelming context
  private async filterAdditionalResults(
    query: string,
    existingResults: any[],
    additionalResults: any[],
    maxAdditional: number = 5
  ): Promise<any[]> {
    if (additionalResults.length <= maxAdditional) {
      return additionalResults;
    }

    console.log(`🔍 Filtering additional results: ${additionalResults.length} → ${maxAdditional} (query-focused)`);

    // Simple filtering for additional results - prioritize unique, relevant content
    const existingContent = new Set(
      existingResults.map(r => {
        const content = r.memory || JSON.stringify(r) || '';
        return content.toLowerCase().substring(0, 100);
      })
    );

    // Filter out duplicate or similar content, prioritize by relevance
    const uniqueAdditional = additionalResults.filter(result => {
      const content = result.memory || JSON.stringify(result) || '';
      const normalizedContent = content.toLowerCase().substring(0, 100);
      return !existingContent.has(normalizedContent);
    });

    // Take top results based on relevance and diversity
    const filtered = uniqueAdditional.slice(0, maxAdditional);
    
    console.log(`   ✅ Filtered additional results: ${filtered.length} unique items`);
    return filtered;
  }
}

interface SearchAgentDecision {
  needsAdditionalSteps: boolean;
  reasoning: string;
  needsAnotherHop: boolean;
  hopReasoning: string;
  nextSteps: Array<{
    action: string;
    parameters: Record<string, any>;
    priority: number;
  }>;
}

function removeCodeBlocks(text: string): string {
  return text.replace(/```[^`]*```/g, "");
}