import { Client } from '@hubspot/api-client';
import dotenv from 'dotenv';

dotenv.config();

// Convert any datetime objects to ISO strings
export function convertDatetimeFields(obj: any): any {
  if (obj === null || obj === undefined) {
    return obj;
  }
  
  if (typeof obj === 'object') {
    if (obj instanceof Date) {
      return obj.toISOString();
    }
    
    if (Array.isArray(obj)) {
      return obj.map(item => convertDatetimeFields(item));
    }
    
    const result: Record<string, any> = {};
    for (const [key, value] of Object.entries(obj)) {
      result[key] = convertDatetimeFields(value);
    }
    return result;
  }
  
  return obj;
}

function filterRecordsByUpdatedAfter(
  records: any[] | undefined,
  updatedAfter?: string,
  extractor?: (record: any) => string | number | null,
  debugLabel: string = 'records'
): any[] {
  if (!Array.isArray(records) || !updatedAfter) {
    return records || [];
  }

  const cutoff = Date.parse(updatedAfter);
  if (Number.isNaN(cutoff)) {
    return records;
  }

  let missingTimestampCount = 0;
  let invalidTimestampCount = 0;

  const filtered = records.filter((record) => {
    const rawValue = extractor
      ? extractor(record)
      : record?.updatedAt ??
        record?.updated_at ??
        record?.updated ??
        record?.lastUpdated ??
        record?.lastUpdatedAt ??
        record?.properties?.hs_lastmodifieddate ??
        record?.properties?.lastmodifieddate ??
        record?.metadata?.lastUpdated ??
        record?.metadata?.updatedAt ??
        record?.stats?.lastUpdated ??
        null;

    if (!rawValue) {
      missingTimestampCount++;
      return true;
    }

    const timestamp =
      typeof rawValue === 'number' ? rawValue : Date.parse(rawValue as string);

    if (Number.isNaN(timestamp)) {
      invalidTimestampCount++;
      return true;
    }

    return timestamp > cutoff;
  });

  const removed = records.length - filtered.length;
  console.log(
    `🔁 Incremental filter for ${debugLabel}: kept ${filtered.length}/${records.length} records updated after ${new Date(
      cutoff
    ).toISOString()}`
  );

  if (removed === 0) {
    console.log(
      `ℹ️ No ${debugLabel} records newer than ${new Date(cutoff).toISOString()}`
    );
  }

  if (missingTimestampCount > 0) {
    console.warn(
      `[Incremental] ${debugLabel}: ${missingTimestampCount} record(s) missing updated timestamp (included by default).`
    );
  }

  if (invalidTimestampCount > 0) {
    console.warn(
      `[Incremental] ${debugLabel}: ${invalidTimestampCount} record(s) had invalid timestamp values (included by default).`
    );
  }

  return filtered;
}

export class HubSpotScopeError extends Error {
  public scope: string;
  public code: string;

  constructor(scopeName: string, cause?: any) {
    super(`HubSpot scope "${scopeName}" is not granted for this account.`);
    this.name = 'HubSpotScopeError';
    this.scope = scopeName;
    this.code = 'HUBSPOT_SCOPE_MISSING';
    if (cause) {
      (this as any).cause = cause;
    }
  }
}

export class HubSpotClient {
  private client: Client;
  private accessToken: string;
  private lastRequestTime: number = 0;
  private requestCount: number = 0;
  private rateLimitWindow: number = 1000; // 1 second window
  
  constructor(accessToken?: string) {
    const token = accessToken ||
                 process.env.HUBSPOT_ACCESS_TOKEN ||
                 process.env.HUBSPOT_DEVELOPER_API_KEY ||
                 process.env.HUBSPOT_PERSONAL_ACCESS_KEY;

    if (!token) {
      throw new Error('HubSpot access token is required. Set HUBSPOT_ACCESS_TOKEN, HUBSPOT_DEVELOPER_API_KEY, or HUBSPOT_PERSONAL_ACCESS_KEY');
    }

    this.accessToken = token;
    this.client = new Client({ accessToken: token });
  }

  /**
   * Update the access token and reinitialize the client
   */
  updateAccessToken(newToken: string): void {
    this.accessToken = newToken;
    this.client = new Client({ accessToken: newToken });
    console.log('🔄 HubSpot client: Access token updated');
  }

  /**
   * Wrapper for API calls with automatic token refresh on 401 and
   * retry-with-backoff on 429 rate-limit errors.
   */
  private async callWithRetry<T>(apiCall: () => Promise<T>, retryOnAuth: boolean = true): Promise<T> {
    const MAX_RATE_LIMIT_RETRIES = 4;
    let rateLimitAttempt = 0;

    const attempt = async (): Promise<T> => {
      try {
        return await apiCall();
      } catch (error: any) {
        // 429 — rate-limited by HubSpot; back off and retry
        const statusCode = error?.code ?? error?.statusCode ?? error?.response?.status;
        if (statusCode === 429 && rateLimitAttempt < MAX_RATE_LIMIT_RETRIES) {
          rateLimitAttempt++;
          const retryAfter = error?.headers?.['retry-after'];
          const baseDelay = retryAfter ? Number(retryAfter) * 1000 : 1000 * rateLimitAttempt;
          const jitter = Math.random() * 500;
          const delay = baseDelay + jitter;
          console.log(`⏳ HubSpot 429 rate-limit hit — retry ${rateLimitAttempt}/${MAX_RATE_LIMIT_RETRIES} after ${Math.round(delay)}ms`);
          await new Promise(resolve => setTimeout(resolve, delay));
          return attempt();
        }

        // 401 — try refreshing the access token once
        if (error?.code === 401 &&
            error?.body?.category === 'EXPIRED_AUTHENTICATION' &&
            retryOnAuth) {

          console.log('🔄 HubSpot client: Detected 401 error, attempting token refresh...');

          const refreshedToken = process.env.HUBSPOT_ACCESS_TOKEN ||
                                process.env.HUBSPOT_DEVELOPER_API_KEY ||
                                process.env.HUBSPOT_PERSONAL_ACCESS_KEY;

          if (refreshedToken && refreshedToken !== this.accessToken) {
            console.log('🔄 HubSpot client: Found updated token, retrying...');
            this.updateAccessToken(refreshedToken);
            return await apiCall();
          } else {
            console.log('❌ HubSpot client: No updated token available for refresh');
            throw error;
          }
        }

        throw error;
      }
    };

    return attempt();
  }

  /**
   * Rate limiting to prevent 429 errors
   */
  private async rateLimitDelay(): Promise<void> {
    const now = Date.now();
    const timeSinceLastRequest = now - this.lastRequestTime;
    
    // Reset counter if window has passed
    if (timeSinceLastRequest > this.rateLimitWindow) {
      this.requestCount = 0;
    }
    
    // If we've made too many requests, wait
    if (this.requestCount >= 5) {
      const waitTime = this.rateLimitWindow - timeSinceLastRequest;
      if (waitTime > 0) {
        await new Promise(resolve => setTimeout(resolve, waitTime));
      }
      this.requestCount = 0;
    }
    
    this.requestCount++;
    this.lastRequestTime = Date.now();
  }
  
  /**
   * Helper method to paginate through all results using HubSpot's after cursor
   */
  private async paginateSearch(
    searchFn: (searchRequest: any) => Promise<any>,
    baseRequest: any,
    limit: number = 100
  ): Promise<any[]> {
    // Ensure limit doesn't exceed HubSpot's maximum
    const safeLimit = Math.min(Math.max(limit, 1), 200);
    
    const allResults: any[] = [];
    let after: string | undefined = undefined;
    let hasMore = true;

    while (hasMore) {
      await this.rateLimitDelay();

      const searchRequest = {
        ...baseRequest,
        limit: safeLimit,
        ...(after ? { after } : {})
      };

      const response: any = await this.callWithRetry<any>(() => searchFn(searchRequest));

      if (response.results && response.results.length > 0) {
        allResults.push(...response.results);
        console.log(`📄 Fetched ${response.results.length} records (total: ${allResults.length})`);
      }

      // Check for pagination cursor
      if (response.paging?.next?.after) {
        after = response.paging.next.after;
      } else {
        hasMore = false;
      }
    }

    return allResults;
  }

  private normalizeDateInput(value?: string | number | Date): string | null {
    if (!value) return null;
    const dateValue = value instanceof Date ? value : new Date(value);
    if (Number.isNaN(dateValue.getTime())) {
      return null;
    }
    return dateValue.toISOString();
  }

  private applyUpdatedAfterFilter(
    baseRequest: any,
    updatedAfter?: string,
    propertyName: string = 'hs_lastmodifieddate'
  ) {
    if (!baseRequest || !updatedAfter) {
      return baseRequest;
    }

    const normalized = this.normalizeDateInput(updatedAfter);
    if (!normalized) {
      return baseRequest;
    }

    const filter = {
      propertyName,
      operator: 'GT',
      value: normalized
    };

    if (Array.isArray(baseRequest.filterGroups) && baseRequest.filterGroups.length > 0) {
      baseRequest.filterGroups = baseRequest.filterGroups.map((group: any) => ({
        ...group,
        filters: Array.isArray(group.filters) ? [...group.filters, filter] : [filter]
      }));
    } else {
      baseRequest.filterGroups = [{ filters: [filter] }];
    }

    return baseRequest;
  }

  /**
   * Helper method to paginate through all results using HubSpot's after cursor (for List/GET APIs)
   */
  private async paginateList(
    listFn: (params: any) => Promise<any>,
    limit: number = 100
  ): Promise<any[]> {
    // Ensure limit doesn't exceed HubSpot's maximum
    const safeLimit = Math.min(Math.max(limit, 1), 200);
    
    const allResults: any[] = [];
    let after: string | undefined = undefined;
    let hasMore = true;

    while (hasMore) {
      await this.rateLimitDelay();

      const params: any = { limit: safeLimit, after };
      const response = await listFn(params);

      // Standard v3 SDK responses have a 'results' array
      const results = response.results;

      if (results && Array.isArray(results) && results.length > 0) {
        allResults.push(...results);
        console.log(`📄 Fetched ${results.length} records (total: ${allResults.length})`);
      }

      // Standard v3 SDK pagination cursor, guard against stuck cursors
      const nextAfter = response.paging?.next?.after;
      if (!nextAfter || nextAfter === after || !results || results.length === 0) {
        hasMore = false;
      } else {
        after = nextAfter;
      }
    }
    return allResults;
  }

  async getRecentCompanies(limit: number = 100, updatedAfter?: string): Promise<any> {
    return this.callWithRetry(async () => {
      console.log('🏢 Fetching ALL companies with pagination...');

      // Create search request with sort by lastmodifieddate
      const baseRequest = {
        sorts: ['lastmodifieddate:desc'],
        properties: [
          // Basic company info - enhanced
          'name', 'domain', 'website', 'phone', 'industry', 'company_type',
          // Address info - comprehensive
          'address', 'address2', 'city', 'state', 'zip', 'country', 'country_code',
          // Company details - expanded
          'description', 'numberofemployees', 'annualrevenue', 'type',
          'founded_year', 'business_type', 'revenue_range', 'employee_range',
          'target_account', 'ideal_customer_profile', 'account_tier',
          // Contact info
          'owneremail', 'ownername', 'phone', 'owner_phone',
          // Website and online presence - comprehensive
          'website', 'domain', 'blog_url', 'facebook_url', 'twitter_url',
          'linkedin_url', 'youtube_url', 'instagram_url', 'pinterest_url',
          'googleplus_url', 'klout_score', 'alexa_ranking', 'semrush_ranking',
          // Company associations and details
          'hs_parent_company_id', 'hs_child_company_ids', 'hs_analytics_source',
          'hs_analytics_source_data_1', 'hs_analytics_source_data_2', 'hs_analytics_source_data_3',
          'hs_analytics_source_data_4', 'hs_analytics_source_raw',
          // Custom properties - expanded
          'custom_property_1', 'custom_property_2', 'custom_property_3', 'custom_property_4', 'custom_property_5',
          'custom_property_6', 'custom_property_7', 'custom_property_8', 'custom_property_9', 'custom_property_10',
          // Dates and timestamps - comprehensive
          'createdate', 'hs_lastmodifieddate', 'lastmodifieddate', 'hs_createdate',
          'first_conversion_date', 'last_conversion_date', 'hs_first_engagement_date',
          'hs_last_contacted', 'hs_last_activity_date', 'hs_first_contact_createdate',
          // Additional fields - enhanced
          'hs_object_id', 'hs_unique_creation_key', 'hs_company_id', 'id',
          'is_deleted', 'hs_time_to_first_engagement', 'hs_time_to_close',
          // Lead status and scoring
          'hs_lead_status', 'hs_lead_scoring', 'hs_predictivecontactscore_v2',
          'hs_predictivecontactscoring_v2', 'hs_ideal_customer_profile'
        ]
      };
      this.applyUpdatedAfterFilter(baseRequest, updatedAfter, 'hs_lastmodifieddate');

      // Use pagination to fetch ALL companies
      const allCompanies = await this.paginateSearch(
        (req) => this.client.crm.companies.searchApi.doSearch(req),
        baseRequest,
        limit
      );

      console.log(`✅ Total companies fetched: ${allCompanies.length}`);
      return convertDatetimeFields(allCompanies);
    }).catch((error: any) => {
      console.error('Error getting recent companies:', error);
      return { error: error.message };
    });
  }
  
  async getRecentContacts(limit: number = 100, updatedAfter?: string): Promise<any> {
    try {
      console.log('📞 Fetching ALL contacts with pagination...');

      // Create search request with sort by lastmodifieddate
      const baseRequest = {
        sorts: ['lastmodifieddate:desc'],
        properties: [
          // Basic info - enhanced
          'firstname', 'lastname', 'email', 'phone', 'mobilephone', 'jobtitle',
          'salutation', 'middle_name', 'prefix', 'suffix',
          // Company info - comprehensive
          'company', 'website', 'company_name', 'company_website',
          'company_domain', 'company_industry', 'company_size',
          // Address info - comprehensive
          'address', 'address2', 'city', 'state', 'zip', 'country', 'country_code',
          // Contact properties - expanded
          'lifecyclestage', 'lead_status', 'hs_lead_status', 'contact_owner',
          'hs_contact_stage', 'hs_contact_status', 'hs_contact_source',
          // Company association and details - enhanced
          'associatedcompanyid', 'associatedcompany_name', 'associatedcompany_domain',
          'associatedcompany_website', 'associatedcompany_industry', 'associatedcompany_size',
          // Social media and online presence - comprehensive
          'linkedin_profile', 'twitter_profile', 'facebook_profile',
          'linkedin_url', 'twitter_url', 'facebook_url', 'instagram_url', 'youtube_url',
          // Professional details - expanded
          'seniority', 'department', 'industry', 'job_function', 'job_role',
          'job_level', 'years_experience', 'education_level', 'school',
          // Lead scoring and qualification - comprehensive
          'hs_lead_scoring', 'hs_lead_status', 'hs_analytics_source',
          'hs_analytics_source_data_1', 'hs_analytics_source_data_2', 'hs_analytics_source_data_3',
          'hs_analytics_source_data_4', 'hs_analytics_source_raw',
          'hs_predictivecontactscore_v2', 'hs_predictivecontactscoring_v2',
          // Custom properties - expanded
          'custom_property_1', 'custom_property_2', 'custom_property_3', 'custom_property_4', 'custom_property_5',
          'custom_property_6', 'custom_property_7', 'custom_property_8', 'custom_property_9', 'custom_property_10',
          // Dates and timestamps - comprehensive
          'createdate', 'hs_lastmodifieddate', 'lastmodifieddate', 'hs_createdate',
          'first_conversion_date', 'last_conversion_date', 'hs_first_engagement_date',
          'hs_last_contacted', 'hs_last_activity_date', 'hs_first_contact_createdate',
          // Additional fields - enhanced
          'hs_object_id', 'hs_unique_creation_key', 'hs_contact_id', 'id',
          'is_deleted', 'hs_time_to_first_engagement', 'hs_contact_stage_probability'
        ]
      };
      this.applyUpdatedAfterFilter(baseRequest, updatedAfter, 'hs_lastmodifieddate');

      // Use pagination to fetch ALL contacts
      const allContacts = await this.paginateSearch(
        (req) => this.client.crm.contacts.searchApi.doSearch(req),
        baseRequest,
        limit
      );

      console.log(`✅ Total contacts fetched: ${allContacts.length}`);
      return convertDatetimeFields(allContacts);
    } catch (error: any) {
      console.error('Error getting recent contacts:', error);
      return { error: error.message };
    }
  }
  
  async getAssociations(fromObjectType: string, fromObjectId: string, toObjectType: string): Promise<any[]> {
    try {
      const allAssociations: any[] = [];
      let after: string | undefined = undefined;

      while (true) {
        await this.rateLimitDelay();
        const response = await this.client.crm.associations.v4.basicApi.getPage(
          fromObjectType,
          fromObjectId,
          toObjectType,
          after,
          500
        );

        const results = response.results || [];
        if (results.length > 0) {
          allAssociations.push(...results);
        }

        const nextAfter = response.paging?.next?.after;
        if (!nextAfter || nextAfter === after || results.length === 0) {
          break;
        }
        after = nextAfter;
      }

      return allAssociations;
    } catch (error: any) {
      console.error(`Error fetching associations for ${fromObjectType}:${fromObjectId} -> ${toObjectType}:`, error);
      throw new Error(`HubSpot API error: ${error.message}`);
    }
  }
  
  async searchTasks(minutes: number = 30, limit: number = 50): Promise<any> {
    try {
      // Calculate the date range (past N minutes)
      const endTime = new Date();
      const startTime = new Date(endTime);
      startTime.setMinutes(startTime.getMinutes() - minutes);
      const startTimestamp = startTime.getTime();

      console.log(`[MCP Debug] Searching tasks from ${startTime.toISOString()} to ${endTime.toISOString()}`);
      console.log(`[MCP Debug] Timestamp filter: ${startTimestamp}`);

      const baseRequest = {
        filterGroups: [{
          filters: [{
            propertyName: 'hs_lastmodifieddate',
            operator: 'GTE' as any,
            value: startTimestamp.toString()
          }]
        }],
        properties: ['hs_task_subject', 'hs_task_body', 'hs_task_status', 'hs_lastmodifieddate', 'hs_createdate'],
        sorts: ['hs_lastmodifieddate:desc']
      };

      const allTasks = await this.paginateSearch(
        (req) => this.client.crm.objects.tasks.searchApi.doSearch(req),
        baseRequest,
        limit
      );

      console.log(`[MCP Debug] Tasks found: ${allTasks.length}`);

      return convertDatetimeFields(allTasks);

    } catch (error: any) {
      console.error('Error searching tasks:', error);
      return { error: error.message };
    }
  }

  async searchNotes(minutes: number = 30, limit: number = 50): Promise<any> {
    try {
      // Calculate the date range (past N minutes)
      const endTime = new Date();
      const startTime = new Date(endTime);
      startTime.setMinutes(startTime.getMinutes() - minutes);
      const startTimestamp = startTime.getTime();

      console.log(`[MCP Debug] Searching notes from ${startTime.toISOString()} to ${endTime.toISOString()}`);
      console.log(`[MCP Debug] Timestamp filter: ${startTimestamp}`);

      const baseRequest = {
        filterGroups: [{
          filters: [{
            propertyName: 'hs_lastmodifieddate',
            operator: 'GTE' as any,
            value: startTimestamp.toString()
          }]
        }],
        properties: ['hs_note_body', 'hs_lastmodifieddate', 'hs_createdate'],
        sorts: ['hs_lastmodifieddate:desc']
      };

      const allNotes = await this.paginateSearch(
        (req) => this.client.crm.objects.notes.searchApi.doSearch(req),
        baseRequest,
        limit
      );

      console.log(`[MCP Debug] Notes found: ${allNotes.length}`);
      return convertDatetimeFields(allNotes);

    } catch (error: any) {
      console.error('Error searching notes:', error);
      return { error: error.message };
    }
  }

  async searchCalls(minutes: number = 30, limit: number = 50): Promise<any> {
    try {
      // Calculate the date range (past N minutes)
      const endTime = new Date();
      const startTime = new Date(endTime);
      startTime.setMinutes(startTime.getMinutes() - minutes);
      const startTimestamp = startTime.getTime();

      console.log(`[MCP Debug] Searching calls from ${startTime.toISOString()} to ${endTime.toISOString()}`);
      console.log(`[MCP Debug] Timestamp filter: ${startTimestamp}`);

      const baseRequest = {
        filterGroups: [{
          filters: [{
            propertyName: 'hs_lastmodifieddate',
            operator: 'GTE' as any,
            value: startTimestamp.toString()
          }]
        }],
        properties: ['hs_call_title', 'hs_call_body', 'hs_call_duration', 'hs_call_status', 'hs_lastmodifieddate', 'hs_createdate'],
        sorts: ['hs_lastmodifieddate:desc']
      };

      const allCalls = await this.paginateSearch(
        (req) => this.client.crm.objects.calls.searchApi.doSearch(req),
        baseRequest,
        limit
      );

      console.log(`[MCP Debug] Calls found: ${allCalls.length}`);
      return convertDatetimeFields(allCalls);

    } catch (error: any) {
      console.error('Error searching calls:', error);
      return { error: error.message };
    }
  }

  async searchEmails(minutes: number = 30, limit: number = 50): Promise<any> {
    try {
      // Calculate the date range (past N minutes)
      const endTime = new Date();
      const startTime = new Date(endTime);
      startTime.setMinutes(startTime.getMinutes() - minutes);
      const startTimestamp = startTime.getTime();

      console.log(`[MCP Debug] Searching emails from ${startTime.toISOString()} to ${endTime.toISOString()}`);
      console.log(`[MCP Debug] Timestamp filter: ${startTimestamp}`);

      const baseRequest = {
        filterGroups: [{
          filters: [{
            propertyName: 'hs_lastmodifieddate',
            operator: 'GTE' as any,
            value: startTimestamp.toString()
          }]
        }],
        properties: ['hs_email_subject', 'hs_email_text', 'hs_email_status', 'hs_lastmodifieddate', 'hs_createdate'],
        sorts: ['hs_lastmodifieddate:desc']
      };

      const allEmails = await this.paginateSearch(
        (req) => this.client.crm.objects.emails.searchApi.doSearch(req),
        baseRequest,
        limit
      );

      console.log(`[MCP Debug] Emails found: ${allEmails.length}`);
      return convertDatetimeFields(allEmails);

    } catch (error: any) {
      console.error('Error searching emails:', error);
      return { error: error.message };
    }
  }

  async searchMeetings(minutes: number = 30, limit: number = 50): Promise<any> {
    try {
      // Calculate the date range (past N minutes)
      const endTime = new Date();
      const startTime = new Date(endTime);
      startTime.setMinutes(startTime.getMinutes() - minutes);
      const startTimestamp = startTime.getTime();

      console.log(`[MCP Debug] Searching meetings from ${startTime.toISOString()} to ${endTime.toISOString()}`);
      console.log(`[MCP Debug] Timestamp filter: ${startTimestamp}`);

      const baseRequest = {
        filterGroups: [{
          filters: [{
            propertyName: 'hs_lastmodifieddate',
            operator: 'GTE' as any,
            value: startTimestamp.toString()
          }]
        }],
        properties: ['hs_meeting_title', 'hs_meeting_body', 'hs_meeting_start_time', 'hs_meeting_end_time', 'hs_lastmodifieddate', 'hs_createdate'],
        sorts: ['hs_lastmodifieddate:desc']
      };

      const allMeetings = await this.paginateSearch(
        (req) => this.client.crm.objects.meetings.searchApi.doSearch(req),
        baseRequest,
        limit
      );

      console.log(`[MCP Debug] Meetings found: ${allMeetings.length}`);
      return convertDatetimeFields(allMeetings);

    } catch (error: any) {
      console.error('Error searching meetings:', error);
      return { error: error.message };
    }
  }

  async getRecentEngagements(limit: number = 50): Promise<any> {
    try {
      await this.rateLimitDelay();
      
      console.log(`[MCP Debug] Searching for all engagements`);
      
      // Ensure limit doesn't exceed HubSpot's maximum per search
      const safeLimit = Math.min(Math.max(limit, 1), 200);
      
      // Get all notes and tasks using CRM v3 API (no time filtering) with safe limits
      const [notesResponse, tasksResponse] = await Promise.allSettled([
        // Get all notes
        this.client.crm.objects.notes.searchApi.doSearch({
          properties: ['hs_note_body', 'hs_lastmodifieddate', 'hs_createdate'],
          sorts: ['hs_lastmodifieddate:desc'],
          limit: Math.floor(safeLimit / 2)
        }),
        // Get all tasks
        this.client.crm.objects.tasks.searchApi.doSearch({
          properties: ['hs_task_subject', 'hs_task_body', 'hs_task_status', 'hs_lastmodifieddate', 'hs_createdate'],
          sorts: ['hs_lastmodifieddate:desc'],
          limit: Math.floor(safeLimit / 2)
        })
      ]);
      
      console.log(`[MCP Debug] Notes response status: ${notesResponse.status}`);
      console.log(`[MCP Debug] Tasks response status: ${tasksResponse.status}`);
      
      if (notesResponse.status === 'rejected') {
        console.error(`[MCP Debug] Notes search failed:`, notesResponse.reason);
      } else {
        console.log(`[MCP Debug] Notes found: ${notesResponse.value?.results?.length || 0}`);
      }
      
      if (tasksResponse.status === 'rejected') {
        console.error(`[MCP Debug] Tasks search failed:`, tasksResponse.reason);
      } else {
        console.log(`[MCP Debug] Tasks found: ${tasksResponse.value?.results?.length || 0}`);
      }
      
      // Format the engagements from CRM v3 responses
      const formattedEngagements = [];
      
      // Process notes
      if (notesResponse.status === 'fulfilled' && notesResponse.value?.results) {
        for (const note of notesResponse.value.results) {
          formattedEngagements.push({
            id: note.id,
            type: 'NOTE',
            engagement: {
              id: note.id,
              type: 'NOTE',
              createdAt: note.properties?.hs_createdate,
              lastUpdated: note.properties?.hs_lastmodifieddate
            },
            metadata: {
              body: note.properties?.hs_note_body
            },
            properties: note.properties,
            created_at: note.properties?.hs_createdate,
            last_updated: note.properties?.hs_lastmodifieddate
          });
        }
      }
      
      // Process tasks
      if (tasksResponse.status === 'fulfilled' && tasksResponse.value?.results) {
        for (const task of tasksResponse.value.results) {
          formattedEngagements.push({
            id: task.id,
            type: 'TASK',
            engagement: {
              id: task.id,
              type: 'TASK',
              createdAt: task.properties?.hs_createdate,
              lastUpdated: task.properties?.hs_lastmodifieddate
            },
            metadata: {
              subject: task.properties?.hs_task_subject,
              body: task.properties?.hs_task_body,
              status: task.properties?.hs_task_status
            },
            properties: task.properties,
            created_at: task.properties?.hs_createdate,
            last_updated: task.properties?.hs_lastmodifieddate
          });
        }
      }
      
      // Sort by last modified date (most recent first)
      formattedEngagements.sort((a, b) => {
        const aTime = new Date(a.last_updated || a.created_at || 0).getTime();
        const bTime = new Date(b.last_updated || b.created_at || 0).getTime();
        return bTime - aTime;
      });
      
      console.log(`[MCP Debug] Returning ${formattedEngagements.length} formatted engagements`);
      
      return convertDatetimeFields(formattedEngagements);
      
    } catch (error: any) {
      console.error('Error getting recent engagements:', error);
      return { error: error.message };
    }
  }

  
  async createContact(
    firstname: string, 
    lastname: string, 
    email?: string, 
    properties?: Record<string, any>
  ): Promise<any> {
    try {
      // Search for existing contacts with same name and company
      const company = properties?.company;
      
      // Use type assertion to satisfy the HubSpot API client types
      const searchRequest = {
        filterGroups: [{
          filters: [
            {
              propertyName: 'firstname',
              operator: 'EQ',
              value: firstname
            } as any,
            {
              propertyName: 'lastname',
              operator: 'EQ',
              value: lastname
            } as any
          ]
        }]
      } as any;
      
      // Add company filter if provided
      if (company) {
        searchRequest.filterGroups[0].filters.push({
          propertyName: 'company',
          operator: 'EQ',
          value: company
        } as any);
      }
      
      const searchResponse = await this.client.crm.contacts.searchApi.doSearch(searchRequest);
      
      if (searchResponse.total > 0) {
        // Contact already exists
        return { 
          message: 'Contact already exists', 
          contact: searchResponse.results[0] 
        };
      }
      
      // If no existing contact found, proceed with creation
      const contactProperties: Record<string, any> = {
        firstname,
        lastname
      };
      
      // Add email if provided
      if (email) {
        contactProperties.email = email;
      }
      
      // Add any additional properties
      if (properties) {
        Object.assign(contactProperties, properties);
      }
      
      // Create contact
      const apiResponse = await this.client.crm.contacts.basicApi.create({
        properties: contactProperties
      });
      
      return apiResponse;
    } catch (error: any) {
      console.error('Error creating contact:', error);
      throw new Error(`HubSpot API error: ${error.message}`);
    }
  }
  
  async createCompany(name: string, properties?: Record<string, any>): Promise<any> {
    try {
      await this.rateLimitDelay();
      // Search for existing companies with same name
      // Use type assertion to satisfy the HubSpot API client types
      const searchRequest = {
        filterGroups: [{
          filters: [
            {
              propertyName: 'name',
              operator: 'EQ',
              value: name
            } as any
          ]
        }]
      } as any;

      const searchResponse = await this.client.crm.companies.searchApi.doSearch(searchRequest);

      if (searchResponse.total > 0) {
        // Company already exists
        return {
          message: 'Company already exists',
          company: searchResponse.results[0]
        };
      }

      // If no existing company found, proceed with creation
      const companyProperties: Record<string, any> = {
        name
      };

      // Add any additional properties
      if (properties) {
        Object.assign(companyProperties, properties);
      }

      // Create company
      const apiResponse = await this.client.crm.companies.basicApi.create({
        properties: companyProperties
      });

      return apiResponse;
    } catch (error: any) {
      console.error('Error creating company:', error);
      throw new Error(`HubSpot API error: ${error.message}`);
    }
  }

  async createNote(body: string, associations?: any[]): Promise<any> {
    try {
      await this.rateLimitDelay();

      // Create note properties
      const noteProperties: Record<string, any> = {
        hs_note_body: body
      };

      // Create the note using CRM v3 API
      const noteResponse = await this.client.crm.objects.notes.basicApi.create({
        properties: noteProperties,
        associations: []
      });

      // If associations are provided, create them
      if (associations && associations.length > 0) {
        for (const association of associations) {
          try {
            await this.client.crm.associations.v4.basicApi.create(
              'notes',
              noteResponse.id,
              association.toObjectType,
              association.toObjectId,
              [{ associationCategory: association.category || 'USER_DEFINED', associationTypeId: association.typeId || 1 }]
            );
          } catch (associationError) {
            console.warn('Failed to create association for note:', associationError);
            // Continue with other associations
          }
        }
      }

      return {
        id: noteResponse.id,
        properties: noteResponse.properties,
        associations: associations || []
      };
    } catch (error: any) {
      console.error('Error creating note:', error);
      throw new Error(`HubSpot API error: ${error.message}`);
    }
  }

  async updateContact(
    contactId: string,
    properties: Record<string, any>
  ): Promise<any> {
    try {
      // Check if contact exists
      try {
        await this.client.crm.contacts.basicApi.getById(contactId);
      } catch (error: any) {
        // If contact doesn't exist, return a message
        if (error.statusCode === 404) {
          return {
            message: 'Contact not found, no update performed',
            contactId
          };
        }
        // For other errors, throw them to be caught by the outer try/catch
        throw error;
      }

      // Update the contact
      const apiResponse = await this.client.crm.contacts.basicApi.update(contactId, {
        properties
      });

      return {
        message: 'Contact updated successfully',
        contactId,
        properties
      };
    } catch (error: any) {
      console.error('Error updating contact:', error);
      throw new Error(`HubSpot API error: ${error.message}`);
    }
  }

  async updateCompany(
    companyId: string,
    properties: Record<string, any>
  ): Promise<any> {
    try {
      // Check if company exists
      try {
        await this.client.crm.companies.basicApi.getById(companyId);
      } catch (error: any) {
        // If company doesn't exist, return a message
        if (error.statusCode === 404) {
          return {
            message: 'Company not found, no update performed',
            companyId
          };
        }
        // For other errors, throw them to be caught by the outer try/catch
        throw error;
      }

      // Update the company
      const apiResponse = await this.client.crm.companies.basicApi.update(companyId, {
        properties
      });

      return {
        message: 'Company updated successfully',
        companyId,
        properties
      };
    } catch (error: any) {
      console.error('Error updating company:', error);
      throw new Error(`HubSpot API error: ${error.message}`);
    }
  }

  async getRecentDeals(limit: number = 100, updatedAfter?: string): Promise<any> {
    try {
      console.log('Fetching ALL deals with pagination...');

      const baseRequest = {
        sorts: ['hs_lastmodifieddate:desc'],
        properties: [
          'dealname', 'amount', 'dealstage', 'createdate', 'closedate',
          'hubspot_owner_id', 'dealtype', 'pipeline', 'hs_analytics_source',
          'hs_deal_stage_probability', 'days_to_close', 'num_contacted_notes'
        ]
      };

      this.applyUpdatedAfterFilter(baseRequest, updatedAfter, 'hs_lastmodifieddate');

      const allDeals = await this.paginateSearch(
        (req) => this.client.crm.deals.searchApi.doSearch(req),
        baseRequest,
        limit
      );

      console.log(`?o. Total deals fetched: ${allDeals.length}`);
      return convertDatetimeFields(allDeals);
    } catch (error: any) {
      console.error('Error fetching deals:', error);
      throw new Error(`HubSpot API error: ${error.message}`);
    }
  }

  async getCurrentUser(): Promise<any> {
    try {
      await this.rateLimitDelay();
      // Use the OAuth v1 access-tokens introspection endpoint (no Authorization header required)
      // https://api.hubapi.com/oauth/v1/access-tokens/{token}
      const resp = await fetch(`https://api.hubapi.com/oauth/v1/access-tokens/${this.accessToken}`);

      if (!resp.ok) {
        // Return a minimal object instead of throwing to avoid noisy errors upstream
        return {
          userId: 'unknown',
          userEmail: 'unknown',
          hubId: null,
          scopes: [],
          error: `HTTP ${resp.status}: ${resp.statusText}`
        };
      }

      const data = await resp.json();
      return {
        userId: data.user_id || data.user || 'unknown',
        userEmail: data.user_email || data.email || 'unknown',
        hubId: data.hub_id || data.hubId || null,
        scopes: data.scopes || []
      };
    } catch (error: any) {
      console.error('Error fetching current user:', error);
      // Graceful fallback to avoid breaking downstream consumers
      return {
        userId: 'unknown',
        userEmail: 'unknown',
        hubId: null,
        scopes: [],
        error: error?.message || 'unknown'
      };
    }
  }

  async getRecentTickets(limit: number = 100): Promise<any> {
    try {
      console.log('?YZ? Fetching ALL tickets with pagination...');

      const ticketProperties = [
        'subject', 'content', 'hs_ticket_priority', 'hs_pipeline', 'hs_pipeline_stage',
        'createdate', 'hs_lastmodifieddate', 'hubspot_owner_id',
        'hs_ticket_category', 'hs_ticket_id', 'source_type',
        'closed_date', 'first_agent_reply_date', 'time_to_close', 'time_to_first_agent_reply',
        'hs_resolution', 'hs_custom_inbox', 'hs_was_imported',
        'num_associated_conversations', 'num_notes', 'num_contacted_notes'
      ];

      const baseRequest = {
        sorts: ['hs_lastmodifieddate:desc'],
        properties: ticketProperties
      };

      const allTickets = await this.paginateSearch(
        (req) => this.client.crm.tickets.searchApi.doSearch(req),
        baseRequest,
        limit
      );

      console.log(`?o. Total tickets fetched: ${allTickets.length}`);
      return convertDatetimeFields(allTickets);
    } catch (error: any) {
      console.error('Error fetching tickets:', error);
      throw new Error(`HubSpot API error: ${error.message}`);
    }
  }

  async getRecentOrders(limit: number = 100, updatedAfter?: string): Promise<any[]> {
    try {
      const orderProperties = [
        'hs_order_name', 'hs_source_store', 'hs_fulfillment_status',
        'hs_currency_code', 'hs_total_price', 'hs_shipping_price',
        'hs_tax_price', 'hs_discount_price', 'hs_createdate', 'hs_lastmodifieddate',
        'hs_external_order_id', 'hs_order_status', 'hs_shipping_address_city',
        'hubspot_owner_id', 'hs_payment_status', 'hs_order_currency_code'
      ];
      const searchRequest = {
        sorts: ['hs_lastmodifieddate:desc'],
        properties: orderProperties
      };

      if (updatedAfter) {
        this.applyUpdatedAfterFilter(searchRequest, updatedAfter, 'hs_lastmodifieddate');
      }

      // Note: Orders might not be available in all HubSpot portals
      // This depends on having Commerce Hub or ecommerce integrations
      const allOrders = await this.paginateSearch(
        (req) => this.client.crm.objects.searchApi.doSearch('orders', req),
        searchRequest,
        limit
      );
      return convertDatetimeFields(allOrders);
    } catch (error: any) {
      console.error('Error fetching orders:', error);
      // Return empty array if orders are not available rather than throwing
      return [];
    }
  }

  async getOrders(limit: number = 100, updatedAfter?: string): Promise<any[]> {
    return this.getRecentOrders(limit, updatedAfter);
  }


  async getContactById(contactId: string): Promise<any> {
    try {
      await this.rateLimitDelay();
      const contact = await this.client.crm.contacts.basicApi.getById(contactId, [
        // Basic info - enhanced
        'firstname', 'lastname', 'email', 'phone', 'mobilephone', 'jobtitle',
        'salutation', 'middle_name', 'prefix', 'suffix',
        // Company info - comprehensive
        'company', 'website', 'company_name', 'company_website',
        'company_domain', 'company_industry', 'company_size',
        // Address info - comprehensive
        'address', 'address2', 'city', 'state', 'zip', 'country', 'country_code',
        // Contact properties - expanded
        'lifecyclestage', 'lead_status', 'hs_lead_status', 'contact_owner',
        'hs_contact_stage', 'hs_contact_status', 'hs_contact_source',
        // Company association and details - enhanced
        'associatedcompanyid', 'associatedcompany_name', 'associatedcompany_domain',
        'associatedcompany_website', 'associatedcompany_industry', 'associatedcompany_size',
        // Social media and online presence - comprehensive
        'linkedin_profile', 'twitter_profile', 'facebook_profile',
        'linkedin_url', 'twitter_url', 'facebook_url', 'instagram_url', 'youtube_url',
        // Professional details - expanded
        'seniority', 'department', 'industry', 'job_function', 'job_role',
        'job_level', 'years_experience', 'education_level', 'school',
        // Lead scoring and qualification - comprehensive
        'hs_lead_scoring', 'hs_lead_status', 'hs_analytics_source',
        'hs_analytics_source_data_1', 'hs_analytics_source_data_2', 'hs_analytics_source_data_3',
        'hs_analytics_source_data_4', 'hs_analytics_source_raw',
        'hs_predictivecontactscore_v2', 'hs_predictivecontactscoring_v2',
        // Custom properties - expanded
        'custom_property_1', 'custom_property_2', 'custom_property_3', 'custom_property_4', 'custom_property_5',
        'custom_property_6', 'custom_property_7', 'custom_property_8', 'custom_property_9', 'custom_property_10',
        // Dates and timestamps - comprehensive
        'createdate', 'hs_lastmodifieddate', 'lastmodifieddate', 'hs_createdate',
        'first_conversion_date', 'last_conversion_date', 'hs_first_engagement_date',
        'hs_last_contacted', 'hs_last_activity_date', 'hs_first_contact_createdate',
        // Additional fields - enhanced
        'hs_object_id', 'hs_unique_creation_key', 'hs_contact_id', 'id',
        'is_deleted', 'hs_time_to_first_engagement', 'hs_contact_stage_probability'
      ]);
      return convertDatetimeFields(contact);
    } catch (error: any) {
      console.error('Error fetching contact by ID:', error);
      throw new Error(`HubSpot API error: ${error.message}`);
    }
  }

  async getCompanyById(companyId: string): Promise<any> {
    try {
      await this.rateLimitDelay();
      const company = await this.client.crm.companies.basicApi.getById(companyId, [
        // Basic company info - enhanced
        'name', 'domain', 'website', 'phone', 'industry', 'company_type',
        // Address info - comprehensive
        'address', 'address2', 'city', 'state', 'zip', 'country', 'country_code',
        // Company details - expanded
        'description', 'numberofemployees', 'annualrevenue', 'type',
        'founded_year', 'business_type', 'revenue_range', 'employee_range',
        'target_account', 'ideal_customer_profile', 'account_tier',
        // Contact info
        'owneremail', 'ownername', 'phone', 'owner_phone',
        // Website and online presence - comprehensive
        'website', 'domain', 'blog_url', 'facebook_url', 'twitter_url',
        'linkedin_url', 'youtube_url', 'instagram_url', 'pinterest_url',
        'googleplus_url', 'klout_score', 'alexa_ranking', 'semrush_ranking',
        // Company associations and details
        'hs_parent_company_id', 'hs_child_company_ids', 'hs_analytics_source',
        'hs_analytics_source_data_1', 'hs_analytics_source_data_2', 'hs_analytics_source_data_3',
        'hs_analytics_source_data_4', 'hs_analytics_source_raw',
        // Custom properties - expanded
        'custom_property_1', 'custom_property_2', 'custom_property_3', 'custom_property_4', 'custom_property_5',
        'custom_property_6', 'custom_property_7', 'custom_property_8', 'custom_property_9', 'custom_property_10',
        // Dates and timestamps - comprehensive
        'createdate', 'hs_lastmodifieddate', 'lastmodifieddate', 'hs_createdate',
        'first_conversion_date', 'last_conversion_date', 'hs_first_engagement_date',
        'hs_last_contacted', 'hs_last_activity_date', 'hs_first_contact_createdate',
        // Additional fields - enhanced
        'hs_object_id', 'hs_unique_creation_key', 'hs_company_id', 'id',
        'is_deleted', 'hs_time_to_first_engagement', 'hs_time_to_close',
        // Lead status and scoring
        'hs_lead_status', 'hs_lead_scoring', 'hs_predictivecontactscore_v2',
        'hs_predictivecontactscoring_v2', 'hs_ideal_customer_profile'
      ]);
      return convertDatetimeFields(company);
    } catch (error: any) {
      console.error('Error fetching company by ID:', error);
      throw new Error(`HubSpot API error: ${error.message}`);
    }
  }

  async getDealById(dealId: string): Promise<any> {
    try {
      await this.rateLimitDelay();
      const deal = await this.client.crm.deals.basicApi.getById(dealId, [
        'dealname', 'amount', 'dealstage', 'pipeline', 'closedate', 'hs_lastmodifieddate', 'createdate'
      ]);
      return convertDatetimeFields(deal);
    } catch (error: any) {
      console.error('Error fetching deal by ID:', error);
      throw new Error(`HubSpot API error: ${error.message}`);
    }
  }

  async getTicketById(ticketId: string): Promise<any> {
    try {
      await this.rateLimitDelay();
      const ticket = await this.client.crm.objects.basicApi.getById('tickets', ticketId, [
        'subject', 'content', 'hs_ticket_priority', 'hs_ticket_category', 'hs_ticket_status', 'hs_lastmodifieddate', 'createdate'
      ]);
      return convertDatetimeFields(ticket);
    } catch (error: any) {
      console.error('Error fetching ticket by ID:', error);
      throw new Error(`HubSpot API error: ${error.message}`);
    }
  }


  // Helper method for graceful error handling of optional scopes
  private handleOptionalScopeError(error: any, scopeName: string, fallbackValue: any = []): any {
    const errorMessage = error.message?.toLowerCase() || '';
    const isPermissionError = errorMessage.includes('scope') || 
                             errorMessage.includes('permission') || 
                             errorMessage.includes('unauthorized') ||
                             errorMessage.includes('forbidden') ||
                             error.status === 403 ||
                             error.status === 401;
    
    if (isPermissionError) {
      console.error(`HubSpot scope missing: ${scopeName}. Unable to complete API call.`, {
        scope: scopeName,
        status: error?.status,
        category: error?.body?.category,
        message: error?.message
      });
      throw new HubSpotScopeError(scopeName, error);
    }
    
    // Re-throw non-permission errors
    throw error;
  }

  // Marketing Hub Methods
  async getForms(limit: number = 100): Promise<any[]> {
    try {
      console.log('📝 Fetching ALL forms with pagination...');
      const allForms = await this.paginateList(
        // Use the dedicated v3 marketing forms API client
        (params) => this.client.marketing.forms.formsApi.getPage(
          params.after,
          params.limit
        ),
        limit
      );
      console.log(`✅ Total forms fetched: ${allForms.length}`);
      return allForms;
    } catch (error: any) {
      console.error('Error fetching forms:', error);
      return this.handleOptionalScopeError(error, 'forms', []);
    }
  }

  async getFormSubmissions(formId: string, limit: number = 100): Promise<any[]> {
    try {
      await this.rateLimitDelay();
      const response = await this.client.apiRequest({
        method: 'GET',
        path: `/forms/v2/submissions/forms/${formId}?limit=${limit}`
      });
      const data = await response.json();
      return data?.results || [];
    } catch (error: any) {
      console.error('Error fetching form submissions:', error);
      return this.handleOptionalScopeError(error, 'forms', []);
    }
  }

  async getMarketingEmails(limit: number = 100, updatedAfter?: string): Promise<any[]> {
    try {
      console.log('📧 Fetching ALL marketing emails with pagination (v3)...');

      const normalizedFilter = this.normalizeDateInput(updatedAfter);
      const allEmails = await this.paginateList(
        async (params) => {
          const query = new URLSearchParams();
          query.append('limit', params.limit.toString());
          if (params.after) {
            query.append('after', params.after);
          }

          const path = `/marketing/v3/emails?${query.toString()}`;
          const response = await this.client.apiRequest({ method: 'GET', path });
          return await response.json();
        },
        limit
      );

      const normalizedEmails = convertDatetimeFields(allEmails);
      const filteredEmails = filterRecordsByUpdatedAfter(
        normalizedEmails,
        normalizedFilter || undefined,
        (email) => email?.updatedAt,
        'marketing emails'
      );

      console.log(`✅ Total marketing emails fetched: ${filteredEmails.length}`);
      return filteredEmails;
    } catch (error: any) {
      console.error('Error fetching marketing emails:', error);
      return this.handleOptionalScopeError(error, 'marketing-email', []);
    }
  }
  
  async getEmailEvents(limit: number = 100): Promise<any[]> {
    try {
      console.log('📊 Fetching marketing email events...');
      const allEvents = await this.paginateList(
        async (params) => {
          const path = `/marketing/v3/marketing-events/events?limit=${params.limit}${params.after ? `&after=${params.after}` : ''}`;
          const response = await this.client.apiRequest({ method: 'GET', path });
          return await response.json();
        },
        limit
      );
      console.log(`✅ Total email events fetched: ${allEvents.length}`);
      return allEvents;
    } catch (error: any) {
      console.error('Error fetching email events:', error);
      return this.handleOptionalScopeError(error, 'crm.objects.marketing_events.read', []);
    }
  }

  async getCampaigns(limit: number = 100): Promise<any[]> {
    try {
      console.log('📢 Fetching ALL campaigns with pagination...');
      const allCampaigns = await this.paginateList(
        async (params) => {
          const path = `/marketing/v3/campaigns?limit=${params.limit}${params.after ? `&after=${params.after}` : ''}`;
          const response = await this.client.apiRequest({ method: 'GET', path });
          return await response.json();
        },
        limit
      );
      console.log(`✅ Total campaigns fetched: ${allCampaigns.length}`);
      return allCampaigns;
    } catch (error: any) {
      console.error('Error fetching campaigns:', error);
      return this.handleOptionalScopeError(error, 'marketing.campaigns.read', []);
    }
  }

  async getAnalytics(startDate: string, endDate: string): Promise<any> {
    try {
      await this.rateLimitDelay();
      const path = `/analytics/v3/reports/domains/total-traffic?start_date=${startDate}&end_date=${endDate}`;
      
      const response = await this.client.apiRequest({
        method: 'GET',
        path: path
      });
      return await response.json();
    } catch (error: any) {
      console.error('Error fetching analytics:', error);
      return this.handleOptionalScopeError(error, 'analytics.behavioral_events.send', {});
    }
  }

  async getBlogPosts(limit: number = 100): Promise<any[]> {
    try {
      console.log('📝 Fetching ALL blog posts with pagination...');
      const allPosts = await this.paginateList(
        async (params) => {
          const path = `/cms/v3/blogs/posts?limit=${params.limit}${params.after ? `&after=${params.after}` : ''}`;
          const response = await this.client.apiRequest({ method: 'GET', path });
          return await response.json();
        },
        limit
      );
      console.log(`✅ Total blog posts fetched: ${allPosts.length}`);
      return allPosts;
    } catch (error: any) {
      console.error('Error fetching blog posts:', error);
      return this.handleOptionalScopeError(error, 'content', []);
    }
  }

  async getLandingPages(limit: number = 100): Promise<any[]> {
    try {
      console.log('🏠 Fetching ALL landing pages with pagination...');
      const allPages = await this.paginateList(
        async (params) => {
          const path = `/cms/v3/pages/landing-pages?limit=${params.limit}${params.after ? `&after=${params.after}` : ''}`;
          const response = await this.client.apiRequest({ method: 'GET', path });
          return await response.json();
        },
        limit
      );
      console.log(`✅ Total landing pages fetched: ${allPages.length}`);
      return allPages;
    } catch (error: any) {
      console.error('Error fetching landing pages:', error);
      return this.handleOptionalScopeError(error, 'content', []);
    }
  }
  async getKnowledgeBase(limit: number = 100): Promise<any[]> {
    try {
      console.log('📚 Fetching ALL knowledge base articles with pagination...');
      const allArticles = await this.paginateList(
        async (params) => {
          const path = `/cms/v3/knowledge-base/articles?limit=${params.limit}${params.after ? `&after=${params.after}` : ''}`;
          const response = await this.client.apiRequest({ method: 'GET', path });
          return await response.json();
        },
        limit
      );
      console.log(`✅ Total knowledge base articles fetched: ${allArticles.length}`);
      return allArticles;
    } catch (error: any) {
      console.error('Error fetching knowledge base:', error);
      return this.handleOptionalScopeError(error, 'cms.knowledge_base.articles.read', []);
    }
  }

  // Sales Hub Methods
  async getPipelines(objectType: string = 'deals'): Promise<any[]> {
    try {
      await this.rateLimitDelay();
      // Use the dedicated SDK method for pipelines
      const response = await this.client.crm.pipelines.pipelinesApi.getAll(objectType);
      return response.results || [];
    } catch (error: any) {
      console.error(`Error fetching pipelines for ${objectType}:`, error);
      // Handle cases where the object type might not support pipelines
      if (error.code === 404) {
          console.warn(`Pipelines for object type '${objectType}' not found.`);
          return [];
      }
      return this.handleOptionalScopeError(error, 'crm.pipelines.read', []);
    }
  }

  async getProperties(objectType: string): Promise<any[]> {
    if (!objectType) {
      console.error('Error in getProperties: objectType was not provided.');
      return [];
    }
    try {
      await this.rateLimitDelay();
      // Use the dedicated SDK method for properties
      const response = await this.client.crm.properties.coreApi.getAll(objectType);
      return response.results || [];
    } catch (error: any) {
      console.error(`Error fetching properties for ${objectType}:`, error);
      return this.handleOptionalScopeError(error, 'crm.properties.read', []);
    }
  }

  async getQuotes(limit: number = 100, updatedAfter?: string): Promise<any[]> {
    try {
      console.log('💰 Fetching ALL quotes with pagination...');
      
      const quoteProperties = [
        'hs_title', 'hs_expiration_date', 'hs_status', 'hs_quote_amount',
        'hs_sender_firstname', 'hs_sender_lastname', 'hs_sender_email',
        'hs_esign_enabled', 'hs_payment_enabled', 'hs_template_id',
        'createdate', 'hs_lastmodifieddate', 'hubspot_owner_id',
        'hs_quote_link', 'hs_pdf_download_link', 'hs_domain'
      ];

      if (updatedAfter) {
        const searchRequest = {
          sorts: ['hs_lastmodifieddate:desc'],
          properties: quoteProperties
        };
        this.applyUpdatedAfterFilter(searchRequest, updatedAfter, 'hs_lastmodifieddate');
        const incrementalQuotes = await this.paginateSearch(
          (req) => this.client.crm.quotes.searchApi.doSearch(req),
          searchRequest,
          limit
        );
        console.log(`✅ Total quotes fetched (incremental): ${incrementalQuotes.length}`);
        return convertDatetimeFields(incrementalQuotes);
      }

      const allQuotes = await this.paginateList(
        async (params) => {
          const props = quoteProperties.join(',');
          const path = `/crm/v3/objects/quotes?limit=${params.limit}&properties=${props}${params.after ? `&after=${params.after}` : ''}`;
          const response = await this.client.apiRequest({ method: 'GET', path });
          return await response.json();
        },
        limit
      );
      console.log(`✅ Total quotes fetched: ${allQuotes.length}`);
      return allQuotes;
    } catch (error: any) {
      console.error('Error fetching quotes:', error);
      return this.handleOptionalScopeError(error, 'crm.objects.quotes.read', []);
    }
  }

  async getUsers(limit: number = 100): Promise<any[]> {
    try {
      console.log('👥 Fetching ALL users with pagination...');
      const allUsers = await this.paginateList(
        // Use the dedicated SDK method for users
        (params) => this.client.settings.users.usersApi.getPage(
          params.limit,
          params.after
        ),
        limit
      );
      console.log(`✅ Total users fetched: ${allUsers.length}`);
      return allUsers;
    } catch (error: any) {
      console.error('Error fetching users:', error);
      // This scope is usually available, but we can handle errors just in case.
      return this.handleOptionalScopeError(error, 'settings.users.read', []);
    }
  }

  async getOwners(limit: number = 100): Promise<any[]> {
    try {
      console.log('🧑‍💼 Fetching ALL owners with pagination...');
      const allOwners = await this.paginateList(
        (params) => this.client.crm.owners.ownersApi.getPage(
          undefined,
          params.after,
          params.limit,
          false
        ),
        limit
      );
      console.log(`✅ Total owners fetched: ${allOwners.length}`);
      return allOwners;
    } catch (error: any) {
      console.error('Error fetching owners:', error);
      return this.handleOptionalScopeError(error, 'crm.objects.owners.read', []);
    }
  }

  // Service Hub Methods
  async getConversations(limit: number = 100): Promise<any[]> {
    try {
      console.log('💬 Fetching ALL conversations with pagination...');
      const allConversations = await this.paginateList(
        async (params) => {
          const path = `/conversations/v3/conversations/threads?limit=${params.limit}${params.after ? `&after=${params.after}` : ''}`;
          const response = await this.client.apiRequest({ method: 'GET', path });
          return await response.json();
        },
        limit
      );
      console.log(`✅ Total conversations fetched: ${allConversations.length}`);
      return allConversations;
    } catch (error: any) {
      console.error('Error fetching conversations:', error);
      return this.handleOptionalScopeError(error, 'conversations.read', []);
    }
  }


  async getCalls(limit: number = 100, updatedAfter?: string): Promise<any[]> {
    try {
      await this.rateLimitDelay();
      const callProperties = [
        'hs_call_title', 'hs_call_body', 'hs_call_status', 'hs_call_direction',
        'hs_call_disposition', 'hs_call_duration', 'hs_timestamp',
        'hs_call_from_number', 'hs_call_to_number', 'hs_call_recording_url',
        'createdate', 'hs_lastmodifieddate', 'hubspot_owner_id',
        'hs_call_callee_object_id', 'hs_call_callee_object_type'
      ];
      const searchRequest = {
        sorts: ['hs_timestamp:desc'],
        limit,
        properties: callProperties
      };
      if (updatedAfter) {
        this.applyUpdatedAfterFilter(searchRequest, updatedAfter, 'hs_lastmodifieddate');
      }
      const response = await this.client.crm.objects.searchApi.doSearch('calls', searchRequest);
      const results = convertDatetimeFields(response.results || []);
      console.log(`Total calls fetched: ${results.length}`);
      return results;
    } catch (error: any) {
      console.error('Error fetching calls:', error);
      return this.handleOptionalScopeError(error, 'crm.objects.calls.read', []);
    }
  }

  async getMeetings(limit: number = 100, updatedAfter?: string): Promise<any[]> {
    try {
      await this.rateLimitDelay();
      const meetingProperties = [
        'hs_meeting_title', 'hs_meeting_body', 'hs_meeting_outcome',
        'hs_meeting_start_time', 'hs_meeting_end_time', 'hs_timestamp',
        'hs_meeting_external_url', 'hs_meeting_location', 'hs_meeting_source',
        'createdate', 'hs_lastmodifieddate', 'hubspot_owner_id',
        'hs_internal_meeting_notes', 'hs_meeting_ms_teams_payload'
      ];
      const searchRequest = {
        sorts: ['hs_meeting_start_time:desc'],
        limit,
        properties: meetingProperties
      };
      if (updatedAfter) {
        this.applyUpdatedAfterFilter(searchRequest, updatedAfter, 'hs_lastmodifieddate');
      }
      const response = await this.client.crm.objects.searchApi.doSearch('meetings', searchRequest);
      const results = convertDatetimeFields(response.results || []);
      console.log(`Total meetings fetched: ${results.length}`);
      return results;
    } catch (error: any) {
      console.error('Error fetching meetings:', error);
      return this.handleOptionalScopeError(error, 'crm.objects.meetings.read', []);
    }
  }

  async getCustomObjects(objectType?: string, limit: number = 100): Promise<any[]> {
    const resolvedTypes: string[] = [];
    const schemaPropertiesByType = new Map<string, string[]>();

    if (objectType && objectType !== 'custom_objects') {
      resolvedTypes.push(objectType);
    } else {
      try {
        console.log('Discovering custom object schemas...');
        await this.rateLimitDelay();
        const schemaResponse = await this.client.crm.schemas.coreApi.getAll();
        const customSchemas =
          schemaResponse?.results?.filter(schema =>
            typeof schema?.objectTypeId === 'string' &&
            schema.objectTypeId.startsWith('2-')
          ) || [];

        for (const schema of customSchemas) {
          const typeId = schema.objectTypeId || schema.name;
          if (typeId) {
            resolvedTypes.push(typeId);
            const propertyNames = Array.isArray(schema.properties)
              ? schema.properties
                  .map((property) => property?.name)
                  .filter((name): name is string => Boolean(name))
              : [];
            if (propertyNames.length) {
              schemaPropertiesByType.set(typeId, propertyNames);
            }
          }
        }

        if (resolvedTypes.length === 0) {
          console.log('No custom object schemas found for this portal.');
          return [];
        }
      } catch (error: any) {
        return this.handleOptionalScopeError(error, 'crm.schemas.custom.read', []);
      }
    }

    try {
      const aggregated: any[] = [];

      for (const typeId of resolvedTypes) {
        console.log(`Fetching custom objects for type ${typeId}...`);
        try {
          const objectsForType = await this.paginateList(
            (params) =>
              this.client.crm.objects.basicApi.getPage(
                typeId,
                params.limit,
                params.after,
                schemaPropertiesByType.get(typeId)
              ),
            limit
          );

          const tagged = objectsForType.map(obj => ({
            ...obj,
            objectTypeId: typeId
          }));

          aggregated.push(...tagged);
        } catch (error: any) {
          if (error?.status === 404 || error?.code === 404) {
            console.warn(`Custom object type '${typeId}' not found.`);
            continue;
          }
          throw error;
        }
      }

      console.log(
        `Fetched ${aggregated.length} custom objects across ${resolvedTypes.length} type(s)`
      );
      return aggregated;
    } catch (error: any) {
      if (error?.status === 404 || error?.code === 404) {
        console.warn('Custom objects endpoint not found for this portal.');
        return [];
      }
      return this.handleOptionalScopeError(error, 'crm.objects.custom.read', []);
    }
  }

  // CMS Hub Methods
  async getSitePages(limit: number = 100): Promise<any[]> {
    try {
      console.log('📄 Fetching ALL site pages with pagination...');
      const allPages = await this.paginateList(
        async (params) => {
          const path = `/cms/v3/pages/site-pages?limit=${params.limit}${params.after ? `&after=${params.after}` : ''}`;
          const response = await this.client.apiRequest({ method: 'GET', path });
          return await response.json();
        },
        limit
      );
      console.log(`✅ Total site pages fetched: ${allPages.length}`);
      return allPages;
    } catch (error: any) {
      console.error('Error fetching site pages:', error);
      return this.handleOptionalScopeError(error, 'content', []);
    }
  }

  async getFiles(limit: number = 100): Promise<any[]> {
    try {
      console.log('📁 Fetching ALL files with pagination...');
      const pageSize = Math.min(Math.max(limit, 1), 200);
      const chunkThreshold = 9000; // stay well below HubSpot's 10k search cap
      const seenIds = new Set<string>();
      const allFiles: any[] = [];
      let createdAtCursor: string | undefined;
      let keepFetching = true;

      while (keepFetching) {
        const segment = await this.fetchFilesSegment(pageSize, chunkThreshold, createdAtCursor);

        if (segment.results.length === 0) {
          break;
        }

        for (const file of segment.results) {
          if (file?.id && !seenIds.has(file.id)) {
            seenIds.add(file.id);
            allFiles.push(file);
          }
        }

        if (!segment.reachedThreshold) {
          keepFetching = false;
        } else if (segment.lastCreatedAt) {
          const nextStart = new Date(segment.lastCreatedAt);
          nextStart.setMilliseconds(nextStart.getMilliseconds() + 1);
          createdAtCursor = nextStart.toISOString();
        } else {
          keepFetching = false;
        }
      }

      console.log(`✅ Total files fetched: ${allFiles.length}`);
      return allFiles;
    } catch (error: any) {
      console.error('Error fetching files:', error);
      return this.handleOptionalScopeError(error, 'files', []);
    }
  }

  private async fetchFilesSegment(
    pageSize: number,
    chunkThreshold: number,
    createdAtGte?: string
  ): Promise<{ results: any[]; reachedThreshold: boolean; lastCreatedAt?: string }> {
    const collected: any[] = [];
    let reachedThreshold = false;
    let lastCreatedAt: string | undefined;
    let after: string | undefined;
    let hasMore = true;

    while (hasMore) {
      await this.rateLimitDelay();
      const createdAtDate = createdAtGte ? new Date(createdAtGte) : undefined;
      const response = await this.client.files.filesApi.doSearch(
        undefined,
        after,
        undefined,
        pageSize,
        ['createdAt'],
        undefined,
        undefined,
        undefined,
        createdAtDate
      );

      const results = response.results || [];
      if (results.length === 0) {
        break;
      }

      collected.push(...results);
      lastCreatedAt = results[results.length - 1]?.createdAt;

      if (!response.paging?.next?.after) {
        hasMore = false;
      } else {
        after = response.paging.next.after;
      }

      if (collected.length >= chunkThreshold) {
        reachedThreshold = true;
        hasMore = false;
      }
    }

    return { results: collected, reachedThreshold, lastCreatedAt };
  }

  async getHubDbTables(limit: number = 100): Promise<any[]> {
    try {
      console.log('🗄 Fetching ALL HubDB tables with pagination...');
      const allTables = await this.paginateList(
        (params) => this.client.cms.hubdb.tablesApi.getAllTables(undefined, params.after, params.limit),
        limit
      );
      console.log(`✅ Total HubDB tables fetched: ${allTables.length}`);
      return allTables;
    } catch (error: any) {
      console.error('Error fetching HubDB tables:', error);
      return this.handleOptionalScopeError(error, 'hubdb', []);
    }
  }

  // Commerce Hub Methods
  async getProducts(limit: number = 100, updatedAfter?: string): Promise<any[]> {
    try {
      console.log('📦 Fetching ALL products with pagination...');
      
      const productProperties = [
        'name', 'description', 'price', 'hs_sku', 'hs_cost_of_goods_sold',
        'hs_recurring_billing_period', 'hs_product_type', 'hs_url',
        'createdate', 'hs_lastmodifieddate', 'hs_folder_id',
        'tax', 'hs_images', 'hs_avatar_filemanager_key'
      ];

      if (updatedAfter) {
        const searchRequest = {
          sorts: ['hs_lastmodifieddate:desc'],
          properties: productProperties
        };
        this.applyUpdatedAfterFilter(searchRequest, updatedAfter, 'hs_lastmodifieddate');
        const incrementalProducts = await this.paginateSearch(
          (req) => this.client.crm.products.searchApi.doSearch(req),
          searchRequest,
          limit
        );
        const normalizedProducts = convertDatetimeFields(incrementalProducts);
        return filterRecordsByUpdatedAfter(
          normalizedProducts,
          updatedAfter,
          (record) => record?.updatedAt,
          'products'
        );
      }

      const allProducts = await this.paginateList(
        (params) =>
          this.client.crm.products.basicApi.getPage(
            params.limit,
            params.after,
            productProperties
          ),
        limit
      );
      const normalizedProducts = convertDatetimeFields(allProducts);
      return normalizedProducts;
    } catch (error: any) {
      console.error('Error fetching products:', error);
      return this.handleOptionalScopeError(error, 'crm.objects.products.read', []);
    }
  }

  async getLineItems(limit: number = 100, updatedAfter?: string): Promise<any[]> {
    try {
      console.log('📋 Fetching ALL line items with pagination...');
      
      const lineItemProperties = [
        'name', 'quantity', 'price', 'amount', 'discount',
        'hs_sku', 'hs_product_id', 'hs_line_item_currency_code',
        'createdate', 'hs_lastmodifieddate',
        'hs_recurring_billing_period', 'hs_term_in_months', 'tax'
      ];

      if (updatedAfter) {
        const searchRequest = {
          sorts: ['hs_lastmodifieddate:desc'],
          properties: lineItemProperties
        };
        this.applyUpdatedAfterFilter(searchRequest, updatedAfter, 'hs_lastmodifieddate');
        const incrementalLineItems = await this.paginateSearch(
          (req) => this.client.crm.lineItems.searchApi.doSearch(req),
          searchRequest,
          limit
        );
        console.log(`✅ Total line items fetched (incremental): ${incrementalLineItems.length}`);
        return convertDatetimeFields(incrementalLineItems);
      }

      const allLineItems = await this.paginateList(
        (params) => this.client.crm.lineItems.basicApi.getPage(
          params.limit,
          params.after,
          lineItemProperties
        ),
        limit
      );
      console.log(`✅ Total line items fetched: ${allLineItems.length}`);
      return allLineItems;
    } catch (error: any) {
      console.error('Error fetching line items:', error);
      return this.handleOptionalScopeError(error, 'crm.objects.line_items.read', []);
    }
  }

  // Enterprise Methods
  async searchObjects(objectType: string, query: string, limit: number = 100): Promise<any[]> {
    try {
      await this.rateLimitDelay();
      const response = await this.client.apiRequest({
        method: 'POST',
        path: `/crm/v3/objects/${objectType}/search`,
        body: {
          query: query,
          limit: limit,
          sorts: [{ propertyName: "createdate", direction: "DESCENDING" }]
        }
      });
      const data = await response.json();
      return data?.results || [];
    } catch (error: any) {
      console.error('Error searching objects:', error);
      throw new Error(`HubSpot API error: ${error.message}`);
    }
  }

  async getLists(limit: number = 100): Promise<any[]> {
    const pageSize = Math.min(Math.max(limit, 1), 100);
    const legacyPageSize = Math.min(Math.max(limit, 1), 250);

    const parseMaybeJson = (value: any): any => {
      if (typeof value !== 'string') return value;
      const trimmed = value.trim();
      if (!trimmed || !trimmed.startsWith('{')) return value;
      try {
        return JSON.parse(trimmed);
      } catch (_) {
        return value;
      }
    };

    const listIdFromRecord = (list: any): string => {
      const raw = list?.listId ?? list?.list_id ?? list?.id ?? list?.internalListId;
      return String(raw ?? '').trim();
    };

    const isScopeResponseError = (payload: any): boolean => {
      const category = String(payload?.category ?? '').trim().toUpperCase();
      const message = String(payload?.message ?? '').trim().toLowerCase();
      return (
        category === 'MISSING_SCOPES' ||
        message.includes('scope') ||
        message.includes('permission') ||
        message.includes('unauthorized') ||
        message.includes('forbidden')
      );
    };

    const mergeListRecords = (baseRecord: any = {}, incomingRecord: any = {}): any => {
      const baseListId = listIdFromRecord(baseRecord);
      const incomingListId = listIdFromRecord(incomingRecord);
      const listId = baseListId || incomingListId;
      const merged: any = {
        ...baseRecord,
        ...incomingRecord,
        listId,
        id: baseRecord?.id ?? incomingRecord?.id ?? listId,
      };
      if (baseListId && incomingListId && baseListId !== incomingListId) {
        merged.legacyListId = merged.legacyListId || incomingListId;
      }

      const legacyFilterBranchRaw =
        merged?.ilsFilterBranch ??
        incomingRecord?.ilsFilterBranch ??
        baseRecord?.ilsFilterBranch;
      const parsedLegacyFilterBranch = parseMaybeJson(legacyFilterBranchRaw);

      if (parsedLegacyFilterBranch && !merged.filterBranch) {
        merged.filterBranch = parsedLegacyFilterBranch;
      }
      if (
        parsedLegacyFilterBranch &&
        Array.isArray(parsedLegacyFilterBranch?.filterBranches) &&
        !merged.filterBranches
      ) {
        merged.filterBranches = parsedLegacyFilterBranch.filterBranches;
      }
      if (incomingRecord?.filterBranches && !merged.filterBranches) {
        merged.filterBranches = incomingRecord.filterBranches;
      }
      if (incomingRecord?.filterBranch && !merged.filterBranch) {
        merged.filterBranch = incomingRecord.filterBranch;
      }
      if (incomingRecord?.filters && !merged.filters) {
        merged.filters = incomingRecord.filters;
      }

      if (!merged.name) {
        merged.name =
          incomingRecord?.name ||
          baseRecord?.name ||
          incomingRecord?.listName ||
          baseRecord?.listName ||
          (listId ? `List ${listId}` : 'List');
      }

      if (!merged.listType && merged.processingType) {
        merged.listType = merged.processingType;
      }
      if (merged.dynamic === undefined && merged.listType) {
        merged.dynamic = String(merged.listType).toUpperCase() === 'DYNAMIC';
      }

      return merged;
    };

    const fetchListsViaV3 = async (): Promise<any[]> => {
      console.log('Fetching lists via CRM Lists v3 search...');
      const allLists: any[] = [];
      let offset: number | undefined = undefined;
      let hasMore = true;

      while (hasMore) {
        await this.rateLimitDelay();

        const body: any = { limit: pageSize };
        if (typeof offset === 'number') {
          body.offset = offset;
        }

        const response = await this.client.apiRequest({
          method: 'POST',
          path: `/crm/v3/lists/search`,
          body
        });

        const data = await response.json();
        if (isScopeResponseError(data)) {
          throw new HubSpotScopeError('crm.lists.read', data);
        }
        const pageLists = Array.isArray(data?.lists) ? data.lists : [];
        if (pageLists.length > 0) {
          allLists.push(...pageLists);
          console.log(`Fetched ${pageLists.length} lists from v3 (total: ${allLists.length})`);
        }

        hasMore = Boolean(data?.hasMore);
        offset = data?.offset;
        if (!hasMore || typeof offset !== 'number') {
          break;
        }
      }

      const enriched: any[] = [];
      for (const list of allLists) {
        const listId = listIdFromRecord(list);
        if (!listId) {
          enriched.push(list);
          continue;
        }

        try {
          await this.rateLimitDelay();
          const detailResponse = await this.client.apiRequest({
            method: 'GET',
            path: `/crm/v3/lists/${encodeURIComponent(listId)}`
          });
          const detailPayload = await detailResponse.json();
          const detail = detailPayload?.list || detailPayload;
          enriched.push(mergeListRecords(list, detail));
        } catch (detailError: any) {
          console.warn(`Failed to fetch v3 list detail for list ${listId}:`, detailError?.message || detailError);
          enriched.push(list);
        }
      }

      return enriched;
    };

    const fetchListsViaLegacy = async (): Promise<any[]> => {
      console.log('Fetching lists via legacy contacts/v1 endpoint...');
      const legacyLists: any[] = [];
      const seenOffsets = new Set<string>();
      let offset: string | number | null = null;

      while (true) {
        await this.rateLimitDelay();
        const query = new URLSearchParams();
        query.append('count', legacyPageSize.toString());
        if (offset !== null && offset !== undefined && String(offset).trim()) {
          query.append('offset', String(offset));
        }

        const response = await this.client.apiRequest({
          method: 'GET',
          path: `/contacts/v1/lists?${query.toString()}`
        });

        const data = await response.json();
        if (isScopeResponseError(data)) {
          throw new HubSpotScopeError('crm.lists.read', data);
        }
        const pageLists = Array.isArray(data?.lists) ? data.lists : [];
        if (pageLists.length > 0) {
          legacyLists.push(...pageLists);
          console.log(`Fetched ${pageLists.length} lists from legacy API (total: ${legacyLists.length})`);
        }

        const nextOffset = data?.offset;
        const nextOffsetKey = nextOffset === null || nextOffset === undefined ? '' : String(nextOffset);
        if (!nextOffsetKey || pageLists.length === 0 || seenOffsets.has(nextOffsetKey)) {
          break;
        }

        seenOffsets.add(nextOffsetKey);
        offset = nextOffset;
      }

      return legacyLists;
    };

    let v3Lists: any[] = [];
    let v3Error: any = null;
    try {
      v3Lists = await fetchListsViaV3();
    } catch (error: any) {
      v3Error = error;
      console.warn('Lists v3 fetch failed, will try legacy fallback:', error?.message || error);
    }

    let legacyLists: any[] = [];
    let legacyError: any = null;
    try {
      legacyLists = await fetchListsViaLegacy();
    } catch (error: any) {
      legacyError = error;
      console.warn('Legacy lists fetch failed:', error?.message || error);
    }

    if (!v3Lists.length && !legacyLists.length) {
      if (v3Error) {
        return this.handleOptionalScopeError(v3Error, 'crm.lists.read', []);
      }
      if (legacyError) {
        return this.handleOptionalScopeError(legacyError, 'crm.lists.read', []);
      }
      return [];
    }

    const mergedByKey = new Map<string, any>();
    const nameKeyToPrimaryKey = new Map<string, string>();

    const getNameKey = (record: any): string => {
      return String(record?.name ?? record?.listName ?? '').trim().toLowerCase();
    };

    const upsert = (record: any) => {
      const listId = listIdFromRecord(record);
      const nameKey = getNameKey(record);
      const idKey = listId ? `id:${listId}` : '';
      const existingPrimaryKey =
        (idKey && mergedByKey.has(idKey) ? idKey : '') ||
        (nameKey && nameKeyToPrimaryKey.has(nameKey) ? nameKeyToPrimaryKey.get(nameKey) : '') ||
        idKey ||
        (nameKey ? `name:${nameKey}` : `anon:${mergedByKey.size + 1}`);

      const existing = mergedByKey.get(existingPrimaryKey);
      const mergedRecord = mergeListRecords(existing || {}, record);
      mergedByKey.set(existingPrimaryKey, mergedRecord);

      const mergedListId = listIdFromRecord(mergedRecord);
      const mergedNameKey = getNameKey(mergedRecord);
      if (mergedNameKey) {
        nameKeyToPrimaryKey.set(mergedNameKey, existingPrimaryKey);
      }
      if (mergedListId) {
        const mergedIdKey = `id:${mergedListId}`;
        if (mergedIdKey !== existingPrimaryKey) {
          const prior = mergedByKey.get(mergedIdKey);
          mergedByKey.set(mergedIdKey, mergeListRecords(prior || {}, mergedRecord));
          if (mergedNameKey) {
            nameKeyToPrimaryKey.set(mergedNameKey, mergedIdKey);
          }
        }
      }
    };

    v3Lists.forEach(upsert);
    legacyLists.forEach(upsert);

    const dedupedByName = new Map<string, any>();
    const unnamedLists: any[] = [];
    for (const record of mergedByKey.values()) {
      const nameKey = getNameKey(record);
      if (!nameKey) {
        unnamedLists.push(record);
        continue;
      }
      const existing = dedupedByName.get(nameKey);
      dedupedByName.set(nameKey, mergeListRecords(existing || {}, record));
    }
    const merged = [...dedupedByName.values(), ...unnamedLists];
    console.log(`Total lists fetched (merged): ${merged.length}`);
    return merged;
  }

  async getWorkflows(limit: number = 100): Promise<any[]> {
    try {
      console.log('?sT??? Fetching workflows from automation API with pagination...');

      const allWorkflows = await this.paginateList(
        async (params) => {
          const query = new URLSearchParams();
          query.append('limit', params.limit.toString());
          if (params.after) {
            query.append('after', params.after);
          }

          const response = await this.client.apiRequest({
            method: 'GET',
            path: `/automation/v3/workflows?${query.toString()}`
          });

          const data = await response.json();
          const results = Array.isArray(data?.workflows)
            ? data.workflows
            : Array.isArray(data?.results)
              ? data.results
              : [];

          return {
            results,
            paging: data?.paging
          };
        },
        limit
      );

      console.log(`?o. Total workflows fetched: ${allWorkflows.length}`);
      return allWorkflows;
    } catch (error: any) {
      console.error('Error fetching workflows:', error);
      return this.handleOptionalScopeError(error, 'automation', []);
    }
  }

  async getDomains(limit: number = 10): Promise<any[]> {
    try {
      console.log('?Y"? Fetching ALL domains with pagination...');

      const allDomains = await this.paginateList(
        async (params) => {
          const query = new URLSearchParams();
          query.append('limit', params.limit.toString());
          if (params.after) {
            query.append('after', params.after);
          }

          const response = await this.client.apiRequest({
            method: 'GET',
            path: `/cms/v3/domains?${query.toString()}`
          });

          return await response.json();
        },
        limit
      );

      console.log(`?o. Total domains fetched: ${allDomains.length}`);
      return allDomains;
    } catch (error: any) {
      console.error('Error fetching domains:', error);
      return this.handleOptionalScopeError(error, 'content', []);
    }
  }
}