import { S3Client } from "@aws-sdk/client-s3"; import type { MultiScraperOptions, ScraperStorageConfig } from "./config.js"; import { DatabaseManager } from "./database.js"; import type { IndexRow, IngestResult, InputListingRecord, ListingRecord, ListOptions, ListResponse, ReadResult } from "./types.js"; /** * ScraperStorage - A modern, per-scraper storage manager for listing data using PostgreSQL (via Prisma) and S3. * * This class provides a complete storage solution for scraper applications that need to: * - Store listing records with automatic deduplication * - Maintain temporal data (first seen, last seen timestamps) * - Query by date ranges and source hosts * - Handle data compression and storage optimization * * Features: * - Automatic AWS RDS and S3 bucket creation (configurable) * - Atomic deduplication using PostgreSQL transactions * - GZIP compression for S3 storage efficiency * - Weekly partitioning for query performance * - Host-based querying and filtering * - Automatic timestamp management * - Automatic database migrations * - Support for both AWS RDS and local PostgreSQL databases * * @example * ```typescript * // Basic usage * const storage = new ScraperStorage({ * region: 'us-east-1', * scraperName: 'sauto-cars', * databaseUrl: 'postgresql://user:pass@localhost:5432/mydb', * autoCreateBucket: true, // Create S3 bucket if needed * s3Prefix: 'car-listings', // Optional: defaults to "listings" * }); * * // With automatic AWS RDS database creation * const storage = new ScraperStorage({ * region: 'us-east-1', * scraperName: 'sauto-cars', * databaseUrl: 'postgresql://user:pass@placeholder:5432/mydb', // Will be updated with actual RDS endpoint * autoCreateDatabase: true, // Create RDS instance if needed * autoMigrate: true, // Run migrations after creation * rdsConfig: { * dbInstanceIdentifier: 'my-scraper-db', * allocatedStorage: 20, * dbInstanceClass: 'db.t3.micro', * engine: 'postgres', * masterUsername: 'postgres', * masterUserPassword: 'secure-password', * publiclyAccessible: true, * backupRetentionPeriod: 7, * storageEncrypted: true, * }, * }); * * // With automatic local database creation * const storage = new ScraperStorage({ * region: 'us-east-1', * scraperName: 'sauto-cars', * databaseUrl: 'postgresql://user:pass@localhost:5432/mydb', * autoCreateDatabase: true, // Create database if needed * autoMigrate: true, // Run migrations after creation * databaseServer: { * host: 'localhost', * port: 5432, * user: 'postgres', * password: 'password', * ssl: false, * }, * }); * * // Or with custom Prisma client * const storage = new ScraperStorage({ * region: 'us-east-1', * scraperName: 'sauto-cars', * prismaClient: myPrismaClient, * bucket: 'my-custom-bucket', * }); * * // Ingest a listing record * const listing = { * url: 'https://www.sauto.cz/osobni/detail/skoda/octavia/17961234', * title: 'Škoda Octavia 2.0 TDI Style DSG', * price: { * withVat: 485000, * withoutVat: 400826, * currency: 'CZK', * }, * seller: { * name: 'AUTO CENTRUM PRAHA s.r.o.', * contact: { * phones: [{ label: 'hlavní', number: '+420 234 567 890' }], * emails: ['info@autocentrum-praha.cz'], * }, * }, * properties: { * 'značka': 'Škoda', * 'model': 'Octavia', * 'rok_vyroby': '2022', * }, * }; * * const result = await storage.ingest(listing); * console.log(result); // { status: "inserted", id: "abc123", s3Key: "listings/..." } * * // Retrieve the full record * const fullRecord = await storage.getFull(result.id); * * // List recent records with pagination * const recent = await storage.listRecent({ limit: 20 }); * ``` */ export declare class ScraperStorage { private config; private s3; private db; private bucketName; private prefix; private tableName; private isTest; private isInitialized; /** * Creates a new ScraperStorage instance for managing listing data with PostgreSQL and S3. * * @param config - Configuration object for the storage manager * @param config.region - AWS region for S3 and RDS resources * @param config.scraperName - Unique identifier for the scraper (used for bucket naming) * @param config.databaseUrl - PostgreSQL connection URL (required) * @param config.bucket - Optional custom S3 bucket name (defaults to `scraper-{scraperName}-bucket`) * @param config.s3Prefix - Optional S3 folder prefix for organizing data (defaults to 'listings') * @param config.autoCreateBucket - Whether to automatically create S3 bucket (default: true) * @param config.accessKeyId - Optional AWS access key ID * @param config.secretAccessKey - Optional AWS secret access key * @param config.sessionToken - Optional AWS session token for temporary credentials * * * @example * ```typescript * // Basic usage with existing database * const storage = new ScraperStorage({ * region: 'us-east-1', * scraperName: 'my-scraper', * databaseUrl: 'postgresql://user:pass@localhost:5432/mydb' * }); * * // With automatic AWS RDS creation * const storage = new ScraperStorage({ * region: 'us-east-1', * scraperName: 'my-scraper', * databaseUrl: 'postgresql://user:pass@placeholder:5432/mydb', * autoCreateDatabase: true, * rdsConfig: { * dbInstanceIdentifier: 'my-scraper-db', * masterUsername: 'postgres', * masterUserPassword: 'secure-password' * } * }); * ``` */ constructor(config: ScraperStorageConfig); /** * Set test mode and override clients for testing purposes. * * This method allows you to inject mock clients for unit testing, * bypassing the need for actual AWS and database connections. * * @param s3Client - Optional mock S3 client for testing * @param dbManager - Optional mock database manager for testing * * @example * ```typescript * const storage = new ScraperStorage(config); * * // Inject mock clients for testing * const mockS3 = new MockS3Client(); * const mockDb = new MockDatabaseManager(); * storage.setTestClients(mockS3, mockDb); * * // Now all operations will use the mock clients * await storage.ingest(testListing); * ``` */ setTestClients(s3Client?: S3Client, dbManager?: DatabaseManager): void; /** * Initialize storage resources by creating table and S3 bucket if needed. * * This method should be called before using the storage manager to ensure * all required resources are available. It will: * - Create table based on scraper name if it doesn't exist * - Create S3 bucket if `autoCreateBucket` is true * * @returns Promise that resolves when initialization is complete * * @throws {Error} When table creation fails * @throws {Error} When S3 bucket creation fails * * @example * ```typescript * const storage = new ScraperStorage({ * region: 'us-east-1', * scraperName: 'my-scraper', * databaseUrl: 'postgresql://user:pass@localhost:5432/mydb', * autoCreateBucket: true * }); * * // Initialize all resources * await storage.initialize(); * * // Now safe to use the storage * await storage.ingest(listingData); * ``` */ initialize(): Promise; /** * Checks if the storage is initialized and throws an error if not. * This should be called at the beginning of all public methods. */ private checkInitialized; /** * Clean up resources and close database connections. * * This method should be called when you're done using the storage manager * to properly close database connections and free up resources. * * @returns Promise that resolves when cleanup is complete * * @example * ```typescript * const storage = new ScraperStorage(config); * * try { * await storage.initialize(); * // Use the storage... * } finally { * await storage.close(); * } * ``` */ close(): Promise; /** * Convert database row to IndexRow for backward compatibility */ private dbRowToIndexRow; /** * Create S3 bucket if it doesn't exist */ private createBucketIfNotExists; /** * Generate S3 key for a listing */ private generateS3Key; /** * Clean and validate phone data to ensure only valid phone numbers are stored */ private cleanPhoneData; /** * Check if a phone array contains valid phone numbers */ private hasValidPhones; /** * Compare two phone arrays to determine if they are different */ private arePhoneArraysDifferent; /** * Detect if phone information has changed between two contact objects */ private detectPhoneChanges; /** * Merge contact information and detect changes */ private mergeContactsAndDetectChanges; /** * Merge contact information preserving existing data while adding new contacts * ALWAYS preserves existing phone numbers - never removes them during ingest */ private mergeContactInfo; /** * Store listing record in S3 */ private storeInS3; /** * Retrieve listing record from S3 */ private getFromS3; /** * Ingest a listing record into the storage system. * * This method handles the complete ingestion process including: * - Automatic deduplication based on URL * - Change detection for price and phone information * - Storage in both PostgreSQL (index) and S3 (full data) * - GZIP compression for S3 storage efficiency * - Atomic transactions to ensure data consistency * * @param record - The listing record to ingest * @param record.url - Unique URL of the listing (used for deduplication) * @param record.title - Title of the listing * @param record.price - Price information with VAT and currency * @param record.seller - Seller information including contact details * @param record.properties - Additional properties as key-value pairs * @param record.datePublished - Optional publication date * * @returns Promise resolving to ingestion result * @returns result.status - Status of the operation: 'inserted', 'updated', or 'duplicate' * @returns result.id - Unique identifier for the listing * @returns result.s3Key - S3 key where the full data is stored * @returns result.priceChanged - Whether price changed (only for 'updated' status) * @returns result.phoneChanged - Whether phone info changed (only for 'updated' status) * @returns result.lastSeenUpdated - Whether lastSeen was updated (only for 'duplicate' status) * * @throws {Error} When database operations fail * @throws {Error} When S3 operations fail * * @example * ```typescript * const listing = { * url: 'https://example.com/listing/123', * title: 'Amazing Car for Sale', * price: { * withVat: 25000, * withoutVat: 20661, * currency: 'USD' * }, * seller: { * name: 'John Doe', * contact: { * phones: [{ label: 'main', number: '+1234567890' }], * emails: ['john@example.com'] * } * }, * properties: { * brand: 'Toyota', * model: 'Camry', * year: '2020' * } * }; * * const result = await storage.ingest(listing); * console.log(result); // { status: 'inserted', id: 'abc123', s3Key: 'listings/2024/01/15/abc123.json.gz' } * ``` */ ingest(record: InputListingRecord): Promise; /** * Get index data for a listing by its unique identifier. * * This method retrieves the database index information for a listing, * including metadata like first seen, last seen, status, and S3 key. * It does not retrieve the full listing data from S3. * * @param id - Unique identifier for the listing (typically SHA256 hash of URL) * * @returns Promise resolving to index row data or undefined if not found * @returns result.id - Unique identifier * @returns result.url - Original URL * @returns result.s3Key - S3 key for full data * @returns result.firstSeen - When first discovered * @returns result.lastSeen - When last seen * @returns result.status - Current status ('stored' or 'pending') * @returns result.host - Extracted hostname * @returns result.datePublished - Original publication date * @returns result.weekPartition - Weekly partition for querying * @returns result.priceWithVat - Current price with VAT * @returns result.priceCurrency - Price currency * @returns result.hasPhone - Whether listing has phone contact * @returns result.lastPriceChange - When price last changed * @returns result.revision - Change revision number * * @throws {Error} When database query fails * * @example * ```typescript * const indexData = await storage.get('abc123'); * if (indexData) { * console.log(`Listing ${indexData.url} was first seen on ${indexData.firstSeen}`); * console.log(`Current price: ${indexData.priceWithVat} ${indexData.priceCurrency}`); * } * ``` */ get(id: string): Promise; /** * Get full listing data including both index information and S3 content. * * This method retrieves the complete listing data by first getting the index * information from the database, then fetching the full listing record from S3. * * @param id - Unique identifier for the listing * * @returns Promise resolving to read result * @returns result.status - Status of the operation: 'ok', 'not_found', or 'pending' * @returns result.id - Unique identifier * @returns result.s3Key - S3 key where data is stored (for 'ok' status) * @returns result.record - Full listing record data (for 'ok' status) * * @throws {Error} When database query fails * @throws {Error} When S3 retrieval fails * * @example * ```typescript * const result = await storage.getFull('abc123'); * * switch (result.status) { * case 'ok': * console.log(`Found listing: ${result.record.title}`); * console.log(`Price: ${result.record.price?.withVat} ${result.record.price?.currency}`); * break; * case 'not_found': * console.log('Listing not found'); * break; * case 'pending': * console.log('Listing is still being processed'); * break; * } * ``` */ getFull(id: string): Promise; /** * Update or add a phone number for an existing listing. * * This method updates the phone contact information for a listing both in the * database index and in the full record stored in S3. It adds the phone number * to existing phones without removing any. * * @param id - Unique identifier for the listing * @param phoneNumber - New phone number to add (or null/empty to do nothing) * @param label - Optional label for the phone number (e.g., 'main', 'mobile'). Defaults to 'phone' if not provided. * * @returns Promise resolving to update result * @returns result.success - Whether the update was successful * @returns result.phoneChanged - Whether the phone information actually changed * @returns result.hasPhone - New hasPhone status after update * @returns result.phonesCount - Total number of phones after update * * @throws {Error} When listing is not found or update fails * * @example * ```typescript * // Add a phone number with custom label * const result = await storage.updatePhoneNumber('listing123', '+1234567890', 'main'); * if (result.success) { * console.log('Phone number added successfully'); * } * * // Add a phone number with default label ('phone') * const result2 = await storage.updatePhoneNumber('listing123', '+1234567890'); * ``` */ updatePhoneNumber(id: string, phoneNumber: string | null, label?: string): Promise<{ success: boolean; phoneChanged: boolean; hasPhone: boolean; phonesCount: number; }>; /** * Read full listing record directly from S3 using its S3 key. * * This method bypasses the database and directly retrieves and decompresses * the full listing data from S3. Useful when you already have the S3 key * from index data. * * @param s3Key - S3 key where the listing data is stored * * @returns Promise resolving to the full listing record * * @throws {Error} When S3 object is not found * @throws {Error} When decompression fails * @throws {Error} When JSON parsing fails * * @example * ```typescript * // First get the S3 key from index data * const indexData = await storage.get('abc123'); * if (indexData?.s3Key) { * // Then get the full record directly from S3 * const fullRecord = await storage.getByS3Key(indexData.s3Key); * console.log(fullRecord.title); * } * ``` */ getByS3Key(s3Key: string): Promise; /** * List recent listings with pagination and optional date filtering. * * This method retrieves listings ordered by last seen date (most recent first) * with support for pagination and date range filtering. * * @param options - Optional listing options * @param options.limit - Maximum number of listings to return (1-1000, default: 50) * @param options.offset - Number of listings to skip for pagination (default: 0) * @param options.fromISO - Filter listings seen after this ISO date string * @param options.toISO - Filter listings seen before this ISO date string * * @returns Promise resolving to paginated listing response * @returns result.items - Array of index row data * @returns result.nextCursor - Cursor for next page (null if no more pages) * * @throws {Error} When database query fails * * @example * ```typescript * // Get first 20 recent listings * const recent = await storage.listRecent({ limit: 20 }); * console.log(`Found ${recent.items.length} listings`); * * // Get listings from last week * const lastWeek = new Date(); * lastWeek.setDate(lastWeek.getDate() - 7); * const recentWeek = await storage.listRecent({ * fromISO: lastWeek.toISOString(), * limit: 100 * }); * * // Pagination * const page1 = await storage.listRecent({ limit: 50, offset: 0 }); * if (page1.nextCursor) { * const page2 = await storage.listRecent({ limit: 50, offset: 50 }); * } * ``` */ listRecent(options?: ListOptions): Promise>; /** * List listings filtered by host with pagination and optional date filtering. * * This method retrieves listings from a specific host (domain) ordered by * last seen date (most recent first) with support for pagination and date range filtering. * * @param host - Hostname to filter by (e.g., 'example.com', 'cars.com') * @param options - Optional listing options * @param options.limit - Maximum number of listings to return (1-1000, default: 50) * @param options.offset - Number of listings to skip for pagination (default: 0) * @param options.fromISO - Filter listings seen after this ISO date string * @param options.toISO - Filter listings seen before this ISO date string * * @returns Promise resolving to paginated listing response * @returns result.items - Array of index row data for the specified host * @returns result.nextCursor - Cursor for next page (null if no more pages) * * @throws {Error} When database query fails * * @example * ```typescript * // Get recent listings from a specific host * const carsComListings = await storage.listByHost('cars.com', { limit: 100 }); * console.log(`Found ${carsComListings.items.length} listings from cars.com`); * * // Get listings from a host within a date range * const recentCars = await storage.listByHost('cars.com', { * fromISO: '2024-01-01T00:00:00Z', * toISO: '2024-01-31T23:59:59Z', * limit: 50 * }); * ``` */ listByHost(host: string, options?: ListOptions): Promise>; /** * List all listings with pagination and optional host filtering (advanced method). * * This is the most flexible listing method that supports all filtering options * including host filtering, date range filtering, and pagination. * * @param options - Optional listing options * @param options.host - Optional hostname to filter by * @param options.limit - Maximum number of listings to return (1-1000, default: 50) * @param options.offset - Number of listings to skip for pagination (default: 0) * @param options.fromISO - Filter listings seen after this ISO date string * @param options.toISO - Filter listings seen before this ISO date string * * @returns Promise resolving to paginated listing response * @returns result.items - Array of index row data * @returns result.nextCursor - Cursor for next page (null if no more pages) * * @throws {Error} When database query fails * * @example * ```typescript * // Get all recent listings * const allListings = await storage.list({ limit: 100 }); * * // Get listings from specific host * const hostListings = await storage.list({ * host: 'cars.com', * limit: 50 * }); * * // Complex filtering * const filtered = await storage.list({ * host: 'cars.com', * fromISO: '2024-01-01T00:00:00Z', * toISO: '2024-01-31T23:59:59Z', * limit: 200 * }); * ``` */ list(options?: ListOptions & { host?: string; }): Promise>; /** * Count listings matching optional filters. * * This method provides efficient counting of listings without retrieving * the actual data, useful for pagination and analytics. * * @param filters - Optional filters to apply * @param filters.host - Filter by hostname * @param filters.fromISO - Filter listings seen after this ISO date string * @param filters.toISO - Filter listings seen before this ISO date string * * @returns Promise resolving to the count of matching listings * * @throws {Error} When database query fails * * @example * ```typescript * // Count all listings * const totalCount = await storage.count(); * console.log(`Total listings: ${totalCount}`); * * // Count listings from specific host * const hostCount = await storage.count({ host: 'cars.com' }); * console.log(`Listings from cars.com: ${hostCount}`); * * // Count listings in date range * const recentCount = await storage.count({ * fromISO: '2024-01-01T00:00:00Z', * toISO: '2024-01-31T23:59:59Z' * }); * console.log(`Listings in January 2024: ${recentCount}`); * ``` */ count(filters?: { host?: string; fromISO?: string; toISO?: string; }): Promise; /** * Delete a listing from both the database index and S3 storage. * * This method performs a complete cleanup by removing the listing from * both the PostgreSQL index and the S3 storage. The operation is atomic * - if either deletion fails, the operation is considered failed. * * @param id - Unique identifier of the listing to delete * * @returns Promise resolving to true if deletion was successful, false if listing not found * * @throws {Error} When database deletion fails * @throws {Error} When S3 deletion fails (logged but doesn't throw) * * @example * ```typescript * // Delete a specific listing * const deleted = await storage.delete('abc123'); * if (deleted) { * console.log('Listing deleted successfully'); * } else { * console.log('Listing not found'); * } * * // Batch deletion * const idsToDelete = ['abc123', 'def456', 'ghi789']; * for (const id of idsToDelete) { * const deleted = await storage.delete(id); * console.log(`Deleted ${id}: ${deleted}`); * } * ``` */ delete(id: string): Promise; /** * List active listings that are older than specified days. * * This method finds listings that are: * - Active: last seen within the active threshold (default: 7 days) * - Old: first seen more than the specified number of days ago * * This is useful for finding listings that have been around for a while * but are still being actively updated, which might indicate stale or * problematic listings that need attention. * * @param options - Filtering options * @param options.olderThanDays - Minimum age in days (firstSeen must be older than this) * @param options.activeThresholdDays - Maximum days since last seen to be considered active (default: 7) * @param options.lastSeenNotOlderThanDays - Optional: filter out listings not seen for more than X days * @param options.limit - Maximum number of listings to return (1-1000, default: 50) * @param options.offset - Number of listings to skip for pagination (default: 0) * @param options.host - Optional hostname to filter by * * @returns Promise resolving to paginated listing response * @returns result.items - Array of index row data for active old listings * @returns result.nextCursor - Cursor for next page (null if no more pages) * * @throws {Error} When database query fails * * @example * ```typescript * // Find listings older than 30 days that are still active * const staleListings = await storage.listActiveListingsOlderThan({ * olderThanDays: 30, * limit: 100 * }); * console.log(`Found ${staleListings.items.length} potentially stale listings`); * * // Find very old listings that are still being updated * const veryOldActive = await storage.listActiveListingsOlderThan({ * olderThanDays: 90, * activeThresholdDays: 3, // Seen within last 3 days * limit: 50 * }); * ``` */ listActiveListingsOlderThan(options: { olderThanDays: number; activeThresholdDays?: number; lastSeenNotOlderThanDays?: number; limit?: number; offset?: number; cursor?: string | null; host?: string; }): Promise>; /** * Count active listings that are older than specified days. * * This method provides efficient counting of listings that are both active * and old, without retrieving the actual data. Useful for analytics and * monitoring purposes. * * @param options - Filtering options * @param options.olderThanDays - Minimum age in days (firstSeen must be older than this) * @param options.activeThresholdDays - Maximum days since last seen to be considered active (default: 7) * @param options.lastSeenNotOlderThanDays - Optional: filter out listings not seen for more than X days * @param options.host - Optional hostname to filter by * * @returns Promise resolving to the count of active old listings * * @throws {Error} When database query fails * * @example * ```typescript * // Count potentially stale listings * const staleCount = await storage.countActiveListingsOlderThan({ * olderThanDays: 30 * }); * console.log(`Found ${staleCount} potentially stale listings`); * * // Count very old active listings from specific host * const hostStaleCount = await storage.countActiveListingsOlderThan({ * olderThanDays: 60, * activeThresholdDays: 3, * host: 'cars.com' * }); * ``` */ countActiveListingsOlderThan(options: { olderThanDays: number; activeThresholdDays?: number; lastSeenNotOlderThanDays?: number; host?: string; }): Promise; /** * List active listings without phone numbers within a specific time range. * * This method finds listings that: * - Are active (last seen within the active threshold) * - Do not have phone contact information (hasPhone=false) * - Optionally meet age criteria * * This is useful for identifying listings that might need contact information * updates or for quality control purposes. * * @param options - Filtering options * @param options.olderThanDays - Optional: only show listings first seen more than X days ago * @param options.activeThresholdDays - Maximum days since last seen to be considered active (default: 7) * @param options.lastSeenNotOlderThanDays - Optional: filter out listings not seen for more than X days * @param options.limit - Maximum number of listings to return (1-1000, default: 50) * @param options.offset - Number of listings to skip for pagination (default: 0) * @param options.host - Optional hostname to filter by * * @returns Promise resolving to paginated listing response * @returns result.items - Array of index row data for listings without phone * @returns result.nextCursor - Cursor for next page (null if no more pages) * * @throws {Error} When database query fails * * @example * ```typescript * // Find active listings without phone numbers * const noPhoneListings = await storage.listListingsWithoutPhoneInRange({ * limit: 100 * }); * console.log(`Found ${noPhoneListings.items.length} listings without phone numbers`); * * // Find old listings without phone numbers * const oldNoPhone = await storage.listListingsWithoutPhoneInRange({ * olderThanDays: 30, * activeThresholdDays: 7, * limit: 50 * }); * ``` */ listListingsWithoutPhoneInRange(options: { olderThanDays?: number; activeThresholdDays?: number; lastSeenNotOlderThanDays?: number; limit?: number; offset?: number; cursor?: string | null; host?: string; }): Promise>; /** * Count active listings without phone numbers within a specific time range. * * This method provides efficient counting of listings that are active but * lack phone contact information, without retrieving the actual data. * Useful for analytics and quality monitoring. * * @param options - Filtering options * @param options.olderThanDays - Optional: only count listings first seen more than X days ago * @param options.activeThresholdDays - Maximum days since last seen to be considered active (default: 7) * @param options.lastSeenNotOlderThanDays - Optional: filter out listings not seen for more than X days * @param options.host - Optional hostname to filter by * * @returns Promise resolving to the count of active listings without phone numbers * * @throws {Error} When database query fails * * @example * ```typescript * // Count active listings without phone numbers * const noPhoneCount = await storage.countListingsWithoutPhoneInRange({}); * console.log(`Found ${noPhoneCount} active listings without phone numbers`); * * // Count old listings without phone numbers from specific host * const hostNoPhoneCount = await storage.countListingsWithoutPhoneInRange({ * olderThanDays: 30, * host: 'cars.com' * }); * console.log(`Found ${hostNoPhoneCount} old listings without phone from cars.com`); * ``` */ countListingsWithoutPhoneInRange(options: { olderThanDays?: number; activeThresholdDays?: number; lastSeenNotOlderThanDays?: number; host?: string; }): Promise; /** * Fix hasPhone flags in the database by checking actual phone data in S3. * * This method iterates through all listings in the database and corrects the * hasPhone flag based on the actual phone data stored in S3. This is useful * for fixing data after a bug or migration issue. * * @param options - Options for the fix operation * @param options.batchSize - Number of records to process at once (default: 100) * @param options.host - Optional hostname to filter records to fix * @param options.dryRun - If true, only report what would be changed without making updates * @param options.onProgress - Optional callback to report progress * * @returns Promise resolving to fix results * @returns result.totalProcessed - Total number of records processed * @returns result.totalFixed - Number of records that were fixed * @returns result.errors - Array of errors encountered during processing * * @example * ```typescript * // Fix all records * const result = await storage.fixHasPhoneFlags({ * onProgress: (processed, fixed) => { * console.log(`Processed: ${processed}, Fixed: ${fixed}`); * } * }); * console.log(`Fixed ${result.totalFixed} out of ${result.totalProcessed} records`); * * // Dry run to see what would be fixed * const dryRunResult = await storage.fixHasPhoneFlags({ * dryRun: true, * host: 'bazos.sk' * }); * console.log(`Would fix ${dryRunResult.totalFixed} records`); * ``` */ fixHasPhoneFlags(options?: { batchSize?: number; host?: string; dryRun?: boolean; onProgress?: (processed: number, fixed: number) => void; }): Promise<{ totalProcessed: number; totalFixed: number; errors: { id: string; error: string; }[]; }>; } /** * MultiScraperStorage - Manages multiple ScraperStorage instances * * @deprecated Use individual ScraperStorage instances instead */ export declare class MultiScraperStorage { private scrapers; private config; constructor(config: MultiScraperOptions & { isTest?: boolean; }); /** * Initialize all scraper storages */ initialize(): Promise; /** * Set test clients for all scrapers */ setTestClients(s3Client: S3Client, dbManager: DatabaseManager): void; /** * Get a specific scraper storage instance */ getStorage(scraperName: string): ScraperStorage | undefined; /** * Close all scraper storage instances */ close(): Promise; ingest(scraperName: string, record: InputListingRecord): Promise; get(scraperName: string, id: string): Promise; getFull(scraperName: string, id: string): Promise; updatePhoneNumber(scraperName: string, id: string, phoneNumber: string | null, label?: string): Promise<{ success: boolean; phoneChanged: boolean; hasPhone: boolean; phonesCount: number; }>; listRecent(scraperName: string, options?: ListOptions): Promise>; listByHost(scraperName: string, host: string, options?: ListOptions): Promise>; list(scraperName: string, options?: ListOptions & { host?: string; }): Promise>; count(scraperName: string, filters?: { host?: string; fromISO?: string; toISO?: string; }): Promise; delete(scraperName: string, id: string): Promise; } //# sourceMappingURL=storage-manager.d.ts.map