/** * Configuration options for ScraperStorage - the modern, per-scraper storage manager. * * This configuration provides flexible options for setting up storage resources, * with automatic naming and resource creation capabilities. * * @example * ```typescript * // Minimal configuration * const config = { * region: 'us-east-1', * scraperName: 'sauto-cars', * databaseUrl: 'postgresql://user:pass@localhost:5432/mydb' * }; * * // Configuration with AWS credentials * const config = { * region: 'us-east-1', * scraperName: 'sauto-cars', * databaseUrl: 'postgresql://user:pass@localhost:5432/mydb', * accessKeyId: 'YOUR_ACCESS_KEY_ID', * secretAccessKey: 'YOUR_SECRET_ACCESS_KEY', * }; * * // Configuration with custom bucket * const config = { * region: 'eu-central-1', * scraperName: 'mobile-de', * databaseUrl: 'postgresql://user:pass@localhost:5432/mydb', * bucket: 'custom-bucket-name', * s3Prefix: 'car-listings', * }; * ``` */ export type ScraperStorageConfig = { /** AWS region where S3 resources will be created (e.g., 'us-east-1', 'eu-central-1', 'ap-southeast-1') */ region: string; /** * Optional: AWS Access Key ID for authentication. If not provided, uses default AWS credential provider chain. */ accessKeyId?: string; /** * Optional: AWS Secret Access Key for authentication. Required if accessKeyId is provided. */ secretAccessKey?: string; /** * Optional: AWS Session Token for temporary credentials. */ sessionToken?: string; /** * Unique scraper name used for organizing data. * Should be URL-safe and descriptive (e.g., 'sauto-cars', 'mobile-de', 'autotrader-uk'). */ scraperName: string; /** * Database URL for PostgreSQL connection. Required if prismaClient is not provided. * Example: "postgresql://user:password@localhost:5432/dbname" */ databaseUrl: string; /** * Optional: Custom S3 bucket name. If not provided, will be auto-generated * as "scraper-{scraperName}-bucket". */ bucket?: string; /** * Optional: S3 folder/prefix for organizing listings within the bucket. * Defaults to "listings". Examples: "cars", "properties", "products". */ s3Prefix?: string; /** * Optional: Whether to automatically create the S3 bucket if it doesn't exist. * Default: true. Set to false for production environments where buckets are pre-created. */ autoCreateBucket?: boolean; }; /** * Legacy configuration for individual scrapers in MultiScraperStorage. * * @deprecated Use ScraperStorageConfig with ScraperStorage instead. * * This configuration requires explicit table and bucket names and does not support * automatic resource creation. */ export type ScraperConfig = { /** DynamoDB table name for this scraper (must exist) */ tableName: string; /** S3 bucket name for this scraper (can be shared or unique, must exist) */ bucket: string; /** * Optional: S3 folder/prefix root for organizing data within the bucket. * Examples: "cars", "properties", "sauto", "mobile-de". */ s3Prefix?: string; }; /** * Legacy configuration for MultiScraperStorage managing multiple scrapers. * * @deprecated Use individual ScraperStorage instances instead. * * This configuration allows managing multiple scrapers from a single instance, * but requires all tables and buckets to exist beforehand. * * @example * ```typescript * const config: MultiScraperOptions = { * region: 'us-east-1', * scrapers: { * 'sauto': { * tableName: 'sauto-listings-table', * bucket: 'sauto-data-bucket', * s3Prefix: 'cars', * }, * 'mobileDe': { * tableName: 'mobile-de-listings-table', * bucket: 'mobile-de-data-bucket', * s3Prefix: 'vehicles', * }, * }, * }; * * // With AWS credentials * const configWithCredentials: MultiScraperOptions = { * region: 'us-east-1', * accessKeyId: 'YOUR_ACCESS_KEY_ID', * secretAccessKey: 'YOUR_SECRET_ACCESS_KEY', * scrapers: { * 'sauto': { * tableName: 'sauto-listings-table', * bucket: 'sauto-data-bucket', * s3Prefix: 'cars', * }, * }, * }; * ``` */ export type MultiScraperOptions = { /** AWS region where all scraper resources are located */ region: string; /** * Database URL for PostgreSQL connection. Required for multi-scraper setup. * Example: "postgresql://user:password@localhost:5432/dbname" */ databaseUrl: string; /** * Optional: AWS Access Key ID for authentication. If not provided, uses default AWS credential provider chain. */ accessKeyId?: string; /** * Optional: AWS Secret Access Key for authentication. Required if accessKeyId is provided. */ secretAccessKey?: string; /** * Optional: AWS Session Token for temporary credentials. */ sessionToken?: string; /** * Scraper configurations keyed by scraper name. * Each key should be a unique scraper identifier (e.g., "sauto", "mobileDe", "autotrader"). */ scrapers: Record; }; export declare const CONTENT_TYPE_JSON = "application/json"; export declare const CONTENT_ENCODING_GZIP = "gzip"; //# sourceMappingURL=config.d.ts.map