import { U as UrlFieldExtractor, f as JobLocation, M as MetadataMap, V as ValidationResult, d as JobDetailsResult, e as JobListing, l as ScraperProvider, g as MetadataValue, m as ScraperProviderManifest } from '../provider-Ce-KxEDG.js'; export { JobValidationIssue, ValidatableJobDetail, ValidatableJobListing, ValidationSummary, isValidUrl, validateDetail, validateDetails, validateListing, validateListings } from '../validation/index.js'; export { U as UrlExtractionResult, m as extractNamedParam, n as extractParamFromUrl, p as extractUrlInfo, r as matchesAnyUrlPattern, s as matchesUrlPattern } from '../helpers-ndADLAXB.js'; /** * @jobo/scraper-sdk/testing — Typed test case definitions and assertion helpers. * * tests.json supports five test types: * * - "urlPattern" — offline check: does the URL match the provider's urlPatterns? * - "validation" — calls provider.validate(url), checks isValid + companyId + baseUrl extracted * - "jobId" — uses manifest jobIdExtractor definitions to extract jobId + companyId from a job detail URL * - "listingUrl" — offline: reconstructs the canonical listing URL from a job detail URL via companyIdExtractor definitions * - "scrape" — full live scrape, checks listing/detail counts, field presence, and meta keys */ /** * Checks that the URL matches (or doesn't match) the provider's urlPatterns. * Offline — no HTTP requests made. * * @example * { "type": "urlPattern", "name": "Bare domain matches", "url": "https://acme.applytojob.com", "expect": { "matches": true } } */ interface UrlPatternTestCase { type: "urlPattern"; name: string; url: string; expect: { /** Whether the URL should match the provider's urlPatterns. */ matches: boolean; }; } /** * Calls provider.validate(url) and checks the result. * Makes no HTTP requests — validate() is synchronous/cheap. * * @example * { "type": "validation", "name": "Extracts company ID", "url": "https://acme.applytojob.com", "expect": { "isValid": true, "companyId": "acme" } } */ interface ValidationTestCase { type: "validation"; name: string; url: string; expect: { /** Whether validate() should return isValid: true. */ isValid: boolean; /** * Expected companyId extracted from the URL. * Set to `null` to assert that no companyId is extracted. * Omit entirely to skip this assertion. */ companyId?: string | null; /** Expected baseUrl returned. */ baseUrl?: string; }; } /** * Calls provider.validate(jobDetailUrl) and checks the jobId extracted. * Uses the provider's jobIdExtractor definitions (or validate() result) to extract the job ID. * Makes no HTTP requests. * * @example * { "type": "jobId", "name": "Extracts job ID from detail URL", "url": "https://boards.greenhouse.io/acme/jobs/12345", "expect": { "jobId": "12345" } } */ interface JobIdTestCase { type: "jobId"; name: string; /** A job detail URL (e.g. the applyUrl or listingUrl of a specific job). */ url: string; expect: { /** Expected jobId extracted from the URL. */ jobId: string; /** Expected companyId extracted from the URL (optional). */ companyId?: string; }; } /** * Runs a full live scrape and checks listing/detail counts, field presence, and meta keys. * Makes real HTTP requests to the ATS. * * @example * { "type": "scrape", "name": "Anthropic jobs", "url": "https://job-boards.greenhouse.io/anthropic", "maxPages": 1, "maxDetails": 3, "expect": { "minListings": 1, "listingHasFields": ["externalId", "applyUrl"], "detailHasFields": ["title", "description"], "detailMetaHasKeys": ["employment_type", "department"] } } */ interface ScrapeTestCase { type: "scrape"; name: string; url: string; /** Max listing pages to scrape. Default: 1. */ maxPages?: number; /** Max detail fetches (split scrapers). Default: 3. */ maxDetails?: number; expect: { /** Minimum number of listings expected. */ minListings?: number; /** Fields that must be present on every listing. */ listingHasFields?: string[]; /** Fields that must be present on every detail. */ detailHasFields?: string[]; /** * Meta keys that must be present (non-null, non-empty) on at least one detail. * Checks the `meta` map on each detail record. * @example ["employment_type", "department", "is_remote"] */ detailMetaHasKeys?: string[]; }; } /** * Reconstructs the canonical listing URL from a job detail URL using the * provider's companyIdExtractor definitions. Offline — no HTTP requests made. * * @example * { "type": "listingUrl", "name": "Builds listing URL from job URL", "url": "https://boards.greenhouse.io/acme/jobs/12345", "expect": { "listingUrl": "https://boards.greenhouse.io/acme" } } */ interface ListingUrlTestCase { type: "listingUrl"; name: string; /** A job detail URL or any sub-page URL. */ url: string; expect: { /** The expected canonical listing URL. */ listingUrl: string; }; } /** * A single test case in tests.json. Discriminated by `type`. */ type ProviderTestCase = UrlPatternTestCase | ValidationTestCase | JobIdTestCase | ListingUrlTestCase | ScrapeTestCase; /** * The full tests.json schema. */ interface ProviderTestSuite { tests: ProviderTestCase[]; } interface AssertionResult { /** Assertion name (e.g. "urlPatternMatches", "companyId", "jobId"). */ name: string; /** Whether the assertion passed. */ passed: boolean; /** Expected value (human-readable). */ expected: string; /** Actual value (human-readable). */ actual: string; } declare function evaluateUrlPatternAssertions(tc: UrlPatternTestCase, urlPatterns: string[]): AssertionResult[]; declare function evaluateValidationAssertions(tc: ValidationTestCase, actual: { isValid: boolean; companyId?: string; baseUrl?: string; }): AssertionResult[]; declare function evaluateJobIdAssertions(tc: JobIdTestCase, actual: { jobId?: string; companyId?: string; }): AssertionResult[]; declare function evaluateListingUrlAssertions(tc: ListingUrlTestCase, companyIdExtractors: UrlFieldExtractor[]): AssertionResult[]; declare function evaluateScrapeAssertions(tc: ScrapeTestCase, actual: { totalListings: number; listings: Array>; details: Array>; }): AssertionResult[]; /** * @jobo/scraper-sdk/testing — Test Runner * * A lightweight test harness for exercising scraper providers against * live ATS pages or mock data. Validates that the provider correctly * discovers listings, paginates, and extracts job details. * * The test runner creates a mock {@link ScraperContext} that uses * real HTTP requests (via `node-fetch` or Node's built-in `fetch`) * so you can test against live endpoints during development. */ /** * Configuration for a test run. */ interface TestConfig { /** The scraper provider instance to test. */ provider: ScraperProvider; /** The ATS URL to test against. */ url: string; /** Company identifier (if not extractable from the URL). */ companyId?: string; /** Maximum number of listing pages to scrape. Default: `2`. */ maxPages?: number; /** Maximum number of detail fetches to perform (split scrapers). Default: `3`. */ maxDetails?: number; /** Additional config to pass to the scraper context. */ config?: Record; /** Whether to print verbose output. Default: `true`. */ verbose?: boolean; } /** * Result of a test run. */ interface TestResult { /** Whether all test phases passed. */ success: boolean; /** Validation result. */ validation: ValidationResult; /** Total listings discovered across all pages. */ totalListings: number; /** Number of pages scraped. */ pagesScraped: number; /** Detail results (split scrapers only). */ detailResults: JobDetailsResult[]; /** All listings collected. */ listings: JobListing[]; /** Validation issues found in the scraped data. */ issues: TestIssue[]; /** Total duration in milliseconds. */ duration: number; /** Metadata quality score (0-100). */ qualityScore?: QualityScore; } /** * Metadata quality assessment for a scraper run. */ interface QualityScore { /** Overall score 0-100. */ overall: number; /** Score breakdown by category. */ breakdown: { /** Do listings have required fields? (0-100) */ requiredFields: number; /** Do details have descriptions, locations, titles? (0-100) */ detailCompleteness: number; /** How much structured metadata is captured? (0-100) */ metadataRichness: number; /** Are URLs valid and not duplicated? (0-100) */ dataIntegrity: number; }; /** Human-readable summary of quality findings. */ findings: string[]; } /** * A data quality issue found during testing. */ interface TestIssue { /** Severity level. */ severity: "error" | "warning" | "info"; /** Which phase the issue was found in. */ phase: "validation" | "listings" | "details"; /** Human-readable description. */ message: string; /** The listing or detail that triggered the issue, if applicable. */ externalId?: string; } /** * Run a comprehensive test of a scraper provider. * * Phases: * 1. **Validation** — Calls `provider.validate(url)` and checks the result. * 2. **Initialization** — Calls `provider.initialize(ctx)` if defined. * 3. **Listings** — Calls `provider.scrapeListings(ctx)` up to `maxPages` times. * 4. **Details** — For split scrapers, calls `provider.scrapeJobDetails(ctx, listing)` * for up to `maxDetails` listings. * 5. **Data Quality** — Validates required fields, deduplication, and data integrity. * * @param config — Test configuration. * @returns Test result with all collected data and any issues. * * @example * ```ts * import { runTest } from "@jobo/scraper-sdk/testing"; * import provider from "./src/provider"; * * const result = await runTest({ * provider, * url: "https://boards.greenhouse.io/acme", * maxPages: 2, * maxDetails: 5, * }); * * if (result.success) { * console.log(`✓ Found ${result.totalListings} listings`); * } else { * console.error("Issues:", result.issues); * } * ``` */ declare function runTest(config: TestConfig): Promise; /** * Compute a quality score for the test result. * Evaluates: required fields, detail completeness, metadata richness, data integrity. */ declare function computeQualityScore(listings: JobListing[], detailResults: JobDetailsResult[], issues: TestIssue[]): QualityScore; /** * Serializable test result with truncated descriptions for storage. */ interface SavedTestResult { providerId: string; testUrl: string; timestamp: string; success: boolean; duration: number; totalListings: number; pagesScraped: number; qualityScore: QualityScore; issues: TestIssue[]; /** All listings with truncated descriptions. */ listings: SerializedJobListing[]; /** All detail records with truncated descriptions. */ details: SerializedJobDetails[]; } interface SerializedJobListing { externalId: string; applyUrl: string; listingUrl: string; title?: string; description_preview?: string; description_length?: number; locations?: JobLocation[]; companyName?: string; postedAt?: string; closesAt?: string; meta: MetadataMap; } interface SerializedJobDetails { externalId: string; title: string; description_preview: string; description_length: number; listingUrl: string; applyUrl: string; locations: JobLocation[]; companyName?: string; postedAt?: string; closesAt?: string; meta: MetadataMap; } /** * Convert a TestResult into a serializable format with truncated descriptions. */ declare function serializeTestResult(providerId: string, testUrl: string, result: TestResult, maxDescriptionLength?: number): SavedTestResult; /** * Generate a markdown report for a single ATS/provider test run. */ declare function generateSingleProviderReport(result: SavedTestResult): string; /** * Configuration for running a full test suite from tests.json. */ interface TestSuiteConfig { /** The scraper provider instance to test. */ provider: ScraperProvider; /** The provider manifest (for URL pattern matching). */ manifest: ScraperProviderManifest; /** Test cases from tests.json. */ testSuite: ProviderTestSuite; /** Whether to print verbose output. Default: true. */ verbose?: boolean; } /** * Result of a single test case within a suite. */ interface TestCaseResult { /** Test case name from tests.json. */ name: string; /** URL tested. */ url: string; /** Whether all assertions passed and the test run succeeded. */ passed: boolean; /** Assertion results from the expect block. */ assertions: AssertionResult[]; /** The underlying test run result (null if test was assertion-only). */ testResult: TestResult | null; /** Duration in milliseconds. */ durationMs: number; /** Error message if the test case threw. */ error?: string; } /** * Result of running a full test suite. */ interface TestSuiteResult { /** Provider ID. */ providerId: string; /** Whether all test cases passed. */ success: boolean; /** Total test cases. */ total: number; /** Number of passed test cases. */ passed: number; /** Number of failed test cases. */ failed: number; /** Per-case results. */ cases: TestCaseResult[]; /** Total duration in milliseconds. */ durationMs: number; } /** * Run a full test suite from tests.json against a provider. * * Dispatches each test case by its `type`: * - "urlPattern" — offline URL pattern matching (no HTTP) * - "validation" — calls provider.validate(), checks companyId * - "jobId" — calls provider.validate() on a job detail URL, checks jobId * - "scrape" — full live scrape with listing/detail assertions */ declare function runTestSuite(config: TestSuiteConfig): Promise; export { type AssertionResult, type JobIdTestCase, type ListingUrlTestCase, type ProviderTestCase, type ProviderTestSuite, type QualityScore, type SavedTestResult, type ScrapeTestCase, type SerializedJobDetails, type SerializedJobListing, type TestCaseResult, type TestConfig, type TestIssue, type TestResult, type TestSuiteConfig, type TestSuiteResult, type UrlPatternTestCase, type ValidationTestCase, computeQualityScore, evaluateJobIdAssertions, evaluateListingUrlAssertions, evaluateScrapeAssertions, evaluateUrlPatternAssertions, evaluateValidationAssertions, generateSingleProviderReport, runTest, runTestSuite, serializeTestResult };