#!/usr/bin/env node

import * as process from 'process';
import * as url from 'url';
import fsp from 'fs/promises';
import fs from 'fs';
import path from 'path';

import { CheerioCrawler, ProxyConfiguration, CriticalError } from 'crawlee';


function printHelpMessage(): void {
    console.log(`
    Usage: npx yp-extractor <urls-file-path> [save-file-path] [proxy-file-path] [reqs-per-min]

    Required: urls-file-path - the path to a file where urls are on separate lines. Can be just 
     the file name and extension if in the same folder as this.
    
    Can ignore this if speed is adequate:
    ---------------------------------------------
    Optional: save-file-path (needs to be specified if using proxies)
    Optional: proxy-file-path (also needs to be specified if using proxies): A file where 
       proxies are on separate lines in the format: http://username:password@ip:port
    Optional: reqs-per-min (also needs to be specified if using proxies): Number of requests 
        per minute. A higher number will get you blocked faster (unless using a few proxies). Default is 40. Suggest 40 x the number of proxies in proxy-file-path.
    
    For more info: https://www.npmjs.com/package/yp-extractor
    `);
}

interface Listing {
    name: string;
    category?: string;
    address?: string;
    gmapsLink?: string;
    telephone?: string;
    email?: string;
    website?: string;
    extractionUrl?: string;
}

type AllListingKeys = {
    [K in keyof Listing]: K
  } extends { [P in keyof Listing]: infer U }
    ? U
    : never;

async function toCsvString(listing: Listing): Promise<string> {
    const values = Object.values(listing);
    return values.map(value => `"${value ?? ''}"`).join(',');
}

// async function fileExists(filePath: string): Promise<boolean> {
//     try {
//         await fsp.stat(filePath);
//         return true;
//     } catch (error: any) {
//         if (error.code === 'ENOENT') {
//             return false;
//         } else {
//             try {
//                 await fsp.access(filePath, fsp.constants.R_OK);
//                 return true;
//             } catch {
//                 return false;
//             }
//         }
//     }
// }

type PropertyNames = Array<keyof Listing>;

async function createCsvIfNotExist(headers: PropertyNames, saveFilePath: string): Promise<void> {
    
    try {
        await fsp.access(saveFilePath, fsp.constants.R_OK | fsp.constants.W_OK);

        console.log(`Found ${saveFilePath}, it will be appended to. Warning: Do not open ${saveFilePath} while this is running. If it's open now, close it and restart this.`)

    } catch(error: any) {

        if (error.code === 'ENOENT') {
            await fsp.writeFile(saveFilePath, headers.join(',') + '\n');

            console.log(`Savefile did not exist, created a new one at ${path.resolve(saveFilePath)}`);

        } else {

            console.log(`There was an error accessing ${saveFilePath}. Is it open in another program?`);

            process.exit(1);
        }
    }
}

async function updateCsv(listing: Listing, saveFilePath: string): Promise<void> {

    const csvString = await toCsvString(listing);

    const existingData = await fsp.readFile(saveFilePath, 'utf-8');

    if(existingData.includes(csvString.trim())) {
        // console.log(`Already have ${listing.name} in ${saveFilePath}. Not saving.`);
        return; 
    }

    await fsp.appendFile(saveFilePath, csvString + "\n");
}

async function getNumberOfEntries(saveFilePath: string): Promise<number> {
    try {
        const existingData = await fsp.readFile(saveFilePath, 'utf-8');
        return existingData.split('\n').length - 1;
    } catch {
        console.log("Couldn't read save file.")
        return 0;
    }
}

const args = process.argv.slice(2);
if (args.includes('--help') || args.includes('-h') || args.length === 0) {
    printHelpMessage();
    process.exit(0);
}

await executeCommand(args);

async function executeCommand(args: string[]) {
    // get urls on each line in the txt file "urls.txt" (path is first argument)
    // if no second argument (path to save-file.csv) create a new save-file.csv.
    // run extract with urls array and save-file.csv path arguments.

    const urlsFilePath: string = args[0];
    const saveFilePath: string = args[1] ?? 'save-file.csv';
    const proxiesFilePath: string | undefined = args[2];
    const reqsPerMin: number | undefined = args[3] ? parseInt(args[3]) : undefined;

    let urls: string[] = [];
    let proxies: string[] = [];

    try {
        urls = fs.readFileSync(urlsFilePath, 'utf8').split('\n');
        urls = urls.filter(url => url.trim().length > 0);
    } catch (error) {
        console.log(`Error reading file: ${path.resolve(urlsFilePath)}. Does it exist?`);
        return;
    }

    if (urls.length === 0) {
        console.log(`No urls found in file: ${path.resolve(urlsFilePath)}. Put one url on each line.`);
        return;
    }

    if(proxiesFilePath) {
        try {
            proxies = fs.readFileSync(proxiesFilePath, 'utf8').split('\n');
        } catch (error) {
            console.log(`Error reading file: ${path.resolve(proxiesFilePath)}. Does it exist?`);
            return;
        }

        if (proxies.length === 0) {
            console.log(`No proxies found in file: ${path.resolve(proxiesFilePath)}. Put one proxy on each line in the format: http://username:password@ip:port.`);
            return;
        }
    }

    const propertyNames: PropertyNames = [
        "name",
        "category",
        "address",
        "gmapsLink",
        "telephone",
        "email",
        'website',
        "extractionUrl",
    ];

    await createCsvIfNotExist(propertyNames, saveFilePath);

    await extract(urls, saveFilePath, proxies, reqsPerMin ?? 40);
}


async function extract(urls: string[], saveFile: string, proxies: string[], reqsPerMin: number) {

    let totalListings = 0;
    const startTime = Date.now();
    const startingAmount = await getNumberOfEntries(saveFile);
    let listingsPerMinuteValues: number[] = [];


    const crawler = new CheerioCrawler({

        proxyConfiguration: proxies.length > 0 ? new ProxyConfiguration({proxyUrls: proxies}) : undefined,

        requestHandler: async ({ request, $, enqueueLinks }) => {

            if (request.loadedUrl?.includes("dataprotection")) {

                console.log("IP is blocked by yellow pages. Create a proxy.txt file, add proxies and try again.");
                throw new CriticalError("IP blocked by yellow pages")


            } else if(request.label === "listing") {

                // Can have more than one telephone 
                const telephone = $('a.contact-phone .desktop-display-value');
                const telephoneNumbers = telephone.length > 1 ? 
                    telephone.map((_, el) => $(el).text()).get().join(", "): 
                    telephone.text();

                const lat = $('meta[itemprop="latitude"]').attr('content')
                const long = $('meta[itemprop="longitude"]').attr('content')

                const listing: Listing = {
                    name: $('.listing-name').text(),
                    category: $('.listing-heading').text(),
                    address: $('.listing-address').text(),
                    gmapsLink: lat && long ? `https://www.google.com/maps/?q=${lat},${long}` : "",
                    telephone: telephoneNumbers,
                    email: ($('a.contact-email').attr('data-email') ?? '').replace('mailto:', ''),
                    website: $('a.contact-url').attr('href'),
                    extractionUrl: request.loadedUrl,
                };
            
                await updateCsv(listing, saveFile);

            } else {
                
                const listingLinks: string[] = [];
                const categoryLinks: string[] = [];
    
                $('a').each((_, element) => {
                    const href = $(element).attr('href');
                    const catLinkRegex = /-category-[a-z]\d$/;
                    if (href) {
                        if (href.endsWith('listing.html')) {
                            const absoluteUrl = url.resolve("https://www.yellowpages.com.au/", href);
                            listingLinks.push(absoluteUrl);
                        } else if (catLinkRegex.test(href)) {
                            const absoluteUrl = url.resolve("https://www.yellowpages.com.au/", href);
                            categoryLinks.push(absoluteUrl);
                        }
                    }
                });

                totalListings += listingLinks.length;


    
                await enqueueLinks({
                    label: "listing",
                    urls: listingLinks,
                });

                await enqueueLinks({
                    label: "category",
                    urls: categoryLinks,
                });
            }

        },

        // maxRequestsPerCrawl: 20,

        maxRequestsPerMinute: reqsPerMin,

        maxConcurrency: 5,

        statusMessageCallback: async (ctx) => {

            const totalGot = await getNumberOfEntries(saveFile);
            const gotThisSession = totalGot - startingAmount;

            const listingsPerMinute = gotThisSession / (Date.now() - startTime) * 1000 * 60;
            listingsPerMinuteValues.push(listingsPerMinute);

            if (listingsPerMinuteValues.length > 10) listingsPerMinuteValues.shift();

            const averageListingsPerMinute = (
                listingsPerMinuteValues.reduce((a, b) => a + b) / listingsPerMinuteValues.length
            );

            const remainingTimeMins = Math.round((totalListings - totalGot) / averageListingsPerMinute);

            const messageParts = [
                `${saveFile} currently has ${totalGot} of ${totalListings} listings`,
                `Approx. time remaining: ${remainingTimeMins < Infinity ? remainingTimeMins: "(calculating)"} minutes`,
                `Crawled ${ctx.state.requestsFinished} urls`,
                `Number of bad reqs: ${ctx.state.requestsFailed}`,
            ]

            const msg = messageParts.join(', ');
            
            return ctx.crawler.setStatusMessage(msg, { level: 'INFO' });
        }
    });

    await crawler.run(urls);
}