import React from 'react'; import { RouteComponentProps, Link } from 'react-router-dom'; import Typography from 'antd/lib/typography'; import notification from 'antd/lib/notification'; import { Project as ProjectType, Browser } from 'test-crawler-core'; import Spin from 'antd/lib/spin'; import Icon from 'antd/lib/icon'; import Checkbox, { CheckboxChangeEvent } from 'antd/lib/checkbox'; import { History } from 'history'; import Button from 'antd/lib/button'; import List from 'antd/lib/list'; import message from 'antd/lib/message'; import { Job } from '../server/typing'; import { saveProject, startCrawler, getJobs } from '../server/service'; import { getViewportName } from '../viewport'; import { getResultsRoute, getPinsRoute } from '../routes'; import { useProject } from './useProject'; import { useCrawlers } from './useCrawlers'; import { timestampToString } from '../utils'; import { Codes } from '../code/Codes'; import { StorageType } from '../server/storage.typing'; import { ProjectJobs } from './ProjectJobs'; import { useAsync } from '../hook/useAsync'; import { useThisDoc } from '../doc/useDoc'; import { useGitHub } from '../auth/useGitHub'; const onStart = ( history: History, projectId: string, storageType: StorageType, loadJobs: () => Promise, browser: Browser, ) => async () => { try { const hide = message.loading('Starting crawlers', 0); const { timestamp, redirect } = await startCrawler( storageType, projectId, browser, ); if (redirect) { setTimeout(loadJobs, 5000); setTimeout(loadJobs, 10000); notification.open({ duration: 10, message: 'Test-crawler started', description: 'Test-crawler is running the crawlers on a remote container. To see live progress click open:', btn: ( Open ), }); } else { history.push(getResultsRoute(storageType, projectId, timestamp)); } hide(); } catch (error) { notification['error']({ message: 'Something went wrong!', description: error.toString(), }); } }; const onAutoPinChange = ( storageType: StorageType, { name, id, crawlerInput }: ProjectType, setProject: (response: ProjectType) => Promise, ) => async ({ target: { checked } }: CheckboxChangeEvent) => { const project = await saveProject( storageType, { ...crawlerInput, autopin: checked }, name, id, ); setProject(project); }; const getCrawlerStatusIcon = ( diffZoneCount: number, errorCount: number, status: string, inQueue: number, ) => { if (inQueue > 0) { return 'loading'; } if (!diffZoneCount && errorCount === 0) { return 'check'; } if (status === 'done') { return 'issues-close'; } return 'exclamation-circle'; }; export const Project = ({ match: { params: { projectId, storageType }, }, history, }: RouteComponentProps<{ projectId: string; storageType: StorageType }>) => { useGitHub(storageType); useThisDoc(Doc); const { project, setProject } = useProject(storageType, projectId); const { crawlers, loading, loadCrawlers } = useCrawlers( storageType, projectId, ); const { result: jobs, call: loadJobs } = useAsync(() => getJobs(storageType, projectId), ); const browser = project?.crawlerInput?.browser || Browser.ChromePuppeteer; return ( <> Project {!project ? ( ) : ( <>

Name: {project.name}

ID: {projectId}

URL: {project.crawlerInput.url}

Browser: {browser}

Screen:{' '} {getViewportName(project.crawlerInput.viewport)}

Method: {project.crawlerInput.method === 'urls' ? ( <> {' '} URLs list ) : ( <> {' '} Spider bot {!!project.crawlerInput.limit && ( {' '} (Limit: {project.crawlerInput.limit}) )} )}

Automatically pin new page founds.

{' '}  

( Open , ]} > {timestampToString(timestamp)} } description={ <> {' '} Diff: {diffZoneCount} - Error:{' '} {errorCount} - In queue:{' '} {inQueue} } /> )} /> )}
); }; const Doc = () => ( <>

This page give you an overview of a selected project. From there you can see all the crawlers that has been started and their results. To run a crawler click on{' '}

After clicking on the run button, you will be either redirected to the result page or a job will appear above the list of crawler, this will depends of the crawling storage you will use.

Codes

Under the list of crawlers, is a codes section. This will let you write your own code to interact with the crawler when the job is running. There is different phases where you can interact with the crawler: before, during and after crawling.

Before all

This script will run when the test-crawler is starting, to give you the possibility to setup a working environment. This can be useful if you need to start a server to run your test against it.

For each page

This give you the possibility to inject some code in the crawler while parsing the page. This code will be executed just after the page finish loaded, before to make the screenshot and before extracting the links. This can be really useful to manipulate the page before making the screenshot. For example, if you have dynamic element in your page, you can simply remove it. You could open some hidden element from an accordion. Run some e2e assertion with Jest. There is so much possibility with this feature...

After all

This script will run when the test-crawler finish. This can be useful to send the result to an API or in an email. In the following example, we will show you how to send result in slack:

            
                {`
// Need to install @slack/web-api where the crawler is running.
// with local storage just do yarn add @slack/web-api
// with remote storage like GitHub you will need to customize the CI job
const { WebClient } = require('@slack/web-api');

const token = 'api_slack_token';
// Given some known conversation ID (representing a public channel, private channel, DM or group DM)
const conversationId = '...';

module.exports = async function run(totalDiffCount, totalErrorCount) {
    const web = new WebClient(token);
    const result = await web.chat.postMessage({
        text: \`Hi, crawler finish his job. We found \${totalDiffCount} diff(s) and \${totalErrorCount} error(s).\`,
        channel: conversationId,
    });
}
            `}
            
        
Schedule

Right now, test-crawler doesn't offer scheduling out of the box but you can easily setup your own. On your server you can use build-in feature, like cronjob on Linux. For that, just use{' '} test-crawler-core --project the_id_of_your_project .

On GitHub, you can reuse the workflow generated by test-crawler in{' '} .github/workflows. Make a copy of the workflow under a different name and define the event to trigger the run, see{' '} documentation . You need then to specify the project you want to run.

            
                {`
name: Test-crawler schedule

on:
  schedule:
    # everyday at 01.00
    - cron:  '* 1 * * *'

jobs:
  test-crawler:
    runs-on: macos-latest

    steps:
    - uses: actions/checkout@v2
    - name: Run test-crawler
      uses: apiel/test-crawler/actions/run@master
      with:
        projectId: put_here_id_of_the_project
    - name: Push changes
      uses: apiel/test-crawler/actions/push@master
      with:
        token: \${{ secrets.GITHUB_TOKEN }}

            `}
            
        
);