import React from 'react'; import Input from 'antd/lib/input'; import InputNumber from 'antd/lib/input-number'; import Checkbox from 'antd/lib/checkbox'; import Form, { FormComponentProps } from 'antd/lib/form'; import Icon from 'antd/lib/icon'; import Popover from 'antd/lib/popover'; import Radio from 'antd/lib/radio'; import Typography from 'antd/lib/typography'; import notification from 'antd/lib/notification'; import Button from 'antd/lib/button'; import { RouteComponentProps } from 'react-router'; import { CrawlerInput, Browser } from 'test-crawler-core'; import { getHomeRoute } from '../../routes'; import { saveProject, getBrowsers } from '../../server/service'; import { Info } from '../../common/Info'; import { getDefaultViewport, viewportsStr } from '../../viewport'; import { History } from 'history'; import { StorageType } from '../../server/storage.typing'; import { ProjectRepos } from '../ProjectRepos'; import { useThisDoc } from '../../doc/useDoc'; import Select from 'antd/lib/select'; import { useAsync } from '../../hook/useAsync'; const save = async ( storageType: StorageType, history: History, { name, viewport, ...input }: CrawlerInput & { name: string; viewport: string }, ) => { try { await saveProject( storageType, { ...input, viewport: JSON.parse(viewport) }, name, undefined, ); history.push(getHomeRoute()); } catch (error) { notification['error']({ message: 'Something went wrong!', description: error.toString(), }); } }; const handleSubmit = ( storageType: StorageType, history: History, validateFields: any, ) => (event: React.FormEvent) => { event.preventDefault(); validateFields((err: any, values: any) => { if (!err) { save(storageType, history, values); } }); }; type Props = FormComponentProps & RouteComponentProps<{ storageType: StorageType }>; const NewProject = ({ history, match: { params: { storageType }, }, form: { getFieldDecorator, validateFields, getFieldValue }, }: Props) => { const { result: browsers } = useAsync(() => getBrowsers(storageType), ); useThisDoc(Doc); return (
{getFieldDecorator('name', { rules: [ { required: true, message: 'Please give a name to the project.', }, ], })()} {getFieldDecorator('url', { rules: [ { required: true, message: 'Please input an URL to crawl.', }, ], })( , )}
{getFieldDecorator('browser', { initialValue: Browser.ChromePuppeteer, rules: [ { required: true, message: 'Please select a browser.', }, ], })( , )} {getFieldDecorator('viewport', { initialValue: JSON.stringify(getDefaultViewport()), rules: [ { required: true, message: 'Please select viewport.', }, ], })( , )}
{getFieldDecorator('method', { initialValue: 'spiderbot', rules: [{ required: true }], })( Spider bot URLs list , )} {getFieldValue('method') === 'spiderbot' && ( {getFieldDecorator('limit')( , )}   Limit the number of sibling pages. For example, with the urls:
  • /item/1
  • /item/2
  • /item/3
  • /item/4
{' '} using the limit 2 will only crawl{' '} /item/1 and /item/2.

Use 0 to skip the limit.
} trigger="click" overlayStyle={{ width: 200 }} > )} Spider bot crawling method will get all the links inside the page of the given URL and crawl the children. It will then continue do the same with the children till no new link is found. Be careful if you have big website, this is most likely not the right solution for you. URLs list crawling method will crawl a specific sets of URLs. In the URL input field you must provide an endpoint containing a list of URLs (a simple text format, with one URL per line). The crawler will crawl each of those URL only and will not try to find links in the page. To use a static list of URLs, you can use a tool like{' '} https://pastebin.com . {getFieldDecorator('autopin', { valuePropName: 'checked', })(Automatically pin new page founds.)} ); }; const NewForm = Form.create({ name: 'start_crawler' })(NewProject); export default NewForm; const Doc = () => ( <>

First of all, you need to give a name to your project. Then, you need to provide the URL depending of the crawling method you will be using. If you are using the Spider bot method, you should give the URL of the website you want to crawl. If you are using the{' '} URLs list method, you should give the URL of the endpoint containing the list of URLs.

There is different kind of browser:{' '} chrome-puppeteer ,{' '} firefox-selenium ... Depending of the browser you will select, different tool will be available in the code injection. Also some browser are only available depending of the OS where is test-crawler hosted.

IE and Safari are in experimentation, might not be stable.

Running your test in safari work only on macOS. Safari doesn't support multiple instance in parallel, therefor crawling might be very slow. Locally, you will need to activate safari webdriver for selenium with the following commands:

            sudo safaridriver --enable safaridriver -p 0 &
        

Internet explorer, is the browser giving the most problem till now. Setting viewport has no effect, fullpage screenshot doesn't work, Spider Bot crawling will most likely fail.

There is multiple viewports (screen size) available. If you want to test multiple viewports for the same website, you will have to create one project per viewport.

Finally, you can specify if you want to automatically pin the new pages founds. Pins are the references screenshot to make the comparison with. While crawling, the crawler is comparing page to pin. If you are not sure, the auto-pin can be activate/deactivate afterwards.

Spider bot

Spider bot crawling method will get all the links inside the page of the given URL and crawl the children. It will then continue do the same with the children till no new link is found. Be careful if you have big website, this is most likely not the right solution for you.

URLs list

URLs list crawling method will crawl a specific sets of URLs. In the URL input field you must provide an endpoint containing a list of URLs (a simple text format, with one URL per line). The crawler will crawl each of those URL only and will not try to find links in the page. To use a static list of URLs, you can use a tool like{' '} https://pastebin.com .

);