import { join, extname } from 'path'; import * as md5 from 'md5'; import { groupOverlappingZone } from 'pixdiff-zone'; import { WsContext, Context } from 'isomor-server'; import { generateTinestampFolder, CRAWL_FOLDER, PIN_FOLDER, CODE_FOLDER, PROJECT_FOLDER, SNAPSHOT_FOLDER, Crawler, CrawlerInput, PageData, Project, CodeInfoList, Browser, ZoneStatus, } from 'test-crawler-core'; import {} from './config'; import { Code, StartCrawler, BeforeAfterType } from '../typing'; import { StorageType } from '../storage.typing'; import { CrawlerProviderStorage } from './CrawlerProviderStorage'; export class CrawlerProvider extends CrawlerProviderStorage { constructor( storageType?: StorageType, public ctx?: undefined | WsContext | Context, ) { super(storageType, ctx); } repos() { return this.storage.repos(); } repo() { return this.storage.getRepo(); } info() { return this.storage.info(); } jobs(projectId: string) { return this.storage.jobs(projectId); } loadProject(projectId: string): Promise { return this.storage.readJSON(this.join(projectId, `project.json`)); } async loadProjects(): Promise { // we should use accumulator const projects = await this.storage.readdir(PROJECT_FOLDER); return Promise.all( projects.map(projectId => this.loadProject(projectId)), ); } async saveProject( crawlerInput: CrawlerInput, name: string, projectId?: string, ): Promise { if (!projectId) { projectId = (md5 as any)(name) as string; } const project = { id: projectId, name, crawlerInput }; await this.storage.saveJSON( this.join(projectId, 'project.json'), project, ); return project; } getCrawler(projectId: string, timestamp: string): Promise { const path = this.join(projectId, CRAWL_FOLDER, timestamp, '_.json'); return this.storage.readJSON(path); } async getAllCrawlers(projectId: string): Promise { const path = this.join(projectId, CRAWL_FOLDER); const folders = await this.storage.readdir(path); const crawlers: Crawler[] = await Promise.all( folders.map(timestamp => this.getCrawler(projectId, timestamp)), ); return crawlers; } async copyToPins( projectId: string, timestamp: string, id: string, ): Promise { const jsonFile = this.join( projectId, CRAWL_FOLDER, timestamp, `${id}.json`, ); // set diff to 0 // instead to load this file again, we could get the data from the frontend? const data: PageData = await this.storage.readJSON(jsonFile); if (data?.png) { data.png.diff = { pixelDiffRatio: 0, zones: [], }; if (data.png.diff.pixelDiffRatio > 0) { await this.storage.saveJSON(jsonFile, data); } } await this.storage.saveJSON( this.join(projectId, PIN_FOLDER, `${id}.json`), data, ); return data; } async removeFromPins(projectId: string, id: string): Promise { await this.storage.remove( this.join(projectId, PIN_FOLDER, `${id}.json`), ); return this.getPins(projectId); } async image(projectId: string, timestamp: string, id: string) { if (timestamp === 'pin') { const pin: PageData = await this.storage.readJSON( this.join(projectId, PIN_FOLDER, `${id}.json`), ); if (!pin) return; timestamp = pin.timestamp; } return this.storage.image( this.join(projectId, SNAPSHOT_FOLDER, `${timestamp}-${id}.png`), ); } saveBeforeAfterCode( projectId: string, type: BeforeAfterType, code: string, ): Promise { if (!Object.values(BeforeAfterType).includes(type)) { throw new Error(`Unknown code type ${type}.`); } const file = this.join(projectId, `${type}.js`); if (!code.length) { return this.storage.remove(file); } return this.storage.saveFile(file, code); } async getBeforeAfterCode( projectId: string, type: BeforeAfterType, ): Promise { if (!Object.values(BeforeAfterType).includes(type)) { throw new Error(`Unknown code type ${type}.`); } try { const buf = await this.storage.read( this.join(projectId, `${type}.js`), ); return buf?.toString() || ''; } catch (err) {} return ''; } async saveCode(projectId: string, code: Code): Promise { const { source, ...codeInfo } = code; const list = await this.getCodeList(projectId); list[code.id] = codeInfo; await this.storage.saveJSON( this.join(projectId, CODE_FOLDER, `list.json`), { ...list }, ); // for some reason it need a copy await this.storage.saveFile( this.join(projectId, CODE_FOLDER, `${code.id}.js`), source, ); } async loadCode(projectId: string, id: string): Promise { const list = await this.getCodeList(projectId); const codeInfo = list[id]; const sourcePath = this.join(projectId, CODE_FOLDER, `${id}.js`); if (codeInfo) { const buffer = await this.storage.read(sourcePath); if (buffer) { const source = buffer.toString(); return { ...codeInfo, source }; } } return { id, name: '', pattern: '', source: '', }; } async getCodeList(projectId: string): Promise { const listPath = this.join(projectId, CODE_FOLDER, `list.json`); const list = await this.storage.readJSON(listPath); return list || {}; } getPins(projectId: string): Promise { return this.getPinsInFolder(this.join(projectId, PIN_FOLDER)); } getPin(projectId: string, id: string): Promise { return this.getPageInFolder(this.join(projectId, PIN_FOLDER), id); } getPages(projectId: string, timestamp: string): Promise { return this.getPinsInFolder( this.join(projectId, CRAWL_FOLDER, timestamp), ); } private getPageInFolder(folder: string, id: string): Promise { return this.storage.readJSON(join(folder, `${id}.json`)); } private async getPinsInFolder(folder: string): Promise { const files = await this.storage.readdir(folder); return Promise.all( files .filter(file => extname(file) === '.json' && file !== '_.json') .map(file => this.storage.readJSON(join(folder, file))), ); } async setCrawlerStatus( projectId: string, timestamp: string, status: string, ): Promise { const file = this.join(projectId, CRAWL_FOLDER, timestamp, '_.json'); const crawler: Crawler = await this.storage.readJSON(file); crawler.status = status; await this.storage.saveJSON(file, crawler); return crawler; } async setZoneStatus( projectId: string, timestamp: string, id: string, status: ZoneStatus, index?: number, ): Promise { const folder = this.join(projectId, CRAWL_FOLDER, timestamp); const fileJson = join(folder, `${id}.json`); const data: PageData = await this.storage.readJSON(fileJson); if (index && status === ZoneStatus.zonePin) { const pinJsonFile = this.join(projectId, PIN_FOLDER, `${id}.json`); const pin: PageData = await this.storage.readJSON(pinJsonFile); if (pin?.png?.diff?.zones && data?.png?.diff?.zones) { if (index) { pin.png.diff.zones.push({ ...data.png.diff.zones[index], status, }); } const zones = pin.png.diff.zones.map(item => item.zone); zones.sort((a, b) => a.xMin * a.yMin - b.xMin * b.yMin); const groupedZones = groupOverlappingZone(zones); pin.png.diff.zones = groupedZones.map(zone => ({ zone, status, })); } await this.storage.saveJSON(pinJsonFile, pin); } if (data?.png?.diff?.zones) { if (index) { data.png.diff.zones[index].status = status; } else { data.png.diff.zones.forEach(zone => (zone.status = status)); } } await this.storage.saveJSON(fileJson, data); return this.getPages(projectId, timestamp); } async startCrawler( projectId: string, browser?: Browser, ): Promise { const timestamp = generateTinestampFolder(); const crawlTarget = { projectId, timestamp }; const redirect = await this.storage.crawl( crawlTarget, (this.ctx as any)?.push, browser, ); return { timestamp, redirect, }; } get browsers(): Browser[] { return this.storage.browsers; } protected join(projectId: string, ...path: string[]) { return join(PROJECT_FOLDER, projectId, ...path); } }