import fs from "fs"; import path from "path"; const RELOAD_THROTTLE = 5 * 1000; export class SourcesList { constructor(private filePath: string) { } private urls: string[] = []; private indexes = new Map(); private endsClean = true; private lastReload = 0; private appendQueue = Promise.resolve(); private async load(): Promise { let content = ""; try { content = await fs.promises.readFile(this.filePath, "utf8"); } catch (e) { if ((e as { code?: string }).code !== "ENOENT") throw e; } this.endsClean = !content || content.endsWith("\n"); let lines = content.split("\n"); if (lines[lines.length - 1] === "") { lines.pop(); } this.urls = lines; this.indexes.clear(); for (let i = 0; i < lines.length; i++) { if (!this.indexes.has(lines[i])) { this.indexes.set(lines[i], i); } } } public getUrl(sourcesListIndex: number): string | undefined { return this.urls[sourcesListIndex]; } public async getUrlReloading(sourcesListIndex: number): Promise { if (sourcesListIndex < this.urls.length) return this.urls[sourcesListIndex]; if (Date.now() - this.lastReload < RELOAD_THROTTLE) return undefined; this.lastReload = Date.now(); await this.load(); return this.urls[sourcesListIndex]; } public ensure(url: string): Promise { if (url.includes("\n")) { throw new Error(`Source URLs cannot contain newlines (they are stored one per line): ${JSON.stringify(url)}`); } let result = this.appendQueue.then(async () => { await this.load(); let existing = this.indexes.get(url); if (existing !== undefined) return existing; // appendFile creates the file but not its parent directories, and on a fresh bucket we run before anything else has created them await fs.promises.mkdir(path.dirname(this.filePath), { recursive: true }); let prefix = this.endsClean && "" || "\n"; await fs.promises.appendFile(this.filePath, prefix + url + "\n"); this.endsClean = true; let index = this.urls.length; this.urls.push(url); this.indexes.set(url, index); console.log(`Registered new source ${JSON.stringify(url)} as sourcesListIndex ${index} in ${this.filePath}`); return index; }); this.appendQueue = result.then(() => { }, () => { }); return result; } }