import fsex from "fs-extra" import { sleep } from "../../../async" import nodeFetch from "node-fetch" // 下载单个文件 // 失败后重试 export async function downloadToFile(url: string, filePath?: string, options?: { retry?: number }) { let retry = options?.retry ?? 3 let response!: Response let buffer!: ArrayBuffer let isGzipError = false while (retry >= 0) { try { response = await fetch(url) if (!response.ok) { throw new Error(`Failed to download url: ${url} \n ${response.statusText}`) } buffer = await response.arrayBuffer() break } catch (e) { if (e.message === "terminated" && e.cause?.code == "Z_DATA_ERROR") { console.error(`donwload file gzip error: Z_DATA_ERROR, try download raw data.`) isGzipError = true buffer = await fetchRawData(url) break } retry-- if (retry >= 0) { console.warn(`Failed to url: ${url}, ${e.message}, retrying... (${retry} times left)`) await sleep(2000) continue } else { console.error("Failed to download url: ", url) throw e } } } try { const size = buffer.byteLength if (filePath) await fsex.outputFile(filePath, Buffer.from(buffer)) return { size, buffer, isGzipError, headers: { "content-type": response.headers.get("content-type"), "content-encoding": response.headers.get("content-encoding"), }, } } catch (e) { console.error(`Failed to download url: ${url} to path:${filePath}`) throw e } } async function fetchRawData(url: string) { let res = await nodeFetch(url, { compress: false }) let buffer = await res.arrayBuffer() return buffer }