import { Auth, gmail_v1 } from "googleapis"; import debug from "debug"; import { waitUntil } from "../utils/waitUntil"; import { Email } from "../Email"; import { callApi } from "../utils/callApi"; //"Zygon Alerts " => "alerts@notifications.zygon.tech" const extractEmailAddress = (str: string) => str.split("<").pop()!.split(">")[0]; /** * Connector to the gmail API. * * Features * - Auth * - High performance email retrieval (~200 emails/s) * */ type MessageMeta = { id: string }; const BATCH_SIZE = 200; const downloadMessagesInBatch = (auth: Auth.AuthClient) => async (metas: MessageMeta[]): Promise => { const options: RequestInit = { method: "POST", headers: { "Content-Type": `multipart/mixed; boundary=${BOUNDARY}`, "Accept-Encoding": `gzip`, Authorization: `Bearer ${(await auth.getAccessToken()).token}`, }, body: "", }; options.body += `--${BOUNDARY}`; for (const meta of metas) { options.body += `\nContent-Type: application/http\n\n`; options.body += `GET /gmail/v1/users/me/messages/${meta.id}?format=metadata&fields=id,payload(headers),internalDate,labelIds\n\n`; options.body += `--${BOUNDARY}`; } options.body += "--"; const res = await callApi("https://gmail.googleapis.com/batch", options); const responseText = (await res.text()).substring(2); // remove the empty first line const responseBoundary = responseText.split("\n")[0]!; // retrieve the boundary let elements = responseText.split(responseBoundary); if (!elements) throw new Error("no response in batch"); elements = elements.slice(1); // remove the first boundary if (!elements) throw new Error("no first boundary"); elements = elements.slice(0, -1); // remove the stop character at the end if (!elements) throw new Error("no last char"); const responses = elements.map((j: string) => JSON.parse(j.split("\n").slice(9).join("\n")), ); for (const e of responses) { // https://developers.google.com/gmail/api/guides/handle-errors#resolve_a_429_error_too_many_requests if (e.error) throw e.error; } return responses.map((e: gmail_v1.Schema$Message) => toEmail(e)); }; type Awaitable = T | PromiseLike; const PAGE_SIZE = 10; export async function* fetchEmailsFromGmail( gmail: gmail_v1.Gmail, since: Date = new Date(0), ): AsyncGenerator { let hasMore = true; let pageToken: string | undefined = undefined; while (hasMore) { const startTime = Date.now(); // used for rate limiting our downloads const params: gmail_v1.Params$Resource$Users$Messages$List = { userId: "me", maxResults: PAGE_SIZE, pageToken: pageToken, }; const list = await gmail.users.messages.list(params); if (list?.data.nextPageToken && list.data.messages) { pageToken = list!.data.nextPageToken; } else { hasMore = false; break; } const emails: Email[] = await downloadMessagesInBatch( gmail!.context._options.auth as unknown as Auth.AuthClient, )(list.data.messages as MessageMeta[]); for (const email of emails) { yield email; } hasMore = emails.every((email) => new Date(Number(email.date)) > since); // quota management await waitUntil(startTime, 300); } } /** * Download all emails from a mailbox. * Returns the most recent emails from the batch. */ export async function downloadEmailsGoogle( gmail: gmail_v1.Gmail, since: Date, processBatch: (e: Email[]) => Awaitable, ): Promise<{ mostRecent: number; emailCount: number }> { let mostRecent = 0; const start = Date.now(); // Per user rate limit 250 quota units per user per second, // moving average (allows short bursts). // a message download is 5 units. let result: Email[] = []; let hasMore = true; let pageToken = ""; const PAGE_SIZE = 10; debug("triage")(`Start loading google emails`); let emailCount = 0; while (hasMore) { const startTime = Date.now(); // used for rate limiting our downloads const params: gmail_v1.Params$Resource$Users$Messages$List = { userId: "me", maxResults: PAGE_SIZE, }; debug("triage")(`getting ${PAGE_SIZE} emails metadata...`); if (pageToken !== "") params.pageToken = pageToken; const list = await gmail.users.messages.list(params); if (list?.data.nextPageToken && list.data.messages) { pageToken = list!.data.nextPageToken; } else { hasMore = false; debug("triage")( `break because of ${!!list?.data.nextPageToken} ${!!list.data ?.messages}`, ); break; } const page: Email[] = await downloadMessagesInBatch( gmail!.context._options.auth as unknown as Auth.AuthClient, )(list.data.messages as MessageMeta[]); emailCount += page.length; const afterSinceEmails = page.filter( (p) => new Date(Number(p.date)) > since, ); if (afterSinceEmails.length === 0) { debug("triage")( `hasMore = false because of afterSinceEmails.length ${afterSinceEmails.length}`, ); hasMore = false; } result.push(...afterSinceEmails); if (result.length > BATCH_SIZE) { const mostRecentBatch = await processBatch(result); mostRecent = Math.max(mostRecent, mostRecentBatch); result = []; } // quota management await waitUntil(startTime, 300); } const mostRecentBatch = await processBatch(result); mostRecent = Math.max(mostRecent, mostRecentBatch); logEmailRunTime(start, result); return { mostRecent, emailCount }; } const BOUNDARY = "dontstopmenow"; const inHeader = (email: gmail_v1.Schema$Message, field: string) => email.payload?.headers ?.find((h) => h.name?.toLocaleLowerCase() === field) ?.value?.toLocaleLowerCase() ?? ""; function toEmail(email: gmail_v1.Schema$Message): Email { let id = email.id!; if (isFwd(email)) id = ""; // we ignore fwd emails return { id, subject: inHeader(email, "subject"), from: extractEmailAddress(inHeader(email, "from")), deliveredTo: extractEmailAddress(inHeader(email, "delivered-to")), replyTo: extractEmailAddress(inHeader(email, "reply-to")), to: inHeader(email, "to") .split(",") .map((e) => extractEmailAddress(e)), cc: inHeader(email, "cc") .split(",") .map((e) => extractEmailAddress(e)), bcc: inHeader(email, "bcc") .split(",") .map((e) => extractEmailAddress(e)), group: inHeader(email, "list-id"), date: email.internalDate!, body: email.snippet!, labelIds: email.labelIds, } as Email; } const isFwd = (email: gmail_v1.Schema$Message) => inHeader(email, "X-Gm-Original-To") || // X-Gm... -> google headers inHeader(email, "X-Gm-Spam") || inHeader(email, "X-GM-Phishy") || /tr:|fwd:/.test(inHeader(email, "subject")); function logEmailRunTime(start: number, result: Email[]) { const end = Date.now(); const time = Math.floor((end - start) / 1000); debug("triage")( `elapsed time: ${time.toLocaleString("en-EN", { maximumFractionDigits: 2, })}s, speed:${ time !== 0 ? (result.length / time).toLocaleString("en-EN", { maximumFractionDigits: 2, }) : 0 } emails/s`, ); }