import * as MicrosoftGraph from "@microsoft/microsoft-graph-types"; import { Email } from "../Email"; import debug from "debug"; import { waitUntil } from "../utils/waitUntil"; import { callApi } from "../utils/callApi"; type MsMessage = Pick< MicrosoftGraph.Message, | "id" | "from" | "subject" | "replyTo" | "toRecipients" | "ccRecipients" | "bccRecipients" | "sentDateTime" >; export class MsError extends Error { response: Response; error: Error | null; constructor(response: Response, error?: Error) { super( `MS API call failed with status ${response.status}: ${response.statusText}`, ); this.response = response; this.error = error ?? null; } } export async function downloadEmailsMs( msId: string, since: Date, accessToken: string, processBatch: (e: Email[]) => Promise, ): Promise<{ mostRecent: number; emailCount: number }> { let batchSize = 1000; const fields: (keyof MsMessage)[] = [ "id", "from", "subject", "replyTo", "toRecipients", "ccRecipients", "bccRecipients", "sentDateTime", ]; const getOptions = `$top=${batchSize}&$select=${fields.join(",")}&$filter=sentDateTime ge ${ since.toISOString().split("T")[0] }&$skip=0`; let url = `https://graph.microsoft.com/v1.0/users/${msId}/messages?${getOptions}`; let hasMore = true; let emailCount = 0; let mostRecent = 0; let backoff = 0; while (hasMore) { await waitUntil(Date.now(), backoff); const res = await msFetch(url, accessToken); if ("status" in res) { if (res.status === 404) { debug(`crawl-email-ms`)( `${msId} - The mailbox is either inactive, soft-deleted, or is hosted on-premise.`, ); break; } if (res.error === "ErrorMailboxMoveInProgress") { debug(`crawl-email-ms`)(`${msId} - [${res.error}] ${res.message}`); break; } // We received a timeout error OR service unvailable from Microsoft // We decrease email batch size and increase backoff to get through the error if (batchSize > 16) { debug(`crawl-email-ms`)( `BATCH TOO BIG => downgrading $top to '${Math.floor(batchSize / 2)}'`, ); // We divide our email crawling batch size by 2 url = url.replace( `top=${batchSize}`, `top=${Math.floor(batchSize / 2)}`, ); batchSize = Math.floor(batchSize / 2); // We increase backoff time backoff = backoff ? backoff * 2 : 1000; } else { // If we already downgraded batch size below 16 results, // We continue by skipping the last 15 results const searchParams = new URLSearchParams(url.split("?")[1]); const skipping = searchParams.get("$skip"); if (skipping) { debug(`crawl-email-ms`).log( `BATCH TOO SMALL => continuing by skipping '${batchSize}' emails`, ); url = url.replace(`skip=${skipping}`, `skip=${skipping + batchSize}`); } if (Number(skipping) === 0) throw new Error("Unhandled MS error..."); if (!skipping) throw new Error("You need to provide an initial skip"); } // We relaunch last email crawling with modified params continue; } else { // If no timeout error, we set back batch size to 1000 if (res["@odata.nextLink"]) { url = res["@odata.nextLink"].replace(`top=${batchSize}`, `top=1000`); batchSize = 1000; backoff = 0; } else hasMore = false; } if (!res || !res.value) { break; } const toString = (r?: MicrosoftGraph.Recipient | null) => (r && r.emailAddress && r.emailAddress.address ? r.emailAddress.address : "" ).toLocaleLowerCase(); const emails: Email[] = res.value.map((e) => ({ id: e.id ?? "", from: toString(e.from), subject: e.subject ?? "", deliveredTo: "", // @todo replyTo: e?.replyTo && e.replyTo.length > 0 ? toString(e.replyTo[0]!) : "", to: (e.toRecipients ?? []).map(toString), cc: (e.ccRecipients ?? []).map(toString), bcc: (e.bccRecipients ?? []).map(toString), group: "", date: "" + (e.sentDateTime ? new Date(e.sentDateTime).getTime() : new Date().getTime()), labelIds: [], })); emailCount += emails.length; mostRecent = Math.max( mostRecent, ...emails.map((e) => new Date(Number(e.date)).getTime()), ); await processBatch(emails); } return { mostRecent, emailCount }; } type managedResponseStatus = 504 | 503 | 404; type MsResult = | { value: Result; "@odata.nextLink"?: string } | { status: managedResponseStatus; error?: string; message?: string }; async function msFetch( endpoint: string, accessToken: string, ): Promise> { const response = await callApi(endpoint, { method: "get", credentials: "include", headers: { Authorization: `Bearer ${accessToken}`, }, }); try { if ([404, 504, 503].includes(response.status)) { // eslint-disable-next-line @typescript-eslint/no-explicit-any const r = (await response.json()) as any; return { status: response.status as managedResponseStatus, error: r.error.code, message: r.error.message, }; } if (response.ok) return (await response.json()) as MsResult; throw new MsError(response); // eslint-disable-next-line @typescript-eslint/no-explicit-any } catch (error: any) { throw new MsError(response, error); } }