/** * Streaming ActivityReport builder for long-horizon Summary documents. * * Rows are consumed in `(projectKey, timestamp, id)` order. Only the current * gap-merge session for each project/contributor/unassigned label is retained; * raw events are never accumulated. This keeps Summary correctness independent * of any hosted row cap while preserving the billable clock used by invoices. */ import { UNASSIGNED_PROJECT_ID } from "../db/schema"; import { CHAT_PROVIDER_IDS, isFileMetadata, isGitMetadata, isLinearMetadata, type SourceCountMap, type TimelineEventDisplay, } from "../types/timeline"; import { type ActivityReport, type ActivityReportClientTypeBreakdown, type ActivityReportDevice, type ActivityReportOutcome, type ActivityReportProject, type ActivityReportTicket, type ActivityReportTicketsByProject, type ActivityReportUnassignedBreakdownRow, extractTicketFromSubject, type HighlightCandidate, highlightCandidate, isJunkHighlightLabel, selectHighlights, stripConventionalCommitPrefix, unassignedEventLabel, } from "./activity-report"; import { DEFAULT_GAP_THRESHOLD_MINUTES } from "./time-ranges"; const CHAT_SOURCES = new Set(CHAT_PROVIDER_IDS); const UNASSIGNED_BREAKDOWN_THRESHOLD = 0.15; const UNASSIGNED_BREAKDOWN_TOP_N = 6; const TICKETS_PER_PROJECT_CAP = 15; const OUTCOMES_PROJECTS_CAP = 6; const OUTCOMES_BULLETS_PER_PROJECT = 5; const HIGHLIGHT_RESERVOIR_MULTIPLIER = 8; const NOTABLE_SUBJECT_RESERVOIR = 64; const MERGE_PR_SUBJECT = /^Merge pull request/; export interface StreamReportCursor { projectKey: string; timestamp: Date; id: string; } export type StreamBillableEvent = TimelineEventDisplay & { /** Stable SQL sort key used only for cursor pagination. */ projectKey: string; }; export interface StreamBillableReportOptions { start?: string | null; end?: string | null; gapThresholdMinutes?: number; highlightLimit?: number; includeClientBreakdown?: boolean; chunkSize?: number; } export type StreamReportPageLoader = ( cursor: StreamReportCursor | null, chunkSize: number, ) => Promise; interface SessionAccumulator { startMs: number | null; endMs: number | null; finalizedMinutes: number; } interface ProjectAccumulator { projectId: string | null; projectName: string; projectColor: string | null; clientId: string | null | undefined; session: SessionAccumulator; manualMinutes: number; eventCount: number; days: Set; bySource: SourceCountMap; gitCommits: number; conversations: Set; files: Set; mergeCount: number; notableSubjects: string[]; tickets: Map; } interface ContributorAccumulator { deviceId: string | null; deviceName: string | null; userId: string | null; userName: string | null; userImage: string | null; eventCount: number; manualMinutes: number; sessions: Map; } interface UnassignedLabelAccumulator { label: string; eventCount: number; manualMinutes: number; session: SessionAccumulator; } interface TicketAccumulator extends ActivityReportTicket { fromLinear: boolean; } function emptySession(): SessionAccumulator { return { startMs: null, endMs: null, finalizedMinutes: 0 }; } function dayKey(date: Date): string { return `${date.getFullYear()}-${String(date.getMonth() + 1).padStart(2, "0")}-${String(date.getDate()).padStart(2, "0")}`; } function parseDayKey(day: string): Date { const [year, month, date] = day.split("-").map(Number); return new Date(year, month - 1, date); } function isUnassigned(projectId: string | null): boolean { return projectId === null || projectId === UNASSIGNED_PROJECT_ID; } function projectBucketKey(projectId: string | null): string { return isUnassigned(projectId) ? UNASSIGNED_PROJECT_ID : (projectId as string); } function addSessionEvent( session: SessionAccumulator, timestampMs: number, gapMs: number, onFinalize?: (startMs: number, endMs: number) => void, ): void { const eventEnd = timestampMs + gapMs; if (session.startMs == null || session.endMs == null) { session.startMs = timestampMs; session.endMs = eventEnd; return; } if (timestampMs <= session.endMs) { if (eventEnd > session.endMs) session.endMs = eventEnd; return; } finalizeCurrentSession(session, onFinalize); session.startMs = timestampMs; session.endMs = eventEnd; } function finalizeCurrentSession( session: SessionAccumulator, onFinalize?: (startMs: number, endMs: number) => void, ): void { if (session.startMs == null || session.endMs == null) return; session.finalizedMinutes += Math.round( (session.endMs - session.startMs) / 60_000, ); onFinalize?.(session.startMs, session.endMs); session.startMs = null; session.endMs = null; } function allocateRangeToDays( startMs: number, endMs: number, dailyMinutes: Map, ): void { const start = new Date(startMs); let cursor = new Date(start.getFullYear(), start.getMonth(), start.getDate()); while (cursor.getTime() < endMs) { const dayEnd = new Date( cursor.getFullYear(), cursor.getMonth(), cursor.getDate() + 1, ); const overlapStart = Math.max(startMs, cursor.getTime()); const overlapEnd = Math.min(endMs, dayEnd.getTime()); const overlapMinutes = Math.max(0, overlapEnd - overlapStart) / 60_000; if (overlapMinutes > 0) { const key = dayKey(cursor); dailyMinutes.set(key, (dailyMinutes.get(key) ?? 0) + overlapMinutes); } cursor = dayEnd; } } function isNearDuplicate(candidate: string, existing: readonly string[]) { const value = candidate.toLowerCase(); return existing.some((item) => { const other = item.toLowerCase(); return ( value === other || value.startsWith(other) || other.startsWith(value) ); }); } function trimTicketMap(tickets: Map): void { if (tickets.size <= TICKETS_PER_PROJECT_CAP) return; const lastId = [...tickets.keys()].sort((a, b) => b.localeCompare(a))[0]; if (lastId) tickets.delete(lastId); } function buildClientTypeBreakdown( rows: ActivityReportProject[], ): ActivityReportClientTypeBreakdown { const result: ActivityReportClientTypeBreakdown = { client: { estimatedMinutes: 0, eventCount: 0, projectCount: 0 }, internal: { estimatedMinutes: 0, eventCount: 0, projectCount: 0 }, }; for (const row of rows) { const bucket = row.clientId != null ? result.client : result.internal; bucket.estimatedMinutes += row.estimatedMinutes; bucket.eventCount += row.eventCount; bucket.projectCount += 1; } return result; } function formatTicketBullet( verb: "Completed" | "Worked", ticket: ActivityReportTicket, ): string { return ticket.title?.trim() ? `${verb} ${ticket.id} ${ticket.title.trim()}` : `${verb} ${ticket.id}`; } function projectOutcome( project: ActivityReportProject, tickets: ActivityReportTicket[], mergeCount: number, notableSubjects: string[], ): string[] { const bullets: string[] = []; const completed = tickets.filter((ticket) => ticket.completed === true); const worked = tickets.filter((ticket) => ticket.completed !== true); for (const ticket of completed.slice(0, 3)) { bullets.push(formatTicketBullet("Completed", ticket)); } const completedOverflow = completed.length - Math.min(completed.length, 3); if (completedOverflow > 0 && bullets.length < OUTCOMES_BULLETS_PER_PROJECT) { bullets.push(`+${completedOverflow} more completed`); } for (const ticket of worked.slice(0, 2)) { if (bullets.length >= OUTCOMES_BULLETS_PER_PROJECT) break; bullets.push(formatTicketBullet("Worked", ticket)); } if (mergeCount > 0 && bullets.length < OUTCOMES_BULLETS_PER_PROJECT) { bullets.push( mergeCount === 1 ? "Merged 1 pull request" : `Merged ${mergeCount} pull requests`, ); } const available = Math.min(3, OUTCOMES_BULLETS_PER_PROJECT - bullets.length); const notable = [...notableSubjects].sort( (a, b) => b.length - a.length || a.localeCompare(b), ); bullets.push(...notable.slice(0, available)); if (bullets.length === 0 && project.gitCommits > 0) { bullets.push( `${project.gitCommits} ${project.gitCommits === 1 ? "commit" : "commits"} over ${project.activeDays} active ${project.activeDays === 1 ? "day" : "days"}`, ); } return bullets; } /** Stateful accumulator exported for deterministic unit tests. */ export class StreamingActivityReportBuilder { private readonly gapMs: number; private readonly highlightLimit: number; private readonly projects = new Map(); private readonly contributors = new Map(); private readonly unassignedLabels = new Map< string, UnassignedLabelAccumulator >(); private readonly allDays = new Set(); private readonly bySource: SourceCountMap = {}; private readonly dailyMinutes = new Map(); private highlightCandidates: HighlightCandidate[] = []; private totalEvents = 0; private finished = false; constructor(private readonly options: StreamBillableReportOptions = {}) { const gapThresholdMinutes = options.gapThresholdMinutes ?? DEFAULT_GAP_THRESHOLD_MINUTES; this.gapMs = gapThresholdMinutes * 60_000; this.highlightLimit = options.highlightLimit ?? 8; } add(event: TimelineEventDisplay): void { if (this.finished) throw new Error("stream_report_already_finished"); const date = new Date(event.timestamp); if (Number.isNaN(date.getTime())) return; this.totalEvents += 1; const day = dayKey(date); this.allDays.add(day); this.bySource[event.source] = (this.bySource[event.source] ?? 0) + 1; const unassigned = isUnassigned(event.projectId); const projectKey = projectBucketKey(event.projectId); let project = this.projects.get(projectKey); if (!project) { project = { projectId: unassigned ? null : event.projectId, projectName: event.projectName || (unassigned ? "Unassigned" : (event.projectId as string)), projectColor: event.projectColor, clientId: event.clientId, session: emptySession(), manualMinutes: 0, eventCount: 0, days: new Set(), bySource: {}, gitCommits: 0, conversations: new Set(), files: new Set(), mergeCount: 0, notableSubjects: [], tickets: new Map(), }; this.projects.set(projectKey, project); } if ( event.projectName && project.projectId && project.projectName === project.projectId ) { project.projectName = event.projectName; } if (!project.projectColor && event.projectColor) { project.projectColor = event.projectColor; } if (project.clientId == null && event.clientId != null) { project.clientId = event.clientId; } project.eventCount += 1; project.days.add(day); project.bySource[event.source] = (project.bySource[event.source] ?? 0) + 1; if (event.source === "manual") { const minutes = event.estimatedMinutes ?? 0; project.manualMinutes += minutes; this.dailyMinutes.set(day, (this.dailyMinutes.get(day) ?? 0) + minutes); } else { addSessionEvent( project.session, date.getTime(), this.gapMs, (startMs, endMs) => allocateRangeToDays(startMs, endMs, this.dailyMinutes), ); } this.collectProjectEvidence(project, event); this.collectContributor(event, projectKey, date.getTime()); if (unassigned) this.collectUnassignedLabel(event, date.getTime()); const candidate = highlightCandidate(event, date.toISOString()); if (candidate) { this.highlightCandidates.push(candidate); const reservoirLimit = Math.max( this.highlightLimit * HIGHLIGHT_RESERVOIR_MULTIPLIER, this.highlightLimit, ); if (this.highlightCandidates.length > reservoirLimit) { this.highlightCandidates.sort((a, b) => a.priority !== b.priority ? a.priority - b.priority : b.timestamp.localeCompare(a.timestamp), ); this.highlightCandidates.length = reservoirLimit; } } } private collectProjectEvidence( project: ProjectAccumulator, event: TimelineEventDisplay, ): void { if (event.source === "git" && isGitMetadata(event.metadata)) { project.gitCommits += 1; const subject = event.metadata.message.split("\n")[0]?.trim() ?? ""; const ticket = extractTicketFromSubject(subject); if (ticket) { if (!project.tickets.has(ticket.id)) { project.tickets.set(ticket.id, { ...ticket, completed: null, fromLinear: false, }); trimTicketMap(project.tickets); } } else if (MERGE_PR_SUBJECT.test(subject)) { project.mergeCount += 1; } else { const cleaned = stripConventionalCommitPrefix(subject); if ( cleaned && !isJunkHighlightLabel(cleaned) && !isNearDuplicate(cleaned, project.notableSubjects) && project.notableSubjects.length < NOTABLE_SUBJECT_RESERVOIR ) { project.notableSubjects.push(cleaned); } } return; } if (CHAT_SOURCES.has(event.source)) { const metadata = event.metadata as { sessionId?: unknown } | null; if (typeof metadata?.sessionId === "string") { project.conversations.add(`${event.source}:${metadata.sessionId}`); } return; } if (event.source === "file" && isFileMetadata(event.metadata)) { project.files.add(event.metadata.path); return; } if (event.source === "linear" && isLinearMetadata(event.metadata)) { const id = event.metadata.issueIdentifier?.trim(); if (!id) return; const state = (event.metadata as unknown as Record) .state; const stateName = typeof state === "string" ? state.toLowerCase() : ""; const completed = event.metadata.action === "completed" || [ "done", "completed", "merged", "closed", "canceled", "cancelled", ].includes(stateName); const existing = project.tickets.get(id); if (existing) { existing.fromLinear = true; if (event.metadata.title?.trim()) { existing.title = event.metadata.title.trim(); } if (completed) existing.completed = true; } else { project.tickets.set(id, { id, title: event.metadata.title?.trim() || null, completed: completed ? true : null, fromLinear: true, }); trimTicketMap(project.tickets); } } } private collectContributor( event: TimelineEventDisplay, projectKey: string, timestampMs: number, ): void { const contributorKey = event.userId ?? event.deviceId ?? "__local__"; let contributor = this.contributors.get(contributorKey); if (!contributor) { contributor = { deviceId: event.deviceId, deviceName: event.deviceName, userId: event.userId ?? null, userName: event.userName ?? null, userImage: event.userImage ?? null, eventCount: 0, manualMinutes: 0, sessions: new Map(), }; this.contributors.set(contributorKey, contributor); } if (!contributor.deviceName && event.deviceName) { contributor.deviceName = event.deviceName; } if (!contributor.userName && event.userName) { contributor.userName = event.userName; } if (!contributor.userImage && event.userImage) { contributor.userImage = event.userImage; } contributor.eventCount += 1; if (event.source === "manual") { contributor.manualMinutes += event.estimatedMinutes ?? 0; return; } let session = contributor.sessions.get(projectKey); if (!session) { session = emptySession(); contributor.sessions.set(projectKey, session); } addSessionEvent(session, timestampMs, this.gapMs); } private collectUnassignedLabel( event: TimelineEventDisplay, timestampMs: number, ): void { const label = unassignedEventLabel(event); let state = this.unassignedLabels.get(label); if (!state) { state = { label, eventCount: 0, manualMinutes: 0, session: emptySession(), }; this.unassignedLabels.set(label, state); } state.eventCount += 1; if (event.source === "manual") { state.manualMinutes += event.estimatedMinutes ?? 0; } else { addSessionEvent(state.session, timestampMs, this.gapMs); } } finish(): ActivityReport { if (this.finished) throw new Error("stream_report_already_finished"); this.finished = true; for (const project of this.projects.values()) { finalizeCurrentSession(project.session, (startMs, endMs) => allocateRangeToDays(startMs, endMs, this.dailyMinutes), ); } for (const contributor of this.contributors.values()) { for (const session of contributor.sessions.values()) { finalizeCurrentSession(session); } } for (const state of this.unassignedLabels.values()) { finalizeCurrentSession(state.session); } const projectRows = this.buildProjectRows(); const totalEstimatedMinutes = projectRows.reduce( (sum, project) => sum + project.estimatedMinutes, 0, ); const unassigned = projectRows.find((project) => project.projectId === null) ?? null; const ticketsByProject = this.buildTicketsByProject(projectRows); const outcomes = this.buildOutcomes(projectRows, ticketsByProject); const report: ActivityReport = { range: { start: this.options.start ?? null, end: this.options.end ?? null, }, totalEstimatedMinutes, totalEvents: this.totalEvents, activeDays: this.allDays.size, bySource: this.bySource, projects: projectRows, unassigned, devices: this.buildContributorRows(), highlights: selectHighlights( this.highlightCandidates, this.highlightLimit, ), daily: this.buildDailySeries(), }; const unassignedBreakdown = this.buildUnassignedBreakdown( unassigned?.estimatedMinutes ?? 0, totalEstimatedMinutes, ); if (unassignedBreakdown) report.unassignedBreakdown = unassignedBreakdown; if (ticketsByProject) report.ticketsByProject = ticketsByProject; if (outcomes) report.outcomes = outcomes; if (this.options.includeClientBreakdown) { report.byClientType = buildClientTypeBreakdown(projectRows); } return report; } private buildProjectRows(): ActivityReportProject[] { const rows = [...this.projects.values()].map((project) => ({ projectId: project.projectId, projectName: project.projectName, projectColor: project.projectColor, clientId: project.clientId, estimatedMinutes: project.session.finalizedMinutes + project.manualMinutes, eventCount: project.eventCount, activeDays: project.days.size, bySource: project.bySource, gitCommits: project.gitCommits, conversations: project.conversations.size, filesModified: project.files.size, })); rows.sort((a, b) => b.estimatedMinutes - a.estimatedMinutes); return rows; } private buildContributorRows(): ActivityReportDevice[] { const rows = [...this.contributors.values()].map((contributor) => ({ deviceId: contributor.deviceId, deviceName: contributor.deviceName, userId: contributor.userId, userName: contributor.userName, userImage: contributor.userImage, eventCount: contributor.eventCount, estimatedMinutes: contributor.manualMinutes + [...contributor.sessions.values()].reduce( (sum, session) => sum + session.finalizedMinutes, 0, ), })); rows.sort((a, b) => b.estimatedMinutes - a.estimatedMinutes); return rows; } private buildDailySeries() { const eventDays = [...this.allDays].sort(); const startDay = this.options.start ? dayKey(new Date(this.options.start)) : null; const endDay = this.options.end ? dayKey(new Date(this.options.end)) : null; let domainStart = startDay ?? eventDays[0] ?? null; let domainEnd = endDay ?? eventDays.at(-1) ?? null; if (eventDays[0] && (!domainStart || eventDays[0] < domainStart)) { domainStart = eventDays[0]; } const lastEventDay = eventDays.at(-1); if (lastEventDay && (!domainEnd || lastEventDay > domainEnd)) { domainEnd = lastEventDay; } if (!domainStart || !domainEnd) return []; if (domainEnd < domainStart) domainEnd = domainStart; let cursor = parseDayKey(domainStart); const last = parseDayKey(domainEnd); while (cursor.getTime() <= last.getTime()) { const key = dayKey(cursor); if (!this.dailyMinutes.has(key)) this.dailyMinutes.set(key, 0); cursor = new Date( cursor.getFullYear(), cursor.getMonth(), cursor.getDate() + 1, ); } return [...this.dailyMinutes.entries()] .filter(([date]) => date >= domainStart && date <= domainEnd) .sort(([a], [b]) => a.localeCompare(b)) .map(([date, minutes]) => ({ date, minutes: Math.round(minutes) })); } private buildUnassignedBreakdown( unassignedMinutes: number, totalMinutes: number, ): ActivityReportUnassignedBreakdownRow[] | undefined { if ( this.unassignedLabels.size === 0 || totalMinutes <= 0 || unassignedMinutes / totalMinutes <= UNASSIGNED_BREAKDOWN_THRESHOLD ) { return undefined; } const rows = [...this.unassignedLabels.values()] .map((state) => ({ label: state.label, estimatedMinutes: state.session.finalizedMinutes + state.manualMinutes, eventCount: state.eventCount, })) .sort( (a, b) => b.estimatedMinutes - a.estimatedMinutes || a.label.localeCompare(b.label), ); if (rows.length <= UNASSIGNED_BREAKDOWN_TOP_N) return rows; const top = rows.slice(0, UNASSIGNED_BREAKDOWN_TOP_N); const rest = rows.slice(UNASSIGNED_BREAKDOWN_TOP_N); top.push({ label: "Other", estimatedMinutes: rest.reduce( (sum, row) => sum + row.estimatedMinutes, 0, ), eventCount: rest.reduce((sum, row) => sum + row.eventCount, 0), }); return top; } private buildTicketsByProject( projectRows: ActivityReportProject[], ): ActivityReportTicketsByProject[] | undefined { const rows: ActivityReportTicketsByProject[] = []; for (const project of projectRows) { const state = this.projects.get(projectBucketKey(project.projectId)); if (!state || state.tickets.size === 0) continue; rows.push({ projectId: project.projectId, projectName: project.projectName, tickets: [...state.tickets.values()] .map(({ id, title, completed }) => ({ id, title, completed })) .sort((a, b) => a.id.localeCompare(b.id)), }); } if (rows.length === 0) return undefined; rows.sort((a, b) => a.projectName.localeCompare(b.projectName)); return rows; } private buildOutcomes( projectRows: ActivityReportProject[], ticketsByProject: ActivityReportTicketsByProject[] | undefined, ): ActivityReportOutcome[] | undefined { const ticketMap = new Map( (ticketsByProject ?? []).map((row) => [ projectBucketKey(row.projectId), row.tickets, ]), ); const outcomes: ActivityReportOutcome[] = []; for (const project of projectRows) { const key = projectBucketKey(project.projectId); const state = this.projects.get(key); if (!state) continue; const bullets = projectOutcome( project, ticketMap.get(key) ?? [], state.mergeCount, state.notableSubjects, ); if (bullets.length > 0) { outcomes.push({ projectId: project.projectId, projectName: project.projectName, bullets, }); } } if (outcomes.length === 0) return undefined; if (outcomes.length <= OUTCOMES_PROJECTS_CAP) return outcomes; const kept = outcomes.slice(0, OUTCOMES_PROJECTS_CAP); const foldedKeys = new Set( outcomes .slice(OUTCOMES_PROJECTS_CAP) .map((row) => projectBucketKey(row.projectId)), ); const foldedMinutes = projectRows .filter((row) => foldedKeys.has(projectBucketKey(row.projectId))) .reduce((sum, row) => sum + row.estimatedMinutes, 0); const hours = Math.round((foldedMinutes / 60) * 10) / 10; kept.push({ projectId: null, projectName: "Other projects", bullets: [`Other projects: ${hours}h`], }); return kept; } } /** * Consume every page until a short page is reached. `chunkSize` controls I/O * and peak row memory only; it is never interpreted as a correctness cap. */ export async function streamBillableReport( loadPage: StreamReportPageLoader, options: StreamBillableReportOptions = {}, ): Promise { const chunkSize = options.chunkSize ?? 3_000; if (!Number.isInteger(chunkSize) || chunkSize < 1) { throw new Error("invalid_stream_report_chunk_size"); } const builder = new StreamingActivityReportBuilder(options); let cursor: StreamReportCursor | null = null; while (true) { const page = await loadPage(cursor, chunkSize); if (page.length > chunkSize) { throw new Error("stream_report_page_exceeds_chunk_size"); } for (const event of page) builder.add(event); if (page.length < chunkSize) break; const last = page.at(-1); if (!last) break; const nextCursor = { projectKey: last.projectKey, timestamp: new Date(last.timestamp), id: last.id, }; if ( Number.isNaN(nextCursor.timestamp.getTime()) || (cursor && cursor.projectKey === nextCursor.projectKey && cursor.timestamp.getTime() === nextCursor.timestamp.getTime() && cursor.id === nextCursor.id) ) { throw new Error("stream_report_cursor_did_not_advance"); } cursor = nextCursor; } return builder.finish(); }