/** * Access Pattern Tracker for Intelligent Tiering * * This module provides advanced access pattern tracking for the postgres.do * tiered storage architecture. It enables intelligent promotion/demotion * decisions based on: * * - Exponential decay scoring (configurable half-life) * - Recency-weighted frequency calculation * - Working set estimation using sliding window * - Page correlation detection using co-access matrix * * The tracker is designed to work with Cloudflare Durable Objects and supports * serialization for hibernation. * * @module @dotdo/postgres-shared/access-pattern-tracker */ import type { AccessStats, StorageTier } from './storage-policy.js' import { ACCESS_PATTERN_DECAY_HALF_LIFE_MS, ACCESS_PATTERN_SLIDING_WINDOW_MS, ACCESS_PATTERN_CORRELATION_WINDOW_MS, ACCESS_PATTERN_PRUNE_INTERVAL_MS, MAX_TRACKED_PAGES, } from './constants.js' /** * Configuration for the access pattern tracker */ export interface AccessPatternConfig { /** * Half-life for exponential decay in milliseconds. * After this time, the access score is halved. * Default: 5 minutes (300,000ms) */ decayHalfLifeMs: number /** * Sliding window size in milliseconds for working set estimation. * Default: 1 minute (60,000ms) */ slidingWindowMs: number /** * Maximum time gap between accesses to be considered correlated. * Pages accessed within this time of each other are correlated. * Default: 100ms */ correlationWindowMs: number /** * Minimum correlation strength to track (0-1). * Correlations below this are pruned. * Default: 0.1 */ minCorrelationStrength: number /** * Maximum number of correlations to track per page. * Default: 50 */ maxCorrelationsPerPage: number /** * Recency bonus weight (0-1). * Higher values give more weight to recent accesses. * Default: 0.3 */ recencyWeight: number /** * Maximum number of pages to track. * Oldest/lowest-scored pages are evicted when limit is reached. * Default: 10,000 */ maxTrackedPages: number /** * Interval for pruning stale data in milliseconds. * Default: 5 minutes (300,000ms) */ pruneIntervalMs: number } /** * Default configuration for the access pattern tracker */ export const DEFAULT_ACCESS_PATTERN_CONFIG: AccessPatternConfig = { decayHalfLifeMs: ACCESS_PATTERN_DECAY_HALF_LIFE_MS, slidingWindowMs: ACCESS_PATTERN_SLIDING_WINDOW_MS, correlationWindowMs: ACCESS_PATTERN_CORRELATION_WINDOW_MS, minCorrelationStrength: 0.1, maxCorrelationsPerPage: 50, recencyWeight: 0.3, maxTrackedPages: MAX_TRACKED_PAGES, pruneIntervalMs: ACCESS_PATTERN_PRUNE_INTERVAL_MS, } /** * Internal representation of a tracked page's access data */ interface PageAccessData { /** Page identifier */ pageId: string /** Timestamp of first access (ms since epoch) */ firstAccessTime: number /** Timestamp of last access (ms since epoch) */ lastAccessTime: number /** Total number of accesses */ totalAccessCount: number /** Access timestamps within the sliding window (for window access count) */ recentAccessTimes: number[] /** Accumulated decayed score */ decayedScore: number /** Last time the score was updated (for lazy decay calculation) */ lastScoreUpdate: number /** Current storage tier (if known) */ currentTier?: StorageTier /** Size of the page in bytes (if known) */ sizeBytes?: number } /** * Sparse co-access matrix entry for correlation tracking */ interface CoAccessEntry { /** Target page ID */ targetPageId: string /** Number of co-accesses */ coAccessCount: number /** Last co-access timestamp */ lastCoAccessTime: number } /** * Serialized state for DO hibernation */ export interface SerializedAccessPatternState { version: number config: AccessPatternConfig pages: Array<{ pageId: string firstAccessTime: number lastAccessTime: number totalAccessCount: number recentAccessTimes: number[] decayedScore: number lastScoreUpdate: number currentTier?: StorageTier sizeBytes?: number }> correlations: Array<{ sourcePageId: string entries: CoAccessEntry[] }> workingSet: { windowAccessTimes: number[] uniquePagesInWindow: string[] } lastPruneTime: number } /** * Result of scoring a page */ export interface PageScore { /** Page identifier */ pageId: string /** Combined score (decay + recency) */ score: number /** Decay component of the score */ decayScore: number /** Recency bonus component */ recencyBonus: number /** Access count within the sliding window */ windowAccessCount: number /** Total access count */ totalAccessCount: number } /** * Working set estimation result */ export interface WorkingSetEstimate { /** Number of unique pages accessed in the window */ uniquePageCount: number /** Total accesses in the window */ totalAccessCount: number /** Page IDs in the working set */ pageIds: string[] /** Estimated size in bytes (if size data available) */ estimatedSizeBytes?: number /** Window duration in milliseconds */ windowMs: number } /** * Correlated pages result */ export interface CorrelatedPages { /** Source page ID */ sourcePageId: string /** Correlated pages sorted by strength (strongest first) */ correlations: Array<{ pageId: string strength: number coAccessCount: number lastCoAccessTime: number }> } /** * Access Pattern Tracker for intelligent tiering decisions * * This class tracks page access patterns to enable intelligent promotion * and demotion decisions in the tiered storage system. * * Features: * - Exponential decay scoring with configurable half-life * - Recency-weighted frequency calculation * - Working set estimation using sliding window * - Page correlation detection using sparse co-access matrix * - Serialization support for DO hibernation * * @example * ```typescript * const tracker = new AccessPatternTracker() * * // Record page accesses * tracker.recordAccess('page-1') * tracker.recordAccess('page-2') * * // Get page scores for tiering decisions * const scores = tracker.getPageScores() * console.log(scores[0]) // Highest scored page * * // Get working set estimate * const workingSet = tracker.getWorkingSetEstimate() * console.log(`Working set: ${workingSet.uniquePageCount} pages`) * * // Get correlated pages for batch promotion * const correlated = tracker.getCorrelatedPages('page-1') * console.log(`Pages to promote with page-1:`, correlated.correlations) * * // Serialize for hibernation * const state = tracker.serialize() * * // Restore from hibernation * const restoredTracker = AccessPatternTracker.deserialize(state) * ``` */ export class AccessPatternTracker { private config: AccessPatternConfig private pages: Map private correlations: Map private recentAccessedPageIds: Array<{ pageId: string; timestamp: number }> private lastPruneTime: number /** * Create a new access pattern tracker * * @param config - Optional configuration overrides */ constructor(config: Partial = {}) { this.config = { ...DEFAULT_ACCESS_PATTERN_CONFIG, ...config } this.pages = new Map() this.correlations = new Map() this.recentAccessedPageIds = [] this.lastPruneTime = Date.now() } /** * Record a page access * * @param pageId - The page identifier * @param timestamp - Access timestamp (defaults to now) * @param sizeBytes - Optional page size in bytes * @param currentTier - Optional current storage tier */ recordAccess( pageId: string, timestamp: number = Date.now(), sizeBytes?: number, currentTier?: StorageTier ): void { // Update or create page data let pageData = this.pages.get(pageId) if (!pageData) { const newPageData: PageAccessData = { pageId, firstAccessTime: timestamp, lastAccessTime: timestamp, totalAccessCount: 0, recentAccessTimes: [], decayedScore: 0, lastScoreUpdate: timestamp, } if (currentTier !== undefined) { newPageData.currentTier = currentTier } if (sizeBytes !== undefined) { newPageData.sizeBytes = sizeBytes } pageData = newPageData this.pages.set(pageId, pageData) } // At this point pageData is guaranteed to exist const existingPage = pageData // Update page data existingPage.lastAccessTime = timestamp existingPage.totalAccessCount++ existingPage.recentAccessTimes.push(timestamp) if (sizeBytes !== undefined) { existingPage.sizeBytes = sizeBytes } if (currentTier !== undefined) { existingPage.currentTier = currentTier } // Update decayed score with lazy evaluation this.updateDecayedScore(existingPage, timestamp) existingPage.decayedScore += 1 // Add 1 for this access // Prune old access times from sliding window const windowCutoff = timestamp - this.config.slidingWindowMs existingPage.recentAccessTimes = existingPage.recentAccessTimes.filter((t) => t >= windowCutoff) // Track for correlation detection this.trackForCorrelation(pageId, timestamp) // Periodic pruning if (timestamp - this.lastPruneTime > this.config.pruneIntervalMs) { this.prune(timestamp) } } /** * Get the score for a specific page * * @param pageId - The page identifier * @param currentTime - Current timestamp (defaults to now) * @returns Page score or null if page not tracked */ getPageScore(pageId: string, currentTime: number = Date.now()): PageScore | null { const pageData = this.pages.get(pageId) if (!pageData) { return null } return this.calculatePageScore(pageData, currentTime) } /** * Get scores for all tracked pages, sorted by score descending * * @param currentTime - Current timestamp (defaults to now) * @param limit - Maximum number of results (default: all) * @returns Array of page scores sorted by score descending */ getPageScores(currentTime: number = Date.now(), limit?: number): PageScore[] { const scores: PageScore[] = [] for (const pageData of this.pages.values()) { scores.push(this.calculatePageScore(pageData, currentTime)) } // Sort by score descending scores.sort((a, b) => b.score - a.score) return limit !== undefined ? scores.slice(0, limit) : scores } /** * Get access statistics for a page (compatible with TierPolicy interface) * * @param pageId - The page identifier * @param currentTime - Current timestamp (defaults to now) * @returns Access statistics or null if page not tracked */ getAccessStats(pageId: string, currentTime: number = Date.now()): AccessStats | null { const pageData = this.pages.get(pageId) if (!pageData) { return null } // Calculate window access count const windowCutoff = currentTime - this.config.slidingWindowMs const windowAccessCount = pageData.recentAccessTimes.filter((t) => t >= windowCutoff).length const result: AccessStats = { totalAccessCount: pageData.totalAccessCount, windowAccessCount, firstAccessTime: pageData.firstAccessTime, lastAccessTime: pageData.lastAccessTime, } if (pageData.sizeBytes !== undefined) { result.sizeBytes = pageData.sizeBytes } return result } /** * Get working set estimate based on sliding window * * @param currentTime - Current timestamp (defaults to now) * @returns Working set estimation */ getWorkingSetEstimate(currentTime: number = Date.now()): WorkingSetEstimate { const windowCutoff = currentTime - this.config.slidingWindowMs const uniquePages = new Set() let totalAccesses = 0 let totalSizeBytes = 0 let hasAllSizes = true for (const pageData of this.pages.values()) { const windowAccesses = pageData.recentAccessTimes.filter((t) => t >= windowCutoff) if (windowAccesses.length > 0) { uniquePages.add(pageData.pageId) totalAccesses += windowAccesses.length if (pageData.sizeBytes !== undefined) { totalSizeBytes += pageData.sizeBytes } else { hasAllSizes = false } } } const estimate: WorkingSetEstimate = { uniquePageCount: uniquePages.size, totalAccessCount: totalAccesses, pageIds: Array.from(uniquePages), windowMs: this.config.slidingWindowMs, } if (hasAllSizes) { estimate.estimatedSizeBytes = totalSizeBytes } return estimate } /** * Get pages correlated with a given page * * Pages that are frequently accessed together (within the correlation window) * are considered correlated. This enables batch promotion of related pages. * * @param pageId - The source page identifier * @param minStrength - Minimum correlation strength to include (default: from config) * @returns Correlated pages sorted by strength */ getCorrelatedPages(pageId: string, minStrength?: number): CorrelatedPages { const entries = this.correlations.get(pageId) || [] const pageData = this.pages.get(pageId) const minStr = minStrength ?? this.config.minCorrelationStrength // Calculate correlation strength as ratio of co-accesses to source accesses const sourceAccessCount = pageData?.totalAccessCount || 1 const correlations = entries .map((entry) => ({ pageId: entry.targetPageId, strength: entry.coAccessCount / sourceAccessCount, coAccessCount: entry.coAccessCount, lastCoAccessTime: entry.lastCoAccessTime, })) .filter((c) => c.strength >= minStr) .sort((a, b) => b.strength - a.strength) return { sourcePageId: pageId, correlations, } } /** * Get pages that should be promoted together with the given page * * Returns the page itself plus all strongly correlated pages. * * @param pageId - The page being promoted * @param correlationThreshold - Minimum correlation strength (default: 0.5) * @returns Array of page IDs to promote together */ getPagesToPromoteTogether(pageId: string, correlationThreshold: number = 0.5): string[] { const result = [pageId] const correlated = this.getCorrelatedPages(pageId, correlationThreshold) for (const c of correlated.correlations) { if (!result.includes(c.pageId)) { result.push(c.pageId) } } return result } /** * Update the current tier for a page * * @param pageId - The page identifier * @param tier - The new storage tier */ updatePageTier(pageId: string, tier: StorageTier): void { const pageData = this.pages.get(pageId) if (pageData) { pageData.currentTier = tier } } /** * Get the current tier for a page * * @param pageId - The page identifier * @returns Current tier or undefined if not tracked */ getPageTier(pageId: string): StorageTier | undefined { return this.pages.get(pageId)?.currentTier } /** * Check if a page is being tracked * * @param pageId - The page identifier * @returns True if the page is tracked */ isTracked(pageId: string): boolean { return this.pages.has(pageId) } /** * Get the number of tracked pages * * @returns Number of tracked pages */ getTrackedPageCount(): number { return this.pages.size } /** * Get the current configuration * * @returns A copy of the current configuration */ getConfig(): AccessPatternConfig { return { ...this.config } } /** * Serialize the tracker state for DO hibernation * * @returns Serialized state that can be stored and restored */ serialize(): SerializedAccessPatternState { const pages: SerializedAccessPatternState['pages'] = [] for (const pageData of this.pages.values()) { const serializedPage: SerializedAccessPatternState['pages'][number] = { pageId: pageData.pageId, firstAccessTime: pageData.firstAccessTime, lastAccessTime: pageData.lastAccessTime, totalAccessCount: pageData.totalAccessCount, recentAccessTimes: pageData.recentAccessTimes, decayedScore: pageData.decayedScore, lastScoreUpdate: pageData.lastScoreUpdate, } if (pageData.currentTier !== undefined) { serializedPage.currentTier = pageData.currentTier } if (pageData.sizeBytes !== undefined) { serializedPage.sizeBytes = pageData.sizeBytes } pages.push(serializedPage) } const correlations: SerializedAccessPatternState['correlations'] = [] for (const [sourcePageId, entries] of this.correlations.entries()) { correlations.push({ sourcePageId, entries: [...entries], }) } // Build working set info from recent accesses const windowCutoff = Date.now() - this.config.slidingWindowMs const uniquePagesInWindow: string[] = [] const windowAccessTimes: number[] = [] for (const pageData of this.pages.values()) { const windowAccesses = pageData.recentAccessTimes.filter((t) => t >= windowCutoff) if (windowAccesses.length > 0) { uniquePagesInWindow.push(pageData.pageId) windowAccessTimes.push(...windowAccesses) } } return { version: 1, config: this.config, pages, correlations, workingSet: { windowAccessTimes, uniquePagesInWindow, }, lastPruneTime: this.lastPruneTime, } } /** * Deserialize and restore tracker state from DO hibernation * * @param state - Serialized state from serialize() * @returns Restored AccessPatternTracker instance */ static deserialize(state: SerializedAccessPatternState): AccessPatternTracker { const tracker = new AccessPatternTracker(state.config) // Restore pages for (const page of state.pages) { const restoredPage: PageAccessData = { pageId: page.pageId, firstAccessTime: page.firstAccessTime, lastAccessTime: page.lastAccessTime, totalAccessCount: page.totalAccessCount, recentAccessTimes: page.recentAccessTimes, decayedScore: page.decayedScore, lastScoreUpdate: page.lastScoreUpdate, } if (page.currentTier !== undefined) { restoredPage.currentTier = page.currentTier } if (page.sizeBytes !== undefined) { restoredPage.sizeBytes = page.sizeBytes } tracker.pages.set(page.pageId, restoredPage) } // Restore correlations for (const { sourcePageId, entries } of state.correlations) { tracker.correlations.set(sourcePageId, [...entries]) } tracker.lastPruneTime = state.lastPruneTime return tracker } /** * Remove a page from tracking * * @param pageId - The page identifier to remove */ removePage(pageId: string): void { this.pages.delete(pageId) this.correlations.delete(pageId) // Remove correlations TO this page for (const [, entries] of this.correlations) { const idx = entries.findIndex((e) => e.targetPageId === pageId) if (idx !== -1) { entries.splice(idx, 1) } } } /** * Clear all tracking data */ clear(): void { this.pages.clear() this.correlations.clear() this.recentAccessedPageIds = [] this.lastPruneTime = Date.now() } /** * Calculate exponential decay factor * * @param timeSinceAccess - Time since last access in milliseconds * @returns Decay factor (0 to 1) */ private calculateDecayFactor(timeSinceAccess: number): number { // decay = e^(-timeSinceAccess * ln(2) / halfLife) // This gives us: decay = 0.5 when timeSinceAccess = halfLife const lambda = Math.LN2 / this.config.decayHalfLifeMs return Math.exp(-lambda * timeSinceAccess) } /** * Update the decayed score for a page with lazy evaluation * * @param pageData - The page data to update * @param currentTime - Current timestamp */ private updateDecayedScore(pageData: PageAccessData, currentTime: number): void { const timeSinceLastUpdate = currentTime - pageData.lastScoreUpdate if (timeSinceLastUpdate > 0) { const decayFactor = this.calculateDecayFactor(timeSinceLastUpdate) pageData.decayedScore *= decayFactor pageData.lastScoreUpdate = currentTime } } /** * Calculate the full page score * * @param pageData - The page data * @param currentTime - Current timestamp * @returns Calculated page score */ private calculatePageScore(pageData: PageAccessData, currentTime: number): PageScore { // Update decay lazily this.updateDecayedScore(pageData, currentTime) // Calculate recency bonus (0 to 1 based on how recent the last access was) const timeSinceLastAccess = currentTime - pageData.lastAccessTime const recencyFactor = this.calculateDecayFactor(timeSinceLastAccess) const recencyBonus = recencyFactor * this.config.recencyWeight // Combined score const decayScore = pageData.decayedScore const score = decayScore + recencyBonus // Window access count const windowCutoff = currentTime - this.config.slidingWindowMs const windowAccessCount = pageData.recentAccessTimes.filter((t) => t >= windowCutoff).length return { pageId: pageData.pageId, score, decayScore, recencyBonus, windowAccessCount, totalAccessCount: pageData.totalAccessCount, } } /** * Track page access for correlation detection * * @param pageId - The page that was just accessed * @param timestamp - Access timestamp */ private trackForCorrelation(pageId: string, timestamp: number): void { // Find pages accessed within the correlation window const correlationCutoff = timestamp - this.config.correlationWindowMs // Clean up old entries and find correlated pages this.recentAccessedPageIds = this.recentAccessedPageIds.filter( (entry) => entry.timestamp >= correlationCutoff ) // Record co-access with all pages in the correlation window const seenPages = new Set() for (const entry of this.recentAccessedPageIds) { if (entry.pageId !== pageId && !seenPages.has(entry.pageId)) { seenPages.add(entry.pageId) this.recordCoAccess(pageId, entry.pageId, timestamp) this.recordCoAccess(entry.pageId, pageId, timestamp) } } // Add current access to recent list this.recentAccessedPageIds.push({ pageId, timestamp }) } /** * Record a co-access between two pages * * @param sourcePageId - The source page * @param targetPageId - The target page * @param timestamp - Co-access timestamp */ private recordCoAccess(sourcePageId: string, targetPageId: string, timestamp: number): void { let entries = this.correlations.get(sourcePageId) if (!entries) { entries = [] this.correlations.set(sourcePageId, entries) } // Find or create entry let entry = entries.find((e) => e.targetPageId === targetPageId) if (!entry) { // Check if we need to evict an entry if (entries.length >= this.config.maxCorrelationsPerPage) { // Remove the oldest/weakest correlation entries.sort((a, b) => a.lastCoAccessTime - b.lastCoAccessTime) entries.shift() } entry = { targetPageId, coAccessCount: 0, lastCoAccessTime: timestamp, } entries.push(entry) } entry.coAccessCount++ entry.lastCoAccessTime = timestamp } /** * Prune stale data to manage memory usage * * @param currentTime - Current timestamp */ private prune(currentTime: number): void { this.lastPruneTime = currentTime // Prune old access times from sliding window for all pages const windowCutoff = currentTime - this.config.slidingWindowMs for (const pageData of this.pages.values()) { pageData.recentAccessTimes = pageData.recentAccessTimes.filter((t) => t >= windowCutoff) } // Evict pages if over limit if (this.pages.size > this.config.maxTrackedPages) { // Get all page scores and sort by score ascending (lowest first) const scores = this.getPageScores(currentTime) scores.sort((a, b) => a.score - b.score) // Remove lowest scored pages const toRemove = this.pages.size - this.config.maxTrackedPages for (let i = 0; i < toRemove && i < scores.length; i++) { this.removePage(scores[i]!.pageId) } } // Prune weak correlations for (const [sourcePageId, entries] of this.correlations.entries()) { const sourceData = this.pages.get(sourcePageId) const sourceAccessCount = sourceData?.totalAccessCount || 1 const filtered = entries.filter((entry) => { const strength = entry.coAccessCount / sourceAccessCount return strength >= this.config.minCorrelationStrength }) if (filtered.length === 0) { this.correlations.delete(sourcePageId) } else { this.correlations.set(sourcePageId, filtered) } } // Clean up recent accessed page IDs const correlationCutoff = currentTime - this.config.correlationWindowMs this.recentAccessedPageIds = this.recentAccessedPageIds.filter( (entry) => entry.timestamp >= correlationCutoff ) } } /** * Create an access pattern tracker with preset configurations * * @param preset - Preset name or 'default' * @param overrides - Optional configuration overrides * @returns Configured AccessPatternTracker */ export function createAccessPatternTracker( preset: 'default' | 'high-frequency' | 'long-term' | 'correlation-focused' = 'default', overrides?: Partial ): AccessPatternTracker { let baseConfig: Partial switch (preset) { case 'high-frequency': // For workloads with many accesses per second baseConfig = { decayHalfLifeMs: 60 * 1000, // 1 minute half-life slidingWindowMs: 30 * 1000, // 30 second window correlationWindowMs: 50, // 50ms correlation window maxTrackedPages: 50000, } break case 'long-term': // For tracking patterns over longer periods baseConfig = { decayHalfLifeMs: 30 * 60 * 1000, // 30 minute half-life slidingWindowMs: 5 * 60 * 1000, // 5 minute window correlationWindowMs: 500, // 500ms correlation window pruneIntervalMs: 15 * 60 * 1000, // 15 minute prune interval } break case 'correlation-focused': // For workloads where page correlation is important baseConfig = { correlationWindowMs: 200, // 200ms correlation window minCorrelationStrength: 0.05, // Track weaker correlations maxCorrelationsPerPage: 100, // More correlations per page } break default: baseConfig = {} } return new AccessPatternTracker({ ...baseConfig, ...overrides }) }