import * as Shardus from '../shardus/shardus-types'; import Profiler from '../utils/profiler'; import Crypto from '../crypto'; import Logger from '../logger'; import AccountCache from './AccountCache'; import StateManager from '.'; import { QueueEntry, CycleShardData } from './state-manager-types'; import { StateManager as StateManagerTypes } from '@shardeum-foundation/lib-types'; import Log4js from 'log4js'; import { Response } from 'express-serve-static-core'; type RawAccountData = { data: { data: { balance: string; }; } | { balance: string; }; } | { balance: string; }; type Line = { raw: string; file: { owner: string; }; }; /** * PartitionStats is a system that allows the dapp to build custom anonymous tallies of accounts and committed TXs. * This code manages opaque blobs and then uploads them once a cycle as part of a report. * This can ultimately allow for a sort of map reduce that helps and explorer server figure out things like * total account balance, or number of accounts, etc. * * Other state manager modules need to call a few functions to make this work: * -When a new account is seen for the first time (create or synced or other?) it calls statsDataSummaryInit() * -When an account is updated it calls statsDataSummaryUpdate() and passes in the last and current copy of the account * -When a TX is committed statsTxSummaryUpdate() is called * * These call results in calls to the dapp. This code creates and maintains opaqueBlobs. This code should not understand * what is in the blob, it just has to hand the correct blob to the dapp for updating. The dapp does not have to worry about complex * address math to figure out what opaqueBlob should be used! * * Once per cycle buildStatsReport() is called to generate a summary for any stats data that this node has consensus coverage over. * * --------------Everything else is just debug support, or support accessors. * * A few other notes. * -TX stats are bucketed per cycle per stat partition and the tally starts freash each cycle * -DATA(account) stats bucketed by stat partition only. The update operations go through a queue * that is synchronized to only commit the stats as a cycle is "ready" i.e. old enough that nodes can be in sync. * * Debug notes: * -there is are some debug endpoints but they will only work with smaller numbers of nodes. * get-stats-report-all * * -shardus-scan tool can do stats analysis if you pass in the folder your instances are in. ex: * node .\statsReport.js C:\shardus\gitlab\liberdus-server5\instances * * it is almost impossible to trace failures without turning invasiveDebugInfo on. (but never check it in as true!) * invasiveDebugInfo allows stats report to have enough clue to determine which accounts or * TXs are missing, and which nodes voted on which opaqueBlobs. * */ declare class PartitionStats { app: Shardus.App; crypto: Crypto; config: Shardus.StrictServerConfiguration; profiler: Profiler; logger: Logger; stateManager: StateManager; mainLogger: Log4js.Logger; fatalLogger: Log4js.Logger; shardLogger: Log4js.Logger; statsLogger: Log4js.Logger; summaryBlobByPartition: Map; summaryPartitionCount: number; txSummaryBlobCollections: StateManagerTypes.StateManagerTypes.SummaryBlobCollection[]; extensiveRangeChecking: boolean; accountCache: AccountCache; invasiveDebugInfo: boolean; statsProcessCounter: number; maxCyclesToStoreBlob: number; statemanager_fatal: (key: string, log: string) => void; /** buildStatsReport uses the work queue to build stats reports at the correct times. do not add items to work queue if stats are disabled */ workQueue: { cycle: number; fn: (...args: unknown[]) => unknown; args: unknown[]; }[]; constructor(stateManager: StateManager, profiler: Profiler, app: Shardus.App, logger: Logger, crypto: Crypto, config: Shardus.StrictServerConfiguration, accountCache: AccountCache); /*** * ######## ## ## ######## ######## ####### #### ## ## ######## ###### * ## ### ## ## ## ## ## ## ## ## ### ## ## ## ## * ## #### ## ## ## ## ## ## ## ## #### ## ## ## * ###### ## ## ## ## ## ######## ## ## ## ## ## ## ## ###### * ## ## #### ## ## ## ## ## ## ## #### ## ## * ## ## ### ## ## ## ## ## ## ## ### ## ## ## * ######## ## ## ######## ## ####### #### ## ## ## ###### */ setupHandlers(): void; /*** * #### ## ## #### ######## ### ## ## ######## ### ###### ###### ######## ###### ###### * ## ### ## ## ## ## ## ### ## ## ## ## ## ## ## ## ## ## ## ## ## ## * ## #### ## ## ## ## ## #### ## ## ## ## ## ## ## ## ## ## * ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ###### ###### ###### * ## ## #### ## ## ######### ## #### ## ## ######### ## ## ## ## ## * ## ## ### ## ## ## ## ## ### ## ## ## ## ## ## ## ## ## ## ## ## ## * #### ## ## #### ## ## ## ## ## ######## ## ## ###### ###### ######## ###### ###### */ /** * get a new data summary blob * note that opaqueBlob is what we eventually show to the dapp. * This code should not understand opaqueBlob *other than some debug only hacks hardcoded for liberdus * @param partition */ getNewSummaryBlob(partition: number): StateManagerTypes.StateManagerTypes.SummaryBlob; /** * init the data stats blobs */ initSummaryBlobs(): void; /** * Init the TX summary blobs * @param cycleNumber */ initTXSummaryBlobsForCycle(cycleNumber: number): StateManagerTypes.StateManagerTypes.SummaryBlobCollection; /** * gets the TX summary blob partition for the given cycle. (should be the TX's cycleToRecordOn). * @param cycle */ getOrCreateTXSummaryBlobCollectionByCycle(cycle: number): StateManagerTypes.StateManagerTypes.SummaryBlobCollection; getSummaryBlobPartition(address: string): number; /** * Get the correct summary blob that matches this address. * Address must be in the account space. (i.e. AccountIDs) * Never pass a TX id into this (we use the first sorted writable account key for a TX) * @param address */ getSummaryBlob(address: string): StateManagerTypes.StateManagerTypes.SummaryBlob; hasAccountBeenSeenByStats(accountId: string): boolean; /** * Figures out what snapshot partitions are fully covered by our consensus partitions. * Note these partitions are not in the same address space!. * There is one consenus partition per node in the network. (each node covers them in a radius) * Snapshot partitions are currently a fixed count so that we don't have to recompute all of the summaries each cycle when the * network topology changes * @param cycleShardData * //the return value is a bit obtuse. should decide if a list or map output is better, or are they both needed. */ /** * * @param cycleShardData */ getConsensusSnapshotPartitions(cycleShardData: CycleShardData): { list: number[]; map: Map; }; /*** * #### ## ## #### ######## ######## ### ######## ### ###### ######## ### ######## ###### * ## ### ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## * ## #### ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## * ## ## ## ## ## ## ## ## ## ## ## ## ## ###### ## ## ## ## ###### * ## ## #### ## ## ## ## ######### ## ######### ## ## ######### ## ## * ## ## ### ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## * #### ## ## #### ## ######## ## ## ## ## ## ###### ## ## ## ## ###### */ /** * If we have never seen this account before then call init on it. * This will queue an opertation that will lead to calling * this.app.dataSummaryInit() * so that the dapp can define how to tally a newly seen account * @param cycle * @param accountId * @param accountDataRaw * @param debugMsg */ statsDataSummaryInit(cycle: number, accountId: string, accountDataRaw: unknown, debugMsg: string): void; /** * The internal function that calls into the app. * this has to go into a queue so that it can be caled only when the call * is for a valid cycle# (i.e. old enough cycle that we can have consistent results) * @param cycle * @param blob * @param accountDataRaw * @param accountId * @param opCounter */ private internalDoInit; /*** * ## ## ######## ######## ### ######## ######## ######## ### ######## ### ###### ######## ### ######## ###### * ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## * ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## * ## ## ######## ## ## ## ## ## ###### ## ## ## ## ## ## ## ###### ## ## ## ## ###### * ## ## ## ## ## ######### ## ## ## ## ######### ## ######### ## ## ######### ## ## * ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## * ####### ## ######## ## ## ## ######## ######## ## ## ## ## ## ###### ## ## ## ## ###### */ /** * * This will queue an opertation that will lead to calling * this.app.dataSummaryUpdate() * so that the dapp can define how to tally an updated account. * and old copy and current copy of the account are passed in. * WARNING. current limitation can not guarantee if there are intermediate states * between these two accounts. Need to write user facing docs on this. * * @param cycle * @param accountDataBefore * @param accountDataAfter * @param debugMsg */ statsDataSummaryUpdate(cycle: number, accountDataBefore: unknown, accountDataAfter: Shardus.WrappedData, debugMsg: string): void; /** * does the queued work of calling dataSummaryUpdate() * @param cycle * @param blob * @param accountDataBefore * @param accountDataAfter * @param opCounter */ private internalDoUpdate; /*** * ## ## ######## ######## ### ######## ######## ######## ## ## ###### ######## ### ######## ###### * ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## * ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## * ## ## ######## ## ## ## ## ## ###### ## ### ###### ## ## ## ## ###### * ## ## ## ## ## ######### ## ## ## ## ## ## ## ######### ## ## * ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## * ####### ## ######## ## ## ## ######## ## ## ## ###### ## ## ## ## ###### */ /** * Call this to update the TX stats. * note, this does not have to get queued because it is a per cycle calculation by default * (bucketed by stat partition and by cycle) * @param cycle * @param queueEntry */ statsTxSummaryUpdate(cycle: number, queueEntry: QueueEntry): void; /*** * ######## ## ## #### ## ######## ######## ######## ######## ####### ######## ######## * ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## * ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## * ######## ## ## ## ## ## ## ######## ###### ######## ## ## ######## ## * ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## * ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## * ######## ####### #### ######## ######## ## ## ######## ## ####### ## ## ## */ /** * Build the statsDump report that is used sent to the archive server. * Will only include covered stats partitions. * @param cycleShardData * @param excludeEmpty */ buildStatsReport(cycleShardData: CycleShardData, excludeEmpty?: boolean): StateManagerTypes.StateManagerTypes.StatsClump; /*** * ######## ######## ######## ## ## ###### ###### ## ## ######## ######## ####### ######## ######## * ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## * ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## * ## ## ###### ######## ## ## ## #### ###### ## ## ######## ######## ## ## ######## ## * ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## * ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## * ######## ######## ######## ####### ###### ###### ####### ## ## ####### ## ## ## */ /** * Builds a debug object with stata information for logging or runtime debugging with endpoints. * @param cycle * @param writeTofile * @param cycleShardData */ dumpLogsForCycle(cycle: number, writeTofile?: boolean, cycleShardData?: CycleShardData): { cycle: number; dataStats: any[]; txStats: any[]; covered: any[]; cycleDebugNotes: Record; }; /*** * ######## ######## ####### ###### ######## ###### ###### ######## ### ######## ### ###### ######## ### ######## ###### ######## ## ## ## ## ######## * ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ### ### ## ## * ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## #### #### ## ## * ######## ######## ## ## ## ###### ###### ###### ## ## ## ## ## ## ## ###### ## ## ## ## ###### ## ## ## ## ## ### ## ######## * ## ## ## ## ## ## ## ## ## ## ## ######### ## ######### ## ## ######### ## ## ## ## ## ## ## ## ## * ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## * ## ## ## ####### ###### ######## ###### ###### ######## ## ## ## ## ## ###### ## ## ## ## ###### ######## ####### ## ## ## */ processDataStatsDump(stream: Response, number>, tallyFunction: { (opaqueBlob: unknown): unknown; (arg0: unknown): unknown; }, lines: Line[]): { allPassed: boolean; allPassedMetric2: boolean; singleVotePartitions: number; multiVotePartitions: number; badPartitions: any[]; dataByParition: Map; }; /*** * ######## ######## ####### ###### ######## ###### ###### ######## ## ## ###### ######## ### ######## ###### ######## ## ## ## ## ######## * ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ### ### ## ## * ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## #### #### ## ## * ######## ######## ## ## ## ###### ###### ###### ## ### ###### ## ## ## ## ###### ## ## ## ## ## ### ## ######## * ## ## ## ## ## ## ## ## ## ## ## ## ## ## ######### ## ## ## ## ## ## ## ## ## * ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## * ## ## ## ####### ###### ######## ###### ###### ## ## ## ###### ## ## ## ## ###### ######## ####### ## ## ## */ processTxStatsDump(stream: Response, number>, tallyFunction: { (opaqueBlob: { totalTx?: number; }): number; (arg0: unknown): unknown; }, lines: Line[]): { allPassed: boolean; allPassedMetric2: boolean; singleVotePartitions: number; multiVotePartitions: number; badPartitions: unknown[]; totalTx: number; }; dataStatsTallyFunction(opaqueBlob: { totalBalance?: number; }): number; txStatsTallyFunction(opaqueBlob: { totalTx?: number; }): number; debugAccountData(accountData: RawAccountData): string; addDebugToBlob(blob: StateManagerTypes.StateManagerTypes.SummaryBlob, accountID: string): void; } export default PartitionStats;