import * as Shardus from '../shardus/shardus-types'; import Profiler from '../utils/profiler'; import { P2PModuleContext as P2P } from '../p2p/Context'; import Crypto, { HashableObject } from '../crypto'; import Logger from '../logger'; import log4js from 'log4js'; import StateManager from '.'; import { AccountIDAndHash, AccountIdAndHashToRepair, AccountPreTest, HashTrieNode, HashTrieSyncConsensus, HashTrieUpdateStats, RadixAndHashWithNodeId, RadixAndChildHashesWithNodeId, RadixAndHash, ShardedHashTrie, TrieAccount, IsInsyncResult, CycleShardData, SignedReceipt } from './state-manager-types'; import { Ordering } from '../utils'; import { Response } from 'express-serve-static-core'; type Line = { raw: string; file: { owner: string; }; }; type AccountHashStats = { matched: number; visisted: number; empty: number; nullResults: number; numRequests: number; responses: number; exceptions: number; radixToReq: number; actualRadixRequests: number; }; type AccountStats = { skipping: number; multiRequests: number; requested: number; }; interface AccountRepairDataResponse { nodes: Shardus.Node[]; wrappedDataList: Shardus.WrappedData[]; } declare class AccountPatcher { app: Shardus.App; crypto: Crypto; config: Shardus.StrictServerConfiguration; profiler: Profiler; p2p: P2P; logger: Logger; mainLogger: log4js.Logger; fatalLogger: log4js.Logger; shardLogger: log4js.Logger; statsLogger: log4js.Logger; statemanager_fatal: (key: string, log: string) => void; stateManager: StateManager; treeMaxDepth: number; treeSyncDepth: number; shardTrie: ShardedHashTrie; totalAccounts: number; accountUpdateQueue: TrieAccount[]; accountUpdateQueueFuture: TrieAccount[]; accountRemovalQueue: string[]; hashTrieSyncConsensusByCycle: Map; incompleteNodes: HashTrieNode[]; debug_ignoreUpdates: boolean; lastInSyncResult: IsInsyncResult; failedLastTrieSync: boolean; failStartCycle: number; failEndCycle: number; failRepairsCounter: number; syncFailHistory: { s: number; e: number; cycles: number; repaired: number; }[]; sendHashesToEdgeNodes: boolean; lastCycleNonConsensusRanges: { low: string; high: string; }[]; nonStoredRanges: { low: string; high: string; }[]; radixIsStored: Map; repairRequestsMadeThisCycle: { cycle: number; numRequests: number; }; lastRepairInfo: unknown; constructor(stateManager: StateManager, profiler: Profiler, app: Shardus.App, logger: Logger, p2p: P2P, crypto: Crypto, config: Shardus.StrictServerConfiguration); hashObj(value: HashableObject): string; sortByAccountID(a: TrieAccount, b: TrieAccount): Ordering; sortByRadix(a: RadixAndHash, b: RadixAndHash): Ordering; /*** * ######## ## ## ######## ######## ####### #### ## ## ######## ###### * ## ### ## ## ## ## ## ## ## ## ### ## ## ## ## * ## #### ## ## ## ## ## ## ## ## #### ## ## ## * ###### ## ## ## ## ## ######## ## ## ## ## ## ## ## ###### * ## ## #### ## ## ## ## ## ## ## #### ## ## * ## ## ### ## ## ## ## ## ## ## ### ## ## ## * ######## ## ## ######## ## ####### #### ## ## ## ###### */ setupHandlers(): void; getAccountTreeInfo(accountID: string): TrieAccount; /*** * ## ## ######## ### ######## ######## ###### ## ## ### ######## ######## ######## ######## #### ######## * ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## * ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## * ## ## ######## ## ## ## ###### ###### ######### ## ## ######## ## ## ## ######## ## ###### * ## ## ## ######### ## ## ## ## ## ######### ## ## ## ## ## ## ## ## ## * ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## * ####### ## ## ## ## ######## ###### ## ## ## ## ## ## ######## ## ## ## #### ######## */ upateShardTrie(cycle: number): HashTrieUpdateStats; getNonConsensusRanges(cycle: number): { low: string; high: string; }[]; getConsensusRanges(cycle: number): { low: string; high: string; }[]; getNonStoredRanges(cycle: number): { low: string; high: string; }[]; getSyncTrackerRanges(): { low: string; high: string; }[]; /** * Uses a wrappable start and end partition range as input and figures out the array * of ranges that would not be covered by these partitions. * * TODO! consider if the offset used in "partition space" should really be happening on the result address instead! * I think that would be more correct. getConsensusSnapshotPartitions would need adjustments after this since it * is making this compensation on its own. * * @param shardValues * @param startPartition * @param endPartition * @param depth How many characters long should the high/low return values be? usually treeSyncDepth */ getNonParitionRanges(shardValues: CycleShardData, startPartition: number, endPartition: number, depth: number): { low: string; high: string; }[]; initStoredRadixValues(cycle: number): void; isRadixStored(_cycle: number, radix: string): boolean; /*** * ######## #### ######## ######## ###### ####### ## ## ###### ######## ## ## ## ## ###### * ## ## ## ## ## ## ## ## ## ### ## ## ## ## ### ## ## ## ## ## * ## ## ## ## ## ## ## ## #### ## ## ## #### ## ## ## ## * ## ## ## ###### ###### ## ## ## ## ## ## ###### ###### ## ## ## ## ## ###### * ## ## ## ## ## ## ## ## ## #### ## ## ## #### ## ## ## * ## ## ## ## ## ## ## ## ## ## ### ## ## ## ## ### ## ## ## ## * ######## #### ## ## ###### ####### ## ## ###### ######## ## ## ####### ###### */ /** * diffConsenus * get a list where localMap does not have entries that match consensusArray. * Note this only works one way. we do not find cases where localMap has an entry that consensusArray does not. * // (TODO, compute this and at least start logging it.(if in debug mode)) * @param consensusArray the list of radix and hash values that have been voted on by the majority * @param localMap a map of our hashTrie nodes to compare to the consensus */ diffConsenus(consensusArray: RadixAndHash[], localMap: Map): { radix: string; hash: string; }[]; /** * findExtraChildren * a debug method to figure out if we have keys not covered by other nodes. * @param consensusArray * @param localLayerMap * @returns */ findExtraBadKeys(consensusArray: RadixAndHashWithNodeId[], localLayerMap: Map): RadixAndHashWithNodeId[]; /*** * ###### ####### ## ## ######## ## ## ######## ######## ###### ####### ## ## ######## ######## ### ###### ######## * ## ## ## ## ### ### ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## * ## ## ## #### #### ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## * ## ## ## ## ### ## ######## ## ## ## ###### ## ## ## ## ## ###### ######## ## ## ## #### ###### * ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ######### ## ## ## * ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## * ###### ####### ## ## ## ####### ## ######## ###### ####### ### ######## ## ## ## ## ###### ######## */ /** * computeCoverage * * Take a look at the winning votes and build of lists of which nodes we can ask for information * this happens once per cycle then getNodeForQuery() can be used to cleanly figure out what node to ask for a query given * a certain radix value. * * @param cycle */ computeCoverage(cycle: number): void; /*** * ###### ######## ######## ## ## ####### ######## ######## ######## ####### ######## ####### ## ## ######## ######## ## ## * ## ## ## ## ### ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## * ## ## ## #### ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## #### * ## #### ###### ## ## ## ## ## ## ## ## ###### ###### ## ## ######## ## ## ## ## ###### ######## ## * ## ## ## ## ## #### ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## * ## ## ## ## ## ### ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## * ###### ######## ## ## ## ####### ######## ######## ## ####### ## ## ##### ## ####### ######## ## ## ## */ /** * getNodeForQuery * Figure out what node we can ask for a query related to the given radix. * this will node that has given us a winning vote for the given radix * @param radix * @param cycle * @param nextNode pass true to start asking the next node in the list for data. */ getNodeForQuery(radix: string, cycle: number, nextNode?: boolean): Shardus.Node | null; /** * getChildrenOf * ask nodes for the child node information of the given list of radix values * TODO convert to allSettled?, but support a timeout? * @param radixHashEntries * @param cycle */ getChildrenOf(radixHashEntries: RadixAndHash[], cycle: number): Promise; /** * getChildAccountHashes * requests account hashes from one or more nodes. * TODO convert to allSettled?, but support a timeout? * @param radixHashEntries * @param cycle */ getChildAccountHashes(radixHashEntries: RadixAndHash[], cycle: number): Promise<{ radixAndChildHashes: RadixAndChildHashesWithNodeId[]; getAccountHashStats: AccountHashStats; }>; /*** * #### ###### #### ## ## ###### ## ## ## ## ###### * ## ## ## ## ### ## ## ## ## ## ### ## ## ## * ## ## ## #### ## ## #### #### ## ## * ## ###### ## ## ## ## ###### ## ## ## ## ## * ## ## ## ## #### ## ## ## #### ## * ## ## ## ## ## ### ## ## ## ## ### ## ## * #### ###### #### ## ## ###### ## ## ## ###### */ /** * isInSync * * looks at sync level hashes to figure out if any are out of matching. * there are cases where this is false but we dig into accounts an realize we do not * or can't yet repair something. * * @param cycle */ isInSync(cycle: number): IsInsyncResult; /*** * ######## #### ## ## ######## ######## ### ######## ### ###### ###### ####### ## ## ## ## ######## ###### * ## ## ### ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ### ## ## ## ## * ## ## #### ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## #### ## ## ## * ###### ## ## ## ## ## ## ######## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ###### * ## ## ## #### ## ## ## ## ######### ## ## ######### ## ## ## ## ## ## ## #### ## ## * ## ## ## ### ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ### ## ## ## * ## #### ## ## ######## ######## ## ## ######## ## ## ###### ###### ####### ####### ## ## ## ###### */ /** * findBadAccounts * * starts at the sync level hashes that dont match and queries for child nodes to get more details about * what accounts could possibly be bad. At the lowest level gets a list of accounts and hashes * We double check out cache values before returning a list of bad accounts that need repairs. * * @param cycle */ findBadAccounts(cycle: number): Promise; extractLeafNodes(rootNode: HashTrieNode): HashTrieNode[]; /*** * ## ## ######## ######## ### ######## ######## ### ###### ###### ####### ## ## ## ## ######## ## ## ### ###### ## ## * ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ### ## ## ## ## ## ## ## ## ## ## * ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## #### ## ## ## ## ## ## ## ## ## * ## ## ######## ## ## ## ## ## ###### ## ## ## ## ## ## ## ## ## ## ## ## ######### ## ## ###### ######### * ## ## ## ## ## ######### ## ## ######### ## ## ## ## ## ## ## #### ## ## ## ######### ## ## ## * ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ### ## ## ## ## ## ## ## ## ## * ####### ## ######## ## ## ## ######## ## ## ###### ###### ####### ####### ## ## ## ## ## ## ## ###### ## ## */ /** * updateAccountHash * This is the main function called externally to tell the hash trie what the hash value is for a given accountID * * @param accountID * @param hash * */ updateAccountHash(accountID: string, hash: string): void; removeAccountHash(accountID: string): void; /*** * ######## ######## ####### ### ######## ###### ### ###### ######## ###### ## ## ## ## ###### ## ## ### ###### ## ## ######## ###### * ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ### ## ## ## ## ## ## ## ## ## ## ## ## ## ## * ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## #### #### ## ## ## ## ## ## ## ## ## ## ## * ######## ######## ## ## ## ## ## ## ## ## ## ###### ## ###### ## ## ## ## ## ######### ## ## ###### ######### ###### ###### * ## ## ## ## ## ## ######### ## ## ## ######### ## ## ## ## ## #### ## ## ## ######### ## ## ## ## ## * ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ### ## ## ## ## ## ## ## ## ## ## ## ## ## * ######## ## ## ####### ## ## ######## ###### ## ## ###### ## ###### ## ## ## ###### ## ## ## ## ###### ## ## ######## ###### */ /** * broadcastSyncHashes * after each tree computation we figure out what radix + hash values we can send out * these will be nodes at the treeSyncDepth (which is higher up than our leafs, and is efficient, but also adjusts to support sharding) * there are important conditions about when we can send a value for a given radix and who we can send it to. * * @param cycle */ broadcastSyncHashes(cycle: number): Promise; /*** * ## ## ######## ######## ### ######## ######## ######## ######## #### ######## ### ## ## ######## ######## ######## ####### ### ######## ###### ### ###### ######## * ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ### ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## * ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## #### ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## * ## ## ######## ## ## ## ## ## ###### ## ######## ## ###### ## ## ## ## ## ## ## ######## ######## ## ## ## ## ## ## ## ## ## ###### ## * ## ## ## ## ## ######### ## ## ## ## ## ## ## ######### ## #### ## ## ## ## ## ## ## ## ######### ## ## ## ######### ## ## * ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ### ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## * ####### ## ######## ## ## ## ######## ## ## ## #### ######## ## ## ## ## ######## ######## ## ## ####### ## ## ######## ###### ## ## ###### ## */ /** * updateTrieAndBroadCast * calculates what our tree leaf(max) depth and sync depths are. * if there is a change we have to do some partition work to send old leaf data to new leafs. * Then calls upateShardTrie() and broadcastSyncHashes() * * @param cycle */ updateTrieAndBroadCast(cycle: number): Promise; requestOtherNodesToRepair(accountsToFix: AccountIdAndHashToRepair[]): Promise; /*** * ######## ######## ###### ######## ### ## ## ######## ######## ### ######## ###### ## ## ### ###### ###### ####### ## ## ## ## ######## ###### * ## ## ## ## ## ## ## ### ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ### ## ## ## ## * ## ## ## ## ## ## #### ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## #### ## ## ## * ## ###### ###### ## ## ## ## ## ## ## ## ######## ## ## ## ## ######### ## ## ## ## ## ## ## ## ## ## ## ## ###### * ## ## ## ## ######### ## #### ## ## ## ######### ## ## ## ## ######### ## ## ## ## ## ## ## #### ## ## * ## ## ## ## ## ## ## ## ### ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ### ## ## ## * ## ######## ###### ## ## ## ## ## ######## ## ## ## ## ###### ## ## ## ## ###### ###### ####### ####### ## ## ## ###### */ /** * testAndPatchAccounts * does a quick check to see if we are isInSync() with the sync level votes we have been given. * if we are out of sync it uses findBadAccounts to recursively find what accounts need repair. * we then query nodes for the account data we need to do a repair with * finally we check the repair data and use it to repair out accounts. * * @param cycle */ testAndPatchAccounts(cycle: number): Promise; /** * simulateRepairs * incomplete. the idea was to see if potential repairs can even solve our current sync issue. * not sure this is worth the perf/complexity/effort. * * if we miss something, the patcher can just try again next cycle. * * @param cycle * @param badAccounts */ simulateRepairs(_cycle: number, badAccounts: AccountIDAndHash[]): AccountPreTest[]; /*** * ###### ######## ######## ### ###### ###### ####### ## ## ## ## ######## ######## ######## ######## ### #### ######## ######## ### ######## ### * ## ## ## ## ## ## ## ## ## ## ## ## ## ## ### ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## * ## ## ## ## ## ## ## ## ## ## ## #### ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## * ## #### ###### ## ## ## ## ## ## ## ## ## ## ## ## ## ######## ###### ######## ## ## ## ######## ## ## ## ## ## ## ## * ## ## ## ## ######### ## ## ## ## ## ## ## #### ## ## ## ## ## ######### ## ## ## ## ## ######### ## ######### * ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ### ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## * ###### ######## ## ## ## ###### ###### ####### ####### ## ## ## ## ## ######## ## ## ## #### ## ## ######## ## ## ## ## ## */ /** * getAccountRepairData * take a list of bad accounts and figures out which nodes we can ask to get the data. * makes one or more data requests in parallel * organizes and returns the results. * @param cycle * @param badAccounts */ getAccountRepairData(cycle: number, badAccounts: AccountIDAndHash[]): Promise<{ repairDataResponse: AccountRepairDataResponse; stateTableDataMap: Map; getAccountStats: AccountStats; }>; /*** * ######## ######## ####### ###### ######## ###### ###### ###### ## ## ### ######## ######## ######## ## ## ## ## ######## * ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ### ### ## ## * ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## #### #### ## ## * ######## ######## ## ## ## ###### ###### ###### ###### ######### ## ## ######## ## ## ## ## ## ## ## ### ## ######## * ## ## ## ## ## ## ## ## ## ## ## ## ######### ## ## ## ## ## ## ## ## ## ## ## * ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## * ## ## ## ####### ###### ######## ###### ###### ###### ## ## ## ## ## ## ######## ######## ####### ## ## ## */ /** * processShardDump * debug only code to create a shard report. * @param stream * @param lines */ processShardDump(stream: Response, number>, lines: Line[]): { allPassed: boolean; allPassed2: boolean; }; calculateMinVotes(): number; } type BadAccountStats = { testedSyncRadix: number; skippedSyncRadix: number; badSyncRadix: number; ok_noTrieAcc: number; ok_trieHashBad: number; fix_butHashMatch: number; fixLastSeen: number; needsVotes: number; subHashesTested: number; trailColdLevel: number; checkedLevel: number; leafsChecked: number; leafResponses: number; getAccountHashStats: Record; }; type BadAccountsInfo = { badAccounts: AccountIDAndHash[]; hashesPerLevel: number[]; checkedKeysPerLevel: number[]; requestedKeysPerLevel: number[]; badHashesPerLevel: number[]; accountHashesChecked: number; stats: BadAccountStats; extraBadAccounts: AccountIdAndHashToRepair[]; extraBadKeys: RadixAndHash[]; accountsTheyNeedToRepair: AccountIdAndHashToRepair[]; }; export type AccountRepairInstruction = { accountID: string; hash: string; txId: string; accountData: Shardus.WrappedData; targetNodeId: string; signedReceipt: SignedReceipt; }; export default AccountPatcher;