import bippath from "bip32-path"; import isEqual from "lodash/isEqual"; import { BigNumber } from "bignumber.js"; import { Observable, Observer, Subject, defaultIfEmpty, from, isObservable, lastValueFrom, map, of, takeLast, takeUntil, } from "rxjs"; import { log } from "@ledgerhq/logs"; import { WrongDeviceForAccount } from "../errors"; import { getSeedIdentifierDerivation, getDerivationModesForCurrency, getDerivationScheme, runDerivationScheme, isIterableDerivationMode, derivationModeSupportsIndex, getMandatoryEmptyAccountSkip, getDerivationModeStartsAt, } from "../derivation"; import { isAccountEmpty, clearAccount, emptyHistoryCache, encodeAccountId, decodeAccountId, } from "../account"; import { generateHistoryFromOperations, recalculateAccountBalanceHistories, } from "../account/balanceHistoryCache"; import { shouldRetainPendingOperation } from "../account/pending"; import { shouldShowNewAccount } from "../account/support"; import getAddressWrapper, { GetAddressFn } from "./getAddressWrapper"; import type { GetAddressResult } from "../derivation"; import type { CryptoCurrency } from "../types"; import type { Account, AccountBridge, AccountRaw, CurrencyBridge, DerivationMode, Operation, ProtoNFT, ScanAccountEvent, SyncConfig, TransactionCommon, TransactionStatusCommon, } from "@ledgerhq/types-live"; // Customize the way to iterate on the keychain derivation type IterateResult = ({ index, derivationsCache, derivationScheme, derivationMode, currency, deviceId, }: { index: number; derivationsCache: Record; derivationScheme: string; derivationMode: DerivationMode; currency: CryptoCurrency; deviceId: string; }) => Promise; export type IterateResultBuilder = ({ result, // derivation on the "root" of the derivation derivationMode, // identify the current derivation scheme derivationScheme, }: { result: GetAddressResult; derivationMode: DerivationMode; derivationScheme: string; }) => Promise; export type AccountShapeInfo = { currency: CryptoCurrency; address: string; index: number; initialAccount?: A | undefined; derivationPath: string; derivationMode: DerivationMode; rest?: any; deviceId?: string; }; export type GetAccountShape = ( accountShape: AccountShapeInfo, config: SyncConfig, ) => Promise>; export type GetAccountShapeStream = ( accountShape: AccountShapeInfo, config: SyncConfig, ) => Observable>; type AccountUpdater = (account: A) => A; /** * Normalizes a Promise or Observable to an Observable. * This helper function allows the framework to handle both Promise-based * and Observable-based getAccountShape implementations uniformly. * * @param value - Either a Promise or Observable * @returns An Observable */ function normalizeToObservable(value: Promise | Observable): Observable { if (isObservable(value)) { return value as Observable; } return from(value as Promise); } // compare that two dates are roughly the same date in order to update the case it would have drastically changed const sameDate = (a: Date, b: Date) => Math.abs(a.getTime() - b.getTime()) < 1000 * 60 * 30; // a later sync can report a sequence number the stored operation is missing, but a fetch that // reports none must never erase one: `mergeOps` replaces `stored` by `fetched` when they differ const sameSequenceNumber = (stored: Operation, fetched: Operation) => { const next = fetched.transactionSequenceNumber; if (next === undefined) return true; const current = stored.transactionSequenceNumber; return current !== undefined && current.isEqualTo(next); }; // an operation is relatively immutable, however we saw that sometimes it can temporarily change due to reorg,.. // NOTE not symmetrical: `a` is the stored operation, `b` the freshly fetched one export const sameOp = (a: Operation, b: Operation): boolean => a === b || (a.id === b.id && // hash, accountId, type are in id (a.fee ? a.fee.isEqualTo(b.fee) : a.fee === b.fee) && (a.value ? a.value.isEqualTo(b.value) : a.value === b.value) && a.nftOperations?.length === b.nftOperations?.length && sameDate(a.date, b.date) && a.blockHeight === b.blockHeight && sameSequenceNumber(a, b) && isEqual(a.senders, b.senders) && isEqual(a.recipients, b.recipients)); // efficiently prepend newFetched operations to existing operations export function mergeOps( existing: Operation[], // existing operations. sorted (newer to older). deduped. newFetched: Operation[], // new fetched operations. not sorted. not deduped. time is allowed to overlap inside existing. ): // return a list of operations, deduped and sorted from newer to older Operation[] { // there is new fetched if (newFetched.length === 0) return existing; // efficient lookup map of id. const existingIds: Record = {}; for (const o of existing) { existingIds[o.id] = o; } // only keep the newFetched that are not in existing. this array will be mutated let newOps = newFetched .filter(o => !existingIds[o.id] || !sameOp(existingIds[o.id], o)) .sort((a, b) => b.date.valueOf() - a.date.valueOf()); // Deduplicate new ops to guarantee operations don't have dups const newOpsIds: Record = {}; newOps.forEach(op => { newOpsIds[op.id] = op; }); newOps = Object.values(newOpsIds); // return existing when there is no real new operations if (newOps.length === 0) return existing; // edge case, existing can be empty. return the sorted list. if (existing.length === 0) return newOps; // building up merging the ops const all: Operation[] = []; for (const o of existing) { // prepend all the new ops that have higher date while (newOps.length > 0 && newOps[0].date >= o.date) { all.push(newOps.shift() as Operation); } if (!newOpsIds[o.id]) { all.push(o); } } return all; } export const mergeNfts = (oldNfts: ProtoNFT[], newNfts: ProtoNFT[]): ProtoNFT[] => { // Getting a map of id => NFT const newNftsPerId: Record = {}; newNfts.forEach(n => { newNftsPerId[n.id] = n; }); // copying the argument to avoid mutating it const nfts = oldNfts.slice(); for (let i = 0; i < nfts.length; i++) { const nft = nfts[i]; // The NFTs are the same, do don't anything if (!newNftsPerId[nft.id]) { nfts.splice(i, 1); i--; } else if (!isEqual(nft, newNftsPerId[nft.id])) { // Use the new NFT instead nfts[i] = newNftsPerId[nft.id]; } // Delete it from the newNfts to keep only the un-added ones at the end delete newNftsPerId[nft.id]; } // Prepending newNfts to respect nfts's newest to oldest order return Object.values(newNftsPerId).concat(nfts); }; const getCustomDataFromAccountId = (accountId: string): string | undefined => { try { return decodeAccountId(accountId).customData; } catch { return undefined; } }; const getXpubOrAddressFromAccountId = (accountId: string): string | undefined => { try { return decodeAccountId(accountId).xpubOrAddress; } catch { return undefined; } }; export const makeSync = < T extends TransactionCommon = TransactionCommon, A extends Account = Account, U extends TransactionStatusCommon = TransactionStatusCommon, O extends Operation = Operation, R extends AccountRaw = AccountRaw, >({ getAccountShape, postSync = (_, a) => a, shouldMergeOps = true, }: { getAccountShape: GetAccountShape | GetAccountShapeStream; postSync?: (initial: A, synced: A) => A; shouldMergeOps?: boolean; }): AccountBridge["sync"] => (initial: A, syncConfig: SyncConfig): Observable> => new Observable((o: Observer>) => { let innerSub: { unsubscribe: () => void } | null = null; let cancelled = false; async function main() { const customData = getCustomDataFromAccountId(initial.id); const accountId = encodeAccountId({ type: "js", version: "2", currencyId: initial.currency.id, // Identity comes from the immutable account id, decoupled from the mutable `xpub` field // (so healing xpub to the public key never re-keys the account). Only if the id is // undecodable (malformed) do we fall back to the pre-existing recovery behaviour. xpubOrAddress: getXpubOrAddressFromAccountId(initial.id) ?? (initial.xpub || initial.freshAddress), derivationMode: initial.derivationMode, ...(customData && { customData }), }); const needClear = initial.id !== accountId; try { const freshAddressPath = getSeedIdentifierDerivation( initial.currency, initial.derivationMode as DerivationMode, ); const shapeResult = getAccountShape( { currency: initial.currency, index: initial.index, address: initial.freshAddress, derivationPath: freshAddressPath, derivationMode: initial.derivationMode as DerivationMode, initialAccount: needClear ? clearAccount(initial) : initial, }, syncConfig, ); const shape$ = normalizeToObservable(shapeResult); const updater = (shape: Partial) => (acc: A): A => { let a = acc; // a is a immutable version of Account, based on acc if (needClear && acc.id !== accountId) { a = clearAccount(acc); } // FIXME reconsider doing mergeOps here. work is redundant for impl like eth const operations = shouldMergeOps ? mergeOps(a.operations, shape.operations || []) : shape.operations || []; a = postSync(a, { ...a, id: accountId, spendableBalance: shape.balance || a.balance, operationsCount: shape.operationsCount || operations.length, lastSyncDate: new Date(), creationDate: operations.length > 0 ? operations[operations.length - 1].date : new Date(), ...shape, operations, pendingOperations: a.pendingOperations.filter(op => shouldRetainPendingOperation(a, op), ), }); a = recalculateAccountBalanceHistories(a, acc); if (!a.used) { a.used = !isAccountEmpty(a); } return a; }; const observer: Observer> = { next: shape => { if (!cancelled) o.next(updater(shape)); }, complete: () => { if (!cancelled) o.complete(); }, error: e => { if (!cancelled) o.error(e); }, }; innerSub = shape$.subscribe(observer); if (cancelled) { innerSub.unsubscribe(); return; } } catch (e) { if (!cancelled) o.error(e); } } main(); return () => { cancelled = true; innerSub?.unsubscribe(); }; }); const defaultIterateResultBuilder = (getAddressFn: GetAddressFn) => () => Promise.resolve( async ({ index, derivationsCache, derivationScheme, derivationMode, currency, deviceId, }: { index: number | string; derivationsCache: Record; derivationScheme: string; derivationMode: DerivationMode; currency: CryptoCurrency; deviceId: string; }): Promise => { const freshAddressPath = runDerivationScheme(derivationScheme, currency, { account: index, }); const cacheKey = `${freshAddressPath}:${derivationMode}`; let res = derivationsCache[cacheKey]; if (!res) { res = await getAddressWrapper(getAddressFn)(deviceId, { currency, path: freshAddressPath, derivationMode, }); derivationsCache[cacheKey] = res; } return res as GetAddressResult; }, ); export const makeScanAccounts = ({ getAccountShape, buildIterateResult, getAddressFn, postSync = (_, a) => a, }: { getAccountShape: GetAccountShape | GetAccountShapeStream; buildIterateResult?: IterateResultBuilder; getAddressFn: GetAddressFn; postSync?: (initial: A, synced: A) => A; }): CurrencyBridge["scanAccounts"] => ({ currency, deviceId, syncConfig, scheme }): Observable => new Observable((outerObs: Observer<{ type: "discovered"; account: Account }>) => { if (buildIterateResult === undefined) { buildIterateResult = defaultIterateResultBuilder(getAddressFn); } let finished = false; const teardown$ = new Subject(); const unsubscribe = () => { teardown$.next(); teardown$.complete(); finished = true; }; const derivationsCache: Record = {}; function stepAccount( index: number, res: GetAddressResult, derivationMode: DerivationMode, seedIdentifier: string, ): Observable { if (finished) return of(null); const { address, path: freshAddressPath, ...rest } = res; const accountShapeResult = getAccountShape( { currency, index, address, derivationPath: freshAddressPath, derivationMode, rest, deviceId, }, syncConfig, ); const accountShape$: Observable> = normalizeToObservable(accountShapeResult); function accountShapePipe(accountShape: Partial): Account { const freshAddress = address; const operations = accountShape.operations || []; const operationsCount = accountShape.operationsCount || operations.length; const creationDate = operations.length > 0 ? operations[operations.length - 1].date : new Date(); const balance = accountShape.balance || new BigNumber(0); const spendableBalance = accountShape.spendableBalance || new BigNumber(0); if (!accountShape.id) throw new Error("account ID must be provided"); if (balance.isNaN()) throw new Error("invalid balance NaN"); const initialAccount: Account = { type: "Account", id: accountShape.id, seedIdentifier, freshAddress, freshAddressPath, derivationMode, used: false, index, currency, operationsCount, operations: [], swapHistory: [], pendingOperations: [], lastSyncDate: new Date(), creationDate, // overrides balance, spendableBalance, blockHeight: 0, balanceHistoryCache: emptyHistoryCache, }; let account = { ...initialAccount, ...accountShape }; if (account.balanceHistoryCache === emptyHistoryCache) { account.balanceHistoryCache = generateHistoryFromOperations(account); } if (accountShape.used === undefined) { account.used = !isAccountEmpty(account); } // Bitcoin needs to compute the freshAddressPath itself, // so we update it afterwards if (account?.freshAddressPath) { res.address = account.freshAddress; derivationsCache[account.freshAddressPath] = res; } // Temporary: we're going to remove transformations in postSync that should be done in getAccountShape // Using the generic doesn't resolve correctly // oxlint-disable-next-line typescript/consistent-type-assertions account = postSync(initialAccount as unknown as A, account as unknown as A); log("scanAccounts", "derivationsCache", res); log( "scanAccounts", `scanning ${currency.id} at ${freshAddressPath}: ${res.address} resulted of ${ account ? `Account with ${account.operations.length} txs` : "no account" }`, ); return account; } return accountShape$.pipe(takeUntil(teardown$), map(accountShapePipe)); } async function main() { try { let derivationModes: DerivationMode[] = []; if (scheme === null || scheme === undefined) { derivationModes = getDerivationModesForCurrency(currency); } else { derivationModes = [scheme]; } for (const derivationMode of derivationModes) { if (finished) break; const path = getSeedIdentifierDerivation(currency, derivationMode); log("scanAccounts", `scanning ${currency.id} on derivationMode=${derivationMode}`); const seedCacheKey = `${path}:${derivationMode}`; let result: GetAddressResult = derivationsCache[seedCacheKey]; if (!result) { try { result = await getAddressFn(deviceId, { currency, path, derivationMode, }); derivationsCache[seedCacheKey] = result; } catch (e) { if ((e as { name?: string })?.name === "UnsupportedDerivation") { log("scanAccounts", "ignore derivationMode=" + derivationMode); continue; } throw e; } } if (!result) continue; const seedIdentifier = result.publicKey; let emptyCount = 0; // We only ever propose ONE new (empty) account: the first empty index // available. Intermediate empty accounts sitting inside a gap before a // later used account must not be offered as creatable. let firstEmptyAccountEmitted = false; const mandatoryEmptyAccountSkip = getMandatoryEmptyAccountSkip(derivationMode); const derivationScheme = getDerivationScheme({ derivationMode, currency, }); const stopAt = isIterableDerivationMode(derivationMode) ? 255 : 1; const startsAt = getDerivationModeStartsAt(derivationMode); log( "debug", `start scanning account process. MandatoryEmptyAccountSkip ${mandatoryEmptyAccountSkip} / StartsAt: ${startsAt} - StopAt: ${stopAt}`, ); const iterateResult = await buildIterateResult!({ result, derivationMode, derivationScheme, }); for (let index = startsAt; index < stopAt; index++) { log("debug", `start to scan a new account. Index: ${index}`); if (finished) { log("debug", `new account scanning process has been finished`); break; } if (!derivationModeSupportsIndex(derivationMode, index)) continue; const res = await iterateResult({ index, derivationsCache, derivationMode, derivationScheme, currency, deviceId, }); if (!res) break; const account$ = stepAccount(index, res, derivationMode, seedIdentifier); const account = await lastValueFrom(account$.pipe(takeLast(1), defaultIfEmpty(null))); if (finished) { log("debug", `new account scanning process has been finished`); break; } if (account) { const showNewAccount = shouldShowNewAccount(currency, derivationMode); if (account.used) { log("debug", `Emit 'discovered' event for a used account. Index: ${index}`); outerObs.next({ type: "discovered", account, }); // A used account closes the current run of empty accounts: the // gap limit counts *consecutive* empty accounts, so reset here. // This lets discovery cross empty gaps between used accounts as // long as no gap is longer than `mandatoryEmptyAccountSkip`. emptyCount = 0; } else { // Offer only the first empty account (lowest index) as creatable. if (!firstEmptyAccountEmitted && showNewAccount) { log( "debug", `Emit 'discovered' event for the first empty account. Index: ${index}`, ); outerObs.next({ type: "discovered", account, }); firstEmptyAccountEmitted = true; } if (emptyCount >= mandatoryEmptyAccountSkip) { break; // reached the gap limit of consecutive empty accounts } emptyCount++; } } } } if (!finished) { outerObs.complete(); } } catch (e) { outerObs.error(e); } } main(); return unsubscribe; }); export function makeAccountBridgeReceive( getAddressFn: GetAddressFn, { injectGetAddressParams, }: { injectGetAddressParams?: (account: A) => any; } = {}, ) { return ( account: A, { verify, deviceId, path, }: { verify?: boolean; deviceId: string; subAccountId?: string; path?: string; }, ) => { const arg = { verify, currency: account.currency, derivationMode: account.derivationMode, path: path || account.freshAddressPath, ...(injectGetAddressParams && injectGetAddressParams(account)), }; return from( getAddressFn(deviceId, arg).then(r => { const accountAddress = account.freshAddress; if (verify && r.address !== accountAddress) { throw new WrongDeviceForAccount(); } return r; }), ); }; } /** * Default trivial implem for updateTransaction, that keeps reference stability (for React) */ export function updateTransaction(t: T, patch: Partial): T { const patched = { ...t, ...patch }; return isEqual(t, patched) ? t : patched; } /** * Default trivial implem for getSerializedAddressParameters */ export function getSerializedAddressParameters(account: Account): Buffer { return bip32asBuffer(account.freshAddressPath); } /** * Parses a BIP32 path string (e.g. "m/84'/0'/0'") into an array of path indices. * Uses the bip32-path library for consistent parsing across the codebase. */ export function pathStringToArray(path: string): number[] { return path ? bippath.fromString(path).toPathArray() : []; } export function bip32asBuffer(path: string): Buffer { const pathElements = pathStringToArray(path); const buffer = Buffer.alloc(1 + pathElements.length * 4); buffer[0] = pathElements.length; pathElements.forEach((element, index) => { buffer.writeUInt32BE(element, 1 + 4 * index); }); return buffer; }