import { Command, InvalidArgumentError } from '@commander-js/extra-typings' import { loadProcessorConfig, overrideConfigWithOptions, YamlProjectConfig } from '../config.js' import fs from 'fs' import fetch from 'node-fetch' import { buildProcessor } from './build.js' import chalk from 'chalk' import path from 'path' import { ReadKey } from '../key.js' import { createHash } from 'crypto' import { CommonExecOptions, execFileSync } from 'child_process' import { getApiUrl, getSdkVersion } from '../utils.js' import readline from 'readline' import JSZip from 'jszip' import type { UserInfo } from '../../../protos/dist/service/common/protos/common_pb.js' import { CommandOptionsType } from './types.js' import { getSentioNetworkConfig, resolveNetworkAddresses, getWalletFromPrivateKey, requirePrivateKey, checkSTBalance, checkBillingBalance, isOperatorOnChain, uploadToIPFS, createProcessorOnChain, startProcessorOnChain, stopProcessorOnChain, confirmNoPlatformUpload, getProcessorOnChain, deleteProcessorOnChain } from '../network.js' import { ethers } from 'ethers' import { Auth, DefaultBatchUploader, FileType, IPFSBatchUploader, WalrusBatchUploader } from '../uploader.js' export { type Auth } from '../uploader.js' const PROCESSOR_ID_RE = /^[a-zA-Z0-9_-]+$/ function validateProcessorId(id: string): void { if (id.length === 0) { throw new Error('Processor ID must not be empty.') } if (id.length >= 32) { throw new Error(`Processor ID "${id}" is too long (${id.length} chars). Maximum length is 31 characters.`) } if (!PROCESSOR_ID_RE.test(id)) { throw new Error(`Processor ID "${id}" contains invalid characters. Only alphanumeric, "-", and "_" are allowed.`) } } function myParseInt(value: string, dummyPrevious: number): number { // parseInt takes a string and a radix const parsedValue = parseInt(value, 10) if (isNaN(parsedValue)) { throw new InvalidArgumentError('Not a number.') } return parsedValue } export function createUploadCommand() { return new Command('upload') .description('Upload processor to Sentio') .option( '--owner ', "(Optional) Project owner. Platform mode: project owner username. --no-platform mode: on-chain owner address (0x...) — $PRIVATE_KEY then signs as an operator on the owner's behalf, requires Permissions.addOperator beforehand." ) .option('--name ', '(Optional) Override Project name') .option( '--continue-from ', '(Optional) Continue processing data from the specific processor version which will keeping the old data from previous version and will STOP that version IMMEDIATELY.', myParseInt ) .option( '--checkpoint ', '(Optional) Checkpoint(s) to rollback to, only available with --continue-from. Format: "chain_id:block_number" or "block_number" (uses default chain). Can specify multiple checkpoints.' ) .option('--debug', '(Optional) Run driver in debug mode') .option('-y --silent-overwrite', '(Optional) Create project or upload new version without confirmation') .option('--skip-build', 'Skip build & pack file before uploading') .option('--skip-gen', 'Skip code generation.') .option('--skip-deps', 'Skip dependency enforce.') .option('--example', 'Generate example usage of the processor.') .option('--walrus', 'Store file in walrus') .option('--path ', 'override project path, default to current directory') .option('--api-key ', '(Optional) Manually provide API key rather than use saved credential') .option('--token ', '(Optional) Manually provide token rather than use saved credential') .option('--host ', '(Optional) Override Sentio Host name') .option('--num-workers ', '(Optional) Number of processor workers to start', myParseInt) .option('--sentio-network ', '(Optional) Sentio network to connect to, can be testnet, devnet or mainnet') .option( '--required-chain-id ', '(Optional) Specify chain IDs required for the Sentio network. This option is only available when --sentio-network is used. If omitted, all chain IDs from the project configuration (contracts or network overrides) will be used.' ) .option( '--no-platform', 'Upload processor directly to Sentio Network without platform support. Requires $PRIVATE_KEY env var. Supported networks: testnet, devnet.' ) .option( '--ipfs-put-url ', 'IPFS upload endpoint (used with --no-platform)', 'https://api.sentio.xyz/v1/ipfs/add' ) .option( '--address-book
', '(Optional) Override the default AddressBook contract address used to resolve on-chain contracts (used with --no-platform).' ) .action(async (options, command) => { const processorConfig = loadProcessorConfig(options.path) overrideConfigWithOptions(processorConfig, options) if (options.path) { process.chdir(options.path) } if (options.platform === false) { await runNoPlatformUpload(processorConfig, options) } else { await runUploadInternal(processorConfig, options, command) } }) } async function runNoPlatformUpload( processorConfig: YamlProjectConfig, options: CommandOptionsType ) { // Determine network - default to testnet for --no-platform const network = options.sentioNetwork || 'testnet' const networkConfig = getSentioNetworkConfig(network, options.addressBook) // Step 1: Require PRIVATE_KEY (operator key when --owner is set, otherwise the owner's own key) const privateKey = requirePrivateKey() const wallet = getWalletFromPrivateKey(privateKey) const walletAddress = wallet.address // Operator mode: --owner is the on-chain owner address; PRIVATE_KEY signs as operator on its behalf. let ownerOverride: string | undefined if (options.owner) { if (!ethers.isAddress(options.owner)) { console.error(chalk.red(`Invalid --owner: "${options.owner}" is not a valid 0x-prefixed Ethereum address.`)) process.exit(1) } ownerOverride = ethers.getAddress(options.owner) } const effectiveOwner = ownerOverride ?? walletAddress // Step 2: Resolve contract addresses via AddressBook console.log(chalk.blue('Resolving contract addresses from AddressBook...')) const addresses = await resolveNetworkAddresses(networkConfig) // Step 2.5: Pre-flight check that operator is registered for this owner. if (ownerOverride) { const isOp = await isOperatorOnChain(networkConfig, addresses, ownerOverride, walletAddress) if (!isOp) { console.error( chalk.red( `Operator ${walletAddress} is not authorized for owner ${ownerOverride}.\n` + `The owner must call Permissions.addOperator(${walletAddress}) first.` ) ) process.exit(1) } } // Step 3: Check ST balance and Billing balance (always against the owner — operator just submits txs) const [stBalance, billingBalance] = await Promise.all([ checkSTBalance(networkConfig, addresses, effectiveOwner), checkBillingBalance(networkConfig, addresses, effectiveOwner) ]) const confirmed = await confirmNoPlatformUpload( walletAddress, stBalance, billingBalance, addresses, networkConfig, ownerOverride ) if (!confirmed) { console.log('Upload cancelled.') process.exit(0) } // Step 4: Build processor if (!options.skipBuild) { await buildProcessor(false, options) } const processorFile = path.join(process.cwd(), 'dist/lib.js') if (!fs.existsSync(processorFile)) { console.error(chalk.red('File not found:', processorFile, "- don't use --skip-build")) process.exit(1) } const codeBuffer = fs.readFileSync(processorFile) const stat = fs.statSync(processorFile) console.log('Packed processor file size', Math.floor(stat.size / 1024) + 'K, last modified', stat.mtime) // Step 5: Upload to IPFS const ipfsPutUrl = options.ipfsPutUrl || 'https://api.sentio.xyz/v1/ipfs/add' console.log(chalk.blue(`Uploading processor to IPFS (${ipfsPutUrl})...`)) const cid = await uploadToIPFS(codeBuffer, ipfsPutUrl) console.log(chalk.green(`IPFS upload complete.`)) console.log('\t', chalk.blue('CID:'), cid) // Step 6: Collect required chain IDs let requiredChainIds = options.requiredChainId || processorConfig.requiredChainIds || [] if (requiredChainIds.length === 0) { const chainIds = new Set() if (processorConfig.contracts) { for (const contract of processorConfig.contracts) { chainIds.add(String(contract.chain)) } } if (processorConfig.networkOverrides) { for (const override of processorConfig.networkOverrides) { chainIds.add(String(override.chain)) } } requiredChainIds = Array.from(chainIds) } if (requiredChainIds.length === 0) { console.error( chalk.red( 'Error: No chain IDs found. Specify --required-chain-id or define contracts/networkOverrides in sentio.yaml.' ) ) process.exit(1) } // Step 7: Derive processor ID from project name let processorId = processorConfig.project.replace('/', '_') validateProcessorId(processorId) const sdkVersion = getSdkVersion() || 'unknown' // Step 7.5: Check if processor already exists on-chain console.log(chalk.blue('Checking if processor already exists on-chain...')) const existingProcessor = await getProcessorOnChain(networkConfig, addresses, processorId) if (existingProcessor) { const existingOwner = existingProcessor.owner.toLowerCase() const expectedOwner = effectiveOwner.toLowerCase() if (existingOwner === expectedOwner) { // Same owner — prompt to replace. In operator mode the operator is allowed to // stop+delete via the contract's isProcessorAdmin check. const ownedBy = ownerOverride ? `owned by ${ownerOverride}` : 'owned by you' console.log(chalk.yellow(`Processor "${processorId}" already exists (${ownedBy}).`)) const shouldReplace = await confirm(`Replace existing processor "${processorId}"?`) if (!shouldReplace) { console.log('Upload cancelled.') process.exit(0) } // Stop and delete existing processor before re-creating await stopProcessorOnChain(networkConfig, addresses, wallet, processorId) await deleteProcessorOnChain(networkConfig, addresses, wallet, processorId) } else { // Different owner — prompt to rename const expectedLabel = ownerOverride ? `expected owner ${ownerOverride}` : `your wallet ${wallet.address}` console.log( chalk.yellow( `Processor "${processorId}" already exists and is owned by ${existingProcessor.owner} (not ${expectedLabel}).` ) ) const randomSuffix = Math.random().toString(36).substring(2, 8) const defaultNewId = `${processorId}-${randomSuffix}` let newId: string for (;;) { const rl = readline.createInterface({ input: process.stdin, output: process.stdout }) newId = await new Promise((resolve) => rl.question(`Enter a new processor ID [${defaultNewId}]: `, (answer) => { rl.close() resolve(answer.trim() || defaultNewId) }) ) try { validateProcessorId(newId) break } catch (e: any) { console.log(chalk.red(e.message)) } } processorId = newId console.log(chalk.blue(`Using processor ID: ${processorId}`)) } } // Step 8: Create processor on-chain await createProcessorOnChain( networkConfig, addresses, wallet, processorId, cid, requiredChainIds, sdkVersion, ownerOverride ) // Step 9: Start processor on-chain await startProcessorOnChain(networkConfig, addresses, wallet, processorId) console.log() console.log(chalk.green('=== Upload Complete ===')) console.log('\t', chalk.blue('Processor ID:'), processorId) console.log('\t', chalk.blue('IPFS CID:'), cid) console.log('\t', chalk.blue('Network:'), network) const codeHash = createHash('sha256').update(codeBuffer).digest('hex') console.log('\t', chalk.blue('sha256:'), codeHash) } function parseCheckpoints( checkpoints: string[] | undefined, continueFrom: number | undefined, defaultChainId?: string ): Record | undefined { // Validate that checkpoint is only used with continue-from if (checkpoints && checkpoints.length > 0 && !continueFrom) { console.error(chalk.red('Error: --checkpoint can only be used with --continue-from')) process.exit(1) } if (!checkpoints || checkpoints.length === 0) { return undefined } const rollbackMap: Record = {} for (const checkpoint of checkpoints) { const parts = checkpoint.split(':') if (parts.length === 1) { // Format: "block_number" - use default chain if (!defaultChainId) { console.error( chalk.red( `Error: Checkpoint "${checkpoint}" does not specify chain_id, and no default chain found in configuration` ) ) process.exit(1) } const blockNumber = parseInt(parts[0], 10) if (isNaN(blockNumber)) { console.error(chalk.red(`Error: Invalid block number in checkpoint "${checkpoint}"`)) process.exit(1) } rollbackMap[defaultChainId] = blockNumber } else if (parts.length === 2) { // Format: "chain_id:block_number" const [chainId, blockStr] = parts const blockNumber = parseInt(blockStr, 10) if (isNaN(blockNumber)) { console.error(chalk.red(`Error: Invalid block number in checkpoint "${checkpoint}"`)) process.exit(1) } rollbackMap[chainId] = blockNumber } else { console.error( chalk.red( `Error: Invalid checkpoint format "${checkpoint}". Expected "chain_id:block_number" or "block_number"` ) ) process.exit(1) } } return rollbackMap } async function runUploadInternal( processorConfig: YamlProjectConfig, options: CommandOptionsType, command?: Command ) { console.log(processorConfig) const uploadAuth: Auth = {} let apiKey = ReadKey(processorConfig.host) if (options.apiKey) { apiKey = options.apiKey } if (apiKey) { uploadAuth['api-key'] = apiKey } else if (options.token) { uploadAuth.authorization = 'Bearer ' + options.token } else { const isProd = processorConfig.host === 'https://app.sentio.xyz' const cmd = isProd ? 'sentio login' : 'sentio login --host=' + processorConfig.host console.error(chalk.red('No Credential found for', processorConfig.host, '. Please run `' + cmd + '`.')) process.exit(1) } if (options.requiredChainId && options.requiredChainId.length > 0) { if (!options.sentioNetwork) { console.error(chalk.red('Error: --required-chain-id can only be used with --sentio-network')) command?.outputHelp() process.exit(1) } processorConfig.requiredChainIds = options.requiredChainId } else if (options.sentioNetwork) { const chainIds = new Set() if (processorConfig.contracts) { for (const contract of processorConfig.contracts) { chainIds.add(String(contract.chain)) } } if (processorConfig.networkOverrides) { for (const override of processorConfig.networkOverrides) { chainIds.add(String(override.chain)) } } if (chainIds.size === 0) { console.error( chalk.red( 'Error: No chain IDs found for Sentio Network deployment. When using --sentio-network, you must either specify chain IDs using --required-chain-id or define them in your sentio.yaml file under `requiredChainIds`.' ) ) command?.outputHelp() process.exit(1) } processorConfig.requiredChainIds = Array.from(chainIds) } if (!options.skipBuild) { await buildProcessor(false, options) } if (!processorConfig.project.includes('/')) { const res = await getMe(processorConfig.host, uploadAuth) const me = (await res.json()) as UserInfo processorConfig.project = `${me.username}/${processorConfig.project}` } return uploadFile(processorConfig, uploadAuth, options) } async function createProject(options: YamlProjectConfig, auth: Auth, type?: string) { const url = getApiUrl('/api/v1/projects', options.host) const [ownerName, slug] = options.project.includes('/') ? options.project.split('/') : [undefined, options.project] return fetch(url.href, { method: 'POST', headers: { ...auth }, body: JSON.stringify({ slug, ownerName, visibility: 'PRIVATE', type, sentioNetwork: !!options.sentioNetwork }) }) } async function updateVariables(projectId: string, options: YamlProjectConfig, auth: Auth) { const url = getApiUrl(`/api/v1/projects/${projectId}/variables`, options.host) return fetch(url.href, { method: 'POST', headers: { ...auth }, body: JSON.stringify({ projectId, variables: options.variables }) }) } export async function createProjectPrompt( options: YamlProjectConfig, auth: Auth, type?: string ): Promise { const create = async () => { const res = await createProject(options, auth, type) if (!res.ok) { console.error(chalk.red('Create Project Failed')) console.error(chalk.red(((await res.json()) as { message: string }).message)) return undefined } console.log(chalk.green('Project created')) return ((await res.json()) as any).id } if (options.silentOverwrite) { return await create() } const rl = readline.createInterface({ input: process.stdin, output: process.stdout }) const answer: string = await new Promise((resolve) => rl.question(`Project not found, do you want to create it and continue the uploading process? (yes/no) `, resolve) ) if (['y', 'yes'].includes(answer.toLowerCase())) { if (!type && options.type == 'action') { type = 'ACTION' } rl.close() return await create() } else if (['n', 'no'].includes(answer.toLowerCase())) { rl.close() return undefined } else { rl.close() return createProjectPrompt(options, auth, type) } } export async function confirm(question: string): Promise { const rl = readline.createInterface({ input: process.stdin, output: process.stdout }) const answer: string = await new Promise((resolve) => rl.question(`${question} (yes/no) `, resolve)) if (['y', 'yes'].includes(answer.toLowerCase())) { rl.close() return true } else if (['n', 'no'].includes(answer.toLowerCase())) { rl.close() return false } else { rl.close() return confirm(question) } } export function getGitAttributes() { let commitSha = '' let gitUrl = '' try { const options: CommonExecOptions = { stdio: [undefined, undefined, 'ignore'] } commitSha = execFileSync('git', ['rev-parse', 'HEAD'], options).toString().trim() gitUrl = execFileSync('git', ['remote', 'get-url', 'origin'], options).toString().trim() } catch (e) {} return { commitSha, gitUrl } } async function checkOrCreateProject(options: YamlProjectConfig, auth: Auth) { const [owner, slug] = options.project.split('/') const url = getApiUrl(`/api/v1/project/${owner}/${slug}`, options.host) const response = await fetch(url.href, { method: 'GET', headers: { ...auth } }) if (response.status === 404) { const projectId = await createProjectPrompt(options, auth, options.type) if (!projectId) { console.error(chalk.red('Project creation cancelled')) process.exit(1) } return projectId } else if (!response.ok) { console.error(chalk.red(`Failed to check project: ${response.status} ${response.statusText}`)) process.exit(1) } const project = ((await response.json()) as any)?.project if (options.sentioNetwork && project.sentioNetwork !== true) { console.error( chalk.red(`Project ${project?.slug} is not enabled for Sentio Network. Please create a Sentio Network Project.`) ) process.exit(1) } if (!options.sentioNetwork && project.sentioNetwork === true) { console.error( chalk.red( `Project ${project?.slug} is a Sentio Network project. Please add the --sentio-network flag when uploading.\n` + `Example: sentio upload --sentio-network testnet` ) ) process.exit(1) } return project?.id } export async function uploadFile( config: YamlProjectConfig, auth: Auth, options: CommandOptionsType ) { console.log(chalk.blue('Prepare to upload')) const continueFrom = options.continueFrom // Get default chain ID from the first contract if available const defaultChainId = config.contracts && config.contracts.length > 0 ? String(config.contracts[0].chain) : undefined // Parse checkpoint arguments into rollback map const rollbackMap = parseCheckpoints(options.checkpoint, continueFrom, defaultChainId) const processorFile = path.join(process.cwd(), 'dist/lib.js') if (!fs.existsSync(processorFile)) { console.error(chalk.red('File not existed ', processorFile, "don't use --skip-build")) process.exit(1) } const projectId = await checkOrCreateProject(config, auth) const stat = fs.statSync(processorFile) console.log('Packed processor file size', Math.floor(stat.size / 1024) + 'K, last modified', stat.mtime) if (continueFrom) { const res = await getProcessorStatus(config.host, auth, config.project) const data = (await res.json()) as { processors: { version: number versionState: string sdkVersion: string }[] } const found = data?.processors?.find( (x) => x.version == continueFrom && (x.versionState == 'ACTIVE' || x.versionState == 'PENDING') ) if (found) { const currentSdkVersion = getSdkVersion() if (found.sdkVersion.split('.')[0] != currentSdkVersion.split('.')[0]) { console.error( chalk.red( `Failed to continue from version ${continueFrom}, because it is running on SDK v${found.sdkVersion}, but current SDK is v${currentSdkVersion}` ) ) process.exit(0) } if (!config.silentOverwrite) { const confirmed = await confirm(`Continue from version ${continueFrom}?`) if (!confirmed) { process.exit(0) } } } else { const latest = Math.max(...data.processors?.map((x) => x?.version)) console.error( chalk.red( `Failed to find existed version ${continueFrom} in ${config.project}` + (latest ? `, latest is ${latest}` : '') ) ) process.exit(0) } } let triedCount = 0 const upload = async () => { config.variables = config.variables || [] for (const v of config.variables) { if (v.key || '' == '') { console.error(chalk.red("Variable key can't be empty")) return } if (v.value || '' == '') { console.error(chalk.red("Variable value can't be empty")) return } } const { commitSha, gitUrl } = getGitAttributes() console.log(chalk.blue(triedCount > 1 ? 'Retry uploading' : 'Uploading')) try { const sourceBuffer = await createSourceZip() // Read processor file const codeBuffer = fs.readFileSync(processorFile) // Create uploader const uploader = options.walrus ? new WalrusBatchUploader(config, auth) : options.sentioNetwork ? new IPFSBatchUploader(config, auth) : new DefaultBatchUploader(config, auth) // Initialize upload and handle confirmation const fileTypes: Record = { source: FileType.SOURCE, code: FileType.PROCESSOR } const initResponse = await uploader.initUpload(fileTypes) if (initResponse.warning) { console.log(chalk.yellow(initResponse.warning)) } if (initResponse.replacing_version && !config.silentOverwrite) { const confirmed = await confirm( `This will replace processor version ${initResponse.replacing_version}. Continue?` ) if (!confirmed) { process.exit(0) } } // Handle variables update if needed if (config.variables && config.variables.length > 0) { const ret = await updateVariables(projectId, config, auth) if (!ret.ok) { console.error(chalk.red('Update variables failed')) console.error(chalk.red(((await ret.json()) as { message: string }).message)) return } } // Upload files const result = await uploader.upload( { source: sourceBuffer, code: codeBuffer }, commitSha, gitUrl, config.debug || options.debug, continueFrom, config.networkOverrides, rollbackMap, initResponse ) console.log(chalk.green('Upload success: ')) const codeHash = createHash('sha256').update(codeBuffer).digest('hex') console.log('\t', chalk.blue('sha256:'), codeHash) if (commitSha) { console.log('\t', chalk.blue('Git commit SHA:'), commitSha) } console.log( '\t', chalk.blue('Check status:'), `${config.host}/${result.projectFullSlug}/datasource/${result.processorId}` ) console.log('\t', chalk.blue('Version:'), result.version) } catch (e) { throw e } } let error: Error const tryUploading = async () => { if (triedCount++ >= 5) { console.error(error) return } try { await upload() } catch (e: any) { // Transient network errors while PUT-ing the packed processor to the signed // storage URL: node-fetch surfaces these as `FetchError` with `type: 'system'` // and the underlying socket errno in `code`. Retry all of them, not just EPIPE // (ECONNRESET / "socket hang up" is the most common one against GCS). const RETRYABLE_CODES = ['EPIPE', 'ECONNRESET', 'ETIMEDOUT', 'ECONNREFUSED', 'EAI_AGAIN'] if (e?.constructor.name === 'FetchError' && e.type === 'system' && RETRYABLE_CODES.includes(e.code)) { error = e await new Promise((resolve) => setTimeout(resolve, 1000)) await tryUploading() } else { console.error(e) } } } await tryUploading() } async function getProcessorStatus(host: string, auth: Auth, projectSlug: string) { const url = getApiUrl(`/api/v1/processors/${projectSlug}/status?version=ALL`, host) return fetch(url.href, { headers: { ...auth } }) } async function getMe(host: string, auth: Auth) { const url = getApiUrl(`/api/v1/users`, host) return fetch(url.href, { headers: { ...auth } }) } async function createSourceZip() { const zip = new JSZip() fs.readdirSync('.').forEach((p) => { if ( fs.lstatSync(p).isFile() && (['package.json', 'tsconfig.json', 'sentio.yaml'].includes(p) || p.endsWith('.graphql')) ) { zip.file(p, fs.readFileSync(p)) } }) if (fs.existsSync('abis') && fs.lstatSync('abis').isDirectory()) { fs.readdirSync('abis').forEach((p) => { const item = path.join('abis', p) if (fs.lstatSync(item).isDirectory()) { fs.readdirSync(item).forEach((pp) => { if (pp.endsWith('.json')) { zip.file(`abis/${p}/${pp}`, fs.readFileSync(path.join(item, pp))) } }) } else if (p.endsWith('.json')) { zip.file(`abis/${p}`, fs.readFileSync(item)) } }) } return await zip.generateAsync({ type: 'nodebuffer' }) }