import { ndn, verifyBundle } from 'buckyos/node' import { ToolError, UsageError } from '../core/errors.ts' import { getHost } from '../runtime/host.ts' import { digestFile, type FileDigest, fileReadableStream, listTarGzStream, openZip, readTarGz, readZipEntry, sha256Bytes, verifyZipEntry, writeStoredZip, zipEntryStream, } from './pikg_archive.ts' const basename = (path: string) => getHost().path.basename(path) const join = (...parts: string[]) => getHost().path.join(...parts) const resolve = (...parts: string[]) => getHost().path.resolve(...parts) export const PACKAGE_META_SCHEMA = 'buckyos.pikg.package-meta.v1' export const DIST_MANIFEST_NAME = '.buckyos-pikg-dist.json' export const APPDOC_ENTRY = 'APPDOC.json' export const APPDOC_JWT_ENTRY = 'APPDOC.jwt' export const PACKAGE_META_ENTRY = 'PACKAGE_META.json' export const MAX_APPDOC_BYTES = 1024 * 1024 export const MAX_METADATA_ENTRY_BYTES = 8 * 1024 * 1024 export const MAX_METADATA_TOTAL_BYTES = 64 * 1024 * 1024 const SAFE_SUBPACKAGE = /^[A-Za-z0-9._-]+$/ const SAFE_BNS_LABEL = /^[a-z0-9](?:[a-z0-9-]*[a-z0-9])?$/ const SHA256_DIGEST = /^sha256:([0-9a-fA-F]{64})$/ const OBJECT_ID = /^(appdoc|pkg):[0-9a-f]{64}$/ export interface ContentIndexEntry { sub_pkg_name: string path: string format: 'tar.gz' size: number digest: string } export interface PackageMetaFile { '@schema': typeof PACKAGE_META_SCHEMA app_doc_id: string package_objects: Record> content_index: Record } export interface ValidatedSubpackage { key: string selector: Record | null required: boolean pkg_id: string pkg_objid: string payload_path: string payload_size: number payload_digest: string docker_image_name?: string docker_image_digest?: string } export interface ValidatedPikg { appDoc: Record appDocObjectId: string packageMeta: PackageMetaFile subpackages: ValidatedSubpackage[] hasJsonAppDoc: boolean hasSignedAppDoc: boolean canonicalMatch: boolean } export interface PikgInspectionResult { schema_version: 1 /** An `aiws` package: its bundle checked like the AI Workspace library does. */ aiws?: Record /** An `ext_tools` package: what its `ext-manifest.json` says, checked against every payload. */ ext_tools?: Record protocol: string pikg_path: string size: number pikg_digest: string valid: true app: { did: string app_id: string name?: string app_type: string version: string owner: string app_doc_object_id: string } app_doc_form: 'unsigned' | 'signed' | 'both' canonical_match: boolean subpackages: Array> offline_content_validation: 'passed' signature_validation: 'not-present' | 'not-resolvable-offline' publication_validation: 'not-checked' } export function validateSubpackageName(name: string): void { if (!SAFE_SUBPACKAGE.test(name) || name === '.' || name === '..') { throw new UsageError('INVALID_SUBPACKAGE_NAME', `invalid subpackage name: ${name}`) } } export function normalizeArch(value: string): string { switch (value.trim().toLowerCase()) { case 'amd64': case 'x86_64': case 'x64': return 'x86_64' case 'arm64': case 'aarch64': return 'aarch64' default: return value.trim().toLowerCase() } } export function normalizeOs(value: string): string { switch (value.trim().toLowerCase()) { case 'darwin': case 'apple': case 'macos': case 'osx': return 'macos' case 'win': case 'win32': case 'windows': return 'windows' default: return value.trim().toLowerCase() } } export function derivedSelector(key: string): Record | undefined { const selectors: Record> = { amd64_docker_image: { os: 'linux', arch: 'x86_64' }, aarch64_docker_image: { os: 'linux', arch: 'aarch64' }, amd64_linux_app: { os: 'linux', arch: 'x86_64' }, aarch64_linux_app: { os: 'linux', arch: 'aarch64' }, amd64_win_app: { os: 'windows', arch: 'x86_64' }, aarch64_win_app: { os: 'windows', arch: 'aarch64' }, amd64_apple_app: { os: 'macos', arch: 'x86_64' }, aarch64_apple_app: { os: 'macos', arch: 'aarch64' }, web: {}, agent: {}, agent_skills: {}, aiws: {}, script: {}, } return selectors[key] ?? extToolsPlatformOfKey(key)?.selector } export function canonicalSelector( value: unknown, label: string, ): Record | undefined { if (value === undefined) return undefined const selector = expectObject(value, label) rejectUnknown(selector, ['os', 'arch', 'min_kernel_version'], label) const output: Record = {} for (const key of ['os', 'arch', 'min_kernel_version']) { if (selector[key] !== undefined) { output[key] = expectNonEmptyString(selector[key], `${label}.${key}`) } } if (output.os) output.os = normalizeOs(output.os) if (output.arch) output.arch = normalizeArch(output.arch) return output } export function assertSelectorCompatible( key: string, selector: Record | undefined, label: string, ): void { const derived = derivedSelector(key) if (!derived || !selector) return for (const field of ['os', 'arch']) { if (derived[field] !== undefined && selector[field] !== derived[field]) { throw new UsageError( 'SELECTOR_CONFLICT', `${label}.${field} conflicts with the selector derived from ${key}`, ) } } } export function deriveAppNamespace( appDid: string, ): string { return appIdFromDid(appDid) } export function appIdFromDid(appDid: string): string { const app = parseDid(appDid, 'did') if ( app.id.includes('#') || app.id.includes('/') || app.id.includes('%') || app.id.includes(':') ) { throw new UsageError( 'INVALID_APP_ID', 'App DID must use hostname form without path, fragment, port, or encoding', ) } const labels = app.id.split('.') if (labels.some((label) => !SAFE_BNS_LABEL.test(label))) { throw new UsageError('INVALID_APP_ID', 'App DID must contain canonical lowercase DNS labels') } if (app.method === 'web') { if (labels.length >= 3 && labels.at(-1) === 'did') { throw new UsageError( 'INVALID_APP_ID', 'did:web hostname conflicts with the reserved non-Web .did form', ) } return app.id } return `${app.id}.${app.method}.did` } export function createPackageMeta( packageName: string, version: string, author: string, owner: string, payload: FileDigest, timestamp: number, ): { value: Record; objectId: string } { const content = ndn.ChunkId.fromMix256Result( payload.size, ndn.hexToBytes(payload.sha256), ).toString() const value: Record = { name: packageName, author, owner, create_time: timestamp, last_update_time: timestamp, size: payload.size, content, version, } const objectId = ndn.buildNamedObjectByJson(ndn.OBJ_TYPE_PKG, value)[0].toString() return { value, objectId } } export function validatePermissions(value: unknown, label: string): unknown[] { if (!Array.isArray(value)) throw invalid('schema', `${label} must be an array`) return value.map((raw, index) => { const item = expectObject(raw, `${label}[${index}]`) assertKnownFields(item, ['scope_path', 'required', 'actions', 'exp'], `${label}[${index}]`) expectNonEmptyString(item.scope_path, `${label}[${index}].scope_path`) if (typeof item.required !== 'boolean') { throw invalid('schema', `${label}[${index}].required must be boolean`) } if ( item.actions !== undefined && (!Array.isArray(item.actions) || item.actions.some((action) => typeof action !== 'string')) ) { throw invalid('schema', `${label}[${index}].actions must be a string array`) } if ( item.exp !== null && item.exp !== undefined && (typeof item.exp !== 'number' || !Number.isInteger(item.exp) || item.exp < 0 || item.exp > 0xffffffff) ) { throw invalid('schema', `${label}[${index}].exp must be null or a uint32`) } return raw }) } const BUCKET_POLICIES = ['documents', 'app_data', 'collected', 'ambient', 'temp', 'cache'] /** `buckets` / `storage_quota_mib` (BuckyOS 应用对象桶设计 §4.3, §4.5.3, D17). */ function validateBuckets(config: Record, label: string): void { if (config.storage_quota_mib !== undefined) { expectUint(config.storage_quota_mib, `${label}.storage_quota_mib`, Number.MAX_SAFE_INTEGER) } let needsAppPool = false for (const [name, raw] of Object.entries(expectObject(config.buckets ?? {}, `${label}.buckets`))) { const at = `${label}.buckets.${name}` if (!/^[A-Za-z0-9._-]{1,128}$/.test(name) || name === '.' || name === '..') { throw invalid('schema', `${at} is not a valid bucket name`) } const bucket = expectObject(raw, at) assertKnownFields( bucket, ['kind', 'data_policy', 'scope', 'versioning', 'quota_mib', 'max_objects', 'default_ttl_days', 'lifecycle'], at, ) if (!['data', 'object'].includes(String(bucket.kind))) throw invalid('schema', `${at}.kind is data or object`) const policy = String(bucket.data_policy) if (!BUCKET_POLICIES.includes(policy)) throw invalid('schema', `${at}.data_policy is invalid`) if (bucket.scope !== undefined && !['user', 'zone'].includes(String(bucket.scope))) { throw invalid('schema', `${at}.scope is user or zone`) } // The AppDoc carries the declaration without defaults, as BuckyOS serializes it. if (bucket.scope === 'user') delete bucket.scope optionalBoolean(bucket.versioning, `${at}.versioning`) const userOwned = policy === 'documents' || policy === 'collected' if (bucket.quota_mib !== undefined) { expectUint(bucket.quota_mib, `${at}.quota_mib`, Number.MAX_SAFE_INTEGER) if (userOwned) throw invalid('schema', `${at}: a ${policy} bucket uses the user's quota`) } if (bucket.max_objects !== undefined) expectUint(bucket.max_objects, `${at}.max_objects`, Number.MAX_SAFE_INTEGER) if (bucket.versioning === true && !['documents', 'app_data', 'collected'].includes(policy)) { throw invalid('schema', `${at}: a ${policy} bucket cannot be versioned`) } if (bucket.default_ttl_days !== undefined) { expectUint(bucket.default_ttl_days, `${at}.default_ttl_days`, 0xffffffff) if (!['ambient', 'temp', 'cache'].includes(policy)) { throw invalid('schema', `${at}: a ${policy} bucket cannot declare default_ttl_days`) } } if (policy === 'temp') { const days = bucket.default_ttl_days if (typeof days !== 'number' || days < 1 || days > 30) { throw invalid('schema', `${at}: a temp bucket declares default_ttl_days 1..30`) } } if (bucket.lifecycle !== undefined) { const lifecycle = bucket.lifecycle if (!Array.isArray(lifecycle)) throw invalid('schema', `${at}.lifecycle must be an array`) if (lifecycle.length === 0) delete bucket.lifecycle for (const [index, rule] of lifecycle.entries()) { const kind = String(expectObject(rule, `${at}.lifecycle[${index}]`).rule) if (!['expire', 'noncurrent_expire', 'expired_delete_marker'].includes(kind)) { throw invalid('schema', `${at}.lifecycle[${index}].rule is invalid`) } if (policy === 'temp' || (userOwned && kind !== 'noncurrent_expire')) { throw invalid('schema', `${at}: a ${policy} bucket does not accept the ${kind} rule`) } } } needsAppPool ||= policy === 'app_data' || policy === 'temp' } if (needsAppPool && config.storage_quota_mib === undefined) { throw invalid('schema', `${label}.storage_quota_mib is required when app_data or temp buckets are declared`) } } export function validateServiceConfigTips( value: unknown, label: string, ): Record { const config = expectObject(value, label) for ( const field of [ 'service_endpoints', 'data_mount_points', 'local_cache_mount_points', 'external_mount_points', 'rdb_instances', 'bash_envs', 'runtime_caps', ] ) { if (config[field] !== undefined) expectObject(config[field], `${label}.${field}`) } for ( const [name, raw] of Object.entries( expectObject(config.service_endpoints ?? {}, `${label}.service_endpoints`), ) ) { const endpoint = expectObject(raw, `${label}.service_endpoints.${name}`) assertKnownFields( endpoint, ['protocol', 'inner_port', 'required', 'description', 'expose'], `${label}.service_endpoints.${name}`, ) if (!['http', 'https', 'tcp', 'udp'].includes(String(endpoint.protocol))) { throw invalid('schema', `${label}.service_endpoints.${name}.protocol is invalid`) } expectUint(endpoint.inner_port, `${label}.service_endpoints.${name}.inner_port`, 0xffff) optionalBoolean(endpoint.required, `${label}.service_endpoints.${name}.required`) validateStringMap( endpoint.description ?? {}, `${label}.service_endpoints.${name}.description`, ) if (endpoint.expose !== undefined && endpoint.expose !== null) { const expose = expectObject(endpoint.expose, `${label}.service_endpoints.${name}.expose`) assertKnownFields( expose, ['route', 'scope', 'allow_guest'], `${label}.service_endpoints.${name}.expose`, ) const route = expectObject(expose.route, `${label}.service_endpoints.${name}.expose.route`) if (route.type === 'web') { assertKnownFields(route, ['type'], `${label}.service_endpoints.${name}.expose.route`) } else if (route.type === 'port') { assertKnownFields( route, ['type', 'preferred_port'], `${label}.service_endpoints.${name}.expose.route`, ) if (route.preferred_port !== undefined) { expectUint( route.preferred_port, `${label}.service_endpoints.${name}.expose.route.preferred_port`, 0xffff, ) } } else { throw invalid('schema', `${label}.service_endpoints.${name}.expose.route.type is invalid`) } optionalString(expose.scope, `${label}.service_endpoints.${name}.expose.scope`) optionalBoolean( expose.allow_guest, `${label}.service_endpoints.${name}.expose.allow_guest`, ) } } for ( const field of [ 'data_mount_points', 'local_cache_mount_points', 'external_mount_points', ] ) { for ( const [name, raw] of Object.entries(expectObject(config[field] ?? {}, `${label}.${field}`)) ) { if (raw === null) continue const mount = expectObject(raw, `${label}.${field}.${name}`) assertKnownFields( mount, ['mount_point_name', 'access', 'reason'], `${label}.${field}.${name}`, ) expectNonEmptyString(mount.mount_point_name, `${label}.${field}.${name}.mount_point_name`) expectNonEmptyString(mount.access, `${label}.${field}.${name}.access`) validateStringMap(mount.reason, `${label}.${field}.${name}.reason`) } } for ( const [name, raw] of Object.entries( expectObject(config.rdb_instances ?? {}, `${label}.rdb_instances`), ) ) { const database = expectObject(raw, `${label}.rdb_instances.${name}`) assertKnownFields( database, ['backend', 'version', 'schema', 'connection'], `${label}.rdb_instances.${name}`, ) if (!['sqlite', 'postgres'].includes(String(database.backend))) { throw invalid('schema', `${label}.rdb_instances.${name}.backend is invalid`) } if (database.version !== undefined) { expectUint( database.version, `${label}.rdb_instances.${name}.version`, Number.MAX_SAFE_INTEGER, ) } const schemas = validateStringMap( database.schema ?? {}, `${label}.rdb_instances.${name}.schema`, ) for (const backend of Object.keys(schemas)) { if (!['sqlite', 'postgres'].includes(backend)) { throw invalid('schema', `${label}.rdb_instances.${name}.schema.${backend} is invalid`) } } optionalString(database.connection, `${label}.rdb_instances.${name}.connection`) } validateBuckets(config, label) if (config.instance_volume !== undefined) { const volume = expectObject(config.instance_volume, `${label}.instance_volume`) assertKnownFields( volume, ['mode', 'quota_mib', 'ephemeral_contents'], `${label}.instance_volume`, ) if ( volume.mode !== undefined && !['required', 'optional', 'disabled'].includes(String(volume.mode)) ) { throw invalid('schema', `${label}.instance_volume.mode is invalid`) } if (volume.quota_mib !== undefined) { expectUint(volume.quota_mib, `${label}.instance_volume.quota_mib`, Number.MAX_SAFE_INTEGER) } if ( volume.ephemeral_contents !== undefined && (!Array.isArray(volume.ephemeral_contents) || volume.ephemeral_contents.some((item) => typeof item !== 'string')) ) { throw invalid('schema', `${label}.instance_volume.ephemeral_contents must be a string array`) } } for ( const [name, raw] of Object.entries(expectObject(config.bash_envs ?? {}, `${label}.bash_envs`)) ) { const environment = expectObject(raw, `${label}.bash_envs.${name}`) assertKnownFields(environment, ['required', 'description'], `${label}.bash_envs.${name}`) if (typeof environment.required !== 'boolean') { throw invalid('schema', `${label}.bash_envs.${name}.required must be boolean`) } validateStringMap(environment.description ?? {}, `${label}.bash_envs.${name}.description`) } validateStringMap(config.runtime_caps ?? {}, `${label}.runtime_caps`) optionalString(config.container_param, `${label}.container_param`) optionalString(config.start_param, `${label}.start_param`) return config } export function appDocObjectId(value: Record): string { return ndn.buildNamedObjectByJson('appdoc', value)[0].toString() } /** The sub-package of an `aiws` package: the AIWS bundle (aiws-manifest.json plus controls// and, from * manifest_version 3, Workspace templates under templates//). */ export const AIWS_SUB_PACKAGE = 'aiws' const MAX_AIWS_PAYLOAD_BYTES = 256 * 1024 * 1024 /** Check the bundle inside an `aiws` payload the way the AI Workspace library does (digests, component.json, the * templates' shape, snapshot, digest and required controls, no file outside a control, a template or the manifest). */ export async function verifyAiwsPayload(tarGz: Uint8Array): Promise> { const files = new Map() for (const entry of await readTarGz(tarGz, MAX_AIWS_PAYLOAD_BYTES)) files.set(entry.path, entry.bytes) let checked try { checked = await verifyBundle(files) } catch (error) { throw invalid('aiws', `aiws bundle: ${error instanceof Error ? error.message : String(error)}`) } const errors = checked.diagnostics.filter((d: { severity: string }) => d.severity === 'error') if (errors.length > 0) { throw invalid('aiws', `aiws bundle: ${errors.map((d: { file: string; message: string }) => `${d.file}: ${d.message}`).join('; ')}`) } return { manifest_version: checked.manifest.manifest_version, controls: checked.manifest.controls.map((c: Record) => ({ ident: c.ident, kind: c.kind, version: c.version, digest: c.digest })), templates: checked.manifest.workspace_templates.map((t: Record) => ({ id: t.id, title: t.title, digest: t.digest, requires: t.requires })), } } /** Extension tool packages (`app_type = ext_tools`): tools every Agent of a machine shares, one payload per platform. */ export const EXT_TOOLS_APP_TYPE = 'ext_tools' export const EXT_TOOLS_MANIFEST_FILE = 'ext-manifest.json' export const EXT_TOOLS_MANIFEST_SCHEMA = 'buckyos.ext_tools/1' const MAX_EXT_TOOLS_PAYLOAD_BYTES = 64 * 1024 * 1024 * 1024 /** The standard layer's commands (Rust: `EXT_TOOLS_RESERVED_COMMANDS`, `baseline::STANDARD_TOOLS`): `tools/bin` comes first * on the PATH, so a package providing one would shadow an Agent's identity and control commands. */ export const EXT_TOOLS_RESERVED_COMMANDS = [ 'agent-session', 'agent_tool', 'ai_provider', 'ai_provider_test', 'ai_quota', 'aicc-tool', 'ask_user', 'buckyos', 'caption_image', 'detect_image', 'edit_image', 'edit_video', 'enhance_audio', 'extend_video', 'gen_image', 'gen_music', 'gen_video', 'import_office', 'import_table', 'inpaint_image', 'llm_understand_media', 'materialize_resource', 'ocr_image', 'reflow', 'remove_bg', 'resume_aicc_task', 'segment_image', 'speech_to_text', 'text_to_speech', 'upscale_image', 'upscale_video', 'xagent', ] const MAX_EXT_TOOLS_LAUNCHER_BYTES = 1024 * 1024 const EXT_TOOLS_ID = /^[a-z0-9](?:[a-z0-9-]*[a-z0-9])?$/ const EXT_TOOLS_COMMAND = /^[A-Za-z0-9_+-][A-Za-z0-9._+-]*$/ const EXT_TOOLS_WINDOWS_SUFFIXES = ['', '.exe', '.cmd', '.bat', '.ps1'] // A launcher resolves its tree from its own location: these never exist where a package lands. const BUILD_MACHINE_PATH = /(?:^|[\s"'=:(])(\/home\/|\/Users\/|\/root\/|\/tmp\/|\/var\/folders\/|\/opt\/buckyos\/|[A-Za-z]:\\\\?Users\\)/ export interface ExtToolsPlatform { name: string key: string selector: Record } export const EXT_TOOLS_PLATFORMS: ExtToolsPlatform[] = [ { name: 'linux-x86_64', key: 'ext_tools_linux_x86_64', selector: { os: 'linux', arch: 'x86_64' } }, { name: 'linux-aarch64', key: 'ext_tools_linux_aarch64', selector: { os: 'linux', arch: 'aarch64' } }, { name: 'darwin-arm64', key: 'ext_tools_darwin_arm64', selector: { os: 'macos', arch: 'aarch64' } }, { name: 'darwin-x86_64', key: 'ext_tools_darwin_x86_64', selector: { os: 'macos', arch: 'x86_64' } }, { name: 'windows-x86_64', key: 'ext_tools_windows_x86_64', selector: { os: 'windows', arch: 'x86_64' } }, ] export function extToolsPlatformOfKey(key: string): ExtToolsPlatform | undefined { return EXT_TOOLS_PLATFORMS.find((platform) => platform.key === key) } /** Where one platform gets an online item: the upstream's own archive, and what to do with it once unpacked. */ export interface ExtToolsOnlineSource { url: string sha256: string size_bytes: number format: string into?: string rename?: Array<[string, string]> overlay?: string } /** Something a package only declares: a node downloads it from the upstream to `tools/online//` once the package is * installed. A platform without an entry has no such item. */ export interface ExtToolsOnlineItem { id: string version: string desc?: string license?: string auto?: boolean platforms: Record } export interface ExtToolsManifest { schema: string id: string version: string requires: Array<{ id: string; version: string }> conflicts: string[] provides: Array<{ command: string; version_probe: string; smoke?: string; needs?: string[] }> platforms: string[] size_bytes?: Record license?: string desc?: string online?: ExtToolsOnlineItem[] } const EXT_TOOLS_VERSION = /^[A-Za-z0-9._+-]+$/ export const EXT_TOOLS_ONLINE_FORMATS = ['zip', 'tar.gz'] function unknownKeys(value: Record, allowed: string[]): string[] { return Object.keys(value).filter((key) => !allowed.includes(key)) } function isHttpsUrl(value: unknown): boolean { if (typeof value !== 'string') return false try { const url = new URL(value) return url.protocol === 'https:' && url.hostname !== '' } catch { return false } } /** What is wrong with one platform's source of online item `item` (Rust: `ExtToolsOnlineSource::problem`). */ function extToolsOnlineSourceErrors(item: string, raw: unknown): string[] { if (!raw || typeof raw !== 'object' || Array.isArray(raw)) return ['must be an object'] const source = raw as Record const errors = unknownKeys(source, ['url', 'sha256', 'size_bytes', 'format', 'into', 'rename', 'overlay']).map((key) => `${key} is not a field`) if (!isHttpsUrl(source.url)) errors.push(`url ${JSON.stringify(source.url)} must be an https address`) if (typeof source.sha256 !== 'string' || !/^[0-9a-fA-F]{64}$/.test(source.sha256)) errors.push('sha256 must be 64 hex digits') if (!Number.isSafeInteger(source.size_bytes) || (source.size_bytes as number) <= 0) errors.push('size_bytes must be the size of the archive') if (typeof source.format !== 'string' || !EXT_TOOLS_ONLINE_FORMATS.includes(source.format)) errors.push(`format ${JSON.stringify(source.format)} must be zip or tar.gz`) if (source.into !== undefined && (typeof source.into !== 'string' || !isSafeRelativePath(source.into))) errors.push('into must be a relative path inside the item') const pairs = source.rename === undefined ? [] : source.rename const renamed = Array.isArray(pairs) && pairs.every((pair) => Array.isArray(pair) && pair.length === 2 && pair.every((path) => typeof path === 'string' && isSafeRelativePath(path)) && pair[0] !== pair[1] ) if (!renamed) errors.push('rename must be [from, to] pairs of relative paths inside the item') const root = `online/${item}` const overlay = source.overlay if (overlay !== undefined && !(typeof overlay === 'string' && isSafeRelativePath(overlay) && (overlay === root || overlay.startsWith(`${root}/`)))) { errors.push(`overlay must be ${root} or a directory below it`) } return errors } /** The `online` list of a manifest (Rust: `ExtToolsManifest::validate`), and the ids it declares. */ function extToolsOnlineErrors(manifest: Record): { errors: string[]; items: Set } { const errors: string[] = [] const items = new Set() if (manifest.online === undefined) return { errors, items } if (!Array.isArray(manifest.online)) return { errors: ['online must be a list of items'], items } const platforms = Array.isArray(manifest.platforms) ? manifest.platforms : [] for (const raw of manifest.online) { const item = (raw ?? {}) as Record const id = item.id if (typeof id !== 'string' || !EXT_TOOLS_ID.test(id)) { errors.push(`online: id ${JSON.stringify(id)} must be lowercase letters, digits and -`) continue } if (items.has(id)) errors.push(`online: ${id} is listed twice`) items.add(id) for (const key of unknownKeys(item, ['id', 'version', 'desc', 'license', 'auto', 'platforms'])) errors.push(`online ${id}: ${key} is not a field`) if (typeof item.version !== 'string' || !EXT_TOOLS_VERSION.test(item.version)) errors.push(`online ${id}: version must be letters, digits and ._+-`) for (const text of ['desc', 'license']) if (item[text] !== undefined && typeof item[text] !== 'string') errors.push(`online ${id}: ${text} must be a string`) if (item.auto !== undefined && typeof item.auto !== 'boolean') errors.push(`online ${id}: auto must be true or false`) const sources = item.platforms if (!sources || typeof sources !== 'object' || Array.isArray(sources) || Object.keys(sources).length === 0) { errors.push(`online ${id}: platforms must not be empty`) continue } for (const [platform, source] of Object.entries(sources)) { if (!platforms.includes(platform)) errors.push(`online ${id}: ${platform} is not a platform of the package`) for (const problem of extToolsOnlineSourceErrors(id, source)) errors.push(`online ${id} on ${platform}: ${problem}`) } } return { errors, items } } /** The manifest's own rules (`buckyos.ext_tools/1`, doc/pikg/format.md §5), each problem one line. */ export function extToolsManifestErrors(value: unknown): string[] { if (!value || typeof value !== 'object' || Array.isArray(value)) return ['must be a JSON object'] const manifest = value as Record const errors: string[] = [] const allowed = ['schema', 'id', 'version', 'requires', 'conflicts', 'provides', 'platforms', 'size_bytes', 'license', 'desc', 'online'] for (const key of Object.keys(manifest)) if (!allowed.includes(key)) errors.push(`${key} is not a field of the manifest`) if (manifest.schema !== EXT_TOOLS_MANIFEST_SCHEMA) errors.push(`schema must be ${EXT_TOOLS_MANIFEST_SCHEMA}`) if (typeof manifest.id !== 'string' || !EXT_TOOLS_ID.test(manifest.id)) errors.push('id must be lowercase letters, digits and -') if (typeof manifest.version !== 'string' || !EXT_TOOLS_VERSION.test(manifest.version)) errors.push('version must be letters, digits and ._+-') if (!Array.isArray(manifest.provides)) errors.push('provides must be a list') else if (manifest.provides.length === 0 && !(Array.isArray(manifest.online) && manifest.online.length > 0)) { errors.push('a package provides commands or declares online items') } const online = extToolsOnlineErrors(manifest) errors.push(...online.errors) const commands = new Set() for (const raw of Array.isArray(manifest.provides) ? manifest.provides : []) { const entry = (raw ?? {}) as Record const command = entry.command if (typeof command !== 'string' || !EXT_TOOLS_COMMAND.test(command)) { errors.push(`provides: ${JSON.stringify(command)} is not a command name`) continue } if (commands.has(command)) errors.push(`provides: command ${command} is listed twice`) if (EXT_TOOLS_RESERVED_COMMANDS.includes(command)) errors.push(`provides: ${command} is a standard command and cannot be provided by a package`) commands.add(command) for (const key of unknownKeys(entry, ['command', 'version_probe', 'smoke', 'needs'])) errors.push(`provides: ${key} is not a field of ${command}`) if (typeof entry.version_probe !== 'string' || !entry.version_probe.trim()) errors.push(`provides: ${command} has no version_probe`) const smoke = entry.smoke if (smoke !== undefined && (typeof smoke !== 'string' || !isSafeRelativePath(smoke))) { errors.push(`provides: the smoke of ${command} must be a relative path inside the payload`) } const needs = entry.needs === undefined ? [] : entry.needs if (!Array.isArray(needs) || !needs.every((need) => typeof need === 'string')) { errors.push(`provides: the needs of ${command} must list online items by id`) continue } for (const need of needs as string[]) if (!online.items.has(need)) errors.push(`provides: ${command} needs ${need}, which is not an online item of the package`) if (new Set(needs).size !== needs.length) errors.push(`provides: ${command} needs an online item twice`) } if (!Array.isArray(manifest.platforms) || manifest.platforms.length === 0) errors.push('platforms must not be empty') for (const platform of Array.isArray(manifest.platforms) ? manifest.platforms : []) { if (!EXT_TOOLS_PLATFORMS.some((known) => known.name === platform)) errors.push(`unknown platform ${JSON.stringify(platform)}`) } for (const raw of Array.isArray(manifest.requires) ? manifest.requires : manifest.requires === undefined ? [] : [null]) { const required = (raw ?? {}) as Record if (typeof required.id !== 'string' || !EXT_TOOLS_ID.test(required.id) || required.id === manifest.id || typeof required.version !== 'string') { errors.push(`requires: ${JSON.stringify(raw)} is not another package's {id, version}`) } } if (manifest.conflicts !== undefined && !(Array.isArray(manifest.conflicts) && manifest.conflicts.every((id) => typeof id === 'string' && id !== manifest.id))) { errors.push('conflicts must list other packages by id') } return errors } function isSafeRelativePath(path: string): boolean { return !path.includes('\\') && !path.startsWith('/') && path.split('/').every((part) => part && part !== '.' && part !== '..') } export function parseExtToolsManifest(bytes: Uint8Array): ExtToolsManifest { let value: unknown try { value = JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(bytes)) } catch (error) { throw invalid('ext_tools', `${EXT_TOOLS_MANIFEST_FILE} is not valid JSON: ${error instanceof Error ? error.message : String(error)}`) } const errors = extToolsManifestErrors(value) if (errors.length > 0) throw invalid('ext_tools', `${EXT_TOOLS_MANIFEST_FILE}: ${errors.join('; ')}`) const manifest = value as ExtToolsManifest manifest.requires ??= [] manifest.conflicts ??= [] return manifest } /** A payload's tree, from a directory or from its tar: the regular files, as `pikg build` would archive them. */ export interface ExtToolsPayloadTree { files: Map directories: Set /** The contents of the small files under `bin/`. */ launchers: Map } /** What is wrong with a payload of `platform`: its layout, a launcher for every command it provides and for nothing * else, the smoke scripts, and launchers that only name paths of the machine that built them. */ export function extToolsPayloadErrors(manifest: ExtToolsManifest, platform: string, tree: ExtToolsPayloadTree, buildPaths: string[] = []): string[] { const errors: string[] = [] const windows = platform.startsWith('windows-') const suffixes = windows ? EXT_TOOLS_WINDOWS_SUFFIXES : [''] const online = manifest.online ?? [] for (const path of [...tree.files.keys(), ...tree.directories]) { const [top, second] = path.split('/') const isTop = !path.includes('/') const known = top === EXT_TOOLS_MANIFEST_FILE && isTop || top === 'bin' || top === 'smoke' || top === 'store' || top === 'online' if (!known) errors.push(`${path}: a package only holds ${EXT_TOOLS_MANIFEST_FILE}, bin/, smoke/, store/${manifest.id}/ and online//`) if (top === 'store' && second !== undefined && second !== manifest.id) errors.push(`${path}: a package only writes store/${manifest.id}/`) if (top === 'online' && second !== undefined && !online.some((item) => item.id === second)) errors.push(`${path}: not an online item of the package`) } for (const item of online) { const overlay = item.platforms[platform]?.overlay const present = overlay !== undefined && (tree.directories.has(overlay) || [...tree.files.keys()].some((path) => path.startsWith(`${overlay}/`))) if (overlay !== undefined && !present) errors.push(`${overlay}: the overlay of online item ${item.id} is missing`) } const declared = new Set(manifest.provides.map((provided) => provided.command)) for (const provided of manifest.provides) { const launcher = suffixes.map((suffix) => tree.files.get(`bin/${provided.command}${suffix}`)).find(Boolean) if (!launcher) errors.push(`bin/${provided.command}: the launcher of a provided command is missing`) else if (!windows && !launcher.executable) errors.push(`bin/${provided.command}: the launcher is not executable`) const smoke = provided.smoke && windows ? provided.smoke.replace(/\.[^./]+$/, '.ps1') : provided.smoke if (smoke && !tree.files.has(smoke)) errors.push(`${smoke}: the smoke script of ${provided.command} is missing`) } for (const path of tree.files.keys()) { if (!path.startsWith('bin/')) continue const name = path.slice('bin/'.length) const command = suffixes.map((suffix) => suffix && name.endsWith(suffix) ? name.slice(0, -suffix.length) : name).find((candidate) => declared.has(candidate)) if (!command) errors.push(`${path}: not in provides, so nothing knows this package provides it`) } const decoder = new TextDecoder() for (const [path, bytes] of tree.launchers) { const text = decoder.decode(bytes) const named = buildPaths.find((dir) => text.includes(dir)) if (named) errors.push(`${path}: names ${named} on the machine that built it`) else if (BUILD_MACHINE_PATH.test(text)) errors.push(`${path}: names an absolute path of the machine that built it (${BUILD_MACHINE_PATH.exec(text)?.[1]})`) } return errors } function assertExtToolsOf(appDoc: Record, manifest: ExtToolsManifest, platform: string, platforms: string[]): void { if (manifest.id !== appDoc.name) throw invalid('ext_tools', `${EXT_TOOLS_MANIFEST_FILE} id ${manifest.id} is not the App name ${appDoc.name}`) if (manifest.version !== appDoc.version) throw invalid('ext_tools', `${EXT_TOOLS_MANIFEST_FILE} version ${manifest.version} is not the App version ${appDoc.version}`) if (!manifest.platforms.includes(platform)) throw invalid('ext_tools', `${EXT_TOOLS_MANIFEST_FILE} does not list the payload's platform ${platform}`) const missing = manifest.platforms.filter((name) => !platforms.includes(name)) if (missing.length > 0) throw invalid('ext_tools', `${EXT_TOOLS_MANIFEST_FILE} lists platforms without a payload: ${missing.join(', ')}`) } /** Check one payload of an `ext_tools` package (a tar.gz stream, never held whole) and return its manifest. */ async function verifyExtToolsPayload( gzip: ReadableStream, key: string, appDoc: Record, ): Promise { const platform = extToolsPlatformOfKey(key) if (!platform) throw invalid('appdoc', `an ext_tools package has no sub package ${key}`) const listing = await listTarGzStream( gzip, MAX_EXT_TOOLS_PAYLOAD_BYTES, (path, size) => path === EXT_TOOLS_MANIFEST_FILE || (path.startsWith('bin/') && size <= MAX_EXT_TOOLS_LAUNCHER_BYTES), ) const raw = listing.captured.get(EXT_TOOLS_MANIFEST_FILE) if (!raw) throw invalid('ext_tools', `${key}: ${EXT_TOOLS_MANIFEST_FILE} is not at the root of the payload`) const manifest = parseExtToolsManifest(raw) assertExtToolsOf(appDoc, manifest, platform.name, Object.keys(expectObject(appDoc.pkg_list, 'APPDOC.pkg_list')).flatMap((k) => extToolsPlatformOfKey(k)?.name ?? [])) const launchers = new Map([...listing.captured].filter(([path]) => path.startsWith('bin/'))) const errors = extToolsPayloadErrors(manifest, platform.name, { files: listing.files, directories: listing.directories, launchers }) if (errors.length > 0) throw invalid('ext_tools', `${key}: ${errors.join('; ')}`) return manifest } export async function validateSnapshot( distDir: string, appDoc: Record, packageMeta: PackageMetaFile, ): Promise { const payloads = new Map() for (const entryValue of Object.values(packageMeta.content_index)) { const entry = validateContentIndexEntry(entryValue, 'content_index') const path = join(distDir, entry.path) const digest = await digestFile(path) payloads.set(entry.path, digest) } const validated = validateObjectGraph(appDoc, packageMeta, payloads, true, false, true) if (appDoc.app_type === 'aiws') { for (const sub of validated.subpackages) await verifyAiwsPayload(await getHost().readFile(join(distDir, sub.payload_path))) } if (appDoc.app_type === EXT_TOOLS_APP_TYPE) { for (const sub of validated.subpackages) await verifyExtToolsPayload(fileReadableStream(join(distDir, sub.payload_path)), sub.key, appDoc) } return validated } export async function packSnapshot( distDir: string, destination: string, appDoc: Record, packageMeta: PackageMetaFile, ): Promise { const validated = await validateSnapshot(distDir, appDoc, packageMeta) const encoder = new TextEncoder() await writeStoredZip(destination, [ { name: APPDOC_ENTRY, bytes: encoder.encode(`${JSON.stringify(appDoc, null, 2)}\n`) }, { name: PACKAGE_META_ENTRY, bytes: encoder.encode(`${JSON.stringify(packageMeta, null, 2)}\n`), }, ...validated.subpackages.map((subpackage) => ({ name: subpackage.payload_path, path: join(distDir, subpackage.payload_path), })), ]) } export async function inspectPikg(path: string): Promise { const archive = await openZip(path) let metadataTotal = 0 for (const entry of archive.entries) { if ( entry.name === APPDOC_ENTRY || entry.name === APPDOC_JWT_ENTRY || entry.name === PACKAGE_META_ENTRY || entry.name.startsWith('objects/') && entry.name.endsWith('.json') ) { const limit = entry.name === APPDOC_ENTRY || entry.name === APPDOC_JWT_ENTRY ? MAX_APPDOC_BYTES : MAX_METADATA_ENTRY_BYTES if (entry.size > limit) throw invalid('limits', `metadata entry exceeds limit: ${entry.name}`) metadataTotal += entry.size } } if (metadataTotal > MAX_METADATA_TOTAL_BYTES) { throw invalid('limits', 'PIKG metadata exceeds the total size limit') } if (archive.byName.has('APPDOC.wt')) { throw invalid('appdoc', 'legacy APPDOC.wt is not supported') } const jsonEntry = archive.byName.get(APPDOC_ENTRY) const jwtEntry = archive.byName.get(APPDOC_JWT_ENTRY) if (!jsonEntry && !jwtEntry) throw invalid('appdoc', 'PIKG has no App Document') const jsonDoc = jsonEntry ? parseJsonObject( await readZipEntry(archive, jsonEntry, MAX_APPDOC_BYTES), APPDOC_ENTRY, ) : undefined const jwt = jwtEntry ? new TextDecoder('utf-8', { fatal: true }).decode( await readZipEntry(archive, jwtEntry, MAX_APPDOC_BYTES), ).trim() : undefined const jwtDoc = jwt ? decodeJwtClaims(jwt) : undefined if (jsonDoc && jwtDoc && appDocObjectId(jsonDoc) !== appDocObjectId(jwtDoc)) { throw invalid('appdoc', 'APPDOC.json and APPDOC.jwt have different canonical documents') } const appDoc = jwtDoc ?? jsonDoc! const metaEntry = archive.byName.get(PACKAGE_META_ENTRY) if (!metaEntry) throw invalid('package-meta', 'PACKAGE_META.json is required') const packageMeta = parsePackageMeta( parseJsonObject( await readZipEntry(archive, metaEntry, MAX_METADATA_ENTRY_BYTES), PACKAGE_META_ENTRY, ), ) for ( const entry of archive.entries.filter((candidate) => candidate.name.startsWith('objects/')) ) { const match = /^objects\/([^/]+)\.json$/.exec(entry.name) if (!match) { throw invalid( 'object-graph', `invalid object entry path: ${entry.name}`, entry.name, ) } let objectId: InstanceType try { objectId = ndn.ObjId.fromString(match[1]) } catch { throw invalid('object-graph', `invalid object entry ID: ${entry.name}`, entry.name) } const value = parseJsonValue( await readZipEntry(archive, entry, MAX_METADATA_ENTRY_BYTES), entry.name, ) const computed = ndn.buildNamedObjectByJson(objectId.objType, value)[0] if (!computed.equals(objectId)) { throw invalid('object-graph', `object entry ID mismatch: ${entry.name}`, entry.name) } } const payloads = new Map() for (const value of Object.values(packageMeta.content_index)) { const content = validateContentIndexEntry(value, 'content_index') const entry = archive.byName.get(content.path) if (!entry) throw invalid('content-index', `content entry is missing: ${content.path}`) if (entry.size !== content.size) { throw invalid('content-index', `content entry size mismatch: ${content.path}`, content.path) } const expected = parseSha256(content.digest, `content_index.${content.digest}`) payloads.set(content.path, await verifyZipEntry(archive, entry, expected)) } const validated = validateObjectGraph( appDoc, packageMeta, payloads, Boolean(jsonEntry), Boolean(jwtEntry), !jsonDoc || !jwtDoc || appDocObjectId(jsonDoc) === appDocObjectId(jwtDoc), ) let aiws: Record | undefined if (appDoc.app_type === 'aiws') { const sub = validated.subpackages[0] aiws = await verifyAiwsPayload(await readZipEntry(archive, archive.byName.get(sub.payload_path)!, MAX_AIWS_PAYLOAD_BYTES)) } let extTools: Record | undefined if (appDoc.app_type === EXT_TOOLS_APP_TYPE) { for (const sub of validated.subpackages) { const manifest = await verifyExtToolsPayload(zipEntryStream(archive.path, archive.byName.get(sub.payload_path)!), sub.key, appDoc) extTools ??= { ...manifest } } } const pikgDigest = await digestFile(path) return { schema_version: 1, ...(aiws ? { aiws } : {}), ...(extTools ? { ext_tools: extTools } : {}), protocol: PACKAGE_META_SCHEMA, pikg_path: resolve(path), size: pikgDigest.size, pikg_digest: `sha256:${pikgDigest.sha256}`, valid: true, app: { did: String(appDoc.did), app_id: appIdFromDid(String(appDoc.did)), ...(typeof appDoc.name === 'string' ? { name: appDoc.name } : {}), app_type: String(appDoc.app_type), version: String(appDoc.version), owner: String(appDoc.owner), app_doc_object_id: validated.appDocObjectId, }, app_doc_form: jsonEntry && jwtEntry ? 'both' : jwtEntry ? 'signed' : 'unsigned', canonical_match: validated.canonicalMatch, subpackages: validated.subpackages.map((subpackage) => ({ key: subpackage.key, selector: subpackage.selector, required: subpackage.required, pkg_id: subpackage.pkg_id, pkg_objid: subpackage.pkg_objid, payload: { path: subpackage.payload_path, size: subpackage.payload_size, digest: subpackage.payload_digest, }, ...(subpackage.docker_image_name ? { docker_image_name: subpackage.docker_image_name } : {}), ...(subpackage.docker_image_digest ? { docker_image_digest: subpackage.docker_image_digest } : {}), })), offline_content_validation: 'passed', signature_validation: jwtEntry ? 'not-resolvable-offline' : 'not-present', publication_validation: 'not-checked', } } export function parsePackageMeta(value: Record): PackageMetaFile { try { rejectUnknown( value, ['@schema', 'app_doc_id', 'package_objects', 'content_index'], PACKAGE_META_ENTRY, ) } catch (error) { if (error instanceof UsageError) throw invalid('package-meta', error.message) throw error } if (value['@schema'] !== PACKAGE_META_SCHEMA) { throw invalid('package-meta', `unsupported PACKAGE_META.json schema: ${value['@schema']}`) } const appDocId = expectNonEmptyString(value.app_doc_id, 'PACKAGE_META.json.app_doc_id') if (!OBJECT_ID.test(appDocId) || !appDocId.startsWith('appdoc:')) { throw invalid('package-meta', 'PACKAGE_META.json.app_doc_id is invalid') } const packageObjects = expectObject(value.package_objects, 'PACKAGE_META.json.package_objects') const contentIndex = expectObject(value.content_index, 'PACKAGE_META.json.content_index') return { '@schema': PACKAGE_META_SCHEMA, app_doc_id: appDocId, package_objects: packageObjects as Record>, content_index: contentIndex as Record, } } function validateObjectGraph( appDoc: Record, packageMeta: PackageMetaFile, payloads: Map, hasJson: boolean, hasSigned: boolean, canonicalMatch: boolean, ): ValidatedPikg { validateAppDocShape(appDoc) const appId = appDocObjectId(appDoc) if (packageMeta.app_doc_id !== appId) { throw invalid('object-graph', 'PACKAGE_META.json app_doc_id does not match APPDOC') } const appVersion = String(appDoc.version) const namespace = deriveNamespaceForPackage(appDoc) const pkgList = appDoc.pkg_list as Record const referenced = new Set() const subpackages: ValidatedSubpackage[] = [] const contentBySubpackage = new Map() for (const [digest, raw] of Object.entries(packageMeta.content_index)) { const entry = validateContentIndexEntry(raw, `content_index.${digest}`) if (entry.digest !== digest) throw invalid('content-index', `digest key mismatch: ${digest}`) const expectedHex = parseSha256(digest, `content_index.${digest}`) const payload = payloads.get(entry.path) if (!payload) throw invalid('content-index', `payload is missing: ${entry.path}`, entry.path) if (payload.size !== entry.size || payload.sha256 !== expectedHex) { throw invalid('content', `payload does not match content index: ${entry.path}`, entry.path) } if (contentBySubpackage.has(entry.sub_pkg_name)) { throw invalid('content-index', `subpackage has more than one payload: ${entry.sub_pkg_name}`) } contentBySubpackage.set(entry.sub_pkg_name, entry) } const dependencyNames = new Set() for ( const [key, rawDesc] of Object.entries(pkgList).sort(([left], [right]) => left.localeCompare(right) ) ) { validateSubpackageNameForPackage(key) const desc = expectObject(rawDesc, `APPDOC.pkg_list.${key}`) rejectUnknown( desc, [ 'pkg_id', 'pkg_objid', 'docker_image_name', 'docker_image_digest', 'source_url', 'selector', 'required', ], `APPDOC.pkg_list.${key}`, ) if (desc.source_url !== undefined) { throw invalid('self-contained', `source_url is not allowed for bundled App: ${key}`) } const pkgId = expectNonEmptyString(desc.pkg_id, `APPDOC.pkg_list.${key}.pkg_id`) const parsedId = parsePackageId(pkgId, key) if (parsedId.version !== appVersion) { throw invalid('namespace', `subpackage ${key} must use App version ${appVersion}`) } const uniqueName = packageUniqueName(parsedId.name, key) const namespacePrefix = uniqueName.endsWith(`.${namespace}`) ? uniqueName.slice(0, -namespace.length - 1) : undefined if ( uniqueName !== namespace && (!namespacePrefix || namespacePrefix.includes('.') || !/^[a-z0-9][a-z0-9_-]*$/.test(namespacePrefix)) ) { throw invalid('namespace', `subpackage ${key} is outside App package namespace`) } const pkgObjid = expectNonEmptyString(desc.pkg_objid, `APPDOC.pkg_list.${key}.pkg_objid`) if (!OBJECT_ID.test(pkgObjid) || !pkgObjid.startsWith('pkg:')) { throw invalid('object-graph', `subpackage ${key} has an invalid pkg_objid`) } const packageObject = packageMeta.package_objects[pkgObjid] if (!packageObject) throw invalid('object-graph', `PackageMeta is missing for ${key}`) const computed = ndn.buildNamedObjectByJson(ndn.OBJ_TYPE_PKG, packageObject)[0].toString() if (computed !== pkgObjid) { throw invalid('object-graph', `PackageMeta Object ID mismatch: ${key}`) } const metaName = expectNonEmptyString(packageObject.name, `PackageMeta(${key}).name`) const metaVersion = expectNonEmptyString(packageObject.version, `PackageMeta(${key}).version`) if (metaName !== parsedId.name || metaVersion !== parsedId.version) { throw invalid('namespace', `PackageMeta identity does not match pkg_id: ${key}`) } if (packageObject.owner !== appDoc.owner || packageObject.author !== appDoc.author) { throw invalid('namespace', `PackageMeta owner/author does not match AppDoc: ${key}`) } const metaDeps = packageObject.deps if ( metaDeps !== undefined && Object.keys(expectObject(metaDeps, `PackageMeta(${key}).deps`)).length ) { throw invalid( 'self-contained', `third-party PackageMeta dependencies are not allowed: ${key}`, ) } const entry = contentBySubpackage.get(key) if (!entry) throw invalid('content-index', `content index is missing for ${key}`) if (entry.path !== `${key}.tar.gz` || entry.format !== 'tar.gz') { throw invalid('content-index', `subpackage ${key} has a noncanonical payload path`) } const metaSize = expectSafeInteger(packageObject.size, `PackageMeta(${key}).size`) if (metaSize !== entry.size) throw invalid('content', `PackageMeta size mismatch: ${key}`) validateChunkContent( expectNonEmptyString(packageObject.content, `PackageMeta(${key}).content`), entry.size, parseSha256(entry.digest, `content_index.${entry.digest}`), key, ) const selector = canonicalSelectorForPackage(desc.selector, `APPDOC.pkg_list.${key}.selector`) assertSelectorForPackage(key, selector) if (desc.required !== undefined && typeof desc.required !== 'boolean') { throw invalid('appdoc', `required must be boolean: ${key}`) } const dockerImageName = optionalNonEmptyString( desc.docker_image_name, `APPDOC.pkg_list.${key}.docker_image_name`, ) const dockerImageDigest = optionalNonEmptyString( desc.docker_image_digest, `APPDOC.pkg_list.${key}.docker_image_digest`, ) if (dockerImageDigest && !SHA256_DIGEST.test(dockerImageDigest)) { throw invalid('appdoc', `invalid Docker image digest: ${key}`) } referenced.add(pkgObjid) dependencyNames.add(metaName) subpackages.push({ key, selector: selector ?? derivedSelector(key) ?? null, required: desc.required === undefined ? true : desc.required, pkg_id: pkgId, pkg_objid: pkgObjid, payload_path: entry.path, payload_size: entry.size, payload_digest: entry.digest, ...(dockerImageName ? { docker_image_name: dockerImageName } : {}), ...(dockerImageDigest ? { docker_image_digest: dockerImageDigest } : {}), }) } if (subpackages.length === 0) throw invalid('appdoc', 'AppDoc.pkg_list must not be empty') for (const key of Object.keys(packageMeta.package_objects)) { if (!referenced.has(key)) { throw invalid('object-graph', `unreferenced PackageMeta object: ${key}`) } } if (contentBySubpackage.size !== subpackages.length) { throw invalid('content-index', 'content index and pkg_list have different subpackages') } if (dependencyNames.size !== subpackages.length) { throw invalid('self-contained', 'AppDoc package identities are not unique') } return { appDoc, appDocObjectId: appId, packageMeta, subpackages, hasJsonAppDoc: hasJson, hasSignedAppDoc: hasSigned, canonicalMatch, } } export function validateAppPresentation(value: unknown, label: string): Record { const presentation = expectObject(value, label) rejectUnknown(presentation, ['title', 'summary', 'description', 'icons', 'links', 'license'], label) for (const field of ['title', 'summary', 'description', 'icons', 'links']) { const values = expectObject(presentation[field], `${label}.${field}`) for (const [key, item] of Object.entries(values)) { if (typeof item !== 'string') { throw invalid('appdoc', `${label}.${field}.${key} must be a string`) } if (field === 'icons') { try { ndn.ObjId.fromString(item) } catch { throw invalid('appdoc', `${label}.${field}.${key} must be an ObjectId`) } } } } if (typeof presentation.license !== 'string') { throw invalid('appdoc', `${label}.license must be a string`) } return presentation } export function validateAppDocShape(appDoc: Record): void { if (appDoc.presentation !== undefined && appDoc.presentation !== null) { validateAppPresentation(appDoc.presentation, 'APPDOC.presentation') } rejectUnknown( appDoc, [ 'schema_version', 'doc_type', 'did', 'name', 'copyright', 'tags', 'categories', 'base_on', 'directory', 'references', 'version', 'version_tag', 'app_type', 'owner', 'controller', 'author', 'create_time', 'last_update_time', 'exp', 'pkg_list', 'show_name', 'presentation', 'sdk_version', 'req_capbilities', 'permissions', 'content_handlers', 'selector_type', 'service_config_tips', ], 'APPDOC', ) if (appDoc.schema_version !== 1) { throw invalid('appdoc', 'APPDOC.schema_version must be 1') } for ( const field of [ 'did', 'version', 'app_type', 'owner', 'controller', 'author', 'show_name', 'selector_type', ] ) { expectNonEmptyString(appDoc[field], `APPDOC.${field}`) } if (appDoc.doc_type !== 'app') throw invalid('appdoc', 'APPDOC.doc_type must be app') if (!['service', 'dapp', 'web', 'agent', 'agent_skills', 'aiws', EXT_TOOLS_APP_TYPE].includes(String(appDoc.app_type))) { throw invalid('appdoc', 'APPDOC.app_type is invalid') } if (appDoc.name !== undefined) expectNonEmptyString(appDoc.name, 'APPDOC.name') optionalString(appDoc.copyright, 'APPDOC.copyright') for (const field of ['tags', 'categories']) { const values = appDoc[field] if ( values !== undefined && (!Array.isArray(values) || values.length === 0 || values.some((value) => typeof value !== 'string')) ) { throw invalid('appdoc', `APPDOC.${field} must be a non-empty string array when present`) } } if (appDoc.base_on !== undefined) { const baseOn = expectNonEmptyString(appDoc.base_on, 'APPDOC.base_on') try { ndn.ObjId.fromString(baseOn) } catch { throw invalid('appdoc', 'APPDOC.base_on must be an ObjectId') } } for (const field of ['directory', 'references']) { if (appDoc[field] === undefined) continue const entries = expectObject(appDoc[field], `APPDOC.${field}`) if (Object.keys(entries).length === 0) { throw invalid('appdoc', `APPDOC.${field} must not be empty when present`) } for (const [key, value] of Object.entries(entries)) { expectObject(value, `APPDOC.${field}.${key}`) } } expectSafeInteger(appDoc.create_time, 'APPDOC.create_time') expectSafeInteger(appDoc.last_update_time, 'APPDOC.last_update_time') if (expectSafeInteger(appDoc.exp, 'APPDOC.exp') === 0) { throw invalid('appdoc', 'APPDOC.exp must be greater than zero') } appIdFromDid(String(appDoc.did)) parseDid(String(appDoc.owner), 'owner') parseDid(String(appDoc.controller), 'controller') parseDid(String(appDoc.author), 'author') if (appDoc.permissions !== undefined && !Array.isArray(appDoc.permissions)) { throw invalid('appdoc', 'APPDOC.permissions must be an array') } validatePermissions(appDoc.permissions ?? [], 'APPDOC.permissions') if (appDoc.content_handlers !== undefined && !Array.isArray(appDoc.content_handlers)) { throw invalid('appdoc', 'APPDOC.content_handlers must be an array') } validateServiceConfigTips(appDoc.service_config_tips, 'APPDOC.service_config_tips') const pkgList = expectObject(appDoc.pkg_list, 'APPDOC.pkg_list') if (Object.keys(pkgList).length === 0) { throw invalid('appdoc', 'APPDOC.pkg_list must not be empty') } if (appDoc.app_type === 'agent_skills') { expectNonEmptyString(appDoc.name, 'APPDOC.name') if (Object.keys(pkgList).join() !== 'agent_skills') { throw invalid('appdoc', 'an agent_skills App must contain only the agent_skills sub package') } } // AI Workspace controls (buckyos aiworkspace README “Extension controls”): one bundle, nothing to run if (appDoc.app_type === 'aiws' && Object.keys(pkgList).join() !== AIWS_SUB_PACKAGE) { throw invalid('appdoc', 'an aiws package must contain only the aiws sub package') } // Extension tools: one payload per platform, nothing to run if (appDoc.app_type === EXT_TOOLS_APP_TYPE) { expectNonEmptyString(appDoc.name, 'APPDOC.name') const unknown = Object.keys(pkgList).find((key) => !extToolsPlatformOfKey(key)) if (unknown) throw invalid('appdoc', `an ext_tools package must contain only ext_tools__ sub packages, not ${unknown}`) } else { const stray = Object.keys(pkgList).find((key) => extToolsPlatformOfKey(key)) if (stray) throw invalid('appdoc', `${stray} belongs to an ext_tools package`) } deriveNamespaceForPackage(appDoc) } function deriveNamespaceForPackage(appDoc: Record): string { try { return deriveAppNamespace(String(appDoc.did)) } catch (error) { if (error instanceof UsageError) throw invalid('namespace', error.message) throw error } } function validateContentIndexEntry(value: unknown, label: string): ContentIndexEntry { const entry = expectObject(value, label) rejectUnknown(entry, ['sub_pkg_name', 'path', 'format', 'size', 'digest'], label) const subpackage = expectNonEmptyString(entry.sub_pkg_name, `${label}.sub_pkg_name`) validateSubpackageNameForPackage(subpackage) const path = expectNonEmptyString(entry.path, `${label}.path`) if (path !== `${subpackage}.tar.gz`) { throw invalid('content-index', `invalid payload path: ${path}`) } if (entry.format !== 'tar.gz') throw invalid('content-index', `invalid payload format: ${path}`) const size = expectSafeInteger(entry.size, `${label}.size`) const digest = expectNonEmptyString(entry.digest, `${label}.digest`) parseSha256(digest, `${label}.digest`) return { sub_pkg_name: subpackage, path, format: 'tar.gz', size, digest } } function validateChunkContent(content: string, size: number, sha256: string, key: string): void { let chunk: InstanceType try { chunk = ndn.ChunkId.fromString(content) } catch { throw invalid('content', `PackageMeta content is not a ChunkId: ${key}`) } if (!['sha256', 'mix256'].includes(chunk.chunkType)) { throw invalid('content', `unsupported PackageMeta ChunkId type: ${key}`) } const hash = ndn.hexToBytes(sha256) const expected = chunk.chunkType === 'mix256' ? ndn.ChunkId.fromMix256Result(size, hash) : ndn.ChunkId.fromSha256Result(hash) if (chunk.toString() !== expected.toString()) { throw invalid('content', `PackageMeta ChunkId digest mismatch: ${key}`) } } function parsePackageId(value: string, key: string): { name: string; version: string } { const parts = value.split('#') if (parts.length !== 2 || !parts[0] || !parts[1] || parts[1].startsWith('$')) { throw invalid('namespace', `subpackage ${key} must use an exact-version pkg_id`) } return { name: parts[0], version: parts[1] } } function packageUniqueName(packageName: string, key: string): string { const parts = packageName.split('.') const environments = new Set([ 'all', 'nightly-linux-amd64', 'nightly-linux-aarch64', 'nightly-windows-amd64', 'nightly-windows-aarch64', 'nightly-apple-amd64', 'nightly-apple-aarch64', ]) const unique = parts.length > 1 && environments.has(parts[0]) ? parts.slice(1).join('.') : parts.join('.') if (!unique || unique.split('.').some((label) => !/^[a-z0-9][a-z0-9_-]*$/.test(label))) { throw invalid('namespace', `subpackage ${key} has an invalid package name`) } return unique } function decodeJwtClaims(jwt: string): Record { const segments = jwt.split('.') if (segments.length !== 3 || !segments[1]) throw invalid('appdoc', 'APPDOC.jwt is malformed') try { const normalized = segments[1].replaceAll('-', '+').replaceAll('_', '/') const padded = normalized.padEnd(Math.ceil(normalized.length / 4) * 4, '=') const binary = atob(padded) const bytes = Uint8Array.from(binary, (character) => character.charCodeAt(0)) return parseJsonObject(bytes, APPDOC_JWT_ENTRY) } catch (error) { if (error instanceof ToolError) throw error throw invalid('appdoc', 'APPDOC.jwt claims are not valid JSON') } } function parseJsonObject(bytes: Uint8Array, label: string): Record { return expectObject(parseJsonValue(bytes, label), label) } function parseJsonValue(bytes: Uint8Array, label: string): unknown { try { return JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(bytes)) } catch (error) { if (error instanceof ToolError) throw error throw invalid('metadata', `${label} is not valid JSON`) } } function parseDid(value: string, label: string): { method: string; id: string } { const match = /^did:([a-z0-9]+):(.+)$/.exec(value) if (!match) throw new UsageError('INVALID_DID', `${label} must be a DID`) return { method: match[1], id: match[2] } } function parseSha256(value: string, label: string): string { const match = SHA256_DIGEST.exec(value) if (!match) throw invalid('digest', `${label} must be sha256:<64 lowercase hex>`) return match[1].toLowerCase() } function canonicalSelectorForPackage( value: unknown, label: string, ): Record | undefined { try { return canonicalSelector(value, label) } catch (error) { if (error instanceof UsageError) throw invalid('appdoc', error.message) throw error } } function assertSelectorForPackage(key: string, selector: Record | undefined): void { try { assertSelectorCompatible(key, selector, `APPDOC.pkg_list.${key}.selector`) } catch (error) { if (error instanceof UsageError) throw invalid('appdoc', error.message) throw error } } function validateSubpackageNameForPackage(name: string): void { try { validateSubpackageName(name) } catch (error) { if (error instanceof UsageError) throw invalid('appdoc', error.message) throw error } } export function expectObject(value: unknown, label: string): Record { if (!value || typeof value !== 'object' || Array.isArray(value)) { throw invalid('schema', `${label} must be an object`) } return value as Record } export function expectNonEmptyString(value: unknown, label: string): string { if (typeof value !== 'string' || !value.trim()) { throw invalid('schema', `${label} must be a string`) } return value } function optionalNonEmptyString(value: unknown, label: string): string | undefined { return value === undefined ? undefined : expectNonEmptyString(value, label) } function expectSafeInteger(value: unknown, label: string): number { if (typeof value !== 'number' || !Number.isSafeInteger(value) || value < 0) { throw invalid('schema', `${label} must be a non-negative safe integer`) } return value } function expectUint(value: unknown, label: string, maximum: number): number { const result = expectSafeInteger(value, label) if (result > maximum) throw invalid('schema', `${label} is too large`) return result } function optionalBoolean(value: unknown, label: string): void { if (value !== undefined && typeof value !== 'boolean') { throw invalid('schema', `${label} must be boolean`) } } function optionalString(value: unknown, label: string): void { if (value !== undefined && typeof value !== 'string') { throw invalid('schema', `${label} must be a string`) } } function validateStringMap(value: unknown, label: string): Record { const result = expectObject(value, label) if (Object.values(result).some((item) => typeof item !== 'string')) { throw invalid('schema', `${label} values must be strings`) } return result } function assertKnownFields( value: Record, allowed: string[], label: string, ): void { const accepted = new Set(allowed) const unknown = Object.keys(value).find((key) => !accepted.has(key)) if (unknown) throw invalid('schema', `${label}.${unknown} is not allowed`) } export function rejectUnknown( value: Record, allowed: string[], label: string, ): void { const accepted = new Set(allowed) const unknown = Object.keys(value).filter((key) => !accepted.has(key)) if (unknown.length) { throw new UsageError('SCHEMA_VALIDATION_FAILED', `${label}.${unknown[0]} is not allowed`) } } function invalid(stage: string, message: string, entry?: string): ToolError { return new ToolError('INVALID_PACKAGE', message, 6, false, { stage, ...(entry ? { entry: basename(entry) } : {}), }) } export function stableJsonDigest(value: unknown): string { return sha256Bytes(new TextEncoder().encode(ndn.toCanonicalJsonString(value))) } export function validateContentHandlers(value: unknown, permissions: unknown[]): unknown[] { if (!Array.isArray(value)) throw invalid('appdoc', 'content_handlers must be an array') const ids = new Set() for (const raw of value) { const h = expectObject(raw, 'content handler') const id = expectNonEmptyString(h.handler_id, 'handler_id') if (!/^[a-zA-Z0-9_-]+$/.test(id) || ids.has(id)) throw invalid('appdoc', 'invalid or duplicate handler_id') ids.add(id) if (!Number.isSafeInteger(h.version) || Number(h.version) < 1) throw invalid('appdoc', 'invalid handler version') if (!Array.isArray(h.selectors) || !h.selectors.length) throw invalid('appdoc', 'missing selectors') for (const rawSelector of h.selectors) { const s = expectObject(rawSelector, 'selector') if (!['mime', 'objType', 'schema', 'ext'].some(k => k in s)) throw invalid('appdoc', 'empty selector') for (const key of ['mime', 'objType', 'schema']) { const v = s[key] if (v !== undefined && !(typeof v === 'string' && v.length) && !(Array.isArray(v) && v.length && v.every(i => typeof i === 'string' && i.length))) throw invalid('appdoc', 'invalid selector field') } if (s.maxSize !== undefined && (!Number.isSafeInteger(s.maxSize) || Number(s.maxSize) < 0)) throw invalid('appdoc', 'invalid maxSize') } const intents = expectObject(h.intents, 'intents') if (Object.keys(intents).length !== 1 || !intents.open) throw invalid('appdoc', 'only open intent is supported') const open = expectObject(intents.open, 'open') const entry = expectObject(open.entry, 'entry') const path = expectNonEmptyString(entry.path, 'entry.path') let decoded: string try { decoded = decodeURIComponent(path) } catch { throw invalid('appdoc', 'invalid entry path') } if (entry.type !== 'web' || !decoded.startsWith('/') || decoded.startsWith('//') || decoded.includes('..') || decoded.includes('\\') || decoded.includes('://') || decoded.includes('#') || /[\r\n]/.test(decoded)) throw invalid('appdoc', 'unsafe Web entry') if (open.priority !== undefined && (!Number.isSafeInteger(open.priority) || Number(open.priority) < 0 || Number(open.priority) > 80)) throw invalid('appdoc', 'priority must be 0..80') if (open.window !== undefined && open.window !== 'new' && open.window !== 'reuse') throw invalid('appdoc', 'invalid window policy') if (open.multiSource !== undefined && typeof open.multiSource !== 'boolean') throw invalid('appdoc', 'invalid multiSource') if (open.modes !== undefined && (!Array.isArray(open.modes) || !open.modes.length || open.modes.some(m => m !== 'view' && m !== 'edit'))) throw invalid('appdoc', 'invalid modes') if (h.permissions !== undefined && (!Array.isArray(h.permissions) || h.permissions.some(scope => !permissions.some(p => p && typeof p === 'object' && (p as Record).scope_path === scope)))) throw invalid('appdoc', 'handler permissions exceed App permissions') } return value }